dovecot: security backports, new versioning scheme

- Fix dovecot package download URLs for new
  [release](https://github.com/chatmail/dovecot/releases/tag/upstream%2F2.3.21%2Bdfsg1-3%2Bchatmail2)
  with
    - debian-security backport for 12 CVEs
    - distro-specific suffix (+deb{release}u1), simplified primary URL
      path: remove codename subdirectory
- Use VERSION_ID from os-release as deb_release instead of codename
  mapping, reorder hash-dict to match Github release page
- Remove redundant parsing/validation, let function validate against hash dict
- Update test expectations

.github/workflows/docker-dispatch.yaml

@@ -8,7 +8,7 @@ name: Trigger Docker build
 
 on:
   push:
-    branches: [main]
+    branches: [main, j4n/dovecot-multidist]
     tags: ['[0-9]+.[0-9]+.[0-9]+']
   workflow_dispatch:
 

cmdeploy/src/cmdeploy/deployers.py

@@ -98,6 +98,19 @@ def _install_remote_venv_with_chatmaild(deployer) -> None:
         dest=remote_dist_file,
     )
 
+    # Remove venv if its Python major.minor doesn't match the system Python
+    server.shell(
+        name="remove stale chatmaild venv if python version changed",
+        commands=["\n".join([
+            f"if [ -d {remote_venv_dir} ]; then",
+            r"  re='[0-9]+\.[0-9]+'",  # match major.minor from 'Python X.Y.Z'"
+            '   SYS_VERSION=$(python3 --version | grep -oE "$re")',
+            f'  VENV_VERSION=$({remote_venv_dir}/bin/python --version 2>/dev/null | grep -oE "$re")',
+            f'  [ "$SYS_VERSION" = "$VENV_VERSION" ] || rm -rf {remote_venv_dir}',
+            "fi",
+        ])],
+    )
+
     pip.virtualenv(
         name=f"chatmaild virtualenv {remote_venv_dir}",
         path=remote_venv_dir,
@@ -406,6 +419,12 @@ class ChatmailDeployer(Deployer):
             src=BytesIO(b'APT::Install-Recommends "false";\n'),
             dest="/etc/apt/apt.conf.d/00InstallRecommends",
         )
+        # Pin dovecot-* to priority -1 before any apt operation, apt should
+        # never manage dovecot as our version might be lower than the distro's.
+        self.put_file(
+            src=StringIO("Package: dovecot-*\nPin: version *\nPin-Priority: -1\n"),
+            dest="/etc/apt/preferences.d/pin-dovecot",
+        )
         apt.update(name="apt update", cache_time=24 * 3600)
         apt.upgrade(name="upgrade apt packages", auto_remove=True)
 

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -1,4 +1,3 @@
-import io
 import urllib.request
 
 from chatmaild.config import Config
@@ -15,16 +14,22 @@ from cmdeploy.basedeploy import (
     is_in_container,
 )
 
-DOVECOT_ARCHIVE_VERSION = "2.3.21+dfsg1-3"
+DOVECOT_ARCHIVE_VERSION = "2.3.21+dfsg1-3+chatmail2"
 DOVECOT_PACKAGE_VERSION = f"1:{DOVECOT_ARCHIVE_VERSION}"
 
 DOVECOT_SHA256 = {
-    ("core", "amd64"): "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d",
-    ("core", "arm64"): "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9",
-    ("imapd", "amd64"): "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86",
-    ("imapd", "arm64"): "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f",
-    ("lmtpd", "amd64"): "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab",
-    ("lmtpd", "arm64"): "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f",
+    ("amd64", 12, "core"): "ac3977264d9b9a6fcec53fd3f5cdd2a79ca8aa0324de530c07e535008540826e",
+    ("arm64", 12, "core"): "21626c9c9b52cbdcf1a17b5c09e3c4043e69aa371bf83cc2fcb3b7ddaecdc109",
+    ("amd64", 13, "core"): "47c242ef23c17e700ac19d52d82c9fdb2ebd757d8beb3a7f6781d2de59f87bd0",
+    ("arm64", 13, "core"): "c14c53f112c875f698c4cb6e5870c605cd0a9dd98d35a66e94ceb1827f8020a3",
+    ("amd64", 12, "imapd"): "92a7ab5fc7dc32886a0c34404f919f1335d397b48c467e0c1ef77e56978f60ea",
+    ("arm64", 12, "imapd"): "9369fd566fec4df109ef23debf34ea0417ae85beb29cbe7de619d4d1f31b120c",
+    ("amd64", 13, "imapd"): "e38cc1266455f937ed62f971ea859c47e1a99247841ed0ad946963b524cfdbc5",
+    ("arm64", 13, "imapd"): "11d97dabf23171b37f8b1335dfdb81d408f8b95391aea6d4066aecc9fde01dfe",
+    ("amd64", 12, "lmtpd"): "dc3de473789969f7dd3504ac8783da5e42a446d2d7a305a4e9d7081a6dfe71ab",
+    ("arm64", 12, "lmtpd"): "ae2cbd6c5c43f6d8e2172997b055448f4c79238e2f99cd9ab9200a7d9f548908",
+    ("amd64", 13, "lmtpd"): "833b243e28c7baff141ecf37456e310f5d836e7944a3b9f2fe5074adf0d6a418",
+    ("arm64", 13, "lmtpd"): "55af47a121ba7e23966b20ddaab2dff7feba4b34677864e045e31a702afa180d",
 }
 
 
@@ -38,34 +43,31 @@ class DovecotDeployer(Deployer):
 
     def install(self):
         arch = host.get_fact(Arch)
+        version_line = (host.get_fact(Command, "grep '^VERSION_ID=' /etc/os-release") or "").strip()
+        deb_release = int(version_line.split("=", 1)[1].strip('"'))
         with blocked_service_startup():
             debs = []
             for pkg in ("core", "imapd", "lmtpd"):
-                deb, changed = _download_dovecot_package(pkg, arch)
+                deb, changed = _download_dovecot_package(pkg, arch, deb_release)
                 self.need_restart |= changed
                 if deb:
                     debs.append(deb)
             if debs:
                 deb_list = " ".join(debs)
-                # First dpkg may fail on missing dependencies (stderr suppressed);
-                # apt-get --fix-broken pulls them in, then dpkg retries cleanly.
+                # apt-get install with local .deb paths resolves depends
+                # against the configured repos (e.g. pulls libwrap0),
+                # The pin file written earlier by ChatmailDeployer prevents apt
+                # from installing a 'wrong' version
                 server.shell(
                     name="Install dovecot packages",
                     commands=[
-                        f"dpkg --force-confdef --force-confold -i {deb_list} 2> /dev/null || true",
-                        "DEBIAN_FRONTEND=noninteractive apt-get -y --fix-broken install",
-                        f"dpkg --force-confdef --force-confold -i {deb_list}",
+                        "DEBIAN_FRONTEND=noninteractive apt-get install -y "
+                        '-o Dpkg::Options::="--force-confdef" '
+                        '-o Dpkg::Options::="--force-confold" '
+                        f"--allow-downgrades {deb_list}",
                     ],
                 )
                 self.need_restart = True
-        self.put_file(
-            src=io.StringIO(
-                "Package: dovecot-*\n"
-                "Pin: version *\n"
-                "Pin-Priority: -1\n"
-            ),
-            dest="/etc/apt/preferences.d/pin-dovecot",
-        )
 
     def configure(self):
         configure_remote_units(self, self.config.mail_domain_bare, self.units)
@@ -78,7 +80,7 @@ class DovecotDeployer(Deployer):
         if not self.disable_mail and not self.need_restart:
             stale = host.get_fact(
                 Command,
-                'pid=$(systemctl show -p MainPID --value dovecot.service 2>/dev/null);'
+                "pid=$(systemctl show -p MainPID --value dovecot.service 2>/dev/null);"
                 ' [ "${pid:-0}" != "0" ] && readlink "/proc/$pid/exe" 2>/dev/null | grep -q "(deleted)"'
                 " && echo STALE || true",
             )
@@ -102,27 +104,32 @@ def _pick_url(primary, fallback):
         return fallback
 
 
-def _download_dovecot_package(package: str, arch: str) -> tuple[str | None, bool]:
+def _download_dovecot_package(package: str, arch: str, deb_release: int) -> tuple[str | None, bool]:
     """Download a dovecot .deb if needed, return (path, changed)."""
     arch = "amd64" if arch == "x86_64" else arch
     arch = "arm64" if arch == "aarch64" else arch
 
     pkg_name = f"dovecot-{package}"
-    sha256 = DOVECOT_SHA256.get((package, arch))
-    if sha256 is None:
+    if (arch, deb_release, package) not in DOVECOT_SHA256:
         op = apt.packages(packages=[pkg_name])
         return None, bool(getattr(op, "changed", False))
+    sha256 = DOVECOT_SHA256[(arch, deb_release, package)]
 
     installed_versions = host.get_fact(DebPackages).get(pkg_name, [])
     if DOVECOT_PACKAGE_VERSION in installed_versions:
         return None, False
 
-    url_version = DOVECOT_ARCHIVE_VERSION.replace("+", "%2B")
-    deb_base = f"{pkg_name}_{url_version}_{arch}.deb"
-    primary_url = f"https://download.delta.chat/dovecot/{deb_base}"
-    fallback_url = f"https://github.com/chatmail/dovecot/releases/download/upstream%2F{url_version}/{deb_base}"
+    # Primary URL: flat structure with distro suffix in filename
+    primary_deb = f"{pkg_name}_{DOVECOT_ARCHIVE_VERSION}+deb{deb_release}u1_{arch}.deb"
+    primary_url = f"https://download.delta.chat/dovecot/{primary_deb}"
+    # GitHub release files: escaped + in tag and filename
+    url_version_escaped = DOVECOT_ARCHIVE_VERSION.replace("+", "%2B")
+    fallback_deb = f"{pkg_name}_{url_version_escaped}%2Bdeb{deb_release}u1_{arch}.deb"
+    fallback_url = (
+        f"https://github.com/chatmail/dovecot/releases/download/upstream%2F{url_version_escaped}/{fallback_deb}"
+    )
     url = _pick_url(primary_url, fallback_url)
-    deb_filename = f"/root/{deb_base}"
+    deb_filename = f"/root/{primary_deb}"
 
     files.download(
         name=f"Download {pkg_name}",
@@ -134,6 +141,7 @@ def _download_dovecot_package(package: str, arch: str) -> tuple[str | None, bool
 
     return deb_filename, True
 
+
 def _configure_dovecot(deployer, config: Config, debug: bool = False):
     """Configures Dovecot IMAP server."""
     deployer.put_template(
@@ -144,9 +152,7 @@ def _configure_dovecot(deployer, config: Config, debug: bool = False):
         disable_ipv6=config.disable_ipv6,
     )
     deployer.put_file("dovecot/auth.conf", "/etc/dovecot/auth.conf")
-    deployer.put_file(
-        "dovecot/push_notification.lua", "/etc/dovecot/push_notification.lua"
-    )
+    deployer.put_file("dovecot/push_notification.lua", "/etc/dovecot/push_notification.lua")
 
     # as per https://doc.dovecot.org/2.3/configuration_manual/os/
     # it is recommended to set the following inotify limits

cmdeploy/src/cmdeploy/tests/test_dovecot_deployer.py

@@ -3,6 +3,7 @@ from types import SimpleNamespace
 
 import pytest
 from pyinfra.facts.deb import DebPackages
+from pyinfra.facts.server import Command
 
 from cmdeploy.dovecot import deployer as dovecot_deployer
 
@@ -19,12 +20,10 @@ def make_host(*fact_pairs):
     """
     facts = dict(fact_pairs)
 
-    def get_fact(cls):
+    def get_fact(cls, *args):
         if cls not in facts:
             registered = ", ".join(c.__name__ for c in facts)
-            raise LookupError(
-                f"unexpected get_fact({cls.__name__}); only registered: {registered}"
-            )
+            raise LookupError(f"unexpected get_fact({cls.__name__}); only registered: {registered}")
         return facts[cls]
 
     return SimpleNamespace(get_fact=get_fact)
@@ -82,7 +81,7 @@ def test_download_dovecot_package_skips_epoch_matched_install(monkeypatch):
         lambda **kwargs: downloads.append(kwargs),
     )
 
-    deb, changed = dovecot_deployer._download_dovecot_package("core", "amd64")
+    deb, changed = dovecot_deployer._download_dovecot_package("core", "amd64", deb_release=12)
 
     assert deb is None, f"expected no deb path when version matches, got {deb!r}"
     assert changed is False, "should not flag changed when version already installed"
@@ -109,12 +108,11 @@ def test_download_dovecot_package_uses_archive_version_for_url_and_filename(
         lambda **kwargs: downloads.append(kwargs),
     )
 
-    deb, changed = dovecot_deployer._download_dovecot_package("core", "amd64")
+    deb, changed = dovecot_deployer._download_dovecot_package("core", "amd64", deb_release=12)
 
-    archive_version = dovecot_deployer.DOVECOT_ARCHIVE_VERSION.replace("+", "%2B")
-    expected_deb = f"/root/dovecot-core_{archive_version}_amd64.deb"
+    expected_deb = f"/root/dovecot-core_{dovecot_deployer.DOVECOT_ARCHIVE_VERSION}+deb12u1_amd64.deb"
 
-    # Verify the returned path uses archive version, not package version (with epoch)
+    # Verify the returned path uses archive version with distro suffix, not package version (with epoch)
     assert changed is True, "should flag changed when package not yet installed"
     assert deb == expected_deb, f"deb path mismatch: {deb!r} != {expected_deb!r}"
     assert dovecot_deployer.DOVECOT_PACKAGE_VERSION not in deb, (
@@ -139,6 +137,7 @@ def test_install_skips_dpkg_path_when_epoch_matched_packages_present(
                 },
             ),
             (dovecot_deployer.Arch, "x86_64"),
+            (Command, 'VERSION_ID="12"'),
         ),
     )
     downloads = []
@@ -152,19 +151,17 @@ def test_install_skips_dpkg_path_when_epoch_matched_packages_present(
 
     assert downloads == [], "should not download when all packages epoch-matched"
     assert track_shell == [], "should not run dpkg when all packages epoch-matched"
-    assert deployer.need_restart is False, (
-        "need_restart should be False when nothing changed"
-    )
+    assert deployer.need_restart is False, "need_restart should be False when nothing changed"
 
 
-def test_install_unsupported_arch_falls_back_to_apt(
-    deployer, patch_blocked, mock_files_put, track_shell, monkeypatch
-):
-    # For unsupported architectures, all fact lookups return the arch string.
+def test_install_unsupported_arch_falls_back_to_apt(deployer, patch_blocked, mock_files_put, track_shell, monkeypatch):
     monkeypatch.setattr(
         dovecot_deployer,
         "host",
-        SimpleNamespace(get_fact=lambda cls: "riscv64"),
+        make_host(
+            (dovecot_deployer.Arch, "riscv64"),
+            (Command, 'VERSION_ID="12"'),
+        ),
     )
     apt_calls = []
 
@@ -184,9 +181,7 @@ def test_install_unsupported_arch_falls_back_to_apt(
         f"expected apt install of core/imapd/lmtpd, got {actual_pkgs}"
     )
     assert track_shell == [], "should not run dpkg for unsupported arch"
-    assert deployer.need_restart is True, (
-        "need_restart should be True when apt installed a package"
-    )
+    assert deployer.need_restart is True, "need_restart should be True when apt installed a package"
 
 
 def test_install_runs_dpkg_when_packages_need_download(
@@ -198,6 +193,7 @@ def test_install_runs_dpkg_when_packages_need_download(
         make_host(
             (dovecot_deployer.DebPackages, {}),
             (dovecot_deployer.Arch, "x86_64"),
+            (Command, 'VERSION_ID="12"'),
         ),
     )
     monkeypatch.setattr(
@@ -213,17 +209,15 @@ def test_install_runs_dpkg_when_packages_need_download(
 
     deployer.install()
 
-    assert len(track_shell) == 1, (
-        f"expected one server.shell() call for dpkg install, got {len(track_shell)}"
-    )
+    assert len(track_shell) == 1, f"expected one server.shell() call for dpkg install, got {len(track_shell)}"
     cmds = track_shell[0]["commands"]
-    assert len(cmds) == 3, f"expected 3 dpkg/apt commands, got: {cmds}"
-    assert cmds[0].startswith("dpkg --force-confdef --force-confold -i ")
-    assert "apt-get -y --fix-broken install" in cmds[1]
-    assert cmds[2].startswith("dpkg --force-confdef --force-confold -i ")
-    assert deployer.need_restart is True, (
-        "need_restart should be True after dpkg install"
-    )
+    assert len(cmds) == 1, f"expected single apt-get install command, got: {cmds}"
+    assert "apt-get install -y" in cmds[0]
+    assert '-o Dpkg::Options::="--force-confdef"' in cmds[0]
+    assert '-o Dpkg::Options::="--force-confold"' in cmds[0]
+    assert "--allow-downgrades" in cmds[0]
+    assert ".deb" in cmds[0]
+    assert deployer.need_restart is True, "need_restart should be True after dpkg install"
 
 
 def test_pick_url_falls_back_on_primary_error(monkeypatch):
@@ -232,6 +226,25 @@ def test_pick_url_falls_back_on_primary_error(monkeypatch):
 
     monkeypatch.setattr(dovecot_deployer.urllib.request, "urlopen", raise_error)
     result = dovecot_deployer._pick_url("http://primary", "http://fallback")
-    assert result == "http://fallback", (
-        f"should fall back when primary fails, got {result!r}"
+    assert result == "http://fallback", f"should fall back when primary fails, got {result!r}"
+
+
+def test_install_fails_on_unsupported_debian_version(deployer, patch_blocked, monkeypatch):
+    monkeypatch.setattr(
+        dovecot_deployer,
+        "host",
+        make_host(
+            (dovecot_deployer.Arch, "x86_64"),
+            (Command, 'VERSION_ID="99"'),
+        ),
     )
+
+    # Mock apt.packages to handle unsupported release gracefully
+    def fake_apt(**kwargs):
+        return SimpleNamespace(changed=False)
+
+    monkeypatch.setattr(dovecot_deployer.apt, "packages", fake_apt)
+
+    # Should not error, just skip the dovecot-specific packages and use apt
+    deployer.install()
+    assert deployer.need_restart is False, "unsupported release should not trigger restart"