From 33c85ff58a64d8d00222ffdf8462da3c59173f03 Mon Sep 17 00:00:00 2001 From: j4n Date: Mon, 10 Aug 2026 10:26:22 +0200 Subject: [PATCH] dovecot: security backports, new versioning scheme - Fix dovecot package download URLs for new [release](https://github.com/chatmail/dovecot/releases/tag/upstream%2F2.3.21%2Bdfsg1-3%2Bchatmail2) with - debian-security backport for 12 CVEs - distro-specific suffix (+deb{release}u1), enabling a simplified primary URL path and combined github releases - Use VERSION_ID from os-release as deb_release instead of codename mapping, reorder hash-dict to match Github release page - Remove redundant parsing/validation, let function validate against hash dict - Update test expectations and test new versioning derivation --- cmdeploy/src/cmdeploy/deployers.py | 20 ++- cmdeploy/src/cmdeploy/dovecot/deployer.py | 87 +++++++---- .../cmdeploy/tests/test_dovecot_deployer.py | 146 ++++++++++-------- 3 files changed, 149 insertions(+), 104 deletions(-) diff --git a/cmdeploy/src/cmdeploy/deployers.py b/cmdeploy/src/cmdeploy/deployers.py index 6e1b888e..6b163c50 100644 --- a/cmdeploy/src/cmdeploy/deployers.py +++ b/cmdeploy/src/cmdeploy/deployers.py @@ -101,14 +101,18 @@ def _install_remote_venv_with_chatmaild(deployer) -> None: # Remove venv if its Python major.minor doesn't match the system Python server.shell( name="remove stale chatmaild venv if python version changed", - commands=["\n".join([ - f"if [ -d {remote_venv_dir} ]; then", - r" re='[0-9]+\.[0-9]+'", # match major.minor from 'Python X.Y.Z'" - ' SYS_VERSION=$(python3 --version | grep -oE "$re")', - f' VENV_VERSION=$({remote_venv_dir}/bin/python --version 2>/dev/null | grep -oE "$re")', - f' [ "$SYS_VERSION" = "$VENV_VERSION" ] || rm -rf {remote_venv_dir}', - "fi", - ])], + commands=[ + "\n".join( + [ + r"re='[0-9]+\.[0-9]+'", # major.minor out of 'Python X.Y.Z' + 'sys_version=$(python3 --version 2>/dev/null | grep -oE "$re")', + f'venv_version=$({remote_venv_dir}/bin/python --version 2>/dev/null | grep -oE "$re")', + # an empty sys_version means we could not tell: keep the venv + f'[ -z "$sys_version" ] || [ "$sys_version" = "$venv_version" ] ' + f"|| rm -rf {remote_venv_dir}", + ] + ) + ], ) pip.virtualenv( diff --git a/cmdeploy/src/cmdeploy/dovecot/deployer.py b/cmdeploy/src/cmdeploy/dovecot/deployer.py index 7ffe2ca2..7716b115 100644 --- a/cmdeploy/src/cmdeploy/dovecot/deployer.py +++ b/cmdeploy/src/cmdeploy/dovecot/deployer.py @@ -4,7 +4,7 @@ from chatmaild.config import Config from pyinfra import host from pyinfra.facts.deb import DebPackages from pyinfra.facts.server import Arch, Command, Sysctl -from pyinfra.operations import apt, files, server +from pyinfra.operations import files, server from cmdeploy.basedeploy import ( Deployer, @@ -14,22 +14,31 @@ from cmdeploy.basedeploy import ( is_in_container, ) -DOVECOT_ARCHIVE_VERSION = "2.3.21+dfsg1-3" -DOVECOT_PACKAGE_VERSION = f"1:{DOVECOT_ARCHIVE_VERSION}" +# distro-neutral base version, as committed in chatmail/dovecot debian/changelog +DOVECOT_ARCHIVE_VERSION = "2.3.21+dfsg1-3+chatmail2" + +VERSION_ID_CMD = "grep '^VERSION_ID=' /etc/os-release" + + +def _stamped_version(deb_release: int) -> str: + """Version as built, including the per-distro suffix stamped by + chatmail/dovecot CI into package version and filename.""" + return f"{DOVECOT_ARCHIVE_VERSION}+deb{deb_release}u1" + DOVECOT_SHA256 = { - ("amd64", "bookworm", "core"): "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d", - ("arm64", "bookworm", "core"): "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9", - ("amd64", "bookworm", "imapd"): "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86", - ("arm64", "bookworm", "imapd"): "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f", - ("amd64", "bookworm", "lmtpd"): "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab", - ("arm64", "bookworm", "lmtpd"): "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f", - ("amd64", "trixie", "core"): "406d3781ed81e0913c472077dcf62cb1106e3855983efa6e44ddf43b4b0c9be1", - ("arm64", "trixie", "core"): "c75b0d9df11a77d07ebd8522920380c167fa47330ddefebe10575d99d0ecdf7f", - ("amd64", "trixie", "imapd"): "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86", - ("arm64", "trixie", "imapd"): "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f", - ("amd64", "trixie", "lmtpd"): "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab", - ("arm64", "trixie", "lmtpd"): "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f", + ("amd64", 12, "core"): "ac3977264d9b9a6fcec53fd3f5cdd2a79ca8aa0324de530c07e535008540826e", + ("arm64", 12, "core"): "21626c9c9b52cbdcf1a17b5c09e3c4043e69aa371bf83cc2fcb3b7ddaecdc109", + ("amd64", 13, "core"): "47c242ef23c17e700ac19d52d82c9fdb2ebd757d8beb3a7f6781d2de59f87bd0", + ("arm64", 13, "core"): "c14c53f112c875f698c4cb6e5870c605cd0a9dd98d35a66e94ceb1827f8020a3", + ("amd64", 12, "imapd"): "92a7ab5fc7dc32886a0c34404f919f1335d397b48c467e0c1ef77e56978f60ea", + ("arm64", 12, "imapd"): "9369fd566fec4df109ef23debf34ea0417ae85beb29cbe7de619d4d1f31b120c", + ("amd64", 13, "imapd"): "e38cc1266455f937ed62f971ea859c47e1a99247841ed0ad946963b524cfdbc5", + ("arm64", 13, "imapd"): "11d97dabf23171b37f8b1335dfdb81d408f8b95391aea6d4066aecc9fde01dfe", + ("amd64", 12, "lmtpd"): "dc3de473789969f7dd3504ac8783da5e42a446d2d7a305a4e9d7081a6dfe71ab", + ("arm64", 12, "lmtpd"): "ae2cbd6c5c43f6d8e2172997b055448f4c79238e2f99cd9ab9200a7d9f548908", + ("amd64", 13, "lmtpd"): "833b243e28c7baff141ecf37456e310f5d836e7944a3b9f2fe5074adf0d6a418", + ("arm64", 13, "lmtpd"): "55af47a121ba7e23966b20ddaab2dff7feba4b34677864e045e31a702afa180d", } @@ -43,13 +52,11 @@ class DovecotDeployer(Deployer): def install(self): arch = host.get_fact(Arch) - codename = (host.get_fact(Command, "grep '^VERSION_CODENAME=' /etc/os-release | cut -d= -f2") or "").strip() - if codename not in {key[1] for key in DOVECOT_SHA256}: - raise ValueError(f"Unsupported Debian codename: {codename!r}") + deb_release = _parse_version_id(host.get_fact(Command, VERSION_ID_CMD)) with blocked_service_startup(): debs = [] for pkg in ("core", "imapd", "lmtpd"): - deb, changed = _download_dovecot_package(pkg, arch, codename) + deb, changed = _download_dovecot_package(pkg, arch, deb_release) self.need_restart |= changed if deb: debs.append(deb) @@ -96,6 +103,15 @@ class DovecotDeployer(Deployer): ) +def _parse_version_id(version_line: str) -> int: + """Debian major release from an /etc/os-release VERSION_ID line.""" + _, _, raw = (version_line or "").strip().partition("=") + try: + return int(raw.strip('"')) + except ValueError: + raise ValueError(f"cannot determine Debian release from {version_line!r}") + + def _pick_url(primary, fallback): try: req = urllib.request.Request(primary, method="HEAD") @@ -105,29 +121,36 @@ def _pick_url(primary, fallback): return fallback -def _download_dovecot_package(package: str, arch: str, codename: str) -> tuple[str | None, bool]: +def _download_dovecot_package(package: str, arch: str, deb_release: int) -> tuple[str | None, bool]: """Download a dovecot .deb if needed, return (path, changed).""" arch = "amd64" if arch == "x86_64" else arch arch = "arm64" if arch == "aarch64" else arch pkg_name = f"dovecot-{package}" - sha256 = DOVECOT_SHA256.get((arch, codename, package)) - if sha256 is None: - op = apt.packages(packages=[pkg_name]) - return None, bool(getattr(op, "changed", False)) + try: + # never fall back to the distro package: it is pinned to -1 and would + # in any case be a version we did not build and do not support + sha256 = DOVECOT_SHA256[(arch, deb_release, package)] + except KeyError: + raise ValueError(f"no dovecot build for {pkg_name} on deb{deb_release}/{arch}") + stamped_version = _stamped_version(deb_release) installed_versions = host.get_fact(DebPackages).get(pkg_name, []) - if DOVECOT_PACKAGE_VERSION in installed_versions: + if f"1:{stamped_version}" in installed_versions: return None, False - url_version = DOVECOT_ARCHIVE_VERSION.replace("+", "%2B") - deb_base = f"{pkg_name}_{url_version}_{arch}.deb" - primary_url = f"https://download.delta.chat/dovecot/{codename}/{url_version}/{deb_base}" - upstream_version = DOVECOT_ARCHIVE_VERSION.rsplit("-", 1)[0].replace("+", "%2B") - fallback_deb = f"{pkg_name}_{url_version}_{arch}_{codename}.deb" - fallback_url = f"https://github.com/chatmail/dovecot/releases/download/upstream%2F{upstream_version}/{fallback_deb}" + # Primary URL: flat structure with distro suffix in filename + primary_deb = f"{pkg_name}_{stamped_version}_{arch}.deb" + primary_url = f"https://download.delta.chat/dovecot/{primary_deb}" + # GitHub release files: escaped + in filename; the release tag stays + # distro-neutral, both distros ship in one combined release + tag_version = DOVECOT_ARCHIVE_VERSION.replace("+", "%2B") + fallback_deb = f"{pkg_name}_{stamped_version.replace('+', '%2B')}_{arch}.deb" + fallback_url = ( + f"https://github.com/chatmail/dovecot/releases/download/upstream%2F{tag_version}/{fallback_deb}" + ) url = _pick_url(primary_url, fallback_url) - deb_filename = f"/root/{deb_base}" + deb_filename = f"/root/{primary_deb}" files.download( name=f"Download {pkg_name}", diff --git a/cmdeploy/src/cmdeploy/tests/test_dovecot_deployer.py b/cmdeploy/src/cmdeploy/tests/test_dovecot_deployer.py index b7efb5a3..5d075be5 100644 --- a/cmdeploy/src/cmdeploy/tests/test_dovecot_deployer.py +++ b/cmdeploy/src/cmdeploy/tests/test_dovecot_deployer.py @@ -8,25 +8,35 @@ from pyinfra.facts.server import Command from cmdeploy.dovecot import deployer as dovecot_deployer +def _fact_name(key): + if isinstance(key, tuple): + return f"{key[0].__name__}{key[1:]!r}" + return key.__name__ + + def make_host(*fact_pairs): - """Build a mock host; get_fact(cls) dispatches to the provided facts mapping. + """Build a mock host; get_fact() dispatches to the provided facts mapping. Args: - *fact_pairs: tuples of (fact_class, fact_value) to register + *fact_pairs: (fact_class, value) to match any call of that fact, or + ((fact_class, *args), value) to match one specific call. Needed + for Command, which install() and check_restart() invoke with + different scripts; a bare Command entry would serve both. Returns: SimpleNamespace with get_fact that raises a clear error if an - unexpected fact type is requested. + unregistered fact is requested. """ facts = dict(fact_pairs) def get_fact(cls, *args): - if cls not in facts: - registered = ", ".join(c.__name__ for c in facts) - raise LookupError( - f"unexpected get_fact({cls.__name__}); only registered: {registered}" - ) - return facts[cls] + for key in ((cls, *args), cls): + if key in facts: + return facts[key] + registered = ", ".join(_fact_name(k) for k in facts) + raise LookupError( + f"unexpected get_fact({_fact_name((cls, *args))}); only registered: {registered}" + ) return SimpleNamespace(get_fact=get_fact) @@ -65,7 +75,9 @@ def track_shell(monkeypatch): def test_download_dovecot_package_skips_epoch_matched_install(monkeypatch): - epoch_version = dovecot_deployer.DOVECOT_PACKAGE_VERSION + # what dpkg reports after installing our deb: epoch + the +debNu1 suffix + # that chatmail/dovecot CI stamps via dch before building + epoch_version = f"1:{dovecot_deployer._stamped_version(12)}" downloads = [] monkeypatch.setattr( dovecot_deployer, @@ -83,17 +95,17 @@ def test_download_dovecot_package_skips_epoch_matched_install(monkeypatch): lambda **kwargs: downloads.append(kwargs), ) - deb, changed = dovecot_deployer._download_dovecot_package( - "core", "amd64", codename="bookworm" - ) + deb, changed = dovecot_deployer._download_dovecot_package("core", "amd64", deb_release=12) assert deb is None, f"expected no deb path when version matches, got {deb!r}" assert changed is False, "should not flag changed when version already installed" assert downloads == [], "should not download when version already installed" +@pytest.mark.parametrize("deb_release", [12, 13]) +@pytest.mark.parametrize("arch", ["amd64", "arm64"]) def test_download_dovecot_package_uses_archive_version_for_url_and_filename( - monkeypatch, + monkeypatch, deb_release, arch ): downloads = [] monkeypatch.setattr( @@ -113,19 +125,25 @@ def test_download_dovecot_package_uses_archive_version_for_url_and_filename( ) deb, changed = dovecot_deployer._download_dovecot_package( - "core", "amd64", codename="bookworm" + "core", arch, deb_release=deb_release ) - archive_version = dovecot_deployer.DOVECOT_ARCHIVE_VERSION.replace("+", "%2B") - expected_deb = f"/root/dovecot-core_{archive_version}_amd64.deb" + stamped = dovecot_deployer._stamped_version(deb_release) + expected_deb = f"/root/dovecot-core_{stamped}_{arch}.deb" - # Verify the returned path uses archive version, not package version (with epoch) + # path uses the stamped version, and deb filenames never carry the epoch assert changed is True, "should flag changed when package not yet installed" assert deb == expected_deb, f"deb path mismatch: {deb!r} != {expected_deb!r}" - assert dovecot_deployer.DOVECOT_PACKAGE_VERSION not in deb, ( - f"deb path should use archive version (no epoch), got {deb!r}" - ) + assert "1:" not in deb, f"deb filename must not contain the epoch, got {deb!r}" assert len(downloads) == 1, "files.download should be called exactly once" + # the checksum is the security boundary: verify the right table row is used + assert ( + downloads[0]["sha256sum"] + == dovecot_deployer.DOVECOT_SHA256[(arch, deb_release, "core")] + ), "must pass the sha256 matching (arch, release, package)" + assert f"deb{deb_release}u1" in downloads[0]["src"], ( + f"download URL should carry the deb{deb_release} suffix, got {downloads[0]['src']!r}" + ) def test_install_skips_dpkg_path_when_epoch_matched_packages_present( @@ -138,13 +156,13 @@ def test_install_skips_dpkg_path_when_epoch_matched_packages_present( ( dovecot_deployer.DebPackages, { - "dovecot-core": [dovecot_deployer.DOVECOT_PACKAGE_VERSION], - "dovecot-imapd": [dovecot_deployer.DOVECOT_PACKAGE_VERSION], - "dovecot-lmtpd": [dovecot_deployer.DOVECOT_PACKAGE_VERSION], + "dovecot-core": [f"1:{dovecot_deployer._stamped_version(12)}"], + "dovecot-imapd": [f"1:{dovecot_deployer._stamped_version(12)}"], + "dovecot-lmtpd": [f"1:{dovecot_deployer._stamped_version(12)}"], }, ), (dovecot_deployer.Arch, "x86_64"), - (Command, "bookworm"), + ((Command, dovecot_deployer.VERSION_ID_CMD), 'VERSION_ID="12"'), ), ) downloads = [] @@ -158,12 +176,10 @@ def test_install_skips_dpkg_path_when_epoch_matched_packages_present( assert downloads == [], "should not download when all packages epoch-matched" assert track_shell == [], "should not run dpkg when all packages epoch-matched" - assert deployer.need_restart is False, ( - "need_restart should be False when nothing changed" - ) + assert deployer.need_restart is False, "need_restart should be False when nothing changed" -def test_install_unsupported_arch_falls_back_to_apt( +def test_install_unsupported_arch_raises( deployer, patch_blocked, mock_files_put, track_shell, monkeypatch ): monkeypatch.setattr( @@ -171,30 +187,15 @@ def test_install_unsupported_arch_falls_back_to_apt( "host", make_host( (dovecot_deployer.Arch, "riscv64"), - (Command, "bookworm"), + ((Command, dovecot_deployer.VERSION_ID_CMD), 'VERSION_ID="12"'), ), ) - apt_calls = [] - # Mirrors apt.packages() return value: OperationMeta with .changed property. - # Only lmtpd triggers a change to verify |= accumulation of changed flags. - def fake_apt(**kwargs): - apt_calls.append(kwargs) - changed = "lmtpd" in kwargs["packages"][0] - return SimpleNamespace(changed=changed) + # we never fall back to the pinned distro package + with pytest.raises(ValueError, match="no dovecot build for dovecot-core"): + deployer.install() - monkeypatch.setattr(dovecot_deployer.apt, "packages", fake_apt) - - deployer.install() - - actual_pkgs = [c["packages"] for c in apt_calls] - assert actual_pkgs == [["dovecot-core"], ["dovecot-imapd"], ["dovecot-lmtpd"]], ( - f"expected apt install of core/imapd/lmtpd, got {actual_pkgs}" - ) - assert track_shell == [], "should not run dpkg for unsupported arch" - assert deployer.need_restart is True, ( - "need_restart should be True when apt installed a package" - ) + assert track_shell == [], "should not run apt-get for unsupported arch" def test_install_runs_dpkg_when_packages_need_download( @@ -206,7 +207,7 @@ def test_install_runs_dpkg_when_packages_need_download( make_host( (dovecot_deployer.DebPackages, {}), (dovecot_deployer.Arch, "x86_64"), - (Command, "bookworm"), + ((Command, dovecot_deployer.VERSION_ID_CMD), 'VERSION_ID="12"'), ), ) monkeypatch.setattr( @@ -222,9 +223,7 @@ def test_install_runs_dpkg_when_packages_need_download( deployer.install() - assert len(track_shell) == 1, ( - f"expected one server.shell() call for dpkg install, got {len(track_shell)}" - ) + assert len(track_shell) == 1, f"expected one server.shell() call for dpkg install, got {len(track_shell)}" cmds = track_shell[0]["commands"] assert len(cmds) == 1, f"expected single apt-get install command, got: {cmds}" assert "apt-get install -y" in cmds[0] @@ -232,9 +231,7 @@ def test_install_runs_dpkg_when_packages_need_download( assert '-o Dpkg::Options::="--force-confold"' in cmds[0] assert "--allow-downgrades" in cmds[0] assert ".deb" in cmds[0] - assert deployer.need_restart is True, ( - "need_restart should be True after dpkg install" - ) + assert deployer.need_restart is True, "need_restart should be True after dpkg install" def test_pick_url_falls_back_on_primary_error(monkeypatch): @@ -243,22 +240,43 @@ def test_pick_url_falls_back_on_primary_error(monkeypatch): monkeypatch.setattr(dovecot_deployer.urllib.request, "urlopen", raise_error) result = dovecot_deployer._pick_url("http://primary", "http://fallback") - assert result == "http://fallback", ( - f"should fall back when primary fails, got {result!r}" - ) + assert result == "http://fallback", f"should fall back when primary fails, got {result!r}" -def test_install_fails_on_unsupported_debian_version( - deployer, patch_blocked, monkeypatch -): +def test_install_fails_on_unsupported_debian_version(deployer, patch_blocked, monkeypatch): monkeypatch.setattr( dovecot_deployer, "host", make_host( (dovecot_deployer.Arch, "x86_64"), - (Command, "sid"), + ((Command, dovecot_deployer.VERSION_ID_CMD), 'VERSION_ID="99"'), ), ) - with pytest.raises(ValueError, match="Unsupported Debian codename"): + with pytest.raises(ValueError, match="no dovecot build for dovecot-core on deb99"): deployer.install() + + +@pytest.mark.parametrize( + "version_line", ["", None, "ID=debian"], ids=["empty", "none", "no-version-id"] +) +def test_parse_version_id_raises_without_version_id(version_line): + with pytest.raises(ValueError, match="cannot determine Debian release"): + dovecot_deployer._parse_version_id(version_line) + + +@pytest.mark.parametrize("deb_release", [12, 13]) +def test_parse_version_id(deb_release): + parsed = dovecot_deployer._parse_version_id(f'VERSION_ID="{deb_release}"\n') + assert parsed == deb_release + + +def test_dovecot_sha256_covers_all_packages_per_release(): + """Every release in the table needs all three packages on both arches.""" + table = dovecot_deployer.DOVECOT_SHA256 + expected = { + (arch, pkg) for arch in ("amd64", "arm64") for pkg in ("core", "imapd", "lmtpd") + } + for release in {r for _, r, _ in table}: + got = {(arch, pkg) for arch, r, pkg in table if r == release} + assert got == expected, f"deb{release} incomplete: {sorted(expected - got)}"