cqrenet/relay
5
0
Fork 0
mirror of https://github.com/chatmail/relay.git synced 2026-08-11 02:50:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

dovecot: security backports, new versioning scheme

Browse Source 
- Fix dovecot package download URLs for new
  [release](https://github.com/chatmail/dovecot/releases/tag/upstream%2F2.3.21%2Bdfsg1-3%2Bchatmail2)
  with
    - debian-security backport for 12 CVEs
    - distro-specific suffix (+deb{release}u1), enabling a simplified
      primary URL path and combined github releases
- Use VERSION_ID from os-release as deb_release instead of codename
  mapping, reorder hash-dict to match Github release page
- Remove redundant parsing/validation, let function validate against hash dict
- Update test expectations and test new versioning derivation
This commit is contained in:
j4n
2026-08-10 10:26:22 +02:00
parent 6b872446e1
commit 33c85ff58a
3 changed files with 149 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmdeploy/src/cmdeploy/deployers.py
+12 -8
View File
@@ -101,14 +101,18 @@ def _install_remote_venv_with_chatmaild(deployer) -> None:
# Remove venv if its Python major.minor doesn't match the system Python
server.shell(
name="remove stale chatmaild venv if python version changed",
commands=["\n".join([
f"if [ -d {remote_venv_dir} ]; then",
r" re='[0-9]+\.[0-9]+'", # match major.minor from 'Python X.Y.Z'"
' SYS_VERSION=$(python3 --version | grep -oE "$re")',
f' VENV_VERSION=$({remote_venv_dir}/bin/python --version 2>/dev/null | grep -oE "$re")',
f' [ "$SYS_VERSION" = "$VENV_VERSION" ] || rm -rf {remote_venv_dir}',
"fi",
])],
commands=[
"\n".join(
[
r"re='[0-9]+\.[0-9]+'", # major.minor out of 'Python X.Y.Z'
'sys_version=$(python3 --version 2>/dev/null | grep -oE "$re")',
f'venv_version=$({remote_venv_dir}/bin/python --version 2>/dev/null | grep -oE "$re")',
# an empty sys_version means we could not tell: keep the venv
f'[ -z "$sys_version" ] || [ "$sys_version" = "$venv_version" ] '
f"|| rm -rf {remote_venv_dir}",
]
)
],
)
pip.virtualenv(
cmdeploy/src/cmdeploy/dovecot/deployer.py
+55 -32
View File
@@ -4,7 +4,7 @@ from chatmaild.config import Config
from pyinfra import host
from pyinfra.facts.deb import DebPackages
from pyinfra.facts.server import Arch, Command, Sysctl
from pyinfra.operations import apt, files, server
from pyinfra.operations import files, server
from cmdeploy.basedeploy import (
Deployer,
@@ -14,22 +14,31 @@ from cmdeploy.basedeploy import (
is_in_container,
)
DOVECOT_ARCHIVE_VERSION = "2.3.21+dfsg1-3"
DOVECOT_PACKAGE_VERSION = f"1:{DOVECOT_ARCHIVE_VERSION}"
# distro-neutral base version, as committed in chatmail/dovecot debian/changelog
DOVECOT_ARCHIVE_VERSION = "2.3.21+dfsg1-3+chatmail2"
VERSION_ID_CMD = "grep '^VERSION_ID=' /etc/os-release"
def _stamped_version(deb_release: int) -> str:
"""Version as built, including the per-distro suffix stamped by
chatmail/dovecot CI into package version and filename."""
return f"{DOVECOT_ARCHIVE_VERSION}+deb{deb_release}u1"
DOVECOT_SHA256 = {
("amd64", "bookworm", "core"): "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d",
("arm64", "bookworm", "core"): "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9",
("amd64", "bookworm", "imapd"): "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86",
("arm64", "bookworm", "imapd"): "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f",
("amd64", "bookworm", "lmtpd"): "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab",
("arm64", "bookworm", "lmtpd"): "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f",
("amd64", "trixie", "core"): "406d3781ed81e0913c472077dcf62cb1106e3855983efa6e44ddf43b4b0c9be1",
("arm64", "trixie", "core"): "c75b0d9df11a77d07ebd8522920380c167fa47330ddefebe10575d99d0ecdf7f",
("amd64", "trixie", "imapd"): "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86",
("arm64", "trixie", "imapd"): "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f",
("amd64", "trixie", "lmtpd"): "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab",
("arm64", "trixie", "lmtpd"): "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f",
("amd64", 12, "core"): "ac3977264d9b9a6fcec53fd3f5cdd2a79ca8aa0324de530c07e535008540826e",
("arm64", 12, "core"): "21626c9c9b52cbdcf1a17b5c09e3c4043e69aa371bf83cc2fcb3b7ddaecdc109",
("amd64", 13, "core"): "47c242ef23c17e700ac19d52d82c9fdb2ebd757d8beb3a7f6781d2de59f87bd0",
("arm64", 13, "core"): "c14c53f112c875f698c4cb6e5870c605cd0a9dd98d35a66e94ceb1827f8020a3",
("amd64", 12, "imapd"): "92a7ab5fc7dc32886a0c34404f919f1335d397b48c467e0c1ef77e56978f60ea",
("arm64", 12, "imapd"): "9369fd566fec4df109ef23debf34ea0417ae85beb29cbe7de619d4d1f31b120c",
("amd64", 13, "imapd"): "e38cc1266455f937ed62f971ea859c47e1a99247841ed0ad946963b524cfdbc5",
("arm64", 13, "imapd"): "11d97dabf23171b37f8b1335dfdb81d408f8b95391aea6d4066aecc9fde01dfe",
("amd64", 12, "lmtpd"): "dc3de473789969f7dd3504ac8783da5e42a446d2d7a305a4e9d7081a6dfe71ab",
("arm64", 12, "lmtpd"): "ae2cbd6c5c43f6d8e2172997b055448f4c79238e2f99cd9ab9200a7d9f548908",
("amd64", 13, "lmtpd"): "833b243e28c7baff141ecf37456e310f5d836e7944a3b9f2fe5074adf0d6a418",
("arm64", 13, "lmtpd"): "55af47a121ba7e23966b20ddaab2dff7feba4b34677864e045e31a702afa180d",
}
@@ -43,13 +52,11 @@ class DovecotDeployer(Deployer):
def install(self):
arch = host.get_fact(Arch)
codename = (host.get_fact(Command, "grep '^VERSION_CODENAME=' /etc/os-release | cut -d= -f2") or "").strip()
if codename not in {key[1] for key in DOVECOT_SHA256}:
raise ValueError(f"Unsupported Debian codename: {codename!r}")
deb_release = _parse_version_id(host.get_fact(Command, VERSION_ID_CMD))
with blocked_service_startup():
debs = []
for pkg in ("core", "imapd", "lmtpd"):
deb, changed = _download_dovecot_package(pkg, arch, codename)
deb, changed = _download_dovecot_package(pkg, arch, deb_release)
self.need_restart |= changed
if deb:
debs.append(deb)
@@ -96,6 +103,15 @@ class DovecotDeployer(Deployer):
)
def _parse_version_id(version_line: str) -> int:
"""Debian major release from an /etc/os-release VERSION_ID line."""
_, _, raw = (version_line or "").strip().partition("=")
try:
return int(raw.strip('"'))
except ValueError:
raise ValueError(f"cannot determine Debian release from {version_line!r}")
def _pick_url(primary, fallback):
try:
req = urllib.request.Request(primary, method="HEAD")
@@ -105,29 +121,36 @@ def _pick_url(primary, fallback):
return fallback
def _download_dovecot_package(package: str, arch: str, codename: str) -> tuple[str | None, bool]:
def _download_dovecot_package(package: str, arch: str, deb_release: int) -> tuple[str | None, bool]:
"""Download a dovecot .deb if needed, return (path, changed)."""
arch = "amd64" if arch == "x86_64" else arch
arch = "arm64" if arch == "aarch64" else arch
pkg_name = f"dovecot-{package}"
sha256 = DOVECOT_SHA256.get((arch, codename, package))
if sha256 is None:
op = apt.packages(packages=[pkg_name])
return None, bool(getattr(op, "changed", False))
try:
# never fall back to the distro package: it is pinned to -1 and would
# in any case be a version we did not build and do not support
sha256 = DOVECOT_SHA256[(arch, deb_release, package)]
except KeyError:
raise ValueError(f"no dovecot build for {pkg_name} on deb{deb_release}/{arch}")
stamped_version = _stamped_version(deb_release)
installed_versions = host.get_fact(DebPackages).get(pkg_name, [])
if DOVECOT_PACKAGE_VERSION in installed_versions:
if f"1:{stamped_version}" in installed_versions:
return None, False
url_version = DOVECOT_ARCHIVE_VERSION.replace("+", "%2B")
deb_base = f"{pkg_name}_{url_version}_{arch}.deb"
primary_url = f"https://download.delta.chat/dovecot/{codename}/{url_version}/{deb_base}"
upstream_version = DOVECOT_ARCHIVE_VERSION.rsplit("-", 1)[0].replace("+", "%2B")
fallback_deb = f"{pkg_name}_{url_version}_{arch}_{codename}.deb"
fallback_url = f"https://github.com/chatmail/dovecot/releases/download/upstream%2F{upstream_version}/{fallback_deb}"
# Primary URL: flat structure with distro suffix in filename
primary_deb = f"{pkg_name}_{stamped_version}_{arch}.deb"
primary_url = f"https://download.delta.chat/dovecot/{primary_deb}"
# GitHub release files: escaped + in filename; the release tag stays
# distro-neutral, both distros ship in one combined release
tag_version = DOVECOT_ARCHIVE_VERSION.replace("+", "%2B")
fallback_deb = f"{pkg_name}_{stamped_version.replace('+', '%2B')}_{arch}.deb"
fallback_url = (
f"https://github.com/chatmail/dovecot/releases/download/upstream%2F{tag_version}/{fallback_deb}"
)
url = _pick_url(primary_url, fallback_url)
deb_filename = f"/root/{deb_base}"
deb_filename = f"/root/{primary_deb}"
files.download(
name=f"Download {pkg_name}",
cmdeploy/src/cmdeploy/tests/test_dovecot_deployer.py
+82 -64
View File
@@ -8,25 +8,35 @@ from pyinfra.facts.server import Command
from cmdeploy.dovecot import deployer as dovecot_deployer
def _fact_name(key):
if isinstance(key, tuple):
return f"{key[0].__name__}{key[1:]!r}"
return key.__name__
def make_host(*fact_pairs):
"""Build a mock host; get_fact(cls) dispatches to the provided facts mapping.
"""Build a mock host; get_fact() dispatches to the provided facts mapping.
Args:
*fact_pairs: tuples of (fact_class, fact_value) to register
*fact_pairs: (fact_class, value) to match any call of that fact, or
((fact_class, *args), value) to match one specific call. Needed
for Command, which install() and check_restart() invoke with
different scripts; a bare Command entry would serve both.
Returns:
SimpleNamespace with get_fact that raises a clear error if an
unexpected fact type is requested.
unregistered fact is requested.
"""
facts = dict(fact_pairs)
def get_fact(cls, *args):
if cls not in facts:
registered = ", ".join(c.__name__ for c in facts)
raise LookupError(
f"unexpected get_fact({cls.__name__}); only registered: {registered}"
)
return facts[cls]
for key in ((cls, *args), cls):
if key in facts:
return facts[key]
registered = ", ".join(_fact_name(k) for k in facts)
raise LookupError(
f"unexpected get_fact({_fact_name((cls, *args))}); only registered: {registered}"
)
return SimpleNamespace(get_fact=get_fact)
@@ -65,7 +75,9 @@ def track_shell(monkeypatch):
def test_download_dovecot_package_skips_epoch_matched_install(monkeypatch):
epoch_version = dovecot_deployer.DOVECOT_PACKAGE_VERSION
# what dpkg reports after installing our deb: epoch + the +debNu1 suffix
# that chatmail/dovecot CI stamps via dch before building
epoch_version = f"1:{dovecot_deployer._stamped_version(12)}"
downloads = []
monkeypatch.setattr(
dovecot_deployer,
@@ -83,17 +95,17 @@ def test_download_dovecot_package_skips_epoch_matched_install(monkeypatch):
lambda **kwargs: downloads.append(kwargs),
)
deb, changed = dovecot_deployer._download_dovecot_package(
"core", "amd64", codename="bookworm"
)
deb, changed = dovecot_deployer._download_dovecot_package("core", "amd64", deb_release=12)
assert deb is None, f"expected no deb path when version matches, got {deb!r}"
assert changed is False, "should not flag changed when version already installed"
assert downloads == [], "should not download when version already installed"
@pytest.mark.parametrize("deb_release", [12, 13])
@pytest.mark.parametrize("arch", ["amd64", "arm64"])
def test_download_dovecot_package_uses_archive_version_for_url_and_filename(
monkeypatch,
monkeypatch, deb_release, arch
):
downloads = []
monkeypatch.setattr(
@@ -113,19 +125,25 @@ def test_download_dovecot_package_uses_archive_version_for_url_and_filename(
)
deb, changed = dovecot_deployer._download_dovecot_package(
"core", "amd64", codename="bookworm"
"core", arch, deb_release=deb_release
)
archive_version = dovecot_deployer.DOVECOT_ARCHIVE_VERSION.replace("+", "%2B")
expected_deb = f"/root/dovecot-core_{archive_version}_amd64.deb"
stamped = dovecot_deployer._stamped_version(deb_release)
expected_deb = f"/root/dovecot-core_{stamped}_{arch}.deb"
# Verify the returned path uses archive version, not package version (with epoch)
# path uses the stamped version, and deb filenames never carry the epoch
assert changed is True, "should flag changed when package not yet installed"
assert deb == expected_deb, f"deb path mismatch: {deb!r} != {expected_deb!r}"
assert dovecot_deployer.DOVECOT_PACKAGE_VERSION not in deb, (
f"deb path should use archive version (no epoch), got {deb!r}"
)
assert "1:" not in deb, f"deb filename must not contain the epoch, got {deb!r}"
assert len(downloads) == 1, "files.download should be called exactly once"
# the checksum is the security boundary: verify the right table row is used
assert (
downloads[0]["sha256sum"]
== dovecot_deployer.DOVECOT_SHA256[(arch, deb_release, "core")]
), "must pass the sha256 matching (arch, release, package)"
assert f"deb{deb_release}u1" in downloads[0]["src"], (
f"download URL should carry the deb{deb_release} suffix, got {downloads[0]['src']!r}"
)
def test_install_skips_dpkg_path_when_epoch_matched_packages_present(
@@ -138,13 +156,13 @@ def test_install_skips_dpkg_path_when_epoch_matched_packages_present(
(
dovecot_deployer.DebPackages,
{
"dovecot-core": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
"dovecot-imapd": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
"dovecot-lmtpd": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
"dovecot-core": [f"1:{dovecot_deployer._stamped_version(12)}"],
"dovecot-imapd": [f"1:{dovecot_deployer._stamped_version(12)}"],
"dovecot-lmtpd": [f"1:{dovecot_deployer._stamped_version(12)}"],
},
),
(dovecot_deployer.Arch, "x86_64"),
(Command, "bookworm"),
((Command, dovecot_deployer.VERSION_ID_CMD), 'VERSION_ID="12"'),
),
)
downloads = []
@@ -158,12 +176,10 @@ def test_install_skips_dpkg_path_when_epoch_matched_packages_present(
assert downloads == [], "should not download when all packages epoch-matched"
assert track_shell == [], "should not run dpkg when all packages epoch-matched"
assert deployer.need_restart is False, (
"need_restart should be False when nothing changed"
)
assert deployer.need_restart is False, "need_restart should be False when nothing changed"
def test_install_unsupported_arch_falls_back_to_apt(
def test_install_unsupported_arch_raises(
deployer, patch_blocked, mock_files_put, track_shell, monkeypatch
):
monkeypatch.setattr(
@@ -171,30 +187,15 @@ def test_install_unsupported_arch_falls_back_to_apt(
"host",
make_host(
(dovecot_deployer.Arch, "riscv64"),
(Command, "bookworm"),
((Command, dovecot_deployer.VERSION_ID_CMD), 'VERSION_ID="12"'),
),
)
apt_calls = []
# Mirrors apt.packages() return value: OperationMeta with .changed property.
# Only lmtpd triggers a change to verify |= accumulation of changed flags.
def fake_apt(**kwargs):
apt_calls.append(kwargs)
changed = "lmtpd" in kwargs["packages"][0]
return SimpleNamespace(changed=changed)
# we never fall back to the pinned distro package
with pytest.raises(ValueError, match="no dovecot build for dovecot-core"):
deployer.install()
monkeypatch.setattr(dovecot_deployer.apt, "packages", fake_apt)
deployer.install()
actual_pkgs = [c["packages"] for c in apt_calls]
assert actual_pkgs == [["dovecot-core"], ["dovecot-imapd"], ["dovecot-lmtpd"]], (
f"expected apt install of core/imapd/lmtpd, got {actual_pkgs}"
)
assert track_shell == [], "should not run dpkg for unsupported arch"
assert deployer.need_restart is True, (
"need_restart should be True when apt installed a package"
)
assert track_shell == [], "should not run apt-get for unsupported arch"
def test_install_runs_dpkg_when_packages_need_download(
@@ -206,7 +207,7 @@ def test_install_runs_dpkg_when_packages_need_download(
make_host(
(dovecot_deployer.DebPackages, {}),
(dovecot_deployer.Arch, "x86_64"),
(Command, "bookworm"),
((Command, dovecot_deployer.VERSION_ID_CMD), 'VERSION_ID="12"'),
),
)
monkeypatch.setattr(
@@ -222,9 +223,7 @@ def test_install_runs_dpkg_when_packages_need_download(
deployer.install()
assert len(track_shell) == 1, (
f"expected one server.shell() call for dpkg install, got {len(track_shell)}"
)
assert len(track_shell) == 1, f"expected one server.shell() call for dpkg install, got {len(track_shell)}"
cmds = track_shell[0]["commands"]
assert len(cmds) == 1, f"expected single apt-get install command, got: {cmds}"
assert "apt-get install -y" in cmds[0]
@@ -232,9 +231,7 @@ def test_install_runs_dpkg_when_packages_need_download(
assert '-o Dpkg::Options::="--force-confold"' in cmds[0]
assert "--allow-downgrades" in cmds[0]
assert ".deb" in cmds[0]
assert deployer.need_restart is True, (
"need_restart should be True after dpkg install"
)
assert deployer.need_restart is True, "need_restart should be True after dpkg install"
def test_pick_url_falls_back_on_primary_error(monkeypatch):
@@ -243,22 +240,43 @@ def test_pick_url_falls_back_on_primary_error(monkeypatch):
monkeypatch.setattr(dovecot_deployer.urllib.request, "urlopen", raise_error)
result = dovecot_deployer._pick_url("http://primary", "http://fallback")
assert result == "http://fallback", (
f"should fall back when primary fails, got {result!r}"
)
assert result == "http://fallback", f"should fall back when primary fails, got {result!r}"
def test_install_fails_on_unsupported_debian_version(
deployer, patch_blocked, monkeypatch
):
def test_install_fails_on_unsupported_debian_version(deployer, patch_blocked, monkeypatch):
monkeypatch.setattr(
dovecot_deployer,
"host",
make_host(
(dovecot_deployer.Arch, "x86_64"),
(Command, "sid"),
((Command, dovecot_deployer.VERSION_ID_CMD), 'VERSION_ID="99"'),
),
)
with pytest.raises(ValueError, match="Unsupported Debian codename"):
with pytest.raises(ValueError, match="no dovecot build for dovecot-core on deb99"):
deployer.install()
@pytest.mark.parametrize(
"version_line", ["", None, "ID=debian"], ids=["empty", "none", "no-version-id"]
)
def test_parse_version_id_raises_without_version_id(version_line):
with pytest.raises(ValueError, match="cannot determine Debian release"):
dovecot_deployer._parse_version_id(version_line)
@pytest.mark.parametrize("deb_release", [12, 13])
def test_parse_version_id(deb_release):
parsed = dovecot_deployer._parse_version_id(f'VERSION_ID="{deb_release}"\n')
assert parsed == deb_release
def test_dovecot_sha256_covers_all_packages_per_release():
"""Every release in the table needs all three packages on both arches."""
table = dovecot_deployer.DOVECOT_SHA256
expected = {
(arch, pkg) for arch in ("amd64", "arm64") for pkg in ("core", "imapd", "lmtpd")
}
for release in {r for _, r, _ in table}:
got = {(arch, pkg) for arch, r, pkg in table if r == release}
assert got == expected, f"deb{release} incomplete: {sorted(expected - got)}"