Compare commits

..

5 Commits

Author SHA1 Message Date
j4n 33c85ff58a dovecot: security backports, new versioning scheme
- Fix dovecot package download URLs for new
  [release](https://github.com/chatmail/dovecot/releases/tag/upstream%2F2.3.21%2Bdfsg1-3%2Bchatmail2)
  with
    - debian-security backport for 12 CVEs
    - distro-specific suffix (+deb{release}u1), enabling a simplified
      primary URL path and combined github releases
- Use VERSION_ID from os-release as deb_release instead of codename
  mapping, reorder hash-dict to match Github release page
- Remove redundant parsing/validation, let function validate against hash dict
- Update test expectations and test new versioning derivation
2026-08-10 13:39:55 +02:00
j4n 6b872446e1 fix(cmdeploy): check venv python versions and purge if mismatched
`cmdeploy run` fails after system upgrade to Debian 13 with "Fatal Python
error: init_fs_encoding: failed to get the Python codec of the filesystem
encoding" indicating a Python version missmatch. Check for both versions and
remove old `remote_venv_dir` on mismatch to allow clean reinitialization by
subsequent pip.virtualenv().
2026-08-10 10:45:46 +02:00
j4n f0fe7256b7 ci: temporarily build docker packages for bookworm branch 2026-08-10 10:44:51 +02:00
j4n ab14cc7319 dovecot: add multi-dist/Debian trixie support
- Install .debs via apt-get install instead of dpkg+fix-broken
- Package hashes are now keyed by (arch, codename, pkg):
  - download.delta.chat uploads now go to dovecot/{distro}/{version}/
  - GitHub release packages get a _{distro}.deb suffix to allow for
    combined releases.

Tests:
- updated to support this and add a test to check for the unsupported
  release version case
- fix make_host to accept extra args from Command fact
- assert single apt-get install command
2026-08-10 10:44:51 +02:00
j4n 53b5a8189b dovecot: pin dovecot-* to priority -1 before any apt operation
Prevent Trixie from somehow pulling in dovecot 2.4 before we get to install.
2026-08-10 10:44:51 +02:00
4 changed files with 195 additions and 100 deletions
+1 -1
View File
@@ -8,7 +8,7 @@ name: Trigger Docker build
on:
push:
branches: [main]
branches: [main, j4n/dovecot-multidist]
tags: ['[0-9]+.[0-9]+.[0-9]+']
workflow_dispatch:
+23
View File
@@ -98,6 +98,23 @@ def _install_remote_venv_with_chatmaild(deployer) -> None:
dest=remote_dist_file,
)
# Remove venv if its Python major.minor doesn't match the system Python
server.shell(
name="remove stale chatmaild venv if python version changed",
commands=[
"\n".join(
[
r"re='[0-9]+\.[0-9]+'", # major.minor out of 'Python X.Y.Z'
'sys_version=$(python3 --version 2>/dev/null | grep -oE "$re")',
f'venv_version=$({remote_venv_dir}/bin/python --version 2>/dev/null | grep -oE "$re")',
# an empty sys_version means we could not tell: keep the venv
f'[ -z "$sys_version" ] || [ "$sys_version" = "$venv_version" ] '
f"|| rm -rf {remote_venv_dir}",
]
)
],
)
pip.virtualenv(
name=f"chatmaild virtualenv {remote_venv_dir}",
path=remote_venv_dir,
@@ -406,6 +423,12 @@ class ChatmailDeployer(Deployer):
src=BytesIO(b'APT::Install-Recommends "false";\n'),
dest="/etc/apt/apt.conf.d/00InstallRecommends",
)
# Pin dovecot-* to priority -1 before any apt operation, apt should
# never manage dovecot as our version might be lower than the distro's.
self.put_file(
src=StringIO("Package: dovecot-*\nPin: version *\nPin-Priority: -1\n"),
dest="/etc/apt/preferences.d/pin-dovecot",
)
apt.update(name="apt update", cache_time=24 * 3600)
apt.upgrade(name="upgrade apt packages", auto_remove=True)
+66 -39
View File
@@ -1,11 +1,10 @@
import io
import urllib.request
from chatmaild.config import Config
from pyinfra import host
from pyinfra.facts.deb import DebPackages
from pyinfra.facts.server import Arch, Command, Sysctl
from pyinfra.operations import apt, files, server
from pyinfra.operations import files, server
from cmdeploy.basedeploy import (
Deployer,
@@ -15,16 +14,31 @@ from cmdeploy.basedeploy import (
is_in_container,
)
DOVECOT_ARCHIVE_VERSION = "2.3.21+dfsg1-3"
DOVECOT_PACKAGE_VERSION = f"1:{DOVECOT_ARCHIVE_VERSION}"
# distro-neutral base version, as committed in chatmail/dovecot debian/changelog
DOVECOT_ARCHIVE_VERSION = "2.3.21+dfsg1-3+chatmail2"
VERSION_ID_CMD = "grep '^VERSION_ID=' /etc/os-release"
def _stamped_version(deb_release: int) -> str:
"""Version as built, including the per-distro suffix stamped by
chatmail/dovecot CI into package version and filename."""
return f"{DOVECOT_ARCHIVE_VERSION}+deb{deb_release}u1"
DOVECOT_SHA256 = {
("core", "amd64"): "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d",
("core", "arm64"): "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9",
("imapd", "amd64"): "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86",
("imapd", "arm64"): "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f",
("lmtpd", "amd64"): "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab",
("lmtpd", "arm64"): "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f",
("amd64", 12, "core"): "ac3977264d9b9a6fcec53fd3f5cdd2a79ca8aa0324de530c07e535008540826e",
("arm64", 12, "core"): "21626c9c9b52cbdcf1a17b5c09e3c4043e69aa371bf83cc2fcb3b7ddaecdc109",
("amd64", 13, "core"): "47c242ef23c17e700ac19d52d82c9fdb2ebd757d8beb3a7f6781d2de59f87bd0",
("arm64", 13, "core"): "c14c53f112c875f698c4cb6e5870c605cd0a9dd98d35a66e94ceb1827f8020a3",
("amd64", 12, "imapd"): "92a7ab5fc7dc32886a0c34404f919f1335d397b48c467e0c1ef77e56978f60ea",
("arm64", 12, "imapd"): "9369fd566fec4df109ef23debf34ea0417ae85beb29cbe7de619d4d1f31b120c",
("amd64", 13, "imapd"): "e38cc1266455f937ed62f971ea859c47e1a99247841ed0ad946963b524cfdbc5",
("arm64", 13, "imapd"): "11d97dabf23171b37f8b1335dfdb81d408f8b95391aea6d4066aecc9fde01dfe",
("amd64", 12, "lmtpd"): "dc3de473789969f7dd3504ac8783da5e42a446d2d7a305a4e9d7081a6dfe71ab",
("arm64", 12, "lmtpd"): "ae2cbd6c5c43f6d8e2172997b055448f4c79238e2f99cd9ab9200a7d9f548908",
("amd64", 13, "lmtpd"): "833b243e28c7baff141ecf37456e310f5d836e7944a3b9f2fe5074adf0d6a418",
("arm64", 13, "lmtpd"): "55af47a121ba7e23966b20ddaab2dff7feba4b34677864e045e31a702afa180d",
}
@@ -38,34 +52,30 @@ class DovecotDeployer(Deployer):
def install(self):
arch = host.get_fact(Arch)
deb_release = _parse_version_id(host.get_fact(Command, VERSION_ID_CMD))
with blocked_service_startup():
debs = []
for pkg in ("core", "imapd", "lmtpd"):
deb, changed = _download_dovecot_package(pkg, arch)
deb, changed = _download_dovecot_package(pkg, arch, deb_release)
self.need_restart |= changed
if deb:
debs.append(deb)
if debs:
deb_list = " ".join(debs)
# First dpkg may fail on missing dependencies (stderr suppressed);
# apt-get --fix-broken pulls them in, then dpkg retries cleanly.
# apt-get install with local .deb paths resolves depends
# against the configured repos (e.g. pulls libwrap0),
# The pin file written earlier by ChatmailDeployer prevents apt
# from installing a 'wrong' version
server.shell(
name="Install dovecot packages",
commands=[
f"dpkg --force-confdef --force-confold -i {deb_list} 2> /dev/null || true",
"DEBIAN_FRONTEND=noninteractive apt-get -y --fix-broken install",
f"dpkg --force-confdef --force-confold -i {deb_list}",
"DEBIAN_FRONTEND=noninteractive apt-get install -y "
'-o Dpkg::Options::="--force-confdef" '
'-o Dpkg::Options::="--force-confold" '
f"--allow-downgrades {deb_list}",
],
)
self.need_restart = True
self.put_file(
src=io.StringIO(
"Package: dovecot-*\n"
"Pin: version *\n"
"Pin-Priority: -1\n"
),
dest="/etc/apt/preferences.d/pin-dovecot",
)
def configure(self):
configure_remote_units(self, self.config.mail_domain_bare, self.units)
@@ -78,7 +88,7 @@ class DovecotDeployer(Deployer):
if not self.disable_mail and not self.need_restart:
stale = host.get_fact(
Command,
'pid=$(systemctl show -p MainPID --value dovecot.service 2>/dev/null);'
"pid=$(systemctl show -p MainPID --value dovecot.service 2>/dev/null);"
' [ "${pid:-0}" != "0" ] && readlink "/proc/$pid/exe" 2>/dev/null | grep -q "(deleted)"'
" && echo STALE || true",
)
@@ -93,6 +103,15 @@ class DovecotDeployer(Deployer):
)
def _parse_version_id(version_line: str) -> int:
"""Debian major release from an /etc/os-release VERSION_ID line."""
_, _, raw = (version_line or "").strip().partition("=")
try:
return int(raw.strip('"'))
except ValueError:
raise ValueError(f"cannot determine Debian release from {version_line!r}")
def _pick_url(primary, fallback):
try:
req = urllib.request.Request(primary, method="HEAD")
@@ -102,27 +121,36 @@ def _pick_url(primary, fallback):
return fallback
def _download_dovecot_package(package: str, arch: str) -> tuple[str | None, bool]:
def _download_dovecot_package(package: str, arch: str, deb_release: int) -> tuple[str | None, bool]:
"""Download a dovecot .deb if needed, return (path, changed)."""
arch = "amd64" if arch == "x86_64" else arch
arch = "arm64" if arch == "aarch64" else arch
pkg_name = f"dovecot-{package}"
sha256 = DOVECOT_SHA256.get((package, arch))
if sha256 is None:
op = apt.packages(packages=[pkg_name])
return None, bool(getattr(op, "changed", False))
try:
# never fall back to the distro package: it is pinned to -1 and would
# in any case be a version we did not build and do not support
sha256 = DOVECOT_SHA256[(arch, deb_release, package)]
except KeyError:
raise ValueError(f"no dovecot build for {pkg_name} on deb{deb_release}/{arch}")
stamped_version = _stamped_version(deb_release)
installed_versions = host.get_fact(DebPackages).get(pkg_name, [])
if DOVECOT_PACKAGE_VERSION in installed_versions:
if f"1:{stamped_version}" in installed_versions:
return None, False
url_version = DOVECOT_ARCHIVE_VERSION.replace("+", "%2B")
deb_base = f"{pkg_name}_{url_version}_{arch}.deb"
primary_url = f"https://download.delta.chat/dovecot/{deb_base}"
fallback_url = f"https://github.com/chatmail/dovecot/releases/download/upstream%2F{url_version}/{deb_base}"
# Primary URL: flat structure with distro suffix in filename
primary_deb = f"{pkg_name}_{stamped_version}_{arch}.deb"
primary_url = f"https://download.delta.chat/dovecot/{primary_deb}"
# GitHub release files: escaped + in filename; the release tag stays
# distro-neutral, both distros ship in one combined release
tag_version = DOVECOT_ARCHIVE_VERSION.replace("+", "%2B")
fallback_deb = f"{pkg_name}_{stamped_version.replace('+', '%2B')}_{arch}.deb"
fallback_url = (
f"https://github.com/chatmail/dovecot/releases/download/upstream%2F{tag_version}/{fallback_deb}"
)
url = _pick_url(primary_url, fallback_url)
deb_filename = f"/root/{deb_base}"
deb_filename = f"/root/{primary_deb}"
files.download(
name=f"Download {pkg_name}",
@@ -134,6 +162,7 @@ def _download_dovecot_package(package: str, arch: str) -> tuple[str | None, bool
return deb_filename, True
def _configure_dovecot(deployer, config: Config, debug: bool = False):
"""Configures Dovecot IMAP server."""
deployer.put_template(
@@ -144,9 +173,7 @@ def _configure_dovecot(deployer, config: Config, debug: bool = False):
disable_ipv6=config.disable_ipv6,
)
deployer.put_file("dovecot/auth.conf", "/etc/dovecot/auth.conf")
deployer.put_file(
"dovecot/push_notification.lua", "/etc/dovecot/push_notification.lua"
)
deployer.put_file("dovecot/push_notification.lua", "/etc/dovecot/push_notification.lua")
# as per https://doc.dovecot.org/2.3/configuration_manual/os/
# it is recommended to set the following inotify limits
@@ -3,29 +3,40 @@ from types import SimpleNamespace
import pytest
from pyinfra.facts.deb import DebPackages
from pyinfra.facts.server import Command
from cmdeploy.dovecot import deployer as dovecot_deployer
def _fact_name(key):
if isinstance(key, tuple):
return f"{key[0].__name__}{key[1:]!r}"
return key.__name__
def make_host(*fact_pairs):
"""Build a mock host; get_fact(cls) dispatches to the provided facts mapping.
"""Build a mock host; get_fact() dispatches to the provided facts mapping.
Args:
*fact_pairs: tuples of (fact_class, fact_value) to register
*fact_pairs: (fact_class, value) to match any call of that fact, or
((fact_class, *args), value) to match one specific call. Needed
for Command, which install() and check_restart() invoke with
different scripts; a bare Command entry would serve both.
Returns:
SimpleNamespace with get_fact that raises a clear error if an
unexpected fact type is requested.
unregistered fact is requested.
"""
facts = dict(fact_pairs)
def get_fact(cls):
if cls not in facts:
registered = ", ".join(c.__name__ for c in facts)
raise LookupError(
f"unexpected get_fact({cls.__name__}); only registered: {registered}"
)
return facts[cls]
def get_fact(cls, *args):
for key in ((cls, *args), cls):
if key in facts:
return facts[key]
registered = ", ".join(_fact_name(k) for k in facts)
raise LookupError(
f"unexpected get_fact({_fact_name((cls, *args))}); only registered: {registered}"
)
return SimpleNamespace(get_fact=get_fact)
@@ -64,7 +75,9 @@ def track_shell(monkeypatch):
def test_download_dovecot_package_skips_epoch_matched_install(monkeypatch):
epoch_version = dovecot_deployer.DOVECOT_PACKAGE_VERSION
# what dpkg reports after installing our deb: epoch + the +debNu1 suffix
# that chatmail/dovecot CI stamps via dch before building
epoch_version = f"1:{dovecot_deployer._stamped_version(12)}"
downloads = []
monkeypatch.setattr(
dovecot_deployer,
@@ -82,15 +95,17 @@ def test_download_dovecot_package_skips_epoch_matched_install(monkeypatch):
lambda **kwargs: downloads.append(kwargs),
)
deb, changed = dovecot_deployer._download_dovecot_package("core", "amd64")
deb, changed = dovecot_deployer._download_dovecot_package("core", "amd64", deb_release=12)
assert deb is None, f"expected no deb path when version matches, got {deb!r}"
assert changed is False, "should not flag changed when version already installed"
assert downloads == [], "should not download when version already installed"
@pytest.mark.parametrize("deb_release", [12, 13])
@pytest.mark.parametrize("arch", ["amd64", "arm64"])
def test_download_dovecot_package_uses_archive_version_for_url_and_filename(
monkeypatch,
monkeypatch, deb_release, arch
):
downloads = []
monkeypatch.setattr(
@@ -109,18 +124,26 @@ def test_download_dovecot_package_uses_archive_version_for_url_and_filename(
lambda **kwargs: downloads.append(kwargs),
)
deb, changed = dovecot_deployer._download_dovecot_package("core", "amd64")
deb, changed = dovecot_deployer._download_dovecot_package(
"core", arch, deb_release=deb_release
)
archive_version = dovecot_deployer.DOVECOT_ARCHIVE_VERSION.replace("+", "%2B")
expected_deb = f"/root/dovecot-core_{archive_version}_amd64.deb"
stamped = dovecot_deployer._stamped_version(deb_release)
expected_deb = f"/root/dovecot-core_{stamped}_{arch}.deb"
# Verify the returned path uses archive version, not package version (with epoch)
# path uses the stamped version, and deb filenames never carry the epoch
assert changed is True, "should flag changed when package not yet installed"
assert deb == expected_deb, f"deb path mismatch: {deb!r} != {expected_deb!r}"
assert dovecot_deployer.DOVECOT_PACKAGE_VERSION not in deb, (
f"deb path should use archive version (no epoch), got {deb!r}"
)
assert "1:" not in deb, f"deb filename must not contain the epoch, got {deb!r}"
assert len(downloads) == 1, "files.download should be called exactly once"
# the checksum is the security boundary: verify the right table row is used
assert (
downloads[0]["sha256sum"]
== dovecot_deployer.DOVECOT_SHA256[(arch, deb_release, "core")]
), "must pass the sha256 matching (arch, release, package)"
assert f"deb{deb_release}u1" in downloads[0]["src"], (
f"download URL should carry the deb{deb_release} suffix, got {downloads[0]['src']!r}"
)
def test_install_skips_dpkg_path_when_epoch_matched_packages_present(
@@ -133,12 +156,13 @@ def test_install_skips_dpkg_path_when_epoch_matched_packages_present(
(
dovecot_deployer.DebPackages,
{
"dovecot-core": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
"dovecot-imapd": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
"dovecot-lmtpd": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
"dovecot-core": [f"1:{dovecot_deployer._stamped_version(12)}"],
"dovecot-imapd": [f"1:{dovecot_deployer._stamped_version(12)}"],
"dovecot-lmtpd": [f"1:{dovecot_deployer._stamped_version(12)}"],
},
),
(dovecot_deployer.Arch, "x86_64"),
((Command, dovecot_deployer.VERSION_ID_CMD), 'VERSION_ID="12"'),
),
)
downloads = []
@@ -152,41 +176,26 @@ def test_install_skips_dpkg_path_when_epoch_matched_packages_present(
assert downloads == [], "should not download when all packages epoch-matched"
assert track_shell == [], "should not run dpkg when all packages epoch-matched"
assert deployer.need_restart is False, (
"need_restart should be False when nothing changed"
)
assert deployer.need_restart is False, "need_restart should be False when nothing changed"
def test_install_unsupported_arch_falls_back_to_apt(
def test_install_unsupported_arch_raises(
deployer, patch_blocked, mock_files_put, track_shell, monkeypatch
):
# For unsupported architectures, all fact lookups return the arch string.
monkeypatch.setattr(
dovecot_deployer,
"host",
SimpleNamespace(get_fact=lambda cls: "riscv64"),
make_host(
(dovecot_deployer.Arch, "riscv64"),
((Command, dovecot_deployer.VERSION_ID_CMD), 'VERSION_ID="12"'),
),
)
apt_calls = []
# Mirrors apt.packages() return value: OperationMeta with .changed property.
# Only lmtpd triggers a change to verify |= accumulation of changed flags.
def fake_apt(**kwargs):
apt_calls.append(kwargs)
changed = "lmtpd" in kwargs["packages"][0]
return SimpleNamespace(changed=changed)
# we never fall back to the pinned distro package
with pytest.raises(ValueError, match="no dovecot build for dovecot-core"):
deployer.install()
monkeypatch.setattr(dovecot_deployer.apt, "packages", fake_apt)
deployer.install()
actual_pkgs = [c["packages"] for c in apt_calls]
assert actual_pkgs == [["dovecot-core"], ["dovecot-imapd"], ["dovecot-lmtpd"]], (
f"expected apt install of core/imapd/lmtpd, got {actual_pkgs}"
)
assert track_shell == [], "should not run dpkg for unsupported arch"
assert deployer.need_restart is True, (
"need_restart should be True when apt installed a package"
)
assert track_shell == [], "should not run apt-get for unsupported arch"
def test_install_runs_dpkg_when_packages_need_download(
@@ -198,6 +207,7 @@ def test_install_runs_dpkg_when_packages_need_download(
make_host(
(dovecot_deployer.DebPackages, {}),
(dovecot_deployer.Arch, "x86_64"),
((Command, dovecot_deployer.VERSION_ID_CMD), 'VERSION_ID="12"'),
),
)
monkeypatch.setattr(
@@ -213,17 +223,15 @@ def test_install_runs_dpkg_when_packages_need_download(
deployer.install()
assert len(track_shell) == 1, (
f"expected one server.shell() call for dpkg install, got {len(track_shell)}"
)
assert len(track_shell) == 1, f"expected one server.shell() call for dpkg install, got {len(track_shell)}"
cmds = track_shell[0]["commands"]
assert len(cmds) == 3, f"expected 3 dpkg/apt commands, got: {cmds}"
assert cmds[0].startswith("dpkg --force-confdef --force-confold -i ")
assert "apt-get -y --fix-broken install" in cmds[1]
assert cmds[2].startswith("dpkg --force-confdef --force-confold -i ")
assert deployer.need_restart is True, (
"need_restart should be True after dpkg install"
)
assert len(cmds) == 1, f"expected single apt-get install command, got: {cmds}"
assert "apt-get install -y" in cmds[0]
assert '-o Dpkg::Options::="--force-confdef"' in cmds[0]
assert '-o Dpkg::Options::="--force-confold"' in cmds[0]
assert "--allow-downgrades" in cmds[0]
assert ".deb" in cmds[0]
assert deployer.need_restart is True, "need_restart should be True after dpkg install"
def test_pick_url_falls_back_on_primary_error(monkeypatch):
@@ -232,6 +240,43 @@ def test_pick_url_falls_back_on_primary_error(monkeypatch):
monkeypatch.setattr(dovecot_deployer.urllib.request, "urlopen", raise_error)
result = dovecot_deployer._pick_url("http://primary", "http://fallback")
assert result == "http://fallback", (
f"should fall back when primary fails, got {result!r}"
assert result == "http://fallback", f"should fall back when primary fails, got {result!r}"
def test_install_fails_on_unsupported_debian_version(deployer, patch_blocked, monkeypatch):
monkeypatch.setattr(
dovecot_deployer,
"host",
make_host(
(dovecot_deployer.Arch, "x86_64"),
((Command, dovecot_deployer.VERSION_ID_CMD), 'VERSION_ID="99"'),
),
)
with pytest.raises(ValueError, match="no dovecot build for dovecot-core on deb99"):
deployer.install()
@pytest.mark.parametrize(
"version_line", ["", None, "ID=debian"], ids=["empty", "none", "no-version-id"]
)
def test_parse_version_id_raises_without_version_id(version_line):
with pytest.raises(ValueError, match="cannot determine Debian release"):
dovecot_deployer._parse_version_id(version_line)
@pytest.mark.parametrize("deb_release", [12, 13])
def test_parse_version_id(deb_release):
parsed = dovecot_deployer._parse_version_id(f'VERSION_ID="{deb_release}"\n')
assert parsed == deb_release
def test_dovecot_sha256_covers_all_packages_per_release():
"""Every release in the table needs all three packages on both arches."""
table = dovecot_deployer.DOVECOT_SHA256
expected = {
(arch, pkg) for arch in ("amd64", "arm64") for pkg in ("core", "imapd", "lmtpd")
}
for release in {r for _, r, _ in table}:
got = {(arch, pkg) for arch, r, pkg in table if r == release}
assert got == expected, f"deb{release} incomplete: {sorted(expected - got)}"