feat: Extend IMAP METADATA to have a unixtimestamp key

.github/workflows/docker-dispatch.yaml

@@ -8,7 +8,7 @@ name: Trigger Docker build
 
 on:
   push:
-    branches: [main, j4n/dovecot-multidist]
+    branches: [main]
     tags: ['[0-9]+.[0-9]+.[0-9]+']
   workflow_dispatch:
 

chatmaild/pyproject.toml

@@ -10,6 +10,7 @@ dependencies = [
   "filelock",
   "requests",
   "crypt-r >= 3.13.1 ; python_version >= '3.11'",
+  "domain-validator",
 ]
 
 [tool.setuptools]

chatmaild/src/chatmaild/config.py

@@ -2,6 +2,7 @@ import ipaddress
 from pathlib import Path
 
 import iniconfig
+from domain_validator import DomainValidator
 
 from chatmaild.user import User
 
@@ -24,6 +25,7 @@ class Config:
             self.mail_domain = f"[{raw_domain}]"
             self.postfix_myhostname = ipaddress.IPv4Address(raw_domain).reverse_pointer
         else:
+            DomainValidator().validate_domain_re(raw_domain)
             self.ipv4_relay = None
             self.mail_domain = raw_domain
             self.postfix_myhostname = raw_domain

chatmaild/src/chatmaild/metadata.py

@@ -125,6 +125,8 @@ class MetadataDictProxy(DictProxy):
                         case "maxsmtprecipients":
                             # postfix default  (see "postconf smtpd_recipient_limit")
                             return "O1000\n"
+                        case "unixtimestamp":
+                            return f"O{int(time.time())}\n"
 
         logging.warning(f"lookup ignored: {parts!r}")
         return "N\n"

cmdeploy/src/cmdeploy/deployers.py

@@ -389,12 +389,6 @@ class ChatmailDeployer(Deployer):
             src=BytesIO(b'APT::Install-Recommends "false";\n'),
             dest="/etc/apt/apt.conf.d/00InstallRecommends",
         )
-        # Pin dovecot-* to priority -1 before any apt operation, apt should
-        # never manage dovecot as our version might be lower than the distro's.
-        self.put_file(
-            src=StringIO("Package: dovecot-*\nPin: version *\nPin-Priority: -1\n"),
-            dest="/etc/apt/preferences.d/pin-dovecot",
-        )
         apt.update(name="apt update", cache_time=24 * 3600)
         apt.upgrade(name="upgrade apt packages", auto_remove=True)
 

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -1,3 +1,4 @@
+import io
 import urllib.request
 
 from chatmaild.config import Config
@@ -18,18 +19,12 @@ DOVECOT_ARCHIVE_VERSION = "2.3.21+dfsg1-3"
 DOVECOT_PACKAGE_VERSION = f"1:{DOVECOT_ARCHIVE_VERSION}"
 
 DOVECOT_SHA256 = {
-    ("amd64", "bookworm", "core"): "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d",
-    ("arm64", "bookworm", "core"): "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9",
-    ("amd64", "bookworm", "imapd"): "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86",
-    ("arm64", "bookworm", "imapd"): "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f",
-    ("amd64", "bookworm", "lmtpd"): "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab",
-    ("arm64", "bookworm", "lmtpd"): "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f",
-    ("amd64", "trixie", "core"): "406d3781ed81e0913c472077dcf62cb1106e3855983efa6e44ddf43b4b0c9be1",
-    ("arm64", "trixie", "core"): "c75b0d9df11a77d07ebd8522920380c167fa47330ddefebe10575d99d0ecdf7f",
-    ("amd64", "trixie", "imapd"): "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86",
-    ("arm64", "trixie", "imapd"): "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f",
-    ("amd64", "trixie", "lmtpd"): "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab",
-    ("arm64", "trixie", "lmtpd"): "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f",
+    ("core", "amd64"): "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d",
+    ("core", "arm64"): "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9",
+    ("imapd", "amd64"): "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86",
+    ("imapd", "arm64"): "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f",
+    ("lmtpd", "amd64"): "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab",
+    ("lmtpd", "arm64"): "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f",
 }
 
 
@@ -43,32 +38,34 @@ class DovecotDeployer(Deployer):
 
     def install(self):
         arch = host.get_fact(Arch)
-        codename = (host.get_fact(Command, "grep '^VERSION_CODENAME=' /etc/os-release | cut -d= -f2") or "").strip()
-        if codename not in {key[1] for key in DOVECOT_SHA256}:
-            raise ValueError(f"Unsupported Debian codename: {codename!r}")
         with blocked_service_startup():
             debs = []
             for pkg in ("core", "imapd", "lmtpd"):
-                deb, changed = _download_dovecot_package(pkg, arch, codename)
+                deb, changed = _download_dovecot_package(pkg, arch)
                 self.need_restart |= changed
                 if deb:
                     debs.append(deb)
             if debs:
                 deb_list = " ".join(debs)
-                # apt-get install with local .deb paths resolves depends
-                # against the configured repos (e.g. pulls libwrap0),
-                # The pin file written earlier by ChatmailDeployer prevents apt
-                # from installing a 'wrong' version
+                # First dpkg may fail on missing dependencies (stderr suppressed);
+                # apt-get --fix-broken pulls them in, then dpkg retries cleanly.
                 server.shell(
                     name="Install dovecot packages",
                     commands=[
-                        "DEBIAN_FRONTEND=noninteractive apt-get install -y "
-                        '-o Dpkg::Options::="--force-confdef" '
-                        '-o Dpkg::Options::="--force-confold" '
-                        f"--allow-downgrades {deb_list}",
+                        f"dpkg --force-confdef --force-confold -i {deb_list} 2> /dev/null || true",
+                        "DEBIAN_FRONTEND=noninteractive apt-get -y --fix-broken install",
+                        f"dpkg --force-confdef --force-confold -i {deb_list}",
                     ],
                 )
                 self.need_restart = True
+        self.put_file(
+            src=io.StringIO(
+                "Package: dovecot-*\n"
+                "Pin: version *\n"
+                "Pin-Priority: -1\n"
+            ),
+            dest="/etc/apt/preferences.d/pin-dovecot",
+        )
 
     def configure(self):
         configure_remote_units(self, self.config.mail_domain_bare, self.units)
@@ -81,7 +78,7 @@ class DovecotDeployer(Deployer):
         if not self.disable_mail and not self.need_restart:
             stale = host.get_fact(
                 Command,
-                "pid=$(systemctl show -p MainPID --value dovecot.service 2>/dev/null);"
+                'pid=$(systemctl show -p MainPID --value dovecot.service 2>/dev/null);'
                 ' [ "${pid:-0}" != "0" ] && readlink "/proc/$pid/exe" 2>/dev/null | grep -q "(deleted)"'
                 " && echo STALE || true",
             )
@@ -105,13 +102,13 @@ def _pick_url(primary, fallback):
         return fallback
 
 
-def _download_dovecot_package(package: str, arch: str, codename: str) -> tuple[str | None, bool]:
+def _download_dovecot_package(package: str, arch: str) -> tuple[str | None, bool]:
     """Download a dovecot .deb if needed, return (path, changed)."""
     arch = "amd64" if arch == "x86_64" else arch
     arch = "arm64" if arch == "aarch64" else arch
 
     pkg_name = f"dovecot-{package}"
-    sha256 = DOVECOT_SHA256.get((arch, codename, package))
+    sha256 = DOVECOT_SHA256.get((package, arch))
     if sha256 is None:
         op = apt.packages(packages=[pkg_name])
         return None, bool(getattr(op, "changed", False))
@@ -122,10 +119,8 @@ def _download_dovecot_package(package: str, arch: str, codename: str) -> tuple[s
 
     url_version = DOVECOT_ARCHIVE_VERSION.replace("+", "%2B")
     deb_base = f"{pkg_name}_{url_version}_{arch}.deb"
-    primary_url = f"https://download.delta.chat/dovecot/{codename}/{url_version}/{deb_base}"
-    upstream_version = DOVECOT_ARCHIVE_VERSION.rsplit("-", 1)[0].replace("+", "%2B")
-    fallback_deb = f"{pkg_name}_{url_version}_{arch}_{codename}.deb"
-    fallback_url = f"https://github.com/chatmail/dovecot/releases/download/upstream%2F{upstream_version}/{fallback_deb}"
+    primary_url = f"https://download.delta.chat/dovecot/{deb_base}"
+    fallback_url = f"https://github.com/chatmail/dovecot/releases/download/upstream%2F{url_version}/{deb_base}"
     url = _pick_url(primary_url, fallback_url)
     deb_filename = f"/root/{deb_base}"
 
@@ -139,7 +134,6 @@ def _download_dovecot_package(package: str, arch: str, codename: str) -> tuple[s
 
     return deb_filename, True
 
-
 def _configure_dovecot(deployer, config: Config, debug: bool = False):
     """Configures Dovecot IMAP server."""
     deployer.put_template(
@@ -150,7 +144,9 @@ def _configure_dovecot(deployer, config: Config, debug: bool = False):
         disable_ipv6=config.disable_ipv6,
     )
     deployer.put_file("dovecot/auth.conf", "/etc/dovecot/auth.conf")
-    deployer.put_file("dovecot/push_notification.lua", "/etc/dovecot/push_notification.lua")
+    deployer.put_file(
+        "dovecot/push_notification.lua", "/etc/dovecot/push_notification.lua"
+    )
 
     # as per https://doc.dovecot.org/2.3/configuration_manual/os/
     # it is recommended to set the following inotify limits

cmdeploy/src/cmdeploy/postfix/lmtp_header_cleanup

@@ -1,3 +1,23 @@
-/^DKIM-Signature:/            IGNORE
-/^Authentication-Results:/            IGNORE
-/^Received:/            IGNORE
+# List of headers for incoming messages 
+# that must be retained for functionality and compatibility reasons 
+/^From:/            DUNNO
+/^Message-Id:/      DUNNO
+/^Chat-/            DUNNO
+/^Content-Type:/    DUNNO
+
+# For receiving clear-text messages (still supported in May 2026)
+/^Subject:/         DUNNO
+/^Date:/            DUNNO
+/^To:/              DUNNO
+/^CC:/              DUNNO
+/^References:/      DUNNO
+/^In-Reply-To:/     DUNNO
+
+# Senders might support Autocrypt 1 but not RFC9788 (Header Protection) 
+/^Autocrypt:/       DUNNO
+
+# SecureJoin V2 protocol headers (for backward compatibility) 
+/^Secure-Join/      DUNNO
+
+# Ignore all other headers
+/.*/                IGNORE

cmdeploy/src/cmdeploy/tests/plugin.py

@@ -10,11 +10,13 @@ from pathlib import Path
 
 import pytest
 from chatmaild.config import is_valid_ipv4, read_config
+from domain_validator import DomainValidator
 
 
 def format_mail_domain(raw_domain: str) -> str:
     if is_valid_ipv4(raw_domain):
         return f"[{raw_domain}]"
+    DomainValidator().validate_domain_re(raw_domain)
     return raw_domain
 
 

cmdeploy/src/cmdeploy/tests/test_dovecot_deployer.py

@@ -3,7 +3,6 @@ from types import SimpleNamespace
 
 import pytest
 from pyinfra.facts.deb import DebPackages
-from pyinfra.facts.server import Command
 
 from cmdeploy.dovecot import deployer as dovecot_deployer
 
@@ -20,7 +19,7 @@ def make_host(*fact_pairs):
     """
     facts = dict(fact_pairs)
 
-    def get_fact(cls, *args):
+    def get_fact(cls):
         if cls not in facts:
             registered = ", ".join(c.__name__ for c in facts)
             raise LookupError(
@@ -83,9 +82,7 @@ def test_download_dovecot_package_skips_epoch_matched_install(monkeypatch):
         lambda **kwargs: downloads.append(kwargs),
     )
 
-    deb, changed = dovecot_deployer._download_dovecot_package(
-        "core", "amd64", codename="bookworm"
-    )
+    deb, changed = dovecot_deployer._download_dovecot_package("core", "amd64")
 
     assert deb is None, f"expected no deb path when version matches, got {deb!r}"
     assert changed is False, "should not flag changed when version already installed"
@@ -112,9 +109,7 @@ def test_download_dovecot_package_uses_archive_version_for_url_and_filename(
         lambda **kwargs: downloads.append(kwargs),
     )
 
-    deb, changed = dovecot_deployer._download_dovecot_package(
-        "core", "amd64", codename="bookworm"
-    )
+    deb, changed = dovecot_deployer._download_dovecot_package("core", "amd64")
 
     archive_version = dovecot_deployer.DOVECOT_ARCHIVE_VERSION.replace("+", "%2B")
     expected_deb = f"/root/dovecot-core_{archive_version}_amd64.deb"
@@ -144,7 +139,6 @@ def test_install_skips_dpkg_path_when_epoch_matched_packages_present(
                 },
             ),
             (dovecot_deployer.Arch, "x86_64"),
-            (Command, "bookworm"),
         ),
     )
     downloads = []
@@ -166,13 +160,11 @@ def test_install_skips_dpkg_path_when_epoch_matched_packages_present(
 def test_install_unsupported_arch_falls_back_to_apt(
     deployer, patch_blocked, mock_files_put, track_shell, monkeypatch
 ):
+    # For unsupported architectures, all fact lookups return the arch string.
     monkeypatch.setattr(
         dovecot_deployer,
         "host",
-        make_host(
-            (dovecot_deployer.Arch, "riscv64"),
-            (Command, "bookworm"),
-        ),
+        SimpleNamespace(get_fact=lambda cls: "riscv64"),
     )
     apt_calls = []
 
@@ -206,7 +198,6 @@ def test_install_runs_dpkg_when_packages_need_download(
         make_host(
             (dovecot_deployer.DebPackages, {}),
             (dovecot_deployer.Arch, "x86_64"),
-            (Command, "bookworm"),
         ),
     )
     monkeypatch.setattr(
@@ -226,12 +217,10 @@ def test_install_runs_dpkg_when_packages_need_download(
         f"expected one server.shell() call for dpkg install, got {len(track_shell)}"
     )
     cmds = track_shell[0]["commands"]
-    assert len(cmds) == 1, f"expected single apt-get install command, got: {cmds}"
-    assert "apt-get install -y" in cmds[0]
-    assert '-o Dpkg::Options::="--force-confdef"' in cmds[0]
-    assert '-o Dpkg::Options::="--force-confold"' in cmds[0]
-    assert "--allow-downgrades" in cmds[0]
-    assert ".deb" in cmds[0]
+    assert len(cmds) == 3, f"expected 3 dpkg/apt commands, got: {cmds}"
+    assert cmds[0].startswith("dpkg --force-confdef --force-confold -i ")
+    assert "apt-get -y --fix-broken install" in cmds[1]
+    assert cmds[2].startswith("dpkg --force-confdef --force-confold -i ")
     assert deployer.need_restart is True, (
         "need_restart should be True after dpkg install"
     )
@@ -246,19 +235,3 @@ def test_pick_url_falls_back_on_primary_error(monkeypatch):
     assert result == "http://fallback", (
         f"should fall back when primary fails, got {result!r}"
     )
-
-
-def test_install_fails_on_unsupported_debian_version(
-    deployer, patch_blocked, monkeypatch
-):
-    monkeypatch.setattr(
-        dovecot_deployer,
-        "host",
-        make_host(
-            (dovecot_deployer.Arch, "x86_64"),
-            (Command, "sid"),
-        ),
-    )
-
-    with pytest.raises(ValueError, match="Unsupported Debian codename"):
-        deployer.install()