feat: randomize SMTP + IMAP ports

.github/workflows/docker-dispatch.yaml

@@ -8,7 +8,7 @@ name: Trigger Docker build
 
 on:
   push:
-    branches: [main, j4n/dovecot-multidist]
+    branches: [main]
     tags: ['[0-9]+.[0-9]+.[0-9]+']
   workflow_dispatch:
 

chatmaild/pyproject.toml

@@ -9,7 +9,7 @@ dependencies = [
   "iniconfig",
   "filelock",
   "requests",
-  "crypt-r >= 3.13.1 ; python_version >= '3.13'",
+  "crypt-r >= 3.13.1 ; python_version >= '3.11'",
 ]
 
 [tool.setuptools]

chatmaild/src/chatmaild/config.py

@@ -1,5 +1,6 @@
 import ipaddress
 from pathlib import Path
+from random import randint
 
 import iniconfig
 
@@ -41,6 +42,11 @@ class Config:
         self.username_max_length = int(params.pop("username_max_length", 9))
         self.password_min_length = int(params.pop("password_min_length", 9))
         self.www_folder = params.pop("www_folder", "")
+
+        self.imap_port = int(params.pop("imap_port", 143))
+        self.imaps_port = int(params.pop("imaps_port", 993))
+        self.smtp_port = int(params.pop("smtp_port", 587))
+        self.smtps_port = int(params.pop("smtps_port", 465))
         self.filtermail_smtp_port = int(params.pop("filtermail_smtp_port", "10080"))
         self.filtermail_smtp_port_incoming = int(
             params.pop("filtermail_smtp_port_incoming", "10081")
@@ -138,8 +144,15 @@ def parse_size_mb(limit):
 
 def write_initial_config(inipath, mail_domain, overrides):
     """Write out default config file, using the specified config value overrides."""
-    content = get_default_config_content(mail_domain, **overrides)
+    content = get_default_config_content(mail_domain, **overrides).splitlines()
-    inipath.write_text(content)
+    used_ports = [25, 53, 80, 143, 402, 443, 465, 587, 993, 3340, 3903, 3904, 8443, 10080, 10081, 10082, 10083, 10025, 10026]
+    for config_key in ["smtp_port", "imap_port", "smtps_port", "imaps_port"]:
+        value = randint(1, 65536)
+        while value in used_ports:
+            value = randint(65535)
+        used_ports.append(value)
+        content.append(f"{config_key} = {value}")
+    inipath.write_text("\n".join(content))
 
 
 def get_default_config_content(mail_domain, **overrides):

cmdeploy/src/cmdeploy/basedeploy.py

@@ -166,7 +166,7 @@ class Deployer:
             return self.put_template(src, dest, **kwargs)
         return self.put_file(src, dest)
 
-    def put_file(self, src, dest, mode="644", **kwargs):
+    def put_file(self, src, dest, mode="644"):
         if isinstance(src, str):
             src = get_resource(src)
         res = files.put(
@@ -176,7 +176,6 @@ class Deployer:
             user="root",
             group="root",
             mode=mode,
-            **kwargs,
         )
 
         return self._update_restart_signals(dest, res)

cmdeploy/src/cmdeploy/deployers.py

@@ -164,7 +164,6 @@ class UnboundDeployer(Deployer):
         self.put_file(
             src=BytesIO(b"nameserver 127.0.0.1\nnameserver 9.9.9.9\n"),
             dest="/etc/resolv.conf",
-            force=True,
         )
         server.shell(
             name="Generate root keys for validating DNSSEC",
@@ -390,12 +389,6 @@ class ChatmailDeployer(Deployer):
             src=BytesIO(b'APT::Install-Recommends "false";\n'),
             dest="/etc/apt/apt.conf.d/00InstallRecommends",
         )
-        # Pin dovecot-* to priority -1 before any apt operation, apt should
-        # never manage dovecot as our version might be lower than the distro's.
-        self.put_file(
-            src=StringIO("Package: dovecot-*\nPin: version *\nPin-Priority: -1\n"),
-            dest="/etc/apt/preferences.d/pin-dovecot",
-        )
         apt.update(name="apt update", cache_time=24 * 3600)
         apt.upgrade(name="upgrade apt packages", auto_remove=True)
 
@@ -502,15 +495,15 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
         if config.tls_cert_mode == "acme":
             port_services.append(("acmetool", 402))
         port_services += [
-            (["imap-login", "dovecot"], 143),
+            (["imap-login", "dovecot"], config.imap_port),
             # acmetool previously listened on port 80,
             # so don't complain during upgrade that moved it to port 402
             # and gave the port to nginx.
             (["acmetool", "nginx"], 80),
             ("nginx", 443),
-            (["master", "smtpd"], 465),
+            (["master", "smtpd"], config.smtp_port),
-            (["master", "smtpd"], 587),
+            (["master", "smtpd"], config.smtps_port),
-            (["imap-login", "dovecot"], 993),
+            (["imap-login", "dovecot"], config.imaps_port),
             ("iroh-relay", 3340),
             ("mtail", 3903),
             ("stats", 3904),

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -1,3 +1,4 @@
+import io
 import urllib.request
 
 from chatmaild.config import Config
@@ -18,18 +19,12 @@ DOVECOT_ARCHIVE_VERSION = "2.3.21+dfsg1-3"
 DOVECOT_PACKAGE_VERSION = f"1:{DOVECOT_ARCHIVE_VERSION}"
 
 DOVECOT_SHA256 = {
-    ("amd64", "bookworm", "core"): "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d",
+    ("core", "amd64"): "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d",
-    ("arm64", "bookworm", "core"): "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9",
+    ("core", "arm64"): "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9",
-    ("amd64", "bookworm", "imapd"): "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86",
+    ("imapd", "amd64"): "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86",
-    ("arm64", "bookworm", "imapd"): "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f",
+    ("imapd", "arm64"): "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f",
-    ("amd64", "bookworm", "lmtpd"): "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab",
+    ("lmtpd", "amd64"): "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab",
-    ("arm64", "bookworm", "lmtpd"): "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f",
+    ("lmtpd", "arm64"): "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f",
-    ("amd64", "trixie", "core"): "406d3781ed81e0913c472077dcf62cb1106e3855983efa6e44ddf43b4b0c9be1",
-    ("arm64", "trixie", "core"): "c75b0d9df11a77d07ebd8522920380c167fa47330ddefebe10575d99d0ecdf7f",
-    ("amd64", "trixie", "imapd"): "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86",
-    ("arm64", "trixie", "imapd"): "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f",
-    ("amd64", "trixie", "lmtpd"): "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab",
-    ("arm64", "trixie", "lmtpd"): "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f",
 }
 
 
@@ -43,32 +38,34 @@ class DovecotDeployer(Deployer):
 
     def install(self):
         arch = host.get_fact(Arch)
-        codename = (host.get_fact(Command, "grep '^VERSION_CODENAME=' /etc/os-release | cut -d= -f2") or "").strip()
-        if codename not in {key[1] for key in DOVECOT_SHA256}:
-            raise ValueError(f"Unsupported Debian codename: {codename!r}")
         with blocked_service_startup():
             debs = []
             for pkg in ("core", "imapd", "lmtpd"):
-                deb, changed = _download_dovecot_package(pkg, arch, codename)
+                deb, changed = _download_dovecot_package(pkg, arch)
                 self.need_restart |= changed
                 if deb:
                     debs.append(deb)
             if debs:
                 deb_list = " ".join(debs)
-                # apt-get install with local .deb paths resolves depends
+                # First dpkg may fail on missing dependencies (stderr suppressed);
-                # against the configured repos (e.g. pulls libwrap0),
+                # apt-get --fix-broken pulls them in, then dpkg retries cleanly.
-                # The pin file written earlier by ChatmailDeployer prevents apt
-                # from installing a 'wrong' version
                 server.shell(
                     name="Install dovecot packages",
                     commands=[
-                        "DEBIAN_FRONTEND=noninteractive apt-get install -y "
+                        f"dpkg --force-confdef --force-confold -i {deb_list} 2> /dev/null || true",
-                        '-o Dpkg::Options::="--force-confdef" '
+                        "DEBIAN_FRONTEND=noninteractive apt-get -y --fix-broken install",
-                        '-o Dpkg::Options::="--force-confold" '
+                        f"dpkg --force-confdef --force-confold -i {deb_list}",
-                        f"--allow-downgrades {deb_list}",
                     ],
                 )
                 self.need_restart = True
+        self.put_file(
+            src=io.StringIO(
+                "Package: dovecot-*\n"
+                "Pin: version *\n"
+                "Pin-Priority: -1\n"
+            ),
+            dest="/etc/apt/preferences.d/pin-dovecot",
+        )
 
     def configure(self):
         configure_remote_units(self, self.config.mail_domain_bare, self.units)
@@ -81,7 +78,7 @@ class DovecotDeployer(Deployer):
         if not self.disable_mail and not self.need_restart:
             stale = host.get_fact(
                 Command,
-                "pid=$(systemctl show -p MainPID --value dovecot.service 2>/dev/null);"
+                'pid=$(systemctl show -p MainPID --value dovecot.service 2>/dev/null);'
                 ' [ "${pid:-0}" != "0" ] && readlink "/proc/$pid/exe" 2>/dev/null | grep -q "(deleted)"'
                 " && echo STALE || true",
             )
@@ -105,13 +102,13 @@ def _pick_url(primary, fallback):
         return fallback
 
 
-def _download_dovecot_package(package: str, arch: str, codename: str) -> tuple[str | None, bool]:
+def _download_dovecot_package(package: str, arch: str) -> tuple[str | None, bool]:
     """Download a dovecot .deb if needed, return (path, changed)."""
     arch = "amd64" if arch == "x86_64" else arch
     arch = "arm64" if arch == "aarch64" else arch
 
     pkg_name = f"dovecot-{package}"
-    sha256 = DOVECOT_SHA256.get((arch, codename, package))
+    sha256 = DOVECOT_SHA256.get((package, arch))
     if sha256 is None:
         op = apt.packages(packages=[pkg_name])
         return None, bool(getattr(op, "changed", False))
@@ -122,10 +119,8 @@ def _download_dovecot_package(package: str, arch: str, codename: str) -> tuple[s
 
     url_version = DOVECOT_ARCHIVE_VERSION.replace("+", "%2B")
     deb_base = f"{pkg_name}_{url_version}_{arch}.deb"
-    primary_url = f"https://download.delta.chat/dovecot/{codename}/{url_version}/{deb_base}"
+    primary_url = f"https://download.delta.chat/dovecot/{deb_base}"
-    upstream_version = DOVECOT_ARCHIVE_VERSION.rsplit("-", 1)[0].replace("+", "%2B")
+    fallback_url = f"https://github.com/chatmail/dovecot/releases/download/upstream%2F{url_version}/{deb_base}"
-    fallback_deb = f"{pkg_name}_{url_version}_{arch}_{codename}.deb"
-    fallback_url = f"https://github.com/chatmail/dovecot/releases/download/upstream%2F{upstream_version}/{fallback_deb}"
     url = _pick_url(primary_url, fallback_url)
     deb_filename = f"/root/{deb_base}"
 
@@ -139,7 +134,6 @@ def _download_dovecot_package(package: str, arch: str, codename: str) -> tuple[s
 
     return deb_filename, True
 
-
 def _configure_dovecot(deployer, config: Config, debug: bool = False):
     """Configures Dovecot IMAP server."""
     deployer.put_template(
@@ -150,7 +144,9 @@ def _configure_dovecot(deployer, config: Config, debug: bool = False):
         disable_ipv6=config.disable_ipv6,
     )
     deployer.put_file("dovecot/auth.conf", "/etc/dovecot/auth.conf")
-    deployer.put_file("dovecot/push_notification.lua", "/etc/dovecot/push_notification.lua")
+    deployer.put_file(
+        "dovecot/push_notification.lua", "/etc/dovecot/push_notification.lua"
+    )
 
     # as per https://doc.dovecot.org/2.3/configuration_manual/os/
     # it is recommended to set the following inotify limits

cmdeploy/src/cmdeploy/nginx/autoconfig.xml.j2

@@ -7,14 +7,14 @@
     <displayShortName>{{ config.mail_domain }}</displayShortName>
     <incomingServer type="imap">
       <hostname>{{ config.mail_domain }}</hostname>
-      <port>993</port>
+      <port>{{ config.imaps_port }}</port>
       <socketType>SSL</socketType>
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
     </incomingServer>
     <incomingServer type="imap">
       <hostname>{{ config.mail_domain }}</hostname>
-      <port>143</port>
+      <port>{{ config.imap_port }}</port>
       <socketType>STARTTLS</socketType>
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
@@ -28,14 +28,14 @@
     </incomingServer>
     <outgoingServer type="smtp">
       <hostname>{{ config.mail_domain }}</hostname>
-      <port>465</port>
+      <port>{{ config.smtps_port }}</port>
       <socketType>SSL</socketType>
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
     </outgoingServer>
     <outgoingServer type="smtp">
       <hostname>{{ config.mail_domain }}</hostname>
-      <port>587</port>
+      <port>{{ config.smtp_port }}</port>
       <socketType>STARTTLS</socketType>
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -31,6 +31,26 @@ stream {
             ~\bimap\b 127.0.0.1:993;
         }
 
+        server {
+                listen {{ config.smtp_port }};
+                proxy_pass 127.0.0.1:587;
+        }
+
+        server {
+                listen {{ config.imap_port }};
+                proxy_pass 127.0.0.1:143;
+        }
+
+        server {
+                listen {{ config.smtps_port }};
+                proxy_pass 127.0.0.1:465;
+        }
+
+        server {
+                listen {{ config.imaps_port }};
+                proxy_pass 127.0.0.1:993;
+        }
+
         server {
                 listen 443;
                 {% if not disable_ipv6 %}

cmdeploy/src/cmdeploy/tests/test_dovecot_deployer.py

@@ -3,7 +3,6 @@ from types import SimpleNamespace
 
 import pytest
 from pyinfra.facts.deb import DebPackages
-from pyinfra.facts.server import Command
 
 from cmdeploy.dovecot import deployer as dovecot_deployer
 
@@ -20,7 +19,7 @@ def make_host(*fact_pairs):
     """
     facts = dict(fact_pairs)
 
-    def get_fact(cls, *args):
+    def get_fact(cls):
         if cls not in facts:
             registered = ", ".join(c.__name__ for c in facts)
             raise LookupError(
@@ -83,9 +82,7 @@ def test_download_dovecot_package_skips_epoch_matched_install(monkeypatch):
         lambda **kwargs: downloads.append(kwargs),
     )
 
-    deb, changed = dovecot_deployer._download_dovecot_package(
+    deb, changed = dovecot_deployer._download_dovecot_package("core", "amd64")
-        "core", "amd64", codename="bookworm"
-    )
 
     assert deb is None, f"expected no deb path when version matches, got {deb!r}"
     assert changed is False, "should not flag changed when version already installed"
@@ -112,9 +109,7 @@ def test_download_dovecot_package_uses_archive_version_for_url_and_filename(
         lambda **kwargs: downloads.append(kwargs),
     )
 
-    deb, changed = dovecot_deployer._download_dovecot_package(
+    deb, changed = dovecot_deployer._download_dovecot_package("core", "amd64")
-        "core", "amd64", codename="bookworm"
-    )
 
     archive_version = dovecot_deployer.DOVECOT_ARCHIVE_VERSION.replace("+", "%2B")
     expected_deb = f"/root/dovecot-core_{archive_version}_amd64.deb"
@@ -144,7 +139,6 @@ def test_install_skips_dpkg_path_when_epoch_matched_packages_present(
                 },
             ),
             (dovecot_deployer.Arch, "x86_64"),
-            (Command, "bookworm"),
         ),
     )
     downloads = []
@@ -166,13 +160,11 @@ def test_install_skips_dpkg_path_when_epoch_matched_packages_present(
 def test_install_unsupported_arch_falls_back_to_apt(
     deployer, patch_blocked, mock_files_put, track_shell, monkeypatch
 ):
+    # For unsupported architectures, all fact lookups return the arch string.
     monkeypatch.setattr(
         dovecot_deployer,
         "host",
-        make_host(
+        SimpleNamespace(get_fact=lambda cls: "riscv64"),
-            (dovecot_deployer.Arch, "riscv64"),
-            (Command, "bookworm"),
-        ),
     )
     apt_calls = []
 
@@ -206,7 +198,6 @@ def test_install_runs_dpkg_when_packages_need_download(
         make_host(
             (dovecot_deployer.DebPackages, {}),
             (dovecot_deployer.Arch, "x86_64"),
-            (Command, "bookworm"),
         ),
     )
     monkeypatch.setattr(
@@ -226,12 +217,10 @@ def test_install_runs_dpkg_when_packages_need_download(
         f"expected one server.shell() call for dpkg install, got {len(track_shell)}"
     )
     cmds = track_shell[0]["commands"]
-    assert len(cmds) == 1, f"expected single apt-get install command, got: {cmds}"
+    assert len(cmds) == 3, f"expected 3 dpkg/apt commands, got: {cmds}"
-    assert "apt-get install -y" in cmds[0]
+    assert cmds[0].startswith("dpkg --force-confdef --force-confold -i ")
-    assert '-o Dpkg::Options::="--force-confdef"' in cmds[0]
+    assert "apt-get -y --fix-broken install" in cmds[1]
-    assert '-o Dpkg::Options::="--force-confold"' in cmds[0]
+    assert cmds[2].startswith("dpkg --force-confdef --force-confold -i ")
-    assert "--allow-downgrades" in cmds[0]
-    assert ".deb" in cmds[0]
     assert deployer.need_restart is True, (
         "need_restart should be True after dpkg install"
     )
@@ -246,19 +235,3 @@ def test_pick_url_falls_back_on_primary_error(monkeypatch):
     assert result == "http://fallback", (
         f"should fall back when primary fails, got {result!r}"
     )
-
-
-def test_install_fails_on_unsupported_debian_version(
-    deployer, patch_blocked, monkeypatch
-):
-    monkeypatch.setattr(
-        dovecot_deployer,
-        "host",
-        make_host(
-            (dovecot_deployer.Arch, "x86_64"),
-            (Command, "sid"),
-        ),
-    )
-
-    with pytest.raises(ValueError, match="Unsupported Debian codename"):
-        deployer.install()