fix(cmdeploy): deployer fixes for container compatibility

- UnboundDeployer: use blocked_service_startup() instead of manual policy-rc.d
- ChatmailDeployer: accept full Config object, ensure mailboxes_dir exists
- DovecotDeployer: replace CHATMAIL_NOSYSCTL env var with systemd-detect-virt -c
- SelfSignedTlsDeployer: add basicConstraints=critical,CA:FALSE
- WebsiteDeployer: use stable files.sync instead of experimental files.rsync
- GithashDeployer: use util.get_version_string()
- TurnDeployer: update to chatmail-turn v0.4

cmdeploy/src/cmdeploy/deployers.py

@@ -17,13 +17,12 @@ from pyinfra.facts.files import Sha256File
 from pyinfra.facts.systemd import SystemdEnabled
 from pyinfra.operations import apt, files, pip, server, systemd
 
-from cmdeploy.cmdeploy import Out
-
 from .acmetool import AcmetoolDeployer
 from .basedeploy import (
     Deployer,
     Deployment,
     activate_remote_units,
+    blocked_service_startup,
     configure_remote_units,
     get_resource,
     has_systemd,
@@ -36,6 +35,7 @@ from .nginx.deployer import NginxDeployer
 from .opendkim.deployer import OpendkimDeployer
 from .postfix.deployer import PostfixDeployer
 from .selfsigned.deployer import SelfSignedTlsDeployer
+from .util import Out, get_version_string
 from .www import build_webpages, find_merge_conflict, get_paths
 
 
@@ -149,33 +149,16 @@ class UnboundDeployer(Deployer):
         self.need_restart = False
 
     def install(self):
-        # Run local DNS resolver `unbound`.
+        # Run local DNS resolver `unbound`. `resolvconf` takes care of
-        # `resolvconf` takes care of setting up /etc/resolv.conf
+        # setting up /etc/resolv.conf to use 127.0.0.1 as the resolver.
-        # to use 127.0.0.1 as the resolver.
 
-        #
+        # On an IPv4-only system, if unbound is started but not configured,
-        # On an IPv4-only system, if unbound is started but not
+        # it causes subsequent steps to fail to resolve hosts.
-        # configured, it causes subsequent steps to fail to resolve hosts.
+        with blocked_service_startup():
-        # Here, we use policy-rc.d to prevent unbound from starting up
+            apt.packages(
-        # on initial install.  Later, we will configure it and start it.
+                name="Install unbound",
-        #
+                packages=["unbound", "unbound-anchor", "dnsutils"],
-        # For documentation about policy-rc.d, see:
+            )
-        # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
-        #
-        files.put(
-            src=get_resource("policy-rc.d"),
-            dest="/usr/sbin/policy-rc.d",
-            user="root",
-            group="root",
-            mode="755",
-        )
-
-        apt.packages(
-            name="Install unbound",
-            packages=["unbound", "unbound-anchor", "dnsutils"],
-        )
-
-        files.file("/usr/sbin/policy-rc.d", present=False)
 
     def configure(self):
         server.shell(
@@ -271,8 +254,14 @@ class WebsiteDeployer(Deployer):
                     logger.warning("Web page build failed, skipping website deployment")
                     return
             # if it is not a hugo page, upload it as is
-            files.rsync(
+            # pyinfra files.rsync (experimental) causes problems with ssh-config configuration
-                f"{www_path}/", "/var/www/html", flags=["-avz", "--chown=www-data"]
+            # the stable files.sync should do
+            files.sync(
+                src=str(www_path),
+                dest="/var/www/html",
+                user="www-data",
+                group="www-data",
+                delete=True,
             )
 
 
@@ -336,12 +325,12 @@ class TurnDeployer(Deployer):
     def install(self):
         (url, sha256sum) = {
             "x86_64": (
-                "https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-x86_64-linux",
+                "https://github.com/chatmail/chatmail-turn/releases/download/v0.4/chatmail-turn-x86_64-linux",
-                "841e527c15fdc2940b0469e206188ea8f0af48533be12ecb8098520f813d41e4",
+                "1ec1f5c50122165e858a5a91bcba9037a28aa8cb8b64b8db570aa457c6141a8a",
             ),
             "aarch64": (
-                "https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-aarch64-linux",
+                "https://github.com/chatmail/chatmail-turn/releases/download/v0.4/chatmail-turn-aarch64-linux",
-                "a5fc2d06d937b56a34e098d2cd72a82d3e89967518d159bf246dc69b65e81b42",
+                "0fb3e792419494e21ecad536464929dba706bb2c88884ed8f1788141d26fc756",
             ),
         }[host.get_fact(facts.server.Arch)]
 
@@ -474,8 +463,9 @@ class ChatmailDeployer(Deployer):
         ("iroh", None, None),
     ]
 
-    def __init__(self, mail_domain):
+    def __init__(self, config):
-        self.mail_domain = mail_domain
+        self.config = config
+        self.mail_domain = config.mail_domain
 
     def install(self):
         files.put(
@@ -500,6 +490,17 @@ class ChatmailDeployer(Deployer):
         )
 
     def configure(self):
+        # Ensure the per-domain mailbox directory exists before
+        # chatmail-metadata starts (it crashes without it).
+        files.directory(
+            name="Ensure vmail mailbox directory exists",
+            path=f"/home/vmail/mail/{self.mail_domain}",
+            user="vmail",
+            group="vmail",
+            mode="700",
+            present=True,
+        )
+
         # This file is used by auth proxy.
         # https://wiki.debian.org/EtcMailName
         server.shell(
@@ -509,6 +510,15 @@ class ChatmailDeployer(Deployer):
             ],
         )
 
+        files.directory(
+            name=f"Ensure mailboxes directory {self.config.mailboxes_dir} exists",
+            path=str(self.config.mailboxes_dir),
+            user="vmail",
+            group="vmail",
+            mode="700",
+            present=True,
+        )
+
 
 class FcgiwrapDeployer(Deployer):
     def install(self):
@@ -528,17 +538,9 @@ class FcgiwrapDeployer(Deployer):
 
 class GithashDeployer(Deployer):
     def activate(self):
-        try:
-            git_hash = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode()
-        except Exception:
-            git_hash = "unknown\n"
-        try:
-            git_diff = subprocess.check_output(["git", "diff"]).decode()
-        except Exception:
-            git_diff = ""
         files.put(
             name="Upload chatmail relay git commit hash",
-            src=StringIO(git_hash + git_diff),
+            src=StringIO(get_version_string()),
             dest="/etc/chatmail-version",
             mode="700",
         )
@@ -582,11 +584,17 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
         )
 
     # Check if mtail_address interface is available (if configured)
-    if config.mtail_address and config.mtail_address not in ('127.0.0.1', '::1', 'localhost'):
+    if config.mtail_address and config.mtail_address not in (
+        "127.0.0.1",
+        "::1",
+        "localhost",
+    ):
         ipv4_addrs = host.get_fact(hardware.Ipv4Addrs)
         all_addresses = [addr for addrs in ipv4_addrs.values() for addr in addrs]
         if config.mtail_address not in all_addresses:
-            Out().red(f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n")
+            Out().red(
+                f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n"
+            )
             exit(1)
 
     if not os.environ.get("CHATMAIL_NOPORTCHECK"):
@@ -629,7 +637,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
     tls_deployer = get_tls_deployer(config, mail_domain)
 
     all_deployers = [
-        ChatmailDeployer(mail_domain),
+        ChatmailDeployer(config),
         LegacyRemoveDeployer(),
         FiltermailDeployer(),
         JournaldDeployer(),

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -1,10 +1,9 @@
-import os
 import urllib.request
 
 from chatmaild.config import Config
 from pyinfra import host
 from pyinfra.facts.deb import DebPackages
-from pyinfra.facts.server import Arch, Sysctl
+from pyinfra.facts.server import Arch, Command, Sysctl
 from pyinfra.operations import apt, files, server, systemd
 
 from cmdeploy.basedeploy import (
@@ -153,19 +152,25 @@ def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
 
     # as per https://doc.dovecot.org/2.3/configuration_manual/os/
     # it is recommended to set the following inotify limits
-    if not os.environ.get("CHATMAIL_NOSYSCTL"):
+    can_modify = host.get_fact(Command, "systemd-detect-virt -c || true") == "none"
-        for name in ("max_user_instances", "max_user_watches"):
+    for name in ("max_user_instances", "max_user_watches"):
-            key = f"fs.inotify.{name}"
+        key = f"fs.inotify.{name}"
-            if host.get_fact(Sysctl)[key] > 65535:
+        value = host.get_fact(Sysctl)[key]
-                # Skip updating limits if already sufficient
+        if value > 65534:
-                # (enables running in incus containers where sysctl readonly)
+            continue
-                continue
+        if not can_modify:
-            server.sysctl(
+            print(
-                name=f"Change {key}",
+                "\n!!!! refusing to attempt sysctl setting in shared-kernel containers\n"
-                key=key,
+                f"!!!! dovecot: sysctl {key!r}={value}, should be >65534 for production setups\n"
-                value=65535,
+                "!!!!"
-                persist=True,
             )
+            continue
+        server.sysctl(
+            name=f"Change {key}",
+            key=key,
+            value=65535,
+            persist=True,
+        )
 
     timezone_env = files.line(
         name="Set TZ environment variable",

cmdeploy/src/cmdeploy/selfsigned/deployer.py

@@ -12,13 +12,27 @@ def openssl_selfsigned_args(domain, cert_path, key_path, days=36500):
     ``www.<domain>`` and ``mta-sts.<domain>``.
     """
     return [
-        "openssl", "req", "-x509",
+        "openssl",
-        "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:P-256",
+        "req",
-        "-noenc", "-days", str(days),
+        "-x509",
-        "-keyout", str(key_path),
+        "-newkey",
-        "-out", str(cert_path),
+        "ec",
-        "-subj", f"/CN={domain}",
+        "-pkeyopt",
-        "-addext", "extendedKeyUsage=serverAuth,clientAuth",
+        "ec_paramgen_curve:P-256",
+        "-noenc",
+        "-days",
+        str(days),
+        "-keyout",
+        str(key_path),
+        "-out",
+        str(cert_path),
+        "-subj",
+        f"/CN={domain}",
+        # Mark as end-entity cert so it cannot be used as a CA to sign others.
+        "-addext",
+        "basicConstraints=critical,CA:FALSE",
+        "-addext",
+        "extendedKeyUsage=serverAuth,clientAuth",
         "-addext",
         f"subjectAltName=DNS:{domain},DNS:www.{domain},DNS:mta-sts.{domain}",
     ]
@@ -40,7 +54,9 @@ class SelfSignedTlsDeployer(Deployer):
 
     def configure(self):
         args = openssl_selfsigned_args(
-            self.mail_domain, self.cert_path, self.key_path,
+            self.mail_domain,
+            self.cert_path,
+            self.key_path,
         )
         cmd = shlex.join(args)
         server.shell(