feat: support externally managed TLS via tls_external_cert_and_key option (#860)

Adds a new tls_external_cert_and_key config option for chatmail servers
that manage their own TLS certificates (e.g. via an external ACME client
or a load balancer).

A systemd path unit (tls-cert-reload.path) watches the certificate file
via inotify and automatically reloads dovecot and nginx when it changes.
Postfix reads certs per TLS handshake so needs no reload.

Also extracts openssl_selfsigned_args() so cert generation parameters
are shared between SelfSignedTlsDeployer and the e2e test.

chatmaild/src/chatmaild/config.py

@@ -60,10 +60,23 @@ class Config:
         self.privacy_pdo = params.get("privacy_pdo")
         self.privacy_supervisor = params.get("privacy_supervisor")
 
-        # TLS certificate management: derived from the domain name.
-        # Domains starting with "_" use self-signed certificates
-        # All other domains use ACME.
-        if self.mail_domain.startswith("_"):
+        # TLS certificate management.
+        # If tls_external_cert_and_key is set, use externally managed certs.
+        # Otherwise derived from the domain name:
+        # - Domains starting with "_" use self-signed certificates
+        # - All other domains use ACME.
+        external = params.get("tls_external_cert_and_key", "").strip()
+
+        if external:
+            parts = external.split()
+            if len(parts) != 2:
+                raise ValueError(
+                    "tls_external_cert_and_key must have two space-separated"
+                    " paths: CERT_PATH KEY_PATH"
+                )
+            self.tls_cert_mode = "external"
+            self.tls_cert_path, self.tls_key_path = parts
+        elif self.mail_domain.startswith("_"):
             self.tls_cert_mode = "self"
             self.tls_cert_path = "/etc/ssl/certs/mailserver.pem"
             self.tls_key_path = "/etc/ssl/private/mailserver.key"

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -48,6 +48,13 @@ passthrough_senders =
 # (space-separated, item may start with "@" to whitelist whole recipient domains)
 passthrough_recipients =
 
+# Use externally managed TLS certificates instead of built-in acmetool.
+# Paths refer to files on the deployment server (not the build machine).
+# Both files must already exist before running cmdeploy.
+# Certificate renewal is your responsibility; changed files are
+# picked up automatically by all relay services.
+# tls_external_cert_and_key = /path/to/fullchain.pem /path/to/privkey.pem
+
 # path to www directory - documented here: https://chatmail.at/doc/relay/getting_started.html#custom-web-pages
 #www_folder = www
 

chatmaild/src/chatmaild/tests/test_config.py

@@ -87,3 +87,37 @@ def test_config_tls_self(make_config):
     assert config.tls_cert_mode == "self"
     assert config.tls_cert_path == "/etc/ssl/certs/mailserver.pem"
     assert config.tls_key_path == "/etc/ssl/private/mailserver.key"
+
+
+def test_config_tls_external(make_config):
+    config = make_config(
+        "chat.example.org",
+        {
+            "tls_external_cert_and_key": "/custom/fullchain.pem /custom/privkey.pem",
+        },
+    )
+    assert config.tls_cert_mode == "external"
+    assert config.tls_cert_path == "/custom/fullchain.pem"
+    assert config.tls_key_path == "/custom/privkey.pem"
+
+
+def test_config_tls_external_overrides_underscore(make_config):
+    config = make_config(
+        "_test.example.org",
+        {
+            "tls_external_cert_and_key": "/certs/fullchain.pem /certs/privkey.pem",
+        },
+    )
+    assert config.tls_cert_mode == "external"
+    assert config.tls_cert_path == "/certs/fullchain.pem"
+    assert config.tls_key_path == "/certs/privkey.pem"
+
+
+def test_config_tls_external_bad_format(make_config):
+    with pytest.raises(ValueError, match="two space-separated"):
+        make_config(
+            "chat.example.org",
+            {
+                "tls_external_cert_and_key": "/only/one/path.pem",
+            },
+        )

cmdeploy/src/cmdeploy/deployers.py

@@ -11,8 +11,8 @@ from pathlib import Path
 
 from chatmaild.config import read_config
 from pyinfra import facts, host, logger
-from pyinfra.facts import hardware
 from pyinfra.api import FactBase
+from pyinfra.facts import hardware
 from pyinfra.facts.files import Sha256File
 from pyinfra.facts.systemd import SystemdEnabled
 from pyinfra.operations import apt, files, pip, server, systemd
@@ -20,7 +20,6 @@ from pyinfra.operations import apt, files, pip, server, systemd
 from cmdeploy.cmdeploy import Out
 
 from .acmetool import AcmetoolDeployer
-from .selfsigned.deployer import SelfSignedTlsDeployer
 from .basedeploy import (
     Deployer,
     Deployment,
@@ -30,11 +29,13 @@ from .basedeploy import (
     has_systemd,
 )
 from .dovecot.deployer import DovecotDeployer
+from .external.deployer import ExternalTlsDeployer
 from .filtermail.deployer import FiltermailDeployer
 from .mtail.deployer import MtailDeployer
 from .nginx.deployer import NginxDeployer
 from .opendkim.deployer import OpendkimDeployer
 from .postfix.deployer import PostfixDeployer
+from .selfsigned.deployer import SelfSignedTlsDeployer
 from .www import build_webpages, find_merge_conflict, get_paths
 
 
@@ -540,6 +541,20 @@ class GithashDeployer(Deployer):
         )
 
 
+def get_tls_deployer(config, mail_domain):
+    """Select the appropriate TLS deployer based on config."""
+    tls_domains = [mail_domain, f"mta-sts.{mail_domain}", f"www.{mail_domain}"]
+
+    if config.tls_cert_mode == "acme":
+        return AcmetoolDeployer(config.acme_email, tls_domains)
+    elif config.tls_cert_mode == "self":
+        return SelfSignedTlsDeployer(mail_domain)
+    elif config.tls_cert_mode == "external":
+        return ExternalTlsDeployer(config.tls_cert_path, config.tls_key_path)
+    else:
+        raise ValueError(f"Unknown tls_cert_mode: {config.tls_cert_mode}")
+
+
 def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -> None:
     """Deploy a chat-mail instance.
 
@@ -608,12 +623,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
                     )
                     exit(1)
 
-    tls_domains = [mail_domain, f"mta-sts.{mail_domain}", f"www.{mail_domain}"]
-
-    if config.tls_cert_mode == "acme":
-        tls_deployer = AcmetoolDeployer(config.acme_email, tls_domains)
-    else:
-        tls_deployer = SelfSignedTlsDeployer(mail_domain)
+    tls_deployer = get_tls_deployer(config, mail_domain)
 
     all_deployers = [
         ChatmailDeployer(mail_domain),

cmdeploy/src/cmdeploy/external/deployer.py

@@ -0,0 +1,67 @@
+import io
+
+from pyinfra import host
+from pyinfra.facts.files import File
+from pyinfra.operations import files, systemd
+
+from cmdeploy.basedeploy import Deployer, get_resource
+
+
+class ExternalTlsDeployer(Deployer):
+    """Expects TLS certificates to be managed on the server.
+
+    Validates that the configured certificate and key files
+    exist on the remote host.  Installs a systemd path unit
+    that watches the certificate file and automatically
+    restarts/reloads affected services when it changes.
+    """
+
+    def __init__(self, cert_path, key_path):
+        self.cert_path = cert_path
+        self.key_path = key_path
+
+    def configure(self):
+        # Verify cert and key exist on the remote host using pyinfra facts.
+        for path in (self.cert_path, self.key_path):
+            info = host.get_fact(File, path=path)
+            if info is None:
+                raise Exception(f"External TLS file not found on server: {path}")
+
+        # Deploy the .path unit (templated with the cert path).
+        # pkg=__package__ is required here because the resource files
+        # live in cmdeploy.external, not the default cmdeploy package.
+        source = get_resource("tls-cert-reload.path.f", pkg=__package__)
+        content = source.read_text().format(cert_path=self.cert_path).encode()
+
+        path_unit = files.put(
+            name="Upload tls-cert-reload.path",
+            src=io.BytesIO(content),
+            dest="/etc/systemd/system/tls-cert-reload.path",
+            user="root",
+            group="root",
+            mode="644",
+        )
+
+        service_unit = files.put(
+            name="Upload tls-cert-reload.service",
+            src=get_resource("tls-cert-reload.service", pkg=__package__),
+            dest="/etc/systemd/system/tls-cert-reload.service",
+            user="root",
+            group="root",
+            mode="644",
+        )
+
+        if path_unit.changed or service_unit.changed:
+            self.need_restart = True
+
+    def activate(self):
+        systemd.service(
+            name="Enable tls-cert-reload path watcher",
+            service="tls-cert-reload.path",
+            running=True,
+            enabled=True,
+            restarted=self.need_restart,
+            daemon_reload=self.need_restart,
+        )
+        # No explicit reload needed here: dovecot/nginx read the cert
+        # on startup, and the .path watcher handles live changes.

cmdeploy/src/cmdeploy/external/tls-cert-reload.path.f

@@ -0,0 +1,15 @@
+# Watch the TLS certificate file for changes.
+# When the cert is updated (e.g. renewed by an external process),
+# this triggers tls-cert-reload.service to reload the affected services.
+#
+# NOTE: changes to the certificates are not detected if they cross bind-mount boundaries. 
+# After cert renewal, you must then trigger the reload explicitly:
+#   systemctl start tls-cert-reload.service
+[Unit]
+Description=Watch TLS certificate for changes
+
+[Path]
+PathChanged={cert_path}
+
+[Install]
+WantedBy=multi-user.target

cmdeploy/src/cmdeploy/external/tls-cert-reload.service

@@ -0,0 +1,15 @@
+# Reload services that cache the TLS certificate.
+#
+# dovecot: caches the cert at startup; reload re-reads SSL certs
+#          without dropping existing connections.
+# nginx:   caches the cert at startup; reload gracefully picks up
+#          the new cert for new connections.
+# postfix: reads the cert fresh on each TLS handshake,
+#          does NOT need a reload/restart.
+[Unit]
+Description=Reload TLS services after certificate change
+
+[Service]
+Type=oneshot
+ExecStart=/bin/systemctl try-reload-or-restart dovecot
+ExecStart=/bin/systemctl try-reload-or-restart nginx

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -84,7 +84,7 @@ http {
 		}
 
 		location /new {
-{% if config.tls_cert_mode == "acme" %}
+{% if config.tls_cert_mode != "self" %}
 			if ($request_method = GET) {
 				# Redirect to Delta Chat,
 				# which will in turn do a POST request.
@@ -106,7 +106,7 @@ http {
 		#
 		# Redirects are only for browsers.
 		location /cgi-bin/newemail.py {
-{% if config.tls_cert_mode == "acme" %}
+{% if config.tls_cert_mode != "self" %}
 			if ($request_method = GET) {
 				return 301 dcaccount:https://{{ config.mail_domain }}/new;
 			}

cmdeploy/src/cmdeploy/selfsigned/deployer.py

@@ -1,8 +1,29 @@
-from pyinfra.operations import apt, files, server
+import shlex
+
+from pyinfra.operations import apt, server
 
 from cmdeploy.basedeploy import Deployer
 
 
+def openssl_selfsigned_args(domain, cert_path, key_path, days=36500):
+    """Return the openssl argument list for a self-signed certificate.
+
+    The certificate uses an EC P-256 key with SAN entries for *domain*,
+    ``www.<domain>`` and ``mta-sts.<domain>``.
+    """
+    return [
+        "openssl", "req", "-x509",
+        "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:P-256",
+        "-noenc", "-days", str(days),
+        "-keyout", str(key_path),
+        "-out", str(cert_path),
+        "-subj", f"/CN={domain}",
+        "-addext", "extendedKeyUsage=serverAuth,clientAuth",
+        "-addext",
+        f"subjectAltName=DNS:{domain},DNS:www.{domain},DNS:mta-sts.{domain}",
+    ]
+
+
 class SelfSignedTlsDeployer(Deployer):
     """Generates a self-signed TLS certificate for all chatmail endpoints."""
 
@@ -18,18 +39,13 @@ class SelfSignedTlsDeployer(Deployer):
         )
 
     def configure(self):
+        args = openssl_selfsigned_args(
+            self.mail_domain, self.cert_path, self.key_path,
+        )
+        cmd = shlex.join(args)
         server.shell(
             name="Generate self-signed TLS certificate if not present",
-            commands=[
-                f"[ -f {self.cert_path} ] || openssl req -x509"
-                f" -newkey ec -pkeyopt ec_paramgen_curve:P-256"
-                f" -noenc -days 36500"
-                f" -keyout {self.key_path}"
-                f" -out {self.cert_path}"
-                f' -subj "/CN={self.mail_domain}"'
-                f' -addext "extendedKeyUsage=serverAuth,clientAuth"'
-                f' -addext "subjectAltName=DNS:{self.mail_domain},DNS:www.{self.mail_domain},DNS:mta-sts.{self.mail_domain}"',
-            ],
+            commands=[f"[ -f {self.cert_path} ] || {cmd}"],
         )
 
     def activate(self):

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -98,13 +98,13 @@ class TestEndToEndDeltaChat:
 
         lp.sec("ac2: check quota is triggered")
 
-        starting = True
-        for line in remote.iter_output("journalctl -n0 -f -u dovecot"):
-            if starting:
-                chat.send_text("hello")
-                starting = False
+        def send_hello():
+            chat.send_text("hello")
+
+        for line in remote.iter_output(
+            "journalctl -n1 -f -u dovecot", ready=send_hello
+        ):
             if user not in line:
-                # print(line)
                 continue
             if "quota exceeded" in line:
                 return

cmdeploy/src/cmdeploy/tests/plugin.py

@@ -395,7 +395,7 @@ class Remote:
     def __init__(self, sshdomain):
         self.sshdomain = sshdomain
 
-    def iter_output(self, logcmd=""):
+    def iter_output(self, logcmd="", ready=None):
         getjournal = "journalctl -f" if not logcmd else logcmd
         print(self.sshdomain)
         match self.sshdomain:
@@ -410,10 +410,12 @@ class Remote:
         while 1:
             line = self.popen.stdout.readline()
             res = line.decode().strip().lower()
-            if res:
-                yield res
-            else:
+            if not res:
                 break
+            if ready is not None:
+                ready()
+                ready = None
+            yield res
 
 
 @pytest.fixture

cmdeploy/src/cmdeploy/tests/test_external_tls.py

@@ -0,0 +1,78 @@
+"""Functional tests for tls_external_cert_and_key option."""
+
+import json
+
+import chatmaild.newemail
+import pytest
+from chatmaild.config import read_config, write_initial_config
+
+
+def make_external_config(tmp_path, cert_key=None):
+    inipath = tmp_path / "chatmail.ini"
+    overrides = {}
+    if cert_key is not None:
+        overrides["tls_external_cert_and_key"] = cert_key
+    write_initial_config(inipath, "chat.example.org", overrides=overrides)
+    return inipath
+
+
+def test_external_tls_config_reads_paths(tmp_path):
+    inipath = make_external_config(
+        tmp_path,
+        cert_key=(
+            "/etc/letsencrypt/live/chat.example.org/fullchain.pem"
+            " /etc/letsencrypt/live/chat.example.org/privkey.pem"
+        ),
+    )
+    config = read_config(inipath)
+    assert config.tls_cert_mode == "external"
+    assert (
+        config.tls_cert_path == "/etc/letsencrypt/live/chat.example.org/fullchain.pem"
+    )
+    assert config.tls_key_path == "/etc/letsencrypt/live/chat.example.org/privkey.pem"
+
+
+def test_external_tls_missing_option_uses_acme(tmp_path):
+    config = read_config(make_external_config(tmp_path))
+    assert config.tls_cert_mode == "acme"
+
+
+def test_external_tls_bad_format_raises(tmp_path):
+    inipath = make_external_config(tmp_path, cert_key="/only/one/path.pem")
+    with pytest.raises(ValueError, match="two space-separated"):
+        read_config(inipath)
+
+
+def test_external_tls_three_paths_raises(tmp_path):
+    inipath = make_external_config(tmp_path, cert_key="/a /b /c")
+    with pytest.raises(ValueError, match="two space-separated"):
+        read_config(inipath)
+
+
+def test_external_tls_no_dclogin_url(tmp_path, capsys, monkeypatch):
+    inipath = make_external_config(
+        tmp_path, cert_key="/certs/fullchain.pem /certs/privkey.pem"
+    )
+    monkeypatch.setattr(chatmaild.newemail, "CONFIG_PATH", str(inipath))
+    chatmaild.newemail.print_new_account()
+    out, _ = capsys.readouterr()
+    lines = out.split("\n")
+    dic = json.loads(lines[2])
+    assert "dclogin_url" not in dic
+
+
+def test_external_tls_selects_correct_deployer(tmp_path):
+    from cmdeploy.deployers import get_tls_deployer
+    from cmdeploy.external.deployer import ExternalTlsDeployer
+    from cmdeploy.selfsigned.deployer import SelfSignedTlsDeployer
+
+    inipath = make_external_config(
+        tmp_path, cert_key="/certs/fullchain.pem /certs/privkey.pem"
+    )
+    config = read_config(inipath)
+    deployer = get_tls_deployer(config, "chat.example.org")
+
+    assert isinstance(deployer, ExternalTlsDeployer)
+    assert not isinstance(deployer, SelfSignedTlsDeployer)
+    assert deployer.cert_path == "/certs/fullchain.pem"
+    assert deployer.key_path == "/certs/privkey.pem"

doc/source/getting_started.rst

@@ -198,6 +198,44 @@ and all other relays will accept connections from it
 without requiring certificate verification.
 This is useful for experimental setups and testing.
 
+.. _external-tls:
+
+Running a relay with externally managed certificates
+-----------------------------------------------------
+
+If you already have a TLS certificate manager
+(e.g. Traefik, certbot, or another ACME client)
+running on the deployment server,
+you can configure the relay to use those certificates
+instead of the built-in ``acmetool``.
+
+Set the following in ``chatmail.ini``::
+
+    tls_external_cert_and_key = /path/to/fullchain.pem /path/to/privkey.pem
+
+The paths must point to certificate and key files
+on the deployment server.
+During ``cmdeploy run``, these paths are written into
+the Postfix, Dovecot, and Nginx configurations.
+No certificate files are transferred from the build machine —
+they must already exist on the server,
+managed by your external certificate tool.
+
+The deploy will verify that both files exist on the server.
+``acmetool`` is **not** installed or run in this mode.
+
+.. note::
+
+   You are responsible for certificate renewal.
+   When the certificate file changes on disk,
+   all relay services pick up the new certificate automatically
+   via a systemd path watcher installed during deploy.
+   The watcher uses inotify, which does not cross bind-mount boundaries.
+   If you use such a setup, you must trigger the reload explicitly after renewal::
+
+      systemctl start tls-cert-reload.service
+
+
 Migrating to a new build machine
 ----------------------------------
 

doc/source/overview.rst

@@ -308,6 +308,11 @@ When providing a TLS certificate to your chatmail relay server, make
 sure to provide the full certificate chain and not just the last
 certificate.
 
+If you use an external certificate manager (e.g. Traefik or certbot),
+set ``tls_external_cert_and_key`` in ``chatmail.ini``
+to provide the certificate and key paths.
+See :ref:`external-tls` for details.
+
 If you are running an Exim server and don’t see incoming connections
 from a chatmail relay server in the logs, make sure ``smtp_no_mail`` log
 item is enabled in the config with ``log_selector = +smtp_no_mail``. By