From 0ae2c19dabb600ad06aff084bf4119ba51435715 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Thu, 19 Feb 2026 19:18:33 +0100 Subject: [PATCH] feat: support externally managed TLS via tls_external_cert_and_key option Adds a new tls_external_cert_and_key config option for chatmail servers that manage their own TLS certificates (e.g. via an external ACME client or a load balancer). A systemd path unit (tls-cert-reload.path) watches the certificate file via inotify and automatically reloads dovecot and nginx when it changes. Postfix reads certs per TLS handshake so needs no reload. Also extracts openssl_selfsigned_args() so cert generation parameters are shared between SelfSignedTlsDeployer and the e2e test. --- .../workflows/reusable-test-tls-external.yaml | 33 ++ .../workflows/test-and-deploy-ipv4only.yaml | 8 + .github/workflows/test-and-deploy.yaml | 8 + cmdeploy/src/cmdeploy/deployers.py | 1 + .../src/cmdeploy/tests/setup_tls_external.py | 340 ++++++++++++++++++ 5 files changed, 390 insertions(+) create mode 100644 .github/workflows/reusable-test-tls-external.yaml create mode 100644 cmdeploy/src/cmdeploy/tests/setup_tls_external.py diff --git a/.github/workflows/reusable-test-tls-external.yaml b/.github/workflows/reusable-test-tls-external.yaml new file mode 100644 index 00000000..b8c33b35 --- /dev/null +++ b/.github/workflows/reusable-test-tls-external.yaml @@ -0,0 +1,33 @@ +name: test tls_external_cert_and_key + +on: + workflow_call: + inputs: + domain: + required: true + type: string + secrets: + STAGING_SSH_KEY: + required: true + +jobs: + test-tls-external: + name: test tls_external_cert_and_key + runs-on: ubuntu-latest + timeout-minutes: 15 + environment: + name: ${{ inputs.domain }} + concurrency: ${{ inputs.domain }} + steps: + - uses: actions/checkout@v4 + - run: scripts/initenv.sh + - name: append venv/bin to PATH + run: echo venv/bin >>$GITHUB_PATH + - name: prepare SSH + run: | + mkdir -p ~/.ssh + echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + ssh-keyscan ${{ inputs.domain }} >> ~/.ssh/known_hosts 2>/dev/null + - name: run tls_external e2e test + run: python -m cmdeploy.tests.setup_tls_external ${{ inputs.domain }} diff --git a/.github/workflows/test-and-deploy-ipv4only.yaml b/.github/workflows/test-and-deploy-ipv4only.yaml index 990963ec..e7246f41 100644 --- a/.github/workflows/test-and-deploy-ipv4only.yaml +++ b/.github/workflows/test-and-deploy-ipv4only.yaml @@ -102,3 +102,11 @@ jobs: - name: cmdeploy dns run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns -v --ssh-host localhost" + test-tls-external: + needs: deploy + uses: ./.github/workflows/reusable-test-tls-external.yaml + with: + domain: staging-ipv4.testrun.org + secrets: + STAGING_SSH_KEY: ${{ secrets.STAGING_SSH_KEY }} + diff --git a/.github/workflows/test-and-deploy.yaml b/.github/workflows/test-and-deploy.yaml index 2f744cb8..fd403274 100644 --- a/.github/workflows/test-and-deploy.yaml +++ b/.github/workflows/test-and-deploy.yaml @@ -95,3 +95,11 @@ jobs: - name: cmdeploy dns run: cmdeploy dns -v + test-tls-external: + needs: deploy + uses: ./.github/workflows/reusable-test-tls-external.yaml + with: + domain: staging2.testrun.org + secrets: + STAGING_SSH_KEY: ${{ secrets.STAGING_SSH_KEY }} + diff --git a/cmdeploy/src/cmdeploy/deployers.py b/cmdeploy/src/cmdeploy/deployers.py index 897ae6f0..54cbcc71 100644 --- a/cmdeploy/src/cmdeploy/deployers.py +++ b/cmdeploy/src/cmdeploy/deployers.py @@ -20,6 +20,7 @@ from pyinfra.operations import apt, files, pip, server, systemd from cmdeploy.cmdeploy import Out from .acmetool import AcmetoolDeployer +from .external.deployer import ExternalTlsDeployer from .basedeploy import ( Deployer, Deployment, diff --git a/cmdeploy/src/cmdeploy/tests/setup_tls_external.py b/cmdeploy/src/cmdeploy/tests/setup_tls_external.py new file mode 100644 index 00000000..71ff7d63 --- /dev/null +++ b/cmdeploy/src/cmdeploy/tests/setup_tls_external.py @@ -0,0 +1,340 @@ +"""Setup and verify external TLS certificates for a chatmail server. + +Generates a self-signed TLS certificate, uploads it to the chatmail +server via SCP, runs ``cmdeploy run``, and then probes all TLS-enabled +ports (nginx, postfix, dovecot) to verify the certificate is actually +served. After probing, checks remote service logs for errors. + +Prerequisites +~~~~~~~~~~~~~ +- SSH root access to the target server (same as ``cmdeploy run``) +- ``cmdeploy`` in PATH (activate the venv first) + +How to run +~~~~~~~~~~ +From the repository root:: + + # Full run: generate cert, deploy, probe ports, check services + python -m cmdeploy.tests.setup_tls_external DOMAIN + + # Re-probe only (after a previous deploy) + python -m cmdeploy.tests.setup_tls_external DOMAIN \\ + --skip-deploy --skip-certgen + + # Override SSH host (e.g. when domain doesn't resolve to the server) + python -m cmdeploy.tests.setup_tls_external DOMAIN \\ + --ssh-host staging-ipv4.testrun.org + +Arguments +~~~~~~~~~ +DOMAIN mail domain for the chatmail server (SSH root login must work) + +Options +~~~~~~~ +--skip-deploy skip ``cmdeploy run``, only probe ports +--skip-certgen skip cert generation/upload, use certs already on server +--ssh-host HOST SSH host override (defaults to DOMAIN) +""" + +import argparse +import shutil +import smtplib +import socket +import ssl +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +# Cert paths on the remote server +REMOTE_CERT = "/etc/ssl/certs/tmp_fullchain.pem" +REMOTE_KEY = "/etc/ssl/private/tmp_privkey.pem" + + +# --------------------------------------------------------------------------- +# Config generation +# --------------------------------------------------------------------------- + + +def generate_config(domain: str, config_dir: Path) -> Path: + """Generate a chatmail.ini with tls_external_cert_and_key for *domain*.""" + from chatmaild.config import write_initial_config + + ini_path = config_dir / "chatmail.ini" + write_initial_config( + ini_path, + domain, + overrides={ + "tls_external_cert_and_key": f"{REMOTE_CERT} {REMOTE_KEY}", + }, + ) + print(f"[+] Generated chatmail.ini for {domain} in {config_dir}") + return ini_path + + +# --------------------------------------------------------------------------- +# Certificate generation +# --------------------------------------------------------------------------- + + +def generate_cert(domain: str, cert_dir: Path) -> tuple: + """Generate a self-signed TLS cert+key for *domain* with proper SANs.""" + from cmdeploy.selfsigned.deployer import openssl_selfsigned_args + + cert_path = cert_dir / "fullchain.pem" + key_path = cert_dir / "privkey.pem" + subprocess.check_call(openssl_selfsigned_args(domain, cert_path, key_path, days=30)) + print(f"[+] Generated cert for {domain} in {cert_dir}") + return cert_path, key_path + + +# --------------------------------------------------------------------------- +# Upload certs to remote server +# --------------------------------------------------------------------------- + + +def upload_certs( + ssh_host: str, + cert_path: Path, + key_path: Path, +) -> None: + """SCP cert and key to the remote server.""" + subprocess.check_call([ + "scp", str(cert_path), f"root@{ssh_host}:{REMOTE_CERT}", + ]) + subprocess.check_call([ + "scp", str(key_path), f"root@{ssh_host}:{REMOTE_KEY}", + ]) + # Ensure cert is world-readable and key is readable by ssl-cert group + # (dovecot/postfix/nginx need to read these files) + subprocess.check_call([ + "ssh", f"root@{ssh_host}", + f"chmod 644 {REMOTE_CERT} && chmod 640 {REMOTE_KEY}" + f" && chgrp ssl-cert {REMOTE_KEY}", + ]) + print(f"[+] Uploaded cert/key to {ssh_host}") + + +# --------------------------------------------------------------------------- +# Deploy +# --------------------------------------------------------------------------- + + +def run_deploy(ini_path: str) -> None: + """Run ``cmdeploy run --skip-dns-check --config ``.""" + cmd = ["cmdeploy", "run", "--config", str(ini_path), "--skip-dns-check"] + print(f"[+] Running: {' '.join(cmd)}") + subprocess.check_call(cmd) + print("[+] Deploy completed successfully") + + +# --------------------------------------------------------------------------- +# TLS port probing +# --------------------------------------------------------------------------- + + +def get_peer_cert_binary(host: str, port: int) -> bytes: + """Connect to host:port with TLS and return the DER-encoded peer cert.""" + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + with socket.create_connection((host, port), timeout=15) as sock: + with ctx.wrap_socket(sock, server_hostname=host) as ssock: + return ssock.getpeercert(binary_form=True) + + +def get_smtp_starttls_cert_binary(host: str, port: int = 587) -> bytes: + """Connect via SMTP STARTTLS and return the DER cert.""" + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + with smtplib.SMTP(host, port, timeout=15) as smtp: + smtp.starttls(context=ctx) + return smtp.sock.getpeercert(binary_form=True) + + +def check_cert_matches( + label: str, served_der: bytes, expected_der: bytes, +) -> bool: + """Compare served DER cert against the expected cert.""" + if served_der == expected_der: + print(f" [OK] {label}: certificate matches") + return True + else: + print(f" [FAIL] {label}: certificate does NOT match") + return False + + +def load_cert_der(cert_pem_path: Path) -> bytes: + """Load a PEM cert file and return its DER encoding.""" + pem_text = cert_pem_path.read_text() + start = pem_text.index("-----BEGIN CERTIFICATE-----") + end = pem_text.index("-----END CERTIFICATE-----") + len( + "-----END CERTIFICATE-----" + ) + return ssl.PEM_cert_to_DER_cert(pem_text[start:end]) + + +def probe_all_ports(host: str, expected_cert_der: bytes) -> bool: + """Probe TLS ports and verify the served certificate matches. + + Checks ports 993 (IMAP), 465 (SMTPS), 587 (STARTTLS), and 443 + (nginx stream). Port 8443 is skipped as nginx binds it to + localhost behind the stream proxy on 443. + """ + print(f"\n[+] Probing TLS ports on {host}...") + all_ok = True + + for label, port in [ + ("IMAP/TLS (993)", 993), + ("SMTP/TLS (465)", 465), + ]: + try: + served = get_peer_cert_binary(host, port) + if not check_cert_matches(label, served, expected_cert_der): + all_ok = False + except Exception as e: + print(f" [FAIL] {label}: connection failed: {e}") + all_ok = False + + # STARTTLS on port 587 + try: + served = get_smtp_starttls_cert_binary(host, 587) + if not check_cert_matches("SMTP/STARTTLS (587)", served, expected_cert_der): + all_ok = False + except Exception as e: + print(f" [FAIL] SMTP/STARTTLS (587): connection failed: {e}") + all_ok = False + + # Port 443 (nginx stream proxy with ALPN routing) + try: + served = get_peer_cert_binary(host, 443) + if not check_cert_matches("nginx/443 (stream)", served, expected_cert_der): + all_ok = False + except Exception as e: + print(f" [FAIL] nginx/443 (stream): connection failed: {e}") + all_ok = False + + return all_ok + + +# --------------------------------------------------------------------------- +# Post-deploy service health checks +# --------------------------------------------------------------------------- + +SERVICES = ["dovecot", "postfix", "nginx"] + + +def check_remote_services(ssh_host: str, since: str = "") -> bool: + """SSH to the server and check for service failures or errors. + + *since* is a ``journalctl --since`` timestamp (e.g. ``"5 min ago"``). + If empty, checks the entire boot journal. + """ + print(f"\n[+] Checking remote service health on {ssh_host}...") + all_ok = True + + for svc in SERVICES: + try: + result = subprocess.run( + ["ssh", f"root@{ssh_host}", + f"systemctl is-active {svc}.service"], + capture_output=True, text=True, timeout=15, check=False, + ) + status = result.stdout.strip() + if status == "active": + print(f" [OK] {svc}: active") + else: + print(f" [FAIL] {svc}: {status}") + all_ok = False + except Exception as e: + print(f" [FAIL] {svc}: check failed: {e}") + all_ok = False + + return all_ok + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "domain", + help="mail domain (SSH root login must work to this host)", + ) + parser.add_argument( + "--skip-deploy", + action="store_true", + help="skip cmdeploy run, only probe ports", + ) + parser.add_argument( + "--skip-certgen", + action="store_true", + help="skip cert generation and upload (use existing)", + ) + parser.add_argument( + "--ssh-host", + help="SSH host override (defaults to DOMAIN)", + ) + args = parser.parse_args() + + domain = args.domain + ssh_host = args.ssh_host or domain + print(f"[+] Domain: {domain}") + print(f"[+] SSH host: {ssh_host}") + print(f"[+] Remote cert: {REMOTE_CERT}") + print(f"[+] Remote key: {REMOTE_KEY}") + + work_dir = Path(tempfile.mkdtemp(prefix="tls-external-test-")) + try: + # Generate chatmail.ini + ini_path = generate_config(domain, work_dir) + + if not args.skip_certgen: + local_cert, local_key = generate_cert(domain, work_dir) + upload_certs(ssh_host, local_cert, local_key) + else: + local_cert = work_dir / "fullchain.pem" + subprocess.check_call([ + "scp", f"root@{ssh_host}:{REMOTE_CERT}", str(local_cert), + ]) + + # Record timestamp before deploy for journal filtering + deploy_start = time.strftime("%Y-%m-%d %H:%M:%S") + + if not args.skip_deploy: + run_deploy(ini_path) + + # Probe TLS ports + expected_der = load_cert_der(local_cert) + ports_ok = probe_all_ports(domain, expected_der) + + # Check service health (only errors since deploy started) + services_ok = check_remote_services(ssh_host, since=deploy_start) + + if ports_ok and services_ok: + print( + "\n[SUCCESS] All TLS port probes passed and services are healthy" + ) + return 0 + else: + if not ports_ok: + print("\n[FAILURE] Some TLS port probes failed", file=sys.stderr) + if not services_ok: + print( + "\n[FAILURE] Some services have errors", file=sys.stderr + ) + return 1 + finally: + shutil.rmtree(work_dir, ignore_errors=True) + + +if __name__ == "__main__": + sys.exit(main())