ci: replace staging workflows with LXC-local testing

Replace the two staging-server CI workflows and their zone-file
helpers with a single lxc-test job in ci.yaml that runs
'cmdeploy lxc-test' inside an ubuntu-24.04 runner.

The new workflow installs Incus from the Zabbly apt repository,
initialises it, bootstraps the venv, caches the base LXC image
together with SSH keys, and runs the full LXC pipeline
(container creation, deploy, DNS zones, tests).

.github/workflows/ci.yaml

@@ -16,27 +16,77 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: download filtermail
         run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.5.2/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
-      - name: run chatmaild tests 
+      - name: run chatmaild tests
         working-directory: chatmaild
         run: pipx run tox
 
   scripts:
-    name: deploy-chatmail tests 
+    name: deploy-chatmail tests
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
 
-      - name: initenv 
+      - name: initenv
         run: scripts/initenv.sh
 
       - name: append venv/bin to PATH
         run: echo venv/bin >>$GITHUB_PATH
 
-      - name: run formatting checks 
-        run: cmdeploy fmt -v 
+      - name: run formatting checks
+        run: cmdeploy fmt -v
 
-      - name: run deploy-chatmail offline tests 
-        run: pytest --pyargs cmdeploy 
+      - name: run deploy-chatmail offline tests
+        run: pytest --pyargs cmdeploy
 
-      # all other cmdeploy commands require a staging server 
-      # see https://github.com/deltachat/chatmail/issues/100
+  lxc-test:
+    name: LXC deploy and test
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: install incus
+        run: |
+          # zabbly is the official incus community packages source
+          curl -fsSL https://pkgs.zabbly.com/key.asc \
+            | sudo gpg --dearmor -o /etc/apt/keyrings/zabbly.gpg
+          sudo sh -c 'cat <<EOF > /etc/apt/sources.list.d/zabbly-incus-stable.sources
+          Enabled: yes
+          Types: deb
+          URIs: https://pkgs.zabbly.com/incus/stable
+          Suites: $(. /etc/os-release && echo ${VERSION_CODENAME})
+          Components: main
+          Architectures: $(dpkg --print-architecture)
+          Signed-By: /etc/apt/keyrings/zabbly.gpg
+          EOF'
+          sudo apt-get update
+          sudo apt-get install -y incus
+
+      - name: initialise incus
+        run: |
+          sudo systemctl stop docker.socket docker || true
+          sudo iptables -P FORWARD ACCEPT
+          sudo sysctl -w fs.inotify.max_user_instances=65535
+          sudo sysctl -w fs.inotify.max_user_watches=65535
+          sudo incus admin init --minimal
+          sudo usermod -aG incus-admin "$USER"
+
+      - name: initenv
+        run: scripts/initenv.sh
+
+      - name: append venv/bin to PATH
+        run: echo venv/bin >>$GITHUB_PATH
+
+      - name: lxc-test
+        run: sg incus-admin -c 'cmdeploy lxc-test'
+
+      - name: export images for cache
+        if: always()
+        run: |
+          for alias in localchat-base localchat-ns localchat-test0 localchat-test1; do
+            if ! [ -f /tmp/$alias.tar.gz ]; then
+              sg incus-admin -c "incus image export $alias /tmp/$alias" || true
+            fi
+          done

.github/workflows/staging-ipv4.testrun.org-default.zone

@@ -1,20 +0,0 @@
-;; Zone file for staging-ipv4.testrun.org
-
-$ORIGIN staging-ipv4.testrun.org.
-$TTL 300
-
-@  IN  SOA     ns.testrun.org. root.nine.testrun.org (
-  2023010101 ; Serial
-  7200       ; Refresh
-  3600       ; Retry
-  1209600    ; Expire
-  3600       ; Negative response caching TTL
-)
-
-;; Nameservers.
-@  IN NS ns.testrun.org.
-
-;; DNS records.
-@       IN      A       37.27.95.249
-mta-sts.staging-ipv4.testrun.org.    CNAME   staging-ipv4.testrun.org.
-www.staging-ipv4.testrun.org.        CNAME   staging-ipv4.testrun.org.

.github/workflows/staging.testrun.org-default.zone

@@ -1,21 +0,0 @@
-;; Zone file for staging2.testrun.org
-
-$ORIGIN staging2.testrun.org.
-$TTL 300
-
-@  IN  SOA     ns.testrun.org. root.nine.testrun.org (
-  2023010101 ; Serial
-  7200       ; Refresh
-  3600       ; Retry
-  1209600    ; Expire
-  3600       ; Negative response caching TTL
-)
-
-;; Nameservers.
-@  IN NS ns.testrun.org.
-
-;; DNS records.
-@       IN      A       37.27.24.139
-mta-sts.staging2.testrun.org.    CNAME   staging2.testrun.org.
-www.staging2.testrun.org.        CNAME   staging2.testrun.org.
-

.github/workflows/test-and-deploy-ipv4only.yaml

@@ -1,104 +0,0 @@
-name: deploy on staging-ipv4.testrun.org, and run tests
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    paths-ignore:
-      - 'scripts/**'
-      - '**/README.md'
-      - 'CHANGELOG.md'
-      - 'LICENSE'
-
-jobs:
-  deploy:
-    name: deploy on staging-ipv4.testrun.org, and run tests
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    environment:
-      name: staging-ipv4.testrun.org
-      url: https://staging-ipv4.testrun.org/
-    concurrency: staging-ipv4.testrun.org
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: prepare SSH
-        run: |
-          mkdir ~/.ssh
-          echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
-          chmod 600 ~/.ssh/id_ed25519
-          ssh-keyscan staging-ipv4.testrun.org > ~/.ssh/known_hosts
-          # save previous acme & dkim state
-          rsync -avz root@staging-ipv4.testrun.org:/var/lib/acme acme-ipv4 || true
-          rsync -avz root@staging-ipv4.testrun.org:/etc/dkimkeys dkimkeys-ipv4 || true
-          # store previous acme & dkim state on ns.testrun.org, if it contains useful certs
-          if [ -f dkimkeys-ipv4/dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys-ipv4 root@ns.testrun.org:/tmp/ || true; fi
-          if [ "$(ls -A acme-ipv4/acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme-ipv4 root@ns.testrun.org:/tmp/ || true; fi
-          # make sure CAA record isn't set
-          scp -o StrictHostKeyChecking=accept-new .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
-          ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging-ipv4.testrun.org.zone
-          ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
-          ssh root@ns.testrun.org systemctl reload nsd
-
-      - name: rebuild staging-ipv4.testrun.org to have a clean VPS
-        run: |
-            curl -X POST \
-            -H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
-            -H "Content-Type: application/json" \
-            -d '{"image":"debian-12"}' \
-            "https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_IPV4_SERVER_ID }}/actions/rebuild"
-
-      - run: scripts/initenv.sh
-
-      - name: append venv/bin to PATH
-        run: echo venv/bin >>$GITHUB_PATH
-
-      - name: upload TLS cert after rebuilding
-        run: |
-          echo " --- wait until staging-ipv4.testrun.org VPS is rebuilt --- "
-          rm ~/.ssh/known_hosts
-          while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u ; do sleep 1 ; done
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u
-          # download acme & dkim state from ns.testrun.org
-          rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme-ipv4/acme acme-restore || true
-          rsync -avz root@ns.testrun.org:/tmp/dkimkeys-ipv4/dkimkeys dkimkeys-restore || true
-          # restore acme & dkim state to staging2.testrun.org
-          rsync -avz acme-restore/acme root@staging-ipv4.testrun.org:/var/lib/ || true
-          rsync -avz dkimkeys-restore/dkimkeys root@staging-ipv4.testrun.org:/etc/ || true
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org chown root:root -R /var/lib/acme || true
-
-      - name: run deploy-chatmail offline tests 
-        run: pytest --pyargs cmdeploy 
-
-      - name: setup dependencies
-        run: |
-          ssh root@staging-ipv4.testrun.org apt update
-          ssh root@staging-ipv4.testrun.org apt install -y git python3.11-venv python3-dev gcc
-          ssh root@staging-ipv4.testrun.org git clone https://github.com/chatmail/relay
-          ssh root@staging-ipv4.testrun.org "cd relay && git checkout " ${{ github.head_ref }}
-          ssh root@staging-ipv4.testrun.org "cd relay && scripts/initenv.sh"
-
-      - name: initialize config
-        run: |
-          ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy init staging-ipv4.testrun.org"
-          ssh root@staging-ipv4.testrun.org "sed -i 's#disable_ipv6 = False#disable_ipv6 = True#' relay/chatmail.ini"
-          ssh root@staging-ipv4.testrun.org "sed -i 's/#\s*mtail_address/mtail_address/' relay/chatmail.ini"
-
-      - run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy run --verbose --skip-dns-check --ssh-host localhost"
-
-      - name: set DNS entries
-        run: |
-          ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns --zonefile staging-generated.zone --ssh-host localhost"
-          ssh root@staging-ipv4.testrun.org cat relay/staging-generated.zone >> .github/workflows/staging-ipv4.testrun.org-default.zone
-          cat .github/workflows/staging-ipv4.testrun.org-default.zone
-          scp .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
-          ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
-          ssh root@ns.testrun.org systemctl reload nsd
-
-      - name: cmdeploy test
-        run: ssh root@staging-ipv4.testrun.org "cd relay && CHATMAIL_DOMAIN2=ci-chatmail.testrun.org scripts/cmdeploy test --slow --ssh-host localhost"
-
-      - name: cmdeploy dns
-        run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns -v --ssh-host localhost"
-

.github/workflows/test-and-deploy.yaml

@@ -1,38 +0,0 @@
-name: deploy on allocated VPS and run tests
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    paths-ignore:
-      - 'scripts/**'
-      - '**/README.md'
-      - 'CHANGELOG.md'
-      - 'LICENSE'
-
-jobs:
-  deploy-and-test:
-    name: deploy on allocated VPS, and run tests
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          repository: chatmail/hetzner-relay
-          path: hetzner-relay
-
-      - uses: actions/checkout@v4
-        with:
-          path: relay
-
-      - name: prepare SSH
-        run: |
-          mkdir ~/.ssh
-          echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/staging.testrun.org
-          chmod 600 ~/.ssh/staging.testrun.org
-
-      - run: pip install hetzner-relay/
-
-      - run: python3 hetzner-relay/main.py -i ~/.ssh/staging.testrun.org --dns --test --rebuild --run-id $GITHUB_RUN_ID --domain2 ci-chatmail.testrun.org --hetzner-api-token ${{ secrets.HETZNER_API_TOKEN }} relay/
-

.gitignore

@@ -5,6 +5,7 @@ __pycache__/
 *.swp
 *qr-*.png
 chatmail*.ini
+lxconfigs/
 
 
 # C extensions

chatmaild/src/chatmaild/tests/plugin.py

@@ -85,13 +85,13 @@ def mockout():
         captured_green = []
         captured_plain = []
 
-        def red(self, msg):
+        def red(self, msg, **kw):
             self.captured_red.append(msg)
 
-        def green(self, msg):
+        def green(self, msg, **kw):
             self.captured_green.append(msg)
 
-        def __call__(self, msg):
+        def print(self, msg="", **kw):
             self.captured_plain.append(msg)
 
     return MockOut()

cmdeploy/src/cmdeploy/basedeploy.py

@@ -1,6 +1,7 @@
 import importlib.resources
 import io
 import os
+from contextlib import contextmanager
 
 from pyinfra.operations import files, server, systemd
 
@@ -10,6 +11,28 @@ def has_systemd():
     return os.path.isdir("/run/systemd/system")
 
 
+@contextmanager
+def blocked_service_startup():
+    """Prevent services from auto-starting during package installation.
+
+    Installs a ``/usr/sbin/policy-rc.d`` that exits 101, blocking any
+    service from being started by the package manager.  This avoids bind
+    conflicts and CPU/RAM spikes during initial setup.  The file is removed
+    when the context exits.
+    """
+    # For documentation about policy-rc.d, see:
+    # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
+    files.put(
+        src=get_resource("policy-rc.d"),
+        dest="/usr/sbin/policy-rc.d",
+        user="root",
+        group="root",
+        mode="755",
+    )
+    yield
+    files.file("/usr/sbin/policy-rc.d", present=False)
+
+
 def get_resource(arg, pkg=__package__):
     return importlib.resources.files(pkg).joinpath(arg)
 

cmdeploy/src/cmdeploy/chatmail.zone.j2

@@ -1,32 +0,0 @@
-;
-; Required DNS entries for chatmail servers 
-;
-{% if A %}
-{{ mail_domain }}.                   A     {{ A }}
-{% endif %}
-{% if AAAA %}
-{{ mail_domain }}.                   AAAA  {{ AAAA }}
-{% endif %}
-{{ mail_domain }}.                   MX 10 {{ mail_domain }}.
-{% if strict_tls %}
-_mta-sts.{{ mail_domain }}.          TXT "v=STSv1; id={{ sts_id }}"
-mta-sts.{{ mail_domain }}.           CNAME {{ mail_domain }}.
-{% endif %}
-www.{{ mail_domain }}.               CNAME {{ mail_domain }}.
-{{ dkim_entry }}
-
-;
-; Recommended DNS entries for interoperability and security-hardening
-;
-{{ mail_domain }}.                   TXT "v=spf1 a ~all"
-_dmarc.{{ mail_domain }}.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
-
-{% if acme_account_url %}
-{{ mail_domain }}.                   CAA 0 issue "letsencrypt.org;accounturi={{ acme_account_url }}"
-{% endif %}
-_adsp._domainkey.{{ mail_domain }}.  TXT "dkim=discardable"
-
-_submission._tcp.{{ mail_domain }}.  SRV 0 1 587 {{ mail_domain }}.
-_submissions._tcp.{{ mail_domain }}. SRV 0 1 465 {{ mail_domain }}.
-_imap._tcp.{{ mail_domain }}.        SRV 0 1 143 {{ mail_domain }}.
-_imaps._tcp.{{ mail_domain }}.       SRV 0 1 993 {{ mail_domain }}.

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -15,10 +15,27 @@ from pathlib import Path
 import pyinfra
 from chatmaild.config import read_config, write_initial_config
 from packaging import version
-from termcolor import colored
 
 from . import dns, remote
-from .sshexec import LocalExec, SSHExec
+from .lxc.cli import (
+    lxc_start_cmd,
+    lxc_start_cmd_options,
+    lxc_status_cmd,
+    lxc_status_cmd_options,
+    lxc_stop_cmd,
+    lxc_stop_cmd_options,
+    lxc_test_cmd,
+    lxc_test_cmd_options,
+)
+from .lxc.incus import DNSConfigurationError
+from .sshexec import (
+    LocalExec,
+    SSHExec,
+    resolve_host_from_ssh_config,
+    resolve_key_from_ssh_config,
+)
+from .util import Out
+from .www import main as webdev_main
 
 #
 # cmdeploy sub commands and options
@@ -82,18 +99,21 @@ def run_cmd_options(parser):
         help="disable checks nslookup for dns",
     )
     add_ssh_host_option(parser)
+    add_ssh_config_option(parser)
 
 
 def run_cmd(args, out):
     """Deploy chatmail services on the remote server."""
 
     ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
-    sshexec = get_sshexec(ssh_host)
+    sshexec = get_sshexec(ssh_host, ssh_config=args.ssh_config)
     require_iroh = args.config.enable_iroh_relay
     strict_tls = args.config.tls_cert_mode == "acme"
     if not args.dns_check_disabled:
         remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
-        if not dns.check_initial_remote_data(remote_data, strict_tls=strict_tls, print=out.red):
+        if not dns.check_initial_remote_data(
+            remote_data, strict_tls=strict_tls, print=out.red
+        ):
             return 1
 
     env = os.environ.copy()
@@ -104,10 +124,24 @@ def run_cmd(args, out):
     if not args.dns_check_disabled:
         env["CHATMAIL_ADDR_V4"] = remote_data.get("A") or ""
         env["CHATMAIL_ADDR_V6"] = remote_data.get("AAAA") or ""
+    env["DEBIAN_FRONTEND"] = "noninteractive"
+    env["TERM"] = "linux"
     deploy_path = importlib.resources.files(__package__).joinpath("run.py").resolve()
     pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
 
     cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y"
+    ssh_config = args.ssh_config
+    if ssh_config:
+        ssh_config = str(Path(ssh_config).resolve())
+
+        # Use pyinfra's native SSH data keys to configure the connection directly
+        # rather than relying on paramiko config parsing (see also sshexec.py)
+        ip = resolve_host_from_ssh_config(ssh_host, ssh_config)
+        key = resolve_key_from_ssh_config(ssh_host, ssh_config)
+        data_args = f"--data ssh_hostname={ip} --data ssh_known_hosts_file=/dev/null"
+        if key:
+            data_args += f" --data ssh_key={key}"
+        cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y {data_args}"
     if ssh_host in ["localhost", "@docker"]:
         if ssh_host == "@docker":
             env["CHATMAIL_NOPORTCHECK"] = "True"
@@ -119,10 +153,17 @@ def run_cmd(args, out):
         return 1
 
     try:
-        out.check_call(cmd, env=env)
+        ret = out.shell(cmd, env=env)
+        if ret:
+            out.red("Deploy failed")
+            return 1
         if args.website_only:
             out.green("Website deployment completed.")
-        elif not args.dns_check_disabled and strict_tls and not remote_data["acme_account_url"]:
+        elif (
+            not args.dns_check_disabled
+            and strict_tls
+            and not remote_data["acme_account_url"]
+        ):
             out.red("Deploy completed but letsencrypt not configured")
             out.red("Run 'cmdeploy run' again")
         else:
@@ -139,15 +180,16 @@ def dns_cmd_options(parser):
         dest="zonefile",
         type=pathlib.Path,
         default=None,
-        help="write out a zonefile",
+        help="write DNS records in standard BIND format to the given file",
     )
     add_ssh_host_option(parser)
+    add_ssh_config_option(parser)
 
 
 def dns_cmd(args, out):
     """Check DNS entries and optionally generate dns zone file."""
     ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
-    sshexec = get_sshexec(ssh_host, verbose=args.verbose)
+    sshexec = get_sshexec(ssh_host, verbose=args.verbose, ssh_config=args.ssh_config)
     tls_cert_mode = args.config.tls_cert_mode
     strict_tls = tls_cert_mode == "acme"
     remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
@@ -178,13 +220,14 @@ def dns_cmd(args, out):
 
 def status_cmd_options(parser):
     add_ssh_host_option(parser)
+    add_ssh_config_option(parser)
 
 
 def status_cmd(args, out):
     """Display status for online chatmail instance."""
 
     ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
-    sshexec = get_sshexec(ssh_host, verbose=args.verbose)
+    sshexec = get_sshexec(ssh_host, verbose=args.verbose, ssh_config=args.ssh_config)
 
     out.green(f"chatmail domain: {args.config.mail_domain}")
     if args.config.privacy_mail:
@@ -204,14 +247,14 @@ def test_cmd_options(parser):
         help="also run slow tests",
     )
     add_ssh_host_option(parser)
+    add_ssh_config_option(parser)
 
 
 def test_cmd(args, out):
     """Run local and online tests for chatmail deployment."""
 
     env = os.environ.copy()
-    if args.ssh_host:
-        env["CHATMAIL_SSH"] = args.ssh_host
+    env["CHATMAIL_INI"] = str(args.inipath.resolve())
 
     pytest_path = shutil.which("pytest")
     pytest_args = [
@@ -225,7 +268,11 @@ def test_cmd(args, out):
     ]
     if args.slow:
         pytest_args.append("--slow")
-    ret = out.run_ret(pytest_args, env=env)
+    if args.ssh_host:
+        pytest_args.extend(["--ssh-host", args.ssh_host])
+    if args.ssh_config:
+        pytest_args.extend(["--ssh-config", str(Path(args.ssh_config).resolve())])
+    ret = out.shell(" ".join(pytest_args), env=env)
     return ret
 
 
@@ -262,8 +309,8 @@ def fmt_cmd(args, out):
     format_args.extend(sources)
     check_args.extend(sources)
 
-    out.check_call(" ".join(format_args), quiet=not args.verbose)
-    out.check_call(" ".join(check_args), quiet=not args.verbose)
+    out.shell(" ".join(format_args), quiet=not args.verbose)
+    out.shell(" ".join(check_args), quiet=not args.verbose)
 
 
 def bench_cmd(args, out):
@@ -276,9 +323,7 @@ def bench_cmd(args, out):
 
 def webdev_cmd(args, out):
     """Run local web development loop for static web pages."""
-    from .www import main
-
-    main()
+    webdev_main()
 
 
 #
@@ -286,32 +331,6 @@ def webdev_cmd(args, out):
 #
 
 
-class Out:
-    """Convenience output printer providing coloring."""
-
-    def red(self, msg, file=sys.stderr):
-        print(colored(msg, "red"), file=file)
-
-    def green(self, msg, file=sys.stderr):
-        print(colored(msg, "green"), file=file)
-
-    def __call__(self, msg, red=False, green=False, file=sys.stdout):
-        color = "red" if red else ("green" if green else None)
-        print(colored(msg, color), file=file)
-
-    def check_call(self, arg, env=None, quiet=False):
-        if not quiet:
-            self(f"[$ {arg}]", file=sys.stderr)
-        return subprocess.check_call(arg, shell=True, env=env)
-
-    def run_ret(self, args, env=None, quiet=False):
-        if not quiet:
-            cmdstring = " ".join(args)
-            self(f"[$ {cmdstring}]", file=sys.stderr)
-        proc = subprocess.run(args, env=env, check=False)
-        return proc.returncode
-
-
 def add_ssh_host_option(parser):
     parser.add_argument(
         "--ssh-host",
@@ -321,6 +340,16 @@ def add_ssh_host_option(parser):
     )
 
 
+def add_ssh_config_option(parser):
+    parser.add_argument(
+        "--ssh-config",
+        dest="ssh_config",
+        type=Path,
+        default=None,
+        help="Path to an SSH config file (e.g. lxconfigs/ssh-config).",
+    )
+
+
 def add_config_option(parser):
     parser.add_argument(
         "--config",
@@ -330,25 +359,26 @@ def add_config_option(parser):
         type=Path,
         help="path to the chatmail.ini file",
     )
-    parser.add_argument(
-        "--verbose",
-        "-v",
-        dest="verbose",
-        action="store_true",
-        default=False,
-        help="provide verbose logging",
-    )
 
 
-def add_subcommand(subparsers, func):
+def add_subcommand(subparsers, func, add_config=True):
     name = func.__name__
     assert name.endswith("_cmd")
-    name = name[:-4]
+    name = name[:-4].replace("_", "-")
     doc = func.__doc__.strip()
     help = doc.split("\n")[0].strip(".")
     p = subparsers.add_parser(name, description=doc, help=help)
     p.set_defaults(func=func)
-    add_config_option(p)
+    if add_config:
+        add_config_option(p)
+    p.add_argument(
+        "-v",
+        "--verbose",
+        dest="verbose",
+        action="count",
+        default=0,
+        help="increase verbosity (can be repeated: -v, -vv)",
+    )
     return p
 
 
@@ -357,45 +387,60 @@ Setup your chatmail server configuration and
 deploy it via SSH to your remote location.
 """
 
+# Explicit subcommand registry: (cmd_func, options_func_or_None, needs_config).
+# LXC commands don't need a chatmail.ini (no config); all others do.
+SUBCOMMANDS = [
+    (init_cmd, init_cmd_options, True),
+    (run_cmd, run_cmd_options, True),
+    (dns_cmd, dns_cmd_options, True),
+    (status_cmd, status_cmd_options, True),
+    (test_cmd, test_cmd_options, True),
+    (fmt_cmd, fmt_cmd_options, True),
+    (bench_cmd, None, True),
+    (webdev_cmd, None, True),
+    (lxc_start_cmd, lxc_start_cmd_options, False),
+    (lxc_stop_cmd, lxc_stop_cmd_options, False),
+    (lxc_status_cmd, lxc_status_cmd_options, False),
+    (lxc_test_cmd, lxc_test_cmd_options, False),
+]
+
 
 def get_parser():
     """Return an ArgumentParser for the 'cmdeploy' CLI"""
 
     parser = argparse.ArgumentParser(description=description.strip())
+    parser.set_defaults(func=None, inipath=None)
     subparsers = parser.add_subparsers(title="subcommands")
 
-    # find all subcommands in the module namespace
-    glob = globals()
-    for name, func in glob.items():
-        if name.endswith("_cmd"):
-            subparser = add_subcommand(subparsers, func)
-            addopts = glob.get(name + "_options")
-            if addopts is not None:
-                addopts(subparser)
+    for func, addopts, needs_config in SUBCOMMANDS:
+        subparser = add_subcommand(subparsers, func, add_config=needs_config)
+        if addopts is not None:
+            addopts(subparser)
 
     return parser
 
 
-def get_sshexec(ssh_host: str, verbose=True):
+def get_sshexec(ssh_host: str, verbose=True, ssh_config=None):
     if ssh_host in ["localhost", "@local"]:
         return LocalExec(verbose, docker=False)
     elif ssh_host == "@docker":
         return LocalExec(verbose, docker=True)
     if verbose:
         print(f"[ssh] login to {ssh_host}")
-    return SSHExec(ssh_host, verbose=verbose)
+    return SSHExec(ssh_host, verbose=verbose, ssh_config=ssh_config)
 
 
 def main(args=None):
     """Provide main entry point for 'cmdeploy' CLI invocation."""
     parser = get_parser()
     args = parser.parse_args(args=args)
-    if not hasattr(args, "func"):
+    if args.func is None:
         return parser.parse_args(["-h"])
 
-    out = Out()
+    out = Out(sepchar="\u2501", verbosity=args.verbose)
     kwargs = {}
-    if args.func.__name__ not in ("init_cmd", "fmt_cmd"):
+
+    if args.inipath is not None and args.func.__name__ not in ("init_cmd", "fmt_cmd"):
         if not args.inipath.exists():
             out.red(f"expecting {args.inipath} to exist, run init first?")
             raise SystemExit(1)
@@ -410,6 +455,9 @@ def main(args=None):
         if res is None:
             res = 0
         return res
+    except DNSConfigurationError as exc:
+        out.red(str(exc))
+        return 1
     except KeyboardInterrupt:
         out.red("KeyboardInterrupt")
         sys.exit(130)

cmdeploy/src/cmdeploy/deployers.py

@@ -17,13 +17,12 @@ from pyinfra.facts.files import Sha256File
 from pyinfra.facts.systemd import SystemdEnabled
 from pyinfra.operations import apt, files, pip, server, systemd
 
-from cmdeploy.cmdeploy import Out
-
 from .acmetool import AcmetoolDeployer
 from .basedeploy import (
     Deployer,
     Deployment,
     activate_remote_units,
+    blocked_service_startup,
     configure_remote_units,
     get_resource,
     has_systemd,
@@ -36,6 +35,7 @@ from .nginx.deployer import NginxDeployer
 from .opendkim.deployer import OpendkimDeployer
 from .postfix.deployer import PostfixDeployer
 from .selfsigned.deployer import SelfSignedTlsDeployer
+from .util import Out, get_version_string
 from .www import build_webpages, find_merge_conflict, get_paths
 
 
@@ -149,33 +149,16 @@ class UnboundDeployer(Deployer):
         self.need_restart = False
 
     def install(self):
-        # Run local DNS resolver `unbound`.
-        # `resolvconf` takes care of setting up /etc/resolv.conf
-        # to use 127.0.0.1 as the resolver.
+        # Run local DNS resolver `unbound`. `resolvconf` takes care of
+        # setting up /etc/resolv.conf to use 127.0.0.1 as the resolver.
 
-        #
-        # On an IPv4-only system, if unbound is started but not
-        # configured, it causes subsequent steps to fail to resolve hosts.
-        # Here, we use policy-rc.d to prevent unbound from starting up
-        # on initial install.  Later, we will configure it and start it.
-        #
-        # For documentation about policy-rc.d, see:
-        # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
-        #
-        files.put(
-            src=get_resource("policy-rc.d"),
-            dest="/usr/sbin/policy-rc.d",
-            user="root",
-            group="root",
-            mode="755",
-        )
-
-        apt.packages(
-            name="Install unbound",
-            packages=["unbound", "unbound-anchor", "dnsutils"],
-        )
-
-        files.file("/usr/sbin/policy-rc.d", present=False)
+        # On an IPv4-only system, if unbound is started but not configured,
+        # it causes subsequent steps to fail to resolve hosts.
+        with blocked_service_startup():
+            apt.packages(
+                name="Install unbound",
+                packages=["unbound", "unbound-anchor", "dnsutils"],
+            )
 
     def configure(self):
         server.shell(
@@ -271,8 +254,14 @@ class WebsiteDeployer(Deployer):
                     logger.warning("Web page build failed, skipping website deployment")
                     return
             # if it is not a hugo page, upload it as is
-            files.rsync(
-                f"{www_path}/", "/var/www/html", flags=["-avz", "--chown=www-data"]
+            # pyinfra files.rsync (experimental) causes problems with ssh-config configuration
+            # the stable files.sync should do
+            files.sync(
+                src=str(www_path),
+                dest="/var/www/html",
+                user="www-data",
+                group="www-data",
+                delete=True,
             )
 
 
@@ -336,12 +325,12 @@ class TurnDeployer(Deployer):
     def install(self):
         (url, sha256sum) = {
             "x86_64": (
-                "https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-x86_64-linux",
-                "841e527c15fdc2940b0469e206188ea8f0af48533be12ecb8098520f813d41e4",
+                "https://github.com/chatmail/chatmail-turn/releases/download/v0.4/chatmail-turn-x86_64-linux",
+                "1ec1f5c50122165e858a5a91bcba9037a28aa8cb8b64b8db570aa457c6141a8a",
             ),
             "aarch64": (
-                "https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-aarch64-linux",
-                "a5fc2d06d937b56a34e098d2cd72a82d3e89967518d159bf246dc69b65e81b42",
+                "https://github.com/chatmail/chatmail-turn/releases/download/v0.4/chatmail-turn-aarch64-linux",
+                "0fb3e792419494e21ecad536464929dba706bb2c88884ed8f1788141d26fc756",
             ),
         }[host.get_fact(facts.server.Arch)]
 
@@ -532,17 +521,9 @@ class FcgiwrapDeployer(Deployer):
 
 class GithashDeployer(Deployer):
     def activate(self):
-        try:
-            git_hash = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode()
-        except Exception:
-            git_hash = "unknown\n"
-        try:
-            git_diff = subprocess.check_output(["git", "diff"]).decode()
-        except Exception:
-            git_diff = ""
         files.put(
             name="Upload chatmail relay git commit hash",
-            src=StringIO(git_hash + git_diff),
+            src=StringIO(get_version_string()),
             dest="/etc/chatmail-version",
             mode="700",
         )
@@ -586,11 +567,17 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
         )
 
     # Check if mtail_address interface is available (if configured)
-    if config.mtail_address and config.mtail_address not in ('127.0.0.1', '::1', 'localhost'):
+    if config.mtail_address and config.mtail_address not in (
+        "127.0.0.1",
+        "::1",
+        "localhost",
+    ):
         ipv4_addrs = host.get_fact(hardware.Ipv4Addrs)
         all_addresses = [addr for addrs in ipv4_addrs.values() for addr in addrs]
         if config.mtail_address not in all_addresses:
-            Out().red(f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n")
+            Out().red(
+                f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n"
+            )
             exit(1)
 
     if not os.environ.get("CHATMAIL_NOPORTCHECK"):

cmdeploy/src/cmdeploy/dns.py

@@ -1,11 +1,26 @@
 import datetime
-import importlib
-
-from jinja2 import Template
 
 from . import remote
 
 
+def parse_zone_records(text):
+    """Yield ``(name, ttl, rtype, rdata)`` from standard BIND-format text.
+
+    Skips comment lines (starting with ``;``) and blank lines.
+    Each record line must have the format ``name TTL IN type rdata``.
+    """
+    for raw_line in text.splitlines():
+        line = raw_line.strip()
+        if not line or line.startswith(";"):
+            continue
+        try:
+            name, ttl, _in, rtype, rdata = line.split(None, 4)
+        except ValueError:
+            raise ValueError(f"Bad zone record line: {line!r}") from None
+        name = name.rstrip(".")
+        yield name, ttl, rtype.upper(), rdata
+
+
 def get_initial_remote_data(sshexec, mail_domain):
     return sshexec.logged(
         call=remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=mail_domain)
@@ -31,13 +46,36 @@ def get_filled_zone_file(remote_data):
     if not sts_id:
         remote_data["sts_id"] = datetime.datetime.now().strftime("%Y%m%d%H%M")
 
-    template = importlib.resources.files(__package__).joinpath("chatmail.zone.j2")
-    content = template.read_text()
-    zonefile = Template(content).render(**remote_data)
-    lines = [x.strip() for x in zonefile.split("\n") if x.strip()]
+    d = remote_data["mail_domain"]
+    lines = ["; Required DNS entries"]
+    if remote_data.get("A"):
+        lines.append(f"{d}.  3600  IN  A  {remote_data['A']}")
+    if remote_data.get("AAAA"):
+        lines.append(f"{d}.  3600  IN  AAAA  {remote_data['AAAA']}")
+    lines.append(f"{d}.  3600  IN  MX  10 {d}.")
+    if remote_data.get("strict_tls"):
+        lines.append(
+            f'_mta-sts.{d}.  3600  IN  TXT  "v=STSv1; id={remote_data["sts_id"]}"'
+        )
+        lines.append(f"mta-sts.{d}.  3600  IN  CNAME  {d}.")
+    lines.append(f"www.{d}.  3600  IN  CNAME  {d}.")
+    lines.append(remote_data["dkim_entry"])
     lines.append("")
-    zonefile = "\n".join(lines)
-    return zonefile
+    lines.append("; Recommended DNS entries")
+    lines.append(f'{d}.  3600  IN  TXT  "v=spf1 a ~all"')
+    lines.append(f'_dmarc.{d}.  3600  IN  TXT  "v=DMARC1;p=reject;adkim=s;aspf=s"')
+    if remote_data.get("acme_account_url"):
+        lines.append(
+            f"{d}.  3600  IN  CAA  0 issue"
+            f' "letsencrypt.org;accounturi={remote_data["acme_account_url"]}"'
+        )
+    lines.append(f'_adsp._domainkey.{d}.  3600  IN  TXT  "dkim=discardable"')
+    lines.append(f"_submission._tcp.{d}.  3600  IN  SRV  0 1 587 {d}.")
+    lines.append(f"_submissions._tcp.{d}.  3600  IN  SRV  0 1 465 {d}.")
+    lines.append(f"_imap._tcp.{d}.  3600  IN  SRV  0 1 143 {d}.")
+    lines.append(f"_imaps._tcp.{d}.  3600  IN  SRV  0 1 993 {d}.")
+    lines.append("")
+    return "\n".join(lines)
 
 
 def check_full_zone(sshexec, remote_data, out, zonefile) -> int:
@@ -53,18 +91,19 @@ def check_full_zone(sshexec, remote_data, out, zonefile) -> int:
     if required_diff:
         out.red("Please set required DNS entries at your DNS provider:\n")
         for line in required_diff:
-            out(line)
-        out("")
+            out.print(line)
+        out.print()
         returncode = 1
     if remote_data.get("dkim_entry") in required_diff:
-        out(
-            "If the DKIM entry above does not work with your DNS provider, you can try this one:\n"
+        out.print(
+            "If the DKIM entry above does not work with your DNS provider,"
+            " you can try this one:\n"
         )
-        out(remote_data.get("web_dkim_entry") + "\n")
+        out.print(remote_data.get("web_dkim_entry") + "\n")
     if recommended_diff:
-        out("WARNING: these recommended DNS entries are not set:\n")
+        out.print("WARNING: these recommended DNS entries are not set:\n")
         for line in recommended_diff:
-            out(line)
+            out.print(line)
 
     if not (recommended_diff or required_diff):
         out.green("Great! All your DNS entries are verified and correct.")

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -1,15 +1,15 @@
-import os
 import urllib.request
 
 from chatmaild.config import Config
 from pyinfra import host
-from pyinfra.facts.server import Arch, Sysctl
+from pyinfra.facts.server import Arch, Command, Sysctl
 from pyinfra.facts.systemd import SystemdEnabled
 from pyinfra.operations import apt, files, server, systemd
 
 from cmdeploy.basedeploy import (
     Deployer,
     activate_remote_units,
+    blocked_service_startup,
     configure_remote_units,
     get_resource,
     has_systemd,
@@ -28,9 +28,11 @@ class DovecotDeployer(Deployer):
         arch = host.get_fact(Arch)
         if has_systemd() and "dovecot.service" in host.get_fact(SystemdEnabled):
             return  # already installed and running
-        _install_dovecot_package("core", arch)
-        _install_dovecot_package("imapd", arch)
-        _install_dovecot_package("lmtpd", arch)
+
+        with blocked_service_startup():
+            _install_dovecot_package("core", arch)
+            _install_dovecot_package("imapd", arch)
+            _install_dovecot_package("lmtpd", arch)
 
     def configure(self):
         configure_remote_units(self.config.mail_domain, self.units)
@@ -134,19 +136,25 @@ def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
 
     # as per https://doc.dovecot.org/2.3/configuration_manual/os/
     # it is recommended to set the following inotify limits
-    if not os.environ.get("CHATMAIL_NOSYSCTL"):
-        for name in ("max_user_instances", "max_user_watches"):
-            key = f"fs.inotify.{name}"
-            if host.get_fact(Sysctl)[key] > 65535:
-                # Skip updating limits if already sufficient
-                # (enables running in incus containers where sysctl readonly)
-                continue
-            server.sysctl(
-                name=f"Change {key}",
-                key=key,
-                value=65535,
-                persist=True,
+    can_modify = host.get_fact(Command, "systemd-detect-virt -c || true") == "none"
+    for name in ("max_user_instances", "max_user_watches"):
+        key = f"fs.inotify.{name}"
+        value = host.get_fact(Sysctl)[key]
+        if value > 65534:
+            continue
+        if not can_modify:
+            print(
+                "\n!!!! refusing to attempt sysctl setting in shared-kernel containers\n"
+                f"!!!! dovecot: sysctl {key!r}={value}, should be >65535 for production setups\n"
+                "!!!!"
             )
+            continue
+        server.sysctl(
+            name=f"Change {key}",
+            key=key,
+            value=65535,
+            persist=True,
+        )
 
     timezone_env = files.line(
         name="Set TZ environment variable",

cmdeploy/src/cmdeploy/lxc/cli.py

@@ -0,0 +1,454 @@
+"""lxc-start/stop/status/test subcommands for testing with local containers."""
+
+import os
+import time
+
+from ..util import get_git_hash, get_version_string, shell
+from .incus import RELAY_IMAGE_ALIAS, Incus, RelayContainer
+
+RELAY_NAMES = ("test0", "test1")
+
+
+# -------------------------------------------------------------------
+# lxc-start
+# -------------------------------------------------------------------
+
+
+def lxc_start_cmd_options(parser):
+    _add_name_args(
+        parser,
+        help_text="User relay name(s) to create (default: test0).",
+    )
+    parser.add_argument(
+        "--ipv4-only",
+        dest="ipv4_only",
+        action="store_true",
+        help="Create an IPv4-only container.",
+    )
+    parser.add_argument(
+        "--run",
+        action="store_true",
+        help="Run 'cmdeploy run' on each container after starting it.",
+    )
+
+
+def lxc_start_cmd(args, out):
+    """Create/Ensure and start LXC relay and DNS containers."""
+
+    with out.section("Preparing container setup"):
+        _lxc_start_cmd(args, out)
+
+
+def _lxc_start_cmd(args, out):
+    ix = Incus(out)
+    sub = out.new_prefixed_out()
+    out.green("Ensuring base image ...")
+    ix.ensure_base_image()
+    out.green("Ensuring DNS container (ns-localchat) ...")
+    dns_ct = ix.get_dns_container()
+    dns_ct.ensure()
+    sub.print(f"DNS container IP: {dns_ct.ipv4}")
+
+    names = args.names if args.names else RELAY_NAMES
+    relays = list(ix.get_container(n) for n in names)
+    for ct in relays:
+        out.green(f"Ensuring container {ct.name!r} ({ct.domain}) ...")
+        ct.ensure()
+        ip = ct.ipv4
+
+        sub.print("Configuring container hostname ...")
+        ct.configure_hosts(ip)
+
+        sub.print(f"Writing {ct.ini.name} ...")
+        ct.write_ini(disable_ipv6=args.ipv4_only)
+        sub.print(f"Config: {ct.ini}")
+        if args.ipv4_only:
+            ct.disable_ipv6()
+            ipv6 = None
+        else:
+            output = ct.bash(
+                "ip -6 addr show scope global -deprecated"
+                " | grep -oP '(?<=inet6 )[^/]+'",
+                check=False,
+            )
+            ipv6 = output.strip() if output else None
+        sub.print(f"{_format_addrs(ip, ipv6)}")
+
+        sub.green(f"Container {ct.name!r} ready: {ct.domain} -> {ip}")
+        out.print()
+
+    # Reset DNS zones only for the containers we just started
+    started_cnames = {ct.name for ct in relays}
+    managed = ix.list_managed()
+    started = [c for c in managed if c["name"] in started_cnames]
+
+    if started:
+        out.print(
+            f"Resetting DNS zones for {len(started)} domain(s) (A + AAAA records) ..."
+        )
+        dns_ct.reset_dns_records(dns_ct.ipv4, started)
+
+        for ct in relays:
+            if ct.name in started_cnames:
+                sub.print(f"Configuring DNS in {ct.name} ...")
+                ct.configure_dns(dns_ct.ipv4)
+
+    # Generate the unified SSH config
+    out.green("Writing ssh-config ...")
+    ssh_cfg = ix.write_ssh_config()
+    sub.print(f"{ssh_cfg}")
+
+    # Verify SSH via the generated config
+    for ct in relays:
+        sub.print(f"Verifying SSH to {ct.name} via ssh-config ...")
+        if ct.verify_ssh(ssh_cfg):
+            sub.print(f"SSH OK: ssh -F lxconfigs/ssh-config {ct.domain}")
+        else:
+            sub.red(f"WARNING: SSH verification failed for {ct.name}")
+
+    # Print integration suggestions
+    ssh_cfg = ix.ssh_config_path
+    if not ix.check_ssh_include():
+        sub.green(
+            "\n(Optional) To use containers from any SSH client, add to ~/.ssh/config:"
+        )
+        sub.green(f"    Include {ssh_cfg}")
+
+    # Optionally run cmdeploy run on each relay
+    if args.run:
+        for ct in relays:
+            with out.section(f"cmdeploy run: {ct.sname} ({ct.domain})"):
+                ret = _run_cmdeploy("run", ct, ix, out, extra=["--skip-dns-check"])
+                if ret:
+                    out.red(f"Deploy to {ct.sname} failed (exit {ret})")
+                    return ret
+
+
+# -------------------------------------------------------------------
+# lxc-stop
+# -------------------------------------------------------------------
+
+
+def lxc_stop_cmd_options(parser):
+    parser.add_argument(
+        "--destroy",
+        action="store_true",
+        help="Delete containers and their config files after stopping.",
+    )
+    parser.add_argument(
+        "--destroy-all",
+        dest="destroy_all",
+        action="store_true",
+        help="Like --destroy, but also remove the ns-localchat DNS container.",
+    )
+    _add_name_args(
+        parser,
+        help_text="Container name(s) to stop (default: test0 + test1).",
+    )
+
+
+def lxc_stop_cmd(args, out):
+    """Stop (and optionally destroy) local LXC relay containers."""
+    ix = Incus(out)
+    names = args.names or RELAY_NAMES
+    destroy = args.destroy or args.destroy_all
+
+    for ct in map(ix.get_container, names):
+        if destroy:
+            out.green(f"Destroying container {ct.name!r} ...")
+            ct.destroy()
+        else:
+            out.green(f"Stopping container {ct.name!r} ...")
+            ct.stop(force=True)
+
+    if args.destroy_all:
+        dns_ct = ix.get_dns_container()
+        out.green(f"Destroying DNS container {dns_ct.name!r} ...")
+        dns_ct.destroy()
+        ix.delete_images()
+
+    if destroy:
+        ix.write_ssh_config()
+        out.green("LXC containers destroyed.")
+    else:
+        out.green("LXC containers stopped.")
+
+
+# -------------------------------------------------------------------
+# lxc-test
+# -------------------------------------------------------------------
+
+
+def lxc_test_cmd_options(parser):
+    parser.add_argument(
+        "--one",
+        action="store_true",
+        help="Only deploy and test against test0 (skip test1).",
+    )
+
+
+def lxc_test_cmd(args, out):
+    """Run full LXC pipeline: start, deploy, DNS, zone files, and tests.
+
+    All commands run directly on the host using
+    ``--ssh-config lxconfigs/ssh-config`` for SSH access.
+    """
+    ix = Incus(out)
+    t_total = time.time()
+    relay_names = list(RELAY_NAMES)
+    if args.one:
+        relay_names = relay_names[:1]
+
+    local_hash = get_git_hash()
+
+    # Per-relay: start, deploy, then snapshot the first relay as a
+    # reusable image so the second relay launches pre-deployed.
+    ipv4_only_flags = {RELAY_NAMES[0]: False, RELAY_NAMES[1]: True}
+
+    for ct in map(ix.get_container, relay_names):
+        name = ct.sname
+        ipv4_only = ipv4_only_flags.get(name, False)
+        v_flag = " -" + "v" * out.verbosity if out.verbosity > 0 else ""
+        start_cmd = f"cmdeploy lxc-start{v_flag} {name}"
+        if ipv4_only:
+            start_cmd += " --ipv4-only"
+        with out.section(f"cmdeploy lxc-start: {name}"):
+            ret = out.shell(start_cmd, cwd=str(ix.project_root))
+            if ret:
+                return ret
+
+        status = _deploy_status(ct, local_hash, ix)
+        with out.section(f"cmdeploy run: {name}"):
+            if "IN-SYNC" in status:
+                out.print(f"{name} is {status}, skipping")
+            else:
+                ret = _run_cmdeploy("run", ct, ix, out, extra=["--skip-dns-check"])
+                if ret:
+                    out.red(f"Deploy to {name} failed (exit {ret})")
+                    return ret
+
+        # Snapshot the first relay so subsequent ones launch pre-deployed
+        if not ix.find_image([RELAY_IMAGE_ALIAS]):
+            with out.section("lxc-test: caching relay image"):
+                ct.publish_as_relay_image()
+
+    for ct in map(ix.get_container, relay_names):
+        with out.section(f"cmdeploy dns: {ct.sname} ({ct.domain})"):
+            ret = _run_cmdeploy("dns", ct, ix, out, extra=["--zonefile", str(ct.zone)])
+            if ret:
+                out.red(f"DNS for {ct.sname} failed (exit {ret})")
+                return ret
+
+    with out.section(f"lxc-test: loading DNS zones {' & '.join(relay_names)}"):
+        dns_ct = ix.get_dns_container()
+        for ct in map(ix.get_container, relay_names):
+            if ct.zone.exists():
+                zone_data = ct.zone.read_text()
+                out.print(f"Loading {ct.zone} into PowerDNS ...")
+                dns_ct.set_dns_records(zone_data)
+
+    with out.section("cmdeploy test"):
+        first = ix.get_container(relay_names[0])
+        env = None
+        if len(relay_names) > 1:
+            env = os.environ.copy()
+            env["CHATMAIL_DOMAIN2"] = ix.get_container(relay_names[1]).domain
+        ret = _run_cmdeploy("test", first, ix, out, **({"env": env} if env else {}))
+        if ret:
+            out.red(f"Tests failed (exit {ret})")
+            return ret
+
+    elapsed = time.time() - t_total
+    out.section_line(f"lxc-test complete ({elapsed:.1f}s)")
+    if out.section_timings:
+        out.print("Section timings:")
+        for name, secs in out.section_timings:
+            out.print(f"  {name:.<50s} {secs:5.1f}s")
+        out.print(f"  {'total':.<50s} {elapsed:5.1f}s")
+    out.section_timings.clear()
+    return 0
+
+
+# -------------------------------------------------------------------
+# lxc-status
+# -------------------------------------------------------------------
+
+
+def lxc_status_cmd_options(parser):
+    pass
+
+
+def lxc_status_cmd(args, out):
+    """Show status of local LXC chatmail containers."""
+    ix = Incus(out)
+    containers = ix.list_managed()
+    if not containers:
+        out.red("No LXC containers found.  Run 'cmdeploy lxc-start' first.")
+        return 1
+
+    local_hash = get_git_hash()
+
+    # Get storage pool path for display
+    storage_path = None
+    data = ix.run_json(["storage", "show", "default"], check=False)
+    if data:
+        storage_path = data.get("config", {}).get("source")
+    msg = "Container status"
+    if storage_path:
+        msg += f": {storage_path}"
+    out.section_line(msg)
+
+    dns_ip = None
+    for c in containers:
+        _print_container_status(out, c, ix, local_hash)
+        if c["name"] == ix.get_dns_container().name:
+            dns_ip = c["ip"]
+
+    out.section_line("Host ssh and DNS configuration")
+    _print_ssh_status(out, ix)
+    _print_dns_forwarding_status(out, dns_ip)
+    return 0
+
+
+def _print_container_status(out, c, ix, local_hash):
+    """Print name/status, domain/IPs, and RAM for one container."""
+    cname = c["name"]
+    is_running = c.get("status") == "Running"
+    ct = ix.get_container(cname)
+
+    # First line: name + running/STOPPED + deploy status
+    if not is_running:
+        tag = "STOPPED"
+    elif not isinstance(ct, RelayContainer):
+        tag = "running"
+    else:
+        tag = f"running {_deploy_status(ct, local_hash, ix)}"
+    out.print(f"{cname:20s} {tag}")
+
+    # Second line: domain, IPv4, IPv6
+    domain = c.get("domain", "")
+    ip = c.get("ip") or "?"
+    ipv6 = c.get("ipv6")
+    out.print(f"{domain:20s} {_format_addrs(ip, ipv6)}")
+
+    # Third line: RAM (RSS), config
+    detail_out = out.new_prefixed_out(" " * 21)
+    try:
+        used, total = ct.rss_mib()
+    except Exception:
+        ram_str = "RSS ?"
+    else:
+        ram_str = f"RSS {used}/{total} MiB ({used * 100 // total}%)"
+
+    if isinstance(ct, RelayContainer):
+        detail = f"{ram_str}, config: {os.path.relpath(ct.ini)}"
+    else:
+        detail = ram_str
+
+    detail_out.print(detail)
+    out.print()
+
+
+def _print_ssh_status(out, ix):
+    """Print SSH integration status."""
+    ssh_cfg = ix.ssh_config_path
+    if ix.check_ssh_include():
+        out.green("SSH: ~/.ssh/config includes lxconfigs/ssh-config ✓")
+    else:
+        out.red("SSH: ~/.ssh/config does NOT include lxconfigs/ssh-config")
+        sub = out.new_prefixed_out()
+        sub.print("Add to ~/.ssh/config:")
+        sub.print(f"    Include {ssh_cfg}")
+
+
+def _print_dns_forwarding_status(out, dns_ip):
+    """Print host DNS forwarding status for .localchat."""
+    sub = out.new_prefixed_out()
+    if not dns_ip:
+        out.red("DNS: ns-localchat container not found")
+        return
+    try:
+        rv = shell("resolvectl status incusbr0")
+        dns_ok = dns_ip in rv.stdout and "localchat" in rv.stdout
+    except Exception:
+        dns_ok = None
+    if dns_ok is True:
+        out.green(f"DNS: .localchat forwarding to {dns_ip} ✓")
+    elif dns_ok is False:
+        out.red("DNS: .localchat forwarding NOT configured")
+        sub.print("Run:")
+        sub.print(f"    sudo resolvectl dns incusbr0 {dns_ip}")
+        sub.print("    sudo resolvectl domain incusbr0 ~localchat")
+    else:
+        sub.print("DNS: .localchat forwarding status UNKNOWN")
+
+
+# -------------------------------------------------------------------
+# Internal helpers
+# -------------------------------------------------------------------
+
+
+def _format_addrs(ip, ipv6=None):
+    parts = [f"IPv4 {ip}"]
+    if ipv6:
+        parts.append(f"IPv6 {ipv6}")
+    return ", ".join(parts)
+
+
+def _deploy_status(ct, local_hash, ix):
+    """Return a human-readable deploy status string.
+
+    Compares the full deployed version (hash + diff) against
+    the local state built by :func:`~cmdeploy.util.get_version_string`.
+    """
+    deployed = ct.deployed_version()
+    if deployed is None:
+        return "NOT DEPLOYED"
+
+    # A container launched from the relay image has the same
+    # git hash but a different domain — always redeploy.
+    deployed_domain = ct.deployed_domain()
+    if deployed_domain and deployed_domain != ct.domain:
+        return f"DOMAIN-MISMATCH (deployed: {deployed_domain})"
+
+    deployed_lines = deployed.splitlines()
+    deployed_hash = deployed_lines[0] if deployed_lines else ""
+    short = deployed_hash[:12]
+
+    if not local_hash:
+        return f"UNKNOWN (deployed: {short})"
+
+    local_short = local_hash[:12]
+    if deployed_hash != local_hash:
+        return f"STALE (deployed: {short}, local: {local_short})"
+
+    # Hash matches — check for uncommitted diffs
+    local_version = get_version_string()
+    if deployed != local_version:
+        return f"DIRTY ({local_short}, undeployed changes)"
+
+    return f"IN-SYNC ({short})"
+
+
+def _add_name_args(parser, help_text):
+    parser.add_argument("names", nargs="*", metavar="NAME", help=help_text)
+
+
+def _run_cmdeploy(subcmd, ct, ix, out, extra=None, **kwargs):
+    """Run ``cmdeploy <subcmd>`` with standard --config/--ssh flags.
+
+    *ct* is a Container (uses ``ct.ini`` and ``ct.domain``).
+    Returns the subprocess exit code.
+    """
+    extra_str = " ".join(extra) if extra else ""
+    v_flag = " -" + "v" * out.verbosity if out.verbosity > 0 else ""
+    cmd = f"""
+        cmdeploy {subcmd}{v_flag}
+        --config {ct.ini}
+        --ssh-config {ix.ssh_config_path}
+        --ssh-host {ct.domain}
+        {extra_str}
+    """
+    if "cwd" not in kwargs:
+        kwargs["cwd"] = str(ix.project_root)
+    return out.shell(cmd, **kwargs)

cmdeploy/src/cmdeploy/lxc/incus.py

@@ -0,0 +1,739 @@
+"""Core Incus operations for local chatmail LXC containers."""
+
+import json
+import subprocess
+import textwrap
+import time
+from pathlib import Path
+
+from ..util import shell
+
+LABEL_KEY = "user.localchat-managed"
+SSH_KEY_NAME = "id_localchat"
+DOMAIN_SUFFIX = ".localchat"
+UPSTREAM_IMAGE = "images:debian/12"
+BASE_IMAGE_ALIAS = "localchat-base"
+BASE_SETUP_NAME = "localchat-base-setup"
+RELAY_IMAGE_ALIAS = "localchat-relay"
+
+DNS_CONTAINER_NAME = "ns-localchat"
+DNS_DOMAIN = "ns.localchat"
+
+
+class DNSConfigurationError(Exception):
+    """Raised when the DNS container is not reachable or not answering."""
+
+
+def _extract_ip(net_data, family="inet"):
+    """Extract the first global-scope IP of *family* from network state data.
+
+    *net_data* is the ``state.network`` dict from ``incus list --format=json``.
+    *family* is ``"inet"`` for IPv4 or ``"inet6"`` for IPv6.
+    Returns the address string, or None.
+    """
+    for iface_name, iface in net_data.items():
+        if iface_name == "lo":
+            continue
+        for addr in iface.get("addresses", []):
+            if addr["family"] == family and addr["scope"] == "global":
+                return addr["address"]
+    return None
+
+
+class Incus:
+    """Gateway for all Incus container operations.
+
+    Instantiated once per CLI command and passed around so that
+    all modules share a single entry point for Incus interactions.
+    """
+
+    def __init__(self, out):
+        self.out = out
+        self.project_root = Path(__file__).resolve().parent.parent.parent.parent.parent
+        self.lxconfigs_dir = self.project_root / "lxconfigs"
+        self.lxconfigs_dir.mkdir(exist_ok=True)
+        self.ssh_key_path = self.lxconfigs_dir / SSH_KEY_NAME
+        if not self.ssh_key_path.exists():
+            shell(
+                f"ssh-keygen -t ed25519 -f {self.ssh_key_path} -N '' -C localchat",
+                check=True,
+            )
+        self.ssh_config_path = self.lxconfigs_dir / "ssh-config"
+
+    def write_ssh_config(self):
+        """Write ``lxconfigs/ssh-config`` mapping all containers to their IPs.
+
+        Each Host block maps the container name, the domain name, and the
+        short relay name (e.g. ``_test0``) to the container's IP, using the
+        shared localchat SSH key.  Returns the path to the file.
+        """
+        containers = self.list_managed()
+        key_path = self.ssh_key_path
+        lines = ["# Auto-generated by cmdeploy lxc-start — do not edit\n"]
+        for c in containers:
+            hosts = [c["name"]]
+            domain = c.get("domain", "")
+            if domain and domain != c["name"]:
+                hosts.append(domain)
+                short = domain.split(".")[0]
+                if short and short not in hosts:
+                    hosts.append(short)
+            lines.append(f"\nHost {' '.join(hosts)}\n")
+            lines.append(f"    Hostname {c['ip']}\n")
+            lines.append("    User root\n")
+            lines.append(f"    IdentityFile {key_path}\n")
+            lines.append("    IdentitiesOnly yes\n")
+            lines.append("    StrictHostKeyChecking accept-new\n")
+            lines.append("    UserKnownHostsFile /dev/null\n")
+            lines.append("    LogLevel ERROR\n")
+        path = self.ssh_config_path
+        path.write_text("".join(lines))
+        return path
+
+    def check_ssh_include(self):
+        """Check if the user's ~/.ssh/config already includes our ssh-config."""
+        user_ssh_config = Path.home() / ".ssh" / "config"
+        if not user_ssh_config.exists():
+            return False
+        lines = user_ssh_config.read_text().splitlines()
+        target = f"include {self.ssh_config_path}".lower()
+        return any(line.strip().lower() == target for line in lines)
+
+    def run(self, args, check=True, capture=True, input=None):
+        """Run an incus command.
+
+        When *capture* is True and *verbosity* >= 1, output is streamed
+        to the terminal line-by-line while also being captured for
+        later return via result.stdout.
+        """
+        cmd = ["incus"] + list(args)
+        sub = self.out.new_prefixed_out("  ")
+
+        if not capture:
+            # Simple case: let subprocess handle streams (no capture)
+            if self.out.verbosity >= 1:
+                sub.print(f"$ {' '.join(cmd)}")
+            return subprocess.run(
+                cmd, text=True, input=input, check=check, stdout=None, stderr=None
+            )
+
+        # Capture case: we may need to stream while capturing
+        if sub.verbosity >= 1:
+            cmd_lines = " ".join(cmd).splitlines()
+            sub.print(f"$ {cmd_lines.pop(0)}")
+            if sub.verbosity >= 2:
+                for line in cmd_lines:
+                    sub.print(f"  {line}")
+
+        proc = subprocess.Popen(
+            cmd,
+            text=True,
+            stdin=subprocess.PIPE if input else None,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+        )
+
+        stdout_lines = []
+        if input:
+            proc.stdin.write(input)
+            proc.stdin.close()
+
+        for line in proc.stdout:
+            stdout_lines.append(line)
+            if sub.verbosity >= 2:
+                sub.print(f"  > {line.rstrip()}")
+
+        ret = proc.wait()
+        stdout = "".join(stdout_lines)
+        if check and ret != 0:
+            for line in stdout.splitlines():
+                if sub.verbosity < 1:  # and we haven't printed it yet
+                    sub.red(line)
+            raise subprocess.CalledProcessError(ret, cmd, output=stdout)
+
+        return subprocess.CompletedProcess(cmd, ret, stdout=stdout)
+
+    def run_json(self, args, check=True):
+        """Run an incus command with ``--format=json``.
+
+        Returns the parsed JSON on success.
+        When *check* is True raises ``subprocess.CalledProcessError``
+        on non-zero exit; when False returns *None* instead.
+        """
+        result = self.run(
+            list(args) + ["--format=json"],
+            check=check,
+        )
+        if result.returncode != 0:
+            return None
+        return json.loads(result.stdout)
+
+    def run_output(self, args, check=True):
+        """Run an incus command and return its stripped stdout.
+
+        When *check* is False, returns *None* on non-zero exit
+        instead of raising.
+        """
+        result = self.run(args, check=check)
+        if result.returncode != 0:
+            return None
+        return result.stdout.strip()
+
+    def find_image(self, aliases):
+        """Return the first alias from *aliases* that exists, else None."""
+        images = self.run_json(["image", "list"], check=False) or []
+        existing = {a.get("name") for img in images for a in img.get("aliases", [])}
+        for alias in aliases:
+            if alias in existing:
+                return alias
+        return None
+
+    def delete_images(self):
+        """Delete the cached base and relay images."""
+        for alias in (RELAY_IMAGE_ALIAS, BASE_IMAGE_ALIAS):
+            self.run(["image", "delete", alias], check=False)  # ok if absent
+
+    def list_managed(self):
+        """Return list of dicts with name, ip, ipv6, domain, status, memory_usage."""
+        containers = []
+        for ct in self.run_json(["list"]):
+            config = ct.get("config", {})
+            if config.get(LABEL_KEY) != "true":
+                continue
+            name = ct["name"]
+            state = ct.get("state", {})
+            net = state.get("network") or {}
+            containers.append(
+                {
+                    "name": name,
+                    "ip": _extract_ip(net, "inet"),
+                    "ipv6": _extract_ip(net, "inet6"),
+                    "domain": config.get(
+                        "user.localchat-domain", f"{name}{DOMAIN_SUFFIX}"
+                    ),
+                    "status": ct.get("status", "Unknown"),
+                    "memory_usage": state.get("memory", {}).get("usage", 0),
+                }
+            )
+        return containers
+
+    def ensure_base_image(self):
+        """Build and cache a base image with openssh and the SSH key.
+
+        The image is published as a local incus image with alias
+        'localchat-base'.  Subsequent container launches use this
+        image instead of the upstream Debian 12, skipping the
+        slow apt-get install step.
+        Returns the image alias.
+        """
+        if self.find_image([BASE_IMAGE_ALIAS]):
+            self.out.print(f"  Base image '{BASE_IMAGE_ALIAS}' already cached.")
+            return BASE_IMAGE_ALIAS
+
+        self.out.print("  Building base image (one-time setup) ...")
+
+        self.run(["delete", BASE_SETUP_NAME, "--force"], check=False)
+        self.run(["image", "delete", BASE_IMAGE_ALIAS], check=False)
+        self.run(["launch", UPSTREAM_IMAGE, BASE_SETUP_NAME])
+
+        ct = Container(self, BASE_SETUP_NAME)
+        ct.wait_ready()
+
+        key_path = self.ssh_key_path
+        pub_key = key_path.with_suffix(".pub").read_text().strip()
+        ct.bash(f"""
+            echo 'nameserver 9.9.9.9' > /etc/resolv.conf
+            apt-get -o DPkg::Lock::Timeout=60 update
+            DEBIAN_FRONTEND=noninteractive apt-get install -y openssh-server python3
+            systemctl enable ssh
+            apt-get clean
+            mkdir -p /root/.ssh
+            chmod 700 /root/.ssh
+            echo '{pub_key}' > /root/.ssh/authorized_keys
+            chmod 600 /root/.ssh/authorized_keys
+        """)
+
+        self.run(["stop", BASE_SETUP_NAME])
+        self.run(["publish", BASE_SETUP_NAME, f"--alias={BASE_IMAGE_ALIAS}"])
+        self.run(["delete", BASE_SETUP_NAME, "--force"])
+        self.out.print(f"  Base image '{BASE_IMAGE_ALIAS}' ready.")
+        return BASE_IMAGE_ALIAS
+
+    def get_container(self, name):
+        """Return a container handle for the given name.
+
+        Accepts both short relay names (``test0``) and full Incus
+        container names (``test0-localchat``).  Returns
+        ``DNSContainer`` for the DNS container and
+        ``RelayContainer`` for everything else.
+        """
+        if name == DNS_CONTAINER_NAME:
+            return DNSContainer(self)
+        return RelayContainer(self, name.removesuffix("-localchat"))
+
+    def get_dns_container(self):
+        """Return a DNSContainer handle."""
+        return DNSContainer(self)
+
+
+class Container:
+    """The base container handle wraps all interactions with incus."""
+
+    def __init__(self, incus, name, domain=None, memory="200MiB"):
+        self.incus = incus
+        self.out = incus.out
+        self.name = name
+        self.domain = domain or f"{name}{DOMAIN_SUFFIX}"
+        self.memory = memory
+        self.ipv4 = None
+        self.ipv6 = None
+
+    def bash(self, script, check=True):
+        """Returns stdout from executing ``bash -ec <script>`` inside this container.
+
+        *script* is dedented and stripped so callers can use triple-quoted strings.
+        When *check* is False, returns *None* on non-zero exit instead of raising.
+        """
+        script = textwrap.dedent(script).strip()
+        cmd = ["exec", self.name, "--", "bash", "-ec", script]
+        return self.incus.run_output(cmd, check=check)
+
+    def run_cmd(self, *args, check=True):
+        """Return stdout from running a command directly in the container (no shell).
+
+        When *check* is False, returns *None* on non-zero exit instead of raising.
+        """
+        return self.incus.run_output(
+            ["exec", self.name, "--", *args],
+            check=check,
+        )
+
+    def start(self):
+        self.incus.run(["start", self.name])
+
+    def stop(self, force=False):
+        cmd = ["stop", self.name]
+        if force:
+            cmd.append("--force")
+        self.incus.run(cmd, check=False)
+
+    def launch(self):
+        """Launch from the best available image, return the alias used."""
+        image = self.incus.find_image([RELAY_IMAGE_ALIAS, BASE_IMAGE_ALIAS])
+        if not image:
+            raise RuntimeError(
+                f"No base image '{BASE_IMAGE_ALIAS}' found. "
+                "Call ensure_base_image() before launching containers."
+            )
+        self.out.print(f"  Launching from '{image}' image ...")
+        cfg = []
+        cfg += ("-c", f"{LABEL_KEY}=true")
+        cfg += ("-c", f"user.localchat-domain={self.domain}")
+        cfg += ("-c", f"limits.memory={self.memory}")
+        self.incus.run(["launch", image, self.name, *cfg])
+        return image
+
+    def ensure(self):
+        """Create/start this container from the cached base image.
+
+        On first call, builds the base image (~30s).
+        Subsequent containers launch in ~2s from the cached image.
+        Returns ``self`` for chaining.
+        """
+        data = self.incus.run_json(["list", self.name], check=False) or []
+
+        existing = [c for c in data if c["name"] == self.name]
+        if existing:
+            if existing[0]["status"] != "Running":
+                self.start()
+        else:
+            self.launch()
+        self.wait_ready()
+        return self
+
+    def destroy(self):
+        """Stop, delete, and clean up config files."""
+        self.stop(force=True)
+        self.incus.run(["delete", self.name, "--force"], check=False)
+
+    def push_file_content(self, dest_path, content):
+        """Write *content* to *dest_path* inside the container.
+
+        *content* is dedented and stripped so callers can use
+        indented triple-quoted strings.
+        """
+        content = textwrap.dedent(content).strip() + "\n"
+        self.incus.run(
+            ["file", "push", "-", f"{self.name}{dest_path}"],
+            input=content,
+        )
+        self.bash(f"chmod 644 {dest_path}")
+
+    def wait_ready(self, timeout=60):
+        """Wait until the container is running with an IPv4 address.
+
+        Sets ``self.ipv4`` and ``self.ipv6`` (may be *None*),
+        or raises ``TimeoutError``.
+        """
+        deadline = time.time() + timeout
+        while time.time() < deadline:
+            data = self.incus.run_json(
+                ["list", self.name],
+                check=False,
+            )
+            if data and data[0].get("status") == "Running":
+                net = data[0].get("state", {}).get("network", {})
+                self.ipv4 = _extract_ip(net, "inet")
+                self.ipv6 = _extract_ip(net, "inet6")
+                if self.ipv4:
+                    return
+            time.sleep(1)
+        raise TimeoutError(
+            f"Container {self.name!r} did not become ready within {timeout}s"
+        )
+
+    def rss_mib(self):
+        """Return ``(used, total)`` memory from container (or None if unobtainable)."""
+        output = self.bash("free -m", check=False)
+        if output:
+            for line in output.splitlines():
+                if line.startswith("Mem:"):
+                    parts = line.split()
+                    return int(parts[2]), int(parts[1])
+
+
+class RelayContainer(Container):
+    """Container handle for a chatmail relay.
+
+    Accepts the short relay name (e.g. ``test0``) and derives
+    the Incus container name and mail domain automatically.
+    """
+
+    def __init__(self, incus, name):
+        super().__init__(
+            incus,
+            f"{name}-localchat",
+            domain=f"_{name}{DOMAIN_SUFFIX}",
+            memory="600MiB",
+        )
+        self.sname = name
+        self.ini = incus.lxconfigs_dir / f"chatmail-{name}.ini"
+        self.zone = incus.lxconfigs_dir / f"{name}.zone"
+
+    def launch(self):
+        """Launch (from a potentially cached image) and clear inherited chatmail-version."""
+        image = super().launch()
+        self.bash("rm -f /etc/chatmail-version")
+        return image
+
+    def destroy(self):
+        """Stop, delete, and clean up config files."""
+        super().destroy()
+        if self.ini.exists():
+            self.ini.unlink()
+
+    def disable_ipv6(self):
+        """Disable IPv6 inside the container via sysctl."""
+        # incus provides net.* virtualization for LXC containers so that
+        # these sysctls only affect the container's network namespace.
+        self.bash("""
+            sysctl -w net.ipv6.conf.all.disable_ipv6=1
+            sysctl -w net.ipv6.conf.default.disable_ipv6=1
+            mkdir -p /etc/sysctl.d
+            printf 'net.ipv6.conf.all.disable_ipv6=1\\n
+            net.ipv6.conf.default.disable_ipv6=1\\n'
+            > /etc/sysctl.d/99-disable-ipv6.conf
+        """)
+
+    def configure_hosts(self, ip):
+        """Set hostname and /etc/hosts inside the container."""
+        self.bash(f"""
+            echo '{self.name}' > /etc/hostname
+            hostname {self.name}
+            sed -i '/ {self.domain}$/d' /etc/hosts
+            echo '{ip} {self.name} {self.domain}' >> /etc/hosts
+        """)
+
+    def publish_as_relay_image(self):
+        """Publish this container as a reusable relay image.
+
+        Stops the container, 'publishes' it as 'localchat-relay', then restarts it.
+        """
+        if self.incus.find_image([RELAY_IMAGE_ALIAS]):
+            return
+        self.out.print(
+            f"  Locally caching {self.name!r} as '{RELAY_IMAGE_ALIAS}' image ..."
+        )
+        self.incus.run(
+            ["publish", self.name, f"--alias={RELAY_IMAGE_ALIAS}", "--force"]
+        )
+        self.wait_ready()
+        self.out.print(f"  Relay image '{RELAY_IMAGE_ALIAS}' ready.")
+
+    def deployed_version(self):
+        """Read /etc/chatmail-version, or None if absent."""
+        return self.bash("cat /etc/chatmail-version", check=False)
+
+    def deployed_domain(self):
+        """Read the domain deployed on the container (postfix myhostname)."""
+        return self.bash(
+            "postconf -h myhostname 2>/dev/null",
+            check=False,
+        )
+
+    def verify_ssh(self, ssh_config):
+        """Verify SSH connectivity to this container."""
+        cmd = f"ssh -F {ssh_config} -o ConnectTimeout=10 root@{self.domain} hostname"
+        return shell(cmd, timeout=15).returncode == 0
+
+    def configure_dns(self, dns_ip):
+        """Point this container's resolver at *dns_ip* and verify DNS is reachable."""
+        self.bash(f"""
+            systemctl disable --now systemd-resolved 2>/dev/null || true
+            rm -f /etc/resolv.conf
+            echo 'nameserver {dns_ip}' > /etc/resolv.conf
+            mkdir -p /etc/unbound/unbound.conf.d
+            printf 'server:\\n  domain-insecure: "localchat"\\n\\n
+            forward-zone:\\n  name: "localchat"\\n
+              forward-addr: {dns_ip}\\n'
+            > /etc/unbound/unbound.conf.d/localchat-forward.conf
+            systemctl restart unbound 2>/dev/null || true
+        """)
+        self._wait_dns_reachable(dns_ip)
+
+    def _wait_dns_reachable(self, dns_ip, timeout=10):
+        """Poll until *dns_ip* answers a DNS query from this container."""
+        if self.bash("which dig", check=False) is None:
+            self.bash(
+                "DEBIAN_FRONTEND=noninteractive "
+                "apt-get install -y dnsutils 2>/dev/null || true"
+            )
+        deadline = time.time() + timeout
+        while time.time() < deadline:
+            result = self.bash(
+                f"dig @{dns_ip} . SOA +short +time=1 +tries=1",
+                check=False,
+            )
+            if result and result.strip():
+                return
+            time.sleep(0.5)
+        raise DNSConfigurationError(
+            f"DNS at {dns_ip} not reachable from {self.name} after {timeout}s"
+        )
+
+    def write_ini(self, disable_ipv6=False):
+        """Generate a chatmail.ini config file in lxconfigs/."""
+        from chatmaild.config import write_initial_config
+
+        overrides = {
+            "max_user_send_per_minute": 600,
+            "max_user_send_burst_size": 100,
+            "mtail_address": "127.0.0.1",
+        }
+        if disable_ipv6:
+            overrides["disable_ipv6"] = "True"
+        write_initial_config(self.ini, self.domain, overrides)
+        return self.ini
+
+
+class DNSContainer(Container):
+    """Container handle for the PowerDNS name server.
+
+    Manages the authoritative and recursive DNS services required for
+    name resolution in the local testing environment.
+    """
+
+    def __init__(self, incus):
+        super().__init__(incus, DNS_CONTAINER_NAME, domain=DNS_DOMAIN)
+
+    def pdnsutil(self, *args, check=True):
+        """Run ``pdnsutil <args>`` inside the DNS container."""
+        return self.run_cmd("pdnsutil", *args, check=check)
+
+    def replace_rrset(self, zone, name, rtype, ttl, rdata):
+        """Shortcut for ``pdnsutil replace-rrset``."""
+        self.pdnsutil("replace-rrset", zone, name, rtype, ttl, rdata)
+
+    def restart_services(self):
+        """Restart pdns and pdns-recursor, then wait until DNS is answering."""
+        self.bash("""
+            systemctl restart pdns
+            systemctl restart pdns-recursor || true
+        """)
+        self._wait_dns_ready()
+
+    def _wait_dns_ready(self, timeout=10):
+        """Poll until the recursor answers a query on port 53."""
+        deadline = time.time() + timeout
+        while time.time() < deadline:
+            result = self.bash(
+                "dig @127.0.0.1 . SOA +short +time=1 +tries=1",
+                check=False,
+            )
+            if result and result.strip():
+                return
+            time.sleep(0.5)
+        raise DNSConfigurationError(f"DNS recursor not answering after {timeout}s")
+
+    def ensure(self):
+        """Create the DNS container with PowerDNS if needed.
+
+        Calls ``super().ensure()`` to create/start the container
+        and set up SSH, then installs PowerDNS and configures
+        the Incus bridge to use this container as DNS.
+        """
+        super().ensure()
+        self._install_powerdns()
+        self.incus.run(
+            ["network", "set", "incusbr0", "dns.mode=none"],
+            check=False,
+        )
+        self.incus.run(
+            ["network", "set", "incusbr0", f"raw.dnsmasq=dhcp-option=6,{self.ipv4}"],
+            check=False,
+        )
+
+    def destroy(self):
+        """Stop, delete, and reset bridge DNS config."""
+        super().destroy()
+        self.incus.run(["network", "unset", "incusbr0", "dns.mode"], check=False)
+        self.incus.run(["network", "unset", "incusbr0", "raw.dnsmasq"], check=False)
+
+    def _install_powerdns(self):
+        """Install and configure PowerDNS if not already present."""
+        if self.run_cmd("which", "pdns_server", check=False) is not None:
+            return
+
+        self.bash("""
+            systemctl disable --now systemd-resolved 2>/dev/null || true
+            rm -f /etc/resolv.conf
+            echo 'nameserver 9.9.9.9' > /etc/resolv.conf
+
+            # Block automatic service startup during package installation
+            printf '#!/bin/sh\\nexit 101\\n' > /usr/sbin/policy-rc.d
+            chmod +x /usr/sbin/policy-rc.d
+
+            apt-get -o DPkg::Lock::Timeout=60 update
+            DEBIAN_FRONTEND=noninteractive apt-get install -y \
+                pdns-server pdns-backend-sqlite3 sqlite3 pdns-recursor dnsutils
+
+            # Remove the startup block
+            rm /usr/sbin/policy-rc.d
+
+            systemctl stop pdns pdns-recursor || true
+            mkdir -p /var/lib/powerdns
+            sqlite3 /var/lib/powerdns/pdns.sqlite3 \
+                </usr/share/doc/pdns-backend-sqlite3/schema.sqlite3.sql
+            chown -R pdns:pdns /var/lib/powerdns
+        """)
+
+        self.push_file_content(
+            "/etc/powerdns/pdns.conf",
+            """
+            launch=gsqlite3
+            gsqlite3-database=/var/lib/powerdns/pdns.sqlite3
+            local-address=127.0.0.1
+            local-port=5353
+            """,
+        )
+
+        self.push_file_content(
+            "/etc/powerdns/recursor.conf",
+            """
+            local-address=0.0.0.0
+            local-port=53
+            forward-zones=localchat=127.0.0.1:5353
+            forward-zones-recurse=.=9.9.9.9;149.112.112.112
+            allow-from=0.0.0.0/0
+            dont-query=
+            dnssec=off
+            """,
+        )
+
+        self.bash("""
+            systemctl start pdns
+            systemctl start pdns-recursor
+            echo 'nameserver 127.0.0.1' > /etc/resolv.conf
+        """)
+        self._wait_dns_ready()
+
+    def reset_dns_records(self, dns_ip, domains):
+        """Create DNS zones with initial A records via pdnsutil.
+
+        Only sets SOA, NS, and A records as the minimal set
+        needed for SSH connectivity.  Full records (MX, TXT, SRV,
+        CNAME, DKIM) are added later by ``cmdeploy dns``.
+
+        Args:
+            dns_ip: IP of the DNS container
+            domains: list of dicts with 'name', 'domain', 'ip'
+        """
+        for d in domains:
+            domain = d["domain"]
+            ip = d["ip"]
+            self.out.print(f"  {domain} -> {ip}")
+
+            # Delete and recreate zone fresh (removes stale records)
+            self.pdnsutil("delete-zone", domain, check=False)
+            self.pdnsutil("create-zone", domain, f"ns.{domain}")
+
+            serial = str(int(time.time()))
+            soa = f"ns.{domain} hostmaster.{domain} {serial} 3600 900 604800 300"
+            self.replace_rrset(domain, ".", "SOA", "3600", soa)
+            self.replace_rrset(domain, ".", "NS", "3600", f"ns.{domain}.")
+            self.replace_rrset(domain, ".", "A", "3600", ip)
+            self.replace_rrset(domain, "ns", "A", "3600", dns_ip)
+
+            # AAAA (domain -> container IPv6, if available)
+            ipv6 = d.get("ipv6")
+            if ipv6:
+                self.replace_rrset(domain, ".", "AAAA", "3600", ipv6)
+                self.out.print(f"    zone reset: SOA, NS, A, AAAA  ({ip}, {ipv6})")
+            else:
+                # Remove any stale AAAA record
+                self.pdnsutil("delete-rrset", domain, ".", "AAAA", check=False)
+                self.out.print(f"    zone reset: SOA, NS, A  ({ip}, IPv4-only)")
+
+        self.restart_services()
+
+    def set_dns_records(self, text):
+        """Add or overwrite DNS records from standard BIND format.
+
+        Uses ``cmdeploy.dns.parse_zone_records`` to parse.
+        Zones are created automatically from the record names.
+        """
+        from ..dns import parse_zone_records
+
+        zones_seen = set()
+
+        for name, ttl, rtype, rdata in parse_zone_records(text):
+            # Derive zone from name: find top-level .localchat domain
+            name_parts = name.split(".")
+            zone = name  # fallback
+            for i in range(len(name_parts) - 1):
+                if name_parts[i + 1 :] == ["localchat"]:
+                    zone = ".".join(name_parts[i:])
+                    break
+
+            # Create zone if first time seeing it
+            if zone not in zones_seen:
+                self.pdnsutil(
+                    "create-zone",
+                    zone,
+                    f"ns.{zone}",
+                    check=False,
+                )
+                zones_seen.add(zone)
+
+            # Figure out the record name relative to zone
+            if name == zone:
+                relative = "."
+            elif name.endswith(f".{zone}"):
+                relative = name[: -(len(zone) + 1)]
+            else:
+                relative = name
+
+            self.replace_rrset(zone, relative, rtype, ttl, rdata)
+
+        if zones_seen:
+            self.restart_services()

cmdeploy/src/cmdeploy/remote/rdns.py

@@ -58,8 +58,8 @@ def get_dkim_entry(mail_domain, pre_command, dkim_selector):
     dkim_value = '" "'.join(re.findall(".{1,255}", dkim_value_raw))
     web_dkim_value = "".join(re.findall(".{1,255}", dkim_value_raw))
     return (
-        f'{dkim_selector}._domainkey.{mail_domain}. TXT "{dkim_value}"',
-        f'{dkim_selector}._domainkey.{mail_domain}. TXT "{web_dkim_value}"',
+        f'{dkim_selector}._domainkey.{mail_domain}.  3600  IN  TXT  "{dkim_value}"',
+        f'{dkim_selector}._domainkey.{mail_domain}.  3600  IN  TXT  "{web_dkim_value}"',
     )
 
 
@@ -94,9 +94,11 @@ def check_zonefile(zonefile, verbose=True):
         if not zf_line.strip() or zf_line.startswith(";"):
             continue
         print(f"dns-checking {zf_line!r}") if verbose else log_progress("")
-        zf_domain, zf_typ, zf_value = zf_line.split(maxsplit=2)
-        zf_domain = zf_domain.rstrip(".")
-        zf_value = zf_value.strip()
+        parts = zf_line.split(None, 4)
+        zf_domain = parts[0].rstrip(".")
+        # parts[1]=TTL, parts[2]=IN, parts[3]=type, parts[4]=rdata
+        zf_typ = parts[3]
+        zf_value = parts[4].strip()
         query_value = query_dns(zf_typ, zf_domain)
         if zf_value != query_value:
             assert zf_typ in ("A", "AAAA", "CNAME", "CAA", "SRV", "MX", "TXT"), zf_line

cmdeploy/src/cmdeploy/selfsigned/deployer.py

@@ -12,13 +12,27 @@ def openssl_selfsigned_args(domain, cert_path, key_path, days=36500):
     ``www.<domain>`` and ``mta-sts.<domain>``.
     """
     return [
-        "openssl", "req", "-x509",
-        "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:P-256",
-        "-noenc", "-days", str(days),
-        "-keyout", str(key_path),
-        "-out", str(cert_path),
-        "-subj", f"/CN={domain}",
-        "-addext", "extendedKeyUsage=serverAuth,clientAuth",
+        "openssl",
+        "req",
+        "-x509",
+        "-newkey",
+        "ec",
+        "-pkeyopt",
+        "ec_paramgen_curve:P-256",
+        "-noenc",
+        "-days",
+        str(days),
+        "-keyout",
+        str(key_path),
+        "-out",
+        str(cert_path),
+        "-subj",
+        f"/CN={domain}",
+        # Mark as end-entity cert so it cannot be used as a CA to sign others.
+        "-addext",
+        "basicConstraints=critical,CA:FALSE",
+        "-addext",
+        "extendedKeyUsage=serverAuth,clientAuth",
         "-addext",
         f"subjectAltName=DNS:{domain},DNS:www.{domain},DNS:mta-sts.{domain}",
     ]
@@ -40,7 +54,9 @@ class SelfSignedTlsDeployer(Deployer):
 
     def configure(self):
         args = openssl_selfsigned_args(
-            self.mail_domain, self.cert_path, self.key_path,
+            self.mail_domain,
+            self.cert_path,
+            self.key_path,
         )
         cmd = shlex.join(args)
         server.shell(

cmdeploy/src/cmdeploy/sshexec.py

@@ -49,8 +49,13 @@ class SSHExec:
     RemoteError = execnet.RemoteError
     FuncError = FuncError
 
-    def __init__(self, host, verbose=False, python="python3", timeout=60):
-        self.gateway = execnet.makegateway(f"ssh=root@{host}//python={python}")
+    def __init__(
+        self, host, verbose=False, python="python3", timeout=60, ssh_config=None
+    ):
+        spec = f"ssh=root@{host}//python={python}"
+        if ssh_config:
+            spec += f"//ssh_config={ssh_config}"
+        self.gateway = execnet.makegateway(spec)
         self._remote_cmdloop_channel = bootstrap_remote(self.gateway, remote)
         self.timeout = timeout
         self.verbose = verbose
@@ -113,3 +118,46 @@ class LocalExec:
             res = self(call, kwargs, log_callback=remote.rshell.log_progress)
             print_stderr()
             return res
+
+
+# pyinfra exposes a ``ssh_config_file`` data key that *should* let
+# paramiko parse an SSH config file directly.  In practice it silently
+# fails to connect (zero hosts / zero operations), so we resolve the
+# hostname and identity-file ourselves and pass them via
+# ``--data ssh_hostname`` / ``--data ssh_key`` instead.
+# Execnet uses ssh natively (and not paramiko) and doesn't have this problem.
+
+
+def _get_from_ssh_config(host, ssh_config_path, key):
+    """Internal helper to parse a value for a specific key from ssh-config."""
+    current_hosts = []
+    found_value = None
+    with open(ssh_config_path) as f:
+        for raw_line in f:
+            line = raw_line.strip()
+            if not line or line.startswith("#"):
+                continue
+            parts = line.split(None, 1)
+            if not parts:
+                continue
+            directive = parts[0].lower()
+            if directive == "host":
+                if host in current_hosts and found_value:
+                    return found_value
+                current_hosts = parts[1].split()
+                found_value = None
+            elif directive == key.lower():
+                found_value = parts[1]
+    if host in current_hosts and found_value:
+        return found_value
+    return None
+
+
+def resolve_host_from_ssh_config(host, ssh_config_path):
+    """Resolve a host alias to its IP from an ssh-config file."""
+    return _get_from_ssh_config(host, ssh_config_path, "Hostname") or host
+
+
+def resolve_key_from_ssh_config(host, ssh_config_path):
+    """Resolve a host alias to its IdentityFile from an ssh-config file."""
+    return _get_from_ssh_config(host, ssh_config_path, "IdentityFile")

cmdeploy/src/cmdeploy/tests/data/zftest.zone

@@ -1,17 +1,18 @@
-; Required DNS entries for chatmail servers
-zftest.testrun.org.                   A     135.181.204.127
-zftest.testrun.org.                   AAAA  2a01:4f9:c012:52f4::1
-zftest.testrun.org.                   MX 10 zftest.testrun.org.
-_mta-sts.zftest.testrun.org.          TXT "v=STSv1; id=202403211706"
-mta-sts.zftest.testrun.org.           CNAME zftest.testrun.org.
-www.zftest.testrun.org.               CNAME zftest.testrun.org.
-opendkim._domainkey.zftest.testrun.org. TXT "v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoYt82CVUyz2ouaqjX2kB+5J80knAyoOU3MGU5aWppmwUwwTvj/oSTSpkc5JMtVTRmKKr8NUDWAL1Yw7dfGqqPHdHfwwjS3BIvDzYx+hzgtz62RnfNgV+/2MAoNpfX7cAFIHdRzEHNtwugc3RDLquqPoupAE3Y2YRw2T5zG5fILh4vwIcJZL5Uq6B92j8wwJqOex" "33n+vm1NKQ9rxo/UsHAmZlJzpooXcG/4igTBxJyJlamVSRR6N7Nul1v//YJb7J6v2o0iPHW6uE0StzKaPPNC2IVosSRFbD9H2oqppltptFSNPlI0E+t0JBWHem6YK7xcugiO3ImMCaaU8g6Jt/wIDAQAB;s=email;t=s"
+; Required DNS entries
+zftest.testrun.org.  3600  IN  A  135.181.204.127
+zftest.testrun.org.  3600  IN  AAAA  2a01:4f9:c012:52f4::1
+zftest.testrun.org.  3600  IN  MX  10 zftest.testrun.org.
+_mta-sts.zftest.testrun.org.  3600  IN  TXT  "v=STSv1; id=202403211706"
+mta-sts.zftest.testrun.org.  3600  IN  CNAME  zftest.testrun.org.
+www.zftest.testrun.org.  3600  IN  CNAME  zftest.testrun.org.
+opendkim._domainkey.zftest.testrun.org.  3600  IN  TXT  "v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoYt82CVUyz2ouaqjX2kB+5J80knAyoOU3MGU5aWppmwUwwTvj/oSTSpkc5JMtVTRmKKr8NUDWAL1Yw7dfGqqPHdHfwwjS3BIvDzYx+hzgtz62RnfNgV+/2MAoNpfX7cAFIHdRzEHNtwugc3RDLquqPoupAE3Y2YRw2T5zG5fILh4vwIcJZL5Uq6B92j8wwJqOex" "33n+vm1NKQ9rxo/UsHAmZlJzpooXcG/4igTBxJyJlamVSRR6N7Nul1v//YJb7J6v2o0iPHW6uE0StzKaPPNC2IVosSRFbD9H2oqppltptFSNPlI0E+t0JBWHem6YK7xcugiO3ImMCaaU8g6Jt/wIDAQAB;s=email;t=s"
+
 ; Recommended DNS entries
-_submission._tcp.zftest.testrun.org.  SRV 0 1 587 zftest.testrun.org.
-_submissions._tcp.zftest.testrun.org. SRV 0 1 465 zftest.testrun.org.
-_imap._tcp.zftest.testrun.org.        SRV 0 1 143 zftest.testrun.org.
-_imaps._tcp.zftest.testrun.org.       SRV 0 1 993 zftest.testrun.org.
-zftest.testrun.org.                   CAA 0 issue "letsencrypt.org;accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/1371472956"
-zftest.testrun.org.                   TXT "v=spf1 a:zftest.testrun.org ~all"
-_dmarc.zftest.testrun.org.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
-_adsp._domainkey.zftest.testrun.org.  TXT "dkim=discardable"
+zftest.testrun.org.  3600  IN  TXT  "v=spf1 a ~all"
+_dmarc.zftest.testrun.org.  3600  IN  TXT  "v=DMARC1;p=reject;adkim=s;aspf=s"
+zftest.testrun.org.  3600  IN  CAA  0 issue "letsencrypt.org;accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/1371472956"
+_adsp._domainkey.zftest.testrun.org.  3600  IN  TXT  "dkim=discardable"
+_submission._tcp.zftest.testrun.org.  3600  IN  SRV  0 1 587 zftest.testrun.org.
+_submissions._tcp.zftest.testrun.org.  3600  IN  SRV  0 1 465 zftest.testrun.org.
+_imap._tcp.zftest.testrun.org.  3600  IN  SRV  0 1 143 zftest.testrun.org.
+_imaps._tcp.zftest.testrun.org.  3600  IN  SRV  0 1 993 zftest.testrun.org.

cmdeploy/src/cmdeploy/tests/online/benchmark.py

@@ -1,4 +1,3 @@
-import time
 def test_tls_imap(benchmark, imap):
     def imap_connect():
         imap.connect()

cmdeploy/src/cmdeploy/tests/online/test_0_qr.py

@@ -20,7 +20,7 @@ def test_fastcgi_working(maildomain, chatmail_config):
 
 
 @pytest.mark.filterwarnings("ignore::urllib3.exceptions.InsecureRequestWarning")
-def test_newemail_configure(maildomain, rpc, chatmail_config):
+def test_newemail_configure(maildomain, maildomain_ip, rpc, chatmail_config):
     """Test configuring accounts by scanning a QR code works."""
     url = f"DCACCOUNT:https://{maildomain}/new"
     for i in range(3):
@@ -30,12 +30,15 @@ def test_newemail_configure(maildomain, rpc, chatmail_config):
             # set_config_from_qr, so fetch credentials via requests instead
             res = requests.post(f"https://{maildomain}/new", verify=False)
             data = res.json()
-            rpc.add_or_update_transport(account_id, {
-                "addr": data["email"],
-                "password": data["password"],
-                "imapServer": maildomain,
-                "smtpServer": maildomain,
-                "certificateChecks": "acceptInvalidCertificates",
-            })
+            rpc.add_or_update_transport(
+                account_id,
+                {
+                    "addr": data["email"],
+                    "password": data["password"],
+                    "imapServer": maildomain_ip,
+                    "smtpServer": maildomain_ip,
+                    "certificateChecks": "acceptInvalidCertificates",
+                },
+            )
         else:
             rpc.add_transport_from_qr(account_id, url)

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -12,8 +12,9 @@ from cmdeploy.cmdeploy import get_sshexec
 
 class TestSSHExecutor:
     @pytest.fixture(scope="class")
-    def sshexec(self, sshdomain):
-        return get_sshexec(sshdomain)
+    def sshexec(self, sshdomain, pytestconfig):
+        ssh_config = pytestconfig.getoption("ssh_config")
+        return get_sshexec(sshdomain, ssh_config=ssh_config)
 
     def test_ls(self, sshexec):
         out = sshexec(call=remote.rdns.shell, kwargs=dict(command="ls"))
@@ -132,11 +133,10 @@ def test_authenticated_from(cmsetup, maildata):
 @pytest.mark.parametrize("from_addr", ["fake@example.org", "fake@testrun.org"])
 def test_reject_missing_dkim(cmsetup, maildata, from_addr):
     domain = cmsetup.maildomain
-    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    sock.settimeout(10)
     try:
-        sock.connect((domain, 25))
-    except socket.timeout:
+        sock = socket.create_connection((domain, 25), timeout=10)
+        sock.close()
+    except (socket.timeout, OSError):
         pytest.skip(f"port 25 not reachable for {domain}")
 
     recipient = cmsetup.gen_users(1)[0]

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -6,8 +6,8 @@ import imap_tools
 import pytest
 import requests
 
-from cmdeploy.remote import rshell
 from cmdeploy.cmdeploy import get_sshexec
+from cmdeploy.remote import rshell
 
 
 @pytest.fixture
@@ -67,7 +67,7 @@ class TestEndToEndDeltaChat:
         assert msg2.get_snapshot().text == "message0"
 
     def test_exceed_quota(
-        self, cmfactory, lp, tmpdir, remote, chatmail_config, sshdomain
+        self, cmfactory, lp, tmpdir, remote, chatmail_config, sshdomain, pytestconfig
     ):
         """This is a very slow test as it needs to upload >100MB of mail data
         before quota is exceeded, and thus depends on the speed of the upload.
@@ -92,7 +92,9 @@ class TestEndToEndDeltaChat:
         lp.sec(f"filling remote inbox for {user}")
         fn = f"7743102289.M843172P2484002.c20,S={quota},W=2398:2,"
         path = chatmail_config.mailboxes_dir.joinpath(user, "cur", fn)
-        sshexec = get_sshexec(sshdomain)
+        sshexec = get_sshexec(
+            sshdomain, ssh_config=pytestconfig.getoption("ssh_config")
+        )
         sshexec(call=rshell.write_numbytes, kwargs=dict(path=str(path), num=120))
         res = sshexec(call=rshell.dovecot_recalc_quota, kwargs=dict(user=user))
         assert res["percent"] >= 100

cmdeploy/src/cmdeploy/tests/online/test_3_status.py

@@ -3,12 +3,15 @@ import os
 from cmdeploy.cmdeploy import main
 
 
-def test_status_cmd(chatmail_config, capsys, request):
+def test_status_cmd(chatmail_config, capsys, request, pytestconfig):
     os.chdir(request.config.invocation_params.dir)
     command = ["status"]
-    if os.getenv("CHATMAIL_SSH"):
-        command.append("--ssh-host")
-        command.append(os.getenv("CHATMAIL_SSH"))
+    ssh_host = pytestconfig.getoption("ssh_host")
+    if ssh_host:
+        command.extend(["--ssh-host", ssh_host])
+    ssh_config = pytestconfig.getoption("ssh_config")
+    if ssh_config:
+        command.extend(["--ssh-config", ssh_config])
     assert main(command) == 0
     status_out = capsys.readouterr()
     print(status_out.out)

cmdeploy/src/cmdeploy/tests/plugin.py

@@ -2,7 +2,9 @@ import imaplib
 import itertools
 import os
 import random
+import re
 import smtplib
+import socket
 import ssl
 import subprocess
 import time
@@ -18,6 +20,76 @@ def pytest_addoption(parser):
     parser.addoption(
         "--slow", action="store_true", default=False, help="also run slow tests"
     )
+    parser.addoption(
+        "--ssh-host",
+        dest="ssh_host",
+        default=None,
+        help="SSH host (overrides mail_domain for SSH operations).",
+    )
+    parser.addoption(
+        "--ssh-config",
+        dest="ssh_config",
+        default=None,
+        help="Path to an SSH config file (e.g. lxconfigs/ssh-config).",
+    )
+
+
+def _parse_ssh_config_hosts(path):
+    """Parse an OpenSSH config file and return a dict of hostname -> IP."""
+    mapping = {}
+    current_names = []
+    for ln in Path(path).read_text().splitlines():
+        line = ln.strip()
+        m = re.match(r"^Host\s+(.+)", line)
+        if m:
+            current_names = m.group(1).split()
+            continue
+        m = re.match(r"^Hostname\s+(\S+)", line)
+        if m and current_names:
+            ip = m.group(1)
+            for name in current_names:
+                mapping[name] = ip
+            current_names = []
+    return mapping
+
+
+_original_getaddrinfo = socket.getaddrinfo
+
+
+def _make_patched_getaddrinfo(host_map):
+    """Return a getaddrinfo that resolves hosts in host_map to their IPs."""
+
+    def patched_getaddrinfo(host, port, family=0, type=0, proto=0, flags=0):
+        if host in host_map:
+            ip = host_map[host]
+            return _original_getaddrinfo(ip, port, family, type, proto, flags)
+        return _original_getaddrinfo(host, port, family, type, proto, flags)
+
+    return patched_getaddrinfo
+
+
+@pytest.fixture(autouse=True, scope="session")
+def _setup_localchat_dns(pytestconfig):
+    """Monkey-patch socket.getaddrinfo to resolve .localchat via ssh-config."""
+    ssh_config = pytestconfig.getoption("ssh_config")
+    if not ssh_config or not Path(ssh_config).exists():
+        yield {}
+        return
+    host_map = _parse_ssh_config_hosts(ssh_config)
+    if not host_map:
+        yield {}
+        return
+    socket.getaddrinfo = _make_patched_getaddrinfo(host_map)
+    try:
+        yield host_map
+    finally:
+        socket.getaddrinfo = _original_getaddrinfo
+
+
+@pytest.fixture(scope="session")
+def ssh_config_host_map(_setup_localchat_dns):
+    """Return the host-name → IP map parsed from ssh-config."""
+    return _setup_localchat_dns
 
 
 def pytest_configure(config):
@@ -35,6 +107,11 @@ def pytest_runtest_setup(item):
 
 
 def _get_chatmail_config():
+    ini = os.environ.get("CHATMAIL_INI")
+    if ini:
+        path = Path(ini).resolve()
+        if path.exists():
+            return read_config(path), path
     current = Path().resolve()
     while 1:
         path = current.joinpath("chatmail.ini").resolve()
@@ -61,8 +138,14 @@ def maildomain(chatmail_config):
 
 
 @pytest.fixture(scope="session")
-def sshdomain(maildomain):
-    return os.environ.get("CHATMAIL_SSH", maildomain)
+def sshdomain(maildomain, pytestconfig):
+    return pytestconfig.getoption("ssh_host") or maildomain
+
+
+@pytest.fixture(scope="session")
+def maildomain_ip(maildomain, ssh_config_host_map):
+    """Return the IP for maildomain from ssh-config, or maildomain itself."""
+    return ssh_config_host_map.get(maildomain, maildomain)
 
 
 @pytest.fixture
@@ -306,12 +389,22 @@ from deltachat_rpc_client import DeltaChat, Rpc
 class ChatmailACFactory:
     """RPC-based account factory for chatmail testing."""
 
-    def __init__(self, rpc, maildomain, gencreds, chatmail_config):
+    def __init__(
+        self,
+        rpc,
+        maildomain,
+        maildomain_ip,
+        gencreds,
+        chatmail_config,
+        ssh_config_host_map,
+    ):
         self.dc = DeltaChat(rpc)
         self.rpc = rpc
         self._maildomain = maildomain
+        self._maildomain_ip = maildomain_ip
         self.gencreds = gencreds
         self.chatmail_config = chatmail_config
+        self._ssh_config_host_map = ssh_config_host_map
 
     def _make_transport(self, domain):
         """Build a transport config dict for the given domain."""
@@ -319,11 +412,13 @@ class ChatmailACFactory:
         transport = {
             "addr": addr,
             "password": password,
-            # Setting server explicitly skips requesting autoconfig XML,
-            # see https://datatracker.ietf.org/doc/draft-ietf-mailmaint-autoconfig/
-            "imapServer": domain,
-            "smtpServer": domain,
         }
+        # To support running against local relays without host DNS resolution
+        # we attempt resolving the domain via ssh-config
+        # because otherwise core fails to find the address
+        server = self._ssh_config_host_map.get(domain)
+        if server is not None:
+            transport.update({"imapServer": server, "smtpServer": server})
         if self.chatmail_config.tls_cert_mode == "self":
             transport["certificateChecks"] = "acceptInvalidCertificates"
         return transport
@@ -376,32 +471,43 @@ def rpc(tmp_path_factory):
 
 
 @pytest.fixture
-def cmfactory(rpc, gencreds, maildomain, chatmail_config):
+def cmfactory(
+    rpc, gencreds, maildomain, maildomain_ip, chatmail_config, ssh_config_host_map
+):
     """Return a ChatmailACFactory for creating online Delta Chat accounts."""
     return ChatmailACFactory(
         rpc=rpc,
         maildomain=maildomain,
+        maildomain_ip=maildomain_ip,
         gencreds=gencreds,
         chatmail_config=chatmail_config,
+        ssh_config_host_map=ssh_config_host_map,
     )
 
 
 @pytest.fixture
-def remote(sshdomain):
-    return Remote(sshdomain)
+def remote(sshdomain, pytestconfig):
+    return Remote(sshdomain, ssh_config=pytestconfig.getoption("ssh_config"))
 
 
 class Remote:
-    def __init__(self, sshdomain):
+    def __init__(self, sshdomain, ssh_config=None):
         self.sshdomain = sshdomain
+        self.ssh_config = ssh_config
 
     def iter_output(self, logcmd="", ready=None):
         getjournal = "journalctl -f" if not logcmd else logcmd
         print(self.sshdomain)
         match self.sshdomain:
-            case "@local": command = []
-            case "localhost": command = []
-            case _: command = ["ssh", f"root@{self.sshdomain}"]
+            case "@local":
+                command = []
+            case "localhost":
+                command = []
+            case _:
+                command = ["ssh"]
+                if self.ssh_config:
+                    command.extend(["-F", self.ssh_config])
+                command.append(f"root@{self.sshdomain}")
         [command.append(arg) for arg in getjournal.split()]
         self.popen = subprocess.Popen(
             command,

cmdeploy/src/cmdeploy/tests/test_cmdeploy.py

@@ -23,7 +23,10 @@ class TestCmdline:
         run = parser.parse_args(["run"])
         assert init and run
 
-    def test_init_not_overwrite(self, capsys):
+    def test_init_not_overwrite(self, tmp_path, capsys, monkeypatch):
+        monkeypatch.delenv("CHATMAIL_INI", raising=False)
+        monkeypatch.chdir(tmp_path)
+
         assert main(["init", "chat.example.org"]) == 0
         capsys.readouterr()
 

cmdeploy/src/cmdeploy/tests/test_dns.py

@@ -3,7 +3,7 @@ from copy import deepcopy
 import pytest
 
 from cmdeploy import remote
-from cmdeploy.dns import check_full_zone, check_initial_remote_data
+from cmdeploy.dns import check_full_zone, check_initial_remote_data, parse_zone_records
 
 
 @pytest.fixture
@@ -125,18 +125,33 @@ class TestPerformInitialChecks:
         assert not l
 
 
+def test_parse_zone_records():
+    text = """
+    ; This is a comment
+    some.domain. 3600 IN A 1.1.1.1
+
+    ; Another comment
+    www.some.domain. 3600 IN CNAME some.domain.
+    """
+    records = list(parse_zone_records(text))
+    assert records == [
+        ("some.domain", "3600", "A", "1.1.1.1"),
+        ("www.some.domain", "3600", "CNAME", "some.domain."),
+    ]
+
+
+def test_parse_zone_records_invalid_line():
+    text = "invalid line"
+    with pytest.raises(ValueError, match="Bad zone record line"):
+        list(parse_zone_records(text))
+
+
 def parse_zonefile_into_dict(zonefile, mockdns_base, only_required=False):
-    for zf_line in zonefile.split("\n"):
-        if zf_line.startswith("#"):
-            if "Recommended" in zf_line and only_required:
-                return
-            continue
-        if not zf_line.strip():
-            continue
-        zf_domain, zf_typ, zf_value = zf_line.split(maxsplit=2)
-        zf_domain = zf_domain.rstrip(".")
-        zf_value = zf_value.strip()
-        mockdns_base.setdefault(zf_typ, {})[zf_domain] = zf_value
+    if only_required:
+        # Only take records before the "; Recommended" section
+        zonefile = zonefile.split("; Recommended")[0]
+    for name, ttl, rtype, rdata in parse_zone_records(zonefile):
+        mockdns_base.setdefault(rtype, {})[name] = rdata
 
 
 class MockSSHExec:

cmdeploy/src/cmdeploy/tests/test_lxc.py

@@ -0,0 +1,174 @@
+"""Tests for cmdeploy lxc-* subcommands."""
+
+import shutil
+import subprocess
+import sys
+
+import pytest
+
+from cmdeploy.lxc import cli
+from cmdeploy.lxc.incus import Incus
+from cmdeploy.util import Out
+
+pytestmark = pytest.mark.skipif(
+    not shutil.which("incus"),
+    reason="incus not installed",
+)
+
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+
+
+@pytest.fixture
+def ix():
+    out = Out()
+    return Incus(out)
+
+
+@pytest.fixture(scope="session")
+def lxc_setup():
+    out = Out()
+    ix = Incus(out)
+    ix.get_dns_container().ensure()
+    return ix.list_managed()
+
+
+@pytest.fixture(scope="session")
+def relay_container(lxc_setup):
+    test_names = {f"{n}-localchat" for n in cli.RELAY_NAMES}
+    relays = [c for c in lxc_setup if c["name"] in test_names and c.get("ip")]
+    if not relays:
+        pytest.skip("no test relay containers running")
+    return relays[0]
+
+
+@pytest.fixture
+def cmdeploy():
+    def run(*args):
+        return subprocess.run(
+            [sys.executable, "-m", "cmdeploy.cmdeploy", *args],
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+
+    return run
+
+
+# ---------------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+    "subcmd, expected, absent",
+    [
+        (None, ["lxc-start", "lxc-stop", "lxc-test", "lxc-status"], ["lxc-destroy"]),
+        ("lxc-start", ["--ipv4-only", "--run"], ["--config"]),
+        ("lxc-stop", ["--destroy", "--destroy-all"], ["--config"]),
+        ("lxc-test", ["--one"], ["--config"]),
+        ("lxc-status", [], ["--config"]),
+        ("run", ["--ssh-config"], ["--lxc"]),
+        ("dns", ["--ssh-config"], []),
+        ("test", ["--ssh-config"], []),
+        ("status", ["--ssh-config"], []),
+    ],
+)
+def test_help_options(cmdeploy, subcmd, expected, absent):
+    args = [subcmd, "--help"] if subcmd else ["--help"]
+    result = cmdeploy(*args)
+    output = result.stdout + result.stderr
+    assert result.returncode == 0
+    for flag in expected:
+        assert flag in output
+    for flag in absent:
+        assert flag not in output
+
+
+class TestSSHConfig:
+    def test_lxconfigs(self, ix, lxc_setup):
+        d = ix.lxconfigs_dir
+        assert d.name == "lxconfigs"
+        assert d.exists()
+        path = ix.ssh_config_path
+        assert path.name == "ssh-config"
+        assert path.parent.name == "lxconfigs"
+
+    def test_write_ssh_config(self, ix, lxc_setup):
+        path = ix.write_ssh_config()
+        assert path.exists()
+        text = path.read_text()
+
+        for c in lxc_setup:
+            if c.get("ip"):
+                assert c["name"] in text
+                assert f"Hostname {c['ip']}" in text
+
+        assert "User root" in text
+        assert "IdentityFile" in text
+        assert "StrictHostKeyChecking accept-new" in text
+
+
+def test_dns(ix, relay_container):
+    def dig(qname, qtype):
+        ct = ix.get_dns_container()
+        return ct.bash(f"dig @127.0.0.1 {qname} {qtype} +short").strip()
+
+    domain = relay_container["domain"]
+    assert dig(domain, "A") == relay_container["ip"]
+    assert domain in dig(domain, "MX")
+    assert "587" in dig(f"_submission._tcp.{domain}", "SRV")
+
+
+class TestLxcStatus:
+    def test_cli_lxc_status_help(self, cmdeploy):
+        result = cmdeploy("lxc-status", "--help")
+        assert result.returncode == 0
+        assert "status" in result.stdout.lower()
+
+    def test_shows_containers(self, lxc_setup, capsys):
+        class QuietOut(Out):
+            def red(self, msg, **kw):
+                pass
+
+            def green(self, msg, **kw):
+                pass
+
+        ret = cli.lxc_status_cmd(None, QuietOut())
+        assert ret == 0
+        captured = capsys.readouterr().out
+        assert "ns-localchat" in captured
+        assert "running" in captured
+
+    def test_deploy_freshness(self, ix, monkeypatch):
+        ct = ix.get_container("x")
+
+        monkeypatch.setattr(
+            "cmdeploy.lxc.incus.RelayContainer.deployed_version",
+            lambda _self: "abc123def456",
+        )
+        monkeypatch.setattr(
+            "cmdeploy.lxc.incus.RelayContainer.deployed_domain",
+            lambda _self: ct.domain,
+        )
+        monkeypatch.setattr(
+            "cmdeploy.lxc.cli.get_version_string",
+            lambda: "abc123def456",
+        )
+        assert "IN-SYNC" in cli._deploy_status(ct, "abc123def456", ix)
+        assert "STALE" in cli._deploy_status(ct, "other_hash_here", ix)
+
+        # Hash matches but local has uncommitted changes
+        monkeypatch.setattr(
+            "cmdeploy.lxc.cli.get_version_string",
+            lambda: "abc123def456\ndiff --git a/foo",
+        )
+        assert "DIRTY" in cli._deploy_status(ct, "abc123def456", ix)
+
+        monkeypatch.setattr(
+            "cmdeploy.lxc.incus.RelayContainer.deployed_version",
+            lambda _self: None,
+        )
+        assert "NOT DEPLOYED" in cli._deploy_status(ct, "abc123", ix)

cmdeploy/src/cmdeploy/tests/test_util.py

@@ -0,0 +1,120 @@
+import sys
+
+from cmdeploy.util import Out, collapse, get_git_hash, get_version_string, shell
+
+
+class TestOut:
+    def test_prefix_default(self, capsys):
+        out = Out()
+        out.print("hello")
+        assert capsys.readouterr().out == "hello\n"
+
+    def test_prefix_custom(self, capsys):
+        out = Out(prefix=">> ")
+        out.print("hello")
+        assert capsys.readouterr().out == ">> hello\n"
+
+    def test_prefix_print_file(self):
+        import io
+
+        buf = io.StringIO()
+        out = Out(prefix=":: ")
+        out.print("msg", file=buf)
+        assert ":: msg" in buf.getvalue()
+
+    def test_new_prefixed_out(self, capsys):
+        parent = Out(prefix="A")
+        child = parent.new_prefixed_out("B")
+        child.print("x")
+        assert capsys.readouterr().out == "ABx\n"
+        # shares section_timings
+        assert child.section_timings is parent.section_timings
+
+    def test_section_no_auto_indent(self, capsys):
+        out = Out(prefix="")
+        with out.section("test"):
+            out.print("inside")
+        captured = capsys.readouterr().out
+        # "inside" should NOT be indented by section()
+        lines = captured.strip().splitlines()
+        inside_line = [l for l in lines if "inside" in l][0]
+        assert inside_line == "inside"
+
+    def test_section_records_timing(self):
+        out = Out()
+        with out.section("s1"):
+            pass
+        assert len(out.section_timings) == 1
+        assert out.section_timings[0][0] == "s1"
+
+    def test_shell_failure_shows_output(self):
+        """When a shell command fails, its output and exit code are shown."""
+        import subprocess
+
+        result = subprocess.run(
+            [
+                sys.executable,
+                "-c",
+                "from cmdeploy.util import Out; Out(prefix='').shell("
+                "\"echo 'boom on stderr' >&2; exit 42\")",
+            ],
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+        # the command's stderr is merged into stdout by Popen
+        assert "boom on stderr" in result.stdout
+        # Out.red() prints the failure notice to stderr
+        assert "exit code 42" in result.stderr
+
+
+def test_collapse():
+    text = """
+        line 1
+        line 2
+    """
+    assert collapse(text) == "line 1 line 2"
+    assert collapse("  single line  ") == "single line"
+
+
+def test_git_helpers_no_git(tmp_path):
+    # Not a git repo
+    assert get_git_hash(root=tmp_path) is None
+    assert get_version_string(root=tmp_path) == "unknown"
+
+
+def test_git_helpers_empty_repo(tmp_path):
+    shell("git init", cwd=tmp_path, check=True)
+    # No commits yet
+    assert get_git_hash(root=tmp_path) is None
+    assert get_version_string(root=tmp_path) == "unknown"
+
+
+def test_git_helpers_with_commits_and_diffs(tmp_path):
+    shell("git init", cwd=tmp_path, check=True)
+    shell("git config user.email you@example.com", cwd=tmp_path, check=True)
+    shell("git config user.name 'Your Name'", cwd=tmp_path, check=True)
+
+    # First commit
+    path = tmp_path / "file.txt"
+    path.write_text("content")
+    shell("git add file.txt", cwd=tmp_path, check=True)
+    shell("git commit -m initial", cwd=tmp_path, check=True)
+
+    git_hash = get_git_hash(root=tmp_path)
+    assert len(git_hash) >= 7  # usually 40, but git is git
+    assert get_version_string(root=tmp_path) == git_hash
+
+    # Create a diff
+    path.write_text("new content")
+    v = get_version_string(root=tmp_path)
+    assert v.startswith(git_hash + "\n")
+    assert "new content" in v
+    assert not v.endswith("\n")
+
+    # Commit again -> no diff
+    shell("git add file.txt", cwd=tmp_path, check=True)
+    shell("git commit -m second", cwd=tmp_path, check=True)
+    new_hash = get_git_hash(root=tmp_path)
+    assert new_hash != git_hash
+    assert get_version_string(root=tmp_path) == new_hash

cmdeploy/src/cmdeploy/util.py

@@ -0,0 +1,168 @@
+"""Shared utility functions for cmdeploy."""
+
+import os
+import shutil
+import subprocess
+import sys
+import textwrap
+import time
+from contextlib import contextmanager
+from pathlib import Path
+
+from termcolor import colored
+
+
+class Out:
+    """Convenience output printer providing coloring and section formatting."""
+
+    def __init__(self, sepchar="\u2501", prefix="", verbosity=0):
+        self.section_timings = []
+        self.prefix = prefix
+        self.sepchar = sepchar
+        self.verbosity = verbosity
+        env_width = os.environ.get("_CMDEPLOY_WIDTH")
+        if env_width:
+            self.section_width = int(env_width)
+        else:
+            self.section_width = shutil.get_terminal_size((80, 24)).columns
+
+    def new_prefixed_out(self, newprefix="  "):
+        """Return a new Out with an extended prefix,
+        sharing section_timings with the parent.
+        """
+        out = Out(
+            sepchar=self.sepchar,
+            prefix=self.prefix + newprefix,
+            verbosity=self.verbosity,
+        )
+        out.section_timings = self.section_timings
+        return out
+
+    def red(self, msg, file=sys.stderr):
+        print(colored(self.prefix + msg, "red"), file=file, flush=True)
+
+    def green(self, msg, file=sys.stderr):
+        print(colored(self.prefix + msg, "green"), file=file, flush=True)
+
+    def print(self, msg="", **kwargs):
+        """Print to stdout with automatic flush."""
+        if msg:
+            msg = self.prefix + msg
+        print(msg, flush=True, **kwargs)
+
+    def _format_header(self, title):
+        """Return a formatted section header string."""
+        width = self.section_width - len(self.prefix)
+        bar = self.sepchar * (width - len(title) - 5)
+        return f"{self.sepchar * 3} {title} {bar}"
+
+    @contextmanager
+    def section(self, title):
+        """Context manager that prints a section header and records elapsed time."""
+        self.green(self._format_header(title))
+        t0 = time.time()
+        yield
+        elapsed = time.time() - t0
+        self.section_timings.append((title, elapsed))
+
+    def section_line(self, title):
+        """Print a section header without timing."""
+        self.green(self._format_header(title))
+
+    def shell(self, cmd, quiet=False, **kwargs):
+        """Print *cmd*, run it, and re-print its output with the current prefix.
+
+        *cmd* is passed through :func:`collapse`, so callers
+        can use triple-quoted f-strings freely.
+        Stdout and stderr are merged, read line-by-line,
+        and each line is printed with ``self.prefix`` prepended.
+        When the command exits non-zero, a red error line is printed.
+        """
+        cmd = collapse(cmd)
+        if not quiet:
+            self.print(f"$ {cmd}")
+        indent = self.prefix + "  "
+        env = kwargs.pop("env", None)
+        if env is None:
+            env = os.environ.copy()
+        env["_CMDEPLOY_WIDTH"] = str(self.section_width - len(indent))
+        proc = subprocess.Popen(
+            cmd,
+            shell=True,
+            text=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            env=env,
+            **kwargs,
+        )
+        for line in proc.stdout:
+            sys.stdout.write(indent + line)
+            sys.stdout.flush()
+        ret = proc.wait()
+        if ret:
+            self.red(f"command failed with exit code {ret}: {cmd}")
+        return ret
+
+
+def _project_root():
+    """Return the project root directory."""
+    return Path(__file__).resolve().parent.parent.parent.parent
+
+
+def collapse(text):
+    """Dedent, join lines, and strip a (triple-quoted) string.
+
+    Handy for writing shell commands across multiple lines::
+
+        cmd = collapse(f\"""
+            cmdeploy run
+            --config {ct.ini}
+            --ssh-host {ct.domain}
+        \""")
+    """
+    return textwrap.dedent(text).replace("\n", " ").strip()
+
+
+def shell(cmd, check=False, **kwargs):
+    """Run a shell command string with sensible defaults.
+
+    *cmd* is passed through :func:`collapse` first, so callers
+    can use triple-quoted f-strings freely.
+    Captures stdout/stderr by default; pass ``capture_output=False``
+    to stream output to the terminal instead.
+    """
+    if "capture_output" not in kwargs and "stdout" not in kwargs:
+        kwargs["capture_output"] = True
+    return subprocess.run(collapse(cmd), shell=True, text=True, check=check, **kwargs)
+
+
+def get_git_hash(root=None):
+    """Return the local HEAD commit hash, or None."""
+    if root is None:
+        root = _project_root()
+    result = shell(
+        "git rev-parse HEAD",
+        cwd=str(root),
+    )
+    if result.returncode == 0:
+        return result.stdout.strip()
+    return None
+
+
+def get_version_string(root=None):
+    """Return ``git_hash\\ngit_diff`` for the local working tree.
+
+    Used by :class:`~cmdeploy.deployers.GithashDeployer` to write
+    ``/etc/chatmail-version`` and by ``lxc-status`` to compare
+    the deployed state against the local checkout.
+    """
+    if root is None:
+        root = _project_root()
+    git_hash = get_git_hash(root=root) or "unknown"
+    try:
+        git_diff = shell("git diff", cwd=str(root)).stdout.strip()
+    except Exception:
+        git_diff = ""
+    if git_diff:
+        return f"{git_hash}\n{git_diff}"
+    return git_hash

doc/source/conf.py

@@ -15,7 +15,7 @@ author = 'chatmail collective'
 
 extensions = [
     #'sphinx.ext.autodoc',
-    #'sphinx.ext.viewdoc',
+    #'sphinx.ext.viewcode',
     'sphinxcontrib.mermaid',
 ]
 

doc/source/index.rst

@@ -16,5 +16,6 @@ Contributions and feedback welcome through the https://github.com/chatmail/relay
     proxy
     migrate
     overview
+    lxc
     related
     faq

doc/source/lxc.rst

@@ -0,0 +1,271 @@
+Local testing with LXC/Incus
+============================
+
+The ``cmdeploy`` tool includes support for running
+chatmail relays inside local
+`Incus <https://linuxcontainers.org/incus/>`_ LXC containers.
+This is meant for development, testing, and CI
+without requiring a remote server.
+LXC system containers are lightweight virtual machines
+that share the host's kernel but run their own init system,
+package manager, and network stack,
+so the cmdeploy deployment scripts work pretty much
+as they would on a real Debian server or cloud VPS.
+
+Prerequisites
+-------------
+
+Install `Incus <https://linuxcontainers.org/incus/>`_
+(LXC container manager).
+See the `official installation guide
+<https://linuxcontainers.org/incus/docs/main/installing/>`_
+for full details.
+
+After installing incus, initialise and grant yourself access::
+
+    sudo incus admin init --minimal
+    sudo usermod -aG incus-admin $USER
+
+
+.. caution::
+
+   Adding yourself to ``incus-admin`` grants effective root access
+   to the host: any member can mount host directories into a container
+   and manipulate them as root.
+   This is fine for local testing of your own relay branches,
+   but do **not** use it for production setups
+   or for testing untrusted relay branches from others.
+
+.. warning::
+
+   You **must now log out and back in** (or run ``newgrp incus-admin``)
+   after adding yourself to the group.
+   Without this, all ``cmdeploy lxc-*`` commands
+   will fail with permission errors.
+
+Verify the installation works by running ``incus list``,
+which should print an empty table without errors.
+
+
+Quick start
+-----------
+
+::
+
+    cd relay
+    scripts/initenv.sh               # bootstrap venv
+    source venv/bin/activate         # activate venv
+    cmdeploy lxc-test                # create containers, deploy, test
+
+The ``lxc-test`` command provides an automated way
+to run the full deployment and test pipeline.
+It executes several ``cmdeploy`` subcommands in sequential steps.
+If a step fails, you can copy-paste the printed command
+and run it manually to debug.
+No host DNS delegation or ``~/.ssh/config`` changes are needed
+because ``lxc-test`` passes the required SSH and DNS options directly.
+
+
+CLI reference
+--------------
+
+``lxc-start [--ipv4-only] [--run] [NAME ...]``
+    Create and start containers.
+    Without arguments, creates ``test0-localchat`` and ``ns-localchat`` (DNS).
+    Pass one or more ``NAME`` arguments to create user relay containers instead
+    (e.g. ``cmdeploy lxc-start myrelay``).
+    Use ``--ipv4-only`` to set ``disable_ipv6 = True`` in the generated ``chatmail.ini``,
+    producing an IPv4-only relay.
+    Use ``--run`` to automatically run ``cmdeploy run`` on each container after starting it.
+    Generates ``lxconfigs/ssh-config``.
+    It reuses existing containers and resets DNS zones to minimal records.
+
+``lxc-stop [--destroy] [--destroy-all] [NAME ...]``
+    Stop relay containers.
+    Without arguments, stops ``test0-localchat`` and ``test1-localchat``.
+    Pass ``NAME`` to stop specific containers.
+    Use ``--destroy`` to also delete the containers and their config files.
+    Use ``--destroy-all`` to additionally destroy
+    the ``ns-localchat`` DNS container **and** remove
+    the cached ``localchat-base`` and ``localchat-relay``
+    images, giving a fully clean slate for the next ``lxc-test``.
+    User containers are **never** destroyed unless named explicitly.
+
+``lxc-test [--one]``
+    By default creates, deploys, and tests both ``test0`` and ``test1``
+    for dual-domain federation testing (sets ``CHATMAIL_DOMAIN2=_test1.localchat``).
+    test0 runs dual-stack (IPv4 + IPv6) while test1 runs IPv4-only (``disable_ipv6 = True``).
+    Pass ``--one`` to only deploy and test against ``test0``
+    (skips ``test1``, does not set ``CHATMAIL_DOMAIN2``).
+
+``lxc-status``
+    Show live status of all LXC containers (including the DNS container),
+    deploy freshness (comparing ``/etc/chatmail-version``
+    against local ``git rev-parse HEAD`` and ``git diff``),
+    SSH config inclusion, and host DNS forwarding for ``.localchat``.
+    Reports **IN-SYNC**, **DIRTY** (hash matches but uncommitted changes exist),
+    **STALE** (different commit), or **NOT DEPLOYED**.
+
+
+Container types
+-----------------
+
+**Test relay containers** (``test0-localchat``, ``test1-localchat``)
+    Created automatically by ``lxc-test``.
+    **test0** has IPv4 and IPv6 configured,
+    **test1** is IPv4-only (``disable_ipv6 = True``).
+
+**User relay containers** (``<name>-localchat``)
+    Created by ``cmdeploy lxc-start <name>``
+    where ``<name>`` does not start with ``test``.
+    These are personal development instances,
+    never touched by ``lxc-stop --destroy`` unless named explicitly.
+
+**DNS container** (``ns-localchat``)
+    Singleton container running PowerDNS.
+    Created automatically when any relay is started.
+
+
+.. _lxc-ssh-config:
+
+SSH configuration
+-----------------
+
+``cmdeploy lxc-start`` generates ``lxconfigs/ssh-config``,
+a standard OpenSSH config file mapping every container name,
+its domain, and a short alias to the container's IP address::
+
+    Host test0-localchat _test0.localchat _test0
+        Hostname 10.204.0.42
+        User root
+        IdentityFile /path/to/relay/lxconfigs/id_localchat
+        IdentitiesOnly yes
+        StrictHostKeyChecking accept-new
+        UserKnownHostsFile /dev/null
+        LogLevel ERROR
+
+All ``cmdeploy`` commands (``run``, ``dns``, ``status``, ``test``)
+accept ``--ssh-config lxconfigs/ssh-config`` to use this file.
+``lxc-test`` passes it automatically.
+
+**Using containers from the host shell:**
+
+To make ``ssh _test0`` work from any terminal, add one line to ``~/.ssh/config``::
+
+    Include /absolute/path/to/relay/lxconfigs/ssh-config
+
+
+.. _lxc-dns-setup:
+.. _localchat-tld:
+
+``.localchat`` DNS and name resolution
+---------------------------------------
+
+All LXC-managed chatmail domains use the ``.localchat`` pseudo-TLD
+(e.g. ``_test0.localchat``, ``_test1.localchat``),
+a non-delegated suffix that exists only within the local PowerDNS infrastructure.
+A dedicated DNS container (``ns-localchat``)
+is created so that local test relays interact
+with DNS similar to a regular public Internet setup.
+On first start, ``cmdeploy lxc-start`` creates this container
+running two `PowerDNS <https://www.powerdns.com/>`_ services:
+
+* **pdns-server** (authoritative) serves ``.localchat``
+  zones from a local SQLite database.
+
+* **pdns-recursor** (recursive) listens on the Incus
+  bridge so all containers can use it.
+  Forwards ``.localchat`` queries to the local
+  authoritative server and everything else to Quad9 (``9.9.9.9``).
+
+After the DNS container is up, ``lxc-start`` configures the Incus bridge
+to advertise its IP via DHCP and disables Incus's own DNS.
+DNS records are then created in two phases matching the "cmdeploy run" deployment flow:
+
+1. **``lxc-start``** resets each relay zone to
+   **SOA, NS, and A** records (plus **AAAA** for dual-stack containers).
+   If host DNS resolution is configured, users can
+   afterwards run ``cmdeploy run --config lxconfigs/chatmail-test0.ini
+   --ssh-config lxconfigs/ssh-config --ssh-host _test0.localchat``.
+   LXC subcommands do not depend on host DNS resolution
+   and resolve addresses via ``lxconfigs/ssh-config``.
+
+2. **``cmdeploy dns --zonefile``** generates a standard
+   BIND-format zone file (MX, TXT/SPF, TXT/DMARC,
+   TXT/MTA-STS, SRV, CNAME, DKIM) and loads it
+   into PowerDNS.
+
+This two-phase approach prevents premature configuration of mail records
+before the relay is actually deployed and running.
+Once ``cmdeploy run`` deploys `Unbound <https://nlnetlabs.nl/projects/unbound/>`_
+inside a relay container, Unbound has a configuration plugin snippet
+that forwards all ``.localchat`` queries to the PowerDNS recursor,
+and lets all other queries go through normal recursive resolution.
+
+
+State outside the repository
+-----------------------------
+
+All generated configuration by lxc subcommands live in ``lxconfigs/``
+(git-ignored), including the SSH key pair (``id_localchat``),
+per-container ``chatmail-*.ini`` files, zone files, and ``ssh-config``.
+
+The only state *outside* the repository is the Incus containers and images themselves
+(managed via the ``incus`` CLI, labelled with ``user.localchat-managed=true``).
+The Incus image store retains the following snapshot images:
+
+* ``localchat-base``: Debian 12 with openssh-server and Python (built on first run)
+
+* ``localchat-relay``: fully deployed relay snapshot,
+  cached after the first successful ``cmdeploy run``.
+  Subsequent relay containers launch from this image
+  so the deploy step is mostly no-ops (roughly 3× faster than a fresh deploy).
+
+
+.. _lxc-tls:
+
+TLS handling and underscore domains
+------------------------------------
+
+Container domains start with ``_`` (e.g. ``_test0.localchat``).
+As described in :doc:`getting_started` ("Running a relay with self-signed certificates"),
+underscore domains automatically use self-signed TLS
+and ``smtp_tls_security_level = encrypt``.
+This permits cross-relay federation between LXC containers
+without any external certificate authority.
+Delta Chat clients connecting to these relays
+must be configured with
+``certificateChecks = acceptInvalidCertificates``
+(the test fixtures handle this automatically).
+`PR #7926 on chatmail-core <https://github.com/chatmail/core/pull/7926>`_
+is meant to make this special setting unnecessary for chatmail clients
+that are connecting to underscore domains.
+
+
+Known limitations
+------------------
+
+The LXC environment differs from a production
+deployment in several ways:
+
+**No ACME / Let's Encrypt**:
+  Self-signed TLS only (see :ref:`lxc-tls`);
+  ACME code paths are never exercised locally.
+
+**No inbound connections from the internet**:
+  Containers sit on a private Incus bridge and are not port-forwarded.
+  Only the host and other containers on the same bridge can reach them.
+
+**Local federation only**:
+  Cross-relay mail delivery (e.g. test0 → test1) works between containers on the same host,
+  but these relays are invisible to any external mail server.
+
+**DNS is local only**:
+  The ``.localchat`` pseudo-TLD is not resolvable from the wider internet
+  (see :ref:`lxc-dns-setup`).
+
+**IPv6 is ULA-only**:
+  Containers receive IPv6 addresses from the ``fd42:...`` ULA range on the Incus bridge.
+  These are not globally routable, but are sufficient for testing IPv6 service binding
+  (Postfix, Dovecot, Nginx) and DNS AAAA records inside the local environment.
+  test1 runs with ``disable_ipv6 = True`` to exercise the IPv4-only deployment path.