fix(cmdeploy/dns): align zones, add multiline test records

And add requirements to lxc docs

.gitignore

@@ -5,6 +5,7 @@ __pycache__/
 *.swp
 *qr-*.png
 chatmail*.ini
+lxconfigs/
 
 
 # C extensions

chatmaild/pyproject.toml

@@ -6,10 +6,7 @@ build-backend = "setuptools.build_meta"
 name = "chatmaild"
 version = "0.3"
 dependencies = [
-  "aiosmtpd",
   "iniconfig",
-  "deltachat-rpc-server",
-  "deltachat-rpc-client",
   "filelock",
   "requests",
   "crypt-r >= 3.13.1 ; python_version >= '3.11'",
@@ -70,6 +67,7 @@ commands =
 deps = pytest 
        pdbpp 
        pytest-localserver
+       aiosmtpd
        execnet
 commands = pytest -v -rsXx {posargs}
 """

chatmaild/src/chatmaild/tests/plugin.py

@@ -85,13 +85,13 @@ def mockout():
         captured_green = []
         captured_plain = []
 
-        def red(self, msg):
+        def red(self, msg, **kw):
             self.captured_red.append(msg)
 
-        def green(self, msg):
+        def green(self, msg, **kw):
             self.captured_green.append(msg)
 
-        def __call__(self, msg):
+        def print(self, msg="", **kw):
             self.captured_plain.append(msg)
 
     return MockOut()

cmdeploy/pyproject.toml

@@ -10,7 +10,6 @@ dependencies = [
   "pillow",
   "qrcode",
   "markdown",
-  "pytest",
   "setuptools>=68",
   "termcolor",
   "build",
@@ -21,6 +20,7 @@ dependencies = [
   "execnet",
   "imap_tools",
   "deltachat-rpc-client",
+  "deltachat-rpc-server",
 ]
 
 [project.scripts]

cmdeploy/src/cmdeploy/chatmail.zone.j2

@@ -1,32 +0,0 @@
-;
-; Required DNS entries for chatmail servers 
-;
-{% if A %}
-{{ mail_domain }}.                   A     {{ A }}
-{% endif %}
-{% if AAAA %}
-{{ mail_domain }}.                   AAAA  {{ AAAA }}
-{% endif %}
-{{ mail_domain }}.                   MX 10 {{ mail_domain }}.
-{% if strict_tls %}
-_mta-sts.{{ mail_domain }}.          TXT "v=STSv1; id={{ sts_id }}"
-mta-sts.{{ mail_domain }}.           CNAME {{ mail_domain }}.
-{% endif %}
-www.{{ mail_domain }}.               CNAME {{ mail_domain }}.
-{{ dkim_entry }}
-
-;
-; Recommended DNS entries for interoperability and security-hardening
-;
-{{ mail_domain }}.                   TXT "v=spf1 a ~all"
-_dmarc.{{ mail_domain }}.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
-
-{% if acme_account_url %}
-{{ mail_domain }}.                   CAA 0 issue "letsencrypt.org;accounturi={{ acme_account_url }}"
-{% endif %}
-_adsp._domainkey.{{ mail_domain }}.  TXT "dkim=discardable"
-
-_submission._tcp.{{ mail_domain }}.  SRV 0 1 587 {{ mail_domain }}.
-_submissions._tcp.{{ mail_domain }}. SRV 0 1 465 {{ mail_domain }}.
-_imap._tcp.{{ mail_domain }}.        SRV 0 1 143 {{ mail_domain }}.
-_imaps._tcp.{{ mail_domain }}.       SRV 0 1 993 {{ mail_domain }}.

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -15,10 +15,27 @@ from pathlib import Path
 import pyinfra
 from chatmaild.config import read_config, write_initial_config
 from packaging import version
-from termcolor import colored
 
 from . import dns, remote
-from .sshexec import LocalExec, SSHExec
+from .lxc.cli import (
+    lxc_start_cmd,
+    lxc_start_cmd_options,
+    lxc_status_cmd,
+    lxc_status_cmd_options,
+    lxc_stop_cmd,
+    lxc_stop_cmd_options,
+    lxc_test_cmd,
+    lxc_test_cmd_options,
+)
+from .lxc.incus import DNSConfigurationError
+from .sshexec import (
+    LocalExec,
+    SSHExec,
+    resolve_host_from_ssh_config,
+    resolve_key_from_ssh_config,
+)
+from .util import Out
+from .www import main as webdev_main
 
 #
 # cmdeploy sub commands and options
@@ -82,18 +99,21 @@ def run_cmd_options(parser):
         help="disable checks nslookup for dns",
     )
     add_ssh_host_option(parser)
+    add_ssh_config_option(parser)
 
 
 def run_cmd(args, out):
     """Deploy chatmail services on the remote server."""
 
     ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
-    sshexec = get_sshexec(ssh_host)
+    sshexec = get_sshexec(ssh_host, ssh_config=args.ssh_config)
     require_iroh = args.config.enable_iroh_relay
     strict_tls = args.config.tls_cert_mode == "acme"
     if not args.dns_check_disabled:
         remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
-        if not dns.check_initial_remote_data(remote_data, strict_tls=strict_tls, print=out.red):
+        if not dns.check_initial_remote_data(
+            remote_data, strict_tls=strict_tls, print=out.red
+        ):
             return 1
 
     env = os.environ.copy()
@@ -104,10 +124,24 @@ def run_cmd(args, out):
     if not args.dns_check_disabled:
         env["CHATMAIL_ADDR_V4"] = remote_data.get("A") or ""
         env["CHATMAIL_ADDR_V6"] = remote_data.get("AAAA") or ""
+    env["DEBIAN_FRONTEND"] = "noninteractive"
+    env["TERM"] = "linux"
     deploy_path = importlib.resources.files(__package__).joinpath("run.py").resolve()
     pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
 
     cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y"
+    ssh_config = args.ssh_config
+    if ssh_config:
+        ssh_config = str(Path(ssh_config).resolve())
+
+        # Use pyinfra's native SSH data keys to configure the connection directly
+        # rather than relying on paramiko config parsing (see also sshexec.py)
+        ip = resolve_host_from_ssh_config(ssh_host, ssh_config)
+        key = resolve_key_from_ssh_config(ssh_host, ssh_config)
+        data_args = f"--data ssh_hostname={ip} --data ssh_known_hosts_file=/dev/null"
+        if key:
+            data_args += f" --data ssh_key={key}"
+        cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y {data_args}"
     if ssh_host in ["localhost", "@docker"]:
         if ssh_host == "@docker":
             env["CHATMAIL_NOPORTCHECK"] = "True"
@@ -119,10 +153,17 @@ def run_cmd(args, out):
         return 1
 
     try:
-        out.check_call(cmd, env=env)
+        ret = out.shell(cmd, env=env)
+        if ret:
+            out.red("Deploy failed")
+            return 1
         if args.website_only:
             out.green("Website deployment completed.")
-        elif not args.dns_check_disabled and strict_tls and not remote_data["acme_account_url"]:
+        elif (
+            not args.dns_check_disabled
+            and strict_tls
+            and not remote_data["acme_account_url"]
+        ):
             out.red("Deploy completed but letsencrypt not configured")
             out.red("Run 'cmdeploy run' again")
         else:
@@ -139,15 +180,16 @@ def dns_cmd_options(parser):
         dest="zonefile",
         type=pathlib.Path,
         default=None,
-        help="write out a zonefile",
+        help="write DNS records in standard BIND format to the given file",
     )
     add_ssh_host_option(parser)
+    add_ssh_config_option(parser)
 
 
 def dns_cmd(args, out):
     """Check DNS entries and optionally generate dns zone file."""
     ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
-    sshexec = get_sshexec(ssh_host, verbose=args.verbose)
+    sshexec = get_sshexec(ssh_host, verbose=args.verbose, ssh_config=args.ssh_config)
     tls_cert_mode = args.config.tls_cert_mode
     strict_tls = tls_cert_mode == "acme"
     remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
@@ -178,13 +220,14 @@ def dns_cmd(args, out):
 
 def status_cmd_options(parser):
     add_ssh_host_option(parser)
+    add_ssh_config_option(parser)
 
 
 def status_cmd(args, out):
     """Display status for online chatmail instance."""
 
     ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
-    sshexec = get_sshexec(ssh_host, verbose=args.verbose)
+    sshexec = get_sshexec(ssh_host, verbose=args.verbose, ssh_config=args.ssh_config)
 
     out.green(f"chatmail domain: {args.config.mail_domain}")
     if args.config.privacy_mail:
@@ -204,14 +247,14 @@ def test_cmd_options(parser):
         help="also run slow tests",
     )
     add_ssh_host_option(parser)
+    add_ssh_config_option(parser)
 
 
 def test_cmd(args, out):
     """Run local and online tests for chatmail deployment."""
 
     env = os.environ.copy()
-    if args.ssh_host:
-        env["CHATMAIL_SSH"] = args.ssh_host
+    env["CHATMAIL_INI"] = str(args.inipath.resolve())
 
     pytest_path = shutil.which("pytest")
     pytest_args = [
@@ -225,7 +268,11 @@ def test_cmd(args, out):
     ]
     if args.slow:
         pytest_args.append("--slow")
-    ret = out.run_ret(pytest_args, env=env)
+    if args.ssh_host:
+        pytest_args.extend(["--ssh-host", args.ssh_host])
+    if args.ssh_config:
+        pytest_args.extend(["--ssh-config", str(Path(args.ssh_config).resolve())])
+    ret = out.shell(" ".join(pytest_args), env=env)
     return ret
 
 
@@ -262,8 +309,8 @@ def fmt_cmd(args, out):
     format_args.extend(sources)
     check_args.extend(sources)
 
-    out.check_call(" ".join(format_args), quiet=not args.verbose)
-    out.check_call(" ".join(check_args), quiet=not args.verbose)
+    out.shell(" ".join(format_args), quiet=not args.verbose)
+    out.shell(" ".join(check_args), quiet=not args.verbose)
 
 
 def bench_cmd(args, out):
@@ -276,9 +323,7 @@ def bench_cmd(args, out):
 
 def webdev_cmd(args, out):
     """Run local web development loop for static web pages."""
-    from .www import main
-
-    main()
+    webdev_main()
 
 
 #
@@ -286,32 +331,6 @@ def webdev_cmd(args, out):
 #
 
 
-class Out:
-    """Convenience output printer providing coloring."""
-
-    def red(self, msg, file=sys.stderr):
-        print(colored(msg, "red"), file=file)
-
-    def green(self, msg, file=sys.stderr):
-        print(colored(msg, "green"), file=file)
-
-    def __call__(self, msg, red=False, green=False, file=sys.stdout):
-        color = "red" if red else ("green" if green else None)
-        print(colored(msg, color), file=file)
-
-    def check_call(self, arg, env=None, quiet=False):
-        if not quiet:
-            self(f"[$ {arg}]", file=sys.stderr)
-        return subprocess.check_call(arg, shell=True, env=env)
-
-    def run_ret(self, args, env=None, quiet=False):
-        if not quiet:
-            cmdstring = " ".join(args)
-            self(f"[$ {cmdstring}]", file=sys.stderr)
-        proc = subprocess.run(args, env=env, check=False)
-        return proc.returncode
-
-
 def add_ssh_host_option(parser):
     parser.add_argument(
         "--ssh-host",
@@ -321,6 +340,16 @@ def add_ssh_host_option(parser):
     )
 
 
+def add_ssh_config_option(parser):
+    parser.add_argument(
+        "--ssh-config",
+        dest="ssh_config",
+        type=Path,
+        default=None,
+        help="Path to an SSH config file (e.g. lxconfigs/ssh-config).",
+    )
+
+
 def add_config_option(parser):
     parser.add_argument(
         "--config",
@@ -330,25 +359,26 @@ def add_config_option(parser):
         type=Path,
         help="path to the chatmail.ini file",
     )
-    parser.add_argument(
-        "--verbose",
-        "-v",
-        dest="verbose",
-        action="store_true",
-        default=False,
-        help="provide verbose logging",
-    )
 
 
-def add_subcommand(subparsers, func):
+def add_subcommand(subparsers, func, add_config=True):
     name = func.__name__
     assert name.endswith("_cmd")
-    name = name[:-4]
+    name = name[:-4].replace("_", "-")
     doc = func.__doc__.strip()
     help = doc.split("\n")[0].strip(".")
     p = subparsers.add_parser(name, description=doc, help=help)
     p.set_defaults(func=func)
-    add_config_option(p)
+    if add_config:
+        add_config_option(p)
+    p.add_argument(
+        "-v",
+        "--verbose",
+        dest="verbose",
+        action="count",
+        default=0,
+        help="increase verbosity (can be repeated: -v, -vv)",
+    )
     return p
 
 
@@ -357,45 +387,60 @@ Setup your chatmail server configuration and
 deploy it via SSH to your remote location.
 """
 
+# Explicit subcommand registry: (cmd_func, options_func_or_None, needs_config).
+# LXC commands don't need a chatmail.ini (no config); all others do.
+SUBCOMMANDS = [
+    (init_cmd, init_cmd_options, True),
+    (run_cmd, run_cmd_options, True),
+    (dns_cmd, dns_cmd_options, True),
+    (status_cmd, status_cmd_options, True),
+    (test_cmd, test_cmd_options, True),
+    (fmt_cmd, fmt_cmd_options, True),
+    (bench_cmd, None, True),
+    (webdev_cmd, None, True),
+    (lxc_start_cmd, lxc_start_cmd_options, False),
+    (lxc_stop_cmd, lxc_stop_cmd_options, False),
+    (lxc_status_cmd, lxc_status_cmd_options, False),
+    (lxc_test_cmd, lxc_test_cmd_options, False),
+]
+
 
 def get_parser():
     """Return an ArgumentParser for the 'cmdeploy' CLI"""
 
     parser = argparse.ArgumentParser(description=description.strip())
+    parser.set_defaults(func=None, inipath=None)
     subparsers = parser.add_subparsers(title="subcommands")
 
-    # find all subcommands in the module namespace
-    glob = globals()
-    for name, func in glob.items():
-        if name.endswith("_cmd"):
-            subparser = add_subcommand(subparsers, func)
-            addopts = glob.get(name + "_options")
-            if addopts is not None:
-                addopts(subparser)
+    for func, addopts, needs_config in SUBCOMMANDS:
+        subparser = add_subcommand(subparsers, func, add_config=needs_config)
+        if addopts is not None:
+            addopts(subparser)
 
     return parser
 
 
-def get_sshexec(ssh_host: str, verbose=True):
+def get_sshexec(ssh_host: str, verbose=True, ssh_config=None):
     if ssh_host in ["localhost", "@local"]:
         return LocalExec(verbose, docker=False)
     elif ssh_host == "@docker":
         return LocalExec(verbose, docker=True)
     if verbose:
         print(f"[ssh] login to {ssh_host}")
-    return SSHExec(ssh_host, verbose=verbose)
+    return SSHExec(ssh_host, verbose=verbose, ssh_config=ssh_config)
 
 
 def main(args=None):
     """Provide main entry point for 'cmdeploy' CLI invocation."""
     parser = get_parser()
     args = parser.parse_args(args=args)
-    if not hasattr(args, "func"):
+    if args.func is None:
         return parser.parse_args(["-h"])
 
-    out = Out()
+    out = Out(verbosity=args.verbose)
     kwargs = {}
-    if args.func.__name__ not in ("init_cmd", "fmt_cmd"):
+
+    if args.inipath is not None and args.func.__name__ not in ("init_cmd", "fmt_cmd"):
         if not args.inipath.exists():
             out.red(f"expecting {args.inipath} to exist, run init first?")
             raise SystemExit(1)
@@ -410,6 +455,9 @@ def main(args=None):
         if res is None:
             res = 0
         return res
+    except DNSConfigurationError as exc:
+        out.red(str(exc))
+        return 1
     except KeyboardInterrupt:
         out.red("KeyboardInterrupt")
         sys.exit(130)

cmdeploy/src/cmdeploy/deployers.py

@@ -17,13 +17,12 @@ from pyinfra.facts.files import Sha256File
 from pyinfra.facts.systemd import SystemdEnabled
 from pyinfra.operations import apt, files, pip, server, systemd
 
-from cmdeploy.cmdeploy import Out
-
 from .acmetool import AcmetoolDeployer
 from .basedeploy import (
     Deployer,
     Deployment,
     activate_remote_units,
+    blocked_service_startup,
     configure_remote_units,
     get_resource,
     has_systemd,
@@ -36,6 +35,7 @@ from .nginx.deployer import NginxDeployer
 from .opendkim.deployer import OpendkimDeployer
 from .postfix.deployer import PostfixDeployer
 from .selfsigned.deployer import SelfSignedTlsDeployer
+from .util import Out, get_version_string
 from .www import build_webpages, find_merge_conflict, get_paths
 
 
@@ -149,33 +149,16 @@ class UnboundDeployer(Deployer):
         self.need_restart = False
 
     def install(self):
-        # Run local DNS resolver `unbound`.
-        # `resolvconf` takes care of setting up /etc/resolv.conf
-        # to use 127.0.0.1 as the resolver.
+        # Run local DNS resolver `unbound`. `resolvconf` takes care of
+        # setting up /etc/resolv.conf to use 127.0.0.1 as the resolver.
 
-        #
-        # On an IPv4-only system, if unbound is started but not
-        # configured, it causes subsequent steps to fail to resolve hosts.
-        # Here, we use policy-rc.d to prevent unbound from starting up
-        # on initial install.  Later, we will configure it and start it.
-        #
-        # For documentation about policy-rc.d, see:
-        # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
-        #
-        files.put(
-            src=get_resource("policy-rc.d"),
-            dest="/usr/sbin/policy-rc.d",
-            user="root",
-            group="root",
-            mode="755",
-        )
-
-        apt.packages(
-            name="Install unbound",
-            packages=["unbound", "unbound-anchor", "dnsutils"],
-        )
-
-        files.file("/usr/sbin/policy-rc.d", present=False)
+        # On an IPv4-only system, if unbound is started but not configured,
+        # it causes subsequent steps to fail to resolve hosts.
+        with blocked_service_startup():
+            apt.packages(
+                name="Install unbound",
+                packages=["unbound", "unbound-anchor", "dnsutils"],
+            )
 
     def configure(self):
         server.shell(
@@ -271,8 +254,14 @@ class WebsiteDeployer(Deployer):
                     logger.warning("Web page build failed, skipping website deployment")
                     return
             # if it is not a hugo page, upload it as is
-            files.rsync(
-                f"{www_path}/", "/var/www/html", flags=["-avz", "--chown=www-data"]
+            # pyinfra files.rsync (experimental) causes problems with ssh-config configuration
+            # the stable files.sync should do
+            files.sync(
+                src=str(www_path),
+                dest="/var/www/html",
+                user="www-data",
+                group="www-data",
+                delete=True,
             )
 
 
@@ -336,12 +325,12 @@ class TurnDeployer(Deployer):
     def install(self):
         (url, sha256sum) = {
             "x86_64": (
-                "https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-x86_64-linux",
-                "841e527c15fdc2940b0469e206188ea8f0af48533be12ecb8098520f813d41e4",
+                "https://github.com/chatmail/chatmail-turn/releases/download/v0.4/chatmail-turn-x86_64-linux",
+                "1ec1f5c50122165e858a5a91bcba9037a28aa8cb8b64b8db570aa457c6141a8a",
             ),
             "aarch64": (
-                "https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-aarch64-linux",
-                "a5fc2d06d937b56a34e098d2cd72a82d3e89967518d159bf246dc69b65e81b42",
+                "https://github.com/chatmail/chatmail-turn/releases/download/v0.4/chatmail-turn-aarch64-linux",
+                "0fb3e792419494e21ecad536464929dba706bb2c88884ed8f1788141d26fc756",
             ),
         }[host.get_fact(facts.server.Arch)]
 
@@ -474,8 +463,9 @@ class ChatmailDeployer(Deployer):
         ("iroh", None, None),
     ]
 
-    def __init__(self, mail_domain):
-        self.mail_domain = mail_domain
+    def __init__(self, config):
+        self.config = config
+        self.mail_domain = config.mail_domain
 
     def install(self):
         files.put(
@@ -498,12 +488,19 @@ class ChatmailDeployer(Deployer):
             name="Install rsync",
             packages=["rsync"],
         )
-        apt.packages(
-            name="Ensure cron is installed",
-            packages=["cron"],
-        )
 
     def configure(self):
+        # Ensure the per-domain mailbox directory exists before
+        # chatmail-metadata starts (it crashes without it).
+        files.directory(
+            name="Ensure vmail mailbox directory exists",
+            path=f"/home/vmail/mail/{self.mail_domain}",
+            user="vmail",
+            group="vmail",
+            mode="700",
+            present=True,
+        )
+
         # This file is used by auth proxy.
         # https://wiki.debian.org/EtcMailName
         server.shell(
@@ -513,6 +510,15 @@ class ChatmailDeployer(Deployer):
             ],
         )
 
+        files.directory(
+            name=f"Ensure mailboxes directory {self.config.mailboxes_dir} exists",
+            path=str(self.config.mailboxes_dir),
+            user="vmail",
+            group="vmail",
+            mode="700",
+            present=True,
+        )
+
 
 class FcgiwrapDeployer(Deployer):
     def install(self):
@@ -532,17 +538,9 @@ class FcgiwrapDeployer(Deployer):
 
 class GithashDeployer(Deployer):
     def activate(self):
-        try:
-            git_hash = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode()
-        except Exception:
-            git_hash = "unknown\n"
-        try:
-            git_diff = subprocess.check_output(["git", "diff"]).decode()
-        except Exception:
-            git_diff = ""
         files.put(
             name="Upload chatmail relay git commit hash",
-            src=StringIO(git_hash + git_diff),
+            src=StringIO(get_version_string()),
             dest="/etc/chatmail-version",
             mode="700",
         )
@@ -586,11 +584,17 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
         )
 
     # Check if mtail_address interface is available (if configured)
-    if config.mtail_address and config.mtail_address not in ('127.0.0.1', '::1', 'localhost'):
+    if config.mtail_address and config.mtail_address not in (
+        "127.0.0.1",
+        "::1",
+        "localhost",
+    ):
         ipv4_addrs = host.get_fact(hardware.Ipv4Addrs)
         all_addresses = [addr for addrs in ipv4_addrs.values() for addr in addrs]
         if config.mtail_address not in all_addresses:
-            Out().red(f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n")
+            Out().red(
+                f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n"
+            )
             exit(1)
 
     if not os.environ.get("CHATMAIL_NOPORTCHECK"):
@@ -633,7 +637,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
     tls_deployer = get_tls_deployer(config, mail_domain)
 
     all_deployers = [
-        ChatmailDeployer(mail_domain),
+        ChatmailDeployer(config),
         LegacyRemoveDeployer(),
         FiltermailDeployer(),
         JournaldDeployer(),

cmdeploy/src/cmdeploy/dns.py

@@ -1,11 +1,26 @@
 import datetime
-import importlib
-
-from jinja2 import Template
 
 from . import remote
 
 
+def parse_zone_records(text):
+    """Yield ``(name, ttl, rtype, rdata)`` from standard BIND-format text.
+
+    Skips comment lines (starting with ``;``) and blank lines.
+    Each record line must have the format ``name TTL IN type rdata``.
+    """
+    for raw_line in text.splitlines():
+        line = raw_line.strip()
+        if not line or line.startswith(";"):
+            continue
+        try:
+            name, ttl, _in, rtype, rdata = line.split(None, 4)
+        except ValueError:
+            raise ValueError(f"Bad zone record line: {line!r}") from None
+        name = name.rstrip(".")
+        yield name, ttl, rtype.upper(), rdata
+
+
 def get_initial_remote_data(sshexec, mail_domain):
     return sshexec.logged(
         call=remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=mail_domain)
@@ -31,13 +46,43 @@ def get_filled_zone_file(remote_data):
     if not sts_id:
         remote_data["sts_id"] = datetime.datetime.now().strftime("%Y%m%d%H%M")
 
-    template = importlib.resources.files(__package__).joinpath("chatmail.zone.j2")
-    content = template.read_text()
-    zonefile = Template(content).render(**remote_data)
-    lines = [x.strip() for x in zonefile.split("\n") if x.strip()]
+    d = remote_data["mail_domain"]
+
+    def rec(name, rtype, rdata, ttl=3600):
+        return f"{name:<40} {ttl:<6} IN  {rtype:<5}  {rdata}"
+
+    lines = ["; Required DNS entries"]
+    if remote_data.get("A"):
+        lines.append(rec(f"{d}.", "A", remote_data["A"]))
+    if remote_data.get("AAAA"):
+        lines.append(rec(f"{d}.", "AAAA", remote_data["AAAA"]))
+    lines.append(rec(f"{d}.", "MX", f"10 {d}."))
+    if remote_data.get("strict_tls"):
+        lines.append(
+            rec(f"_mta-sts.{d}.", "TXT", f'"v=STSv1; id={remote_data["sts_id"]}"')
+        )
+        lines.append(rec(f"mta-sts.{d}.", "CNAME", f"{d}."))
+    lines.append(rec(f"www.{d}.", "CNAME", f"{d}."))
+    lines.append(remote_data["dkim_entry"])
     lines.append("")
-    zonefile = "\n".join(lines)
-    return zonefile
+    lines.append("; Recommended DNS entries")
+    lines.append(rec(f"{d}.", "TXT", '"v=spf1 a ~all"'))
+    lines.append(rec(f"_dmarc.{d}.", "TXT", '"v=DMARC1;p=reject;adkim=s;aspf=s"'))
+    if remote_data.get("acme_account_url"):
+        lines.append(
+            rec(
+                f"{d}.",
+                "CAA",
+                f'0 issue "letsencrypt.org;accounturi={remote_data["acme_account_url"]}"',
+            )
+        )
+    lines.append(rec(f"_adsp._domainkey.{d}.", "TXT", '"dkim=discardable"'))
+    lines.append(rec(f"_submission._tcp.{d}.", "SRV", f"0 1 587 {d}."))
+    lines.append(rec(f"_submissions._tcp.{d}.", "SRV", f"0 1 465 {d}."))
+    lines.append(rec(f"_imap._tcp.{d}.", "SRV", f"0 1 143 {d}."))
+    lines.append(rec(f"_imaps._tcp.{d}.", "SRV", f"0 1 993 {d}."))
+    lines.append("")
+    return "\n".join(lines)
 
 
 def check_full_zone(sshexec, remote_data, out, zonefile) -> int:
@@ -53,18 +98,19 @@ def check_full_zone(sshexec, remote_data, out, zonefile) -> int:
     if required_diff:
         out.red("Please set required DNS entries at your DNS provider:\n")
         for line in required_diff:
-            out(line)
-        out("")
+            out.print(line)
+        out.print()
         returncode = 1
     if remote_data.get("dkim_entry") in required_diff:
-        out(
-            "If the DKIM entry above does not work with your DNS provider, you can try this one:\n"
+        out.print(
+            "If the DKIM entry above does not work with your DNS provider,"
+            " you can try this one:\n"
         )
-        out(remote_data.get("web_dkim_entry") + "\n")
+        out.print(remote_data.get("web_dkim_entry") + "\n")
     if recommended_diff:
-        out("WARNING: these recommended DNS entries are not set:\n")
+        out.print("WARNING: these recommended DNS entries are not set:\n")
         for line in recommended_diff:
-            out(line)
+            out.print(line)
 
     if not (recommended_diff or required_diff):
         out.green("Great! All your DNS entries are verified and correct.")

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -1,10 +1,9 @@
-import os
 import urllib.request
 
 from chatmaild.config import Config
 from pyinfra import host
 from pyinfra.facts.deb import DebPackages
-from pyinfra.facts.server import Arch, Sysctl
+from pyinfra.facts.server import Arch, Command, Sysctl
 from pyinfra.operations import apt, files, server, systemd
 
 from cmdeploy.basedeploy import (
@@ -38,9 +37,21 @@ class DovecotDeployer(Deployer):
     def install(self):
         arch = host.get_fact(Arch)
         with blocked_service_startup():
-            _install_dovecot_package("core", arch)
-            _install_dovecot_package("imapd", arch)
-            _install_dovecot_package("lmtpd", arch)
+            debs = []
+            for pkg in ("core", "imapd", "lmtpd"):
+                deb = _download_dovecot_package(pkg, arch)
+                if deb:
+                    debs.append(deb)
+            if debs:
+                deb_list = " ".join(debs)
+                server.shell(
+                    name="Install dovecot packages",
+                    commands=[
+                        f"dpkg --force-confdef --force-confold -i {deb_list} 2> /dev/null || true",
+                        "DEBIAN_FRONTEND=noninteractive apt-get -y --fix-broken install",
+                        f"dpkg --force-confdef --force-confold -i {deb_list}",
+                    ],
+                )
 
     def configure(self):
         configure_remote_units(self.config.mail_domain, self.units)
@@ -73,7 +84,8 @@ def _pick_url(primary, fallback):
         return fallback
 
 
-def _install_dovecot_package(package: str, arch: str):
+def _download_dovecot_package(package: str, arch: str):
+    """Download a dovecot .deb if needed, return its path (or None)."""
     arch = "amd64" if arch == "x86_64" else arch
     arch = "arm64" if arch == "aarch64" else arch
 
@@ -81,11 +93,11 @@ def _install_dovecot_package(package: str, arch: str):
     sha256 = DOVECOT_SHA256.get((package, arch))
     if sha256 is None:
         apt.packages(packages=[pkg_name])
-        return
+        return None
 
     installed_versions = host.get_fact(DebPackages).get(pkg_name, [])
     if DOVECOT_VERSION in installed_versions:
-        return
+        return None
 
     url_version = DOVECOT_VERSION.replace("+", "%2B")
     deb_base = f"{pkg_name}_{url_version}_{arch}.deb"
@@ -102,7 +114,7 @@ def _install_dovecot_package(package: str, arch: str):
         cache_time=60 * 60 * 24 * 365 * 10,  # never redownload the package
     )
 
-    apt.deb(name=f"Install {pkg_name}", src=deb_filename)
+    return deb_filename
 
 
 def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
@@ -140,19 +152,25 @@ def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
 
     # as per https://doc.dovecot.org/2.3/configuration_manual/os/
     # it is recommended to set the following inotify limits
-    if not os.environ.get("CHATMAIL_NOSYSCTL"):
-        for name in ("max_user_instances", "max_user_watches"):
-            key = f"fs.inotify.{name}"
-            if host.get_fact(Sysctl)[key] > 65535:
-                # Skip updating limits if already sufficient
-                # (enables running in incus containers where sysctl readonly)
-                continue
-            server.sysctl(
-                name=f"Change {key}",
-                key=key,
-                value=65535,
-                persist=True,
+    can_modify = host.get_fact(Command, "systemd-detect-virt -c || true") == "none"
+    for name in ("max_user_instances", "max_user_watches"):
+        key = f"fs.inotify.{name}"
+        value = host.get_fact(Sysctl)[key]
+        if value > 65534:
+            continue
+        if not can_modify:
+            print(
+                "\n!!!! refusing to attempt sysctl setting in shared-kernel containers\n"
+                f"!!!! dovecot: sysctl {key!r}={value}, should be >65534 for production setups\n"
+                "!!!!"
             )
+            continue
+        server.sysctl(
+            name=f"Change {key}",
+            key=key,
+            value=65535,
+            persist=True,
+        )
 
     timezone_env = files.line(
         name="Set TZ environment variable",

cmdeploy/src/cmdeploy/lxc/cli.py

@@ -0,0 +1,475 @@
+"""lxc-start/stop/status/test subcommands for testing with local containers."""
+
+import os
+import time
+
+from ..util import get_git_hash, get_version_string, shell
+from .incus import RELAY_IMAGE_ALIAS, Incus, RelayContainer
+
+RELAY_NAMES = ("test0", "test1")
+
+
+# -------------------------------------------------------------------
+# lxc-start
+# -------------------------------------------------------------------
+
+
+def lxc_start_cmd_options(parser):
+    _add_name_args(
+        parser,
+        help_text="User relay name(s) to create (default: test0).",
+    )
+    parser.add_argument(
+        "--ipv4-only",
+        dest="ipv4_only",
+        action="store_true",
+        help="Create an IPv4-only container.",
+    )
+    parser.add_argument(
+        "--run",
+        action="store_true",
+        help="Run 'cmdeploy run' on each container after starting it.",
+    )
+
+
+def lxc_start_cmd(args, out):
+    """Create/Ensure and start LXC relay and DNS containers."""
+
+    with out.section("Preparing container setup"):
+        _lxc_start_cmd(args, out)
+
+
+def _lxc_start_cmd(args, out):
+    ix = Incus(out)
+    sub = out.new_prefixed_out()
+    out.green("Ensuring base image ...")
+    ix.ensure_base_image()
+    out.green("Ensuring DNS container (ns-localchat) ...")
+    dns_ct = ix.get_dns_container()
+    dns_ct.ensure()
+    sub.print(f"DNS container IP: {dns_ct.ipv4}")
+
+    names = args.names if args.names else RELAY_NAMES
+    relays = list(ix.get_container(n) for n in names)
+    for ct in relays:
+        out.green(f"Ensuring container {ct.name!r} ({ct.domain}) ...")
+        ct.ensure()
+        ip = ct.ipv4
+
+        sub.print("Configuring container hostname ...")
+        ct.configure_hosts(ip)
+
+        sub.print(f"Writing {ct.ini.name} ...")
+        ct.write_ini(disable_ipv6=args.ipv4_only)
+        sub.print(f"Config: {ct.ini}")
+        if args.ipv4_only:
+            ct.disable_ipv6()
+            ipv6 = None
+        else:
+            output = ct.bash(
+                "ip -6 addr show scope global -deprecated"
+                " | grep -oP '(?<=inet6 )[^/]+'",
+                check=False,
+            )
+            ipv6 = output.strip() if output else None
+        sub.print(f"{_format_addrs(ip, ipv6)}")
+
+        sub.green(f"Container {ct.name!r} ready: {ct.domain} -> {ip}")
+        out.print()
+
+    # Reset DNS zones only for the containers we just started
+    started_cnames = {ct.name for ct in relays}
+    managed = ix.list_managed()
+    started = [c for c in managed if c["name"] in started_cnames]
+
+    if started:
+        out.print(
+            f"Resetting DNS zones for {len(started)} domain(s) (A + AAAA records) ..."
+        )
+        dns_ct.reset_dns_records(dns_ct.ipv4, started)
+
+        for ct in relays:
+            if ct.name in started_cnames:
+                sub.print(f"Configuring DNS in {ct.name} ...")
+                ct.configure_dns(dns_ct.ipv4)
+
+    # Generate the unified SSH config
+    out.green("Writing ssh-config ...")
+    ssh_cfg = ix.write_ssh_config()
+    sub.print(f"{ssh_cfg}")
+
+    # Verify SSH via the generated config
+    for ct in relays:
+        sub.print(f"Verifying SSH to {ct.name} via ssh-config ...")
+        if ct.verify_ssh(ssh_cfg):
+            sub.print(f"SSH OK: ssh -F lxconfigs/ssh-config {ct.domain}")
+        else:
+            sub.red(f"WARNING: SSH verification failed for {ct.name}")
+
+    # Print integration suggestions
+    ssh_cfg = ix.ssh_config_path
+    if not ix.check_ssh_include():
+        sub.green(
+            "\n(Optional) To use containers from any SSH client, add to ~/.ssh/config:"
+        )
+        sub.green(f"    Include {ssh_cfg}")
+
+    # Optionally run cmdeploy run + dns on each relay
+    if args.run:
+        for ct in relays:
+            with out.section(f"cmdeploy run: {ct.sname} ({ct.domain})"):
+                ret = _run_cmdeploy("run", ct, ix, out, extra=["--skip-dns-check"])
+                if ret:
+                    out.red(f"Deploy to {ct.sname} failed (exit {ret})")
+                    return ret
+
+        with out.section("loading DNS zones"):
+            for ct in relays:
+                ret = _run_cmdeploy(
+                    "dns", ct, ix, out,
+                    extra=["--zonefile", str(ct.zone)],
+                )
+                if ret:
+                    out.red(f"DNS for {ct.sname} failed (exit {ret})")
+                    return ret
+                if ct.zone.exists():
+                    dns_ct.set_dns_records(ct.zone.read_text())
+                out.print(f"Restarting filtermail-incoming on {ct.name}")
+                ct.bash("systemctl restart filtermail-incoming")
+
+
+# -------------------------------------------------------------------
+# lxc-stop
+# -------------------------------------------------------------------
+
+
+def lxc_stop_cmd_options(parser):
+    parser.add_argument(
+        "--destroy",
+        action="store_true",
+        help="Delete containers and their config files after stopping.",
+    )
+    parser.add_argument(
+        "--destroy-all",
+        dest="destroy_all",
+        action="store_true",
+        help="Like --destroy, but also remove the ns-localchat DNS container.",
+    )
+    _add_name_args(
+        parser,
+        help_text="Container name(s) to stop (default: test0 + test1).",
+    )
+
+
+def lxc_stop_cmd(args, out):
+    """Stop (and optionally destroy) local LXC relay containers."""
+    ix = Incus(out)
+    names = args.names or RELAY_NAMES
+    destroy = args.destroy or args.destroy_all
+
+    for ct in map(ix.get_container, names):
+        if destroy:
+            out.green(f"Destroying container {ct.name!r} ...")
+            ct.destroy()
+        else:
+            out.green(f"Stopping container {ct.name!r} ...")
+            ct.stop(force=True)
+
+    if args.destroy_all:
+        dns_ct = ix.get_dns_container()
+        out.green(f"Destroying DNS container {dns_ct.name!r} ...")
+        dns_ct.destroy()
+        ix.delete_images()
+
+    if destroy:
+        ix.write_ssh_config()
+        out.green("LXC containers destroyed.")
+    else:
+        out.green("LXC containers stopped.")
+
+
+# -------------------------------------------------------------------
+# lxc-test
+# -------------------------------------------------------------------
+
+
+def lxc_test_cmd_options(parser):
+    parser.add_argument(
+        "--one",
+        action="store_true",
+        help="Only deploy and test against test0 (skip test1).",
+    )
+
+
+def lxc_test_cmd(args, out):
+    """Run full LXC pipeline: start, deploy, DNS, zone files, and tests.
+
+    All commands run directly on the host using
+    ``--ssh-config lxconfigs/ssh-config`` for SSH access.
+    """
+    ix = Incus(out)
+    t_total = time.time()
+    relay_names = list(RELAY_NAMES)
+    if args.one:
+        relay_names = relay_names[:1]
+
+    local_hash = get_git_hash()
+
+    # Per-relay: start, deploy, then snapshot the first relay as a
+    # reusable image so the second relay launches pre-deployed.
+    ipv4_only_flags = {RELAY_NAMES[0]: False, RELAY_NAMES[1]: True}
+
+    for ct in map(ix.get_container, relay_names):
+        name = ct.sname
+        ipv4_only = ipv4_only_flags.get(name, False)
+        v_flag = " -" + "v" * out.verbosity if out.verbosity > 0 else ""
+        start_cmd = f"cmdeploy lxc-start{v_flag} {name}"
+        if ipv4_only:
+            start_cmd += " --ipv4-only"
+        with out.section(f"cmdeploy lxc-start: {name}"):
+            ret = out.shell(start_cmd, cwd=str(ix.project_root))
+            if ret:
+                return ret
+
+        status = _deploy_status(ct, local_hash, ix)
+        with out.section(f"cmdeploy run: {name}"):
+            if "IN-SYNC" in status:
+                out.print(f"{name} is {status}, skipping")
+            else:
+                ret = _run_cmdeploy("run", ct, ix, out, extra=["--skip-dns-check"])
+                if ret:
+                    out.red(f"Deploy to {name} failed (exit {ret})")
+                    return ret
+
+        # Snapshot the first relay so subsequent ones launch pre-deployed
+        if not ix.find_image([RELAY_IMAGE_ALIAS]):
+            with out.section("lxc-test: caching relay image"):
+                ct.publish_as_relay_image()
+
+    for ct in map(ix.get_container, relay_names):
+        with out.section(f"cmdeploy dns: {ct.sname} ({ct.domain})"):
+            ret = _run_cmdeploy("dns", ct, ix, out, extra=["--zonefile", str(ct.zone)])
+            if ret:
+                out.red(f"DNS for {ct.sname} failed (exit {ret})")
+                return ret
+
+    with out.section(f"lxc-test: loading DNS zones {' & '.join(relay_names)}"):
+        dns_ct = ix.get_dns_container()
+        for ct in map(ix.get_container, relay_names):
+            if ct.zone.exists():
+                zone_data = ct.zone.read_text()
+                out.print(f"Loading {ct.zone} into PowerDNS ...")
+                dns_ct.set_dns_records(zone_data)
+
+        # Restart filtermail so its in-process DNS cache
+        # does not hold stale negative DKIM responses
+        # from before the zones were loaded.
+        for ct in map(ix.get_container, relay_names):
+            out.print(f"Restarting filtermail-incoming on {ct.name} ...")
+            ct.bash("systemctl restart filtermail-incoming")
+
+    with out.section("cmdeploy test"):
+        first = ix.get_container(relay_names[0])
+        env = None
+        if len(relay_names) > 1:
+            env = os.environ.copy()
+            env["CHATMAIL_DOMAIN2"] = ix.get_container(relay_names[1]).domain
+        ret = _run_cmdeploy("test", first, ix, out, **({"env": env} if env else {}))
+        if ret:
+            out.red(f"Tests failed (exit {ret})")
+            return ret
+
+    elapsed = time.time() - t_total
+    out.section_line(f"lxc-test complete ({elapsed:.1f}s)")
+    if out.section_timings:
+        out.print("Section timings:")
+        for name, secs in out.section_timings:
+            out.print(f"  {name:.<50s} {secs:5.1f}s")
+        out.print(f"  {'total':.<50s} {elapsed:5.1f}s")
+    out.section_timings.clear()
+    return 0
+
+
+# -------------------------------------------------------------------
+# lxc-status
+# -------------------------------------------------------------------
+
+
+def lxc_status_cmd_options(parser):
+    pass
+
+
+def lxc_status_cmd(args, out):
+    """Show status of local LXC chatmail containers."""
+    ix = Incus(out)
+    containers = ix.list_managed()
+    if not containers:
+        out.red("No LXC containers found.  Run 'cmdeploy lxc-start' first.")
+        return 1
+
+    local_hash = get_git_hash()
+
+    # Get storage pool path for display
+    storage_path = None
+    data = ix.run_json(["storage", "show", "default"], check=False)
+    if data:
+        storage_path = data.get("config", {}).get("source")
+    msg = "Container status"
+    if storage_path:
+        msg += f": {storage_path}"
+    out.section_line(msg)
+
+    dns_ip = None
+    for c in containers:
+        _print_container_status(out, c, ix, local_hash)
+        if c["name"] == ix.get_dns_container().name:
+            dns_ip = c["ip"]
+
+    out.section_line("Host ssh and DNS configuration")
+    _print_ssh_status(out, ix)
+    _print_dns_forwarding_status(out, dns_ip)
+    return 0
+
+
+def _print_container_status(out, c, ix, local_hash):
+    """Print name/status, domain/IPs, and RAM for one container."""
+    cname = c["name"]
+    is_running = c.get("status") == "Running"
+    ct = ix.get_container(cname)
+
+    # First line: name + running/STOPPED + deploy status
+    if not is_running:
+        tag = "STOPPED"
+    elif not isinstance(ct, RelayContainer):
+        tag = "running"
+    else:
+        tag = f"running {_deploy_status(ct, local_hash, ix)}"
+    out.print(f"{cname:20s} {tag}")
+
+    # Second line: domain, IPv4, IPv6
+    domain = c.get("domain", "")
+    ip = c.get("ip") or "?"
+    ipv6 = c.get("ipv6")
+    out.print(f"{domain:20s} {_format_addrs(ip, ipv6)}")
+
+    # Third line: RAM (RSS), config
+    detail_out = out.new_prefixed_out(" " * 21)
+    try:
+        used, total = ct.rss_mib()
+    except Exception:
+        ram_str = "RSS ?"
+    else:
+        ram_str = f"RSS {used}/{total} MiB ({used * 100 // total}%)"
+
+    if isinstance(ct, RelayContainer):
+        detail = f"{ram_str}, config: {os.path.relpath(ct.ini)}"
+    else:
+        detail = ram_str
+
+    detail_out.print(detail)
+    out.print()
+
+
+def _print_ssh_status(out, ix):
+    """Print SSH integration status."""
+    ssh_cfg = ix.ssh_config_path
+    if ix.check_ssh_include():
+        out.green("SSH: ~/.ssh/config includes lxconfigs/ssh-config ✓")
+    else:
+        out.red("SSH: ~/.ssh/config does NOT include lxconfigs/ssh-config")
+        sub = out.new_prefixed_out()
+        sub.print("Add to ~/.ssh/config:")
+        sub.print(f"    Include {ssh_cfg}")
+
+
+def _print_dns_forwarding_status(out, dns_ip):
+    """Print host DNS forwarding status for .localchat."""
+    sub = out.new_prefixed_out()
+    if not dns_ip:
+        out.red("DNS: ns-localchat container not found")
+        return
+    try:
+        rv = shell("resolvectl status incusbr0")
+        dns_ok = dns_ip in rv.stdout and "localchat" in rv.stdout
+    except Exception:
+        dns_ok = None
+    if dns_ok is True:
+        out.green(f"DNS: .localchat forwarding to {dns_ip} ✓")
+    elif dns_ok is False:
+        out.red("DNS: .localchat forwarding NOT configured")
+        sub.print("Run:")
+        sub.print(f"    sudo resolvectl dns incusbr0 {dns_ip}")
+        sub.print("    sudo resolvectl domain incusbr0 ~localchat")
+    else:
+        sub.print("DNS: .localchat forwarding status UNKNOWN")
+
+
+# -------------------------------------------------------------------
+# Internal helpers
+# -------------------------------------------------------------------
+
+
+def _format_addrs(ip, ipv6=None):
+    parts = [f"IPv4 {ip}"]
+    if ipv6:
+        parts.append(f"IPv6 {ipv6}")
+    return ", ".join(parts)
+
+
+def _deploy_status(ct, local_hash, ix):
+    """Return a human-readable deploy status string.
+
+    Compares the full deployed version (hash + diff) against
+    the local state built by :func:`~cmdeploy.util.get_version_string`.
+    """
+    deployed = ct.deployed_version()
+    if deployed is None:
+        return "NOT DEPLOYED"
+
+    # A container launched from the relay image has the same
+    # git hash but a different domain — always redeploy.
+    deployed_domain = ct.deployed_domain()
+    if deployed_domain and deployed_domain != ct.domain:
+        return f"DOMAIN-MISMATCH (deployed: {deployed_domain})"
+
+    deployed_lines = deployed.splitlines()
+    deployed_hash = deployed_lines[0] if deployed_lines else ""
+    short = deployed_hash[:12]
+
+    if not local_hash:
+        return f"UNKNOWN (deployed: {short})"
+
+    local_short = local_hash[:12]
+    if deployed_hash != local_hash:
+        return f"STALE (deployed: {short}, local: {local_short})"
+
+    # Hash matches — check for uncommitted diffs
+    local_version = get_version_string()
+    if deployed != local_version:
+        return f"DIRTY ({local_short}, undeployed changes)"
+
+    return f"IN-SYNC ({short})"
+
+
+def _add_name_args(parser, help_text):
+    parser.add_argument("names", nargs="*", metavar="NAME", help=help_text)
+
+
+def _run_cmdeploy(subcmd, ct, ix, out, extra=None, **kwargs):
+    """Run ``cmdeploy <subcmd>`` with standard --config/--ssh flags.
+
+    *ct* is a Container (uses ``ct.ini`` and ``ct.domain``).
+    Returns the subprocess exit code.
+    """
+    extra_str = " ".join(extra) if extra else ""
+    v_flag = " -" + "v" * out.verbosity if out.verbosity > 0 else ""
+    cmd = f"""
+        cmdeploy {subcmd}{v_flag}
+        --config {ct.ini}
+        --ssh-config {ix.ssh_config_path}
+        --ssh-host {ct.domain}
+        {extra_str}
+    """
+    if "cwd" not in kwargs:
+        kwargs["cwd"] = str(ix.project_root)
+    return out.shell(cmd, **kwargs)

cmdeploy/src/cmdeploy/lxc/incus.py

@@ -0,0 +1,768 @@
+"""Core Incus operations for local chatmail LXC containers."""
+
+import json
+import subprocess
+import textwrap
+import time
+from pathlib import Path
+
+from ..util import shell
+
+LABEL_KEY = "user.localchat-managed"
+SSH_KEY_NAME = "id_localchat"
+DOMAIN_SUFFIX = ".localchat"
+UPSTREAM_IMAGE = "images:debian/12"
+BASE_IMAGE_ALIAS = "localchat-base"
+BASE_SETUP_NAME = "localchat-base-setup"
+RELAY_IMAGE_ALIAS = "localchat-relay"
+
+DNS_CONTAINER_NAME = "ns-localchat"
+DNS_DOMAIN = "ns.localchat"
+
+
+class DNSConfigurationError(Exception):
+    """Raised when the DNS container is not reachable or not answering."""
+
+
+def _extract_ip(net_data, family="inet"):
+    """Extract the first global-scope IP of *family* from network state data.
+
+    *net_data* is the ``state.network`` dict from ``incus list --format=json``.
+    *family* is ``"inet"`` for IPv4 or ``"inet6"`` for IPv6.
+    Returns the address string, or None.
+    """
+    for iface_name, iface in net_data.items():
+        if iface_name == "lo":
+            continue
+        for addr in iface.get("addresses", []):
+            if addr["family"] == family and addr["scope"] == "global":
+                return addr["address"]
+    return None
+
+
+class Incus:
+    """Gateway for all Incus container operations.
+
+    Instantiated once per CLI command and passed around so that
+    all modules share a single entry point for Incus interactions.
+    """
+
+    def __init__(self, out):
+        self.out = out
+        self.project_root = Path(__file__).resolve().parent.parent.parent.parent.parent
+        self.lxconfigs_dir = self.project_root / "lxconfigs"
+        self.lxconfigs_dir.mkdir(exist_ok=True)
+        self.ssh_key_path = self.lxconfigs_dir / SSH_KEY_NAME
+        if not self.ssh_key_path.exists():
+            shell(
+                f"ssh-keygen -t ed25519 -f {self.ssh_key_path} -N '' -C localchat",
+                check=True,
+            )
+        self.ssh_config_path = self.lxconfigs_dir / "ssh-config"
+
+    def write_ssh_config(self):
+        """Write ``lxconfigs/ssh-config`` mapping all containers to their IPs.
+
+        Each Host block maps the container name, the domain name, and the
+        short relay name (e.g. ``_test0``) to the container's IP, using the
+        shared localchat SSH key.  Returns the path to the file.
+        """
+        containers = self.list_managed()
+        key_path = self.ssh_key_path
+        lines = ["# Auto-generated by cmdeploy lxc-start — do not edit\n"]
+        for c in containers:
+            hosts = [c["name"]]
+            domain = c.get("domain", "")
+            if domain and domain != c["name"]:
+                hosts.append(domain)
+                short = domain.split(".")[0]
+                if short and short not in hosts:
+                    hosts.append(short)
+            lines.append(f"\nHost {' '.join(hosts)}\n")
+            lines.append(f"    Hostname {c['ip']}\n")
+            lines.append("    User root\n")
+            lines.append(f"    IdentityFile {key_path}\n")
+            lines.append("    IdentitiesOnly yes\n")
+            lines.append("    StrictHostKeyChecking accept-new\n")
+            lines.append("    UserKnownHostsFile /dev/null\n")
+            lines.append("    LogLevel ERROR\n")
+        path = self.ssh_config_path
+        path.write_text("".join(lines))
+        return path
+
+    def check_ssh_include(self):
+        """Check if the user's ~/.ssh/config already includes our ssh-config."""
+        user_ssh_config = Path.home() / ".ssh" / "config"
+        if not user_ssh_config.exists():
+            return False
+        lines = user_ssh_config.read_text().splitlines()
+        target = f"include {self.ssh_config_path}".lower()
+        return any(line.strip().lower() == target for line in lines)
+
+    def get_host_nameservers(self):
+        """Return upstream nameservers found on the host."""
+        ns = []
+        for path in ["/run/systemd/resolve/resolv.conf", "/etc/resolv.conf"]:
+            p = Path(path)
+            if p.exists():
+                for line in p.read_text().splitlines():
+                    if line.strip().startswith("nameserver "):
+                        addr = line.split()[1]
+                        if addr not in ("127.0.0.1", "127.0.0.53", "::1"):
+                            if addr not in ns:
+                                ns.append(addr)
+                if ns:
+                    break
+        return ns
+
+    def run(self, args, check=True, capture=True, input=None):
+        """Run an incus command.
+
+        When *capture* is True and *verbosity* >= 1, output is streamed
+        to the terminal line-by-line while also being captured for
+        later return via result.stdout.
+        """
+        cmd = ["incus", "--quiet"] + list(args)
+        sub = self.out.new_prefixed_out("  ")
+
+        if not capture:
+            # Simple case: let subprocess handle streams (no capture)
+            if self.out.verbosity >= 1:
+                sub.print(f"$ {' '.join(cmd)}")
+            return subprocess.run(
+                cmd, text=True, input=input, check=check, stdout=None, stderr=None
+            )
+
+        # Capture case: we may need to stream while capturing
+        if sub.verbosity >= 1:
+            cmd_lines = " ".join(cmd).splitlines()
+            sub.print(f"$ {cmd_lines.pop(0)}")
+            if sub.verbosity >= 2:
+                for line in cmd_lines:
+                    sub.print(f"  {line}")
+
+        proc = subprocess.Popen(
+            cmd,
+            text=True,
+            stdin=subprocess.PIPE if input else subprocess.DEVNULL,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+
+        stdout_lines = []
+        if input:
+            proc.stdin.write(input)
+            proc.stdin.close()
+
+        for line in proc.stdout:
+            stdout_lines.append(line)
+            if sub.verbosity >= 2:
+                sub.print(f"  > {line.rstrip()}")
+
+        stderr = proc.stderr.read()
+        ret = proc.wait()
+        stdout = "".join(stdout_lines)
+        if check and ret != 0:
+            full_output = stdout + stderr
+            for line in full_output.splitlines():
+                if sub.verbosity < 1:  # and we haven't printed it yet
+                    sub.red(line)
+            raise subprocess.CalledProcessError(ret, cmd, output=stdout, stderr=stderr)
+
+        return subprocess.CompletedProcess(cmd, ret, stdout=stdout, stderr=stderr)
+
+    def run_json(self, args, check=True):
+        """Run an incus command with ``--format=json``.
+
+        Returns the parsed JSON on success.
+        When *check* is True raises ``subprocess.CalledProcessError``
+        on non-zero exit; when False returns *None* instead.
+        """
+        result = self.run(
+            list(args) + ["--format=json"],
+            check=check,
+        )
+        if result.returncode != 0:
+            return None
+        return json.loads(result.stdout)
+
+    def run_output(self, args, check=True):
+        """Run an incus command and return its stripped stdout.
+
+        When *check* is False, returns *None* on non-zero exit
+        instead of raising.
+        """
+        result = self.run(args, check=check)
+        if result.returncode != 0:
+            return None
+        return result.stdout.strip()
+
+    def find_image(self, aliases):
+        """Return the first alias from *aliases* that exists, else None."""
+        images = self.run_json(["image", "list"], check=False) or []
+        existing = {a.get("name") for img in images for a in img.get("aliases", [])}
+        for alias in aliases:
+            if alias in existing:
+                return alias
+        return None
+
+    def delete_images(self):
+        """Delete the cached base and relay images."""
+        for alias in (RELAY_IMAGE_ALIAS, BASE_IMAGE_ALIAS):
+            self.run(["image", "delete", alias], check=False)  # ok if absent
+
+    def list_managed(self):
+        """Return list of dicts with name, ip, ipv6, domain, status, memory_usage."""
+        containers = []
+        for ct in self.run_json(["list"]):
+            config = ct.get("config", {})
+            if config.get(LABEL_KEY) != "true":
+                continue
+            name = ct["name"]
+            state = ct.get("state", {})
+            net = state.get("network") or {}
+            containers.append(
+                {
+                    "name": name,
+                    "ip": _extract_ip(net, "inet"),
+                    "ipv6": _extract_ip(net, "inet6"),
+                    "domain": config.get(
+                        "user.localchat-domain", f"{name}{DOMAIN_SUFFIX}"
+                    ),
+                    "status": ct.get("status", "Unknown"),
+                    "memory_usage": state.get("memory", {}).get("usage", 0),
+                }
+            )
+        return containers
+
+    def ensure_base_image(self):
+        """Build and cache a base image with openssh and the SSH key.
+
+        The image is published as a local incus image with alias
+        'localchat-base'.  Subsequent container launches use this
+        image instead of the upstream Debian 12, skipping the
+        slow apt-get install step.
+        Returns the image alias.
+        """
+        if self.find_image([BASE_IMAGE_ALIAS]):
+            self.out.print(f"  Base image '{BASE_IMAGE_ALIAS}' already cached.")
+            return BASE_IMAGE_ALIAS
+
+        self.out.print("  Building base image (one-time setup) ...")
+
+        self.run(["delete", BASE_SETUP_NAME, "--force"], check=False)
+        self.run(["image", "delete", BASE_IMAGE_ALIAS], check=False)
+        self.run(["launch", UPSTREAM_IMAGE, BASE_SETUP_NAME])
+
+        ct = Container(self, BASE_SETUP_NAME)
+        ct.wait_ready()
+
+        key_path = self.ssh_key_path
+        pub_key = key_path.with_suffix(".pub").read_text().strip()
+        host_ns = self.get_host_nameservers()
+        ns_lines = "\n".join(f"nameserver {n}" for n in host_ns)
+        ct.bash(f"""
+            printf '{ns_lines}\n' > /etc/resolv.conf
+            apt-get -o DPkg::Lock::Timeout=60 update
+            DEBIAN_FRONTEND=noninteractive apt-get install -y openssh-server python3
+            systemctl enable ssh
+            apt-get clean
+            mkdir -p /root/.ssh
+            chmod 700 /root/.ssh
+            echo '{pub_key}' > /root/.ssh/authorized_keys
+            chmod 600 /root/.ssh/authorized_keys
+        """)
+
+        self.run(["stop", BASE_SETUP_NAME])
+        self.run(["publish", BASE_SETUP_NAME, f"--alias={BASE_IMAGE_ALIAS}"])
+        self.run(["delete", BASE_SETUP_NAME, "--force"])
+        self.out.print(f"  Base image '{BASE_IMAGE_ALIAS}' ready.")
+        return BASE_IMAGE_ALIAS
+
+    def get_container(self, name):
+        """Return a container handle for the given name.
+
+        Accepts both short relay names (``test0``) and full Incus
+        container names (``test0-localchat``).  Returns
+        ``DNSContainer`` for the DNS container and
+        ``RelayContainer`` for everything else.
+        """
+        if name == DNS_CONTAINER_NAME:
+            return DNSContainer(self)
+        return RelayContainer(self, name.removesuffix("-localchat"))
+
+    def get_dns_container(self):
+        """Return a DNSContainer handle."""
+        return DNSContainer(self)
+
+
+class Container:
+    """The base container handle wraps all interactions with incus."""
+
+    def __init__(self, incus, name, domain=None):
+        self.incus = incus
+        self.out = incus.out
+        self.name = name
+        self.domain = domain or f"{name}{DOMAIN_SUFFIX}"
+        self.ipv4 = None
+        self.ipv6 = None
+
+    def bash(self, script, check=True):
+        """Returns stdout from executing ``bash -ec <script>`` inside this container.
+
+        *script* is dedented and stripped so callers can use triple-quoted strings.
+        When *check* is False, returns *None* on non-zero exit instead of raising.
+        """
+        script = textwrap.dedent(script).strip()
+        cmd = ["exec", self.name, "--", "bash", "-ec", script]
+        return self.incus.run_output(cmd, check=check)
+
+    def run_cmd(self, *args, check=True):
+        """Return stdout from running a command directly in the container (no shell).
+
+        When *check* is False, returns *None* on non-zero exit instead of raising.
+        """
+        return self.incus.run_output(
+            ["exec", self.name, "--", *args],
+            check=check,
+        )
+
+    def start(self):
+        self.incus.run(["start", self.name])
+
+    def stop(self, force=False):
+        cmd = ["stop", self.name]
+        if force:
+            cmd.append("--force")
+        self.incus.run(cmd, check=False)
+
+    def launch(self):
+        """Launch from the best available image, return the alias used."""
+        image = self.incus.find_image([RELAY_IMAGE_ALIAS, BASE_IMAGE_ALIAS])
+        if not image:
+            raise RuntimeError(
+                f"No base image '{BASE_IMAGE_ALIAS}' found. "
+                "Call ensure_base_image() before launching containers."
+            )
+        self.out.print(f"  Launching from '{image}' image ...")
+        cfg = []
+        cfg += ("-c", f"{LABEL_KEY}=true")
+        cfg += ("-c", f"user.localchat-domain={self.domain}")
+        self.incus.run(["launch", image, self.name, *cfg])
+        return image
+
+    def ensure(self):
+        """Create/start this container from the cached base image.
+
+        On first call, builds the base image (~30s).
+        Subsequent containers launch in ~2s from the cached image.
+        Returns ``self`` for chaining.
+        """
+        data = self.incus.run_json(["list", self.name], check=False) or []
+
+        existing = [c for c in data if c["name"] == self.name]
+        if existing:
+            if existing[0]["status"] != "Running":
+                self.start()
+        else:
+            self.launch()
+        self.wait_ready()
+        return self
+
+    def destroy(self):
+        """Stop, delete, and clean up config files."""
+        self.stop(force=True)
+        self.incus.run(["delete", self.name, "--force"], check=False)
+
+    def push_file_content(self, dest_path, content):
+        """Write *content* to *dest_path* inside the container.
+
+        *content* is dedented and stripped so callers can use
+        indented triple-quoted strings.
+        """
+        content = textwrap.dedent(content).strip() + "\n"
+        self.incus.run(
+            ["file", "push", "-", f"{self.name}{dest_path}"],
+            input=content,
+        )
+        self.bash(f"chmod 644 {dest_path}")
+
+    def wait_ready(self, timeout=60):
+        """Wait until the container is running with an IPv4 address.
+
+        Sets ``self.ipv4`` and ``self.ipv6`` (may be *None*),
+        or raises ``TimeoutError``.
+        """
+        deadline = time.time() + timeout
+        while time.time() < deadline:
+            data = self.incus.run_json(
+                ["list", self.name],
+                check=False,
+            )
+            if data and data[0].get("status") == "Running":
+                net = data[0].get("state", {}).get("network", {})
+                self.ipv4 = _extract_ip(net, "inet")
+                self.ipv6 = _extract_ip(net, "inet6")
+                if self.ipv4:
+                    return
+            time.sleep(1)
+        raise TimeoutError(
+            f"Container {self.name!r} did not become ready within {timeout}s"
+        )
+
+    def rss_mib(self):
+        """Return ``(used, total)`` memory from container (or None if unobtainable)."""
+        output = self.bash("free -m", check=False)
+        if output:
+            for line in output.splitlines():
+                if line.startswith("Mem:"):
+                    parts = line.split()
+                    return int(parts[2]), int(parts[1])
+
+
+class RelayContainer(Container):
+    """Container handle for a chatmail relay.
+
+    Accepts the short relay name (e.g. ``test0``) and derives
+    the Incus container name and mail domain automatically.
+    """
+
+    def __init__(self, incus, name):
+        super().__init__(
+            incus,
+            f"{name}-localchat",
+            domain=f"_{name}{DOMAIN_SUFFIX}",
+        )
+        self.sname = name
+        self.ini = incus.lxconfigs_dir / f"chatmail-{name}.ini"
+        self.zone = incus.lxconfigs_dir / f"{name}.zone"
+
+    def launch(self):
+        """Launch (from a potentially cached image) and clear inherited chatmail-version."""
+        image = super().launch()
+        self.bash("rm -f /etc/chatmail-version")
+        return image
+
+    def destroy(self):
+        """Stop, delete, and clean up config files."""
+        super().destroy()
+        if self.ini.exists():
+            self.ini.unlink()
+
+    def disable_ipv6(self):
+        """Disable IPv6 inside the container via sysctl."""
+        # incus provides net.* virtualization for LXC containers so that
+        # these sysctls only affect the container's network namespace.
+        self.bash("""
+            sysctl -w net.ipv6.conf.all.disable_ipv6=1
+            sysctl -w net.ipv6.conf.default.disable_ipv6=1
+        """)
+        self.push_file_content(
+            "/etc/sysctl.d/99-disable-ipv6.conf",
+            """
+            net.ipv6.conf.all.disable_ipv6=1
+            net.ipv6.conf.default.disable_ipv6=1
+            """,
+        )
+
+    def configure_hosts(self, ip):
+        """Set hostname and /etc/hosts inside the container."""
+        self.bash(f"""
+            echo '{self.name}' > /etc/hostname
+            hostname {self.name}
+            sed -i '/ {self.domain}$/d' /etc/hosts
+            echo '{ip} {self.name} {self.domain}' >> /etc/hosts
+        """)
+
+    def publish_as_relay_image(self):
+        """Publish this container as a reusable relay image.
+
+        Stops the container, 'publishes' it as 'localchat-relay', then restarts it.
+        """
+        if self.incus.find_image([RELAY_IMAGE_ALIAS]):
+            return
+        self.out.print(
+            f"  Locally caching {self.name!r} as '{RELAY_IMAGE_ALIAS}' image ..."
+        )
+        self.incus.run(
+            ["publish", self.name, f"--alias={RELAY_IMAGE_ALIAS}", "--force"]
+        )
+        self.wait_ready()
+        self.out.print(f"  Relay image '{RELAY_IMAGE_ALIAS}' ready.")
+
+    def deployed_version(self):
+        """Read /etc/chatmail-version, or None if absent."""
+        return self.bash("cat /etc/chatmail-version", check=False)
+
+    def deployed_domain(self):
+        """Read the domain deployed on the container (postfix myhostname)."""
+        return self.bash(
+            "postconf -h myhostname 2>/dev/null",
+            check=False,
+        )
+
+    def verify_ssh(self, ssh_config):
+        """Verify SSH connectivity to this container."""
+        cmd = f"ssh -F {ssh_config} -o ConnectTimeout=60 root@{self.domain} hostname"
+        return shell(cmd, timeout=60).returncode == 0
+
+    def configure_dns(self, dns_ip):
+        """Point this container's resolver at *dns_ip* and verify DNS is reachable."""
+        self.bash(f"""
+            systemctl disable --now systemd-resolved 2>/dev/null || true
+            rm -f /etc/resolv.conf
+            printf 'nameserver {dns_ip}\\n' >/etc/resolv.conf
+            mkdir -p /etc/unbound/unbound.conf.d
+        """)
+        self.push_file_content(
+            "/etc/unbound/unbound.conf.d/localchat-forward.conf",
+            f"""
+            server:
+              domain-insecure: "localchat"
+
+            forward-zone:
+              name: "localchat"
+              forward-addr: {dns_ip}
+            """,
+        )
+        self.bash("systemctl restart unbound 2>/dev/null || true")
+        self._wait_dns_reachable(dns_ip)
+
+    def _wait_dns_reachable(self, dns_ip, timeout=10):
+        """Poll until *dns_ip* answers a DNS query from this container."""
+        if self.bash("which dig", check=False) is None:
+            self.bash(
+                "DEBIAN_FRONTEND=noninteractive "
+                "apt-get install -y dnsutils 2>/dev/null || true"
+            )
+        deadline = time.time() + timeout
+        while time.time() < deadline:
+            result = self.bash(
+                f"dig @{dns_ip} . SOA +short +time=1 +tries=1",
+                check=False,
+            )
+            if result and result.strip():
+                return
+            time.sleep(0.5)
+        raise DNSConfigurationError(
+            f"DNS at {dns_ip} not reachable from {self.name} after {timeout}s"
+        )
+
+    def write_ini(self, disable_ipv6=False):
+        """Generate a chatmail.ini config file in lxconfigs/."""
+        from chatmaild.config import write_initial_config
+
+        overrides = {
+            "max_user_send_per_minute": 600,
+            "max_user_send_burst_size": 100,
+            "mtail_address": "127.0.0.1",
+        }
+        if disable_ipv6:
+            overrides["disable_ipv6"] = "True"
+        write_initial_config(self.ini, self.domain, overrides)
+        return self.ini
+
+
+class DNSContainer(Container):
+    """Container handle for the PowerDNS name server.
+
+    Manages the authoritative and recursive DNS services required for
+    name resolution in the local testing environment.
+    """
+
+    def __init__(self, incus):
+        super().__init__(incus, DNS_CONTAINER_NAME, domain=DNS_DOMAIN)
+
+    def pdnsutil(self, *args, check=True):
+        """Run ``pdnsutil <args>`` inside the DNS container."""
+        return self.run_cmd("pdnsutil", *args, check=check)
+
+    def replace_rrset(self, zone, name, rtype, ttl, rdata):
+        """Shortcut for ``pdnsutil replace-rrset``."""
+        self.pdnsutil("replace-rrset", zone, name, rtype, ttl, rdata)
+
+    def restart_services(self):
+        """Restart pdns and pdns-recursor, then wait until DNS is answering."""
+        self.bash("""
+            systemctl restart pdns
+            systemctl restart pdns-recursor || true
+        """)
+        self._wait_dns_ready()
+
+    def _wait_dns_ready(self, timeout=60):
+        """Poll until the recursor answers a query on port 53."""
+        deadline = time.time() + timeout
+        while time.time() < deadline:
+            result = self.bash(
+                "dig @127.0.0.1 . SOA +short +time=1 +tries=1",
+                check=False,
+            )
+            if result and result.strip():
+                return
+            time.sleep(0.5)
+        raise DNSConfigurationError(f"DNS recursor not answering after {timeout}s")
+
+    def ensure(self):
+        """Create the DNS container with PowerDNS if needed.
+
+        Calls ``super().ensure()`` to create/start the container
+        and set up SSH, then installs PowerDNS and configures
+        the Incus bridge to use this container as DNS.
+        """
+        super().ensure()
+        self._install_powerdns()
+        self.incus.run(
+            ["network", "set", "incusbr0", "dns.mode=none"],
+            check=False,
+        )
+        self.incus.run(
+            ["network", "set", "incusbr0", f"raw.dnsmasq=dhcp-option=6,{self.ipv4}"],
+            check=False,
+        )
+
+    def destroy(self):
+        """Stop, delete, and reset bridge DNS config."""
+        super().destroy()
+        self.incus.run(["network", "unset", "incusbr0", "dns.mode"], check=False)
+        self.incus.run(["network", "unset", "incusbr0", "raw.dnsmasq"], check=False)
+
+    def _install_powerdns(self):
+        """Install and configure PowerDNS if not already present."""
+        if self.run_cmd("which", "pdns_server", check=False) is not None:
+            return
+
+        host_ns = self.incus.get_host_nameservers()
+        ns_lines = "\n".join(f"nameserver {n}" for n in host_ns)
+
+        self.bash(f"""
+            systemctl disable --now systemd-resolved 2>/dev/null || true
+            rm -f /etc/resolv.conf
+            printf '{ns_lines}\n' > /etc/resolv.conf
+
+            # Block automatic service startup during package installation
+            printf '#!/bin/sh\\nexit 101\\n' > /usr/sbin/policy-rc.d
+            chmod +x /usr/sbin/policy-rc.d
+
+            apt-get -o DPkg::Lock::Timeout=60 update
+            DEBIAN_FRONTEND=noninteractive apt-get install -y \
+                pdns-server pdns-backend-sqlite3 sqlite3 pdns-recursor dnsutils
+
+            # Remove the startup block
+            rm /usr/sbin/policy-rc.d
+
+            systemctl stop pdns pdns-recursor || true
+            mkdir -p /var/lib/powerdns
+            sqlite3 /var/lib/powerdns/pdns.sqlite3 \
+                </usr/share/doc/pdns-backend-sqlite3/schema.sqlite3.sql
+            chown -R pdns:pdns /var/lib/powerdns
+        """)
+
+        self.push_file_content(
+            "/etc/powerdns/pdns.conf",
+            """
+            launch=gsqlite3
+            gsqlite3-database=/var/lib/powerdns/pdns.sqlite3
+            local-address=127.0.0.1
+            local-port=5353
+            """,
+        )
+
+        self.push_file_content(
+            "/etc/powerdns/recursor.conf",
+            """
+            local-address=0.0.0.0
+            local-port=53
+            forward-zones=localchat=127.0.0.1:5353
+            allow-from=0.0.0.0/0
+            dont-query=
+            dnssec=off
+            """,
+        )
+
+        self.bash("""
+            systemctl start pdns
+            systemctl start pdns-recursor
+            echo 'nameserver 127.0.0.1' > /etc/resolv.conf
+        """)
+        self._wait_dns_ready()
+
+    def reset_dns_records(self, dns_ip, domains):
+        """Create DNS zones with initial A records via pdnsutil.
+
+        Only sets SOA, NS, and A records as the minimal set
+        needed for SSH connectivity.  Full records (MX, TXT, SRV,
+        CNAME, DKIM) are added later by ``cmdeploy dns``.
+
+        Args:
+            dns_ip: IP of the DNS container
+            domains: list of dicts with 'name', 'domain', 'ip'
+        """
+        for d in domains:
+            domain = d["domain"]
+            ip = d["ip"]
+            self.out.print(f"  {domain} -> {ip}")
+
+            # Delete and recreate zone fresh (removes stale records)
+            self.pdnsutil("delete-zone", domain, check=False)
+            self.pdnsutil("create-zone", domain, f"ns.{domain}")
+
+            serial = str(int(time.time()))
+            soa = f"ns.{domain} hostmaster.{domain} {serial} 3600 900 604800 300"
+            self.replace_rrset(domain, ".", "SOA", "3600", soa)
+            self.replace_rrset(domain, ".", "NS", "3600", f"ns.{domain}.")
+            self.replace_rrset(domain, ".", "A", "3600", ip)
+            self.replace_rrset(domain, "ns", "A", "3600", dns_ip)
+
+            # AAAA (domain -> container IPv6, if available)
+            ipv6 = d.get("ipv6")
+            if ipv6:
+                self.replace_rrset(domain, ".", "AAAA", "3600", ipv6)
+                self.out.print(f"    zone reset: SOA, NS, A, AAAA  ({ip}, {ipv6})")
+            else:
+                # Remove any stale AAAA record
+                self.pdnsutil("delete-rrset", domain, ".", "AAAA", check=False)
+                self.out.print(f"    zone reset: SOA, NS, A  ({ip}, IPv4-only)")
+
+        self.restart_services()
+
+    def set_dns_records(self, text):
+        """Add or overwrite DNS records from standard BIND format.
+
+        Uses ``cmdeploy.dns.parse_zone_records`` to parse.
+        Zones are created automatically from the record names.
+        """
+        from ..dns import parse_zone_records
+
+        zones_seen = set()
+
+        for name, ttl, rtype, rdata in parse_zone_records(text):
+            # Derive zone from name: find top-level .localchat domain
+            name_parts = name.split(".")
+            zone = name  # fallback
+            for i in range(len(name_parts) - 1):
+                if name_parts[i + 1 :] == ["localchat"]:
+                    zone = ".".join(name_parts[i:])
+                    break
+
+            # Create zone if first time seeing it
+            if zone not in zones_seen:
+                self.pdnsutil(
+                    "create-zone",
+                    zone,
+                    f"ns.{zone}",
+                    check=False,
+                )
+                zones_seen.add(zone)
+
+            # Figure out the record name relative to zone
+            if name == zone:
+                relative = "."
+            elif name.endswith(f".{zone}"):
+                relative = name[: -(len(zone) + 1)]
+            else:
+                relative = name
+
+            self.replace_rrset(zone, relative, rtype, ttl, rdata)
+
+        if zones_seen:
+            self.restart_services()

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -20,7 +20,7 @@ smtpd_tls_key_file={{ config.tls_key_path }}
 smtpd_tls_security_level=may
 
 smtp_tls_CApath=/etc/ssl/certs
-smtp_tls_security_level={{ "verify" if config.tls_cert_mode == "acme" else "encrypt" }}
+smtp_tls_security_level=verify
 # Send SNI extension when connecting to other servers.
 # <https://www.postfix.org/postconf.5.html#smtp_tls_servername>
 smtp_tls_servername = hostname
@@ -88,6 +88,22 @@ inet_protocols = ipv4
 inet_protocols = all
 {% endif %}
 
+# Postfix does not try IPv4 and IPv6 connections
+# concurrently as of version 3.7.11.
+#
+# When relay has both A (IPv4) and AAAA (IPv6) records,
+# but broken IPv6 connectivity,
+# every second message is delayed by the connection timeout
+# <https://www.postfix.org/postconf.5.html#smtp_connect_timeout>
+# which defaults to 30 seconds. Reducing timeouts is not a solution
+# as this will result in a failure to connect to slow servers.
+#
+# As a workaround we always prefer IPv4 when it is available.
+# 
+# The setting is documented at
+# <https://www.postfix.org/postconf.5.html#smtp_address_preference>
+smtp_address_preference=ipv4
+
 virtual_transport = lmtp:unix:private/dovecot-lmtp
 virtual_mailbox_domains = {{ config.mail_domain }}
 lmtp_header_checks = regexp:/etc/postfix/lmtp_header_cleanup

cmdeploy/src/cmdeploy/remote/rdns.py

@@ -57,9 +57,10 @@ def get_dkim_entry(mail_domain, pre_command, dkim_selector):
     dkim_value_raw = f"v=DKIM1;k=rsa;p={dkim_pubkey};s=email;t=s"
     dkim_value = '" "'.join(re.findall(".{1,255}", dkim_value_raw))
     web_dkim_value = "".join(re.findall(".{1,255}", dkim_value_raw))
+    name = f"{dkim_selector}._domainkey.{mail_domain}."
     return (
-        f'{dkim_selector}._domainkey.{mail_domain}. TXT "{dkim_value}"',
-        f'{dkim_selector}._domainkey.{mail_domain}. TXT "{web_dkim_value}"',
+        f'{name:<40} 3600   IN  TXT    "{dkim_value}"',
+        f'{name:<40} 3600   IN  TXT    "{web_dkim_value}"',
     )
 
 
@@ -94,9 +95,11 @@ def check_zonefile(zonefile, verbose=True):
         if not zf_line.strip() or zf_line.startswith(";"):
             continue
         print(f"dns-checking {zf_line!r}") if verbose else log_progress("")
-        zf_domain, zf_typ, zf_value = zf_line.split(maxsplit=2)
-        zf_domain = zf_domain.rstrip(".")
-        zf_value = zf_value.strip()
+        parts = zf_line.split(None, 4)
+        zf_domain = parts[0].rstrip(".")
+        # parts[1]=TTL, parts[2]=IN, parts[3]=type, parts[4]=rdata
+        zf_typ = parts[3]
+        zf_value = parts[4].strip()
         query_value = query_dns(zf_typ, zf_domain)
         if zf_value != query_value:
             assert zf_typ in ("A", "AAAA", "CNAME", "CAA", "SRV", "MX", "TXT"), zf_line

cmdeploy/src/cmdeploy/selfsigned/deployer.py

@@ -12,13 +12,27 @@ def openssl_selfsigned_args(domain, cert_path, key_path, days=36500):
     ``www.<domain>`` and ``mta-sts.<domain>``.
     """
     return [
-        "openssl", "req", "-x509",
-        "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:P-256",
-        "-noenc", "-days", str(days),
-        "-keyout", str(key_path),
-        "-out", str(cert_path),
-        "-subj", f"/CN={domain}",
-        "-addext", "extendedKeyUsage=serverAuth,clientAuth",
+        "openssl",
+        "req",
+        "-x509",
+        "-newkey",
+        "ec",
+        "-pkeyopt",
+        "ec_paramgen_curve:P-256",
+        "-noenc",
+        "-days",
+        str(days),
+        "-keyout",
+        str(key_path),
+        "-out",
+        str(cert_path),
+        "-subj",
+        f"/CN={domain}",
+        # Mark as end-entity cert so it cannot be used as a CA to sign others.
+        "-addext",
+        "basicConstraints=critical,CA:FALSE",
+        "-addext",
+        "extendedKeyUsage=serverAuth,clientAuth",
         "-addext",
         f"subjectAltName=DNS:{domain},DNS:www.{domain},DNS:mta-sts.{domain}",
     ]
@@ -40,7 +54,9 @@ class SelfSignedTlsDeployer(Deployer):
 
     def configure(self):
         args = openssl_selfsigned_args(
-            self.mail_domain, self.cert_path, self.key_path,
+            self.mail_domain,
+            self.cert_path,
+            self.key_path,
         )
         cmd = shlex.join(args)
         server.shell(

cmdeploy/src/cmdeploy/sshexec.py

@@ -49,8 +49,13 @@ class SSHExec:
     RemoteError = execnet.RemoteError
     FuncError = FuncError
 
-    def __init__(self, host, verbose=False, python="python3", timeout=60):
-        self.gateway = execnet.makegateway(f"ssh=root@{host}//python={python}")
+    def __init__(
+        self, host, verbose=False, python="python3", timeout=60, ssh_config=None
+    ):
+        spec = f"ssh=root@{host}//python={python}"
+        if ssh_config:
+            spec += f"//ssh_config={ssh_config}"
+        self.gateway = execnet.makegateway(spec)
         self._remote_cmdloop_channel = bootstrap_remote(self.gateway, remote)
         self.timeout = timeout
         self.verbose = verbose
@@ -113,3 +118,46 @@ class LocalExec:
             res = self(call, kwargs, log_callback=remote.rshell.log_progress)
             print_stderr()
             return res
+
+
+# pyinfra exposes a ``ssh_config_file`` data key that *should* let
+# paramiko parse an SSH config file directly.  In practice it silently
+# fails to connect (zero hosts / zero operations), so we resolve the
+# hostname and identity-file ourselves and pass them via
+# ``--data ssh_hostname`` / ``--data ssh_key`` instead.
+# Execnet uses ssh natively (and not paramiko) and doesn't have this problem.
+
+
+def _get_from_ssh_config(host, ssh_config_path, key):
+    """Internal helper to parse a value for a specific key from ssh-config."""
+    current_hosts = []
+    found_value = None
+    with open(ssh_config_path) as f:
+        for raw_line in f:
+            line = raw_line.strip()
+            if not line or line.startswith("#"):
+                continue
+            parts = line.split(None, 1)
+            if not parts:
+                continue
+            directive = parts[0].lower()
+            if directive == "host":
+                if host in current_hosts and found_value:
+                    return found_value
+                current_hosts = parts[1].split()
+                found_value = None
+            elif directive == key.lower():
+                found_value = parts[1]
+    if host in current_hosts and found_value:
+        return found_value
+    return None
+
+
+def resolve_host_from_ssh_config(host, ssh_config_path):
+    """Resolve a host alias to its IP from an ssh-config file."""
+    return _get_from_ssh_config(host, ssh_config_path, "Hostname") or host
+
+
+def resolve_key_from_ssh_config(host, ssh_config_path):
+    """Resolve a host alias to its IdentityFile from an ssh-config file."""
+    return _get_from_ssh_config(host, ssh_config_path, "IdentityFile")

cmdeploy/src/cmdeploy/tests/data/zftest.zone

@@ -1,17 +1,18 @@
-; Required DNS entries for chatmail servers
-zftest.testrun.org.                   A     135.181.204.127
-zftest.testrun.org.                   AAAA  2a01:4f9:c012:52f4::1
-zftest.testrun.org.                   MX 10 zftest.testrun.org.
-_mta-sts.zftest.testrun.org.          TXT "v=STSv1; id=202403211706"
-mta-sts.zftest.testrun.org.           CNAME zftest.testrun.org.
-www.zftest.testrun.org.               CNAME zftest.testrun.org.
-opendkim._domainkey.zftest.testrun.org. TXT "v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoYt82CVUyz2ouaqjX2kB+5J80knAyoOU3MGU5aWppmwUwwTvj/oSTSpkc5JMtVTRmKKr8NUDWAL1Yw7dfGqqPHdHfwwjS3BIvDzYx+hzgtz62RnfNgV+/2MAoNpfX7cAFIHdRzEHNtwugc3RDLquqPoupAE3Y2YRw2T5zG5fILh4vwIcJZL5Uq6B92j8wwJqOex" "33n+vm1NKQ9rxo/UsHAmZlJzpooXcG/4igTBxJyJlamVSRR6N7Nul1v//YJb7J6v2o0iPHW6uE0StzKaPPNC2IVosSRFbD9H2oqppltptFSNPlI0E+t0JBWHem6YK7xcugiO3ImMCaaU8g6Jt/wIDAQAB;s=email;t=s"
+; Required DNS entries
+zftest.testrun.org.                      3600   IN  A      135.181.204.127
+zftest.testrun.org.                      3600   IN  AAAA   2a01:4f9:c012:52f4::1
+zftest.testrun.org.                      3600   IN  MX     10 zftest.testrun.org.
+_mta-sts.zftest.testrun.org.             3600   IN  TXT    "v=STSv1; id=202403211706"
+mta-sts.zftest.testrun.org.              3600   IN  CNAME  zftest.testrun.org.
+www.zftest.testrun.org.                  3600   IN  CNAME  zftest.testrun.org.
+opendkim._domainkey.zftest.testrun.org.  3600   IN  TXT    "v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoYt82CVUyz2ouaqjX2kB+5J80knAyoOU3MGU5aWppmwUwwTvj/oSTSpkc5JMtVTRmKKr8NUDWAL1Yw7dfGqqPHdHfwwjS3BIvDzYx+hzgtz62RnfNgV+/2MAoNpfX7cAFIHdRzEHNtwugc3RDLquqPoupAE3Y2YRw2T5zG5fILh4vwIcJZL5Uq6B92j8wwJqOex" "33n+vm1NKQ9rxo/UsHAmZlJzpooXcG/4igTBxJyJlamVSRR6N7Nul1v//YJb7J6v2o0iPHW6uE0StzKaPPNC2IVosSRFbD9H2oqppltptFSNPlI0E+t0JBWHem6YK7xcugiO3ImMCaaU8g6Jt/wIDAQAB;s=email;t=s"
+
 ; Recommended DNS entries
-_submission._tcp.zftest.testrun.org.  SRV 0 1 587 zftest.testrun.org.
-_submissions._tcp.zftest.testrun.org. SRV 0 1 465 zftest.testrun.org.
-_imap._tcp.zftest.testrun.org.        SRV 0 1 143 zftest.testrun.org.
-_imaps._tcp.zftest.testrun.org.       SRV 0 1 993 zftest.testrun.org.
-zftest.testrun.org.                   CAA 0 issue "letsencrypt.org;accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/1371472956"
-zftest.testrun.org.                   TXT "v=spf1 a:zftest.testrun.org ~all"
-_dmarc.zftest.testrun.org.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
-_adsp._domainkey.zftest.testrun.org.  TXT "dkim=discardable"
+zftest.testrun.org.                      3600   IN  TXT    "v=spf1 a ~all"
+_dmarc.zftest.testrun.org.               3600   IN  TXT    "v=DMARC1;p=reject;adkim=s;aspf=s"
+zftest.testrun.org.                      3600   IN  CAA    0 issue "letsencrypt.org;accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/1371472956"
+_adsp._domainkey.zftest.testrun.org.     3600   IN  TXT    "dkim=discardable"
+_submission._tcp.zftest.testrun.org.     3600   IN  SRV    0 1 587 zftest.testrun.org.
+_submissions._tcp.zftest.testrun.org.    3600   IN  SRV    0 1 465 zftest.testrun.org.
+_imap._tcp.zftest.testrun.org.           3600   IN  SRV    0 1 143 zftest.testrun.org.
+_imaps._tcp.zftest.testrun.org.          3600   IN  SRV    0 1 993 zftest.testrun.org.

cmdeploy/src/cmdeploy/tests/online/test_0_qr.py

@@ -20,7 +20,7 @@ def test_fastcgi_working(maildomain, chatmail_config):
 
 
 @pytest.mark.filterwarnings("ignore::urllib3.exceptions.InsecureRequestWarning")
-def test_newemail_configure(maildomain, rpc, chatmail_config):
+def test_newemail_configure(maildomain, maildomain_ip, rpc, chatmail_config):
     """Test configuring accounts by scanning a QR code works."""
     url = f"DCACCOUNT:https://{maildomain}/new"
     for i in range(3):
@@ -30,12 +30,15 @@ def test_newemail_configure(maildomain, rpc, chatmail_config):
             # set_config_from_qr, so fetch credentials via requests instead
             res = requests.post(f"https://{maildomain}/new", verify=False)
             data = res.json()
-            rpc.add_or_update_transport(account_id, {
-                "addr": data["email"],
-                "password": data["password"],
-                "imapServer": maildomain,
-                "smtpServer": maildomain,
-                "certificateChecks": "acceptInvalidCertificates",
-            })
+            rpc.add_or_update_transport(
+                account_id,
+                {
+                    "addr": data["email"],
+                    "password": data["password"],
+                    "imapServer": maildomain_ip,
+                    "smtpServer": maildomain_ip,
+                    "certificateChecks": "acceptInvalidCertificates",
+                },
+            )
         else:
             rpc.add_transport_from_qr(account_id, url)

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -12,8 +12,9 @@ from cmdeploy.cmdeploy import get_sshexec
 
 class TestSSHExecutor:
     @pytest.fixture(scope="class")
-    def sshexec(self, sshdomain):
-        return get_sshexec(sshdomain)
+    def sshexec(self, sshdomain, pytestconfig):
+        ssh_config = pytestconfig.getoption("ssh_config")
+        return get_sshexec(sshdomain, ssh_config=ssh_config)
 
     def test_ls(self, sshexec):
         out = sshexec(call=remote.rdns.shell, kwargs=dict(command="ls"))
@@ -132,11 +133,10 @@ def test_authenticated_from(cmsetup, maildata):
 @pytest.mark.parametrize("from_addr", ["fake@example.org", "fake@testrun.org"])
 def test_reject_missing_dkim(cmsetup, maildata, from_addr):
     domain = cmsetup.maildomain
-    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    sock.settimeout(10)
     try:
-        sock.connect((domain, 25))
-    except socket.timeout:
+        sock = socket.create_connection((domain, 25), timeout=10)
+        sock.close()
+    except (socket.timeout, OSError):
         pytest.skip(f"port 25 not reachable for {domain}")
 
     recipient = cmsetup.gen_users(1)[0]

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -67,7 +67,7 @@ class TestEndToEndDeltaChat:
         assert msg2.get_snapshot().text == "message0"
 
     def test_exceed_quota(
-        self, cmfactory, lp, tmpdir, remote, chatmail_config, sshdomain
+        self, cmfactory, lp, tmpdir, remote, chatmail_config, sshdomain, pytestconfig
     ):
         """This is a very slow test as it needs to upload >100MB of mail data
         before quota is exceeded, and thus depends on the speed of the upload.
@@ -92,7 +92,9 @@ class TestEndToEndDeltaChat:
         lp.sec(f"filling remote inbox for {user}")
         fn = f"7743102289.M843172P2484002.c20,S={quota},W=2398:2,"
         path = chatmail_config.mailboxes_dir.joinpath(user, "cur", fn)
-        sshexec = get_sshexec(sshdomain)
+        sshexec = get_sshexec(
+            sshdomain, ssh_config=pytestconfig.getoption("ssh_config")
+        )
         sshexec(call=rshell.write_numbytes, kwargs=dict(path=str(path), num=120))
         res = sshexec(call=rshell.dovecot_recalc_quota, kwargs=dict(user=user))
         assert res["percent"] >= 100

cmdeploy/src/cmdeploy/tests/online/test_3_status.py

@@ -3,12 +3,15 @@ import os
 from cmdeploy.cmdeploy import main
 
 
-def test_status_cmd(chatmail_config, capsys, request):
+def test_status_cmd(chatmail_config, capsys, request, pytestconfig):
     os.chdir(request.config.invocation_params.dir)
     command = ["status"]
-    if os.getenv("CHATMAIL_SSH"):
-        command.append("--ssh-host")
-        command.append(os.getenv("CHATMAIL_SSH"))
+    ssh_host = pytestconfig.getoption("ssh_host")
+    if ssh_host:
+        command.extend(["--ssh-host", ssh_host])
+    ssh_config = pytestconfig.getoption("ssh_config")
+    if ssh_config:
+        command.extend(["--ssh-config", ssh_config])
     assert main(command) == 0
     status_out = capsys.readouterr()
     print(status_out.out)

cmdeploy/src/cmdeploy/tests/plugin.py

@@ -2,7 +2,9 @@ import imaplib
 import itertools
 import os
 import random
+import re
 import smtplib
+import socket
 import ssl
 import subprocess
 import time
@@ -18,6 +20,76 @@ def pytest_addoption(parser):
     parser.addoption(
         "--slow", action="store_true", default=False, help="also run slow tests"
     )
+    parser.addoption(
+        "--ssh-host",
+        dest="ssh_host",
+        default=None,
+        help="SSH host (overrides mail_domain for SSH operations).",
+    )
+    parser.addoption(
+        "--ssh-config",
+        dest="ssh_config",
+        default=None,
+        help="Path to an SSH config file (e.g. lxconfigs/ssh-config).",
+    )
+
+
+def _parse_ssh_config_hosts(path):
+    """Parse an OpenSSH config file and return a dict of hostname -> IP."""
+    mapping = {}
+    current_names = []
+    for ln in Path(path).read_text().splitlines():
+        line = ln.strip()
+        m = re.match(r"^Host\s+(.+)", line)
+        if m:
+            current_names = m.group(1).split()
+            continue
+        m = re.match(r"^Hostname\s+(\S+)", line)
+        if m and current_names:
+            ip = m.group(1)
+            for name in current_names:
+                mapping[name] = ip
+            current_names = []
+    return mapping
+
+
+_original_getaddrinfo = socket.getaddrinfo
+
+
+def _make_patched_getaddrinfo(host_map):
+    """Return a getaddrinfo that resolves hosts in host_map to their IPs."""
+
+    def patched_getaddrinfo(host, port, family=0, type=0, proto=0, flags=0):
+        if host in host_map:
+            ip = host_map[host]
+            return _original_getaddrinfo(ip, port, family, type, proto, flags)
+        return _original_getaddrinfo(host, port, family, type, proto, flags)
+
+    return patched_getaddrinfo
+
+
+@pytest.fixture(autouse=True, scope="session")
+def _setup_localchat_dns(pytestconfig):
+    """Monkey-patch socket.getaddrinfo to resolve .localchat via ssh-config."""
+    ssh_config = pytestconfig.getoption("ssh_config")
+    if not ssh_config or not Path(ssh_config).exists():
+        yield {}
+        return
+    host_map = _parse_ssh_config_hosts(ssh_config)
+    if not host_map:
+        yield {}
+        return
+    socket.getaddrinfo = _make_patched_getaddrinfo(host_map)
+    try:
+        yield host_map
+    finally:
+        socket.getaddrinfo = _original_getaddrinfo
+
+
+@pytest.fixture(scope="session")
+def ssh_config_host_map(_setup_localchat_dns):
+    """Return the host-name → IP map parsed from ssh-config."""
+    return _setup_localchat_dns
 
 
 def pytest_configure(config):
@@ -35,6 +107,11 @@ def pytest_runtest_setup(item):
 
 
 def _get_chatmail_config():
+    ini = os.environ.get("CHATMAIL_INI")
+    if ini:
+        path = Path(ini).resolve()
+        if path.exists():
+            return read_config(path), path
     current = Path().resolve()
     while 1:
         path = current.joinpath("chatmail.ini").resolve()
@@ -61,8 +138,14 @@ def maildomain(chatmail_config):
 
 
 @pytest.fixture(scope="session")
-def sshdomain(maildomain):
-    return os.environ.get("CHATMAIL_SSH", maildomain)
+def sshdomain(maildomain, pytestconfig):
+    return pytestconfig.getoption("ssh_host") or maildomain
+
+
+@pytest.fixture(scope="session")
+def maildomain_ip(maildomain, ssh_config_host_map):
+    """Return the IP for maildomain from ssh-config, or maildomain itself."""
+    return ssh_config_host_map.get(maildomain, maildomain)
 
 
 @pytest.fixture
@@ -306,12 +389,22 @@ from deltachat_rpc_client import DeltaChat, Rpc
 class ChatmailACFactory:
     """RPC-based account factory for chatmail testing."""
 
-    def __init__(self, rpc, maildomain, gencreds, chatmail_config):
+    def __init__(
+        self,
+        rpc,
+        maildomain,
+        maildomain_ip,
+        gencreds,
+        chatmail_config,
+        ssh_config_host_map,
+    ):
         self.dc = DeltaChat(rpc)
         self.rpc = rpc
         self._maildomain = maildomain
+        self._maildomain_ip = maildomain_ip
         self.gencreds = gencreds
         self.chatmail_config = chatmail_config
+        self._ssh_config_host_map = ssh_config_host_map
 
     def _make_transport(self, domain):
         """Build a transport config dict for the given domain."""
@@ -319,11 +412,13 @@ class ChatmailACFactory:
         transport = {
             "addr": addr,
             "password": password,
-            # Setting server explicitly skips requesting autoconfig XML,
-            # see https://datatracker.ietf.org/doc/draft-ietf-mailmaint-autoconfig/
-            "imapServer": domain,
-            "smtpServer": domain,
         }
+        # To support running against local relays without host DNS resolution
+        # we attempt resolving the domain via ssh-config
+        # because otherwise core fails to find the address
+        server = self._ssh_config_host_map.get(domain)
+        if server is not None:
+            transport.update({"imapServer": server, "smtpServer": server})
         if self.chatmail_config.tls_cert_mode == "self":
             transport["certificateChecks"] = "acceptInvalidCertificates"
         return transport
@@ -376,39 +471,56 @@ def rpc(tmp_path_factory):
 
 
 @pytest.fixture
-def cmfactory(rpc, gencreds, maildomain, chatmail_config):
+def cmfactory(
+    rpc, gencreds, maildomain, maildomain_ip, chatmail_config, ssh_config_host_map
+):
     """Return a ChatmailACFactory for creating online Delta Chat accounts."""
     return ChatmailACFactory(
         rpc=rpc,
         maildomain=maildomain,
+        maildomain_ip=maildomain_ip,
         gencreds=gencreds,
         chatmail_config=chatmail_config,
+        ssh_config_host_map=ssh_config_host_map,
     )
 
 
 @pytest.fixture
-def remote(sshdomain):
-    return Remote(sshdomain)
+def remote(sshdomain, pytestconfig):
+    r = Remote(sshdomain, ssh_config=pytestconfig.getoption("ssh_config"))
+    yield r
+    r.close()
 
 
 class Remote:
-    def __init__(self, sshdomain):
+    def __init__(self, sshdomain, ssh_config=None):
         self.sshdomain = sshdomain
+        self.ssh_config = ssh_config
+        self._procs = []
 
     def iter_output(self, logcmd="", ready=None):
         getjournal = "journalctl -f" if not logcmd else logcmd
         print(self.sshdomain)
         match self.sshdomain:
-            case "@local": command = []
-            case "localhost": command = []
-            case _: command = ["ssh", f"root@{self.sshdomain}"]
+            case "@local":
+                command = []
+            case "localhost":
+                command = []
+            case _:
+                command = ["ssh"]
+                if self.ssh_config:
+                    command.extend(["-F", self.ssh_config])
+                command.append(f"root@{self.sshdomain}")
         [command.append(arg) for arg in getjournal.split()]
-        self.popen = subprocess.Popen(
+        popen = subprocess.Popen(
             command,
+            stdin=subprocess.DEVNULL,
             stdout=subprocess.PIPE,
+            stderr=subprocess.DEVNULL,
         )
+        self._procs.append(popen)
         while 1:
-            line = self.popen.stdout.readline()
+            line = popen.stdout.readline()
             res = line.decode().strip().lower()
             if not res:
                 break
@@ -417,6 +529,12 @@ class Remote:
                 ready = None
             yield res
 
+    def close(self):
+        while self._procs:
+            proc = self._procs.pop()
+            proc.kill()
+            proc.wait()
+
 
 @pytest.fixture
 def lp(request):

cmdeploy/src/cmdeploy/tests/test_cmdeploy.py

@@ -23,7 +23,10 @@ class TestCmdline:
         run = parser.parse_args(["run"])
         assert init and run
 
-    def test_init_not_overwrite(self, capsys):
+    def test_init_not_overwrite(self, tmp_path, capsys, monkeypatch):
+        monkeypatch.delenv("CHATMAIL_INI", raising=False)
+        monkeypatch.chdir(tmp_path)
+
         assert main(["init", "chat.example.org"]) == 0
         capsys.readouterr()
 

cmdeploy/src/cmdeploy/tests/test_dns.py

@@ -3,7 +3,7 @@ from copy import deepcopy
 import pytest
 
 from cmdeploy import remote
-from cmdeploy.dns import check_full_zone, check_initial_remote_data
+from cmdeploy.dns import check_full_zone, check_initial_remote_data, parse_zone_records
 
 
 @pytest.fixture
@@ -125,18 +125,50 @@ class TestPerformInitialChecks:
         assert not l
 
 
+def test_parse_zone_records():
+    text = """
+    ; This is a comment
+    some.domain. 3600 IN A 1.1.1.1
+
+    ; Another comment
+    www.some.domain. 3600 IN CNAME some.domain.
+
+    ; Multi-word rdata
+    some.domain. 3600 IN MX 10 mail.some.domain.
+
+    ; DKIM record (single line, multi-word TXT rdata)
+    dkim._domainkey.some.domain. 3600 IN TXT "v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG" "9w0BAQEFAAOCAQ8AMIIBCgKCAQEA"
+
+    ; Another TXT record
+    _dmarc.some.domain. 3600 IN TXT "v=DMARC1;p=reject"
+    """
+    records = list(parse_zone_records(text))
+    assert records == [
+        ("some.domain", "3600", "A", "1.1.1.1"),
+        ("www.some.domain", "3600", "CNAME", "some.domain."),
+        ("some.domain", "3600", "MX", "10 mail.some.domain."),
+        (
+            "dkim._domainkey.some.domain",
+            "3600",
+            "TXT",
+            '"v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG" "9w0BAQEFAAOCAQ8AMIIBCgKCAQEA"',
+        ),
+        ("_dmarc.some.domain", "3600", "TXT", '"v=DMARC1;p=reject"'),
+    ]
+
+
+def test_parse_zone_records_invalid_line():
+    text = "invalid line"
+    with pytest.raises(ValueError, match="Bad zone record line"):
+        list(parse_zone_records(text))
+
+
 def parse_zonefile_into_dict(zonefile, mockdns_base, only_required=False):
-    for zf_line in zonefile.split("\n"):
-        if zf_line.startswith("#"):
-            if "Recommended" in zf_line and only_required:
-                return
-            continue
-        if not zf_line.strip():
-            continue
-        zf_domain, zf_typ, zf_value = zf_line.split(maxsplit=2)
-        zf_domain = zf_domain.rstrip(".")
-        zf_value = zf_value.strip()
-        mockdns_base.setdefault(zf_typ, {})[zf_domain] = zf_value
+    if only_required:
+        # Only take records before the "; Recommended" section
+        zonefile = zonefile.split("; Recommended")[0]
+    for name, ttl, rtype, rdata in parse_zone_records(zonefile):
+        mockdns_base.setdefault(rtype, {})[name] = rdata
 
 
 class MockSSHExec:

cmdeploy/src/cmdeploy/tests/test_lxc.py

@@ -0,0 +1,174 @@
+"""Tests for cmdeploy lxc-* subcommands."""
+
+import shutil
+import subprocess
+import sys
+
+import pytest
+
+from cmdeploy.lxc import cli
+from cmdeploy.lxc.incus import Incus
+from cmdeploy.util import Out
+
+pytestmark = pytest.mark.skipif(
+    not shutil.which("incus"),
+    reason="incus not installed",
+)
+
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+
+
+@pytest.fixture
+def ix():
+    out = Out()
+    return Incus(out)
+
+
+@pytest.fixture(scope="session")
+def lxc_setup():
+    out = Out()
+    ix = Incus(out)
+    ix.get_dns_container().ensure()
+    return ix.list_managed()
+
+
+@pytest.fixture(scope="session")
+def relay_container(lxc_setup):
+    test_names = {f"{n}-localchat" for n in cli.RELAY_NAMES}
+    relays = [c for c in lxc_setup if c["name"] in test_names and c.get("ip")]
+    if not relays:
+        pytest.skip("no test relay containers running")
+    return relays[0]
+
+
+@pytest.fixture
+def cmdeploy():
+    def run(*args):
+        return subprocess.run(
+            [sys.executable, "-m", "cmdeploy.cmdeploy", *args],
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+
+    return run
+
+
+# ---------------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+    "subcmd, expected, absent",
+    [
+        (None, ["lxc-start", "lxc-stop", "lxc-test", "lxc-status"], ["lxc-destroy"]),
+        ("lxc-start", ["--ipv4-only", "--run"], ["--config"]),
+        ("lxc-stop", ["--destroy", "--destroy-all"], ["--config"]),
+        ("lxc-test", ["--one"], ["--config"]),
+        ("lxc-status", [], ["--config"]),
+        ("run", ["--ssh-config"], ["--lxc"]),
+        ("dns", ["--ssh-config"], []),
+        ("test", ["--ssh-config"], []),
+        ("status", ["--ssh-config"], []),
+    ],
+)
+def test_help_options(cmdeploy, subcmd, expected, absent):
+    args = [subcmd, "--help"] if subcmd else ["--help"]
+    result = cmdeploy(*args)
+    output = result.stdout + result.stderr
+    assert result.returncode == 0
+    for flag in expected:
+        assert flag in output
+    for flag in absent:
+        assert flag not in output
+
+
+class TestSSHConfig:
+    def test_lxconfigs(self, ix, lxc_setup):
+        d = ix.lxconfigs_dir
+        assert d.name == "lxconfigs"
+        assert d.exists()
+        path = ix.ssh_config_path
+        assert path.name == "ssh-config"
+        assert path.parent.name == "lxconfigs"
+
+    def test_write_ssh_config(self, ix, lxc_setup):
+        path = ix.write_ssh_config()
+        assert path.exists()
+        text = path.read_text()
+
+        for c in lxc_setup:
+            if c.get("ip"):
+                assert c["name"] in text
+                assert f"Hostname {c['ip']}" in text
+
+        assert "User root" in text
+        assert "IdentityFile" in text
+        assert "StrictHostKeyChecking accept-new" in text
+
+
+def test_dns(ix, relay_container):
+    def dig(qname, qtype):
+        ct = ix.get_dns_container()
+        return ct.bash(f"dig @127.0.0.1 {qname} {qtype} +short").strip()
+
+    domain = relay_container["domain"]
+    assert dig(domain, "A") == relay_container["ip"]
+    assert domain in dig(domain, "MX")
+    assert "587" in dig(f"_submission._tcp.{domain}", "SRV")
+
+
+class TestLxcStatus:
+    def test_cli_lxc_status_help(self, cmdeploy):
+        result = cmdeploy("lxc-status", "--help")
+        assert result.returncode == 0
+        assert "status" in result.stdout.lower()
+
+    def test_shows_containers(self, lxc_setup, capsys):
+        class QuietOut(Out):
+            def red(self, msg, **kw):
+                pass
+
+            def green(self, msg, **kw):
+                pass
+
+        ret = cli.lxc_status_cmd(None, QuietOut())
+        assert ret == 0
+        captured = capsys.readouterr().out
+        assert "ns-localchat" in captured
+        assert "running" in captured
+
+    def test_deploy_freshness(self, ix, monkeypatch):
+        ct = ix.get_container("x")
+
+        monkeypatch.setattr(
+            "cmdeploy.lxc.incus.RelayContainer.deployed_version",
+            lambda _self: "abc123def456",
+        )
+        monkeypatch.setattr(
+            "cmdeploy.lxc.incus.RelayContainer.deployed_domain",
+            lambda _self: ct.domain,
+        )
+        monkeypatch.setattr(
+            "cmdeploy.lxc.cli.get_version_string",
+            lambda: "abc123def456",
+        )
+        assert "IN-SYNC" in cli._deploy_status(ct, "abc123def456", ix)
+        assert "STALE" in cli._deploy_status(ct, "other_hash_here", ix)
+
+        # Hash matches but local has uncommitted changes
+        monkeypatch.setattr(
+            "cmdeploy.lxc.cli.get_version_string",
+            lambda: "abc123def456\ndiff --git a/foo",
+        )
+        assert "DIRTY" in cli._deploy_status(ct, "abc123def456", ix)
+
+        monkeypatch.setattr(
+            "cmdeploy.lxc.incus.RelayContainer.deployed_version",
+            lambda _self: None,
+        )
+        assert "NOT DEPLOYED" in cli._deploy_status(ct, "abc123", ix)

cmdeploy/src/cmdeploy/tests/test_util.py

@@ -0,0 +1,120 @@
+import sys
+
+from cmdeploy.util import Out, collapse, get_git_hash, get_version_string, shell
+
+
+class TestOut:
+    def test_prefix_default(self, capsys):
+        out = Out()
+        out.print("hello")
+        assert capsys.readouterr().out == "hello\n"
+
+    def test_prefix_custom(self, capsys):
+        out = Out(prefix=">> ")
+        out.print("hello")
+        assert capsys.readouterr().out == ">> hello\n"
+
+    def test_prefix_print_file(self):
+        import io
+
+        buf = io.StringIO()
+        out = Out(prefix=":: ")
+        out.print("msg", file=buf)
+        assert ":: msg" in buf.getvalue()
+
+    def test_new_prefixed_out(self, capsys):
+        parent = Out(prefix="A")
+        child = parent.new_prefixed_out("B")
+        child.print("x")
+        assert capsys.readouterr().out == "ABx\n"
+        # shares section_timings
+        assert child.section_timings is parent.section_timings
+
+    def test_section_no_auto_indent(self, capsys):
+        out = Out(prefix="")
+        with out.section("test"):
+            out.print("inside")
+        captured = capsys.readouterr().out
+        # "inside" should NOT be indented by section()
+        lines = captured.strip().splitlines()
+        inside_line = [l for l in lines if "inside" in l][0]
+        assert inside_line == "inside"
+
+    def test_section_records_timing(self):
+        out = Out()
+        with out.section("s1"):
+            pass
+        assert len(out.section_timings) == 1
+        assert out.section_timings[0][0] == "s1"
+
+    def test_shell_failure_shows_output(self):
+        """When a shell command fails, its output and exit code are shown."""
+        import subprocess
+
+        result = subprocess.run(
+            [
+                sys.executable,
+                "-c",
+                "from cmdeploy.util import Out; Out(prefix='').shell("
+                "\"echo 'boom on stderr' >&2; exit 42\")",
+            ],
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+        # the command's stderr is merged into stdout by Popen
+        assert "boom on stderr" in result.stdout
+        # Out.red() prints the failure notice to stderr
+        assert "exit code 42" in result.stderr
+
+
+def test_collapse():
+    text = """
+        line 1
+        line 2
+    """
+    assert collapse(text) == "line 1 line 2"
+    assert collapse("  single line  ") == "single line"
+
+
+def test_git_helpers_no_git(tmp_path):
+    # Not a git repo
+    assert get_git_hash(root=tmp_path) is None
+    assert get_version_string(root=tmp_path) == "unknown"
+
+
+def test_git_helpers_empty_repo(tmp_path):
+    shell("git init", cwd=tmp_path, check=True)
+    # No commits yet
+    assert get_git_hash(root=tmp_path) is None
+    assert get_version_string(root=tmp_path) == "unknown"
+
+
+def test_git_helpers_with_commits_and_diffs(tmp_path):
+    shell("git init", cwd=tmp_path, check=True)
+    shell("git config user.email you@example.com", cwd=tmp_path, check=True)
+    shell("git config user.name 'Your Name'", cwd=tmp_path, check=True)
+
+    # First commit
+    path = tmp_path / "file.txt"
+    path.write_text("content")
+    shell("git add file.txt", cwd=tmp_path, check=True)
+    shell("git commit -m initial", cwd=tmp_path, check=True)
+
+    git_hash = get_git_hash(root=tmp_path)
+    assert len(git_hash) >= 7  # usually 40, but git is git
+    assert get_version_string(root=tmp_path) == git_hash
+
+    # Create a diff
+    path.write_text("new content")
+    v = get_version_string(root=tmp_path)
+    assert v.startswith(git_hash + "\n")
+    assert "new content" in v
+    assert not v.endswith("\n")
+
+    # Commit again -> no diff
+    shell("git add file.txt", cwd=tmp_path, check=True)
+    shell("git commit -m second", cwd=tmp_path, check=True)
+    new_hash = get_git_hash(root=tmp_path)
+    assert new_hash != git_hash
+    assert get_version_string(root=tmp_path) == new_hash

cmdeploy/src/cmdeploy/util.py

@@ -0,0 +1,169 @@
+"""Shared utility functions for cmdeploy."""
+
+import os
+import shutil
+import subprocess
+import sys
+import textwrap
+import time
+from contextlib import contextmanager
+from pathlib import Path
+
+from termcolor import colored
+
+
+class Out:
+    """Convenience output printer providing coloring and section formatting."""
+
+    def __init__(self, prefix="", verbosity=0):
+        self.section_timings = []
+        self.prefix = prefix
+        self.sepchar = "\u2501"
+        self.verbosity = verbosity
+        env_width = os.environ.get("_CMDEPLOY_WIDTH")
+        if env_width:
+            self.section_width = int(env_width)
+        else:
+            self.section_width = shutil.get_terminal_size((80, 24)).columns
+
+    def new_prefixed_out(self, newprefix="  "):
+        """Return a new Out with an extended prefix,
+        sharing section_timings with the parent.
+        """
+        out = Out(
+            prefix=self.prefix + newprefix,
+            verbosity=self.verbosity,
+        )
+        out.section_timings = self.section_timings
+        return out
+
+    def red(self, msg, file=sys.stderr):
+        print(colored(self.prefix + msg, "red"), file=file, flush=True)
+
+    def green(self, msg, file=sys.stderr):
+        print(colored(self.prefix + msg, "green"), file=file, flush=True)
+
+    def print(self, msg="", **kwargs):
+        """Print to stdout with automatic flush."""
+        if msg:
+            msg = self.prefix + msg
+        print(msg, flush=True, **kwargs)
+
+    def _format_header(self, title):
+        """Return a formatted section header string."""
+        width = self.section_width - len(self.prefix)
+        bar = self.sepchar * (width - len(title) - 5)
+        return f"{self.sepchar * 3} {title} {bar}"
+
+    @contextmanager
+    def section(self, title):
+        """Context manager that prints a section header and records elapsed time."""
+        self.green(self._format_header(title))
+        t0 = time.time()
+        yield
+        elapsed = time.time() - t0
+        self.section_timings.append((title, elapsed))
+
+    def section_line(self, title):
+        """Print a section header without timing."""
+        self.green(self._format_header(title))
+
+    def shell(self, cmd, quiet=False, **kwargs):
+        """Print *cmd*, run it, and re-print its output with the current prefix.
+
+        *cmd* is passed through :func:`collapse`, so callers
+        can use triple-quoted f-strings freely.
+        Stdout and stderr are merged, read line-by-line,
+        and each line is printed with ``self.prefix`` prepended.
+        When the command exits non-zero, a red error line is printed.
+        """
+        cmd = collapse(cmd)
+        if not quiet:
+            self.print(f"$ {cmd}")
+        indent = self.prefix + "  "
+        env = kwargs.pop("env", None)
+        if env is None:
+            env = os.environ.copy()
+        env["_CMDEPLOY_WIDTH"] = str(self.section_width - len(indent))
+        proc = subprocess.Popen(
+            cmd,
+            shell=True,
+            text=True,
+            stdin=subprocess.DEVNULL,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            env=env,
+            **kwargs,
+        )
+        for line in proc.stdout:
+            sys.stdout.write(indent + line)
+            sys.stdout.flush()
+        ret = proc.wait()
+        if ret:
+            self.red(f"command failed with exit code {ret}: {cmd}")
+        return ret
+
+
+def _project_root():
+    """Return the project root directory."""
+    return Path(__file__).resolve().parent.parent.parent.parent
+
+
+def collapse(text):
+    """Dedent, join lines, and strip a (triple-quoted) string.
+
+    Handy for writing shell commands across multiple lines::
+
+        cmd = collapse(f\"""
+            cmdeploy run
+            --config {ct.ini}
+            --ssh-host {ct.domain}
+        \""")
+    """
+    return textwrap.dedent(text).replace("\n", " ").strip()
+
+
+def shell(cmd, check=False, **kwargs):
+    """Run a shell command string with sensible defaults.
+
+    *cmd* is passed through :func:`collapse` first, so callers
+    can use triple-quoted f-strings freely.
+    Captures stdout/stderr by default; pass ``capture_output=False``
+    to stream output to the terminal instead.
+    """
+    if "capture_output" not in kwargs and "stdout" not in kwargs:
+        kwargs["capture_output"] = True
+    kwargs.setdefault("stdin", subprocess.DEVNULL)
+    return subprocess.run(collapse(cmd), shell=True, text=True, check=check, **kwargs)
+
+
+def get_git_hash(root=None):
+    """Return the local HEAD commit hash, or None."""
+    if root is None:
+        root = _project_root()
+    result = shell(
+        "git rev-parse HEAD",
+        cwd=str(root),
+    )
+    if result.returncode == 0:
+        return result.stdout.strip()
+    return None
+
+
+def get_version_string(root=None):
+    """Return ``git_hash\\ngit_diff`` for the local working tree.
+
+    Used by :class:`~cmdeploy.deployers.GithashDeployer` to write
+    ``/etc/chatmail-version`` and by ``lxc-status`` to compare
+    the deployed state against the local checkout.
+    """
+    if root is None:
+        root = _project_root()
+    git_hash = get_git_hash(root=root) or "unknown"
+    try:
+        git_diff = shell("git diff", cwd=str(root)).stdout.strip()
+    except Exception:
+        git_diff = ""
+    if git_diff:
+        return f"{git_hash}\n{git_diff}"
+    return git_hash

doc/source/conf.py

@@ -15,7 +15,7 @@ author = 'chatmail collective'
 
 extensions = [
     #'sphinx.ext.autodoc',
-    #'sphinx.ext.viewdoc',
+    #'sphinx.ext.viewcode',
     'sphinxcontrib.mermaid',
 ]
 

doc/source/index.rst

@@ -16,5 +16,6 @@ Contributions and feedback welcome through the https://github.com/chatmail/relay
     proxy
     migrate
     overview
+    lxc
     related
     faq

doc/source/lxc.rst

@@ -0,0 +1,275 @@
+Local testing with LXC/Incus
+============================
+
+The ``cmdeploy`` tool includes support for running
+chatmail relays inside local
+`Incus <https://linuxcontainers.org/incus/>`_ LXC containers.
+This is meant for development, testing, and CI
+without requiring a remote server.
+LXC system containers are lightweight virtual machines
+that share the host's kernel but run their own init system,
+package manager, and network stack,
+so the cmdeploy deployment scripts work pretty much
+as they would on a real Debian server or cloud VPS.
+
+Prerequisites
+-------------
+
+- Around 4-5 GiB free disk space
+- `systemd-networkd` for the automagic hostname resolution
+- No other service occupying Port 53
+
+Install `Incus <https://linuxcontainers.org/incus/>`_
+(LXC container manager).
+See the `official installation guide
+<https://linuxcontainers.org/incus/docs/main/installing/>`_
+for full details.
+
+After installing incus, initialise and grant yourself access::
+
+    sudo incus admin init --minimal
+    sudo usermod -aG incus-admin $USER
+
+
+.. caution::
+
+   Adding yourself to ``incus-admin`` grants effective root access
+   to the host: any member can mount host directories into a container
+   and manipulate them as root.
+   This is fine for local testing of your own relay branches,
+   but do **not** use it for production setups
+   or for testing untrusted relay branches from others.
+
+.. warning::
+
+   You **must now log out and back in** (or run ``newgrp incus-admin``)
+   after adding yourself to the group.
+   Without this, all ``cmdeploy lxc-*`` commands
+   will fail with permission errors.
+
+Verify the installation works by running ``incus list``,
+which should print an empty table without errors.
+
+
+Quick start
+-----------
+
+::
+
+    cd relay
+    scripts/initenv.sh               # bootstrap venv
+    source venv/bin/activate         # activate venv
+    cmdeploy lxc-test                # create containers, deploy, test
+
+The ``lxc-test`` command provides an automated way
+to run the full deployment and test pipeline.
+It executes several ``cmdeploy`` subcommands in sequential steps.
+If a step fails, you can copy-paste the printed command
+and run it manually to debug.
+No host DNS delegation or ``~/.ssh/config`` changes are needed
+because ``lxc-test`` passes the required SSH and DNS options directly.
+
+
+CLI reference
+--------------
+
+``lxc-start [--ipv4-only] [--run] [NAME ...]``
+    Create and start containers.
+    Without arguments, creates ``test0-localchat`` and ``ns-localchat`` (DNS).
+    Pass one or more ``NAME`` arguments to create user relay containers instead
+    (e.g. ``cmdeploy lxc-start myrelay``).
+    Use ``--ipv4-only`` to set ``disable_ipv6 = True`` in the generated ``chatmail.ini``,
+    producing an IPv4-only relay.
+    Use ``--run`` to automatically run ``cmdeploy run`` on each container after starting it.
+    Generates ``lxconfigs/ssh-config``.
+    It reuses existing containers and resets DNS zones to minimal records.
+
+``lxc-stop [--destroy] [--destroy-all] [NAME ...]``
+    Stop relay containers.
+    Without arguments, stops ``test0-localchat`` and ``test1-localchat``.
+    Pass ``NAME`` to stop specific containers.
+    Use ``--destroy`` to also delete the containers and their config files.
+    Use ``--destroy-all`` to additionally destroy
+    the ``ns-localchat`` DNS container **and** remove
+    the cached ``localchat-base`` and ``localchat-relay``
+    images, giving a fully clean slate for the next ``lxc-test``.
+    User containers are **never** destroyed unless named explicitly.
+
+``lxc-test [--one]``
+    By default creates, deploys, and tests both ``test0`` and ``test1``
+    for dual-domain federation testing (sets ``CHATMAIL_DOMAIN2=_test1.localchat``).
+    test0 runs dual-stack (IPv4 + IPv6) while test1 runs IPv4-only (``disable_ipv6 = True``).
+    Pass ``--one`` to only deploy and test against ``test0``
+    (skips ``test1``, does not set ``CHATMAIL_DOMAIN2``).
+
+``lxc-status``
+    Show live status of all LXC containers (including the DNS container),
+    deploy freshness (comparing ``/etc/chatmail-version``
+    against local ``git rev-parse HEAD`` and ``git diff``),
+    SSH config inclusion, and host DNS forwarding for ``.localchat``.
+    Reports **IN-SYNC**, **DIRTY** (hash matches but uncommitted changes exist),
+    **STALE** (different commit), or **NOT DEPLOYED**.
+
+
+Container types
+-----------------
+
+**Test relay containers** (``test0-localchat``, ``test1-localchat``)
+    Created automatically by ``lxc-test``.
+    **test0** has IPv4 and IPv6 configured,
+    **test1** is IPv4-only (``disable_ipv6 = True``).
+
+**User relay containers** (``<name>-localchat``)
+    Created by ``cmdeploy lxc-start <name>``
+    where ``<name>`` does not start with ``test``.
+    These are personal development instances,
+    never touched by ``lxc-stop --destroy`` unless named explicitly.
+
+**DNS container** (``ns-localchat``)
+    Singleton container running PowerDNS.
+    Created automatically when any relay is started.
+
+
+.. _lxc-ssh-config:
+
+SSH configuration
+-----------------
+
+``cmdeploy lxc-start`` generates ``lxconfigs/ssh-config``,
+a standard OpenSSH config file mapping every container name,
+its domain, and a short alias to the container's IP address::
+
+    Host test0-localchat _test0.localchat _test0
+        Hostname 10.204.0.42
+        User root
+        IdentityFile /path/to/relay/lxconfigs/id_localchat
+        IdentitiesOnly yes
+        StrictHostKeyChecking accept-new
+        UserKnownHostsFile /dev/null
+        LogLevel ERROR
+
+All ``cmdeploy`` commands (``run``, ``dns``, ``status``, ``test``)
+accept ``--ssh-config lxconfigs/ssh-config`` to use this file.
+``lxc-test`` passes it automatically.
+
+**Using containers from the host shell:**
+
+To make ``ssh _test0`` work from any terminal, add one line to ``~/.ssh/config``::
+
+    Include /absolute/path/to/relay/lxconfigs/ssh-config
+
+
+.. _lxc-dns-setup:
+.. _localchat-tld:
+
+``.localchat`` DNS and name resolution
+---------------------------------------
+
+All LXC-managed chatmail domains use the ``.localchat`` pseudo-TLD
+(e.g. ``_test0.localchat``, ``_test1.localchat``),
+a non-delegated suffix that exists only within the local PowerDNS infrastructure.
+A dedicated DNS container (``ns-localchat``)
+is created so that local test relays interact
+with DNS similar to a regular public Internet setup.
+On first start, ``cmdeploy lxc-start`` creates this container
+running two `PowerDNS <https://www.powerdns.com/>`_ services:
+
+* **pdns-server** (authoritative) serves ``.localchat``
+  zones from a local SQLite database.
+
+* **pdns-recursor** (recursive) listens on the Incus
+  bridge so all containers can use it.
+  Forwards ``.localchat`` queries to the local
+  authoritative server and resolves everything else recursively.
+
+After the DNS container is up, ``lxc-start`` configures the Incus bridge
+to advertise its IP via DHCP and disables Incus's own DNS.
+DNS records are then created in two phases matching the "cmdeploy run" deployment flow:
+
+1. **``lxc-start``** resets each relay zone to
+   **SOA, NS, and A** records (plus **AAAA** for dual-stack containers).
+   If host DNS resolution is configured, users can
+   afterwards run ``cmdeploy run --config lxconfigs/chatmail-test0.ini
+   --ssh-config lxconfigs/ssh-config --ssh-host _test0.localchat``.
+   LXC subcommands do not depend on host DNS resolution
+   and resolve addresses via ``lxconfigs/ssh-config``.
+
+2. **``cmdeploy dns --zonefile``** generates a standard
+   BIND-format zone file (MX, TXT/SPF, TXT/DMARC,
+   TXT/MTA-STS, SRV, CNAME, DKIM) and loads it
+   into PowerDNS.
+
+This two-phase approach prevents premature configuration of mail records
+before the relay is actually deployed and running.
+Once ``cmdeploy run`` deploys `Unbound <https://nlnetlabs.nl/projects/unbound/>`_
+inside a relay container, Unbound has a configuration plugin snippet
+that forwards all ``.localchat`` queries to the PowerDNS recursor,
+and lets all other queries go through normal recursive resolution.
+
+
+State outside the repository
+-----------------------------
+
+All generated configuration by lxc subcommands live in ``lxconfigs/``
+(git-ignored), including the SSH key pair (``id_localchat``),
+per-container ``chatmail-*.ini`` files, zone files, and ``ssh-config``.
+
+The only state *outside* the repository is the Incus containers and images themselves
+(managed via the ``incus`` CLI, labelled with ``user.localchat-managed=true``).
+The Incus image store retains the following snapshot images:
+
+* ``localchat-base``: Debian 12 with openssh-server and Python (built on first run)
+
+* ``localchat-relay``: fully deployed relay snapshot,
+  cached after the first successful ``cmdeploy run``.
+  Subsequent relay containers launch from this image
+  so the deploy step is mostly no-ops (roughly 3× faster than a fresh deploy).
+
+
+.. _lxc-tls:
+
+TLS handling and underscore domains
+------------------------------------
+
+Container domains start with ``_`` (e.g. ``_test0.localchat``).
+As described in :doc:`getting_started` ("Running a relay with self-signed certificates"),
+underscore domains automatically use self-signed TLS
+and ``smtp_tls_security_level = encrypt``.
+This permits cross-relay federation between LXC containers
+without any external certificate authority.
+Delta Chat clients connecting to these relays
+must be configured with
+``certificateChecks = acceptInvalidCertificates``
+(the test fixtures handle this automatically).
+`PR #7926 on chatmail-core <https://github.com/chatmail/core/pull/7926>`_
+is meant to make this special setting unnecessary for chatmail clients
+that are connecting to underscore domains.
+
+
+Known limitations
+------------------
+
+The LXC environment differs from a production
+deployment in several ways:
+
+**No ACME / Let's Encrypt**:
+  Self-signed TLS only (see :ref:`lxc-tls`);
+  ACME code paths are never exercised locally.
+
+**No inbound connections from the internet**:
+  Containers sit on a private Incus bridge and are not port-forwarded.
+  Only the host and other containers on the same bridge can reach them.
+
+**Local federation only**:
+  Cross-relay mail delivery (e.g. test0 → test1) works between containers on the same host,
+  but these relays are invisible to any external mail server.
+
+**DNS is local only**:
+  The ``.localchat`` pseudo-TLD is not resolvable from the wider internet
+  (see :ref:`lxc-dns-setup`).
+
+**IPv6 is ULA-only**:
+  Containers receive IPv6 addresses from the ``fd42:...`` ULA range on the Incus bridge.
+  These are not globally routable, but are sufficient for testing IPv6 service binding
+  (Postfix, Dovecot, Nginx) and DNS AAAA records inside the local environment.
+  test1 runs with ``disable_ipv6 = True`` to exercise the IPv4-only deployment path.