ci: replace staging workflows with LXC-local testing

Replace the two staging-server CI workflows and their zone-file
helpers with a single lxc-test job in ci.yaml that runs
'cmdeploy lxc-test' inside an ubuntu-24.04 runner.

The new workflow installs Incus from the Zabbly apt repository,
initialises it, bootstraps the venv, caches the base LXC image
together with SSH keys, and runs the full LXC pipeline
(container creation, deploy, DNS zones, tests).

.github/workflows/ci.yaml

@@ -41,14 +41,28 @@ jobs:
   lxc-test:
     name: LXC deploy and test
     runs-on: ubuntu-24.04
-    timeout-minutes: 15
+    timeout-minutes: 30
     steps:
       - uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
       - name: install incus
-        run: sudo apt-get update && sudo apt-get install -y incus
+        run: |
+          # zabbly is the official incus community packages source
+          curl -fsSL https://pkgs.zabbly.com/key.asc \
+            | sudo gpg --dearmor -o /etc/apt/keyrings/zabbly.gpg
+          sudo sh -c 'cat <<EOF > /etc/apt/sources.list.d/zabbly-incus-stable.sources
+          Enabled: yes
+          Types: deb
+          URIs: https://pkgs.zabbly.com/incus/stable
+          Suites: $(. /etc/os-release && echo ${VERSION_CODENAME})
+          Components: main
+          Architectures: $(dpkg --print-architecture)
+          Signed-By: /etc/apt/keyrings/zabbly.gpg
+          EOF'
+          sudo apt-get update
+          sudo apt-get install -y incus
 
       - name: initialise incus
         run: |
@@ -56,7 +70,7 @@ jobs:
           sudo iptables -P FORWARD ACCEPT
           sudo sysctl -w fs.inotify.max_user_instances=65535
           sudo sysctl -w fs.inotify.max_user_watches=65535
-          sudo incus admin init --minimal --quiet
+          sudo incus admin init --minimal
           sudo usermod -aG incus-admin "$USER"
 
       - name: initenv
@@ -65,48 +79,14 @@ jobs:
       - name: append venv/bin to PATH
         run: echo venv/bin >>$GITHUB_PATH
 
-      - name: restore cached images
+      - name: lxc-test
-        id: cache-images
+        run: sg incus-admin -c 'cmdeploy lxc-test'
-        uses: actions/cache@v4
-        with:
-          path: |
-            /tmp/localchat-base.tar.gz
-            /tmp/localchat-ns.tar.gz
-            /tmp/localchat-test0.tar.gz
-            /tmp/localchat-test1.tar.gz
-            lxconfigs/id_localchat*
-          key: incus-contain-v2-${{ runner.os }}-${{ github.ref_name }}
-          restore-keys: |
-            incus-contain-v2-${{ runner.os }}-${{ github.ref_name }}-
-            incus-contain-v2-${{ runner.os }}-main-
-            incus-contain-v2-${{ runner.os }}-
-
-      - name: import cached images
-        run: |
-          for alias in localchat-base localchat-ns localchat-test0 localchat-test1; do
-            if [ -f /tmp/$alias.tar.gz ]; then
-              sg incus-admin -c "incus --quiet image import /tmp/$alias.tar.gz --alias $alias" || true
-            fi
-          done
-
-      - name: cmdeploy lxc-test
-        run: sg incus-admin -c 'cmdeploy lxc-test -vv'
-
-      - name: show container logs on failure
-        if: failure() || cancelled()
-        run: |
-          for c in test0-localchat test1-localchat; do
-            echo "::group::$c journal (warnings+errors)"
-            sg incus-admin -c "incus exec $c -- journalctl -p warning --no-pager -n200" 2>/dev/null || echo "no log"
-            echo "::endgroup::"
-            echo "::group::$c failed services"
-            sg incus-admin -c "incus exec $c -- systemctl --no-pager --failed" || true
-            echo "::endgroup::"
-          done
 
       - name: export images for cache
         if: always()
         run: |
           for alias in localchat-base localchat-ns localchat-test0 localchat-test1; do
-            sg incus-admin -c "incus --quiet image export $alias /tmp/$alias" || true
+            if ! [ -f /tmp/$alias.tar.gz ]; then
+              sg incus-admin -c "incus image export $alias /tmp/$alias" || true
+            fi
           done

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -260,10 +260,10 @@ def test_cmd(args, out):
     pytest_args = [
         pytest_path,
         "cmdeploy/src/",
+        "-n4",
         "-rs",
         "-x",
-        "-vv" if args.verbose > 1 else "-v",
+        "-v",
-        "-s",
         "--durations=5",
     ]
     if args.slow:
@@ -437,7 +437,7 @@ def main(args=None):
     if args.func is None:
         return parser.parse_args(["-h"])
 
-    out = Out(verbosity=args.verbose)
+    out = Out(sepchar="\u2501", verbosity=args.verbose)
     kwargs = {}
 
     if args.inipath is not None and args.func.__name__ not in ("init_cmd", "fmt_cmd"):

cmdeploy/src/cmdeploy/deployers.py

@@ -463,9 +463,8 @@ class ChatmailDeployer(Deployer):
         ("iroh", None, None),
     ]
 
-    def __init__(self, config):
+    def __init__(self, mail_domain):
-        self.config = config
+        self.mail_domain = mail_domain
-        self.mail_domain = config.mail_domain
 
     def install(self):
         files.put(
@@ -494,17 +493,6 @@ class ChatmailDeployer(Deployer):
         )
 
     def configure(self):
-        # Ensure the per-domain mailbox directory exists before
-        # chatmail-metadata starts (it crashes without it).
-        files.directory(
-            name="Ensure vmail mailbox directory exists",
-            path=f"/home/vmail/mail/{self.mail_domain}",
-            user="vmail",
-            group="vmail",
-            mode="700",
-            present=True,
-        )
-
         # This file is used by auth proxy.
         # https://wiki.debian.org/EtcMailName
         server.shell(
@@ -514,15 +502,6 @@ class ChatmailDeployer(Deployer):
             ],
         )
 
-        files.directory(
-            name=f"Ensure mailboxes directory {self.config.mailboxes_dir} exists",
-            path=str(self.config.mailboxes_dir),
-            user="vmail",
-            group="vmail",
-            mode="700",
-            present=True,
-        )
-
 
 class FcgiwrapDeployer(Deployer):
     def install(self):
@@ -641,7 +620,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
     tls_deployer = get_tls_deployer(config, mail_domain)
 
     all_deployers = [
-        ChatmailDeployer(config),
+        ChatmailDeployer(mail_domain),
         LegacyRemoveDeployer(),
         FiltermailDeployer(),
         JournaldDeployer(),

cmdeploy/src/cmdeploy/lxc/cli.py

@@ -4,7 +4,7 @@ import os
 import time
 
 from ..util import get_git_hash, get_version_string, shell
-from .incus import Incus, RelayContainer
+from .incus import RELAY_IMAGE_ALIAS, Incus, RelayContainer
 
 RELAY_NAMES = ("test0", "test1")
 
@@ -47,7 +47,6 @@ def _lxc_start_cmd(args, out):
     out.green("Ensuring DNS container (ns-localchat) ...")
     dns_ct = ix.get_dns_container()
     dns_ct.ensure()
-    dns_ct.ensure_cached_as_image()
     sub.print(f"DNS container IP: {dns_ct.ipv4}")
 
     names = args.names if args.names else RELAY_NAMES
@@ -115,53 +114,14 @@ def _lxc_start_cmd(args, out):
         )
         sub.green(f"    Include {ssh_cfg}")
 
-    # Optionally run cmdeploy run + dns on each relay
+    # Optionally run cmdeploy run on each relay
     if args.run:
-        local_hash = get_git_hash()
         for ct in relays:
-            status = _deploy_status(ct, local_hash, ix)
             with out.section(f"cmdeploy run: {ct.sname} ({ct.domain})"):
-                if "IN-SYNC" in status:
+                ret = _run_cmdeploy("run", ct, ix, out, extra=["--skip-dns-check"])
-                    out.print(f"{ct.sname} is {status}, skipping")
-                else:
-                    ret = _run_cmdeploy("run", ct, ix, out, extra=["--skip-dns-check"])
-                    if ret:
-                        out.red(f"Deploy to {ct.sname} failed (exit {ret})")
-                        return ret
-                    # Cache a per-relay image after each successful deploy
-                    # so the next run can launch directly from the deployed state.
-                    with out.section(f"lxc-test: caching {ct.sname} image"):
-                        ct.ensure_cached_as_image()
-
-        # Restart mail services to flush stale DNS state.
-        # Cached container images boot with a resolv.conf
-        # pointing to the previous run's DNS IP;
-        # configure_dns() already restarted unbound,
-        # but postfix/dovecot may hold stale results
-        # from the window between boot and DNS fix.
-        for ct in relays:
-            out.print(f"Restarting mail services on {ct.name} ...")
-            ct.bash("systemctl restart postfix dovecot opendkim")
-
-        for ct in relays:
-            with out.section(f"cmdeploy dns: {ct.sname} ({ct.domain})"):
-                ret = _run_cmdeploy(
-                    "dns",
-                    ct,
-                    ix,
-                    out,
-                    extra=["--zonefile", str(ct.zone)],
-                )
                 if ret:
-                    out.red(f"DNS for {ct.sname} failed (exit {ret})")
+                    out.red(f"Deploy to {ct.sname} failed (exit {ret})")
                     return ret
-                if ct.zone.exists():
-                    dns_ct.set_dns_records(ct.zone.read_text())
-                # Restart filtermail so its in-process DNS cache
-                # does not hold stale negative DKIM responses
-                # from before the zones were loaded.
-                out.print(f"Restarting filtermail-incoming on {ct.name} ...")
-                ct.bash("systemctl restart filtermail-incoming")
 
 
 # -------------------------------------------------------------------
@@ -235,26 +195,64 @@ def lxc_test_cmd(args, out):
     """
     ix = Incus(out)
     t_total = time.time()
-    v_flag = " -" + "v" * out.verbosity if out.verbosity > 0 else ""
+    relay_names = list(RELAY_NAMES)
+    if args.one:
+        relay_names = relay_names[:1]
 
-    ret = out.shell(f"cmdeploy lxc-start{v_flag} --run test0", cwd=str(ix.project_root))
+    local_hash = get_git_hash()
-    if ret:
-        return ret
 
-    if not args.one:
+    # Per-relay: start, deploy, then snapshot the first relay as a
-        ret = out.shell(
+    # reusable image so the second relay launches pre-deployed.
-            f"cmdeploy lxc-start{v_flag} --run test1 --ipv4-only",
+    ipv4_only_flags = {RELAY_NAMES[0]: False, RELAY_NAMES[1]: True}
-            cwd=str(ix.project_root),
+
-        )
+    for ct in map(ix.get_container, relay_names):
-        if ret:
+        name = ct.sname
-            return ret
+        ipv4_only = ipv4_only_flags.get(name, False)
+        v_flag = " -" + "v" * out.verbosity if out.verbosity > 0 else ""
+        start_cmd = f"cmdeploy lxc-start{v_flag} {name}"
+        if ipv4_only:
+            start_cmd += " --ipv4-only"
+        with out.section(f"cmdeploy lxc-start: {name}"):
+            ret = out.shell(start_cmd, cwd=str(ix.project_root))
+            if ret:
+                return ret
+
+        status = _deploy_status(ct, local_hash, ix)
+        with out.section(f"cmdeploy run: {name}"):
+            if "IN-SYNC" in status:
+                out.print(f"{name} is {status}, skipping")
+            else:
+                ret = _run_cmdeploy("run", ct, ix, out, extra=["--skip-dns-check"])
+                if ret:
+                    out.red(f"Deploy to {name} failed (exit {ret})")
+                    return ret
+
+        # Snapshot the first relay so subsequent ones launch pre-deployed
+        if not ix.find_image([RELAY_IMAGE_ALIAS]):
+            with out.section("lxc-test: caching relay image"):
+                ct.publish_as_relay_image()
+
+    for ct in map(ix.get_container, relay_names):
+        with out.section(f"cmdeploy dns: {ct.sname} ({ct.domain})"):
+            ret = _run_cmdeploy("dns", ct, ix, out, extra=["--zonefile", str(ct.zone)])
+            if ret:
+                out.red(f"DNS for {ct.sname} failed (exit {ret})")
+                return ret
+
+    with out.section(f"lxc-test: loading DNS zones {' & '.join(relay_names)}"):
+        dns_ct = ix.get_dns_container()
+        for ct in map(ix.get_container, relay_names):
+            if ct.zone.exists():
+                zone_data = ct.zone.read_text()
+                out.print(f"Loading {ct.zone} into PowerDNS ...")
+                dns_ct.set_dns_records(zone_data)
 
     with out.section("cmdeploy test"):
-        first = ix.get_container("test0")
+        first = ix.get_container(relay_names[0])
         env = None
-        if not args.one:
+        if len(relay_names) > 1:
             env = os.environ.copy()
-            env["CHATMAIL_DOMAIN2"] = ix.get_container("test1").domain
+            env["CHATMAIL_DOMAIN2"] = ix.get_container(relay_names[1]).domain
         ret = _run_cmdeploy("test", first, ix, out, **({"env": env} if env else {}))
         if ret:
             out.red(f"Tests failed (exit {ret})")

cmdeploy/src/cmdeploy/lxc/incus.py

@@ -14,9 +14,9 @@ DOMAIN_SUFFIX = ".localchat"
 UPSTREAM_IMAGE = "images:debian/12"
 BASE_IMAGE_ALIAS = "localchat-base"
 BASE_SETUP_NAME = "localchat-base-setup"
+RELAY_IMAGE_ALIAS = "localchat-relay"
 
 DNS_CONTAINER_NAME = "ns-localchat"
-DNS_IMAGE_ALIAS = "localchat-ns"
 DNS_DOMAIN = "ns.localchat"
 
 
@@ -99,22 +99,6 @@ class Incus:
         target = f"include {self.ssh_config_path}".lower()
         return any(line.strip().lower() == target for line in lines)
 
-    def get_host_nameservers(self):
-        """Return upstream nameservers found on the host."""
-        ns = []
-        for path in ["/run/systemd/resolve/resolv.conf", "/etc/resolv.conf"]:
-            p = Path(path)
-            if p.exists():
-                for line in p.read_text().splitlines():
-                    if line.strip().startswith("nameserver "):
-                        addr = line.split()[1]
-                        if addr not in ("127.0.0.1", "127.0.0.53", "::1"):
-                            if addr not in ns:
-                                ns.append(addr)
-                if ns:
-                    break
-        return ns
-
     def run(self, args, check=True, capture=True, input=None):
         """Run an incus command.
 
@@ -122,7 +106,7 @@ class Incus:
         to the terminal line-by-line while also being captured for
         later return via result.stdout.
         """
-        cmd = ["incus", "--quiet"] + list(args)
+        cmd = ["incus"] + list(args)
         sub = self.out.new_prefixed_out("  ")
 
         if not capture:
@@ -144,9 +128,9 @@ class Incus:
         proc = subprocess.Popen(
             cmd,
             text=True,
-            stdin=subprocess.PIPE if input else subprocess.DEVNULL,
+            stdin=subprocess.PIPE if input else None,
             stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
         )
 
         stdout_lines = []
@@ -159,17 +143,15 @@ class Incus:
             if sub.verbosity >= 2:
                 sub.print(f"  > {line.rstrip()}")
 
-        stderr = proc.stderr.read()
         ret = proc.wait()
         stdout = "".join(stdout_lines)
         if check and ret != 0:
-            full_output = stdout + stderr
+            for line in stdout.splitlines():
-            for line in full_output.splitlines():
                 if sub.verbosity < 1:  # and we haven't printed it yet
                     sub.red(line)
-            raise subprocess.CalledProcessError(ret, cmd, output=stdout, stderr=stderr)
+            raise subprocess.CalledProcessError(ret, cmd, output=stdout)
 
-        return subprocess.CompletedProcess(cmd, ret, stdout=stdout, stderr=stderr)
+        return subprocess.CompletedProcess(cmd, ret, stdout=stdout)
 
     def run_json(self, args, check=True):
         """Run an incus command with ``--format=json``.
@@ -184,16 +166,7 @@ class Incus:
         )
         if result.returncode != 0:
             return None
-        try:
+        return json.loads(result.stdout)
-            return json.loads(result.stdout)
-        except json.JSONDecodeError as e:
-            msg = f"Incus JSON processing failed for {args!r}: {e!s}"
-            self.out.red(msg)
-            self.out.red(f"Captured stdout: {result.stdout!r}")
-            self.out.red(f"Captured stderr: {result.stderr!r}")
-            if check:
-                raise
-            return None
 
     def run_output(self, args, check=True):
         """Run an incus command and return its stripped stdout.
@@ -216,13 +189,8 @@ class Incus:
         return None
 
     def delete_images(self):
-        """Delete localchat-base and per-container images."""
+        """Delete the cached base and relay images."""
-        for alias in [
+        for alias in (RELAY_IMAGE_ALIAS, BASE_IMAGE_ALIAS):
-            BASE_IMAGE_ALIAS,
-            DNS_IMAGE_ALIAS,
-            "localchat-test0",
-            "localchat-test1",
-        ]:
             self.run(["image", "delete", alias], check=False)  # ok if absent
 
     def list_managed(self):
@@ -252,7 +220,7 @@ class Incus:
     def ensure_base_image(self):
         """Build and cache a base image with openssh and the SSH key.
 
-        The image is cached as a local incus image with alias
+        The image is published as a local incus image with alias
         'localchat-base'.  Subsequent container launches use this
         image instead of the upstream Debian 12, skipping the
         slow apt-get install step.
@@ -273,10 +241,8 @@ class Incus:
 
         key_path = self.ssh_key_path
         pub_key = key_path.with_suffix(".pub").read_text().strip()
-        host_ns = self.get_host_nameservers()
-        ns_lines = "\n".join(f"nameserver {n}" for n in host_ns)
         ct.bash(f"""
-            printf '{ns_lines}\n' > /etc/resolv.conf
+            echo 'nameserver 9.9.9.9' > /etc/resolv.conf
             apt-get -o DPkg::Lock::Timeout=60 update
             DEBIAN_FRONTEND=noninteractive apt-get install -y openssh-server python3
             systemctl enable ssh
@@ -313,11 +279,12 @@ class Incus:
 class Container:
     """The base container handle wraps all interactions with incus."""
 
-    def __init__(self, incus, name, domain=None):
+    def __init__(self, incus, name, domain=None, memory="200MiB"):
         self.incus = incus
         self.out = incus.out
         self.name = name
         self.domain = domain or f"{name}{DOMAIN_SUFFIX}"
+        self.memory = memory
         self.ipv4 = None
         self.ipv6 = None
 
@@ -352,7 +319,7 @@ class Container:
 
     def launch(self):
         """Launch from the best available image, return the alias used."""
-        image = self.incus.find_image([BASE_IMAGE_ALIAS])
+        image = self.incus.find_image([RELAY_IMAGE_ALIAS, BASE_IMAGE_ALIAS])
         if not image:
             raise RuntimeError(
                 f"No base image '{BASE_IMAGE_ALIAS}' found. "
@@ -362,6 +329,7 @@ class Container:
         cfg = []
         cfg += ("-c", f"{LABEL_KEY}=true")
         cfg += ("-c", f"user.localchat-domain={self.domain}")
+        cfg += ("-c", f"limits.memory={self.memory}")
         self.incus.run(["launch", image, self.name, *cfg])
         return image
 
@@ -433,18 +401,6 @@ class Container:
                     parts = line.split()
                     return int(parts[2]), int(parts[1])
 
-    def ensure_cached_as_image(self):
-        """Cache this container as a respective image."""
-        alias = self.image_alias
-        if self.incus.find_image([alias]):
-            return
-        self.out.print("  Cleaning apt cache before caching image ...")
-        self.bash("apt-get clean")
-        self.out.print(f"  Caching {self.name!r} as '{alias}' ...")
-        self.incus.run(["publish", self.name, f"--alias={alias}", "--force"])
-        self.out.print(f"  Image '{alias}' cached.")
-        self.wait_ready()
-
 
 class RelayContainer(Container):
     """Container handle for a chatmail relay.
@@ -458,24 +414,15 @@ class RelayContainer(Container):
             incus,
             f"{name}-localchat",
             domain=f"_{name}{DOMAIN_SUFFIX}",
+            memory="600MiB",
         )
         self.sname = name
-        self.image_alias = f"localchat-{name}"
         self.ini = incus.lxconfigs_dir / f"chatmail-{name}.ini"
         self.zone = incus.lxconfigs_dir / f"{name}.zone"
 
     def launch(self):
-        """Launch from localchat-{sname} if cached, else localchat-base."""
+        """Launch (from a potentially cached image) and clear inherited chatmail-version."""
-
+        image = super().launch()
-        candidates = [self.image_alias]
-        candidates.append(BASE_IMAGE_ALIAS)
-        image = self.incus.find_image(candidates)
-        assert image, f"No deployment base, candidates: {','.join(candidates)}"
-        self.out.print(f"  Launching from '{image}' image ...")
-        cfg = []
-        cfg += ("-c", f"{LABEL_KEY}=true")
-        cfg += ("-c", f"user.localchat-domain={self.domain}")
-        self.incus.run(["launch", image, self.name, *cfg])
         self.bash("rm -f /etc/chatmail-version")
         return image
 
@@ -492,14 +439,11 @@ class RelayContainer(Container):
         self.bash("""
             sysctl -w net.ipv6.conf.all.disable_ipv6=1
             sysctl -w net.ipv6.conf.default.disable_ipv6=1
+            mkdir -p /etc/sysctl.d
+            printf 'net.ipv6.conf.all.disable_ipv6=1\\n
+            net.ipv6.conf.default.disable_ipv6=1\\n'
+            > /etc/sysctl.d/99-disable-ipv6.conf
         """)
-        self.push_file_content(
-            "/etc/sysctl.d/99-disable-ipv6.conf",
-            """
-            net.ipv6.conf.all.disable_ipv6=1
-            net.ipv6.conf.default.disable_ipv6=1
-            """,
-        )
 
     def configure_hosts(self, ip):
         """Set hostname and /etc/hosts inside the container."""
@@ -510,6 +454,22 @@ class RelayContainer(Container):
             echo '{ip} {self.name} {self.domain}' >> /etc/hosts
         """)
 
+    def publish_as_relay_image(self):
+        """Publish this container as a reusable relay image.
+
+        Stops the container, 'publishes' it as 'localchat-relay', then restarts it.
+        """
+        if self.incus.find_image([RELAY_IMAGE_ALIAS]):
+            return
+        self.out.print(
+            f"  Locally caching {self.name!r} as '{RELAY_IMAGE_ALIAS}' image ..."
+        )
+        self.incus.run(
+            ["publish", self.name, f"--alias={RELAY_IMAGE_ALIAS}", "--force"]
+        )
+        self.wait_ready()
+        self.out.print(f"  Relay image '{RELAY_IMAGE_ALIAS}' ready.")
+
     def deployed_version(self):
         """Read /etc/chatmail-version, or None if absent."""
         return self.bash("cat /etc/chatmail-version", check=False)
@@ -523,29 +483,22 @@ class RelayContainer(Container):
 
     def verify_ssh(self, ssh_config):
         """Verify SSH connectivity to this container."""
-        cmd = f"ssh -F {ssh_config} -o ConnectTimeout=60 root@{self.domain} hostname"
+        cmd = f"ssh -F {ssh_config} -o ConnectTimeout=10 root@{self.domain} hostname"
-        return shell(cmd, timeout=60).returncode == 0
+        return shell(cmd, timeout=15).returncode == 0
 
     def configure_dns(self, dns_ip):
         """Point this container's resolver at *dns_ip* and verify DNS is reachable."""
         self.bash(f"""
             systemctl disable --now systemd-resolved 2>/dev/null || true
             rm -f /etc/resolv.conf
-            printf 'nameserver {dns_ip}\\n' >/etc/resolv.conf
+            echo 'nameserver {dns_ip}' > /etc/resolv.conf
             mkdir -p /etc/unbound/unbound.conf.d
+            printf 'server:\\n  domain-insecure: "localchat"\\n\\n
+            forward-zone:\\n  name: "localchat"\\n
+              forward-addr: {dns_ip}\\n'
+            > /etc/unbound/unbound.conf.d/localchat-forward.conf
+            systemctl restart unbound 2>/dev/null || true
         """)
-        self.push_file_content(
-            "/etc/unbound/unbound.conf.d/localchat-forward.conf",
-            f"""
-            server:
-              domain-insecure: "localchat"
-
-            forward-zone:
-              name: "localchat"
-              forward-addr: {dns_ip}
-            """,
-        )
-        self.bash("systemctl restart unbound 2>/dev/null || true")
         self._wait_dns_reachable(dns_ip)
 
     def _wait_dns_reachable(self, dns_ip, timeout=10):
@@ -592,21 +545,6 @@ class DNSContainer(Container):
 
     def __init__(self, incus):
         super().__init__(incus, DNS_CONTAINER_NAME, domain=DNS_DOMAIN)
-        self.image_alias = DNS_IMAGE_ALIAS
-
-    def launch(self):
-        """Launch from localchat-ns if cached, else localchat-base."""
-        image = self.incus.find_image([DNS_IMAGE_ALIAS, BASE_IMAGE_ALIAS])
-        if not image:
-            raise RuntimeError(
-                f"No base image '{BASE_IMAGE_ALIAS}' found. "
-                "Call ensure_base_image() before launching containers."
-            )
-        self.out.print(f"  Launching from '{image}' image ...")
-        cfg = []
-        cfg += ("-c", f"{LABEL_KEY}=true")
-        cfg += ("-c", f"user.localchat-domain={self.domain}")
-        self.incus.run(["launch", image, self.name, *cfg])
 
     def pdnsutil(self, *args, check=True):
         """Run ``pdnsutil <args>`` inside the DNS container."""
@@ -624,7 +562,7 @@ class DNSContainer(Container):
         """)
         self._wait_dns_ready()
 
-    def _wait_dns_ready(self, timeout=60):
+    def _wait_dns_ready(self, timeout=10):
         """Poll until the recursor answers a query on port 53."""
         deadline = time.time() + timeout
         while time.time() < deadline:
@@ -666,13 +604,10 @@ class DNSContainer(Container):
         if self.run_cmd("which", "pdns_server", check=False) is not None:
             return
 
-        host_ns = self.incus.get_host_nameservers()
+        self.bash("""
-        ns_lines = "\n".join(f"nameserver {n}" for n in host_ns)
-
-        self.bash(f"""
             systemctl disable --now systemd-resolved 2>/dev/null || true
             rm -f /etc/resolv.conf
-            printf '{ns_lines}\n' > /etc/resolv.conf
+            echo 'nameserver 9.9.9.9' > /etc/resolv.conf
 
             # Block automatic service startup during package installation
             printf '#!/bin/sh\\nexit 101\\n' > /usr/sbin/policy-rc.d
@@ -708,6 +643,7 @@ class DNSContainer(Container):
             local-address=0.0.0.0
             local-port=53
             forward-zones=localchat=127.0.0.1:5353
+            forward-zones-recurse=.=9.9.9.9;149.112.112.112
             allow-from=0.0.0.0/0
             dont-query=
             dnssec=off

cmdeploy/src/cmdeploy/opendkim/opendkim.conf

@@ -60,10 +60,6 @@ PidFile			/run/opendkim/opendkim.pid
 # by the package dns-root-data.
 TrustAnchorFile		/usr/share/dns/root.key
 
-# Use the local unbound resolver rather than querying root servers directly.
-# on IPv4-only hosts opendkim may otherwise try ipv6 requests and time out. 
-Nameservers		127.0.0.1
-
 # Sign messages when `-o milter_macro_daemon_name=ORIGINATING` is set.
 MTA ORIGINATING
 

cmdeploy/src/cmdeploy/tests/plugin.py

@@ -487,16 +487,13 @@ def cmfactory(
 
 @pytest.fixture
 def remote(sshdomain, pytestconfig):
-    r = Remote(sshdomain, ssh_config=pytestconfig.getoption("ssh_config"))
+    return Remote(sshdomain, ssh_config=pytestconfig.getoption("ssh_config"))
-    yield r
-    r.close()
 
 
 class Remote:
     def __init__(self, sshdomain, ssh_config=None):
         self.sshdomain = sshdomain
         self.ssh_config = ssh_config
-        self._procs = []
 
     def iter_output(self, logcmd="", ready=None):
         getjournal = "journalctl -f" if not logcmd else logcmd
@@ -512,15 +509,12 @@ class Remote:
                     command.extend(["-F", self.ssh_config])
                 command.append(f"root@{self.sshdomain}")
         [command.append(arg) for arg in getjournal.split()]
-        popen = subprocess.Popen(
+        self.popen = subprocess.Popen(
             command,
-            stdin=subprocess.DEVNULL,
             stdout=subprocess.PIPE,
-            stderr=subprocess.DEVNULL,
         )
-        self._procs.append(popen)
         while 1:
-            line = popen.stdout.readline()
+            line = self.popen.stdout.readline()
             res = line.decode().strip().lower()
             if not res:
                 break
@@ -529,12 +523,6 @@ class Remote:
                 ready = None
             yield res
 
-    def close(self):
-        while self._procs:
-            proc = self._procs.pop()
-            proc.kill()
-            proc.wait()
-
 
 @pytest.fixture
 def lp(request):

cmdeploy/src/cmdeploy/util.py

@@ -15,10 +15,10 @@ from termcolor import colored
 class Out:
     """Convenience output printer providing coloring and section formatting."""
 
-    def __init__(self, prefix="", verbosity=0):
+    def __init__(self, sepchar="\u2501", prefix="", verbosity=0):
         self.section_timings = []
         self.prefix = prefix
-        self.sepchar = "\u2501"
+        self.sepchar = sepchar
         self.verbosity = verbosity
         env_width = os.environ.get("_CMDEPLOY_WIDTH")
         if env_width:
@@ -31,6 +31,7 @@ class Out:
         sharing section_timings with the parent.
         """
         out = Out(
+            sepchar=self.sepchar,
             prefix=self.prefix + newprefix,
             verbosity=self.verbosity,
         )
@@ -89,7 +90,6 @@ class Out:
             cmd,
             shell=True,
             text=True,
-            stdin=subprocess.DEVNULL,
             stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT,
             env=env,
@@ -133,7 +133,6 @@ def shell(cmd, check=False, **kwargs):
     """
     if "capture_output" not in kwargs and "stdout" not in kwargs:
         kwargs["capture_output"] = True
-    kwargs.setdefault("stdin", subprocess.DEVNULL)
     return subprocess.run(collapse(cmd), shell=True, text=True, check=check, **kwargs)
 
 

doc/source/lxc.rst

@@ -176,7 +176,7 @@ running two `PowerDNS <https://www.powerdns.com/>`_ services:
 * **pdns-recursor** (recursive) listens on the Incus
   bridge so all containers can use it.
   Forwards ``.localchat`` queries to the local
-  authoritative server and resolves everything else recursively.
+  authoritative server and everything else to Quad9 (``9.9.9.9``).
 
 After the DNS container is up, ``lxc-start`` configures the Incus bridge
 to advertise its IP via DHCP and disables Incus's own DNS.