fix: test_no_vrfy to use an actually existing address

fixes silent fail on invalid configuration file.

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -437,7 +437,7 @@ def main(args=None):
     if args.func is None:
         return parser.parse_args(["-h"])
 
-    out = Out(sepchar="\u2501", verbosity=args.verbose)
+    out = Out(verbosity=args.verbose)
     kwargs = {}
 
     if args.inipath is not None and args.func.__name__ not in ("init_cmd", "fmt_cmd"):

cmdeploy/src/cmdeploy/deployers.py

@@ -463,8 +463,9 @@ class ChatmailDeployer(Deployer):
         ("iroh", None, None),
     ]
 
-    def __init__(self, mail_domain):
-        self.mail_domain = mail_domain
+    def __init__(self, config):
+        self.config = config
+        self.mail_domain = config.mail_domain
 
     def install(self):
         files.put(
@@ -493,6 +494,17 @@ class ChatmailDeployer(Deployer):
         )
 
     def configure(self):
+        # Ensure the per-domain mailbox directory exists before
+        # chatmail-metadata starts (it crashes without it).
+        files.directory(
+            name="Ensure vmail mailbox directory exists",
+            path=f"/home/vmail/mail/{self.mail_domain}",
+            user="vmail",
+            group="vmail",
+            mode="700",
+            present=True,
+        )
+
         # This file is used by auth proxy.
         # https://wiki.debian.org/EtcMailName
         server.shell(
@@ -502,6 +514,15 @@ class ChatmailDeployer(Deployer):
             ],
         )
 
+        files.directory(
+            name=f"Ensure mailboxes directory {self.config.mailboxes_dir} exists",
+            path=str(self.config.mailboxes_dir),
+            user="vmail",
+            group="vmail",
+            mode="700",
+            present=True,
+        )
+
 
 class FcgiwrapDeployer(Deployer):
     def install(self):
@@ -620,7 +641,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
     tls_deployer = get_tls_deployer(config, mail_domain)
 
     all_deployers = [
-        ChatmailDeployer(mail_domain),
+        ChatmailDeployer(config),
         LegacyRemoveDeployer(),
         FiltermailDeployer(),
         JournaldDeployer(),

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -145,7 +145,7 @@ def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
         if not can_modify:
             print(
                 "\n!!!! refusing to attempt sysctl setting in shared-kernel containers\n"
-                f"!!!! dovecot: sysctl {key!r}={value}, should be >65535 for production setups\n"
+                f"!!!! dovecot: sysctl {key!r}={value}, should be >65534 for production setups\n"
                 "!!!!"
             )
             continue

cmdeploy/src/cmdeploy/lxc/cli.py

@@ -114,7 +114,7 @@ def _lxc_start_cmd(args, out):
         )
         sub.green(f"    Include {ssh_cfg}")
 
-    # Optionally run cmdeploy run on each relay
+    # Optionally run cmdeploy run + dns on each relay
     if args.run:
         for ct in relays:
             with out.section(f"cmdeploy run: {ct.sname} ({ct.domain})"):
@@ -123,6 +123,20 @@ def _lxc_start_cmd(args, out):
                     out.red(f"Deploy to {ct.sname} failed (exit {ret})")
                     return ret
 
+        with out.section("loading DNS zones"):
+            for ct in relays:
+                ret = _run_cmdeploy(
+                    "dns", ct, ix, out,
+                    extra=["--zonefile", str(ct.zone)],
+                )
+                if ret:
+                    out.red(f"DNS for {ct.sname} failed (exit {ret})")
+                    return ret
+                if ct.zone.exists():
+                    dns_ct.set_dns_records(ct.zone.read_text())
+                out.print(f"Restarting filtermail-incoming on {ct.name}")
+                ct.bash("systemctl restart filtermail-incoming")
+
 
 # -------------------------------------------------------------------
 # lxc-stop
@@ -247,6 +261,13 @@ def lxc_test_cmd(args, out):
                 out.print(f"Loading {ct.zone} into PowerDNS ...")
                 dns_ct.set_dns_records(zone_data)
 
+        # Restart filtermail so its in-process DNS cache
+        # does not hold stale negative DKIM responses
+        # from before the zones were loaded.
+        for ct in map(ix.get_container, relay_names):
+            out.print(f"Restarting filtermail-incoming on {ct.name} ...")
+            ct.bash("systemctl restart filtermail-incoming")
+
     with out.section("cmdeploy test"):
         first = ix.get_container(relay_names[0])
         env = None

cmdeploy/src/cmdeploy/lxc/incus.py

@@ -99,6 +99,22 @@ class Incus:
         target = f"include {self.ssh_config_path}".lower()
         return any(line.strip().lower() == target for line in lines)
 
+    def get_host_nameservers(self):
+        """Return upstream nameservers found on the host."""
+        ns = []
+        for path in ["/run/systemd/resolve/resolv.conf", "/etc/resolv.conf"]:
+            p = Path(path)
+            if p.exists():
+                for line in p.read_text().splitlines():
+                    if line.strip().startswith("nameserver "):
+                        addr = line.split()[1]
+                        if addr not in ("127.0.0.1", "127.0.0.53", "::1"):
+                            if addr not in ns:
+                                ns.append(addr)
+                if ns:
+                    break
+        return ns
+
     def run(self, args, check=True, capture=True, input=None):
         """Run an incus command.
 
@@ -106,7 +122,7 @@ class Incus:
         to the terminal line-by-line while also being captured for
         later return via result.stdout.
         """
-        cmd = ["incus"] + list(args)
+        cmd = ["incus", "--quiet"] + list(args)
         sub = self.out.new_prefixed_out("  ")
 
         if not capture:
@@ -128,9 +144,9 @@ class Incus:
         proc = subprocess.Popen(
             cmd,
             text=True,
-            stdin=subprocess.PIPE if input else None,
+            stdin=subprocess.PIPE if input else subprocess.DEVNULL,
             stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
+            stderr=subprocess.PIPE,
         )
 
         stdout_lines = []
@@ -143,15 +159,17 @@ class Incus:
             if sub.verbosity >= 2:
                 sub.print(f"  > {line.rstrip()}")
 
+        stderr = proc.stderr.read()
         ret = proc.wait()
         stdout = "".join(stdout_lines)
         if check and ret != 0:
-            for line in stdout.splitlines():
+            full_output = stdout + stderr
+            for line in full_output.splitlines():
                 if sub.verbosity < 1:  # and we haven't printed it yet
                     sub.red(line)
-            raise subprocess.CalledProcessError(ret, cmd, output=stdout)
+            raise subprocess.CalledProcessError(ret, cmd, output=stdout, stderr=stderr)
 
-        return subprocess.CompletedProcess(cmd, ret, stdout=stdout)
+        return subprocess.CompletedProcess(cmd, ret, stdout=stdout, stderr=stderr)
 
     def run_json(self, args, check=True):
         """Run an incus command with ``--format=json``.
@@ -241,8 +259,10 @@ class Incus:
 
         key_path = self.ssh_key_path
         pub_key = key_path.with_suffix(".pub").read_text().strip()
+        host_ns = self.get_host_nameservers()
+        ns_lines = "\n".join(f"nameserver {n}" for n in host_ns)
         ct.bash(f"""
-            echo 'nameserver 9.9.9.9' > /etc/resolv.conf
+            printf '{ns_lines}\n' > /etc/resolv.conf
             apt-get -o DPkg::Lock::Timeout=60 update
             DEBIAN_FRONTEND=noninteractive apt-get install -y openssh-server python3
             systemctl enable ssh
@@ -279,12 +299,11 @@ class Incus:
 class Container:
     """The base container handle wraps all interactions with incus."""
 
-    def __init__(self, incus, name, domain=None, memory="200MiB"):
+    def __init__(self, incus, name, domain=None):
         self.incus = incus
         self.out = incus.out
         self.name = name
         self.domain = domain or f"{name}{DOMAIN_SUFFIX}"
-        self.memory = memory
         self.ipv4 = None
         self.ipv6 = None
 
@@ -329,7 +348,6 @@ class Container:
         cfg = []
         cfg += ("-c", f"{LABEL_KEY}=true")
         cfg += ("-c", f"user.localchat-domain={self.domain}")
-        cfg += ("-c", f"limits.memory={self.memory}")
         self.incus.run(["launch", image, self.name, *cfg])
         return image
 
@@ -414,7 +432,6 @@ class RelayContainer(Container):
             incus,
             f"{name}-localchat",
             domain=f"_{name}{DOMAIN_SUFFIX}",
-            memory="600MiB",
         )
         self.sname = name
         self.ini = incus.lxconfigs_dir / f"chatmail-{name}.ini"
@@ -439,11 +456,14 @@ class RelayContainer(Container):
         self.bash("""
             sysctl -w net.ipv6.conf.all.disable_ipv6=1
             sysctl -w net.ipv6.conf.default.disable_ipv6=1
-            mkdir -p /etc/sysctl.d
-            printf 'net.ipv6.conf.all.disable_ipv6=1\\n
-            net.ipv6.conf.default.disable_ipv6=1\\n'
-            > /etc/sysctl.d/99-disable-ipv6.conf
         """)
+        self.push_file_content(
+            "/etc/sysctl.d/99-disable-ipv6.conf",
+            """
+            net.ipv6.conf.all.disable_ipv6=1
+            net.ipv6.conf.default.disable_ipv6=1
+            """,
+        )
 
     def configure_hosts(self, ip):
         """Set hostname and /etc/hosts inside the container."""
@@ -483,22 +503,29 @@ class RelayContainer(Container):
 
     def verify_ssh(self, ssh_config):
         """Verify SSH connectivity to this container."""
-        cmd = f"ssh -F {ssh_config} -o ConnectTimeout=10 root@{self.domain} hostname"
-        return shell(cmd, timeout=15).returncode == 0
+        cmd = f"ssh -F {ssh_config} -o ConnectTimeout=60 root@{self.domain} hostname"
+        return shell(cmd, timeout=60).returncode == 0
 
     def configure_dns(self, dns_ip):
         """Point this container's resolver at *dns_ip* and verify DNS is reachable."""
         self.bash(f"""
             systemctl disable --now systemd-resolved 2>/dev/null || true
             rm -f /etc/resolv.conf
-            echo 'nameserver {dns_ip}' > /etc/resolv.conf
+            printf 'nameserver {dns_ip}\\n' >/etc/resolv.conf
             mkdir -p /etc/unbound/unbound.conf.d
-            printf 'server:\\n  domain-insecure: "localchat"\\n\\n
-            forward-zone:\\n  name: "localchat"\\n
-              forward-addr: {dns_ip}\\n'
-            > /etc/unbound/unbound.conf.d/localchat-forward.conf
-            systemctl restart unbound 2>/dev/null || true
         """)
+        self.push_file_content(
+            "/etc/unbound/unbound.conf.d/localchat-forward.conf",
+            f"""
+            server:
+              domain-insecure: "localchat"
+
+            forward-zone:
+              name: "localchat"
+              forward-addr: {dns_ip}
+            """,
+        )
+        self.bash("systemctl restart unbound 2>/dev/null || true")
         self._wait_dns_reachable(dns_ip)
 
     def _wait_dns_reachable(self, dns_ip, timeout=10):
@@ -562,7 +589,7 @@ class DNSContainer(Container):
         """)
         self._wait_dns_ready()
 
-    def _wait_dns_ready(self, timeout=10):
+    def _wait_dns_ready(self, timeout=60):
         """Poll until the recursor answers a query on port 53."""
         deadline = time.time() + timeout
         while time.time() < deadline:
@@ -604,10 +631,13 @@ class DNSContainer(Container):
         if self.run_cmd("which", "pdns_server", check=False) is not None:
             return
 
-        self.bash("""
+        host_ns = self.incus.get_host_nameservers()
+        ns_lines = "\n".join(f"nameserver {n}" for n in host_ns)
+
+        self.bash(f"""
             systemctl disable --now systemd-resolved 2>/dev/null || true
             rm -f /etc/resolv.conf
-            echo 'nameserver 9.9.9.9' > /etc/resolv.conf
+            printf '{ns_lines}\n' > /etc/resolv.conf
 
             # Block automatic service startup during package installation
             printf '#!/bin/sh\\nexit 101\\n' > /usr/sbin/policy-rc.d
@@ -643,7 +673,6 @@ class DNSContainer(Container):
             local-address=0.0.0.0
             local-port=53
             forward-zones=localchat=127.0.0.1:5353
-            forward-zones-recurse=.=9.9.9.9;149.112.112.112
             allow-from=0.0.0.0/0
             dont-query=
             dnssec=off

cmdeploy/src/cmdeploy/tests/online/test_0_login.py

@@ -89,7 +89,10 @@ def test_concurrent_logins_same_account(
         assert login_results.get()
 
 
-def test_no_vrfy(chatmail_config):
+def test_no_vrfy(chatmail_config, cmfactory):
+    ac1 = cmfactory.get_online_account()
+    addr = ac1.get_config("addr")
+
     domain = chatmail_config.mail_domain
 
     s = smtplib.SMTP(domain)
@@ -98,7 +101,7 @@ def test_no_vrfy(chatmail_config):
     s.putcmd("vrfy", f"wrongaddress@{chatmail_config.mail_domain}")
     result = s.getreply()
     print(result)
-    s.putcmd("vrfy", f"echo@{chatmail_config.mail_domain}")
+    s.putcmd("vrfy", addr)
     result2 = s.getreply()
     print(result2)
     assert result[0] == result2[0] == 252

cmdeploy/src/cmdeploy/tests/plugin.py

@@ -487,13 +487,16 @@ def cmfactory(
 
 @pytest.fixture
 def remote(sshdomain, pytestconfig):
-    return Remote(sshdomain, ssh_config=pytestconfig.getoption("ssh_config"))
+    r = Remote(sshdomain, ssh_config=pytestconfig.getoption("ssh_config"))
+    yield r
+    r.close()
 
 
 class Remote:
     def __init__(self, sshdomain, ssh_config=None):
         self.sshdomain = sshdomain
         self.ssh_config = ssh_config
+        self._procs = []
 
     def iter_output(self, logcmd="", ready=None):
         getjournal = "journalctl -f" if not logcmd else logcmd
@@ -509,12 +512,15 @@ class Remote:
                     command.extend(["-F", self.ssh_config])
                 command.append(f"root@{self.sshdomain}")
         [command.append(arg) for arg in getjournal.split()]
-        self.popen = subprocess.Popen(
+        popen = subprocess.Popen(
             command,
+            stdin=subprocess.DEVNULL,
             stdout=subprocess.PIPE,
+            stderr=subprocess.DEVNULL,
         )
+        self._procs.append(popen)
         while 1:
-            line = self.popen.stdout.readline()
+            line = popen.stdout.readline()
             res = line.decode().strip().lower()
             if not res:
                 break
@@ -523,6 +529,12 @@ class Remote:
                 ready = None
             yield res
 
+    def close(self):
+        while self._procs:
+            proc = self._procs.pop()
+            proc.kill()
+            proc.wait()
+
 
 @pytest.fixture
 def lp(request):

cmdeploy/src/cmdeploy/util.py

@@ -15,10 +15,10 @@ from termcolor import colored
 class Out:
     """Convenience output printer providing coloring and section formatting."""
 
-    def __init__(self, sepchar="\u2501", prefix="", verbosity=0):
+    def __init__(self, prefix="", verbosity=0):
         self.section_timings = []
         self.prefix = prefix
-        self.sepchar = sepchar
+        self.sepchar = "\u2501"
         self.verbosity = verbosity
         env_width = os.environ.get("_CMDEPLOY_WIDTH")
         if env_width:
@@ -31,7 +31,6 @@ class Out:
         sharing section_timings with the parent.
         """
         out = Out(
-            sepchar=self.sepchar,
             prefix=self.prefix + newprefix,
             verbosity=self.verbosity,
         )
@@ -90,6 +89,7 @@ class Out:
             cmd,
             shell=True,
             text=True,
+            stdin=subprocess.DEVNULL,
             stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT,
             env=env,
@@ -133,6 +133,7 @@ def shell(cmd, check=False, **kwargs):
     """
     if "capture_output" not in kwargs and "stdout" not in kwargs:
         kwargs["capture_output"] = True
+    kwargs.setdefault("stdin", subprocess.DEVNULL)
     return subprocess.run(collapse(cmd), shell=True, text=True, check=check, **kwargs)
 
 

doc/source/lxc.rst

@@ -176,7 +176,7 @@ running two `PowerDNS <https://www.powerdns.com/>`_ services:
 * **pdns-recursor** (recursive) listens on the Incus
   bridge so all containers can use it.
   Forwards ``.localchat`` queries to the local
-  authoritative server and everything else to Quad9 (``9.9.9.9``).
+  authoritative server and resolves everything else recursively.
 
 After the DNS container is up, ``lxc-start`` configures the Incus bridge
 to advertise its IP via DHCP and disables Incus's own DNS.