tests: make sure chatmail-metadata was started

fix a flaky test: https://github.com/chatmail/relay/pull/856#issuecomment-3919881473
since #856 chatmail-metadata is restarted every 5 second, if it didn't come up after that, the failure likely sits deeper.

.github/workflows/ci.yaml

@@ -15,7 +15,7 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: download filtermail
-        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.6.0/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
+        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.5.1/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
       - name: run chatmaild tests 
         working-directory: chatmaild
         run: pipx run tox

.github/workflows/test-and-deploy-ipv4only.yaml

@@ -89,6 +89,7 @@ jobs:
 
       - name: set DNS entries
         run: |
+          ssh root@staging-ipv4.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
           ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns --zonefile staging-generated.zone --ssh-host localhost"
           ssh root@staging-ipv4.testrun.org cat relay/staging-generated.zone >> .github/workflows/staging-ipv4.testrun.org-default.zone
           cat .github/workflows/staging-ipv4.testrun.org-default.zone

.github/workflows/test-and-deploy.yaml

@@ -82,6 +82,7 @@ jobs:
 
       - name: set DNS entries
         run: |
+          ssh -o StrictHostKeyChecking=accept-new root@staging2.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
           cmdeploy dns --zonefile staging-generated.zone --verbose
           cat staging-generated.zone >> .github/workflows/staging.testrun.org-default.zone
           cat .github/workflows/staging.testrun.org-default.zone

.gitignore

@@ -5,7 +5,6 @@ __pycache__/
 *.swp
 *qr-*.png
 chatmail*.ini
-lxconfigs/
 
 
 # C extensions

chatmaild/pyproject.toml

@@ -6,7 +6,10 @@ build-backend = "setuptools.build_meta"
 name = "chatmaild"
 version = "0.3"
 dependencies = [
+  "aiosmtpd",
   "iniconfig",
+  "deltachat-rpc-server",
+  "deltachat-rpc-client",
   "filelock",
   "requests",
   "crypt-r >= 3.13.1 ; python_version >= '3.11'",
@@ -21,6 +24,7 @@ where = ['src']
 [project.scripts]
 doveauth = "chatmaild.doveauth:main"
 chatmail-metadata = "chatmaild.metadata:main"
+chatmail-metrics = "chatmaild.metrics:main"
 chatmail-expire = "chatmaild.expire:main"
 chatmail-fsreport = "chatmaild.fsreport:main"
 lastlogin = "chatmaild.lastlogin:main"
@@ -67,7 +71,6 @@ commands =
 deps = pytest 
        pdbpp 
        pytest-localserver
-       aiosmtpd
        execnet
 commands = pytest -v -rsXx {posargs}
 """

chatmaild/src/chatmaild/doveauth.py

@@ -1,11 +1,8 @@
 import json
 import logging
 import os
-import re
 import sys
 
-import filelock
-
 try:
     import crypt_r
 except ImportError:
@@ -16,7 +13,6 @@ from .dictproxy import DictProxy
 from .migrate_db import migrate_from_db_to_maildir
 
 NOCREATE_FILE = "/etc/chatmail-nocreate"
-VALID_LOCALPART_RE = re.compile(r"^[a-z0-9._-]+$")
 
 
 def encrypt_password(password: str):
@@ -56,10 +52,6 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
         )
         return False
 
-    if not VALID_LOCALPART_RE.match(localpart):
-        logging.warning("localpart %r contains invalid characters", localpart)
-        return False
-
     return True
 
 
@@ -148,13 +140,8 @@ class AuthDictProxy(DictProxy):
         if not is_allowed_to_create(self.config, addr, cleartext_password):
             return
 
-        lock = filelock.FileLock(str(user.password_path) + ".lock", timeout=5)
-        with lock:
-            userdata = user.get_userdb_dict()
-            if userdata:
-                return userdata
-            user.set_password(encrypt_password(cleartext_password))
-            print(f"Created address: {addr}", file=sys.stderr)
+        user.set_password(encrypt_password(cleartext_password))
+        print(f"Created address: {addr}", file=sys.stderr)
         return user.get_userdb_dict()
 
 

chatmaild/src/chatmaild/fsreport.py

@@ -13,20 +13,9 @@ to show storage summaries only for first 1000 mailboxes
 
     python -m chatmaild.fsreport /path/to/chatmail.ini --maxnum 1000
 
-to write Prometheus textfile for node_exporter
-
-    python -m chatmaild.fsreport --textfile /var/lib/prometheus/node-exporter/
-
-writes to /var/lib/prometheus/node-exporter/fsreport.prom
-
-to also write legacy metrics.py style output (default: /var/www/html/metrics):
-
-    python -m chatmaild.fsreport --textfile /var/lib/prometheus/node-exporter/ --legacy-metrics
-
 """
 
 import os
-import tempfile
 from argparse import ArgumentParser
 from datetime import datetime
 
@@ -59,19 +48,7 @@ class Report:
         self.num_ci_logins = self.num_all_logins = 0
         self.login_buckets = {x: 0 for x in (1, 10, 30, 40, 80, 100, 150)}
 
-        KiB = 1024
-        MiB = 1024 * KiB
-        self.message_size_thresholds = (
-            0,
-            100 * KiB,
-            MiB // 2,
-            1 * MiB,
-            2 * MiB,
-            5 * MiB,
-            10 * MiB,
-        )
-        self.message_buckets = {x: 0 for x in self.message_size_thresholds}
-        self.message_count_buckets = {x: 0 for x in self.message_size_thresholds}
+        self.message_buckets = {x: 0 for x in (0, 160000, 500000, 2000000)}
 
     def process_mailbox_stat(self, mailbox):
         # categorize login times
@@ -91,10 +68,9 @@ class Report:
             for size in self.message_buckets:
                 for msg in mailbox.messages:
                     if msg.size >= size:
-                        if self.mdir and f"/{self.mdir}/" not in msg.path:
+                        if self.mdir and not msg.relpath.startswith(self.mdir):
                             continue
                         self.message_buckets[size] += msg.size
-                        self.message_count_buckets[size] += 1
 
         self.size_messages += sum(entry.size for entry in mailbox.messages)
         self.size_extra += sum(entry.size for entry in mailbox.extrafiles)
@@ -117,10 +93,9 @@ class Report:
 
         pref = f"[{self.mdir}] " if self.mdir else ""
         for minsize, sumsize in self.message_buckets.items():
-            count = self.message_count_buckets[minsize]
             percent = (sumsize / all_messages * 100) if all_messages else 0
             print(
-                f"{pref}larger than {HSize(minsize)}: {HSize(sumsize)} ({percent:.2f}%), {count} msgs"
+                f"{pref}larger than {HSize(minsize)}: {HSize(sumsize)} ({percent:.2f}%)"
             )
 
         user_logins = self.num_all_logins - self.num_ci_logins
@@ -136,75 +111,6 @@ class Report:
         for days, active in self.login_buckets.items():
             print(f"last {days:3} days: {HSize(active)} {p(active)}")
 
-    def _write_atomic(self, filepath, content):
-        """Atomically write content to filepath via tmp+rename."""
-        dirpath = os.path.dirname(os.path.abspath(filepath))
-        fd, tmppath = tempfile.mkstemp(dir=dirpath, suffix=".tmp")
-        try:
-            with os.fdopen(fd, "w") as f:
-                f.write(content)
-            os.chmod(tmppath, 0o644)
-            os.rename(tmppath, filepath)
-        except BaseException:
-            try:
-                os.unlink(tmppath)
-            except OSError:
-                pass
-            raise
-
-    def dump_textfile(self, filepath):
-        """Dump metrics in Prometheus exposition format."""
-        lines = []
-
-        lines.append("# HELP chatmail_storage_bytes Mailbox storage in bytes.")
-        lines.append("# TYPE chatmail_storage_bytes gauge")
-        lines.append(f'chatmail_storage_bytes{{kind="messages"}} {self.size_messages}')
-        lines.append(f'chatmail_storage_bytes{{kind="extra"}} {self.size_extra}')
-        total = self.size_extra + self.size_messages
-        lines.append(f'chatmail_storage_bytes{{kind="total"}} {total}')
-
-        lines.append("# HELP chatmail_messages_bytes Sum of msg bytes >= threshold.")
-        lines.append("# TYPE chatmail_messages_bytes gauge")
-        for minsize, sumsize in self.message_buckets.items():
-            lines.append(f'chatmail_messages_bytes{{min_size="{minsize}"}} {sumsize}')
-
-        lines.append("# HELP chatmail_messages_count Number of msgs >= size threshold.")
-        lines.append("# TYPE chatmail_messages_count gauge")
-        for minsize, count in self.message_count_buckets.items():
-            lines.append(f'chatmail_messages_count{{min_size="{minsize}"}} {count}')
-
-        lines.append("# HELP chatmail_accounts Number of accounts.")
-        lines.append("# TYPE chatmail_accounts gauge")
-        user_logins = self.num_all_logins - self.num_ci_logins
-        lines.append(f'chatmail_accounts{{kind="all"}} {self.num_all_logins}')
-        lines.append(f'chatmail_accounts{{kind="ci"}} {self.num_ci_logins}')
-        lines.append(f'chatmail_accounts{{kind="user"}} {user_logins}')
-
-        lines.append(
-            "# HELP chatmail_accounts_active Non-CI accounts active within N days."
-        )
-        lines.append("# TYPE chatmail_accounts_active gauge")
-        for days, active in self.login_buckets.items():
-            lines.append(f'chatmail_accounts_active{{days="{days}"}} {active}')
-
-        self._write_atomic(filepath, "\n".join(lines) + "\n")
-
-    def dump_compat_textfile(self, filepath):
-        """Dump legacy metrics.py style metrics."""
-        user_logins = self.num_all_logins - self.num_ci_logins
-        lines = [
-            "# HELP total number of accounts",
-            "# TYPE accounts gauge",
-            f"accounts {self.num_all_logins}",
-            "# HELP number of CI accounts",
-            "# TYPE ci_accounts gauge",
-            f"ci_accounts {self.num_ci_logins}",
-            "# HELP number of non-CI accounts",
-            "# TYPE nonci_accounts gauge",
-            f"nonci_accounts {user_logins}",
-        ]
-        self._write_atomic(filepath, "\n".join(lines) + "\n")
-
 
 def main(args=None):
     """Report about filesystem storage usage of all mailboxes and messages"""
@@ -221,21 +127,19 @@ def main(args=None):
         "--days",
         default=0,
         action="store",
-        help="assume date to be DAYS older than now",
+        help="assume date to be days older than now",
     )
     parser.add_argument(
         "--min-login-age",
         default=0,
-        metavar="DAYS",
         dest="min_login_age",
         action="store",
-        help="only sum up message size if last login is at least DAYS days old",
+        help="only sum up message size if last login is at least min-login-age days old",
     )
     parser.add_argument(
         "--mdir",
-        metavar="{cur,new,tmp}",
         action="store",
-        help="only consider messages in specified Maildir subdirectory for summary",
+        help="only consider 'cur' or 'new' or 'tmp' messages for summary",
     )
 
     parser.add_argument(
@@ -244,21 +148,6 @@ def main(args=None):
         action="store",
         help="maximum number of mailboxes to iterate on",
     )
-    parser.add_argument(
-        "--textfile",
-        metavar="PATH",
-        default=None,
-        help="write Prometheus textfile to PATH (directory or file); "
-        "if PATH is a directory, writes 'fsreport.prom' inside it",
-    )
-    parser.add_argument(
-        "--legacy-metrics",
-        metavar="FILENAME",
-        nargs="?",
-        const="/var/www/html/metrics",
-        default=None,
-        help="write legacy metrics.py textfile (default: /var/www/html/metrics)",
-    )
 
     args = parser.parse_args(args)
 
@@ -272,15 +161,7 @@ def main(args=None):
     rep = Report(now=now, min_login_age=int(args.min_login_age), mdir=args.mdir)
     for mbox in iter_mailboxes(str(config.mailboxes_dir), maxnum=maxnum):
         rep.process_mailbox_stat(mbox)
-    if args.textfile:
-        path = args.textfile
-        if os.path.isdir(path):
-            path = os.path.join(path, "fsreport.prom")
-        rep.dump_textfile(path)
-    if args.legacy_metrics:
-        rep.dump_compat_textfile(args.legacy_metrics)
-    if not args.textfile and not args.legacy_metrics:
-        rep.dump_summary()
+    rep.dump_summary()
 
 
 if __name__ == "__main__":

chatmaild/src/chatmaild/metadata.py

@@ -101,11 +101,7 @@ class MetadataDictProxy(DictProxy):
                 # Handle `GETMETADATA "" /shared/vendor/deltachat/irohrelay`
                 return f"O{self.iroh_relay}\n"
             elif keyname == "vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn":
-                try:
-                    res = turn_credentials()
-                except Exception:
-                    logging.exception("failed to get TURN credentials")
-                    return "N\n"
+                res = turn_credentials()
                 port = 3478
                 return f"O{self.turn_hostname}:{port}:{res}\n"
 

chatmaild/src/chatmaild/metrics.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+import sys
+from pathlib import Path
+
+
+def main(vmail_dir=None):
+    if vmail_dir is None:
+        vmail_dir = sys.argv[1]
+
+    accounts = 0
+    ci_accounts = 0
+
+    for path in Path(vmail_dir).iterdir():
+        if not path.joinpath("cur").is_dir():
+            continue
+        accounts += 1
+        if path.name[:3] in ("ci-", "ac_"):
+            ci_accounts += 1
+
+    print("# HELP total number of accounts")
+    print("# TYPE accounts gauge")
+    print(f"accounts {accounts}")
+    print("# HELP number of CI accounts")
+    print("# TYPE ci_accounts gauge")
+    print(f"ci_accounts {ci_accounts}")
+    print("# HELP number of non-CI accounts")
+    print("# TYPE nonci_accounts gauge")
+    print(f"nonci_accounts {accounts - ci_accounts}")
+
+
+if __name__ == "__main__":
+    main()

chatmaild/src/chatmaild/newemail.py

@@ -3,6 +3,7 @@
 """CGI script for creating new accounts."""
 
 import json
+import random
 import secrets
 import string
 from urllib.parse import quote
@@ -15,9 +16,7 @@ ALPHANUMERIC_PUNCT = string.ascii_letters + string.digits + string.punctuation
 
 
 def create_newemail_dict(config: Config):
-    user = "".join(
-        secrets.choice(ALPHANUMERIC) for _ in range(config.username_max_length)
-    )
+    user = "".join(random.choices(ALPHANUMERIC, k=config.username_max_length))
     password = "".join(
         secrets.choice(ALPHANUMERIC_PUNCT)
         for _ in range(config.password_min_length + 3)

chatmaild/src/chatmaild/tests/plugin.py

@@ -85,13 +85,13 @@ def mockout():
         captured_green = []
         captured_plain = []
 
-        def red(self, msg, **kw):
+        def red(self, msg):
             self.captured_red.append(msg)
 
-        def green(self, msg, **kw):
+        def green(self, msg):
             self.captured_green.append(msg)
 
-        def print(self, msg="", **kw):
+        def __call__(self, msg):
             self.captured_plain.append(msg)
 
     return MockOut()

chatmaild/src/chatmaild/tests/test_doveauth.py

@@ -120,60 +120,6 @@ def test_handle_dovecot_protocol_iterate(gencreds, example_config):
     assert not lines[2]
 
 
-def test_invalid_localpart_characters(make_config):
-    """Test that is_allowed_to_create rejects localparts with invalid characters."""
-    config = make_config("chat.example.org", {"username_min_length": "3"})
-    password = "zequ0Aimuchoodaechik"
-    domain = config.mail_domain
-
-    # valid localparts
-    assert is_allowed_to_create(config, f"abc123@{domain}", password)
-    assert is_allowed_to_create(config, f"a.b-c_d@{domain}", password)
-
-    # uppercase rejected
-    assert not is_allowed_to_create(config, f"Abc123@{domain}", password)
-    assert not is_allowed_to_create(config, f"ABCDEFG@{domain}", password)
-
-    # spaces and special chars rejected
-    assert not is_allowed_to_create(config, f"a b cde@{domain}", password)
-    assert not is_allowed_to_create(config, f"abc+def@{domain}", password)
-    assert not is_allowed_to_create(config, f"abc!def@{domain}", password)
-    assert not is_allowed_to_create(config, f"ab@cdef@{domain}", password)
-    assert not is_allowed_to_create(config, f"abc/def@{domain}", password)
-    assert not is_allowed_to_create(config, f"abc\\def@{domain}", password)
-
-
-def test_concurrent_creation_same_account(dictproxy):
-    """Test that concurrent creation of the same account doesn't corrupt password."""
-    addr = "racetest1@chat.example.org"
-    password = "zequ0Aimuchoodaechik"
-    num_threads = 10
-    results = queue.Queue()
-
-    def create():
-        try:
-            res = dictproxy.lookup_passdb(addr, password)
-            results.put(("ok", res))
-        except Exception:
-            results.put(("err", traceback.format_exc()))
-
-    threads = [threading.Thread(target=create, daemon=True) for _ in range(num_threads)]
-    for t in threads:
-        t.start()
-    for t in threads:
-        t.join(timeout=10)
-
-    passwords_seen = set()
-    for _ in range(num_threads):
-        status, res = results.get()
-        if status == "err":
-            pytest.fail(f"concurrent creation failed\n{res}")
-        passwords_seen.add(res["password"])
-
-    # all threads must see the same password hash
-    assert len(passwords_seen) == 1
-
-
 def test_50_concurrent_lookups_different_accounts(gencreds, dictproxy):
     num_threads = 50
     req_per_thread = 5

chatmaild/src/chatmaild/tests/test_expire.py

@@ -112,43 +112,6 @@ def test_report(mbox1, example_config):
     report_main(args)
 
 
-def test_report_mdir_filters_by_path(mbox1, example_config):
-    """Test that Report with mdir='cur' only counts messages in cur/ subdirectory."""
-    from chatmaild.fsreport import Report
-
-    now = datetime.utcnow().timestamp()
-
-    # Set password mtime to old enough so min_login_age check passes
-    password = Path(mbox1.basedir).joinpath("password")
-    old_time = now - 86400 * 10  # 10 days ago
-    os.utime(password, (old_time, old_time))
-
-    # Reload mailbox with updated mtime
-    from chatmaild.expire import MailboxStat
-
-    mbox = MailboxStat(mbox1.basedir)
-
-    # Report without mdir — should count all messages
-    rep_all = Report(now=now, min_login_age=1, mdir=None)
-    rep_all.process_mailbox_stat(mbox)
-    total_all = rep_all.message_buckets[0]
-
-    # Report with mdir='cur' — should only count cur/ messages
-    rep_cur = Report(now=now, min_login_age=1, mdir="cur")
-    rep_cur.process_mailbox_stat(mbox)
-    total_cur = rep_cur.message_buckets[0]
-
-    # Report with mdir='new' — should only count new/ messages
-    rep_new = Report(now=now, min_login_age=1, mdir="new")
-    rep_new.process_mailbox_stat(mbox)
-    total_new = rep_new.message_buckets[0]
-
-    # cur has 500-byte msg, new has 600-byte msg (from fill_mbox)
-    assert total_cur == 500
-    assert total_new == 600
-    assert total_all == 500 + 600
-
-
 def test_expiry_cli_basic(example_config, mbox1):
     args = (str(example_config._inipath),)
     expiry_main(args)

chatmaild/src/chatmaild/tests/test_metadata.py

@@ -314,51 +314,6 @@ def test_persistent_queue_items(tmp_path, testaddr, token):
     assert not queue_item < item2 and not item2 < queue_item
 
 
-def test_turn_credentials_exception_returns_N(notifier, metadata, monkeypatch):
-    """Test that turn_credentials() failure returns N\\n instead of crashing."""
-    import chatmaild.metadata
-
-    dictproxy = MetadataDictProxy(
-        notifier=notifier,
-        metadata=metadata,
-        turn_hostname="turn.example.org",
-    )
-
-    def mock_turn_credentials():
-        raise ConnectionRefusedError("socket not available")
-
-    monkeypatch.setattr(chatmaild.metadata, "turn_credentials", mock_turn_credentials)
-
-    transactions = {}
-    res = dictproxy.handle_dovecot_request(
-        "Lshared/0123/vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn"
-        "\tuser@example.org",
-        transactions,
-    )
-    assert res == "N\n"
-
-
-def test_turn_credentials_success(notifier, metadata, monkeypatch):
-    """Test that valid turn_credentials() returns TURN URI."""
-    import chatmaild.metadata
-
-    dictproxy = MetadataDictProxy(
-        notifier=notifier,
-        metadata=metadata,
-        turn_hostname="turn.example.org",
-    )
-
-    monkeypatch.setattr(chatmaild.metadata, "turn_credentials", lambda: "user:pass")
-
-    transactions = {}
-    res = dictproxy.handle_dovecot_request(
-        "Lshared/0123/vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn"
-        "\tuser@example.org",
-        transactions,
-    )
-    assert res == "Oturn.example.org:3478:user:pass\n"
-
-
 def test_iroh_relay(dictproxy):
     rfile = io.BytesIO(
         b"\n".join(

chatmaild/src/chatmaild/tests/test_metrics.py

@@ -0,0 +1,24 @@
+from chatmaild.metrics import main
+
+
+def test_main(tmp_path, capsys):
+    paths = []
+    for x in ("ci-asllkj", "ac_12l3kj", "qweqwe", "ci-l1k2j31l2k3"):
+        p = tmp_path.joinpath(x)
+        p.mkdir()
+        p.joinpath("cur").mkdir()
+        paths.append(p)
+
+    tmp_path.joinpath("nomailbox").mkdir()
+
+    main(tmp_path)
+    out, _ = capsys.readouterr()
+    d = {}
+    for line in out.split("\n"):
+        if line.strip() and not line.startswith("#"):
+            name, num = line.split()
+            d[name] = int(num)
+
+    assert d["accounts"] == 4
+    assert d["ci_accounts"] == 3
+    assert d["nonci_accounts"] == 1

chatmaild/src/chatmaild/tests/test_turnserver.py

@@ -1,73 +0,0 @@
-import socket
-import threading
-import time
-from unittest.mock import patch
-
-import pytest
-
-from chatmaild.turnserver import turn_credentials
-
-SOCKET_PATH = "/run/chatmail-turn/turn.socket"
-
-
-@pytest.fixture
-def turn_socket(tmp_path):
-    """Create a real Unix socket server at a temp path."""
-    sock_path = str(tmp_path / "turn.socket")
-    server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-    server.bind(sock_path)
-    server.listen(1)
-    yield sock_path, server
-    server.close()
-
-
-def _call_turn_credentials(sock_path):
-    """Call turn_credentials but connect to sock_path instead of hardcoded path."""
-    original_connect = socket.socket.connect
-
-    def patched_connect(self, address):
-        if address == SOCKET_PATH:
-            address = sock_path
-        return original_connect(self, address)
-
-    with patch.object(socket.socket, "connect", patched_connect):
-        return turn_credentials()
-
-
-def test_turn_credentials_timeout(turn_socket):
-    """Server accepts but never responds — must raise socket.timeout."""
-    sock_path, server = turn_socket
-
-    def accept_and_hang():
-        conn, _ = server.accept()
-        time.sleep(30)
-        conn.close()
-
-    t = threading.Thread(target=accept_and_hang, daemon=True)
-    t.start()
-
-    with pytest.raises(socket.timeout):
-        _call_turn_credentials(sock_path)
-
-
-def test_turn_credentials_connection_refused(tmp_path):
-    """Socket file doesn't exist — must raise ConnectionRefusedError or FileNotFoundError."""
-    missing = str(tmp_path / "nonexistent.socket")
-    with pytest.raises((ConnectionRefusedError, FileNotFoundError)):
-        _call_turn_credentials(missing)
-
-
-def test_turn_credentials_success(turn_socket):
-    """Server responds with credentials — must return stripped string."""
-    sock_path, server = turn_socket
-
-    def respond():
-        conn, _ = server.accept()
-        conn.sendall(b"testuser:testpass\n")
-        conn.close()
-
-    t = threading.Thread(target=respond, daemon=True)
-    t.start()
-
-    result = _call_turn_credentials(sock_path)
-    assert result == "testuser:testpass"

chatmaild/src/chatmaild/turnserver.py

@@ -4,7 +4,6 @@ import socket
 
 def turn_credentials() -> str:
     with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client_socket:
-        client_socket.settimeout(5)
         client_socket.connect("/run/chatmail-turn/turn.socket")
         with client_socket.makefile("rb") as file:
             return file.readline().decode("utf-8").strip()

cmdeploy/pyproject.toml

@@ -10,6 +10,7 @@ dependencies = [
   "pillow",
   "qrcode",
   "markdown",
+  "pytest",
   "setuptools>=68",
   "termcolor",
   "build",
@@ -20,7 +21,6 @@ dependencies = [
   "execnet",
   "imap_tools",
   "deltachat-rpc-client",
-  "deltachat-rpc-server",
 ]
 
 [project.scripts]

cmdeploy/src/cmdeploy/acmetool/__init__.py

@@ -67,7 +67,7 @@ class AcmetoolDeployer(Deployer):
         )
         files.template(
             src=importlib.resources.files(__package__).joinpath("desired.yaml.j2"),
-            dest=f"/var/lib/acme/desired/{self.domains[0]}",  # 0 is mailhost TLD
+            dest=f"/var/lib/acme/desired/{self.domains[0]}", # 0 is mailhost TLD
             user="root",
             group="root",
             mode="644",

cmdeploy/src/cmdeploy/basedeploy.py

@@ -1,7 +1,6 @@
 import importlib.resources
 import io
 import os
-from contextlib import contextmanager
 
 from pyinfra.operations import files, server, systemd
 
@@ -11,28 +10,6 @@ def has_systemd():
     return os.path.isdir("/run/systemd/system")
 
 
-@contextmanager
-def blocked_service_startup():
-    """Prevent services from auto-starting during package installation.
-
-    Installs a ``/usr/sbin/policy-rc.d`` that exits 101, blocking any
-    service from being started by the package manager.  This avoids bind
-    conflicts and CPU/RAM spikes during initial setup.  The file is removed
-    when the context exits.
-    """
-    # For documentation about policy-rc.d, see:
-    # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
-    files.put(
-        src=get_resource("policy-rc.d"),
-        dest="/usr/sbin/policy-rc.d",
-        user="root",
-        group="root",
-        mode="755",
-    )
-    yield
-    files.file("/usr/sbin/policy-rc.d", present=False)
-
-
 def get_resource(arg, pkg=__package__):
     return importlib.resources.files(pkg).joinpath(arg)
 

cmdeploy/src/cmdeploy/chatmail.zone.j2

@@ -0,0 +1,32 @@
+;
+; Required DNS entries for chatmail servers 
+;
+{% if A %}
+{{ mail_domain }}.                   A     {{ A }}
+{% endif %}
+{% if AAAA %}
+{{ mail_domain }}.                   AAAA  {{ AAAA }}
+{% endif %}
+{{ mail_domain }}.                   MX 10 {{ mail_domain }}.
+{% if strict_tls %}
+_mta-sts.{{ mail_domain }}.          TXT "v=STSv1; id={{ sts_id }}"
+mta-sts.{{ mail_domain }}.           CNAME {{ mail_domain }}.
+{% endif %}
+www.{{ mail_domain }}.               CNAME {{ mail_domain }}.
+{{ dkim_entry }}
+
+;
+; Recommended DNS entries for interoperability and security-hardening
+;
+{{ mail_domain }}.                   TXT "v=spf1 a ~all"
+_dmarc.{{ mail_domain }}.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
+
+{% if acme_account_url %}
+{{ mail_domain }}.                   CAA 0 issue "letsencrypt.org;accounturi={{ acme_account_url }}"
+{% endif %}
+_adsp._domainkey.{{ mail_domain }}.  TXT "dkim=discardable"
+
+_submission._tcp.{{ mail_domain }}.  SRV 0 1 587 {{ mail_domain }}.
+_submissions._tcp.{{ mail_domain }}. SRV 0 1 465 {{ mail_domain }}.
+_imap._tcp.{{ mail_domain }}.        SRV 0 1 143 {{ mail_domain }}.
+_imaps._tcp.{{ mail_domain }}.       SRV 0 1 993 {{ mail_domain }}.

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -15,27 +15,10 @@ from pathlib import Path
 import pyinfra
 from chatmaild.config import read_config, write_initial_config
 from packaging import version
+from termcolor import colored
 
 from . import dns, remote
-from .lxc.cli import (
-    lxc_start_cmd,
-    lxc_start_cmd_options,
-    lxc_status_cmd,
-    lxc_status_cmd_options,
-    lxc_stop_cmd,
-    lxc_stop_cmd_options,
-    lxc_test_cmd,
-    lxc_test_cmd_options,
-)
-from .lxc.incus import DNSConfigurationError
-from .sshexec import (
-    LocalExec,
-    SSHExec,
-    resolve_host_from_ssh_config,
-    resolve_key_from_ssh_config,
-)
-from .util import Out
-from .www import main as webdev_main
+from .sshexec import LocalExec, SSHExec
 
 #
 # cmdeploy sub commands and options
@@ -99,21 +82,18 @@ def run_cmd_options(parser):
         help="disable checks nslookup for dns",
     )
     add_ssh_host_option(parser)
-    add_ssh_config_option(parser)
 
 
 def run_cmd(args, out):
     """Deploy chatmail services on the remote server."""
 
     ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
-    sshexec = get_sshexec(ssh_host, ssh_config=args.ssh_config)
+    sshexec = get_sshexec(ssh_host)
     require_iroh = args.config.enable_iroh_relay
     strict_tls = args.config.tls_cert_mode == "acme"
     if not args.dns_check_disabled:
         remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
-        if not dns.check_initial_remote_data(
-            remote_data, strict_tls=strict_tls, print=out.red
-        ):
+        if not dns.check_initial_remote_data(remote_data, strict_tls=strict_tls, print=out.red):
             return 1
 
     env = os.environ.copy()
@@ -124,24 +104,10 @@ def run_cmd(args, out):
     if not args.dns_check_disabled:
         env["CHATMAIL_ADDR_V4"] = remote_data.get("A") or ""
         env["CHATMAIL_ADDR_V6"] = remote_data.get("AAAA") or ""
-    env["DEBIAN_FRONTEND"] = "noninteractive"
-    env["TERM"] = "linux"
     deploy_path = importlib.resources.files(__package__).joinpath("run.py").resolve()
     pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
 
     cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y"
-    ssh_config = args.ssh_config
-    if ssh_config:
-        ssh_config = str(Path(ssh_config).resolve())
-
-        # Use pyinfra's native SSH data keys to configure the connection directly
-        # rather than relying on paramiko config parsing (see also sshexec.py)
-        ip = resolve_host_from_ssh_config(ssh_host, ssh_config)
-        key = resolve_key_from_ssh_config(ssh_host, ssh_config)
-        data_args = f"--data ssh_hostname={ip} --data ssh_known_hosts_file=/dev/null"
-        if key:
-            data_args += f" --data ssh_key={key}"
-        cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y {data_args}"
     if ssh_host in ["localhost", "@docker"]:
         if ssh_host == "@docker":
             env["CHATMAIL_NOPORTCHECK"] = "True"
@@ -153,25 +119,24 @@ def run_cmd(args, out):
         return 1
 
     try:
-        ret = out.shell(cmd, env=env)
-        if ret:
-            out.red("Deploy failed")
-            return 1
+        retcode = out.check_call(cmd, env=env)
         if args.website_only:
-            out.green("Website deployment completed.")
-        elif (
-            not args.dns_check_disabled
-            and strict_tls
-            and not remote_data["acme_account_url"]
-        ):
+            if retcode == 0:
+                out.green("Website deployment completed.")
+            else:
+                out.red("Website deployment failed.")
+        elif retcode == 0:
+            out.green("Deploy completed, call `cmdeploy dns` next.")
+        elif not args.dns_check_disabled and strict_tls and not remote_data["acme_account_url"]:
             out.red("Deploy completed but letsencrypt not configured")
             out.red("Run 'cmdeploy run' again")
+            retcode = 0
         else:
-            out.green("Deploy completed, call `cmdeploy dns` next.")
-        return 0
+            out.red("Deploy failed")
     except subprocess.CalledProcessError:
         out.red("Deploy failed")
-        return 1
+        retcode = 1
+    return retcode
 
 
 def dns_cmd_options(parser):
@@ -180,16 +145,15 @@ def dns_cmd_options(parser):
         dest="zonefile",
         type=pathlib.Path,
         default=None,
-        help="write DNS records in standard BIND format to the given file",
+        help="write out a zonefile",
     )
     add_ssh_host_option(parser)
-    add_ssh_config_option(parser)
 
 
 def dns_cmd(args, out):
     """Check DNS entries and optionally generate dns zone file."""
     ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
-    sshexec = get_sshexec(ssh_host, verbose=args.verbose, ssh_config=args.ssh_config)
+    sshexec = get_sshexec(ssh_host, verbose=args.verbose)
     tls_cert_mode = args.config.tls_cert_mode
     strict_tls = tls_cert_mode == "acme"
     remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
@@ -220,14 +184,13 @@ def dns_cmd(args, out):
 
 def status_cmd_options(parser):
     add_ssh_host_option(parser)
-    add_ssh_config_option(parser)
 
 
 def status_cmd(args, out):
     """Display status for online chatmail instance."""
 
     ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
-    sshexec = get_sshexec(ssh_host, verbose=args.verbose, ssh_config=args.ssh_config)
+    sshexec = get_sshexec(ssh_host, verbose=args.verbose)
 
     out.green(f"chatmail domain: {args.config.mail_domain}")
     if args.config.privacy_mail:
@@ -247,14 +210,14 @@ def test_cmd_options(parser):
         help="also run slow tests",
     )
     add_ssh_host_option(parser)
-    add_ssh_config_option(parser)
 
 
 def test_cmd(args, out):
     """Run local and online tests for chatmail deployment."""
 
     env = os.environ.copy()
-    env["CHATMAIL_INI"] = str(args.inipath.resolve())
+    if args.ssh_host:
+        env["CHATMAIL_SSH"] = args.ssh_host
 
     pytest_path = shutil.which("pytest")
     pytest_args = [
@@ -268,11 +231,7 @@ def test_cmd(args, out):
     ]
     if args.slow:
         pytest_args.append("--slow")
-    if args.ssh_host:
-        pytest_args.extend(["--ssh-host", args.ssh_host])
-    if args.ssh_config:
-        pytest_args.extend(["--ssh-config", str(Path(args.ssh_config).resolve())])
-    ret = out.shell(" ".join(pytest_args), env=env)
+    ret = out.run_ret(pytest_args, env=env)
     return ret
 
 
@@ -309,8 +268,8 @@ def fmt_cmd(args, out):
     format_args.extend(sources)
     check_args.extend(sources)
 
-    out.shell(" ".join(format_args), quiet=not args.verbose)
-    out.shell(" ".join(check_args), quiet=not args.verbose)
+    out.check_call(" ".join(format_args), quiet=not args.verbose)
+    out.check_call(" ".join(check_args), quiet=not args.verbose)
 
 
 def bench_cmd(args, out):
@@ -323,7 +282,9 @@ def bench_cmd(args, out):
 
 def webdev_cmd(args, out):
     """Run local web development loop for static web pages."""
-    webdev_main()
+    from .www import main
+
+    main()
 
 
 #
@@ -331,6 +292,32 @@ def webdev_cmd(args, out):
 #
 
 
+class Out:
+    """Convenience output printer providing coloring."""
+
+    def red(self, msg, file=sys.stderr):
+        print(colored(msg, "red"), file=file)
+
+    def green(self, msg, file=sys.stderr):
+        print(colored(msg, "green"), file=file)
+
+    def __call__(self, msg, red=False, green=False, file=sys.stdout):
+        color = "red" if red else ("green" if green else None)
+        print(colored(msg, color), file=file)
+
+    def check_call(self, arg, env=None, quiet=False):
+        if not quiet:
+            self(f"[$ {arg}]", file=sys.stderr)
+        return subprocess.check_call(arg, shell=True, env=env)
+
+    def run_ret(self, args, env=None, quiet=False):
+        if not quiet:
+            cmdstring = " ".join(args)
+            self(f"[$ {cmdstring}]", file=sys.stderr)
+        proc = subprocess.run(args, env=env, check=False)
+        return proc.returncode
+
+
 def add_ssh_host_option(parser):
     parser.add_argument(
         "--ssh-host",
@@ -340,16 +327,6 @@ def add_ssh_host_option(parser):
     )
 
 
-def add_ssh_config_option(parser):
-    parser.add_argument(
-        "--ssh-config",
-        dest="ssh_config",
-        type=Path,
-        default=None,
-        help="Path to an SSH config file (e.g. lxconfigs/ssh-config).",
-    )
-
-
 def add_config_option(parser):
     parser.add_argument(
         "--config",
@@ -359,26 +336,25 @@ def add_config_option(parser):
         type=Path,
         help="path to the chatmail.ini file",
     )
+    parser.add_argument(
+        "--verbose",
+        "-v",
+        dest="verbose",
+        action="store_true",
+        default=False,
+        help="provide verbose logging",
+    )
 
 
-def add_subcommand(subparsers, func, add_config=True):
+def add_subcommand(subparsers, func):
     name = func.__name__
     assert name.endswith("_cmd")
-    name = name[:-4].replace("_", "-")
+    name = name[:-4]
     doc = func.__doc__.strip()
     help = doc.split("\n")[0].strip(".")
     p = subparsers.add_parser(name, description=doc, help=help)
     p.set_defaults(func=func)
-    if add_config:
-        add_config_option(p)
-    p.add_argument(
-        "-v",
-        "--verbose",
-        dest="verbose",
-        action="count",
-        default=0,
-        help="increase verbosity (can be repeated: -v, -vv)",
-    )
+    add_config_option(p)
     return p
 
 
@@ -387,60 +363,45 @@ Setup your chatmail server configuration and
 deploy it via SSH to your remote location.
 """
 
-# Explicit subcommand registry: (cmd_func, options_func_or_None, needs_config).
-# LXC commands don't need a chatmail.ini (no config); all others do.
-SUBCOMMANDS = [
-    (init_cmd, init_cmd_options, True),
-    (run_cmd, run_cmd_options, True),
-    (dns_cmd, dns_cmd_options, True),
-    (status_cmd, status_cmd_options, True),
-    (test_cmd, test_cmd_options, True),
-    (fmt_cmd, fmt_cmd_options, True),
-    (bench_cmd, None, True),
-    (webdev_cmd, None, True),
-    (lxc_start_cmd, lxc_start_cmd_options, False),
-    (lxc_stop_cmd, lxc_stop_cmd_options, False),
-    (lxc_status_cmd, lxc_status_cmd_options, False),
-    (lxc_test_cmd, lxc_test_cmd_options, False),
-]
-
 
 def get_parser():
     """Return an ArgumentParser for the 'cmdeploy' CLI"""
 
     parser = argparse.ArgumentParser(description=description.strip())
-    parser.set_defaults(func=None, inipath=None)
     subparsers = parser.add_subparsers(title="subcommands")
 
-    for func, addopts, needs_config in SUBCOMMANDS:
-        subparser = add_subcommand(subparsers, func, add_config=needs_config)
-        if addopts is not None:
-            addopts(subparser)
+    # find all subcommands in the module namespace
+    glob = globals()
+    for name, func in glob.items():
+        if name.endswith("_cmd"):
+            subparser = add_subcommand(subparsers, func)
+            addopts = glob.get(name + "_options")
+            if addopts is not None:
+                addopts(subparser)
 
     return parser
 
 
-def get_sshexec(ssh_host: str, verbose=True, ssh_config=None):
+def get_sshexec(ssh_host: str, verbose=True):
     if ssh_host in ["localhost", "@local"]:
         return LocalExec(verbose, docker=False)
     elif ssh_host == "@docker":
         return LocalExec(verbose, docker=True)
     if verbose:
         print(f"[ssh] login to {ssh_host}")
-    return SSHExec(ssh_host, verbose=verbose, ssh_config=ssh_config)
+    return SSHExec(ssh_host, verbose=verbose)
 
 
 def main(args=None):
     """Provide main entry point for 'cmdeploy' CLI invocation."""
     parser = get_parser()
     args = parser.parse_args(args=args)
-    if args.func is None:
+    if not hasattr(args, "func"):
         return parser.parse_args(["-h"])
 
-    out = Out(verbosity=args.verbose)
+    out = Out()
     kwargs = {}
-
-    if args.inipath is not None and args.func.__name__ not in ("init_cmd", "fmt_cmd"):
+    if args.func.__name__ not in ("init_cmd", "fmt_cmd"):
         if not args.inipath.exists():
             out.red(f"expecting {args.inipath} to exist, run init first?")
             raise SystemExit(1)
@@ -455,9 +416,6 @@ def main(args=None):
         if res is None:
             res = 0
         return res
-    except DNSConfigurationError as exc:
-        out.red(str(exc))
-        return 1
     except KeyboardInterrupt:
         out.red("KeyboardInterrupt")
         sys.exit(130)

cmdeploy/src/cmdeploy/deployers.py

@@ -6,7 +6,7 @@ import os
 import shutil
 import subprocess
 import sys
-from io import BytesIO, StringIO
+from io import StringIO
 from pathlib import Path
 
 from chatmaild.config import read_config
@@ -17,12 +17,13 @@ from pyinfra.facts.files import Sha256File
 from pyinfra.facts.systemd import SystemdEnabled
 from pyinfra.operations import apt, files, pip, server, systemd
 
+from cmdeploy.cmdeploy import Out
+
 from .acmetool import AcmetoolDeployer
 from .basedeploy import (
     Deployer,
     Deployment,
     activate_remote_units,
-    blocked_service_startup,
     configure_remote_units,
     get_resource,
     has_systemd,
@@ -35,7 +36,6 @@ from .nginx.deployer import NginxDeployer
 from .opendkim.deployer import OpendkimDeployer
 from .postfix.deployer import PostfixDeployer
 from .selfsigned.deployer import SelfSignedTlsDeployer
-from .util import Out, get_version_string
 from .www import build_webpages, find_merge_conflict, get_paths
 
 
@@ -123,6 +123,7 @@ def _install_remote_venv_with_chatmaild() -> None:
 
 def _configure_remote_venv_with_chatmaild(config) -> None:
     remote_base_dir = "/usr/local/lib/chatmaild"
+    remote_venv_dir = f"{remote_base_dir}/venv"
     remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini"
     root_owned = dict(user="root", group="root", mode="644")
 
@@ -133,13 +134,16 @@ def _configure_remote_venv_with_chatmaild(config) -> None:
         **root_owned,
     )
 
-    files.file(
-        path="/etc/cron.d/chatmail-metrics",
-        present=False,
-    )
-    files.file(
-        path="/var/www/html/metrics",
-        present=False,
+    files.template(
+        src=get_resource("metrics.cron.j2"),
+        dest="/etc/cron.d/chatmail-metrics",
+        user="root",
+        group="root",
+        mode="644",
+        config={
+            "mailboxes_dir": config.mailboxes_dir,
+            "execpath": f"{remote_venv_dir}/bin/chatmail-metrics",
+        },
     )
 
 
@@ -149,16 +153,33 @@ class UnboundDeployer(Deployer):
         self.need_restart = False
 
     def install(self):
-        # Run local DNS resolver `unbound`. `resolvconf` takes care of
-        # setting up /etc/resolv.conf to use 127.0.0.1 as the resolver.
+        # Run local DNS resolver `unbound`.
+        # `resolvconf` takes care of setting up /etc/resolv.conf
+        # to use 127.0.0.1 as the resolver.
 
-        # On an IPv4-only system, if unbound is started but not configured,
-        # it causes subsequent steps to fail to resolve hosts.
-        with blocked_service_startup():
-            apt.packages(
-                name="Install unbound",
-                packages=["unbound", "unbound-anchor", "dnsutils"],
-            )
+        #
+        # On an IPv4-only system, if unbound is started but not
+        # configured, it causes subsequent steps to fail to resolve hosts.
+        # Here, we use policy-rc.d to prevent unbound from starting up
+        # on initial install.  Later, we will configure it and start it.
+        #
+        # For documentation about policy-rc.d, see:
+        # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
+        #
+        files.put(
+            src=get_resource("policy-rc.d"),
+            dest="/usr/sbin/policy-rc.d",
+            user="root",
+            group="root",
+            mode="755",
+        )
+
+        apt.packages(
+            name="Install unbound",
+            packages=["unbound", "unbound-anchor", "dnsutils"],
+        )
+
+        files.file("/usr/sbin/policy-rc.d", present=False)
 
     def configure(self):
         server.shell(
@@ -250,18 +271,9 @@ class WebsiteDeployer(Deployer):
             # if www_folder is a hugo page, build it
             if build_dir:
                 www_path = build_webpages(src_dir, build_dir, self.config)
-                if www_path is None:
-                    logger.warning("Web page build failed, skipping website deployment")
-                    return
             # if it is not a hugo page, upload it as is
-            # pyinfra files.rsync (experimental) causes problems with ssh-config configuration
-            # the stable files.sync should do
-            files.sync(
-                src=str(www_path),
-                dest="/var/www/html",
-                user="www-data",
-                group="www-data",
-                delete=True,
+            files.rsync(
+                f"{www_path}/", "/var/www/html", flags=["-avz", "--chown=www-data"]
             )
 
 
@@ -325,12 +337,12 @@ class TurnDeployer(Deployer):
     def install(self):
         (url, sha256sum) = {
             "x86_64": (
-                "https://github.com/chatmail/chatmail-turn/releases/download/v0.4/chatmail-turn-x86_64-linux",
-                "1ec1f5c50122165e858a5a91bcba9037a28aa8cb8b64b8db570aa457c6141a8a",
+                "https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-x86_64-linux",
+                "841e527c15fdc2940b0469e206188ea8f0af48533be12ecb8098520f813d41e4",
             ),
             "aarch64": (
-                "https://github.com/chatmail/chatmail-turn/releases/download/v0.4/chatmail-turn-aarch64-linux",
-                "0fb3e792419494e21ecad536464929dba706bb2c88884ed8f1788141d26fc756",
+                "https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-aarch64-linux",
+                "a5fc2d06d937b56a34e098d2cd72a82d3e89967518d159bf246dc69b65e81b42",
             ),
         }[host.get_fact(facts.server.Arch)]
 
@@ -463,19 +475,10 @@ class ChatmailDeployer(Deployer):
         ("iroh", None, None),
     ]
 
-    def __init__(self, config):
-        self.config = config
-        self.mail_domain = config.mail_domain
+    def __init__(self, mail_domain):
+        self.mail_domain = mail_domain
 
     def install(self):
-        files.put(
-            name="Disable installing recommended packages globally",
-            src=BytesIO(b'APT::Install-Recommends "false";\n'),
-            dest="/etc/apt/apt.conf.d/00InstallRecommends",
-            user="root",
-            group="root",
-            mode="644",
-        )
         apt.update(name="apt update", cache_time=24 * 3600)
         apt.upgrade(name="upgrade apt packages", auto_remove=True)
 
@@ -488,19 +491,12 @@ class ChatmailDeployer(Deployer):
             name="Install rsync",
             packages=["rsync"],
         )
-
-    def configure(self):
-        # Ensure the per-domain mailbox directory exists before
-        # chatmail-metadata starts (it crashes without it).
-        files.directory(
-            name="Ensure vmail mailbox directory exists",
-            path=f"/home/vmail/mail/{self.mail_domain}",
-            user="vmail",
-            group="vmail",
-            mode="700",
-            present=True,
+        apt.packages(
+            name="Ensure cron is installed",
+            packages=["cron"],
         )
 
+    def configure(self):
         # This file is used by auth proxy.
         # https://wiki.debian.org/EtcMailName
         server.shell(
@@ -510,15 +506,6 @@ class ChatmailDeployer(Deployer):
             ],
         )
 
-        files.directory(
-            name=f"Ensure mailboxes directory {self.config.mailboxes_dir} exists",
-            path=str(self.config.mailboxes_dir),
-            user="vmail",
-            group="vmail",
-            mode="700",
-            present=True,
-        )
-
 
 class FcgiwrapDeployer(Deployer):
     def install(self):
@@ -538,9 +525,17 @@ class FcgiwrapDeployer(Deployer):
 
 class GithashDeployer(Deployer):
     def activate(self):
+        try:
+            git_hash = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode()
+        except Exception:
+            git_hash = "unknown\n"
+        try:
+            git_diff = subprocess.check_output(["git", "diff"]).decode()
+        except Exception:
+            git_diff = ""
         files.put(
             name="Upload chatmail relay git commit hash",
-            src=StringIO(get_version_string()),
+            src=StringIO(git_hash + git_diff),
             dest="/etc/chatmail-version",
             mode="700",
         )
@@ -584,17 +579,11 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
         )
 
     # Check if mtail_address interface is available (if configured)
-    if config.mtail_address and config.mtail_address not in (
-        "127.0.0.1",
-        "::1",
-        "localhost",
-    ):
+    if config.mtail_address and config.mtail_address not in ('127.0.0.1', '::1', 'localhost'):
         ipv4_addrs = host.get_fact(hardware.Ipv4Addrs)
         all_addresses = [addr for addrs in ipv4_addrs.values() for addr in addrs]
         if config.mtail_address not in all_addresses:
-            Out().red(
-                f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n"
-            )
+            Out().red(f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n")
             exit(1)
 
     if not os.environ.get("CHATMAIL_NOPORTCHECK"):
@@ -637,7 +626,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
     tls_deployer = get_tls_deployer(config, mail_domain)
 
     all_deployers = [
-        ChatmailDeployer(config),
+        ChatmailDeployer(mail_domain),
         LegacyRemoveDeployer(),
         FiltermailDeployer(),
         JournaldDeployer(),

cmdeploy/src/cmdeploy/dns.py

@@ -1,26 +1,11 @@
 import datetime
+import importlib
+
+from jinja2 import Template
 
 from . import remote
 
 
-def parse_zone_records(text):
-    """Yield ``(name, ttl, rtype, rdata)`` from standard BIND-format text.
-
-    Skips comment lines (starting with ``;``) and blank lines.
-    Each record line must have the format ``name TTL IN type rdata``.
-    """
-    for raw_line in text.splitlines():
-        line = raw_line.strip()
-        if not line or line.startswith(";"):
-            continue
-        try:
-            name, ttl, _in, rtype, rdata = line.split(None, 4)
-        except ValueError:
-            raise ValueError(f"Bad zone record line: {line!r}") from None
-        name = name.rstrip(".")
-        yield name, ttl, rtype.upper(), rdata
-
-
 def get_initial_remote_data(sshexec, mail_domain):
     return sshexec.logged(
         call=remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=mail_domain)
@@ -46,43 +31,13 @@ def get_filled_zone_file(remote_data):
     if not sts_id:
         remote_data["sts_id"] = datetime.datetime.now().strftime("%Y%m%d%H%M")
 
-    d = remote_data["mail_domain"]
-
-    def rec(name, rtype, rdata, ttl=3600):
-        return f"{name:<40} {ttl:<6} IN  {rtype:<5}  {rdata}"
-
-    lines = ["; Required DNS entries"]
-    if remote_data.get("A"):
-        lines.append(rec(f"{d}.", "A", remote_data["A"]))
-    if remote_data.get("AAAA"):
-        lines.append(rec(f"{d}.", "AAAA", remote_data["AAAA"]))
-    lines.append(rec(f"{d}.", "MX", f"10 {d}."))
-    if remote_data.get("strict_tls"):
-        lines.append(
-            rec(f"_mta-sts.{d}.", "TXT", f'"v=STSv1; id={remote_data["sts_id"]}"')
-        )
-        lines.append(rec(f"mta-sts.{d}.", "CNAME", f"{d}."))
-    lines.append(rec(f"www.{d}.", "CNAME", f"{d}."))
-    lines.append(remote_data["dkim_entry"])
+    template = importlib.resources.files(__package__).joinpath("chatmail.zone.j2")
+    content = template.read_text()
+    zonefile = Template(content).render(**remote_data)
+    lines = [x.strip() for x in zonefile.split("\n") if x.strip()]
     lines.append("")
-    lines.append("; Recommended DNS entries")
-    lines.append(rec(f"{d}.", "TXT", '"v=spf1 a ~all"'))
-    lines.append(rec(f"_dmarc.{d}.", "TXT", '"v=DMARC1;p=reject;adkim=s;aspf=s"'))
-    if remote_data.get("acme_account_url"):
-        lines.append(
-            rec(
-                f"{d}.",
-                "CAA",
-                f'0 issue "letsencrypt.org;accounturi={remote_data["acme_account_url"]}"',
-            )
-        )
-    lines.append(rec(f"_adsp._domainkey.{d}.", "TXT", '"dkim=discardable"'))
-    lines.append(rec(f"_submission._tcp.{d}.", "SRV", f"0 1 587 {d}."))
-    lines.append(rec(f"_submissions._tcp.{d}.", "SRV", f"0 1 465 {d}."))
-    lines.append(rec(f"_imap._tcp.{d}.", "SRV", f"0 1 143 {d}."))
-    lines.append(rec(f"_imaps._tcp.{d}.", "SRV", f"0 1 993 {d}."))
-    lines.append("")
-    return "\n".join(lines)
+    zonefile = "\n".join(lines)
+    return zonefile
 
 
 def check_full_zone(sshexec, remote_data, out, zonefile) -> int:
@@ -98,19 +53,18 @@ def check_full_zone(sshexec, remote_data, out, zonefile) -> int:
     if required_diff:
         out.red("Please set required DNS entries at your DNS provider:\n")
         for line in required_diff:
-            out.print(line)
-        out.print()
+            out(line)
+        out("")
         returncode = 1
     if remote_data.get("dkim_entry") in required_diff:
-        out.print(
-            "If the DKIM entry above does not work with your DNS provider,"
-            " you can try this one:\n"
+        out(
+            "If the DKIM entry above does not work with your DNS provider, you can try this one:\n"
         )
-        out.print(remote_data.get("web_dkim_entry") + "\n")
+        out(remote_data.get("web_dkim_entry") + "\n")
     if recommended_diff:
-        out.print("WARNING: these recommended DNS entries are not set:\n")
+        out("WARNING: these recommended DNS entries are not set:\n")
         for line in recommended_diff:
-            out.print(line)
+            out(line)
 
     if not (recommended_diff or required_diff):
         out.green("Great! All your DNS entries are verified and correct.")

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -1,30 +1,19 @@
-import urllib.request
+import os
 
 from chatmaild.config import Config
 from pyinfra import host
-from pyinfra.facts.deb import DebPackages
-from pyinfra.facts.server import Arch, Command, Sysctl
+from pyinfra.facts.server import Arch, Sysctl
+from pyinfra.facts.systemd import SystemdEnabled
 from pyinfra.operations import apt, files, server, systemd
 
 from cmdeploy.basedeploy import (
     Deployer,
     activate_remote_units,
-    blocked_service_startup,
     configure_remote_units,
     get_resource,
+    has_systemd,
 )
 
-DOVECOT_VERSION = "2.3.21+dfsg1-3"
-
-DOVECOT_SHA256 = {
-    ("core", "amd64"): "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d",
-    ("core", "arm64"): "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9",
-    ("imapd", "amd64"): "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86",
-    ("imapd", "arm64"): "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f",
-    ("lmtpd", "amd64"): "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab",
-    ("lmtpd", "arm64"): "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f",
-}
-
 
 class DovecotDeployer(Deployer):
     daemon_reload = False
@@ -36,22 +25,11 @@ class DovecotDeployer(Deployer):
 
     def install(self):
         arch = host.get_fact(Arch)
-        with blocked_service_startup():
-            debs = []
-            for pkg in ("core", "imapd", "lmtpd"):
-                deb = _download_dovecot_package(pkg, arch)
-                if deb:
-                    debs.append(deb)
-            if debs:
-                deb_list = " ".join(debs)
-                server.shell(
-                    name="Install dovecot packages",
-                    commands=[
-                        f"dpkg --force-confdef --force-confold -i {deb_list} 2> /dev/null || true",
-                        "DEBIAN_FRONTEND=noninteractive apt-get -y --fix-broken install",
-                        f"dpkg --force-confdef --force-confold -i {deb_list}",
-                    ],
-                )
+        if has_systemd() and "dovecot.service" in host.get_fact(SystemdEnabled):
+            return  # already installed and running
+        _install_dovecot_package("core", arch)
+        _install_dovecot_package("imapd", arch)
+        _install_dovecot_package("lmtpd", arch)
 
     def configure(self):
         configure_remote_units(self.config.mail_domain, self.units)
@@ -63,9 +41,7 @@ class DovecotDeployer(Deployer):
         restart = False if self.disable_mail else self.need_restart
 
         systemd.service(
-            name="Disable dovecot for now"
-            if self.disable_mail
-            else "Start and enable Dovecot",
+            name="Disable dovecot for now" if self.disable_mail else "Start and enable Dovecot",
             service="dovecot.service",
             running=False if self.disable_mail else True,
             enabled=False if self.disable_mail else True,
@@ -75,46 +51,38 @@ class DovecotDeployer(Deployer):
         self.need_restart = False
 
 
-def _pick_url(primary, fallback):
-    try:
-        req = urllib.request.Request(primary, method="HEAD")
-        urllib.request.urlopen(req, timeout=10)
-        return primary
-    except Exception:
-        return fallback
-
-
-def _download_dovecot_package(package: str, arch: str):
-    """Download a dovecot .deb if needed, return its path (or None)."""
+def _install_dovecot_package(package: str, arch: str):
     arch = "amd64" if arch == "x86_64" else arch
     arch = "arm64" if arch == "aarch64" else arch
+    url = f"https://download.delta.chat/dovecot/dovecot-{package}_2.3.21%2Bdfsg1-3_{arch}.deb"
+    deb_filename = "/root/" + url.split("/")[-1]
 
-    pkg_name = f"dovecot-{package}"
-    sha256 = DOVECOT_SHA256.get((package, arch))
-    if sha256 is None:
-        apt.packages(packages=[pkg_name])
-        return None
-
-    installed_versions = host.get_fact(DebPackages).get(pkg_name, [])
-    if DOVECOT_VERSION in installed_versions:
-        return None
-
-    url_version = DOVECOT_VERSION.replace("+", "%2B")
-    deb_base = f"{pkg_name}_{url_version}_{arch}.deb"
-    primary_url = f"https://download.delta.chat/dovecot/{deb_base}"
-    fallback_url = f"https://github.com/chatmail/dovecot/releases/download/upstream%2F{url_version}/{deb_base}"
-    url = _pick_url(primary_url, fallback_url)
-    deb_filename = f"/root/{deb_base}"
+    match (package, arch):
+        case ("core", "amd64"):
+            sha256 = "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d"
+        case ("core", "arm64"):
+            sha256 = "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9"
+        case ("imapd", "amd64"):
+            sha256 = "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86"
+        case ("imapd", "arm64"):
+            sha256 = "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f"
+        case ("lmtpd", "amd64"):
+            sha256 = "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab"
+        case ("lmtpd", "arm64"):
+            sha256 = "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f"
+        case _:
+            apt.packages(packages=[f"dovecot-{package}"])
+            return
 
     files.download(
-        name=f"Download {pkg_name}",
+        name=f"Download dovecot-{package}",
         src=url,
         dest=deb_filename,
         sha256sum=sha256,
         cache_time=60 * 60 * 24 * 365 * 10,  # never redownload the package
     )
 
-    return deb_filename
+    apt.deb(name=f"Install dovecot-{package}", src=deb_filename)
 
 
 def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
@@ -152,25 +120,19 @@ def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
 
     # as per https://doc.dovecot.org/2.3/configuration_manual/os/
     # it is recommended to set the following inotify limits
-    can_modify = host.get_fact(Command, "systemd-detect-virt -c || true") == "none"
-    for name in ("max_user_instances", "max_user_watches"):
-        key = f"fs.inotify.{name}"
-        value = host.get_fact(Sysctl)[key]
-        if value > 65534:
-            continue
-        if not can_modify:
-            print(
-                "\n!!!! refusing to attempt sysctl setting in shared-kernel containers\n"
-                f"!!!! dovecot: sysctl {key!r}={value}, should be >65534 for production setups\n"
-                "!!!!"
+    if not os.environ.get("CHATMAIL_NOSYSCTL"):
+        for name in ("max_user_instances", "max_user_watches"):
+            key = f"fs.inotify.{name}"
+            if host.get_fact(Sysctl)[key] > 65535:
+                # Skip updating limits if already sufficient
+                # (enables running in incus containers where sysctl readonly)
+                continue
+            server.sysctl(
+                name=f"Change {key}",
+                key=key,
+                value=65535,
+                persist=True,
             )
-            continue
-        server.sysctl(
-            name=f"Change {key}",
-            key=key,
-            value=65535,
-            persist=True,
-        )
 
     timezone_env = files.line(
         name="Set TZ environment variable",

cmdeploy/src/cmdeploy/filtermail/deployer.py

@@ -14,10 +14,10 @@ class FiltermailDeployer(Deployer):
 
     def install(self):
         arch = host.get_fact(facts.server.Arch)
-        url = f"https://github.com/chatmail/filtermail/releases/download/v0.6.0/filtermail-{arch}"
+        url = f"https://github.com/chatmail/filtermail/releases/download/v0.5.1/filtermail-{arch}"
         sha256sum = {
-            "x86_64": "3fd8b18282252c75a5bbfa603d8c1b65f6563e5e920bddf3e64e451b7cdb43ce",
-            "aarch64": "2bd191de205f7fd60158dd8e3516ab7e3efb14627696f3d7dc186bdcd9e10a43",
+            "x86_64": "adce2ddb461c5fd744df699f3b0b3c33b6d52413c641f18695b93826e5e0d234",
+            "aarch64": "b51cf4248c6c443308f21b1811da1cc919b98b719a2138f4b60940ea093a5422",
         }[arch]
         self.need_restart |= files.download(
             name="Download filtermail",

cmdeploy/src/cmdeploy/lxc/cli.py

@@ -1,475 +0,0 @@
-"""lxc-start/stop/status/test subcommands for testing with local containers."""
-
-import os
-import time
-
-from ..util import get_git_hash, get_version_string, shell
-from .incus import RELAY_IMAGE_ALIAS, Incus, RelayContainer
-
-RELAY_NAMES = ("test0", "test1")
-
-
-# -------------------------------------------------------------------
-# lxc-start
-# -------------------------------------------------------------------
-
-
-def lxc_start_cmd_options(parser):
-    _add_name_args(
-        parser,
-        help_text="User relay name(s) to create (default: test0).",
-    )
-    parser.add_argument(
-        "--ipv4-only",
-        dest="ipv4_only",
-        action="store_true",
-        help="Create an IPv4-only container.",
-    )
-    parser.add_argument(
-        "--run",
-        action="store_true",
-        help="Run 'cmdeploy run' on each container after starting it.",
-    )
-
-
-def lxc_start_cmd(args, out):
-    """Create/Ensure and start LXC relay and DNS containers."""
-
-    with out.section("Preparing container setup"):
-        _lxc_start_cmd(args, out)
-
-
-def _lxc_start_cmd(args, out):
-    ix = Incus(out)
-    sub = out.new_prefixed_out()
-    out.green("Ensuring base image ...")
-    ix.ensure_base_image()
-    out.green("Ensuring DNS container (ns-localchat) ...")
-    dns_ct = ix.get_dns_container()
-    dns_ct.ensure()
-    sub.print(f"DNS container IP: {dns_ct.ipv4}")
-
-    names = args.names if args.names else RELAY_NAMES
-    relays = list(ix.get_container(n) for n in names)
-    for ct in relays:
-        out.green(f"Ensuring container {ct.name!r} ({ct.domain}) ...")
-        ct.ensure()
-        ip = ct.ipv4
-
-        sub.print("Configuring container hostname ...")
-        ct.configure_hosts(ip)
-
-        sub.print(f"Writing {ct.ini.name} ...")
-        ct.write_ini(disable_ipv6=args.ipv4_only)
-        sub.print(f"Config: {ct.ini}")
-        if args.ipv4_only:
-            ct.disable_ipv6()
-            ipv6 = None
-        else:
-            output = ct.bash(
-                "ip -6 addr show scope global -deprecated"
-                " | grep -oP '(?<=inet6 )[^/]+'",
-                check=False,
-            )
-            ipv6 = output.strip() if output else None
-        sub.print(f"{_format_addrs(ip, ipv6)}")
-
-        sub.green(f"Container {ct.name!r} ready: {ct.domain} -> {ip}")
-        out.print()
-
-    # Reset DNS zones only for the containers we just started
-    started_cnames = {ct.name for ct in relays}
-    managed = ix.list_managed()
-    started = [c for c in managed if c["name"] in started_cnames]
-
-    if started:
-        out.print(
-            f"Resetting DNS zones for {len(started)} domain(s) (A + AAAA records) ..."
-        )
-        dns_ct.reset_dns_records(dns_ct.ipv4, started)
-
-        for ct in relays:
-            if ct.name in started_cnames:
-                sub.print(f"Configuring DNS in {ct.name} ...")
-                ct.configure_dns(dns_ct.ipv4)
-
-    # Generate the unified SSH config
-    out.green("Writing ssh-config ...")
-    ssh_cfg = ix.write_ssh_config()
-    sub.print(f"{ssh_cfg}")
-
-    # Verify SSH via the generated config
-    for ct in relays:
-        sub.print(f"Verifying SSH to {ct.name} via ssh-config ...")
-        if ct.verify_ssh(ssh_cfg):
-            sub.print(f"SSH OK: ssh -F lxconfigs/ssh-config {ct.domain}")
-        else:
-            sub.red(f"WARNING: SSH verification failed for {ct.name}")
-
-    # Print integration suggestions
-    ssh_cfg = ix.ssh_config_path
-    if not ix.check_ssh_include():
-        sub.green(
-            "\n(Optional) To use containers from any SSH client, add to ~/.ssh/config:"
-        )
-        sub.green(f"    Include {ssh_cfg}")
-
-    # Optionally run cmdeploy run + dns on each relay
-    if args.run:
-        for ct in relays:
-            with out.section(f"cmdeploy run: {ct.sname} ({ct.domain})"):
-                ret = _run_cmdeploy("run", ct, ix, out, extra=["--skip-dns-check"])
-                if ret:
-                    out.red(f"Deploy to {ct.sname} failed (exit {ret})")
-                    return ret
-
-        with out.section("loading DNS zones"):
-            for ct in relays:
-                ret = _run_cmdeploy(
-                    "dns", ct, ix, out,
-                    extra=["--zonefile", str(ct.zone)],
-                )
-                if ret:
-                    out.red(f"DNS for {ct.sname} failed (exit {ret})")
-                    return ret
-                if ct.zone.exists():
-                    dns_ct.set_dns_records(ct.zone.read_text())
-                out.print(f"Restarting filtermail-incoming on {ct.name}")
-                ct.bash("systemctl restart filtermail-incoming")
-
-
-# -------------------------------------------------------------------
-# lxc-stop
-# -------------------------------------------------------------------
-
-
-def lxc_stop_cmd_options(parser):
-    parser.add_argument(
-        "--destroy",
-        action="store_true",
-        help="Delete containers and their config files after stopping.",
-    )
-    parser.add_argument(
-        "--destroy-all",
-        dest="destroy_all",
-        action="store_true",
-        help="Like --destroy, but also remove the ns-localchat DNS container.",
-    )
-    _add_name_args(
-        parser,
-        help_text="Container name(s) to stop (default: test0 + test1).",
-    )
-
-
-def lxc_stop_cmd(args, out):
-    """Stop (and optionally destroy) local LXC relay containers."""
-    ix = Incus(out)
-    names = args.names or RELAY_NAMES
-    destroy = args.destroy or args.destroy_all
-
-    for ct in map(ix.get_container, names):
-        if destroy:
-            out.green(f"Destroying container {ct.name!r} ...")
-            ct.destroy()
-        else:
-            out.green(f"Stopping container {ct.name!r} ...")
-            ct.stop(force=True)
-
-    if args.destroy_all:
-        dns_ct = ix.get_dns_container()
-        out.green(f"Destroying DNS container {dns_ct.name!r} ...")
-        dns_ct.destroy()
-        ix.delete_images()
-
-    if destroy:
-        ix.write_ssh_config()
-        out.green("LXC containers destroyed.")
-    else:
-        out.green("LXC containers stopped.")
-
-
-# -------------------------------------------------------------------
-# lxc-test
-# -------------------------------------------------------------------
-
-
-def lxc_test_cmd_options(parser):
-    parser.add_argument(
-        "--one",
-        action="store_true",
-        help="Only deploy and test against test0 (skip test1).",
-    )
-
-
-def lxc_test_cmd(args, out):
-    """Run full LXC pipeline: start, deploy, DNS, zone files, and tests.
-
-    All commands run directly on the host using
-    ``--ssh-config lxconfigs/ssh-config`` for SSH access.
-    """
-    ix = Incus(out)
-    t_total = time.time()
-    relay_names = list(RELAY_NAMES)
-    if args.one:
-        relay_names = relay_names[:1]
-
-    local_hash = get_git_hash()
-
-    # Per-relay: start, deploy, then snapshot the first relay as a
-    # reusable image so the second relay launches pre-deployed.
-    ipv4_only_flags = {RELAY_NAMES[0]: False, RELAY_NAMES[1]: True}
-
-    for ct in map(ix.get_container, relay_names):
-        name = ct.sname
-        ipv4_only = ipv4_only_flags.get(name, False)
-        v_flag = " -" + "v" * out.verbosity if out.verbosity > 0 else ""
-        start_cmd = f"cmdeploy lxc-start{v_flag} {name}"
-        if ipv4_only:
-            start_cmd += " --ipv4-only"
-        with out.section(f"cmdeploy lxc-start: {name}"):
-            ret = out.shell(start_cmd, cwd=str(ix.project_root))
-            if ret:
-                return ret
-
-        status = _deploy_status(ct, local_hash, ix)
-        with out.section(f"cmdeploy run: {name}"):
-            if "IN-SYNC" in status:
-                out.print(f"{name} is {status}, skipping")
-            else:
-                ret = _run_cmdeploy("run", ct, ix, out, extra=["--skip-dns-check"])
-                if ret:
-                    out.red(f"Deploy to {name} failed (exit {ret})")
-                    return ret
-
-        # Snapshot the first relay so subsequent ones launch pre-deployed
-        if not ix.find_image([RELAY_IMAGE_ALIAS]):
-            with out.section("lxc-test: caching relay image"):
-                ct.publish_as_relay_image()
-
-    for ct in map(ix.get_container, relay_names):
-        with out.section(f"cmdeploy dns: {ct.sname} ({ct.domain})"):
-            ret = _run_cmdeploy("dns", ct, ix, out, extra=["--zonefile", str(ct.zone)])
-            if ret:
-                out.red(f"DNS for {ct.sname} failed (exit {ret})")
-                return ret
-
-    with out.section(f"lxc-test: loading DNS zones {' & '.join(relay_names)}"):
-        dns_ct = ix.get_dns_container()
-        for ct in map(ix.get_container, relay_names):
-            if ct.zone.exists():
-                zone_data = ct.zone.read_text()
-                out.print(f"Loading {ct.zone} into PowerDNS ...")
-                dns_ct.set_dns_records(zone_data)
-
-        # Restart filtermail so its in-process DNS cache
-        # does not hold stale negative DKIM responses
-        # from before the zones were loaded.
-        for ct in map(ix.get_container, relay_names):
-            out.print(f"Restarting filtermail-incoming on {ct.name} ...")
-            ct.bash("systemctl restart filtermail-incoming")
-
-    with out.section("cmdeploy test"):
-        first = ix.get_container(relay_names[0])
-        env = None
-        if len(relay_names) > 1:
-            env = os.environ.copy()
-            env["CHATMAIL_DOMAIN2"] = ix.get_container(relay_names[1]).domain
-        ret = _run_cmdeploy("test", first, ix, out, **({"env": env} if env else {}))
-        if ret:
-            out.red(f"Tests failed (exit {ret})")
-            return ret
-
-    elapsed = time.time() - t_total
-    out.section_line(f"lxc-test complete ({elapsed:.1f}s)")
-    if out.section_timings:
-        out.print("Section timings:")
-        for name, secs in out.section_timings:
-            out.print(f"  {name:.<50s} {secs:5.1f}s")
-        out.print(f"  {'total':.<50s} {elapsed:5.1f}s")
-    out.section_timings.clear()
-    return 0
-
-
-# -------------------------------------------------------------------
-# lxc-status
-# -------------------------------------------------------------------
-
-
-def lxc_status_cmd_options(parser):
-    pass
-
-
-def lxc_status_cmd(args, out):
-    """Show status of local LXC chatmail containers."""
-    ix = Incus(out)
-    containers = ix.list_managed()
-    if not containers:
-        out.red("No LXC containers found.  Run 'cmdeploy lxc-start' first.")
-        return 1
-
-    local_hash = get_git_hash()
-
-    # Get storage pool path for display
-    storage_path = None
-    data = ix.run_json(["storage", "show", "default"], check=False)
-    if data:
-        storage_path = data.get("config", {}).get("source")
-    msg = "Container status"
-    if storage_path:
-        msg += f": {storage_path}"
-    out.section_line(msg)
-
-    dns_ip = None
-    for c in containers:
-        _print_container_status(out, c, ix, local_hash)
-        if c["name"] == ix.get_dns_container().name:
-            dns_ip = c["ip"]
-
-    out.section_line("Host ssh and DNS configuration")
-    _print_ssh_status(out, ix)
-    _print_dns_forwarding_status(out, dns_ip)
-    return 0
-
-
-def _print_container_status(out, c, ix, local_hash):
-    """Print name/status, domain/IPs, and RAM for one container."""
-    cname = c["name"]
-    is_running = c.get("status") == "Running"
-    ct = ix.get_container(cname)
-
-    # First line: name + running/STOPPED + deploy status
-    if not is_running:
-        tag = "STOPPED"
-    elif not isinstance(ct, RelayContainer):
-        tag = "running"
-    else:
-        tag = f"running {_deploy_status(ct, local_hash, ix)}"
-    out.print(f"{cname:20s} {tag}")
-
-    # Second line: domain, IPv4, IPv6
-    domain = c.get("domain", "")
-    ip = c.get("ip") or "?"
-    ipv6 = c.get("ipv6")
-    out.print(f"{domain:20s} {_format_addrs(ip, ipv6)}")
-
-    # Third line: RAM (RSS), config
-    detail_out = out.new_prefixed_out(" " * 21)
-    try:
-        used, total = ct.rss_mib()
-    except Exception:
-        ram_str = "RSS ?"
-    else:
-        ram_str = f"RSS {used}/{total} MiB ({used * 100 // total}%)"
-
-    if isinstance(ct, RelayContainer):
-        detail = f"{ram_str}, config: {os.path.relpath(ct.ini)}"
-    else:
-        detail = ram_str
-
-    detail_out.print(detail)
-    out.print()
-
-
-def _print_ssh_status(out, ix):
-    """Print SSH integration status."""
-    ssh_cfg = ix.ssh_config_path
-    if ix.check_ssh_include():
-        out.green("SSH: ~/.ssh/config includes lxconfigs/ssh-config ✓")
-    else:
-        out.red("SSH: ~/.ssh/config does NOT include lxconfigs/ssh-config")
-        sub = out.new_prefixed_out()
-        sub.print("Add to ~/.ssh/config:")
-        sub.print(f"    Include {ssh_cfg}")
-
-
-def _print_dns_forwarding_status(out, dns_ip):
-    """Print host DNS forwarding status for .localchat."""
-    sub = out.new_prefixed_out()
-    if not dns_ip:
-        out.red("DNS: ns-localchat container not found")
-        return
-    try:
-        rv = shell("resolvectl status incusbr0")
-        dns_ok = dns_ip in rv.stdout and "localchat" in rv.stdout
-    except Exception:
-        dns_ok = None
-    if dns_ok is True:
-        out.green(f"DNS: .localchat forwarding to {dns_ip} ✓")
-    elif dns_ok is False:
-        out.red("DNS: .localchat forwarding NOT configured")
-        sub.print("Run:")
-        sub.print(f"    sudo resolvectl dns incusbr0 {dns_ip}")
-        sub.print("    sudo resolvectl domain incusbr0 ~localchat")
-    else:
-        sub.print("DNS: .localchat forwarding status UNKNOWN")
-
-
-# -------------------------------------------------------------------
-# Internal helpers
-# -------------------------------------------------------------------
-
-
-def _format_addrs(ip, ipv6=None):
-    parts = [f"IPv4 {ip}"]
-    if ipv6:
-        parts.append(f"IPv6 {ipv6}")
-    return ", ".join(parts)
-
-
-def _deploy_status(ct, local_hash, ix):
-    """Return a human-readable deploy status string.
-
-    Compares the full deployed version (hash + diff) against
-    the local state built by :func:`~cmdeploy.util.get_version_string`.
-    """
-    deployed = ct.deployed_version()
-    if deployed is None:
-        return "NOT DEPLOYED"
-
-    # A container launched from the relay image has the same
-    # git hash but a different domain — always redeploy.
-    deployed_domain = ct.deployed_domain()
-    if deployed_domain and deployed_domain != ct.domain:
-        return f"DOMAIN-MISMATCH (deployed: {deployed_domain})"
-
-    deployed_lines = deployed.splitlines()
-    deployed_hash = deployed_lines[0] if deployed_lines else ""
-    short = deployed_hash[:12]
-
-    if not local_hash:
-        return f"UNKNOWN (deployed: {short})"
-
-    local_short = local_hash[:12]
-    if deployed_hash != local_hash:
-        return f"STALE (deployed: {short}, local: {local_short})"
-
-    # Hash matches — check for uncommitted diffs
-    local_version = get_version_string()
-    if deployed != local_version:
-        return f"DIRTY ({local_short}, undeployed changes)"
-
-    return f"IN-SYNC ({short})"
-
-
-def _add_name_args(parser, help_text):
-    parser.add_argument("names", nargs="*", metavar="NAME", help=help_text)
-
-
-def _run_cmdeploy(subcmd, ct, ix, out, extra=None, **kwargs):
-    """Run ``cmdeploy <subcmd>`` with standard --config/--ssh flags.
-
-    *ct* is a Container (uses ``ct.ini`` and ``ct.domain``).
-    Returns the subprocess exit code.
-    """
-    extra_str = " ".join(extra) if extra else ""
-    v_flag = " -" + "v" * out.verbosity if out.verbosity > 0 else ""
-    cmd = f"""
-        cmdeploy {subcmd}{v_flag}
-        --config {ct.ini}
-        --ssh-config {ix.ssh_config_path}
-        --ssh-host {ct.domain}
-        {extra_str}
-    """
-    if "cwd" not in kwargs:
-        kwargs["cwd"] = str(ix.project_root)
-    return out.shell(cmd, **kwargs)

cmdeploy/src/cmdeploy/lxc/incus.py

@@ -1,768 +0,0 @@
-"""Core Incus operations for local chatmail LXC containers."""
-
-import json
-import subprocess
-import textwrap
-import time
-from pathlib import Path
-
-from ..util import shell
-
-LABEL_KEY = "user.localchat-managed"
-SSH_KEY_NAME = "id_localchat"
-DOMAIN_SUFFIX = ".localchat"
-UPSTREAM_IMAGE = "images:debian/12"
-BASE_IMAGE_ALIAS = "localchat-base"
-BASE_SETUP_NAME = "localchat-base-setup"
-RELAY_IMAGE_ALIAS = "localchat-relay"
-
-DNS_CONTAINER_NAME = "ns-localchat"
-DNS_DOMAIN = "ns.localchat"
-
-
-class DNSConfigurationError(Exception):
-    """Raised when the DNS container is not reachable or not answering."""
-
-
-def _extract_ip(net_data, family="inet"):
-    """Extract the first global-scope IP of *family* from network state data.
-
-    *net_data* is the ``state.network`` dict from ``incus list --format=json``.
-    *family* is ``"inet"`` for IPv4 or ``"inet6"`` for IPv6.
-    Returns the address string, or None.
-    """
-    for iface_name, iface in net_data.items():
-        if iface_name == "lo":
-            continue
-        for addr in iface.get("addresses", []):
-            if addr["family"] == family and addr["scope"] == "global":
-                return addr["address"]
-    return None
-
-
-class Incus:
-    """Gateway for all Incus container operations.
-
-    Instantiated once per CLI command and passed around so that
-    all modules share a single entry point for Incus interactions.
-    """
-
-    def __init__(self, out):
-        self.out = out
-        self.project_root = Path(__file__).resolve().parent.parent.parent.parent.parent
-        self.lxconfigs_dir = self.project_root / "lxconfigs"
-        self.lxconfigs_dir.mkdir(exist_ok=True)
-        self.ssh_key_path = self.lxconfigs_dir / SSH_KEY_NAME
-        if not self.ssh_key_path.exists():
-            shell(
-                f"ssh-keygen -t ed25519 -f {self.ssh_key_path} -N '' -C localchat",
-                check=True,
-            )
-        self.ssh_config_path = self.lxconfigs_dir / "ssh-config"
-
-    def write_ssh_config(self):
-        """Write ``lxconfigs/ssh-config`` mapping all containers to their IPs.
-
-        Each Host block maps the container name, the domain name, and the
-        short relay name (e.g. ``_test0``) to the container's IP, using the
-        shared localchat SSH key.  Returns the path to the file.
-        """
-        containers = self.list_managed()
-        key_path = self.ssh_key_path
-        lines = ["# Auto-generated by cmdeploy lxc-start — do not edit\n"]
-        for c in containers:
-            hosts = [c["name"]]
-            domain = c.get("domain", "")
-            if domain and domain != c["name"]:
-                hosts.append(domain)
-                short = domain.split(".")[0]
-                if short and short not in hosts:
-                    hosts.append(short)
-            lines.append(f"\nHost {' '.join(hosts)}\n")
-            lines.append(f"    Hostname {c['ip']}\n")
-            lines.append("    User root\n")
-            lines.append(f"    IdentityFile {key_path}\n")
-            lines.append("    IdentitiesOnly yes\n")
-            lines.append("    StrictHostKeyChecking accept-new\n")
-            lines.append("    UserKnownHostsFile /dev/null\n")
-            lines.append("    LogLevel ERROR\n")
-        path = self.ssh_config_path
-        path.write_text("".join(lines))
-        return path
-
-    def check_ssh_include(self):
-        """Check if the user's ~/.ssh/config already includes our ssh-config."""
-        user_ssh_config = Path.home() / ".ssh" / "config"
-        if not user_ssh_config.exists():
-            return False
-        lines = user_ssh_config.read_text().splitlines()
-        target = f"include {self.ssh_config_path}".lower()
-        return any(line.strip().lower() == target for line in lines)
-
-    def get_host_nameservers(self):
-        """Return upstream nameservers found on the host."""
-        ns = []
-        for path in ["/run/systemd/resolve/resolv.conf", "/etc/resolv.conf"]:
-            p = Path(path)
-            if p.exists():
-                for line in p.read_text().splitlines():
-                    if line.strip().startswith("nameserver "):
-                        addr = line.split()[1]
-                        if addr not in ("127.0.0.1", "127.0.0.53", "::1"):
-                            if addr not in ns:
-                                ns.append(addr)
-                if ns:
-                    break
-        return ns
-
-    def run(self, args, check=True, capture=True, input=None):
-        """Run an incus command.
-
-        When *capture* is True and *verbosity* >= 1, output is streamed
-        to the terminal line-by-line while also being captured for
-        later return via result.stdout.
-        """
-        cmd = ["incus", "--quiet"] + list(args)
-        sub = self.out.new_prefixed_out("  ")
-
-        if not capture:
-            # Simple case: let subprocess handle streams (no capture)
-            if self.out.verbosity >= 1:
-                sub.print(f"$ {' '.join(cmd)}")
-            return subprocess.run(
-                cmd, text=True, input=input, check=check, stdout=None, stderr=None
-            )
-
-        # Capture case: we may need to stream while capturing
-        if sub.verbosity >= 1:
-            cmd_lines = " ".join(cmd).splitlines()
-            sub.print(f"$ {cmd_lines.pop(0)}")
-            if sub.verbosity >= 2:
-                for line in cmd_lines:
-                    sub.print(f"  {line}")
-
-        proc = subprocess.Popen(
-            cmd,
-            text=True,
-            stdin=subprocess.PIPE if input else subprocess.DEVNULL,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-        )
-
-        stdout_lines = []
-        if input:
-            proc.stdin.write(input)
-            proc.stdin.close()
-
-        for line in proc.stdout:
-            stdout_lines.append(line)
-            if sub.verbosity >= 2:
-                sub.print(f"  > {line.rstrip()}")
-
-        stderr = proc.stderr.read()
-        ret = proc.wait()
-        stdout = "".join(stdout_lines)
-        if check and ret != 0:
-            full_output = stdout + stderr
-            for line in full_output.splitlines():
-                if sub.verbosity < 1:  # and we haven't printed it yet
-                    sub.red(line)
-            raise subprocess.CalledProcessError(ret, cmd, output=stdout, stderr=stderr)
-
-        return subprocess.CompletedProcess(cmd, ret, stdout=stdout, stderr=stderr)
-
-    def run_json(self, args, check=True):
-        """Run an incus command with ``--format=json``.
-
-        Returns the parsed JSON on success.
-        When *check* is True raises ``subprocess.CalledProcessError``
-        on non-zero exit; when False returns *None* instead.
-        """
-        result = self.run(
-            list(args) + ["--format=json"],
-            check=check,
-        )
-        if result.returncode != 0:
-            return None
-        return json.loads(result.stdout)
-
-    def run_output(self, args, check=True):
-        """Run an incus command and return its stripped stdout.
-
-        When *check* is False, returns *None* on non-zero exit
-        instead of raising.
-        """
-        result = self.run(args, check=check)
-        if result.returncode != 0:
-            return None
-        return result.stdout.strip()
-
-    def find_image(self, aliases):
-        """Return the first alias from *aliases* that exists, else None."""
-        images = self.run_json(["image", "list"], check=False) or []
-        existing = {a.get("name") for img in images for a in img.get("aliases", [])}
-        for alias in aliases:
-            if alias in existing:
-                return alias
-        return None
-
-    def delete_images(self):
-        """Delete the cached base and relay images."""
-        for alias in (RELAY_IMAGE_ALIAS, BASE_IMAGE_ALIAS):
-            self.run(["image", "delete", alias], check=False)  # ok if absent
-
-    def list_managed(self):
-        """Return list of dicts with name, ip, ipv6, domain, status, memory_usage."""
-        containers = []
-        for ct in self.run_json(["list"]):
-            config = ct.get("config", {})
-            if config.get(LABEL_KEY) != "true":
-                continue
-            name = ct["name"]
-            state = ct.get("state", {})
-            net = state.get("network") or {}
-            containers.append(
-                {
-                    "name": name,
-                    "ip": _extract_ip(net, "inet"),
-                    "ipv6": _extract_ip(net, "inet6"),
-                    "domain": config.get(
-                        "user.localchat-domain", f"{name}{DOMAIN_SUFFIX}"
-                    ),
-                    "status": ct.get("status", "Unknown"),
-                    "memory_usage": state.get("memory", {}).get("usage", 0),
-                }
-            )
-        return containers
-
-    def ensure_base_image(self):
-        """Build and cache a base image with openssh and the SSH key.
-
-        The image is published as a local incus image with alias
-        'localchat-base'.  Subsequent container launches use this
-        image instead of the upstream Debian 12, skipping the
-        slow apt-get install step.
-        Returns the image alias.
-        """
-        if self.find_image([BASE_IMAGE_ALIAS]):
-            self.out.print(f"  Base image '{BASE_IMAGE_ALIAS}' already cached.")
-            return BASE_IMAGE_ALIAS
-
-        self.out.print("  Building base image (one-time setup) ...")
-
-        self.run(["delete", BASE_SETUP_NAME, "--force"], check=False)
-        self.run(["image", "delete", BASE_IMAGE_ALIAS], check=False)
-        self.run(["launch", UPSTREAM_IMAGE, BASE_SETUP_NAME])
-
-        ct = Container(self, BASE_SETUP_NAME)
-        ct.wait_ready()
-
-        key_path = self.ssh_key_path
-        pub_key = key_path.with_suffix(".pub").read_text().strip()
-        host_ns = self.get_host_nameservers()
-        ns_lines = "\n".join(f"nameserver {n}" for n in host_ns)
-        ct.bash(f"""
-            printf '{ns_lines}\n' > /etc/resolv.conf
-            apt-get -o DPkg::Lock::Timeout=60 update
-            DEBIAN_FRONTEND=noninteractive apt-get install -y openssh-server python3
-            systemctl enable ssh
-            apt-get clean
-            mkdir -p /root/.ssh
-            chmod 700 /root/.ssh
-            echo '{pub_key}' > /root/.ssh/authorized_keys
-            chmod 600 /root/.ssh/authorized_keys
-        """)
-
-        self.run(["stop", BASE_SETUP_NAME])
-        self.run(["publish", BASE_SETUP_NAME, f"--alias={BASE_IMAGE_ALIAS}"])
-        self.run(["delete", BASE_SETUP_NAME, "--force"])
-        self.out.print(f"  Base image '{BASE_IMAGE_ALIAS}' ready.")
-        return BASE_IMAGE_ALIAS
-
-    def get_container(self, name):
-        """Return a container handle for the given name.
-
-        Accepts both short relay names (``test0``) and full Incus
-        container names (``test0-localchat``).  Returns
-        ``DNSContainer`` for the DNS container and
-        ``RelayContainer`` for everything else.
-        """
-        if name == DNS_CONTAINER_NAME:
-            return DNSContainer(self)
-        return RelayContainer(self, name.removesuffix("-localchat"))
-
-    def get_dns_container(self):
-        """Return a DNSContainer handle."""
-        return DNSContainer(self)
-
-
-class Container:
-    """The base container handle wraps all interactions with incus."""
-
-    def __init__(self, incus, name, domain=None):
-        self.incus = incus
-        self.out = incus.out
-        self.name = name
-        self.domain = domain or f"{name}{DOMAIN_SUFFIX}"
-        self.ipv4 = None
-        self.ipv6 = None
-
-    def bash(self, script, check=True):
-        """Returns stdout from executing ``bash -ec <script>`` inside this container.
-
-        *script* is dedented and stripped so callers can use triple-quoted strings.
-        When *check* is False, returns *None* on non-zero exit instead of raising.
-        """
-        script = textwrap.dedent(script).strip()
-        cmd = ["exec", self.name, "--", "bash", "-ec", script]
-        return self.incus.run_output(cmd, check=check)
-
-    def run_cmd(self, *args, check=True):
-        """Return stdout from running a command directly in the container (no shell).
-
-        When *check* is False, returns *None* on non-zero exit instead of raising.
-        """
-        return self.incus.run_output(
-            ["exec", self.name, "--", *args],
-            check=check,
-        )
-
-    def start(self):
-        self.incus.run(["start", self.name])
-
-    def stop(self, force=False):
-        cmd = ["stop", self.name]
-        if force:
-            cmd.append("--force")
-        self.incus.run(cmd, check=False)
-
-    def launch(self):
-        """Launch from the best available image, return the alias used."""
-        image = self.incus.find_image([RELAY_IMAGE_ALIAS, BASE_IMAGE_ALIAS])
-        if not image:
-            raise RuntimeError(
-                f"No base image '{BASE_IMAGE_ALIAS}' found. "
-                "Call ensure_base_image() before launching containers."
-            )
-        self.out.print(f"  Launching from '{image}' image ...")
-        cfg = []
-        cfg += ("-c", f"{LABEL_KEY}=true")
-        cfg += ("-c", f"user.localchat-domain={self.domain}")
-        self.incus.run(["launch", image, self.name, *cfg])
-        return image
-
-    def ensure(self):
-        """Create/start this container from the cached base image.
-
-        On first call, builds the base image (~30s).
-        Subsequent containers launch in ~2s from the cached image.
-        Returns ``self`` for chaining.
-        """
-        data = self.incus.run_json(["list", self.name], check=False) or []
-
-        existing = [c for c in data if c["name"] == self.name]
-        if existing:
-            if existing[0]["status"] != "Running":
-                self.start()
-        else:
-            self.launch()
-        self.wait_ready()
-        return self
-
-    def destroy(self):
-        """Stop, delete, and clean up config files."""
-        self.stop(force=True)
-        self.incus.run(["delete", self.name, "--force"], check=False)
-
-    def push_file_content(self, dest_path, content):
-        """Write *content* to *dest_path* inside the container.
-
-        *content* is dedented and stripped so callers can use
-        indented triple-quoted strings.
-        """
-        content = textwrap.dedent(content).strip() + "\n"
-        self.incus.run(
-            ["file", "push", "-", f"{self.name}{dest_path}"],
-            input=content,
-        )
-        self.bash(f"chmod 644 {dest_path}")
-
-    def wait_ready(self, timeout=60):
-        """Wait until the container is running with an IPv4 address.
-
-        Sets ``self.ipv4`` and ``self.ipv6`` (may be *None*),
-        or raises ``TimeoutError``.
-        """
-        deadline = time.time() + timeout
-        while time.time() < deadline:
-            data = self.incus.run_json(
-                ["list", self.name],
-                check=False,
-            )
-            if data and data[0].get("status") == "Running":
-                net = data[0].get("state", {}).get("network", {})
-                self.ipv4 = _extract_ip(net, "inet")
-                self.ipv6 = _extract_ip(net, "inet6")
-                if self.ipv4:
-                    return
-            time.sleep(1)
-        raise TimeoutError(
-            f"Container {self.name!r} did not become ready within {timeout}s"
-        )
-
-    def rss_mib(self):
-        """Return ``(used, total)`` memory from container (or None if unobtainable)."""
-        output = self.bash("free -m", check=False)
-        if output:
-            for line in output.splitlines():
-                if line.startswith("Mem:"):
-                    parts = line.split()
-                    return int(parts[2]), int(parts[1])
-
-
-class RelayContainer(Container):
-    """Container handle for a chatmail relay.
-
-    Accepts the short relay name (e.g. ``test0``) and derives
-    the Incus container name and mail domain automatically.
-    """
-
-    def __init__(self, incus, name):
-        super().__init__(
-            incus,
-            f"{name}-localchat",
-            domain=f"_{name}{DOMAIN_SUFFIX}",
-        )
-        self.sname = name
-        self.ini = incus.lxconfigs_dir / f"chatmail-{name}.ini"
-        self.zone = incus.lxconfigs_dir / f"{name}.zone"
-
-    def launch(self):
-        """Launch (from a potentially cached image) and clear inherited chatmail-version."""
-        image = super().launch()
-        self.bash("rm -f /etc/chatmail-version")
-        return image
-
-    def destroy(self):
-        """Stop, delete, and clean up config files."""
-        super().destroy()
-        if self.ini.exists():
-            self.ini.unlink()
-
-    def disable_ipv6(self):
-        """Disable IPv6 inside the container via sysctl."""
-        # incus provides net.* virtualization for LXC containers so that
-        # these sysctls only affect the container's network namespace.
-        self.bash("""
-            sysctl -w net.ipv6.conf.all.disable_ipv6=1
-            sysctl -w net.ipv6.conf.default.disable_ipv6=1
-        """)
-        self.push_file_content(
-            "/etc/sysctl.d/99-disable-ipv6.conf",
-            """
-            net.ipv6.conf.all.disable_ipv6=1
-            net.ipv6.conf.default.disable_ipv6=1
-            """,
-        )
-
-    def configure_hosts(self, ip):
-        """Set hostname and /etc/hosts inside the container."""
-        self.bash(f"""
-            echo '{self.name}' > /etc/hostname
-            hostname {self.name}
-            sed -i '/ {self.domain}$/d' /etc/hosts
-            echo '{ip} {self.name} {self.domain}' >> /etc/hosts
-        """)
-
-    def publish_as_relay_image(self):
-        """Publish this container as a reusable relay image.
-
-        Stops the container, 'publishes' it as 'localchat-relay', then restarts it.
-        """
-        if self.incus.find_image([RELAY_IMAGE_ALIAS]):
-            return
-        self.out.print(
-            f"  Locally caching {self.name!r} as '{RELAY_IMAGE_ALIAS}' image ..."
-        )
-        self.incus.run(
-            ["publish", self.name, f"--alias={RELAY_IMAGE_ALIAS}", "--force"]
-        )
-        self.wait_ready()
-        self.out.print(f"  Relay image '{RELAY_IMAGE_ALIAS}' ready.")
-
-    def deployed_version(self):
-        """Read /etc/chatmail-version, or None if absent."""
-        return self.bash("cat /etc/chatmail-version", check=False)
-
-    def deployed_domain(self):
-        """Read the domain deployed on the container (postfix myhostname)."""
-        return self.bash(
-            "postconf -h myhostname 2>/dev/null",
-            check=False,
-        )
-
-    def verify_ssh(self, ssh_config):
-        """Verify SSH connectivity to this container."""
-        cmd = f"ssh -F {ssh_config} -o ConnectTimeout=60 root@{self.domain} hostname"
-        return shell(cmd, timeout=60).returncode == 0
-
-    def configure_dns(self, dns_ip):
-        """Point this container's resolver at *dns_ip* and verify DNS is reachable."""
-        self.bash(f"""
-            systemctl disable --now systemd-resolved 2>/dev/null || true
-            rm -f /etc/resolv.conf
-            printf 'nameserver {dns_ip}\\n' >/etc/resolv.conf
-            mkdir -p /etc/unbound/unbound.conf.d
-        """)
-        self.push_file_content(
-            "/etc/unbound/unbound.conf.d/localchat-forward.conf",
-            f"""
-            server:
-              domain-insecure: "localchat"
-
-            forward-zone:
-              name: "localchat"
-              forward-addr: {dns_ip}
-            """,
-        )
-        self.bash("systemctl restart unbound 2>/dev/null || true")
-        self._wait_dns_reachable(dns_ip)
-
-    def _wait_dns_reachable(self, dns_ip, timeout=10):
-        """Poll until *dns_ip* answers a DNS query from this container."""
-        if self.bash("which dig", check=False) is None:
-            self.bash(
-                "DEBIAN_FRONTEND=noninteractive "
-                "apt-get install -y dnsutils 2>/dev/null || true"
-            )
-        deadline = time.time() + timeout
-        while time.time() < deadline:
-            result = self.bash(
-                f"dig @{dns_ip} . SOA +short +time=1 +tries=1",
-                check=False,
-            )
-            if result and result.strip():
-                return
-            time.sleep(0.5)
-        raise DNSConfigurationError(
-            f"DNS at {dns_ip} not reachable from {self.name} after {timeout}s"
-        )
-
-    def write_ini(self, disable_ipv6=False):
-        """Generate a chatmail.ini config file in lxconfigs/."""
-        from chatmaild.config import write_initial_config
-
-        overrides = {
-            "max_user_send_per_minute": 600,
-            "max_user_send_burst_size": 100,
-            "mtail_address": "127.0.0.1",
-        }
-        if disable_ipv6:
-            overrides["disable_ipv6"] = "True"
-        write_initial_config(self.ini, self.domain, overrides)
-        return self.ini
-
-
-class DNSContainer(Container):
-    """Container handle for the PowerDNS name server.
-
-    Manages the authoritative and recursive DNS services required for
-    name resolution in the local testing environment.
-    """
-
-    def __init__(self, incus):
-        super().__init__(incus, DNS_CONTAINER_NAME, domain=DNS_DOMAIN)
-
-    def pdnsutil(self, *args, check=True):
-        """Run ``pdnsutil <args>`` inside the DNS container."""
-        return self.run_cmd("pdnsutil", *args, check=check)
-
-    def replace_rrset(self, zone, name, rtype, ttl, rdata):
-        """Shortcut for ``pdnsutil replace-rrset``."""
-        self.pdnsutil("replace-rrset", zone, name, rtype, ttl, rdata)
-
-    def restart_services(self):
-        """Restart pdns and pdns-recursor, then wait until DNS is answering."""
-        self.bash("""
-            systemctl restart pdns
-            systemctl restart pdns-recursor || true
-        """)
-        self._wait_dns_ready()
-
-    def _wait_dns_ready(self, timeout=60):
-        """Poll until the recursor answers a query on port 53."""
-        deadline = time.time() + timeout
-        while time.time() < deadline:
-            result = self.bash(
-                "dig @127.0.0.1 . SOA +short +time=1 +tries=1",
-                check=False,
-            )
-            if result and result.strip():
-                return
-            time.sleep(0.5)
-        raise DNSConfigurationError(f"DNS recursor not answering after {timeout}s")
-
-    def ensure(self):
-        """Create the DNS container with PowerDNS if needed.
-
-        Calls ``super().ensure()`` to create/start the container
-        and set up SSH, then installs PowerDNS and configures
-        the Incus bridge to use this container as DNS.
-        """
-        super().ensure()
-        self._install_powerdns()
-        self.incus.run(
-            ["network", "set", "incusbr0", "dns.mode=none"],
-            check=False,
-        )
-        self.incus.run(
-            ["network", "set", "incusbr0", f"raw.dnsmasq=dhcp-option=6,{self.ipv4}"],
-            check=False,
-        )
-
-    def destroy(self):
-        """Stop, delete, and reset bridge DNS config."""
-        super().destroy()
-        self.incus.run(["network", "unset", "incusbr0", "dns.mode"], check=False)
-        self.incus.run(["network", "unset", "incusbr0", "raw.dnsmasq"], check=False)
-
-    def _install_powerdns(self):
-        """Install and configure PowerDNS if not already present."""
-        if self.run_cmd("which", "pdns_server", check=False) is not None:
-            return
-
-        host_ns = self.incus.get_host_nameservers()
-        ns_lines = "\n".join(f"nameserver {n}" for n in host_ns)
-
-        self.bash(f"""
-            systemctl disable --now systemd-resolved 2>/dev/null || true
-            rm -f /etc/resolv.conf
-            printf '{ns_lines}\n' > /etc/resolv.conf
-
-            # Block automatic service startup during package installation
-            printf '#!/bin/sh\\nexit 101\\n' > /usr/sbin/policy-rc.d
-            chmod +x /usr/sbin/policy-rc.d
-
-            apt-get -o DPkg::Lock::Timeout=60 update
-            DEBIAN_FRONTEND=noninteractive apt-get install -y \
-                pdns-server pdns-backend-sqlite3 sqlite3 pdns-recursor dnsutils
-
-            # Remove the startup block
-            rm /usr/sbin/policy-rc.d
-
-            systemctl stop pdns pdns-recursor || true
-            mkdir -p /var/lib/powerdns
-            sqlite3 /var/lib/powerdns/pdns.sqlite3 \
-                </usr/share/doc/pdns-backend-sqlite3/schema.sqlite3.sql
-            chown -R pdns:pdns /var/lib/powerdns
-        """)
-
-        self.push_file_content(
-            "/etc/powerdns/pdns.conf",
-            """
-            launch=gsqlite3
-            gsqlite3-database=/var/lib/powerdns/pdns.sqlite3
-            local-address=127.0.0.1
-            local-port=5353
-            """,
-        )
-
-        self.push_file_content(
-            "/etc/powerdns/recursor.conf",
-            """
-            local-address=0.0.0.0
-            local-port=53
-            forward-zones=localchat=127.0.0.1:5353
-            allow-from=0.0.0.0/0
-            dont-query=
-            dnssec=off
-            """,
-        )
-
-        self.bash("""
-            systemctl start pdns
-            systemctl start pdns-recursor
-            echo 'nameserver 127.0.0.1' > /etc/resolv.conf
-        """)
-        self._wait_dns_ready()
-
-    def reset_dns_records(self, dns_ip, domains):
-        """Create DNS zones with initial A records via pdnsutil.
-
-        Only sets SOA, NS, and A records as the minimal set
-        needed for SSH connectivity.  Full records (MX, TXT, SRV,
-        CNAME, DKIM) are added later by ``cmdeploy dns``.
-
-        Args:
-            dns_ip: IP of the DNS container
-            domains: list of dicts with 'name', 'domain', 'ip'
-        """
-        for d in domains:
-            domain = d["domain"]
-            ip = d["ip"]
-            self.out.print(f"  {domain} -> {ip}")
-
-            # Delete and recreate zone fresh (removes stale records)
-            self.pdnsutil("delete-zone", domain, check=False)
-            self.pdnsutil("create-zone", domain, f"ns.{domain}")
-
-            serial = str(int(time.time()))
-            soa = f"ns.{domain} hostmaster.{domain} {serial} 3600 900 604800 300"
-            self.replace_rrset(domain, ".", "SOA", "3600", soa)
-            self.replace_rrset(domain, ".", "NS", "3600", f"ns.{domain}.")
-            self.replace_rrset(domain, ".", "A", "3600", ip)
-            self.replace_rrset(domain, "ns", "A", "3600", dns_ip)
-
-            # AAAA (domain -> container IPv6, if available)
-            ipv6 = d.get("ipv6")
-            if ipv6:
-                self.replace_rrset(domain, ".", "AAAA", "3600", ipv6)
-                self.out.print(f"    zone reset: SOA, NS, A, AAAA  ({ip}, {ipv6})")
-            else:
-                # Remove any stale AAAA record
-                self.pdnsutil("delete-rrset", domain, ".", "AAAA", check=False)
-                self.out.print(f"    zone reset: SOA, NS, A  ({ip}, IPv4-only)")
-
-        self.restart_services()
-
-    def set_dns_records(self, text):
-        """Add or overwrite DNS records from standard BIND format.
-
-        Uses ``cmdeploy.dns.parse_zone_records`` to parse.
-        Zones are created automatically from the record names.
-        """
-        from ..dns import parse_zone_records
-
-        zones_seen = set()
-
-        for name, ttl, rtype, rdata in parse_zone_records(text):
-            # Derive zone from name: find top-level .localchat domain
-            name_parts = name.split(".")
-            zone = name  # fallback
-            for i in range(len(name_parts) - 1):
-                if name_parts[i + 1 :] == ["localchat"]:
-                    zone = ".".join(name_parts[i:])
-                    break
-
-            # Create zone if first time seeing it
-            if zone not in zones_seen:
-                self.pdnsutil(
-                    "create-zone",
-                    zone,
-                    f"ns.{zone}",
-                    check=False,
-                )
-                zones_seen.add(zone)
-
-            # Figure out the record name relative to zone
-            if name == zone:
-                relative = "."
-            elif name.endswith(f".{zone}"):
-                relative = name[: -(len(zone) + 1)]
-            else:
-                relative = name
-
-            self.replace_rrset(zone, relative, rtype, ttl, rdata)
-
-        if zones_seen:
-            self.restart_services()

cmdeploy/src/cmdeploy/metrics.cron.j2

@@ -0,0 +1 @@
+*/5 * * * * root {{ config.execpath }} {{ config.mailboxes_dir }} >/var/www/html/metrics

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -54,7 +54,7 @@ http {
 	include /etc/nginx/mime.types;
 	default_type application/octet-stream;
 
-	ssl_protocols TLSv1.2 TLSv1.3;
+	ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
 	ssl_prefer_server_ciphers on;
 	ssl_certificate {{ config.tls_cert_path }};
 	ssl_certificate_key {{ config.tls_key_path }};
@@ -79,6 +79,10 @@ http {
 			try_files $uri $uri/ =404;
 		}
 
+		location /metrics {
+			default_type text/plain;
+		}
+
 		location /new {
 {% if config.tls_cert_mode != "self" %}
 			if ($request_method = GET) {

cmdeploy/src/cmdeploy/opendkim/deployer.py

@@ -103,13 +103,6 @@ class OpendkimDeployer(Deployer):
         )
         need_restart |= service_file.changed
 
-        files.file(
-            name="chown opendkim: /etc/dkimkeys/opendkim.private",
-            path="/etc/dkimkeys/opendkim.private",
-            user="opendkim",
-            group="opendkim",
-        )
-
         self.need_restart = need_restart
 
     def activate(self):

cmdeploy/src/cmdeploy/postfix/deployer.py

@@ -97,9 +97,7 @@ class PostfixDeployer(Deployer):
             server.shell(
                 name="Validate postfix configuration",
                 # Extract stderr and quit with error if non-zero
-                commands=[
-                    """bash -c 'w=$(postconf 2>&1 >/dev/null); [[ -z "$w" ]] || { echo "$w"; false; }'"""
-                ],
+                commands=["""bash -c 'w=$(postconf 2>&1 >/dev/null); [[ -z "$w" ]] || { echo "$w"; false; }'"""],
             )
         self.need_restart = need_restart
 

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -20,7 +20,7 @@ smtpd_tls_key_file={{ config.tls_key_path }}
 smtpd_tls_security_level=may
 
 smtp_tls_CApath=/etc/ssl/certs
-smtp_tls_security_level=verify
+smtp_tls_security_level={{ "verify" if config.tls_cert_mode == "acme" else "encrypt" }}
 # Send SNI extension when connecting to other servers.
 # <https://www.postfix.org/postconf.5.html#smtp_tls_servername>
 smtp_tls_servername = hostname
@@ -88,22 +88,6 @@ inet_protocols = ipv4
 inet_protocols = all
 {% endif %}
 
-# Postfix does not try IPv4 and IPv6 connections
-# concurrently as of version 3.7.11.
-#
-# When relay has both A (IPv4) and AAAA (IPv6) records,
-# but broken IPv6 connectivity,
-# every second message is delayed by the connection timeout
-# <https://www.postfix.org/postconf.5.html#smtp_connect_timeout>
-# which defaults to 30 seconds. Reducing timeouts is not a solution
-# as this will result in a failure to connect to slow servers.
-#
-# As a workaround we always prefer IPv4 when it is available.
-# 
-# The setting is documented at
-# <https://www.postfix.org/postconf.5.html#smtp_address_preference>
-smtp_address_preference=ipv4
-
 virtual_transport = lmtp:unix:private/dovecot-lmtp
 virtual_mailbox_domains = {{ config.mail_domain }}
 lmtp_header_checks = regexp:/etc/postfix/lmtp_header_cleanup

cmdeploy/src/cmdeploy/remote/rdns.py

@@ -53,14 +53,13 @@ def get_dkim_entry(mail_domain, pre_command, dkim_selector):
             print=log_progress,
         )
     except CalledProcessError:
-        return None, None
+        return
     dkim_value_raw = f"v=DKIM1;k=rsa;p={dkim_pubkey};s=email;t=s"
     dkim_value = '" "'.join(re.findall(".{1,255}", dkim_value_raw))
     web_dkim_value = "".join(re.findall(".{1,255}", dkim_value_raw))
-    name = f"{dkim_selector}._domainkey.{mail_domain}."
     return (
-        f'{name:<40} 3600   IN  TXT    "{dkim_value}"',
-        f'{name:<40} 3600   IN  TXT    "{web_dkim_value}"',
+        f'{dkim_selector}._domainkey.{mail_domain}. TXT "{dkim_value}"',
+        f'{dkim_selector}._domainkey.{mail_domain}. TXT "{web_dkim_value}"',
     )
 
 
@@ -95,11 +94,9 @@ def check_zonefile(zonefile, verbose=True):
         if not zf_line.strip() or zf_line.startswith(";"):
             continue
         print(f"dns-checking {zf_line!r}") if verbose else log_progress("")
-        parts = zf_line.split(None, 4)
-        zf_domain = parts[0].rstrip(".")
-        # parts[1]=TTL, parts[2]=IN, parts[3]=type, parts[4]=rdata
-        zf_typ = parts[3]
-        zf_value = parts[4].strip()
+        zf_domain, zf_typ, zf_value = zf_line.split(maxsplit=2)
+        zf_domain = zf_domain.rstrip(".")
+        zf_value = zf_value.strip()
         query_value = query_dns(zf_typ, zf_domain)
         if zf_value != query_value:
             assert zf_typ in ("A", "AAAA", "CNAME", "CAA", "SRV", "MX", "TXT"), zf_line

cmdeploy/src/cmdeploy/remote/rshell.py

@@ -40,5 +40,5 @@ def dovecot_recalc_quota(user):
     #
     for line in output.split("\n"):
         parts = line.split()
-        if len(parts) >= 6 and parts[2] == "STORAGE":
+        if parts[2] == "STORAGE":
             return dict(value=int(parts[3]), limit=int(parts[4]), percent=int(parts[5]))

cmdeploy/src/cmdeploy/selfsigned/deployer.py

@@ -12,27 +12,13 @@ def openssl_selfsigned_args(domain, cert_path, key_path, days=36500):
     ``www.<domain>`` and ``mta-sts.<domain>``.
     """
     return [
-        "openssl",
-        "req",
-        "-x509",
-        "-newkey",
-        "ec",
-        "-pkeyopt",
-        "ec_paramgen_curve:P-256",
-        "-noenc",
-        "-days",
-        str(days),
-        "-keyout",
-        str(key_path),
-        "-out",
-        str(cert_path),
-        "-subj",
-        f"/CN={domain}",
-        # Mark as end-entity cert so it cannot be used as a CA to sign others.
-        "-addext",
-        "basicConstraints=critical,CA:FALSE",
-        "-addext",
-        "extendedKeyUsage=serverAuth,clientAuth",
+        "openssl", "req", "-x509",
+        "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:P-256",
+        "-noenc", "-days", str(days),
+        "-keyout", str(key_path),
+        "-out", str(cert_path),
+        "-subj", f"/CN={domain}",
+        "-addext", "extendedKeyUsage=serverAuth,clientAuth",
         "-addext",
         f"subjectAltName=DNS:{domain},DNS:www.{domain},DNS:mta-sts.{domain}",
     ]
@@ -54,9 +40,7 @@ class SelfSignedTlsDeployer(Deployer):
 
     def configure(self):
         args = openssl_selfsigned_args(
-            self.mail_domain,
-            self.cert_path,
-            self.key_path,
+            self.mail_domain, self.cert_path, self.key_path,
         )
         cmd = shlex.join(args)
         server.shell(

cmdeploy/src/cmdeploy/service/chatmail-fsreport.service.f

@@ -5,5 +5,5 @@ After=network.target
 [Service]
 Type=oneshot
 User=vmail
-ExecStart=/usr/local/lib/chatmaild/venv/bin/chatmail-fsreport /usr/local/lib/chatmaild/chatmail.ini
+ExecStart=/usr/local/lib/chatmaild/venv/bin/chatmail-fsreport /usr/local/lib/chatmaild/chatmail.ini 
 

cmdeploy/src/cmdeploy/sshexec.py

@@ -49,13 +49,8 @@ class SSHExec:
     RemoteError = execnet.RemoteError
     FuncError = FuncError
 
-    def __init__(
-        self, host, verbose=False, python="python3", timeout=60, ssh_config=None
-    ):
-        spec = f"ssh=root@{host}//python={python}"
-        if ssh_config:
-            spec += f"//ssh_config={ssh_config}"
-        self.gateway = execnet.makegateway(spec)
+    def __init__(self, host, verbose=False, python="python3", timeout=60):
+        self.gateway = execnet.makegateway(f"ssh=root@{host}//python={python}")
         self._remote_cmdloop_channel = bootstrap_remote(self.gateway, remote)
         self.timeout = timeout
         self.verbose = verbose
@@ -118,46 +113,3 @@ class LocalExec:
             res = self(call, kwargs, log_callback=remote.rshell.log_progress)
             print_stderr()
             return res
-
-
-# pyinfra exposes a ``ssh_config_file`` data key that *should* let
-# paramiko parse an SSH config file directly.  In practice it silently
-# fails to connect (zero hosts / zero operations), so we resolve the
-# hostname and identity-file ourselves and pass them via
-# ``--data ssh_hostname`` / ``--data ssh_key`` instead.
-# Execnet uses ssh natively (and not paramiko) and doesn't have this problem.
-
-
-def _get_from_ssh_config(host, ssh_config_path, key):
-    """Internal helper to parse a value for a specific key from ssh-config."""
-    current_hosts = []
-    found_value = None
-    with open(ssh_config_path) as f:
-        for raw_line in f:
-            line = raw_line.strip()
-            if not line or line.startswith("#"):
-                continue
-            parts = line.split(None, 1)
-            if not parts:
-                continue
-            directive = parts[0].lower()
-            if directive == "host":
-                if host in current_hosts and found_value:
-                    return found_value
-                current_hosts = parts[1].split()
-                found_value = None
-            elif directive == key.lower():
-                found_value = parts[1]
-    if host in current_hosts and found_value:
-        return found_value
-    return None
-
-
-def resolve_host_from_ssh_config(host, ssh_config_path):
-    """Resolve a host alias to its IP from an ssh-config file."""
-    return _get_from_ssh_config(host, ssh_config_path, "Hostname") or host
-
-
-def resolve_key_from_ssh_config(host, ssh_config_path):
-    """Resolve a host alias to its IdentityFile from an ssh-config file."""
-    return _get_from_ssh_config(host, ssh_config_path, "IdentityFile")

cmdeploy/src/cmdeploy/tests/data/zftest.zone

@@ -1,18 +1,17 @@
-; Required DNS entries
-zftest.testrun.org.                      3600   IN  A      135.181.204.127
-zftest.testrun.org.                      3600   IN  AAAA   2a01:4f9:c012:52f4::1
-zftest.testrun.org.                      3600   IN  MX     10 zftest.testrun.org.
-_mta-sts.zftest.testrun.org.             3600   IN  TXT    "v=STSv1; id=202403211706"
-mta-sts.zftest.testrun.org.              3600   IN  CNAME  zftest.testrun.org.
-www.zftest.testrun.org.                  3600   IN  CNAME  zftest.testrun.org.
-opendkim._domainkey.zftest.testrun.org.  3600   IN  TXT    "v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoYt82CVUyz2ouaqjX2kB+5J80knAyoOU3MGU5aWppmwUwwTvj/oSTSpkc5JMtVTRmKKr8NUDWAL1Yw7dfGqqPHdHfwwjS3BIvDzYx+hzgtz62RnfNgV+/2MAoNpfX7cAFIHdRzEHNtwugc3RDLquqPoupAE3Y2YRw2T5zG5fILh4vwIcJZL5Uq6B92j8wwJqOex" "33n+vm1NKQ9rxo/UsHAmZlJzpooXcG/4igTBxJyJlamVSRR6N7Nul1v//YJb7J6v2o0iPHW6uE0StzKaPPNC2IVosSRFbD9H2oqppltptFSNPlI0E+t0JBWHem6YK7xcugiO3ImMCaaU8g6Jt/wIDAQAB;s=email;t=s"
-
+; Required DNS entries for chatmail servers
+zftest.testrun.org.                   A     135.181.204.127
+zftest.testrun.org.                   AAAA  2a01:4f9:c012:52f4::1
+zftest.testrun.org.                   MX 10 zftest.testrun.org.
+_mta-sts.zftest.testrun.org.          TXT "v=STSv1; id=202403211706"
+mta-sts.zftest.testrun.org.           CNAME zftest.testrun.org.
+www.zftest.testrun.org.               CNAME zftest.testrun.org.
+opendkim._domainkey.zftest.testrun.org. TXT "v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoYt82CVUyz2ouaqjX2kB+5J80knAyoOU3MGU5aWppmwUwwTvj/oSTSpkc5JMtVTRmKKr8NUDWAL1Yw7dfGqqPHdHfwwjS3BIvDzYx+hzgtz62RnfNgV+/2MAoNpfX7cAFIHdRzEHNtwugc3RDLquqPoupAE3Y2YRw2T5zG5fILh4vwIcJZL5Uq6B92j8wwJqOex" "33n+vm1NKQ9rxo/UsHAmZlJzpooXcG/4igTBxJyJlamVSRR6N7Nul1v//YJb7J6v2o0iPHW6uE0StzKaPPNC2IVosSRFbD9H2oqppltptFSNPlI0E+t0JBWHem6YK7xcugiO3ImMCaaU8g6Jt/wIDAQAB;s=email;t=s"
 ; Recommended DNS entries
-zftest.testrun.org.                      3600   IN  TXT    "v=spf1 a ~all"
-_dmarc.zftest.testrun.org.               3600   IN  TXT    "v=DMARC1;p=reject;adkim=s;aspf=s"
-zftest.testrun.org.                      3600   IN  CAA    0 issue "letsencrypt.org;accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/1371472956"
-_adsp._domainkey.zftest.testrun.org.     3600   IN  TXT    "dkim=discardable"
-_submission._tcp.zftest.testrun.org.     3600   IN  SRV    0 1 587 zftest.testrun.org.
-_submissions._tcp.zftest.testrun.org.    3600   IN  SRV    0 1 465 zftest.testrun.org.
-_imap._tcp.zftest.testrun.org.           3600   IN  SRV    0 1 143 zftest.testrun.org.
-_imaps._tcp.zftest.testrun.org.          3600   IN  SRV    0 1 993 zftest.testrun.org.
+_submission._tcp.zftest.testrun.org.  SRV 0 1 587 zftest.testrun.org.
+_submissions._tcp.zftest.testrun.org. SRV 0 1 465 zftest.testrun.org.
+_imap._tcp.zftest.testrun.org.        SRV 0 1 143 zftest.testrun.org.
+_imaps._tcp.zftest.testrun.org.       SRV 0 1 993 zftest.testrun.org.
+zftest.testrun.org.                   CAA 0 issue "letsencrypt.org;accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/1371472956"
+zftest.testrun.org.                   TXT "v=spf1 a:zftest.testrun.org ~all"
+_dmarc.zftest.testrun.org.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
+_adsp._domainkey.zftest.testrun.org.  TXT "dkim=discardable"

cmdeploy/src/cmdeploy/tests/online/benchmark.py

@@ -1,3 +1,4 @@
+import time
 def test_tls_imap(benchmark, imap):
     def imap_connect():
         imap.connect()

cmdeploy/src/cmdeploy/tests/online/test_0_login.py

@@ -89,9 +89,7 @@ def test_concurrent_logins_same_account(
         assert login_results.get()
 
 
-def test_no_vrfy(cmfactory, chatmail_config):
-    ac = cmfactory.get_online_account()
-    addr = ac.get_config("addr")
+def test_no_vrfy(chatmail_config):
     domain = chatmail_config.mail_domain
 
     s = smtplib.SMTP(domain)
@@ -100,7 +98,7 @@ def test_no_vrfy(cmfactory, chatmail_config):
     s.putcmd("vrfy", f"wrongaddress@{chatmail_config.mail_domain}")
     result = s.getreply()
     print(result)
-    s.putcmd("vrfy", addr)
+    s.putcmd("vrfy", f"echo@{chatmail_config.mail_domain}")
     result2 = s.getreply()
     print(result2)
     assert result[0] == result2[0] == 252

cmdeploy/src/cmdeploy/tests/online/test_0_qr.py

@@ -20,7 +20,7 @@ def test_fastcgi_working(maildomain, chatmail_config):
 
 
 @pytest.mark.filterwarnings("ignore::urllib3.exceptions.InsecureRequestWarning")
-def test_newemail_configure(maildomain, maildomain_ip, rpc, chatmail_config):
+def test_newemail_configure(maildomain, rpc, chatmail_config):
     """Test configuring accounts by scanning a QR code works."""
     url = f"DCACCOUNT:https://{maildomain}/new"
     for i in range(3):
@@ -30,15 +30,12 @@ def test_newemail_configure(maildomain, maildomain_ip, rpc, chatmail_config):
             # set_config_from_qr, so fetch credentials via requests instead
             res = requests.post(f"https://{maildomain}/new", verify=False)
             data = res.json()
-            rpc.add_or_update_transport(
-                account_id,
-                {
-                    "addr": data["email"],
-                    "password": data["password"],
-                    "imapServer": maildomain_ip,
-                    "smtpServer": maildomain_ip,
-                    "certificateChecks": "acceptInvalidCertificates",
-                },
-            )
+            rpc.add_or_update_transport(account_id, {
+                "addr": data["email"],
+                "password": data["password"],
+                "imapServer": maildomain,
+                "smtpServer": maildomain,
+                "certificateChecks": "acceptInvalidCertificates",
+            })
         else:
             rpc.add_transport_from_qr(account_id, url)

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -12,9 +12,8 @@ from cmdeploy.cmdeploy import get_sshexec
 
 class TestSSHExecutor:
     @pytest.fixture(scope="class")
-    def sshexec(self, sshdomain, pytestconfig):
-        ssh_config = pytestconfig.getoption("ssh_config")
-        return get_sshexec(sshdomain, ssh_config=ssh_config)
+    def sshexec(self, sshdomain):
+        return get_sshexec(sshdomain)
 
     def test_ls(self, sshexec):
         out = sshexec(call=remote.rdns.shell, kwargs=dict(command="ls"))
@@ -133,10 +132,11 @@ def test_authenticated_from(cmsetup, maildata):
 @pytest.mark.parametrize("from_addr", ["fake@example.org", "fake@testrun.org"])
 def test_reject_missing_dkim(cmsetup, maildata, from_addr):
     domain = cmsetup.maildomain
+    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    sock.settimeout(10)
     try:
-        sock = socket.create_connection((domain, 25), timeout=10)
-        sock.close()
-    except (socket.timeout, OSError):
+        sock.connect((domain, 25))
+    except socket.timeout:
         pytest.skip(f"port 25 not reachable for {domain}")
 
     recipient = cmsetup.gen_users(1)[0]

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -6,8 +6,8 @@ import imap_tools
 import pytest
 import requests
 
-from cmdeploy.cmdeploy import get_sshexec
 from cmdeploy.remote import rshell
+from cmdeploy.cmdeploy import get_sshexec
 
 
 @pytest.fixture
@@ -67,7 +67,7 @@ class TestEndToEndDeltaChat:
         assert msg2.get_snapshot().text == "message0"
 
     def test_exceed_quota(
-        self, cmfactory, lp, tmpdir, remote, chatmail_config, sshdomain, pytestconfig
+        self, cmfactory, lp, tmpdir, remote, chatmail_config, sshdomain
     ):
         """This is a very slow test as it needs to upload >100MB of mail data
         before quota is exceeded, and thus depends on the speed of the upload.
@@ -92,9 +92,7 @@ class TestEndToEndDeltaChat:
         lp.sec(f"filling remote inbox for {user}")
         fn = f"7743102289.M843172P2484002.c20,S={quota},W=2398:2,"
         path = chatmail_config.mailboxes_dir.joinpath(user, "cur", fn)
-        sshexec = get_sshexec(
-            sshdomain, ssh_config=pytestconfig.getoption("ssh_config")
-        )
+        sshexec = get_sshexec(sshdomain)
         sshexec(call=rshell.write_numbytes, kwargs=dict(path=str(path), num=120))
         res = sshexec(call=rshell.dovecot_recalc_quota, kwargs=dict(user=user))
         assert res["percent"] >= 100

cmdeploy/src/cmdeploy/tests/online/test_3_status.py

@@ -3,15 +3,12 @@ import os
 from cmdeploy.cmdeploy import main
 
 
-def test_status_cmd(chatmail_config, capsys, request, pytestconfig):
+def test_status_cmd(chatmail_config, capsys, request):
     os.chdir(request.config.invocation_params.dir)
     command = ["status"]
-    ssh_host = pytestconfig.getoption("ssh_host")
-    if ssh_host:
-        command.extend(["--ssh-host", ssh_host])
-    ssh_config = pytestconfig.getoption("ssh_config")
-    if ssh_config:
-        command.extend(["--ssh-config", ssh_config])
+    if os.getenv("CHATMAIL_SSH"):
+        command.append("--ssh-host")
+        command.append(os.getenv("CHATMAIL_SSH"))
     assert main(command) == 0
     status_out = capsys.readouterr()
     print(status_out.out)

cmdeploy/src/cmdeploy/tests/plugin.py

@@ -2,9 +2,7 @@ import imaplib
 import itertools
 import os
 import random
-import re
 import smtplib
-import socket
 import ssl
 import subprocess
 import time
@@ -20,76 +18,6 @@ def pytest_addoption(parser):
     parser.addoption(
         "--slow", action="store_true", default=False, help="also run slow tests"
     )
-    parser.addoption(
-        "--ssh-host",
-        dest="ssh_host",
-        default=None,
-        help="SSH host (overrides mail_domain for SSH operations).",
-    )
-    parser.addoption(
-        "--ssh-config",
-        dest="ssh_config",
-        default=None,
-        help="Path to an SSH config file (e.g. lxconfigs/ssh-config).",
-    )
-
-
-def _parse_ssh_config_hosts(path):
-    """Parse an OpenSSH config file and return a dict of hostname -> IP."""
-    mapping = {}
-    current_names = []
-    for ln in Path(path).read_text().splitlines():
-        line = ln.strip()
-        m = re.match(r"^Host\s+(.+)", line)
-        if m:
-            current_names = m.group(1).split()
-            continue
-        m = re.match(r"^Hostname\s+(\S+)", line)
-        if m and current_names:
-            ip = m.group(1)
-            for name in current_names:
-                mapping[name] = ip
-            current_names = []
-    return mapping
-
-
-_original_getaddrinfo = socket.getaddrinfo
-
-
-def _make_patched_getaddrinfo(host_map):
-    """Return a getaddrinfo that resolves hosts in host_map to their IPs."""
-
-    def patched_getaddrinfo(host, port, family=0, type=0, proto=0, flags=0):
-        if host in host_map:
-            ip = host_map[host]
-            return _original_getaddrinfo(ip, port, family, type, proto, flags)
-        return _original_getaddrinfo(host, port, family, type, proto, flags)
-
-    return patched_getaddrinfo
-
-
-@pytest.fixture(autouse=True, scope="session")
-def _setup_localchat_dns(pytestconfig):
-    """Monkey-patch socket.getaddrinfo to resolve .localchat via ssh-config."""
-    ssh_config = pytestconfig.getoption("ssh_config")
-    if not ssh_config or not Path(ssh_config).exists():
-        yield {}
-        return
-    host_map = _parse_ssh_config_hosts(ssh_config)
-    if not host_map:
-        yield {}
-        return
-    socket.getaddrinfo = _make_patched_getaddrinfo(host_map)
-    try:
-        yield host_map
-    finally:
-        socket.getaddrinfo = _original_getaddrinfo
-
-
-@pytest.fixture(scope="session")
-def ssh_config_host_map(_setup_localchat_dns):
-    """Return the host-name → IP map parsed from ssh-config."""
-    return _setup_localchat_dns
 
 
 def pytest_configure(config):
@@ -107,11 +35,6 @@ def pytest_runtest_setup(item):
 
 
 def _get_chatmail_config():
-    ini = os.environ.get("CHATMAIL_INI")
-    if ini:
-        path = Path(ini).resolve()
-        if path.exists():
-            return read_config(path), path
     current = Path().resolve()
     while 1:
         path = current.joinpath("chatmail.ini").resolve()
@@ -138,14 +61,8 @@ def maildomain(chatmail_config):
 
 
 @pytest.fixture(scope="session")
-def sshdomain(maildomain, pytestconfig):
-    return pytestconfig.getoption("ssh_host") or maildomain
-
-
-@pytest.fixture(scope="session")
-def maildomain_ip(maildomain, ssh_config_host_map):
-    """Return the IP for maildomain from ssh-config, or maildomain itself."""
-    return ssh_config_host_map.get(maildomain, maildomain)
+def sshdomain(maildomain):
+    return os.environ.get("CHATMAIL_SSH", maildomain)
 
 
 @pytest.fixture
@@ -389,22 +306,12 @@ from deltachat_rpc_client import DeltaChat, Rpc
 class ChatmailACFactory:
     """RPC-based account factory for chatmail testing."""
 
-    def __init__(
-        self,
-        rpc,
-        maildomain,
-        maildomain_ip,
-        gencreds,
-        chatmail_config,
-        ssh_config_host_map,
-    ):
+    def __init__(self, rpc, maildomain, gencreds, chatmail_config):
         self.dc = DeltaChat(rpc)
         self.rpc = rpc
         self._maildomain = maildomain
-        self._maildomain_ip = maildomain_ip
         self.gencreds = gencreds
         self.chatmail_config = chatmail_config
-        self._ssh_config_host_map = ssh_config_host_map
 
     def _make_transport(self, domain):
         """Build a transport config dict for the given domain."""
@@ -412,13 +319,11 @@ class ChatmailACFactory:
         transport = {
             "addr": addr,
             "password": password,
+            # Setting server explicitly skips requesting autoconfig XML,
+            # see https://datatracker.ietf.org/doc/draft-ietf-mailmaint-autoconfig/
+            "imapServer": domain,
+            "smtpServer": domain,
         }
-        # To support running against local relays without host DNS resolution
-        # we attempt resolving the domain via ssh-config
-        # because otherwise core fails to find the address
-        server = self._ssh_config_host_map.get(domain)
-        if server is not None:
-            transport.update({"imapServer": server, "smtpServer": server})
         if self.chatmail_config.tls_cert_mode == "self":
             transport["certificateChecks"] = "acceptInvalidCertificates"
         return transport
@@ -471,56 +376,39 @@ def rpc(tmp_path_factory):
 
 
 @pytest.fixture
-def cmfactory(
-    rpc, gencreds, maildomain, maildomain_ip, chatmail_config, ssh_config_host_map
-):
+def cmfactory(rpc, gencreds, maildomain, chatmail_config):
     """Return a ChatmailACFactory for creating online Delta Chat accounts."""
     return ChatmailACFactory(
         rpc=rpc,
         maildomain=maildomain,
-        maildomain_ip=maildomain_ip,
         gencreds=gencreds,
         chatmail_config=chatmail_config,
-        ssh_config_host_map=ssh_config_host_map,
     )
 
 
 @pytest.fixture
-def remote(sshdomain, pytestconfig):
-    r = Remote(sshdomain, ssh_config=pytestconfig.getoption("ssh_config"))
-    yield r
-    r.close()
+def remote(sshdomain):
+    return Remote(sshdomain)
 
 
 class Remote:
-    def __init__(self, sshdomain, ssh_config=None):
+    def __init__(self, sshdomain):
         self.sshdomain = sshdomain
-        self.ssh_config = ssh_config
-        self._procs = []
 
     def iter_output(self, logcmd="", ready=None):
         getjournal = "journalctl -f" if not logcmd else logcmd
         print(self.sshdomain)
         match self.sshdomain:
-            case "@local":
-                command = []
-            case "localhost":
-                command = []
-            case _:
-                command = ["ssh"]
-                if self.ssh_config:
-                    command.extend(["-F", self.ssh_config])
-                command.append(f"root@{self.sshdomain}")
+            case "@local": command = []
+            case "localhost": command = []
+            case _: command = ["ssh", f"root@{self.sshdomain}"]
         [command.append(arg) for arg in getjournal.split()]
-        popen = subprocess.Popen(
+        self.popen = subprocess.Popen(
             command,
-            stdin=subprocess.DEVNULL,
             stdout=subprocess.PIPE,
-            stderr=subprocess.DEVNULL,
         )
-        self._procs.append(popen)
         while 1:
-            line = popen.stdout.readline()
+            line = self.popen.stdout.readline()
             res = line.decode().strip().lower()
             if not res:
                 break
@@ -529,12 +417,6 @@ class Remote:
                 ready = None
             yield res
 
-    def close(self):
-        while self._procs:
-            proc = self._procs.pop()
-            proc.kill()
-            proc.wait()
-
 
 @pytest.fixture
 def lp(request):

cmdeploy/src/cmdeploy/tests/test_cmdeploy.py

@@ -23,10 +23,7 @@ class TestCmdline:
         run = parser.parse_args(["run"])
         assert init and run
 
-    def test_init_not_overwrite(self, tmp_path, capsys, monkeypatch):
-        monkeypatch.delenv("CHATMAIL_INI", raising=False)
-        monkeypatch.chdir(tmp_path)
-
+    def test_init_not_overwrite(self, capsys):
         assert main(["init", "chat.example.org"]) == 0
         capsys.readouterr()
 

cmdeploy/src/cmdeploy/tests/test_dns.py

@@ -3,7 +3,7 @@ from copy import deepcopy
 import pytest
 
 from cmdeploy import remote
-from cmdeploy.dns import check_full_zone, check_initial_remote_data, parse_zone_records
+from cmdeploy.dns import check_full_zone, check_initial_remote_data
 
 
 @pytest.fixture
@@ -60,29 +60,6 @@ def mockdns(request, mockdns_base, mockdns_expected):
     return mockdns_base
 
 
-class TestGetDkimEntry:
-    def test_dkim_entry_returns_tuple_on_success(self, mockdns):
-        entry, web_entry = remote.rdns.get_dkim_entry(
-            "some.domain", "", dkim_selector="opendkim"
-        )
-        # May return None,None if openssl not available, but should never crash
-        if entry is not None:
-            assert "opendkim._domainkey.some.domain" in entry
-            assert "opendkim._domainkey.some.domain" in web_entry
-
-    def test_dkim_entry_returns_none_tuple_on_error(self, monkeypatch):
-        """CalledProcessError must return (None, None), not bare None."""
-        from subprocess import CalledProcessError
-
-        def failing_shell(command, fail_ok=False, print=print):
-            raise CalledProcessError(1, command)
-
-        monkeypatch.setattr(remote.rdns, "shell", failing_shell)
-        result = remote.rdns.get_dkim_entry("some.domain", "", dkim_selector="opendkim")
-        assert result == (None, None)
-        assert result[0] is None and result[1] is None
-
-
 class TestPerformInitialChecks:
     def test_perform_initial_checks_ok1(self, mockdns, mockdns_expected):
         remote_data = remote.rdns.perform_initial_checks("some.domain")
@@ -125,50 +102,18 @@ class TestPerformInitialChecks:
         assert not l
 
 
-def test_parse_zone_records():
-    text = """
-    ; This is a comment
-    some.domain. 3600 IN A 1.1.1.1
-
-    ; Another comment
-    www.some.domain. 3600 IN CNAME some.domain.
-
-    ; Multi-word rdata
-    some.domain. 3600 IN MX 10 mail.some.domain.
-
-    ; DKIM record (single line, multi-word TXT rdata)
-    dkim._domainkey.some.domain. 3600 IN TXT "v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG" "9w0BAQEFAAOCAQ8AMIIBCgKCAQEA"
-
-    ; Another TXT record
-    _dmarc.some.domain. 3600 IN TXT "v=DMARC1;p=reject"
-    """
-    records = list(parse_zone_records(text))
-    assert records == [
-        ("some.domain", "3600", "A", "1.1.1.1"),
-        ("www.some.domain", "3600", "CNAME", "some.domain."),
-        ("some.domain", "3600", "MX", "10 mail.some.domain."),
-        (
-            "dkim._domainkey.some.domain",
-            "3600",
-            "TXT",
-            '"v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG" "9w0BAQEFAAOCAQ8AMIIBCgKCAQEA"',
-        ),
-        ("_dmarc.some.domain", "3600", "TXT", '"v=DMARC1;p=reject"'),
-    ]
-
-
-def test_parse_zone_records_invalid_line():
-    text = "invalid line"
-    with pytest.raises(ValueError, match="Bad zone record line"):
-        list(parse_zone_records(text))
-
-
 def parse_zonefile_into_dict(zonefile, mockdns_base, only_required=False):
-    if only_required:
-        # Only take records before the "; Recommended" section
-        zonefile = zonefile.split("; Recommended")[0]
-    for name, ttl, rtype, rdata in parse_zone_records(zonefile):
-        mockdns_base.setdefault(rtype, {})[name] = rdata
+    for zf_line in zonefile.split("\n"):
+        if zf_line.startswith("#"):
+            if "Recommended" in zf_line and only_required:
+                return
+            continue
+        if not zf_line.strip():
+            continue
+        zf_domain, zf_typ, zf_value = zf_line.split(maxsplit=2)
+        zf_domain = zf_domain.rstrip(".")
+        zf_value = zf_value.strip()
+        mockdns_base.setdefault(zf_typ, {})[zf_domain] = zf_value
 
 
 class MockSSHExec:

cmdeploy/src/cmdeploy/tests/test_lxc.py

@@ -1,174 +0,0 @@
-"""Tests for cmdeploy lxc-* subcommands."""
-
-import shutil
-import subprocess
-import sys
-
-import pytest
-
-from cmdeploy.lxc import cli
-from cmdeploy.lxc.incus import Incus
-from cmdeploy.util import Out
-
-pytestmark = pytest.mark.skipif(
-    not shutil.which("incus"),
-    reason="incus not installed",
-)
-
-
-# ---------------------------------------------------------------------------
-# Fixtures
-# ---------------------------------------------------------------------------
-
-
-@pytest.fixture
-def ix():
-    out = Out()
-    return Incus(out)
-
-
-@pytest.fixture(scope="session")
-def lxc_setup():
-    out = Out()
-    ix = Incus(out)
-    ix.get_dns_container().ensure()
-    return ix.list_managed()
-
-
-@pytest.fixture(scope="session")
-def relay_container(lxc_setup):
-    test_names = {f"{n}-localchat" for n in cli.RELAY_NAMES}
-    relays = [c for c in lxc_setup if c["name"] in test_names and c.get("ip")]
-    if not relays:
-        pytest.skip("no test relay containers running")
-    return relays[0]
-
-
-@pytest.fixture
-def cmdeploy():
-    def run(*args):
-        return subprocess.run(
-            [sys.executable, "-m", "cmdeploy.cmdeploy", *args],
-            capture_output=True,
-            text=True,
-            check=False,
-        )
-
-    return run
-
-
-# ---------------------------------------------------------------------------
-# Tests
-# ---------------------------------------------------------------------------
-
-
-@pytest.mark.parametrize(
-    "subcmd, expected, absent",
-    [
-        (None, ["lxc-start", "lxc-stop", "lxc-test", "lxc-status"], ["lxc-destroy"]),
-        ("lxc-start", ["--ipv4-only", "--run"], ["--config"]),
-        ("lxc-stop", ["--destroy", "--destroy-all"], ["--config"]),
-        ("lxc-test", ["--one"], ["--config"]),
-        ("lxc-status", [], ["--config"]),
-        ("run", ["--ssh-config"], ["--lxc"]),
-        ("dns", ["--ssh-config"], []),
-        ("test", ["--ssh-config"], []),
-        ("status", ["--ssh-config"], []),
-    ],
-)
-def test_help_options(cmdeploy, subcmd, expected, absent):
-    args = [subcmd, "--help"] if subcmd else ["--help"]
-    result = cmdeploy(*args)
-    output = result.stdout + result.stderr
-    assert result.returncode == 0
-    for flag in expected:
-        assert flag in output
-    for flag in absent:
-        assert flag not in output
-
-
-class TestSSHConfig:
-    def test_lxconfigs(self, ix, lxc_setup):
-        d = ix.lxconfigs_dir
-        assert d.name == "lxconfigs"
-        assert d.exists()
-        path = ix.ssh_config_path
-        assert path.name == "ssh-config"
-        assert path.parent.name == "lxconfigs"
-
-    def test_write_ssh_config(self, ix, lxc_setup):
-        path = ix.write_ssh_config()
-        assert path.exists()
-        text = path.read_text()
-
-        for c in lxc_setup:
-            if c.get("ip"):
-                assert c["name"] in text
-                assert f"Hostname {c['ip']}" in text
-
-        assert "User root" in text
-        assert "IdentityFile" in text
-        assert "StrictHostKeyChecking accept-new" in text
-
-
-def test_dns(ix, relay_container):
-    def dig(qname, qtype):
-        ct = ix.get_dns_container()
-        return ct.bash(f"dig @127.0.0.1 {qname} {qtype} +short").strip()
-
-    domain = relay_container["domain"]
-    assert dig(domain, "A") == relay_container["ip"]
-    assert domain in dig(domain, "MX")
-    assert "587" in dig(f"_submission._tcp.{domain}", "SRV")
-
-
-class TestLxcStatus:
-    def test_cli_lxc_status_help(self, cmdeploy):
-        result = cmdeploy("lxc-status", "--help")
-        assert result.returncode == 0
-        assert "status" in result.stdout.lower()
-
-    def test_shows_containers(self, lxc_setup, capsys):
-        class QuietOut(Out):
-            def red(self, msg, **kw):
-                pass
-
-            def green(self, msg, **kw):
-                pass
-
-        ret = cli.lxc_status_cmd(None, QuietOut())
-        assert ret == 0
-        captured = capsys.readouterr().out
-        assert "ns-localchat" in captured
-        assert "running" in captured
-
-    def test_deploy_freshness(self, ix, monkeypatch):
-        ct = ix.get_container("x")
-
-        monkeypatch.setattr(
-            "cmdeploy.lxc.incus.RelayContainer.deployed_version",
-            lambda _self: "abc123def456",
-        )
-        monkeypatch.setattr(
-            "cmdeploy.lxc.incus.RelayContainer.deployed_domain",
-            lambda _self: ct.domain,
-        )
-        monkeypatch.setattr(
-            "cmdeploy.lxc.cli.get_version_string",
-            lambda: "abc123def456",
-        )
-        assert "IN-SYNC" in cli._deploy_status(ct, "abc123def456", ix)
-        assert "STALE" in cli._deploy_status(ct, "other_hash_here", ix)
-
-        # Hash matches but local has uncommitted changes
-        monkeypatch.setattr(
-            "cmdeploy.lxc.cli.get_version_string",
-            lambda: "abc123def456\ndiff --git a/foo",
-        )
-        assert "DIRTY" in cli._deploy_status(ct, "abc123def456", ix)
-
-        monkeypatch.setattr(
-            "cmdeploy.lxc.incus.RelayContainer.deployed_version",
-            lambda _self: None,
-        )
-        assert "NOT DEPLOYED" in cli._deploy_status(ct, "abc123", ix)

cmdeploy/src/cmdeploy/tests/test_rshell.py

@@ -1,68 +0,0 @@
-from unittest.mock import patch
-
-from cmdeploy.remote.rshell import dovecot_recalc_quota
-
-
-def test_dovecot_recalc_quota_normal_output():
-    """Normal doveadm output returns parsed dict."""
-    normal_output = (
-        "Quota name Type    Value  Limit  %\n"
-        "User quota STORAGE     5 102400  0\n"
-        "User quota MESSAGE     2      -  0\n"
-    )
-
-    with patch("cmdeploy.remote.rshell.shell", return_value=normal_output):
-        result = dovecot_recalc_quota("user@example.org")
-
-    # shell is called twice (recalc + get), patch returns same for both
-    assert result == {"value": 5, "limit": 102400, "percent": 0}
-
-
-def test_dovecot_recalc_quota_empty_output():
-    """Empty doveadm output (trailing newline) must not IndexError."""
-    call_count = [0]
-
-    def mock_shell(cmd):
-        call_count[0] += 1
-        if "recalc" in cmd:
-            return ""
-        # quota get returns only empty lines
-        return "\n\n"
-
-    with patch("cmdeploy.remote.rshell.shell", side_effect=mock_shell):
-        result = dovecot_recalc_quota("user@example.org")
-
-    assert result is None
-
-
-def test_dovecot_recalc_quota_malformed_output():
-    """Malformed output with too few columns must not crash."""
-    call_count = [0]
-
-    def mock_shell(cmd):
-        call_count[0] += 1
-        if "recalc" in cmd:
-            return ""
-        # partial line, fewer than 6 parts
-        return "Quota name\nUser quota STORAGE\n"
-
-    with patch("cmdeploy.remote.rshell.shell", side_effect=mock_shell):
-        result = dovecot_recalc_quota("user@example.org")
-
-    assert result is None
-
-
-def test_dovecot_recalc_quota_header_only():
-    """Only header line, no data rows."""
-    call_count = [0]
-
-    def mock_shell(cmd):
-        call_count[0] += 1
-        if "recalc" in cmd:
-            return ""
-        return "Quota name Type    Value  Limit  %\n"
-
-    with patch("cmdeploy.remote.rshell.shell", side_effect=mock_shell):
-        result = dovecot_recalc_quota("user@example.org")
-
-    assert result is None

cmdeploy/src/cmdeploy/tests/test_util.py

@@ -1,120 +0,0 @@
-import sys
-
-from cmdeploy.util import Out, collapse, get_git_hash, get_version_string, shell
-
-
-class TestOut:
-    def test_prefix_default(self, capsys):
-        out = Out()
-        out.print("hello")
-        assert capsys.readouterr().out == "hello\n"
-
-    def test_prefix_custom(self, capsys):
-        out = Out(prefix=">> ")
-        out.print("hello")
-        assert capsys.readouterr().out == ">> hello\n"
-
-    def test_prefix_print_file(self):
-        import io
-
-        buf = io.StringIO()
-        out = Out(prefix=":: ")
-        out.print("msg", file=buf)
-        assert ":: msg" in buf.getvalue()
-
-    def test_new_prefixed_out(self, capsys):
-        parent = Out(prefix="A")
-        child = parent.new_prefixed_out("B")
-        child.print("x")
-        assert capsys.readouterr().out == "ABx\n"
-        # shares section_timings
-        assert child.section_timings is parent.section_timings
-
-    def test_section_no_auto_indent(self, capsys):
-        out = Out(prefix="")
-        with out.section("test"):
-            out.print("inside")
-        captured = capsys.readouterr().out
-        # "inside" should NOT be indented by section()
-        lines = captured.strip().splitlines()
-        inside_line = [l for l in lines if "inside" in l][0]
-        assert inside_line == "inside"
-
-    def test_section_records_timing(self):
-        out = Out()
-        with out.section("s1"):
-            pass
-        assert len(out.section_timings) == 1
-        assert out.section_timings[0][0] == "s1"
-
-    def test_shell_failure_shows_output(self):
-        """When a shell command fails, its output and exit code are shown."""
-        import subprocess
-
-        result = subprocess.run(
-            [
-                sys.executable,
-                "-c",
-                "from cmdeploy.util import Out; Out(prefix='').shell("
-                "\"echo 'boom on stderr' >&2; exit 42\")",
-            ],
-            capture_output=True,
-            text=True,
-            check=False,
-        )
-        # the command's stderr is merged into stdout by Popen
-        assert "boom on stderr" in result.stdout
-        # Out.red() prints the failure notice to stderr
-        assert "exit code 42" in result.stderr
-
-
-def test_collapse():
-    text = """
-        line 1
-        line 2
-    """
-    assert collapse(text) == "line 1 line 2"
-    assert collapse("  single line  ") == "single line"
-
-
-def test_git_helpers_no_git(tmp_path):
-    # Not a git repo
-    assert get_git_hash(root=tmp_path) is None
-    assert get_version_string(root=tmp_path) == "unknown"
-
-
-def test_git_helpers_empty_repo(tmp_path):
-    shell("git init", cwd=tmp_path, check=True)
-    # No commits yet
-    assert get_git_hash(root=tmp_path) is None
-    assert get_version_string(root=tmp_path) == "unknown"
-
-
-def test_git_helpers_with_commits_and_diffs(tmp_path):
-    shell("git init", cwd=tmp_path, check=True)
-    shell("git config user.email you@example.com", cwd=tmp_path, check=True)
-    shell("git config user.name 'Your Name'", cwd=tmp_path, check=True)
-
-    # First commit
-    path = tmp_path / "file.txt"
-    path.write_text("content")
-    shell("git add file.txt", cwd=tmp_path, check=True)
-    shell("git commit -m initial", cwd=tmp_path, check=True)
-
-    git_hash = get_git_hash(root=tmp_path)
-    assert len(git_hash) >= 7  # usually 40, but git is git
-    assert get_version_string(root=tmp_path) == git_hash
-
-    # Create a diff
-    path.write_text("new content")
-    v = get_version_string(root=tmp_path)
-    assert v.startswith(git_hash + "\n")
-    assert "new content" in v
-    assert not v.endswith("\n")
-
-    # Commit again -> no diff
-    shell("git add file.txt", cwd=tmp_path, check=True)
-    shell("git commit -m second", cwd=tmp_path, check=True)
-    new_hash = get_git_hash(root=tmp_path)
-    assert new_hash != git_hash
-    assert get_version_string(root=tmp_path) == new_hash

cmdeploy/src/cmdeploy/util.py

@@ -1,169 +0,0 @@
-"""Shared utility functions for cmdeploy."""
-
-import os
-import shutil
-import subprocess
-import sys
-import textwrap
-import time
-from contextlib import contextmanager
-from pathlib import Path
-
-from termcolor import colored
-
-
-class Out:
-    """Convenience output printer providing coloring and section formatting."""
-
-    def __init__(self, prefix="", verbosity=0):
-        self.section_timings = []
-        self.prefix = prefix
-        self.sepchar = "\u2501"
-        self.verbosity = verbosity
-        env_width = os.environ.get("_CMDEPLOY_WIDTH")
-        if env_width:
-            self.section_width = int(env_width)
-        else:
-            self.section_width = shutil.get_terminal_size((80, 24)).columns
-
-    def new_prefixed_out(self, newprefix="  "):
-        """Return a new Out with an extended prefix,
-        sharing section_timings with the parent.
-        """
-        out = Out(
-            prefix=self.prefix + newprefix,
-            verbosity=self.verbosity,
-        )
-        out.section_timings = self.section_timings
-        return out
-
-    def red(self, msg, file=sys.stderr):
-        print(colored(self.prefix + msg, "red"), file=file, flush=True)
-
-    def green(self, msg, file=sys.stderr):
-        print(colored(self.prefix + msg, "green"), file=file, flush=True)
-
-    def print(self, msg="", **kwargs):
-        """Print to stdout with automatic flush."""
-        if msg:
-            msg = self.prefix + msg
-        print(msg, flush=True, **kwargs)
-
-    def _format_header(self, title):
-        """Return a formatted section header string."""
-        width = self.section_width - len(self.prefix)
-        bar = self.sepchar * (width - len(title) - 5)
-        return f"{self.sepchar * 3} {title} {bar}"
-
-    @contextmanager
-    def section(self, title):
-        """Context manager that prints a section header and records elapsed time."""
-        self.green(self._format_header(title))
-        t0 = time.time()
-        yield
-        elapsed = time.time() - t0
-        self.section_timings.append((title, elapsed))
-
-    def section_line(self, title):
-        """Print a section header without timing."""
-        self.green(self._format_header(title))
-
-    def shell(self, cmd, quiet=False, **kwargs):
-        """Print *cmd*, run it, and re-print its output with the current prefix.
-
-        *cmd* is passed through :func:`collapse`, so callers
-        can use triple-quoted f-strings freely.
-        Stdout and stderr are merged, read line-by-line,
-        and each line is printed with ``self.prefix`` prepended.
-        When the command exits non-zero, a red error line is printed.
-        """
-        cmd = collapse(cmd)
-        if not quiet:
-            self.print(f"$ {cmd}")
-        indent = self.prefix + "  "
-        env = kwargs.pop("env", None)
-        if env is None:
-            env = os.environ.copy()
-        env["_CMDEPLOY_WIDTH"] = str(self.section_width - len(indent))
-        proc = subprocess.Popen(
-            cmd,
-            shell=True,
-            text=True,
-            stdin=subprocess.DEVNULL,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
-            env=env,
-            **kwargs,
-        )
-        for line in proc.stdout:
-            sys.stdout.write(indent + line)
-            sys.stdout.flush()
-        ret = proc.wait()
-        if ret:
-            self.red(f"command failed with exit code {ret}: {cmd}")
-        return ret
-
-
-def _project_root():
-    """Return the project root directory."""
-    return Path(__file__).resolve().parent.parent.parent.parent
-
-
-def collapse(text):
-    """Dedent, join lines, and strip a (triple-quoted) string.
-
-    Handy for writing shell commands across multiple lines::
-
-        cmd = collapse(f\"""
-            cmdeploy run
-            --config {ct.ini}
-            --ssh-host {ct.domain}
-        \""")
-    """
-    return textwrap.dedent(text).replace("\n", " ").strip()
-
-
-def shell(cmd, check=False, **kwargs):
-    """Run a shell command string with sensible defaults.
-
-    *cmd* is passed through :func:`collapse` first, so callers
-    can use triple-quoted f-strings freely.
-    Captures stdout/stderr by default; pass ``capture_output=False``
-    to stream output to the terminal instead.
-    """
-    if "capture_output" not in kwargs and "stdout" not in kwargs:
-        kwargs["capture_output"] = True
-    kwargs.setdefault("stdin", subprocess.DEVNULL)
-    return subprocess.run(collapse(cmd), shell=True, text=True, check=check, **kwargs)
-
-
-def get_git_hash(root=None):
-    """Return the local HEAD commit hash, or None."""
-    if root is None:
-        root = _project_root()
-    result = shell(
-        "git rev-parse HEAD",
-        cwd=str(root),
-    )
-    if result.returncode == 0:
-        return result.stdout.strip()
-    return None
-
-
-def get_version_string(root=None):
-    """Return ``git_hash\\ngit_diff`` for the local working tree.
-
-    Used by :class:`~cmdeploy.deployers.GithashDeployer` to write
-    ``/etc/chatmail-version`` and by ``lxc-status`` to compare
-    the deployed state against the local checkout.
-    """
-    if root is None:
-        root = _project_root()
-    git_hash = get_git_hash(root=root) or "unknown"
-    try:
-        git_diff = shell("git diff", cwd=str(root)).stdout.strip()
-    except Exception:
-        git_diff = ""
-    if git_diff:
-        return f"{git_hash}\n{git_diff}"
-    return git_hash

doc/source/conf.py

@@ -15,7 +15,7 @@ author = 'chatmail collective'
 
 extensions = [
     #'sphinx.ext.autodoc',
-    #'sphinx.ext.viewcode',
+    #'sphinx.ext.viewdoc',
     'sphinxcontrib.mermaid',
 ]
 

doc/source/index.rst

@@ -16,6 +16,5 @@ Contributions and feedback welcome through the https://github.com/chatmail/relay
     proxy
     migrate
     overview
-    lxc
     related
     faq

doc/source/lxc.rst

@@ -1,275 +0,0 @@
-Local testing with LXC/Incus
-============================
-
-The ``cmdeploy`` tool includes support for running
-chatmail relays inside local
-`Incus <https://linuxcontainers.org/incus/>`_ LXC containers.
-This is meant for development, testing, and CI
-without requiring a remote server.
-LXC system containers are lightweight virtual machines
-that share the host's kernel but run their own init system,
-package manager, and network stack,
-so the cmdeploy deployment scripts work pretty much
-as they would on a real Debian server or cloud VPS.
-
-Prerequisites
--------------
-
-- Around 4-5 GiB free disk space
-- `systemd-networkd` for the automagic hostname resolution
-- No other service occupying Port 53
-
-Install `Incus <https://linuxcontainers.org/incus/>`_
-(LXC container manager).
-See the `official installation guide
-<https://linuxcontainers.org/incus/docs/main/installing/>`_
-for full details.
-
-After installing incus, initialise and grant yourself access::
-
-    sudo incus admin init --minimal
-    sudo usermod -aG incus-admin $USER
-
-
-.. caution::
-
-   Adding yourself to ``incus-admin`` grants effective root access
-   to the host: any member can mount host directories into a container
-   and manipulate them as root.
-   This is fine for local testing of your own relay branches,
-   but do **not** use it for production setups
-   or for testing untrusted relay branches from others.
-
-.. warning::
-
-   You **must now log out and back in** (or run ``newgrp incus-admin``)
-   after adding yourself to the group.
-   Without this, all ``cmdeploy lxc-*`` commands
-   will fail with permission errors.
-
-Verify the installation works by running ``incus list``,
-which should print an empty table without errors.
-
-
-Quick start
------------
-
-::
-
-    cd relay
-    scripts/initenv.sh               # bootstrap venv
-    source venv/bin/activate         # activate venv
-    cmdeploy lxc-test                # create containers, deploy, test
-
-The ``lxc-test`` command provides an automated way
-to run the full deployment and test pipeline.
-It executes several ``cmdeploy`` subcommands in sequential steps.
-If a step fails, you can copy-paste the printed command
-and run it manually to debug.
-No host DNS delegation or ``~/.ssh/config`` changes are needed
-because ``lxc-test`` passes the required SSH and DNS options directly.
-
-
-CLI reference
---------------
-
-``lxc-start [--ipv4-only] [--run] [NAME ...]``
-    Create and start containers.
-    Without arguments, creates ``test0-localchat`` and ``ns-localchat`` (DNS).
-    Pass one or more ``NAME`` arguments to create user relay containers instead
-    (e.g. ``cmdeploy lxc-start myrelay``).
-    Use ``--ipv4-only`` to set ``disable_ipv6 = True`` in the generated ``chatmail.ini``,
-    producing an IPv4-only relay.
-    Use ``--run`` to automatically run ``cmdeploy run`` on each container after starting it.
-    Generates ``lxconfigs/ssh-config``.
-    It reuses existing containers and resets DNS zones to minimal records.
-
-``lxc-stop [--destroy] [--destroy-all] [NAME ...]``
-    Stop relay containers.
-    Without arguments, stops ``test0-localchat`` and ``test1-localchat``.
-    Pass ``NAME`` to stop specific containers.
-    Use ``--destroy`` to also delete the containers and their config files.
-    Use ``--destroy-all`` to additionally destroy
-    the ``ns-localchat`` DNS container **and** remove
-    the cached ``localchat-base`` and ``localchat-relay``
-    images, giving a fully clean slate for the next ``lxc-test``.
-    User containers are **never** destroyed unless named explicitly.
-
-``lxc-test [--one]``
-    By default creates, deploys, and tests both ``test0`` and ``test1``
-    for dual-domain federation testing (sets ``CHATMAIL_DOMAIN2=_test1.localchat``).
-    test0 runs dual-stack (IPv4 + IPv6) while test1 runs IPv4-only (``disable_ipv6 = True``).
-    Pass ``--one`` to only deploy and test against ``test0``
-    (skips ``test1``, does not set ``CHATMAIL_DOMAIN2``).
-
-``lxc-status``
-    Show live status of all LXC containers (including the DNS container),
-    deploy freshness (comparing ``/etc/chatmail-version``
-    against local ``git rev-parse HEAD`` and ``git diff``),
-    SSH config inclusion, and host DNS forwarding for ``.localchat``.
-    Reports **IN-SYNC**, **DIRTY** (hash matches but uncommitted changes exist),
-    **STALE** (different commit), or **NOT DEPLOYED**.
-
-
-Container types
------------------
-
-**Test relay containers** (``test0-localchat``, ``test1-localchat``)
-    Created automatically by ``lxc-test``.
-    **test0** has IPv4 and IPv6 configured,
-    **test1** is IPv4-only (``disable_ipv6 = True``).
-
-**User relay containers** (``<name>-localchat``)
-    Created by ``cmdeploy lxc-start <name>``
-    where ``<name>`` does not start with ``test``.
-    These are personal development instances,
-    never touched by ``lxc-stop --destroy`` unless named explicitly.
-
-**DNS container** (``ns-localchat``)
-    Singleton container running PowerDNS.
-    Created automatically when any relay is started.
-
-
-.. _lxc-ssh-config:
-
-SSH configuration
------------------
-
-``cmdeploy lxc-start`` generates ``lxconfigs/ssh-config``,
-a standard OpenSSH config file mapping every container name,
-its domain, and a short alias to the container's IP address::
-
-    Host test0-localchat _test0.localchat _test0
-        Hostname 10.204.0.42
-        User root
-        IdentityFile /path/to/relay/lxconfigs/id_localchat
-        IdentitiesOnly yes
-        StrictHostKeyChecking accept-new
-        UserKnownHostsFile /dev/null
-        LogLevel ERROR
-
-All ``cmdeploy`` commands (``run``, ``dns``, ``status``, ``test``)
-accept ``--ssh-config lxconfigs/ssh-config`` to use this file.
-``lxc-test`` passes it automatically.
-
-**Using containers from the host shell:**
-
-To make ``ssh _test0`` work from any terminal, add one line to ``~/.ssh/config``::
-
-    Include /absolute/path/to/relay/lxconfigs/ssh-config
-
-
-.. _lxc-dns-setup:
-.. _localchat-tld:
-
-``.localchat`` DNS and name resolution
----------------------------------------
-
-All LXC-managed chatmail domains use the ``.localchat`` pseudo-TLD
-(e.g. ``_test0.localchat``, ``_test1.localchat``),
-a non-delegated suffix that exists only within the local PowerDNS infrastructure.
-A dedicated DNS container (``ns-localchat``)
-is created so that local test relays interact
-with DNS similar to a regular public Internet setup.
-On first start, ``cmdeploy lxc-start`` creates this container
-running two `PowerDNS <https://www.powerdns.com/>`_ services:
-
-* **pdns-server** (authoritative) serves ``.localchat``
-  zones from a local SQLite database.
-
-* **pdns-recursor** (recursive) listens on the Incus
-  bridge so all containers can use it.
-  Forwards ``.localchat`` queries to the local
-  authoritative server and resolves everything else recursively.
-
-After the DNS container is up, ``lxc-start`` configures the Incus bridge
-to advertise its IP via DHCP and disables Incus's own DNS.
-DNS records are then created in two phases matching the "cmdeploy run" deployment flow:
-
-1. **``lxc-start``** resets each relay zone to
-   **SOA, NS, and A** records (plus **AAAA** for dual-stack containers).
-   If host DNS resolution is configured, users can
-   afterwards run ``cmdeploy run --config lxconfigs/chatmail-test0.ini
-   --ssh-config lxconfigs/ssh-config --ssh-host _test0.localchat``.
-   LXC subcommands do not depend on host DNS resolution
-   and resolve addresses via ``lxconfigs/ssh-config``.
-
-2. **``cmdeploy dns --zonefile``** generates a standard
-   BIND-format zone file (MX, TXT/SPF, TXT/DMARC,
-   TXT/MTA-STS, SRV, CNAME, DKIM) and loads it
-   into PowerDNS.
-
-This two-phase approach prevents premature configuration of mail records
-before the relay is actually deployed and running.
-Once ``cmdeploy run`` deploys `Unbound <https://nlnetlabs.nl/projects/unbound/>`_
-inside a relay container, Unbound has a configuration plugin snippet
-that forwards all ``.localchat`` queries to the PowerDNS recursor,
-and lets all other queries go through normal recursive resolution.
-
-
-State outside the repository
------------------------------
-
-All generated configuration by lxc subcommands live in ``lxconfigs/``
-(git-ignored), including the SSH key pair (``id_localchat``),
-per-container ``chatmail-*.ini`` files, zone files, and ``ssh-config``.
-
-The only state *outside* the repository is the Incus containers and images themselves
-(managed via the ``incus`` CLI, labelled with ``user.localchat-managed=true``).
-The Incus image store retains the following snapshot images:
-
-* ``localchat-base``: Debian 12 with openssh-server and Python (built on first run)
-
-* ``localchat-relay``: fully deployed relay snapshot,
-  cached after the first successful ``cmdeploy run``.
-  Subsequent relay containers launch from this image
-  so the deploy step is mostly no-ops (roughly 3× faster than a fresh deploy).
-
-
-.. _lxc-tls:
-
-TLS handling and underscore domains
-------------------------------------
-
-Container domains start with ``_`` (e.g. ``_test0.localchat``).
-As described in :doc:`getting_started` ("Running a relay with self-signed certificates"),
-underscore domains automatically use self-signed TLS
-and ``smtp_tls_security_level = encrypt``.
-This permits cross-relay federation between LXC containers
-without any external certificate authority.
-Delta Chat clients connecting to these relays
-must be configured with
-``certificateChecks = acceptInvalidCertificates``
-(the test fixtures handle this automatically).
-`PR #7926 on chatmail-core <https://github.com/chatmail/core/pull/7926>`_
-is meant to make this special setting unnecessary for chatmail clients
-that are connecting to underscore domains.
-
-
-Known limitations
-------------------
-
-The LXC environment differs from a production
-deployment in several ways:
-
-**No ACME / Let's Encrypt**:
-  Self-signed TLS only (see :ref:`lxc-tls`);
-  ACME code paths are never exercised locally.
-
-**No inbound connections from the internet**:
-  Containers sit on a private Incus bridge and are not port-forwarded.
-  Only the host and other containers on the same bridge can reach them.
-
-**Local federation only**:
-  Cross-relay mail delivery (e.g. test0 → test1) works between containers on the same host,
-  but these relays are invisible to any external mail server.
-
-**DNS is local only**:
-  The ``.localchat`` pseudo-TLD is not resolvable from the wider internet
-  (see :ref:`lxc-dns-setup`).
-
-**IPv6 is ULA-only**:
-  Containers receive IPv6 addresses from the ``fd42:...`` ULA range on the Incus bridge.
-  These are not globally routable, but are sufficient for testing IPv6 service binding
-  (Postfix, Dovecot, Nginx) and DNS AAAA records inside the local environment.
-  test1 runs with ``disable_ipv6 = True`` to exercise the IPv4-only deployment path.

doc/source/overview.rst

@@ -109,6 +109,10 @@ short overview of ``chatmaild`` services:
    is contacted by Dovecot when a user logs in and stores the date of
    the login.
 
+-  `metrics <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/metrics.py>`_
+   collects some metrics and displays them at
+   ``https://example.org/metrics``.
+
 ``www/``
 ~~~~~~~~~
 
@@ -138,9 +142,11 @@ Chatmail relay dependency diagram
         nginx-internal --- autoconfig.xml;
         certs-nginx[("`TLS certs
         /var/lib/acme`")] --> nginx-internal;
+        systemd-timer --- chatmail-metrics;
         systemd-timer --- acmetool;
         systemd-timer --- chatmail-expire-daily;
         systemd-timer --- chatmail-fsreport-daily;
+        chatmail-metrics --- website;
         acmetool --> certs[("`TLS certs
         /var/lib/acme`")];
         nginx-external --- |993|dovecot;