docker: trim compose override example

- remove all traces of CHATMAIL_NOACME; purge certwatch service
- introduce TLS_EXTERNAL_CERT_AND_KEY as per new logic

.dockerignore

@@ -0,0 +1,18 @@
+data/
+venv/
+__pycache__
+*.pyc
+*.orig
+*.ini
+.pytest_cache
+.env
+
+# Slim build context — .git/ alone can be 100s of MB
+.git
+.github/
+docs/
+tests/
+
+# Exclude markdown files but keep www/src/*.md (used by WebsiteDeployer)
+*.md
+!www/**/*.md

.github/workflows/ci.yaml

@@ -1,32 +1,21 @@
-name: Run unit-tests and container-based deploy+test verification
+name: CI
 
 on:
-  # Triggers when a PR is merged into main or a direct push occurs
-  push:
-    branches: [ "main" ]
-    
-  # Triggers for any PR (and its subsequent commits) targeting the main branch
   pull_request:
-    branches: [ "main" ]
-
-# Newest push wins: Prevents multiple runs from clashing and wasting runner efforts
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
+  push:
 
 jobs:
   tox:
     name: isolated chatmaild tests
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
         # Checkout pull request HEAD commit instead of merge commit
         # Otherwise `test_deployed_state` will be unhappy.
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: download filtermail
-        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.6.1/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
+        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.3.0/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
       - name: run chatmaild tests 
         working-directory: chatmaild
         run: pipx run tox
@@ -35,9 +24,7 @@ jobs:
     name: deploy-chatmail tests 
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
+      - uses: actions/checkout@v4
 
       - name: initenv 
         run: scripts/initenv.sh
@@ -51,23 +38,5 @@ jobs:
       - name: run deploy-chatmail offline tests 
         run: pytest --pyargs cmdeploy 
 
-  lxc-test:
-    name: LXC deploy and test
-    uses: chatmail/cmlxc/.github/workflows/lxc-test.yml@v0.10.0
-    with:
-      cmlxc_commands: |
-        cmlxc init 
-        # single cmdeploy relay test
-        cmlxc -v deploy-cmdeploy --source ./repo cm0
-        cmlxc -v test-mini cm0
-        cmlxc -v test-cmdeploy cm0
-
-        # cross cmdeploy relay test
-        cmlxc -v deploy-cmdeploy --source ./repo --ipv4-only cm1
-        cmlxc -v test-cmdeploy cm0 cm1
-
-        # cross cmdeploy/madmail relay tests
-        cmlxc -v deploy-madmail mad0
-        cmlxc -v test-cmdeploy cm0 mad0
-        cmlxc -v test-mini cm0 mad0
-        cmlxc -v test-mini mad0 cm0
+      # all other cmdeploy commands require a staging server 
+      # see https://github.com/deltachat/chatmail/issues/100

.github/workflows/docker-build.yaml

@@ -0,0 +1,76 @@
+name: Docker Build
+
+on:
+  pull_request:
+    paths:
+      - 'docker/**'
+      - 'docker-compose.yaml'
+      - '.dockerignore'
+      - 'chatmaild/**'
+      - 'cmdeploy/**'
+      - '.github/workflows/docker-build.yaml'
+  push:
+    branches:
+      - main
+      - j4n/docker
+    paths:
+      - 'docker/**'
+      - 'docker-compose.yaml'
+      - '.dockerignore'
+      - 'chatmaild/**'
+      - 'cmdeploy/**'
+      - '.github/workflows/docker-build.yaml'
+    tags:
+      - 'v*'
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build:
+    name: Build Docker image
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GHCR
+        if: github.event_name == 'push'
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels)
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            # Tagged releases: v1.2.3 → :1.2.3, :1.2, :latest
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            # Branch pushes: j4n/docker → :j4n-docker
+            type=ref,event=branch
+            # Always: :sha-<hash>
+            type=sha
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: docker/chatmail_relay.dockerfile
+          push: ${{ github.event_name == 'push' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          build-args: |
+            GIT_HASH=${{ github.sha }}

.github/workflows/test-and-deploy-ipv4only.yaml

@@ -0,0 +1,95 @@
+name: deploy on staging-ipv4.testrun.org, and run tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths-ignore:
+      - 'scripts/**'
+      - '**/README.md'
+      - 'CHANGELOG.md'
+      - 'LICENSE'
+
+jobs:
+  deploy:
+    name: deploy on staging-ipv4.testrun.org, and run tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    environment:
+      name: staging-ipv4.testrun.org
+      url: https://staging-ipv4.testrun.org/
+    concurrency: staging-ipv4.testrun.org
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: prepare SSH
+        run: |
+          mkdir ~/.ssh
+          echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
+          chmod 600 ~/.ssh/id_ed25519
+          ssh-keyscan staging-ipv4.testrun.org > ~/.ssh/known_hosts
+          # save previous acme & dkim state
+          rsync -avz root@staging-ipv4.testrun.org:/var/lib/acme acme-ipv4 || true
+          rsync -avz root@staging-ipv4.testrun.org:/etc/dkimkeys dkimkeys-ipv4 || true
+          # store previous acme & dkim state on ns.testrun.org, if it contains useful certs
+          if [ -f dkimkeys-ipv4/dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys-ipv4 root@ns.testrun.org:/tmp/ || true; fi
+          if [ "$(ls -A acme-ipv4/acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme-ipv4 root@ns.testrun.org:/tmp/ || true; fi
+          # make sure CAA record isn't set
+          scp -o StrictHostKeyChecking=accept-new .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
+          ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging-ipv4.testrun.org.zone
+          ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
+          ssh root@ns.testrun.org systemctl reload nsd
+
+      - name: rebuild staging-ipv4.testrun.org to have a clean VPS
+        run: |
+            curl -X POST \
+            -H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
+            -H "Content-Type: application/json" \
+            -d '{"image":"debian-12"}' \
+            "https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_IPV4_SERVER_ID }}/actions/rebuild"
+
+      - run: scripts/initenv.sh
+
+      - name: append venv/bin to PATH
+        run: echo venv/bin >>$GITHUB_PATH
+
+      - name: upload TLS cert after rebuilding
+        run: |
+          echo " --- wait until staging-ipv4.testrun.org VPS is rebuilt --- "
+          rm ~/.ssh/known_hosts
+          while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u ; do sleep 1 ; done
+          ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u
+          # download acme & dkim state from ns.testrun.org
+          rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme-ipv4/acme acme-restore || true
+          rsync -avz root@ns.testrun.org:/tmp/dkimkeys-ipv4/dkimkeys dkimkeys-restore || true
+          # restore acme & dkim state to staging2.testrun.org
+          rsync -avz acme-restore/acme root@staging-ipv4.testrun.org:/var/lib/ || true
+          rsync -avz dkimkeys-restore/dkimkeys root@staging-ipv4.testrun.org:/etc/ || true
+          ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org chown root:root -R /var/lib/acme || true
+
+      - name: run deploy-chatmail offline tests 
+        run: pytest --pyargs cmdeploy 
+
+      - run: |
+          cmdeploy init staging-ipv4.testrun.org
+          sed -i 's#disable_ipv6 = False#disable_ipv6 = True#' chatmail.ini
+          sed -i 's/#\s*mtail_address/mtail_address/' chatmail.ini
+          cmdeploy run --verbose --skip-dns-check
+
+      - name: set DNS entries
+        run: |
+          ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
+          cmdeploy dns --zonefile staging-generated.zone
+          cat staging-generated.zone >> .github/workflows/staging-ipv4.testrun.org-default.zone
+          cat .github/workflows/staging-ipv4.testrun.org-default.zone
+          scp .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
+          ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
+          ssh root@ns.testrun.org systemctl reload nsd
+
+      - name: cmdeploy test
+        run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
+
+      - name: cmdeploy dns
+        run: cmdeploy dns -v
+

.github/workflows/test-and-deploy.yaml

@@ -0,0 +1,98 @@
+name: deploy on staging2.testrun.org, and run tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths-ignore:
+      - 'scripts/**'
+      - '**/README.md'
+      - 'CHANGELOG.md'
+      - 'LICENSE'
+
+jobs:
+  deploy:
+    name: deploy on staging2.testrun.org, and run tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    environment:
+      name: staging2.testrun.org
+      url: https://staging2.testrun.org/
+    concurrency: staging2.testrun.org
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: prepare SSH
+        run: |
+          mkdir ~/.ssh
+          echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
+          chmod 600 ~/.ssh/id_ed25519
+          ssh-keyscan staging2.testrun.org > ~/.ssh/known_hosts
+          # save previous acme & dkim state
+          rsync -avz root@staging2.testrun.org:/var/lib/acme . || true
+          rsync -avz root@staging2.testrun.org:/etc/dkimkeys . || true
+          # store previous acme & dkim state on ns.testrun.org, if it contains useful certs
+          if [ -f dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys root@ns.testrun.org:/tmp/ || true; fi
+          if [ "$(ls -A acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme root@ns.testrun.org:/tmp/ || true; fi
+          # make sure CAA record isn't set
+          scp -o StrictHostKeyChecking=accept-new .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
+          ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging2.testrun.org.zone
+          ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
+          ssh root@ns.testrun.org systemctl reload nsd
+
+      - name: rebuild staging2.testrun.org to have a clean VPS
+        run: |
+            curl -X POST \
+            -H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
+            -H "Content-Type: application/json" \
+            -d '{"image":"debian-12"}' \
+            "https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_SERVER_ID }}/actions/rebuild"
+
+      - run: scripts/initenv.sh
+
+      - name: append venv/bin to PATH
+        run: echo venv/bin >>$GITHUB_PATH
+
+      - name: upload TLS cert after rebuilding
+        run: |
+          echo " --- wait until staging2.testrun.org VPS is rebuilt --- "
+          rm ~/.ssh/known_hosts
+          while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org id -u ; do sleep 1 ; done
+          ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org id -u
+          # download acme & dkim state from ns.testrun.org
+          rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme acme-restore || true
+          rsync -avz root@ns.testrun.org:/tmp/dkimkeys dkimkeys-restore || true
+          # restore acme & dkim state to staging2.testrun.org
+          rsync -avz acme-restore/acme root@staging2.testrun.org:/var/lib/ || true
+          rsync -avz dkimkeys-restore/dkimkeys root@staging2.testrun.org:/etc/ || true
+          ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org chown root:root -R /var/lib/acme || true
+
+      - name: add hpk42 key to staging server
+        run: ssh root@staging2.testrun.org 'curl -s https://github.com/hpk42.keys >> .ssh/authorized_keys'
+
+      - name: run deploy-chatmail offline tests 
+        run: pytest --pyargs cmdeploy 
+
+      - run: |
+          cmdeploy init staging2.testrun.org
+          sed -i 's/#\s*mtail_address/mtail_address/' chatmail.ini
+
+      - run: cmdeploy run --verbose --skip-dns-check
+
+      - name: set DNS entries
+        run: |
+          ssh -o StrictHostKeyChecking=accept-new root@staging2.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
+          cmdeploy dns --zonefile staging-generated.zone --verbose
+          cat staging-generated.zone >> .github/workflows/staging.testrun.org-default.zone
+          cat .github/workflows/staging.testrun.org-default.zone
+          scp .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
+          ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
+          ssh root@ns.testrun.org systemctl reload nsd
+
+      - name: cmdeploy test
+        run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
+
+      - name: cmdeploy dns
+        run: cmdeploy dns -v
+

.github/workflows/test-tls-external.yaml

@@ -0,0 +1,37 @@
+name: test tls_external_cert_and_key on staging2.testrun.org
+
+on:
+  workflow_run:
+    workflows:
+      - "deploy on staging2.testrun.org, and run tests"
+    types:
+      - completed
+
+jobs:
+  test-tls-external:
+    name: test tls_external_cert_and_key
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    concurrency: staging2.testrun.org
+    environment:
+      name: staging2.testrun.org
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: prepare SSH
+        run: |
+          mkdir -p ~/.ssh
+          echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
+          chmod 600 ~/.ssh/id_ed25519
+          ssh-keyscan staging2.testrun.org >> ~/.ssh/known_hosts 2>/dev/null
+
+      - run: scripts/initenv.sh
+
+      - name: append venv/bin to PATH
+        run: echo venv/bin >>$GITHUB_PATH
+
+      - name: run tls_external e2e test
+        run: |
+          python -m cmdeploy.tests.setup_tls_external \
+            staging2.testrun.org

.gitignore

@@ -4,7 +4,7 @@ __pycache__/
 *$py.class
 *.swp
 *qr-*.png
-chatmail*.ini
+chatmail.ini
 
 
 # C extensions
@@ -164,3 +164,9 @@ cython_debug/
 #.idea/
 
 chatmail.zone
+
+# docker
+/data/
+/custom/
+docker-compose.override.yaml
+.env

chatmaild/pyproject.toml

@@ -6,7 +6,10 @@ build-backend = "setuptools.build_meta"
 name = "chatmaild"
 version = "0.3"
 dependencies = [
+  "aiosmtpd",
   "iniconfig",
+  "deltachat-rpc-server",
+  "deltachat-rpc-client",
   "filelock",
   "requests",
   "crypt-r >= 3.13.1 ; python_version >= '3.11'",
@@ -21,8 +24,8 @@ where = ['src']
 [project.scripts]
 doveauth = "chatmaild.doveauth:main"
 chatmail-metadata = "chatmaild.metadata:main"
-chatmail-expire = "chatmaild.expire:daily_expire_main"
-chatmail-quota-expire = "chatmaild.expire:quota_expire_main"
+chatmail-metrics = "chatmaild.metrics:main"
+chatmail-expire = "chatmaild.expire:main"
 chatmail-fsreport = "chatmaild.fsreport:main"
 lastlogin = "chatmaild.lastlogin:main"
 turnserver = "chatmaild.turnserver:main"
@@ -68,7 +71,6 @@ commands =
 deps = pytest 
        pdbpp 
        pytest-localserver
-       aiosmtpd
        execnet
 commands = pytest -v -rsXx {posargs}
 """

chatmaild/src/chatmaild/config.py

@@ -38,9 +38,6 @@ class Config:
         self.filtermail_smtp_port_incoming = int(
             params.get("filtermail_smtp_port_incoming", "10081")
         )
-        self.filtermail_http_port_incoming = int(
-            params.get("filtermail_http_port_incoming", "10082")
-        )
         self.postfix_reinject_port = int(params.get("postfix_reinject_port", "10025"))
         self.postfix_reinject_port_incoming = int(
             params.get("postfix_reinject_port_incoming", "10026")
@@ -78,7 +75,8 @@ class Config:
                     " paths: CERT_PATH KEY_PATH"
                 )
             self.tls_cert_mode = "external"
-            self.tls_cert_path, self.tls_key_path = parts
+            self.tls_cert_path = parts[0]
+            self.tls_key_path = parts[1]
         elif self.mail_domain.startswith("_"):
             self.tls_cert_mode = "self"
             self.tls_cert_path = "/etc/ssl/certs/mailserver.pem"
@@ -95,11 +93,6 @@ class Config:
         # old unused option (except for first migration from sqlite to maildir store)
         self.passdb_path = Path(params.get("passdb_path", "/home/vmail/passdb.sqlite"))
 
-    @property
-    def max_mailbox_size_mb(self):
-        """Return max_mailbox_size as an integer in megabytes."""
-        return parse_size_mb(self.max_mailbox_size)
-
     def _getbytefile(self):
         return open(self._inipath, "rb")
 
@@ -113,16 +106,6 @@ class Config:
         return User(maildir, addr, password_path, uid="vmail", gid="vmail")
 
 
-def parse_size_mb(limit):
-    """Parse a size string like ``500M`` or ``2G`` and return megabytes."""
-    value = limit.strip().upper().removesuffix("B")
-    if value.endswith("G"):
-        return int(value[:-1]) * 1024
-    if value.endswith("M"):
-        return int(value[:-1])
-    return int(value)
-
-
 def write_initial_config(inipath, mail_domain, overrides):
     """Write out default config file, using the specified config value overrides."""
     content = get_default_config_content(mail_domain, **overrides)

chatmaild/src/chatmaild/doveauth.py

@@ -1,11 +1,8 @@
 import json
 import logging
 import os
-import re
 import sys
 
-import filelock
-
 try:
     import crypt_r
 except ImportError:
@@ -16,7 +13,6 @@ from .dictproxy import DictProxy
 from .migrate_db import migrate_from_db_to_maildir
 
 NOCREATE_FILE = "/etc/chatmail-nocreate"
-VALID_LOCALPART_RE = re.compile(r"^[a-z0-9._-]+$")
 
 
 def encrypt_password(password: str):
@@ -56,10 +52,6 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
         )
         return False
 
-    if not VALID_LOCALPART_RE.match(localpart):
-        logging.warning("localpart %r contains invalid characters", localpart)
-        return False
-
     return True
 
 
@@ -148,13 +140,8 @@ class AuthDictProxy(DictProxy):
         if not is_allowed_to_create(self.config, addr, cleartext_password):
             return
 
-        lock = filelock.FileLock(str(user.password_path) + ".lock", timeout=5)
-        with lock:
-            userdata = user.get_userdb_dict()
-            if userdata:
-                return userdata
-            user.set_password(encrypt_password(cleartext_password))
-            print(f"Created address: {addr}", file=sys.stderr)
+        user.set_password(encrypt_password(cleartext_password))
+        print(f"Created address: {addr}", file=sys.stderr)
         return user.get_userdb_dict()
 
 

chatmaild/src/chatmaild/expire.py

@@ -4,26 +4,17 @@ Expire old messages and addresses.
 """
 
 import os
-import re
 import shutil
 import sys
 import time
 from argparse import ArgumentParser
 from collections import namedtuple
 from datetime import datetime
-from pathlib import Path
 from stat import S_ISREG
 
 from chatmaild.config import read_config
 
 FileEntry = namedtuple("FileEntry", ("path", "mtime", "size"))
-QuotaFileEntry = namedtuple("QuotaFileEntry", ("mtime", "quota_size", "path"))
-
-# Quota cleanup factor of max_mailbox_size. The mailbox is reset to this size.
-QUOTA_CLEANUP_FACTOR = 0.7
-
-# e.g. "cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S"
-_dovecot_fn_rex = re.compile(r".+/(\d+)\..+,S=(\d+)")
 
 
 def iter_mailboxes(basedir, maxnum):
@@ -83,54 +74,6 @@ class MailboxStat:
         self.extrafiles.sort(key=lambda x: -x.size)
 
 
-def parse_dovecot_filename(relpath):
-    m = _dovecot_fn_rex.match(relpath)
-    if not m:
-        return None
-    return QuotaFileEntry(int(m.group(1)), int(m.group(2)), relpath)
-
-
-def scan_mailbox_messages(mbox):
-    messages = []
-    for sub in ("cur", "new"):
-        for name in os_listdir_if_exists(mbox / sub):
-            if entry := parse_dovecot_filename(f"{sub}/{name}"):
-                messages.append(entry)
-    return messages
-
-
-# Within this window, large messages are deleted before small ones.
-DELETE_LARGE_FIRST_DAYS = 7
-
-
-def expire_to_target(mbox, target_bytes):
-    cutoff = time.time() - DELETE_LARGE_FIRST_DAYS * 86400
-    messages = scan_mailbox_messages(mbox)
-
-    def sort_key(msg):
-        # prio 0: Older than cutoff -> remove oldest first
-        if msg.mtime < cutoff:
-            return (0, msg.mtime)
-
-        # prio 1: more recent than cutoff, large -> remove largest first
-        if msg.quota_size > 200000:
-            return (1, -msg.quota_size, msg.mtime)
-
-        # prio 2: more recent than cutoff, small -> remove oldest first
-        return (2, msg.mtime)
-
-    total_size = sum(m.quota_size for m in messages)
-    removed = 0
-    for entry in sorted(messages, key=sort_key):
-        if total_size <= target_bytes:
-            break
-        (mbox / entry.path).unlink(missing_ok=True)
-        total_size -= entry.quota_size
-        removed += 1
-
-    return removed
-
-
 def print_info(msg):
     print(msg, file=sys.stderr)
 
@@ -200,19 +143,6 @@ class Expiry:
             else:
                 continue
             changed = True
-
-        target_bytes = (
-            self.config.max_mailbox_size_mb * 1024 * 1024 * QUOTA_CLEANUP_FACTOR
-        )
-        removed = expire_to_target(Path(mbox.basedir), target_bytes)
-        if removed:
-            changed = True
-            self.del_files += removed
-            if self.verbose:
-                print_info(
-                    f"quota-expire: removed {removed} message(s) from {mboxname}"
-                )
-
         if changed:
             self.remove_file(f"{mbox.basedir}/maildirsize")
 
@@ -224,9 +154,9 @@ class Expiry:
         )
 
 
-def daily_expire_main(args=None):
+def main(args=None):
     """Expire mailboxes and messages according to chatmail config"""
-    parser = ArgumentParser(description=daily_expire_main.__doc__)
+    parser = ArgumentParser(description=main.__doc__)
     ini = "/usr/local/lib/chatmaild/chatmail.ini"
     parser.add_argument(
         "chatmail_ini",
@@ -272,33 +202,5 @@ def daily_expire_main(args=None):
     print(exp.get_summary())
 
 
-def quota_expire_main(args=None):
-    """Remove mailbox messages to stay within a megabyte target.
-
-    This entry point is called by dovecot when a quota threshold is passed.
-    """
-
-    parser = ArgumentParser(description=quota_expire_main.__doc__)
-    parser.add_argument(
-        "target_mb",
-        type=int,
-        help="target mailbox size in megabytes",
-    )
-    parser.add_argument(
-        "mailbox_path",
-        type=Path,
-        help="path to a user mailbox",
-    )
-    args = parser.parse_args(args)
-
-    target_bytes = args.target_mb * 1024 * 1024
-
-    removed_count = expire_to_target(args.mailbox_path, target_bytes)
-    if removed_count:
-        (args.mailbox_path / "maildirsize").unlink(missing_ok=True)
-        print(
-            f"quota-expire: removed {removed_count} message(s)"
-            f" from {args.mailbox_path.name}",
-            file=sys.stderr,
-        )
-    return 0
+if __name__ == "__main__":
+    main(sys.argv[1:])

chatmaild/src/chatmaild/fsreport.py

@@ -13,20 +13,9 @@ to show storage summaries only for first 1000 mailboxes
 
     python -m chatmaild.fsreport /path/to/chatmail.ini --maxnum 1000
 
-to write Prometheus textfile for node_exporter
-
-    python -m chatmaild.fsreport --textfile /var/lib/prometheus/node-exporter/
-
-writes to /var/lib/prometheus/node-exporter/fsreport.prom
-
-to also write legacy metrics.py style output (default: /var/www/html/metrics):
-
-    python -m chatmaild.fsreport --textfile /var/lib/prometheus/node-exporter/ --legacy-metrics
-
 """
 
 import os
-import tempfile
 from argparse import ArgumentParser
 from datetime import datetime
 
@@ -59,19 +48,7 @@ class Report:
         self.num_ci_logins = self.num_all_logins = 0
         self.login_buckets = {x: 0 for x in (1, 10, 30, 40, 80, 100, 150)}
 
-        KiB = 1024
-        MiB = 1024 * KiB
-        self.message_size_thresholds = (
-            0,
-            100 * KiB,
-            MiB // 2,
-            1 * MiB,
-            2 * MiB,
-            5 * MiB,
-            10 * MiB,
-        )
-        self.message_buckets = {x: 0 for x in self.message_size_thresholds}
-        self.message_count_buckets = {x: 0 for x in self.message_size_thresholds}
+        self.message_buckets = {x: 0 for x in (0, 160000, 500000, 2000000)}
 
     def process_mailbox_stat(self, mailbox):
         # categorize login times
@@ -91,10 +68,9 @@ class Report:
             for size in self.message_buckets:
                 for msg in mailbox.messages:
                     if msg.size >= size:
-                        if self.mdir and f"/{self.mdir}/" not in msg.path:
+                        if self.mdir and not msg.relpath.startswith(self.mdir):
                             continue
                         self.message_buckets[size] += msg.size
-                        self.message_count_buckets[size] += 1
 
         self.size_messages += sum(entry.size for entry in mailbox.messages)
         self.size_extra += sum(entry.size for entry in mailbox.extrafiles)
@@ -117,10 +93,9 @@ class Report:
 
         pref = f"[{self.mdir}] " if self.mdir else ""
         for minsize, sumsize in self.message_buckets.items():
-            count = self.message_count_buckets[minsize]
             percent = (sumsize / all_messages * 100) if all_messages else 0
             print(
-                f"{pref}larger than {HSize(minsize)}: {HSize(sumsize)} ({percent:.2f}%), {count} msgs"
+                f"{pref}larger than {HSize(minsize)}: {HSize(sumsize)} ({percent:.2f}%)"
             )
 
         user_logins = self.num_all_logins - self.num_ci_logins
@@ -136,75 +111,6 @@ class Report:
         for days, active in self.login_buckets.items():
             print(f"last {days:3} days: {HSize(active)} {p(active)}")
 
-    def _write_atomic(self, filepath, content):
-        """Atomically write content to filepath via tmp+rename."""
-        dirpath = os.path.dirname(os.path.abspath(filepath))
-        fd, tmppath = tempfile.mkstemp(dir=dirpath, suffix=".tmp")
-        try:
-            with os.fdopen(fd, "w") as f:
-                f.write(content)
-            os.chmod(tmppath, 0o644)
-            os.rename(tmppath, filepath)
-        except BaseException:
-            try:
-                os.unlink(tmppath)
-            except OSError:
-                pass
-            raise
-
-    def dump_textfile(self, filepath):
-        """Dump metrics in Prometheus exposition format."""
-        lines = []
-
-        lines.append("# HELP chatmail_storage_bytes Mailbox storage in bytes.")
-        lines.append("# TYPE chatmail_storage_bytes gauge")
-        lines.append(f'chatmail_storage_bytes{{kind="messages"}} {self.size_messages}')
-        lines.append(f'chatmail_storage_bytes{{kind="extra"}} {self.size_extra}')
-        total = self.size_extra + self.size_messages
-        lines.append(f'chatmail_storage_bytes{{kind="total"}} {total}')
-
-        lines.append("# HELP chatmail_messages_bytes Sum of msg bytes >= threshold.")
-        lines.append("# TYPE chatmail_messages_bytes gauge")
-        for minsize, sumsize in self.message_buckets.items():
-            lines.append(f'chatmail_messages_bytes{{min_size="{minsize}"}} {sumsize}')
-
-        lines.append("# HELP chatmail_messages_count Number of msgs >= size threshold.")
-        lines.append("# TYPE chatmail_messages_count gauge")
-        for minsize, count in self.message_count_buckets.items():
-            lines.append(f'chatmail_messages_count{{min_size="{minsize}"}} {count}')
-
-        lines.append("# HELP chatmail_accounts Number of accounts.")
-        lines.append("# TYPE chatmail_accounts gauge")
-        user_logins = self.num_all_logins - self.num_ci_logins
-        lines.append(f'chatmail_accounts{{kind="all"}} {self.num_all_logins}')
-        lines.append(f'chatmail_accounts{{kind="ci"}} {self.num_ci_logins}')
-        lines.append(f'chatmail_accounts{{kind="user"}} {user_logins}')
-
-        lines.append(
-            "# HELP chatmail_accounts_active Non-CI accounts active within N days."
-        )
-        lines.append("# TYPE chatmail_accounts_active gauge")
-        for days, active in self.login_buckets.items():
-            lines.append(f'chatmail_accounts_active{{days="{days}"}} {active}')
-
-        self._write_atomic(filepath, "\n".join(lines) + "\n")
-
-    def dump_compat_textfile(self, filepath):
-        """Dump legacy metrics.py style metrics."""
-        user_logins = self.num_all_logins - self.num_ci_logins
-        lines = [
-            "# HELP total number of accounts",
-            "# TYPE accounts gauge",
-            f"accounts {self.num_all_logins}",
-            "# HELP number of CI accounts",
-            "# TYPE ci_accounts gauge",
-            f"ci_accounts {self.num_ci_logins}",
-            "# HELP number of non-CI accounts",
-            "# TYPE nonci_accounts gauge",
-            f"nonci_accounts {user_logins}",
-        ]
-        self._write_atomic(filepath, "\n".join(lines) + "\n")
-
 
 def main(args=None):
     """Report about filesystem storage usage of all mailboxes and messages"""
@@ -221,21 +127,19 @@ def main(args=None):
         "--days",
         default=0,
         action="store",
-        help="assume date to be DAYS older than now",
+        help="assume date to be days older than now",
     )
     parser.add_argument(
         "--min-login-age",
         default=0,
-        metavar="DAYS",
         dest="min_login_age",
         action="store",
-        help="only sum up message size if last login is at least DAYS days old",
+        help="only sum up message size if last login is at least min-login-age days old",
     )
     parser.add_argument(
         "--mdir",
-        metavar="{cur,new,tmp}",
         action="store",
-        help="only consider messages in specified Maildir subdirectory for summary",
+        help="only consider 'cur' or 'new' or 'tmp' messages for summary",
     )
 
     parser.add_argument(
@@ -244,21 +148,6 @@ def main(args=None):
         action="store",
         help="maximum number of mailboxes to iterate on",
     )
-    parser.add_argument(
-        "--textfile",
-        metavar="PATH",
-        default=None,
-        help="write Prometheus textfile to PATH (directory or file); "
-        "if PATH is a directory, writes 'fsreport.prom' inside it",
-    )
-    parser.add_argument(
-        "--legacy-metrics",
-        metavar="FILENAME",
-        nargs="?",
-        const="/var/www/html/metrics",
-        default=None,
-        help="write legacy metrics.py textfile (default: /var/www/html/metrics)",
-    )
 
     args = parser.parse_args(args)
 
@@ -272,15 +161,7 @@ def main(args=None):
     rep = Report(now=now, min_login_age=int(args.min_login_age), mdir=args.mdir)
     for mbox in iter_mailboxes(str(config.mailboxes_dir), maxnum=maxnum):
         rep.process_mailbox_stat(mbox)
-    if args.textfile:
-        path = args.textfile
-        if os.path.isdir(path):
-            path = os.path.join(path, "fsreport.prom")
-        rep.dump_textfile(path)
-    if args.legacy_metrics:
-        rep.dump_compat_textfile(args.legacy_metrics)
-    if not args.textfile and not args.legacy_metrics:
-        rep.dump_summary()
+    rep.dump_summary()
 
 
 if __name__ == "__main__":

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -18,7 +18,6 @@ max_user_send_per_minute = 60
 max_user_send_burst_size = 10
 
 # maximum mailbox size of a chatmail address
-# Oldest messages will be removed automatically, so mailboxes never run full.
 max_mailbox_size = 500M
 
 # maximum message size for an e-mail in bytes

chatmaild/src/chatmaild/metadata.py

@@ -101,11 +101,7 @@ class MetadataDictProxy(DictProxy):
                 # Handle `GETMETADATA "" /shared/vendor/deltachat/irohrelay`
                 return f"O{self.iroh_relay}\n"
             elif keyname == "vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn":
-                try:
-                    res = turn_credentials()
-                except Exception:
-                    logging.exception("failed to get TURN credentials")
-                    return "N\n"
+                res = turn_credentials()
                 port = 3478
                 return f"O{self.turn_hostname}:{port}:{res}\n"
 

chatmaild/src/chatmaild/metrics.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+import sys
+from pathlib import Path
+
+
+def main(vmail_dir=None):
+    if vmail_dir is None:
+        vmail_dir = sys.argv[1]
+
+    accounts = 0
+    ci_accounts = 0
+
+    for path in Path(vmail_dir).iterdir():
+        if not path.joinpath("cur").is_dir():
+            continue
+        accounts += 1
+        if path.name[:3] in ("ci-", "ac_"):
+            ci_accounts += 1
+
+    print("# HELP total number of accounts")
+    print("# TYPE accounts gauge")
+    print(f"accounts {accounts}")
+    print("# HELP number of CI accounts")
+    print("# TYPE ci_accounts gauge")
+    print(f"ci_accounts {ci_accounts}")
+    print("# HELP number of non-CI accounts")
+    print("# TYPE nonci_accounts gauge")
+    print(f"nonci_accounts {accounts - ci_accounts}")
+
+
+if __name__ == "__main__":
+    main()

chatmaild/src/chatmaild/newemail.py

@@ -2,8 +2,8 @@
 
 """CGI script for creating new accounts."""
 
-import ipaddress
 import json
+import random
 import secrets
 import string
 from urllib.parse import quote
@@ -15,25 +15,13 @@ ALPHANUMERIC = string.ascii_lowercase + string.digits
 ALPHANUMERIC_PUNCT = string.ascii_letters + string.digits + string.punctuation
 
 
-def wrap_ip(host):
-    if host.startswith("[") and host.endswith("]"):
-        return host
-    try:
-        ipaddress.ip_address(host)
-        return f"[{host}]"
-    except ValueError:
-        return host
-
-
 def create_newemail_dict(config: Config):
-    user = "".join(
-        secrets.choice(ALPHANUMERIC) for _ in range(config.username_max_length)
-    )
+    user = "".join(random.choices(ALPHANUMERIC, k=config.username_max_length))
     password = "".join(
         secrets.choice(ALPHANUMERIC_PUNCT)
         for _ in range(config.password_min_length + 3)
     )
-    return dict(email=f"{user}@{wrap_ip(config.mail_domain)}", password=f"{password}")
+    return dict(email=f"{user}@{config.mail_domain}", password=f"{password}")
 
 
 def create_dclogin_url(email, password):

chatmaild/src/chatmaild/tests/test_config.py

@@ -1,6 +1,6 @@
 import pytest
 
-from chatmaild.config import parse_size_mb, read_config
+from chatmaild.config import read_config
 
 
 def test_read_config_basic(example_config):
@@ -110,7 +110,6 @@ def test_config_tls_external_overrides_underscore(make_config):
     )
     assert config.tls_cert_mode == "external"
     assert config.tls_cert_path == "/certs/fullchain.pem"
-    assert config.tls_key_path == "/certs/privkey.pem"
 
 
 def test_config_tls_external_bad_format(make_config):
@@ -121,17 +120,3 @@ def test_config_tls_external_bad_format(make_config):
                 "tls_external_cert_and_key": "/only/one/path.pem",
             },
         )
-
-
-def test_parse_size_mb():
-    assert parse_size_mb("500M") == 500
-    assert parse_size_mb("2G") == 2048
-    assert parse_size_mb("  1g  ") == 1024
-    assert parse_size_mb("100MB") == 100
-    assert parse_size_mb("256") == 256
-
-
-def test_max_mailbox_size_mb(make_config):
-    config = make_config("chat.example.org")
-    assert config.max_mailbox_size == "500M"
-    assert config.max_mailbox_size_mb == 500

chatmaild/src/chatmaild/tests/test_delete_inactive_users.py

@@ -1,7 +1,7 @@
 import time
 
 from chatmaild.doveauth import AuthDictProxy
-from chatmaild.expire import daily_expire_main as main_expire
+from chatmaild.expire import main as main_expire
 
 
 def test_login_timestamps(example_config):

chatmaild/src/chatmaild/tests/test_doveauth.py

@@ -120,60 +120,6 @@ def test_handle_dovecot_protocol_iterate(gencreds, example_config):
     assert not lines[2]
 
 
-def test_invalid_localpart_characters(make_config):
-    """Test that is_allowed_to_create rejects localparts with invalid characters."""
-    config = make_config("chat.example.org", {"username_min_length": "3"})
-    password = "zequ0Aimuchoodaechik"
-    domain = config.mail_domain
-
-    # valid localparts
-    assert is_allowed_to_create(config, f"abc123@{domain}", password)
-    assert is_allowed_to_create(config, f"a.b-c_d@{domain}", password)
-
-    # uppercase rejected
-    assert not is_allowed_to_create(config, f"Abc123@{domain}", password)
-    assert not is_allowed_to_create(config, f"ABCDEFG@{domain}", password)
-
-    # spaces and special chars rejected
-    assert not is_allowed_to_create(config, f"a b cde@{domain}", password)
-    assert not is_allowed_to_create(config, f"abc+def@{domain}", password)
-    assert not is_allowed_to_create(config, f"abc!def@{domain}", password)
-    assert not is_allowed_to_create(config, f"ab@cdef@{domain}", password)
-    assert not is_allowed_to_create(config, f"abc/def@{domain}", password)
-    assert not is_allowed_to_create(config, f"abc\\def@{domain}", password)
-
-
-def test_concurrent_creation_same_account(dictproxy):
-    """Test that concurrent creation of the same account doesn't corrupt password."""
-    addr = "racetest1@chat.example.org"
-    password = "zequ0Aimuchoodaechik"
-    num_threads = 10
-    results = queue.Queue()
-
-    def create():
-        try:
-            res = dictproxy.lookup_passdb(addr, password)
-            results.put(("ok", res))
-        except Exception:
-            results.put(("err", traceback.format_exc()))
-
-    threads = [threading.Thread(target=create, daemon=True) for _ in range(num_threads)]
-    for t in threads:
-        t.start()
-    for t in threads:
-        t.join(timeout=10)
-
-    passwords_seen = set()
-    for _ in range(num_threads):
-        status, res = results.get()
-        if status == "err":
-            pytest.fail(f"concurrent creation failed\n{res}")
-        passwords_seen.add(res["password"])
-
-    # all threads must see the same password hash
-    assert len(passwords_seen) == 1
-
-
 def test_50_concurrent_lookups_different_accounts(gencreds, dictproxy):
     num_threads = 50
     req_per_thread = 5

chatmaild/src/chatmaild/tests/test_expire.py

@@ -1,8 +1,5 @@
-import itertools
 import os
 import random
-import shutil
-import time
 from datetime import datetime
 from fnmatch import fnmatch
 from pathlib import Path
@@ -12,19 +9,13 @@ import pytest
 from chatmaild.expire import (
     FileEntry,
     MailboxStat,
-    expire_to_target,
     get_file_entry,
     iter_mailboxes,
     os_listdir_if_exists,
-    parse_dovecot_filename,
-    quota_expire_main,
-    scan_mailbox_messages,
 )
-from chatmaild.expire import daily_expire_main as expiry_main
+from chatmaild.expire import main as expiry_main
 from chatmaild.fsreport import main as report_main
 
-MB = 1024 * 1024
-
 
 def fill_mbox(folderdir):
     password = folderdir.joinpath("password")
@@ -121,43 +112,6 @@ def test_report(mbox1, example_config):
     report_main(args)
 
 
-def test_report_mdir_filters_by_path(mbox1, example_config):
-    """Test that Report with mdir='cur' only counts messages in cur/ subdirectory."""
-    from chatmaild.fsreport import Report
-
-    now = datetime.utcnow().timestamp()
-
-    # Set password mtime to old enough so min_login_age check passes
-    password = Path(mbox1.basedir).joinpath("password")
-    old_time = now - 86400 * 10  # 10 days ago
-    os.utime(password, (old_time, old_time))
-
-    # Reload mailbox with updated mtime
-    from chatmaild.expire import MailboxStat
-
-    mbox = MailboxStat(mbox1.basedir)
-
-    # Report without mdir — should count all messages
-    rep_all = Report(now=now, min_login_age=1, mdir=None)
-    rep_all.process_mailbox_stat(mbox)
-    total_all = rep_all.message_buckets[0]
-
-    # Report with mdir='cur' — should only count cur/ messages
-    rep_cur = Report(now=now, min_login_age=1, mdir="cur")
-    rep_cur.process_mailbox_stat(mbox)
-    total_cur = rep_cur.message_buckets[0]
-
-    # Report with mdir='new' — should only count new/ messages
-    rep_new = Report(now=now, min_login_age=1, mdir="new")
-    rep_new.process_mailbox_stat(mbox)
-    total_new = rep_new.message_buckets[0]
-
-    # cur has 500-byte msg, new has 600-byte msg (from fill_mbox)
-    assert total_cur == 500
-    assert total_new == 600
-    assert total_all == 500 + 600
-
-
 def test_expiry_cli_basic(example_config, mbox1):
     args = (str(example_config._inipath),)
     expiry_main(args)
@@ -205,74 +159,3 @@ def test_os_listdir_if_exists(tmp_path):
     tmp_path.joinpath("x").write_text("hello")
     assert len(os_listdir_if_exists(str(tmp_path))) == 1
     assert len(os_listdir_if_exists(str(tmp_path.joinpath("123123")))) == 0
-
-
-# --- quota expire tests ---
-
-_msg_counter = itertools.count(1)
-
-
-def _create_message(basedir, sub, size, days_old=0, disk_size=None):
-    seq = next(_msg_counter)
-    mtime = int(time.time() - days_old * 86400)
-    name = f"{mtime}.M1P1Q{seq}.hostname,S={size},W={size}:2,S"
-    path = basedir / sub / name
-    path.parent.mkdir(parents=True, exist_ok=True)
-    path.write_bytes(b"x" * (disk_size if disk_size is not None else size))
-    os.utime(path, (mtime, mtime))
-    return path
-
-
-def test_parse_dovecot_filename():
-    e = parse_dovecot_filename("cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S")
-    assert e.path == "cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S"
-    assert e.mtime == 1775324677
-    assert e.quota_size == 3235
-    assert parse_dovecot_filename("cur/msg_without_structure") is None
-
-
-def test_expire_to_target(tmp_path):
-    _create_message(tmp_path, "cur", MB, days_old=10, disk_size=100)
-    _create_message(tmp_path, "new", MB, days_old=5)
-    assert len(scan_mailbox_messages(tmp_path)) == 2
-    # removes oldest first, uses S= size not disk size
-    removed = expire_to_target(tmp_path, MB)
-    assert removed == 1
-    assert len(scan_mailbox_messages(tmp_path)) == 1
-
-
-def test_expire_to_target_prioritization(tmp_path):
-    def create_messages():
-        for sub in ("cur", "new"):
-            if (tmp_path / sub).exists():
-                shutil.rmtree(tmp_path / sub)
-        # prio 0: older than 7 days
-        _create_message(tmp_path, "cur", 5 * MB, days_old=10)
-        # prio 1: last 7 days, large (>200KB)
-        _create_message(tmp_path, "cur", 5 * MB, days_old=1)
-        # prio 2: last 7 days, small
-        _create_message(tmp_path, "cur", 1000, days_old=2)
-
-    # Shrink to 6MB: only the old message (prio 0) is removed.
-    create_messages()
-    assert expire_to_target(tmp_path, 6 * MB) == 1
-    msgs = scan_mailbox_messages(tmp_path)
-    assert len(msgs) == 2
-    assert all(m.mtime > time.time() - 7 * 86400 for m in msgs)
-
-    # Shrink to 1KB: old and recent-large removed, small survives.
-    create_messages()
-    assert expire_to_target(tmp_path, 1024) == 2
-    msgs = scan_mailbox_messages(tmp_path)
-    assert len(msgs) == 1
-    assert msgs[0].quota_size == 1000
-
-
-def test_quota_expire_main(tmp_path, capsys):
-    mbox = tmp_path / "user@example.org"
-    _create_message(mbox, "cur", 2 * MB, days_old=5)
-    (mbox / "maildirsize").write_text("x")
-    quota_expire_main([str(1), str(mbox)])
-    _, err = capsys.readouterr()
-    assert "quota-expire: removed 1 message(s) from user@example.org" in err
-    assert not (mbox / "maildirsize").exists()

chatmaild/src/chatmaild/tests/test_filtermail_blackbox.py

@@ -47,8 +47,6 @@ def test_one_mail(
     make_config, make_popen, smtpserver, maildata, filtermail_mode, monkeypatch
 ):
     monkeypatch.setenv("PYTHONUNBUFFERED", "1")
-    # DKIM is tested by cmdeploy tests.
-    monkeypatch.setenv("FILTERMAIL_SKIP_DKIM", "1")
     smtp_inject_port = 20025
     if filtermail_mode == "outgoing":
         settings = dict(
@@ -66,10 +64,6 @@ def test_one_mail(
 
     popen = make_popen(["filtermail", path, filtermail_mode])
     line = popen.stderr.readline().strip()
-
-    # skip a warning that FILTERMAIL_SKIP_DKIM shouldn't be used in prod
-    if b"DKIM verification DISABLED!" in line:
-        line = popen.stderr.readline().strip()
     if b"loop" not in line:
         print(line.decode("ascii"), file=sys.stderr)
         pytest.fail("starting filtermail failed")

chatmaild/src/chatmaild/tests/test_metadata.py

@@ -314,51 +314,6 @@ def test_persistent_queue_items(tmp_path, testaddr, token):
     assert not queue_item < item2 and not item2 < queue_item
 
 
-def test_turn_credentials_exception_returns_N(notifier, metadata, monkeypatch):
-    """Test that turn_credentials() failure returns N\\n instead of crashing."""
-    import chatmaild.metadata
-
-    dictproxy = MetadataDictProxy(
-        notifier=notifier,
-        metadata=metadata,
-        turn_hostname="turn.example.org",
-    )
-
-    def mock_turn_credentials():
-        raise ConnectionRefusedError("socket not available")
-
-    monkeypatch.setattr(chatmaild.metadata, "turn_credentials", mock_turn_credentials)
-
-    transactions = {}
-    res = dictproxy.handle_dovecot_request(
-        "Lshared/0123/vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn"
-        "\tuser@example.org",
-        transactions,
-    )
-    assert res == "N\n"
-
-
-def test_turn_credentials_success(notifier, metadata, monkeypatch):
-    """Test that valid turn_credentials() returns TURN URI."""
-    import chatmaild.metadata
-
-    dictproxy = MetadataDictProxy(
-        notifier=notifier,
-        metadata=metadata,
-        turn_hostname="turn.example.org",
-    )
-
-    monkeypatch.setattr(chatmaild.metadata, "turn_credentials", lambda: "user:pass")
-
-    transactions = {}
-    res = dictproxy.handle_dovecot_request(
-        "Lshared/0123/vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn"
-        "\tuser@example.org",
-        transactions,
-    )
-    assert res == "Oturn.example.org:3478:user:pass\n"
-
-
 def test_iroh_relay(dictproxy):
     rfile = io.BytesIO(
         b"\n".join(

chatmaild/src/chatmaild/tests/test_metrics.py

@@ -0,0 +1,24 @@
+from chatmaild.metrics import main
+
+
+def test_main(tmp_path, capsys):
+    paths = []
+    for x in ("ci-asllkj", "ac_12l3kj", "qweqwe", "ci-l1k2j31l2k3"):
+        p = tmp_path.joinpath(x)
+        p.mkdir()
+        p.joinpath("cur").mkdir()
+        paths.append(p)
+
+    tmp_path.joinpath("nomailbox").mkdir()
+
+    main(tmp_path)
+    out, _ = capsys.readouterr()
+    d = {}
+    for line in out.split("\n"):
+        if line.strip() and not line.startswith("#"):
+            name, num = line.split()
+            d[name] = int(num)
+
+    assert d["accounts"] == 4
+    assert d["ci_accounts"] == 3
+    assert d["nonci_accounts"] == 1

chatmaild/src/chatmaild/tests/test_newmail.py

@@ -19,12 +19,6 @@ def test_create_newemail_dict(example_config):
     assert ac1["password"] != ac2["password"]
 
 
-def test_create_newemail_dict_ip(make_config):
-    config = make_config("1.2.3.4")
-    ac = create_newemail_dict(config)
-    assert ac["email"].endswith("@[1.2.3.4]")
-
-
 def test_create_dclogin_url():
     url = create_dclogin_url("user@example.org", "p@ss w+rd")
     assert url.startswith("dclogin:")

chatmaild/src/chatmaild/tests/test_turnserver.py

@@ -1,73 +0,0 @@
-import socket
-import threading
-import time
-from unittest.mock import patch
-
-import pytest
-
-from chatmaild.turnserver import turn_credentials
-
-SOCKET_PATH = "/run/chatmail-turn/turn.socket"
-
-
-@pytest.fixture
-def turn_socket(tmp_path):
-    """Create a real Unix socket server at a temp path."""
-    sock_path = str(tmp_path / "turn.socket")
-    server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-    server.bind(sock_path)
-    server.listen(1)
-    yield sock_path, server
-    server.close()
-
-
-def _call_turn_credentials(sock_path):
-    """Call turn_credentials but connect to sock_path instead of hardcoded path."""
-    original_connect = socket.socket.connect
-
-    def patched_connect(self, address):
-        if address == SOCKET_PATH:
-            address = sock_path
-        return original_connect(self, address)
-
-    with patch.object(socket.socket, "connect", patched_connect):
-        return turn_credentials()
-
-
-def test_turn_credentials_timeout(turn_socket):
-    """Server accepts but never responds — must raise socket.timeout."""
-    sock_path, server = turn_socket
-
-    def accept_and_hang():
-        conn, _ = server.accept()
-        time.sleep(30)
-        conn.close()
-
-    t = threading.Thread(target=accept_and_hang, daemon=True)
-    t.start()
-
-    with pytest.raises(socket.timeout):
-        _call_turn_credentials(sock_path)
-
-
-def test_turn_credentials_connection_refused(tmp_path):
-    """Socket file doesn't exist — must raise ConnectionRefusedError or FileNotFoundError."""
-    missing = str(tmp_path / "nonexistent.socket")
-    with pytest.raises((ConnectionRefusedError, FileNotFoundError)):
-        _call_turn_credentials(missing)
-
-
-def test_turn_credentials_success(turn_socket):
-    """Server responds with credentials — must return stripped string."""
-    sock_path, server = turn_socket
-
-    def respond():
-        conn, _ = server.accept()
-        conn.sendall(b"testuser:testpass\n")
-        conn.close()
-
-    t = threading.Thread(target=respond, daemon=True)
-    t.start()
-
-    result = _call_turn_credentials(sock_path)
-    assert result == "testuser:testpass"

chatmaild/src/chatmaild/turnserver.py

@@ -4,7 +4,6 @@ import socket
 
 def turn_credentials() -> str:
     with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client_socket:
-        client_socket.settimeout(5)
         client_socket.connect("/run/chatmail-turn/turn.socket")
         with client_socket.makefile("rb") as file:
             return file.readline().decode("utf-8").strip()

cmdeploy/pyproject.toml

@@ -10,6 +10,7 @@ dependencies = [
   "pillow",
   "qrcode",
   "markdown",
+  "pytest",
   "setuptools>=68",
   "termcolor",
   "build",
@@ -19,8 +20,6 @@ dependencies = [
   "pytest-xdist", 
   "execnet",
   "imap_tools",
-  "deltachat-rpc-client",
-  "deltachat-rpc-server",
 ]
 
 [project.scripts]

cmdeploy/src/cmdeploy/acmetool/__init__.py

@@ -67,7 +67,7 @@ class AcmetoolDeployer(Deployer):
         )
         files.template(
             src=importlib.resources.files(__package__).joinpath("desired.yaml.j2"),
-            dest=f"/var/lib/acme/desired/{self.domains[0]}",  # 0 is mailhost TLD
+            dest=f"/var/lib/acme/desired/{self.domains[0]}", # 0 is mailhost TLD
             user="root",
             group="root",
             mode="644",

cmdeploy/src/cmdeploy/acmetool/acmetool-redirector.service

@@ -3,7 +3,7 @@ Description=acmetool HTTP redirector
 
 [Service]
 Type=notify
-ExecStart=/usr/bin/acmetool redirector --service.uid=daemon --bind=127.0.0.1:402
+ExecStart=/usr/bin/acmetool redirector --service.uid=daemon
 Restart=always
 RestartSec=30
 

cmdeploy/src/cmdeploy/basedeploy.py

@@ -1,10 +1,7 @@
 import importlib.resources
 import io
 import os
-from contextlib import contextmanager
 
-from pyinfra import host
-from pyinfra.facts.server import Command
 from pyinfra.operations import files, server, systemd
 
 
@@ -13,39 +10,6 @@ def has_systemd():
     return os.path.isdir("/run/systemd/system")
 
 
-def is_in_container() -> bool:
-    """Return True if running inside a container (Docker, LXC, etc.)."""
-    return (
-        host.get_fact(
-            Command,
-            "systemd-detect-virt --container --quiet 2>/dev/null && echo yes || true",
-        )
-        == "yes"
-    )
-
-
-@contextmanager
-def blocked_service_startup():
-    """Prevent services from auto-starting during package installation.
-
-    Installs a ``/usr/sbin/policy-rc.d`` that exits 101, blocking any
-    service from being started by the package manager.  This avoids bind
-    conflicts and CPU/RAM spikes during initial setup.  The file is removed
-    when the context exits.
-    """
-    # For documentation about policy-rc.d, see:
-    # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
-    files.put(
-        src=get_resource("policy-rc.d"),
-        dest="/usr/sbin/policy-rc.d",
-        user="root",
-        group="root",
-        mode="755",
-    )
-    yield
-    files.file("/usr/sbin/policy-rc.d", present=False)
-
-
 def get_resource(arg, pkg=__package__):
     return importlib.resources.files(pkg).joinpath(arg)
 

cmdeploy/src/cmdeploy/chatmail.zone.j2

@@ -0,0 +1,32 @@
+;
+; Required DNS entries for chatmail servers 
+;
+{% if A %}
+{{ mail_domain }}.                   A     {{ A }}
+{% endif %}
+{% if AAAA %}
+{{ mail_domain }}.                   AAAA  {{ AAAA }}
+{% endif %}
+{{ mail_domain }}.                   MX 10 {{ mail_domain }}.
+{% if strict_tls %}
+_mta-sts.{{ mail_domain }}.          TXT "v=STSv1; id={{ sts_id }}"
+mta-sts.{{ mail_domain }}.           CNAME {{ mail_domain }}.
+{% endif %}
+www.{{ mail_domain }}.               CNAME {{ mail_domain }}.
+{{ dkim_entry }}
+
+;
+; Recommended DNS entries for interoperability and security-hardening
+;
+{{ mail_domain }}.                   TXT "v=spf1 a ~all"
+_dmarc.{{ mail_domain }}.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
+
+{% if acme_account_url %}
+{{ mail_domain }}.                   CAA 0 issue "letsencrypt.org;accounturi={{ acme_account_url }}"
+{% endif %}
+_adsp._domainkey.{{ mail_domain }}.  TXT "dkim=discardable"
+
+_submission._tcp.{{ mail_domain }}.  SRV 0 1 587 {{ mail_domain }}.
+_submissions._tcp.{{ mail_domain }}. SRV 0 1 465 {{ mail_domain }}.
+_imap._tcp.{{ mail_domain }}.        SRV 0 1 143 {{ mail_domain }}.
+_imaps._tcp.{{ mail_domain }}.       SRV 0 1 993 {{ mail_domain }}.

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -5,6 +5,7 @@ along with command line option and subcommand parsing.
 
 import argparse
 import importlib.resources
+import importlib.util
 import os
 import pathlib
 import shutil
@@ -108,7 +109,10 @@ def run_cmd(args, out):
     pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
 
     cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y"
-    if ssh_host == "localhost":
+    if ssh_host in ["localhost", "@docker"]:
+        if ssh_host == "@docker":
+            env["CHATMAIL_NOPORTCHECK"] = "True"
+            env["CHATMAIL_NOSYSCTL"] = "True"
         cmd = f"{pyinf} @local {deploy_path} -y"
 
     if version.parse(pyinfra.__version__) < version.parse("3"):
@@ -116,18 +120,24 @@ def run_cmd(args, out):
         return 1
 
     try:
-        out.check_call(cmd, env=env)
+        retcode = out.check_call(cmd, env=env)
         if args.website_only:
-            out.green("Website deployment completed.")
+            if retcode == 0:
+                out.green("Website deployment completed.")
+            else:
+                out.red("Website deployment failed.")
+        elif retcode == 0:
+            out.green("Deploy completed, call `cmdeploy dns` next.")
         elif not args.dns_check_disabled and strict_tls and not remote_data["acme_account_url"]:
             out.red("Deploy completed but letsencrypt not configured")
             out.red("Run 'cmdeploy run' again")
+            retcode = 0
         else:
-            out.green("Deploy completed, call `cmdeploy dns` next.")
-        return 0
+            out.red("Deploy failed")
     except subprocess.CalledProcessError:
         out.red("Deploy failed")
-        return 1
+        retcode = 1
+    return retcode
 
 
 def dns_cmd_options(parser):
@@ -194,16 +204,23 @@ def status_cmd(args, out):
 
 
 def test_cmd_options(parser):
-    add_ssh_host_option(parser)
+    parser.add_argument(
+        "--slow",
+        dest="slow",
+        action="store_true",
+        help="also run slow tests",
+    )
 
 
 def test_cmd(args, out):
-    """Run local and online tests for chatmail deployment."""
+    """Run local and online tests for chatmail deployment.
 
-    env = os.environ.copy()
-    env["CHATMAIL_INI"] = str(args.inipath.absolute())
-    if args.ssh_host:
-        env["CHATMAIL_SSH"] = args.ssh_host
+    This will automatically pip-install 'deltachat' if it's not available.
+    """
+
+    x = importlib.util.find_spec("deltachat")
+    if x is None:
+        out.check_call(f"{sys.executable} -m pip install deltachat")
 
     pytest_path = shutil.which("pytest")
     pytest_args = [
@@ -215,7 +232,9 @@ def test_cmd(args, out):
         "-v",
         "--durations=5",
     ]
-    ret = out.run_ret(pytest_args, env=env)
+    if args.slow:
+        pytest_args.append("--slow")
+    ret = out.run_ret(pytest_args)
     return ret
 
 
@@ -306,7 +325,7 @@ def add_ssh_host_option(parser):
     parser.add_argument(
         "--ssh-host",
         dest="ssh_host",
-        help="Run commands on 'localhost' or on a specific SSH host "
+        help="Run commands on 'localhost', via '@docker', or on a specific SSH host "
         "instead of chatmail.ini's mail_domain.",
     )
 
@@ -368,7 +387,9 @@ def get_parser():
 
 def get_sshexec(ssh_host: str, verbose=True):
     if ssh_host in ["localhost", "@local"]:
-        return LocalExec(verbose)
+        return LocalExec(verbose, docker=False)
+    elif ssh_host == "@docker":
+        return LocalExec(verbose, docker=True)
     if verbose:
         print(f"[ssh] login to {ssh_host}")
     return SSHExec(ssh_host, verbose=verbose)

cmdeploy/src/cmdeploy/deployers.py

@@ -2,16 +2,17 @@
 Chat Mail pyinfra deploy.
 """
 
+import os
 import shutil
 import subprocess
 import sys
-from io import BytesIO, StringIO
+from io import StringIO
 from pathlib import Path
 
 from chatmaild.config import read_config
 from pyinfra import facts, host, logger
-from pyinfra.api import FactBase
 from pyinfra.facts import hardware
+from pyinfra.api import FactBase
 from pyinfra.facts.files import Sha256File
 from pyinfra.facts.systemd import SystemdEnabled
 from pyinfra.operations import apt, files, pip, server, systemd
@@ -19,18 +20,16 @@ from pyinfra.operations import apt, files, pip, server, systemd
 from cmdeploy.cmdeploy import Out
 
 from .acmetool import AcmetoolDeployer
+from .external.deployer import ExternalTlsDeployer
 from .basedeploy import (
     Deployer,
     Deployment,
     activate_remote_units,
-    blocked_service_startup,
     configure_remote_units,
     get_resource,
     has_systemd,
-    is_in_container,
 )
 from .dovecot.deployer import DovecotDeployer
-from .external.deployer import ExternalTlsDeployer
 from .filtermail.deployer import FiltermailDeployer
 from .mtail.deployer import MtailDeployer
 from .nginx.deployer import NginxDeployer
@@ -124,6 +123,7 @@ def _install_remote_venv_with_chatmaild() -> None:
 
 def _configure_remote_venv_with_chatmaild(config) -> None:
     remote_base_dir = "/usr/local/lib/chatmaild"
+    remote_venv_dir = f"{remote_base_dir}/venv"
     remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini"
     root_owned = dict(user="root", group="root", mode="644")
 
@@ -134,13 +134,16 @@ def _configure_remote_venv_with_chatmaild(config) -> None:
         **root_owned,
     )
 
-    files.file(
-        path="/etc/cron.d/chatmail-metrics",
-        present=False,
-    )
-    files.file(
-        path="/var/www/html/metrics",
-        present=False,
+    files.template(
+        src=get_resource("metrics.cron.j2"),
+        dest="/etc/cron.d/chatmail-metrics",
+        user="root",
+        group="root",
+        mode="644",
+        config={
+            "mailboxes_dir": config.mailboxes_dir,
+            "execpath": f"{remote_venv_dir}/bin/chatmail-metrics",
+        },
     )
 
 
@@ -150,40 +153,35 @@ class UnboundDeployer(Deployer):
         self.need_restart = False
 
     def install(self):
-        # On an IPv4-only system, if unbound is started but not configured,
-        # it causes subsequent steps to fail to resolve hosts.
-        with blocked_service_startup():
-            apt.packages(
-                name="Install unbound",
-                packages=["unbound", "unbound-anchor", "dnsutils"],
-            )
+        # Run local DNS resolver `unbound`.
+        # `resolvconf` takes care of setting up /etc/resolv.conf
+        # to use 127.0.0.1 as the resolver.
 
-    def configure(self):
-        # Remove dynamic resolver managers that compete for /etc/resolv.conf.
-        apt.packages(
-            name="Purge resolvconf",
-            packages=["resolvconf"],
-            present=False,
-            extra_uninstall_args="--purge",
-        )
-        # systemd-resolved can't be purged due to dependencies; stop and mask.
-        server.shell(
-            name="Stop and mask systemd-resolved",
-            commands=[
-                "systemctl stop systemd-resolved.service || true",
-                "systemctl mask systemd-resolved.service",
-            ],
-        )
-        # Configure unbound resolver with Quad9 fallback and a trailing newline
-        # (SolusVM bug).
+        #
+        # On an IPv4-only system, if unbound is started but not
+        # configured, it causes subsequent steps to fail to resolve hosts.
+        # Here, we use policy-rc.d to prevent unbound from starting up
+        # on initial install.  Later, we will configure it and start it.
+        #
+        # For documentation about policy-rc.d, see:
+        # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
+        #
         files.put(
-            name="Write static resolv.conf",
-            src=BytesIO(b"nameserver 127.0.0.1\nnameserver 9.9.9.9\n"),
-            dest="/etc/resolv.conf",
+            src=get_resource("policy-rc.d"),
+            dest="/usr/sbin/policy-rc.d",
             user="root",
             group="root",
-            mode="644",
+            mode="755",
         )
+
+        apt.packages(
+            name="Install unbound",
+            packages=["unbound", "unbound-anchor", "dnsutils"],
+        )
+
+        files.file("/usr/sbin/policy-rc.d", present=False)
+
+    def configure(self):
         server.shell(
             name="Generate root keys for validating DNSSEC",
             commands=[
@@ -273,9 +271,6 @@ class WebsiteDeployer(Deployer):
             # if www_folder is a hugo page, build it
             if build_dir:
                 www_path = build_webpages(src_dir, build_dir, self.config)
-                if www_path is None:
-                    logger.warning("Web page build failed, skipping website deployment")
-                    return
             # if it is not a hugo page, upload it as is
             files.rsync(
                 f"{www_path}/", "/var/www/html", flags=["-avz", "--chown=www-data"]
@@ -342,12 +337,12 @@ class TurnDeployer(Deployer):
     def install(self):
         (url, sha256sum) = {
             "x86_64": (
-                "https://github.com/chatmail/chatmail-turn/releases/download/v0.4/chatmail-turn-x86_64-linux",
-                "1ec1f5c50122165e858a5a91bcba9037a28aa8cb8b64b8db570aa457c6141a8a",
+                "https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-x86_64-linux",
+                "841e527c15fdc2940b0469e206188ea8f0af48533be12ecb8098520f813d41e4",
             ),
             "aarch64": (
-                "https://github.com/chatmail/chatmail-turn/releases/download/v0.4/chatmail-turn-aarch64-linux",
-                "0fb3e792419494e21ecad536464929dba706bb2c88884ed8f1788141d26fc756",
+                "https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-aarch64-linux",
+                "a5fc2d06d937b56a34e098d2cd72a82d3e89967518d159bf246dc69b65e81b42",
             ),
         }[host.get_fact(facts.server.Arch)]
 
@@ -480,19 +475,10 @@ class ChatmailDeployer(Deployer):
         ("iroh", None, None),
     ]
 
-    def __init__(self, config):
-        self.config = config
-        self.mail_domain = config.mail_domain
+    def __init__(self, mail_domain):
+        self.mail_domain = mail_domain
 
     def install(self):
-        files.put(
-            name="Disable installing recommended packages globally",
-            src=BytesIO(b'APT::Install-Recommends "false";\n'),
-            dest="/etc/apt/apt.conf.d/00InstallRecommends",
-            user="root",
-            group="root",
-            mode="644",
-        )
         apt.update(name="apt update", cache_time=24 * 3600)
         apt.upgrade(name="upgrade apt packages", auto_remove=True)
 
@@ -505,18 +491,12 @@ class ChatmailDeployer(Deployer):
             name="Install rsync",
             packages=["rsync"],
         )
-
-    def configure(self):
-        # metadata crashes if the mailboxes dir does not exist
-        files.directory(
-            name="Ensure vmail mailbox directory exists",
-            path=str(self.config.mailboxes_dir),
-            user="vmail",
-            group="vmail",
-            mode="700",
-            present=True,
+        apt.packages(
+            name="Ensure cron is installed",
+            packages=["cron"],
         )
 
+    def configure(self):
         # This file is used by auth proxy.
         # https://wiki.debian.org/EtcMailName
         server.shell(
@@ -590,6 +570,14 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
         Deployment().perform_stages([WebsiteDeployer(config)])
         return
 
+    if host.get_fact(Port, port=53) != "unbound":
+        files.line(
+            name="Add 9.9.9.9 to resolv.conf",
+            path="/etc/resolv.conf",
+            # Guard against resolv.conf missing a trailing newline (SolusVM bug).
+            line="\nnameserver 9.9.9.9",
+        )
+
     # Check if mtail_address interface is available (if configured)
     if config.mtail_address and config.mtail_address not in ('127.0.0.1', '::1', 'localhost'):
         ipv4_addrs = host.get_fact(hardware.Ipv4Addrs)
@@ -598,19 +586,15 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
             Out().red(f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n")
             exit(1)
 
-    if not is_in_container():
+    if not os.environ.get("CHATMAIL_NOPORTCHECK"):
         port_services = [
             (["master", "smtpd"], 25),
             ("unbound", 53),
         ]
         if config.tls_cert_mode == "acme":
-            port_services.append(("acmetool", 402))
+            port_services.append(("acmetool", 80))
         port_services += [
             (["imap-login", "dovecot"], 143),
-            # acmetool previously listened on port 80,
-            # so don't complain during upgrade that moved it to port 402
-            # and gave the port to nginx.
-            (["acmetool", "nginx"], 80),
             ("nginx", 443),
             (["master", "smtpd"], 465),
             (["master", "smtpd"], 587),
@@ -638,7 +622,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
     tls_deployer = get_tls_deployer(config, mail_domain)
 
     all_deployers = [
-        ChatmailDeployer(config),
+        ChatmailDeployer(mail_domain),
         LegacyRemoveDeployer(),
         FiltermailDeployer(),
         JournaldDeployer(),

cmdeploy/src/cmdeploy/dns.py

@@ -1,22 +1,11 @@
 import datetime
+import importlib
+
+from jinja2 import Template
 
 from . import remote
 
 
-def parse_zone_records(text):
-    """Yield ``(name, ttl, rtype, rdata)`` from standard BIND-format text."""
-    for raw_line in text.splitlines():
-        line = raw_line.strip()
-        if not line or line.startswith(";"):
-            continue
-        try:
-            name, ttl, _in, rtype, rdata = line.split(None, 4)
-        except ValueError:
-            raise ValueError(f"Bad zone record line: {line!r}") from None
-        name = name.rstrip(".")
-        yield name, ttl, rtype.upper(), rdata
-
-
 def get_initial_remote_data(sshexec, mail_domain):
     return sshexec.logged(
         call=remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=mail_domain)
@@ -42,39 +31,13 @@ def get_filled_zone_file(remote_data):
     if not sts_id:
         remote_data["sts_id"] = datetime.datetime.now().strftime("%Y%m%d%H%M")
 
-    d = remote_data["mail_domain"]
-
-    def append_record(name, rtype, rdata, ttl=3600):
-        lines.append(f"{name:<40} {ttl:<6} IN  {rtype:<5}  {rdata}")
-
-    lines = ["; Required DNS entries"]
-    if remote_data.get("A"):
-        append_record(f"{d}.", "A", remote_data["A"])
-    if remote_data.get("AAAA"):
-        append_record(f"{d}.", "AAAA", remote_data["AAAA"])
-    append_record(f"{d}.", "MX", f"10 {d}.")
-    if remote_data.get("strict_tls"):
-        append_record(f"_mta-sts.{d}.", "TXT", f'"v=STSv1; id={remote_data["sts_id"]}"')
-        append_record(f"mta-sts.{d}.", "CNAME", f"{d}.")
-    append_record(f"www.{d}.", "CNAME", f"{d}.")
-    lines.append(remote_data["dkim_entry"])
+    template = importlib.resources.files(__package__).joinpath("chatmail.zone.j2")
+    content = template.read_text()
+    zonefile = Template(content).render(**remote_data)
+    lines = [x.strip() for x in zonefile.split("\n") if x.strip()]
     lines.append("")
-    lines.append("; Recommended DNS entries")
-    append_record(f"{d}.", "TXT", '"v=spf1 a ~all"')
-    append_record(f"_dmarc.{d}.", "TXT", '"v=DMARC1;p=reject;adkim=s;aspf=s"')
-    if remote_data.get("acme_account_url"):
-        append_record(
-            f"{d}.",
-            "CAA",
-            f'0 issue "letsencrypt.org;accounturi={remote_data["acme_account_url"]}"',
-        )
-    append_record(f"_adsp._domainkey.{d}.", "TXT", '"dkim=discardable"')
-    append_record(f"_submission._tcp.{d}.", "SRV", f"0 1 587 {d}.")
-    append_record(f"_submissions._tcp.{d}.", "SRV", f"0 1 465 {d}.")
-    append_record(f"_imap._tcp.{d}.", "SRV", f"0 1 143 {d}.")
-    append_record(f"_imaps._tcp.{d}.", "SRV", f"0 1 993 {d}.")
-    lines.append("")
-    return "\n".join(lines)
+    zonefile = "\n".join(lines)
+    return zonefile
 
 
 def check_full_zone(sshexec, remote_data, out, zonefile) -> int:
@@ -95,8 +58,7 @@ def check_full_zone(sshexec, remote_data, out, zonefile) -> int:
         returncode = 1
     if remote_data.get("dkim_entry") in required_diff:
         out(
-            "If the DKIM entry above does not work with your DNS provider,"
-            " you can try this one:\n"
+            "If the DKIM entry above does not work with your DNS provider, you can try this one:\n"
         )
         out(remote_data.get("web_dkim_entry") + "\n")
     if recommended_diff:

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -1,33 +1,19 @@
-import io
-import urllib.request
+import os
 
 from chatmaild.config import Config
 from pyinfra import host
-from pyinfra.facts.deb import DebPackages
-from pyinfra.facts.server import Arch, Command, Sysctl
+from pyinfra.facts.server import Arch, Sysctl
+from pyinfra.facts.systemd import SystemdEnabled
 from pyinfra.operations import apt, files, server, systemd
 
 from cmdeploy.basedeploy import (
     Deployer,
     activate_remote_units,
-    blocked_service_startup,
     configure_remote_units,
     get_resource,
-    is_in_container,
+    has_systemd,
 )
 
-DOVECOT_ARCHIVE_VERSION = "2.3.21+dfsg1-3"
-DOVECOT_PACKAGE_VERSION = f"1:{DOVECOT_ARCHIVE_VERSION}"
-
-DOVECOT_SHA256 = {
-    ("core", "amd64"): "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d",
-    ("core", "arm64"): "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9",
-    ("imapd", "amd64"): "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86",
-    ("imapd", "arm64"): "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f",
-    ("lmtpd", "amd64"): "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab",
-    ("lmtpd", "arm64"): "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f",
-}
-
 
 class DovecotDeployer(Deployer):
     daemon_reload = False
@@ -39,64 +25,23 @@ class DovecotDeployer(Deployer):
 
     def install(self):
         arch = host.get_fact(Arch)
-        with blocked_service_startup():
-            debs = []
-            for pkg in ("core", "imapd", "lmtpd"):
-                deb, changed = _download_dovecot_package(pkg, arch)
-                self.need_restart |= changed
-                if deb:
-                    debs.append(deb)
-            if debs:
-                deb_list = " ".join(debs)
-                # First dpkg may fail on missing dependencies (stderr suppressed);
-                # apt-get --fix-broken pulls them in, then dpkg retries cleanly.
-                server.shell(
-                    name="Install dovecot packages",
-                    commands=[
-                        f"dpkg --force-confdef --force-confold -i {deb_list} 2> /dev/null || true",
-                        "DEBIAN_FRONTEND=noninteractive apt-get -y --fix-broken install",
-                        f"dpkg --force-confdef --force-confold -i {deb_list}",
-                    ],
-                )
-                self.need_restart = True
-        files.put(
-            name="Pin dovecot packages to block Debian dist-upgrades",
-            src=io.StringIO(
-                "Package: dovecot-*\n"
-                "Pin: version *\n"
-                "Pin-Priority: -1\n"
-            ),
-            dest="/etc/apt/preferences.d/pin-dovecot",
-            user="root",
-            group="root",
-            mode="644",
-        )
+        if has_systemd() and "dovecot.service" in host.get_fact(SystemdEnabled):
+            return  # already installed and running
+        _install_dovecot_package("core", arch)
+        _install_dovecot_package("imapd", arch)
+        _install_dovecot_package("lmtpd", arch)
 
     def configure(self):
         configure_remote_units(self.config.mail_domain, self.units)
-        config_restart, self.daemon_reload = _configure_dovecot(self.config)
-        self.need_restart |= config_restart
+        self.need_restart, self.daemon_reload = _configure_dovecot(self.config)
 
     def activate(self):
         activate_remote_units(self.units)
 
-        # Detect stale binary: package installed but service still runs old (deleted) binary.
-        if not self.disable_mail and not self.need_restart:
-            stale = host.get_fact(
-                Command,
-                'pid=$(systemctl show -p MainPID --value dovecot.service 2>/dev/null);'
-                ' [ "${pid:-0}" != "0" ] && readlink "/proc/$pid/exe" 2>/dev/null | grep -q "(deleted)"'
-                " && echo STALE || true",
-            )
-            if stale == "STALE":
-                self.need_restart = True
-
         restart = False if self.disable_mail else self.need_restart
 
         systemd.service(
-            name="Disable dovecot for now"
-            if self.disable_mail
-            else "Start and enable Dovecot",
+            name="Disable dovecot for now" if self.disable_mail else "Start and enable Dovecot",
             service="dovecot.service",
             running=False if self.disable_mail else True,
             enabled=False if self.disable_mail else True,
@@ -106,49 +51,41 @@ class DovecotDeployer(Deployer):
         self.need_restart = False
 
 
-def _pick_url(primary, fallback):
-    try:
-        req = urllib.request.Request(primary, method="HEAD")
-        urllib.request.urlopen(req, timeout=10)
-        return primary
-    except Exception:
-        return fallback
-
-
-def _download_dovecot_package(package: str, arch: str) -> tuple[str | None, bool]:
-    """Download a dovecot .deb if needed, return (path, changed)."""
+def _install_dovecot_package(package: str, arch: str):
     arch = "amd64" if arch == "x86_64" else arch
     arch = "arm64" if arch == "aarch64" else arch
+    url = f"https://download.delta.chat/dovecot/dovecot-{package}_2.3.21%2Bdfsg1-3_{arch}.deb"
+    deb_filename = "/root/" + url.split("/")[-1]
 
-    pkg_name = f"dovecot-{package}"
-    sha256 = DOVECOT_SHA256.get((package, arch))
-    if sha256 is None:
-        op = apt.packages(packages=[pkg_name])
-        return None, bool(getattr(op, "changed", False))
-
-    installed_versions = host.get_fact(DebPackages).get(pkg_name, [])
-    if DOVECOT_PACKAGE_VERSION in installed_versions:
-        return None, False
-
-    url_version = DOVECOT_ARCHIVE_VERSION.replace("+", "%2B")
-    deb_base = f"{pkg_name}_{url_version}_{arch}.deb"
-    primary_url = f"https://download.delta.chat/dovecot/{deb_base}"
-    fallback_url = f"https://github.com/chatmail/dovecot/releases/download/upstream%2F{url_version}/{deb_base}"
-    url = _pick_url(primary_url, fallback_url)
-    deb_filename = f"/root/{deb_base}"
+    match (package, arch):
+        case ("core", "amd64"):
+            sha256 = "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d"
+        case ("core", "arm64"):
+            sha256 = "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9"
+        case ("imapd", "amd64"):
+            sha256 = "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86"
+        case ("imapd", "arm64"):
+            sha256 = "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f"
+        case ("lmtpd", "amd64"):
+            sha256 = "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab"
+        case ("lmtpd", "arm64"):
+            sha256 = "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f"
+        case _:
+            apt.packages(packages=[f"dovecot-{package}"])
+            return
 
     files.download(
-        name=f"Download {pkg_name}",
+        name=f"Download dovecot-{package}",
         src=url,
         dest=deb_filename,
         sha256sum=sha256,
         cache_time=60 * 60 * 24 * 365 * 10,  # never redownload the package
     )
 
-    return deb_filename, True
+    apt.deb(name=f"Install dovecot-{package}", src=deb_filename)
 
 
-def _configure_dovecot(config: Config, debug: bool = False) -> tuple[bool, bool]:
+def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
     """Configures Dovecot IMAP server."""
     need_restart = False
     daemon_reload = False
@@ -183,25 +120,19 @@ def _configure_dovecot(config: Config, debug: bool = False) -> tuple[bool, bool]
 
     # as per https://doc.dovecot.org/2.3/configuration_manual/os/
     # it is recommended to set the following inotify limits
-    can_modify = not is_in_container()
-    for name in ("max_user_instances", "max_user_watches"):
-        key = f"fs.inotify.{name}"
-        value = host.get_fact(Sysctl).get(key, 0)
-        if value > 65534:
-            continue
-        if not can_modify:
-            print(
-                "\n!!!! refusing to attempt sysctl setting in containers\n"
-                f"!!!! dovecot: sysctl {key!r}={value}, should be >65534 for production setups\n"
-                "!!!!"
+    if not os.environ.get("CHATMAIL_NOSYSCTL"):
+        for name in ("max_user_instances", "max_user_watches"):
+            key = f"fs.inotify.{name}"
+            if host.get_fact(Sysctl)[key] > 65535:
+                # Skip updating limits if already sufficient
+                # (enables running in incus containers where sysctl readonly)
+                continue
+            server.sysctl(
+                name=f"Change {key}",
+                key=key,
+                value=65535,
+                persist=True,
             )
-            continue
-        server.sysctl(
-            name=f"Change {key}",
-            key=key,
-            value=65535,
-            persist=True,
-        )
 
     timezone_env = files.line(
         name="Set TZ environment variable",

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -133,11 +133,6 @@ protocol lmtp {
   # mail_lua and push_notification_lua are needed for Lua push notification handler.
   # <https://doc.dovecot.org/2.3/configuration_manual/push_notification/#configuration>
   mail_plugins = $mail_plugins mail_lua notify push_notification push_notification_lua
-
-  # Disable fsync for LMTP. May lose delivered message,
-  # but unlikely to cause problems with multiple relays.
-  # https://doc.dovecot.org/2.3/admin_manual/mailbox_formats/#fsyncing
-  mail_fsync = never
 }
 
 plugin {
@@ -149,26 +144,12 @@ plugin {
 }
 
 plugin {
+  # for now we define static quota-rules for all users 
   quota = maildir:User quota
+  quota_rule = *:storage={{ config.max_mailbox_size }}
   quota_max_mail_size={{ config.max_message_size }}
   quota_grace = 0
-
-  quota_rule = *:storage={{ config.max_mailbox_size_mb }}M
-
-  # Trigger at 75%% of quota, expire oldest messages down to 70%%.
-  # The percentages are chosen to prevent current Delta Chat users
-  # from seeing "quota warnings" which trigger at 80% and 95%.
-
-  quota_warning = storage=75%% quota-warning {{ config.max_mailbox_size_mb * 70 // 100 }} {{ config.mailboxes_dir }}/%u
-}
-
-service quota-warning {
-  executable = script /usr/local/lib/chatmaild/venv/bin/chatmail-quota-expire
-  user = vmail
-  unix_listener quota-warning {
-     user = vmail
-     mode = 0600
-  }
+  # quota_over_flag_value = TRUE
 }
 
 # push_notification configuration
@@ -271,9 +252,6 @@ protocol imap {
   # sort -sn <(sed 's/ / C: /' *.in) <(sed 's/ / S: /' cat *.out)
 
   rawlog_dir = %h 
-
-  # Disable fsync for IMAP. May lose IMAP changes like setting flags. 
-  mail_fsync = never
 } 
 {% endif %}
 

cmdeploy/src/cmdeploy/external/deployer.py

@@ -1,8 +1,4 @@
-import io
-
-from pyinfra import host
-from pyinfra.facts.files import File
-from pyinfra.operations import files, systemd
+from pyinfra.operations import files, server, systemd
 
 from cmdeploy.basedeploy import Deployer, get_resource
 
@@ -21,18 +17,19 @@ class ExternalTlsDeployer(Deployer):
         self.key_path = key_path
 
     def configure(self):
-        # Verify cert and key exist on the remote host using pyinfra facts.
-        for path in (self.cert_path, self.key_path):
-            info = host.get_fact(File, path=path)
-            if info is None:
-                raise Exception(f"External TLS file not found on server: {path}")
+        server.shell(
+            name="Verify external TLS certificate and key exist",
+            commands=[
+                f"test -f {self.cert_path} && test -f {self.key_path}",
+            ],
+        )
 
         # Deploy the .path unit (templated with the cert path).
-        # pkg=__package__ is required here because the resource files
-        # live in cmdeploy.external, not the default cmdeploy package.
         source = get_resource("tls-cert-reload.path.f", pkg=__package__)
         content = source.read_text().format(cert_path=self.cert_path).encode()
 
+        import io
+
         path_unit = files.put(
             name="Upload tls-cert-reload.path",
             src=io.BytesIO(content),
@@ -63,5 +60,10 @@ class ExternalTlsDeployer(Deployer):
             restarted=self.need_restart,
             daemon_reload=self.need_restart,
         )
-        # No explicit reload needed here: dovecot/nginx read the cert
-        # on startup, and the .path watcher handles live changes.
+        # Always trigger a reload so services pick up the current cert.
+        # The path unit handles future changes via inotify.
+        server.shell(
+            name="Reload TLS services for current certificate",
+            commands=["systemctl start tls-cert-reload.service"],
+        )
+

cmdeploy/src/cmdeploy/external/tls-cert-reload.path.f

@@ -1,10 +1,6 @@
 # Watch the TLS certificate file for changes.
 # When the cert is updated (e.g. renewed by an external process),
-# this triggers tls-cert-reload.service to reload the affected services.
-#
-# NOTE: changes to the certificates are not detected if they cross bind-mount boundaries. 
-# After cert renewal, you must then trigger the reload explicitly:
-#   systemctl start tls-cert-reload.service
+# this triggers tls-cert-reload.service to restart the affected services.
 [Unit]
 Description=Watch TLS certificate for changes
 

cmdeploy/src/cmdeploy/external/tls-cert-reload.service

@@ -11,5 +11,5 @@ Description=Reload TLS services after certificate change
 
 [Service]
 Type=oneshot
-ExecStart=/bin/systemctl try-reload-or-restart dovecot
-ExecStart=/bin/systemctl try-reload-or-restart nginx
+ExecStart=/bin/systemctl reload dovecot
+ExecStart=/bin/systemctl reload nginx

cmdeploy/src/cmdeploy/filtermail/deployer.py

@@ -14,10 +14,10 @@ class FiltermailDeployer(Deployer):
 
     def install(self):
         arch = host.get_fact(facts.server.Arch)
-        url = f"https://github.com/chatmail/filtermail/releases/download/v0.6.1/filtermail-{arch}"
+        url = f"https://github.com/chatmail/filtermail/releases/download/v0.3.0/filtermail-{arch}"
         sha256sum = {
-            "x86_64": "48b3fb80c092d00b9b0a0ef77a8673496da3b9aed5ec1851e1df936d5589d62f",
-            "aarch64": "c65bd5f45df187d3d65d6965a285583a3be0f44a6916ff12909ff9a8d702c22e",
+            "x86_64": "f14a31323ae2dad3b59d3fdafcde507521da2f951a9478cd1f2fe2b4463df71d",
+            "aarch64": "933770d75046c4fd7084ce8d43f905f8748333426ad839154f0fc654755ef09f",
         }[arch]
         self.need_restart |= files.download(
             name="Download filtermail",

cmdeploy/src/cmdeploy/metrics.cron.j2

@@ -0,0 +1 @@
+*/5 * * * * root {{ config.execpath }} {{ config.mailboxes_dir }} >/var/www/html/metrics

cmdeploy/src/cmdeploy/mtail/delivered_mail.mtail

@@ -78,11 +78,3 @@ counter rejected_unencrypted_mail_count
 /Rejected unencrypted mail/ {
   rejected_unencrypted_mail_count++
 }
-
-counter quota_expire_runs
-counter quota_expire_removed_files
-
-/quota-expire: removed (?P<count>\d+) message\(s\)/ {
-  quota_expire_runs++
-  quota_expire_removed_files += $count
-}

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -54,7 +54,7 @@ http {
 	include /etc/nginx/mime.types;
 	default_type application/octet-stream;
 
-	ssl_protocols TLSv1.2 TLSv1.3;
+	ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
 	ssl_prefer_server_ciphers on;
 	ssl_certificate {{ config.tls_cert_path }};
 	ssl_certificate_key {{ config.tls_key_path }};
@@ -73,16 +73,16 @@ http {
 
 		access_log syslog:server=unix:/dev/log,facility=local7;
 
-		location /mxdeliv/ {
-		    proxy_pass http://127.0.0.1:{{ config.filtermail_http_port_incoming }};
-		}
-
 		location / {
 			# First attempt to serve request as file, then
 			# as directory, then fall back to displaying a 404.
 			try_files $uri $uri/ =404;
 		}
 
+		location /metrics {
+			default_type text/plain;
+		}
+
 		location /new {
 {% if config.tls_cert_mode != "self" %}
 			if ($request_method = GET) {
@@ -145,25 +145,4 @@ http {
 		return 301 $scheme://{{ config.mail_domain }}$request_uri;
 		access_log syslog:server=unix:/dev/log,facility=local7;
 	}
-
-	server {
-		listen 80;
-		{% if not disable_ipv6 %}
-		listen [::]:80;
-		{% endif %}
-
-		{% if config.tls_cert_mode == "acme" %}
-		location /.well-known/acme-challenge/ {
-			proxy_pass http://acmetool;
-		}
-		{% endif %}
-
-		return 301 https://$host$request_uri;
-	}
-
-	{% if config.tls_cert_mode == "acme" %}
-	upstream acmetool {
-		server 127.0.0.1:402;
-	}
-	{% endif %}
 }

cmdeploy/src/cmdeploy/opendkim/deployer.py

@@ -37,15 +37,21 @@ class OpendkimDeployer(Deployer):
         )
         need_restart |= main_config.changed
 
-        screen_script = files.file(
-            path="/etc/opendkim/screen.lua",
-            present=False,
+        screen_script = files.put(
+            src=get_resource("opendkim/screen.lua"),
+            dest="/etc/opendkim/screen.lua",
+            user="root",
+            group="root",
+            mode="644",
         )
         need_restart |= screen_script.changed
 
-        final_script = files.file(
-            path="/etc/opendkim/final.lua",
-            present=False,
+        final_script = files.put(
+            src=get_resource("opendkim/final.lua"),
+            dest="/etc/opendkim/final.lua",
+            user="root",
+            group="root",
+            mode="644",
         )
         need_restart |= final_script.changed
 
@@ -103,13 +109,6 @@ class OpendkimDeployer(Deployer):
         )
         need_restart |= service_file.changed
 
-        files.file(
-            name="chown opendkim: /etc/dkimkeys/opendkim.private",
-            path="/etc/dkimkeys/opendkim.private",
-            user="opendkim",
-            group="opendkim",
-        )
-
         self.need_restart = need_restart
 
     def activate(self):

cmdeploy/src/cmdeploy/opendkim/final.lua

@@ -0,0 +1,42 @@
+mtaname = odkim.get_mtasymbol(ctx, "{daemon_name}")
+if mtaname == "ORIGINATING" then
+	-- Outgoing message will be signed,
+	-- no need to look for signatures.
+	return nil
+end
+
+nsigs = odkim.get_sigcount(ctx)
+if nsigs == nil then
+	return nil
+end
+
+local valid = false
+local error_msg = "No valid DKIM signature found."
+for i = 1, nsigs do
+	sig = odkim.get_sighandle(ctx, i - 1)
+	sigres = odkim.sig_result(sig)
+
+	-- All signatures that do not correspond to From: 
+	-- were ignored in screen.lua and return sigres -1.
+	-- 
+	-- Any valid signature that was not ignored like this
+	-- means the message is acceptable.
+	if sigres == 0 then
+		valid = true
+    else
+        error_msg = "DKIM signature is invalid, error code " .. tostring(sigres) .. ", search https://github.com/trusteddomainproject/OpenDKIM/blob/master/libopendkim/dkim.h#L108"
+	end
+end
+
+if valid then
+	-- Strip all DKIM-Signature headers after successful validation
+	-- Delete in reverse order to avoid index shifting.
+	for i = nsigs, 1, -1 do
+		odkim.del_header(ctx, "DKIM-Signature", i)
+	end
+else
+	odkim.set_reply(ctx, "554", "5.7.1", error_msg)
+	odkim.set_result(ctx, SMFIS_REJECT)
+end
+
+return nil

cmdeploy/src/cmdeploy/opendkim/opendkim.conf

@@ -45,6 +45,12 @@ SignHeaders *,+autocrypt,+content-type
 # Default is empty.
 OversignHeaders from,reply-to,subject,date,to,cc,resent-date,resent-from,resent-sender,resent-to,resent-cc,in-reply-to,references,list-id,list-help,list-unsubscribe,list-subscribe,list-post,list-owner,list-archive,autocrypt
 
+# Script to ignore signatures that do not correspond to the From: domain.
+ScreenPolicyScript /etc/opendkim/screen.lua
+
+# Script to reject mails without a valid DKIM signature.
+FinalPolicyScript /etc/opendkim/final.lua
+
 # In Debian, opendkim runs as user "opendkim". A umask of 007 is required when
 # using a local socket with MTAs that access the socket as a non-privileged
 # user (for example, Postfix). You may need to add user "postfix" to group

cmdeploy/src/cmdeploy/opendkim/screen.lua

@@ -0,0 +1,21 @@
+-- Ignore signatures that do not correspond to the From: domain.
+
+from_domain = odkim.get_fromdomain(ctx)
+if from_domain == nil then
+	return nil
+end
+
+n = odkim.get_sigcount(ctx)
+if n == nil then
+	return nil
+end
+
+for i = 1, n do
+	sig = odkim.get_sighandle(ctx, i - 1)
+	sig_domain = odkim.sig_getdomain(sig)
+	if from_domain ~= sig_domain then
+		odkim.sig_ignore(sig)
+	end
+end
+
+return nil

cmdeploy/src/cmdeploy/postfix/deployer.py

@@ -97,9 +97,7 @@ class PostfixDeployer(Deployer):
             server.shell(
                 name="Validate postfix configuration",
                 # Extract stderr and quit with error if non-zero
-                commands=[
-                    """bash -c 'w=$(postconf 2>&1 >/dev/null); [[ -z "$w" ]] || { echo "$w"; false; }'"""
-                ],
+                commands=["""bash -c 'w=$(postconf 2>&1 >/dev/null); [[ -z "$w" ]] || { echo "$w"; false; }'"""],
             )
         self.need_restart = need_restart
 

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -20,7 +20,7 @@ smtpd_tls_key_file={{ config.tls_key_path }}
 smtpd_tls_security_level=may
 
 smtp_tls_CApath=/etc/ssl/certs
-smtp_tls_security_level=verify
+smtp_tls_security_level={{ "verify" if config.tls_cert_mode == "acme" else "encrypt" }}
 # Send SNI extension when connecting to other servers.
 # <https://www.postfix.org/postconf.5.html#smtp_tls_servername>
 smtp_tls_servername = hostname
@@ -88,22 +88,6 @@ inet_protocols = ipv4
 inet_protocols = all
 {% endif %}
 
-# Postfix does not try IPv4 and IPv6 connections
-# concurrently as of version 3.7.11.
-#
-# When relay has both A (IPv4) and AAAA (IPv6) records,
-# but broken IPv6 connectivity,
-# every second message is delayed by the connection timeout
-# <https://www.postfix.org/postconf.5.html#smtp_connect_timeout>
-# which defaults to 30 seconds. Reducing timeouts is not a solution
-# as this will result in a failure to connect to slow servers.
-#
-# As a workaround we always prefer IPv4 when it is available.
-# 
-# The setting is documented at
-# <https://www.postfix.org/postconf.5.html#smtp_address_preference>
-smtp_address_preference=ipv4
-
 virtual_transport = lmtp:unix:private/dovecot-lmtp
 virtual_mailbox_domains = {{ config.mail_domain }}
 lmtp_header_checks = regexp:/etc/postfix/lmtp_header_cleanup

cmdeploy/src/cmdeploy/postfix/master.cf.j2

@@ -86,6 +86,7 @@ filter    unix -        n       n       -       -       lmtp
 # Local SMTP server for reinjecting incoming filtered mail 
 127.0.0.1:{{ config.postfix_reinject_port_incoming }} inet  n       -       n       -       100      smtpd
   -o syslog_name=postfix/reinject_incoming
+  -o smtpd_milters=unix:opendkim/opendkim.sock
 
 # Cleanup `Received` headers for authenticated mail
 # to avoid leaking client IP.

cmdeploy/src/cmdeploy/remote/rdns.py

@@ -53,14 +53,13 @@ def get_dkim_entry(mail_domain, pre_command, dkim_selector):
             print=log_progress,
         )
     except CalledProcessError:
-        return None, None
+        return
     dkim_value_raw = f"v=DKIM1;k=rsa;p={dkim_pubkey};s=email;t=s"
     dkim_value = '" "'.join(re.findall(".{1,255}", dkim_value_raw))
     web_dkim_value = "".join(re.findall(".{1,255}", dkim_value_raw))
-    name = f"{dkim_selector}._domainkey.{mail_domain}."
     return (
-        f'{name:<40} 3600   IN  TXT    "{dkim_value}"',
-        f'{name:<40} 3600   IN  TXT    "{web_dkim_value}"',
+        f'{dkim_selector}._domainkey.{mail_domain}. TXT "{dkim_value}"',
+        f'{dkim_selector}._domainkey.{mail_domain}. TXT "{web_dkim_value}"',
     )
 
 
@@ -95,7 +94,7 @@ def check_zonefile(zonefile, verbose=True):
         if not zf_line.strip() or zf_line.startswith(";"):
             continue
         print(f"dns-checking {zf_line!r}") if verbose else log_progress("")
-        zf_domain, _ttl, _in, zf_typ, zf_value = zf_line.split(None, 4)
+        zf_domain, zf_typ, zf_value = zf_line.split(maxsplit=2)
         zf_domain = zf_domain.rstrip(".")
         zf_value = zf_value.strip()
         query_value = query_dns(zf_typ, zf_domain)

cmdeploy/src/cmdeploy/remote/rshell.py

@@ -40,5 +40,5 @@ def dovecot_recalc_quota(user):
     #
     for line in output.split("\n"):
         parts = line.split()
-        if len(parts) >= 6 and parts[2] == "STORAGE":
+        if parts[2] == "STORAGE":
             return dict(value=int(parts[3]), limit=int(parts[4]), percent=int(parts[5]))

cmdeploy/src/cmdeploy/selfsigned/deployer.py

@@ -18,8 +18,6 @@ def openssl_selfsigned_args(domain, cert_path, key_path, days=36500):
         "-keyout", str(key_path),
         "-out", str(cert_path),
         "-subj", f"/CN={domain}",
-        # Mark as end-entity cert so it cannot be used as a CA to sign others.
-        "-addext", "basicConstraints=critical,CA:FALSE",
         "-addext", "extendedKeyUsage=serverAuth,clientAuth",
         "-addext",
         f"subjectAltName=DNS:{domain},DNS:www.{domain},DNS:mta-sts.{domain}",

cmdeploy/src/cmdeploy/service/chatmail-fsreport.service.f

@@ -5,5 +5,5 @@ After=network.target
 [Service]
 Type=oneshot
 User=vmail
-ExecStart=/usr/local/lib/chatmaild/venv/bin/chatmail-fsreport /usr/local/lib/chatmaild/chatmail.ini
+ExecStart=/usr/local/lib/chatmaild/venv/bin/chatmail-fsreport /usr/local/lib/chatmaild/chatmail.ini 
 

cmdeploy/src/cmdeploy/service/chatmail-metadata.service.f

@@ -4,7 +4,7 @@ Description=Chatmail dict proxy for IMAP METADATA
 [Service]
 ExecStart={execpath} /run/chatmail-metadata/metadata.socket {config_path}
 Restart=always
-RestartSec=5
+RestartSec=30
 User=vmail
 RuntimeDirectory=chatmail-metadata
 UMask=0077

cmdeploy/src/cmdeploy/sshexec.py

@@ -85,10 +85,9 @@ class SSHExec:
 
 
 class LocalExec:
-    FuncError = FuncError
-
-    def __init__(self, verbose=False):
+    def __init__(self, verbose=False, docker=False):
         self.verbose = verbose
+        self.docker = docker
 
     def __call__(self, call, kwargs=None, log_callback=None):
         if kwargs is None:
@@ -96,15 +95,11 @@ class LocalExec:
         return call(**kwargs)
 
     def logged(self, call, kwargs: dict):
-        title = call.__doc__
-        if not title:
-            title = call.__name__
         where = "locally"
+        if self.docker:
+            if call == remote.rdns.perform_initial_checks:
+                kwargs["pre_command"] = "docker exec chatmail "
+                where = "in docker"
         if self.verbose:
-            print_stderr(f"Running {where}: {title}(**{kwargs})")
-            return self(call, kwargs, log_callback=print_stderr)
-        else:
-            print_stderr(title, end="")
-            res = self(call, kwargs, log_callback=remote.rshell.log_progress)
-            print_stderr()
-            return res
+            print(f"Running {where}: {call.__name__}(**{kwargs})")
+        return call(**kwargs)

cmdeploy/src/cmdeploy/tests/data/zftest.zone

@@ -1,18 +1,17 @@
-; Required DNS entries
-zftest.testrun.org.                      3600   IN  A      135.181.204.127
-zftest.testrun.org.                      3600   IN  AAAA   2a01:4f9:c012:52f4::1
-zftest.testrun.org.                      3600   IN  MX     10 zftest.testrun.org.
-_mta-sts.zftest.testrun.org.             3600   IN  TXT    "v=STSv1; id=202403211706"
-mta-sts.zftest.testrun.org.              3600   IN  CNAME  zftest.testrun.org.
-www.zftest.testrun.org.                  3600   IN  CNAME  zftest.testrun.org.
-opendkim._domainkey.zftest.testrun.org.  3600   IN  TXT    "v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoYt82CVUyz2ouaqjX2kB+5J80knAyoOU3MGU5aWppmwUwwTvj/oSTSpkc5JMtVTRmKKr8NUDWAL1Yw7dfGqqPHdHfwwjS3BIvDzYx+hzgtz62RnfNgV+/2MAoNpfX7cAFIHdRzEHNtwugc3RDLquqPoupAE3Y2YRw2T5zG5fILh4vwIcJZL5Uq6B92j8wwJqOex" "33n+vm1NKQ9rxo/UsHAmZlJzpooXcG/4igTBxJyJlamVSRR6N7Nul1v//YJb7J6v2o0iPHW6uE0StzKaPPNC2IVosSRFbD9H2oqppltptFSNPlI0E+t0JBWHem6YK7xcugiO3ImMCaaU8g6Jt/wIDAQAB;s=email;t=s"
-
+; Required DNS entries for chatmail servers
+zftest.testrun.org.                   A     135.181.204.127
+zftest.testrun.org.                   AAAA  2a01:4f9:c012:52f4::1
+zftest.testrun.org.                   MX 10 zftest.testrun.org.
+_mta-sts.zftest.testrun.org.          TXT "v=STSv1; id=202403211706"
+mta-sts.zftest.testrun.org.           CNAME zftest.testrun.org.
+www.zftest.testrun.org.               CNAME zftest.testrun.org.
+opendkim._domainkey.zftest.testrun.org. TXT "v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoYt82CVUyz2ouaqjX2kB+5J80knAyoOU3MGU5aWppmwUwwTvj/oSTSpkc5JMtVTRmKKr8NUDWAL1Yw7dfGqqPHdHfwwjS3BIvDzYx+hzgtz62RnfNgV+/2MAoNpfX7cAFIHdRzEHNtwugc3RDLquqPoupAE3Y2YRw2T5zG5fILh4vwIcJZL5Uq6B92j8wwJqOex" "33n+vm1NKQ9rxo/UsHAmZlJzpooXcG/4igTBxJyJlamVSRR6N7Nul1v//YJb7J6v2o0iPHW6uE0StzKaPPNC2IVosSRFbD9H2oqppltptFSNPlI0E+t0JBWHem6YK7xcugiO3ImMCaaU8g6Jt/wIDAQAB;s=email;t=s"
 ; Recommended DNS entries
-zftest.testrun.org.                      3600   IN  TXT    "v=spf1 a ~all"
-_dmarc.zftest.testrun.org.               3600   IN  TXT    "v=DMARC1;p=reject;adkim=s;aspf=s"
-zftest.testrun.org.                      3600   IN  CAA    0 issue "letsencrypt.org;accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/1371472956"
-_adsp._domainkey.zftest.testrun.org.     3600   IN  TXT    "dkim=discardable"
-_submission._tcp.zftest.testrun.org.     3600   IN  SRV    0 1 587 zftest.testrun.org.
-_submissions._tcp.zftest.testrun.org.    3600   IN  SRV    0 1 465 zftest.testrun.org.
-_imap._tcp.zftest.testrun.org.           3600   IN  SRV    0 1 143 zftest.testrun.org.
-_imaps._tcp.zftest.testrun.org.          3600   IN  SRV    0 1 993 zftest.testrun.org.
+_submission._tcp.zftest.testrun.org.  SRV 0 1 587 zftest.testrun.org.
+_submissions._tcp.zftest.testrun.org. SRV 0 1 465 zftest.testrun.org.
+_imap._tcp.zftest.testrun.org.        SRV 0 1 143 zftest.testrun.org.
+_imaps._tcp.zftest.testrun.org.       SRV 0 1 993 zftest.testrun.org.
+zftest.testrun.org.                   CAA 0 issue "letsencrypt.org;accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/1371472956"
+zftest.testrun.org.                   TXT "v=spf1 a:zftest.testrun.org ~all"
+_dmarc.zftest.testrun.org.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
+_adsp._domainkey.zftest.testrun.org.  TXT "dkim=discardable"

cmdeploy/src/cmdeploy/tests/online/benchmark.py

@@ -41,9 +41,9 @@ class TestDC:
 
         def dc_ping_pong():
             chat.send_text("ping")
-            msg = ac2.wait_for_incoming_msg()
-            msg.get_snapshot().chat.send_text("pong")
-            ac1.wait_for_incoming_msg()
+            msg = ac2._evtracker.wait_next_incoming_message()
+            msg.chat.send_text("pong")
+            ac1._evtracker.wait_next_incoming_message()
 
         benchmark(dc_ping_pong, 5)
 
@@ -55,6 +55,6 @@ class TestDC:
             for i in range(10):
                 chat.send_text(f"hello {i}")
             for i in range(10):
-                ac2.wait_for_incoming_msg()
+                ac2._evtracker.wait_next_incoming_message()
 
-        benchmark(dc_send_10_receive_10, 5, cooldown="auto")
+        benchmark(dc_send_10_receive_10, 5)

cmdeploy/src/cmdeploy/tests/online/test_0_login.py

@@ -89,9 +89,7 @@ def test_concurrent_logins_same_account(
         assert login_results.get()
 
 
-def test_no_vrfy(cmfactory, chatmail_config):
-    ac = cmfactory.get_online_account()
-    addr = ac.get_config("addr")
+def test_no_vrfy(chatmail_config):
     domain = chatmail_config.mail_domain
 
     s = smtplib.SMTP(domain)
@@ -100,7 +98,7 @@ def test_no_vrfy(cmfactory, chatmail_config):
     s.putcmd("vrfy", f"wrongaddress@{chatmail_config.mail_domain}")
     result = s.getreply()
     print(result)
-    s.putcmd("vrfy", addr)
+    s.putcmd("vrfy", f"echo@{chatmail_config.mail_domain}")
     result2 = s.getreply()
     print(result2)
     assert result[0] == result2[0] == 252

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -7,13 +7,13 @@ import time
 import pytest
 
 from cmdeploy import remote
-from cmdeploy.cmdeploy import get_sshexec
+from cmdeploy.sshexec import SSHExec
 
 
 class TestSSHExecutor:
     @pytest.fixture(scope="class")
     def sshexec(self, sshdomain):
-        return get_sshexec(sshdomain)
+        return SSHExec(sshdomain)
 
     def test_ls(self, sshexec):
         out = sshexec(call=remote.rdns.shell, kwargs=dict(command="ls"))
@@ -27,7 +27,6 @@ class TestSSHExecutor:
         assert res["A"] or res["AAAA"]
 
     def test_logged(self, sshexec, maildomain, capsys):
-        sshexec.verbose = False
         sshexec.logged(
             remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain)
         )
@@ -53,8 +52,6 @@ class TestSSHExecutor:
                 remote.rdns.perform_initial_checks,
                 kwargs=dict(mail_domain=None),
             )
-        except AssertionError:
-            pass
         except sshexec.FuncError as e:
             assert "rdns.py" in str(e)
             assert "AssertionError" in str(e)
@@ -71,44 +68,6 @@ class TestSSHExecutor:
         assert (now - since_date).total_seconds() < 60 * 60 * 51
 
 
-def test_dovecot_main_process_matches_installed_binary(sshdomain):
-    sshexec = get_sshexec(sshdomain)
-    main_pid = int(
-        sshexec(
-            call=remote.rshell.shell,
-            kwargs=dict(
-                command="timeout 10 systemctl show -p MainPID --value dovecot.service"
-            ),
-        ).strip()
-    )
-    assert main_pid != 0, "dovecot.service MainPID is 0 -- service not running?"
-
-    exe = sshexec(
-        call=remote.rshell.shell,
-        kwargs=dict(command=f"timeout 10 readlink /proc/{main_pid}/exe"),
-    ).strip()
-    status_text = sshexec(
-        call=remote.rshell.shell,
-        kwargs=dict(
-            command="timeout 10 systemctl show -p StatusText --value dovecot.service"
-        ),
-    ).strip()
-    installed_version = sshexec(
-        call=remote.rshell.shell, kwargs=dict(command="timeout 10 dovecot --version")
-    ).strip()
-
-    assert not exe.endswith("(deleted)"), (
-        f"running dovecot binary was deleted (stale after upgrade): {exe}"
-    )
-    expected_status_text = f"v{installed_version}"
-    assert status_text == expected_status_text or status_text.startswith(
-        f"{expected_status_text} "
-    ), (
-        f"dovecot status version mismatch: "
-        f"StatusText={status_text!r}, installed={installed_version!r}"
-    )
-
-
 def test_timezone_env(remote):
     for line in remote.iter_output("env"):
         print(line)
@@ -124,8 +83,10 @@ def test_remote(remote, imap_or_smtp):
 
 
 def test_use_two_chatmailservers(cmfactory, maildomain2):
-    ac1 = cmfactory.get_online_account()
-    ac2 = cmfactory.get_online_account(domain=maildomain2)
+    ac1 = cmfactory.new_online_configuring_account(cache=False)
+    cmfactory.switch_maildomain(maildomain2)
+    ac2 = cmfactory.new_online_configuring_account(cache=False)
+    cmfactory.bring_accounts_online()
     cmfactory.get_accepted_chat(ac1, ac2)
     domain1 = ac1.get_config("addr").split("@")[1]
     domain2 = ac2.get_config("addr").split("@")[1]
@@ -185,7 +146,7 @@ def test_reject_missing_dkim(cmsetup, maildata, from_addr):
     conn.starttls()
 
     with conn as s:
-        with pytest.raises(smtplib.SMTPDataError, match="No DKIM signature found"):
+        with pytest.raises(smtplib.SMTPDataError, match="No valid DKIM signature"):
             s.sendmail(from_addr=from_addr, to_addrs=recipient.addr, msg=msg)
 
 
@@ -221,6 +182,7 @@ def test_rewrite_subject(cmsetup, maildata):
     assert "Subject: Unencrypted subject" not in rcvd_msg
 
 
+@pytest.mark.slow
 def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
     """Test that the per-account send-mail limit is exceeded."""
     user1, user2 = cmsetup.gen_users(2)
@@ -243,6 +205,7 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
     pytest.fail("Rate limit was not exceeded")
 
 
+@pytest.mark.slow
 def test_expunged(remote, chatmail_config):
     outdated_days = int(chatmail_config.delete_mails_after) + 1
     find_cmds = [
@@ -255,7 +218,7 @@ def test_expunged(remote, chatmail_config):
     ]
     outdated_days = int(chatmail_config.delete_large_after) + 1
     find_cmds.append(
-        f"find {chatmail_config.mailboxes_dir} -path '*/cur/*' -mtime +{outdated_days} -size +200k -type f"
+        "find {chatmail_config.mailboxes_dir} -path '*/cur/*' -mtime +{outdated_days} -size +200k -type f"
     )
     for cmd in find_cmds:
         for line in remote.iter_output(cmd):

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -6,8 +6,8 @@ import imap_tools
 import pytest
 import requests
 
-from cmdeploy.cmdeploy import get_sshexec
 from cmdeploy.remote import rshell
+from cmdeploy.sshexec import SSHExec
 
 
 @pytest.fixture
@@ -27,7 +27,6 @@ class TestMetadataTokens:
 
     def test_set_get_metadata(self, imap_mailbox):
         "set and get metadata token for an account"
-        time.sleep(5)  # make sure Metadata service had a chance to restart
         client = imap_mailbox.client
         client.send(b'a01 SETMETADATA INBOX (/private/devicetoken "1111" )\n')
         res = client.readline()
@@ -63,8 +62,8 @@ class TestEndToEndDeltaChat:
         chat.send_text("message0")
 
         lp.sec("wait for ac2 to receive message")
-        msg2 = ac2.wait_for_incoming_msg()
-        assert msg2.get_snapshot().text == "message0"
+        msg2 = ac2._evtracker.wait_next_incoming_message()
+        assert msg2.text == "message0"
 
     def test_exceed_quota(
         self, cmfactory, lp, tmpdir, remote, chatmail_config, sshdomain
@@ -92,41 +91,45 @@ class TestEndToEndDeltaChat:
         lp.sec(f"filling remote inbox for {user}")
         fn = f"7743102289.M843172P2484002.c20,S={quota},W=2398:2,"
         path = chatmail_config.mailboxes_dir.joinpath(user, "cur", fn)
-        sshexec = get_sshexec(sshdomain)
+        sshexec = SSHExec(sshdomain)
         sshexec(call=rshell.write_numbytes, kwargs=dict(path=str(path), num=120))
         res = sshexec(call=rshell.dovecot_recalc_quota, kwargs=dict(user=user))
         assert res["percent"] >= 100
 
         lp.sec("ac2: check quota is triggered")
 
-        def send_hello():
-            chat.send_text("hello")
-
-        for line in remote.iter_output(
-            "journalctl -n1 -f -u dovecot", ready=send_hello
-        ):
+        starting = True
+        for line in remote.iter_output("journalctl -n0 -f -u dovecot"):
+            if starting:
+                chat.send_text("hello")
+                starting = False
             if user not in line:
+                # print(line)
                 continue
             if "quota exceeded" in line:
                 return
 
     def test_securejoin(self, cmfactory, lp, maildomain2):
-        ac1 = cmfactory.get_online_account()
-        ac2 = cmfactory.get_online_account(domain=maildomain2)
+        ac1 = cmfactory.new_online_configuring_account(cache=False)
+        cmfactory.switch_maildomain(maildomain2)
+        ac2 = cmfactory.new_online_configuring_account(cache=False)
+        cmfactory.bring_accounts_online()
 
         lp.sec("ac1: create QR code and let ac2 scan it, starting the securejoin")
-        qr = ac1.get_qr_code()
+        qr = ac1.get_setup_contact_qr()
 
         lp.sec("ac2: start QR-code based setup contact protocol")
-        ch = ac2.secure_join(qr)
+        ch = ac2.qr_setup_contact(qr)
         assert ch.id >= 10
-        ac1.wait_for_securejoin_inviter_success()
+        ac1._evtracker.wait_securejoin_inviter_progress(1000)
 
     def test_dkim_header_stripped(self, cmfactory, maildomain2, lp, imap_mailbox):
         """Test that if a DC address receives a message, it has no
         DKIM-Signature and Authentication-Results headers."""
-        ac1 = cmfactory.get_online_account()
-        ac2 = cmfactory.get_online_account(domain=maildomain2)
+        ac1 = cmfactory.new_online_configuring_account(cache=False)
+        cmfactory.switch_maildomain(maildomain2)
+        ac2 = cmfactory.new_online_configuring_account(cache=False)
+        cmfactory.bring_accounts_online()
         chat = cmfactory.get_accepted_chat(ac1, imap_mailbox.dc_ac)
         chat.send_text("message0")
         chat2 = cmfactory.get_accepted_chat(ac2, imap_mailbox.dc_ac)
@@ -143,28 +146,29 @@ class TestEndToEndDeltaChat:
                 assert "dkim-signature" not in msg.headers
 
     def test_read_receipts_between_instances(self, cmfactory, lp, maildomain2):
-        ac1 = cmfactory.get_online_account()
-        ac2 = cmfactory.get_online_account(domain=maildomain2)
+        ac1 = cmfactory.new_online_configuring_account(cache=False)
+        cmfactory.switch_maildomain(maildomain2)
+        ac2 = cmfactory.new_online_configuring_account(cache=False)
+        cmfactory.bring_accounts_online()
 
         lp.sec("setup encrypted comms between ac1 and ac2 on different instances")
-        qr = ac1.get_qr_code()
-        ch = ac2.secure_join(qr)
+        qr = ac1.get_setup_contact_qr()
+        ch = ac2.qr_setup_contact(qr)
         assert ch.id >= 10
-        ac1.wait_for_securejoin_inviter_success()
+        ac1._evtracker.wait_securejoin_inviter_progress(1000)
 
         lp.sec("ac1 sends a message and ac2 marks it as seen")
         chat = ac1.create_chat(ac2)
         msg = chat.send_text("hi")
-        m = ac2.wait_for_incoming_msg()
+        m = ac2._evtracker.wait_next_incoming_message()
         m.mark_seen()
         # we can only indirectly wait for mark-seen to cause an smtp-error
         lp.sec("try to wait for markseen to complete and check error states")
         deadline = time.time() + 3.1
         while time.time() < deadline:
-            m_snap = m.get_snapshot()
-            msgs = m_snap.chat.get_messages()
+            msgs = m.chat.get_messages()
             for msg in msgs:
-                assert "error" not in m.get_info()
+                assert "error" not in m.get_message_info()
             time.sleep(1)
 
 
@@ -176,7 +180,7 @@ def test_hide_senders_ip_address(cmfactory, ssl_context):
     chat = cmfactory.get_accepted_chat(user1, user2)
 
     chat.send_text("testing submission header cleanup")
-    user2.wait_for_incoming_msg()
+    user2._evtracker.wait_next_incoming_message()
     addr = user2.get_config("addr")
     host = addr.split("@")[1]
     pw = user2.get_config("mail_pw")

cmdeploy/src/cmdeploy/tests/online/test_3_status.py

@@ -5,11 +5,7 @@ from cmdeploy.cmdeploy import main
 
 def test_status_cmd(chatmail_config, capsys, request):
     os.chdir(request.config.invocation_params.dir)
-    command = ["status"]
-    if os.getenv("CHATMAIL_SSH"):
-        command.append("--ssh-host")
-        command.append(os.getenv("CHATMAIL_SSH"))
-    assert main(command) == 0
+    assert main(["status"]) == 0
     status_out = capsys.readouterr()
     print(status_out.out)
 

cmdeploy/src/cmdeploy/tests/plugin.py

@@ -1,5 +1,5 @@
 import imaplib
-import ipaddress
+import io
 import itertools
 import os
 import random
@@ -15,12 +15,10 @@ from chatmaild.config import read_config
 conftestdir = Path(__file__).parent
 
 
-def _is_ip(domain):
-    try:
-        ipaddress.ip_address(domain)
-        return True
-    except ValueError:
-        return False
+def pytest_addoption(parser):
+    parser.addoption(
+        "--slow", action="store_true", default=False, help="also run slow tests"
+    )
 
 
 def pytest_configure(config):
@@ -30,29 +28,24 @@ def pytest_configure(config):
     )
 
 
-def _get_chatmail_config():
-    inipath = os.environ.get("CHATMAIL_INI")
-    if inipath:
-        path = Path(inipath).resolve()
-        return read_config(path), path
-
-    current = Path().resolve()
-    while 1:
-        path = current.joinpath("chatmail.ini").resolve()
-        if path.exists():
-            return read_config(path), path
-        if current == current.parent:
-            break
-        current = current.parent
-    return None, None
+def pytest_runtest_setup(item):
+    markers = list(item.iter_markers(name="slow"))
+    if markers:
+        if not item.config.getoption("--slow"):
+            pytest.skip("skipping slow test, use --slow to run")
 
 
 @pytest.fixture(scope="session")
 def chatmail_config(pytestconfig):
-    config, path = _get_chatmail_config()
-    if config:
-        return config
-    basedir = Path().resolve()
+    current = basedir = Path().resolve()
+    while 1:
+        path = current.joinpath("chatmail.ini").resolve()
+        if path.exists():
+            return read_config(path)
+        if current == current.parent:
+            break
+        current = current.parent
+
     pytest.skip(f"no chatmail.ini file found in {basedir} or parent dirs")
 
 
@@ -80,17 +73,10 @@ def sshdomain2(maildomain2):
 
 
 def pytest_report_header():
-    config, path = _get_chatmail_config()
-    domain2 = os.environ.get("CHATMAIL_DOMAIN2", "NOT SET")
-    domain = config.mail_domain if config else "NOT SET"
-    path = path if path else "NOT SET"
-
-    lines = [
-        f"chatmail.ini {domain} location: {path}",
-        f"chatmail2: {domain2}",
-    ]
-    sep = "-" * max(map(len, lines))
-    return [sep, *lines, sep]
+    domain = os.environ.get("CHATMAIL_DOMAIN")
+    if domain:
+        text = f"chatmail test instance: {domain}"
+        return ["-" * len(text), text, "-" * len(text)]
 
 
 @pytest.fixture
@@ -105,22 +91,15 @@ def cm_data(request):
 
 
 @pytest.fixture
-def benchmark(request, chatmail_config):
-    def bench(func, num, name=None, reportfunc=None, cooldown=0.0):
+def benchmark(request):
+    def bench(func, num, name=None, reportfunc=None):
         if name is None:
             name = func.__name__
-        if cooldown == "auto":
-            per_minute = max(chatmail_config.max_user_send_per_minute, 1)
-            cooldown = chatmail_config.max_user_send_burst_size * 60 / per_minute
-
         durations = []
         for i in range(num):
             now = time.time()
             func()
             durations.append(time.time() - now)
-            if cooldown > 0 and i + 1 < num:
-                # Keep post-run cooldown out of measured benchmark duration.
-                time.sleep(cooldown)
         durations.sort()
         request.config._benchresults[name] = (reportfunc, durations)
 
@@ -278,7 +257,6 @@ def gencreds(chatmail_config):
 
     def gen(domain=None):
         domain = domain if domain else chatmail_config.mail_domain
-        addr_domain = f"[{domain}]" if _is_ip(domain) else domain
         while 1:
             num = next(count)
             alphanumeric = "abcdefghijklmnopqrstuvwxyz1234567890"
@@ -292,161 +270,108 @@ def gencreds(chatmail_config):
             password = "".join(
                 random.choices(alphanumeric, k=chatmail_config.password_min_length)
             )
-            yield f"{user}@{addr_domain}", f"{password}"
+            yield f"{user}@{domain}", f"{password}"
 
     return lambda domain=None: next(gen(domain))
 
 
 #
-# Delta Chat RPC-based test support
+# Delta Chat testplugin re-use
 # use the cmfactory fixture to get chatmail instance accounts
 #
 
-from deltachat_rpc_client import DeltaChat, Rpc
 
+class ChatmailTestProcess:
+    """Provider for chatmail instance accounts as used by deltachat.testplugin.acfactory"""
 
-class ChatmailACFactory:
-    """RPC-based account factory for chatmail testing."""
-
-    def __init__(self, rpc, maildomain, gencreds, chatmail_config):
-        self.dc = DeltaChat(rpc)
-        self.rpc = rpc
-        self._maildomain = maildomain
+    def __init__(self, pytestconfig, maildomain, gencreds, chatmail_config):
+        self.pytestconfig = pytestconfig
+        self.maildomain = maildomain
+        assert "." in self.maildomain, maildomain
         self.gencreds = gencreds
         self.chatmail_config = chatmail_config
+        self._addr2files = {}
 
-    def _make_transport(self, domain):
-        """Build a transport config dict for the given domain."""
-        addr, password = self.gencreds(domain)
-        transport = {
-            "addr": addr,
-            "password": password,
-            # Setting server explicitly skips requesting autoconfig XML,
-            # see https://datatracker.ietf.org/doc/draft-ietf-mailmaint-autoconfig/
-            "imapServer": domain,
-            "smtpServer": domain,
-        }
-        if self.chatmail_config.tls_cert_mode == "self":
-            transport["certificateChecks"] = "acceptInvalidCertificates"
-        return transport
+    def get_liveconfig_producer(self):
+        while 1:
+            user, password = self.gencreds(self.maildomain)
+            config = {
+                "addr": user,
+                "mail_pw": password,
+            }
+            # speed up account configuration
+            config["mail_server"] = self.maildomain
+            config["send_server"] = self.maildomain
+            if self.chatmail_config.tls_cert_mode == "self":
+                # Accept self-signed TLS certificates
+                config["imap_certificate_checks"] = "3"
+            yield config
 
-    def get_online_account(self, domain=None):
-        """Create, configure and bring online a single account."""
-        return self.get_online_accounts(1, domain)[0]
+    def cache_maybe_retrieve_configured_db_files(self, cache_addr, db_target_path):
+        pass
 
-    def get_online_accounts(self, num, domain=None):
-        """Create multiple online accounts in parallel."""
-        domain = domain or self._maildomain
-        futures = []
-        accounts = []
-        for _ in range(num):
-            account = self.dc.add_account()
-            addr, password = self.gencreds(domain)
-            if _is_ip(domain):
-                # Use DCLOGIN scheme with explicit server hosts,
-                # matching how madmail presents its addresses to users.
-                qr = (
-                    f"dclogin:{addr}"
-                    f"?p={password}&v=1"
-                    f"&ih={domain}&ip=993"
-                    f"&sh={domain}&sp=465"
-                    f"&ic=3&ss=default"
-                )
-                future = account.add_transport_from_qr.future(qr)
-            else:
-                future = account.add_or_update_transport.future(
-                    self._make_transport(domain)
-                )
-            futures.append(future)
-
-            # ensure messages stay in INBOX so that they can be
-            # concurrently fetched via extra IMAP connections during tests
-            account.set_config("delete_server_after", "10")
-            accounts.append(account)
-
-        for future in futures:
-            future()
-
-        for account in accounts:
-            account.bring_online()
-        return accounts
-
-    def get_accepted_chat(self, ac1, ac2):
-        """Create a 1:1 chat between ac1 and ac2 accepted on both sides."""
-        ac2.create_chat(ac1)
-        return ac1.create_chat(ac2)
-
-
-@pytest.fixture(scope="session")
-def rpc(tmp_path_factory):
-    """Start a deltachat-rpc-server process for the test session."""
-
-    # NB: accounts_dir must NOT already exist as directory --
-    # core-rust only creates accounts.toml if the dir doesn't exist yet.
-    accounts_dir = str(tmp_path_factory.mktemp("dc") / "accounts")
-    rpc = Rpc(accounts_dir=accounts_dir)
-    rpc.start()
-    yield rpc
-    rpc.close()
+    def cache_maybe_store_configured_db_files(self, acc):
+        pass
 
 
 @pytest.fixture
-def cmfactory(rpc, gencreds, maildomain, chatmail_config):
-    """Return a ChatmailACFactory for creating online Delta Chat accounts."""
-    return ChatmailACFactory(
-        rpc=rpc,
-        maildomain=maildomain,
-        gencreds=gencreds,
-        chatmail_config=chatmail_config,
+def cmfactory(request, gencreds, tmpdir, maildomain, chatmail_config):
+    # cloned from deltachat.testplugin.amfactory
+    pytest.importorskip("deltachat")
+    from deltachat.testplugin import ACFactory
+
+    testproc = ChatmailTestProcess(
+        request.config, maildomain, gencreds, chatmail_config
     )
 
+    class Data:
+        def read_path(self, path):
+            return
+
+    am = ACFactory(request=request, tmpdir=tmpdir, testprocess=testproc, data=Data())
+
+    # Skip upstream's init_imap to prevent extra imap connections not
+    # needed for relay testing
+    am._acsetup.init_imap = lambda acc: None
+
+    # nb. a bit hacky
+    # would probably be better if deltachat's test machinery grows native support
+    def switch_maildomain(maildomain2):
+        am.testprocess.maildomain = maildomain2
+
+    am.switch_maildomain = switch_maildomain
+
+    yield am
+    if hasattr(request.node, "rep_call") and request.node.rep_call.failed:
+        if testproc.pytestconfig.getoption("--extra-info"):
+            logfile = io.StringIO()
+            am.dump_imap_summary(logfile=logfile)
+            print(logfile.getvalue())
+            # request.node.add_report_section("call", "imap-server-state", s)
+
 
 @pytest.fixture
 def remote(sshdomain):
-    r = Remote(sshdomain)
-    yield r
-    r.close()
+    return Remote(sshdomain)
 
 
 class Remote:
     def __init__(self, sshdomain):
         self.sshdomain = sshdomain
-        self._procs = []
 
-    def iter_output(self, logcmd="", ready=None):
+    def iter_output(self, logcmd=""):
         getjournal = "journalctl -f" if not logcmd else logcmd
-        print(self.sshdomain)
-        match self.sshdomain:
-            case "@local": command = []
-            case "localhost": command = []
-            case _: command = ["ssh", f"root@{self.sshdomain}"]
-        [command.append(arg) for arg in getjournal.split()]
-        popen = subprocess.Popen(
-            command,
-            stdin=subprocess.DEVNULL,
+        self.popen = subprocess.Popen(
+            ["ssh", f"root@{self.sshdomain}", getjournal],
             stdout=subprocess.PIPE,
-            stderr=subprocess.DEVNULL,
         )
-        self._procs.append(popen)
-        try:
-            while 1:
-                line = popen.stdout.readline()
-                res = line.decode().strip().lower()
-                if not res:
-                    break
-                if ready is not None:
-                    ready()
-                    ready = None
+        while 1:
+            line = self.popen.stdout.readline()
+            res = line.decode().strip().lower()
+            if res:
                 yield res
-        finally:
-            popen.terminate()
-            popen.wait()
-
-    def close(self):
-        while self._procs:
-            proc = self._procs.pop()
-            proc.kill()
-            proc.wait()
+            else:
+                break
 
 
 @pytest.fixture

cmdeploy/src/cmdeploy/tests/setup_tls_external.py

@@ -0,0 +1,362 @@
+"""Setup and verify external TLS certificates for a chatmail server.
+
+Generates a self-signed TLS certificate, uploads it to the chatmail
+server via SCP, runs ``cmdeploy run``, and then probes all TLS-enabled
+ports (nginx, postfix, dovecot) to verify the certificate is actually
+served.  After probing, checks remote service logs for errors.
+
+Prerequisites
+~~~~~~~~~~~~~
+- SSH root access to the target server (same as ``cmdeploy run``)
+- ``cmdeploy`` in PATH (activate the venv first)
+
+How to run
+~~~~~~~~~~
+From the repository root::
+
+    # Full run: generate cert, deploy, probe ports, check services
+    python -m cmdeploy.tests.setup_tls_external DOMAIN
+
+    # Re-probe only (after a previous deploy)
+    python -m cmdeploy.tests.setup_tls_external DOMAIN \\
+        --skip-deploy --skip-certgen
+
+    # Override SSH host (e.g. when domain doesn't resolve to the server)
+    python -m cmdeploy.tests.setup_tls_external DOMAIN \\
+        --ssh-host staging-ipv4.testrun.org
+
+Arguments
+~~~~~~~~~
+DOMAIN            mail domain for the chatmail server (SSH root login must work)
+
+Options
+~~~~~~~
+--skip-deploy     skip ``cmdeploy run``, only probe ports
+--skip-certgen    skip cert generation/upload, use certs already on server
+--ssh-host HOST   SSH host override (defaults to DOMAIN)
+"""
+
+import argparse
+import shutil
+import smtplib
+import socket
+import ssl
+import subprocess
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+# Cert paths on the remote server
+REMOTE_CERT = "/etc/ssl/certs/tmp_fullchain.pem"
+REMOTE_KEY = "/etc/ssl/private/tmp_privkey.pem"
+
+
+# ---------------------------------------------------------------------------
+# Config generation
+# ---------------------------------------------------------------------------
+
+
+def generate_config(domain: str, config_dir: Path) -> Path:
+    """Generate a chatmail.ini with tls_external_cert_and_key for *domain*."""
+    from chatmaild.config import write_initial_config
+
+    ini_path = config_dir / "chatmail.ini"
+    write_initial_config(
+        ini_path,
+        domain,
+        overrides={
+            "tls_external_cert_and_key": f"{REMOTE_CERT} {REMOTE_KEY}",
+        },
+    )
+    print(f"[+] Generated chatmail.ini for {domain} in {config_dir}")
+    return ini_path
+
+
+# ---------------------------------------------------------------------------
+# Certificate generation
+# ---------------------------------------------------------------------------
+
+
+def generate_cert(domain: str, cert_dir: Path) -> tuple:
+    """Generate a self-signed TLS cert+key for *domain* with proper SANs."""
+    from cmdeploy.selfsigned.deployer import openssl_selfsigned_args
+
+    cert_path = cert_dir / "fullchain.pem"
+    key_path = cert_dir / "privkey.pem"
+    subprocess.check_call(openssl_selfsigned_args(domain, cert_path, key_path, days=30))
+    print(f"[+] Generated cert for {domain} in {cert_dir}")
+    return cert_path, key_path
+
+
+# ---------------------------------------------------------------------------
+# Upload certs to remote server
+# ---------------------------------------------------------------------------
+
+
+def upload_certs(
+    ssh_host: str,
+    cert_path: Path,
+    key_path: Path,
+) -> None:
+    """SCP cert and key to the remote server."""
+    subprocess.check_call([
+        "scp", str(cert_path), f"root@{ssh_host}:{REMOTE_CERT}",
+    ])
+    subprocess.check_call([
+        "scp", str(key_path), f"root@{ssh_host}:{REMOTE_KEY}",
+    ])
+    # Ensure cert is world-readable and key is readable by ssl-cert group
+    # (dovecot/postfix/nginx need to read these files)
+    subprocess.check_call([
+        "ssh", f"root@{ssh_host}",
+        f"chmod 644 {REMOTE_CERT} && chmod 640 {REMOTE_KEY}"
+        f" && chgrp ssl-cert {REMOTE_KEY}",
+    ])
+    print(f"[+] Uploaded cert/key to {ssh_host}")
+
+
+# ---------------------------------------------------------------------------
+# Deploy
+# ---------------------------------------------------------------------------
+
+
+def run_deploy(ini_path: str) -> None:
+    """Run ``cmdeploy run --skip-dns-check --config <ini>``."""
+    cmd = ["cmdeploy", "run", "--config", str(ini_path), "--skip-dns-check"]
+    print(f"[+] Running: {' '.join(cmd)}")
+    subprocess.check_call(cmd)
+    print("[+] Deploy completed successfully")
+
+
+# ---------------------------------------------------------------------------
+# TLS port probing
+# ---------------------------------------------------------------------------
+
+
+def get_peer_cert_binary(host: str, port: int) -> bytes:
+    """Connect to host:port with TLS and return the DER-encoded peer cert."""
+    ctx = ssl.create_default_context()
+    ctx.check_hostname = False
+    ctx.verify_mode = ssl.CERT_NONE
+    with socket.create_connection((host, port), timeout=15) as sock:
+        with ctx.wrap_socket(sock, server_hostname=host) as ssock:
+            return ssock.getpeercert(binary_form=True)
+
+
+def get_smtp_starttls_cert_binary(host: str, port: int = 587) -> bytes:
+    """Connect via SMTP STARTTLS and return the DER cert."""
+    ctx = ssl.create_default_context()
+    ctx.check_hostname = False
+    ctx.verify_mode = ssl.CERT_NONE
+    with smtplib.SMTP(host, port, timeout=15) as smtp:
+        smtp.starttls(context=ctx)
+        return smtp.sock.getpeercert(binary_form=True)
+
+
+def check_cert_matches(
+    label: str, served_der: bytes, expected_der: bytes,
+) -> bool:
+    """Compare served DER cert against the expected cert."""
+    if served_der == expected_der:
+        print(f"  [OK]   {label}: certificate matches")
+        return True
+    else:
+        print(f"  [FAIL] {label}: certificate does NOT match")
+        return False
+
+
+def load_cert_der(cert_pem_path: Path) -> bytes:
+    """Load a PEM cert file and return its DER encoding."""
+    pem_text = cert_pem_path.read_text()
+    start = pem_text.index("-----BEGIN CERTIFICATE-----")
+    end = pem_text.index("-----END CERTIFICATE-----") + len(
+        "-----END CERTIFICATE-----"
+    )
+    return ssl.PEM_cert_to_DER_cert(pem_text[start:end])
+
+
+def probe_all_ports(host: str, expected_cert_der: bytes) -> bool:
+    """Probe TLS ports and verify the served certificate matches.
+
+    Checks ports 993 (IMAP), 465 (SMTPS), 587 (STARTTLS), and 443
+    (nginx stream).  Port 8443 is skipped as nginx binds it to
+    localhost behind the stream proxy on 443.
+    """
+    print(f"\n[+] Probing TLS ports on {host}...")
+    all_ok = True
+
+    for label, port in [
+        ("IMAP/TLS (993)", 993),
+        ("SMTP/TLS (465)", 465),
+    ]:
+        try:
+            served = get_peer_cert_binary(host, port)
+            if not check_cert_matches(label, served, expected_cert_der):
+                all_ok = False
+        except Exception as e:
+            print(f"  [FAIL] {label}: connection failed: {e}")
+            all_ok = False
+
+    # STARTTLS on port 587
+    try:
+        served = get_smtp_starttls_cert_binary(host, 587)
+        if not check_cert_matches("SMTP/STARTTLS (587)", served, expected_cert_der):
+            all_ok = False
+    except Exception as e:
+        print(f"  [FAIL] SMTP/STARTTLS (587): connection failed: {e}")
+        all_ok = False
+
+    # Port 443 (nginx stream proxy with ALPN routing)
+    try:
+        served = get_peer_cert_binary(host, 443)
+        if not check_cert_matches("nginx/443 (stream)", served, expected_cert_der):
+            all_ok = False
+    except Exception as e:
+        print(f"  [FAIL] nginx/443 (stream): connection failed: {e}")
+        all_ok = False
+
+    return all_ok
+
+
+# ---------------------------------------------------------------------------
+# Post-deploy service health checks
+# ---------------------------------------------------------------------------
+
+SERVICES = ["dovecot", "postfix", "nginx"]
+
+
+def check_remote_services(ssh_host: str, since: str = "") -> bool:
+    """SSH to the server and check for service failures or errors.
+
+    *since* is a ``journalctl --since`` timestamp (e.g. ``"5 min ago"``).
+    If empty, checks the entire boot journal.
+    """
+    print(f"\n[+] Checking remote service health on {ssh_host}...")
+    all_ok = True
+
+    for svc in SERVICES:
+        try:
+            result = subprocess.run(
+                ["ssh", f"root@{ssh_host}",
+                 f"systemctl is-active {svc}.service"],
+                capture_output=True, text=True, timeout=15, check=False,
+            )
+            status = result.stdout.strip()
+            if status == "active":
+                print(f"  [OK]   {svc}: active")
+            else:
+                print(f"  [FAIL] {svc}: {status}")
+                all_ok = False
+        except Exception as e:
+            print(f"  [FAIL] {svc}: check failed: {e}")
+            all_ok = False
+
+    since_arg = f'--since="{since}"' if since else ""
+    print(f"\n[+] Checking journal for errors on {ssh_host}...")
+    for svc in SERVICES:
+        try:
+            result = subprocess.run(
+                ["ssh", f"root@{ssh_host}",
+                 f"journalctl -u {svc}.service {since_arg}"
+                 f" --no-pager -p err -q"],
+                capture_output=True, text=True, timeout=15, check=False,
+            )
+            errors = result.stdout.strip()
+            if errors:
+                print(f"  [WARN] {svc} errors in journal:")
+                for line in errors.splitlines()[:10]:
+                    print(f"         {line}")
+                all_ok = False
+            else:
+                print(f"  [OK]   {svc}: no errors in journal")
+        except Exception as e:
+            print(f"  [FAIL] {svc}: journal check failed: {e}")
+            all_ok = False
+
+    return all_ok
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "domain",
+        help="mail domain (SSH root login must work to this host)",
+    )
+    parser.add_argument(
+        "--skip-deploy",
+        action="store_true",
+        help="skip cmdeploy run, only probe ports",
+    )
+    parser.add_argument(
+        "--skip-certgen",
+        action="store_true",
+        help="skip cert generation and upload (use existing)",
+    )
+    parser.add_argument(
+        "--ssh-host",
+        help="SSH host override (defaults to DOMAIN)",
+    )
+    args = parser.parse_args()
+
+    domain = args.domain
+    ssh_host = args.ssh_host or domain
+    print(f"[+] Domain: {domain}")
+    print(f"[+] SSH host: {ssh_host}")
+    print(f"[+] Remote cert: {REMOTE_CERT}")
+    print(f"[+] Remote key:  {REMOTE_KEY}")
+
+    work_dir = Path(tempfile.mkdtemp(prefix="tls-external-test-"))
+    try:
+        # Generate chatmail.ini
+        ini_path = generate_config(domain, work_dir)
+
+        if not args.skip_certgen:
+            local_cert, local_key = generate_cert(domain, work_dir)
+            upload_certs(ssh_host, local_cert, local_key)
+        else:
+            local_cert = work_dir / "fullchain.pem"
+            subprocess.check_call([
+                "scp", f"root@{ssh_host}:{REMOTE_CERT}", str(local_cert),
+            ])
+
+        # Record timestamp before deploy for journal filtering
+        deploy_start = time.strftime("%Y-%m-%d %H:%M:%S")
+
+        if not args.skip_deploy:
+            run_deploy(ini_path)
+
+        # Probe TLS ports
+        expected_der = load_cert_der(local_cert)
+        ports_ok = probe_all_ports(domain, expected_der)
+
+        # Check service health (only errors since deploy started)
+        services_ok = check_remote_services(ssh_host, since=deploy_start)
+
+        if ports_ok and services_ok:
+            print(
+                "\n[SUCCESS] All TLS port probes passed and services are healthy"
+            )
+            return 0
+        else:
+            if not ports_ok:
+                print("\n[FAILURE] Some TLS port probes failed", file=sys.stderr)
+            if not services_ok:
+                print(
+                    "\n[FAILURE] Some services have errors", file=sys.stderr
+                )
+            return 1
+    finally:
+        shutil.rmtree(work_dir, ignore_errors=True)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

cmdeploy/src/cmdeploy/tests/test_cmdeploy.py

@@ -23,19 +23,15 @@ class TestCmdline:
         run = parser.parse_args(["run"])
         assert init and run
 
-    def test_init_not_overwrite(self, capsys, tmp_path, monkeypatch):
-        monkeypatch.delenv("CHATMAIL_INI", raising=False)
-        inipath = tmp_path / "chatmail.ini"
-        args = ["init", "--config", str(inipath), "chat.example.org"]
-        assert main(args) == 0
+    def test_init_not_overwrite(self, capsys):
+        assert main(["init", "chat.example.org"]) == 0
         capsys.readouterr()
 
-        assert main(args) == 1
+        assert main(["init", "chat.example.org"]) == 1
         out, err = capsys.readouterr()
         assert "path exists" in out.lower()
 
-        args.insert(1, "--force")
-        assert main(args) == 0
+        assert main(["init", "chat.example.org", "--force"]) == 0
         out, err = capsys.readouterr()
         assert "deleting config file" in out.lower()
 

cmdeploy/src/cmdeploy/tests/test_dns.py

@@ -3,7 +3,7 @@ from copy import deepcopy
 import pytest
 
 from cmdeploy import remote
-from cmdeploy.dns import check_full_zone, check_initial_remote_data, parse_zone_records
+from cmdeploy.dns import check_full_zone, check_initial_remote_data
 
 
 @pytest.fixture
@@ -60,29 +60,6 @@ def mockdns(request, mockdns_base, mockdns_expected):
     return mockdns_base
 
 
-class TestGetDkimEntry:
-    def test_dkim_entry_returns_tuple_on_success(self, mockdns):
-        entry, web_entry = remote.rdns.get_dkim_entry(
-            "some.domain", "", dkim_selector="opendkim"
-        )
-        # May return None,None if openssl not available, but should never crash
-        if entry is not None:
-            assert "opendkim._domainkey.some.domain" in entry
-            assert "opendkim._domainkey.some.domain" in web_entry
-
-    def test_dkim_entry_returns_none_tuple_on_error(self, monkeypatch):
-        """CalledProcessError must return (None, None), not bare None."""
-        from subprocess import CalledProcessError
-
-        def failing_shell(command, fail_ok=False, print=print):
-            raise CalledProcessError(1, command)
-
-        monkeypatch.setattr(remote.rdns, "shell", failing_shell)
-        result = remote.rdns.get_dkim_entry("some.domain", "", dkim_selector="opendkim")
-        assert result == (None, None)
-        assert result[0] is None and result[1] is None
-
-
 class TestPerformInitialChecks:
     def test_perform_initial_checks_ok1(self, mockdns, mockdns_expected):
         remote_data = remote.rdns.perform_initial_checks("some.domain")
@@ -125,49 +102,18 @@ class TestPerformInitialChecks:
         assert not l
 
 
-def test_parse_zone_records():
-    text = """
-    ; This is a comment
-    some.domain. 3600 IN A 1.1.1.1
-
-    ; Another comment
-    www.some.domain. 3600 IN CNAME some.domain.
-
-    ; Multi-word rdata
-    some.domain. 3600 IN MX 10 mail.some.domain.
-
-    ; DKIM record (single line, multi-word TXT rdata)
-    dkim._domainkey.some.domain. 3600 IN TXT "v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG" "9w0BAQEFAAOCAQ8AMIIBCgKCAQEA"
-
-    ; Another TXT record
-    _dmarc.some.domain. 3600 IN TXT "v=DMARC1;p=reject"
-    """
-    records = list(parse_zone_records(text))
-    assert records == [
-        ("some.domain", "3600", "A", "1.1.1.1"),
-        ("www.some.domain", "3600", "CNAME", "some.domain."),
-        ("some.domain", "3600", "MX", "10 mail.some.domain."),
-        (
-            "dkim._domainkey.some.domain",
-            "3600",
-            "TXT",
-            '"v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG" "9w0BAQEFAAOCAQ8AMIIBCgKCAQEA"',
-        ),
-        ("_dmarc.some.domain", "3600", "TXT", '"v=DMARC1;p=reject"'),
-    ]
-
-
-def test_parse_zone_records_invalid_line():
-    text = "invalid line"
-    with pytest.raises(ValueError, match="Bad zone record line"):
-        list(parse_zone_records(text))
-
-
 def parse_zonefile_into_dict(zonefile, mockdns_base, only_required=False):
-    if only_required:
-        zonefile = zonefile.split("; Recommended")[0]
-    for name, ttl, rtype, rdata in parse_zone_records(zonefile):
-        mockdns_base.setdefault(rtype, {})[name] = rdata
+    for zf_line in zonefile.split("\n"):
+        if zf_line.startswith("#"):
+            if "Recommended" in zf_line and only_required:
+                return
+            continue
+        if not zf_line.strip():
+            continue
+        zf_domain, zf_typ, zf_value = zf_line.split(maxsplit=2)
+        zf_domain = zf_domain.rstrip(".")
+        zf_value = zf_value.strip()
+        mockdns_base.setdefault(zf_typ, {})[zf_domain] = zf_value
 
 
 class MockSSHExec:

cmdeploy/src/cmdeploy/tests/test_dovecot_deployer.py

@@ -1,238 +0,0 @@
-from contextlib import nullcontext
-from types import SimpleNamespace
-
-import pytest
-from pyinfra.facts.deb import DebPackages
-
-from cmdeploy.dovecot import deployer as dovecot_deployer
-
-
-def make_host(*fact_pairs):
-    """Build a mock host; get_fact(cls) dispatches to the provided facts mapping.
-
-    Args:
-        *fact_pairs: tuples of (fact_class, fact_value) to register
-
-    Returns:
-        SimpleNamespace with get_fact that raises a clear error if an
-        unexpected fact type is requested.
-    """
-    facts = dict(fact_pairs)
-
-    def get_fact(cls):
-        if cls not in facts:
-            registered = ", ".join(c.__name__ for c in facts)
-            raise LookupError(
-                f"unexpected get_fact({cls.__name__}); "
-                f"only registered: {registered}"
-            )
-        return facts[cls]
-
-    return SimpleNamespace(get_fact=get_fact)
-
-
-@pytest.fixture
-def deployer():
-    return dovecot_deployer.DovecotDeployer(
-        SimpleNamespace(mail_domain="chat.example.org"),
-        disable_mail=False,
-    )
-
-
-@pytest.fixture
-def patch_blocked(monkeypatch):
-    monkeypatch.setattr(dovecot_deployer, "blocked_service_startup", nullcontext)
-
-
-@pytest.fixture
-def mock_files_put(monkeypatch):
-    monkeypatch.setattr(
-        dovecot_deployer.files,
-        "put",
-        lambda **kwargs: SimpleNamespace(changed=False),
-    )
-
-
-@pytest.fixture
-def track_shell(monkeypatch):
-    calls = []
-    monkeypatch.setattr(
-        dovecot_deployer.server,
-        "shell",
-        lambda **kwargs: calls.append(kwargs) or SimpleNamespace(changed=False),
-    )
-    return calls
-
-
-def test_download_dovecot_package_skips_epoch_matched_install(monkeypatch):
-    epoch_version = dovecot_deployer.DOVECOT_PACKAGE_VERSION
-    downloads = []
-    monkeypatch.setattr(
-        dovecot_deployer,
-        "host",
-        make_host((DebPackages, {"dovecot-core": [epoch_version]})),
-    )
-    monkeypatch.setattr(
-        dovecot_deployer,
-        "_pick_url",
-        lambda primary, fallback: primary,
-    )
-    monkeypatch.setattr(
-        dovecot_deployer.files,
-        "download",
-        lambda **kwargs: downloads.append(kwargs),
-    )
-
-    deb, changed = dovecot_deployer._download_dovecot_package("core", "amd64")
-
-    assert deb is None, f"expected no deb path when version matches, got {deb!r}"
-    assert changed is False, "should not flag changed when version already installed"
-    assert downloads == [], "should not download when version already installed"
-
-
-def test_download_dovecot_package_uses_archive_version_for_url_and_filename(
-    monkeypatch,
-):
-    downloads = []
-    monkeypatch.setattr(
-        dovecot_deployer,
-        "host",
-        make_host((DebPackages, {})),
-    )
-    monkeypatch.setattr(
-        dovecot_deployer,
-        "_pick_url",
-        lambda primary, fallback: primary,
-    )
-    monkeypatch.setattr(
-        dovecot_deployer.files,
-        "download",
-        lambda **kwargs: downloads.append(kwargs),
-    )
-
-    deb, changed = dovecot_deployer._download_dovecot_package("core", "amd64")
-
-    archive_version = dovecot_deployer.DOVECOT_ARCHIVE_VERSION.replace("+", "%2B")
-    expected_deb = f"/root/dovecot-core_{archive_version}_amd64.deb"
-
-    # Verify the returned path uses archive version, not package version (with epoch)
-    assert changed is True, "should flag changed when package not yet installed"
-    assert deb == expected_deb, f"deb path mismatch: {deb!r} != {expected_deb!r}"
-    assert dovecot_deployer.DOVECOT_PACKAGE_VERSION not in deb, (
-        f"deb path should use archive version (no epoch), got {deb!r}"
-    )
-    assert len(downloads) == 1, "files.download should be called exactly once"
-
-
-def test_install_skips_dpkg_path_when_epoch_matched_packages_present(
-    deployer, patch_blocked, mock_files_put, track_shell, monkeypatch
-):
-    monkeypatch.setattr(
-        dovecot_deployer,
-        "host",
-        make_host(
-            (
-                dovecot_deployer.DebPackages,
-                {
-                    "dovecot-core": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
-                    "dovecot-imapd": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
-                    "dovecot-lmtpd": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
-                },
-            ),
-            (dovecot_deployer.Arch, "x86_64"),
-        ),
-    )
-    downloads = []
-    monkeypatch.setattr(
-        dovecot_deployer.files,
-        "download",
-        lambda **kwargs: downloads.append(kwargs),
-    )
-
-    deployer.install()
-
-    assert downloads == [], "should not download when all packages epoch-matched"
-    assert track_shell == [], "should not run dpkg when all packages epoch-matched"
-    assert deployer.need_restart is False, (
-        "need_restart should be False when nothing changed"
-    )
-
-
-def test_install_unsupported_arch_falls_back_to_apt(
-    deployer, patch_blocked, mock_files_put, track_shell, monkeypatch
-):
-    # For unsupported architectures, all fact lookups return the arch string.
-    monkeypatch.setattr(
-        dovecot_deployer,
-        "host",
-        SimpleNamespace(get_fact=lambda cls: "riscv64"),
-    )
-    apt_calls = []
-
-    # Mirrors apt.packages() return value: OperationMeta with .changed property.
-    # Only lmtpd triggers a change to verify |= accumulation of changed flags.
-    def fake_apt(**kwargs):
-        apt_calls.append(kwargs)
-        changed = "lmtpd" in kwargs["packages"][0]
-        return SimpleNamespace(changed=changed)
-
-    monkeypatch.setattr(dovecot_deployer.apt, "packages", fake_apt)
-
-    deployer.install()
-
-    actual_pkgs = [c["packages"] for c in apt_calls]
-    assert actual_pkgs == [["dovecot-core"], ["dovecot-imapd"], ["dovecot-lmtpd"]], (
-        f"expected apt install of core/imapd/lmtpd, got {actual_pkgs}"
-    )
-    assert track_shell == [], "should not run dpkg for unsupported arch"
-    assert deployer.need_restart is True, (
-        "need_restart should be True when apt installed a package"
-    )
-
-
-def test_install_runs_dpkg_when_packages_need_download(
-    deployer, patch_blocked, mock_files_put, track_shell, monkeypatch
-):
-    monkeypatch.setattr(
-        dovecot_deployer,
-        "host",
-        make_host(
-            (dovecot_deployer.DebPackages, {}),
-            (dovecot_deployer.Arch, "x86_64"),
-        ),
-    )
-    monkeypatch.setattr(
-        dovecot_deployer,
-        "_pick_url",
-        lambda primary, fallback: primary,
-    )
-    monkeypatch.setattr(
-        dovecot_deployer.files,
-        "download",
-        lambda **kwargs: SimpleNamespace(changed=True),
-    )
-
-    deployer.install()
-
-    assert len(track_shell) == 1, (
-        f"expected one server.shell() call for dpkg install, got {len(track_shell)}"
-    )
-    cmds = track_shell[0]["commands"]
-    assert len(cmds) == 3, f"expected 3 dpkg/apt commands, got: {cmds}"
-    assert cmds[0].startswith("dpkg --force-confdef --force-confold -i ")
-    assert "apt-get -y --fix-broken install" in cmds[1]
-    assert cmds[2].startswith("dpkg --force-confdef --force-confold -i ")
-    assert deployer.need_restart is True, (
-        "need_restart should be True after dpkg install"
-    )
-
-
-def test_pick_url_falls_back_on_primary_error(monkeypatch):
-    def raise_error(req, timeout):
-        raise OSError("connection timeout")
-
-    monkeypatch.setattr(dovecot_deployer.urllib.request, "urlopen", raise_error)
-    result = dovecot_deployer._pick_url("http://primary", "http://fallback")
-    assert result == "http://fallback", (
-        f"should fall back when primary fails, got {result!r}"
-    )

cmdeploy/src/cmdeploy/tests/test_rshell.py

@@ -1,68 +0,0 @@
-from unittest.mock import patch
-
-from cmdeploy.remote.rshell import dovecot_recalc_quota
-
-
-def test_dovecot_recalc_quota_normal_output():
-    """Normal doveadm output returns parsed dict."""
-    normal_output = (
-        "Quota name Type    Value  Limit  %\n"
-        "User quota STORAGE     5 102400  0\n"
-        "User quota MESSAGE     2      -  0\n"
-    )
-
-    with patch("cmdeploy.remote.rshell.shell", return_value=normal_output):
-        result = dovecot_recalc_quota("user@example.org")
-
-    # shell is called twice (recalc + get), patch returns same for both
-    assert result == {"value": 5, "limit": 102400, "percent": 0}
-
-
-def test_dovecot_recalc_quota_empty_output():
-    """Empty doveadm output (trailing newline) must not IndexError."""
-    call_count = [0]
-
-    def mock_shell(cmd):
-        call_count[0] += 1
-        if "recalc" in cmd:
-            return ""
-        # quota get returns only empty lines
-        return "\n\n"
-
-    with patch("cmdeploy.remote.rshell.shell", side_effect=mock_shell):
-        result = dovecot_recalc_quota("user@example.org")
-
-    assert result is None
-
-
-def test_dovecot_recalc_quota_malformed_output():
-    """Malformed output with too few columns must not crash."""
-    call_count = [0]
-
-    def mock_shell(cmd):
-        call_count[0] += 1
-        if "recalc" in cmd:
-            return ""
-        # partial line, fewer than 6 parts
-        return "Quota name\nUser quota STORAGE\n"
-
-    with patch("cmdeploy.remote.rshell.shell", side_effect=mock_shell):
-        result = dovecot_recalc_quota("user@example.org")
-
-    assert result is None
-
-
-def test_dovecot_recalc_quota_header_only():
-    """Only header line, no data rows."""
-    call_count = [0]
-
-    def mock_shell(cmd):
-        call_count[0] += 1
-        if "recalc" in cmd:
-            return ""
-        return "Quota name Type    Value  Limit  %\n"
-
-    with patch("cmdeploy.remote.rshell.shell", side_effect=mock_shell):
-        result = dovecot_recalc_quota("user@example.org")
-
-    assert result is None

doc/source/docker.rst

@@ -0,0 +1,260 @@
+Docker installation
+===================
+
+This section provides instructions for installing a chatmail relay
+using Docker Compose.
+
+.. note::
+
+    - Docker support is experimental and not yet covered by automated tests, please report bugs.
+    - This preliminary image simply wraps the cmdeploy process detailed in the :doc:`getting_started` instructions in a full Debian-systemd image with r/w access to `/sys/fs`
+    - Currently, the image has only been tested and built on amd64, though arm64 should theoretically work as well.
+
+
+Setup Preparation
+-----------------
+
+We use ``chat.example.org`` as the chatmail domain in the following
+steps. Please substitute it with your own domain.
+
+1. Install docker and docker compose v2 (check with `docker compose version`), install, e.g., through
+    - Debian 12 through the `official install instructions <https://docs.docker.com/engine/install/debian/#install-using-the-repository>`_
+    - Debian 13+ with `apt install docker docker-compose`
+
+   If you must use v1 (EOL since 2023), use `docker-compose` in the following and modify the `docker-compose.yaml` to use `privileged: true` instead of `cgroup: host`, though that will run give the container all priviledges.
+
+2. Setup the initial DNS records.
+   The following is an example in the familiar BIND zone file format with
+   a TTL of 1 hour (3600 seconds).
+   Please substitute your domain and IP addresses.
+
+   ::
+
+       chat.example.org. 3600 IN A 198.51.100.5
+       chat.example.org. 3600 IN AAAA 2001:db8::5
+       www.chat.example.org. 3600 IN CNAME chat.example.org.
+       mta-sts.chat.example.org. 3600 IN CNAME chat.example.org.
+
+3. Configure kernel parameters on the host, as these can not be set from the container::
+
+       echo "fs.inotify.max_user_instances=65536" | sudo tee -a /etc/sysctl.d/99-inotify.conf
+       echo "fs.inotify.max_user_watches=65536" | sudo tee -a /etc/sysctl.d/99-inotify.conf
+       sudo sysctl --system
+
+
+Docker Compose Setup
+--------------------
+
+Pre-built images are available from GitHub Container Registry. The
+``main`` branch and tagged releases are pushed automatically by CI::
+
+    docker pull ghcr.io/chatmail/relay:main      # latest main branch
+    docker pull ghcr.io/chatmail/relay:1.2.3     # tagged release
+
+
+Create service directory
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+Either:
+
+- Create a service directory, e.g., `/srv/chatmail-relay`::
+
+    mkdir -p /srv/chatmail-relay && cd /srv/chatmail-relay
+    wget https://raw.githubusercontent.com/chatmail/relay/refs/heads/main/docker-compose.yaml
+    wget https://raw.githubusercontent.com/chatmail/relay/refs/heads/main/docker-compose.override.yaml.example -O docker-compose.override.yaml
+
+- or clone the chatmail repo ::
+
+   git clone https://github.com/chatmail/relay
+   cd relay
+
+
+Customize and start
+^^^^^^^^^^^^^^^^^^^
+
+1. Set the fully qualified domain name of the relay::
+
+       echo 'MAIL_DOMAIN=chat.example.org' > .env
+
+   The container generates a ``chatmail.ini`` with defaults from
+   ``MAIL_DOMAIN`` on first start. To customize chatmail settings, mount
+   your own ``chatmail.ini`` instead (see `Custom chatmail.ini`_ below).
+
+2. All local customizations (data paths, extra volumes, config mounts) go in
+   ``docker-compose.override.yaml``, which Compose merges automatically with
+   the base file. By default, all data is stored in docker volumes, you will
+   likely want to at least create and configure the mail storage location, but
+   you might also want to configure external TLS certificates there.
+
+3. Start the container::
+
+       docker compose up -d
+       docker compose logs -f chatmail   # view logs, Ctrl+C to exit
+
+4. After installation is complete, open ``https://chat.example.org`` in
+   your browser.
+
+Finish install and test
+-----------------------
+
+You can test the installation with::
+
+        pip install cmping chat.example.org # or
+        uvx cmping chat.example.org # if you use https://docs.astral.sh/uv/
+
+You should check and extend your DNS records for better interoperability::
+
+    # Show required DNS records
+    docker exec chatmail /opt/cmdeploy/bin/cmdeploy dns --ssh-host @local
+
+You can check server status with::
+
+    docker exec chatmail /opt/cmdeploy/bin/cmdeploy status --ssh-host @local
+
+You can run some benchmarks (can also run from any machine with cmdeploy installed)
+
+    docker exec chatmail /opt/cmdeploy/bin/cmdeploy bench chat.example.org
+
+You can run the test suite with
+
+    docker exec chatmail /opt/cmdeploy/bin/cmdeploy test chat.example.org --ssh-host localhost
+
+
+Customization
+-------------
+
+Website
+^^^^^^^^^^^^^^
+
+You can customize the chatmail landing page by mounting a directory with
+your own website source files.
+
+1. Create a directory with your custom website source::
+
+       mkdir -p ./custom/www/src
+       nano ./custom/www/src/index.md
+
+2. Add the volume mount in ``docker-compose.override.yaml``::
+
+       services:
+         chatmail:
+           volumes:
+             - ./custom/www:/opt/chatmail-www
+
+3. Restart the service::
+
+       docker compose down
+       docker compose up -d
+
+
+Custom chatmail.ini
+^^^^^^^^^^^^^^^^^^^
+
+If you want to go beyond simply setting the ``MAIL_DOMAIN`` in ``.env``, you
+can use a regular `chatmail.ini` to give you full control.
+
+1. Extract the generated config from a running container::
+
+       docker cp chatmail:/etc/chatmail/chatmail.ini ./chatmail.ini
+
+2. Edit ``chatmail.ini`` as needed.
+
+3. Add the volume mount in ``docker-compose.override.yaml`` ::
+
+       services:
+         chatmail:
+           volumes:
+             - ./chatmail.ini:/etc/chatmail/chatmail.ini
+
+4. Restart the container, the container skips generating a new one: ::
+
+       docker compose down && docker compose up -d
+
+
+External TLS certificates
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If TLS certificates are managed outside the container (e.g. by certbot,
+acmetool, or Traefik on the host), mount them into the container and set
+``TLS_EXTERNAL_CERT_AND_KEY`` in ``docker-compose.override.yaml``.
+Changed certificates are picked up automatically via inotify.
+See the examples in the example override and :ref:`external-tls` in the getting started guide for details.
+
+
+Migrating from a bare-metal install
+------------------------------------
+
+If you have an existing bare-metal chatmail installation and want to
+switch to Docker:
+
+1. Stop all existing services::
+
+       systemctl stop postfix dovecot doveauth nginx opendkim unbound \
+         acmetool-redirector filtermail filtermail-incoming chatmail-turn \
+         iroh-relay chatmail-metadata lastlogin mtail
+       systemctl disable postfix dovecot doveauth nginx opendkim unbound \
+         acmetool-redirector filtermail filtermail-incoming chatmail-turn \
+         iroh-relay chatmail-metadata lastlogin mtail
+
+2. Copy your existing ``chatmail.ini`` and mount it into the container
+   (see `Custom chatmail.ini`_ above)::
+
+       cp /usr/local/lib/chatmaild/chatmail.ini ./chatmail.ini
+
+3. Copy persistent data into the ``./data/`` subdirectories (for example, as configured in `Customize and start`_) ::
+
+       mkdir -p data/dkim data/certs data/mail
+
+       # DKIM keys
+       cp -a /etc/dkimkeys/* data/dkim/
+
+       # TLS certificates
+       rsync -a /var/lib/acme/ data/certs/
+
+   Note that ownership of dkim and acme is adjusted on container start.
+
+   For the mail directory::
+
+       rsync -a /home/vmail/ data/mail/
+
+   Alternatively, mount ``/home/vmail`` directly by changing the volume
+   in ``docker-compose-override.yaml``::
+
+       - /home/vmail:/home/vmail
+
+   The three ``./data/`` subdirectories cover all persistent state.
+   Everything else is regenerated by the ``configure`` and ``activate``
+   stages on container start.
+
+Building the image
+------------------
+
+Clone the repository and build the Docker image::
+
+    git clone https://github.com/chatmail/relay
+    cd relay
+    docker compose build chatmail
+
+The build bakes all binaries, Python packages, and the install stage
+into the image. After building, only ``docker-compose.yaml`` and a ``.env`` with
+``MAIL_DOMAIN`` are needed to run the container.
+
+You can transfer a locally built image to your server directly (pigz is parallel `gzip` which can be used instead as well) ::
+
+    docker save chatmail-relay:latest | pigz | ssh chat.example.org 'pigz -d | docker load'
+
+
+Forcing a full reinstall
+------------------------
+
+On container start, only the ``configure`` and ``activate`` stages run by default.
+
+To force a full reinstall (e.g. after updating the source), either
+rebuild the image::
+
+    docker compose build chatmail
+    docker compose up -d
+
+Or override the stages at runtime without rebuilding::
+
+    CMDEPLOY_STAGES="install,configure,activate" docker compose up -d

doc/source/getting_started.rst

@@ -98,6 +98,12 @@ steps. Please substitute it with your own domain.
    configure at your DNS provider (it can take some time until they are
    public).
 
+Docker installation
+-------------------
+
+There is experimental support for running chatmail via Docker Compose.
+See :doc:`docker` for full setup instructions.
+
 Other helpful commands
 ----------------------
 
@@ -229,11 +235,7 @@ The deploy will verify that both files exist on the server.
    You are responsible for certificate renewal.
    When the certificate file changes on disk,
    all relay services pick up the new certificate automatically
-   via a systemd path watcher installed during deploy.
-   The watcher uses inotify, which does not cross bind-mount boundaries.
-   If you use such a setup, you must trigger the reload explicitly after renewal::
-
-      systemctl start tls-cert-reload.service
+   (via a systemd path watcher installed during deploy).
 
 
 Migrating to a new build machine

doc/source/index.rst

@@ -13,6 +13,7 @@ Contributions and feedback welcome through the https://github.com/chatmail/relay
     :maxdepth: 5
 
     getting_started
+    docker
     proxy
     migrate
     overview

doc/source/overview.rst

@@ -102,17 +102,17 @@ short overview of ``chatmaild`` services:
    Apple/Google/Huawei.
 
 -  `chatmail-expire <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/expire.py>`_
-   deletes old messages, large messages, and entire mailboxes
-   of users who have not logged in for longer than
-   ``delete_inactive_users_after`` days.
-
--  ``chatmail-quota-expire`` is called by Dovecot's ``quota_warning`` mechanism
-   and will automatically remove oldest messages to keep mailboxes well under ``max_mailbox_size``.
+   deletes users if they have not logged in for a longer while.
+   The timeframe can be configured in ``chatmail.ini``.
 
 -  `lastlogin <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/lastlogin.py>`_
    is contacted by Dovecot when a user logs in and stores the date of
    the login.
 
+-  `metrics <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/metrics.py>`_
+   collects some metrics and displays them at
+   ``https://example.org/metrics``.
+
 ``www/``
 ~~~~~~~~~
 
@@ -142,9 +142,11 @@ Chatmail relay dependency diagram
         nginx-internal --- autoconfig.xml;
         certs-nginx[("`TLS certs
         /var/lib/acme`")] --> nginx-internal;
+        systemd-timer --- chatmail-metrics;
         systemd-timer --- acmetool;
         systemd-timer --- chatmail-expire-daily;
         systemd-timer --- chatmail-fsreport-daily;
+        chatmail-metrics --- website;
         acmetool --> certs[("`TLS certs
         /var/lib/acme`")];
         nginx-external --- |993|dovecot;
@@ -160,8 +162,6 @@ Chatmail relay dependency diagram
         /home/vmail/.../user"];
         dovecot --- |lastlogin.socket|lastlogin;
         dovecot --- chatmail-metadata;
-        dovecot --- |quota-warning|chatmail-quota-expire;
-        chatmail-quota-expire --- maildir;
         lastlogin --- maildir;
         doveauth --- maildir;
         chatmail-expire-daily --- maildir;

docker-compose.override.yaml.example

@@ -0,0 +1,40 @@
+# Local overrides — copy to docker-compose.override.yaml in the repo root.
+# Compose automatically merges this with docker-compose.yaml.
+#
+#   cp docker-compose.override.yaml.example docker-compose.override.yaml
+#
+# Volumes are APPENDED to the base file's volumes list.
+# Environment and other scalar keys are MERGED by key.
+services:
+  chatmail:
+    volumes:
+      ## Data paths — bind-mount to host directories for easy access/backup.
+
+      # - ./data/dkim:/etc/dkimkeys
+      # - ./data/certs:/var/lib/acme
+
+      # - ./data/mail:/home/vmail
+      ## Or mount from an existing bare-metal install.
+      # - /home/vmail:/home/vmail
+
+      ## Mount your own chatmail.ini (skips auto-generation):
+      # - ./chatmail.ini:/etc/chatmail/chatmail.ini
+
+      ## Custom website:
+      # - ./custom/www:/opt/chatmail-www
+
+      ## Debug — mount scripts from the repo for live editing:
+      # - ./docker/files/setup_chatmail_docker.sh:/setup_chatmail_docker.sh
+      # - ./docker/files/entrypoint.sh:/entrypoint.sh
+
+    # environment:
+      ## Mount certs (above) and set TLS_EXTERNAL_CERT_AND_KEY to in-container paths.
+      ## Changed certs are picked up automatically (inotify via tls-cert-reload.path).
+      ##
+      ## Host acmetool (bare-metal migration): create mount above, and
+      ##   rsync -a /var/lib/acme/live data/certs
+      # TLS_EXTERNAL_CERT_AND_KEY: "/var/lib/acme/live/${MAIL_DOMAIN}/fullchain /var/lib/acme/live/${MAIL_DOMAIN}/privkey"
+      ##
+      ## (Untested) Traefik certs-dumper (see docker/docker-compose-traefik.yaml) - also add volume:
+      ##   - traefik-certs:/certs:ro
+      # TLS_EXTERNAL_CERT_AND_KEY: "/certs/${MAIL_DOMAIN}/certificate.crt /certs/${MAIL_DOMAIN}/privatekey.key"

docker-compose.yaml

@@ -0,0 +1,47 @@
+# Base compose file — do not edit. Put customizations (data paths, extra
+# volumes, env overrides) in docker-compose.override.yaml instead.
+# See docker/docker-compose.override.yaml.example for a starting point.
+#
+# Security note: this container uses network_mode:host (chatmail needs many
+# ports: 25, 53, 80, 143, 443, 465, 587, 993, 3340, 8443) and cgroup:host
+# (required for systemd). Together these give the container near-host-level
+# access. This is acceptable for a dedicated mail server, but be aware that
+# the container can bind any port and see all host network traffic.
+services:
+  chatmail:
+    build:
+      context: ./
+      dockerfile: docker/chatmail_relay.dockerfile
+      args:
+        GIT_HASH: ${GIT_HASH:-unknown}
+    image: chatmail-relay:latest
+    restart: unless-stopped
+    container_name: chatmail
+    # Required for systemd — use only one of the following:
+    cgroup: host          # compose v2
+    # privileged: true    # compose v1 (less restricted)
+    tty: true # required for logs
+    tmpfs: # required for systemd
+      - /tmp
+      - /run
+      - /run/lock
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "3"
+    environment:
+      MAIL_DOMAIN: $MAIL_DOMAIN
+    network_mode: "host"
+    volumes:
+      ## system (required)
+      - /sys/fs/cgroup:/sys/fs/cgroup:rw
+      ## data (defaults — override in docker-compose.override.yaml)
+      - mail:/home/vmail
+      - dkim:/etc/dkimkeys
+      - certs:/var/lib/acme
+
+volumes:
+  mail:
+  dkim:
+  certs:

docker/build.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+# Build the chatmail Docker image with the current git hash baked in.
+# Usage: ./docker/build.sh [extra docker-compose build args...]
+#
+# .git/ is excluded from the build context (.dockerignore) so the hash
+# must be passed as a build arg from the host.
+
+export GIT_HASH=$(git rev-parse --short HEAD)
+exec docker compose build "$@"

docker/chatmail_relay.dockerfile

@@ -0,0 +1,99 @@
+FROM jrei/systemd-debian:12 AS base
+
+ENV LANG=en_US.UTF-8
+
+RUN echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/01norecommend && \
+    echo 'APT::Install-Suggests "0";' >> /etc/apt/apt.conf.d/01norecommend && \
+    apt-get update && \
+    apt-get install -y \
+        ca-certificates && \
+    DEBIAN_FRONTEND=noninteractive \
+    TZ=UTC \
+    apt-get install -y tzdata && \
+    apt-get install -y locales && \
+    sed -i -e "s/# $LANG.*/$LANG UTF-8/" /etc/locale.gen && \
+    dpkg-reconfigure --frontend=noninteractive locales && \
+    update-locale LANG=$LANG \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN apt-get update && \
+    apt-get install -y \
+        git \
+        python3 \
+        python3-venv \
+        python3-virtualenv \
+        gcc \
+        python3-dev \
+        opendkim \
+        opendkim-tools \
+        curl \
+        rsync \
+        unbound \
+        unbound-anchor \
+        dnsutils \
+        postfix \
+        acl \
+        nginx \
+        libnginx-mod-stream \
+        fcgiwrap \
+        cron \
+    && rm -rf /var/lib/apt/lists/*
+
+# --- Build-time: install cmdeploy venv and run install stage ---
+# Editable install so importlib.resources reads directly from the source tree.
+# On container start only "configure,activate" stages run.
+COPY . /opt/chatmail/
+WORKDIR /opt/chatmail
+
+RUN printf '[params]\nmail_domain = build.local\n' > /tmp/chatmail.ini
+
+# Dummy git repo init: .git/ is excluded from the build context (.dockerignore)
+# but setuptools calls `git ls-files` when building the sdist.
+RUN git init -q && \
+    python3 -m venv /opt/cmdeploy && \
+    /opt/cmdeploy/bin/pip install --no-cache-dir \
+        -e chatmaild/ -e cmdeploy/
+
+RUN CMDEPLOY_STAGES=install \
+    CHATMAIL_INI=/tmp/chatmail.ini \
+    CHATMAIL_NOSYSCTL=True \
+    CHATMAIL_NOPORTCHECK=True \
+    /opt/cmdeploy/bin/pyinfra @local \
+        /opt/chatmail/cmdeploy/src/cmdeploy/run.py -y
+
+RUN cp -a www/ /opt/chatmail-www/
+
+RUN rm -f /tmp/chatmail.ini
+
+# Record image version (used in deploy fingerprint at runtime).
+# GIT_HASH is passed as a build arg (from docker-compose or CI) so that
+# .git/ can be excluded from the build context via .dockerignore.
+ARG GIT_HASH=unknown
+RUN echo "$GIT_HASH" > /etc/chatmail-image-version && \
+    echo "$GIT_HASH" > /etc/chatmail-version
+# --- End build-time install ---
+
+ENV TZ=:/etc/localtime
+ENV PATH="/opt/cmdeploy/bin:${PATH}"
+RUN ln -s /etc/chatmail/chatmail.ini /opt/chatmail/chatmail.ini
+
+ARG SETUP_CHATMAIL_SERVICE_PATH=/lib/systemd/system/setup_chatmail.service
+COPY ./docker/files/setup_chatmail.service "$SETUP_CHATMAIL_SERVICE_PATH"
+RUN ln -sf "$SETUP_CHATMAIL_SERVICE_PATH" "/etc/systemd/system/multi-user.target.wants/setup_chatmail.service"
+
+# Remove default nginx site config at build time (not in entrypoint)
+RUN rm -f /etc/nginx/sites-enabled/default
+
+COPY --chmod=555 ./docker/files/setup_chatmail_docker.sh /setup_chatmail_docker.sh
+COPY --chmod=555 ./docker/files/entrypoint.sh /entrypoint.sh
+
+HEALTHCHECK --interval=60s --timeout=10s --retries=3 \
+  CMD systemctl is-active dovecot postfix nginx unbound opendkim filtermail doveauth chatmail-metadata || exit 1
+
+STOPSIGNAL SIGRTMIN+3
+
+ENTRYPOINT ["/entrypoint.sh"]
+
+CMD [   "--default-standard-output=journal+console", \
+        "--default-standard-error=journal+console" ]
+

docker/files/entrypoint.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -eo pipefail
+
+SETUP_CHATMAIL_SERVICE_PATH="${SETUP_CHATMAIL_SERVICE_PATH:-/lib/systemd/system/setup_chatmail.service}"
+
+# Whitelist only the env vars needed by setup_chatmail_docker.sh.
+# Forwarding all env vars (via printenv) would leak Docker internals,
+# orchestrator secrets, and other unrelated variables into systemd.
+env_vars="MAIL_DOMAIN CMDEPLOY_STAGES CHATMAIL_INI TLS_EXTERNAL_CERT_AND_KEY PATH"
+sed -i "s|<envs_list>|$env_vars|g" "$SETUP_CHATMAIL_SERVICE_PATH"
+
+exec /lib/systemd/systemd "$@"

docker/files/setup_chatmail.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Run container setup commands
+After=multi-user.target
+ConditionPathExists=/setup_chatmail_docker.sh
+
+[Service]
+Type=oneshot
+ExecStart=/bin/bash /setup_chatmail_docker.sh
+RemainAfterExit=true
+WorkingDirectory=/opt/chatmail
+PassEnvironment=<envs_list>
+
+[Install]
+WantedBy=multi-user.target

docker/files/setup_chatmail_docker.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+set -euo pipefail
+export CHATMAIL_INI="${CHATMAIL_INI:-/etc/chatmail/chatmail.ini}"
+export CHATMAIL_NOSYSCTL=True
+export CHATMAIL_NOPORTCHECK=True
+
+CMDEPLOY=/opt/cmdeploy/bin/cmdeploy
+
+if [ -z "$MAIL_DOMAIN" ]; then
+    echo "ERROR: Environment variable 'MAIL_DOMAIN' must be set!" >&2
+    exit 1
+fi
+
+### MAIN
+
+if [ ! -f /etc/dkimkeys/opendkim.private ]; then
+    /usr/sbin/opendkim-genkey -D /etc/dkimkeys -d "$MAIL_DOMAIN" -s opendkim
+fi
+# Fix ownership for bind-mounted keys (host opendkim UID may differ from container)
+chown -R opendkim:opendkim /etc/dkimkeys
+
+# Journald: forward to console for docker logs
+grep -q '^ForwardToConsole=yes' /etc/systemd/journald.conf \
+    || echo "ForwardToConsole=yes" >> /etc/systemd/journald.conf
+systemctl restart systemd-journald
+
+# Create chatmail.ini (skips if file already exists, e.g. volume-mounted)
+mkdir -p "$(dirname "$CHATMAIL_INI")"
+if [ ! -f "$CHATMAIL_INI" ]; then
+    $CMDEPLOY init --config "$CHATMAIL_INI" "$MAIL_DOMAIN"
+fi
+
+# Inject external TLS paths from env var (unless user mounted their own ini)
+if [ -n "${TLS_EXTERNAL_CERT_AND_KEY:-}" ]; then
+    if ! grep -q '^tls_external_cert_and_key' "$CHATMAIL_INI"; then
+        echo "tls_external_cert_and_key = $TLS_EXTERNAL_CERT_AND_KEY" >> "$CHATMAIL_INI"
+    fi
+fi
+
+# --- Deploy fingerprint: skip cmdeploy run if nothing changed ---
+# On restart with identical image+config, systemd already brings up all
+# enabled services — the full cmdeploy run is redundant (~30s saved).
+# The install stage runs at image build time (Dockerfile), so only
+# configure+activate are needed here.
+IMAGE_VERSION_FILE="/etc/chatmail-image-version"
+FINGERPRINT_FILE="/etc/chatmail/.deploy-fingerprint"
+image_ver="none"
+[ -f "$IMAGE_VERSION_FILE" ] && image_ver=$(cat "$IMAGE_VERSION_FILE")
+config_hash=$(sha256sum "$CHATMAIL_INI" | cut -c1-16)
+current_fp="${image_ver}:${config_hash}"
+
+# CMDEPLOY_STAGES non-empty in env = operator override → always run.
+# Otherwise, if fingerprint matches the last successful deploy, skip.
+if [ -z "${CMDEPLOY_STAGES:-}" ] \
+    && [ -f "$FINGERPRINT_FILE" ] \
+    && [ "$(cat "$FINGERPRINT_FILE")" = "$current_fp" ]; then
+    echo "[INFO] No changes detected ($current_fp), skipping deploy."
+else
+    export CMDEPLOY_STAGES="${CMDEPLOY_STAGES:-configure,activate}"
+    $CMDEPLOY run --config "$CHATMAIL_INI" --ssh-host @local
+    echo "$current_fp" > "$FINGERPRINT_FILE"
+fi

env.example

@@ -0,0 +1 @@
+MAIL_DOMAIN=chat.example.com