feat: add tool to analyze deferred queue

It prints all destinations with the number of recipients
and all the reasons. Operator can then try
to fix the problems for destinations,
e.g. by manually adding reverse proxy
addresses to /etc/hosts for failing domains
or routing IP addresses to another interface.

.github/workflows/ci.yaml

@@ -1,20 +1,35 @@
-name: CI
+name: Run unit-tests and container-based deploy+test verification
 
 on:
-  pull_request:
+  # Triggers when a PR is merged into main or a direct push occurs
   push:
+    branches: [ "main" ]
+    
+  # Triggers for any PR (and its subsequent commits) targeting the main branch
+  pull_request:
+    branches: [ "main" ]
+
+permissions: {}
+
+# Newest push wins: Prevents multiple runs from clashing and wasting runner efforts
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 
 jobs:
   tox:
     name: isolated chatmaild tests
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         # Checkout pull request HEAD commit instead of merge commit
         # Otherwise `test_deployed_state` will be unhappy.
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-
+          persist-credentials: false
+      - name: download filtermail
+        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.6.4/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
       - name: run chatmaild tests 
         working-directory: chatmaild
         run: pipx run tox
@@ -23,7 +38,10 @@ jobs:
     name: deploy-chatmail tests 
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          persist-credentials: false
 
       - name: initenv 
         run: scripts/initenv.sh
@@ -37,5 +55,23 @@ jobs:
       - name: run deploy-chatmail offline tests 
         run: pytest --pyargs cmdeploy 
 
-      # all other cmdeploy commands require a staging server 
-      # see https://github.com/deltachat/chatmail/issues/100
+  lxc-test:
+    name: LXC deploy and test
+    uses: chatmail/cmlxc/.github/workflows/lxc-test.yml@v0.10.0
+    with:
+      cmlxc_commands: |
+        cmlxc init 
+        # single cmdeploy relay test
+        cmlxc -v deploy-cmdeploy --source ./repo cm0
+        cmlxc -v test-mini cm0
+        cmlxc -v test-cmdeploy cm0
+
+        # cross cmdeploy relay test
+        cmlxc -v deploy-cmdeploy --source ./repo --ipv4-only cm1
+        cmlxc -v test-cmdeploy cm0 cm1
+
+        # cross cmdeploy/madmail relay tests
+        cmlxc -v deploy-madmail mad0
+        cmlxc -v test-cmdeploy cm0 mad0
+        cmlxc -v test-mini cm0 mad0
+        cmlxc -v test-mini mad0 cm0

.github/workflows/docker-dispatch.yaml

@@ -0,0 +1,37 @@
+# Notify the docker repo to build and test a new image after relay CI passes.
+#
+# Sends a repository_dispatch event to chatmail/docker with the relay ref
+# and short SHA, which triggers docker-ci.yaml to build, push to GHCR,
+# and run integration tests via cmlxc.
+
+name: Trigger Docker build
+
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+permissions: {}
+
+jobs:
+  dispatch:
+    name: Dispatch build to chatmail/docker
+    runs-on: ubuntu-latest
+    if: github.repository == 'chatmail/relay'
+    steps:
+      - name: Compute short SHA
+        id: sha
+        run: echo "short=$(echo '${{ github.sha }}' | cut -c1-7)" >> "$GITHUB_OUTPUT"
+
+      - name: Send repository_dispatch
+        uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # v3
+        with:
+          token: ${{ secrets.CHATMAIL_DOCKER_DISPATCH_TOKEN }}
+          repository: chatmail/docker
+          event-type: relay-updated
+          client-payload: >-
+            {
+              "relay_ref": "${{ github.ref_name }}",
+              "relay_sha": "${{ github.sha }}",
+              "relay_sha_short": "${{ steps.sha.outputs.short }}"
+            }

.github/workflows/docs-preview.yaml

@@ -7,6 +7,8 @@ on:
       - 'scripts/build-docs.sh'
       - '.github/workflows/docs-preview.yaml'
 
+permissions: {}
+
 jobs:
   scripts:
     name: build 
@@ -16,6 +18,8 @@ jobs:
       url: https://staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}
     steps:
       - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
 
       - name: initenv 
         run: scripts/initenv.sh
@@ -34,18 +38,22 @@ jobs:
       - name: Get Pullrequest ID
         id: prepare
         run: |
-          export PULLREQUEST_ID=$(echo "${{ github.ref }}" | cut -d "/" -f3)
+          export PULLREQUEST_ID=$(echo "${GITHUB_REF}" | cut -d "/" -f3)
           echo "prid=$PULLREQUEST_ID" >> $GITHUB_OUTPUT
           if [ $(expr length "${{ secrets.USERNAME }}") -gt "1" ]; then echo "uploadtoserver=true" >> $GITHUB_OUTPUT; fi
       - run: |
-          echo "baseurl: /${{ steps.prepare.outputs.prid }}" >> _config.yml
+          echo "baseurl: /${STEPS_PREPARE_OUTPUTS_PRID}" >> _config.yml
+        env:
+          STEPS_PREPARE_OUTPUTS_PRID: ${{ steps.prepare.outputs.prid }}
 
       - name: Upload preview
         run: |
           mkdir -p "$HOME/.ssh"
           echo "${{ secrets.CHATMAIL_STAGING_SSHKEY }}" > "$HOME/.ssh/key"
           chmod 600 "$HOME/.ssh/key"
-          rsync -rILvh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/doc/build/ "${{ secrets.USERNAME }}@chatmail.at:/var/www/html/staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}/"
+          rsync -rILvh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/doc/build/ "${{ secrets.USERNAME }}@chatmail.at:/var/www/html/staging.chatmail.at/doc/relay/${STEPS_PREPARE_OUTPUTS_PRID}/"
+        env:
+          STEPS_PREPARE_OUTPUTS_PRID: ${{ steps.prepare.outputs.prid }}
 
       - name: check links 
         working-directory: doc

.github/workflows/docs.yaml

@@ -10,6 +10,8 @@ on:
       - 'scripts/build-docs.sh'
       - '.github/workflows/docs.yaml'
 
+permissions: {}
+
 jobs:
   scripts:
     name: build 
@@ -19,6 +21,8 @@ jobs:
       url: https://chatmail.at/doc/relay/
     steps:
       - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
 
       - name: initenv 
         run: scripts/initenv.sh

.github/workflows/test-and-deploy-ipv4only.yaml

@@ -1,100 +0,0 @@
-name: deploy on staging-ipv4.testrun.org, and run tests
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    paths-ignore:
-      - 'scripts/**'
-      - '**/README.md'
-      - 'CHANGELOG.md'
-      - 'LICENSE'
-
-jobs:
-  deploy:
-    name: deploy on staging-ipv4.testrun.org, and run tests
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    environment:
-      name: staging-ipv4.testrun.org
-      url: https://staging-ipv4.testrun.org/
-    concurrency:
-      group: ci-ipv4-${{ github.workflow }}-${{ github.ref }}
-      cancel-in-progress: ${{ !contains(github.ref, '$GITHUB_REF') }}
-    steps:
-      - uses: jsok/serialize-workflow-action@515cd04c46d7ea7435c4a22a3b4419127afdefe9
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: actions/checkout@v4
-
-      - name: prepare SSH
-        run: |
-          mkdir ~/.ssh
-          echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
-          chmod 600 ~/.ssh/id_ed25519
-          ssh-keyscan staging-ipv4.testrun.org > ~/.ssh/known_hosts
-          # save previous acme & dkim state
-          rsync -avz root@staging-ipv4.testrun.org:/var/lib/acme acme-ipv4 || true
-          rsync -avz root@staging-ipv4.testrun.org:/etc/dkimkeys dkimkeys-ipv4 || true
-          # store previous acme & dkim state on ns.testrun.org, if it contains useful certs
-          if [ -f dkimkeys-ipv4/dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys-ipv4 root@ns.testrun.org:/tmp/ || true; fi
-          if [ "$(ls -A acme-ipv4/acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme-ipv4 root@ns.testrun.org:/tmp/ || true; fi
-          # make sure CAA record isn't set
-          scp -o StrictHostKeyChecking=accept-new .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
-          ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging-ipv4.testrun.org.zone
-          ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
-          ssh root@ns.testrun.org systemctl reload nsd
-
-      - name: rebuild staging-ipv4.testrun.org to have a clean VPS
-        run: |
-            curl -X POST \
-            -H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
-            -H "Content-Type: application/json" \
-            -d '{"image":"debian-12"}' \
-            "https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_IPV4_SERVER_ID }}/actions/rebuild"
-
-      - run: scripts/initenv.sh
-
-      - name: append venv/bin to PATH
-        run: echo venv/bin >>$GITHUB_PATH
-
-      - name: upload TLS cert after rebuilding
-        run: |
-          echo " --- wait until staging-ipv4.testrun.org VPS is rebuilt --- "
-          rm ~/.ssh/known_hosts
-          while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u ; do sleep 1 ; done
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u
-          # download acme & dkim state from ns.testrun.org
-          rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme-ipv4/acme acme-restore || true
-          rsync -avz root@ns.testrun.org:/tmp/dkimkeys-ipv4/dkimkeys dkimkeys-restore || true
-          # restore acme & dkim state to staging2.testrun.org
-          rsync -avz acme-restore/acme root@staging-ipv4.testrun.org:/var/lib/ || true
-          rsync -avz dkimkeys-restore/dkimkeys root@staging-ipv4.testrun.org:/etc/ || true
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org chown root:root -R /var/lib/acme || true
-
-      - name: run deploy-chatmail offline tests 
-        run: pytest --pyargs cmdeploy 
-
-      - run: |
-          cmdeploy init staging-ipv4.testrun.org
-          sed -i 's#disable_ipv6 = False#disable_ipv6 = True#' chatmail.ini
-
-      - run: cmdeploy run --verbose --skip-dns-check
-
-      - name: set DNS entries
-        run: |
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
-          cmdeploy dns --zonefile staging-generated.zone
-          cat staging-generated.zone >> .github/workflows/staging-ipv4.testrun.org-default.zone
-          cat .github/workflows/staging-ipv4.testrun.org-default.zone
-          scp .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
-          ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
-          ssh root@ns.testrun.org systemctl reload nsd
-
-      - name: cmdeploy test
-        run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
-
-      - name: cmdeploy dns
-        run: cmdeploy dns -v
-

.github/workflows/test-and-deploy.yaml

@@ -1,101 +0,0 @@
-name: deploy on staging2.testrun.org, and run tests
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    paths-ignore:
-      - 'scripts/**'
-      - '**/README.md'
-      - 'CHANGELOG.md'
-      - 'LICENSE'
-
-jobs:
-  deploy:
-    name: deploy on staging2.testrun.org, and run tests
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    environment:
-      name: staging2.testrun.org
-      url: https://staging2.testrun.org/
-    concurrency:
-      group: ci-${{ github.workflow }}-${{ github.ref }}
-      cancel-in-progress: ${{ !contains(github.ref, '$GITHUB_REF') }}
-    steps:
-      - uses: jsok/serialize-workflow-action@515cd04c46d7ea7435c4a22a3b4419127afdefe9
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: actions/checkout@v4
-
-      - name: prepare SSH
-        run: |
-          mkdir ~/.ssh
-          echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
-          chmod 600 ~/.ssh/id_ed25519
-          ssh-keyscan staging2.testrun.org > ~/.ssh/known_hosts
-          # save previous acme & dkim state
-          rsync -avz root@staging2.testrun.org:/var/lib/acme . || true
-          rsync -avz root@staging2.testrun.org:/etc/dkimkeys . || true
-          # store previous acme & dkim state on ns.testrun.org, if it contains useful certs
-          if [ -f dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys root@ns.testrun.org:/tmp/ || true; fi
-          if [ "$(ls -A acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme root@ns.testrun.org:/tmp/ || true; fi
-          # make sure CAA record isn't set
-          scp -o StrictHostKeyChecking=accept-new .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
-          ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging2.testrun.org.zone
-          ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
-          ssh root@ns.testrun.org systemctl reload nsd
-
-      - name: rebuild staging2.testrun.org to have a clean VPS
-        run: |
-            curl -X POST \
-            -H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
-            -H "Content-Type: application/json" \
-            -d '{"image":"debian-12"}' \
-            "https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_SERVER_ID }}/actions/rebuild"
-
-      - run: scripts/initenv.sh
-
-      - name: append venv/bin to PATH
-        run: echo venv/bin >>$GITHUB_PATH
-
-      - name: upload TLS cert after rebuilding
-        run: |
-          echo " --- wait until staging2.testrun.org VPS is rebuilt --- "
-          rm ~/.ssh/known_hosts
-          while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org id -u ; do sleep 1 ; done
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org id -u
-          # download acme & dkim state from ns.testrun.org
-          rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme acme-restore || true
-          rsync -avz root@ns.testrun.org:/tmp/dkimkeys dkimkeys-restore || true
-          # restore acme & dkim state to staging2.testrun.org
-          rsync -avz acme-restore/acme root@staging2.testrun.org:/var/lib/ || true
-          rsync -avz dkimkeys-restore/dkimkeys root@staging2.testrun.org:/etc/ || true
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org chown root:root -R /var/lib/acme || true
-
-      - name: add hpk42 key to staging server
-        run: ssh root@staging2.testrun.org 'curl -s https://github.com/hpk42.keys >> .ssh/authorized_keys'
-
-      - name: run deploy-chatmail offline tests 
-        run: pytest --pyargs cmdeploy 
-
-      - run: cmdeploy init staging2.testrun.org
-
-      - run: cmdeploy run --verbose --skip-dns-check
-
-      - name: set DNS entries
-        run: |
-          ssh -o StrictHostKeyChecking=accept-new root@staging2.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
-          cmdeploy dns --zonefile staging-generated.zone --verbose
-          cat staging-generated.zone >> .github/workflows/staging.testrun.org-default.zone
-          cat .github/workflows/staging.testrun.org-default.zone
-          scp .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
-          ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
-          ssh root@ns.testrun.org systemctl reload nsd
-
-      - name: cmdeploy test
-        run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
-
-      - name: cmdeploy dns
-        run: cmdeploy dns -v
-

.github/workflows/zizmor-scan.yml

@@ -0,0 +1,26 @@
+name: GitHub Actions Security Analysis with zizmor
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+    branches: ["**"]
+
+permissions: {}
+
+jobs:
+  zizmor:
+    name: Run zizmor
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write # Required for upload-sarif (used by zizmor-action) to upload SARIF files.
+      contents: read
+      actions: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Run zizmor
+        uses: zizmorcore/zizmor-action@b1d7e1fb5de872772f31590499237e7cce841e8e # v0.5.3

.github/zizmor.yml

@@ -0,0 +1,7 @@
+rules:
+  unpinned-uses:
+    config:
+      policies:
+        actions/*: ref-pin
+        dependabot/*: ref-pin
+        chatmail/*: ref-pin

.gitignore

@@ -4,7 +4,7 @@ __pycache__/
 *$py.class
 *.swp
 *qr-*.png
-chatmail.ini
+chatmail*.ini
 
 
 # C extensions

CHANGELOG.md

@@ -1,5 +1,89 @@
 # Changelog for chatmail deployment 
 
+## 1.10.0 2026-04-30
+
+* start mtail after networking is fully up <https://github.com/chatmail/relay/pull/942>
+* support specifying custom filtermail binary through environment variable <https://github.com/chatmail/relay/pull/941>
+* add automated zizmor scanning of github workflows <https://github.com/chatmail/relay/pull/938>
+* added dispatch for *automated builds of chatmail relay docker images* <https://github.com/chatmail/relay/pull/934>
+* do not bind SMTP client sockets to public addresses <https://github.com/chatmail/relay/pull/932>
+* underline in docs that scripts/initenv.sh should be used for building the docs <https://github.com/chatmail/relay/pull/933>
+* automatic oldest-first message removal from mailboxes to always stay under max_mailbox_size <https://github.com/chatmail/relay/pull/929>
+* remove --slow from cmdeploy test <https://github.com/chatmail/relay/pull/931>
+* handle missing inotify sysctl keys in containers <https://github.com/chatmail/relay/pull/930>
+* replace resolvconf with static resolv.conf <https://github.com/chatmail/relay/pull/928>
+* disable fsync for LMTP and IMAP services <https://github.com/chatmail/relay/pull/925>
+* re-use cmlxc workflow, replacing CI with hetzner staging servers with local lxc containers <https://github.com/chatmail/relay/pull/917>
+* explicitly install resolvconf <https://github.com/chatmail/relay/pull/924>
+* detect stale dovecot binary and force restart in activate() <https://github.com/chatmail/relay/pull/922>
+* Rename filtermail_http_port to filtermail_http_port_incoming <https://github.com/chatmail/relay/pull/921>
+* consolidated is_in_container() check https://github.com/chatmail/relay/pull/920>
+* restart dovecot after package replacement (rebase, test condense) <https://github.com/chatmail/relay/pull/913>
+* Set permissions on dovecot pin prefs <https://github.com/chatmail/relay/pull/915>
+* Route `/mxdeliv/` to configurable port <https://github.com/chatmail/relay/pull/901>
+* fix VM detection, automated testing fixes, use newer chatmail-turn and move to standard BIND DNS zone format <https://github.com/chatmail/relay/pull/912>
+* Upgrade to filtermail 0.6.1 <https://github.com/chatmail/relay/pull/910>
+* pin dovecot packages to prevent apt upgrades <https://github.com/chatmail/relay/pull/908>
+* add rpc server to cmdeploy along with client <https://github.com/chatmail/relay/pull/906>
+* remove unused deps from chatmaild <https://github.com/chatmail/relay/pull/905>
+* set default smtp_tls_security_level to "verify" unconditionally <https://github.com/chatmail/relay/pull/902>
+* featprefer IPv4 in SMTP client <https://github.com/chatmail/relay/pull/900>
+* Install dovecot .deb packages atomically <https://github.com/chatmail/relay/pull/899>
+* stop installing cron package <https://github.com/chatmail/relay/pull/898>
+* Rewrite dovecot install logic, update <https://github.com/chatmail/relay/pull/862>
+* fix a test and some linting fixes <https://github.com/chatmail/relay/pull/897>
+* Disable IP verification on domain-literal addresses <https://github.com/chatmail/relay/pull/895>
+* disable installing recommended packages globally on the relay <https://github.com/chatmail/relay/pull/887>
+* multiple bug fixes across chatmaild and cmdeploy <https://github.com/chatmail/relay/pull/883>
+* remove /metrics from the website <https://github.com/chatmail/relay/pull/703>
+* add Prometheus textfile output to fsreport <https://github.com/chatmail/relay/pull/881>
+* chown opendkim: private key <https://github.com/chatmail/relay/pull/879>
+* make sure chatmail-metadata was started <https://github.com/chatmail/relay/pull/882>
+* dovecot update url <https://github.com/chatmail/relay/pull/880>
+* upgrade to filtermail v0.5.2 <https://github.com/chatmail/relay/pull/876>
+* download dovecot packages from github release <https://github.com/chatmail/relay/pull/875>
+* replace DKIM verification with filtermail v0.5 <https://github.com/chatmail/relay/pull/831>
+* remove CFFI deltachat bindings usage, and consolidate test support with rpc-bindings <https://github.com/chatmail/relay/pull/872>
+* prepare chatmaild/cmdeploy changes for Docker support <https://github.com/chatmail/relay/pull/857>
+* stabilize online benchmark timing adding rate-limit-aware cooldown between iterations <https://github.com/chatmail/relay/pull/867>
+* move rate-limit cooldown to benchmark fixture <https://github.com/chatmail/relay/pull/868>
+* reconfigure acmetool from redirector to proxy mode <https://github.com/chatmail/relay/pull/861>
+* make tests work with `--ssh-host localhost` <https://github.com/chatmail/relay/pull/856>
+* mark f-string with f prefix in test_expunged <https://github.com/chatmail/relay/pull/863>
+* install also if dovecot.service=False in SystemdEnabled Fact <https://github.com/chatmail/relay/pull/841>
+* Introduce support for self-signed chatmail relays <https://github.com/chatmail/relay/pull/855>
+* Strip Received headers before delivery <https://github.com/chatmail/relay/pull/849>
+* upgrade to filtermail v0.3 <https://github.com/chatmail/relay/pull/850>
+* fix link to Maddy and update madmail URL <https://github.com/chatmail/relay/pull/847>
+* accept self-signed certificates for IP-only relays <https://github.com/chatmail/relay/pull/846>
+* enforce sending from public IP addresses <https://github.com/chatmail/relay/pull/845>
+* port check: check addresses, fix single services <https://github.com/chatmail/relay/pull/844>
+* remediates issue with improper concat on resolver injection <https://github.com/chatmail/relay/pull/834>
+* ipv6 boolean not being respected during operations <https://github.com/chatmail/relay/pull/832>
+* upgrade to filtermail v0.2 by <https://github.com/chatmail/relay/pull/825>
+* fix link to filtermail <https://github.com/chatmail/relay/pull/824>
+* print timestamps when sending messages <https://github.com/chatmail/relay/pull/823>
+* fix flaky test_exceed_rate_limit <https://github.com/chatmail/relay/pull/822>
+* Replace filtermail with rust reimplementation <https://github.com/chatmail/relay/pull/808>
+* Set default internal SMTP ports in Config <https://github.com/chatmail/relay/pull/819>
+* separate metrics for incoming and outgoing messages <https://github.com/chatmail/relay/pull/820>
+* disable appending the Received header <https://github.com/chatmail/relay/pull/815>
+* fail on errors in postfix/dovecot config <https://github.com/chatmail/relay/pull/813>
+* tweak idle/hibernate metrics some more <https://github.com/chatmail/relay/pull/811>
+* add config flag to export statistics <https://github.com/chatmail/relay/pull/806>
+* add --website-only option to run subcommand <https://github.com/chatmail/relay/pull/768>
+* Strip DKIM-Signature header before LMTP <https://github.com/chatmail/relay/pull/803>
+* properly make sure that postfix gets restarted on failure <https://github.com/chatmail/relay/pull/802>
+* expire.py: use absolute path to maildirsize <https://github.com/chatmail/relay/pull/807>
+* pin Dovecot documentation URLs to version 2.3 <https://github.com/chatmail/relay/pull/800>
+* try to use "build machine" and "deployment server"  consistently  <https://github.com/chatmail/relay/pull/797>
+* adds instructions for migrating control machines <https://github.com/chatmail/relay/pull/795>
+* use consistent naming schema in getting started <https://github.com/chatmail/relay/pull/793>
+* remove jsok/serialize-workflow-action dependency <https://github.com/chatmail/relay/pull/790>
+* streamline migration guide wording, provide titled steps  <https://github.com/chatmail/relay/pull/789>
+* increases default max mailbox size <https://github.com/chatmail/relay/pull/792>
+* use daemon_name for OpenDKIM sign-verify decision instead of IP <https://github.com/chatmail/relay/pull/784>
+
 ## 1.9.0 2025-12-18
 
 ### Documentation

chatmaild/pyproject.toml

@@ -6,10 +6,7 @@ build-backend = "setuptools.build_meta"
 name = "chatmaild"
 version = "0.3"
 dependencies = [
-  "aiosmtpd",
   "iniconfig",
-  "deltachat-rpc-server",
-  "deltachat-rpc-client",
   "filelock",
   "requests",
   "crypt-r >= 3.13.1 ; python_version >= '3.11'",
@@ -24,10 +21,10 @@ where = ['src']
 [project.scripts]
 doveauth = "chatmaild.doveauth:main"
 chatmail-metadata = "chatmaild.metadata:main"
-filtermail = "chatmaild.filtermail:main"
-chatmail-metrics = "chatmaild.metrics:main"
-chatmail-expire = "chatmaild.expire:main"
+chatmail-expire = "chatmaild.expire:daily_expire_main"
+chatmail-quota-expire = "chatmaild.expire:quota_expire_main"
 chatmail-fsreport = "chatmaild.fsreport:main"
+chatmail-deferred = "chatmaild.deferred:main"
 lastlogin = "chatmaild.lastlogin:main"
 turnserver = "chatmaild.turnserver:main"
 
@@ -72,6 +69,7 @@ commands =
 deps = pytest 
        pdbpp 
        pytest-localserver
+       aiosmtpd
        execnet
 commands = pytest -v -rsXx {posargs}
 """

chatmaild/src/chatmaild/config.py

@@ -20,7 +20,8 @@ class Config:
     def __init__(self, inipath, params):
         self._inipath = inipath
         self.mail_domain = params["mail_domain"]
-        self.max_user_send_per_minute = int(params["max_user_send_per_minute"])
+        self.max_user_send_per_minute = int(params.get("max_user_send_per_minute", 60))
+        self.max_user_send_burst_size = int(params.get("max_user_send_burst_size", 10))
         self.max_mailbox_size = params["max_mailbox_size"]
         self.max_message_size = int(params.get("max_message_size", "31457280"))
         self.delete_mails_after = params["delete_mails_after"]
@@ -32,13 +33,19 @@ class Config:
         self.passthrough_senders = params["passthrough_senders"].split()
         self.passthrough_recipients = params["passthrough_recipients"].split()
         self.www_folder = params.get("www_folder", "")
-        self.filtermail_smtp_port = int(params["filtermail_smtp_port"])
+        self.filtermail_smtp_port = int(params.get("filtermail_smtp_port", "10080"))
         self.filtermail_smtp_port_incoming = int(
-            params["filtermail_smtp_port_incoming"]
+            params.get("filtermail_smtp_port_incoming", "10081")
         )
-        self.postfix_reinject_port = int(params["postfix_reinject_port"])
+        self.filtermail_http_port_incoming = int(
+            params.get("filtermail_http_port_incoming", "10082")
+        )
+        self.filtermail_lmtp_port_transport = int(
+            params.get("filtermail_lmtp_port_transport", "10083")
+        )
+        self.postfix_reinject_port = int(params.get("postfix_reinject_port", "10025"))
         self.postfix_reinject_port_incoming = int(
-            params["postfix_reinject_port_incoming"]
+            params.get("postfix_reinject_port_incoming", "10026")
         )
         self.mtail_address = params.get("mtail_address")
         self.disable_ipv6 = params.get("disable_ipv6", "false").lower() == "true"
@@ -56,6 +63,31 @@ class Config:
         self.privacy_pdo = params.get("privacy_pdo")
         self.privacy_supervisor = params.get("privacy_supervisor")
 
+        # TLS certificate management.
+        # If tls_external_cert_and_key is set, use externally managed certs.
+        # Otherwise derived from the domain name:
+        # - Domains starting with "_" use self-signed certificates
+        # - All other domains use ACME.
+        external = params.get("tls_external_cert_and_key", "").strip()
+
+        if external:
+            parts = external.split()
+            if len(parts) != 2:
+                raise ValueError(
+                    "tls_external_cert_and_key must have two space-separated"
+                    " paths: CERT_PATH KEY_PATH"
+                )
+            self.tls_cert_mode = "external"
+            self.tls_cert_path, self.tls_key_path = parts
+        elif self.mail_domain.startswith("_"):
+            self.tls_cert_mode = "self"
+            self.tls_cert_path = "/etc/ssl/certs/mailserver.pem"
+            self.tls_key_path = "/etc/ssl/private/mailserver.key"
+        else:
+            self.tls_cert_mode = "acme"
+            self.tls_cert_path = f"/var/lib/acme/live/{self.mail_domain}/fullchain"
+            self.tls_key_path = f"/var/lib/acme/live/{self.mail_domain}/privkey"
+
         # deprecated option
         mbdir = params.get("mailboxes_dir", f"/home/vmail/mail/{self.mail_domain}")
         self.mailboxes_dir = Path(mbdir.strip())
@@ -63,6 +95,11 @@ class Config:
         # old unused option (except for first migration from sqlite to maildir store)
         self.passdb_path = Path(params.get("passdb_path", "/home/vmail/passdb.sqlite"))
 
+    @property
+    def max_mailbox_size_mb(self):
+        """Return max_mailbox_size as an integer in megabytes."""
+        return parse_size_mb(self.max_mailbox_size)
+
     def _getbytefile(self):
         return open(self._inipath, "rb")
 
@@ -76,6 +113,16 @@ class Config:
         return User(maildir, addr, password_path, uid="vmail", gid="vmail")
 
 
+def parse_size_mb(limit):
+    """Parse a size string like ``500M`` or ``2G`` and return megabytes."""
+    value = limit.strip().upper().removesuffix("B")
+    if value.endswith("G"):
+        return int(value[:-1]) * 1024
+    if value.endswith("M"):
+        return int(value[:-1])
+    return int(value)
+
+
 def write_initial_config(inipath, mail_domain, overrides):
     """Write out default config file, using the specified config value overrides."""
     content = get_default_config_content(mail_domain, **overrides)

chatmaild/src/chatmaild/deferred.py

@@ -0,0 +1,37 @@
+"""
+Analyze deferred mails and print most common failing destinations.
+
+Example:
+
+    python -m chatmaild.deferred
+"""
+
+import json
+import subprocess
+from collections import Counter, defaultdict
+
+
+def main():
+    p = subprocess.Popen(["postqueue", "-j"], text=True, stdout=subprocess.PIPE)
+    domain_reasons = defaultdict(Counter)
+    domain_total = Counter()
+
+    for line in p.stdout:
+        item = json.loads(line)
+        if item["queue_name"] != "deferred":
+            continue
+
+        for recipient in item["recipients"]:
+            _, domain = recipient["address"].rsplit("@", 1)
+            reason = recipient["delay_reason"]
+            domain_total[domain] += 1
+            domain_reasons[domain][reason] += 1
+
+    for domain, total in reversed(domain_total.most_common()):
+        print(f"{domain} ({total} recipients)")
+        for reason, count in domain_reasons[domain].most_common():
+            print(f"  {count}: {reason}")
+
+
+if __name__ == "__main__":
+    main()

chatmaild/src/chatmaild/dictproxy.py

@@ -22,7 +22,7 @@ class DictProxy:
                 wfile.flush()
 
     def handle_dovecot_request(self, msg, transactions):
-        # see https://doc.dovecot.org/developer_manual/design/dict_protocol/#dovecot-dict-protocol
+        # see https://doc.dovecot.org/2.3/developer_manual/design/dict_protocol/#dovecot-dict-protocol
         short_command = msg[0]
         parts = msg[1:].split("\t")
 

chatmaild/src/chatmaild/doveauth.py

@@ -1,8 +1,11 @@
 import json
 import logging
 import os
+import re
 import sys
 
+import filelock
+
 try:
     import crypt_r
 except ImportError:
@@ -13,10 +16,11 @@ from .dictproxy import DictProxy
 from .migrate_db import migrate_from_db_to_maildir
 
 NOCREATE_FILE = "/etc/chatmail-nocreate"
+VALID_LOCALPART_RE = re.compile(r"^[a-z0-9._-]+$")
 
 
 def encrypt_password(password: str):
-    # https://doc.dovecot.org/configuration_manual/authentication/password_schemes/
+    # https://doc.dovecot.org/2.3/configuration_manual/authentication/password_schemes/
     passhash = crypt_r.crypt(password, crypt_r.METHOD_SHA512)
     return "{SHA512-CRYPT}" + passhash
 
@@ -52,6 +56,10 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
         )
         return False
 
+    if not VALID_LOCALPART_RE.match(localpart):
+        logging.warning("localpart %r contains invalid characters", localpart)
+        return False
+
     return True
 
 
@@ -140,8 +148,13 @@ class AuthDictProxy(DictProxy):
         if not is_allowed_to_create(self.config, addr, cleartext_password):
             return
 
-        user.set_password(encrypt_password(cleartext_password))
-        print(f"Created address: {addr}", file=sys.stderr)
+        lock = filelock.FileLock(str(user.password_path) + ".lock", timeout=5)
+        with lock:
+            userdata = user.get_userdb_dict()
+            if userdata:
+                return userdata
+            user.set_password(encrypt_password(cleartext_password))
+            print(f"Created address: {addr}", file=sys.stderr)
         return user.get_userdb_dict()
 
 

chatmaild/src/chatmaild/expire.py

@@ -4,17 +4,26 @@ Expire old messages and addresses.
 """
 
 import os
+import re
 import shutil
 import sys
 import time
 from argparse import ArgumentParser
 from collections import namedtuple
 from datetime import datetime
+from pathlib import Path
 from stat import S_ISREG
 
 from chatmaild.config import read_config
 
 FileEntry = namedtuple("FileEntry", ("path", "mtime", "size"))
+QuotaFileEntry = namedtuple("QuotaFileEntry", ("mtime", "quota_size", "path"))
+
+# Quota cleanup factor of max_mailbox_size. The mailbox is reset to this size.
+QUOTA_CLEANUP_FACTOR = 0.7
+
+# e.g. "cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S"
+_dovecot_fn_rex = re.compile(r".+/(\d+)\..+,S=(\d+)")
 
 
 def iter_mailboxes(basedir, maxnum):
@@ -74,6 +83,42 @@ class MailboxStat:
         self.extrafiles.sort(key=lambda x: -x.size)
 
 
+def parse_dovecot_filename(relpath):
+    m = _dovecot_fn_rex.match(relpath)
+    if not m:
+        return None
+    return QuotaFileEntry(int(m.group(1)), int(m.group(2)), relpath)
+
+
+def scan_mailbox_messages(mbox):
+    messages = []
+    for sub in ("cur", "new"):
+        for name in os_listdir_if_exists(mbox / sub):
+            if entry := parse_dovecot_filename(f"{sub}/{name}"):
+                messages.append(entry)
+    return messages
+
+
+def expire_to_target(mbox, target_bytes):
+    messages = scan_mailbox_messages(mbox)
+    total_size = sum(m.quota_size for m in messages)
+    # Keep recent 24 hours of messages protected from expiry because
+    # likely something is wrong with interactions on that address
+    # and quota-full signal can help the address owner's device to notice it
+    undeletable_messages_cutoff = time.time() - (3600 * 24)
+    removed = 0
+    for entry in sorted(messages):
+        if total_size <= target_bytes:
+            break
+        if entry.mtime > undeletable_messages_cutoff:
+            break
+        (mbox / entry.path).unlink(missing_ok=True)
+        total_size -= entry.quota_size
+        removed += 1
+
+    return removed
+
+
 def print_info(msg):
     print(msg, file=sys.stderr)
 
@@ -143,8 +188,21 @@ class Expiry:
             else:
                 continue
             changed = True
+
+        target_bytes = (
+            self.config.max_mailbox_size_mb * 1024 * 1024 * QUOTA_CLEANUP_FACTOR
+        )
+        removed = expire_to_target(Path(mbox.basedir), target_bytes)
+        if removed:
+            changed = True
+            self.del_files += removed
+            if self.verbose:
+                print_info(
+                    f"quota-expire: removed {removed} message(s) from {mboxname}"
+                )
+
         if changed:
-            self.remove_file("maildirsize")
+            self.remove_file(f"{mbox.basedir}/maildirsize")
 
     def get_summary(self):
         return (
@@ -154,9 +212,9 @@ class Expiry:
         )
 
 
-def main(args=None):
+def daily_expire_main(args=None):
     """Expire mailboxes and messages according to chatmail config"""
-    parser = ArgumentParser(description=main.__doc__)
+    parser = ArgumentParser(description=daily_expire_main.__doc__)
     ini = "/usr/local/lib/chatmaild/chatmail.ini"
     parser.add_argument(
         "chatmail_ini",
@@ -202,5 +260,33 @@ def main(args=None):
     print(exp.get_summary())
 
 
-if __name__ == "__main__":
-    main(sys.argv[1:])
+def quota_expire_main(args=None):
+    """Remove mailbox messages to stay within a megabyte target.
+
+    This entry point is called by dovecot when a quota threshold is passed.
+    """
+
+    parser = ArgumentParser(description=quota_expire_main.__doc__)
+    parser.add_argument(
+        "target_mb",
+        type=int,
+        help="target mailbox size in megabytes",
+    )
+    parser.add_argument(
+        "mailbox_path",
+        type=Path,
+        help="path to a user mailbox",
+    )
+    args = parser.parse_args(args)
+
+    target_bytes = args.target_mb * 1024 * 1024
+
+    removed_count = expire_to_target(args.mailbox_path, target_bytes)
+    if removed_count:
+        (args.mailbox_path / "maildirsize").unlink(missing_ok=True)
+        print(
+            f"quota-expire: removed {removed_count} message(s)"
+            f" from {args.mailbox_path.name}",
+            file=sys.stderr,
+        )
+    return 0

chatmaild/src/chatmaild/filtermail.py

@@ -1,381 +0,0 @@
-#!/usr/bin/env python3
-import asyncio
-import base64
-import binascii
-import sys
-import time
-from email import policy
-from email.parser import BytesParser
-from email.utils import parseaddr
-from smtplib import SMTP as SMTPClient
-
-from aiosmtpd.controller import Controller
-from aiosmtpd.smtp import SMTP
-
-from .config import read_config
-
-ENCRYPTION_NEEDED_523 = "523 Encryption Needed: Invalid Unencrypted Mail"
-
-
-def check_openpgp_payload(payload: bytes):
-    """Checks the OpenPGP payload.
-
-    OpenPGP payload must consist only of PKESK and SKESK packets
-    terminated by a single SEIPD packet.
-
-    Returns True if OpenPGP payload is correct,
-    False otherwise.
-
-    May raise IndexError while trying to read OpenPGP packet header
-    if it is truncated.
-    """
-    i = 0
-    while i < len(payload):
-        # Only OpenPGP format is allowed.
-        if payload[i] & 0xC0 != 0xC0:
-            return False
-
-        packet_type_id = payload[i] & 0x3F
-        i += 1
-
-        while payload[i] >= 224 and payload[i] < 255:
-            # Partial body length.
-            partial_length = 1 << (payload[i] & 0x1F)
-            i += 1 + partial_length
-
-        if payload[i] < 192:
-            # One-octet length.
-            body_len = payload[i]
-            i += 1
-        elif payload[i] < 224:
-            # Two-octet length.
-            body_len = ((payload[i] - 192) << 8) + payload[i + 1] + 192
-            i += 2
-        elif payload[i] == 255:
-            # Five-octet length.
-            body_len = (
-                (payload[i + 1] << 24)
-                | (payload[i + 2] << 16)
-                | (payload[i + 3] << 8)
-                | payload[i + 4]
-            )
-            i += 5
-        else:
-            # Impossible, partial body length was processed above.
-            return False
-
-        i += body_len
-
-        if i == len(payload):
-            # Last packet should be
-            # Symmetrically Encrypted and Integrity Protected Data Packet (SEIPD)
-            #
-            # This is the only place where this function may return `True`.
-            return packet_type_id == 18
-        elif packet_type_id not in [1, 3]:
-            # All packets except the last one must be either
-            # Public-Key Encrypted Session Key Packet (PKESK)
-            # or
-            # Symmetric-Key Encrypted Session Key Packet (SKESK)
-            return False
-
-    return False
-
-
-def check_armored_payload(payload: str, outgoing: bool):
-    """Check the armored PGP message for invalid content.
-
-    :param payload: the armored PGP message
-    :param outgoing: whether the message is outgoing or incoming
-    :return: whether the message is a valid PGP message
-    """
-    prefix = "-----BEGIN PGP MESSAGE-----\r\n"
-    if not payload.startswith(prefix):
-        return False
-    payload = payload.removeprefix(prefix)
-
-    while payload.endswith("\r\n"):
-        payload = payload.removesuffix("\r\n")
-    suffix = "-----END PGP MESSAGE-----"
-    if not payload.endswith(suffix):
-        return False
-    payload = payload.removesuffix(suffix)
-
-    version_comment = "Version: "
-    if payload.startswith(version_comment):
-        if outgoing:  # Disallow comments in outgoing messages
-            return False
-        # Remove comments from incoming messages
-        payload = payload.partition("\r\n")[2]
-
-    while payload.startswith("\r\n"):
-        payload = payload.removeprefix("\r\n")
-
-    # Remove CRC24.
-    payload = payload.rpartition("=")[0]
-
-    try:
-        payload = base64.b64decode(payload)
-    except binascii.Error:
-        return False
-
-    try:
-        return check_openpgp_payload(payload)
-    except IndexError:
-        return False
-
-
-def is_securejoin(message):
-    if message.get("secure-join") not in ["vc-request", "vg-request"]:
-        return False
-    if not message.is_multipart():
-        return False
-    parts_count = 0
-    for part in message.iter_parts():
-        parts_count += 1
-        if parts_count > 1:
-            return False
-        if part.is_multipart():
-            return False
-        if part.get_content_type() != "text/plain":
-            return False
-
-        payload = part.get_payload().strip().lower()
-        if payload not in ("secure-join: vc-request", "secure-join: vg-request"):
-            return False
-    return True
-
-
-def check_encrypted(message, outgoing=True):
-    """Check that the message is an OpenPGP-encrypted message.
-
-    MIME structure of the message must correspond to <https://www.rfc-editor.org/rfc/rfc3156>.
-    """
-    if not message.is_multipart():
-        return False
-    if message.get_content_type() != "multipart/encrypted":
-        return False
-    parts_count = 0
-    for part in message.iter_parts():
-        # We explicitly check Content-Type of each part later,
-        # but this is to be absolutely sure `get_payload()` returns string and not list.
-        if part.is_multipart():
-            return False
-
-        if parts_count == 0:
-            if part.get_content_type() != "application/pgp-encrypted":
-                return False
-
-            payload = part.get_payload()
-            if payload.strip() != "Version: 1":
-                return False
-        elif parts_count == 1:
-            if part.get_content_type() != "application/octet-stream":
-                return False
-
-            if not check_armored_payload(part.get_payload(), outgoing=outgoing):
-                return False
-        else:
-            return False
-        parts_count += 1
-    return True
-
-
-async def asyncmain_beforequeue(config, mode):
-    if mode == "outgoing":
-        port = config.filtermail_smtp_port
-        handler = OutgoingBeforeQueueHandler(config)
-    else:
-        port = config.filtermail_smtp_port_incoming
-        handler = IncomingBeforeQueueHandler(config)
-    HackedController(
-        handler,
-        hostname="127.0.0.1",
-        port=port,
-        data_size_limit=config.max_message_size,
-    ).start()
-
-
-def recipient_matches_passthrough(recipient, passthrough_recipients):
-    for addr in passthrough_recipients:
-        if recipient == addr:
-            return True
-        if addr[0] == "@" and recipient.endswith(addr):
-            return True
-    return False
-
-
-class HackedController(Controller):
-    def factory(self):
-        return SMTPDiscardRCPTO_options(self.handler, **self.SMTP_kwargs)
-
-
-class SMTPDiscardRCPTO_options(SMTP):
-    def _getparams(self, params):
-        # Ignore RCPT TO parameters.
-        #
-        # Otherwise parameters such as `ORCPT=...`
-        # or `NOTIFY=DELAY,FAILURE` (generated by Stalwart)
-        # make aiosmtpd reject the message here:
-        # <https://github.com/aio-libs/aiosmtpd/blob/98f578389ae86e5345cc343fa4e5a17b21d9c96d/aiosmtpd/smtp.py#L1379-L1384>
-        return {}
-
-
-class OutgoingBeforeQueueHandler:
-    def __init__(self, config):
-        self.config = config
-        self.send_rate_limiter = SendRateLimiter()
-
-    async def handle_MAIL(self, server, session, envelope, address, mail_options):
-        log_info(f"handle_MAIL from {address}")
-        envelope.mail_from = address
-        max_sent = self.config.max_user_send_per_minute
-        if not self.send_rate_limiter.is_sending_allowed(address, max_sent):
-            return f"450 4.7.1: Too much mail from {address}"
-
-        parts = envelope.mail_from.split("@")
-        if len(parts) != 2:
-            return f"500 Invalid from address <{envelope.mail_from!r}>"
-
-        return "250 OK"
-
-    async def handle_DATA(self, server, session, envelope):
-        loop = asyncio.get_running_loop()
-        return await loop.run_in_executor(None, self.sync_handle_DATA, envelope)
-
-    def sync_handle_DATA(self, envelope):
-        log_info("handle_DATA before-queue")
-        error = self.check_DATA(envelope)
-        if error:
-            return error
-        log_info("re-injecting the mail that passed checks")
-        client = SMTPClient("localhost", self.config.postfix_reinject_port)
-        client.sendmail(
-            envelope.mail_from, envelope.rcpt_tos, envelope.original_content
-        )
-        return "250 OK"
-
-    def check_DATA(self, envelope):
-        """the central filtering function for e-mails."""
-        log_info(f"Processing DATA message from {envelope.mail_from}")
-
-        message = BytesParser(policy=policy.default).parsebytes(envelope.content)
-        mail_encrypted = check_encrypted(message, outgoing=True)
-
-        _, from_addr = parseaddr(message.get("from").strip())
-
-        if envelope.mail_from.lower() != from_addr.lower():
-            return f"500 Invalid FROM <{from_addr!r}> for <{envelope.mail_from!r}>"
-
-        if mail_encrypted or is_securejoin(message):
-            print("Outgoing: Filtering encrypted mail.", file=sys.stderr)
-            return
-
-        print("Outgoing: Filtering unencrypted mail.", file=sys.stderr)
-
-        if envelope.mail_from in self.config.passthrough_senders:
-            return
-
-        # allow self-sent Autocrypt Setup Message
-        if envelope.rcpt_tos == [from_addr]:
-            if message.get("subject") == "Autocrypt Setup Message":
-                if message.get_content_type() == "multipart/mixed":
-                    return
-
-        passthrough_recipients = self.config.passthrough_recipients
-
-        for recipient in envelope.rcpt_tos:
-            if recipient_matches_passthrough(recipient, passthrough_recipients):
-                continue
-
-            print("Rejected unencrypted mail.", file=sys.stderr)
-            return ENCRYPTION_NEEDED_523
-
-
-class IncomingBeforeQueueHandler:
-    def __init__(self, config):
-        self.config = config
-
-    async def handle_DATA(self, server, session, envelope):
-        loop = asyncio.get_running_loop()
-        return await loop.run_in_executor(None, self.sync_handle_DATA, envelope)
-
-    def sync_handle_DATA(self, envelope):
-        log_info("handle_DATA before-queue")
-        error = self.check_DATA(envelope)
-        if error:
-            return error
-        log_info("re-injecting the mail that passed checks")
-
-        # the smtp daemon on reinject_port_incoming gives it to dkim milter
-        # which looks at source address to determine whether to verify or sign
-        client = SMTPClient(
-            "localhost",
-            self.config.postfix_reinject_port_incoming,
-            source_address=("127.0.0.2", 0),
-        )
-        client.sendmail(
-            envelope.mail_from, envelope.rcpt_tos, envelope.original_content
-        )
-        return "250 OK"
-
-    def check_DATA(self, envelope):
-        """the central filtering function for e-mails."""
-        log_info(f"Processing DATA message from {envelope.mail_from}")
-
-        message = BytesParser(policy=policy.default).parsebytes(envelope.content)
-        mail_encrypted = check_encrypted(message, outgoing=False)
-
-        if mail_encrypted or is_securejoin(message):
-            print("Incoming: Filtering encrypted mail.", file=sys.stderr)
-            return
-
-        print("Incoming: Filtering unencrypted mail.", file=sys.stderr)
-
-        # we want cleartext mailer-daemon messages to pass through
-        # chatmail core will typically not display them as normal messages
-        if message.get("auto-submitted"):
-            _, from_addr = parseaddr(message.get("from").strip())
-            if from_addr.lower().startswith("mailer-daemon@"):
-                if message.get_content_type() == "multipart/report":
-                    return
-
-        for recipient in envelope.rcpt_tos:
-            user = self.config.get_user(recipient)
-            if user is None or user.is_incoming_cleartext_ok():
-                continue
-
-            print("Rejected unencrypted mail.", file=sys.stderr)
-            return ENCRYPTION_NEEDED_523
-
-
-class SendRateLimiter:
-    def __init__(self):
-        self.addr2timestamps = {}
-
-    def is_sending_allowed(self, mail_from, max_send_per_minute):
-        last = self.addr2timestamps.setdefault(mail_from, [])
-        now = time.time()
-        last[:] = [ts for ts in last if ts >= (now - 60)]
-        if len(last) <= max_send_per_minute:
-            last.append(now)
-            return True
-        return False
-
-
-def log_info(msg):
-    print(msg, file=sys.stderr)
-
-
-def main():
-    args = sys.argv[1:]
-    assert len(args) == 2
-    config = read_config(args[0])
-    mode = args[1]
-    loop = asyncio.new_event_loop()
-    asyncio.set_event_loop(loop)
-    assert mode in ["incoming", "outgoing"]
-    task = asyncmain_beforequeue(config, mode)
-    loop.create_task(task)
-    log_info("entering serving loop")
-    loop.run_forever()

chatmaild/src/chatmaild/fsreport.py

@@ -13,9 +13,20 @@ to show storage summaries only for first 1000 mailboxes
 
     python -m chatmaild.fsreport /path/to/chatmail.ini --maxnum 1000
 
+to write Prometheus textfile for node_exporter
+
+    python -m chatmaild.fsreport --textfile /var/lib/prometheus/node-exporter/
+
+writes to /var/lib/prometheus/node-exporter/fsreport.prom
+
+to also write legacy metrics.py style output (default: /var/www/html/metrics):
+
+    python -m chatmaild.fsreport --textfile /var/lib/prometheus/node-exporter/ --legacy-metrics
+
 """
 
 import os
+import tempfile
 from argparse import ArgumentParser
 from datetime import datetime
 
@@ -48,7 +59,19 @@ class Report:
         self.num_ci_logins = self.num_all_logins = 0
         self.login_buckets = {x: 0 for x in (1, 10, 30, 40, 80, 100, 150)}
 
-        self.message_buckets = {x: 0 for x in (0, 160000, 500000, 2000000)}
+        KiB = 1024
+        MiB = 1024 * KiB
+        self.message_size_thresholds = (
+            0,
+            100 * KiB,
+            MiB // 2,
+            1 * MiB,
+            2 * MiB,
+            5 * MiB,
+            10 * MiB,
+        )
+        self.message_buckets = {x: 0 for x in self.message_size_thresholds}
+        self.message_count_buckets = {x: 0 for x in self.message_size_thresholds}
 
     def process_mailbox_stat(self, mailbox):
         # categorize login times
@@ -68,9 +91,10 @@ class Report:
             for size in self.message_buckets:
                 for msg in mailbox.messages:
                     if msg.size >= size:
-                        if self.mdir and not msg.relpath.startswith(self.mdir):
+                        if self.mdir and f"/{self.mdir}/" not in msg.path:
                             continue
                         self.message_buckets[size] += msg.size
+                        self.message_count_buckets[size] += 1
 
         self.size_messages += sum(entry.size for entry in mailbox.messages)
         self.size_extra += sum(entry.size for entry in mailbox.extrafiles)
@@ -93,9 +117,10 @@ class Report:
 
         pref = f"[{self.mdir}] " if self.mdir else ""
         for minsize, sumsize in self.message_buckets.items():
+            count = self.message_count_buckets[minsize]
             percent = (sumsize / all_messages * 100) if all_messages else 0
             print(
-                f"{pref}larger than {HSize(minsize)}: {HSize(sumsize)} ({percent:.2f}%)"
+                f"{pref}larger than {HSize(minsize)}: {HSize(sumsize)} ({percent:.2f}%), {count} msgs"
             )
 
         user_logins = self.num_all_logins - self.num_ci_logins
@@ -111,6 +136,75 @@ class Report:
         for days, active in self.login_buckets.items():
             print(f"last {days:3} days: {HSize(active)} {p(active)}")
 
+    def _write_atomic(self, filepath, content):
+        """Atomically write content to filepath via tmp+rename."""
+        dirpath = os.path.dirname(os.path.abspath(filepath))
+        fd, tmppath = tempfile.mkstemp(dir=dirpath, suffix=".tmp")
+        try:
+            with os.fdopen(fd, "w") as f:
+                f.write(content)
+            os.chmod(tmppath, 0o644)
+            os.rename(tmppath, filepath)
+        except BaseException:
+            try:
+                os.unlink(tmppath)
+            except OSError:
+                pass
+            raise
+
+    def dump_textfile(self, filepath):
+        """Dump metrics in Prometheus exposition format."""
+        lines = []
+
+        lines.append("# HELP chatmail_storage_bytes Mailbox storage in bytes.")
+        lines.append("# TYPE chatmail_storage_bytes gauge")
+        lines.append(f'chatmail_storage_bytes{{kind="messages"}} {self.size_messages}')
+        lines.append(f'chatmail_storage_bytes{{kind="extra"}} {self.size_extra}')
+        total = self.size_extra + self.size_messages
+        lines.append(f'chatmail_storage_bytes{{kind="total"}} {total}')
+
+        lines.append("# HELP chatmail_messages_bytes Sum of msg bytes >= threshold.")
+        lines.append("# TYPE chatmail_messages_bytes gauge")
+        for minsize, sumsize in self.message_buckets.items():
+            lines.append(f'chatmail_messages_bytes{{min_size="{minsize}"}} {sumsize}')
+
+        lines.append("# HELP chatmail_messages_count Number of msgs >= size threshold.")
+        lines.append("# TYPE chatmail_messages_count gauge")
+        for minsize, count in self.message_count_buckets.items():
+            lines.append(f'chatmail_messages_count{{min_size="{minsize}"}} {count}')
+
+        lines.append("# HELP chatmail_accounts Number of accounts.")
+        lines.append("# TYPE chatmail_accounts gauge")
+        user_logins = self.num_all_logins - self.num_ci_logins
+        lines.append(f'chatmail_accounts{{kind="all"}} {self.num_all_logins}')
+        lines.append(f'chatmail_accounts{{kind="ci"}} {self.num_ci_logins}')
+        lines.append(f'chatmail_accounts{{kind="user"}} {user_logins}')
+
+        lines.append(
+            "# HELP chatmail_accounts_active Non-CI accounts active within N days."
+        )
+        lines.append("# TYPE chatmail_accounts_active gauge")
+        for days, active in self.login_buckets.items():
+            lines.append(f'chatmail_accounts_active{{days="{days}"}} {active}')
+
+        self._write_atomic(filepath, "\n".join(lines) + "\n")
+
+    def dump_compat_textfile(self, filepath):
+        """Dump legacy metrics.py style metrics."""
+        user_logins = self.num_all_logins - self.num_ci_logins
+        lines = [
+            "# HELP total number of accounts",
+            "# TYPE accounts gauge",
+            f"accounts {self.num_all_logins}",
+            "# HELP number of CI accounts",
+            "# TYPE ci_accounts gauge",
+            f"ci_accounts {self.num_ci_logins}",
+            "# HELP number of non-CI accounts",
+            "# TYPE nonci_accounts gauge",
+            f"nonci_accounts {user_logins}",
+        ]
+        self._write_atomic(filepath, "\n".join(lines) + "\n")
+
 
 def main(args=None):
     """Report about filesystem storage usage of all mailboxes and messages"""
@@ -127,19 +221,21 @@ def main(args=None):
         "--days",
         default=0,
         action="store",
-        help="assume date to be days older than now",
+        help="assume date to be DAYS older than now",
     )
     parser.add_argument(
         "--min-login-age",
         default=0,
+        metavar="DAYS",
         dest="min_login_age",
         action="store",
-        help="only sum up message size if last login is at least min-login-age days old",
+        help="only sum up message size if last login is at least DAYS days old",
     )
     parser.add_argument(
         "--mdir",
+        metavar="{cur,new,tmp}",
         action="store",
-        help="only consider 'cur' or 'new' or 'tmp' messages for summary",
+        help="only consider messages in specified Maildir subdirectory for summary",
     )
 
     parser.add_argument(
@@ -148,6 +244,21 @@ def main(args=None):
         action="store",
         help="maximum number of mailboxes to iterate on",
     )
+    parser.add_argument(
+        "--textfile",
+        metavar="PATH",
+        default=None,
+        help="write Prometheus textfile to PATH (directory or file); "
+        "if PATH is a directory, writes 'fsreport.prom' inside it",
+    )
+    parser.add_argument(
+        "--legacy-metrics",
+        metavar="FILENAME",
+        nargs="?",
+        const="/var/www/html/metrics",
+        default=None,
+        help="write legacy metrics.py textfile (default: /var/www/html/metrics)",
+    )
 
     args = parser.parse_args(args)
 
@@ -161,7 +272,15 @@ def main(args=None):
     rep = Report(now=now, min_login_age=int(args.min_login_age), mdir=args.mdir)
     for mbox in iter_mailboxes(str(config.mailboxes_dir), maxnum=maxnum):
         rep.process_mailbox_stat(mbox)
-    rep.dump_summary()
+    if args.textfile:
+        path = args.textfile
+        if os.path.isdir(path):
+            path = os.path.join(path, "fsreport.prom")
+        rep.dump_textfile(path)
+    if args.legacy_metrics:
+        rep.dump_compat_textfile(args.legacy_metrics)
+    if not args.textfile and not args.legacy_metrics:
+        rep.dump_summary()
 
 
 if __name__ == "__main__":

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -11,11 +11,15 @@ mail_domain = {mail_domain}
 # Restrictions on user addresses
 #
 
-# how many mails a user can send out per minute
+# email sending rate per user and minute
 max_user_send_per_minute = 60
 
+# per-user max burst size for sending rate limiting (GCRA bucket capacity)
+max_user_send_burst_size = 10
+
 # maximum mailbox size of a chatmail address
-max_mailbox_size = 100M
+# Oldest messages will be removed automatically, so mailboxes never run full.
+max_mailbox_size = 500M
 
 # maximum message size for an e-mail in bytes
 max_message_size = 31457280
@@ -45,6 +49,13 @@ passthrough_senders =
 # (space-separated, item may start with "@" to whitelist whole recipient domains)
 passthrough_recipients =
 
+# Use externally managed TLS certificates instead of built-in acmetool.
+# Paths refer to files on the deployment server (not the build machine).
+# Both files must already exist before running cmdeploy.
+# Certificate renewal is your responsibility; changed files are
+# picked up automatically by all relay services.
+# tls_external_cert_and_key = /path/to/fullchain.pem /path/to/privkey.pem
+
 # path to www directory - documented here: https://chatmail.at/doc/relay/getting_started.html#custom-web-pages
 #www_folder = www
 

chatmaild/src/chatmaild/metadata.py

@@ -101,7 +101,11 @@ class MetadataDictProxy(DictProxy):
                 # Handle `GETMETADATA "" /shared/vendor/deltachat/irohrelay`
                 return f"O{self.iroh_relay}\n"
             elif keyname == "vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn":
-                res = turn_credentials()
+                try:
+                    res = turn_credentials()
+                except Exception:
+                    logging.exception("failed to get TURN credentials")
+                    return "N\n"
                 port = 3478
                 return f"O{self.turn_hostname}:{port}:{res}\n"
 

chatmaild/src/chatmaild/metrics.py

@@ -1,32 +0,0 @@
-#!/usr/bin/env python3
-import sys
-from pathlib import Path
-
-
-def main(vmail_dir=None):
-    if vmail_dir is None:
-        vmail_dir = sys.argv[1]
-
-    accounts = 0
-    ci_accounts = 0
-
-    for path in Path(vmail_dir).iterdir():
-        if not path.joinpath("cur").is_dir():
-            continue
-        accounts += 1
-        if path.name[:3] in ("ci-", "ac_"):
-            ci_accounts += 1
-
-    print("# HELP total number of accounts")
-    print("# TYPE accounts gauge")
-    print(f"accounts {accounts}")
-    print("# HELP number of CI accounts")
-    print("# TYPE ci_accounts gauge")
-    print(f"ci_accounts {ci_accounts}")
-    print("# HELP number of non-CI accounts")
-    print("# TYPE nonci_accounts gauge")
-    print(f"nonci_accounts {accounts - ci_accounts}")
-
-
-if __name__ == "__main__":
-    main()

chatmaild/src/chatmaild/newemail.py

@@ -2,10 +2,11 @@
 
 """CGI script for creating new accounts."""
 
+import ipaddress
 import json
-import random
 import secrets
 import string
+from urllib.parse import quote
 
 from chatmaild.config import Config, read_config
 
@@ -14,22 +15,47 @@ ALPHANUMERIC = string.ascii_lowercase + string.digits
 ALPHANUMERIC_PUNCT = string.ascii_letters + string.digits + string.punctuation
 
 
+def wrap_ip(host):
+    if host.startswith("[") and host.endswith("]"):
+        return host
+    try:
+        ipaddress.ip_address(host)
+        return f"[{host}]"
+    except ValueError:
+        return host
+
+
 def create_newemail_dict(config: Config):
-    user = "".join(random.choices(ALPHANUMERIC, k=config.username_max_length))
+    user = "".join(
+        secrets.choice(ALPHANUMERIC) for _ in range(config.username_max_length)
+    )
     password = "".join(
         secrets.choice(ALPHANUMERIC_PUNCT)
         for _ in range(config.password_min_length + 3)
     )
-    return dict(email=f"{user}@{config.mail_domain}", password=f"{password}")
+    return dict(email=f"{user}@{wrap_ip(config.mail_domain)}", password=f"{password}")
+
+
+def create_dclogin_url(email, password):
+    """Build a dclogin: URL with credentials and self-signed cert acceptance.
+
+    Uses ic=3 (AcceptInvalidCertificates) so chatmail clients
+    can connect to servers with self-signed TLS certificates.
+    """
+    return f"dclogin:{quote(email, safe='@')}?p={quote(password, safe='')}&v=1&ic=3"
 
 
 def print_new_account():
     config = read_config(CONFIG_PATH)
     creds = create_newemail_dict(config)
 
+    result = dict(email=creds["email"], password=creds["password"])
+    if config.tls_cert_mode == "self":
+        result["dclogin_url"] = create_dclogin_url(creds["email"], creds["password"])
+
     print("Content-Type: application/json")
     print("")
-    print(json.dumps(creds))
+    print(json.dumps(result))
 
 
 if __name__ == "__main__":

chatmaild/src/chatmaild/tests/test_config.py

@@ -1,6 +1,6 @@
 import pytest
 
-from chatmaild.config import read_config
+from chatmaild.config import parse_size_mb, read_config
 
 
 def test_read_config_basic(example_config):
@@ -33,7 +33,7 @@ def test_read_config_testrun(make_config):
     assert config.filtermail_smtp_port == 10080
     assert config.postfix_reinject_port == 10025
     assert config.max_user_send_per_minute == 60
-    assert config.max_mailbox_size == "100M"
+    assert config.max_mailbox_size == "500M"
     assert config.delete_mails_after == "20"
     assert config.delete_large_after == "7"
     assert config.username_min_length == 9
@@ -73,3 +73,65 @@ def test_config_userstate_paths(make_config, tmp_path):
 def test_config_max_message_size(make_config, tmp_path):
     config = make_config("something.testrun.org", dict(max_message_size="10000"))
     assert config.max_message_size == 10000
+
+
+def test_config_tls_default_acme(make_config):
+    config = make_config("chat.example.org")
+    assert config.tls_cert_mode == "acme"
+    assert config.tls_cert_path == "/var/lib/acme/live/chat.example.org/fullchain"
+    assert config.tls_key_path == "/var/lib/acme/live/chat.example.org/privkey"
+
+
+def test_config_tls_self(make_config):
+    config = make_config("_test.example.org")
+    assert config.tls_cert_mode == "self"
+    assert config.tls_cert_path == "/etc/ssl/certs/mailserver.pem"
+    assert config.tls_key_path == "/etc/ssl/private/mailserver.key"
+
+
+def test_config_tls_external(make_config):
+    config = make_config(
+        "chat.example.org",
+        {
+            "tls_external_cert_and_key": "/custom/fullchain.pem /custom/privkey.pem",
+        },
+    )
+    assert config.tls_cert_mode == "external"
+    assert config.tls_cert_path == "/custom/fullchain.pem"
+    assert config.tls_key_path == "/custom/privkey.pem"
+
+
+def test_config_tls_external_overrides_underscore(make_config):
+    config = make_config(
+        "_test.example.org",
+        {
+            "tls_external_cert_and_key": "/certs/fullchain.pem /certs/privkey.pem",
+        },
+    )
+    assert config.tls_cert_mode == "external"
+    assert config.tls_cert_path == "/certs/fullchain.pem"
+    assert config.tls_key_path == "/certs/privkey.pem"
+
+
+def test_config_tls_external_bad_format(make_config):
+    with pytest.raises(ValueError, match="two space-separated"):
+        make_config(
+            "chat.example.org",
+            {
+                "tls_external_cert_and_key": "/only/one/path.pem",
+            },
+        )
+
+
+def test_parse_size_mb():
+    assert parse_size_mb("500M") == 500
+    assert parse_size_mb("2G") == 2048
+    assert parse_size_mb("  1g  ") == 1024
+    assert parse_size_mb("100MB") == 100
+    assert parse_size_mb("256") == 256
+
+
+def test_max_mailbox_size_mb(make_config):
+    config = make_config("chat.example.org")
+    assert config.max_mailbox_size == "500M"
+    assert config.max_mailbox_size_mb == 500

chatmaild/src/chatmaild/tests/test_delete_inactive_users.py

@@ -1,7 +1,7 @@
 import time
 
 from chatmaild.doveauth import AuthDictProxy
-from chatmaild.expire import main as main_expire
+from chatmaild.expire import daily_expire_main as main_expire
 
 
 def test_login_timestamps(example_config):

chatmaild/src/chatmaild/tests/test_doveauth.py

@@ -120,6 +120,60 @@ def test_handle_dovecot_protocol_iterate(gencreds, example_config):
     assert not lines[2]
 
 
+def test_invalid_localpart_characters(make_config):
+    """Test that is_allowed_to_create rejects localparts with invalid characters."""
+    config = make_config("chat.example.org", {"username_min_length": "3"})
+    password = "zequ0Aimuchoodaechik"
+    domain = config.mail_domain
+
+    # valid localparts
+    assert is_allowed_to_create(config, f"abc123@{domain}", password)
+    assert is_allowed_to_create(config, f"a.b-c_d@{domain}", password)
+
+    # uppercase rejected
+    assert not is_allowed_to_create(config, f"Abc123@{domain}", password)
+    assert not is_allowed_to_create(config, f"ABCDEFG@{domain}", password)
+
+    # spaces and special chars rejected
+    assert not is_allowed_to_create(config, f"a b cde@{domain}", password)
+    assert not is_allowed_to_create(config, f"abc+def@{domain}", password)
+    assert not is_allowed_to_create(config, f"abc!def@{domain}", password)
+    assert not is_allowed_to_create(config, f"ab@cdef@{domain}", password)
+    assert not is_allowed_to_create(config, f"abc/def@{domain}", password)
+    assert not is_allowed_to_create(config, f"abc\\def@{domain}", password)
+
+
+def test_concurrent_creation_same_account(dictproxy):
+    """Test that concurrent creation of the same account doesn't corrupt password."""
+    addr = "racetest1@chat.example.org"
+    password = "zequ0Aimuchoodaechik"
+    num_threads = 10
+    results = queue.Queue()
+
+    def create():
+        try:
+            res = dictproxy.lookup_passdb(addr, password)
+            results.put(("ok", res))
+        except Exception:
+            results.put(("err", traceback.format_exc()))
+
+    threads = [threading.Thread(target=create, daemon=True) for _ in range(num_threads)]
+    for t in threads:
+        t.start()
+    for t in threads:
+        t.join(timeout=10)
+
+    passwords_seen = set()
+    for _ in range(num_threads):
+        status, res = results.get()
+        if status == "err":
+            pytest.fail(f"concurrent creation failed\n{res}")
+        passwords_seen.add(res["password"])
+
+    # all threads must see the same password hash
+    assert len(passwords_seen) == 1
+
+
 def test_50_concurrent_lookups_different_accounts(gencreds, dictproxy):
     num_threads = 50
     req_per_thread = 5

chatmaild/src/chatmaild/tests/test_expire.py

@@ -1,5 +1,7 @@
+import itertools
 import os
 import random
+import time
 from datetime import datetime
 from fnmatch import fnmatch
 from pathlib import Path
@@ -9,13 +11,19 @@ import pytest
 from chatmaild.expire import (
     FileEntry,
     MailboxStat,
+    expire_to_target,
     get_file_entry,
     iter_mailboxes,
     os_listdir_if_exists,
+    parse_dovecot_filename,
+    quota_expire_main,
+    scan_mailbox_messages,
 )
-from chatmaild.expire import main as expiry_main
+from chatmaild.expire import daily_expire_main as expiry_main
 from chatmaild.fsreport import main as report_main
 
+MB = 1024 * 1024
+
 
 def fill_mbox(folderdir):
     password = folderdir.joinpath("password")
@@ -112,6 +120,43 @@ def test_report(mbox1, example_config):
     report_main(args)
 
 
+def test_report_mdir_filters_by_path(mbox1, example_config):
+    """Test that Report with mdir='cur' only counts messages in cur/ subdirectory."""
+    from chatmaild.fsreport import Report
+
+    now = datetime.utcnow().timestamp()
+
+    # Set password mtime to old enough so min_login_age check passes
+    password = Path(mbox1.basedir).joinpath("password")
+    old_time = now - 86400 * 10  # 10 days ago
+    os.utime(password, (old_time, old_time))
+
+    # Reload mailbox with updated mtime
+    from chatmaild.expire import MailboxStat
+
+    mbox = MailboxStat(mbox1.basedir)
+
+    # Report without mdir — should count all messages
+    rep_all = Report(now=now, min_login_age=1, mdir=None)
+    rep_all.process_mailbox_stat(mbox)
+    total_all = rep_all.message_buckets[0]
+
+    # Report with mdir='cur' — should only count cur/ messages
+    rep_cur = Report(now=now, min_login_age=1, mdir="cur")
+    rep_cur.process_mailbox_stat(mbox)
+    total_cur = rep_cur.message_buckets[0]
+
+    # Report with mdir='new' — should only count new/ messages
+    rep_new = Report(now=now, min_login_age=1, mdir="new")
+    rep_new.process_mailbox_stat(mbox)
+    total_new = rep_new.message_buckets[0]
+
+    # cur has 500-byte msg, new has 600-byte msg (from fill_mbox)
+    assert total_cur == 500
+    assert total_new == 600
+    assert total_all == 500 + 600
+
+
 def test_expiry_cli_basic(example_config, mbox1):
     args = (str(example_config._inipath),)
     expiry_main(args)
@@ -159,3 +204,51 @@ def test_os_listdir_if_exists(tmp_path):
     tmp_path.joinpath("x").write_text("hello")
     assert len(os_listdir_if_exists(str(tmp_path))) == 1
     assert len(os_listdir_if_exists(str(tmp_path.joinpath("123123")))) == 0
+
+
+# --- quota expire tests ---
+
+_msg_counter = itertools.count(1)
+
+
+def _create_message(basedir, sub, size, days_old=0, disk_size=None):
+    seq = next(_msg_counter)
+    mtime = int(time.time() - days_old * 86400)
+    name = f"{mtime}.M1P1Q{seq}.hostname,S={size},W={size}:2,S"
+    path = basedir / sub / name
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_bytes(b"x" * (disk_size if disk_size is not None else size))
+    os.utime(path, (mtime, mtime))
+    return path
+
+
+def test_parse_dovecot_filename():
+    e = parse_dovecot_filename("cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S")
+    assert e.path == "cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S"
+    assert e.mtime == 1775324677
+    assert e.quota_size == 3235
+    assert parse_dovecot_filename("cur/msg_without_structure") is None
+
+
+def test_expire_to_target(tmp_path):
+    _create_message(tmp_path, "cur", MB, days_old=10, disk_size=100)
+    _create_message(tmp_path, "new", MB, days_old=5)
+    _create_message(tmp_path, "cur", MB, days_old=0)  # undeletable (<1 hour)
+    assert len(scan_mailbox_messages(tmp_path)) == 3
+    # removes oldest first, uses S= size not disk size
+    removed = expire_to_target(tmp_path, MB)
+    assert removed == 2
+    msgs = scan_mailbox_messages(tmp_path)
+    assert len(msgs) == 1
+    # the surviving message is the fresh undeletable one
+    assert msgs[0].mtime > time.time() - 3600
+
+
+def test_quota_expire_main(tmp_path, capsys):
+    mbox = tmp_path / "user@example.org"
+    _create_message(mbox, "cur", 2 * MB, days_old=5)
+    (mbox / "maildirsize").write_text("x")
+    quota_expire_main([str(1), str(mbox)])
+    _, err = capsys.readouterr()
+    assert "quota-expire: removed 1 message(s) from user@example.org" in err
+    assert not (mbox / "maildirsize").exists()

chatmaild/src/chatmaild/tests/test_filtermail.py

@@ -1,361 +0,0 @@
-import pytest
-
-from chatmaild.filtermail import (
-    IncomingBeforeQueueHandler,
-    OutgoingBeforeQueueHandler,
-    SendRateLimiter,
-    check_armored_payload,
-    check_encrypted,
-    is_securejoin,
-)
-
-
-@pytest.fixture
-def maildomain():
-    # let's not depend on a real chatmail instance for the offline tests below
-    return "chatmail.example.org"
-
-
-@pytest.fixture
-def handler(make_config, maildomain):
-    config = make_config(maildomain)
-    return OutgoingBeforeQueueHandler(config)
-
-
-@pytest.fixture
-def inhandler(make_config, maildomain):
-    config = make_config(maildomain)
-    return IncomingBeforeQueueHandler(config)
-
-
-def test_reject_forged_from(maildata, gencreds, handler):
-    class env:
-        mail_from = gencreds()[0]
-        rcpt_tos = [gencreds()[0]]
-
-    # test that the filter lets good mail through
-    to_addr = gencreds()[0]
-    env.content = maildata(
-        "encrypted.eml", from_addr=env.mail_from, to_addr=to_addr
-    ).as_bytes()
-
-    assert not handler.check_DATA(envelope=env)
-
-    # test that the filter rejects forged mail
-    env.content = maildata(
-        "encrypted.eml", from_addr="forged@c3.testrun.org", to_addr=to_addr
-    ).as_bytes()
-    error = handler.check_DATA(envelope=env)
-    assert "500" in error
-
-
-def test_filtermail_no_encryption_detection(maildata):
-    msg = maildata(
-        "plain.eml", from_addr="some@example.org", to_addr="other@example.org"
-    )
-    assert not check_encrypted(msg)
-
-    # https://xkcd.com/1181/
-    msg = maildata(
-        "fake-encrypted.eml", from_addr="some@example.org", to_addr="other@example.org"
-    )
-    assert not check_encrypted(msg)
-
-
-def test_filtermail_securejoin_detection(maildata):
-    msg = maildata(
-        "securejoin-vc.eml", from_addr="some@example.org", to_addr="other@example.org"
-    )
-    assert is_securejoin(msg)
-
-    msg = maildata(
-        "securejoin-vc-fake.eml",
-        from_addr="some@example.org",
-        to_addr="other@example.org",
-    )
-    assert not is_securejoin(msg)
-
-
-def test_filtermail_encryption_detection(maildata):
-    msg = maildata(
-        "encrypted.eml",
-        from_addr="1@example.org",
-        to_addr="2@example.org",
-        subject="Subject does not matter, will be replaced anyway",
-    )
-    assert check_encrypted(msg)
-
-
-def test_filtermail_no_literal_packets(maildata):
-    """Test that literal OpenPGP packet is not considered an encrypted mail."""
-    msg = maildata("literal.eml", from_addr="1@example.org", to_addr="2@example.org")
-    assert not check_encrypted(msg)
-
-
-def test_filtermail_unencrypted_mdn(maildata, gencreds):
-    """Unencrypted MDNs should not pass."""
-    from_addr = gencreds()[0]
-    to_addr = gencreds()[0] + ".other"
-    msg = maildata("mdn.eml", from_addr=from_addr, to_addr=to_addr)
-
-    assert not check_encrypted(msg)
-
-
-def test_send_rate_limiter():
-    limiter = SendRateLimiter()
-    for i in range(100):
-        if limiter.is_sending_allowed("some@example.org", 10):
-            if i <= 10:
-                continue
-            pytest.fail("limiter didn't work")
-        else:
-            assert i == 11
-            break
-
-
-def test_cleartext_excempt_privacy(maildata, gencreds, handler):
-    from_addr = gencreds()[0]
-    to_addr = "privacy@testrun.org"
-    handler.config.passthrough_recipients = [to_addr]
-    false_to = "privacy@something.org"
-
-    msg = maildata("plain.eml", from_addr=from_addr, to_addr=to_addr)
-
-    class env:
-        mail_from = from_addr
-        rcpt_tos = [to_addr]
-        content = msg.as_bytes()
-
-    # assert that None/no error is returned
-    assert not handler.check_DATA(envelope=env)
-
-    class env2:
-        mail_from = from_addr
-        rcpt_tos = [to_addr, false_to]
-        content = msg.as_bytes()
-
-    assert "523" in handler.check_DATA(envelope=env2)
-
-
-def test_cleartext_self_send_autocrypt_setup_message(maildata, gencreds, handler):
-    from_addr = gencreds()[0]
-    to_addr = from_addr
-
-    msg = maildata("asm.eml", from_addr=from_addr, to_addr=to_addr)
-
-    class env:
-        mail_from = from_addr
-        rcpt_tos = [to_addr]
-        content = msg.as_bytes()
-
-    assert not handler.check_DATA(envelope=env)
-
-
-def test_cleartext_send_fails(maildata, gencreds, handler):
-    from_addr = gencreds()[0]
-    to_addr = gencreds()[0]
-
-    msg = maildata("plain.eml", from_addr=from_addr, to_addr=to_addr)
-
-    class env:
-        mail_from = from_addr
-        rcpt_tos = [to_addr]
-        content = msg.as_bytes()
-
-    res = handler.check_DATA(envelope=env)
-    assert "523 Encryption Needed" in res
-
-
-def test_cleartext_incoming_fails(maildata, gencreds, inhandler):
-    from_addr = gencreds()[0]
-    to_addr, password = gencreds()
-
-    msg = maildata("plain.eml", from_addr=from_addr, to_addr=to_addr)
-
-    class env:
-        mail_from = from_addr
-        rcpt_tos = [to_addr]
-        content = msg.as_bytes()
-
-    user = inhandler.config.get_user(to_addr)
-    user.set_password(password)
-    res = inhandler.check_DATA(envelope=env)
-    assert "523 Encryption Needed" in res
-
-    user.allow_incoming_cleartext()
-    assert not inhandler.check_DATA(envelope=env)
-
-
-def test_cleartext_incoming_mailer_daemon(maildata, gencreds, inhandler):
-    from_addr = "mailer-daemon@example.org"
-    to_addr = gencreds()[0]
-
-    msg = maildata("mailer-daemon.eml", from_addr=from_addr, to_addr=to_addr)
-
-    class env:
-        mail_from = from_addr
-        rcpt_tos = [to_addr]
-        content = msg.as_bytes()
-
-    assert not inhandler.check_DATA(envelope=env)
-
-
-def test_cleartext_passthrough_domains(maildata, gencreds, handler):
-    from_addr = gencreds()[0]
-    to_addr = "privacy@x.y.z"
-    handler.config.passthrough_recipients = ["@x.y.z"]
-    false_to = "something@x.y"
-
-    msg = maildata("plain.eml", from_addr=from_addr, to_addr=to_addr)
-
-    class env:
-        mail_from = from_addr
-        rcpt_tos = [to_addr]
-        content = msg.as_bytes()
-
-    # assert that None/no error is returned
-    assert not handler.check_DATA(envelope=env)
-
-    class env2:
-        mail_from = from_addr
-        rcpt_tos = [to_addr, false_to]
-        content = msg.as_bytes()
-
-    assert "523" in handler.check_DATA(envelope=env2)
-
-
-def test_cleartext_passthrough_senders(gencreds, handler, maildata):
-    acc1 = gencreds()[0]
-    to_addr = "recipient@something.org"
-    handler.config.passthrough_senders = [acc1]
-
-    msg = maildata("plain.eml", from_addr=acc1, to_addr=to_addr)
-
-    class env:
-        mail_from = acc1
-        rcpt_tos = to_addr
-        content = msg.as_bytes()
-
-    # assert that None/no error is returned
-    assert not handler.check_DATA(envelope=env)
-
-
-def test_check_armored_payload():
-    prefix = "-----BEGIN PGP MESSAGE-----\r\n"
-    comment = "Version: ProtonMail\r\n"
-    payload = """\r
-wU4DSqFx0d1yqAoSAQdAYkX/ZN/Az4B0k7X47zKyWrXxlDEdS3WOy0Yf2+GJTFgg\r
-Zk5ql0mLG8Ze+ZifCS0XMO4otlemSyJ0K1ZPdFMGzUDBTgNqzkFabxXoXRIBB0AM\r
-755wlX41X6Ay3KhnwBq7yEqSykVH6F3x11iHPKraLCAGZoaS8bKKNy/zg5slda1X\r
-pt14b4aC1VwtSnYhcRRELNLD/wE2TFif+g7poMmFY50VyMPLYjVP96Z5QCT4+z4H\r
-Ikh/pRRN8S3JNMrRJHc6prooSJmLcx47Y5un7VFy390MsJ+LiUJuQMDdYWRAinfs\r
-Ebm89Ezjm7F03qbFPXE0X4ZNzVXS/eKO0uhJQdiov/vmbn41rNtHmNpqjaO0vi5+\r
-sS9tR7yDUrIXiCUCN78eBLVioxtktsPZm5cDORbQWzv+7nmCEz9/JowCUcBVdCGn\r
-1ofOaH82JCAX/cRx08pLaDNj6iolVBsi56Dd+2bGxJOZOG2AMcEyz0pXY0dOAJCD\r
-iUThcQeGIdRnU3j8UBcnIEsjLu2+C+rrwMZQESMWKnJ0rnqTk0pK5kXScr6F/L0L\r
-UE49ccIexNm3xZvYr5drszr6wz3Tv5fdue87P4etBt90gF/Vzknck+g1LLlkzZkp\r
-d8dI0k2tOSPjUbDPnSy1x+X73WGpPZmj0kWT+RGvq0nH6UkJj3AQTG2qf1T8jK+3\r
-rTp3LR9vDkMwDjX4R8SA9c0wdnUzzr79OYQC9lTnzcx+fM6BBmgQ2GrS33jaFLp7\r
-L6/DFpCl5zhnPjM/2dKvMkw/Kd6XS/vjwsO405FQdjSDiQEEAZA+ZvAfcjdccbbU\r
-yCO+x0QNdeBsufDVnh3xvzuWy4CICdTQT4s1AWRPCzjOj+SGmx5WqCLWfsd8Ma0+\r
-w/C7SfTYu1FDQILLM+llpq1M/9GPley4QZ8JQjo262AyPXsPF/OW48uuZz0Db1xT\r
-Yh4iHBztj4VSdy7l2+IyaIf7cnL4EEBFxv/MwmVDXvDlxyvfAfIsd3D9SvJESzKZ\r
-VWDYwaocgeCN+ojKu1p885lu1EfRbX3fr3YO02K5/c2JYDkc0Py0W3wUP/J1XUax\r
-pbKpzwlkxEgtmzsGqsOfMJqBV3TNDrOA2uBsa+uBqP5MGYLZ49S/4v/bW9I01Cr1\r
-D2ZkV510Y1Vgo66WlP8mRqOTyt/5WRhPD+MxXdk67BNN/PmO6tMlVoJDuk+XwWPR\r
-t2TvNaND/yabT9eYI55Og4fzKD6RIjouUX8DvKLkm+7aXxVs2uuLQ3Jco3O82z55\r
-dbShU1jYsrw9oouXUz06MHPbkdhNbF/2hfhZ2qA31sNeovJw65iUv7sDKX3LVWgJ\r
-10jlywcDwqlU8CO7WC9lGixYTbnOkYZpXCGEl8e6Jbs79l42YFo4ogYpFK1NXFhV\r
-kOXRmDf/wmfj+c/ld3L2PkvwlgofhCudOQknZbo3ub1gjiTn7L+lMGHIj/3suMIl\r
-ID4EUxAXScIM1ZEz2fjtW5jATlqYcLjLTbf/olw6HFyPNH+9IssqXeZNKnGwPUB9\r
-3lTXsg0tpzl+x7F/2WjEw1DSNhjC0KnHt1vEYNMkUGDGFdN9y3ERLqX/FIgiASUb\r
-bTvAVupnAK3raBezGmhrs6LsQtLS9P0VvQiLU3uDhMqw8Z4SISLpcD+NnVBHzQqm\r
-6W5Qn/8xsCL6av18yUVTi2G3igt3QCNoYx9evt2ZcIkNoyyagUVjfZe5GHXh8Dnz\r
-GaBXW/hg3HlXLRGaQu4RYCzBMJILcO25OhZOg6jbkCLiEexQlm2e9krB5cXR49Al\r
-UN4fiB0KR9JyG2ayUdNJVkXZSZLnHyRgiaadlpUo16LVvw==\r
-=b5Kp\r
------END PGP MESSAGE-----\r
-\r
-\r
-"""
-
-    commented_payload = prefix + comment + payload
-    assert check_armored_payload(commented_payload, outgoing=False) == True
-    assert check_armored_payload(commented_payload, outgoing=True) == False
-
-    payload = prefix + payload
-    assert check_armored_payload(payload, outgoing=False) == True
-    assert check_armored_payload(payload, outgoing=True) == True
-
-    payload = payload.removesuffix("\r\n")
-    assert check_armored_payload(payload, outgoing=False) == True
-    assert check_armored_payload(payload, outgoing=True) == True
-
-    payload = payload.removesuffix("\r\n")
-    assert check_armored_payload(payload, outgoing=False) == True
-    assert check_armored_payload(payload, outgoing=True) == True
-
-    payload = payload.removesuffix("\r\n")
-    assert check_armored_payload(payload, outgoing=False) == True
-    assert check_armored_payload(payload, outgoing=True) == True
-
-    payload = """-----BEGIN PGP MESSAGE-----\r
-\r
-HELLOWORLD
------END PGP MESSAGE-----\r
-\r
-"""
-    assert check_armored_payload(payload, outgoing=False) == False
-    assert check_armored_payload(payload, outgoing=True) == False
-
-    payload = """-----BEGIN PGP MESSAGE-----\r
-\r
-=njUN
------END PGP MESSAGE-----\r
-\r
-"""
-    assert check_armored_payload(payload, outgoing=False) == False
-    assert check_armored_payload(payload, outgoing=True) == False
-
-    # Test payload using partial body length
-    # as generated by GopenPGP.
-    payload = """-----BEGIN PGP MESSAGE-----\r
-\r
-wV4DdCVjRfOT3TQSAQdAY5+pjT6mlCxPGdR3be4w7oJJRUGIPI/Vnh+mJxGSm34w\r
-LNlVc89S1g22uQYFif2sUJsQWbpoHpNkuWpkSgOaHmNvrZiY/YU5iv+cZ3LbmtUG\r
-0uoBisSHh9O1c+5sYZSbrvYZ1NOwlD7Fv/U5/Mw4E5+CjxfdgNGp5o3DDddzPK78\r
-jseDhdSXxnaiIJC93hxNX6R1RPt3G2gukyzx69wciPQShcF8zf3W3o75Ed7B8etV\r
-QEeB16xzdFhKa9JxdjTu3osgCs21IO7wpcFkjc7nZzlW6jPnELJJaNmv4yOOCjMp\r
-6YAkaN/BkL+jHTznHDuDsT5ilnTXpwHDU1Cm9PIx/KFcNCQnIB+2DcdIHPHUH1ci\r
-jvqoeXAVWjKXEjS7PqPFuP/xGbrWG2ugs+toXJOKbgRkExvKs1dwPFKrgghvCVbW\r
-AcKejQKAPArLwpkA7aD875TZQShvGt74fNs45XBlGOYOnNOAJ1KAmzrXLIDViyyB\r
-kDsmTBk785xofuCkjBpXSe6vsMprPzCteDfaUibh8FHeJjucxPerwuOPEmnogNaf\r
-YyL4+iy8H8I9/p7pmUqILprxTG0jTOtlk0bTVzeiF56W1xbtSEMuOo4oFbQTyOM2\r
-bKXaYo774Jm+rRtKAnnI2dtf9RpK19cog6YNzfYjesLKbXDsPZbN5rmwyFiCvvxC\r
-kQ6JLob+B2fPdY2gzy7LypxktS8Zi1HJcWDHJGVmQodaDLqKUObb4M26bXDe6oxI\r
-NS8PJz5exVbM3KhZnUOEn6PJRBBf5a/ZqxlhZPcQo/oBuhKpBRpO5kSDwPIUByu3\r
-UlXLSkpMqe9pUarAOEuQjfl2RVY7U+RrQYp4YP5keMO+i8NCefAFbowTTufO1JIq\r
-2nVgCi/QVnxZyEc9OYt/8AE3g4cdojE+vsSDifZLSWYIetpfrohHv3dT3StD1QRG\r
-0QE6qq6oKpg/IL0cjvuX4c7a7bslv2fXp8t75y37RU6253qdIebhxc/cRhPbc/yu\r
-p0YLyD4SrvKTLP2ZV95jT4IPEpqm4AN3QmiOzdtqR2gLyb62L8QfqI/FdwsIiRiM\r
-hqydwoqt/lfSqG1WKPh+6EkMkH+TDiCC1BQdbN1MNcyUtcjb35PR2c8Ld2TF3guA\r
-jLIqMt/Vb7hBoMb2FcsOYY25ka9oV62OwgKWLXnFzk+modMR5fzb4kxVVAYEqP+D\r
-T5KO1Vs76v1fyPGOq6BbBCvLwTqe/e6IZInJles4v5jrhnLcGKmNGivCUDe6X6NY\r
-UKNt5RsZllwDQpaAb5dMNhyrk8SgIE7TBI7rvqIdUCE52Vy+0JDxFg5olRpFUfO6\r
-/MyTW3Yo/ekk/npHr7iYYqJTCc21bDGLWQcIo/XO7WPxrKNWGBNPFnkRdw0MaKr4\r
-+cEM3V8NFnSEpC12xA+RX/CezuJtwXZK5MpG76eYqMO6qyC+c25YcFecEufDZDxx\r
-ZLqRszVRyxyWPtk/oIeQK2v9wOqY6N9/ff01gHz69vqYqN5bUw/QKZsmx1zW+gPw\r
-6x2tDK2BHeYl182gCbhlKISRFwCtbjqZSkiKWao/VtygHkw0fK34avJuyQ/X9YaN\r
-BRy+7Lf3VA53pnB5WJ1xwRXN8VDvmZeXzv2krHveCMemj0OjnRoCLu117xN0A5m9\r
-Fm/RoDix5PolDHtWTtr2m1n2hp2LHnj8at9lFEd0SKhAYHVL9KjzycwWODZRXt+x\r
-zGDDuooEeTvdY5NLyKcl4gETz1ZP4Ez5jGGjhPSwSpq1mU7UaJ9ZXXdr4KHyifW6\r
-ggNzNsGhXTap7IWZpTtqXABydfiBshmH2NjqtNDwBweJVSgP10+r0WhMWlaZs6xl\r
-V3o5yskJt6GlkwpJxZrTvN6Tiww/eW7HFV6NGf7IRSWY5tJc/iA7/92tOmkdvJ1q\r
-myLbG7cJB787QjplEyVe2P/JBO6xYvbkJLf9Q+HaviTO25rugRSrYsoKMDfO8VlQ\r
-1CcnTPVtApPZJEQzAWJEgVAM8uIlkqWJJMgyWT34sTkdBeCUFGloXQFs9Yxd0AGf\r
-/zHEkYZSTKpVSvAIGu4=\r
-=6iHb\r
------END PGP MESSAGE-----\r
-"""
-    assert check_armored_payload(payload, outgoing=False) == True
-    assert check_armored_payload(payload, outgoing=True) == True

chatmaild/src/chatmaild/tests/test_filtermail_blackbox.py

@@ -1,9 +1,15 @@
+import shutil
 import smtplib
 import subprocess
 import sys
 
 import pytest
 
+pytestmark = pytest.mark.skipif(
+    shutil.which("filtermail") is None,
+    reason="filtermail binary not found",
+)
+
 
 @pytest.fixture
 def smtpserver():
@@ -41,6 +47,8 @@ def test_one_mail(
     make_config, make_popen, smtpserver, maildata, filtermail_mode, monkeypatch
 ):
     monkeypatch.setenv("PYTHONUNBUFFERED", "1")
+    # DKIM is tested by cmdeploy tests.
+    monkeypatch.setenv("FILTERMAIL_SKIP_DKIM", "1")
     smtp_inject_port = 20025
     if filtermail_mode == "outgoing":
         settings = dict(
@@ -58,6 +66,10 @@ def test_one_mail(
 
     popen = make_popen(["filtermail", path, filtermail_mode])
     line = popen.stderr.readline().strip()
+
+    # skip a warning that FILTERMAIL_SKIP_DKIM shouldn't be used in prod
+    if b"DKIM verification DISABLED!" in line:
+        line = popen.stderr.readline().strip()
     if b"loop" not in line:
         print(line.decode("ascii"), file=sys.stderr)
         pytest.fail("starting filtermail failed")

chatmaild/src/chatmaild/tests/test_metadata.py

@@ -314,6 +314,51 @@ def test_persistent_queue_items(tmp_path, testaddr, token):
     assert not queue_item < item2 and not item2 < queue_item
 
 
+def test_turn_credentials_exception_returns_N(notifier, metadata, monkeypatch):
+    """Test that turn_credentials() failure returns N\\n instead of crashing."""
+    import chatmaild.metadata
+
+    dictproxy = MetadataDictProxy(
+        notifier=notifier,
+        metadata=metadata,
+        turn_hostname="turn.example.org",
+    )
+
+    def mock_turn_credentials():
+        raise ConnectionRefusedError("socket not available")
+
+    monkeypatch.setattr(chatmaild.metadata, "turn_credentials", mock_turn_credentials)
+
+    transactions = {}
+    res = dictproxy.handle_dovecot_request(
+        "Lshared/0123/vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn"
+        "\tuser@example.org",
+        transactions,
+    )
+    assert res == "N\n"
+
+
+def test_turn_credentials_success(notifier, metadata, monkeypatch):
+    """Test that valid turn_credentials() returns TURN URI."""
+    import chatmaild.metadata
+
+    dictproxy = MetadataDictProxy(
+        notifier=notifier,
+        metadata=metadata,
+        turn_hostname="turn.example.org",
+    )
+
+    monkeypatch.setattr(chatmaild.metadata, "turn_credentials", lambda: "user:pass")
+
+    transactions = {}
+    res = dictproxy.handle_dovecot_request(
+        "Lshared/0123/vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn"
+        "\tuser@example.org",
+        transactions,
+    )
+    assert res == "Oturn.example.org:3478:user:pass\n"
+
+
 def test_iroh_relay(dictproxy):
     rfile = io.BytesIO(
         b"\n".join(

chatmaild/src/chatmaild/tests/test_metrics.py

@@ -1,24 +0,0 @@
-from chatmaild.metrics import main
-
-
-def test_main(tmp_path, capsys):
-    paths = []
-    for x in ("ci-asllkj", "ac_12l3kj", "qweqwe", "ci-l1k2j31l2k3"):
-        p = tmp_path.joinpath(x)
-        p.mkdir()
-        p.joinpath("cur").mkdir()
-        paths.append(p)
-
-    tmp_path.joinpath("nomailbox").mkdir()
-
-    main(tmp_path)
-    out, _ = capsys.readouterr()
-    d = {}
-    for line in out.split("\n"):
-        if line.strip() and not line.startswith("#"):
-            name, num = line.split()
-            d[name] = int(num)
-
-    assert d["accounts"] == 4
-    assert d["ci_accounts"] == 3
-    assert d["nonci_accounts"] == 1

chatmaild/src/chatmaild/tests/test_newmail.py

@@ -1,7 +1,11 @@
 import json
 
 import chatmaild
-from chatmaild.newemail import create_newemail_dict, print_new_account
+from chatmaild.newemail import (
+    create_dclogin_url,
+    create_newemail_dict,
+    print_new_account,
+)
 
 
 def test_create_newemail_dict(example_config):
@@ -15,6 +19,24 @@ def test_create_newemail_dict(example_config):
     assert ac1["password"] != ac2["password"]
 
 
+def test_create_newemail_dict_ip(make_config):
+    config = make_config("1.2.3.4")
+    ac = create_newemail_dict(config)
+    assert ac["email"].endswith("@[1.2.3.4]")
+
+
+def test_create_dclogin_url():
+    url = create_dclogin_url("user@example.org", "p@ss w+rd")
+    assert url.startswith("dclogin:")
+    assert "v=1" in url
+    assert "ic=3" in url
+
+    assert "user@example.org" in url
+    # password special chars must be encoded
+    assert "p%40ss" in url
+    assert "w%2Brd" in url
+
+
 def test_print_new_account(capsys, monkeypatch, maildomain, tmpdir, example_config):
     monkeypatch.setattr(chatmaild.newemail, "CONFIG_PATH", str(example_config._inipath))
     print_new_account()
@@ -25,3 +47,20 @@ def test_print_new_account(capsys, monkeypatch, maildomain, tmpdir, example_conf
     dic = json.loads(lines[2])
     assert dic["email"].endswith(f"@{example_config.mail_domain}")
     assert len(dic["password"]) >= 10
+    # default tls_cert=acme should not include dclogin_url
+    assert "dclogin_url" not in dic
+
+
+def test_print_new_account_self_signed(capsys, monkeypatch, make_config):
+    config = make_config("_test.example.org")
+    monkeypatch.setattr(chatmaild.newemail, "CONFIG_PATH", str(config._inipath))
+    print_new_account()
+    out, err = capsys.readouterr()
+    lines = out.split("\n")
+    dic = json.loads(lines[2])
+    assert "dclogin_url" in dic
+    url = dic["dclogin_url"]
+    assert url.startswith("dclogin:")
+    assert "ic=3" in url
+
+    assert dic["email"].split("@")[0] in url

chatmaild/src/chatmaild/tests/test_turnserver.py

@@ -0,0 +1,73 @@
+import socket
+import threading
+import time
+from unittest.mock import patch
+
+import pytest
+
+from chatmaild.turnserver import turn_credentials
+
+SOCKET_PATH = "/run/chatmail-turn/turn.socket"
+
+
+@pytest.fixture
+def turn_socket(tmp_path):
+    """Create a real Unix socket server at a temp path."""
+    sock_path = str(tmp_path / "turn.socket")
+    server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    server.bind(sock_path)
+    server.listen(1)
+    yield sock_path, server
+    server.close()
+
+
+def _call_turn_credentials(sock_path):
+    """Call turn_credentials but connect to sock_path instead of hardcoded path."""
+    original_connect = socket.socket.connect
+
+    def patched_connect(self, address):
+        if address == SOCKET_PATH:
+            address = sock_path
+        return original_connect(self, address)
+
+    with patch.object(socket.socket, "connect", patched_connect):
+        return turn_credentials()
+
+
+def test_turn_credentials_timeout(turn_socket):
+    """Server accepts but never responds — must raise socket.timeout."""
+    sock_path, server = turn_socket
+
+    def accept_and_hang():
+        conn, _ = server.accept()
+        time.sleep(30)
+        conn.close()
+
+    t = threading.Thread(target=accept_and_hang, daemon=True)
+    t.start()
+
+    with pytest.raises(socket.timeout):
+        _call_turn_credentials(sock_path)
+
+
+def test_turn_credentials_connection_refused(tmp_path):
+    """Socket file doesn't exist — must raise ConnectionRefusedError or FileNotFoundError."""
+    missing = str(tmp_path / "nonexistent.socket")
+    with pytest.raises((ConnectionRefusedError, FileNotFoundError)):
+        _call_turn_credentials(missing)
+
+
+def test_turn_credentials_success(turn_socket):
+    """Server responds with credentials — must return stripped string."""
+    sock_path, server = turn_socket
+
+    def respond():
+        conn, _ = server.accept()
+        conn.sendall(b"testuser:testpass\n")
+        conn.close()
+
+    t = threading.Thread(target=respond, daemon=True)
+    t.start()
+
+    result = _call_turn_credentials(sock_path)
+    assert result == "testuser:testpass"

chatmaild/src/chatmaild/turnserver.py

@@ -4,6 +4,7 @@ import socket
 
 def turn_credentials() -> str:
     with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client_socket:
+        client_socket.settimeout(5)
         client_socket.connect("/run/chatmail-turn/turn.socket")
         with client_socket.makefile("rb") as file:
             return file.readline().decode("utf-8").strip()

cmdeploy/pyproject.toml

@@ -10,7 +10,6 @@ dependencies = [
   "pillow",
   "qrcode",
   "markdown",
-  "pytest",
   "setuptools>=68",
   "termcolor",
   "build",
@@ -20,6 +19,8 @@ dependencies = [
   "pytest-xdist", 
   "execnet",
   "imap_tools",
+  "deltachat-rpc-client",
+  "deltachat-rpc-server",
 ]
 
 [project.scripts]

cmdeploy/src/cmdeploy/acmetool/__init__.py

@@ -67,7 +67,7 @@ class AcmetoolDeployer(Deployer):
         )
         files.template(
             src=importlib.resources.files(__package__).joinpath("desired.yaml.j2"),
-            dest=f"/var/lib/acme/desired/{self.domains[0]}", # 0 is mailhost TLD
+            dest=f"/var/lib/acme/desired/{self.domains[0]}",  # 0 is mailhost TLD
             user="root",
             group="root",
             mode="644",

cmdeploy/src/cmdeploy/acmetool/acmetool-redirector.service

@@ -3,7 +3,7 @@ Description=acmetool HTTP redirector
 
 [Service]
 Type=notify
-ExecStart=/usr/bin/acmetool redirector --service.uid=daemon
+ExecStart=/usr/bin/acmetool redirector --service.uid=daemon --bind=127.0.0.1:402
 Restart=always
 RestartSec=30
 

cmdeploy/src/cmdeploy/basedeploy.py

@@ -1,10 +1,51 @@
 import importlib.resources
 import io
 import os
+from contextlib import contextmanager
 
+from pyinfra import host
+from pyinfra.facts.server import Command
 from pyinfra.operations import files, server, systemd
 
 
+def has_systemd():
+    """Returns False during Docker image builds or any other non-systemd environment."""
+    return os.path.isdir("/run/systemd/system")
+
+
+def is_in_container() -> bool:
+    """Return True if running inside a container (Docker, LXC, etc.)."""
+    return (
+        host.get_fact(
+            Command,
+            "systemd-detect-virt --container --quiet 2>/dev/null && echo yes || true",
+        )
+        == "yes"
+    )
+
+
+@contextmanager
+def blocked_service_startup():
+    """Prevent services from auto-starting during package installation.
+
+    Installs a ``/usr/sbin/policy-rc.d`` that exits 101, blocking any
+    service from being started by the package manager.  This avoids bind
+    conflicts and CPU/RAM spikes during initial setup.  The file is removed
+    when the context exits.
+    """
+    # For documentation about policy-rc.d, see:
+    # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
+    files.put(
+        src=get_resource("policy-rc.d"),
+        dest="/usr/sbin/policy-rc.d",
+        user="root",
+        group="root",
+        mode="755",
+    )
+    yield
+    files.file("/usr/sbin/policy-rc.d", present=False)
+
+
 def get_resource(arg, pkg=__package__):
     return importlib.resources.files(pkg).joinpath(arg)
 
@@ -17,9 +58,8 @@ def configure_remote_units(mail_domain, units) -> None:
 
     # install systemd units
     for fn in units:
-        execpath = fn if fn != "filtermail-incoming" else "filtermail"
         params = dict(
-            execpath=f"{remote_venv_dir}/bin/{execpath}",
+            execpath=f"{remote_venv_dir}/bin/{fn}",
             config_path=remote_chatmail_inipath,
             remote_venv_dir=remote_venv_dir,
             mail_domain=mail_domain,

cmdeploy/src/cmdeploy/chatmail.zone.j2

@@ -1,30 +0,0 @@
-;
-; Required DNS entries for chatmail servers 
-;
-{% if A %}
-{{ mail_domain }}.                   A     {{ A }}
-{% endif %}
-{% if AAAA %}
-{{ mail_domain }}.                   AAAA  {{ AAAA }}
-{% endif %}
-{{ mail_domain }}.                   MX 10 {{ mail_domain }}.
-_mta-sts.{{ mail_domain }}.          TXT "v=STSv1; id={{ sts_id }}"
-mta-sts.{{ mail_domain }}.           CNAME {{ mail_domain }}.
-www.{{ mail_domain }}.               CNAME {{ mail_domain }}.
-{{ dkim_entry }}
-
-;
-; Recommended DNS entries for interoperability and security-hardening
-;
-{{ mail_domain }}.                   TXT "v=spf1 a ~all"
-_dmarc.{{ mail_domain }}.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
-
-{% if acme_account_url %}
-{{ mail_domain }}.                   CAA 0 issue "letsencrypt.org;accounturi={{ acme_account_url }}"
-{% endif %}
-_adsp._domainkey.{{ mail_domain }}.  TXT "dkim=discardable"
-
-_submission._tcp.{{ mail_domain }}.  SRV 0 1 587 {{ mail_domain }}.
-_submissions._tcp.{{ mail_domain }}. SRV 0 1 465 {{ mail_domain }}.
-_imap._tcp.{{ mail_domain }}.        SRV 0 1 143 {{ mail_domain }}.
-_imaps._tcp.{{ mail_domain }}.       SRV 0 1 993 {{ mail_domain }}.

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -5,7 +5,6 @@ along with command line option and subcommand parsing.
 
 import argparse
 import importlib.resources
-import importlib.util
 import os
 import pathlib
 import shutil
@@ -71,6 +70,11 @@ def run_cmd_options(parser):
         action="store_true",
         help="install/upgrade the server, but disable postfix & dovecot for now",
     )
+    parser.add_argument(
+        "--website-only",
+        action="store_true",
+        help="only update/deploy the website, skipping full server upgrade/deployment, useful when you only changed/updated the web pages and don't need to re-run a full server upgrade",
+    )
     parser.add_argument(
         "--skip-dns-check",
         dest="dns_check_disabled",
@@ -86,20 +90,22 @@ def run_cmd(args, out):
     ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
     sshexec = get_sshexec(ssh_host)
     require_iroh = args.config.enable_iroh_relay
+    strict_tls = args.config.tls_cert_mode == "acme"
     if not args.dns_check_disabled:
         remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
-        if not dns.check_initial_remote_data(remote_data, print=out.red):
+        if not dns.check_initial_remote_data(remote_data, strict_tls=strict_tls, print=out.red):
             return 1
 
     env = os.environ.copy()
     env["CHATMAIL_INI"] = args.inipath
+    env["CHATMAIL_WEBSITE_ONLY"] = "True" if args.website_only else ""
     env["CHATMAIL_DISABLE_MAIL"] = "True" if args.disable_mail else ""
     env["CHATMAIL_REQUIRE_IROH"] = "True" if require_iroh else ""
     deploy_path = importlib.resources.files(__package__).joinpath("run.py").resolve()
     pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
 
     cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y"
-    if ssh_host in ["localhost", "@docker"]:
+    if ssh_host == "localhost":
         cmd = f"{pyinf} @local {deploy_path} -y"
 
     if version.parse(pyinfra.__version__) < version.parse("3"):
@@ -107,19 +113,18 @@ def run_cmd(args, out):
         return 1
 
     try:
-        retcode = out.check_call(cmd, env=env)
-        if retcode == 0:
-            out.green("Deploy completed, call `cmdeploy dns` next.")
-        elif not remote_data["acme_account_url"]:
+        out.check_call(cmd, env=env)
+        if args.website_only:
+            out.green("Website deployment completed.")
+        elif not args.dns_check_disabled and strict_tls and not remote_data["acme_account_url"]:
             out.red("Deploy completed but letsencrypt not configured")
             out.red("Run 'cmdeploy run' again")
-            retcode = 0
         else:
-            out.red("Deploy failed")
+            out.green("Deploy completed, call `cmdeploy dns` next.")
+        return 0
     except subprocess.CalledProcessError:
         out.red("Deploy failed")
-        retcode = 1
-    return retcode
+        return 1
 
 
 def dns_cmd_options(parser):
@@ -137,11 +142,13 @@ def dns_cmd(args, out):
     """Check DNS entries and optionally generate dns zone file."""
     ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
     sshexec = get_sshexec(ssh_host, verbose=args.verbose)
+    tls_cert_mode = args.config.tls_cert_mode
+    strict_tls = tls_cert_mode == "acme"
     remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
-    if not remote_data:
+    if not dns.check_initial_remote_data(remote_data, strict_tls=strict_tls):
         return 1
 
-    if not remote_data["acme_account_url"]:
+    if strict_tls and not remote_data["acme_account_url"]:
         out.red("could not get letsencrypt account url, please run 'cmdeploy run'")
         return 1
 
@@ -149,6 +156,7 @@ def dns_cmd(args, out):
         out.red("could not determine dkim_entry, please run 'cmdeploy run'")
         return 1
 
+    remote_data["strict_tls"] = strict_tls
     zonefile = dns.get_filled_zone_file(remote_data)
 
     if args.zonefile:
@@ -183,23 +191,16 @@ def status_cmd(args, out):
 
 
 def test_cmd_options(parser):
-    parser.add_argument(
-        "--slow",
-        dest="slow",
-        action="store_true",
-        help="also run slow tests",
-    )
+    add_ssh_host_option(parser)
 
 
 def test_cmd(args, out):
-    """Run local and online tests for chatmail deployment.
+    """Run local and online tests for chatmail deployment."""
 
-    This will automatically pip-install 'deltachat' if it's not available.
-    """
-
-    x = importlib.util.find_spec("deltachat")
-    if x is None:
-        out.check_call(f"{sys.executable} -m pip install deltachat")
+    env = os.environ.copy()
+    env["CHATMAIL_INI"] = str(args.inipath.absolute())
+    if args.ssh_host:
+        env["CHATMAIL_SSH"] = args.ssh_host
 
     pytest_path = shutil.which("pytest")
     pytest_args = [
@@ -211,9 +212,7 @@ def test_cmd(args, out):
         "-v",
         "--durations=5",
     ]
-    if args.slow:
-        pytest_args.append("--slow")
-    ret = out.run_ret(pytest_args)
+    ret = out.run_ret(pytest_args, env=env)
     return ret
 
 
@@ -304,7 +303,7 @@ def add_ssh_host_option(parser):
     parser.add_argument(
         "--ssh-host",
         dest="ssh_host",
-        help="Run commands on 'localhost', via '@docker', or on a specific SSH host "
+        help="Run commands on 'localhost' or on a specific SSH host "
         "instead of chatmail.ini's mail_domain.",
     )
 
@@ -314,7 +313,7 @@ def add_config_option(parser):
         "--config",
         dest="inipath",
         action="store",
-        default=Path("chatmail.ini"),
+        default=Path(os.environ.get("CHATMAIL_INI", "chatmail.ini")),
         type=Path,
         help="path to the chatmail.ini file",
     )
@@ -366,9 +365,7 @@ def get_parser():
 
 def get_sshexec(ssh_host: str, verbose=True):
     if ssh_host in ["localhost", "@local"]:
-        return LocalExec(verbose, docker=False)
-    elif ssh_host == "@docker":
-        return LocalExec(verbose, docker=True)
+        return LocalExec(verbose)
     if verbose:
         print(f"[ssh] login to {ssh_host}")
     return SSHExec(ssh_host, verbose=verbose)

cmdeploy/src/cmdeploy/deployers.py

@@ -5,12 +5,13 @@ Chat Mail pyinfra deploy.
 import shutil
 import subprocess
 import sys
-from io import StringIO
+from io import BytesIO, StringIO
 from pathlib import Path
 
 from chatmaild.config import read_config
 from pyinfra import facts, host, logger
 from pyinfra.api import FactBase
+from pyinfra.facts import hardware
 from pyinfra.facts.files import Sha256File
 from pyinfra.facts.systemd import SystemdEnabled
 from pyinfra.operations import apt, files, pip, server, systemd
@@ -22,20 +23,26 @@ from .basedeploy import (
     Deployer,
     Deployment,
     activate_remote_units,
+    blocked_service_startup,
     configure_remote_units,
     get_resource,
+    has_systemd,
+    is_in_container,
 )
 from .dovecot.deployer import DovecotDeployer
+from .external.deployer import ExternalTlsDeployer
+from .filtermail.deployer import FiltermailDeployer
 from .mtail.deployer import MtailDeployer
 from .nginx.deployer import NginxDeployer
 from .opendkim.deployer import OpendkimDeployer
 from .postfix.deployer import PostfixDeployer
+from .selfsigned.deployer import SelfSignedTlsDeployer
 from .www import build_webpages, find_merge_conflict, get_paths
 
 
 class Port(FactBase):
     """
-    Returns the process occuping a port.
+    Returns the process occupying a port.
     """
 
     def command(self, port: int) -> str:
@@ -63,6 +70,8 @@ def _build_chatmaild(dist_dir) -> None:
 
 
 def remove_legacy_artifacts():
+    if not has_systemd():
+        return
     # disable legacy doveauth-dictproxy.service
     if host.get_fact(SystemdEnabled).get("doveauth-dictproxy.service"):
         systemd.service(
@@ -115,7 +124,6 @@ def _install_remote_venv_with_chatmaild() -> None:
 
 def _configure_remote_venv_with_chatmaild(config) -> None:
     remote_base_dir = "/usr/local/lib/chatmaild"
-    remote_venv_dir = f"{remote_base_dir}/venv"
     remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini"
     root_owned = dict(user="root", group="root", mode="644")
 
@@ -126,56 +134,83 @@ def _configure_remote_venv_with_chatmaild(config) -> None:
         **root_owned,
     )
 
-    files.template(
-        src=get_resource("metrics.cron.j2"),
-        dest="/etc/cron.d/chatmail-metrics",
-        user="root",
-        group="root",
-        mode="644",
-        config={
-            "mailboxes_dir": config.mailboxes_dir,
-            "execpath": f"{remote_venv_dir}/bin/chatmail-metrics",
-        },
+    files.file(
+        path="/etc/cron.d/chatmail-metrics",
+        present=False,
+    )
+    files.file(
+        path="/var/www/html/metrics",
+        present=False,
     )
 
 
 class UnboundDeployer(Deployer):
+    def __init__(self, config):
+        self.config = config
+        self.need_restart = False
+
     def install(self):
-        # Run local DNS resolver `unbound`.
-        # `resolvconf` takes care of setting up /etc/resolv.conf
-        # to use 127.0.0.1 as the resolver.
-
-        #
-        # On an IPv4-only system, if unbound is started but not
-        # configured, it causes subsequent steps to fail to resolve hosts.
-        # Here, we use policy-rc.d to prevent unbound from starting up
-        # on initial install.  Later, we will configure it and start it.
-        #
-        # For documentation about policy-rc.d, see:
-        # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
-        #
-        files.put(
-            src=get_resource("policy-rc.d"),
-            dest="/usr/sbin/policy-rc.d",
-            user="root",
-            group="root",
-            mode="755",
-        )
-
-        apt.packages(
-            name="Install unbound",
-            packages=["unbound", "unbound-anchor", "dnsutils"],
-        )
-
-        files.file("/usr/sbin/policy-rc.d", present=False)
+        # On an IPv4-only system, if unbound is started but not configured,
+        # it causes subsequent steps to fail to resolve hosts.
+        with blocked_service_startup():
+            apt.packages(
+                name="Install unbound",
+                packages=["unbound", "unbound-anchor", "dnsutils"],
+            )
 
     def configure(self):
+        # Remove dynamic resolver managers that compete for /etc/resolv.conf.
+        apt.packages(
+            name="Purge resolvconf",
+            packages=["resolvconf"],
+            present=False,
+            extra_uninstall_args="--purge",
+        )
+        # systemd-resolved can't be purged due to dependencies; stop and mask.
+        server.shell(
+            name="Stop and mask systemd-resolved",
+            commands=[
+                "systemctl stop systemd-resolved.service || true",
+                "systemctl mask systemd-resolved.service",
+            ],
+        )
+        # Configure unbound resolver with Quad9 fallback and a trailing newline
+        # (SolusVM bug).
+        files.put(
+            name="Write static resolv.conf",
+            src=BytesIO(b"nameserver 127.0.0.1\nnameserver 9.9.9.9\n"),
+            dest="/etc/resolv.conf",
+            user="root",
+            group="root",
+            mode="644",
+        )
         server.shell(
             name="Generate root keys for validating DNSSEC",
             commands=[
                 "unbound-anchor -a /var/lib/unbound/root.key || true",
             ],
         )
+        if self.config.disable_ipv6:
+            files.directory(
+                path="/etc/unbound/unbound.conf.d",
+                present=True,
+                user="root",
+                group="root",
+                mode="755",
+            )
+            conf = files.put(
+                src=get_resource("unbound/unbound.conf.j2"),
+                dest="/etc/unbound/unbound.conf.d/chatmail.conf",
+                user="root",
+                group="root",
+                mode="644",
+            )
+        else:
+            conf = files.file(
+                path="/etc/unbound/unbound.conf.d/chatmail.conf",
+                present=False,
+            )
+        self.need_restart |= conf.changed
 
     def activate(self):
         server.shell(
@@ -190,6 +225,7 @@ class UnboundDeployer(Deployer):
             service="unbound.service",
             running=True,
             enabled=True,
+            restarted=self.need_restart,
         )
 
 
@@ -237,6 +273,9 @@ class WebsiteDeployer(Deployer):
             # if www_folder is a hugo page, build it
             if build_dir:
                 www_path = build_webpages(src_dir, build_dir, self.config)
+                if www_path is None:
+                    logger.warning("Web page build failed, skipping website deployment")
+                    return
             # if it is not a hugo page, upload it as is
             files.rsync(
                 f"{www_path}/", "/var/www/html", flags=["-avz", "--chown=www-data"]
@@ -271,7 +310,7 @@ class LegacyRemoveDeployer(Deployer):
             present=False,
         )
         # remove echobot if it is still running
-        if host.get_fact(SystemdEnabled).get("echobot.service"):
+        if has_systemd() and host.get_fact(SystemdEnabled).get("echobot.service"):
             systemd.service(
                 name="Disable echobot.service",
                 service="echobot.service",
@@ -303,12 +342,12 @@ class TurnDeployer(Deployer):
     def install(self):
         (url, sha256sum) = {
             "x86_64": (
-                "https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-x86_64-linux",
-                "841e527c15fdc2940b0469e206188ea8f0af48533be12ecb8098520f813d41e4",
+                "https://github.com/chatmail/chatmail-turn/releases/download/v0.4/chatmail-turn-x86_64-linux",
+                "1ec1f5c50122165e858a5a91bcba9037a28aa8cb8b64b8db570aa457c6141a8a",
             ),
             "aarch64": (
-                "https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-aarch64-linux",
-                "a5fc2d06d937b56a34e098d2cd72a82d3e89967518d159bf246dc69b65e81b42",
+                "https://github.com/chatmail/chatmail-turn/releases/download/v0.4/chatmail-turn-aarch64-linux",
+                "0fb3e792419494e21ecad536464929dba706bb2c88884ed8f1788141d26fc756",
             ),
         }[host.get_fact(facts.server.Arch)]
 
@@ -416,8 +455,6 @@ class ChatmailVenvDeployer(Deployer):
     def __init__(self, config):
         self.config = config
         self.units = (
-            "filtermail",
-            "filtermail-incoming",
             "chatmail-metadata",
             "lastlogin",
             "chatmail-expire",
@@ -443,10 +480,19 @@ class ChatmailDeployer(Deployer):
         ("iroh", None, None),
     ]
 
-    def __init__(self, mail_domain):
-        self.mail_domain = mail_domain
+    def __init__(self, config):
+        self.config = config
+        self.mail_domain = config.mail_domain
 
     def install(self):
+        files.put(
+            name="Disable installing recommended packages globally",
+            src=BytesIO(b'APT::Install-Recommends "false";\n'),
+            dest="/etc/apt/apt.conf.d/00InstallRecommends",
+            user="root",
+            group="root",
+            mode="644",
+        )
         apt.update(name="apt update", cache_time=24 * 3600)
         apt.upgrade(name="upgrade apt packages", auto_remove=True)
 
@@ -459,12 +505,18 @@ class ChatmailDeployer(Deployer):
             name="Install rsync",
             packages=["rsync"],
         )
-        apt.packages(
-            name="Ensure cron is installed",
-            packages=["cron"],
-        )
 
     def configure(self):
+        # metadata crashes if the mailboxes dir does not exist
+        files.directory(
+            name="Ensure vmail mailbox directory exists",
+            path=str(self.config.mailboxes_dir),
+            user="vmail",
+            group="vmail",
+            mode="700",
+            present=True,
+        )
+
         # This file is used by auth proxy.
         # https://wiki.debian.org/EtcMailName
         server.shell(
@@ -502,66 +554,98 @@ class GithashDeployer(Deployer):
         except Exception:
             git_diff = ""
         files.put(
-            name="Upload chatmail relay git commiit hash",
+            name="Upload chatmail relay git commit hash",
             src=StringIO(git_hash + git_diff),
             dest="/etc/chatmail-version",
             mode="700",
         )
 
 
-def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
+def get_tls_deployer(config, mail_domain):
+    """Select the appropriate TLS deployer based on config."""
+    tls_domains = [mail_domain, f"mta-sts.{mail_domain}", f"www.{mail_domain}"]
+
+    if config.tls_cert_mode == "acme":
+        return AcmetoolDeployer(config.acme_email, tls_domains)
+    elif config.tls_cert_mode == "self":
+        return SelfSignedTlsDeployer(mail_domain)
+    elif config.tls_cert_mode == "external":
+        return ExternalTlsDeployer(config.tls_cert_path, config.tls_key_path)
+    else:
+        raise ValueError(f"Unknown tls_cert_mode: {config.tls_cert_mode}")
+
+
+def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -> None:
     """Deploy a chat-mail instance.
 
     :param config_path: path to chatmail.ini
     :param disable_mail: whether to disable postfix & dovecot
+    :param website_only: if True, only deploy the website
     """
     config = read_config(config_path)
     check_config(config)
     mail_domain = config.mail_domain
 
-    if host.get_fact(Port, port=53) != "unbound":
-        files.line(
-            name="Add 9.9.9.9 to resolv.conf",
-            path="/etc/resolv.conf",
-            line="nameserver 9.9.9.9",
-        )
+    if website_only:
+        Deployment().perform_stages([WebsiteDeployer(config)])
+        return
 
-    port_services = [
-        (["master", "smtpd"], 25),
-        ("unbound", 53),
-        ("acmetool", 80),
-        (["imap-login", "dovecot"], 143),
-        ("nginx", 443),
-        (["master", "smtpd"], 465),
-        (["master", "smtpd"], 587),
-        (["imap-login", "dovecot"], 993),
-        ("iroh-relay", 3340),
-        ("nginx", 8443),
-        (["master", "smtpd"], config.postfix_reinject_port),
-        (["master", "smtpd"], config.postfix_reinject_port_incoming),
-        ("filtermail", config.filtermail_smtp_port),
-        ("filtermail", config.filtermail_smtp_port_incoming),
-    ]
-    for service, port in port_services:
-        print(f"Checking if port {port} is available for {service}...")
-        running_service = host.get_fact(Port, port=port)
-        if running_service:
-            if running_service not in service:
-                Out().red(
-                    f"Deploy failed: port {port} is occupied by: {running_service}"
-                )
-                exit(1)
+    # Check if mtail_address interface is available (if configured)
+    if config.mtail_address and config.mtail_address not in ('127.0.0.1', '::1', 'localhost'):
+        ipv4_addrs = host.get_fact(hardware.Ipv4Addrs)
+        all_addresses = [addr for addrs in ipv4_addrs.values() for addr in addrs]
+        if config.mtail_address not in all_addresses:
+            Out().red(f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n")
+            exit(1)
 
-    tls_domains = [mail_domain, f"mta-sts.{mail_domain}", f"www.{mail_domain}"]
+    if not is_in_container():
+        port_services = [
+            (["master", "smtpd"], 25),
+            ("unbound", 53),
+        ]
+        if config.tls_cert_mode == "acme":
+            port_services.append(("acmetool", 402))
+        port_services += [
+            (["imap-login", "dovecot"], 143),
+            # acmetool previously listened on port 80,
+            # so don't complain during upgrade that moved it to port 402
+            # and gave the port to nginx.
+            (["acmetool", "nginx"], 80),
+            ("nginx", 443),
+            (["master", "smtpd"], 465),
+            (["master", "smtpd"], 587),
+            (["imap-login", "dovecot"], 993),
+            ("iroh-relay", 3340),
+            ("mtail", 3903),
+            ("stats", 3904),
+            ("nginx", 8443),
+            (["master", "smtpd"], config.postfix_reinject_port),
+            (["master", "smtpd"], config.postfix_reinject_port_incoming),
+            ("filtermail", config.filtermail_smtp_port),
+            ("filtermail", config.filtermail_smtp_port_incoming),
+        ]
+        for service, port in port_services:
+            print(f"Checking if port {port} is available for {service}...")
+            running_service = host.get_fact(Port, port=port)
+            services = [service] if isinstance(service, str) else service
+            if running_service:
+                if running_service not in services:
+                    Out().red(
+                        f"Deploy failed: port {port} is occupied by: {running_service}"
+                    )
+                    exit(1)
+
+    tls_deployer = get_tls_deployer(config, mail_domain)
 
     all_deployers = [
-        ChatmailDeployer(mail_domain),
+        ChatmailDeployer(config),
         LegacyRemoveDeployer(),
+        FiltermailDeployer(),
         JournaldDeployer(),
-        UnboundDeployer(),
+        UnboundDeployer(config),
         TurnDeployer(mail_domain),
         IrohDeployer(config.enable_iroh_relay),
-        AcmetoolDeployer(config.acme_email, tls_domains),
+        tls_deployer,
         WebsiteDeployer(config),
         ChatmailVenvDeployer(config),
         MtastsDeployer(),

cmdeploy/src/cmdeploy/dns.py

@@ -1,25 +1,36 @@
 import datetime
-import importlib
-
-from jinja2 import Template
 
 from . import remote
 
 
+def parse_zone_records(text):
+    """Yield ``(name, ttl, rtype, rdata)`` from standard BIND-format text."""
+    for raw_line in text.splitlines():
+        line = raw_line.strip()
+        if not line or line.startswith(";"):
+            continue
+        try:
+            name, ttl, _in, rtype, rdata = line.split(None, 4)
+        except ValueError:
+            raise ValueError(f"Bad zone record line: {line!r}") from None
+        name = name.rstrip(".")
+        yield name, ttl, rtype.upper(), rdata
+
+
 def get_initial_remote_data(sshexec, mail_domain):
     return sshexec.logged(
         call=remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=mail_domain)
     )
 
 
-def check_initial_remote_data(remote_data, *, print=print):
+def check_initial_remote_data(remote_data, *, strict_tls=True, print=print):
     mail_domain = remote_data["mail_domain"]
     if not remote_data["A"] and not remote_data["AAAA"]:
         print(f"Missing A and/or AAAA DNS records for {mail_domain}!")
-    elif remote_data["MTA_STS"] != f"{mail_domain}.":
+    elif strict_tls and remote_data["MTA_STS"] != f"{mail_domain}.":
         print("Missing MTA-STS CNAME record:")
         print(f"mta-sts.{mail_domain}.   CNAME  {mail_domain}.")
-    elif remote_data["WWW"] != f"{mail_domain}.":
+    elif strict_tls and remote_data["WWW"] != f"{mail_domain}.":
         print("Missing www CNAME record:")
         print(f"www.{mail_domain}.   CNAME  {mail_domain}.")
     else:
@@ -31,13 +42,39 @@ def get_filled_zone_file(remote_data):
     if not sts_id:
         remote_data["sts_id"] = datetime.datetime.now().strftime("%Y%m%d%H%M")
 
-    template = importlib.resources.files(__package__).joinpath("chatmail.zone.j2")
-    content = template.read_text()
-    zonefile = Template(content).render(**remote_data)
-    lines = [x.strip() for x in zonefile.split("\n") if x.strip()]
+    d = remote_data["mail_domain"]
+
+    def append_record(name, rtype, rdata, ttl=3600):
+        lines.append(f"{name:<40} {ttl:<6} IN  {rtype:<5}  {rdata}")
+
+    lines = ["; Required DNS entries"]
+    if remote_data.get("A"):
+        append_record(f"{d}.", "A", remote_data["A"])
+    if remote_data.get("AAAA"):
+        append_record(f"{d}.", "AAAA", remote_data["AAAA"])
+    append_record(f"{d}.", "MX", f"10 {d}.")
+    if remote_data.get("strict_tls"):
+        append_record(f"_mta-sts.{d}.", "TXT", f'"v=STSv1; id={remote_data["sts_id"]}"')
+        append_record(f"mta-sts.{d}.", "CNAME", f"{d}.")
+    append_record(f"www.{d}.", "CNAME", f"{d}.")
+    lines.append(remote_data["dkim_entry"])
     lines.append("")
-    zonefile = "\n".join(lines)
-    return zonefile
+    lines.append("; Recommended DNS entries")
+    append_record(f"{d}.", "TXT", '"v=spf1 a ~all"')
+    append_record(f"_dmarc.{d}.", "TXT", '"v=DMARC1;p=reject;adkim=s;aspf=s"')
+    if remote_data.get("acme_account_url"):
+        append_record(
+            f"{d}.",
+            "CAA",
+            f'0 issue "letsencrypt.org;accounturi={remote_data["acme_account_url"]}"',
+        )
+    append_record(f"_adsp._domainkey.{d}.", "TXT", '"dkim=discardable"')
+    append_record(f"_submission._tcp.{d}.", "SRV", f"0 1 587 {d}.")
+    append_record(f"_submissions._tcp.{d}.", "SRV", f"0 1 465 {d}.")
+    append_record(f"_imap._tcp.{d}.", "SRV", f"0 1 143 {d}.")
+    append_record(f"_imaps._tcp.{d}.", "SRV", f"0 1 993 {d}.")
+    lines.append("")
+    return "\n".join(lines)
 
 
 def check_full_zone(sshexec, remote_data, out, zonefile) -> int:
@@ -58,7 +95,8 @@ def check_full_zone(sshexec, remote_data, out, zonefile) -> int:
         returncode = 1
     if remote_data.get("dkim_entry") in required_diff:
         out(
-            "If the DKIM entry above does not work with your DNS provider, you can try this one:\n"
+            "If the DKIM entry above does not work with your DNS provider,"
+            " you can try this one:\n"
         )
         out(remote_data.get("web_dkim_entry") + "\n")
     if recommended_diff:

cmdeploy/src/cmdeploy/dovecot/auth.conf

@@ -4,7 +4,7 @@ iterate_prefix = userdb/
 
 default_pass_scheme = plain
 # %E escapes characters " (double quote), ' (single quote) and \ (backslash) with \ (backslash).
-# See <https://doc.dovecot.org/configuration_manual/config_file/config_variables/#modifiers>
+# See <https://doc.dovecot.org/2.3/configuration_manual/config_file/config_variables/#modifiers>
 # for documentation.
 #
 # We escape user-provided input and use double quote as a separator.

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -1,16 +1,33 @@
+import io
+import urllib.request
+
 from chatmaild.config import Config
 from pyinfra import host
-from pyinfra.facts.server import Arch, Sysctl
-from pyinfra.facts.systemd import SystemdEnabled
+from pyinfra.facts.deb import DebPackages
+from pyinfra.facts.server import Arch, Command, Sysctl
 from pyinfra.operations import apt, files, server, systemd
 
 from cmdeploy.basedeploy import (
     Deployer,
     activate_remote_units,
+    blocked_service_startup,
     configure_remote_units,
     get_resource,
+    is_in_container,
 )
 
+DOVECOT_ARCHIVE_VERSION = "2.3.21+dfsg1-3"
+DOVECOT_PACKAGE_VERSION = f"1:{DOVECOT_ARCHIVE_VERSION}"
+
+DOVECOT_SHA256 = {
+    ("core", "amd64"): "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d",
+    ("core", "arm64"): "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9",
+    ("imapd", "amd64"): "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86",
+    ("imapd", "arm64"): "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f",
+    ("lmtpd", "amd64"): "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab",
+    ("lmtpd", "arm64"): "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f",
+}
+
 
 class DovecotDeployer(Deployer):
     daemon_reload = False
@@ -22,22 +39,62 @@ class DovecotDeployer(Deployer):
 
     def install(self):
         arch = host.get_fact(Arch)
-        if not "dovecot.service" in host.get_fact(SystemdEnabled):
-            _install_dovecot_package("core", arch)
-            _install_dovecot_package("imapd", arch)
-            _install_dovecot_package("lmtpd", arch)
+        with blocked_service_startup():
+            debs = []
+            for pkg in ("core", "imapd", "lmtpd"):
+                deb, changed = _download_dovecot_package(pkg, arch)
+                self.need_restart |= changed
+                if deb:
+                    debs.append(deb)
+            if debs:
+                deb_list = " ".join(debs)
+                # First dpkg may fail on missing dependencies (stderr suppressed);
+                # apt-get --fix-broken pulls them in, then dpkg retries cleanly.
+                server.shell(
+                    name="Install dovecot packages",
+                    commands=[
+                        f"dpkg --force-confdef --force-confold -i {deb_list} 2> /dev/null || true",
+                        "DEBIAN_FRONTEND=noninteractive apt-get -y --fix-broken install",
+                        f"dpkg --force-confdef --force-confold -i {deb_list}",
+                    ],
+                )
+                self.need_restart = True
+        files.put(
+            name="Pin dovecot packages to block Debian dist-upgrades",
+            src=io.StringIO(
+                "Package: dovecot-*\n"
+                "Pin: version *\n"
+                "Pin-Priority: -1\n"
+            ),
+            dest="/etc/apt/preferences.d/pin-dovecot",
+            user="root",
+            group="root",
+            mode="644",
+        )
 
     def configure(self):
         configure_remote_units(self.config.mail_domain, self.units)
-        self.need_restart, self.daemon_reload = _configure_dovecot(self.config)
+        config_restart, self.daemon_reload = _configure_dovecot(self.config)
+        self.need_restart |= config_restart
 
     def activate(self):
         activate_remote_units(self.units)
 
+        # Detect stale binary: package installed but service still runs old (deleted) binary.
+        if not self.disable_mail and not self.need_restart:
+            stale = host.get_fact(
+                Command,
+                'pid=$(systemctl show -p MainPID --value dovecot.service 2>/dev/null);'
+                ' [ "${pid:-0}" != "0" ] && readlink "/proc/$pid/exe" 2>/dev/null | grep -q "(deleted)"'
+                " && echo STALE || true",
+            )
+            if stale == "STALE":
+                self.need_restart = True
+
         restart = False if self.disable_mail else self.need_restart
 
         systemd.service(
-            name="disable dovecot for now"
+            name="Disable dovecot for now"
             if self.disable_mail
             else "Start and enable Dovecot",
             service="dovecot.service",
@@ -49,41 +106,49 @@ class DovecotDeployer(Deployer):
         self.need_restart = False
 
 
-def _install_dovecot_package(package: str, arch: str):
+def _pick_url(primary, fallback):
+    try:
+        req = urllib.request.Request(primary, method="HEAD")
+        urllib.request.urlopen(req, timeout=10)
+        return primary
+    except Exception:
+        return fallback
+
+
+def _download_dovecot_package(package: str, arch: str) -> tuple[str | None, bool]:
+    """Download a dovecot .deb if needed, return (path, changed)."""
     arch = "amd64" if arch == "x86_64" else arch
     arch = "arm64" if arch == "aarch64" else arch
-    url = f"https://download.delta.chat/dovecot/dovecot-{package}_2.3.21%2Bdfsg1-3_{arch}.deb"
-    deb_filename = "/root/" + url.split("/")[-1]
 
-    match (package, arch):
-        case ("core", "amd64"):
-            sha256 = "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d"
-        case ("core", "arm64"):
-            sha256 = "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9"
-        case ("imapd", "amd64"):
-            sha256 = "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86"
-        case ("imapd", "arm64"):
-            sha256 = "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f"
-        case ("lmtpd", "amd64"):
-            sha256 = "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab"
-        case ("lmtpd", "arm64"):
-            sha256 = "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f"
-        case _:
-            apt.packages(packages=[f"dovecot-{package}"])
-            return
+    pkg_name = f"dovecot-{package}"
+    sha256 = DOVECOT_SHA256.get((package, arch))
+    if sha256 is None:
+        op = apt.packages(packages=[pkg_name])
+        return None, bool(getattr(op, "changed", False))
+
+    installed_versions = host.get_fact(DebPackages).get(pkg_name, [])
+    if DOVECOT_PACKAGE_VERSION in installed_versions:
+        return None, False
+
+    url_version = DOVECOT_ARCHIVE_VERSION.replace("+", "%2B")
+    deb_base = f"{pkg_name}_{url_version}_{arch}.deb"
+    primary_url = f"https://download.delta.chat/dovecot/{deb_base}"
+    fallback_url = f"https://github.com/chatmail/dovecot/releases/download/upstream%2F{url_version}/{deb_base}"
+    url = _pick_url(primary_url, fallback_url)
+    deb_filename = f"/root/{deb_base}"
 
     files.download(
-        name=f"Download dovecot-{package}",
+        name=f"Download {pkg_name}",
         src=url,
         dest=deb_filename,
         sha256sum=sha256,
         cache_time=60 * 60 * 24 * 365 * 10,  # never redownload the package
     )
 
-    apt.deb(name=f"Install dovecot-{package}", src=deb_filename)
+    return deb_filename, True
 
 
-def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
+def _configure_dovecot(config: Config, debug: bool = False) -> tuple[bool, bool]:
     """Configures Dovecot IMAP server."""
     need_restart = False
     daemon_reload = False
@@ -116,13 +181,20 @@ def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
     )
     need_restart |= lua_push_notification_script.changed
 
-    # as per https://doc.dovecot.org/configuration_manual/os/
+    # as per https://doc.dovecot.org/2.3/configuration_manual/os/
     # it is recommended to set the following inotify limits
+    can_modify = not is_in_container()
     for name in ("max_user_instances", "max_user_watches"):
         key = f"fs.inotify.{name}"
-        if host.get_fact(Sysctl)[key] > 65535:
-            # Skip updating limits if already sufficient
-            # (enables running in incus containers where sysctl readonly)
+        value = host.get_fact(Sysctl).get(key, 0)
+        if value > 65534:
+            continue
+        if not can_modify:
+            print(
+                "\n!!!! refusing to attempt sysctl setting in containers\n"
+                f"!!!! dovecot: sysctl {key!r}={value}, should be >65534 for production setups\n"
+                "!!!!"
+            )
             continue
         server.sysctl(
             name=f"Change {key}",
@@ -145,4 +217,11 @@ def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
     )
     daemon_reload |= restart_conf.changed
 
+    # Validate dovecot configuration before restart
+    if need_restart:
+        server.shell(
+            name="Validate dovecot configuration",
+            commands=["doveconf -n >/dev/null"],
+        )
+
     return need_restart, daemon_reload

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -1,7 +1,7 @@
 ## Dovecot configuration file
 
 {% if disable_ipv6 %}
-listen = *
+listen = 0.0.0.0
 {% endif %}
 
 protocols = imap lmtp
@@ -26,7 +26,7 @@ default_client_limit = 20000
 # Increase number of logged in IMAP connections.
 # Each connection is handled by a separate `imap` process.
 # `imap` process should have `client_limit=1` as described in
-# <https://doc.dovecot.org/configuration_manual/service_configuration/#service-limits>
+# <https://doc.dovecot.org/2.3/configuration_manual/service_configuration/#service-limits>
 # so each logged in IMAP session will need its own `imap` process.
 #
 # If this limit is reached,
@@ -44,11 +44,11 @@ mail_server_comment = Chatmail server
 
 # `zlib` enables compressing messages stored in the maildir.
 # See
-# <https://doc.dovecot.org/configuration_manual/zlib_plugin/>
+# <https://doc.dovecot.org/2.3/configuration_manual/zlib_plugin/>
 # for documentation.
 #
 # quota plugin documentation:
-# <https://doc.dovecot.org/configuration_manual/quota_plugin/>
+# <https://doc.dovecot.org/2.3/configuration_manual/quota_plugin/>
 mail_plugins = zlib quota
 
 imap_capability = +XDELTAPUSH XCHATMAIL
@@ -125,14 +125,19 @@ plugin {
 
 protocol lmtp {
   # notify plugin is a dependency of push_notification plugin:
-  # <https://doc.dovecot.org/settings/plugin/notify-plugin/>
+  # <https://doc.dovecot.org/2.3/settings/plugin/notify-plugin/>
   #
   # push_notification plugin documentation:
-  # <https://doc.dovecot.org/configuration_manual/push_notification/>
+  # <https://doc.dovecot.org/2.3/configuration_manual/push_notification/>
   #
   # mail_lua and push_notification_lua are needed for Lua push notification handler.
-  # <https://doc.dovecot.org/configuration_manual/push_notification/#configuration>
+  # <https://doc.dovecot.org/2.3/configuration_manual/push_notification/#configuration>
   mail_plugins = $mail_plugins mail_lua notify push_notification push_notification_lua
+
+  # Disable fsync for LMTP. May lose delivered message,
+  # but unlikely to cause problems with multiple relays.
+  # https://doc.dovecot.org/2.3/admin_manual/mailbox_formats/#fsyncing
+  mail_fsync = never
 }
 
 plugin {
@@ -144,17 +149,31 @@ plugin {
 }
 
 plugin {
-  # for now we define static quota-rules for all users 
   quota = maildir:User quota
-  quota_rule = *:storage={{ config.max_mailbox_size }}
   quota_max_mail_size={{ config.max_message_size }}
   quota_grace = 0
-  # quota_over_flag_value = TRUE
+
+  quota_rule = *:storage={{ config.max_mailbox_size_mb }}M
+
+  # Trigger at 75%% of quota, expire oldest messages down to 70%%.
+  # The percentages are chosen to prevent current Delta Chat users
+  # from seeing "quota warnings" which trigger at 80% and 95%.
+
+  quota_warning = storage=75%% quota-warning {{ config.max_mailbox_size_mb * 70 // 100 }} {{ config.mailboxes_dir }}/%u
+}
+
+service quota-warning {
+  executable = script /usr/local/lib/chatmaild/venv/bin/chatmail-quota-expire
+  user = vmail
+  unix_listener quota-warning {
+     user = vmail
+     mode = 0600
+  }
 }
 
 # push_notification configuration
 plugin {
-  # <https://doc.dovecot.org/configuration_manual/push_notification/#lua-lua>
+  # <https://doc.dovecot.org/2.3/configuration_manual/push_notification/#lua-lua>
   push_notification_driver = lua:file=/etc/dovecot/push_notification.lua
 }
 
@@ -168,6 +187,8 @@ service lmtp {
   }
 }
 
+lmtp_add_received_header = no
+
 service auth {
   unix_listener /var/spool/postfix/private/auth {
     mode = 0660
@@ -226,8 +247,8 @@ service anvil {
 }
 
 ssl = required
-ssl_cert = </var/lib/acme/live/{{ config.mail_domain }}/fullchain
-ssl_key = </var/lib/acme/live/{{ config.mail_domain }}/privkey
+ssl_cert = <{{ config.tls_cert_path }}
+ssl_key = <{{ config.tls_key_path }}
 ssl_dh = </usr/share/dovecot/dh.pem
 ssl_min_protocol = TLSv1.3
 ssl_prefer_server_ciphers = yes
@@ -250,6 +271,9 @@ protocol imap {
   # sort -sn <(sed 's/ / C: /' *.in) <(sed 's/ / S: /' cat *.out)
 
   rawlog_dir = %h 
+
+  # Disable fsync for IMAP. May lose IMAP changes like setting flags. 
+  mail_fsync = never
 } 
 {% endif %}
 
@@ -277,3 +301,156 @@ service imap-hibernate {
   }
 }
 {% endif %}
+
+{% if config.mtail_address %}
+#
+# Dovecot Statistics
+#
+# OpenMetrics endpoint at http://{{- config.mtail_address}}:3904/metrics
+service stats {
+  inet_listener http {
+    port = 3904
+    address = {{- config.mtail_address}}
+  }
+}
+
+# IMAP Command Metrics
+# - Bytes in/out for compression efficiency analysis
+# - Lock wait time for contention debugging
+# - Grouped by command name and reply state
+metric imap_command {
+  filter = event=imap_command_finished
+  fields = bytes_in bytes_out lock_wait_usecs running_usecs
+  group_by = cmd_name tagged_reply_state
+}
+
+# Duration buckets for latency histograms (base 10: 10us, 100us, 1ms, 10ms, 100ms, 1s, 10s, 100s)
+metric imap_command_duration {
+  filter = event=imap_command_finished
+  group_by = cmd_name duration:exponential:1:8:10
+}
+
+# Slow command outliers (>1 second = 1000000 usecs)
+# Useful for alerting without high cardinality
+metric imap_command_slow {
+  filter = event=imap_command_finished AND duration>1000000 AND NOT cmd_name=IDLE
+  group_by = cmd_name
+}
+
+# IDLE-specific Metrics
+
+metric imap_idle {
+  filter = event=imap_command_finished AND cmd_name=IDLE
+  fields = bytes_in bytes_out running_usecs
+  group_by = tagged_reply_state
+}
+
+metric imap_idle_duration {
+  filter = event=imap_command_finished AND cmd_name=IDLE
+  # Base 10: 100ms to 27h (covers short wakeups to long idle sessions)
+  group_by = duration:exponential:5:11:10
+}
+
+metric imap_idle_commands {
+  filter = event=imap_command_finished AND cmd_name=IDLE
+  group_by = tagged_reply_state
+}
+
+metric imap_idle_failed {
+  filter = event=imap_command_finished AND cmd_name=IDLE AND NOT tagged_reply_state=OK
+}
+
+# Hibernation Metrics (requires imap_hibernate_timeout)
+
+metric imap_hibernated {
+  filter = event=imap_client_hibernated
+}
+
+metric imap_hibernated_failed {
+  filter = event=imap_client_hibernated AND error=*
+}
+
+metric imap_unhibernated {
+  filter = event=imap_client_unhibernated
+  fields = hibernation_usecs
+}
+
+metric imap_unhibernated_reason {
+  filter = event=imap_client_unhibernated
+  group_by = reason
+  fields = hibernation_usecs
+}
+
+metric imap_unhibernated_reason_sleep {
+  filter = event=imap_client_unhibernated
+  group_by = reason hibernation_usecs:exponential:4:8:10
+}
+
+metric imap_unhibernated_failed {
+  filter = event=imap_client_unhibernated AND error=*
+}
+
+# Hibernation duration buckets (how long clients stayed hibernated)
+# Base 10: 100ms to 27h
+metric imap_hibernation_duration {
+  filter = event=imap_client_unhibernated
+  group_by = reason duration:exponential:5:11:10
+}
+
+# Authentication / Login Metrics
+
+metric auth_request {
+  filter = event=auth_request_finished
+  group_by = success
+}
+
+metric auth_request_duration {
+  filter = event=auth_request_finished
+  group_by = success duration:exponential:2:6:10
+}
+
+metric auth_failed {
+  filter = event=auth_request_finished AND success=no
+}
+
+# Passdb cache effectiveness
+metric auth_passdb {
+  filter = event=auth_passdb_request_finished
+  group_by = result cache
+}
+
+# Master login (post-auth userdb lookup)
+metric auth_master_login {
+  filter = event=auth_master_client_login_finished
+}
+
+metric auth_master_login_failed {
+  filter = event=auth_master_client_login_finished AND error=*
+}
+
+# Mail Delivery (LMTP) - affects IDLE wakeup latency
+
+metric mail_delivery {
+  filter = event=mail_delivery_finished
+}
+
+metric mail_delivery_duration {
+  filter = event=mail_delivery_finished
+  group_by = duration:exponential:3:7:10
+}
+
+metric mail_delivery_failed {
+  filter = event=mail_delivery_finished AND error=*
+}
+
+# Connection Events
+
+metric client_connected {
+  filter = event=client_connection_connected AND category="service:imap"
+}
+
+metric client_disconnected {
+  filter = event=client_connection_disconnected AND category="service:imap"
+  fields = bytes_in bytes_out
+}
+{% endif %}

cmdeploy/src/cmdeploy/external/deployer.py

@@ -0,0 +1,67 @@
+import io
+
+from pyinfra import host
+from pyinfra.facts.files import File
+from pyinfra.operations import files, systemd
+
+from cmdeploy.basedeploy import Deployer, get_resource
+
+
+class ExternalTlsDeployer(Deployer):
+    """Expects TLS certificates to be managed on the server.
+
+    Validates that the configured certificate and key files
+    exist on the remote host.  Installs a systemd path unit
+    that watches the certificate file and automatically
+    restarts/reloads affected services when it changes.
+    """
+
+    def __init__(self, cert_path, key_path):
+        self.cert_path = cert_path
+        self.key_path = key_path
+
+    def configure(self):
+        # Verify cert and key exist on the remote host using pyinfra facts.
+        for path in (self.cert_path, self.key_path):
+            info = host.get_fact(File, path=path)
+            if info is None:
+                raise Exception(f"External TLS file not found on server: {path}")
+
+        # Deploy the .path unit (templated with the cert path).
+        # pkg=__package__ is required here because the resource files
+        # live in cmdeploy.external, not the default cmdeploy package.
+        source = get_resource("tls-cert-reload.path.f", pkg=__package__)
+        content = source.read_text().format(cert_path=self.cert_path).encode()
+
+        path_unit = files.put(
+            name="Upload tls-cert-reload.path",
+            src=io.BytesIO(content),
+            dest="/etc/systemd/system/tls-cert-reload.path",
+            user="root",
+            group="root",
+            mode="644",
+        )
+
+        service_unit = files.put(
+            name="Upload tls-cert-reload.service",
+            src=get_resource("tls-cert-reload.service", pkg=__package__),
+            dest="/etc/systemd/system/tls-cert-reload.service",
+            user="root",
+            group="root",
+            mode="644",
+        )
+
+        if path_unit.changed or service_unit.changed:
+            self.need_restart = True
+
+    def activate(self):
+        systemd.service(
+            name="Enable tls-cert-reload path watcher",
+            service="tls-cert-reload.path",
+            running=True,
+            enabled=True,
+            restarted=self.need_restart,
+            daemon_reload=self.need_restart,
+        )
+        # No explicit reload needed here: dovecot/nginx read the cert
+        # on startup, and the .path watcher handles live changes.

cmdeploy/src/cmdeploy/external/tls-cert-reload.path.f

@@ -0,0 +1,15 @@
+# Watch the TLS certificate file for changes.
+# When the cert is updated (e.g. renewed by an external process),
+# this triggers tls-cert-reload.service to reload the affected services.
+#
+# NOTE: changes to the certificates are not detected if they cross bind-mount boundaries. 
+# After cert renewal, you must then trigger the reload explicitly:
+#   systemctl start tls-cert-reload.service
+[Unit]
+Description=Watch TLS certificate for changes
+
+[Path]
+PathChanged={cert_path}
+
+[Install]
+WantedBy=multi-user.target

cmdeploy/src/cmdeploy/external/tls-cert-reload.service

@@ -0,0 +1,15 @@
+# Reload services that cache the TLS certificate.
+#
+# dovecot: caches the cert at startup; reload re-reads SSL certs
+#          without dropping existing connections.
+# nginx:   caches the cert at startup; reload gracefully picks up
+#          the new cert for new connections.
+# postfix: reads the cert fresh on each TLS handshake,
+#          does NOT need a reload/restart.
+[Unit]
+Description=Reload TLS services after certificate change
+
+[Service]
+Type=oneshot
+ExecStart=/bin/systemctl try-reload-or-restart dovecot
+ExecStart=/bin/systemctl try-reload-or-restart nginx

cmdeploy/src/cmdeploy/filtermail/deployer.py

@@ -0,0 +1,64 @@
+import os
+
+from pyinfra import facts, host
+from pyinfra.operations import files, systemd
+
+from cmdeploy.basedeploy import Deployer, get_resource
+
+
+class FiltermailDeployer(Deployer):
+    services = ["filtermail", "filtermail-incoming", "filtermail-transport"]
+    bin_path = "/usr/local/bin/filtermail"
+    config_path = "/usr/local/lib/chatmaild/chatmail.ini"
+
+    def __init__(self):
+        self.need_restart = False
+
+    def install(self):
+        local_bin = os.environ.get("CHATMAIL_FILTERMAIL_BINARY")
+        if local_bin:
+            self.need_restart |= files.put(
+                name="Upload locally built filtermail",
+                src=local_bin,
+                dest=self.bin_path,
+                mode="755",
+            ).changed
+            return
+
+        arch = host.get_fact(facts.server.Arch)
+        url = f"https://github.com/chatmail/filtermail/releases/download/v0.6.4/filtermail-{arch}"
+        sha256sum = {
+            "x86_64": "5295115952c72e4c4ec3c85546e094b4155a4c702c82bd71fcdcb744dc73adf6",
+            "aarch64": "6892244f17b8f26ccb465766e96028e7222b3c8adefca9fc6bfe9ff332ca8dff",
+        }[arch]
+        self.need_restart |= files.download(
+            name="Download filtermail",
+            src=url,
+            sha256sum=sha256sum,
+            dest=self.bin_path,
+            mode="755",
+        ).changed
+
+    def configure(self):
+        for service in self.services:
+            self.need_restart |= files.template(
+                src=get_resource(f"filtermail/{service}.service.j2"),
+                dest=f"/etc/systemd/system/{service}.service",
+                user="root",
+                group="root",
+                mode="644",
+                bin_path=self.bin_path,
+                config_path=self.config_path,
+            ).changed
+
+    def activate(self):
+        for service in self.services:
+            systemd.service(
+                name=f"Start and enable {service}",
+                service=f"{service}.service",
+                running=True,
+                enabled=True,
+                restarted=self.need_restart,
+                daemon_reload=True,
+            )
+        self.need_restart = False

cmdeploy/src/cmdeploy/filtermail/filtermail-incoming.service.j2

@@ -2,11 +2,10 @@
 Description=Incoming Chatmail Postfix before queue filter 
 
 [Service]
-ExecStart={execpath} {config_path} incoming
+ExecStart={{ bin_path }} {{ config_path }} incoming
 Restart=always
 RestartSec=30
 User=vmail
 
 [Install]
 WantedBy=multi-user.target
-

cmdeploy/src/cmdeploy/filtermail/filtermail-transport.service.j2

@@ -0,0 +1,11 @@
+[Unit]
+Description=Chatmail transport service
+
+[Service]
+ExecStart={{ bin_path }} {{ config_path }} transport
+Restart=always
+RestartSec=30
+User=vmail
+
+[Install]
+WantedBy=multi-user.target

cmdeploy/src/cmdeploy/filtermail/filtermail.service.j2

@@ -2,7 +2,7 @@
 Description=Outgoing Chatmail Postfix before queue filter
 
 [Service]
-ExecStart={execpath} {config_path} outgoing
+ExecStart={{ bin_path }} {{ config_path }} outgoing
 Restart=always
 RestartSec=30
 User=vmail

cmdeploy/src/cmdeploy/metrics.cron.j2

@@ -1 +0,0 @@
-*/5 * * * * root {{ config.execpath }} {{ config.mailboxes_dir }} >/var/www/html/metrics

cmdeploy/src/cmdeploy/mtail/delivered_mail.mtail

@@ -44,21 +44,45 @@ counter warning_count
 }
 
 
-counter filtered_mail_count
+counter filtered_outgoing_mail_count
 
-counter encrypted_mail_count
-/Filtering encrypted mail\./ {
-  encrypted_mail_count++
-  filtered_mail_count++
+counter outgoing_encrypted_mail_count
+/Outgoing: Filtering encrypted mail\./ {
+  outgoing_encrypted_mail_count++
+  filtered_outgoing_mail_count++
 }
 
-counter unencrypted_mail_count
-/Filtering unencrypted mail\./ {
-  unencrypted_mail_count++
-  filtered_mail_count++
+counter outgoing_unencrypted_mail_count
+/Outgoing: Filtering unencrypted mail\./ {
+  outgoing_unencrypted_mail_count++
+  filtered_outgoing_mail_count++
 }
 
+
+counter filtered_incoming_mail_count
+
+counter incoming_encrypted_mail_count
+/Incoming: Filtering encrypted mail\./ {
+  incoming_encrypted_mail_count++
+  filtered_incoming_mail_count++
+}
+
+counter incoming_unencrypted_mail_count
+/Incoming: Filtering unencrypted mail\./ {
+  incoming_unencrypted_mail_count++
+  filtered_incoming_mail_count++
+}
+
+
 counter rejected_unencrypted_mail_count
-/Rejected unencrypted mail\./ {
+/Rejected unencrypted mail/ {
   rejected_unencrypted_mail_count++
 }
+
+counter quota_expire_runs
+counter quota_expire_removed_files
+
+/quota-expire: removed (?P<count>\d+) message\(s\)/ {
+  quota_expire_runs++
+  quota_expire_removed_files += $count
+}

cmdeploy/src/cmdeploy/mtail/mtail.service.j2

@@ -1,5 +1,6 @@
 [Unit]
 Description=mtail
+After=multi-user.target
 
 [Service]
 Type=simple

cmdeploy/src/cmdeploy/nginx/autoconfig.xml.j2

@@ -1,47 +1,47 @@
 <?xml version="1.0" encoding="UTF-8"?>
 
 <clientConfig version="1.1">
-  <emailProvider id="{{ config.domain_name }}">
-    <domain>{{ config.domain_name }}</domain>
-    <displayName>{{ config.domain_name }} chatmail</displayName>
-    <displayShortName>{{ config.domain_name }}</displayShortName>
+  <emailProvider id="{{ config.mail_domain }}">
+    <domain>{{ config.mail_domain }}</domain>
+    <displayName>{{ config.mail_domain }} chatmail</displayName>
+    <displayShortName>{{ config.mail_domain }}</displayShortName>
     <incomingServer type="imap">
-      <hostname>{{ config.domain_name }}</hostname>
+      <hostname>{{ config.mail_domain }}</hostname>
       <port>993</port>
       <socketType>SSL</socketType>
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
     </incomingServer>
     <incomingServer type="imap">
-      <hostname>{{ config.domain_name }}</hostname>
+      <hostname>{{ config.mail_domain }}</hostname>
       <port>143</port>
       <socketType>STARTTLS</socketType>
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
     </incomingServer>
     <incomingServer type="imap">
-      <hostname>{{ config.domain_name }}</hostname>
+      <hostname>{{ config.mail_domain }}</hostname>
       <port>443</port>
       <socketType>SSL</socketType>
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
     </incomingServer>
     <outgoingServer type="smtp">
-      <hostname>{{ config.domain_name }}</hostname>
+      <hostname>{{ config.mail_domain }}</hostname>
       <port>465</port>
       <socketType>SSL</socketType>
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
     </outgoingServer>
     <outgoingServer type="smtp">
-      <hostname>{{ config.domain_name }}</hostname>
+      <hostname>{{ config.mail_domain }}</hostname>
       <port>587</port>
       <socketType>STARTTLS</socketType>
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
     </outgoingServer>
     <outgoingServer type="smtp">
-      <hostname>{{ config.domain_name }}</hostname>
+      <hostname>{{ config.mail_domain }}</hostname>
       <port>443</port>
       <socketType>SSL</socketType>
       <authentication>password-cleartext</authentication>

cmdeploy/src/cmdeploy/nginx/deployer.py

@@ -70,7 +70,7 @@ def _configure_nginx(config: Config, debug: bool = False) -> bool:
         user="root",
         group="root",
         mode="644",
-        config={"domain_name": config.mail_domain},
+        config=config,
         disable_ipv6=config.disable_ipv6,
     )
     need_restart |= main_config.changed
@@ -81,7 +81,7 @@ def _configure_nginx(config: Config, debug: bool = False) -> bool:
         user="root",
         group="root",
         mode="644",
-        config={"domain_name": config.mail_domain},
+        config=config,
     )
     need_restart |= autoconfig.changed
 
@@ -91,7 +91,7 @@ def _configure_nginx(config: Config, debug: bool = False) -> bool:
         user="root",
         group="root",
         mode="644",
-        config={"domain_name": config.mail_domain},
+        config=config,
     )
     need_restart |= mta_sts_config.changed
 

cmdeploy/src/cmdeploy/nginx/mta-sts.txt.j2

@@ -1,4 +1,4 @@
 version: STSv1
 mode: enforce
-mx: {{ config.domain_name }}
+mx: {{ config.mail_domain }}
 max_age: 2419200

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -42,6 +42,9 @@ stream {
 }
 
 http {
+{% if config.tls_cert_mode == "self" %}
+	limit_req_zone $binary_remote_addr zone=newaccount:10m rate=2r/s;
+{% endif %}
 	sendfile on;
 	tcp_nopush on;
 
@@ -51,10 +54,10 @@ http {
 	include /etc/nginx/mime.types;
 	default_type application/octet-stream;
 
-	ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
+	ssl_protocols TLSv1.2 TLSv1.3;
 	ssl_prefer_server_ciphers on;
-	ssl_certificate /var/lib/acme/live/{{ config.domain_name }}/fullchain;
-	ssl_certificate_key /var/lib/acme/live/{{ config.domain_name }}/privkey;
+	ssl_certificate {{ config.tls_cert_path }};
+	ssl_certificate_key {{ config.tls_key_path }};
 
 	gzip on;
 
@@ -66,26 +69,30 @@ http {
 
 		index index.html index.htm;
 
-		server_name {{ config.domain_name }} www.{{ config.domain_name }} mta-sts.{{ config.domain_name }};
+		server_name {{ config.mail_domain }} www.{{ config.mail_domain }} mta-sts.{{ config.mail_domain }};
 
 		access_log syslog:server=unix:/dev/log,facility=local7;
 
+		location /mxdeliv {
+		    proxy_pass http://127.0.0.1:{{ config.filtermail_http_port_incoming }};
+		}
+
 		location / {
 			# First attempt to serve request as file, then
 			# as directory, then fall back to displaying a 404.
 			try_files $uri $uri/ =404;
 		}
 
-		location /metrics {
-			default_type text/plain;
-		}
-
 		location /new {
+{% if config.tls_cert_mode != "self" %}
 			if ($request_method = GET) {
 				# Redirect to Delta Chat,
 				# which will in turn do a POST request.
-				return 301 dcaccount:https://{{ config.domain_name }}/new;
+				return 301 dcaccount:https://{{ config.mail_domain }}/new;
 			}
+{% else %}
+			limit_req zone=newaccount burst=5 nodelay;
+{% endif %}
 
 			fastcgi_pass unix:/run/fcgiwrap.socket;
 			include /etc/nginx/fastcgi_params;
@@ -99,9 +106,11 @@ http {
 		#
 		# Redirects are only for browsers.
 		location /cgi-bin/newemail.py {
+{% if config.tls_cert_mode != "self" %}
 			if ($request_method = GET) {
-				return 301 dcaccount:https://{{ config.domain_name }}/new;
+				return 301 dcaccount:https://{{ config.mail_domain }}/new;
 			}
+{% endif %}
 
 			fastcgi_pass unix:/run/fcgiwrap.socket;
 			include /etc/nginx/fastcgi_params;
@@ -132,8 +141,29 @@ http {
 	# Redirect www. to non-www
 	server {
 		listen 127.0.0.1:8443 ssl;
-		server_name www.{{ config.domain_name }};
-		return 301 $scheme://{{ config.domain_name }}$request_uri;
+		server_name www.{{ config.mail_domain }};
+		return 301 $scheme://{{ config.mail_domain }}$request_uri;
 		access_log syslog:server=unix:/dev/log,facility=local7;
 	}
+
+	server {
+		listen 80;
+		{% if not disable_ipv6 %}
+		listen [::]:80;
+		{% endif %}
+
+		{% if config.tls_cert_mode == "acme" %}
+		location /.well-known/acme-challenge/ {
+			proxy_pass http://acmetool;
+		}
+		{% endif %}
+
+		return 301 https://$host$request_uri;
+	}
+
+	{% if config.tls_cert_mode == "acme" %}
+	upstream acmetool {
+		server 127.0.0.1:402;
+	}
+	{% endif %}
 }

cmdeploy/src/cmdeploy/opendkim/deployer.py

@@ -37,21 +37,15 @@ class OpendkimDeployer(Deployer):
         )
         need_restart |= main_config.changed
 
-        screen_script = files.put(
-            src=get_resource("opendkim/screen.lua"),
-            dest="/etc/opendkim/screen.lua",
-            user="root",
-            group="root",
-            mode="644",
+        screen_script = files.file(
+            path="/etc/opendkim/screen.lua",
+            present=False,
         )
         need_restart |= screen_script.changed
 
-        final_script = files.put(
-            src=get_resource("opendkim/final.lua"),
-            dest="/etc/opendkim/final.lua",
-            user="root",
-            group="root",
-            mode="644",
+        final_script = files.file(
+            path="/etc/opendkim/final.lua",
+            present=False,
         )
         need_restart |= final_script.changed
 
@@ -109,6 +103,13 @@ class OpendkimDeployer(Deployer):
         )
         need_restart |= service_file.changed
 
+        files.file(
+            name="chown opendkim: /etc/dkimkeys/opendkim.private",
+            path="/etc/dkimkeys/opendkim.private",
+            user="opendkim",
+            group="opendkim",
+        )
+
         self.need_restart = need_restart
 
     def activate(self):

cmdeploy/src/cmdeploy/opendkim/final.lua

@@ -1,41 +0,0 @@
-if odkim.internal_ip(ctx) == 1 then
-	-- Outgoing message will be signed,
-	-- no need to look for signatures.
-	return nil
-end
-
-nsigs = odkim.get_sigcount(ctx)
-if nsigs == nil then
-	return nil
-end
-
-local valid = false
-local error_msg = "No valid DKIM signature found."
-for i = 1, nsigs do
-	sig = odkim.get_sighandle(ctx, i - 1)
-	sigres = odkim.sig_result(sig)
-
-	-- All signatures that do not correspond to From: 
-	-- were ignored in screen.lua and return sigres -1.
-	-- 
-	-- Any valid signature that was not ignored like this
-	-- means the message is acceptable.
-	if sigres == 0 then
-		valid = true
-    else
-        error_msg = "DKIM signature is invalid, error code " .. tostring(sigres) .. ", search https://github.com/trusteddomainproject/OpenDKIM/blob/master/libopendkim/dkim.h#L108"
-	end
-end
-
-if valid then
-	-- Strip all DKIM-Signature headers after successful validation
-	-- Delete in reverse order to avoid index shifting.
-	for i = nsigs, 1, -1 do
-		odkim.del_header(ctx, "DKIM-Signature", i)
-	end
-else
-	odkim.set_reply(ctx, "554", "5.7.1", error_msg)
-	odkim.set_result(ctx, SMFIS_REJECT)
-end
-
-return nil

cmdeploy/src/cmdeploy/opendkim/opendkim.conf

@@ -45,12 +45,6 @@ SignHeaders *,+autocrypt,+content-type
 # Default is empty.
 OversignHeaders from,reply-to,subject,date,to,cc,resent-date,resent-from,resent-sender,resent-to,resent-cc,in-reply-to,references,list-id,list-help,list-unsubscribe,list-subscribe,list-post,list-owner,list-archive,autocrypt
 
-# Script to ignore signatures that do not correspond to the From: domain.
-ScreenPolicyScript /etc/opendkim/screen.lua
-
-# Script to reject mails without a valid DKIM signature.
-FinalPolicyScript /etc/opendkim/final.lua
-
 # In Debian, opendkim runs as user "opendkim". A umask of 007 is required when
 # using a local socket with MTAs that access the socket as a non-privileged
 # user (for example, Postfix). You may need to add user "postfix" to group
@@ -65,3 +59,9 @@ PidFile			/run/opendkim/opendkim.pid
 # The trust anchor enables DNSSEC. In Debian, the trust anchor file is provided
 # by the package dns-root-data.
 TrustAnchorFile		/usr/share/dns/root.key
+
+# Sign messages when `-o milter_macro_daemon_name=ORIGINATING` is set.
+MTA ORIGINATING
+
+# No hosts are treated as internal, ORIGINATING daemon name should be set explicitly.
+InternalHosts -

cmdeploy/src/cmdeploy/opendkim/screen.lua

@@ -1,21 +0,0 @@
--- Ignore signatures that do not correspond to the From: domain.
-
-from_domain = odkim.get_fromdomain(ctx)
-if from_domain == nil then
-	return nil
-end
-
-n = odkim.get_sigcount(ctx)
-if n == nil then
-	return nil
-end
-
-for i = 1, n do
-	sig = odkim.get_sighandle(ctx, i - 1)
-	sig_domain = odkim.sig_getdomain(sig)
-	if from_domain ~= sig_domain then
-		odkim.sig_ignore(sig)
-	end
-end
-
-return nil

cmdeploy/src/cmdeploy/postfix/deployer.py

@@ -1,4 +1,4 @@
-from pyinfra.operations import apt, files, systemd
+from pyinfra.operations import apt, files, server, systemd
 
 from cmdeploy.basedeploy import Deployer, get_resource
 
@@ -52,6 +52,29 @@ class PostfixDeployer(Deployer):
         )
         need_restart |= header_cleanup.changed
 
+        lmtp_header_cleanup = files.put(
+            src=get_resource("postfix/lmtp_header_cleanup"),
+            dest="/etc/postfix/lmtp_header_cleanup",
+            user="root",
+            group="root",
+            mode="644",
+        )
+        need_restart |= lmtp_header_cleanup.changed
+
+        tls_policy_map = files.put(
+            name="Upload SMTP TLS Policy that accepts self-signed certificates for IP-only hosts",
+            src=get_resource("postfix/smtp_tls_policy_map"),
+            dest="/etc/postfix/smtp_tls_policy_map",
+            user="root",
+            group="root",
+            mode="644",
+        )
+        need_restart |= tls_policy_map.changed
+        if tls_policy_map.changed:
+            server.shell(
+                commands=["postmap /etc/postfix/smtp_tls_policy_map"],
+            )
+
         # Login map that 1:1 maps email address to login.
         login_map = files.put(
             src=get_resource("postfix/login_map"),
@@ -65,9 +88,19 @@ class PostfixDeployer(Deployer):
         restart_conf = files.put(
             name="postfix: restart automatically on failure",
             src=get_resource("service/10_restart.conf"),
-            dest="/etc/systemd/system/dovecot.service.d/10_restart.conf",
+            dest="/etc/systemd/system/postfix@.service.d/10_restart.conf",
         )
         self.daemon_reload = restart_conf.changed
+
+        # Validate postfix configuration before restart
+        if need_restart:
+            server.shell(
+                name="Validate postfix configuration",
+                # Extract stderr and quit with error if non-zero
+                commands=[
+                    """bash -c 'w=$(postconf 2>&1 >/dev/null); [[ -z "$w" ]] || { echo "$w"; false; }'"""
+                ],
+            )
         self.need_restart = need_restart
 
     def activate(self):

cmdeploy/src/cmdeploy/postfix/lmtp_header_cleanup

@@ -0,0 +1,3 @@
+/^DKIM-Signature:/            IGNORE
+/^Authentication-Results:/            IGNORE
+/^Received:/            IGNORE

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -15,8 +15,8 @@ readme_directory = no
 compatibility_level = 3.6
 
 # TLS parameters
-smtpd_tls_cert_file=/var/lib/acme/live/{{ config.mail_domain }}/fullchain
-smtpd_tls_key_file=/var/lib/acme/live/{{ config.mail_domain }}/privkey
+smtpd_tls_cert_file={{ config.tls_cert_path }}
+smtpd_tls_key_file={{ config.tls_key_path }}
 smtpd_tls_security_level=may
 
 smtp_tls_CApath=/etc/ssl/certs
@@ -25,7 +25,7 @@ smtp_tls_security_level=verify
 # <https://www.postfix.org/postconf.5.html#smtp_tls_servername>
 smtp_tls_servername = hostname
 smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
-smtp_tls_policy_maps = inline:{nauta.cu=may}
+smtp_tls_policy_maps = regexp:/etc/postfix/smtp_tls_policy_map
 smtp_tls_protocols = >=TLSv1.2
 smtp_tls_mandatory_protocols = >=TLSv1.2
 
@@ -64,7 +64,11 @@ alias_database = hash:/etc/aliases
 mydestination =
 
 relayhost = 
+{% if disable_ipv6 %}
+mynetworks = 127.0.0.0/8
+{% else %}
 mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128
+{% endif %}
 mailbox_size_limit = 0
 message_size_limit = {{config.max_message_size}}
 recipient_delimiter = +
@@ -77,6 +81,7 @@ inet_protocols = all
 
 virtual_transport = lmtp:unix:private/dovecot-lmtp
 virtual_mailbox_domains = {{ config.mail_domain }}
+lmtp_header_checks = regexp:/etc/postfix/lmtp_header_cleanup
 
 mua_client_restrictions = permit_sasl_authenticated, reject
 mua_sender_restrictions = reject_sender_login_mismatch, permit_sasl_authenticated, reject
@@ -88,3 +93,10 @@ smtpd_sender_login_maps = regexp:/etc/postfix/login_map
 # Do not lookup SMTP client hostnames to reduce delays
 # and avoid unnecessary DNS requests.
 smtpd_peername_lookup = no
+
+# Use filtermail-transport to relay messages.
+# We can't force postfix to split messages per destination,
+# when specifying a custom next-hop,
+# so instead this is handled in filtermail.
+# We use LMTP instead SMTP so we can communicate per-recipient errors back to postfix.
+default_transport = lmtp-filtermail:inet:[127.0.0.1]:{{ config.filtermail_lmtp_port_transport }}

cmdeploy/src/cmdeploy/postfix/master.cf.j2

@@ -31,7 +31,6 @@ submission inet n       -       y       -       5000    smtpd
   -o smtpd_sender_restrictions=$mua_sender_restrictions
   -o smtpd_recipient_restrictions=
   -o smtpd_relay_restrictions=permit_sasl_authenticated,reject
-  -o milter_macro_daemon_name=ORIGINATING
   -o smtpd_client_connection_count_limit=1000
   -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
 smtps     inet  n       -       y       -       5000    smtpd
@@ -49,7 +48,6 @@ smtps     inet  n       -       y       -       5000    smtpd
   -o smtpd_recipient_restrictions=
   -o smtpd_relay_restrictions=permit_sasl_authenticated,reject
   -o smtpd_client_connection_count_limit=1000
-  -o milter_macro_daemon_name=ORIGINATING
   -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
 #628       inet  n       -       y       -       -       qmqpd
 pickup    unix  n       -       y       60      1       pickup
@@ -81,13 +79,13 @@ filter    unix -        n       n       -       -       lmtp
 # Local SMTP server for reinjecting outgoing filtered mail.
 127.0.0.1:{{ config.postfix_reinject_port }} inet  n       -       n       -       100      smtpd
   -o syslog_name=postfix/reinject
+  -o milter_macro_daemon_name=ORIGINATING
   -o smtpd_milters=unix:opendkim/opendkim.sock
   -o cleanup_service_name=authclean
 
 # Local SMTP server for reinjecting incoming filtered mail 
 127.0.0.1:{{ config.postfix_reinject_port_incoming }} inet  n       -       n       -       100      smtpd
   -o syslog_name=postfix/reinject_incoming
-  -o smtpd_milters=unix:opendkim/opendkim.sock
 
 # Cleanup `Received` headers for authenticated mail
 # to avoid leaking client IP.
@@ -102,3 +100,8 @@ filter    unix -        n       n       -       -       lmtp
 # cannot send unprotected Subject.
 authclean unix  n       -       -       -       0       cleanup
   -o header_checks=regexp:/etc/postfix/submission_header_cleanup
+
+lmtp-filtermail unix  -       -       y       -       -       lmtp
+  -o syslog_name=postfix/lmtp-filtermail
+  -o lmtp_header_checks=
+  -o lmtp_tls_security_level=none

cmdeploy/src/cmdeploy/postfix/smtp_tls_policy_map

@@ -0,0 +1,3 @@
+/^\[[^]]+\]$/ encrypt
+/^_/          encrypt
+/^nauta\.cu$/    may

cmdeploy/src/cmdeploy/remote/rdns.py

@@ -53,13 +53,14 @@ def get_dkim_entry(mail_domain, pre_command, dkim_selector):
             print=log_progress,
         )
     except CalledProcessError:
-        return
+        return None, None
     dkim_value_raw = f"v=DKIM1;k=rsa;p={dkim_pubkey};s=email;t=s"
     dkim_value = '" "'.join(re.findall(".{1,255}", dkim_value_raw))
     web_dkim_value = "".join(re.findall(".{1,255}", dkim_value_raw))
+    name = f"{dkim_selector}._domainkey.{mail_domain}."
     return (
-        f'{dkim_selector}._domainkey.{mail_domain}. TXT "{dkim_value}"',
-        f'{dkim_selector}._domainkey.{mail_domain}. TXT "{web_dkim_value}"',
+        f'{name:<40} 3600   IN  TXT    "{dkim_value}"',
+        f'{name:<40} 3600   IN  TXT    "{web_dkim_value}"',
     )
 
 
@@ -94,7 +95,7 @@ def check_zonefile(zonefile, verbose=True):
         if not zf_line.strip() or zf_line.startswith(";"):
             continue
         print(f"dns-checking {zf_line!r}") if verbose else log_progress("")
-        zf_domain, zf_typ, zf_value = zf_line.split(maxsplit=2)
+        zf_domain, _ttl, _in, zf_typ, zf_value = zf_line.split(None, 4)
         zf_domain = zf_domain.rstrip(".")
         zf_value = zf_value.strip()
         query_value = query_dns(zf_typ, zf_domain)

cmdeploy/src/cmdeploy/remote/rshell.py

@@ -40,5 +40,5 @@ def dovecot_recalc_quota(user):
     #
     for line in output.split("\n"):
         parts = line.split()
-        if parts[2] == "STORAGE":
+        if len(parts) >= 6 and parts[2] == "STORAGE":
             return dict(value=int(parts[3]), limit=int(parts[4]), percent=int(parts[5]))

cmdeploy/src/cmdeploy/run.py

@@ -14,8 +14,9 @@ def main():
         importlib.resources.files("cmdeploy").joinpath("../../../chatmail.ini"),
     )
     disable_mail = bool(os.environ.get("CHATMAIL_DISABLE_MAIL"))
+    website_only = bool(os.environ.get("CHATMAIL_WEBSITE_ONLY"))
 
-    deploy_chatmail(config_path, disable_mail)
+    deploy_chatmail(config_path, disable_mail, website_only)
 
 
 if pyinfra.is_cli:

cmdeploy/src/cmdeploy/selfsigned/deployer.py

@@ -0,0 +1,54 @@
+import shlex
+
+from pyinfra.operations import apt, server
+
+from cmdeploy.basedeploy import Deployer
+
+
+def openssl_selfsigned_args(domain, cert_path, key_path, days=36500):
+    """Return the openssl argument list for a self-signed certificate.
+
+    The certificate uses an EC P-256 key with SAN entries for *domain*,
+    ``www.<domain>`` and ``mta-sts.<domain>``.
+    """
+    return [
+        "openssl", "req", "-x509",
+        "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:P-256",
+        "-noenc", "-days", str(days),
+        "-keyout", str(key_path),
+        "-out", str(cert_path),
+        "-subj", f"/CN={domain}",
+        # Mark as end-entity cert so it cannot be used as a CA to sign others.
+        "-addext", "basicConstraints=critical,CA:FALSE",
+        "-addext", "extendedKeyUsage=serverAuth,clientAuth",
+        "-addext",
+        f"subjectAltName=DNS:{domain},DNS:www.{domain},DNS:mta-sts.{domain}",
+    ]
+
+
+class SelfSignedTlsDeployer(Deployer):
+    """Generates a self-signed TLS certificate for all chatmail endpoints."""
+
+    def __init__(self, mail_domain):
+        self.mail_domain = mail_domain
+        self.cert_path = "/etc/ssl/certs/mailserver.pem"
+        self.key_path = "/etc/ssl/private/mailserver.key"
+
+    def install(self):
+        apt.packages(
+            name="Install openssl",
+            packages=["openssl"],
+        )
+
+    def configure(self):
+        args = openssl_selfsigned_args(
+            self.mail_domain, self.cert_path, self.key_path,
+        )
+        cmd = shlex.join(args)
+        server.shell(
+            name="Generate self-signed TLS certificate if not present",
+            commands=[f"[ -f {self.cert_path} ] || {cmd}"],
+        )
+
+    def activate(self):
+        pass

cmdeploy/src/cmdeploy/service/chatmail-fsreport.service.f

@@ -5,5 +5,5 @@ After=network.target
 [Service]
 Type=oneshot
 User=vmail
-ExecStart=/usr/local/lib/chatmaild/venv/bin/chatmail-fsreport /usr/local/lib/chatmaild/chatmail.ini 
+ExecStart=/usr/local/lib/chatmaild/venv/bin/chatmail-fsreport /usr/local/lib/chatmaild/chatmail.ini
 

cmdeploy/src/cmdeploy/service/chatmail-metadata.service.f

@@ -4,7 +4,7 @@ Description=Chatmail dict proxy for IMAP METADATA
 [Service]
 ExecStart={execpath} /run/chatmail-metadata/metadata.socket {config_path}
 Restart=always
-RestartSec=30
+RestartSec=5
 User=vmail
 RuntimeDirectory=chatmail-metadata
 UMask=0077

cmdeploy/src/cmdeploy/sshexec.py

@@ -85,16 +85,26 @@ class SSHExec:
 
 
 class LocalExec:
-    def __init__(self, verbose=False, docker=False):
+    FuncError = FuncError
+
+    def __init__(self, verbose=False):
         self.verbose = verbose
-        self.docker = docker
+
+    def __call__(self, call, kwargs=None, log_callback=None):
+        if kwargs is None:
+            kwargs = {}
+        return call(**kwargs)
 
     def logged(self, call, kwargs: dict):
+        title = call.__doc__
+        if not title:
+            title = call.__name__
         where = "locally"
-        if self.docker:
-            if call == remote.rdns.perform_initial_checks:
-                kwargs["pre_command"] = "docker exec chatmail "
-                where = "in docker"
         if self.verbose:
-            print(f"Running {where}: {call.__name__}(**{kwargs})")
-        return call(**kwargs)
+            print_stderr(f"Running {where}: {title}(**{kwargs})")
+            return self(call, kwargs, log_callback=print_stderr)
+        else:
+            print_stderr(title, end="")
+            res = self(call, kwargs, log_callback=remote.rshell.log_progress)
+            print_stderr()
+            return res

cmdeploy/src/cmdeploy/tests/data/zftest.zone

@@ -1,17 +1,18 @@
-; Required DNS entries for chatmail servers
-zftest.testrun.org.                   A     135.181.204.127
-zftest.testrun.org.                   AAAA  2a01:4f9:c012:52f4::1
-zftest.testrun.org.                   MX 10 zftest.testrun.org.
-_mta-sts.zftest.testrun.org.          TXT "v=STSv1; id=202403211706"
-mta-sts.zftest.testrun.org.           CNAME zftest.testrun.org.
-www.zftest.testrun.org.               CNAME zftest.testrun.org.
-opendkim._domainkey.zftest.testrun.org. TXT "v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoYt82CVUyz2ouaqjX2kB+5J80knAyoOU3MGU5aWppmwUwwTvj/oSTSpkc5JMtVTRmKKr8NUDWAL1Yw7dfGqqPHdHfwwjS3BIvDzYx+hzgtz62RnfNgV+/2MAoNpfX7cAFIHdRzEHNtwugc3RDLquqPoupAE3Y2YRw2T5zG5fILh4vwIcJZL5Uq6B92j8wwJqOex" "33n+vm1NKQ9rxo/UsHAmZlJzpooXcG/4igTBxJyJlamVSRR6N7Nul1v//YJb7J6v2o0iPHW6uE0StzKaPPNC2IVosSRFbD9H2oqppltptFSNPlI0E+t0JBWHem6YK7xcugiO3ImMCaaU8g6Jt/wIDAQAB;s=email;t=s"
+; Required DNS entries
+zftest.testrun.org.                      3600   IN  A      135.181.204.127
+zftest.testrun.org.                      3600   IN  AAAA   2a01:4f9:c012:52f4::1
+zftest.testrun.org.                      3600   IN  MX     10 zftest.testrun.org.
+_mta-sts.zftest.testrun.org.             3600   IN  TXT    "v=STSv1; id=202403211706"
+mta-sts.zftest.testrun.org.              3600   IN  CNAME  zftest.testrun.org.
+www.zftest.testrun.org.                  3600   IN  CNAME  zftest.testrun.org.
+opendkim._domainkey.zftest.testrun.org.  3600   IN  TXT    "v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoYt82CVUyz2ouaqjX2kB+5J80knAyoOU3MGU5aWppmwUwwTvj/oSTSpkc5JMtVTRmKKr8NUDWAL1Yw7dfGqqPHdHfwwjS3BIvDzYx+hzgtz62RnfNgV+/2MAoNpfX7cAFIHdRzEHNtwugc3RDLquqPoupAE3Y2YRw2T5zG5fILh4vwIcJZL5Uq6B92j8wwJqOex" "33n+vm1NKQ9rxo/UsHAmZlJzpooXcG/4igTBxJyJlamVSRR6N7Nul1v//YJb7J6v2o0iPHW6uE0StzKaPPNC2IVosSRFbD9H2oqppltptFSNPlI0E+t0JBWHem6YK7xcugiO3ImMCaaU8g6Jt/wIDAQAB;s=email;t=s"
+
 ; Recommended DNS entries
-_submission._tcp.zftest.testrun.org.  SRV 0 1 587 zftest.testrun.org.
-_submissions._tcp.zftest.testrun.org. SRV 0 1 465 zftest.testrun.org.
-_imap._tcp.zftest.testrun.org.        SRV 0 1 143 zftest.testrun.org.
-_imaps._tcp.zftest.testrun.org.       SRV 0 1 993 zftest.testrun.org.
-zftest.testrun.org.                   CAA 0 issue "letsencrypt.org;accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/1371472956"
-zftest.testrun.org.                   TXT "v=spf1 a:zftest.testrun.org ~all"
-_dmarc.zftest.testrun.org.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
-_adsp._domainkey.zftest.testrun.org.  TXT "dkim=discardable"
+zftest.testrun.org.                      3600   IN  TXT    "v=spf1 a ~all"
+_dmarc.zftest.testrun.org.               3600   IN  TXT    "v=DMARC1;p=reject;adkim=s;aspf=s"
+zftest.testrun.org.                      3600   IN  CAA    0 issue "letsencrypt.org;accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/1371472956"
+_adsp._domainkey.zftest.testrun.org.     3600   IN  TXT    "dkim=discardable"
+_submission._tcp.zftest.testrun.org.     3600   IN  SRV    0 1 587 zftest.testrun.org.
+_submissions._tcp.zftest.testrun.org.    3600   IN  SRV    0 1 465 zftest.testrun.org.
+_imap._tcp.zftest.testrun.org.           3600   IN  SRV    0 1 143 zftest.testrun.org.
+_imaps._tcp.zftest.testrun.org.          3600   IN  SRV    0 1 993 zftest.testrun.org.

cmdeploy/src/cmdeploy/tests/online/benchmark.py

@@ -41,9 +41,9 @@ class TestDC:
 
         def dc_ping_pong():
             chat.send_text("ping")
-            msg = ac2._evtracker.wait_next_incoming_message()
-            msg.chat.send_text("pong")
-            ac1._evtracker.wait_next_incoming_message()
+            msg = ac2.wait_for_incoming_msg()
+            msg.get_snapshot().chat.send_text("pong")
+            ac1.wait_for_incoming_msg()
 
         benchmark(dc_ping_pong, 5)
 
@@ -55,6 +55,6 @@ class TestDC:
             for i in range(10):
                 chat.send_text(f"hello {i}")
             for i in range(10):
-                ac2._evtracker.wait_next_incoming_message()
+                ac2.wait_for_incoming_msg()
 
-        benchmark(dc_send_10_receive_10, 5)
+        benchmark(dc_send_10_receive_10, 5, cooldown="auto")

cmdeploy/src/cmdeploy/tests/online/test_0_login.py

@@ -89,7 +89,9 @@ def test_concurrent_logins_same_account(
         assert login_results.get()
 
 
-def test_no_vrfy(chatmail_config):
+def test_no_vrfy(cmfactory, chatmail_config):
+    ac = cmfactory.get_online_account()
+    addr = ac.get_config("addr")
     domain = chatmail_config.mail_domain
 
     s = smtplib.SMTP(domain)
@@ -98,7 +100,7 @@ def test_no_vrfy(chatmail_config):
     s.putcmd("vrfy", f"wrongaddress@{chatmail_config.mail_domain}")
     result = s.getreply()
     print(result)
-    s.putcmd("vrfy", f"echo@{chatmail_config.mail_domain}")
+    s.putcmd("vrfy", addr)
     result2 = s.getreply()
     print(result2)
     assert result[0] == result2[0] == 252

cmdeploy/src/cmdeploy/tests/online/test_0_qr.py

@@ -1,3 +1,4 @@
+import pytest
 import requests
 
 from cmdeploy.genqr import gen_qr_png_data
@@ -8,18 +9,33 @@ def test_gen_qr_png_data(maildomain):
     assert data
 
 
+@pytest.mark.filterwarnings("ignore::urllib3.exceptions.InsecureRequestWarning")
 def test_fastcgi_working(maildomain, chatmail_config):
     url = f"https://{maildomain}/new"
     print(url)
-    res = requests.post(url)
+    verify = chatmail_config.tls_cert_mode == "acme"
+    res = requests.post(url, verify=verify)
     assert maildomain in res.json().get("email")
     assert len(res.json().get("password")) > chatmail_config.password_min_length
 
 
-def test_newemail_configure(maildomain, rpc):
+@pytest.mark.filterwarnings("ignore::urllib3.exceptions.InsecureRequestWarning")
+def test_newemail_configure(maildomain, rpc, chatmail_config):
     """Test configuring accounts by scanning a QR code works."""
     url = f"DCACCOUNT:https://{maildomain}/new"
     for i in range(3):
         account_id = rpc.add_account()
-        rpc.set_config_from_qr(account_id, url)
-        rpc.configure(account_id)
+        if chatmail_config.tls_cert_mode == "self":
+            # deltachat core's rustls rejects self-signed HTTPS certs during
+            # set_config_from_qr, so fetch credentials via requests instead
+            res = requests.post(f"https://{maildomain}/new", verify=False)
+            data = res.json()
+            rpc.add_or_update_transport(account_id, {
+                "addr": data["email"],
+                "password": data["password"],
+                "imapServer": maildomain,
+                "smtpServer": maildomain,
+                "certificateChecks": "acceptInvalidCertificates",
+            })
+        else:
+            rpc.add_transport_from_qr(account_id, url)

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -7,13 +7,13 @@ import time
 import pytest
 
 from cmdeploy import remote
-from cmdeploy.sshexec import SSHExec
+from cmdeploy.cmdeploy import get_sshexec
 
 
 class TestSSHExecutor:
     @pytest.fixture(scope="class")
     def sshexec(self, sshdomain):
-        return SSHExec(sshdomain)
+        return get_sshexec(sshdomain)
 
     def test_ls(self, sshexec):
         out = sshexec(call=remote.rdns.shell, kwargs=dict(command="ls"))
@@ -27,6 +27,7 @@ class TestSSHExecutor:
         assert res["A"] or res["AAAA"]
 
     def test_logged(self, sshexec, maildomain, capsys):
+        sshexec.verbose = False
         sshexec.logged(
             remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain)
         )
@@ -52,6 +53,8 @@ class TestSSHExecutor:
                 remote.rdns.perform_initial_checks,
                 kwargs=dict(mail_domain=None),
             )
+        except AssertionError:
+            pass
         except sshexec.FuncError as e:
             assert "rdns.py" in str(e)
             assert "AssertionError" in str(e)
@@ -68,6 +71,44 @@ class TestSSHExecutor:
         assert (now - since_date).total_seconds() < 60 * 60 * 51
 
 
+def test_dovecot_main_process_matches_installed_binary(sshdomain):
+    sshexec = get_sshexec(sshdomain)
+    main_pid = int(
+        sshexec(
+            call=remote.rshell.shell,
+            kwargs=dict(
+                command="timeout 10 systemctl show -p MainPID --value dovecot.service"
+            ),
+        ).strip()
+    )
+    assert main_pid != 0, "dovecot.service MainPID is 0 -- service not running?"
+
+    exe = sshexec(
+        call=remote.rshell.shell,
+        kwargs=dict(command=f"timeout 10 readlink /proc/{main_pid}/exe"),
+    ).strip()
+    status_text = sshexec(
+        call=remote.rshell.shell,
+        kwargs=dict(
+            command="timeout 10 systemctl show -p StatusText --value dovecot.service"
+        ),
+    ).strip()
+    installed_version = sshexec(
+        call=remote.rshell.shell, kwargs=dict(command="timeout 10 dovecot --version")
+    ).strip()
+
+    assert not exe.endswith("(deleted)"), (
+        f"running dovecot binary was deleted (stale after upgrade): {exe}"
+    )
+    expected_status_text = f"v{installed_version}"
+    assert status_text == expected_status_text or status_text.startswith(
+        f"{expected_status_text} "
+    ), (
+        f"dovecot status version mismatch: "
+        f"StatusText={status_text!r}, installed={installed_version!r}"
+    )
+
+
 def test_timezone_env(remote):
     for line in remote.iter_output("env"):
         print(line)
@@ -83,10 +124,8 @@ def test_remote(remote, imap_or_smtp):
 
 
 def test_use_two_chatmailservers(cmfactory, maildomain2):
-    ac1 = cmfactory.new_online_configuring_account(cache=False)
-    cmfactory.switch_maildomain(maildomain2)
-    ac2 = cmfactory.new_online_configuring_account(cache=False)
-    cmfactory.bring_accounts_online()
+    ac1 = cmfactory.get_online_account()
+    ac2 = cmfactory.get_online_account(domain=maildomain2)
     cmfactory.get_accepted_chat(ac1, ac2)
     domain1 = ac1.get_config("addr").split("@")[1]
     domain2 = ac2.get_config("addr").split("@")[1]
@@ -146,7 +185,7 @@ def test_reject_missing_dkim(cmsetup, maildata, from_addr):
     conn.starttls()
 
     with conn as s:
-        with pytest.raises(smtplib.SMTPDataError, match="No valid DKIM signature"):
+        with pytest.raises(smtplib.SMTPDataError, match="No DKIM signature found"):
             s.sendmail(from_addr=from_addr, to_addrs=recipient.addr, msg=msg)
 
 
@@ -182,19 +221,20 @@ def test_rewrite_subject(cmsetup, maildata):
     assert "Subject: Unencrypted subject" not in rcvd_msg
 
 
-@pytest.mark.slow
 def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
     """Test that the per-account send-mail limit is exceeded."""
     user1, user2 = cmsetup.gen_users(2)
     mail = maildata(
         "encrypted.eml", from_addr=user1.addr, to_addr=user2.addr
     ).as_string()
-    for i in range(chatmail_config.max_user_send_per_minute + 5):
-        print("Sending mail", str(i))
+
+    start = time.time()
+    for i in range(chatmail_config.max_user_send_per_minute * 3):
+        print("Sending mail", str(i + 1), "at", time.time() - start, "s.")
         try:
             user1.smtp.sendmail(user1.addr, [user2.addr], mail)
         except smtplib.SMTPException as e:
-            if i < chatmail_config.max_user_send_per_minute:
+            if i < chatmail_config.max_user_send_burst_size:
                 pytest.fail(f"rate limit was exceeded too early with msg {i}")
             outcome = e.recipients[user2.addr]
             assert outcome[0] == 450
@@ -203,7 +243,6 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
     pytest.fail("Rate limit was not exceeded")
 
 
-@pytest.mark.slow
 def test_expunged(remote, chatmail_config):
     outdated_days = int(chatmail_config.delete_mails_after) + 1
     find_cmds = [
@@ -216,7 +255,7 @@ def test_expunged(remote, chatmail_config):
     ]
     outdated_days = int(chatmail_config.delete_large_after) + 1
     find_cmds.append(
-        "find {chatmail_config.mailboxes_dir} -path '*/cur/*' -mtime +{outdated_days} -size +200k -type f"
+        f"find {chatmail_config.mailboxes_dir} -path '*/cur/*' -mtime +{outdated_days} -size +200k -type f"
     )
     for cmd in find_cmds:
         for line in remote.iter_output(cmd):

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -6,17 +6,19 @@ import imap_tools
 import pytest
 import requests
 
+from cmdeploy.cmdeploy import get_sshexec
 from cmdeploy.remote import rshell
-from cmdeploy.sshexec import SSHExec
 
 
 @pytest.fixture
-def imap_mailbox(cmfactory):
+def imap_mailbox(cmfactory, ssl_context):
     (ac1,) = cmfactory.get_online_accounts(1)
     user = ac1.get_config("addr")
     password = ac1.get_config("mail_pw")
-    mailbox = imap_tools.MailBox(user.split("@")[1])
+    host = user.split("@")[1]
+    mailbox = imap_tools.MailBox(host, ssl_context=ssl_context)
     mailbox.login(user, password)
+    mailbox.dc_ac = ac1
     return mailbox
 
 
@@ -25,6 +27,7 @@ class TestMetadataTokens:
 
     def test_set_get_metadata(self, imap_mailbox):
         "set and get metadata token for an account"
+        time.sleep(5)  # make sure Metadata service had a chance to restart
         client = imap_mailbox.client
         client.send(b'a01 SETMETADATA INBOX (/private/devicetoken "1111" )\n')
         res = client.readline()
@@ -60,8 +63,8 @@ class TestEndToEndDeltaChat:
         chat.send_text("message0")
 
         lp.sec("wait for ac2 to receive message")
-        msg2 = ac2._evtracker.wait_next_incoming_message()
-        assert msg2.text == "message0"
+        msg2 = ac2.wait_for_incoming_msg()
+        assert msg2.get_snapshot().text == "message0"
 
     def test_exceed_quota(
         self, cmfactory, lp, tmpdir, remote, chatmail_config, sshdomain
@@ -89,66 +92,83 @@ class TestEndToEndDeltaChat:
         lp.sec(f"filling remote inbox for {user}")
         fn = f"7743102289.M843172P2484002.c20,S={quota},W=2398:2,"
         path = chatmail_config.mailboxes_dir.joinpath(user, "cur", fn)
-        sshexec = SSHExec(sshdomain)
+        sshexec = get_sshexec(sshdomain)
         sshexec(call=rshell.write_numbytes, kwargs=dict(path=str(path), num=120))
         res = sshexec(call=rshell.dovecot_recalc_quota, kwargs=dict(user=user))
         assert res["percent"] >= 100
 
         lp.sec("ac2: check quota is triggered")
 
-        starting = True
-        for line in remote.iter_output("journalctl -n0 -f -u dovecot"):
-            if starting:
-                chat.send_text("hello")
-                starting = False
+        def send_hello():
+            chat.send_text("hello")
+
+        for line in remote.iter_output(
+            "journalctl -n1 -f -u dovecot", ready=send_hello
+        ):
             if user not in line:
-                # print(line)
                 continue
             if "quota exceeded" in line:
                 return
 
     def test_securejoin(self, cmfactory, lp, maildomain2):
-        ac1 = cmfactory.new_online_configuring_account(cache=False)
-        cmfactory.switch_maildomain(maildomain2)
-        ac2 = cmfactory.new_online_configuring_account(cache=False)
-        cmfactory.bring_accounts_online()
+        ac1 = cmfactory.get_online_account()
+        ac2 = cmfactory.get_online_account(domain=maildomain2)
 
         lp.sec("ac1: create QR code and let ac2 scan it, starting the securejoin")
-        qr = ac1.get_setup_contact_qr()
+        qr = ac1.get_qr_code()
 
         lp.sec("ac2: start QR-code based setup contact protocol")
-        ch = ac2.qr_setup_contact(qr)
+        ch = ac2.secure_join(qr)
         assert ch.id >= 10
-        ac1._evtracker.wait_securejoin_inviter_progress(1000)
+        ac1.wait_for_securejoin_inviter_success()
+
+    def test_dkim_header_stripped(self, cmfactory, maildomain2, lp, imap_mailbox):
+        """Test that if a DC address receives a message, it has no
+        DKIM-Signature and Authentication-Results headers."""
+        ac1 = cmfactory.get_online_account()
+        ac2 = cmfactory.get_online_account(domain=maildomain2)
+        chat = cmfactory.get_accepted_chat(ac1, imap_mailbox.dc_ac)
+        chat.send_text("message0")
+        chat2 = cmfactory.get_accepted_chat(ac2, imap_mailbox.dc_ac)
+        chat2.send_text("message1")
+
+        lp.sec("receive message with ac1...")
+        received = 0
+        while received < 2:
+            msgs = imap_mailbox.fetch()
+            for msg in msgs:
+                lp.sec(f"ac1 received msg from {msg.from_}")
+                received += 1
+                assert "authentication-results" not in msg.headers
+                assert "dkim-signature" not in msg.headers
 
     def test_read_receipts_between_instances(self, cmfactory, lp, maildomain2):
-        ac1 = cmfactory.new_online_configuring_account(cache=False)
-        cmfactory.switch_maildomain(maildomain2)
-        ac2 = cmfactory.new_online_configuring_account(cache=False)
-        cmfactory.bring_accounts_online()
+        ac1 = cmfactory.get_online_account()
+        ac2 = cmfactory.get_online_account(domain=maildomain2)
 
         lp.sec("setup encrypted comms between ac1 and ac2 on different instances")
-        qr = ac1.get_setup_contact_qr()
-        ch = ac2.qr_setup_contact(qr)
+        qr = ac1.get_qr_code()
+        ch = ac2.secure_join(qr)
         assert ch.id >= 10
-        ac1._evtracker.wait_securejoin_inviter_progress(1000)
+        ac1.wait_for_securejoin_inviter_success()
 
         lp.sec("ac1 sends a message and ac2 marks it as seen")
         chat = ac1.create_chat(ac2)
         msg = chat.send_text("hi")
-        m = ac2._evtracker.wait_next_incoming_message()
+        m = ac2.wait_for_incoming_msg()
         m.mark_seen()
         # we can only indirectly wait for mark-seen to cause an smtp-error
         lp.sec("try to wait for markseen to complete and check error states")
         deadline = time.time() + 3.1
         while time.time() < deadline:
-            msgs = m.chat.get_messages()
+            m_snap = m.get_snapshot()
+            msgs = m_snap.chat.get_messages()
             for msg in msgs:
-                assert "error" not in m.get_message_info()
+                assert "error" not in m.get_info()
             time.sleep(1)
 
 
-def test_hide_senders_ip_address(cmfactory):
+def test_hide_senders_ip_address(cmfactory, ssl_context):
     public_ip = requests.get("http://icanhazip.com").content.decode().strip()
     assert ipaddress.ip_address(public_ip)
 
@@ -156,7 +176,12 @@ def test_hide_senders_ip_address(cmfactory):
     chat = cmfactory.get_accepted_chat(user1, user2)
 
     chat.send_text("testing submission header cleanup")
-    user2._evtracker.wait_next_incoming_message()
-    user2.direct_imap.select_folder("Inbox")
-    msg = user2.direct_imap.get_all_messages()[0]
-    assert public_ip not in msg.obj.as_string()
+    user2.wait_for_incoming_msg()
+    addr = user2.get_config("addr")
+    host = addr.split("@")[1]
+    pw = user2.get_config("mail_pw")
+    mailbox = imap_tools.MailBox(host, ssl_context=ssl_context)
+    mailbox.login(addr, pw)
+    msgs = list(mailbox.fetch(mark_seen=False))
+    assert msgs, "expected at least one message"
+    assert public_ip not in msgs[0].obj.as_string()

cmdeploy/src/cmdeploy/tests/online/test_3_status.py

@@ -5,7 +5,11 @@ from cmdeploy.cmdeploy import main
 
 def test_status_cmd(chatmail_config, capsys, request):
     os.chdir(request.config.invocation_params.dir)
-    assert main(["status"]) == 0
+    command = ["status"]
+    if os.getenv("CHATMAIL_SSH"):
+        command.append("--ssh-host")
+        command.append(os.getenv("CHATMAIL_SSH"))
+    assert main(command) == 0
     status_out = capsys.readouterr()
     print(status_out.out)
 

cmdeploy/src/cmdeploy/tests/plugin.py

@@ -1,9 +1,10 @@
 import imaplib
-import io
+import ipaddress
 import itertools
 import os
 import random
 import smtplib
+import ssl
 import subprocess
 import time
 from pathlib import Path
@@ -14,10 +15,12 @@ from chatmaild.config import read_config
 conftestdir = Path(__file__).parent
 
 
-def pytest_addoption(parser):
-    parser.addoption(
-        "--slow", action="store_true", default=False, help="also run slow tests"
-    )
+def _is_ip(domain):
+    try:
+        ipaddress.ip_address(domain)
+        return True
+    except ValueError:
+        return False
 
 
 def pytest_configure(config):
@@ -27,24 +30,29 @@ def pytest_configure(config):
     )
 
 
-def pytest_runtest_setup(item):
-    markers = list(item.iter_markers(name="slow"))
-    if markers:
-        if not item.config.getoption("--slow"):
-            pytest.skip("skipping slow test, use --slow to run")
+def _get_chatmail_config():
+    inipath = os.environ.get("CHATMAIL_INI")
+    if inipath:
+        path = Path(inipath).resolve()
+        return read_config(path), path
+
+    current = Path().resolve()
+    while 1:
+        path = current.joinpath("chatmail.ini").resolve()
+        if path.exists():
+            return read_config(path), path
+        if current == current.parent:
+            break
+        current = current.parent
+    return None, None
 
 
 @pytest.fixture(scope="session")
 def chatmail_config(pytestconfig):
-    current = basedir = Path().resolve()
-    while 1:
-        path = current.joinpath("chatmail.ini").resolve()
-        if path.exists():
-            return read_config(path)
-        if current == current.parent:
-            break
-        current = current.parent
-
+    config, path = _get_chatmail_config()
+    if config:
+        return config
+    basedir = Path().resolve()
     pytest.skip(f"no chatmail.ini file found in {basedir} or parent dirs")
 
 
@@ -72,10 +80,17 @@ def sshdomain2(maildomain2):
 
 
 def pytest_report_header():
-    domain = os.environ.get("CHATMAIL_DOMAIN")
-    if domain:
-        text = f"chatmail test instance: {domain}"
-        return ["-" * len(text), text, "-" * len(text)]
+    config, path = _get_chatmail_config()
+    domain2 = os.environ.get("CHATMAIL_DOMAIN2", "NOT SET")
+    domain = config.mail_domain if config else "NOT SET"
+    path = path if path else "NOT SET"
+
+    lines = [
+        f"chatmail.ini {domain} location: {path}",
+        f"chatmail2: {domain2}",
+    ]
+    sep = "-" * max(map(len, lines))
+    return [sep, *lines, sep]
 
 
 @pytest.fixture
@@ -90,15 +105,22 @@ def cm_data(request):
 
 
 @pytest.fixture
-def benchmark(request):
-    def bench(func, num, name=None, reportfunc=None):
+def benchmark(request, chatmail_config):
+    def bench(func, num, name=None, reportfunc=None, cooldown=0.0):
         if name is None:
             name = func.__name__
+        if cooldown == "auto":
+            per_minute = max(chatmail_config.max_user_send_per_minute, 1)
+            cooldown = chatmail_config.max_user_send_burst_size * 60 / per_minute
+
         durations = []
         for i in range(num):
             now = time.time()
             func()
             durations.append(time.time() - now)
+            if cooldown > 0 and i + 1 < num:
+                # Keep post-run cooldown out of measured benchmark duration.
+                time.sleep(cooldown)
         durations.sort()
         request.config._benchresults[name] = (reportfunc, durations)
 
@@ -144,15 +166,25 @@ def pytest_terminal_summary(terminalreporter):
             tr.write_line(line)
 
 
-@pytest.fixture
-def imap(maildomain):
-    return ImapConn(maildomain)
+@pytest.fixture(scope="session")
+def ssl_context(chatmail_config):
+    if chatmail_config.tls_cert_mode == "self":
+        ctx = ssl.create_default_context()
+        ctx.check_hostname = False
+        ctx.verify_mode = ssl.CERT_NONE
+        return ctx
+    return None
 
 
 @pytest.fixture
-def make_imap_connection(maildomain):
+def imap(maildomain, ssl_context):
+    return ImapConn(maildomain, ssl_context=ssl_context)
+
+
+@pytest.fixture
+def make_imap_connection(maildomain, ssl_context):
     def make_imap_connection():
-        conn = ImapConn(maildomain)
+        conn = ImapConn(maildomain, ssl_context=ssl_context)
         conn.connect()
         return conn
 
@@ -164,12 +196,13 @@ class ImapConn:
     logcmd = "journalctl -f -u dovecot"
     name = "dovecot"
 
-    def __init__(self, host):
+    def __init__(self, host, ssl_context=None):
         self.host = host
+        self.ssl_context = ssl_context
 
     def connect(self):
         print(f"imap-connect {self.host}")
-        self.conn = imaplib.IMAP4_SSL(self.host)
+        self.conn = imaplib.IMAP4_SSL(self.host, ssl_context=self.ssl_context)
 
     def login(self, user, password):
         print(f"imap-login {user!r} {password!r}")
@@ -195,14 +228,14 @@ class ImapConn:
 
 
 @pytest.fixture
-def smtp(maildomain):
-    return SmtpConn(maildomain)
+def smtp(maildomain, ssl_context):
+    return SmtpConn(maildomain, ssl_context=ssl_context)
 
 
 @pytest.fixture
-def make_smtp_connection(maildomain):
+def make_smtp_connection(maildomain, ssl_context):
     def make_smtp_connection():
-        conn = SmtpConn(maildomain)
+        conn = SmtpConn(maildomain, ssl_context=ssl_context)
         conn.connect()
         return conn
 
@@ -214,12 +247,14 @@ class SmtpConn:
     logcmd = "journalctl -f -t postfix/smtpd -t postfix/smtp -t postfix/lmtp"
     name = "postfix"
 
-    def __init__(self, host):
+    def __init__(self, host, ssl_context=None):
         self.host = host
+        self.ssl_context = ssl_context
 
     def connect(self):
         print(f"smtp-connect {self.host}")
-        self.conn = smtplib.SMTP_SSL(self.host)
+        context = self.ssl_context or ssl.create_default_context()
+        self.conn = smtplib.SMTP_SSL(self.host, context=context)
 
     def login(self, user, password):
         print(f"smtp-login {user!r} {password!r}")
@@ -243,6 +278,7 @@ def gencreds(chatmail_config):
 
     def gen(domain=None):
         domain = domain if domain else chatmail_config.mail_domain
+        addr_domain = f"[{domain}]" if _is_ip(domain) else domain
         while 1:
             num = next(count)
             alphanumeric = "abcdefghijklmnopqrstuvwxyz1234567890"
@@ -256,98 +292,161 @@ def gencreds(chatmail_config):
             password = "".join(
                 random.choices(alphanumeric, k=chatmail_config.password_min_length)
             )
-            yield f"{user}@{domain}", f"{password}"
+            yield f"{user}@{addr_domain}", f"{password}"
 
     return lambda domain=None: next(gen(domain))
 
 
 #
-# Delta Chat testplugin re-use
+# Delta Chat RPC-based test support
 # use the cmfactory fixture to get chatmail instance accounts
 #
 
+from deltachat_rpc_client import DeltaChat, Rpc
 
-class ChatmailTestProcess:
-    """Provider for chatmail instance accounts as used by deltachat.testplugin.acfactory"""
 
-    def __init__(self, pytestconfig, maildomain, gencreds):
-        self.pytestconfig = pytestconfig
-        self.maildomain = maildomain
-        assert "." in self.maildomain, maildomain
+class ChatmailACFactory:
+    """RPC-based account factory for chatmail testing."""
+
+    def __init__(self, rpc, maildomain, gencreds, chatmail_config):
+        self.dc = DeltaChat(rpc)
+        self.rpc = rpc
+        self._maildomain = maildomain
         self.gencreds = gencreds
-        self._addr2files = {}
+        self.chatmail_config = chatmail_config
 
-    def get_liveconfig_producer(self):
-        while 1:
-            user, password = self.gencreds(self.maildomain)
-            config = {
-                "addr": user,
-                "mail_pw": password,
-            }
-            # speed up account configuration
-            config["mail_server"] = self.maildomain
-            config["send_server"] = self.maildomain
-            yield config
+    def _make_transport(self, domain):
+        """Build a transport config dict for the given domain."""
+        addr, password = self.gencreds(domain)
+        transport = {
+            "addr": addr,
+            "password": password,
+            # Setting server explicitly skips requesting autoconfig XML,
+            # see https://datatracker.ietf.org/doc/draft-ietf-mailmaint-autoconfig/
+            "imapServer": domain,
+            "smtpServer": domain,
+        }
+        if self.chatmail_config.tls_cert_mode == "self":
+            transport["certificateChecks"] = "acceptInvalidCertificates"
+        return transport
 
-    def cache_maybe_retrieve_configured_db_files(self, cache_addr, db_target_path):
-        pass
+    def get_online_account(self, domain=None):
+        """Create, configure and bring online a single account."""
+        return self.get_online_accounts(1, domain)[0]
 
-    def cache_maybe_store_configured_db_files(self, acc):
-        pass
+    def get_online_accounts(self, num, domain=None):
+        """Create multiple online accounts in parallel."""
+        domain = domain or self._maildomain
+        futures = []
+        accounts = []
+        for _ in range(num):
+            account = self.dc.add_account()
+            addr, password = self.gencreds(domain)
+            if _is_ip(domain):
+                # Use DCLOGIN scheme with explicit server hosts,
+                # matching how madmail presents its addresses to users.
+                qr = (
+                    f"dclogin:{addr}"
+                    f"?p={password}&v=1"
+                    f"&ih={domain}&ip=993"
+                    f"&sh={domain}&sp=465"
+                    f"&ic=3&ss=default"
+                )
+                future = account.add_transport_from_qr.future(qr)
+            else:
+                future = account.add_or_update_transport.future(
+                    self._make_transport(domain)
+                )
+            futures.append(future)
+
+            # ensure messages stay in INBOX so that they can be
+            # concurrently fetched via extra IMAP connections during tests
+            account.set_config("delete_server_after", "10")
+            accounts.append(account)
+
+        for future in futures:
+            future()
+
+        for account in accounts:
+            account.bring_online()
+        return accounts
+
+    def get_accepted_chat(self, ac1, ac2):
+        """Create a 1:1 chat between ac1 and ac2 accepted on both sides."""
+        ac2.create_chat(ac1)
+        return ac1.create_chat(ac2)
+
+
+@pytest.fixture(scope="session")
+def rpc(tmp_path_factory):
+    """Start a deltachat-rpc-server process for the test session."""
+
+    # NB: accounts_dir must NOT already exist as directory --
+    # core-rust only creates accounts.toml if the dir doesn't exist yet.
+    accounts_dir = str(tmp_path_factory.mktemp("dc") / "accounts")
+    rpc = Rpc(accounts_dir=accounts_dir)
+    rpc.start()
+    yield rpc
+    rpc.close()
 
 
 @pytest.fixture
-def cmfactory(request, gencreds, tmpdir, maildomain):
-    # cloned from deltachat.testplugin.amfactory
-    pytest.importorskip("deltachat")
-    from deltachat.testplugin import ACFactory
-
-    testproc = ChatmailTestProcess(request.config, maildomain, gencreds)
-
-    class Data:
-        def read_path(self, path):
-            return
-
-    am = ACFactory(request=request, tmpdir=tmpdir, testprocess=testproc, data=Data())
-
-    # nb. a bit hacky
-    # would probably be better if deltachat's test machinery grows native support
-    def switch_maildomain(maildomain2):
-        am.testprocess.maildomain = maildomain2
-
-    am.switch_maildomain = switch_maildomain
-
-    yield am
-    if hasattr(request.node, "rep_call") and request.node.rep_call.failed:
-        if testproc.pytestconfig.getoption("--extra-info"):
-            logfile = io.StringIO()
-            am.dump_imap_summary(logfile=logfile)
-            print(logfile.getvalue())
-            # request.node.add_report_section("call", "imap-server-state", s)
+def cmfactory(rpc, gencreds, maildomain, chatmail_config):
+    """Return a ChatmailACFactory for creating online Delta Chat accounts."""
+    return ChatmailACFactory(
+        rpc=rpc,
+        maildomain=maildomain,
+        gencreds=gencreds,
+        chatmail_config=chatmail_config,
+    )
 
 
 @pytest.fixture
 def remote(sshdomain):
-    return Remote(sshdomain)
+    r = Remote(sshdomain)
+    yield r
+    r.close()
 
 
 class Remote:
     def __init__(self, sshdomain):
         self.sshdomain = sshdomain
+        self._procs = []
 
-    def iter_output(self, logcmd=""):
+    def iter_output(self, logcmd="", ready=None):
         getjournal = "journalctl -f" if not logcmd else logcmd
-        self.popen = subprocess.Popen(
-            ["ssh", f"root@{self.sshdomain}", getjournal],
+        print(self.sshdomain)
+        match self.sshdomain:
+            case "@local": command = []
+            case "localhost": command = []
+            case _: command = ["ssh", f"root@{self.sshdomain}"]
+        [command.append(arg) for arg in getjournal.split()]
+        popen = subprocess.Popen(
+            command,
+            stdin=subprocess.DEVNULL,
             stdout=subprocess.PIPE,
+            stderr=subprocess.DEVNULL,
         )
-        while 1:
-            line = self.popen.stdout.readline()
-            res = line.decode().strip().lower()
-            if res:
+        self._procs.append(popen)
+        try:
+            while 1:
+                line = popen.stdout.readline()
+                res = line.decode().strip().lower()
+                if not res:
+                    break
+                if ready is not None:
+                    ready()
+                    ready = None
                 yield res
-            else:
-                break
+        finally:
+            popen.terminate()
+            popen.wait()
+
+    def close(self):
+        while self._procs:
+            proc = self._procs.pop()
+            proc.kill()
+            proc.wait()
 
 
 @pytest.fixture
@@ -363,38 +462,40 @@ def lp(request):
 
 
 @pytest.fixture
-def cmsetup(maildomain, gencreds):
-    return CMSetup(maildomain, gencreds)
+def cmsetup(maildomain, gencreds, ssl_context):
+    return CMSetup(maildomain, gencreds, ssl_context)
 
 
 class CMSetup:
-    def __init__(self, maildomain, gencreds):
+    def __init__(self, maildomain, gencreds, ssl_context):
         self.maildomain = maildomain
         self.gencreds = gencreds
+        self.ssl_context = ssl_context
 
     def gen_users(self, num):
         print(f"Creating {num} online users")
         users = []
         for i in range(num):
             addr, password = self.gencreds()
-            user = CMUser(self.maildomain, addr, password)
+            user = CMUser(self.maildomain, addr, password, self.ssl_context)
             assert user.smtp
             users.append(user)
         return users
 
 
 class CMUser:
-    def __init__(self, maildomain, addr, password):
+    def __init__(self, maildomain, addr, password, ssl_context=None):
         self.maildomain = maildomain
         self.addr = addr
         self.password = password
+        self.ssl_context = ssl_context
         self._smtp = None
         self._imap = None
 
     @property
     def smtp(self):
         if not self._smtp:
-            handle = SmtpConn(self.maildomain)
+            handle = SmtpConn(self.maildomain, ssl_context=self.ssl_context)
             handle.connect()
             handle.login(self.addr, self.password)
             self._smtp = handle
@@ -403,7 +504,7 @@ class CMUser:
     @property
     def imap(self):
         if not self._imap:
-            imap = ImapConn(self.maildomain)
+            imap = ImapConn(self.maildomain, ssl_context=self.ssl_context)
             imap.connect()
             imap.login(self.addr, self.password)
             self._imap = imap

cmdeploy/src/cmdeploy/tests/test_cmdeploy.py

@@ -23,15 +23,19 @@ class TestCmdline:
         run = parser.parse_args(["run"])
         assert init and run
 
-    def test_init_not_overwrite(self, capsys):
-        assert main(["init", "chat.example.org"]) == 0
+    def test_init_not_overwrite(self, capsys, tmp_path, monkeypatch):
+        monkeypatch.delenv("CHATMAIL_INI", raising=False)
+        inipath = tmp_path / "chatmail.ini"
+        args = ["init", "--config", str(inipath), "chat.example.org"]
+        assert main(args) == 0
         capsys.readouterr()
 
-        assert main(["init", "chat.example.org"]) == 1
+        assert main(args) == 1
         out, err = capsys.readouterr()
         assert "path exists" in out.lower()
 
-        assert main(["init", "chat.example.org", "--force"]) == 0
+        args.insert(1, "--force")
+        assert main(args) == 0
         out, err = capsys.readouterr()
         assert "deleting config file" in out.lower()
 

cmdeploy/src/cmdeploy/tests/test_dns.py

@@ -3,7 +3,7 @@ from copy import deepcopy
 import pytest
 
 from cmdeploy import remote
-from cmdeploy.dns import check_full_zone, check_initial_remote_data
+from cmdeploy.dns import check_full_zone, check_initial_remote_data, parse_zone_records
 
 
 @pytest.fixture
@@ -60,6 +60,29 @@ def mockdns(request, mockdns_base, mockdns_expected):
     return mockdns_base
 
 
+class TestGetDkimEntry:
+    def test_dkim_entry_returns_tuple_on_success(self, mockdns):
+        entry, web_entry = remote.rdns.get_dkim_entry(
+            "some.domain", "", dkim_selector="opendkim"
+        )
+        # May return None,None if openssl not available, but should never crash
+        if entry is not None:
+            assert "opendkim._domainkey.some.domain" in entry
+            assert "opendkim._domainkey.some.domain" in web_entry
+
+    def test_dkim_entry_returns_none_tuple_on_error(self, monkeypatch):
+        """CalledProcessError must return (None, None), not bare None."""
+        from subprocess import CalledProcessError
+
+        def failing_shell(command, fail_ok=False, print=print):
+            raise CalledProcessError(1, command)
+
+        monkeypatch.setattr(remote.rdns, "shell", failing_shell)
+        result = remote.rdns.get_dkim_entry("some.domain", "", dkim_selector="opendkim")
+        assert result == (None, None)
+        assert result[0] is None and result[1] is None
+
+
 class TestPerformInitialChecks:
     def test_perform_initial_checks_ok1(self, mockdns, mockdns_expected):
         remote_data = remote.rdns.perform_initial_checks("some.domain")
@@ -91,19 +114,60 @@ class TestPerformInitialChecks:
         assert not res
         assert len(l) == 2
 
+    def test_perform_initial_checks_no_mta_sts_self_signed(self, mockdns):
+        del mockdns["CNAME"]["mta-sts.some.domain"]
+        remote_data = remote.rdns.perform_initial_checks("some.domain")
+        assert not remote_data["MTA_STS"]
+
+        l = []
+        res = check_initial_remote_data(remote_data, strict_tls=False, print=l.append)
+        assert res
+        assert not l
+
+
+def test_parse_zone_records():
+    text = """
+    ; This is a comment
+    some.domain. 3600 IN A 1.1.1.1
+
+    ; Another comment
+    www.some.domain. 3600 IN CNAME some.domain.
+
+    ; Multi-word rdata
+    some.domain. 3600 IN MX 10 mail.some.domain.
+
+    ; DKIM record (single line, multi-word TXT rdata)
+    dkim._domainkey.some.domain. 3600 IN TXT "v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG" "9w0BAQEFAAOCAQ8AMIIBCgKCAQEA"
+
+    ; Another TXT record
+    _dmarc.some.domain. 3600 IN TXT "v=DMARC1;p=reject"
+    """
+    records = list(parse_zone_records(text))
+    assert records == [
+        ("some.domain", "3600", "A", "1.1.1.1"),
+        ("www.some.domain", "3600", "CNAME", "some.domain."),
+        ("some.domain", "3600", "MX", "10 mail.some.domain."),
+        (
+            "dkim._domainkey.some.domain",
+            "3600",
+            "TXT",
+            '"v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG" "9w0BAQEFAAOCAQ8AMIIBCgKCAQEA"',
+        ),
+        ("_dmarc.some.domain", "3600", "TXT", '"v=DMARC1;p=reject"'),
+    ]
+
+
+def test_parse_zone_records_invalid_line():
+    text = "invalid line"
+    with pytest.raises(ValueError, match="Bad zone record line"):
+        list(parse_zone_records(text))
+
 
 def parse_zonefile_into_dict(zonefile, mockdns_base, only_required=False):
-    for zf_line in zonefile.split("\n"):
-        if zf_line.startswith("#"):
-            if "Recommended" in zf_line and only_required:
-                return
-            continue
-        if not zf_line.strip():
-            continue
-        zf_domain, zf_typ, zf_value = zf_line.split(maxsplit=2)
-        zf_domain = zf_domain.rstrip(".")
-        zf_value = zf_value.strip()
-        mockdns_base.setdefault(zf_typ, {})[zf_domain] = zf_value
+    if only_required:
+        zonefile = zonefile.split("; Recommended")[0]
+    for name, ttl, rtype, rdata in parse_zone_records(zonefile):
+        mockdns_base.setdefault(rtype, {})[name] = rdata
 
 
 class MockSSHExec:

cmdeploy/src/cmdeploy/tests/test_dovecot_deployer.py

@@ -0,0 +1,238 @@
+from contextlib import nullcontext
+from types import SimpleNamespace
+
+import pytest
+from pyinfra.facts.deb import DebPackages
+
+from cmdeploy.dovecot import deployer as dovecot_deployer
+
+
+def make_host(*fact_pairs):
+    """Build a mock host; get_fact(cls) dispatches to the provided facts mapping.
+
+    Args:
+        *fact_pairs: tuples of (fact_class, fact_value) to register
+
+    Returns:
+        SimpleNamespace with get_fact that raises a clear error if an
+        unexpected fact type is requested.
+    """
+    facts = dict(fact_pairs)
+
+    def get_fact(cls):
+        if cls not in facts:
+            registered = ", ".join(c.__name__ for c in facts)
+            raise LookupError(
+                f"unexpected get_fact({cls.__name__}); "
+                f"only registered: {registered}"
+            )
+        return facts[cls]
+
+    return SimpleNamespace(get_fact=get_fact)
+
+
+@pytest.fixture
+def deployer():
+    return dovecot_deployer.DovecotDeployer(
+        SimpleNamespace(mail_domain="chat.example.org"),
+        disable_mail=False,
+    )
+
+
+@pytest.fixture
+def patch_blocked(monkeypatch):
+    monkeypatch.setattr(dovecot_deployer, "blocked_service_startup", nullcontext)
+
+
+@pytest.fixture
+def mock_files_put(monkeypatch):
+    monkeypatch.setattr(
+        dovecot_deployer.files,
+        "put",
+        lambda **kwargs: SimpleNamespace(changed=False),
+    )
+
+
+@pytest.fixture
+def track_shell(monkeypatch):
+    calls = []
+    monkeypatch.setattr(
+        dovecot_deployer.server,
+        "shell",
+        lambda **kwargs: calls.append(kwargs) or SimpleNamespace(changed=False),
+    )
+    return calls
+
+
+def test_download_dovecot_package_skips_epoch_matched_install(monkeypatch):
+    epoch_version = dovecot_deployer.DOVECOT_PACKAGE_VERSION
+    downloads = []
+    monkeypatch.setattr(
+        dovecot_deployer,
+        "host",
+        make_host((DebPackages, {"dovecot-core": [epoch_version]})),
+    )
+    monkeypatch.setattr(
+        dovecot_deployer,
+        "_pick_url",
+        lambda primary, fallback: primary,
+    )
+    monkeypatch.setattr(
+        dovecot_deployer.files,
+        "download",
+        lambda **kwargs: downloads.append(kwargs),
+    )
+
+    deb, changed = dovecot_deployer._download_dovecot_package("core", "amd64")
+
+    assert deb is None, f"expected no deb path when version matches, got {deb!r}"
+    assert changed is False, "should not flag changed when version already installed"
+    assert downloads == [], "should not download when version already installed"
+
+
+def test_download_dovecot_package_uses_archive_version_for_url_and_filename(
+    monkeypatch,
+):
+    downloads = []
+    monkeypatch.setattr(
+        dovecot_deployer,
+        "host",
+        make_host((DebPackages, {})),
+    )
+    monkeypatch.setattr(
+        dovecot_deployer,
+        "_pick_url",
+        lambda primary, fallback: primary,
+    )
+    monkeypatch.setattr(
+        dovecot_deployer.files,
+        "download",
+        lambda **kwargs: downloads.append(kwargs),
+    )
+
+    deb, changed = dovecot_deployer._download_dovecot_package("core", "amd64")
+
+    archive_version = dovecot_deployer.DOVECOT_ARCHIVE_VERSION.replace("+", "%2B")
+    expected_deb = f"/root/dovecot-core_{archive_version}_amd64.deb"
+
+    # Verify the returned path uses archive version, not package version (with epoch)
+    assert changed is True, "should flag changed when package not yet installed"
+    assert deb == expected_deb, f"deb path mismatch: {deb!r} != {expected_deb!r}"
+    assert dovecot_deployer.DOVECOT_PACKAGE_VERSION not in deb, (
+        f"deb path should use archive version (no epoch), got {deb!r}"
+    )
+    assert len(downloads) == 1, "files.download should be called exactly once"
+
+
+def test_install_skips_dpkg_path_when_epoch_matched_packages_present(
+    deployer, patch_blocked, mock_files_put, track_shell, monkeypatch
+):
+    monkeypatch.setattr(
+        dovecot_deployer,
+        "host",
+        make_host(
+            (
+                dovecot_deployer.DebPackages,
+                {
+                    "dovecot-core": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
+                    "dovecot-imapd": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
+                    "dovecot-lmtpd": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
+                },
+            ),
+            (dovecot_deployer.Arch, "x86_64"),
+        ),
+    )
+    downloads = []
+    monkeypatch.setattr(
+        dovecot_deployer.files,
+        "download",
+        lambda **kwargs: downloads.append(kwargs),
+    )
+
+    deployer.install()
+
+    assert downloads == [], "should not download when all packages epoch-matched"
+    assert track_shell == [], "should not run dpkg when all packages epoch-matched"
+    assert deployer.need_restart is False, (
+        "need_restart should be False when nothing changed"
+    )
+
+
+def test_install_unsupported_arch_falls_back_to_apt(
+    deployer, patch_blocked, mock_files_put, track_shell, monkeypatch
+):
+    # For unsupported architectures, all fact lookups return the arch string.
+    monkeypatch.setattr(
+        dovecot_deployer,
+        "host",
+        SimpleNamespace(get_fact=lambda cls: "riscv64"),
+    )
+    apt_calls = []
+
+    # Mirrors apt.packages() return value: OperationMeta with .changed property.
+    # Only lmtpd triggers a change to verify |= accumulation of changed flags.
+    def fake_apt(**kwargs):
+        apt_calls.append(kwargs)
+        changed = "lmtpd" in kwargs["packages"][0]
+        return SimpleNamespace(changed=changed)
+
+    monkeypatch.setattr(dovecot_deployer.apt, "packages", fake_apt)
+
+    deployer.install()
+
+    actual_pkgs = [c["packages"] for c in apt_calls]
+    assert actual_pkgs == [["dovecot-core"], ["dovecot-imapd"], ["dovecot-lmtpd"]], (
+        f"expected apt install of core/imapd/lmtpd, got {actual_pkgs}"
+    )
+    assert track_shell == [], "should not run dpkg for unsupported arch"
+    assert deployer.need_restart is True, (
+        "need_restart should be True when apt installed a package"
+    )
+
+
+def test_install_runs_dpkg_when_packages_need_download(
+    deployer, patch_blocked, mock_files_put, track_shell, monkeypatch
+):
+    monkeypatch.setattr(
+        dovecot_deployer,
+        "host",
+        make_host(
+            (dovecot_deployer.DebPackages, {}),
+            (dovecot_deployer.Arch, "x86_64"),
+        ),
+    )
+    monkeypatch.setattr(
+        dovecot_deployer,
+        "_pick_url",
+        lambda primary, fallback: primary,
+    )
+    monkeypatch.setattr(
+        dovecot_deployer.files,
+        "download",
+        lambda **kwargs: SimpleNamespace(changed=True),
+    )
+
+    deployer.install()
+
+    assert len(track_shell) == 1, (
+        f"expected one server.shell() call for dpkg install, got {len(track_shell)}"
+    )
+    cmds = track_shell[0]["commands"]
+    assert len(cmds) == 3, f"expected 3 dpkg/apt commands, got: {cmds}"
+    assert cmds[0].startswith("dpkg --force-confdef --force-confold -i ")
+    assert "apt-get -y --fix-broken install" in cmds[1]
+    assert cmds[2].startswith("dpkg --force-confdef --force-confold -i ")
+    assert deployer.need_restart is True, (
+        "need_restart should be True after dpkg install"
+    )
+
+
+def test_pick_url_falls_back_on_primary_error(monkeypatch):
+    def raise_error(req, timeout):
+        raise OSError("connection timeout")
+
+    monkeypatch.setattr(dovecot_deployer.urllib.request, "urlopen", raise_error)
+    result = dovecot_deployer._pick_url("http://primary", "http://fallback")
+    assert result == "http://fallback", (
+        f"should fall back when primary fails, got {result!r}"
+    )

cmdeploy/src/cmdeploy/tests/test_external_tls.py

@@ -0,0 +1,78 @@
+"""Functional tests for tls_external_cert_and_key option."""
+
+import json
+
+import chatmaild.newemail
+import pytest
+from chatmaild.config import read_config, write_initial_config
+
+
+def make_external_config(tmp_path, cert_key=None):
+    inipath = tmp_path / "chatmail.ini"
+    overrides = {}
+    if cert_key is not None:
+        overrides["tls_external_cert_and_key"] = cert_key
+    write_initial_config(inipath, "chat.example.org", overrides=overrides)
+    return inipath
+
+
+def test_external_tls_config_reads_paths(tmp_path):
+    inipath = make_external_config(
+        tmp_path,
+        cert_key=(
+            "/etc/letsencrypt/live/chat.example.org/fullchain.pem"
+            " /etc/letsencrypt/live/chat.example.org/privkey.pem"
+        ),
+    )
+    config = read_config(inipath)
+    assert config.tls_cert_mode == "external"
+    assert (
+        config.tls_cert_path == "/etc/letsencrypt/live/chat.example.org/fullchain.pem"
+    )
+    assert config.tls_key_path == "/etc/letsencrypt/live/chat.example.org/privkey.pem"
+
+
+def test_external_tls_missing_option_uses_acme(tmp_path):
+    config = read_config(make_external_config(tmp_path))
+    assert config.tls_cert_mode == "acme"
+
+
+def test_external_tls_bad_format_raises(tmp_path):
+    inipath = make_external_config(tmp_path, cert_key="/only/one/path.pem")
+    with pytest.raises(ValueError, match="two space-separated"):
+        read_config(inipath)
+
+
+def test_external_tls_three_paths_raises(tmp_path):
+    inipath = make_external_config(tmp_path, cert_key="/a /b /c")
+    with pytest.raises(ValueError, match="two space-separated"):
+        read_config(inipath)
+
+
+def test_external_tls_no_dclogin_url(tmp_path, capsys, monkeypatch):
+    inipath = make_external_config(
+        tmp_path, cert_key="/certs/fullchain.pem /certs/privkey.pem"
+    )
+    monkeypatch.setattr(chatmaild.newemail, "CONFIG_PATH", str(inipath))
+    chatmaild.newemail.print_new_account()
+    out, _ = capsys.readouterr()
+    lines = out.split("\n")
+    dic = json.loads(lines[2])
+    assert "dclogin_url" not in dic
+
+
+def test_external_tls_selects_correct_deployer(tmp_path):
+    from cmdeploy.deployers import get_tls_deployer
+    from cmdeploy.external.deployer import ExternalTlsDeployer
+    from cmdeploy.selfsigned.deployer import SelfSignedTlsDeployer
+
+    inipath = make_external_config(
+        tmp_path, cert_key="/certs/fullchain.pem /certs/privkey.pem"
+    )
+    config = read_config(inipath)
+    deployer = get_tls_deployer(config, "chat.example.org")
+
+    assert isinstance(deployer, ExternalTlsDeployer)
+    assert not isinstance(deployer, SelfSignedTlsDeployer)
+    assert deployer.cert_path == "/certs/fullchain.pem"
+    assert deployer.key_path == "/certs/privkey.pem"

cmdeploy/src/cmdeploy/tests/test_helpers.py

@@ -1,11 +1,10 @@
-import importlib.resources
+from pathlib import Path
 
 from cmdeploy.www import build_webpages
 
 
 def test_build_webpages(tmp_path, make_config):
-    pkgroot = importlib.resources.files("cmdeploy")
-    src_dir = pkgroot.joinpath("../../../www/src").resolve()
+    src_dir = (Path(__file__).resolve() / "../../../../../www/src").resolve()
     assert src_dir.exists(), src_dir
     config = make_config("chat.example.org")
     build_dir = tmp_path.joinpath("build")

cmdeploy/src/cmdeploy/tests/test_rshell.py

@@ -0,0 +1,68 @@
+from unittest.mock import patch
+
+from cmdeploy.remote.rshell import dovecot_recalc_quota
+
+
+def test_dovecot_recalc_quota_normal_output():
+    """Normal doveadm output returns parsed dict."""
+    normal_output = (
+        "Quota name Type    Value  Limit  %\n"
+        "User quota STORAGE     5 102400  0\n"
+        "User quota MESSAGE     2      -  0\n"
+    )
+
+    with patch("cmdeploy.remote.rshell.shell", return_value=normal_output):
+        result = dovecot_recalc_quota("user@example.org")
+
+    # shell is called twice (recalc + get), patch returns same for both
+    assert result == {"value": 5, "limit": 102400, "percent": 0}
+
+
+def test_dovecot_recalc_quota_empty_output():
+    """Empty doveadm output (trailing newline) must not IndexError."""
+    call_count = [0]
+
+    def mock_shell(cmd):
+        call_count[0] += 1
+        if "recalc" in cmd:
+            return ""
+        # quota get returns only empty lines
+        return "\n\n"
+
+    with patch("cmdeploy.remote.rshell.shell", side_effect=mock_shell):
+        result = dovecot_recalc_quota("user@example.org")
+
+    assert result is None
+
+
+def test_dovecot_recalc_quota_malformed_output():
+    """Malformed output with too few columns must not crash."""
+    call_count = [0]
+
+    def mock_shell(cmd):
+        call_count[0] += 1
+        if "recalc" in cmd:
+            return ""
+        # partial line, fewer than 6 parts
+        return "Quota name\nUser quota STORAGE\n"
+
+    with patch("cmdeploy.remote.rshell.shell", side_effect=mock_shell):
+        result = dovecot_recalc_quota("user@example.org")
+
+    assert result is None
+
+
+def test_dovecot_recalc_quota_header_only():
+    """Only header line, no data rows."""
+    call_count = [0]
+
+    def mock_shell(cmd):
+        call_count[0] += 1
+        if "recalc" in cmd:
+            return ""
+        return "Quota name Type    Value  Limit  %\n"
+
+    with patch("cmdeploy.remote.rshell.shell", side_effect=mock_shell):
+        result = dovecot_recalc_quota("user@example.org")
+
+    assert result is None

cmdeploy/src/cmdeploy/unbound/unbound.conf.j2

@@ -0,0 +1,4 @@
+# Managed by cmdeploy: disable IPv6 in unbound.
+server:
+  interface: 127.0.0.1
+  do-ip6: no

cmdeploy/src/cmdeploy/www.py

@@ -1,5 +1,4 @@
 import hashlib
-import importlib.resources
 import re
 import time
 import traceback
@@ -37,7 +36,7 @@ def prepare_template(source):
 
 
 def get_paths(config) -> (Path, Path, Path):
-    reporoot = importlib.resources.files(__package__).joinpath("../../../").resolve()
+    reporoot = (Path(__file__).resolve() / "../../../../").resolve()
     www_path = Path(config.www_folder)
     # if www_folder was not set, use default directory
     if config.www_folder == "":
@@ -133,8 +132,7 @@ def find_merge_conflict(src_dir) -> Path:
 
 
 def main():
-    path = importlib.resources.files(__package__)
-    reporoot = path.joinpath("../../../").resolve()
+    reporoot = (Path(__file__).resolve() / "../../../../").resolve()
     inipath = reporoot.joinpath("chatmail.ini")
     config = read_config(inipath)
     config.webdev = True

doc/README.md

@@ -4,12 +4,14 @@
 
 You can use the `make` command and `make html` to build web pages. 
 
-You need a Python environment where the following install was excuted: 
-
-    pip install furo sphinx-autobuild
+You need a Python environment with `sphinx` and other
+dependencies, you can create it by running `scripts/initenv.sh`
+from the repository root.
 
 To develop/change documentation, you can then do: 
 
+    . venv/bin/activate
+    cd doc
     make auto 
 
 A page will open at https://127.0.0.1:8000/ serving the docs and it will 

doc/source/getting_started.rst

@@ -16,15 +16,16 @@ You will need the following:
 
 -  Control over a domain through a DNS provider of your choice.
 
--  A Debian 12 server with reachable SMTP/SUBMISSIONS/IMAPS/HTTPS ports.
+-  A Debian 12 **deployment server** with reachable SMTP/SUBMISSIONS/IMAPS/HTTPS ports.
    IPv6 is encouraged if available. Chatmail relay servers only require
    1GB RAM, one CPU, and perhaps 10GB storage for a few thousand active
    chatmail addresses.
 
--  Key-based SSH authentication to the root user. You must add a
-   passphrase-protected private key to your local ssh-agent because you
-   can’t type in your passphrase during deployment. (An ed25519 private
-   key is required due to an `upstream bug in
+-  A Linux or Unix **build machine** with key-based SSH access to the root
+   user of the deployment server.
+   You must add a passphrase-protected private key to your local ssh-agent because you
+   can’t type in your passphrase during deployment.
+   (An ed25519 private key is required due to an `upstream bug in
    paramiko <https://github.com/paramiko/paramiko/issues/2191>`_)
 
 
@@ -34,16 +35,25 @@ Setup with ``scripts/cmdeploy``
 We use ``chat.example.org`` as the chatmail domain in the following
 steps. Please substitute it with your own domain.
 
-1. Setup the initial DNS records. The following is an example in the
+1. Setup the initial DNS records for your deployment server.
+   The following is an example in the
    familiar BIND zone file format with a TTL of 1 hour (3600 seconds).
    Please substitute your domain and IP addresses.
 
    ::
 
-       chat.example.com. 3600 IN A 198.51.100.5
-       chat.example.com. 3600 IN AAAA 2001:db8::5
-       www.chat.example.com. 3600 IN CNAME chat.example.com.
-       mta-sts.chat.example.com. 3600 IN CNAME chat.example.com.
+       chat.example.org. 3600 IN A 198.51.100.5
+       chat.example.org. 3600 IN AAAA 2001:db8::5
+       www.chat.example.org. 3600 IN CNAME chat.example.org.
+       mta-sts.chat.example.org. 3600 IN CNAME chat.example.org.
+
+   .. note::
+
+      For experimental deployments using self-signed certificates,
+      use a domain name starting with ``_``
+      (e.g. ``_chat.example.org``).
+      The ``mta-sts`` CNAME and ``_mta-sts`` TXT records
+      are not needed for such domains.
 
 2. On your local PC, clone the repository and bootstrap the Python
    virtualenv.
@@ -54,20 +64,30 @@ steps. Please substitute it with your own domain.
        cd relay
        scripts/initenv.sh
 
-3. On your local PC, create chatmail configuration file
+3. On your local build machine (PC), create a chatmail configuration file
    ``chatmail.ini``:
 
    ::
 
        scripts/cmdeploy init chat.example.org  # <-- use your domain
 
-4. Verify that SSH root login to your remote server works:
+   To use self-signed TLS certificates
+   instead of Let's Encrypt,
+   use a domain name starting with ``_``
+   (e.g. ``scripts/cmdeploy init _chat.example.org``).
+   Domains starting with ``_`` cannot obtain WebPKI certificates,
+   so self-signed mode is derived automatically.
+   This is useful for private or test deployments.
+   See the :doc:`overview`
+   for details on certificate provisioning.
+
+4. Verify that SSH root login to the deployment server server works:
 
    ::
 
        ssh root@chat.example.org  # <-- use your domain
 
-5. From your local PC, deploy the remote chatmail relay server:
+5. From your local build machine, setup and configure the remote deployment server:
 
    ::
 
@@ -81,7 +101,7 @@ steps. Please substitute it with your own domain.
 Other helpful commands
 ----------------------
 
-To check the status of your remotely running chatmail service:
+To check the status of your deployment server running the chatmail service:
 
 ::
 
@@ -158,7 +178,7 @@ Disable automatic address creation
 --------------------------------------------------------
 
 If you need to stop address creation, e.g. because some script is wildly
-creating addresses, login with ssh and run:
+creating addresses, login with ssh to the deployment machine and run:
 
 ::
 
@@ -167,3 +187,72 @@ creating addresses, login with ssh and run:
 Chatmail address creation will be denied while this file is present.
 
 
+Running a relay with self-signed certificates
+----------------------------------------------
+
+Use a domain name starting with ``_`` (e.g. ``_chat.example.org``)
+to run a relay with self-signed certificates.
+Domains starting with ``_`` cannot obtain WebPKI certificates
+so the relay automatically uses self-signed certificates
+and all other relays will accept connections from it
+without requiring certificate verification.
+This is useful for experimental setups and testing.
+
+.. _external-tls:
+
+Running a relay with externally managed certificates
+-----------------------------------------------------
+
+If you already have a TLS certificate manager
+(e.g. Traefik, certbot, or another ACME client)
+running on the deployment server,
+you can configure the relay to use those certificates
+instead of the built-in ``acmetool``.
+
+Set the following in ``chatmail.ini``::
+
+    tls_external_cert_and_key = /path/to/fullchain.pem /path/to/privkey.pem
+
+The paths must point to certificate and key files
+on the deployment server.
+During ``cmdeploy run``, these paths are written into
+the Postfix, Dovecot, and Nginx configurations.
+No certificate files are transferred from the build machine —
+they must already exist on the server,
+managed by your external certificate tool.
+
+The deploy will verify that both files exist on the server.
+``acmetool`` is **not** installed or run in this mode.
+
+.. note::
+
+   You are responsible for certificate renewal.
+   When the certificate file changes on disk,
+   all relay services pick up the new certificate automatically
+   via a systemd path watcher installed during deploy.
+   The watcher uses inotify, which does not cross bind-mount boundaries.
+   If you use such a setup, you must trigger the reload explicitly after renewal::
+
+      systemctl start tls-cert-reload.service
+
+
+Migrating to a new build machine
+----------------------------------
+
+To move or add a build machine,
+clone the relay repository on the new build machine, and copy the ``chatmail.ini`` file from the old build machine.
+Make sure ``rsync`` is installed, then initialize the environment:
+
+::
+
+   ./scripts/initenv.sh
+
+Run safety checks before a new deployment:
+
+::
+
+   ./scripts/cmdeploy dns
+   ./scripts/cmdeploy status
+
+If you keep multiple build machines (ie laptop and desktop), keep ``chatmail.ini`` in sync between
+them.

doc/source/index.rst

@@ -16,5 +16,6 @@ Contributions and feedback welcome through the https://github.com/chatmail/relay
     proxy
     migrate
     overview
+    reverse_dns
     related
     faq

doc/source/migrate.rst

@@ -1,72 +1,98 @@
 
-Migrating to a new host
------------------------
+Migrating to a new machine
+===========================
 
-If you want to migrate chatmail relay from an old machine to a new
-machine, you can use these steps. They were tested with a Linux laptop;
-you might need to adjust some of the steps to your environment.
+This migration tutorial provides a step-wise approach
+to safely migrate a chatmail relay from one remote machine to another.
 
-Let’s assume that your ``mail_domain`` is ``mail.example.org``, all
-involved machines run Debian 12, your old site’s IP address is
-``13.37.13.37``, and your new site’s IP address is ``13.12.23.42``.
+Preliminary notes and assumptions
+---------------------------------
 
-Note, you should lower the TTLs of your DNS records to a value such as
-300 (5 minutes) so the migration happens as smoothly as possible.
+- If the migration is a planned move,
+  it's recommended to lower the Time To Live (TTL) of your DNS records to a value such as 300 (5 minutes),
+  at best much earlier than the actual planned migration.
+  This speeds up propagation of DNS changes in the Internet after the migration is complete.
 
-During the guide you might get a warning about changed SSH Host keys; in
-this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
+- The migration steps were tested with a Linux laptop; you might need to adjust some of the steps to your local environment.
 
-1. First, disable mail services on the old site.
+- Your ``mail_domain`` is ``mail.example.org``.
+
+- All remote machines run Debian 12.
+
+- The old site’s IP version 4 address is ``$OLD_IP4``.
+
+- The new site’s IP addresses are ``$NEW_IP4`` and ``$NEW_IPV6``.
+
+
+The six steps to migrate
+------------------------
+
+Note that during some of the following steps you might get a warning about changed SSH Host keys;
+in this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
+
+
+1. **Initially transfer mailboxes from old to new site.**
+
+   Login to old site, forwarding your ssh-agent with ``ssh -A``
+   to allow using ssh to directly copy files from old to new site.
+   ::
+
+       ssh -A root@$OLD_IP4
+       tar c /home/vmail/mail | ssh root@$NEW_IP4 "tar x -C /"
+
+
+2. **Pre-configure the new site but keep it inactive until step 6**
+   ::
+
+       CMDEPLOY_STAGES=install,configure scripts/cmdeploy run --ssh-host $NEW_IP4
+
+
+3. **It's getting serious: disable mail services on the old site.**
+   Users will not be able to send or receive messages until all steps are completed.
+   Other relays and mail servers will retry delivering messages from time to time,
+   so nothing is lost for users.
 
    ::
 
-       cmdeploy run --disable-mail --ssh-host 13.37.13.37
+       scripts/cmdeploy run --disable-mail --ssh-host $OLD_IP4
 
-   Now your users will notice the migration and will not be able to send
-   or receive messages until the migration is completed.
 
-2. Now we want to copy ``/home/vmail``, ``/var/lib/acme``,
-   ``/etc/dkimkeys``, and ``/var/spool/postfix`` to
-   the new site. Login to the old site while forwarding your SSH agent
-   so you can copy directly from the old to the new site with your SSH
-   key:
-
-   ::
-
-       ssh -A root@13.37.13.37
-       tar c - /home/vmail/mail /var/lib/acme /etc/dkimkeys /var/spool/postfix | ssh root@13.12.23.42 "tar x -C /"
-
-   This transfers all addresses, the TLS certificate,
-   and DKIM keys (so DKIM DNS record remains valid).
-   It also preserves the Postfix mail spool so any messages
-   pending delivery will still be delivered.
-
-3. Install chatmail on the new machine:
-
-   ::
-
-       cmdeploy run --disable-mail --ssh-host 13.12.23.42
-
-   Postfix and Dovecot are disabled for now; we will enable them later.
-   We first need to make the new site fully operational.
-
-4. On the new site, run the following to ensure the ownership is correct
-   in case UIDs/GIDs changed:
+4. **Final synchronization of TLS/DKIM secrets, mail queues and mailboxes.**
+   Again we use ssh-agent forwarding (``-A``) to allow transfering all important data directly
+   from the old to the new site.
+   ::
+
+       ssh -A root@$OLD_IP4
+       tar c /var/lib/acme /etc/dkimkeys /var/spool/postfix | ssh root@$NEW_IP4 "tar x -C /"
+       rsync -azH /home/vmail/mail root@$NEW_IP4:/home/vmail/
+
+   Login to the new site and ensure file ownerships are correctly set:
 
    ::
 
+       ssh root@$NEW_IP4
        chown root: -R /var/lib/acme
        chown opendkim: -R /etc/dkimkeys
        chown vmail: -R /home/vmail/mail
 
-5. Now, update DNS entries.
 
-   If other MTAs try to deliver messages to your chatmail domain they
-   may fail intermittently, as DNS catches up with the new site settings
-   but normally will retry delivering messages for at least a week, so
-   messages will not be lost.
+5. **Update the DNS entries to point to the new site.**
+   You only need to change the ``A`` and ``AAAA`` records, for example:
 
-6. Finally, you can execute ``cmdeploy run --ssh-host 13.12.23.42`` to
-   turn on chatmail on the new relay. Your users will be able to use the
-   chatmail relay as soon as the DNS changes have propagated. Voilà!
+   ::
+
+       mail.example.org.    IN A    $NEW_IP4
+       mail.example.org.    IN AAAA $NEW_IP6
+
+
+6. **Activate chatmail relay on new site.**
+
+   ::
+
+        CMDEPLOY_STAGES=activate scripts/cmdeploy run --ssh-host $NEW_IP4
+
+   Voilà!
+   Users will be able to use the relay as soon as the DNS changes have propagated.
+   If you have lowered the Time-to-Live for DNS records in step 1,
+   better use a higher value again (between 14400 and 86400 seconds) once you are sure everything works.
 

doc/source/overview.rst

@@ -42,6 +42,11 @@ The deployed system components of a chatmail relay are:
 -  Dovecot_ is the Mail Delivery Agent (MDA) and
    stores messages for users until they download them
 
+-  `filtermail <https://github.com/chatmail/filtermail>`_
+   prevents unencrypted email from leaving or entering the chatmail
+   service and is integrated into Postfix’s outbound and inbound mail
+   pipelines.
+
 -  Nginx_ shows the web page with privacy policy and additional information
 
 -  `acmetool <https://hlandau.github.io/acmetool/>`_ manages TLS
@@ -85,11 +90,6 @@ short overview of ``chatmaild`` services:
    <https://doc.dovecot.org/2.3/configuration_manual/authentication/dict/#complete-example-for-authenticating-via-a-unix-socket>`_
    to authenticate logins.
 
--  `filtermail <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/filtermail.py>`_
-   prevents unencrypted email from leaving or entering the chatmail
-   service and is integrated into Postfix’s outbound and inbound mail
-   pipelines.
-
 -  `chatmail-metadata <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/metadata.py>`_
    is contacted by a `Dovecot lua
    script <https://github.com/chatmail/relay/blob/main/cmdeploy/src/cmdeploy/dovecot/push_notification.lua>`_
@@ -102,17 +102,17 @@ short overview of ``chatmaild`` services:
    Apple/Google/Huawei.
 
 -  `chatmail-expire <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/expire.py>`_
-   deletes users if they have not logged in for a longer while.
-   The timeframe can be configured in ``chatmail.ini``.
+   deletes old messages, large messages, and entire mailboxes
+   of users who have not logged in for longer than
+   ``delete_inactive_users_after`` days.
+
+-  ``chatmail-quota-expire`` is called by Dovecot's ``quota_warning`` mechanism
+   and will automatically remove oldest messages to keep mailboxes well under ``max_mailbox_size``.
 
 -  `lastlogin <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/lastlogin.py>`_
    is contacted by Dovecot when a user logs in and stores the date of
    the login.
 
--  `metrics <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/metrics.py>`_
-   collects some metrics and displays them at
-   ``https://example.org/metrics``.
-
 ``www/``
 ~~~~~~~~~
 
@@ -142,11 +142,9 @@ Chatmail relay dependency diagram
         nginx-internal --- autoconfig.xml;
         certs-nginx[("`TLS certs
         /var/lib/acme`")] --> nginx-internal;
-        systemd-timer --- chatmail-metrics;
         systemd-timer --- acmetool;
         systemd-timer --- chatmail-expire-daily;
         systemd-timer --- chatmail-fsreport-daily;
-        chatmail-metrics --- website;
         acmetool --> certs[("`TLS certs
         /var/lib/acme`")];
         nginx-external --- |993|dovecot;
@@ -155,6 +153,7 @@ Chatmail relay dependency diagram
         autoconfig.xml --- dovecot;
         postfix --- |10080|filtermail-outgoing;
         postfix --- |10081|filtermail-incoming;
+        postfix --- |10083|filtermail-transport;
         filtermail-outgoing --- |10025 reinject|postfix;
         filtermail-incoming --- |10026 reinject|postfix;
         dovecot --- |doveauth.socket|doveauth;
@@ -162,6 +161,8 @@ Chatmail relay dependency diagram
         /home/vmail/.../user"];
         dovecot --- |lastlogin.socket|lastlogin;
         dovecot --- chatmail-metadata;
+        dovecot --- |quota-warning|chatmail-quota-expire;
+        chatmail-quota-expire --- maildir;
         lastlogin --- maildir;
         doveauth --- maildir;
         chatmail-expire-daily --- maildir;
@@ -295,10 +296,7 @@ ensured by ``filtermail`` proxy.
 TLS requirements
 ~~~~~~~~~~~~~~~~
 
-Postfix is configured to require valid TLS by setting
-`smtp_tls_security_level <https://www.postfix.org/postconf.5.html#smtp_tls_security_level>`_
-to ``verify``. If emails don’t arrive at your chatmail relay server, the
-problem is likely that your relay does not have a valid TLS certificate.
+Filtermail (used for delivery) requires a valid TLS.
 
 You can test it by resolving ``MX`` records of your relay domain and
 then connecting to MX relays (e.g ``mx.example.org``) with
@@ -309,6 +307,11 @@ When providing a TLS certificate to your chatmail relay server, make
 sure to provide the full certificate chain and not just the last
 certificate.
 
+If you use an external certificate manager (e.g. Traefik or certbot),
+set ``tls_external_cert_and_key`` in ``chatmail.ini``
+to provide the certificate and key paths.
+See :ref:`external-tls` for details.
+
 If you are running an Exim server and don’t see incoming connections
 from a chatmail relay server in the logs, make sure ``smtp_no_mail`` log
 item is enabled in the config with ``log_selector = +smtp_no_mail``. By
@@ -317,6 +320,14 @@ default Exim does not log sessions that are closed before sending the
 by Postfix, so you might think that connection is not established while
 actually it is a problem with your TLS certificate.
 
+If emails don’t arrive at your chatmail relay server, the
+problem is likely that your relay does not have a valid TLS certificate.
+
+Note that connections to relays with underscore-prefixed test domains
+(e.g. ``_chat.example.org``) use ``encrypt`` tls security level,
+because such domains cannot obtain valid Let's Encrypt certificates
+and run with self-signed certificates.
+
 
 .. _dovecot: https://dovecot.org
 .. _postfix: https://www.postfix.org

doc/source/related.rst

@@ -14,10 +14,10 @@ We know of three work-in-progress alternative implementation efforts:
    it to support all of the features and configuration settings required
    to operate as a chatmail relay.
 
--  `Maddy-Chatmail <https://github.com/sadraiiali/maddy_chatmail>`_: a
-   plugin for the `Maddy email server <https://maddy.email/>`_ which
-   aims to implement the chatmail relay features and configuration
-   options.
+-  `Madmail <https://github.com/themadorg/madmail>`_: an
+   experimental fork of `Maddy Mail Server <https://maddy.email/>`_, modified
+   for chatmail deployments.  It provides a single binary solution
+   for running a chatmail relay.
 
 -  `Chatmail Cookbook <https://github.com/feld/chatmail-cookbook>`_:
    A Chef Cookbook implementing a relay server. The project follows the

doc/source/reverse_dns.rst

@@ -0,0 +1,64 @@
+Configuring reverse DNS
+=======================
+
+Some email servers reject the emails
+if they don't pass `FCrDNS`_ check, also known as `iprev`_ check.
+
+.. _FCrDNS: https://en.wikipedia.org/wiki/Forward-confirmed_reverse_DNS
+.. _iprev: https://datatracker.ietf.org/doc/html/rfc8601#section-3
+
+Passing the check requires that the IP address that email is sent from
+should have a ``PTR`` record pointing to the domain name of the server,
+and domain name record should have an ``A/AAAA`` record
+pointing to the IP address.
+
+Modern email relies on DKIM and SPF for authentication,
+while iprev check exists for
+`historical reasons <https://datatracker.ietf.org/doc/html/draft-ietf-dnsop-reverse-mapping-considerations-06#section-2.1>`_.
+Chatmail relays don't resolve ``PTR`` records,
+so you can ignore this section if configuring ``PTR`` records
+is difficult and federation with legacy email servers that don't accept
+valid DKIM signature for authentication is not important.
+
+Multi-homed setups
+------------------
+
+If you have a server with multiple IP addresses,
+also known as multi-homed setup,
+and don't publish all IP addresses in DNS,
+you need to make sure you are using
+the published address when making outgoing connections.
+
+For example, your server may have a static IP
+address, and a so-called Floating IP or Virtual IP
+that can be moved between servers in case of
+migration or for failover.
+By using Floating IP you can avoid downtime
+and keep the IP address reputation
+for destinatinons that rely on IP reputation and IP blocklists.
+In this case you will only publish
+the Floating IP to DNS and only use the static IP
+to SSH into the server.
+
+If you have such setup, make sure that
+you not only set ``PTR`` records for the Floating IP,
+but make outgoing connections using the Floating IP.
+Otherwise reverse DNS check succeed,
+but forward check making sure your domain name points
+to the IP address will fail.
+Such setup is indistinguishable from someone
+setting IP address ``PTR`` with the domain they don't own
+and as a result don't succeed.
+
+On Linux you can configure source IP address with ``ip route`` command,
+for example:
+::
+
+  ip route change default via <default-gateway> dev eth0 src <source-address>
+
+Make sure to persist the change after verifying it is working.
+You can check what your outgoing IP address is
+with ``curl icanhazip.com``.
+Check both the IPv4 and IPv6 addresses.
+For IPv4 address use ``curl ipv4.icanhazip.com`` or ``curl -4 icanhazip.com``
+and similarly for IPv6 if you have it.

www/src/dclogin.js

@@ -0,0 +1,21 @@
+/* dclogin profile generator for self-signed chatmail relays.
+ * Fetches credentials from /new and generates a dclogin: QR code.
+ * Requires qrcode-svg.min.js to be loaded first.
+ */
+(function () {
+    function generateProfile() {
+        fetch('/new')
+            .then(function (r) { return r.json(); })
+            .then(function (data) {
+                var url = data.dclogin_url;
+                var link = document.getElementById('dclogin-link');
+                link.href = url;
+                var qrLink = document.getElementById('qr-link');
+                qrLink.href = url;
+                var qrCode = document.getElementById('qr-code');
+                var qr = new QRCode({ content: url, width: 300, height: 300, padding: 1, join: true });
+                qrCode.innerHTML = qr.svg();
+            });
+    }
+    generateProfile();
+})();

www/src/index.md

@@ -11,6 +11,18 @@ for Delta Chat users.  For details how it avoids storing personal information
 please see our [privacy policy](privacy.html). 
 {% endif %}
 
+{% if config.tls_cert_mode == "self" %}
+<a class="cta-button" id="dclogin-link" href="#">Get a {{config.mail_domain}} chat profile</a>
+
+If you are viewing this page on a different device
+without a Delta Chat app,
+you can also **scan this QR code** with Delta Chat:
+
+<a id="qr-link" href="#"><div id="qr-code"></div></a>
+
+<script src="qrcode-svg.min.js"></script>
+<script src="dclogin.js"></script>
+{% else %}
 <a class="cta-button" href="DCACCOUNT:https://{{ config.mail_domain }}/new">Get a {{config.mail_domain}} chat profile</a>
 
 If you are viewing this page on a different device
@@ -19,6 +31,7 @@ you can also **scan this QR code** with Delta Chat:
 
 <a href="DCACCOUNT:https://{{ config.mail_domain }}/new">
     <img width=300 style="float: none;" src="qr-chatmail-invite-{{config.mail_domain}}.png" /></a>
+{% endif %}
 
 🐣 **Choose** your Avatar and Name