feat: add tool to analyze deferred queue

It prints all destinations with the number of recipients
and all the reasons. Operator can then try
to fix the problems for destinations,
e.g. by manually adding reverse proxy
addresses to /etc/hosts for failing domains
or routing IP addresses to another interface.

.github/workflows/ci.yaml

@@ -1,21 +1,35 @@
-name: CI
+name: Run unit-tests and container-based deploy+test verification
 
 on:
-  pull_request:
+  # Triggers when a PR is merged into main or a direct push occurs
   push:
+    branches: [ "main" ]
+    
+  # Triggers for any PR (and its subsequent commits) targeting the main branch
+  pull_request:
+    branches: [ "main" ]
+
+permissions: {}
+
+# Newest push wins: Prevents multiple runs from clashing and wasting runner efforts
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 
 jobs:
   tox:
     name: isolated chatmaild tests
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         # Checkout pull request HEAD commit instead of merge commit
         # Otherwise `test_deployed_state` will be unhappy.
         with:
           ref: ${{ github.event.pull_request.head.sha }}
+          persist-credentials: false
       - name: download filtermail
-        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.5.2/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
+        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.6.4/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
       - name: run chatmaild tests 
         working-directory: chatmaild
         run: pipx run tox
@@ -24,7 +38,10 @@ jobs:
     name: deploy-chatmail tests 
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          persist-credentials: false
 
       - name: initenv 
         run: scripts/initenv.sh
@@ -38,5 +55,23 @@ jobs:
       - name: run deploy-chatmail offline tests 
         run: pytest --pyargs cmdeploy 
 
-      # all other cmdeploy commands require a staging server 
-      # see https://github.com/deltachat/chatmail/issues/100
+  lxc-test:
+    name: LXC deploy and test
+    uses: chatmail/cmlxc/.github/workflows/lxc-test.yml@v0.10.0
+    with:
+      cmlxc_commands: |
+        cmlxc init 
+        # single cmdeploy relay test
+        cmlxc -v deploy-cmdeploy --source ./repo cm0
+        cmlxc -v test-mini cm0
+        cmlxc -v test-cmdeploy cm0
+
+        # cross cmdeploy relay test
+        cmlxc -v deploy-cmdeploy --source ./repo --ipv4-only cm1
+        cmlxc -v test-cmdeploy cm0 cm1
+
+        # cross cmdeploy/madmail relay tests
+        cmlxc -v deploy-madmail mad0
+        cmlxc -v test-cmdeploy cm0 mad0
+        cmlxc -v test-mini cm0 mad0
+        cmlxc -v test-mini mad0 cm0

.github/workflows/docker-dispatch.yaml

@@ -0,0 +1,37 @@
+# Notify the docker repo to build and test a new image after relay CI passes.
+#
+# Sends a repository_dispatch event to chatmail/docker with the relay ref
+# and short SHA, which triggers docker-ci.yaml to build, push to GHCR,
+# and run integration tests via cmlxc.
+
+name: Trigger Docker build
+
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+permissions: {}
+
+jobs:
+  dispatch:
+    name: Dispatch build to chatmail/docker
+    runs-on: ubuntu-latest
+    if: github.repository == 'chatmail/relay'
+    steps:
+      - name: Compute short SHA
+        id: sha
+        run: echo "short=$(echo '${{ github.sha }}' | cut -c1-7)" >> "$GITHUB_OUTPUT"
+
+      - name: Send repository_dispatch
+        uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # v3
+        with:
+          token: ${{ secrets.CHATMAIL_DOCKER_DISPATCH_TOKEN }}
+          repository: chatmail/docker
+          event-type: relay-updated
+          client-payload: >-
+            {
+              "relay_ref": "${{ github.ref_name }}",
+              "relay_sha": "${{ github.sha }}",
+              "relay_sha_short": "${{ steps.sha.outputs.short }}"
+            }

.github/workflows/docs-preview.yaml

@@ -7,6 +7,8 @@ on:
       - 'scripts/build-docs.sh'
       - '.github/workflows/docs-preview.yaml'
 
+permissions: {}
+
 jobs:
   scripts:
     name: build 
@@ -16,6 +18,8 @@ jobs:
       url: https://staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}
     steps:
       - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
 
       - name: initenv 
         run: scripts/initenv.sh
@@ -34,18 +38,22 @@ jobs:
       - name: Get Pullrequest ID
         id: prepare
         run: |
-          export PULLREQUEST_ID=$(echo "${{ github.ref }}" | cut -d "/" -f3)
+          export PULLREQUEST_ID=$(echo "${GITHUB_REF}" | cut -d "/" -f3)
           echo "prid=$PULLREQUEST_ID" >> $GITHUB_OUTPUT
           if [ $(expr length "${{ secrets.USERNAME }}") -gt "1" ]; then echo "uploadtoserver=true" >> $GITHUB_OUTPUT; fi
       - run: |
-          echo "baseurl: /${{ steps.prepare.outputs.prid }}" >> _config.yml
+          echo "baseurl: /${STEPS_PREPARE_OUTPUTS_PRID}" >> _config.yml
+        env:
+          STEPS_PREPARE_OUTPUTS_PRID: ${{ steps.prepare.outputs.prid }}
 
       - name: Upload preview
         run: |
           mkdir -p "$HOME/.ssh"
           echo "${{ secrets.CHATMAIL_STAGING_SSHKEY }}" > "$HOME/.ssh/key"
           chmod 600 "$HOME/.ssh/key"
-          rsync -rILvh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/doc/build/ "${{ secrets.USERNAME }}@chatmail.at:/var/www/html/staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}/"
+          rsync -rILvh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/doc/build/ "${{ secrets.USERNAME }}@chatmail.at:/var/www/html/staging.chatmail.at/doc/relay/${STEPS_PREPARE_OUTPUTS_PRID}/"
+        env:
+          STEPS_PREPARE_OUTPUTS_PRID: ${{ steps.prepare.outputs.prid }}
 
       - name: check links 
         working-directory: doc

.github/workflows/docs.yaml

@@ -10,6 +10,8 @@ on:
       - 'scripts/build-docs.sh'
       - '.github/workflows/docs.yaml'
 
+permissions: {}
+
 jobs:
   scripts:
     name: build 
@@ -19,6 +21,8 @@ jobs:
       url: https://chatmail.at/doc/relay/
     steps:
       - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
 
       - name: initenv 
         run: scripts/initenv.sh

.github/workflows/test-and-deploy-ipv4only.yaml

@@ -1,104 +0,0 @@
-name: deploy on staging-ipv4.testrun.org, and run tests
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    paths-ignore:
-      - 'scripts/**'
-      - '**/README.md'
-      - 'CHANGELOG.md'
-      - 'LICENSE'
-
-jobs:
-  deploy:
-    name: deploy on staging-ipv4.testrun.org, and run tests
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    environment:
-      name: staging-ipv4.testrun.org
-      url: https://staging-ipv4.testrun.org/
-    concurrency: staging-ipv4.testrun.org
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: prepare SSH
-        run: |
-          mkdir ~/.ssh
-          echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
-          chmod 600 ~/.ssh/id_ed25519
-          ssh-keyscan staging-ipv4.testrun.org > ~/.ssh/known_hosts
-          # save previous acme & dkim state
-          rsync -avz root@staging-ipv4.testrun.org:/var/lib/acme acme-ipv4 || true
-          rsync -avz root@staging-ipv4.testrun.org:/etc/dkimkeys dkimkeys-ipv4 || true
-          # store previous acme & dkim state on ns.testrun.org, if it contains useful certs
-          if [ -f dkimkeys-ipv4/dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys-ipv4 root@ns.testrun.org:/tmp/ || true; fi
-          if [ "$(ls -A acme-ipv4/acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme-ipv4 root@ns.testrun.org:/tmp/ || true; fi
-          # make sure CAA record isn't set
-          scp -o StrictHostKeyChecking=accept-new .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
-          ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging-ipv4.testrun.org.zone
-          ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
-          ssh root@ns.testrun.org systemctl reload nsd
-
-      - name: rebuild staging-ipv4.testrun.org to have a clean VPS
-        run: |
-            curl -X POST \
-            -H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
-            -H "Content-Type: application/json" \
-            -d '{"image":"debian-12"}' \
-            "https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_IPV4_SERVER_ID }}/actions/rebuild"
-
-      - run: scripts/initenv.sh
-
-      - name: append venv/bin to PATH
-        run: echo venv/bin >>$GITHUB_PATH
-
-      - name: upload TLS cert after rebuilding
-        run: |
-          echo " --- wait until staging-ipv4.testrun.org VPS is rebuilt --- "
-          rm ~/.ssh/known_hosts
-          while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u ; do sleep 1 ; done
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u
-          # download acme & dkim state from ns.testrun.org
-          rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme-ipv4/acme acme-restore || true
-          rsync -avz root@ns.testrun.org:/tmp/dkimkeys-ipv4/dkimkeys dkimkeys-restore || true
-          # restore acme & dkim state to staging2.testrun.org
-          rsync -avz acme-restore/acme root@staging-ipv4.testrun.org:/var/lib/ || true
-          rsync -avz dkimkeys-restore/dkimkeys root@staging-ipv4.testrun.org:/etc/ || true
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org chown root:root -R /var/lib/acme || true
-
-      - name: run deploy-chatmail offline tests 
-        run: pytest --pyargs cmdeploy 
-
-      - name: setup dependencies
-        run: |
-          ssh root@staging-ipv4.testrun.org apt update
-          ssh root@staging-ipv4.testrun.org apt install -y git python3.11-venv python3-dev gcc
-          ssh root@staging-ipv4.testrun.org git clone https://github.com/chatmail/relay
-          ssh root@staging-ipv4.testrun.org "cd relay && git checkout " ${{ github.head_ref }}
-          ssh root@staging-ipv4.testrun.org "cd relay && scripts/initenv.sh"
-
-      - name: initialize config
-        run: |
-          ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy init staging-ipv4.testrun.org"
-          ssh root@staging-ipv4.testrun.org "sed -i 's#disable_ipv6 = False#disable_ipv6 = True#' relay/chatmail.ini"
-          ssh root@staging-ipv4.testrun.org "sed -i 's/#\s*mtail_address/mtail_address/' relay/chatmail.ini"
-
-      - run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy run --verbose --skip-dns-check --ssh-host localhost"
-
-      - name: set DNS entries
-        run: |
-          ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns --zonefile staging-generated.zone --ssh-host localhost"
-          ssh root@staging-ipv4.testrun.org cat relay/staging-generated.zone >> .github/workflows/staging-ipv4.testrun.org-default.zone
-          cat .github/workflows/staging-ipv4.testrun.org-default.zone
-          scp .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
-          ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
-          ssh root@ns.testrun.org systemctl reload nsd
-
-      - name: cmdeploy test
-        run: ssh root@staging-ipv4.testrun.org "cd relay && CHATMAIL_DOMAIN2=ci-chatmail.testrun.org scripts/cmdeploy test --slow --ssh-host localhost"
-
-      - name: cmdeploy dns
-        run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns -v --ssh-host localhost"
-

.github/workflows/test-and-deploy.yaml

@@ -1,97 +0,0 @@
-name: deploy on staging2.testrun.org, and run tests
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    paths-ignore:
-      - 'scripts/**'
-      - '**/README.md'
-      - 'CHANGELOG.md'
-      - 'LICENSE'
-
-jobs:
-  deploy:
-    name: deploy on staging2.testrun.org, and run tests
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    environment:
-      name: staging2.testrun.org
-      url: https://staging2.testrun.org/
-    concurrency: staging2.testrun.org
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: prepare SSH
-        run: |
-          mkdir ~/.ssh
-          echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
-          chmod 600 ~/.ssh/id_ed25519
-          ssh-keyscan staging2.testrun.org > ~/.ssh/known_hosts
-          # save previous acme & dkim state
-          rsync -avz root@staging2.testrun.org:/var/lib/acme . || true
-          rsync -avz root@staging2.testrun.org:/etc/dkimkeys . || true
-          # store previous acme & dkim state on ns.testrun.org, if it contains useful certs
-          if [ -f dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys root@ns.testrun.org:/tmp/ || true; fi
-          if [ "$(ls -A acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme root@ns.testrun.org:/tmp/ || true; fi
-          # make sure CAA record isn't set
-          scp -o StrictHostKeyChecking=accept-new .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
-          ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging2.testrun.org.zone
-          ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
-          ssh root@ns.testrun.org systemctl reload nsd
-
-      - name: rebuild staging2.testrun.org to have a clean VPS
-        run: |
-            curl -X POST \
-            -H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
-            -H "Content-Type: application/json" \
-            -d '{"image":"debian-12"}' \
-            "https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_SERVER_ID }}/actions/rebuild"
-
-      - run: scripts/initenv.sh
-
-      - name: append venv/bin to PATH
-        run: echo venv/bin >>$GITHUB_PATH
-
-      - name: upload TLS cert after rebuilding
-        run: |
-          echo " --- wait until staging2.testrun.org VPS is rebuilt --- "
-          rm ~/.ssh/known_hosts
-          while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org id -u ; do sleep 1 ; done
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org id -u
-          # download acme & dkim state from ns.testrun.org
-          rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme acme-restore || true
-          rsync -avz root@ns.testrun.org:/tmp/dkimkeys dkimkeys-restore || true
-          # restore acme & dkim state to staging2.testrun.org
-          rsync -avz acme-restore/acme root@staging2.testrun.org:/var/lib/ || true
-          rsync -avz dkimkeys-restore/dkimkeys root@staging2.testrun.org:/etc/ || true
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org chown root:root -R /var/lib/acme || true
-
-      - name: add hpk42 key to staging server
-        run: ssh root@staging2.testrun.org 'curl -s https://github.com/hpk42.keys >> .ssh/authorized_keys'
-
-      - name: run deploy-chatmail offline tests 
-        run: pytest --pyargs cmdeploy 
-
-      - run: |
-          cmdeploy init staging2.testrun.org
-          sed -i 's/#\s*mtail_address/mtail_address/' chatmail.ini
-
-      - run: cmdeploy run --verbose --skip-dns-check
-
-      - name: set DNS entries
-        run: |
-          cmdeploy dns --zonefile staging-generated.zone --verbose
-          cat staging-generated.zone >> .github/workflows/staging.testrun.org-default.zone
-          cat .github/workflows/staging.testrun.org-default.zone
-          scp .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
-          ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
-          ssh root@ns.testrun.org systemctl reload nsd
-
-      - name: cmdeploy test
-        run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
-
-      - name: cmdeploy dns
-        run: cmdeploy dns -v
-

.github/workflows/zizmor-scan.yml

@@ -0,0 +1,26 @@
+name: GitHub Actions Security Analysis with zizmor
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+    branches: ["**"]
+
+permissions: {}
+
+jobs:
+  zizmor:
+    name: Run zizmor
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write # Required for upload-sarif (used by zizmor-action) to upload SARIF files.
+      contents: read
+      actions: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Run zizmor
+        uses: zizmorcore/zizmor-action@b1d7e1fb5de872772f31590499237e7cce841e8e # v0.5.3

.github/zizmor.yml

@@ -0,0 +1,7 @@
+rules:
+  unpinned-uses:
+    config:
+      policies:
+        actions/*: ref-pin
+        dependabot/*: ref-pin
+        chatmail/*: ref-pin

CHANGELOG.md

@@ -1,5 +1,89 @@
 # Changelog for chatmail deployment 
 
+## 1.10.0 2026-04-30
+
+* start mtail after networking is fully up <https://github.com/chatmail/relay/pull/942>
+* support specifying custom filtermail binary through environment variable <https://github.com/chatmail/relay/pull/941>
+* add automated zizmor scanning of github workflows <https://github.com/chatmail/relay/pull/938>
+* added dispatch for *automated builds of chatmail relay docker images* <https://github.com/chatmail/relay/pull/934>
+* do not bind SMTP client sockets to public addresses <https://github.com/chatmail/relay/pull/932>
+* underline in docs that scripts/initenv.sh should be used for building the docs <https://github.com/chatmail/relay/pull/933>
+* automatic oldest-first message removal from mailboxes to always stay under max_mailbox_size <https://github.com/chatmail/relay/pull/929>
+* remove --slow from cmdeploy test <https://github.com/chatmail/relay/pull/931>
+* handle missing inotify sysctl keys in containers <https://github.com/chatmail/relay/pull/930>
+* replace resolvconf with static resolv.conf <https://github.com/chatmail/relay/pull/928>
+* disable fsync for LMTP and IMAP services <https://github.com/chatmail/relay/pull/925>
+* re-use cmlxc workflow, replacing CI with hetzner staging servers with local lxc containers <https://github.com/chatmail/relay/pull/917>
+* explicitly install resolvconf <https://github.com/chatmail/relay/pull/924>
+* detect stale dovecot binary and force restart in activate() <https://github.com/chatmail/relay/pull/922>
+* Rename filtermail_http_port to filtermail_http_port_incoming <https://github.com/chatmail/relay/pull/921>
+* consolidated is_in_container() check https://github.com/chatmail/relay/pull/920>
+* restart dovecot after package replacement (rebase, test condense) <https://github.com/chatmail/relay/pull/913>
+* Set permissions on dovecot pin prefs <https://github.com/chatmail/relay/pull/915>
+* Route `/mxdeliv/` to configurable port <https://github.com/chatmail/relay/pull/901>
+* fix VM detection, automated testing fixes, use newer chatmail-turn and move to standard BIND DNS zone format <https://github.com/chatmail/relay/pull/912>
+* Upgrade to filtermail 0.6.1 <https://github.com/chatmail/relay/pull/910>
+* pin dovecot packages to prevent apt upgrades <https://github.com/chatmail/relay/pull/908>
+* add rpc server to cmdeploy along with client <https://github.com/chatmail/relay/pull/906>
+* remove unused deps from chatmaild <https://github.com/chatmail/relay/pull/905>
+* set default smtp_tls_security_level to "verify" unconditionally <https://github.com/chatmail/relay/pull/902>
+* featprefer IPv4 in SMTP client <https://github.com/chatmail/relay/pull/900>
+* Install dovecot .deb packages atomically <https://github.com/chatmail/relay/pull/899>
+* stop installing cron package <https://github.com/chatmail/relay/pull/898>
+* Rewrite dovecot install logic, update <https://github.com/chatmail/relay/pull/862>
+* fix a test and some linting fixes <https://github.com/chatmail/relay/pull/897>
+* Disable IP verification on domain-literal addresses <https://github.com/chatmail/relay/pull/895>
+* disable installing recommended packages globally on the relay <https://github.com/chatmail/relay/pull/887>
+* multiple bug fixes across chatmaild and cmdeploy <https://github.com/chatmail/relay/pull/883>
+* remove /metrics from the website <https://github.com/chatmail/relay/pull/703>
+* add Prometheus textfile output to fsreport <https://github.com/chatmail/relay/pull/881>
+* chown opendkim: private key <https://github.com/chatmail/relay/pull/879>
+* make sure chatmail-metadata was started <https://github.com/chatmail/relay/pull/882>
+* dovecot update url <https://github.com/chatmail/relay/pull/880>
+* upgrade to filtermail v0.5.2 <https://github.com/chatmail/relay/pull/876>
+* download dovecot packages from github release <https://github.com/chatmail/relay/pull/875>
+* replace DKIM verification with filtermail v0.5 <https://github.com/chatmail/relay/pull/831>
+* remove CFFI deltachat bindings usage, and consolidate test support with rpc-bindings <https://github.com/chatmail/relay/pull/872>
+* prepare chatmaild/cmdeploy changes for Docker support <https://github.com/chatmail/relay/pull/857>
+* stabilize online benchmark timing adding rate-limit-aware cooldown between iterations <https://github.com/chatmail/relay/pull/867>
+* move rate-limit cooldown to benchmark fixture <https://github.com/chatmail/relay/pull/868>
+* reconfigure acmetool from redirector to proxy mode <https://github.com/chatmail/relay/pull/861>
+* make tests work with `--ssh-host localhost` <https://github.com/chatmail/relay/pull/856>
+* mark f-string with f prefix in test_expunged <https://github.com/chatmail/relay/pull/863>
+* install also if dovecot.service=False in SystemdEnabled Fact <https://github.com/chatmail/relay/pull/841>
+* Introduce support for self-signed chatmail relays <https://github.com/chatmail/relay/pull/855>
+* Strip Received headers before delivery <https://github.com/chatmail/relay/pull/849>
+* upgrade to filtermail v0.3 <https://github.com/chatmail/relay/pull/850>
+* fix link to Maddy and update madmail URL <https://github.com/chatmail/relay/pull/847>
+* accept self-signed certificates for IP-only relays <https://github.com/chatmail/relay/pull/846>
+* enforce sending from public IP addresses <https://github.com/chatmail/relay/pull/845>
+* port check: check addresses, fix single services <https://github.com/chatmail/relay/pull/844>
+* remediates issue with improper concat on resolver injection <https://github.com/chatmail/relay/pull/834>
+* ipv6 boolean not being respected during operations <https://github.com/chatmail/relay/pull/832>
+* upgrade to filtermail v0.2 by <https://github.com/chatmail/relay/pull/825>
+* fix link to filtermail <https://github.com/chatmail/relay/pull/824>
+* print timestamps when sending messages <https://github.com/chatmail/relay/pull/823>
+* fix flaky test_exceed_rate_limit <https://github.com/chatmail/relay/pull/822>
+* Replace filtermail with rust reimplementation <https://github.com/chatmail/relay/pull/808>
+* Set default internal SMTP ports in Config <https://github.com/chatmail/relay/pull/819>
+* separate metrics for incoming and outgoing messages <https://github.com/chatmail/relay/pull/820>
+* disable appending the Received header <https://github.com/chatmail/relay/pull/815>
+* fail on errors in postfix/dovecot config <https://github.com/chatmail/relay/pull/813>
+* tweak idle/hibernate metrics some more <https://github.com/chatmail/relay/pull/811>
+* add config flag to export statistics <https://github.com/chatmail/relay/pull/806>
+* add --website-only option to run subcommand <https://github.com/chatmail/relay/pull/768>
+* Strip DKIM-Signature header before LMTP <https://github.com/chatmail/relay/pull/803>
+* properly make sure that postfix gets restarted on failure <https://github.com/chatmail/relay/pull/802>
+* expire.py: use absolute path to maildirsize <https://github.com/chatmail/relay/pull/807>
+* pin Dovecot documentation URLs to version 2.3 <https://github.com/chatmail/relay/pull/800>
+* try to use "build machine" and "deployment server"  consistently  <https://github.com/chatmail/relay/pull/797>
+* adds instructions for migrating control machines <https://github.com/chatmail/relay/pull/795>
+* use consistent naming schema in getting started <https://github.com/chatmail/relay/pull/793>
+* remove jsok/serialize-workflow-action dependency <https://github.com/chatmail/relay/pull/790>
+* streamline migration guide wording, provide titled steps  <https://github.com/chatmail/relay/pull/789>
+* increases default max mailbox size <https://github.com/chatmail/relay/pull/792>
+* use daemon_name for OpenDKIM sign-verify decision instead of IP <https://github.com/chatmail/relay/pull/784>
+
 ## 1.9.0 2025-12-18
 
 ### Documentation

chatmaild/pyproject.toml

@@ -6,10 +6,7 @@ build-backend = "setuptools.build_meta"
 name = "chatmaild"
 version = "0.3"
 dependencies = [
-  "aiosmtpd",
   "iniconfig",
-  "deltachat-rpc-server",
-  "deltachat-rpc-client",
   "filelock",
   "requests",
   "crypt-r >= 3.13.1 ; python_version >= '3.11'",
@@ -24,8 +21,10 @@ where = ['src']
 [project.scripts]
 doveauth = "chatmaild.doveauth:main"
 chatmail-metadata = "chatmaild.metadata:main"
-chatmail-expire = "chatmaild.expire:main"
+chatmail-expire = "chatmaild.expire:daily_expire_main"
+chatmail-quota-expire = "chatmaild.expire:quota_expire_main"
 chatmail-fsreport = "chatmaild.fsreport:main"
+chatmail-deferred = "chatmaild.deferred:main"
 lastlogin = "chatmaild.lastlogin:main"
 turnserver = "chatmaild.turnserver:main"
 
@@ -70,6 +69,7 @@ commands =
 deps = pytest 
        pdbpp 
        pytest-localserver
+       aiosmtpd
        execnet
 commands = pytest -v -rsXx {posargs}
 """

chatmaild/src/chatmaild/config.py

@@ -1,4 +1,3 @@
-import os
 from pathlib import Path
 
 import iniconfig
@@ -38,14 +37,18 @@ class Config:
         self.filtermail_smtp_port_incoming = int(
             params.get("filtermail_smtp_port_incoming", "10081")
         )
+        self.filtermail_http_port_incoming = int(
+            params.get("filtermail_http_port_incoming", "10082")
+        )
+        self.filtermail_lmtp_port_transport = int(
+            params.get("filtermail_lmtp_port_transport", "10083")
+        )
         self.postfix_reinject_port = int(params.get("postfix_reinject_port", "10025"))
         self.postfix_reinject_port_incoming = int(
             params.get("postfix_reinject_port_incoming", "10026")
         )
         self.mtail_address = params.get("mtail_address")
         self.disable_ipv6 = params.get("disable_ipv6", "false").lower() == "true"
-        self.addr_v4 = os.environ.get("CHATMAIL_ADDR_V4", "")
-        self.addr_v6 = os.environ.get("CHATMAIL_ADDR_V6", "")
         self.acme_email = params.get("acme_email", "")
         self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true"
         self.imap_compress = params.get("imap_compress", "false").lower() == "true"
@@ -92,6 +95,11 @@ class Config:
         # old unused option (except for first migration from sqlite to maildir store)
         self.passdb_path = Path(params.get("passdb_path", "/home/vmail/passdb.sqlite"))
 
+    @property
+    def max_mailbox_size_mb(self):
+        """Return max_mailbox_size as an integer in megabytes."""
+        return parse_size_mb(self.max_mailbox_size)
+
     def _getbytefile(self):
         return open(self._inipath, "rb")
 
@@ -105,6 +113,16 @@ class Config:
         return User(maildir, addr, password_path, uid="vmail", gid="vmail")
 
 
+def parse_size_mb(limit):
+    """Parse a size string like ``500M`` or ``2G`` and return megabytes."""
+    value = limit.strip().upper().removesuffix("B")
+    if value.endswith("G"):
+        return int(value[:-1]) * 1024
+    if value.endswith("M"):
+        return int(value[:-1])
+    return int(value)
+
+
 def write_initial_config(inipath, mail_domain, overrides):
     """Write out default config file, using the specified config value overrides."""
     content = get_default_config_content(mail_domain, **overrides)

chatmaild/src/chatmaild/deferred.py

@@ -0,0 +1,37 @@
+"""
+Analyze deferred mails and print most common failing destinations.
+
+Example:
+
+    python -m chatmaild.deferred
+"""
+
+import json
+import subprocess
+from collections import Counter, defaultdict
+
+
+def main():
+    p = subprocess.Popen(["postqueue", "-j"], text=True, stdout=subprocess.PIPE)
+    domain_reasons = defaultdict(Counter)
+    domain_total = Counter()
+
+    for line in p.stdout:
+        item = json.loads(line)
+        if item["queue_name"] != "deferred":
+            continue
+
+        for recipient in item["recipients"]:
+            _, domain = recipient["address"].rsplit("@", 1)
+            reason = recipient["delay_reason"]
+            domain_total[domain] += 1
+            domain_reasons[domain][reason] += 1
+
+    for domain, total in reversed(domain_total.most_common()):
+        print(f"{domain} ({total} recipients)")
+        for reason, count in domain_reasons[domain].most_common():
+            print(f"  {count}: {reason}")
+
+
+if __name__ == "__main__":
+    main()

chatmaild/src/chatmaild/expire.py

@@ -4,17 +4,26 @@ Expire old messages and addresses.
 """
 
 import os
+import re
 import shutil
 import sys
 import time
 from argparse import ArgumentParser
 from collections import namedtuple
 from datetime import datetime
+from pathlib import Path
 from stat import S_ISREG
 
 from chatmaild.config import read_config
 
 FileEntry = namedtuple("FileEntry", ("path", "mtime", "size"))
+QuotaFileEntry = namedtuple("QuotaFileEntry", ("mtime", "quota_size", "path"))
+
+# Quota cleanup factor of max_mailbox_size. The mailbox is reset to this size.
+QUOTA_CLEANUP_FACTOR = 0.7
+
+# e.g. "cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S"
+_dovecot_fn_rex = re.compile(r".+/(\d+)\..+,S=(\d+)")
 
 
 def iter_mailboxes(basedir, maxnum):
@@ -74,6 +83,42 @@ class MailboxStat:
         self.extrafiles.sort(key=lambda x: -x.size)
 
 
+def parse_dovecot_filename(relpath):
+    m = _dovecot_fn_rex.match(relpath)
+    if not m:
+        return None
+    return QuotaFileEntry(int(m.group(1)), int(m.group(2)), relpath)
+
+
+def scan_mailbox_messages(mbox):
+    messages = []
+    for sub in ("cur", "new"):
+        for name in os_listdir_if_exists(mbox / sub):
+            if entry := parse_dovecot_filename(f"{sub}/{name}"):
+                messages.append(entry)
+    return messages
+
+
+def expire_to_target(mbox, target_bytes):
+    messages = scan_mailbox_messages(mbox)
+    total_size = sum(m.quota_size for m in messages)
+    # Keep recent 24 hours of messages protected from expiry because
+    # likely something is wrong with interactions on that address
+    # and quota-full signal can help the address owner's device to notice it
+    undeletable_messages_cutoff = time.time() - (3600 * 24)
+    removed = 0
+    for entry in sorted(messages):
+        if total_size <= target_bytes:
+            break
+        if entry.mtime > undeletable_messages_cutoff:
+            break
+        (mbox / entry.path).unlink(missing_ok=True)
+        total_size -= entry.quota_size
+        removed += 1
+
+    return removed
+
+
 def print_info(msg):
     print(msg, file=sys.stderr)
 
@@ -143,6 +188,19 @@ class Expiry:
             else:
                 continue
             changed = True
+
+        target_bytes = (
+            self.config.max_mailbox_size_mb * 1024 * 1024 * QUOTA_CLEANUP_FACTOR
+        )
+        removed = expire_to_target(Path(mbox.basedir), target_bytes)
+        if removed:
+            changed = True
+            self.del_files += removed
+            if self.verbose:
+                print_info(
+                    f"quota-expire: removed {removed} message(s) from {mboxname}"
+                )
+
         if changed:
             self.remove_file(f"{mbox.basedir}/maildirsize")
 
@@ -154,9 +212,9 @@ class Expiry:
         )
 
 
-def main(args=None):
+def daily_expire_main(args=None):
     """Expire mailboxes and messages according to chatmail config"""
-    parser = ArgumentParser(description=main.__doc__)
+    parser = ArgumentParser(description=daily_expire_main.__doc__)
     ini = "/usr/local/lib/chatmaild/chatmail.ini"
     parser.add_argument(
         "chatmail_ini",
@@ -202,5 +260,33 @@ def main(args=None):
     print(exp.get_summary())
 
 
-if __name__ == "__main__":
-    main(sys.argv[1:])
+def quota_expire_main(args=None):
+    """Remove mailbox messages to stay within a megabyte target.
+
+    This entry point is called by dovecot when a quota threshold is passed.
+    """
+
+    parser = ArgumentParser(description=quota_expire_main.__doc__)
+    parser.add_argument(
+        "target_mb",
+        type=int,
+        help="target mailbox size in megabytes",
+    )
+    parser.add_argument(
+        "mailbox_path",
+        type=Path,
+        help="path to a user mailbox",
+    )
+    args = parser.parse_args(args)
+
+    target_bytes = args.target_mb * 1024 * 1024
+
+    removed_count = expire_to_target(args.mailbox_path, target_bytes)
+    if removed_count:
+        (args.mailbox_path / "maildirsize").unlink(missing_ok=True)
+        print(
+            f"quota-expire: removed {removed_count} message(s)"
+            f" from {args.mailbox_path.name}",
+            file=sys.stderr,
+        )
+    return 0

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -18,6 +18,7 @@ max_user_send_per_minute = 60
 max_user_send_burst_size = 10
 
 # maximum mailbox size of a chatmail address
+# Oldest messages will be removed automatically, so mailboxes never run full.
 max_mailbox_size = 500M
 
 # maximum message size for an e-mail in bytes

chatmaild/src/chatmaild/newemail.py

@@ -2,6 +2,7 @@
 
 """CGI script for creating new accounts."""
 
+import ipaddress
 import json
 import secrets
 import string
@@ -14,6 +15,16 @@ ALPHANUMERIC = string.ascii_lowercase + string.digits
 ALPHANUMERIC_PUNCT = string.ascii_letters + string.digits + string.punctuation
 
 
+def wrap_ip(host):
+    if host.startswith("[") and host.endswith("]"):
+        return host
+    try:
+        ipaddress.ip_address(host)
+        return f"[{host}]"
+    except ValueError:
+        return host
+
+
 def create_newemail_dict(config: Config):
     user = "".join(
         secrets.choice(ALPHANUMERIC) for _ in range(config.username_max_length)
@@ -22,7 +33,7 @@ def create_newemail_dict(config: Config):
         secrets.choice(ALPHANUMERIC_PUNCT)
         for _ in range(config.password_min_length + 3)
     )
-    return dict(email=f"{user}@{config.mail_domain}", password=f"{password}")
+    return dict(email=f"{user}@{wrap_ip(config.mail_domain)}", password=f"{password}")
 
 
 def create_dclogin_url(email, password):

chatmaild/src/chatmaild/tests/test_config.py

@@ -1,6 +1,6 @@
 import pytest
 
-from chatmaild.config import read_config
+from chatmaild.config import parse_size_mb, read_config
 
 
 def test_read_config_basic(example_config):
@@ -121,3 +121,17 @@ def test_config_tls_external_bad_format(make_config):
                 "tls_external_cert_and_key": "/only/one/path.pem",
             },
         )
+
+
+def test_parse_size_mb():
+    assert parse_size_mb("500M") == 500
+    assert parse_size_mb("2G") == 2048
+    assert parse_size_mb("  1g  ") == 1024
+    assert parse_size_mb("100MB") == 100
+    assert parse_size_mb("256") == 256
+
+
+def test_max_mailbox_size_mb(make_config):
+    config = make_config("chat.example.org")
+    assert config.max_mailbox_size == "500M"
+    assert config.max_mailbox_size_mb == 500

chatmaild/src/chatmaild/tests/test_delete_inactive_users.py

@@ -1,7 +1,7 @@
 import time
 
 from chatmaild.doveauth import AuthDictProxy
-from chatmaild.expire import main as main_expire
+from chatmaild.expire import daily_expire_main as main_expire
 
 
 def test_login_timestamps(example_config):

chatmaild/src/chatmaild/tests/test_expire.py

@@ -1,5 +1,7 @@
+import itertools
 import os
 import random
+import time
 from datetime import datetime
 from fnmatch import fnmatch
 from pathlib import Path
@@ -9,13 +11,19 @@ import pytest
 from chatmaild.expire import (
     FileEntry,
     MailboxStat,
+    expire_to_target,
     get_file_entry,
     iter_mailboxes,
     os_listdir_if_exists,
+    parse_dovecot_filename,
+    quota_expire_main,
+    scan_mailbox_messages,
 )
-from chatmaild.expire import main as expiry_main
+from chatmaild.expire import daily_expire_main as expiry_main
 from chatmaild.fsreport import main as report_main
 
+MB = 1024 * 1024
+
 
 def fill_mbox(folderdir):
     password = folderdir.joinpath("password")
@@ -196,3 +204,51 @@ def test_os_listdir_if_exists(tmp_path):
     tmp_path.joinpath("x").write_text("hello")
     assert len(os_listdir_if_exists(str(tmp_path))) == 1
     assert len(os_listdir_if_exists(str(tmp_path.joinpath("123123")))) == 0
+
+
+# --- quota expire tests ---
+
+_msg_counter = itertools.count(1)
+
+
+def _create_message(basedir, sub, size, days_old=0, disk_size=None):
+    seq = next(_msg_counter)
+    mtime = int(time.time() - days_old * 86400)
+    name = f"{mtime}.M1P1Q{seq}.hostname,S={size},W={size}:2,S"
+    path = basedir / sub / name
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_bytes(b"x" * (disk_size if disk_size is not None else size))
+    os.utime(path, (mtime, mtime))
+    return path
+
+
+def test_parse_dovecot_filename():
+    e = parse_dovecot_filename("cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S")
+    assert e.path == "cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S"
+    assert e.mtime == 1775324677
+    assert e.quota_size == 3235
+    assert parse_dovecot_filename("cur/msg_without_structure") is None
+
+
+def test_expire_to_target(tmp_path):
+    _create_message(tmp_path, "cur", MB, days_old=10, disk_size=100)
+    _create_message(tmp_path, "new", MB, days_old=5)
+    _create_message(tmp_path, "cur", MB, days_old=0)  # undeletable (<1 hour)
+    assert len(scan_mailbox_messages(tmp_path)) == 3
+    # removes oldest first, uses S= size not disk size
+    removed = expire_to_target(tmp_path, MB)
+    assert removed == 2
+    msgs = scan_mailbox_messages(tmp_path)
+    assert len(msgs) == 1
+    # the surviving message is the fresh undeletable one
+    assert msgs[0].mtime > time.time() - 3600
+
+
+def test_quota_expire_main(tmp_path, capsys):
+    mbox = tmp_path / "user@example.org"
+    _create_message(mbox, "cur", 2 * MB, days_old=5)
+    (mbox / "maildirsize").write_text("x")
+    quota_expire_main([str(1), str(mbox)])
+    _, err = capsys.readouterr()
+    assert "quota-expire: removed 1 message(s) from user@example.org" in err
+    assert not (mbox / "maildirsize").exists()

chatmaild/src/chatmaild/tests/test_newmail.py

@@ -19,6 +19,12 @@ def test_create_newemail_dict(example_config):
     assert ac1["password"] != ac2["password"]
 
 
+def test_create_newemail_dict_ip(make_config):
+    config = make_config("1.2.3.4")
+    ac = create_newemail_dict(config)
+    assert ac["email"].endswith("@[1.2.3.4]")
+
+
 def test_create_dclogin_url():
     url = create_dclogin_url("user@example.org", "p@ss w+rd")
     assert url.startswith("dclogin:")

cmdeploy/pyproject.toml

@@ -10,7 +10,6 @@ dependencies = [
   "pillow",
   "qrcode",
   "markdown",
-  "pytest",
   "setuptools>=68",
   "termcolor",
   "build",
@@ -21,6 +20,7 @@ dependencies = [
   "execnet",
   "imap_tools",
   "deltachat-rpc-client",
+  "deltachat-rpc-server",
 ]
 
 [project.scripts]

cmdeploy/src/cmdeploy/basedeploy.py

@@ -1,7 +1,10 @@
 import importlib.resources
 import io
 import os
+from contextlib import contextmanager
 
+from pyinfra import host
+from pyinfra.facts.server import Command
 from pyinfra.operations import files, server, systemd
 
 
@@ -10,6 +13,39 @@ def has_systemd():
     return os.path.isdir("/run/systemd/system")
 
 
+def is_in_container() -> bool:
+    """Return True if running inside a container (Docker, LXC, etc.)."""
+    return (
+        host.get_fact(
+            Command,
+            "systemd-detect-virt --container --quiet 2>/dev/null && echo yes || true",
+        )
+        == "yes"
+    )
+
+
+@contextmanager
+def blocked_service_startup():
+    """Prevent services from auto-starting during package installation.
+
+    Installs a ``/usr/sbin/policy-rc.d`` that exits 101, blocking any
+    service from being started by the package manager.  This avoids bind
+    conflicts and CPU/RAM spikes during initial setup.  The file is removed
+    when the context exits.
+    """
+    # For documentation about policy-rc.d, see:
+    # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
+    files.put(
+        src=get_resource("policy-rc.d"),
+        dest="/usr/sbin/policy-rc.d",
+        user="root",
+        group="root",
+        mode="755",
+    )
+    yield
+    files.file("/usr/sbin/policy-rc.d", present=False)
+
+
 def get_resource(arg, pkg=__package__):
     return importlib.resources.files(pkg).joinpath(arg)
 

cmdeploy/src/cmdeploy/chatmail.zone.j2

@@ -1,32 +0,0 @@
-;
-; Required DNS entries for chatmail servers 
-;
-{% if A %}
-{{ mail_domain }}.                   A     {{ A }}
-{% endif %}
-{% if AAAA %}
-{{ mail_domain }}.                   AAAA  {{ AAAA }}
-{% endif %}
-{{ mail_domain }}.                   MX 10 {{ mail_domain }}.
-{% if strict_tls %}
-_mta-sts.{{ mail_domain }}.          TXT "v=STSv1; id={{ sts_id }}"
-mta-sts.{{ mail_domain }}.           CNAME {{ mail_domain }}.
-{% endif %}
-www.{{ mail_domain }}.               CNAME {{ mail_domain }}.
-{{ dkim_entry }}
-
-;
-; Recommended DNS entries for interoperability and security-hardening
-;
-{{ mail_domain }}.                   TXT "v=spf1 a ~all"
-_dmarc.{{ mail_domain }}.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
-
-{% if acme_account_url %}
-{{ mail_domain }}.                   CAA 0 issue "letsencrypt.org;accounturi={{ acme_account_url }}"
-{% endif %}
-_adsp._domainkey.{{ mail_domain }}.  TXT "dkim=discardable"
-
-_submission._tcp.{{ mail_domain }}.  SRV 0 1 587 {{ mail_domain }}.
-_submissions._tcp.{{ mail_domain }}. SRV 0 1 465 {{ mail_domain }}.
-_imap._tcp.{{ mail_domain }}.        SRV 0 1 143 {{ mail_domain }}.
-_imaps._tcp.{{ mail_domain }}.       SRV 0 1 993 {{ mail_domain }}.

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -101,17 +101,11 @@ def run_cmd(args, out):
     env["CHATMAIL_WEBSITE_ONLY"] = "True" if args.website_only else ""
     env["CHATMAIL_DISABLE_MAIL"] = "True" if args.disable_mail else ""
     env["CHATMAIL_REQUIRE_IROH"] = "True" if require_iroh else ""
-    if not args.dns_check_disabled:
-        env["CHATMAIL_ADDR_V4"] = remote_data.get("A") or ""
-        env["CHATMAIL_ADDR_V6"] = remote_data.get("AAAA") or ""
     deploy_path = importlib.resources.files(__package__).joinpath("run.py").resolve()
     pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
 
     cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y"
-    if ssh_host in ["localhost", "@docker"]:
-        if ssh_host == "@docker":
-            env["CHATMAIL_NOPORTCHECK"] = "True"
-            env["CHATMAIL_NOSYSCTL"] = "True"
+    if ssh_host == "localhost":
         cmd = f"{pyinf} @local {deploy_path} -y"
 
     if version.parse(pyinfra.__version__) < version.parse("3"):
@@ -197,12 +191,6 @@ def status_cmd(args, out):
 
 
 def test_cmd_options(parser):
-    parser.add_argument(
-        "--slow",
-        dest="slow",
-        action="store_true",
-        help="also run slow tests",
-    )
     add_ssh_host_option(parser)
 
 
@@ -210,6 +198,7 @@ def test_cmd(args, out):
     """Run local and online tests for chatmail deployment."""
 
     env = os.environ.copy()
+    env["CHATMAIL_INI"] = str(args.inipath.absolute())
     if args.ssh_host:
         env["CHATMAIL_SSH"] = args.ssh_host
 
@@ -223,8 +212,6 @@ def test_cmd(args, out):
         "-v",
         "--durations=5",
     ]
-    if args.slow:
-        pytest_args.append("--slow")
     ret = out.run_ret(pytest_args, env=env)
     return ret
 
@@ -316,7 +303,7 @@ def add_ssh_host_option(parser):
     parser.add_argument(
         "--ssh-host",
         dest="ssh_host",
-        help="Run commands on 'localhost', via '@docker', or on a specific SSH host "
+        help="Run commands on 'localhost' or on a specific SSH host "
         "instead of chatmail.ini's mail_domain.",
     )
 
@@ -378,9 +365,7 @@ def get_parser():
 
 def get_sshexec(ssh_host: str, verbose=True):
     if ssh_host in ["localhost", "@local"]:
-        return LocalExec(verbose, docker=False)
-    elif ssh_host == "@docker":
-        return LocalExec(verbose, docker=True)
+        return LocalExec(verbose)
     if verbose:
         print(f"[ssh] login to {ssh_host}")
     return SSHExec(ssh_host, verbose=verbose)

cmdeploy/src/cmdeploy/deployers.py

@@ -2,11 +2,10 @@
 Chat Mail pyinfra deploy.
 """
 
-import os
 import shutil
 import subprocess
 import sys
-from io import StringIO
+from io import BytesIO, StringIO
 from pathlib import Path
 
 from chatmaild.config import read_config
@@ -24,9 +23,11 @@ from .basedeploy import (
     Deployer,
     Deployment,
     activate_remote_units,
+    blocked_service_startup,
     configure_remote_units,
     get_resource,
     has_systemd,
+    is_in_container,
 )
 from .dovecot.deployer import DovecotDeployer
 from .external.deployer import ExternalTlsDeployer
@@ -149,35 +150,40 @@ class UnboundDeployer(Deployer):
         self.need_restart = False
 
     def install(self):
-        # Run local DNS resolver `unbound`.
-        # `resolvconf` takes care of setting up /etc/resolv.conf
-        # to use 127.0.0.1 as the resolver.
-
-        #
-        # On an IPv4-only system, if unbound is started but not
-        # configured, it causes subsequent steps to fail to resolve hosts.
-        # Here, we use policy-rc.d to prevent unbound from starting up
-        # on initial install.  Later, we will configure it and start it.
-        #
-        # For documentation about policy-rc.d, see:
-        # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
-        #
-        files.put(
-            src=get_resource("policy-rc.d"),
-            dest="/usr/sbin/policy-rc.d",
-            user="root",
-            group="root",
-            mode="755",
-        )
-
-        apt.packages(
-            name="Install unbound",
-            packages=["unbound", "unbound-anchor", "dnsutils"],
-        )
-
-        files.file("/usr/sbin/policy-rc.d", present=False)
+        # On an IPv4-only system, if unbound is started but not configured,
+        # it causes subsequent steps to fail to resolve hosts.
+        with blocked_service_startup():
+            apt.packages(
+                name="Install unbound",
+                packages=["unbound", "unbound-anchor", "dnsutils"],
+            )
 
     def configure(self):
+        # Remove dynamic resolver managers that compete for /etc/resolv.conf.
+        apt.packages(
+            name="Purge resolvconf",
+            packages=["resolvconf"],
+            present=False,
+            extra_uninstall_args="--purge",
+        )
+        # systemd-resolved can't be purged due to dependencies; stop and mask.
+        server.shell(
+            name="Stop and mask systemd-resolved",
+            commands=[
+                "systemctl stop systemd-resolved.service || true",
+                "systemctl mask systemd-resolved.service",
+            ],
+        )
+        # Configure unbound resolver with Quad9 fallback and a trailing newline
+        # (SolusVM bug).
+        files.put(
+            name="Write static resolv.conf",
+            src=BytesIO(b"nameserver 127.0.0.1\nnameserver 9.9.9.9\n"),
+            dest="/etc/resolv.conf",
+            user="root",
+            group="root",
+            mode="644",
+        )
         server.shell(
             name="Generate root keys for validating DNSSEC",
             commands=[
@@ -336,12 +342,12 @@ class TurnDeployer(Deployer):
     def install(self):
         (url, sha256sum) = {
             "x86_64": (
-                "https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-x86_64-linux",
-                "841e527c15fdc2940b0469e206188ea8f0af48533be12ecb8098520f813d41e4",
+                "https://github.com/chatmail/chatmail-turn/releases/download/v0.4/chatmail-turn-x86_64-linux",
+                "1ec1f5c50122165e858a5a91bcba9037a28aa8cb8b64b8db570aa457c6141a8a",
             ),
             "aarch64": (
-                "https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-aarch64-linux",
-                "a5fc2d06d937b56a34e098d2cd72a82d3e89967518d159bf246dc69b65e81b42",
+                "https://github.com/chatmail/chatmail-turn/releases/download/v0.4/chatmail-turn-aarch64-linux",
+                "0fb3e792419494e21ecad536464929dba706bb2c88884ed8f1788141d26fc756",
             ),
         }[host.get_fact(facts.server.Arch)]
 
@@ -474,10 +480,19 @@ class ChatmailDeployer(Deployer):
         ("iroh", None, None),
     ]
 
-    def __init__(self, mail_domain):
-        self.mail_domain = mail_domain
+    def __init__(self, config):
+        self.config = config
+        self.mail_domain = config.mail_domain
 
     def install(self):
+        files.put(
+            name="Disable installing recommended packages globally",
+            src=BytesIO(b'APT::Install-Recommends "false";\n'),
+            dest="/etc/apt/apt.conf.d/00InstallRecommends",
+            user="root",
+            group="root",
+            mode="644",
+        )
         apt.update(name="apt update", cache_time=24 * 3600)
         apt.upgrade(name="upgrade apt packages", auto_remove=True)
 
@@ -490,12 +505,18 @@ class ChatmailDeployer(Deployer):
             name="Install rsync",
             packages=["rsync"],
         )
-        apt.packages(
-            name="Ensure cron is installed",
-            packages=["cron"],
-        )
 
     def configure(self):
+        # metadata crashes if the mailboxes dir does not exist
+        files.directory(
+            name="Ensure vmail mailbox directory exists",
+            path=str(self.config.mailboxes_dir),
+            user="vmail",
+            group="vmail",
+            mode="700",
+            present=True,
+        )
+
         # This file is used by auth proxy.
         # https://wiki.debian.org/EtcMailName
         server.shell(
@@ -569,14 +590,6 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
         Deployment().perform_stages([WebsiteDeployer(config)])
         return
 
-    if host.get_fact(Port, port=53) != "unbound":
-        files.line(
-            name="Add 9.9.9.9 to resolv.conf",
-            path="/etc/resolv.conf",
-            # Guard against resolv.conf missing a trailing newline (SolusVM bug).
-            line="\nnameserver 9.9.9.9",
-        )
-
     # Check if mtail_address interface is available (if configured)
     if config.mtail_address and config.mtail_address not in ('127.0.0.1', '::1', 'localhost'):
         ipv4_addrs = host.get_fact(hardware.Ipv4Addrs)
@@ -585,7 +598,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
             Out().red(f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n")
             exit(1)
 
-    if not os.environ.get("CHATMAIL_NOPORTCHECK"):
+    if not is_in_container():
         port_services = [
             (["master", "smtpd"], 25),
             ("unbound", 53),
@@ -625,7 +638,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
     tls_deployer = get_tls_deployer(config, mail_domain)
 
     all_deployers = [
-        ChatmailDeployer(mail_domain),
+        ChatmailDeployer(config),
         LegacyRemoveDeployer(),
         FiltermailDeployer(),
         JournaldDeployer(),

cmdeploy/src/cmdeploy/dns.py

@@ -1,11 +1,22 @@
 import datetime
-import importlib
-
-from jinja2 import Template
 
 from . import remote
 
 
+def parse_zone_records(text):
+    """Yield ``(name, ttl, rtype, rdata)`` from standard BIND-format text."""
+    for raw_line in text.splitlines():
+        line = raw_line.strip()
+        if not line or line.startswith(";"):
+            continue
+        try:
+            name, ttl, _in, rtype, rdata = line.split(None, 4)
+        except ValueError:
+            raise ValueError(f"Bad zone record line: {line!r}") from None
+        name = name.rstrip(".")
+        yield name, ttl, rtype.upper(), rdata
+
+
 def get_initial_remote_data(sshexec, mail_domain):
     return sshexec.logged(
         call=remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=mail_domain)
@@ -31,13 +42,39 @@ def get_filled_zone_file(remote_data):
     if not sts_id:
         remote_data["sts_id"] = datetime.datetime.now().strftime("%Y%m%d%H%M")
 
-    template = importlib.resources.files(__package__).joinpath("chatmail.zone.j2")
-    content = template.read_text()
-    zonefile = Template(content).render(**remote_data)
-    lines = [x.strip() for x in zonefile.split("\n") if x.strip()]
+    d = remote_data["mail_domain"]
+
+    def append_record(name, rtype, rdata, ttl=3600):
+        lines.append(f"{name:<40} {ttl:<6} IN  {rtype:<5}  {rdata}")
+
+    lines = ["; Required DNS entries"]
+    if remote_data.get("A"):
+        append_record(f"{d}.", "A", remote_data["A"])
+    if remote_data.get("AAAA"):
+        append_record(f"{d}.", "AAAA", remote_data["AAAA"])
+    append_record(f"{d}.", "MX", f"10 {d}.")
+    if remote_data.get("strict_tls"):
+        append_record(f"_mta-sts.{d}.", "TXT", f'"v=STSv1; id={remote_data["sts_id"]}"')
+        append_record(f"mta-sts.{d}.", "CNAME", f"{d}.")
+    append_record(f"www.{d}.", "CNAME", f"{d}.")
+    lines.append(remote_data["dkim_entry"])
     lines.append("")
-    zonefile = "\n".join(lines)
-    return zonefile
+    lines.append("; Recommended DNS entries")
+    append_record(f"{d}.", "TXT", '"v=spf1 a ~all"')
+    append_record(f"_dmarc.{d}.", "TXT", '"v=DMARC1;p=reject;adkim=s;aspf=s"')
+    if remote_data.get("acme_account_url"):
+        append_record(
+            f"{d}.",
+            "CAA",
+            f'0 issue "letsencrypt.org;accounturi={remote_data["acme_account_url"]}"',
+        )
+    append_record(f"_adsp._domainkey.{d}.", "TXT", '"dkim=discardable"')
+    append_record(f"_submission._tcp.{d}.", "SRV", f"0 1 587 {d}.")
+    append_record(f"_submissions._tcp.{d}.", "SRV", f"0 1 465 {d}.")
+    append_record(f"_imap._tcp.{d}.", "SRV", f"0 1 143 {d}.")
+    append_record(f"_imaps._tcp.{d}.", "SRV", f"0 1 993 {d}.")
+    lines.append("")
+    return "\n".join(lines)
 
 
 def check_full_zone(sshexec, remote_data, out, zonefile) -> int:
@@ -58,7 +95,8 @@ def check_full_zone(sshexec, remote_data, out, zonefile) -> int:
         returncode = 1
     if remote_data.get("dkim_entry") in required_diff:
         out(
-            "If the DKIM entry above does not work with your DNS provider, you can try this one:\n"
+            "If the DKIM entry above does not work with your DNS provider,"
+            " you can try this one:\n"
         )
         out(remote_data.get("web_dkim_entry") + "\n")
     if recommended_diff:

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -1,20 +1,33 @@
-import os
+import io
 import urllib.request
 
 from chatmaild.config import Config
 from pyinfra import host
-from pyinfra.facts.server import Arch, Sysctl
-from pyinfra.facts.systemd import SystemdEnabled
+from pyinfra.facts.deb import DebPackages
+from pyinfra.facts.server import Arch, Command, Sysctl
 from pyinfra.operations import apt, files, server, systemd
 
 from cmdeploy.basedeploy import (
     Deployer,
     activate_remote_units,
+    blocked_service_startup,
     configure_remote_units,
     get_resource,
-    has_systemd,
+    is_in_container,
 )
 
+DOVECOT_ARCHIVE_VERSION = "2.3.21+dfsg1-3"
+DOVECOT_PACKAGE_VERSION = f"1:{DOVECOT_ARCHIVE_VERSION}"
+
+DOVECOT_SHA256 = {
+    ("core", "amd64"): "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d",
+    ("core", "arm64"): "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9",
+    ("imapd", "amd64"): "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86",
+    ("imapd", "arm64"): "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f",
+    ("lmtpd", "amd64"): "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab",
+    ("lmtpd", "arm64"): "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f",
+}
+
 
 class DovecotDeployer(Deployer):
     daemon_reload = False
@@ -26,19 +39,58 @@ class DovecotDeployer(Deployer):
 
     def install(self):
         arch = host.get_fact(Arch)
-        if has_systemd() and "dovecot.service" in host.get_fact(SystemdEnabled):
-            return  # already installed and running
-        _install_dovecot_package("core", arch)
-        _install_dovecot_package("imapd", arch)
-        _install_dovecot_package("lmtpd", arch)
+        with blocked_service_startup():
+            debs = []
+            for pkg in ("core", "imapd", "lmtpd"):
+                deb, changed = _download_dovecot_package(pkg, arch)
+                self.need_restart |= changed
+                if deb:
+                    debs.append(deb)
+            if debs:
+                deb_list = " ".join(debs)
+                # First dpkg may fail on missing dependencies (stderr suppressed);
+                # apt-get --fix-broken pulls them in, then dpkg retries cleanly.
+                server.shell(
+                    name="Install dovecot packages",
+                    commands=[
+                        f"dpkg --force-confdef --force-confold -i {deb_list} 2> /dev/null || true",
+                        "DEBIAN_FRONTEND=noninteractive apt-get -y --fix-broken install",
+                        f"dpkg --force-confdef --force-confold -i {deb_list}",
+                    ],
+                )
+                self.need_restart = True
+        files.put(
+            name="Pin dovecot packages to block Debian dist-upgrades",
+            src=io.StringIO(
+                "Package: dovecot-*\n"
+                "Pin: version *\n"
+                "Pin-Priority: -1\n"
+            ),
+            dest="/etc/apt/preferences.d/pin-dovecot",
+            user="root",
+            group="root",
+            mode="644",
+        )
 
     def configure(self):
         configure_remote_units(self.config.mail_domain, self.units)
-        self.need_restart, self.daemon_reload = _configure_dovecot(self.config)
+        config_restart, self.daemon_reload = _configure_dovecot(self.config)
+        self.need_restart |= config_restart
 
     def activate(self):
         activate_remote_units(self.units)
 
+        # Detect stale binary: package installed but service still runs old (deleted) binary.
+        if not self.disable_mail and not self.need_restart:
+            stale = host.get_fact(
+                Command,
+                'pid=$(systemctl show -p MainPID --value dovecot.service 2>/dev/null);'
+                ' [ "${pid:-0}" != "0" ] && readlink "/proc/$pid/exe" 2>/dev/null | grep -q "(deleted)"'
+                " && echo STALE || true",
+            )
+            if stale == "STALE":
+                self.need_restart = True
+
         restart = False if self.disable_mail else self.need_restart
 
         systemd.service(
@@ -63,43 +115,40 @@ def _pick_url(primary, fallback):
         return fallback
 
 
-def _install_dovecot_package(package: str, arch: str):
+def _download_dovecot_package(package: str, arch: str) -> tuple[str | None, bool]:
+    """Download a dovecot .deb if needed, return (path, changed)."""
     arch = "amd64" if arch == "x86_64" else arch
     arch = "arm64" if arch == "aarch64" else arch
-    primary_url = f"https://download.delta.chat/dovecot/dovecot-{package}_2.3.21%2Bdfsg1-3_{arch}.deb"
-    fallback_url = f"https://github.com/chatmail/dovecot/releases/download/upstream%2F2.3.21%2Bdfsg1/dovecot-{package}_2.3.21%2Bdfsg1-3_{arch}.deb"
-    url = _pick_url(primary_url, fallback_url)
-    deb_filename = "/root/" + url.split("/")[-1]
 
-    match (package, arch):
-        case ("core", "amd64"):
-            sha256 = "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d"
-        case ("core", "arm64"):
-            sha256 = "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9"
-        case ("imapd", "amd64"):
-            sha256 = "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86"
-        case ("imapd", "arm64"):
-            sha256 = "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f"
-        case ("lmtpd", "amd64"):
-            sha256 = "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab"
-        case ("lmtpd", "arm64"):
-            sha256 = "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f"
-        case _:
-            apt.packages(packages=[f"dovecot-{package}"])
-            return
+    pkg_name = f"dovecot-{package}"
+    sha256 = DOVECOT_SHA256.get((package, arch))
+    if sha256 is None:
+        op = apt.packages(packages=[pkg_name])
+        return None, bool(getattr(op, "changed", False))
+
+    installed_versions = host.get_fact(DebPackages).get(pkg_name, [])
+    if DOVECOT_PACKAGE_VERSION in installed_versions:
+        return None, False
+
+    url_version = DOVECOT_ARCHIVE_VERSION.replace("+", "%2B")
+    deb_base = f"{pkg_name}_{url_version}_{arch}.deb"
+    primary_url = f"https://download.delta.chat/dovecot/{deb_base}"
+    fallback_url = f"https://github.com/chatmail/dovecot/releases/download/upstream%2F{url_version}/{deb_base}"
+    url = _pick_url(primary_url, fallback_url)
+    deb_filename = f"/root/{deb_base}"
 
     files.download(
-        name=f"Download dovecot-{package}",
+        name=f"Download {pkg_name}",
         src=url,
         dest=deb_filename,
         sha256sum=sha256,
         cache_time=60 * 60 * 24 * 365 * 10,  # never redownload the package
     )
 
-    apt.deb(name=f"Install dovecot-{package}", src=deb_filename)
+    return deb_filename, True
 
 
-def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
+def _configure_dovecot(config: Config, debug: bool = False) -> tuple[bool, bool]:
     """Configures Dovecot IMAP server."""
     need_restart = False
     daemon_reload = False
@@ -134,19 +183,25 @@ def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
 
     # as per https://doc.dovecot.org/2.3/configuration_manual/os/
     # it is recommended to set the following inotify limits
-    if not os.environ.get("CHATMAIL_NOSYSCTL"):
-        for name in ("max_user_instances", "max_user_watches"):
-            key = f"fs.inotify.{name}"
-            if host.get_fact(Sysctl)[key] > 65535:
-                # Skip updating limits if already sufficient
-                # (enables running in incus containers where sysctl readonly)
-                continue
-            server.sysctl(
-                name=f"Change {key}",
-                key=key,
-                value=65535,
-                persist=True,
+    can_modify = not is_in_container()
+    for name in ("max_user_instances", "max_user_watches"):
+        key = f"fs.inotify.{name}"
+        value = host.get_fact(Sysctl).get(key, 0)
+        if value > 65534:
+            continue
+        if not can_modify:
+            print(
+                "\n!!!! refusing to attempt sysctl setting in containers\n"
+                f"!!!! dovecot: sysctl {key!r}={value}, should be >65534 for production setups\n"
+                "!!!!"
             )
+            continue
+        server.sysctl(
+            name=f"Change {key}",
+            key=key,
+            value=65535,
+            persist=True,
+        )
 
     timezone_env = files.line(
         name="Set TZ environment variable",

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -133,6 +133,11 @@ protocol lmtp {
   # mail_lua and push_notification_lua are needed for Lua push notification handler.
   # <https://doc.dovecot.org/2.3/configuration_manual/push_notification/#configuration>
   mail_plugins = $mail_plugins mail_lua notify push_notification push_notification_lua
+
+  # Disable fsync for LMTP. May lose delivered message,
+  # but unlikely to cause problems with multiple relays.
+  # https://doc.dovecot.org/2.3/admin_manual/mailbox_formats/#fsyncing
+  mail_fsync = never
 }
 
 plugin {
@@ -144,12 +149,26 @@ plugin {
 }
 
 plugin {
-  # for now we define static quota-rules for all users 
   quota = maildir:User quota
-  quota_rule = *:storage={{ config.max_mailbox_size }}
   quota_max_mail_size={{ config.max_message_size }}
   quota_grace = 0
-  # quota_over_flag_value = TRUE
+
+  quota_rule = *:storage={{ config.max_mailbox_size_mb }}M
+
+  # Trigger at 75%% of quota, expire oldest messages down to 70%%.
+  # The percentages are chosen to prevent current Delta Chat users
+  # from seeing "quota warnings" which trigger at 80% and 95%.
+
+  quota_warning = storage=75%% quota-warning {{ config.max_mailbox_size_mb * 70 // 100 }} {{ config.mailboxes_dir }}/%u
+}
+
+service quota-warning {
+  executable = script /usr/local/lib/chatmaild/venv/bin/chatmail-quota-expire
+  user = vmail
+  unix_listener quota-warning {
+     user = vmail
+     mode = 0600
+  }
 }
 
 # push_notification configuration
@@ -252,6 +271,9 @@ protocol imap {
   # sort -sn <(sed 's/ / C: /' *.in) <(sed 's/ / S: /' cat *.out)
 
   rawlog_dir = %h 
+
+  # Disable fsync for IMAP. May lose IMAP changes like setting flags. 
+  mail_fsync = never
 } 
 {% endif %}
 

cmdeploy/src/cmdeploy/filtermail/deployer.py

@@ -1,3 +1,5 @@
+import os
+
 from pyinfra import facts, host
 from pyinfra.operations import files, systemd
 
@@ -5,7 +7,7 @@ from cmdeploy.basedeploy import Deployer, get_resource
 
 
 class FiltermailDeployer(Deployer):
-    services = ["filtermail", "filtermail-incoming"]
+    services = ["filtermail", "filtermail-incoming", "filtermail-transport"]
     bin_path = "/usr/local/bin/filtermail"
     config_path = "/usr/local/lib/chatmaild/chatmail.ini"
 
@@ -13,11 +15,21 @@ class FiltermailDeployer(Deployer):
         self.need_restart = False
 
     def install(self):
+        local_bin = os.environ.get("CHATMAIL_FILTERMAIL_BINARY")
+        if local_bin:
+            self.need_restart |= files.put(
+                name="Upload locally built filtermail",
+                src=local_bin,
+                dest=self.bin_path,
+                mode="755",
+            ).changed
+            return
+
         arch = host.get_fact(facts.server.Arch)
-        url = f"https://github.com/chatmail/filtermail/releases/download/v0.5.2/filtermail-{arch}"
+        url = f"https://github.com/chatmail/filtermail/releases/download/v0.6.4/filtermail-{arch}"
         sha256sum = {
-            "x86_64": "ce24ca0075aa445510291d775fb3aea8f4411818c7b885ae51a0fe18c5f789ce",
-            "aarch64": "c5d783eefa5332db3d97a0e6a23917d72849e3eb45da3d16ce908a9b4e5a797d",
+            "x86_64": "5295115952c72e4c4ec3c85546e094b4155a4c702c82bd71fcdcb744dc73adf6",
+            "aarch64": "6892244f17b8f26ccb465766e96028e7222b3c8adefca9fc6bfe9ff332ca8dff",
         }[arch]
         self.need_restart |= files.download(
             name="Download filtermail",

cmdeploy/src/cmdeploy/filtermail/filtermail-transport.service.j2

@@ -0,0 +1,11 @@
+[Unit]
+Description=Chatmail transport service
+
+[Service]
+ExecStart={{ bin_path }} {{ config_path }} transport
+Restart=always
+RestartSec=30
+User=vmail
+
+[Install]
+WantedBy=multi-user.target

cmdeploy/src/cmdeploy/mtail/delivered_mail.mtail

@@ -78,3 +78,11 @@ counter rejected_unencrypted_mail_count
 /Rejected unencrypted mail/ {
   rejected_unencrypted_mail_count++
 }
+
+counter quota_expire_runs
+counter quota_expire_removed_files
+
+/quota-expire: removed (?P<count>\d+) message\(s\)/ {
+  quota_expire_runs++
+  quota_expire_removed_files += $count
+}

cmdeploy/src/cmdeploy/mtail/mtail.service.j2

@@ -1,5 +1,6 @@
 [Unit]
 Description=mtail
+After=multi-user.target
 
 [Service]
 Type=simple

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -73,6 +73,10 @@ http {
 
 		access_log syslog:server=unix:/dev/log,facility=local7;
 
+		location /mxdeliv {
+		    proxy_pass http://127.0.0.1:{{ config.filtermail_http_port_incoming }};
+		}
+
 		location / {
 			# First attempt to serve request as file, then
 			# as directory, then fall back to displaying a 404.

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -20,7 +20,7 @@ smtpd_tls_key_file={{ config.tls_key_path }}
 smtpd_tls_security_level=may
 
 smtp_tls_CApath=/etc/ssl/certs
-smtp_tls_security_level={{ "verify" if config.tls_cert_mode == "acme" else "encrypt" }}
+smtp_tls_security_level=verify
 # Send SNI extension when connecting to other servers.
 # <https://www.postfix.org/postconf.5.html#smtp_tls_servername>
 smtp_tls_servername = hostname
@@ -69,15 +69,6 @@ mynetworks = 127.0.0.0/8
 {% else %}
 mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128
 {% endif %}
-{% if config.addr_v4 %}
-smtp_bind_address = {{ config.addr_v4 }}
-{% endif %}
-{% if config.addr_v6 %}
-smtp_bind_address6 = {{ config.addr_v6 }}
-{% endif %}
-{% if config.addr_v4 or config.addr_v6 %}
-smtp_bind_address_enforce = yes
-{% endif %}
 mailbox_size_limit = 0
 message_size_limit = {{config.max_message_size}}
 recipient_delimiter = +
@@ -102,3 +93,10 @@ smtpd_sender_login_maps = regexp:/etc/postfix/login_map
 # Do not lookup SMTP client hostnames to reduce delays
 # and avoid unnecessary DNS requests.
 smtpd_peername_lookup = no
+
+# Use filtermail-transport to relay messages.
+# We can't force postfix to split messages per destination,
+# when specifying a custom next-hop,
+# so instead this is handled in filtermail.
+# We use LMTP instead SMTP so we can communicate per-recipient errors back to postfix.
+default_transport = lmtp-filtermail:inet:[127.0.0.1]:{{ config.filtermail_lmtp_port_transport }}

cmdeploy/src/cmdeploy/postfix/master.cf.j2

@@ -100,3 +100,8 @@ filter    unix -        n       n       -       -       lmtp
 # cannot send unprotected Subject.
 authclean unix  n       -       -       -       0       cleanup
   -o header_checks=regexp:/etc/postfix/submission_header_cleanup
+
+lmtp-filtermail unix  -       -       y       -       -       lmtp
+  -o syslog_name=postfix/lmtp-filtermail
+  -o lmtp_header_checks=
+  -o lmtp_tls_security_level=none

cmdeploy/src/cmdeploy/remote/rdns.py

@@ -57,9 +57,10 @@ def get_dkim_entry(mail_domain, pre_command, dkim_selector):
     dkim_value_raw = f"v=DKIM1;k=rsa;p={dkim_pubkey};s=email;t=s"
     dkim_value = '" "'.join(re.findall(".{1,255}", dkim_value_raw))
     web_dkim_value = "".join(re.findall(".{1,255}", dkim_value_raw))
+    name = f"{dkim_selector}._domainkey.{mail_domain}."
     return (
-        f'{dkim_selector}._domainkey.{mail_domain}. TXT "{dkim_value}"',
-        f'{dkim_selector}._domainkey.{mail_domain}. TXT "{web_dkim_value}"',
+        f'{name:<40} 3600   IN  TXT    "{dkim_value}"',
+        f'{name:<40} 3600   IN  TXT    "{web_dkim_value}"',
     )
 
 
@@ -94,7 +95,7 @@ def check_zonefile(zonefile, verbose=True):
         if not zf_line.strip() or zf_line.startswith(";"):
             continue
         print(f"dns-checking {zf_line!r}") if verbose else log_progress("")
-        zf_domain, zf_typ, zf_value = zf_line.split(maxsplit=2)
+        zf_domain, _ttl, _in, zf_typ, zf_value = zf_line.split(None, 4)
         zf_domain = zf_domain.rstrip(".")
         zf_value = zf_value.strip()
         query_value = query_dns(zf_typ, zf_domain)

cmdeploy/src/cmdeploy/selfsigned/deployer.py

@@ -18,6 +18,8 @@ def openssl_selfsigned_args(domain, cert_path, key_path, days=36500):
         "-keyout", str(key_path),
         "-out", str(cert_path),
         "-subj", f"/CN={domain}",
+        # Mark as end-entity cert so it cannot be used as a CA to sign others.
+        "-addext", "basicConstraints=critical,CA:FALSE",
         "-addext", "extendedKeyUsage=serverAuth,clientAuth",
         "-addext",
         f"subjectAltName=DNS:{domain},DNS:www.{domain},DNS:mta-sts.{domain}",

cmdeploy/src/cmdeploy/sshexec.py

@@ -87,9 +87,8 @@ class SSHExec:
 class LocalExec:
     FuncError = FuncError
 
-    def __init__(self, verbose=False, docker=False):
+    def __init__(self, verbose=False):
         self.verbose = verbose
-        self.docker = docker
 
     def __call__(self, call, kwargs=None, log_callback=None):
         if kwargs is None:
@@ -101,10 +100,6 @@ class LocalExec:
         if not title:
             title = call.__name__
         where = "locally"
-        if self.docker:
-            if call == remote.rdns.perform_initial_checks:
-                kwargs["pre_command"] = "docker exec chatmail "
-                where = "in docker"
         if self.verbose:
             print_stderr(f"Running {where}: {title}(**{kwargs})")
             return self(call, kwargs, log_callback=print_stderr)

cmdeploy/src/cmdeploy/tests/data/zftest.zone

@@ -1,17 +1,18 @@
-; Required DNS entries for chatmail servers
-zftest.testrun.org.                   A     135.181.204.127
-zftest.testrun.org.                   AAAA  2a01:4f9:c012:52f4::1
-zftest.testrun.org.                   MX 10 zftest.testrun.org.
-_mta-sts.zftest.testrun.org.          TXT "v=STSv1; id=202403211706"
-mta-sts.zftest.testrun.org.           CNAME zftest.testrun.org.
-www.zftest.testrun.org.               CNAME zftest.testrun.org.
-opendkim._domainkey.zftest.testrun.org. TXT "v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoYt82CVUyz2ouaqjX2kB+5J80knAyoOU3MGU5aWppmwUwwTvj/oSTSpkc5JMtVTRmKKr8NUDWAL1Yw7dfGqqPHdHfwwjS3BIvDzYx+hzgtz62RnfNgV+/2MAoNpfX7cAFIHdRzEHNtwugc3RDLquqPoupAE3Y2YRw2T5zG5fILh4vwIcJZL5Uq6B92j8wwJqOex" "33n+vm1NKQ9rxo/UsHAmZlJzpooXcG/4igTBxJyJlamVSRR6N7Nul1v//YJb7J6v2o0iPHW6uE0StzKaPPNC2IVosSRFbD9H2oqppltptFSNPlI0E+t0JBWHem6YK7xcugiO3ImMCaaU8g6Jt/wIDAQAB;s=email;t=s"
+; Required DNS entries
+zftest.testrun.org.                      3600   IN  A      135.181.204.127
+zftest.testrun.org.                      3600   IN  AAAA   2a01:4f9:c012:52f4::1
+zftest.testrun.org.                      3600   IN  MX     10 zftest.testrun.org.
+_mta-sts.zftest.testrun.org.             3600   IN  TXT    "v=STSv1; id=202403211706"
+mta-sts.zftest.testrun.org.              3600   IN  CNAME  zftest.testrun.org.
+www.zftest.testrun.org.                  3600   IN  CNAME  zftest.testrun.org.
+opendkim._domainkey.zftest.testrun.org.  3600   IN  TXT    "v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoYt82CVUyz2ouaqjX2kB+5J80knAyoOU3MGU5aWppmwUwwTvj/oSTSpkc5JMtVTRmKKr8NUDWAL1Yw7dfGqqPHdHfwwjS3BIvDzYx+hzgtz62RnfNgV+/2MAoNpfX7cAFIHdRzEHNtwugc3RDLquqPoupAE3Y2YRw2T5zG5fILh4vwIcJZL5Uq6B92j8wwJqOex" "33n+vm1NKQ9rxo/UsHAmZlJzpooXcG/4igTBxJyJlamVSRR6N7Nul1v//YJb7J6v2o0iPHW6uE0StzKaPPNC2IVosSRFbD9H2oqppltptFSNPlI0E+t0JBWHem6YK7xcugiO3ImMCaaU8g6Jt/wIDAQAB;s=email;t=s"
+
 ; Recommended DNS entries
-_submission._tcp.zftest.testrun.org.  SRV 0 1 587 zftest.testrun.org.
-_submissions._tcp.zftest.testrun.org. SRV 0 1 465 zftest.testrun.org.
-_imap._tcp.zftest.testrun.org.        SRV 0 1 143 zftest.testrun.org.
-_imaps._tcp.zftest.testrun.org.       SRV 0 1 993 zftest.testrun.org.
-zftest.testrun.org.                   CAA 0 issue "letsencrypt.org;accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/1371472956"
-zftest.testrun.org.                   TXT "v=spf1 a:zftest.testrun.org ~all"
-_dmarc.zftest.testrun.org.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
-_adsp._domainkey.zftest.testrun.org.  TXT "dkim=discardable"
+zftest.testrun.org.                      3600   IN  TXT    "v=spf1 a ~all"
+_dmarc.zftest.testrun.org.               3600   IN  TXT    "v=DMARC1;p=reject;adkim=s;aspf=s"
+zftest.testrun.org.                      3600   IN  CAA    0 issue "letsencrypt.org;accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/1371472956"
+_adsp._domainkey.zftest.testrun.org.     3600   IN  TXT    "dkim=discardable"
+_submission._tcp.zftest.testrun.org.     3600   IN  SRV    0 1 587 zftest.testrun.org.
+_submissions._tcp.zftest.testrun.org.    3600   IN  SRV    0 1 465 zftest.testrun.org.
+_imap._tcp.zftest.testrun.org.           3600   IN  SRV    0 1 143 zftest.testrun.org.
+_imaps._tcp.zftest.testrun.org.          3600   IN  SRV    0 1 993 zftest.testrun.org.

cmdeploy/src/cmdeploy/tests/online/benchmark.py

@@ -1,4 +1,3 @@
-import time
 def test_tls_imap(benchmark, imap):
     def imap_connect():
         imap.connect()

cmdeploy/src/cmdeploy/tests/online/test_0_login.py

@@ -89,7 +89,9 @@ def test_concurrent_logins_same_account(
         assert login_results.get()
 
 
-def test_no_vrfy(chatmail_config):
+def test_no_vrfy(cmfactory, chatmail_config):
+    ac = cmfactory.get_online_account()
+    addr = ac.get_config("addr")
     domain = chatmail_config.mail_domain
 
     s = smtplib.SMTP(domain)
@@ -98,7 +100,7 @@ def test_no_vrfy(chatmail_config):
     s.putcmd("vrfy", f"wrongaddress@{chatmail_config.mail_domain}")
     result = s.getreply()
     print(result)
-    s.putcmd("vrfy", f"echo@{chatmail_config.mail_domain}")
+    s.putcmd("vrfy", addr)
     result2 = s.getreply()
     print(result2)
     assert result[0] == result2[0] == 252

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -71,6 +71,44 @@ class TestSSHExecutor:
         assert (now - since_date).total_seconds() < 60 * 60 * 51
 
 
+def test_dovecot_main_process_matches_installed_binary(sshdomain):
+    sshexec = get_sshexec(sshdomain)
+    main_pid = int(
+        sshexec(
+            call=remote.rshell.shell,
+            kwargs=dict(
+                command="timeout 10 systemctl show -p MainPID --value dovecot.service"
+            ),
+        ).strip()
+    )
+    assert main_pid != 0, "dovecot.service MainPID is 0 -- service not running?"
+
+    exe = sshexec(
+        call=remote.rshell.shell,
+        kwargs=dict(command=f"timeout 10 readlink /proc/{main_pid}/exe"),
+    ).strip()
+    status_text = sshexec(
+        call=remote.rshell.shell,
+        kwargs=dict(
+            command="timeout 10 systemctl show -p StatusText --value dovecot.service"
+        ),
+    ).strip()
+    installed_version = sshexec(
+        call=remote.rshell.shell, kwargs=dict(command="timeout 10 dovecot --version")
+    ).strip()
+
+    assert not exe.endswith("(deleted)"), (
+        f"running dovecot binary was deleted (stale after upgrade): {exe}"
+    )
+    expected_status_text = f"v{installed_version}"
+    assert status_text == expected_status_text or status_text.startswith(
+        f"{expected_status_text} "
+    ), (
+        f"dovecot status version mismatch: "
+        f"StatusText={status_text!r}, installed={installed_version!r}"
+    )
+
+
 def test_timezone_env(remote):
     for line in remote.iter_output("env"):
         print(line)
@@ -183,7 +221,6 @@ def test_rewrite_subject(cmsetup, maildata):
     assert "Subject: Unencrypted subject" not in rcvd_msg
 
 
-@pytest.mark.slow
 def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
     """Test that the per-account send-mail limit is exceeded."""
     user1, user2 = cmsetup.gen_users(2)
@@ -206,7 +243,6 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
     pytest.fail("Rate limit was not exceeded")
 
 
-@pytest.mark.slow
 def test_expunged(remote, chatmail_config):
     outdated_days = int(chatmail_config.delete_mails_after) + 1
     find_cmds = [

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -6,8 +6,8 @@ import imap_tools
 import pytest
 import requests
 
-from cmdeploy.remote import rshell
 from cmdeploy.cmdeploy import get_sshexec
+from cmdeploy.remote import rshell
 
 
 @pytest.fixture

cmdeploy/src/cmdeploy/tests/plugin.py

@@ -1,4 +1,5 @@
 import imaplib
+import ipaddress
 import itertools
 import os
 import random
@@ -14,10 +15,12 @@ from chatmaild.config import read_config
 conftestdir = Path(__file__).parent
 
 
-def pytest_addoption(parser):
-    parser.addoption(
-        "--slow", action="store_true", default=False, help="also run slow tests"
-    )
+def _is_ip(domain):
+    try:
+        ipaddress.ip_address(domain)
+        return True
+    except ValueError:
+        return False
 
 
 def pytest_configure(config):
@@ -27,14 +30,12 @@ def pytest_configure(config):
     )
 
 
-def pytest_runtest_setup(item):
-    markers = list(item.iter_markers(name="slow"))
-    if markers:
-        if not item.config.getoption("--slow"):
-            pytest.skip("skipping slow test, use --slow to run")
-
-
 def _get_chatmail_config():
+    inipath = os.environ.get("CHATMAIL_INI")
+    if inipath:
+        path = Path(inipath).resolve()
+        return read_config(path), path
+
     current = Path().resolve()
     while 1:
         path = current.joinpath("chatmail.ini").resolve()
@@ -277,6 +278,7 @@ def gencreds(chatmail_config):
 
     def gen(domain=None):
         domain = domain if domain else chatmail_config.mail_domain
+        addr_domain = f"[{domain}]" if _is_ip(domain) else domain
         while 1:
             num = next(count)
             alphanumeric = "abcdefghijklmnopqrstuvwxyz1234567890"
@@ -290,7 +292,7 @@ def gencreds(chatmail_config):
             password = "".join(
                 random.choices(alphanumeric, k=chatmail_config.password_min_length)
             )
-            yield f"{user}@{domain}", f"{password}"
+            yield f"{user}@{addr_domain}", f"{password}"
 
     return lambda domain=None: next(gen(domain))
 
@@ -339,9 +341,22 @@ class ChatmailACFactory:
         accounts = []
         for _ in range(num):
             account = self.dc.add_account()
-            future = account.add_or_update_transport.future(
-                self._make_transport(domain)
-            )
+            addr, password = self.gencreds(domain)
+            if _is_ip(domain):
+                # Use DCLOGIN scheme with explicit server hosts,
+                # matching how madmail presents its addresses to users.
+                qr = (
+                    f"dclogin:{addr}"
+                    f"?p={password}&v=1"
+                    f"&ih={domain}&ip=993"
+                    f"&sh={domain}&sp=465"
+                    f"&ic=3&ss=default"
+                )
+                future = account.add_transport_from_qr.future(qr)
+            else:
+                future = account.add_or_update_transport.future(
+                    self._make_transport(domain)
+                )
             futures.append(future)
 
             # ensure messages stay in INBOX so that they can be
@@ -388,12 +403,15 @@ def cmfactory(rpc, gencreds, maildomain, chatmail_config):
 
 @pytest.fixture
 def remote(sshdomain):
-    return Remote(sshdomain)
+    r = Remote(sshdomain)
+    yield r
+    r.close()
 
 
 class Remote:
     def __init__(self, sshdomain):
         self.sshdomain = sshdomain
+        self._procs = []
 
     def iter_output(self, logcmd="", ready=None):
         getjournal = "journalctl -f" if not logcmd else logcmd
@@ -403,19 +421,32 @@ class Remote:
             case "localhost": command = []
             case _: command = ["ssh", f"root@{self.sshdomain}"]
         [command.append(arg) for arg in getjournal.split()]
-        self.popen = subprocess.Popen(
+        popen = subprocess.Popen(
             command,
+            stdin=subprocess.DEVNULL,
             stdout=subprocess.PIPE,
+            stderr=subprocess.DEVNULL,
         )
-        while 1:
-            line = self.popen.stdout.readline()
-            res = line.decode().strip().lower()
-            if not res:
-                break
-            if ready is not None:
-                ready()
-                ready = None
-            yield res
+        self._procs.append(popen)
+        try:
+            while 1:
+                line = popen.stdout.readline()
+                res = line.decode().strip().lower()
+                if not res:
+                    break
+                if ready is not None:
+                    ready()
+                    ready = None
+                yield res
+        finally:
+            popen.terminate()
+            popen.wait()
+
+    def close(self):
+        while self._procs:
+            proc = self._procs.pop()
+            proc.kill()
+            proc.wait()
 
 
 @pytest.fixture

cmdeploy/src/cmdeploy/tests/test_cmdeploy.py

@@ -23,15 +23,19 @@ class TestCmdline:
         run = parser.parse_args(["run"])
         assert init and run
 
-    def test_init_not_overwrite(self, capsys):
-        assert main(["init", "chat.example.org"]) == 0
+    def test_init_not_overwrite(self, capsys, tmp_path, monkeypatch):
+        monkeypatch.delenv("CHATMAIL_INI", raising=False)
+        inipath = tmp_path / "chatmail.ini"
+        args = ["init", "--config", str(inipath), "chat.example.org"]
+        assert main(args) == 0
         capsys.readouterr()
 
-        assert main(["init", "chat.example.org"]) == 1
+        assert main(args) == 1
         out, err = capsys.readouterr()
         assert "path exists" in out.lower()
 
-        assert main(["init", "chat.example.org", "--force"]) == 0
+        args.insert(1, "--force")
+        assert main(args) == 0
         out, err = capsys.readouterr()
         assert "deleting config file" in out.lower()
 

cmdeploy/src/cmdeploy/tests/test_dns.py

@@ -3,7 +3,7 @@ from copy import deepcopy
 import pytest
 
 from cmdeploy import remote
-from cmdeploy.dns import check_full_zone, check_initial_remote_data
+from cmdeploy.dns import check_full_zone, check_initial_remote_data, parse_zone_records
 
 
 @pytest.fixture
@@ -125,18 +125,49 @@ class TestPerformInitialChecks:
         assert not l
 
 
+def test_parse_zone_records():
+    text = """
+    ; This is a comment
+    some.domain. 3600 IN A 1.1.1.1
+
+    ; Another comment
+    www.some.domain. 3600 IN CNAME some.domain.
+
+    ; Multi-word rdata
+    some.domain. 3600 IN MX 10 mail.some.domain.
+
+    ; DKIM record (single line, multi-word TXT rdata)
+    dkim._domainkey.some.domain. 3600 IN TXT "v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG" "9w0BAQEFAAOCAQ8AMIIBCgKCAQEA"
+
+    ; Another TXT record
+    _dmarc.some.domain. 3600 IN TXT "v=DMARC1;p=reject"
+    """
+    records = list(parse_zone_records(text))
+    assert records == [
+        ("some.domain", "3600", "A", "1.1.1.1"),
+        ("www.some.domain", "3600", "CNAME", "some.domain."),
+        ("some.domain", "3600", "MX", "10 mail.some.domain."),
+        (
+            "dkim._domainkey.some.domain",
+            "3600",
+            "TXT",
+            '"v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG" "9w0BAQEFAAOCAQ8AMIIBCgKCAQEA"',
+        ),
+        ("_dmarc.some.domain", "3600", "TXT", '"v=DMARC1;p=reject"'),
+    ]
+
+
+def test_parse_zone_records_invalid_line():
+    text = "invalid line"
+    with pytest.raises(ValueError, match="Bad zone record line"):
+        list(parse_zone_records(text))
+
+
 def parse_zonefile_into_dict(zonefile, mockdns_base, only_required=False):
-    for zf_line in zonefile.split("\n"):
-        if zf_line.startswith("#"):
-            if "Recommended" in zf_line and only_required:
-                return
-            continue
-        if not zf_line.strip():
-            continue
-        zf_domain, zf_typ, zf_value = zf_line.split(maxsplit=2)
-        zf_domain = zf_domain.rstrip(".")
-        zf_value = zf_value.strip()
-        mockdns_base.setdefault(zf_typ, {})[zf_domain] = zf_value
+    if only_required:
+        zonefile = zonefile.split("; Recommended")[0]
+    for name, ttl, rtype, rdata in parse_zone_records(zonefile):
+        mockdns_base.setdefault(rtype, {})[name] = rdata
 
 
 class MockSSHExec:

cmdeploy/src/cmdeploy/tests/test_dovecot_deployer.py

@@ -0,0 +1,238 @@
+from contextlib import nullcontext
+from types import SimpleNamespace
+
+import pytest
+from pyinfra.facts.deb import DebPackages
+
+from cmdeploy.dovecot import deployer as dovecot_deployer
+
+
+def make_host(*fact_pairs):
+    """Build a mock host; get_fact(cls) dispatches to the provided facts mapping.
+
+    Args:
+        *fact_pairs: tuples of (fact_class, fact_value) to register
+
+    Returns:
+        SimpleNamespace with get_fact that raises a clear error if an
+        unexpected fact type is requested.
+    """
+    facts = dict(fact_pairs)
+
+    def get_fact(cls):
+        if cls not in facts:
+            registered = ", ".join(c.__name__ for c in facts)
+            raise LookupError(
+                f"unexpected get_fact({cls.__name__}); "
+                f"only registered: {registered}"
+            )
+        return facts[cls]
+
+    return SimpleNamespace(get_fact=get_fact)
+
+
+@pytest.fixture
+def deployer():
+    return dovecot_deployer.DovecotDeployer(
+        SimpleNamespace(mail_domain="chat.example.org"),
+        disable_mail=False,
+    )
+
+
+@pytest.fixture
+def patch_blocked(monkeypatch):
+    monkeypatch.setattr(dovecot_deployer, "blocked_service_startup", nullcontext)
+
+
+@pytest.fixture
+def mock_files_put(monkeypatch):
+    monkeypatch.setattr(
+        dovecot_deployer.files,
+        "put",
+        lambda **kwargs: SimpleNamespace(changed=False),
+    )
+
+
+@pytest.fixture
+def track_shell(monkeypatch):
+    calls = []
+    monkeypatch.setattr(
+        dovecot_deployer.server,
+        "shell",
+        lambda **kwargs: calls.append(kwargs) or SimpleNamespace(changed=False),
+    )
+    return calls
+
+
+def test_download_dovecot_package_skips_epoch_matched_install(monkeypatch):
+    epoch_version = dovecot_deployer.DOVECOT_PACKAGE_VERSION
+    downloads = []
+    monkeypatch.setattr(
+        dovecot_deployer,
+        "host",
+        make_host((DebPackages, {"dovecot-core": [epoch_version]})),
+    )
+    monkeypatch.setattr(
+        dovecot_deployer,
+        "_pick_url",
+        lambda primary, fallback: primary,
+    )
+    monkeypatch.setattr(
+        dovecot_deployer.files,
+        "download",
+        lambda **kwargs: downloads.append(kwargs),
+    )
+
+    deb, changed = dovecot_deployer._download_dovecot_package("core", "amd64")
+
+    assert deb is None, f"expected no deb path when version matches, got {deb!r}"
+    assert changed is False, "should not flag changed when version already installed"
+    assert downloads == [], "should not download when version already installed"
+
+
+def test_download_dovecot_package_uses_archive_version_for_url_and_filename(
+    monkeypatch,
+):
+    downloads = []
+    monkeypatch.setattr(
+        dovecot_deployer,
+        "host",
+        make_host((DebPackages, {})),
+    )
+    monkeypatch.setattr(
+        dovecot_deployer,
+        "_pick_url",
+        lambda primary, fallback: primary,
+    )
+    monkeypatch.setattr(
+        dovecot_deployer.files,
+        "download",
+        lambda **kwargs: downloads.append(kwargs),
+    )
+
+    deb, changed = dovecot_deployer._download_dovecot_package("core", "amd64")
+
+    archive_version = dovecot_deployer.DOVECOT_ARCHIVE_VERSION.replace("+", "%2B")
+    expected_deb = f"/root/dovecot-core_{archive_version}_amd64.deb"
+
+    # Verify the returned path uses archive version, not package version (with epoch)
+    assert changed is True, "should flag changed when package not yet installed"
+    assert deb == expected_deb, f"deb path mismatch: {deb!r} != {expected_deb!r}"
+    assert dovecot_deployer.DOVECOT_PACKAGE_VERSION not in deb, (
+        f"deb path should use archive version (no epoch), got {deb!r}"
+    )
+    assert len(downloads) == 1, "files.download should be called exactly once"
+
+
+def test_install_skips_dpkg_path_when_epoch_matched_packages_present(
+    deployer, patch_blocked, mock_files_put, track_shell, monkeypatch
+):
+    monkeypatch.setattr(
+        dovecot_deployer,
+        "host",
+        make_host(
+            (
+                dovecot_deployer.DebPackages,
+                {
+                    "dovecot-core": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
+                    "dovecot-imapd": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
+                    "dovecot-lmtpd": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
+                },
+            ),
+            (dovecot_deployer.Arch, "x86_64"),
+        ),
+    )
+    downloads = []
+    monkeypatch.setattr(
+        dovecot_deployer.files,
+        "download",
+        lambda **kwargs: downloads.append(kwargs),
+    )
+
+    deployer.install()
+
+    assert downloads == [], "should not download when all packages epoch-matched"
+    assert track_shell == [], "should not run dpkg when all packages epoch-matched"
+    assert deployer.need_restart is False, (
+        "need_restart should be False when nothing changed"
+    )
+
+
+def test_install_unsupported_arch_falls_back_to_apt(
+    deployer, patch_blocked, mock_files_put, track_shell, monkeypatch
+):
+    # For unsupported architectures, all fact lookups return the arch string.
+    monkeypatch.setattr(
+        dovecot_deployer,
+        "host",
+        SimpleNamespace(get_fact=lambda cls: "riscv64"),
+    )
+    apt_calls = []
+
+    # Mirrors apt.packages() return value: OperationMeta with .changed property.
+    # Only lmtpd triggers a change to verify |= accumulation of changed flags.
+    def fake_apt(**kwargs):
+        apt_calls.append(kwargs)
+        changed = "lmtpd" in kwargs["packages"][0]
+        return SimpleNamespace(changed=changed)
+
+    monkeypatch.setattr(dovecot_deployer.apt, "packages", fake_apt)
+
+    deployer.install()
+
+    actual_pkgs = [c["packages"] for c in apt_calls]
+    assert actual_pkgs == [["dovecot-core"], ["dovecot-imapd"], ["dovecot-lmtpd"]], (
+        f"expected apt install of core/imapd/lmtpd, got {actual_pkgs}"
+    )
+    assert track_shell == [], "should not run dpkg for unsupported arch"
+    assert deployer.need_restart is True, (
+        "need_restart should be True when apt installed a package"
+    )
+
+
+def test_install_runs_dpkg_when_packages_need_download(
+    deployer, patch_blocked, mock_files_put, track_shell, monkeypatch
+):
+    monkeypatch.setattr(
+        dovecot_deployer,
+        "host",
+        make_host(
+            (dovecot_deployer.DebPackages, {}),
+            (dovecot_deployer.Arch, "x86_64"),
+        ),
+    )
+    monkeypatch.setattr(
+        dovecot_deployer,
+        "_pick_url",
+        lambda primary, fallback: primary,
+    )
+    monkeypatch.setattr(
+        dovecot_deployer.files,
+        "download",
+        lambda **kwargs: SimpleNamespace(changed=True),
+    )
+
+    deployer.install()
+
+    assert len(track_shell) == 1, (
+        f"expected one server.shell() call for dpkg install, got {len(track_shell)}"
+    )
+    cmds = track_shell[0]["commands"]
+    assert len(cmds) == 3, f"expected 3 dpkg/apt commands, got: {cmds}"
+    assert cmds[0].startswith("dpkg --force-confdef --force-confold -i ")
+    assert "apt-get -y --fix-broken install" in cmds[1]
+    assert cmds[2].startswith("dpkg --force-confdef --force-confold -i ")
+    assert deployer.need_restart is True, (
+        "need_restart should be True after dpkg install"
+    )
+
+
+def test_pick_url_falls_back_on_primary_error(monkeypatch):
+    def raise_error(req, timeout):
+        raise OSError("connection timeout")
+
+    monkeypatch.setattr(dovecot_deployer.urllib.request, "urlopen", raise_error)
+    result = dovecot_deployer._pick_url("http://primary", "http://fallback")
+    assert result == "http://fallback", (
+        f"should fall back when primary fails, got {result!r}"
+    )

cmdeploy/src/cmdeploy/tests/test_helpers.py

@@ -1,11 +1,10 @@
-import importlib.resources
+from pathlib import Path
 
 from cmdeploy.www import build_webpages
 
 
 def test_build_webpages(tmp_path, make_config):
-    pkgroot = importlib.resources.files("cmdeploy")
-    src_dir = pkgroot.joinpath("../../../www/src").resolve()
+    src_dir = (Path(__file__).resolve() / "../../../../../www/src").resolve()
     assert src_dir.exists(), src_dir
     config = make_config("chat.example.org")
     build_dir = tmp_path.joinpath("build")

cmdeploy/src/cmdeploy/www.py

@@ -1,5 +1,4 @@
 import hashlib
-import importlib.resources
 import re
 import time
 import traceback
@@ -37,7 +36,7 @@ def prepare_template(source):
 
 
 def get_paths(config) -> (Path, Path, Path):
-    reporoot = importlib.resources.files(__package__).joinpath("../../../").resolve()
+    reporoot = (Path(__file__).resolve() / "../../../../").resolve()
     www_path = Path(config.www_folder)
     # if www_folder was not set, use default directory
     if config.www_folder == "":
@@ -133,8 +132,7 @@ def find_merge_conflict(src_dir) -> Path:
 
 
 def main():
-    path = importlib.resources.files(__package__)
-    reporoot = path.joinpath("../../../").resolve()
+    reporoot = (Path(__file__).resolve() / "../../../../").resolve()
     inipath = reporoot.joinpath("chatmail.ini")
     config = read_config(inipath)
     config.webdev = True

doc/README.md

@@ -4,12 +4,14 @@
 
 You can use the `make` command and `make html` to build web pages. 
 
-You need a Python environment where the following install was excuted: 
-
-    pip install furo sphinx-autobuild
+You need a Python environment with `sphinx` and other
+dependencies, you can create it by running `scripts/initenv.sh`
+from the repository root.
 
 To develop/change documentation, you can then do: 
 
+    . venv/bin/activate
+    cd doc
     make auto 
 
 A page will open at https://127.0.0.1:8000/ serving the docs and it will 

doc/source/index.rst

@@ -16,5 +16,6 @@ Contributions and feedback welcome through the https://github.com/chatmail/relay
     proxy
     migrate
     overview
+    reverse_dns
     related
     faq

doc/source/overview.rst

@@ -102,8 +102,12 @@ short overview of ``chatmaild`` services:
    Apple/Google/Huawei.
 
 -  `chatmail-expire <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/expire.py>`_
-   deletes users if they have not logged in for a longer while.
-   The timeframe can be configured in ``chatmail.ini``.
+   deletes old messages, large messages, and entire mailboxes
+   of users who have not logged in for longer than
+   ``delete_inactive_users_after`` days.
+
+-  ``chatmail-quota-expire`` is called by Dovecot's ``quota_warning`` mechanism
+   and will automatically remove oldest messages to keep mailboxes well under ``max_mailbox_size``.
 
 -  `lastlogin <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/lastlogin.py>`_
    is contacted by Dovecot when a user logs in and stores the date of
@@ -149,6 +153,7 @@ Chatmail relay dependency diagram
         autoconfig.xml --- dovecot;
         postfix --- |10080|filtermail-outgoing;
         postfix --- |10081|filtermail-incoming;
+        postfix --- |10083|filtermail-transport;
         filtermail-outgoing --- |10025 reinject|postfix;
         filtermail-incoming --- |10026 reinject|postfix;
         dovecot --- |doveauth.socket|doveauth;
@@ -156,6 +161,8 @@ Chatmail relay dependency diagram
         /home/vmail/.../user"];
         dovecot --- |lastlogin.socket|lastlogin;
         dovecot --- chatmail-metadata;
+        dovecot --- |quota-warning|chatmail-quota-expire;
+        chatmail-quota-expire --- maildir;
         lastlogin --- maildir;
         doveauth --- maildir;
         chatmail-expire-daily --- maildir;
@@ -289,9 +296,7 @@ ensured by ``filtermail`` proxy.
 TLS requirements
 ~~~~~~~~~~~~~~~~
 
-Postfix is configured to require valid TLS by setting
-`smtp_tls_security_level <https://www.postfix.org/postconf.5.html#smtp_tls_security_level>`_
-to ``verify``.
+Filtermail (used for delivery) requires a valid TLS.
 
 You can test it by resolving ``MX`` records of your relay domain and
 then connecting to MX relays (e.g ``mx.example.org``) with

doc/source/reverse_dns.rst

@@ -0,0 +1,64 @@
+Configuring reverse DNS
+=======================
+
+Some email servers reject the emails
+if they don't pass `FCrDNS`_ check, also known as `iprev`_ check.
+
+.. _FCrDNS: https://en.wikipedia.org/wiki/Forward-confirmed_reverse_DNS
+.. _iprev: https://datatracker.ietf.org/doc/html/rfc8601#section-3
+
+Passing the check requires that the IP address that email is sent from
+should have a ``PTR`` record pointing to the domain name of the server,
+and domain name record should have an ``A/AAAA`` record
+pointing to the IP address.
+
+Modern email relies on DKIM and SPF for authentication,
+while iprev check exists for
+`historical reasons <https://datatracker.ietf.org/doc/html/draft-ietf-dnsop-reverse-mapping-considerations-06#section-2.1>`_.
+Chatmail relays don't resolve ``PTR`` records,
+so you can ignore this section if configuring ``PTR`` records
+is difficult and federation with legacy email servers that don't accept
+valid DKIM signature for authentication is not important.
+
+Multi-homed setups
+------------------
+
+If you have a server with multiple IP addresses,
+also known as multi-homed setup,
+and don't publish all IP addresses in DNS,
+you need to make sure you are using
+the published address when making outgoing connections.
+
+For example, your server may have a static IP
+address, and a so-called Floating IP or Virtual IP
+that can be moved between servers in case of
+migration or for failover.
+By using Floating IP you can avoid downtime
+and keep the IP address reputation
+for destinatinons that rely on IP reputation and IP blocklists.
+In this case you will only publish
+the Floating IP to DNS and only use the static IP
+to SSH into the server.
+
+If you have such setup, make sure that
+you not only set ``PTR`` records for the Floating IP,
+but make outgoing connections using the Floating IP.
+Otherwise reverse DNS check succeed,
+but forward check making sure your domain name points
+to the IP address will fail.
+Such setup is indistinguishable from someone
+setting IP address ``PTR`` with the domain they don't own
+and as a result don't succeed.
+
+On Linux you can configure source IP address with ``ip route`` command,
+for example:
+::
+
+  ip route change default via <default-gateway> dev eth0 src <source-address>
+
+Make sure to persist the change after verifying it is working.
+You can check what your outgoing IP address is
+with ``curl icanhazip.com``.
+Check both the IPv4 and IPv6 addresses.
+For IPv4 address use ``curl ipv4.icanhazip.com`` or ``curl -4 icanhazip.com``
+and similarly for IPv6 if you have it.