feat: Sign bounce messages

Closes #873

Signed-off-by: Jagoda Ślązak <jslazak@jslazak.com>

.github/workflows/ci.yaml

@@ -1,35 +1,21 @@
-name: Run unit-tests and container-based deploy+test verification
+name: CI
 
 on:
-  # Triggers when a PR is merged into main or a direct push occurs
-  push:
-    branches: [ "main" ]
-    
-  # Triggers for any PR (and its subsequent commits) targeting the main branch
   pull_request:
-    branches: [ "main" ]
-
-permissions: {}
-
-# Newest push wins: Prevents multiple runs from clashing and wasting runner efforts
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
+  push:
 
 jobs:
   tox:
     name: isolated chatmaild tests
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
         # Checkout pull request HEAD commit instead of merge commit
         # Otherwise `test_deployed_state` will be unhappy.
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-          persist-credentials: false
       - name: download filtermail
-        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.6.4/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
+        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.6.1/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
       - name: run chatmaild tests 
         working-directory: chatmaild
         run: pipx run tox
@@ -38,10 +24,7 @@ jobs:
     name: deploy-chatmail tests 
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-          persist-credentials: false
+      - uses: actions/checkout@v4
 
       - name: initenv 
         run: scripts/initenv.sh
@@ -55,24 +38,5 @@ jobs:
       - name: run deploy-chatmail offline tests 
         run: pytest --pyargs cmdeploy 
 
-  lxc-test:
-    name: LXC deploy and test
-    uses: chatmail/cmlxc/.github/workflows/lxc-test.yml@v0.13.5
-    with:
-      cmlxc_version: v0.13.5
-      cmlxc_commands: |
-        cmlxc init 
-        # single cmdeploy relay test
-        cmlxc -v deploy-cmdeploy --source ./repo cm0
-        cmlxc -v test-mini cm0
-        cmlxc -v test-cmdeploy cm0
-
-        # cross cmdeploy relay test
-        cmlxc -v deploy-cmdeploy --source ./repo --ipv4-only cm1
-        cmlxc -v test-cmdeploy cm0 cm1
-
-        # cross cmdeploy/madmail relay tests
-        cmlxc -v deploy-madmail mad0
-        cmlxc -v test-cmdeploy cm0 mad0
-        cmlxc -v test-mini cm0 mad0
-        cmlxc -v test-mini mad0 cm0
+      # all other cmdeploy commands require a staging server 
+      # see https://github.com/deltachat/chatmail/issues/100

.github/workflows/docker-dispatch.yaml

@@ -1,37 +0,0 @@
-# Notify the docker repo to build and test a new image after relay CI passes.
-#
-# Sends a repository_dispatch event to chatmail/docker with the relay ref
-# and short SHA, which triggers docker-ci.yaml to build, push to GHCR,
-# and run integration tests via cmlxc.
-
-name: Trigger Docker build
-
-on:
-  push:
-    branches: [main]
-  workflow_dispatch:
-
-permissions: {}
-
-jobs:
-  dispatch:
-    name: Dispatch build to chatmail/docker
-    runs-on: ubuntu-latest
-    if: github.repository == 'chatmail/relay'
-    steps:
-      - name: Compute short SHA
-        id: sha
-        run: echo "short=$(echo '${{ github.sha }}' | cut -c1-7)" >> "$GITHUB_OUTPUT"
-
-      - name: Send repository_dispatch
-        uses: peter-evans/repository-dispatch@ff45666b9427631e3450c54a1bcbee4d9ff4d7c0 # v3
-        with:
-          token: ${{ secrets.CHATMAIL_DOCKER_DISPATCH_TOKEN }}
-          repository: chatmail/docker
-          event-type: relay-updated
-          client-payload: >-
-            {
-              "relay_ref": "${{ github.ref_name }}",
-              "relay_sha": "${{ github.sha }}",
-              "relay_sha_short": "${{ steps.sha.outputs.short }}"
-            }

.github/workflows/docs-preview.yaml

@@ -7,8 +7,6 @@ on:
       - 'scripts/build-docs.sh'
       - '.github/workflows/docs-preview.yaml'
 
-permissions: {}
-
 jobs:
   scripts:
     name: build 
@@ -18,8 +16,6 @@ jobs:
       url: https://staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}
     steps:
       - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
 
       - name: initenv 
         run: scripts/initenv.sh
@@ -38,22 +34,18 @@ jobs:
       - name: Get Pullrequest ID
         id: prepare
         run: |
-          export PULLREQUEST_ID=$(echo "${GITHUB_REF}" | cut -d "/" -f3)
+          export PULLREQUEST_ID=$(echo "${{ github.ref }}" | cut -d "/" -f3)
           echo "prid=$PULLREQUEST_ID" >> $GITHUB_OUTPUT
           if [ $(expr length "${{ secrets.USERNAME }}") -gt "1" ]; then echo "uploadtoserver=true" >> $GITHUB_OUTPUT; fi
       - run: |
-          echo "baseurl: /${STEPS_PREPARE_OUTPUTS_PRID}" >> _config.yml
-        env:
-          STEPS_PREPARE_OUTPUTS_PRID: ${{ steps.prepare.outputs.prid }}
+          echo "baseurl: /${{ steps.prepare.outputs.prid }}" >> _config.yml
 
       - name: Upload preview
         run: |
           mkdir -p "$HOME/.ssh"
           echo "${{ secrets.CHATMAIL_STAGING_SSHKEY }}" > "$HOME/.ssh/key"
           chmod 600 "$HOME/.ssh/key"
-          rsync -rILvh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/doc/build/ "${{ secrets.USERNAME }}@chatmail.at:/var/www/html/staging.chatmail.at/doc/relay/${STEPS_PREPARE_OUTPUTS_PRID}/"
-        env:
-          STEPS_PREPARE_OUTPUTS_PRID: ${{ steps.prepare.outputs.prid }}
+          rsync -rILvh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/doc/build/ "${{ secrets.USERNAME }}@chatmail.at:/var/www/html/staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}/"
 
       - name: check links 
         working-directory: doc

.github/workflows/docs.yaml

@@ -10,8 +10,6 @@ on:
       - 'scripts/build-docs.sh'
       - '.github/workflows/docs.yaml'
 
-permissions: {}
-
 jobs:
   scripts:
     name: build 
@@ -21,8 +19,6 @@ jobs:
       url: https://chatmail.at/doc/relay/
     steps:
       - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
 
       - name: initenv 
         run: scripts/initenv.sh

.github/workflows/test-and-deploy-ipv4only.yaml

@@ -0,0 +1,105 @@
+name: deploy on staging-ipv4.testrun.org, and run tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths-ignore:
+      - 'scripts/**'
+      - '**/README.md'
+      - 'CHANGELOG.md'
+      - 'LICENSE'
+
+jobs:
+  deploy:
+    name: deploy on staging-ipv4.testrun.org, and run tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    environment:
+      name: staging-ipv4.testrun.org
+      url: https://staging-ipv4.testrun.org/
+    concurrency: staging-ipv4.testrun.org
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: prepare SSH
+        run: |
+          mkdir ~/.ssh
+          echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
+          chmod 600 ~/.ssh/id_ed25519
+          ssh-keyscan staging-ipv4.testrun.org > ~/.ssh/known_hosts
+          # save previous acme & dkim state
+          rsync -avz root@staging-ipv4.testrun.org:/var/lib/acme acme-ipv4 || true
+          rsync -avz root@staging-ipv4.testrun.org:/etc/dkimkeys dkimkeys-ipv4 || true
+          # store previous acme & dkim state on ns.testrun.org, if it contains useful certs
+          if [ -f dkimkeys-ipv4/dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys-ipv4 root@ns.testrun.org:/tmp/ || true; fi
+          if [ "$(ls -A acme-ipv4/acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme-ipv4 root@ns.testrun.org:/tmp/ || true; fi
+          # make sure CAA record isn't set
+          scp -o StrictHostKeyChecking=accept-new .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
+          ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging-ipv4.testrun.org.zone
+          ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
+          ssh root@ns.testrun.org systemctl reload nsd
+
+      - name: rebuild staging-ipv4.testrun.org to have a clean VPS
+        run: |
+            curl -X POST \
+            -H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
+            -H "Content-Type: application/json" \
+            -d '{"image":"debian-12"}' \
+            "https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_IPV4_SERVER_ID }}/actions/rebuild"
+
+      - run: scripts/initenv.sh
+
+      - name: append venv/bin to PATH
+        run: echo venv/bin >>$GITHUB_PATH
+
+      - name: upload TLS cert after rebuilding
+        run: |
+          echo " --- wait until staging-ipv4.testrun.org VPS is rebuilt --- "
+          rm ~/.ssh/known_hosts
+          while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u ; do sleep 1 ; done
+          ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u
+          # download acme & dkim state from ns.testrun.org
+          rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme-ipv4/acme acme-restore || true
+          rsync -avz root@ns.testrun.org:/tmp/dkimkeys-ipv4/dkimkeys dkimkeys-restore || true
+          # restore acme & dkim state to staging2.testrun.org
+          rsync -avz acme-restore/acme root@staging-ipv4.testrun.org:/var/lib/ || true
+          rsync -avz dkimkeys-restore/dkimkeys root@staging-ipv4.testrun.org:/etc/ || true
+          ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org chown root:root -R /var/lib/acme || true
+
+      - name: run deploy-chatmail offline tests 
+        run: pytest --pyargs cmdeploy 
+
+      - name: setup dependencies
+        run: |
+          ssh root@staging-ipv4.testrun.org apt update
+          ssh root@staging-ipv4.testrun.org apt install -y git python3.11-venv python3-dev gcc
+          ssh root@staging-ipv4.testrun.org git clone https://github.com/chatmail/relay
+          ssh root@staging-ipv4.testrun.org "cd relay && git checkout " ${{ github.head_ref }}
+          ssh root@staging-ipv4.testrun.org "cd relay && scripts/initenv.sh"
+
+      - name: initialize config
+        run: |
+          ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy init staging-ipv4.testrun.org"
+          ssh root@staging-ipv4.testrun.org "sed -i 's#disable_ipv6 = False#disable_ipv6 = True#' relay/chatmail.ini"
+          ssh root@staging-ipv4.testrun.org "sed -i 's/#\s*mtail_address/mtail_address/' relay/chatmail.ini"
+          ssh root@staging-ipv4.testrun.org "sed -i 's/max_mailbox_size = 500M/max_mailbox_size = 10k/' relay/chatmail.ini"
+
+      - run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy run --verbose --skip-dns-check --ssh-host localhost"
+
+      - name: set DNS entries
+        run: |
+          ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns --zonefile staging-generated.zone --ssh-host localhost"
+          ssh root@staging-ipv4.testrun.org cat relay/staging-generated.zone >> .github/workflows/staging-ipv4.testrun.org-default.zone
+          cat .github/workflows/staging-ipv4.testrun.org-default.zone
+          scp .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
+          ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
+          ssh root@ns.testrun.org systemctl reload nsd
+
+      - name: cmdeploy test
+        run: ssh root@staging-ipv4.testrun.org "cd relay && CHATMAIL_DOMAIN2=ci-chatmail.testrun.org scripts/cmdeploy test --slow --ssh-host localhost"
+
+      - name: cmdeploy dns
+        run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns -v --ssh-host localhost"
+

.github/workflows/test-and-deploy.yaml

@@ -0,0 +1,97 @@
+name: deploy on staging2.testrun.org, and run tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths-ignore:
+      - 'scripts/**'
+      - '**/README.md'
+      - 'CHANGELOG.md'
+      - 'LICENSE'
+
+jobs:
+  deploy:
+    name: deploy on staging2.testrun.org, and run tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    environment:
+      name: staging2.testrun.org
+      url: https://staging2.testrun.org/
+    concurrency: staging2.testrun.org
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: prepare SSH
+        run: |
+          mkdir ~/.ssh
+          echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
+          chmod 600 ~/.ssh/id_ed25519
+          ssh-keyscan staging2.testrun.org > ~/.ssh/known_hosts
+          # save previous acme & dkim state
+          rsync -avz root@staging2.testrun.org:/var/lib/acme . || true
+          rsync -avz root@staging2.testrun.org:/etc/dkimkeys . || true
+          # store previous acme & dkim state on ns.testrun.org, if it contains useful certs
+          if [ -f dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys root@ns.testrun.org:/tmp/ || true; fi
+          if [ "$(ls -A acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme root@ns.testrun.org:/tmp/ || true; fi
+          # make sure CAA record isn't set
+          scp -o StrictHostKeyChecking=accept-new .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
+          ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging2.testrun.org.zone
+          ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
+          ssh root@ns.testrun.org systemctl reload nsd
+
+      - name: rebuild staging2.testrun.org to have a clean VPS
+        run: |
+            curl -X POST \
+            -H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
+            -H "Content-Type: application/json" \
+            -d '{"image":"debian-12"}' \
+            "https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_SERVER_ID }}/actions/rebuild"
+
+      - run: scripts/initenv.sh
+
+      - name: append venv/bin to PATH
+        run: echo venv/bin >>$GITHUB_PATH
+
+      - name: upload TLS cert after rebuilding
+        run: |
+          echo " --- wait until staging2.testrun.org VPS is rebuilt --- "
+          rm ~/.ssh/known_hosts
+          while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org id -u ; do sleep 1 ; done
+          ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org id -u
+          # download acme & dkim state from ns.testrun.org
+          rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme acme-restore || true
+          rsync -avz root@ns.testrun.org:/tmp/dkimkeys dkimkeys-restore || true
+          # restore acme & dkim state to staging2.testrun.org
+          rsync -avz acme-restore/acme root@staging2.testrun.org:/var/lib/ || true
+          rsync -avz dkimkeys-restore/dkimkeys root@staging2.testrun.org:/etc/ || true
+          ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org chown root:root -R /var/lib/acme || true
+
+      - name: add hpk42 key to staging server
+        run: ssh root@staging2.testrun.org 'curl -s https://github.com/hpk42.keys >> .ssh/authorized_keys'
+
+      - name: run deploy-chatmail offline tests 
+        run: pytest --pyargs cmdeploy 
+
+      - run: |
+          cmdeploy init staging2.testrun.org
+          sed -i 's/#\s*mtail_address/mtail_address/;s/max_mailbox_size = 500M/max_mailbox_size = 10k/' chatmail.ini
+
+      - run: cmdeploy run --verbose --skip-dns-check
+
+      - name: set DNS entries
+        run: |
+          cmdeploy dns --zonefile staging-generated.zone --verbose
+          cat staging-generated.zone >> .github/workflows/staging.testrun.org-default.zone
+          cat .github/workflows/staging.testrun.org-default.zone
+          scp .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
+          ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
+          ssh root@ns.testrun.org systemctl reload nsd
+
+      - name: cmdeploy test
+        run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
+
+      - name: cmdeploy dns
+        run: cmdeploy dns -v
+

.github/workflows/zizmor-scan.yml

@@ -1,26 +0,0 @@
-name: GitHub Actions Security Analysis with zizmor
-
-on:
-  push:
-    branches: ["main"]
-  pull_request:
-    branches: ["**"]
-
-permissions: {}
-
-jobs:
-  zizmor:
-    name: Run zizmor
-    runs-on: ubuntu-latest
-    permissions:
-      security-events: write # Required for upload-sarif (used by zizmor-action) to upload SARIF files.
-      contents: read
-      actions: read
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v6
-        with:
-          persist-credentials: false
-
-      - name: Run zizmor
-        uses: zizmorcore/zizmor-action@b1d7e1fb5de872772f31590499237e7cce841e8e # v0.5.3

.github/zizmor.yml

@@ -1,7 +0,0 @@
-rules:
-  unpinned-uses:
-    config:
-      policies:
-        actions/*: ref-pin
-        dependabot/*: ref-pin
-        chatmail/*: ref-pin

CHANGELOG.md

@@ -1,89 +1,5 @@
 # Changelog for chatmail deployment 
 
-## 1.10.0 2026-04-30
-
-* start mtail after networking is fully up <https://github.com/chatmail/relay/pull/942>
-* support specifying custom filtermail binary through environment variable <https://github.com/chatmail/relay/pull/941>
-* add automated zizmor scanning of github workflows <https://github.com/chatmail/relay/pull/938>
-* added dispatch for *automated builds of chatmail relay docker images* <https://github.com/chatmail/relay/pull/934>
-* do not bind SMTP client sockets to public addresses <https://github.com/chatmail/relay/pull/932>
-* underline in docs that scripts/initenv.sh should be used for building the docs <https://github.com/chatmail/relay/pull/933>
-* automatic oldest-first message removal from mailboxes to always stay under max_mailbox_size <https://github.com/chatmail/relay/pull/929>
-* remove --slow from cmdeploy test <https://github.com/chatmail/relay/pull/931>
-* handle missing inotify sysctl keys in containers <https://github.com/chatmail/relay/pull/930>
-* replace resolvconf with static resolv.conf <https://github.com/chatmail/relay/pull/928>
-* disable fsync for LMTP and IMAP services <https://github.com/chatmail/relay/pull/925>
-* re-use cmlxc workflow, replacing CI with hetzner staging servers with local lxc containers <https://github.com/chatmail/relay/pull/917>
-* explicitly install resolvconf <https://github.com/chatmail/relay/pull/924>
-* detect stale dovecot binary and force restart in activate() <https://github.com/chatmail/relay/pull/922>
-* Rename filtermail_http_port to filtermail_http_port_incoming <https://github.com/chatmail/relay/pull/921>
-* consolidated is_in_container() check https://github.com/chatmail/relay/pull/920>
-* restart dovecot after package replacement (rebase, test condense) <https://github.com/chatmail/relay/pull/913>
-* Set permissions on dovecot pin prefs <https://github.com/chatmail/relay/pull/915>
-* Route `/mxdeliv/` to configurable port <https://github.com/chatmail/relay/pull/901>
-* fix VM detection, automated testing fixes, use newer chatmail-turn and move to standard BIND DNS zone format <https://github.com/chatmail/relay/pull/912>
-* Upgrade to filtermail 0.6.1 <https://github.com/chatmail/relay/pull/910>
-* pin dovecot packages to prevent apt upgrades <https://github.com/chatmail/relay/pull/908>
-* add rpc server to cmdeploy along with client <https://github.com/chatmail/relay/pull/906>
-* remove unused deps from chatmaild <https://github.com/chatmail/relay/pull/905>
-* set default smtp_tls_security_level to "verify" unconditionally <https://github.com/chatmail/relay/pull/902>
-* featprefer IPv4 in SMTP client <https://github.com/chatmail/relay/pull/900>
-* Install dovecot .deb packages atomically <https://github.com/chatmail/relay/pull/899>
-* stop installing cron package <https://github.com/chatmail/relay/pull/898>
-* Rewrite dovecot install logic, update <https://github.com/chatmail/relay/pull/862>
-* fix a test and some linting fixes <https://github.com/chatmail/relay/pull/897>
-* Disable IP verification on domain-literal addresses <https://github.com/chatmail/relay/pull/895>
-* disable installing recommended packages globally on the relay <https://github.com/chatmail/relay/pull/887>
-* multiple bug fixes across chatmaild and cmdeploy <https://github.com/chatmail/relay/pull/883>
-* remove /metrics from the website <https://github.com/chatmail/relay/pull/703>
-* add Prometheus textfile output to fsreport <https://github.com/chatmail/relay/pull/881>
-* chown opendkim: private key <https://github.com/chatmail/relay/pull/879>
-* make sure chatmail-metadata was started <https://github.com/chatmail/relay/pull/882>
-* dovecot update url <https://github.com/chatmail/relay/pull/880>
-* upgrade to filtermail v0.5.2 <https://github.com/chatmail/relay/pull/876>
-* download dovecot packages from github release <https://github.com/chatmail/relay/pull/875>
-* replace DKIM verification with filtermail v0.5 <https://github.com/chatmail/relay/pull/831>
-* remove CFFI deltachat bindings usage, and consolidate test support with rpc-bindings <https://github.com/chatmail/relay/pull/872>
-* prepare chatmaild/cmdeploy changes for Docker support <https://github.com/chatmail/relay/pull/857>
-* stabilize online benchmark timing adding rate-limit-aware cooldown between iterations <https://github.com/chatmail/relay/pull/867>
-* move rate-limit cooldown to benchmark fixture <https://github.com/chatmail/relay/pull/868>
-* reconfigure acmetool from redirector to proxy mode <https://github.com/chatmail/relay/pull/861>
-* make tests work with `--ssh-host localhost` <https://github.com/chatmail/relay/pull/856>
-* mark f-string with f prefix in test_expunged <https://github.com/chatmail/relay/pull/863>
-* install also if dovecot.service=False in SystemdEnabled Fact <https://github.com/chatmail/relay/pull/841>
-* Introduce support for self-signed chatmail relays <https://github.com/chatmail/relay/pull/855>
-* Strip Received headers before delivery <https://github.com/chatmail/relay/pull/849>
-* upgrade to filtermail v0.3 <https://github.com/chatmail/relay/pull/850>
-* fix link to Maddy and update madmail URL <https://github.com/chatmail/relay/pull/847>
-* accept self-signed certificates for IP-only relays <https://github.com/chatmail/relay/pull/846>
-* enforce sending from public IP addresses <https://github.com/chatmail/relay/pull/845>
-* port check: check addresses, fix single services <https://github.com/chatmail/relay/pull/844>
-* remediates issue with improper concat on resolver injection <https://github.com/chatmail/relay/pull/834>
-* ipv6 boolean not being respected during operations <https://github.com/chatmail/relay/pull/832>
-* upgrade to filtermail v0.2 by <https://github.com/chatmail/relay/pull/825>
-* fix link to filtermail <https://github.com/chatmail/relay/pull/824>
-* print timestamps when sending messages <https://github.com/chatmail/relay/pull/823>
-* fix flaky test_exceed_rate_limit <https://github.com/chatmail/relay/pull/822>
-* Replace filtermail with rust reimplementation <https://github.com/chatmail/relay/pull/808>
-* Set default internal SMTP ports in Config <https://github.com/chatmail/relay/pull/819>
-* separate metrics for incoming and outgoing messages <https://github.com/chatmail/relay/pull/820>
-* disable appending the Received header <https://github.com/chatmail/relay/pull/815>
-* fail on errors in postfix/dovecot config <https://github.com/chatmail/relay/pull/813>
-* tweak idle/hibernate metrics some more <https://github.com/chatmail/relay/pull/811>
-* add config flag to export statistics <https://github.com/chatmail/relay/pull/806>
-* add --website-only option to run subcommand <https://github.com/chatmail/relay/pull/768>
-* Strip DKIM-Signature header before LMTP <https://github.com/chatmail/relay/pull/803>
-* properly make sure that postfix gets restarted on failure <https://github.com/chatmail/relay/pull/802>
-* expire.py: use absolute path to maildirsize <https://github.com/chatmail/relay/pull/807>
-* pin Dovecot documentation URLs to version 2.3 <https://github.com/chatmail/relay/pull/800>
-* try to use "build machine" and "deployment server"  consistently  <https://github.com/chatmail/relay/pull/797>
-* adds instructions for migrating control machines <https://github.com/chatmail/relay/pull/795>
-* use consistent naming schema in getting started <https://github.com/chatmail/relay/pull/793>
-* remove jsok/serialize-workflow-action dependency <https://github.com/chatmail/relay/pull/790>
-* streamline migration guide wording, provide titled steps  <https://github.com/chatmail/relay/pull/789>
-* increases default max mailbox size <https://github.com/chatmail/relay/pull/792>
-* use daemon_name for OpenDKIM sign-verify decision instead of IP <https://github.com/chatmail/relay/pull/784>
-
 ## 1.9.0 2025-12-18
 
 ### Documentation

chatmaild/pyproject.toml

@@ -21,8 +21,7 @@ where = ['src']
 [project.scripts]
 doveauth = "chatmaild.doveauth:main"
 chatmail-metadata = "chatmaild.metadata:main"
-chatmail-expire = "chatmaild.expire:daily_expire_main"
-chatmail-quota-expire = "chatmaild.expire:quota_expire_main"
+chatmail-expire = "chatmaild.expire:main"
 chatmail-fsreport = "chatmaild.fsreport:main"
 lastlogin = "chatmaild.lastlogin:main"
 turnserver = "chatmaild.turnserver:main"

chatmaild/src/chatmaild/config.py

@@ -1,3 +1,4 @@
+import os
 from pathlib import Path
 
 import iniconfig
@@ -37,18 +38,15 @@ class Config:
         self.filtermail_smtp_port_incoming = int(
             params.get("filtermail_smtp_port_incoming", "10081")
         )
-        self.filtermail_http_port_incoming = int(
-            params.get("filtermail_http_port_incoming", "10082")
-        )
-        self.filtermail_lmtp_port_transport = int(
-            params.get("filtermail_lmtp_port_transport", "10083")
-        )
+        self.filtermail_http_port = int(params.get("filtermail_http_port", "10082"))
         self.postfix_reinject_port = int(params.get("postfix_reinject_port", "10025"))
         self.postfix_reinject_port_incoming = int(
             params.get("postfix_reinject_port_incoming", "10026")
         )
         self.mtail_address = params.get("mtail_address")
         self.disable_ipv6 = params.get("disable_ipv6", "false").lower() == "true"
+        self.addr_v4 = os.environ.get("CHATMAIL_ADDR_V4", "")
+        self.addr_v6 = os.environ.get("CHATMAIL_ADDR_V6", "")
         self.acme_email = params.get("acme_email", "")
         self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true"
         self.imap_compress = params.get("imap_compress", "false").lower() == "true"
@@ -95,11 +93,6 @@ class Config:
         # old unused option (except for first migration from sqlite to maildir store)
         self.passdb_path = Path(params.get("passdb_path", "/home/vmail/passdb.sqlite"))
 
-    @property
-    def max_mailbox_size_mb(self):
-        """Return max_mailbox_size as an integer in megabytes."""
-        return parse_size_mb(self.max_mailbox_size)
-
     def _getbytefile(self):
         return open(self._inipath, "rb")
 
@@ -113,16 +106,6 @@ class Config:
         return User(maildir, addr, password_path, uid="vmail", gid="vmail")
 
 
-def parse_size_mb(limit):
-    """Parse a size string like ``500M`` or ``2G`` and return megabytes."""
-    value = limit.strip().upper().removesuffix("B")
-    if value.endswith("G"):
-        return int(value[:-1]) * 1024
-    if value.endswith("M"):
-        return int(value[:-1])
-    return int(value)
-
-
 def write_initial_config(inipath, mail_domain, overrides):
     """Write out default config file, using the specified config value overrides."""
     content = get_default_config_content(mail_domain, **overrides)

chatmaild/src/chatmaild/expire.py

@@ -4,26 +4,17 @@ Expire old messages and addresses.
 """
 
 import os
-import re
 import shutil
 import sys
 import time
 from argparse import ArgumentParser
 from collections import namedtuple
 from datetime import datetime
-from pathlib import Path
 from stat import S_ISREG
 
 from chatmaild.config import read_config
 
 FileEntry = namedtuple("FileEntry", ("path", "mtime", "size"))
-QuotaFileEntry = namedtuple("QuotaFileEntry", ("mtime", "quota_size", "path"))
-
-# Quota cleanup factor of max_mailbox_size. The mailbox is reset to this size.
-QUOTA_CLEANUP_FACTOR = 0.7
-
-# e.g. "cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S"
-_dovecot_fn_rex = re.compile(r".+/(\d+)\..+,S=(\d+)")
 
 
 def iter_mailboxes(basedir, maxnum):
@@ -83,42 +74,6 @@ class MailboxStat:
         self.extrafiles.sort(key=lambda x: -x.size)
 
 
-def parse_dovecot_filename(relpath):
-    m = _dovecot_fn_rex.match(relpath)
-    if not m:
-        return None
-    return QuotaFileEntry(int(m.group(1)), int(m.group(2)), relpath)
-
-
-def scan_mailbox_messages(mbox):
-    messages = []
-    for sub in ("cur", "new"):
-        for name in os_listdir_if_exists(mbox / sub):
-            if entry := parse_dovecot_filename(f"{sub}/{name}"):
-                messages.append(entry)
-    return messages
-
-
-def expire_to_target(mbox, target_bytes):
-    messages = scan_mailbox_messages(mbox)
-    total_size = sum(m.quota_size for m in messages)
-    # Keep recent 24 hours of messages protected from expiry because
-    # likely something is wrong with interactions on that address
-    # and quota-full signal can help the address owner's device to notice it
-    undeletable_messages_cutoff = time.time() - (3600 * 24)
-    removed = 0
-    for entry in sorted(messages):
-        if total_size <= target_bytes:
-            break
-        if entry.mtime > undeletable_messages_cutoff:
-            break
-        (mbox / entry.path).unlink(missing_ok=True)
-        total_size -= entry.quota_size
-        removed += 1
-
-    return removed
-
-
 def print_info(msg):
     print(msg, file=sys.stderr)
 
@@ -188,19 +143,6 @@ class Expiry:
             else:
                 continue
             changed = True
-
-        target_bytes = (
-            self.config.max_mailbox_size_mb * 1024 * 1024 * QUOTA_CLEANUP_FACTOR
-        )
-        removed = expire_to_target(Path(mbox.basedir), target_bytes)
-        if removed:
-            changed = True
-            self.del_files += removed
-            if self.verbose:
-                print_info(
-                    f"quota-expire: removed {removed} message(s) from {mboxname}"
-                )
-
         if changed:
             self.remove_file(f"{mbox.basedir}/maildirsize")
 
@@ -212,9 +154,9 @@ class Expiry:
         )
 
 
-def daily_expire_main(args=None):
+def main(args=None):
     """Expire mailboxes and messages according to chatmail config"""
-    parser = ArgumentParser(description=daily_expire_main.__doc__)
+    parser = ArgumentParser(description=main.__doc__)
     ini = "/usr/local/lib/chatmaild/chatmail.ini"
     parser.add_argument(
         "chatmail_ini",
@@ -260,33 +202,5 @@ def daily_expire_main(args=None):
     print(exp.get_summary())
 
 
-def quota_expire_main(args=None):
-    """Remove mailbox messages to stay within a megabyte target.
-
-    This entry point is called by dovecot when a quota threshold is passed.
-    """
-
-    parser = ArgumentParser(description=quota_expire_main.__doc__)
-    parser.add_argument(
-        "target_mb",
-        type=int,
-        help="target mailbox size in megabytes",
-    )
-    parser.add_argument(
-        "mailbox_path",
-        type=Path,
-        help="path to a user mailbox",
-    )
-    args = parser.parse_args(args)
-
-    target_bytes = args.target_mb * 1024 * 1024
-
-    removed_count = expire_to_target(args.mailbox_path, target_bytes)
-    if removed_count:
-        (args.mailbox_path / "maildirsize").unlink(missing_ok=True)
-        print(
-            f"quota-expire: removed {removed_count} message(s)"
-            f" from {args.mailbox_path.name}",
-            file=sys.stderr,
-        )
-    return 0
+if __name__ == "__main__":
+    main(sys.argv[1:])

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -18,7 +18,6 @@ max_user_send_per_minute = 60
 max_user_send_burst_size = 10
 
 # maximum mailbox size of a chatmail address
-# Oldest messages will be removed automatically, so mailboxes never run full.
 max_mailbox_size = 500M
 
 # maximum message size for an e-mail in bytes

chatmaild/src/chatmaild/newemail.py

@@ -2,7 +2,6 @@
 
 """CGI script for creating new accounts."""
 
-import ipaddress
 import json
 import secrets
 import string
@@ -15,16 +14,6 @@ ALPHANUMERIC = string.ascii_lowercase + string.digits
 ALPHANUMERIC_PUNCT = string.ascii_letters + string.digits + string.punctuation
 
 
-def wrap_ip(host):
-    if host.startswith("[") and host.endswith("]"):
-        return host
-    try:
-        ipaddress.ip_address(host)
-        return f"[{host}]"
-    except ValueError:
-        return host
-
-
 def create_newemail_dict(config: Config):
     user = "".join(
         secrets.choice(ALPHANUMERIC) for _ in range(config.username_max_length)
@@ -33,7 +22,7 @@ def create_newemail_dict(config: Config):
         secrets.choice(ALPHANUMERIC_PUNCT)
         for _ in range(config.password_min_length + 3)
     )
-    return dict(email=f"{user}@{wrap_ip(config.mail_domain)}", password=f"{password}")
+    return dict(email=f"{user}@{config.mail_domain}", password=f"{password}")
 
 
 def create_dclogin_url(email, password):

chatmaild/src/chatmaild/tests/test_config.py

@@ -1,6 +1,6 @@
 import pytest
 
-from chatmaild.config import parse_size_mb, read_config
+from chatmaild.config import read_config
 
 
 def test_read_config_basic(example_config):
@@ -121,17 +121,3 @@ def test_config_tls_external_bad_format(make_config):
                 "tls_external_cert_and_key": "/only/one/path.pem",
             },
         )
-
-
-def test_parse_size_mb():
-    assert parse_size_mb("500M") == 500
-    assert parse_size_mb("2G") == 2048
-    assert parse_size_mb("  1g  ") == 1024
-    assert parse_size_mb("100MB") == 100
-    assert parse_size_mb("256") == 256
-
-
-def test_max_mailbox_size_mb(make_config):
-    config = make_config("chat.example.org")
-    assert config.max_mailbox_size == "500M"
-    assert config.max_mailbox_size_mb == 500

chatmaild/src/chatmaild/tests/test_delete_inactive_users.py

@@ -1,7 +1,7 @@
 import time
 
 from chatmaild.doveauth import AuthDictProxy
-from chatmaild.expire import daily_expire_main as main_expire
+from chatmaild.expire import main as main_expire
 
 
 def test_login_timestamps(example_config):

chatmaild/src/chatmaild/tests/test_expire.py

@@ -1,7 +1,5 @@
-import itertools
 import os
 import random
-import time
 from datetime import datetime
 from fnmatch import fnmatch
 from pathlib import Path
@@ -11,19 +9,13 @@ import pytest
 from chatmaild.expire import (
     FileEntry,
     MailboxStat,
-    expire_to_target,
     get_file_entry,
     iter_mailboxes,
     os_listdir_if_exists,
-    parse_dovecot_filename,
-    quota_expire_main,
-    scan_mailbox_messages,
 )
-from chatmaild.expire import daily_expire_main as expiry_main
+from chatmaild.expire import main as expiry_main
 from chatmaild.fsreport import main as report_main
 
-MB = 1024 * 1024
-
 
 def fill_mbox(folderdir):
     password = folderdir.joinpath("password")
@@ -204,51 +196,3 @@ def test_os_listdir_if_exists(tmp_path):
     tmp_path.joinpath("x").write_text("hello")
     assert len(os_listdir_if_exists(str(tmp_path))) == 1
     assert len(os_listdir_if_exists(str(tmp_path.joinpath("123123")))) == 0
-
-
-# --- quota expire tests ---
-
-_msg_counter = itertools.count(1)
-
-
-def _create_message(basedir, sub, size, days_old=0, disk_size=None):
-    seq = next(_msg_counter)
-    mtime = int(time.time() - days_old * 86400)
-    name = f"{mtime}.M1P1Q{seq}.hostname,S={size},W={size}:2,S"
-    path = basedir / sub / name
-    path.parent.mkdir(parents=True, exist_ok=True)
-    path.write_bytes(b"x" * (disk_size if disk_size is not None else size))
-    os.utime(path, (mtime, mtime))
-    return path
-
-
-def test_parse_dovecot_filename():
-    e = parse_dovecot_filename("cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S")
-    assert e.path == "cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S"
-    assert e.mtime == 1775324677
-    assert e.quota_size == 3235
-    assert parse_dovecot_filename("cur/msg_without_structure") is None
-
-
-def test_expire_to_target(tmp_path):
-    _create_message(tmp_path, "cur", MB, days_old=10, disk_size=100)
-    _create_message(tmp_path, "new", MB, days_old=5)
-    _create_message(tmp_path, "cur", MB, days_old=0)  # undeletable (<1 hour)
-    assert len(scan_mailbox_messages(tmp_path)) == 3
-    # removes oldest first, uses S= size not disk size
-    removed = expire_to_target(tmp_path, MB)
-    assert removed == 2
-    msgs = scan_mailbox_messages(tmp_path)
-    assert len(msgs) == 1
-    # the surviving message is the fresh undeletable one
-    assert msgs[0].mtime > time.time() - 3600
-
-
-def test_quota_expire_main(tmp_path, capsys):
-    mbox = tmp_path / "user@example.org"
-    _create_message(mbox, "cur", 2 * MB, days_old=5)
-    (mbox / "maildirsize").write_text("x")
-    quota_expire_main([str(1), str(mbox)])
-    _, err = capsys.readouterr()
-    assert "quota-expire: removed 1 message(s) from user@example.org" in err
-    assert not (mbox / "maildirsize").exists()

chatmaild/src/chatmaild/tests/test_migrate_db.py

@@ -48,8 +48,6 @@ def test_migration(tmp_path, example_config, caplog):
     assert passdb_path.stat().st_size > 10000
 
     example_config.passdb_path = passdb_path
-    # ensure logging.info records are captured regardless of global configuration
-    caplog.set_level("INFO")
 
     assert not caplog.records
 

chatmaild/src/chatmaild/tests/test_newmail.py

@@ -19,12 +19,6 @@ def test_create_newemail_dict(example_config):
     assert ac1["password"] != ac2["password"]
 
 
-def test_create_newemail_dict_ip(make_config):
-    config = make_config("1.2.3.4")
-    ac = create_newemail_dict(config)
-    assert ac["email"].endswith("@[1.2.3.4]")
-
-
 def test_create_dclogin_url():
     url = create_dclogin_url("user@example.org", "p@ss w+rd")
     assert url.startswith("dclogin:")

cmdeploy/src/cmdeploy/acmetool/__init__.py

@@ -1,4 +1,6 @@
-from pyinfra.operations import apt, server
+import importlib.resources
+
+from pyinfra.operations import apt, files, server, systemd
 
 from ..basedeploy import Deployer
 
@@ -7,6 +9,9 @@ class AcmetoolDeployer(Deployer):
     def __init__(self, email, domains):
         self.domains = domains
         self.email = email
+        self.need_restart_redirector = False
+        self.need_restart_reconcile_service = False
+        self.need_restart_reconcile_timer = False
 
     def install(self):
         apt.packages(
@@ -14,41 +19,121 @@ class AcmetoolDeployer(Deployer):
             packages=["acmetool"],
         )
 
-        self.remove_file("/etc/cron.d/acmetool")
+        files.file(
+            name="Remove old acmetool cronjob, it is replaced with systemd timer.",
+            path="/etc/cron.d/acmetool",
+            present=False,
+        )
 
-        self.put_executable("acmetool/acmetool.hook", "/etc/acme/hooks/nginx")
-        self.remove_file("/usr/lib/acme/hooks/nginx")
+        files.put(
+            name="Install acmetool hook.",
+            src=importlib.resources.files(__package__)
+            .joinpath("acmetool.hook")
+            .open("rb"),
+            dest="/etc/acme/hooks/nginx",
+            user="root",
+            group="root",
+            mode="755",
+        )
+        files.file(
+            name="Remove acmetool hook from the wrong location where it was previously installed.",
+            path="/usr/lib/acme/hooks/nginx",
+            present=False,
+        )
 
     def configure(self):
-        self.put_template(
-            "acmetool/response-file.yaml.j2",
-            "/var/lib/acme/conf/responses",
+        files.template(
+            src=importlib.resources.files(__package__).joinpath(
+                "response-file.yaml.j2"
+            ),
+            dest="/var/lib/acme/conf/responses",
+            user="root",
+            group="root",
+            mode="644",
             email=self.email,
         )
 
-        self.put_template(
-            "acmetool/target.yaml.j2",
-            "/var/lib/acme/conf/target",
+        files.template(
+            src=importlib.resources.files(__package__).joinpath("target.yaml.j2"),
+            dest="/var/lib/acme/conf/target",
+            user="root",
+            group="root",
+            mode="644",
         )
 
         server.shell(
             name=f"Remove old acmetool desired files for {self.domains[0]}",
             commands=[f"rm -f /var/lib/acme/desired/{self.domains[0]}-*"],
         )
-        self.put_template(
-            "acmetool/desired.yaml.j2",
-            f"/var/lib/acme/desired/{self.domains[0]}",
+        files.template(
+            src=importlib.resources.files(__package__).joinpath("desired.yaml.j2"),
+            dest=f"/var/lib/acme/desired/{self.domains[0]}",  # 0 is mailhost TLD
+            user="root",
+            group="root",
+            mode="644",
             domains=self.domains,
         )
 
-        self.ensure_systemd_unit("acmetool/acmetool-redirector.service")
-        self.ensure_systemd_unit("acmetool/acmetool-reconcile.service")
-        self.ensure_systemd_unit("acmetool/acmetool-reconcile.timer")
+        service_file = files.put(
+            src=importlib.resources.files(__package__).joinpath(
+                "acmetool-redirector.service"
+            ),
+            dest="/etc/systemd/system/acmetool-redirector.service",
+            user="root",
+            group="root",
+            mode="644",
+        )
+        self.need_restart_redirector = service_file.changed
+
+        reconcile_service_file = files.put(
+            src=importlib.resources.files(__package__).joinpath(
+                "acmetool-reconcile.service"
+            ),
+            dest="/etc/systemd/system/acmetool-reconcile.service",
+            user="root",
+            group="root",
+            mode="644",
+        )
+        self.need_restart_reconcile_service = reconcile_service_file.changed
+
+        reconcile_timer_file = files.put(
+            src=importlib.resources.files(__package__).joinpath(
+                "acmetool-reconcile.timer"
+            ),
+            dest="/etc/systemd/system/acmetool-reconcile.timer",
+            user="root",
+            group="root",
+            mode="644",
+        )
+        self.need_restart_reconcile_timer = reconcile_timer_file.changed
 
     def activate(self):
-        self.ensure_service("acmetool-redirector.service")
-        self.ensure_service("acmetool-reconcile.service", running=False, enabled=False)
-        self.ensure_service("acmetool-reconcile.timer")
+        systemd.service(
+            name="Setup acmetool-redirector service",
+            service="acmetool-redirector.service",
+            running=True,
+            enabled=True,
+            restarted=self.need_restart_redirector,
+        )
+        self.need_restart_redirector = False
+
+        systemd.service(
+            name="Setup acmetool-reconcile service",
+            service="acmetool-reconcile.service",
+            running=False,
+            enabled=False,
+            daemon_reload=self.need_restart_reconcile_service,
+        )
+        self.need_restart_reconcile_service = False
+
+        systemd.service(
+            name="Setup acmetool-reconcile timer",
+            service="acmetool-reconcile.timer",
+            running=True,
+            enabled=True,
+            daemon_reload=self.need_restart_reconcile_timer,
+        )
+        self.need_restart_reconcile_timer = False
 
         server.shell(
             name=f"Reconcile certificates for: {', '.join(self.domains)}",

cmdeploy/src/cmdeploy/basedeploy.py

@@ -3,9 +3,6 @@ import io
 import os
 from contextlib import contextmanager
 
-from pyinfra import host
-from pyinfra.facts.files import Sha256File
-from pyinfra.facts.server import Command
 from pyinfra.operations import files, server, systemd
 
 
@@ -14,17 +11,6 @@ def has_systemd():
     return os.path.isdir("/run/systemd/system")
 
 
-def is_in_container() -> bool:
-    """Return True if running inside a container (Docker, LXC, etc.)."""
-    return (
-        host.get_fact(
-            Command,
-            "systemd-detect-virt --container --quiet 2>/dev/null && echo yes || true",
-        )
-        == "yes"
-    )
-
-
 @contextmanager
 def blocked_service_startup():
     """Prevent services from auto-starting during package installation.
@@ -51,10 +37,11 @@ def get_resource(arg, pkg=__package__):
     return importlib.resources.files(pkg).joinpath(arg)
 
 
-def configure_remote_units(deployer, mail_domain, units) -> None:
+def configure_remote_units(mail_domain, units) -> None:
     remote_base_dir = "/usr/local/lib/chatmaild"
     remote_venv_dir = f"{remote_base_dir}/venv"
     remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini"
+    root_owned = dict(user="root", group="root", mode="644")
 
     # install systemd units
     for fn in units:
@@ -70,13 +57,15 @@ def configure_remote_units(deployer, mail_domain, units) -> None:
         source_path = get_resource(f"service/{basename}.f")
         content = source_path.read_text().format(**params).encode()
 
-        deployer.put_file(
+        files.put(
+            name=f"Upload {basename}",
             src=io.BytesIO(content),
             dest=f"/etc/systemd/system/{basename}",
+            **root_owned,
         )
 
 
-def activate_remote_units(deployer, units) -> None:
+def activate_remote_units(units) -> None:
     # activate systemd units
     for fn in units:
         basename = fn if "." in fn else f"{fn}.service"
@@ -86,8 +75,14 @@ def activate_remote_units(deployer, units) -> None:
             enabled = False
         else:
             enabled = True
-
-        deployer.ensure_service(basename, running=enabled, enabled=enabled)
+        systemd.service(
+            name=f"Setup {basename}",
+            service=basename,
+            running=enabled,
+            enabled=enabled,
+            restarted=enabled,
+            daemon_reload=True,
+        )
 
 
 class Deployment:
@@ -133,7 +128,6 @@ class Deployment:
 
 class Deployer:
     need_restart = False
-    daemon_reload = False
 
     def install(self):
         pass
@@ -143,113 +137,3 @@ class Deployer:
 
     def activate(self):
         pass
-
-    def ensure_service(self, service, running=True, enabled=True):
-        if running:
-            verb = "Start and enable"
-        else:
-            verb = "Stop"
-        systemd.service(
-            name=f"{verb} {service}",
-            service=service,
-            running=running,
-            enabled=enabled,
-            restarted=self.need_restart if running else False,
-            daemon_reload=self.daemon_reload,
-        )
-        self.daemon_reload = False
-
-    def ensure_systemd_unit(self, src, **kwargs):
-        dest_name = src.split("/")[-1].replace(".j2", "")
-        dest = f"/etc/systemd/system/{dest_name}"
-        if src.endswith(".j2"):
-            return self.put_template(src, dest, **kwargs)
-        return self.put_file(src, dest)
-
-    def put_file(self, src, dest, mode="644"):
-        if isinstance(src, str):
-            src = get_resource(src)
-        res = files.put(
-            name=f"Upload {dest}",
-            src=src,
-            dest=dest,
-            user="root",
-            group="root",
-            mode=mode,
-        )
-
-        return self._update_restart_signals(dest, res)
-
-    def put_executable(self, src, dest):
-        return self.put_file(src, dest, mode="755")
-
-    def put_template(self, src, dest, owner="root", **kwargs):
-        if isinstance(src, str):
-            src = get_resource(src)
-        res = files.template(
-            name=f"Upload {dest}",
-            src=src,
-            dest=dest,
-            user=owner,
-            group=owner,
-            mode="644",
-            **kwargs,
-        )
-
-        return self._update_restart_signals(dest, res)
-
-    def remove_file(self, dest):
-        res = files.file(name=f"Remove {dest}", path=dest, present=False)
-        return self._update_restart_signals(dest, res)
-
-    def ensure_line(self, path, line, **kwargs):
-        name = kwargs.pop("name", f"Ensure line in {path}")
-        res = files.line(name=name, path=path, line=line, **kwargs)
-        return self._update_restart_signals(path, res)
-
-    def ensure_directory(self, path, owner="root", mode="755", **kwargs):
-        name = kwargs.pop("name", f"Ensure directory {path}")
-        res = files.directory(
-            name=name,
-            path=path,
-            user=owner,
-            group=owner,
-            mode=mode,
-            present=True,
-            **kwargs,
-        )
-        return self._update_restart_signals(path, res)
-
-    def remove_directory(self, path, **kwargs):
-        name = kwargs.pop("name", f"Remove directory {path}")
-        res = files.directory(name=name, path=path, present=False, **kwargs)
-        return self._update_restart_signals(path, res)
-
-    def download_executable(self, url, dest, sha256sum, extract=None):
-        existing = host.get_fact(Sha256File, dest)
-        if existing == sha256sum:
-            return
-
-        tmp = f"{dest}.new"
-        if extract:
-            dl_cmd = f"curl -fSL {url} | {extract} >{tmp}"
-        else:
-            dl_cmd = f"curl -fSL {url} -o {tmp}"
-
-        server.shell(
-            name=f"Download {dest}",
-            commands=[
-                f"({dl_cmd}"
-                f" && echo '{sha256sum}  {tmp}' | sha256sum -c"
-                f" && mv {tmp} {dest})",
-                f"chmod 755 {dest}",
-            ],
-        )
-        self.need_restart = True
-
-    def _update_restart_signals(self, path, res):
-        if res.changed:
-            self.need_restart = True
-            if str(path).startswith("/etc/systemd/system/"):
-                self.daemon_reload = True
-        return res

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -101,11 +101,16 @@ def run_cmd(args, out):
     env["CHATMAIL_WEBSITE_ONLY"] = "True" if args.website_only else ""
     env["CHATMAIL_DISABLE_MAIL"] = "True" if args.disable_mail else ""
     env["CHATMAIL_REQUIRE_IROH"] = "True" if require_iroh else ""
+    if not args.dns_check_disabled:
+        env["CHATMAIL_ADDR_V4"] = remote_data.get("A") or ""
+        env["CHATMAIL_ADDR_V6"] = remote_data.get("AAAA") or ""
     deploy_path = importlib.resources.files(__package__).joinpath("run.py").resolve()
     pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
 
     cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y"
-    if ssh_host == "localhost":
+    if ssh_host in ["localhost", "@docker"]:
+        if ssh_host == "@docker":
+            env["CHATMAIL_NOPORTCHECK"] = "True"
         cmd = f"{pyinf} @local {deploy_path} -y"
 
     if version.parse(pyinfra.__version__) < version.parse("3"):
@@ -191,6 +196,12 @@ def status_cmd(args, out):
 
 
 def test_cmd_options(parser):
+    parser.add_argument(
+        "--slow",
+        dest="slow",
+        action="store_true",
+        help="also run slow tests",
+    )
     add_ssh_host_option(parser)
 
 
@@ -212,6 +223,8 @@ def test_cmd(args, out):
         "-v",
         "--durations=5",
     ]
+    if args.slow:
+        pytest_args.append("--slow")
     ret = out.run_ret(pytest_args, env=env)
     return ret
 
@@ -303,7 +316,7 @@ def add_ssh_host_option(parser):
     parser.add_argument(
         "--ssh-host",
         dest="ssh_host",
-        help="Run commands on 'localhost' or on a specific SSH host "
+        help="Run commands on 'localhost', via '@docker', or on a specific SSH host "
         "instead of chatmail.ini's mail_domain.",
     )
 
@@ -363,12 +376,14 @@ def get_parser():
     return parser
 
 
-def get_sshexec(ssh_host: str, verbose=True):
+def get_sshexec(ssh_host: str, verbose=True, **kwargs):
     if ssh_host in ["localhost", "@local"]:
-        return LocalExec(verbose)
+        return LocalExec(verbose, docker=False)
+    elif ssh_host == "@docker":
+        return LocalExec(verbose, docker=True)
     if verbose:
         print(f"[ssh] login to {ssh_host}")
-    return SSHExec(ssh_host, verbose=verbose)
+    return SSHExec(ssh_host, verbose=verbose, **kwargs)
 
 
 def main(args=None):

cmdeploy/src/cmdeploy/deployers.py

@@ -2,6 +2,7 @@
 Chat Mail pyinfra deploy.
 """
 
+import os
 import shutil
 import subprocess
 import sys
@@ -12,6 +13,7 @@ from chatmaild.config import read_config
 from pyinfra import facts, host, logger
 from pyinfra.api import FactBase
 from pyinfra.facts import hardware
+from pyinfra.facts.files import Sha256File
 from pyinfra.facts.systemd import SystemdEnabled
 from pyinfra.operations import apt, files, pip, server, systemd
 
@@ -24,8 +26,8 @@ from .basedeploy import (
     activate_remote_units,
     blocked_service_startup,
     configure_remote_units,
+    get_resource,
     has_systemd,
-    is_in_container,
 )
 from .dovecot.deployer import DovecotDeployer
 from .external.deployer import ExternalTlsDeployer
@@ -80,22 +82,25 @@ def remove_legacy_artifacts():
         )
 
 
-def _install_remote_venv_with_chatmaild(deployer) -> None:
+def _install_remote_venv_with_chatmaild() -> None:
     remove_legacy_artifacts()
     dist_file = _build_chatmaild(dist_dir=Path("chatmaild/dist"))
     remote_base_dir = "/usr/local/lib/chatmaild"
     remote_dist_file = f"{remote_base_dir}/dist/{dist_file.name}"
     remote_venv_dir = f"{remote_base_dir}/venv"
+    root_owned = dict(user="root", group="root", mode="644")
 
     apt.packages(
         name="apt install python3-virtualenv",
         packages=["python3-virtualenv"],
     )
 
-    deployer.ensure_directory(f"{remote_base_dir}/dist")
-    deployer.put_file(
+    files.put(
+        name="Upload chatmaild source package",
         src=dist_file.open("rb"),
         dest=remote_dist_file,
+        create_remote_dir=True,
+        **root_owned,
     )
 
     pip.virtualenv(
@@ -117,24 +122,37 @@ def _install_remote_venv_with_chatmaild(deployer) -> None:
     )
 
 
-def _configure_remote_venv_with_chatmaild(deployer, config) -> None:
+def _configure_remote_venv_with_chatmaild(config) -> None:
     remote_base_dir = "/usr/local/lib/chatmaild"
     remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini"
+    root_owned = dict(user="root", group="root", mode="644")
 
-    deployer.put_file(
+    files.put(
+        name=f"Upload {remote_chatmail_inipath}",
         src=config._getbytefile(),
         dest=remote_chatmail_inipath,
+        **root_owned,
     )
 
-    deployer.remove_file("/etc/cron.d/chatmail-metrics")
-    deployer.remove_file("/var/www/html/metrics")
+    files.file(
+        path="/etc/cron.d/chatmail-metrics",
+        present=False,
+    )
+    files.file(
+        path="/var/www/html/metrics",
+        present=False,
+    )
 
 
 class UnboundDeployer(Deployer):
     def __init__(self, config):
         self.config = config
+        self.need_restart = False
 
     def install(self):
+        # Run local DNS resolver `unbound`. `resolvconf` takes care of
+        # setting up /etc/resolv.conf to use 127.0.0.1 as the resolver.
+
         # On an IPv4-only system, if unbound is started but not configured,
         # it causes subsequent steps to fail to resolve hosts.
         with blocked_service_startup():
@@ -144,27 +162,6 @@ class UnboundDeployer(Deployer):
             )
 
     def configure(self):
-        # Remove dynamic resolver managers that compete for /etc/resolv.conf.
-        apt.packages(
-            name="Purge resolvconf",
-            packages=["resolvconf"],
-            present=False,
-            extra_uninstall_args="--purge",
-        )
-        # systemd-resolved can't be purged due to dependencies; stop and mask.
-        server.shell(
-            name="Stop and mask systemd-resolved",
-            commands=[
-                "systemctl stop systemd-resolved.service || true",
-                "systemctl mask systemd-resolved.service",
-            ],
-        )
-        # Configure unbound resolver with Quad9 fallback and a trailing newline
-        # (SolusVM bug).
-        self.put_file(
-            src=BytesIO(b"nameserver 127.0.0.1\nnameserver 9.9.9.9\n"),
-            dest="/etc/resolv.conf",
-        )
         server.shell(
             name="Generate root keys for validating DNSSEC",
             commands=[
@@ -172,15 +169,26 @@ class UnboundDeployer(Deployer):
             ],
         )
         if self.config.disable_ipv6:
-            self.ensure_directory(
+            files.directory(
                 path="/etc/unbound/unbound.conf.d",
+                present=True,
+                user="root",
+                group="root",
+                mode="755",
             )
-            self.put_template(
-                "unbound/unbound.conf.j2",
-                "/etc/unbound/unbound.conf.d/chatmail.conf",
+            conf = files.put(
+                src=get_resource("unbound/unbound.conf.j2"),
+                dest="/etc/unbound/unbound.conf.d/chatmail.conf",
+                user="root",
+                group="root",
+                mode="644",
             )
         else:
-            self.remove_file("/etc/unbound/unbound.conf.d/chatmail.conf")
+            conf = files.file(
+                path="/etc/unbound/unbound.conf.d/chatmail.conf",
+                present=False,
+            )
+        self.need_restart |= conf.changed
 
     def activate(self):
         server.shell(
@@ -190,25 +198,27 @@ class UnboundDeployer(Deployer):
             ],
         )
 
-        self.ensure_service("unbound.service")
-
-        self.ensure_service(
-            "unbound-resolvconf.service",
-            running=False,
-            enabled=False,
+        systemd.service(
+            name="Start and enable unbound",
+            service="unbound.service",
+            running=True,
+            enabled=True,
+            restarted=self.need_restart,
         )
 
 
 class MtastsDeployer(Deployer):
     def configure(self):
         # Remove configuration.
-        self.remove_file("/etc/mta-sts-daemon.yml")
-        self.remove_directory("/usr/local/lib/postfix-mta-sts-resolver")
-        self.remove_file("/etc/systemd/system/mta-sts-daemon.service")
+        files.file("/etc/mta-sts-daemon.yml", present=False)
+        files.directory("/usr/local/lib/postfix-mta-sts-resolver", present=False)
+        files.file("/etc/systemd/system/mta-sts-daemon.service", present=False)
 
     def activate(self):
-        self.ensure_service(
-            "mta-sts-daemon.service",
+        systemd.service(
+            name="Stop MTA-STS daemon",
+            service="mta-sts-daemon.service",
+            daemon_reload=True,
             running=False,
             enabled=False,
         )
@@ -219,7 +229,14 @@ class WebsiteDeployer(Deployer):
         self.config = config
 
     def install(self):
-        self.ensure_directory("/var/www")
+        files.directory(
+            name="Ensure /var/www exists",
+            path="/var/www",
+            user="root",
+            group="root",
+            mode="755",
+            present=True,
+        )
 
     def configure(self):
         www_path, src_dir, build_dir = get_paths(self.config)
@@ -249,11 +266,15 @@ class LegacyRemoveDeployer(Deployer):
 
         # remove historic expunge script
         # which is now implemented through a systemd timer (chatmail-expire)
-        self.remove_file("/etc/cron.d/expunge")
+        files.file(
+            path="/etc/cron.d/expunge",
+            present=False,
+        )
 
         # Remove OBS repository key that is no longer used.
-        self.remove_file("/etc/apt/keyrings/obs-home-deltachat.gpg")
-        self.ensure_line(
+        files.file("/etc/apt/keyrings/obs-home-deltachat.gpg", present=False)
+        files.line(
+            name="Remove DeltaChat OBS home repository from sources.list",
             path="/etc/apt/sources.list",
             line="deb [signed-by=/etc/apt/keyrings/obs-home-deltachat.gpg] https://download.opensuse.org/repositories/home:/deltachat/Debian_12/ ./",
             escape_regex_characters=True,
@@ -261,7 +282,11 @@ class LegacyRemoveDeployer(Deployer):
         )
 
         # prior relay versions used filelogging
-        self.remove_directory("/var/log/journal/")
+        files.directory(
+            name="Ensure old logs on disk are deleted",
+            path="/var/log/journal/",
+            present=False,
+        )
         # remove echobot if it is still running
         if has_systemd() and host.get_fact(SystemdEnabled).get("echobot.service"):
             systemd.service(
@@ -303,13 +328,22 @@ class TurnDeployer(Deployer):
                 "0fb3e792419494e21ecad536464929dba706bb2c88884ed8f1788141d26fc756",
             ),
         }[host.get_fact(facts.server.Arch)]
-        self.download_executable(url, "/usr/local/bin/chatmail-turn", sha256sum)
+
+        existing_sha256sum = host.get_fact(Sha256File, "/usr/local/bin/chatmail-turn")
+        if existing_sha256sum != sha256sum:
+            server.shell(
+                name="Download chatmail-turn",
+                commands=[
+                    f"(curl -L {url} >/usr/local/bin/chatmail-turn.new && (echo '{sha256sum} /usr/local/bin/chatmail-turn.new' | sha256sum -c) && mv /usr/local/bin/chatmail-turn.new /usr/local/bin/chatmail-turn)",
+                    "chmod 755 /usr/local/bin/chatmail-turn",
+                ],
+            )
 
     def configure(self):
-        configure_remote_units(self, self.mail_domain, self.units)
+        configure_remote_units(self.mail_domain, self.units)
 
     def activate(self):
-        activate_remote_units(self, self.units)
+        activate_remote_units(self.units)
 
 
 class IrohDeployer(Deployer):
@@ -327,30 +361,72 @@ class IrohDeployer(Deployer):
                 "f8ef27631fac213b3ef668d02acd5b3e215292746a3fc71d90c63115446008b1",
             ),
         }[host.get_fact(facts.server.Arch)]
-        self.download_executable(
-            url,
-            "/usr/local/bin/iroh-relay",
-            sha256sum,
-            extract="gunzip | tar -xf - ./iroh-relay -O",
-        )
+
+        existing_sha256sum = host.get_fact(Sha256File, "/usr/local/bin/iroh-relay")
+        if existing_sha256sum != sha256sum:
+            server.shell(
+                name="Download iroh-relay",
+                commands=[
+                    f"(curl -L {url} | gunzip | tar -x -f - ./iroh-relay -O >/usr/local/bin/iroh-relay.new && (echo '{sha256sum} /usr/local/bin/iroh-relay.new' | sha256sum -c) && mv /usr/local/bin/iroh-relay.new /usr/local/bin/iroh-relay)",
+                    "chmod 755 /usr/local/bin/iroh-relay",
+                ],
+            )
+
+            self.need_restart = True
 
     def configure(self):
-        self.ensure_systemd_unit("iroh-relay.service")
-        self.put_file("iroh-relay.toml", "/etc/iroh-relay.toml")
+        systemd_unit = files.put(
+            name="Upload iroh-relay systemd unit",
+            src=get_resource("iroh-relay.service"),
+            dest="/etc/systemd/system/iroh-relay.service",
+            user="root",
+            group="root",
+            mode="644",
+        )
+        self.need_restart |= systemd_unit.changed
+
+        iroh_config = files.put(
+            name="Upload iroh-relay config",
+            src=get_resource("iroh-relay.toml"),
+            dest="/etc/iroh-relay.toml",
+            user="root",
+            group="root",
+            mode="644",
+        )
+        self.need_restart |= iroh_config.changed
 
     def activate(self):
-        self.ensure_service(
-            "iroh-relay.service",
+        systemd.service(
+            name="Start and enable iroh-relay",
+            service="iroh-relay.service",
+            running=True,
             enabled=self.enable_iroh_relay,
+            restarted=self.need_restart,
         )
+        self.need_restart = False
 
 
 class JournaldDeployer(Deployer):
     def configure(self):
-        self.put_file("journald.conf", "/etc/systemd/journald.conf")
+        journald_conf = files.put(
+            name="Configure journald",
+            src=get_resource("journald.conf"),
+            dest="/etc/systemd/journald.conf",
+            user="root",
+            group="root",
+            mode="644",
+        )
+        self.need_restart = journald_conf.changed
 
     def activate(self):
-        self.ensure_service("systemd-journald.service")
+        systemd.service(
+            name="Start and enable journald",
+            service="systemd-journald.service",
+            running=True,
+            enabled=True,
+            restarted=self.need_restart,
+        )
+        self.need_restart = False
 
 
 class ChatmailVenvDeployer(Deployer):
@@ -366,14 +442,14 @@ class ChatmailVenvDeployer(Deployer):
         )
 
     def install(self):
-        _install_remote_venv_with_chatmaild(self)
+        _install_remote_venv_with_chatmaild()
 
     def configure(self):
-        _configure_remote_venv_with_chatmaild(self, self.config)
-        configure_remote_units(self, self.config.mail_domain, self.units)
+        _configure_remote_venv_with_chatmaild(self.config)
+        configure_remote_units(self.config.mail_domain, self.units)
 
     def activate(self):
-        activate_remote_units(self, self.units)
+        activate_remote_units(self.units)
 
 
 class ChatmailDeployer(Deployer):
@@ -387,9 +463,13 @@ class ChatmailDeployer(Deployer):
         self.mail_domain = config.mail_domain
 
     def install(self):
-        self.put_file(
+        files.put(
+            name="Disable installing recommended packages globally",
             src=BytesIO(b'APT::Install-Recommends "false";\n'),
             dest="/etc/apt/apt.conf.d/00InstallRecommends",
+            user="root",
+            group="root",
+            mode="644",
         )
         apt.update(name="apt update", cache_time=24 * 3600)
         apt.upgrade(name="upgrade apt packages", auto_remove=True)
@@ -406,10 +486,13 @@ class ChatmailDeployer(Deployer):
 
     def configure(self):
         # metadata crashes if the mailboxes dir does not exist
-        self.ensure_directory(
-            str(self.config.mailboxes_dir),
-            owner="vmail",
+        files.directory(
+            name="Ensure vmail mailbox directory exists",
+            path=str(self.config.mailboxes_dir),
+            user="vmail",
+            group="vmail",
             mode="700",
+            present=True,
         )
 
         # This file is used by auth proxy.
@@ -430,7 +513,12 @@ class FcgiwrapDeployer(Deployer):
         )
 
     def activate(self):
-        self.ensure_service("fcgiwrap.service")
+        systemd.service(
+            name="Start and enable fcgiwrap",
+            service="fcgiwrap.service",
+            running=True,
+            enabled=True,
+        )
 
 
 class GithashDeployer(Deployer):
@@ -443,7 +531,12 @@ class GithashDeployer(Deployer):
             git_diff = subprocess.check_output(["git", "diff"]).decode()
         except Exception:
             git_diff = ""
-        self.put_file(src=StringIO(git_hash + git_diff), dest="/etc/chatmail-version")
+        files.put(
+            name="Upload chatmail relay git commit hash",
+            src=StringIO(git_hash + git_diff),
+            dest="/etc/chatmail-version",
+            mode="700",
+        )
 
 
 def get_tls_deployer(config, mail_domain):
@@ -475,21 +568,23 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
         Deployment().perform_stages([WebsiteDeployer(config)])
         return
 
+    if host.get_fact(Port, port=53) != "unbound":
+        files.line(
+            name="Add 9.9.9.9 to resolv.conf",
+            path="/etc/resolv.conf",
+            # Guard against resolv.conf missing a trailing newline (SolusVM bug).
+            line="\nnameserver 9.9.9.9",
+        )
+
     # Check if mtail_address interface is available (if configured)
-    if config.mtail_address and config.mtail_address not in (
-        "127.0.0.1",
-        "::1",
-        "localhost",
-    ):
+    if config.mtail_address and config.mtail_address not in ('127.0.0.1', '::1', 'localhost'):
         ipv4_addrs = host.get_fact(hardware.Ipv4Addrs)
         all_addresses = [addr for addrs in ipv4_addrs.values() for addr in addrs]
         if config.mtail_address not in all_addresses:
-            Out().red(
-                f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n"
-            )
+            Out().red(f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n")
             exit(1)
 
-    if not is_in_container():
+    if not os.environ.get("CHATMAIL_NOPORTCHECK"):
         port_services = [
             (["master", "smtpd"], 25),
             ("unbound", 53),

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -5,18 +5,17 @@ from chatmaild.config import Config
 from pyinfra import host
 from pyinfra.facts.deb import DebPackages
 from pyinfra.facts.server import Arch, Command, Sysctl
-from pyinfra.operations import apt, files, server
+from pyinfra.operations import apt, files, server, systemd
 
 from cmdeploy.basedeploy import (
     Deployer,
     activate_remote_units,
     blocked_service_startup,
     configure_remote_units,
-    is_in_container,
+    get_resource,
 )
 
-DOVECOT_ARCHIVE_VERSION = "2.3.21+dfsg1-3"
-DOVECOT_PACKAGE_VERSION = f"1:{DOVECOT_ARCHIVE_VERSION}"
+DOVECOT_VERSION = "2.3.21+dfsg1-3"
 
 DOVECOT_SHA256 = {
     ("core", "amd64"): "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d",
@@ -41,14 +40,11 @@ class DovecotDeployer(Deployer):
         with blocked_service_startup():
             debs = []
             for pkg in ("core", "imapd", "lmtpd"):
-                deb, changed = _download_dovecot_package(pkg, arch)
-                self.need_restart |= changed
+                deb = _download_dovecot_package(pkg, arch)
                 if deb:
                     debs.append(deb)
             if debs:
                 deb_list = " ".join(debs)
-                # First dpkg may fail on missing dependencies (stderr suppressed);
-                # apt-get --fix-broken pulls them in, then dpkg retries cleanly.
                 server.shell(
                     name="Install dovecot packages",
                     commands=[
@@ -57,40 +53,39 @@ class DovecotDeployer(Deployer):
                         f"dpkg --force-confdef --force-confold -i {deb_list}",
                     ],
                 )
-                self.need_restart = True
-        self.put_file(
+        files.put(
+            name="Pin dovecot packages to block Debian dist-upgrades",
             src=io.StringIO(
                 "Package: dovecot-*\n"
                 "Pin: version *\n"
                 "Pin-Priority: -1\n"
             ),
             dest="/etc/apt/preferences.d/pin-dovecot",
+            user="root",
+            group="root",
+            mode="644",
         )
 
     def configure(self):
-        configure_remote_units(self, self.config.mail_domain, self.units)
-        _configure_dovecot(self, self.config)
+        configure_remote_units(self.config.mail_domain, self.units)
+        self.need_restart, self.daemon_reload = _configure_dovecot(self.config)
 
     def activate(self):
-        activate_remote_units(self, self.units)
+        activate_remote_units(self.units)
 
-        # Detect stale binary: package installed but service still runs old (deleted) binary.
-        if not self.disable_mail and not self.need_restart:
-            stale = host.get_fact(
-                Command,
-                'pid=$(systemctl show -p MainPID --value dovecot.service 2>/dev/null);'
-                ' [ "${pid:-0}" != "0" ] && readlink "/proc/$pid/exe" 2>/dev/null | grep -q "(deleted)"'
-                " && echo STALE || true",
-            )
-            if stale == "STALE":
-                self.need_restart = True
+        restart = False if self.disable_mail else self.need_restart
 
-        active = not self.disable_mail
-        self.ensure_service(
-            "dovecot.service",
-            running=active,
-            enabled=active,
+        systemd.service(
+            name="Disable dovecot for now"
+            if self.disable_mail
+            else "Start and enable Dovecot",
+            service="dovecot.service",
+            running=False if self.disable_mail else True,
+            enabled=False if self.disable_mail else True,
+            restarted=restart,
+            daemon_reload=self.daemon_reload,
         )
+        self.need_restart = False
 
 
 def _pick_url(primary, fallback):
@@ -102,22 +97,22 @@ def _pick_url(primary, fallback):
         return fallback
 
 
-def _download_dovecot_package(package: str, arch: str) -> tuple[str | None, bool]:
-    """Download a dovecot .deb if needed, return (path, changed)."""
+def _download_dovecot_package(package: str, arch: str):
+    """Download a dovecot .deb if needed, return its path (or None)."""
     arch = "amd64" if arch == "x86_64" else arch
     arch = "arm64" if arch == "aarch64" else arch
 
     pkg_name = f"dovecot-{package}"
     sha256 = DOVECOT_SHA256.get((package, arch))
     if sha256 is None:
-        op = apt.packages(packages=[pkg_name])
-        return None, bool(getattr(op, "changed", False))
+        apt.packages(packages=[pkg_name])
+        return None
 
     installed_versions = host.get_fact(DebPackages).get(pkg_name, [])
-    if DOVECOT_PACKAGE_VERSION in installed_versions:
-        return None, False
+    if DOVECOT_VERSION in installed_versions:
+        return None
 
-    url_version = DOVECOT_ARCHIVE_VERSION.replace("+", "%2B")
+    url_version = DOVECOT_VERSION.replace("+", "%2B")
     deb_base = f"{pkg_name}_{url_version}_{arch}.deb"
     primary_url = f"https://download.delta.chat/dovecot/{deb_base}"
     fallback_url = f"https://github.com/chatmail/dovecot/releases/download/upstream%2F{url_version}/{deb_base}"
@@ -132,28 +127,59 @@ def _download_dovecot_package(package: str, arch: str) -> tuple[str | None, bool
         cache_time=60 * 60 * 24 * 365 * 10,  # never redownload the package
     )
 
-    return deb_filename, True
+    return deb_filename
 
-def _configure_dovecot(deployer, config: Config, debug: bool = False):
+
+def _can_set_inotify_limits() -> bool:
+    is_container = (
+        host.get_fact(
+            Command,
+            "systemd-detect-virt --container --quiet 2>/dev/null && echo yes || true",
+        )
+        == "yes"
+    )
+    return not is_container
+
+
+def _configure_dovecot(config: Config, debug: bool = False) -> tuple[bool, bool]:
     """Configures Dovecot IMAP server."""
-    deployer.put_template(
-        "dovecot/dovecot.conf.j2",
-        "/etc/dovecot/dovecot.conf",
+    need_restart = False
+    daemon_reload = False
+
+    main_config = files.template(
+        src=get_resource("dovecot/dovecot.conf.j2"),
+        dest="/etc/dovecot/dovecot.conf",
+        user="root",
+        group="root",
+        mode="644",
         config=config,
         debug=debug,
         disable_ipv6=config.disable_ipv6,
     )
-    deployer.put_file("dovecot/auth.conf", "/etc/dovecot/auth.conf")
-    deployer.put_file(
-        "dovecot/push_notification.lua", "/etc/dovecot/push_notification.lua"
+    need_restart |= main_config.changed
+    auth_config = files.put(
+        src=get_resource("dovecot/auth.conf"),
+        dest="/etc/dovecot/auth.conf",
+        user="root",
+        group="root",
+        mode="644",
     )
+    need_restart |= auth_config.changed
+    lua_push_notification_script = files.put(
+        src=get_resource("dovecot/push_notification.lua"),
+        dest="/etc/dovecot/push_notification.lua",
+        user="root",
+        group="root",
+        mode="644",
+    )
+    need_restart |= lua_push_notification_script.changed
 
     # as per https://doc.dovecot.org/2.3/configuration_manual/os/
     # it is recommended to set the following inotify limits
-    can_modify = not is_in_container()
+    can_modify = _can_set_inotify_limits()
     for name in ("max_user_instances", "max_user_watches"):
         key = f"fs.inotify.{name}"
-        value = host.get_fact(Sysctl).get(key, 0)
+        value = host.get_fact(Sysctl)[key]
         if value > 65534:
             continue
         if not can_modify:
@@ -170,20 +196,25 @@ def _configure_dovecot(deployer, config: Config, debug: bool = False):
             persist=True,
         )
 
-    deployer.ensure_line(
+    timezone_env = files.line(
         name="Set TZ environment variable",
         path="/etc/environment",
         line="TZ=:/etc/localtime",
     )
+    need_restart |= timezone_env.changed
 
-    deployer.put_file(
-        "service/10_restart_on_failure.conf",
-        "/etc/systemd/system/dovecot.service.d/10_restart.conf",
+    restart_conf = files.put(
+        name="dovecot: restart automatically on failure",
+        src=get_resource("service/10_restart.conf"),
+        dest="/etc/systemd/system/dovecot.service.d/10_restart.conf",
     )
+    daemon_reload |= restart_conf.changed
 
     # Validate dovecot configuration before restart
-    if deployer.need_restart:
+    if need_restart:
         server.shell(
             name="Validate dovecot configuration",
             commands=["doveconf -n >/dev/null"],
         )
+
+    return need_restart, daemon_reload

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -133,11 +133,6 @@ protocol lmtp {
   # mail_lua and push_notification_lua are needed for Lua push notification handler.
   # <https://doc.dovecot.org/2.3/configuration_manual/push_notification/#configuration>
   mail_plugins = $mail_plugins mail_lua notify push_notification push_notification_lua
-
-  # Disable fsync for LMTP. May lose delivered message,
-  # but unlikely to cause problems with multiple relays.
-  # https://doc.dovecot.org/2.3/admin_manual/mailbox_formats/#fsyncing
-  mail_fsync = never
 }
 
 plugin {
@@ -149,26 +144,12 @@ plugin {
 }
 
 plugin {
+  # for now we define static quota-rules for all users 
   quota = maildir:User quota
+  quota_rule = *:storage={{ config.max_mailbox_size }}
   quota_max_mail_size={{ config.max_message_size }}
   quota_grace = 0
-
-  quota_rule = *:storage={{ config.max_mailbox_size_mb }}M
-
-  # Trigger at 75%% of quota, expire oldest messages down to 70%%.
-  # The percentages are chosen to prevent current Delta Chat users
-  # from seeing "quota warnings" which trigger at 80% and 95%.
-
-  quota_warning = storage=75%% quota-warning {{ config.max_mailbox_size_mb * 70 // 100 }} {{ config.mailboxes_dir }}/%u
-}
-
-service quota-warning {
-  executable = script /usr/local/lib/chatmaild/venv/bin/chatmail-quota-expire
-  user = vmail
-  unix_listener quota-warning {
-     user = vmail
-     mode = 0600
-  }
+  # quota_over_flag_value = TRUE
 }
 
 # push_notification configuration
@@ -271,9 +252,6 @@ protocol imap {
   # sort -sn <(sed 's/ / C: /' *.in) <(sed 's/ / S: /' cat *.out)
 
   rawlog_dir = %h 
-
-  # Disable fsync for IMAP. May lose IMAP changes like setting flags. 
-  mail_fsync = never
 } 
 {% endif %}
 

cmdeploy/src/cmdeploy/external/deployer.py

@@ -1,8 +1,10 @@
+import io
 
 from pyinfra import host
 from pyinfra.facts.files import File
+from pyinfra.operations import files, systemd
 
-from ..basedeploy import Deployer
+from cmdeploy.basedeploy import Deployer, get_resource
 
 
 class ExternalTlsDeployer(Deployer):
@@ -21,24 +23,45 @@ class ExternalTlsDeployer(Deployer):
     def configure(self):
         # Verify cert and key exist on the remote host using pyinfra facts.
         for path in (self.cert_path, self.key_path):
-                if host.get_fact(File, path=path) is None:
-                    raise Exception(f"External TLS file not found on server: {path}")
+            info = host.get_fact(File, path=path)
+            if info is None:
+                raise Exception(f"External TLS file not found on server: {path}")
 
-        self.ensure_systemd_unit(
-            "external/tls-cert-reload.path.j2",
-            cert_path=self.cert_path,
+        # Deploy the .path unit (templated with the cert path).
+        # pkg=__package__ is required here because the resource files
+        # live in cmdeploy.external, not the default cmdeploy package.
+        source = get_resource("tls-cert-reload.path.f", pkg=__package__)
+        content = source.read_text().format(cert_path=self.cert_path).encode()
+
+        path_unit = files.put(
+            name="Upload tls-cert-reload.path",
+            src=io.BytesIO(content),
+            dest="/etc/systemd/system/tls-cert-reload.path",
+            user="root",
+            group="root",
+            mode="644",
         )
-        self.ensure_systemd_unit(
-            "external/tls-cert-reload.service",
+
+        service_unit = files.put(
+            name="Upload tls-cert-reload.service",
+            src=get_resource("tls-cert-reload.service", pkg=__package__),
+            dest="/etc/systemd/system/tls-cert-reload.service",
+            user="root",
+            group="root",
+            mode="644",
         )
 
+        if path_unit.changed or service_unit.changed:
+            self.need_restart = True
+
     def activate(self):
-        # No explicit reload needed here: dovecot/nginx read the cert
-        # on startup, and the .path watcher handles live changes.
-        self.ensure_service(
-            "tls-cert-reload.path",
+        systemd.service(
+            name="Enable tls-cert-reload path watcher",
+            service="tls-cert-reload.path",
             running=True,
             enabled=True,
+            restarted=self.need_restart,
+            daemon_reload=self.need_restart,
         )
-
-
+        # No explicit reload needed here: dovecot/nginx read the cert
+        # on startup, and the .path watcher handles live changes.

cmdeploy/src/cmdeploy/external/tls-cert-reload.path.f

@@ -9,7 +9,7 @@
 Description=Watch TLS certificate for changes
 
 [Path]
-PathChanged={{ cert_path }}
+PathChanged={cert_path}
 
 [Install]
 WantedBy=multi-user.target

cmdeploy/src/cmdeploy/filtermail/deployer.py

@@ -1,40 +1,52 @@
-import os
-
 from pyinfra import facts, host
+from pyinfra.operations import files, systemd
 
-from cmdeploy.basedeploy import Deployer
+from cmdeploy.basedeploy import Deployer, get_resource
 
 
 class FiltermailDeployer(Deployer):
-    services = ["filtermail", "filtermail-incoming", "filtermail-transport"]
+    services = ["filtermail", "filtermail-incoming"]
     bin_path = "/usr/local/bin/filtermail"
     config_path = "/usr/local/lib/chatmaild/chatmail.ini"
 
-    def install(self):
-        local_bin = os.environ.get("CHATMAIL_FILTERMAIL_BINARY")
-        if local_bin:
-            self.put_executable(
-                src=local_bin,
-                dest=self.bin_path,
-            )
-            return
+    def __init__(self):
+        self.need_restart = False
 
+    def install(self):
         arch = host.get_fact(facts.server.Arch)
-        url = f"https://github.com/chatmail/filtermail/releases/download/v0.6.4/filtermail-{arch}"
+        url = f"https://github.com/chatmail/filtermail/releases/download/v0.6.1/filtermail-{arch}"
         sha256sum = {
-            "x86_64": "5295115952c72e4c4ec3c85546e094b4155a4c702c82bd71fcdcb744dc73adf6",
-            "aarch64": "6892244f17b8f26ccb465766e96028e7222b3c8adefca9fc6bfe9ff332ca8dff",
+            "x86_64": "48b3fb80c092d00b9b0a0ef77a8673496da3b9aed5ec1851e1df936d5589d62f",
+            "aarch64": "c65bd5f45df187d3d65d6965a285583a3be0f44a6916ff12909ff9a8d702c22e",
         }[arch]
-        self.download_executable(url, self.bin_path, sha256sum)
+        self.need_restart |= files.download(
+            name="Download filtermail",
+            src=url,
+            sha256sum=sha256sum,
+            dest=self.bin_path,
+            mode="755",
+        ).changed
 
     def configure(self):
         for service in self.services:
-            self.ensure_systemd_unit(
-                f"filtermail/{service}.service.j2",
+            self.need_restart |= files.template(
+                src=get_resource(f"filtermail/{service}.service.j2"),
+                dest=f"/etc/systemd/system/{service}.service",
+                user="root",
+                group="root",
+                mode="644",
                 bin_path=self.bin_path,
                 config_path=self.config_path,
-            )
+            ).changed
 
     def activate(self):
         for service in self.services:
-            self.ensure_service(f"{service}.service")
+            systemd.service(
+                name=f"Start and enable {service}",
+                service=f"{service}.service",
+                running=True,
+                enabled=True,
+                restarted=self.need_restart,
+                daemon_reload=True,
+            )
+        self.need_restart = False

cmdeploy/src/cmdeploy/filtermail/filtermail-transport.service.j2

@@ -1,11 +0,0 @@
-[Unit]
-Description=Chatmail transport service
-
-[Service]
-ExecStart={{ bin_path }} {{ config_path }} transport
-Restart=always
-RestartSec=30
-User=vmail
-
-[Install]
-WantedBy=multi-user.target

cmdeploy/src/cmdeploy/mtail/delivered_mail.mtail

@@ -78,11 +78,3 @@ counter rejected_unencrypted_mail_count
 /Rejected unencrypted mail/ {
   rejected_unencrypted_mail_count++
 }
-
-counter quota_expire_runs
-counter quota_expire_removed_files
-
-/quota-expire: removed (?P<count>\d+) message\(s\)/ {
-  quota_expire_runs++
-  quota_expire_removed_files += $count
-}

cmdeploy/src/cmdeploy/mtail/deployer.py

@@ -1,7 +1,10 @@
 from pyinfra import facts, host
-from pyinfra.operations import apt
+from pyinfra.operations import apt, files, server, systemd
 
-from cmdeploy.basedeploy import Deployer
+from cmdeploy.basedeploy import (
+    Deployer,
+    get_resource,
+)
 
 
 class MtailDeployer(Deployer):
@@ -15,30 +18,51 @@ class MtailDeployer(Deployer):
         (url, sha256sum) = {
             "x86_64": (
                 "https://github.com/google/mtail/releases/download/v3.0.8/mtail_3.0.8_linux_amd64.tar.gz",
-                "d55cb601049c5e61eabab29998dbbcea95d480e5448544f9470337ba2eea882e",
+                "123c2ee5f48c3eff12ebccee38befd2233d715da736000ccde49e3d5607724e4",
             ),
             "aarch64": (
                 "https://github.com/google/mtail/releases/download/v3.0.8/mtail_3.0.8_linux_arm64.tar.gz",
-                "f748db8ad2a1e0b63684d4c8868cf6a373a20f7e6922e5ece601fff0ee00eb1a",
+                "aa04811c0929b6754408676de520e050c45dddeb3401881888a092c9aea89cae",
             ),
         }[host.get_fact(facts.server.Arch)]
-        self.download_executable(
-            url,
-            "/usr/local/bin/mtail",
-            sha256sum,
-            extract="gunzip | tar -xf - mtail -O",
+
+        server.shell(
+            name="Download mtail",
+            commands=[
+                f"(echo '{sha256sum} /usr/local/bin/mtail' | sha256sum -c) || (curl -L {url} | gunzip | tar -x -f - mtail -O >/usr/local/bin/mtail.new && mv /usr/local/bin/mtail.new /usr/local/bin/mtail)",
+                "chmod 755 /usr/local/bin/mtail",
+            ],
         )
 
     def configure(self):
         # Using our own systemd unit instead of `/usr/lib/systemd/system/mtail.service`.
         # This allows to read from journalctl instead of log files.
-        self.ensure_systemd_unit(
-            "mtail/mtail.service.j2",
+        files.template(
+            src=get_resource("mtail/mtail.service.j2"),
+            dest="/etc/systemd/system/mtail.service",
+            user="root",
+            group="root",
+            mode="644",
             address=self.mtail_address or "127.0.0.1",
             port=3903,
         )
-        self.put_file("mtail/delivered_mail.mtail", "/etc/mtail/delivered_mail.mtail")
+
+        mtail_conf = files.put(
+            name="Mtail configuration",
+            src=get_resource("mtail/delivered_mail.mtail"),
+            dest="/etc/mtail/delivered_mail.mtail",
+            user="root",
+            group="root",
+            mode="644",
+        )
+        self.need_restart = mtail_conf.changed
 
     def activate(self):
-        active = bool(self.mtail_address)
-        self.ensure_service("mtail.service", running=active, enabled=active)
+        systemd.service(
+            name="Start and enable mtail",
+            service="mtail.service",
+            running=bool(self.mtail_address),
+            enabled=bool(self.mtail_address),
+            restarted=self.need_restart,
+        )
+        self.need_restart = False

cmdeploy/src/cmdeploy/mtail/mtail.service.j2

@@ -1,13 +1,10 @@
 [Unit]
 Description=mtail
-After=network-online.target
-Wants=network-online.target
 
 [Service]
 Type=simple
 ExecStart=/bin/sh -c "journalctl -f -o short-iso -n 0 | /usr/local/bin/mtail --address={{ address }} --port={{ port }} --progs /etc/mtail --logtostderr --logs -"
 Restart=on-failure
-RestartSec=2s
 
 [Install]
 WantedBy=multi-user.target

cmdeploy/src/cmdeploy/nginx/deployer.py

@@ -1,5 +1,5 @@
 from chatmaild.config import Config
-from pyinfra.operations import apt
+from pyinfra.operations import apt, files, systemd
 
 from cmdeploy.basedeploy import (
     Deployer,
@@ -31,50 +31,87 @@ class NginxDeployer(Deployer):
         # For documentation about policy-rc.d, see:
         # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
         #
-        self.put_executable(src="policy-rc.d", dest="/usr/sbin/policy-rc.d")
+        files.put(
+            src=get_resource("policy-rc.d"),
+            dest="/usr/sbin/policy-rc.d",
+            user="root",
+            group="root",
+            mode="755",
+        )
 
         apt.packages(
             name="Install nginx",
             packages=["nginx", "libnginx-mod-stream"],
         )
 
-        self.remove_file("/usr/sbin/policy-rc.d")
+        files.file("/usr/sbin/policy-rc.d", present=False)
 
     def configure(self):
-        _configure_nginx(self, self.config)
+        self.need_restart = _configure_nginx(self.config)
 
     def activate(self):
-        self.ensure_service("nginx.service")
+        systemd.service(
+            name="Start and enable nginx",
+            service="nginx.service",
+            running=True,
+            enabled=True,
+            restarted=self.need_restart,
+        )
+        self.need_restart = False
 
 
-def _configure_nginx(deployer, config: Config, debug: bool = False):
+def _configure_nginx(config: Config, debug: bool = False) -> bool:
     """Configures nginx HTTP server."""
+    need_restart = False
 
-    deployer.put_template(
-        "nginx/nginx.conf.j2",
-        "/etc/nginx/nginx.conf",
+    main_config = files.template(
+        src=get_resource("nginx/nginx.conf.j2"),
+        dest="/etc/nginx/nginx.conf",
+        user="root",
+        group="root",
+        mode="644",
         config=config,
         disable_ipv6=config.disable_ipv6,
     )
+    need_restart |= main_config.changed
 
-    deployer.put_template(
-        "nginx/autoconfig.xml.j2",
-        "/var/www/html/.well-known/autoconfig/mail/config-v1.1.xml",
+    autoconfig = files.template(
+        src=get_resource("nginx/autoconfig.xml.j2"),
+        dest="/var/www/html/.well-known/autoconfig/mail/config-v1.1.xml",
+        user="root",
+        group="root",
+        mode="644",
         config=config,
     )
+    need_restart |= autoconfig.changed
 
-    deployer.put_template(
-        "nginx/mta-sts.txt.j2",
-        "/var/www/html/.well-known/mta-sts.txt",
+    mta_sts_config = files.template(
+        src=get_resource("nginx/mta-sts.txt.j2"),
+        dest="/var/www/html/.well-known/mta-sts.txt",
+        user="root",
+        group="root",
+        mode="644",
         config=config,
     )
+    need_restart |= mta_sts_config.changed
 
     # install CGI newemail script
     #
     cgi_dir = "/usr/lib/cgi-bin"
-    deployer.ensure_directory(cgi_dir)
+    files.directory(
+        name=f"Ensure {cgi_dir} exists",
+        path=cgi_dir,
+        user="root",
+        group="root",
+    )
 
-    deployer.put_executable(
+    files.put(
+        name="Upload cgi newemail.py script",
         src=get_resource("newemail.py", pkg="chatmaild").open("rb"),
         dest=f"{cgi_dir}/newemail.py",
+        user="root",
+        group="root",
+        mode="755",
     )
+
+    return need_restart

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -73,8 +73,8 @@ http {
 
 		access_log syslog:server=unix:/dev/log,facility=local7;
 
-		location /mxdeliv {
-		    proxy_pass http://127.0.0.1:{{ config.filtermail_http_port_incoming }};
+		location /mxdeliv/ {
+		    proxy_pass http://127.0.0.1:{{ config.filtermail_http_port }};
 		}
 
 		location / {

cmdeploy/src/cmdeploy/opendkim/deployer.py

@@ -4,9 +4,9 @@ Installs OpenDKIM
 
 from pyinfra import host
 from pyinfra.facts.files import File
-from pyinfra.operations import apt, files, server
+from pyinfra.operations import apt, files, server, systemd
 
-from cmdeploy.basedeploy import Deployer
+from cmdeploy.basedeploy import Deployer, get_resource
 
 
 class OpendkimDeployer(Deployer):
@@ -25,39 +25,65 @@ class OpendkimDeployer(Deployer):
         domain = self.mail_domain
         dkim_selector = "opendkim"
         """Configures OpenDKIM"""
+        need_restart = False
 
-        self.put_template(
-            "opendkim/opendkim.conf",
-            "/etc/opendkim.conf",
+        main_config = files.template(
+            src=get_resource("opendkim/opendkim.conf"),
+            dest="/etc/opendkim.conf",
+            user="root",
+            group="root",
+            mode="644",
             config={"domain_name": domain, "opendkim_selector": dkim_selector},
         )
+        need_restart |= main_config.changed
 
-        self.remove_file("/etc/opendkim/screen.lua")
-        self.remove_file("/etc/opendkim/final.lua")
+        screen_script = files.file(
+            path="/etc/opendkim/screen.lua",
+            present=False,
+        )
+        need_restart |= screen_script.changed
 
-        self.ensure_directory(
-            "/etc/opendkim",
-            owner="opendkim",
+        final_script = files.file(
+            path="/etc/opendkim/final.lua",
+            present=False,
+        )
+        need_restart |= final_script.changed
+
+        files.directory(
+            name="Add opendkim directory to /etc",
+            path="/etc/opendkim",
+            user="opendkim",
+            group="opendkim",
             mode="750",
+            present=True,
         )
 
-        self.put_template(
-            "opendkim/KeyTable",
-            "/etc/dkimkeys/KeyTable",
-            owner="opendkim",
+        keytable = files.template(
+            src=get_resource("opendkim/KeyTable"),
+            dest="/etc/dkimkeys/KeyTable",
+            user="opendkim",
+            group="opendkim",
+            mode="644",
             config={"domain_name": domain, "opendkim_selector": dkim_selector},
         )
+        need_restart |= keytable.changed
 
-        self.put_template(
-            "opendkim/SigningTable",
-            "/etc/dkimkeys/SigningTable",
-            owner="opendkim",
+        signing_table = files.template(
+            src=get_resource("opendkim/SigningTable"),
+            dest="/etc/dkimkeys/SigningTable",
+            user="opendkim",
+            group="opendkim",
+            mode="644",
             config={"domain_name": domain, "opendkim_selector": dkim_selector},
         )
-        self.ensure_directory(
-            "/var/spool/postfix/opendkim",
-            owner="opendkim",
+        need_restart |= signing_table.changed
+        files.directory(
+            name="Add opendkim socket directory to /var/spool/postfix",
+            path="/var/spool/postfix/opendkim",
+            user="opendkim",
+            group="opendkim",
             mode="750",
+            present=True,
         )
 
         if not host.get_fact(File, f"/etc/dkimkeys/{dkim_selector}.private"):
@@ -70,10 +96,12 @@ class OpendkimDeployer(Deployer):
                 _su_user="opendkim",
             )
 
-        self.put_file(
-            "opendkim/systemd.conf",
-            "/etc/systemd/system/opendkim.service.d/10-prevent-memory-leak.conf",
+        service_file = files.put(
+            name="Configure opendkim to restart once a day",
+            src=get_resource("opendkim/systemd.conf"),
+            dest="/etc/systemd/system/opendkim.service.d/10-prevent-memory-leak.conf",
         )
+        need_restart |= service_file.changed
 
         files.file(
             name="chown opendkim: /etc/dkimkeys/opendkim.private",
@@ -82,5 +110,15 @@ class OpendkimDeployer(Deployer):
             group="opendkim",
         )
 
+        self.need_restart = need_restart
+
     def activate(self):
-        self.ensure_service("opendkim.service")
+        systemd.service(
+            name="Start and enable OpenDKIM",
+            service="opendkim.service",
+            running=True,
+            enabled=True,
+            daemon_reload=self.need_restart,
+            restarted=self.need_restart,
+        )
+        self.need_restart = False

cmdeploy/src/cmdeploy/postfix/deployer.py

@@ -1,10 +1,11 @@
-from pyinfra.operations import apt, server
+from pyinfra.operations import apt, files, server, systemd
 
-from cmdeploy.basedeploy import Deployer
+from cmdeploy.basedeploy import Deployer, get_resource
 
 
 class PostfixDeployer(Deployer):
     required_users = [("postfix", None, ["opendkim"])]
+    daemon_reload = False
 
     def __init__(self, config, disable_mail):
         self.config = config
@@ -18,46 +19,81 @@ class PostfixDeployer(Deployer):
 
     def configure(self):
         config = self.config
+        need_restart = False
 
-        self.put_template(
-            "postfix/main.cf.j2",
-            "/etc/postfix/main.cf",
+        main_config = files.template(
+            src=get_resource("postfix/main.cf.j2"),
+            dest="/etc/postfix/main.cf",
+            user="root",
+            group="root",
+            mode="644",
             config=config,
             disable_ipv6=config.disable_ipv6,
         )
+        need_restart |= main_config.changed
 
-        self.put_template(
-            "postfix/master.cf.j2",
-            "/etc/postfix/master.cf",
+        master_config = files.template(
+            src=get_resource("postfix/master.cf.j2"),
+            dest="/etc/postfix/master.cf",
+            user="root",
+            group="root",
+            mode="644",
             debug=False,
             config=config,
         )
+        need_restart |= master_config.changed
 
-        self.put_file(
-            "postfix/submission_header_cleanup",
-            "/etc/postfix/submission_header_cleanup",
+        header_cleanup = files.put(
+            src=get_resource("postfix/submission_header_cleanup"),
+            dest="/etc/postfix/submission_header_cleanup",
+            user="root",
+            group="root",
+            mode="644",
         )
-        self.put_file("postfix/lmtp_header_cleanup", "/etc/postfix/lmtp_header_cleanup")
+        need_restart |= header_cleanup.changed
 
-        res = self.put_file(
-            "postfix/smtp_tls_policy_map", "/etc/postfix/smtp_tls_policy_map"
+        lmtp_header_cleanup = files.put(
+            src=get_resource("postfix/lmtp_header_cleanup"),
+            dest="/etc/postfix/lmtp_header_cleanup",
+            user="root",
+            group="root",
+            mode="644",
         )
-        tls_policy_changed = res.changed
-        if tls_policy_changed:
+        need_restart |= lmtp_header_cleanup.changed
+
+        tls_policy_map = files.put(
+            name="Upload SMTP TLS Policy that accepts self-signed certificates for IP-only hosts",
+            src=get_resource("postfix/smtp_tls_policy_map"),
+            dest="/etc/postfix/smtp_tls_policy_map",
+            user="root",
+            group="root",
+            mode="644",
+        )
+        need_restart |= tls_policy_map.changed
+        if tls_policy_map.changed:
             server.shell(
                 commands=["postmap /etc/postfix/smtp_tls_policy_map"],
             )
 
         # Login map that 1:1 maps email address to login.
-        self.put_file("postfix/login_map", "/etc/postfix/login_map")
-
-        self.put_file(
-            "service/10_restart_on_failure.conf",
-            "/etc/systemd/system/postfix@.service.d/10_restart.conf",
+        login_map = files.put(
+            src=get_resource("postfix/login_map"),
+            dest="/etc/postfix/login_map",
+            user="root",
+            group="root",
+            mode="644",
         )
+        need_restart |= login_map.changed
+
+        restart_conf = files.put(
+            name="postfix: restart automatically on failure",
+            src=get_resource("service/10_restart.conf"),
+            dest="/etc/systemd/system/postfix@.service.d/10_restart.conf",
+        )
+        self.daemon_reload = restart_conf.changed
 
         # Validate postfix configuration before restart
-        if self.need_restart:
+        if need_restart:
             server.shell(
                 name="Validate postfix configuration",
                 # Extract stderr and quit with error if non-zero
@@ -65,11 +101,19 @@ class PostfixDeployer(Deployer):
                     """bash -c 'w=$(postconf 2>&1 >/dev/null); [[ -z "$w" ]] || { echo "$w"; false; }'"""
                 ],
             )
+        self.need_restart = need_restart
 
     def activate(self):
-        active = not self.disable_mail
-        self.ensure_service(
-            "postfix.service",
-            running=active,
-            enabled=active,
+        restart = False if self.disable_mail else self.need_restart
+
+        systemd.service(
+            name="disable postfix for now"
+            if self.disable_mail
+            else "Start and enable Postfix",
+            service="postfix.service",
+            running=False if self.disable_mail else True,
+            enabled=False if self.disable_mail else True,
+            restarted=restart,
+            daemon_reload=self.daemon_reload,
         )
+        self.need_restart = False

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -69,6 +69,15 @@ mynetworks = 127.0.0.0/8
 {% else %}
 mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128
 {% endif %}
+{% if config.addr_v4 %}
+smtp_bind_address = {{ config.addr_v4 }}
+{% endif %}
+{% if config.addr_v6 %}
+smtp_bind_address6 = {{ config.addr_v6 }}
+{% endif %}
+{% if config.addr_v4 or config.addr_v6 %}
+smtp_bind_address_enforce = yes
+{% endif %}
 mailbox_size_limit = 0
 message_size_limit = {{config.max_message_size}}
 recipient_delimiter = +
@@ -79,6 +88,22 @@ inet_protocols = ipv4
 inet_protocols = all
 {% endif %}
 
+# Postfix does not try IPv4 and IPv6 connections
+# concurrently as of version 3.7.11.
+#
+# When relay has both A (IPv4) and AAAA (IPv6) records,
+# but broken IPv6 connectivity,
+# every second message is delayed by the connection timeout
+# <https://www.postfix.org/postconf.5.html#smtp_connect_timeout>
+# which defaults to 30 seconds. Reducing timeouts is not a solution
+# as this will result in a failure to connect to slow servers.
+#
+# As a workaround we always prefer IPv4 when it is available.
+# 
+# The setting is documented at
+# <https://www.postfix.org/postconf.5.html#smtp_address_preference>
+smtp_address_preference=ipv4
+
 virtual_transport = lmtp:unix:private/dovecot-lmtp
 virtual_mailbox_domains = {{ config.mail_domain }}
 lmtp_header_checks = regexp:/etc/postfix/lmtp_header_cleanup
@@ -93,12 +118,3 @@ smtpd_sender_login_maps = regexp:/etc/postfix/login_map
 # Do not lookup SMTP client hostnames to reduce delays
 # and avoid unnecessary DNS requests.
 smtpd_peername_lookup = no
-
-# Use filtermail-transport to relay messages.
-# We can't force postfix to split messages per destination,
-# when specifying a custom next-hop,
-# so instead this is handled in filtermail.
-# We use LMTP instead SMTP so we can communicate per-recipient errors back to postfix.
-default_transport = lmtp-filtermail:inet:[127.0.0.1]:{{ config.filtermail_lmtp_port_transport }}
-lmtp-filtermail_initial_destination_concurrency=10000
-lmtp-filtermail_destination_concurrency_limit=10000

cmdeploy/src/cmdeploy/postfix/master.cf.j2

@@ -51,12 +51,15 @@ smtps     inet  n       -       y       -       5000    smtpd
   -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
 #628       inet  n       -       y       -       -       qmqpd
 pickup    unix  n       -       y       60      1       pickup
+ -o cleanup_service_name=signlocal
 cleanup   unix  n       -       y       -       0       cleanup
 qmgr      unix  n       -       n       300     1       qmgr
 #qmgr     unix  n       -       n       300     1       oqmgr
 tlsmgr    unix  -       -       y       1000?   1       tlsmgr
 rewrite   unix  -       -       y       -       -       trivial-rewrite
 bounce    unix  -       -       y       -       0       bounce
+  -o internal_mail_filter_classes=bounce
+  -o cleanup_service_name=signlocal
 defer     unix  -       -       y       -       0       bounce
 trace     unix  -       -       y       -       0       bounce
 verify    unix  -       -       y       -       1       verify
@@ -101,7 +104,8 @@ filter    unix -        n       n       -       -       lmtp
 authclean unix  n       -       -       -       0       cleanup
   -o header_checks=regexp:/etc/postfix/submission_header_cleanup
 
-lmtp-filtermail unix  -       -       y       -       10000       lmtp
-  -o syslog_name=postfix/lmtp-filtermail
-  -o lmtp_header_checks=
-  -o lmtp_tls_security_level=none
+# Signs locally generated bounce messages.
+# These can't be signed using smtpd as they are not non-smtpd.
+signlocal unix  n       -       -       -       0       cleanup
+  -o milter_macro_daemon_name=ORIGINATING
+  -o non_smtpd_milters=unix:opendkim/opendkim.sock

cmdeploy/src/cmdeploy/remote/rdns.py

@@ -64,25 +64,21 @@ def get_dkim_entry(mail_domain, pre_command, dkim_selector):
     )
 
 
-def get_authoritative_ns(domain):
-    ns_replies = [
+def query_dns(typ, domain):
+    # Get autoritative nameserver from the SOA record.
+    soa_answers = [
         x.split()
         for x in shell(
-            f"dig -r -q {domain} -t NS +noall +authority +answer", print=log_progress
+            f"dig -r -q {domain} -t SOA +noall +authority +answer", print=log_progress
         ).split("\n")
     ]
-    filtered_replies = [a for a in ns_replies if len(a) >= 5 and a[3] in ("SOA", "NS")]
-    if not filtered_replies:
+    soa = [a for a in soa_answers if len(a) >= 3 and a[3] == "SOA"]
+    if not soa:
         return
-    return filtered_replies[0][4]
-
-
-def query_dns(typ, domain):
-    ns = get_authoritative_ns(domain)
+    ns = soa[0][4]
 
     # Query authoritative nameserver directly to bypass DNS cache.
-    direct_ns = f"@{ns}" if ns else ""
-    res = shell(f"dig {direct_ns} -r -q {domain} -t {typ} +short", print=log_progress)
+    res = shell(f"dig @{ns} -r -q {domain} -t {typ} +short", print=log_progress)
     return next((line for line in res.split("\n") if not line.startswith(";")), "")
 
 

cmdeploy/src/cmdeploy/selfsigned/deployer.py

@@ -1,8 +1,8 @@
 import shlex
 
-from pyinfra.operations import server
+from pyinfra.operations import apt, server
 
-from ..basedeploy import Deployer
+from cmdeploy.basedeploy import Deployer
 
 
 def openssl_selfsigned_args(domain, cert_path, key_path, days=36500):
@@ -34,7 +34,11 @@ class SelfSignedTlsDeployer(Deployer):
         self.cert_path = "/etc/ssl/certs/mailserver.pem"
         self.key_path = "/etc/ssl/private/mailserver.key"
 
-
+    def install(self):
+        apt.packages(
+            name="Install openssl",
+            packages=["openssl"],
+        )
 
     def configure(self):
         args = openssl_selfsigned_args(
@@ -48,5 +52,3 @@ class SelfSignedTlsDeployer(Deployer):
 
     def activate(self):
         pass
-
-

cmdeploy/src/cmdeploy/sshexec.py

@@ -49,8 +49,9 @@ class SSHExec:
     RemoteError = execnet.RemoteError
     FuncError = FuncError
 
-    def __init__(self, host, verbose=False, python="python3", timeout=60):
-        self.gateway = execnet.makegateway(f"ssh=root@{host}//python={python}")
+    def __init__(self, host, verbose=False, python="python3", timeout=60, ssh_options=None):
+        ssh_options = f"{ssh_options.strip()} " if ssh_options is not None else ""
+        self.gateway = execnet.makegateway(f"ssh={ssh_options}root@{host}//python={python}")
         self._remote_cmdloop_channel = bootstrap_remote(self.gateway, remote)
         self.timeout = timeout
         self.verbose = verbose
@@ -87,8 +88,9 @@ class SSHExec:
 class LocalExec:
     FuncError = FuncError
 
-    def __init__(self, verbose=False):
+    def __init__(self, verbose=False, docker=False):
         self.verbose = verbose
+        self.docker = docker
 
     def __call__(self, call, kwargs=None, log_callback=None):
         if kwargs is None:
@@ -100,6 +102,10 @@ class LocalExec:
         if not title:
             title = call.__name__
         where = "locally"
+        if self.docker:
+            if call == remote.rdns.perform_initial_checks:
+                kwargs["pre_command"] = "docker exec chatmail "
+                where = "in docker"
         if self.verbose:
             print_stderr(f"Running {where}: {title}(**{kwargs})")
             return self(call, kwargs, log_callback=print_stderr)

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -8,6 +8,7 @@ import pytest
 
 from cmdeploy import remote
 from cmdeploy.cmdeploy import get_sshexec
+from cmdeploy.sshexec import FuncError
 
 
 class TestSSHExecutor:
@@ -71,44 +72,6 @@ class TestSSHExecutor:
         assert (now - since_date).total_seconds() < 60 * 60 * 51
 
 
-def test_dovecot_main_process_matches_installed_binary(sshdomain):
-    sshexec = get_sshexec(sshdomain)
-    main_pid = int(
-        sshexec(
-            call=remote.rshell.shell,
-            kwargs=dict(
-                command="timeout 10 systemctl show -p MainPID --value dovecot.service"
-            ),
-        ).strip()
-    )
-    assert main_pid != 0, "dovecot.service MainPID is 0 -- service not running?"
-
-    exe = sshexec(
-        call=remote.rshell.shell,
-        kwargs=dict(command=f"timeout 10 readlink /proc/{main_pid}/exe"),
-    ).strip()
-    status_text = sshexec(
-        call=remote.rshell.shell,
-        kwargs=dict(
-            command="timeout 10 systemctl show -p StatusText --value dovecot.service"
-        ),
-    ).strip()
-    installed_version = sshexec(
-        call=remote.rshell.shell, kwargs=dict(command="timeout 10 dovecot --version")
-    ).strip()
-
-    assert not exe.endswith("(deleted)"), (
-        f"running dovecot binary was deleted (stale after upgrade): {exe}"
-    )
-    expected_status_text = f"v{installed_version}"
-    assert status_text == expected_status_text or status_text.startswith(
-        f"{expected_status_text} "
-    ), (
-        f"dovecot status version mismatch: "
-        f"StatusText={status_text!r}, installed={installed_version!r}"
-    )
-
-
 def test_timezone_env(remote):
     for line in remote.iter_output("env"):
         print(line)
@@ -155,6 +118,33 @@ def test_reject_forged_from(cmsetup, maildata, gencreds, lp, forgeaddr):
     assert "500" in str(e.value)
 
 
+@pytest.mark.slow
+def test_bounces_are_signed(cmsetup, cmsetup2, maildata, sshdomain2):
+    """Test that bounce messages are dkim signed"""
+
+    dkim_rejects_dir = "/tmp/filtermail-rejected/dkim-verify"
+    sshexec2 = get_sshexec(sshdomain2, ssh_options="-oStrictHostKeyChecking=accept-new")
+    sshexec2(call=remote.rdns.shell, kwargs=dict(command=f"rm -rf {dkim_rejects_dir}"))
+
+    our_user = cmsetup.gen_users(1)[0]
+    other_user = cmsetup2.gen_users(1)[0]
+    msg = maildata("encrypted.eml", from_addr=other_user.addr, to_addr=our_user.addr)
+
+    # exceed the 10kB quota of our_user mailbox to trigger a bounce message.
+    def bounce_received():
+        other_user.smtp.sendmail(
+            from_addr=other_user.addr, to_addrs=[our_user.addr], msg=msg.as_string()
+        )
+        out = sshexec2(call=remote.rdns.shell, kwargs=dict(command=f"journalctl -n 5 -u filtermail-incoming"))
+        assert "Filtering unencrypted mail." in out
+
+    try_n_times(20, bounce_received)
+
+    time.sleep(1)
+    # if bounce was dkim-signed, filtermail shouldn't log the eml.
+    with pytest.raises(FuncError):
+        sshexec2(call=remote.rdns.shell, kwargs=dict(command=f"ls {dkim_rejects_dir}"))
+
 def test_authenticated_from(cmsetup, maildata):
     """Test that envelope FROM must be the same as login."""
     user1, user2, user3 = cmsetup.gen_users(3)
@@ -221,6 +211,7 @@ def test_rewrite_subject(cmsetup, maildata):
     assert "Subject: Unencrypted subject" not in rcvd_msg
 
 
+@pytest.mark.slow
 def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
     """Test that the per-account send-mail limit is exceeded."""
     user1, user2 = cmsetup.gen_users(2)
@@ -243,6 +234,7 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
     pytest.fail("Rate limit was not exceeded")
 
 
+@pytest.mark.slow
 def test_expunged(remote, chatmail_config):
     outdated_days = int(chatmail_config.delete_mails_after) + 1
     find_cmds = [

cmdeploy/src/cmdeploy/tests/plugin.py

@@ -1,5 +1,4 @@
 import imaplib
-import ipaddress
 import itertools
 import os
 import random
@@ -15,12 +14,10 @@ from chatmaild.config import read_config
 conftestdir = Path(__file__).parent
 
 
-def _is_ip(domain):
-    try:
-        ipaddress.ip_address(domain)
-        return True
-    except ValueError:
-        return False
+def pytest_addoption(parser):
+    parser.addoption(
+        "--slow", action="store_true", default=False, help="also run slow tests"
+    )
 
 
 def pytest_configure(config):
@@ -30,6 +27,13 @@ def pytest_configure(config):
     )
 
 
+def pytest_runtest_setup(item):
+    markers = list(item.iter_markers(name="slow"))
+    if markers:
+        if not item.config.getoption("--slow"):
+            pytest.skip("skipping slow test, use --slow to run")
+
+
 def _get_chatmail_config():
     inipath = os.environ.get("CHATMAIL_INI")
     if inipath:
@@ -278,7 +282,6 @@ def gencreds(chatmail_config):
 
     def gen(domain=None):
         domain = domain if domain else chatmail_config.mail_domain
-        addr_domain = f"[{domain}]" if _is_ip(domain) else domain
         while 1:
             num = next(count)
             alphanumeric = "abcdefghijklmnopqrstuvwxyz1234567890"
@@ -292,7 +295,7 @@ def gencreds(chatmail_config):
             password = "".join(
                 random.choices(alphanumeric, k=chatmail_config.password_min_length)
             )
-            yield f"{user}@{addr_domain}", f"{password}"
+            yield f"{user}@{domain}", f"{password}"
 
     return lambda domain=None: next(gen(domain))
 
@@ -341,22 +344,9 @@ class ChatmailACFactory:
         accounts = []
         for _ in range(num):
             account = self.dc.add_account()
-            addr, password = self.gencreds(domain)
-            if _is_ip(domain):
-                # Use DCLOGIN scheme with explicit server hosts,
-                # matching how madmail presents its addresses to users.
-                qr = (
-                    f"dclogin:{addr}"
-                    f"?p={password}&v=1"
-                    f"&ih={domain}&ip=993"
-                    f"&sh={domain}&sp=465"
-                    f"&ic=3&ss=default"
-                )
-                future = account.add_transport_from_qr.future(qr)
-            else:
-                future = account.add_or_update_transport.future(
-                    self._make_transport(domain)
-                )
+            future = account.add_or_update_transport.future(
+                self._make_transport(domain)
+            )
             futures.append(future)
 
             # ensure messages stay in INBOX so that they can be
@@ -466,6 +456,11 @@ def cmsetup(maildomain, gencreds, ssl_context):
     return CMSetup(maildomain, gencreds, ssl_context)
 
 
+@pytest.fixture
+def cmsetup2(maildomain2, gencreds, ssl_context):
+    return CMSetup(maildomain2, gencreds, ssl_context)
+
+
 class CMSetup:
     def __init__(self, maildomain, gencreds, ssl_context):
         self.maildomain = maildomain
@@ -476,7 +471,7 @@ class CMSetup:
         print(f"Creating {num} online users")
         users = []
         for i in range(num):
-            addr, password = self.gencreds()
+            addr, password = self.gencreds(self.maildomain)
             user = CMUser(self.maildomain, addr, password, self.ssl_context)
             assert user.smtp
             users.append(user)

cmdeploy/src/cmdeploy/tests/test_basedeploy.py

@@ -1,118 +0,0 @@
-from unittest.mock import MagicMock, patch
-
-from cmdeploy.basedeploy import Deployer
-
-
-def test_put_file_restart_and_reload():
-    deployer = Deployer()
-    mock_res = MagicMock()
-    mock_res.changed = True
-
-    with patch("cmdeploy.basedeploy.files.put", return_value=mock_res):
-        deployer.put_file("foo.conf", "/etc/foo.conf")
-        assert deployer.need_restart is True
-        assert deployer.daemon_reload is False
-
-        deployer = Deployer()
-
-        deployer.put_file("test.service", "/etc/systemd/system/test.service")
-        assert deployer.need_restart is True
-        assert deployer.daemon_reload is True
-
-
-def test_remove_file():
-    deployer = Deployer()
-    mock_res = MagicMock()
-    mock_res.changed = True
-
-    with patch("cmdeploy.basedeploy.files.file", return_value=mock_res) as mock_file:
-        deployer.remove_file("/etc/foo.conf")
-        mock_file.assert_called_once_with(
-            name="Remove /etc/foo.conf", path="/etc/foo.conf", present=False
-        )
-        assert deployer.need_restart is True
-
-
-def test_ensure_systemd_unit():
-    deployer = Deployer()
-    mock_res = MagicMock()
-    mock_res.changed = True
-
-    # Plain service file
-    with patch("cmdeploy.basedeploy.files.put", return_value=mock_res) as mock_put:
-        deployer.ensure_systemd_unit("iroh-relay.service")
-        assert (
-            mock_put.call_args.kwargs["dest"]
-            == "/etc/systemd/system/iroh-relay.service"
-        )
-        assert deployer.need_restart is True
-        assert deployer.daemon_reload is True
-
-    deployer = Deployer()
-
-    # Template (.j2) dispatches to put_template and strips .j2 suffix
-    with patch("cmdeploy.basedeploy.files.template", return_value=mock_res) as mock_tpl:
-        deployer.ensure_systemd_unit(
-            "filtermail/chatmaild.service.j2",
-            bin_path="/usr/local/bin/filtermail",
-        )
-        assert (
-            mock_tpl.call_args.kwargs["dest"] == "/etc/systemd/system/chatmaild.service"
-        )
-
-    deployer = Deployer()
-
-    # Explicit dest_name override
-    with patch("cmdeploy.basedeploy.files.put", return_value=mock_res) as mock_put:
-        deployer.ensure_systemd_unit(
-            "acmetool/acmetool-reconcile.timer",
-            dest_name="acmetool-reconcile.timer",
-        )
-        assert (
-            mock_put.call_args.kwargs["dest"]
-            == "/etc/systemd/system/acmetool-reconcile.timer"
-        )
-
-
-def test_ensure_service():
-    with patch("cmdeploy.basedeploy.systemd.service") as mock_svc:
-        deployer = Deployer()
-        deployer.need_restart = True
-        deployer.daemon_reload = True
-        deployer.ensure_service("nginx.service")
-        mock_svc.assert_called_once_with(
-            name="Start and enable nginx.service",
-            service="nginx.service",
-            running=True,
-            enabled=True,
-            restarted=True,
-            daemon_reload=True,
-        )
-        # daemon_reload is cleared to avoid multiple systemctl daemon-reload calls
-        # need_restart is kept to ensure all subsequent services also restart
-        assert deployer.need_restart is True
-        assert deployer.daemon_reload is False
-
-    with patch("cmdeploy.basedeploy.systemd.service") as mock_svc:
-        # Stopping suppresses restarted even when need_restart is True
-        deployer = Deployer()
-        deployer.need_restart = True
-        deployer.daemon_reload = True
-        deployer.ensure_service(
-            "mta-sts-daemon.service",
-            running=False,
-            enabled=False,
-        )
-        assert mock_svc.call_args.kwargs["restarted"] is False
-        assert deployer.need_restart is True
-
-    with patch("cmdeploy.basedeploy.systemd.service") as mock_svc:
-        # Multiple calls: daemon_reload resets after first, need_restart persists
-        deployer = Deployer()
-        deployer.need_restart = True
-        deployer.daemon_reload = True
-        deployer.ensure_service("chatmaild.service")
-        deployer.ensure_service("chatmaild-metadata.service")
-        second_call = mock_svc.call_args_list[1]
-        assert second_call.kwargs["restarted"] is True
-        assert second_call.kwargs["daemon_reload"] is False

cmdeploy/src/cmdeploy/tests/test_dns.py

@@ -4,7 +4,6 @@ import pytest
 
 from cmdeploy import remote
 from cmdeploy.dns import check_full_zone, check_initial_remote_data, parse_zone_records
-from cmdeploy.remote.rdns import get_authoritative_ns
 
 
 @pytest.fixture
@@ -15,22 +14,7 @@ def mockdns_base(monkeypatch):
         if command.startswith("dig"):
             if command == "dig":
                 return "."
-            if "with.public.soa" in command:
-                return (
-                    "domain.with.public.soa. 2419 IN SOA ns1.first-ns.de. dns.hetzner.com."
-                    " 2026050300 7200 1800 604800 3600"
-                )
-            if "with.hidden.soa" in command and "SOA" in command:
-                return (
-                    "domain.with.hidden.soa. 300 IN SOA get.desec.io. get.desec.io."
-                    " 2026025451 86400 3600 2419200 3600"
-                )
-            if "with.hidden.soa" in command and "NS" in command:
-                return (
-                    "domain.with.hidden.soa. 2137 IN NS ns1.desec.io.\n"
-                    "domain.with.hidden.soa. 2137 IN NS ns2.desec.org."
-                )
-            if "NS" in command:
+            if "SOA" in command:
                 return (
                     "delta.chat. 21600 IN SOA ns1.first-ns.de. dns.hetzner.com."
                     " 2025102800 14400 1800 604800 3600"
@@ -141,17 +125,6 @@ class TestPerformInitialChecks:
         assert not l
 
 
-@pytest.mark.parametrize(
-    ("domain", "ns"),
-    [
-        ("domain.with.public.soa", "ns1.first-ns.de."),
-        ("domain.with.hidden.soa", "ns1.desec.io.")
-    ]
-)
-def test_get_authoritative_ns(domain, ns, mockdns):
-    assert get_authoritative_ns(domain) == ns
-
-
 def test_parse_zone_records():
     text = """
     ; This is a comment

cmdeploy/src/cmdeploy/tests/test_dovecot_deployer.py

@@ -1,238 +0,0 @@
-from contextlib import nullcontext
-from types import SimpleNamespace
-
-import pytest
-from pyinfra.facts.deb import DebPackages
-
-from cmdeploy.dovecot import deployer as dovecot_deployer
-
-
-def make_host(*fact_pairs):
-    """Build a mock host; get_fact(cls) dispatches to the provided facts mapping.
-
-    Args:
-        *fact_pairs: tuples of (fact_class, fact_value) to register
-
-    Returns:
-        SimpleNamespace with get_fact that raises a clear error if an
-        unexpected fact type is requested.
-    """
-    facts = dict(fact_pairs)
-
-    def get_fact(cls):
-        if cls not in facts:
-            registered = ", ".join(c.__name__ for c in facts)
-            raise LookupError(
-                f"unexpected get_fact({cls.__name__}); "
-                f"only registered: {registered}"
-            )
-        return facts[cls]
-
-    return SimpleNamespace(get_fact=get_fact)
-
-
-@pytest.fixture
-def deployer():
-    return dovecot_deployer.DovecotDeployer(
-        SimpleNamespace(mail_domain="chat.example.org"),
-        disable_mail=False,
-    )
-
-
-@pytest.fixture
-def patch_blocked(monkeypatch):
-    monkeypatch.setattr(dovecot_deployer, "blocked_service_startup", nullcontext)
-
-
-@pytest.fixture
-def mock_files_put(monkeypatch):
-    monkeypatch.setattr(
-        dovecot_deployer.files,
-        "put",
-        lambda **kwargs: SimpleNamespace(changed=False),
-    )
-
-
-@pytest.fixture
-def track_shell(monkeypatch):
-    calls = []
-    monkeypatch.setattr(
-        dovecot_deployer.server,
-        "shell",
-        lambda **kwargs: calls.append(kwargs) or SimpleNamespace(changed=False),
-    )
-    return calls
-
-
-def test_download_dovecot_package_skips_epoch_matched_install(monkeypatch):
-    epoch_version = dovecot_deployer.DOVECOT_PACKAGE_VERSION
-    downloads = []
-    monkeypatch.setattr(
-        dovecot_deployer,
-        "host",
-        make_host((DebPackages, {"dovecot-core": [epoch_version]})),
-    )
-    monkeypatch.setattr(
-        dovecot_deployer,
-        "_pick_url",
-        lambda primary, fallback: primary,
-    )
-    monkeypatch.setattr(
-        dovecot_deployer.files,
-        "download",
-        lambda **kwargs: downloads.append(kwargs),
-    )
-
-    deb, changed = dovecot_deployer._download_dovecot_package("core", "amd64")
-
-    assert deb is None, f"expected no deb path when version matches, got {deb!r}"
-    assert changed is False, "should not flag changed when version already installed"
-    assert downloads == [], "should not download when version already installed"
-
-
-def test_download_dovecot_package_uses_archive_version_for_url_and_filename(
-    monkeypatch,
-):
-    downloads = []
-    monkeypatch.setattr(
-        dovecot_deployer,
-        "host",
-        make_host((DebPackages, {})),
-    )
-    monkeypatch.setattr(
-        dovecot_deployer,
-        "_pick_url",
-        lambda primary, fallback: primary,
-    )
-    monkeypatch.setattr(
-        dovecot_deployer.files,
-        "download",
-        lambda **kwargs: downloads.append(kwargs),
-    )
-
-    deb, changed = dovecot_deployer._download_dovecot_package("core", "amd64")
-
-    archive_version = dovecot_deployer.DOVECOT_ARCHIVE_VERSION.replace("+", "%2B")
-    expected_deb = f"/root/dovecot-core_{archive_version}_amd64.deb"
-
-    # Verify the returned path uses archive version, not package version (with epoch)
-    assert changed is True, "should flag changed when package not yet installed"
-    assert deb == expected_deb, f"deb path mismatch: {deb!r} != {expected_deb!r}"
-    assert dovecot_deployer.DOVECOT_PACKAGE_VERSION not in deb, (
-        f"deb path should use archive version (no epoch), got {deb!r}"
-    )
-    assert len(downloads) == 1, "files.download should be called exactly once"
-
-
-def test_install_skips_dpkg_path_when_epoch_matched_packages_present(
-    deployer, patch_blocked, mock_files_put, track_shell, monkeypatch
-):
-    monkeypatch.setattr(
-        dovecot_deployer,
-        "host",
-        make_host(
-            (
-                dovecot_deployer.DebPackages,
-                {
-                    "dovecot-core": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
-                    "dovecot-imapd": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
-                    "dovecot-lmtpd": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
-                },
-            ),
-            (dovecot_deployer.Arch, "x86_64"),
-        ),
-    )
-    downloads = []
-    monkeypatch.setattr(
-        dovecot_deployer.files,
-        "download",
-        lambda **kwargs: downloads.append(kwargs),
-    )
-
-    deployer.install()
-
-    assert downloads == [], "should not download when all packages epoch-matched"
-    assert track_shell == [], "should not run dpkg when all packages epoch-matched"
-    assert deployer.need_restart is False, (
-        "need_restart should be False when nothing changed"
-    )
-
-
-def test_install_unsupported_arch_falls_back_to_apt(
-    deployer, patch_blocked, mock_files_put, track_shell, monkeypatch
-):
-    # For unsupported architectures, all fact lookups return the arch string.
-    monkeypatch.setattr(
-        dovecot_deployer,
-        "host",
-        SimpleNamespace(get_fact=lambda cls: "riscv64"),
-    )
-    apt_calls = []
-
-    # Mirrors apt.packages() return value: OperationMeta with .changed property.
-    # Only lmtpd triggers a change to verify |= accumulation of changed flags.
-    def fake_apt(**kwargs):
-        apt_calls.append(kwargs)
-        changed = "lmtpd" in kwargs["packages"][0]
-        return SimpleNamespace(changed=changed)
-
-    monkeypatch.setattr(dovecot_deployer.apt, "packages", fake_apt)
-
-    deployer.install()
-
-    actual_pkgs = [c["packages"] for c in apt_calls]
-    assert actual_pkgs == [["dovecot-core"], ["dovecot-imapd"], ["dovecot-lmtpd"]], (
-        f"expected apt install of core/imapd/lmtpd, got {actual_pkgs}"
-    )
-    assert track_shell == [], "should not run dpkg for unsupported arch"
-    assert deployer.need_restart is True, (
-        "need_restart should be True when apt installed a package"
-    )
-
-
-def test_install_runs_dpkg_when_packages_need_download(
-    deployer, patch_blocked, mock_files_put, track_shell, monkeypatch
-):
-    monkeypatch.setattr(
-        dovecot_deployer,
-        "host",
-        make_host(
-            (dovecot_deployer.DebPackages, {}),
-            (dovecot_deployer.Arch, "x86_64"),
-        ),
-    )
-    monkeypatch.setattr(
-        dovecot_deployer,
-        "_pick_url",
-        lambda primary, fallback: primary,
-    )
-    monkeypatch.setattr(
-        dovecot_deployer.files,
-        "download",
-        lambda **kwargs: SimpleNamespace(changed=True),
-    )
-
-    deployer.install()
-
-    assert len(track_shell) == 1, (
-        f"expected one server.shell() call for dpkg install, got {len(track_shell)}"
-    )
-    cmds = track_shell[0]["commands"]
-    assert len(cmds) == 3, f"expected 3 dpkg/apt commands, got: {cmds}"
-    assert cmds[0].startswith("dpkg --force-confdef --force-confold -i ")
-    assert "apt-get -y --fix-broken install" in cmds[1]
-    assert cmds[2].startswith("dpkg --force-confdef --force-confold -i ")
-    assert deployer.need_restart is True, (
-        "need_restart should be True after dpkg install"
-    )
-
-
-def test_pick_url_falls_back_on_primary_error(monkeypatch):
-    def raise_error(req, timeout):
-        raise OSError("connection timeout")
-
-    monkeypatch.setattr(dovecot_deployer.urllib.request, "urlopen", raise_error)
-    result = dovecot_deployer._pick_url("http://primary", "http://fallback")
-    assert result == "http://fallback", (
-        f"should fall back when primary fails, got {result!r}"
-    )

cmdeploy/src/cmdeploy/tests/test_helpers.py

@@ -1,10 +1,11 @@
-from pathlib import Path
+import importlib.resources
 
 from cmdeploy.www import build_webpages
 
 
 def test_build_webpages(tmp_path, make_config):
-    src_dir = (Path(__file__).resolve() / "../../../../../www/src").resolve()
+    pkgroot = importlib.resources.files("cmdeploy")
+    src_dir = pkgroot.joinpath("../../../www/src").resolve()
     assert src_dir.exists(), src_dir
     config = make_config("chat.example.org")
     build_dir = tmp_path.joinpath("build")

cmdeploy/src/cmdeploy/www.py

@@ -1,4 +1,5 @@
 import hashlib
+import importlib.resources
 import re
 import time
 import traceback
@@ -36,7 +37,7 @@ def prepare_template(source):
 
 
 def get_paths(config) -> (Path, Path, Path):
-    reporoot = (Path(__file__).resolve() / "../../../../").resolve()
+    reporoot = importlib.resources.files(__package__).joinpath("../../../").resolve()
     www_path = Path(config.www_folder)
     # if www_folder was not set, use default directory
     if config.www_folder == "":
@@ -132,7 +133,8 @@ def find_merge_conflict(src_dir) -> Path:
 
 
 def main():
-    reporoot = (Path(__file__).resolve() / "../../../../").resolve()
+    path = importlib.resources.files(__package__)
+    reporoot = path.joinpath("../../../").resolve()
     inipath = reporoot.joinpath("chatmail.ini")
     config = read_config(inipath)
     config.webdev = True

doc/README.md

@@ -4,14 +4,12 @@
 
 You can use the `make` command and `make html` to build web pages. 
 
-You need a Python environment with `sphinx` and other
-dependencies, you can create it by running `scripts/initenv.sh`
-from the repository root.
+You need a Python environment where the following install was excuted: 
+
+    pip install furo sphinx-autobuild
 
 To develop/change documentation, you can then do: 
 
-    . venv/bin/activate
-    cd doc
     make auto 
 
 A page will open at https://127.0.0.1:8000/ serving the docs and it will 

doc/source/index.rst

@@ -16,6 +16,5 @@ Contributions and feedback welcome through the https://github.com/chatmail/relay
     proxy
     migrate
     overview
-    reverse_dns
     related
     faq

doc/source/overview.rst

@@ -102,12 +102,8 @@ short overview of ``chatmaild`` services:
    Apple/Google/Huawei.
 
 -  `chatmail-expire <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/expire.py>`_
-   deletes old messages, large messages, and entire mailboxes
-   of users who have not logged in for longer than
-   ``delete_inactive_users_after`` days.
-
--  ``chatmail-quota-expire`` is called by Dovecot's ``quota_warning`` mechanism
-   and will automatically remove oldest messages to keep mailboxes well under ``max_mailbox_size``.
+   deletes users if they have not logged in for a longer while.
+   The timeframe can be configured in ``chatmail.ini``.
 
 -  `lastlogin <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/lastlogin.py>`_
    is contacted by Dovecot when a user logs in and stores the date of
@@ -153,7 +149,6 @@ Chatmail relay dependency diagram
         autoconfig.xml --- dovecot;
         postfix --- |10080|filtermail-outgoing;
         postfix --- |10081|filtermail-incoming;
-        postfix --- |10083|filtermail-transport;
         filtermail-outgoing --- |10025 reinject|postfix;
         filtermail-incoming --- |10026 reinject|postfix;
         dovecot --- |doveauth.socket|doveauth;
@@ -161,8 +156,6 @@ Chatmail relay dependency diagram
         /home/vmail/.../user"];
         dovecot --- |lastlogin.socket|lastlogin;
         dovecot --- chatmail-metadata;
-        dovecot --- |quota-warning|chatmail-quota-expire;
-        chatmail-quota-expire --- maildir;
         lastlogin --- maildir;
         doveauth --- maildir;
         chatmail-expire-daily --- maildir;
@@ -296,7 +289,9 @@ ensured by ``filtermail`` proxy.
 TLS requirements
 ~~~~~~~~~~~~~~~~
 
-Filtermail (used for delivery) requires a valid TLS.
+Postfix is configured to require valid TLS by setting
+`smtp_tls_security_level <https://www.postfix.org/postconf.5.html#smtp_tls_security_level>`_
+to ``verify``.
 
 You can test it by resolving ``MX`` records of your relay domain and
 then connecting to MX relays (e.g ``mx.example.org``) with

doc/source/reverse_dns.rst

@@ -1,64 +0,0 @@
-Configuring reverse DNS
-=======================
-
-Some email servers reject the emails
-if they don't pass `FCrDNS`_ check, also known as `iprev`_ check.
-
-.. _FCrDNS: https://en.wikipedia.org/wiki/Forward-confirmed_reverse_DNS
-.. _iprev: https://datatracker.ietf.org/doc/html/rfc8601#section-3
-
-Passing the check requires that the IP address that email is sent from
-should have a ``PTR`` record pointing to the domain name of the server,
-and domain name record should have an ``A/AAAA`` record
-pointing to the IP address.
-
-Modern email relies on DKIM and SPF for authentication,
-while iprev check exists for
-`historical reasons <https://datatracker.ietf.org/doc/html/draft-ietf-dnsop-reverse-mapping-considerations-06#section-2.1>`_.
-Chatmail relays don't resolve ``PTR`` records,
-so you can ignore this section if configuring ``PTR`` records
-is difficult and federation with legacy email servers that don't accept
-valid DKIM signature for authentication is not important.
-
-Multi-homed setups
-------------------
-
-If you have a server with multiple IP addresses,
-also known as multi-homed setup,
-and don't publish all IP addresses in DNS,
-you need to make sure you are using
-the published address when making outgoing connections.
-
-For example, your server may have a static IP
-address, and a so-called Floating IP or Virtual IP
-that can be moved between servers in case of
-migration or for failover.
-By using Floating IP you can avoid downtime
-and keep the IP address reputation
-for destinatinons that rely on IP reputation and IP blocklists.
-In this case you will only publish
-the Floating IP to DNS and only use the static IP
-to SSH into the server.
-
-If you have such setup, make sure that
-you not only set ``PTR`` records for the Floating IP,
-but make outgoing connections using the Floating IP.
-Otherwise reverse DNS check succeed,
-but forward check making sure your domain name points
-to the IP address will fail.
-Such setup is indistinguishable from someone
-setting IP address ``PTR`` with the domain they don't own
-and as a result don't succeed.
-
-On Linux you can configure source IP address with ``ip route`` command,
-for example:
-::
-
-  ip route change default via <default-gateway> dev eth0 src <source-address>
-
-Make sure to persist the change after verifying it is working.
-You can check what your outgoing IP address is
-with ``curl icanhazip.com``.
-Check both the IPv4 and IPv6 addresses.
-For IPv4 address use ``curl ipv4.icanhazip.com`` or ``curl -4 icanhazip.com``
-and similarly for IPv6 if you have it.