added full path to tox

2026-05-11 08:24:37 +00:00 · 2023-10-24 21:53:23 +02:00
216 changed files with 1742 additions and 11483 deletions
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,18 +0,0 @@
 data/
 venv/
 __pycache__
 *.pyc
 *.orig
 *.ini
 .pytest_cache
 .env
 # Slim build context — .git/ alone can be 100s of MB
 .git
 .github/
 docs/
 tests/
 # Exclude markdown files but keep www/src/*.md (used by WebsiteDeployer)
 *.md
 !www/**/*.md
--- a/.github/CODE_OF_CONDUCT.md
+++ b/.github/CODE_OF_CONDUCT.md
@@ -1,4 +0,0 @@
 Please refer to 
 [Delta Chat community standards and practices](https://delta.chat/en/community-standards)
 which also apply for all chatmail developments. 
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -1,33 +0,0 @@
 ---
 name: Bug report
 about: Report something that isn't working.
 title: ''
 assignees: ''
 ---
 <!--
 Please fill out as much of this form as you can (leaving out stuff that is not applicable is ok).
 -->
 - Server OS (Operating System) - preferably Debian 12: 
 - On which OS you run cmdeploy:
 - chatmail/relay version: `git rev-parse HEAD`
 ## Expected behavior
 *What did you try to achieve?*
 ## Actual behavior
 *What happened instead?*
 ### Steps to reproduce the problem:
 1. 
 2.
 ### Screenshots
 ### Logs
--- a/.github/ISSUE_TEMPLATE/config.yml
+++ b/.github/ISSUE_TEMPLATE/config.yml
@@ -1 +0,0 @@
 blank_issues_enabled: true
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -6,37 +6,26 @@ on:
 jobs:
  tox:
-    name: isolated chatmaild tests
+    name: chatmail tests
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
        # Checkout pull request HEAD commit instead of merge commit
        # Otherwise `test_deployed_state` will be unhappy.
        with:
          ref: ${{ github.event.pull_request.head.sha }}
      - name: download filtermail
        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.3.0/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
      - name: run chatmaild tests 
        working-directory: chatmaild
        run: pipx run tox
      - name: run deploy-chatmail offline tests 
        working-directory: deploy-chatmail
        run: pipx run tox
      - name: run deploy-chatmail offline tests 
        working-directory: deploy-chatmail
        run: pipx run tox
  scripts:
-    name: deploy-chatmail tests 
+    name: chatmail script invocations 
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
-
+      - name: run init.sh 
-      - name: initenv 
+        run: ./scripts/init.sh 
-        run: scripts/initenv.sh
+      - name: run test.sh 
-
+        run: ./scripts/test.sh 
      - name: append venv/bin to PATH
        run: echo venv/bin >>$GITHUB_PATH
      - name: run formatting checks 
        run: cmdeploy fmt -v 
      - name: run deploy-chatmail offline tests 
        run: pytest --pyargs cmdeploy 
      # all other cmdeploy commands require a staging server 
      # see https://github.com/deltachat/chatmail/issues/100
--- a/.github/workflows/docker-build.yaml
+++ b/.github/workflows/docker-build.yaml
@@ -1,76 +0,0 @@
 name: Docker Build
 on:
  pull_request:
    paths:
      - 'docker/**'
      - 'docker-compose.yaml'
      - '.dockerignore'
      - 'chatmaild/**'
      - 'cmdeploy/**'
      - '.github/workflows/docker-build.yaml'
  push:
    branches:
      - main
      - j4n/docker
    paths:
      - 'docker/**'
      - 'docker-compose.yaml'
      - '.dockerignore'
      - 'chatmaild/**'
      - 'cmdeploy/**'
      - '.github/workflows/docker-build.yaml'
    tags:
      - 'v*'
 env:
  REGISTRY: ghcr.io
  IMAGE_NAME: ${{ github.repository }}
 jobs:
  build:
    name: Build Docker image
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: write
    steps:
      - uses: actions/checkout@v4
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Login to GHCR
        if: github.event_name == 'push'
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Extract metadata (tags, labels)
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
          tags: |
            # Tagged releases: v1.2.3 → :1.2.3, :1.2, :latest
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}
            # Branch pushes: j4n/docker → :j4n-docker
            type=ref,event=branch
            # Always: :sha-<hash>
            type=sha
      - name: Build and push
        uses: docker/build-push-action@v6
        with:
          context: .
          file: docker/chatmail_relay.dockerfile
          push: ${{ github.event_name == 'push' }}
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          cache-from: type=gha
          cache-to: type=gha,mode=max
          build-args: |
            GIT_HASH=${{ github.sha }}
--- a/.github/workflows/docs-preview.yaml
+++ b/.github/workflows/docs-preview.yaml
@@ -1,53 +0,0 @@
 name: documentation preview
 on:
  pull_request:
    paths:
      - 'doc/**'
      - 'scripts/build-docs.sh'
      - '.github/workflows/docs-preview.yaml'
 jobs:
  scripts:
    name: build 
    runs-on: ubuntu-latest
    environment:
      name: 'staging.chatmail.at/doc/relay/'
      url: https://staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}
    steps:
      - uses: actions/checkout@v4
      - name: initenv 
        run: scripts/initenv.sh
      - name: append venv/bin to PATH
        run: echo `pwd`/venv/bin >>$GITHUB_PATH
      - name: build documentation 
        working-directory: doc
        run: sphinx-build source build
      - name: build documentation second time (for TOC)
        working-directory: doc
        run: sphinx-build source build
      - name: Get Pullrequest ID
        id: prepare
        run: |
          export PULLREQUEST_ID=$(echo "${{ github.ref }}" | cut -d "/" -f3)
          echo "prid=$PULLREQUEST_ID" >> $GITHUB_OUTPUT
          if [ $(expr length "${{ secrets.USERNAME }}") -gt "1" ]; then echo "uploadtoserver=true" >> $GITHUB_OUTPUT; fi
      - run: |
          echo "baseurl: /${{ steps.prepare.outputs.prid }}" >> _config.yml
      - name: Upload preview
        run: |
          mkdir -p "$HOME/.ssh"
          echo "${{ secrets.CHATMAIL_STAGING_SSHKEY }}" > "$HOME/.ssh/key"
          chmod 600 "$HOME/.ssh/key"
          rsync -rILvh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/doc/build/ "${{ secrets.USERNAME }}@chatmail.at:/var/www/html/staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}/"
      - name: check links 
        working-directory: doc
        run: sphinx-build --builder linkcheck source build
--- a/.github/workflows/docs.yaml
+++ b/.github/workflows/docs.yaml
@@ -1,47 +0,0 @@
 name: build and upload documentation
 on:
  push:
    branches:
      - main
      - 'missytake/docs-ci'
    paths:
      - 'doc/**'
      - 'scripts/build-docs.sh'
      - '.github/workflows/docs.yaml'
 jobs:
  scripts:
    name: build 
    runs-on: ubuntu-latest
    environment: 
      name: 'chatmail.at/doc/relay/'
      url: https://chatmail.at/doc/relay/
    steps:
      - uses: actions/checkout@v4
      - name: initenv 
        run: scripts/initenv.sh
      - name: append venv/bin to PATH
        run: echo `pwd`/venv/bin >>$GITHUB_PATH
      - name: build documentation 
        working-directory: doc
        run: sphinx-build source build
      - name: build documentation second time (for TOC)
        working-directory: doc
        run: sphinx-build source build
      - name: check links 
        working-directory: doc
        run: sphinx-build --builder linkcheck source build
      - name: upload documentation
        run: |
          mkdir -p "$HOME/.ssh"
          echo "${{ secrets.CHATMAIL_STAGING_SSHKEY }}" > "$HOME/.ssh/key"
          chmod 600 "$HOME/.ssh/key"
          rsync -rILvh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/doc/build/ "${{ secrets.USERNAME }}@chatmail.at:/var/www/html/chatmail.at/doc/relay/"
--- a/.github/workflows/staging-ipv4.testrun.org-default.zone
+++ b/.github/workflows/staging-ipv4.testrun.org-default.zone
@@ -1,20 +0,0 @@
 ;; Zone file for staging-ipv4.testrun.org
 $ORIGIN staging-ipv4.testrun.org.
 $TTL 300
@  IN  SOA     ns.testrun.org. root.nine.testrun.org (
  2023010101 ; Serial
  7200       ; Refresh
  3600       ; Retry
  1209600    ; Expire
  3600       ; Negative response caching TTL
 )
 ;; Nameservers.
@  IN NS ns.testrun.org.
 ;; DNS records.
@       IN      A       37.27.95.249
 mta-sts.staging-ipv4.testrun.org.    CNAME   staging-ipv4.testrun.org.
 www.staging-ipv4.testrun.org.        CNAME   staging-ipv4.testrun.org.
--- a/.github/workflows/staging.testrun.org-default.zone
+++ b/.github/workflows/staging.testrun.org-default.zone
@@ -1,21 +0,0 @@
 ;; Zone file for staging2.testrun.org
 $ORIGIN staging2.testrun.org.
 $TTL 300
@  IN  SOA     ns.testrun.org. root.nine.testrun.org (
  2023010101 ; Serial
  7200       ; Refresh
  3600       ; Retry
  1209600    ; Expire
  3600       ; Negative response caching TTL
 )
 ;; Nameservers.
@  IN NS ns.testrun.org.
 ;; DNS records.
@       IN      A       37.27.24.139
 mta-sts.staging2.testrun.org.    CNAME   staging2.testrun.org.
 www.staging2.testrun.org.        CNAME   staging2.testrun.org.
--- a/.github/workflows/test-and-deploy-ipv4only.yaml
+++ b/.github/workflows/test-and-deploy-ipv4only.yaml
@@ -1,96 +0,0 @@
 name: deploy on staging-ipv4.testrun.org, and run tests
 on:
  push:
    branches:
      - main
  pull_request:
    paths-ignore:
      - 'scripts/**'
      - '**/README.md'
      - 'CHANGELOG.md'
      - 'LICENSE'
 jobs:
  deploy:
    name: deploy on staging-ipv4.testrun.org, and run tests
    runs-on: ubuntu-latest
    timeout-minutes: 30
    environment:
      name: staging-ipv4.testrun.org
      url: https://staging-ipv4.testrun.org/
    concurrency: staging-ipv4.testrun.org
    steps:
      - uses: actions/checkout@v4
      - name: prepare SSH
        run: |
          mkdir ~/.ssh
          echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
          chmod 600 ~/.ssh/id_ed25519
          ssh-keyscan staging-ipv4.testrun.org > ~/.ssh/known_hosts
          # save previous acme & dkim state
          rsync -avz root@staging-ipv4.testrun.org:/var/lib/acme acme-ipv4 || true
          rsync -avz root@staging-ipv4.testrun.org:/etc/dkimkeys dkimkeys-ipv4 || true
          # store previous acme & dkim state on ns.testrun.org, if it contains useful certs
          if [ -f dkimkeys-ipv4/dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys-ipv4 root@ns.testrun.org:/tmp/ || true; fi
          if [ "$(ls -A acme-ipv4/acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme-ipv4 root@ns.testrun.org:/tmp/ || true; fi
          # make sure CAA record isn't set
          scp -o StrictHostKeyChecking=accept-new .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
          ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging-ipv4.testrun.org.zone
          ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
          ssh root@ns.testrun.org systemctl reload nsd
      - name: rebuild staging-ipv4.testrun.org to have a clean VPS
        run: |
            curl -X POST \
            -H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
            -H "Content-Type: application/json" \
            -d '{"image":"debian-12"}' \
            "https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_IPV4_SERVER_ID }}/actions/rebuild"
      - run: scripts/initenv.sh
      - name: append venv/bin to PATH
        run: echo venv/bin >>$GITHUB_PATH
      - name: upload TLS cert after rebuilding
        run: |
          echo " --- wait until staging-ipv4.testrun.org VPS is rebuilt --- "
          rm ~/.ssh/known_hosts
          while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u ; do sleep 1 ; done
          ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u
          # download acme & dkim state from ns.testrun.org
          rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme-ipv4/acme acme-restore || true
          rsync -avz root@ns.testrun.org:/tmp/dkimkeys-ipv4/dkimkeys dkimkeys-restore || true
          # restore acme & dkim state to staging2.testrun.org
          rsync -avz acme-restore/acme root@staging-ipv4.testrun.org:/var/lib/ || true
          rsync -avz dkimkeys-restore/dkimkeys root@staging-ipv4.testrun.org:/etc/ || true
          ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org chown root:root -R /var/lib/acme || true
      - name: run deploy-chatmail offline tests 
        run: pytest --pyargs cmdeploy 
      - run: |
          cmdeploy init staging-ipv4.testrun.org
          sed -i 's#disable_ipv6 = False#disable_ipv6 = True#' chatmail.ini
          sed -i 's/#\s*mtail_address/mtail_address/' chatmail.ini
      - run: cmdeploy run --verbose --skip-dns-check
      - name: set DNS entries
        run: |
          ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
          cmdeploy dns --zonefile staging-generated.zone
          cat staging-generated.zone >> .github/workflows/staging-ipv4.testrun.org-default.zone
          cat .github/workflows/staging-ipv4.testrun.org-default.zone
          scp .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
          ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
          ssh root@ns.testrun.org systemctl reload nsd
      - name: cmdeploy test
        run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
      - name: cmdeploy dns
        run: cmdeploy dns -v
--- a/.github/workflows/test-and-deploy.yaml
+++ b/.github/workflows/test-and-deploy.yaml
@@ -1,98 +0,0 @@
 name: deploy on staging2.testrun.org, and run tests
 on:
  push:
    branches:
      - main
  pull_request:
    paths-ignore:
      - 'scripts/**'
      - '**/README.md'
      - 'CHANGELOG.md'
      - 'LICENSE'
 jobs:
  deploy:
    name: deploy on staging2.testrun.org, and run tests
    runs-on: ubuntu-latest
    timeout-minutes: 30
    environment:
      name: staging2.testrun.org
      url: https://staging2.testrun.org/
    concurrency: staging2.testrun.org
    steps:
      - uses: actions/checkout@v4
      - name: prepare SSH
        run: |
          mkdir ~/.ssh
          echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
          chmod 600 ~/.ssh/id_ed25519
          ssh-keyscan staging2.testrun.org > ~/.ssh/known_hosts
          # save previous acme & dkim state
          rsync -avz root@staging2.testrun.org:/var/lib/acme . || true
          rsync -avz root@staging2.testrun.org:/etc/dkimkeys . || true
          # store previous acme & dkim state on ns.testrun.org, if it contains useful certs
          if [ -f dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys root@ns.testrun.org:/tmp/ || true; fi
          if [ "$(ls -A acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme root@ns.testrun.org:/tmp/ || true; fi
          # make sure CAA record isn't set
          scp -o StrictHostKeyChecking=accept-new .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
          ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging2.testrun.org.zone
          ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
          ssh root@ns.testrun.org systemctl reload nsd
      - name: rebuild staging2.testrun.org to have a clean VPS
        run: |
            curl -X POST \
            -H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
            -H "Content-Type: application/json" \
            -d '{"image":"debian-12"}' \
            "https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_SERVER_ID }}/actions/rebuild"
      - run: scripts/initenv.sh
      - name: append venv/bin to PATH
        run: echo venv/bin >>$GITHUB_PATH
      - name: upload TLS cert after rebuilding
        run: |
          echo " --- wait until staging2.testrun.org VPS is rebuilt --- "
          rm ~/.ssh/known_hosts
          while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org id -u ; do sleep 1 ; done
          ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org id -u
          # download acme & dkim state from ns.testrun.org
          rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme acme-restore || true
          rsync -avz root@ns.testrun.org:/tmp/dkimkeys dkimkeys-restore || true
          # restore acme & dkim state to staging2.testrun.org
          rsync -avz acme-restore/acme root@staging2.testrun.org:/var/lib/ || true
          rsync -avz dkimkeys-restore/dkimkeys root@staging2.testrun.org:/etc/ || true
          ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org chown root:root -R /var/lib/acme || true
      - name: add hpk42 key to staging server
        run: ssh root@staging2.testrun.org 'curl -s https://github.com/hpk42.keys >> .ssh/authorized_keys'
      - name: run deploy-chatmail offline tests 
        run: pytest --pyargs cmdeploy 
      - run: |
          cmdeploy init staging2.testrun.org
          sed -i 's/#\s*mtail_address/mtail_address/' chatmail.ini
      - run: cmdeploy run --verbose --skip-dns-check
      - name: set DNS entries
        run: |
          ssh -o StrictHostKeyChecking=accept-new root@staging2.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
          cmdeploy dns --zonefile staging-generated.zone --verbose
          cat staging-generated.zone >> .github/workflows/staging.testrun.org-default.zone
          cat .github/workflows/staging.testrun.org-default.zone
          scp .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
          ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
          ssh root@ns.testrun.org systemctl reload nsd
      - name: cmdeploy test
        run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
      - name: cmdeploy dns
        run: cmdeploy dns -v
--- a/.gitignore
+++ b/.gitignore
@@ -3,9 +3,6 @@ __pycache__/
 *.py[cod]
 *$py.class
 *.swp
 *qr-*.png
 chatmail.ini
 # C extensions
 *.so
@@ -162,11 +159,3 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
 chatmail.zone
 # docker
 /data/
 /custom/
 docker-compose.override.yaml
 .env
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,526 +0,0 @@
 # Changelog for chatmail deployment 
 ## 1.9.0 2025-12-18
 ### Documentation
 - Add RELEASE.md and CONTRIBUTING.md
 - README update, mention Chatmail Cookbook project
 ### Bug Fixes
 - Expire messages also from IMAP subfolders
 - Use absolute path instead of relative path in message expiration script
 - Restart Postfix and Dovecot automatically on failure
 - acmetool: Use a fixed name and `reconcile` instead of `want`
 ### Features
 - Report DKIM error code in SMTP response
 - Remove development notice from the web pages
 ### Miscellaneous Tasks
 - Update the heading in the CHANGELOG.md
 - Setup git-cliff
 - Run tests against ci-chatmail.testrun.org instead of nine.testrun.org
 - Cleanup remaining echobot code, remove echobot user from deployment and passthrough recipients
 ## 1.8.0 2025-12-12
 - Add imap_compress option to chatmail.ini
  ([#760](https://github.com/chatmail/relay/pull/760))
 - Remove echobot from relays 
  ([#753](https://github.com/chatmail/relay/pull/753))
 - Fix `cmdeploy webdev`
  ([#743](https://github.com/chatmail/relay/pull/743))
 - Add robots.txt to exclude all web crawlers
  ([#732](https://github.com/chatmail/relay/pull/732))
 - acmetool: accept new Let's Encrypt ToS: https://letsencrypt.org/documents/LE-SA-v1.6-August-18-2025.pdf
  ([#729](https://github.com/chatmail/relay/pull/729))
 - Organized cmdeploy into install, configure, and activate stages
  ([#695](https://github.com/chatmail/relay/pull/695))
 - docs: move readme.md docs to sphinx documentation rendered at https://chatmail.at/doc/relay 
  ([#711](https://github.com/chatmail/relay/pull/711))
 - acmetool: replace cronjob with a systemd timer
  ([#719](https://github.com/chatmail/relay/pull/719))
 - remove xstore@testrun.org from default passthrough recipients
  ([#722](https://github.com/chatmail/relay/pull/722))
 - don't deploy the website if there are merge conflicts in the www folder
  ([#714](https://github.com/chatmail/relay/pull/714))
 - acmetool: use ECDSA keys instead of RSA
  ([#689](https://github.com/chatmail/relay/pull/689))
 - Require TLS 1.2 for outgoing SMTP connections
  ([#685](https://github.com/chatmail/relay/pull/685), [#730](https://github.com/chatmail/relay/pull/730))
 - require STARTTLS for incoming port 25 connections
  ([#684](https://github.com/chatmail/relay/pull/684), [#730](https://github.com/chatmail/relay/pull/730))
 - filtermail: run CPU-intensive handle_DATA in a thread pool executor
  ([#676](https://github.com/chatmail/relay/pull/676))
 - don't use the complicated logging module in filtermail to exclude a potential source of errors. 
  ([#674](https://github.com/chatmail/relay/pull/674))
 - Specify nginx.conf to only handle `mail_domain`, www, and mta-sts domains
  ([#636](https://github.com/chatmail/relay/pull/636))
 - Setup TURN server
  ([#621](https://github.com/chatmail/relay/pull/621))
 - cmdeploy: make --ssh-host work with localhost
  ([#659](https://github.com/chatmail/relay/pull/659))
 - Update iroh-relay to 0.35.0
  ([#650](https://github.com/chatmail/relay/pull/650))
 - filtermail: accept mails from Protonmail
  ([#616](https://github.com/chatmail/relay/pull/616))
 - Ignore all RCPT TO: parameters
  ([#651](https://github.com/chatmail/relay/pull/651))
 - Increase opendkim DNS Timeout from 5 to 60 seconds
  ([#672](https://github.com/chatmail/relay/pull/672))
 - Add config parameter for Let's Encrypt ACME email
  ([#663](https://github.com/chatmail/relay/pull/663))
 - Use max username length in newemail.py, not min
  ([#648](https://github.com/chatmail/relay/pull/648))
 - Add startup for `fcgiwrap.service` because sometimes it did not start automatically.
  ([#657](https://github.com/chatmail/relay/pull/657))
 - Add `cmdeploy init --force` command for recreating chatmail.ini
  ([#656](https://github.com/chatmail/relay/pull/656))
 - Increase maxproc for reinjecting ports from 10 to 100
  ([#646](https://github.com/chatmail/relay/pull/646))
 - Allow ports 143 and 993 to be used by `dovecot` process
  ([#639](https://github.com/chatmail/relay/pull/639))
 - Add `--skip-dns-check` argument to `cmdeploy run` command, which disables DNS record checking before installation.
  ([#661](https://github.com/chatmail/relay/pull/661))
 - Rework expiry of message files and mailboxes in Python 
  to only do a single iteration over sometimes millions of messages
  instead of doing "find" commands that iterate 9 times over the messages. 
  Provide an "fsreport" CLI for more fine grained analysis of message files. 
  ([#637](https://github.com/chatmail/relay/pull/637))
 - Add installation via docker compose (MVP 1). The instructions, known issues and limitations are located in `/docs` 
  ([#614](https://github.com/chatmail/relay/pull/614))
 - Add configuration parameters
  ([#614](https://github.com/chatmail/relay/pull/614)):
  - `change_kernel_settings` - Whether to change kernel parameters during installation (default: `True`)
  - `fs_inotify_max_user_instances_and_watchers` - Value for kernel parameters `fs.inotify.max_user_instances` and `fs.inotify.max_user_watches` (default: `65535`)
 ## 1.7.0 2025-09-11
 - Make www upload path configurable
  ([#618](https://github.com/chatmail/relay/pull/618))
 - Check whether GCC is installed in initenv.sh
  ([#608](https://github.com/chatmail/relay/pull/608))
 - Expire push notification tokens after 90 days
  ([#583](https://github.com/chatmail/relay/pull/583))
 - Use official `mtail` binary instead of `mtail` package
  ([#581](https://github.com/chatmail/relay/pull/581))
 - dovecot: install from download.delta.chat instead of openSUSE Build Service
  ([#590](https://github.com/chatmail/relay/pull/590))
 - Reconfigure Dovecot imap-login service to high-performance mode
  ([#578](https://github.com/chatmail/relay/pull/578))
 - Set timezone to improve dovecot performance
  ([#584](https://github.com/chatmail/relay/pull/584))
 - Increase nginx connection limits
  ([#576](https://github.com/chatmail/relay/pull/576))
 - If `dns-utils` needs to be installed before cmdeploy run, apt update to make sure it works
  ([#560](https://github.com/chatmail/relay/pull/560))
 - filtermail: respect config message size limit
  ([#572](https://github.com/chatmail/relay/pull/572))
 - Don't deploy if one of the ports used for chatmail relay services is occupied by an unexpected process
  ([#568](https://github.com/chatmail/relay/pull/568))
 - Add config value after how many days large files are deleted
  ([#555](https://github.com/chatmail/relay/pull/555))
 - cmdeploy: push relay version to /etc/chatmail-version
  ([#573](https://github.com/chatmail/relay/pull/573))
 - filtermail: allow partial body length in OpenPGP payloads
  ([#570](https://github.com/chatmail/relay/pull/570))
 - chatmaild: allow echobot to receive unencrypted messages by default
  ([#556](https://github.com/chatmail/relay/pull/556))
 ## 1.6.0 2025-04-11
 - Handle Port-25 connect errors more gracefully (common with VPNs)
  ([#552](https://github.com/chatmail/relay/pull/552))
 - Avoid "acmetool not found" during initial run
  ([#550](https://github.com/chatmail/relay/pull/550))
 - Fix timezone handling such that client/servers do not need to use
  same timezone. 
  ([#553](https://github.com/chatmail/relay/pull/553))
 - Enforce end-to-end encryption for incoming messages. 
  New user address mailboxes now get a `enforceE2EEincoming` file 
  which prohibits incoming cleartext messages from other domains. 
  An outside MTA trying to submit a cleartext message will 
  get a "523 Encryption Needed" response, see RFC5248. 
  If the file does not exist (as it the case for all existing accounts) 
  incoming cleartext messages are accepted. 
  ([#538](https://github.com/chatmail/server/pull/538))
 - Enforce end-to-end encryption between local addresses 
  ([#535](https://github.com/chatmail/server/pull/535))
 - unbound: check that port 53 is not occupied by a different process
  ([#537](https://github.com/chatmail/server/pull/537))
 - unbound: before unbound is there, use 9.9.9.9 for resolving
  ([#518](https://github.com/chatmail/relay/pull/518))
 - Limit the bind for the HTTPS server on 8443 to 127.0.0.1 
  ([#522](https://github.com/chatmail/server/pull/522))
  ([#532](https://github.com/chatmail/server/pull/532))
 - Send SNI when connecting to outside servers
  ([#524](https://github.com/chatmail/server/pull/524))
 - postfix master.cf: use 127.0.0.1 for consistency
  ([#544](https://github.com/chatmail/relay/pull/544))
 - Pass through `original_content` instead of `content` in filtermail
  ([#509](https://github.com/chatmail/server/pull/509))
 - Document TLS requirements in the readme
  ([#514](https://github.com/chatmail/server/pull/514))
 - Remove cleanup service from submission ports
  ([#512](https://github.com/chatmail/server/pull/512))
 - cmdeploy dovecot: delete big messages after 7 days
  ([#504](https://github.com/chatmail/server/pull/504))
 - mtail: fix getting logs from STDIN
  ([#502](https://github.com/chatmail/server/pull/502))
 - filtermail: don't require exactly 2 lines after openPGP payload
  ([#497](https://github.com/chatmail/server/pull/497))
 - cmdeploy dns: offer alternative DKIM record format for some web interfaces
  ([#470](https://github.com/chatmail/server/pull/470))
 - journald: remove old logs from disk
  ([#490](https://github.com/chatmail/server/pull/490))
 - opendkim: restart once every day to mend RAM leaks
  ([#498](https://github.com/chatmail/server/pull/498)
 - migration guide: let opendkim own the DKIM keys directory
  ([#468](https://github.com/chatmail/server/pull/468))
 - improve secure-join message detection
  ([#473](https://github.com/chatmail/server/pull/473))
 - use old crypt lib in python < 3.11
  ([#483](https://github.com/chatmail/server/pull/483))
 - chatmaild: set umask to 0700 for doveauth + metadata
  ([#490](https://github.com/chatmail/server/pull/492))
 - remove MTA-STS daemon
  ([#488](https://github.com/chatmail/server/pull/488))
 - replace `Subject` with `[...]` for all outgoing mails.
  ([#481](https://github.com/chatmail/server/pull/481))
 - opendkim: use su instead of sudo
  ([#491](https://github.com/chatmail/server/pull/491))
 ## 1.5.0 2024-12-20
 - cmdeploy dns: always show recommended DNS records
  ([#463](https://github.com/chatmail/server/pull/463))
 - add `--all` to `cmdeploy dns`
  ([#462](https://github.com/chatmail/server/pull/462))
 - fix `_mta-sts` TXT DNS record
  ([#461](https://github.com/chatmail/server/pull/461)
 - deploy `iroh-relay` and also update "realtime relay services" in privacy policy. 
  ([#434](https://github.com/chatmail/server/pull/434))
  ([#451](https://github.com/chatmail/server/pull/451))
 - add guide to migrate chatmail to a new server
  ([#429](https://github.com/chatmail/server/pull/429))
 - disable anvil authentication penalty
  ([#414](https://github.com/chatmail/server/pull/444)
 - increase `request_queue_size` for UNIX sockets to 1000.
  ([#437](https://github.com/chatmail/server/pull/437))
 - add argument to `cmdeploy run` for specifying
  a different SSH host than `mail_domain`
  ([#439](https://github.com/chatmail/server/pull/439))
 - query autoritative nameserver to bypass DNS cache
  ([#424](https://github.com/chatmail/server/pull/424))
 - add mtail support (new optional `mtail_address` ini value)
  This defines the address on which [`mtail`](https://google.github.io/mtail/)
  exposes its metrics collected from the logs.
  If you want to collect the metrics with Prometheus,
  setup a private network (e.g. WireGuard interface)
  and assign an IP address from this network to the host.
  If you do not plan to collect metrics,
  keep this setting unset.
  ([#388](https://github.com/chatmail/server/pull/388))
 - fix checking for required DNS records
  ([#412](https://github.com/chatmail/server/pull/412))
 - add support for specifying whole domains for recipient passthrough list
  ([#408](https://github.com/chatmail/server/pull/408))
 - add a paragraph about "account deletion" to info page 
  ([#405](https://github.com/chatmail/server/pull/405))
 - avoid nginx listening on ipv6 if v6 is dsiabled 
  ([#402](https://github.com/chatmail/server/pull/402))
 - refactor ssh-based execution to allow organizing remote functions in
  modules. 
  ([#396](https://github.com/chatmail/server/pull/396))
 - trigger "apt upgrade" during "cmdeploy run" 
  ([#398](https://github.com/chatmail/server/pull/398))
 - drop hispanilandia passthrough address
  ([#401](https://github.com/chatmail/server/pull/401))
 - set CAA record flags to 0
 - add IMAP capabilities instead of overwriting them
  ([#413](https://github.com/chatmail/server/pull/413))
 - fix OpenPGP payload check
  ([#435](https://github.com/chatmail/server/pull/435))
 - fix Dovecot quota_max_mail_size to use max_message_size config value
  ([#438](https://github.com/chatmail/server/pull/438))
 ## 1.4.1 2024-07-31
 - fix metadata dictproxy which would confuse transactions
  resulting in missed notifications and other issues. 
  ([#393](https://github.com/chatmail/server/pull/393))
  ([#394](https://github.com/chatmail/server/pull/394))
 - add optional "imap_rawlog" config option. If true, 
  .in/.out files are created in user home dirs 
  containing the imap protocol messages. 
  ([#389](https://github.com/chatmail/server/pull/389))
 ## 1.4.0 2024-07-28
 - Add `disable_ipv6` config option to chatmail.ini.
  Required if the server doesn't have IPv6 connectivity.
  ([#312](https://github.com/chatmail/server/pull/312))
 - allow current K9/Thunderbird-mail releases to send encrypted messages
  outside by accepting their localized "encrypted subject" strings. 
  ([#370](https://github.com/chatmail/server/pull/370))
 - Migrate and remove sqlite database in favor of password/lastlogin tracking 
  in a user's maildir.  
  ([#379](https://github.com/chatmail/server/pull/379))
 - Require pyinfra V3 installed on the client side,
  run `./scripts/initenv.sh` to upgrade locally.
  ([#378](https://github.com/chatmail/server/pull/378))
 - don't hardcode "/home/vmail" paths but rather set them 
  once in the config object and use it everywhere else, 
  thereby also improving testability.  
  ([#351](https://github.com/chatmail/server/pull/351))
  temporarily introduced obligatory "passdb_path" and "mailboxes_dir" 
  settings but they were removed/obsoleted in 
  ([#380](https://github.com/chatmail/server/pull/380))
 - BREAKING: new required chatmail.ini value 'delete_inactive_users_after = 100'
  which removes users from database and mails after 100 days without any login. 
  ([#350](https://github.com/chatmail/server/pull/350))
 - Refine DNS checking to distinguish between "required" and "recommended" settings 
  ([#372](https://github.com/chatmail/server/pull/372))
 - reload nginx in the acmetool cronjob
  ([#360](https://github.com/chatmail/server/pull/360))
 - remove checking of reverse-DNS PTR records.  Chatmail-servers don't
  depend on it and even in the wider e-mail system it's not common anymore. 
  If it's an issue, a chatmail operator can still care to properly set reverse DNS. 
  ([#348](https://github.com/chatmail/server/pull/348))
 - Make DNS-checking faster and more interactive, run it fully during "cmdeploy run",
  also introducing a generic mechanism for rapid remote ssh-based python function execution. 
  ([#346](https://github.com/chatmail/server/pull/346))
 - Don't fix file owner ship of /home/vmail 
  ([#345](https://github.com/chatmail/server/pull/345))
 - Support iterating over all users with doveadm commands 
  ([#344](https://github.com/chatmail/server/pull/344))
 - Test and fix for attempts to create inadmissible accounts 
  ([#333](https://github.com/chatmail/server/pull/321))
 - check that OpenPGP has only PKESK, SKESK and SEIPD packets
  ([#323](https://github.com/chatmail/server/pull/323),
   [#324](https://github.com/chatmail/server/pull/324))
 - improve filtermail checks for encrypted messages and drop support for unencrypted MDNs
  ([#320](https://github.com/chatmail/server/pull/320))
 - replace `bash` with `/bin/sh`
  ([#334](https://github.com/chatmail/server/pull/334))
 - Increase number of logged in IMAP sessions to 50000
  ([#335](https://github.com/chatmail/server/pull/335))
 - filtermail: do not allow ASCII armor without actual payload
  ([#325](https://github.com/chatmail/server/pull/325))
 - Remove sieve to enable hardlink deduplication in LMTP
  ([#343](https://github.com/chatmail/server/pull/343))
 - dovecot: enable gzip compression on disk
  ([#341](https://github.com/chatmail/server/pull/341))
 - DKIM-sign Content-Type and oversign all signed headers
  ([#296](https://github.com/chatmail/server/pull/296))
 - Add nonci_accounts metric
  ([#347](https://github.com/chatmail/server/pull/347))
 - doveauth: log when a new account is created
  ([#349](https://github.com/chatmail/server/pull/349))
 - Multiplex HTTPS, IMAP and SMTP on port 443
  ([#357](https://github.com/chatmail/server/pull/357))
 ## 1.3.0 - 2024-06-06
 - don't check necessary DNS records on cmdeploy init anymore
  ([#316](https://github.com/chatmail/server/pull/316))
 - ensure cron and acl are installed
  ([#293](https://github.com/chatmail/server/pull/293),
  [#310](https://github.com/chatmail/server/pull/310))
 - change default for delete_mails_after from 40 to 20 days
  ([#300](https://github.com/chatmail/server/pull/300))
 - save journald logs only to memory and save nginx logs to journald instead of file
  ([#299](https://github.com/chatmail/server/pull/299))
 - fix writing of multiple obs repositories in `/etc/apt/sources.list`
  ([#290](https://github.com/chatmail/server/pull/290))
 - metadata: add support for `/shared/vendor/deltachat/irohrelay`
  ([#284](https://github.com/chatmail/server/pull/284))
 - Emit "XCHATMAIL" capability from IMAP server 
  ([#278](https://github.com/chatmail/server/pull/278))
 - Move echobot `into /var/lib/echobot`
  ([#281](https://github.com/chatmail/server/pull/281))
 - Accept Let's Encrypt's new Terms of Services
  ([#275](https://github.com/chatmail/server/pull/276))
 - Reload Dovecot and Postfix when TLS certificate updates
  ([#271](https://github.com/chatmail/server/pull/271))
 - Use forked version of dovecot without hardcoded delays
  ([#270](https://github.com/chatmail/server/pull/270))
 ## 1.2.0 - 2024-04-04
 - Install dig on the server to resolve DNS records
  ([#267](https://github.com/chatmail/server/pull/267))
 - preserve notification order and exponentially backoff with 
  retries for tokens where we didn't get a successful return
  ([#265](https://github.com/chatmail/server/pull/263))
 - Run chatmail-metadata and doveauth as vmail
  ([#261](https://github.com/chatmail/server/pull/261))
 - Apply systemd restrictions to echobot
  ([#259](https://github.com/chatmail/server/pull/259))
 - re-enable running the CI in pull requests, but not concurrently 
  ([#258](https://github.com/chatmail/server/pull/258))
 ## 1.1.0 - 2024-03-28
 ### The changelog starts to record changes from March 15th, 2024 
 - Move systemd unit templates to cmdeploy package 
  ([#255](https://github.com/chatmail/server/pull/255))
 - Persist push tokens and support multiple device per address 
  ([#254](https://github.com/chatmail/server/pull/254))
 - Avoid warning for regular doveauth protocol's hello message. 
  ([#250](https://github.com/chatmail/server/pull/250))
 - Fix various tests to pass again with "cmdeploy test". 
  ([#245](https://github.com/chatmail/server/pull/245),
  [#242](https://github.com/chatmail/server/pull/242)
 - Ensure lets-encrypt certificates are reloaded after renewal 
  ([#244]) https://github.com/chatmail/server/pull/244
 - Persist tokens to avoid iOS users loosing push-notifications when the
  chatmail metadata service is restarted (happens regularly during deploys)
  ([#238](https://github.com/chatmail/server/pull/239)
 - Fix failing sieve-script compile errors on incoming messages
  ([#237](https://github.com/chatmail/server/pull/239)
 - Fix quota reporting after expunging of old mails
  ([#233](https://github.com/chatmail/server/pull/239)
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,7 +0,0 @@
 # Contributing to the chatmail relay
 Commit messages follow the [Conventional Commits] notation.
 We use [git-cliff] to generate the changelog from commit messages before the release.
 [Conventional Commits]: https://www.conventionalcommits.org/
 [git-cliff]: https://git-cliff.org/
--- a/21
+++ b/21
@@ -1,21 +0,0 @@
 The MIT License (MIT)
 Copyright (c) 2023, chatmail and delta chat teams
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in
 the Software without restriction, including without limitation the rights to
 use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
 of the Software, and to permit persons to whom the Software is furnished to do
 so, subject to the following conditions:
 The above copyright notice and this permission notice shall be included in all
 copies or substantial portions of the Software.
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
--- a/README.md
+++ b/README.md
@@ -1,20 +1,59 @@
 # Chat Mail server configuration
-# Chatmail relays for end-to-end encrypted email
+This repository setups a ready-to-go chatmail instance
 comprised of a minimal setup of the battle-tested 
 [postfix smtp server](https://www.postfix.org) and [dovecot imap server](https://www.dovecot.org).
-Chatmail relay servers are interoperable Mail Transport Agents (MTAs) designed for: 
+## Getting started 
-  **Zero State:** no private data or metadata collected, messages are auto-deleted, low disk usage
+1. prepare your local system:
-  **Instant/Realtime:** sub-second message delivery, realtime P2P
+    scripts/init.sh 
   streaming, privacy-preserving Push Notifications for Apple, Google, and Huawei;
-  **Security Enforcement**: only strict TLS, DKIM and OpenPGP with minimized metadata accepted
+2. set environment variable to the chatmail domain you want to setup:
-  **Reliable Federation and Decentralization:** No spam or IP reputation checks, federating
+    export CHATMAIL_DOMAIN=c1.testrun.org   # replace with your host
   depends on established IETF standards and protocols.
-This repository contains everything needed to setup a ready-to-use chatmail relay on an ssh-reachable host. 
+3. run the deploy of the chat mail instance: 
 For getting started and more information please refer to the web version of this repositories' documentation at
-[https://chatmail.at/doc/relay](https://chatmail.at/doc/relay)
+    scripts/deploy.sh 
 ## Running tests and benchmarks (offline and online) 
 1. Set `CHATMAIL_SSH` so that `ssh root@$CHATMAIL_SSH` allows 
   to login to the chatmail instance server. 
 2. To run local and online tests: 
    scripts/test.sh 
 3. To run benchmarks against your chatmail instance: 
    scripts/bench.sh 
 ## Running tests (offline and online) 
 ```
 ## Dovecot/Postfix configuration
 ### Ports
 Postfix listens on ports 25 (smtp) and 587 (submission) and 465 (submissions).
 Dovecot listens on ports 143(imap) and 993 (imaps).
 ## DNS
 For DKIM you must add a DNS entry as found in /etc/opendkim/selector.txt on your chatmail instance.
 The above `scripts/deploy.sh` prints out the DKIM selector and DNS entry you
 need to setup with your DNS provider. 
 ## Emergency Commands
 If you need to stop account creation,
 e.g. because some script is wildly creating accounts,
 just run `touch /tmp/nocreate`.
 You can remove the file
 as soon as the attacker was banned
 by different means.
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -1,15 +0,0 @@
 # Releasing a new version of chatmail relay
 For example, to release version 1.9.0 of chatmail relay, do the following steps.
 1. Update the changelog: `git cliff --unreleased --tag 1.9.0 --prepend CHANGELOG.md` or `git cliff -u -t 1.9.0 -p CHANGELOG.md`.
 2. Open the changelog in the editor, edit it if required.
 3. Commit the changes to the changelog with a commit message `chore(release): prepare for 1.9.0`.
 3. Tag the release: `git tag --annotate 1.9.0`.
 4. Push the release tag: `git push origin 1.9.0`.
 5. Create a GitHub release: `gh release create 1.9.0`.
--- a/chatmaild/MANIFEST.in
+++ b/chatmaild/MANIFEST.in
@@ -1,3 +0,0 @@
 include src/chatmaild/ini/*.ini.f
 include src/chatmaild/ini/*.ini
 include src/chatmaild/tests/mail-data/*
--- a/chatmaild/pyproject.toml
+++ b/chatmaild/pyproject.toml
@@ -1,56 +1,20 @@
 [build-system]
-requires = ["setuptools>=61"]
+requires = ["setuptools>=45"]
 build-backend = "setuptools.build_meta"
 [project]
 name = "chatmaild"
-version = "0.3"
+version = "0.1"
 dependencies = [
-  "aiosmtpd",
+  "aiosmtpd"
  "iniconfig",
  "deltachat-rpc-server",
  "deltachat-rpc-client",
  "filelock",
  "requests",
  "crypt-r >= 3.13.1 ; python_version >= '3.11'",
 ]
 [tool.setuptools]
 include-package-data = true
 [tool.setuptools.packages.find]
 where = ['src']
 [project.scripts]
-doveauth = "chatmaild.doveauth:main"
+doveauth-dictproxy = "chatmaild.dictproxy:main"
-chatmail-metadata = "chatmaild.metadata:main"
+filtermail = "chatmaild.filtermail:main"
 chatmail-metrics = "chatmaild.metrics:main"
 chatmail-expire = "chatmaild.expire:main"
 chatmail-fsreport = "chatmaild.fsreport:main"
 lastlogin = "chatmaild.lastlogin:main"
 turnserver = "chatmaild.turnserver:main"
 [project.entry-points.pytest11]
 "chatmaild.testplugin" = "chatmaild.tests.plugin"
 [tool.pytest.ini_options]
 addopts = "-v -ra --strict-markers"
 log_format = "%(asctime)s %(levelname)s %(message)s"
 log_date_format = "%Y-%m-%d %H:%M:%S"
 log_level = "INFO"
 [tool.ruff]
 lint.select = [
  "F", # Pyflakes
  "I", # isort
  "PLC", # Pylint Convention
  "PLE", # Pylint Error
  "PLW", # Pylint Warning
 ]
 lint.ignore = [
  "PLC0415" # import-outside-top-level
 ]
 [tool.tox]
 legacy_tox_ini = """
@@ -63,14 +27,14 @@ skipdist = True
 skip_install = True
 deps =
  ruff
  black
 commands =
-  ruff format --quiet --diff src/
+  black --quiet --check --diff src/
-  ruff check src/
+  ruff src/
 [testenv]
 passenv = CHATMAIL_DOMAIN
 deps = pytest 
       pdbpp 
-       pytest-localserver
+commands = pytest -v -rsXx {posargs: ../tests/chatmaild}
       execnet
 commands = pytest -v -rsXx {posargs}
 """
--- a/chatmaild/src/chatmaild/init.py
+++ b/chatmaild/src/chatmaild/init.py
@@ -1 +0,0 @@
--- a/chatmaild/src/chatmaild/config.py
+++ b/chatmaild/src/chatmaild/config.py
@@ -1,135 +0,0 @@
 import os
 from pathlib import Path
 import iniconfig
 from chatmaild.user import User
 def read_config(inipath):
    assert Path(inipath).exists(), inipath
    cfg = iniconfig.IniConfig(inipath)
    params = cfg.sections["params"]
    default_config_content = get_default_config_content(params["mail_domain"])
    df_params = iniconfig.IniConfig("ini", data=default_config_content)["params"]
    new_params = dict(df_params.items())
    new_params.update(params)
    return Config(inipath, params=new_params)
 class Config:
    def __init__(self, inipath, params):
        self._inipath = inipath
        self.mail_domain = params["mail_domain"]
        self.max_user_send_per_minute = int(params.get("max_user_send_per_minute", 60))
        self.max_user_send_burst_size = int(params.get("max_user_send_burst_size", 10))
        self.max_mailbox_size = params["max_mailbox_size"]
        self.max_message_size = int(params.get("max_message_size", "31457280"))
        self.delete_mails_after = params["delete_mails_after"]
        self.delete_large_after = params["delete_large_after"]
        self.delete_inactive_users_after = int(params["delete_inactive_users_after"])
        self.username_min_length = int(params["username_min_length"])
        self.username_max_length = int(params["username_max_length"])
        self.password_min_length = int(params["password_min_length"])
        self.passthrough_senders = params["passthrough_senders"].split()
        self.passthrough_recipients = params["passthrough_recipients"].split()
        self.www_folder = params.get("www_folder", "")
        self.filtermail_smtp_port = int(params.get("filtermail_smtp_port", "10080"))
        self.filtermail_smtp_port_incoming = int(
            params.get("filtermail_smtp_port_incoming", "10081")
        )
        self.postfix_reinject_port = int(params.get("postfix_reinject_port", "10025"))
        self.postfix_reinject_port_incoming = int(
            params.get("postfix_reinject_port_incoming", "10026")
        )
        self.mtail_address = params.get("mtail_address")
        self.disable_ipv6 = params.get("disable_ipv6", "false").lower() == "true"
        self.noacme = os.environ.get("CHATMAIL_NOACME", "false").lower() == "true"
        self.addr_v4 = os.environ.get("CHATMAIL_ADDR_V4", "")
        self.addr_v6 = os.environ.get("CHATMAIL_ADDR_V6", "")
        self.acme_email = params.get("acme_email", "")
        self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true"
        self.imap_compress = params.get("imap_compress", "false").lower() == "true"
        if "iroh_relay" not in params:
            self.iroh_relay = "https://" + params["mail_domain"]
            self.enable_iroh_relay = True
        else:
            self.iroh_relay = params["iroh_relay"].strip()
            self.enable_iroh_relay = False
        self.privacy_postal = params.get("privacy_postal")
        self.privacy_mail = params.get("privacy_mail")
        self.privacy_pdo = params.get("privacy_pdo")
        self.privacy_supervisor = params.get("privacy_supervisor")
        # deprecated option
        mbdir = params.get("mailboxes_dir", f"/home/vmail/mail/{self.mail_domain}")
        self.mailboxes_dir = Path(mbdir.strip())
        # old unused option (except for first migration from sqlite to maildir store)
        self.passdb_path = Path(params.get("passdb_path", "/home/vmail/passdb.sqlite"))
    def _getbytefile(self):
        return open(self._inipath, "rb")
    def get_user(self, addr) -> User:
        if not addr or "@" not in addr or "/" in addr:
            raise ValueError(f"invalid address {addr!r}")
        maildir = self.mailboxes_dir.joinpath(addr)
        password_path = maildir.joinpath("password")
        return User(maildir, addr, password_path, uid="vmail", gid="vmail")
 def write_initial_config(inipath, mail_domain, overrides):
    """Write out default config file, using the specified config value overrides."""
    content = get_default_config_content(mail_domain, **overrides)
    inipath.write_text(content)
 def get_default_config_content(mail_domain, **overrides):
    from importlib.resources import files
    inidir = files(__package__).joinpath("ini")
    source_inipath = inidir.joinpath("chatmail.ini.f")
    content = source_inipath.read_text().format(mail_domain=mail_domain)
    # apply config overrides
    new_lines = []
    extra = overrides.copy()
    for line in content.split("\n"):
        new_line = line.strip()
        if new_line and new_line[0] not in "#[":
            name, value = map(str.strip, new_line.split("=", maxsplit=1))
            value = extra.pop(name, value)
            new_line = f"{name} = {value}"
        new_lines.append(new_line)
    for name, value in extra.items():
        new_line = f"{name} = {value}"
        new_lines.append(new_line)
    content = "\n".join(new_lines)
    # apply testrun privacy overrides
    if mail_domain.endswith(".testrun.org"):
        override_inipath = inidir.joinpath("override-testrun.ini")
        privacy = iniconfig.IniConfig(override_inipath)["privacy"]
        lines = []
        for line in content.split("\n"):
            for key, value in privacy.items():
                value_lines = value.format(mail_domain=mail_domain).strip().split("\n")
                if not line.startswith(f"{key} =") or not value_lines:
                    continue
                if len(value_lines) == 1:
                    lines.append(f"{key} = {value}")
                else:
                    lines.append(f"{key} =")
                    for vl in value_lines:
                        lines.append(f"    {vl}")
                break
            else:
                lines.append(line)
        content = "\n".join(lines)
    return content
--- a/chatmaild/src/chatmaild/database.py
+++ b/chatmaild/src/chatmaild/database.py
@@ -0,0 +1,140 @@
 import sqlite3
 import contextlib
 import time
 from pathlib import Path
 class DBError(Exception):
    """error during an operation on the database."""
 class Connection:
    def __init__(self, sqlconn, write):
        self._sqlconn = sqlconn
        self._write = write
    def close(self):
        self._sqlconn.close()
    def commit(self):
        self._sqlconn.commit()
    def rollback(self):
        self._sqlconn.rollback()
    def execute(self, query, params=()):
        cur = self.cursor()
        try:
            cur.execute(query, params)
        except sqlite3.IntegrityError as e:
            raise DBError(e)
        return cur
    def cursor(self):
        return self._sqlconn.cursor()
    def create_user(self, addr: str, password: str):
        """Create a row in the users table."""
        self.execute("PRAGMA foreign_keys=on")
        q = """INSERT INTO users (addr, password, last_login)
               VALUES (?, ?, ?)"""
        self.execute(q, (addr, password, int(time.time())))
    def get_user(self, addr: str) -> {}:
        """Get a row from the users table."""
        q = "SELECT addr, password, last_login from users WHERE addr = ?"
        row = self._sqlconn.execute(q, (addr,)).fetchone()
        result = {}
        if row:
            result = dict(
                user=row[0],
                password=row[1],
                last_login=row[2],
            )
        return result
 class Database:
    def __init__(self, path: str):
        self.path = Path(path)
        self.ensure_tables()
    def _get_connection(
        self, write=False, transaction=False, closing=False
    ) -> Connection:
        # we let the database serialize all writers at connection time
        # to play it very safe (we don't have massive amounts of writes).
        mode = "ro"
        if write:
            mode = "rw"
        if not self.path.exists():
            mode = "rwc"
        uri = "file:%s?mode=%s" % (self.path, mode)
        sqlconn = sqlite3.connect(
            uri,
            timeout=60,
            isolation_level=None if transaction else "DEFERRED",
            uri=True,
        )
        # Enable Write-Ahead Logging to avoid readers blocking writers and vice versa.
        if write:
            sqlconn.execute("PRAGMA journal_mode=wal")
        if transaction:
            start_time = time.time()
            while 1:
                try:
                    sqlconn.execute("begin immediate")
                    break
                except sqlite3.OperationalError:
                    # another thread may be writing, give it a chance to finish
                    time.sleep(0.1)
                    if time.time() - start_time > 5:
                        # if it takes this long, something is wrong
                        raise
        conn = Connection(sqlconn, write=write)
        if closing:
            conn = contextlib.closing(conn)
        return conn
    @contextlib.contextmanager
    def write_transaction(self):
        conn = self._get_connection(closing=False, write=True, transaction=True)
        try:
            yield conn
        except Exception:
            conn.rollback()
            conn.close()
            raise
        else:
            conn.commit()
            conn.close()
    def read_connection(self, closing=True) -> Connection:
        return self._get_connection(closing=closing, write=False)
    def get_schema_version(self) -> int:
        with self.read_connection() as conn:
            dbversion = conn.execute("PRAGMA user_version").fetchone()[0]
        return dbversion
    CURRENT_DBVERSION = 1
    def ensure_tables(self):
        with self.write_transaction() as conn:
            if self.get_schema_version() > 1:
                raise DBError(
                    "version is %s; downgrading schema is not supported"
                    % (self.get_schema_version(),)
                )
            conn.execute(
                """
                CREATE TABLE IF NOT EXISTS users (
                    addr TEXT PRIMARY KEY,
                    password TEXT,
                    last_login INTEGER
                )
            """,
            )
            conn.execute("PRAGMA user_version=%s" % (self.CURRENT_DBVERSION,))
--- a/chatmaild/src/chatmaild/dictproxy.py
+++ b/chatmaild/src/chatmaild/dictproxy.py
@@ -1,98 +1,119 @@
 import logging
 import os
-from socketserver import StreamRequestHandler, ThreadingUnixStreamServer
+import sys
 import json
 import crypt
 from socketserver import (
    UnixStreamServer,
    StreamRequestHandler,
    ThreadingMixIn,
 )
 import pwd
 from .database import Database
 NOCREATE_FILE = "/etc/chatmail-nocreate"
-class DictProxy:
+def encrypt_password(password: str):
-    def loop_forever(self, rfile, wfile):
+    # https://doc.dovecot.org/configuration_manual/authentication/password_schemes/
-        # Transaction storage is local to each handler loop.
+    passhash = crypt.crypt(password, crypt.METHOD_SHA512)
-        # Dovecot reuses transaction IDs across connections,
+    return "{SHA512-CRYPT}" + passhash
        # starting transaction with the name `1`
        # on two different connections to the same proxy sometimes.
        transactions = {}
        while True:
            msg = rfile.readline().strip().decode()
            if not msg:
                break
-            res = self.handle_dovecot_request(msg, transactions)
+def create_user(db, user, password):
-            if res:
+    if os.path.exists(NOCREATE_FILE):
-                wfile.write(res.encode("ascii"))
+        logging.warning(
-                wfile.flush()
+            f"Didn't create account: {NOCREATE_FILE} exists. Delete the file to enable account creation."
-
+        )
    def handle_dovecot_request(self, msg, transactions):
        # see https://doc.dovecot.org/2.3/developer_manual/design/dict_protocol/#dovecot-dict-protocol
        short_command = msg[0]
        parts = msg[1:].split("\t")
        if short_command == "L":
            return self.handle_lookup(parts)
        elif short_command == "I":
            return self.handle_iterate(parts)
        elif short_command == "H":
            return  # no version checking
        if short_command not in ("BSC"):
            logging.warning(f"unknown dictproxy request: {msg!r}")
        return
    with db.write_transaction() as conn:
        conn.create_user(user, password)
    return dict(home=f"/home/vmail/{user}", uid="vmail", gid="vmail", password=password)
        transaction_id = parts[0]
-        if short_command == "B":
+def get_user_data(db, user):
-            return self.handle_begin_transaction(transaction_id, parts, transactions)
+    with db.read_connection() as conn:
-        elif short_command == "C":
+        result = conn.get_user(user)
-            return self.handle_commit_transaction(transaction_id, parts, transactions)
+    if result:
-        elif short_command == "S":
+        result["uid"] = "vmail"
-            addr = transactions[transaction_id]["addr"]
+        result["gid"] = "vmail"
-            if not self.handle_set(addr, parts):
+    return result
                transactions[transaction_id]["res"] = "F\n"
                logging.error(f"dictproxy-set failed for {addr!r}: {msg!r}")
    def handle_lookup(self, parts):
        logging.warning(f"lookup ignored: {parts!r}")
        return "N\n"
-    def handle_iterate(self, parts):
+def lookup_userdb(db, user):
-        # Empty line means ITER_FINISHED.
+    return get_user_data(db, user)
        # If we don't return empty line Dovecot will timeout.
        return "\n"
    def handle_begin_transaction(self, transaction_id, parts, transactions):
        addr = parts[1]
        transactions[transaction_id] = dict(addr=addr, res="O\n")
-    def handle_set(self, addr, parts):
+def lookup_passdb(db, user, password):
-        # For documentation on key structure see
+    userdata = get_user_data(db, user)
-        # https://github.com/dovecot/core/blob/main/src/lib-storage/mailbox-attribute.h
+    if not userdata:
-        return False
+        return create_user(db, user, encrypt_password(password))
    userdata["password"] = userdata["password"].strip()
    return userdata
    def handle_commit_transaction(self, transaction_id, parts, transactions):
        # return whatever "set" command(s) set as result.
        return transactions.pop(transaction_id)["res"]
-    def serve_forever_from_socket(self, socket):
+def handle_dovecot_request(msg, db, mail_domain):
-        dictproxy = self
+    print(f"received msg: {msg!r}", file=sys.stderr)
    short_command = msg[0]
    if short_command == "L":  # LOOKUP
        parts = msg[1:].split("\t")
        keyname, user = parts[:2]
        namespace, type, *args = keyname.split("/")
        reply_command = "F"
        res = ""
        if namespace == "shared":
            if type == "userdb":
                if user.endswith(f"@{mail_domain}"):
                    res = lookup_userdb(db, user)
                if res:
                    reply_command = "O"
                else:
                    reply_command = "N"
            elif type == "passdb":
                if user.endswith(f"@{mail_domain}"):
                    res = lookup_passdb(db, user, password=args[0])
                if res:
                    reply_command = "O"
                else:
                    reply_command = "N"
        print(f"res: {res!r}", file=sys.stderr)
        json_res = json.dumps(res) if res else ""
        return f"{reply_command}{json_res}\n"
    return None
 class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
    pass
 def main():
    socket = sys.argv[1]
    passwd_entry = pwd.getpwnam(sys.argv[2])
    db = Database(sys.argv[3])
    with open("/etc/mailname", "r") as fp:
        mail_domain = fp.read().strip()
    class Handler(StreamRequestHandler):
        def handle(self):
-                try:
+            while True:
-                    dictproxy.loop_forever(self.rfile, self.wfile)
+                msg = self.rfile.readline().strip().decode()
-                except Exception:
+                if not msg:
-                    logging.exception("Exception in the handler")
+                    break
-                    raise
+                res = handle_dovecot_request(msg, db, mail_domain)
                if res:
                    print(f"sending result: {res!r}", file=sys.stderr)
                    self.wfile.write(res.encode("ascii"))
                    self.wfile.flush()
    try:
        os.unlink(socket)
    except FileNotFoundError:
        pass
-        with CustomThreadingUnixStreamServer(socket, Handler) as server:
+    with ThreadedUnixStreamServer(socket, Handler) as server:
        os.chown(socket, uid=passwd_entry.pw_uid, gid=passwd_entry.pw_gid)
        try:
            server.serve_forever()
        except KeyboardInterrupt:
            pass
 class CustomThreadingUnixStreamServer(ThreadingUnixStreamServer):
    request_queue_size = 1000
--- a/chatmaild/src/chatmaild/doveauth-dictproxy.service
+++ b/chatmaild/src/chatmaild/doveauth-dictproxy.service
@@ -0,0 +1,10 @@
 [Unit]
 Description=Dict authentication proxy for dovecot
 [Service]
 ExecStart=/usr/local/bin/doveauth-dictproxy /run/dovecot/doveauth.socket vmail /home/vmail/passdb.sqlite
 Restart=always
 RestartSec=30
 [Install]
 WantedBy=multi-user.target
--- a/chatmaild/src/chatmaild/doveauth.py
+++ b/chatmaild/src/chatmaild/doveauth.py
@@ -1,156 +0,0 @@
 import json
 import logging
 import os
 import sys
 try:
    import crypt_r
 except ImportError:
    import crypt as crypt_r
 from .config import Config, read_config
 from .dictproxy import DictProxy
 from .migrate_db import migrate_from_db_to_maildir
 NOCREATE_FILE = "/etc/chatmail-nocreate"
 def encrypt_password(password: str):
    # https://doc.dovecot.org/2.3/configuration_manual/authentication/password_schemes/
    passhash = crypt_r.crypt(password, crypt_r.METHOD_SHA512)
    return "{SHA512-CRYPT}" + passhash
 def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
    """Return True if user and password are admissable."""
    if os.path.exists(NOCREATE_FILE):
        logging.warning(f"blocked account creation because {NOCREATE_FILE!r} exists.")
        return False
    if len(cleartext_password) < config.password_min_length:
        logging.warning(
            "Password needs to be at least %s characters long",
            config.password_min_length,
        )
        return False
    parts = user.split("@")
    if len(parts) != 2:
        logging.warning(f"user {user!r} is not a proper e-mail address")
        return False
    localpart, domain = parts
    if (
        len(localpart) > config.username_max_length
        or len(localpart) < config.username_min_length
    ):
        logging.warning(
            "localpart %s has to be between %s and %s chars long",
            localpart,
            config.username_min_length,
            config.username_max_length,
        )
        return False
    return True
 def split_and_unescape(s):
    """Split strings using double quote as a separator and backslash as escape character
    into parts."""
    out = ""
    i = 0
    while i < len(s):
        c = s[i]
        if c == "\\":
            # Skip escape character.
            i += 1
            # This will raise IndexError if there is no character
            # after escape character. This is expected
            # as this is an invalid input.
            out += s[i]
        elif c == '"':
            # Separator
            yield out
            out = ""
        else:
            out += c
        i += 1
    yield out
 class AuthDictProxy(DictProxy):
    def __init__(self, config):
        super().__init__()
        self.config = config
    def handle_lookup(self, parts):
        # Dovecot <2.3.17 has only one part,
        # do not attempt to read any other parts for compatibility.
        keyname = parts[0]
        namespace, type, args = keyname.split("/", 2)
        args = list(split_and_unescape(args))
        config = self.config
        reply_command = "F"
        res = ""
        if namespace == "shared":
            if type == "userdb":
                user = args[0]
                if user.endswith(f"@{config.mail_domain}"):
                    res = self.lookup_userdb(user)
                if res:
                    reply_command = "O"
                else:
                    reply_command = "N"
            elif type == "passdb":
                user = args[1]
                if user.endswith(f"@{config.mail_domain}"):
                    res = self.lookup_passdb(user, cleartext_password=args[0])
                if res:
                    reply_command = "O"
                else:
                    reply_command = "N"
        json_res = json.dumps(res) if res else ""
        return f"{reply_command}{json_res}\n"
    def handle_iterate(self, parts):
        # example: I0\t0\tshared/userdb/
        if parts[2] == "shared/userdb/":
            result = "".join(
                f"Oshared/userdb/{user}\t\n" for user in self.iter_userdb()
            )
            return f"{result}\n"
    def iter_userdb(self) -> list:
        """Get a list of all user addresses."""
        return [x for x in os.listdir(self.config.mailboxes_dir) if "@" in x]
    def lookup_userdb(self, addr):
        return self.config.get_user(addr).get_userdb_dict()
    def lookup_passdb(self, addr, cleartext_password):
        user = self.config.get_user(addr)
        userdata = user.get_userdb_dict()
        if userdata:
            return userdata
        if not is_allowed_to_create(self.config, addr, cleartext_password):
            return
        user.set_password(encrypt_password(cleartext_password))
        print(f"Created address: {addr}", file=sys.stderr)
        return user.get_userdb_dict()
 def main():
    socket, cfgpath = sys.argv[1:]
    config = read_config(cfgpath)
    migrate_from_db_to_maildir(config)
    dictproxy = AuthDictProxy(config=config)
    dictproxy.serve_forever_from_socket(socket)
--- a/chatmaild/src/chatmaild/expire.py
+++ b/chatmaild/src/chatmaild/expire.py
@@ -1,206 +0,0 @@
 """
 Expire old messages and addresses.
 """
 import os
 import shutil
 import sys
 import time
 from argparse import ArgumentParser
 from collections import namedtuple
 from datetime import datetime
 from stat import S_ISREG
 from chatmaild.config import read_config
 FileEntry = namedtuple("FileEntry", ("path", "mtime", "size"))
 def iter_mailboxes(basedir, maxnum):
    if not os.path.exists(basedir):
        print_info(f"no mailboxes found at: {basedir}")
        return
    for name in os_listdir_if_exists(basedir)[:maxnum]:
        if "@" in name:
            yield MailboxStat(basedir + "/" + name)
 def get_file_entry(path):
    """return a FileEntry or None if the path does not exist or is not a regular file."""
    try:
        st = os.stat(path)
    except FileNotFoundError:
        return None
    if not S_ISREG(st.st_mode):
        return None
    return FileEntry(path, st.st_mtime, st.st_size)
 def os_listdir_if_exists(path):
    """return a list of names obtained from os.listdir or an empty list if the path does not exist."""
    try:
        return os.listdir(path)
    except FileNotFoundError:
        return []
 class MailboxStat:
    last_login = None
    def __init__(self, basedir):
        self.basedir = str(basedir)
        self.messages = []
        self.extrafiles = []
        self.scandir(self.basedir)
    def scandir(self, folderdir):
        for name in os_listdir_if_exists(folderdir):
            path = f"{folderdir}/{name}"
            if name in ("cur", "new", "tmp"):
                for msg_name in os_listdir_if_exists(path):
                    entry = get_file_entry(f"{path}/{msg_name}")
                    if entry is not None:
                        self.messages.append(entry)
            elif os.path.isdir(path):
                self.scandir(path)
            else:
                entry = get_file_entry(path)
                if entry is not None:
                    self.extrafiles.append(entry)
                    if name == "password":
                        self.last_login = entry.mtime
        self.extrafiles.sort(key=lambda x: -x.size)
 def print_info(msg):
    print(msg, file=sys.stderr)
 class Expiry:
    def __init__(self, config, dry, now, verbose):
        self.config = config
        self.dry = dry
        self.now = now
        self.verbose = verbose
        self.del_mboxes = 0
        self.all_mboxes = 0
        self.del_files = 0
        self.all_files = 0
        self.start = time.time()
    def remove_mailbox(self, mboxdir):
        if self.verbose:
            print_info(f"removing {mboxdir}")
        if not self.dry:
            shutil.rmtree(mboxdir)
        self.del_mboxes += 1
    def remove_file(self, path, mtime=None):
        if self.verbose:
            if mtime is not None:
                date = datetime.fromtimestamp(mtime).strftime("%b %d")
                print_info(f"removing {date} {path}")
            else:
                print_info(f"removing {path}")
        if not self.dry:
            try:
                os.unlink(path)
            except FileNotFoundError:
                print_info(f"file not found/vanished {path}")
        self.del_files += 1
    def process_mailbox_stat(self, mbox):
        cutoff_without_login = (
            self.now - int(self.config.delete_inactive_users_after) * 86400
        )
        cutoff_mails = self.now - int(self.config.delete_mails_after) * 86400
        cutoff_large_mails = self.now - int(self.config.delete_large_after) * 86400
        self.all_mboxes += 1
        changed = False
        if mbox.last_login and mbox.last_login < cutoff_without_login:
            self.remove_mailbox(mbox.basedir)
            return
        mboxname = os.path.basename(mbox.basedir)
        if self.verbose:
            date = datetime.fromtimestamp(mbox.last_login) if mbox.last_login else None
            if date:
                print_info(f"checking mailbox {date.strftime('%b %d')} {mboxname}")
            else:
                print_info(f"checking mailbox (no last_login) {mboxname}")
        self.all_files += len(mbox.messages)
        for message in mbox.messages:
            if message.mtime < cutoff_mails:
                self.remove_file(message.path, mtime=message.mtime)
            elif message.size > 200000 and message.mtime < cutoff_large_mails:
                # we only remove noticed large files (not unnoticed ones in new/)
                parts = message.path.split("/")
                if len(parts) >= 2 and parts[-2] == "cur":
                    self.remove_file(message.path, mtime=message.mtime)
            else:
                continue
            changed = True
        if changed:
            self.remove_file(f"{mbox.basedir}/maildirsize")
    def get_summary(self):
        return (
            f"Removed {self.del_mboxes} out of {self.all_mboxes} mailboxes "
            f"and {self.del_files} out of {self.all_files} files in existing mailboxes "
            f"in {time.time() - self.start:2.2f} seconds"
        )
 def main(args=None):
    """Expire mailboxes and messages according to chatmail config"""
    parser = ArgumentParser(description=main.__doc__)
    ini = "/usr/local/lib/chatmaild/chatmail.ini"
    parser.add_argument(
        "chatmail_ini",
        action="store",
        nargs="?",
        help=f"path pointing to chatmail.ini file, default: {ini}",
        default=ini,
    )
    parser.add_argument(
        "--days", action="store", help="assume date to be days older than now"
    )
    parser.add_argument(
        "--maxnum",
        default=None,
        action="store",
        help="maximum number of mailboxes to iterate on",
    )
    parser.add_argument(
        "-v",
        dest="verbose",
        action="store_true",
        help="print out removed files and mailboxes",
    )
    parser.add_argument(
        "--remove",
        dest="remove",
        action="store_true",
        help="actually remove all expired files and dirs",
    )
    args = parser.parse_args(args)
    config = read_config(args.chatmail_ini)
    now = datetime.utcnow().timestamp()
    if args.days:
        now = now - 86400 * int(args.days)
    maxnum = int(args.maxnum) if args.maxnum else None
    exp = Expiry(config, dry=not args.remove, now=now, verbose=args.verbose)
    for mailbox in iter_mailboxes(str(config.mailboxes_dir), maxnum=maxnum):
        exp.process_mailbox_stat(mailbox)
    print(exp.get_summary())
 if __name__ == "__main__":
    main(sys.argv[1:])
--- a/chatmaild/src/chatmaild/filedict.py
+++ b/chatmaild/src/chatmaild/filedict.py
@@ -1,44 +0,0 @@
 import json
 import logging
 import os
 from contextlib import contextmanager
 from random import randint
 import filelock
 class FileDict:
    """Concurrency-safe multi-reader/single-writer persistent dict."""
    def __init__(self, path):
        self.path = path
        self.lock_path = path.with_name(path.name + ".lock")
    @contextmanager
    def modify(self):
        # the OS will release the lock if the process dies,
        # and the contextmanager will otherwise guarantee release
        with filelock.FileLock(self.lock_path):
            data = self.read()
            yield data
            write_path = self.path.with_name(self.path.name + ".tmp")
            with write_path.open("w") as f:
                json.dump(data, f)
            os.rename(write_path, self.path)
    def read(self):
        try:
            with self.path.open("r") as f:
                return json.load(f)
        except FileNotFoundError:
            return {}
        except Exception:
            logging.warning(f"corrupt serialization state at: {self.path!r}")
            return {}
 def write_bytes_atomic(path, content):
    rint = randint(0, 10000000)
    tmp = path.with_name(path.name + f".tmp-{rint}")
    tmp.write_bytes(content)
    os.rename(tmp, path)
--- a/chatmaild/src/chatmaild/filtermail.py
+++ b/chatmaild/src/chatmaild/filtermail.py
@@ -0,0 +1,157 @@
 #!/usr/bin/env python3
 import asyncio
 import logging
 import time
 import sys
 from email.parser import BytesParser
 from email import policy
 from email.utils import parseaddr
 from aiosmtpd.smtp import SMTP
 from aiosmtpd.controller import Controller
 from smtplib import SMTP as SMTPClient
 def check_encrypted(message):
    """Check that the message is an OpenPGP-encrypted message."""
    if not message.is_multipart():
        return False
    if message.get("subject") != "...":
        return False
    if message.get_content_type() != "multipart/encrypted":
        return False
    parts_count = 0
    for part in message.iter_parts():
        if parts_count == 0:
            if part.get_content_type() != "application/pgp-encrypted":
                return False
        elif parts_count == 1:
            if part.get_content_type() != "application/octet-stream":
                return False
        else:
            return False
        parts_count += 1
    return True
 def check_mdn(message, envelope):
    if len(envelope.rcpt_tos) != 1:
        return False
    for name in ["auto-submitted", "chat-version"]:
        if not message.get(name):
            return False
    if message.get_content_type() != "multipart/report":
        return False
    body = message.get_body()
    if body.get_content_type() != "text/plain":
        return False
    if list(body.iter_attachments()) or list(body.iter_parts()):
        return False
    # even with all mime-structural checks an attacker
    # could try to abuse the subject or body to contain links or other
    # annoyance -- we skip on checking subject/body for now as Delta Chat
    # should evolve to create E2E-encrypted read receipts anyway.
    # and then MDNs are just encrypted mail and can pass the border
    # to other instances.
    return True
 class SMTPController(Controller):
    def factory(self):
        return SMTP(self.handler, **self.SMTP_kwargs)
 class BeforeQueueHandler:
    def __init__(self):
        self.send_rate_limiter = SendRateLimiter()
    async def handle_MAIL(self, server, session, envelope, address, mail_options):
        logging.info(f"handle_MAIL from {address}")
        envelope.mail_from = address
        if not self.send_rate_limiter.is_sending_allowed(address):
            return f"450 4.7.1: Too much mail from {address}"
        parts = envelope.mail_from.split("@")
        if len(parts) != 2:
            return f"500 Invalid from address <{envelope.mail_from!r}>"
        return "250 OK"
    async def handle_DATA(self, server, session, envelope):
        logging.info("handle_DATA before-queue")
        error = check_DATA(envelope)
        if error:
            return error
        logging.info("re-injecting the mail that passed checks")
        client = SMTPClient("localhost", "10025")
        client.sendmail(envelope.mail_from, envelope.rcpt_tos, envelope.content)
        return "250 OK"
 async def asyncmain_beforequeue(port):
    Controller(BeforeQueueHandler(), hostname="127.0.0.1", port=port).start()
 def check_DATA(envelope):
    """the central filtering function for e-mails."""
    logging.info(f"Processing DATA message from {envelope.mail_from}")
    message = BytesParser(policy=policy.default).parsebytes(envelope.content)
    mail_encrypted = check_encrypted(message)
    _, from_addr = parseaddr(message.get("from").strip())
    logging.info(f"mime-from: {from_addr} envelope-from: {envelope.mail_from!r}")
    if envelope.mail_from.lower() != from_addr.lower():
        return f"500 Invalid FROM <{from_addr!r}> for <{envelope.mail_from!r}>"
    if not mail_encrypted and check_mdn(message, envelope):
        return
    envelope_from_domain = from_addr.split("@").pop()
    for recipient in envelope.rcpt_tos:
        if envelope.mail_from == recipient:
            # Always allow sending emails to self.
            continue
        res = recipient.split("@")
        if len(res) != 2:
            return f"500 Invalid address <{recipient}>"
        _recipient_addr, recipient_domain = res
        is_outgoing = recipient_domain != envelope_from_domain
        if is_outgoing and not mail_encrypted:
            is_securejoin = message.get("secure-join") in ["vc-request", "vg-request"]
            if not is_securejoin:
                return f"500 Invalid unencrypted mail to <{recipient}>"
 class SendRateLimiter:
    MAX_USER_SEND_PER_MINUTE = 80
    def __init__(self):
        self.addr2timestamps = {}
    def is_sending_allowed(self, mail_from):
        last = self.addr2timestamps.setdefault(mail_from, [])
        now = time.time()
        last[:] = [ts for ts in last if ts >= (now - 60)]
        if len(last) <= self.MAX_USER_SEND_PER_MINUTE:
            last.append(now)
            return True
        return False
 def main():
    args = sys.argv[1:]
    assert len(args) == 1
    logging.basicConfig(level=logging.INFO)
    loop = asyncio.new_event_loop()
    asyncio.set_event_loop(loop)
    task = asyncmain_beforequeue(port=int(args[0]))
    loop.create_task(task)
    loop.run_forever()
--- a/chatmaild/src/chatmaild/filtermail.service
+++ b/chatmaild/src/chatmaild/filtermail.service
@@ -0,0 +1,10 @@
 [Unit]
 Description=Chatmail Postfix BeforeQeue filter 
 [Service]
 ExecStart=/usr/local/bin/filtermail 10080
 Restart=always
 RestartSec=30
 [Install]
 WantedBy=multi-user.target
--- a/chatmaild/src/chatmaild/fsreport.py
+++ b/chatmaild/src/chatmaild/fsreport.py
@@ -1,168 +0,0 @@
 """
 command line tool to analyze mailbox message storage
 example invocation:
    python -m chatmaild.fsreport /path/to/chatmail.ini
 to show storage summaries for all "cur" folders
    python -m chatmaild.fsreport /path/to/chatmail.ini --mdir cur
 to show storage summaries only for first 1000 mailboxes
    python -m chatmaild.fsreport /path/to/chatmail.ini --maxnum 1000
 """
 import os
 from argparse import ArgumentParser
 from datetime import datetime
 from chatmaild.config import read_config
 from chatmaild.expire import iter_mailboxes
 DAYSECONDS = 24 * 60 * 60
 MONTHSECONDS = DAYSECONDS * 30
 def HSize(size: int):
    """Format a size integer as a Human-readable string Kilobyte, Megabyte or Gigabyte"""
    if size < 10000:
        return f"{size / 1000:5.2f}K"
    if size < 1000 * 1000:
        return f"{size / 1000:5.0f}K"
    if size < 1000 * 1000 * 1000:
        return f"{int(size / 1000000):5.0f}M"
    return f"{size / 1000000000:5.2f}G"
 class Report:
    def __init__(self, now, min_login_age, mdir):
        self.size_extra = 0
        self.size_messages = 0
        self.now = now
        self.min_login_age = min_login_age
        self.mdir = mdir
        self.num_ci_logins = self.num_all_logins = 0
        self.login_buckets = {x: 0 for x in (1, 10, 30, 40, 80, 100, 150)}
        self.message_buckets = {x: 0 for x in (0, 160000, 500000, 2000000)}
    def process_mailbox_stat(self, mailbox):
        # categorize login times
        last_login = mailbox.last_login
        if last_login:
            self.num_all_logins += 1
            if os.path.basename(mailbox.basedir)[:3] == "ci-":
                self.num_ci_logins += 1
            else:
                for days in self.login_buckets:
                    if last_login >= self.now - days * DAYSECONDS:
                        self.login_buckets[days] += 1
        cutoff_login_date = self.now - self.min_login_age * DAYSECONDS
        if last_login and last_login <= cutoff_login_date:
            # categorize message sizes
            for size in self.message_buckets:
                for msg in mailbox.messages:
                    if msg.size >= size:
                        if self.mdir and not msg.relpath.startswith(self.mdir):
                            continue
                        self.message_buckets[size] += msg.size
        self.size_messages += sum(entry.size for entry in mailbox.messages)
        self.size_extra += sum(entry.size for entry in mailbox.extrafiles)
    def dump_summary(self):
        all_messages = self.size_messages
        print()
        print("## Mailbox storage use analysis")
        print(f"Mailbox data total size: {HSize(self.size_extra + all_messages)}")
        print(f"Messages total size    : {HSize(all_messages)}")
        try:
            percent = self.size_extra / (self.size_extra + all_messages) * 100
        except ZeroDivisionError:
            percent = 100
        print(f"Extra files : {HSize(self.size_extra)} ({percent:.2f}%)")
        print()
        if self.min_login_age:
            print(f"### Message storage for {self.min_login_age} days old logins")
        pref = f"[{self.mdir}] " if self.mdir else ""
        for minsize, sumsize in self.message_buckets.items():
            percent = (sumsize / all_messages * 100) if all_messages else 0
            print(
                f"{pref}larger than {HSize(minsize)}: {HSize(sumsize)} ({percent:.2f}%)"
            )
        user_logins = self.num_all_logins - self.num_ci_logins
        def p(num):
            return f"({num / user_logins * 100:2.2f}%)" if user_logins else "100%"
        print()
        print(f"## Login stats, from date reference {datetime.fromtimestamp(self.now)}")
        print(f"all:     {HSize(self.num_all_logins)}")
        print(f"non-ci:  {HSize(user_logins)}")
        print(f"ci:      {HSize(self.num_ci_logins)}")
        for days, active in self.login_buckets.items():
            print(f"last {days:3} days: {HSize(active)} {p(active)}")
 def main(args=None):
    """Report about filesystem storage usage of all mailboxes and messages"""
    parser = ArgumentParser(description=main.__doc__)
    ini = "/usr/local/lib/chatmaild/chatmail.ini"
    parser.add_argument(
        "chatmail_ini",
        action="store",
        nargs="?",
        help=f"path pointing to chatmail.ini file, default: {ini}",
        default=ini,
    )
    parser.add_argument(
        "--days",
        default=0,
        action="store",
        help="assume date to be days older than now",
    )
    parser.add_argument(
        "--min-login-age",
        default=0,
        dest="min_login_age",
        action="store",
        help="only sum up message size if last login is at least min-login-age days old",
    )
    parser.add_argument(
        "--mdir",
        action="store",
        help="only consider 'cur' or 'new' or 'tmp' messages for summary",
    )
    parser.add_argument(
        "--maxnum",
        default=None,
        action="store",
        help="maximum number of mailboxes to iterate on",
    )
    args = parser.parse_args(args)
    config = read_config(args.chatmail_ini)
    now = datetime.utcnow().timestamp()
    if args.days:
        now = now - 86400 * int(args.days)
    maxnum = int(args.maxnum) if args.maxnum else None
    rep = Report(now=now, min_login_age=int(args.min_login_age), mdir=args.mdir)
    for mbox in iter_mailboxes(str(config.mailboxes_dir), maxnum=maxnum):
        rep.process_mailbox_stat(mbox)
    rep.dump_summary()
 if __name__ == "__main__":
    main()
--- a/chatmaild/src/chatmaild/ini/chatmail.ini.f
+++ b/chatmaild/src/chatmaild/ini/chatmail.ini.f
@@ -1,126 +0,0 @@
 [params]
 # mail domain (MUST be set to fully qualified chat mail domain)
 mail_domain = {mail_domain}
 #
 # If you only do private test deploys, you don't need to modify any settings below
 #
 #
 # Restrictions on user addresses
 #
 # email sending rate per user and minute
 max_user_send_per_minute = 60
 # per-user max burst size for sending rate limiting (GCRA bucket capacity)
 max_user_send_burst_size = 10
 # maximum mailbox size of a chatmail address
 max_mailbox_size = 500M
 # maximum message size for an e-mail in bytes
 max_message_size = 31457280
 # days after which mails are unconditionally deleted
 delete_mails_after = 20
 # days after which large messages (>200k) are unconditionally deleted
 delete_large_after = 7
 # days after which users without a successful login are deleted (database and mails)
 delete_inactive_users_after = 90
 # minimum length a username must have
 username_min_length = 9
 # maximum length a username can have
 username_max_length = 9
 # minimum length a password must have
 password_min_length = 9
 # list of chatmail addresses which can send outbound un-encrypted mail
 passthrough_senders =
 # list of e-mail recipients for which to accept outbound un-encrypted mails
 # (space-separated, item may start with "@" to whitelist whole recipient domains)
 passthrough_recipients =
 # path to www directory - documented here: https://chatmail.at/doc/relay/getting_started.html#custom-web-pages
 #www_folder = www
 #
 # Deployment Details
 #
 # SMTP outgoing filtermail and reinjection 
 filtermail_smtp_port = 10080
 postfix_reinject_port = 10025
 # SMTP incoming filtermail and reinjection 
 filtermail_smtp_port_incoming = 10081
 postfix_reinject_port_incoming = 10026
 # if set to "True" IPv6 is disabled
 disable_ipv6 = False
 # Your email adress, which will be used in acmetool to manage Let's Encrypt SSL certificates
 acme_email = 
 # Defaults to https://iroh.{{mail_domain}} and running `iroh-relay` on the chatmail
 # service.
 # If you set it to anything else, the service will be disabled
 # and users will be directed to use the given iroh relay URL.
 # Set it to empty string if you want users to use their default iroh relay.
 # iroh_relay =
 # Address on which `mtail` listens,
 # e.g. 127.0.0.1 or some private network
 # address like 192.168.10.1.
 # You can point Prometheus
 # or some other OpenMetrics-compatible
 # collector to
 # http://{{mtail_address}}:3903/metrics
 # and display collected metrics with Grafana.
 #
 # WARNING: do not expose this service
 # to the public IP address.
 #
 # `mtail is not running if the setting is not set.
 # mtail_address = 127.0.0.1
 #
 # Debugging options 
 #
 # set to True if you want to track imap protocol execution
 # in per-maildir ".in/.out" files. 
 # Note that you need to manually cleanup these files
 # so use this option with caution on production servers. 
 imap_rawlog = false 
 # set to true if you want to enable the IMAP COMPRESS Extension,
 # which allows IMAP connections to be efficiently compressed.
 # WARNING: Enabling this makes it impossible to hibernate IMAP
 # processes which will result in much higher memory/RAM usage.
 imap_compress = false
 #
 # Privacy Policy
 #
 # postal address of privacy contact
 privacy_postal =
 # email address of privacy contact
 privacy_mail =
 # postal address of the privacy data officer
 privacy_pdo =
 # postal address of the privacy supervisor
 privacy_supervisor =
--- a/chatmaild/src/chatmaild/ini/override-testrun.ini
+++ b/chatmaild/src/chatmaild/ini/override-testrun.ini
@@ -1,16 +0,0 @@
 [privacy]
 passthrough_recipients = privacy@testrun.org echo@{mail_domain}
 privacy_postal =
    Merlinux GmbH, Represented by the managing director H. Krekel,
    Reichgrafen Str. 20, 79102 Freiburg, Germany
 privacy_mail = privacy@testrun.org
 privacy_pdo =
    Prof. Dr. Fabian Schmieder, lexICT UG (limited), Ostfeldstr. 49, 30559 Hannover.
    You can contact him at *delta-privacy@merlinux.eu* (Keyword: DPO)
 privacy_supervisor =
    State Commissioner for Data Protection and Freedom of Information of
    Baden-Württemberg in 70173 Stuttgart, Germany.
--- a/chatmaild/src/chatmaild/lastlogin.py
+++ b/chatmaild/src/chatmaild/lastlogin.py
@@ -1,29 +0,0 @@
 import sys
 from .config import read_config
 from .dictproxy import DictProxy
 class LastLoginDictProxy(DictProxy):
    def __init__(self, config):
        super().__init__()
        self.config = config
    def handle_set(self, addr, parts):
        keyname = parts[1].split("/")
        value = parts[2] if len(parts) > 2 else ""
        if keyname[0] == "shared" and keyname[1] == "last-login":
            addr = keyname[2]
            timestamp = int(value)
            user = self.config.get_user(addr)
            user.set_last_login_timestamp(timestamp)
            return True
        return False
 def main():
    socket, config_path = sys.argv[1:]
    config = read_config(config_path)
    dictproxy = LastLoginDictProxy(config=config)
    dictproxy.serve_forever_from_socket(socket)
--- a/chatmaild/src/chatmaild/metadata.py
+++ b/chatmaild/src/chatmaild/metadata.py
@@ -1,151 +0,0 @@
 import logging
 import sys
 import time
 from contextlib import contextmanager
 from .config import read_config
 from .dictproxy import DictProxy
 from .filedict import FileDict
 from .notifier import Notifier
 from .turnserver import turn_credentials
 def _is_valid_token_timestamp(timestamp, now):
    # Token if invalid after 90 days
    # or if the timestamp is in the future.
    return timestamp > now - 3600 * 24 * 90 and timestamp < now + 60
 class Metadata:
    # each SETMETADATA on this key appends to dictionary
    # mapping of unique device tokens
    # which only ever get removed if the upstream indicates the token is invalid
    DEVICETOKEN_KEY = "devicetoken"
    def __init__(self, vmail_dir):
        self.vmail_dir = vmail_dir
    def get_metadata_dict(self, addr):
        return FileDict(self.vmail_dir / addr / "metadata.json")
    @contextmanager
    def _modify_tokens(self, addr):
        with self.get_metadata_dict(addr).modify() as data:
            tokens = data.setdefault(self.DEVICETOKEN_KEY, {})
            now = int(time.time())
            if isinstance(tokens, list):
                data[self.DEVICETOKEN_KEY] = tokens = {t: now for t in tokens}
            expired_tokens = [
                token
                for token, timestamp in tokens.items()
                if not _is_valid_token_timestamp(tokens[token], now)
            ]
            for expired_token in expired_tokens:
                del tokens[expired_token]
            yield tokens
    def add_token_to_addr(self, addr, token):
        with self._modify_tokens(addr) as tokens:
            tokens[token] = int(time.time())
    def remove_token_from_addr(self, addr, token):
        with self._modify_tokens(addr) as tokens:
            if token in tokens:
                del tokens[token]
    def get_tokens_for_addr(self, addr):
        mdict = self.get_metadata_dict(addr).read()
        tokens = mdict.get(self.DEVICETOKEN_KEY, {})
        now = int(time.time())
        if isinstance(tokens, dict):
            token_list = [
                token
                for token, timestamp in tokens.items()
                if _is_valid_token_timestamp(timestamp, now)
            ]
            if len(token_list) < len(tokens):
                # Some tokens have expired, remove them.
                with self._modify_tokens(addr) as _tokens:
                    pass
        else:
            token_list = []
        return token_list
 class MetadataDictProxy(DictProxy):
    def __init__(self, notifier, metadata, iroh_relay=None, turn_hostname=None):
        super().__init__()
        self.notifier = notifier
        self.metadata = metadata
        self.iroh_relay = iroh_relay
        self.turn_hostname = turn_hostname
    def handle_lookup(self, parts):
        # Lpriv/43f5f508a7ea0366dff30200c15250e3/devicetoken\tlkj123poi@c2.testrun.org
        keyparts = parts[0].split("/", 2)
        if keyparts[0] == "priv":
            keyname = keyparts[2]
            addr = parts[1]
            if keyname == self.metadata.DEVICETOKEN_KEY:
                res = " ".join(self.metadata.get_tokens_for_addr(addr))
                return f"O{res}\n"
        elif keyparts[0] == "shared":
            keyname = keyparts[2]
            if (
                keyname == "vendor/vendor.dovecot/pvt/server/vendor/deltachat/irohrelay"
                and self.iroh_relay
            ):
                # Handle `GETMETADATA "" /shared/vendor/deltachat/irohrelay`
                return f"O{self.iroh_relay}\n"
            elif keyname == "vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn":
                res = turn_credentials()
                port = 3478
                return f"O{self.turn_hostname}:{port}:{res}\n"
        logging.warning(f"lookup ignored: {parts!r}")
        return "N\n"
    def handle_set(self, addr, parts):
        # For documentation on key structure see
        # https://github.com/dovecot/core/blob/main/src/lib-storage/mailbox-attribute.h
        keyname = parts[1].split("/")
        value = parts[2] if len(parts) > 2 else ""
        if keyname[0] == "priv" and keyname[2] == self.metadata.DEVICETOKEN_KEY:
            self.metadata.add_token_to_addr(addr, value)
            return True
        elif keyname[0] == "priv" and keyname[2] == "messagenew":
            self.notifier.new_message_for_addr(addr, self.metadata)
            return True
        return False
 def main():
    socket, config_path = sys.argv[1:]
    config = read_config(config_path)
    iroh_relay = config.iroh_relay
    mail_domain = config.mail_domain
    vmail_dir = config.mailboxes_dir
    if not vmail_dir.exists():
        logging.error("vmail dir does not exist: %r", vmail_dir)
        return 1
    queue_dir = vmail_dir / "pending_notifications"
    queue_dir.mkdir(exist_ok=True)
    metadata = Metadata(vmail_dir)
    notifier = Notifier(queue_dir)
    notifier.start_notification_threads(metadata.remove_token_from_addr)
    dictproxy = MetadataDictProxy(
        notifier=notifier,
        metadata=metadata,
        iroh_relay=iroh_relay,
        turn_hostname=mail_domain,
    )
    dictproxy.serve_forever_from_socket(socket)
--- a/chatmaild/src/chatmaild/metrics.py
+++ b/chatmaild/src/chatmaild/metrics.py
@@ -1,32 +0,0 @@
 #!/usr/bin/env python3
 import sys
 from pathlib import Path
 def main(vmail_dir=None):
    if vmail_dir is None:
        vmail_dir = sys.argv[1]
    accounts = 0
    ci_accounts = 0
    for path in Path(vmail_dir).iterdir():
        if not path.joinpath("cur").is_dir():
            continue
        accounts += 1
        if path.name[:3] in ("ci-", "ac_"):
            ci_accounts += 1
    print("# HELP total number of accounts")
    print("# TYPE accounts gauge")
    print(f"accounts {accounts}")
    print("# HELP number of CI accounts")
    print("# TYPE ci_accounts gauge")
    print(f"ci_accounts {ci_accounts}")
    print("# HELP number of non-CI accounts")
    print("# TYPE nonci_accounts gauge")
    print(f"nonci_accounts {accounts - ci_accounts}")
 if __name__ == "__main__":
    main()
--- a/chatmaild/src/chatmaild/migrate_db.py
+++ b/chatmaild/src/chatmaild/migrate_db.py
@@ -1,63 +0,0 @@
 """
 migration code from old sqlite databases into per-maildir "password" files
 where mtime reflects and is updated to be the "last-login" time.
 """
 import logging
 import os
 import sqlite3
 import sys
 from chatmaild.config import read_config
 def get_all_rows(path):
    assert path.exists()
    uri = f"file:{path}?mode=ro"
    sqlconn = sqlite3.connect(uri, timeout=60, isolation_level="DEFERRED", uri=True)
    cur = sqlconn.cursor()
    cur.execute("SELECT * from users")
    rows = cur.fetchall()
    sqlconn.close()
    return rows
 def migrate_from_db_to_maildir(config, chunking=10000):
    path = config.passdb_path
    if not path.exists():
        return
    all_rows = get_all_rows(path)
    # don't transfer special/CI accounts
    rows = [row for row in all_rows if row[0][:3] not in ("ci-", "ac_")]
    logging.info(f"ignoring {len(all_rows) - len(rows)} CI accounts")
    logging.info(f"migrating {len(rows)} sqlite database passwords to user dirs")
    for i, row in enumerate(rows):
        addr = row[0]
        enc_password = row[1]
        user = config.get_user(addr)
        user.set_password(enc_password)
        if len(row) == 3 and row[2]:
            timestamp = int(row[2])
            user.set_last_login_timestamp(timestamp)
        if i > 0 and i % chunking == 0:
            logging.info(f"migration-progress: {i} passwords transferred")
    logging.info("migration: all passwords migrated")
    oldpath = config.passdb_path.with_suffix(config.passdb_path.suffix + ".old")
    os.rename(config.passdb_path, oldpath)
    for path in config.passdb_path.parent.iterdir():
        if path.name.startswith(config.passdb_path.name + "-"):
            path.unlink()
    logging.info(f"migration: moved database to {oldpath!r}")
 if __name__ == "__main__":
    config = read_config(sys.argv[1])
    logging.basicConfig(level=logging.INFO)
    migrate_from_db_to_maildir(config)
--- a/chatmaild/src/chatmaild/newemail.py
+++ b/chatmaild/src/chatmaild/newemail.py
@@ -1,36 +0,0 @@
 #!/usr/local/lib/chatmaild/venv/bin/python3
 """CGI script for creating new accounts."""
 import json
 import random
 import secrets
 import string
 from chatmaild.config import Config, read_config
 CONFIG_PATH = "/usr/local/lib/chatmaild/chatmail.ini"
 ALPHANUMERIC = string.ascii_lowercase + string.digits
 ALPHANUMERIC_PUNCT = string.ascii_letters + string.digits + string.punctuation
 def create_newemail_dict(config: Config):
    user = "".join(random.choices(ALPHANUMERIC, k=config.username_max_length))
    password = "".join(
        secrets.choice(ALPHANUMERIC_PUNCT)
        for _ in range(config.password_min_length + 3)
    )
    return dict(email=f"{user}@{config.mail_domain}", password=f"{password}")
 def print_new_account():
    config = read_config(CONFIG_PATH)
    creds = create_newemail_dict(config)
    print("Content-Type: application/json")
    print("")
    print(json.dumps(creds))
 if __name__ == "__main__":
    print_new_account()
--- a/chatmaild/src/chatmaild/notifier.py
+++ b/chatmaild/src/chatmaild/notifier.py
@@ -1,171 +0,0 @@
 """
 This modules provides notification machinery for transmitting device tokens to
 a central notification server which in turn contacts a phone provider's notification server
 to trigger Delta Chat apps to retrieve messages and provide instant notifications to users.
 The Notifier class arranges the queuing of tokens in separate PriorityQueues
 from which NotifyThreads take and transmit them via HTTPS
 to the `notifications.delta.chat` service.
 The current lack of proper HTTP/2-support in Python leads us
 to use multiple threads and connections to the Rust-implemented `notifications.delta.chat`
 which itself uses HTTP/2 and thus only a single connection to phone-notification providers.
 If a token fails to cause a successful notification
 it is moved to a retry-number specific PriorityQueue
 which handles all tokens that failed a particular number of times
 and which are scheduled for retry using exponential back-off timing.
 If a token notification would be scheduled more than DROP_DEADLINE seconds
 after its first attempt, it is dropped with a log error.
 Note that tokens are opaque to the notification machinery here
 and are encrypted foreclosing all ability to distinguish
 which device token ultimately goes to which phone-provider notification service,
 or to understand the relation of "device tokens" and chatmail addresses.
 The meaning and format of tokens is basically a matter of chatmail Core and
 the `notification.delta.chat` service.
 """
 import logging
 import math
 import os
 import time
 from dataclasses import dataclass
 from pathlib import Path
 from queue import PriorityQueue
 from threading import Thread
 from uuid import uuid4
 import requests
@dataclass
 class PersistentQueueItem:
    path: Path
    addr: str
    start_ts: int
    token: str
    def delete(self):
        self.path.unlink(missing_ok=True)
    @classmethod
    def create(cls, queue_dir, addr, start_ts, token):
        queue_id = uuid4().hex
        start_ts = int(start_ts)
        path = queue_dir.joinpath(queue_id)
        tmp_path = path.with_name(path.name + ".tmp")
        tmp_path.write_text(f"{addr}\n{start_ts}\n{token}")
        os.rename(tmp_path, path)
        return cls(path, addr, start_ts, token)
    @classmethod
    def read_from_path(cls, path):
        addr, start_ts, token = path.read_text().split("\n", maxsplit=2)
        return cls(path, addr, int(start_ts), token)
    def __lt__(self, other):
        return self.start_ts < other.start_ts
 class Notifier:
    URL = "https://notifications.delta.chat/notify"
    CONNECTION_TIMEOUT = 60.0  # seconds until http-request is given up
    BASE_DELAY = 8.0  # base seconds for exponential back-off delay
    DROP_DEADLINE = 5 * 60 * 60  #  drop notifications after 5 hours
    def __init__(self, queue_dir):
        self.queue_dir = queue_dir
        max_tries = int(math.log(self.DROP_DEADLINE, self.BASE_DELAY)) + 1
        self.retry_queues = [PriorityQueue() for _ in range(max_tries)]
    def compute_delay(self, retry_num):
        return 0 if retry_num == 0 else pow(self.BASE_DELAY, retry_num)
    def new_message_for_addr(self, addr, metadata):
        start_ts = int(time.time())
        for token in metadata.get_tokens_for_addr(addr):
            queue_item = PersistentQueueItem.create(
                self.queue_dir, addr, start_ts, token
            )
            self.queue_for_retry(queue_item)
    def requeue_persistent_queue_items(self):
        for queue_path in self.queue_dir.iterdir():
            if queue_path.name.endswith(".tmp"):
                logging.warning(f"removing spurious queue item: {queue_path!r}")
                queue_path.unlink()
                continue
            try:
                queue_item = PersistentQueueItem.read_from_path(queue_path)
            except ValueError:
                logging.warning(f"removing spurious queue item: {queue_path!r}")
                queue_path.unlink()
                continue
            self.queue_for_retry(queue_item)
    def queue_for_retry(self, queue_item, retry_num=0):
        delay = self.compute_delay(retry_num)
        when = int(time.time()) + delay
        deadline = queue_item.start_ts + self.DROP_DEADLINE
        if retry_num >= len(self.retry_queues) or when > deadline:
            queue_item.delete()
            logging.error(f"notification exceeded deadline: {queue_item.token!r}")
            return
        self.retry_queues[retry_num].put((when, queue_item))
    def start_notification_threads(self, remove_token_from_addr):
        self.requeue_persistent_queue_items()
        threads = {}
        for retry_num in range(len(self.retry_queues)):
            # use 4 threads for first-try tokens and less for subsequent tries
            num_threads = 4 if retry_num == 0 else 2
            threads[retry_num] = []
            for _ in range(num_threads):
                thread = NotifyThread(self, retry_num, remove_token_from_addr)
                threads[retry_num].append(thread)
                thread.start()
        return threads
 class NotifyThread(Thread):
    def __init__(self, notifier, retry_num, remove_token_from_addr):
        super().__init__(daemon=True)
        self.notifier = notifier
        self.retry_num = retry_num
        self.remove_token_from_addr = remove_token_from_addr
    def stop(self):
        self.notifier.retry_queues[self.retry_num].put((None, None))
    def run(self):
        requests_session = requests.Session()
        while self.retry_one(requests_session):
            pass
    def retry_one(self, requests_session, sleep=time.sleep):
        when, queue_item = self.notifier.retry_queues[self.retry_num].get()
        if when is None:
            return False
        wait_time = when - int(time.time())
        if wait_time > 0:
            sleep(wait_time)
        self.perform_request_to_notification_server(requests_session, queue_item)
        return True
    def perform_request_to_notification_server(self, requests_session, queue_item):
        timeout = self.notifier.CONNECTION_TIMEOUT
        token = queue_item.token
        try:
            res = requests_session.post(self.notifier.URL, data=token, timeout=timeout)
        except requests.exceptions.RequestException as e:
            res = e
        else:
            if res.status_code in (200, 410):
                if res.status_code == 410:
                    self.remove_token_from_addr(queue_item.addr, token)
                queue_item.delete()
                return
        logging.warning(f"Notification request failed: {res!r}")
        self.notifier.queue_for_retry(queue_item, retry_num=self.retry_num + 1)
--- a/chatmaild/src/chatmaild/tests/mail-data/asm.eml
+++ b/chatmaild/src/chatmaild/tests/mail-data/asm.eml
@@ -1,56 +0,0 @@
 From: {from_addr}
 To: {to_addr}
 Autocrypt-Setup-Message: v1
 Subject: Autocrypt Setup Message
 Date: Tue, 22 Jan 2019 12:56:29 +0100
 Content-type: multipart/mixed; boundary="Y6fyGi9SoGeH8WwRaEdC6bbBcYOedDzrQ"
 --Y6fyGi9SoGeH8WwRaEdC6bbBcYOedDzrQ
 Content-Type: text/plain
 This message contains all information to transfer your Autocrypt
 settings along with your secret key securely from your original
 device.
 To set up your new device for Autocrypt, please follow the
 instuctions that should be presented by your new device.
 You can keep this message and use it as a backup for your secret
 key. If you want to do this, you should write down the Setup Code
 and store it securely.
 --Y6fyGi9SoGeH8WwRaEdC6bbBcYOedDzrQ
 Content-Type: application/autocrypt-setup
 Content-Disposition: attachment; filename="autocrypt-setup-message.html"
 <html><body>
 <p>
 This is the Autocrypt setup file used to transfer settings and
 keys between clients. You can decrypt it using the Setup Code
 presented on your old device, and then import the contained key
 into your keyring.
 </p>
 <pre>
 -----BEGIN PGP MESSAGE-----
 Passphrase-Format: numeric9x4
 Passphrase-Begin: 17
 jA0EBwMCFAxADoCdzeX/0ukBlqI5+pfpKb751qd/7nLNbkpy3gVcaf1QwRPZYt40
 Ynp08UqRQ2g48ZlnzHLSwlTGOPTuv2Jt8ka+pgZ45xzvJSG2gau03xP4VsC271kR
 VmCjdb0Y6Rk96mAwfGzrkbaRQ9Z7fIoL866GOv6h9neiVIkp+JYlTV6ISD0ZQJ4Q
 I6dOQkB/TWZyVjtiJDOQHdfNWliA6NtqaLq19wlu9L5xXjuNpY95KwR8EJXWe0+o
 Y3d2U/KxOAkXKghP2Qg1GtlPVeGC5T4p03TGI6pzKT+kHX6Rrm9wK6sM9aTquMmF
 Vok84Jg1DFnwivWC2RILR81rXi7k/+Y6MUbveFgJ9cQduqpxnmD7TjOblYu7M6zp
 YGAUxh8DRKlIMn2QsA++DBYQ6ACZvwuY8qTDLkqPDo4WqM313dsMJbyGjDdVE7EM
 PESS+RlABETpZXz8g/ycr6DIUNdlbPcmYlsBfHWDOuR2GFFTwmlv5slWS39dJv38
 E0eIe1CwdxI801Se7t7dUUS/ZF8wb6GlmxOcqGbF8eko1Z0S64IAm7/h13MRQCxI
 geQnHfGYVJ2FOimoCMEKwfa9x++RFTDW0u7spDC2uWvK/1viV8OfRppFhLr/kmKb
 18lWXuAz80DAjUDUsVqEq2MvJBJGoCJUEyjuRsLkHYRM5jYk4v50LyyR0Om73nWF
 nZBqmqNzdr7Xb9PHHdFhnEc0VvoYbrcM0RVYcEMW3YbmejM891j1d6Iv+/n/qND/
 NdebGrfWJMmFLf/iEkzTZ3/v5inW9LpWoRc94ioCjJTaEo8Rib6ARRFaJVIsmNXi
 YicFGO98D+zX+a2t9Yz6IpPajVslnOp6ScpmXgts/2XWD7oE+JgxSAqo/dLVsHgP
 Ufo=
 =pulM
 -----END PGP MESSAGE-----
 </pre></body></html>
 --Y6fyGi9SoGeH8WwRaEdC6bbBcYOedDzrQ--
--- a/chatmaild/src/chatmaild/tests/mail-data/literal.eml
+++ b/chatmaild/src/chatmaild/tests/mail-data/literal.eml
@@ -1,44 +0,0 @@
 From: {from_addr}
 To: {to_addr}
 Subject: ...
 Date: Sun, 15 Oct 2023 16:43:21 +0000
 Message-ID: <Mr.UVyJWZmkCKM.hGzNc6glBE_@c2.testrun.org>
 In-Reply-To: <Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>
 References: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>
 	<Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>
 Chat-Version: 1.0
 Autocrypt: addr={from_addr}; prefer-encrypt=mutual;
 	keydata=xjMEZSwWjhYJKwYBBAHaRw8BAQdAQBEhqeJh0GueHB6kF/DUQqYCxARNBVokg/AzT+7LqH
 	rNFzxiYXJiYXpAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUsFo4CGwMECwkIBwYVCAkKCwID
 	FgIBFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX9A4AEAnHWHp49eBCMHK5t66gYPiW
 	XQuB1mwUjzGfYWB+0RXUoA/0xcQ3FbUNlGKW7Blp6eMFfViv6Mv2d3kNSXACB6nmcMzjgEZSwWjhIK
 	KwYBBAGXVQEFAQEHQBpY5L2M1XHo0uxf8SX1wNLBp/OVvidoWHQF2Jz+kJsUAwEIB8J4BBgWCAAgBQ
 	JlLBaOAhsMFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX/INgEA37AJaNvruYsJVanP
 	IXnYw4CKd55UAwl8Zcy+M2diAbkA/0fHHcGV4r78hpbbL1Os52DPOdqYQRauIeJUeG+G6bQO
 MIME-Version: 1.0
 Content-Type: multipart/encrypted; protocol="application/pgp-encrypted";
 	boundary="YFrteb74qSXmggbOxZL9dRnhymywAi"
 --YFrteb74qSXmggbOxZL9dRnhymywAi
 Content-Description: PGP/MIME version identification
 Content-Type: application/pgp-encrypted
 Version: 1
 --YFrteb74qSXmggbOxZL9dRnhymywAi
 Content-Description: OpenPGP encrypted message
 Content-Disposition: inline; filename="encrypted.asc";
 Content-Type: application/octet-stream; name="encrypted.asc"
 -----BEGIN PGP MESSAGE-----
 yxJiAAAAAABIZWxsbyB3b3JsZCE=
 =1I/B
 -----END PGP MESSAGE-----
 --YFrteb74qSXmggbOxZL9dRnhymywAi--
--- a/chatmaild/src/chatmaild/tests/mail-data/mailer-daemon.eml
+++ b/chatmaild/src/chatmaild/tests/mail-data/mailer-daemon.eml
@@ -1,46 +0,0 @@
 Date: Fri, 8 Jul 1994 09:21:47 -0400
 From: Mail Delivery Subsystem <MAILER-DAEMON@example.org>
 Subject: Returned mail: User unknown
 To: <owner-ups-mib@CS.UTK.EDU>
 Auto-Submitted: auto-replied 
 MIME-Version: 1.0
 Content-Type: multipart/report; report-type=delivery-status;
      boundary="JAA13167.773673707/CS.UTK.EDU"
 --JAA13167.773673707/CS.UTK.EDU
 content-type: text/plain; charset=us-ascii
   ----- The following addresses had delivery problems -----
 <arathib@vnet.ibm.com>  (unrecoverable error)
 <wsnell@sdcc13.ucsd.edu>  (unrecoverable error)
 --JAA13167.773673707/CS.UTK.EDU
 content-type: message/delivery-status
 Reporting-MTA: dns; cs.utk.edu
 Original-Recipient: rfc822;arathib@vnet.ibm.com
 Final-Recipient: rfc822;arathib@vnet.ibm.com
 Action: failed
 Status: 5.0.0 (permanent failure)
 Diagnostic-Code: smtp;
 550 'arathib@vnet.IBM.COM' is not a registered gateway user
 Remote-MTA: dns; vnet.ibm.com
 Original-Recipient: rfc822;johnh@hpnjld.njd.hp.com
 Final-Recipient: rfc822;johnh@hpnjld.njd.hp.com
 Action: delayed
 Status: 4.0.0 (hpnjld.njd.jp.com: host name lookup failure)
 Original-Recipient: rfc822;wsnell@sdcc13.ucsd.edu
 Final-Recipient: rfc822;wsnell@sdcc13.ucsd.edu
 Action: failed
 Status: 5.0.0
 Diagnostic-Code: smtp; 550 user unknown
 Remote-MTA: dns; sdcc13.ucsd.edu
 --JAA13167.773673707/CS.UTK.EDU
 content-type: message/rfc822
 [original message goes here]
 --JAA13167.773673707/CS.UTK.EDU--
--- a/chatmaild/src/chatmaild/tests/mail-data/securejoin-vc-fake.eml
+++ b/chatmaild/src/chatmaild/tests/mail-data/securejoin-vc-fake.eml
@@ -1,21 +0,0 @@
 Subject: Message from {from_addr}
 From: <{from_addr}>
 To: <{to_addr}>
 Date: Sun, 15 Oct 2023 16:43:25 +0000
 Message-ID: <Mr.78MWtlV7RAi.goCFzBhCYfy@c2.testrun.org>
 Chat-Version: 1.0
 Secure-Join: vc-request
 Secure-Join-Invitenumber: RANDOM-TOKEN
 MIME-Version: 1.0
 Content-Type: multipart/mixed; boundary="Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi"
 --Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi
 Content-Type: text/plain; charset=utf-8
 Buy viagra!
 --Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi--
--- a/chatmaild/src/chatmaild/tests/mail-data/securejoin-vc.eml
+++ b/chatmaild/src/chatmaild/tests/mail-data/securejoin-vc.eml
@@ -1,21 +0,0 @@
 Subject: Message from {from_addr}
 From: <{from_addr}>
 To: <{to_addr}>
 Date: Sun, 15 Oct 2023 16:43:25 +0000
 Message-ID: <Mr.78MWtlV7RAi.goCFzBhCYfy@c2.testrun.org>
 Chat-Version: 1.0
 Secure-Join: vc-request
 Secure-Join-Invitenumber: RANDOM-TOKEN
 MIME-Version: 1.0
 Content-Type: multipart/mixed; boundary="Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi"
 --Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi
 Content-Type: text/plain; charset=utf-8
 Secure-Join: vc-request
 --Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi--
--- a/chatmaild/src/chatmaild/tests/plugin.py
+++ b/chatmaild/src/chatmaild/tests/plugin.py
@@ -1,97 +0,0 @@
 import importlib.resources
 import itertools
 import os
 import random
 from email import policy
 from email.parser import BytesParser
 from pathlib import Path
 import pytest
 from chatmaild.config import read_config, write_initial_config
@pytest.fixture
 def make_config(tmp_path):
    inipath = tmp_path.joinpath("chatmail.ini")
    def make_conf(mail_domain, settings=None):
        basedir = tmp_path.joinpath(f"vmail/{mail_domain}")
        basedir.mkdir(parents=True, exist_ok=True)
        overrides = settings.copy() if settings else {}
        overrides["mailboxes_dir"] = str(basedir)
        write_initial_config(inipath, mail_domain, overrides=overrides)
        return read_config(inipath)
    return make_conf
@pytest.fixture
 def example_config(make_config):
    return make_config("chat.example.org")
@pytest.fixture
 def maildomain(example_config):
    return example_config.mail_domain
@pytest.fixture
 def testaddr(maildomain):
    return f"user.name@{maildomain}"
@pytest.fixture
 def gencreds(maildomain):
    count = itertools.count()
    next(count)
    def gen(domain=None):
        domain = domain if domain else maildomain
        while 1:
            num = next(count)
            alphanumeric = "abcdefghijklmnopqrstuvwxyz1234567890"
            user = "".join(random.choices(alphanumeric, k=10))
            user = f"ac{num}_{user}"[:9]
            password = "".join(random.choices(alphanumeric, k=12))
            yield f"{user}@{domain}", f"{password}"
    return lambda domain=None: next(gen(domain))
@pytest.fixture
 def maildata(request):
    try:
        datadir = importlib.resources.files(__package__).joinpath("mail-data")
    except TypeError:
        # in python3.9 or lower, the above doesn't work, so we get datadir this way:
        datadir = Path(os.getcwd()).joinpath("chatmaild/src/chatmaild/tests/mail-data")
    assert datadir.exists(), datadir
    def maildata(name, from_addr, to_addr, subject="[...]"):
        # Using `.read_bytes().decode()` instead of `.read_text()` to preserve newlines.
        data = datadir.joinpath(name).read_bytes().decode()
        text = data.format(from_addr=from_addr, to_addr=to_addr, subject=subject)
        return BytesParser(policy=policy.SMTP).parsebytes(text.encode())
    return maildata
@pytest.fixture
 def mockout():
    class MockOut:
        captured_red = []
        captured_green = []
        captured_plain = []
        def red(self, msg):
            self.captured_red.append(msg)
        def green(self, msg):
            self.captured_green.append(msg)
        def __call__(self, msg):
            self.captured_plain.append(msg)
    return MockOut()
--- a/chatmaild/src/chatmaild/tests/test_config.py
+++ b/chatmaild/src/chatmaild/tests/test_config.py
@@ -1,75 +0,0 @@
 import pytest
 from chatmaild.config import read_config
 def test_read_config_basic(example_config):
    assert example_config.mail_domain == "chat.example.org"
    assert not example_config.privacy_supervisor and not example_config.privacy_mail
    assert not example_config.privacy_pdo and not example_config.privacy_postal
    inipath = example_config._inipath
    inipath.write_text(inipath.read_text().replace("60", "37"))
    example_config = read_config(inipath)
    assert example_config.max_user_send_per_minute == 37
    assert example_config.mail_domain == "chat.example.org"
 def test_read_config_basic_using_defaults(tmp_path, maildomain):
    inipath = tmp_path.joinpath("chatmail.ini")
    inipath.write_text(f"[params]\nmail_domain = {maildomain}")
    example_config = read_config(inipath)
    assert example_config.max_user_send_per_minute == 60
    assert example_config.filtermail_smtp_port_incoming == 10081
 def test_read_config_testrun(make_config):
    config = make_config("something.testrun.org")
    assert config.mail_domain == "something.testrun.org"
    assert len(config.privacy_postal.split("\n")) > 1
    assert len(config.privacy_supervisor.split("\n")) > 1
    assert len(config.privacy_pdo.split("\n")) > 1
    assert config.privacy_mail == "privacy@testrun.org"
    assert config.filtermail_smtp_port == 10080
    assert config.postfix_reinject_port == 10025
    assert config.max_user_send_per_minute == 60
    assert config.max_mailbox_size == "500M"
    assert config.delete_mails_after == "20"
    assert config.delete_large_after == "7"
    assert config.username_min_length == 9
    assert config.username_max_length == 9
    assert config.password_min_length == 9
    assert "privacy@testrun.org" in config.passthrough_recipients
    assert config.passthrough_senders == []
 def test_config_userstate_paths(make_config, tmp_path):
    config = make_config("something.testrun.org")
    mailboxes_dir = config.mailboxes_dir
    passdb_path = config.passdb_path
    assert mailboxes_dir.name == "something.testrun.org"
    assert str(passdb_path) == "/home/vmail/passdb.sqlite"
    assert config.mail_domain == "something.testrun.org"
    path = config.get_user("user1@something.testrun.org").maildir
    assert not path.exists()
    assert path == mailboxes_dir.joinpath("user1@something.testrun.org")
    with pytest.raises(ValueError):
        config.get_user("")
    with pytest.raises(ValueError):
        config.get_user(None)
    with pytest.raises(ValueError):
        config.get_user("../some@something.testrun.org").maildir
    with pytest.raises(ValueError):
        config.get_user("..").maildir
    with pytest.raises(ValueError):
        config.get_user(".")
 def test_config_max_message_size(make_config, tmp_path):
    config = make_config("something.testrun.org", dict(max_message_size="10000"))
    assert config.max_message_size == 10000
--- a/chatmaild/src/chatmaild/tests/test_delete_inactive_users.py
+++ b/chatmaild/src/chatmaild/tests/test_delete_inactive_users.py
@@ -1,64 +0,0 @@
 import time
 from chatmaild.doveauth import AuthDictProxy
 from chatmaild.expire import main as main_expire
 def test_login_timestamps(example_config):
    testaddr = "someuser@chat.example.org"
    user = example_config.get_user(testaddr)
    # password file needs to be set because it's mtime tracks last-login time
    user.set_password("1l2k3j1l2k3j123")
    for i in range(10):
        user.set_last_login_timestamp(86400 * 4 + i)
        assert user.get_last_login_timestamp() == 86400 * 4
 def test_delete_inactive_users(example_config):
    new = time.time()
    old = new - (example_config.delete_inactive_users_after * 86400) - 1
    dictproxy = AuthDictProxy(example_config)
    def create_user(addr, last_login):
        dictproxy.lookup_passdb(addr, "q9mr3faue")
        user = example_config.get_user(addr)
        user.maildir.joinpath("cur").mkdir()
        user.maildir.joinpath("cur", "something").mkdir()
        user.set_last_login_timestamp(timestamp=last_login)
    # create some stale and some new accounts
    to_remove = []
    for i in range(150):
        addr = f"oldold{i:03}@chat.example.org"
        create_user(addr, last_login=old)
        to_remove.append(addr)
    remain = []
    for i in range(5):
        addr = f"newnew{i:03}@chat.example.org"
        create_user(addr, last_login=new)
        remain.append(addr)
    # check pre and post-conditions for delete_inactive_users()
    for addr in to_remove:
        assert example_config.get_user(addr).maildir.exists()
    main_expire(
        args=[
            "--remove",
            str(example_config._inipath),
        ]
    )
    for p in example_config.mailboxes_dir.iterdir():
        assert not p.name.startswith("old")
    for addr in to_remove:
        assert not example_config.get_user(addr).maildir.exists()
    for addr in remain:
        userdir = example_config.get_user(addr).maildir
        assert userdir.exists()
        assert userdir.joinpath("password").read_text()
--- a/chatmaild/src/chatmaild/tests/test_doveauth.py
+++ b/chatmaild/src/chatmaild/tests/test_doveauth.py
@@ -1,150 +0,0 @@
 import io
 import json
 import queue
 import threading
 import traceback
 import pytest
 import chatmaild.doveauth
 from chatmaild.doveauth import (
    AuthDictProxy,
    is_allowed_to_create,
 )
 from chatmaild.newemail import create_newemail_dict
@pytest.fixture
 def dictproxy(example_config):
    return AuthDictProxy(config=example_config)
 def test_basic(dictproxy, gencreds):
    addr, password = gencreds()
    dictproxy.lookup_passdb(addr, password)
    data = dictproxy.lookup_userdb(addr)
    assert data
    data2 = dictproxy.lookup_passdb(addr, password)
    assert data == data2
 def test_iterate_addresses(dictproxy):
    addresses = []
    for i in range(10):
        addresses.append(f"asdf1234{i}@chat.example.org")
        dictproxy.lookup_passdb(addresses[-1], "q9mr3faue")
    res = dictproxy.iter_userdb()
    assert set(res) == set(addresses)
 def test_invalid_username_length(example_config):
    config = example_config
    config.username_min_length = 6
    config.username_max_length = 10
    password = create_newemail_dict(config)["password"]
    assert not is_allowed_to_create(config, f"a1234@{config.mail_domain}", password)
    assert is_allowed_to_create(config, f"012345@{config.mail_domain}", password)
    assert is_allowed_to_create(config, f"0123456@{config.mail_domain}", password)
    assert is_allowed_to_create(config, f"0123456789@{config.mail_domain}", password)
    assert not is_allowed_to_create(
        config, f"0123456789x@{config.mail_domain}", password
    )
 def test_dont_overwrite_password_on_wrong_login(dictproxy):
    """Test that logging in with a different password doesn't create a new user"""
    res = dictproxy.lookup_passdb(
        "newuser12@chat.example.org", "kajdlkajsldk12l3kj1983"
    )
    assert res["password"]
    res2 = dictproxy.lookup_passdb("newuser12@chat.example.org", "kajdslqwe")
    # this function always returns a password hash, which is actually compared by dovecot.
    assert res["password"] == res2["password"]
 def test_nocreate_file(monkeypatch, tmpdir, dictproxy):
    p = tmpdir.join("nocreate")
    p.write("")
    monkeypatch.setattr(chatmaild.doveauth, "NOCREATE_FILE", str(p))
    dictproxy.lookup_passdb("newuser12@chat.example.org", "zequ0Aimuchoodaechik")
    assert not dictproxy.lookup_userdb("newuser12@chat.example.org")
 def test_handle_dovecot_request(dictproxy):
    transactions = {}
    # Test that password can contain ", ', \ and /
    msg = (
        'Lshared/passdb/laksjdlaksjdlak\\\\sjdlk\\"12j\\\'3l1/k2j3123"'
        "some42123@chat.example.org\tsome42123@chat.example.org"
    )
    res = dictproxy.handle_dovecot_request(msg, transactions)
    assert res
    assert res[0] == "O" and res.endswith("\n")
    userdata = json.loads(res[1:].strip())
    assert userdata["home"].endswith("chat.example.org/some42123@chat.example.org")
    assert userdata["uid"] == userdata["gid"] == "vmail"
    assert userdata["password"].startswith("{SHA512-CRYPT}")
 def test_handle_dovecot_protocol_hello_is_skipped(example_config, caplog):
    dictproxy = AuthDictProxy(config=example_config)
    rfile = io.BytesIO(b"H3\t2\t0\t\tauth\n")
    wfile = io.BytesIO()
    dictproxy.loop_forever(rfile, wfile)
    assert wfile.getvalue() == b""
    assert not caplog.messages
 def test_handle_dovecot_protocol_user_not_exists(example_config):
    dictproxy = AuthDictProxy(config=example_config)
    rfile = io.BytesIO(
        b"H3\t2\t0\t\tauth\nLshared/userdb/foobar@chat.example.org\tfoobar@chat.example.org\n"
    )
    wfile = io.BytesIO()
    dictproxy.loop_forever(rfile, wfile)
    assert wfile.getvalue() == b"N\n"
 def test_handle_dovecot_protocol_iterate(gencreds, example_config):
    dictproxy = AuthDictProxy(config=example_config)
    dictproxy.lookup_passdb("asdf00000@chat.example.org", "q9mr3faue")
    dictproxy.lookup_passdb("asdf11111@chat.example.org", "q9mr3faue")
    rfile = io.BytesIO(b"H3\t2\t0\t\tauth\nI0\t0\tshared/userdb/")
    wfile = io.BytesIO()
    dictproxy.loop_forever(rfile, wfile)
    lines = wfile.getvalue().decode("ascii").split("\n")
    assert "Oshared/userdb/asdf00000@chat.example.org\t" in lines
    assert "Oshared/userdb/asdf11111@chat.example.org\t" in lines
    assert not lines[2]
 def test_50_concurrent_lookups_different_accounts(gencreds, dictproxy):
    num_threads = 50
    req_per_thread = 5
    results = queue.Queue()
    def lookup():
        for i in range(req_per_thread):
            addr, password = gencreds()
            try:
                dictproxy.lookup_passdb(addr, password)
            except Exception:
                results.put(traceback.format_exc())
            else:
                results.put(None)
    threads = []
    for i in range(num_threads):
        thread = threading.Thread(target=lookup, daemon=True)
        threads.append(thread)
    print(f"created {num_threads} threads, starting them and waiting for results")
    for thread in threads:
        thread.start()
    for i in range(num_threads * req_per_thread):
        res = results.get()
        if res is not None:
            pytest.fail(f"concurrent lookup failed\n{res}")
--- a/chatmaild/src/chatmaild/tests/test_expire.py
+++ b/chatmaild/src/chatmaild/tests/test_expire.py
@@ -1,161 +0,0 @@
 import os
 import random
 from datetime import datetime
 from fnmatch import fnmatch
 from pathlib import Path
 import pytest
 from chatmaild.expire import (
    FileEntry,
    MailboxStat,
    get_file_entry,
    iter_mailboxes,
    os_listdir_if_exists,
 )
 from chatmaild.expire import main as expiry_main
 from chatmaild.fsreport import main as report_main
 def fill_mbox(folderdir):
    password = folderdir.joinpath("password")
    password.write_text("xxx")
    folderdir.joinpath("maildirsize").write_text("xxx")
    garbagedir = folderdir.joinpath("garbagedir")
    garbagedir.mkdir()
    garbagedir.joinpath("bimbum").write_text("hello")
    create_new_messages(folderdir, ["cur/msg1"], size=500)
    create_new_messages(folderdir, ["new/msg2"], size=600)
 def create_new_messages(basedir, relpaths, size=1000, days=0):
    now = datetime.utcnow().timestamp()
    for relpath in relpaths:
        msg_path = Path(basedir).joinpath(relpath)
        msg_path.parent.mkdir(parents=True, exist_ok=True)
        msg_path.write_text("x" * size)
        # accessed now, modified N days ago
        os.utime(msg_path, (now, now - days * 86400))
@pytest.fixture
 def mbox1(example_config):
    mboxdir = example_config.mailboxes_dir.joinpath("mailbox1@example.org")
    mboxdir.mkdir()
    fill_mbox(mboxdir)
    return MailboxStat(mboxdir)
 def test_deltachat_folder(example_config):
    """Test old setups that might have a .DeltaChat folder where messages also need to get removed."""
    mboxdir = example_config.mailboxes_dir.joinpath("mailbox1@example.org")
    mboxdir.mkdir()
    mbox2dir = mboxdir.joinpath(".DeltaChat")
    mbox2dir.mkdir()
    fill_mbox(mbox2dir)
    mb = MailboxStat(mboxdir)
    assert len(mb.messages) == 2
 def test_filentry_ordering(tmp_path):
    l = [FileEntry(f"x{i}", size=i + 10, mtime=1000 - i) for i in range(10)]
    sorted = list(l)
    random.shuffle(l)
    l.sort(key=lambda x: x.size)
    assert l == sorted
 def test_no_mailbxoes(tmp_path, capsys):
    assert [] == list(iter_mailboxes(str(tmp_path.joinpath("notexists")), maxnum=10))
    out, err = capsys.readouterr()
    assert "no mailboxes" in err
 def test_stats_mailbox(mbox1):
    password = Path(mbox1.basedir).joinpath("password")
    assert mbox1.last_login == password.stat().st_mtime
    assert len(mbox1.messages) == 2
    msgs = list(sorted(mbox1.messages, key=lambda x: x.size))
    assert len(msgs) == 2
    assert msgs[0].size == 500  # cur
    assert msgs[1].size == 600  # new
    create_new_messages(mbox1.basedir, ["large-extra"], size=1000)
    create_new_messages(mbox1.basedir, ["index-something"], size=3)
    mbox2 = MailboxStat(mbox1.basedir)
    assert len(mbox2.extrafiles) == 5
    assert mbox2.extrafiles[0].size == 1000
    # cope well with mailbox dirs that have no password (for whatever reason)
    Path(mbox1.basedir).joinpath("password").unlink()
    mbox3 = MailboxStat(mbox1.basedir)
    assert mbox3.last_login is None
 def test_report_no_mailboxes(example_config):
    args = (str(example_config._inipath),)
    report_main(args)
 def test_report(mbox1, example_config):
    args = (str(example_config._inipath),)
    report_main(args)
    args = list(args) + "--days 1".split()
    report_main(args)
    args = list(args) + "--min-login-age 1".split()
    report_main(args)
    args = list(args) + "--mdir cur".split()
    report_main(args)
 def test_expiry_cli_basic(example_config, mbox1):
    args = (str(example_config._inipath),)
    expiry_main(args)
 def test_expiry_cli_old_files(capsys, example_config, mbox1):
    relpaths_old = ["cur/msg_old1", "cur/msg_old1"]
    cutoff_days = int(example_config.delete_mails_after) + 1
    create_new_messages(mbox1.basedir, relpaths_old, size=1000, days=cutoff_days)
    relpaths_large = ["cur/msg_old_large1", "new/msg_old_large2"]
    cutoff_days = int(example_config.delete_large_after) + 1
    create_new_messages(
        mbox1.basedir, relpaths_large, size=1000 * 300, days=cutoff_days
    )
    create_new_messages(mbox1.basedir, ["cur/shouldstay"], size=1000 * 300, days=1)
    args = str(example_config._inipath), "--remove", "-v"
    expiry_main(args)
    out, err = capsys.readouterr()
    allpaths = relpaths_old + relpaths_large + ["maildirsize"]
    for path in allpaths:
        for line in err.split("\n"):
            if fnmatch(line, f"removing*{path}"):
                break
        else:
            if path != "new/msg_old_large2":
                pytest.fail(f"failed to remove {path}\n{err}")
    assert "shouldstay" not in err
 def test_get_file_entry(tmp_path):
    assert get_file_entry(str(tmp_path.joinpath("123123"))) is None
    p = tmp_path.joinpath("x")
    p.write_text("hello")
    entry = get_file_entry(str(p))
    assert entry.size == 5
    assert entry.mtime
 def test_os_listdir_if_exists(tmp_path):
    tmp_path.joinpath("x").write_text("hello")
    assert len(os_listdir_if_exists(str(tmp_path))) == 1
    assert len(os_listdir_if_exists(str(tmp_path.joinpath("123123")))) == 0
--- a/chatmaild/src/chatmaild/tests/test_filedict.py
+++ b/chatmaild/src/chatmaild/tests/test_filedict.py
@@ -1,39 +0,0 @@
 import threading
 from chatmaild.filedict import FileDict, write_bytes_atomic
 def test_basic(tmp_path):
    fdict = FileDict(tmp_path.joinpath("metadata"))
    assert fdict.read() == {}
    with fdict.modify() as d:
        d["devicetoken"] = [1, 2, 3]
        d["456"] = 4.2
    new = fdict.read()
    assert new["devicetoken"] == [1, 2, 3]
    assert new["456"] == 4.2
 def test_bad_marshal_file(tmp_path, caplog):
    fdict1 = FileDict(tmp_path.joinpath("metadata"))
    fdict1.path.write_bytes(b"l12k3l12k3l")
    assert fdict1.read() == {}
    assert "corrupt" in caplog.records[0].msg
 def test_write_bytes_atomic_concurrent(tmp_path):
    p = tmp_path.joinpath("somefile.ext")
    write_bytes_atomic(p, b"hello")
    threads = []
    for i in range(30):
        content = f"hello{i}".encode("ascii")
        t = threading.Thread(target=lambda: write_bytes_atomic(p, content))
        t.start()
        threads.append(t)
    for t in threads:
        t.join()
    assert p.read_text().strip() != "hello"
    assert len(list(p.parent.iterdir())) == 1
--- a/chatmaild/src/chatmaild/tests/test_filtermail_blackbox.py
+++ b/chatmaild/src/chatmaild/tests/test_filtermail_blackbox.py
@@ -1,87 +0,0 @@
 import smtplib
 import subprocess
 import sys
 import pytest
@pytest.fixture
 def smtpserver():
    from pytest_localserver import smtp
    server = smtp.Server("127.0.0.1")
    server.start()
    yield server
    server.stop()
@pytest.fixture
 def make_popen(request):
    def popen(cmdargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kw):
        p = subprocess.Popen(
            cmdargs,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
        )
        def fin():
            p.terminate()
            out, err = p.communicate()
            print(out.decode("ascii"))
            print(err.decode("ascii"), file=sys.stderr)
        request.addfinalizer(fin)
        return p
    return popen
@pytest.mark.parametrize("filtermail_mode", ["outgoing", "incoming"])
 def test_one_mail(
    make_config, make_popen, smtpserver, maildata, filtermail_mode, monkeypatch
 ):
    monkeypatch.setenv("PYTHONUNBUFFERED", "1")
    smtp_inject_port = 20025
    if filtermail_mode == "outgoing":
        settings = dict(
            postfix_reinject_port=smtpserver.port,
            filtermail_smtp_port=smtp_inject_port,
        )
    else:
        settings = dict(
            postfix_reinject_port_incoming=smtpserver.port,
            filtermail_smtp_port_incoming=smtp_inject_port,
        )
    config = make_config("example.org", settings=settings)
    path = str(config._inipath)
    popen = make_popen(["filtermail", path, filtermail_mode])
    # Wait for filtermail to start accepting connections
    import socket
    import time
    for _ in range(50):  # 5 second timeout
        try:
            sock = socket.create_connection(("127.0.0.1", smtp_inject_port), timeout=0.1)
            sock.close()
            break
        except (ConnectionRefusedError, OSError):
            time.sleep(0.1)
    else:
        pytest.fail("filtermail failed to start accepting connections")
    addr = f"user1@{config.mail_domain}"
    config.get_user(addr).set_password("l1k2j3l1k2j3l")
    # send encrypted mail
    data = str(maildata("encrypted.eml", from_addr=addr, to_addr=addr))
    client = smtplib.SMTP("localhost", smtp_inject_port)
    client.sendmail(addr, [addr], data)
    assert len(smtpserver.outbox) == 1
    # send un-encrypted mail that errors
    data = str(maildata("fake-encrypted.eml", from_addr=addr, to_addr=addr))
    with pytest.raises(smtplib.SMTPDataError) as e:
        client.sendmail(addr, [addr], data)
    assert e.value.smtp_code == 523
--- a/chatmaild/src/chatmaild/tests/test_lastlogin.py
+++ b/chatmaild/src/chatmaild/tests/test_lastlogin.py
@@ -1,38 +0,0 @@
 import time
 from chatmaild.doveauth import AuthDictProxy
 from chatmaild.lastlogin import (
    LastLoginDictProxy,
 )
 def test_handle_dovecot_request_last_login(testaddr, example_config):
    dictproxy = LastLoginDictProxy(config=example_config)
    authproxy = AuthDictProxy(config=example_config)
    authproxy.lookup_passdb(testaddr, "1l2k3j1l2k3jl123")
    dictproxy_transactions = {}
    # Begin transaction
    tx = "1111"
    msg = f"B{tx}\t{testaddr}"
    res = dictproxy.handle_dovecot_request(msg, dictproxy_transactions)
    assert not res
    assert dictproxy_transactions == {tx: dict(addr=testaddr, res="O\n")}
    # set last-login info for user
    user = dictproxy.config.get_user(testaddr)
    timestamp = int(time.time())
    msg = f"S{tx}\tshared/last-login/{testaddr}\t{timestamp}"
    res = dictproxy.handle_dovecot_request(msg, dictproxy_transactions)
    assert not res
    assert len(dictproxy_transactions) == 1
    read_timestamp = user.get_last_login_timestamp()
    assert read_timestamp == timestamp // 86400 * 86400
    # finish transaction
    msg = f"C{tx}"
    res = dictproxy.handle_dovecot_request(msg, dictproxy_transactions)
    assert res == "O\n"
    assert len(dictproxy_transactions) == 0
--- a/chatmaild/src/chatmaild/tests/test_metadata.py
+++ b/chatmaild/src/chatmaild/tests/test_metadata.py
@@ -1,329 +0,0 @@
 import io
 import time
 import pytest
 import requests
 from chatmaild.metadata import (
    Metadata,
    MetadataDictProxy,
 )
 from chatmaild.notifier import (
    Notifier,
    NotifyThread,
    PersistentQueueItem,
 )
@pytest.fixture
 def notifier(metadata):
    queue_dir = metadata.vmail_dir.joinpath("pending_notifications")
    queue_dir.mkdir()
    return Notifier(queue_dir)
@pytest.fixture
 def metadata(tmp_path):
    vmail_dir = tmp_path.joinpath("vmaildir")
    vmail_dir.mkdir()
    return Metadata(vmail_dir)
@pytest.fixture
 def dictproxy(notifier, metadata):
    return MetadataDictProxy(notifier=notifier, metadata=metadata)
@pytest.fixture
 def testaddr2():
    return "user2@example.org"
@pytest.fixture
 def token():
    return "01234"
 def get_mocked_requests(statuslist):
    class ReqMock:
        requests = []
        def post(self, url, data, timeout):
            self.requests.append((url, data, timeout))
            res = statuslist.pop(0)
            if isinstance(res, Exception):
                raise res
            class Result:
                status_code = res
            return Result()
    return ReqMock()
 def test_metadata_persistence(tmp_path, testaddr, testaddr2):
    metadata1 = Metadata(tmp_path)
    metadata2 = Metadata(tmp_path)
    assert not metadata1.get_tokens_for_addr(testaddr)
    assert not metadata2.get_tokens_for_addr(testaddr)
    metadata1.add_token_to_addr(testaddr, "01234")
    metadata1.add_token_to_addr(testaddr2, "456")
    assert metadata2.get_tokens_for_addr(testaddr) == ["01234"]
    assert metadata2.get_tokens_for_addr(testaddr2) == ["456"]
    metadata2.remove_token_from_addr(testaddr, "01234")
    assert not metadata1.get_tokens_for_addr(testaddr)
    assert metadata1.get_tokens_for_addr(testaddr2) == ["456"]
 def test_remove_nonexisting(metadata, tmp_path, testaddr):
    metadata.add_token_to_addr(testaddr, "123")
    metadata.remove_token_from_addr(testaddr, "1l23k1l2k3")
    assert metadata.get_tokens_for_addr(testaddr) == ["123"]
 def test_notifier_remove_without_set(metadata, testaddr):
    metadata.remove_token_from_addr(testaddr, "123")
    assert not metadata.get_tokens_for_addr(testaddr)
 def test_handle_dovecot_request_lookup_fails(dictproxy, testaddr):
    transactions = {}
    res = dictproxy.handle_dovecot_request(
        f"Lpriv/123/chatmail\t{testaddr}", transactions
    )
    assert res == "N\n"
 def test_handle_dovecot_request_happy_path(dictproxy, testaddr, token):
    metadata = dictproxy.metadata
    transactions = {}
    notifier = dictproxy.notifier
    # set device token in a transaction
    tx = "1111"
    msg = f"B{tx}\t{testaddr}"
    res = dictproxy.handle_dovecot_request(msg, transactions)
    assert not res and not metadata.get_tokens_for_addr(testaddr)
    assert transactions == {tx: dict(addr=testaddr, res="O\n")}
    msg = f"S{tx}\tpriv/guid00/devicetoken\t{token}"
    res = dictproxy.handle_dovecot_request(msg, transactions)
    assert not res
    assert len(transactions) == 1
    assert metadata.get_tokens_for_addr(testaddr) == [token]
    msg = f"C{tx}"
    res = dictproxy.handle_dovecot_request(msg, transactions)
    assert res == "O\n"
    assert len(transactions) == 0
    assert metadata.get_tokens_for_addr(testaddr) == [token]
    # trigger notification for incoming message
    tx2 = "2222"
    assert dictproxy.handle_dovecot_request(f"B{tx2}\t{testaddr}", transactions) is None
    msg = f"S{tx2}\tpriv/guid00/messagenew"
    assert dictproxy.handle_dovecot_request(msg, transactions) is None
    queue_item = notifier.retry_queues[0].get()[1]
    assert queue_item.token == token
    assert dictproxy.handle_dovecot_request(f"C{tx2}", transactions) == "O\n"
    assert not transactions
    assert queue_item.path.exists()
 def test_handle_dovecot_protocol_set_devicetoken(dictproxy):
    rfile = io.BytesIO(
        b"\n".join(
            [
                b"HELLO",
                b"Btx00\tuser@example.org",
                b"Stx00\tpriv/guid00/devicetoken\t01234",
                b"Ctx00",
            ]
        )
    )
    wfile = io.BytesIO()
    dictproxy.loop_forever(rfile, wfile)
    assert wfile.getvalue() == b"O\n"
    assert dictproxy.metadata.get_tokens_for_addr("user@example.org") == ["01234"]
 def test_handle_dovecot_protocol_set_get_devicetoken(dictproxy):
    rfile = io.BytesIO(
        b"\n".join(
            [
                b"HELLO",
                b"Btx00\tuser@example.org",
                b"Stx00\tpriv/guid00/devicetoken\t01234",
                b"Ctx00",
            ]
        )
    )
    wfile = io.BytesIO()
    dictproxy.loop_forever(rfile, wfile)
    assert dictproxy.metadata.get_tokens_for_addr("user@example.org") == ["01234"]
    assert wfile.getvalue() == b"O\n"
    rfile = io.BytesIO(
        b"\n".join([b"HELLO", b"Lpriv/0123/devicetoken\tuser@example.org"])
    )
    wfile = io.BytesIO()
    dictproxy.loop_forever(rfile, wfile)
    assert wfile.getvalue() == b"O01234\n"
 def test_handle_dovecot_protocol_iterate(dictproxy):
    rfile = io.BytesIO(
        b"\n".join(
            [
                b"H",
                b"I9\t0\tpriv/5cbe730f146fea6535be0d003dd4fc98/\tci-2dzsrs@nine.testrun.org",
            ]
        )
    )
    wfile = io.BytesIO()
    dictproxy.loop_forever(rfile, wfile)
    assert wfile.getvalue() == b"\n"
 def test_notifier_thread_deletes_persistent_file(metadata, notifier, testaddr):
    reqmock = get_mocked_requests([200])
    metadata.add_token_to_addr(testaddr, "01234")
    notifier.new_message_for_addr(testaddr, metadata)
    NotifyThread(notifier, 0, None).retry_one(reqmock)
    url, data, timeout = reqmock.requests[0]
    assert data == "01234"
    assert metadata.get_tokens_for_addr(testaddr) == ["01234"]
    notifier.requeue_persistent_queue_items()
    assert notifier.retry_queues[0].qsize() == 0
@pytest.mark.parametrize("status", [requests.exceptions.RequestException(), 404, 500])
 def test_notifier_thread_connection_failures(
    metadata, notifier, testaddr, status, caplog
 ):
    """test that tokens keep getting retried until they are given up."""
    metadata.add_token_to_addr(testaddr, "01234")
    notifier.new_message_for_addr(testaddr, metadata)
    notifier.NOTIFICATION_RETRY_DELAY = 5
    max_tries = len(notifier.retry_queues)
    for i in range(max_tries):
        caplog.clear()
        reqmock = get_mocked_requests([status])
        sleep_calls = []
        NotifyThread(notifier, i, None).retry_one(reqmock, sleep=sleep_calls.append)
        assert notifier.retry_queues[i].qsize() == 0
        assert "request failed" in caplog.records[0].msg
        if i > 0:
            assert len(sleep_calls) == 1
        if i + 1 < max_tries:
            assert notifier.retry_queues[i + 1].qsize() == 1
            assert len(caplog.records) == 1
        else:
            assert len(caplog.records) == 2
            assert "deadline" in caplog.records[1].msg
    notifier.requeue_persistent_queue_items()
    assert notifier.retry_queues[0].qsize() == 0
 def test_requeue_removes_tmp_files(notifier, metadata, testaddr, caplog):
    metadata.add_token_to_addr(testaddr, "01234")
    notifier.new_message_for_addr(testaddr, metadata)
    p = notifier.queue_dir.joinpath("1203981203.tmp")
    p.touch()
    notifier2 = notifier.__class__(notifier.queue_dir)
    notifier2.requeue_persistent_queue_items()
    assert "spurious" in caplog.records[0].msg
    assert not p.exists()
    assert notifier2.retry_queues[0].qsize() == 1
    when, queue_item = notifier2.retry_queues[0].get()
    assert when <= int(time.time())
    assert queue_item.addr == testaddr
 def test_requeue_removes_invalid_files(notifier, metadata, testaddr, caplog):
    metadata.add_token_to_addr(testaddr, "01234")
    notifier.new_message_for_addr(testaddr, metadata)
    # empty/invalid files should be ignored
    p = notifier.queue_dir.joinpath("1203981203")
    p.touch()
    notifier2 = notifier.__class__(notifier.queue_dir)
    notifier2.requeue_persistent_queue_items()
    assert "spurious" in caplog.records[0].msg
    assert not p.exists()
    assert notifier2.retry_queues[0].qsize() == 1
    when, queue_item = notifier2.retry_queues[0].get()
    assert when <= int(time.time())
    assert queue_item.addr == testaddr
 def test_start_and_stop_notification_threads(notifier, testaddr):
    threads = notifier.start_notification_threads(None)
    for retry_num, threadlist in threads.items():
        for t in threadlist:
            t.stop()
            t.join()
 def test_multi_device_notifier(metadata, notifier, testaddr):
    metadata.add_token_to_addr(testaddr, "01234")
    metadata.add_token_to_addr(testaddr, "56789")
    notifier.new_message_for_addr(testaddr, metadata)
    reqmock = get_mocked_requests([200, 200])
    NotifyThread(notifier, 0, None).retry_one(reqmock)
    NotifyThread(notifier, 0, None).retry_one(reqmock)
    assert notifier.retry_queues[0].qsize() == 0
    assert notifier.retry_queues[1].qsize() == 0
    url, data, timeout = reqmock.requests[0]
    assert data == "01234"
    url, data, timeout = reqmock.requests[1]
    assert data == "56789"
    assert metadata.get_tokens_for_addr(testaddr) == ["01234", "56789"]
 def test_notifier_thread_run_gone_removes_token(metadata, notifier, testaddr):
    metadata.add_token_to_addr(testaddr, "01234")
    metadata.add_token_to_addr(testaddr, "45678")
    notifier.new_message_for_addr(testaddr, metadata)
    reqmock = get_mocked_requests([410, 200])
    NotifyThread(notifier, 0, metadata.remove_token_from_addr).retry_one(reqmock)
    NotifyThread(notifier, 0, None).retry_one(reqmock)
    url, data, timeout = reqmock.requests[0]
    assert data == "01234"
    url, data, timeout = reqmock.requests[1]
    assert data == "45678"
    assert metadata.get_tokens_for_addr(testaddr) == ["45678"]
    assert notifier.retry_queues[0].qsize() == 0
    assert notifier.retry_queues[1].qsize() == 0
 def test_persistent_queue_items(tmp_path, testaddr, token):
    queue_item = PersistentQueueItem.create(tmp_path, testaddr, 432, token)
    assert queue_item.addr == testaddr
    assert queue_item.start_ts == 432
    assert queue_item.token == token
    item2 = PersistentQueueItem.read_from_path(queue_item.path)
    assert item2.addr == testaddr
    assert item2.start_ts == 432
    assert item2.token == token
    assert item2 == queue_item
    item2.delete()
    assert not item2.path.exists()
    assert not queue_item < item2 and not item2 < queue_item
 def test_iroh_relay(dictproxy):
    rfile = io.BytesIO(
        b"\n".join(
            [
                b"H",
                b"Lshared/0123/vendor/vendor.dovecot/pvt/server/vendor/deltachat/irohrelay\tuser@example.org",
            ]
        )
    )
    wfile = io.BytesIO()
    dictproxy.iroh_relay = "https://example.org/"
    dictproxy.loop_forever(rfile, wfile)
    assert wfile.getvalue() == b"Ohttps://example.org/\n"
--- a/chatmaild/src/chatmaild/tests/test_metrics.py
+++ b/chatmaild/src/chatmaild/tests/test_metrics.py
@@ -1,24 +0,0 @@
 from chatmaild.metrics import main
 def test_main(tmp_path, capsys):
    paths = []
    for x in ("ci-asllkj", "ac_12l3kj", "qweqwe", "ci-l1k2j31l2k3"):
        p = tmp_path.joinpath(x)
        p.mkdir()
        p.joinpath("cur").mkdir()
        paths.append(p)
    tmp_path.joinpath("nomailbox").mkdir()
    main(tmp_path)
    out, _ = capsys.readouterr()
    d = {}
    for line in out.split("\n"):
        if line.strip() and not line.startswith("#"):
            name, num = line.split()
            d[name] = int(num)
    assert d["accounts"] == 4
    assert d["ci_accounts"] == 3
    assert d["nonci_accounts"] == 1
--- a/chatmaild/src/chatmaild/tests/test_migrate_db.py
+++ b/chatmaild/src/chatmaild/tests/test_migrate_db.py
@@ -1,67 +0,0 @@
 import sqlite3
 from chatmaild.migrate_db import migrate_from_db_to_maildir
 def test_migration_not_exists(tmp_path, example_config):
    example_config.passdb_path = tmp_path.joinpath("sqlite")
 def test_migration(tmp_path, example_config, caplog):
    passdb_path = tmp_path.joinpath("passdb.sqlite")
    uri = f"file:{passdb_path}?mode=rwc"
    sqlconn = sqlite3.connect(uri, timeout=60, uri=True)
    sqlconn.execute(
        """
        CREATE TABLE users (
            addr TEXT PRIMARY KEY,
            password TEXT,
            last_login INTEGER
        )
    """
    )
    all = {}
    for i in range(500):
        values = (f"somsom{i:03}@example.org", f"passwo{i:03}", i * 86400)
        sqlconn.execute(
            """
            INSERT INTO users (addr, password, last_login)
            VALUES (?, ?, ?)""",
            values,
        )
        all[values[0]] = values[1:]
    for i in range(500):
        values = (f"pompom{i:03}@example.org", f"wopass{i:03}", "")
        sqlconn.execute(
            """
            INSERT INTO users (addr, password, last_login)
            VALUES (?, ?, ?)""",
            values,
        )
        all[values[0]] = values[1:]
    sqlconn.commit()
    sqlconn.close()
    assert passdb_path.stat().st_size > 10000
    example_config.passdb_path = passdb_path
    assert not caplog.records
    migrate_from_db_to_maildir(example_config, chunking=500)
    assert len(caplog.records) > 3
    for path in example_config.mailboxes_dir.iterdir():
        if "@" not in path.name:
            continue
        password, last_login = all.pop(path.name)
        user = example_config.get_user(path.name)
        if last_login:
            assert user.get_last_login_timestamp() == last_login
        assert password == user.get_userdb_dict()["password"]
    assert not all
    assert not example_config.passdb_path.exists()
--- a/chatmaild/src/chatmaild/tests/test_newmail.py
+++ b/chatmaild/src/chatmaild/tests/test_newmail.py
@@ -1,27 +0,0 @@
 import json
 import chatmaild
 from chatmaild.newemail import create_newemail_dict, print_new_account
 def test_create_newemail_dict(example_config):
    ac1 = create_newemail_dict(example_config)
    assert "@" in ac1["email"]
    assert len(ac1["password"]) >= 10
    ac2 = create_newemail_dict(example_config)
    assert ac1["email"] != ac2["email"]
    assert ac1["password"] != ac2["password"]
 def test_print_new_account(capsys, monkeypatch, maildomain, tmpdir, example_config):
    monkeypatch.setattr(chatmaild.newemail, "CONFIG_PATH", str(example_config._inipath))
    print_new_account()
    out, err = capsys.readouterr()
    lines = out.split("\n")
    assert lines[0] == "Content-Type: application/json"
    assert not lines[1]
    dic = json.loads(lines[2])
    assert dic["email"].endswith(f"@{example_config.mail_domain}")
    assert len(dic["password"]) >= 10
--- a/chatmaild/src/chatmaild/tests/test_user.py
+++ b/chatmaild/src/chatmaild/tests/test_user.py
@@ -1,56 +0,0 @@
 def test_login_timestamp(testaddr, example_config):
    user = example_config.get_user(testaddr)
    user.set_password("someeqkjwelkqwjleqwe")
    user.set_last_login_timestamp(100000)
    assert user.get_last_login_timestamp() == 86400
    user.set_last_login_timestamp(200000)
    assert user.get_last_login_timestamp() == 86400 * 2
 def test_get_user_dict_not_set(testaddr, example_config, caplog):
    user = example_config.get_user(testaddr)
    assert not caplog.records
    assert user.get_userdb_dict() == {}
    assert len(caplog.records) == 0
    user.set_password("")
    assert user.get_userdb_dict() == {}
    assert len(caplog.records) == 1
 def test_get_user_dict(make_config, tmp_path):
    config = make_config("something.testrun.org")
    addr = "user1@something.org"
    user = config.get_user(addr)
    enc_password = "l1k2j31lk2j3l1k23j123"
    user.set_password(enc_password)
    data = user.get_userdb_dict()
    assert addr in str(data["home"])
    assert data["uid"] == "vmail"
    assert data["gid"] == "vmail"
    assert data["password"] == enc_password
 def test_no_mailboxes_dir(testaddr, example_config, tmp_path):
    p = tmp_path.joinpath("a", "mailboxes")
    example_config.mailboxes_dir = p
    user = example_config.get_user(testaddr)
    user.set_password("someeqkjwelkqwjleqwe")
    user.set_last_login_timestamp(100000)
    assert user.get_last_login_timestamp() == 86400
 def test_set_get_cleartext_flag(testaddr, example_config, tmp_path):
    p = tmp_path.joinpath("a", "mailboxes")
    example_config.mailboxes_dir = p
    user = example_config.get_user(testaddr)
    user.set_password("someeqkjwelkqwjleqwe")
    user.set_last_login_timestamp(100000)
    assert user.get_last_login_timestamp() == 86400
    assert not user.is_incoming_cleartext_ok()
    user.allow_incoming_cleartext()
    assert user.is_incoming_cleartext_ok()
--- a/chatmaild/src/chatmaild/turnserver.py
+++ b/chatmaild/src/chatmaild/turnserver.py
@@ -1,9 +0,0 @@
 #!/usr/bin/env python3
 import socket
 def turn_credentials() -> str:
    with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client_socket:
        client_socket.connect("/run/chatmail-turn/turn.socket")
        with client_socket.makefile("rb") as file:
            return file.readline().decode("utf-8").strip()
--- a/chatmaild/src/chatmaild/user.py
+++ b/chatmaild/src/chatmaild/user.py
@@ -1,82 +0,0 @@
 import logging
 import os
 from chatmaild.filedict import write_bytes_atomic
 def get_daytimestamp(timestamp) -> int:
    return int(timestamp) // 86400 * 86400
 class User:
    def __init__(self, maildir, addr, password_path, uid, gid):
        self.maildir = maildir
        self.addr = addr
        self.password_path = password_path
        self.enforce_E2EE_path = maildir.joinpath("enforceE2EEincoming")
        self.uid = uid
        self.gid = gid
    @property
    def can_track(self):
        return "@" in self.addr
    def get_userdb_dict(self):
        """Return a non-empty dovecot 'userdb' style dict
        if the user has an existing non-empty password"""
        try:
            pw = self.password_path.read_text()
        except FileNotFoundError:
            return {}
        if not pw:
            logging.error(f"password is empty for: {self.addr}")
            return {}
        home = str(self.maildir)
        return dict(addr=self.addr, home=home, uid=self.uid, gid=self.gid, password=pw)
    def is_incoming_cleartext_ok(self):
        return not self.enforce_E2EE_path.exists()
    def allow_incoming_cleartext(self):
        if self.enforce_E2EE_path.exists():
            self.enforce_E2EE_path.unlink()
    def set_password(self, enc_password):
        """Set the specified password for this user.
        This method can be called concurrently
        but there is no guarantee which of the password-set calls will win.
        """
        self.maildir.mkdir(exist_ok=True, parents=True)
        password = enc_password.encode("ascii")
        try:
            write_bytes_atomic(self.password_path, password)
        except PermissionError:
            logging.error(f"could not write password for: {self.addr}")
            raise
        self.enforce_E2EE_path.touch()
    def set_last_login_timestamp(self, timestamp):
        """Track login time with daily granularity
        to minimize touching files and to minimize metadata leakage."""
        if not self.can_track:
            return
        try:
            mtime = int(os.stat(self.password_path).st_mtime)
        except FileNotFoundError:
            logging.error(f"Can not get last login timestamp for {self.addr}")
            return
        timestamp = get_daytimestamp(timestamp)
        if mtime != timestamp:
            os.utime(self.password_path, (timestamp, timestamp))
    def get_last_login_timestamp(self):
        if self.can_track:
            try:
                return int(self.password_path.stat().st_mtime)
            except FileNotFoundError:
                pass
--- a/cliff.toml
+++ b/cliff.toml
@@ -1,94 +0,0 @@
 # git-cliff ~ configuration file
 # https://git-cliff.org/docs/configuration
 [changelog]
 # A Tera template to be rendered for each release in the changelog.
 # See https://keats.github.io/tera/docs/#introduction
 body = """
 {% if version %}\
    ## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }}
 {% else %}\
    ## [unreleased]
 {% endif %}\
 {% for group, commits in commits | group_by(attribute="group") %}
    ### {{ group | striptags | trim | upper_first }}
    {% for commit in commits %}
        - {% if commit.scope %}*({{ commit.scope }})* {% endif %}\
            {% if commit.breaking %}[**breaking**] {% endif %}\
            {{ commit.message | upper_first }}\
    {% endfor %}
 {% endfor %}
 """
 # Remove leading and trailing whitespaces from the changelog's body.
 trim = true
 # Render body even when there are no releases to process.
 render_always = true
 # An array of regex based postprocessors to modify the changelog.
 postprocessors = [
    # Replace the placeholder <REPO> with a URL.
    #{ pattern = '<REPO>', replace = "https://github.com/orhun/git-cliff" },
 ]
 # render body even when there are no releases to process
 # render_always = true
 # output file path
 # output = "test.md"
 [git]
 # Parse commits according to the conventional commits specification.
 # See https://www.conventionalcommits.org
 conventional_commits = true
 # Exclude commits that do not match the conventional commits specification.
 filter_unconventional = true
 # Require all commits to be conventional.
 # Takes precedence over filter_unconventional.
 require_conventional = false
 # Split commits on newlines, treating each line as an individual commit.
 split_commits = false
 # An array of regex based parsers to modify commit messages prior to further processing.
 commit_preprocessors = [
    # Replace issue numbers with link templates to be updated in `changelog.postprocessors`.
    #{ pattern = '\((\w+\s)?#([0-9]+)\)', replace = "([#${2}](<REPO>/issues/${2}))"},
    # Check spelling of the commit message using https://github.com/crate-ci/typos.
    # If the spelling is incorrect, it will be fixed automatically.
    #{ pattern = '.*', replace_command = 'typos --write-changes -' },
 ]
 # Prevent commits that are breaking from being excluded by commit parsers.
 protect_breaking_commits = false
 # An array of regex based parsers for extracting data from the commit message.
 # Assigns commits to groups.
 # Optionally sets the commit's scope and can decide to exclude commits from further processing.
 commit_parsers = [
    { message = "^feat", group = "Features" },
    { message = "^fix", group = "Bug Fixes" },
    { message = "^docs", group = "Documentation" },
    { message = "^perf", group = "Performance" },
    { message = "^refactor", group = "Refactor" },
    { message = "^style", group = "Styling" },
    { message = "^test", group = "Testing" },
    { message = "^chore\\(release\\): prepare for", skip = true },
    { message = "^chore\\(deps.*\\)", skip = true },
    { message = "^chore\\(pr\\)", skip = true },
    { message = "^chore\\(pull\\)", skip = true },
    { message = "^chore|^ci", group = "Miscellaneous Tasks" },
    { body = ".*security", group = "Security" },
    { message = "^revert", group = "Revert" },
    { message = ".*", group = "Other" },
 ]
 # Exclude commits that are not matched by any commit parser.
 filter_commits = false
 # Fail on a commit that is not matched by any commit parser.
 fail_on_unmatched_commit = false
 # An array of link parsers for extracting external references, and turning them into URLs, using regex.
 link_parsers = []
 # Include only the tags that belong to the current branch.
 use_branch_tags = false
 # Order releases topologically instead of chronologically.
 topo_order = false
 # Order commits topologically instead of chronologically.
 topo_order_commits = true
 # Order of commits in each group/release within the changelog.
 # Allowed values: newest, oldest
 sort_commits = "oldest"
 # Process submodules commits
 recurse_submodules = false
--- a/cmdeploy/pyproject.toml
+++ b/cmdeploy/pyproject.toml
@@ -1,46 +0,0 @@
 [build-system]
 requires = ["setuptools>=68"]
 build-backend = "setuptools.build_meta"
 [project]
 name = "cmdeploy"
 version = "0.2"
 dependencies = [
  "pyinfra>=3",
  "pillow",
  "qrcode",
  "markdown",
  "pytest",
  "setuptools>=68",
  "termcolor",
  "build",
  "tox",
  "ruff",
  "pytest",
  "pytest-xdist", 
  "execnet",
  "imap_tools",
 ]
 [project.scripts]
 cmdeploy = "cmdeploy.cmdeploy:main"
 [project.entry-points.pytest11]
 "chatmaild.testplugin" = "chatmaild.tests.plugin"
 "cmdeploy.testplugin" = "cmdeploy.tests.plugin"
 [tool.pytest.ini_options]
 addopts = "-v -ra --strict-markers"
 [tool.ruff]
 lint.select = [
  "F", # Pyflakes
  "I", # isort
  "PLC", # Pylint Convention
  "PLE", # Pylint Error
  "PLW", # Pylint Warning
 ]
 lint.ignore = [
  "PLC0415" # import-outside-top-level
 ]
--- a/cmdeploy/src/cmdeploy/acmetool/init.py
+++ b/cmdeploy/src/cmdeploy/acmetool/init.py
@@ -1,141 +0,0 @@
 import importlib.resources
 from pyinfra.operations import apt, files, server, systemd
 from ..basedeploy import Deployer
 class AcmetoolDeployer(Deployer):
    def __init__(self, email, domains):
        self.domains = domains
        self.email = email
        self.need_restart_redirector = False
        self.need_restart_reconcile_service = False
        self.need_restart_reconcile_timer = False
    def install(self):
        apt.packages(
            name="Install acmetool",
            packages=["acmetool"],
        )
        files.file(
            name="Remove old acmetool cronjob, it is replaced with systemd timer.",
            path="/etc/cron.d/acmetool",
            present=False,
        )
        files.put(
            name="Install acmetool hook.",
            src=importlib.resources.files(__package__)
            .joinpath("acmetool.hook")
            .open("rb"),
            dest="/etc/acme/hooks/nginx",
            user="root",
            group="root",
            mode="755",
        )
        files.file(
            name="Remove acmetool hook from the wrong location where it was previously installed.",
            path="/usr/lib/acme/hooks/nginx",
            present=False,
        )
    def configure(self):
        files.template(
            src=importlib.resources.files(__package__).joinpath(
                "response-file.yaml.j2"
            ),
            dest="/var/lib/acme/conf/responses",
            user="root",
            group="root",
            mode="644",
            email=self.email,
        )
        files.template(
            src=importlib.resources.files(__package__).joinpath("target.yaml.j2"),
            dest="/var/lib/acme/conf/target",
            user="root",
            group="root",
            mode="644",
        )
        server.shell(
            name=f"Remove old acmetool desired files for {self.domains[0]}",
            commands=[f"rm -f /var/lib/acme/desired/{self.domains[0]}-*"],
        )
        files.template(
            src=importlib.resources.files(__package__).joinpath("desired.yaml.j2"),
            dest=f"/var/lib/acme/desired/{self.domains[0]}", # 0 is mailhost TLD
            user="root",
            group="root",
            mode="644",
            domains=self.domains,
        )
        service_file = files.put(
            src=importlib.resources.files(__package__).joinpath(
                "acmetool-redirector.service"
            ),
            dest="/etc/systemd/system/acmetool-redirector.service",
            user="root",
            group="root",
            mode="644",
        )
        self.need_restart_redirector = service_file.changed
        reconcile_service_file = files.put(
            src=importlib.resources.files(__package__).joinpath(
                "acmetool-reconcile.service"
            ),
            dest="/etc/systemd/system/acmetool-reconcile.service",
            user="root",
            group="root",
            mode="644",
        )
        self.need_restart_reconcile_service = reconcile_service_file.changed
        reconcile_timer_file = files.put(
            src=importlib.resources.files(__package__).joinpath(
                "acmetool-reconcile.timer"
            ),
            dest="/etc/systemd/system/acmetool-reconcile.timer",
            user="root",
            group="root",
            mode="644",
        )
        self.need_restart_reconcile_timer = reconcile_timer_file.changed
    def activate(self):
        systemd.service(
            name="Setup acmetool-redirector service",
            service="acmetool-redirector.service",
            running=True,
            enabled=True,
            restarted=self.need_restart_redirector,
        )
        self.need_restart_redirector = False
        systemd.service(
            name="Setup acmetool-reconcile service",
            service="acmetool-reconcile.service",
            running=False,
            enabled=False,
            daemon_reload=self.need_restart_reconcile_service,
        )
        self.need_restart_reconcile_service = False
        systemd.service(
            name="Setup acmetool-reconcile timer",
            service="acmetool-reconcile.timer",
            running=True,
            enabled=True,
            daemon_reload=self.need_restart_reconcile_timer,
        )
        self.need_restart_reconcile_timer = False
        server.shell(
            name=f"Reconcile certificates for: {', '.join(self.domains)}",
            commands=["acmetool --batch --xlog.severity=debug reconcile"],
        )
--- a/cmdeploy/src/cmdeploy/acmetool/acmetool-reconcile.service
+++ b/cmdeploy/src/cmdeploy/acmetool/acmetool-reconcile.service
@@ -1,8 +0,0 @@
 [Unit]
 Description=Renew TLS certificates with acmetool
 After=network.target
 [Service]
 Type=oneshot
 ExecStart=/usr/bin/acmetool --batch reconcile
--- a/cmdeploy/src/cmdeploy/acmetool/acmetool-reconcile.timer
+++ b/cmdeploy/src/cmdeploy/acmetool/acmetool-reconcile.timer
@@ -1,8 +0,0 @@
 [Unit]
 Description=Renew TLS certificates with acmetool
 [Timer]
 OnCalendar=*-*-* 16:20:00
 [Install]
 WantedBy=timers.target
--- a/cmdeploy/src/cmdeploy/acmetool/acmetool-redirector.service
+++ b/cmdeploy/src/cmdeploy/acmetool/acmetool-redirector.service
@@ -1,11 +0,0 @@
 [Unit]
 Description=acmetool HTTP redirector
 [Service]
 Type=notify
 ExecStart=/usr/bin/acmetool redirector --service.uid=daemon
 Restart=always
 RestartSec=30
 [Install]
 WantedBy=multi-user.target
--- a/cmdeploy/src/cmdeploy/acmetool/desired.yaml.j2
+++ b/cmdeploy/src/cmdeploy/acmetool/desired.yaml.j2
@@ -1,6 +0,0 @@
 satisfy:
  names:
 {%- for domain in domains %}
  - {{ domain }}
 {%- endfor %}
--- a/cmdeploy/src/cmdeploy/acmetool/response-file.yaml.j2
+++ b/cmdeploy/src/cmdeploy/acmetool/response-file.yaml.j2
@@ -1,2 +0,0 @@
 "acme-enter-email": "{{ email }}"
 "acme-agreement:https://letsencrypt.org/documents/LE-SA-v1.6-August-18-2025.pdf": true
--- a/cmdeploy/src/cmdeploy/basedeploy.py
+++ b/cmdeploy/src/cmdeploy/basedeploy.py
@@ -1,116 +0,0 @@
 import importlib.resources
 import io
 import os
 from pyinfra.operations import files, server, systemd
 def has_systemd():
    """Returns False during Docker image builds or any other non-systemd environment."""
    return os.path.isdir("/run/systemd/system")
 def get_resource(arg, pkg=__package__):
    return importlib.resources.files(pkg).joinpath(arg)
 def configure_remote_units(mail_domain, units) -> None:
    remote_base_dir = "/usr/local/lib/chatmaild"
    remote_venv_dir = f"{remote_base_dir}/venv"
    remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini"
    root_owned = dict(user="root", group="root", mode="644")
    # install systemd units
    for fn in units:
        params = dict(
            execpath=f"{remote_venv_dir}/bin/{fn}",
            config_path=remote_chatmail_inipath,
            remote_venv_dir=remote_venv_dir,
            mail_domain=mail_domain,
        )
        basename = fn if "." in fn else f"{fn}.service"
        source_path = get_resource(f"service/{basename}.f")
        content = source_path.read_text().format(**params).encode()
        files.put(
            name=f"Upload {basename}",
            src=io.BytesIO(content),
            dest=f"/etc/systemd/system/{basename}",
            **root_owned,
        )
 def activate_remote_units(units) -> None:
    # activate systemd units
    for fn in units:
        basename = fn if "." in fn else f"{fn}.service"
        if fn == "chatmail-expire" or fn == "chatmail-fsreport":
            # don't auto-start but let the corresponding timer trigger execution
            enabled = False
        else:
            enabled = True
        systemd.service(
            name=f"Setup {basename}",
            service=basename,
            running=enabled,
            enabled=enabled,
            restarted=enabled,
            daemon_reload=True,
        )
 class Deployment:
    def install(self, deployer):
        # optional 'required_users' contains a list of (user, group, secondary-group-list) tuples.
        # If the group is None, no group is created corresponding to that user.
        # If the secondary group list is not None, all listed groups are created as well.
        required_users = getattr(deployer, "required_users", [])
        for user, group, groups in required_users:
            if group is not None:
                server.group(
                    name="Create {} group".format(group), group=group, system=True
                )
            if groups is not None:
                for group2 in groups:
                    server.group(
                        name="Create {} group".format(group2), group=group2, system=True
                    )
            server.user(
                name="Create {} user".format(user),
                user=user,
                group=group,
                groups=groups,
                system=True,
            )
        deployer.install()
    def configure(self, deployer):
        deployer.configure()
    def activate(self, deployer):
        deployer.activate()
    def perform_stages(self, deployers):
        default_stages = "install,configure,activate"
        stages = os.getenv("CMDEPLOY_STAGES", default_stages).split(",")
        for stage in stages:
            for deployer in deployers:
                getattr(self, stage)(deployer)
 class Deployer:
    need_restart = False
    def install(self):
        pass
    def configure(self):
        pass
    def activate(self):
        pass
--- a/cmdeploy/src/cmdeploy/chatmail.zone.j2
+++ b/cmdeploy/src/cmdeploy/chatmail.zone.j2
@@ -1,30 +0,0 @@
 ;
 ; Required DNS entries for chatmail servers 
 ;
 {% if A %}
 {{ mail_domain }}.                   A     {{ A }}
 {% endif %}
 {% if AAAA %}
 {{ mail_domain }}.                   AAAA  {{ AAAA }}
 {% endif %}
 {{ mail_domain }}.                   MX 10 {{ mail_domain }}.
 _mta-sts.{{ mail_domain }}.          TXT "v=STSv1; id={{ sts_id }}"
 mta-sts.{{ mail_domain }}.           CNAME {{ mail_domain }}.
 www.{{ mail_domain }}.               CNAME {{ mail_domain }}.
 {{ dkim_entry }}
 ;
 ; Recommended DNS entries for interoperability and security-hardening
 ;
 {{ mail_domain }}.                   TXT "v=spf1 a ~all"
 _dmarc.{{ mail_domain }}.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
 {% if acme_account_url %}
 {{ mail_domain }}.                   CAA 0 issue "letsencrypt.org;accounturi={{ acme_account_url }}"
 {% endif %}
 _adsp._domainkey.{{ mail_domain }}.  TXT "dkim=discardable"
 _submission._tcp.{{ mail_domain }}.  SRV 0 1 587 {{ mail_domain }}.
 _submissions._tcp.{{ mail_domain }}. SRV 0 1 465 {{ mail_domain }}.
 _imap._tcp.{{ mail_domain }}.        SRV 0 1 143 {{ mail_domain }}.
 _imaps._tcp.{{ mail_domain }}.       SRV 0 1 993 {{ mail_domain }}.
--- a/cmdeploy/src/cmdeploy/cmdeploy.py
+++ b/cmdeploy/src/cmdeploy/cmdeploy.py
@@ -1,424 +0,0 @@
 """
 Provides the `cmdeploy` entry point function,
 along with command line option and subcommand parsing.
 """
 import argparse
 import importlib.resources
 import importlib.util
 import os
 import pathlib
 import shutil
 import subprocess
 import sys
 from pathlib import Path
 import pyinfra
 from chatmaild.config import read_config, write_initial_config
 from packaging import version
 from termcolor import colored
 from . import dns, remote
 from .sshexec import LocalExec, SSHExec
 #
 # cmdeploy sub commands and options
 #
 def init_cmd_options(parser):
    parser.add_argument(
        "chatmail_domain",
        action="store",
        help="fully qualified DNS domain name for your chatmail instance",
    )
    parser.add_argument(
        "--force",
        dest="recreate_ini",
        action="store_true",
        help="force reacreate ini file",
    )
 def init_cmd(args, out):
    """Initialize chatmail config file."""
    mail_domain = args.chatmail_domain
    inipath = args.inipath
    if args.inipath.exists():
        if not args.recreate_ini:
            print(f"[WARNING] Path exists, not modifying: {inipath}")
            return 1
        else:
            print(
                f"[WARNING] Force argument was provided, deleting config file: {inipath}"
            )
            inipath.unlink()
    write_initial_config(inipath, mail_domain, overrides={})
    out.green(f"created config file for {mail_domain} in {inipath}")
 def run_cmd_options(parser):
    parser.add_argument(
        "--dry-run",
        dest="dry_run",
        action="store_true",
        help="don't actually modify the server",
    )
    parser.add_argument(
        "--disable-mail",
        dest="disable_mail",
        action="store_true",
        help="install/upgrade the server, but disable postfix & dovecot for now",
    )
    parser.add_argument(
        "--website-only",
        action="store_true",
        help="only update/deploy the website, skipping full server upgrade/deployment, useful when you only changed/updated the web pages and don't need to re-run a full server upgrade",
    )
    parser.add_argument(
        "--skip-dns-check",
        dest="dns_check_disabled",
        action="store_true",
        help="disable checks nslookup for dns",
    )
    add_ssh_host_option(parser)
 def run_cmd(args, out):
    """Deploy chatmail services on the remote server."""
    ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
    sshexec = get_sshexec(ssh_host)
    require_iroh = args.config.enable_iroh_relay
    if not args.dns_check_disabled:
        remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
        if not dns.check_initial_remote_data(remote_data, print=out.red):
            return 1
    env = os.environ.copy()
    env["CHATMAIL_INI"] = args.inipath
    env["CHATMAIL_WEBSITE_ONLY"] = "True" if args.website_only else ""
    env["CHATMAIL_DISABLE_MAIL"] = "True" if args.disable_mail else ""
    env["CHATMAIL_REQUIRE_IROH"] = "True" if require_iroh else ""
    if not args.dns_check_disabled:
        env["CHATMAIL_ADDR_V4"] = remote_data.get("A") or ""
        env["CHATMAIL_ADDR_V6"] = remote_data.get("AAAA") or ""
    deploy_path = importlib.resources.files(__package__).joinpath("run.py").resolve()
    pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
    cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y"
    if ssh_host in ["localhost", "@docker"]:
        if ssh_host == "@docker":
            env["CHATMAIL_NOPORTCHECK"] = "True"
            env["CHATMAIL_NOSYSCTL"] = "True"
        cmd = f"{pyinf} @local {deploy_path} -y"
    if version.parse(pyinfra.__version__) < version.parse("3"):
        out.red("Please re-run scripts/initenv.sh to update pyinfra to version 3.")
        return 1
    try:
        retcode = out.check_call(cmd, env=env)
        if args.website_only:
            if retcode == 0:
                out.green("Website deployment completed.")
            else:
                out.red("Website deployment failed.")
        elif retcode == 0:
            out.green("Deploy completed, call `cmdeploy dns` next.")
        elif not args.dns_check_disabled and not remote_data["acme_account_url"]:
            out.red("Deploy completed but letsencrypt not configured")
            out.red("Run 'cmdeploy run' again")
            retcode = 0
        else:
            out.red("Deploy failed")
    except subprocess.CalledProcessError:
        out.red("Deploy failed")
        retcode = 1
    return retcode
 def dns_cmd_options(parser):
    parser.add_argument(
        "--zonefile",
        dest="zonefile",
        type=pathlib.Path,
        default=None,
        help="write out a zonefile",
    )
    add_ssh_host_option(parser)
 def dns_cmd(args, out):
    """Check DNS entries and optionally generate dns zone file."""
    ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
    sshexec = get_sshexec(ssh_host, verbose=args.verbose)
    remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
    if not remote_data:
        return 1
    if not remote_data["acme_account_url"]:
        out.red("could not get letsencrypt account url, please run 'cmdeploy run'")
        return 1
    if not remote_data["dkim_entry"]:
        out.red("could not determine dkim_entry, please run 'cmdeploy run'")
        return 1
    zonefile = dns.get_filled_zone_file(remote_data)
    if args.zonefile:
        args.zonefile.write_text(zonefile)
        out.green(f"DNS records successfully written to: {args.zonefile}")
        return 0
    retcode = dns.check_full_zone(
        sshexec, remote_data=remote_data, zonefile=zonefile, out=out
    )
    return retcode
 def status_cmd_options(parser):
    add_ssh_host_option(parser)
 def status_cmd(args, out):
    """Display status for online chatmail instance."""
    ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
    sshexec = get_sshexec(ssh_host, verbose=args.verbose)
    out.green(f"chatmail domain: {args.config.mail_domain}")
    if args.config.privacy_mail:
        out.green("privacy settings: present")
    else:
        out.red("no privacy settings")
    for line in sshexec(remote.rshell.get_systemd_running):
        print(line)
 def test_cmd_options(parser):
    parser.add_argument(
        "--slow",
        dest="slow",
        action="store_true",
        help="also run slow tests",
    )
 def test_cmd(args, out):
    """Run local and online tests for chatmail deployment.
    This will automatically pip-install 'deltachat' if it's not available.
    """
    x = importlib.util.find_spec("deltachat")
    if x is None:
        out.check_call(f"{sys.executable} -m pip install deltachat")
    pytest_path = shutil.which("pytest")
    pytest_args = [
        pytest_path,
        "cmdeploy/src/",
        "-n4",
        "-rs",
        "-x",
        "-v",
        "--durations=5",
    ]
    if args.slow:
        pytest_args.append("--slow")
    ret = out.run_ret(pytest_args)
    return ret
 def fmt_cmd_options(parser):
    parser.add_argument(
        "--check",
        "-c",
        action="store_true",
        help="only check but don't fix problems",
    )
 def fmt_cmd(args, out):
    """Run formattting fixes on all chatmail source code."""
    chatmaild_dir = importlib.resources.files("chatmaild").resolve()
    cmdeploy_dir = chatmaild_dir.joinpath(
        "..", "..", "..", "cmdeploy", "src", "cmdeploy"
    ).resolve()
    sources = [str(chatmaild_dir), str(cmdeploy_dir)]
    format_args = [shutil.which("ruff"), "format"]
    check_args = [shutil.which("ruff"), "check"]
    if args.check:
        format_args.append("--diff")
    else:
        check_args.append("--fix")
    if not args.verbose:
        check_args.append("--quiet")
        format_args.append("--quiet")
    format_args.extend(sources)
    check_args.extend(sources)
    out.check_call(" ".join(format_args), quiet=not args.verbose)
    out.check_call(" ".join(check_args), quiet=not args.verbose)
 def bench_cmd(args, out):
    """Run benchmarks against an online chatmail instance."""
    args = ["pytest", "--pyargs", "cmdeploy.tests.online.benchmark", "-vrx"]
    cmdstring = " ".join(args)
    out.green(f"[$ {cmdstring}]")
    subprocess.check_call(args)
 def webdev_cmd(args, out):
    """Run local web development loop for static web pages."""
    from .www import main
    main()
 #
 # Parsing command line options and starting commands
 #
 class Out:
    """Convenience output printer providing coloring."""
    def red(self, msg, file=sys.stderr):
        print(colored(msg, "red"), file=file)
    def green(self, msg, file=sys.stderr):
        print(colored(msg, "green"), file=file)
    def __call__(self, msg, red=False, green=False, file=sys.stdout):
        color = "red" if red else ("green" if green else None)
        print(colored(msg, color), file=file)
    def check_call(self, arg, env=None, quiet=False):
        if not quiet:
            self(f"[$ {arg}]", file=sys.stderr)
        return subprocess.check_call(arg, shell=True, env=env)
    def run_ret(self, args, env=None, quiet=False):
        if not quiet:
            cmdstring = " ".join(args)
            self(f"[$ {cmdstring}]", file=sys.stderr)
        proc = subprocess.run(args, env=env, check=False)
        return proc.returncode
 def add_ssh_host_option(parser):
    parser.add_argument(
        "--ssh-host",
        dest="ssh_host",
        help="Run commands on 'localhost', via '@docker', or on a specific SSH host "
        "instead of chatmail.ini's mail_domain.",
    )
 def add_config_option(parser):
    parser.add_argument(
        "--config",
        dest="inipath",
        action="store",
        default=Path(os.environ.get("CHATMAIL_INI", "chatmail.ini")),
        type=Path,
        help="path to the chatmail.ini file",
    )
    parser.add_argument(
        "--verbose",
        "-v",
        dest="verbose",
        action="store_true",
        default=False,
        help="provide verbose logging",
    )
 def add_subcommand(subparsers, func):
    name = func.__name__
    assert name.endswith("_cmd")
    name = name[:-4]
    doc = func.__doc__.strip()
    help = doc.split("\n")[0].strip(".")
    p = subparsers.add_parser(name, description=doc, help=help)
    p.set_defaults(func=func)
    add_config_option(p)
    return p
 description = """
 Setup your chatmail server configuration and
 deploy it via SSH to your remote location.
 """
 def get_parser():
    """Return an ArgumentParser for the 'cmdeploy' CLI"""
    parser = argparse.ArgumentParser(description=description.strip())
    subparsers = parser.add_subparsers(title="subcommands")
    # find all subcommands in the module namespace
    glob = globals()
    for name, func in glob.items():
        if name.endswith("_cmd"):
            subparser = add_subcommand(subparsers, func)
            addopts = glob.get(name + "_options")
            if addopts is not None:
                addopts(subparser)
    return parser
 def get_sshexec(ssh_host: str, verbose=True):
    if ssh_host in ["localhost", "@local"]:
        return LocalExec(verbose, docker=False)
    elif ssh_host == "@docker":
        return LocalExec(verbose, docker=True)
    if verbose:
        print(f"[ssh] login to {ssh_host}")
    return SSHExec(ssh_host, verbose=verbose)
 def main(args=None):
    """Provide main entry point for 'cmdeploy' CLI invocation."""
    parser = get_parser()
    args = parser.parse_args(args=args)
    if not hasattr(args, "func"):
        return parser.parse_args(["-h"])
    out = Out()
    kwargs = {}
    if args.func.__name__ not in ("init_cmd", "fmt_cmd"):
        if not args.inipath.exists():
            out.red(f"expecting {args.inipath} to exist, run init first?")
            raise SystemExit(1)
        try:
            args.config = read_config(args.inipath)
        except Exception as ex:
            out.red(ex)
            raise SystemExit(1)
    try:
        res = args.func(args, out, **kwargs)
        if res is None:
            res = 0
        return res
    except KeyboardInterrupt:
        out.red("KeyboardInterrupt")
        sys.exit(130)
 if __name__ == "__main__":
    main()
--- a/cmdeploy/src/cmdeploy/data/delta-chat-bw.png
+++ b/cmdeploy/src/cmdeploy/data/delta-chat-bw.png
--- a/cmdeploy/src/cmdeploy/data/opensans-regular.ttf
+++ b/cmdeploy/src/cmdeploy/data/opensans-regular.ttf
--- a/cmdeploy/src/cmdeploy/deployers.py
+++ b/cmdeploy/src/cmdeploy/deployers.py
@@ -1,634 +0,0 @@
 """
 Chat Mail pyinfra deploy.
 """
 import os
 import shutil
 import subprocess
 import sys
 from io import StringIO
 from pathlib import Path
 from chatmaild.config import read_config
 from pyinfra import facts, host, logger
 from pyinfra.facts import hardware
 from pyinfra.api import FactBase
 from pyinfra.facts.files import Sha256File
 from pyinfra.facts.systemd import SystemdEnabled
 from pyinfra.operations import apt, files, pip, server, systemd
 from cmdeploy.cmdeploy import Out
 from .acmetool import AcmetoolDeployer
 from .basedeploy import (
    Deployer,
    Deployment,
    activate_remote_units,
    configure_remote_units,
    get_resource,
    has_systemd,
 )
 from .dovecot.deployer import DovecotDeployer
 from .filtermail.deployer import FiltermailDeployer
 from .mtail.deployer import MtailDeployer
 from .nginx.deployer import NginxDeployer
 from .opendkim.deployer import OpendkimDeployer
 from .postfix.deployer import PostfixDeployer
 from .www import build_webpages, find_merge_conflict, get_paths
 class Port(FactBase):
    """
    Returns the process occupying a port.
    """
    def command(self, port: int) -> str:
        return (
            "ss -lptn 'src :%d' | awk 'NR>1 {print $6,$7}' | sed 's/users:((\"//;s/\".*//'"
            % (port,)
        )
    def process(self, output: [str]) -> str:
        return output[0]
 def _build_chatmaild(dist_dir) -> None:
    dist_dir = Path(dist_dir).resolve()
    if dist_dir.exists():
        shutil.rmtree(dist_dir)
    dist_dir.mkdir()
    subprocess.check_output(
        [sys.executable, "-m", "build", "-n"]
        + ["--sdist", "chatmaild", "--outdir", str(dist_dir)]
    )
    entries = list(dist_dir.iterdir())
    assert len(entries) == 1
    return entries[0]
 def remove_legacy_artifacts():
    if not has_systemd():
        return
    # disable legacy doveauth-dictproxy.service
    if host.get_fact(SystemdEnabled).get("doveauth-dictproxy.service"):
        systemd.service(
            name="Disable legacy doveauth-dictproxy.service",
            service="doveauth-dictproxy.service",
            running=False,
            enabled=False,
        )
 def _install_remote_venv_with_chatmaild() -> None:
    remove_legacy_artifacts()
    dist_file = _build_chatmaild(dist_dir=Path("chatmaild/dist"))
    remote_base_dir = "/usr/local/lib/chatmaild"
    remote_dist_file = f"{remote_base_dir}/dist/{dist_file.name}"
    remote_venv_dir = f"{remote_base_dir}/venv"
    root_owned = dict(user="root", group="root", mode="644")
    apt.packages(
        name="apt install python3-virtualenv",
        packages=["python3-virtualenv"],
    )
    files.put(
        name="Upload chatmaild source package",
        src=dist_file.open("rb"),
        dest=remote_dist_file,
        create_remote_dir=True,
        **root_owned,
    )
    pip.virtualenv(
        name=f"chatmaild virtualenv {remote_venv_dir}",
        path=remote_venv_dir,
        always_copy=True,
    )
    apt.packages(
        name="install gcc and headers to build crypt_r source package",
        packages=["gcc", "python3-dev"],
    )
    server.shell(
        name=f"forced pip-install {dist_file.name}",
        commands=[
            f"{remote_venv_dir}/bin/pip install --force-reinstall {remote_dist_file}"
        ],
    )
 def _configure_remote_venv_with_chatmaild(config) -> None:
    remote_base_dir = "/usr/local/lib/chatmaild"
    remote_venv_dir = f"{remote_base_dir}/venv"
    remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini"
    root_owned = dict(user="root", group="root", mode="644")
    files.put(
        name=f"Upload {remote_chatmail_inipath}",
        src=config._getbytefile(),
        dest=remote_chatmail_inipath,
        **root_owned,
    )
    files.template(
        src=get_resource("metrics.cron.j2"),
        dest="/etc/cron.d/chatmail-metrics",
        user="root",
        group="root",
        mode="644",
        config={
            "mailboxes_dir": config.mailboxes_dir,
            "execpath": f"{remote_venv_dir}/bin/chatmail-metrics",
        },
    )
 class UnboundDeployer(Deployer):
    def __init__(self, config):
        self.config = config
        self.need_restart = False
    def install(self):
        # Run local DNS resolver `unbound`.
        # `resolvconf` takes care of setting up /etc/resolv.conf
        # to use 127.0.0.1 as the resolver.
        #
        # On an IPv4-only system, if unbound is started but not
        # configured, it causes subsequent steps to fail to resolve hosts.
        # Here, we use policy-rc.d to prevent unbound from starting up
        # on initial install.  Later, we will configure it and start it.
        #
        # For documentation about policy-rc.d, see:
        # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
        #
        files.put(
            src=get_resource("policy-rc.d"),
            dest="/usr/sbin/policy-rc.d",
            user="root",
            group="root",
            mode="755",
        )
        apt.packages(
            name="Install unbound",
            packages=["unbound", "unbound-anchor", "dnsutils"],
        )
        files.file("/usr/sbin/policy-rc.d", present=False)
    def configure(self):
        server.shell(
            name="Generate root keys for validating DNSSEC",
            commands=[
                "unbound-anchor -a /var/lib/unbound/root.key || true",
            ],
        )
        if self.config.disable_ipv6:
            files.directory(
                path="/etc/unbound/unbound.conf.d",
                present=True,
                user="root",
                group="root",
                mode="755",
            )
            conf = files.put(
                src=get_resource("unbound/unbound.conf.j2"),
                dest="/etc/unbound/unbound.conf.d/chatmail.conf",
                user="root",
                group="root",
                mode="644",
            )
        else:
            conf = files.file(
                path="/etc/unbound/unbound.conf.d/chatmail.conf",
                present=False,
            )
        self.need_restart |= conf.changed
    def activate(self):
        server.shell(
            name="Generate root keys for validating DNSSEC",
            commands=[
                "systemctl reset-failed unbound.service",
            ],
        )
        systemd.service(
            name="Start and enable unbound",
            service="unbound.service",
            running=True,
            enabled=True,
            restarted=self.need_restart,
        )
 class MtastsDeployer(Deployer):
    def configure(self):
        # Remove configuration.
        files.file("/etc/mta-sts-daemon.yml", present=False)
        files.directory("/usr/local/lib/postfix-mta-sts-resolver", present=False)
        files.file("/etc/systemd/system/mta-sts-daemon.service", present=False)
    def activate(self):
        systemd.service(
            name="Stop MTA-STS daemon",
            service="mta-sts-daemon.service",
            daemon_reload=True,
            running=False,
            enabled=False,
        )
 class WebsiteDeployer(Deployer):
    def __init__(self, config):
        self.config = config
    def install(self):
        files.directory(
            name="Ensure /var/www exists",
            path="/var/www",
            user="root",
            group="root",
            mode="755",
            present=True,
        )
    def configure(self):
        www_path, src_dir, build_dir = get_paths(self.config)
        # if www_folder was set to a non-existing folder, skip upload
        if not www_path.is_dir():
            logger.warning("Building web pages is disabled in chatmail.ini, skipping")
        elif (path := find_merge_conflict(src_dir)) is not None:
            logger.warning(
                f"Merge conflict found in {path}, skipping website deployment. Fix merge conflict if you want to upload your web page."
            )
        else:
            # if www_folder is a hugo page, build it
            if build_dir:
                www_path = build_webpages(src_dir, build_dir, self.config)
            # if it is not a hugo page, upload it as is
            files.rsync(
                f"{www_path}/", "/var/www/html", flags=["-avz", "--chown=www-data"]
            )
 class LegacyRemoveDeployer(Deployer):
    def install(self):
        apt.packages(name="Remove rspamd", packages="rspamd", present=False)
        # remove historic expunge script
        # which is now implemented through a systemd timer (chatmail-expire)
        files.file(
            path="/etc/cron.d/expunge",
            present=False,
        )
        # Remove OBS repository key that is no longer used.
        files.file("/etc/apt/keyrings/obs-home-deltachat.gpg", present=False)
        files.line(
            name="Remove DeltaChat OBS home repository from sources.list",
            path="/etc/apt/sources.list",
            line="deb [signed-by=/etc/apt/keyrings/obs-home-deltachat.gpg] https://download.opensuse.org/repositories/home:/deltachat/Debian_12/ ./",
            escape_regex_characters=True,
            present=False,
        )
        # prior relay versions used filelogging
        files.directory(
            name="Ensure old logs on disk are deleted",
            path="/var/log/journal/",
            present=False,
        )
        # remove echobot if it is still running
        if has_systemd() and host.get_fact(SystemdEnabled).get("echobot.service"):
            systemd.service(
                name="Disable echobot.service",
                service="echobot.service",
                running=False,
                enabled=False,
            )
 def check_config(config):
    mail_domain = config.mail_domain
    if mail_domain != "testrun.org" and not mail_domain.endswith(".testrun.org"):
        blocked_words = "merlinux schmieder testrun.org".split()
        for key in config.__dict__:
            value = config.__dict__[key]
            if key.startswith("privacy") and any(
                x in str(value) for x in blocked_words
            ):
                raise ValueError(
                    f"please set your own privacy contacts/addresses in {config._inipath}"
                )
    return config
 class TurnDeployer(Deployer):
    def __init__(self, mail_domain):
        self.mail_domain = mail_domain
        self.units = ["turnserver"]
    def install(self):
        (url, sha256sum) = {
            "x86_64": (
                "https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-x86_64-linux",
                "841e527c15fdc2940b0469e206188ea8f0af48533be12ecb8098520f813d41e4",
            ),
            "aarch64": (
                "https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-aarch64-linux",
                "a5fc2d06d937b56a34e098d2cd72a82d3e89967518d159bf246dc69b65e81b42",
            ),
        }[host.get_fact(facts.server.Arch)]
        existing_sha256sum = host.get_fact(Sha256File, "/usr/local/bin/chatmail-turn")
        if existing_sha256sum != sha256sum:
            server.shell(
                name="Download chatmail-turn",
                commands=[
                    f"(curl -L {url} >/usr/local/bin/chatmail-turn.new && (echo '{sha256sum} /usr/local/bin/chatmail-turn.new' | sha256sum -c) && mv /usr/local/bin/chatmail-turn.new /usr/local/bin/chatmail-turn)",
                    "chmod 755 /usr/local/bin/chatmail-turn",
                ],
            )
    def configure(self):
        configure_remote_units(self.mail_domain, self.units)
    def activate(self):
        activate_remote_units(self.units)
 class IrohDeployer(Deployer):
    def __init__(self, enable_iroh_relay):
        self.enable_iroh_relay = enable_iroh_relay
    def install(self):
        (url, sha256sum) = {
            "x86_64": (
                "https://github.com/n0-computer/iroh/releases/download/v0.35.0/iroh-relay-v0.35.0-x86_64-unknown-linux-musl.tar.gz",
                "45c81199dbd70f8c4c30fef7f3b9727ca6e3cea8f2831333eeaf8aa71bf0fac1",
            ),
            "aarch64": (
                "https://github.com/n0-computer/iroh/releases/download/v0.35.0/iroh-relay-v0.35.0-aarch64-unknown-linux-musl.tar.gz",
                "f8ef27631fac213b3ef668d02acd5b3e215292746a3fc71d90c63115446008b1",
            ),
        }[host.get_fact(facts.server.Arch)]
        existing_sha256sum = host.get_fact(Sha256File, "/usr/local/bin/iroh-relay")
        if existing_sha256sum != sha256sum:
            server.shell(
                name="Download iroh-relay",
                commands=[
                    f"(curl -L {url} | gunzip | tar -x -f - ./iroh-relay -O >/usr/local/bin/iroh-relay.new && (echo '{sha256sum} /usr/local/bin/iroh-relay.new' | sha256sum -c) && mv /usr/local/bin/iroh-relay.new /usr/local/bin/iroh-relay)",
                    "chmod 755 /usr/local/bin/iroh-relay",
                ],
            )
            self.need_restart = True
    def configure(self):
        systemd_unit = files.put(
            name="Upload iroh-relay systemd unit",
            src=get_resource("iroh-relay.service"),
            dest="/etc/systemd/system/iroh-relay.service",
            user="root",
            group="root",
            mode="644",
        )
        self.need_restart |= systemd_unit.changed
        iroh_config = files.put(
            name="Upload iroh-relay config",
            src=get_resource("iroh-relay.toml"),
            dest="/etc/iroh-relay.toml",
            user="root",
            group="root",
            mode="644",
        )
        self.need_restart |= iroh_config.changed
    def activate(self):
        systemd.service(
            name="Start and enable iroh-relay",
            service="iroh-relay.service",
            running=True,
            enabled=self.enable_iroh_relay,
            restarted=self.need_restart,
        )
        self.need_restart = False
 class JournaldDeployer(Deployer):
    def configure(self):
        journald_conf = files.put(
            name="Configure journald",
            src=get_resource("journald.conf"),
            dest="/etc/systemd/journald.conf",
            user="root",
            group="root",
            mode="644",
        )
        self.need_restart = journald_conf.changed
    def activate(self):
        systemd.service(
            name="Start and enable journald",
            service="systemd-journald.service",
            running=True,
            enabled=True,
            restarted=self.need_restart,
        )
        self.need_restart = False
 class ChatmailVenvDeployer(Deployer):
    def __init__(self, config):
        self.config = config
        self.units = (
            "chatmail-metadata",
            "lastlogin",
            "chatmail-expire",
            "chatmail-expire.timer",
            "chatmail-fsreport",
            "chatmail-fsreport.timer",
        )
    def install(self):
        _install_remote_venv_with_chatmaild()
    def configure(self):
        _configure_remote_venv_with_chatmaild(self.config)
        configure_remote_units(self.config.mail_domain, self.units)
    def activate(self):
        activate_remote_units(self.units)
 class ChatmailDeployer(Deployer):
    required_users = [
        ("vmail", "vmail", None),
        ("iroh", None, None),
    ]
    def __init__(self, mail_domain):
        self.mail_domain = mail_domain
    def install(self):
        apt.update(name="apt update", cache_time=24 * 3600)
        apt.upgrade(name="upgrade apt packages", auto_remove=True)
        apt.packages(
            name="Install curl",
            packages=["curl"],
        )
        apt.packages(
            name="Install rsync",
            packages=["rsync"],
        )
        apt.packages(
            name="Ensure cron is installed",
            packages=["cron"],
        )
    def configure(self):
        # This file is used by auth proxy.
        # https://wiki.debian.org/EtcMailName
        server.shell(
            name="Setup /etc/mailname",
            commands=[
                f"echo {self.mail_domain} >/etc/mailname; chmod 644 /etc/mailname"
            ],
        )
 class FcgiwrapDeployer(Deployer):
    def install(self):
        apt.packages(
            name="Install fcgiwrap",
            packages=["fcgiwrap"],
        )
    def activate(self):
        systemd.service(
            name="Start and enable fcgiwrap",
            service="fcgiwrap.service",
            running=True,
            enabled=True,
        )
 class GithashDeployer(Deployer):
    def activate(self):
        try:
            git_hash = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode()
        except Exception:
            git_hash = "unknown\n"
        try:
            git_diff = subprocess.check_output(["git", "diff"]).decode()
        except Exception:
            git_diff = ""
        files.put(
            name="Upload chatmail relay git commit hash",
            src=StringIO(git_hash + git_diff),
            dest="/etc/chatmail-version",
            mode="700",
        )
 def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -> None:
    """Deploy a chat-mail instance.
    :param config_path: path to chatmail.ini
    :param disable_mail: whether to disable postfix & dovecot
    :param website_only: if True, only deploy the website
    """
    config = read_config(config_path)
    check_config(config)
    mail_domain = config.mail_domain
    if website_only:
        Deployment().perform_stages([WebsiteDeployer(config)])
        return
    if host.get_fact(Port, port=53) != "unbound":
        files.line(
            name="Add 9.9.9.9 to resolv.conf",
            path="/etc/resolv.conf",
            # Guard against resolv.conf missing a trailing newline (SolusVM bug).
            line="\nnameserver 9.9.9.9",
        )
    # Check if mtail_address interface is available (if configured)
    if config.mtail_address and config.mtail_address not in ('127.0.0.1', '::1', 'localhost'):
        ipv4_addrs = host.get_fact(hardware.Ipv4Addrs)
        all_addresses = [addr for addrs in ipv4_addrs.values() for addr in addrs]
        if config.mtail_address not in all_addresses:
            Out().red(f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n")
            exit(1)
    if not os.environ.get("CHATMAIL_NOPORTCHECK"):
        port_services = [
            (["master", "smtpd"], 25),
            ("unbound", 53),
            ("acmetool", 80),
            (["imap-login", "dovecot"], 143),
            ("nginx", 443),
            (["master", "smtpd"], 465),
            (["master", "smtpd"], 587),
            (["imap-login", "dovecot"], 993),
            ("iroh-relay", 3340),
            ("mtail", 3903),
            ("stats", 3904),
            ("nginx", 8443),
            (["master", "smtpd"], config.postfix_reinject_port),
            (["master", "smtpd"], config.postfix_reinject_port_incoming),
            ("filtermail", config.filtermail_smtp_port),
            ("filtermail", config.filtermail_smtp_port_incoming),
        ]
        for service, port in port_services:
            print(f"Checking if port {port} is available for {service}...")
            running_service = host.get_fact(Port, port=port)
            services = [service] if isinstance(service, str) else service
            if running_service:
                if running_service not in services:
                    Out().red(
                        f"Deploy failed: port {port} is occupied by: {running_service}"
                    )
                    exit(1)
    tls_domains = [mail_domain, f"mta-sts.{mail_domain}", f"www.{mail_domain}"]
    all_deployers = [
        ChatmailDeployer(mail_domain),
        LegacyRemoveDeployer(),
        FiltermailDeployer(),
        JournaldDeployer(),
        UnboundDeployer(config),
        TurnDeployer(mail_domain),
        IrohDeployer(config.enable_iroh_relay),
    ]
    if not config.noacme:
        all_deployers.append(AcmetoolDeployer(config.acme_email, tls_domains))
    all_deployers += [
        WebsiteDeployer(config),
        ChatmailVenvDeployer(config),
        MtastsDeployer(),
        OpendkimDeployer(mail_domain),
        # Dovecot should be started before Postfix
        # because it creates authentication socket
        # required by Postfix.
        DovecotDeployer(config, disable_mail),
        PostfixDeployer(config, disable_mail),
        FcgiwrapDeployer(),
        NginxDeployer(config),
        MtailDeployer(config.mtail_address),
        GithashDeployer(),
    ]
    Deployment().perform_stages(all_deployers)
--- a/cmdeploy/src/cmdeploy/dns.py
+++ b/cmdeploy/src/cmdeploy/dns.py
@@ -1,71 +0,0 @@
 import datetime
 import importlib
 from jinja2 import Template
 from . import remote
 def get_initial_remote_data(sshexec, mail_domain):
    return sshexec.logged(
        call=remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=mail_domain)
    )
 def check_initial_remote_data(remote_data, *, print=print):
    mail_domain = remote_data["mail_domain"]
    if not remote_data["A"] and not remote_data["AAAA"]:
        print(f"Missing A and/or AAAA DNS records for {mail_domain}!")
    elif remote_data["MTA_STS"] != f"{mail_domain}.":
        print("Missing MTA-STS CNAME record:")
        print(f"mta-sts.{mail_domain}.   CNAME  {mail_domain}.")
    elif remote_data["WWW"] != f"{mail_domain}.":
        print("Missing www CNAME record:")
        print(f"www.{mail_domain}.   CNAME  {mail_domain}.")
    else:
        return remote_data
 def get_filled_zone_file(remote_data):
    sts_id = remote_data.get("sts_id")
    if not sts_id:
        remote_data["sts_id"] = datetime.datetime.now().strftime("%Y%m%d%H%M")
    template = importlib.resources.files(__package__).joinpath("chatmail.zone.j2")
    content = template.read_text()
    zonefile = Template(content).render(**remote_data)
    lines = [x.strip() for x in zonefile.split("\n") if x.strip()]
    lines.append("")
    zonefile = "\n".join(lines)
    return zonefile
 def check_full_zone(sshexec, remote_data, out, zonefile) -> int:
    """Check existing DNS records, optionally write them to zone file
    and return (exitcode, remote_data) tuple."""
    required_diff, recommended_diff = sshexec.logged(
        remote.rdns.check_zonefile,
        kwargs=dict(zonefile=zonefile, verbose=False),
    )
    returncode = 0
    if required_diff:
        out.red("Please set required DNS entries at your DNS provider:\n")
        for line in required_diff:
            out(line)
        out("")
        returncode = 1
    if remote_data.get("dkim_entry") in required_diff:
        out(
            "If the DKIM entry above does not work with your DNS provider, you can try this one:\n"
        )
        out(remote_data.get("web_dkim_entry") + "\n")
    if recommended_diff:
        out("WARNING: these recommended DNS entries are not set:\n")
        for line in recommended_diff:
            out(line)
    if not (recommended_diff or required_diff):
        out.green("Great! All your DNS entries are verified and correct.")
    return returncode
--- a/cmdeploy/src/cmdeploy/dovecot/auth.conf
+++ b/cmdeploy/src/cmdeploy/dovecot/auth.conf
@@ -1,12 +0,0 @@
 uri = proxy:/run/doveauth/doveauth.socket:auth
 iterate_disable = no
 iterate_prefix = userdb/
 default_pass_scheme = plain
 # %E escapes characters " (double quote), ' (single quote) and \ (backslash) with \ (backslash).
 # See <https://doc.dovecot.org/2.3/configuration_manual/config_file/config_variables/#modifiers>
 # for documentation.
 #
 # We escape user-provided input and use double quote as a separator.
 password_key = passdb/%Ew"%Eu
 user_key = userdb/%Eu
--- a/cmdeploy/src/cmdeploy/dovecot/deployer.py
+++ b/cmdeploy/src/cmdeploy/dovecot/deployer.py
@@ -1,158 +0,0 @@
 import os
 from chatmaild.config import Config
 from pyinfra import host
 from pyinfra.facts.server import Arch, Sysctl
 from pyinfra.facts.systemd import SystemdEnabled
 from pyinfra.operations import apt, files, server, systemd
 from cmdeploy.basedeploy import (
    Deployer,
    activate_remote_units,
    configure_remote_units,
    get_resource,
    has_systemd,
 )
 class DovecotDeployer(Deployer):
    daemon_reload = False
    def __init__(self, config, disable_mail):
        self.config = config
        self.disable_mail = disable_mail
        self.units = ["doveauth"]
    def install(self):
        arch = host.get_fact(Arch)
        if has_systemd() and "dovecot.service" in host.get_fact(SystemdEnabled):
            return  # already installed and running
        _install_dovecot_package("core", arch)
        _install_dovecot_package("imapd", arch)
        _install_dovecot_package("lmtpd", arch)
    def configure(self):
        configure_remote_units(self.config.mail_domain, self.units)
        self.need_restart, self.daemon_reload = _configure_dovecot(self.config)
    def activate(self):
        activate_remote_units(self.units)
        restart = False if self.disable_mail else self.need_restart
        systemd.service(
            name="Disable dovecot for now" if self.disable_mail else "Start and enable Dovecot",
            service="dovecot.service",
            running=False if self.disable_mail else True,
            enabled=False if self.disable_mail else True,
            restarted=restart,
            daemon_reload=self.daemon_reload,
        )
        self.need_restart = False
 def _install_dovecot_package(package: str, arch: str):
    arch = "amd64" if arch == "x86_64" else arch
    arch = "arm64" if arch == "aarch64" else arch
    url = f"https://download.delta.chat/dovecot/dovecot-{package}_2.3.21%2Bdfsg1-3_{arch}.deb"
    deb_filename = "/root/" + url.split("/")[-1]
    match (package, arch):
        case ("core", "amd64"):
            sha256 = "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d"
        case ("core", "arm64"):
            sha256 = "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9"
        case ("imapd", "amd64"):
            sha256 = "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86"
        case ("imapd", "arm64"):
            sha256 = "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f"
        case ("lmtpd", "amd64"):
            sha256 = "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab"
        case ("lmtpd", "arm64"):
            sha256 = "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f"
        case _:
            apt.packages(packages=[f"dovecot-{package}"])
            return
    files.download(
        name=f"Download dovecot-{package}",
        src=url,
        dest=deb_filename,
        sha256sum=sha256,
        cache_time=60 * 60 * 24 * 365 * 10,  # never redownload the package
    )
    apt.deb(name=f"Install dovecot-{package}", src=deb_filename)
 def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
    """Configures Dovecot IMAP server."""
    need_restart = False
    daemon_reload = False
    main_config = files.template(
        src=get_resource("dovecot/dovecot.conf.j2"),
        dest="/etc/dovecot/dovecot.conf",
        user="root",
        group="root",
        mode="644",
        config=config,
        debug=debug,
        disable_ipv6=config.disable_ipv6,
    )
    need_restart |= main_config.changed
    auth_config = files.put(
        src=get_resource("dovecot/auth.conf"),
        dest="/etc/dovecot/auth.conf",
        user="root",
        group="root",
        mode="644",
    )
    need_restart |= auth_config.changed
    lua_push_notification_script = files.put(
        src=get_resource("dovecot/push_notification.lua"),
        dest="/etc/dovecot/push_notification.lua",
        user="root",
        group="root",
        mode="644",
    )
    need_restart |= lua_push_notification_script.changed
    # as per https://doc.dovecot.org/2.3/configuration_manual/os/
    # it is recommended to set the following inotify limits
    if not os.environ.get("CHATMAIL_NOSYSCTL"):
        for name in ("max_user_instances", "max_user_watches"):
            key = f"fs.inotify.{name}"
            if host.get_fact(Sysctl)[key] > 65535:
                # Skip updating limits if already sufficient
                # (enables running in incus containers where sysctl readonly)
                continue
            server.sysctl(
                name=f"Change {key}",
                key=key,
                value=65535,
                persist=True,
            )
    timezone_env = files.line(
        name="Set TZ environment variable",
        path="/etc/environment",
        line="TZ=:/etc/localtime",
    )
    need_restart |= timezone_env.changed
    restart_conf = files.put(
        name="dovecot: restart automatically on failure",
        src=get_resource("service/10_restart.conf"),
        dest="/etc/systemd/system/dovecot.service.d/10_restart.conf",
    )
    daemon_reload |= restart_conf.changed
    # Validate dovecot configuration before restart
    if need_restart:
        server.shell(
            name="Validate dovecot configuration",
            commands=["doveconf -n >/dev/null"],
        )
    return need_restart, daemon_reload
--- a/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2
+++ b/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2
@@ -1,434 +0,0 @@
 ## Dovecot configuration file
 {% if disable_ipv6 %}
 listen = 0.0.0.0
 {% endif %}
 protocols = imap lmtp
 auth_mechanisms = plain
 {% if debug == true %}
 auth_verbose = yes
 auth_debug = yes
 auth_debug_passwords = yes
 auth_verbose_passwords = plain
 auth_cache_size = 100M
 mail_debug = yes
 {% endif %}
 # Prevent warnings similar to:
 #   config: Warning: service auth { client_limit=1000 } is lower than required under max. load (10200). Counted for protocol services with service_count != 1: service lmtp { process_limit=100 } + service imap-urlauth-login { process_limit=100 } + service imap-login { process_limit=10000 }
 #   config: Warning: service anvil { client_limit=1000 } is lower than required under max. load (10103). Counted with: service imap-urlauth-login { process_limit=100 } + service imap-login { process_limit=10000 } + service auth { process_limit=1 }
 #   master: Warning: service(stats): client_limit (1000) reached, client connections are being dropped
 default_client_limit = 20000
 # Increase number of logged in IMAP connections.
 # Each connection is handled by a separate `imap` process.
 # `imap` process should have `client_limit=1` as described in
 # <https://doc.dovecot.org/2.3/configuration_manual/service_configuration/#service-limits>
 # so each logged in IMAP session will need its own `imap` process.
 #
 # If this limit is reached,
 # users will fail to LOGIN as `imap-login` process
 # will accept them logging in but fail to transfer logged in
 # connection to `imap` process until someone logs out and
 # the following warning will be logged:
 #   Warning: service(imap): process_limit (1024) reached, client connections are being dropped
 service imap {
  process_limit = 50000
 }
 mail_server_admin = mailto:root@{{ config.mail_domain }}
 mail_server_comment = Chatmail server
 # `zlib` enables compressing messages stored in the maildir.
 # See
 # <https://doc.dovecot.org/2.3/configuration_manual/zlib_plugin/>
 # for documentation.
 #
 # quota plugin documentation:
 # <https://doc.dovecot.org/2.3/configuration_manual/quota_plugin/>
 mail_plugins = zlib quota
 imap_capability = +XDELTAPUSH XCHATMAIL
 # Authentication for system users.
 passdb {
  driver = dict
  args = /etc/dovecot/auth.conf
 }
 userdb {
  driver = dict
  args = /etc/dovecot/auth.conf
 }
 ##
 ## Mailbox locations and namespaces
 ##
 # Mailboxes are stored in the "mail" directory of the vmail user home.
 mail_location = maildir:{{ config.mailboxes_dir }}/%u
 # index/cache files are not very useful for chatmail relay operations 
 # but it's not clear how to disable them completely. 
 # According to https://doc.dovecot.org/2.3/settings/advanced/#core_setting-mail_cache_max_size
 # if the cache file becomes larger than the specified size, it is truncated by dovecot 
 mail_cache_max_size = 500K
 namespace inbox {
  inbox = yes
  mailbox Drafts {
    special_use = \Drafts
  }
  mailbox Junk {
    special_use = \Junk
  }
  mailbox Trash {
    special_use = \Trash
  }
  # For \Sent mailboxes there are two widely used names. We'll mark both of
  # them as \Sent. User typically deletes one of them if duplicates are created.
  mailbox Sent {
    special_use = \Sent
  }
  mailbox "Sent Messages" {
    special_use = \Sent
  }
 }
 mail_uid = vmail
 mail_gid = vmail
 mail_privileged_group = vmail
 ##
 ## Mail processes
 ##
 # Pass all IMAP METADATA requests to the server implementing Dovecot's dict protocol.
 mail_attribute_dict = proxy:/run/chatmail-metadata/metadata.socket:metadata
 # `imap_zlib` enables IMAP COMPRESS (RFC 4978).
 # <https://datatracker.ietf.org/doc/html/rfc4978.html>
 protocol imap {
  mail_plugins = $mail_plugins imap_quota last_login {% if config.imap_compress %}imap_zlib{% endif %}
  imap_metadata = yes
 }
 plugin {
  last_login_dict = proxy:/run/chatmail-lastlogin/lastlogin.socket:lastlogin
  #last_login_key = last-login/%u # default
  last_login_precision = s
 }
 protocol lmtp {
  # notify plugin is a dependency of push_notification plugin:
  # <https://doc.dovecot.org/2.3/settings/plugin/notify-plugin/>
  #
  # push_notification plugin documentation:
  # <https://doc.dovecot.org/2.3/configuration_manual/push_notification/>
  #
  # mail_lua and push_notification_lua are needed for Lua push notification handler.
  # <https://doc.dovecot.org/2.3/configuration_manual/push_notification/#configuration>
  mail_plugins = $mail_plugins mail_lua notify push_notification push_notification_lua
 }
 plugin {
  zlib_save = gz
 }
 plugin {
  imap_compress_deflate_level = 6
 }
 plugin {
  # for now we define static quota-rules for all users 
  quota = maildir:User quota
  quota_rule = *:storage={{ config.max_mailbox_size }}
  quota_max_mail_size={{ config.max_message_size }}
  quota_grace = 0
  # quota_over_flag_value = TRUE
 }
 # push_notification configuration
 plugin {
  # <https://doc.dovecot.org/2.3/configuration_manual/push_notification/#lua-lua>
  push_notification_driver = lua:file=/etc/dovecot/push_notification.lua
 }
 service lmtp {
  user=vmail
  unix_listener /var/spool/postfix/private/dovecot-lmtp {
    group = postfix
    mode = 0600
    user = postfix
  }
 }
 lmtp_add_received_header = no
 service auth {
  unix_listener /var/spool/postfix/private/auth {
    mode = 0660
    user = postfix
    group = postfix
  }
 }
 service auth-worker {
  # Default is root.
  # Drop privileges we don't need.
  user = vmail
 }
 service imap-login {
    # High-performance mode as described in
    # <https://doc.dovecot.org/2.3/admin_manual/login_processes/#high-performance-mode>
    #
    # So-called high-security mode described in
    # <https://doc.dovecot.org/2.3/admin_manual/login_processes/#high-security-mode>
    # and enabled by default with `service_count = 1` starts one process per connection
    # and has problems logging in thousands of users after Dovecot restart.
    service_count = 0
    # Increase virtual memory size limit.
    # Since imap-login processes handle TLS connections
    # even after logging users in
    # and many connections are handled by each process,
    # memory size limit should be increased.
    #
    # Otherwise the whole process eventually dies
    # with an error similar to
    #   imap-login: Fatal: master: service(imap-login):
    #   child 1422951 returned error 83
    #   (Out of memory (service imap-login { vsz_limit=256 MB },
    #    you may need to increase it)
    # and takes down all its TLS connections at once.
    vsz_limit = 1G
    # Avoid startup latency for new connections.
    #
    # Should be set to at least the number of CPU cores
    # according to the documentation.
    process_min_avail = 10
 }
 service anvil {
    # We are disabling anvil penalty on failed login attempts
    # because it can only detect brute forcing by IP address
    # not by username. As the correct IP address is not handed
    # to dovecot anyway, it is more of hindrance than of use.
    # See <https://www.dovecot.org/list/dovecot/2012-May/135485.html> for details.
    unix_listener anvil-auth-penalty {
        mode = 0
    }
 }
 ssl = required
 ssl_cert = </var/lib/acme/live/{{ config.mail_domain }}/fullchain
 ssl_key = </var/lib/acme/live/{{ config.mail_domain }}/privkey
 ssl_dh = </usr/share/dovecot/dh.pem
 ssl_min_protocol = TLSv1.3
 ssl_prefer_server_ciphers = yes
 {% if config.imap_rawlog %}
 service postlogin {
  executable = script-login -d rawlog
  unix_listener postlogin {
 }  
 } 
 service imap {
  executable = imap postlogin 
 } 
 protocol imap { 
  #rawlog_dir = /tmp/rawlog/%u
  # Put .in and .out imap protocol logging files into per-user homedir 
  # You can use a command like this to combine into one protocol stream:
  # sort -sn <(sed 's/ / C: /' *.in) <(sed 's/ / S: /' cat *.out)
  rawlog_dir = %h 
 } 
 {% endif %}
 {% if not config.imap_compress %}
 # Hibernate IDLE users to save memory and CPU resources
 # NOTE: this will have no effect if imap_zlib plugin is used
 imap_hibernate_timeout = 30s
 service imap {
  # Note that this change will allow any process running as
  # $default_internal_user (dovecot) to access mails as any other user.
  # This may be insecure in some installations, which is why this isn't
  # done by default.
  unix_listener imap-master {
    user = $default_internal_user
  }
 }
 # The following is the default already in v2.3.1+:
 service imap {
  extra_groups = $default_internal_group
 }
 service imap-hibernate {
  unix_listener imap-hibernate {
    mode = 0660
    group = $default_internal_group
  }
 }
 {% endif %}
 {% if config.mtail_address %}
 #
 # Dovecot Statistics
 #
 # OpenMetrics endpoint at http://{{- config.mtail_address}}:3904/metrics
 service stats {
  inet_listener http {
    port = 3904
    address = {{- config.mtail_address}}
  }
 }
 # IMAP Command Metrics
 # - Bytes in/out for compression efficiency analysis
 # - Lock wait time for contention debugging
 # - Grouped by command name and reply state
 metric imap_command {
  filter = event=imap_command_finished
  fields = bytes_in bytes_out lock_wait_usecs running_usecs
  group_by = cmd_name tagged_reply_state
 }
 # Duration buckets for latency histograms (base 10: 10us, 100us, 1ms, 10ms, 100ms, 1s, 10s, 100s)
 metric imap_command_duration {
  filter = event=imap_command_finished
  group_by = cmd_name duration:exponential:1:8:10
 }
 # Slow command outliers (>1 second = 1000000 usecs)
 # Useful for alerting without high cardinality
 metric imap_command_slow {
  filter = event=imap_command_finished AND duration>1000000 AND NOT cmd_name=IDLE
  group_by = cmd_name
 }
 # IDLE-specific Metrics
 metric imap_idle {
  filter = event=imap_command_finished AND cmd_name=IDLE
  fields = bytes_in bytes_out running_usecs
  group_by = tagged_reply_state
 }
 metric imap_idle_duration {
  filter = event=imap_command_finished AND cmd_name=IDLE
  # Base 10: 100ms to 27h (covers short wakeups to long idle sessions)
  group_by = duration:exponential:5:11:10
 }
 metric imap_idle_commands {
  filter = event=imap_command_finished AND cmd_name=IDLE
  group_by = tagged_reply_state
 }
 metric imap_idle_failed {
  filter = event=imap_command_finished AND cmd_name=IDLE AND NOT tagged_reply_state=OK
 }
 # Hibernation Metrics (requires imap_hibernate_timeout)
 metric imap_hibernated {
  filter = event=imap_client_hibernated
 }
 metric imap_hibernated_failed {
  filter = event=imap_client_hibernated AND error=*
 }
 metric imap_unhibernated {
  filter = event=imap_client_unhibernated
  fields = hibernation_usecs
 }
 metric imap_unhibernated_reason {
  filter = event=imap_client_unhibernated
  group_by = reason
  fields = hibernation_usecs
 }
 metric imap_unhibernated_reason_sleep {
  filter = event=imap_client_unhibernated
  group_by = reason hibernation_usecs:exponential:4:8:10
 }
 metric imap_unhibernated_failed {
  filter = event=imap_client_unhibernated AND error=*
 }
 # Hibernation duration buckets (how long clients stayed hibernated)
 # Base 10: 100ms to 27h
 metric imap_hibernation_duration {
  filter = event=imap_client_unhibernated
  group_by = reason duration:exponential:5:11:10
 }
 # Authentication / Login Metrics
 metric auth_request {
  filter = event=auth_request_finished
  group_by = success
 }
 metric auth_request_duration {
  filter = event=auth_request_finished
  group_by = success duration:exponential:2:6:10
 }
 metric auth_failed {
  filter = event=auth_request_finished AND success=no
 }
 # Passdb cache effectiveness
 metric auth_passdb {
  filter = event=auth_passdb_request_finished
  group_by = result cache
 }
 # Master login (post-auth userdb lookup)
 metric auth_master_login {
  filter = event=auth_master_client_login_finished
 }
 metric auth_master_login_failed {
  filter = event=auth_master_client_login_finished AND error=*
 }
 # Mail Delivery (LMTP) - affects IDLE wakeup latency
 metric mail_delivery {
  filter = event=mail_delivery_finished
 }
 metric mail_delivery_duration {
  filter = event=mail_delivery_finished
  group_by = duration:exponential:3:7:10
 }
 metric mail_delivery_failed {
  filter = event=mail_delivery_finished AND error=*
 }
 # Connection Events
 metric client_connected {
  filter = event=client_connection_connected AND category="service:imap"
 }
 metric client_disconnected {
  filter = event=client_connection_disconnected AND category="service:imap"
  fields = bytes_in bytes_out
 }
 {% endif %}
--- a/cmdeploy/src/cmdeploy/dovecot/push_notification.lua
+++ b/cmdeploy/src/cmdeploy/dovecot/push_notification.lua
@@ -1,19 +0,0 @@
 function dovecot_lua_notify_begin_txn(user)
  return user
 end
 function dovecot_lua_notify_event_message_new(user, event)
  local mbox = user:mailbox(event.mailbox)
  mbox:sync()
  if user.username ~= event.from_address then
    -- Incoming message
    -- Notify METADATA server about new message.
    mbox:metadata_set("/private/messagenew", "")
  end
  mbox:free()
 end
 function dovecot_lua_notify_end_txn(ctx, success)
 end
--- a/cmdeploy/src/cmdeploy/filtermail/deployer.py
+++ b/cmdeploy/src/cmdeploy/filtermail/deployer.py
@@ -1,52 +0,0 @@
 from pyinfra import facts, host
 from pyinfra.operations import files, systemd
 from cmdeploy.basedeploy import Deployer, get_resource
 class FiltermailDeployer(Deployer):
    services = ["filtermail", "filtermail-incoming"]
    bin_path = "/usr/local/bin/filtermail"
    config_path = "/usr/local/lib/chatmaild/chatmail.ini"
    def __init__(self):
        self.need_restart = False
    def install(self):
        arch = host.get_fact(facts.server.Arch)
        url = f"https://github.com/chatmail/filtermail/releases/download/v0.3.0/filtermail-{arch}"
        sha256sum = {
            "x86_64": "f14a31323ae2dad3b59d3fdafcde507521da2f951a9478cd1f2fe2b4463df71d",
            "aarch64": "933770d75046c4fd7084ce8d43f905f8748333426ad839154f0fc654755ef09f",
        }[arch]
        self.need_restart |= files.download(
            name="Download filtermail",
            src=url,
            sha256sum=sha256sum,
            dest=self.bin_path,
            mode="755",
        ).changed
    def configure(self):
        for service in self.services:
            self.need_restart |= files.template(
                src=get_resource(f"filtermail/{service}.service.j2"),
                dest=f"/etc/systemd/system/{service}.service",
                user="root",
                group="root",
                mode="644",
                bin_path=self.bin_path,
                config_path=self.config_path,
            ).changed
    def activate(self):
        for service in self.services:
            systemd.service(
                name=f"Start and enable {service}",
                service=f"{service}.service",
                running=True,
                enabled=True,
                restarted=self.need_restart,
                daemon_reload=True,
            )
        self.need_restart = False
--- a/cmdeploy/src/cmdeploy/filtermail/filtermail-incoming.service.j2
+++ b/cmdeploy/src/cmdeploy/filtermail/filtermail-incoming.service.j2
@@ -1,11 +0,0 @@
 [Unit]
 Description=Incoming Chatmail Postfix before queue filter 
 [Service]
 ExecStart={{ bin_path }} {{ config_path }} incoming
 Restart=always
 RestartSec=30
 User=vmail
 [Install]
 WantedBy=multi-user.target
--- a/cmdeploy/src/cmdeploy/filtermail/filtermail.service.j2
+++ b/cmdeploy/src/cmdeploy/filtermail/filtermail.service.j2
@@ -1,11 +0,0 @@
 [Unit]
 Description=Outgoing Chatmail Postfix before queue filter
 [Service]
 ExecStart={{ bin_path }} {{ config_path }} outgoing
 Restart=always
 RestartSec=30
 User=vmail
 [Install]
 WantedBy=multi-user.target
--- a/cmdeploy/src/cmdeploy/genqr.py
+++ b/cmdeploy/src/cmdeploy/genqr.py
@@ -1,88 +0,0 @@
 import importlib
 import io
 import os
 import qrcode
 from PIL import Image, ImageDraw, ImageFont
 def gen_qr_png_data(maildomain):
    url = f"DCACCOUNT:https://{maildomain}/new"
    image = gen_qr(maildomain, url)
    temp = io.BytesIO()
    image.save(temp, format="png")
    temp.seek(0)
    return temp
 def gen_qr(maildomain, url):
    # taken and modified from
    # https://github.com/deltachat/mailadm/blob/master/src/mailadm/gen_qr.py
    # info = f"{maildomain} invite code"
    info = ""
    # load QR code
    qr = qrcode.QRCode(
        version=1,
        error_correction=qrcode.constants.ERROR_CORRECT_H,
        box_size=1,
        border=1,
    )
    qr.add_data(url)
    qr.make(fit=True)
    qr_img = qr.make_image(fill_color="black", back_color="white")
    # paint all elements
    ttf_path = str(
        importlib.resources.files(__package__).joinpath("data/opensans-regular.ttf")
    )
    logo_red_path = str(
        importlib.resources.files(__package__).joinpath("data/delta-chat-bw.png")
    )
    assert os.path.exists(ttf_path), ttf_path
    font_size = 16
    font = ImageFont.truetype(font=ttf_path, size=font_size)
    num_lines = ((info).count("\n") + 1) if info else 0
    size = width = 384
    qr_padding = 6
    text_height = font_size * num_lines
    height = size + text_height
    image = Image.new("RGBA", (width, height), "white")
    qr_final_size = width - (qr_padding * 2)
    if num_lines:
        draw = ImageDraw.Draw(image)
        # draw text
        if hasattr(font, "getsize"):
            info_pos = (width - font.getsize(info.strip())[0]) // 2
        else:
            info_pos = (width - font.getbbox(info.strip())[3]) // 2
        draw.multiline_text(
            (info_pos, size - qr_padding // 2),
            info,
            font=font,
            fill="black",
            align="right",
        )
    # paste QR code
    image.paste(
        qr_img.resize((qr_final_size, qr_final_size), resample=Image.NEAREST),
        (qr_padding, qr_padding),
    )
    # background delta logo
    logo2_img = Image.open(logo_red_path)
    logo2_width = int(size / 6)
    logo2 = logo2_img.resize((logo2_width, logo2_width), resample=Image.NEAREST)
    pos = int((size / 2) - (logo2_width / 2))
    image.paste(logo2, (pos, pos), mask=logo2)
    return image
--- a/cmdeploy/src/cmdeploy/iroh-relay.service
+++ b/cmdeploy/src/cmdeploy/iroh-relay.service
@@ -1,12 +0,0 @@
 [Unit]
 Description=Iroh relay
 [Service]
 ExecStart=/usr/local/bin/iroh-relay --config-path /etc/iroh-relay.toml
 Restart=on-failure
 RestartSec=5s
 User=iroh
 Group=iroh
 [Install]
 WantedBy=multi-user.target
--- a/cmdeploy/src/cmdeploy/iroh-relay.toml
+++ b/cmdeploy/src/cmdeploy/iroh-relay.toml
@@ -1,11 +0,0 @@
 enable_relay = true
 http_bind_addr = "[::]:3340"
 # Disable built-in STUN server in iroh-relay 0.35
 # as we deploy our own TURN server instead.
 # STUN server is going to be removed in iroh-relay 1.0
 # and this line can be removed after upgrade.
 enable_stun = false
 enable_metrics = false
 metrics_bind_addr = "127.0.0.1:9092"
--- a/cmdeploy/src/cmdeploy/metrics.cron.j2
+++ b/cmdeploy/src/cmdeploy/metrics.cron.j2
@@ -1 +0,0 @@
 */5 * * * * root {{ config.execpath }} {{ config.mailboxes_dir }} >/var/www/html/metrics
--- a/cmdeploy/src/cmdeploy/mtail/delivered_mail.mtail
+++ b/cmdeploy/src/cmdeploy/mtail/delivered_mail.mtail
@@ -1,80 +0,0 @@
 counter delivered_mail
 /saved mail to INBOX$/ {
  delivered_mail++
 }
 counter quota_exceeded
 /Quota exceeded \(mailbox for user is full\)$/ {
  quota_exceeded++
 }
 # Essentially the number of outgoing messages.
 counter dkim_signed
 /DKIM-Signature field added/ {
  dkim_signed++
 }
 counter created_accounts
 counter created_ci_accounts
 counter created_nonci_accounts
 /: Created address: (?P<addr>.*)$/ {
  created_accounts++
  $addr =~ /ci-/ {
    created_ci_accounts++
  } else {
    created_nonci_accounts++
  }
 }
 counter postfix_timeouts
 /timeout after DATA/ {
  postfix_timeouts++
 }
 counter postfix_noqueue
 /postfix\/.*NOQUEUE/ {
  postfix_noqueue++
 }
 counter warning_count
 /warning/ {
  warning_count++
 }
 counter filtered_outgoing_mail_count
 counter outgoing_encrypted_mail_count
 /Outgoing: Filtering encrypted mail\./ {
  outgoing_encrypted_mail_count++
  filtered_outgoing_mail_count++
 }
 counter outgoing_unencrypted_mail_count
 /Outgoing: Filtering unencrypted mail\./ {
  outgoing_unencrypted_mail_count++
  filtered_outgoing_mail_count++
 }
 counter filtered_incoming_mail_count
 counter incoming_encrypted_mail_count
 /Incoming: Filtering encrypted mail\./ {
  incoming_encrypted_mail_count++
  filtered_incoming_mail_count++
 }
 counter incoming_unencrypted_mail_count
 /Incoming: Filtering unencrypted mail\./ {
  incoming_unencrypted_mail_count++
  filtered_incoming_mail_count++
 }
 counter rejected_unencrypted_mail_count
 /Rejected unencrypted mail/ {
  rejected_unencrypted_mail_count++
 }
--- a/cmdeploy/src/cmdeploy/mtail/deployer.py
+++ b/cmdeploy/src/cmdeploy/mtail/deployer.py
@@ -1,68 +0,0 @@
 from pyinfra import facts, host
 from pyinfra.operations import apt, files, server, systemd
 from cmdeploy.basedeploy import (
    Deployer,
    get_resource,
 )
 class MtailDeployer(Deployer):
    def __init__(self, mtail_address):
        self.mtail_address = mtail_address
    def install(self):
        # Uninstall mtail package to install a static binary.
        apt.packages(name="Uninstall mtail", packages=["mtail"], present=False)
        (url, sha256sum) = {
            "x86_64": (
                "https://github.com/google/mtail/releases/download/v3.0.8/mtail_3.0.8_linux_amd64.tar.gz",
                "123c2ee5f48c3eff12ebccee38befd2233d715da736000ccde49e3d5607724e4",
            ),
            "aarch64": (
                "https://github.com/google/mtail/releases/download/v3.0.8/mtail_3.0.8_linux_arm64.tar.gz",
                "aa04811c0929b6754408676de520e050c45dddeb3401881888a092c9aea89cae",
            ),
        }[host.get_fact(facts.server.Arch)]
        server.shell(
            name="Download mtail",
            commands=[
                f"(echo '{sha256sum} /usr/local/bin/mtail' | sha256sum -c) || (curl -L {url} | gunzip | tar -x -f - mtail -O >/usr/local/bin/mtail.new && mv /usr/local/bin/mtail.new /usr/local/bin/mtail)",
                "chmod 755 /usr/local/bin/mtail",
            ],
        )
    def configure(self):
        # Using our own systemd unit instead of `/usr/lib/systemd/system/mtail.service`.
        # This allows to read from journalctl instead of log files.
        files.template(
            src=get_resource("mtail/mtail.service.j2"),
            dest="/etc/systemd/system/mtail.service",
            user="root",
            group="root",
            mode="644",
            address=self.mtail_address or "127.0.0.1",
            port=3903,
        )
        mtail_conf = files.put(
            name="Mtail configuration",
            src=get_resource("mtail/delivered_mail.mtail"),
            dest="/etc/mtail/delivered_mail.mtail",
            user="root",
            group="root",
            mode="644",
        )
        self.need_restart = mtail_conf.changed
    def activate(self):
        systemd.service(
            name="Start and enable mtail",
            service="mtail.service",
            running=bool(self.mtail_address),
            enabled=bool(self.mtail_address),
            restarted=self.need_restart,
        )
        self.need_restart = False
--- a/cmdeploy/src/cmdeploy/mtail/mtail.service.j2
+++ b/cmdeploy/src/cmdeploy/mtail/mtail.service.j2
@@ -1,10 +0,0 @@
 [Unit]
 Description=mtail
 [Service]
 Type=simple
 ExecStart=/bin/sh -c "journalctl -f -o short-iso -n 0 | /usr/local/bin/mtail --address={{ address }} --port={{ port }} --progs /etc/mtail --logtostderr --logs -"
 Restart=on-failure
 [Install]
 WantedBy=multi-user.target
--- a/cmdeploy/src/cmdeploy/nginx/deployer.py
+++ b/cmdeploy/src/cmdeploy/nginx/deployer.py
@@ -1,117 +0,0 @@
 from chatmaild.config import Config
 from pyinfra.operations import apt, files, systemd
 from cmdeploy.basedeploy import (
    Deployer,
    get_resource,
 )
 class NginxDeployer(Deployer):
    def __init__(self, config):
        self.config = config
    def install(self):
        #
        # If we allow nginx to start up on install, it will grab port
        # 80, which then will block acmetool from listening on the port.
        # That in turn prevents getting certificates, which then causes
        # an error when we try to start nginx on the custom config
        # that leaves port 80 open but also requires certificates to
        # be present.  To avoid getting into that interlocking mess,
        # we use policy-rc.d to prevent nginx from starting up when it
        # is installed.
        #
        # This approach allows us to avoid performing any explicit
        # systemd operations during the install stage (as opposed to
        # allowing it to start and then forcing it to stop), which allows
        # the install stage to run in non-systemd environments like a
        # container image build.
        #
        # For documentation about policy-rc.d, see:
        # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
        #
        files.put(
            src=get_resource("policy-rc.d"),
            dest="/usr/sbin/policy-rc.d",
            user="root",
            group="root",
            mode="755",
        )
        apt.packages(
            name="Install nginx",
            packages=["nginx", "libnginx-mod-stream"],
        )
        files.file("/usr/sbin/policy-rc.d", present=False)
    def configure(self):
        self.need_restart = _configure_nginx(self.config)
    def activate(self):
        systemd.service(
            name="Start and enable nginx",
            service="nginx.service",
            running=True,
            enabled=True,
            restarted=self.need_restart,
        )
        self.need_restart = False
 def _configure_nginx(config: Config, debug: bool = False) -> bool:
    """Configures nginx HTTP server."""
    need_restart = False
    main_config = files.template(
        src=get_resource("nginx/nginx.conf.j2"),
        dest="/etc/nginx/nginx.conf",
        user="root",
        group="root",
        mode="644",
        config={"domain_name": config.mail_domain},
        disable_ipv6=config.disable_ipv6,
    )
    need_restart |= main_config.changed
    autoconfig = files.template(
        src=get_resource("nginx/autoconfig.xml.j2"),
        dest="/var/www/html/.well-known/autoconfig/mail/config-v1.1.xml",
        user="root",
        group="root",
        mode="644",
        config={"domain_name": config.mail_domain},
    )
    need_restart |= autoconfig.changed
    mta_sts_config = files.template(
        src=get_resource("nginx/mta-sts.txt.j2"),
        dest="/var/www/html/.well-known/mta-sts.txt",
        user="root",
        group="root",
        mode="644",
        config={"domain_name": config.mail_domain},
    )
    need_restart |= mta_sts_config.changed
    # install CGI newemail script
    #
    cgi_dir = "/usr/lib/cgi-bin"
    files.directory(
        name=f"Ensure {cgi_dir} exists",
        path=cgi_dir,
        user="root",
        group="root",
    )
    files.put(
        name="Upload cgi newemail.py script",
        src=get_resource("newemail.py", pkg="chatmaild").open("rb"),
        dest=f"{cgi_dir}/newemail.py",
        user="root",
        group="root",
        mode="755",
    )
    return need_restart
--- a/cmdeploy/src/cmdeploy/nginx/mta-sts.txt.j2
+++ b/cmdeploy/src/cmdeploy/nginx/mta-sts.txt.j2
@@ -1,4 +0,0 @@
 version: STSv1
 mode: enforce
 mx: {{ config.domain_name }}
 max_age: 2419200
--- a/cmdeploy/src/cmdeploy/nginx/nginx.conf.j2
+++ b/cmdeploy/src/cmdeploy/nginx/nginx.conf.j2
@@ -1,139 +0,0 @@
 load_module modules/ngx_stream_module.so;
 user www-data;
 worker_processes auto;
 # Increase the number of connections
 # that a worker process can open
 # to avoid errors such as
 #   accept4() failed (24: Too many open files)
 # and
 #   socket() failed (24: Too many open files) while connecting to upstream
 # in the logs.
 # <https://nginx.org/en/docs/ngx_core_module.html#worker_rlimit_nofile>
 worker_rlimit_nofile 2048;
 pid /run/nginx.pid;
 error_log syslog:server=unix:/dev/log,facility=local3;
 events {
        # Increase to avoid errors such as
        #   768 worker_connections are not enough while connecting to upstream
        # in the logs.
        # <https://nginx.org/en/docs/ngx_core_module.html#worker_connections>
 	worker_connections 2048;
 	# multi_accept on;
 }
 stream {
        map $ssl_preread_alpn_protocols $proxy {
            default 127.0.0.1:8443;
            ~\bsmtp\b 127.0.0.1:465;
            ~\bimap\b 127.0.0.1:993;
        }
        server {
                listen 443;
                {% if not disable_ipv6 %}
                listen [::]:443;
                {% endif %}
                proxy_pass $proxy;
                ssl_preread on;
        }
 }
 http {
 	sendfile on;
 	tcp_nopush on;
 	# Do not emit nginx version on error pages.
 	server_tokens off;
 	include /etc/nginx/mime.types;
 	default_type application/octet-stream;
 	ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
 	ssl_prefer_server_ciphers on;
 	ssl_certificate /var/lib/acme/live/{{ config.domain_name }}/fullchain;
 	ssl_certificate_key /var/lib/acme/live/{{ config.domain_name }}/privkey;
 	gzip on;
 	server {
 		listen 127.0.0.1:8443 ssl default_server;
 		root /var/www/html;
 		index index.html index.htm;
 		server_name {{ config.domain_name }} www.{{ config.domain_name }} mta-sts.{{ config.domain_name }};
 		access_log syslog:server=unix:/dev/log,facility=local7;
 		location / {
 			# First attempt to serve request as file, then
 			# as directory, then fall back to displaying a 404.
 			try_files $uri $uri/ =404;
 		}
 		location /metrics {
 			default_type text/plain;
 		}
 		location /new {
 			if ($request_method = GET) {
 				# Redirect to Delta Chat,
 				# which will in turn do a POST request.
 				return 301 dcaccount:https://{{ config.domain_name }}/new;
 			}
 			fastcgi_pass unix:/run/fcgiwrap.socket;
 			include /etc/nginx/fastcgi_params;
 			fastcgi_param SCRIPT_FILENAME /usr/lib/cgi-bin/newemail.py;
 		}
 		# Old URL for compatibility with e.g. printed QR codes.
 		#
 		# Copy-paste instead of redirect to /new
 		# because Delta Chat core does not follow redirects.
 		#
 		# Redirects are only for browsers.
 		location /cgi-bin/newemail.py {
 			if ($request_method = GET) {
 				return 301 dcaccount:https://{{ config.domain_name }}/new;
 			}
 			fastcgi_pass unix:/run/fcgiwrap.socket;
 			include /etc/nginx/fastcgi_params;
 			fastcgi_param SCRIPT_FILENAME /usr/lib/cgi-bin/newemail.py;
 		}
                # Proxy to iroh-relay service.
                location /relay {
                        proxy_pass http://127.0.0.1:3340;
                        proxy_http_version 1.1;
                        # Upgrade header is normally set to "iroh derp http" or "websocket".
                        proxy_set_header Upgrade $http_upgrade;
                        proxy_set_header Connection "upgrade";
                }
 		location /relay/probe {
                        proxy_pass http://127.0.0.1:3340;
                        proxy_http_version 1.1;
 		}
 		location /generate_204 {
                        proxy_pass http://127.0.0.1:3340;
                        proxy_http_version 1.1;
 		}
 	}
 	# Redirect www. to non-www
 	server {
 		listen 127.0.0.1:8443 ssl;
 		server_name www.{{ config.domain_name }};
 		return 301 $scheme://{{ config.domain_name }}$request_uri;
 		access_log syslog:server=unix:/dev/log,facility=local7;
 	}
 }
--- a/cmdeploy/src/cmdeploy/opendkim/KeyTable
+++ b/cmdeploy/src/cmdeploy/opendkim/KeyTable
@@ -1 +0,0 @@
 {{ config.opendkim_selector }}._domainkey.{{ config.domain_name }} {{ config.domain_name }}:{{ config.opendkim_selector }}:/etc/dkimkeys/{{ config.opendkim_selector }}.private
--- a/cmdeploy/src/cmdeploy/opendkim/SigningTable
+++ b/cmdeploy/src/cmdeploy/opendkim/SigningTable
@@ -1 +0,0 @@
 *@{{ config.domain_name }} {{ config.opendkim_selector }}._domainkey.{{ config.domain_name }}
--- a/cmdeploy/src/cmdeploy/opendkim/deployer.py
+++ b/cmdeploy/src/cmdeploy/opendkim/deployer.py
@@ -1,123 +0,0 @@
 """
 Installs OpenDKIM
 """
 from pyinfra import host
 from pyinfra.facts.files import File
 from pyinfra.operations import apt, files, server, systemd
 from cmdeploy.basedeploy import Deployer, get_resource
 class OpendkimDeployer(Deployer):
    required_users = [("opendkim", None, ["opendkim"])]
    def __init__(self, mail_domain):
        self.mail_domain = mail_domain
    def install(self):
        apt.packages(
            name="apt install opendkim opendkim-tools",
            packages=["opendkim", "opendkim-tools"],
        )
    def configure(self):
        domain = self.mail_domain
        dkim_selector = "opendkim"
        """Configures OpenDKIM"""
        need_restart = False
        main_config = files.template(
            src=get_resource("opendkim/opendkim.conf"),
            dest="/etc/opendkim.conf",
            user="root",
            group="root",
            mode="644",
            config={"domain_name": domain, "opendkim_selector": dkim_selector},
        )
        need_restart |= main_config.changed
        screen_script = files.put(
            src=get_resource("opendkim/screen.lua"),
            dest="/etc/opendkim/screen.lua",
            user="root",
            group="root",
            mode="644",
        )
        need_restart |= screen_script.changed
        final_script = files.put(
            src=get_resource("opendkim/final.lua"),
            dest="/etc/opendkim/final.lua",
            user="root",
            group="root",
            mode="644",
        )
        need_restart |= final_script.changed
        files.directory(
            name="Add opendkim directory to /etc",
            path="/etc/opendkim",
            user="opendkim",
            group="opendkim",
            mode="750",
            present=True,
        )
        keytable = files.template(
            src=get_resource("opendkim/KeyTable"),
            dest="/etc/dkimkeys/KeyTable",
            user="opendkim",
            group="opendkim",
            mode="644",
            config={"domain_name": domain, "opendkim_selector": dkim_selector},
        )
        need_restart |= keytable.changed
        signing_table = files.template(
            src=get_resource("opendkim/SigningTable"),
            dest="/etc/dkimkeys/SigningTable",
            user="opendkim",
            group="opendkim",
            mode="644",
            config={"domain_name": domain, "opendkim_selector": dkim_selector},
        )
        need_restart |= signing_table.changed
        files.directory(
            name="Add opendkim socket directory to /var/spool/postfix",
            path="/var/spool/postfix/opendkim",
            user="opendkim",
            group="opendkim",
            mode="750",
            present=True,
        )
        if not host.get_fact(File, f"/etc/dkimkeys/{dkim_selector}.private"):
            server.shell(
                name="Generate OpenDKIM domain keys",
                commands=[
                    f"/usr/sbin/opendkim-genkey -D /etc/dkimkeys -d {domain} -s {dkim_selector}"
                ],
                _use_su_login=True,
                _su_user="opendkim",
            )
        service_file = files.put(
            name="Configure opendkim to restart once a day",
            src=get_resource("opendkim/systemd.conf"),
            dest="/etc/systemd/system/opendkim.service.d/10-prevent-memory-leak.conf",
        )
        need_restart |= service_file.changed
        self.need_restart = need_restart
    def activate(self):
        systemd.service(
            name="Start and enable OpenDKIM",
            service="opendkim.service",
            running=True,
            enabled=True,
            daemon_reload=self.need_restart,
            restarted=self.need_restart,
        )
        self.need_restart = False
--- a/cmdeploy/src/cmdeploy/opendkim/final.lua
+++ b/cmdeploy/src/cmdeploy/opendkim/final.lua
@@ -1,42 +0,0 @@
 mtaname = odkim.get_mtasymbol(ctx, "{daemon_name}")
 if mtaname == "ORIGINATING" then
 	-- Outgoing message will be signed,
 	-- no need to look for signatures.
 	return nil
 end
 nsigs = odkim.get_sigcount(ctx)
 if nsigs == nil then
 	return nil
 end
 local valid = false
 local error_msg = "No valid DKIM signature found."
 for i = 1, nsigs do
 	sig = odkim.get_sighandle(ctx, i - 1)
 	sigres = odkim.sig_result(sig)
 	-- All signatures that do not correspond to From: 
 	-- were ignored in screen.lua and return sigres -1.
 	-- 
 	-- Any valid signature that was not ignored like this
 	-- means the message is acceptable.
 	if sigres == 0 then
 		valid = true
    else
        error_msg = "DKIM signature is invalid, error code " .. tostring(sigres) .. ", search https://github.com/trusteddomainproject/OpenDKIM/blob/master/libopendkim/dkim.h#L108"
 	end
 end
 if valid then
 	-- Strip all DKIM-Signature headers after successful validation
 	-- Delete in reverse order to avoid index shifting.
 	for i = nsigs, 1, -1 do
 		odkim.del_header(ctx, "DKIM-Signature", i)
 	end
 else
 	odkim.set_reply(ctx, "554", "5.7.1", error_msg)
 	odkim.set_result(ctx, SMFIS_REJECT)
 end
 return nil
--- a/cmdeploy/src/cmdeploy/opendkim/opendkim.conf
+++ b/cmdeploy/src/cmdeploy/opendkim/opendkim.conf
@@ -1,73 +0,0 @@
 # OpenDKIM configuration.
 Syslog			yes
 SyslogSuccess		yes
 #LogWhy			no
 # Common signing and verification parameters. In Debian, the "From" header is
 # oversigned, because it is often the identity key used by reputation systems
 # and thus somewhat security sensitive.
 Canonicalization	relaxed/simple
 OversignHeaders		From
 On-BadSignature         reject
 On-KeyNotFound          reject
 On-NoSignature          reject
 DNSTimeout          60
 # Signing domain, selector, and key (required). For example, perform signing
 # for domain "example.com" with selector "2020" (2020._domainkey.example.com),
 # using the private key stored in /etc/dkimkeys/example.private. More granular
 # setup options can be found in /usr/share/doc/opendkim/README.opendkim.
 Domain			{{ config.domain_name }}
 Selector		{{ config.opendkim_selector }}
 KeyFile		  /etc/dkimkeys/{{ config.opendkim_selector }}.private
 KeyTable          /etc/dkimkeys/KeyTable
 SigningTable      refile:/etc/dkimkeys/SigningTable
 # Sign Autocrypt header in addition to the default specified in RFC 6376.
 #
 # Default list is here:
 # <https://github.com/trusteddomainproject/OpenDKIM/blob/5c539587561785a66c1f67f720f2fb741f320785/libopendkim/dkim.c#L221-L245>
 SignHeaders *,+autocrypt,+content-type
 # Prevent addition of second Content-Type header
 # and other important headers that should not be added
 # after signing the message.
 # See
 # <https://www.zone.eu/blog/2024/05/17/bimi-and-dmarc-cant-save-you/>
 # and RFC 6376 (page 41) for reference.
 #
 # We don't use "l=" body length so the problem described in RFC 6376
 # is not applicable, but adding e.g. a second "From" header
 # or second "Autocrypt" header is better prevented in any case.
 #
 # Default is empty.
 OversignHeaders from,reply-to,subject,date,to,cc,resent-date,resent-from,resent-sender,resent-to,resent-cc,in-reply-to,references,list-id,list-help,list-unsubscribe,list-subscribe,list-post,list-owner,list-archive,autocrypt
 # Script to ignore signatures that do not correspond to the From: domain.
 ScreenPolicyScript /etc/opendkim/screen.lua
 # Script to reject mails without a valid DKIM signature.
 FinalPolicyScript /etc/opendkim/final.lua
 # In Debian, opendkim runs as user "opendkim". A umask of 007 is required when
 # using a local socket with MTAs that access the socket as a non-privileged
 # user (for example, Postfix). You may need to add user "postfix" to group
 # "opendkim" in that case.
 UserID			opendkim
 UMask			007
 Socket			local:/var/spool/postfix/opendkim/opendkim.sock
 PidFile			/run/opendkim/opendkim.pid
 # The trust anchor enables DNSSEC. In Debian, the trust anchor file is provided
 # by the package dns-root-data.
 TrustAnchorFile		/usr/share/dns/root.key
 # Sign messages when `-o milter_macro_daemon_name=ORIGINATING` is set.
 MTA ORIGINATING
 # No hosts are treated as internal, ORIGINATING daemon name should be set explicitly.
 InternalHosts -
--- a/cmdeploy/src/cmdeploy/opendkim/screen.lua
+++ b/cmdeploy/src/cmdeploy/opendkim/screen.lua
@@ -1,21 +0,0 @@
 -- Ignore signatures that do not correspond to the From: domain.
 from_domain = odkim.get_fromdomain(ctx)
 if from_domain == nil then
 	return nil
 end
 n = odkim.get_sigcount(ctx)
 if n == nil then
 	return nil
 end
 for i = 1, n do
 	sig = odkim.get_sighandle(ctx, i - 1)
 	sig_domain = odkim.sig_getdomain(sig)
 	if from_domain ~= sig_domain then
 		odkim.sig_ignore(sig)
 	end
 end
 return nil
--- a/cmdeploy/src/cmdeploy/opendkim/systemd.conf
+++ b/cmdeploy/src/cmdeploy/opendkim/systemd.conf
@@ -1,3 +0,0 @@
 [Service]
 Restart=always
 RuntimeMaxSec=1d
--- a/cmdeploy/src/cmdeploy/policy-rc.d
+++ b/cmdeploy/src/cmdeploy/policy-rc.d
@@ -1,3 +0,0 @@
 #!/bin/sh
 echo "All runlevel operations denied by policy" >&2
 exit 101
--- a/cmdeploy/src/cmdeploy/postfix/deployer.py
+++ b/cmdeploy/src/cmdeploy/postfix/deployer.py
@@ -1,117 +0,0 @@
 from pyinfra.operations import apt, files, server, systemd
 from cmdeploy.basedeploy import Deployer, get_resource
 class PostfixDeployer(Deployer):
    required_users = [("postfix", None, ["opendkim"])]
    daemon_reload = False
    def __init__(self, config, disable_mail):
        self.config = config
        self.disable_mail = disable_mail
    def install(self):
        apt.packages(
            name="Install Postfix",
            packages="postfix",
        )
    def configure(self):
        config = self.config
        need_restart = False
        main_config = files.template(
            src=get_resource("postfix/main.cf.j2"),
            dest="/etc/postfix/main.cf",
            user="root",
            group="root",
            mode="644",
            config=config,
            disable_ipv6=config.disable_ipv6,
        )
        need_restart |= main_config.changed
        master_config = files.template(
            src=get_resource("postfix/master.cf.j2"),
            dest="/etc/postfix/master.cf",
            user="root",
            group="root",
            mode="644",
            debug=False,
            config=config,
        )
        need_restart |= master_config.changed
        header_cleanup = files.put(
            src=get_resource("postfix/submission_header_cleanup"),
            dest="/etc/postfix/submission_header_cleanup",
            user="root",
            group="root",
            mode="644",
        )
        need_restart |= header_cleanup.changed
        lmtp_header_cleanup = files.put(
            src=get_resource("postfix/lmtp_header_cleanup"),
            dest="/etc/postfix/lmtp_header_cleanup",
            user="root",
            group="root",
            mode="644",
        )
        need_restart |= lmtp_header_cleanup.changed
        tls_policy_map = files.put(
            name="Upload SMTP TLS Policy that accepts self-signed certificates for IP-only hosts",
            src=get_resource("postfix/smtp_tls_policy_map"),
            dest="/etc/postfix/smtp_tls_policy_map",
            user="root",
            group="root",
            mode="644",
        )
        need_restart |= tls_policy_map.changed
        if tls_policy_map.changed:
            server.shell(
                commands=["postmap /etc/postfix/smtp_tls_policy_map"],
            )
        # Login map that 1:1 maps email address to login.
        login_map = files.put(
            src=get_resource("postfix/login_map"),
            dest="/etc/postfix/login_map",
            user="root",
            group="root",
            mode="644",
        )
        need_restart |= login_map.changed
        restart_conf = files.put(
            name="postfix: restart automatically on failure",
            src=get_resource("service/10_restart.conf"),
            dest="/etc/systemd/system/postfix@.service.d/10_restart.conf",
        )
        self.daemon_reload = restart_conf.changed
        # Validate postfix configuration before restart
        if need_restart:
            server.shell(
                name="Validate postfix configuration",
                # Extract stderr and quit with error if non-zero
                commands=["""bash -c 'w=$(postconf 2>&1 >/dev/null); [[ -z "$w" ]] || { echo "$w"; false; }'"""],
            )
        self.need_restart = need_restart
    def activate(self):
        restart = False if self.disable_mail else self.need_restart
        systemd.service(
            name="disable postfix for now"
            if self.disable_mail
            else "Start and enable Postfix",
            service="postfix.service",
            running=False if self.disable_mail else True,
            enabled=False if self.disable_mail else True,
            restarted=restart,
            daemon_reload=self.daemon_reload,
        )
        self.need_restart = False
--- a/Show More
+++ b/Show More
		`@@ -1,2 +0,0 @@`
			`"acme-enter-email": "{{ email }}"`
			`"acme-agreement:https://letsencrypt.org/documents/LE-SA-v1.6-August-18-2025.pdf": true`
		`@@ -1 +0,0 @@`
			`/5 * * * root {{ config.execpath }} {{ config.mailboxes_dir }} >/var/www/html/metrics`
		`@@ -1 +0,0 @@`
			`{{ config.opendkim_selector }}._domainkey.{{ config.domain_name }} {{ config.domain_name }}:{{ config.opendkim_selector }}:/etc/dkimkeys/{{ config.opendkim_selector }}.private`
		`@@ -1 +0,0 @@`
			`*@{{ config.domain_name }} {{ config.opendkim_selector }}._domainkey.{{ config.domain_name }}`