Remove authclean service from reinjecting port

Messages are already cleaned once on the submission port
to avoid leaking the user IP address.
There is no need to clean them again
and remove the `Received` header
which records reception from filtermail.

.github/workflows/test-and-deploy-ipv4only.yaml

@@ -38,7 +38,9 @@ jobs:
           if [ -f dkimkeys-ipv4/dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys-ipv4 root@ns.testrun.org:/tmp/ || true; fi
           if [ "$(ls -A acme-ipv4/acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme-ipv4 root@ns.testrun.org:/tmp/ || true; fi
           # make sure CAA record isn't set
-          ssh -o StrictHostKeyChecking=accept-new root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging-ipv4.testrun.org.zone
+          scp -o StrictHostKeyChecking=accept-new .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
+          ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging-ipv4.testrun.org.zone
+          ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
           ssh root@ns.testrun.org systemctl reload nsd
 
       - name: rebuild staging-ipv4.testrun.org to have a clean VPS
@@ -47,7 +49,7 @@ jobs:
             -H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
             -H "Content-Type: application/json" \
             -d '{"image":"debian-12"}' \
-            "https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_SERVER_ID }}/actions/rebuild"
+            "https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_IPV4_SERVER_ID }}/actions/rebuild"
 
       - run: scripts/initenv.sh
 
@@ -61,11 +63,11 @@ jobs:
           while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u ; do sleep 1 ; done
           ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u
           # download acme & dkim state from ns.testrun.org
-          rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme-ipv4 acme-restore || true
-          rsync -avz root@ns.testrun.org:/tmp/dkimkeys-ipv4 dkimkeys-restore || true
+          rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme-ipv4/acme acme-restore || true
+          rsync -avz root@ns.testrun.org:/tmp/dkimkeys-ipv4/dkimkeys dkimkeys-restore || true
           # restore acme & dkim state to staging2.testrun.org
-          rsync -avz acme-restore/acme-ipv4/acme root@staging-ipv4.testrun.org:/var/lib/acme || true
-          rsync -avz dkimkeys-restore/dkimkeys-ipv4/dkimkeys root@staging-ipv4.testrun.org:/etc/dkimkeys || true
+          rsync -avz acme-restore/acme root@staging-ipv4.testrun.org:/var/lib/ || true
+          rsync -avz dkimkeys-restore/dkimkeys root@staging-ipv4.testrun.org:/etc/ || true
           ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org chown root:root -R /var/lib/acme || true
 
       - name: run formatting checks 
@@ -93,6 +95,6 @@ jobs:
       - name: cmdeploy test
         run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow
 
-      - name: cmdeploy dns (try 3 times)
-        run: cmdeploy dns || cmdeploy dns || cmdeploy dns
+      - name: cmdeploy dns
+        run: cmdeploy dns -v
 

.github/workflows/test-and-deploy.yaml

@@ -38,7 +38,9 @@ jobs:
           if [ -f dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys root@ns.testrun.org:/tmp/ || true; fi
           if [ "$(ls -A acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme root@ns.testrun.org:/tmp/ || true; fi
           # make sure CAA record isn't set
-          ssh -o StrictHostKeyChecking=accept-new root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging2.testrun.org.zone
+          scp -o StrictHostKeyChecking=accept-new .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
+          ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging2.testrun.org.zone
+          ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
           ssh root@ns.testrun.org systemctl reload nsd
 
       - name: rebuild staging2.testrun.org to have a clean VPS
@@ -64,8 +66,8 @@ jobs:
           rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme acme-restore || true
           rsync -avz root@ns.testrun.org:/tmp/dkimkeys dkimkeys-restore || true
           # restore acme & dkim state to staging2.testrun.org
-          rsync -avz acme-restore/acme/ root@staging2.testrun.org:/var/lib/acme || true
-          rsync -avz dkimkeys-restore/dkimkeys/ root@staging2.testrun.org:/etc/dkimkeys || true
+          rsync -avz acme-restore/acme root@staging2.testrun.org:/var/lib/ || true
+          rsync -avz dkimkeys-restore/dkimkeys root@staging2.testrun.org:/etc/ || true
           ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org chown root:root -R /var/lib/acme || true
 
       - name: run formatting checks 
@@ -91,6 +93,6 @@ jobs:
       - name: cmdeploy test
         run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow
 
-      - name: cmdeploy dns (try 3 times)
-        run: cmdeploy dns -v || cmdeploy dns -v || cmdeploy dns -v
+      - name: cmdeploy dns
+        run: cmdeploy dns -v
 

CHANGELOG.md

@@ -2,169 +2,273 @@
 
 ## untagged
 
+- cmdeploy dovecot: delete big messages after 7 days
+  ([#504](https://github.com/chatmail/server/pull/504))
+
+- mtail: fix getting logs from STDIN
+  ([#502](https://github.com/chatmail/server/pull/502))
+
+- filtermail: don't require exactly 2 lines after openPGP payload
+  ([#497](https://github.com/chatmail/server/pull/497))
+
+- cmdeploy dns: offer alternative DKIM record format for some web interfaces
+  ([#470](https://github.com/chatmail/server/pull/470))
+
+- journald: remove old logs from disk
+  ([#490](https://github.com/chatmail/server/pull/490))
+
+- opendkim: restart once every day to mend RAM leaks
+  ([#498](https://github.com/chatmail/server/pull/498)
+
+- migration guide: let opendkim own the DKIM keys directory
+  ([#468](https://github.com/chatmail/server/pull/468))
+
+- improve secure-join message detection
+  ([#473](https://github.com/chatmail/server/pull/473))
+
+- use old crypt lib in python < 3.11
+  ([#483](https://github.com/chatmail/server/pull/483))
+
+- chatmaild: set umask to 0700 for doveauth + metadata
+  ([#490](https://github.com/chatmail/server/pull/492))
+
+- remove MTA-STS daemon
+  ([#488](https://github.com/chatmail/server/pull/488))
+
+- replace `Subject` with `[...]` for all outgoing mails.
+  ([#481](https://github.com/chatmail/server/pull/481))
+
+- opendkim: use su instead of sudo
+  ([#491](https://github.com/chatmail/server/pull/491))
+
+## 1.5.0 2024-12-20
+
+- cmdeploy dns: always show recommended DNS records
+  ([#463](https://github.com/chatmail/server/pull/463))
+
+- add `--all` to `cmdeploy dns`
+  ([#462](https://github.com/chatmail/server/pull/462))
+
+- fix `_mta-sts` TXT DNS record
+  ([#461](https://github.com/chatmail/server/pull/461)
+
+- deploy `iroh-relay` and also update "realtime relay services" in privacy policy. 
+  ([#434](https://github.com/chatmail/server/pull/434))
+  ([#451](https://github.com/chatmail/server/pull/451))
+
+- add guide to migrate chatmail to a new server
+  ([#429](https://github.com/chatmail/server/pull/429))
+
+- disable anvil authentication penalty
+  ([#414](https://github.com/chatmail/server/pull/444)
+
+- increase `request_queue_size` for UNIX sockets to 1000.
+  ([#437](https://github.com/chatmail/server/pull/437))
+
+- add argument to `cmdeploy run` for specifying
+  a different SSH host than `mail_domain`
+  ([#439](https://github.com/chatmail/server/pull/439))
+
+- query autoritative nameserver to bypass DNS cache
+  ([#424](https://github.com/chatmail/server/pull/424))
+
+- add mtail support (new optional `mtail_address` ini value)
+  This defines the address on which [`mtail`](https://google.github.io/mtail/)
+  exposes its metrics collected from the logs.
+  If you want to collect the metrics with Prometheus,
+  setup a private network (e.g. WireGuard interface)
+  and assign an IP address from this network to the host.
+  If you do not plan to collect metrics,
+  keep this setting unset.
+  ([#388](https://github.com/chatmail/server/pull/388))
+
+- fix checking for required DNS records
+  ([#412](https://github.com/chatmail/server/pull/412))
+
+- add support for specifying whole domains for recipient passthrough list
+  ([#408](https://github.com/chatmail/server/pull/408))
+
+- add a paragraph about "account deletion" to info page 
+  ([#405](https://github.com/chatmail/server/pull/405))
+
 - avoid nginx listening on ipv6 if v6 is dsiabled 
-  ([#402](https://github.com/deltachat/chatmail/pull/402))
+  ([#402](https://github.com/chatmail/server/pull/402))
+
+- refactor ssh-based execution to allow organizing remote functions in
+  modules. 
+  ([#396](https://github.com/chatmail/server/pull/396))
 
 - trigger "apt upgrade" during "cmdeploy run" 
-  ([#398](https://github.com/deltachat/chatmail/pull/398))
+  ([#398](https://github.com/chatmail/server/pull/398))
 
 - drop hispanilandia passthrough address
-  ([#401](https://github.com/deltachat/chatmail/pull/401))
+  ([#401](https://github.com/chatmail/server/pull/401))
+
+- set CAA record flags to 0
+
+- add IMAP capabilities instead of overwriting them
+  ([#413](https://github.com/chatmail/server/pull/413))
+
+- fix OpenPGP payload check
+  ([#435](https://github.com/chatmail/server/pull/435))
+
+- fix Dovecot quota_max_mail_size to use max_message_size config value
+  ([#438](https://github.com/chatmail/server/pull/438))
 
 
 ## 1.4.1 2024-07-31
 
 - fix metadata dictproxy which would confuse transactions
   resulting in missed notifications and other issues. 
-  ([#393](https://github.com/deltachat/chatmail/pull/393))
-  ([#394](https://github.com/deltachat/chatmail/pull/394))
+  ([#393](https://github.com/chatmail/server/pull/393))
+  ([#394](https://github.com/chatmail/server/pull/394))
 
 - add optional "imap_rawlog" config option. If true, 
   .in/.out files are created in user home dirs 
   containing the imap protocol messages. 
-  ([#389](https://github.com/deltachat/chatmail/pull/389))
+  ([#389](https://github.com/chatmail/server/pull/389))
 
 ## 1.4.0 2024-07-28
 
 - Add `disable_ipv6` config option to chatmail.ini.
   Required if the server doesn't have IPv6 connectivity.
-  ([#312](https://github.com/deltachat/chatmail/pull/312))
+  ([#312](https://github.com/chatmail/server/pull/312))
 
 - allow current K9/Thunderbird-mail releases to send encrypted messages
   outside by accepting their localized "encrypted subject" strings. 
-  ([#370](https://github.com/deltachat/chatmail/pull/370))
+  ([#370](https://github.com/chatmail/server/pull/370))
 
 - Migrate and remove sqlite database in favor of password/lastlogin tracking 
   in a user's maildir.  
-  ([#379](https://github.com/deltachat/chatmail/pull/379))
+  ([#379](https://github.com/chatmail/server/pull/379))
 
 - Require pyinfra V3 installed on the client side,
   run `./scripts/initenv.sh` to upgrade locally.
-  ([#378](https://github.com/deltachat/chatmail/pull/378))
+  ([#378](https://github.com/chatmail/server/pull/378))
 
 - don't hardcode "/home/vmail" paths but rather set them 
   once in the config object and use it everywhere else, 
   thereby also improving testability.  
-  ([#351](https://github.com/deltachat/chatmail/pull/351))
+  ([#351](https://github.com/chatmail/server/pull/351))
   temporarily introduced obligatory "passdb_path" and "mailboxes_dir" 
   settings but they were removed/obsoleted in 
-  ([#380](https://github.com/deltachat/chatmail/pull/380))
+  ([#380](https://github.com/chatmail/server/pull/380))
 
 - BREAKING: new required chatmail.ini value 'delete_inactive_users_after = 100'
   which removes users from database and mails after 100 days without any login. 
-  ([#350](https://github.com/deltachat/chatmail/pull/350))
+  ([#350](https://github.com/chatmail/server/pull/350))
 
 - Refine DNS checking to distinguish between "required" and "recommended" settings 
-  ([#372](https://github.com/deltachat/chatmail/pull/372))
+  ([#372](https://github.com/chatmail/server/pull/372))
 
 - reload nginx in the acmetool cronjob
-  ([#360](https://github.com/deltachat/chatmail/pull/360))
+  ([#360](https://github.com/chatmail/server/pull/360))
 
 - remove checking of reverse-DNS PTR records.  Chatmail-servers don't
   depend on it and even in the wider e-mail system it's not common anymore. 
   If it's an issue, a chatmail operator can still care to properly set reverse DNS. 
-  ([#348](https://github.com/deltachat/chatmail/pull/348))
+  ([#348](https://github.com/chatmail/server/pull/348))
 
 - Make DNS-checking faster and more interactive, run it fully during "cmdeploy run",
   also introducing a generic mechanism for rapid remote ssh-based python function execution. 
-  ([#346](https://github.com/deltachat/chatmail/pull/346))
+  ([#346](https://github.com/chatmail/server/pull/346))
 
 - Don't fix file owner ship of /home/vmail 
-  ([#345](https://github.com/deltachat/chatmail/pull/345))
+  ([#345](https://github.com/chatmail/server/pull/345))
 
 - Support iterating over all users with doveadm commands 
-  ([#344](https://github.com/deltachat/chatmail/pull/344))
+  ([#344](https://github.com/chatmail/server/pull/344))
 
 - Test and fix for attempts to create inadmissible accounts 
-  ([#333](https://github.com/deltachat/chatmail/pull/321))
+  ([#333](https://github.com/chatmail/server/pull/321))
 
 - check that OpenPGP has only PKESK, SKESK and SEIPD packets
-  ([#323](https://github.com/deltachat/chatmail/pull/323),
-   [#324](https://github.com/deltachat/chatmail/pull/324))
+  ([#323](https://github.com/chatmail/server/pull/323),
+   [#324](https://github.com/chatmail/server/pull/324))
 
 - improve filtermail checks for encrypted messages and drop support for unencrypted MDNs
-  ([#320](https://github.com/deltachat/chatmail/pull/320))
+  ([#320](https://github.com/chatmail/server/pull/320))
 
 - replace `bash` with `/bin/sh`
-  ([#334](https://github.com/deltachat/chatmail/pull/334))
+  ([#334](https://github.com/chatmail/server/pull/334))
 
 - Increase number of logged in IMAP sessions to 50000
-  ([#335](https://github.com/deltachat/chatmail/pull/335))
+  ([#335](https://github.com/chatmail/server/pull/335))
 
 - filtermail: do not allow ASCII armor without actual payload
-  ([#325](https://github.com/deltachat/chatmail/pull/325))
+  ([#325](https://github.com/chatmail/server/pull/325))
 
 - Remove sieve to enable hardlink deduplication in LMTP
-  ([#343](https://github.com/deltachat/chatmail/pull/343))
+  ([#343](https://github.com/chatmail/server/pull/343))
 
 - dovecot: enable gzip compression on disk
-  ([#341](https://github.com/deltachat/chatmail/pull/341))
+  ([#341](https://github.com/chatmail/server/pull/341))
 
 - DKIM-sign Content-Type and oversign all signed headers
-  ([#296](https://github.com/deltachat/chatmail/pull/296))
+  ([#296](https://github.com/chatmail/server/pull/296))
 
 - Add nonci_accounts metric
-  ([#347](https://github.com/deltachat/chatmail/pull/347))
+  ([#347](https://github.com/chatmail/server/pull/347))
 
 - doveauth: log when a new account is created
-  ([#349](https://github.com/deltachat/chatmail/pull/349))
+  ([#349](https://github.com/chatmail/server/pull/349))
 
 - Multiplex HTTPS, IMAP and SMTP on port 443
-  ([#357](https://github.com/deltachat/chatmail/pull/357))
+  ([#357](https://github.com/chatmail/server/pull/357))
 
 ## 1.3.0 - 2024-06-06
 
 - don't check necessary DNS records on cmdeploy init anymore
-  ([#316](https://github.com/deltachat/chatmail/pull/316))
+  ([#316](https://github.com/chatmail/server/pull/316))
 
 - ensure cron and acl are installed
-  ([#293](https://github.com/deltachat/chatmail/pull/293),
-  [#310](https://github.com/deltachat/chatmail/pull/310))
+  ([#293](https://github.com/chatmail/server/pull/293),
+  [#310](https://github.com/chatmail/server/pull/310))
 
 - change default for delete_mails_after from 40 to 20 days
-  ([#300](https://github.com/deltachat/chatmail/pull/300))
+  ([#300](https://github.com/chatmail/server/pull/300))
 
 - save journald logs only to memory and save nginx logs to journald instead of file
-  ([#299](https://github.com/deltachat/chatmail/pull/299))
+  ([#299](https://github.com/chatmail/server/pull/299))
 
 - fix writing of multiple obs repositories in `/etc/apt/sources.list`
-  ([#290](https://github.com/deltachat/chatmail/pull/290))
+  ([#290](https://github.com/chatmail/server/pull/290))
 
 - metadata: add support for `/shared/vendor/deltachat/irohrelay`
-  ([#284](https://github.com/deltachat/chatmail/pull/284))
+  ([#284](https://github.com/chatmail/server/pull/284))
 
 - Emit "XCHATMAIL" capability from IMAP server 
-  ([#278](https://github.com/deltachat/chatmail/pull/278))
+  ([#278](https://github.com/chatmail/server/pull/278))
 
 - Move echobot `into /var/lib/echobot`
-  ([#281](https://github.com/deltachat/chatmail/pull/281))
+  ([#281](https://github.com/chatmail/server/pull/281))
 
 - Accept Let's Encrypt's new Terms of Services
-  ([#275](https://github.com/deltachat/chatmail/pull/276))
+  ([#275](https://github.com/chatmail/server/pull/276))
 
 - Reload Dovecot and Postfix when TLS certificate updates
-  ([#271](https://github.com/deltachat/chatmail/pull/271))
+  ([#271](https://github.com/chatmail/server/pull/271))
 
 - Use forked version of dovecot without hardcoded delays
-  ([#270](https://github.com/deltachat/chatmail/pull/270))
+  ([#270](https://github.com/chatmail/server/pull/270))
 
 ## 1.2.0 - 2024-04-04
 
 - Install dig on the server to resolve DNS records
-  ([#267](https://github.com/deltachat/chatmail/pull/267))
+  ([#267](https://github.com/chatmail/server/pull/267))
 
 - preserve notification order and exponentially backoff with 
   retries for tokens where we didn't get a successful return
-  ([#265](https://github.com/deltachat/chatmail/pull/263))
+  ([#265](https://github.com/chatmail/server/pull/263))
 
 - Run chatmail-metadata and doveauth as vmail
-  ([#261](https://github.com/deltachat/chatmail/pull/261))
+  ([#261](https://github.com/chatmail/server/pull/261))
 
 - Apply systemd restrictions to echobot
-  ([#259](https://github.com/deltachat/chatmail/pull/259))
+  ([#259](https://github.com/chatmail/server/pull/259))
 
 - re-enable running the CI in pull requests, but not concurrently 
-  ([#258](https://github.com/deltachat/chatmail/pull/258))
+  ([#258](https://github.com/chatmail/server/pull/258))
 
 
 ## 1.1.0 - 2024-03-28
@@ -172,27 +276,27 @@
 ### The changelog starts to record changes from March 15th, 2024 
 
 - Move systemd unit templates to cmdeploy package 
-  ([#255](https://github.com/deltachat/chatmail/pull/255))
+  ([#255](https://github.com/chatmail/server/pull/255))
 
 - Persist push tokens and support multiple device per address 
-  ([#254](https://github.com/deltachat/chatmail/pull/254))
+  ([#254](https://github.com/chatmail/server/pull/254))
 
 - Avoid warning for regular doveauth protocol's hello message. 
-  ([#250](https://github.com/deltachat/chatmail/pull/250))
+  ([#250](https://github.com/chatmail/server/pull/250))
 
 - Fix various tests to pass again with "cmdeploy test". 
-  ([#245](https://github.com/deltachat/chatmail/pull/245),
-  [#242](https://github.com/deltachat/chatmail/pull/242)
+  ([#245](https://github.com/chatmail/server/pull/245),
+  [#242](https://github.com/chatmail/server/pull/242)
 
 - Ensure lets-encrypt certificates are reloaded after renewal 
-  ([#244]) https://github.com/deltachat/chatmail/pull/244
+  ([#244]) https://github.com/chatmail/server/pull/244
 
 - Persist tokens to avoid iOS users loosing push-notifications when the
   chatmail metadata service is restarted (happens regularly during deploys)
-  ([#238](https://github.com/deltachat/chatmail/pull/239)
+  ([#238](https://github.com/chatmail/server/pull/239)
 
 - Fix failing sieve-script compile errors on incoming messages
-  ([#237](https://github.com/deltachat/chatmail/pull/239)
+  ([#237](https://github.com/chatmail/server/pull/239)
 
 - Fix quota reporting after expunging of old mails
-  ([#233](https://github.com/deltachat/chatmail/pull/239)
+  ([#233](https://github.com/chatmail/server/pull/239)

README.md

@@ -34,8 +34,8 @@ Please substitute it with your own domain.
     scripts/cmdeploy init chat.example.org  # <-- use your domain 
    ```
 
-3. Setup first DNS records for your chatmail domain, 
-   according to the hints provided by `cmdeploy init`.
+3. Point your domain to the server's IP address,
+   if you haven't done so already.
    Verify that SSH root login works:
 
    ```
@@ -47,7 +47,8 @@ Please substitute it with your own domain.
    ```
     scripts/cmdeploy run
    ```
-   This script will also show you additional DNS records
+   This script will check that you have all necessary DNS records.
+   If DNS records are missing, it will recommend
    which you should configure at your DNS provider
    (it can take some time until they are public).
 
@@ -59,7 +60,7 @@ To check the status of your remotely running chatmail service:
 scripts/cmdeploy status
 ```
 
-To check whether your DNS records are correct:
+To display and check all recommended DNS records:
 
 ```
 scripts/cmdeploy dns
@@ -79,31 +80,99 @@ scripts/cmdeploy bench
 
 ## Overview of this repository
 
-This repository drives the development of chatmail services, 
-comprised of minimal setups of
+This repository has four directories:
 
-- [postfix smtp server](https://www.postfix.org)
-- [dovecot imap server](https://www.dovecot.org)
+- [cmdeploy](https://github.com/deltachat/chatmail/tree/main/cmdeploy)
+  is a collection of configuration files
+  and a [pyinfra](https://pyinfra.com)-based deployment script.
 
-as well as custom services that are integrated with these two:
+- [chatmaild](https://github.com/deltachat/chatmail/tree/main/chatmaild)
+  is a python package containing several small services
+  which handle authentication,
+  trigger push notifications on new messages,
+  ensure that outbound mails are encrypted,
+  delete inactive users,
+  and some other minor things.
+  chatmaild can also be installed as a stand-alone python package.
 
-- `chatmaild/src/chatmaild/doveauth.py` implements
+- [www](https://github.com/deltachat/chatmail/tree/main/www)
+  contains the html, css, and markdown files
+  which make up a chatmail server's web page.
+  Edit them before deploying to make your chatmail server stand out.
+
+- [scripts](https://github.com/deltachat/chatmail/tree/main/scripts)
+  offers two convenience tools for beginners;
+  `initenv.sh` installs the necessary dependencies to a local virtual environment,
+  and the `scripts/cmdeploy` script enables you
+  to run the `cmdeploy` command line tool in the local virtual environment.
+
+### cmdeploy
+
+The `cmdeploy/src/cmdeploy/cmdeploy.py` command line tool
+helps with setting up and managing the chatmail service.
+`cmdeploy init` creates the `chatmail.ini` config file.
+`cmdeploy run` uses a [pyinfra](https://pyinfra.com/)-based [script](`cmdeploy/src/cmdeploy/__init__.py`)
+to automatically install or upgrade all chatmail components on a server,
+according to the `chatmail.ini` config.
+
+The components of chatmail are:
+
+- [postfix smtp server](https://www.postfix.org) accepts sent messages (both from your users and from other servers)
+
+- [dovecot imap server](https://www.dovecot.org) stores messages for your users until they download them
+
+- [nginx](https://nginx.org/) shows the web page with your privacy policy and additional information
+
+- [acmetool](https://hlandau.github.io/acmetool/) manages TLS certificates for dovecot, postfix, and nginx
+
+- [opendkim](http://www.opendkim.org/) for signing messages with DKIM and rejecting inbound messages without DKIM
+
+- [mtail](https://google.github.io/mtail/) for collecting anonymized metrics in case you have monitoring
+
+- and the chatmaild services, explained in the next section:
+
+### chatmaild
+
+chatmaild offers several commands
+which differentiate a *chatmail* server from a classic mail server.
+If you deploy them with cmdeploy,
+they are run by systemd services in the background.
+A short overview:
+
+- [`doveauth`](https://github.com/deltachat/chatmail/blob/main/chatmaild/src/chatmaild/doveauth.py) implements
   create-on-login account creation semantics and is used
   by Dovecot during login authentication and by Postfix
   which in turn uses [Dovecot SASL](https://doc.dovecot.org/configuration_manual/authentication/dict/#complete-example-for-authenticating-via-a-unix-socket)
   to authenticate users
   to send mails for them.
 
-- `chatmaild/src/chatmaild/filtermail.py` prevents
+- [`filtermail`](https://github.com/deltachat/chatmail/blob/main/chatmaild/src/chatmaild/filtermail.py) prevents
   unencrypted e-mail from leaving the chatmail service
   and is integrated into postfix's outbound mail pipelines.
 
-There is also the `cmdeploy/src/cmdeploy/cmdeploy.py` command line tool
-which helps with setting up and managing the chatmail service.
-`cmdeploy run` uses [pyinfra-based scripting](https://pyinfra.com/)
-in `cmdeploy/src/cmdeploy/__init__.py`
-to automatically install all chatmail components on a server.
+- [`chatmail-metadata`](https://github.com/deltachat/chatmail/blob/main/chatmaild/src/chatmaild/metadata.py) is contacted by a
+  [dovecot lua script](https://github.com/deltachat/chatmail/blob/main/cmdeploy/src/cmdeploy/dovecot/push_notification.lua)
+  to store user-specific server-side config.
+  On new messages,
+  it [passes the user's push notification token](https://github.com/deltachat/chatmail/blob/main/chatmaild/src/chatmaild/notifier.py)
+  to [notifications.delta.chat](https://delta.chat/help#instant-delivery)
+  so the push notifications on the user's phone can be triggered
+  by Apple/Google.
 
+- [`delete_inactive_users`](https://github.com/deltachat/chatmail/blob/main/chatmaild/src/chatmaild/delete_inactive_users.py)
+  deletes users if they have not logged in for a very long time.
+  The timeframe can be configured in `chatmail.ini`.
+
+- [`lastlogin`](https://github.com/deltachat/chatmail/blob/main/chatmaild/src/chatmaild/lastlogin.py)
+  is contacted by dovecot when a user logs in
+  and stores the date of the login.
+
+- [`echobot`](https://github.com/deltachat/chatmail/blob/main/chatmaild/src/chatmaild/echo.py)
+  is a small bot for test purposes.
+  It simply echoes back messages from users.
+
+- [`chatmail-metrics`](https://github.com/deltachat/chatmail/blob/main/chatmaild/src/chatmaild/metrics.py)
+  collects some metrics and displays them at `https://example.org/metrics`.
 
 ### Home page and getting started for users
 
@@ -186,3 +255,186 @@ to MAIL FROM with
 and rejects incorrectly authenticated emails with [`reject_sender_login_mismatch`](reject_sender_login_mismatch) policy.
 `From:` header must correspond to envelope MAIL FROM,
 this is ensured by `filtermail` proxy.
+
+## Migrating chatmail server to a new host
+
+If you want to migrate chatmail from an old machine
+to a new machine,
+you can use these steps.
+They were tested with a linux laptop;
+you might need to adjust some of the steps to your environment.
+
+Let's assume that your `mail_domain` is `mail.example.org`,
+all involved machines run Debian 12,
+your old server's IP address is `13.37.13.37`,
+and your new server's IP address is `13.12.23.42`.
+
+During the guide, you might get a warning about changed SSH Host keys;
+in this case, just run `ssh-keygen -R "mail.example.org"` as recommended
+to make sure you can connect with SSH.
+
+1. First, copy `/var/lib/acme` to the new server with
+   `ssh root@13.37.13.37 tar c /var/lib/acme | ssh root@13.12.23.42 tar x -C /var/lib/`.
+   This transfers your TLS certificate.
+
+2. You should also copy `/etc/dkimkeys` to the new server with
+   `ssh root@13.37.13.37 tar c /etc/dkimkeys | ssh root@13.12.23.42 tar x -C /etc/`
+   so the DKIM DNS record stays correct.
+
+3. On the new server, run `chown root: -R /var/lib/acme` and `chown opendkim: -R /etc/dkimkeys` to make sure the permissions are correct.
+
+4. Run `cmdeploy run --disable-mail --ssh-host 13.12.23.42` to install chatmail on the new machine.
+   postfix and dovecot are disabled for now,
+   we will enable them later.
+
+5. Now, point DNS to the new IP addresses.
+
+   You can already remove the old IP addresses from DNS.
+   Existing Delta Chat users will still be able to connect
+   to the old server, send and receive messages,
+   but new users will fail to create new profiles
+   with your chatmail server.
+
+   If other servers try to deliver messages to your new server they will fail,
+   but normally email servers will retry delivering messages
+   for at least a week, so messages will not be lost.
+
+6. Now you can run `cmdeploy run --disable-mail --ssh-host 13.37.13.37` to disable your old server.
+
+   Now your users will notice the migration
+   and will not be able to send or receive messages
+   until the migration is completed.
+
+7. After everything is stopped,
+    you can copy the `/home/vmail/mail` directory to the new server.
+    It includes all user data, messages, password hashes, etc.
+
+    Just run: `ssh root@13.37.13.37 tar c /home/vmail/mail | ssh root@13.12.23.42 tar x -C /home/vmail/`
+
+    After this, your new server has all the necessary files to start operating :)
+
+8. To be sure the permissions are still fine,
+    run `chown vmail: -R /home/vmail` on the new server.
+
+9. Finally, you can run `cmdeploy run` to turn on chatmail on the new server.
+    Your users can continue using the chatmail server,
+    and messages which were sent after step 6. should arrive now.
+    Voilà!
+
+## Setting up a reverse proxy
+
+A chatmail server does not depend on the client IP address
+for its operation, so it can be run behind a reverse proxy.
+This will not even affect incoming mail authentication
+as DKIM only checks the cryptographic signature
+of the message and does not use the IP address as the input.
+
+For example, you may want to self-host your chatmail server
+and only use hosted VPS to provide a public IP address
+for client connections and incoming mail.
+You can connect chatmail server to VPS
+using a tunnel protocol
+such as [WireGuard](https://www.wireguard.com/)
+and setup a reverse proxy on a VPS
+to forward connections to the chatmail server
+over the tunnel.
+You can also setup multiple reverse proxies
+for your chatmail server in different networks
+to ensure your server is reachable even when
+one of the IPs becomes inaccessible due to
+hosting or routing problems.
+
+Note that your server still needs
+to be able to make outgoing connections on port 25
+to send messages outside.
+
+To setup a reverse proxy
+(or rather Destination NAT, DNAT)
+for your chatmail server,
+put the following configuration in `/etc/nftables.conf`:
+```
+#!/usr/sbin/nft -f
+
+flush ruleset
+
+define wan = eth0
+
+# Which ports to proxy.
+#
+# Note that SSH is not proxied
+# so it is possible to log into the proxy server
+# and not the original one.
+define ports = { smtp, http, https, imap, imaps, submission, submissions }
+
+# The host we want to proxy to.
+define ipv4_address = AAA.BBB.CCC.DDD
+define ipv6_address = [XXX::1]
+
+table ip nat {
+        chain prerouting {
+                type nat hook prerouting priority dstnat; policy accept;
+                iif $wan tcp dport $ports dnat to $ipv4_address
+        }
+
+        chain postrouting {
+                type nat hook postrouting priority 0;
+
+                oifname $wan masquerade
+        }
+}
+
+table ip6 nat {
+        chain prerouting {
+                type nat hook prerouting priority dstnat; policy accept;
+                iif $wan tcp dport $ports dnat to $ipv6_address
+        }
+
+        chain postrouting {
+                type nat hook postrouting priority 0;
+
+                oifname $wan masquerade
+        }
+}
+
+table inet filter {
+        chain input {
+                type filter hook input priority filter; policy drop;
+
+                # Accept ICMP.
+                # It is especially important to accept ICMPv6 ND messages,
+                # otherwise IPv6 connectivity breaks.
+                icmp type { echo-request } accept
+                icmpv6 type { echo-request, nd-neighbor-solicit, nd-router-advert, nd-neighbor-advert } accept
+
+                # Allow incoming SSH connections.
+                tcp dport { ssh } accept
+
+                ct state established accept
+        }
+        chain forward {
+                type filter hook forward priority filter; policy drop;
+
+                ct state established accept
+                ip daddr $ipv4_address counter accept
+                ip6 daddr $ipv6_address counter accept
+        }
+        chain output {
+                type filter hook output priority filter;
+        }
+}
+```
+
+Run `systemctl enable nftables.service`
+to ensure configuration is reloaded when the proxy server reboots.
+
+Uncomment in `/etc/sysctl.conf` the following two lines:
+
+```
+net.ipv4.ip_forward=1
+net.ipv6.conf.all.forwarding=1
+```
+
+Then reboot the server or do `sysctl -p` and `nft -f /etc/nftables.conf`.
+
+Once proxy server is set up,
+you can add its IP address to the DNS.

chatmaild/pyproject.toml

@@ -12,6 +12,7 @@ dependencies = [
   "deltachat-rpc-client",
   "filelock",
   "requests",
+  "crypt-r >= 3.13.1 ; python_version >= '3.11'",
 ]
 
 [tool.setuptools]

chatmaild/src/chatmaild/__init__.py

@@ -0,0 +1 @@
+

chatmaild/src/chatmaild/common_encrypted_subjects.py

@@ -1,59 +0,0 @@
-"""Generated from deltachat, draft-ietf-lamps-header-protection, and
-encrypted_subject localizations in
-https://github.com/thunderbird/thunderbird-android/
-
-"""
-
-common_encrypted_subjects = {
-    "...",
-    "[...]",
-    "암호화된 메시지",
-    "Ĉifrita mesaĝo",
-    "Courriel chiffré",
-    "Dulrituð skilaboð",
-    "Encrypted Message",
-    "Fersifere berjocht",
-    "Kemennadenn enrineget",
-    "Krüptitud kiri",
-    "Krypterat meddelande",
-    "Krypteret besked",
-    "Kryptert melding",
-    "Mensagem criptografada",
-    "Mensagem encriptada",
-    "Mensaje cifrado",
-    "Mensaxe cifrada",
-    "Mesaj Criptat",
-    "Mesazh i Fshehtëzuar",
-    "Messaggio criptato",
-    "Messaghju cifratu",
-    "Missatge encriptat",
-    "Neges wedi'i Hamgryptio",
-    "Pesan terenkripsi",
-    "Salattu viesti",
-    "Şifreli İleti",
-    "Šifrēta ziņa",
-    "Šifrirana poruka",
-    "Šifrirano sporočilo",
-    "Šifruotas laiškas",
-    "Tin nhắn được mã hóa",
-    "Titkosított üzenet",
-    "Verschlüsselte Nachricht",
-    "Versleuteld bericht",
-    "Zašifrovaná zpráva",
-    "Zaszyfrowana wiadomość",
-    "Zifratu mezua",
-    "Κρυπτογραφημένο μήνυμα",
-    "Зашифроване повідомлення",
-    "Зашифрованное сообщение",
-    "Зашыфраваны ліст",
-    "Криптирано съобщение",
-    "Шифрована порука",
-    "დაშიფრული წერილი",
-    "הודעה מוצפנת",
-    "پیام رمزنگاری‌شده",
-    "رسالة مشفّرة",
-    "എൻക്രിപ്റ്റുചെയ്‌ത സന്ദേശം",
-    "加密邮件",
-    "已加密的訊息",
-    "暗号化されたメッセージ",
-}

chatmaild/src/chatmaild/config.py

@@ -30,9 +30,15 @@ class Config:
         self.passthrough_recipients = params["passthrough_recipients"].split()
         self.filtermail_smtp_port = int(params["filtermail_smtp_port"])
         self.postfix_reinject_port = int(params["postfix_reinject_port"])
+        self.mtail_address = params.get("mtail_address")
         self.disable_ipv6 = params.get("disable_ipv6", "false").lower() == "true"
         self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true"
-        self.iroh_relay = params.get("iroh_relay")
+        if "iroh_relay" not in params:
+            self.iroh_relay = "https://" + params["mail_domain"]
+            self.enable_iroh_relay = True
+        else:
+            self.iroh_relay = params["iroh_relay"].strip()
+            self.enable_iroh_relay = False
         self.privacy_postal = params.get("privacy_postal")
         self.privacy_mail = params.get("privacy_mail")
         self.privacy_pdo = params.get("privacy_pdo")

chatmaild/src/chatmaild/dictproxy.py

@@ -87,8 +87,12 @@ class DictProxy:
         except FileNotFoundError:
             pass
 
-        with ThreadingUnixStreamServer(socket, Handler) as server:
+        with CustomThreadingUnixStreamServer(socket, Handler) as server:
             try:
                 server.serve_forever()
             except KeyboardInterrupt:
                 pass
+
+
+class CustomThreadingUnixStreamServer(ThreadingUnixStreamServer):
+    request_queue_size = 1000

chatmaild/src/chatmaild/doveauth.py

@@ -1,9 +1,13 @@
-import crypt
 import json
 import logging
 import os
 import sys
 
+try:
+    import crypt_r
+except ImportError:
+    import crypt as crypt_r
+
 from .config import Config, read_config
 from .dictproxy import DictProxy
 from .migrate_db import migrate_from_db_to_maildir
@@ -13,7 +17,7 @@ NOCREATE_FILE = "/etc/chatmail-nocreate"
 
 def encrypt_password(password: str):
     # https://doc.dovecot.org/configuration_manual/authentication/password_schemes/
-    passhash = crypt.crypt(password, crypt.METHOD_SHA512)
+    passhash = crypt_r.crypt(password, crypt_r.METHOD_SHA512)
     return "{SHA512-CRYPT}" + passhash
 
 

chatmaild/src/chatmaild/filtermail.py

@@ -12,7 +12,6 @@ from smtplib import SMTP as SMTPClient
 
 from aiosmtpd.controller import Controller
 
-from .common_encrypted_subjects import common_encrypted_subjects
 from .config import read_config
 
 
@@ -60,10 +59,11 @@ def check_openpgp_payload(payload: bytes):
         i += body_len
 
         if i == len(payload):
-            if packet_type_id == 18:
-                # Last packet should be
-                # Symmetrically Encrypted and Integrity Protected Data Packet (SEIPD)
-                return True
+            # Last packet should be
+            # Symmetrically Encrypted and Integrity Protected Data Packet (SEIPD)
+            #
+            # This is the only place where this function may return `True`.
+            return packet_type_id == 18
         elif packet_type_id not in [1, 3]:
             # All packets except the last one must be either
             # Public-Key Encrypted Session Key Packet (PKESK)
@@ -71,13 +71,7 @@ def check_openpgp_payload(payload: bytes):
             # Symmetric-Key Encrypted Session Key Packet (SKESK)
             return False
 
-    if i == 0:
-        return False
-
-    if i > len(payload):
-        # Payload is truncated.
-        return False
-    return True
+    return False
 
 
 def check_armored_payload(payload: str):
@@ -86,7 +80,9 @@ def check_armored_payload(payload: str):
         return False
     payload = payload.removeprefix(prefix)
 
-    suffix = "-----END PGP MESSAGE-----\r\n\r\n"
+    while payload.endswith("\r\n"):
+        payload = payload.removesuffix("\r\n")
+    suffix = "-----END PGP MESSAGE-----"
     if not payload.endswith(suffix):
         return False
     payload = payload.removesuffix(suffix)
@@ -105,6 +101,27 @@ def check_armored_payload(payload: str):
         return False
 
 
+def is_securejoin(message):
+    if message.get("secure-join") not in ["vc-request", "vg-request"]:
+        return False
+    if not message.is_multipart():
+        return False
+    parts_count = 0
+    for part in message.iter_parts():
+        parts_count += 1
+        if parts_count > 1:
+            return False
+        if part.is_multipart():
+            return False
+        if part.get_content_type() != "text/plain":
+            return False
+
+        payload = part.get_payload().strip().lower()
+        if payload not in ("secure-join: vc-request", "secure-join: vg-request"):
+            return False
+    return True
+
+
 def check_encrypted(message):
     """Check that the message is an OpenPGP-encrypted message.
 
@@ -112,8 +129,6 @@ def check_encrypted(message):
     """
     if not message.is_multipart():
         return False
-    if message.get("subject") not in common_encrypted_subjects:
-        return False
     if message.get_content_type() != "multipart/encrypted":
         return False
     parts_count = 0
@@ -147,6 +162,15 @@ async def asyncmain_beforequeue(config):
     Controller(BeforeQueueHandler(config), hostname="127.0.0.1", port=port).start()
 
 
+def recipient_matches_passthrough(recipient, passthrough_recipients):
+    for addr in passthrough_recipients:
+        if recipient == addr:
+            return True
+        if addr[0] == "@" and recipient.endswith(addr):
+            return True
+    return False
+
+
 class BeforeQueueHandler:
     def __init__(self, config):
         self.config = config
@@ -183,20 +207,30 @@ class BeforeQueueHandler:
         mail_encrypted = check_encrypted(message)
 
         _, from_addr = parseaddr(message.get("from").strip())
+        envelope_from_domain = from_addr.split("@").pop()
+
         logging.info(f"mime-from: {from_addr} envelope-from: {envelope.mail_from!r}")
         if envelope.mail_from.lower() != from_addr.lower():
             return f"500 Invalid FROM <{from_addr!r}> for <{envelope.mail_from!r}>"
 
+        if mail_encrypted:
+            print("Filtering encrypted mail.", file=sys.stderr)
+        else:
+            print("Filtering unencrypted mail.", file=sys.stderr)
+
         if envelope.mail_from in self.config.passthrough_senders:
             return
 
         passthrough_recipients = self.config.passthrough_recipients
-        envelope_from_domain = from_addr.split("@").pop()
+
+        if mail_encrypted or is_securejoin(message):
+            return
+
         for recipient in envelope.rcpt_tos:
             if envelope.mail_from == recipient:
                 # Always allow sending emails to self.
                 continue
-            if recipient in passthrough_recipients:
+            if recipient_matches_passthrough(recipient, passthrough_recipients):
                 continue
             res = recipient.split("@")
             if len(res) != 2:
@@ -204,13 +238,9 @@ class BeforeQueueHandler:
             _recipient_addr, recipient_domain = res
 
             is_outgoing = recipient_domain != envelope_from_domain
-            if is_outgoing and not mail_encrypted:
-                is_securejoin = message.get("secure-join") in [
-                    "vc-request",
-                    "vg-request",
-                ]
-                if not is_securejoin:
-                    return f"500 Invalid unencrypted mail to <{recipient}>"
+            if is_outgoing:
+                print("Rejected unencrypted mail.", file=sys.stderr)
+                return f"500 Invalid unencrypted mail to <{recipient}>"
 
 
 class SendRateLimiter:

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -23,8 +23,8 @@ max_message_size = 31457280
 # days after which mails are unconditionally deleted
 delete_mails_after = 20
 
-# days after which users without a login are deleted (database and mails)
-delete_inactive_users_after = 100
+# days after which users without a successful login are deleted (database and mails)
+delete_inactive_users_after = 90
 
 # minimum length a username must have
 username_min_length = 9
@@ -39,7 +39,7 @@ password_min_length = 9
 passthrough_senders =
 
 # list of e-mail recipients for which to accept outbound un-encrypted mails
-# (space-separated)
+# (space-separated, item may start with "@" to whitelist whole recipient domains)
 passthrough_recipients = xstore@testrun.org 
 
 #
@@ -55,6 +55,29 @@ postfix_reinject_port = 10025
 # if set to "True" IPv6 is disabled
 disable_ipv6 = False
 
+# Defaults to https://iroh.{{mail_domain}} and running `iroh-relay` on the chatmail
+# service.
+# If you set it to anything else, the service will be disabled
+# and users will be directed to use the given iroh relay URL.
+# Set it to empty string if you want users to use their default iroh relay.
+# iroh_relay =
+
+# Address on which `mtail` listens,
+# e.g. 127.0.0.1 or some private network
+# address like 192.168.10.1.
+# You can point Prometheus
+# or some other OpenMetrics-compatible
+# collector to
+# http://{{mtail_address}}:3903/metrics
+# and display collected metrics with Grafana.
+#
+# WARNING: do not expose this service
+# to the public IP address.
+#
+# `mtail is not running if the setting is not set.
+
+# mtail_address = 127.0.0.1
+
 #
 # Debugging options 
 #

chatmaild/src/chatmaild/migrate_db.py

@@ -32,7 +32,7 @@ def migrate_from_db_to_maildir(config, chunking=10000):
     # don't transfer special/CI accounts
     rows = [row for row in all_rows if row[0][:3] not in ("ci-", "ac_")]
 
-    logging.info(f"ignoring {len(all_rows)-len(rows)} CI accounts")
+    logging.info(f"ignoring {len(all_rows) - len(rows)} CI accounts")
     logging.info(f"migrating {len(rows)} sqlite database passwords to user dirs")
 
     for i, row in enumerate(rows):

chatmaild/src/chatmaild/tests/mail-data/literal.eml

@@ -1,44 +1,44 @@
-From: {from_addr}
-
-To: {to_addr}
-
-Subject: ...
-
-Date: Sun, 15 Oct 2023 16:43:21 +0000
-
-Message-ID: <Mr.UVyJWZmkCKM.hGzNc6glBE_@c2.testrun.org>
-
-In-Reply-To: <Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>
-
-References: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>
-
-	<Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>
-
-Chat-Version: 1.0
-
-Autocrypt: addr={from_addr}; prefer-encrypt=mutual;
-
-	keydata=xjMEZSwWjhYJKwYBBAHaRw8BAQdAQBEhqeJh0GueHB6kF/DUQqYCxARNBVokg/AzT+7LqH
-
-	rNFzxiYXJiYXpAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUsFo4CGwMECwkIBwYVCAkKCwID
-
-	FgIBFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX9A4AEAnHWHp49eBCMHK5t66gYPiW
-
-	XQuB1mwUjzGfYWB+0RXUoA/0xcQ3FbUNlGKW7Blp6eMFfViv6Mv2d3kNSXACB6nmcMzjgEZSwWjhIK
-
-	KwYBBAGXVQEFAQEHQBpY5L2M1XHo0uxf8SX1wNLBp/OVvidoWHQF2Jz+kJsUAwEIB8J4BBgWCAAgBQ
-
-	JlLBaOAhsMFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX/INgEA37AJaNvruYsJVanP
-
-	IXnYw4CKd55UAwl8Zcy+M2diAbkA/0fHHcGV4r78hpbbL1Os52DPOdqYQRauIeJUeG+G6bQO
-
-MIME-Version: 1.0
-
-Content-Type: multipart/encrypted; protocol="application/pgp-encrypted";
-
-	boundary="YFrteb74qSXmggbOxZL9dRnhymywAi"
-
-
-
-
+From: {from_addr}
+To: {to_addr}
+Subject: ...
+Date: Sun, 15 Oct 2023 16:43:21 +0000
+Message-ID: <Mr.UVyJWZmkCKM.hGzNc6glBE_@c2.testrun.org>
+In-Reply-To: <Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>
+References: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>
+	<Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>
+Chat-Version: 1.0
+Autocrypt: addr={from_addr}; prefer-encrypt=mutual;
+	keydata=xjMEZSwWjhYJKwYBBAHaRw8BAQdAQBEhqeJh0GueHB6kF/DUQqYCxARNBVokg/AzT+7LqH
+	rNFzxiYXJiYXpAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUsFo4CGwMECwkIBwYVCAkKCwID
+	FgIBFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX9A4AEAnHWHp49eBCMHK5t66gYPiW
+	XQuB1mwUjzGfYWB+0RXUoA/0xcQ3FbUNlGKW7Blp6eMFfViv6Mv2d3kNSXACB6nmcMzjgEZSwWjhIK
+	KwYBBAGXVQEFAQEHQBpY5L2M1XHo0uxf8SX1wNLBp/OVvidoWHQF2Jz+kJsUAwEIB8J4BBgWCAAgBQ
+	JlLBaOAhsMFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX/INgEA37AJaNvruYsJVanP
+	IXnYw4CKd55UAwl8Zcy+M2diAbkA/0fHHcGV4r78hpbbL1Os52DPOdqYQRauIeJUeG+G6bQO
+MIME-Version: 1.0
+Content-Type: multipart/encrypted; protocol="application/pgp-encrypted";
+	boundary="YFrteb74qSXmggbOxZL9dRnhymywAi"
+
+
+--YFrteb74qSXmggbOxZL9dRnhymywAi
+Content-Description: PGP/MIME version identification
+Content-Type: application/pgp-encrypted
+
+Version: 1
+
+
+--YFrteb74qSXmggbOxZL9dRnhymywAi
+Content-Description: OpenPGP encrypted message
+Content-Disposition: inline; filename="encrypted.asc";
+Content-Type: application/octet-stream; name="encrypted.asc"
+
+-----BEGIN PGP MESSAGE-----
+
+yxJiAAAAAABIZWxsbyB3b3JsZCE=
+=1I/B
+-----END PGP MESSAGE-----
+
+
+--YFrteb74qSXmggbOxZL9dRnhymywAi--
+
 

chatmaild/src/chatmaild/tests/mail-data/securejoin-vc-fake.eml

@@ -0,0 +1,21 @@
+Subject: Message from {from_addr}
+From: <{from_addr}>
+To: <{to_addr}>
+Date: Sun, 15 Oct 2023 16:43:25 +0000
+Message-ID: <Mr.78MWtlV7RAi.goCFzBhCYfy@c2.testrun.org>
+Chat-Version: 1.0
+Secure-Join: vc-request
+Secure-Join-Invitenumber: RANDOM-TOKEN
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi"
+
+
+--Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi
+Content-Type: text/plain; charset=utf-8
+
+Buy viagra!
+
+
+--Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi--
+
+

chatmaild/src/chatmaild/tests/mail-data/securejoin-vc.eml

@@ -0,0 +1,21 @@
+Subject: Message from {from_addr}
+From: <{from_addr}>
+To: <{to_addr}>
+Date: Sun, 15 Oct 2023 16:43:25 +0000
+Message-ID: <Mr.78MWtlV7RAi.goCFzBhCYfy@c2.testrun.org>
+Chat-Version: 1.0
+Secure-Join: vc-request
+Secure-Join-Invitenumber: RANDOM-TOKEN
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi"
+
+
+--Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi
+Content-Type: text/plain; charset=utf-8
+
+Secure-Join: vc-request
+
+
+--Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi--
+
+

chatmaild/src/chatmaild/tests/plugin.py

@@ -69,7 +69,7 @@ def maildata(request):
 
     assert datadir.exists(), datadir
 
-    def maildata(name, from_addr, to_addr, subject="..."):
+    def maildata(name, from_addr, to_addr, subject="[...]"):
         # Using `.read_bytes().decode()` instead of `.read_text()` to preserve newlines.
         data = datadir.joinpath(name).read_bytes().decode()
 

chatmaild/src/chatmaild/tests/test_filtermail.py

@@ -5,7 +5,7 @@ from chatmaild.filtermail import (
     SendRateLimiter,
     check_armored_payload,
     check_encrypted,
-    common_encrypted_subjects,
+    is_securejoin,
 )
 
 
@@ -55,19 +55,28 @@ def test_filtermail_no_encryption_detection(maildata):
     assert not check_encrypted(msg)
 
 
-def test_filtermail_encryption_detection(maildata):
-    for subject in common_encrypted_subjects:
-        msg = maildata(
-            "encrypted.eml",
-            from_addr="1@example.org",
-            to_addr="2@example.org",
-            subject=subject,
-        )
-        assert check_encrypted(msg)
+def test_filtermail_securejoin_detection(maildata):
+    msg = maildata(
+        "securejoin-vc.eml", from_addr="some@example.org", to_addr="other@example.org"
+    )
+    assert is_securejoin(msg)
 
-    # if the subject is not a known encrypted subject value, it is not considered ac-encrypted
-    msg.replace_header("Subject", "Click this link")
-    assert not check_encrypted(msg)
+    msg = maildata(
+        "securejoin-vc-fake.eml",
+        from_addr="some@example.org",
+        to_addr="other@example.org",
+    )
+    assert not is_securejoin(msg)
+
+
+def test_filtermail_encryption_detection(maildata):
+    msg = maildata(
+        "encrypted.eml",
+        from_addr="1@example.org",
+        to_addr="2@example.org",
+        subject="Subject does not matter, will be replaced anyway",
+    )
+    assert check_encrypted(msg)
 
 
 def test_filtermail_no_literal_packets(maildata):
@@ -121,6 +130,30 @@ def test_excempt_privacy(maildata, gencreds, handler):
     assert "500" in handler.check_DATA(envelope=env2)
 
 
+def test_passthrough_domains(maildata, gencreds, handler):
+    from_addr = gencreds()[0]
+    to_addr = "privacy@x.y.z"
+    handler.config.passthrough_recipients = ["@x.y.z"]
+    false_to = "something@x.y"
+
+    msg = maildata("plain.eml", from_addr=from_addr, to_addr=to_addr)
+
+    class env:
+        mail_from = from_addr
+        rcpt_tos = [to_addr]
+        content = msg.as_bytes()
+
+    # assert that None/no error is returned
+    assert not handler.check_DATA(envelope=env)
+
+    class env2:
+        mail_from = from_addr
+        rcpt_tos = [to_addr, false_to]
+        content = msg.as_bytes()
+
+    assert "500" in handler.check_DATA(envelope=env2)
+
+
 def test_passthrough_senders(gencreds, handler, maildata):
     acc1 = gencreds()[0]
     to_addr = "recipient@something.org"
@@ -172,10 +205,20 @@ UN4fiB0KR9JyG2ayUdNJVkXZSZLnHyRgiaadlpUo16LVvw==\r
 =b5Kp\r
 -----END PGP MESSAGE-----\r
 \r
+\r
 """
 
     assert check_armored_payload(payload) == True
 
+    payload = payload.removesuffix("\r\n")
+    assert check_armored_payload(payload) == True
+
+    payload = payload.removesuffix("\r\n")
+    assert check_armored_payload(payload) == True
+
+    payload = payload.removesuffix("\r\n")
+    assert check_armored_payload(payload) == True
+
     payload = """-----BEGIN PGP MESSAGE-----\r
 \r
 HELLOWORLD

cmdeploy/src/cmdeploy/__init__.py

@@ -10,7 +10,7 @@ import sys
 from pathlib import Path
 
 from chatmaild.config import Config, read_config
-from pyinfra import host
+from pyinfra import facts, host
 from pyinfra.facts.files import File
 from pyinfra.facts.systemd import SystemdEnabled
 from pyinfra.operations import apt, files, pip, server, systemd
@@ -78,6 +78,11 @@ def _install_remote_venv_with_chatmaild(config) -> None:
         always_copy=True,
     )
 
+    apt.packages(
+        name="install gcc and headers to build crypt_r source package",
+        packages=["gcc", "python3-dev"],
+    )
+
     server.shell(
         name=f"forced pip-install {dist_file.name}",
         commands=[
@@ -212,49 +217,36 @@ def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
             commands=[
                 f"opendkim-genkey -D /etc/dkimkeys -d {domain} -s {dkim_selector}"
             ],
-            _sudo=True,
-            _sudo_user="opendkim",
+            _use_su_login=True,
+            _su_user="opendkim",
         )
 
+    service_file = files.put(
+        name="Configure opendkim to restart once a day",
+        src=importlib.resources.files(__package__).joinpath("opendkim/systemd.conf"),
+        dest="/etc/systemd/system/opendkim.service.d/10-prevent-memory-leak.conf",
+    )
+    need_restart |= service_file.changed
+
+
     return need_restart
 
 
-def _install_mta_sts_daemon() -> bool:
-    need_restart = False
+def _uninstall_mta_sts_daemon() -> None:
+    # Remove configuration.
+    files.file("/etc/mta-sts-daemon.yml", present=False)
 
-    config = files.put(
-        name="upload postfix-mta-sts-resolver config",
-        src=importlib.resources.files(__package__).joinpath(
-            "postfix/mta-sts-daemon.yml"
-        ),
-        dest="/etc/mta-sts-daemon.yml",
-        user="root",
-        group="root",
-        mode="644",
+    files.directory("/usr/local/lib/postfix-mta-sts-resolver", present=False)
+
+    files.file("/etc/systemd/system/mta-sts-daemon.service", present=False)
+
+    systemd.service(
+        name="Stop MTA-STS daemon",
+        service="mta-sts-daemon.service",
+        daemon_reload=True,
+        running=False,
+        enabled=False,
     )
-    need_restart |= config.changed
-
-    server.shell(
-        name="install postfix-mta-sts-resolver with pip",
-        commands=[
-            "python3 -m virtualenv /usr/local/lib/postfix-mta-sts-resolver",
-            "/usr/local/lib/postfix-mta-sts-resolver/bin/pip install postfix-mta-sts-resolver",
-        ],
-    )
-
-    systemd_unit = files.put(
-        name="upload mta-sts-daemon systemd unit",
-        src=importlib.resources.files(__package__).joinpath(
-            "postfix/mta-sts-daemon.service"
-        ),
-        dest="/etc/systemd/system/mta-sts-daemon.service",
-        user="root",
-        group="root",
-        mode="644",
-    )
-    need_restart |= systemd_unit.changed
-
-    return need_restart
 
 
 def _configure_postfix(config: Config, debug: bool = False) -> bool:
@@ -441,10 +433,105 @@ def check_config(config):
     return config
 
 
-def deploy_chatmail(config_path: Path) -> None:
+def deploy_mtail(config):
+    apt.packages(
+        name="Install mtail",
+        packages=["mtail"],
+    )
+
+    # Using our own systemd unit instead of `/usr/lib/systemd/system/mtail.service`.
+    # This allows to read from journalctl instead of log files.
+    files.template(
+        src=importlib.resources.files(__package__).joinpath("mtail/mtail.service.j2"),
+        dest="/etc/systemd/system/mtail.service",
+        user="root",
+        group="root",
+        mode="644",
+        address=config.mtail_address or "127.0.0.1",
+        port=3903,
+    )
+
+    mtail_conf = files.put(
+        name="Mtail configuration",
+        src=importlib.resources.files(__package__).joinpath(
+            "mtail/delivered_mail.mtail"
+        ),
+        dest="/etc/mtail/delivered_mail.mtail",
+        user="root",
+        group="root",
+        mode="644",
+    )
+
+    systemd.service(
+        name="Start and enable mtail",
+        service="mtail.service",
+        running=bool(config.mtail_address),
+        enabled=bool(config.mtail_address),
+        restarted=mtail_conf.changed,
+    )
+
+
+def deploy_iroh_relay(config) -> None:
+    (url, sha256sum) = {
+        "x86_64": (
+            "https://github.com/n0-computer/iroh/releases/download/v0.28.1/iroh-relay-v0.28.1-x86_64-unknown-linux-musl.tar.gz",
+            "2ffacf7c0622c26b67a5895ee8e07388769599f60e5f52a3bd40a3258db89b2c",
+        ),
+        "aarch64": (
+            "https://github.com/n0-computer/iroh/releases/download/v0.28.1/iroh-relay-v0.28.1-aarch64-unknown-linux-musl.tar.gz",
+            "b915037bcc1ff1110cc9fcb5de4a17c00ff576fd2f568cd339b3b2d54c420dc4",
+        ),
+    }[host.get_fact(facts.server.Arch)]
+
+    apt.packages(
+        name="Install curl",
+        packages=["curl"],
+    )
+
+    server.shell(
+        name="Download iroh-relay",
+        commands=[
+            f"(echo '{sha256sum} /usr/local/bin/iroh-relay' | sha256sum -c) || (curl -L {url} | gunzip | tar -x -f - ./iroh-relay -O >/usr/local/bin/iroh-relay.new && mv /usr/local/bin/iroh-relay.new /usr/local/bin/iroh-relay)",
+            "chmod 755 /usr/local/bin/iroh-relay",
+        ],
+    )
+
+    need_restart = False
+
+    systemd_unit = files.put(
+        name="Upload iroh-relay systemd unit",
+        src=importlib.resources.files(__package__).joinpath("iroh-relay.service"),
+        dest="/etc/systemd/system/iroh-relay.service",
+        user="root",
+        group="root",
+        mode="644",
+    )
+    need_restart |= systemd_unit.changed
+
+    iroh_config = files.put(
+        name="Upload iroh-relay config",
+        src=importlib.resources.files(__package__).joinpath("iroh-relay.toml"),
+        dest="/etc/iroh-relay.toml",
+        user="root",
+        group="root",
+        mode="644",
+    )
+    need_restart |= iroh_config.changed
+
+    systemd.service(
+        name="Start and enable iroh-relay",
+        service="iroh-relay.service",
+        running=True,
+        enabled=config.enable_iroh_relay,
+        restarted=need_restart,
+    )
+
+
+def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
     """Deploy a chat-mail instance.
 
     :param config_path: path to chatmail.ini
+    :param disable_mail: whether to disable postfix & dovecot
     """
     config = read_config(config_path)
     check_config(config)
@@ -469,6 +556,7 @@ def deploy_chatmail(config_path: Path) -> None:
         system=True,
     )
     server.user(name="Create echobot user", user="echobot", system=True)
+    server.user(name="Create iroh user", user="iroh", system=True)
 
     # Add our OBS repository for dovecot_no_delay
     files.put(
@@ -517,9 +605,12 @@ def deploy_chatmail(config_path: Path) -> None:
         enabled=True,
     )
 
+    deploy_iroh_relay(config)
+
     # Deploy acmetool to have TLS certificates.
+    tls_domains = [mail_domain, f"mta-sts.{mail_domain}", f"www.{mail_domain}"]
     deploy_acmetool(
-        domains=[mail_domain, f"mta-sts.{mail_domain}", f"www.{mail_domain}"],
+        domains=tls_domains,
     )
 
     apt.packages(
@@ -559,8 +650,8 @@ def deploy_chatmail(config_path: Path) -> None:
     debug = False
     dovecot_need_restart = _configure_dovecot(config, debug=debug)
     postfix_need_restart = _configure_postfix(config, debug=debug)
-    mta_sts_need_restart = _install_mta_sts_daemon()
     nginx_need_restart = _configure_nginx(config)
+    _uninstall_mta_sts_daemon()
 
     _remove_rspamd()
     opendkim_need_restart = _configure_opendkim(mail_domain, "opendkim")
@@ -570,35 +661,27 @@ def deploy_chatmail(config_path: Path) -> None:
         service="opendkim.service",
         running=True,
         enabled=True,
+        daemon_reload=opendkim_need_restart,
         restarted=opendkim_need_restart,
     )
 
-    systemd.service(
-        name="Start and enable MTA-STS daemon",
-        service="mta-sts-daemon.service",
-        daemon_reload=True,
-        running=True,
-        enabled=True,
-        restarted=mta_sts_need_restart,
-    )
-
     # Dovecot should be started before Postfix
     # because it creates authentication socket
     # required by Postfix.
     systemd.service(
-        name="Start and enable Dovecot",
+        name="disable dovecot for now" if disable_mail else "Start and enable Dovecot",
         service="dovecot.service",
-        running=True,
-        enabled=True,
-        restarted=dovecot_need_restart,
+        running=False if disable_mail else True,
+        enabled=False if disable_mail else True,
+        restarted=dovecot_need_restart if not disable_mail else False,
     )
 
     systemd.service(
-        name="Start and enable Postfix",
+        name="disable postfix for now" if disable_mail else "Start and enable Postfix",
         service="postfix.service",
-        running=True,
-        enabled=True,
-        restarted=postfix_need_restart,
+        running=False if disable_mail else True,
+        enabled=False if disable_mail else True,
+        restarted=postfix_need_restart if not disable_mail else False,
     )
 
     systemd.service(
@@ -631,8 +714,15 @@ def deploy_chatmail(config_path: Path) -> None:
         enabled=True,
         restarted=journald_conf.changed,
     )
+    files.directory(
+        name="Ensure old logs on disk are deleted",
+        path="/var/log/journal/",
+        present=False,
+    )
 
     apt.packages(
         name="Ensure cron is installed",
         packages=["cron"],
     )
+
+    deploy_mtail(config)

cmdeploy/src/cmdeploy/acmetool/__init__.py

@@ -70,6 +70,6 @@ def deploy_acmetool(email="", domains=[]):
     )
 
     server.shell(
-        name=f"Request certificate for: { ', '.join(domains) }",
-        commands=[f"acmetool want --xlog.severity=debug { ' '.join(domains)}"],
+        name=f"Request certificate for: {', '.join(domains)}",
+        commands=[f"acmetool want --xlog.severity=debug {' '.join(domains)}"],
     )

cmdeploy/src/cmdeploy/acmetool/response-file.yaml.j2

@@ -1,2 +1,2 @@
 "acme-enter-email": "{{ email }}"
-"acme-agreement:https://letsencrypt.org/documents/LE-SA-v1.4-April-3-2024.pdf": true
+"acme-agreement:https://letsencrypt.org/documents/LE-SA-v1.5-February-24-2025.pdf": true

cmdeploy/src/cmdeploy/chatmail.zone.j2

@@ -16,11 +16,11 @@ www.{{ mail_domain }}.               CNAME {{ mail_domain }}.
 ;
 ; Recommended DNS entries for interoperability and security-hardening
 ;
-{{ mail_domain }}.                   TXT "v=spf1 a:{{ mail_domain }} ~all"
+{{ mail_domain }}.                   TXT "v=spf1 a ~all"
 _dmarc.{{ mail_domain }}.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
 
 {% if acme_account_url %}
-{{ mail_domain }}.                   CAA 128 issue "letsencrypt.org;accounturi={{ acme_account_url }}"
+{{ mail_domain }}.                   CAA 0 issue "letsencrypt.org;accounturi={{ acme_account_url }}"
 {% endif %}
 _adsp._domainkey.{{ mail_domain }}.  TXT "dkim=discardable"
 

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -18,7 +18,7 @@ from chatmaild.config import read_config, write_initial_config
 from packaging import version
 from termcolor import colored
 
-from . import dns, remote_funcs
+from . import dns, remote
 from .sshexec import SSHExec
 
 #
@@ -52,21 +52,36 @@ def run_cmd_options(parser):
         action="store_true",
         help="don't actually modify the server",
     )
+    parser.add_argument(
+        "--disable-mail",
+        dest="disable_mail",
+        action="store_true",
+        help="install/upgrade the server, but disable postfix & dovecot for now",
+    )
+    parser.add_argument(
+        "--ssh-host",
+        dest="ssh_host",
+        help="specify an SSH host to deploy to; uses mail_domain from chatmail.ini by default",
+    )
 
 
 def run_cmd(args, out):
     """Deploy chatmail services on the remote server."""
 
     sshexec = args.get_sshexec()
+    require_iroh = args.config.enable_iroh_relay
     remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
     if not dns.check_initial_remote_data(remote_data, print=out.red):
         return 1
 
     env = os.environ.copy()
     env["CHATMAIL_INI"] = args.inipath
+    env["CHATMAIL_DISABLE_MAIL"] = "True" if args.disable_mail else ""
+    env["CHATMAIL_REQUIRE_IROH"] = "True" if require_iroh else ""
     deploy_path = importlib.resources.files(__package__).joinpath("deploy.py").resolve()
     pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
-    cmd = f"{pyinf} --ssh-user root {args.config.mail_domain} {deploy_path} -y"
+    ssh_host = args.config.mail_domain if not args.ssh_host else args.ssh_host
+    cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y"
     if version.parse(pyinfra.__version__) < version.parse("3"):
         out.red("Please re-run scripts/initenv.sh to update pyinfra to version 3.")
         return 1
@@ -132,7 +147,7 @@ def status_cmd(args, out):
     else:
         out.red("no privacy settings")
 
-    for line in sshexec(remote_funcs.get_systemd_running):
+    for line in sshexec(remote.rshell.get_systemd_running):
         print(line)
 
 
@@ -313,7 +328,7 @@ def main(args=None):
 
     def get_sshexec():
         print(f"[ssh] login to {args.config.mail_domain}")
-        return SSHExec(args.config.mail_domain, remote_funcs, verbose=args.verbose)
+        return SSHExec(args.config.mail_domain, verbose=args.verbose)
 
     args.get_sshexec = get_sshexec
 

cmdeploy/src/cmdeploy/deploy.py

@@ -11,8 +11,9 @@ def main():
         "CHATMAIL_INI",
         importlib.resources.files("cmdeploy").joinpath("../../../chatmail.ini"),
     )
+    disable_mail = bool(os.environ.get("CHATMAIL_DISABLE_MAIL"))
 
-    deploy_chatmail(config_path)
+    deploy_chatmail(config_path, disable_mail)
 
 
 if pyinfra.is_cli:

cmdeploy/src/cmdeploy/dns.py

@@ -3,22 +3,25 @@ import importlib
 
 from jinja2 import Template
 
-from . import remote_funcs
+from . import remote
 
 
 def get_initial_remote_data(sshexec, mail_domain):
     return sshexec.logged(
-        call=remote_funcs.perform_initial_checks, kwargs=dict(mail_domain=mail_domain)
+        call=remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=mail_domain)
     )
 
 
-def check_initial_remote_data(remote_data, print=print):
+def check_initial_remote_data(remote_data, *, print=print):
     mail_domain = remote_data["mail_domain"]
     if not remote_data["A"] and not remote_data["AAAA"]:
         print(f"Missing A and/or AAAA DNS records for {mail_domain}!")
-    elif not remote_data["MTA_STS"]:
+    elif remote_data["MTA_STS"] != f"{mail_domain}.":
         print("Missing MTA-STS CNAME record:")
-        print(f"mta-sts.{mail_domain}.   CNAME  {mail_domain}")
+        print(f"mta-sts.{mail_domain}.   CNAME  {mail_domain}.")
+    elif remote_data["WWW"] != f"{mail_domain}.":
+        print("Missing www CNAME record:")
+        print(f"www.{mail_domain}.   CNAME  {mail_domain}.")
     else:
         return remote_data
 
@@ -26,7 +29,7 @@ def check_initial_remote_data(remote_data, print=print):
 def get_filled_zone_file(remote_data):
     sts_id = remote_data.get("sts_id")
     if not sts_id:
-        sts_id = datetime.datetime.now().strftime("%Y%m%d%H%M")
+        remote_data["sts_id"] = datetime.datetime.now().strftime("%Y%m%d%H%M")
 
     template = importlib.resources.files(__package__).joinpath("chatmail.zone.j2")
     content = template.read_text()
@@ -42,19 +45,27 @@ def check_full_zone(sshexec, remote_data, out, zonefile) -> int:
     and return (exitcode, remote_data) tuple."""
 
     required_diff, recommended_diff = sshexec.logged(
-        remote_funcs.check_zonefile, kwargs=dict(zonefile=zonefile)
+        remote.rdns.check_zonefile,
+        kwargs=dict(zonefile=zonefile, mail_domain=remote_data["mail_domain"]),
     )
 
+    returncode = 0
     if required_diff:
         out.red("Please set required DNS entries at your DNS provider:\n")
         for line in required_diff:
             out(line)
-        return 1
-    elif recommended_diff:
+        out("")
+        returncode = 1
+    if remote_data.get("dkim_entry") in required_diff:
+        out(
+            "If the DKIM entry above does not work with your DNS provider, you can try this one:\n"
+        )
+        out(remote_data.get("web_dkim_entry") + "\n")
+    if recommended_diff:
         out("WARNING: these recommended DNS entries are not set:\n")
         for line in recommended_diff:
             out(line)
-        return 0
 
-    out.green("Great! All your DNS entries are verified and correct.")
-    return 0
+    if not (recommended_diff or required_diff):
+        out.green("Great! All your DNS entries are verified and correct.")
+    return returncode

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -51,10 +51,7 @@ mail_server_comment = Chatmail server
 # <https://doc.dovecot.org/configuration_manual/quota_plugin/>
 mail_plugins = zlib quota
 
-# these are the capabilities Delta Chat cares about actually 
-# so let's keep the network overhead per login small
-# https://github.com/deltachat/deltachat-core-rust/blob/master/src/imap/capabilities.rs
-imap_capability = IMAP4rev1 IDLE MOVE QUOTA CONDSTORE NOTIFY METADATA XDELTAPUSH XCHATMAIL
+imap_capability = +XDELTAPUSH XCHATMAIL
 
 
 # Authentication for system users.
@@ -144,7 +141,7 @@ plugin {
   # for now we define static quota-rules for all users 
   quota = maildir:User quota
   quota_rule = *:storage={{ config.max_mailbox_size }}
-  quota_max_mail_size=30M
+  quota_max_mail_size={{ config.max_message_size }}
   quota_grace = 0
   # quota_over_flag_value = TRUE
 }
@@ -197,11 +194,22 @@ service imap-login {
     process_min_avail = 10
 }
 
+service anvil {
+    # We are disabling anvil penalty on failed login attempts
+    # because it can only detect brute forcing by IP address
+    # not by username. As the correct IP address is not handed
+    # to dovecot anyway, it is more of hindrance than of use.
+    # See <https://www.dovecot.org/list/dovecot/2012-May/135485.html> for details.
+    unix_listener anvil-auth-penalty {
+        mode = 0
+    }
+}
+
 ssl = required
 ssl_cert = </var/lib/acme/live/{{ config.mail_domain }}/fullchain
 ssl_key = </var/lib/acme/live/{{ config.mail_domain }}/privkey
 ssl_dh = </usr/share/dovecot/dh.pem
-ssl_min_protocol = TLSv1.2
+ssl_min_protocol = TLSv1.3
 ssl_prefer_server_ciphers = yes
 
 

cmdeploy/src/cmdeploy/dovecot/expunge.cron.j2

@@ -1,3 +1,5 @@
+# delete already seen big mails after 7 days, in the INBOX
+2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/cur/*' -mtime +7 -size +200k -type f -delete
 # delete all mails after {{ config.delete_mails_after }} days, in the Inbox
 2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 # or in any IMAP subfolder

cmdeploy/src/cmdeploy/iroh-relay.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Iroh relay
+
+[Service]
+ExecStart=/usr/local/bin/iroh-relay --config-path /etc/iroh-relay.toml
+Restart=on-failure
+RestartSec=5s
+User=iroh
+Group=iroh
+
+[Install]
+WantedBy=multi-user.target

cmdeploy/src/cmdeploy/iroh-relay.toml

@@ -0,0 +1,5 @@
+enable_relay = true
+http_bind_addr = "[::]:3340"
+enable_stun = true
+enable_metrics = false
+metrics_bind_addr = "127.0.0.1:9092"

cmdeploy/src/cmdeploy/mtail/delivered_mail.mtail

@@ -0,0 +1,64 @@
+counter delivered_mail
+/saved mail to INBOX$/ {
+  delivered_mail++
+}
+
+counter quota_exceeded
+/Quota exceeded \(mailbox for user is full\)$/ {
+  quota_exceeded++
+}
+
+# Essentially the number of outgoing messages.
+counter dkim_signed
+/DKIM-Signature field added/ {
+  dkim_signed++
+}
+
+counter created_accounts
+counter created_ci_accounts
+counter created_nonci_accounts
+
+/: Created address: (?P<addr>.*)$/ {
+  created_accounts++
+
+  $addr =~ /ci-/ {
+    created_ci_accounts++
+  } else {
+    created_nonci_accounts++
+  }
+}
+
+counter postfix_timeouts
+/timeout after DATA/ {
+  postfix_timeouts++
+}
+
+counter postfix_noqueue
+/postfix\/.*NOQUEUE/ {
+  postfix_noqueue++
+}
+
+counter warning_count
+/warning/ {
+  warning_count++
+}
+
+
+counter filtered_mail_count
+
+counter encrypted_mail_count
+/Filtering encrypted mail\./ {
+  encrypted_mail_count++
+  filtered_mail_count++
+}
+
+counter unencrypted_mail_count
+/Filtering unencrypted mail\./ {
+  unencrypted_mail_count++
+  filtered_mail_count++
+}
+
+counter rejected_unencrypted_mail_count
+/Rejected unencrypted mail\./ {
+  rejected_unencrypted_mail_count++
+}

cmdeploy/src/cmdeploy/mtail/mtail.service.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=mtail
+
+[Service]
+Type=simple
+ExecStart=/bin/sh -c "journalctl -f -o short-iso -n 0 | /usr/bin/mtail --address={{ address }} --port={{ port }} --progs /etc/mtail --logtostderr --logs /dev/stdin"
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -96,6 +96,26 @@ http {
 			include /etc/nginx/fastcgi_params;
 			fastcgi_param SCRIPT_FILENAME /usr/lib/cgi-bin/newemail.py;
 		}
+
+                # Proxy to iroh-relay service.
+                location /relay {
+                        proxy_pass http://127.0.0.1:3340;
+                        proxy_http_version 1.1;
+
+                        # Upgrade header is normally set to "iroh derp http" or "websocket".
+                        proxy_set_header Upgrade $http_upgrade;
+                        proxy_set_header Connection "upgrade";
+                }
+
+		location /relay/probe {
+                        proxy_pass http://127.0.0.1:3340;
+                        proxy_http_version 1.1;
+		}
+
+		location /generate_204 {
+                        proxy_pass http://127.0.0.1:3340;
+                        proxy_http_version 1.1;
+		}
 	}
 
 	# Redirect www. to non-www

cmdeploy/src/cmdeploy/opendkim/systemd.conf

@@ -0,0 +1,3 @@
+[Service]
+Restart=always
+RuntimeMaxSec=1d

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -20,9 +20,9 @@ smtpd_tls_key_file=/var/lib/acme/live/{{ config.mail_domain }}/privkey
 smtpd_tls_security_level=may
 
 smtp_tls_CApath=/etc/ssl/certs
-smtp_tls_security_level=may
+smtp_tls_security_level=verify
 smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
-smtp_tls_policy_maps = socketmap:inet:127.0.0.1:8461:postfix
+smtp_tls_policy_maps = inline:{nauta.cu=may}
 smtpd_tls_protocols = >=TLSv1.2
 
 # Disable anonymous cipher suites

cmdeploy/src/cmdeploy/postfix/master.cf.j2

@@ -18,6 +18,7 @@ smtp      inet  n       -       y       -       -       smtpd
 submission inet n       -       y       -       5000    smtpd
   -o syslog_name=postfix/submission
   -o smtpd_tls_security_level=encrypt
+  -o smtpd_tls_mandatory_protocols=>=TLSv1.3
   -o smtpd_sasl_auth_enable=yes
   -o smtpd_sasl_type=dovecot
   -o smtpd_sasl_path=private/auth
@@ -36,6 +37,7 @@ smtps     inet  n       -       y       -       5000    smtpd
   -o syslog_name=postfix/smtps
   -o smtpd_tls_wrappermode=yes
   -o smtpd_tls_security_level=encrypt
+  -o smtpd_tls_mandatory_protocols=>=TLSv1.3
   -o smtpd_sasl_auth_enable=yes
   -o smtpd_sasl_type=dovecot
   -o smtpd_sasl_path=private/auth
@@ -80,7 +82,6 @@ filter    unix -        n       n       -       -       lmtp
 localhost:{{ config.postfix_reinject_port }} inet  n       -       n       -       10      smtpd
   -o syslog_name=postfix/reinject
   -o smtpd_milters=unix:opendkim/opendkim.sock
-  -o cleanup_service_name=authclean
 
 # Cleanup `Received` headers for authenticated mail
 # to avoid leaking client IP.

cmdeploy/src/cmdeploy/postfix/submission_header_cleanup

@@ -2,3 +2,4 @@
 /^X-Originating-IP:/    IGNORE
 /^X-Mailer:/            IGNORE
 /^User-Agent:/          IGNORE
+/^Subject:/             REPLACE Subject: [...]

cmdeploy/src/cmdeploy/remote/__init__.py

@@ -0,0 +1,12 @@
+"""
+
+The 'cmdeploy.remote' sub package contains modules with remotely executing functions.
+
+Its "_sshexec_bootstrap" module is executed remotely through `SSHExec`
+and its main() loop there stays connected via a command channel,
+ready to receive function invocations ("command") and return results.
+"""
+
+from . import rdns, rshell
+
+__all__ = ["rdns", "rshell"]

cmdeploy/src/cmdeploy/remote/_sshexec_bootstrap.py

@@ -0,0 +1,30 @@
+import builtins
+import importlib
+import traceback
+
+## Function Execution server
+
+
+def _run_loop(cmd_channel):
+    while cmd := cmd_channel.receive():
+        cmd_channel.send(_handle_one_request(cmd))
+
+
+def _handle_one_request(cmd):
+    pymod_path, func_name, kwargs = cmd
+    try:
+        mod = importlib.import_module(pymod_path)
+        func = getattr(mod, func_name)
+        res = func(**kwargs)
+        return ("finish", res)
+    except:
+        data = traceback.format_exc()
+        return ("error", data)
+
+
+def main(channel):
+    # enable simple "print" logging
+
+    builtins.print = lambda x="": channel.send(("log", x))
+
+    _run_loop(channel)

cmdeploy/src/cmdeploy/remote/rdns.py

@@ -11,41 +11,28 @@ All functions of this module
 """
 
 import re
-import traceback
-from subprocess import CalledProcessError, check_output
 
-
-def shell(command, fail_ok=False):
-    print(f"$ {command}")
-    try:
-        return check_output(command, shell=True).decode().rstrip()
-    except CalledProcessError:
-        if not fail_ok:
-            raise
-        return ""
-
-
-def get_systemd_running():
-    lines = shell("systemctl --type=service --state=running").split("\n")
-    return [line for line in lines if line.startswith("  ")]
+from .rshell import CalledProcessError, shell
 
 
 def perform_initial_checks(mail_domain):
     """Collecting initial DNS settings."""
     assert mail_domain
+    if not shell("dig", fail_ok=True):
+        shell("apt-get install -y dnsutils")
     A = query_dns("A", mail_domain)
     AAAA = query_dns("AAAA", mail_domain)
     MTA_STS = query_dns("CNAME", f"mta-sts.{mail_domain}")
+    WWW = query_dns("CNAME", f"www.{mail_domain}")
 
-    res = dict(mail_domain=mail_domain, A=A, AAAA=AAAA, MTA_STS=MTA_STS)
-    if not MTA_STS or (not A and not AAAA):
-        return res
-
+    res = dict(mail_domain=mail_domain, A=A, AAAA=AAAA, MTA_STS=MTA_STS, WWW=WWW)
     res["acme_account_url"] = shell("acmetool account-url", fail_ok=True)
-    if not shell("dig", fail_ok=True):
-        shell("apt-get install -y dnsutils")
-    shell(f"unbound-control flush_zone {mail_domain}", fail_ok=True)
-    res["dkim_entry"] = get_dkim_entry(mail_domain, dkim_selector="opendkim")
+    res["dkim_entry"], res["web_dkim_entry"] = get_dkim_entry(
+        mail_domain, dkim_selector="opendkim"
+    )
+
+    if not MTA_STS or not WWW or (not A and not AAAA):
+        return res
 
     # parse out sts-id if exists, example: "v=STSv1; id=2090123"
     parts = query_dns("TXT", f"_mta-sts.{mail_domain}").split("id=")
@@ -63,18 +50,34 @@ def get_dkim_entry(mail_domain, dkim_selector):
         return
     dkim_value_raw = f"v=DKIM1;k=rsa;p={dkim_pubkey};s=email;t=s"
     dkim_value = '" "'.join(re.findall(".{1,255}", dkim_value_raw))
-    return f'{dkim_selector}._domainkey.{mail_domain}. TXT "{dkim_value}"'
+    web_dkim_value = "".join(re.findall(".{1,255}", dkim_value_raw))
+    return (
+        f'{dkim_selector}._domainkey.{mail_domain}. TXT "{dkim_value}"',
+        f'{dkim_selector}._domainkey.{mail_domain}. TXT "{web_dkim_value}"',
+    )
 
 
 def query_dns(typ, domain):
-    res = shell(f"dig -r -q {domain} -t {typ} +short")
-    print(res)
+    # Get autoritative nameserver from the SOA record.
+    soa_answers = [
+        x.split()
+        for x in shell(f"dig -r -q {domain} -t SOA +noall +authority +answer").split(
+            "\n"
+        )
+    ]
+    soa = [a for a in soa_answers if len(a) >= 3 and a[3] == "SOA"]
+    if not soa:
+        return
+    ns = soa[0][4]
+
+    # Query authoritative nameserver directly to bypass DNS cache.
+    res = shell(f"dig @{ns} -r -q {domain} -t {typ} +short")
     if res:
         return res.split("\n")[0]
     return ""
 
 
-def check_zonefile(zonefile):
+def check_zonefile(zonefile, mail_domain):
     """Check expected zone file entries."""
     required = True
     required_diff = []
@@ -99,37 +102,3 @@ def check_zonefile(zonefile):
                 recommended_diff.append(zf_line)
 
     return required_diff, recommended_diff
-
-
-## Function Execution server
-
-
-def _run_loop(cmd_channel):
-    while 1:
-        cmd = cmd_channel.receive()
-        if cmd is None:
-            break
-
-        cmd_channel.send(_handle_one_request(cmd))
-
-
-def _handle_one_request(cmd):
-    func_name, kwargs = cmd
-    try:
-        res = globals()[func_name](**kwargs)
-        return ("finish", res)
-    except:
-        data = traceback.format_exc()
-        return ("error", data)
-
-
-# check if this module is executed remotely
-# and setup a simple serialized function-execution loop
-
-if __name__ == "__channelexec__":
-    channel = channel  # noqa (channel object gets injected)
-
-    # enable simple "print" logging for anyone changing this module
-    globals()["print"] = lambda x="": channel.send(("log", x))
-
-    _run_loop(channel)

cmdeploy/src/cmdeploy/remote/rshell.py

@@ -0,0 +1,16 @@
+from subprocess import CalledProcessError, check_output
+
+
+def shell(command, fail_ok=False):
+    print(f"$ {command}")
+    try:
+        return check_output(command, shell=True).decode().rstrip()
+    except CalledProcessError:
+        if not fail_ok:
+            raise
+        return ""
+
+
+def get_systemd_running():
+    lines = shell("systemctl --type=service --state=running").split("\n")
+    return [line for line in lines if line.startswith("  ")]

cmdeploy/src/cmdeploy/service/chatmail-metadata.service.f

@@ -7,6 +7,7 @@ Restart=always
 RestartSec=30
 User=vmail
 RuntimeDirectory=chatmail-metadata
+UMask=0077
 
 [Install]
 WantedBy=multi-user.target

cmdeploy/src/cmdeploy/service/doveauth.service.f

@@ -7,6 +7,7 @@ Restart=always
 RestartSec=30
 User=vmail
 RuntimeDirectory=doveauth
+UMask=0077
 
 [Install]
 WantedBy=multi-user.target

cmdeploy/src/cmdeploy/sshexec.py

@@ -1,12 +1,45 @@
+import inspect
+import os
 import sys
+from queue import Queue
 
 import execnet
 
+from . import remote
+
 
 class FuncError(Exception):
     pass
 
 
+def bootstrap_remote(gateway, remote=remote):
+    """Return a command channel which can execute remote functions."""
+    source_init_path = inspect.getfile(remote)
+    basedir = os.path.dirname(source_init_path)
+    name = os.path.basename(basedir)
+
+    # rsync sourcedir to remote host
+    remote_pkg_path = f"/root/from-cmdeploy/{name}"
+    q = Queue()
+    finish = lambda: q.put(None)
+    rsync = execnet.RSync(sourcedir=basedir, verbose=False)
+    rsync.add_target(gateway, remote_pkg_path, finishedcallback=finish, delete=True)
+    rsync.send()
+    q.get()
+
+    # start sshexec bootstrap and return its command channel
+    remote_sys_path = os.path.dirname(remote_pkg_path)
+    channel = gateway.remote_exec(
+        f"""
+        import sys
+        sys.path.insert(0, {remote_sys_path!r})
+        from remote._sshexec_bootstrap import main
+        main(channel)
+    """
+    )
+    return channel
+
+
 def print_stderr(item="", end="\n"):
     print(item, file=sys.stderr, end=end)
 
@@ -15,16 +48,18 @@ class SSHExec:
     RemoteError = execnet.RemoteError
     FuncError = FuncError
 
-    def __init__(self, host, remote_funcs, verbose=False, python="python3", timeout=60):
+    def __init__(self, host, verbose=False, python="python3", timeout=60):
         self.gateway = execnet.makegateway(f"ssh=root@{host}//python={python}")
-        self._remote_cmdloop_channel = self.gateway.remote_exec(remote_funcs)
+        self._remote_cmdloop_channel = bootstrap_remote(self.gateway, remote)
         self.timeout = timeout
         self.verbose = verbose
 
     def __call__(self, call, kwargs=None, log_callback=None):
         if kwargs is None:
             kwargs = {}
-        self._remote_cmdloop_channel.send((call.__name__, kwargs))
+        assert call.__module__.startswith("cmdeploy.remote")
+        modname = call.__module__.replace("cmdeploy.", "")
+        self._remote_cmdloop_channel.send((modname, call.__name__, kwargs))
         while 1:
             code, data = self._remote_cmdloop_channel.receive(timeout=self.timeout)
             if log_callback is not None and code == "log":

cmdeploy/src/cmdeploy/tests/data/zftest.zone

@@ -11,7 +11,7 @@ _submission._tcp.zftest.testrun.org.  SRV 0 1 587 zftest.testrun.org.
 _submissions._tcp.zftest.testrun.org. SRV 0 1 465 zftest.testrun.org.
 _imap._tcp.zftest.testrun.org.        SRV 0 1 143 zftest.testrun.org.
 _imaps._tcp.zftest.testrun.org.       SRV 0 1 993 zftest.testrun.org.
-zftest.testrun.org.                   CAA 128 issue "letsencrypt.org;accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/1371472956"
+zftest.testrun.org.                   CAA 0 issue "letsencrypt.org;accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/1371472956"
 zftest.testrun.org.                   TXT "v=spf1 a:zftest.testrun.org ~all"
 _dmarc.zftest.testrun.org.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
 _adsp._domainkey.zftest.testrun.org.  TXT "dkim=discardable"

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -1,30 +1,31 @@
+import datetime
 import smtplib
 
 import pytest
 
-from cmdeploy import remote_funcs
+from cmdeploy import remote
 from cmdeploy.sshexec import SSHExec
 
 
 class TestSSHExecutor:
     @pytest.fixture(scope="class")
     def sshexec(self, sshdomain):
-        return SSHExec(sshdomain, remote_funcs)
+        return SSHExec(sshdomain)
 
     def test_ls(self, sshexec):
-        out = sshexec(call=remote_funcs.shell, kwargs=dict(command="ls"))
-        out2 = sshexec(call=remote_funcs.shell, kwargs=dict(command="ls"))
+        out = sshexec(call=remote.rdns.shell, kwargs=dict(command="ls"))
+        out2 = sshexec(call=remote.rdns.shell, kwargs=dict(command="ls"))
         assert out == out2
 
     def test_perform_initial(self, sshexec, maildomain):
         res = sshexec(
-            remote_funcs.perform_initial_checks, kwargs=dict(mail_domain=maildomain)
+            remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain)
         )
         assert res["A"] or res["AAAA"]
 
     def test_logged(self, sshexec, maildomain, capsys):
         sshexec.logged(
-            remote_funcs.perform_initial_checks, kwargs=dict(mail_domain=maildomain)
+            remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain)
         )
         out, err = capsys.readouterr()
         assert err.startswith("Collecting")
@@ -33,25 +34,33 @@ class TestSSHExecutor:
 
         sshexec.verbose = True
         sshexec.logged(
-            remote_funcs.perform_initial_checks, kwargs=dict(mail_domain=maildomain)
+            remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain)
         )
         out, err = capsys.readouterr()
         lines = err.split("\n")
         assert len(lines) > 4
-        assert remote_funcs.perform_initial_checks.__doc__ in lines[0]
+        assert remote.rdns.perform_initial_checks.__doc__ in lines[0]
 
     def test_exception(self, sshexec, capsys):
         try:
             sshexec.logged(
-                remote_funcs.perform_initial_checks,
+                remote.rdns.perform_initial_checks,
                 kwargs=dict(mail_domain=None),
             )
         except sshexec.FuncError as e:
-            assert "remote_funcs.py" in str(e)
+            assert "rdns.py" in str(e)
             assert "AssertionError" in str(e)
         else:
             pytest.fail("didn't raise exception")
 
+    def test_opendkim_restarted(self, sshexec):
+        """check that opendkim is not running for longer than a day."""
+        out = sshexec(call=remote.rshell.shell, kwargs=dict(command="systemctl status opendkim"))
+        assert type(out) == str
+        since_date_str = out.split("since ")[1].split(";")[0]
+        since_date = datetime.datetime.strptime(since_date_str, "%a %Y-%m-%d %H:%M:%S %Z")
+        assert (datetime.datetime.now() - since_date).total_seconds() < 60 * 60 * 24
+
 
 def test_remote(remote, imap_or_smtp):
     lineproducer = remote.iter_output(imap_or_smtp.logcmd)
@@ -115,6 +124,27 @@ def test_reject_missing_dkim(cmsetup, maildata, from_addr):
             s.sendmail(from_addr=from_addr, to_addrs=recipient.addr, msg=msg)
 
 
+def test_rewrite_subject(cmsetup, maildata):
+    """Test that subject gets replaced with [...]."""
+    user1, user2 = cmsetup.gen_users(2)
+
+    sent_msg = maildata(
+        "encrypted.eml",
+        from_addr=user1.addr,
+        to_addr=user2.addr,
+        subject="Unencrypted subject",
+    ).as_string()
+    user1.smtp.sendmail(from_addr=user1.addr, to_addrs=[user2.addr], msg=sent_msg)
+
+    messages = user2.imap.fetch_all_messages()
+    assert len(messages) == 1
+    rcvd_msg = messages[0]
+    assert "Subject: [...]" not in sent_msg
+    assert "Subject: [...]" in rcvd_msg
+    assert "Subject: Unencrypted subject" in sent_msg
+    assert "Subject: Unencrypted subject" not in rcvd_msg
+
+
 @pytest.mark.slow
 def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
     """Test that the per-account send-mail limit is exceeded."""

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -85,7 +85,7 @@ class TestEndToEndDeltaChat:
         attachsize = 1 * 1024 * 1024
         num_to_send = quota // attachsize + 2
         lp.sec(f"ac1: send {num_to_send} large files to ac2")
-        lp.indent(f"per-user quota is assumed to be: {quota/(1024*1024)}MB")
+        lp.indent(f"per-user quota is assumed to be: {quota / (1024 * 1024)}MB")
         alphanumeric = "abcdefghijklmnopqrstuvwxyz1234567890"
         msgs = []
         for i in range(num_to_send):
@@ -97,7 +97,7 @@ class TestEndToEndDeltaChat:
 
             msg = chat.send_file(str(attachment))
             msgs.append(msg)
-            lp.indent(f"Sent out msg {i}, size {attachsize/(1024*1024)}MB")
+            lp.indent(f"Sent out msg {i}, size {attachsize / (1024 * 1024)}MB")
 
         lp.sec("ac2: check messages are arriving until quota is reached")
 

cmdeploy/src/cmdeploy/tests/test_dns.py

@@ -1,6 +1,6 @@
 import pytest
 
-from cmdeploy import remote_funcs
+from cmdeploy import remote
 from cmdeploy.dns import check_full_zone, check_initial_remote_data
 
 
@@ -14,7 +14,7 @@ def mockdns_base(monkeypatch):
         except KeyError:
             return ""
 
-    monkeypatch.setattr(remote_funcs, query_dns.__name__, query_dns)
+    monkeypatch.setattr(remote.rdns, query_dns.__name__, query_dns)
     return qdict
 
 
@@ -24,7 +24,10 @@ def mockdns(mockdns_base):
         {
             "A": {"some.domain": "1.1.1.1"},
             "AAAA": {"some.domain": "fde5:cd7a:9e1c:3240:5a99:936f:cdac:53ae"},
-            "CNAME": {"mta-sts.some.domain": "some.domain"},
+            "CNAME": {
+                "mta-sts.some.domain": "some.domain.",
+                "www.some.domain": "some.domain.",
+            },
         }
     )
     return mockdns_base
@@ -32,14 +35,16 @@ def mockdns(mockdns_base):
 
 class TestPerformInitialChecks:
     def test_perform_initial_checks_ok1(self, mockdns):
-        remote_data = remote_funcs.perform_initial_checks("some.domain")
-        assert len(remote_data) == 7
+        remote_data = remote.rdns.perform_initial_checks("some.domain")
+        assert remote_data["A"] == mockdns["A"]["some.domain"]
+        assert remote_data["AAAA"] == mockdns["AAAA"]["some.domain"]
+        assert remote_data["MTA_STS"] == mockdns["CNAME"]["mta-sts.some.domain"]
+        assert remote_data["WWW"] == mockdns["CNAME"]["www.some.domain"]
 
     @pytest.mark.parametrize("drop", ["A", "AAAA"])
     def test_perform_initial_checks_with_one_of_A_AAAA(self, mockdns, drop):
         del mockdns[drop]
-        remote_data = remote_funcs.perform_initial_checks("some.domain")
-        assert len(remote_data) == 7
+        remote_data = remote.rdns.perform_initial_checks("some.domain")
         assert not remote_data[drop]
 
         l = []
@@ -48,9 +53,8 @@ class TestPerformInitialChecks:
         assert not l
 
     def test_perform_initial_checks_no_mta_sts(self, mockdns):
-        del mockdns["CNAME"]
-        remote_data = remote_funcs.perform_initial_checks("some.domain")
-        assert len(remote_data) == 4
+        del mockdns["CNAME"]["mta-sts.some.domain"]
+        remote_data = remote.rdns.perform_initial_checks("some.domain")
         assert not remote_data["MTA_STS"]
 
         l = []
@@ -85,14 +89,18 @@ class TestZonefileChecks:
     def test_check_zonefile_all_ok(self, cm_data, mockdns_base):
         zonefile = cm_data.get("zftest.zone")
         parse_zonefile_into_dict(zonefile, mockdns_base)
-        required_diff, recommended_diff = remote_funcs.check_zonefile(zonefile)
+        required_diff, recommended_diff = remote.rdns.check_zonefile(
+            zonefile, "some.domain"
+        )
         assert not required_diff and not recommended_diff
 
     def test_check_zonefile_recommended_not_set(self, cm_data, mockdns_base):
         zonefile = cm_data.get("zftest.zone")
         zonefile_mocked = zonefile.split("; Recommended")[0]
         parse_zonefile_into_dict(zonefile_mocked, mockdns_base)
-        required_diff, recommended_diff = remote_funcs.check_zonefile(zonefile)
+        required_diff, recommended_diff = remote.rdns.check_zonefile(
+            zonefile, "some.domain"
+        )
         assert not required_diff
         assert len(recommended_diff) == 8
 
@@ -101,6 +109,7 @@ class TestZonefileChecks:
         zonefile_mocked = zonefile.split("; Recommended")[0]
         parse_zonefile_into_dict(zonefile_mocked, mockdns_base, only_required=True)
         mssh = MockSSHExec()
+        mockdns_base["mail_domain"] = "some.domain"
         res = check_full_zone(mssh, mockdns_base, out=mockout, zonefile=zonefile)
         assert res == 0
         assert "WARNING" in mockout.captured_plain[0]
@@ -110,6 +119,7 @@ class TestZonefileChecks:
         zonefile = cm_data.get("zftest.zone")
         parse_zonefile_into_dict(zonefile, mockdns_base)
         mssh = MockSSHExec()
+        mockdns_base["mail_domain"] = "some.domain"
         res = check_full_zone(mssh, mockdns_base, out=mockout, zonefile=zonefile)
         assert res == 0
         assert not mockout.captured_red

www/src/info.md

@@ -43,6 +43,20 @@ The first login sets your password.
 - You can store up to [{{ config.max_mailbox_size }} messages on the server](https://delta.chat/en/help#what-happens-if-i-turn-on-delete-old-messages-from-server).
 
 
+### <a name="account-deletion"></a> Account deletion 
+
+If you remove a {{ config.mail_domain }} profile from within the Delta Chat app, 
+then the according account on the server, along with all associated data,
+is automatically deleted {{ config.delete_inactive_users_after }} days afterwards. 
+
+If you use multiple devices 
+then you need to remove the according chat profile from each device
+in order for all account data to be removed on the server side. 
+
+If you have any further questions or requests regarding account deletion
+please send a message from your account to {{ config.privacy_mail }}. 
+
+
 ### Who are the operators? Which software is running? 
 
 This chatmail provider is run by a small voluntary group of devs and sysadmins,

www/src/privacy.md

@@ -3,8 +3,8 @@
 
 {% if config.mail_domain == "nine.testrun.org" %}
 Welcome to `{{config.mail_domain}}`, the default chatmail onboarding server for Delta Chat users. 
-It is operated on the side by a small sysops team employed by [merlinux](https://merlinux.eu),
-an open-source R&D company also acting as the fiscal sponsor of Delta Chat app developments. 
+It is operated on the side by a small sysops team
+on a voluntary basis.
 See [other chatmail servers](https://delta.chat/en/chatmail) for alternative server operators. 
 {% endif %}
 
@@ -23,18 +23,22 @@ A chatmail server behaves more like the Signal messaging server
 but does not know about phone numbers and securely and automatically interoperates 
 with other chatmail and classic e-mail servers. 
 
-In particular, this chatmail server 
+Unlike classic e-mail servers, this chatmail server 
 
 - unconditionally removes messages after {{ config.delete_mails_after }} days,
 
 - prohibits sending out un-encrypted messages,
 
-- only has temporary log files used for debugging purposes.
+- does not store Internet addresses ("IP addresses"), 
+
+- does not process IP addresses in relation to email addresses.
+
+Due to the resulting lack of personal data processing
+this chatmail server may not require a privacy policy.
+
+Nevertheless, we provide legal details below to make life easier
+for data protection specialists and lawyers scrutinizing chatmail operations.
 
-Legally, authorities might still regard chatmail as a "classic e-mail" server
-which collects and retains personal data. 
-We do not agree on this interpretation. Nevertheless, we provide more legal details below
-to make life easier for data protection specialists and lawyers scrutinizing chatmail operations. 
 
 
 ## 1. Name and contact information 
@@ -54,18 +58,18 @@ We have appointed a data protection officer:
 
 ## 2. Processing when using chat e-mail services
 
-We provide e-mail services optimized for the use from [Delta Chat](https://delta.chat) apps 
+We provide services optimized for the use from [Delta Chat](https://delta.chat) apps
 and process only the data necessary
-for the setup and technical execution of the e-mail dispatch.
-The purpose of the processing is to
-read, write, manage, delete, send, and receive emails.
+for the setup and technical execution of message delivery.
+The purpose of the processing is that users can
+read, write, manage, delete, send, and receive chat messages.
 For this purpose,
 we operate server-side software
-that enables us to send and receive e-mail messages.
-Allowing the use of the e-mail service,
-we process the following data and details:
+that enables us to send and receive messages.
 
-- Outgoing and incoming messages (SMTP) are stored for transit 
+We process the following data and details:
+
+- Outgoing and incoming messages (SMTP) are stored for transit
   on behalf of their users until the message can be delivered.
 
 - E-Mail-Messages are stored for the recipient and made accessible via IMAP protocols,
@@ -74,9 +78,15 @@ we process the following data and details:
 
 - IMAP and SMTP protocols are password protected with unique credentials for each account.
 
-- Users can retrieve or delete all stored messages 
+- Users can retrieve or delete all stored messages
   without intervention from the operators using standard IMAP client tools.
 
+- Users can connect to a "realtime relay service"
+  to establish Peer-to-Peer connection between user devices,
+  allowing them to send and retrieve ephemeral messages
+  which are never stored on the chatmail server, also not in encrypted form.
+
+
 ### 2.1 Account setup
 
 Creating an account happens in one of two ways on our mail servers: 
@@ -253,7 +263,7 @@ is the `{{ config.privacy_supervisor }}`.
 ## 6. Validity of this privacy policy 
 
 This data protection declaration is valid
-as of *December 2023*.
+as of *October 2024*.
 Due to the further development of our service and offers
 or due to changed legal or official requirements,
 it may become necessary to revise this data protection declaration from time to time.