tests: fix output capture

- https://github.com/chatmail/relay/pull/614#discussion_r2279697445

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,33 @@
+---
+name: Bug report
+about: Report something that isn't working.
+title: ''
+assignees: ''
+
+---
+
+<!--
+Please fill out as much of this form as you can (leaving out stuff that is not applicable is ok).
+-->
+
+- Server OS (Operating System) - preferably Debian 12: 
+- On which OS you run cmdeploy:
+- chatmail/relay version: `git rev-parse HEAD`
+
+## Expected behavior
+
+*What did you try to achieve?*
+
+## Actual behavior
+
+*What happened instead?*
+
+### Steps to reproduce the problem:
+
+1. 
+2.
+
+### Screenshots
+
+### Logs
+

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Mutual Help Chat Group
+    url: https://i.delta.chat/#C2846EB4C1CB8DF84B1818F5E3A638FC3FBDC981&a=stalebot1%40nine.testrun.org&g=Chatmail%20Mutual%20Help&x=7sFF7Ik50pWv6J1z7RVC5527&i=d7s1HvOsk5UrSf9AoqRZggg4&s=XmX_9BAW6-g5Ao5E8PyaeKNB
+    about: If you have troubles setting up the relay server, feel free to ask here.

.github/workflows/ci.yaml

@@ -10,6 +10,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        # Checkout pull request HEAD commit instead of merge commit
+        # Otherwise `test_deployed_state` will be unhappy.
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
 
       - name: run chatmaild tests 
         working-directory: chatmaild

.github/workflows/staging-ipv4.testrun.org-default.zone

@@ -17,5 +17,4 @@ $TTL 300
 ;; DNS records.
 @       IN      A       37.27.95.249
 mta-sts.staging-ipv4.testrun.org.    CNAME   staging-ipv4.testrun.org.
-iroh.staging-ipv4.testrun.org.       CNAME   staging-ipv4.testrun.org.
 www.staging-ipv4.testrun.org.        CNAME   staging-ipv4.testrun.org.

.github/workflows/staging.testrun.org-default.zone

@@ -17,6 +17,5 @@ $TTL 300
 ;; DNS records.
 @       IN      A       37.27.24.139
 mta-sts.staging2.testrun.org.    CNAME   staging2.testrun.org.
-iroh.staging2.testrun.org.       CNAME   staging2.testrun.org.
 www.staging2.testrun.org.        CNAME   staging2.testrun.org.
 

.github/workflows/test-and-deploy-ipv4only.yaml

@@ -38,8 +38,8 @@ jobs:
           if [ -f dkimkeys-ipv4/dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys-ipv4 root@ns.testrun.org:/tmp/ || true; fi
           if [ "$(ls -A acme-ipv4/acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme-ipv4 root@ns.testrun.org:/tmp/ || true; fi
           # make sure CAA record isn't set
-          scp .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
+          scp -o StrictHostKeyChecking=accept-new .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
-          ssh -o StrictHostKeyChecking=accept-new root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging-ipv4.testrun.org.zone
+          ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging-ipv4.testrun.org.zone
           ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
           ssh root@ns.testrun.org systemctl reload nsd
 
@@ -49,7 +49,7 @@ jobs:
             -H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
             -H "Content-Type: application/json" \
             -d '{"image":"debian-12"}' \
-            "https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_SERVER_ID }}/actions/rebuild"
+            "https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_IPV4_SERVER_ID }}/actions/rebuild"
 
       - run: scripts/initenv.sh
 
@@ -63,11 +63,11 @@ jobs:
           while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u ; do sleep 1 ; done
           ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u
           # download acme & dkim state from ns.testrun.org
-          rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme-ipv4 acme-restore || true
+          rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme-ipv4/acme acme-restore || true
-          rsync -avz root@ns.testrun.org:/tmp/dkimkeys-ipv4 dkimkeys-restore || true
+          rsync -avz root@ns.testrun.org:/tmp/dkimkeys-ipv4/dkimkeys dkimkeys-restore || true
           # restore acme & dkim state to staging2.testrun.org
-          rsync -avz acme-restore/acme-ipv4/acme root@staging-ipv4.testrun.org:/var/lib/ || true
+          rsync -avz acme-restore/acme root@staging-ipv4.testrun.org:/var/lib/ || true
-          rsync -avz dkimkeys-restore/dkimkeys-ipv4/dkimkeys root@staging-ipv4.testrun.org:/etc/ || true
+          rsync -avz dkimkeys-restore/dkimkeys root@staging-ipv4.testrun.org:/etc/ || true
           ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org chown root:root -R /var/lib/acme || true
 
       - name: run formatting checks 

.gitignore

@@ -164,3 +164,9 @@ cython_debug/
 #.idea/
 
 chatmail.zone
+
+# docker
+/data/
+/custom/
+docker-compose.yaml
+.env

CHANGELOG.md

@@ -2,36 +2,193 @@
 
 ## untagged
 
+- Add installation via docker compose (MVP 1). The instructions, known issues and limitations are located in `/docs` 
+  ([#614](https://github.com/chatmail/relay/pull/614))
+
+- Add markdown tabs blocks for rendering multilingual pages. Add russian language support to `index.md`, `privacy.md`, and `info.md`.
+  ([#614](https://github.com/chatmail/relay/pull/614))
+
+- Fix [Issue 604](https://github.com/chatmail/relay/issues/604), now the `--ssh_host` argument of the `cmdeploy run` command works correctly and does not depend on `config.mail_domain`.
+  ([#614](https://github.com/chatmail/relay/pull/614))
+
+- Add `--skip-dns-check` argument to `cmdeploy run` command, which disables DNS record checking before installation.
+  ([#614](https://github.com/chatmail/relay/pull/614))
+
+- Add `--force` argument to `cmdeploy init` command, which recreates the `chatmail.ini` file.
+  ([#614](https://github.com/chatmail/relay/pull/614))
+
+- Add startup for `fcgiwrap.service` because sometimes it did not start automatically.
+  ([#614](https://github.com/chatmail/relay/pull/614))
+
+- Add extended check when installing `unbound.service`. Now, if it is not shown who exactly is occupying port 53, but `unbound.service` is running, it is considered that the port is occupied by `unbound.service`.
+  ([#614](https://github.com/chatmail/relay/pull/614))
+
+- Add configuration parameters
+  ([#614](https://github.com/chatmail/relay/pull/614)):
+  - `is_development_instance` - Indicates that this instance is installed as a temporary/test one (default: `True`)
+  - `use_foreign_cert_manager` - Use a third-party certificate manager instead of acmetool (default: `False`)
+  - `acme_email` - Email address used by acmetool to obtain Let's Encrypt certificates (default: empty)
+  - `change_kernel_settings` - Whether to change kernel parameters during installation (default: `True`)
+  - `fs_inotify_max_user_instances_and_watchers` - Value for kernel parameters `fs.inotify.max_user_instances` and `fs.inotify.max_user_watches` (default: `65535`)
+
+- Check whether GCC is installed in initenv.sh
+  ([#608](https://github.com/chatmail/relay/pull/608))
+
+- Expire push notification tokens after 90 days
+  ([#583](https://github.com/chatmail/relay/pull/583))
+
+- Use official `mtail` binary instead of `mtail` package
+  ([#581](https://github.com/chatmail/relay/pull/581))
+
+- dovecot: install from download.delta.chat instead of openSUSE Build Service
+  ([#590](https://github.com/chatmail/relay/pull/590))
+
+- Reconfigure Dovecot imap-login service to high-performance mode
+  ([#578](https://github.com/chatmail/relay/pull/578))
+
+- Set timezone to improve dovecot performance
+  ([#584](https://github.com/chatmail/relay/pull/584))
+
+- Increase nginx connection limits
+  ([#576](https://github.com/chatmail/relay/pull/576))
+
+- If `dns-utils` needs to be installed before cmdeploy run, apt update to make sure it works
+  ([#560](https://github.com/chatmail/relay/pull/560))
+
+- filtermail: respect config message size limit
+  ([#572](https://github.com/chatmail/relay/pull/572))
+
+- Add config value after how many days large files are deleted
+  ([#555](https://github.com/chatmail/relay/pull/555))
+
+- cmdeploy: push relay version to /etc/chatmail-version
+  ([#573](https://github.com/chatmail/relay/pull/573))
+
+- filtermail: allow partial body length in OpenPGP payloads
+  ([#570](https://github.com/chatmail/relay/pull/570))
+
+- chatmaild: allow echobot to receive unencrypted messages by default
+  ([#556](https://github.com/chatmail/relay/pull/556))
+
+
+## 1.6.0 2025-04-11
+
+- Handle Port-25 connect errors more gracefully (common with VPNs)
+  ([#552](https://github.com/chatmail/relay/pull/552))
+
+- Avoid "acmetool not found" during initial run
+  ([#550](https://github.com/chatmail/relay/pull/550))
+
+- Fix timezone handling such that client/servers do not need to use
+  same timezone. 
+  ([#553](https://github.com/chatmail/relay/pull/553))
+
+- Enforce end-to-end encryption for incoming messages. 
+  New user address mailboxes now get a `enforceE2EEincoming` file 
+  which prohibits incoming cleartext messages from other domains. 
+  An outside MTA trying to submit a cleartext message will 
+  get a "523 Encryption Needed" response, see RFC5248. 
+  If the file does not exist (as it the case for all existing accounts) 
+  incoming cleartext messages are accepted. 
+  ([#538](https://github.com/chatmail/server/pull/538))
+
+- Enforce end-to-end encryption between local addresses 
+  ([#535](https://github.com/chatmail/server/pull/535))
+
+- unbound: check that port 53 is not occupied by a different process
+  ([#537](https://github.com/chatmail/server/pull/537))
+
+- unbound: before unbound is there, use 9.9.9.9 for resolving
+  ([#518](https://github.com/chatmail/relay/pull/518))
+
+- Limit the bind for the HTTPS server on 8443 to 127.0.0.1 
+  ([#522](https://github.com/chatmail/server/pull/522))
+  ([#532](https://github.com/chatmail/server/pull/532))
+
+- Send SNI when connecting to outside servers
+  ([#524](https://github.com/chatmail/server/pull/524))
+
+- postfix master.cf: use 127.0.0.1 for consistency
+  ([#544](https://github.com/chatmail/relay/pull/544))
+
+- Pass through `original_content` instead of `content` in filtermail
+  ([#509](https://github.com/chatmail/server/pull/509))
+
+- Document TLS requirements in the readme
+  ([#514](https://github.com/chatmail/server/pull/514))
+
+- Remove cleanup service from submission ports
+  ([#512](https://github.com/chatmail/server/pull/512))
+
+- cmdeploy dovecot: delete big messages after 7 days
+  ([#504](https://github.com/chatmail/server/pull/504))
+
+- mtail: fix getting logs from STDIN
+  ([#502](https://github.com/chatmail/server/pull/502))
+
+- filtermail: don't require exactly 2 lines after openPGP payload
+  ([#497](https://github.com/chatmail/server/pull/497))
+
+- cmdeploy dns: offer alternative DKIM record format for some web interfaces
+  ([#470](https://github.com/chatmail/server/pull/470))
+
+- journald: remove old logs from disk
+  ([#490](https://github.com/chatmail/server/pull/490))
+
+- opendkim: restart once every day to mend RAM leaks
+  ([#498](https://github.com/chatmail/server/pull/498)
+
+- migration guide: let opendkim own the DKIM keys directory
+  ([#468](https://github.com/chatmail/server/pull/468))
+
+- improve secure-join message detection
+  ([#473](https://github.com/chatmail/server/pull/473))
+
+- use old crypt lib in python < 3.11
+  ([#483](https://github.com/chatmail/server/pull/483))
+
+- chatmaild: set umask to 0700 for doveauth + metadata
+  ([#490](https://github.com/chatmail/server/pull/492))
+
+- remove MTA-STS daemon
+  ([#488](https://github.com/chatmail/server/pull/488))
+
+- replace `Subject` with `[...]` for all outgoing mails.
+  ([#481](https://github.com/chatmail/server/pull/481))
+
+- opendkim: use su instead of sudo
+  ([#491](https://github.com/chatmail/server/pull/491))
+
 ## 1.5.0 2024-12-20
 
 - cmdeploy dns: always show recommended DNS records
-  ([#463](https://github.com/deltachat/chatmail/pull/463))
+  ([#463](https://github.com/chatmail/server/pull/463))
 
 - add `--all` to `cmdeploy dns`
-  ([#462](https://github.com/deltachat/chatmail/pull/462))
+  ([#462](https://github.com/chatmail/server/pull/462))
 
 - fix `_mta-sts` TXT DNS record
-  ([#461](https://github.com/deltachat/chatmail/pull/461)
+  ([#461](https://github.com/chatmail/server/pull/461)
 
 - deploy `iroh-relay` and also update "realtime relay services" in privacy policy. 
-  ([#434](https://github.com/deltachat/chatmail/pull/434))
+  ([#434](https://github.com/chatmail/server/pull/434))
-  ([#451](https://github.com/deltachat/chatmail/pull/451))
+  ([#451](https://github.com/chatmail/server/pull/451))
 
 - add guide to migrate chatmail to a new server
-  ([#429](https://github.com/deltachat/chatmail/pull/429))
+  ([#429](https://github.com/chatmail/server/pull/429))
 
 - disable anvil authentication penalty
-  ([#414](https://github.com/deltachat/chatmail/pull/444)
+  ([#414](https://github.com/chatmail/server/pull/444)
 
 - increase `request_queue_size` for UNIX sockets to 1000.
-  ([#437](https://github.com/deltachat/chatmail/pull/437))
+  ([#437](https://github.com/chatmail/server/pull/437))
 
 - add argument to `cmdeploy run` for specifying
   a different SSH host than `mail_domain`
-  ([#439](https://github.com/deltachat/chatmail/pull/439))
+  ([#439](https://github.com/chatmail/server/pull/439))
 
 - query autoritative nameserver to bypass DNS cache
-  ([#424](https://github.com/deltachat/chatmail/pull/424))
+  ([#424](https://github.com/chatmail/server/pull/424))
 
 - add mtail support (new optional `mtail_address` ini value)
   This defines the address on which [`mtail`](https://google.github.io/mtail/)
@@ -41,195 +198,195 @@
   and assign an IP address from this network to the host.
   If you do not plan to collect metrics,
   keep this setting unset.
-  ([#388](https://github.com/deltachat/chatmail/pull/388))
+  ([#388](https://github.com/chatmail/server/pull/388))
 
 - fix checking for required DNS records
-  ([#412](https://github.com/deltachat/chatmail/pull/412))
+  ([#412](https://github.com/chatmail/server/pull/412))
 
 - add support for specifying whole domains for recipient passthrough list
-  ([#408](https://github.com/deltachat/chatmail/pull/408))
+  ([#408](https://github.com/chatmail/server/pull/408))
 
 - add a paragraph about "account deletion" to info page 
-  ([#405](https://github.com/deltachat/chatmail/pull/405))
+  ([#405](https://github.com/chatmail/server/pull/405))
 
 - avoid nginx listening on ipv6 if v6 is dsiabled 
-  ([#402](https://github.com/deltachat/chatmail/pull/402))
+  ([#402](https://github.com/chatmail/server/pull/402))
 
 - refactor ssh-based execution to allow organizing remote functions in
   modules. 
-  ([#396](https://github.com/deltachat/chatmail/pull/396))
+  ([#396](https://github.com/chatmail/server/pull/396))
 
 - trigger "apt upgrade" during "cmdeploy run" 
-  ([#398](https://github.com/deltachat/chatmail/pull/398))
+  ([#398](https://github.com/chatmail/server/pull/398))
 
 - drop hispanilandia passthrough address
-  ([#401](https://github.com/deltachat/chatmail/pull/401))
+  ([#401](https://github.com/chatmail/server/pull/401))
 
 - set CAA record flags to 0
 
 - add IMAP capabilities instead of overwriting them
-  ([#413](https://github.com/deltachat/chatmail/pull/413))
+  ([#413](https://github.com/chatmail/server/pull/413))
 
 - fix OpenPGP payload check
-  ([#435](https://github.com/deltachat/chatmail/pull/435))
+  ([#435](https://github.com/chatmail/server/pull/435))
 
 - fix Dovecot quota_max_mail_size to use max_message_size config value
-  ([#438](https://github.com/deltachat/chatmail/pull/438))
+  ([#438](https://github.com/chatmail/server/pull/438))
 
 
 ## 1.4.1 2024-07-31
 
 - fix metadata dictproxy which would confuse transactions
   resulting in missed notifications and other issues. 
-  ([#393](https://github.com/deltachat/chatmail/pull/393))
+  ([#393](https://github.com/chatmail/server/pull/393))
-  ([#394](https://github.com/deltachat/chatmail/pull/394))
+  ([#394](https://github.com/chatmail/server/pull/394))
 
 - add optional "imap_rawlog" config option. If true, 
   .in/.out files are created in user home dirs 
   containing the imap protocol messages. 
-  ([#389](https://github.com/deltachat/chatmail/pull/389))
+  ([#389](https://github.com/chatmail/server/pull/389))
 
 ## 1.4.0 2024-07-28
 
 - Add `disable_ipv6` config option to chatmail.ini.
   Required if the server doesn't have IPv6 connectivity.
-  ([#312](https://github.com/deltachat/chatmail/pull/312))
+  ([#312](https://github.com/chatmail/server/pull/312))
 
 - allow current K9/Thunderbird-mail releases to send encrypted messages
   outside by accepting their localized "encrypted subject" strings. 
-  ([#370](https://github.com/deltachat/chatmail/pull/370))
+  ([#370](https://github.com/chatmail/server/pull/370))
 
 - Migrate and remove sqlite database in favor of password/lastlogin tracking 
   in a user's maildir.  
-  ([#379](https://github.com/deltachat/chatmail/pull/379))
+  ([#379](https://github.com/chatmail/server/pull/379))
 
 - Require pyinfra V3 installed on the client side,
   run `./scripts/initenv.sh` to upgrade locally.
-  ([#378](https://github.com/deltachat/chatmail/pull/378))
+  ([#378](https://github.com/chatmail/server/pull/378))
 
 - don't hardcode "/home/vmail" paths but rather set them 
   once in the config object and use it everywhere else, 
   thereby also improving testability.  
-  ([#351](https://github.com/deltachat/chatmail/pull/351))
+  ([#351](https://github.com/chatmail/server/pull/351))
   temporarily introduced obligatory "passdb_path" and "mailboxes_dir" 
   settings but they were removed/obsoleted in 
-  ([#380](https://github.com/deltachat/chatmail/pull/380))
+  ([#380](https://github.com/chatmail/server/pull/380))
 
 - BREAKING: new required chatmail.ini value 'delete_inactive_users_after = 100'
   which removes users from database and mails after 100 days without any login. 
-  ([#350](https://github.com/deltachat/chatmail/pull/350))
+  ([#350](https://github.com/chatmail/server/pull/350))
 
 - Refine DNS checking to distinguish between "required" and "recommended" settings 
-  ([#372](https://github.com/deltachat/chatmail/pull/372))
+  ([#372](https://github.com/chatmail/server/pull/372))
 
 - reload nginx in the acmetool cronjob
-  ([#360](https://github.com/deltachat/chatmail/pull/360))
+  ([#360](https://github.com/chatmail/server/pull/360))
 
 - remove checking of reverse-DNS PTR records.  Chatmail-servers don't
   depend on it and even in the wider e-mail system it's not common anymore. 
   If it's an issue, a chatmail operator can still care to properly set reverse DNS. 
-  ([#348](https://github.com/deltachat/chatmail/pull/348))
+  ([#348](https://github.com/chatmail/server/pull/348))
 
 - Make DNS-checking faster and more interactive, run it fully during "cmdeploy run",
   also introducing a generic mechanism for rapid remote ssh-based python function execution. 
-  ([#346](https://github.com/deltachat/chatmail/pull/346))
+  ([#346](https://github.com/chatmail/server/pull/346))
 
 - Don't fix file owner ship of /home/vmail 
-  ([#345](https://github.com/deltachat/chatmail/pull/345))
+  ([#345](https://github.com/chatmail/server/pull/345))
 
 - Support iterating over all users with doveadm commands 
-  ([#344](https://github.com/deltachat/chatmail/pull/344))
+  ([#344](https://github.com/chatmail/server/pull/344))
 
 - Test and fix for attempts to create inadmissible accounts 
-  ([#333](https://github.com/deltachat/chatmail/pull/321))
+  ([#333](https://github.com/chatmail/server/pull/321))
 
 - check that OpenPGP has only PKESK, SKESK and SEIPD packets
-  ([#323](https://github.com/deltachat/chatmail/pull/323),
+  ([#323](https://github.com/chatmail/server/pull/323),
-   [#324](https://github.com/deltachat/chatmail/pull/324))
+   [#324](https://github.com/chatmail/server/pull/324))
 
 - improve filtermail checks for encrypted messages and drop support for unencrypted MDNs
-  ([#320](https://github.com/deltachat/chatmail/pull/320))
+  ([#320](https://github.com/chatmail/server/pull/320))
 
 - replace `bash` with `/bin/sh`
-  ([#334](https://github.com/deltachat/chatmail/pull/334))
+  ([#334](https://github.com/chatmail/server/pull/334))
 
 - Increase number of logged in IMAP sessions to 50000
-  ([#335](https://github.com/deltachat/chatmail/pull/335))
+  ([#335](https://github.com/chatmail/server/pull/335))
 
 - filtermail: do not allow ASCII armor without actual payload
-  ([#325](https://github.com/deltachat/chatmail/pull/325))
+  ([#325](https://github.com/chatmail/server/pull/325))
 
 - Remove sieve to enable hardlink deduplication in LMTP
-  ([#343](https://github.com/deltachat/chatmail/pull/343))
+  ([#343](https://github.com/chatmail/server/pull/343))
 
 - dovecot: enable gzip compression on disk
-  ([#341](https://github.com/deltachat/chatmail/pull/341))
+  ([#341](https://github.com/chatmail/server/pull/341))
 
 - DKIM-sign Content-Type and oversign all signed headers
-  ([#296](https://github.com/deltachat/chatmail/pull/296))
+  ([#296](https://github.com/chatmail/server/pull/296))
 
 - Add nonci_accounts metric
-  ([#347](https://github.com/deltachat/chatmail/pull/347))
+  ([#347](https://github.com/chatmail/server/pull/347))
 
 - doveauth: log when a new account is created
-  ([#349](https://github.com/deltachat/chatmail/pull/349))
+  ([#349](https://github.com/chatmail/server/pull/349))
 
 - Multiplex HTTPS, IMAP and SMTP on port 443
-  ([#357](https://github.com/deltachat/chatmail/pull/357))
+  ([#357](https://github.com/chatmail/server/pull/357))
 
 ## 1.3.0 - 2024-06-06
 
 - don't check necessary DNS records on cmdeploy init anymore
-  ([#316](https://github.com/deltachat/chatmail/pull/316))
+  ([#316](https://github.com/chatmail/server/pull/316))
 
 - ensure cron and acl are installed
-  ([#293](https://github.com/deltachat/chatmail/pull/293),
+  ([#293](https://github.com/chatmail/server/pull/293),
-  [#310](https://github.com/deltachat/chatmail/pull/310))
+  [#310](https://github.com/chatmail/server/pull/310))
 
 - change default for delete_mails_after from 40 to 20 days
-  ([#300](https://github.com/deltachat/chatmail/pull/300))
+  ([#300](https://github.com/chatmail/server/pull/300))
 
 - save journald logs only to memory and save nginx logs to journald instead of file
-  ([#299](https://github.com/deltachat/chatmail/pull/299))
+  ([#299](https://github.com/chatmail/server/pull/299))
 
 - fix writing of multiple obs repositories in `/etc/apt/sources.list`
-  ([#290](https://github.com/deltachat/chatmail/pull/290))
+  ([#290](https://github.com/chatmail/server/pull/290))
 
 - metadata: add support for `/shared/vendor/deltachat/irohrelay`
-  ([#284](https://github.com/deltachat/chatmail/pull/284))
+  ([#284](https://github.com/chatmail/server/pull/284))
 
 - Emit "XCHATMAIL" capability from IMAP server 
-  ([#278](https://github.com/deltachat/chatmail/pull/278))
+  ([#278](https://github.com/chatmail/server/pull/278))
 
 - Move echobot `into /var/lib/echobot`
-  ([#281](https://github.com/deltachat/chatmail/pull/281))
+  ([#281](https://github.com/chatmail/server/pull/281))
 
 - Accept Let's Encrypt's new Terms of Services
-  ([#275](https://github.com/deltachat/chatmail/pull/276))
+  ([#275](https://github.com/chatmail/server/pull/276))
 
 - Reload Dovecot and Postfix when TLS certificate updates
-  ([#271](https://github.com/deltachat/chatmail/pull/271))
+  ([#271](https://github.com/chatmail/server/pull/271))
 
 - Use forked version of dovecot without hardcoded delays
-  ([#270](https://github.com/deltachat/chatmail/pull/270))
+  ([#270](https://github.com/chatmail/server/pull/270))
 
 ## 1.2.0 - 2024-04-04
 
 - Install dig on the server to resolve DNS records
-  ([#267](https://github.com/deltachat/chatmail/pull/267))
+  ([#267](https://github.com/chatmail/server/pull/267))
 
 - preserve notification order and exponentially backoff with 
   retries for tokens where we didn't get a successful return
-  ([#265](https://github.com/deltachat/chatmail/pull/263))
+  ([#265](https://github.com/chatmail/server/pull/263))
 
 - Run chatmail-metadata and doveauth as vmail
-  ([#261](https://github.com/deltachat/chatmail/pull/261))
+  ([#261](https://github.com/chatmail/server/pull/261))
 
 - Apply systemd restrictions to echobot
-  ([#259](https://github.com/deltachat/chatmail/pull/259))
+  ([#259](https://github.com/chatmail/server/pull/259))
 
 - re-enable running the CI in pull requests, but not concurrently 
-  ([#258](https://github.com/deltachat/chatmail/pull/258))
+  ([#258](https://github.com/chatmail/server/pull/258))
 
 
 ## 1.1.0 - 2024-03-28
@@ -237,27 +394,27 @@
 ### The changelog starts to record changes from March 15th, 2024 
 
 - Move systemd unit templates to cmdeploy package 
-  ([#255](https://github.com/deltachat/chatmail/pull/255))
+  ([#255](https://github.com/chatmail/server/pull/255))
 
 - Persist push tokens and support multiple device per address 
-  ([#254](https://github.com/deltachat/chatmail/pull/254))
+  ([#254](https://github.com/chatmail/server/pull/254))
 
 - Avoid warning for regular doveauth protocol's hello message. 
-  ([#250](https://github.com/deltachat/chatmail/pull/250))
+  ([#250](https://github.com/chatmail/server/pull/250))
 
 - Fix various tests to pass again with "cmdeploy test". 
-  ([#245](https://github.com/deltachat/chatmail/pull/245),
+  ([#245](https://github.com/chatmail/server/pull/245),
-  [#242](https://github.com/deltachat/chatmail/pull/242)
+  [#242](https://github.com/chatmail/server/pull/242)
 
 - Ensure lets-encrypt certificates are reloaded after renewal 
-  ([#244]) https://github.com/deltachat/chatmail/pull/244
+  ([#244]) https://github.com/chatmail/server/pull/244
 
 - Persist tokens to avoid iOS users loosing push-notifications when the
   chatmail metadata service is restarted (happens regularly during deploys)
-  ([#238](https://github.com/deltachat/chatmail/pull/239)
+  ([#238](https://github.com/chatmail/server/pull/239)
 
 - Fix failing sieve-script compile errors on incoming messages
-  ([#237](https://github.com/deltachat/chatmail/pull/239)
+  ([#237](https://github.com/chatmail/server/pull/239)
 
 - Fix quota reporting after expunging of old mails
-  ([#233](https://github.com/deltachat/chatmail/pull/239)
+  ([#233](https://github.com/chatmail/server/pull/239)

README.md

@@ -1,58 +1,109 @@
 
 <img width="800px" src="www/src/collage-top.png"/>
 
-# Chatmail services optimized for Delta Chat apps 
+# Chatmail relays for end-to-end encrypted e-mail
 
-This repository helps to setup a ready-to-use chatmail server
+Chatmail relay servers are interoperable Mail Transport Agents (MTAs) designed for: 
+
+- **Convenience:** Low friction instant onboarding
+
+- **Privacy:** No name, phone numbers, email required or collected 
+
+- **End-to-End Encryption enforced**: only OpenPGP messages with metadata minimization allowed 
+
+- **Instant:** Privacy-preserving Push Notifications for Apple, Google, and Huawei
+
+- **Speed:** Message delivery in half a second, with optional P2P realtime connections 
+
+- **Transport Security:** Strict TLS and DKIM enforced 
+
+- **Reliability:** No spam or IP reputation checks; rate-limits are suitable for realtime chats
+
+- **Efficiency:** Messages are only stored for transit and removed automatically
+
+This repository contains everything needed to setup a ready-to-use chatmail relay
 comprised of a minimal setup of the battle-tested 
-[postfix smtp](https://www.postfix.org) and [dovecot imap](https://www.dovecot.org) services. 
+[Postfix SMTP](https://www.postfix.org) and [Dovecot IMAP](https://www.dovecot.org) MTAs/MDAs.
 
-The setup is designed and optimized for providing chatmail accounts 
+The automated setup is designed and optimized for providing chatmail addresses
-for use by [Delta Chat apps](https://delta.chat).
+for immediate permission-free onboarding through chat apps and bots.
+Chatmail addresses are automatically created at first login,
+after which the initially specified password is required
+for sending and receiving messages through them.
 
-Chatmail accounts are automatically created by a first login, 
+Please see [this list of known apps and client projects](https://chatmail.at/clients.html) 
-after which the initially specified password is required for using them. 
+and [this list of known public 3rd party chatmail relay servers](https://chatmail.at/relays).
 
-## Deploying your own chatmail server 
 
-To deploy chatmail on your own server, you must have set-up ssh authentication and need to use an ed25519 key, due to an [upstream bug in paramiko](https://github.com/paramiko/paramiko/issues/2191). You also need to add your private key to the local ssh-agent, because you can't type in your password during deployment.
+## Minimal requirements, Prerequisites 
 
-We use `chat.example.org` as the chatmail domain in the following steps. 
+You will need the following: 
+
+- Control over a domain through a DNS provider of your choice.
+
+- A Debian 12 server with reachable SMTP/SUBMISSIONS/IMAPS/HTTPS ports.
+  IPv6 is encouraged if available.
+  Chatmail relay servers only require 1GB RAM, one CPU, and perhaps 10GB storage for a
+  few thousand active chatmail addresses.
+
+- Key-based SSH authentication to the root user.
+  You must add a passphrase-protected private key to your local ssh-agent
+  because you can't type in your passphrase during deployment.
+  (An ed25519 private key is required due to an [upstream bug in paramiko](https://github.com/paramiko/paramiko/issues/2191))
+
+
+## Getting started 
+
+We use `chat.example.org` as the chatmail domain in the following steps.
 Please substitute it with your own domain. 
 
-1. Install the `cmdeploy` command in a virtualenv
+1. Setup the initial DNS records.
+   The following is an example in the familiar BIND zone file format with
+   a TTL of 1 hour (3600 seconds).
+   Please substitute your domain and IP addresses.
+
+   ```
+    chat.example.com. 3600 IN A 198.51.100.5
+    chat.example.com. 3600 IN AAAA 2001:db8::5
+    www.chat.example.com. 3600 IN CNAME chat.example.com.
+    mta-sts.chat.example.com. 3600 IN CNAME chat.example.com.
+   ```
+
+2. On your local PC, clone the repository and bootstrap the Python virtualenv.
+
+   ```
+    git clone https://github.com/chatmail/relay
+    cd relay
+   ```
+
+### Manual installation
+1. On your local PC, create chatmail configuration file `chatmail.ini`:
 
    ```
-    git clone https://github.com/deltachat/chatmail
-    cd chatmail
     scripts/initenv.sh
-   ```
-
-2. Create chatmail configuration file `chatmail.ini`:
-
-   ```
     scripts/cmdeploy init chat.example.org  # <-- use your domain 
    ```
 
-3. Point your domain to the server's IP address,
+2. Verify that SSH root login to your remote server works:
-   if you haven't done so already.
-   Verify that SSH root login works:
 
    ```
-    ssh root@chat.example.org   # <-- use your domain 
+    ssh root@chat.example.org  # <-- use your domain 
    ```
 
-4. Deploy to the remote chatmail server:
+3. From your local PC, deploy the remote chatmail relay server:
 
    ```
     scripts/cmdeploy run
    ```
-   This script will check that you have all necessary DNS records.
+   This script will also check that you have all necessary DNS records.
    If DNS records are missing, it will recommend
    which you should configure at your DNS provider
    (it can take some time until they are public).
 
-### Other helpful commands:
+### Docker installation
+Installation using docker compose is presented [here](./docs/DOCKER_INSTALLATION_EN.md)
+
+### Other helpful commands
 
 To check the status of your remotely running chatmail service:
 
@@ -82,25 +133,25 @@ scripts/cmdeploy bench
 
 This repository has four directories:
 
-- [cmdeploy](https://github.com/deltachat/chatmail/tree/main/cmdeploy)
+- [cmdeploy](https://github.com/chatmail/relay/tree/main/cmdeploy)
   is a collection of configuration files
   and a [pyinfra](https://pyinfra.com)-based deployment script.
 
-- [chatmaild](https://github.com/deltachat/chatmail/tree/main/chatmaild)
+- [chatmaild](https://github.com/chatmail/relay/tree/main/chatmaild)
-  is a python package containing several small services
+  is a Python package containing several small services
   which handle authentication,
   trigger push notifications on new messages,
   ensure that outbound mails are encrypted,
   delete inactive users,
   and some other minor things.
-  chatmaild can also be installed as a stand-alone python package.
+  chatmaild can also be installed as a stand-alone Python package.
 
-- [www](https://github.com/deltachat/chatmail/tree/main/www)
+- [www](https://github.com/chatmail/relay/tree/main/www)
   contains the html, css, and markdown files
-  which make up a chatmail server's web page.
+  which make up a chatmail relay's web page.
-  Edit them before deploying to make your chatmail server stand out.
+  Edit them before deploying to make your chatmail relay stand out.
 
-- [scripts](https://github.com/deltachat/chatmail/tree/main/scripts)
+- [scripts](https://github.com/chatmail/relay/tree/main/scripts)
   offers two convenience tools for beginners;
   `initenv.sh` installs the necessary dependencies to a local virtual environment,
   and the `scripts/cmdeploy` script enables you
@@ -111,80 +162,82 @@ This repository has four directories:
 The `cmdeploy/src/cmdeploy/cmdeploy.py` command line tool
 helps with setting up and managing the chatmail service.
 `cmdeploy init` creates the `chatmail.ini` config file.
-`cmdeploy run` uses a [pyinfra](https://pyinfra.com/)-based [script](`cmdeploy/src/cmdeploy/__init__.py`)
+`cmdeploy run` uses a [pyinfra](https://pyinfra.com/)-based [`script`](cmdeploy/src/cmdeploy/__init__.py)
-to automatically install or upgrade all chatmail components on a server,
+to automatically install or upgrade all chatmail components on a relay,
 according to the `chatmail.ini` config.
 
 The components of chatmail are:
 
-- [postfix smtp server](https://www.postfix.org) accepts sent messages (both from your users and from other servers)
+- [Postfix SMTP MTA](https://www.postfix.org) accepts and relays messages
+  (both from your users and from the wider e-mail MTA network)
 
-- [dovecot imap server](https://www.dovecot.org) stores messages for your users until they download them
+- [Dovecot IMAP MDA](https://www.dovecot.org) stores messages for your users until they download them
 
-- [nginx](https://nginx.org/) shows the web page with your privacy policy and additional information
+- [Nginx](https://nginx.org/) shows the web page with your privacy policy and additional information
 
-- [acmetool](https://hlandau.github.io/acmetool/) manages TLS certificates for dovecot, postfix, and nginx
+- [acmetool](https://hlandau.github.io/acmetool/) manages TLS certificates for Dovecot, Postfix, and Nginx
 
-- [opendkim](http://www.opendkim.org/) for signing messages with DKIM and rejecting inbound messages without DKIM
+- [OpenDKIM](http://www.opendkim.org/) for signing messages with DKIM and rejecting inbound messages without DKIM
 
 - [mtail](https://google.github.io/mtail/) for collecting anonymized metrics in case you have monitoring
 
+- [Iroh relay](https://www.iroh.computer/docs/concepts/relay) 
+  which helps client devices to establish Peer-to-Peer connections 
+
 - and the chatmaild services, explained in the next section:
 
 ### chatmaild
 
-chatmaild offers several commands
+`chatmaild` implements various systemd-controlled services  
-which differentiate a *chatmail* server from a classic mail server.
+that integrate with Dovecot and Postfix to achieve instant-onboarding and 
-If you deploy them with cmdeploy,
+only relaying OpenPGP end-to-end messages encrypted messages. 
-they are run by systemd services in the background.
+A short overview of `chatmaild` services: 
-A short overview:
 
-- [`doveauth`](https://github.com/deltachat/chatmail/blob/main/chatmaild/src/chatmaild/doveauth.py) implements
+- [`doveauth`](https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/doveauth.py) 
-  create-on-login account creation semantics and is used
+  implements create-on-login address semantics and is used 
-  by Dovecot during login authentication and by Postfix
+  by Dovecot during IMAP login and by Postfix during SMTP/SUBMISSION login
   which in turn uses [Dovecot SASL](https://doc.dovecot.org/configuration_manual/authentication/dict/#complete-example-for-authenticating-via-a-unix-socket)
-  to authenticate users
+  to authenticate logins. 
-  to send mails for them.
 
-- [`filtermail`](https://github.com/deltachat/chatmail/blob/main/chatmaild/src/chatmaild/filtermail.py) prevents
+- [`filtermail`](https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/filtermail.py) 
-  unencrypted e-mail from leaving the chatmail service
+  prevents unencrypted email from leaving or entering the chatmail service
-  and is integrated into postfix's outbound mail pipelines.
+  and is integrated into Postfix's outbound and inbound mail pipelines.
 
-- [`chatmail-metadata`](https://github.com/deltachat/chatmail/blob/main/chatmaild/src/chatmaild/metadata.py) is contacted by a
+- [`chatmail-metadata`](https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/metadata.py) is contacted by a
-  [dovecot lua script](https://github.com/deltachat/chatmail/blob/main/cmdeploy/src/cmdeploy/dovecot/push_notification.lua)
+  [Dovecot lua script](https://github.com/chatmail/relay/blob/main/cmdeploy/src/cmdeploy/dovecot/push_notification.lua)
-  to store user-specific server-side config.
+  to store user-specific relay-side config.
   On new messages,
-  it [passes the user's push notification token](https://github.com/deltachat/chatmail/blob/main/chatmaild/src/chatmaild/notifier.py)
+  it [passes the user's push notification token](https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/notifier.py)
   to [notifications.delta.chat](https://delta.chat/help#instant-delivery)
   so the push notifications on the user's phone can be triggered
-  by Apple/Google.
+  by Apple/Google/Huawei.
 
-- [`delete_inactive_users`](https://github.com/deltachat/chatmail/blob/main/chatmaild/src/chatmaild/delete_inactive_users.py)
+- [`delete_inactive_users`](https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/delete_inactive_users.py)
   deletes users if they have not logged in for a very long time.
   The timeframe can be configured in `chatmail.ini`.
 
-- [`lastlogin`](https://github.com/deltachat/chatmail/blob/main/chatmaild/src/chatmaild/lastlogin.py)
+- [`lastlogin`](https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/lastlogin.py)
-  is contacted by dovecot when a user logs in
+  is contacted by Dovecot when a user logs in
   and stores the date of the login.
 
-- [`echobot`](https://github.com/deltachat/chatmail/blob/main/chatmaild/src/chatmaild/echo.py)
+- [`echobot`](https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/echo.py)
   is a small bot for test purposes.
   It simply echoes back messages from users.
 
-- [`chatmail-metrics`](https://github.com/deltachat/chatmail/blob/main/chatmaild/src/chatmaild/metrics.py)
+- [`chatmail-metrics`](https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/metrics.py)
   collects some metrics and displays them at `https://example.org/metrics`.
 
 ### Home page and getting started for users
 
-`cmdeploy run` also creates default static Web pages and deploys them
+`cmdeploy run` also creates default static web pages and deploys them
-to a nginx web server with: 
+to a Nginx web server with:
 
 - a default `index.html` along with a QR code that users can click to
-  create accounts on your chatmail provider,
+  create an address on your chatmail relay 
 
-- a default `info.html` that is linked from the home page,
+- a default `info.html` that is linked from the home page
 
-- a default `policy.html` that is linked from the home page.
+- a default `policy.html` that is linked from the home page
 
 All `.html` files are generated
 by the according markdown `.md` file in the `www/src` directory.
@@ -192,48 +245,64 @@ by the according markdown `.md` file in the `www/src` directory.
 
 ### Refining the web pages
 
-
 ```
 scripts/cmdeploy webdev
 ```
 
-This starts a local live development cycle for chatmail Web pages:
+This starts a local live development cycle for chatmail web pages:
 
 - uses the `www/src/page-layout.html` file for producing static
   HTML pages from `www/src/*.md` files
 
 - continously builds the web presence reading files from `www/src` directory
-  and generating html files and copying assets to the `www/build` directory.
+  and generating HTML files and copying assets to the `www/build` directory.
 
 - Starts a browser window automatically where you can "refresh" as needed.
 
+## Mailbox directory layout
 
-## Emergency Commands to disable automatic account creation
+Fresh chatmail addresses have a mailbox directory that contains: 
 
-If you need to stop account creation,
+- a `password` file with the salted password required for authenticating
-e.g. because some script is wildly creating accounts,
+  whether a login may use the address to send/receive messages. 
-login to the server with ssh and run:
+  If you modify the password file manually, you effectively block the user. 
+
+- `enforceE2EEincoming` is a default-created file with each address. 
+  If present the file indicates that this chatmail address rejects incoming cleartext messages.
+  If absent the address accepts incoming cleartext messages. 
+
+- `dovecot*`, `cur`, `new` and `tmp` represent IMAP/mailbox state. 
+  If the address is only used by one device, the Maildir directories
+  will typically be empty unless the user of that address hasn't been online 
+  for a while. 
+
+
+## Emergency Commands to disable automatic address creation
+
+If you need to stop address creation,
+e.g. because some script is wildly creating addresses, 
+login with ssh and run:
 
 ```
     touch /etc/chatmail-nocreate
 ```
 
-While this file is present, account creation will be blocked.
+Chatmail address creation will be denied while this file is present.
 
 ### Ports
 
-[Postfix](http://www.postfix.org/) listens on ports 25 (smtp) and 587 (submission) and 465 (submissions).
+[Postfix](http://www.postfix.org/) listens on ports 25 (SMTP) and 587 (SUBMISSION) and 465 (SUBMISSIONS).
-[Dovecot](https://www.dovecot.org/) listens on ports 143 (imap) and 993 (imaps).
+[Dovecot](https://www.dovecot.org/) listens on ports 143 (IMAP) and 993 (IMAPS).
-[nginx](https://www.nginx.com/) listens on port 8443 (https-alt) and 443 (https).
+[Nginx](https://www.nginx.com/) listens on port 8443 (HTTPS-ALT) and 443 (HTTPS).
 Port 443 multiplexes HTTPS, IMAP and SMTP using ALPN to redirect connections to ports 8443, 465 or 993.
-[acmetool](https://hlandau.github.io/acmetool/) listens on port 80 (http).
+[acmetool](https://hlandau.github.io/acmetool/) listens on port 80 (HTTP).
 
-Delta Chat apps will, however, discover all ports and configurations
+chatmail-core based apps will, however, discover all ports and configurations
-automatically by reading the [autoconfig XML file](https://www.ietf.org/archive/id/draft-bucksch-autoconfig-00.html) from the chatmail service.
+automatically by reading the [autoconfig XML file](https://www.ietf.org/archive/id/draft-bucksch-autoconfig-00.html) from the chatmail relay server.
 
 ## Email authentication
 
-chatmail servers rely on [DKIM](https://www.rfc-editor.org/rfc/rfc6376)
+Chatmail relays enforce [DKIM](https://www.rfc-editor.org/rfc/rfc6376)
 to authenticate incoming emails.
 Incoming emails must have a valid DKIM signature with
 Signing Domain Identifier (SDID, `d=` parameter in the DKIM-Signature header)
@@ -256,101 +325,130 @@ and rejects incorrectly authenticated emails with [`reject_sender_login_mismatch
 `From:` header must correspond to envelope MAIL FROM,
 this is ensured by `filtermail` proxy.
 
-## Migrating chatmail server to a new host
+## TLS requirements
 
-If you want to migrate chatmail from an old machine
+Postfix is configured to require valid TLS
+by setting [`smtp_tls_security_level`](https://www.postfix.org/postconf.5.html#smtp_tls_security_level) to `verify`.
+If emails don't arrive at your chatmail relay server, 
+the problem is likely that your relay does not have a valid TLS certificate.
+
+You can test it by resolving `MX` records of your relay domain
+and then connecting to MX relays (e.g `mx.example.org`) with
+`openssl s_client -connect mx.example.org:25 -verify_hostname mx.example.org -verify_return_error -starttls smtp`
+from the host that has open port 25 to verify that certificate is valid.
+
+When providing a TLS certificate to your chatmail relay server, 
+make sure to provide the full certificate chain
+and not just the last certificate.
+
+If you are running an Exim server and don't see incoming connections
+from a chatmail relay server in the logs,
+make sure `smtp_no_mail` log item is enabled in the config
+with `log_selector = +smtp_no_mail`.
+By default Exim does not log sessions that are closed
+before sending the `MAIL` command.
+This happens if certificate is not recognized as valid by Postfix,
+so you might think that connection is not established
+while actually it is a problem with your TLS certificate.
+
+## Migrating a chatmail relay to a new host
+
+If you want to migrate chatmail relay from an old machine
 to a new machine,
 you can use these steps.
-They were tested with a linux laptop;
+They were tested with a Linux laptop;
 you might need to adjust some of the steps to your environment.
 
 Let's assume that your `mail_domain` is `mail.example.org`,
 all involved machines run Debian 12,
-your old server's IP address is `13.37.13.37`,
+your old site's IP address is `13.37.13.37`,
-and your new server's IP address is `13.12.23.42`.
+and your new site's IP address is `13.12.23.42`.
 
-During the guide, you might get a warning about changed SSH Host keys;
+Note, you should lower the TTLs of your DNS records to a value
-in this case, just run `ssh-keygen -R "mail.example.org"` as recommended
+such as 300 (5 minutes) so the migration happens as smoothly as possible.
-to make sure you can connect with SSH.
 
-1. First, copy `/var/lib/acme` to the new server with
+During the guide you might get a warning about changed SSH Host keys;
-   `ssh root@13.37.13.37 tar c /var/lib/acme | ssh root@13.12.23.42 tar x -C /var/lib/`.
+in this case, just run `ssh-keygen -R "mail.example.org"` as recommended.
-   This transfers your TLS certificate.
 
-2. You should also copy `/etc/dkimkeys` to the new server with
+1. First, disable mail services on the old site. 
-   `ssh root@13.37.13.37 tar c /etc/dkimkeys | ssh root@13.12.23.42 tar x -C /etc/`
-   so the DKIM DNS record stays correct.
 
-3. On the new server, run `chown root: -R /var/lib/acme` and `chown root: -R /etc/dkimkeys` to make sure the permissions are correct.
+   ```
-
+    cmdeploy run --disable-mail --ssh-host 13.37.13.37
-4. Run `cmdeploy run --disable-mail --ssh-host 13.12.23.42` to install chatmail on the new machine.
+   ```
-   postfix and dovecot are disabled for now,
-   we will enable them later.
-
-5. Now, point DNS to the new IP addresses.
-
-   You can already remove the old IP addresses from DNS.
-   Existing Delta Chat users will still be able to connect
-   to the old server, send and receive messages,
-   but new users will fail to create new profiles
-   with your chatmail server.
-
-   If other servers try to deliver messages to your new server they will fail,
-   but normally email servers will retry delivering messages
-   for at least a week, so messages will not be lost.
-
-6. Now you can run `cmdeploy run --disable-mail --ssh-host 13.37.13.37` to disable your old server.
 
    Now your users will notice the migration
    and will not be able to send or receive messages
    until the migration is completed.
 
-7. After everything is stopped,
+2. Now we want to copy `/home/vmail`, `/var/lib/acme`, `/etc/dkimkeys`, `/run/echobot`, and `/var/spool/postfix` to the new site. 
-    you can copy the `/home/vmail/mail` directory to the new server.
+   Login to the old site while forwarding your SSH agent 
-    It includes all user data, messages, password hashes, etc.
+   so you can copy directly from the old to the new site with your SSH key:
+   ```
+    ssh -A root@13.37.13.37
+    tar c - /home/vmail/mail /var/lib/acme /etc/dkimkeys /run/echobot /var/spool/postfix | ssh root@13.12.23.42 "tar x -C /"
+   ```
 
-    Just run: `ssh root@13.37.13.37 tar c /home/vmail/mail | ssh root@13.12.23.42 tar x -C /home/vmail/`
+   This transfers all addresses, the TLS certificate, DKIM keys (so DKIM DNS record remains valid), and the echobot's password so it continues to function.
+   It also preserves the Postfix mail spool so any messages pending delivery will still be delivered.
 
-    After this, your new server has all the necessary files to start operating :)
+3. Install chatmail on the new machine:
 
-8. To be sure the permissions are still fine,
+   ```
-    run `chown vmail: -R /home/vmail` on the new server.
+    cmdeploy run --disable-mail --ssh-host 13.12.23.42
+   ```
+   Postfix and Dovecot are disabled for now; we will enable them later. 
+   We first need to make the new site fully operational. 
 
-9. Finally, you can run `cmdeploy run` to turn on chatmail on the new server.
+3. On the new site, run the following to ensure the ownership is correct in case UIDs/GIDs changed:
-    Your users can continue using the chatmail server,
+
-    and messages which were sent after step 6. should arrive now.
+   ```
-    Voilà!
+    chown root: -R /var/lib/acme
+    chown opendkim: -R /etc/dkimkeys
+    chown vmail: -R /home/vmail/mail
+    chown echobot: -R /run/echobot
+   ```
+
+4. Now, update DNS entries. 
+
+   If other MTAs try to deliver messages to your chatmail domain they may fail intermittently,
+   as DNS catches up with the new site settings 
+   but normally will retry delivering messages
+   for at least a week, so messages will not be lost.
+
+5. Finally, you can execute `cmdeploy run --ssh-host 13.12.23.42` to turn on chatmail on the new relay.
+   Your users will be able to use the chatmail relay as soon as the DNS changes have propagated.
+   Voilà!
 
 ## Setting up a reverse proxy
 
-A chatmail server does not depend on the client IP address
+A chatmail relay MTA does not track or depend on the client IP address
 for its operation, so it can be run behind a reverse proxy.
 This will not even affect incoming mail authentication
 as DKIM only checks the cryptographic signature
 of the message and does not use the IP address as the input.
 
-For example, you may want to self-host your chatmail server
+For example, you may want to self-host your chatmail relay
 and only use hosted VPS to provide a public IP address
 for client connections and incoming mail.
-You can connect chatmail server to VPS
+You can connect chatmail relay to VPS
 using a tunnel protocol
 such as [WireGuard](https://www.wireguard.com/)
 and setup a reverse proxy on a VPS
-to forward connections to the chatmail server
+to forward connections to the chatmail relay
 over the tunnel.
 You can also setup multiple reverse proxies
-for your chatmail server in different networks
+for your chatmail relay in different networks
-to ensure your server is reachable even when
+to ensure your relay is reachable even when
 one of the IPs becomes inaccessible due to
 hosting or routing problems.
 
-Note that your server still needs
+Note that your chatmail relay still needs
 to be able to make outgoing connections on port 25
 to send messages outside.
 
 To setup a reverse proxy
 (or rather Destination NAT, DNAT)
-for your chatmail server,
+for your chatmail relay,
 put the following configuration in `/etc/nftables.conf`:
 ```
 #!/usr/sbin/nft -f
@@ -362,7 +460,7 @@ define wan = eth0
 # Which ports to proxy.
 #
 # Note that SSH is not proxied
-# so it is possible to log into the proxy server
+# so it is possible to log into the proxy server 
 # and not the original one.
 define ports = { smtp, http, https, imap, imaps, submission, submissions }
 
@@ -425,7 +523,7 @@ table inet filter {
 ```
 
 Run `systemctl enable nftables.service`
-to ensure configuration is reloaded when the proxy server reboots.
+to ensure configuration is reloaded when the proxy relay reboots.
 
 Uncomment in `/etc/sysctl.conf` the following two lines:
 
@@ -434,7 +532,19 @@ net.ipv4.ip_forward=1
 net.ipv6.conf.all.forwarding=1
 ```
 
-Then reboot the server or do `sysctl -p` and `nft -f /etc/nftables.conf`.
+Then reboot the relay or do `sysctl -p` and `nft -f /etc/nftables.conf`.
 
-Once proxy server is set up,
+Once proxy relay is set up,
 you can add its IP address to the DNS.
+
+## Neighbors and Acquaintances
+
+Here are some related projects that you may be interested in:
+
+- [Mox](https://github.com/mjl-/mox): A Golang email server.  [Work is in
+  progress](https://github.com/mjl-/mox/issues/251) to modify it to support all
+  of the features and configuration settings required to operate as a chatmail
+  relay.
+- [Maddy-Chatmail](https://github.com/sadraiiali/maddy_chatmail): a plugin for the
+  [Maddy email server](https://maddy.email/) which aims to implement the
+  chatmail relay features and configuration options.

chatmaild/pyproject.toml

@@ -12,6 +12,7 @@ dependencies = [
   "deltachat-rpc-client",
   "filelock",
   "requests",
+  "crypt-r >= 3.13.1 ; python_version >= '3.11'",
 ]
 
 [tool.setuptools]
@@ -47,6 +48,9 @@ lint.select = [
   "PLE", # Pylint Error
   "PLW", # Pylint Warning
 ]
+lint.ignore = [
+  "PLC0415" # import-outside-top-level
+]
 
 [tool.tox]
 legacy_tox_ini = """

chatmaild/src/chatmaild/common_encrypted_subjects.py

@@ -1,59 +0,0 @@
-"""Generated from deltachat, draft-ietf-lamps-header-protection, and
-encrypted_subject localizations in
-https://github.com/thunderbird/thunderbird-android/
-
-"""
-
-common_encrypted_subjects = {
-    "...",
-    "[...]",
-    "암호화된 메시지",
-    "Ĉifrita mesaĝo",
-    "Courriel chiffré",
-    "Dulrituð skilaboð",
-    "Encrypted Message",
-    "Fersifere berjocht",
-    "Kemennadenn enrineget",
-    "Krüptitud kiri",
-    "Krypterat meddelande",
-    "Krypteret besked",
-    "Kryptert melding",
-    "Mensagem criptografada",
-    "Mensagem encriptada",
-    "Mensaje cifrado",
-    "Mensaxe cifrada",
-    "Mesaj Criptat",
-    "Mesazh i Fshehtëzuar",
-    "Messaggio criptato",
-    "Messaghju cifratu",
-    "Missatge encriptat",
-    "Neges wedi'i Hamgryptio",
-    "Pesan terenkripsi",
-    "Salattu viesti",
-    "Şifreli İleti",
-    "Šifrēta ziņa",
-    "Šifrirana poruka",
-    "Šifrirano sporočilo",
-    "Šifruotas laiškas",
-    "Tin nhắn được mã hóa",
-    "Titkosított üzenet",
-    "Verschlüsselte Nachricht",
-    "Versleuteld bericht",
-    "Zašifrovaná zpráva",
-    "Zaszyfrowana wiadomość",
-    "Zifratu mezua",
-    "Κρυπτογραφημένο μήνυμα",
-    "Зашифроване повідомлення",
-    "Зашифрованное сообщение",
-    "Зашыфраваны ліст",
-    "Криптирано съобщение",
-    "Шифрована порука",
-    "დაშიფრული წერილი",
-    "הודעה מוצפנת",
-    "پیام رمزنگاری‌شده",
-    "رسالة مشفّرة",
-    "എൻക്രിപ്റ്റുചെയ്‌ത സന്ദേശം",
-    "加密邮件",
-    "已加密的訊息",
-    "暗号化されたメッセージ",
-}

chatmaild/src/chatmaild/config.py

@@ -11,7 +11,11 @@ def read_config(inipath):
     assert Path(inipath).exists(), inipath
     cfg = iniconfig.IniConfig(inipath)
     params = cfg.sections["params"]
-    return Config(inipath, params=params)
+    default_config_content = get_default_config_content(params["mail_domain"])
+    df_params = iniconfig.IniConfig("ini", data=default_config_content)["params"]
+    new_params = dict(df_params.items())
+    new_params.update(params)
+    return Config(inipath, params=new_params)
 
 
 class Config:
@@ -22,16 +26,30 @@ class Config:
         self.max_mailbox_size = params["max_mailbox_size"]
         self.max_message_size = int(params.get("max_message_size", "31457280"))
         self.delete_mails_after = params["delete_mails_after"]
+        self.delete_large_after = params["delete_large_after"]
         self.delete_inactive_users_after = int(params["delete_inactive_users_after"])
         self.username_min_length = int(params["username_min_length"])
         self.username_max_length = int(params["username_max_length"])
         self.password_min_length = int(params["password_min_length"])
         self.passthrough_senders = params["passthrough_senders"].split()
         self.passthrough_recipients = params["passthrough_recipients"].split()
+        self.is_development_instance = params.get("is_development_instance", "true").lower() == "true"
         self.filtermail_smtp_port = int(params["filtermail_smtp_port"])
+        self.filtermail_smtp_port_incoming = int(
+            params["filtermail_smtp_port_incoming"]
+        )
         self.postfix_reinject_port = int(params["postfix_reinject_port"])
+        self.postfix_reinject_port_incoming = int(
+            params["postfix_reinject_port_incoming"]
+        )
         self.mtail_address = params.get("mtail_address")
         self.disable_ipv6 = params.get("disable_ipv6", "false").lower() == "true"
+        self.use_foreign_cert_manager = params.get("use_foreign_cert_manager", "false").lower() == "true"
+        self.acme_email = params["acme_email"]
+        self.change_kernel_settings = params.get("change_kernel_settings", "true").lower() == "true"
+        self.fs_inotify_max_user_instances_and_watchers = int(
+            params["fs_inotify_max_user_instances_and_watchers"]
+        )
         self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true"
         if "iroh_relay" not in params:
             self.iroh_relay = "https://" + params["mail_domain"]
@@ -54,7 +72,7 @@ class Config:
     def _getbytefile(self):
         return open(self._inipath, "rb")
 
-    def get_user(self, addr):
+    def get_user(self, addr) -> User:
         if not addr or "@" not in addr or "/" in addr:
             raise ValueError(f"invalid address {addr!r}")
 
@@ -69,6 +87,11 @@ class Config:
 
 def write_initial_config(inipath, mail_domain, overrides):
     """Write out default config file, using the specified config value overrides."""
+    content = get_default_config_content(mail_domain, **overrides)
+    inipath.write_text(content)
+
+
+def get_default_config_content(mail_domain, **overrides):
     from importlib.resources import files
 
     inidir = files(__package__).joinpath("ini")
@@ -100,7 +123,7 @@ def write_initial_config(inipath, mail_domain, overrides):
         lines = []
         for line in content.split("\n"):
             for key, value in privacy.items():
-                value_lines = value.strip().split("\n")
+                value_lines = value.format(mail_domain=mail_domain).strip().split("\n")
                 if not line.startswith(f"{key} =") or not value_lines:
                     continue
                 if len(value_lines) == 1:
@@ -113,5 +136,4 @@ def write_initial_config(inipath, mail_domain, overrides):
             else:
                 lines.append(line)
         content = "\n".join(lines)
-
+    return content
-    inipath.write_text(content)

chatmaild/src/chatmaild/doveauth.py

@@ -1,9 +1,13 @@
-import crypt
 import json
 import logging
 import os
 import sys
 
+try:
+    import crypt_r
+except ImportError:
+    import crypt as crypt_r
+
 from .config import Config, read_config
 from .dictproxy import DictProxy
 from .migrate_db import migrate_from_db_to_maildir
@@ -13,7 +17,7 @@ NOCREATE_FILE = "/etc/chatmail-nocreate"
 
 def encrypt_password(password: str):
     # https://doc.dovecot.org/configuration_manual/authentication/password_schemes/
-    passhash = crypt.crypt(password, crypt.METHOD_SHA512)
+    passhash = crypt_r.crypt(password, crypt_r.METHOD_SHA512)
     return "{SHA512-CRYPT}" + passhash
 
 

chatmaild/src/chatmaild/echo.py

@@ -8,6 +8,7 @@ import logging
 import os
 import subprocess
 import sys
+from pathlib import Path
 
 from deltachat_rpc_client import Bot, DeltaChat, EventType, Rpc, events
 
@@ -97,6 +98,10 @@ def main():
         if not bot.is_configured():
             bot.configure(addr, password)
 
+        # write invite link to working directory
+        invitelink = bot.account.get_qr_code()
+        Path("invite-link.txt").write_text(invitelink)
+
         bot.run_forever()
 
 

chatmaild/src/chatmaild/filtermail.py

@@ -11,10 +11,12 @@ from email.utils import parseaddr
 from smtplib import SMTP as SMTPClient
 
 from aiosmtpd.controller import Controller
+from aiosmtpd.smtp import SMTP
 
-from .common_encrypted_subjects import common_encrypted_subjects
 from .config import read_config
 
+ENCRYPTION_NEEDED_523 = "523 Encryption Needed: Invalid Unencrypted Mail"
+
 
 def check_openpgp_payload(payload: bytes):
     """Checks the OpenPGP payload.
@@ -36,6 +38,12 @@ def check_openpgp_payload(payload: bytes):
 
         packet_type_id = payload[i] & 0x3F
         i += 1
+
+        while payload[i] >= 224 and payload[i] < 255:
+            # Partial body length.
+            partial_length = 1 << (payload[i] & 0x1F)
+            i += 1 + partial_length
+
         if payload[i] < 192:
             # One-octet length.
             body_len = payload[i]
@@ -54,7 +62,7 @@ def check_openpgp_payload(payload: bytes):
             )
             i += 5
         else:
-            # Partial body length is not allowed.
+            # Impossible, partial body length was processed above.
             return False
 
         i += body_len
@@ -81,7 +89,9 @@ def check_armored_payload(payload: str):
         return False
     payload = payload.removeprefix(prefix)
 
-    suffix = "-----END PGP MESSAGE-----\r\n\r\n"
+    while payload.endswith("\r\n"):
+        payload = payload.removesuffix("\r\n")
+    suffix = "-----END PGP MESSAGE-----"
     if not payload.endswith(suffix):
         return False
     payload = payload.removesuffix(suffix)
@@ -100,6 +110,27 @@ def check_armored_payload(payload: str):
         return False
 
 
+def is_securejoin(message):
+    if message.get("secure-join") not in ["vc-request", "vg-request"]:
+        return False
+    if not message.is_multipart():
+        return False
+    parts_count = 0
+    for part in message.iter_parts():
+        parts_count += 1
+        if parts_count > 1:
+            return False
+        if part.is_multipart():
+            return False
+        if part.get_content_type() != "text/plain":
+            return False
+
+        payload = part.get_payload().strip().lower()
+        if payload not in ("secure-join: vc-request", "secure-join: vg-request"):
+            return False
+    return True
+
+
 def check_encrypted(message):
     """Check that the message is an OpenPGP-encrypted message.
 
@@ -107,8 +138,6 @@ def check_encrypted(message):
     """
     if not message.is_multipart():
         return False
-    if message.get("subject") not in common_encrypted_subjects:
-        return False
     if message.get_content_type() != "multipart/encrypted":
         return False
     parts_count = 0
@@ -137,9 +166,19 @@ def check_encrypted(message):
     return True
 
 
-async def asyncmain_beforequeue(config):
+async def asyncmain_beforequeue(config, mode):
-    port = config.filtermail_smtp_port
+    if mode == "outgoing":
-    Controller(BeforeQueueHandler(config), hostname="127.0.0.1", port=port).start()
+        port = config.filtermail_smtp_port
+        handler = OutgoingBeforeQueueHandler(config)
+    else:
+        port = config.filtermail_smtp_port_incoming
+        handler = IncomingBeforeQueueHandler(config)
+    HackedController(
+        handler,
+        hostname="127.0.0.1",
+        port=port,
+        data_size_limit=config.max_message_size,
+    ).start()
 
 
 def recipient_matches_passthrough(recipient, passthrough_recipients):
@@ -151,7 +190,21 @@ def recipient_matches_passthrough(recipient, passthrough_recipients):
     return False
 
 
-class BeforeQueueHandler:
+class HackedController(Controller):
+    def factory(self):
+        return SMTPDiscardRCPTO_options(self.handler, **self.SMTP_kwargs)
+
+
+class SMTPDiscardRCPTO_options(SMTP):
+    def _getparams(self, params):
+        # aiosmtpd's SMTP daemon fails to handle a request if there are RCPT TO options
+        # We just ignore them for our incoming filtermail purposes
+        if len(params) == 1 and params[0].startswith("ORCPT"):
+            return {}
+        return super()._getparams(params)
+
+
+class OutgoingBeforeQueueHandler:
     def __init__(self, config):
         self.config = config
         self.send_rate_limiter = SendRateLimiter()
@@ -176,7 +229,9 @@ class BeforeQueueHandler:
             return error
         logging.info("re-injecting the mail that passed checks")
         client = SMTPClient("localhost", self.config.postfix_reinject_port)
-        client.sendmail(envelope.mail_from, envelope.rcpt_tos, envelope.content)
+        client.sendmail(
+            envelope.mail_from, envelope.rcpt_tos, envelope.original_content
+        )
         return "250 OK"
 
     def check_DATA(self, envelope):
@@ -187,44 +242,86 @@ class BeforeQueueHandler:
         mail_encrypted = check_encrypted(message)
 
         _, from_addr = parseaddr(message.get("from").strip())
-        envelope_from_domain = from_addr.split("@").pop()
 
-        logging.info(f"mime-from: {from_addr} envelope-from: {envelope.mail_from!r}")
         if envelope.mail_from.lower() != from_addr.lower():
             return f"500 Invalid FROM <{from_addr!r}> for <{envelope.mail_from!r}>"
 
-        if mail_encrypted:
+        if mail_encrypted or is_securejoin(message):
-            print("Filtering encrypted mail.", file=sys.stderr)
+            print("Outgoing: Filtering encrypted mail.", file=sys.stderr)
-        else:
+            return
-            print("Filtering unencrypted mail.", file=sys.stderr)
+
+        print("Outgoing: Filtering unencrypted mail.", file=sys.stderr)
 
         if envelope.mail_from in self.config.passthrough_senders:
             return
 
+        # allow self-sent Autocrypt Setup Message
+        if envelope.rcpt_tos == [from_addr]:
+            if message.get("subject") == "Autocrypt Setup Message":
+                if message.get_content_type() == "multipart/mixed":
+                    return
+
         passthrough_recipients = self.config.passthrough_recipients
 
-        is_securejoin = message.get("secure-join") in [
-            "vc-request",
-            "vg-request",
-        ]
-        if is_securejoin:
-            return
-
         for recipient in envelope.rcpt_tos:
-            if envelope.mail_from == recipient:
-                # Always allow sending emails to self.
-                continue
             if recipient_matches_passthrough(recipient, passthrough_recipients):
                 continue
-            res = recipient.split("@")
-            if len(res) != 2:
-                return f"500 Invalid address <{recipient}>"
-            _recipient_addr, recipient_domain = res
 
-            is_outgoing = recipient_domain != envelope_from_domain
+            print("Rejected unencrypted mail.", file=sys.stderr)
-            if is_outgoing and not mail_encrypted:
+            return ENCRYPTION_NEEDED_523
-                print("Rejected unencrypted mail.", file=sys.stderr)
+
-                return f"500 Invalid unencrypted mail to <{recipient}>"
+
+class IncomingBeforeQueueHandler:
+    def __init__(self, config):
+        self.config = config
+
+    async def handle_DATA(self, server, session, envelope):
+        logging.info("handle_DATA before-queue")
+        error = self.check_DATA(envelope)
+        if error:
+            return error
+        logging.info("re-injecting the mail that passed checks")
+
+        # the smtp daemon on reinject_port_incoming gives it to dkim milter
+        # which looks at source address to determine whether to verify or sign
+        client = SMTPClient(
+            "localhost",
+            self.config.postfix_reinject_port_incoming,
+            source_address=("127.0.0.2", 0),
+        )
+        client.sendmail(
+            envelope.mail_from, envelope.rcpt_tos, envelope.original_content
+        )
+        return "250 OK"
+
+    def check_DATA(self, envelope):
+        """the central filtering function for e-mails."""
+        logging.info(f"Processing DATA message from {envelope.mail_from}")
+
+        message = BytesParser(policy=policy.default).parsebytes(envelope.content)
+        mail_encrypted = check_encrypted(message)
+
+        if mail_encrypted or is_securejoin(message):
+            print("Incoming: Filtering encrypted mail.", file=sys.stderr)
+            return
+
+        print("Incoming: Filtering unencrypted mail.", file=sys.stderr)
+
+        # we want cleartext mailer-daemon messages to pass through
+        # chatmail core will typically not display them as normal messages
+        if message.get("auto-submitted"):
+            _, from_addr = parseaddr(message.get("from").strip())
+            if from_addr.lower().startswith("mailer-daemon@"):
+                if message.get_content_type() == "multipart/report":
+                    return
+
+        for recipient in envelope.rcpt_tos:
+            user = self.config.get_user(recipient)
+            if user is None or user.is_incoming_cleartext_ok():
+                continue
+
+            print("Rejected unencrypted mail.", file=sys.stderr)
+            return ENCRYPTION_NEEDED_523
 
 
 class SendRateLimiter:
@@ -243,11 +340,14 @@ class SendRateLimiter:
 
 def main():
     args = sys.argv[1:]
-    assert len(args) == 1
+    assert len(args) == 2
     config = read_config(args[0])
+    mode = args[1]
     logging.basicConfig(level=logging.WARN)
     loop = asyncio.new_event_loop()
     asyncio.set_event_loop(loop)
-    task = asyncmain_beforequeue(config)
+    assert mode in ["incoming", "outgoing"]
+    task = asyncmain_beforequeue(config, mode)
     loop.create_task(task)
+    logging.info("entering serving loop")
     loop.run_forever()

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -23,6 +23,9 @@ max_message_size = 31457280
 # days after which mails are unconditionally deleted
 delete_mails_after = 20
 
+# days after which large messages (>200k) are unconditionally deleted
+delete_large_after = 7
+
 # days after which users without a successful login are deleted (database and mails)
 delete_inactive_users_after = 90
 
@@ -40,21 +43,42 @@ passthrough_senders =
 
 # list of e-mail recipients for which to accept outbound un-encrypted mails
 # (space-separated, item may start with "@" to whitelist whole recipient domains)
-passthrough_recipients = xstore@testrun.org 
+passthrough_recipients = xstore@testrun.org echo@{mail_domain}
 
 #
 # Deployment Details
 #
 
-# where the filtermail SMTP service listens
+# set to "False" to remove the "development instance" banner on the main page.
-filtermail_smtp_port = 10080
+is_development_instance = True
 
-# postfix accepts on the localhost reinject SMTP port
+# SMTP outgoing filtermail and reinjection 
+filtermail_smtp_port = 10080
 postfix_reinject_port = 10025
 
+# SMTP incoming filtermail and reinjection 
+filtermail_smtp_port_incoming = 10081
+postfix_reinject_port_incoming = 10026
+
 # if set to "True" IPv6 is disabled
 disable_ipv6 = False
 
+# if you set "True", acmetool will not be installed and you will have to manage certificates yourself.
+use_foreign_cert_manager = False
+
+# Your email adress, which will be used in acmetool to manage Let's Encrypt SSL certificates. Required if `use_foreign_cert_manager` param set as "False". 
+acme_email = 
+
+#
+# Kernel settings
+#
+
+# if you set "True", the kernel settings will be configured according to the values below
+change_kernel_settings  = True
+
+# change fs.inotify.max_user_instances and fs.inotify.max_user_watches kernel settings
+fs_inotify_max_user_instances_and_watchers = 65535
+
 # Defaults to https://iroh.{{mail_domain}} and running `iroh-relay` on the chatmail
 # service.
 # If you set it to anything else, the service will be disabled

chatmaild/src/chatmaild/ini/override-testrun.ini

@@ -1,7 +1,7 @@
 
 [privacy]
 
-passthrough_recipients = privacy@testrun.org xstore@testrun.org
+passthrough_recipients = privacy@testrun.org xstore@testrun.org echo@{mail_domain}
 
 privacy_postal =
     Merlinux GmbH, Represented by the managing director H. Krekel,

chatmaild/src/chatmaild/metadata.py

@@ -1,5 +1,7 @@
 import logging
 import sys
+import time
+from contextlib import contextmanager
 
 from .config import read_config
 from .dictproxy import DictProxy
@@ -7,8 +9,15 @@ from .filedict import FileDict
 from .notifier import Notifier
 
 
+def _is_valid_token_timestamp(timestamp, now):
+    # Token if invalid after 90 days
+    # or if the timestamp is in the future.
+    return timestamp > now - 3600 * 24 * 90 and timestamp < now + 60
+
+
 class Metadata:
-    # each SETMETADATA on this key appends to a list of unique device tokens
+    # each SETMETADATA on this key appends to dictionary
+    # mapping of unique device tokens
     # which only ever get removed if the upstream indicates the token is invalid
     DEVICETOKEN_KEY = "devicetoken"
 
@@ -18,21 +27,51 @@ class Metadata:
     def get_metadata_dict(self, addr):
         return FileDict(self.vmail_dir / addr / "metadata.json")
 
-    def add_token_to_addr(self, addr, token):
+    @contextmanager
+    def _modify_tokens(self, addr):
         with self.get_metadata_dict(addr).modify() as data:
-            tokens = data.setdefault(self.DEVICETOKEN_KEY, [])
+            tokens = data.setdefault(self.DEVICETOKEN_KEY, {})
-            if token not in tokens:
+            now = int(time.time())
-                tokens.append(token)
+            if isinstance(tokens, list):
+                data[self.DEVICETOKEN_KEY] = tokens = {t: now for t in tokens}
+
+            expired_tokens = [
+                token
+                for token, timestamp in tokens.items()
+                if not _is_valid_token_timestamp(tokens[token], now)
+            ]
+            for expired_token in expired_tokens:
+                del tokens[expired_token]
+
+            yield tokens
+
+    def add_token_to_addr(self, addr, token):
+        with self._modify_tokens(addr) as tokens:
+            tokens[token] = int(time.time())
 
     def remove_token_from_addr(self, addr, token):
-        with self.get_metadata_dict(addr).modify() as data:
+        with self._modify_tokens(addr) as tokens:
-            tokens = data.get(self.DEVICETOKEN_KEY, [])
             if token in tokens:
-                tokens.remove(token)
+                del tokens[token]
 
     def get_tokens_for_addr(self, addr):
         mdict = self.get_metadata_dict(addr).read()
-        return mdict.get(self.DEVICETOKEN_KEY, [])
+        tokens = mdict.get(self.DEVICETOKEN_KEY, {})
+
+        now = int(time.time())
+        if isinstance(tokens, dict):
+            token_list = [
+                token
+                for token, timestamp in tokens.items()
+                if _is_valid_token_timestamp(timestamp, now)
+            ]
+            if len(token_list) < len(tokens):
+                # Some tokens have expired, remove them.
+                with self._modify_tokens(addr) as _tokens:
+                    pass
+        else:
+            token_list = []
+        return token_list
 
 
 class MetadataDictProxy(DictProxy):

chatmaild/src/chatmaild/metrics.py

@@ -11,6 +11,8 @@ def main(vmail_dir=None):
     ci_accounts = 0
 
     for path in Path(vmail_dir).iterdir():
+        if not path.joinpath("cur").is_dir():
+            continue
         accounts += 1
         if path.name[:3] in ("ci-", "ac_"):
             ci_accounts += 1

chatmaild/src/chatmaild/migrate_db.py

@@ -32,7 +32,7 @@ def migrate_from_db_to_maildir(config, chunking=10000):
     # don't transfer special/CI accounts
     rows = [row for row in all_rows if row[0][:3] not in ("ci-", "ac_")]
 
-    logging.info(f"ignoring {len(all_rows)-len(rows)} CI accounts")
+    logging.info(f"ignoring {len(all_rows) - len(rows)} CI accounts")
     logging.info(f"migrating {len(rows)} sqlite database passwords to user dirs")
 
     for i, row in enumerate(rows):

chatmaild/src/chatmaild/notifier.py

@@ -17,11 +17,11 @@ and which are scheduled for retry using exponential back-off timing.
 If a token notification would be scheduled more than DROP_DEADLINE seconds
 after its first attempt, it is dropped with a log error.
 
-Note that tokens are completely opaque to the notification machinery here
+Note that tokens are opaque to the notification machinery here
-and will in the future be encrypted foreclosing all ability to distinguish
+and are encrypted foreclosing all ability to distinguish
 which device token ultimately goes to which phone-provider notification service,
 or to understand the relation of "device tokens" and chatmail addresses.
-The meaning and format of tokens is basically a matter of Delta-Chat Core and
+The meaning and format of tokens is basically a matter of chatmail Core and
 the `notification.delta.chat` service.
 """
 
@@ -95,7 +95,12 @@ class Notifier:
                 logging.warning(f"removing spurious queue item: {queue_path!r}")
                 queue_path.unlink()
                 continue
-            queue_item = PersistentQueueItem.read_from_path(queue_path)
+            try:
+                queue_item = PersistentQueueItem.read_from_path(queue_path)
+            except ValueError:
+                logging.warning(f"removing spurious queue item: {queue_path!r}")
+                queue_path.unlink()
+                continue
             self.queue_for_retry(queue_item)
 
     def queue_for_retry(self, queue_item, retry_num=0):

chatmaild/src/chatmaild/tests/mail-data/asm.eml

@@ -0,0 +1,56 @@
+From: {from_addr}
+To: {to_addr}
+Autocrypt-Setup-Message: v1
+Subject: Autocrypt Setup Message
+Date: Tue, 22 Jan 2019 12:56:29 +0100
+Content-type: multipart/mixed; boundary="Y6fyGi9SoGeH8WwRaEdC6bbBcYOedDzrQ"
+
+--Y6fyGi9SoGeH8WwRaEdC6bbBcYOedDzrQ
+Content-Type: text/plain
+
+This message contains all information to transfer your Autocrypt
+settings along with your secret key securely from your original
+device.
+
+To set up your new device for Autocrypt, please follow the
+instuctions that should be presented by your new device.
+
+You can keep this message and use it as a backup for your secret
+key. If you want to do this, you should write down the Setup Code
+and store it securely.
+--Y6fyGi9SoGeH8WwRaEdC6bbBcYOedDzrQ
+Content-Type: application/autocrypt-setup
+Content-Disposition: attachment; filename="autocrypt-setup-message.html"
+
+<html><body>
+<p>
+This is the Autocrypt setup file used to transfer settings and
+keys between clients. You can decrypt it using the Setup Code
+presented on your old device, and then import the contained key
+into your keyring.
+</p>
+
+<pre>
+-----BEGIN PGP MESSAGE-----
+Passphrase-Format: numeric9x4
+Passphrase-Begin: 17
+
+jA0EBwMCFAxADoCdzeX/0ukBlqI5+pfpKb751qd/7nLNbkpy3gVcaf1QwRPZYt40
+Ynp08UqRQ2g48ZlnzHLSwlTGOPTuv2Jt8ka+pgZ45xzvJSG2gau03xP4VsC271kR
+VmCjdb0Y6Rk96mAwfGzrkbaRQ9Z7fIoL866GOv6h9neiVIkp+JYlTV6ISD0ZQJ4Q
+I6dOQkB/TWZyVjtiJDOQHdfNWliA6NtqaLq19wlu9L5xXjuNpY95KwR8EJXWe0+o
+Y3d2U/KxOAkXKghP2Qg1GtlPVeGC5T4p03TGI6pzKT+kHX6Rrm9wK6sM9aTquMmF
+Vok84Jg1DFnwivWC2RILR81rXi7k/+Y6MUbveFgJ9cQduqpxnmD7TjOblYu7M6zp
+YGAUxh8DRKlIMn2QsA++DBYQ6ACZvwuY8qTDLkqPDo4WqM313dsMJbyGjDdVE7EM
+PESS+RlABETpZXz8g/ycr6DIUNdlbPcmYlsBfHWDOuR2GFFTwmlv5slWS39dJv38
+E0eIe1CwdxI801Se7t7dUUS/ZF8wb6GlmxOcqGbF8eko1Z0S64IAm7/h13MRQCxI
+geQnHfGYVJ2FOimoCMEKwfa9x++RFTDW0u7spDC2uWvK/1viV8OfRppFhLr/kmKb
+18lWXuAz80DAjUDUsVqEq2MvJBJGoCJUEyjuRsLkHYRM5jYk4v50LyyR0Om73nWF
+nZBqmqNzdr7Xb9PHHdFhnEc0VvoYbrcM0RVYcEMW3YbmejM891j1d6Iv+/n/qND/
+NdebGrfWJMmFLf/iEkzTZ3/v5inW9LpWoRc94ioCjJTaEo8Rib6ARRFaJVIsmNXi
+YicFGO98D+zX+a2t9Yz6IpPajVslnOp6ScpmXgts/2XWD7oE+JgxSAqo/dLVsHgP
+Ufo=
+=pulM
+-----END PGP MESSAGE-----
+</pre></body></html>
+--Y6fyGi9SoGeH8WwRaEdC6bbBcYOedDzrQ--

chatmaild/src/chatmaild/tests/mail-data/mailer-daemon.eml

@@ -0,0 +1,46 @@
+Date: Fri, 8 Jul 1994 09:21:47 -0400
+From: Mail Delivery Subsystem <MAILER-DAEMON@example.org>
+Subject: Returned mail: User unknown
+To: <owner-ups-mib@CS.UTK.EDU>
+Auto-Submitted: auto-replied 
+MIME-Version: 1.0
+Content-Type: multipart/report; report-type=delivery-status;
+      boundary="JAA13167.773673707/CS.UTK.EDU"
+
+--JAA13167.773673707/CS.UTK.EDU
+content-type: text/plain; charset=us-ascii
+
+   ----- The following addresses had delivery problems -----
+<arathib@vnet.ibm.com>  (unrecoverable error)
+<wsnell@sdcc13.ucsd.edu>  (unrecoverable error)
+
+--JAA13167.773673707/CS.UTK.EDU
+content-type: message/delivery-status
+
+Reporting-MTA: dns; cs.utk.edu
+
+Original-Recipient: rfc822;arathib@vnet.ibm.com
+Final-Recipient: rfc822;arathib@vnet.ibm.com
+Action: failed
+Status: 5.0.0 (permanent failure)
+Diagnostic-Code: smtp;
+ 550 'arathib@vnet.IBM.COM' is not a registered gateway user
+Remote-MTA: dns; vnet.ibm.com
+
+Original-Recipient: rfc822;johnh@hpnjld.njd.hp.com
+Final-Recipient: rfc822;johnh@hpnjld.njd.hp.com
+Action: delayed
+Status: 4.0.0 (hpnjld.njd.jp.com: host name lookup failure)
+
+Original-Recipient: rfc822;wsnell@sdcc13.ucsd.edu
+Final-Recipient: rfc822;wsnell@sdcc13.ucsd.edu
+Action: failed
+Status: 5.0.0
+Diagnostic-Code: smtp; 550 user unknown
+Remote-MTA: dns; sdcc13.ucsd.edu
+
+--JAA13167.773673707/CS.UTK.EDU
+content-type: message/rfc822
+
+[original message goes here]
+--JAA13167.773673707/CS.UTK.EDU--

chatmaild/src/chatmaild/tests/mail-data/securejoin-vc-fake.eml

@@ -0,0 +1,21 @@
+Subject: Message from {from_addr}
+From: <{from_addr}>
+To: <{to_addr}>
+Date: Sun, 15 Oct 2023 16:43:25 +0000
+Message-ID: <Mr.78MWtlV7RAi.goCFzBhCYfy@c2.testrun.org>
+Chat-Version: 1.0
+Secure-Join: vc-request
+Secure-Join-Invitenumber: RANDOM-TOKEN
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi"
+
+
+--Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi
+Content-Type: text/plain; charset=utf-8
+
+Buy viagra!
+
+
+--Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi--
+
+

chatmaild/src/chatmaild/tests/mail-data/securejoin-vc.eml

@@ -0,0 +1,21 @@
+Subject: Message from {from_addr}
+From: <{from_addr}>
+To: <{to_addr}>
+Date: Sun, 15 Oct 2023 16:43:25 +0000
+Message-ID: <Mr.78MWtlV7RAi.goCFzBhCYfy@c2.testrun.org>
+Chat-Version: 1.0
+Secure-Join: vc-request
+Secure-Join-Invitenumber: RANDOM-TOKEN
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi"
+
+
+--Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi
+Content-Type: text/plain; charset=utf-8
+
+Secure-Join: vc-request
+
+
+--Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi--
+
+

chatmaild/src/chatmaild/tests/plugin.py

@@ -69,12 +69,11 @@ def maildata(request):
 
     assert datadir.exists(), datadir
 
-    def maildata(name, from_addr, to_addr, subject="..."):
+    def maildata(name, from_addr, to_addr, subject="[...]"):
         # Using `.read_bytes().decode()` instead of `.read_text()` to preserve newlines.
         data = datadir.joinpath(name).read_bytes().decode()
-
         text = data.format(from_addr=from_addr, to_addr=to_addr, subject=subject)
-        return BytesParser(policy=policy.default).parsebytes(text.encode())
+        return BytesParser(policy=policy.SMTP).parsebytes(text.encode())
 
     return maildata
 

chatmaild/src/chatmaild/tests/test_config.py

@@ -15,6 +15,14 @@ def test_read_config_basic(example_config):
     assert example_config.mail_domain == "chat.example.org"
 
 
+def test_read_config_basic_using_defaults(tmp_path, maildomain):
+    inipath = tmp_path.joinpath("chatmail.ini")
+    inipath.write_text(f"[params]\nmail_domain = {maildomain}")
+    example_config = read_config(inipath)
+    assert example_config.max_user_send_per_minute == 60
+    assert example_config.filtermail_smtp_port_incoming == 10081
+
+
 def test_read_config_testrun(make_config):
     config = make_config("something.testrun.org")
     assert config.mail_domain == "something.testrun.org"
@@ -27,6 +35,7 @@ def test_read_config_testrun(make_config):
     assert config.max_user_send_per_minute == 60
     assert config.max_mailbox_size == "100M"
     assert config.delete_mails_after == "20"
+    assert config.delete_large_after == "7"
     assert config.username_min_length == 9
     assert config.username_max_length == 9
     assert config.password_min_length == 9

chatmaild/src/chatmaild/tests/test_filtermail.py

@@ -1,11 +1,12 @@
 import pytest
 
 from chatmaild.filtermail import (
-    BeforeQueueHandler,
+    IncomingBeforeQueueHandler,
+    OutgoingBeforeQueueHandler,
     SendRateLimiter,
     check_armored_payload,
     check_encrypted,
-    common_encrypted_subjects,
+    is_securejoin,
 )
 
 
@@ -18,7 +19,13 @@ def maildomain():
 @pytest.fixture
 def handler(make_config, maildomain):
     config = make_config(maildomain)
-    return BeforeQueueHandler(config)
+    return OutgoingBeforeQueueHandler(config)
+
+
+@pytest.fixture
+def inhandler(make_config, maildomain):
+    config = make_config(maildomain)
+    return IncomingBeforeQueueHandler(config)
 
 
 def test_reject_forged_from(maildata, gencreds, handler):
@@ -29,14 +36,14 @@ def test_reject_forged_from(maildata, gencreds, handler):
     # test that the filter lets good mail through
     to_addr = gencreds()[0]
     env.content = maildata(
-        "plain.eml", from_addr=env.mail_from, to_addr=to_addr
+        "encrypted.eml", from_addr=env.mail_from, to_addr=to_addr
     ).as_bytes()
 
     assert not handler.check_DATA(envelope=env)
 
     # test that the filter rejects forged mail
     env.content = maildata(
-        "plain.eml", from_addr="forged@c3.testrun.org", to_addr=to_addr
+        "encrypted.eml", from_addr="forged@c3.testrun.org", to_addr=to_addr
     ).as_bytes()
     error = handler.check_DATA(envelope=env)
     assert "500" in error
@@ -55,19 +62,28 @@ def test_filtermail_no_encryption_detection(maildata):
     assert not check_encrypted(msg)
 
 
-def test_filtermail_encryption_detection(maildata):
+def test_filtermail_securejoin_detection(maildata):
-    for subject in common_encrypted_subjects:
+    msg = maildata(
-        msg = maildata(
+        "securejoin-vc.eml", from_addr="some@example.org", to_addr="other@example.org"
-            "encrypted.eml",
+    )
-            from_addr="1@example.org",
+    assert is_securejoin(msg)
-            to_addr="2@example.org",
-            subject=subject,
-        )
-        assert check_encrypted(msg)
 
-    # if the subject is not a known encrypted subject value, it is not considered ac-encrypted
+    msg = maildata(
-    msg.replace_header("Subject", "Click this link")
+        "securejoin-vc-fake.eml",
-    assert not check_encrypted(msg)
+        from_addr="some@example.org",
+        to_addr="other@example.org",
+    )
+    assert not is_securejoin(msg)
+
+
+def test_filtermail_encryption_detection(maildata):
+    msg = maildata(
+        "encrypted.eml",
+        from_addr="1@example.org",
+        to_addr="2@example.org",
+        subject="Subject does not matter, will be replaced anyway",
+    )
+    assert check_encrypted(msg)
 
 
 def test_filtermail_no_literal_packets(maildata):
@@ -97,7 +113,7 @@ def test_send_rate_limiter():
             break
 
 
-def test_excempt_privacy(maildata, gencreds, handler):
+def test_cleartext_excempt_privacy(maildata, gencreds, handler):
     from_addr = gencreds()[0]
     to_addr = "privacy@testrun.org"
     handler.config.passthrough_recipients = [to_addr]
@@ -118,10 +134,73 @@ def test_excempt_privacy(maildata, gencreds, handler):
         rcpt_tos = [to_addr, false_to]
         content = msg.as_bytes()
 
-    assert "500" in handler.check_DATA(envelope=env2)
+    assert "523" in handler.check_DATA(envelope=env2)
 
 
-def test_passthrough_domains(maildata, gencreds, handler):
+def test_cleartext_self_send_autocrypt_setup_message(maildata, gencreds, handler):
+    from_addr = gencreds()[0]
+    to_addr = from_addr
+
+    msg = maildata("asm.eml", from_addr=from_addr, to_addr=to_addr)
+
+    class env:
+        mail_from = from_addr
+        rcpt_tos = [to_addr]
+        content = msg.as_bytes()
+
+    assert not handler.check_DATA(envelope=env)
+
+
+def test_cleartext_send_fails(maildata, gencreds, handler):
+    from_addr = gencreds()[0]
+    to_addr = gencreds()[0]
+
+    msg = maildata("plain.eml", from_addr=from_addr, to_addr=to_addr)
+
+    class env:
+        mail_from = from_addr
+        rcpt_tos = [to_addr]
+        content = msg.as_bytes()
+
+    res = handler.check_DATA(envelope=env)
+    assert "523 Encryption Needed" in res
+
+
+def test_cleartext_incoming_fails(maildata, gencreds, inhandler):
+    from_addr = gencreds()[0]
+    to_addr, password = gencreds()
+
+    msg = maildata("plain.eml", from_addr=from_addr, to_addr=to_addr)
+
+    class env:
+        mail_from = from_addr
+        rcpt_tos = [to_addr]
+        content = msg.as_bytes()
+
+    user = inhandler.config.get_user(to_addr)
+    user.set_password(password)
+    res = inhandler.check_DATA(envelope=env)
+    assert "523 Encryption Needed" in res
+
+    user.allow_incoming_cleartext()
+    assert not inhandler.check_DATA(envelope=env)
+
+
+def test_cleartext_incoming_mailer_daemon(maildata, gencreds, inhandler):
+    from_addr = "mailer-daemon@example.org"
+    to_addr = gencreds()[0]
+
+    msg = maildata("mailer-daemon.eml", from_addr=from_addr, to_addr=to_addr)
+
+    class env:
+        mail_from = from_addr
+        rcpt_tos = [to_addr]
+        content = msg.as_bytes()
+
+    assert not inhandler.check_DATA(envelope=env)
+
+
+def test_cleartext_passthrough_domains(maildata, gencreds, handler):
     from_addr = gencreds()[0]
     to_addr = "privacy@x.y.z"
     handler.config.passthrough_recipients = ["@x.y.z"]
@@ -142,10 +221,10 @@ def test_passthrough_domains(maildata, gencreds, handler):
         rcpt_tos = [to_addr, false_to]
         content = msg.as_bytes()
 
-    assert "500" in handler.check_DATA(envelope=env2)
+    assert "523" in handler.check_DATA(envelope=env2)
 
 
-def test_passthrough_senders(gencreds, handler, maildata):
+def test_cleartext_passthrough_senders(gencreds, handler, maildata):
     acc1 = gencreds()[0]
     to_addr = "recipient@something.org"
     handler.config.passthrough_senders = [acc1]
@@ -196,10 +275,20 @@ UN4fiB0KR9JyG2ayUdNJVkXZSZLnHyRgiaadlpUo16LVvw==\r
 =b5Kp\r
 -----END PGP MESSAGE-----\r
 \r
+\r
 """
 
     assert check_armored_payload(payload) == True
 
+    payload = payload.removesuffix("\r\n")
+    assert check_armored_payload(payload) == True
+
+    payload = payload.removesuffix("\r\n")
+    assert check_armored_payload(payload) == True
+
+    payload = payload.removesuffix("\r\n")
+    assert check_armored_payload(payload) == True
+
     payload = """-----BEGIN PGP MESSAGE-----\r
 \r
 HELLOWORLD
@@ -215,3 +304,45 @@ HELLOWORLD
 \r
 """
     assert check_armored_payload(payload) == False
+
+    # Test payload using partial body length
+    # as generated by GopenPGP.
+    payload = """-----BEGIN PGP MESSAGE-----\r
+\r
+wV4DdCVjRfOT3TQSAQdAY5+pjT6mlCxPGdR3be4w7oJJRUGIPI/Vnh+mJxGSm34w\r
+LNlVc89S1g22uQYFif2sUJsQWbpoHpNkuWpkSgOaHmNvrZiY/YU5iv+cZ3LbmtUG\r
+0uoBisSHh9O1c+5sYZSbrvYZ1NOwlD7Fv/U5/Mw4E5+CjxfdgNGp5o3DDddzPK78\r
+jseDhdSXxnaiIJC93hxNX6R1RPt3G2gukyzx69wciPQShcF8zf3W3o75Ed7B8etV\r
+QEeB16xzdFhKa9JxdjTu3osgCs21IO7wpcFkjc7nZzlW6jPnELJJaNmv4yOOCjMp\r
+6YAkaN/BkL+jHTznHDuDsT5ilnTXpwHDU1Cm9PIx/KFcNCQnIB+2DcdIHPHUH1ci\r
+jvqoeXAVWjKXEjS7PqPFuP/xGbrWG2ugs+toXJOKbgRkExvKs1dwPFKrgghvCVbW\r
+AcKejQKAPArLwpkA7aD875TZQShvGt74fNs45XBlGOYOnNOAJ1KAmzrXLIDViyyB\r
+kDsmTBk785xofuCkjBpXSe6vsMprPzCteDfaUibh8FHeJjucxPerwuOPEmnogNaf\r
+YyL4+iy8H8I9/p7pmUqILprxTG0jTOtlk0bTVzeiF56W1xbtSEMuOo4oFbQTyOM2\r
+bKXaYo774Jm+rRtKAnnI2dtf9RpK19cog6YNzfYjesLKbXDsPZbN5rmwyFiCvvxC\r
+kQ6JLob+B2fPdY2gzy7LypxktS8Zi1HJcWDHJGVmQodaDLqKUObb4M26bXDe6oxI\r
+NS8PJz5exVbM3KhZnUOEn6PJRBBf5a/ZqxlhZPcQo/oBuhKpBRpO5kSDwPIUByu3\r
+UlXLSkpMqe9pUarAOEuQjfl2RVY7U+RrQYp4YP5keMO+i8NCefAFbowTTufO1JIq\r
+2nVgCi/QVnxZyEc9OYt/8AE3g4cdojE+vsSDifZLSWYIetpfrohHv3dT3StD1QRG\r
+0QE6qq6oKpg/IL0cjvuX4c7a7bslv2fXp8t75y37RU6253qdIebhxc/cRhPbc/yu\r
+p0YLyD4SrvKTLP2ZV95jT4IPEpqm4AN3QmiOzdtqR2gLyb62L8QfqI/FdwsIiRiM\r
+hqydwoqt/lfSqG1WKPh+6EkMkH+TDiCC1BQdbN1MNcyUtcjb35PR2c8Ld2TF3guA\r
+jLIqMt/Vb7hBoMb2FcsOYY25ka9oV62OwgKWLXnFzk+modMR5fzb4kxVVAYEqP+D\r
+T5KO1Vs76v1fyPGOq6BbBCvLwTqe/e6IZInJles4v5jrhnLcGKmNGivCUDe6X6NY\r
+UKNt5RsZllwDQpaAb5dMNhyrk8SgIE7TBI7rvqIdUCE52Vy+0JDxFg5olRpFUfO6\r
+/MyTW3Yo/ekk/npHr7iYYqJTCc21bDGLWQcIo/XO7WPxrKNWGBNPFnkRdw0MaKr4\r
++cEM3V8NFnSEpC12xA+RX/CezuJtwXZK5MpG76eYqMO6qyC+c25YcFecEufDZDxx\r
+ZLqRszVRyxyWPtk/oIeQK2v9wOqY6N9/ff01gHz69vqYqN5bUw/QKZsmx1zW+gPw\r
+6x2tDK2BHeYl182gCbhlKISRFwCtbjqZSkiKWao/VtygHkw0fK34avJuyQ/X9YaN\r
+BRy+7Lf3VA53pnB5WJ1xwRXN8VDvmZeXzv2krHveCMemj0OjnRoCLu117xN0A5m9\r
+Fm/RoDix5PolDHtWTtr2m1n2hp2LHnj8at9lFEd0SKhAYHVL9KjzycwWODZRXt+x\r
+zGDDuooEeTvdY5NLyKcl4gETz1ZP4Ez5jGGjhPSwSpq1mU7UaJ9ZXXdr4KHyifW6\r
+ggNzNsGhXTap7IWZpTtqXABydfiBshmH2NjqtNDwBweJVSgP10+r0WhMWlaZs6xl\r
+V3o5yskJt6GlkwpJxZrTvN6Tiww/eW7HFV6NGf7IRSWY5tJc/iA7/92tOmkdvJ1q\r
+myLbG7cJB787QjplEyVe2P/JBO6xYvbkJLf9Q+HaviTO25rugRSrYsoKMDfO8VlQ\r
+1CcnTPVtApPZJEQzAWJEgVAM8uIlkqWJJMgyWT34sTkdBeCUFGloXQFs9Yxd0AGf\r
+/zHEkYZSTKpVSvAIGu4=\r
+=6iHb\r
+-----END PGP MESSAGE-----\r
+"""
+    assert check_armored_payload(payload) == True

chatmaild/src/chatmaild/tests/test_metadata.py

@@ -242,6 +242,22 @@ def test_requeue_removes_tmp_files(notifier, metadata, testaddr, caplog):
     assert queue_item.addr == testaddr
 
 
+def test_requeue_removes_invalid_files(notifier, metadata, testaddr, caplog):
+    metadata.add_token_to_addr(testaddr, "01234")
+    notifier.new_message_for_addr(testaddr, metadata)
+    # empty/invalid files should be ignored
+    p = notifier.queue_dir.joinpath("1203981203")
+    p.touch()
+    notifier2 = notifier.__class__(notifier.queue_dir)
+    notifier2.requeue_persistent_queue_items()
+    assert "spurious" in caplog.records[0].msg
+    assert not p.exists()
+    assert notifier2.retry_queues[0].qsize() == 1
+    when, queue_item = notifier2.retry_queues[0].get()
+    assert when <= int(time.time())
+    assert queue_item.addr == testaddr
+
+
 def test_start_and_stop_notification_threads(notifier, testaddr):
     threads = notifier.start_notification_threads(None)
     for retry_num, threadlist in threads.items():

chatmaild/src/chatmaild/tests/test_metrics.py

@@ -2,8 +2,15 @@ from chatmaild.metrics import main
 
 
 def test_main(tmp_path, capsys):
+    paths = []
     for x in ("ci-asllkj", "ac_12l3kj", "qweqwe", "ci-l1k2j31l2k3"):
-        tmp_path.joinpath(x).mkdir()
+        p = tmp_path.joinpath(x)
+        p.mkdir()
+        p.joinpath("cur").mkdir()
+        paths.append(p)
+
+    tmp_path.joinpath("nomailbox").mkdir()
+
     main(tmp_path)
     out, _ = capsys.readouterr()
     d = {}

chatmaild/src/chatmaild/tests/test_user.py

@@ -40,3 +40,17 @@ def test_no_mailboxes_dir(testaddr, example_config, tmp_path):
     user.set_password("someeqkjwelkqwjleqwe")
     user.set_last_login_timestamp(100000)
     assert user.get_last_login_timestamp() == 86400
+
+
+def test_set_get_cleartext_flag(testaddr, example_config, tmp_path):
+    p = tmp_path.joinpath("a", "mailboxes")
+    example_config.mailboxes_dir = p
+
+    user = example_config.get_user(testaddr)
+    user.set_password("someeqkjwelkqwjleqwe")
+    user.set_last_login_timestamp(100000)
+    assert user.get_last_login_timestamp() == 86400
+
+    assert not user.is_incoming_cleartext_ok()
+    user.allow_incoming_cleartext()
+    assert user.is_incoming_cleartext_ok()

chatmaild/src/chatmaild/user.py

@@ -13,6 +13,7 @@ class User:
         self.maildir = maildir
         self.addr = addr
         self.password_path = password_path
+        self.enforce_E2EE_path = maildir.joinpath("enforceE2EEincoming")
         self.uid = uid
         self.gid = gid
 
@@ -35,6 +36,13 @@ class User:
         home = str(self.maildir)
         return dict(addr=self.addr, home=home, uid=self.uid, gid=self.gid, password=pw)
 
+    def is_incoming_cleartext_ok(self):
+        return not self.enforce_E2EE_path.exists()
+
+    def allow_incoming_cleartext(self):
+        if self.enforce_E2EE_path.exists():
+            self.enforce_E2EE_path.unlink()
+
     def set_password(self, enc_password):
         """Set the specified password for this user.
 
@@ -50,6 +58,8 @@ class User:
             if not self.addr.startswith("echo@"):
                 logging.error(f"could not write password for: {self.addr}")
                 raise
+        if not self.addr.startswith("echo@"):
+            self.enforce_E2EE_path.touch()
 
     def set_last_login_timestamp(self, timestamp):
         """Track login time with daily granularity

cmdeploy/pyproject.toml

@@ -20,6 +20,7 @@ dependencies = [
   "pytest-xdist", 
   "execnet",
   "imap_tools",
+  "pymdown-extensions",
 ]
 
 [project.scripts]
@@ -41,3 +42,6 @@ lint.select = [
   "PLE", # Pylint Error
   "PLW", # Pylint Warning
 ]
+lint.ignore = [
+  "PLC0415" # import-outside-top-level
+]

cmdeploy/src/cmdeploy/__init__.py

@@ -7,17 +7,35 @@ import io
 import shutil
 import subprocess
 import sys
+from io import StringIO
 from pathlib import Path
 
 from chatmaild.config import Config, read_config
-from pyinfra import host, facts
+from pyinfra import facts, host
+from pyinfra.api import FactBase
 from pyinfra.facts.files import File
-from pyinfra.facts.systemd import SystemdEnabled
+from pyinfra.facts.server import Sysctl
+from pyinfra.facts.systemd import SystemdEnabled, SystemdStatus
 from pyinfra.operations import apt, files, pip, server, systemd
 
 from .acmetool import deploy_acmetool
 
 
+class Port(FactBase):
+    """
+    Returns the process occuping a port.
+    """
+
+    def command(self, port: int) -> str:
+        return (
+            "ss -lptn 'src :%d' | awk 'NR>1 {print $6,$7}' | sed 's/users:((\"//;s/\".*//'"
+            % (port,)
+        )
+
+    def process(self, output: [str]) -> str:
+        return output[0]
+
+
 def _build_chatmaild(dist_dir) -> None:
     dist_dir = Path(dist_dir).resolve()
     if dist_dir.exists():
@@ -78,6 +96,11 @@ def _install_remote_venv_with_chatmaild(config) -> None:
         always_copy=True,
     )
 
+    apt.packages(
+        name="install gcc and headers to build crypt_r source package",
+        packages=["gcc", "python3-dev"],
+    )
+
     server.shell(
         name=f"forced pip-install {dist_file.name}",
         commands=[
@@ -101,12 +124,14 @@ def _install_remote_venv_with_chatmaild(config) -> None:
     for fn in (
         "doveauth",
         "filtermail",
+        "filtermail-incoming",
         "echobot",
         "chatmail-metadata",
         "lastlogin",
     ):
+        execpath = fn if fn != "filtermail-incoming" else "filtermail"
         params = dict(
-            execpath=f"{remote_venv_dir}/bin/{fn}",
+            execpath=f"{remote_venv_dir}/bin/{execpath}",
             config_path=remote_chatmail_inipath,
             remote_venv_dir=remote_venv_dir,
             mail_domain=config.mail_domain,
@@ -210,51 +235,37 @@ def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
         server.shell(
             name="Generate OpenDKIM domain keys",
             commands=[
-                f"opendkim-genkey -D /etc/dkimkeys -d {domain} -s {dkim_selector}"
+                f"/usr/sbin/opendkim-genkey -D /etc/dkimkeys -d {domain} -s {dkim_selector}"
             ],
-            _sudo=True,
+            _use_su_login=True,
-            _sudo_user="opendkim",
+            _su_user="opendkim",
         )
 
+    service_file = files.put(
+        name="Configure opendkim to restart once a day",
+        src=importlib.resources.files(__package__).joinpath("opendkim/systemd.conf"),
+        dest="/etc/systemd/system/opendkim.service.d/10-prevent-memory-leak.conf",
+    )
+    need_restart |= service_file.changed
+
     return need_restart
 
 
-def _install_mta_sts_daemon() -> bool:
+def _uninstall_mta_sts_daemon() -> None:
-    need_restart = False
+    # Remove configuration.
+    files.file("/etc/mta-sts-daemon.yml", present=False)
 
-    config = files.put(
+    files.directory("/usr/local/lib/postfix-mta-sts-resolver", present=False)
-        name="upload postfix-mta-sts-resolver config",
+
-        src=importlib.resources.files(__package__).joinpath(
+    files.file("/etc/systemd/system/mta-sts-daemon.service", present=False)
-            "postfix/mta-sts-daemon.yml"
+
-        ),
+    systemd.service(
-        dest="/etc/mta-sts-daemon.yml",
+        name="Stop MTA-STS daemon",
-        user="root",
+        service="mta-sts-daemon.service",
-        group="root",
+        daemon_reload=True,
-        mode="644",
+        running=False,
+        enabled=False,
     )
-    need_restart |= config.changed
-
-    server.shell(
-        name="install postfix-mta-sts-resolver with pip",
-        commands=[
-            "python3 -m virtualenv /usr/local/lib/postfix-mta-sts-resolver",
-            "/usr/local/lib/postfix-mta-sts-resolver/bin/pip install postfix-mta-sts-resolver",
-        ],
-    )
-
-    systemd_unit = files.put(
-        name="upload mta-sts-daemon systemd unit",
-        src=importlib.resources.files(__package__).joinpath(
-            "postfix/mta-sts-daemon.service"
-        ),
-        dest="/etc/systemd/system/mta-sts-daemon.service",
-        user="root",
-        group="root",
-        mode="644",
-    )
-    need_restart |= systemd_unit.changed
-
-    return need_restart
 
 
 def _configure_postfix(config: Config, debug: bool = False) -> bool:
@@ -307,6 +318,40 @@ def _configure_postfix(config: Config, debug: bool = False) -> bool:
     return need_restart
 
 
+def _install_dovecot_package(package: str, arch: str):
+    arch = "amd64" if arch == "x86_64" else arch
+    arch = "arm64" if arch == "aarch64" else arch
+    url = f"https://download.delta.chat/dovecot/dovecot-{package}_2.3.21%2Bdfsg1-3_{arch}.deb"
+    deb_filename = "/root/" + url.split("/")[-1]
+
+    match (package, arch):
+        case ("core", "amd64"):
+            sha256 = "43f593332e22ac7701c62d58b575d2ca409e0f64857a2803be886c22860f5587"
+        case ("core", "arm64"):
+            sha256 = "4d21eba1a83f51c100f08f2e49f0c9f8f52f721ebc34f75018e043306da993a7"
+        case ("imapd", "amd64"):
+            sha256 = "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86"
+        case ("imapd", "arm64"):
+            sha256 = "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f"
+        case ("lmtpd", "amd64"):
+            sha256 = "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab"
+        case ("lmtpd", "arm64"):
+            sha256 = "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f"
+        case _:
+            apt.packages(packages=[f"dovecot-{package}"])
+            return
+
+    files.download(
+        name=f"Download dovecot-{package}",
+        src=url,
+        dest=deb_filename,
+        sha256sum=sha256,
+        cache_time=60 * 60 * 24 * 365 * 10,  # never redownload the package
+    )
+
+    apt.deb(name=f"Install dovecot-{package}", src=deb_filename)
+
+
 def _configure_dovecot(config: Config, debug: bool = False) -> bool:
     """Configures Dovecot IMAP server."""
     need_restart = False
@@ -350,16 +395,28 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
         config=config,
     )
 
-    # as per https://doc.dovecot.org/configuration_manual/os/
+    # as per https://doc.dovecot.org/2.3/configuration_manual/os/
     # it is recommended to set the following inotify limits
-    for name in ("max_user_instances", "max_user_watches"):
+    if config.change_kernel_settings:
-        key = f"fs.inotify.{name}"
+        for name in ("max_user_instances", "max_user_watches"):
-        server.sysctl(
+            key = f"fs.inotify.{name}"
-            name=f"Change {key}",
+            if host.get_fact(Sysctl)[key] == config.fs_inotify_max_user_instances_and_watchers:
-            key=key,
+                # Skip updating limits if already sufficient
-            value=65535,
+                # (enables running in incus containers where sysctl readonly)
-            persist=True,
+                continue
-        )
+            server.sysctl(
+                name=f"Change {key}",
+                key=key,
+                value=config.fs_inotify_max_user_instances_and_watchers,
+                persist=True,
+            )
+
+    timezone_env = files.line(
+        name="Set TZ environment variable",
+        path="/etc/environment",
+        line="TZ=:/etc/localtime",
+    )
+    need_restart |= timezone_env.changed
 
     return need_restart
 
@@ -442,9 +499,26 @@ def check_config(config):
 
 
 def deploy_mtail(config):
-    apt.packages(
+    # Uninstall mtail package, we are going to install a static binary.
-        name="Install mtail",
+    apt.packages(name="Uninstall mtail", packages=["mtail"], present=False)
-        packages=["mtail"],
+
+    (url, sha256sum) = {
+        "x86_64": (
+            "https://github.com/google/mtail/releases/download/v3.0.8/mtail_3.0.8_linux_amd64.tar.gz",
+            "123c2ee5f48c3eff12ebccee38befd2233d715da736000ccde49e3d5607724e4",
+        ),
+        "aarch64": (
+            "https://github.com/google/mtail/releases/download/v3.0.8/mtail_3.0.8_linux_arm64.tar.gz",
+            "aa04811c0929b6754408676de520e050c45dddeb3401881888a092c9aea89cae",
+        ),
+    }[host.get_fact(facts.server.Arch)]
+
+    server.shell(
+        name="Download mtail",
+        commands=[
+            f"(echo '{sha256sum} /usr/local/bin/mtail' | sha256sum -c) || (curl -L {url} | gunzip | tar -x -f - mtail -O >/usr/local/bin/mtail.new && mv /usr/local/bin/mtail.new /usr/local/bin/mtail)",
+            "chmod 755 /usr/local/bin/mtail",
+        ],
     )
 
     # Using our own systemd unit instead of `/usr/lib/systemd/system/mtail.service`.
@@ -517,9 +591,9 @@ def deploy_iroh_relay(config) -> None:
     need_restart |= systemd_unit.changed
 
     iroh_config = files.put(
-        name=f"Upload iroh-relay config",
+        name="Upload iroh-relay config",
         src=importlib.resources.files(__package__).joinpath("iroh-relay.toml"),
-        dest=f"/etc/iroh-relay.toml",
+        dest="/etc/iroh-relay.toml",
         user="root",
         group="root",
         mode="644",
@@ -549,7 +623,6 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
 
     server.group(name="Create vmail group", group="vmail", system=True)
     server.user(name="Create vmail user", user="vmail", group="vmail", system=True)
-    server.user(name="Create filtermail user", user="filtermail", system=True)
     server.group(name="Create opendkim group", group="opendkim", system=True)
     server.user(
         name="Create opendkim user",
@@ -581,9 +654,15 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
         path="/etc/apt/sources.list",
         line="deb [signed-by=/etc/apt/keyrings/obs-home-deltachat.gpg] https://download.opensuse.org/repositories/home:/deltachat/Debian_12/ ./",
         escape_regex_characters=True,
-        ensure_newline=True,
+        present=False,
     )
 
+    if host.get_fact(Port, port=53) != "unbound":
+        files.line(
+            name="Add 9.9.9.9 to resolv.conf",
+            path="/etc/resolv.conf",
+            line="nameserver 9.9.9.9",
+        )
     apt.update(name="apt update", cache_time=24 * 3600)
     apt.upgrade(name="upgrade apt packages", auto_remove=True)
 
@@ -595,6 +674,14 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
     # Run local DNS resolver `unbound`.
     # `resolvconf` takes care of setting up /etc/resolv.conf
     # to use 127.0.0.1 as the resolver.
+    from cmdeploy.cmdeploy import Out
+
+    process_on_53 = host.get_fact(Port, port=53)
+    if host.get_fact(SystemdStatus, services="unbound").get("unbound.service"):
+        process_on_53 = "unbound"
+    if process_on_53 not in (None, "unbound"):
+        Out().red(f"Can't install unbound: port 53 is occupied by: {process_on_53}")
+        exit(1)
     apt.packages(
         name="Install unbound",
         packages=["unbound", "unbound-anchor", "dnsutils"],
@@ -616,10 +703,12 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
     deploy_iroh_relay(config)
 
     # Deploy acmetool to have TLS certificates.
-    tls_domains = [mail_domain, f"mta-sts.{mail_domain}", f"www.{mail_domain}"]
+    if not config.use_foreign_cert_manager:
-    deploy_acmetool(
+        tls_domains = [mail_domain, f"mta-sts.{mail_domain}", f"www.{mail_domain}"]
-        domains=tls_domains,
+        deploy_acmetool(
-    )
+            email = config.acme_email,
+            domains=tls_domains,
+        )
 
     apt.packages(
         # required for setfacl for echobot
@@ -632,10 +721,10 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
         packages="postfix",
     )
 
-    apt.packages(
+    if not "dovecot.service" in host.get_fact(SystemdEnabled):
-        name="Install Dovecot",
+        _install_dovecot_package("core", host.get_fact(facts.server.Arch))
-        packages=["dovecot-imapd", "dovecot-lmtpd"],
+        _install_dovecot_package("imapd", host.get_fact(facts.server.Arch))
-    )
+        _install_dovecot_package("lmtpd", host.get_fact(facts.server.Arch))
 
     apt.packages(
         name="Install nginx",
@@ -658,8 +747,8 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
     debug = False
     dovecot_need_restart = _configure_dovecot(config, debug=debug)
     postfix_need_restart = _configure_postfix(config, debug=debug)
-    mta_sts_need_restart = _install_mta_sts_daemon()
     nginx_need_restart = _configure_nginx(config)
+    _uninstall_mta_sts_daemon()
 
     _remove_rspamd()
     opendkim_need_restart = _configure_opendkim(mail_domain, "opendkim")
@@ -669,18 +758,10 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
         service="opendkim.service",
         running=True,
         enabled=True,
+        daemon_reload=opendkim_need_restart,
         restarted=opendkim_need_restart,
     )
 
-    systemd.service(
-        name="Start and enable MTA-STS daemon",
-        service="mta-sts-daemon.service",
-        daemon_reload=True,
-        running=True,
-        enabled=True,
-        restarted=mta_sts_need_restart,
-    )
-
     # Dovecot should be started before Postfix
     # because it creates authentication socket
     # required by Postfix.
@@ -707,6 +788,13 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
         enabled=True,
         restarted=nginx_need_restart,
     )
+    
+    systemd.service(
+        name="Start and enable fcgiwrap",
+        service="fcgiwrap.service",
+        running=True,
+        enabled=True,
+    )
 
     # This file is used by auth proxy.
     # https://wiki.debian.org/EtcMailName
@@ -730,10 +818,29 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
         enabled=True,
         restarted=journald_conf.changed,
     )
+    files.directory(
+        name="Ensure old logs on disk are deleted",
+        path="/var/log/journal/",
+        present=False,
+    )
 
     apt.packages(
         name="Ensure cron is installed",
         packages=["cron"],
     )
+    try:
+        git_hash = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode()
+    except Exception:
+        git_hash = "unknown\n"
+    try:
+        git_diff = subprocess.check_output(["git", "diff"]).decode()
+    except Exception:
+        git_diff = ""
+    files.put(
+        name="Upload chatmail relay git commiit hash",
+        src=StringIO(git_hash + git_diff),
+        dest="/etc/chatmail-version",
+        mode="700",
+    )
 
     deploy_mtail(config)

cmdeploy/src/cmdeploy/acmetool/__init__.py

@@ -70,6 +70,6 @@ def deploy_acmetool(email="", domains=[]):
     )
 
     server.shell(
-        name=f"Request certificate for: { ', '.join(domains) }",
+        name=f"Request certificate for: {', '.join(domains)}",
-        commands=[f"acmetool want --xlog.severity=debug { ' '.join(domains)}"],
+        commands=[f"acmetool want --xlog.severity=debug {' '.join(domains)}"],
     )

cmdeploy/src/cmdeploy/acmetool/response-file.yaml.j2

@@ -1,2 +1,2 @@
 "acme-enter-email": "{{ email }}"
-"acme-agreement:https://letsencrypt.org/documents/LE-SA-v1.4-April-3-2024.pdf": true
+"acme-agreement:https://letsencrypt.org/documents/LE-SA-v1.5-February-24-2025.pdf": true

cmdeploy/src/cmdeploy/chatmail.zone.j2

@@ -16,7 +16,7 @@ www.{{ mail_domain }}.               CNAME {{ mail_domain }}.
 ;
 ; Recommended DNS entries for interoperability and security-hardening
 ;
-{{ mail_domain }}.                   TXT "v=spf1 a:{{ mail_domain }} ~all"
+{{ mail_domain }}.                   TXT "v=spf1 a ~all"
 _dmarc.{{ mail_domain }}.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
 
 {% if acme_account_url %}

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -32,17 +32,29 @@ def init_cmd_options(parser):
         action="store",
         help="fully qualified DNS domain name for your chatmail instance",
     )
+    parser.add_argument(
+        "--force",
+        dest="recreate_ini",
+        action="store_true",
+        help="force reacreate ini file",
+    )
 
 
 def init_cmd(args, out):
     """Initialize chatmail config file."""
     mail_domain = args.chatmail_domain
+    inipath = args.inipath
     if args.inipath.exists():
-        print(f"Path exists, not modifying: {args.inipath}")
+        if not args.recreate_ini:
-        return 1
+            print(f"[WARNING] Path exists, not modifying: {inipath}")
-    else:
+            return 1
-        write_initial_config(args.inipath, mail_domain, overrides={})
+        else:
-        out.green(f"created config file for {mail_domain} in {args.inipath}")
+            print(f"[WARNING] Force argument was provided, deleting config file: {inipath}")
+            inipath.unlink()
+            return 0
+    
+    write_initial_config(inipath, mail_domain, overrides={})
+    out.green(f"created config file for {mail_domain} in {inipath}")
 
 
 def run_cmd_options(parser):
@@ -56,12 +68,18 @@ def run_cmd_options(parser):
         "--disable-mail",
         dest="disable_mail",
         action="store_true",
-        help="install/upgrade the server, but disable postfix & dovecot for now"
+        help="install/upgrade the server, but disable postfix & dovecot for now",
     )
     parser.add_argument(
         "--ssh-host",
         dest="ssh_host",
-        help="specify an SSH host to deploy to; uses mail_domain from chatmail.ini by default"
+        help="specify an SSH host to deploy to; uses mail_domain from chatmail.ini by default",
+    )
+    parser.add_argument(
+        "--skip-dns-check",
+        dest="dns_check_disabled",
+        action="store_true",
+        help="disable checks nslookup for dns",
     )
 
 
@@ -70,9 +88,10 @@ def run_cmd(args, out):
 
     sshexec = args.get_sshexec()
     require_iroh = args.config.enable_iroh_relay
-    remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
+    if not args.dns_check_disabled:
-    if not dns.check_initial_remote_data(remote_data, print=out.red):
+        remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
-        return 1
+        if not dns.check_initial_remote_data(remote_data, print=out.red):
+            return 1
 
     env = os.environ.copy()
     env["CHATMAIL_INI"] = args.inipath
@@ -86,15 +105,22 @@ def run_cmd(args, out):
         out.red("Please re-run scripts/initenv.sh to update pyinfra to version 3.")
         return 1
 
-    retcode = out.check_call(cmd, env=env)
+    try:
-    if retcode == 0:
+        retcode = out.check_call(cmd, env=env)
-        out.green("Deploy completed, call `cmdeploy dns` next.")
+        if retcode == 0:
-    elif not remote_data["acme_account_url"]:
+            server_deployed_message = f"Chatmail server started: https://{args.config.mail_domain}/"
-        out.red("Deploy completed but letsencrypt not configured")
+            delimiter_line = "=" * len(server_deployed_message)
-        out.red("Run 'cmdeploy run' again")
+            out.green(f"{delimiter_line}\n{server_deployed_message}\n{delimiter_line}")
-        retcode = 0
+            out.green("Deploy completed, call `cmdeploy dns` next.")
-    else:
+        elif not remote_data["acme_account_url"]:
+            out.red("Deploy completed but letsencrypt not configured")
+            out.red("Run 'cmdeploy run' again")
+            retcode = 0
+        else:
+            out.red("Deploy failed")
+    except subprocess.CalledProcessError:
         out.red("Deploy failed")
+        retcode = 1
     return retcode
 
 
@@ -247,8 +273,17 @@ class Out:
     def green(self, msg, file=sys.stderr):
         print(colored(msg, "green"), file=file)
 
-    def __call__(self, msg, red=False, green=False, file=sys.stdout):
+    def yellow(self, msg, file=sys.stderr):
-        color = "red" if red else ("green" if green else None)
+        print(colored(msg, "yellow"), file=file)
+
+    def __call__(self, msg, red=False, green=False, yellow=False, file=sys.stdout):
+        color = None
+        if red:
+            color = "red"
+        elif green:
+            color = "green"
+        elif yellow:
+            color = "yellow"
         print(colored(msg, color), file=file)
 
     def check_call(self, arg, env=None, quiet=False):
@@ -327,8 +362,9 @@ def main(args=None):
         return parser.parse_args(["-h"])
 
     def get_sshexec():
-        print(f"[ssh] login to {args.config.mail_domain}")
+        host = args.ssh_host if hasattr(args, "ssh_host") and args.ssh_host else args.config.mail_domain
-        return SSHExec(args.config.mail_domain, verbose=args.verbose)
+        print(f"[ssh] login to {host}")
+        return SSHExec(host, verbose=args.verbose)
 
     args.get_sshexec = get_sshexec
 

cmdeploy/src/cmdeploy/deploy.py

@@ -11,7 +11,7 @@ def main():
         "CHATMAIL_INI",
         importlib.resources.files("cmdeploy").joinpath("../../../chatmail.ini"),
     )
-    disable_mail = bool(os.environ.get('CHATMAIL_DISABLE_MAIL'))
+    disable_mail = bool(os.environ.get("CHATMAIL_DISABLE_MAIL"))
 
     deploy_chatmail(config_path, disable_mail)
 

cmdeploy/src/cmdeploy/dns.py

@@ -56,6 +56,11 @@ def check_full_zone(sshexec, remote_data, out, zonefile) -> int:
             out(line)
         out("")
         returncode = 1
+    if remote_data.get("dkim_entry") in required_diff:
+        out(
+            "If the DKIM entry above does not work with your DNS provider, you can try this one:\n"
+        )
+        out(remote_data.get("web_dkim_entry") + "\n")
     if recommended_diff:
         out("WARNING: these recommended DNS entries are not set:\n")
         for line in recommended_diff:

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -177,20 +177,34 @@ service auth-worker {
 }
 
 service imap-login {
-    # High-security mode.
+    # High-performance mode as described in
-    # Each process serves a single connection and exits afterwards.
+    # <https://doc.dovecot.org/2.3/admin_manual/login_processes/#high-performance-mode>
-    # This is the default, but we set it explicitly to be sure.
-    # See <https://doc.dovecot.org/admin_manual/login_processes/#high-security-mode> for details.
-    service_count = 1
-
-    # Inrease the number of simultaneous connections.
     #
-    # As of Dovecot 2.3.19.1 the default is 100 processes.
+    # So-called high-security mode described in
-    # Combined with `service_count = 1` it means only 100 connections
+    # <https://doc.dovecot.org/2.3/admin_manual/login_processes/#high-security-mode>
-    # can be handled simultaneously.
+    # and enabled by default with `service_count = 1` starts one process per connection
-    process_limit = 10000
+    # and has problems logging in thousands of users after Dovecot restart.
+    service_count = 0
+
+    # Increase virtual memory size limit.
+    # Since imap-login processes handle TLS connections
+    # even after logging users in
+    # and many connections are handled by each process,
+    # memory size limit should be increased.
+    #
+    # Otherwise the whole process eventually dies
+    # with an error similar to
+    #   imap-login: Fatal: master: service(imap-login):
+    #   child 1422951 returned error 83
+    #   (Out of memory (service imap-login { vsz_limit=256 MB },
+    #    you may need to increase it)
+    # and takes down all its TLS connections at once.
+    vsz_limit = 1G
 
     # Avoid startup latency for new connections.
+    #
+    # Should be set to at least the number of CPU cores
+    # according to the documentation.
     process_min_avail = 10
 }
 
@@ -209,7 +223,7 @@ ssl = required
 ssl_cert = </var/lib/acme/live/{{ config.mail_domain }}/fullchain
 ssl_key = </var/lib/acme/live/{{ config.mail_domain }}/privkey
 ssl_dh = </usr/share/dovecot/dh.pem
-ssl_min_protocol = TLSv1.2
+ssl_min_protocol = TLSv1.3
 ssl_prefer_server_ciphers = yes
 
 

cmdeploy/src/cmdeploy/dovecot/expunge.cron.j2

@@ -1,3 +1,5 @@
+# delete already seen big mails after 7 days, in the INBOX
+2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/cur/*' -mtime +{{ config.delete_large_after }} -size +200k -type f -delete
 # delete all mails after {{ config.delete_mails_after }} days, in the Inbox
 2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 # or in any IMAP subfolder

cmdeploy/src/cmdeploy/dovecot/push_notification.lua

@@ -2,15 +2,6 @@ function dovecot_lua_notify_begin_txn(user)
   return user
 end
 
-function contains(v, needle)
-  for _, keyword in ipairs(v) do
-    if keyword == needle then
-      return true
-    end
-  end
-  return false
-end
-
 function dovecot_lua_notify_event_message_new(user, event)
   local mbox = user:mailbox(event.mailbox)
   mbox:sync()

cmdeploy/src/cmdeploy/mtail/mtail.service.j2

@@ -3,7 +3,7 @@ Description=mtail
 
 [Service]
 Type=simple
-ExecStart=/bin/sh -c "journalctl -f -o short-iso -n 0 | /usr/bin/mtail --address={{ address }} --port={{ port }} --progs /etc/mtail --logtostderr --logs -"
+ExecStart=/bin/sh -c "journalctl -f -o short-iso -n 0 | /usr/local/bin/mtail --address={{ address }} --port={{ port }} --progs /etc/mtail --logtostderr --logs -"
 Restart=on-failure
 
 [Install]

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -2,11 +2,25 @@ load_module modules/ngx_stream_module.so;
 
 user www-data;
 worker_processes auto;
+
+# Increase the number of connections
+# that a worker process can open
+# to avoid errors such as
+#   accept4() failed (24: Too many open files)
+# and
+#   socket() failed (24: Too many open files) while connecting to upstream
+# in the logs.
+# <https://nginx.org/en/docs/ngx_core_module.html#worker_rlimit_nofile>
+worker_rlimit_nofile 2048;
 pid /run/nginx.pid;
 error_log syslog:server=unix:/dev/log,facility=local3;
 
 events {
-	worker_connections 768;
+        # Increase to avoid errors such as
+        #   768 worker_connections are not enough while connecting to upstream
+        # in the logs.
+        # <https://nginx.org/en/docs/ngx_core_module.html#worker_connections>
+	worker_connections 2048;
 	# multi_accept on;
 }
 
@@ -46,10 +60,7 @@ http {
 
 	server {
   
-		listen 8443 ssl default_server;
+		listen 127.0.0.1:8443 ssl default_server;
-		{% if not disable_ipv6 %}
-		listen [::]:8443 ssl default_server;
-		{% endif %}
 
 		root /var/www/html;
 
@@ -120,10 +131,7 @@ http {
 
 	# Redirect www. to non-www
 	server {
-		listen 8443 ssl;
+		listen 127.0.0.1:8443 ssl;
-		{% if not disable_ipv6 %}
-		listen [::]:8443 ssl;
-		{% endif %}
 		server_name www.{{ config.domain_name }};
 		return 301 $scheme://{{ config.domain_name }}$request_uri;
 		access_log syslog:server=unix:/dev/log,facility=local7;

cmdeploy/src/cmdeploy/opendkim/systemd.conf

@@ -0,0 +1,3 @@
+[Service]
+Restart=always
+RuntimeMaxSec=1d

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -20,9 +20,12 @@ smtpd_tls_key_file=/var/lib/acme/live/{{ config.mail_domain }}/privkey
 smtpd_tls_security_level=may
 
 smtp_tls_CApath=/etc/ssl/certs
-smtp_tls_security_level=may
+smtp_tls_security_level=verify
+# Send SNI extension when connecting to other servers.
+# <https://www.postfix.org/postconf.5.html#smtp_tls_servername>
+smtp_tls_servername = hostname
 smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
-smtp_tls_policy_maps = socketmap:inet:127.0.0.1:8461:postfix
+smtp_tls_policy_maps = inline:{nauta.cu=may}
 smtpd_tls_protocols = >=TLSv1.2
 
 # Disable anonymous cipher suites

cmdeploy/src/cmdeploy/postfix/master.cf.j2

@@ -14,10 +14,11 @@ smtp      inet  n       -       y       -       -       smtpd -v
 {%- else %}
 smtp      inet  n       -       y       -       -       smtpd 
 {%- endif %}
-  -o smtpd_milters=unix:opendkim/opendkim.sock
+  -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port_incoming }}
 submission inet n       -       y       -       5000    smtpd
   -o syslog_name=postfix/submission
   -o smtpd_tls_security_level=encrypt
+  -o smtpd_tls_mandatory_protocols=>=TLSv1.3
   -o smtpd_sasl_auth_enable=yes
   -o smtpd_sasl_type=dovecot
   -o smtpd_sasl_path=private/auth
@@ -31,11 +32,11 @@ submission inet n       -       y       -       5000    smtpd
   -o milter_macro_daemon_name=ORIGINATING
   -o smtpd_client_connection_count_limit=1000
   -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
-  -o cleanup_service_name=authclean
 smtps     inet  n       -       y       -       5000    smtpd
   -o syslog_name=postfix/smtps
   -o smtpd_tls_wrappermode=yes
   -o smtpd_tls_security_level=encrypt
+  -o smtpd_tls_mandatory_protocols=>=TLSv1.3
   -o smtpd_sasl_auth_enable=yes
   -o smtpd_sasl_type=dovecot
   -o smtpd_sasl_path=private/auth
@@ -48,7 +49,6 @@ smtps     inet  n       -       y       -       5000    smtpd
   -o smtpd_client_connection_count_limit=1000
   -o milter_macro_daemon_name=ORIGINATING
   -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
-  -o cleanup_service_name=authclean
 #628       inet  n       -       y       -       -       qmqpd
 pickup    unix  n       -       y       60      1       pickup
 cleanup   unix  n       -       y       -       0       cleanup
@@ -76,17 +76,27 @@ anvil     unix  -       -       y       -       1       anvil
 scache    unix  -       -       y       -       1       scache
 postlog   unix-dgram n  -       n       -       1       postlogd
 filter    unix -        n       n       -       -       lmtp
-# Local SMTP server for reinjecting filered mail.
+# Local SMTP server for reinjecting outgoing filtered mail.
-localhost:{{ config.postfix_reinject_port }} inet  n       -       n       -       10      smtpd
+127.0.0.1:{{ config.postfix_reinject_port }} inet  n       -       n       -       10      smtpd
   -o syslog_name=postfix/reinject
   -o smtpd_milters=unix:opendkim/opendkim.sock
   -o cleanup_service_name=authclean
 
+# Local SMTP server for reinjecting incoming filtered mail 
+127.0.0.1:{{ config.postfix_reinject_port_incoming }} inet  n       -       n       -       10      smtpd
+  -o syslog_name=postfix/reinject_incoming
+  -o smtpd_milters=unix:opendkim/opendkim.sock
+
 # Cleanup `Received` headers for authenticated mail
 # to avoid leaking client IP.
 #
 # We do not do this for received mails
 # as this will break DKIM signatures
 # if `Received` header is signed.
+#
+# This service also rewrites
+# Subject with `[...]`
+# to make sure the users
+# cannot send unprotected Subject.
 authclean unix  n       -       -       -       0       cleanup
   -o header_checks=regexp:/etc/postfix/submission_header_cleanup

cmdeploy/src/cmdeploy/postfix/submission_header_cleanup

@@ -2,3 +2,4 @@
 /^X-Originating-IP:/    IGNORE
 /^X-Mailer:/            IGNORE
 /^User-Agent:/          IGNORE
+/^Subject:/             REPLACE Subject: [...]

cmdeploy/src/cmdeploy/remote/rdns.py

@@ -19,7 +19,7 @@ def perform_initial_checks(mail_domain):
     """Collecting initial DNS settings."""
     assert mail_domain
     if not shell("dig", fail_ok=True):
-        shell("apt-get install -y dnsutils")
+        shell("apt-get update && apt-get install -y dnsutils")
     A = query_dns("A", mail_domain)
     AAAA = query_dns("AAAA", mail_domain)
     MTA_STS = query_dns("CNAME", f"mta-sts.{mail_domain}")
@@ -27,7 +27,9 @@ def perform_initial_checks(mail_domain):
 
     res = dict(mail_domain=mail_domain, A=A, AAAA=AAAA, MTA_STS=MTA_STS, WWW=WWW)
     res["acme_account_url"] = shell("acmetool account-url", fail_ok=True)
-    res["dkim_entry"] = get_dkim_entry(mail_domain, dkim_selector="opendkim")
+    res["dkim_entry"], res["web_dkim_entry"] = get_dkim_entry(
+        mail_domain, dkim_selector="opendkim"
+    )
 
     if not MTA_STS or not WWW or (not A and not AAAA):
         return res
@@ -48,7 +50,11 @@ def get_dkim_entry(mail_domain, dkim_selector):
         return
     dkim_value_raw = f"v=DKIM1;k=rsa;p={dkim_pubkey};s=email;t=s"
     dkim_value = '" "'.join(re.findall(".{1,255}", dkim_value_raw))
-    return f'{dkim_selector}._domainkey.{mail_domain}. TXT "{dkim_value}"'
+    web_dkim_value = "".join(re.findall(".{1,255}", dkim_value_raw))
+    return (
+        f'{dkim_selector}._domainkey.{mail_domain}. TXT "{dkim_value}"',
+        f'{dkim_selector}._domainkey.{mail_domain}. TXT "{web_dkim_value}"',
+    )
 
 
 def query_dns(typ, domain):

cmdeploy/src/cmdeploy/remote/rshell.py

@@ -1,10 +1,13 @@
-from subprocess import CalledProcessError, check_output
+from subprocess import DEVNULL, CalledProcessError, check_output
 
 
 def shell(command, fail_ok=False):
     print(f"$ {command}")
+    args = dict(shell=True)
+    if fail_ok:
+        args["stderr"] = DEVNULL
     try:
-        return check_output(command, shell=True).decode().rstrip()
+        return check_output(command, **args).decode().rstrip()
     except CalledProcessError:
         if not fail_ok:
             raise
@@ -14,3 +17,22 @@ def shell(command, fail_ok=False):
 def get_systemd_running():
     lines = shell("systemctl --type=service --state=running").split("\n")
     return [line for line in lines if line.startswith("  ")]
+
+
+def write_numbytes(path, num):
+    with open(path, "w") as f:
+        f.write("x" * num)
+
+
+def dovecot_recalc_quota(user):
+    shell(f"doveadm quota recalc -u {user}")
+    output = shell(f"doveadm quota get -u {user}")
+    #
+    # Quota name Type    Value  Limit                                              %
+    # User quota STORAGE     5 102400                                              0
+    # User quota MESSAGE     2      -                                              0
+    #
+    for line in output.split("\n"):
+        parts = line.split()
+        if parts[2] == "STORAGE":
+            return dict(value=int(parts[3]), limit=int(parts[4]), percent=int(parts[5]))

cmdeploy/src/cmdeploy/service/chatmail-metadata.service.f

@@ -7,6 +7,7 @@ Restart=always
 RestartSec=30
 User=vmail
 RuntimeDirectory=chatmail-metadata
+UMask=0077
 
 [Install]
 WantedBy=multi-user.target

cmdeploy/src/cmdeploy/service/doveauth.service.f

@@ -7,6 +7,7 @@ Restart=always
 RestartSec=30
 User=vmail
 RuntimeDirectory=doveauth
+UMask=0077
 
 [Install]
 WantedBy=multi-user.target

cmdeploy/src/cmdeploy/service/filtermail-incoming.service.f

@@ -0,0 +1,12 @@
+[Unit]
+Description=Incoming Chatmail Postfix before queue filter 
+
+[Service]
+ExecStart={execpath} {config_path} incoming
+Restart=always
+RestartSec=30
+User=vmail
+
+[Install]
+WantedBy=multi-user.target
+

cmdeploy/src/cmdeploy/service/filtermail.service.f

@@ -1,11 +1,11 @@
 [Unit]
-Description=Chatmail Postfix before queue filter
+Description=Outgoing Chatmail Postfix before queue filter
 
 [Service]
-ExecStart={execpath} {config_path}
+ExecStart={execpath} {config_path} outgoing
 Restart=always
 RestartSec=30
-User=filtermail
+User=vmail
 
 [Install]
 WantedBy=multi-user.target

cmdeploy/src/cmdeploy/tests/online/benchmark.py

@@ -37,7 +37,7 @@ class TestDC:
 
     def test_ping_pong(self, benchmark, cmfactory):
         ac1, ac2 = cmfactory.get_online_accounts(2)
-        chat = cmfactory.get_accepted_chat(ac1, ac2)
+        chat = cmfactory.get_protected_chat(ac1, ac2)
 
         def dc_ping_pong():
             chat.send_text("ping")
@@ -49,7 +49,7 @@ class TestDC:
 
     def test_send_10_receive_10(self, benchmark, cmfactory, lp):
         ac1, ac2 = cmfactory.get_online_accounts(2)
-        chat = cmfactory.get_accepted_chat(ac1, ac2)
+        chat = cmfactory.get_protected_chat(ac1, ac2)
 
         def dc_send_10_receive_10():
             for i in range(10):

cmdeploy/src/cmdeploy/tests/online/test_0_login.py

@@ -90,8 +90,13 @@ def test_concurrent_logins_same_account(
 
 
 def test_no_vrfy(chatmail_config):
+    domain = chatmail_config.mail_domain
     sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    sock.connect((chatmail_config.mail_domain, 25))
+    sock.settimeout(10)
+    try:
+        sock.connect((domain, 25))
+    except socket.timeout:
+        pytest.skip(f"port 25 not reachable for {domain}")
     banner = sock.recv(1024)
     print(banner)
     sock.send(b"VRFY wrongaddress@%s\r\n" % (chatmail_config.mail_domain.encode(),))

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -1,4 +1,7 @@
+import datetime
 import smtplib
+import socket
+import subprocess
 
 import pytest
 
@@ -52,6 +55,23 @@ class TestSSHExecutor:
         else:
             pytest.fail("didn't raise exception")
 
+    def test_opendkim_restarted(self, sshexec):
+        """check that opendkim is not running for longer than a day."""
+        cmd = "systemctl show opendkim --timestamp=utc --property=ActiveEnterTimestamp"
+        out = sshexec(call=remote.rshell.shell, kwargs=dict(command=cmd))
+        datestring = out.split("=")[1]
+        since_date = datetime.datetime.strptime(datestring, "%a %Y-%m-%d %H:%M:%S %Z")
+        now = datetime.datetime.now(since_date.tzinfo)
+        assert (now - since_date).total_seconds() < 60 * 60 * 51
+
+
+def test_timezone_env(remote):
+    for line in remote.iter_output("env"):
+        print(line)
+        if line == "tz=:/etc/localtime":
+            return True
+    pytest.fail("TZ is not set")
+
 
 def test_remote(remote, imap_or_smtp):
     lineproducer = remote.iter_output(imap_or_smtp.logcmd)
@@ -107,14 +127,46 @@ def test_authenticated_from(cmsetup, maildata):
 
 @pytest.mark.parametrize("from_addr", ["fake@example.org", "fake@testrun.org"])
 def test_reject_missing_dkim(cmsetup, maildata, from_addr):
-    """Test that emails with missing or wrong DMARC, DKIM, and SPF entries are rejected."""
+    domain = cmsetup.maildomain
+    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    sock.settimeout(10)
+    try:
+        sock.connect((domain, 25))
+    except socket.timeout:
+        pytest.skip(f"port 25 not reachable for {domain}")
+
     recipient = cmsetup.gen_users(1)[0]
-    msg = maildata("plain.eml", from_addr=from_addr, to_addr=recipient.addr).as_string()
+    msg = maildata(
-    with smtplib.SMTP(cmsetup.maildomain, 25) as s:
+        "encrypted.eml", from_addr=from_addr, to_addr=recipient.addr
+    ).as_string()
+    conn = smtplib.SMTP(cmsetup.maildomain, 25, timeout=10)
+
+    with conn as s:
         with pytest.raises(smtplib.SMTPDataError, match="No valid DKIM signature"):
             s.sendmail(from_addr=from_addr, to_addrs=recipient.addr, msg=msg)
 
 
+def test_rewrite_subject(cmsetup, maildata):
+    """Test that subject gets replaced with [...]."""
+    user1, user2 = cmsetup.gen_users(2)
+
+    sent_msg = maildata(
+        "encrypted.eml",
+        from_addr=user1.addr,
+        to_addr=user2.addr,
+        subject="Unencrypted subject",
+    ).as_string()
+    user1.smtp.sendmail(from_addr=user1.addr, to_addrs=[user2.addr], msg=sent_msg)
+
+    messages = user2.imap.fetch_all_messages()
+    assert len(messages) == 1
+    rcvd_msg = messages[0]
+    assert "Subject: [...]" not in sent_msg
+    assert "Subject: [...]" in rcvd_msg
+    assert "Subject: Unencrypted subject" in sent_msg
+    assert "Subject: Unencrypted subject" not in rcvd_msg
+
+
 @pytest.mark.slow
 def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
     """Test that the per-account send-mail limit is exceeded."""
@@ -147,6 +199,25 @@ def test_expunged(remote, chatmail_config):
         f"find {chatmail_config.mailboxes_dir} -path '*/tmp/*' -mtime +{outdated_days} -type f",
         f"find {chatmail_config.mailboxes_dir} -path '*/.*/tmp/*' -mtime +{outdated_days} -type f",
     ]
+    outdated_days = int(chatmail_config.delete_large_after) + 1
+    find_cmds.append(
+        "find {chatmail_config.mailboxes_dir} -path '*/cur/*' -mtime +{outdated_days} -size +200k -type f"
+    )
     for cmd in find_cmds:
         for line in remote.iter_output(cmd):
             assert not line
+
+
+def test_deployed_state(remote):
+    git_hash = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode()
+    git_diff = subprocess.check_output(["git", "diff"]).decode()
+    git_status = [git_hash.strip()]
+    for line in git_diff.splitlines():
+        git_status.append(line.strip().lower())
+    remote_version = []
+    for line in remote.iter_output("cat /etc/chatmail-version"):
+        print(line)
+        remote_version.append(line)
+    # assert len(git_status) == len(remote_version)  # for some reason, we only get 11 lines from remote.iter_output()
+    for i in range(len(remote_version)):
+        assert git_status[i] == remote_version[i], "You have undeployed changes."

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -1,5 +1,4 @@
 import ipaddress
-import random
 import re
 import time
 
@@ -7,6 +6,9 @@ import imap_tools
 import pytest
 import requests
 
+from cmdeploy.remote import rshell
+from cmdeploy.sshexec import SSHExec
+
 
 @pytest.fixture
 def imap_mailbox(cmfactory):
@@ -54,22 +56,23 @@ class TestEndToEndDeltaChat:
         """Test that a DC account can send a message to a second DC account
         on the same chat-mail instance."""
         ac1, ac2 = cmfactory.get_online_accounts(2)
-        chat = cmfactory.get_accepted_chat(ac1, ac2)
+        chat = cmfactory.get_protected_chat(ac1, ac2)
-
-        lp.sec("ac1: prepare and send text message to ac2")
         chat.send_text("message0")
 
         lp.sec("wait for ac2 to receive message")
         msg2 = ac2._evtracker.wait_next_incoming_message()
         assert msg2.text == "message0"
 
-    @pytest.mark.slow
+    def test_exceed_quota(
-    def test_exceed_quota(self, cmfactory, lp, tmpdir, remote, chatmail_config):
+        self, cmfactory, lp, tmpdir, remote, chatmail_config, sshdomain
+    ):
         """This is a very slow test as it needs to upload >100MB of mail data
         before quota is exceeded, and thus depends on the speed of the upload.
         """
         ac1, ac2 = cmfactory.get_online_accounts(2)
-        chat = cmfactory.get_accepted_chat(ac1, ac2)
+        chat = cmfactory.get_protected_chat(ac1, ac2)
+
+        user = ac2.get_config("configured_addr")
 
         def parse_size_limit(limit: str) -> int:
             """Parse a size limit and return the number of bytes as integer.
@@ -82,49 +85,27 @@ class TestEndToEndDeltaChat:
             return int(float(number) * units[unit])
 
         quota = parse_size_limit(chatmail_config.max_mailbox_size)
-        attachsize = 1 * 1024 * 1024
-        num_to_send = quota // attachsize + 2
-        lp.sec(f"ac1: send {num_to_send} large files to ac2")
-        lp.indent(f"per-user quota is assumed to be: {quota/(1024*1024)}MB")
-        alphanumeric = "abcdefghijklmnopqrstuvwxyz1234567890"
-        msgs = []
-        for i in range(num_to_send):
-            attachment = tmpdir / f"attachment{i}"
-            data = "".join(random.choice(alphanumeric) for i in range(1024))
-            with open(attachment, "w+") as f:
-                for j in range(attachsize // len(data)):
-                    f.write(data)
 
-            msg = chat.send_file(str(attachment))
+        lp.sec(f"filling remote inbox for {user}")
-            msgs.append(msg)
+        fn = f"7743102289.M843172P2484002.c20,S={quota},W=2398:2,"
-            lp.indent(f"Sent out msg {i}, size {attachsize/(1024*1024)}MB")
+        path = chatmail_config.mailboxes_dir.joinpath(user, "cur", fn)
+        sshexec = SSHExec(sshdomain)
+        sshexec(call=rshell.write_numbytes, kwargs=dict(path=str(path), num=120))
+        res = sshexec(call=rshell.dovecot_recalc_quota, kwargs=dict(user=user))
+        assert res["percent"] >= 100
 
-        lp.sec("ac2: check messages are arriving until quota is reached")
+        lp.sec("ac2: check quota is triggered")
 
-        addr = ac2.get_config("addr").lower()
+        starting = True
-        saved_ok = 0
         for line in remote.iter_output("journalctl -n0 -f -u dovecot"):
-            if addr not in line:
+            if starting:
+                chat.send_text("hello")
+                starting = False
+            if user not in line:
                 # print(line)
                 continue
-            if "quota" in line:
+            if "quota exceeded" in line:
-                if "quota exceeded" in line:
+                return
-                    if saved_ok < num_to_send // 2:
-                        pytest.fail(
-                            f"quota exceeded too early: after {saved_ok} messages already"
-                        )
-                    lp.indent("good, message sending failed because quota was exceeded")
-                    return
-            if (
-                "stored mail into mailbox 'inbox'" in line
-                or "saved mail to inbox" in line
-            ):
-                saved_ok += 1
-                print(f"{saved_ok}: {line}")
-                if saved_ok >= num_to_send:
-                    break
-
-        pytest.fail("sending succeeded although messages should exceed quota")
 
     def test_securejoin(self, cmfactory, lp, maildomain2):
         ac1 = cmfactory.new_online_configuring_account(cache=False)
@@ -172,7 +153,7 @@ def test_hide_senders_ip_address(cmfactory):
     assert ipaddress.ip_address(public_ip)
 
     user1, user2 = cmfactory.get_online_accounts(2)
-    chat = cmfactory.get_accepted_chat(user1, user2)
+    chat = cmfactory.get_protected_chat(user1, user2)
 
     chat.send_text("testing submission header cleanup")
     user2._evtracker.wait_next_incoming_message()
@@ -181,11 +162,18 @@ def test_hide_senders_ip_address(cmfactory):
     assert public_ip not in msg.obj.as_string()
 
 
-def test_echobot(cmfactory, chatmail_config, lp):
+def test_echobot(cmfactory, chatmail_config, lp, sshdomain):
     ac = cmfactory.get_online_accounts(1)[0]
 
-    lp.sec(f"Send message to echo@{chatmail_config.mail_domain}")
+    # establish contact with echobot
-    chat = ac.create_chat(f"echo@{chatmail_config.mail_domain}")
+    sshexec = SSHExec(sshdomain)
+    command = "cat /var/lib/echobot/invite-link.txt"
+    echo_invite_link = sshexec(call=rshell.shell, kwargs=dict(command=command))
+    chat = ac.qr_setup_contact(echo_invite_link)
+    ac._evtracker.wait_securejoin_joiner_progress(1000)
+
+    # send message and check it gets replied back
+    lp.sec("Send message to echobot")
     text = "hi, I hope you text me back"
     chat.send_text(text)
     lp.sec("Wait for reply from echobot")

cmdeploy/src/cmdeploy/tests/plugin.py

@@ -62,7 +62,7 @@ def sshdomain(maildomain):
 def maildomain2():
     domain = os.environ.get("CHATMAIL_DOMAIN2")
     if not domain:
-        pytest.skip("set CHATMAIL_DOMAIN2 to a ssh-reachable chatmail instance")
+        pytest.skip("set CHATMAIL_DOMAIN2 to a second chatmail server")
     return domain
 
 
@@ -302,10 +302,13 @@ def cmfactory(request, gencreds, tmpdir, maildomain):
     pytest.importorskip("deltachat")
     from deltachat.testplugin import ACFactory
 
-    data = request.getfixturevalue("data")
-
     testproc = ChatmailTestProcess(request.config, maildomain, gencreds)
-    am = ACFactory(request=request, tmpdir=tmpdir, testprocess=testproc, data=data)
+
+    class Data:
+        def read_path(self, path):
+            return
+
+    am = ACFactory(request=request, tmpdir=tmpdir, testprocess=testproc, data=Data())
 
     # nb. a bit hacky
     # would probably be better if deltachat's test machinery grows native support

cmdeploy/src/cmdeploy/tests/test_cmdeploy.py

@@ -27,3 +27,6 @@ class TestCmdline:
         assert main(["init", "chat.example.org"]) == 1
         out, err = capsys.readouterr()
         assert "path exists" in out.lower()
+        assert main(["init", "chat.example.org", "--force"]) == 0
+        out, err = capsys.readouterr()
+        assert "deleting config file" in out.lower()

cmdeploy/src/cmdeploy/www.py

@@ -25,7 +25,8 @@ def prepare_template(source):
     assert source.exists(), source
     render_vars = {}
     render_vars["pagename"] = "home" if source.stem == "index" else source.stem
-    render_vars["markdown_html"] = markdown.markdown(source.read_text())
+    # tabs usage for multiple languages https://facelessuser.github.io/pymdown-extensions/extensions/blocks/plugins/tab/
+    render_vars["markdown_html"] = markdown.markdown(source.read_text(), extensions=['pymdownx.blocks.tab'])
     page_layout = source.with_name("page-layout.html").read_text()
     return render_vars, page_layout
 

docker/chatmail_server.dockerfile

@@ -0,0 +1,102 @@
+FROM jrei/systemd-debian:12 AS base
+
+ENV LANG=en_US.UTF-8
+
+RUN echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/01norecommend && \
+    echo 'APT::Install-Suggests "0";' >> /etc/apt/apt.conf.d/01norecommend && \
+    apt-get update && \
+    apt-get install -y \
+        ca-certificates && \
+    DEBIAN_FRONTEND=noninteractive \
+    TZ=Europe/London \
+    apt-get install -y tzdata && \
+    apt-get install -y locales && \
+    sed -i -e "s/# $LANG.*/$LANG UTF-8/" /etc/locale.gen && \
+    dpkg-reconfigure --frontend=noninteractive locales && \
+    update-locale LANG=$LANG \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN apt-get update && \
+    apt-get install -y \
+        openssh-client \
+        openssh-server \
+        git \
+        python3 \
+        python3-venv \
+        python3-virtualenv \
+        gcc \
+        python3-dev \
+        opendkim \
+        opendkim-tools \
+        curl \
+        rsync \
+        unbound \
+        unbound-anchor \
+        dnsutils \
+        postfix \
+        acl \
+        nginx \
+        libnginx-mod-stream \
+        fcgiwrap \
+        cron \
+    && for pkg in core imapd lmtpd; do \
+      case "$pkg" in \
+        core) sha256="43f593332e22ac7701c62d58b575d2ca409e0f64857a2803be886c22860f5587" ;; \
+        imapd) sha256="8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86" ;; \
+        lmtpd) sha256="2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab" ;; \
+      esac; \
+      url="https://download.delta.chat/dovecot/dovecot-${pkg}_2.3.21%2Bdfsg1-3_amd64.deb"; \
+      file="/tmp/$(basename "$url")"; \
+      curl -fsSL "$url" -o "$file"; \
+      echo "$sha256  $file" | sha256sum -c -; \
+      apt-get install -y "$file"; \
+      rm -f "$file"; \
+    done \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN systemctl enable \
+        ssh \
+        fcgiwrap
+
+RUN sed -i 's/^#PasswordAuthentication .*/PasswordAuthentication no/' /etc/ssh/sshd_config && \
+    sed -i 's/^#PermitRootLogin .*/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config && \
+    ssh-keygen -P "" -t rsa -b 2048 -f /root/.ssh/id_rsa && \
+    mkdir -p /root/.ssh && \
+    cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys && \
+    SSH_USER_CONFIG="/root/.ssh/config" && \
+    echo "Host localhost" >  "$SSH_USER_CONFIG" && \
+    echo "    HostName localhost" >>  "$SSH_USER_CONFIG" && \
+    echo "    User root" >>  "$SSH_USER_CONFIG" && \
+    echo "    StrictHostKeyChecking no" >>  "$SSH_USER_CONFIG" && \
+    echo "    UserKnownHostsFile /dev/null" >>  "$SSH_USER_CONFIG"
+    ## TODO: deny access for all insteed root form 127.0.0.1 https://unix.stackexchange.com/a/406264
+
+WORKDIR /opt/chatmail
+
+ARG SETUP_CHATMAIL_SERVICE_PATH=/lib/systemd/system/setup_chatmail.service
+COPY ./files/setup_chatmail.service "$SETUP_CHATMAIL_SERVICE_PATH"
+RUN ln -sf "$SETUP_CHATMAIL_SERVICE_PATH" "/etc/systemd/system/multi-user.target.wants/setup_chatmail.service"
+
+COPY --chmod=555 ./files/setup_chatmail_docker.sh /setup_chatmail_docker.sh
+COPY --chmod=555 ./files/update_ini.sh /update_ini.sh
+COPY --chmod=555 ./files/entrypoint.sh /entrypoint.sh
+
+## TODO: add git clone.
+## Problem: how correct save only required files inside container....
+# RUN git clone https://github.com/chatmail/relay.git -b master . \
+#  && ./scripts/initenv.sh
+
+# EXPOSE 443 25 587 143 993 
+
+VOLUME ["/sys/fs/cgroup", "/home"]
+
+STOPSIGNAL SIGRTMIN+3
+
+ENTRYPOINT ["/entrypoint.sh"]
+
+CMD [   "--default-standard-output=journal+console", \
+        "--default-standard-error=journal+console" ]
+
+## TODO: Add installation and configuration of chatmaild inside the Dockerfile.
+## This is required to ensure repeatable deployment.
+## In the current MVP, the chatmaild server is updated on every container restart.

docker/docker-compose-default.yaml

@@ -0,0 +1,58 @@
+services:
+  chatmail:
+    build: 
+      context: ./docker
+      dockerfile: chatmail_server.dockerfile
+      tags:
+        - chatmail-relay:latest
+    image: chatmail-relay:latest
+    restart: unless-stopped
+    container_name: chatmail 
+    cgroup: host # required for systemd
+    tty: true # required for logs
+    tmpfs: # required for systemd
+      - /tmp
+      - /run
+      - /run/lock
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "3"
+    environment:
+      MAIL_DOMAIN: <your_domain>
+      CHANGE_KERNEL_SETTINGS: "False"
+      ACME_EMAIL: <your_email>
+
+      MAX_MESSAGE_SIZE: "50M"
+      # DEBUG_COMMANDS_ENABLED: "true"
+      # FORCE_REINIT_INI_FILE: "true"
+      # USE_FOREIGN_CERT_MANAGER: "True"
+      # ENABLE_CERTS_MONITORING: "true"
+      # CERTS_MONITORING_TIMEOUT: 10
+      # IS_DEVELOPMENT_INSTANCE: "True"
+    ports:
+      - "25:25"
+      - "587:587"
+      - "143:143"
+      - "993:993"
+      - "443:443"
+    volumes:
+      ## system
+      - /sys/fs/cgroup:/sys/fs/cgroup:rw # required for systemd
+      - ./:/opt/chatmail
+      - ./data/acme:/var/lib/acme
+      
+      ## data
+      - ./data/chatmail:/home
+      - ./data/chatmail-dkimkeys:/etc/dkimkeys
+      - ./data/chatmail-echobot:/run/echobot
+      - ./data/chatmail-acme:/var/lib/acme
+
+      ## custom resources
+      # - ./custom/www/src/index.md:/opt/chatmail/www/src/index.md
+
+      ## debug
+      # - ./docker/files/setup_chatmail_docker.sh:/setup_chatmail_docker.sh
+      # - ./docker/files/entrypoint.sh:/entrypoint.sh
+      # - ./docker/files/update_ini.sh:/update_ini.sh

docker/docker-compose-traefik.yaml

@@ -0,0 +1,115 @@
+services:
+  chatmail:
+    build: 
+      context: ./docker
+      dockerfile: chatmail_server.dockerfile
+      tags:
+        - chatmail-relay:latest
+    image: chatmail-relay:latest
+    restart: unless-stopped
+    container_name: chatmail
+    depends_on:
+      - traefik-certs-dumper
+    cgroup: host # required for systemd
+    tty: true # required for logs
+    tmpfs: # required for systemd
+      - /tmp
+      - /run
+      - /run/lock
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "3"
+    environment: #all possible variables you can check inside README and /chatmaild/src/chatmaild/ini/chatmail.ini.f
+      MAIL_DOMAIN: $MAIL_DOMAIN
+      # MAX_MESSAGE_SIZE: "50M"
+      # DEBUG_COMMANDS_ENABLED: "true"
+      # FORCE_REINIT_INI_FILE: "true"
+      USE_FOREIGN_CERT_MANAGER: "true"
+      CHANGE_KERNEL_SETTINGS: "false"
+      PATH_TO_SSL_CONTAINER: $PATH_TO_SSL_CONTAINER
+      ENABLE_CERTS_MONITORING: "true"
+      # CERTS_MONITORING_TIMEOUT: 60
+      # IS_DEVELOPMENT_INSTANCE: "true"
+    ports:
+      - "25:25"
+      - "587:587"
+      - "143:143"
+      - "993:993"
+    volumes:
+      ## system
+      - /sys/fs/cgroup:/sys/fs/cgroup:rw # required for systemd
+      - ./:/opt/chatmail
+      - ${PATH_TO_SSL_HOST}:${PATH_TO_SSL_CONTAINER}:ro
+      
+      ## data
+      - ./data/chatmail:/home
+      # - ./data/chatmail-dkimkeys:/etc/dkimkeys
+      # - ./data/chatmail-echobot:/run/echobot
+      # - ./data/chatmail-acme:/var/lib/acme
+
+      ## custom resources
+      # - ./custom/www/src/index.md:/opt/chatmail/www/src/index.md
+
+      ## debug
+      # - ./docker/files/setup_chatmail_docker.sh:/setup_chatmail_docker.sh
+      # - ./docker/files/entrypoint.sh:/entrypoint.sh
+      # - ./docker/files/update_ini.sh:/update_ini.sh
+
+    labels:
+      - traefik.enable=true
+      - traefik.http.services.chatmail-relay.loadbalancer.server.scheme=https
+      - traefik.http.services.chatmail-relay.loadbalancer.server.port=443
+      - traefik.http.services.chatmail-relay.loadbalancer.serverstransport=insecure@file
+      - traefik.http.routers.chatmail-relay.rule=Host(`${MAIL_DOMAIN}`) || Host(`mta-sts.${MAIL_DOMAIN}`) || Host(`www.${MAIL_DOMAIN}`)
+      - traefik.http.routers.chatmail-relay.service=chatmail-relay
+      - traefik.http.routers.chatmail-relay.tls=true
+      - traefik.http.routers.chatmail-relay.tls.certresolver=letsEncrypt
+
+  traefik:
+    image: traefik:v3.3
+    container_name: traefik
+    restart: unless-stopped
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "3"
+    command:
+      - --configFile=/config.yaml
+    # ports:
+    #   - "80:80"
+    #   - "443:443"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./data/traefik/config.yaml:/config.yaml
+      - ./data/traefik/acme.json:/acme.json
+      - ./data/traefik/dynamic-configs:/dynamic/conf
+
+    network_mode: host
+    
+  traefik-certs-dumper:
+    image: ldez/traefik-certs-dumper:v2.10.0
+    restart: unless-stopped
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "3"
+    depends_on:
+      - traefik
+    entrypoint: sh -c '
+      apk add openssl &&
+      while ! [ -e /data/acme.json ]
+      || ! [ `jq ".[] | .Certificates | length" /data/acme.json | jq -s "add" ` != 0 ]; do
+      sleep 1
+      ; done
+      && traefik-certs-dumper file --version v3 --watch --domain-subdir=true
+      --source /data/acme.json --dest /data/letsencrypt/certs --post-hook "sh /post-hook.sh"'
+    environment:
+      CERTS_DIR: /data/letsencrypt/certs
+    volumes:
+      - ./data/traefik/letsencrypt:/data/letsencrypt
+      - ./data/traefik/acme.json:/data/acme.json
+      - ./data/traefik/post-hook.sh:/post-hook.sh

docker/example.env

@@ -0,0 +1,4 @@
+MAIL_DOMAIN="chat.example.com"
+
+PATH_TO_SSL_HOST="/opt/traefik/data/letsencrypt/certs/${MAIL_DOMAIN}"
+PATH_TO_SSL_CONTAINER="/var/lib/acme/live/${MAIL_DOMAIN}"

docker/files/entrypoint.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -eo pipefail
+
+if [ "${USE_FOREIGN_CERT_MANAGER,,}" == "true" ]; then
+    if [ ! -f "$PATH_TO_SSL_CONTAINER/fullchain" ]; then
+        echo "Error: file '$PATH_TO_SSL_CONTAINER/fullchain' does not exist. Exiting..." > /dev/stderr
+        exit 1
+    fi
+    if [ ! -f "$PATH_TO_SSL_CONTAINER/privkey" ]; then
+        echo "Error: file '$PATH_TO_SSL_CONTAINER/privkey' does not exist. Exiting..." > /dev/stderr
+        exit 1
+    fi
+fi
+
+SETUP_CHATMAIL_SERVICE_PATH="${SETUP_CHATMAIL_SERVICE_PATH:-/lib/systemd/system/setup_chatmail.service}"
+
+env_vars=$(printenv | cut -d= -f1 | xargs)
+sed -i "s|<envs_list>|$env_vars|g" $SETUP_CHATMAIL_SERVICE_PATH
+
+exec /lib/systemd/systemd $@

docker/files/setup_chatmail.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Run container setup commands
+After=multi-user.target
+ConditionPathExists=/setup_chatmail_docker.sh
+
+[Service]
+Type=oneshot
+ExecStart=/bin/bash /setup_chatmail_docker.sh
+RemainAfterExit=true
+WorkingDirectory=/opt/chatmail
+PassEnvironment=<envs_list>
+
+[Install]
+WantedBy=multi-user.target

docker/files/setup_chatmail_docker.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+
+set -eo pipefail
+export INI_FILE="${INI_FILE:-chatmail.ini}"
+export ENABLE_CERTS_MONITORING="${ENABLE_CERTS_MONITORING:-true}"
+export CERTS_MONITORING_TIMEOUT="${CERTS_MONITORING_TIMEOUT:-60}"
+export PATH_TO_SSL_CONTAINER="${PATH_TO_SSL_CONTAINER:-/var/lib/acme/live/${MAIL_DOMAIN}}"
+export CHANGE_KERNEL_SETTINGS=${CHANGE_KERNEL_SETTINGS:-"False"}
+
+if [ -z "$MAIL_DOMAIN" ]; then
+    echo "ERROR: Environment variable 'MAIL_DOMAIN' must be set!" >&2
+    exit 1
+fi
+
+debug_commands() {
+    echo "Executing debug commands"
+    # git config --global --add safe.directory /opt/chatmail
+    # ./scripts/initenv.sh
+}
+
+calculate_hash() {
+    find "$PATH_TO_SSL_CONTAINER" -type f -exec sha1sum {} \; | sort | sha1sum | awk '{print $1}'
+}
+
+monitor_certificates() {
+    if [ "$ENABLE_CERTS_MONITORING" != "true" ]; then
+        echo "Certs monitoring disabled."
+        exit 0
+    fi
+
+    current_hash=$(calculate_hash)
+    previous_hash=$current_hash
+
+    while true; do
+        current_hash=$(calculate_hash)
+        if [[ "$current_hash" != "$previous_hash" ]]; then
+        # TODO: add an option to restart at a specific time interval 
+            echo "[INFO] Certificate's folder hash was changed, restarting nginx, dovecot and postfix services."
+            systemctl restart nginx.service
+            systemctl reload dovecot.service
+            systemctl reload postfix.service
+            previous_hash=$current_hash
+        fi
+        sleep $CERTS_MONITORING_TIMEOUT
+    done
+}
+
+### MAIN
+
+if [ "$DEBUG_COMMANDS_ENABLED" == "true" ]; then
+    debug_commands
+fi
+
+if [ "$FORCE_REINIT_INI_FILE" == "true" ]; then 
+    INI_CMD_ARGS=--force
+fi
+
+/usr/sbin/opendkim-genkey -D /etc/dkimkeys -d $MAIL_DOMAIN -s opendkim
+chown opendkim:opendkim /etc/dkimkeys/opendkim.private
+chown opendkim:opendkim /etc/dkimkeys/opendkim.txt
+
+# TODO: Move to debug_commands after git clone is moved to dockerfile. 
+git config --global --add safe.directory /opt/chatmail
+./scripts/initenv.sh
+
+./scripts/cmdeploy init --config "${INI_FILE}" $INI_CMD_ARGS $MAIL_DOMAIN
+bash /update_ini.sh
+
+./scripts/cmdeploy run --ssh-host localhost --skip-dns-check
+
+echo "ForwardToConsole=yes" >> /etc/systemd/journald.conf
+systemctl restart systemd-journald
+
+monitor_certificates &

docker/files/update_ini.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+set -eo pipefail
+
+INI_FILE="${INI_FILE:-chatmail.ini}"
+
+if [ ! -f "$INI_FILE" ]; then
+    echo "Error: file $INI_FILE not found." >&2
+    exit 1
+fi
+
+TMP_FILE="$(mktemp)"
+
+convert_to_bytes() {
+    local value="$1"
+    if [[ "$value" =~ ^([0-9]+)([KkMmGgTt])$ ]]; then
+        local num="${BASH_REMATCH[1]}"
+        local unit="${BASH_REMATCH[2]}"
+        case "$unit" in
+            [Kk]) echo $((num * 1024)) ;;
+            [Mm]) echo $((num * 1024 * 1024)) ;;
+            [Gg]) echo $((num * 1024 * 1024 * 1024)) ;;
+            [Tt]) echo $((num * 1024 * 1024 * 1024 * 1024)) ;;
+        esac
+    elif [[ "$value" =~ ^[0-9]+$ ]]; then
+        echo "$value"
+    else
+        echo "Error: incorrect size format: $value." >&2
+        return 1
+    fi
+}
+
+process_specific_params() {
+    local key=$1
+    local value=$2
+    local destination_file=$3
+
+    if [[ "$key" == "max_message_size" ]]; then
+        converted=$(convert_to_bytes "$value") || exit 1
+        if grep -q -e "## .* = .* bytes" "$destination_file"; then 
+            sed "s|## .* = .* bytes|## $value = $converted bytes|g" "$destination_file";
+        else
+            echo "## $value = $converted bytes"  >> "$destination_file"
+        fi
+        echo "$key = $converted" >> "$destination_file"
+    else
+        echo "$key = $value" >> "$destination_file"
+    fi
+}
+
+while IFS= read -r line; do
+    if [[ "$line" =~ ^[[:space:]]*#.* || "$line" =~ ^[[:space:]]*$ ]]; then
+        echo "$line" >> "$TMP_FILE"
+        continue
+    fi
+
+    if [[ "$line" =~ ^([a-z0-9_]+)[[:space:]]*=[[:space:]]*(.*)$ ]]; then
+        key="${BASH_REMATCH[1]}"
+        current_value="${BASH_REMATCH[2]}"
+        env_var_name=$(echo "$key" | tr 'a-z' 'A-Z')
+        env_value="${!env_var_name}"
+
+        if [[ -n "$env_value" ]]; then
+            process_specific_params "$key" "$env_value" "$TMP_FILE"
+        else
+            echo "$line" >> "$TMP_FILE"
+        fi
+    else
+        echo "$line" >> "$TMP_FILE"
+    fi
+done < "$INI_FILE"
+
+PERMS=$(stat -c %a "$INI_FILE")
+OWNER=$(stat -c %u "$INI_FILE")
+GROUP=$(stat -c %g "$INI_FILE")
+
+chmod "$PERMS" "$TMP_FILE"
+chown "$OWNER":"$GROUP" "$TMP_FILE"
+
+mv "$TMP_FILE" "$INI_FILE"

docs/DOCKER_INSTALLATION_EN.md

@@ -0,0 +1,310 @@
+# Known issues and limitations
+
+- Installation using acmetool (`docker-compose-default.yaml`) may NOT work. In this case, use installation via traefik (`docker-compose-traefik.yaml`). Personally, during my tests, I encountered the error `could not install DNS challenge, no hooks succeeded;`, which I was unable to fix.
+- Chatmail will be reinstalled every time the container is started (longer the first time, faster on subsequent starts). This is how the original installer works because it wasn’t designed for Docker. At the end of the documentation, there’s a [proposed solution](#locking-the-chatmail-version).
+- Requires cgroups v2 configured in the system. Operation with cgroups v1 has not been tested.
+- Yes, of course, using systemd inside a container is a hack, and it would be better to split it into several services, but since this is an MVP, it turned out to be easier to do it this way initially than to rewrite the entire deployment system.
+- The Docker image is only suitable for amd64. If you need to run it on a different architecture, try modifying the Dockerfile (specifically the part responsible for installing dovecot).
+
+# Docker installation
+This section provides instructions for installing Chatmail using docker-compose.
+
+## Preliminary setup
+We use `chat.example.org` as the Chatmail domain in the following steps.
+Please substitute it with your own domain.
+
+1. Setup the initial DNS records.
+   The following is an example in the familiar BIND zone file format with
+   a TTL of 1 hour (3600 seconds).
+   Please substitute your domain and IP addresses.
+
+   ```
+    chat.example.com. 3600 IN A 198.51.100.5
+    chat.example.com. 3600 IN AAAA 2001:db8::5
+    www.chat.example.com. 3600 IN CNAME chat.example.com.
+    mta-sts.chat.example.com. 3600 IN CNAME chat.example.com.
+   ```
+
+2. clone the repository on your server.
+
+   ```shell
+    git clone https://github.com/chatmail/relay
+    cd relay
+   ```
+
+## Installation
+When installing via Docker, there are several options:
+
+- Use the built-in nginx and acmetool in Chatmail container to host the chat and manage certificates.
+- Use third-party tools for certificate management.
+
+For the third-party certificate manager example, traefik will be used, but you can use whatever is more convenient for you.
+
+1. Copy the file `./docker/docker-compose-default.yaml` or `./docker/docker-compose-traefik.yaml` and rename it to `docker-compose.yaml`. This is necessary because `docker-compose.yaml` is in `.gitignore` and won’t cause conflicts when updating the git repository.
+
+```shell
+cp ./docker/docker-compose-default.yaml docker-compose.yaml
+## or
+# cp ./docker/docker-compose-traefik.yaml docker-compose.yaml
+```
+
+2. Copy `./docker/example.env` and rename it to `.env`. This file stores variables used in `docker-compose.yaml`. Required only when installing with traefik; if using the default setup, you can skip this step.
+
+```shell
+cp ./docker/example.env .env
+```
+
+3. Configure kernel parameters because they cannot be changed inside the container, specifically `fs.inotify.max_user_instances` and `fs.inotify.max_user_watches`. Run the following:
+
+```shell
+echo "fs.inotify.max_user_instances=65536" | sudo tee -a /etc/sysctl.d/99-inotify.conf
+echo "fs.inotify.max_user_watches=65536" | sudo tee -a /etc/sysctl.d/99-inotify.conf
+sudo sysctl --system
+```
+
+4. Configure container environment variables. Below is the list of variables used during deployment:
+
+- `MAIL_DOMAIN` – The domain name of the future server. (required)
+- `DEBUG_COMMANDS_ENABLED` – Run debug commands before installation. (default: `false`)
+- `FORCE_REINIT_INI_FILE` – Recreate the ini configuration file on startup. (default: `false`)
+- `USE_FOREIGN_CERT_MANAGER` – Use a third-party certificate manager. (default: `false`)
+- `INI_FILE` – Path to the ini configuration file. (default: `./chatmail.ini`)
+- `PATH_TO_SSL_CONTAINER` – Path to where the certificates are stored. (default: `/var/lib/acme/live/${MAIL_DOMAIN}`)
+- `ENABLE_CERTS_MONITORING` – Enable certificate monitoring if `USE_FOREIGN_CERT_MANAGER=true`. If certificates change, services will be automatically restarted. (default: `false`)
+- `CERTS_MONITORING_TIMEOUT` – Interval in seconds to check if certificates have changed. (default: `'60'`)
+
+You can also use any variables from the [ini configuration file](https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/ini/chatmail.ini.f); they must be in uppercase.
+
+Mandatory variables for deployment via Docker:
+
+- `CHANGE_KERNEL_SETTINGS` – Change kernel settings (`fs.inotify.max_user_instances` and `fs.inotify.max_user_watches`) on startup. Changing kernel settings inside the container is not possible! (default: `False`)
+
+5. Configure environment variables in the `.env` file. These variables are used in the `docker-compose.yaml` file to pass repeated values.
+
+6. Build the Docker image:
+
+```shell
+docker compose build chatmail
+```
+
+<details>
+<summary>Additional steps for configuring with traefik</summary>
+
+> [!note]
+> If you are using the default installation without traefik – skip these steps and go to step 7 (running docker compose).
+
+Before starting traefik, configuration files must be prepared; otherwise, it will not start correctly.
+
+First, run these commands in the console, replacing their values with the correct ones:
+
+```shell
+export YOUR_EMAIL=your_email@gmail.com
+mkdir -p "./data/traefik"
+cd "./data/traefik"
+```
+
+1. Create a traefik configuration file:
+
+```shell
+cat > config.yaml << EOF
+log:
+  level: TRACE
+
+entryPoints:
+  web:
+    address: ":80"
+    http:
+      redirections:
+        entryPoint:
+          to: websecure
+          permanent: true
+  websecure:
+    address: ":443"
+
+providers:
+  docker:
+    endpoint: "unix:///var/run/docker.sock"
+    exposedByDefault: false
+  file:
+    directory: /dynamic/conf
+    watch: true
+
+serverstransport:
+  insecureskipverify: true
+
+certificatesResolvers:
+  letsEncrypt:
+    acme:
+      email: $YOUR_EMAIL
+      storage: /acme.json
+      caServer: "https://acme-v02.api.letsencrypt.org/directory"
+      tlschallenge: true
+      httpChallenge:
+        entryPoint: web
+EOF
+```
+
+2. Create a post-hook script:
+
+```shell
+cat > post-hook.sh << 'EOF'
+CERTS_DIR=${CERTS_DIR:-"/data/letsencrypt/certs"}
+
+for dir in "$CERTS_DIR"/*/; do
+    cd "$dir"
+    if [ -f "certificate.crt" ]; then
+        ln -sf certificate.crt fullchain
+    fi
+    if [ -f "privatekey.key" ]; then
+        ln -sf privatekey.key privkey
+    fi
+    cd -
+done
+EOF
+```
+
+3. Create the `acme.json` file:
+
+```shell
+touch acme.json
+sudo chown 0:0 ./acme.json # required
+sudo chmod 600 ./acme.json # required
+```
+
+4. Create insecure config:
+
+```shell
+mkdir dynamic-configs
+cat > ./dynamic-configs/insecure.yaml << 'EOF'
+http:
+  serversTransports:
+    insecure:
+      insecureSkipVerify: true
+EOF
+cd ../..
+```
+
+</details>
+
+7. Start docker compose and wait for the installation to finish:
+
+```shell
+docker compose up -d # start service
+docker compose logs -f chatmail # view container logs, press CTRL+C to exit
+```
+
+8. After installation is complete, you can open `https://<your_domain_name>` in your browser.
+
+## Using custom files
+
+When using Docker, you can apply modified configuration files to make the installation more personalized. This is usually needed for the `www/src` section so that the Chatmail landing page is customized to your taste, but it can be used for any other cases as well.
+
+To replace files correctly:
+
+1. Create the `./custom` directory. It is in `.gitignore`, so it won’t cause conflicts when updating.
+
+```shell
+mkdir -p ./custom
+```
+
+2. Modify the required file. For example, `index.md`:
+
+```shell
+mkdir -p ./custom/www/src
+nano ./custom/www/src/index.md
+```
+
+3. In `docker-compose.yaml`, add the file mount in the `volumes` section:
+
+```yaml
+services:
+  chatmail:
+    volumes:
+      ...
+      ## custom resources
+      - ./custom/www/src/index.md:/opt/chatmail/www/src/index.md
+```
+
+4. Restart the service:
+
+```shell
+docker compose down
+docker compose up -d
+```
+
+## Locking the Chatmail version
+
+> [!note]
+> These steps are optional and should only be done if you are not satisfied that the service is installed each time the container starts.
+
+Since the current Docker version installs the Chatmail service every time the container starts, you can lock the container version after installation as follows:
+
+1. Commit the current state of the configured container:
+
+```shell
+docker container commit chatmail configured-chatmail:$(date +'%Y-%m-%d')
+docker image ls | grep configured-chatmail
+```
+
+2. Change the entrypoint for the container in `docker-compose.yaml` to:
+
+```yaml
+services:
+  chatmail:
+    image: <image name from step 1>
+    volumes:
+      ...
+      ## custom resources
+      - ./custom/setup_chatmail_docker.sh:/setup_chatmail_docker.sh
+```
+
+3. Create the file `./custom/setup_chatmail_docker.sh` with the new configuration:
+
+```shell
+mkdir -p ./custom
+cat > ./custom/setup_chatmail_docker.sh << 'EOF'
+#!/bin/bash
+
+set -eo pipefail
+
+export ENABLE_CERTS_MONITORING="${ENABLE_CERTS_MONITORING:-true}"
+export CERTS_MONITORING_TIMEOUT="${CERTS_MONITORING_TIMEOUT:-60}"
+export PATH_TO_SSL_CONTAINER="${PATH_TO_SSL_CONTAINER:-/var/lib/acme/live/${MAIL_DOMAIN}}"
+
+calculate_hash() {
+    find "$PATH_TO_SSL_CONTAINER" -type f -exec sha1sum {} \; | sort | sha1sum | awk '{print $1}'
+}
+
+monitor_certificates() {
+    if [ "$ENABLE_CERTS_MONITORING" != "true" ]; then
+        echo "Certs monitoring disabled."
+        exit 0
+    fi
+
+    current_hash=$(calculate_hash)
+    previous_hash=$current_hash
+
+    while true; do
+        current_hash=$(calculate_hash)
+        if [[ "$current_hash" != "$previous_hash" ]]; then
+        # TODO: add an option to restart at a specific time interval 
+            echo "[INFO] Certificate's folder hash was changed, restarting nginx, dovecot and postfix services."
+            systemctl restart nginx.service
+            systemctl reload dovecot.service
+            systemctl reload postfix.service
+            previous_hash=$current_hash
+        fi
+        sleep $CERTS_MONITORING_TIMEOUT
+    done
+}
+
+monitor_certificates &
+EOF
+```
+
+4. Restart the service:
+
+```shell
+docker compose down
+docker compose up -d
+```

docs/DOCKER_INSTALLATION_RU.md

@@ -0,0 +1,284 @@
+# Известные проблемы и ограничения
+- Установка с помощью acmetool (`docker-compose-default.yaml`) может НЕ работать. В таком случае используйте установку через traefik (`docker-compose-traefik.yaml`). Лично у меня при тестах ошибка была `could not install DNS challenge, no hooks succeeded;`, которую исправить не удалось.
+- Chatmail будет переустановлен при каждом запуске контейнера (при первом - долго, при последующих быстрее). Так устроен изначальный установщик, потому что он не был заточен под docker. В конце документации [представлено](#фиксирование-версии-chatmail) возможное решение
+- Требуется настроенный в системе cgroups v2. Работа с cgroups v1 не тестировалась.
+- Да, понятно дело что systemd использовать в контейнере костыль и надо это всё разнести на несколько сервисов, но это MVP и в первом приближении оказалось сделать проще так, чем переписывать всю систему развертывания.
+- docker образ подходит только для amd64, если нужно запустить на другой архитектуре, попробуйте изменить dockerfile (конкретно ту часть что ответсвенна за установку dovecot)
+
+# Docker installation
+Здесь представлена инструкция по установке chatmail с помощью docker-compose.
+
+## Предварительная настройка
+We use `chat.example.org` as the chatmail domain in the following steps.
+Please substitute it with your own domain. 
+
+1. Настройте начальные записи DNS.Ниже приведен пример в привычном формате файла зоны BIND сTTL 1 час (3600 секунд).
+   Замените домен и IP-адреса на свои.
+
+   ```
+    chat.example.com. 3600 IN A 198.51.100.5
+    chat.example.com. 3600 IN AAAA 2001:db8::5
+    www.chat.example.com. 3600 IN CNAME chat.example.com.
+    mta-sts.chat.example.com. 3600 IN CNAME chat.example.com.
+   ```
+
+2. Склонируйте репозиторий на свой сервер.
+
+   ```shell
+    git clone https://github.com/chatmail/relay
+    cd relay
+   ```
+
+## Installation
+При установке через docker есть несколько вариантов:
+- использовать встроенный в chatmail контейнер nginx и acmetool для хостинга чата и управления сертификатами.
+- использовать сторонние инструменты для менеджмента сертификатов
+
+В качестве примера для стороннего менеджера сертификатов будет использоваться traefik, но вы можете использовать то что удобнее вам.
+
+1. Скопировать файл `./docker/docker-compose-default.yaml` или `./docker/docker-compose-traefik.yaml` и переименовать в `docker-compose.yaml`. Это нужно потому что `docker-compose.yaml` находится в `.gitignore` и не будет создавать конфликты при обновлении гит репозитория.
+```shell
+cp ./docker/docker-compose-default.yaml docker-compose.yaml
+## or
+# cp ./docker/docker-compose-traefik.yaml docker-compose.yaml
+```
+
+2. Скопировать `./docker/example.env` и переименовать в `.env`. Здесь хранятся переменные, которые используятся в `docker-compose.yaml`. Нужен только для установки совместно с traefik, если используется default - можно пропустить
+```shell
+cp ./docker/example.env .env
+```
+
+3. Настроить параметры ядра, потому что внутри контейнера их нельзя изменить, а конкретно `fs.inotify.max_user_instances` и `fs.inotify.max_user_watches`. Для этого выполнить следующее:
+```shell
+echo "fs.inotify.max_user_instances=65536" | sudo tee -a /etc/sysctl.d/99-inotify.conf
+echo "fs.inotify.max_user_watches=65536" | sudo tee -a /etc/sysctl.d/99-inotify.conf
+sudo sysctl --system
+```
+
+4. Настроить переменные окружения контейнера. Ниже перечислен список переменных учавствующих при развертывании.
+- `MAIL_DOMAIN` - Доменное имя будущего сервера. (required)
+- `DEBUG_COMMANDS_ENABLED` - Выполнить debug команды перед установкой. (default: `false`)
+- `FORCE_REINIT_INI_FILE` - Пересоздавать ini файл конфигурации при запуске. (default: `false`)
+- `USE_FOREIGN_CERT_MANAGER` - Использовать сторонний менеджер сертификатов. (default: `false`)
+- `INI_FILE` - путь к ini файлу конфигурации. (default: `./chatmail.ini`)
+- `PATH_TO_SSL_CONTAINER` - Путь где располагаются сертификаты. (default: `/var/lib/acme/live/${MAIL_DOMAIN}`)
+- `ENABLE_CERTS_MONITORING` - Включить мониторинг сертификатов, если `USE_FOREIGN_CERT_MANAGER=true`.  Если сертфикаты изменятся сервисы будут автоматически перезапущены. (default: `false`)
+- `CERTS_MONITORING_TIMEOUT` - Раз во сколько секунд проверять что изменились сертификаты. (default: `'60'`)
+
+Также могут быть использованы все переменные из [ini файла конфигурации](https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/ini/chatmail.ini.f), они обязаны быть в uppercase формате. 
+
+Ниже перечислены переменные, которые обязательны быть выставлены при развертывании через docker:
+- `CHANGE_KERNEL_SETTINGS` - Менять настройки ядра (`fs.inotify.max_user_instances` и `fs.inotify.max_user_watches`) при запуске. При запуске в контейнере смена настроек ядра не может быть выполнена! (default: `False`)
+
+5. Настроить переменные окружения в `.env` файле. Эти переменные используются в `docker-compose.yaml` файле, чтобы передавать повторяющиеся значения.
+
+6. Собрать docker образ
+```shell
+docker compose build chatmail
+```
+
+<details>
+
+<summary>Дополнительные шаги для конфигурации работы с traefik</summary>
+
+> [!note]
+> Если вы используете default установку, без использования traefik - пропустите эти шаги и переходите к шагу 7 (запуск docker compose)
+
+Перед запуском traefik необходимо подготовить файлы конфигурации, иначе он запустится некорректно.
+
+Сначала выполните эти команды в консоли, заменив значения в них на корректные.
+```shell
+export YOUR_EMAIL=your_email@gmail.com
+mkdir -p "./data/traefik"
+cd "./data/traefik"
+```
+
+1. Создать файл конфигурации traefik
+```shell
+cat > config.yaml << EOF
+log:
+  level: TRACE
+
+entryPoints:
+  web:
+    address: ":80"
+    http:
+      redirections:
+        entryPoint:
+          to: websecure
+          permanent: true
+  websecure:
+    address: ":443"
+
+providers:
+  docker:
+    endpoint: "unix:///var/run/docker.sock"
+    exposedByDefault: false
+  file:
+    directory: /dynamic/conf
+    watch: true
+
+serverstransport:
+  insecureskipverify: true
+
+certificatesResolvers:
+  letsEncrypt:
+    acme:
+      email: $YOUR_EMAIL
+      storage: /acme.json
+      caServer: "https://acme-v02.api.letsencrypt.org/directory"
+      tlschallenge: true
+      httpChallenge:
+        entryPoint: web
+EOF
+```
+
+2. Создать post-hook скрипт
+```shell
+cat > post-hook.sh << 'EOF'
+CERTS_DIR=${CERTS_DIR:-"/data/letsencrypt/certs"}
+
+for dir in "$CERTS_DIR"/*/; do
+    cd "$dir"
+    if [ -f "certificate.crt" ]; then
+        ln -sf certificate.crt fullchain
+    fi
+    if [ -f "privatekey.key" ]; then
+        ln -sf privatekey.key privkey
+    fi
+    cd -
+done
+EOF
+```
+
+3. Создать `acme.json` файл
+```shell
+touch acme.json
+sudo chown 0:0 ./acme.json # это обязательно
+sudo chmod 600 ./acme.json # это обязательно
+```
+
+4. Создать insecure config
+```shell
+mkdir dynamic-configs
+cat > ./dynamic-configs/insecure.yaml << 'EOF'
+http:
+  serversTransports:
+    insecure:
+      insecureSkipVerify: true
+EOF
+cd ../..
+```
+
+</details>
+
+7. Запустить docker compose и дождаться завершения установки
+```shell
+docker compose up -d # запуск сервиса
+docker compose logs -f chatmail # просмотр логов контейнера. Для выхода нажать CTRL+C
+```
+
+8. По окончанию установки можно открыть в браузер `https://<your_domain_name>`
+
+## Использование кастомных файлов
+При использовании docker есть возможность использовать измененые файлы конфигурации, чтобы сделать установку более персонализированной. Обычно это требуется для секции `www/src`, чтобы ознакомительная страница Chatmail была сделана на ваш вкус. Но также это можно использовать и для любых других случаев.
+
+Для того чтобы корректно выполнить подмену файлов необходимо 
+1. создать каталог `./custom`, он находится в `.gitignore`, поэтому при обновлении не вызовет конфликтов.
+```shell
+mkdir -p ./custom
+```
+
+2. Изменить нужный файл. Для примера возьмем `index.md`
+```shell
+mkdir -p ./custom/www/src
+nano ./custom/www/src/index.md
+```
+
+3. В `docker-compose.yaml` добавить монтирование файла с помощью секции `volumes`
+```yaml
+services:
+  chatmail:
+    volumes:
+      ...
+      ## custom resources
+      - ./custom/www/src/index.md:/opt/chatmail/www/src/index.md
+```
+
+4. Перезапустить сервис
+```shell
+docker compose down
+docker compose up -d
+```
+
+## Фиксирование версии Chatmail
+> [!note]
+> Это опциональные шаги, их делать требуется только если вас не устраивает что сервис устанавливается каждый раз при запуске
+
+Поскольку в текущей версии docker chatmail сервис устанавливается каждый раз запуске контейнера, чтобы этого не происходило можно зафиксировать версию контейнера после установки. Делается это следующим образом:
+
+1. Зафиксировать текущее состояние сконфигурированного контейнера
+```shell
+docker container commit chatmail configured-chatmail:$(date +'%Y-%m-%d')
+docker image ls | grep configured-chatmail
+```
+
+2. Изменить entrypoint для контейнера в `docker-compose.yaml` на
+```yaml
+services:
+  chatmail:
+    image: <image name from step 1>
+    volumes:
+      ...
+      ## custom resources
+      - ./custom/setup_chatmail_docker.sh:/setup_chatmail_docker.sh
+```
+
+3. Создать файл `./custom/setup_chatmail_docker.sh` с новым файлом конфигурации
+```shell
+mkdir -p ./custom
+cat > ./custom/setup_chatmail_docker.sh << 'EOF'
+#!/bin/bash
+
+set -eo pipefail
+
+export ENABLE_CERTS_MONITORING="${ENABLE_CERTS_MONITORING:-true}"
+export CERTS_MONITORING_TIMEOUT="${CERTS_MONITORING_TIMEOUT:-60}"
+export PATH_TO_SSL_CONTAINER="${PATH_TO_SSL_CONTAINER:-/var/lib/acme/live/${MAIL_DOMAIN}}"
+
+calculate_hash() {
+    find "$PATH_TO_SSL_CONTAINER" -type f -exec sha1sum {} \; | sort | sha1sum | awk '{print $1}'
+}
+
+monitor_certificates() {
+    if [ "$ENABLE_CERTS_MONITORING" != "true" ]; then
+        echo "Certs monitoring disabled."
+        exit 0
+    fi
+
+    current_hash=$(calculate_hash)
+    previous_hash=$current_hash
+
+    while true; do
+        current_hash=$(calculate_hash)
+        if [[ "$current_hash" != "$previous_hash" ]]; then
+        # TODO: add an option to restart at a specific time interval 
+            echo "[INFO] Certificate's folder hash was changed, restarting nginx, dovecot and postfix services."
+            systemctl restart nginx.service
+            systemctl reload dovecot.service
+            systemctl reload postfix.service
+            previous_hash=$current_hash
+        fi
+        sleep $CERTS_MONITORING_TIMEOUT
+    done
+}
+
+monitor_certificates &
+EOF
+```
+
+4. Перезапустить сервис
+```shell
+docker compose down
+docker compose up -d
+```

scripts/initenv.sh

@@ -1,5 +1,23 @@
 #!/bin/sh
 set -e
+
+if command -v lsb_release 2>&1 >/dev/null; then
+  case "$(lsb_release -is)" in
+    Ubuntu | Debian )
+      if ! dpkg -l | grep python3-dev 2>&1 >/dev/null
+      then
+        echo "You need to install python3-dev for installing the other dependencies."
+        exit 1
+      fi
+      if ! gcc --version 2>&1 >/dev/null
+      then
+        echo "You need to install gcc for building Python dependencies."
+        exit 1
+      fi
+      ;;
+  esac
+fi
+
 python3 -m venv --upgrade-deps venv
 
 venv/bin/pip install -e chatmaild 

www/src/index.md

@@ -1,7 +1,8 @@
-
 <img class="banner" src="collage-top.png"/>
 
-## Dear [Delta Chat](https://get.delta.chat) users and newcomers ... 
+/// tab | 🇬🇧 English
+
+## Dear [Delta Chat](https://get.delta.chat) users and newcomers ...
 
 {% if config.mail_domain != "nine.testrun.org" %}
 Welcome to instant, interoperable and [privacy-preserving](privacy.html) messaging :) 
@@ -23,7 +24,34 @@ you can also **scan this QR code** with Delta Chat:
 🐣 **Choose** your Avatar and Name
 
 💬 **Start** chatting with any Delta Chat contacts using [QR invite codes](https://delta.chat/en/help#howtoe2ee)
+///
+
+/// tab | 🇷🇺 Русский
+
+## Уважаемые пользователи и новички [Delta Chat](https://get.delta.chat)...
 
 {% if config.mail_domain != "nine.testrun.org" %}
+Добро пожаловать в мир мгновенного, совместимого и [конфиденциального](privacy.html) обмена сообщениями :) 
+{% else %}
+Вы находитесь на сервере по умолчанию ({{ config.mail_domain }}) 
+для пользователей Delta Chat. Подробную информацию о том, как он избегает хранения личной информации,
+см. в нашей [политике конфиденциальности](privacy.html). 
+{% endif %}
+
+<a class="cta-button" href="DCACCOUNT:https://{{ config.mail_domain }}/new">Создать чат-профиль на {{config.mail_domain}}</a>
+
+Если вы открыли эту страницу на устройстве,
+где нет приложения Delta Chat, вы можете
+**отсканировать этот QR-код** с помощью Delta Chat:
+
+<a href="DCACCOUNT:https://{{ config.mail_domain }}/new">
+    <img width=300 style="float: none;" src="qr-chatmail-invite-{{config.mail_domain}}.png" /></a>
+
+🐣 **Выберите** аватар и имя
+
+💬 **Начните** чат с любыми контактами Delta Chat через [QR-приглашения](https://delta.chat/ru/help#howtoe2ee)
+///
+
+{% if config.is_development_instance == True %}
 <div class="experimental">Note: this is only a temporary development chatmail service</div>
 {% endif %}

www/src/info.md

@@ -1,3 +1,6 @@
+<img class="banner" src="collage-info.png"/>
+
+/// tab | 🇬🇧 English
 
 ## More information 
 
@@ -6,29 +9,6 @@ interoperable e-mail service for everyone. What's behind a `chatmail` is
 effectively a normal e-mail address just like any other but optimized 
 for the usage in chats, especially DeltaChat.
 
-### Choosing a chatmail address instead of using a random one
-
-In the Delta Chat account setup you may tap `Create a profile` then `Use other server` and choose `Classic e-mail login`. Here fill the two fields like this: 
-
-- `E-Mail Address`: invent a word with
-{% if username_min_length == username_max_length %}
-  *exactly* {{ username_min_length }}
-{% else %}
-  {{ username_min_length}}
-  {% if username_max_length == "more" %}
-    or more
-  {% else %}
-    to {{ username_max_length }}
-  {% endif %}
-{% endif %}
-  characters
-  and append `@{{config.mail_domain}}` to it.
-
-- `Existing Password`: invent at least {{ password_min_length }} characters.
-
-If the e-mail address is not yet taken, you'll get that account. 
-The first login sets your password. 
-
 
 ### Rate and storage limits 
 
@@ -38,10 +18,11 @@ The first login sets your password.
 
 - You may send up to {{ config.max_user_send_per_minute }} messages per minute.
 
-- Messages are unconditionally removed {{ config.delete_mails_after }} days after arriving on the server.
-
 - You can store up to [{{ config.max_mailbox_size }} messages on the server](https://delta.chat/en/help#what-happens-if-i-turn-on-delete-old-messages-from-server).
 
+- Messages are unconditionally removed latest {{ config.delete_mails_after }} days after arriving on the server.
+  Earlier, if storage may exceed otherwise.
+
 
 ### <a name="account-deletion"></a> Account deletion 
 
@@ -63,3 +44,47 @@ This chatmail provider is run by a small voluntary group of devs and sysadmins,
 who [publically develop chatmail provider setups](https://github.com/deltachat/chatmail).
 Chatmail setups aim to be very low-maintenance, resource efficient and 
 interoperable with any other standards-compliant e-mail service. 
+///
+
+/// tab | 🇷🇺 Русский
+
+## Дополнительная информация
+
+{{ config.mail_domain }} предоставляет малозатратный, ресурсосберегающий и совместимый с другими системами почтовый сервис для всех. За `chatmail` фактически скрывается 
+обычный почтовый адрес, как и любой другой, но оптимизированный 
+для использования в чатах, особенно DeltaChat.
+
+### Ограничения по скорости и хранению
+
+* Незашифрованные сообщения блокируются для получателей вне
+  {{config.mail_domain}}, но добавление контакта через [QR-коды приглашения](https://delta.chat/en/help#howtoe2ee)
+  позволяет свободно обмениваться сообщениями между с ним.
+
+* Вы можете отправлять до {{ config.max_user_send_per_minute }} сообщений в минуту.
+
+- Вы можете хранить до [{{ config.max_mailbox_size }} сообщений на сервере](https://delta.chat/en/help#what-happens-if-i-turn-on-delete-old-messages-from-server).
+
+* Сообщения в любом случае будут удалены с сервера через {{ config.delete_mails_after }} дней после поступления на сервер.
+  Или раньше, если хранилище превышает допустимый объем.
+
+### <a name="account-deletion"></a> Удаление аккаунта
+
+Если вы удалите профиль {{ config.mail_domain }} через приложение Delta Chat,
+соответствующая учетная запись на сервере и все связанные с ней данные
+будут автоматически удалены через {{ config.delete_inactive_users_after }} дней.
+
+Если вы используете несколько устройств,
+вам необходимо удалить профиль чата на каждом из них,
+чтобы все данные аккаунта были удалены с сервера.
+
+Если у вас есть дополнительные вопросы или запросы по поводу удаления аккаунта,
+пожалуйста, отправьте сообщение со своей учетной записи на {{ config.privacy_mail }}.
+
+### Кто операторы? Какое ПО используется?
+
+Этот chatmail провайдер управляется небольшой группой добровольцев — разработчиков и системных администраторов,
+которые [публично разрабатывают инфраструктуру chatmail провайдеров](https://github.com/deltachat/chatmail).
+Chatmail стремится быть максимально простыми в обслуживании, ресурсосберегающими и
+совместимыми с любым другим почтовым сервисом, соответствующим стандартам.
+
+///

www/src/main.css

@@ -84,3 +84,57 @@ code {
     color: white !important;
     font-weight: bold;
 }
+
+.tabbed-set {
+  position: relative;
+  display: flex;
+  flex-wrap: wrap;
+  margin: 1em 0;
+  border-radius: 0.1rem;
+}
+
+.tabbed-set > input {
+  display: none;
+}
+
+.tabbed-set label {
+  width: auto;
+  padding: 0.9375em 1.25em 0.78125em;
+  font-weight: 700;
+  font-size: 0.84em;
+  white-space: nowrap;
+  border-bottom: 0.15rem solid transparent;
+  border-top-left-radius: 0.1rem;
+  border-top-right-radius: 0.1rem;
+  cursor: pointer;
+  transition: background-color 250ms, color 250ms;
+}
+
+.tabbed-set .tabbed-content {
+  width: 100%;
+  display: none;
+  box-shadow: 0 -.05rem #ddd;
+}
+
+.tabbed-set input {
+  position: absolute;
+  opacity: 0;
+}
+
+.tabbed-set input:checked:nth-child(n+1) + label {
+  color: red;
+  border-color: red;
+}
+
+@media screen {
+  .tabbed-set input:nth-child(n+1):checked + label + .tabbed-content {
+    order: 99;
+    display: block;
+  }
+}
+
+@media print {
+  .tabbed-content {
+    display: contents;
+  }
+}

www/src/privacy.md

@@ -1,3 +1,6 @@
+<img class="banner" src="collage-privacy.png"/>
+
+/// tab | 🇬🇧 English
 
 # Privacy Policy for {{ config.mail_domain }} 
 
@@ -267,5 +270,199 @@ as of *October 2024*.
 Due to the further development of our service and offers
 or due to changed legal or official requirements,
 it may become necessary to revise this data protection declaration from time to time.
+///
 
+/// tab | 🇷🇺 Русский
 
+# Политика конфиденциальности для {{ config.mail_domain }}
+
+{% if config.mail_domain == "nine.testrun.org" %}
+Добро пожаловать на `{{config.mail_domain}}` — это основной сервер Chatmail для новых пользователей Delta Chat.  
+Он поддерживается небольшой командой системных администраторов на добровольной основе.  
+Альтернативные сервера вы можете найти [здесь](https://delta.chat/en/chatmail).  
+{% endif %}
+
+## Кратко: Личные данные не запрашиваются и не собираются
+
+Этот сервер Chatmail не запрашивает и не сохраняет личную информацию.  
+Серверы Chatmail существуют исключительно для надёжной передачи (временного хранения и доставки) зашифрованных сообщений между устройствами пользователей, использующих мессенджер Delta Chat.  
+
+Технически, Chatmail-сервер можно представить как «маршрутизатор сообщений» с поддержкой сквозного шифрования в масштабе интернета.  
+
+В отличие от классических почтовых сервисов (например, Gmail),  
+Chatmail-серверы не запрашивают личные данные и не хранят письма постоянно.  
+Они ближе по устройству к серверам Signal,  
+однако не используют номера телефонов и могут безопасно и автоматически взаимодействовать как с другими Chatmail-серверами, так и с обычной электронной почтой.  
+
+Отличия от традиционных почтовых серверов:
+
+- безусловное удаление сообщений через {{ config.delete_mails_after }} дней;
+- невозможность отправки незашифрованных сообщений;
+- отсутствие хранения IP-адресов;
+- IP-адреса не обрабатываются в связке с адресами электронной почты.
+
+Из-за отсутствия обработки персональных данных  
+данный сервер, возможно, формально не обязан иметь политику конфиденциальности.
+
+Тем не менее, ниже приведена юридическая информация  
+для удобства специалистов по защите данных и юристов, изучающих работу Chatmail.
+
+---
+
+## 1. Название и контактная информация
+
+Ответственный за обработку ваших персональных данных:
+
+```
+{{ config.privacy_postal }}
+```
+
+Эл. почта: {{ config.privacy_mail }}
+
+Назначен ответственный по защите данных:
+
+```
+{{ config.privacy_pdo }}
+```
+
+---
+
+## 2. Обработка при использовании чата и электронной почты
+
+Мы предоставляем сервисы, оптимизированные для работы с приложением [Delta Chat](https://delta.chat),  
+и обрабатываем только те данные, которые необходимы для настройки и технической реализации доставки сообщений.  
+Цель обработки — дать пользователям возможность читать, писать, управлять, удалять, отправлять и получать сообщения.  
+
+Для этого мы используем серверное ПО, обеспечивающее передачу сообщений.
+
+Обрабатываются следующие данные:
+
+- Исходящие и входящие сообщения (SMTP) временно хранятся до их доставки получателю;
+- Сообщения доступны получателю через IMAP до их удаления пользователем или по истечении установленного срока  
+  (*обычно 4–8 недель*);
+- Протоколы IMAP и SMTP защищены паролем, уникальным для каждого аккаунта;
+- Пользователи могут самостоятельно просматривать или удалять сообщения через любой стандартный IMAP-клиент;
+- Также возможно подключение к «службе передачи в реальном времени»,  
+  которая устанавливает P2P-соединение между устройствами и позволяет отправлять временные сообщения,  
+  которые *никогда* не сохраняются на сервере — даже в зашифрованном виде.
+
+### 2.1 Создание аккаунта
+
+Аккаунт создаётся одним из двух способов:
+
+- с помощью QR-кода приглашения,  
+  отсканированного через приложение Delta Chat;
+
+- автоматически, при создании и регистрации аккаунта в {{ config.mail_domain }} через приложение Delta Chat.
+
+В любом случае, обрабатывается только созданный адрес электронной почты.  
+Номера телефонов, другие адреса электронной почты или любые другие идентификаторы не требуются.  
+Правовое основание для обработки —  
+статья 6 (1) пункт b Общего регламента по защите данных (GDPR),  
+так как вы заключаете пользовательский договор, пользуясь нашим сервисом.
+
+### 2.2 Обработка почтовых сообщений
+
+Кроме того, мы обрабатываем данные,  
+необходимые для обеспечения стабильной работы инфраструктуры сервера,  
+доставки сообщений и предотвращения злоупотреблений.
+
+- Поэтому может потребоваться обработка содержимого и/или метаданных  
+  (например, заголовков писем и технической информации SMTP) во время передачи;
+
+- Мы храним логи передаваемых сообщений ограниченное время —  
+  они используются для устранения проблем с доставкой и ошибок ПО.
+
+Также мы вводим ограничения для защиты системы от перегрузок:
+
+- ограничения скорости (rate limits),
+- лимиты на объём хранения,
+- ограничения на размер сообщений,
+- любые другие меры, необходимые для стабильной работы сервера и предотвращения злоупотреблений.
+
+Обработка вышеуказанных данных необходима для предоставления сервиса.  
+Правовое основание — статья 6 (1) пункт b GDPR.  
+Обработка данных в целях безопасности и предотвращения злоупотреблений основана на статье 6 (1) пункт f GDPR,  
+и соответствует нашим законным интересам.
+
+Мы не используем собранные данные для определения вашей личности.
+
+---
+
+## 3. Обработка при посещении сайта
+
+При посещении нашего сайта браузер вашего устройства  
+автоматически передаёт определённую информацию на сервер,  
+где она временно сохраняется в так называемых лог-файлах.  
+Эти данные автоматически удаляются (обычно через *7 дней*).
+
+Среди собираемых данных:
+
+- тип используемого браузера,
+- операционная система,
+- дата и время доступа,
+- страна и IP-адрес,
+- запрашиваемый файл или ресурс,
+- объём переданных данных,
+- статус доступа (успешно, ошибка и т.п.),
+- страница, с которой был сделан запрос.
+
+Хостинг нашего сайта осуществляется внешним провайдером.  
+Личные данные, собираемые на сайте, хранятся на его серверах.  
+Провайдер обрабатывает данные строго по нашим инструкциям,  
+в пределах заключённого договора на обработку данных (ст. 28 GDPR).
+
+Цели обработки:
+
+- обеспечение стабильного подключения к сайту;
+- удобство использования сайта;
+- контроль безопасности и стабильности системы;
+- административные цели.
+
+Правовое основание — статья 6 (1) пункт f GDPR.  
+Собранные данные не используются для установления вашей личности.
+
+---
+
+## 4. Передача данных
+
+Мы не сохраняем личные данные,  
+но письма, ожидающие доставки, могут содержать личную информацию.  
+Такие данные не передаются третьим лицам, за исключением следующих случаев:
+
+a) при наличии вашего явного согласия (ст. 6 п.1 п. a GDPR);  
+
+b) если передача необходима для защиты прав, интересов или правовой позиции (ст. 6 п.1 п. f GDPR);  
+
+c) если это требуется по закону (ст. 6 п.1 п. c GDPR);  
+
+d) если это необходимо для исполнения договора с вами (ст. 6 п.1 п. b GDPR);  
+
+e) если обработка осуществляется сервис-провайдером по нашему поручению,  
+   с которым заключён договор (ст. 28 GDPR),  
+   предусматривающий меры безопасности и контроль с нашей стороны.
+
+---
+
+## 5. Права субъектов данных
+
+Ваши права закреплены в статьях 12–23 GDPR.  
+Так как сервер не хранит персональные данные — даже в зашифрованном виде —  
+предоставление информации или подача возражений не требуются.  
+Удаление данных можно выполнить напрямую через приложение Delta Chat.
+
+Если у вас есть вопросы или жалобы, напишите нам:  
+{{ config.privacy_mail }}
+
+Также вы можете обратиться в надзорный орган по месту вашего проживания,  
+работы или к органу, ответственному за нашу деятельность:  
+`{{ config.privacy_supervisor }}`.
+
+---
+
+## 6. Актуальность политики конфиденциальности
+
+Настоящая политика действует с *октября 2024 года*.  
+В случае изменений в услугах или законодательства  
+она может быть обновлена.
+///