filtermail: use single SMTP connection to reinject mails

Services are distinguished based on ALPN.
For example,
    openssl s_client -connect example.org:443 -alpn smtp
gives SMTP connection and
    openssl s_client -connect example.org:443 -alpn imap
gives IMAP connection.

.github/workflows/ci.yaml

@@ -9,7 +9,7 @@ jobs:
     name: isolated chatmaild tests
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: run chatmaild tests 
         working-directory: chatmaild
@@ -19,7 +19,7 @@ jobs:
     name: deploy-chatmail tests 
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: initenv 
         run: scripts/initenv.sh

.github/workflows/staging.testrun.org-default.zone

@@ -1,6 +1,6 @@
-;; Zone file for staging.testrun.org
+;; Zone file for staging2.testrun.org
 
-$ORIGIN staging.testrun.org.
+$ORIGIN staging2.testrun.org.
 $TTL 300
 
 @  IN  SOA     ns.testrun.org. root.nine.testrun.org (
@@ -15,6 +15,7 @@ $TTL 300
 @  IN NS ns.testrun.org.
 
 ;; DNS records.
-@       IN      A       37.27.37.98
-mta-sts.staging.testrun.org.    CNAME   staging.testrun.org.
-www.staging.testrun.org.        CNAME   staging.testrun.org.
+@       IN      A       37.27.24.139
+mta-sts.staging2.testrun.org.    CNAME   staging2.testrun.org.
+www.staging2.testrun.org.        CNAME   staging2.testrun.org.
+

.github/workflows/test-and-deploy.yaml

@@ -1,4 +1,4 @@
-name: deploy on staging.testrun.org, and run tests
+name: deploy on staging2.testrun.org, and run tests
 
 on:
   push:
@@ -7,31 +7,41 @@ on:
   pull_request:
     paths-ignore:
       - 'scripts/**'
+      - '**/README.md'
+      - 'CHANGELOG.md'
+      - 'LICENSE'
 
 jobs:
   deploy:
-    name: deploy on staging.testrun.org, and run tests
+    name: deploy on staging2.testrun.org, and run tests
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     concurrency:
-      group: staging-deploy
-      cancel-in-progress: true
+      group: ci-${{ github.workflow }}-${{ github.ref }}
+      cancel-in-progress: ${{ !contains(github.ref, '$GITHUB_REF') }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: jsok/serialize-workflow-action@v1
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions/checkout@v4
 
       - name: prepare SSH
         run: |
           mkdir ~/.ssh
           echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
           chmod 600 ~/.ssh/id_ed25519
-          ssh-keyscan staging.testrun.org > ~/.ssh/known_hosts
+          ssh-keyscan staging2.testrun.org > ~/.ssh/known_hosts
           # save previous acme & dkim state
-          rsync -avz root@staging.testrun.org:/var/lib/acme . || true
-          rsync -avz root@staging.testrun.org:/etc/dkimkeys . || true
+          rsync -avz root@staging2.testrun.org:/var/lib/acme . || true
+          rsync -avz root@staging2.testrun.org:/etc/dkimkeys . || true
           # store previous acme & dkim state on ns.testrun.org, if it contains useful certs
           if [ -f dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys root@ns.testrun.org:/tmp/ || true; fi
-          if [ -z "$(ls -A acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme root@ns.testrun.org:/tmp/ || true; fi
+          if [ "$(ls -A acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme root@ns.testrun.org:/tmp/ || true; fi
+          # make sure CAA record isn't set
+          ssh -o StrictHostKeyChecking=accept-new root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging2.testrun.org.zone
+          ssh root@ns.testrun.org systemctl reload nsd
 
-      - name: rebuild staging.testrun.org to have a clean VPS
+      - name: rebuild staging2.testrun.org to have a clean VPS
         run: |
             curl -X POST \
             -H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
@@ -46,17 +56,17 @@ jobs:
 
       - name: upload TLS cert after rebuilding
         run: |
-          echo " --- wait until staging.testrun.org VPS is rebuilt --- "
+          echo " --- wait until staging2.testrun.org VPS is rebuilt --- "
           rm ~/.ssh/known_hosts
-          while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u ; do sleep 1 ; done
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u
+          while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org id -u ; do sleep 1 ; done
+          ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org id -u
           # download acme & dkim state from ns.testrun.org
           rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme acme-restore || true
           rsync -avz root@ns.testrun.org:/tmp/dkimkeys dkimkeys-restore || true
-          # restore acme & dkim state to staging.testrun.org
-          rsync -avz acme-restore/acme/ root@staging.testrun.org:/var/lib/acme || true
-          rsync -avz dkimkeys-restore/dkimkeys/ root@staging.testrun.org:/etc/dkimkeys || true
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org chown root:root -R /var/lib/acme || true
+          # restore acme & dkim state to staging2.testrun.org
+          rsync -avz acme-restore/acme/ root@staging2.testrun.org:/var/lib/acme || true
+          rsync -avz dkimkeys-restore/dkimkeys/ root@staging2.testrun.org:/etc/dkimkeys || true
+          ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org chown root:root -R /var/lib/acme || true
 
       - name: run formatting checks 
         run: cmdeploy fmt -v 
@@ -64,23 +74,23 @@ jobs:
       - name: run deploy-chatmail offline tests 
         run: pytest --pyargs cmdeploy 
 
-      - run: cmdeploy init staging.testrun.org
+      - run: cmdeploy init staging2.testrun.org
 
-      - run: cmdeploy run
+      - run: cmdeploy run --verbose
 
       - name: set DNS entries
         run: |
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
-          cmdeploy dns --zonefile staging-generated.zone
+          ssh -o StrictHostKeyChecking=accept-new root@staging2.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
+          cmdeploy dns --zonefile staging-generated.zone --verbose
           cat staging-generated.zone >> .github/workflows/staging.testrun.org-default.zone
           cat .github/workflows/staging.testrun.org-default.zone
-          scp .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging.testrun.org.zone
-          ssh root@ns.testrun.org nsd-checkzone staging.testrun.org /etc/nsd/staging.testrun.org.zone
+          scp .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
+          ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
           ssh root@ns.testrun.org systemctl reload nsd
 
       - name: cmdeploy test
         run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow
 
       - name: cmdeploy dns (try 3 times)
-        run: cmdeploy dns || cmdeploy dns || cmdeploy dns
+        run: cmdeploy dns -v || cmdeploy dns -v || cmdeploy dns -v
 

CHANGELOG.md

@@ -2,8 +2,90 @@
 
 ## untagged
 
+- BREAKING: new required chatmail.ini values:
+
+  mailboxes_dir = /home/vmail/mail/{mail_domain}
+  passdb = /home/vmail/passdb.sqlite
+  
+  reducing hardcoding these two paths all over the files, also improving testability. 
+  ([#351](https://github.com/deltachat/chatmail/pull/351))
+
+- BREAKING: new required chatmail.ini value 'delete_inactive_users_after = 100'
+  which removes users from database and mails after 100 days without any login. 
+  ([#350](https://github.com/deltachat/chatmail/pull/350))
+
+- reload nginx in the acmetool cronjob
+  ([#360](https://github.com/deltachat/chatmail/pull/360))
+
+- remove checking of reverse-DNS PTR records.  Chatmail-servers don't
+  depend on it and even in the wider e-mail system it's not common anymore. 
+  If it's an issue, a chatmail operator can still care to properly set reverse DNS. 
+  ([#348](https://github.com/deltachat/chatmail/pull/348))
+
+- Make DNS-checking faster and more interactive, run it fully during "cmdeploy run",
+  also introducing a generic mechanism for rapid remote ssh-based python function execution. 
+  ([#346](https://github.com/deltachat/chatmail/pull/346))
+
+- Don't fix file owner ship of /home/vmail 
+  ([#345](https://github.com/deltachat/chatmail/pull/345))
+
+- Support iterating over all users with doveadm commands 
+  ([#344](https://github.com/deltachat/chatmail/pull/344))
+
+- Test and fix for attempts to create inadmissible accounts 
+  ([#333](https://github.com/deltachat/chatmail/pull/321))
+
+- check that OpenPGP has only PKESK, SKESK and SEIPD packets
+  ([#323](https://github.com/deltachat/chatmail/pull/323),
+   [#324](https://github.com/deltachat/chatmail/pull/324))
+
+- improve filtermail checks for encrypted messages and drop support for unencrypted MDNs
+  ([#320](https://github.com/deltachat/chatmail/pull/320))
+
+- replace `bash` with `/bin/sh`
+  ([#334](https://github.com/deltachat/chatmail/pull/334))
+
+- Increase number of logged in IMAP sessions to 50000
+  ([#335](https://github.com/deltachat/chatmail/pull/335))
+
+- filtermail: do not allow ASCII armor without actual payload
+  ([#325](https://github.com/deltachat/chatmail/pull/325))
+
+- Remove sieve to enable hardlink deduplication in LMTP
+  ([#343](https://github.com/deltachat/chatmail/pull/343))
+
+- dovecot: enable gzip compression on disk
+  ([#341](https://github.com/deltachat/chatmail/pull/341))
+
+- DKIM-sign Content-Type and oversign all signed headers
+  ([#296](https://github.com/deltachat/chatmail/pull/296))
+
+- Add nonci_accounts metric
+  ([#347](https://github.com/deltachat/chatmail/pull/347))
+
+- doveauth: log when a new account is created
+  ([#349](https://github.com/deltachat/chatmail/pull/349))
+
+- Multiplex HTTPS, IMAP and SMTP on port 443
+  ([#357](https://github.com/deltachat/chatmail/pull/357))
+
+## 1.3.0 - 2024-06-06
+
+- don't check necessary DNS records on cmdeploy init anymore
+  ([#316](https://github.com/deltachat/chatmail/pull/316))
+
+- ensure cron and acl are installed
+  ([#293](https://github.com/deltachat/chatmail/pull/293),
+  [#310](https://github.com/deltachat/chatmail/pull/310))
+
+- change default for delete_mails_after from 40 to 20 days
+  ([#300](https://github.com/deltachat/chatmail/pull/300))
+
+- save journald logs only to memory and save nginx logs to journald instead of file
+  ([#299](https://github.com/deltachat/chatmail/pull/299))
+
 - fix writing of multiple obs repositories in `/etc/apt/sources.list`
-  ([#272](https://github.com/deltachat/chatmail/issues/272))
+  ([#290](https://github.com/deltachat/chatmail/pull/290))
 
 - metadata: add support for `/shared/vendor/deltachat/irohrelay`
   ([#284](https://github.com/deltachat/chatmail/pull/284))

README.md

@@ -15,6 +15,8 @@ after which the initially specified password is required for using them.
 
 ## Deploying your own chatmail server 
 
+To deploy chatmail on your own server, you must have set-up ssh authentication and need to use an ed25519 key, due to an [upstream bug in paramiko](https://github.com/paramiko/paramiko/issues/2191). You also need to add your private key to the local ssh-agent, because you can't type in your password during deployment.
+
 We use `chat.example.org` as the chatmail domain in the following steps. 
 Please substitute it with your own domain. 
 
@@ -153,7 +155,8 @@ While this file is present, account creation will be blocked.
 
 [Postfix](http://www.postfix.org/) listens on ports 25 (smtp) and 587 (submission) and 465 (submissions).
 [Dovecot](https://www.dovecot.org/) listens on ports 143 (imap) and 993 (imaps).
-[nginx](https://www.nginx.com/) listens on port 443 (https).
+[nginx](https://www.nginx.com/) listens on port 8443 (https-alt) and 443 (https).
+Port 443 multiplexes HTTPS, IMAP and SMTP using ALPN to redirect connections to ports 8443, 465 or 993.
 [acmetool](https://hlandau.github.io/acmetool/) listens on port 80 (http).
 
 Delta Chat apps will, however, discover all ports and configurations

chatmaild/pyproject.toml

@@ -26,6 +26,7 @@ chatmail-metadata = "chatmaild.metadata:main"
 filtermail = "chatmaild.filtermail:main"
 echobot = "chatmaild.echo:main"
 chatmail-metrics = "chatmaild.metrics:main"
+delete_inactive_users = "chatmaild.delete_inactive_users:main"
 
 [project.entry-points.pytest11]
 "chatmaild.testplugin" = "chatmaild.tests.plugin"

chatmaild/src/chatmaild/config.py

@@ -1,9 +1,13 @@
+from pathlib import Path
+
 import iniconfig
 
 
 def read_config(inipath):
+    assert Path(inipath).exists(), inipath
     cfg = iniconfig.IniConfig(inipath)
-    return Config(inipath, params=cfg.sections["params"])
+    params = cfg.sections["params"]
+    return Config(inipath, params=params)
 
 
 class Config:
@@ -13,11 +17,14 @@ class Config:
         self.max_user_send_per_minute = int(params["max_user_send_per_minute"])
         self.max_mailbox_size = params["max_mailbox_size"]
         self.delete_mails_after = params["delete_mails_after"]
+        self.delete_inactive_users_after = int(params["delete_inactive_users_after"])
         self.username_min_length = int(params["username_min_length"])
         self.username_max_length = int(params["username_max_length"])
         self.password_min_length = int(params["password_min_length"])
         self.passthrough_senders = params["passthrough_senders"].split()
         self.passthrough_recipients = params["passthrough_recipients"].split()
+        self.mailboxes_dir = Path(params["mailboxes_dir"].strip())
+        self.passdb_path = Path(params["passdb_path"].strip())
         self.filtermail_smtp_port = int(params["filtermail_smtp_port"])
         self.postfix_reinject_port = int(params["postfix_reinject_port"])
         self.iroh_relay = params.get("iroh_relay")
@@ -29,14 +36,36 @@ class Config:
     def _getbytefile(self):
         return open(self._inipath, "rb")
 
+    def get_user_maildir(self, addr):
+        if addr and addr != "." and "/" not in addr:
+            res = self.mailboxes_dir.joinpath(addr).resolve()
+            if res.is_relative_to(self.mailboxes_dir):
+                return res
+        raise ValueError(f"invalid address {addr!r}")
 
-def write_initial_config(inipath, mail_domain):
+
+def write_initial_config(inipath, mail_domain, overrides):
+    """Write out default config file, using the specified config value overrides."""
     from importlib.resources import files
 
     inidir = files(__package__).joinpath("ini")
-    content = (
-        inidir.joinpath("chatmail.ini.f").read_text().format(mail_domain=mail_domain)
-    )
+    source_inipath = inidir.joinpath("chatmail.ini.f")
+    content = source_inipath.read_text().format(mail_domain=mail_domain)
+
+    # apply config overrides
+    new_lines = []
+    for line in content.split("\n"):
+        new_line = line.strip()
+        if new_line and new_line[0] not in "#[":
+            name, value = map(str.strip, new_line.split("=", maxsplit=1))
+            value = overrides.get(name, value)
+            new_line = f"{name} = {value}"
+        new_lines.append(new_line)
+
+    content = "\n".join(new_lines)
+
+    # apply testrun privacy overrides
+
     if mail_domain.endswith(".testrun.org"):
         override_inipath = inidir.joinpath("override-testrun.ini")
         privacy = iniconfig.IniConfig(override_inipath)["privacy"]

chatmaild/src/chatmaild/delete_inactive_users.py

@@ -0,0 +1,33 @@
+"""
+Remove inactive users
+"""
+
+import shutil
+import sys
+import time
+
+from .config import read_config
+from .database import Database
+from .doveauth import iter_userdb_lastlogin_before
+
+
+def delete_inactive_users(db, config, CHUNK=100):
+    cutoff_date = time.time() - config.delete_inactive_users_after * 86400
+
+    old_users = iter_userdb_lastlogin_before(db, cutoff_date)
+    chunks = (old_users[i : i + CHUNK] for i in range(0, len(old_users), CHUNK))
+    for sublist in chunks:
+        for user in sublist:
+            user_mail_dir = config.get_user_maildir(user)
+            shutil.rmtree(user_mail_dir, ignore_errors=True)
+
+        with db.write_transaction() as conn:
+            for user in sublist:
+                conn.execute("DELETE FROM users WHERE addr = ?", (user,))
+
+
+def main():
+    (cfgpath,) = sys.argv[1:]
+    config = read_config(cfgpath)
+    db = Database(config.passdb_path)
+    delete_inactive_users(db, config)

chatmaild/src/chatmaild/doveauth.py

@@ -60,6 +60,7 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
             config.username_min_length,
             config.username_max_length,
         )
+        return False
 
     return True
 
@@ -67,7 +68,7 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
 def get_user_data(db, config: Config, user):
     if user == f"echo@{config.mail_domain}":
         return dict(
-            home=f"/home/vmail/mail/{config.mail_domain}/echo@{config.mail_domain}",
+            home=str(config.get_user_maildir(user)),
             uid="vmail",
             gid="vmail",
         )
@@ -75,7 +76,7 @@ def get_user_data(db, config: Config, user):
     with db.read_connection() as conn:
         result = conn.get_user(user)
     if result:
-        result["home"] = f"/home/vmail/mail/{config.mail_domain}/{user}"
+        result["home"] = str(config.get_user_maildir(user))
         result["uid"] = "vmail"
         result["gid"] = "vmail"
     return result
@@ -85,7 +86,7 @@ def lookup_userdb(db, config: Config, user):
     return get_user_data(db, config, user)
 
 
-def lookup_passdb(db, config: Config, user, cleartext_password):
+def lookup_passdb(db, config: Config, user, cleartext_password, last_login=None):
     if user == f"echo@{config.mail_domain}":
         # Echobot writes password it wants to log in with into /run/echobot/password
         try:
@@ -95,21 +96,25 @@ def lookup_passdb(db, config: Config, user, cleartext_password):
             return None
 
         return dict(
-            home=f"/home/vmail/mail/{config.mail_domain}/echo@{config.mail_domain}",
+            home=str(config.get_user_maildir(user)),
             uid="vmail",
             gid="vmail",
             password=encrypt_password(password),
         )
 
+    if last_login is None:
+        last_login = time.time()
+    last_login = int(last_login)
+
     with db.write_transaction() as conn:
         userdata = conn.get_user(user)
         if userdata:
             # Update last login time.
             conn.execute(
-                "UPDATE users SET last_login=? WHERE addr=?", (int(time.time()), user)
+                "UPDATE users SET last_login=? WHERE addr=?", (last_login, user)
             )
 
-            userdata["home"] = f"/home/vmail/mail/{config.mail_domain}/{user}"
+            userdata["home"] = str(config.get_user_maildir(user))
             userdata["uid"] = "vmail"
             userdata["gid"] = "vmail"
             return userdata
@@ -119,15 +124,34 @@ def lookup_passdb(db, config: Config, user, cleartext_password):
         encrypted_password = encrypt_password(cleartext_password)
         q = """INSERT INTO users (addr, password, last_login)
                VALUES (?, ?, ?)"""
-        conn.execute(q, (user, encrypted_password, int(time.time())))
+        conn.execute(q, (user, encrypted_password, last_login))
+        print(f"Created address: {user}", file=sys.stderr)
         return dict(
-            home=f"/home/vmail/mail/{config.mail_domain}/{user}",
+            home=str(config.get_user_maildir(user)),
             uid="vmail",
             gid="vmail",
             password=encrypted_password,
         )
 
 
+def iter_userdb(db) -> list:
+    """Get a list of all user addresses."""
+    with db.read_connection() as conn:
+        rows = conn.execute(
+            "SELECT addr from users",
+        ).fetchall()
+    return [x[0] for x in rows]
+
+
+def iter_userdb_lastlogin_before(db, cutoff_date):
+    """Get a list of users where last login was before cutoff_date."""
+    with db.read_connection() as conn:
+        rows = conn.execute(
+            "SELECT addr FROM users WHERE last_login < ?", (cutoff_date,)
+        ).fetchall()
+    return [x[0] for x in rows]
+
+
 def split_and_unescape(s):
     """Split strings using double quote as a separator and backslash as escape character
     into parts."""
@@ -191,6 +215,13 @@ def handle_dovecot_request(msg, db, config: Config):
                     reply_command = "N"
         json_res = json.dumps(res) if res else ""
         return f"{reply_command}{json_res}\n"
+    elif short_command == "I":  # ITERATE
+        # example: I0\t0\tshared/userdb/
+        parts = msg[1:].split("\t")
+        if parts[2] == "shared/userdb/":
+            result = "".join(f"Oshared/userdb/{user}\t\n" for user in iter_userdb(db))
+            return f"{result}\n"
+
     raise UnknownCommand(msg)
 
 
@@ -214,9 +245,9 @@ class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
 
 
 def main():
-    socket = sys.argv[1]
-    db = Database(sys.argv[2])
-    config = read_config(sys.argv[3])
+    socket, cfgpath = sys.argv[1:]
+    config = read_config(cfgpath)
+    db = Database(config.passdb_path)
 
     class Handler(StreamRequestHandler):
         def handle(self):

chatmaild/src/chatmaild/filtermail.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python3
 import asyncio
+import base64
+import binascii
 import logging
 import sys
 import time
@@ -13,8 +15,100 @@ from aiosmtpd.controller import Controller
 from .config import read_config
 
 
+def check_openpgp_payload(payload: bytes):
+    """Checks the OpenPGP payload.
+
+    OpenPGP payload must consist only of PKESK and SKESK packets
+    terminated by a single SEIPD packet.
+
+    Returns True if OpenPGP payload is correct,
+    False otherwise.
+
+    May raise IndexError while trying to read OpenPGP packet header
+    if it is truncated.
+    """
+    i = 0
+    while i < len(payload):
+        # Only OpenPGP format is allowed.
+        if payload[i] & 0xC0 != 0xC0:
+            return False
+
+        packet_type_id = payload[i] & 0x3F
+        i += 1
+        if payload[i] < 192:
+            # One-octet length.
+            body_len = payload[i]
+            i += 1
+        elif payload[i] < 224:
+            # Two-octet length.
+            body_len = ((payload[i] - 192) << 8) + payload[i + 1] + 192
+            i += 2
+        elif payload[i] == 255:
+            # Five-octet length.
+            body_len = (
+                (payload[i + 1] << 24)
+                | (payload[i + 2] << 16)
+                | (payload[i + 3] << 8)
+                | payload[i + 4]
+            )
+            i += 5
+        else:
+            # Partial body length is not allowed.
+            return False
+
+        i += body_len
+
+        if i == len(payload):
+            if packet_type_id == 18:
+                # Last packet should be
+                # Symmetrically Encrypted and Integrity Protected Data Packet (SEIPD)
+                return True
+        elif packet_type_id not in [1, 3]:
+            # All packets except the last one must be either
+            # Public-Key Encrypted Session Key Packet (PKESK)
+            # or
+            # Symmetric-Key Encrypted Session Key Packet (SKESK)
+            return False
+
+    if i == 0:
+        return False
+
+    if i > len(payload):
+        # Payload is truncated.
+        return False
+    return True
+
+
+def check_armored_payload(payload: str):
+    prefix = "-----BEGIN PGP MESSAGE-----\r\n\r\n"
+    if not payload.startswith(prefix):
+        return False
+    payload = payload.removeprefix(prefix)
+
+    suffix = "-----END PGP MESSAGE-----\r\n\r\n"
+    if not payload.endswith(suffix):
+        return False
+    payload = payload.removesuffix(suffix)
+
+    # Remove CRC24.
+    payload = payload.rpartition("=")[0]
+
+    try:
+        payload = base64.b64decode(payload)
+    except binascii.Error:
+        return False
+
+    try:
+        return check_openpgp_payload(payload)
+    except IndexError:
+        return False
+
+
 def check_encrypted(message):
-    """Check that the message is an OpenPGP-encrypted message."""
+    """Check that the message is an OpenPGP-encrypted message.
+
+    MIME structure of the message must correspond to <https://www.rfc-editor.org/rfc/rfc3156>.
+    """
     if not message.is_multipart():
         return False
     if message.get("subject") != "...":
@@ -23,54 +117,44 @@ def check_encrypted(message):
         return False
     parts_count = 0
     for part in message.iter_parts():
+        # We explicitly check Content-Type of each part later,
+        # but this is to be absolutely sure `get_payload()` returns string and not list.
+        if part.is_multipart():
+            return False
+
         if parts_count == 0:
             if part.get_content_type() != "application/pgp-encrypted":
                 return False
+
+            payload = part.get_payload()
+            if payload.strip() != "Version: 1":
+                return False
         elif parts_count == 1:
             if part.get_content_type() != "application/octet-stream":
                 return False
+
+            if not check_armored_payload(part.get_payload()):
+                return False
         else:
             return False
         parts_count += 1
     return True
 
 
-def check_mdn(message, envelope):
-    if len(envelope.rcpt_tos) != 1:
-        return False
-
-    for name in ["auto-submitted", "chat-version"]:
-        if not message.get(name):
-            return False
-
-    if message.get_content_type() != "multipart/report":
-        return False
-
-    body = message.get_body()
-    if body.get_content_type() != "text/plain":
-        return False
-
-    if list(body.iter_attachments()) or list(body.iter_parts()):
-        return False
-
-    # even with all mime-structural checks an attacker
-    # could try to abuse the subject or body to contain links or other
-    # annoyance -- we skip on checking subject/body for now as Delta Chat
-    # should evolve to create E2E-encrypted read receipts anyway.
-    # and then MDNs are just encrypted mail and can pass the border
-    # to other instances.
-
-    return True
-
-
 async def asyncmain_beforequeue(config):
     port = config.filtermail_smtp_port
-    Controller(BeforeQueueHandler(config), hostname="127.0.0.1", port=port).start()
+    smtp_client = SMTPClient("localhost", config.postfix_reinject_port)
+    Controller(
+        BeforeQueueHandler(config, smtp_client=smtp_client),
+        hostname="127.0.0.1",
+        port=port,
+    ).start()
 
 
 class BeforeQueueHandler:
-    def __init__(self, config):
+    def __init__(self, config, smtp_client):
         self.config = config
+        self.smtp_client = smtp_client
         self.send_rate_limiter = SendRateLimiter()
 
     async def handle_MAIL(self, server, session, envelope, address, mail_options):
@@ -92,8 +176,9 @@ class BeforeQueueHandler:
         if error:
             return error
         logging.info("re-injecting the mail that passed checks")
-        client = SMTPClient("localhost", self.config.postfix_reinject_port)
-        client.sendmail(envelope.mail_from, envelope.rcpt_tos, envelope.content)
+        self.smtp_client.sendmail(
+            envelope.mail_from, envelope.rcpt_tos, envelope.content
+        )
         return "250 OK"
 
     def check_DATA(self, envelope):
@@ -108,9 +193,6 @@ class BeforeQueueHandler:
         if envelope.mail_from.lower() != from_addr.lower():
             return f"500 Invalid FROM <{from_addr!r}> for <{envelope.mail_from!r}>"
 
-        if not mail_encrypted and check_mdn(message, envelope):
-            return
-
         if envelope.mail_from in self.config.passthrough_senders:
             return
 

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -8,17 +8,20 @@ mail_domain = {mail_domain}
 #
 
 #
-# Account Restrictions
+# Restrictions on user addresses
 #
 
 # how many mails a user can send out per minute
 max_user_send_per_minute = 60
 
-# maximum mailbox size of a chatmail account
+# maximum mailbox size of a chatmail address
 max_mailbox_size = 100M
 
 # days after which mails are unconditionally deleted
-delete_mails_after = 40
+delete_mails_after = 20
+
+# days after which users without a login are deleted (database and mails)
+delete_inactive_users_after = 100
 
 # minimum length a username must have
 username_min_length = 9
@@ -29,7 +32,7 @@ username_max_length = 9
 # minimum length a password must have
 password_min_length = 9
 
-# list of chatmail accounts which can send outbound un-encrypted mail
+# list of chatmail addresses which can send outbound un-encrypted mail
 passthrough_senders =
 
 # list of e-mail recipients for which to accept outbound un-encrypted mails
@@ -39,6 +42,12 @@ passthrough_recipients = xstore@testrun.org groupsbot@hispanilandia.net
 # Deployment Details
 #
 
+# Directory where user mailboxes are stored
+mailboxes_dir = /home/vmail/mail/{mail_domain}
+
+# user address sqlite database path
+passdb_path = /home/vmail/passdb.sqlite
+
 # where the filtermail SMTP service listens
 filtermail_smtp_port = 10080
 
@@ -60,4 +69,3 @@ privacy_pdo =
 
 # postal address of the privacy supervisor
 privacy_supervisor =
-

chatmaild/src/chatmaild/metadata.py

@@ -1,7 +1,6 @@
 import logging
 import os
 import sys
-from pathlib import Path
 from socketserver import (
     StreamRequestHandler,
     ThreadingMixIn,
@@ -128,12 +127,12 @@ class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
 
 
 def main():
-    socket, vmail_dir, config_path = sys.argv[1:]
+    socket, config_path = sys.argv[1:]
 
     config = read_config(config_path)
     iroh_relay = config.iroh_relay
 
-    vmail_dir = Path(vmail_dir)
+    vmail_dir = config.mailboxes_dir
     if not vmail_dir.exists():
         logging.error("vmail dir does not exist: %r", vmail_dir)
         return 1

chatmaild/src/chatmaild/metrics.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 import sys
-import time
 from pathlib import Path
 
 
@@ -16,9 +15,15 @@ def main(vmail_dir=None):
         if path.name[:3] in ("ci-", "ac_"):
             ci_accounts += 1
 
-    timestamp = int(time.time() * 1000)
-    print(f"accounts {accounts} {timestamp}")
-    print(f"ci_accounts {ci_accounts} {timestamp}")
+    print("# HELP total number of accounts")
+    print("# TYPE accounts gauge")
+    print(f"accounts {accounts}")
+    print("# HELP number of CI accounts")
+    print("# TYPE ci_accounts gauge")
+    print(f"ci_accounts {ci_accounts}")
+    print("# HELP number of non-CI accounts")
+    print("# TYPE nonci_accounts gauge")
+    print(f"nonci_accounts {accounts - ci_accounts}")
 
 
 if __name__ == "__main__":

chatmaild/src/chatmaild/tests/mail-data/literal.eml

@@ -0,0 +1,44 @@
+From: {from_addr}
+
+To: {to_addr}
+
+Subject: ...
+
+Date: Sun, 15 Oct 2023 16:43:21 +0000
+
+Message-ID: <Mr.UVyJWZmkCKM.hGzNc6glBE_@c2.testrun.org>
+
+In-Reply-To: <Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>
+
+References: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>
+
+	<Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>
+
+Chat-Version: 1.0
+
+Autocrypt: addr={from_addr}; prefer-encrypt=mutual;
+
+	keydata=xjMEZSwWjhYJKwYBBAHaRw8BAQdAQBEhqeJh0GueHB6kF/DUQqYCxARNBVokg/AzT+7LqH
+
+	rNFzxiYXJiYXpAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUsFo4CGwMECwkIBwYVCAkKCwID
+
+	FgIBFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX9A4AEAnHWHp49eBCMHK5t66gYPiW
+
+	XQuB1mwUjzGfYWB+0RXUoA/0xcQ3FbUNlGKW7Blp6eMFfViv6Mv2d3kNSXACB6nmcMzjgEZSwWjhIK
+
+	KwYBBAGXVQEFAQEHQBpY5L2M1XHo0uxf8SX1wNLBp/OVvidoWHQF2Jz+kJsUAwEIB8J4BBgWCAAgBQ
+
+	JlLBaOAhsMFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX/INgEA37AJaNvruYsJVanP
+
+	IXnYw4CKd55UAwl8Zcy+M2diAbkA/0fHHcGV4r78hpbbL1Os52DPOdqYQRauIeJUeG+G6bQO
+
+MIME-Version: 1.0
+
+Content-Type: multipart/encrypted; protocol="application/pgp-encrypted";
+
+	boundary="YFrteb74qSXmggbOxZL9dRnhymywAi"
+
+
+
+
+

chatmaild/src/chatmaild/tests/plugin.py

@@ -16,7 +16,11 @@ def make_config(tmp_path):
     inipath = tmp_path.joinpath("chatmail.ini")
 
     def make_conf(mail_domain):
-        write_initial_config(inipath, mail_domain=mail_domain)
+        basedir = tmp_path.joinpath(f"vmail/{mail_domain}")
+        basedir.mkdir(parents=True, exist_ok=True)
+        passdb = tmp_path.joinpath("vmail/passdb.sqlite")
+        overrides = dict(mailboxes_dir=str(basedir), passdb_path=str(passdb))
+        write_initial_config(inipath, mail_domain, overrides=overrides)
         return read_config(inipath)
 
     return make_conf
@@ -68,7 +72,9 @@ def maildata(request):
     assert datadir.exists(), datadir
 
     def maildata(name, from_addr, to_addr):
-        data = datadir.joinpath(name).read_text()
+        # Using `.read_bytes().decode()` instead of `.read_text()` to preserve newlines.
+        data = datadir.joinpath(name).read_bytes().decode()
+
         text = data.format(from_addr=from_addr, to_addr=to_addr)
         return BytesParser(policy=policy.default).parsebytes(text.encode())
 

chatmaild/src/chatmaild/tests/test_config.py

@@ -1,3 +1,4 @@
+import pytest
 from chatmaild.config import read_config
 
 
@@ -24,9 +25,37 @@ def test_read_config_testrun(make_config):
     assert config.postfix_reinject_port == 10025
     assert config.max_user_send_per_minute == 60
     assert config.max_mailbox_size == "100M"
-    assert config.delete_mails_after == "40"
+    assert config.delete_mails_after == "20"
     assert config.username_min_length == 9
     assert config.username_max_length == 9
     assert config.password_min_length == 9
     assert "privacy@testrun.org" in config.passthrough_recipients
     assert config.passthrough_senders == []
+
+
+def test_config_userstate_paths(make_config, tmp_path):
+    config = make_config("something.testrun.org")
+    mailboxes_dir = config.mailboxes_dir
+    passdb_path = config.passdb_path
+    assert mailboxes_dir.name == "something.testrun.org"
+    assert passdb_path.name == "passdb.sqlite"
+    assert passdb_path.is_relative_to(tmp_path)
+    assert config.mail_domain == "something.testrun.org"
+    path = config.get_user_maildir("user1@something.testrun.org")
+    assert not path.exists()
+    assert path == mailboxes_dir.joinpath("user1@something.testrun.org")
+
+    with pytest.raises(ValueError):
+        config.get_user_maildir("")
+
+    with pytest.raises(ValueError):
+        config.get_user_maildir(None)
+
+    with pytest.raises(ValueError):
+        config.get_user_maildir("../some@something.testrun.org")
+
+    with pytest.raises(ValueError):
+        config.get_user_maildir("..")
+
+    with pytest.raises(ValueError):
+        config.get_user_maildir(".")

chatmaild/src/chatmaild/tests/test_delete_inactive_users.py

@@ -0,0 +1,51 @@
+import time
+
+from chatmaild.delete_inactive_users import delete_inactive_users
+from chatmaild.doveauth import lookup_passdb
+
+
+def test_remove_stale_users(db, example_config):
+    new = time.time()
+    old = new - (example_config.delete_inactive_users_after * 86400) - 1
+
+    def create_user(addr, last_login):
+        lookup_passdb(db, example_config, addr, "q9mr3faue", last_login=last_login)
+        md = example_config.get_user_maildir(addr)
+        md.mkdir(parents=True)
+        md.joinpath("cur").mkdir()
+        md.joinpath("cur", "something").mkdir()
+
+    # create some stale and some new accounts
+    to_remove = []
+    for i in range(150):
+        addr = f"oldold{i:03}@chat.example.org"
+        create_user(addr, last_login=old)
+        with db.read_connection() as conn:
+            assert conn.get_user(addr)
+        to_remove.append(addr)
+
+    remain = []
+    for i in range(5):
+        addr = f"newnew{i:03}@chat.example.org"
+        create_user(addr, last_login=new)
+        remain.append(addr)
+
+    # check pre and post-conditions for delete_inactive_users()
+
+    for addr in to_remove:
+        assert example_config.get_user_maildir(addr).exists()
+
+    delete_inactive_users(db, example_config)
+
+    for p in example_config.mailboxes_dir.iterdir():
+        assert not p.name.startswith("old")
+
+    for addr in to_remove:
+        assert not example_config.get_user_maildir(addr).exists()
+        with db.read_connection() as conn:
+            assert not conn.get_user(addr)
+
+    for addr in remain:
+        assert example_config.get_user_maildir(addr).exists()
+        with db.read_connection() as conn:
+            assert conn.get_user(addr)

chatmaild/src/chatmaild/tests/test_doveauth.py

@@ -11,8 +11,12 @@ from chatmaild.doveauth import (
     get_user_data,
     handle_dovecot_protocol,
     handle_dovecot_request,
+    is_allowed_to_create,
+    iter_userdb,
+    iter_userdb_lastlogin_before,
     lookup_passdb,
 )
+from chatmaild.newemail import create_newemail_dict
 
 
 def test_basic(db, example_config):
@@ -25,6 +29,49 @@ def test_basic(db, example_config):
     assert data == data2
 
 
+def test_iterate_addresses(db, example_config):
+    addresses = []
+
+    for i in range(10):
+        addresses.append(f"asdf1234{i}@chat.example.org")
+        lookup_passdb(db, example_config, addresses[-1], "q9mr3faue")
+    res = iter_userdb(db)
+    assert res == addresses
+
+
+def test_iterate_addresses_lastlogin_before(db, example_config):
+    addresses = []
+
+    cutoff_date = 1000
+    for i in range(10):
+        addr = f"oldold{i:03}@chat.example.org"
+        lookup_passdb(
+            db, example_config, addr, "q9mr3faue", last_login=cutoff_date - 10
+        )
+        addresses.append(addr)
+
+    for i in range(5):
+        addr = f"newnew{i:03}@chat.example.org"
+        lookup_passdb(db, example_config, addr, "q9mr3faue", last_login=cutoff_date + i)
+
+    res = iter_userdb_lastlogin_before(db, cutoff_date)
+    assert sorted(res) == sorted(addresses)
+
+
+def test_invalid_username_length(example_config):
+    config = example_config
+    config.username_min_length = 6
+    config.username_max_length = 10
+    password = create_newemail_dict(config)["password"]
+    assert not is_allowed_to_create(config, f"a1234@{config.mail_domain}", password)
+    assert is_allowed_to_create(config, f"012345@{config.mail_domain}", password)
+    assert is_allowed_to_create(config, f"0123456@{config.mail_domain}", password)
+    assert is_allowed_to_create(config, f"0123456789@{config.mail_domain}", password)
+    assert not is_allowed_to_create(
+        config, f"0123456789x@{config.mail_domain}", password
+    )
+
+
 def test_dont_overwrite_password_on_wrong_login(db, example_config):
     """Test that logging in with a different password doesn't create a new user"""
     res = lookup_passdb(
@@ -67,10 +114,7 @@ def test_handle_dovecot_request(db, example_config):
     assert res
     assert res[0] == "O" and res.endswith("\n")
     userdata = json.loads(res[1:].strip())
-    assert (
-        userdata["home"]
-        == "/home/vmail/mail/chat.example.org/some42123@chat.example.org"
-    )
+    assert userdata["home"].endswith("chat.example.org/some42123@chat.example.org")
     assert userdata["uid"] == userdata["gid"] == "vmail"
     assert userdata["password"].startswith("{SHA512-CRYPT}")
 
@@ -92,6 +136,18 @@ def test_handle_dovecot_protocol(db, example_config):
     assert wfile.getvalue() == b"N\n"
 
 
+def test_handle_dovecot_protocol_iterate(db, gencreds, example_config):
+    lookup_passdb(db, example_config, "asdf00000@chat.example.org", "q9mr3faue")
+    lookup_passdb(db, example_config, "asdf11111@chat.example.org", "q9mr3faue")
+    rfile = io.BytesIO(b"H3\t2\t0\t\tauth\nI0\t0\tshared/userdb/")
+    wfile = io.BytesIO()
+    handle_dovecot_protocol(rfile, wfile, db, example_config)
+    lines = wfile.getvalue().decode("ascii").split("\n")
+    assert lines[0] == "Oshared/userdb/asdf00000@chat.example.org\t"
+    assert lines[1] == "Oshared/userdb/asdf11111@chat.example.org\t"
+    assert not lines[2]
+
+
 def test_50_concurrent_lookups_different_accounts(db, gencreds, example_config):
     num_threads = 50
     req_per_thread = 5

chatmaild/src/chatmaild/tests/test_filtermail.py

@@ -2,8 +2,8 @@ import pytest
 from chatmaild.filtermail import (
     BeforeQueueHandler,
     SendRateLimiter,
+    check_armored_payload,
     check_encrypted,
-    check_mdn,
 )
 
 
@@ -62,34 +62,19 @@ def test_filtermail_encryption_detection(maildata):
     assert not check_encrypted(msg)
 
 
-def test_filtermail_is_mdn(maildata, gencreds, handler):
+def test_filtermail_no_literal_packets(maildata):
+    """Test that literal OpenPGP packet is not considered an encrypted mail."""
+    msg = maildata("literal.eml", from_addr="1@example.org", to_addr="2@example.org")
+    assert not check_encrypted(msg)
+
+
+def test_filtermail_unencrypted_mdn(maildata, gencreds):
+    """Unencrypted MDNs should not pass."""
     from_addr = gencreds()[0]
     to_addr = gencreds()[0] + ".other"
     msg = maildata("mdn.eml", from_addr, to_addr)
 
-    class env:
-        mail_from = from_addr
-        rcpt_tos = [to_addr]
-        content = msg.as_bytes()
-
-    assert check_mdn(msg, env)
-    print(msg.as_string())
-
-    assert not handler.check_DATA(env)
-
-
-def test_filtermail_to_multiple_recipients_no_mdn(maildata, gencreds):
-    from_addr = gencreds()[0]
-    to_addr = gencreds()[0] + ".other"
-    thirdaddr = gencreds()[0]
-    msg = maildata("mdn.eml", from_addr, to_addr)
-
-    class env:
-        mail_from = from_addr
-        rcpt_tos = [to_addr, thirdaddr]
-        content = msg.as_bytes()
-
-    assert not check_mdn(msg, env)
+    assert not check_encrypted(msg)
 
 
 def test_send_rate_limiter():
@@ -142,3 +127,59 @@ def test_passthrough_senders(gencreds, handler, maildata):
 
     # assert that None/no error is returned
     assert not handler.check_DATA(envelope=env)
+
+
+def test_check_armored_payload():
+    payload = """-----BEGIN PGP MESSAGE-----\r
+\r
+wU4DSqFx0d1yqAoSAQdAYkX/ZN/Az4B0k7X47zKyWrXxlDEdS3WOy0Yf2+GJTFgg\r
+Zk5ql0mLG8Ze+ZifCS0XMO4otlemSyJ0K1ZPdFMGzUDBTgNqzkFabxXoXRIBB0AM\r
+755wlX41X6Ay3KhnwBq7yEqSykVH6F3x11iHPKraLCAGZoaS8bKKNy/zg5slda1X\r
+pt14b4aC1VwtSnYhcRRELNLD/wE2TFif+g7poMmFY50VyMPLYjVP96Z5QCT4+z4H\r
+Ikh/pRRN8S3JNMrRJHc6prooSJmLcx47Y5un7VFy390MsJ+LiUJuQMDdYWRAinfs\r
+Ebm89Ezjm7F03qbFPXE0X4ZNzVXS/eKO0uhJQdiov/vmbn41rNtHmNpqjaO0vi5+\r
+sS9tR7yDUrIXiCUCN78eBLVioxtktsPZm5cDORbQWzv+7nmCEz9/JowCUcBVdCGn\r
+1ofOaH82JCAX/cRx08pLaDNj6iolVBsi56Dd+2bGxJOZOG2AMcEyz0pXY0dOAJCD\r
+iUThcQeGIdRnU3j8UBcnIEsjLu2+C+rrwMZQESMWKnJ0rnqTk0pK5kXScr6F/L0L\r
+UE49ccIexNm3xZvYr5drszr6wz3Tv5fdue87P4etBt90gF/Vzknck+g1LLlkzZkp\r
+d8dI0k2tOSPjUbDPnSy1x+X73WGpPZmj0kWT+RGvq0nH6UkJj3AQTG2qf1T8jK+3\r
+rTp3LR9vDkMwDjX4R8SA9c0wdnUzzr79OYQC9lTnzcx+fM6BBmgQ2GrS33jaFLp7\r
+L6/DFpCl5zhnPjM/2dKvMkw/Kd6XS/vjwsO405FQdjSDiQEEAZA+ZvAfcjdccbbU\r
+yCO+x0QNdeBsufDVnh3xvzuWy4CICdTQT4s1AWRPCzjOj+SGmx5WqCLWfsd8Ma0+\r
+w/C7SfTYu1FDQILLM+llpq1M/9GPley4QZ8JQjo262AyPXsPF/OW48uuZz0Db1xT\r
+Yh4iHBztj4VSdy7l2+IyaIf7cnL4EEBFxv/MwmVDXvDlxyvfAfIsd3D9SvJESzKZ\r
+VWDYwaocgeCN+ojKu1p885lu1EfRbX3fr3YO02K5/c2JYDkc0Py0W3wUP/J1XUax\r
+pbKpzwlkxEgtmzsGqsOfMJqBV3TNDrOA2uBsa+uBqP5MGYLZ49S/4v/bW9I01Cr1\r
+D2ZkV510Y1Vgo66WlP8mRqOTyt/5WRhPD+MxXdk67BNN/PmO6tMlVoJDuk+XwWPR\r
+t2TvNaND/yabT9eYI55Og4fzKD6RIjouUX8DvKLkm+7aXxVs2uuLQ3Jco3O82z55\r
+dbShU1jYsrw9oouXUz06MHPbkdhNbF/2hfhZ2qA31sNeovJw65iUv7sDKX3LVWgJ\r
+10jlywcDwqlU8CO7WC9lGixYTbnOkYZpXCGEl8e6Jbs79l42YFo4ogYpFK1NXFhV\r
+kOXRmDf/wmfj+c/ld3L2PkvwlgofhCudOQknZbo3ub1gjiTn7L+lMGHIj/3suMIl\r
+ID4EUxAXScIM1ZEz2fjtW5jATlqYcLjLTbf/olw6HFyPNH+9IssqXeZNKnGwPUB9\r
+3lTXsg0tpzl+x7F/2WjEw1DSNhjC0KnHt1vEYNMkUGDGFdN9y3ERLqX/FIgiASUb\r
+bTvAVupnAK3raBezGmhrs6LsQtLS9P0VvQiLU3uDhMqw8Z4SISLpcD+NnVBHzQqm\r
+6W5Qn/8xsCL6av18yUVTi2G3igt3QCNoYx9evt2ZcIkNoyyagUVjfZe5GHXh8Dnz\r
+GaBXW/hg3HlXLRGaQu4RYCzBMJILcO25OhZOg6jbkCLiEexQlm2e9krB5cXR49Al\r
+UN4fiB0KR9JyG2ayUdNJVkXZSZLnHyRgiaadlpUo16LVvw==\r
+=b5Kp\r
+-----END PGP MESSAGE-----\r
+\r
+"""
+
+    assert check_armored_payload(payload) == True
+
+    payload = """-----BEGIN PGP MESSAGE-----\r
+\r
+HELLOWORLD
+-----END PGP MESSAGE-----\r
+\r
+"""
+    assert check_armored_payload(payload) == False
+
+    payload = """-----BEGIN PGP MESSAGE-----\r
+\r
+=njUN
+-----END PGP MESSAGE-----\r
+\r
+"""
+    assert check_armored_payload(payload) == False

chatmaild/src/chatmaild/tests/test_metrics.py

@@ -8,9 +8,10 @@ def test_main(tmp_path, capsys):
     out, _ = capsys.readouterr()
     d = {}
     for line in out.split("\n"):
-        if line.strip():
-            name, num, _ = line.split()
+        if line.strip() and not line.startswith("#"):
+            name, num = line.split()
             d[name] = int(num)
 
     assert d["accounts"] == 4
     assert d["ci_accounts"] == 3
+    assert d["nonci_accounts"] == 1

cmdeploy/pyproject.toml

@@ -18,6 +18,7 @@ dependencies = [
   "ruff",
   "pytest",
   "pytest-xdist", 
+  "execnet",
   "imap_tools",
 ]
 

cmdeploy/src/cmdeploy/__init__.py

@@ -92,7 +92,7 @@ def _install_remote_venv_with_chatmaild(config) -> None:
         group="root",
         mode="644",
         config={
-            "mail_domain": config.mail_domain,
+            "mailboxes_dir": config.mailboxes_dir,
             "execpath": f"{remote_venv_dir}/bin/chatmail-metrics",
         },
     )
@@ -338,20 +338,6 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
     )
     need_restart |= lua_push_notification_script.changed
 
-    sieve_script = files.put(
-        src=importlib.resources.files(__package__).joinpath("dovecot/default.sieve"),
-        dest="/etc/dovecot/default.sieve",
-        user="root",
-        group="root",
-        mode="644",
-    )
-    need_restart |= sieve_script.changed
-    if sieve_script.changed:
-        server.shell(
-            name="compile sieve script",
-            commands=["/usr/bin/sievec /etc/dovecot/default.sieve"],
-        )
-
     files.template(
         src=importlib.resources.files(__package__).joinpath("dovecot/expunge.cron.j2"),
         dest="/etc/cron.d/expunge",
@@ -479,11 +465,6 @@ def deploy_chatmail(config_path: Path) -> None:
     )
     server.user(name="Create echobot user", user="echobot", system=True)
 
-    server.shell(
-        name="Fix file owner in /home/vmail",
-        commands=["test -d /home/vmail && chown -R vmail:vmail /home/vmail"],
-    )
-
     # Add our OBS repository for dovecot_no_delay
     files.put(
         name="Add Deltachat OBS GPG key to apt keyring",
@@ -535,6 +516,12 @@ def deploy_chatmail(config_path: Path) -> None:
         domains=[mail_domain, f"mta-sts.{mail_domain}", f"www.{mail_domain}"],
     )
 
+    apt.packages(
+        # required for setfacl for echobot
+        name="Install acl",
+        packages="acl",
+    )
+
     apt.packages(
         name="Install Postfix",
         packages="postfix",
@@ -542,12 +529,12 @@ def deploy_chatmail(config_path: Path) -> None:
 
     apt.packages(
         name="Install Dovecot",
-        packages=["dovecot-imapd", "dovecot-lmtpd", "dovecot-sieve"],
+        packages=["dovecot-imapd", "dovecot-lmtpd"],
     )
 
     apt.packages(
         name="Install nginx",
-        packages=["nginx"],
+        packages=["nginx", "libnginx-mod-stream"],
     )
 
     apt.packages(
@@ -636,5 +623,10 @@ def deploy_chatmail(config_path: Path) -> None:
         service="systemd-journald.service",
         running=True,
         enabled=True,
-        restarted=journald_conf,
+        restarted=journald_conf.changed,
+    )
+
+    apt.packages(
+        name="Ensure cron is installed",
+        packages=["cron"],
     )

cmdeploy/src/cmdeploy/acmetool/acmetool.cron

@@ -1,4 +1,4 @@
 SHELL=/bin/sh
 PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
 MAILTO=root
-20 16 * * * root /usr/bin/acmetool --batch reconcile && systemctl reload dovecot && systemctl reload postfix
+20 16 * * * root /usr/bin/acmetool --batch reconcile && systemctl reload dovecot && systemctl reload postfix && systemctl reload nginx

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -15,7 +15,9 @@ from pathlib import Path
 from chatmaild.config import read_config, write_initial_config
 from termcolor import colored
 
-from cmdeploy.dns import check_necessary_dns, show_dns
+from . import remote_funcs
+from .dns import show_dns
+from .sshexec import SSHExec
 
 #
 # cmdeploy sub commands and options
@@ -35,13 +37,10 @@ def init_cmd(args, out):
     mail_domain = args.chatmail_domain
     if args.inipath.exists():
         print(f"Path exists, not modifying: {args.inipath}")
+        return 1
     else:
-        write_initial_config(args.inipath, mail_domain)
+        write_initial_config(args.inipath, mail_domain, overrides={})
         out.green(f"created config file for {mail_domain} in {args.inipath}")
-    check_necessary_dns(
-        out,
-        mail_domain,
-    )
 
 
 def run_cmd_options(parser):
@@ -55,12 +54,7 @@ def run_cmd_options(parser):
 
 def run_cmd(args, out):
     """Deploy chatmail services on the remote server."""
-    mail_domain = args.config.mail_domain
-    if not check_necessary_dns(
-        out,
-        mail_domain,
-    ):
-        sys.exit(1)
+    retcode, remote_data = show_dns(args, out)
 
     env = os.environ.copy()
     env["CHATMAIL_INI"] = args.inipath
@@ -69,7 +63,15 @@ def run_cmd(args, out):
     cmd = f"{pyinf} --ssh-user root {args.config.mail_domain} {deploy_path}"
 
     out.check_call(cmd, env=env)
-    print("Deploy completed, call `cmdeploy dns` next.")
+    if retcode == 0:
+        out.green("Deploy completed, call `cmdeploy test` next.")
+    elif not remote_data["acme_account_url"]:
+        out.red("Deploy completed but letsencrypt not configured")
+        out.red("Run 'cmdeploy dns' or 'cmdeploy run' again")
+        retcode = 0
+    else:
+        out.red("Deploy failed")
+    return retcode
 
 
 def dns_cmd_options(parser):
@@ -81,15 +83,15 @@ def dns_cmd_options(parser):
 
 
 def dns_cmd(args, out):
-    """Generate dns zone file."""
-    exit_code = show_dns(args, out)
-    exit(exit_code)
+    """Check DNS entries and optionally generate dns zone file."""
+    retcode, remote_data = show_dns(args, out)
+    return retcode
 
 
 def status_cmd(args, out):
     """Display status for online chatmail instance."""
 
-    ssh = f"ssh root@{args.config.mail_domain}"
+    sshexec = args.get_sshexec()
 
     out.green(f"chatmail domain: {args.config.mail_domain}")
     if args.config.privacy_mail:
@@ -97,10 +99,8 @@ def status_cmd(args, out):
     else:
         out.red("no privacy settings")
 
-    s1 = "systemctl --type=service --state=running"
-    for line in out.shell_output(f"{ssh} -- {s1}").split("\n"):
-        if line.startswith("  "):
-            print(line)
+    for line in sshexec(remote_funcs.get_systemd_running):
+        print(line)
 
 
 def test_cmd_options(parser):
@@ -129,7 +129,7 @@ def test_cmd(args, out):
         "-n4",
         "-rs",
         "-x",
-        "-vrx",
+        "-v",
         "--durations=5",
     ]
     if args.slow:
@@ -139,14 +139,6 @@ def test_cmd(args, out):
 
 
 def fmt_cmd_options(parser):
-    parser.add_argument(
-        "--verbose",
-        "-v",
-        dest="verbose",
-        action="store_true",
-        help="provide information on invocations",
-    )
-
     parser.add_argument(
         "--check",
         "-c",
@@ -176,7 +168,6 @@ def fmt_cmd(args, out):
 
     out.check_call(" ".join(format_args), quiet=not args.verbose)
     out.check_call(" ".join(check_args), quiet=not args.verbose)
-    return 0
 
 
 def bench_cmd(args, out):
@@ -212,16 +203,6 @@ class Out:
         color = "red" if red else ("green" if green else None)
         print(colored(msg, color), file=file)
 
-    def shell_output(self, arg, no_print=False, timeout=10):
-        if not no_print:
-            self(f"[$ {arg}]", file=sys.stderr)
-            output = subprocess.STDOUT
-        else:
-            output = subprocess.DEVNULL
-        return subprocess.check_output(
-            arg, shell=True, timeout=timeout, stderr=output
-        ).decode()
-
     def check_call(self, arg, env=None, quiet=False):
         if not quiet:
             self(f"[$ {arg}]", file=sys.stderr)
@@ -244,6 +225,14 @@ def add_config_option(parser):
         type=Path,
         help="path to the chatmail.ini file",
     )
+    parser.add_argument(
+        "--verbose",
+        "-v",
+        dest="verbose",
+        action="store_true",
+        default=False,
+        help="provide verbose logging",
+    )
 
 
 def add_subcommand(subparsers, func):
@@ -283,11 +272,18 @@ def get_parser():
 
 
 def main(args=None):
-    """Provide main entry point for 'xdcget' CLI invocation."""
+    """Provide main entry point for 'cmdeploy' CLI invocation."""
     parser = get_parser()
     args = parser.parse_args(args=args)
     if not hasattr(args, "func"):
         return parser.parse_args(["-h"])
+
+    def get_sshexec(log=None):
+        print(f"[ssh] login to {args.config.mail_domain}")
+        return SSHExec(args.config.mail_domain, remote_funcs, log=log)
+
+    args.get_sshexec = get_sshexec
+
     out = Out()
     kwargs = {}
     if args.func.__name__ not in ("init_cmd", "fmt_cmd"):

cmdeploy/src/cmdeploy/dns.py

@@ -1,208 +1,59 @@
 import datetime
 import importlib
-import subprocess
 import sys
 
-import requests
-
-
-class DNS:
-    def __init__(self, out, mail_domain):
-        self.session = requests.Session()
-        self.out = out
-        self.ssh = f"ssh root@{mail_domain} -- "
-        self.out.shell_output(
-            f"{ self.ssh }'apt-get update && apt-get install -y dnsutils'",
-            timeout=60,
-            no_print=True,
-        )
-        try:
-            self.shell(f"unbound-control flush_zone {mail_domain}")
-        except subprocess.CalledProcessError:
-            pass
-
-    def shell(self, cmd):
-        try:
-            return self.out.shell_output(f"{self.ssh}{cmd}", no_print=True)
-        except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
-            if "exit status 255" in str(e) or "timed out" in str(e):
-                self.out.red(f"Error: can't reach the server with: {self.ssh[:-4]}")
-                sys.exit(1)
-            else:
-                raise
-
-    def get_ipv4(self):
-        cmd = "ip a | grep 'inet ' | grep 'scope global' | grep -oE '[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}' | head -1"
-        return self.shell(cmd).strip()
-
-    def get_ipv6(self):
-        cmd = "ip a | grep inet6 | grep 'scope global' | sed -e 's#/64 scope global##' | sed -e 's#inet6##'"
-        return self.shell(cmd).strip()
-
-    def get(self, typ: str, domain: str) -> str:
-        """Get a DNS entry or empty string if there is none."""
-        dig_result = self.shell(f"dig -r -q {domain} -t {typ} +short")
-        line = dig_result.partition("\n")[0]
-        return line
-
-    def check_ptr_record(self, ip: str, mail_domain) -> bool:
-        """Check the PTR record for an IPv4 or IPv6 address."""
-        result = self.shell(f"dig -r -x {ip} +short").rstrip()
-        return result == f"{mail_domain}."
+from . import remote_funcs
 
 
 def show_dns(args, out) -> int:
-    """Check existing DNS records, optionally write them to zone file, return exit code 0 or 1."""
+    """Check existing DNS records, optionally write them to zone file
+    and return (exitcode, remote_data) tuple."""
     template = importlib.resources.files(__package__).joinpath("chatmail.zone.f")
     mail_domain = args.config.mail_domain
-    ssh = f"ssh root@{mail_domain}"
-    dns = DNS(out, mail_domain)
 
-    print("Checking your DKIM keys and DNS entries...")
-    try:
-        acme_account_url = out.shell_output(f"{ssh} -- acmetool account-url")
-    except subprocess.CalledProcessError:
-        print("Please run `cmdeploy run` first.")
-        return 1
+    def log_progress(data):
+        sys.stdout.write(".")
+        sys.stdout.flush()
 
-    dkim_selector = "opendkim"
-    dkim_pubkey = out.shell_output(
-        ssh + f" -- openssl rsa -in /etc/dkimkeys/{dkim_selector}.private"
-        " -pubout 2>/dev/null | awk '/-/{next}{printf(\"%s\",$0)}'"
-    )
-    dkim_entry_value = f"v=DKIM1;k=rsa;p={dkim_pubkey};s=email;t=s"
-    dkim_entry_str = ""
-    while len(dkim_entry_value) >= 255:
-        dkim_entry_str += '"' + dkim_entry_value[:255] + '" '
-        dkim_entry_value = dkim_entry_value[255:]
-    dkim_entry_str += '"' + dkim_entry_value + '"'
-    dkim_entry = f"{dkim_selector}._domainkey.{mail_domain}. TXT {dkim_entry_str}"
+    sshexec = args.get_sshexec(log=print if args.verbose else log_progress)
+    print("Checking DNS entries ", end="\n" if args.verbose else "")
 
-    ipv6 = dns.get_ipv6()
-    reverse_ipv6 = dns.check_ptr_record(ipv6, mail_domain)
-    ipv4 = dns.get_ipv4()
-    reverse_ipv4 = dns.check_ptr_record(ipv4, mail_domain)
-    to_print = []
+    remote_data = sshexec(remote_funcs.perform_initial_checks, mail_domain=mail_domain)
+
+    assert remote_data["ipv4"] or remote_data["ipv6"]
 
     with open(template, "r") as f:
-        zonefile = (
-            f.read()
-            .format(
-                acme_account_url=acme_account_url,
-                sts_id=datetime.datetime.now().strftime("%Y%m%d%H%M"),
-                chatmail_domain=args.config.mail_domain,
-                dkim_entry=dkim_entry,
-                ipv6=ipv6,
-                ipv4=ipv4,
-            )
-            .strip()
+        zonefile = f.read().format(
+            acme_account_url=remote_data["acme_account_url"],
+            dkim_entry=remote_data["dkim_entry"],
+            ipv6=remote_data["ipv6"],
+            ipv4=remote_data["ipv4"],
+            sts_id=datetime.datetime.now().strftime("%Y%m%d%H%M"),
+            chatmail_domain=args.config.mail_domain,
         )
-        try:
-            with open(args.zonefile, "w+") as zf:
-                zf.write(zonefile)
-                print(f"DNS records successfully written to: {args.zonefile}")
-            return 0
-        except TypeError:
-            pass
-        for raw_line in zonefile.splitlines():
-            line = raw_line.format(
-                acme_account_url=acme_account_url,
-                sts_id=datetime.datetime.now().strftime("%Y%m%d%H%M"),
-                chatmail_domain=args.config.mail_domain,
-                dkim_entry=dkim_entry,
-                ipv6=ipv6,
-            ).strip()
-            for typ in ["A", "AAAA", "CNAME", "CAA"]:
-                if f" {typ} " in line:
-                    domain, value = line.split(f" {typ} ")
-                    current = dns.get(typ, domain.strip()[:-1])
-                    if current != value.strip():
-                        to_print.append(line)
-            if " MX " in line:
-                domain, typ, prio, value = line.split()
-                current = dns.get(typ, domain[:-1])
-                if not current:
-                    to_print.append(line)
-                elif current.split()[1] != value:
-                    print(line.replace(prio, str(int(current[0]) + 1)))
-            if " SRV " in line:
-                domain, typ, prio, weight, port, value = line.split()
-                current = dns.get("SRV", domain[:-1])
-                if current != f"{prio} {weight} {port} {value}":
-                    to_print.append(line)
-            if " TXT " in line:
-                domain, value = line.split(" TXT ")
-                current = dns.get("TXT", domain.strip()[:-1])
-                if domain.startswith("_mta-sts."):
-                    if current:
-                        if current.split("id=")[0] == value.split("id=")[0]:
-                            continue
 
-                # TXT records longer than 255 bytes
-                # are split into multiple <character-string>s.
-                # This typically happens with DKIM record
-                # which contains long RSA key.
-                #
-                # Removing `" "` before comparison
-                # to get back a single string.
-                if current.replace('" "', "") != value.replace('" "', ""):
-                    to_print.append(line)
+    to_print = sshexec(remote_funcs.check_zonefile, zonefile=zonefile)
+    if not args.verbose:
+        print()
+
+    if getattr(args, "zonefile", None):
+        with open(args.zonefile, "w+") as zf:
+            zf.write(zonefile)
+        out.green(f"DNS records successfully written to: {args.zonefile}")
+        return 0, remote_data
 
-    exit_code = 0
     if to_print:
         to_print.insert(
-            0, "You should configure the following DNS entries at your provider:\n"
+            0, "You should configure the following entries at your DNS provider:\n"
         )
         to_print.append(
-            "\nIf you already configured the DNS entries, wait a bit until the DNS entries propagate to the Internet."
+            "\nIf you already configured the DNS entries, "
+            "wait a bit until the DNS entries propagate to the Internet."
         )
-        print("\n".join(to_print))
+        out.red("\n".join(to_print))
         exit_code = 1
     else:
-        out.green("Great! All your DNS entries are correct.")
+        out.green("Great! All your DNS entries are verified and correct.")
+        exit_code = 0
 
-    to_print = []
-    if not reverse_ipv4:
-        to_print.append(f"\tIPv4:\t{ipv4}\t{args.config.mail_domain}")
-    if not reverse_ipv6:
-        to_print.append(f"\tIPv6:\t{ipv6}\t{args.config.mail_domain}")
-    if len(to_print) > 0:
-        if len(to_print) == 1:
-            warning = "You should add the following PTR/reverse DNS entry:"
-        else:
-            warning = "You should add the following PTR/reverse DNS entries:"
-        out.red(warning)
-        for entry in to_print:
-            print(entry)
-        print(
-            "You can do so at your hosting provider (maybe this isn't your DNS provider)."
-        )
-        exit_code = 1
-    return exit_code
-
-
-def check_necessary_dns(out, mail_domain):
-    """Check whether $mail_domain and mta-sts.$mail_domain resolve."""
-    dns = DNS(out, mail_domain)
-    ipv4 = dns.get("A", mail_domain)
-    ipv6 = dns.get("AAAA", mail_domain)
-    mta_entry = dns.get("CNAME", "mta-sts." + mail_domain)
-    www_entry = dns.get("CNAME", "www." + mail_domain)
-    to_print = []
-    if not (ipv4 or ipv6):
-        to_print.append(f"\t{mail_domain}.\t\t\tA<your server's IPv4 address>")
-    if mta_entry != mail_domain + ".":
-        to_print.append(f"\tmta-sts.{mail_domain}.\tCNAME\t{mail_domain}.")
-    if www_entry != mail_domain + ".":
-        to_print.append(f"\twww.{mail_domain}.\tCNAME\t{mail_domain}.")
-    if to_print:
-        to_print.insert(
-            0,
-            "\nFor chatmail to work, you need to configure this at your DNS provider:\n",
-        )
-        for line in to_print:
-            print(line)
-        print()
-    else:
-        dns.out.green("\nAll necessary DNS entries seem to be set.")
-        return True
+    return exit_code, remote_data

cmdeploy/src/cmdeploy/dovecot/auth.conf

@@ -1,5 +1,7 @@
 uri = proxy:/run/doveauth/doveauth.socket:auth
-iterate_disable = yes
+iterate_disable = no
+iterate_prefix = userdb/
+
 default_pass_scheme = plain
 # %E escapes characters " (double quote), ' (single quote) and \ (backslash) with \ (backslash).
 # See <https://doc.dovecot.org/configuration_manual/config_file/config_variables/#modifiers>

cmdeploy/src/cmdeploy/dovecot/default.sieve

@@ -1,7 +0,0 @@
-require ["imap4flags"];
-
-# flag the message so it doesn't cause a push notification
-
-if header :is ["Auto-Submitted"] ["auto-replied", "auto-generated"] {
-	addflag "$Auto";
-}

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -19,10 +19,33 @@ mail_debug = yes
 #   master: Warning: service(stats): client_limit (1000) reached, client connections are being dropped
 default_client_limit = 20000
 
+# Increase number of logged in IMAP connections.
+# Each connection is handled by a separate `imap` process.
+# `imap` process should have `client_limit=1` as described in
+# <https://doc.dovecot.org/configuration_manual/service_configuration/#service-limits>
+# so each logged in IMAP session will need its own `imap` process.
+#
+# If this limit is reached,
+# users will fail to LOGIN as `imap-login` process
+# will accept them logging in but fail to transfer logged in
+# connection to `imap` process until someone logs out and
+# the following warning will be logged:
+#   Warning: service(imap): process_limit (1024) reached, client connections are being dropped
+service imap {
+  process_limit = 50000
+}
+
 mail_server_admin = mailto:root@{{ config.mail_domain }}
 mail_server_comment = Chatmail server
 
-mail_plugins = quota
+# `zlib` enables compressing messages stored in the maildir.
+# See
+# <https://doc.dovecot.org/configuration_manual/zlib_plugin/>
+# for documentation.
+#
+# quota plugin documentation:
+# <https://doc.dovecot.org/configuration_manual/quota_plugin/>
+mail_plugins = zlib quota
 
 # these are the capabilities Delta Chat cares about actually 
 # so let's keep the network overhead per login small
@@ -44,7 +67,7 @@ userdb {
 ##
 
 # Mailboxes are stored in the "mail" directory of the vmail user home.
-mail_location = maildir:/home/vmail/mail/%d/%u
+mail_location = maildir:{{ config.mailboxes_dir }}/%u
 
 namespace inbox {
   inbox = yes
@@ -80,7 +103,7 @@ mail_privileged_group = vmail
 # Pass all IMAP METADATA requests to the server implementing Dovecot's dict protocol.
 mail_attribute_dict = proxy:/run/chatmail-metadata/metadata.socket:metadata
 
-# Enable IMAP COMPRESS (RFC 4978).
+# `imap_zlib` enables IMAP COMPRESS (RFC 4978).
 # <https://datatracker.ietf.org/doc/html/rfc4978.html>
 protocol imap {
   mail_plugins = $mail_plugins imap_zlib imap_quota
@@ -88,9 +111,6 @@ protocol imap {
 }
 
 protocol lmtp {
-  # quota plugin documentation:
-  # <https://doc.dovecot.org/configuration_manual/quota_plugin/>
-  #
   # notify plugin is a dependency of push_notification plugin:
   # <https://doc.dovecot.org/settings/plugin/notify-plugin/>
   #
@@ -99,10 +119,11 @@ protocol lmtp {
   #
   # mail_lua and push_notification_lua are needed for Lua push notification handler.
   # <https://doc.dovecot.org/configuration_manual/push_notification/#configuration>
-  #
-  # Sieve to mark messages that should not be notified as \Seen
-  # <https://doc.dovecot.org/configuration_manual/sieve/configuration/>
-  mail_plugins = $mail_plugins quota mail_lua notify push_notification push_notification_lua sieve
+  mail_plugins = $mail_plugins mail_lua notify push_notification push_notification_lua
+}
+
+plugin {
+  zlib_save = gz
 }
 
 plugin {
@@ -124,10 +145,6 @@ plugin {
   push_notification_driver = lua:file=/etc/dovecot/push_notification.lua
 }
 
-plugin {
-  sieve_default = file:/etc/dovecot/default.sieve
-}
-
 service lmtp {
   user=vmail
 

cmdeploy/src/cmdeploy/dovecot/expunge.cron.j2

@@ -1,11 +1,12 @@
 # delete all mails after {{ config.delete_mails_after }} days, in the Inbox
-2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 # or in any IMAP subfolder
-2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/.*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 # even if they are unseen
-2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/.*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 # or only temporary (but then they shouldn't be around after {{ config.delete_mails_after }} days anyway).
-2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-3 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -name 'maildirsize' -type f -delete
+2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/.*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+3 0 * * * vmail find {{ config.mailboxes_dir }} -name 'maildirsize' -type f -delete
+4 0 * * * vmail /usr/local/lib/chatmaild/venv/bin/delete_inactive_users /usr/local/lib/chatmaild/chatmail.ini 

cmdeploy/src/cmdeploy/dovecot/push_notification.lua

@@ -17,12 +17,8 @@ function dovecot_lua_notify_event_message_new(user, event)
 
   if user.username ~= event.from_address then
     -- Incoming message
-    if not contains(event.keywords, "$Auto") then
-      -- Not an Auto-Submitted message, notifying.
-
-      -- Notify METADATA server about new message.
-      mbox:metadata_set("/private/messagenew", "")
-    end
+    -- Notify METADATA server about new message.
+    mbox:metadata_set("/private/messagenew", "")
   end
 
   mbox:free()

cmdeploy/src/cmdeploy/journald.conf

@@ -1,2 +1,3 @@
 [Journal]
 MaxRetentionSec=3d
+Storage=volatile

cmdeploy/src/cmdeploy/metrics.cron.j2

@@ -1 +1 @@
-*/5 * * * * root {{ config.execpath }} /home/vmail/mail/{{ config.mail_domain }} >/var/www/html/metrics
+*/5 * * * * root {{ config.execpath }} {{ config.mailboxes_dir }} >/var/www/html/metrics

cmdeploy/src/cmdeploy/nginx/autoconfig.xml.j2

@@ -19,6 +19,13 @@
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
     </incomingServer>
+    <incomingServer type="imap">
+      <hostname>{{ config.domain_name }}</hostname>
+      <port>443</port>
+      <socketType>SSL</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+    </incomingServer>
     <outgoingServer type="smtp">
       <hostname>{{ config.domain_name }}</hostname>
       <port>465</port>
@@ -33,5 +40,12 @@
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
     </outgoingServer>
+    <outgoingServer type="smtp">
+      <hostname>{{ config.domain_name }}</hostname>
+      <port>443</port>
+      <socketType>SSL</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+    </outgoingServer>
   </emailProvider>
 </clientConfig>

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -1,13 +1,30 @@
+load_module modules/ngx_stream_module.so;
+
 user www-data;
 worker_processes auto;
 pid /run/nginx.pid;
-error_log /var/log/nginx/error.log;
+error_log syslog:server=unix:/dev/log,facility=local3;
 
 events {
 	worker_connections 768;
 	# multi_accept on;
 }
 
+stream {
+        map $ssl_preread_alpn_protocols $proxy {
+            default 127.0.0.1:8443;
+            ~\bsmtp\b 127.0.0.1:submissions;
+            ~\bimap\b 127.0.0.1:imaps;
+        }
+
+        server {
+                listen 443;
+                listen [::]:443;
+                proxy_pass $proxy;
+                ssl_preread on;
+        }
+}
+
 http {
 	sendfile on;
 	tcp_nopush on;
@@ -26,8 +43,8 @@ http {
 	gzip on;
 
 	server {
-		listen 443 ssl default_server;
-		listen [::]:443 ssl default_server;
+		listen 8443 ssl default_server;
+		listen [::]:8443 ssl default_server;
 
 		root /var/www/html;
 
@@ -35,6 +52,8 @@ http {
 
 		server_name _;
 
+		access_log syslog:server=unix:/dev/log,facility=local7;
+
 		location / {
 			# First attempt to serve request as file, then
 			# as directory, then fall back to displaying a 404.
@@ -76,9 +95,10 @@ http {
 
 	# Redirect www. to non-www
 	server {
-		listen 443 ssl;
-		listen [::]:443 ssl;
+		listen 8443 ssl;
+		listen [::]:8443 ssl;
 		server_name www.{{ config.domain_name }};
 		return 301 $scheme://{{ config.domain_name }}$request_uri;
+		access_log syslog:server=unix:/dev/log,facility=local7;
 	}
 }

cmdeploy/src/cmdeploy/opendkim/opendkim.conf

@@ -25,7 +25,24 @@ KeyTable          /etc/dkimkeys/KeyTable
 SigningTable      refile:/etc/dkimkeys/SigningTable
 
 # Sign Autocrypt header in addition to the default specified in RFC 6376.
-SignHeaders *,+autocrypt
+#
+# Default list is here:
+# <https://github.com/trusteddomainproject/OpenDKIM/blob/5c539587561785a66c1f67f720f2fb741f320785/libopendkim/dkim.c#L221-L245>
+SignHeaders *,+autocrypt,+content-type
+
+# Prevent addition of second Content-Type header
+# and other important headers that should not be added
+# after signing the message.
+# See
+# <https://www.zone.eu/blog/2024/05/17/bimi-and-dmarc-cant-save-you/>
+# and RFC 6376 (page 41) for reference.
+#
+# We don't use "l=" body length so the problem described in RFC 6376
+# is not applicable, but adding e.g. a second "From" header
+# or second "Autocrypt" header is better prevented in any case.
+#
+# Default is empty.
+OversignHeaders from,reply-to,subject,date,to,cc,resent-date,resent-from,resent-sender,resent-to,resent-cc,in-reply-to,references,list-id,list-help,list-unsubscribe,list-subscribe,list-post,list-owner,list-archive,autocrypt
 
 # Script to ignore signatures that do not correspond to the From: domain.
 ScreenPolicyScript /etc/opendkim/screen.lua

cmdeploy/src/cmdeploy/remote_funcs.py

@@ -0,0 +1,105 @@
+"""
+Functions to be executed on an ssh-connected host.
+
+All functions of this module need to work with Python builtin types
+and standard library dependencies only.
+
+When a remote function executes remotely, it runs in a system python interpreter
+without any installed dependencies.
+
+"""
+
+import re
+import socket
+from subprocess import CalledProcessError, check_output
+
+
+def shell(command, fail_ok=False):
+    log(f"$ {command}")
+    try:
+        return check_output(command, shell=True).decode().rstrip()
+    except CalledProcessError:
+        if not fail_ok:
+            raise
+        return ""
+
+
+def get_systemd_running():
+    lines = shell("systemctl --type=service --state=running").split("\n")
+    return [line for line in lines if line.startswith("  ")]
+
+
+def perform_initial_checks(mail_domain):
+    res = {}
+
+    res["acme_account_url"] = shell("acmetool account-url", fail_ok=True)
+    if not shell("dig", fail_ok=True):
+        shell("apt-get install -y dnsutils")
+    shell(f"unbound-control flush_zone {mail_domain}", fail_ok=True)
+
+    res["dkim_entry"] = get_dkim_entry(mail_domain, dkim_selector="opendkim")
+    res["ipv4"] = get_ip_address(socket.AF_INET)
+    res["ipv6"] = get_ip_address(socket.AF_INET6)
+    return res
+
+
+def get_dkim_entry(mail_domain, dkim_selector):
+    dkim_pubkey = shell(
+        f"openssl rsa -in /etc/dkimkeys/{dkim_selector}.private "
+        "-pubout 2>/dev/null | awk '/-/{next}{printf(\"%s\",$0)}'"
+    )
+    dkim_value_raw = f"v=DKIM1;k=rsa;p={dkim_pubkey};s=email;t=s"
+    dkim_value = '" "'.join(re.findall(".{1,255}", dkim_value_raw))
+    return f'{dkim_selector}._domainkey.{mail_domain}. TXT "{dkim_value}"'
+
+
+def get_ip_address(typ):
+    sock = socket.socket(typ, socket.SOCK_DGRAM)
+    sock.settimeout(0)
+    sock.connect(("notifications.delta.chat", 1))
+    return sock.getsockname()[0]
+
+
+def query_dns(typ, domain):
+    res = shell(f"dig -r -q {domain} -t {typ} +short")
+    return set(filter(None, res.split("\n")))
+
+
+def check_zonefile(zonefile):
+    diff = []
+
+    for zf_line in zonefile.splitlines():
+        zf_domain, zf_typ, zf_value = zf_line.split(maxsplit=2)
+        zf_domain = zf_domain.rstrip(".")
+        zf_value = zf_value.strip()
+        query_values = query_dns(zf_typ, zf_domain)
+        if zf_value in query_values:
+            continue
+
+        if zf_typ == "CAA" and zf_value.endswith('accounturi="'):
+            # this is an initial run where acmetool did not work yet
+            continue
+
+        if query_values and zf_typ == "TXT" and zf_domain.startswith("_mta-sts."):
+            (query_value,) = query_values
+            if query_value.split("id=")[0] == zf_value.split("id=")[0]:
+                continue
+
+        assert zf_typ in ("A", "AAAA", "CNAME", "CAA", "SRV", "MX", "TXT"), zf_line
+        diff.append(zf_line)
+
+    return diff
+
+
+# check if this module is executed remotely
+# and setup a simple serialized function-execution loop
+
+if __name__ == "__channelexec__":
+
+    def log(item):
+        channel.send(("log", item))  # noqa
+
+    while 1:
+        func_name, kwargs = channel.receive()  # noqa
+        res = globals()[func_name](**kwargs)  # noqa
+        channel.send(("finish", res))  # noqa

cmdeploy/src/cmdeploy/service/chatmail-metadata.service.f

@@ -2,7 +2,7 @@
 Description=Chatmail dict proxy for IMAP METADATA
 
 [Service]
-ExecStart={execpath} /run/chatmail-metadata/metadata.socket /home/vmail/mail/{mail_domain} {config_path}
+ExecStart={execpath} /run/chatmail-metadata/metadata.socket {config_path}
 Restart=always
 RestartSec=30
 User=vmail

cmdeploy/src/cmdeploy/service/doveauth.service.f

@@ -2,7 +2,7 @@
 Description=Chatmail dict authentication proxy for dovecot
 
 [Service]
-ExecStart={execpath} /run/doveauth/doveauth.socket /home/vmail/passdb.sqlite {config_path}
+ExecStart={execpath} /run/doveauth/doveauth.socket {config_path}
 Restart=always
 RestartSec=30
 User=vmail

cmdeploy/src/cmdeploy/sshexec.py

@@ -0,0 +1,20 @@
+import execnet
+
+
+class SSHExec:
+    RemoteError = execnet.RemoteError
+
+    def __init__(self, host, remote_funcs, log=None, python="python3", timeout=60):
+        self.gateway = execnet.makegateway(f"ssh=root@{host}//python={python}")
+        self._remote_cmdloop_channel = self.gateway.remote_exec(remote_funcs)
+        self.log = log
+        self.timeout = timeout
+
+    def __call__(self, func, **kwargs):
+        self._remote_cmdloop_channel.send((func.__name__, kwargs))
+        while 1:
+            code, data = self._remote_cmdloop_channel.receive(timeout=self.timeout)
+            if code == "log" and self.log:
+                self.log(data)
+            elif code == "finish":
+                return data

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -2,6 +2,24 @@ import smtplib
 
 import pytest
 
+from cmdeploy import remote_funcs
+from cmdeploy.sshexec import SSHExec
+
+
+class TestSSHExecutor:
+    @pytest.fixture
+    def sshexec(self, sshdomain):
+        return SSHExec(sshdomain, remote_funcs)
+
+    def test_ls(self, sshexec):
+        out = sshexec(remote_funcs.shell, command="ls")
+        out2 = sshexec(remote_funcs.shell, command="ls")
+        assert out == out2
+
+    def test_perform_initial(self, sshexec, maildomain):
+        res = sshexec(remote_funcs.perform_initial_checks, mail_domain=maildomain)
+        assert res["ipv4"] or res["ipv6"]
+
 
 def test_remote(remote, imap_or_smtp):
     lineproducer = remote.iter_output(imap_or_smtp.logcmd)
@@ -86,15 +104,16 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
     pytest.fail("Rate limit was not exceeded")
 
 
+@pytest.mark.slow
 def test_expunged(remote, chatmail_config):
     outdated_days = int(chatmail_config.delete_mails_after) + 1
     find_cmds = [
-        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/cur/*' -mtime +{outdated_days} -type f",
-        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/.*/cur/*' -mtime +{outdated_days} -type f",
-        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/new/*' -mtime +{outdated_days} -type f",
-        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/.*/new/*' -mtime +{outdated_days} -type f",
-        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/tmp/*' -mtime +{outdated_days} -type f",
-        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/.*/tmp/*' -mtime +{outdated_days} -type f",
+        f"find {chatmail_config.mailboxes_dir} -path '*/cur/*' -mtime +{outdated_days} -type f",
+        f"find {chatmail_config.mailboxes_dir} -path '*/.*/cur/*' -mtime +{outdated_days} -type f",
+        f"find {chatmail_config.mailboxes_dir} -path '*/new/*' -mtime +{outdated_days} -type f",
+        f"find {chatmail_config.mailboxes_dir} -path '*/.*/new/*' -mtime +{outdated_days} -type f",
+        f"find {chatmail_config.mailboxes_dir} -path '*/tmp/*' -mtime +{outdated_days} -type f",
+        f"find {chatmail_config.mailboxes_dir} -path '*/.*/tmp/*' -mtime +{outdated_days} -type f",
     ]
     for cmd in find_cmds:
         for line in remote.iter_output(cmd):

cmdeploy/src/cmdeploy/tests/test_cmdeploy.py

@@ -21,8 +21,9 @@ class TestCmdline:
         run = parser.parse_args(["run"])
         assert init and run
 
-    @pytest.mark.xfail(reason="init doesn't exit anymore, check for CLI output instead")
-    def test_init_not_overwrite(self):
-        main(["init", "chat.example.org"])
-        with pytest.raises(SystemExit):
-            main(["init", "chat.example.org"])
+    def test_init_not_overwrite(self, capsys):
+        assert main(["init", "chat.example.org"]) == 0
+        capsys.readouterr()
+        assert main(["init", "chat.example.org"]) == 1
+        out, err = capsys.readouterr()
+        assert "path exists" in out.lower()

scripts/cmdeploy

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/sh
 #
 # Wrapper for cmdelpoy to run it in activated virtualenv.
 set -e

scripts/dovecot/README.md

@@ -0,0 +1,80 @@
+## Introduction to custom Dovecot builds
+
+Chatmail servers use a custom Debian build of the IMAP 'dovecot' server software because 
+
+a) Dovecot developers did not yet merge a [pull request](https://github.com/dovecot/core/pull/216)
+   which majorly speeds up message delivery by removing a hardcoded 0.5 second delay 
+   on relaying incoming messages.  
+
+b) Even if merged, it would take years for it to reach Debian stable.
+
+c) The modified dovecot has been successfully used since December 2023 without issues
+   and we see no noticeable downside (theoretically higher CPU usage but not measureable)
+   but a considerable upside as the delay-removal facilitates end-to-end message 
+   delivery of 200 ms in real networks. 
+
+The modified forked dovecot code lives at 
+[https://github.com/chatmail/dovecot](https://github.com/chatmail/dovecot).  
+The remainder of this document describes the setup of the Debian repository 
+containing the patched dovecot version. 
+
+## Building Debian packages at build.opensuse.org 
+
+Delta Chat developers maintain an [account](https://build.opensuse.org/project/show/home:deltachat) 
+in the [Open Build Service (OBS)](https://openbuildservice.org/), 
+where the [resulting package](https://build.opensuse.org/package/show/home:deltachat/dovecot) 
+is now used in deploying chatmail servers. 
+   
+The Open Build Service (OBS) is a platform for building and distributing software packages 
+across various operating systems and architectures. 
+It supports openSUSE, Fedora, Debian, Ubuntu and Arch.
+It's [primary instance](https://build.opensuse.org/) is ran by the openSUSE project 
+and is part of the pipeline of the creation of SUSE Linux Enterprise.
+
+The OBS provides a mercurial-like interface to create source repositories 
+that are then automatically built. 
+While in theory a package can be created entirely over the web interface, 
+the use of the cli-tool `osc` is more convenient and is described in the [official documentation](https://openbuildservice.org/help/manuals/obs-user-guide/art.obs.bg#sec.obsbg.obsconfig).
+
+### How to build the dovecot debian package on the OBS via our script
+
+In scripts/dovecot/ is a shell script that prepares the required files and pushes them to build.opensuse.org.
+
+Before using the script, you should have osc set up as described in the [official documentation](https://openbuildservice.org/help/manuals/obs-user-guide/art.obs.bg#sec.obsbg.obsconfig).
+
+The script assumes you are on Debian. It automatically installs any needed dependencies and creates the source package. To upload the resulting source package to the OBS you need to enter the username and password for deltachat on build.opensuse.org in the last step of the script.
+
+Use `source build-obs.sh` to run it.
+
+### Adding the resulting OBS repository to Debian 12
+
+Our dovecot fork is automatically installed as part of the chatmail deployment. You can see it in cmdeploy/src/cmdeploy/__init__.py. If you want to add our fork manually to a system, you can do the following:
+
+First add our signing key to your apt keyring:
+
+```
+sudo cp cmdeploy/src/cmdeploy/obs-home-deltachat.gpg /etc/apt/keyrings/obs-home-deltachat.gpg`
+```
+
+Now add our repository and key to /etc/apt/sources.list with a text editor of your choice:
+
+```
+deb [signed-by=/etc/apt/keyrings/obs-home-deltachat.gpg] https://download.opensuse.org/repositories/home:/deltachat/Debian_12/ ./
+```
+
+You can now install dovecot like normal.
+
+```
+sudo apt update
+sudo apt install dovecot-core
+```
+
+### Security concerns 
+
+The signing of the patched dovecot package is done in the OBS and 
+in theory SUSE could make changes to the package delivered.
+It is probably reasonable to trust SUSE to not mess with the build
+process because it would cause serious negative reputation damage for them 
+if they tried and someone finds out. 
+
+Our dovecot fork will receive the same security backports as the dovecot package in Debian Sid.

scripts/dovecot/build-obs.sh

@@ -0,0 +1,54 @@
+#!/bin/sh
+
+# Install dependencies
+echo "Installing dependencies for this script:"
+sudo apt install -y devscripts build-essential osc curl git debhelper-compat
+
+# Define path of your local OBS repository
+SCRIPT_DIR=$PWD
+OBS_PATH=$SCRIPT_DIR/obs
+REPO_PATH=$OBS_PATH/home:deltachat/dovecot/
+
+# Download Debian Source Files
+echo "Downloading precise files from Debian unstable repository..."
+mkdir dovecot-build
+cd dovecot-build
+
+# taken May 6th 2024, from https://packages.debian.org/unstable/dovecot-core 
+curl http://deb.debian.org/debian/pool/main/d/dovecot/dovecot_2.3.21+dfsg1-3.debian.tar.xz -O
+curl http://deb.debian.org/debian/pool/main/d/dovecot/dovecot_2.3.21+dfsg1.orig.tar.gz -O
+curl http://deb.debian.org/debian/pool/main/d/dovecot/dovecot_2.3.21+dfsg1.orig-pigeonhole.tar.gz -O
+
+# Clone the Chatmail Dovecot Repo
+echo "Cloning the Chatmail Dovecot fork..."
+git clone https://github.com/chatmail/dovecot.git
+
+# Build the source package
+echo "Building the source package"
+cd dovecot
+dpkg-source -b .
+
+# Setting up OSC
+echo "Setting up OBS home repository"
+mkdir $OBS_PATH
+cd $OBS_PATH
+rm -rf home:deltachat/dovecot
+osc checkout home:deltachat/dovecot
+
+# Copy Files to Your Local OBS Repository,
+echo "Copying files to your local OBS repository..."
+cd $SCRIPT_DIR/dovecot-build
+cp -rf dovecot_2.3.21+dfsg1-3.debian.tar.xz $REPO_PATH
+cp -rf dovecot_2.3.21+dfsg1.orig.tar.gz $REPO_PATH
+cp -rf dovecot_2.3.21+dfsg1.orig-pigeonhole.tar.gz $REPO_PATH
+cp -rf dovecot_2.3.21+dfsg1-3.dsc $REPO_PATH
+
+# Push Changes to OBS
+echo "Pushing changes to OBS..."
+cd $REPO_PATH
+osc up
+osc add dovecot_2.3.21+dfsg1-3.debian.tar.xz
+osc add dovecot_2.3.21+dfsg1.orig.tar.gz
+osc add dovecot_2.3.21+dfsg1.orig-pigeonhole.tar.gz
+osc add dovecot_2.3.21+dfsg1-3.dsc
+osc commit

scripts/initenv.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
 set -e
 python3 -m venv --upgrade-deps venv
 

www/src/index.md

@@ -1,11 +1,17 @@
 
 <img class="banner" src="collage-top.png"/>
 
-## Dear [Delta Chat](https://get.delta.chat) users and newcomers, 
+## Dear [Delta Chat](https://get.delta.chat) users and newcomers ... 
 
+{% if config.mail_domain != "nine.testrun.org" %}
 Welcome to instant, interoperable and [privacy-preserving](privacy.html) messaging :) 
+{% else %}
+Welcome to the default onboarding server ({{ config.mail_domain }}) 
+for Delta Chat users.  For details how it avoids storing personal information
+please see our [privacy policy](privacy.html). 
+{% endif %}
 
-👉 **Tap** or scan this QR code to get a random `@{{config.mail_domain}}` e-mail address
+👉 **Tap** or scan this QR code to get a `@{{config.mail_domain}}` chat profile
 
 <a href="DCACCOUNT:https://{{ config.mail_domain }}/new">
     <img width=300 style="float: none;" src="qr-chatmail-invite-{{config.mail_domain}}.png" /></a>

www/src/info.md

@@ -1,6 +1,4 @@
 
-<img class="banner" src="collage-info.png"/>
-
 ## More information 
 
 {{ config.mail_domain }} provides a low-maintenance, resource efficient and 
@@ -11,7 +9,7 @@ for the usage in chats, especially DeltaChat.
 ### Choosing a chatmail address instead of using a random one
 
 In the Delta Chat account setup 
-you may tap `LOG INTO YOUR E-MAIL ACCOUNT` 
+you may tap `I already have a profile`
 and fill the two fields like this: 
 
 - `Address`: invent a word with

www/src/privacy.md

@@ -1,21 +1,41 @@
-<img class="banner" src="collage-privacy.png"/>
 
 # Privacy Policy for {{ config.mail_domain }} 
 
-We want to show you in a fair and transparent way
-what personal data is processed by us.
-We follow a strict privacy-by-design approach
-and try to avoid processing your data in the first place,
-but as you may know,
-the internet, 
-and in particular sending e-mail messages,
-does not work without data.
-Still,
-it's only fair that you know at all times
-what personal data is processed
-when you use our service.
+{% if config.mail_domain == "nine.testrun.org" %}
+Welcome to `{{config.mail_domain}}`, the default chatmail onboarding server for Delta Chat users. 
+It is operated on the side by a small sysops team employed by [merlinux](https://merlinux.eu),
+an open-source R&D company also acting as the fiscal sponsor of Delta Chat app developments. 
+See [other chatmail servers](https://delta.chat/en/chatmail) for alternative server operators. 
+{% endif %}
+
+
+## Summary: No personal data asked or collected 
+
+This chatmail server neither asks for nor retains personal information. 
+Chatmail servers exist to reliably transmit (store and deliver) end-to-end encrypted messages
+between user's devices running the Delta Chat messenger app. 
+Technically, you may think of a Chatmail server as 
+an end-to-end encrypted "messaging router" at Internet-scale. 
+
+A chatmail server is very unlike classic e-mail servers (for example Google Mail servers)
+that ask for personal data and permanently store messages. 
+A chatmail server behaves more like the Signal messaging server 
+but does not know about phone numbers and securely and automatically interoperates 
+with other chatmail and classic e-mail servers. 
+
+In particular, this chatmail server 
+
+- unconditionally removes messages after {{ config.delete_mails_after }} days,
+
+- prohibits sending out un-encrypted messages,
+
+- only has temporary log files used for debugging purposes.
+
+Legally, authorities might still regard chatmail as a "classic e-mail" server
+which collects and retains personal data. 
+We do not agree on this interpretation. Nevertheless, we provide more legal details below
+to make life easier for data protection specialists and lawyers scrutinizing chatmail operations. 
 
-If you have any remaining questions about data protection, please contact us. 
 
 ## 1. Name and contact information 
 
@@ -57,7 +77,7 @@ we process the following data and details:
 - Users can retrieve or delete all stored messages 
   without intervention from the operators using standard IMAP client tools.
 
-### 3.1 Account setup
+### 2.1 Account setup
 
 Creating an account happens in one of two ways on our mail servers: 
 
@@ -78,7 +98,7 @@ Art. 6 (1) lit. b GDPR,
 as you have a usage contract with us
 by using our services.
 
-## 3.2 Processing of E-Mail-Messages
+### 2.2 Processing of E-Mail-Messages
 
 In addition,
 we will process data
@@ -104,7 +124,7 @@ Therefore, limits are enforced:
 
 - message size limits
 
-- any other limit neccessary for the whole server to function in a healthy way
+- any other limit necessary for the whole server to function in a healthy way
   and to prevent abuse.
 
 The processing and use of the above permissions
@@ -178,8 +198,9 @@ for the purpose of drawing conclusions about your person.
 
 ## 4. Transfer of Data
 
-Your personal data
-will not be transferred to third parties
+We do not retain any personal data but e-mail messages waiting to be delivered 
+may contain personal data.
+Any such residual personal data will not be transferred to third parties
 for purposes other than those listed below:
 
 a) you have given your express consent