wip

* fix #231

* CI: disable CI for markdown files

* clarify need for ssh-add

* Update README.md

Co-authored-by: missytake <missytake@systemli.org>

---------

Co-authored-by: missytake <missytake@systemli.org>

.github/workflows/test-and-deploy.yaml

@@ -4,12 +4,20 @@ on:
   push:
     branches:
       - main
-      - staging-ci
+  pull_request:
+    paths-ignore:
+      - 'scripts/**'
+      - '**/README.md'
+      - 'CHANGELOG.md'
+      - 'LICENSE'
 
 jobs:
   deploy:
     name: deploy on staging.testrun.org, and run tests
     runs-on: ubuntu-latest
+    concurrency:
+      group: staging-deploy
+      cancel-in-progress: true
     steps:
       - uses: actions/checkout@v3
 
@@ -19,48 +27,57 @@ jobs:
           echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
           chmod 600 ~/.ssh/id_ed25519
           ssh-keyscan staging.testrun.org > ~/.ssh/known_hosts
-     #    rsync -avz root@staging.testrun.org:/var/lib/acme . || true
-     #    rsync -avz root@staging.testrun.org:/var/lib/rspamd/dkim . || true
+          # save previous acme & dkim state
+          rsync -avz root@staging.testrun.org:/var/lib/acme . || true
+          rsync -avz root@staging.testrun.org:/etc/dkimkeys . || true
+          # store previous acme & dkim state on ns.testrun.org, if it contains useful certs
+          if [ -f dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys root@ns.testrun.org:/tmp/ || true; fi
+          if [ -z "$(ls -A acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme root@ns.testrun.org:/tmp/ || true; fi
 
-     #- name: rebuild staging.testrun.org to have a clean VPS
-     #  run: |
-     #      curl -X POST \
-     #      -H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
-     #      -H "Content-Type: application/json" \
-     #      -d '{"image":"debian-12"}' \
-     #      "https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_SERVER_ID }}/actions/rebuild"
+      - name: rebuild staging.testrun.org to have a clean VPS
+        run: |
+            curl -X POST \
+            -H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
+            -H "Content-Type: application/json" \
+            -d '{"image":"debian-12"}' \
+            "https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_SERVER_ID }}/actions/rebuild"
 
       - run: scripts/initenv.sh
 
       - name: append venv/bin to PATH
         run: echo venv/bin >>$GITHUB_PATH
 
+      - name: upload TLS cert after rebuilding
+        run: |
+          echo " --- wait until staging.testrun.org VPS is rebuilt --- "
+          rm ~/.ssh/known_hosts
+          while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u ; do sleep 1 ; done
+          ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u
+          # download acme & dkim state from ns.testrun.org
+          rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme acme-restore || true
+          rsync -avz root@ns.testrun.org:/tmp/dkimkeys dkimkeys-restore || true
+          # restore acme & dkim state to staging.testrun.org
+          rsync -avz acme-restore/acme/ root@staging.testrun.org:/var/lib/acme || true
+          rsync -avz dkimkeys-restore/dkimkeys/ root@staging.testrun.org:/etc/dkimkeys || true
+          ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org chown root:root -R /var/lib/acme || true
+
       - name: run formatting checks 
         run: cmdeploy fmt -v 
 
       - name: run deploy-chatmail offline tests 
         run: pytest --pyargs cmdeploy 
 
-     #- name: upload TLS cert after rebuilding
-     #  run: |
-     #    echo " --- wait until staging.testrun.org VPS is rebuilt --- "
-     #    rm ~/.ssh/known_hosts
-     #    while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u ; do sleep 1 ; done
-     #    ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u
-     #    rsync -avz acme root@staging.testrun.org:/var/lib/ || true
-     #    rsync -avz dkim root@staging.testrun.org:/var/lib/rspamd/ || true
-
       - run: cmdeploy init staging.testrun.org
 
       - run: cmdeploy run
 
       - name: set DNS entries
         run: |
-          #ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org chown _rspamd:_rspamd -R /var/lib/rspamd/dkim
+          ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
           cmdeploy dns --zonefile staging-generated.zone
           cat staging-generated.zone >> .github/workflows/staging.testrun.org-default.zone
           cat .github/workflows/staging.testrun.org-default.zone
-          scp -o StrictHostKeyChecking=accept-new .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging.testrun.org.zone
+          scp .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging.testrun.org.zone
           ssh root@ns.testrun.org nsd-checkzone staging.testrun.org /etc/nsd/staging.testrun.org.zone
           ssh root@ns.testrun.org systemctl reload nsd
 

CHANGELOG.md

@@ -0,0 +1,79 @@
+# Changelog for chatmail deployment 
+
+## untagged
+
+- run metrics generation with systemd-timer instead of cron
+  ([#304](https://github.com/deltachat/chatmail/pull/304))
+
+- change default for delete_mails_after from 40 to 20 days
+  ([#300]https://github.com/deltachat/chatmail/pull/300)
+
+- fix writing of multiple obs repositories in `/etc/apt/sources.list`
+  ([#272](https://github.com/deltachat/chatmail/issues/272))
+
+- metadata: add support for `/shared/vendor/deltachat/irohrelay`
+  ([#284](https://github.com/deltachat/chatmail/pull/284))
+
+- Emit "XCHATMAIL" capability from IMAP server 
+  ([#278](https://github.com/deltachat/chatmail/pull/278))
+
+- Move echobot `into /var/lib/echobot`
+  ([#281](https://github.com/deltachat/chatmail/pull/281))
+
+- Accept Let's Encrypt's new Terms of Services
+  ([#275](https://github.com/deltachat/chatmail/pull/276))
+
+- Reload Dovecot and Postfix when TLS certificate updates
+  ([#271](https://github.com/deltachat/chatmail/pull/271))
+
+- Use forked version of dovecot without hardcoded delays
+  ([#270](https://github.com/deltachat/chatmail/pull/270))
+
+## 1.2.0 - 2024-04-04
+
+- Install dig on the server to resolve DNS records
+  ([#267](https://github.com/deltachat/chatmail/pull/267))
+
+- preserve notification order and exponentially backoff with 
+  retries for tokens where we didn't get a successful return
+  ([#265](https://github.com/deltachat/chatmail/pull/263))
+
+- Run chatmail-metadata and doveauth as vmail
+  ([#261](https://github.com/deltachat/chatmail/pull/261))
+
+- Apply systemd restrictions to echobot
+  ([#259](https://github.com/deltachat/chatmail/pull/259))
+
+- re-enable running the CI in pull requests, but not concurrently 
+  ([#258](https://github.com/deltachat/chatmail/pull/258))
+
+
+## 1.1.0 - 2024-03-28
+
+### The changelog starts to record changes from March 15th, 2024 
+
+- Move systemd unit templates to cmdeploy package 
+  ([#255](https://github.com/deltachat/chatmail/pull/255))
+
+- Persist push tokens and support multiple device per address 
+  ([#254](https://github.com/deltachat/chatmail/pull/254))
+
+- Avoid warning for regular doveauth protocol's hello message. 
+  ([#250](https://github.com/deltachat/chatmail/pull/250))
+
+- Fix various tests to pass again with "cmdeploy test". 
+  ([#245](https://github.com/deltachat/chatmail/pull/245),
+  [#242](https://github.com/deltachat/chatmail/pull/242)
+
+- Ensure lets-encrypt certificates are reloaded after renewal 
+  ([#244]) https://github.com/deltachat/chatmail/pull/244
+
+- Persist tokens to avoid iOS users loosing push-notifications when the
+  chatmail metadata service is restarted (happens regularly during deploys)
+  ([#238](https://github.com/deltachat/chatmail/pull/239)
+
+- Fix failing sieve-script compile errors on incoming messages
+  ([#237](https://github.com/deltachat/chatmail/pull/239)
+
+- Fix quota reporting after expunging of old mails
+  ([#233](https://github.com/deltachat/chatmail/pull/239)

README.md

@@ -15,6 +15,8 @@ after which the initially specified password is required for using them.
 
 ## Deploying your own chatmail server 
 
+To deploy chatmail on your own server, you must have set-up ssh authentication and need to use an ed25519 key, due to an [upstream bug in paramiko](https://github.com/paramiko/paramiko/issues/2191). You also need to add your private key to the local ssh-agent, because you can't type in your password during deployment.
+
 We use `chat.example.org` as the chatmail domain in the following steps. 
 Please substitute it with your own domain. 
 
@@ -159,4 +161,27 @@ While this file is present, account creation will be blocked.
 Delta Chat apps will, however, discover all ports and configurations
 automatically by reading the [autoconfig XML file](https://www.ietf.org/archive/id/draft-bucksch-autoconfig-00.html) from the chatmail service.
 
+## Email authentication
 
+chatmail servers rely on [DKIM](https://www.rfc-editor.org/rfc/rfc6376)
+to authenticate incoming emails.
+Incoming emails must have a valid DKIM signature with
+Signing Domain Identifier (SDID, `d=` parameter in the DKIM-Signature header)
+equal to the `From:` header domain.
+This property is checked by OpenDKIM screen policy script
+before validating the signatures.
+This correpsonds to strict [DMARC](https://www.rfc-editor.org/rfc/rfc7489) alignment (`adkim=s`),
+but chatmail does not rely on DMARC and does not consult the sender policy published in DMARC records.
+Other legacy authentication mechanisms such as [iprev](https://www.rfc-editor.org/rfc/rfc8601#section-2.7.3)
+and [SPF](https://www.rfc-editor.org/rfc/rfc7208) are also not taken into account.
+If there is no valid DKIM signature on the incoming email,
+the sender receives a "5.7.1 No valid DKIM signature found" error.
+
+Outgoing emails must be sent over authenticated connection
+with envelope MAIL FROM (return path) corresponding to the login.
+This is ensured by Postfix which maps login username
+to MAIL FROM with
+[`smtpd_sender_login_maps`](https://www.postfix.org/postconf.5.html#smtpd_sender_login_maps)
+and rejects incorrectly authenticated emails with [`reject_sender_login_mismatch`](reject_sender_login_mismatch) policy.
+`From:` header must correspond to envelope MAIL FROM,
+this is ensured by `filtermail` proxy.

chatmaild/MANIFEST.in

@@ -1,4 +1,3 @@
-include src/chatmaild/*.f 
 include src/chatmaild/ini/*.ini.f
 include src/chatmaild/ini/*.ini
 include src/chatmaild/tests/mail-data/*

chatmaild/pyproject.toml

@@ -10,6 +10,8 @@ dependencies = [
   "iniconfig",
   "deltachat-rpc-server",
   "deltachat-rpc-client",
+  "filelock",
+  "requests",
 ]
 
 [tool.setuptools]
@@ -20,6 +22,7 @@ where = ['src']
 
 [project.scripts]
 doveauth = "chatmaild.doveauth:main"
+chatmail-metadata = "chatmaild.metadata:main"
 filtermail = "chatmaild.filtermail:main"
 echobot = "chatmaild.echo:main"
 chatmail-metrics = "chatmaild.metrics:main"
@@ -33,6 +36,16 @@ log_format = "%(asctime)s %(levelname)s %(message)s"
 log_date_format = "%Y-%m-%d %H:%M:%S"
 log_level = "INFO"
 
+[tool.ruff]
+lint.select = [
+  "F", # Pyflakes
+  "I", # isort
+
+  "PLC", # Pylint Convention
+  "PLE", # Pylint Error
+  "PLW", # Pylint Warning
+]
+
 [tool.tox]
 legacy_tox_ini = """
 [tox]
@@ -44,10 +57,9 @@ skipdist = True
 skip_install = True
 deps =
   ruff
-  black
 commands =
-  black --quiet --check --diff src/
-  ruff src/
+  ruff format --quiet --diff src/
+  ruff check src/
 
 [testenv]
 deps = pytest 

chatmaild/src/chatmaild/config.py

@@ -20,6 +20,7 @@ class Config:
         self.passthrough_recipients = params["passthrough_recipients"].split()
         self.filtermail_smtp_port = int(params["filtermail_smtp_port"])
         self.postfix_reinject_port = int(params["postfix_reinject_port"])
+        self.iroh_relay = params.get("iroh_relay")
         self.privacy_postal = params.get("privacy_postal")
         self.privacy_mail = params.get("privacy_mail")
         self.privacy_pdo = params.get("privacy_pdo")

chatmaild/src/chatmaild/database.py

@@ -1,5 +1,5 @@
-import sqlite3
 import contextlib
+import sqlite3
 import time
 from pathlib import Path
 

chatmaild/src/chatmaild/doveauth.py

@@ -1,22 +1,26 @@
+import crypt
+import json
 import logging
 import os
-import time
 import sys
-import json
-import crypt
+import time
+from pathlib import Path
 from socketserver import (
-    UnixStreamServer,
     StreamRequestHandler,
     ThreadingMixIn,
+    UnixStreamServer,
 )
-import pwd
 
+from .config import Config, read_config
 from .database import Database
-from .config import read_config, Config
 
 NOCREATE_FILE = "/etc/chatmail-nocreate"
 
 
+class UnknownCommand(ValueError):
+    """dictproxy handler received an unkown command"""
+
+
 def encrypt_password(password: str):
     # https://doc.dovecot.org/configuration_manual/authentication/password_schemes/
     passhash = crypt.crypt(password, crypt.METHOD_SHA512)
@@ -42,36 +46,61 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
         return False
     localpart, domain = parts
 
+    if localpart == "echo":
+        # echobot account should not be created in the database
+        return False
+
     if (
         len(localpart) > config.username_max_length
         or len(localpart) < config.username_min_length
     ):
-        if localpart != "echo":
-            logging.warning(
-                "localpart %s has to be between %s and %s chars long",
-                localpart,
-                config.username_min_length,
-                config.username_max_length,
-            )
-            return False
+        logging.warning(
+            "localpart %s has to be between %s and %s chars long",
+            localpart,
+            config.username_min_length,
+            config.username_max_length,
+        )
 
     return True
 
 
-def get_user_data(db, user):
+def get_user_data(db, config: Config, user):
+    if user == f"echo@{config.mail_domain}":
+        return dict(
+            home=f"/home/vmail/mail/{config.mail_domain}/echo@{config.mail_domain}",
+            uid="vmail",
+            gid="vmail",
+        )
+
     with db.read_connection() as conn:
         result = conn.get_user(user)
     if result:
+        result["home"] = f"/home/vmail/mail/{config.mail_domain}/{user}"
         result["uid"] = "vmail"
         result["gid"] = "vmail"
     return result
 
 
-def lookup_userdb(db, user):
-    return get_user_data(db, user)
+def lookup_userdb(db, config: Config, user):
+    return get_user_data(db, config, user)
 
 
 def lookup_passdb(db, config: Config, user, cleartext_password):
+    if user == f"echo@{config.mail_domain}":
+        # Echobot writes password it wants to log in with into /run/echobot/password
+        try:
+            password = Path("/run/echobot/password").read_text()
+        except Exception:
+            logging.exception("Exception when trying to read /run/echobot/password")
+            return None
+
+        return dict(
+            home=f"/home/vmail/mail/{config.mail_domain}/echo@{config.mail_domain}",
+            uid="vmail",
+            gid="vmail",
+            password=encrypt_password(password),
+        )
+
     with db.write_transaction() as conn:
         userdata = conn.get_user(user)
         if userdata:
@@ -80,6 +109,7 @@ def lookup_passdb(db, config: Config, user, cleartext_password):
                 "UPDATE users SET last_login=? WHERE addr=?", (int(time.time()), user)
             )
 
+            userdata["home"] = f"/home/vmail/mail/{config.mail_domain}/{user}"
             userdata["uid"] = "vmail"
             userdata["gid"] = "vmail"
             return userdata
@@ -125,8 +155,12 @@ def split_and_unescape(s):
 
 
 def handle_dovecot_request(msg, db, config: Config):
+    # see https://doc.dovecot.org/3.0/developer_manual/design/dict_protocol/
     short_command = msg[0]
-    if short_command == "L":  # LOOKUP
+    if short_command == "H":  # HELLO
+        # we don't do any checking on versions and just return
+        return
+    elif short_command == "L":  # LOOKUP
         parts = msg[1:].split("\t")
 
         # Dovecot <2.3.17 has only one part,
@@ -142,7 +176,7 @@ def handle_dovecot_request(msg, db, config: Config):
             if type == "userdb":
                 user = args[0]
                 if user.endswith(f"@{config.mail_domain}"):
-                    res = lookup_userdb(db, user)
+                    res = lookup_userdb(db, config, user)
                 if res:
                     reply_command = "O"
                 else:
@@ -157,7 +191,7 @@ def handle_dovecot_request(msg, db, config: Config):
                     reply_command = "N"
         json_res = json.dumps(res) if res else ""
         return f"{reply_command}{json_res}\n"
-    return None
+    raise UnknownCommand(msg)
 
 
 def handle_dovecot_protocol(rfile, wfile, db: Database, config: Config):
@@ -165,12 +199,14 @@ def handle_dovecot_protocol(rfile, wfile, db: Database, config: Config):
         msg = rfile.readline().strip().decode()
         if not msg:
             break
-        res = handle_dovecot_request(msg, db, config)
-        if res:
-            wfile.write(res.encode("ascii"))
-            wfile.flush()
+        try:
+            res = handle_dovecot_request(msg, db, config)
+        except UnknownCommand:
+            logging.warning("unknown command: %r", msg)
         else:
-            logging.warning("request had no answer: %r", msg)
+            if res:
+                wfile.write(res.encode("ascii"))
+                wfile.flush()
 
 
 class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
@@ -179,9 +215,8 @@ class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
 
 def main():
     socket = sys.argv[1]
-    passwd_entry = pwd.getpwnam(sys.argv[2])
-    db = Database(sys.argv[3])
-    config = read_config(sys.argv[4])
+    db = Database(sys.argv[2])
+    config = read_config(sys.argv[3])
 
     class Handler(StreamRequestHandler):
         def handle(self):
@@ -197,7 +232,6 @@ def main():
         pass
 
     with ThreadedUnixStreamServer(socket, Handler) as server:
-        os.chown(socket, uid=passwd_entry.pw_uid, gid=passwd_entry.pw_gid)
         try:
             server.serve_forever()
         except KeyboardInterrupt:

chatmaild/src/chatmaild/echo.py

@@ -3,14 +3,17 @@
 
 it will echo back any message that has non-empty text and also supports the /help command.
 """
+
 import logging
 import os
+import subprocess
 import sys
+from pathlib import Path
 
 from deltachat_rpc_client import Bot, DeltaChat, EventType, Rpc, events
 
-from chatmaild.newemail import create_newemail_dict
 from chatmaild.config import read_config
+from chatmaild.newemail import create_newemail_dict
 
 hooks = events.HookCollection()
 
@@ -18,14 +21,14 @@ hooks = events.HookCollection()
 @hooks.on(events.RawEvent)
 def log_event(event):
     if event.kind == EventType.INFO:
-        logging.info(event.msg)
+        logging.info("%s", event.msg)
     elif event.kind == EventType.WARNING:
-        logging.warning(event.msg)
+        logging.warning("%s", event.msg)
 
 
 @hooks.on(events.RawEvent(EventType.ERROR))
 def log_error(event):
-    logging.error(event.msg)
+    logging.error("%s", event.msg)
 
 
 @hooks.on(events.MemberListChanged)
@@ -48,6 +51,9 @@ def on_group_name_changed(event):
 @hooks.on(events.NewMessage(func=lambda e: not e.command))
 def echo(event):
     snapshot = event.message_snapshot
+    if snapshot.is_info:
+        # Ignore info messages
+        return
     if snapshot.text or snapshot.file:
         snapshot.chat.send_message(text=snapshot.text, file=snapshot.file)
 
@@ -59,6 +65,7 @@ def help_command(event):
 
 
 def main():
+    logging.basicConfig(level=logging.INFO)
     path = os.environ.get("PATH")
     venv_path = sys.argv[0].strip("echobot")
     os.environ["PATH"] = path + ":" + venv_path
@@ -71,14 +78,27 @@ def main():
         account = accounts[0] if accounts else deltachat.add_account()
 
         bot = Bot(account, hooks)
+
+        config = read_config(sys.argv[1])
+
+        # Create password file
+        if bot.is_configured():
+            password = bot.account.get_config("mail_pw")
+        else:
+            password = create_newemail_dict(config)["password"]
+        Path("/run/echobot/password").write_text(password)
+
+        # Give the user which doveauth runs as access to the password file.
+        subprocess.run(
+            ["/usr/bin/setfacl", "-m", "user:vmail:r", "/run/echobot/password"],
+            check=True,
+        )
+
         if not bot.is_configured():
-            config = read_config(sys.argv[1])
-            password = create_newemail_dict(config).get("password")
             email = "echo@" + config.mail_domain
             bot.configure(email, password)
         bot.run_forever()
 
 
 if __name__ == "__main__":
-    logging.basicConfig(level=logging.INFO)
     main()

chatmaild/src/chatmaild/echobot.service.f

@@ -1,11 +0,0 @@
-[Unit]
-Description=Chatmail echo bot for testing it works
-
-[Service]
-ExecStart={execpath} {config_path}
-Environment="PATH={remote_venv_dir}:$PATH"
-Restart=always
-RestartSec=30
-
-[Install]
-WantedBy=multi-user.target

chatmaild/src/chatmaild/filedict.py

@@ -0,0 +1,36 @@
+import json
+import logging
+import os
+from contextlib import contextmanager
+
+import filelock
+
+
+class FileDict:
+    """Concurrency-safe multi-reader/single-writer persistent dict."""
+
+    def __init__(self, path):
+        self.path = path
+        self.lock_path = path.with_name(path.name + ".lock")
+
+    @contextmanager
+    def modify(self):
+        # the OS will release the lock if the process dies,
+        # and the contextmanager will otherwise guarantee release
+        with filelock.FileLock(self.lock_path):
+            data = self.read()
+            yield data
+            write_path = self.path.with_name(self.path.name + ".tmp")
+            with write_path.open("w") as f:
+                json.dump(data, f)
+            os.rename(write_path, self.path)
+
+    def read(self):
+        try:
+            with self.path.open("r") as f:
+                return json.load(f)
+        except FileNotFoundError:
+            return {}
+        except Exception:
+            logging.warning("corrupt serialization state at: %r", self.path)
+            return {}

chatmaild/src/chatmaild/filtermail.py

@@ -1,14 +1,14 @@
 #!/usr/bin/env python3
 import asyncio
 import logging
-import time
 import sys
-from email.parser import BytesParser
+import time
 from email import policy
+from email.parser import BytesParser
 from email.utils import parseaddr
+from smtplib import SMTP as SMTPClient
 
 from aiosmtpd.controller import Controller
-from smtplib import SMTP as SMTPClient
 
 from .config import read_config
 

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -18,7 +18,7 @@ max_user_send_per_minute = 60
 max_mailbox_size = 100M
 
 # days after which mails are unconditionally deleted
-delete_mails_after = 40
+delete_mails_after = 20
 
 # minimum length a username must have
 username_min_length = 9

chatmaild/src/chatmaild/metadata.py

@@ -0,0 +1,166 @@
+import logging
+import os
+import sys
+from pathlib import Path
+from socketserver import (
+    StreamRequestHandler,
+    ThreadingMixIn,
+    UnixStreamServer,
+)
+
+from .config import read_config
+from .filedict import FileDict
+from .notifier import Notifier
+
+DICTPROXY_HELLO_CHAR = "H"
+DICTPROXY_LOOKUP_CHAR = "L"
+DICTPROXY_ITERATE_CHAR = "I"
+DICTPROXY_BEGIN_TRANSACTION_CHAR = "B"
+DICTPROXY_SET_CHAR = "S"
+DICTPROXY_COMMIT_TRANSACTION_CHAR = "C"
+DICTPROXY_TRANSACTION_CHARS = "BSC"
+
+
+class Metadata:
+    # each SETMETADATA on this key appends to a list of unique device tokens
+    # which only ever get removed if the upstream indicates the token is invalid
+    DEVICETOKEN_KEY = "devicetoken"
+
+    def __init__(self, vmail_dir):
+        self.vmail_dir = vmail_dir
+
+    def get_metadata_dict(self, addr):
+        return FileDict(self.vmail_dir / addr / "metadata.json")
+
+    def add_token_to_addr(self, addr, token):
+        with self.get_metadata_dict(addr).modify() as data:
+            tokens = data.setdefault(self.DEVICETOKEN_KEY, [])
+            if token not in tokens:
+                tokens.append(token)
+
+    def remove_token_from_addr(self, addr, token):
+        with self.get_metadata_dict(addr).modify() as data:
+            tokens = data.get(self.DEVICETOKEN_KEY, [])
+            if token in tokens:
+                tokens.remove(token)
+
+    def get_tokens_for_addr(self, addr):
+        mdict = self.get_metadata_dict(addr).read()
+        return mdict.get(self.DEVICETOKEN_KEY, [])
+
+
+def handle_dovecot_protocol(rfile, wfile, notifier, metadata, iroh_relay=None):
+    transactions = {}
+    while True:
+        msg = rfile.readline().strip().decode()
+        if not msg:
+            break
+
+        res = handle_dovecot_request(msg, transactions, notifier, metadata, iroh_relay)
+        if res:
+            wfile.write(res.encode("ascii"))
+            wfile.flush()
+
+
+def handle_dovecot_request(msg, transactions, notifier, metadata, iroh_relay=None):
+    # see https://doc.dovecot.org/3.0/developer_manual/design/dict_protocol/
+    short_command = msg[0]
+    parts = msg[1:].split("\t")
+    if short_command == DICTPROXY_LOOKUP_CHAR:
+        # Lpriv/43f5f508a7ea0366dff30200c15250e3/devicetoken\tlkj123poi@c2.testrun.org
+        keyparts = parts[0].split("/", 2)
+        if keyparts[0] == "priv":
+            keyname = keyparts[2]
+            addr = parts[1]
+            if keyname == metadata.DEVICETOKEN_KEY:
+                res = " ".join(metadata.get_tokens_for_addr(addr))
+                return f"O{res}\n"
+        elif keyparts[0] == "shared":
+            keyname = keyparts[2]
+            if (
+                keyname == "vendor/vendor.dovecot/pvt/server/vendor/deltachat/irohrelay"
+                and iroh_relay
+            ):
+                # Handle `GETMETADATA "" /shared/vendor/deltachat/irohrelay`
+                return f"O{iroh_relay}\n"
+        logging.warning("lookup ignored: %r", msg)
+        return "N\n"
+    elif short_command == DICTPROXY_ITERATE_CHAR:
+        # Empty line means ITER_FINISHED.
+        # If we don't return empty line Dovecot will timeout.
+        return "\n"
+    elif short_command == DICTPROXY_HELLO_CHAR:
+        return  # no version checking
+
+    if short_command not in (DICTPROXY_TRANSACTION_CHARS):
+        logging.warning("unknown dictproxy request: %r", msg)
+        return
+
+    transaction_id = parts[0]
+
+    if short_command == DICTPROXY_BEGIN_TRANSACTION_CHAR:
+        addr = parts[1]
+        transactions[transaction_id] = dict(addr=addr, res="O\n")
+    elif short_command == DICTPROXY_COMMIT_TRANSACTION_CHAR:
+        # each set devicetoken operation persists directly
+        # and does not wait until a "commit" comes
+        # because our dovecot config does not involve
+        # multiple set-operations in a single commit
+        return transactions.pop(transaction_id)["res"]
+    elif short_command == DICTPROXY_SET_CHAR:
+        # For documentation on key structure see
+        # https://github.com/dovecot/core/blob/main/src/lib-storage/mailbox-attribute.h
+
+        keyname = parts[1].split("/")
+        value = parts[2] if len(parts) > 2 else ""
+        addr = transactions[transaction_id]["addr"]
+        if keyname[0] == "priv" and keyname[2] == metadata.DEVICETOKEN_KEY:
+            metadata.add_token_to_addr(addr, value)
+        elif keyname[0] == "priv" and keyname[2] == "messagenew":
+            notifier.new_message_for_addr(addr, metadata)
+        else:
+            # Transaction failed.
+            transactions[transaction_id]["res"] = "F\n"
+
+
+class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
+    request_queue_size = 100
+
+
+def main():
+    socket, vmail_dir, config_path = sys.argv[1:]
+
+    config = read_config(config_path)
+    iroh_relay = config.iroh_relay
+
+    vmail_dir = Path(vmail_dir)
+    if not vmail_dir.exists():
+        logging.error("vmail dir does not exist: %r", vmail_dir)
+        return 1
+
+    queue_dir = vmail_dir / "pending_notifications"
+    queue_dir.mkdir(exist_ok=True)
+    metadata = Metadata(vmail_dir)
+    notifier = Notifier(queue_dir)
+    notifier.start_notification_threads(metadata.remove_token_from_addr)
+
+    class Handler(StreamRequestHandler):
+        def handle(self):
+            try:
+                handle_dovecot_protocol(
+                    self.rfile, self.wfile, notifier, metadata, iroh_relay
+                )
+            except Exception:
+                logging.exception("Exception in the dovecot dictproxy handler")
+                raise
+
+    try:
+        os.unlink(socket)
+    except FileNotFoundError:
+        pass
+
+    with ThreadedUnixStreamServer(socket, Handler) as server:
+        try:
+            server.serve_forever()
+        except KeyboardInterrupt:
+            pass

chatmaild/src/chatmaild/metrics.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
-from pathlib import Path
-import time
 import sys
+import time
+from pathlib import Path
 
 
 def main(vmail_dir=None):

chatmaild/src/chatmaild/newemail.py

@@ -1,13 +1,13 @@
 #!/usr/local/lib/chatmaild/venv/bin/python3
 
-""" CGI script for creating new accounts. """
+"""CGI script for creating new accounts."""
 
 import json
 import random
 import secrets
 import string
 
-from chatmaild.config import read_config, Config
+from chatmaild.config import Config, read_config
 
 CONFIG_PATH = "/usr/local/lib/chatmaild/chatmail.ini"
 ALPHANUMERIC = string.ascii_lowercase + string.digits

chatmaild/src/chatmaild/notifier.py

@@ -0,0 +1,166 @@
+"""
+This modules provides notification machinery for transmitting device tokens to
+a central notification server which in turn contacts a phone provider's notification server
+to trigger Delta Chat apps to retrieve messages and provide instant notifications to users.
+
+The Notifier class arranges the queuing of tokens in separate PriorityQueues
+from which NotifyThreads take and transmit them via HTTPS
+to the `notifications.delta.chat` service.
+The current lack of proper HTTP/2-support in Python leads us
+to use multiple threads and connections to the Rust-implemented `notifications.delta.chat`
+which itself uses HTTP/2 and thus only a single connection to phone-notification providers.
+
+If a token fails to cause a successful notification
+it is moved to a retry-number specific PriorityQueue
+which handles all tokens that failed a particular number of times
+and which are scheduled for retry using exponential back-off timing.
+If a token notification would be scheduled more than DROP_DEADLINE seconds
+after its first attempt, it is dropped with a log error.
+
+Note that tokens are completely opaque to the notification machinery here
+and will in the future be encrypted foreclosing all ability to distinguish
+which device token ultimately goes to which phone-provider notification service,
+or to understand the relation of "device tokens" and chatmail addresses.
+The meaning and format of tokens is basically a matter of Delta-Chat Core and
+the `notification.delta.chat` service.
+"""
+
+import logging
+import math
+import os
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from queue import PriorityQueue
+from threading import Thread
+from uuid import uuid4
+
+import requests
+
+
+@dataclass
+class PersistentQueueItem:
+    path: Path
+    addr: str
+    start_ts: int
+    token: str
+
+    def delete(self):
+        self.path.unlink(missing_ok=True)
+
+    @classmethod
+    def create(cls, queue_dir, addr, start_ts, token):
+        queue_id = uuid4().hex
+        start_ts = int(start_ts)
+        path = queue_dir.joinpath(queue_id)
+        tmp_path = path.with_name(path.name + ".tmp")
+        tmp_path.write_text(f"{addr}\n{start_ts}\n{token}")
+        os.rename(tmp_path, path)
+        return cls(path, addr, start_ts, token)
+
+    @classmethod
+    def read_from_path(cls, path):
+        addr, start_ts, token = path.read_text().split("\n", maxsplit=2)
+        return cls(path, addr, int(start_ts), token)
+
+    def __lt__(self, other):
+        return self.start_ts < other.start_ts
+
+
+class Notifier:
+    URL = "https://notifications.delta.chat/notify"
+    CONNECTION_TIMEOUT = 60.0  # seconds until http-request is given up
+    BASE_DELAY = 8.0  # base seconds for exponential back-off delay
+    DROP_DEADLINE = 5 * 60 * 60  #  drop notifications after 5 hours
+
+    def __init__(self, queue_dir):
+        self.queue_dir = queue_dir
+        max_tries = int(math.log(self.DROP_DEADLINE, self.BASE_DELAY)) + 1
+        self.retry_queues = [PriorityQueue() for _ in range(max_tries)]
+
+    def compute_delay(self, retry_num):
+        return 0 if retry_num == 0 else pow(self.BASE_DELAY, retry_num)
+
+    def new_message_for_addr(self, addr, metadata):
+        start_ts = int(time.time())
+        for token in metadata.get_tokens_for_addr(addr):
+            queue_item = PersistentQueueItem.create(
+                self.queue_dir, addr, start_ts, token
+            )
+            self.queue_for_retry(queue_item)
+
+    def requeue_persistent_queue_items(self):
+        for queue_path in self.queue_dir.iterdir():
+            if queue_path.name.endswith(".tmp"):
+                logging.warning("removing spurious queue item: %r", queue_path)
+                queue_path.unlink()
+                continue
+            queue_item = PersistentQueueItem.read_from_path(queue_path)
+            self.queue_for_retry(queue_item)
+
+    def queue_for_retry(self, queue_item, retry_num=0):
+        delay = self.compute_delay(retry_num)
+        when = int(time.time()) + delay
+        deadline = queue_item.start_ts + self.DROP_DEADLINE
+        if retry_num >= len(self.retry_queues) or when > deadline:
+            queue_item.delete()
+            logging.error("notification exceeded deadline: %r", queue_item.token)
+            return
+
+        self.retry_queues[retry_num].put((when, queue_item))
+
+    def start_notification_threads(self, remove_token_from_addr):
+        self.requeue_persistent_queue_items()
+        threads = {}
+        for retry_num in range(len(self.retry_queues)):
+            # use 4 threads for first-try tokens and less for subsequent tries
+            num_threads = 4 if retry_num == 0 else 2
+            threads[retry_num] = []
+            for _ in range(num_threads):
+                thread = NotifyThread(self, retry_num, remove_token_from_addr)
+                threads[retry_num].append(thread)
+                thread.start()
+        return threads
+
+
+class NotifyThread(Thread):
+    def __init__(self, notifier, retry_num, remove_token_from_addr):
+        super().__init__(daemon=True)
+        self.notifier = notifier
+        self.retry_num = retry_num
+        self.remove_token_from_addr = remove_token_from_addr
+
+    def stop(self):
+        self.notifier.retry_queues[self.retry_num].put((None, None))
+
+    def run(self):
+        requests_session = requests.Session()
+        while self.retry_one(requests_session):
+            pass
+
+    def retry_one(self, requests_session, sleep=time.sleep):
+        when, queue_item = self.notifier.retry_queues[self.retry_num].get()
+        if when is None:
+            return False
+        wait_time = when - int(time.time())
+        if wait_time > 0:
+            sleep(wait_time)
+        self.perform_request_to_notification_server(requests_session, queue_item)
+        return True
+
+    def perform_request_to_notification_server(self, requests_session, queue_item):
+        timeout = self.notifier.CONNECTION_TIMEOUT
+        token = queue_item.token
+        try:
+            res = requests_session.post(self.notifier.URL, data=token, timeout=timeout)
+        except requests.exceptions.RequestException as e:
+            res = e
+        else:
+            if res.status_code in (200, 410):
+                if res.status_code == 410:
+                    self.remove_token_from_addr(queue_item.addr, token)
+                queue_item.delete()
+                return
+
+        logging.warning("Notification request failed: %r", res)
+        self.notifier.queue_for_retry(queue_item, retry_num=self.retry_num + 1)

chatmaild/src/chatmaild/tests/plugin.py

@@ -1,12 +1,14 @@
-import random
 import importlib.resources
 import itertools
-from email.parser import BytesParser
+import os
+import random
 from email import policy
-import pytest
+from email.parser import BytesParser
+from pathlib import Path
 
-from chatmaild.database import Database
+import pytest
 from chatmaild.config import read_config, write_initial_config
+from chatmaild.database import Database
 
 
 @pytest.fixture
@@ -57,7 +59,12 @@ def db(tmpdir):
 
 @pytest.fixture
 def maildata(request):
-    datadir = importlib.resources.files(__package__).joinpath("mail-data")
+    try:
+        datadir = importlib.resources.files(__package__).joinpath("mail-data")
+    except TypeError:
+        # in python3.9 or lower, the above doesn't work, so we get datadir this way:
+        datadir = Path(os.getcwd()).joinpath("chatmaild/src/chatmaild/tests/mail-data")
+
     assert datadir.exists(), datadir
 
     def maildata(name, from_addr, to_addr):

chatmaild/src/chatmaild/tests/test_config.py

@@ -24,7 +24,7 @@ def test_read_config_testrun(make_config):
     assert config.postfix_reinject_port == 10025
     assert config.max_user_send_per_minute == 60
     assert config.max_mailbox_size == "100M"
-    assert config.delete_mails_after == "40"
+    assert config.delete_mails_after == "20"
     assert config.username_min_length == 9
     assert config.username_max_length == 9
     assert config.password_min_length == 9

chatmaild/src/chatmaild/tests/test_doveauth.py

@@ -1,23 +1,23 @@
 import io
 import json
-import pytest
 import queue
 import threading
 import traceback
 
 import chatmaild.doveauth
+import pytest
+from chatmaild.database import DBError
 from chatmaild.doveauth import (
     get_user_data,
-    lookup_passdb,
-    handle_dovecot_request,
     handle_dovecot_protocol,
+    handle_dovecot_request,
+    lookup_passdb,
 )
-from chatmaild.database import DBError
 
 
 def test_basic(db, example_config):
     lookup_passdb(db, example_config, "asdf12345@chat.example.org", "q9mr3faue")
-    data = get_user_data(db, "asdf12345@chat.example.org")
+    data = get_user_data(db, example_config, "asdf12345@chat.example.org")
     assert data
     data2 = lookup_passdb(
         db, example_config, "asdf12345@chat.example.org", "q9mr3jewvadsfaue"
@@ -43,7 +43,7 @@ def test_nocreate_file(db, monkeypatch, tmpdir, example_config):
     lookup_passdb(
         db, example_config, "newuser12@chat.example.org", "zequ0Aimuchoodaechik"
     )
-    assert not get_user_data(db, "newuser12@chat.example.org")
+    assert not get_user_data(db, example_config, "newuser12@chat.example.org")
 
 
 def test_db_version(db):
@@ -75,6 +75,14 @@ def test_handle_dovecot_request(db, example_config):
     assert userdata["password"].startswith("{SHA512-CRYPT}")
 
 
+def test_handle_dovecot_protocol_hello_is_skipped(db, example_config, caplog):
+    rfile = io.BytesIO(b"H3\t2\t0\t\tauth\n")
+    wfile = io.BytesIO()
+    handle_dovecot_protocol(rfile, wfile, db, example_config)
+    assert wfile.getvalue() == b""
+    assert not caplog.messages
+
+
 def test_handle_dovecot_protocol(db, example_config):
     rfile = io.BytesIO(
         b"H3\t2\t0\t\tauth\nLshared/userdb/foobar@chat.example.org\tfoobar@chat.example.org\n"

chatmaild/src/chatmaild/tests/test_filedict.py

@@ -0,0 +1,19 @@
+from chatmaild.filedict import FileDict
+
+
+def test_basic(tmp_path):
+    fdict = FileDict(tmp_path.joinpath("metadata"))
+    assert fdict.read() == {}
+    with fdict.modify() as d:
+        d["devicetoken"] = [1, 2, 3]
+        d["456"] = 4.2
+    new = fdict.read()
+    assert new["devicetoken"] == [1, 2, 3]
+    assert new["456"] == 4.2
+
+
+def test_bad_marshal_file(tmp_path, caplog):
+    fdict1 = FileDict(tmp_path.joinpath("metadata"))
+    fdict1.path.write_bytes(b"l12k3l12k3l")
+    assert fdict1.read() == {}
+    assert "corrupt" in caplog.records[0].msg

chatmaild/src/chatmaild/tests/test_filtermail.py

@@ -1,12 +1,11 @@
+import pytest
 from chatmaild.filtermail import (
-    check_encrypted,
     BeforeQueueHandler,
     SendRateLimiter,
+    check_encrypted,
     check_mdn,
 )
 
-import pytest
-
 
 @pytest.fixture
 def maildomain():

chatmaild/src/chatmaild/tests/test_metadata.py

@@ -0,0 +1,312 @@
+import io
+import time
+
+import pytest
+import requests
+from chatmaild.metadata import (
+    Metadata,
+    handle_dovecot_protocol,
+    handle_dovecot_request,
+)
+from chatmaild.notifier import (
+    Notifier,
+    NotifyThread,
+    PersistentQueueItem,
+)
+
+
+@pytest.fixture
+def notifier(metadata):
+    queue_dir = metadata.vmail_dir.joinpath("pending_notifications")
+    queue_dir.mkdir()
+    return Notifier(queue_dir)
+
+
+@pytest.fixture
+def metadata(tmp_path):
+    vmail_dir = tmp_path.joinpath("vmaildir")
+    vmail_dir.mkdir()
+    return Metadata(vmail_dir)
+
+
+@pytest.fixture
+def testaddr():
+    return "user.name@example.org"
+
+
+@pytest.fixture
+def testaddr2():
+    return "user2@example.org"
+
+
+@pytest.fixture
+def token():
+    return "01234"
+
+
+def get_mocked_requests(statuslist):
+    class ReqMock:
+        requests = []
+
+        def post(self, url, data, timeout):
+            self.requests.append((url, data, timeout))
+            res = statuslist.pop(0)
+            if isinstance(res, Exception):
+                raise res
+
+            class Result:
+                status_code = res
+
+            return Result()
+
+    return ReqMock()
+
+
+def test_metadata_persistence(tmp_path, testaddr, testaddr2):
+    metadata1 = Metadata(tmp_path)
+    metadata2 = Metadata(tmp_path)
+    assert not metadata1.get_tokens_for_addr(testaddr)
+    assert not metadata2.get_tokens_for_addr(testaddr)
+
+    metadata1.add_token_to_addr(testaddr, "01234")
+    metadata1.add_token_to_addr(testaddr2, "456")
+    assert metadata2.get_tokens_for_addr(testaddr) == ["01234"]
+    assert metadata2.get_tokens_for_addr(testaddr2) == ["456"]
+    metadata2.remove_token_from_addr(testaddr, "01234")
+    assert not metadata1.get_tokens_for_addr(testaddr)
+    assert metadata1.get_tokens_for_addr(testaddr2) == ["456"]
+
+
+def test_remove_nonexisting(metadata, tmp_path, testaddr):
+    metadata.add_token_to_addr(testaddr, "123")
+    metadata.remove_token_from_addr(testaddr, "1l23k1l2k3")
+    assert metadata.get_tokens_for_addr(testaddr) == ["123"]
+
+
+def test_notifier_remove_without_set(metadata, testaddr):
+    metadata.remove_token_from_addr(testaddr, "123")
+    assert not metadata.get_tokens_for_addr(testaddr)
+
+
+def test_handle_dovecot_request_lookup_fails(notifier, metadata, testaddr):
+    res = handle_dovecot_request(
+        f"Lpriv/123/chatmail\t{testaddr}", {}, notifier, metadata
+    )
+    assert res == "N\n"
+
+
+def test_handle_dovecot_request_happy_path(notifier, metadata, testaddr, token):
+    transactions = {}
+
+    # set device token in a transaction
+    tx = "1111"
+    msg = f"B{tx}\t{testaddr}"
+    res = handle_dovecot_request(msg, transactions, notifier, metadata)
+    assert not res and not metadata.get_tokens_for_addr(testaddr)
+    assert transactions == {tx: dict(addr=testaddr, res="O\n")}
+
+    msg = f"S{tx}\tpriv/guid00/devicetoken\t{token}"
+    res = handle_dovecot_request(msg, transactions, notifier, metadata)
+    assert not res
+    assert len(transactions) == 1
+    assert metadata.get_tokens_for_addr(testaddr) == [token]
+
+    msg = f"C{tx}"
+    res = handle_dovecot_request(msg, transactions, notifier, metadata)
+    assert res == "O\n"
+    assert len(transactions) == 0
+    assert metadata.get_tokens_for_addr(testaddr) == [token]
+
+    # trigger notification for incoming message
+    tx2 = "2222"
+    assert (
+        handle_dovecot_request(f"B{tx2}\t{testaddr}", transactions, notifier, metadata)
+        is None
+    )
+    msg = f"S{tx2}\tpriv/guid00/messagenew"
+    assert handle_dovecot_request(msg, transactions, notifier, metadata) is None
+    queue_item = notifier.retry_queues[0].get()[1]
+    assert queue_item.token == token
+    assert handle_dovecot_request(f"C{tx2}", transactions, notifier, metadata) == "O\n"
+    assert not transactions
+    assert queue_item.path.exists()
+
+
+def test_handle_dovecot_protocol_set_devicetoken(metadata, notifier):
+    rfile = io.BytesIO(
+        b"\n".join(
+            [
+                b"HELLO",
+                b"Btx00\tuser@example.org",
+                b"Stx00\tpriv/guid00/devicetoken\t01234",
+                b"Ctx00",
+            ]
+        )
+    )
+    wfile = io.BytesIO()
+    handle_dovecot_protocol(rfile, wfile, notifier, metadata)
+    assert wfile.getvalue() == b"O\n"
+    assert metadata.get_tokens_for_addr("user@example.org") == ["01234"]
+
+
+def test_handle_dovecot_protocol_set_get_devicetoken(metadata, notifier):
+    rfile = io.BytesIO(
+        b"\n".join(
+            [
+                b"HELLO",
+                b"Btx00\tuser@example.org",
+                b"Stx00\tpriv/guid00/devicetoken\t01234",
+                b"Ctx00",
+            ]
+        )
+    )
+    wfile = io.BytesIO()
+    handle_dovecot_protocol(rfile, wfile, notifier, metadata)
+    assert metadata.get_tokens_for_addr("user@example.org") == ["01234"]
+    assert wfile.getvalue() == b"O\n"
+
+    rfile = io.BytesIO(
+        b"\n".join([b"HELLO", b"Lpriv/0123/devicetoken\tuser@example.org"])
+    )
+    wfile = io.BytesIO()
+    handle_dovecot_protocol(rfile, wfile, notifier, metadata)
+    assert wfile.getvalue() == b"O01234\n"
+
+
+def test_handle_dovecot_protocol_iterate(metadata, notifier):
+    rfile = io.BytesIO(
+        b"\n".join(
+            [
+                b"H",
+                b"I9\t0\tpriv/5cbe730f146fea6535be0d003dd4fc98/\tci-2dzsrs@nine.testrun.org",
+            ]
+        )
+    )
+    wfile = io.BytesIO()
+    handle_dovecot_protocol(rfile, wfile, notifier, metadata)
+    assert wfile.getvalue() == b"\n"
+
+
+def test_notifier_thread_deletes_persistent_file(metadata, notifier, testaddr):
+    reqmock = get_mocked_requests([200])
+    metadata.add_token_to_addr(testaddr, "01234")
+    notifier.new_message_for_addr(testaddr, metadata)
+    NotifyThread(notifier, 0, None).retry_one(reqmock)
+    url, data, timeout = reqmock.requests[0]
+    assert data == "01234"
+    assert metadata.get_tokens_for_addr(testaddr) == ["01234"]
+    notifier.requeue_persistent_queue_items()
+    assert notifier.retry_queues[0].qsize() == 0
+
+
+@pytest.mark.parametrize("status", [requests.exceptions.RequestException(), 404, 500])
+def test_notifier_thread_connection_failures(
+    metadata, notifier, testaddr, status, caplog
+):
+    """test that tokens keep getting retried until they are given up."""
+    metadata.add_token_to_addr(testaddr, "01234")
+    notifier.new_message_for_addr(testaddr, metadata)
+    notifier.NOTIFICATION_RETRY_DELAY = 5
+    max_tries = len(notifier.retry_queues)
+    for i in range(max_tries):
+        caplog.clear()
+        reqmock = get_mocked_requests([status])
+        sleep_calls = []
+        NotifyThread(notifier, i, None).retry_one(reqmock, sleep=sleep_calls.append)
+        assert notifier.retry_queues[i].qsize() == 0
+        assert "request failed" in caplog.records[0].msg
+        if i > 0:
+            assert len(sleep_calls) == 1
+        if i + 1 < max_tries:
+            assert notifier.retry_queues[i + 1].qsize() == 1
+            assert len(caplog.records) == 1
+        else:
+            assert len(caplog.records) == 2
+            assert "deadline" in caplog.records[1].msg
+    notifier.requeue_persistent_queue_items()
+    assert notifier.retry_queues[0].qsize() == 0
+
+
+def test_requeue_removes_tmp_files(notifier, metadata, testaddr, caplog):
+    metadata.add_token_to_addr(testaddr, "01234")
+    notifier.new_message_for_addr(testaddr, metadata)
+    p = notifier.queue_dir.joinpath("1203981203.tmp")
+    p.touch()
+    notifier2 = notifier.__class__(notifier.queue_dir)
+    notifier2.requeue_persistent_queue_items()
+    assert "spurious" in caplog.records[0].msg
+    assert not p.exists()
+    assert notifier2.retry_queues[0].qsize() == 1
+    when, queue_item = notifier2.retry_queues[0].get()
+    assert when <= int(time.time())
+    assert queue_item.addr == testaddr
+
+
+def test_start_and_stop_notification_threads(notifier, testaddr):
+    threads = notifier.start_notification_threads(None)
+    for retry_num, threadlist in threads.items():
+        for t in threadlist:
+            t.stop()
+            t.join()
+
+
+def test_multi_device_notifier(metadata, notifier, testaddr):
+    metadata.add_token_to_addr(testaddr, "01234")
+    metadata.add_token_to_addr(testaddr, "56789")
+    notifier.new_message_for_addr(testaddr, metadata)
+    reqmock = get_mocked_requests([200, 200])
+    NotifyThread(notifier, 0, None).retry_one(reqmock)
+    NotifyThread(notifier, 0, None).retry_one(reqmock)
+    assert notifier.retry_queues[0].qsize() == 0
+    assert notifier.retry_queues[1].qsize() == 0
+    url, data, timeout = reqmock.requests[0]
+    assert data == "01234"
+    url, data, timeout = reqmock.requests[1]
+    assert data == "56789"
+    assert metadata.get_tokens_for_addr(testaddr) == ["01234", "56789"]
+
+
+def test_notifier_thread_run_gone_removes_token(metadata, notifier, testaddr):
+    metadata.add_token_to_addr(testaddr, "01234")
+    metadata.add_token_to_addr(testaddr, "45678")
+    notifier.new_message_for_addr(testaddr, metadata)
+
+    reqmock = get_mocked_requests([410, 200])
+    NotifyThread(notifier, 0, metadata.remove_token_from_addr).retry_one(reqmock)
+    NotifyThread(notifier, 0, None).retry_one(reqmock)
+    url, data, timeout = reqmock.requests[0]
+    assert data == "01234"
+    url, data, timeout = reqmock.requests[1]
+    assert data == "45678"
+    assert metadata.get_tokens_for_addr(testaddr) == ["45678"]
+    assert notifier.retry_queues[0].qsize() == 0
+    assert notifier.retry_queues[1].qsize() == 0
+
+
+def test_persistent_queue_items(tmp_path, testaddr, token):
+    queue_item = PersistentQueueItem.create(tmp_path, testaddr, 432, token)
+    assert queue_item.addr == testaddr
+    assert queue_item.start_ts == 432
+    assert queue_item.token == token
+    item2 = PersistentQueueItem.read_from_path(queue_item.path)
+    assert item2.addr == testaddr
+    assert item2.start_ts == 432
+    assert item2.token == token
+    assert item2 == queue_item
+    item2.delete()
+    assert not item2.path.exists()
+    assert not queue_item < item2 and not item2 < queue_item
+
+
+def test_iroh_relay(metadata):
+    rfile = io.BytesIO(
+        b"\n".join(
+            [
+                b"H",
+                b"Lshared/0123/vendor/vendor.dovecot/pvt/server/vendor/deltachat/irohrelay\tuser@example.org",
+            ]
+        )
+    )
+    wfile = io.BytesIO()
+    handle_dovecot_protocol(rfile, wfile, notifier, metadata, "https://example.org/")
+    assert wfile.getvalue() == b"Ohttps://example.org/\n"

cmdeploy/pyproject.toml

@@ -16,9 +16,9 @@ dependencies = [
   "build",
   "tox",
   "ruff",
-  "black",
   "pytest",
   "pytest-xdist", 
+  "imap_tools",
 ]
 
 [project.scripts]
@@ -30,3 +30,13 @@ cmdeploy = "cmdeploy.cmdeploy:main"
 
 [tool.pytest.ini_options]
 addopts = "-v -ra --strict-markers"
+
+[tool.ruff]
+lint.select = [
+  "F", # Pyflakes
+  "I", # isort
+
+  "PLC", # Pylint Convention
+  "PLE", # Pylint Error
+  "PLW", # Pylint Warning
+]

cmdeploy/src/cmdeploy/__init__.py

@@ -2,21 +2,22 @@
 Chat Mail pyinfra deploy.
 """
 
-import sys
 import importlib.resources
-import subprocess
-import shutil
 import io
+import shutil
+import subprocess
+import sys
 from pathlib import Path
 
+from chatmaild.config import Config, read_config
 from pyinfra import host
-from pyinfra.operations import apt, files, server, systemd, pip
 from pyinfra.facts.files import File
 from pyinfra.facts.systemd import SystemdEnabled
+from pyinfra.operations import apt, files, pip, server, systemd
+
 from .acmetool import deploy_acmetool
 
-from chatmaild.config import read_config, Config
-
+root_owned = dict(user="root", group="root", mode="644")
 
 def _build_chatmaild(dist_dir) -> None:
     dist_dir = Path(dist_dir).resolve()
@@ -50,7 +51,6 @@ def _install_remote_venv_with_chatmaild(config) -> None:
     remote_dist_file = f"{remote_base_dir}/dist/{dist_file.name}"
     remote_venv_dir = f"{remote_base_dir}/venv"
     remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini"
-    root_owned = dict(user="root", group="root", mode="644")
 
     apt.packages(
         name="apt install python3-virtualenv",
@@ -85,9 +85,19 @@ def _install_remote_venv_with_chatmaild(config) -> None:
         ],
     )
 
+    # create metrics every 5 minutes via systemd
+
+    files.put(
+        name="Upload metrics.timer",
+        src=importlib.resources.files(__package__).joinpath("service/metrics.timer"),
+        dest=f"/etc/systemd/system/metrics.timer",
+        **root_owned,
+    )
+
     files.template(
-        src=importlib.resources.files(__package__).joinpath("metrics.cron.j2"),
-        dest="/etc/cron.d/chatmail-metrics",
+        name="upload metrics.service",
+        src=importlib.resources.files(__package__).joinpath("service/metrics.service.j2"),
+        dest="/etc/systemd/system/metrics.service",
         user="root",
         group="root",
         mode="644",
@@ -97,18 +107,31 @@ def _install_remote_venv_with_chatmaild(config) -> None:
         },
     )
 
+    systemd.service(
+        name=f"Setup metrics timer",
+        service="metrics.timer",
+        running=True,
+        enabled=True,
+        restarted=True,
+        daemon_reload=True,
+    )
+
     # install systemd units
     for fn in (
         "doveauth",
         "filtermail",
         "echobot",
+        "chatmail-metadata",
     ):
         params = dict(
             execpath=f"{remote_venv_dir}/bin/{fn}",
             config_path=remote_chatmail_inipath,
             remote_venv_dir=remote_venv_dir,
+            mail_domain=config.mail_domain,
+        )
+        source_path = importlib.resources.files(__package__).joinpath(
+            "service", f"{fn}.service.f"
         )
-        source_path = importlib.resources.files("chatmaild").joinpath(f"{fn}.service.f")
         content = source_path.read_text().format(**params).encode()
 
         files.put(
@@ -131,20 +154,6 @@ def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
     """Configures OpenDKIM"""
     need_restart = False
 
-    server.group(name="Create opendkim group", group="opendkim", system=True)
-    server.user(
-        name="Create opendkim user",
-        user="opendkim",
-        groups=["opendkim"],
-        system=True,
-    )
-    server.user(
-        name="Add postfix user to opendkim group for socket access",
-        user="postfix",
-        groups=["opendkim"],
-        system=True,
-    )
-
     main_config = files.template(
         src=importlib.resources.files(__package__).joinpath("opendkim/opendkim.conf"),
         dest="/etc/opendkim.conf",
@@ -337,6 +346,47 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
         mode="644",
     )
     need_restart |= auth_config.changed
+    lua_push_notification_script = files.put(
+        src=importlib.resources.files(__package__).joinpath(
+            "dovecot/push_notification.lua"
+        ),
+        dest="/etc/dovecot/push_notification.lua",
+        user="root",
+        group="root",
+        mode="644",
+    )
+    need_restart |= lua_push_notification_script.changed
+
+    sieve_script = files.put(
+        src=importlib.resources.files(__package__).joinpath("dovecot/default.sieve"),
+        dest="/etc/dovecot/default.sieve",
+        user="root",
+        group="root",
+        mode="644",
+    )
+    need_restart |= sieve_script.changed
+    if sieve_script.changed:
+        server.shell(
+            name="compile sieve script",
+            commands=["/usr/bin/sievec /etc/dovecot/default.sieve"],
+        )
+
+    files.template(
+        src=importlib.resources.files(__package__).joinpath("service/expunge.service.j2"),
+        dest="/etc/systemd/system/expunge.service",
+        config={
+            "mail_domain": config.mail_domain,
+            "delete_mails_after": config.delete_mails_after,
+        },
+        **root_owned,
+    )
+
+    files.put(
+        name="Upload expunge.timer",
+        src=importlib.resources.files(__package__).joinpath("service/expunge.timer"),
+        dest=f"/etc/systemd/system/expunge.timer",
+        **root_owned,
+    )
 
     files.template(
         src=importlib.resources.files(__package__).joinpath("dovecot/expunge.cron.j2"),
@@ -426,8 +476,11 @@ def check_config(config):
     mail_domain = config.mail_domain
     if mail_domain != "testrun.org" and not mail_domain.endswith(".testrun.org"):
         blocked_words = "merlinux schmieder testrun.org".split()
-        for value in config.__dict__.values():
-            if any(x in str(value) for x in blocked_words):
+        for key in config.__dict__:
+            value = config.__dict__[key]
+            if key.startswith("privacy") and any(
+                x in str(value) for x in blocked_words
+            ):
                 raise ValueError(
                     f"please set your own privacy contacts/addresses in {config._inipath}"
                 )
@@ -445,9 +498,52 @@ def deploy_chatmail(config_path: Path) -> None:
 
     from .www import build_webpages
 
-    apt.update(name="apt update", cache_time=24 * 3600)
     server.group(name="Create vmail group", group="vmail", system=True)
     server.user(name="Create vmail user", user="vmail", group="vmail", system=True)
+    server.group(name="Create opendkim group", group="opendkim", system=True)
+    server.user(
+        name="Create opendkim user",
+        user="opendkim",
+        groups=["opendkim"],
+        system=True,
+    )
+    server.user(
+        name="Add postfix user to opendkim group for socket access",
+        user="postfix",
+        groups=["opendkim"],
+        system=True,
+    )
+    server.user(name="Create echobot user", user="echobot", system=True)
+
+    server.shell(
+        name="Fix file owner in /home/vmail",
+        commands=["test -d /home/vmail && chown -R vmail:vmail /home/vmail"],
+    )
+
+    # Add our OBS repository for dovecot_no_delay
+    files.put(
+        name="Add Deltachat OBS GPG key to apt keyring",
+        src=importlib.resources.files(__package__).joinpath("obs-home-deltachat.gpg"),
+        dest="/etc/apt/keyrings/obs-home-deltachat.gpg",
+        user="root",
+        group="root",
+        mode="644",
+    )
+
+    files.line(
+        name="Add DeltaChat OBS home repository to sources.list",
+        path="/etc/apt/sources.list",
+        line="deb [signed-by=/etc/apt/keyrings/obs-home-deltachat.gpg] https://download.opensuse.org/repositories/home:/deltachat/Debian_12/ ./",
+        escape_regex_characters=True,
+        ensure_newline=True,
+    )
+
+    apt.update(name="apt update", cache_time=24 * 3600)
+
+    apt.packages(
+        name="Install rsync",
+        packages=["rsync"],
+    )
 
     # Run local DNS resolver `unbound`.
     # `resolvconf` takes care of setting up /etc/resolv.conf
@@ -463,6 +559,7 @@ def deploy_chatmail(config_path: Path) -> None:
             "systemctl reset-failed unbound.service",
         ],
     )
+
     systemd.service(
         name="Start and enable unbound",
         service="unbound.service",
@@ -472,10 +569,15 @@ def deploy_chatmail(config_path: Path) -> None:
 
     # Deploy acmetool to have TLS certificates.
     deploy_acmetool(
-        nginx_hook=True,
         domains=[mail_domain, f"mta-sts.{mail_domain}", f"www.{mail_domain}"],
     )
 
+    apt.packages(
+        # required for setfacl for echobot
+        name="Install acl",
+        packages="acl",
+    )
+
     apt.packages(
         name="Install Postfix",
         packages="postfix",
@@ -483,7 +585,7 @@ def deploy_chatmail(config_path: Path) -> None:
 
     apt.packages(
         name="Install Dovecot",
-        packages=["dovecot-imapd", "dovecot-lmtpd"],
+        packages=["dovecot-imapd", "dovecot-lmtpd", "dovecot-sieve"],
     )
 
     apt.packages(
@@ -530,14 +632,9 @@ def deploy_chatmail(config_path: Path) -> None:
         restarted=mta_sts_need_restart,
     )
 
-    systemd.service(
-        name="Start and enable Postfix",
-        service="postfix.service",
-        running=True,
-        enabled=True,
-        restarted=postfix_need_restart,
-    )
-
+    # Dovecot should be started before Postfix
+    # because it creates authentication socket
+    # required by Postfix.
     systemd.service(
         name="Start and enable Dovecot",
         service="dovecot.service",
@@ -546,6 +643,14 @@ def deploy_chatmail(config_path: Path) -> None:
         restarted=dovecot_need_restart,
     )
 
+    systemd.service(
+        name="Start and enable Postfix",
+        service="postfix.service",
+        running=True,
+        enabled=True,
+        restarted=postfix_need_restart,
+    )
+
     systemd.service(
         name="Start and enable nginx",
         service="nginx.service",

cmdeploy/src/cmdeploy/acmetool/__init__.py

@@ -1,11 +1,11 @@
 import importlib.resources
 
-from pyinfra.operations import apt, files, systemd, server
 from pyinfra import host
 from pyinfra.facts.systemd import SystemdStatus
+from pyinfra.operations import apt, files, server, systemd
 
 
-def deploy_acmetool(nginx_hook=False, email="", domains=[]):
+def deploy_acmetool(email="", domains=[]):
     """Deploy acmetool."""
     apt.packages(
         name="Install acmetool",
@@ -20,16 +20,13 @@ def deploy_acmetool(nginx_hook=False, email="", domains=[]):
         mode="644",
     )
 
-    if nginx_hook:
-        files.put(
-            src=importlib.resources.files(__package__)
-            .joinpath("acmetool.hook")
-            .open("rb"),
-            dest="/usr/lib/acme/hooks/nginx",
-            user="root",
-            group="root",
-            mode="744",
-        )
+    files.put(
+        src=importlib.resources.files(__package__).joinpath("acmetool.hook").open("rb"),
+        dest="/usr/lib/acme/hooks/nginx",
+        user="root",
+        group="root",
+        mode="744",
+    )
 
     files.template(
         src=importlib.resources.files(__package__).joinpath("response-file.yaml.j2"),
@@ -72,7 +69,8 @@ def deploy_acmetool(nginx_hook=False, email="", domains=[]):
         restarted=service_file.changed,
     )
 
-    server.shell(
-        name=f"Request certificate for: { ', '.join(domains) }",
-        commands=[f"acmetool want { ' '.join(domains)}"],
-    )
+    if str(host) != "staging.testrun.org":
+        server.shell(
+            name=f"Request certificate for: { ', '.join(domains) }",
+            commands=[f"acmetool want --xlog.severity=debug { ' '.join(domains)}"],
+        )

cmdeploy/src/cmdeploy/acmetool/acmetool.cron

@@ -1,4 +1,4 @@
 SHELL=/bin/sh
 PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
 MAILTO=root
-20 16 * * * root /usr/bin/acmetool --batch reconcile
+20 16 * * * root /usr/bin/acmetool --batch reconcile && systemctl reload dovecot && systemctl reload postfix

cmdeploy/src/cmdeploy/acmetool/acmetool.hook

@@ -3,3 +3,5 @@ set -e
 EVENT_NAME="$1"
 [ "$EVENT_NAME" = "live-updated" ] || exit 42
 systemctl restart nginx.service
+systemctl reload dovecot.service
+systemctl reload postfix.service

cmdeploy/src/cmdeploy/acmetool/response-file.yaml.j2

@@ -1,2 +1,2 @@
 "acme-enter-email": "{{ email }}"
-"acme-agreement:https://letsencrypt.org/documents/LE-SA-v1.3-September-21-2022.pdf": true
+"acme-agreement:https://letsencrypt.org/documents/LE-SA-v1.4-April-3-2024.pdf": true

cmdeploy/src/cmdeploy/chatmail.zone.f

@@ -6,7 +6,7 @@ _submissions._tcp.{chatmail_domain}. SRV 0 1 465 {chatmail_domain}.
 _imap._tcp.{chatmail_domain}.        SRV 0 1 143 {chatmail_domain}.
 _imaps._tcp.{chatmail_domain}.       SRV 0 1 993 {chatmail_domain}.
 {chatmail_domain}.                   CAA 128 issue "letsencrypt.org;accounturi={acme_account_url}"
-{chatmail_domain}.                   TXT "v=spf1 a:{chatmail_domain} -all"
+{chatmail_domain}.                   TXT "v=spf1 a:{chatmail_domain} ~all"
 _dmarc.{chatmail_domain}.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
 _mta-sts.{chatmail_domain}.          TXT "v=STSv1; id={sts_id}"
 mta-sts.{chatmail_domain}.           CNAME {chatmail_domain}.

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -4,19 +4,18 @@ along with command line option and subcommand parsing.
 """
 
 import argparse
-import shutil
-import subprocess
 import importlib.resources
 import importlib.util
 import os
+import shutil
+import subprocess
 import sys
 from pathlib import Path
 
-
-from termcolor import colored
 from chatmaild.config import read_config, write_initial_config
-from cmdeploy.dns import show_dns, check_necessary_dns
+from termcolor import colored
 
+from cmdeploy.dns import check_necessary_dns, show_dns
 
 #
 # cmdeploy sub commands and options
@@ -157,26 +156,26 @@ def fmt_cmd_options(parser):
 
 
 def fmt_cmd(args, out):
-    """Run formattting fixes (ruff and black) on all chatmail source code."""
+    """Run formattting fixes on all chatmail source code."""
 
     sources = [str(importlib.resources.files(x)) for x in ("chatmaild", "cmdeploy")]
-    black_args = [shutil.which("black")]
-    ruff_args = [shutil.which("ruff")]
+    format_args = [shutil.which("ruff"), "format"]
+    check_args = [shutil.which("ruff"), "check"]
 
     if args.check:
-        black_args.append("--check")
+        format_args.append("--diff")
     else:
-        ruff_args.append("--fix")
+        check_args.append("--fix")
 
     if not args.verbose:
-        black_args.append("-q")
-        ruff_args.append("-q")
+        check_args.append("--quiet")
+        format_args.append("--quiet")
 
-    black_args.extend(sources)
-    ruff_args.extend(sources)
+    format_args.extend(sources)
+    check_args.extend(sources)
 
-    out.check_call(" ".join(black_args), quiet=not args.verbose)
-    out.check_call(" ".join(ruff_args), quiet=not args.verbose)
+    out.check_call(" ".join(format_args), quiet=not args.verbose)
+    out.check_call(" ".join(check_args), quiet=not args.verbose)
     return 0
 
 
@@ -232,7 +231,7 @@ class Out:
         if not quiet:
             cmdstring = " ".join(args)
             self(f"[$ {cmdstring}]", file=sys.stderr)
-        proc = subprocess.run(args, env=env)
+        proc = subprocess.run(args, env=env, check=False)
         return proc.returncode
 
 

cmdeploy/src/cmdeploy/deploy.py

@@ -1,6 +1,8 @@
-import os
 import importlib.resources
+import os
+
 import pyinfra
+
 from cmdeploy import deploy_chatmail
 
 

cmdeploy/src/cmdeploy/dns.py

@@ -1,9 +1,9 @@
+import datetime
+import importlib
+import subprocess
 import sys
 
 import requests
-import importlib
-import subprocess
-import datetime
 
 
 class DNS:
@@ -11,6 +11,11 @@ class DNS:
         self.session = requests.Session()
         self.out = out
         self.ssh = f"ssh root@{mail_domain} -- "
+        self.out.shell_output(
+            f"{ self.ssh }'apt-get update && apt-get install -y dnsutils'",
+            timeout=60,
+            no_print=True,
+        )
         try:
             self.shell(f"unbound-control flush_zone {mail_domain}")
         except subprocess.CalledProcessError:
@@ -34,12 +39,11 @@ class DNS:
         cmd = "ip a | grep inet6 | grep 'scope global' | sed -e 's#/64 scope global##' | sed -e 's#inet6##'"
         return self.shell(cmd).strip()
 
-    def get(self, typ: str, domain: str) -> str | None:
-        """Get a DNS entry"""
+    def get(self, typ: str, domain: str) -> str:
+        """Get a DNS entry or empty string if there is none."""
         dig_result = self.shell(f"dig -r -q {domain} -t {typ} +short")
         line = dig_result.partition("\n")[0]
-        if line:
-            return line
+        return line
 
     def check_ptr_record(self, ip: str, mail_domain) -> bool:
         """Check the PTR record for an IPv4 or IPv6 address."""
@@ -54,22 +58,25 @@ def show_dns(args, out) -> int:
     ssh = f"ssh root@{mail_domain}"
     dns = DNS(out, mail_domain)
 
-    def read_dkim_entries(entry):
-        lines = []
-        for line in entry.split("\n"):
-            if line.startswith(";") or not line.strip():
-                continue
-            line = line.replace("\t", " ")
-            lines.append(line)
-        return "\n".join(lines)
-
     print("Checking your DKIM keys and DNS entries...")
     try:
         acme_account_url = out.shell_output(f"{ssh} -- acmetool account-url")
     except subprocess.CalledProcessError:
         print("Please run `cmdeploy run` first.")
         return 1
-    dkim_entry = read_dkim_entries(out.shell_output(f"{ssh} -- opendkim-genzone -F"))
+
+    dkim_selector = "opendkim"
+    dkim_pubkey = out.shell_output(
+        ssh + f" -- openssl rsa -in /etc/dkimkeys/{dkim_selector}.private"
+        " -pubout 2>/dev/null | awk '/-/{next}{printf(\"%s\",$0)}'"
+    )
+    dkim_entry_value = f"v=DKIM1;k=rsa;p={dkim_pubkey};s=email;t=s"
+    dkim_entry_str = ""
+    while len(dkim_entry_value) >= 255:
+        dkim_entry_str += '"' + dkim_entry_value[:255] + '" '
+        dkim_entry_value = dkim_entry_value[255:]
+    dkim_entry_str += '"' + dkim_entry_value + '"'
+    dkim_entry = f"{dkim_selector}._domainkey.{mail_domain}. TXT {dkim_entry_str}"
 
     ipv6 = dns.get_ipv6()
     reverse_ipv6 = dns.check_ptr_record(ipv6, mail_domain)
@@ -97,9 +104,8 @@ def show_dns(args, out) -> int:
             return 0
         except TypeError:
             pass
-        started_dkim_parsing = False
-        for line in zonefile.splitlines():
-            line = line.format(
+        for raw_line in zonefile.splitlines():
+            line = raw_line.format(
                 acme_account_url=acme_account_url,
                 sts_id=datetime.datetime.now().strftime("%Y%m%d%H%M"),
                 chatmail_domain=args.config.mail_domain,
@@ -124,28 +130,23 @@ def show_dns(args, out) -> int:
                 current = dns.get("SRV", domain[:-1])
                 if current != f"{prio} {weight} {port} {value}":
                     to_print.append(line)
-            if "  TXT " in line:
+            if " TXT " in line:
                 domain, value = line.split(" TXT ")
                 current = dns.get("TXT", domain.strip()[:-1])
                 if domain.startswith("_mta-sts."):
                     if current:
                         if current.split("id=")[0] == value.split("id=")[0]:
                             continue
-                if current != value:
+
+                # TXT records longer than 255 bytes
+                # are split into multiple <character-string>s.
+                # This typically happens with DKIM record
+                # which contains long RSA key.
+                #
+                # Removing `" "` before comparison
+                # to get back a single string.
+                if current.replace('" "', "") != value.replace('" "', ""):
                     to_print.append(line)
-            if "IN TXT ( " in line:
-                started_dkim_parsing = True
-                dkim_lines = [line]
-            if started_dkim_parsing and line.startswith('"'):
-                dkim_lines.append(" " + line)
-        domain, data = "\n".join(dkim_lines).split(" IN TXT ")
-        current = dns.get("TXT", domain.strip()[:-1])
-        if current:
-            current = "( %s )" % (current.replace('" "', '"\n "'))
-            if current.replace(";", "\\;") != data:
-                to_print.append(dkim_entry)
-        else:
-            to_print.append(dkim_entry)
 
     exit_code = 0
     if to_print:

cmdeploy/src/cmdeploy/dovecot/auth.conf

@@ -1,4 +1,4 @@
-uri = proxy:/run/dovecot/doveauth.socket:auth
+uri = proxy:/run/doveauth/doveauth.socket:auth
 iterate_disable = yes
 default_pass_scheme = plain
 # %E escapes characters " (double quote), ' (single quote) and \ (backslash) with \ (backslash).

cmdeploy/src/cmdeploy/dovecot/default.sieve

@@ -0,0 +1,7 @@
+require ["imap4flags"];
+
+# flag the message so it doesn't cause a push notification
+
+if header :is ["Auto-Submitted"] ["auto-replied", "auto-generated"] {
+	addflag "$Auto";
+}

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -13,6 +13,12 @@ auth_cache_size = 100M
 mail_debug = yes
 {% endif %}
 
+# Prevent warnings similar to:
+#   config: Warning: service auth { client_limit=1000 } is lower than required under max. load (10200). Counted for protocol services with service_count != 1: service lmtp { process_limit=100 } + service imap-urlauth-login { process_limit=100 } + service imap-login { process_limit=10000 }
+#   config: Warning: service anvil { client_limit=1000 } is lower than required under max. load (10103). Counted with: service imap-urlauth-login { process_limit=100 } + service imap-login { process_limit=10000 } + service auth { process_limit=1 }
+#   master: Warning: service(stats): client_limit (1000) reached, client connections are being dropped
+default_client_limit = 20000
+
 mail_server_admin = mailto:root@{{ config.mail_domain }}
 mail_server_comment = Chatmail server
 
@@ -21,7 +27,7 @@ mail_plugins = quota
 # these are the capabilities Delta Chat cares about actually 
 # so let's keep the network overhead per login small
 # https://github.com/deltachat/deltachat-core-rust/blob/master/src/imap/capabilities.rs
-imap_capability = IMAP4rev1 IDLE MOVE QUOTA CONDSTORE NOTIFY METADATA
+imap_capability = IMAP4rev1 IDLE MOVE QUOTA CONDSTORE NOTIFY METADATA XDELTAPUSH XCHATMAIL
 
 
 # Authentication for system users.
@@ -71,6 +77,9 @@ mail_privileged_group = vmail
 ## Mail processes
 ##
 
+# Pass all IMAP METADATA requests to the server implementing Dovecot's dict protocol.
+mail_attribute_dict = proxy:/run/chatmail-metadata/metadata.socket:metadata
+
 # Enable IMAP COMPRESS (RFC 4978).
 # <https://datatracker.ietf.org/doc/html/rfc4978.html>
 protocol imap {
@@ -79,7 +88,21 @@ protocol imap {
 }
 
 protocol lmtp {
-  mail_plugins = $mail_plugins quota
+  # quota plugin documentation:
+  # <https://doc.dovecot.org/configuration_manual/quota_plugin/>
+  #
+  # notify plugin is a dependency of push_notification plugin:
+  # <https://doc.dovecot.org/settings/plugin/notify-plugin/>
+  #
+  # push_notification plugin documentation:
+  # <https://doc.dovecot.org/configuration_manual/push_notification/>
+  #
+  # mail_lua and push_notification_lua are needed for Lua push notification handler.
+  # <https://doc.dovecot.org/configuration_manual/push_notification/#configuration>
+  #
+  # Sieve to mark messages that should not be notified as \Seen
+  # <https://doc.dovecot.org/configuration_manual/sieve/configuration/>
+  mail_plugins = $mail_plugins quota mail_lua notify push_notification push_notification_lua sieve
 }
 
 plugin {
@@ -95,7 +118,15 @@ plugin {
   # quota_over_flag_value = TRUE
 }
 
+# push_notification configuration
+plugin {
+  # <https://doc.dovecot.org/configuration_manual/push_notification/#lua-lua>
+  push_notification_driver = lua:file=/etc/dovecot/push_notification.lua
+}
 
+plugin {
+  sieve_default = file:/etc/dovecot/default.sieve
+}
 
 service lmtp {
   user=vmail

cmdeploy/src/cmdeploy/dovecot/expunge.cron.j2

@@ -8,3 +8,4 @@
 # or only temporary (but then they shouldn't be around after {{ config.delete_mails_after }} days anyway).
 2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+3 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -name 'maildirsize' -type f -delete

cmdeploy/src/cmdeploy/dovecot/push_notification.lua

@@ -0,0 +1,32 @@
+function dovecot_lua_notify_begin_txn(user)
+  return user
+end
+
+function contains(v, needle)
+  for _, keyword in ipairs(v) do
+    if keyword == needle then
+      return true
+    end
+  end
+  return false
+end
+
+function dovecot_lua_notify_event_message_new(user, event)
+  local mbox = user:mailbox(event.mailbox)
+  mbox:sync()
+
+  if user.username ~= event.from_address then
+    -- Incoming message
+    if not contains(event.keywords, "$Auto") then
+      -- Not an Auto-Submitted message, notifying.
+
+      -- Notify METADATA server about new message.
+      mbox:metadata_set("/private/messagenew", "")
+    end
+  end
+
+  mbox:free()
+end
+
+function dovecot_lua_notify_end_txn(ctx, success)
+end

cmdeploy/src/cmdeploy/genqr.py

@@ -1,8 +1,9 @@
 import importlib
-import qrcode
-import os
-from PIL import ImageFont, ImageDraw, Image
 import io
+import os
+
+import qrcode
+from PIL import Image, ImageDraw, ImageFont
 
 
 def gen_qr_png_data(maildomain):

cmdeploy/src/cmdeploy/service/chatmail-metadata.service.f

@@ -0,0 +1,12 @@
+[Unit]
+Description=Chatmail dict proxy for IMAP METADATA
+
+[Service]
+ExecStart={execpath} /run/chatmail-metadata/metadata.socket /home/vmail/mail/{mail_domain} {config_path}
+Restart=always
+RestartSec=30
+User=vmail
+RuntimeDirectory=chatmail-metadata
+
+[Install]
+WantedBy=multi-user.target

cmdeploy/src/cmdeploy/service/doveauth.service.f

@@ -2,9 +2,11 @@
 Description=Chatmail dict authentication proxy for dovecot
 
 [Service]
-ExecStart={execpath} /run/dovecot/doveauth.socket vmail /home/vmail/passdb.sqlite {config_path}
+ExecStart={execpath} /run/doveauth/doveauth.socket /home/vmail/passdb.sqlite {config_path}
 Restart=always
 RestartSec=30
+User=vmail
+RuntimeDirectory=doveauth
 
 [Install]
 WantedBy=multi-user.target

cmdeploy/src/cmdeploy/service/echobot.service.f

@@ -0,0 +1,67 @@
+[Unit]
+Description=Chatmail echo bot for testing it works
+
+[Service]
+ExecStart={execpath} {config_path}
+Environment="PATH={remote_venv_dir}:$PATH"
+Restart=always
+RestartSec=30
+
+User=echobot
+Group=echobot
+
+# Create /var/lib/echobot
+StateDirectory=echobot
+
+# Create /run/echobot
+#
+# echobot stores /run/echobot/password
+# with a password there, which doveauth then reads.
+RuntimeDirectory=echobot
+
+WorkingDirectory=/var/lib/echobot
+
+# Apply security restrictions suggested by
+#   systemd-analyze security echobot.service
+CapabilityBoundingSet=
+LockPersonality=true
+MemoryDenyWriteExecute=true
+NoNewPrivileges=true
+PrivateDevices=true
+PrivateMounts=true
+PrivateTmp=true
+
+# We need to know about doveauth user to give it access to /run/echobot/password
+PrivateUsers=false
+
+ProtectClock=true
+ProtectControlGroups=true
+ProtectHostname=true
+ProtectKernelLogs=true
+ProtectKernelModules=true
+ProtectKernelTunables=true
+ProtectProc=noaccess
+
+# Should be "strict", but we currently write /accounts folder in a protected path
+ProtectSystem=full
+
+RemoveIPC=true
+RestrictAddressFamilies=AF_INET AF_INET6
+RestrictNamespaces=true
+RestrictRealtime=true
+RestrictSUIDSGID=true
+SystemCallArchitectures=native
+SystemCallFilter=~@clock
+SystemCallFilter=~@cpu-emulation
+SystemCallFilter=~@debug
+SystemCallFilter=~@module
+SystemCallFilter=~@mount
+SystemCallFilter=~@obsolete
+SystemCallFilter=~@raw-io
+SystemCallFilter=~@reboot
+SystemCallFilter=~@resources
+SystemCallFilter=~@swap
+UMask=0077
+
+[Install]
+WantedBy=multi-user.target

cmdeploy/src/cmdeploy/service/expunge.service.j2

@@ -0,0 +1,16 @@
+[Unit]
+Description=Expunge old mails after {{ config.delete_mails_after }} days
+
+[Service]
+Type=oneshot
+# delete all mails after {{ config.delete_mails_after }} days, in the Inbox
+ExecStart=/home/vmail/mail/{{ config.mail_domain }} -path '*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+# or in any IMAP subfolder
+ExecStart=vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+# even if they are unseen
+ExecStart=vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+ExecStart=vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+# or only temporary (but then they shouldn't be around after {{ config.delete_mails_after }} days anyway).
+ExecStart=vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+ExecStart=vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+ExecStart=vmail find /home/vmail/mail/{{ config.mail_domain }} -name 'maildirsize' -type f -delete

cmdeploy/src/cmdeploy/service/expunge.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Run expunge.service daily
+
+[Timer]
+OnCalendar=weekly
+Persistent=true
+
+[Install]
+WantedBy=timers.target

cmdeploy/src/cmdeploy/service/filtermail.service.f

@@ -1,5 +1,5 @@
 [Unit]
-Description=Chatmail Postfix BeforeQeue filter 
+Description=Chatmail Postfix before queue filter
 
 [Service]
 ExecStart={execpath} {config_path}

cmdeploy/src/cmdeploy/service/metrics.service.j2

@@ -0,0 +1,5 @@
+[Unit]
+Description=Generate metrics in /var/www/html/metrics
+
+[Service]
+ExecStart={{ config.execpath }} /home/vmail/mail/{{ config.mail_domain }} > /var/www/html/metrics

cmdeploy/src/cmdeploy/service/metrics.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Run metrics.service every 5 minutes
+
+[Timer]
+OnBootSec=5min
+OnUnitActiveSec=5min
+
+[Install]
+WantedBy=timers.target

cmdeploy/src/cmdeploy/tests/online/test_0_login.py

@@ -1,8 +1,10 @@
-import pytest
-import threading
 import queue
+import socket
+import threading
 
+import pytest
 from chatmaild.config import read_config
+
 from cmdeploy.cmdeploy import main
 
 
@@ -13,6 +15,13 @@ def test_init(tmp_path, maildomain):
     assert config.mail_domain == maildomain
 
 
+def test_capabilities(imap):
+    imap.connect()
+    capas = imap.conn.capabilities
+    assert "XCHATMAIL" in capas
+    assert "XDELTAPUSH" in capas
+
+
 def test_login_basic_functioning(imap_or_smtp, gencreds, lp):
     """Test a) that an initial login creates a user automatically
     and b) verify we can also login a second time with the same password
@@ -78,3 +87,24 @@ def test_concurrent_logins_same_account(
 
     for _ in conns:
         assert login_results.get()
+
+
+def test_no_vrfy(chatmail_config):
+    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    sock.connect((chatmail_config.mail_domain, 25))
+    banner = sock.recv(1024)
+    print(banner)
+    sock.send(b"VRFY wrongaddress@%s\r\n" % (chatmail_config.mail_domain.encode(),))
+    result = sock.recv(1024)
+    print(result)
+    sock.send(b"VRFY echo@%s\r\n" % (chatmail_config.mail_domain.encode(),))
+    result2 = sock.recv(1024)
+    print(result2)
+    assert result[0:10] == result2[0:10]
+    sock.send(b"VRFY wrongaddress\r\n")
+    result = sock.recv(1024)
+    print(result)
+    sock.send(b"VRFY echo\r\n")
+    result2 = sock.recv(1024)
+    print(result2)
+    assert result[0:10] == result2[0:10] == b"252 2.0.0 "

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -1,4 +1,5 @@
 import smtplib
+
 import pytest
 
 

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -1,10 +1,50 @@
-import time
-import re
+import ipaddress
 import random
+import re
+import time
 
+import imap_tools
 import pytest
 import requests
-import ipaddress
+
+
+@pytest.fixture
+def imap_mailbox(cmfactory):
+    (ac1,) = cmfactory.get_online_accounts(1)
+    user = ac1.get_config("addr")
+    password = ac1.get_config("mail_pw")
+    mailbox = imap_tools.MailBox(user.split("@")[1])
+    mailbox.login(user, password)
+    return mailbox
+
+
+class TestMetadataTokens:
+    "Tests that use Metadata extension for storing tokens"
+
+    def test_set_get_metadata(self, imap_mailbox):
+        "set and get metadata token for an account"
+        client = imap_mailbox.client
+        client.send(b'a01 SETMETADATA INBOX (/private/devicetoken "1111" )\n')
+        res = client.readline()
+        assert b"OK Setmetadata completed" in res
+
+        client.send(b"a02 GETMETADATA INBOX /private/devicetoken\n")
+        res = client.readline()
+        assert res[:1] == b"*"
+        res = client.readline().strip().rstrip(b")")
+        assert res == b"1111"
+        assert b"Getmetadata completed" in client.readline()
+
+        client.send(b'a01 SETMETADATA INBOX (/private/devicetoken "2222" )\n')
+        res = client.readline()
+        assert b"OK Setmetadata completed" in res
+
+        client.send(b"a02 GETMETADATA INBOX /private/devicetoken\n")
+        res = client.readline()
+        assert res[:1] == b"*"
+        res = client.readline().strip().rstrip(b")")
+        assert res == b"1111 2222"
+        assert b"Getmetadata completed" in client.readline()
 
 
 class TestEndToEndDeltaChat:
@@ -63,7 +103,7 @@ class TestEndToEndDeltaChat:
 
         addr = ac2.get_config("addr").lower()
         saved_ok = 0
-        for line in remote.iter_output("journalctl -f -u dovecot"):
+        for line in remote.iter_output("journalctl -n0 -f -u dovecot"):
             if addr not in line:
                 # print(line)
                 continue
@@ -75,7 +115,10 @@ class TestEndToEndDeltaChat:
                         )
                     lp.indent("good, message sending failed because quota was exceeded")
                     return
-            if "saved mail to inbox" in line:
+            if (
+                "stored mail into mailbox 'inbox'" in line
+                or "saved mail to inbox" in line
+            ):
                 saved_ok += 1
                 print(f"{saved_ok}: {line}")
                 if saved_ok >= num_to_send:
@@ -112,7 +155,7 @@ class TestEndToEndDeltaChat:
         lp.sec("ac1 sends a message and ac2 marks it as seen")
         chat = ac1.create_chat(ac2)
         msg = chat.send_text("hi")
-        m = ac2.wait_next_incoming_message()
+        m = ac2._evtracker.wait_next_incoming_message()
         m.mark_seen()
         # we can only indirectly wait for mark-seen to cause an smtp-error
         lp.sec("try to wait for markseen to complete and check error states")
@@ -132,7 +175,7 @@ def test_hide_senders_ip_address(cmfactory):
     chat = cmfactory.get_accepted_chat(user1, user2)
 
     chat.send_text("testing submission header cleanup")
-    user2.wait_next_incoming_message()
+    user2._evtracker.wait_next_incoming_message()
     user2.direct_imap.select_folder("Inbox")
     msg = user2.direct_imap.get_all_messages()[0]
     assert public_ip not in msg.obj.as_string()
@@ -146,5 +189,5 @@ def test_echobot(cmfactory, chatmail_config, lp):
     text = "hi, I hope you text me back"
     chat.send_text(text)
     lp.sec("Wait for reply from echobot")
-    reply = ac.wait_next_incoming_message()
+    reply = ac._evtracker.wait_next_incoming_message()
     assert reply.text == text

cmdeploy/src/cmdeploy/tests/plugin.py

@@ -1,17 +1,16 @@
-import os
-import io
-import time
-import random
-import subprocess
 import imaplib
-import smtplib
+import io
 import itertools
+import os
+import random
+import smtplib
+import subprocess
+import time
 from pathlib import Path
+
 import pytest
-
-from chatmaild.database import Database
 from chatmaild.config import read_config
-
+from chatmaild.database import Database
 
 conftestdir = Path(__file__).parent
 

cmdeploy/src/cmdeploy/tests/test_cmdeploy.py

@@ -1,6 +1,7 @@
 import os
 
 import pytest
+
 from cmdeploy.cmdeploy import get_parser, main
 
 

cmdeploy/src/cmdeploy/www.py

@@ -1,13 +1,14 @@
-import importlib.resources
-import webbrowser
 import hashlib
+import importlib.resources
 import time
 import traceback
+import webbrowser
 
 import markdown
-from jinja2 import Template
-from .genqr import gen_qr_png_data
 from chatmaild.config import read_config
+from jinja2 import Template
+
+from .genqr import gen_qr_png_data
 
 
 def snapshot_dir_stats(somedir):
@@ -120,7 +121,8 @@ def main():
     print(f"watching {src_path} directory for changes")
 
     changenum = 0
-    for count in range(0, 1000000):
+    count = 0
+    while True:
         newstats = snapshot_dir_stats(src_path)
         if newstats == stats and count % 60 != 0:
             count += 1