Do not use redirect on /cgi-bin/newemail.py

Delta Chat does not follow redirects,
so it breaks old QR codes printed on paper
and published on various web pages.

.github/workflows/test-and-deploy.yaml

@@ -4,17 +4,12 @@ on:
   push:
     branches:
       - main
-  pull_request:
-    paths-ignore:
-      - 'scripts/**'
+      - staging-ci
 
 jobs:
   deploy:
     name: deploy on staging.testrun.org, and run tests
     runs-on: ubuntu-latest
-    concurrency:
-      group: staging-deploy
-      cancel-in-progress: true
     steps:
       - uses: actions/checkout@v3
 
@@ -24,57 +19,48 @@ jobs:
           echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
           chmod 600 ~/.ssh/id_ed25519
           ssh-keyscan staging.testrun.org > ~/.ssh/known_hosts
-          # save previous acme & dkim state
-          rsync -avz root@staging.testrun.org:/var/lib/acme . || true
-          rsync -avz root@staging.testrun.org:/etc/dkimkeys . || true
-          # store previous acme & dkim state on ns.testrun.org, if it contains useful certs
-          if [ -f dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys root@ns.testrun.org:/tmp/ || true; fi
-          if [ -z "$(ls -A acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme root@ns.testrun.org:/tmp/ || true; fi
+     #    rsync -avz root@staging.testrun.org:/var/lib/acme . || true
+     #    rsync -avz root@staging.testrun.org:/var/lib/rspamd/dkim . || true
 
-      - name: rebuild staging.testrun.org to have a clean VPS
-        run: |
-            curl -X POST \
-            -H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
-            -H "Content-Type: application/json" \
-            -d '{"image":"debian-12"}' \
-            "https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_SERVER_ID }}/actions/rebuild"
+     #- name: rebuild staging.testrun.org to have a clean VPS
+     #  run: |
+     #      curl -X POST \
+     #      -H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
+     #      -H "Content-Type: application/json" \
+     #      -d '{"image":"debian-12"}' \
+     #      "https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_SERVER_ID }}/actions/rebuild"
 
       - run: scripts/initenv.sh
 
       - name: append venv/bin to PATH
         run: echo venv/bin >>$GITHUB_PATH
 
-      - name: upload TLS cert after rebuilding
-        run: |
-          echo " --- wait until staging.testrun.org VPS is rebuilt --- "
-          rm ~/.ssh/known_hosts
-          while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u ; do sleep 1 ; done
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u
-          # download acme & dkim state from ns.testrun.org
-          rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme acme-restore || true
-          rsync -avz root@ns.testrun.org:/tmp/dkimkeys dkimkeys-restore || true
-          # restore acme & dkim state to staging.testrun.org
-          rsync -avz acme-restore/acme/ root@staging.testrun.org:/var/lib/acme || true
-          rsync -avz dkimkeys-restore/dkimkeys/ root@staging.testrun.org:/etc/dkimkeys || true
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org chown root:root -R /var/lib/acme || true
-
       - name: run formatting checks 
         run: cmdeploy fmt -v 
 
       - name: run deploy-chatmail offline tests 
         run: pytest --pyargs cmdeploy 
 
+     #- name: upload TLS cert after rebuilding
+     #  run: |
+     #    echo " --- wait until staging.testrun.org VPS is rebuilt --- "
+     #    rm ~/.ssh/known_hosts
+     #    while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u ; do sleep 1 ; done
+     #    ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u
+     #    rsync -avz acme root@staging.testrun.org:/var/lib/ || true
+     #    rsync -avz dkim root@staging.testrun.org:/var/lib/rspamd/ || true
+
       - run: cmdeploy init staging.testrun.org
 
       - run: cmdeploy run
 
       - name: set DNS entries
         run: |
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
+          #ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org chown _rspamd:_rspamd -R /var/lib/rspamd/dkim
           cmdeploy dns --zonefile staging-generated.zone
           cat staging-generated.zone >> .github/workflows/staging.testrun.org-default.zone
           cat .github/workflows/staging.testrun.org-default.zone
-          scp .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging.testrun.org.zone
+          scp -o StrictHostKeyChecking=accept-new .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging.testrun.org.zone
           ssh root@ns.testrun.org nsd-checkzone staging.testrun.org /etc/nsd/staging.testrun.org.zone
           ssh root@ns.testrun.org systemctl reload nsd
 

CHANGELOG.md

@@ -1,61 +0,0 @@
-# Changelog for chatmail deployment 
-
-## untagged
-
-- Accept Let's Encrypt's new Terms of Services
-  ([#275](https://github.com/deltachat/chatmail/pull/276))
-
-- Reload Dovecot and Postfix when TLS certificate updates
-  ([#271](https://github.com/deltachat/chatmail/pull/271))
-
-- Use forked version of dovecot without hardcoded delays
-  ([#270](https://github.com/deltachat/chatmail/pull/270))
-
-## 1.2.0 - 2024-04-04
-
-- Install dig on the server to resolve DNS records
-  ([#267](https://github.com/deltachat/chatmail/pull/267))
-
-- preserve notification order and exponentially backoff with 
-  retries for tokens where we didn't get a successful return
-  ([#265](https://github.com/deltachat/chatmail/pull/263))
-
-- Run chatmail-metadata and doveauth as vmail
-  ([#261](https://github.com/deltachat/chatmail/pull/261))
-
-- Apply systemd restrictions to echobot
-  ([#259](https://github.com/deltachat/chatmail/pull/259))
-
-- re-enable running the CI in pull requests, but not concurrently 
-  ([#258](https://github.com/deltachat/chatmail/pull/258))
-
-
-## 1.1.0 - 2024-03-28
-
-### The changelog starts to record changes from March 15th, 2024 
-
-- Move systemd unit templates to cmdeploy package 
-  ([#255](https://github.com/deltachat/chatmail/pull/255))
-
-- Persist push tokens and support multiple device per address 
-  ([#254](https://github.com/deltachat/chatmail/pull/254))
-
-- Avoid warning for regular doveauth protocol's hello message. 
-  ([#250](https://github.com/deltachat/chatmail/pull/250))
-
-- Fix various tests to pass again with "cmdeploy test". 
-  ([#245](https://github.com/deltachat/chatmail/pull/245),
-  [#242](https://github.com/deltachat/chatmail/pull/242)
-
-- Ensure lets-encrypt certificates are reloaded after renewal 
-  ([#244]) https://github.com/deltachat/chatmail/pull/244
-
-- Persist tokens to avoid iOS users loosing push-notifications when the
-  chatmail metadata service is restarted (happens regularly during deploys)
-  ([#238](https://github.com/deltachat/chatmail/pull/239)
-
-- Fix failing sieve-script compile errors on incoming messages
-  ([#237](https://github.com/deltachat/chatmail/pull/239)
-
-- Fix quota reporting after expunging of old mails
-  ([#233](https://github.com/deltachat/chatmail/pull/239)

README.md

@@ -159,27 +159,4 @@ While this file is present, account creation will be blocked.
 Delta Chat apps will, however, discover all ports and configurations
 automatically by reading the [autoconfig XML file](https://www.ietf.org/archive/id/draft-bucksch-autoconfig-00.html) from the chatmail service.
 
-## Email authentication
 
-chatmail servers rely on [DKIM](https://www.rfc-editor.org/rfc/rfc6376)
-to authenticate incoming emails.
-Incoming emails must have a valid DKIM signature with
-Signing Domain Identifier (SDID, `d=` parameter in the DKIM-Signature header)
-equal to the `From:` header domain.
-This property is checked by OpenDKIM screen policy script
-before validating the signatures.
-This correpsonds to strict [DMARC](https://www.rfc-editor.org/rfc/rfc7489) alignment (`adkim=s`),
-but chatmail does not rely on DMARC and does not consult the sender policy published in DMARC records.
-Other legacy authentication mechanisms such as [iprev](https://www.rfc-editor.org/rfc/rfc8601#section-2.7.3)
-and [SPF](https://www.rfc-editor.org/rfc/rfc7208) are also not taken into account.
-If there is no valid DKIM signature on the incoming email,
-the sender receives a "5.7.1 No valid DKIM signature found" error.
-
-Outgoing emails must be sent over authenticated connection
-with envelope MAIL FROM (return path) corresponding to the login.
-This is ensured by Postfix which maps login username
-to MAIL FROM with
-[`smtpd_sender_login_maps`](https://www.postfix.org/postconf.5.html#smtpd_sender_login_maps)
-and rejects incorrectly authenticated emails with [`reject_sender_login_mismatch`](reject_sender_login_mismatch) policy.
-`From:` header must correspond to envelope MAIL FROM,
-this is ensured by `filtermail` proxy.

chatmaild/MANIFEST.in

@@ -1,3 +1,4 @@
+include src/chatmaild/*.f 
 include src/chatmaild/ini/*.ini.f
 include src/chatmaild/ini/*.ini
 include src/chatmaild/tests/mail-data/*

chatmaild/pyproject.toml

@@ -10,8 +10,6 @@ dependencies = [
   "iniconfig",
   "deltachat-rpc-server",
   "deltachat-rpc-client",
-  "filelock",
-  "requests",
 ]
 
 [tool.setuptools]
@@ -22,7 +20,6 @@ where = ['src']
 
 [project.scripts]
 doveauth = "chatmaild.doveauth:main"
-chatmail-metadata = "chatmaild.metadata:main"
 filtermail = "chatmaild.filtermail:main"
 echobot = "chatmaild.echo:main"
 chatmail-metrics = "chatmaild.metrics:main"

chatmaild/src/chatmaild/doveauth.py

@@ -4,12 +4,12 @@ import time
 import sys
 import json
 import crypt
-from pathlib import Path
 from socketserver import (
     UnixStreamServer,
     StreamRequestHandler,
     ThreadingMixIn,
 )
+import pwd
 
 from .database import Database
 from .config import read_config, Config
@@ -17,10 +17,6 @@ from .config import read_config, Config
 NOCREATE_FILE = "/etc/chatmail-nocreate"
 
 
-class UnknownCommand(ValueError):
-    """dictproxy handler received an unkown command"""
-
-
 def encrypt_password(password: str):
     # https://doc.dovecot.org/configuration_manual/authentication/password_schemes/
     passhash = crypt.crypt(password, crypt.METHOD_SHA512)
@@ -46,61 +42,36 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
         return False
     localpart, domain = parts
 
-    if localpart == "echo":
-        # echobot account should not be created in the database
-        return False
-
     if (
         len(localpart) > config.username_max_length
         or len(localpart) < config.username_min_length
     ):
-        logging.warning(
-            "localpart %s has to be between %s and %s chars long",
-            localpart,
-            config.username_min_length,
-            config.username_max_length,
-        )
+        if localpart != "echo":
+            logging.warning(
+                "localpart %s has to be between %s and %s chars long",
+                localpart,
+                config.username_min_length,
+                config.username_max_length,
+            )
+            return False
 
     return True
 
 
-def get_user_data(db, config: Config, user):
-    if user == f"echo@{config.mail_domain}":
-        return dict(
-            home=f"/home/vmail/mail/{config.mail_domain}/echo@{config.mail_domain}",
-            uid="vmail",
-            gid="vmail",
-        )
-
+def get_user_data(db, user):
     with db.read_connection() as conn:
         result = conn.get_user(user)
     if result:
-        result["home"] = f"/home/vmail/mail/{config.mail_domain}/{user}"
         result["uid"] = "vmail"
         result["gid"] = "vmail"
     return result
 
 
-def lookup_userdb(db, config: Config, user):
-    return get_user_data(db, config, user)
+def lookup_userdb(db, user):
+    return get_user_data(db, user)
 
 
 def lookup_passdb(db, config: Config, user, cleartext_password):
-    if user == f"echo@{config.mail_domain}":
-        # Echobot writes password it wants to log in with into /run/echobot/password
-        try:
-            password = Path("/run/echobot/password").read_text()
-        except Exception:
-            logging.exception("Exception when trying to read /run/echobot/password")
-            return None
-
-        return dict(
-            home=f"/home/vmail/mail/{config.mail_domain}/echo@{config.mail_domain}",
-            uid="vmail",
-            gid="vmail",
-            password=encrypt_password(password),
-        )
-
     with db.write_transaction() as conn:
         userdata = conn.get_user(user)
         if userdata:
@@ -109,7 +80,6 @@ def lookup_passdb(db, config: Config, user, cleartext_password):
                 "UPDATE users SET last_login=? WHERE addr=?", (int(time.time()), user)
             )
 
-            userdata["home"] = f"/home/vmail/mail/{config.mail_domain}/{user}"
             userdata["uid"] = "vmail"
             userdata["gid"] = "vmail"
             return userdata
@@ -155,12 +125,8 @@ def split_and_unescape(s):
 
 
 def handle_dovecot_request(msg, db, config: Config):
-    # see https://doc.dovecot.org/3.0/developer_manual/design/dict_protocol/
     short_command = msg[0]
-    if short_command == "H":  # HELLO
-        # we don't do any checking on versions and just return
-        return
-    elif short_command == "L":  # LOOKUP
+    if short_command == "L":  # LOOKUP
         parts = msg[1:].split("\t")
 
         # Dovecot <2.3.17 has only one part,
@@ -176,7 +142,7 @@ def handle_dovecot_request(msg, db, config: Config):
             if type == "userdb":
                 user = args[0]
                 if user.endswith(f"@{config.mail_domain}"):
-                    res = lookup_userdb(db, config, user)
+                    res = lookup_userdb(db, user)
                 if res:
                     reply_command = "O"
                 else:
@@ -191,22 +157,7 @@ def handle_dovecot_request(msg, db, config: Config):
                     reply_command = "N"
         json_res = json.dumps(res) if res else ""
         return f"{reply_command}{json_res}\n"
-    raise UnknownCommand(msg)
-
-
-def handle_dovecot_protocol(rfile, wfile, db: Database, config: Config):
-    while True:
-        msg = rfile.readline().strip().decode()
-        if not msg:
-            break
-        try:
-            res = handle_dovecot_request(msg, db, config)
-        except UnknownCommand:
-            logging.warning("unknown command: %r", msg)
-        else:
-            if res:
-                wfile.write(res.encode("ascii"))
-                wfile.flush()
+    return None
 
 
 class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
@@ -215,13 +166,23 @@ class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
 
 def main():
     socket = sys.argv[1]
-    db = Database(sys.argv[2])
-    config = read_config(sys.argv[3])
+    passwd_entry = pwd.getpwnam(sys.argv[2])
+    db = Database(sys.argv[3])
+    config = read_config(sys.argv[4])
 
     class Handler(StreamRequestHandler):
         def handle(self):
             try:
-                handle_dovecot_protocol(self.rfile, self.wfile, db, config)
+                while True:
+                    msg = self.rfile.readline().strip().decode()
+                    if not msg:
+                        break
+                    res = handle_dovecot_request(msg, db, config)
+                    if res:
+                        self.wfile.write(res.encode("ascii"))
+                        self.wfile.flush()
+                    else:
+                        logging.warn("request had no answer: %r", msg)
             except Exception:
                 logging.exception("Exception in the handler")
                 raise
@@ -232,6 +193,7 @@ def main():
         pass
 
     with ThreadedUnixStreamServer(socket, Handler) as server:
+        os.chown(socket, uid=passwd_entry.pw_uid, gid=passwd_entry.pw_gid)
         try:
             server.serve_forever()
         except KeyboardInterrupt:

chatmaild/src/chatmaild/doveauth.service.f

@@ -2,11 +2,9 @@
 Description=Chatmail dict authentication proxy for dovecot
 
 [Service]
-ExecStart={execpath} /run/doveauth/doveauth.socket /home/vmail/passdb.sqlite {config_path}
+ExecStart={execpath} /run/dovecot/doveauth.socket vmail /home/vmail/passdb.sqlite {config_path}
 Restart=always
 RestartSec=30
-User=vmail
-RuntimeDirectory=doveauth
 
 [Install]
 WantedBy=multi-user.target

chatmaild/src/chatmaild/echo.py

@@ -3,17 +3,14 @@
 
 it will echo back any message that has non-empty text and also supports the /help command.
 """
-
 import logging
 import os
 import sys
-import subprocess
 
 from deltachat_rpc_client import Bot, DeltaChat, EventType, Rpc, events
-from pathlib import Path
 
-from chatmaild.config import read_config
 from chatmaild.newemail import create_newemail_dict
+from chatmaild.config import read_config
 
 hooks = events.HookCollection()
 
@@ -21,14 +18,14 @@ hooks = events.HookCollection()
 @hooks.on(events.RawEvent)
 def log_event(event):
     if event.kind == EventType.INFO:
-        logging.info("%s", event.msg)
+        logging.info(event.msg)
     elif event.kind == EventType.WARNING:
-        logging.warning("%s", event.msg)
+        logging.warning(event.msg)
 
 
 @hooks.on(events.RawEvent(EventType.ERROR))
 def log_error(event):
-    logging.error("%s", event.msg)
+    logging.error(event.msg)
 
 
 @hooks.on(events.MemberListChanged)
@@ -51,9 +48,6 @@ def on_group_name_changed(event):
 @hooks.on(events.NewMessage(func=lambda e: not e.command))
 def echo(event):
     snapshot = event.message_snapshot
-    if snapshot.is_info:
-        # Ignore info messages
-        return
     if snapshot.text or snapshot.file:
         snapshot.chat.send_message(text=snapshot.text, file=snapshot.file)
 
@@ -65,7 +59,6 @@ def help_command(event):
 
 
 def main():
-    logging.basicConfig(level=logging.INFO)
     path = os.environ.get("PATH")
     venv_path = sys.argv[0].strip("echobot")
     os.environ["PATH"] = path + ":" + venv_path
@@ -78,27 +71,14 @@ def main():
         account = accounts[0] if accounts else deltachat.add_account()
 
         bot = Bot(account, hooks)
-
-        config = read_config(sys.argv[1])
-
-        # Create password file
-        if bot.is_configured():
-            password = bot.account.get_config("mail_pw")
-        else:
-            password = create_newemail_dict(config)["password"]
-        Path("/run/echobot/password").write_text(password)
-
-        # Give the user which doveauth runs as access to the password file.
-        subprocess.run(
-            ["/usr/bin/setfacl", "-m", "user:vmail:r", "/run/echobot/password"],
-            check=True,
-        )
-
         if not bot.is_configured():
+            config = read_config(sys.argv[1])
+            password = create_newemail_dict(config).get("password")
             email = "echo@" + config.mail_domain
             bot.configure(email, password)
         bot.run_forever()
 
 
 if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
     main()

chatmaild/src/chatmaild/echobot.service.f

@@ -0,0 +1,11 @@
+[Unit]
+Description=Chatmail echo bot for testing it works
+
+[Service]
+ExecStart={execpath} {config_path}
+Environment="PATH={remote_venv_dir}:$PATH"
+Restart=always
+RestartSec=30
+
+[Install]
+WantedBy=multi-user.target

chatmaild/src/chatmaild/filedict.py

@@ -1,35 +0,0 @@
-import os
-import logging
-import json
-import filelock
-from contextlib import contextmanager
-
-
-class FileDict:
-    """Concurrency-safe multi-reader/single-writer persistent dict."""
-
-    def __init__(self, path):
-        self.path = path
-        self.lock_path = path.with_name(path.name + ".lock")
-
-    @contextmanager
-    def modify(self):
-        # the OS will release the lock if the process dies,
-        # and the contextmanager will otherwise guarantee release
-        with filelock.FileLock(self.lock_path):
-            data = self.read()
-            yield data
-            write_path = self.path.with_name(self.path.name + ".tmp")
-            with write_path.open("w") as f:
-                json.dump(data, f)
-            os.rename(write_path, self.path)
-
-    def read(self):
-        try:
-            with self.path.open("r") as f:
-                return json.load(f)
-        except FileNotFoundError:
-            return {}
-        except Exception:
-            logging.warning("corrupt serialization state at: %r", self.path)
-            return {}

chatmaild/src/chatmaild/filtermail.service.f

@@ -1,5 +1,5 @@
 [Unit]
-Description=Chatmail Postfix before queue filter
+Description=Chatmail Postfix BeforeQeue filter 
 
 [Service]
 ExecStart={execpath} {config_path}

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -33,7 +33,7 @@ password_min_length = 9
 passthrough_senders =
 
 # list of e-mail recipients for which to accept outbound un-encrypted mails
-passthrough_recipients = xstore@testrun.org groupsbot@hispanilandia.net
+passthrough_recipients =
 
 #
 # Deployment Details

chatmaild/src/chatmaild/ini/override-testrun.ini

@@ -1,7 +1,7 @@
 
 [privacy]
 
-passthrough_recipients = privacy@testrun.org xstore@testrun.org groupsbot@hispanilandia.net
+passthrough_recipients = privacy@testrun.org
 
 privacy_postal =
     Merlinux GmbH, Represented by the managing director H. Krekel,

chatmaild/src/chatmaild/metadata.py

@@ -1,153 +0,0 @@
-from pathlib import Path
-from socketserver import (
-    UnixStreamServer,
-    StreamRequestHandler,
-    ThreadingMixIn,
-)
-import sys
-import logging
-import os
-
-from .filedict import FileDict
-from .notifier import Notifier
-
-
-DICTPROXY_HELLO_CHAR = "H"
-DICTPROXY_LOOKUP_CHAR = "L"
-DICTPROXY_ITERATE_CHAR = "I"
-DICTPROXY_BEGIN_TRANSACTION_CHAR = "B"
-DICTPROXY_SET_CHAR = "S"
-DICTPROXY_COMMIT_TRANSACTION_CHAR = "C"
-DICTPROXY_TRANSACTION_CHARS = "BSC"
-
-
-class Metadata:
-    # each SETMETADATA on this key appends to a list of unique device tokens
-    # which only ever get removed if the upstream indicates the token is invalid
-    DEVICETOKEN_KEY = "devicetoken"
-
-    def __init__(self, vmail_dir):
-        self.vmail_dir = vmail_dir
-
-    def get_metadata_dict(self, addr):
-        return FileDict(self.vmail_dir / addr / "metadata.json")
-
-    def add_token_to_addr(self, addr, token):
-        with self.get_metadata_dict(addr).modify() as data:
-            tokens = data.setdefault(self.DEVICETOKEN_KEY, [])
-            if token not in tokens:
-                tokens.append(token)
-
-    def remove_token_from_addr(self, addr, token):
-        with self.get_metadata_dict(addr).modify() as data:
-            tokens = data.get(self.DEVICETOKEN_KEY, [])
-            if token in tokens:
-                tokens.remove(token)
-
-    def get_tokens_for_addr(self, addr):
-        mdict = self.get_metadata_dict(addr).read()
-        return mdict.get(self.DEVICETOKEN_KEY, [])
-
-
-def handle_dovecot_protocol(rfile, wfile, notifier, metadata):
-    transactions = {}
-    while True:
-        msg = rfile.readline().strip().decode()
-        if not msg:
-            break
-
-        res = handle_dovecot_request(msg, transactions, notifier, metadata)
-        if res:
-            wfile.write(res.encode("ascii"))
-            wfile.flush()
-
-
-def handle_dovecot_request(msg, transactions, notifier, metadata):
-    # see https://doc.dovecot.org/3.0/developer_manual/design/dict_protocol/
-    short_command = msg[0]
-    parts = msg[1:].split("\t")
-    if short_command == DICTPROXY_LOOKUP_CHAR:
-        # Lpriv/43f5f508a7ea0366dff30200c15250e3/devicetoken\tlkj123poi@c2.testrun.org
-        keyparts = parts[0].split("/")
-        if keyparts[0] == "priv":
-            keyname = keyparts[2]
-            addr = parts[1]
-            if keyname == metadata.DEVICETOKEN_KEY:
-                res = " ".join(metadata.get_tokens_for_addr(addr))
-                return f"O{res}\n"
-        logging.warning("lookup ignored: %r", msg)
-        return "N\n"
-    elif short_command == DICTPROXY_ITERATE_CHAR:
-        # Empty line means ITER_FINISHED.
-        # If we don't return empty line Dovecot will timeout.
-        return "\n"
-    elif short_command == DICTPROXY_HELLO_CHAR:
-        return  # no version checking
-
-    if short_command not in (DICTPROXY_TRANSACTION_CHARS):
-        logging.warning("unknown dictproxy request: %r", msg)
-        return
-
-    transaction_id = parts[0]
-
-    if short_command == DICTPROXY_BEGIN_TRANSACTION_CHAR:
-        addr = parts[1]
-        transactions[transaction_id] = dict(addr=addr, res="O\n")
-    elif short_command == DICTPROXY_COMMIT_TRANSACTION_CHAR:
-        # each set devicetoken operation persists directly
-        # and does not wait until a "commit" comes
-        # because our dovecot config does not involve
-        # multiple set-operations in a single commit
-        return transactions.pop(transaction_id)["res"]
-    elif short_command == DICTPROXY_SET_CHAR:
-        # For documentation on key structure see
-        # https://github.com/dovecot/core/blob/main/src/lib-storage/mailbox-attribute.h
-
-        keyname = parts[1].split("/")
-        value = parts[2] if len(parts) > 2 else ""
-        addr = transactions[transaction_id]["addr"]
-        if keyname[0] == "priv" and keyname[2] == metadata.DEVICETOKEN_KEY:
-            metadata.add_token_to_addr(addr, value)
-        elif keyname[0] == "priv" and keyname[2] == "messagenew":
-            notifier.new_message_for_addr(addr, metadata)
-        else:
-            # Transaction failed.
-            transactions[transaction_id]["res"] = "F\n"
-
-
-class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
-    request_queue_size = 100
-
-
-def main():
-    socket, vmail_dir = sys.argv[1:]
-
-    vmail_dir = Path(vmail_dir)
-    if not vmail_dir.exists():
-        logging.error("vmail dir does not exist: %r", vmail_dir)
-        return 1
-
-    queue_dir = vmail_dir / "pending_notifications"
-    queue_dir.mkdir(exist_ok=True)
-    metadata = Metadata(vmail_dir)
-    notifier = Notifier(queue_dir)
-    notifier.start_notification_threads(metadata.remove_token_from_addr)
-
-    class Handler(StreamRequestHandler):
-        def handle(self):
-            try:
-                handle_dovecot_protocol(self.rfile, self.wfile, notifier, metadata)
-            except Exception:
-                logging.exception("Exception in the dovecot dictproxy handler")
-                raise
-
-    try:
-        os.unlink(socket)
-    except FileNotFoundError:
-        pass
-
-    with ThreadedUnixStreamServer(socket, Handler) as server:
-        try:
-            server.serve_forever()
-        except KeyboardInterrupt:
-            pass

chatmaild/src/chatmaild/notifier.py

@@ -1,165 +0,0 @@
-"""
-This modules provides notification machinery for transmitting device tokens to
-a central notification server which in turn contacts a phone provider's notification server
-to trigger Delta Chat apps to retrieve messages and provide instant notifications to users.
-
-The Notifier class arranges the queuing of tokens in separate PriorityQueues
-from which NotifyThreads take and transmit them via HTTPS
-to the `notifications.delta.chat` service.
-The current lack of proper HTTP/2-support in Python leads us
-to use multiple threads and connections to the Rust-implemented `notifications.delta.chat`
-which itself uses HTTP/2 and thus only a single connection to phone-notification providers.
-
-If a token fails to cause a successful notification
-it is moved to a retry-number specific PriorityQueue
-which handles all tokens that failed a particular number of times
-and which are scheduled for retry using exponential back-off timing.
-If a token notification would be scheduled more than DROP_DEADLINE seconds
-after its first attempt, it is dropped with a log error.
-
-Note that tokens are completely opaque to the notification machinery here
-and will in the future be encrypted foreclosing all ability to distinguish
-which device token ultimately goes to which phone-provider notification service,
-or to understand the relation of "device tokens" and chatmail addresses.
-The meaning and format of tokens is basically a matter of Delta-Chat Core and
-the `notification.delta.chat` service.
-"""
-
-import os
-import time
-import math
-import logging
-from uuid import uuid4
-from threading import Thread
-from pathlib import Path
-from queue import PriorityQueue
-from dataclasses import dataclass
-import requests
-
-
-@dataclass
-class PersistentQueueItem:
-    path: Path
-    addr: str
-    start_ts: int
-    token: str
-
-    def delete(self):
-        self.path.unlink(missing_ok=True)
-
-    @classmethod
-    def create(cls, queue_dir, addr, start_ts, token):
-        queue_id = uuid4().hex
-        start_ts = int(start_ts)
-        path = queue_dir.joinpath(queue_id)
-        tmp_path = path.with_name(path.name + ".tmp")
-        tmp_path.write_text(f"{addr}\n{start_ts}\n{token}")
-        os.rename(tmp_path, path)
-        return cls(path, addr, start_ts, token)
-
-    @classmethod
-    def read_from_path(cls, path):
-        addr, start_ts, token = path.read_text().split("\n", maxsplit=2)
-        return cls(path, addr, int(start_ts), token)
-
-    def __lt__(self, other):
-        return self.start_ts < other.start_ts
-
-
-class Notifier:
-    URL = "https://notifications.delta.chat/notify"
-    CONNECTION_TIMEOUT = 60.0  # seconds until http-request is given up
-    BASE_DELAY = 8.0  # base seconds for exponential back-off delay
-    DROP_DEADLINE = 5 * 60 * 60  #  drop notifications after 5 hours
-
-    def __init__(self, queue_dir):
-        self.queue_dir = queue_dir
-        max_tries = int(math.log(self.DROP_DEADLINE, self.BASE_DELAY)) + 1
-        self.retry_queues = [PriorityQueue() for _ in range(max_tries)]
-
-    def compute_delay(self, retry_num):
-        return 0 if retry_num == 0 else pow(self.BASE_DELAY, retry_num)
-
-    def new_message_for_addr(self, addr, metadata):
-        start_ts = int(time.time())
-        for token in metadata.get_tokens_for_addr(addr):
-            queue_item = PersistentQueueItem.create(
-                self.queue_dir, addr, start_ts, token
-            )
-            self.queue_for_retry(queue_item)
-
-    def requeue_persistent_queue_items(self):
-        for queue_path in self.queue_dir.iterdir():
-            if queue_path.name.endswith(".tmp"):
-                logging.warning("removing spurious queue item: %r", queue_path)
-                queue_path.unlink()
-                continue
-            queue_item = PersistentQueueItem.read_from_path(queue_path)
-            self.queue_for_retry(queue_item)
-
-    def queue_for_retry(self, queue_item, retry_num=0):
-        delay = self.compute_delay(retry_num)
-        when = int(time.time()) + delay
-        deadline = queue_item.start_ts + self.DROP_DEADLINE
-        if retry_num >= len(self.retry_queues) or when > deadline:
-            queue_item.delete()
-            logging.error("notification exceeded deadline: %r", queue_item.token)
-            return
-
-        self.retry_queues[retry_num].put((when, queue_item))
-
-    def start_notification_threads(self, remove_token_from_addr):
-        self.requeue_persistent_queue_items()
-        threads = {}
-        for retry_num in range(len(self.retry_queues)):
-            # use 4 threads for first-try tokens and less for subsequent tries
-            num_threads = 4 if retry_num == 0 else 2
-            threads[retry_num] = []
-            for _ in range(num_threads):
-                thread = NotifyThread(self, retry_num, remove_token_from_addr)
-                threads[retry_num].append(thread)
-                thread.start()
-        return threads
-
-
-class NotifyThread(Thread):
-    def __init__(self, notifier, retry_num, remove_token_from_addr):
-        super().__init__(daemon=True)
-        self.notifier = notifier
-        self.retry_num = retry_num
-        self.remove_token_from_addr = remove_token_from_addr
-
-    def stop(self):
-        self.notifier.retry_queues[self.retry_num].put((None, None))
-
-    def run(self):
-        requests_session = requests.Session()
-        while self.retry_one(requests_session):
-            pass
-
-    def retry_one(self, requests_session, sleep=time.sleep):
-        when, queue_item = self.notifier.retry_queues[self.retry_num].get()
-        if when is None:
-            return False
-        wait_time = when - int(time.time())
-        if wait_time > 0:
-            sleep(wait_time)
-        self.perform_request_to_notification_server(requests_session, queue_item)
-        return True
-
-    def perform_request_to_notification_server(self, requests_session, queue_item):
-        timeout = self.notifier.CONNECTION_TIMEOUT
-        token = queue_item.token
-        try:
-            res = requests_session.post(self.notifier.URL, data=token, timeout=timeout)
-        except requests.exceptions.RequestException as e:
-            res = e
-        else:
-            if res.status_code in (200, 410):
-                if res.status_code == 410:
-                    self.remove_token_from_addr(queue_item.addr, token)
-                queue_item.delete()
-                return
-
-        logging.warning("Notification request failed: %r", res)
-        self.notifier.queue_for_retry(queue_item, retry_num=self.retry_num + 1)

chatmaild/src/chatmaild/tests/plugin.py

@@ -1,6 +1,4 @@
 import random
-from pathlib import Path
-import os
 import importlib.resources
 import itertools
 from email.parser import BytesParser
@@ -59,12 +57,7 @@ def db(tmpdir):
 
 @pytest.fixture
 def maildata(request):
-    try:
-        datadir = importlib.resources.files(__package__).joinpath("mail-data")
-    except TypeError:
-        # in python3.9 or lower, the above doesn't work, so we get datadir this way:
-        datadir = Path(os.getcwd()).joinpath("chatmaild/src/chatmaild/tests/mail-data")
-
+    datadir = importlib.resources.files(__package__).joinpath("mail-data")
     assert datadir.exists(), datadir
 
     def maildata(name, from_addr, to_addr):

chatmaild/src/chatmaild/tests/test_config.py

@@ -28,5 +28,5 @@ def test_read_config_testrun(make_config):
     assert config.username_min_length == 9
     assert config.username_max_length == 9
     assert config.password_min_length == 9
-    assert "privacy@testrun.org" in config.passthrough_recipients
+    assert config.passthrough_recipients == ["privacy@testrun.org"]
     assert config.passthrough_senders == []

chatmaild/src/chatmaild/tests/test_doveauth.py

@@ -1,23 +1,17 @@
-import io
 import json
 import pytest
-import queue
 import threading
+import queue
 import traceback
 
 import chatmaild.doveauth
-from chatmaild.doveauth import (
-    get_user_data,
-    lookup_passdb,
-    handle_dovecot_request,
-    handle_dovecot_protocol,
-)
+from chatmaild.doveauth import get_user_data, lookup_passdb, handle_dovecot_request
 from chatmaild.database import DBError
 
 
 def test_basic(db, example_config):
     lookup_passdb(db, example_config, "asdf12345@chat.example.org", "q9mr3faue")
-    data = get_user_data(db, example_config, "asdf12345@chat.example.org")
+    data = get_user_data(db, "asdf12345@chat.example.org")
     assert data
     data2 = lookup_passdb(
         db, example_config, "asdf12345@chat.example.org", "q9mr3jewvadsfaue"
@@ -43,7 +37,7 @@ def test_nocreate_file(db, monkeypatch, tmpdir, example_config):
     lookup_passdb(
         db, example_config, "newuser12@chat.example.org", "zequ0Aimuchoodaechik"
     )
-    assert not get_user_data(db, example_config, "newuser12@chat.example.org")
+    assert not get_user_data(db, "newuser12@chat.example.org")
 
 
 def test_db_version(db):
@@ -75,23 +69,6 @@ def test_handle_dovecot_request(db, example_config):
     assert userdata["password"].startswith("{SHA512-CRYPT}")
 
 
-def test_handle_dovecot_protocol_hello_is_skipped(db, example_config, caplog):
-    rfile = io.BytesIO(b"H3\t2\t0\t\tauth\n")
-    wfile = io.BytesIO()
-    handle_dovecot_protocol(rfile, wfile, db, example_config)
-    assert wfile.getvalue() == b""
-    assert not caplog.messages
-
-
-def test_handle_dovecot_protocol(db, example_config):
-    rfile = io.BytesIO(
-        b"H3\t2\t0\t\tauth\nLshared/userdb/foobar@chat.example.org\tfoobar@chat.example.org\n"
-    )
-    wfile = io.BytesIO()
-    handle_dovecot_protocol(rfile, wfile, db, example_config)
-    assert wfile.getvalue() == b"N\n"
-
-
 def test_50_concurrent_lookups_different_accounts(db, gencreds, example_config):
     num_threads = 50
     req_per_thread = 5

chatmaild/src/chatmaild/tests/test_filedict.py

@@ -1,19 +0,0 @@
-from chatmaild.filedict import FileDict
-
-
-def test_basic(tmp_path):
-    fdict = FileDict(tmp_path.joinpath("metadata"))
-    assert fdict.read() == {}
-    with fdict.modify() as d:
-        d["devicetoken"] = [1, 2, 3]
-        d["456"] = 4.2
-    new = fdict.read()
-    assert new["devicetoken"] == [1, 2, 3]
-    assert new["456"] == 4.2
-
-
-def test_bad_marshal_file(tmp_path, caplog):
-    fdict1 = FileDict(tmp_path.joinpath("metadata"))
-    fdict1.path.write_bytes(b"l12k3l12k3l")
-    assert fdict1.read() == {}
-    assert "corrupt" in caplog.records[0].msg

chatmaild/src/chatmaild/tests/test_metadata.py

@@ -1,298 +0,0 @@
-import io
-import pytest
-import requests
-import time
-
-from chatmaild.metadata import (
-    handle_dovecot_request,
-    handle_dovecot_protocol,
-    Metadata,
-)
-from chatmaild.notifier import (
-    Notifier,
-    NotifyThread,
-    PersistentQueueItem,
-)
-
-
-@pytest.fixture
-def notifier(metadata):
-    queue_dir = metadata.vmail_dir.joinpath("pending_notifications")
-    queue_dir.mkdir()
-    return Notifier(queue_dir)
-
-
-@pytest.fixture
-def metadata(tmp_path):
-    vmail_dir = tmp_path.joinpath("vmaildir")
-    vmail_dir.mkdir()
-    return Metadata(vmail_dir)
-
-
-@pytest.fixture
-def testaddr():
-    return "user.name@example.org"
-
-
-@pytest.fixture
-def testaddr2():
-    return "user2@example.org"
-
-
-@pytest.fixture
-def token():
-    return "01234"
-
-
-def get_mocked_requests(statuslist):
-    class ReqMock:
-        requests = []
-
-        def post(self, url, data, timeout):
-            self.requests.append((url, data, timeout))
-            res = statuslist.pop(0)
-            if isinstance(res, Exception):
-                raise res
-
-            class Result:
-                status_code = res
-
-            return Result()
-
-    return ReqMock()
-
-
-def test_metadata_persistence(tmp_path, testaddr, testaddr2):
-    metadata1 = Metadata(tmp_path)
-    metadata2 = Metadata(tmp_path)
-    assert not metadata1.get_tokens_for_addr(testaddr)
-    assert not metadata2.get_tokens_for_addr(testaddr)
-
-    metadata1.add_token_to_addr(testaddr, "01234")
-    metadata1.add_token_to_addr(testaddr2, "456")
-    assert metadata2.get_tokens_for_addr(testaddr) == ["01234"]
-    assert metadata2.get_tokens_for_addr(testaddr2) == ["456"]
-    metadata2.remove_token_from_addr(testaddr, "01234")
-    assert not metadata1.get_tokens_for_addr(testaddr)
-    assert metadata1.get_tokens_for_addr(testaddr2) == ["456"]
-
-
-def test_remove_nonexisting(metadata, tmp_path, testaddr):
-    metadata.add_token_to_addr(testaddr, "123")
-    metadata.remove_token_from_addr(testaddr, "1l23k1l2k3")
-    assert metadata.get_tokens_for_addr(testaddr) == ["123"]
-
-
-def test_notifier_remove_without_set(metadata, testaddr):
-    metadata.remove_token_from_addr(testaddr, "123")
-    assert not metadata.get_tokens_for_addr(testaddr)
-
-
-def test_handle_dovecot_request_lookup_fails(notifier, metadata, testaddr):
-    res = handle_dovecot_request(
-        f"Lpriv/123/chatmail\t{testaddr}", {}, notifier, metadata
-    )
-    assert res == "N\n"
-
-
-def test_handle_dovecot_request_happy_path(notifier, metadata, testaddr, token):
-    transactions = {}
-
-    # set device token in a transaction
-    tx = "1111"
-    msg = f"B{tx}\t{testaddr}"
-    res = handle_dovecot_request(msg, transactions, notifier, metadata)
-    assert not res and not metadata.get_tokens_for_addr(testaddr)
-    assert transactions == {tx: dict(addr=testaddr, res="O\n")}
-
-    msg = f"S{tx}\tpriv/guid00/devicetoken\t{token}"
-    res = handle_dovecot_request(msg, transactions, notifier, metadata)
-    assert not res
-    assert len(transactions) == 1
-    assert metadata.get_tokens_for_addr(testaddr) == [token]
-
-    msg = f"C{tx}"
-    res = handle_dovecot_request(msg, transactions, notifier, metadata)
-    assert res == "O\n"
-    assert len(transactions) == 0
-    assert metadata.get_tokens_for_addr(testaddr) == [token]
-
-    # trigger notification for incoming message
-    tx2 = "2222"
-    assert (
-        handle_dovecot_request(f"B{tx2}\t{testaddr}", transactions, notifier, metadata)
-        is None
-    )
-    msg = f"S{tx2}\tpriv/guid00/messagenew"
-    assert handle_dovecot_request(msg, transactions, notifier, metadata) is None
-    queue_item = notifier.retry_queues[0].get()[1]
-    assert queue_item.token == token
-    assert handle_dovecot_request(f"C{tx2}", transactions, notifier, metadata) == "O\n"
-    assert not transactions
-    assert queue_item.path.exists()
-
-
-def test_handle_dovecot_protocol_set_devicetoken(metadata, notifier):
-    rfile = io.BytesIO(
-        b"\n".join(
-            [
-                b"HELLO",
-                b"Btx00\tuser@example.org",
-                b"Stx00\tpriv/guid00/devicetoken\t01234",
-                b"Ctx00",
-            ]
-        )
-    )
-    wfile = io.BytesIO()
-    handle_dovecot_protocol(rfile, wfile, notifier, metadata)
-    assert wfile.getvalue() == b"O\n"
-    assert metadata.get_tokens_for_addr("user@example.org") == ["01234"]
-
-
-def test_handle_dovecot_protocol_set_get_devicetoken(metadata, notifier):
-    rfile = io.BytesIO(
-        b"\n".join(
-            [
-                b"HELLO",
-                b"Btx00\tuser@example.org",
-                b"Stx00\tpriv/guid00/devicetoken\t01234",
-                b"Ctx00",
-            ]
-        )
-    )
-    wfile = io.BytesIO()
-    handle_dovecot_protocol(rfile, wfile, notifier, metadata)
-    assert metadata.get_tokens_for_addr("user@example.org") == ["01234"]
-    assert wfile.getvalue() == b"O\n"
-
-    rfile = io.BytesIO(
-        b"\n".join([b"HELLO", b"Lpriv/0123/devicetoken\tuser@example.org"])
-    )
-    wfile = io.BytesIO()
-    handle_dovecot_protocol(rfile, wfile, notifier, metadata)
-    assert wfile.getvalue() == b"O01234\n"
-
-
-def test_handle_dovecot_protocol_iterate(metadata, notifier):
-    rfile = io.BytesIO(
-        b"\n".join(
-            [
-                b"H",
-                b"I9\t0\tpriv/5cbe730f146fea6535be0d003dd4fc98/\tci-2dzsrs@nine.testrun.org",
-            ]
-        )
-    )
-    wfile = io.BytesIO()
-    handle_dovecot_protocol(rfile, wfile, notifier, metadata)
-    assert wfile.getvalue() == b"\n"
-
-
-def test_notifier_thread_deletes_persistent_file(metadata, notifier, testaddr):
-    reqmock = get_mocked_requests([200])
-    metadata.add_token_to_addr(testaddr, "01234")
-    notifier.new_message_for_addr(testaddr, metadata)
-    NotifyThread(notifier, 0, None).retry_one(reqmock)
-    url, data, timeout = reqmock.requests[0]
-    assert data == "01234"
-    assert metadata.get_tokens_for_addr(testaddr) == ["01234"]
-    notifier.requeue_persistent_queue_items()
-    assert notifier.retry_queues[0].qsize() == 0
-
-
-@pytest.mark.parametrize("status", [requests.exceptions.RequestException(), 404, 500])
-def test_notifier_thread_connection_failures(
-    metadata, notifier, testaddr, status, caplog
-):
-    """test that tokens keep getting retried until they are given up."""
-    metadata.add_token_to_addr(testaddr, "01234")
-    notifier.new_message_for_addr(testaddr, metadata)
-    notifier.NOTIFICATION_RETRY_DELAY = 5
-    max_tries = len(notifier.retry_queues)
-    for i in range(max_tries):
-        caplog.clear()
-        reqmock = get_mocked_requests([status])
-        sleep_calls = []
-        NotifyThread(notifier, i, None).retry_one(reqmock, sleep=sleep_calls.append)
-        assert notifier.retry_queues[i].qsize() == 0
-        assert "request failed" in caplog.records[0].msg
-        if i > 0:
-            assert len(sleep_calls) == 1
-        if i + 1 < max_tries:
-            assert notifier.retry_queues[i + 1].qsize() == 1
-            assert len(caplog.records) == 1
-        else:
-            assert len(caplog.records) == 2
-            assert "deadline" in caplog.records[1].msg
-    notifier.requeue_persistent_queue_items()
-    assert notifier.retry_queues[0].qsize() == 0
-
-
-def test_requeue_removes_tmp_files(notifier, metadata, testaddr, caplog):
-    metadata.add_token_to_addr(testaddr, "01234")
-    notifier.new_message_for_addr(testaddr, metadata)
-    p = notifier.queue_dir.joinpath("1203981203.tmp")
-    p.touch()
-    notifier2 = notifier.__class__(notifier.queue_dir)
-    notifier2.requeue_persistent_queue_items()
-    assert "spurious" in caplog.records[0].msg
-    assert not p.exists()
-    assert notifier2.retry_queues[0].qsize() == 1
-    when, queue_item = notifier2.retry_queues[0].get()
-    assert when <= int(time.time())
-    assert queue_item.addr == testaddr
-
-
-def test_start_and_stop_notification_threads(notifier, testaddr):
-    threads = notifier.start_notification_threads(None)
-    for retry_num, threadlist in threads.items():
-        for t in threadlist:
-            t.stop()
-            t.join()
-
-
-def test_multi_device_notifier(metadata, notifier, testaddr):
-    metadata.add_token_to_addr(testaddr, "01234")
-    metadata.add_token_to_addr(testaddr, "56789")
-    notifier.new_message_for_addr(testaddr, metadata)
-    reqmock = get_mocked_requests([200, 200])
-    NotifyThread(notifier, 0, None).retry_one(reqmock)
-    NotifyThread(notifier, 0, None).retry_one(reqmock)
-    assert notifier.retry_queues[0].qsize() == 0
-    assert notifier.retry_queues[1].qsize() == 0
-    url, data, timeout = reqmock.requests[0]
-    assert data == "01234"
-    url, data, timeout = reqmock.requests[1]
-    assert data == "56789"
-    assert metadata.get_tokens_for_addr(testaddr) == ["01234", "56789"]
-
-
-def test_notifier_thread_run_gone_removes_token(metadata, notifier, testaddr):
-    metadata.add_token_to_addr(testaddr, "01234")
-    metadata.add_token_to_addr(testaddr, "45678")
-    notifier.new_message_for_addr(testaddr, metadata)
-
-    reqmock = get_mocked_requests([410, 200])
-    NotifyThread(notifier, 0, metadata.remove_token_from_addr).retry_one(reqmock)
-    NotifyThread(notifier, 0, None).retry_one(reqmock)
-    url, data, timeout = reqmock.requests[0]
-    assert data == "01234"
-    url, data, timeout = reqmock.requests[1]
-    assert data == "45678"
-    assert metadata.get_tokens_for_addr(testaddr) == ["45678"]
-    assert notifier.retry_queues[0].qsize() == 0
-    assert notifier.retry_queues[1].qsize() == 0
-
-
-def test_persistent_queue_items(tmp_path, testaddr, token):
-    queue_item = PersistentQueueItem.create(tmp_path, testaddr, 432, token)
-    assert queue_item.addr == testaddr
-    assert queue_item.start_ts == 432
-    assert queue_item.token == token
-    item2 = PersistentQueueItem.read_from_path(queue_item.path)
-    assert item2.addr == testaddr
-    assert item2.start_ts == 432
-    assert item2.token == token
-    assert item2 == queue_item
-    item2.delete()
-    assert not item2.path.exists()
-    assert not queue_item < item2 and not item2 < queue_item

cmdeploy/pyproject.toml

@@ -19,7 +19,6 @@ dependencies = [
   "black",
   "pytest",
   "pytest-xdist", 
-  "imap_tools",
 ]
 
 [project.scripts]

cmdeploy/src/cmdeploy/__init__.py

@@ -1,7 +1,6 @@
 """
 Chat Mail pyinfra deploy.
 """
-
 import sys
 import importlib.resources
 import subprocess
@@ -102,17 +101,13 @@ def _install_remote_venv_with_chatmaild(config) -> None:
         "doveauth",
         "filtermail",
         "echobot",
-        "chatmail-metadata",
     ):
         params = dict(
             execpath=f"{remote_venv_dir}/bin/{fn}",
             config_path=remote_chatmail_inipath,
             remote_venv_dir=remote_venv_dir,
-            mail_domain=config.mail_domain,
-        )
-        source_path = importlib.resources.files(__package__).joinpath(
-            "service", f"{fn}.service.f"
         )
+        source_path = importlib.resources.files("chatmaild").joinpath(f"{fn}.service.f")
         content = source_path.read_text().format(**params).encode()
 
         files.put(
@@ -131,93 +126,6 @@ def _install_remote_venv_with_chatmaild(config) -> None:
         )
 
 
-def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
-    """Configures OpenDKIM"""
-    need_restart = False
-
-    main_config = files.template(
-        src=importlib.resources.files(__package__).joinpath("opendkim/opendkim.conf"),
-        dest="/etc/opendkim.conf",
-        user="root",
-        group="root",
-        mode="644",
-        config={"domain_name": domain, "opendkim_selector": dkim_selector},
-    )
-    need_restart |= main_config.changed
-
-    screen_script = files.put(
-        src=importlib.resources.files(__package__).joinpath("opendkim/screen.lua"),
-        dest="/etc/opendkim/screen.lua",
-        user="root",
-        group="root",
-        mode="644",
-    )
-    need_restart |= screen_script.changed
-
-    final_script = files.put(
-        src=importlib.resources.files(__package__).joinpath("opendkim/final.lua"),
-        dest="/etc/opendkim/final.lua",
-        user="root",
-        group="root",
-        mode="644",
-    )
-    need_restart |= final_script.changed
-
-    files.directory(
-        name="Add opendkim directory to /etc",
-        path="/etc/opendkim",
-        user="opendkim",
-        group="opendkim",
-        mode="750",
-        present=True,
-    )
-
-    keytable = files.template(
-        src=importlib.resources.files(__package__).joinpath("opendkim/KeyTable"),
-        dest="/etc/dkimkeys/KeyTable",
-        user="opendkim",
-        group="opendkim",
-        mode="644",
-        config={"domain_name": domain, "opendkim_selector": dkim_selector},
-    )
-    need_restart |= keytable.changed
-
-    signing_table = files.template(
-        src=importlib.resources.files(__package__).joinpath("opendkim/SigningTable"),
-        dest="/etc/dkimkeys/SigningTable",
-        user="opendkim",
-        group="opendkim",
-        mode="644",
-        config={"domain_name": domain, "opendkim_selector": dkim_selector},
-    )
-    need_restart |= signing_table.changed
-    files.directory(
-        name="Add opendkim socket directory to /var/spool/postfix",
-        path="/var/spool/postfix/opendkim",
-        user="opendkim",
-        group="opendkim",
-        mode="750",
-        present=True,
-    )
-
-    apt.packages(
-        name="apt install opendkim opendkim-tools",
-        packages=["opendkim", "opendkim-tools"],
-    )
-
-    if not host.get_fact(File, f"/etc/dkimkeys/{dkim_selector}.private"):
-        server.shell(
-            name="Generate OpenDKIM domain keys",
-            commands=[
-                f"opendkim-genkey -D /etc/dkimkeys -d {domain} -s {dkim_selector}"
-            ],
-            _sudo=True,
-            _sudo_user="opendkim",
-        )
-
-    return need_restart
-
-
 def _install_mta_sts_daemon() -> bool:
     need_restart = False
 
@@ -292,16 +200,6 @@ def _configure_postfix(config: Config, debug: bool = False) -> bool:
     )
     need_restart |= header_cleanup.changed
 
-    # Login map that 1:1 maps email address to login.
-    login_map = files.put(
-        src=importlib.resources.files(__package__).joinpath("postfix/login_map"),
-        dest="/etc/postfix/login_map",
-        user="root",
-        group="root",
-        mode="644",
-    )
-    need_restart |= login_map.changed
-
     return need_restart
 
 
@@ -327,30 +225,6 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
         mode="644",
     )
     need_restart |= auth_config.changed
-    lua_push_notification_script = files.put(
-        src=importlib.resources.files(__package__).joinpath(
-            "dovecot/push_notification.lua"
-        ),
-        dest="/etc/dovecot/push_notification.lua",
-        user="root",
-        group="root",
-        mode="644",
-    )
-    need_restart |= lua_push_notification_script.changed
-
-    sieve_script = files.put(
-        src=importlib.resources.files(__package__).joinpath("dovecot/default.sieve"),
-        dest="/etc/dovecot/default.sieve",
-        user="root",
-        group="root",
-        mode="644",
-    )
-    need_restart |= sieve_script.changed
-    if sieve_script.changed:
-        server.shell(
-            name="compile sieve script",
-            commands=["/usr/bin/sievec /etc/dovecot/default.sieve"],
-        )
 
     files.template(
         src=importlib.resources.files(__package__).joinpath("dovecot/expunge.cron.j2"),
@@ -431,20 +305,113 @@ def _configure_nginx(domain: str, debug: bool = False) -> bool:
     return need_restart
 
 
-def _remove_rspamd() -> None:
-    """Remove rspamd"""
-    apt.packages(name="Remove rspamd", packages="rspamd", present=False)
+def remove_opendkim() -> None:
+    """Remove OpenDKIM, deprecated"""
+    files.file(
+        name="Remove legacy opendkim.conf",
+        path="/etc/opendkim.conf",
+        present=False,
+    )
+
+    files.directory(
+        name="Remove legacy opendkim socket directory from /var/spool/postfix",
+        path="/var/spool/postfix/opendkim",
+        present=False,
+    )
+
+    apt.packages(name="Remove openDKIM", packages="opendkim", present=False)
+
+
+def _configure_rspamd(dkim_selector: str, mail_domain: str) -> bool:
+    """Configures rspamd for Rate Limiting."""
+    need_restart = False
+
+    apt.packages(
+        name="apt install rspamd",
+        packages="rspamd",
+    )
+
+    for module in ["phishing", "rbl", "hfilter", "ratelimit"]:
+        disabled_module_conf = files.put(
+            name=f"disable {module} rspamd plugin",
+            src=importlib.resources.files(__package__).joinpath("rspamd/disabled.conf"),
+            dest=f"/etc/rspamd/local.d/{module}.conf",
+            user="root",
+            group="root",
+            mode="644",
+        )
+        need_restart |= disabled_module_conf.changed
+
+    options_inc = files.put(
+        name="disable fuzzy checks",
+        src=importlib.resources.files(__package__).joinpath("rspamd/options.inc"),
+        dest="/etc/rspamd/local.d/options.inc",
+        user="root",
+        group="root",
+        mode="644",
+    )
+    need_restart |= options_inc.changed
+
+    # https://rspamd.com/doc/modules/force_actions.html
+    force_actions_conf = files.put(
+        name="Set up rules to reject on DKIM, SPF and DMARC fails",
+        src=importlib.resources.files(__package__).joinpath(
+            "rspamd/force_actions.conf"
+        ),
+        dest="/etc/rspamd/local.d/force_actions.conf",
+        user="root",
+        group="root",
+        mode="644",
+    )
+    need_restart |= force_actions_conf.changed
+
+    dkim_directory = "/var/lib/rspamd/dkim/"
+    dkim_key_path = f"{dkim_directory}{mail_domain}.{dkim_selector}.key"
+    dkim_dns_file = f"{dkim_directory}{mail_domain}.{dkim_selector}.zone"
+
+    dkim_config = files.template(
+        src=importlib.resources.files(__package__).joinpath(
+            "rspamd/dkim_signing.conf.j2"
+        ),
+        dest="/etc/rspamd/local.d/dkim_signing.conf",
+        user="root",
+        group="root",
+        mode="644",
+        config={
+            "dkim_selector": str(dkim_selector),
+            "mail_domain": mail_domain,
+            "dkim_key_path": dkim_key_path,
+        },
+    )
+    need_restart |= dkim_config.changed
+
+    files.directory(
+        name="ensure DKIM key directory exists",
+        path=dkim_directory,
+        present=True,
+        user="_rspamd",
+        group="_rspamd",
+    )
+
+    if not host.get_fact(File, dkim_key_path):
+        server.shell(
+            name="Generate DKIM domain keys with rspamd",
+            commands=[
+                f"rspamadm dkim_keygen -b 2048 -s {dkim_selector} -d {mail_domain} -k {dkim_key_path} > {dkim_dns_file}"
+            ],
+            _sudo=True,
+            _sudo_user="_rspamd",
+        )
+
+    return need_restart
 
 
 def check_config(config):
     mail_domain = config.mail_domain
     if mail_domain != "testrun.org" and not mail_domain.endswith(".testrun.org"):
         blocked_words = "merlinux schmieder testrun.org".split()
-        for key in config.__dict__:
-            value = config.__dict__[key]
-            if key.startswith("privacy") and any(
-                x in str(value) for x in blocked_words
-            ):
+        for value in config.__dict__.values():
+            if any(x in str(value) for x in blocked_words):
                 raise ValueError(
                     f"please set your own privacy contacts/addresses in {config._inipath}"
                 )
@@ -462,51 +429,9 @@ def deploy_chatmail(config_path: Path) -> None:
 
     from .www import build_webpages
 
+    apt.update(name="apt update", cache_time=24 * 3600)
     server.group(name="Create vmail group", group="vmail", system=True)
     server.user(name="Create vmail user", user="vmail", group="vmail", system=True)
-    server.group(name="Create opendkim group", group="opendkim", system=True)
-    server.user(
-        name="Create opendkim user",
-        user="opendkim",
-        groups=["opendkim"],
-        system=True,
-    )
-    server.user(
-        name="Add postfix user to opendkim group for socket access",
-        user="postfix",
-        groups=["opendkim"],
-        system=True,
-    )
-    server.user(name="Create echobot user", user="echobot", system=True)
-
-    server.shell(
-        name="Fix file owner in /home/vmail",
-        commands=["test -d /home/vmail && chown -R vmail:vmail /home/vmail"],
-    )
-
-    # Add our OBS repository for dovecot_no_delay
-    files.put(
-        name = "Add Deltachat OBS GPG key to apt keyring",
-        src = importlib.resources.files(__package__).joinpath("obs-home-deltachat.gpg"),
-        dest = "/etc/apt/keyrings/obs-home-deltachat.gpg",
-        user="root",
-        group="root",
-        mode="644",
-    )
-
-    files.line(
-        name = "Add DeltaChat OBS home repository to sources.list",
-        path = "/etc/apt/sources.list",
-        line = "deb [signed-by=/etc/apt/keyrings/obs-home-deltachat.gpg] https://download.opensuse.org/repositories/home:/deltachat/Debian_12/ ./",
-        ensure_newline = True,
-    )
-
-    apt.update(name="apt update", cache_time=24 * 3600)
-
-    apt.packages(
-        name="Install rsync",
-        packages=["rsync"],
-    )
 
     # Run local DNS resolver `unbound`.
     # `resolvconf` takes care of setting up /etc/resolv.conf
@@ -531,6 +456,7 @@ def deploy_chatmail(config_path: Path) -> None:
 
     # Deploy acmetool to have TLS certificates.
     deploy_acmetool(
+        nginx_hook=True,
         domains=[mail_domain, f"mta-sts.{mail_domain}", f"www.{mail_domain}"],
     )
 
@@ -541,7 +467,7 @@ def deploy_chatmail(config_path: Path) -> None:
 
     apt.packages(
         name="Install Dovecot",
-        packages=["dovecot-imapd", "dovecot-lmtpd", "dovecot-sieve"],
+        packages=["dovecot-imapd", "dovecot-lmtpd"],
     )
 
     apt.packages(
@@ -568,15 +494,15 @@ def deploy_chatmail(config_path: Path) -> None:
     mta_sts_need_restart = _install_mta_sts_daemon()
     nginx_need_restart = _configure_nginx(mail_domain)
 
-    _remove_rspamd()
-    opendkim_need_restart = _configure_opendkim(mail_domain, "opendkim")
+    remove_opendkim()
+    rspamd_need_restart = _configure_rspamd("dkim", mail_domain)
 
     systemd.service(
-        name="Start and enable OpenDKIM",
-        service="opendkim.service",
+        name="Start and enable rspamd",
+        service="rspamd.service",
         running=True,
         enabled=True,
-        restarted=opendkim_need_restart,
+        restarted=rspamd_need_restart,
     )
 
     systemd.service(
@@ -588,17 +514,6 @@ def deploy_chatmail(config_path: Path) -> None:
         restarted=mta_sts_need_restart,
     )
 
-    # Dovecot should be started before Postfix
-    # because it creates authentication socket
-    # required by Postfix.
-    systemd.service(
-        name="Start and enable Dovecot",
-        service="dovecot.service",
-        running=True,
-        enabled=True,
-        restarted=dovecot_need_restart,
-    )
-
     systemd.service(
         name="Start and enable Postfix",
         service="postfix.service",
@@ -607,6 +522,14 @@ def deploy_chatmail(config_path: Path) -> None:
         restarted=postfix_need_restart,
     )
 
+    systemd.service(
+        name="Start and enable Dovecot",
+        service="dovecot.service",
+        running=True,
+        enabled=True,
+        restarted=dovecot_need_restart,
+    )
+
     systemd.service(
         name="Start and enable nginx",
         service="nginx.service",

cmdeploy/src/cmdeploy/acmetool/__init__.py

@@ -5,7 +5,7 @@ from pyinfra import host
 from pyinfra.facts.systemd import SystemdStatus
 
 
-def deploy_acmetool(email="", domains=[]):
+def deploy_acmetool(nginx_hook=False, email="", domains=[]):
     """Deploy acmetool."""
     apt.packages(
         name="Install acmetool",
@@ -20,13 +20,16 @@ def deploy_acmetool(email="", domains=[]):
         mode="644",
     )
 
-    files.put(
-        src=importlib.resources.files(__package__).joinpath("acmetool.hook").open("rb"),
-        dest="/usr/lib/acme/hooks/nginx",
-        user="root",
-        group="root",
-        mode="744",
-    )
+    if nginx_hook:
+        files.put(
+            src=importlib.resources.files(__package__)
+            .joinpath("acmetool.hook")
+            .open("rb"),
+            dest="/usr/lib/acme/hooks/nginx",
+            user="root",
+            group="root",
+            mode="744",
+        )
 
     files.template(
         src=importlib.resources.files(__package__).joinpath("response-file.yaml.j2"),
@@ -71,5 +74,5 @@ def deploy_acmetool(email="", domains=[]):
 
     server.shell(
         name=f"Request certificate for: { ', '.join(domains) }",
-        commands=[f"acmetool want --xlog.severity=debug { ' '.join(domains)}"],
+        commands=[f"acmetool want { ' '.join(domains)}"],
     )

cmdeploy/src/cmdeploy/acmetool/acmetool.cron

@@ -1,4 +1,4 @@
 SHELL=/bin/sh
 PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
 MAILTO=root
-20 16 * * * root /usr/bin/acmetool --batch reconcile && systemctl reload dovecot && systemctl reload postfix
+20 16 * * * root /usr/bin/acmetool --batch reconcile

cmdeploy/src/cmdeploy/acmetool/acmetool.hook

@@ -3,5 +3,3 @@ set -e
 EVENT_NAME="$1"
 [ "$EVENT_NAME" = "live-updated" ] || exit 42
 systemctl restart nginx.service
-systemctl reload dovecot.service
-systemctl reload postfix.service

cmdeploy/src/cmdeploy/acmetool/response-file.yaml.j2

@@ -1,2 +1,2 @@
 "acme-enter-email": "{{ email }}"
-"acme-agreement:https://letsencrypt.org/documents/LE-SA-v1.4-April-3-2024.pdf": true
+"acme-agreement:https://letsencrypt.org/documents/LE-SA-v1.3-September-21-2022.pdf": true

cmdeploy/src/cmdeploy/chatmail.zone.f

@@ -6,10 +6,10 @@ _submissions._tcp.{chatmail_domain}. SRV 0 1 465 {chatmail_domain}.
 _imap._tcp.{chatmail_domain}.        SRV 0 1 143 {chatmail_domain}.
 _imaps._tcp.{chatmail_domain}.       SRV 0 1 993 {chatmail_domain}.
 {chatmail_domain}.                   CAA 128 issue "letsencrypt.org;accounturi={acme_account_url}"
-{chatmail_domain}.                   TXT "v=spf1 a:{chatmail_domain} ~all"
+{chatmail_domain}.                   TXT "v=spf1 a:{chatmail_domain} -all"
 _dmarc.{chatmail_domain}.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
 _mta-sts.{chatmail_domain}.          TXT "v=STSv1; id={sts_id}"
 mta-sts.{chatmail_domain}.           CNAME {chatmail_domain}.
 www.{chatmail_domain}.               CNAME {chatmail_domain}.
+_smtp._tls.{chatmail_domain}.        TXT "v=TLSRPTv1;rua=mailto:{email}"
 {dkim_entry}
-_adsp._domainkey.{chatmail_domain}.  TXT "dkim=discardable"

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -2,7 +2,6 @@
 Provides the `cmdeploy` entry point function,
 along with command line option and subcommand parsing.
 """
-
 import argparse
 import shutil
 import subprocess

cmdeploy/src/cmdeploy/dns.py

@@ -11,11 +11,6 @@ class DNS:
         self.session = requests.Session()
         self.out = out
         self.ssh = f"ssh root@{mail_domain} -- "
-        self.out.shell_output(
-            f"{ self.ssh }'apt-get update && apt-get install -y dnsutils'",
-            timeout=60,
-            no_print=True,
-        )
         try:
             self.shell(f"unbound-control flush_zone {mail_domain}")
         except subprocess.CalledProcessError:
@@ -39,11 +34,12 @@ class DNS:
         cmd = "ip a | grep inet6 | grep 'scope global' | sed -e 's#/64 scope global##' | sed -e 's#inet6##'"
         return self.shell(cmd).strip()
 
-    def get(self, typ: str, domain: str) -> str:
-        """Get a DNS entry or empty string if there is none."""
+    def get(self, typ: str, domain: str) -> str | None:
+        """Get a DNS entry"""
         dig_result = self.shell(f"dig -r -q {domain} -t {typ} +short")
         line = dig_result.partition("\n")[0]
-        return line
+        if line:
+            return line
 
     def check_ptr_record(self, ip: str, mail_domain) -> bool:
         """Check the PTR record for an IPv4 or IPv6 address."""
@@ -58,25 +54,27 @@ def show_dns(args, out) -> int:
     ssh = f"ssh root@{mail_domain}"
     dns = DNS(out, mail_domain)
 
+    def read_dkim_entries(entry):
+        lines = []
+        for line in entry.split("\n"):
+            if line.startswith(";") or not line.strip():
+                continue
+            line = line.replace("\t", " ")
+            lines.append(line)
+        lines[0] = f"dkim._domainkey.{mail_domain}. IN TXT " + lines[0].strip(
+            "dkim._domainkey IN TXT "
+        )
+        return "\n".join(lines)
+
     print("Checking your DKIM keys and DNS entries...")
     try:
         acme_account_url = out.shell_output(f"{ssh} -- acmetool account-url")
     except subprocess.CalledProcessError:
         print("Please run `cmdeploy run` first.")
         return 1
-
-    dkim_selector = "opendkim"
-    dkim_pubkey = out.shell_output(
-        ssh + f" -- openssl rsa -in /etc/dkimkeys/{dkim_selector}.private"
-        " -pubout 2>/dev/null | awk '/-/{next}{printf(\"%s\",$0)}'"
+    dkim_entry = read_dkim_entries(
+        out.shell_output(f"{ssh} -- cat /var/lib/rspamd/dkim/{mail_domain}.dkim.zone")
     )
-    dkim_entry_value = f"v=DKIM1;k=rsa;p={dkim_pubkey};s=email;t=s"
-    dkim_entry_str = ""
-    while len(dkim_entry_value) >= 255:
-        dkim_entry_str += '"' + dkim_entry_value[:255] + '" '
-        dkim_entry_value = dkim_entry_value[255:]
-    dkim_entry_str += '"' + dkim_entry_value + '"'
-    dkim_entry = f"{dkim_selector}._domainkey.{mail_domain}. TXT {dkim_entry_str}"
 
     ipv6 = dns.get_ipv6()
     reverse_ipv6 = dns.check_ptr_record(ipv6, mail_domain)
@@ -89,6 +87,7 @@ def show_dns(args, out) -> int:
             f.read()
             .format(
                 acme_account_url=acme_account_url,
+                email=f"root@{args.config.mail_domain}",
                 sts_id=datetime.datetime.now().strftime("%Y%m%d%H%M"),
                 chatmail_domain=args.config.mail_domain,
                 dkim_entry=dkim_entry,
@@ -104,9 +103,11 @@ def show_dns(args, out) -> int:
             return 0
         except TypeError:
             pass
+        started_dkim_parsing = False
         for line in zonefile.splitlines():
             line = line.format(
                 acme_account_url=acme_account_url,
+                email=f"root@{args.config.mail_domain}",
                 sts_id=datetime.datetime.now().strftime("%Y%m%d%H%M"),
                 chatmail_domain=args.config.mail_domain,
                 dkim_entry=dkim_entry,
@@ -130,23 +131,28 @@ def show_dns(args, out) -> int:
                 current = dns.get("SRV", domain[:-1])
                 if current != f"{prio} {weight} {port} {value}":
                     to_print.append(line)
-            if " TXT " in line:
+            if "  TXT " in line:
                 domain, value = line.split(" TXT ")
                 current = dns.get("TXT", domain.strip()[:-1])
                 if domain.startswith("_mta-sts."):
                     if current:
                         if current.split("id=")[0] == value.split("id=")[0]:
                             continue
-
-                # TXT records longer than 255 bytes
-                # are split into multiple <character-string>s.
-                # This typically happens with DKIM record
-                # which contains long RSA key.
-                #
-                # Removing `" "` before comparison
-                # to get back a single string.
-                if current.replace('" "', "") != value.replace('" "', ""):
+                if current != value:
                     to_print.append(line)
+            if " IN TXT ( " in line:
+                started_dkim_parsing = True
+                dkim_lines = [line]
+            if started_dkim_parsing and line.startswith('"'):
+                dkim_lines.append(" " + line)
+        domain, data = "\n".join(dkim_lines).split(" IN TXT ")
+        current = dns.get("TXT", domain.strip()[:-1])
+        if current:
+            current = "( %s" % (current.replace('" "', '"\n "'))
+            if current != data:
+                to_print.append(dkim_entry)
+        else:
+            to_print.append(dkim_entry)
 
     exit_code = 0
     if to_print:

cmdeploy/src/cmdeploy/dovecot/auth.conf

@@ -1,4 +1,4 @@
-uri = proxy:/run/doveauth/doveauth.socket:auth
+uri = proxy:/run/dovecot/doveauth.socket:auth
 iterate_disable = yes
 default_pass_scheme = plain
 # %E escapes characters " (double quote), ' (single quote) and \ (backslash) with \ (backslash).

cmdeploy/src/cmdeploy/dovecot/default.sieve

@@ -1,7 +0,0 @@
-require ["imap4flags"];
-
-# flag the message so it doesn't cause a push notification
-
-if header :is ["Auto-Submitted"] ["auto-replied", "auto-generated"] {
-	addflag "$Auto";
-}

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -13,21 +13,13 @@ auth_cache_size = 100M
 mail_debug = yes
 {% endif %}
 
-# Prevent warnings similar to:
-#   config: Warning: service auth { client_limit=1000 } is lower than required under max. load (10200). Counted for protocol services with service_count != 1: service lmtp { process_limit=100 } + service imap-urlauth-login { process_limit=100 } + service imap-login { process_limit=10000 }
-#   config: Warning: service anvil { client_limit=1000 } is lower than required under max. load (10103). Counted with: service imap-urlauth-login { process_limit=100 } + service imap-login { process_limit=10000 } + service auth { process_limit=1 }
-#   master: Warning: service(stats): client_limit (1000) reached, client connections are being dropped
-default_client_limit = 20000
-
-mail_server_admin = mailto:root@{{ config.mail_domain }}
-mail_server_comment = Chatmail server
 
 mail_plugins = quota
 
 # these are the capabilities Delta Chat cares about actually 
 # so let's keep the network overhead per login small
 # https://github.com/deltachat/deltachat-core-rust/blob/master/src/imap/capabilities.rs
-imap_capability = IMAP4rev1 IDLE MOVE QUOTA CONDSTORE NOTIFY METADATA XDELTAPUSH
+imap_capability = IMAP4rev1 IDLE MOVE QUOTA CONDSTORE NOTIFY
 
 
 # Authentication for system users.
@@ -77,32 +69,14 @@ mail_privileged_group = vmail
 ## Mail processes
 ##
 
-# Pass all IMAP METADATA requests to the server implementing Dovecot's dict protocol.
-mail_attribute_dict = proxy:/run/chatmail-metadata/metadata.socket:metadata
-
 # Enable IMAP COMPRESS (RFC 4978).
 # <https://datatracker.ietf.org/doc/html/rfc4978.html>
 protocol imap {
   mail_plugins = $mail_plugins imap_zlib imap_quota
-  imap_metadata = yes
 }
 
 protocol lmtp {
-  # quota plugin documentation:
-  # <https://doc.dovecot.org/configuration_manual/quota_plugin/>
-  #
-  # notify plugin is a dependency of push_notification plugin:
-  # <https://doc.dovecot.org/settings/plugin/notify-plugin/>
-  #
-  # push_notification plugin documentation:
-  # <https://doc.dovecot.org/configuration_manual/push_notification/>
-  #
-  # mail_lua and push_notification_lua are needed for Lua push notification handler.
-  # <https://doc.dovecot.org/configuration_manual/push_notification/#configuration>
-  #
-  # Sieve to mark messages that should not be notified as \Seen
-  # <https://doc.dovecot.org/configuration_manual/sieve/configuration/>
-  mail_plugins = $mail_plugins quota mail_lua notify push_notification push_notification_lua sieve
+  mail_plugins = $mail_plugins quota
 }
 
 plugin {
@@ -118,15 +92,7 @@ plugin {
   # quota_over_flag_value = TRUE
 }
 
-# push_notification configuration
-plugin {
-  # <https://doc.dovecot.org/configuration_manual/push_notification/#lua-lua>
-  push_notification_driver = lua:file=/etc/dovecot/push_notification.lua
-}
 
-plugin {
-  sieve_default = file:/etc/dovecot/default.sieve
-}
 
 service lmtp {
   user=vmail

cmdeploy/src/cmdeploy/dovecot/expunge.cron.j2

@@ -1,11 +1,10 @@
 # delete all mails after {{ config.delete_mails_after }} days, in the Inbox
-2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }}/*/cur -mtime +{{ config.delete_mails_after }} -type f -delete
 # or in any IMAP subfolder
-2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }}/*/.*/cur -mtime +{{ config.delete_mails_after }} -type f -delete
 # even if they are unseen
-2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }}/*/new -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }}/*/.*/new -mtime +{{ config.delete_mails_after }} -type f -delete
 # or only temporary (but then they shouldn't be around after {{ config.delete_mails_after }} days anyway).
-2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-3 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -name 'maildirsize' -type f -delete
+2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }}/*/tmp -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }}/*/.*/tmp -mtime +{{ config.delete_mails_after }} -type f -delete

cmdeploy/src/cmdeploy/dovecot/push_notification.lua

@@ -1,32 +0,0 @@
-function dovecot_lua_notify_begin_txn(user)
-  return user
-end
-
-function contains(v, needle)
-  for _, keyword in ipairs(v) do
-    if keyword == needle then
-      return true
-    end
-  end
-  return false
-end
-
-function dovecot_lua_notify_event_message_new(user, event)
-  local mbox = user:mailbox(event.mailbox)
-  mbox:sync()
-
-  if user.username ~= event.from_address then
-    -- Incoming message
-    if not contains(event.keywords, "$Auto") then
-      -- Not an Auto-Submitted message, notifying.
-
-      -- Notify METADATA server about new message.
-      mbox:metadata_set("/private/messagenew", "")
-    end
-  end
-
-  mbox:free()
-end
-
-function dovecot_lua_notify_end_txn(ctx, success)
-end

cmdeploy/src/cmdeploy/opendkim/KeyTable

@@ -1 +1 @@
-{{ config.opendkim_selector }}._domainkey.{{ config.domain_name }} {{ config.domain_name }}:{{ config.opendkim_selector }}:/etc/dkimkeys/{{ config.opendkim_selector }}.private
+dkim._domainkey.{{ config.domain_name }} {{ config.domain_name }}:{{ config.opendkim_selector }}:/etc/dkimkeys/dkim.private

cmdeploy/src/cmdeploy/opendkim/final.lua

@@ -1,28 +0,0 @@
-if odkim.internal_ip(ctx) == 1 then
-	-- Outgoing message will be signed,
-	-- no need to look for signatures.
-	return nil
-end
-
-nsigs = odkim.get_sigcount(ctx)
-if nsigs == nil then
-	return nil
-end
-
-for i = 1, nsigs do
-        sig = odkim.get_sighandle(ctx, i - 1)
-        sigres = odkim.sig_result(sig)
-
-	-- All signatures that do not correspond to From: 
-	-- were ignored in screen.lua and return sigres -1.
-	-- 
-	-- Any valid signature that was not ignored like this
-	-- means the message is acceptable.
-	if sigres == 0 then
-		return nil
-	end	
-end
-
-odkim.set_reply(ctx, "554", "5.7.1", "No valid DKIM signature found")
-odkim.set_result(ctx, SMFIS_REJECT)
-return nil

cmdeploy/src/cmdeploy/opendkim/opendkim.conf

@@ -8,12 +8,10 @@ SyslogSuccess		yes
 # oversigned, because it is often the identity key used by reputation systems
 # and thus somewhat security sensitive.
 Canonicalization	relaxed/simple
+#Mode			sv
+#SubDomains		no
 OversignHeaders		From
 
-On-BadSignature         reject
-On-KeyNotFound          reject
-On-NoSignature          reject
-
 # Signing domain, selector, and key (required). For example, perform signing
 # for domain "example.com" with selector "2020" (2020._domainkey.example.com),
 # using the private key stored in /etc/dkimkeys/example.private. More granular
@@ -24,15 +22,6 @@ KeyFile		  /etc/dkimkeys/{{ config.opendkim_selector }}.private
 KeyTable          /etc/dkimkeys/KeyTable
 SigningTable      refile:/etc/dkimkeys/SigningTable
 
-# Sign Autocrypt header in addition to the default specified in RFC 6376.
-SignHeaders *,+autocrypt
-
-# Script to ignore signatures that do not correspond to the From: domain.
-ScreenPolicyScript /etc/opendkim/screen.lua
-
-# Script to reject mails without a valid DKIM signature.
-FinalPolicyScript /etc/opendkim/final.lua
-
 # In Debian, opendkim runs as user "opendkim". A umask of 007 is required when
 # using a local socket with MTAs that access the socket as a non-privileged
 # user (for example, Postfix). You may need to add user "postfix" to group
@@ -40,10 +29,22 @@ FinalPolicyScript /etc/opendkim/final.lua
 UserID			opendkim
 UMask			007
 
+# Socket for the MTA connection (required). If the MTA is inside a chroot jail,
+# it must be ensured that the socket is accessible. In Debian, Postfix runs in
+# a chroot in /var/spool/postfix, therefore a Unix socket would have to be
+# configured as shown on the last line below.
+#Socket			local:/run/opendkim/opendkim.sock
+#Socket			inet:8891@localhost
+#Socket			inet:8891
 Socket			local:/var/spool/postfix/opendkim/opendkim.sock
 
 PidFile			/run/opendkim/opendkim.pid
 
+# Hosts for which to sign rather than verify, default is 127.0.0.1. See the
+# OPERATION section of opendkim(8) for more information.
+#InternalHosts		192.168.0.0/16, 10.0.0.0/8, 172.16.0.0/12
+
 # The trust anchor enables DNSSEC. In Debian, the trust anchor file is provided
 # by the package dns-root-data.
 TrustAnchorFile		/usr/share/dns/root.key
+#Nameservers		127.0.0.1

cmdeploy/src/cmdeploy/opendkim/screen.lua

@@ -1,21 +0,0 @@
--- Ignore signatures that do not correspond to the From: domain.
-
-from_domain = odkim.get_fromdomain(ctx)
-if from_domain == nil then
-	return nil
-end
-
-n = odkim.get_sigcount(ctx)
-if n == nil then
-	return nil
-end
-
-for i = 1, n do
-	sig = odkim.get_sighandle(ctx, i - 1)
-	sig_domain = odkim.sig_getdomain(sig)
-	if from_domain ~= sig_domain then
-		odkim.sig_ignore(sig)
-	end
-end
-
-return nil

cmdeploy/src/cmdeploy/postfix/login_map

@@ -1 +0,0 @@
-/^(.*)$/	${1}

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -23,31 +23,6 @@ smtp_tls_CApath=/etc/ssl/certs
 smtp_tls_security_level=may
 smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
 smtp_tls_policy_maps = socketmap:inet:127.0.0.1:8461:postfix
-smtpd_tls_protocols = >=TLSv1.2
-
-# Disable anonymous cipher suites
-# and known insecure algorithms.
-#
-# Disabling anonymous ciphers
-# does not generally improve security
-# because clients that want to verify certificate
-# will not select them anyway,
-# but makes cipher suite list shorter and security scanners happy.
-# See <https://www.postfix.org/TLS_README.html> for discussion.
-#
-# Only ancient insecure ciphers should be disabled here
-# as MTA clients that do not support more secure cipher
-# likely do not support MTA-STS either and will
-# otherwise fall back to using plaintext connection.
-smtpd_tls_exclude_ciphers = aNULL, RC4, MD5, DES
-
-# Override client's preference order.
-# <https://www.postfix.org/postconf.5.html#tls_preempt_cipherlist>
-#
-# This is mostly to ensure cipher suites with forward secrecy
-# are preferred over non cipher suites without forward secrecy.
-# See <https://www.postfix.org/FORWARD_SECRECY_README.html#server_fs>.
-tls_preempt_cipherlist = yes
 
 smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination
 myhostname = {{ config.mail_domain }}
@@ -71,9 +46,7 @@ inet_protocols = all
 virtual_transport = lmtp:unix:private/dovecot-lmtp
 virtual_mailbox_domains = {{ config.mail_domain }}
 
-mua_client_restrictions = permit_sasl_authenticated, reject
-mua_sender_restrictions = reject_sender_login_mismatch, permit_sasl_authenticated, reject
-mua_helo_restrictions = permit_mynetworks, reject_invalid_helo_hostname, reject_non_fqdn_helo_hostname, permit
+smtpd_milters = inet:127.0.0.1:11332
+non_smtpd_milters = $smtpd_milters
 
-# 1:1 map MAIL FROM to SASL login name.
-smtpd_sender_login_maps = regexp:/etc/postfix/login_map
+header_checks = regexp:/etc/postfix/submission_header_cleanup

cmdeploy/src/cmdeploy/postfix/master.cf.j2

@@ -11,10 +11,13 @@
 # ==========================================================================
 {% if debug == true %}
 smtp      inet  n       -       y       -       -       smtpd -v
-{%- else %}
+{% else %}
 smtp      inet  n       -       y       -       -       smtpd 
-{%- endif %}
-  -o smtpd_milters=unix:opendkim/opendkim.sock
+{% endif %}
+#smtp      inet  n       -       y       -       1       postscreen
+#smtpd     pass  -       -       y       -       -       smtpd
+#dnsblog   unix  -       -       y       -       0       dnsblog
+#tlsproxy  unix  -       -       y       -       0       tlsproxy
 submission inet n       -       y       -       -       smtpd
   -o syslog_name=postfix/submission
   -o smtpd_tls_security_level=encrypt
@@ -31,7 +34,6 @@ submission inet n       -       y       -       -       smtpd
   -o milter_macro_daemon_name=ORIGINATING
   -o smtpd_client_connection_count_limit=1000
   -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
-  -o cleanup_service_name=authclean
 smtps     inet  n       -       y       -       -       smtpd
   -o syslog_name=postfix/smtps
   -o smtpd_tls_wrappermode=yes
@@ -48,7 +50,6 @@ smtps     inet  n       -       y       -       -       smtpd
   -o smtpd_client_connection_count_limit=1000
   -o milter_macro_daemon_name=ORIGINATING
   -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
-  -o cleanup_service_name=authclean
 #628       inet  n       -       y       -       -       qmqpd
 pickup    unix  n       -       y       60      1       pickup
 cleanup   unix  n       -       y       -       0       cleanup
@@ -79,14 +80,3 @@ filter    unix -        n       n       -       -       lmtp
 # Local SMTP server for reinjecting filered mail.
 localhost:{{ config.postfix_reinject_port }} inet  n       -       n       -       10      smtpd
   -o syslog_name=postfix/reinject
-  -o smtpd_milters=unix:opendkim/opendkim.sock
-  -o cleanup_service_name=authclean
-
-# Cleanup `Received` headers for authenticated mail
-# to avoid leaking client IP.
-#
-# We do not do this for received mails
-# as this will break DKIM signatures
-# if `Received` header is signed.
-authclean unix  n       -       -       -       0       cleanup
-  -o header_checks=regexp:/etc/postfix/submission_header_cleanup

cmdeploy/src/cmdeploy/rspamd/disabled.conf

@@ -0,0 +1 @@
+enabled = false;

cmdeploy/src/cmdeploy/rspamd/dkim_signing.conf.j2

@@ -0,0 +1,10 @@
+selector = {{ config.dkim_selector }}
+use_esld = false  # don't cut c1.testrun.org down to testrun.org
+domain = {
+    {{ config.mail_domain }} {
+        selectors [
+            selector = {{ config.dkim_selector }}
+            path = {{ config.dkim_key_path }}
+        ]
+    }
+}

cmdeploy/src/cmdeploy/rspamd/force_actions.conf

@@ -0,0 +1,60 @@
+rules {
+    ## Reject on missing or invalid DKIM signatures.
+    ##
+    ## We require DKIM signature on incoming mails regardless of DMARC policy.
+
+    # R_DKIM_REJECT: DKIM reject inserted by `dkim` module.
+    REJECT_INVALID_DKIM {
+        action = "reject";
+        expression = "R_DKIM_REJECT";
+        message = "Rejected due to invalid DKIM signature";
+    }
+
+    # R_DKIM_PERMFAIL: permanent failure inserted by `dkim` module e.g. no DKIM DNS record found.
+    REJECT_PERMFAIL_DKIM {
+        action = "reject";
+        expression = "R_DKIM_PERMFAIL";
+        message = "Rejected due to missing DKIM DNS entry";
+    }
+
+    # No DKIM signature (R_DKIM_NA symbol inserted by `dkim` module).
+    REJECT_MISSING_DKIM {
+        action = "reject";
+        expression = "R_DKIM_NA";
+        message = "Rejected due to missing DKIM signature";
+    }
+
+
+    ## Reject on SPF failure.
+
+    # - SPF failure (R_SPF_FAIL)
+    # - SPF permanent failure, e.g. failed to resolve DNS record referenced from SPF (R_SPF_PERMFAIL)
+    REJECT_SPF {
+        action = "reject";
+        expression = "R_SPF_FAIL | R_SPF_PERMFAIL";
+        message = "Rejected due to failed SPF check";
+    }
+
+    # Reject on DMARC policy check failure.
+    REJECT_DMARC {
+        action = "reject";
+        expression = "DMARC_POLICY_REJECT";
+        message = "Rejected due to DMARC policy";
+    }
+
+
+    # Do not reject if:
+    # - R_DKIM_TEMPFAIL, it is a DNS resolution failure
+    #   and we do not want to lose messages because of faulty network.
+    #
+    # - R_SPF_SOFTFAIL
+    # - R_SPF_NEUTRAL
+    # - R_SPF_DNSFAIL
+    # - R_SPF_NA
+    #
+    # - DMARC_DNSFAIL
+    # - DMARC_NA
+    # - DMARC_POLICY_SOFTFAIL
+    # - DMARC_POLICY_QUARANTINE
+    # - DMARC_BAD_POLICY
+}

cmdeploy/src/cmdeploy/rspamd/options.inc

@@ -0,0 +1 @@
+filters = "dkim";

cmdeploy/src/cmdeploy/service/chatmail-metadata.service.f

@@ -1,12 +0,0 @@
-[Unit]
-Description=Chatmail dict proxy for IMAP METADATA
-
-[Service]
-ExecStart={execpath} /run/chatmail-metadata/metadata.socket /home/vmail/mail/{mail_domain}
-Restart=always
-RestartSec=30
-User=vmail
-RuntimeDirectory=chatmail-metadata
-
-[Install]
-WantedBy=multi-user.target

cmdeploy/src/cmdeploy/service/echobot.service.f

@@ -1,67 +0,0 @@
-[Unit]
-Description=Chatmail echo bot for testing it works
-
-[Service]
-ExecStart={execpath} {config_path}
-Environment="PATH={remote_venv_dir}:$PATH"
-Restart=always
-RestartSec=30
-
-User=echobot
-Group=echobot
-
-# Create /var/lib/echobot
-StateDirectory=echobot
-
-# Create /run/echobot
-#
-# echobot stores /run/echobot/password
-# with a password there, which doveauth then reads.
-RuntimeDirectory=echobot
-
-WorkingDirectory=/var/lib/echobot
-
-# Apply security restrictions suggested by
-#   systemd-analyze security echobot.service
-CapabilityBoundingSet=
-LockPersonality=true
-MemoryDenyWriteExecute=true
-NoNewPrivileges=true
-PrivateDevices=true
-PrivateMounts=true
-PrivateTmp=true
-
-# We need to know about doveauth user to give it access to /run/echobot/password
-PrivateUsers=false
-
-ProtectClock=true
-ProtectControlGroups=true
-ProtectHostname=true
-ProtectKernelLogs=true
-ProtectKernelModules=true
-ProtectKernelTunables=true
-ProtectProc=noaccess
-
-# Should be "strict", but we currently write /accounts folder in a protected path
-ProtectSystem=full
-
-RemoveIPC=true
-RestrictAddressFamilies=AF_INET AF_INET6
-RestrictNamespaces=true
-RestrictRealtime=true
-RestrictSUIDSGID=true
-SystemCallArchitectures=native
-SystemCallFilter=~@clock
-SystemCallFilter=~@cpu-emulation
-SystemCallFilter=~@debug
-SystemCallFilter=~@module
-SystemCallFilter=~@mount
-SystemCallFilter=~@obsolete
-SystemCallFilter=~@raw-io
-SystemCallFilter=~@reboot
-SystemCallFilter=~@resources
-SystemCallFilter=~@swap
-UMask=0077
-
-[Install]
-WantedBy=multi-user.target

cmdeploy/src/cmdeploy/tests/online/test_0_login.py

@@ -1,7 +1,6 @@
 import pytest
 import threading
 import queue
-import socket
 
 from chatmaild.config import read_config
 from cmdeploy.cmdeploy import main
@@ -79,24 +78,3 @@ def test_concurrent_logins_same_account(
 
     for _ in conns:
         assert login_results.get()
-
-
-def test_no_vrfy(chatmail_config):
-    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    sock.connect((chatmail_config.mail_domain, 25))
-    banner = sock.recv(1024)
-    print(banner)
-    sock.send(b"VRFY wrongaddress@%s\r\n" % (chatmail_config.mail_domain.encode(),))
-    result = sock.recv(1024)
-    print(result)
-    sock.send(b"VRFY echo@%s\r\n" % (chatmail_config.mail_domain.encode(),))
-    result2 = sock.recv(1024)
-    print(result2)
-    assert result[0:10] == result2[0:10]
-    sock.send(b"VRFY wrongaddress\r\n")
-    result = sock.recv(1024)
-    print(result)
-    sock.send(b"VRFY echo\r\n")
-    result2 = sock.recv(1024)
-    print(result2)
-    assert result[0:10] == result2[0:10] == b"252 2.0.0 "

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -42,25 +42,13 @@ def test_reject_forged_from(cmsetup, maildata, gencreds, lp, forgeaddr):
     assert "500" in str(e.value)
 
 
-def test_authenticated_from(cmsetup, maildata):
-    """Test that envelope FROM must be the same as login."""
-    user1, user2, user3 = cmsetup.gen_users(3)
-
-    msg = maildata("encrypted.eml", from_addr=user2.addr, to_addr=user3.addr)
-    with pytest.raises(smtplib.SMTPException) as e:
-        user1.smtp.sendmail(
-            from_addr=user2.addr, to_addrs=[user3.addr], msg=msg.as_string()
-        )
-    assert e.value.recipients[user3.addr][0] == 553
-
-
 @pytest.mark.parametrize("from_addr", ["fake@example.org", "fake@testrun.org"])
 def test_reject_missing_dkim(cmsetup, maildata, from_addr):
     """Test that emails with missing or wrong DMARC, DKIM, and SPF entries are rejected."""
     recipient = cmsetup.gen_users(1)[0]
     msg = maildata("plain.eml", from_addr=from_addr, to_addr=recipient.addr).as_string()
     with smtplib.SMTP(cmsetup.maildomain, 25) as s:
-        with pytest.raises(smtplib.SMTPDataError, match="No valid DKIM signature"):
+        with pytest.raises(smtplib.SMTPDataError, match="missing DKIM signature"):
             s.sendmail(from_addr=from_addr, to_addrs=recipient.addr, msg=msg)
 
 
@@ -83,18 +71,3 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
             assert b"4.7.1: Too much mail from" in outcome[1]
             return
     pytest.fail("Rate limit was not exceeded")
-
-
-def test_expunged(remote, chatmail_config):
-    outdated_days = int(chatmail_config.delete_mails_after) + 1
-    find_cmds = [
-        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/cur/*' -mtime +{outdated_days} -type f",
-        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/.*/cur/*' -mtime +{outdated_days} -type f",
-        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/new/*' -mtime +{outdated_days} -type f",
-        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/.*/new/*' -mtime +{outdated_days} -type f",
-        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/tmp/*' -mtime +{outdated_days} -type f",
-        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/.*/tmp/*' -mtime +{outdated_days} -type f",
-    ]
-    for cmd in find_cmds:
-        for line in remote.iter_output(cmd):
-            assert not line

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -5,46 +5,6 @@ import random
 import pytest
 import requests
 import ipaddress
-import imap_tools
-
-
-@pytest.fixture
-def imap_mailbox(cmfactory):
-    (ac1,) = cmfactory.get_online_accounts(1)
-    user = ac1.get_config("addr")
-    password = ac1.get_config("mail_pw")
-    mailbox = imap_tools.MailBox(user.split("@")[1])
-    mailbox.login(user, password)
-    return mailbox
-
-
-class TestMetadataTokens:
-    "Tests that use Metadata extension for storing tokens"
-
-    def test_set_get_metadata(self, imap_mailbox):
-        "set and get metadata token for an account"
-        client = imap_mailbox.client
-        client.send(b'a01 SETMETADATA INBOX (/private/devicetoken "1111" )\n')
-        res = client.readline()
-        assert b"OK Setmetadata completed" in res
-
-        client.send(b"a02 GETMETADATA INBOX /private/devicetoken\n")
-        res = client.readline()
-        assert res[:1] == b"*"
-        res = client.readline().strip().rstrip(b")")
-        assert res == b"1111"
-        assert b"Getmetadata completed" in client.readline()
-
-        client.send(b'a01 SETMETADATA INBOX (/private/devicetoken "2222" )\n')
-        res = client.readline()
-        assert b"OK Setmetadata completed" in res
-
-        client.send(b"a02 GETMETADATA INBOX /private/devicetoken\n")
-        res = client.readline()
-        assert res[:1] == b"*"
-        res = client.readline().strip().rstrip(b")")
-        assert res == b"1111 2222"
-        assert b"Getmetadata completed" in client.readline()
 
 
 class TestEndToEndDeltaChat:
@@ -103,7 +63,7 @@ class TestEndToEndDeltaChat:
 
         addr = ac2.get_config("addr").lower()
         saved_ok = 0
-        for line in remote.iter_output("journalctl -n0 -f -u dovecot"):
+        for line in remote.iter_output("journalctl -f -u dovecot"):
             if addr not in line:
                 # print(line)
                 continue
@@ -115,10 +75,7 @@ class TestEndToEndDeltaChat:
                         )
                     lp.indent("good, message sending failed because quota was exceeded")
                     return
-            if (
-                "stored mail into mailbox 'inbox'" in line
-                or "saved mail to inbox" in line
-            ):
+            if "saved mail to inbox" in line:
                 saved_ok += 1
                 print(f"{saved_ok}: {line}")
                 if saved_ok >= num_to_send:
@@ -155,7 +112,7 @@ class TestEndToEndDeltaChat:
         lp.sec("ac1 sends a message and ac2 marks it as seen")
         chat = ac1.create_chat(ac2)
         msg = chat.send_text("hi")
-        m = ac2._evtracker.wait_next_incoming_message()
+        m = ac2.wait_next_incoming_message()
         m.mark_seen()
         # we can only indirectly wait for mark-seen to cause an smtp-error
         lp.sec("try to wait for markseen to complete and check error states")
@@ -175,19 +132,7 @@ def test_hide_senders_ip_address(cmfactory):
     chat = cmfactory.get_accepted_chat(user1, user2)
 
     chat.send_text("testing submission header cleanup")
-    user2._evtracker.wait_next_incoming_message()
+    user2.wait_next_incoming_message()
     user2.direct_imap.select_folder("Inbox")
     msg = user2.direct_imap.get_all_messages()[0]
     assert public_ip not in msg.obj.as_string()
-
-
-def test_echobot(cmfactory, chatmail_config, lp):
-    ac = cmfactory.get_online_accounts(1)[0]
-
-    lp.sec(f"Send message to echo@{chatmail_config.mail_domain}")
-    chat = ac.create_chat(f"echo@{chatmail_config.mail_domain}")
-    text = "hi, I hope you text me back"
-    chat.send_text(text)
-    lp.sec("Wait for reply from echobot")
-    reply = ac._evtracker.wait_next_incoming_message()
-    assert reply.text == text

scripts/initenv.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 set -e
-python3 -m venv --upgrade-deps venv
+python3 -m venv venv
 
 venv/bin/pip install -e chatmaild 
 venv/bin/pip install -e cmdeploy