Implement "iterate" command in metadata server

Otherwise Dovecot times out when trying to iterate over metadata
of the folder. Apparently it happens when attempting to delete
folder from the server over IMAP.

.github/workflows/test-and-deploy.yaml

@@ -4,17 +4,12 @@ on:
   push:
     branches:
       - main
-  pull_request:
-    paths-ignore:
-      - 'scripts/**'
+      - staging-ci
 
 jobs:
   deploy:
     name: deploy on staging.testrun.org, and run tests
     runs-on: ubuntu-latest
-    concurrency:
-      group: staging-deploy
-      cancel-in-progress: true
     steps:
       - uses: actions/checkout@v3
 
@@ -24,45 +19,44 @@ jobs:
           echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
           chmod 600 ~/.ssh/id_ed25519
           ssh-keyscan staging.testrun.org > ~/.ssh/known_hosts
-          rsync -avz root@staging.testrun.org:/var/lib/acme . || true
-          rsync -avz root@staging.testrun.org:/etc/dkimkeys . || true
+     #    rsync -avz root@staging.testrun.org:/var/lib/acme . || true
+     #    rsync -avz root@staging.testrun.org:/var/lib/rspamd/dkim . || true
 
-      - name: rebuild staging.testrun.org to have a clean VPS
-        run: |
-            curl -X POST \
-            -H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
-            -H "Content-Type: application/json" \
-            -d '{"image":"debian-12"}' \
-            "https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_SERVER_ID }}/actions/rebuild"
+     #- name: rebuild staging.testrun.org to have a clean VPS
+     #  run: |
+     #      curl -X POST \
+     #      -H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
+     #      -H "Content-Type: application/json" \
+     #      -d '{"image":"debian-12"}' \
+     #      "https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_SERVER_ID }}/actions/rebuild"
 
       - run: scripts/initenv.sh
 
       - name: append venv/bin to PATH
         run: echo venv/bin >>$GITHUB_PATH
 
-      - name: upload TLS cert after rebuilding
-        run: |
-          echo " --- wait until staging.testrun.org VPS is rebuilt --- "
-          rm ~/.ssh/known_hosts
-          while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u ; do sleep 1 ; done
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u
-          rsync -avz acme/ root@staging.testrun.org:/var/lib/acme || true
-          rsync -avz dkimkeys/ root@staging.testrun.org:/etc/dkimkeys || true
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org chown root:root -R /var/lib/acme
-
       - name: run formatting checks 
         run: cmdeploy fmt -v 
 
       - name: run deploy-chatmail offline tests 
         run: pytest --pyargs cmdeploy 
 
+     #- name: upload TLS cert after rebuilding
+     #  run: |
+     #    echo " --- wait until staging.testrun.org VPS is rebuilt --- "
+     #    rm ~/.ssh/known_hosts
+     #    while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u ; do sleep 1 ; done
+     #    ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u
+     #    rsync -avz acme root@staging.testrun.org:/var/lib/ || true
+     #    rsync -avz dkim root@staging.testrun.org:/var/lib/rspamd/ || true
+
       - run: cmdeploy init staging.testrun.org
 
       - run: cmdeploy run
 
       - name: set DNS entries
         run: |
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
+          #ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org chown _rspamd:_rspamd -R /var/lib/rspamd/dkim
           cmdeploy dns --zonefile staging-generated.zone
           cat staging-generated.zone >> .github/workflows/staging.testrun.org-default.zone
           cat .github/workflows/staging.testrun.org-default.zone

CHANGELOG.md

@@ -1,40 +0,0 @@
-# Changelog for chatmail deployment 
-
-## untagged 
-
-- Apply systemd restrictions to echobot
-  ([#259](https://github.com/deltachat/chatmail/pull/259))
-
-- re-enable running the CI in pull requests, but not concurrently 
-  ([#258](https://github.com/deltachat/chatmail/pull/258))
-
-
-## 1.1.0 - 2024-03-28
-
-### The changelog starts to record changes from March 15th, 2024 
-
-- Move systemd unit templates to cmdeploy package 
-  ([#255](https://github.com/deltachat/chatmail/pull/255))
-
-- Persist push tokens and support multiple device per address 
-  ([#254](https://github.com/deltachat/chatmail/pull/254))
-
-- Avoid warning for regular doveauth protocol's hello message. 
-  ([#250](https://github.com/deltachat/chatmail/pull/250))
-
-- Fix various tests to pass again with "cmdeploy test". 
-  ([#245](https://github.com/deltachat/chatmail/pull/245),
-  [#242](https://github.com/deltachat/chatmail/pull/242)
-
-- Ensure lets-encrypt certificates are reloaded after renewal 
-  ([#244]) https://github.com/deltachat/chatmail/pull/244
-
-- Persist tokens to avoid iOS users loosing push-notifications when the
-  chatmail metadata service is restarted (happens regularly during deploys)
-  ([#238](https://github.com/deltachat/chatmail/pull/239)
-
-- Fix failing sieve-script compile errors on incoming messages
-  ([#237](https://github.com/deltachat/chatmail/pull/239)
-
-- Fix quota reporting after expunging of old mails
-  ([#233](https://github.com/deltachat/chatmail/pull/239)

chatmaild/MANIFEST.in

@@ -1,3 +1,4 @@
+include src/chatmaild/*.f 
 include src/chatmaild/ini/*.ini.f
 include src/chatmaild/ini/*.ini
 include src/chatmaild/tests/mail-data/*

chatmaild/pyproject.toml

@@ -10,7 +10,6 @@ dependencies = [
   "iniconfig",
   "deltachat-rpc-server",
   "deltachat-rpc-client",
-  "filelock",
   "requests",
 ]
 

chatmaild/src/chatmaild/chatmail-metadata.service.f

@@ -2,7 +2,7 @@
 Description=Chatmail dict proxy for IMAP METADATA
 
 [Service]
-ExecStart={execpath} /run/dovecot/metadata.socket vmail /home/vmail/mail/{mail_domain}
+ExecStart={execpath} /run/dovecot/metadata.socket vmail {config_path}
 Restart=always
 RestartSec=30
 

chatmaild/src/chatmaild/doveauth.py

@@ -17,10 +17,6 @@ from .config import read_config, Config
 NOCREATE_FILE = "/etc/chatmail-nocreate"
 
 
-class UnknownCommand(ValueError):
-    """dictproxy handler received an unkown command"""
-
-
 def encrypt_password(password: str):
     # https://doc.dovecot.org/configuration_manual/authentication/password_schemes/
     passhash = crypt.crypt(password, crypt.METHOD_SHA512)
@@ -131,12 +127,8 @@ def split_and_unescape(s):
 
 
 def handle_dovecot_request(msg, db, config: Config):
-    # see https://doc.dovecot.org/3.0/developer_manual/design/dict_protocol/
     short_command = msg[0]
-    if short_command == "H":  # HELLO
-        # we don't do any checking on versions and just return
-        return
-    elif short_command == "L":  # LOOKUP
+    if short_command == "L":  # LOOKUP
         parts = msg[1:].split("\t")
 
         # Dovecot <2.3.17 has only one part,
@@ -167,7 +159,7 @@ def handle_dovecot_request(msg, db, config: Config):
                     reply_command = "N"
         json_res = json.dumps(res) if res else ""
         return f"{reply_command}{json_res}\n"
-    raise UnknownCommand(msg)
+    return None
 
 
 def handle_dovecot_protocol(rfile, wfile, db: Database, config: Config):
@@ -175,14 +167,12 @@ def handle_dovecot_protocol(rfile, wfile, db: Database, config: Config):
         msg = rfile.readline().strip().decode()
         if not msg:
             break
-        try:
-            res = handle_dovecot_request(msg, db, config)
-        except UnknownCommand:
-            logging.warning("unknown command: %r", msg)
+        res = handle_dovecot_request(msg, db, config)
+        if res:
+            wfile.write(res.encode("ascii"))
+            wfile.flush()
         else:
-            if res:
-                wfile.write(res.encode("ascii"))
-                wfile.flush()
+            logging.warning("request had no answer: %r", msg)
 
 
 class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):

chatmaild/src/chatmaild/echo.py

@@ -18,14 +18,14 @@ hooks = events.HookCollection()
 @hooks.on(events.RawEvent)
 def log_event(event):
     if event.kind == EventType.INFO:
-        logging.info("%s", event.msg)
+        logging.info(event.msg)
     elif event.kind == EventType.WARNING:
-        logging.warning("%s", event.msg)
+        logging.warning(event.msg)
 
 
 @hooks.on(events.RawEvent(EventType.ERROR))
 def log_error(event):
-    logging.error("%s", event.msg)
+    logging.error(event.msg)
 
 
 @hooks.on(events.MemberListChanged)
@@ -48,9 +48,6 @@ def on_group_name_changed(event):
 @hooks.on(events.NewMessage(func=lambda e: not e.command))
 def echo(event):
     snapshot = event.message_snapshot
-    if snapshot.is_info:
-        # Ignore info messages
-        return
     if snapshot.text or snapshot.file:
         snapshot.chat.send_message(text=snapshot.text, file=snapshot.file)
 
@@ -62,7 +59,6 @@ def help_command(event):
 
 
 def main():
-    logging.basicConfig(level=logging.INFO)
     path = os.environ.get("PATH")
     venv_path = sys.argv[0].strip("echobot")
     os.environ["PATH"] = path + ":" + venv_path
@@ -84,4 +80,5 @@ def main():
 
 
 if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
     main()

chatmaild/src/chatmaild/echobot.service.f

@@ -0,0 +1,11 @@
+[Unit]
+Description=Chatmail echo bot for testing it works
+
+[Service]
+ExecStart={execpath} {config_path}
+Environment="PATH={remote_venv_dir}:$PATH"
+Restart=always
+RestartSec=30
+
+[Install]
+WantedBy=multi-user.target

chatmaild/src/chatmaild/filedict.py

@@ -1,35 +0,0 @@
-import os
-import logging
-import json
-import filelock
-from contextlib import contextmanager
-
-
-class FileDict:
-    """Concurrency-safe multi-reader/single-writer persistent dict."""
-
-    def __init__(self, path):
-        self.path = path
-        self.lock_path = path.with_name(path.name + ".lock")
-
-    @contextmanager
-    def modify(self):
-        # the OS will release the lock if the process dies,
-        # and the contextmanager will otherwise guarantee release
-        with filelock.FileLock(self.lock_path):
-            data = self.read()
-            yield data
-            write_path = self.path.with_name(self.path.name + ".tmp")
-            with write_path.open("w") as f:
-                json.dump(data, f)
-            os.rename(write_path, self.path)
-
-    def read(self):
-        try:
-            with self.path.open("r") as f:
-                return json.load(f)
-        except FileNotFoundError:
-            return {}
-        except Exception:
-            logging.warning("corrupt serialization state at: %r", self.path)
-            return {}

chatmaild/src/chatmaild/filtermail.service.f

@@ -1,5 +1,5 @@
 [Unit]
-Description=Chatmail Postfix before queue filter
+Description=Chatmail Postfix BeforeQeue filter 
 
 [Service]
 ExecStart={execpath} {config_path}

chatmaild/src/chatmaild/metadata.py

@@ -1,93 +1,62 @@
 import pwd
 
-from pathlib import Path
-from threading import Thread, Event
+from queue import Queue
+from threading import Thread
 from socketserver import (
     UnixStreamServer,
     StreamRequestHandler,
     ThreadingMixIn,
 )
+from .config import read_config
 import sys
 import logging
 import os
 import requests
 
-from .filedict import FileDict
 
-
-DICTPROXY_HELLO_CHAR = "H"
 DICTPROXY_LOOKUP_CHAR = "L"
 DICTPROXY_ITERATE_CHAR = "I"
-DICTPROXY_BEGIN_TRANSACTION_CHAR = "B"
 DICTPROXY_SET_CHAR = "S"
+DICTPROXY_BEGIN_TRANSACTION_CHAR = "B"
 DICTPROXY_COMMIT_TRANSACTION_CHAR = "C"
-DICTPROXY_TRANSACTION_CHARS = "BSC"
-
-# each SETMETADATA on this key appends to a list of unique device tokens
-# which only ever get removed if the upstream indicates the token is invalid
-METADATA_TOKEN_KEY = "devicetoken"
+DICTPROXY_TRANSACTION_CHARS = "SBC"
 
 
 class Notifier:
-    def __init__(self, vmail_dir):
-        self.vmail_dir = vmail_dir
-        self.notification_dir = vmail_dir / "pending_notifications"
-        if not self.notification_dir.exists():
-            self.notification_dir.mkdir()
-        self.message_arrived_event = Event()
+    def __init__(self):
+        self.guid2token = {}
+        self.to_notify_queue = Queue()
 
-    def get_metadata_dict(self, addr):
-        return FileDict(self.vmail_dir / addr / "metadata.json")
+    def set_token(self, guid, token):
+        self.guid2token[guid] = token
 
-    def add_token(self, addr, token):
-        with self.get_metadata_dict(addr).modify() as data:
-            tokens = data.get(METADATA_TOKEN_KEY)
-            if tokens is None:
-                data[METADATA_TOKEN_KEY] = [token]
-            elif token not in tokens:
-                tokens.append(token)
-
-    def remove_token(self, addr, token):
-        with self.get_metadata_dict(addr).modify() as data:
-            tokens = data.get(METADATA_TOKEN_KEY, [])
-            try:
-                tokens.remove(token)
-            except ValueError:
-                pass
-
-    def get_tokens(self, addr):
-        return self.get_metadata_dict(addr).read().get(METADATA_TOKEN_KEY, [])
-
-    def new_message_for_addr(self, addr):
-        self.notification_dir.joinpath(addr).touch()
-        self.message_arrived_event.set()
+    def new_message_for_guid(self, guid):
+        self.to_notify_queue.put(guid)
 
     def thread_run_loop(self):
         requests_session = requests.Session()
         while 1:
-            self.message_arrived_event.wait()
-            self.message_arrived_event.clear()
             self.thread_run_one(requests_session)
 
     def thread_run_one(self, requests_session):
-        for addr_path in self.notification_dir.iterdir():
-            addr = addr_path.name
-            if "@" not in addr:
-                continue
-            for token in self.get_tokens(addr):
-                response = requests_session.post(
-                    "https://notifications.delta.chat/notify",
-                    data=token,
-                    timeout=60,
-                )
-                if response.status_code == 410:
-                    # 410 Gone status code
-                    # means the token is no longer valid.
-                    self.remove_token(addr, token)
-            addr_path.unlink()
+        guid = self.to_notify_queue.get()
+        token = self.guid2token.get(guid)
+        if token:
+            response = requests_session.post(
+                "https://notifications.delta.chat/notify",
+                data=token,
+                timeout=60,
+            )
+            if response.status_code == 410:
+                # 410 Gone status code
+                # means the token is no longer valid.
+                del self.guid2token[guid]
 
 
 def handle_dovecot_protocol(rfile, wfile, notifier):
+    # HELLO message, ignored.
+    msg = rfile.readline().strip().decode()
+
     transactions = {}
     while True:
         msg = rfile.readline().strip().decode()
@@ -105,52 +74,44 @@ def handle_dovecot_request(msg, transactions, notifier):
     short_command = msg[0]
     parts = msg[1:].split("\t")
     if short_command == DICTPROXY_LOOKUP_CHAR:
-        # Lpriv/43f5f508a7ea0366dff30200c15250e3/devicetoken\tlkj123poi@c2.testrun.org
-        keyparts = parts[0].split("/")
-        if keyparts[0] == "priv":
-            keyname = keyparts[2]
-            addr = parts[1]
-            if keyname == METADATA_TOKEN_KEY:
-                res = " ".join(notifier.get_tokens(addr))
-                return f"O{res}\n"
-        logging.warning("lookup ignored: %r", msg)
         return "N\n"
     elif short_command == DICTPROXY_ITERATE_CHAR:
         # Empty line means ITER_FINISHED.
         # If we don't return empty line Dovecot will timeout.
         return "\n"
-    elif short_command == DICTPROXY_HELLO_CHAR:
-        return  # no version checking
 
     if short_command not in (DICTPROXY_TRANSACTION_CHARS):
-        logging.warning("unknown dictproxy request: %r", msg)
         return
 
     transaction_id = parts[0]
 
     if short_command == DICTPROXY_BEGIN_TRANSACTION_CHAR:
-        addr = parts[1]
-        transactions[transaction_id] = dict(addr=addr, res="O\n")
+        transactions[transaction_id] = "O\n"
     elif short_command == DICTPROXY_COMMIT_TRANSACTION_CHAR:
-        # each set devicetoken operation persists directly
-        # and does not wait until a "commit" comes
-        # because our dovecot config does not involve
-        # multiple set-operations in a single commit
-        return transactions.pop(transaction_id)["res"]
+        # returns whether it failed or succeeded.
+        return transactions.pop(transaction_id, "N\n")
     elif short_command == DICTPROXY_SET_CHAR:
-        # For documentation on key structure see
-        # https://github.com/dovecot/core/blob/main/src/lib-storage/mailbox-attribute.h
+        # See header of
+        # <https://github.com/dovecot/core/blob/5e7965632395793d9355eb906b173bf28d2a10ca/src/lib-storage/mailbox-attribute.h>
+        # for the documentation on the structure of the key.
+
+        # Request GETMETADATA "INBOX" /private/chatmail
+        # results in a query for
+        # priv/dd72550f05eadc65542a1200cac67ad7/chatmail
+        #
+        # Request GETMETADATA "" /private/chatmail
+        # results in
+        # priv/dd72550f05eadc65542a1200cac67ad7/vendor/vendor.dovecot/pvt/server/chatmail
 
         keyname = parts[1].split("/")
         value = parts[2] if len(parts) > 2 else ""
-        addr = transactions[transaction_id]["addr"]
-        if keyname[0] == "priv" and keyname[2] == METADATA_TOKEN_KEY:
-            notifier.add_token(addr, value)
+        if keyname[0] == "priv" and keyname[2] == "devicetoken":
+            notifier.set_token(keyname[1], value)
         elif keyname[0] == "priv" and keyname[2] == "messagenew":
-            notifier.new_message_for_addr(addr)
+            notifier.new_message_for_guid(keyname[1])
         else:
             # Transaction failed.
-            transactions[transaction_id]["res"] = "F\n"
+            transactions[transaction_id] = "F\n"
 
 
 class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
@@ -158,23 +119,19 @@ class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
 
 
 def main():
-    socket, username, vmail_dir = sys.argv[1:]
+    socket, username, config = sys.argv[1:]
     passwd_entry = pwd.getpwnam(username)
 
-    vmail_dir = Path(vmail_dir)
-
-    if not vmail_dir.exists():
-        logging.error("vmail dir does not exist: %r", vmail_dir)
-        return 1
-
-    notifier = Notifier(vmail_dir)
+    # XXX config is not currently used
+    config = read_config(config)
+    notifier = Notifier()
 
     class Handler(StreamRequestHandler):
         def handle(self):
             try:
                 handle_dovecot_protocol(self.rfile, self.wfile, notifier)
             except Exception:
-                logging.exception("Exception in the dovecot dictproxy handler")
+                logging.exception("Exception in the handler")
                 raise
 
     try:
@@ -188,8 +145,6 @@ def main():
     t = Thread(target=notifier.thread_run_loop)
     t.setDaemon(True)
     t.start()
-    # let notifier thread run once for any pending notifications from last run
-    notifier.message_arrived_event.set()
 
     with ThreadedUnixStreamServer(socket, Handler) as server:
         os.chown(socket, uid=passwd_entry.pw_uid, gid=passwd_entry.pw_gid)

chatmaild/src/chatmaild/tests/test_doveauth.py

@@ -75,14 +75,6 @@ def test_handle_dovecot_request(db, example_config):
     assert userdata["password"].startswith("{SHA512-CRYPT}")
 
 
-def test_handle_dovecot_protocol_hello_is_skipped(db, example_config, caplog):
-    rfile = io.BytesIO(b"H3\t2\t0\t\tauth\n")
-    wfile = io.BytesIO()
-    handle_dovecot_protocol(rfile, wfile, db, example_config)
-    assert wfile.getvalue() == b""
-    assert not caplog.messages
-
-
 def test_handle_dovecot_protocol(db, example_config):
     rfile = io.BytesIO(
         b"H3\t2\t0\t\tauth\nLshared/userdb/foobar@chat.example.org\tfoobar@chat.example.org\n"

chatmaild/src/chatmaild/tests/test_filedict.py

@@ -1,19 +0,0 @@
-from chatmaild.filedict import FileDict
-
-
-def test_basic(tmp_path):
-    fdict = FileDict(tmp_path.joinpath("metadata"))
-    assert fdict.read() == {}
-    with fdict.modify() as d:
-        d["devicetoken"] = [1, 2, 3]
-        d["456"] = 4.2
-    new = fdict.read()
-    assert new["devicetoken"] == [1, 2, 3]
-    assert new["456"] == 4.2
-
-
-def test_bad_marshal_file(tmp_path, caplog):
-    fdict1 = FileDict(tmp_path.joinpath("metadata"))
-    fdict1.path.write_bytes(b"l12k3l12k3l")
-    assert fdict1.read() == {}
-    assert "corrupt" in caplog.records[0].msg

chatmaild/src/chatmaild/tests/test_metadata.py

@@ -1,5 +1,4 @@
 import io
-import pytest
 
 from chatmaild.metadata import (
     handle_dovecot_request,
@@ -8,130 +7,70 @@ from chatmaild.metadata import (
 )
 
 
-@pytest.fixture
-def notifier(tmp_path):
-    vmail_dir = tmp_path.joinpath("vmaildir")
-    vmail_dir.mkdir()
-    return Notifier(vmail_dir)
-
-
-@pytest.fixture
-def testaddr():
-    return "user.name@example.org"
-
-
-@pytest.fixture
-def testaddr2():
-    return "user2@example.org"
-
-
-def test_notifier_persistence(tmp_path, testaddr, testaddr2):
-    notifier1 = Notifier(tmp_path)
-    notifier2 = Notifier(tmp_path)
-    assert not notifier1.get_tokens(testaddr)
-    assert not notifier2.get_tokens(testaddr)
-
-    notifier1.add_token(testaddr, "01234")
-    notifier1.add_token(testaddr2, "456")
-    assert notifier2.get_tokens(testaddr) == ["01234"]
-    assert notifier2.get_tokens(testaddr2) == ["456"]
-    notifier2.remove_token(testaddr, "01234")
-    assert not notifier1.get_tokens(testaddr)
-    assert notifier1.get_tokens(testaddr2) == ["456"]
-
-
-def test_remove_nonexisting(tmp_path, testaddr):
-    notifier1 = Notifier(tmp_path)
-    notifier1.add_token(testaddr, "123")
-    notifier1.remove_token(testaddr, "1l23k1l2k3")
-    assert notifier1.get_tokens(testaddr) == ["123"]
-
-
-def test_notifier_delete_without_set(notifier, testaddr):
-    notifier.remove_token(testaddr, "123")
-    assert not notifier.get_tokens(testaddr)
-
-
-def test_handle_dovecot_request_lookup_fails(notifier, testaddr):
-    res = handle_dovecot_request(f"Lpriv/123/chatmail\t{testaddr}", {}, notifier)
+def test_handle_dovecot_request_lookup_fails():
+    notifier = Notifier()
+    res = handle_dovecot_request("Lpriv/123/chatmail", {}, notifier)
     assert res == "N\n"
 
 
-def test_handle_dovecot_request_happy_path(notifier, testaddr):
+def test_handle_dovecot_request_happy_path():
+    notifier = Notifier()
     transactions = {}
 
+    # lookups return the same NOTFOUND result
+    res = handle_dovecot_request("Lpriv/123/chatmail", transactions, notifier)
+    assert res == "N\n"
+    assert not notifier.guid2token and not transactions
+
     # set device token in a transaction
     tx = "1111"
-    msg = f"B{tx}\t{testaddr}"
+    msg = f"B{tx}\tuser"
     res = handle_dovecot_request(msg, transactions, notifier)
-    assert not res and not notifier.get_tokens(testaddr)
-    assert transactions == {tx: dict(addr=testaddr, res="O\n")}
+    assert not res and not notifier.guid2token
+    assert transactions == {tx: "O\n"}
 
     msg = f"S{tx}\tpriv/guid00/devicetoken\t01234"
     res = handle_dovecot_request(msg, transactions, notifier)
     assert not res
     assert len(transactions) == 1
-    assert notifier.get_tokens(testaddr) == ["01234"]
+    assert len(notifier.guid2token) == 1
+    assert notifier.guid2token["guid00"] == "01234"
 
     msg = f"C{tx}"
     res = handle_dovecot_request(msg, transactions, notifier)
     assert res == "O\n"
     assert len(transactions) == 0
-    assert notifier.get_tokens(testaddr) == ["01234"]
+    assert notifier.guid2token["guid00"] == "01234"
 
     # trigger notification for incoming message
-    tx2 = "2222"
-    assert handle_dovecot_request(f"B{tx2}\t{testaddr}", transactions, notifier) is None
-    msg = f"S{tx2}\tpriv/guid00/messagenew"
+    assert handle_dovecot_request(f"B{tx}\tuser", transactions, notifier) is None
+    msg = f"S{tx}\tpriv/guid00/messagenew"
     assert handle_dovecot_request(msg, transactions, notifier) is None
-    assert notifier.message_arrived_event.is_set()
-    assert handle_dovecot_request(f"C{tx2}", transactions, notifier) == "O\n"
+    assert notifier.to_notify_queue.get() == "guid00"
+    assert notifier.to_notify_queue.qsize() == 0
+    assert handle_dovecot_request(f"C{tx}\tuser", transactions, notifier) == "O\n"
     assert not transactions
-    assert notifier.notification_dir.joinpath(testaddr).exists()
 
 
-def test_handle_dovecot_protocol_set_devicetoken(notifier):
+def test_handle_dovecot_protocol_set_devicetoken():
     rfile = io.BytesIO(
         b"\n".join(
             [
                 b"HELLO",
-                b"Btx00\tuser@example.org",
+                b"Btx00\tuser",
                 b"Stx00\tpriv/guid00/devicetoken\t01234",
                 b"Ctx00",
             ]
         )
     )
     wfile = io.BytesIO()
+    notifier = Notifier()
     handle_dovecot_protocol(rfile, wfile, notifier)
-    assert wfile.getvalue() == b"O\n"
-    assert notifier.get_tokens("user@example.org") == ["01234"]
-
-
-def test_handle_dovecot_protocol_set_get_devicetoken(notifier):
-    rfile = io.BytesIO(
-        b"\n".join(
-            [
-                b"HELLO",
-                b"Btx00\tuser@example.org",
-                b"Stx00\tpriv/guid00/devicetoken\t01234",
-                b"Ctx00",
-            ]
-        )
-    )
-    wfile = io.BytesIO()
-    handle_dovecot_protocol(rfile, wfile, notifier)
-    assert notifier.get_tokens("user@example.org") == ["01234"]
+    assert notifier.guid2token["guid00"] == "01234"
     assert wfile.getvalue() == b"O\n"
 
-    rfile = io.BytesIO(
-        b"\n".join([b"HELLO", b"Lpriv/0123/devicetoken\tuser@example.org"])
-    )
-    wfile = io.BytesIO()
-    handle_dovecot_protocol(rfile, wfile, notifier)
-    assert wfile.getvalue() == b"O01234\n"
 
-
-def test_handle_dovecot_protocol_iterate(notifier):
+def test_handle_dovecot_protocol_iterate():
     rfile = io.BytesIO(
         b"\n".join(
             [
@@ -141,29 +80,31 @@ def test_handle_dovecot_protocol_iterate(notifier):
         )
     )
     wfile = io.BytesIO()
+    notifier = Notifier()
     handle_dovecot_protocol(rfile, wfile, notifier)
     assert wfile.getvalue() == b"\n"
 
 
-def test_handle_dovecot_protocol_messagenew(notifier):
+def test_handle_dovecot_protocol_messagenew():
     rfile = io.BytesIO(
         b"\n".join(
             [
                 b"HELLO",
-                b"Btx01\tuser@example.org",
+                b"Btx01\tuser",
                 b"Stx01\tpriv/guid00/messagenew",
                 b"Ctx01",
             ]
         )
     )
     wfile = io.BytesIO()
+    notifier = Notifier()
     handle_dovecot_protocol(rfile, wfile, notifier)
     assert wfile.getvalue() == b"O\n"
-    assert notifier.message_arrived_event.is_set()
-    assert notifier.notification_dir.joinpath("user@example.org").exists()
+    assert notifier.to_notify_queue.get() == "guid00"
+    assert notifier.to_notify_queue.qsize() == 0
 
 
-def test_notifier_thread_run(notifier, testaddr):
+def test_notifier_thread_run():
     requests = []
 
     class ReqMock:
@@ -175,15 +116,16 @@ def test_notifier_thread_run(notifier, testaddr):
 
             return Result()
 
-    notifier.add_token(testaddr, "01234")
-    notifier.new_message_for_addr(testaddr)
+    notifier = Notifier()
+    notifier.set_token("guid00", "01234")
+    notifier.new_message_for_guid("guid00")
     notifier.thread_run_one(ReqMock())
     url, data, timeout = requests[0]
     assert data == "01234"
-    assert notifier.get_tokens(testaddr) == ["01234"]
+    assert len(notifier.guid2token) == 1
 
 
-def test_multi_device_notifier(notifier, testaddr):
+def test_notifier_thread_run_gone_removes_token():
     requests = []
 
     class ReqMock:
@@ -191,40 +133,15 @@ def test_multi_device_notifier(notifier, testaddr):
             requests.append((url, data, timeout))
 
             class Result:
-                status_code = 200
+                status_code = 410
 
             return Result()
 
-    notifier.add_token(testaddr, "01234")
-    notifier.add_token(testaddr, "56789")
-    notifier.new_message_for_addr(testaddr)
+    notifier = Notifier()
+    notifier.set_token("guid00", "01234")
+    notifier.new_message_for_guid("guid00")
+    assert notifier.guid2token["guid00"] == "01234"
     notifier.thread_run_one(ReqMock())
     url, data, timeout = requests[0]
     assert data == "01234"
-    url, data, timeout = requests[1]
-    assert data == "56789"
-    assert notifier.get_tokens(testaddr) == ["01234", "56789"]
-
-
-def test_notifier_thread_run_gone_removes_token(notifier, testaddr):
-    requests = []
-
-    class ReqMock:
-        def post(self, url, data, timeout):
-            requests.append((url, data, timeout))
-
-            class Result:
-                status_code = 410 if data == "01234" else 200
-
-            return Result()
-
-    notifier.add_token(testaddr, "01234")
-    notifier.new_message_for_addr(testaddr)
-    assert notifier.get_tokens(testaddr) == ["01234"]
-    notifier.add_token(testaddr, "45678")
-    notifier.thread_run_one(ReqMock())
-    url, data, timeout = requests[0]
-    assert data == "01234"
-    url, data, timeout = requests[1]
-    assert data == "45678"
-    assert notifier.get_tokens(testaddr) == ["45678"]
+    assert len(notifier.guid2token) == 0

cmdeploy/pyproject.toml

@@ -19,7 +19,6 @@ dependencies = [
   "black",
   "pytest",
   "pytest-xdist", 
-  "imap_tools",
 ]
 
 [project.scripts]

cmdeploy/src/cmdeploy/__init__.py

@@ -108,11 +108,8 @@ def _install_remote_venv_with_chatmaild(config) -> None:
             execpath=f"{remote_venv_dir}/bin/{fn}",
             config_path=remote_chatmail_inipath,
             remote_venv_dir=remote_venv_dir,
-            mail_domain=config.mail_domain,
-        )
-        source_path = importlib.resources.files(__package__).joinpath(
-            "service", f"{fn}.service.f"
         )
+        source_path = importlib.resources.files("chatmaild").joinpath(f"{fn}.service.f")
         content = source_path.read_text().format(**params).encode()
 
         files.put(
@@ -135,6 +132,20 @@ def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
     """Configures OpenDKIM"""
     need_restart = False
 
+    server.group(name="Create opendkim group", group="opendkim", system=True)
+    server.user(
+        name="Create opendkim user",
+        user="opendkim",
+        groups=["opendkim"],
+        system=True,
+    )
+    server.user(
+        name="Add postfix user to opendkim group for socket access",
+        user="postfix",
+        groups=["opendkim"],
+        system=True,
+    )
+
     main_config = files.template(
         src=importlib.resources.files(__package__).joinpath("opendkim/opendkim.conf"),
         dest="/etc/opendkim.conf",
@@ -339,18 +350,15 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
     need_restart |= lua_push_notification_script.changed
 
     sieve_script = files.put(
-        src=importlib.resources.files(__package__).joinpath("dovecot/default.sieve"),
+        src=importlib.resources.files(__package__).joinpath(
+            "dovecot/default.sieve"
+        ),
         dest="/etc/dovecot/default.sieve",
         user="root",
         group="root",
         mode="644",
     )
     need_restart |= sieve_script.changed
-    if sieve_script.changed:
-        server.shell(
-            name="compile sieve script",
-            commands=["/usr/bin/sievec /etc/dovecot/default.sieve"],
-        )
 
     files.template(
         src=importlib.resources.files(__package__).joinpath("dovecot/expunge.cron.j2"),
@@ -442,9 +450,7 @@ def check_config(config):
         blocked_words = "merlinux schmieder testrun.org".split()
         for key in config.__dict__:
             value = config.__dict__[key]
-            if key.startswith("privacy") and any(
-                x in str(value) for x in blocked_words
-            ):
+            if key.startswith("privacy") and any(x in str(value) for x in blocked_words):
                 raise ValueError(
                     f"please set your own privacy contacts/addresses in {config._inipath}"
                 )
@@ -462,24 +468,9 @@ def deploy_chatmail(config_path: Path) -> None:
 
     from .www import build_webpages
 
+    apt.update(name="apt update", cache_time=24 * 3600)
     server.group(name="Create vmail group", group="vmail", system=True)
     server.user(name="Create vmail user", user="vmail", group="vmail", system=True)
-    server.group(name="Create opendkim group", group="opendkim", system=True)
-    server.user(
-        name="Create opendkim user",
-        user="opendkim",
-        groups=["opendkim"],
-        system=True,
-    )
-    server.user(
-        name="Add postfix user to opendkim group for socket access",
-        user="postfix",
-        groups=["opendkim"],
-        system=True,
-    )
-
-    apt.update(name="apt update", cache_time=24 * 3600)
-
     apt.packages(
         name="Install rsync",
         packages=["rsync"],
@@ -566,17 +557,6 @@ def deploy_chatmail(config_path: Path) -> None:
         restarted=mta_sts_need_restart,
     )
 
-    # Dovecot should be started before Postfix
-    # because it creates authentication socket
-    # required by Postfix.
-    systemd.service(
-        name="Start and enable Dovecot",
-        service="dovecot.service",
-        running=True,
-        enabled=True,
-        restarted=dovecot_need_restart,
-    )
-
     systemd.service(
         name="Start and enable Postfix",
         service="postfix.service",
@@ -585,6 +565,14 @@ def deploy_chatmail(config_path: Path) -> None:
         restarted=postfix_need_restart,
     )
 
+    systemd.service(
+        name="Start and enable Dovecot",
+        service="dovecot.service",
+        running=True,
+        enabled=True,
+        restarted=dovecot_need_restart,
+    )
+
     systemd.service(
         name="Start and enable nginx",
         service="nginx.service",

cmdeploy/src/cmdeploy/acmetool/acmetool.cron

@@ -1,4 +1,4 @@
 SHELL=/bin/sh
 PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
 MAILTO=root
-20 16 * * * root /usr/bin/acmetool --batch reconcile && systemctl reload dovecot && systemctl reload postfix
+20 16 * * * root /usr/bin/acmetool --batch reconcile

cmdeploy/src/cmdeploy/chatmail.zone.f

@@ -6,7 +6,7 @@ _submissions._tcp.{chatmail_domain}. SRV 0 1 465 {chatmail_domain}.
 _imap._tcp.{chatmail_domain}.        SRV 0 1 143 {chatmail_domain}.
 _imaps._tcp.{chatmail_domain}.       SRV 0 1 993 {chatmail_domain}.
 {chatmail_domain}.                   CAA 128 issue "letsencrypt.org;accounturi={acme_account_url}"
-{chatmail_domain}.                   TXT "v=spf1 a:{chatmail_domain} ~all"
+{chatmail_domain}.                   TXT "v=spf1 a:{chatmail_domain} -all"
 _dmarc.{chatmail_domain}.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
 _mta-sts.{chatmail_domain}.          TXT "v=STSv1; id={sts_id}"
 mta-sts.{chatmail_domain}.           CNAME {chatmail_domain}.

cmdeploy/src/cmdeploy/dns.py

@@ -5,6 +5,8 @@ import importlib
 import subprocess
 import datetime
 
+from typing import Optional
+
 
 class DNS:
     def __init__(self, out, mail_domain):

cmdeploy/src/cmdeploy/dovecot/default.sieve

@@ -1,7 +1,5 @@
 require ["imap4flags"];
 
-# flag the message so it doesn't cause a push notification
-
 if header :is ["Auto-Submitted"] ["auto-replied", "auto-generated"] {
 	addflag "$Auto";
 }

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -13,12 +13,6 @@ auth_cache_size = 100M
 mail_debug = yes
 {% endif %}
 
-# Prevent warnings similar to:
-#   config: Warning: service auth { client_limit=1000 } is lower than required under max. load (10200). Counted for protocol services with service_count != 1: service lmtp { process_limit=100 } + service imap-urlauth-login { process_limit=100 } + service imap-login { process_limit=10000 }
-#   config: Warning: service anvil { client_limit=1000 } is lower than required under max. load (10103). Counted with: service imap-urlauth-login { process_limit=100 } + service imap-login { process_limit=10000 } + service auth { process_limit=1 }
-#   master: Warning: service(stats): client_limit (1000) reached, client connections are being dropped
-default_client_limit = 20000
-
 mail_server_admin = mailto:root@{{ config.mail_domain }}
 mail_server_comment = Chatmail server
 

cmdeploy/src/cmdeploy/dovecot/expunge.cron.j2

@@ -8,4 +8,3 @@
 # or only temporary (but then they shouldn't be around after {{ config.delete_mails_after }} days anyway).
 2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-3 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -name 'maildirsize' -type f -delete

cmdeploy/src/cmdeploy/service/echobot.service.f

@@ -1,50 +0,0 @@
-[Unit]
-Description=Chatmail echo bot for testing it works
-
-[Service]
-ExecStart={execpath} {config_path}
-Environment="PATH={remote_venv_dir}:$PATH"
-Restart=always
-RestartSec=30
-
-# Apply security restrictions suggested by
-#   systemd-analyze security echobot.service
-CapabilityBoundingSet=
-LockPersonality=true
-MemoryDenyWriteExecute=true
-NoNewPrivileges=true
-PrivateDevices=true
-PrivateMounts=true
-PrivateTmp=true
-PrivateUsers=true
-ProtectClock=true
-ProtectControlGroups=true
-ProtectHostname=true
-ProtectKernelLogs=true
-ProtectKernelModules=true
-ProtectKernelTunables=true
-ProtectProc=noaccess
-
-# Should be "strict", but we currently write /accounts folder in a protected path
-ProtectSystem=full
-
-RemoveIPC=true
-RestrictAddressFamilies=AF_INET AF_INET6
-RestrictNamespaces=true
-RestrictRealtime=true
-RestrictSUIDSGID=true
-SystemCallArchitectures=native
-SystemCallFilter=~@clock
-SystemCallFilter=~@cpu-emulation
-SystemCallFilter=~@debug
-SystemCallFilter=~@module
-SystemCallFilter=~@mount
-SystemCallFilter=~@obsolete
-SystemCallFilter=~@raw-io
-SystemCallFilter=~@reboot
-SystemCallFilter=~@resources
-SystemCallFilter=~@swap
-UMask=0077
-
-[Install]
-WantedBy=multi-user.target

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -5,46 +5,6 @@ import random
 import pytest
 import requests
 import ipaddress
-import imap_tools
-
-
-@pytest.fixture
-def imap_mailbox(cmfactory):
-    (ac1,) = cmfactory.get_online_accounts(1)
-    user = ac1.get_config("addr")
-    password = ac1.get_config("mail_pw")
-    mailbox = imap_tools.MailBox(user.split("@")[1])
-    mailbox.login(user, password)
-    return mailbox
-
-
-class TestMetadataTokens:
-    "Tests that use Metadata extension for storing tokens"
-
-    def test_set_get_metadata(self, imap_mailbox):
-        "set and get metadata token for an account"
-        client = imap_mailbox.client
-        client.send(b'a01 SETMETADATA INBOX (/private/devicetoken "1111" )\n')
-        res = client.readline()
-        assert b"OK Setmetadata completed" in res
-
-        client.send(b"a02 GETMETADATA INBOX /private/devicetoken\n")
-        res = client.readline()
-        assert res[:1] == b"*"
-        res = client.readline().strip().rstrip(b")")
-        assert res == b"1111"
-        assert b"Getmetadata completed" in client.readline()
-
-        client.send(b'a01 SETMETADATA INBOX (/private/devicetoken "2222" )\n')
-        res = client.readline()
-        assert b"OK Setmetadata completed" in res
-
-        client.send(b"a02 GETMETADATA INBOX /private/devicetoken\n")
-        res = client.readline()
-        assert res[:1] == b"*"
-        res = client.readline().strip().rstrip(b")")
-        assert res == b"1111 2222"
-        assert b"Getmetadata completed" in client.readline()
 
 
 class TestEndToEndDeltaChat:
@@ -103,7 +63,7 @@ class TestEndToEndDeltaChat:
 
         addr = ac2.get_config("addr").lower()
         saved_ok = 0
-        for line in remote.iter_output("journalctl -n0 -f -u dovecot"):
+        for line in remote.iter_output("journalctl -f -u dovecot"):
             if addr not in line:
                 # print(line)
                 continue
@@ -115,10 +75,7 @@ class TestEndToEndDeltaChat:
                         )
                     lp.indent("good, message sending failed because quota was exceeded")
                     return
-            if (
-                "stored mail into mailbox 'inbox'" in line
-                or "saved mail to inbox" in line
-            ):
+            if "saved mail to inbox" in line:
                 saved_ok += 1
                 print(f"{saved_ok}: {line}")
                 if saved_ok >= num_to_send:
@@ -155,7 +112,7 @@ class TestEndToEndDeltaChat:
         lp.sec("ac1 sends a message and ac2 marks it as seen")
         chat = ac1.create_chat(ac2)
         msg = chat.send_text("hi")
-        m = ac2._evtracker.wait_next_incoming_message()
+        m = ac2.wait_next_incoming_message()
         m.mark_seen()
         # we can only indirectly wait for mark-seen to cause an smtp-error
         lp.sec("try to wait for markseen to complete and check error states")
@@ -175,7 +132,7 @@ def test_hide_senders_ip_address(cmfactory):
     chat = cmfactory.get_accepted_chat(user1, user2)
 
     chat.send_text("testing submission header cleanup")
-    user2._evtracker.wait_next_incoming_message()
+    user2.wait_next_incoming_message()
     user2.direct_imap.select_folder("Inbox")
     msg = user2.direct_imap.get_all_messages()[0]
     assert public_ip not in msg.obj.as_string()
@@ -189,5 +146,5 @@ def test_echobot(cmfactory, chatmail_config, lp):
     text = "hi, I hope you text me back"
     chat.send_text(text)
     lp.sec("Wait for reply from echobot")
-    reply = ac._evtracker.wait_next_incoming_message()
+    reply = ac.wait_next_incoming_message()
     assert reply.text == text