prepare 1.1.0 tag

They are part of deployment rather than service itself.
Different deployments may have different users,
filesystem layout etc.

CHANGELOG.md

@@ -0,0 +1,31 @@
+# Changelog for chatmail deployment 
+
+## 1.1.0 - 2024-03-28
+
+### The changelog starts to record changes from March 15th, 2024 
+
+- Move systemd unit templates to cmdeploy package 
+  ([#255](https://github.com/deltachat/chatmail/pull/255))
+
+- Persist push tokens and support multiple device per address 
+  ([#254](https://github.com/deltachat/chatmail/pull/254))
+
+- Avoid warning for regular doveauth protocol's hello message. 
+  ([#250](https://github.com/deltachat/chatmail/pull/250))
+
+- Fix various tests to pass again with "cmdeploy test". 
+  ([#245](https://github.com/deltachat/chatmail/pull/245),
+  [#242](https://github.com/deltachat/chatmail/pull/242)
+
+- Ensure lets-encrypt certificates are reloaded after renewal 
+  ([#244]) https://github.com/deltachat/chatmail/pull/244
+
+- Persist tokens to avoid iOS users loosing push-notifications when the
+  chatmail metadata service is restarted (happens regularly during deploys)
+  ([#238](https://github.com/deltachat/chatmail/pull/239)
+
+- Fix failing sieve-script compile errors on incoming messages
+  ([#237](https://github.com/deltachat/chatmail/pull/239)
+
+- Fix quota reporting after expunging of old mails
+  ([#233](https://github.com/deltachat/chatmail/pull/239)

chatmaild/pyproject.toml

@@ -10,6 +10,7 @@ dependencies = [
   "iniconfig",
   "deltachat-rpc-server",
   "deltachat-rpc-client",
+  "filelock",
   "requests",
 ]
 

chatmaild/src/chatmaild/doveauth.py

@@ -17,6 +17,10 @@ from .config import read_config, Config
 NOCREATE_FILE = "/etc/chatmail-nocreate"
 
 
+class UnknownCommand(ValueError):
+    """dictproxy handler received an unkown command"""
+
+
 def encrypt_password(password: str):
     # https://doc.dovecot.org/configuration_manual/authentication/password_schemes/
     passhash = crypt.crypt(password, crypt.METHOD_SHA512)
@@ -127,8 +131,12 @@ def split_and_unescape(s):
 
 
 def handle_dovecot_request(msg, db, config: Config):
+    # see https://doc.dovecot.org/3.0/developer_manual/design/dict_protocol/
     short_command = msg[0]
-    if short_command == "L":  # LOOKUP
+    if short_command == "H":  # HELLO
+        # we don't do any checking on versions and just return
+        return
+    elif short_command == "L":  # LOOKUP
         parts = msg[1:].split("\t")
 
         # Dovecot <2.3.17 has only one part,
@@ -159,7 +167,7 @@ def handle_dovecot_request(msg, db, config: Config):
                     reply_command = "N"
         json_res = json.dumps(res) if res else ""
         return f"{reply_command}{json_res}\n"
-    return None
+    raise UnknownCommand(msg)
 
 
 def handle_dovecot_protocol(rfile, wfile, db: Database, config: Config):
@@ -167,12 +175,14 @@ def handle_dovecot_protocol(rfile, wfile, db: Database, config: Config):
         msg = rfile.readline().strip().decode()
         if not msg:
             break
-        res = handle_dovecot_request(msg, db, config)
+        try:
-        if res:
+            res = handle_dovecot_request(msg, db, config)
-            wfile.write(res.encode("ascii"))
+        except UnknownCommand:
-            wfile.flush()
+            logging.warning("unknown command: %r", msg)
         else:
-            logging.warning("request had no answer: %r", msg)
+            if res:
+                wfile.write(res.encode("ascii"))
+                wfile.flush()
 
 
 class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):

chatmaild/src/chatmaild/echo.py

@@ -18,14 +18,14 @@ hooks = events.HookCollection()
 @hooks.on(events.RawEvent)
 def log_event(event):
     if event.kind == EventType.INFO:
-        logging.info(event.msg)
+        logging.info("%s", event.msg)
     elif event.kind == EventType.WARNING:
-        logging.warning(event.msg)
+        logging.warning("%s", event.msg)
 
 
 @hooks.on(events.RawEvent(EventType.ERROR))
 def log_error(event):
-    logging.error(event.msg)
+    logging.error("%s", event.msg)
 
 
 @hooks.on(events.MemberListChanged)
@@ -48,6 +48,9 @@ def on_group_name_changed(event):
 @hooks.on(events.NewMessage(func=lambda e: not e.command))
 def echo(event):
     snapshot = event.message_snapshot
+    if snapshot.is_info:
+        # Ignore info messages
+        return
     if snapshot.text or snapshot.file:
         snapshot.chat.send_message(text=snapshot.text, file=snapshot.file)
 
@@ -59,6 +62,7 @@ def help_command(event):
 
 
 def main():
+    logging.basicConfig(level=logging.INFO)
     path = os.environ.get("PATH")
     venv_path = sys.argv[0].strip("echobot")
     os.environ["PATH"] = path + ":" + venv_path
@@ -80,5 +84,4 @@ def main():
 
 
 if __name__ == "__main__":
-    logging.basicConfig(level=logging.INFO)
     main()

chatmaild/src/chatmaild/filedict.py

@@ -0,0 +1,35 @@
+import os
+import logging
+import json
+import filelock
+from contextlib import contextmanager
+
+
+class FileDict:
+    """Concurrency-safe multi-reader/single-writer persistent dict."""
+
+    def __init__(self, path):
+        self.path = path
+        self.lock_path = path.with_name(path.name + ".lock")
+
+    @contextmanager
+    def modify(self):
+        # the OS will release the lock if the process dies,
+        # and the contextmanager will otherwise guarantee release
+        with filelock.FileLock(self.lock_path):
+            data = self.read()
+            yield data
+            write_path = self.path.with_name(self.path.name + ".tmp")
+            with write_path.open("w") as f:
+                json.dump(data, f)
+            os.rename(write_path, self.path)
+
+    def read(self):
+        try:
+            with self.path.open("r") as f:
+                return json.load(f)
+        except FileNotFoundError:
+            return {}
+        except Exception:
+            logging.warning("corrupt serialization state at: %r", self.path)
+            return {}

chatmaild/src/chatmaild/metadata.py

@@ -1,62 +1,93 @@
 import pwd
 
-from queue import Queue
+from pathlib import Path
-from threading import Thread
+from threading import Thread, Event
 from socketserver import (
     UnixStreamServer,
     StreamRequestHandler,
     ThreadingMixIn,
 )
-from .config import read_config
 import sys
 import logging
 import os
 import requests
 
+from .filedict import FileDict
 
+
+DICTPROXY_HELLO_CHAR = "H"
 DICTPROXY_LOOKUP_CHAR = "L"
 DICTPROXY_ITERATE_CHAR = "I"
-DICTPROXY_SET_CHAR = "S"
 DICTPROXY_BEGIN_TRANSACTION_CHAR = "B"
+DICTPROXY_SET_CHAR = "S"
 DICTPROXY_COMMIT_TRANSACTION_CHAR = "C"
-DICTPROXY_TRANSACTION_CHARS = "SBC"
+DICTPROXY_TRANSACTION_CHARS = "BSC"
+
+# each SETMETADATA on this key appends to a list of unique device tokens
+# which only ever get removed if the upstream indicates the token is invalid
+METADATA_TOKEN_KEY = "devicetoken"
 
 
 class Notifier:
-    def __init__(self):
+    def __init__(self, vmail_dir):
-        self.guid2token = {}
+        self.vmail_dir = vmail_dir
-        self.to_notify_queue = Queue()
+        self.notification_dir = vmail_dir / "pending_notifications"
+        if not self.notification_dir.exists():
+            self.notification_dir.mkdir()
+        self.message_arrived_event = Event()
 
-    def set_token(self, guid, token):
+    def get_metadata_dict(self, addr):
-        self.guid2token[guid] = token
+        return FileDict(self.vmail_dir / addr / "metadata.json")
 
-    def new_message_for_guid(self, guid):
+    def add_token(self, addr, token):
-        self.to_notify_queue.put(guid)
+        with self.get_metadata_dict(addr).modify() as data:
+            tokens = data.get(METADATA_TOKEN_KEY)
+            if tokens is None:
+                data[METADATA_TOKEN_KEY] = [token]
+            elif token not in tokens:
+                tokens.append(token)
+
+    def remove_token(self, addr, token):
+        with self.get_metadata_dict(addr).modify() as data:
+            tokens = data.get(METADATA_TOKEN_KEY, [])
+            try:
+                tokens.remove(token)
+            except ValueError:
+                pass
+
+    def get_tokens(self, addr):
+        return self.get_metadata_dict(addr).read().get(METADATA_TOKEN_KEY, [])
+
+    def new_message_for_addr(self, addr):
+        self.notification_dir.joinpath(addr).touch()
+        self.message_arrived_event.set()
 
     def thread_run_loop(self):
         requests_session = requests.Session()
         while 1:
+            self.message_arrived_event.wait()
+            self.message_arrived_event.clear()
             self.thread_run_one(requests_session)
 
     def thread_run_one(self, requests_session):
-        guid = self.to_notify_queue.get()
+        for addr_path in self.notification_dir.iterdir():
-        token = self.guid2token.get(guid)
+            addr = addr_path.name
-        if token:
+            if "@" not in addr:
-            response = requests_session.post(
+                continue
-                "https://notifications.delta.chat/notify",
+            for token in self.get_tokens(addr):
-                data=token,
+                response = requests_session.post(
-                timeout=60,
+                    "https://notifications.delta.chat/notify",
-            )
+                    data=token,
-            if response.status_code == 410:
+                    timeout=60,
-                # 410 Gone status code
+                )
-                # means the token is no longer valid.
+                if response.status_code == 410:
-                del self.guid2token[guid]
+                    # 410 Gone status code
+                    # means the token is no longer valid.
+                    self.remove_token(addr, token)
+            addr_path.unlink()
 
 
 def handle_dovecot_protocol(rfile, wfile, notifier):
-    # HELLO message, ignored.
-    msg = rfile.readline().strip().decode()
-
     transactions = {}
     while True:
         msg = rfile.readline().strip().decode()
@@ -74,44 +105,52 @@ def handle_dovecot_request(msg, transactions, notifier):
     short_command = msg[0]
     parts = msg[1:].split("\t")
     if short_command == DICTPROXY_LOOKUP_CHAR:
+        # Lpriv/43f5f508a7ea0366dff30200c15250e3/devicetoken\tlkj123poi@c2.testrun.org
+        keyparts = parts[0].split("/")
+        if keyparts[0] == "priv":
+            keyname = keyparts[2]
+            addr = parts[1]
+            if keyname == METADATA_TOKEN_KEY:
+                res = " ".join(notifier.get_tokens(addr))
+                return f"O{res}\n"
+        logging.warning("lookup ignored: %r", msg)
         return "N\n"
     elif short_command == DICTPROXY_ITERATE_CHAR:
         # Empty line means ITER_FINISHED.
         # If we don't return empty line Dovecot will timeout.
         return "\n"
+    elif short_command == DICTPROXY_HELLO_CHAR:
+        return  # no version checking
 
     if short_command not in (DICTPROXY_TRANSACTION_CHARS):
+        logging.warning("unknown dictproxy request: %r", msg)
         return
 
     transaction_id = parts[0]
 
     if short_command == DICTPROXY_BEGIN_TRANSACTION_CHAR:
-        transactions[transaction_id] = "O\n"
+        addr = parts[1]
+        transactions[transaction_id] = dict(addr=addr, res="O\n")
     elif short_command == DICTPROXY_COMMIT_TRANSACTION_CHAR:
-        # returns whether it failed or succeeded.
+        # each set devicetoken operation persists directly
-        return transactions.pop(transaction_id, "N\n")
+        # and does not wait until a "commit" comes
+        # because our dovecot config does not involve
+        # multiple set-operations in a single commit
+        return transactions.pop(transaction_id)["res"]
     elif short_command == DICTPROXY_SET_CHAR:
-        # See header of
+        # For documentation on key structure see
-        # <https://github.com/dovecot/core/blob/5e7965632395793d9355eb906b173bf28d2a10ca/src/lib-storage/mailbox-attribute.h>
+        # https://github.com/dovecot/core/blob/main/src/lib-storage/mailbox-attribute.h
-        # for the documentation on the structure of the key.
-
-        # Request GETMETADATA "INBOX" /private/chatmail
-        # results in a query for
-        # priv/dd72550f05eadc65542a1200cac67ad7/chatmail
-        #
-        # Request GETMETADATA "" /private/chatmail
-        # results in
-        # priv/dd72550f05eadc65542a1200cac67ad7/vendor/vendor.dovecot/pvt/server/chatmail
 
         keyname = parts[1].split("/")
         value = parts[2] if len(parts) > 2 else ""
-        if keyname[0] == "priv" and keyname[2] == "devicetoken":
+        addr = transactions[transaction_id]["addr"]
-            notifier.set_token(keyname[1], value)
+        if keyname[0] == "priv" and keyname[2] == METADATA_TOKEN_KEY:
+            notifier.add_token(addr, value)
         elif keyname[0] == "priv" and keyname[2] == "messagenew":
-            notifier.new_message_for_guid(keyname[1])
+            notifier.new_message_for_addr(addr)
         else:
             # Transaction failed.
-            transactions[transaction_id] = "F\n"
+            transactions[transaction_id]["res"] = "F\n"
 
 
 class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
@@ -119,19 +158,23 @@ class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
 
 
 def main():
-    socket, username, config = sys.argv[1:]
+    socket, username, vmail_dir = sys.argv[1:]
     passwd_entry = pwd.getpwnam(username)
 
-    # XXX config is not currently used
+    vmail_dir = Path(vmail_dir)
-    config = read_config(config)
+
-    notifier = Notifier()
+    if not vmail_dir.exists():
+        logging.error("vmail dir does not exist: %r", vmail_dir)
+        return 1
+
+    notifier = Notifier(vmail_dir)
 
     class Handler(StreamRequestHandler):
         def handle(self):
             try:
                 handle_dovecot_protocol(self.rfile, self.wfile, notifier)
             except Exception:
-                logging.exception("Exception in the handler")
+                logging.exception("Exception in the dovecot dictproxy handler")
                 raise
 
     try:
@@ -145,6 +188,8 @@ def main():
     t = Thread(target=notifier.thread_run_loop)
     t.setDaemon(True)
     t.start()
+    # let notifier thread run once for any pending notifications from last run
+    notifier.message_arrived_event.set()
 
     with ThreadedUnixStreamServer(socket, Handler) as server:
         os.chown(socket, uid=passwd_entry.pw_uid, gid=passwd_entry.pw_gid)

chatmaild/src/chatmaild/tests/test_doveauth.py

@@ -75,6 +75,14 @@ def test_handle_dovecot_request(db, example_config):
     assert userdata["password"].startswith("{SHA512-CRYPT}")
 
 
+def test_handle_dovecot_protocol_hello_is_skipped(db, example_config, caplog):
+    rfile = io.BytesIO(b"H3\t2\t0\t\tauth\n")
+    wfile = io.BytesIO()
+    handle_dovecot_protocol(rfile, wfile, db, example_config)
+    assert wfile.getvalue() == b""
+    assert not caplog.messages
+
+
 def test_handle_dovecot_protocol(db, example_config):
     rfile = io.BytesIO(
         b"H3\t2\t0\t\tauth\nLshared/userdb/foobar@chat.example.org\tfoobar@chat.example.org\n"

chatmaild/src/chatmaild/tests/test_filedict.py

@@ -0,0 +1,19 @@
+from chatmaild.filedict import FileDict
+
+
+def test_basic(tmp_path):
+    fdict = FileDict(tmp_path.joinpath("metadata"))
+    assert fdict.read() == {}
+    with fdict.modify() as d:
+        d["devicetoken"] = [1, 2, 3]
+        d["456"] = 4.2
+    new = fdict.read()
+    assert new["devicetoken"] == [1, 2, 3]
+    assert new["456"] == 4.2
+
+
+def test_bad_marshal_file(tmp_path, caplog):
+    fdict1 = FileDict(tmp_path.joinpath("metadata"))
+    fdict1.path.write_bytes(b"l12k3l12k3l")
+    assert fdict1.read() == {}
+    assert "corrupt" in caplog.records[0].msg

chatmaild/src/chatmaild/tests/test_metadata.py

@@ -1,4 +1,5 @@
 import io
+import pytest
 
 from chatmaild.metadata import (
     handle_dovecot_request,
@@ -7,70 +8,130 @@ from chatmaild.metadata import (
 )
 
 
-def test_handle_dovecot_request_lookup_fails():
+@pytest.fixture
-    notifier = Notifier()
+def notifier(tmp_path):
-    res = handle_dovecot_request("Lpriv/123/chatmail", {}, notifier)
+    vmail_dir = tmp_path.joinpath("vmaildir")
+    vmail_dir.mkdir()
+    return Notifier(vmail_dir)
+
+
+@pytest.fixture
+def testaddr():
+    return "user.name@example.org"
+
+
+@pytest.fixture
+def testaddr2():
+    return "user2@example.org"
+
+
+def test_notifier_persistence(tmp_path, testaddr, testaddr2):
+    notifier1 = Notifier(tmp_path)
+    notifier2 = Notifier(tmp_path)
+    assert not notifier1.get_tokens(testaddr)
+    assert not notifier2.get_tokens(testaddr)
+
+    notifier1.add_token(testaddr, "01234")
+    notifier1.add_token(testaddr2, "456")
+    assert notifier2.get_tokens(testaddr) == ["01234"]
+    assert notifier2.get_tokens(testaddr2) == ["456"]
+    notifier2.remove_token(testaddr, "01234")
+    assert not notifier1.get_tokens(testaddr)
+    assert notifier1.get_tokens(testaddr2) == ["456"]
+
+
+def test_remove_nonexisting(tmp_path, testaddr):
+    notifier1 = Notifier(tmp_path)
+    notifier1.add_token(testaddr, "123")
+    notifier1.remove_token(testaddr, "1l23k1l2k3")
+    assert notifier1.get_tokens(testaddr) == ["123"]
+
+
+def test_notifier_delete_without_set(notifier, testaddr):
+    notifier.remove_token(testaddr, "123")
+    assert not notifier.get_tokens(testaddr)
+
+
+def test_handle_dovecot_request_lookup_fails(notifier, testaddr):
+    res = handle_dovecot_request(f"Lpriv/123/chatmail\t{testaddr}", {}, notifier)
     assert res == "N\n"
 
 
-def test_handle_dovecot_request_happy_path():
+def test_handle_dovecot_request_happy_path(notifier, testaddr):
-    notifier = Notifier()
     transactions = {}
 
-    # lookups return the same NOTFOUND result
-    res = handle_dovecot_request("Lpriv/123/chatmail", transactions, notifier)
-    assert res == "N\n"
-    assert not notifier.guid2token and not transactions
-
     # set device token in a transaction
     tx = "1111"
-    msg = f"B{tx}\tuser"
+    msg = f"B{tx}\t{testaddr}"
     res = handle_dovecot_request(msg, transactions, notifier)
-    assert not res and not notifier.guid2token
+    assert not res and not notifier.get_tokens(testaddr)
-    assert transactions == {tx: "O\n"}
+    assert transactions == {tx: dict(addr=testaddr, res="O\n")}
 
     msg = f"S{tx}\tpriv/guid00/devicetoken\t01234"
     res = handle_dovecot_request(msg, transactions, notifier)
     assert not res
     assert len(transactions) == 1
-    assert len(notifier.guid2token) == 1
+    assert notifier.get_tokens(testaddr) == ["01234"]
-    assert notifier.guid2token["guid00"] == "01234"
 
     msg = f"C{tx}"
     res = handle_dovecot_request(msg, transactions, notifier)
     assert res == "O\n"
     assert len(transactions) == 0
-    assert notifier.guid2token["guid00"] == "01234"
+    assert notifier.get_tokens(testaddr) == ["01234"]
 
     # trigger notification for incoming message
-    assert handle_dovecot_request(f"B{tx}\tuser", transactions, notifier) is None
+    tx2 = "2222"
-    msg = f"S{tx}\tpriv/guid00/messagenew"
+    assert handle_dovecot_request(f"B{tx2}\t{testaddr}", transactions, notifier) is None
+    msg = f"S{tx2}\tpriv/guid00/messagenew"
     assert handle_dovecot_request(msg, transactions, notifier) is None
-    assert notifier.to_notify_queue.get() == "guid00"
+    assert notifier.message_arrived_event.is_set()
-    assert notifier.to_notify_queue.qsize() == 0
+    assert handle_dovecot_request(f"C{tx2}", transactions, notifier) == "O\n"
-    assert handle_dovecot_request(f"C{tx}\tuser", transactions, notifier) == "O\n"
     assert not transactions
+    assert notifier.notification_dir.joinpath(testaddr).exists()
 
 
-def test_handle_dovecot_protocol_set_devicetoken():
+def test_handle_dovecot_protocol_set_devicetoken(notifier):
     rfile = io.BytesIO(
         b"\n".join(
             [
                 b"HELLO",
-                b"Btx00\tuser",
+                b"Btx00\tuser@example.org",
                 b"Stx00\tpriv/guid00/devicetoken\t01234",
                 b"Ctx00",
             ]
         )
     )
     wfile = io.BytesIO()
-    notifier = Notifier()
     handle_dovecot_protocol(rfile, wfile, notifier)
-    assert notifier.guid2token["guid00"] == "01234"
+    assert wfile.getvalue() == b"O\n"
+    assert notifier.get_tokens("user@example.org") == ["01234"]
+
+
+def test_handle_dovecot_protocol_set_get_devicetoken(notifier):
+    rfile = io.BytesIO(
+        b"\n".join(
+            [
+                b"HELLO",
+                b"Btx00\tuser@example.org",
+                b"Stx00\tpriv/guid00/devicetoken\t01234",
+                b"Ctx00",
+            ]
+        )
+    )
+    wfile = io.BytesIO()
+    handle_dovecot_protocol(rfile, wfile, notifier)
+    assert notifier.get_tokens("user@example.org") == ["01234"]
     assert wfile.getvalue() == b"O\n"
 
+    rfile = io.BytesIO(
+        b"\n".join([b"HELLO", b"Lpriv/0123/devicetoken\tuser@example.org"])
+    )
+    wfile = io.BytesIO()
+    handle_dovecot_protocol(rfile, wfile, notifier)
+    assert wfile.getvalue() == b"O01234\n"
 
-def test_handle_dovecot_protocol_iterate():
+
+def test_handle_dovecot_protocol_iterate(notifier):
     rfile = io.BytesIO(
         b"\n".join(
             [
@@ -80,31 +141,29 @@ def test_handle_dovecot_protocol_iterate():
         )
     )
     wfile = io.BytesIO()
-    notifier = Notifier()
     handle_dovecot_protocol(rfile, wfile, notifier)
     assert wfile.getvalue() == b"\n"
 
 
-def test_handle_dovecot_protocol_messagenew():
+def test_handle_dovecot_protocol_messagenew(notifier):
     rfile = io.BytesIO(
         b"\n".join(
             [
                 b"HELLO",
-                b"Btx01\tuser",
+                b"Btx01\tuser@example.org",
                 b"Stx01\tpriv/guid00/messagenew",
                 b"Ctx01",
             ]
         )
     )
     wfile = io.BytesIO()
-    notifier = Notifier()
     handle_dovecot_protocol(rfile, wfile, notifier)
     assert wfile.getvalue() == b"O\n"
-    assert notifier.to_notify_queue.get() == "guid00"
+    assert notifier.message_arrived_event.is_set()
-    assert notifier.to_notify_queue.qsize() == 0
+    assert notifier.notification_dir.joinpath("user@example.org").exists()
 
 
-def test_notifier_thread_run():
+def test_notifier_thread_run(notifier, testaddr):
     requests = []
 
     class ReqMock:
@@ -116,16 +175,15 @@ def test_notifier_thread_run():
 
             return Result()
 
-    notifier = Notifier()
+    notifier.add_token(testaddr, "01234")
-    notifier.set_token("guid00", "01234")
+    notifier.new_message_for_addr(testaddr)
-    notifier.new_message_for_guid("guid00")
     notifier.thread_run_one(ReqMock())
     url, data, timeout = requests[0]
     assert data == "01234"
-    assert len(notifier.guid2token) == 1
+    assert notifier.get_tokens(testaddr) == ["01234"]
 
 
-def test_notifier_thread_run_gone_removes_token():
+def test_multi_device_notifier(notifier, testaddr):
     requests = []
 
     class ReqMock:
@@ -133,15 +191,40 @@ def test_notifier_thread_run_gone_removes_token():
             requests.append((url, data, timeout))
 
             class Result:
-                status_code = 410
+                status_code = 200
 
             return Result()
 
-    notifier = Notifier()
+    notifier.add_token(testaddr, "01234")
-    notifier.set_token("guid00", "01234")
+    notifier.add_token(testaddr, "56789")
-    notifier.new_message_for_guid("guid00")
+    notifier.new_message_for_addr(testaddr)
-    assert notifier.guid2token["guid00"] == "01234"
     notifier.thread_run_one(ReqMock())
     url, data, timeout = requests[0]
     assert data == "01234"
-    assert len(notifier.guid2token) == 0
+    url, data, timeout = requests[1]
+    assert data == "56789"
+    assert notifier.get_tokens(testaddr) == ["01234", "56789"]
+
+
+def test_notifier_thread_run_gone_removes_token(notifier, testaddr):
+    requests = []
+
+    class ReqMock:
+        def post(self, url, data, timeout):
+            requests.append((url, data, timeout))
+
+            class Result:
+                status_code = 410 if data == "01234" else 200
+
+            return Result()
+
+    notifier.add_token(testaddr, "01234")
+    notifier.new_message_for_addr(testaddr)
+    assert notifier.get_tokens(testaddr) == ["01234"]
+    notifier.add_token(testaddr, "45678")
+    notifier.thread_run_one(ReqMock())
+    url, data, timeout = requests[0]
+    assert data == "01234"
+    url, data, timeout = requests[1]
+    assert data == "45678"
+    assert notifier.get_tokens(testaddr) == ["45678"]

cmdeploy/pyproject.toml

@@ -19,6 +19,7 @@ dependencies = [
   "black",
   "pytest",
   "pytest-xdist", 
+  "imap_tools",
 ]
 
 [project.scripts]

cmdeploy/src/cmdeploy/__init__.py

@@ -108,8 +108,11 @@ def _install_remote_venv_with_chatmaild(config) -> None:
             execpath=f"{remote_venv_dir}/bin/{fn}",
             config_path=remote_chatmail_inipath,
             remote_venv_dir=remote_venv_dir,
+            mail_domain=config.mail_domain,
+        )
+        source_path = importlib.resources.files(__package__).joinpath(
+            "service", f"{fn}.service.f"
         )
-        source_path = importlib.resources.files("chatmaild").joinpath(f"{fn}.service.f")
         content = source_path.read_text().format(**params).encode()
 
         files.put(
@@ -350,15 +353,18 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
     need_restart |= lua_push_notification_script.changed
 
     sieve_script = files.put(
-        src=importlib.resources.files(__package__).joinpath(
+        src=importlib.resources.files(__package__).joinpath("dovecot/default.sieve"),
-            "dovecot/default.sieve"
-        ),
         dest="/etc/dovecot/default.sieve",
         user="root",
         group="root",
         mode="644",
     )
     need_restart |= sieve_script.changed
+    if sieve_script.changed:
+        server.shell(
+            name="compile sieve script",
+            commands=["/usr/bin/sievec /etc/dovecot/default.sieve"],
+        )
 
     files.template(
         src=importlib.resources.files(__package__).joinpath("dovecot/expunge.cron.j2"),
@@ -450,7 +456,9 @@ def check_config(config):
         blocked_words = "merlinux schmieder testrun.org".split()
         for key in config.__dict__:
             value = config.__dict__[key]
-            if key.startswith("privacy") and any(x in str(value) for x in blocked_words):
+            if key.startswith("privacy") and any(
+                x in str(value) for x in blocked_words
+            ):
                 raise ValueError(
                     f"please set your own privacy contacts/addresses in {config._inipath}"
                 )

cmdeploy/src/cmdeploy/acmetool/acmetool.cron

@@ -1,4 +1,4 @@
 SHELL=/bin/sh
 PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
 MAILTO=root
-20 16 * * * root /usr/bin/acmetool --batch reconcile
+20 16 * * * root /usr/bin/acmetool --batch reconcile && systemctl reload dovecot && systemctl reload postfix

cmdeploy/src/cmdeploy/chatmail.zone.f

@@ -6,7 +6,7 @@ _submissions._tcp.{chatmail_domain}. SRV 0 1 465 {chatmail_domain}.
 _imap._tcp.{chatmail_domain}.        SRV 0 1 143 {chatmail_domain}.
 _imaps._tcp.{chatmail_domain}.       SRV 0 1 993 {chatmail_domain}.
 {chatmail_domain}.                   CAA 128 issue "letsencrypt.org;accounturi={acme_account_url}"
-{chatmail_domain}.                   TXT "v=spf1 a:{chatmail_domain} -all"
+{chatmail_domain}.                   TXT "v=spf1 a:{chatmail_domain} ~all"
 _dmarc.{chatmail_domain}.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
 _mta-sts.{chatmail_domain}.          TXT "v=STSv1; id={sts_id}"
 mta-sts.{chatmail_domain}.           CNAME {chatmail_domain}.

cmdeploy/src/cmdeploy/dns.py

@@ -5,8 +5,6 @@ import importlib
 import subprocess
 import datetime
 
-from typing import Optional
-
 
 class DNS:
     def __init__(self, out, mail_domain):

cmdeploy/src/cmdeploy/dovecot/default.sieve

@@ -1,5 +1,7 @@
 require ["imap4flags"];
 
+# flag the message so it doesn't cause a push notification
+
 if header :is ["Auto-Submitted"] ["auto-replied", "auto-generated"] {
 	addflag "$Auto";
 }

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -13,6 +13,12 @@ auth_cache_size = 100M
 mail_debug = yes
 {% endif %}
 
+# Prevent warnings similar to:
+#   config: Warning: service auth { client_limit=1000 } is lower than required under max. load (10200). Counted for protocol services with service_count != 1: service lmtp { process_limit=100 } + service imap-urlauth-login { process_limit=100 } + service imap-login { process_limit=10000 }
+#   config: Warning: service anvil { client_limit=1000 } is lower than required under max. load (10103). Counted with: service imap-urlauth-login { process_limit=100 } + service imap-login { process_limit=10000 } + service auth { process_limit=1 }
+#   master: Warning: service(stats): client_limit (1000) reached, client connections are being dropped
+default_client_limit = 20000
+
 mail_server_admin = mailto:root@{{ config.mail_domain }}
 mail_server_comment = Chatmail server
 

cmdeploy/src/cmdeploy/dovecot/expunge.cron.j2

@@ -8,3 +8,4 @@
 # or only temporary (but then they shouldn't be around after {{ config.delete_mails_after }} days anyway).
 2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+3 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -name 'maildirsize' -type f -delete

cmdeploy/src/cmdeploy/service/chatmail-metadata.service.f

@@ -2,7 +2,7 @@
 Description=Chatmail dict proxy for IMAP METADATA
 
 [Service]
-ExecStart={execpath} /run/dovecot/metadata.socket vmail {config_path}
+ExecStart={execpath} /run/dovecot/metadata.socket vmail /home/vmail/mail/{mail_domain}
 Restart=always
 RestartSec=30
 

cmdeploy/src/cmdeploy/service/filtermail.service.f

@@ -1,5 +1,5 @@
 [Unit]
-Description=Chatmail Postfix BeforeQeue filter 
+Description=Chatmail Postfix before queue filter
 
 [Service]
 ExecStart={execpath} {config_path}

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -5,6 +5,46 @@ import random
 import pytest
 import requests
 import ipaddress
+import imap_tools
+
+
+@pytest.fixture
+def imap_mailbox(cmfactory):
+    (ac1,) = cmfactory.get_online_accounts(1)
+    user = ac1.get_config("addr")
+    password = ac1.get_config("mail_pw")
+    mailbox = imap_tools.MailBox(user.split("@")[1])
+    mailbox.login(user, password)
+    return mailbox
+
+
+class TestMetadataTokens:
+    "Tests that use Metadata extension for storing tokens"
+
+    def test_set_get_metadata(self, imap_mailbox):
+        "set and get metadata token for an account"
+        client = imap_mailbox.client
+        client.send(b'a01 SETMETADATA INBOX (/private/devicetoken "1111" )\n')
+        res = client.readline()
+        assert b"OK Setmetadata completed" in res
+
+        client.send(b"a02 GETMETADATA INBOX /private/devicetoken\n")
+        res = client.readline()
+        assert res[:1] == b"*"
+        res = client.readline().strip().rstrip(b")")
+        assert res == b"1111"
+        assert b"Getmetadata completed" in client.readline()
+
+        client.send(b'a01 SETMETADATA INBOX (/private/devicetoken "2222" )\n')
+        res = client.readline()
+        assert b"OK Setmetadata completed" in res
+
+        client.send(b"a02 GETMETADATA INBOX /private/devicetoken\n")
+        res = client.readline()
+        assert res[:1] == b"*"
+        res = client.readline().strip().rstrip(b")")
+        assert res == b"1111 2222"
+        assert b"Getmetadata completed" in client.readline()
 
 
 class TestEndToEndDeltaChat:
@@ -63,7 +103,7 @@ class TestEndToEndDeltaChat:
 
         addr = ac2.get_config("addr").lower()
         saved_ok = 0
-        for line in remote.iter_output("journalctl -f -u dovecot"):
+        for line in remote.iter_output("journalctl -n0 -f -u dovecot"):
             if addr not in line:
                 # print(line)
                 continue
@@ -75,7 +115,10 @@ class TestEndToEndDeltaChat:
                         )
                     lp.indent("good, message sending failed because quota was exceeded")
                     return
-            if "saved mail to inbox" in line:
+            if (
+                "stored mail into mailbox 'inbox'" in line
+                or "saved mail to inbox" in line
+            ):
                 saved_ok += 1
                 print(f"{saved_ok}: {line}")
                 if saved_ok >= num_to_send:
@@ -112,7 +155,7 @@ class TestEndToEndDeltaChat:
         lp.sec("ac1 sends a message and ac2 marks it as seen")
         chat = ac1.create_chat(ac2)
         msg = chat.send_text("hi")
-        m = ac2.wait_next_incoming_message()
+        m = ac2._evtracker.wait_next_incoming_message()
         m.mark_seen()
         # we can only indirectly wait for mark-seen to cause an smtp-error
         lp.sec("try to wait for markseen to complete and check error states")
@@ -132,7 +175,7 @@ def test_hide_senders_ip_address(cmfactory):
     chat = cmfactory.get_accepted_chat(user1, user2)
 
     chat.send_text("testing submission header cleanup")
-    user2.wait_next_incoming_message()
+    user2._evtracker.wait_next_incoming_message()
     user2.direct_imap.select_folder("Inbox")
     msg = user2.direct_imap.get_all_messages()[0]
     assert public_ip not in msg.obj.as_string()
@@ -146,5 +189,5 @@ def test_echobot(cmfactory, chatmail_config, lp):
     text = "hi, I hope you text me back"
     chat.send_text(text)
     lp.sec("Wait for reply from echobot")
-    reply = ac.wait_next_incoming_message()
+    reply = ac._evtracker.wait_next_incoming_message()
     assert reply.text == text