Store raw tokens instead of dictionaries in metadata

CHANGELOG.md

@@ -1,22 +0,0 @@
-# Changelog for chatmail deployment 
-
-## unreleased
-
-### Changes since March 15th, 2024
-
-- Fix various tests to pass again with "cmdeploy test". 
-  ([#245](https://github.com/deltachat/chatmail/pull/245),
-  [#242](https://github.com/deltachat/chatmail/pull/242)
-
-- Ensure lets-encrypt certificates are reloaded after renewal 
-  ([#244]) https://github.com/deltachat/chatmail/pull/244
-
-- Persist tokens to avoid iOS users loosing push-notifications when the
-  chatmail metadata service is restarted (happens regularly during deploys)
-  ([#238](https://github.com/deltachat/chatmail/pull/239)
-
-- Fix failing sieve-script compile errors on incoming messages
-  ([#237](https://github.com/deltachat/chatmail/pull/239)
-
-- Fix quota reporting after expunging of old mails
-  ([#233](https://github.com/deltachat/chatmail/pull/239)

chatmaild/src/chatmaild/doveauth.py

@@ -17,10 +17,6 @@ from .config import read_config, Config
 NOCREATE_FILE = "/etc/chatmail-nocreate"
 
 
-class UnknownCommand(ValueError):
-    """dictproxy handler received an unkown command"""
-
-
 def encrypt_password(password: str):
     # https://doc.dovecot.org/configuration_manual/authentication/password_schemes/
     passhash = crypt.crypt(password, crypt.METHOD_SHA512)
@@ -131,12 +127,8 @@ def split_and_unescape(s):
 
 
 def handle_dovecot_request(msg, db, config: Config):
-    # see https://doc.dovecot.org/3.0/developer_manual/design/dict_protocol/
     short_command = msg[0]
-    if short_command == "H":  # HELLO
+    if short_command == "L":  # LOOKUP
-        # we don't do any checking on versions and just return
-        return
-    elif short_command == "L":  # LOOKUP
         parts = msg[1:].split("\t")
 
         # Dovecot <2.3.17 has only one part,
@@ -167,7 +159,7 @@ def handle_dovecot_request(msg, db, config: Config):
                     reply_command = "N"
         json_res = json.dumps(res) if res else ""
         return f"{reply_command}{json_res}\n"
-    raise UnknownCommand(msg)
+    return None
 
 
 def handle_dovecot_protocol(rfile, wfile, db: Database, config: Config):
@@ -175,14 +167,12 @@ def handle_dovecot_protocol(rfile, wfile, db: Database, config: Config):
         msg = rfile.readline().strip().decode()
         if not msg:
             break
-        try:
+        res = handle_dovecot_request(msg, db, config)
-            res = handle_dovecot_request(msg, db, config)
+        if res:
-        except UnknownCommand:
+            wfile.write(res.encode("ascii"))
-            logging.warning("unknown command: %r", msg)
+            wfile.flush()
         else:
-            if res:
+            logging.warning("request had no answer: %r", msg)
-                wfile.write(res.encode("ascii"))
-                wfile.flush()
 
 
 class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):

chatmaild/src/chatmaild/echo.py

@@ -18,14 +18,14 @@ hooks = events.HookCollection()
 @hooks.on(events.RawEvent)
 def log_event(event):
     if event.kind == EventType.INFO:
-        logging.info("%s", event.msg)
+        logging.info(event.msg)
     elif event.kind == EventType.WARNING:
-        logging.warning("%s", event.msg)
+        logging.warning(event.msg)
 
 
 @hooks.on(events.RawEvent(EventType.ERROR))
 def log_error(event):
-    logging.error("%s", event.msg)
+    logging.error(event.msg)
 
 
 @hooks.on(events.MemberListChanged)
@@ -48,9 +48,6 @@ def on_group_name_changed(event):
 @hooks.on(events.NewMessage(func=lambda e: not e.command))
 def echo(event):
     snapshot = event.message_snapshot
-    if snapshot.is_info:
-        # Ignore info messages
-        return
     if snapshot.text or snapshot.file:
         snapshot.chat.send_message(text=snapshot.text, file=snapshot.file)
 
@@ -62,7 +59,6 @@ def help_command(event):
 
 
 def main():
-    logging.basicConfig(level=logging.INFO)
     path = os.environ.get("PATH")
     venv_path = sys.argv[0].strip("echobot")
     os.environ["PATH"] = path + ":" + venv_path
@@ -84,4 +80,5 @@ def main():
 
 
 if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
     main()

chatmaild/src/chatmaild/metadata.py

@@ -1,6 +1,6 @@
 import pwd
 
-import pathlib
+from pathlib import Path
 from queue import Queue
 from threading import Thread
 from socketserver import (
@@ -13,7 +13,6 @@ import sys
 import logging
 import os
 import requests
-import marshal
 
 
 DICTPROXY_LOOKUP_CHAR = "L"
@@ -29,33 +28,19 @@ class Notifier:
         self.metadata_dir = metadata_dir
         self.to_notify_queue = Queue()
 
-    def get_metadata(self, guid):
+    def set_token(self, guid, token):
-        guid_path = self.metadata_dir.joinpath(guid)
-        if guid_path.exists():
-            with guid_path.open("rb") as f:
-                return marshal.load(f)
-        return {}
-
-    def set_metadata(self, guid, guid_data):
         guid_path = self.metadata_dir.joinpath(guid)
         write_path = guid_path.with_suffix(".tmp")
-        with write_path.open("wb") as f:
+        write_path.write_text(token)
-            marshal.dump(guid_data, f)
+        write_path.rename(guid_path)
-        os.rename(write_path, guid_path)
-
-    def set_token(self, guid, token):
-        guid_data = self.get_metadata(guid)
-        guid_data["token"] = token
-        self.set_metadata(guid, guid_data)
 
     def del_token(self, guid):
-        guid_data = self.get_metadata(guid)
+        self.metadata_dir.joinpath(guid).unlink(missing_ok=True)
-        if "token" in guid_data:
-            del guid_data["token"]
-            self.set_metadata(guid, guid_data)
 
     def get_token(self, guid):
-        return self.get_metadata(guid).get("token")
+        guid_path = self.metadata_dir / guid
+        if guid_path.exists():
+            return guid_path.read_text()
 
     def new_message_for_guid(self, guid):
         self.to_notify_queue.put(guid)
@@ -151,7 +136,7 @@ def main():
 
     # XXX config is not currently used
     config = read_config(config)
-    metadata_dir = pathlib.Path(metadata_dir)
+    metadata_dir = Path(metadata_dir)
     if not metadata_dir.exists():
         metadata_dir.mkdir()
     notifier = Notifier(metadata_dir)

chatmaild/src/chatmaild/tests/test_doveauth.py

@@ -75,14 +75,6 @@ def test_handle_dovecot_request(db, example_config):
     assert userdata["password"].startswith("{SHA512-CRYPT}")
 
 
-def test_handle_dovecot_protocol_hello_is_skipped(db, example_config, caplog):
-    rfile = io.BytesIO(b"H3\t2\t0\t\tauth\n")
-    wfile = io.BytesIO()
-    handle_dovecot_protocol(rfile, wfile, db, example_config)
-    assert wfile.getvalue() == b""
-    assert not caplog.messages
-
-
 def test_handle_dovecot_protocol(db, example_config):
     rfile = io.BytesIO(
         b"H3\t2\t0\t\tauth\nLshared/userdb/foobar@chat.example.org\tfoobar@chat.example.org\n"

cmdeploy/src/cmdeploy/acmetool/acmetool.cron

@@ -1,4 +1,4 @@
 SHELL=/bin/sh
 PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
 MAILTO=root
-20 16 * * * root /usr/bin/acmetool --batch reconcile && systemctl reload dovecot && systemctl reload postfix
+20 16 * * * root /usr/bin/acmetool --batch reconcile

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -75,7 +75,7 @@ class TestEndToEndDeltaChat:
                         )
                     lp.indent("good, message sending failed because quota was exceeded")
                     return
-            if "stored mail into mailbox 'inbox'" in line or "saved mail to inbox" in line:
+            if "saved mail to inbox" in line:
                 saved_ok += 1
                 print(f"{saved_ok}: {line}")
                 if saved_ok >= num_to_send: