add changelog entries

CHANGELOG.md

@@ -0,0 +1,22 @@
+# Changelog for chatmail deployment 
+
+## unreleased
+
+### Changes since March 15th, 2024
+
+- Fix various tests to pass again with "cmdeploy test". 
+  ([#245](https://github.com/deltachat/chatmail/pull/245),
+  [#242](https://github.com/deltachat/chatmail/pull/242)
+
+- Ensure lets-encrypt certificates are reloaded after renewal 
+  ([#244]) https://github.com/deltachat/chatmail/pull/244
+
+- Persist tokens to avoid iOS users loosing push-notifications when the
+  chatmail metadata service is restarted (happens regularly during deploys)
+  ([#238](https://github.com/deltachat/chatmail/pull/239)
+
+- Fix failing sieve-script compile errors on incoming messages
+  ([#237](https://github.com/deltachat/chatmail/pull/239)
+
+- Fix quota reporting after expunging of old mails
+  ([#233](https://github.com/deltachat/chatmail/pull/239)

chatmaild/src/chatmaild/metadata.py

@@ -1,6 +1,6 @@
 import pwd
 
-from pathlib import Path
+import pathlib
 from queue import Queue
 from threading import Thread
 from socketserver import (
@@ -13,6 +13,7 @@ import sys
 import logging
 import os
 import requests
+import marshal
 
 
 DICTPROXY_LOOKUP_CHAR = "L"
@@ -28,19 +29,33 @@ class Notifier:
         self.metadata_dir = metadata_dir
         self.to_notify_queue = Queue()
 
-    def set_token(self, guid, token):
+    def get_metadata(self, guid):
+        guid_path = self.metadata_dir.joinpath(guid)
+        if guid_path.exists():
+            with guid_path.open("rb") as f:
+                return marshal.load(f)
+        return {}
+
+    def set_metadata(self, guid, guid_data):
         guid_path = self.metadata_dir.joinpath(guid)
         write_path = guid_path.with_suffix(".tmp")
-        write_path.write_text(token)
-        write_path.rename(guid_path)
+        with write_path.open("wb") as f:
+            marshal.dump(guid_data, f)
+        os.rename(write_path, guid_path)
+
+    def set_token(self, guid, token):
+        guid_data = self.get_metadata(guid)
+        guid_data["token"] = token
+        self.set_metadata(guid, guid_data)
 
     def del_token(self, guid):
-        self.metadata_dir.joinpath(guid).unlink(missing_ok=True)
+        guid_data = self.get_metadata(guid)
+        if "token" in guid_data:
+            del guid_data["token"]
+            self.set_metadata(guid, guid_data)
 
     def get_token(self, guid):
-        guid_path = self.metadata_dir / guid
-        if guid_path.exists():
-            return guid_path.read_text()
+        return self.get_metadata(guid).get("token")
 
     def new_message_for_guid(self, guid):
         self.to_notify_queue.put(guid)
@@ -136,7 +151,7 @@ def main():
 
     # XXX config is not currently used
     config = read_config(config)
-    metadata_dir = Path(metadata_dir)
+    metadata_dir = pathlib.Path(metadata_dir)
     if not metadata_dir.exists():
         metadata_dir.mkdir()
     notifier = Notifier(metadata_dir)

cmdeploy/src/cmdeploy/__init__.py

@@ -350,7 +350,9 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
     need_restart |= lua_push_notification_script.changed
 
     sieve_script = files.put(
-        src=importlib.resources.files(__package__).joinpath("dovecot/default.sieve"),
+        src=importlib.resources.files(__package__).joinpath(
+            "dovecot/default.sieve"
+        ),
         dest="/etc/dovecot/default.sieve",
         user="root",
         group="root",
@@ -359,8 +361,10 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
     need_restart |= sieve_script.changed
     if sieve_script.changed:
         server.shell(
-            name="compile sieve script",
-            commands=["/usr/bin/sievec /etc/dovecot/default.sieve"],
+            name=f"compile sieve script",
+            commands=[
+                f"/usr/bin/sievec /etc/dovecot/default.sieve"
+            ],
         )
 
     files.template(
@@ -453,9 +457,7 @@ def check_config(config):
         blocked_words = "merlinux schmieder testrun.org".split()
         for key in config.__dict__:
             value = config.__dict__[key]
-            if key.startswith("privacy") and any(
-                x in str(value) for x in blocked_words
-            ):
+            if key.startswith("privacy") and any(x in str(value) for x in blocked_words):
                 raise ValueError(
                     f"please set your own privacy contacts/addresses in {config._inipath}"
                 )

cmdeploy/src/cmdeploy/dns.py

@@ -5,6 +5,8 @@ import importlib
 import subprocess
 import datetime
 
+from typing import Optional
+
 
 class DNS:
     def __init__(self, out, mail_domain):

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -63,7 +63,7 @@ class TestEndToEndDeltaChat:
 
         addr = ac2.get_config("addr").lower()
         saved_ok = 0
-        for line in remote.iter_output("journalctl -n0 -f -u dovecot"):
+        for line in remote.iter_output("journalctl -f -u dovecot"):
             if addr not in line:
                 # print(line)
                 continue
@@ -112,7 +112,7 @@ class TestEndToEndDeltaChat:
         lp.sec("ac1 sends a message and ac2 marks it as seen")
         chat = ac1.create_chat(ac2)
         msg = chat.send_text("hi")
-        m = ac2._evtracker.wait_next_incoming_message()
+        m = ac2.wait_next_incoming_message()
         m.mark_seen()
         # we can only indirectly wait for mark-seen to cause an smtp-error
         lp.sec("try to wait for markseen to complete and check error states")
@@ -132,7 +132,7 @@ def test_hide_senders_ip_address(cmfactory):
     chat = cmfactory.get_accepted_chat(user1, user2)
 
     chat.send_text("testing submission header cleanup")
-    user2._evtracker.wait_next_incoming_message()
+    user2.wait_next_incoming_message()
     user2.direct_imap.select_folder("Inbox")
     msg = user2.direct_imap.get_all_messages()[0]
     assert public_ip not in msg.obj.as_string()
@@ -146,5 +146,5 @@ def test_echobot(cmfactory, chatmail_config, lp):
     text = "hi, I hope you text me back"
     chat.send_text(text)
     lp.sec("Wait for reply from echobot")
-    reply = ac._evtracker.wait_next_incoming_message()
+    reply = ac.wait_next_incoming_message()
     assert reply.text == text