Implement "iterate" command in metadata server

Otherwise Dovecot times out when trying to iterate over metadata
of the folder. Apparently it happens when attempting to delete
folder from the server over IMAP.

chatmaild/src/chatmaild/chatmail-metadata.service.f

@@ -2,7 +2,7 @@
 Description=Chatmail dict proxy for IMAP METADATA
 
 [Service]
-ExecStart={execpath} /run/dovecot/metadata.socket vmail {config_path} /home/vmail/metadata
+ExecStart={execpath} /run/dovecot/metadata.socket vmail {config_path}
 Restart=always
 RestartSec=30
 

chatmaild/src/chatmaild/metadata.py

@@ -1,6 +1,5 @@
 import pwd
 
-import pathlib
 from queue import Queue
 from threading import Thread
 from socketserver import (
@@ -13,7 +12,6 @@ import sys
 import logging
 import os
 import requests
-import marshal
 
 
 DICTPROXY_LOOKUP_CHAR = "L"
@@ -25,37 +23,12 @@ DICTPROXY_TRANSACTION_CHARS = "SBC"
 
 
 class Notifier:
-    def __init__(self, metadata_dir):
-        self.metadata_dir = metadata_dir
+    def __init__(self):
+        self.guid2token = {}
         self.to_notify_queue = Queue()
 
-    def get_metadata(self, guid):
-        guid_path = self.metadata_dir.joinpath(guid)
-        if guid_path.exists():
-            with guid_path.open("rb") as f:
-                return marshal.load(f)
-        return {}
-
-    def set_metadata(self, guid, guid_data):
-        guid_path = self.metadata_dir.joinpath(guid)
-        write_path = guid_path.with_suffix(".tmp")
-        with write_path.open("wb") as f:
-            marshal.dump(guid_data, f)
-        os.rename(write_path, guid_path)
-
     def set_token(self, guid, token):
-        guid_data = self.get_metadata(guid)
-        guid_data["token"] = token
-        self.set_metadata(guid, guid_data)
-
-    def del_token(self, guid):
-        guid_data = self.get_metadata(guid)
-        if "token" in guid_data:
-            del guid_data["token"]
-            self.set_metadata(guid, guid_data)
-
-    def get_token(self, guid):
-        return self.get_metadata(guid).get("token")
+        self.guid2token[guid] = token
 
     def new_message_for_guid(self, guid):
         self.to_notify_queue.put(guid)
@@ -67,7 +40,7 @@ class Notifier:
 
     def thread_run_one(self, requests_session):
         guid = self.to_notify_queue.get()
-        token = self.get_token(guid)
+        token = self.guid2token.get(guid)
         if token:
             response = requests_session.post(
                 "https://notifications.delta.chat/notify",
@@ -77,7 +50,7 @@ class Notifier:
             if response.status_code == 410:
                 # 410 Gone status code
                 # means the token is no longer valid.
-                self.del_token(guid)
+                del self.guid2token[guid]
 
 
 def handle_dovecot_protocol(rfile, wfile, notifier):
@@ -146,15 +119,12 @@ class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
 
 
 def main():
-    socket, username, config, metadata_dir = sys.argv[1:]
+    socket, username, config = sys.argv[1:]
     passwd_entry = pwd.getpwnam(username)
 
     # XXX config is not currently used
     config = read_config(config)
-    metadata_dir = pathlib.Path(metadata_dir)
-    if not metadata_dir.exists():
-        metadata_dir.mkdir()
-    notifier = Notifier(metadata_dir)
+    notifier = Notifier()
 
     class Handler(StreamRequestHandler):
         def handle(self):

chatmaild/src/chatmaild/tests/test_metadata.py

@@ -1,5 +1,4 @@
 import io
-import pytest
 
 from chatmaild.metadata import (
     handle_dovecot_request,
@@ -8,60 +7,40 @@ from chatmaild.metadata import (
 )
 
 
-@pytest.fixture
-def notifier(tmp_path):
-    metadata_dir = tmp_path.joinpath("metadata")
-    metadata_dir.mkdir()
-    return Notifier(metadata_dir)
-
-
-def test_notifier_persistence(tmp_path):
-    metadata_dir = tmp_path.joinpath("metadata")
-    metadata_dir.mkdir()
-    notifier1 = Notifier(metadata_dir)
-    notifier2 = Notifier(metadata_dir)
-    assert notifier1.get_token(guid="guid00") is None
-    assert notifier2.get_token(guid="guid00") is None
-
-    notifier1.set_token("guid00", "01234")
-    notifier1.set_token("guid03", "456")
-    assert notifier2.get_token("guid00") == "01234"
-    assert notifier2.get_token("guid03") == "456"
-    notifier2.del_token("guid00")
-    assert notifier1.get_token("guid00") is None
-
-
-def test_handle_dovecot_request_lookup_fails(notifier):
+def test_handle_dovecot_request_lookup_fails():
+    notifier = Notifier()
     res = handle_dovecot_request("Lpriv/123/chatmail", {}, notifier)
     assert res == "N\n"
 
 
-def test_handle_dovecot_request_happy_path(notifier):
+def test_handle_dovecot_request_happy_path():
+    notifier = Notifier()
     transactions = {}
 
     # lookups return the same NOTFOUND result
     res = handle_dovecot_request("Lpriv/123/chatmail", transactions, notifier)
     assert res == "N\n"
-    assert notifier.get_token("guid00") is None and not transactions
+    assert not notifier.guid2token and not transactions
 
     # set device token in a transaction
     tx = "1111"
     msg = f"B{tx}\tuser"
     res = handle_dovecot_request(msg, transactions, notifier)
-    assert not res and notifier.get_token("guid00") is None
+    assert not res and not notifier.guid2token
     assert transactions == {tx: "O\n"}
 
     msg = f"S{tx}\tpriv/guid00/devicetoken\t01234"
     res = handle_dovecot_request(msg, transactions, notifier)
     assert not res
     assert len(transactions) == 1
-    assert notifier.get_token("guid00") == "01234"
+    assert len(notifier.guid2token) == 1
+    assert notifier.guid2token["guid00"] == "01234"
 
     msg = f"C{tx}"
     res = handle_dovecot_request(msg, transactions, notifier)
     assert res == "O\n"
     assert len(transactions) == 0
-    assert notifier.get_token("guid00") == "01234"
+    assert notifier.guid2token["guid00"] == "01234"
 
     # trigger notification for incoming message
     assert handle_dovecot_request(f"B{tx}\tuser", transactions, notifier) is None
@@ -73,7 +52,7 @@ def test_handle_dovecot_request_happy_path(notifier):
     assert not transactions
 
 
-def test_handle_dovecot_protocol_set_devicetoken(notifier):
+def test_handle_dovecot_protocol_set_devicetoken():
     rfile = io.BytesIO(
         b"\n".join(
             [
@@ -85,12 +64,13 @@ def test_handle_dovecot_protocol_set_devicetoken(notifier):
         )
     )
     wfile = io.BytesIO()
+    notifier = Notifier()
     handle_dovecot_protocol(rfile, wfile, notifier)
-    assert notifier.get_token("guid00") == "01234"
+    assert notifier.guid2token["guid00"] == "01234"
     assert wfile.getvalue() == b"O\n"
 
 
-def test_handle_dovecot_protocol_iterate(notifier):
+def test_handle_dovecot_protocol_iterate():
     rfile = io.BytesIO(
         b"\n".join(
             [
@@ -100,11 +80,12 @@ def test_handle_dovecot_protocol_iterate(notifier):
         )
     )
     wfile = io.BytesIO()
+    notifier = Notifier()
     handle_dovecot_protocol(rfile, wfile, notifier)
     assert wfile.getvalue() == b"\n"
 
 
-def test_handle_dovecot_protocol_messagenew(notifier):
+def test_handle_dovecot_protocol_messagenew():
     rfile = io.BytesIO(
         b"\n".join(
             [
@@ -116,13 +97,14 @@ def test_handle_dovecot_protocol_messagenew(notifier):
         )
     )
     wfile = io.BytesIO()
+    notifier = Notifier()
     handle_dovecot_protocol(rfile, wfile, notifier)
     assert wfile.getvalue() == b"O\n"
     assert notifier.to_notify_queue.get() == "guid00"
     assert notifier.to_notify_queue.qsize() == 0
 
 
-def test_notifier_thread_run(notifier):
+def test_notifier_thread_run():
     requests = []
 
     class ReqMock:
@@ -134,15 +116,16 @@ def test_notifier_thread_run(notifier):
 
             return Result()
 
+    notifier = Notifier()
     notifier.set_token("guid00", "01234")
     notifier.new_message_for_guid("guid00")
     notifier.thread_run_one(ReqMock())
     url, data, timeout = requests[0]
     assert data == "01234"
-    assert notifier.get_token("guid00") == "01234"
+    assert len(notifier.guid2token) == 1
 
 
-def test_notifier_thread_run_gone_removes_token(notifier):
+def test_notifier_thread_run_gone_removes_token():
     requests = []
 
     class ReqMock:
@@ -154,10 +137,11 @@ def test_notifier_thread_run_gone_removes_token(notifier):
 
             return Result()
 
+    notifier = Notifier()
     notifier.set_token("guid00", "01234")
     notifier.new_message_for_guid("guid00")
-    assert notifier.get_token("guid00") == "01234"
+    assert notifier.guid2token["guid00"] == "01234"
     notifier.thread_run_one(ReqMock())
     url, data, timeout = requests[0]
     assert data == "01234"
-    assert notifier.get_token("guid00") is None
+    assert len(notifier.guid2token) == 0

cmdeploy/src/cmdeploy/chatmail.zone.f

@@ -6,7 +6,7 @@ _submissions._tcp.{chatmail_domain}. SRV 0 1 465 {chatmail_domain}.
 _imap._tcp.{chatmail_domain}.        SRV 0 1 143 {chatmail_domain}.
 _imaps._tcp.{chatmail_domain}.       SRV 0 1 993 {chatmail_domain}.
 {chatmail_domain}.                   CAA 128 issue "letsencrypt.org;accounturi={acme_account_url}"
-{chatmail_domain}.                   TXT "v=spf1 a:{chatmail_domain} ~all"
+{chatmail_domain}.                   TXT "v=spf1 a:{chatmail_domain} -all"
 _dmarc.{chatmail_domain}.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
 _mta-sts.{chatmail_domain}.          TXT "v=STSv1; id={sts_id}"
 mta-sts.{chatmail_domain}.           CNAME {chatmail_domain}.

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -13,12 +13,6 @@ auth_cache_size = 100M
 mail_debug = yes
 {% endif %}
 
-# Prevent warnings similar to:
-#   config: Warning: service auth { client_limit=1000 } is lower than required under max. load (10200). Counted for protocol services with service_count != 1: service lmtp { process_limit=100 } + service imap-urlauth-login { process_limit=100 } + service imap-login { process_limit=10000 }
-#   config: Warning: service anvil { client_limit=1000 } is lower than required under max. load (10103). Counted with: service imap-urlauth-login { process_limit=100 } + service imap-login { process_limit=10000 } + service auth { process_limit=1 }
-#   master: Warning: service(stats): client_limit (1000) reached, client connections are being dropped
-default_client_limit = 20000
-
 mail_server_admin = mailto:root@{{ config.mail_domain }}
 mail_server_comment = Chatmail server
 

cmdeploy/src/cmdeploy/dovecot/expunge.cron.j2

@@ -8,4 +8,3 @@
 # or only temporary (but then they shouldn't be around after {{ config.delete_mails_after }} days anyway).
 2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-3 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -name 'maildirsize' -type f -delete