dovecot: increase default_client_limit

Otherwise Dovecot times out when trying to iterate over metadata
of the folder. Apparently it happens when attempting to delete
folder from the server over IMAP.

chatmaild/pyproject.toml

@@ -10,6 +10,7 @@ dependencies = [
   "iniconfig",
   "deltachat-rpc-server",
   "deltachat-rpc-client",
+  "requests",
 ]
 
 [tool.setuptools]
@@ -20,6 +21,7 @@ where = ['src']
 
 [project.scripts]
 doveauth = "chatmaild.doveauth:main"
+chatmail-metadata = "chatmaild.metadata:main"
 filtermail = "chatmaild.filtermail:main"
 echobot = "chatmaild.echo:main"
 chatmail-metrics = "chatmaild.metrics:main"

chatmaild/src/chatmaild/chatmail-metadata.service.f

@@ -0,0 +1,10 @@
+[Unit]
+Description=Chatmail dict proxy for IMAP METADATA
+
+[Service]
+ExecStart={execpath} /run/dovecot/metadata.socket vmail {config_path}
+Restart=always
+RestartSec=30
+
+[Install]
+WantedBy=multi-user.target

chatmaild/src/chatmaild/doveauth.py

@@ -58,17 +58,18 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
     return True
 
 
-def get_user_data(db, user):
+def get_user_data(db, config: Config, user):
     with db.read_connection() as conn:
         result = conn.get_user(user)
     if result:
+        result["home"] = f"/home/vmail/mail/{config.mail_domain}/{user}"
         result["uid"] = "vmail"
         result["gid"] = "vmail"
     return result
 
 
-def lookup_userdb(db, user):
-    return get_user_data(db, user)
+def lookup_userdb(db, config: Config, user):
+    return get_user_data(db, config, user)
 
 
 def lookup_passdb(db, config: Config, user, cleartext_password):
@@ -80,6 +81,7 @@ def lookup_passdb(db, config: Config, user, cleartext_password):
                 "UPDATE users SET last_login=? WHERE addr=?", (int(time.time()), user)
             )
 
+            userdata["home"] = f"/home/vmail/mail/{config.mail_domain}/{user}"
             userdata["uid"] = "vmail"
             userdata["gid"] = "vmail"
             return userdata
@@ -142,7 +144,7 @@ def handle_dovecot_request(msg, db, config: Config):
             if type == "userdb":
                 user = args[0]
                 if user.endswith(f"@{config.mail_domain}"):
-                    res = lookup_userdb(db, user)
+                    res = lookup_userdb(db, config, user)
                 if res:
                     reply_command = "O"
                 else:

chatmaild/src/chatmaild/metadata.py

@@ -0,0 +1,154 @@
+import pwd
+
+from queue import Queue
+from threading import Thread
+from socketserver import (
+    UnixStreamServer,
+    StreamRequestHandler,
+    ThreadingMixIn,
+)
+from .config import read_config
+import sys
+import logging
+import os
+import requests
+
+
+DICTPROXY_LOOKUP_CHAR = "L"
+DICTPROXY_ITERATE_CHAR = "I"
+DICTPROXY_SET_CHAR = "S"
+DICTPROXY_BEGIN_TRANSACTION_CHAR = "B"
+DICTPROXY_COMMIT_TRANSACTION_CHAR = "C"
+DICTPROXY_TRANSACTION_CHARS = "SBC"
+
+
+class Notifier:
+    def __init__(self):
+        self.guid2token = {}
+        self.to_notify_queue = Queue()
+
+    def set_token(self, guid, token):
+        self.guid2token[guid] = token
+
+    def new_message_for_guid(self, guid):
+        self.to_notify_queue.put(guid)
+
+    def thread_run_loop(self):
+        requests_session = requests.Session()
+        while 1:
+            self.thread_run_one(requests_session)
+
+    def thread_run_one(self, requests_session):
+        guid = self.to_notify_queue.get()
+        token = self.guid2token.get(guid)
+        if token:
+            response = requests_session.post(
+                "https://notifications.delta.chat/notify",
+                data=token,
+                timeout=60,
+            )
+            if response.status_code == 410:
+                # 410 Gone status code
+                # means the token is no longer valid.
+                del self.guid2token[guid]
+
+
+def handle_dovecot_protocol(rfile, wfile, notifier):
+    # HELLO message, ignored.
+    msg = rfile.readline().strip().decode()
+
+    transactions = {}
+    while True:
+        msg = rfile.readline().strip().decode()
+        if not msg:
+            break
+
+        res = handle_dovecot_request(msg, transactions, notifier)
+        if res:
+            wfile.write(res.encode("ascii"))
+            wfile.flush()
+
+
+def handle_dovecot_request(msg, transactions, notifier):
+    # see https://doc.dovecot.org/3.0/developer_manual/design/dict_protocol/
+    short_command = msg[0]
+    parts = msg[1:].split("\t")
+    if short_command == DICTPROXY_LOOKUP_CHAR:
+        return "N\n"
+    elif short_command == DICTPROXY_ITERATE_CHAR:
+        # Empty line means ITER_FINISHED.
+        # If we don't return empty line Dovecot will timeout.
+        return "\n"
+
+    if short_command not in (DICTPROXY_TRANSACTION_CHARS):
+        return
+
+    transaction_id = parts[0]
+
+    if short_command == DICTPROXY_BEGIN_TRANSACTION_CHAR:
+        transactions[transaction_id] = "O\n"
+    elif short_command == DICTPROXY_COMMIT_TRANSACTION_CHAR:
+        # returns whether it failed or succeeded.
+        return transactions.pop(transaction_id, "N\n")
+    elif short_command == DICTPROXY_SET_CHAR:
+        # See header of
+        # <https://github.com/dovecot/core/blob/5e7965632395793d9355eb906b173bf28d2a10ca/src/lib-storage/mailbox-attribute.h>
+        # for the documentation on the structure of the key.
+
+        # Request GETMETADATA "INBOX" /private/chatmail
+        # results in a query for
+        # priv/dd72550f05eadc65542a1200cac67ad7/chatmail
+        #
+        # Request GETMETADATA "" /private/chatmail
+        # results in
+        # priv/dd72550f05eadc65542a1200cac67ad7/vendor/vendor.dovecot/pvt/server/chatmail
+
+        keyname = parts[1].split("/")
+        value = parts[2] if len(parts) > 2 else ""
+        if keyname[0] == "priv" and keyname[2] == "devicetoken":
+            notifier.set_token(keyname[1], value)
+        elif keyname[0] == "priv" and keyname[2] == "messagenew":
+            notifier.new_message_for_guid(keyname[1])
+        else:
+            # Transaction failed.
+            transactions[transaction_id] = "F\n"
+
+
+class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
+    request_queue_size = 100
+
+
+def main():
+    socket, username, config = sys.argv[1:]
+    passwd_entry = pwd.getpwnam(username)
+
+    # XXX config is not currently used
+    config = read_config(config)
+    notifier = Notifier()
+
+    class Handler(StreamRequestHandler):
+        def handle(self):
+            try:
+                handle_dovecot_protocol(self.rfile, self.wfile, notifier)
+            except Exception:
+                logging.exception("Exception in the handler")
+                raise
+
+    try:
+        os.unlink(socket)
+    except FileNotFoundError:
+        pass
+
+    # start notifier thread for signalling new messages to
+    # Delta Chat notification server
+
+    t = Thread(target=notifier.thread_run_loop)
+    t.setDaemon(True)
+    t.start()
+
+    with ThreadedUnixStreamServer(socket, Handler) as server:
+        os.chown(socket, uid=passwd_entry.pw_uid, gid=passwd_entry.pw_gid)
+        try:
+            server.serve_forever()
+        except KeyboardInterrupt:
+            pass

chatmaild/src/chatmaild/tests/test_doveauth.py

@@ -17,7 +17,7 @@ from chatmaild.database import DBError
 
 def test_basic(db, example_config):
     lookup_passdb(db, example_config, "asdf12345@chat.example.org", "q9mr3faue")
-    data = get_user_data(db, "asdf12345@chat.example.org")
+    data = get_user_data(db, example_config, "asdf12345@chat.example.org")
     assert data
     data2 = lookup_passdb(
         db, example_config, "asdf12345@chat.example.org", "q9mr3jewvadsfaue"
@@ -43,7 +43,7 @@ def test_nocreate_file(db, monkeypatch, tmpdir, example_config):
     lookup_passdb(
         db, example_config, "newuser12@chat.example.org", "zequ0Aimuchoodaechik"
     )
-    assert not get_user_data(db, "newuser12@chat.example.org")
+    assert not get_user_data(db, example_config, "newuser12@chat.example.org")
 
 
 def test_db_version(db):

chatmaild/src/chatmaild/tests/test_metadata.py

@@ -0,0 +1,147 @@
+import io
+
+from chatmaild.metadata import (
+    handle_dovecot_request,
+    handle_dovecot_protocol,
+    Notifier,
+)
+
+
+def test_handle_dovecot_request_lookup_fails():
+    notifier = Notifier()
+    res = handle_dovecot_request("Lpriv/123/chatmail", {}, notifier)
+    assert res == "N\n"
+
+
+def test_handle_dovecot_request_happy_path():
+    notifier = Notifier()
+    transactions = {}
+
+    # lookups return the same NOTFOUND result
+    res = handle_dovecot_request("Lpriv/123/chatmail", transactions, notifier)
+    assert res == "N\n"
+    assert not notifier.guid2token and not transactions
+
+    # set device token in a transaction
+    tx = "1111"
+    msg = f"B{tx}\tuser"
+    res = handle_dovecot_request(msg, transactions, notifier)
+    assert not res and not notifier.guid2token
+    assert transactions == {tx: "O\n"}
+
+    msg = f"S{tx}\tpriv/guid00/devicetoken\t01234"
+    res = handle_dovecot_request(msg, transactions, notifier)
+    assert not res
+    assert len(transactions) == 1
+    assert len(notifier.guid2token) == 1
+    assert notifier.guid2token["guid00"] == "01234"
+
+    msg = f"C{tx}"
+    res = handle_dovecot_request(msg, transactions, notifier)
+    assert res == "O\n"
+    assert len(transactions) == 0
+    assert notifier.guid2token["guid00"] == "01234"
+
+    # trigger notification for incoming message
+    assert handle_dovecot_request(f"B{tx}\tuser", transactions, notifier) is None
+    msg = f"S{tx}\tpriv/guid00/messagenew"
+    assert handle_dovecot_request(msg, transactions, notifier) is None
+    assert notifier.to_notify_queue.get() == "guid00"
+    assert notifier.to_notify_queue.qsize() == 0
+    assert handle_dovecot_request(f"C{tx}\tuser", transactions, notifier) == "O\n"
+    assert not transactions
+
+
+def test_handle_dovecot_protocol_set_devicetoken():
+    rfile = io.BytesIO(
+        b"\n".join(
+            [
+                b"HELLO",
+                b"Btx00\tuser",
+                b"Stx00\tpriv/guid00/devicetoken\t01234",
+                b"Ctx00",
+            ]
+        )
+    )
+    wfile = io.BytesIO()
+    notifier = Notifier()
+    handle_dovecot_protocol(rfile, wfile, notifier)
+    assert notifier.guid2token["guid00"] == "01234"
+    assert wfile.getvalue() == b"O\n"
+
+
+def test_handle_dovecot_protocol_iterate():
+    rfile = io.BytesIO(
+        b"\n".join(
+            [
+                b"H",
+                b"I9\t0\tpriv/5cbe730f146fea6535be0d003dd4fc98/\tci-2dzsrs@nine.testrun.org",
+            ]
+        )
+    )
+    wfile = io.BytesIO()
+    notifier = Notifier()
+    handle_dovecot_protocol(rfile, wfile, notifier)
+    assert wfile.getvalue() == b"\n"
+
+
+def test_handle_dovecot_protocol_messagenew():
+    rfile = io.BytesIO(
+        b"\n".join(
+            [
+                b"HELLO",
+                b"Btx01\tuser",
+                b"Stx01\tpriv/guid00/messagenew",
+                b"Ctx01",
+            ]
+        )
+    )
+    wfile = io.BytesIO()
+    notifier = Notifier()
+    handle_dovecot_protocol(rfile, wfile, notifier)
+    assert wfile.getvalue() == b"O\n"
+    assert notifier.to_notify_queue.get() == "guid00"
+    assert notifier.to_notify_queue.qsize() == 0
+
+
+def test_notifier_thread_run():
+    requests = []
+
+    class ReqMock:
+        def post(self, url, data, timeout):
+            requests.append((url, data, timeout))
+
+            class Result:
+                status_code = 200
+
+            return Result()
+
+    notifier = Notifier()
+    notifier.set_token("guid00", "01234")
+    notifier.new_message_for_guid("guid00")
+    notifier.thread_run_one(ReqMock())
+    url, data, timeout = requests[0]
+    assert data == "01234"
+    assert len(notifier.guid2token) == 1
+
+
+def test_notifier_thread_run_gone_removes_token():
+    requests = []
+
+    class ReqMock:
+        def post(self, url, data, timeout):
+            requests.append((url, data, timeout))
+
+            class Result:
+                status_code = 410
+
+            return Result()
+
+    notifier = Notifier()
+    notifier.set_token("guid00", "01234")
+    notifier.new_message_for_guid("guid00")
+    assert notifier.guid2token["guid00"] == "01234"
+    notifier.thread_run_one(ReqMock())
+    url, data, timeout = requests[0]
+    assert data == "01234"
+    assert len(notifier.guid2token) == 0

cmdeploy/src/cmdeploy/__init__.py

@@ -102,6 +102,7 @@ def _install_remote_venv_with_chatmaild(config) -> None:
         "doveauth",
         "filtermail",
         "echobot",
+        "chatmail-metadata",
     ):
         params = dict(
             execpath=f"{remote_venv_dir}/bin/{fn}",
@@ -337,6 +338,27 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
         mode="644",
     )
     need_restart |= auth_config.changed
+    lua_push_notification_script = files.put(
+        src=importlib.resources.files(__package__).joinpath(
+            "dovecot/push_notification.lua"
+        ),
+        dest="/etc/dovecot/push_notification.lua",
+        user="root",
+        group="root",
+        mode="644",
+    )
+    need_restart |= lua_push_notification_script.changed
+
+    sieve_script = files.put(
+        src=importlib.resources.files(__package__).joinpath(
+            "dovecot/default.sieve"
+        ),
+        dest="/etc/dovecot/default.sieve",
+        user="root",
+        group="root",
+        mode="644",
+    )
+    need_restart |= sieve_script.changed
 
     files.template(
         src=importlib.resources.files(__package__).joinpath("dovecot/expunge.cron.j2"),
@@ -426,8 +448,9 @@ def check_config(config):
     mail_domain = config.mail_domain
     if mail_domain != "testrun.org" and not mail_domain.endswith(".testrun.org"):
         blocked_words = "merlinux schmieder testrun.org".split()
-        for value in config.__dict__.values():
-            if any(x in str(value) for x in blocked_words):
+        for key in config.__dict__:
+            value = config.__dict__[key]
+            if key.startswith("privacy") and any(x in str(value) for x in blocked_words):
                 raise ValueError(
                     f"please set your own privacy contacts/addresses in {config._inipath}"
                 )
@@ -448,6 +471,10 @@ def deploy_chatmail(config_path: Path) -> None:
     apt.update(name="apt update", cache_time=24 * 3600)
     server.group(name="Create vmail group", group="vmail", system=True)
     server.user(name="Create vmail user", user="vmail", group="vmail", system=True)
+    apt.packages(
+        name="Install rsync",
+        packages=["rsync"],
+    )
 
     # Run local DNS resolver `unbound`.
     # `resolvconf` takes care of setting up /etc/resolv.conf
@@ -483,7 +510,7 @@ def deploy_chatmail(config_path: Path) -> None:
 
     apt.packages(
         name="Install Dovecot",
-        packages=["dovecot-imapd", "dovecot-lmtpd"],
+        packages=["dovecot-imapd", "dovecot-lmtpd", "dovecot-sieve"],
     )
 
     apt.packages(

cmdeploy/src/cmdeploy/dovecot/default.sieve

@@ -0,0 +1,5 @@
+require ["imap4flags"];
+
+if header :is ["Auto-Submitted"] ["auto-replied", "auto-generated"] {
+	addflag "$Auto";
+}

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -13,6 +13,12 @@ auth_cache_size = 100M
 mail_debug = yes
 {% endif %}
 
+# Prevent warnings similar to:
+#   config: Warning: service auth { client_limit=1000 } is lower than required under max. load (10200). Counted for protocol services with service_count != 1: service lmtp { process_limit=100 } + service imap-urlauth-login { process_limit=100 } + service imap-login { process_limit=10000 }
+#   config: Warning: service anvil { client_limit=1000 } is lower than required under max. load (10103). Counted with: service imap-urlauth-login { process_limit=100 } + service imap-login { process_limit=10000 } + service auth { process_limit=1 }
+#   master: Warning: service(stats): client_limit (1000) reached, client connections are being dropped
+default_client_limit = 20000
+
 mail_server_admin = mailto:root@{{ config.mail_domain }}
 mail_server_comment = Chatmail server
 
@@ -21,7 +27,7 @@ mail_plugins = quota
 # these are the capabilities Delta Chat cares about actually 
 # so let's keep the network overhead per login small
 # https://github.com/deltachat/deltachat-core-rust/blob/master/src/imap/capabilities.rs
-imap_capability = IMAP4rev1 IDLE MOVE QUOTA CONDSTORE NOTIFY METADATA
+imap_capability = IMAP4rev1 IDLE MOVE QUOTA CONDSTORE NOTIFY METADATA XDELTAPUSH
 
 
 # Authentication for system users.
@@ -71,6 +77,9 @@ mail_privileged_group = vmail
 ## Mail processes
 ##
 
+# Pass all IMAP METADATA requests to the server implementing Dovecot's dict protocol.
+mail_attribute_dict = proxy:/run/dovecot/metadata.socket:metadata
+
 # Enable IMAP COMPRESS (RFC 4978).
 # <https://datatracker.ietf.org/doc/html/rfc4978.html>
 protocol imap {
@@ -79,7 +88,21 @@ protocol imap {
 }
 
 protocol lmtp {
-  mail_plugins = $mail_plugins quota
+  # quota plugin documentation:
+  # <https://doc.dovecot.org/configuration_manual/quota_plugin/>
+  #
+  # notify plugin is a dependency of push_notification plugin:
+  # <https://doc.dovecot.org/settings/plugin/notify-plugin/>
+  #
+  # push_notification plugin documentation:
+  # <https://doc.dovecot.org/configuration_manual/push_notification/>
+  #
+  # mail_lua and push_notification_lua are needed for Lua push notification handler.
+  # <https://doc.dovecot.org/configuration_manual/push_notification/#configuration>
+  #
+  # Sieve to mark messages that should not be notified as \Seen
+  # <https://doc.dovecot.org/configuration_manual/sieve/configuration/>
+  mail_plugins = $mail_plugins quota mail_lua notify push_notification push_notification_lua sieve
 }
 
 plugin {
@@ -95,7 +118,15 @@ plugin {
   # quota_over_flag_value = TRUE
 }
 
+# push_notification configuration
+plugin {
+  # <https://doc.dovecot.org/configuration_manual/push_notification/#lua-lua>
+  push_notification_driver = lua:file=/etc/dovecot/push_notification.lua
+}
 
+plugin {
+  sieve_default = file:/etc/dovecot/default.sieve
+}
 
 service lmtp {
   user=vmail

cmdeploy/src/cmdeploy/dovecot/push_notification.lua

@@ -0,0 +1,32 @@
+function dovecot_lua_notify_begin_txn(user)
+  return user
+end
+
+function contains(v, needle)
+  for _, keyword in ipairs(v) do
+    if keyword == needle then
+      return true
+    end
+  end
+  return false
+end
+
+function dovecot_lua_notify_event_message_new(user, event)
+  local mbox = user:mailbox(event.mailbox)
+  mbox:sync()
+
+  if user.username ~= event.from_address then
+    -- Incoming message
+    if not contains(event.keywords, "$Auto") then
+      -- Not an Auto-Submitted message, notifying.
+
+      -- Notify METADATA server about new message.
+      mbox:metadata_set("/private/messagenew", "")
+    end
+  end
+
+  mbox:free()
+end
+
+function dovecot_lua_notify_end_txn(ctx, success)
+end