Push notification extension

This change adds XDELTAPUSH capability.

Delta Chat clients detecting this capability
can set /private/devicetoken IMAP metadata
on the inbox to subscribe for Apple (APNS)
notifications.

Notifications are implemented in a new
`chatmail-metadata` service
which handles requests to set /private/devicetoken
IMAP metadata from Delta Chat clients
and /private/messagenew requests from
push_notification_lua script.

To avoid sending notifications for
MDNs, webxdc updates and Delta Chat sync messages,
messages with Auto-Submitted header are ignored
by setting $Auto keyword (flag) on them in Sieve script
and skipping such messages in push_notification_lua script.
Outgoing messages are also ignored.

chatmaild/pyproject.toml

@@ -10,6 +10,7 @@ dependencies = [
   "iniconfig",
   "deltachat-rpc-server",
   "deltachat-rpc-client",
+  "requests",
 ]
 
 [tool.setuptools]
@@ -20,6 +21,7 @@ where = ['src']
 
 [project.scripts]
 doveauth = "chatmaild.doveauth:main"
+chatmail-metadata = "chatmaild.metadata:main"
 filtermail = "chatmaild.filtermail:main"
 echobot = "chatmaild.echo:main"
 chatmail-metrics = "chatmaild.metrics:main"

chatmaild/src/chatmaild/chatmail-metadata.service.f

@@ -0,0 +1,10 @@
+[Unit]
+Description=Chatmail dict proxy for IMAP METADATA
+
+[Service]
+ExecStart={execpath} /run/dovecot/metadata.socket vmail {config_path}
+Restart=always
+RestartSec=30
+
+[Install]
+WantedBy=multi-user.target

chatmaild/src/chatmaild/doveauth.py

@@ -58,17 +58,18 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
     return True
 
 
-def get_user_data(db, user):
+def get_user_data(db, config: Config, user):
     with db.read_connection() as conn:
         result = conn.get_user(user)
     if result:
+        result["home"] = f"/home/vmail/mail/{config.mail_domain}/{user}"
         result["uid"] = "vmail"
         result["gid"] = "vmail"
     return result
 
 
-def lookup_userdb(db, user):
-    return get_user_data(db, user)
+def lookup_userdb(db, config: Config, user):
+    return get_user_data(db, config, user)
 
 
 def lookup_passdb(db, config: Config, user, cleartext_password):
@@ -80,6 +81,7 @@ def lookup_passdb(db, config: Config, user, cleartext_password):
                 "UPDATE users SET last_login=? WHERE addr=?", (int(time.time()), user)
             )
 
+            userdata["home"] = f"/home/vmail/mail/{config.mail_domain}/{user}"
             userdata["uid"] = "vmail"
             userdata["gid"] = "vmail"
             return userdata
@@ -142,7 +144,7 @@ def handle_dovecot_request(msg, db, config: Config):
             if type == "userdb":
                 user = args[0]
                 if user.endswith(f"@{config.mail_domain}"):
-                    res = lookup_userdb(db, user)
+                    res = lookup_userdb(db, config, user)
                 if res:
                     reply_command = "O"
                 else:

chatmaild/src/chatmaild/metadata.py

@@ -0,0 +1,106 @@
+import pwd
+from socketserver import (
+    UnixStreamServer,
+    StreamRequestHandler,
+    ThreadingMixIn,
+)
+from .config import read_config, Config
+import sys
+import logging
+import os
+import requests
+
+
+def handle_dovecot_protocol(rfile, wfile, tokens, requests_session, config: Config):
+    # HELLO message, ignored.
+    msg = rfile.readline().strip().decode()
+
+    transactions = {}
+
+    while True:
+        msg = rfile.readline().strip().decode()
+        if not msg:
+            break
+
+        short_command = msg[0]
+        if short_command == "L":
+            wfile.write(b"N\n")
+        elif short_command == "S":
+            # See header of
+            # <https://github.com/dovecot/core/blob/5e7965632395793d9355eb906b173bf28d2a10ca/src/lib-storage/mailbox-attribute.h>
+            # for the documentation on the structure of the key.
+
+            # Request GETMETADATA "INBOX" /private/chatmail
+            # results in a query for
+            # priv/dd72550f05eadc65542a1200cac67ad7/chatmail
+            #
+            # Request GETMETADATA "" /private/chatmail
+            # results in
+            # priv/dd72550f05eadc65542a1200cac67ad7/vendor/vendor.dovecot/pvt/server/chatmail
+
+            parts = msg[1:].split("\t")
+            transaction_id = parts[0]
+            keyname = parts[1].split("/")
+            value = parts[2] if len(parts) > 2 else ""
+            if keyname[0] == "priv" and keyname[2] == "devicetoken":
+                tokens[keyname[1]] = value
+            elif keyname[0] == "priv" and keyname[2] == "messagenew":
+                guid = keyname[1]
+                token = tokens.get(guid)
+                if token:
+                    response = requests_session.post(
+                        "https://notifications.delta.chat/notify",
+                        data=token,
+                        timeout=60,
+                    )
+                    if response.status_code == 410:
+                        # 410 Gone status code
+                        # means the token is no longer valid.
+                        del tokens[guid]
+            else:
+                # Transaction failed.
+                transactions[transaction_id] = b"F\n"
+        elif short_command == "B":
+            # Begin transaction.
+            transaction_id = msg[1:].split("\t")[0]
+            transactions[transaction_id] = b"O\n"
+        elif short_command == "C":
+            # Commit transaction.
+            transaction_id = msg[1:].split("\t")[0]
+            wfile.write(transactions.pop(transaction_id, b"N\n"))
+
+        wfile.flush()
+
+
+class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
+    request_queue_size = 100
+
+
+def main():
+    socket, username, config = sys.argv[1:]
+    passwd_entry = pwd.getpwnam(username)
+    config = read_config(config)
+    tokens = {}
+    requests_session = requests.Session()
+
+    class Handler(StreamRequestHandler):
+        def handle(self):
+            try:
+                handle_dovecot_protocol(
+                    self.rfile, self.wfile, tokens, requests_session, config
+                )
+            except Exception:
+                logging.exception("Exception in the handler")
+                raise
+
+    try:
+        os.unlink(socket)
+    except FileNotFoundError:
+        pass
+
+    with ThreadedUnixStreamServer(socket, Handler) as server:
+        os.chown(socket, uid=passwd_entry.pw_uid, gid=passwd_entry.pw_gid)
+        try:
+            server.serve_forever()
+        except KeyboardInterrupt:
+            pass

chatmaild/src/chatmaild/tests/test_doveauth.py

@@ -17,7 +17,7 @@ from chatmaild.database import DBError
 
 def test_basic(db, example_config):
     lookup_passdb(db, example_config, "asdf12345@chat.example.org", "q9mr3faue")
-    data = get_user_data(db, "asdf12345@chat.example.org")
+    data = get_user_data(db, example_config, "asdf12345@chat.example.org")
     assert data
     data2 = lookup_passdb(
         db, example_config, "asdf12345@chat.example.org", "q9mr3jewvadsfaue"
@@ -43,7 +43,7 @@ def test_nocreate_file(db, monkeypatch, tmpdir, example_config):
     lookup_passdb(
         db, example_config, "newuser12@chat.example.org", "zequ0Aimuchoodaechik"
     )
-    assert not get_user_data(db, "newuser12@chat.example.org")
+    assert not get_user_data(db, example_config, "newuser12@chat.example.org")
 
 
 def test_db_version(db):

cmdeploy/src/cmdeploy/__init__.py

@@ -102,6 +102,7 @@ def _install_remote_venv_with_chatmaild(config) -> None:
         "doveauth",
         "filtermail",
         "echobot",
+        "chatmail-metadata",
     ):
         params = dict(
             execpath=f"{remote_venv_dir}/bin/{fn}",
@@ -337,6 +338,27 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
         mode="644",
     )
     need_restart |= auth_config.changed
+    lua_push_notification_script = files.put(
+        src=importlib.resources.files(__package__).joinpath(
+            "dovecot/push_notification.lua"
+        ),
+        dest="/etc/dovecot/push_notification.lua",
+        user="root",
+        group="root",
+        mode="644",
+    )
+    need_restart |= lua_push_notification_script.changed
+
+    sieve_script = files.put(
+        src=importlib.resources.files(__package__).joinpath(
+            "dovecot/default.sieve"
+        ),
+        dest="/etc/dovecot/default.sieve",
+        user="root",
+        group="root",
+        mode="644",
+    )
+    need_restart |= sieve_script.changed
 
     files.template(
         src=importlib.resources.files(__package__).joinpath("dovecot/expunge.cron.j2"),
@@ -484,7 +506,7 @@ def deploy_chatmail(config_path: Path) -> None:
 
     apt.packages(
         name="Install Dovecot",
-        packages=["dovecot-imapd", "dovecot-lmtpd"],
+        packages=["dovecot-imapd", "dovecot-lmtpd", "dovecot-sieve"],
     )
 
     apt.packages(

cmdeploy/src/cmdeploy/dovecot/default.sieve

@@ -0,0 +1,5 @@
+require ["imap4flags"];
+
+if header :is ["Auto-Submitted"] ["auto-replied", "auto-generated"] {
+	addflag "$Auto";
+}

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -21,7 +21,7 @@ mail_plugins = quota
 # these are the capabilities Delta Chat cares about actually 
 # so let's keep the network overhead per login small
 # https://github.com/deltachat/deltachat-core-rust/blob/master/src/imap/capabilities.rs
-imap_capability = IMAP4rev1 IDLE MOVE QUOTA CONDSTORE NOTIFY METADATA
+imap_capability = IMAP4rev1 IDLE MOVE QUOTA CONDSTORE NOTIFY METADATA XDELTAPUSH
 
 
 # Authentication for system users.
@@ -71,6 +71,9 @@ mail_privileged_group = vmail
 ## Mail processes
 ##
 
+# Pass all IMAP METADATA requests to the server implementing Dovecot's dict protocol.
+mail_attribute_dict = proxy:/run/dovecot/metadata.socket:metadata
+
 # Enable IMAP COMPRESS (RFC 4978).
 # <https://datatracker.ietf.org/doc/html/rfc4978.html>
 protocol imap {
@@ -79,7 +82,21 @@ protocol imap {
 }
 
 protocol lmtp {
-  mail_plugins = $mail_plugins quota
+  # quota plugin documentation:
+  # <https://doc.dovecot.org/configuration_manual/quota_plugin/>
+  #
+  # notify plugin is a dependency of push_notification plugin:
+  # <https://doc.dovecot.org/settings/plugin/notify-plugin/>
+  #
+  # push_notification plugin documentation:
+  # <https://doc.dovecot.org/configuration_manual/push_notification/>
+  #
+  # mail_lua and push_notification_lua are needed for Lua push notification handler.
+  # <https://doc.dovecot.org/configuration_manual/push_notification/#configuration>
+  #
+  # Sieve to mark messages that should not be notified as \Seen
+  # <https://doc.dovecot.org/configuration_manual/sieve/configuration/>
+  mail_plugins = $mail_plugins quota mail_lua notify push_notification push_notification_lua sieve
 }
 
 plugin {
@@ -95,7 +112,15 @@ plugin {
   # quota_over_flag_value = TRUE
 }
 
+# push_notification configuration
+plugin {
+  # <https://doc.dovecot.org/configuration_manual/push_notification/#lua-lua>
+  push_notification_driver = lua:file=/etc/dovecot/push_notification.lua
+}
 
+plugin {
+  sieve_default = file:/etc/dovecot/default.sieve
+}
 
 service lmtp {
   user=vmail

cmdeploy/src/cmdeploy/dovecot/push_notification.lua

@@ -0,0 +1,32 @@
+function dovecot_lua_notify_begin_txn(user)
+  return user
+end
+
+function contains(v, needle)
+  for _, keyword in ipairs(v) do
+    if keyword == needle then
+      return true
+    end
+  end
+  return false
+end
+
+function dovecot_lua_notify_event_message_new(user, event)
+  local mbox = user:mailbox(event.mailbox)
+  mbox:sync()
+
+  if user.username ~= event.from_address then
+    -- Incoming message
+    if not contains(event.keywords, "$Auto") then
+      -- Not an Auto-Submitted message, notifying.
+
+      -- Notify METADATA server about new message.
+      mbox:metadata_set("/private/messagenew", "")
+    end
+  end
+
+  mbox:free()
+end
+
+function dovecot_lua_notify_end_txn(ctx, success)
+end