Push notification extension

This change adds XDELTAPUSH capability.

Delta Chat clients detecting this capability
can set /private/devicetoken IMAP metadata
on the inbox to subscribe for Apple (APNS)
notifications.

Notifications are implemented in a new
`chatmail-metadata` service
which handles requests to set /private/devicetoken
IMAP metadata from Delta Chat clients
and /private/messagenew requests from
push_notification_lua script.

To avoid sending notifications for
MDNs, webxdc updates and Delta Chat sync messages,
messages with Auto-Submitted header are ignored
by setting $Auto keyword (flag) on them in Sieve script
and skipping such messages in push_notification_lua script.
Outgoing messages are also ignored.

chatmaild/pyproject.toml

@@ -10,6 +10,7 @@ dependencies = [
   "iniconfig",
   "deltachat-rpc-server",
   "deltachat-rpc-client",
+  "requests",
 ]
 
 [tool.setuptools]
@@ -20,6 +21,7 @@ where = ['src']
 
 [project.scripts]
 doveauth = "chatmaild.doveauth:main"
+chatmail-metadata = "chatmaild.metadata:main"
 filtermail = "chatmaild.filtermail:main"
 echobot = "chatmaild.echo:main"
 chatmail-metrics = "chatmaild.metrics:main"

chatmaild/src/chatmaild/chatmail-metadata.service.f

@@ -0,0 +1,10 @@
+[Unit]
+Description=Chatmail dict proxy for IMAP METADATA
+
+[Service]
+ExecStart={execpath} /run/dovecot/metadata.socket vmail {config_path}
+Restart=always
+RestartSec=30
+
+[Install]
+WantedBy=multi-user.target

chatmaild/src/chatmaild/doveauth.py

@@ -58,17 +58,18 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
     return True
 
 
-def get_user_data(db, user):
+def get_user_data(db, config: Config, user):
     with db.read_connection() as conn:
         result = conn.get_user(user)
     if result:
+        result["home"] = f"/home/vmail/mail/{config.mail_domain}/{user}"
         result["uid"] = "vmail"
         result["gid"] = "vmail"
     return result
 
 
-def lookup_userdb(db, user):
-    return get_user_data(db, user)
+def lookup_userdb(db, config: Config, user):
+    return get_user_data(db, config, user)
 
 
 def lookup_passdb(db, config: Config, user, cleartext_password):
@@ -80,6 +81,7 @@ def lookup_passdb(db, config: Config, user, cleartext_password):
                 "UPDATE users SET last_login=? WHERE addr=?", (int(time.time()), user)
             )
 
+            userdata["home"] = f"/home/vmail/mail/{config.mail_domain}/{user}"
             userdata["uid"] = "vmail"
             userdata["gid"] = "vmail"
             return userdata
@@ -142,7 +144,7 @@ def handle_dovecot_request(msg, db, config: Config):
             if type == "userdb":
                 user = args[0]
                 if user.endswith(f"@{config.mail_domain}"):
-                    res = lookup_userdb(db, user)
+                    res = lookup_userdb(db, config, user)
                 if res:
                     reply_command = "O"
                 else:

chatmaild/src/chatmaild/metadata.py

@@ -0,0 +1,106 @@
+import pwd
+from socketserver import (
+    UnixStreamServer,
+    StreamRequestHandler,
+    ThreadingMixIn,
+)
+from .config import read_config, Config
+import sys
+import logging
+import os
+import requests
+
+
+def handle_dovecot_protocol(rfile, wfile, tokens, requests_session, config: Config):
+    # HELLO message, ignored.
+    msg = rfile.readline().strip().decode()
+
+    transactions = {}
+
+    while True:
+        msg = rfile.readline().strip().decode()
+        if not msg:
+            break
+
+        short_command = msg[0]
+        if short_command == "L":
+            wfile.write(b"N\n")
+        elif short_command == "S":
+            # See header of
+            # <https://github.com/dovecot/core/blob/5e7965632395793d9355eb906b173bf28d2a10ca/src/lib-storage/mailbox-attribute.h>
+            # for the documentation on the structure of the key.
+
+            # Request GETMETADATA "INBOX" /private/chatmail
+            # results in a query for
+            # priv/dd72550f05eadc65542a1200cac67ad7/chatmail
+            #
+            # Request GETMETADATA "" /private/chatmail
+            # results in
+            # priv/dd72550f05eadc65542a1200cac67ad7/vendor/vendor.dovecot/pvt/server/chatmail
+
+            parts = msg[1:].split("\t")
+            transaction_id = parts[0]
+            keyname = parts[1].split("/")
+            value = parts[2] if len(parts) > 2 else ""
+            if keyname[0] == "priv" and keyname[2] == "devicetoken":
+                tokens[keyname[1]] = value
+            elif keyname[0] == "priv" and keyname[2] == "messagenew":
+                guid = keyname[1]
+                token = tokens.get(guid)
+                if token:
+                    response = requests_session.post(
+                        "https://notifications.delta.chat/notify",
+                        data=token,
+                        timeout=60,
+                    )
+                    if response.status_code == 410:
+                        # 410 Gone status code
+                        # means the token is no longer valid.
+                        del tokens[guid]
+            else:
+                # Transaction failed.
+                transactions[transaction_id] = b"F\n"
+        elif short_command == "B":
+            # Begin transaction.
+            transaction_id = msg[1:].split("\t")[0]
+            transactions[transaction_id] = b"O\n"
+        elif short_command == "C":
+            # Commit transaction.
+            transaction_id = msg[1:].split("\t")[0]
+            wfile.write(transactions.pop(transaction_id, b"N\n"))
+
+        wfile.flush()
+
+
+class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
+    request_queue_size = 100
+
+
+def main():
+    socket, username, config = sys.argv[1:]
+    passwd_entry = pwd.getpwnam(username)
+    config = read_config(config)
+    tokens = {}
+    requests_session = requests.Session()
+
+    class Handler(StreamRequestHandler):
+        def handle(self):
+            try:
+                handle_dovecot_protocol(
+                    self.rfile, self.wfile, tokens, requests_session, config
+                )
+            except Exception:
+                logging.exception("Exception in the handler")
+                raise
+
+    try:
+        os.unlink(socket)
+    except FileNotFoundError:
+        pass
+
+    with ThreadedUnixStreamServer(socket, Handler) as server:
+        os.chown(socket, uid=passwd_entry.pw_uid, gid=passwd_entry.pw_gid)
+        try:
+            server.serve_forever()
+        except KeyboardInterrupt:
+            pass

chatmaild/src/chatmaild/tests/test_doveauth.py

@@ -17,7 +17,7 @@ from chatmaild.database import DBError
 
 def test_basic(db, example_config):
     lookup_passdb(db, example_config, "asdf12345@chat.example.org", "q9mr3faue")
-    data = get_user_data(db, "asdf12345@chat.example.org")
+    data = get_user_data(db, example_config, "asdf12345@chat.example.org")
     assert data
     data2 = lookup_passdb(
         db, example_config, "asdf12345@chat.example.org", "q9mr3jewvadsfaue"
@@ -43,7 +43,7 @@ def test_nocreate_file(db, monkeypatch, tmpdir, example_config):
     lookup_passdb(
         db, example_config, "newuser12@chat.example.org", "zequ0Aimuchoodaechik"
     )
-    assert not get_user_data(db, "newuser12@chat.example.org")
+    assert not get_user_data(db, example_config, "newuser12@chat.example.org")
 
 
 def test_db_version(db):