diff --git a/chatmaild/pyproject.toml b/chatmaild/pyproject.toml index 90c1fe9a..b31cd934 100644 --- a/chatmaild/pyproject.toml +++ b/chatmaild/pyproject.toml @@ -10,6 +10,7 @@ dependencies = [ "iniconfig", "deltachat-rpc-server", "deltachat-rpc-client", + "requests", ] [tool.setuptools] @@ -20,6 +21,7 @@ where = ['src'] [project.scripts] doveauth = "chatmaild.doveauth:main" +chatmail-metadata = "chatmaild.metadata:main" filtermail = "chatmaild.filtermail:main" echobot = "chatmaild.echo:main" chatmail-metrics = "chatmaild.metrics:main" diff --git a/chatmaild/src/chatmaild/chatmail-metadata.service.f b/chatmaild/src/chatmaild/chatmail-metadata.service.f new file mode 100644 index 00000000..71f3e110 --- /dev/null +++ b/chatmaild/src/chatmaild/chatmail-metadata.service.f @@ -0,0 +1,10 @@ +[Unit] +Description=Chatmail dict proxy for IMAP METADATA + +[Service] +ExecStart={execpath} /run/dovecot/metadata.socket vmail {config_path} +Restart=always +RestartSec=30 + +[Install] +WantedBy=multi-user.target diff --git a/chatmaild/src/chatmaild/doveauth.py b/chatmaild/src/chatmaild/doveauth.py index f25ca1b1..8a4b5b56 100644 --- a/chatmaild/src/chatmaild/doveauth.py +++ b/chatmaild/src/chatmaild/doveauth.py @@ -58,17 +58,18 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool: return True -def get_user_data(db, user): +def get_user_data(db, config: Config, user): with db.read_connection() as conn: result = conn.get_user(user) if result: + result["home"] = f"/home/vmail/mail/{config.mail_domain}/{user}" result["uid"] = "vmail" result["gid"] = "vmail" return result -def lookup_userdb(db, user): - return get_user_data(db, user) +def lookup_userdb(db, config: Config, user): + return get_user_data(db, config, user) def lookup_passdb(db, config: Config, user, cleartext_password): @@ -80,6 +81,7 @@ def lookup_passdb(db, config: Config, user, cleartext_password): "UPDATE users SET last_login=? WHERE addr=?", (int(time.time()), user) ) + userdata["home"] = f"/home/vmail/mail/{config.mail_domain}/{user}" userdata["uid"] = "vmail" userdata["gid"] = "vmail" return userdata @@ -142,7 +144,7 @@ def handle_dovecot_request(msg, db, config: Config): if type == "userdb": user = args[0] if user.endswith(f"@{config.mail_domain}"): - res = lookup_userdb(db, user) + res = lookup_userdb(db, config, user) if res: reply_command = "O" else: diff --git a/chatmaild/src/chatmaild/metadata.py b/chatmaild/src/chatmaild/metadata.py new file mode 100644 index 00000000..52152cc2 --- /dev/null +++ b/chatmaild/src/chatmaild/metadata.py @@ -0,0 +1,106 @@ +import pwd +from socketserver import ( + UnixStreamServer, + StreamRequestHandler, + ThreadingMixIn, +) +from .config import read_config, Config +import sys +import logging +import os +import requests + + +def handle_dovecot_protocol(rfile, wfile, tokens, requests_session, config: Config): + # HELLO message, ignored. + msg = rfile.readline().strip().decode() + + transactions = {} + + while True: + msg = rfile.readline().strip().decode() + if not msg: + break + + short_command = msg[0] + if short_command == "L": + wfile.write(b"N\n") + elif short_command == "S": + # See header of + # + # for the documentation on the structure of the key. + + # Request GETMETADATA "INBOX" /private/chatmail + # results in a query for + # priv/dd72550f05eadc65542a1200cac67ad7/chatmail + # + # Request GETMETADATA "" /private/chatmail + # results in + # priv/dd72550f05eadc65542a1200cac67ad7/vendor/vendor.dovecot/pvt/server/chatmail + + parts = msg[1:].split("\t") + transaction_id = parts[0] + keyname = parts[1].split("/") + value = parts[2] if len(parts) > 2 else "" + if keyname[0] == "priv" and keyname[2] == "devicetoken": + tokens[keyname[1]] = value + elif keyname[0] == "priv" and keyname[2] == "messagenew": + guid = keyname[1] + token = tokens.get(guid) + if token: + response = requests_session.post( + "https://notifications.delta.chat/notify", + data=token, + timeout=60, + ) + if response.status_code == 410: + # 410 Gone status code + # means the token is no longer valid. + del tokens[guid] + else: + # Transaction failed. + transactions[transaction_id] = b"F\n" + elif short_command == "B": + # Begin transaction. + transaction_id = msg[1:].split("\t")[0] + transactions[transaction_id] = b"O\n" + elif short_command == "C": + # Commit transaction. + transaction_id = msg[1:].split("\t")[0] + wfile.write(transactions.pop(transaction_id, b"N\n")) + + wfile.flush() + + +class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer): + request_queue_size = 100 + + +def main(): + socket, username, config = sys.argv[1:] + passwd_entry = pwd.getpwnam(username) + config = read_config(config) + tokens = {} + requests_session = requests.Session() + + class Handler(StreamRequestHandler): + def handle(self): + try: + handle_dovecot_protocol( + self.rfile, self.wfile, tokens, requests_session, config + ) + except Exception: + logging.exception("Exception in the handler") + raise + + try: + os.unlink(socket) + except FileNotFoundError: + pass + + with ThreadedUnixStreamServer(socket, Handler) as server: + os.chown(socket, uid=passwd_entry.pw_uid, gid=passwd_entry.pw_gid) + try: + server.serve_forever() + except KeyboardInterrupt: + pass diff --git a/chatmaild/src/chatmaild/tests/test_doveauth.py b/chatmaild/src/chatmaild/tests/test_doveauth.py index 909ee47c..826d8951 100644 --- a/chatmaild/src/chatmaild/tests/test_doveauth.py +++ b/chatmaild/src/chatmaild/tests/test_doveauth.py @@ -17,7 +17,7 @@ from chatmaild.database import DBError def test_basic(db, example_config): lookup_passdb(db, example_config, "asdf12345@chat.example.org", "q9mr3faue") - data = get_user_data(db, "asdf12345@chat.example.org") + data = get_user_data(db, example_config, "asdf12345@chat.example.org") assert data data2 = lookup_passdb( db, example_config, "asdf12345@chat.example.org", "q9mr3jewvadsfaue" @@ -43,7 +43,7 @@ def test_nocreate_file(db, monkeypatch, tmpdir, example_config): lookup_passdb( db, example_config, "newuser12@chat.example.org", "zequ0Aimuchoodaechik" ) - assert not get_user_data(db, "newuser12@chat.example.org") + assert not get_user_data(db, example_config, "newuser12@chat.example.org") def test_db_version(db): diff --git a/cmdeploy/src/cmdeploy/__init__.py b/cmdeploy/src/cmdeploy/__init__.py index a0c53aca..e973f465 100644 --- a/cmdeploy/src/cmdeploy/__init__.py +++ b/cmdeploy/src/cmdeploy/__init__.py @@ -102,6 +102,7 @@ def _install_remote_venv_with_chatmaild(config) -> None: "doveauth", "filtermail", "echobot", + "chatmail-metadata", ): params = dict( execpath=f"{remote_venv_dir}/bin/{fn}", @@ -337,6 +338,27 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool: mode="644", ) need_restart |= auth_config.changed + lua_push_notification_script = files.put( + src=importlib.resources.files(__package__).joinpath( + "dovecot/push_notification.lua" + ), + dest="/etc/dovecot/push_notification.lua", + user="root", + group="root", + mode="644", + ) + need_restart |= lua_push_notification_script.changed + + sieve_script = files.put( + src=importlib.resources.files(__package__).joinpath( + "dovecot/default.sieve" + ), + dest="/etc/dovecot/default.sieve", + user="root", + group="root", + mode="644", + ) + need_restart |= sieve_script.changed files.template( src=importlib.resources.files(__package__).joinpath("dovecot/expunge.cron.j2"), @@ -484,7 +506,7 @@ def deploy_chatmail(config_path: Path) -> None: apt.packages( name="Install Dovecot", - packages=["dovecot-imapd", "dovecot-lmtpd"], + packages=["dovecot-imapd", "dovecot-lmtpd", "dovecot-sieve"], ) apt.packages( diff --git a/cmdeploy/src/cmdeploy/dovecot/default.sieve b/cmdeploy/src/cmdeploy/dovecot/default.sieve new file mode 100644 index 00000000..23c459d4 --- /dev/null +++ b/cmdeploy/src/cmdeploy/dovecot/default.sieve @@ -0,0 +1,5 @@ +require ["imap4flags"]; + +if header :is ["Auto-Submitted"] ["auto-replied", "auto-generated"] { + addflag "$Auto"; +} diff --git a/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 b/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 index e7b457c5..b996d2fd 100644 --- a/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 +++ b/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 @@ -21,7 +21,7 @@ mail_plugins = quota # these are the capabilities Delta Chat cares about actually # so let's keep the network overhead per login small # https://github.com/deltachat/deltachat-core-rust/blob/master/src/imap/capabilities.rs -imap_capability = IMAP4rev1 IDLE MOVE QUOTA CONDSTORE NOTIFY METADATA +imap_capability = IMAP4rev1 IDLE MOVE QUOTA CONDSTORE NOTIFY METADATA XDELTAPUSH # Authentication for system users. @@ -71,6 +71,9 @@ mail_privileged_group = vmail ## Mail processes ## +# Pass all IMAP METADATA requests to the server implementing Dovecot's dict protocol. +mail_attribute_dict = proxy:/run/dovecot/metadata.socket:metadata + # Enable IMAP COMPRESS (RFC 4978). # protocol imap { @@ -79,7 +82,21 @@ protocol imap { } protocol lmtp { - mail_plugins = $mail_plugins quota + # quota plugin documentation: + # + # + # notify plugin is a dependency of push_notification plugin: + # + # + # push_notification plugin documentation: + # + # + # mail_lua and push_notification_lua are needed for Lua push notification handler. + # + # + # Sieve to mark messages that should not be notified as \Seen + # + mail_plugins = $mail_plugins quota mail_lua notify push_notification push_notification_lua sieve } plugin { @@ -95,7 +112,15 @@ plugin { # quota_over_flag_value = TRUE } +# push_notification configuration +plugin { + # + push_notification_driver = lua:file=/etc/dovecot/push_notification.lua +} +plugin { + sieve_default = file:/etc/dovecot/default.sieve +} service lmtp { user=vmail diff --git a/cmdeploy/src/cmdeploy/dovecot/push_notification.lua b/cmdeploy/src/cmdeploy/dovecot/push_notification.lua new file mode 100644 index 00000000..6c13c1bc --- /dev/null +++ b/cmdeploy/src/cmdeploy/dovecot/push_notification.lua @@ -0,0 +1,32 @@ +function dovecot_lua_notify_begin_txn(user) + return user +end + +function contains(v, needle) + for _, keyword in ipairs(v) do + if keyword == needle then + return true + end + end + return false +end + +function dovecot_lua_notify_event_message_new(user, event) + local mbox = user:mailbox(event.mailbox) + mbox:sync() + + if user.username ~= event.from_address then + -- Incoming message + if not contains(event.keywords, "$Auto") then + -- Not an Auto-Submitted message, notifying. + + -- Notify METADATA server about new message. + mbox:metadata_set("/private/messagenew", "") + end + end + + mbox:free() +end + +function dovecot_lua_notify_end_txn(ctx, success) +end