add changelog entries

This is recommended to prevent SPF failure
from rejecting the message early in case messages
are remailed without breaking DKIM.

CHANGELOG.md

@@ -0,0 +1,22 @@
+# Changelog for chatmail deployment 
+
+## unreleased
+
+### Changes since March 15th, 2024
+
+- Fix various tests to pass again with "cmdeploy test". 
+  ([#245](https://github.com/deltachat/chatmail/pull/245),
+  [#242](https://github.com/deltachat/chatmail/pull/242)
+
+- Ensure lets-encrypt certificates are reloaded after renewal 
+  ([#244]) https://github.com/deltachat/chatmail/pull/244
+
+- Persist tokens to avoid iOS users loosing push-notifications when the
+  chatmail metadata service is restarted (happens regularly during deploys)
+  ([#238](https://github.com/deltachat/chatmail/pull/239)
+
+- Fix failing sieve-script compile errors on incoming messages
+  ([#237](https://github.com/deltachat/chatmail/pull/239)
+
+- Fix quota reporting after expunging of old mails
+  ([#233](https://github.com/deltachat/chatmail/pull/239)

chatmaild/pyproject.toml

@@ -10,6 +10,7 @@ dependencies = [
   "iniconfig",
   "deltachat-rpc-server",
   "deltachat-rpc-client",
+  "requests",
 ]
 
 [tool.setuptools]
@@ -20,6 +21,7 @@ where = ['src']
 
 [project.scripts]
 doveauth = "chatmaild.doveauth:main"
+chatmail-metadata = "chatmaild.metadata:main"
 filtermail = "chatmaild.filtermail:main"
 echobot = "chatmaild.echo:main"
 chatmail-metrics = "chatmaild.metrics:main"

chatmaild/src/chatmaild/chatmail-metadata.service.f

@@ -0,0 +1,10 @@
+[Unit]
+Description=Chatmail dict proxy for IMAP METADATA
+
+[Service]
+ExecStart={execpath} /run/dovecot/metadata.socket vmail {config_path} /home/vmail/metadata
+Restart=always
+RestartSec=30
+
+[Install]
+WantedBy=multi-user.target

chatmaild/src/chatmaild/doveauth.py

@@ -58,17 +58,18 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
     return True
 
 
-def get_user_data(db, user):
+def get_user_data(db, config: Config, user):
     with db.read_connection() as conn:
         result = conn.get_user(user)
     if result:
+        result["home"] = f"/home/vmail/mail/{config.mail_domain}/{user}"
         result["uid"] = "vmail"
         result["gid"] = "vmail"
     return result
 
 
-def lookup_userdb(db, user):
-    return get_user_data(db, user)
+def lookup_userdb(db, config: Config, user):
+    return get_user_data(db, config, user)
 
 
 def lookup_passdb(db, config: Config, user, cleartext_password):
@@ -80,6 +81,7 @@ def lookup_passdb(db, config: Config, user, cleartext_password):
                 "UPDATE users SET last_login=? WHERE addr=?", (int(time.time()), user)
             )
 
+            userdata["home"] = f"/home/vmail/mail/{config.mail_domain}/{user}"
             userdata["uid"] = "vmail"
             userdata["gid"] = "vmail"
             return userdata
@@ -142,7 +144,7 @@ def handle_dovecot_request(msg, db, config: Config):
             if type == "userdb":
                 user = args[0]
                 if user.endswith(f"@{config.mail_domain}"):
-                    res = lookup_userdb(db, user)
+                    res = lookup_userdb(db, config, user)
                 if res:
                     reply_command = "O"
                 else:

chatmaild/src/chatmaild/filtermail.py

@@ -17,8 +17,7 @@ def check_encrypted(message):
     """Check that the message is an OpenPGP-encrypted message."""
     if not message.is_multipart():
         return False
-    subject = message.get("subject")
-    if subject not in ("...", "Encrypted Message"):
+    if message.get("subject") != "...":
         return False
     if message.get_content_type() != "multipart/encrypted":
         return False

chatmaild/src/chatmaild/metadata.py

@@ -0,0 +1,184 @@
+import pwd
+
+import pathlib
+from queue import Queue
+from threading import Thread
+from socketserver import (
+    UnixStreamServer,
+    StreamRequestHandler,
+    ThreadingMixIn,
+)
+from .config import read_config
+import sys
+import logging
+import os
+import requests
+import marshal
+
+
+DICTPROXY_LOOKUP_CHAR = "L"
+DICTPROXY_ITERATE_CHAR = "I"
+DICTPROXY_SET_CHAR = "S"
+DICTPROXY_BEGIN_TRANSACTION_CHAR = "B"
+DICTPROXY_COMMIT_TRANSACTION_CHAR = "C"
+DICTPROXY_TRANSACTION_CHARS = "SBC"
+
+
+class Notifier:
+    def __init__(self, metadata_dir):
+        self.metadata_dir = metadata_dir
+        self.to_notify_queue = Queue()
+
+    def get_metadata(self, guid):
+        guid_path = self.metadata_dir.joinpath(guid)
+        if guid_path.exists():
+            with guid_path.open("rb") as f:
+                return marshal.load(f)
+        return {}
+
+    def set_metadata(self, guid, guid_data):
+        guid_path = self.metadata_dir.joinpath(guid)
+        write_path = guid_path.with_suffix(".tmp")
+        with write_path.open("wb") as f:
+            marshal.dump(guid_data, f)
+        os.rename(write_path, guid_path)
+
+    def set_token(self, guid, token):
+        guid_data = self.get_metadata(guid)
+        guid_data["token"] = token
+        self.set_metadata(guid, guid_data)
+
+    def del_token(self, guid):
+        guid_data = self.get_metadata(guid)
+        if "token" in guid_data:
+            del guid_data["token"]
+            self.set_metadata(guid, guid_data)
+
+    def get_token(self, guid):
+        return self.get_metadata(guid).get("token")
+
+    def new_message_for_guid(self, guid):
+        self.to_notify_queue.put(guid)
+
+    def thread_run_loop(self):
+        requests_session = requests.Session()
+        while 1:
+            self.thread_run_one(requests_session)
+
+    def thread_run_one(self, requests_session):
+        guid = self.to_notify_queue.get()
+        token = self.get_token(guid)
+        if token:
+            response = requests_session.post(
+                "https://notifications.delta.chat/notify",
+                data=token,
+                timeout=60,
+            )
+            if response.status_code == 410:
+                # 410 Gone status code
+                # means the token is no longer valid.
+                self.del_token(guid)
+
+
+def handle_dovecot_protocol(rfile, wfile, notifier):
+    # HELLO message, ignored.
+    msg = rfile.readline().strip().decode()
+
+    transactions = {}
+    while True:
+        msg = rfile.readline().strip().decode()
+        if not msg:
+            break
+
+        res = handle_dovecot_request(msg, transactions, notifier)
+        if res:
+            wfile.write(res.encode("ascii"))
+            wfile.flush()
+
+
+def handle_dovecot_request(msg, transactions, notifier):
+    # see https://doc.dovecot.org/3.0/developer_manual/design/dict_protocol/
+    short_command = msg[0]
+    parts = msg[1:].split("\t")
+    if short_command == DICTPROXY_LOOKUP_CHAR:
+        return "N\n"
+    elif short_command == DICTPROXY_ITERATE_CHAR:
+        # Empty line means ITER_FINISHED.
+        # If we don't return empty line Dovecot will timeout.
+        return "\n"
+
+    if short_command not in (DICTPROXY_TRANSACTION_CHARS):
+        return
+
+    transaction_id = parts[0]
+
+    if short_command == DICTPROXY_BEGIN_TRANSACTION_CHAR:
+        transactions[transaction_id] = "O\n"
+    elif short_command == DICTPROXY_COMMIT_TRANSACTION_CHAR:
+        # returns whether it failed or succeeded.
+        return transactions.pop(transaction_id, "N\n")
+    elif short_command == DICTPROXY_SET_CHAR:
+        # See header of
+        # <https://github.com/dovecot/core/blob/5e7965632395793d9355eb906b173bf28d2a10ca/src/lib-storage/mailbox-attribute.h>
+        # for the documentation on the structure of the key.
+
+        # Request GETMETADATA "INBOX" /private/chatmail
+        # results in a query for
+        # priv/dd72550f05eadc65542a1200cac67ad7/chatmail
+        #
+        # Request GETMETADATA "" /private/chatmail
+        # results in
+        # priv/dd72550f05eadc65542a1200cac67ad7/vendor/vendor.dovecot/pvt/server/chatmail
+
+        keyname = parts[1].split("/")
+        value = parts[2] if len(parts) > 2 else ""
+        if keyname[0] == "priv" and keyname[2] == "devicetoken":
+            notifier.set_token(keyname[1], value)
+        elif keyname[0] == "priv" and keyname[2] == "messagenew":
+            notifier.new_message_for_guid(keyname[1])
+        else:
+            # Transaction failed.
+            transactions[transaction_id] = "F\n"
+
+
+class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
+    request_queue_size = 100
+
+
+def main():
+    socket, username, config, metadata_dir = sys.argv[1:]
+    passwd_entry = pwd.getpwnam(username)
+
+    # XXX config is not currently used
+    config = read_config(config)
+    metadata_dir = pathlib.Path(metadata_dir)
+    if not metadata_dir.exists():
+        metadata_dir.mkdir()
+    notifier = Notifier(metadata_dir)
+
+    class Handler(StreamRequestHandler):
+        def handle(self):
+            try:
+                handle_dovecot_protocol(self.rfile, self.wfile, notifier)
+            except Exception:
+                logging.exception("Exception in the handler")
+                raise
+
+    try:
+        os.unlink(socket)
+    except FileNotFoundError:
+        pass
+
+    # start notifier thread for signalling new messages to
+    # Delta Chat notification server
+
+    t = Thread(target=notifier.thread_run_loop)
+    t.setDaemon(True)
+    t.start()
+
+    with ThreadedUnixStreamServer(socket, Handler) as server:
+        os.chown(socket, uid=passwd_entry.pw_uid, gid=passwd_entry.pw_gid)
+        try:
+            server.serve_forever()
+        except KeyboardInterrupt:
+            pass

chatmaild/src/chatmaild/tests/mail-data/encrypted-k9.eml

@@ -1,92 +0,0 @@
-Date: Mon, 05 Feb 2024 12:22:17 +0100
-From: Jus <tqwertyjd@nine.testrun.org>
-To: REDACTED
-User-Agent: K-9 Mail for Android
-Message-ID: <A238D610-1471-4A32-B387-B07811665F6D@nine.testrun.org>
-Autocrypt: addr=tqwertyjd@nine.testrun.org; keydata=
- mQGNBGWNXoMBDAC+D3Na6zJX8d8NEIIoYqcGsOeJCtPs4DZIE8x4nVIRewwG6+CU0/Su8J1sdNL8
- InVYnE0DUnRfL9RpT/6oHPsbuN8Yo/xyZbc6Df0MgstrbkiIpIb6YdpMB9vnS9phpTDXuVXwOdb+
- Q8woi46bZ4jdCm1x/5zW8e2fbahHSSFjDYTKydu3SVTeKPNVdHv9gG7SNQy0emOCP7NXxloi8+aR
- 4fbgfWpm6yb/pJFDH6jmPZ8LK228qXqSv6urquaCu/yD4S+XR/DvGqj2lA/ntvNhDOjrK4gWt5EA
- 4djfnTK6z/vt/IkSSca5ITjcbyPBpXnId896NQk76sAdG+K+mJGMJn9YahoI4UvISfCp/B5Fw3Bq
- 5NmeL5zKN14R5AW5E/Y2J693MJ+VubRoB3VR/RZi5ZeEd1aLkxhqITv6m8FRXrSpC6fIhbqAZGmm
- 91OAAVNn5/MqaAaWJ5iUKGlNJrDFHVBXEpNah24FEoe6olNiBDNnWJ9tqOmZIiIDPCl8FIEAEQEA
- AbQadHF3ZXJ0eWpkQG5pbmUudGVzdHJ1bi5vcmeJAbAEEwEKABoECwkIBwIVCgIWAQIZAAWCZY1e
- gwKeAQKbAwAKCRDtNhlhxu8KoFcjDACfMwEEuEStLsY8Wo/r/mmhZKLjgwRgdcVV7sFpksgk+Myy
- UyL5VUEi9KVd4lKWNqDSi9S67lW+6hwf/kUrycVIT5AA0i8ZXdtroUpkUIwMOaSEfGpUhPI/kQbz
- wqYJYES1XPqtpUmL4WR+52CHwtEeKZp+jiKnSNeh1QocBYjld0617dpb6XnAVl+69sQUHioxX7Bu
- c60CuABcFw78/9hvzX37NC7mvP1vbYS7iEze5p2CUweKtrnnDJpi+oBLAucKQRErIUfJUV/XFdE4
- j4m+NWAtcnyRVx3WruEWW+fzzb7+fc3fwV8pGCUcD4cb/Bzssg3LVLQiBENRXTmTc5RFxQXWbZae
- 5f6VLAkVEdoxOMVT2dCjLbwo1nPl9emTTIneRLjLX/cTNdbVZuq/Kv/SoXa05ayljSlZmrCF8k3x
- zESSeJLrrHkoSPoXECeAJbZyMYmOxZPZChVQhUCxDBAR9wzJmLoHBxoDxYMq16S+Ws4Z+lR2cHj2
- 4lFAMIzCKsy5AY0EZY1egwEMAIkCo235tKDEUjcW8w77AHFf6+W0183E7US8ze3C8T3UUDsh1nQn
- h+nZFOnKBRNQHUwRzWgV0ZQmllTrZt67fHOwywqHtaQMe90cZXbvhVoTzehw3B9bYT1j/24LDMy/
- /eQBZuQeSlcLD6+BC0ro7EGxn5T24CAsmMjrI2ppjgZFlcXo9bA+Xp6rI/HX8AQgWbbegtGnSIDB
- K20+e+xWANaWUsSBhIwsx2qz0IEq+RER60Zd1xZ41acVyNbDHNocEBnfzOF4GXRAz4M/v9l7ABep
- 21ALLC/OOKuC8cZDeY+HAbJ0qxggh//+ucpfBF0poOQDJzfNaOGysfn/0NGfxRVbFJc8fNc9P5+K
- fnjm4RdNkwQRXQeQfqPU9a4AlAH5vl8zHabyYIJUUtP+b7VF6VPfSVzJ+h4BPPIVS/TqKQM6HShX
- rGs9/DcXfDfcIXxhAfo2M+VKkrlunBev0OrhIDLNn5IigNIa78ZN9cZ/3SZVTfOzFnFuFtnO50tv
- q5OpvQARAQABiQGfBBgBCgAJBYJljV6DApsMAAoJEO02GWHG7wqgwnQL/ijcTiKNO2Cw4pvgggbL
- 8e2mgXCQn0aNufbYeylGdX/BP2SMRku5OubjESU0oMVx/Hhy19UkUFhCuOeouSNsbTd6w8Ou+nkh
- 6bs4KJvhMUFVQe6dE8Reci3EoImcTxV9nqWuvhdXkPddht9Pa5PoRpJlpWxHKpMfrPwWtbW/J8qn
- dnnc+x9FqxLVY+Z75GbrrMI/I2ClvbfgMnOGQxyZRhcPesiaMyp4bYbX3zxrIZXSG68CQERiMXk8
- UAeZFPgnm4Hh0rP5cn9enn8tj67ruFsEAU3YMi7eOVOSFlamjH0PTVr3ztdoMathEL7n1s5ksk5b
- Rgo0SgQy4OgApJIB0B16Zhcd66I4sTZLb2RRkFO9uHDFIOuJGTqYfR3ZjWlEftfxW80g7uLZYwfW
- gzMOEa9jSZpfWpiWDYfVYcHqQTOAoyc1ndwJno/4pO+kRmYoIUaoBqNqiqNtnTl9eiJHcb8kuaxa
- PP5Qy9N6C2QONUAb4aFBPe0cYXQ6AUnOqBmd3w==
-Subject: Encrypted Message
-MIME-Version: 1.0
-Content-Transfer-Encoding: 7bit
-Content-Type: multipart/encrypted; boundary="----GAXGEBIG1Q6UA7H3D91DE8MM3Q54OF";
-  protocol="application/pgp-encrypted"
-
-------GAXGEBIG1Q6UA7H3D91DE8MM3Q54OF
-Content-Type: application/pgp-encrypted
-Content-Transfer-Encoding: quoted-printable
-
-Version: 1
-------GAXGEBIG1Q6UA7H3D91DE8MM3Q54OF
-Content-Type: application/octet-stream; name="encrypted.asc"
-Content-Transfer-Encoding: 7bit
-Content-Disposition: inline; filename="encrypted.asc"
-
------BEGIN PGP MESSAGE-----
-
-hQGMAzhWohyp3KpBAQwAp76KkR97c29iXj6XwvYIom2c5jx9tNWBKR4t/oz1ltuh
-LR2NUAOxL2dNxBY8AFlFK8G6nWwDt29nOQhmPCSImI/WcG0NqIzZ3Sc+fTbBvnPy
-ne62g686UoE9la7qBSlNo0loQaJOBJ1MnFiVJ8o/p49TV1QoLAxXcaHp9eCs1lDa
-g+wWAxnoCANvPKQLTyDKHC98VBXeiNZuJuXK+yVcLojo/4Bd32py/a2eOKZcQMXs
-DGYZrL5ldphApUlD0R8cNjUA6AfIfwf6kBvN9GgacRFOLUMtP5/7CoYmZJd4ia8o
-8T1WjqlNZtwVcWLRe36Ry9aatavYhjtn5FPr7E/RO9BB0kDYMR5HkaT+WZgDQV1V
-/yaWzPSTP9tuCb4ZWz9St5K2zEcoTKpGbNwmX9YLr0tbcym+nLo6kB8ikD2nN3of
-d5aoDeKutoecMLGIF9pseJAGVkHqKj6FQ/kYYrKDlsA0kqvHuSHj/tI2ANB60TME
-diEy6e+wOn/V+Aq7AImphQGMA+7z+cqPiMawAQv8DtpHdiBdRkW85ZURmEsQQUKy
-6PsMqtRXE/kqU4O1HVyUWsUknDNtbziQDpmcfv1FZzJryLgrHNYbObgsbCgGJ/Ru
-3ahQwOkcHV6lzDZCwYs7QjYLAuw8SvWIISvqYOCHS0x/T0b/kjr3Gl+RJSvt78WW
-YhO0R9z/JagTn+uiHjbmAtN2yJyyk5AfLySZ7gixMkkUlz87AAHVpwopNXdzsJKP
-Qa4Kkc2BUWDCz9JLNEc8U2XmMXLEiXxOBKuvCaDqLnPORP2I7PZnXwyanB8ti10h
-Li3U5R5+/n4TmQwF3e7lbJSpkZAPxBrRNxwASqa5nzm90/SWpYhkZBgWder1vFJA
-rGOK7zOXFYxG6/SsRhftJPW8RC4MM5icK5Au+uUfA3Idosq49tZirjx6t9hdEjDK
-bh1Xj5EbJj78q8LqZhpd3SZD2jFa7TKrfGgtxujEujUtojf/VEcAmwKtxkuP7T12
-OktKIZ3BmeRiUfQVrPUoi/Y/VgBIj9wWQaDuh4QX0sJjAWigCK6gOWqYtWUEjVtL
-PeJB7ZMTTsivKxSS0/uhOap3Ur7nq8vofCY5g60uSAPXsbjIH/ZEAIbS2Stw+pxu
-PzG43YLXuvIOClmQ5iJtjPriUcXAWgf2/Ntcev3pIyuCvKHdrKCDADq871SWUyKj
-1/qmw4YNvGVVfmys1d77KO+TGZeKm5j9XhmJ8WJGw2g8uFKEj9IeZlfPNmhF2dtL
-xVjl/lrJPsZPepzh2MOHh78lHD9zq4voAPog2FXogBiZ4xKWVXPswiqOnHwFe9dn
-yOo3cM6dWKXDhOHWK6a8dF9NQ2Q7RrHlXrXwbfeS7ZQ9llXRa0auPkRQKK0XzdhK
-5otAcu0x65aw40hWOTboJR4AN3ypf8okLLN4fnB1kKaVIdMoWlYdTot3ezt0vy99
-cqpKH6/w6pzxRufezW11bxBFYEPnFzhoblN2VuChPFY2nleiifBS+aaKt2EzF1q5
-iL+dfvJupFXQtKyk8mLBB1QKl/qnBIAJchCeJET4xMpWvAhnluLbT9NO3G1uvN11
-U9ZWi9yhZbYxCubYdVYmI6ZD0NIOxV5kTLdRbWm+Ctz+Lm3I/8bO68J73qqKWbYQ
-rNEhj8QUAmKCdrFGufKp7t7wjgSG+h4/U5Xb+/glPuNgVF110BgbgUaAPGvttQO6
-M+QnelZeb4B525Gj7VYDDVcrEjKDK2kXidxfo8nKN3HFM+eU1cWBJqtJHJXbZ5XZ
-d1pmDypu3O8gOs/re1AVdYdGDou8axCC/yTTwbbbgNFxp88CBUe/Xzoaky4ZKunQ
-Q3rlpD9ayzE7Lp0NvxH90pQWsxDCEkJaADbwcbYtOmvoR1uUAfo2NwWuITG70iFz
-v0lqoiezn7D2J9XutpASQQInJus0gvy0ywjh3XirSoeZ2D4l6XcGMJjz7XNjzfnc
-B4lmHQiGbKsfkPyLdPNkobUHjA2TzyEqxs/tAyMSypo9UwNiGsn2NdMs7oif/xiQ
-X+/e2t5Rm+AJk9U7QYPN8o++px+JDZ+r3PkVAWW4c9rHgzf98Lo6pbt5h4eBrt/X
-MrxVY64iHOwasTdMu9F+fv8+akI=
-=Dj6B
------END PGP MESSAGE-----
-
-------GAXGEBIG1Q6UA7H3D91DE8MM3Q54OF--

chatmaild/src/chatmaild/tests/test_doveauth.py

@@ -17,7 +17,7 @@ from chatmaild.database import DBError
 
 def test_basic(db, example_config):
     lookup_passdb(db, example_config, "asdf12345@chat.example.org", "q9mr3faue")
-    data = get_user_data(db, "asdf12345@chat.example.org")
+    data = get_user_data(db, example_config, "asdf12345@chat.example.org")
     assert data
     data2 = lookup_passdb(
         db, example_config, "asdf12345@chat.example.org", "q9mr3jewvadsfaue"
@@ -43,7 +43,7 @@ def test_nocreate_file(db, monkeypatch, tmpdir, example_config):
     lookup_passdb(
         db, example_config, "newuser12@chat.example.org", "zequ0Aimuchoodaechik"
     )
-    assert not get_user_data(db, "newuser12@chat.example.org")
+    assert not get_user_data(db, example_config, "newuser12@chat.example.org")
 
 
 def test_db_version(db):

chatmaild/src/chatmaild/tests/test_filtermail.py

@@ -63,17 +63,6 @@ def test_filtermail_encryption_detection(maildata):
     assert not check_encrypted(msg)
 
 
-def test_filtermail_encryption_detection_k9subject(maildata):
-    msg = maildata(
-        "encrypted-k9.eml", from_addr="1@example.org", to_addr="2@example.org"
-    )
-    assert check_encrypted(msg)
-
-    # if the subject is not "..." it is not considered ac-encrypted
-    msg.replace_header("Subject", "Click this link")
-    assert not check_encrypted(msg)
-
-
 def test_filtermail_is_mdn(maildata, gencreds, handler):
     from_addr = gencreds()[0]
     to_addr = gencreds()[0] + ".other"

chatmaild/src/chatmaild/tests/test_metadata.py

@@ -0,0 +1,163 @@
+import io
+import pytest
+
+from chatmaild.metadata import (
+    handle_dovecot_request,
+    handle_dovecot_protocol,
+    Notifier,
+)
+
+
+@pytest.fixture
+def notifier(tmp_path):
+    metadata_dir = tmp_path.joinpath("metadata")
+    metadata_dir.mkdir()
+    return Notifier(metadata_dir)
+
+
+def test_notifier_persistence(tmp_path):
+    metadata_dir = tmp_path.joinpath("metadata")
+    metadata_dir.mkdir()
+    notifier1 = Notifier(metadata_dir)
+    notifier2 = Notifier(metadata_dir)
+    assert notifier1.get_token(guid="guid00") is None
+    assert notifier2.get_token(guid="guid00") is None
+
+    notifier1.set_token("guid00", "01234")
+    notifier1.set_token("guid03", "456")
+    assert notifier2.get_token("guid00") == "01234"
+    assert notifier2.get_token("guid03") == "456"
+    notifier2.del_token("guid00")
+    assert notifier1.get_token("guid00") is None
+
+
+def test_handle_dovecot_request_lookup_fails(notifier):
+    res = handle_dovecot_request("Lpriv/123/chatmail", {}, notifier)
+    assert res == "N\n"
+
+
+def test_handle_dovecot_request_happy_path(notifier):
+    transactions = {}
+
+    # lookups return the same NOTFOUND result
+    res = handle_dovecot_request("Lpriv/123/chatmail", transactions, notifier)
+    assert res == "N\n"
+    assert notifier.get_token("guid00") is None and not transactions
+
+    # set device token in a transaction
+    tx = "1111"
+    msg = f"B{tx}\tuser"
+    res = handle_dovecot_request(msg, transactions, notifier)
+    assert not res and notifier.get_token("guid00") is None
+    assert transactions == {tx: "O\n"}
+
+    msg = f"S{tx}\tpriv/guid00/devicetoken\t01234"
+    res = handle_dovecot_request(msg, transactions, notifier)
+    assert not res
+    assert len(transactions) == 1
+    assert notifier.get_token("guid00") == "01234"
+
+    msg = f"C{tx}"
+    res = handle_dovecot_request(msg, transactions, notifier)
+    assert res == "O\n"
+    assert len(transactions) == 0
+    assert notifier.get_token("guid00") == "01234"
+
+    # trigger notification for incoming message
+    assert handle_dovecot_request(f"B{tx}\tuser", transactions, notifier) is None
+    msg = f"S{tx}\tpriv/guid00/messagenew"
+    assert handle_dovecot_request(msg, transactions, notifier) is None
+    assert notifier.to_notify_queue.get() == "guid00"
+    assert notifier.to_notify_queue.qsize() == 0
+    assert handle_dovecot_request(f"C{tx}\tuser", transactions, notifier) == "O\n"
+    assert not transactions
+
+
+def test_handle_dovecot_protocol_set_devicetoken(notifier):
+    rfile = io.BytesIO(
+        b"\n".join(
+            [
+                b"HELLO",
+                b"Btx00\tuser",
+                b"Stx00\tpriv/guid00/devicetoken\t01234",
+                b"Ctx00",
+            ]
+        )
+    )
+    wfile = io.BytesIO()
+    handle_dovecot_protocol(rfile, wfile, notifier)
+    assert notifier.get_token("guid00") == "01234"
+    assert wfile.getvalue() == b"O\n"
+
+
+def test_handle_dovecot_protocol_iterate(notifier):
+    rfile = io.BytesIO(
+        b"\n".join(
+            [
+                b"H",
+                b"I9\t0\tpriv/5cbe730f146fea6535be0d003dd4fc98/\tci-2dzsrs@nine.testrun.org",
+            ]
+        )
+    )
+    wfile = io.BytesIO()
+    handle_dovecot_protocol(rfile, wfile, notifier)
+    assert wfile.getvalue() == b"\n"
+
+
+def test_handle_dovecot_protocol_messagenew(notifier):
+    rfile = io.BytesIO(
+        b"\n".join(
+            [
+                b"HELLO",
+                b"Btx01\tuser",
+                b"Stx01\tpriv/guid00/messagenew",
+                b"Ctx01",
+            ]
+        )
+    )
+    wfile = io.BytesIO()
+    handle_dovecot_protocol(rfile, wfile, notifier)
+    assert wfile.getvalue() == b"O\n"
+    assert notifier.to_notify_queue.get() == "guid00"
+    assert notifier.to_notify_queue.qsize() == 0
+
+
+def test_notifier_thread_run(notifier):
+    requests = []
+
+    class ReqMock:
+        def post(self, url, data, timeout):
+            requests.append((url, data, timeout))
+
+            class Result:
+                status_code = 200
+
+            return Result()
+
+    notifier.set_token("guid00", "01234")
+    notifier.new_message_for_guid("guid00")
+    notifier.thread_run_one(ReqMock())
+    url, data, timeout = requests[0]
+    assert data == "01234"
+    assert notifier.get_token("guid00") == "01234"
+
+
+def test_notifier_thread_run_gone_removes_token(notifier):
+    requests = []
+
+    class ReqMock:
+        def post(self, url, data, timeout):
+            requests.append((url, data, timeout))
+
+            class Result:
+                status_code = 410
+
+            return Result()
+
+    notifier.set_token("guid00", "01234")
+    notifier.new_message_for_guid("guid00")
+    assert notifier.get_token("guid00") == "01234"
+    notifier.thread_run_one(ReqMock())
+    url, data, timeout = requests[0]
+    assert data == "01234"
+    assert notifier.get_token("guid00") is None

cmdeploy/src/cmdeploy/__init__.py

@@ -102,6 +102,7 @@ def _install_remote_venv_with_chatmaild(config) -> None:
         "doveauth",
         "filtermail",
         "echobot",
+        "chatmail-metadata",
     ):
         params = dict(
             execpath=f"{remote_venv_dir}/bin/{fn}",
@@ -337,6 +338,34 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
         mode="644",
     )
     need_restart |= auth_config.changed
+    lua_push_notification_script = files.put(
+        src=importlib.resources.files(__package__).joinpath(
+            "dovecot/push_notification.lua"
+        ),
+        dest="/etc/dovecot/push_notification.lua",
+        user="root",
+        group="root",
+        mode="644",
+    )
+    need_restart |= lua_push_notification_script.changed
+
+    sieve_script = files.put(
+        src=importlib.resources.files(__package__).joinpath(
+            "dovecot/default.sieve"
+        ),
+        dest="/etc/dovecot/default.sieve",
+        user="root",
+        group="root",
+        mode="644",
+    )
+    need_restart |= sieve_script.changed
+    if sieve_script.changed:
+        server.shell(
+            name=f"compile sieve script",
+            commands=[
+                f"/usr/bin/sievec /etc/dovecot/default.sieve"
+            ],
+        )
 
     files.template(
         src=importlib.resources.files(__package__).joinpath("dovecot/expunge.cron.j2"),
@@ -426,8 +455,9 @@ def check_config(config):
     mail_domain = config.mail_domain
     if mail_domain != "testrun.org" and not mail_domain.endswith(".testrun.org"):
         blocked_words = "merlinux schmieder testrun.org".split()
-        for value in config.__dict__.values():
-            if any(x in str(value) for x in blocked_words):
+        for key in config.__dict__:
+            value = config.__dict__[key]
+            if key.startswith("privacy") and any(x in str(value) for x in blocked_words):
                 raise ValueError(
                     f"please set your own privacy contacts/addresses in {config._inipath}"
                 )
@@ -448,6 +478,10 @@ def deploy_chatmail(config_path: Path) -> None:
     apt.update(name="apt update", cache_time=24 * 3600)
     server.group(name="Create vmail group", group="vmail", system=True)
     server.user(name="Create vmail user", user="vmail", group="vmail", system=True)
+    apt.packages(
+        name="Install rsync",
+        packages=["rsync"],
+    )
 
     # Run local DNS resolver `unbound`.
     # `resolvconf` takes care of setting up /etc/resolv.conf
@@ -483,7 +517,7 @@ def deploy_chatmail(config_path: Path) -> None:
 
     apt.packages(
         name="Install Dovecot",
-        packages=["dovecot-imapd", "dovecot-lmtpd"],
+        packages=["dovecot-imapd", "dovecot-lmtpd", "dovecot-sieve"],
     )
 
     apt.packages(

cmdeploy/src/cmdeploy/chatmail.zone.f

@@ -6,7 +6,7 @@ _submissions._tcp.{chatmail_domain}. SRV 0 1 465 {chatmail_domain}.
 _imap._tcp.{chatmail_domain}.        SRV 0 1 143 {chatmail_domain}.
 _imaps._tcp.{chatmail_domain}.       SRV 0 1 993 {chatmail_domain}.
 {chatmail_domain}.                   CAA 128 issue "letsencrypt.org;accounturi={acme_account_url}"
-{chatmail_domain}.                   TXT "v=spf1 a:{chatmail_domain} -all"
+{chatmail_domain}.                   TXT "v=spf1 a:{chatmail_domain} ~all"
 _dmarc.{chatmail_domain}.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
 _mta-sts.{chatmail_domain}.          TXT "v=STSv1; id={sts_id}"
 mta-sts.{chatmail_domain}.           CNAME {chatmail_domain}.

cmdeploy/src/cmdeploy/dns.py

@@ -36,12 +36,11 @@ class DNS:
         cmd = "ip a | grep inet6 | grep 'scope global' | sed -e 's#/64 scope global##' | sed -e 's#inet6##'"
         return self.shell(cmd).strip()
 
-    def get(self, typ: str, domain: str) -> Optional[str]:
-        """Get a DNS entry"""
+    def get(self, typ: str, domain: str) -> str:
+        """Get a DNS entry or empty string if there is none."""
         dig_result = self.shell(f"dig -r -q {domain} -t {typ} +short")
         line = dig_result.partition("\n")[0]
-        if line:
-            return line
+        return line
 
     def check_ptr_record(self, ip: str, mail_domain) -> bool:
         """Check the PTR record for an IPv4 or IPv6 address."""

cmdeploy/src/cmdeploy/dovecot/default.sieve

@@ -0,0 +1,7 @@
+require ["imap4flags"];
+
+# flag the message so it doesn't cause a push notification
+
+if header :is ["Auto-Submitted"] ["auto-replied", "auto-generated"] {
+	addflag "$Auto";
+}

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -13,6 +13,12 @@ auth_cache_size = 100M
 mail_debug = yes
 {% endif %}
 
+# Prevent warnings similar to:
+#   config: Warning: service auth { client_limit=1000 } is lower than required under max. load (10200). Counted for protocol services with service_count != 1: service lmtp { process_limit=100 } + service imap-urlauth-login { process_limit=100 } + service imap-login { process_limit=10000 }
+#   config: Warning: service anvil { client_limit=1000 } is lower than required under max. load (10103). Counted with: service imap-urlauth-login { process_limit=100 } + service imap-login { process_limit=10000 } + service auth { process_limit=1 }
+#   master: Warning: service(stats): client_limit (1000) reached, client connections are being dropped
+default_client_limit = 20000
+
 mail_server_admin = mailto:root@{{ config.mail_domain }}
 mail_server_comment = Chatmail server
 
@@ -21,7 +27,7 @@ mail_plugins = quota
 # these are the capabilities Delta Chat cares about actually 
 # so let's keep the network overhead per login small
 # https://github.com/deltachat/deltachat-core-rust/blob/master/src/imap/capabilities.rs
-imap_capability = IMAP4rev1 IDLE MOVE QUOTA CONDSTORE NOTIFY METADATA
+imap_capability = IMAP4rev1 IDLE MOVE QUOTA CONDSTORE NOTIFY METADATA XDELTAPUSH
 
 
 # Authentication for system users.
@@ -71,6 +77,9 @@ mail_privileged_group = vmail
 ## Mail processes
 ##
 
+# Pass all IMAP METADATA requests to the server implementing Dovecot's dict protocol.
+mail_attribute_dict = proxy:/run/dovecot/metadata.socket:metadata
+
 # Enable IMAP COMPRESS (RFC 4978).
 # <https://datatracker.ietf.org/doc/html/rfc4978.html>
 protocol imap {
@@ -79,7 +88,21 @@ protocol imap {
 }
 
 protocol lmtp {
-  mail_plugins = $mail_plugins quota
+  # quota plugin documentation:
+  # <https://doc.dovecot.org/configuration_manual/quota_plugin/>
+  #
+  # notify plugin is a dependency of push_notification plugin:
+  # <https://doc.dovecot.org/settings/plugin/notify-plugin/>
+  #
+  # push_notification plugin documentation:
+  # <https://doc.dovecot.org/configuration_manual/push_notification/>
+  #
+  # mail_lua and push_notification_lua are needed for Lua push notification handler.
+  # <https://doc.dovecot.org/configuration_manual/push_notification/#configuration>
+  #
+  # Sieve to mark messages that should not be notified as \Seen
+  # <https://doc.dovecot.org/configuration_manual/sieve/configuration/>
+  mail_plugins = $mail_plugins quota mail_lua notify push_notification push_notification_lua sieve
 }
 
 plugin {
@@ -95,7 +118,15 @@ plugin {
   # quota_over_flag_value = TRUE
 }
 
+# push_notification configuration
+plugin {
+  # <https://doc.dovecot.org/configuration_manual/push_notification/#lua-lua>
+  push_notification_driver = lua:file=/etc/dovecot/push_notification.lua
+}
 
+plugin {
+  sieve_default = file:/etc/dovecot/default.sieve
+}
 
 service lmtp {
   user=vmail

cmdeploy/src/cmdeploy/dovecot/expunge.cron.j2

@@ -8,3 +8,4 @@
 # or only temporary (but then they shouldn't be around after {{ config.delete_mails_after }} days anyway).
 2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+3 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -name 'maildirsize' -type f -delete

cmdeploy/src/cmdeploy/dovecot/push_notification.lua

@@ -0,0 +1,32 @@
+function dovecot_lua_notify_begin_txn(user)
+  return user
+end
+
+function contains(v, needle)
+  for _, keyword in ipairs(v) do
+    if keyword == needle then
+      return true
+    end
+  end
+  return false
+end
+
+function dovecot_lua_notify_event_message_new(user, event)
+  local mbox = user:mailbox(event.mailbox)
+  mbox:sync()
+
+  if user.username ~= event.from_address then
+    -- Incoming message
+    if not contains(event.keywords, "$Auto") then
+      -- Not an Auto-Submitted message, notifying.
+
+      -- Notify METADATA server about new message.
+      mbox:metadata_set("/private/messagenew", "")
+    end
+  end
+
+  mbox:free()
+end
+
+function dovecot_lua_notify_end_txn(ctx, success)
+end