refactor test and filtermail to prepare it for BeforeQueue handling

chatmaild/src/chatmaild/filtermail.py

@@ -1,11 +1,14 @@
 #!/usr/bin/env python3
 import asyncio
 import logging
+import time
+import sys
 from email.parser import BytesParser
 from email import policy
 from email.utils import parseaddr
 
 from aiosmtpd.lmtp import LMTP
+from aiosmtpd.smtp import SMTP
 from aiosmtpd.controller import UnixSocketController
 from smtplib import SMTP as SMTPClient
 
@@ -32,12 +35,40 @@ def check_encrypted(message):
     return True
 
 
-class ExampleController(UnixSocketController):
+
-    def factory(self):
+class BeforeQueueHandler:
-        return LMTP(self.handler, **self.SMTP_kwargs)
+    transport_class = SMTP
+
+    def __init__(self):
+        self.send_rate_limiter = SendRateLimiter()
+
+    async def handle_MAIL(self, server, session, envelope, address, mail_options):
+        logging.info(f"handle_MAIL from {address}")
+        if self.send_rate_limiter.is_sending_allowed(address):
+            envelope.mail_from = address
+            return "250 OK"
+        return "400 per-user ratelimit exceeded"
 
 
-class ExampleHandler:
+class SendRateLimiter:
+    MAX_USER_SEND_PER_MINUTE = 80
+
+    def __init__(self):
+        self.addr2timestamps = {}
+
+    def is_sending_allowed(self, mail_from):
+        last = self.addr2timestamps.setdefault(mail_from, [])
+        now = time.time()
+        last[:] = [ts for ts in last if ts >= (now - 60)]
+        if len(last) <= self.MAX_USER_SEND_PER_MINUTE:
+            last.append(now)
+            return True
+        return False
+
+
+class AfterQueueHandler:
+    transport_class = LMTP
+
     async def handle_RCPT(self, server, session, envelope, address, rcpt_options):
         envelope.rcpt_tos.append(address)
         return "250 OK"
@@ -55,13 +86,6 @@ class ExampleHandler:
         return "\r\n".join(res)
 
 
-async def asyncmain(loop):
-    controller = ExampleController(
-        ExampleHandler(), unix_socket="/var/spool/postfix/private/filtermail"
-    )
-    controller.start()
-
-
 def lmtp_handle_DATA(envelope):
     """the central filtering function for e-mails."""
     logging.info(f"Processing DATA message from {envelope.mail_from}")
@@ -113,13 +137,25 @@ def lmtp_handle_DATA(envelope):
     return valid_recipients, res
 
 
+class Controller(UnixSocketController):
+    def factory(self):
+        return self.handler.transport_class(self.handler, **self.SMTP_kwargs)
+
+
+async def asyncmain(loop, handler, unix_socket_fn):
+    Controller(handler, unix_socket=unix_socket_fn).start()
+
+
+name2Handler = {"beforequeue": BeforeQueueHandler, "afterqueue": AfterQueueHandler}
+
+
 def main():
+    args = sys.argv[1:]
+    assert len(args) == 2
+    handler = name2Handler[args[0]]()
+    unix_socket_fn = args[1]
     logging.basicConfig(level=logging.INFO)
     loop = asyncio.new_event_loop()
     asyncio.set_event_loop(loop)
-    loop.create_task(asyncmain(loop=loop))
+    loop.create_task(asyncmain(loop, handler, unix_socket_fn))
     loop.run_forever()
-
-
-if __name__ == "__main__":
-    main()

chatmaild/src/chatmaild/filtermail.service

@@ -2,7 +2,7 @@
 Description=Email filter for chatmail servers
 
 [Service]
-ExecStart=/usr/local/bin/filtermail
+ExecStart=/usr/local/bin/filtermail afterqueue /var/spool/postfix/private/filtermail
 Restart=always
 RestartSec=30
 

chatmaild/src/chatmaild/test_filtermail.py

@@ -1,6 +1,7 @@
-from .filtermail import check_encrypted, lmtp_handle_DATA
+from .filtermail import check_encrypted, lmtp_handle_DATA, SendRateLimiter
 from email.parser import BytesParser
 from email import policy
+import pytest
 
 
 def test_reject_forged_from():
@@ -326,3 +327,16 @@ def test_filtermail():
             ]
         ).encode()
     )
+
+
+def test_send_rate_limiter():
+    limiter = SendRateLimiter()
+    for i in range(100):
+        if limiter.is_sending_allowed("some@example.org"):
+            if i <= SendRateLimiter.MAX_USER_SEND_PER_MINUTE:
+                continue
+            pytest.fail("limiter didn't work")
+        else:
+            assert i == SendRateLimiter.MAX_USER_SEND_PER_MINUTE + 1
+            break
+

deploy-chatmail/src/deploy_chatmail/postfix/main.cf.j2

@@ -29,9 +29,6 @@ myhostname = {{ config.domain_name }}
 alias_maps = hash:/etc/aliases
 alias_database = hash:/etc/aliases
 
-# hard limit, also on internal messages
-smtpd_client_message_rate_limit = 80
-
 # Postfix does not deliver mail for any domain by itself.
 # Primary domain is listed in `virtual_mailbox_domains` instead
 # and handed over to Dovecot.

online-tests/test_0_login.py

@@ -37,174 +37,17 @@ def test_login_same_password(imap_or_smtp, gencreds):
     imap_or_smtp.login(user2, password1)
 
 
-@pytest.mark.xfail(reason="Only rate limit is internal as well now")
+@pytest.mark.slow
-def test_no_internal_rate_limit(smtp, gencreds):
+def test_exceed_rate_limit(cmsetup, gencreds, mailgen):
-    """Test that there is no rate limit between accounts on the same chatmail server."""
+    """Test that the per-account send-mail limit is exceeded."""
-    user, password = gencreds()
+    user1, user2 = cmsetup.gen_users(2)
-    to_addr, = gencreds()
+    mail = mailgen.get_encrypted(user1.addr, user2.addr)
-    smtp.connect()
-    smtp.login(user, password)
-
-    mail = "\r\n".join(
-        [
-            "Subject: ...",
-            f"From: <{user}>",
-            f"To: <{to_addr}>",
-            "Date: Sun, 15 Oct 2023 16:43:21 +0000",
-            "Message-ID: <Mr.UVyJWZmkCKM.hGzNc6glBE_@c2.testrun.org>",
-            "In-Reply-To: <Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>",
-            "References: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>",
-            "\t<Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>",
-            "Chat-Version: 1.0",
-            f"Autocrypt: addr={user}; prefer-encrypt=mutual;",
-            "\tkeydata=xjMEZSwWjhYJKwYBBAHaRw8BAQdAQBEhqeJh0GueHB6kF/DUQqYCxARNBVokg/AzT+7LqH",
-            "\trNFzxiYXJiYXpAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUsFo4CGwMECwkIBwYVCAkKCwID",
-            "\tFgIBFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX9A4AEAnHWHp49eBCMHK5t66gYPiW",
-            "\tXQuB1mwUjzGfYWB+0RXUoA/0xcQ3FbUNlGKW7Blp6eMFfViv6Mv2d3kNSXACB6nmcMzjgEZSwWjhIK",
-            "\tKwYBBAGXVQEFAQEHQBpY5L2M1XHo0uxf8SX1wNLBp/OVvidoWHQF2Jz+kJsUAwEIB8J4BBgWCAAgBQ",
-            "\tJlLBaOAhsMFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX/INgEA37AJaNvruYsJVanP",
-            "\tIXnYw4CKd55UAwl8Zcy+M2diAbkA/0fHHcGV4r78hpbbL1Os52DPOdqYQRauIeJUeG+G6bQO",
-            "MIME-Version: 1.0",
-            'Content-Type: multipart/encrypted; protocol="application/pgp-encrypted";',
-            '\tboundary="YFrteb74qSXmggbOxZL9dRnhymywAi"',
-            "",
-            "",
-            "--YFrteb74qSXmggbOxZL9dRnhymywAi",
-            "Content-Description: PGP/MIME version identification",
-            "Content-Type: application/pgp-encrypted",
-            "",
-            "Version: 1",
-            "",
-            "",
-            "--YFrteb74qSXmggbOxZL9dRnhymywAi",
-            "Content-Description: OpenPGP encrypted message",
-            'Content-Disposition: inline; filename="encrypted.asc";',
-            'Content-Type: application/octet-stream; name="encrypted.asc"',
-            "",
-            "-----BEGIN PGP MESSAGE-----",
-            "",
-            "wU4DhW3gBZ/VvCYSAQdA8bMs2spwbKdGjVsL1ByPkNrqD7frpB73maeL6I6SzDYg",
-            "O5G53tv339RdKq3WRcCtEEvxjHlUx2XNwXzC04BpmfvBTgNfPUyLDzjXnxIBB0Ae",
-            "8ymwGvXMCCimHXN0Dg8Ui62KOi03h0UgheoHWovJSCDF4CKre/xtFr3nL7lq/PKI",
-            "JsjVNz7/RK9FSXF6WwfONtLCyQGEuVAsB/KXfCBEyfKhaMwGHvhujRidGW5uV1no",
-            "lMGl3ODmo29Lgeu2uSE7EpJRZoe6hU6ddmBkqxax61ZtkaFlGFFpdo2K8balNNdz",
-            "ZsJ/9mmI9x3oOJ4/l1nhQbUO9ADbs7gJhFdV5Qkp30b5fCI7bU+aoe1ccBbLe/WM",
-            "YUty1PqcuQT7XjA+XmYuL261tvW8pBetT+i33/E2d8PzzYt2IuK9qeevyS+yxdwA",
-            "kfwejFWzzsUlJaDxs1x4XOxkMgSj+jo+g12dFOb7fyClsAnq23iDb8AuaT/BScAI",
-            "+lO+gher69+6LmM7VGHLG5k762J1jTaQCaKt1s8TAWV99Eo4491vL6fyvk3l/Cfg",
-            "RXSwiWFgj19Pn0Rq7CD9v22UE2vdUMBTcV4aw79mClk1YQ23jbF0y5DCjPdJ62Zo",
-            "tskBgFt3NoWV80jZ76zIBLrrjLwCCll8JjJtFwSkt2GX5RFBsVa4A8IDht9RtEk7",
-            "rrHgbSZQfkauEi/mH3/6CDZoLqSHudUZ7d4MaJwun1TkFYGe2ORwGJd4OBj3oGJp",
-            "H8YBwCpk///L/fKjX0Gg3M8nrpM4wrRFhPKidAgO/kcm25X4+ZHlVkWBTCt5RWKI",
-            "fHh6oLDZCqCfcgMkE1KKmwfIHaUkhq5BPRigwy6i5dh1DM4+1UCLh3dxzVbqE9b9",
-            "61NB19nXdRtDA2sOUnj9ve6m/wEPyCb6/zBQZqvCBYb1/AjdXpUrFT+DbpfyxaXN",
-            "XfhDVb5mNqNM/IVj0V5fvTc6vOfYbzQtPm10H+FdWWfb+rJRfyC3MA2w2IqstFe3",
-            "w3bu2iE6CQvSqRvge+ZqLKt/NqYwOURiUmpuklbl3kPJ97+mfKWoiqk8Iz1VY+bb",
-            "NMUC7aoGv+jcoj+WS6PYO8N6BeRVUUB3ZJSf8nzjgxm1/BcM+UD3BPrlhT11ODRs",
-            "baifGbprMWwt3dhb8cQgRT8GPdpO1OsDkzL6iikMjLHWWiA99GV6ruiHsIPw6boW",
-            "A6/uSOskbDHOROotKmddGTBd0iiHXAoQsJFt1ZjUkt6EHrgWs+GAvrvKpXs1mrz8",
-            "uj3GwEFrHS+Xuf2UDgpszYT3hI2cL/kUtGakVR7m7vVMZqXBUbZdGAEb1PZNPwsI",
-            "E4aMK02+EVB+tSN4Fzj99N2YD0inVYt+oPjr2tHhUS6aSGBNS/48Ki47DOg4Sxkn",
-            "lkOWnEbCD+XTnbDd",
-            "=agR5",
-            "-----END PGP MESSAGE-----",
-            "",
-            "",
-            "--YFrteb74qSXmggbOxZL9dRnhymywAi--",
-            "",
-            "",
-        ]
-    ).encode()
-    for i in range(100):
-        print("Sending mail", str(i))
-        smtp.conn.sendmail(user, to_addr, mail)
-
-
-def test_exceed_rate_limit(smtp, gencreds):
-    """Test that the outbound rate limit is exceeded if we send a lot of messages at once."""
-    user, password = gencreds()
-    smtp.connect()
-    smtp.login(user, password)
-
-    to_addr = "foobar@example.org"
-    mail = "\r\n".join(
-        [
-            "Subject: ...",
-            f"From: <{user}>",
-            f"To: <{to_addr}>",
-            "Date: Sun, 15 Oct 2023 16:43:21 +0000",
-            "Message-ID: <Mr.UVyJWZmkCKM.hGzNc6glBE_@c2.testrun.org>",
-            "In-Reply-To: <Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>",
-            "References: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>",
-            "\t<Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>",
-            "Chat-Version: 1.0",
-            f"Autocrypt: addr={user}; prefer-encrypt=mutual;",
-            "\tkeydata=xjMEZSwWjhYJKwYBBAHaRw8BAQdAQBEhqeJh0GueHB6kF/DUQqYCxARNBVokg/AzT+7LqH",
-            "\trNFzxiYXJiYXpAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUsFo4CGwMECwkIBwYVCAkKCwID",
-            "\tFgIBFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX9A4AEAnHWHp49eBCMHK5t66gYPiW",
-            "\tXQuB1mwUjzGfYWB+0RXUoA/0xcQ3FbUNlGKW7Blp6eMFfViv6Mv2d3kNSXACB6nmcMzjgEZSwWjhIK",
-            "\tKwYBBAGXVQEFAQEHQBpY5L2M1XHo0uxf8SX1wNLBp/OVvidoWHQF2Jz+kJsUAwEIB8J4BBgWCAAgBQ",
-            "\tJlLBaOAhsMFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX/INgEA37AJaNvruYsJVanP",
-            "\tIXnYw4CKd55UAwl8Zcy+M2diAbkA/0fHHcGV4r78hpbbL1Os52DPOdqYQRauIeJUeG+G6bQO",
-            "MIME-Version: 1.0",
-            'Content-Type: multipart/encrypted; protocol="application/pgp-encrypted";',
-            '\tboundary="YFrteb74qSXmggbOxZL9dRnhymywAi"',
-            "",
-            "",
-            "--YFrteb74qSXmggbOxZL9dRnhymywAi",
-            "Content-Description: PGP/MIME version identification",
-            "Content-Type: application/pgp-encrypted",
-            "",
-            "Version: 1",
-            "",
-            "",
-            "--YFrteb74qSXmggbOxZL9dRnhymywAi",
-            "Content-Description: OpenPGP encrypted message",
-            'Content-Disposition: inline; filename="encrypted.asc";',
-            'Content-Type: application/octet-stream; name="encrypted.asc"',
-            "",
-            "-----BEGIN PGP MESSAGE-----",
-            "",
-            "wU4DhW3gBZ/VvCYSAQdA8bMs2spwbKdGjVsL1ByPkNrqD7frpB73maeL6I6SzDYg",
-            "O5G53tv339RdKq3WRcCtEEvxjHlUx2XNwXzC04BpmfvBTgNfPUyLDzjXnxIBB0Ae",
-            "8ymwGvXMCCimHXN0Dg8Ui62KOi03h0UgheoHWovJSCDF4CKre/xtFr3nL7lq/PKI",
-            "JsjVNz7/RK9FSXF6WwfONtLCyQGEuVAsB/KXfCBEyfKhaMwGHvhujRidGW5uV1no",
-            "lMGl3ODmo29Lgeu2uSE7EpJRZoe6hU6ddmBkqxax61ZtkaFlGFFpdo2K8balNNdz",
-            "ZsJ/9mmI9x3oOJ4/l1nhQbUO9ADbs7gJhFdV5Qkp30b5fCI7bU+aoe1ccBbLe/WM",
-            "YUty1PqcuQT7XjA+XmYuL261tvW8pBetT+i33/E2d8PzzYt2IuK9qeevyS+yxdwA",
-            "kfwejFWzzsUlJaDxs1x4XOxkMgSj+jo+g12dFOb7fyClsAnq23iDb8AuaT/BScAI",
-            "+lO+gher69+6LmM7VGHLG5k762J1jTaQCaKt1s8TAWV99Eo4491vL6fyvk3l/Cfg",
-            "RXSwiWFgj19Pn0Rq7CD9v22UE2vdUMBTcV4aw79mClk1YQ23jbF0y5DCjPdJ62Zo",
-            "tskBgFt3NoWV80jZ76zIBLrrjLwCCll8JjJtFwSkt2GX5RFBsVa4A8IDht9RtEk7",
-            "rrHgbSZQfkauEi/mH3/6CDZoLqSHudUZ7d4MaJwun1TkFYGe2ORwGJd4OBj3oGJp",
-            "H8YBwCpk///L/fKjX0Gg3M8nrpM4wrRFhPKidAgO/kcm25X4+ZHlVkWBTCt5RWKI",
-            "fHh6oLDZCqCfcgMkE1KKmwfIHaUkhq5BPRigwy6i5dh1DM4+1UCLh3dxzVbqE9b9",
-            "61NB19nXdRtDA2sOUnj9ve6m/wEPyCb6/zBQZqvCBYb1/AjdXpUrFT+DbpfyxaXN",
-            "XfhDVb5mNqNM/IVj0V5fvTc6vOfYbzQtPm10H+FdWWfb+rJRfyC3MA2w2IqstFe3",
-            "w3bu2iE6CQvSqRvge+ZqLKt/NqYwOURiUmpuklbl3kPJ97+mfKWoiqk8Iz1VY+bb",
-            "NMUC7aoGv+jcoj+WS6PYO8N6BeRVUUB3ZJSf8nzjgxm1/BcM+UD3BPrlhT11ODRs",
-            "baifGbprMWwt3dhb8cQgRT8GPdpO1OsDkzL6iikMjLHWWiA99GV6ruiHsIPw6boW",
-            "A6/uSOskbDHOROotKmddGTBd0iiHXAoQsJFt1ZjUkt6EHrgWs+GAvrvKpXs1mrz8",
-            "uj3GwEFrHS+Xuf2UDgpszYT3hI2cL/kUtGakVR7m7vVMZqXBUbZdGAEb1PZNPwsI",
-            "E4aMK02+EVB+tSN4Fzj99N2YD0inVYt+oPjr2tHhUS6aSGBNS/48Ki47DOg4Sxkn",
-            "lkOWnEbCD+XTnbDd",
-            "=agR5",
-            "-----END PGP MESSAGE-----",
-            "",
-            "",
-            "--YFrteb74qSXmggbOxZL9dRnhymywAi--",
-            "",
-            "",
-        ]
-    ).encode()
     for i in range(100):
         print("Sending mail", str(i))
         try:
-            smtp.conn.sendmail(user, to_addr, mail)
+            user1.smtp.sendmail(user1.addr, [user2.addr], mail)
         except smtplib.SMTPSenderRefused as e:
-            if i == 0:
+            if i < 80:
-                pytest.fail(f"rate limit was exceeded too early with msg {i} - maybe wait a minute before testing?")
-            if i < 41:
                 pytest.fail(f"rate limit was exceeded too early with msg {i}")
             assert e.smtp_code == 450
             assert b'4.7.1 Error: too much mail from' in e.smtp_error