From 31c71fa6e9714990537ea7cdcdd5614ca3450abc Mon Sep 17 00:00:00 2001 From: missytake Date: Tue, 17 Oct 2023 11:56:32 +0200 Subject: [PATCH 01/13] add test for postfix rate limiting --- online-tests/test_0_login.py | 90 ++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/online-tests/test_0_login.py b/online-tests/test_0_login.py index 63388b01..bab1aeb5 100644 --- a/online-tests/test_0_login.py +++ b/online-tests/test_0_login.py @@ -1,4 +1,5 @@ import pytest +import smtplib def test_login_basic_functioning(imap_or_smtp, gencreds, lp): @@ -34,3 +35,92 @@ def test_login_same_password(imap_or_smtp, gencreds): imap_or_smtp.login(user1, password1) imap_or_smtp.connect() imap_or_smtp.login(user2, password1) + + +def test_exceed_rate_limit(smtp, gencreds): + """Test that the outbound rate limit is exceeded if we send a lot of messages at once.""" + user, password = gencreds() + smtp.connect() + smtp.login(user, password) + + to_addr = "foobar@example.org" + mail = "\r\n".join( + [ + "Subject: ...", + f"From: <{user}>", + f"To: <{to_addr}>", + "Date: Sun, 15 Oct 2023 16:43:21 +0000", + "Message-ID: ", + "In-Reply-To: ", + "References: ", + "\t", + "Chat-Version: 1.0", + f"Autocrypt: addr={user}; prefer-encrypt=mutual;", + "\tkeydata=xjMEZSwWjhYJKwYBBAHaRw8BAQdAQBEhqeJh0GueHB6kF/DUQqYCxARNBVokg/AzT+7LqH", + "\trNFzxiYXJiYXpAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUsFo4CGwMECwkIBwYVCAkKCwID", + "\tFgIBFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX9A4AEAnHWHp49eBCMHK5t66gYPiW", + "\tXQuB1mwUjzGfYWB+0RXUoA/0xcQ3FbUNlGKW7Blp6eMFfViv6Mv2d3kNSXACB6nmcMzjgEZSwWjhIK", + "\tKwYBBAGXVQEFAQEHQBpY5L2M1XHo0uxf8SX1wNLBp/OVvidoWHQF2Jz+kJsUAwEIB8J4BBgWCAAgBQ", + "\tJlLBaOAhsMFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX/INgEA37AJaNvruYsJVanP", + "\tIXnYw4CKd55UAwl8Zcy+M2diAbkA/0fHHcGV4r78hpbbL1Os52DPOdqYQRauIeJUeG+G6bQO", + "MIME-Version: 1.0", + 'Content-Type: multipart/encrypted; protocol="application/pgp-encrypted";', + '\tboundary="YFrteb74qSXmggbOxZL9dRnhymywAi"', + "", + "", + "--YFrteb74qSXmggbOxZL9dRnhymywAi", + "Content-Description: PGP/MIME version identification", + "Content-Type: application/pgp-encrypted", + "", + "Version: 1", + "", + "", + "--YFrteb74qSXmggbOxZL9dRnhymywAi", + "Content-Description: OpenPGP encrypted message", + 'Content-Disposition: inline; filename="encrypted.asc";', + 'Content-Type: application/octet-stream; name="encrypted.asc"', + "", + "-----BEGIN PGP MESSAGE-----", + "", + "wU4DhW3gBZ/VvCYSAQdA8bMs2spwbKdGjVsL1ByPkNrqD7frpB73maeL6I6SzDYg", + "O5G53tv339RdKq3WRcCtEEvxjHlUx2XNwXzC04BpmfvBTgNfPUyLDzjXnxIBB0Ae", + "8ymwGvXMCCimHXN0Dg8Ui62KOi03h0UgheoHWovJSCDF4CKre/xtFr3nL7lq/PKI", + "JsjVNz7/RK9FSXF6WwfONtLCyQGEuVAsB/KXfCBEyfKhaMwGHvhujRidGW5uV1no", + "lMGl3ODmo29Lgeu2uSE7EpJRZoe6hU6ddmBkqxax61ZtkaFlGFFpdo2K8balNNdz", + "ZsJ/9mmI9x3oOJ4/l1nhQbUO9ADbs7gJhFdV5Qkp30b5fCI7bU+aoe1ccBbLe/WM", + "YUty1PqcuQT7XjA+XmYuL261tvW8pBetT+i33/E2d8PzzYt2IuK9qeevyS+yxdwA", + "kfwejFWzzsUlJaDxs1x4XOxkMgSj+jo+g12dFOb7fyClsAnq23iDb8AuaT/BScAI", + "+lO+gher69+6LmM7VGHLG5k762J1jTaQCaKt1s8TAWV99Eo4491vL6fyvk3l/Cfg", + "RXSwiWFgj19Pn0Rq7CD9v22UE2vdUMBTcV4aw79mClk1YQ23jbF0y5DCjPdJ62Zo", + "tskBgFt3NoWV80jZ76zIBLrrjLwCCll8JjJtFwSkt2GX5RFBsVa4A8IDht9RtEk7", + "rrHgbSZQfkauEi/mH3/6CDZoLqSHudUZ7d4MaJwun1TkFYGe2ORwGJd4OBj3oGJp", + "H8YBwCpk///L/fKjX0Gg3M8nrpM4wrRFhPKidAgO/kcm25X4+ZHlVkWBTCt5RWKI", + "fHh6oLDZCqCfcgMkE1KKmwfIHaUkhq5BPRigwy6i5dh1DM4+1UCLh3dxzVbqE9b9", + "61NB19nXdRtDA2sOUnj9ve6m/wEPyCb6/zBQZqvCBYb1/AjdXpUrFT+DbpfyxaXN", + "XfhDVb5mNqNM/IVj0V5fvTc6vOfYbzQtPm10H+FdWWfb+rJRfyC3MA2w2IqstFe3", + "w3bu2iE6CQvSqRvge+ZqLKt/NqYwOURiUmpuklbl3kPJ97+mfKWoiqk8Iz1VY+bb", + "NMUC7aoGv+jcoj+WS6PYO8N6BeRVUUB3ZJSf8nzjgxm1/BcM+UD3BPrlhT11ODRs", + "baifGbprMWwt3dhb8cQgRT8GPdpO1OsDkzL6iikMjLHWWiA99GV6ruiHsIPw6boW", + "A6/uSOskbDHOROotKmddGTBd0iiHXAoQsJFt1ZjUkt6EHrgWs+GAvrvKpXs1mrz8", + "uj3GwEFrHS+Xuf2UDgpszYT3hI2cL/kUtGakVR7m7vVMZqXBUbZdGAEb1PZNPwsI", + "E4aMK02+EVB+tSN4Fzj99N2YD0inVYt+oPjr2tHhUS6aSGBNS/48Ki47DOg4Sxkn", + "lkOWnEbCD+XTnbDd", + "=agR5", + "-----END PGP MESSAGE-----", + "", + "", + "--YFrteb74qSXmggbOxZL9dRnhymywAi--", + "", + "", + ] + ).encode() + for i in range(100): + print("Sending mail", str(i)) + try: + smtp.conn.sendmail(user, to_addr, mail) + except smtplib.SMTPSenderRefused as e: + assert i > 41 + assert e.smtp_code == 450 + assert b'4.7.1 Error: too much mail from' in e.smtp_error + return + pytest.fail("Rate limit was not exceeded") From b8673d86251d5d585b5a9fedee454265c3b4bca2 Mon Sep 17 00:00:00 2001 From: missytake Date: Tue, 17 Oct 2023 11:57:04 +0200 Subject: [PATCH 02/13] postfix: add simple rate limiting without allow list or leaky bucket, also for internal mail --- deploy-chatmail/src/deploy_chatmail/postfix/main.cf.j2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/deploy-chatmail/src/deploy_chatmail/postfix/main.cf.j2 b/deploy-chatmail/src/deploy_chatmail/postfix/main.cf.j2 index 303aed13..975a5241 100644 --- a/deploy-chatmail/src/deploy_chatmail/postfix/main.cf.j2 +++ b/deploy-chatmail/src/deploy_chatmail/postfix/main.cf.j2 @@ -29,6 +29,9 @@ myhostname = {{ config.domain_name }} alias_maps = hash:/etc/aliases alias_database = hash:/etc/aliases +# hard limit, also on internal messages +smtpd_client_message_rate_limit = 80 + # Postfix does not deliver mail for any domain by itself. # Primary domain is listed in `virtual_mailbox_domains` instead # and handed over to Dovecot. From 015269fa7b7be201ad6355617fc6d70a86dc8944 Mon Sep 17 00:00:00 2001 From: missytake Date: Tue, 17 Oct 2023 12:17:23 +0200 Subject: [PATCH 03/13] test: test that there is no internal limit (xfail for now) --- online-tests/test_0_login.py | 83 ++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/online-tests/test_0_login.py b/online-tests/test_0_login.py index bab1aeb5..2d8b2daf 100644 --- a/online-tests/test_0_login.py +++ b/online-tests/test_0_login.py @@ -37,6 +37,89 @@ def test_login_same_password(imap_or_smtp, gencreds): imap_or_smtp.login(user2, password1) +@pytest.mark.xfail(reason="Only rate limit is internal as well now") +def test_no_internal_rate_limit(smtp, gencreds): + """Test that there is no rate limit between accounts on the same chatmail server.""" + user, password = gencreds() + to_addr, = gencreds() + smtp.connect() + smtp.login(user, password) + + mail = "\r\n".join( + [ + "Subject: ...", + f"From: <{user}>", + f"To: <{to_addr}>", + "Date: Sun, 15 Oct 2023 16:43:21 +0000", + "Message-ID: ", + "In-Reply-To: ", + "References: ", + "\t", + "Chat-Version: 1.0", + f"Autocrypt: addr={user}; prefer-encrypt=mutual;", + "\tkeydata=xjMEZSwWjhYJKwYBBAHaRw8BAQdAQBEhqeJh0GueHB6kF/DUQqYCxARNBVokg/AzT+7LqH", + "\trNFzxiYXJiYXpAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUsFo4CGwMECwkIBwYVCAkKCwID", + "\tFgIBFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX9A4AEAnHWHp49eBCMHK5t66gYPiW", + "\tXQuB1mwUjzGfYWB+0RXUoA/0xcQ3FbUNlGKW7Blp6eMFfViv6Mv2d3kNSXACB6nmcMzjgEZSwWjhIK", + "\tKwYBBAGXVQEFAQEHQBpY5L2M1XHo0uxf8SX1wNLBp/OVvidoWHQF2Jz+kJsUAwEIB8J4BBgWCAAgBQ", + "\tJlLBaOAhsMFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX/INgEA37AJaNvruYsJVanP", + "\tIXnYw4CKd55UAwl8Zcy+M2diAbkA/0fHHcGV4r78hpbbL1Os52DPOdqYQRauIeJUeG+G6bQO", + "MIME-Version: 1.0", + 'Content-Type: multipart/encrypted; protocol="application/pgp-encrypted";', + '\tboundary="YFrteb74qSXmggbOxZL9dRnhymywAi"', + "", + "", + "--YFrteb74qSXmggbOxZL9dRnhymywAi", + "Content-Description: PGP/MIME version identification", + "Content-Type: application/pgp-encrypted", + "", + "Version: 1", + "", + "", + "--YFrteb74qSXmggbOxZL9dRnhymywAi", + "Content-Description: OpenPGP encrypted message", + 'Content-Disposition: inline; filename="encrypted.asc";', + 'Content-Type: application/octet-stream; name="encrypted.asc"', + "", + "-----BEGIN PGP MESSAGE-----", + "", + "wU4DhW3gBZ/VvCYSAQdA8bMs2spwbKdGjVsL1ByPkNrqD7frpB73maeL6I6SzDYg", + "O5G53tv339RdKq3WRcCtEEvxjHlUx2XNwXzC04BpmfvBTgNfPUyLDzjXnxIBB0Ae", + "8ymwGvXMCCimHXN0Dg8Ui62KOi03h0UgheoHWovJSCDF4CKre/xtFr3nL7lq/PKI", + "JsjVNz7/RK9FSXF6WwfONtLCyQGEuVAsB/KXfCBEyfKhaMwGHvhujRidGW5uV1no", + "lMGl3ODmo29Lgeu2uSE7EpJRZoe6hU6ddmBkqxax61ZtkaFlGFFpdo2K8balNNdz", + "ZsJ/9mmI9x3oOJ4/l1nhQbUO9ADbs7gJhFdV5Qkp30b5fCI7bU+aoe1ccBbLe/WM", + "YUty1PqcuQT7XjA+XmYuL261tvW8pBetT+i33/E2d8PzzYt2IuK9qeevyS+yxdwA", + "kfwejFWzzsUlJaDxs1x4XOxkMgSj+jo+g12dFOb7fyClsAnq23iDb8AuaT/BScAI", + "+lO+gher69+6LmM7VGHLG5k762J1jTaQCaKt1s8TAWV99Eo4491vL6fyvk3l/Cfg", + "RXSwiWFgj19Pn0Rq7CD9v22UE2vdUMBTcV4aw79mClk1YQ23jbF0y5DCjPdJ62Zo", + "tskBgFt3NoWV80jZ76zIBLrrjLwCCll8JjJtFwSkt2GX5RFBsVa4A8IDht9RtEk7", + "rrHgbSZQfkauEi/mH3/6CDZoLqSHudUZ7d4MaJwun1TkFYGe2ORwGJd4OBj3oGJp", + "H8YBwCpk///L/fKjX0Gg3M8nrpM4wrRFhPKidAgO/kcm25X4+ZHlVkWBTCt5RWKI", + "fHh6oLDZCqCfcgMkE1KKmwfIHaUkhq5BPRigwy6i5dh1DM4+1UCLh3dxzVbqE9b9", + "61NB19nXdRtDA2sOUnj9ve6m/wEPyCb6/zBQZqvCBYb1/AjdXpUrFT+DbpfyxaXN", + "XfhDVb5mNqNM/IVj0V5fvTc6vOfYbzQtPm10H+FdWWfb+rJRfyC3MA2w2IqstFe3", + "w3bu2iE6CQvSqRvge+ZqLKt/NqYwOURiUmpuklbl3kPJ97+mfKWoiqk8Iz1VY+bb", + "NMUC7aoGv+jcoj+WS6PYO8N6BeRVUUB3ZJSf8nzjgxm1/BcM+UD3BPrlhT11ODRs", + "baifGbprMWwt3dhb8cQgRT8GPdpO1OsDkzL6iikMjLHWWiA99GV6ruiHsIPw6boW", + "A6/uSOskbDHOROotKmddGTBd0iiHXAoQsJFt1ZjUkt6EHrgWs+GAvrvKpXs1mrz8", + "uj3GwEFrHS+Xuf2UDgpszYT3hI2cL/kUtGakVR7m7vVMZqXBUbZdGAEb1PZNPwsI", + "E4aMK02+EVB+tSN4Fzj99N2YD0inVYt+oPjr2tHhUS6aSGBNS/48Ki47DOg4Sxkn", + "lkOWnEbCD+XTnbDd", + "=agR5", + "-----END PGP MESSAGE-----", + "", + "", + "--YFrteb74qSXmggbOxZL9dRnhymywAi--", + "", + "", + ] + ).encode() + for i in range(100): + print("Sending mail", str(i)) + smtp.conn.sendmail(user, to_addr, mail) + + def test_exceed_rate_limit(smtp, gencreds): """Test that the outbound rate limit is exceeded if we send a lot of messages at once.""" user, password = gencreds() From 410bc50a8b8a6b2f19c516587a9306fc5e6c8abc Mon Sep 17 00:00:00 2001 From: missytake Date: Tue, 17 Oct 2023 12:23:38 +0200 Subject: [PATCH 04/13] test: report if rate limit from last test was still active --- online-tests/test_0_login.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/online-tests/test_0_login.py b/online-tests/test_0_login.py index 2d8b2daf..919c9936 100644 --- a/online-tests/test_0_login.py +++ b/online-tests/test_0_login.py @@ -202,7 +202,10 @@ def test_exceed_rate_limit(smtp, gencreds): try: smtp.conn.sendmail(user, to_addr, mail) except smtplib.SMTPSenderRefused as e: - assert i > 41 + if i == 0: + pytest.fail(f"rate limit was exceeded too early with msg {i} - maybe wait a minute before testing?") + if i < 41: + pytest.fail(f"rate limit was exceeded too early with msg {i}") assert e.smtp_code == 450 assert b'4.7.1 Error: too much mail from' in e.smtp_error return From bbd277350647bf74faeda4d0815e45f69a1bcb73 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Wed, 18 Oct 2023 21:43:06 +0200 Subject: [PATCH 05/13] refactor test and filtermail to prepare it for BeforeQueue handling --- chatmaild/src/chatmaild/filtermail.py | 68 +++++-- chatmaild/src/chatmaild/filtermail.service | 2 +- chatmaild/src/chatmaild/test_filtermail.py | 16 +- .../src/deploy_chatmail/postfix/main.cf.j2 | 3 - online-tests/test_0_login.py | 171 +----------------- 5 files changed, 75 insertions(+), 185 deletions(-) diff --git a/chatmaild/src/chatmaild/filtermail.py b/chatmaild/src/chatmaild/filtermail.py index 0a79a97f..76777f51 100644 --- a/chatmaild/src/chatmaild/filtermail.py +++ b/chatmaild/src/chatmaild/filtermail.py @@ -1,11 +1,14 @@ #!/usr/bin/env python3 import asyncio import logging +import time +import sys from email.parser import BytesParser from email import policy from email.utils import parseaddr from aiosmtpd.lmtp import LMTP +from aiosmtpd.smtp import SMTP from aiosmtpd.controller import UnixSocketController from smtplib import SMTP as SMTPClient @@ -32,12 +35,40 @@ def check_encrypted(message): return True -class ExampleController(UnixSocketController): - def factory(self): - return LMTP(self.handler, **self.SMTP_kwargs) + +class BeforeQueueHandler: + transport_class = SMTP + + def __init__(self): + self.send_rate_limiter = SendRateLimiter() + + async def handle_MAIL(self, server, session, envelope, address, mail_options): + logging.info(f"handle_MAIL from {address}") + if self.send_rate_limiter.is_sending_allowed(address): + envelope.mail_from = address + return "250 OK" + return "400 per-user ratelimit exceeded" -class ExampleHandler: +class SendRateLimiter: + MAX_USER_SEND_PER_MINUTE = 80 + + def __init__(self): + self.addr2timestamps = {} + + def is_sending_allowed(self, mail_from): + last = self.addr2timestamps.setdefault(mail_from, []) + now = time.time() + last[:] = [ts for ts in last if ts >= (now - 60)] + if len(last) <= self.MAX_USER_SEND_PER_MINUTE: + last.append(now) + return True + return False + + +class AfterQueueHandler: + transport_class = LMTP + async def handle_RCPT(self, server, session, envelope, address, rcpt_options): envelope.rcpt_tos.append(address) return "250 OK" @@ -55,13 +86,6 @@ class ExampleHandler: return "\r\n".join(res) -async def asyncmain(loop): - controller = ExampleController( - ExampleHandler(), unix_socket="/var/spool/postfix/private/filtermail" - ) - controller.start() - - def lmtp_handle_DATA(envelope): """the central filtering function for e-mails.""" logging.info(f"Processing DATA message from {envelope.mail_from}") @@ -113,13 +137,25 @@ def lmtp_handle_DATA(envelope): return valid_recipients, res +class Controller(UnixSocketController): + def factory(self): + return self.handler.transport_class(self.handler, **self.SMTP_kwargs) + + +async def asyncmain(loop, handler, unix_socket_fn): + Controller(handler, unix_socket=unix_socket_fn).start() + + +name2Handler = {"beforequeue": BeforeQueueHandler, "afterqueue": AfterQueueHandler} + + def main(): + args = sys.argv[1:] + assert len(args) == 2 + handler = name2Handler[args[0]]() + unix_socket_fn = args[1] logging.basicConfig(level=logging.INFO) loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - loop.create_task(asyncmain(loop=loop)) + loop.create_task(asyncmain(loop, handler, unix_socket_fn)) loop.run_forever() - - -if __name__ == "__main__": - main() diff --git a/chatmaild/src/chatmaild/filtermail.service b/chatmaild/src/chatmaild/filtermail.service index b5953854..0e6a9a18 100644 --- a/chatmaild/src/chatmaild/filtermail.service +++ b/chatmaild/src/chatmaild/filtermail.service @@ -2,7 +2,7 @@ Description=Email filter for chatmail servers [Service] -ExecStart=/usr/local/bin/filtermail +ExecStart=/usr/local/bin/filtermail afterqueue /var/spool/postfix/private/filtermail Restart=always RestartSec=30 diff --git a/chatmaild/src/chatmaild/test_filtermail.py b/chatmaild/src/chatmaild/test_filtermail.py index 5e76f774..cc86cfa7 100644 --- a/chatmaild/src/chatmaild/test_filtermail.py +++ b/chatmaild/src/chatmaild/test_filtermail.py @@ -1,6 +1,7 @@ -from .filtermail import check_encrypted, lmtp_handle_DATA +from .filtermail import check_encrypted, lmtp_handle_DATA, SendRateLimiter from email.parser import BytesParser from email import policy +import pytest def test_reject_forged_from(): @@ -326,3 +327,16 @@ def test_filtermail(): ] ).encode() ) + + +def test_send_rate_limiter(): + limiter = SendRateLimiter() + for i in range(100): + if limiter.is_sending_allowed("some@example.org"): + if i <= SendRateLimiter.MAX_USER_SEND_PER_MINUTE: + continue + pytest.fail("limiter didn't work") + else: + assert i == SendRateLimiter.MAX_USER_SEND_PER_MINUTE + 1 + break + diff --git a/deploy-chatmail/src/deploy_chatmail/postfix/main.cf.j2 b/deploy-chatmail/src/deploy_chatmail/postfix/main.cf.j2 index 975a5241..303aed13 100644 --- a/deploy-chatmail/src/deploy_chatmail/postfix/main.cf.j2 +++ b/deploy-chatmail/src/deploy_chatmail/postfix/main.cf.j2 @@ -29,9 +29,6 @@ myhostname = {{ config.domain_name }} alias_maps = hash:/etc/aliases alias_database = hash:/etc/aliases -# hard limit, also on internal messages -smtpd_client_message_rate_limit = 80 - # Postfix does not deliver mail for any domain by itself. # Primary domain is listed in `virtual_mailbox_domains` instead # and handed over to Dovecot. diff --git a/online-tests/test_0_login.py b/online-tests/test_0_login.py index 919c9936..6c397b2a 100644 --- a/online-tests/test_0_login.py +++ b/online-tests/test_0_login.py @@ -37,174 +37,17 @@ def test_login_same_password(imap_or_smtp, gencreds): imap_or_smtp.login(user2, password1) -@pytest.mark.xfail(reason="Only rate limit is internal as well now") -def test_no_internal_rate_limit(smtp, gencreds): - """Test that there is no rate limit between accounts on the same chatmail server.""" - user, password = gencreds() - to_addr, = gencreds() - smtp.connect() - smtp.login(user, password) - - mail = "\r\n".join( - [ - "Subject: ...", - f"From: <{user}>", - f"To: <{to_addr}>", - "Date: Sun, 15 Oct 2023 16:43:21 +0000", - "Message-ID: ", - "In-Reply-To: ", - "References: ", - "\t", - "Chat-Version: 1.0", - f"Autocrypt: addr={user}; prefer-encrypt=mutual;", - "\tkeydata=xjMEZSwWjhYJKwYBBAHaRw8BAQdAQBEhqeJh0GueHB6kF/DUQqYCxARNBVokg/AzT+7LqH", - "\trNFzxiYXJiYXpAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUsFo4CGwMECwkIBwYVCAkKCwID", - "\tFgIBFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX9A4AEAnHWHp49eBCMHK5t66gYPiW", - "\tXQuB1mwUjzGfYWB+0RXUoA/0xcQ3FbUNlGKW7Blp6eMFfViv6Mv2d3kNSXACB6nmcMzjgEZSwWjhIK", - "\tKwYBBAGXVQEFAQEHQBpY5L2M1XHo0uxf8SX1wNLBp/OVvidoWHQF2Jz+kJsUAwEIB8J4BBgWCAAgBQ", - "\tJlLBaOAhsMFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX/INgEA37AJaNvruYsJVanP", - "\tIXnYw4CKd55UAwl8Zcy+M2diAbkA/0fHHcGV4r78hpbbL1Os52DPOdqYQRauIeJUeG+G6bQO", - "MIME-Version: 1.0", - 'Content-Type: multipart/encrypted; protocol="application/pgp-encrypted";', - '\tboundary="YFrteb74qSXmggbOxZL9dRnhymywAi"', - "", - "", - "--YFrteb74qSXmggbOxZL9dRnhymywAi", - "Content-Description: PGP/MIME version identification", - "Content-Type: application/pgp-encrypted", - "", - "Version: 1", - "", - "", - "--YFrteb74qSXmggbOxZL9dRnhymywAi", - "Content-Description: OpenPGP encrypted message", - 'Content-Disposition: inline; filename="encrypted.asc";', - 'Content-Type: application/octet-stream; name="encrypted.asc"', - "", - "-----BEGIN PGP MESSAGE-----", - "", - "wU4DhW3gBZ/VvCYSAQdA8bMs2spwbKdGjVsL1ByPkNrqD7frpB73maeL6I6SzDYg", - "O5G53tv339RdKq3WRcCtEEvxjHlUx2XNwXzC04BpmfvBTgNfPUyLDzjXnxIBB0Ae", - "8ymwGvXMCCimHXN0Dg8Ui62KOi03h0UgheoHWovJSCDF4CKre/xtFr3nL7lq/PKI", - "JsjVNz7/RK9FSXF6WwfONtLCyQGEuVAsB/KXfCBEyfKhaMwGHvhujRidGW5uV1no", - "lMGl3ODmo29Lgeu2uSE7EpJRZoe6hU6ddmBkqxax61ZtkaFlGFFpdo2K8balNNdz", - "ZsJ/9mmI9x3oOJ4/l1nhQbUO9ADbs7gJhFdV5Qkp30b5fCI7bU+aoe1ccBbLe/WM", - "YUty1PqcuQT7XjA+XmYuL261tvW8pBetT+i33/E2d8PzzYt2IuK9qeevyS+yxdwA", - "kfwejFWzzsUlJaDxs1x4XOxkMgSj+jo+g12dFOb7fyClsAnq23iDb8AuaT/BScAI", - "+lO+gher69+6LmM7VGHLG5k762J1jTaQCaKt1s8TAWV99Eo4491vL6fyvk3l/Cfg", - "RXSwiWFgj19Pn0Rq7CD9v22UE2vdUMBTcV4aw79mClk1YQ23jbF0y5DCjPdJ62Zo", - "tskBgFt3NoWV80jZ76zIBLrrjLwCCll8JjJtFwSkt2GX5RFBsVa4A8IDht9RtEk7", - "rrHgbSZQfkauEi/mH3/6CDZoLqSHudUZ7d4MaJwun1TkFYGe2ORwGJd4OBj3oGJp", - "H8YBwCpk///L/fKjX0Gg3M8nrpM4wrRFhPKidAgO/kcm25X4+ZHlVkWBTCt5RWKI", - "fHh6oLDZCqCfcgMkE1KKmwfIHaUkhq5BPRigwy6i5dh1DM4+1UCLh3dxzVbqE9b9", - "61NB19nXdRtDA2sOUnj9ve6m/wEPyCb6/zBQZqvCBYb1/AjdXpUrFT+DbpfyxaXN", - "XfhDVb5mNqNM/IVj0V5fvTc6vOfYbzQtPm10H+FdWWfb+rJRfyC3MA2w2IqstFe3", - "w3bu2iE6CQvSqRvge+ZqLKt/NqYwOURiUmpuklbl3kPJ97+mfKWoiqk8Iz1VY+bb", - "NMUC7aoGv+jcoj+WS6PYO8N6BeRVUUB3ZJSf8nzjgxm1/BcM+UD3BPrlhT11ODRs", - "baifGbprMWwt3dhb8cQgRT8GPdpO1OsDkzL6iikMjLHWWiA99GV6ruiHsIPw6boW", - "A6/uSOskbDHOROotKmddGTBd0iiHXAoQsJFt1ZjUkt6EHrgWs+GAvrvKpXs1mrz8", - "uj3GwEFrHS+Xuf2UDgpszYT3hI2cL/kUtGakVR7m7vVMZqXBUbZdGAEb1PZNPwsI", - "E4aMK02+EVB+tSN4Fzj99N2YD0inVYt+oPjr2tHhUS6aSGBNS/48Ki47DOg4Sxkn", - "lkOWnEbCD+XTnbDd", - "=agR5", - "-----END PGP MESSAGE-----", - "", - "", - "--YFrteb74qSXmggbOxZL9dRnhymywAi--", - "", - "", - ] - ).encode() - for i in range(100): - print("Sending mail", str(i)) - smtp.conn.sendmail(user, to_addr, mail) - - -def test_exceed_rate_limit(smtp, gencreds): - """Test that the outbound rate limit is exceeded if we send a lot of messages at once.""" - user, password = gencreds() - smtp.connect() - smtp.login(user, password) - - to_addr = "foobar@example.org" - mail = "\r\n".join( - [ - "Subject: ...", - f"From: <{user}>", - f"To: <{to_addr}>", - "Date: Sun, 15 Oct 2023 16:43:21 +0000", - "Message-ID: ", - "In-Reply-To: ", - "References: ", - "\t", - "Chat-Version: 1.0", - f"Autocrypt: addr={user}; prefer-encrypt=mutual;", - "\tkeydata=xjMEZSwWjhYJKwYBBAHaRw8BAQdAQBEhqeJh0GueHB6kF/DUQqYCxARNBVokg/AzT+7LqH", - "\trNFzxiYXJiYXpAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUsFo4CGwMECwkIBwYVCAkKCwID", - "\tFgIBFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX9A4AEAnHWHp49eBCMHK5t66gYPiW", - "\tXQuB1mwUjzGfYWB+0RXUoA/0xcQ3FbUNlGKW7Blp6eMFfViv6Mv2d3kNSXACB6nmcMzjgEZSwWjhIK", - "\tKwYBBAGXVQEFAQEHQBpY5L2M1XHo0uxf8SX1wNLBp/OVvidoWHQF2Jz+kJsUAwEIB8J4BBgWCAAgBQ", - "\tJlLBaOAhsMFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX/INgEA37AJaNvruYsJVanP", - "\tIXnYw4CKd55UAwl8Zcy+M2diAbkA/0fHHcGV4r78hpbbL1Os52DPOdqYQRauIeJUeG+G6bQO", - "MIME-Version: 1.0", - 'Content-Type: multipart/encrypted; protocol="application/pgp-encrypted";', - '\tboundary="YFrteb74qSXmggbOxZL9dRnhymywAi"', - "", - "", - "--YFrteb74qSXmggbOxZL9dRnhymywAi", - "Content-Description: PGP/MIME version identification", - "Content-Type: application/pgp-encrypted", - "", - "Version: 1", - "", - "", - "--YFrteb74qSXmggbOxZL9dRnhymywAi", - "Content-Description: OpenPGP encrypted message", - 'Content-Disposition: inline; filename="encrypted.asc";', - 'Content-Type: application/octet-stream; name="encrypted.asc"', - "", - "-----BEGIN PGP MESSAGE-----", - "", - "wU4DhW3gBZ/VvCYSAQdA8bMs2spwbKdGjVsL1ByPkNrqD7frpB73maeL6I6SzDYg", - "O5G53tv339RdKq3WRcCtEEvxjHlUx2XNwXzC04BpmfvBTgNfPUyLDzjXnxIBB0Ae", - "8ymwGvXMCCimHXN0Dg8Ui62KOi03h0UgheoHWovJSCDF4CKre/xtFr3nL7lq/PKI", - "JsjVNz7/RK9FSXF6WwfONtLCyQGEuVAsB/KXfCBEyfKhaMwGHvhujRidGW5uV1no", - "lMGl3ODmo29Lgeu2uSE7EpJRZoe6hU6ddmBkqxax61ZtkaFlGFFpdo2K8balNNdz", - "ZsJ/9mmI9x3oOJ4/l1nhQbUO9ADbs7gJhFdV5Qkp30b5fCI7bU+aoe1ccBbLe/WM", - "YUty1PqcuQT7XjA+XmYuL261tvW8pBetT+i33/E2d8PzzYt2IuK9qeevyS+yxdwA", - "kfwejFWzzsUlJaDxs1x4XOxkMgSj+jo+g12dFOb7fyClsAnq23iDb8AuaT/BScAI", - "+lO+gher69+6LmM7VGHLG5k762J1jTaQCaKt1s8TAWV99Eo4491vL6fyvk3l/Cfg", - "RXSwiWFgj19Pn0Rq7CD9v22UE2vdUMBTcV4aw79mClk1YQ23jbF0y5DCjPdJ62Zo", - "tskBgFt3NoWV80jZ76zIBLrrjLwCCll8JjJtFwSkt2GX5RFBsVa4A8IDht9RtEk7", - "rrHgbSZQfkauEi/mH3/6CDZoLqSHudUZ7d4MaJwun1TkFYGe2ORwGJd4OBj3oGJp", - "H8YBwCpk///L/fKjX0Gg3M8nrpM4wrRFhPKidAgO/kcm25X4+ZHlVkWBTCt5RWKI", - "fHh6oLDZCqCfcgMkE1KKmwfIHaUkhq5BPRigwy6i5dh1DM4+1UCLh3dxzVbqE9b9", - "61NB19nXdRtDA2sOUnj9ve6m/wEPyCb6/zBQZqvCBYb1/AjdXpUrFT+DbpfyxaXN", - "XfhDVb5mNqNM/IVj0V5fvTc6vOfYbzQtPm10H+FdWWfb+rJRfyC3MA2w2IqstFe3", - "w3bu2iE6CQvSqRvge+ZqLKt/NqYwOURiUmpuklbl3kPJ97+mfKWoiqk8Iz1VY+bb", - "NMUC7aoGv+jcoj+WS6PYO8N6BeRVUUB3ZJSf8nzjgxm1/BcM+UD3BPrlhT11ODRs", - "baifGbprMWwt3dhb8cQgRT8GPdpO1OsDkzL6iikMjLHWWiA99GV6ruiHsIPw6boW", - "A6/uSOskbDHOROotKmddGTBd0iiHXAoQsJFt1ZjUkt6EHrgWs+GAvrvKpXs1mrz8", - "uj3GwEFrHS+Xuf2UDgpszYT3hI2cL/kUtGakVR7m7vVMZqXBUbZdGAEb1PZNPwsI", - "E4aMK02+EVB+tSN4Fzj99N2YD0inVYt+oPjr2tHhUS6aSGBNS/48Ki47DOg4Sxkn", - "lkOWnEbCD+XTnbDd", - "=agR5", - "-----END PGP MESSAGE-----", - "", - "", - "--YFrteb74qSXmggbOxZL9dRnhymywAi--", - "", - "", - ] - ).encode() +@pytest.mark.slow +def test_exceed_rate_limit(cmsetup, gencreds, mailgen): + """Test that the per-account send-mail limit is exceeded.""" + user1, user2 = cmsetup.gen_users(2) + mail = mailgen.get_encrypted(user1.addr, user2.addr) for i in range(100): print("Sending mail", str(i)) try: - smtp.conn.sendmail(user, to_addr, mail) + user1.smtp.sendmail(user1.addr, [user2.addr], mail) except smtplib.SMTPSenderRefused as e: - if i == 0: - pytest.fail(f"rate limit was exceeded too early with msg {i} - maybe wait a minute before testing?") - if i < 41: + if i < 80: pytest.fail(f"rate limit was exceeded too early with msg {i}") assert e.smtp_code == 450 assert b'4.7.1 Error: too much mail from' in e.smtp_error From 10cb099c0e229ab1c3d05a2faa5a838a63a7bdab Mon Sep 17 00:00:00 2001 From: holger krekel Date: Thu, 19 Oct 2023 00:07:22 +0200 Subject: [PATCH 06/13] all tests pass --- ...rmail.service => filtermail-after.service} | 4 +- .../src/chatmaild/filtermail-before.service | 10 +++++ chatmaild/src/chatmaild/filtermail.py | 45 ++++++++++++------- chatmaild/src/chatmaild/test_filtermail.py | 1 - .../src/deploy_chatmail/__init__.py | 37 +++++++-------- .../src/deploy_chatmail/postfix/master.cf.j2 | 7 ++- online-tests/test_0_login.py | 7 +-- 7 files changed, 68 insertions(+), 43 deletions(-) rename chatmaild/src/chatmaild/{filtermail.service => filtermail-after.service} (65%) create mode 100644 chatmaild/src/chatmaild/filtermail-before.service diff --git a/chatmaild/src/chatmaild/filtermail.service b/chatmaild/src/chatmaild/filtermail-after.service similarity index 65% rename from chatmaild/src/chatmaild/filtermail.service rename to chatmaild/src/chatmaild/filtermail-after.service index 0e6a9a18..ccd2772e 100644 --- a/chatmaild/src/chatmaild/filtermail.service +++ b/chatmaild/src/chatmaild/filtermail-after.service @@ -1,8 +1,8 @@ [Unit] -Description=Email filter for chatmail servers +Description=Chatmail Postfix AfterQueue filter [Service] -ExecStart=/usr/local/bin/filtermail afterqueue /var/spool/postfix/private/filtermail +ExecStart=/usr/local/bin/filtermail afterqueue /var/spool/postfix/private/filtermail-afterqueue Restart=always RestartSec=30 diff --git a/chatmaild/src/chatmaild/filtermail-before.service b/chatmaild/src/chatmaild/filtermail-before.service new file mode 100644 index 00000000..0351c0b8 --- /dev/null +++ b/chatmaild/src/chatmaild/filtermail-before.service @@ -0,0 +1,10 @@ +[Unit] +Description=Chatmail Postfix BeforeQeue filter + +[Service] +ExecStart=/usr/local/bin/filtermail beforequeue 10080 +Restart=always +RestartSec=30 + +[Install] +WantedBy=multi-user.target diff --git a/chatmaild/src/chatmaild/filtermail.py b/chatmaild/src/chatmaild/filtermail.py index 76777f51..15439bfc 100644 --- a/chatmaild/src/chatmaild/filtermail.py +++ b/chatmaild/src/chatmaild/filtermail.py @@ -9,7 +9,7 @@ from email.utils import parseaddr from aiosmtpd.lmtp import LMTP from aiosmtpd.smtp import SMTP -from aiosmtpd.controller import UnixSocketController +from aiosmtpd.controller import UnixSocketController, Controller from smtplib import SMTP as SMTPClient @@ -35,10 +35,7 @@ def check_encrypted(message): return True - class BeforeQueueHandler: - transport_class = SMTP - def __init__(self): self.send_rate_limiter = SendRateLimiter() @@ -47,7 +44,13 @@ class BeforeQueueHandler: if self.send_rate_limiter.is_sending_allowed(address): envelope.mail_from = address return "250 OK" - return "400 per-user ratelimit exceeded" + return f"450 4.7.1: Too much mail from {address}" + + async def handle_DATA(self, server, session, envelope): + logging.info("handle_DATA before-queue: re-injecting the mail") + client = SMTPClient("localhost", "10026") + client.sendmail(envelope.mail_from, envelope.rcpt_tos, envelope.content) + return "250 OK" class SendRateLimiter: @@ -67,8 +70,6 @@ class SendRateLimiter: class AfterQueueHandler: - transport_class = LMTP - async def handle_RCPT(self, server, session, envelope, address, rcpt_options): envelope.rcpt_tos.append(address) return "250 OK" @@ -77,8 +78,8 @@ class AfterQueueHandler: valid_recipients, res = lmtp_handle_DATA(envelope) # Reinject the mail back into Postfix. if valid_recipients: - logging.info("Reinjecting the mail") - client = SMTPClient("localhost", "10026") + logging.info("afterqueue: re-injecting the mail") + client = SMTPClient("localhost", "10027") client.sendmail(envelope.mail_from, valid_recipients, envelope.content) else: logging.info("no valid recipients, ignoring mail") @@ -137,25 +138,35 @@ def lmtp_handle_DATA(envelope): return valid_recipients, res -class Controller(UnixSocketController): +class UnixController(UnixSocketController): def factory(self): - return self.handler.transport_class(self.handler, **self.SMTP_kwargs) + return LMTP(self.handler, **self.SMTP_kwargs) -async def asyncmain(loop, handler, unix_socket_fn): - Controller(handler, unix_socket=unix_socket_fn).start() +class SMTPController(Controller): + def factory(self): + return SMTP(self.handler, **self.SMTP_kwargs) -name2Handler = {"beforequeue": BeforeQueueHandler, "afterqueue": AfterQueueHandler} +async def asyncmain_afterqueue(loop, unix_socket_fn): + UnixController(AfterQueueHandler(), unix_socket=unix_socket_fn).start() + + +async def asyncmain_beforequeue(loop, port): + Controller(BeforeQueueHandler(), hostname="127.0.0.1", port=port).start() def main(): args = sys.argv[1:] assert len(args) == 2 - handler = name2Handler[args[0]]() - unix_socket_fn = args[1] logging.basicConfig(level=logging.INFO) loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - loop.create_task(asyncmain(loop, handler, unix_socket_fn)) + if args[0] == "afterqueue": + task = asyncmain_afterqueue(loop, args[1]) + elif args[0] == "beforequeue": + task = asyncmain_beforequeue(loop, port=int(args[1])) + else: + raise SystemExit(1) + loop.create_task(task) loop.run_forever() diff --git a/chatmaild/src/chatmaild/test_filtermail.py b/chatmaild/src/chatmaild/test_filtermail.py index cc86cfa7..32452e94 100644 --- a/chatmaild/src/chatmaild/test_filtermail.py +++ b/chatmaild/src/chatmaild/test_filtermail.py @@ -339,4 +339,3 @@ def test_send_rate_limiter(): else: assert i == SendRateLimiter.MAX_USER_SEND_PER_MINUTE + 1 break - diff --git a/deploy-chatmail/src/deploy_chatmail/__init__.py b/deploy-chatmail/src/deploy_chatmail/__init__.py index 80df1844..5ca476ca 100644 --- a/deploy-chatmail/src/deploy_chatmail/__init__.py +++ b/deploy-chatmail/src/deploy_chatmail/__init__.py @@ -53,24 +53,25 @@ def _install_chatmaild() -> None: daemon_reload=True, ) - files.put( - name="upload filtermail.service", - src=importlib.resources.files("chatmaild") - .joinpath("filtermail.service") - .open("rb"), - dest="/etc/systemd/system/filtermail.service", - user="root", - group="root", - mode="644", - ) - systemd.service( - name="Setup filtermail service", - service="filtermail.service", - running=True, - enabled=True, - restarted=True, - daemon_reload=True, - ) + for fn in ("filtermail-after", "filtermail-before"): + files.put( + name=f"upload {fn}.service", + src=importlib.resources.files("chatmaild") + .joinpath(f"{fn}.service") + .open("rb"), + dest=f"/etc/systemd/system/{fn}.service", + user="root", + group="root", + mode="644", + ) + systemd.service( + name=f"Setup {fn} service", + service=f"{fn}.service", + running=True, + enabled=True, + restarted=True, + daemon_reload=True, + ) def _configure_opendkim(domain: str, dkim_selector: str) -> bool: diff --git a/deploy-chatmail/src/deploy_chatmail/postfix/master.cf.j2 b/deploy-chatmail/src/deploy_chatmail/postfix/master.cf.j2 index 18345dc3..1dbac100 100644 --- a/deploy-chatmail/src/deploy_chatmail/postfix/master.cf.j2 +++ b/deploy-chatmail/src/deploy_chatmail/postfix/master.cf.j2 @@ -32,7 +32,8 @@ submission inet n - y - - smtpd -o smtpd_recipient_restrictions= -o smtpd_relay_restrictions=permit_sasl_authenticated,reject -o milter_macro_daemon_name=ORIGINATING - -o content_filter=filter:unix:private/filtermail + -o smtpd_proxy_filter=127.0.0.1:10080 + -o content_filter=filter:unix:private/filtermail-afterqueue smtps inet n - y - - smtpd -o syslog_name=postfix/smtps -o smtpd_tls_wrappermode=yes @@ -47,7 +48,7 @@ smtps inet n - y - - smtpd -o smtpd_recipient_restrictions= -o smtpd_relay_restrictions=permit_sasl_authenticated,reject -o milter_macro_daemon_name=ORIGINATING - -o content_filter=filter:unix:private/filtermail + -o smtpd_proxy_filter=127.0.0.1:10080 #628 inet n - y - - qmqpd pickup unix n - y 60 1 pickup cleanup unix n - y - 0 cleanup @@ -77,4 +78,6 @@ postlog unix-dgram n - n - 1 postlogd filter unix - n n - - lmtp # Local SMTP server for reinjecting filered mail. localhost:10026 inet n - n - 10 smtpd + -o content_filter=filter:unix:private/filtermail-afterqueue +localhost:10027 inet n - n - 10 smtpd -o content_filter= diff --git a/online-tests/test_0_login.py b/online-tests/test_0_login.py index 6c397b2a..e30209c5 100644 --- a/online-tests/test_0_login.py +++ b/online-tests/test_0_login.py @@ -46,10 +46,11 @@ def test_exceed_rate_limit(cmsetup, gencreds, mailgen): print("Sending mail", str(i)) try: user1.smtp.sendmail(user1.addr, [user2.addr], mail) - except smtplib.SMTPSenderRefused as e: + except smtplib.SMTPException as e: if i < 80: pytest.fail(f"rate limit was exceeded too early with msg {i}") - assert e.smtp_code == 450 - assert b'4.7.1 Error: too much mail from' in e.smtp_error + outcome = e.recipients[user2.addr] + assert outcome[0] == 450 + assert b'4.7.1: Too much mail from' in outcome[1] return pytest.fail("Rate limit was not exceeded") From 4358d5fe61fbb852b5b48f0db546a233464c86da Mon Sep 17 00:00:00 2001 From: holger krekel Date: Thu, 19 Oct 2023 00:54:02 +0200 Subject: [PATCH 07/13] only do a smtp beforequeue-handler, also simplifies the send-rate-limiting test and improves DC behaviour --- .../src/chatmaild/filtermail-after.service | 10 -- chatmaild/src/chatmaild/filtermail.py | 102 ++++++------------ chatmaild/src/chatmaild/test_filtermail.py | 11 +- .../src/deploy_chatmail/__init__.py | 21 +--- .../src/deploy_chatmail/postfix/master.cf.j2 | 4 +- online-tests/test_0_basic.py | 34 +++--- online-tests/test_0_login.py | 19 ---- 7 files changed, 58 insertions(+), 143 deletions(-) delete mode 100644 chatmaild/src/chatmaild/filtermail-after.service diff --git a/chatmaild/src/chatmaild/filtermail-after.service b/chatmaild/src/chatmaild/filtermail-after.service deleted file mode 100644 index ccd2772e..00000000 --- a/chatmaild/src/chatmaild/filtermail-after.service +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=Chatmail Postfix AfterQueue filter - -[Service] -ExecStart=/usr/local/bin/filtermail afterqueue /var/spool/postfix/private/filtermail-afterqueue -Restart=always -RestartSec=30 - -[Install] -WantedBy=multi-user.target diff --git a/chatmaild/src/chatmaild/filtermail.py b/chatmaild/src/chatmaild/filtermail.py index 15439bfc..389be15a 100644 --- a/chatmaild/src/chatmaild/filtermail.py +++ b/chatmaild/src/chatmaild/filtermail.py @@ -41,14 +41,23 @@ class BeforeQueueHandler: async def handle_MAIL(self, server, session, envelope, address, mail_options): logging.info(f"handle_MAIL from {address}") - if self.send_rate_limiter.is_sending_allowed(address): - envelope.mail_from = address - return "250 OK" - return f"450 4.7.1: Too much mail from {address}" + envelope.mail_from = address + if not self.send_rate_limiter.is_sending_allowed(address): + return f"450 4.7.1: Too much mail from {address}" + + parts = envelope.mail_from.split("@") + if len(parts) != 2: + return f"500 Invalid from address <{envelope.mail_from!r}>" + + return "250 OK" async def handle_DATA(self, server, session, envelope): - logging.info("handle_DATA before-queue: re-injecting the mail") - client = SMTPClient("localhost", "10026") + logging.info("handle_DATA before-queue") + error = check_DATA(envelope) + if error: + return error + logging.info("re-injecting the mail that passed checks") + client = SMTPClient("localhost", "10025") client.sendmail(envelope.mail_from, envelope.rcpt_tos, envelope.content) return "250 OK" @@ -69,78 +78,33 @@ class SendRateLimiter: return False -class AfterQueueHandler: - async def handle_RCPT(self, server, session, envelope, address, rcpt_options): - envelope.rcpt_tos.append(address) - return "250 OK" - - async def handle_DATA(self, server, session, envelope): - valid_recipients, res = lmtp_handle_DATA(envelope) - # Reinject the mail back into Postfix. - if valid_recipients: - logging.info("afterqueue: re-injecting the mail") - client = SMTPClient("localhost", "10027") - client.sendmail(envelope.mail_from, valid_recipients, envelope.content) - else: - logging.info("no valid recipients, ignoring mail") - - return "\r\n".join(res) - - -def lmtp_handle_DATA(envelope): +def check_DATA(envelope): """the central filtering function for e-mails.""" logging.info(f"Processing DATA message from {envelope.mail_from}") message = BytesParser(policy=policy.default).parsebytes(envelope.content) mail_encrypted = check_encrypted(message) - valid_recipients = [] - res = [] + _, from_addr = parseaddr(message.get("from").strip()) + logging.info(f"mime-from: {from_addr} envelope-from: {envelope.mail_from!r}") + if envelope.mail_from.lower() != from_addr.lower(): + return f"500 Invalid FROM <{from_addr!r}> for <{envelope.mail_from!r}>" + + envelope_from_domain = from_addr.split("@").pop() for recipient in envelope.rcpt_tos: - my_local_domain = envelope.mail_from.split("@") - if len(my_local_domain) != 2: - res += [f"500 Invalid from address <{envelope.mail_from}>"] - continue - - _, from_addr = parseaddr(message.get("from").strip()) - logging.info(f"mime-from: {from_addr} envelope-from: {envelope.mail_from}") - if envelope.mail_from.lower() != from_addr.lower(): - res += [f"500 Invalid FROM <{from_addr}> for <{envelope.mail_from}>"] - continue - if envelope.mail_from == recipient: # Always allow sending emails to self. - valid_recipients += [recipient] - res += ["250 OK"] continue + res = recipient.split("@") + if len(res) != 2: + return f"500 Invalid address <{recipient}>" + _recipient_addr, recipient_domain = res - recipient_local_domain = recipient.split("@") - if len(recipient_local_domain) != 2: - res += [f"500 Invalid address <{recipient}>"] - continue - - is_outgoing = recipient_local_domain[1] != my_local_domain[1] - - if ( - is_outgoing - and not mail_encrypted - and message.get("secure-join") != "vc-request" - and message.get("secure-join") != "vg-request" - ): - res += ["500 Outgoing mail must be encrypted"] - continue - - valid_recipients += [recipient] - res += ["250 OK"] - - assert len(envelope.rcpt_tos) == len(res) - assert len(valid_recipients) <= len(res) - return valid_recipients, res - - -class UnixController(UnixSocketController): - def factory(self): - return LMTP(self.handler, **self.SMTP_kwargs) + is_outgoing = recipient_domain != envelope_from_domain + if is_outgoing and not mail_encrypted: + is_securejoin = message.get("secure-join") in ["vc-request", "vg-request"] + if not is_securejoin: + return f"500 Invalid unencrypted mail to <{recipient}>" class SMTPController(Controller): @@ -148,10 +112,6 @@ class SMTPController(Controller): return SMTP(self.handler, **self.SMTP_kwargs) -async def asyncmain_afterqueue(loop, unix_socket_fn): - UnixController(AfterQueueHandler(), unix_socket=unix_socket_fn).start() - - async def asyncmain_beforequeue(loop, port): Controller(BeforeQueueHandler(), hostname="127.0.0.1", port=port).start() diff --git a/chatmaild/src/chatmaild/test_filtermail.py b/chatmaild/src/chatmaild/test_filtermail.py index 32452e94..df737dc2 100644 --- a/chatmaild/src/chatmaild/test_filtermail.py +++ b/chatmaild/src/chatmaild/test_filtermail.py @@ -1,4 +1,4 @@ -from .filtermail import check_encrypted, lmtp_handle_DATA, SendRateLimiter +from .filtermail import check_encrypted, check_DATA, SendRateLimiter from email.parser import BytesParser from email import policy import pytest @@ -31,15 +31,12 @@ def test_reject_forged_from(): # test that the filter lets good mail through envelope.content = makemail(envelope.mail_from).as_bytes() - valid_recipients, res = lmtp_handle_DATA(envelope=envelope) - assert valid_recipients == envelope.rcpt_tos - assert len(res) == 1 and "250" in res[0] + assert not check_DATA(envelope=envelope) # test that the filter rejects forged mail envelope.content = makemail("forged@c3.testrun.org").as_bytes() - valid_recipients, res = lmtp_handle_DATA(envelope=envelope) - assert not valid_recipients - assert len(res) == 1 and "500" in res[0] + error = check_DATA(envelope=envelope) + assert "500" in error def test_filtermail(): diff --git a/deploy-chatmail/src/deploy_chatmail/__init__.py b/deploy-chatmail/src/deploy_chatmail/__init__.py index 5ca476ca..e152d925 100644 --- a/deploy-chatmail/src/deploy_chatmail/__init__.py +++ b/deploy-chatmail/src/deploy_chatmail/__init__.py @@ -34,26 +34,7 @@ def _install_chatmaild() -> None: commands=[f"pip install --break-system-packages {remote_path}"], ) - files.put( - name="upload doveauth-dictproxy.service", - src=importlib.resources.files("chatmaild") - .joinpath("doveauth-dictproxy.service") - .open("rb"), - dest="/etc/systemd/system/doveauth-dictproxy.service", - user="root", - group="root", - mode="644", - ) - systemd.service( - name="Setup doveauth-dictproxy service", - service="doveauth-dictproxy.service", - running=True, - enabled=True, - restarted=True, - daemon_reload=True, - ) - - for fn in ("filtermail-after", "filtermail-before"): + for fn in ("doveauth-dictproxy", "filtermail-before", ): files.put( name=f"upload {fn}.service", src=importlib.resources.files("chatmaild") diff --git a/deploy-chatmail/src/deploy_chatmail/postfix/master.cf.j2 b/deploy-chatmail/src/deploy_chatmail/postfix/master.cf.j2 index 1dbac100..34c0efe9 100644 --- a/deploy-chatmail/src/deploy_chatmail/postfix/master.cf.j2 +++ b/deploy-chatmail/src/deploy_chatmail/postfix/master.cf.j2 @@ -77,7 +77,5 @@ scache unix - - y - 1 scache postlog unix-dgram n - n - 1 postlogd filter unix - n n - - lmtp # Local SMTP server for reinjecting filered mail. -localhost:10026 inet n - n - 10 smtpd - -o content_filter=filter:unix:private/filtermail-afterqueue -localhost:10027 inet n - n - 10 smtpd +localhost:10025 inet n - n - 10 smtpd -o content_filter= diff --git a/online-tests/test_0_basic.py b/online-tests/test_0_basic.py index e8ff84fb..d4edc4c4 100644 --- a/online-tests/test_0_basic.py +++ b/online-tests/test_0_basic.py @@ -20,7 +20,7 @@ def test_use_two_chatmailservers(cmfactory, maildomain2): @pytest.mark.parametrize("forgeaddr", ["internal", "someone@example.org"]) -def test_reject_forged_from(cmsetup, mailgen, lp, remote, forgeaddr): +def test_reject_forged_from(cmsetup, mailgen, lp, forgeaddr): user1, user3 = cmsetup.gen_users(2) lp.sec("send encrypted message with forged from") @@ -36,17 +36,25 @@ def test_reject_forged_from(cmsetup, mailgen, lp, remote, forgeaddr): print(f" {line}") lp.sec("Send forged mail and check remote postfix lmtp processing result") - remote_log = remote.iter_output("journalctl -t postfix/lmtp") - user1.smtp.sendmail(from_addr=user1.addr, to_addrs=[user3.addr], msg=msg) - for line in remote_log: - # print(line) - if "500 invalid from" in line and user3.addr in line: - break - else: - pytest.fail("remote postfix/filtermail failed to reject message") + with pytest.raises(smtplib.SMTPException) as e: + user1.smtp.sendmail(from_addr=user1.addr, to_addrs=[user3.addr], msg=msg) + assert "500" in str(e.value) - # check that the logged in user (who sent the forged msg) got a non-delivery notice - for message in user1.imap.fetch_all_messages(): - if "Invalid FROM" in message and addr_to_forge in message: + +@pytest.mark.slow +def test_exceed_rate_limit(cmsetup, gencreds, mailgen): + """Test that the per-account send-mail limit is exceeded.""" + user1, user2 = cmsetup.gen_users(2) + mail = mailgen.get_encrypted(user1.addr, user2.addr) + for i in range(100): + print("Sending mail", str(i)) + try: + user1.smtp.sendmail(user1.addr, [user2.addr], mail) + except smtplib.SMTPException as e: + if i < 80: + pytest.fail(f"rate limit was exceeded too early with msg {i}") + outcome = e.recipients[user2.addr] + assert outcome[0] == 450 + assert b'4.7.1: Too much mail from' in outcome[1] return - pytest.fail(f"forged From={addr_to_forge} did not cause non-delivery notice") + pytest.fail("Rate limit was not exceeded") diff --git a/online-tests/test_0_login.py b/online-tests/test_0_login.py index e30209c5..f0e4a3a7 100644 --- a/online-tests/test_0_login.py +++ b/online-tests/test_0_login.py @@ -35,22 +35,3 @@ def test_login_same_password(imap_or_smtp, gencreds): imap_or_smtp.login(user1, password1) imap_or_smtp.connect() imap_or_smtp.login(user2, password1) - - -@pytest.mark.slow -def test_exceed_rate_limit(cmsetup, gencreds, mailgen): - """Test that the per-account send-mail limit is exceeded.""" - user1, user2 = cmsetup.gen_users(2) - mail = mailgen.get_encrypted(user1.addr, user2.addr) - for i in range(100): - print("Sending mail", str(i)) - try: - user1.smtp.sendmail(user1.addr, [user2.addr], mail) - except smtplib.SMTPException as e: - if i < 80: - pytest.fail(f"rate limit was exceeded too early with msg {i}") - outcome = e.recipients[user2.addr] - assert outcome[0] == 450 - assert b'4.7.1: Too much mail from' in outcome[1] - return - pytest.fail("Rate limit was not exceeded") From 7cf6cc2c9176b43ea7278cbcfc7b6dd8e2ce07fe Mon Sep 17 00:00:00 2001 From: holger krekel Date: Thu, 19 Oct 2023 01:02:24 +0200 Subject: [PATCH 08/13] remove filtermail split and LMTP backend --- chatmaild/src/chatmaild/filtermail.py | 16 +++++----------- ...termail-before.service => filtermail.service} | 2 +- deploy-chatmail/src/deploy_chatmail/__init__.py | 7 +++++-- .../src/deploy_chatmail/postfix/master.cf.j2 | 1 - 4 files changed, 11 insertions(+), 15 deletions(-) rename chatmaild/src/chatmaild/{filtermail-before.service => filtermail.service} (71%) diff --git a/chatmaild/src/chatmaild/filtermail.py b/chatmaild/src/chatmaild/filtermail.py index 389be15a..859b628d 100644 --- a/chatmaild/src/chatmaild/filtermail.py +++ b/chatmaild/src/chatmaild/filtermail.py @@ -7,9 +7,8 @@ from email.parser import BytesParser from email import policy from email.utils import parseaddr -from aiosmtpd.lmtp import LMTP -from aiosmtpd.smtp import SMTP -from aiosmtpd.controller import UnixSocketController, Controller +from aiosmtpd.lmtp import SMTP +from aiosmtpd.controller import Controller from smtplib import SMTP as SMTPClient @@ -112,21 +111,16 @@ class SMTPController(Controller): return SMTP(self.handler, **self.SMTP_kwargs) -async def asyncmain_beforequeue(loop, port): +async def asyncmain_beforequeue(port): Controller(BeforeQueueHandler(), hostname="127.0.0.1", port=port).start() def main(): args = sys.argv[1:] - assert len(args) == 2 + assert len(args) == 1 logging.basicConfig(level=logging.INFO) loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - if args[0] == "afterqueue": - task = asyncmain_afterqueue(loop, args[1]) - elif args[0] == "beforequeue": - task = asyncmain_beforequeue(loop, port=int(args[1])) - else: - raise SystemExit(1) + task = asyncmain_beforequeue(port=int(args[1])) loop.create_task(task) loop.run_forever() diff --git a/chatmaild/src/chatmaild/filtermail-before.service b/chatmaild/src/chatmaild/filtermail.service similarity index 71% rename from chatmaild/src/chatmaild/filtermail-before.service rename to chatmaild/src/chatmaild/filtermail.service index 0351c0b8..fec5ba4f 100644 --- a/chatmaild/src/chatmaild/filtermail-before.service +++ b/chatmaild/src/chatmaild/filtermail.service @@ -2,7 +2,7 @@ Description=Chatmail Postfix BeforeQeue filter [Service] -ExecStart=/usr/local/bin/filtermail beforequeue 10080 +ExecStart=/usr/local/bin/filtermail 10080 Restart=always RestartSec=30 diff --git a/deploy-chatmail/src/deploy_chatmail/__init__.py b/deploy-chatmail/src/deploy_chatmail/__init__.py index e152d925..a86bd3f1 100644 --- a/deploy-chatmail/src/deploy_chatmail/__init__.py +++ b/deploy-chatmail/src/deploy_chatmail/__init__.py @@ -34,9 +34,12 @@ def _install_chatmaild() -> None: commands=[f"pip install --break-system-packages {remote_path}"], ) - for fn in ("doveauth-dictproxy", "filtermail-before", ): + for fn in ( + "doveauth-dictproxy", + "filtermail", + ): files.put( - name=f"upload {fn}.service", + name=f"Upload {fn}.service", src=importlib.resources.files("chatmaild") .joinpath(f"{fn}.service") .open("rb"), diff --git a/deploy-chatmail/src/deploy_chatmail/postfix/master.cf.j2 b/deploy-chatmail/src/deploy_chatmail/postfix/master.cf.j2 index 34c0efe9..04d03b9f 100644 --- a/deploy-chatmail/src/deploy_chatmail/postfix/master.cf.j2 +++ b/deploy-chatmail/src/deploy_chatmail/postfix/master.cf.j2 @@ -33,7 +33,6 @@ submission inet n - y - - smtpd -o smtpd_relay_restrictions=permit_sasl_authenticated,reject -o milter_macro_daemon_name=ORIGINATING -o smtpd_proxy_filter=127.0.0.1:10080 - -o content_filter=filter:unix:private/filtermail-afterqueue smtps inet n - y - - smtpd -o syslog_name=postfix/smtps -o smtpd_tls_wrappermode=yes From fb2ea274776b297cf4f21ab143ab9e8e1027532a Mon Sep 17 00:00:00 2001 From: holger krekel Date: Thu, 19 Oct 2023 01:10:06 +0200 Subject: [PATCH 09/13] open a persistent client between the BeforeQueueHandler and postfix smtpd without content filter --- chatmaild/src/chatmaild/filtermail.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chatmaild/src/chatmaild/filtermail.py b/chatmaild/src/chatmaild/filtermail.py index 859b628d..02331758 100644 --- a/chatmaild/src/chatmaild/filtermail.py +++ b/chatmaild/src/chatmaild/filtermail.py @@ -37,6 +37,7 @@ def check_encrypted(message): class BeforeQueueHandler: def __init__(self): self.send_rate_limiter = SendRateLimiter() + self.smtp = SMTPClient("localhost", "10025") async def handle_MAIL(self, server, session, envelope, address, mail_options): logging.info(f"handle_MAIL from {address}") @@ -56,8 +57,7 @@ class BeforeQueueHandler: if error: return error logging.info("re-injecting the mail that passed checks") - client = SMTPClient("localhost", "10025") - client.sendmail(envelope.mail_from, envelope.rcpt_tos, envelope.content) + self.smtp.sendmail(envelope.mail_from, envelope.rcpt_tos, envelope.content) return "250 OK" From c7995356b9e3086afceeeef872a2e733b99527c7 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Thu, 19 Oct 2023 01:14:19 +0200 Subject: [PATCH 10/13] shift for simpler diff --- chatmaild/src/chatmaild/filtermail.py | 38 +++++++++++++-------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/chatmaild/src/chatmaild/filtermail.py b/chatmaild/src/chatmaild/filtermail.py index 02331758..69ee7221 100644 --- a/chatmaild/src/chatmaild/filtermail.py +++ b/chatmaild/src/chatmaild/filtermail.py @@ -34,6 +34,11 @@ def check_encrypted(message): return True +class SMTPController(Controller): + def factory(self): + return SMTP(self.handler, **self.SMTP_kwargs) + + class BeforeQueueHandler: def __init__(self): self.send_rate_limiter = SendRateLimiter() @@ -61,20 +66,8 @@ class BeforeQueueHandler: return "250 OK" -class SendRateLimiter: - MAX_USER_SEND_PER_MINUTE = 80 - - def __init__(self): - self.addr2timestamps = {} - - def is_sending_allowed(self, mail_from): - last = self.addr2timestamps.setdefault(mail_from, []) - now = time.time() - last[:] = [ts for ts in last if ts >= (now - 60)] - if len(last) <= self.MAX_USER_SEND_PER_MINUTE: - last.append(now) - return True - return False +async def asyncmain_beforequeue(port): + Controller(BeforeQueueHandler(), hostname="127.0.0.1", port=port).start() def check_DATA(envelope): @@ -106,13 +99,20 @@ def check_DATA(envelope): return f"500 Invalid unencrypted mail to <{recipient}>" -class SMTPController(Controller): - def factory(self): - return SMTP(self.handler, **self.SMTP_kwargs) +class SendRateLimiter: + MAX_USER_SEND_PER_MINUTE = 80 + def __init__(self): + self.addr2timestamps = {} -async def asyncmain_beforequeue(port): - Controller(BeforeQueueHandler(), hostname="127.0.0.1", port=port).start() + def is_sending_allowed(self, mail_from): + last = self.addr2timestamps.setdefault(mail_from, []) + now = time.time() + last[:] = [ts for ts in last if ts >= (now - 60)] + if len(last) <= self.MAX_USER_SEND_PER_MINUTE: + last.append(now) + return True + return False def main(): From c514fb00a39ee4bb1693feb795221f2e7aa0c979 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 19 Oct 2023 02:22:15 +0000 Subject: [PATCH 11/13] Import SMTP from aiosmtpd.lmtp, not aiosmtpd.smtp --- chatmaild/src/chatmaild/filtermail.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chatmaild/src/chatmaild/filtermail.py b/chatmaild/src/chatmaild/filtermail.py index 69ee7221..0ba460c6 100644 --- a/chatmaild/src/chatmaild/filtermail.py +++ b/chatmaild/src/chatmaild/filtermail.py @@ -7,7 +7,7 @@ from email.parser import BytesParser from email import policy from email.utils import parseaddr -from aiosmtpd.lmtp import SMTP +from aiosmtpd.smtp import SMTP from aiosmtpd.controller import Controller from smtplib import SMTP as SMTPClient From 30680cb170007acefcf41df37a139c06c5f05f30 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 19 Oct 2023 02:22:30 +0000 Subject: [PATCH 12/13] filtermail: port is args[0], not args[1] --- chatmaild/src/chatmaild/filtermail.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chatmaild/src/chatmaild/filtermail.py b/chatmaild/src/chatmaild/filtermail.py index 0ba460c6..aac8e9e1 100644 --- a/chatmaild/src/chatmaild/filtermail.py +++ b/chatmaild/src/chatmaild/filtermail.py @@ -121,6 +121,6 @@ def main(): logging.basicConfig(level=logging.INFO) loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - task = asyncmain_beforequeue(port=int(args[1])) + task = asyncmain_beforequeue(port=int(args[0])) loop.create_task(task) loop.run_forever() From 1cdc5d1351b80cb0a41eb19a76908d5e59248377 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 19 Oct 2023 02:22:38 +0000 Subject: [PATCH 13/13] Revert "open a persistent client between the BeforeQueueHandler and postfix smtpd without content filter" This reverts commit fb2ea274776b297cf4f21ab143ab9e8e1027532a. --- chatmaild/src/chatmaild/filtermail.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chatmaild/src/chatmaild/filtermail.py b/chatmaild/src/chatmaild/filtermail.py index aac8e9e1..79fe1b66 100644 --- a/chatmaild/src/chatmaild/filtermail.py +++ b/chatmaild/src/chatmaild/filtermail.py @@ -42,7 +42,6 @@ class SMTPController(Controller): class BeforeQueueHandler: def __init__(self): self.send_rate_limiter = SendRateLimiter() - self.smtp = SMTPClient("localhost", "10025") async def handle_MAIL(self, server, session, envelope, address, mail_options): logging.info(f"handle_MAIL from {address}") @@ -62,7 +61,8 @@ class BeforeQueueHandler: if error: return error logging.info("re-injecting the mail that passed checks") - self.smtp.sendmail(envelope.mail_from, envelope.rcpt_tos, envelope.content) + client = SMTPClient("localhost", "10025") + client.sendmail(envelope.mail_from, envelope.rcpt_tos, envelope.content) return "250 OK"