diff --git a/chatmaild/src/chatmaild/filtermail.py b/chatmaild/src/chatmaild/filtermail.py index 0a79a97f..76777f51 100644 --- a/chatmaild/src/chatmaild/filtermail.py +++ b/chatmaild/src/chatmaild/filtermail.py @@ -1,11 +1,14 @@ #!/usr/bin/env python3 import asyncio import logging +import time +import sys from email.parser import BytesParser from email import policy from email.utils import parseaddr from aiosmtpd.lmtp import LMTP +from aiosmtpd.smtp import SMTP from aiosmtpd.controller import UnixSocketController from smtplib import SMTP as SMTPClient @@ -32,12 +35,40 @@ def check_encrypted(message): return True -class ExampleController(UnixSocketController): - def factory(self): - return LMTP(self.handler, **self.SMTP_kwargs) + +class BeforeQueueHandler: + transport_class = SMTP + + def __init__(self): + self.send_rate_limiter = SendRateLimiter() + + async def handle_MAIL(self, server, session, envelope, address, mail_options): + logging.info(f"handle_MAIL from {address}") + if self.send_rate_limiter.is_sending_allowed(address): + envelope.mail_from = address + return "250 OK" + return "400 per-user ratelimit exceeded" -class ExampleHandler: +class SendRateLimiter: + MAX_USER_SEND_PER_MINUTE = 80 + + def __init__(self): + self.addr2timestamps = {} + + def is_sending_allowed(self, mail_from): + last = self.addr2timestamps.setdefault(mail_from, []) + now = time.time() + last[:] = [ts for ts in last if ts >= (now - 60)] + if len(last) <= self.MAX_USER_SEND_PER_MINUTE: + last.append(now) + return True + return False + + +class AfterQueueHandler: + transport_class = LMTP + async def handle_RCPT(self, server, session, envelope, address, rcpt_options): envelope.rcpt_tos.append(address) return "250 OK" @@ -55,13 +86,6 @@ class ExampleHandler: return "\r\n".join(res) -async def asyncmain(loop): - controller = ExampleController( - ExampleHandler(), unix_socket="/var/spool/postfix/private/filtermail" - ) - controller.start() - - def lmtp_handle_DATA(envelope): """the central filtering function for e-mails.""" logging.info(f"Processing DATA message from {envelope.mail_from}") @@ -113,13 +137,25 @@ def lmtp_handle_DATA(envelope): return valid_recipients, res +class Controller(UnixSocketController): + def factory(self): + return self.handler.transport_class(self.handler, **self.SMTP_kwargs) + + +async def asyncmain(loop, handler, unix_socket_fn): + Controller(handler, unix_socket=unix_socket_fn).start() + + +name2Handler = {"beforequeue": BeforeQueueHandler, "afterqueue": AfterQueueHandler} + + def main(): + args = sys.argv[1:] + assert len(args) == 2 + handler = name2Handler[args[0]]() + unix_socket_fn = args[1] logging.basicConfig(level=logging.INFO) loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - loop.create_task(asyncmain(loop=loop)) + loop.create_task(asyncmain(loop, handler, unix_socket_fn)) loop.run_forever() - - -if __name__ == "__main__": - main() diff --git a/chatmaild/src/chatmaild/filtermail.service b/chatmaild/src/chatmaild/filtermail.service index b5953854..0e6a9a18 100644 --- a/chatmaild/src/chatmaild/filtermail.service +++ b/chatmaild/src/chatmaild/filtermail.service @@ -2,7 +2,7 @@ Description=Email filter for chatmail servers [Service] -ExecStart=/usr/local/bin/filtermail +ExecStart=/usr/local/bin/filtermail afterqueue /var/spool/postfix/private/filtermail Restart=always RestartSec=30 diff --git a/chatmaild/src/chatmaild/test_filtermail.py b/chatmaild/src/chatmaild/test_filtermail.py index 5e76f774..cc86cfa7 100644 --- a/chatmaild/src/chatmaild/test_filtermail.py +++ b/chatmaild/src/chatmaild/test_filtermail.py @@ -1,6 +1,7 @@ -from .filtermail import check_encrypted, lmtp_handle_DATA +from .filtermail import check_encrypted, lmtp_handle_DATA, SendRateLimiter from email.parser import BytesParser from email import policy +import pytest def test_reject_forged_from(): @@ -326,3 +327,16 @@ def test_filtermail(): ] ).encode() ) + + +def test_send_rate_limiter(): + limiter = SendRateLimiter() + for i in range(100): + if limiter.is_sending_allowed("some@example.org"): + if i <= SendRateLimiter.MAX_USER_SEND_PER_MINUTE: + continue + pytest.fail("limiter didn't work") + else: + assert i == SendRateLimiter.MAX_USER_SEND_PER_MINUTE + 1 + break + diff --git a/deploy-chatmail/src/deploy_chatmail/postfix/main.cf.j2 b/deploy-chatmail/src/deploy_chatmail/postfix/main.cf.j2 index 975a5241..303aed13 100644 --- a/deploy-chatmail/src/deploy_chatmail/postfix/main.cf.j2 +++ b/deploy-chatmail/src/deploy_chatmail/postfix/main.cf.j2 @@ -29,9 +29,6 @@ myhostname = {{ config.domain_name }} alias_maps = hash:/etc/aliases alias_database = hash:/etc/aliases -# hard limit, also on internal messages -smtpd_client_message_rate_limit = 80 - # Postfix does not deliver mail for any domain by itself. # Primary domain is listed in `virtual_mailbox_domains` instead # and handed over to Dovecot. diff --git a/online-tests/test_0_login.py b/online-tests/test_0_login.py index 919c9936..6c397b2a 100644 --- a/online-tests/test_0_login.py +++ b/online-tests/test_0_login.py @@ -37,174 +37,17 @@ def test_login_same_password(imap_or_smtp, gencreds): imap_or_smtp.login(user2, password1) -@pytest.mark.xfail(reason="Only rate limit is internal as well now") -def test_no_internal_rate_limit(smtp, gencreds): - """Test that there is no rate limit between accounts on the same chatmail server.""" - user, password = gencreds() - to_addr, = gencreds() - smtp.connect() - smtp.login(user, password) - - mail = "\r\n".join( - [ - "Subject: ...", - f"From: <{user}>", - f"To: <{to_addr}>", - "Date: Sun, 15 Oct 2023 16:43:21 +0000", - "Message-ID: ", - "In-Reply-To: ", - "References: ", - "\t", - "Chat-Version: 1.0", - f"Autocrypt: addr={user}; prefer-encrypt=mutual;", - "\tkeydata=xjMEZSwWjhYJKwYBBAHaRw8BAQdAQBEhqeJh0GueHB6kF/DUQqYCxARNBVokg/AzT+7LqH", - "\trNFzxiYXJiYXpAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUsFo4CGwMECwkIBwYVCAkKCwID", - "\tFgIBFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX9A4AEAnHWHp49eBCMHK5t66gYPiW", - "\tXQuB1mwUjzGfYWB+0RXUoA/0xcQ3FbUNlGKW7Blp6eMFfViv6Mv2d3kNSXACB6nmcMzjgEZSwWjhIK", - "\tKwYBBAGXVQEFAQEHQBpY5L2M1XHo0uxf8SX1wNLBp/OVvidoWHQF2Jz+kJsUAwEIB8J4BBgWCAAgBQ", - "\tJlLBaOAhsMFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX/INgEA37AJaNvruYsJVanP", - "\tIXnYw4CKd55UAwl8Zcy+M2diAbkA/0fHHcGV4r78hpbbL1Os52DPOdqYQRauIeJUeG+G6bQO", - "MIME-Version: 1.0", - 'Content-Type: multipart/encrypted; protocol="application/pgp-encrypted";', - '\tboundary="YFrteb74qSXmggbOxZL9dRnhymywAi"', - "", - "", - "--YFrteb74qSXmggbOxZL9dRnhymywAi", - "Content-Description: PGP/MIME version identification", - "Content-Type: application/pgp-encrypted", - "", - "Version: 1", - "", - "", - "--YFrteb74qSXmggbOxZL9dRnhymywAi", - "Content-Description: OpenPGP encrypted message", - 'Content-Disposition: inline; filename="encrypted.asc";', - 'Content-Type: application/octet-stream; name="encrypted.asc"', - "", - "-----BEGIN PGP MESSAGE-----", - "", - "wU4DhW3gBZ/VvCYSAQdA8bMs2spwbKdGjVsL1ByPkNrqD7frpB73maeL6I6SzDYg", - "O5G53tv339RdKq3WRcCtEEvxjHlUx2XNwXzC04BpmfvBTgNfPUyLDzjXnxIBB0Ae", - "8ymwGvXMCCimHXN0Dg8Ui62KOi03h0UgheoHWovJSCDF4CKre/xtFr3nL7lq/PKI", - "JsjVNz7/RK9FSXF6WwfONtLCyQGEuVAsB/KXfCBEyfKhaMwGHvhujRidGW5uV1no", - "lMGl3ODmo29Lgeu2uSE7EpJRZoe6hU6ddmBkqxax61ZtkaFlGFFpdo2K8balNNdz", - "ZsJ/9mmI9x3oOJ4/l1nhQbUO9ADbs7gJhFdV5Qkp30b5fCI7bU+aoe1ccBbLe/WM", - "YUty1PqcuQT7XjA+XmYuL261tvW8pBetT+i33/E2d8PzzYt2IuK9qeevyS+yxdwA", - "kfwejFWzzsUlJaDxs1x4XOxkMgSj+jo+g12dFOb7fyClsAnq23iDb8AuaT/BScAI", - "+lO+gher69+6LmM7VGHLG5k762J1jTaQCaKt1s8TAWV99Eo4491vL6fyvk3l/Cfg", - "RXSwiWFgj19Pn0Rq7CD9v22UE2vdUMBTcV4aw79mClk1YQ23jbF0y5DCjPdJ62Zo", - "tskBgFt3NoWV80jZ76zIBLrrjLwCCll8JjJtFwSkt2GX5RFBsVa4A8IDht9RtEk7", - "rrHgbSZQfkauEi/mH3/6CDZoLqSHudUZ7d4MaJwun1TkFYGe2ORwGJd4OBj3oGJp", - "H8YBwCpk///L/fKjX0Gg3M8nrpM4wrRFhPKidAgO/kcm25X4+ZHlVkWBTCt5RWKI", - "fHh6oLDZCqCfcgMkE1KKmwfIHaUkhq5BPRigwy6i5dh1DM4+1UCLh3dxzVbqE9b9", - "61NB19nXdRtDA2sOUnj9ve6m/wEPyCb6/zBQZqvCBYb1/AjdXpUrFT+DbpfyxaXN", - "XfhDVb5mNqNM/IVj0V5fvTc6vOfYbzQtPm10H+FdWWfb+rJRfyC3MA2w2IqstFe3", - "w3bu2iE6CQvSqRvge+ZqLKt/NqYwOURiUmpuklbl3kPJ97+mfKWoiqk8Iz1VY+bb", - "NMUC7aoGv+jcoj+WS6PYO8N6BeRVUUB3ZJSf8nzjgxm1/BcM+UD3BPrlhT11ODRs", - "baifGbprMWwt3dhb8cQgRT8GPdpO1OsDkzL6iikMjLHWWiA99GV6ruiHsIPw6boW", - "A6/uSOskbDHOROotKmddGTBd0iiHXAoQsJFt1ZjUkt6EHrgWs+GAvrvKpXs1mrz8", - "uj3GwEFrHS+Xuf2UDgpszYT3hI2cL/kUtGakVR7m7vVMZqXBUbZdGAEb1PZNPwsI", - "E4aMK02+EVB+tSN4Fzj99N2YD0inVYt+oPjr2tHhUS6aSGBNS/48Ki47DOg4Sxkn", - "lkOWnEbCD+XTnbDd", - "=agR5", - "-----END PGP MESSAGE-----", - "", - "", - "--YFrteb74qSXmggbOxZL9dRnhymywAi--", - "", - "", - ] - ).encode() - for i in range(100): - print("Sending mail", str(i)) - smtp.conn.sendmail(user, to_addr, mail) - - -def test_exceed_rate_limit(smtp, gencreds): - """Test that the outbound rate limit is exceeded if we send a lot of messages at once.""" - user, password = gencreds() - smtp.connect() - smtp.login(user, password) - - to_addr = "foobar@example.org" - mail = "\r\n".join( - [ - "Subject: ...", - f"From: <{user}>", - f"To: <{to_addr}>", - "Date: Sun, 15 Oct 2023 16:43:21 +0000", - "Message-ID: ", - "In-Reply-To: ", - "References: ", - "\t", - "Chat-Version: 1.0", - f"Autocrypt: addr={user}; prefer-encrypt=mutual;", - "\tkeydata=xjMEZSwWjhYJKwYBBAHaRw8BAQdAQBEhqeJh0GueHB6kF/DUQqYCxARNBVokg/AzT+7LqH", - "\trNFzxiYXJiYXpAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUsFo4CGwMECwkIBwYVCAkKCwID", - "\tFgIBFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX9A4AEAnHWHp49eBCMHK5t66gYPiW", - "\tXQuB1mwUjzGfYWB+0RXUoA/0xcQ3FbUNlGKW7Blp6eMFfViv6Mv2d3kNSXACB6nmcMzjgEZSwWjhIK", - "\tKwYBBAGXVQEFAQEHQBpY5L2M1XHo0uxf8SX1wNLBp/OVvidoWHQF2Jz+kJsUAwEIB8J4BBgWCAAgBQ", - "\tJlLBaOAhsMFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX/INgEA37AJaNvruYsJVanP", - "\tIXnYw4CKd55UAwl8Zcy+M2diAbkA/0fHHcGV4r78hpbbL1Os52DPOdqYQRauIeJUeG+G6bQO", - "MIME-Version: 1.0", - 'Content-Type: multipart/encrypted; protocol="application/pgp-encrypted";', - '\tboundary="YFrteb74qSXmggbOxZL9dRnhymywAi"', - "", - "", - "--YFrteb74qSXmggbOxZL9dRnhymywAi", - "Content-Description: PGP/MIME version identification", - "Content-Type: application/pgp-encrypted", - "", - "Version: 1", - "", - "", - "--YFrteb74qSXmggbOxZL9dRnhymywAi", - "Content-Description: OpenPGP encrypted message", - 'Content-Disposition: inline; filename="encrypted.asc";', - 'Content-Type: application/octet-stream; name="encrypted.asc"', - "", - "-----BEGIN PGP MESSAGE-----", - "", - "wU4DhW3gBZ/VvCYSAQdA8bMs2spwbKdGjVsL1ByPkNrqD7frpB73maeL6I6SzDYg", - "O5G53tv339RdKq3WRcCtEEvxjHlUx2XNwXzC04BpmfvBTgNfPUyLDzjXnxIBB0Ae", - "8ymwGvXMCCimHXN0Dg8Ui62KOi03h0UgheoHWovJSCDF4CKre/xtFr3nL7lq/PKI", - "JsjVNz7/RK9FSXF6WwfONtLCyQGEuVAsB/KXfCBEyfKhaMwGHvhujRidGW5uV1no", - "lMGl3ODmo29Lgeu2uSE7EpJRZoe6hU6ddmBkqxax61ZtkaFlGFFpdo2K8balNNdz", - "ZsJ/9mmI9x3oOJ4/l1nhQbUO9ADbs7gJhFdV5Qkp30b5fCI7bU+aoe1ccBbLe/WM", - "YUty1PqcuQT7XjA+XmYuL261tvW8pBetT+i33/E2d8PzzYt2IuK9qeevyS+yxdwA", - "kfwejFWzzsUlJaDxs1x4XOxkMgSj+jo+g12dFOb7fyClsAnq23iDb8AuaT/BScAI", - "+lO+gher69+6LmM7VGHLG5k762J1jTaQCaKt1s8TAWV99Eo4491vL6fyvk3l/Cfg", - "RXSwiWFgj19Pn0Rq7CD9v22UE2vdUMBTcV4aw79mClk1YQ23jbF0y5DCjPdJ62Zo", - "tskBgFt3NoWV80jZ76zIBLrrjLwCCll8JjJtFwSkt2GX5RFBsVa4A8IDht9RtEk7", - "rrHgbSZQfkauEi/mH3/6CDZoLqSHudUZ7d4MaJwun1TkFYGe2ORwGJd4OBj3oGJp", - "H8YBwCpk///L/fKjX0Gg3M8nrpM4wrRFhPKidAgO/kcm25X4+ZHlVkWBTCt5RWKI", - "fHh6oLDZCqCfcgMkE1KKmwfIHaUkhq5BPRigwy6i5dh1DM4+1UCLh3dxzVbqE9b9", - "61NB19nXdRtDA2sOUnj9ve6m/wEPyCb6/zBQZqvCBYb1/AjdXpUrFT+DbpfyxaXN", - "XfhDVb5mNqNM/IVj0V5fvTc6vOfYbzQtPm10H+FdWWfb+rJRfyC3MA2w2IqstFe3", - "w3bu2iE6CQvSqRvge+ZqLKt/NqYwOURiUmpuklbl3kPJ97+mfKWoiqk8Iz1VY+bb", - "NMUC7aoGv+jcoj+WS6PYO8N6BeRVUUB3ZJSf8nzjgxm1/BcM+UD3BPrlhT11ODRs", - "baifGbprMWwt3dhb8cQgRT8GPdpO1OsDkzL6iikMjLHWWiA99GV6ruiHsIPw6boW", - "A6/uSOskbDHOROotKmddGTBd0iiHXAoQsJFt1ZjUkt6EHrgWs+GAvrvKpXs1mrz8", - "uj3GwEFrHS+Xuf2UDgpszYT3hI2cL/kUtGakVR7m7vVMZqXBUbZdGAEb1PZNPwsI", - "E4aMK02+EVB+tSN4Fzj99N2YD0inVYt+oPjr2tHhUS6aSGBNS/48Ki47DOg4Sxkn", - "lkOWnEbCD+XTnbDd", - "=agR5", - "-----END PGP MESSAGE-----", - "", - "", - "--YFrteb74qSXmggbOxZL9dRnhymywAi--", - "", - "", - ] - ).encode() +@pytest.mark.slow +def test_exceed_rate_limit(cmsetup, gencreds, mailgen): + """Test that the per-account send-mail limit is exceeded.""" + user1, user2 = cmsetup.gen_users(2) + mail = mailgen.get_encrypted(user1.addr, user2.addr) for i in range(100): print("Sending mail", str(i)) try: - smtp.conn.sendmail(user, to_addr, mail) + user1.smtp.sendmail(user1.addr, [user2.addr], mail) except smtplib.SMTPSenderRefused as e: - if i == 0: - pytest.fail(f"rate limit was exceeded too early with msg {i} - maybe wait a minute before testing?") - if i < 41: + if i < 80: pytest.fail(f"rate limit was exceeded too early with msg {i}") assert e.smtp_code == 450 assert b'4.7.1 Error: too much mail from' in e.smtp_error