tests: SMTPDataError doesn't have recipients

ARCHITECTURE.md

@@ -48,3 +48,19 @@ graph LR;
 The edges in this graph should not be taken too literally; they
 reflect some sort of communication path or dependency relationship
 between components of the chatmail server.
+
+## Message between users on the same relay
+
+```mermaid
+graph LR;
+    chatmail core --> |465|smtps/smtpd;
+    chatmail core --> |587|submission/smtpd;
+    smtps/smtpd --> |10080|filtermail;
+    submission/smtpd --> |10080|filtermail;
+    filtermail --> |10025|smtpd reinject;
+    smtpd reinject --> cleanup;
+    cleanup --> qmgr;
+    qmgr --> smtpd accepts message;
+    qmgr --> |lmtp|dovecot;
+    dovecot --> chatmail core;
+```

CHANGELOG.md

@@ -2,15 +2,6 @@
 
 ## untagged
 
-- filtermail: run CPU-intensive handle_DATA in a thread pool executor
-  ([#676](https://github.com/chatmail/relay/pull/676))
-
-- don't use the complicated logging module in filtermail to exclude a potential source of errors. 
-  ([#674](https://github.com/chatmail/relay/pull/674))
-
-- Specify nginx.conf to only handle `mail_domain`, www, and mta-sts domains
-  ([#636](https://github.com/chatmail/relay/pull/636))
-
 - Setup TURN server
   ([#621](https://github.com/chatmail/relay/pull/621))
 
@@ -20,6 +11,9 @@
 - Update iroh-relay to 0.35.0
   ([#650](https://github.com/chatmail/relay/pull/650))
 
+- postfix: accept whole mail before passing it to filtermail
+  ([#673](https://github.com/chatmail/relay/pull/673))
+
 - filtermail: accept mails from Protonmail
   ([#616](https://github.com/chatmail/relay/pull/655))
 

chatmaild/src/chatmaild/filtermail.py

@@ -2,6 +2,7 @@
 import asyncio
 import base64
 import binascii
+import logging
 import sys
 import time
 from email import policy
@@ -104,8 +105,8 @@ def check_armored_payload(payload: str, outgoing: bool):
     # Disallow comments in outgoing messages
     version_comment = "Version: "
     if payload.startswith(version_comment):
-        splitindex = payload.find("\r\n") + 2
-        payload = payload[splitindex:]
+        version_line = payload.splitlines()[0]
+        payload = payload.removeprefix(version_line)
         if outgoing:
             return False
 
@@ -228,7 +229,7 @@ class OutgoingBeforeQueueHandler:
         self.send_rate_limiter = SendRateLimiter()
 
     async def handle_MAIL(self, server, session, envelope, address, mail_options):
-        log_info(f"handle_MAIL from {address}")
+        logging.info(f"handle_MAIL from {address}")
         envelope.mail_from = address
         max_sent = self.config.max_user_send_per_minute
         if not self.send_rate_limiter.is_sending_allowed(address, max_sent):
@@ -241,15 +242,11 @@ class OutgoingBeforeQueueHandler:
         return "250 OK"
 
     async def handle_DATA(self, server, session, envelope):
-        loop = asyncio.get_running_loop()
-        return await loop.run_in_executor(None, self.sync_handle_DATA, envelope)
-
-    def sync_handle_DATA(self, envelope):
-        log_info("handle_DATA before-queue")
+        logging.info("handle_DATA before-queue")
         error = self.check_DATA(envelope)
         if error:
             return error
-        log_info("re-injecting the mail that passed checks")
+        logging.info("re-injecting the mail that passed checks")
         client = SMTPClient("localhost", self.config.postfix_reinject_port)
         client.sendmail(
             envelope.mail_from, envelope.rcpt_tos, envelope.original_content
@@ -258,7 +255,7 @@ class OutgoingBeforeQueueHandler:
 
     def check_DATA(self, envelope):
         """the central filtering function for e-mails."""
-        log_info(f"Processing DATA message from {envelope.mail_from}")
+        logging.info(f"Processing DATA message from {envelope.mail_from}")
 
         message = BytesParser(policy=policy.default).parsebytes(envelope.content)
         mail_encrypted = check_encrypted(message, outgoing=True)
@@ -298,15 +295,11 @@ class IncomingBeforeQueueHandler:
         self.config = config
 
     async def handle_DATA(self, server, session, envelope):
-        loop = asyncio.get_running_loop()
-        return await loop.run_in_executor(None, self.sync_handle_DATA, envelope)
-
-    def sync_handle_DATA(self, envelope):
-        log_info("handle_DATA before-queue")
+        logging.info("handle_DATA before-queue")
         error = self.check_DATA(envelope)
         if error:
             return error
-        log_info("re-injecting the mail that passed checks")
+        logging.info("re-injecting the mail that passed checks")
 
         # the smtp daemon on reinject_port_incoming gives it to dkim milter
         # which looks at source address to determine whether to verify or sign
@@ -322,7 +315,7 @@ class IncomingBeforeQueueHandler:
 
     def check_DATA(self, envelope):
         """the central filtering function for e-mails."""
-        log_info(f"Processing DATA message from {envelope.mail_from}")
+        logging.info(f"Processing DATA message from {envelope.mail_from}")
 
         message = BytesParser(policy=policy.default).parsebytes(envelope.content)
         mail_encrypted = check_encrypted(message, outgoing=False)
@@ -364,19 +357,16 @@ class SendRateLimiter:
         return False
 
 
-def log_info(msg):
-    print(msg, file=sys.stderr)
-
-
 def main():
     args = sys.argv[1:]
     assert len(args) == 2
     config = read_config(args[0])
     mode = args[1]
+    logging.basicConfig(level=logging.WARN)
     loop = asyncio.new_event_loop()
     asyncio.set_event_loop(loop)
     assert mode in ["incoming", "outgoing"]
     task = asyncmain_beforequeue(config, mode)
     loop.create_task(task)
-    log_info("entering serving loop")
+    logging.info("entering serving loop")
     loop.run_forever()

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -66,7 +66,7 @@ http {
 
 		index index.html index.htm;
 
-		server_name {{ config.domain_name }} www.{{ config.domain_name }} mta-sts.{{ config.domain_name }};
+		server_name _;
 
 		access_log syslog:server=unix:/dev/log,facility=local7;
 

cmdeploy/src/cmdeploy/postfix/master.cf.j2

@@ -31,6 +31,7 @@ submission inet n       -       y       -       5000    smtpd
   -o smtpd_relay_restrictions=permit_sasl_authenticated,reject
   -o milter_macro_daemon_name=ORIGINATING
   -o smtpd_client_connection_count_limit=1000
+  -o smtpd_proxy_options=speed_adjust
   -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
 smtps     inet  n       -       y       -       5000    smtpd
   -o syslog_name=postfix/smtps
@@ -48,6 +49,7 @@ smtps     inet  n       -       y       -       5000    smtpd
   -o smtpd_relay_restrictions=permit_sasl_authenticated,reject
   -o smtpd_client_connection_count_limit=1000
   -o milter_macro_daemon_name=ORIGINATING
+  -o smtpd_proxy_options=speed_adjust
   -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
 #628       inet  n       -       y       -       -       qmqpd
 pickup    unix  n       -       y       60      1       pickup

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -195,9 +195,8 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
         except smtplib.SMTPException as e:
             if i < chatmail_config.max_user_send_per_minute:
                 pytest.fail(f"rate limit was exceeded too early with msg {i}")
-            outcome = e.recipients[user2.addr]
-            assert outcome[0] == 450
-            assert b"4.7.1: Too much mail from" in outcome[1]
+            assert e.smtp_code == 450
+            assert b"4.7.1: Too much mail from" in e.smtp_error
             return
     pytest.fail("Rate limit was not exceeded")