Deploy nginx and autoconfig XML

chatmaild/src/chatmaild/dictproxy.py

@@ -31,9 +31,7 @@ def encrypt_password(password: str):
 
 def create_user(db, user, password):
     if os.path.exists(NOCREATE_FILE):
-        logging.warning(
-            f"Didn't create account: {NOCREATE_FILE} exists. Delete the file to enable account creation."
-        )
+        logging.warning(f"Didn't create account: {NOCREATE_FILE} exists. Delete the file to enable account creation.")
         return
     with db.write_transaction() as conn:
         conn.create_user(user, password)

chatmaild/src/chatmaild/filtermail.py

@@ -3,7 +3,6 @@ import asyncio
 import logging
 from email.parser import BytesParser
 from email import policy
-from email.utils import parseaddr
 
 from aiosmtpd.lmtp import LMTP
 from aiosmtpd.controller import UnixSocketController
@@ -43,14 +42,50 @@ class ExampleHandler:
         return "250 OK"
 
     async def handle_DATA(self, server, session, envelope):
-        valid_recipients, res = lmtp_handle_DATA(envelope)
+        logging.info("Processing DATA message from %s", envelope.mail_from)
+
+        valid_recipients = []
+
+        message = BytesParser(policy=policy.default).parsebytes(envelope.content)
+        mail_encrypted = check_encrypted(message)
+
+        res = []
+        for recipient in envelope.rcpt_tos:
+            my_local_domain = envelope.mail_from.split("@")
+            if len(my_local_domain) != 2:
+                res += [f"500 Invalid from address <{envelope.mail_from}>"]
+                continue
+
+            if envelope.mail_from == recipient:
+                # Always allow sending emails to self.
+                valid_recipients += [recipient]
+                res += ["250 OK"]
+                continue
+
+            recipient_local_domain = recipient.split("@")
+            if len(recipient_local_domain) != 2:
+                res += [f"500 Invalid address <{recipient}>"]
+                continue
+
+            is_outgoing = recipient_local_domain[1] != my_local_domain[1]
+
+            if (
+                is_outgoing
+                and not mail_encrypted
+                and message.get("secure-join") != "vc-request"
+                and message.get("secure-join") != "vg-request"
+            ):
+                res += ["500 Outgoing mail must be encrypted"]
+                continue
+
+            valid_recipients += [recipient]
+            res += ["250 OK"]
+
         # Reinject the mail back into Postfix.
         if valid_recipients:
             logging.info("Reinjecting the mail")
             client = SMTPClient("localhost", "10026")
             client.sendmail(envelope.mail_from, valid_recipients, envelope.content)
-        else:
-            logging.info("no valid recipients, ignoring mail")
 
         return "\r\n".join(res)
 
@@ -62,57 +97,6 @@ async def asyncmain(loop):
     controller.start()
 
 
-def lmtp_handle_DATA(envelope):
-    """the central filtering function for e-mails."""
-    logging.info(f"Processing DATA message from {envelope.mail_from}")
-
-    message = BytesParser(policy=policy.default).parsebytes(envelope.content)
-    mail_encrypted = check_encrypted(message)
-
-    valid_recipients = []
-    res = []
-    for recipient in envelope.rcpt_tos:
-        my_local_domain = envelope.mail_from.split("@")
-        if len(my_local_domain) != 2:
-            res += [f"500 Invalid from address <{envelope.mail_from}>"]
-            continue
-
-        _, from_addr = parseaddr(message.get("from").strip())
-        logging.info(f"mime-from: {from_addr} envelope-from: {envelope.mail_from}")
-        if envelope.mail_from.lower() != from_addr.lower():
-            res += [f"500 Invalid FROM <{from_addr}> for <{envelope.mail_from}>"]
-            continue
-
-        if envelope.mail_from == recipient:
-            # Always allow sending emails to self.
-            valid_recipients += [recipient]
-            res += ["250 OK"]
-            continue
-
-        recipient_local_domain = recipient.split("@")
-        if len(recipient_local_domain) != 2:
-            res += [f"500 Invalid address <{recipient}>"]
-            continue
-
-        is_outgoing = recipient_local_domain[1] != my_local_domain[1]
-
-        if (
-            is_outgoing
-            and not mail_encrypted
-            and message.get("secure-join") != "vc-request"
-            and message.get("secure-join") != "vg-request"
-        ):
-            res += ["500 Outgoing mail must be encrypted"]
-            continue
-
-        valid_recipients += [recipient]
-        res += ["250 OK"]
-
-    assert len(envelope.rcpt_tos) == len(res)
-    assert len(valid_recipients) <= len(res)
-    return valid_recipients, res
-
-
 def main():
     logging.basicConfig(level=logging.INFO)
     loop = asyncio.new_event_loop()

chatmaild/src/chatmaild/test_filtermail.py

@@ -1,46 +1,8 @@
-from .filtermail import check_encrypted, lmtp_handle_DATA
+from .filtermail import check_encrypted
 from email.parser import BytesParser
 from email import policy
 
 
-def test_reject_forged_from():
-    def makemail(from_addr):
-        return BytesParser(policy=policy.default).parsebytes(
-            "\r\n".join(
-                [
-                    f"From: <{from_addr}",
-                    "To: <barbaz@c3.testrun.org>",
-                    "Date: Sun, 15 Oct 2023 16:41:44 +0000",
-                    "Message-ID: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>",
-                    "References: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>",
-                    "Chat-Version: 1.0",
-                    "MIME-Version: 1.0",
-                    "Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no",
-                    "",
-                    "Hi!",
-                    "",
-                    "",
-                ]
-            ).encode()
-        )
-
-    class envelope:
-        mail_from = "bob@c3.testrun.org"
-        rcpt_tos = ["somebody@c3.testrun.org"]
-
-    # test that the filter lets good mail through
-    envelope.content = makemail(envelope.mail_from).as_bytes()
-    valid_recipients, res = lmtp_handle_DATA(envelope=envelope)
-    assert valid_recipients == envelope.rcpt_tos
-    assert len(res) == 1 and "250" in res[0]
-
-    # test that the filter rejects forged mail
-    envelope.content = makemail("forged@c3.testrun.org").as_bytes()
-    valid_recipients, res = lmtp_handle_DATA(envelope=envelope)
-    assert not valid_recipients
-    assert len(res) == 1 and "500" in res[0]
-
-
 def test_filtermail():
     def check_encrypted_bstr(content):
         message = BytesParser(policy=policy.default).parsebytes(content)

online-tests/benchmark.py

@@ -1,60 +1,34 @@
-def test_tls_imap(benchmark, imap):
-    def imap_connect():
-        imap.connect()
+def test_tls_serialized_connect(benchmark, imap_or_smtp):
+    def connect():
+        imap_or_smtp.connect()
 
-    benchmark(imap_connect, 10)
+    benchmark(connect)
 
 
-def test_login_imap(benchmark, imap, gencreds):
-    def imap_connect_and_login():
-        imap.connect()
-        imap.login(*gencreds())
+def test_login(benchmark, imap_or_smtp, gencreds):
+    cls = imap_or_smtp.__class__
+    conns = []
+    for i in range(20):
+        conn = cls(imap_or_smtp.host)
+        conn.connect()
+        conns.append(conn)
 
-    benchmark(imap_connect_and_login, 10)
+    def login():
+        conn = conns.pop()
+        conn.login(*gencreds())
+
+    benchmark(login)
 
 
-def test_tls_smtp(benchmark, smtp):
-    def smtp_connect():
-        smtp.connect()
+def test_send_and_receive_10(benchmark, cmfactory, lp):
+    """send many messages between two accounts"""
+    ac1, ac2 = cmfactory.get_online_accounts(2)
+    chat = cmfactory.get_accepted_chat(ac1, ac2)
 
-    benchmark(smtp_connect, 10)
+    def send_10_receive_all():
+        for i in range(10):
+            chat.send_text(f"hello {i}")
+        for i in range(10):
+            ac2.wait_next_incoming_message()
 
-
-def test_login_smtp(benchmark, smtp, gencreds):
-    def smtp_connect_and_login():
-        smtp.connect()
-        smtp.login(*gencreds())
-
-    benchmark(smtp_connect_and_login, 10)
-
-
-class TestDC:
-    def test_autoconfigure(self, benchmark, cmfactory):
-        def autoconfig_and_idle_ready():
-            cmfactory.get_online_accounts(1)
-
-        benchmark(autoconfig_and_idle_ready, 5)
-
-    def test_ping_pong(self, benchmark, cmfactory):
-        ac1, ac2 = cmfactory.get_online_accounts(2)
-        chat = cmfactory.get_accepted_chat(ac1, ac2)
-
-        def ping_pong():
-            chat.send_text("ping")
-            msg = ac2.wait_next_incoming_message()
-            msg.chat.send_text("pong")
-            ac1.wait_next_incoming_message()
-
-        benchmark(ping_pong, 3)
-
-    def test_send_10_receive_10(self, benchmark, cmfactory, lp):
-        ac1, ac2 = cmfactory.get_online_accounts(2)
-        chat = cmfactory.get_accepted_chat(ac1, ac2)
-
-        def send_10_receive_10():
-            for i in range(10):
-                chat.send_text(f"hello {i}")
-            for i in range(10):
-                ac2.wait_next_incoming_message()
-
-        benchmark(send_10_receive_10, 1)
+    benchmark(send_10_receive_all)

online-tests/conftest.py

@@ -1,6 +1,5 @@
 import os
 import io
-import time
 import random
 import subprocess
 import imaplib
@@ -15,10 +14,6 @@ def pytest_addoption(parser):
     )
 
 
-def pytest_configure(config):
-    config._benchresults = {}
-
-
 def pytest_runtest_setup(item):
     markers = list(item.iter_markers(name="slow"))
     if markers:
@@ -59,35 +54,6 @@ def pytest_report_header():
         return ["-" * len(text), text, "-" * len(text)]
 
 
-@pytest.fixture
-def benchmark(request):
-    def bench(func, num, name=None):
-        if name is None:
-            name = func.__name__
-        durations = []
-        for i in range(num):
-            now = time.time()
-            func()
-            durations.append(time.time() - now)
-        durations.sort()
-        request.config._benchresults[name] = durations
-
-    return bench
-
-
-def pytest_terminal_summary(terminalreporter):
-    tr = terminalreporter
-    results = tr.config._benchresults
-    tr.section("benchmark results")
-    headers = f"{'benchmark name': <30} {'median': >6}"
-    tr.write_line(headers)
-    tr.write_line("-" * len(headers))
-    for name, durations in results.items():
-        median = sorted(durations)[len(durations) // 2]
-        median = f"{median:2.4f}"
-        tr.write_line(f"{name: <30} {median: >6}")
-
-
 @pytest.fixture
 def imap(maildomain):
     return ImapConn(maildomain)
@@ -109,24 +75,6 @@ class ImapConn:
         print(f"imap-login {user!r} {password!r}")
         self.conn.login(user, password)
 
-    def fetch_all(self):
-        print("imap-fetch all")
-        status, res = self.conn.select()
-        if int(res[0]) == 0:
-            raise ValueError("no messages in imap folder")
-        status, results = self.conn.fetch("1:*", "(RFC822)")
-        assert status == "OK"
-        return results
-
-    def fetch_all_messages(self):
-        print("imap-fetch all messages")
-        results = self.fetch_all()
-        messages = []
-        for item in results:
-            if len(item) == 2:
-                messages.append(item[1].decode())
-        return messages
-
 
 @pytest.fixture
 def smtp(maildomain):
@@ -149,11 +97,6 @@ class SmtpConn:
         print(f"smtp-login {user!r} {password!r}")
         self.conn.login(user, password)
 
-    def sendmail(self, from_addr, to_addrs, msg):
-        print(f"smtp-sendmail from={from_addr!r} to_addrs={to_addrs!r}")
-        print(f"smtp-sendmail message size: {len(msg)}")
-        return self.conn.sendmail(from_addr=from_addr, to_addrs=to_addrs, msg=msg)
-
 
 @pytest.fixture(params=["imap", "smtp"])
 def imap_or_smtp(request):
@@ -255,66 +198,4 @@ class Remote:
         )
         while 1:
             line = self.popen.stdout.readline()
-            res = line.decode().strip().lower()
-            if res:
-                yield res
-            else:
-                break
-
-
-@pytest.fixture
-def mailgen(request):
-    class Mailgen:
-        def get_encrypted(self, from_addr, to_addr):
-            data = request.fspath.dirpath("mailgen/encrypted.eml").read()
-            return data.format(from_addr=from_addr, to_addr=to_addr)
-
-    return Mailgen()
-
-
-@pytest.fixture
-def cmsetup(maildomain, gencreds):
-    return CMSetup(maildomain, gencreds)
-
-
-class CMSetup:
-    def __init__(self, maildomain, gencreds):
-        self.maildomain = maildomain
-        self.gencreds = gencreds
-
-    def gen_users(self, num):
-        print(f"Creating {num} online users")
-        users = []
-        for i in range(num):
-            addr, password = self.gencreds()
-            user = CMUser(self.maildomain, addr, password)
-            assert user.smtp
-            users.append(user)
-        return users
-
-
-class CMUser:
-    def __init__(self, maildomain, addr, password):
-        self.maildomain = maildomain
-        self.addr = addr
-        self.password = password
-        self._smtp = None
-        self._imap = None
-
-    @property
-    def smtp(self):
-        if not self._smtp:
-            handle = SmtpConn(self.maildomain)
-            handle.connect()
-            handle.login(self.addr, self.password)
-            self._smtp = handle
-        return self._smtp
-
-    @property
-    def imap(self):
-        if not self._imap:
-            imap = ImapConn(self.maildomain)
-            imap.connect()
-            imap.login(self.addr, self.password)
-            self._imap = imap
-        return self._imap
+            yield line.decode().strip().lower()

online-tests/mailgen/encrypted.eml

@@ -1,66 +0,0 @@
-From: {from_addr}
-To: {to_addr}
-Subject: ...
-Date: Sun, 15 Oct 2023 16:43:21 +0000
-Message-ID: <Mr.UVyJWZmkCKM.hGzNc6glBE_@c2.testrun.org>
-In-Reply-To: <Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>
-References: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>
-	<Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>
-Chat-Version: 1.0
-Autocrypt: addr={from_addr}; prefer-encrypt=mutual;
-	keydata=xjMEZSwWjhYJKwYBBAHaRw8BAQdAQBEhqeJh0GueHB6kF/DUQqYCxARNBVokg/AzT+7LqH
-	rNFzxiYXJiYXpAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUsFo4CGwMECwkIBwYVCAkKCwID
-	FgIBFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX9A4AEAnHWHp49eBCMHK5t66gYPiW
-	XQuB1mwUjzGfYWB+0RXUoA/0xcQ3FbUNlGKW7Blp6eMFfViv6Mv2d3kNSXACB6nmcMzjgEZSwWjhIK
-	KwYBBAGXVQEFAQEHQBpY5L2M1XHo0uxf8SX1wNLBp/OVvidoWHQF2Jz+kJsUAwEIB8J4BBgWCAAgBQ
-	JlLBaOAhsMFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX/INgEA37AJaNvruYsJVanP
-	IXnYw4CKd55UAwl8Zcy+M2diAbkA/0fHHcGV4r78hpbbL1Os52DPOdqYQRauIeJUeG+G6bQO
-MIME-Version: 1.0
-Content-Type: multipart/encrypted; protocol="application/pgp-encrypted";
-	boundary="YFrteb74qSXmggbOxZL9dRnhymywAi"
-
-
---YFrteb74qSXmggbOxZL9dRnhymywAi
-Content-Description: PGP/MIME version identification
-Content-Type: application/pgp-encrypted
-
-Version: 1
-
-
---YFrteb74qSXmggbOxZL9dRnhymywAi
-Content-Description: OpenPGP encrypted message
-Content-Disposition: inline; filename="encrypted.asc";
-Content-Type: application/octet-stream; name="encrypted.asc"
-
------BEGIN PGP MESSAGE-----
-
-wU4DhW3gBZ/VvCYSAQdA8bMs2spwbKdGjVsL1ByPkNrqD7frpB73maeL6I6SzDYg
-O5G53tv339RdKq3WRcCtEEvxjHlUx2XNwXzC04BpmfvBTgNfPUyLDzjXnxIBB0Ae
-8ymwGvXMCCimHXN0Dg8Ui62KOi03h0UgheoHWovJSCDF4CKre/xtFr3nL7lq/PKI
-JsjVNz7/RK9FSXF6WwfONtLCyQGEuVAsB/KXfCBEyfKhaMwGHvhujRidGW5uV1no
-lMGl3ODmo29Lgeu2uSE7EpJRZoe6hU6ddmBkqxax61ZtkaFlGFFpdo2K8balNNdz
-ZsJ/9mmI9x3oOJ4/l1nhQbUO9ADbs7gJhFdV5Qkp30b5fCI7bU+aoe1ccBbLe/WM
-YUty1PqcuQT7XjA+XmYuL261tvW8pBetT+i33/E2d8PzzYt2IuK9qeevyS+yxdwA
-kfwejFWzzsUlJaDxs1x4XOxkMgSj+jo+g12dFOb7fyClsAnq23iDb8AuaT/BScAI
-+lO+gher69+6LmM7VGHLG5k762J1jTaQCaKt1s8TAWV99Eo4491vL6fyvk3l/Cfg
-RXSwiWFgj19Pn0Rq7CD9v22UE2vdUMBTcV4aw79mClk1YQ23jbF0y5DCjPdJ62Zo
-tskBgFt3NoWV80jZ76zIBLrrjLwCCll8JjJtFwSkt2GX5RFBsVa4A8IDht9RtEk7
-rrHgbSZQfkauEi/mH3/6CDZoLqSHudUZ7d4MaJwun1TkFYGe2ORwGJd4OBj3oGJp
-H8YBwCpk///L/fKjX0Gg3M8nrpM4wrRFhPKidAgO/kcm25X4+ZHlVkWBTCt5RWKI
-fHh6oLDZCqCfcgMkE1KKmwfIHaUkhq5BPRigwy6i5dh1DM4+1UCLh3dxzVbqE9b9
-61NB19nXdRtDA2sOUnj9ve6m/wEPyCb6/zBQZqvCBYb1/AjdXpUrFT+DbpfyxaXN
-XfhDVb5mNqNM/IVj0V5fvTc6vOfYbzQtPm10H+FdWWfb+rJRfyC3MA2w2IqstFe3
-w3bu2iE6CQvSqRvge+ZqLKt/NqYwOURiUmpuklbl3kPJ97+mfKWoiqk8Iz1VY+bb
-NMUC7aoGv+jcoj+WS6PYO8N6BeRVUUB3ZJSf8nzjgxm1/BcM+UD3BPrlhT11ODRs
-baifGbprMWwt3dhb8cQgRT8GPdpO1OsDkzL6iikMjLHWWiA99GV6ruiHsIPw6boW
-A6/uSOskbDHOROotKmddGTBd0iiHXAoQsJFt1ZjUkt6EHrgWs+GAvrvKpXs1mrz8
-uj3GwEFrHS+Xuf2UDgpszYT3hI2cL/kUtGakVR7m7vVMZqXBUbZdGAEb1PZNPwsI
-E4aMK02+EVB+tSN4Fzj99N2YD0inVYt+oPjr2tHhUS6aSGBNS/48Ki47DOg4Sxkn
-lkOWnEbCD+XTnbDd
-=agR5
------END PGP MESSAGE-----
-
-
---YFrteb74qSXmggbOxZL9dRnhymywAi--
-
-

online-tests/test_0_basic.py

@@ -1,7 +1,3 @@
-import smtplib
-import pytest
-
-
 def test_remote(remote, imap_or_smtp):
     lineproducer = remote.iter_output(imap_or_smtp.logcmd)
     imap_or_smtp.connect()
@@ -17,36 +13,3 @@ def test_use_two_chatmailservers(cmfactory, maildomain2):
     domain1 = ac1.get_config("addr").split("@")[1]
     domain2 = ac2.get_config("addr").split("@")[1]
     assert domain1 != domain2
-
-
-@pytest.mark.parametrize("forgeaddr", ["internal", "someone@example.org"])
-def test_reject_forged_from(cmsetup, mailgen, lp, remote, forgeaddr):
-    user1, user3 = cmsetup.gen_users(2)
-
-    lp.sec("send encrypted message with forged from")
-    print("envelope_from", user1.addr)
-    if forgeaddr == "internal":
-        addr_to_forge = cmsetup.gen_users(1)[0].addr
-    else:
-        addr_to_forge = "someone@example.org"
-
-    print("message to inject:")
-    msg = mailgen.get_encrypted(from_addr=addr_to_forge, to_addr=user3.addr)
-    for line in msg.split("\n")[:4]:
-        print(f"   {line}")
-
-    lp.sec("Send forged mail and check remote postfix lmtp processing result")
-    remote_log = remote.iter_output("journalctl -t postfix/lmtp")
-    user1.smtp.sendmail(from_addr=user1.addr, to_addrs=[user3.addr], msg=msg)
-    for line in remote_log:
-        # print(line)
-        if "500 invalid from" in line and user3.addr in line:
-            break
-    else:
-        pytest.fail("remote postfix/filtermail failed to reject message")
-
-    # check that the logged in user (who sent the forged msg) got a non-delivery notice
-    for message in user1.imap.fetch_all_messages():
-        if "Invalid FROM" in message and addr_to_forge in message:
-            return
-    pytest.fail(f"forged From={addr_to_forge} did not cause non-delivery notice")

scripts/init.sh

@@ -11,4 +11,4 @@ chatmaild/venv/bin/pip install --upgrade pytest build 'setuptools>=68'
 chatmaild/venv/bin/pip install -e chatmaild
 
 python3 -m venv online-tests/venv
-online-tests/venv/bin/pip install pytest pytest-timeout pdbpp deltachat 
+online-tests/venv/bin/pip install pytest pytest-timeout pdbpp deltachat pytest-benchmark