dictproxy: use crypt instead of doveadm pw

.github/workflows/ci.yaml

@@ -1,31 +0,0 @@
-name: CI
-
-on:
-  pull_request:
-  push:
-
-jobs:
-  tox:
-    name: chatmail tests
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: run chatmaild tests 
-        working-directory: chatmaild
-        run: pipx run tox
-      - name: run deploy-chatmail offline tests 
-        working-directory: deploy-chatmail
-        run: pipx run tox
-      - name: run deploy-chatmail offline tests 
-        working-directory: deploy-chatmail
-        run: pipx run tox
-
-  scripts:
-    name: chatmail script invocations 
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: run init.sh 
-        run: ./scripts/init.sh 
-      - name: run test.sh 
-        run: ./scripts/test.sh 

chatmaild/pyproject.toml

@@ -20,7 +20,7 @@ addopts = "-v -ra --strict-markers"
 legacy_tox_ini = """
 [tox]
 isolated_build = true
-envlist = lint,py
+envlist = lint
 
 [testenv:lint]
 skipdist = True
@@ -31,10 +31,4 @@ deps =
 commands =
   black --quiet --check --diff src/
   ruff src/
-
-[testenv]
-passenv = CHATMAIL_DOMAIN
-deps = pytest 
-       pdbpp 
-commands = pytest -v -rsXx {posargs: ../tests/chatmaild}
 """

chatmaild/src/chatmaild/filtermail.py

@@ -34,34 +34,6 @@ def check_encrypted(message):
     return True
 
 
-def check_mdn(message, envelope):
-    if len(envelope.rcpt_tos) != 1:
-        return False
-
-    for name in ["auto-submitted", "chat-version"]:
-        if not message.get(name):
-            return False
-
-    if message.get_content_type() != "multipart/report":
-        return False
-
-    body = message.get_body()
-    if body.get_content_type() != "text/plain":
-        return False
-
-    if list(body.iter_attachments()) or list(body.iter_parts()):
-        return False
-
-    # even with all mime-structural checks an attacker
-    # could try to abuse the subject or body to contain links or other
-    # annoyance -- we skip on checking subject/body for now as Delta Chat
-    # should evolve to create E2E-encrypted read receipts anyway.
-    # and then MDNs are just encrypted mail and can pass the border
-    # to other instances.
-
-    return True
-
-
 class SMTPController(Controller):
     def factory(self):
         return SMTP(self.handler, **self.SMTP_kwargs)
@@ -110,9 +82,6 @@ def check_DATA(envelope):
     if envelope.mail_from.lower() != from_addr.lower():
         return f"500 Invalid FROM <{from_addr!r}> for <{envelope.mail_from!r}>"
 
-    if not mail_encrypted and check_mdn(message, envelope):
-        return
-
     envelope_from_domain = from_addr.split("@").pop()
     for recipient in envelope.rcpt_tos:
         if envelope.mail_from == recipient:

chatmaild/src/chatmaild/test_doveauth.py

@@ -3,8 +3,8 @@ import os
 import pytest
 
 import chatmaild.dictproxy
-from chatmaild.dictproxy import get_user_data, lookup_passdb
+from .dictproxy import get_user_data, lookup_passdb
-from chatmaild.database import Database, DBError
+from .database import Database, DBError
 
 
 @pytest.fixture()

chatmaild/src/chatmaild/test_filtermail.py

@@ -0,0 +1,338 @@
+from .filtermail import check_encrypted, check_DATA, SendRateLimiter
+from email.parser import BytesParser
+from email import policy
+import pytest
+
+
+def test_reject_forged_from():
+    def makemail(from_addr):
+        return BytesParser(policy=policy.default).parsebytes(
+            "\r\n".join(
+                [
+                    f"From: <{from_addr}",
+                    "To: <barbaz@c3.testrun.org>",
+                    "Date: Sun, 15 Oct 2023 16:41:44 +0000",
+                    "Message-ID: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>",
+                    "References: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>",
+                    "Chat-Version: 1.0",
+                    "MIME-Version: 1.0",
+                    "Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no",
+                    "",
+                    "Hi!",
+                    "",
+                    "",
+                ]
+            ).encode()
+        )
+
+    class envelope:
+        mail_from = "bob@c3.testrun.org"
+        rcpt_tos = ["somebody@c3.testrun.org"]
+
+    # test that the filter lets good mail through
+    envelope.content = makemail(envelope.mail_from).as_bytes()
+    assert not check_DATA(envelope=envelope)
+
+    # test that the filter rejects forged mail
+    envelope.content = makemail("forged@c3.testrun.org").as_bytes()
+    error = check_DATA(envelope=envelope)
+    assert "500" in error
+
+
+def test_filtermail():
+    def check_encrypted_bstr(content):
+        message = BytesParser(policy=policy.default).parsebytes(content)
+        return check_encrypted(message)
+
+    assert not check_encrypted_bstr(b"foo")
+
+    assert not check_encrypted_bstr(
+        "\r\n".join(
+            [
+                "Subject: =?utf-8?q?Message_from_foobar=40c2=2Etestrun=2Eorg?=",
+                "Chat-Disposition-Notification-To: foobar@c2.testrun.org",
+                "Chat-User-Avatar: 0",
+                "From: <foobar@c2.testrun.org>",
+                "To: <barbaz@c2.testrun.org>",
+                "Date: Sun, 15 Oct 2023 16:41:44 +0000",
+                "Message-ID: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>",
+                "References: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>",
+                "Chat-Version: 1.0",
+                "Autocrypt: addr=foobar@c2.testrun.org; prefer-encrypt=mutual;",
+                "\tkeydata=xjMEZSrw3hYJKwYBBAHaRw8BAQdAiEKNQFU28c6qsx4vo/JHdt73RXdjMOmByf/XsGiJ7m",
+                "\tnNFzxmb29iYXJAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUq8N4CGwMECwkIBwYVCAkKCwID",
+                "\tFgIBFiEEGil0OvTIa6RngmCLUYNnEa9leJAACgkQUYNnEa9leJCX3gEAhm0MehE5byBBU1avPczr/I",
+                "\tHjNLht7Qf6++mAhlJmtDcA/0C8VYJhsUpmiDjuZaMDWNv4FO2BJG6LH7gSm6n7ClMJzjgEZSrw3hIK",
+                "\tKwYBBAGXVQEFAQEHQAxGG/QW0owCfMp1A+vXEMwgzWcBpNFr58kX2eXuPpM6AwEIB8J4BBgWCAAgBQ",
+                "\tJlKvDeAhsMFiEEGil0OvTIa6RngmCLUYNnEa9leJAACgkQUYNnEa9leJDg1gEAwLf8KDoAAKyYgjyI",
+                "\tvYvO9VEgBni1C4Xx1VjcaEmlDK8BALoFuUCK+enw76TtDcAUKhlhUiM6SDRExkS4Nskp/BcK",
+                "MIME-Version: 1.0",
+                "Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no",
+                "",
+                "Hi!",
+                "",
+                "",
+            ]
+        ).encode()
+    )
+
+    assert not check_encrypted_bstr(
+        "\r\n".join(
+            [
+                "Subject: =?utf-8?q?Message_from_foobar=40c2=2Etestrun=2Eorg?=",
+                "Chat-Disposition-Notification-To: foobar@c2.testrun.org",
+                "Chat-User-Avatar: 0",
+                "From: <foobar@c2.testrun.org>",
+                "To: <barbaz@c2.testrun.org>",
+                "Date: Sun, 15 Oct 2023 16:41:44 +0000",
+                "Message-ID: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>",
+                "References: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>",
+                "Chat-Version: 1.0",
+                "Autocrypt: addr=foobar@c2.testrun.org; prefer-encrypt=mutual;",
+                "\tkeydata=xjMEZSrw3hYJKwYBBAHaRw8BAQdAiEKNQFU28c6qsx4vo/JHdt73RXdjMOmByf/XsGiJ7m",
+                "\tnNFzxmb29iYXJAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUq8N4CGwMECwkIBwYVCAkKCwID",
+                "\tFgIBFiEEGil0OvTIa6RngmCLUYNnEa9leJAACgkQUYNnEa9leJCX3gEAhm0MehE5byBBU1avPczr/I",
+                "\tHjNLht7Qf6++mAhlJmtDcA/0C8VYJhsUpmiDjuZaMDWNv4FO2BJG6LH7gSm6n7ClMJzjgEZSrw3hIK",
+                "\tKwYBBAGXVQEFAQEHQAxGG/QW0owCfMp1A+vXEMwgzWcBpNFr58kX2eXuPpM6AwEIB8J4BBgWCAAgBQ",
+                "\tJlKvDeAhsMFiEEGil0OvTIa6RngmCLUYNnEa9leJAACgkQUYNnEa9leJDg1gEAwLf8KDoAAKyYgjyI",
+                "\tvYvO9VEgBni1C4Xx1VjcaEmlDK8BALoFuUCK+enw76TtDcAUKhlhUiM6SDRExkS4Nskp/BcK",
+                "MIME-Version: 1.0",
+                "Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no",
+                "",
+                "Hi!",
+                "",
+                "",
+            ]
+        ).encode()
+    )
+
+    # https://xkcd.com/1181/
+    assert not check_encrypted_bstr(
+        "\r\n".join(
+            [
+                "Subject: =?utf-8?q?Message_from_foobar=40c2=2Etestrun=2Eorg?=",
+                "Chat-Disposition-Notification-To: foobar@c2.testrun.org",
+                "Chat-User-Avatar: 0",
+                "From: <foobar@c2.testrun.org>",
+                "To: <barbaz@c2.testrun.org>",
+                "Date: Sun, 15 Oct 2023 16:41:44 +0000",
+                "Message-ID: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>",
+                "References: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>",
+                "Chat-Version: 1.0",
+                "Autocrypt: addr=foobar@c2.testrun.org; prefer-encrypt=mutual;",
+                "\tkeydata=xjMEZSrw3hYJKwYBBAHaRw8BAQdAiEKNQFU28c6qsx4vo/JHdt73RXdjMOmByf/XsGiJ7m",
+                "\tnNFzxmb29iYXJAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUq8N4CGwMECwkIBwYVCAkKCwID",
+                "\tFgIBFiEEGil0OvTIa6RngmCLUYNnEa9leJAACgkQUYNnEa9leJCX3gEAhm0MehE5byBBU1avPczr/I",
+                "\tHjNLht7Qf6++mAhlJmtDcA/0C8VYJhsUpmiDjuZaMDWNv4FO2BJG6LH7gSm6n7ClMJzjgEZSrw3hIK",
+                "\tKwYBBAGXVQEFAQEHQAxGG/QW0owCfMp1A+vXEMwgzWcBpNFr58kX2eXuPpM6AwEIB8J4BBgWCAAgBQ",
+                "\tJlKvDeAhsMFiEEGil0OvTIa6RngmCLUYNnEa9leJAACgkQUYNnEa9leJDg1gEAwLf8KDoAAKyYgjyI",
+                "\tvYvO9VEgBni1C4Xx1VjcaEmlDK8BALoFuUCK+enw76TtDcAUKhlhUiM6SDRExkS4Nskp/BcK",
+                "MIME-Version: 1.0",
+                "Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no",
+                "",
+                "-----BEGIN PGP MESSAGE-----",
+                "Hi!",
+                "-----END PGP MESSAGE-----",
+                "",
+                "",
+            ]
+        ).encode()
+    )
+
+    assert check_encrypted_bstr(
+        "\r\n".join(
+            [
+                "Subject: ...",
+                "From: <barbaz@c2.testrun.org>",
+                "To: <foobar@c2.testrun.org>",
+                "Date: Sun, 15 Oct 2023 16:43:21 +0000",
+                "Message-ID: <Mr.UVyJWZmkCKM.hGzNc6glBE_@c2.testrun.org>",
+                "In-Reply-To: <Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>",
+                "References: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>",
+                "\t<Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>",
+                "Chat-Version: 1.0",
+                "Autocrypt: addr=barbaz@c2.testrun.org; prefer-encrypt=mutual;",
+                "\tkeydata=xjMEZSwWjhYJKwYBBAHaRw8BAQdAQBEhqeJh0GueHB6kF/DUQqYCxARNBVokg/AzT+7LqH",
+                "\trNFzxiYXJiYXpAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUsFo4CGwMECwkIBwYVCAkKCwID",
+                "\tFgIBFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX9A4AEAnHWHp49eBCMHK5t66gYPiW",
+                "\tXQuB1mwUjzGfYWB+0RXUoA/0xcQ3FbUNlGKW7Blp6eMFfViv6Mv2d3kNSXACB6nmcMzjgEZSwWjhIK",
+                "\tKwYBBAGXVQEFAQEHQBpY5L2M1XHo0uxf8SX1wNLBp/OVvidoWHQF2Jz+kJsUAwEIB8J4BBgWCAAgBQ",
+                "\tJlLBaOAhsMFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX/INgEA37AJaNvruYsJVanP",
+                "\tIXnYw4CKd55UAwl8Zcy+M2diAbkA/0fHHcGV4r78hpbbL1Os52DPOdqYQRauIeJUeG+G6bQO",
+                "MIME-Version: 1.0",
+                'Content-Type: multipart/encrypted; protocol="application/pgp-encrypted";',
+                '\tboundary="YFrteb74qSXmggbOxZL9dRnhymywAi"',
+                "",
+                "",
+                "--YFrteb74qSXmggbOxZL9dRnhymywAi",
+                "Content-Description: PGP/MIME version identification",
+                "Content-Type: application/pgp-encrypted",
+                "",
+                "Version: 1",
+                "",
+                "",
+                "--YFrteb74qSXmggbOxZL9dRnhymywAi",
+                "Content-Description: OpenPGP encrypted message",
+                'Content-Disposition: inline; filename="encrypted.asc";',
+                'Content-Type: application/octet-stream; name="encrypted.asc"',
+                "",
+                "-----BEGIN PGP MESSAGE-----",
+                "",
+                "wU4DhW3gBZ/VvCYSAQdA8bMs2spwbKdGjVsL1ByPkNrqD7frpB73maeL6I6SzDYg",
+                "O5G53tv339RdKq3WRcCtEEvxjHlUx2XNwXzC04BpmfvBTgNfPUyLDzjXnxIBB0Ae",
+                "8ymwGvXMCCimHXN0Dg8Ui62KOi03h0UgheoHWovJSCDF4CKre/xtFr3nL7lq/PKI",
+                "JsjVNz7/RK9FSXF6WwfONtLCyQGEuVAsB/KXfCBEyfKhaMwGHvhujRidGW5uV1no",
+                "lMGl3ODmo29Lgeu2uSE7EpJRZoe6hU6ddmBkqxax61ZtkaFlGFFpdo2K8balNNdz",
+                "ZsJ/9mmI9x3oOJ4/l1nhQbUO9ADbs7gJhFdV5Qkp30b5fCI7bU+aoe1ccBbLe/WM",
+                "YUty1PqcuQT7XjA+XmYuL261tvW8pBetT+i33/E2d8PzzYt2IuK9qeevyS+yxdwA",
+                "kfwejFWzzsUlJaDxs1x4XOxkMgSj+jo+g12dFOb7fyClsAnq23iDb8AuaT/BScAI",
+                "+lO+gher69+6LmM7VGHLG5k762J1jTaQCaKt1s8TAWV99Eo4491vL6fyvk3l/Cfg",
+                "RXSwiWFgj19Pn0Rq7CD9v22UE2vdUMBTcV4aw79mClk1YQ23jbF0y5DCjPdJ62Zo",
+                "tskBgFt3NoWV80jZ76zIBLrrjLwCCll8JjJtFwSkt2GX5RFBsVa4A8IDht9RtEk7",
+                "rrHgbSZQfkauEi/mH3/6CDZoLqSHudUZ7d4MaJwun1TkFYGe2ORwGJd4OBj3oGJp",
+                "H8YBwCpk///L/fKjX0Gg3M8nrpM4wrRFhPKidAgO/kcm25X4+ZHlVkWBTCt5RWKI",
+                "fHh6oLDZCqCfcgMkE1KKmwfIHaUkhq5BPRigwy6i5dh1DM4+1UCLh3dxzVbqE9b9",
+                "61NB19nXdRtDA2sOUnj9ve6m/wEPyCb6/zBQZqvCBYb1/AjdXpUrFT+DbpfyxaXN",
+                "XfhDVb5mNqNM/IVj0V5fvTc6vOfYbzQtPm10H+FdWWfb+rJRfyC3MA2w2IqstFe3",
+                "w3bu2iE6CQvSqRvge+ZqLKt/NqYwOURiUmpuklbl3kPJ97+mfKWoiqk8Iz1VY+bb",
+                "NMUC7aoGv+jcoj+WS6PYO8N6BeRVUUB3ZJSf8nzjgxm1/BcM+UD3BPrlhT11ODRs",
+                "baifGbprMWwt3dhb8cQgRT8GPdpO1OsDkzL6iikMjLHWWiA99GV6ruiHsIPw6boW",
+                "A6/uSOskbDHOROotKmddGTBd0iiHXAoQsJFt1ZjUkt6EHrgWs+GAvrvKpXs1mrz8",
+                "uj3GwEFrHS+Xuf2UDgpszYT3hI2cL/kUtGakVR7m7vVMZqXBUbZdGAEb1PZNPwsI",
+                "E4aMK02+EVB+tSN4Fzj99N2YD0inVYt+oPjr2tHhUS6aSGBNS/48Ki47DOg4Sxkn",
+                "lkOWnEbCD+XTnbDd",
+                "=agR5",
+                "-----END PGP MESSAGE-----",
+                "",
+                "",
+                "--YFrteb74qSXmggbOxZL9dRnhymywAi--",
+                "",
+                "",
+            ]
+        ).encode()
+    )
+
+    assert not check_encrypted_bstr(
+        "\r\n".join(
+            [
+                "Subject: Buy Penis Enlargement at www.malicious-domain.com",
+                "From: <barbaz@c2.testrun.org>",
+                "To: <foobar@c2.testrun.org>",
+                "Date: Sun, 15 Oct 2023 16:43:21 +0000",
+                "Message-ID: <Mr.UVyJWZmkCKM.hGzNc6glBE_@c2.testrun.org>",
+                "In-Reply-To: <Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>",
+                "References: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>",
+                "\t<Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>",
+                "Chat-Version: 1.0",
+                "Autocrypt: addr=barbaz@c2.testrun.org; prefer-encrypt=mutual;",
+                "\tkeydata=xjMEZSwWjhYJKwYBBAHaRw8BAQdAQBEhqeJh0GueHB6kF/DUQqYCxARNBVokg/AzT+7LqH",
+                "\trNFzxiYXJiYXpAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUsFo4CGwMECwkIBwYVCAkKCwID",
+                "\tFgIBFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX9A4AEAnHWHp49eBCMHK5t66gYPiW",
+                "\tXQuB1mwUjzGfYWB+0RXUoA/0xcQ3FbUNlGKW7Blp6eMFfViv6Mv2d3kNSXACB6nmcMzjgEZSwWjhIK",
+                "\tKwYBBAGXVQEFAQEHQBpY5L2M1XHo0uxf8SX1wNLBp/OVvidoWHQF2Jz+kJsUAwEIB8J4BBgWCAAgBQ",
+                "\tJlLBaOAhsMFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX/INgEA37AJaNvruYsJVanP",
+                "\tIXnYw4CKd55UAwl8Zcy+M2diAbkA/0fHHcGV4r78hpbbL1Os52DPOdqYQRauIeJUeG+G6bQO",
+                "MIME-Version: 1.0",
+                'Content-Type: multipart/encrypted; protocol="application/pgp-encrypted";',
+                '\tboundary="YFrteb74qSXmggbOxZL9dRnhymywAi"',
+                "",
+                "",
+                "--YFrteb74qSXmggbOxZL9dRnhymywAi",
+                "Content-Description: PGP/MIME version identification",
+                "Content-Type: application/pgp-encrypted",
+                "",
+                "Version: 1",
+                "",
+                "",
+                "--YFrteb74qSXmggbOxZL9dRnhymywAi",
+                "Content-Description: OpenPGP encrypted message",
+                'Content-Disposition: inline; filename="encrypted.asc";',
+                'Content-Type: application/octet-stream; name="encrypted.asc"',
+                "",
+                "-----BEGIN PGP MESSAGE-----",
+                "",
+                "wU4DhW3gBZ/VvCYSAQdA8bMs2spwbKdGjVsL1ByPkNrqD7frpB73maeL6I6SzDYg",
+                "O5G53tv339RdKq3WRcCtEEvxjHlUx2XNwXzC04BpmfvBTgNfPUyLDzjXnxIBB0Ae",
+                "8ymwGvXMCCimHXN0Dg8Ui62KOi03h0UgheoHWovJSCDF4CKre/xtFr3nL7lq/PKI",
+                "JsjVNz7/RK9FSXF6WwfONtLCyQGEuVAsB/KXfCBEyfKhaMwGHvhujRidGW5uV1no",
+                "lMGl3ODmo29Lgeu2uSE7EpJRZoe6hU6ddmBkqxax61ZtkaFlGFFpdo2K8balNNdz",
+                "ZsJ/9mmI9x3oOJ4/l1nhQbUO9ADbs7gJhFdV5Qkp30b5fCI7bU+aoe1ccBbLe/WM",
+                "YUty1PqcuQT7XjA+XmYuL261tvW8pBetT+i33/E2d8PzzYt2IuK9qeevyS+yxdwA",
+                "kfwejFWzzsUlJaDxs1x4XOxkMgSj+jo+g12dFOb7fyClsAnq23iDb8AuaT/BScAI",
+                "+lO+gher69+6LmM7VGHLG5k762J1jTaQCaKt1s8TAWV99Eo4491vL6fyvk3l/Cfg",
+                "RXSwiWFgj19Pn0Rq7CD9v22UE2vdUMBTcV4aw79mClk1YQ23jbF0y5DCjPdJ62Zo",
+                "tskBgFt3NoWV80jZ76zIBLrrjLwCCll8JjJtFwSkt2GX5RFBsVa4A8IDht9RtEk7",
+                "rrHgbSZQfkauEi/mH3/6CDZoLqSHudUZ7d4MaJwun1TkFYGe2ORwGJd4OBj3oGJp",
+                "H8YBwCpk///L/fKjX0Gg3M8nrpM4wrRFhPKidAgO/kcm25X4+ZHlVkWBTCt5RWKI",
+                "fHh6oLDZCqCfcgMkE1KKmwfIHaUkhq5BPRigwy6i5dh1DM4+1UCLh3dxzVbqE9b9",
+                "61NB19nXdRtDA2sOUnj9ve6m/wEPyCb6/zBQZqvCBYb1/AjdXpUrFT+DbpfyxaXN",
+                "XfhDVb5mNqNM/IVj0V5fvTc6vOfYbzQtPm10H+FdWWfb+rJRfyC3MA2w2IqstFe3",
+                "w3bu2iE6CQvSqRvge+ZqLKt/NqYwOURiUmpuklbl3kPJ97+mfKWoiqk8Iz1VY+bb",
+                "NMUC7aoGv+jcoj+WS6PYO8N6BeRVUUB3ZJSf8nzjgxm1/BcM+UD3BPrlhT11ODRs",
+                "baifGbprMWwt3dhb8cQgRT8GPdpO1OsDkzL6iikMjLHWWiA99GV6ruiHsIPw6boW",
+                "A6/uSOskbDHOROotKmddGTBd0iiHXAoQsJFt1ZjUkt6EHrgWs+GAvrvKpXs1mrz8",
+                "uj3GwEFrHS+Xuf2UDgpszYT3hI2cL/kUtGakVR7m7vVMZqXBUbZdGAEb1PZNPwsI",
+                "E4aMK02+EVB+tSN4Fzj99N2YD0inVYt+oPjr2tHhUS6aSGBNS/48Ki47DOg4Sxkn",
+                "lkOWnEbCD+XTnbDd",
+                "=agR5",
+                "-----END PGP MESSAGE-----",
+                "",
+                "",
+                "--YFrteb74qSXmggbOxZL9dRnhymywAi--",
+                "",
+                "",
+            ]
+        ).encode()
+    )
+
+    assert not check_encrypted_bstr(
+        "\r\n".join(
+            [
+                "Subject: Message opened",
+                "From: <barbaz@c2.testrun.org>",
+                "To: <foobar@c2.testrun.org>",
+                "Date: Sun, 15 Oct 2023 16:43:25 +0000",
+                "Message-ID: <Mr.78MWtlV7RAi.goCFzBhCYfy@c2.testrun.org>",
+                "Auto-Submitted: auto-replied",
+                "Chat-Version: 1.0",
+                "MIME-Version: 1.0",
+                "Content-Type: multipart/report; report-type=disposition-notification;",
+                '\tboundary="Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi"',
+                "",
+                "",
+                "--Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi",
+                "Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no",
+                "",
+                'The "Hi!" message you sent was displayed on the screen of the recipient.',
+                "",
+                "This is no guarantee the content was read.",
+                "",
+                "",
+                "--Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi",
+                "Content-Type: message/disposition-notification",
+                "",
+                "Reporting-UA: Delta Chat 1.124.1",
+                "Original-Recipient: rfc822;barbaz@c2.testrun.org",
+                "Final-Recipient: rfc822;barbaz@c2.testrun.org",
+                "Original-Message-ID: <Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>",
+                "Disposition: manual-action/MDN-sent-automatically; displayed",
+                "",
+                "",
+                "--Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi--",
+                "",
+                "",
+            ]
+        ).encode()
+    )
+
+
+def test_send_rate_limiter():
+    limiter = SendRateLimiter()
+    for i in range(100):
+        if limiter.is_sending_allowed("some@example.org"):
+            if i <= SendRateLimiter.MAX_USER_SEND_PER_MINUTE:
+                continue
+            pytest.fail("limiter didn't work")
+        else:
+            assert i == SendRateLimiter.MAX_USER_SEND_PER_MINUTE + 1
+            break

deploy-chatmail/src/deploy_chatmail/acmetool/__init__.py

@@ -1,6 +1,6 @@
 import importlib.resources
 
-from pyinfra.operations import apt, files, server
+from pyinfra.operations import apt, files, systemd, server
 
 
 def deploy_acmetool(nginx_hook=False, email="", domains=[]):

online-tests/benchmark.py

@@ -0,0 +1,34 @@
+def test_tls_serialized_connect(benchmark, imap_or_smtp):
+    def connect():
+        imap_or_smtp.connect()
+
+    benchmark(connect)
+
+
+def test_login(benchmark, imap_or_smtp, gencreds):
+    cls = imap_or_smtp.__class__
+    conns = []
+    for i in range(20):
+        conn = cls(imap_or_smtp.host)
+        conn.connect()
+        conns.append(conn)
+
+    def login():
+        conn = conns.pop()
+        conn.login(*gencreds())
+
+    benchmark(login)
+
+
+def test_send_and_receive_10(benchmark, cmfactory, lp):
+    """send many messages between two accounts"""
+    ac1, ac2 = cmfactory.get_online_accounts(2)
+    chat = cmfactory.get_accepted_chat(ac1, ac2)
+
+    def send_10_receive_all():
+        for i in range(10):
+            chat.send_text(f"hello {i}")
+        for i in range(10):
+            ac2.wait_next_incoming_message()
+
+    benchmark(send_10_receive_all)

online-tests/conftest.py

@@ -1,34 +1,19 @@
 import os
 import io
-import time
 import random
 import subprocess
 import imaplib
 import smtplib
 import itertools
-from email.parser import BytesParser
-from email import policy
-from pathlib import Path
-from math import ceil
 import pytest
 
 
-conftestdir = Path(__file__).parent
-
-
 def pytest_addoption(parser):
     parser.addoption(
         "--slow", action="store_true", default=False, help="also run slow tests"
     )
 
 
-def pytest_configure(config):
-    config._benchresults = {}
-    config.addinivalue_line(
-        "markers", "slow: mark test to require --slow option to run"
-    )
-
-
 def pytest_runtest_setup(item):
     markers = list(item.iter_markers(name="slow"))
     if markers:
@@ -69,49 +54,6 @@ def pytest_report_header():
         return ["-" * len(text), text, "-" * len(text)]
 
 
-@pytest.fixture
-def benchmark(request):
-    def bench(func, num, name=None):
-        if name is None:
-            name = func.__name__
-        durations = []
-        for i in range(num):
-            now = time.time()
-            func()
-            durations.append(time.time() - now)
-        durations.sort()
-        request.config._benchresults[name] = durations
-
-    return bench
-
-
-def pytest_terminal_summary(terminalreporter):
-    tr = terminalreporter
-    results = tr.config._benchresults
-    if not results:
-        return
-
-    tr.section("benchmark results")
-    float_names = "median min max".split()
-    width = max(map(len, float_names))
-
-    def fcol(parts):
-        return " ".join(part.rjust(width) for part in parts)
-
-    headers = f"{'benchmark name': <30} " + fcol(float_names)
-    tr.write_line(headers)
-    tr.write_line("-" * len(headers))
-    for name, durations in results.items():
-        measures = [
-            sorted(durations)[len(durations) // 2],
-            min(durations),
-            max(durations),
-        ]
-        line = f"{name: <30} "
-        line += fcol(f"{float: 2.2f}" for float in measures)
-        tr.write_line(line)
-
-
 @pytest.fixture
 def imap(maildomain):
     return ImapConn(maildomain)
@@ -287,19 +229,13 @@ class Remote:
 
 
 @pytest.fixture
-def maildata(request, gencreds):
+def mailgen(request):
-    datadir = conftestdir.joinpath("mail-data")
+    class Mailgen:
+        def get_encrypted(self, from_addr, to_addr):
+            data = request.fspath.dirpath("mailgen/encrypted.eml").read()
+            return data.format(from_addr=from_addr, to_addr=to_addr)
 
-    def maildata(name, from_addr=None, to_addr=None):
+    return Mailgen()
-        if from_addr is None:
-            from_addr = gencreds()[0]
-        if to_addr is None:
-            to_addr = gencreds()[0]
-        data = datadir.joinpath(name).read_text()
-        text = data.format(from_addr=from_addr, to_addr=to_addr)
-        return BytesParser(policy=policy.default).parsebytes(text.encode())
-
-    return maildata
 
 
 @pytest.fixture

online-tests/pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+addopts = -vrsx --strict-markers
+markers = slow: mark test as slow (requires --slow option to run)

online-tests/test_0_basic.py

@@ -20,7 +20,7 @@ def test_use_two_chatmailservers(cmfactory, maildomain2):
 
 
 @pytest.mark.parametrize("forgeaddr", ["internal", "someone@example.org"])
-def test_reject_forged_from(cmsetup, maildata, lp, forgeaddr):
+def test_reject_forged_from(cmsetup, mailgen, lp, forgeaddr):
     user1, user3 = cmsetup.gen_users(2)
 
     lp.sec("send encrypted message with forged from")
@@ -31,8 +31,7 @@ def test_reject_forged_from(cmsetup, maildata, lp, forgeaddr):
         addr_to_forge = "someone@example.org"
 
     print("message to inject:")
-    msg = maildata("encrypted.eml", from_addr=addr_to_forge, to_addr=user3.addr)
+    msg = mailgen.get_encrypted(from_addr=addr_to_forge, to_addr=user3.addr)
-    msg = msg.as_string()
     for line in msg.split("\n")[:4]:
         print(f"   {line}")
 
@@ -43,12 +42,10 @@ def test_reject_forged_from(cmsetup, maildata, lp, forgeaddr):
 
 
 @pytest.mark.slow
-def test_exceed_rate_limit(cmsetup, gencreds, maildata):
+def test_exceed_rate_limit(cmsetup, gencreds, mailgen):
     """Test that the per-account send-mail limit is exceeded."""
     user1, user2 = cmsetup.gen_users(2)
-    mail = maildata(
+    mail = mailgen.get_encrypted(user1.addr, user2.addr)
-        "encrypted.eml", from_addr=user1.addr, to_addr=user2.addr
-    ).as_string()
     for i in range(100):
         print("Sending mail", str(i))
         try:
@@ -58,6 +55,6 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata):
                 pytest.fail(f"rate limit was exceeded too early with msg {i}")
             outcome = e.recipients[user2.addr]
             assert outcome[0] == 450
-            assert b"4.7.1: Too much mail from" in outcome[1]
+            assert b'4.7.1: Too much mail from' in outcome[1]
             return
     pytest.fail("Rate limit was not exceeded")

online-tests/test_1_deltachat.py

@@ -1,4 +1,3 @@
-import time
 import random
 import pytest
 
@@ -82,29 +81,3 @@ class TestEndToEndDeltaChat:
         ch = ac2.qr_setup_contact(qr)
         assert ch.id >= 10
         ac1._evtracker.wait_securejoin_inviter_progress(1000)
-
-    def test_read_receipts_between_instances(self, cmfactory, lp, maildomain2):
-        ac1 = cmfactory.new_online_configuring_account(cache=False)
-        cmfactory.switch_maildomain(maildomain2)
-        ac2 = cmfactory.new_online_configuring_account(cache=False)
-        cmfactory.bring_accounts_online()
-
-        lp.sec("setup encrypted comms between ac1 and ac2 on different instances")
-        qr = ac1.get_setup_contact_qr()
-        ch = ac2.qr_setup_contact(qr)
-        msg = ac2.wait_next_incoming_message()
-        assert "verified" in msg.text
-
-        lp.sec("ac1 sends a message and ac2 marks it as seen")
-        chat = ac1.create_chat(ac2)
-        msg = chat.send_text("hi")
-        m = ac2.wait_next_incoming_message()
-        m.mark_seen()
-        # we can only indirectly wait for mark-seen to cause an smtp-error
-        lp.sec("try to wait for markseen to complete and check error states")
-        deadline = time.time() + 3.1
-        while time.time() < deadline:
-            msgs = m.chat.get_messages()
-            for msg in msgs:
-                assert "error" not in m.get_message_info()
-            time.sleep(1)

scripts/init.sh

@@ -1,8 +1,13 @@
 #!/bin/sh
 set -e
-python3 -m venv venv
+python3 -m venv deploy-chatmail/venv
-pip=venv/bin/pip
+deploy-chatmail/venv/bin/pip install pyinfra pytest
+deploy-chatmail/venv/bin/pip install -e deploy-chatmail
+deploy-chatmail/venv/bin/pip install -e chatmaild
 
-$pip install pyinfra pytest build 'setuptools>=68' tox deltachat
+python3 -m venv chatmaild/venv
-$pip install -e deploy-chatmail 
+chatmaild/venv/bin/pip install --upgrade pytest build 'setuptools>=68'
-$pip install -e chatmaild 
+chatmaild/venv/bin/pip install -e chatmaild
+
+python3 -m venv online-tests/venv
+online-tests/venv/bin/pip install pytest pytest-timeout pdbpp deltachat pytest-benchmark

scripts/test.sh

@@ -1,4 +1,3 @@
 #!/bin/bash
-venv/bin/tox -c chatmaild
+chatmaild/venv/bin/pytest chatmaild/ $@
-venv/bin/tox -c deploy-chatmail
+online-tests/venv/bin/pytest online-tests/ -vrx --durations=5 $@
-venv/bin/pytest tests/online -vrx --durations=5 $@

tests/chatmaild/test_filtermail.py

@@ -1,82 +0,0 @@
-from chatmaild.filtermail import check_encrypted, check_DATA, SendRateLimiter, check_mdn
-import pytest
-
-
-@pytest.fixture
-def maildomain():
-    # let's not depend on a real chatmail instance for the offline tests below
-    return "chatmail.example.org"
-
-
-def test_reject_forged_from(maildata, gencreds):
-    class env:
-        mail_from = gencreds()[0]
-        rcpt_tos = [gencreds()[0]]
-
-    # test that the filter lets good mail through
-    env.content = maildata("plain.eml", from_addr=env.mail_from).as_bytes()
-    assert not check_DATA(envelope=env)
-
-    # test that the filter rejects forged mail
-    env.content = maildata("plain.eml", from_addr="forged@c3.testrun.org").as_bytes()
-    error = check_DATA(envelope=env)
-    assert "500" in error
-
-
-def test_filtermail_no_encryption_detection(maildata):
-    msg = maildata("plain.eml")
-    assert not check_encrypted(msg)
-
-    # https://xkcd.com/1181/
-    msg = maildata("fake-encrypted.eml")
-    assert not check_encrypted(msg)
-
-
-def test_filtermail_encryption_detection(maildata):
-    msg = maildata("encrypted.eml")
-    assert check_encrypted(msg)
-
-    # if the subject is not "..." it is not considered ac-encrypted
-    msg.replace_header("Subject", "Click this link")
-    assert not check_encrypted(msg)
-
-
-def test_filtermail_is_mdn(maildata, gencreds):
-    from_addr = gencreds()[0]
-    to_addr = gencreds()[0] + ".other"
-    msg = maildata("mdn.eml", from_addr, to_addr)
-
-    class env:
-        mail_from = from_addr
-        rcpt_tos = [to_addr]
-        content = msg.as_bytes()
-
-    assert check_mdn(msg, env)
-    print(msg.as_string())
-    assert not check_DATA(env)
-
-
-def test_filtermail_to_multiple_recipients_no_mdn(maildata, gencreds):
-    from_addr = gencreds()[0]
-    to_addr = gencreds()[0] + ".other"
-    thirdaddr = gencreds()[0]
-    msg = maildata("mdn.eml", from_addr, to_addr)
-
-    class env:
-        mail_from = from_addr
-        rcpt_tos = [to_addr, thirdaddr]
-        content = msg.as_bytes()
-
-    assert not check_mdn(msg, env)
-
-
-def test_send_rate_limiter():
-    limiter = SendRateLimiter()
-    for i in range(100):
-        if limiter.is_sending_allowed("some@example.org"):
-            if i <= SendRateLimiter.MAX_USER_SEND_PER_MINUTE:
-                continue
-            pytest.fail("limiter didn't work")
-        else:
-            assert i == SendRateLimiter.MAX_USER_SEND_PER_MINUTE + 1
-            break

tests/mail-data/fake-encrypted.eml

@@ -1,25 +0,0 @@
-Subject: =?utf-8?q?Message_from_foobar=40c2=2Etestrun=2Eorg?=
-Chat-Disposition-Notification-To: foobar@c2.testrun.org
-Chat-User-Avatar: 0
-From: <{from_addr}>
-To: <{to_addr}>
-Date: Sun, 15 Oct 2023 16:41:44 +0000
-Message-ID: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>
-References: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>
-Chat-Version: 1.0
-Autocrypt: addr={from_addr}; prefer-encrypt=mutual;
-	keydata=xjMEZSrw3hYJKwYBBAHaRw8BAQdAiEKNQFU28c6qsx4vo/JHdt73RXdjMOmByf/XsGiJ7m
-	nNFzxmb29iYXJAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUq8N4CGwMECwkIBwYVCAkKCwID
-	FgIBFiEEGil0OvTIa6RngmCLUYNnEa9leJAACgkQUYNnEa9leJCX3gEAhm0MehE5byBBU1avPczr/I
-	HjNLht7Qf6++mAhlJmtDcA/0C8VYJhsUpmiDjuZaMDWNv4FO2BJG6LH7gSm6n7ClMJzjgEZSrw3hIK
-	KwYBBAGXVQEFAQEHQAxGG/QW0owCfMp1A+vXEMwgzWcBpNFr58kX2eXuPpM6AwEIB8J4BBgWCAAgBQ
-	JlKvDeAhsMFiEEGil0OvTIa6RngmCLUYNnEa9leJAACgkQUYNnEa9leJDg1gEAwLf8KDoAAKyYgjyI
-	vYvO9VEgBni1C4Xx1VjcaEmlDK8BALoFuUCK+enw76TtDcAUKhlhUiM6SDRExkS4Nskp/BcK
-MIME-Version: 1.0
-Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
-
------BEGIN PGP MESSAGE-----
-Hi!
------END PGP MESSAGE-----
-
-

tests/mail-data/mdn.eml

@@ -1,33 +0,0 @@
-Subject: Message opened
-From: <{from_addr}>
-To: <{to_addr}>
-Date: Sun, 15 Oct 2023 16:43:25 +0000
-Message-ID: <Mr.78MWtlV7RAi.goCFzBhCYfy@c2.testrun.org>
-Auto-Submitted: auto-replied
-Chat-Version: 1.0
-MIME-Version: 1.0
-Content-Type: multipart/report; report-type=disposition-notification;
-	boundary="Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi"
-
-
---Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi
-Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
-
-The "Hi!" message you sent was displayed on the screen of the recipient.
-
-This is no guarantee the content was read.
-
-
---Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi
-Content-Type: message/disposition-notification
-
-Reporting-UA: Delta Chat 1.124.1
-Original-Recipient: rfc822;barbaz@c2.testrun.org
-Final-Recipient: rfc822;barbaz@c2.testrun.org
-Original-Message-ID: <Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>
-Disposition: manual-action/MDN-sent-automatically; displayed
-
-
---Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi--
-
-

tests/mail-data/plain.eml

@@ -1,23 +0,0 @@
-Subject: =?utf-8?q?Message_from_foobar=40c2=2Etestrun=2Eorg?=
-Chat-Disposition-Notification-To: foobar@c2.testrun.org
-Chat-User-Avatar: 0
-From: <{from_addr}>
-To: <{to_addr}>
-Date: Sun, 15 Oct 2023 16:41:44 +0000
-Message-ID: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>
-References: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>
-Chat-Version: 1.0
-Autocrypt: addr=foobar@c2.testrun.org; prefer-encrypt=mutual;
-	keydata=xjMEZSrw3hYJKwYBBAHaRw8BAQdAiEKNQFU28c6qsx4vo/JHdt73RXdjMOmByf/XsGiJ7m
-	nNFzxmb29iYXJAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUq8N4CGwMECwkIBwYVCAkKCwID
-	FgIBFiEEGil0OvTIa6RngmCLUYNnEa9leJAACgkQUYNnEa9leJCX3gEAhm0MehE5byBBU1avPczr/I
-	HjNLht7Qf6++mAhlJmtDcA/0C8VYJhsUpmiDjuZaMDWNv4FO2BJG6LH7gSm6n7ClMJzjgEZSrw3hIK
-	KwYBBAGXVQEFAQEHQAxGG/QW0owCfMp1A+vXEMwgzWcBpNFr58kX2eXuPpM6AwEIB8J4BBgWCAAgBQ
-	JlKvDeAhsMFiEEGil0OvTIa6RngmCLUYNnEa9leJAACgkQUYNnEa9leJDg1gEAwLf8KDoAAKyYgjyI
-	vYvO9VEgBni1C4Xx1VjcaEmlDK8BALoFuUCK+enw76TtDcAUKhlhUiM6SDRExkS4Nskp/BcK
-MIME-Version: 1.0
-Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
-
-Hi!
-
-

tests/online/benchmark.py

@@ -1,60 +0,0 @@
-def test_tls_imap(benchmark, imap):
-    def imap_connect():
-        imap.connect()
-
-    benchmark(imap_connect, 10)
-
-
-def test_login_imap(benchmark, imap, gencreds):
-    def imap_connect_and_login():
-        imap.connect()
-        imap.login(*gencreds())
-
-    benchmark(imap_connect_and_login, 10)
-
-
-def test_tls_smtp(benchmark, smtp):
-    def smtp_connect():
-        smtp.connect()
-
-    benchmark(smtp_connect, 10)
-
-
-def test_login_smtp(benchmark, smtp, gencreds):
-    def smtp_connect_and_login():
-        smtp.connect()
-        smtp.login(*gencreds())
-
-    benchmark(smtp_connect_and_login, 10)
-
-
-class TestDC:
-    def test_autoconfigure(self, benchmark, cmfactory):
-        def autoconfig_and_idle_ready():
-            cmfactory.get_online_accounts(1)
-
-        benchmark(autoconfig_and_idle_ready, 5)
-
-    def test_ping_pong(self, benchmark, cmfactory):
-        ac1, ac2 = cmfactory.get_online_accounts(2)
-        chat = cmfactory.get_accepted_chat(ac1, ac2)
-
-        def ping_pong():
-            chat.send_text("ping")
-            msg = ac2.wait_next_incoming_message()
-            msg.chat.send_text("pong")
-            ac1.wait_next_incoming_message()
-
-        benchmark(ping_pong, 5)
-
-    def test_send_10_receive_10(self, benchmark, cmfactory, lp):
-        ac1, ac2 = cmfactory.get_online_accounts(2)
-        chat = cmfactory.get_accepted_chat(ac1, ac2)
-
-        def send_10_receive_10():
-            for i in range(10):
-                chat.send_text(f"hello {i}")
-            for i in range(10):
-                ac2.wait_next_incoming_message()
-
-        benchmark(send_10_receive_10, 5)

tests/pytest.ini

@@ -1,2 +0,0 @@
-[pytest]
-addopts = -vrsx --strict-markers