add a test for imap capabilities offered from chatmail

.github/workflows/ci.yaml

@@ -9,7 +9,7 @@ jobs:
     name: isolated chatmaild tests
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
 
       - name: run chatmaild tests 
         working-directory: chatmaild
@@ -19,7 +19,7 @@ jobs:
     name: deploy-chatmail tests 
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
 
       - name: initenv 
         run: scripts/initenv.sh

.github/workflows/staging.testrun.org-default.zone

@@ -1,6 +1,6 @@
-;; Zone file for staging2.testrun.org
+;; Zone file for staging.testrun.org
 
-$ORIGIN staging2.testrun.org.
+$ORIGIN staging.testrun.org.
 $TTL 300
 
 @  IN  SOA     ns.testrun.org. root.nine.testrun.org (
@@ -15,7 +15,6 @@ $TTL 300
 @  IN NS ns.testrun.org.
 
 ;; DNS records.
-@       IN      A       37.27.24.139
-mta-sts.staging2.testrun.org.    CNAME   staging2.testrun.org.
-www.staging2.testrun.org.        CNAME   staging2.testrun.org.
-
+@       IN      A       37.27.37.98
+mta-sts.staging.testrun.org.    CNAME   staging.testrun.org.
+www.staging.testrun.org.        CNAME   staging.testrun.org.

.github/workflows/test-and-deploy.yaml

@@ -1,4 +1,4 @@
-name: deploy on staging2.testrun.org, and run tests
+name: deploy on staging.testrun.org, and run tests
 
 on:
   push:
@@ -7,37 +7,31 @@ on:
   pull_request:
     paths-ignore:
       - 'scripts/**'
-      - '**/README.md'
-      - 'CHANGELOG.md'
-      - 'LICENSE'
 
 jobs:
   deploy:
-    name: deploy on staging2.testrun.org, and run tests
+    name: deploy on staging.testrun.org, and run tests
     runs-on: ubuntu-latest
     concurrency:
       group: staging-deploy
       cancel-in-progress: true
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
 
       - name: prepare SSH
         run: |
           mkdir ~/.ssh
           echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
           chmod 600 ~/.ssh/id_ed25519
-          ssh-keyscan staging2.testrun.org > ~/.ssh/known_hosts
+          ssh-keyscan staging.testrun.org > ~/.ssh/known_hosts
           # save previous acme & dkim state
-          rsync -avz root@staging2.testrun.org:/var/lib/acme . || true
-          rsync -avz root@staging2.testrun.org:/etc/dkimkeys . || true
+          rsync -avz root@staging.testrun.org:/var/lib/acme . || true
+          rsync -avz root@staging.testrun.org:/etc/dkimkeys . || true
           # store previous acme & dkim state on ns.testrun.org, if it contains useful certs
           if [ -f dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys root@ns.testrun.org:/tmp/ || true; fi
-          if [ "$(ls -A acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme root@ns.testrun.org:/tmp/ || true; fi
-          # make sure CAA record isn't set
-          ssh -o StrictHostKeyChecking=accept-new root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging2.testrun.org.zone
-          ssh root@ns.testrun.org systemctl reload nsd
+          if [ -z "$(ls -A acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme root@ns.testrun.org:/tmp/ || true; fi
 
-      - name: rebuild staging2.testrun.org to have a clean VPS
+      - name: rebuild staging.testrun.org to have a clean VPS
         run: |
             curl -X POST \
             -H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
@@ -52,17 +46,17 @@ jobs:
 
       - name: upload TLS cert after rebuilding
         run: |
-          echo " --- wait until staging2.testrun.org VPS is rebuilt --- "
+          echo " --- wait until staging.testrun.org VPS is rebuilt --- "
           rm ~/.ssh/known_hosts
-          while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org id -u ; do sleep 1 ; done
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org id -u
+          while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u ; do sleep 1 ; done
+          ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u
           # download acme & dkim state from ns.testrun.org
           rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme acme-restore || true
           rsync -avz root@ns.testrun.org:/tmp/dkimkeys dkimkeys-restore || true
-          # restore acme & dkim state to staging2.testrun.org
-          rsync -avz acme-restore/acme/ root@staging2.testrun.org:/var/lib/acme || true
-          rsync -avz dkimkeys-restore/dkimkeys/ root@staging2.testrun.org:/etc/dkimkeys || true
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org chown root:root -R /var/lib/acme || true
+          # restore acme & dkim state to staging.testrun.org
+          rsync -avz acme-restore/acme/ root@staging.testrun.org:/var/lib/acme || true
+          rsync -avz dkimkeys-restore/dkimkeys/ root@staging.testrun.org:/etc/dkimkeys || true
+          ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org chown root:root -R /var/lib/acme || true
 
       - name: run formatting checks 
         run: cmdeploy fmt -v 
@@ -70,18 +64,18 @@ jobs:
       - name: run deploy-chatmail offline tests 
         run: pytest --pyargs cmdeploy 
 
-      - run: cmdeploy init staging2.testrun.org
+      - run: cmdeploy init staging.testrun.org
 
       - run: cmdeploy run
 
       - name: set DNS entries
         run: |
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
+          ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
           cmdeploy dns --zonefile staging-generated.zone
           cat staging-generated.zone >> .github/workflows/staging.testrun.org-default.zone
           cat .github/workflows/staging.testrun.org-default.zone
-          scp .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
-          ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
+          scp .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging.testrun.org.zone
+          ssh root@ns.testrun.org nsd-checkzone staging.testrun.org /etc/nsd/staging.testrun.org.zone
           ssh root@ns.testrun.org systemctl reload nsd
 
       - name: cmdeploy test

CHANGELOG.md

@@ -2,46 +2,6 @@
 
 ## untagged
 
-- Test and fix for attempts to create inadmissible accounts 
-  ([#333](https://github.com/deltachat/chatmail/pull/321))
-
-- check that OpenPGP has only PKESK, SKESK and SEIPD packets
-  ([#323](https://github.com/deltachat/chatmail/pull/323),
-   [#324](https://github.com/deltachat/chatmail/pull/324))
-
-- improve filtermail checks for encrypted messages and drop support for unencrypted MDNs
-  ([#320](https://github.com/deltachat/chatmail/pull/320))
-
-- replace `bash` with `/bin/sh`
-  ([#334](https://github.com/deltachat/chatmail/pull/334))
-
-- Increase number of logged in IMAP sessions to 50000
-  ([#335](https://github.com/deltachat/chatmail/pull/335))
-
-- filtermail: do not allow ASCII armor without actual payload
-  ([#325](https://github.com/deltachat/chatmail/pull/325))
-
-## 1.3.0 - 2024-06-06
-
-- don't check necessary DNS records on cmdeploy init anymore
-  ([#316](https://github.com/deltachat/chatmail/pull/316))
-
-- ensure cron and acl are installed
-  ([#293](https://github.com/deltachat/chatmail/pull/293),
-  [#310](https://github.com/deltachat/chatmail/pull/310))
-
-- change default for delete_mails_after from 40 to 20 days
-  ([#300](https://github.com/deltachat/chatmail/pull/300))
-
-- save journald logs only to memory and save nginx logs to journald instead of file
-  ([#299](https://github.com/deltachat/chatmail/pull/299))
-
-- fix writing of multiple obs repositories in `/etc/apt/sources.list`
-  ([#290](https://github.com/deltachat/chatmail/pull/290))
-
-- metadata: add support for `/shared/vendor/deltachat/irohrelay`
-  ([#284](https://github.com/deltachat/chatmail/pull/284))
-
 - Emit "XCHATMAIL" capability from IMAP server 
   ([#278](https://github.com/deltachat/chatmail/pull/278))
 

README.md

@@ -15,8 +15,6 @@ after which the initially specified password is required for using them.
 
 ## Deploying your own chatmail server 
 
-To deploy chatmail on your own server, you must have set-up ssh authentication and need to use an ed25519 key, due to an [upstream bug in paramiko](https://github.com/paramiko/paramiko/issues/2191). You also need to add your private key to the local ssh-agent, because you can't type in your password during deployment.
-
 We use `chat.example.org` as the chatmail domain in the following steps. 
 Please substitute it with your own domain. 
 

chatmaild/pyproject.toml

@@ -36,16 +36,6 @@ log_format = "%(asctime)s %(levelname)s %(message)s"
 log_date_format = "%Y-%m-%d %H:%M:%S"
 log_level = "INFO"
 
-[tool.ruff]
-lint.select = [
-  "F", # Pyflakes
-  "I", # isort
-
-  "PLC", # Pylint Convention
-  "PLE", # Pylint Error
-  "PLW", # Pylint Warning
-]
-
 [tool.tox]
 legacy_tox_ini = """
 [tox]
@@ -57,9 +47,10 @@ skipdist = True
 skip_install = True
 deps =
   ruff
+  black
 commands =
-  ruff format --quiet --diff src/
-  ruff check src/
+  black --quiet --check --diff src/
+  ruff src/
 
 [testenv]
 deps = pytest 

chatmaild/src/chatmaild/config.py

@@ -20,7 +20,6 @@ class Config:
         self.passthrough_recipients = params["passthrough_recipients"].split()
         self.filtermail_smtp_port = int(params["filtermail_smtp_port"])
         self.postfix_reinject_port = int(params["postfix_reinject_port"])
-        self.iroh_relay = params.get("iroh_relay")
         self.privacy_postal = params.get("privacy_postal")
         self.privacy_mail = params.get("privacy_mail")
         self.privacy_pdo = params.get("privacy_pdo")

chatmaild/src/chatmaild/database.py

@@ -1,5 +1,5 @@
-import contextlib
 import sqlite3
+import contextlib
 import time
 from pathlib import Path
 

chatmaild/src/chatmaild/doveauth.py

@@ -1,18 +1,18 @@
-import crypt
-import json
 import logging
 import os
-import sys
 import time
+import sys
+import json
+import crypt
 from pathlib import Path
 from socketserver import (
+    UnixStreamServer,
     StreamRequestHandler,
     ThreadingMixIn,
-    UnixStreamServer,
 )
 
-from .config import Config, read_config
 from .database import Database
+from .config import read_config, Config
 
 NOCREATE_FILE = "/etc/chatmail-nocreate"
 
@@ -60,7 +60,6 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
             config.username_min_length,
             config.username_max_length,
         )
-        return False
 
     return True
 

chatmaild/src/chatmaild/echo.py

@@ -6,11 +6,11 @@ it will echo back any message that has non-empty text and also supports the /hel
 
 import logging
 import os
-import subprocess
 import sys
-from pathlib import Path
+import subprocess
 
 from deltachat_rpc_client import Bot, DeltaChat, EventType, Rpc, events
+from pathlib import Path
 
 from chatmaild.config import read_config
 from chatmaild.newemail import create_newemail_dict

chatmaild/src/chatmaild/filedict.py

@@ -1,9 +1,8 @@
-import json
-import logging
 import os
-from contextlib import contextmanager
-
+import logging
+import json
 import filelock
+from contextlib import contextmanager
 
 
 class FileDict:

chatmaild/src/chatmaild/filtermail.py

@@ -1,114 +1,20 @@
 #!/usr/bin/env python3
 import asyncio
-import base64
-import binascii
 import logging
-import sys
 import time
-from email import policy
+import sys
 from email.parser import BytesParser
+from email import policy
 from email.utils import parseaddr
-from smtplib import SMTP as SMTPClient
 
 from aiosmtpd.controller import Controller
+from smtplib import SMTP as SMTPClient
 
 from .config import read_config
 
 
-def check_openpgp_payload(payload: bytes):
-    """Checks the OpenPGP payload.
-
-    OpenPGP payload must consist only of PKESK and SKESK packets
-    terminated by a single SEIPD packet.
-
-    Returns True if OpenPGP payload is correct,
-    False otherwise.
-
-    May raise IndexError while trying to read OpenPGP packet header
-    if it is truncated.
-    """
-    i = 0
-    while i < len(payload):
-        # Only OpenPGP format is allowed.
-        if payload[i] & 0xC0 != 0xC0:
-            return False
-
-        packet_type_id = payload[i] & 0x3F
-        i += 1
-        if payload[i] < 192:
-            # One-octet length.
-            body_len = payload[i]
-            i += 1
-        elif payload[i] < 224:
-            # Two-octet length.
-            body_len = ((payload[i] - 192) << 8) + payload[i + 1] + 192
-            i += 2
-        elif payload[i] == 255:
-            # Five-octet length.
-            body_len = (
-                (payload[i + 1] << 24)
-                | (payload[i + 2] << 16)
-                | (payload[i + 3] << 8)
-                | payload[i + 4]
-            )
-            i += 5
-        else:
-            # Partial body length is not allowed.
-            return False
-
-        i += body_len
-
-        if i == len(payload):
-            if packet_type_id == 18:
-                # Last packet should be
-                # Symmetrically Encrypted and Integrity Protected Data Packet (SEIPD)
-                return True
-        elif packet_type_id not in [1, 3]:
-            # All packets except the last one must be either
-            # Public-Key Encrypted Session Key Packet (PKESK)
-            # or
-            # Symmetric-Key Encrypted Session Key Packet (SKESK)
-            return False
-
-    if i == 0:
-        return False
-
-    if i > len(payload):
-        # Payload is truncated.
-        return False
-    return True
-
-
-def check_armored_payload(payload: str):
-    prefix = "-----BEGIN PGP MESSAGE-----\r\n\r\n"
-    if not payload.startswith(prefix):
-        return False
-    payload = payload.removeprefix(prefix)
-
-    suffix = "-----END PGP MESSAGE-----\r\n\r\n"
-    if not payload.endswith(suffix):
-        return False
-    payload = payload.removesuffix(suffix)
-
-    # Remove CRC24.
-    payload = payload.rpartition("=")[0]
-
-    try:
-        payload = base64.b64decode(payload)
-    except binascii.Error:
-        return False
-
-    try:
-        return check_openpgp_payload(payload)
-    except IndexError:
-        return False
-
-
 def check_encrypted(message):
-    """Check that the message is an OpenPGP-encrypted message.
-
-    MIME structure of the message must correspond to <https://www.rfc-editor.org/rfc/rfc3156>.
-    """
+    """Check that the message is an OpenPGP-encrypted message."""
     if not message.is_multipart():
         return False
     if message.get("subject") != "...":
@@ -117,30 +23,46 @@ def check_encrypted(message):
         return False
     parts_count = 0
     for part in message.iter_parts():
-        # We explicitly check Content-Type of each part later,
-        # but this is to be absolutely sure `get_payload()` returns string and not list.
-        if part.is_multipart():
-            return False
-
         if parts_count == 0:
             if part.get_content_type() != "application/pgp-encrypted":
                 return False
-
-            payload = part.get_payload()
-            if payload.strip() != "Version: 1":
-                return False
         elif parts_count == 1:
             if part.get_content_type() != "application/octet-stream":
                 return False
-
-            if not check_armored_payload(part.get_payload()):
-                return False
         else:
             return False
         parts_count += 1
     return True
 
 
+def check_mdn(message, envelope):
+    if len(envelope.rcpt_tos) != 1:
+        return False
+
+    for name in ["auto-submitted", "chat-version"]:
+        if not message.get(name):
+            return False
+
+    if message.get_content_type() != "multipart/report":
+        return False
+
+    body = message.get_body()
+    if body.get_content_type() != "text/plain":
+        return False
+
+    if list(body.iter_attachments()) or list(body.iter_parts()):
+        return False
+
+    # even with all mime-structural checks an attacker
+    # could try to abuse the subject or body to contain links or other
+    # annoyance -- we skip on checking subject/body for now as Delta Chat
+    # should evolve to create E2E-encrypted read receipts anyway.
+    # and then MDNs are just encrypted mail and can pass the border
+    # to other instances.
+
+    return True
+
+
 async def asyncmain_beforequeue(config):
     port = config.filtermail_smtp_port
     Controller(BeforeQueueHandler(config), hostname="127.0.0.1", port=port).start()
@@ -186,6 +108,9 @@ class BeforeQueueHandler:
         if envelope.mail_from.lower() != from_addr.lower():
             return f"500 Invalid FROM <{from_addr!r}> for <{envelope.mail_from!r}>"
 
+        if not mail_encrypted and check_mdn(message, envelope):
+            return
+
         if envelope.mail_from in self.config.passthrough_senders:
             return
 

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -18,7 +18,7 @@ max_user_send_per_minute = 60
 max_mailbox_size = 100M
 
 # days after which mails are unconditionally deleted
-delete_mails_after = 20
+delete_mails_after = 40
 
 # minimum length a username must have
 username_min_length = 9

chatmaild/src/chatmaild/metadata.py

@@ -1,17 +1,17 @@
-import logging
-import os
-import sys
 from pathlib import Path
 from socketserver import (
+    UnixStreamServer,
     StreamRequestHandler,
     ThreadingMixIn,
-    UnixStreamServer,
 )
+import sys
+import logging
+import os
 
-from .config import read_config
 from .filedict import FileDict
 from .notifier import Notifier
 
+
 DICTPROXY_HELLO_CHAR = "H"
 DICTPROXY_LOOKUP_CHAR = "L"
 DICTPROXY_ITERATE_CHAR = "I"
@@ -49,40 +49,32 @@ class Metadata:
         return mdict.get(self.DEVICETOKEN_KEY, [])
 
 
-def handle_dovecot_protocol(rfile, wfile, notifier, metadata, iroh_relay=None):
+def handle_dovecot_protocol(rfile, wfile, notifier, metadata):
     transactions = {}
     while True:
         msg = rfile.readline().strip().decode()
         if not msg:
             break
 
-        res = handle_dovecot_request(msg, transactions, notifier, metadata, iroh_relay)
+        res = handle_dovecot_request(msg, transactions, notifier, metadata)
         if res:
             wfile.write(res.encode("ascii"))
             wfile.flush()
 
 
-def handle_dovecot_request(msg, transactions, notifier, metadata, iroh_relay=None):
+def handle_dovecot_request(msg, transactions, notifier, metadata):
     # see https://doc.dovecot.org/3.0/developer_manual/design/dict_protocol/
     short_command = msg[0]
     parts = msg[1:].split("\t")
     if short_command == DICTPROXY_LOOKUP_CHAR:
         # Lpriv/43f5f508a7ea0366dff30200c15250e3/devicetoken\tlkj123poi@c2.testrun.org
-        keyparts = parts[0].split("/", 2)
+        keyparts = parts[0].split("/")
         if keyparts[0] == "priv":
             keyname = keyparts[2]
             addr = parts[1]
             if keyname == metadata.DEVICETOKEN_KEY:
                 res = " ".join(metadata.get_tokens_for_addr(addr))
                 return f"O{res}\n"
-        elif keyparts[0] == "shared":
-            keyname = keyparts[2]
-            if (
-                keyname == "vendor/vendor.dovecot/pvt/server/vendor/deltachat/irohrelay"
-                and iroh_relay
-            ):
-                # Handle `GETMETADATA "" /shared/vendor/deltachat/irohrelay`
-                return f"O{iroh_relay}\n"
         logging.warning("lookup ignored: %r", msg)
         return "N\n"
     elif short_command == DICTPROXY_ITERATE_CHAR:
@@ -128,10 +120,7 @@ class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
 
 
 def main():
-    socket, vmail_dir, config_path = sys.argv[1:]
-
-    config = read_config(config_path)
-    iroh_relay = config.iroh_relay
+    socket, vmail_dir = sys.argv[1:]
 
     vmail_dir = Path(vmail_dir)
     if not vmail_dir.exists():
@@ -147,9 +136,7 @@ def main():
     class Handler(StreamRequestHandler):
         def handle(self):
             try:
-                handle_dovecot_protocol(
-                    self.rfile, self.wfile, notifier, metadata, iroh_relay
-                )
+                handle_dovecot_protocol(self.rfile, self.wfile, notifier, metadata)
             except Exception:
                 logging.exception("Exception in the dovecot dictproxy handler")
                 raise

chatmaild/src/chatmaild/metrics.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
-import sys
-import time
 from pathlib import Path
+import time
+import sys
 
 
 def main(vmail_dir=None):

chatmaild/src/chatmaild/newemail.py

@@ -1,13 +1,13 @@
 #!/usr/local/lib/chatmaild/venv/bin/python3
 
-"""CGI script for creating new accounts."""
+""" CGI script for creating new accounts. """
 
 import json
 import random
 import secrets
 import string
 
-from chatmaild.config import Config, read_config
+from chatmaild.config import read_config, Config
 
 CONFIG_PATH = "/usr/local/lib/chatmaild/chatmail.ini"
 ALPHANUMERIC = string.ascii_lowercase + string.digits

chatmaild/src/chatmaild/notifier.py

@@ -25,16 +25,15 @@ The meaning and format of tokens is basically a matter of Delta-Chat Core and
 the `notification.delta.chat` service.
 """
 
-import logging
-import math
 import os
 import time
-from dataclasses import dataclass
+import math
+import logging
+from uuid import uuid4
+from threading import Thread
 from pathlib import Path
 from queue import PriorityQueue
-from threading import Thread
-from uuid import uuid4
-
+from dataclasses import dataclass
 import requests
 
 

chatmaild/src/chatmaild/tests/mail-data/literal.eml

@@ -1,44 +0,0 @@
-From: {from_addr}
-
-To: {to_addr}
-
-Subject: ...
-
-Date: Sun, 15 Oct 2023 16:43:21 +0000
-
-Message-ID: <Mr.UVyJWZmkCKM.hGzNc6glBE_@c2.testrun.org>
-
-In-Reply-To: <Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>
-
-References: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>
-
-	<Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>
-
-Chat-Version: 1.0
-
-Autocrypt: addr={from_addr}; prefer-encrypt=mutual;
-
-	keydata=xjMEZSwWjhYJKwYBBAHaRw8BAQdAQBEhqeJh0GueHB6kF/DUQqYCxARNBVokg/AzT+7LqH
-
-	rNFzxiYXJiYXpAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUsFo4CGwMECwkIBwYVCAkKCwID
-
-	FgIBFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX9A4AEAnHWHp49eBCMHK5t66gYPiW
-
-	XQuB1mwUjzGfYWB+0RXUoA/0xcQ3FbUNlGKW7Blp6eMFfViv6Mv2d3kNSXACB6nmcMzjgEZSwWjhIK
-
-	KwYBBAGXVQEFAQEHQBpY5L2M1XHo0uxf8SX1wNLBp/OVvidoWHQF2Jz+kJsUAwEIB8J4BBgWCAAgBQ
-
-	JlLBaOAhsMFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX/INgEA37AJaNvruYsJVanP
-
-	IXnYw4CKd55UAwl8Zcy+M2diAbkA/0fHHcGV4r78hpbbL1Os52DPOdqYQRauIeJUeG+G6bQO
-
-MIME-Version: 1.0
-
-Content-Type: multipart/encrypted; protocol="application/pgp-encrypted";
-
-	boundary="YFrteb74qSXmggbOxZL9dRnhymywAi"
-
-
-
-
-

chatmaild/src/chatmaild/tests/plugin.py

@@ -1,14 +1,14 @@
+import random
+from pathlib import Path
+import os
 import importlib.resources
 import itertools
-import os
-import random
-from email import policy
 from email.parser import BytesParser
-from pathlib import Path
-
+from email import policy
 import pytest
-from chatmaild.config import read_config, write_initial_config
+
 from chatmaild.database import Database
+from chatmaild.config import read_config, write_initial_config
 
 
 @pytest.fixture
@@ -68,9 +68,7 @@ def maildata(request):
     assert datadir.exists(), datadir
 
     def maildata(name, from_addr, to_addr):
-        # Using `.read_bytes().decode()` instead of `.read_text()` to preserve newlines.
-        data = datadir.joinpath(name).read_bytes().decode()
-
+        data = datadir.joinpath(name).read_text()
         text = data.format(from_addr=from_addr, to_addr=to_addr)
         return BytesParser(policy=policy.default).parsebytes(text.encode())
 

chatmaild/src/chatmaild/tests/test_config.py

@@ -24,7 +24,7 @@ def test_read_config_testrun(make_config):
     assert config.postfix_reinject_port == 10025
     assert config.max_user_send_per_minute == 60
     assert config.max_mailbox_size == "100M"
-    assert config.delete_mails_after == "20"
+    assert config.delete_mails_after == "40"
     assert config.username_min_length == 9
     assert config.username_max_length == 9
     assert config.password_min_length == 9

chatmaild/src/chatmaild/tests/test_doveauth.py

@@ -1,20 +1,18 @@
 import io
 import json
+import pytest
 import queue
 import threading
 import traceback
 
 import chatmaild.doveauth
-import pytest
-from chatmaild.database import DBError
 from chatmaild.doveauth import (
     get_user_data,
-    handle_dovecot_protocol,
-    handle_dovecot_request,
-    is_allowed_to_create,
     lookup_passdb,
+    handle_dovecot_request,
+    handle_dovecot_protocol,
 )
-from chatmaild.newemail import create_newemail_dict
+from chatmaild.database import DBError
 
 
 def test_basic(db, example_config):
@@ -27,20 +25,6 @@ def test_basic(db, example_config):
     assert data == data2
 
 
-def test_invalid_username_length(example_config):
-    config = example_config
-    config.username_min_length = 6
-    config.username_max_length = 10
-    password = create_newemail_dict(config)["password"]
-    assert not is_allowed_to_create(config, f"a1234@{config.mail_domain}", password)
-    assert is_allowed_to_create(config, f"012345@{config.mail_domain}", password)
-    assert is_allowed_to_create(config, f"0123456@{config.mail_domain}", password)
-    assert is_allowed_to_create(config, f"0123456789@{config.mail_domain}", password)
-    assert not is_allowed_to_create(
-        config, f"0123456789x@{config.mail_domain}", password
-    )
-
-
 def test_dont_overwrite_password_on_wrong_login(db, example_config):
     """Test that logging in with a different password doesn't create a new user"""
     res = lookup_passdb(

chatmaild/src/chatmaild/tests/test_filtermail.py

@@ -1,11 +1,12 @@
-import pytest
 from chatmaild.filtermail import (
+    check_encrypted,
     BeforeQueueHandler,
     SendRateLimiter,
-    check_armored_payload,
-    check_encrypted,
+    check_mdn,
 )
 
+import pytest
+
 
 @pytest.fixture
 def maildomain():
@@ -62,19 +63,34 @@ def test_filtermail_encryption_detection(maildata):
     assert not check_encrypted(msg)
 
 
-def test_filtermail_no_literal_packets(maildata):
-    """Test that literal OpenPGP packet is not considered an encrypted mail."""
-    msg = maildata("literal.eml", from_addr="1@example.org", to_addr="2@example.org")
-    assert not check_encrypted(msg)
-
-
-def test_filtermail_unencrypted_mdn(maildata, gencreds):
-    """Unencrypted MDNs should not pass."""
+def test_filtermail_is_mdn(maildata, gencreds, handler):
     from_addr = gencreds()[0]
     to_addr = gencreds()[0] + ".other"
     msg = maildata("mdn.eml", from_addr, to_addr)
 
-    assert not check_encrypted(msg)
+    class env:
+        mail_from = from_addr
+        rcpt_tos = [to_addr]
+        content = msg.as_bytes()
+
+    assert check_mdn(msg, env)
+    print(msg.as_string())
+
+    assert not handler.check_DATA(env)
+
+
+def test_filtermail_to_multiple_recipients_no_mdn(maildata, gencreds):
+    from_addr = gencreds()[0]
+    to_addr = gencreds()[0] + ".other"
+    thirdaddr = gencreds()[0]
+    msg = maildata("mdn.eml", from_addr, to_addr)
+
+    class env:
+        mail_from = from_addr
+        rcpt_tos = [to_addr, thirdaddr]
+        content = msg.as_bytes()
+
+    assert not check_mdn(msg, env)
 
 
 def test_send_rate_limiter():
@@ -127,59 +143,3 @@ def test_passthrough_senders(gencreds, handler, maildata):
 
     # assert that None/no error is returned
     assert not handler.check_DATA(envelope=env)
-
-
-def test_check_armored_payload():
-    payload = """-----BEGIN PGP MESSAGE-----\r
-\r
-wU4DSqFx0d1yqAoSAQdAYkX/ZN/Az4B0k7X47zKyWrXxlDEdS3WOy0Yf2+GJTFgg\r
-Zk5ql0mLG8Ze+ZifCS0XMO4otlemSyJ0K1ZPdFMGzUDBTgNqzkFabxXoXRIBB0AM\r
-755wlX41X6Ay3KhnwBq7yEqSykVH6F3x11iHPKraLCAGZoaS8bKKNy/zg5slda1X\r
-pt14b4aC1VwtSnYhcRRELNLD/wE2TFif+g7poMmFY50VyMPLYjVP96Z5QCT4+z4H\r
-Ikh/pRRN8S3JNMrRJHc6prooSJmLcx47Y5un7VFy390MsJ+LiUJuQMDdYWRAinfs\r
-Ebm89Ezjm7F03qbFPXE0X4ZNzVXS/eKO0uhJQdiov/vmbn41rNtHmNpqjaO0vi5+\r
-sS9tR7yDUrIXiCUCN78eBLVioxtktsPZm5cDORbQWzv+7nmCEz9/JowCUcBVdCGn\r
-1ofOaH82JCAX/cRx08pLaDNj6iolVBsi56Dd+2bGxJOZOG2AMcEyz0pXY0dOAJCD\r
-iUThcQeGIdRnU3j8UBcnIEsjLu2+C+rrwMZQESMWKnJ0rnqTk0pK5kXScr6F/L0L\r
-UE49ccIexNm3xZvYr5drszr6wz3Tv5fdue87P4etBt90gF/Vzknck+g1LLlkzZkp\r
-d8dI0k2tOSPjUbDPnSy1x+X73WGpPZmj0kWT+RGvq0nH6UkJj3AQTG2qf1T8jK+3\r
-rTp3LR9vDkMwDjX4R8SA9c0wdnUzzr79OYQC9lTnzcx+fM6BBmgQ2GrS33jaFLp7\r
-L6/DFpCl5zhnPjM/2dKvMkw/Kd6XS/vjwsO405FQdjSDiQEEAZA+ZvAfcjdccbbU\r
-yCO+x0QNdeBsufDVnh3xvzuWy4CICdTQT4s1AWRPCzjOj+SGmx5WqCLWfsd8Ma0+\r
-w/C7SfTYu1FDQILLM+llpq1M/9GPley4QZ8JQjo262AyPXsPF/OW48uuZz0Db1xT\r
-Yh4iHBztj4VSdy7l2+IyaIf7cnL4EEBFxv/MwmVDXvDlxyvfAfIsd3D9SvJESzKZ\r
-VWDYwaocgeCN+ojKu1p885lu1EfRbX3fr3YO02K5/c2JYDkc0Py0W3wUP/J1XUax\r
-pbKpzwlkxEgtmzsGqsOfMJqBV3TNDrOA2uBsa+uBqP5MGYLZ49S/4v/bW9I01Cr1\r
-D2ZkV510Y1Vgo66WlP8mRqOTyt/5WRhPD+MxXdk67BNN/PmO6tMlVoJDuk+XwWPR\r
-t2TvNaND/yabT9eYI55Og4fzKD6RIjouUX8DvKLkm+7aXxVs2uuLQ3Jco3O82z55\r
-dbShU1jYsrw9oouXUz06MHPbkdhNbF/2hfhZ2qA31sNeovJw65iUv7sDKX3LVWgJ\r
-10jlywcDwqlU8CO7WC9lGixYTbnOkYZpXCGEl8e6Jbs79l42YFo4ogYpFK1NXFhV\r
-kOXRmDf/wmfj+c/ld3L2PkvwlgofhCudOQknZbo3ub1gjiTn7L+lMGHIj/3suMIl\r
-ID4EUxAXScIM1ZEz2fjtW5jATlqYcLjLTbf/olw6HFyPNH+9IssqXeZNKnGwPUB9\r
-3lTXsg0tpzl+x7F/2WjEw1DSNhjC0KnHt1vEYNMkUGDGFdN9y3ERLqX/FIgiASUb\r
-bTvAVupnAK3raBezGmhrs6LsQtLS9P0VvQiLU3uDhMqw8Z4SISLpcD+NnVBHzQqm\r
-6W5Qn/8xsCL6av18yUVTi2G3igt3QCNoYx9evt2ZcIkNoyyagUVjfZe5GHXh8Dnz\r
-GaBXW/hg3HlXLRGaQu4RYCzBMJILcO25OhZOg6jbkCLiEexQlm2e9krB5cXR49Al\r
-UN4fiB0KR9JyG2ayUdNJVkXZSZLnHyRgiaadlpUo16LVvw==\r
-=b5Kp\r
------END PGP MESSAGE-----\r
-\r
-"""
-
-    assert check_armored_payload(payload) == True
-
-    payload = """-----BEGIN PGP MESSAGE-----\r
-\r
-HELLOWORLD
------END PGP MESSAGE-----\r
-\r
-"""
-    assert check_armored_payload(payload) == False
-
-    payload = """-----BEGIN PGP MESSAGE-----\r
-\r
-=njUN
------END PGP MESSAGE-----\r
-\r
-"""
-    assert check_armored_payload(payload) == False

chatmaild/src/chatmaild/tests/test_metadata.py

@@ -1,12 +1,12 @@
 import io
-import time
-
 import pytest
 import requests
+import time
+
 from chatmaild.metadata import (
-    Metadata,
-    handle_dovecot_protocol,
     handle_dovecot_request,
+    handle_dovecot_protocol,
+    Metadata,
 )
 from chatmaild.notifier import (
     Notifier,
@@ -296,17 +296,3 @@ def test_persistent_queue_items(tmp_path, testaddr, token):
     item2.delete()
     assert not item2.path.exists()
     assert not queue_item < item2 and not item2 < queue_item
-
-
-def test_iroh_relay(metadata):
-    rfile = io.BytesIO(
-        b"\n".join(
-            [
-                b"H",
-                b"Lshared/0123/vendor/vendor.dovecot/pvt/server/vendor/deltachat/irohrelay\tuser@example.org",
-            ]
-        )
-    )
-    wfile = io.BytesIO()
-    handle_dovecot_protocol(rfile, wfile, notifier, metadata, "https://example.org/")
-    assert wfile.getvalue() == b"Ohttps://example.org/\n"

cmdeploy/pyproject.toml

@@ -16,6 +16,7 @@ dependencies = [
   "build",
   "tox",
   "ruff",
+  "black",
   "pytest",
   "pytest-xdist", 
   "imap_tools",
@@ -30,13 +31,3 @@ cmdeploy = "cmdeploy.cmdeploy:main"
 
 [tool.pytest.ini_options]
 addopts = "-v -ra --strict-markers"
-
-[tool.ruff]
-lint.select = [
-  "F", # Pyflakes
-  "I", # isort
-
-  "PLC", # Pylint Convention
-  "PLE", # Pylint Error
-  "PLW", # Pylint Warning
-]

cmdeploy/src/cmdeploy/__init__.py

@@ -2,21 +2,21 @@
 Chat Mail pyinfra deploy.
 """
 
-import importlib.resources
-import io
-import shutil
-import subprocess
 import sys
+import importlib.resources
+import subprocess
+import shutil
+import io
 from pathlib import Path
 
-from chatmaild.config import Config, read_config
 from pyinfra import host
+from pyinfra.operations import apt, files, server, systemd, pip
 from pyinfra.facts.files import File
 from pyinfra.facts.systemd import SystemdEnabled
-from pyinfra.operations import apt, files, pip, server, systemd
-
 from .acmetool import deploy_acmetool
 
+from chatmaild.config import read_config, Config
+
 
 def _build_chatmaild(dist_dir) -> None:
     dist_dir = Path(dist_dir).resolve()
@@ -361,14 +361,6 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
         config=config,
     )
 
-    files.put(
-            src=importlib.resources.files(__package__).joinpath("dovecot/remove-seen.py"),
-            dest="/usr/local/bin/remove-seen.py",
-            user="root",
-            group="root",
-            mode="755"
-    )
-
     # as per https://doc.dovecot.org/configuration_manual/os/
     # it is recommended to set the following inotify limits
     for name in ("max_user_instances", "max_user_watches"):
@@ -494,20 +486,19 @@ def deploy_chatmail(config_path: Path) -> None:
 
     # Add our OBS repository for dovecot_no_delay
     files.put(
-        name="Add Deltachat OBS GPG key to apt keyring",
-        src=importlib.resources.files(__package__).joinpath("obs-home-deltachat.gpg"),
-        dest="/etc/apt/keyrings/obs-home-deltachat.gpg",
+        name = "Add Deltachat OBS GPG key to apt keyring",
+        src = importlib.resources.files(__package__).joinpath("obs-home-deltachat.gpg"),
+        dest = "/etc/apt/keyrings/obs-home-deltachat.gpg",
         user="root",
         group="root",
         mode="644",
     )
 
     files.line(
-        name="Add DeltaChat OBS home repository to sources.list",
-        path="/etc/apt/sources.list",
-        line="deb [signed-by=/etc/apt/keyrings/obs-home-deltachat.gpg] https://download.opensuse.org/repositories/home:/deltachat/Debian_12/ ./",
-        escape_regex_characters=True,
-        ensure_newline=True,
+        name = "Add DeltaChat OBS home repository to sources.list",
+        path = "/etc/apt/sources.list",
+        line = "deb [signed-by=/etc/apt/keyrings/obs-home-deltachat.gpg] https://download.opensuse.org/repositories/home:/deltachat/Debian_12/ ./",
+        ensure_newline = True,
     )
 
     apt.update(name="apt update", cache_time=24 * 3600)
@@ -543,12 +534,6 @@ def deploy_chatmail(config_path: Path) -> None:
         domains=[mail_domain, f"mta-sts.{mail_domain}", f"www.{mail_domain}"],
     )
 
-    apt.packages(
-        # required for setfacl for echobot
-        name="Install acl",
-        packages="acl",
-    )
-
     apt.packages(
         name="Install Postfix",
         packages="postfix",
@@ -650,12 +635,5 @@ def deploy_chatmail(config_path: Path) -> None:
         service="systemd-journald.service",
         running=True,
         enabled=True,
-        restarted=journald_conf.changed,
+        restarted=journald_conf,
     )
-
-    apt.packages(
-        name="Ensure cron is installed",
-        packages=["cron"],
-    )
-
-

cmdeploy/src/cmdeploy/acmetool/__init__.py

@@ -1,8 +1,8 @@
 import importlib.resources
 
+from pyinfra.operations import apt, files, systemd, server
 from pyinfra import host
 from pyinfra.facts.systemd import SystemdStatus
-from pyinfra.operations import apt, files, server, systemd
 
 
 def deploy_acmetool(email="", domains=[]):

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -4,18 +4,19 @@ along with command line option and subcommand parsing.
 """
 
 import argparse
+import shutil
+import subprocess
 import importlib.resources
 import importlib.util
 import os
-import shutil
-import subprocess
 import sys
 from pathlib import Path
 
-from chatmaild.config import read_config, write_initial_config
-from termcolor import colored
 
-from cmdeploy.dns import check_necessary_dns, show_dns
+from termcolor import colored
+from chatmaild.config import read_config, write_initial_config
+from cmdeploy.dns import show_dns, check_necessary_dns
+
 
 #
 # cmdeploy sub commands and options
@@ -38,6 +39,10 @@ def init_cmd(args, out):
     else:
         write_initial_config(args.inipath, mail_domain)
         out.green(f"created config file for {mail_domain} in {args.inipath}")
+    check_necessary_dns(
+        out,
+        mail_domain,
+    )
 
 
 def run_cmd_options(parser):
@@ -152,26 +157,26 @@ def fmt_cmd_options(parser):
 
 
 def fmt_cmd(args, out):
-    """Run formattting fixes on all chatmail source code."""
+    """Run formattting fixes (ruff and black) on all chatmail source code."""
 
     sources = [str(importlib.resources.files(x)) for x in ("chatmaild", "cmdeploy")]
-    format_args = [shutil.which("ruff"), "format"]
-    check_args = [shutil.which("ruff"), "check"]
+    black_args = [shutil.which("black")]
+    ruff_args = [shutil.which("ruff")]
 
     if args.check:
-        format_args.append("--diff")
+        black_args.append("--check")
     else:
-        check_args.append("--fix")
+        ruff_args.append("--fix")
 
     if not args.verbose:
-        check_args.append("--quiet")
-        format_args.append("--quiet")
+        black_args.append("-q")
+        ruff_args.append("-q")
 
-    format_args.extend(sources)
-    check_args.extend(sources)
+    black_args.extend(sources)
+    ruff_args.extend(sources)
 
-    out.check_call(" ".join(format_args), quiet=not args.verbose)
-    out.check_call(" ".join(check_args), quiet=not args.verbose)
+    out.check_call(" ".join(black_args), quiet=not args.verbose)
+    out.check_call(" ".join(ruff_args), quiet=not args.verbose)
     return 0
 
 
@@ -227,7 +232,7 @@ class Out:
         if not quiet:
             cmdstring = " ".join(args)
             self(f"[$ {cmdstring}]", file=sys.stderr)
-        proc = subprocess.run(args, env=env, check=False)
+        proc = subprocess.run(args, env=env)
         return proc.returncode
 
 

cmdeploy/src/cmdeploy/deploy.py

@@ -1,8 +1,6 @@
-import importlib.resources
 import os
-
+import importlib.resources
 import pyinfra
-
 from cmdeploy import deploy_chatmail
 
 

cmdeploy/src/cmdeploy/dns.py

@@ -1,9 +1,9 @@
-import datetime
-import importlib
-import subprocess
 import sys
 
 import requests
+import importlib
+import subprocess
+import datetime
 
 
 class DNS:
@@ -104,8 +104,8 @@ def show_dns(args, out) -> int:
             return 0
         except TypeError:
             pass
-        for raw_line in zonefile.splitlines():
-            line = raw_line.format(
+        for line in zonefile.splitlines():
+            line = line.format(
                 acme_account_url=acme_account_url,
                 sts_id=datetime.datetime.now().strftime("%Y%m%d%H%M"),
                 chatmail_domain=args.config.mail_domain,
@@ -183,7 +183,6 @@ def show_dns(args, out) -> int:
 
 def check_necessary_dns(out, mail_domain):
     """Check whether $mail_domain and mta-sts.$mail_domain resolve."""
-    print("Checking necessary DNS records... ")
     dns = DNS(out, mail_domain)
     ipv4 = dns.get("A", mail_domain)
     ipv6 = dns.get("AAAA", mail_domain)
@@ -205,5 +204,5 @@ def check_necessary_dns(out, mail_domain):
             print(line)
         print()
     else:
-        dns.out.green("All necessary DNS records seem to be set.")
+        dns.out.green("\nAll necessary DNS entries seem to be set.")
         return True

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -19,22 +19,6 @@ mail_debug = yes
 #   master: Warning: service(stats): client_limit (1000) reached, client connections are being dropped
 default_client_limit = 20000
 
-# Increase number of logged in IMAP connections.
-# Each connection is handled by a separate `imap` process.
-# `imap` process should have `client_limit=1` as described in
-# <https://doc.dovecot.org/configuration_manual/service_configuration/#service-limits>
-# so each logged in IMAP session will need its own `imap` process.
-#
-# If this limit is reached,
-# users will fail to LOGIN as `imap-login` process
-# will accept them logging in but fail to transfer logged in
-# connection to `imap` process until someone logs out and
-# the following warning will be logged:
-#   Warning: service(imap): process_limit (1024) reached, client connections are being dropped
-service imap {
-  process_limit = 50000
-}
-
 mail_server_admin = mailto:root@{{ config.mail_domain }}
 mail_server_comment = Chatmail server
 

cmdeploy/src/cmdeploy/dovecot/expunge.cron.j2

@@ -9,4 +9,3 @@
 2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 3 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -name 'maildirsize' -type f -delete
-4 0 * * * vmail /usr/local/bin/remove-seen.py /home/vmail/mail/{{ config.mail_domain }}

cmdeploy/src/cmdeploy/dovecot/remove-seen.py

@@ -1,41 +0,0 @@
-#!/usr/bin/env python3
-"""Remove seen messages that are older than two days
-if maildir has more than 80 MB of messages."""
-import sys
-import time
-from pathlib import Path
-
-
-def getdirsize(path):
-    return sum(f.stat().st_size for f in path.glob("**/*") if f.is_file())
-
-
-def parse_dovecot_seen(path):
-    return "S" in path.name.split(":2,")[-1]
-
-
-def main():
-    now = time.time()
-
-    mailhome = Path(sys.argv[1])
-
-    for p in mailhome.iterdir():
-        dirsize = getdirsize(p / "cur") + getdirsize(p / "new")
-        if dirsize < 80000000:
-            continue
-
-        removed_bytes = 0
-        for mailpath in (p / "cur").iterdir():
-            seen = parse_dovecot_seen(mailpath)
-            stat = mailpath.stat()
-            size = stat.st_size
-            if seen and now > stat.st_mtime + 2 * 24 * 3600:
-                removed_bytes += size
-                mailpath.unlink(missing_ok=True)
-
-        if removed_bytes > 0:
-            (p / "maildirsize").unlink(missing_ok=True)
-
-
-if __name__ == "__main__":
-    main()

cmdeploy/src/cmdeploy/genqr.py

@@ -1,9 +1,8 @@
 import importlib
-import io
-import os
-
 import qrcode
-from PIL import Image, ImageDraw, ImageFont
+import os
+from PIL import ImageFont, ImageDraw, Image
+import io
 
 
 def gen_qr_png_data(maildomain):

cmdeploy/src/cmdeploy/journald.conf

@@ -1,3 +1,2 @@
 [Journal]
 MaxRetentionSec=3d
-Storage=volatile

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -1,7 +1,7 @@
 user www-data;
 worker_processes auto;
 pid /run/nginx.pid;
-error_log syslog:server=unix:/dev/log,facility=local3;
+error_log /var/log/nginx/error.log;
 
 events {
 	worker_connections 768;
@@ -35,8 +35,6 @@ http {
 
 		server_name _;
 
-		access_log syslog:server=unix:/dev/log,facility=local7;
-
 		location / {
 			# First attempt to serve request as file, then
 			# as directory, then fall back to displaying a 404.
@@ -82,6 +80,5 @@ http {
 		listen [::]:443 ssl;
 		server_name www.{{ config.domain_name }};
 		return 301 $scheme://{{ config.domain_name }}$request_uri;
-		access_log syslog:server=unix:/dev/log,facility=local7;
 	}
 }

cmdeploy/src/cmdeploy/service/chatmail-metadata.service.f

@@ -2,7 +2,7 @@
 Description=Chatmail dict proxy for IMAP METADATA
 
 [Service]
-ExecStart={execpath} /run/chatmail-metadata/metadata.socket /home/vmail/mail/{mail_domain} {config_path}
+ExecStart={execpath} /run/chatmail-metadata/metadata.socket /home/vmail/mail/{mail_domain}
 Restart=always
 RestartSec=30
 User=vmail

cmdeploy/src/cmdeploy/tests/online/test_0_login.py

@@ -1,10 +1,9 @@
+import pytest
+import threading
 import queue
 import socket
-import threading
 
-import pytest
 from chatmaild.config import read_config
-
 from cmdeploy.cmdeploy import main
 
 

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -1,5 +1,4 @@
 import smtplib
-
 import pytest
 
 
@@ -86,7 +85,6 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
     pytest.fail("Rate limit was not exceeded")
 
 
-@pytest.mark.slow
 def test_expunged(remote, chatmail_config):
     outdated_days = int(chatmail_config.delete_mails_after) + 1
     find_cmds = [

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -1,11 +1,11 @@
-import ipaddress
-import random
-import re
 import time
+import re
+import random
 
-import imap_tools
 import pytest
 import requests
+import ipaddress
+import imap_tools
 
 
 @pytest.fixture

cmdeploy/src/cmdeploy/tests/plugin.py

@@ -1,16 +1,17 @@
-import imaplib
-import io
-import itertools
 import os
-import random
-import smtplib
-import subprocess
+import io
 import time
+import random
+import subprocess
+import imaplib
+import smtplib
+import itertools
 from pathlib import Path
-
 import pytest
-from chatmaild.config import read_config
+
 from chatmaild.database import Database
+from chatmaild.config import read_config
+
 
 conftestdir = Path(__file__).parent
 

cmdeploy/src/cmdeploy/tests/test_cmdeploy.py

@@ -1,7 +1,6 @@
 import os
 
 import pytest
-
 from cmdeploy.cmdeploy import get_parser, main
 
 

cmdeploy/src/cmdeploy/www.py

@@ -1,14 +1,13 @@
-import hashlib
 import importlib.resources
+import webbrowser
+import hashlib
 import time
 import traceback
-import webbrowser
 
 import markdown
-from chatmaild.config import read_config
 from jinja2 import Template
-
 from .genqr import gen_qr_png_data
+from chatmaild.config import read_config
 
 
 def snapshot_dir_stats(somedir):
@@ -121,8 +120,7 @@ def main():
     print(f"watching {src_path} directory for changes")
 
     changenum = 0
-    count = 0
-    while True:
+    for count in range(0, 1000000):
         newstats = snapshot_dir_stats(src_path)
         if newstats == stats and count % 60 != 0:
             count += 1

scripts/cmdeploy

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 #
 # Wrapper for cmdelpoy to run it in activated virtualenv.
 set -e

scripts/dovecot/README.md

@@ -1,80 +0,0 @@
-## Introduction to custom Dovecot builds
-
-Chatmail servers use a custom Debian build of the IMAP 'dovecot' server software because 
-
-a) Dovecot developers did not yet merge a [pull request](https://github.com/dovecot/core/pull/216)
-   which majorly speeds up message delivery by removing a hardcoded 0.5 second delay 
-   on relaying incoming messages.  
-
-b) Even if merged, it would take years for it to reach Debian stable.
-
-c) The modified dovecot has been successfully used since December 2023 without issues
-   and we see no noticeable downside (theoretically higher CPU usage but not measureable)
-   but a considerable upside as the delay-removal facilitates end-to-end message 
-   delivery of 200 ms in real networks. 
-
-The modified forked dovecot code lives at 
-[https://github.com/chatmail/dovecot](https://github.com/chatmail/dovecot).  
-The remainder of this document describes the setup of the Debian repository 
-containing the patched dovecot version. 
-
-## Building Debian packages at build.opensuse.org 
-
-Delta Chat developers maintain an [account](https://build.opensuse.org/project/show/home:deltachat) 
-in the [Open Build Service (OBS)](https://openbuildservice.org/), 
-where the [resulting package](https://build.opensuse.org/package/show/home:deltachat/dovecot) 
-is now used in deploying chatmail servers. 
-   
-The Open Build Service (OBS) is a platform for building and distributing software packages 
-across various operating systems and architectures. 
-It supports openSUSE, Fedora, Debian, Ubuntu and Arch.
-It's [primary instance](https://build.opensuse.org/) is ran by the openSUSE project 
-and is part of the pipeline of the creation of SUSE Linux Enterprise.
-
-The OBS provides a mercurial-like interface to create source repositories 
-that are then automatically built. 
-While in theory a package can be created entirely over the web interface, 
-the use of the cli-tool `osc` is more convenient and is described in the [official documentation](https://openbuildservice.org/help/manuals/obs-user-guide/art.obs.bg#sec.obsbg.obsconfig).
-
-### How to build the dovecot debian package on the OBS via our script
-
-In scripts/dovecot/ is a shell script that prepares the required files and pushes them to build.opensuse.org.
-
-Before using the script, you should have osc set up as described in the [official documentation](https://openbuildservice.org/help/manuals/obs-user-guide/art.obs.bg#sec.obsbg.obsconfig).
-
-The script assumes you are on Debian. It automatically installs any needed dependencies and creates the source package. To upload the resulting source package to the OBS you need to enter the username and password for deltachat on build.opensuse.org in the last step of the script.
-
-Use `source build-obs.sh` to run it.
-
-### Adding the resulting OBS repository to Debian 12
-
-Our dovecot fork is automatically installed as part of the chatmail deployment. You can see it in cmdeploy/src/cmdeploy/__init__.py. If you want to add our fork manually to a system, you can do the following:
-
-First add our signing key to your apt keyring:
-
-```
-sudo cp cmdeploy/src/cmdeploy/obs-home-deltachat.gpg /etc/apt/keyrings/obs-home-deltachat.gpg`
-```
-
-Now add our repository and key to /etc/apt/sources.list with a text editor of your choice:
-
-```
-deb [signed-by=/etc/apt/keyrings/obs-home-deltachat.gpg] https://download.opensuse.org/repositories/home:/deltachat/Debian_12/ ./
-```
-
-You can now install dovecot like normal.
-
-```
-sudo apt update
-sudo apt install dovecot-core
-```
-
-### Security concerns 
-
-The signing of the patched dovecot package is done in the OBS and 
-in theory SUSE could make changes to the package delivered.
-It is probably reasonable to trust SUSE to not mess with the build
-process because it would cause serious negative reputation damage for them 
-if they tried and someone finds out. 
-
-Our dovecot fork will receive the same security backports as the dovecot package in Debian Sid.

scripts/dovecot/build-obs.sh

@@ -1,54 +0,0 @@
-#!/bin/sh
-
-# Install dependencies
-echo "Installing dependencies for this script:"
-sudo apt install -y devscripts build-essential osc curl git debhelper-compat
-
-# Define path of your local OBS repository
-SCRIPT_DIR=$PWD
-OBS_PATH=$SCRIPT_DIR/obs
-REPO_PATH=$OBS_PATH/home:deltachat/dovecot/
-
-# Download Debian Source Files
-echo "Downloading precise files from Debian unstable repository..."
-mkdir dovecot-build
-cd dovecot-build
-
-# taken May 6th 2024, from https://packages.debian.org/unstable/dovecot-core 
-curl http://deb.debian.org/debian/pool/main/d/dovecot/dovecot_2.3.21+dfsg1-3.debian.tar.xz -O
-curl http://deb.debian.org/debian/pool/main/d/dovecot/dovecot_2.3.21+dfsg1.orig.tar.gz -O
-curl http://deb.debian.org/debian/pool/main/d/dovecot/dovecot_2.3.21+dfsg1.orig-pigeonhole.tar.gz -O
-
-# Clone the Chatmail Dovecot Repo
-echo "Cloning the Chatmail Dovecot fork..."
-git clone https://github.com/chatmail/dovecot.git
-
-# Build the source package
-echo "Building the source package"
-cd dovecot
-dpkg-source -b .
-
-# Setting up OSC
-echo "Setting up OBS home repository"
-mkdir $OBS_PATH
-cd $OBS_PATH
-rm -rf home:deltachat/dovecot
-osc checkout home:deltachat/dovecot
-
-# Copy Files to Your Local OBS Repository,
-echo "Copying files to your local OBS repository..."
-cd $SCRIPT_DIR/dovecot-build
-cp -rf dovecot_2.3.21+dfsg1-3.debian.tar.xz $REPO_PATH
-cp -rf dovecot_2.3.21+dfsg1.orig.tar.gz $REPO_PATH
-cp -rf dovecot_2.3.21+dfsg1.orig-pigeonhole.tar.gz $REPO_PATH
-cp -rf dovecot_2.3.21+dfsg1-3.dsc $REPO_PATH
-
-# Push Changes to OBS
-echo "Pushing changes to OBS..."
-cd $REPO_PATH
-osc up
-osc add dovecot_2.3.21+dfsg1-3.debian.tar.xz
-osc add dovecot_2.3.21+dfsg1.orig.tar.gz
-osc add dovecot_2.3.21+dfsg1.orig-pigeonhole.tar.gz
-osc add dovecot_2.3.21+dfsg1-3.dsc
-osc commit

scripts/initenv.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 set -e
 python3 -m venv --upgrade-deps venv
 

www/src/index.md

@@ -1,17 +1,11 @@
 
 <img class="banner" src="collage-top.png"/>
 
-## Dear [Delta Chat](https://get.delta.chat) users and newcomers ... 
+## Dear [Delta Chat](https://get.delta.chat) users and newcomers, 
 
-{% if config.mail_domain != "nine.testrun.org" %}
 Welcome to instant, interoperable and [privacy-preserving](privacy.html) messaging :) 
-{% else %}
-Welcome to the default onboarding server ({{ config.mail_domain }}) 
-for Delta Chat users.  For details how it avoids storing personal information
-please see our [privacy policy](privacy.html). 
-{% endif %}
 
-👉 **Tap** or scan this QR code to get a `@{{config.mail_domain}}` chat profile
+👉 **Tap** or scan this QR code to get a random `@{{config.mail_domain}}` e-mail address
 
 <a href="DCACCOUNT:https://{{ config.mail_domain }}/new">
     <img width=300 style="float: none;" src="qr-chatmail-invite-{{config.mail_domain}}.png" /></a>

www/src/info.md

@@ -1,4 +1,6 @@
 
+<img class="banner" src="collage-info.png"/>
+
 ## More information 
 
 {{ config.mail_domain }} provides a low-maintenance, resource efficient and 
@@ -9,7 +11,7 @@ for the usage in chats, especially DeltaChat.
 ### Choosing a chatmail address instead of using a random one
 
 In the Delta Chat account setup 
-you may tap `I already have a profile`
+you may tap `LOG INTO YOUR E-MAIL ACCOUNT` 
 and fill the two fields like this: 
 
 - `Address`: invent a word with

www/src/privacy.md

@@ -1,41 +1,21 @@
+<img class="banner" src="collage-privacy.png"/>
 
 # Privacy Policy for {{ config.mail_domain }} 
 
-{% if config.mail_domain == "nine.testrun.org" %}
-Welcome to `{{config.mail_domain}}`, the default chatmail onboarding server for Delta Chat users. 
-It is operated on the side by a small sysops team employed by [merlinux](https://merlinux.eu),
-an open-source R&D company also acting as the fiscal sponsor of Delta Chat app developments. 
-See [other chatmail servers](https://delta.chat/en/chatmail) for alternative server operators. 
-{% endif %}
-
-
-## Summary: No personal data asked or collected 
-
-This chatmail server neither asks for nor retains personal information. 
-Chatmail servers exist to reliably transmit (store and deliver) end-to-end encrypted messages
-between user's devices running the Delta Chat messenger app. 
-Technically, you may think of a Chatmail server as 
-an end-to-end encrypted "messaging router" at Internet-scale. 
-
-A chatmail server is very unlike classic e-mail servers (for example Google Mail servers)
-that ask for personal data and permanently store messages. 
-A chatmail server behaves more like the Signal messaging server 
-but does not know about phone numbers and securely and automatically interoperates 
-with other chatmail and classic e-mail servers. 
-
-In particular, this chatmail server 
-
-- unconditionally removes messages after {{ config.delete_mails_after }} days,
-
-- prohibits sending out un-encrypted messages,
-
-- only has temporary log files used for debugging purposes.
-
-Legally, authorities might still regard chatmail as a "classic e-mail" server
-which collects and retains personal data. 
-We do not agree on this interpretation. Nevertheless, we provide more legal details below
-to make life easier for data protection specialists and lawyers scrutinizing chatmail operations. 
+We want to show you in a fair and transparent way
+what personal data is processed by us.
+We follow a strict privacy-by-design approach
+and try to avoid processing your data in the first place,
+but as you may know,
+the internet, 
+and in particular sending e-mail messages,
+does not work without data.
+Still,
+it's only fair that you know at all times
+what personal data is processed
+when you use our service.
 
+If you have any remaining questions about data protection, please contact us. 
 
 ## 1. Name and contact information 
 
@@ -77,7 +57,7 @@ we process the following data and details:
 - Users can retrieve or delete all stored messages 
   without intervention from the operators using standard IMAP client tools.
 
-### 2.1 Account setup
+### 3.1 Account setup
 
 Creating an account happens in one of two ways on our mail servers: 
 
@@ -98,7 +78,7 @@ Art. 6 (1) lit. b GDPR,
 as you have a usage contract with us
 by using our services.
 
-### 2.2 Processing of E-Mail-Messages
+## 3.2 Processing of E-Mail-Messages
 
 In addition,
 we will process data
@@ -124,7 +104,7 @@ Therefore, limits are enforced:
 
 - message size limits
 
-- any other limit necessary for the whole server to function in a healthy way
+- any other limit neccessary for the whole server to function in a healthy way
   and to prevent abuse.
 
 The processing and use of the above permissions
@@ -198,9 +178,8 @@ for the purpose of drawing conclusions about your person.
 
 ## 4. Transfer of Data
 
-We do not retain any personal data but e-mail messages waiting to be delivered 
-may contain personal data.
-Any such residual personal data will not be transferred to third parties
+Your personal data
+will not be transferred to third parties
 for purposes other than those listed below:
 
 a) you have given your express consent