nginx: multiplex SSH over port 443 in case port 22 is blocked

.github/workflows/ci.yaml

@@ -14,8 +14,7 @@ jobs:
         # Otherwise `test_deployed_state` will be unhappy.
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-      - name: download filtermail
+
-        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.2.0/filtermail-x86_64-musl -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
       - name: run chatmaild tests 
         working-directory: chatmaild
         run: pipx run tox

.github/workflows/test-and-deploy-ipv4only.yaml

@@ -74,7 +74,6 @@ jobs:
       - run: |
           cmdeploy init staging-ipv4.testrun.org
           sed -i 's#disable_ipv6 = False#disable_ipv6 = True#' chatmail.ini
-          sed -i 's/#\s*mtail_address/mtail_address/' chatmail.ini
 
       - run: cmdeploy run --verbose --skip-dns-check
 

.github/workflows/test-and-deploy.yaml

@@ -74,9 +74,7 @@ jobs:
       - name: run deploy-chatmail offline tests 
         run: pytest --pyargs cmdeploy 
 
-      - run: |
+      - run: cmdeploy init staging2.testrun.org
-          cmdeploy init staging2.testrun.org
-          sed -i 's/#\s*mtail_address/mtail_address/' chatmail.ini
 
       - run: cmdeploy run --verbose --skip-dns-check
 

chatmaild/pyproject.toml

@@ -24,6 +24,7 @@ where = ['src']
 [project.scripts]
 doveauth = "chatmaild.doveauth:main"
 chatmail-metadata = "chatmaild.metadata:main"
+filtermail = "chatmaild.filtermail:main"
 chatmail-metrics = "chatmaild.metrics:main"
 chatmail-expire = "chatmaild.expire:main"
 chatmail-fsreport = "chatmaild.fsreport:main"

chatmaild/src/chatmaild/config.py

@@ -20,8 +20,7 @@ class Config:
     def __init__(self, inipath, params):
         self._inipath = inipath
         self.mail_domain = params["mail_domain"]
-        self.max_user_send_per_minute = int(params.get("max_user_send_per_minute", 60))
+        self.max_user_send_per_minute = int(params["max_user_send_per_minute"])
-        self.max_user_send_burst_size = int(params.get("max_user_send_burst_size", 10))
         self.max_mailbox_size = params["max_mailbox_size"]
         self.max_message_size = int(params.get("max_message_size", "31457280"))
         self.delete_mails_after = params["delete_mails_after"]
@@ -33,13 +32,13 @@ class Config:
         self.passthrough_senders = params["passthrough_senders"].split()
         self.passthrough_recipients = params["passthrough_recipients"].split()
         self.www_folder = params.get("www_folder", "")
-        self.filtermail_smtp_port = int(params.get("filtermail_smtp_port", "10080"))
+        self.filtermail_smtp_port = int(params["filtermail_smtp_port"])
         self.filtermail_smtp_port_incoming = int(
-            params.get("filtermail_smtp_port_incoming", "10081")
+            params["filtermail_smtp_port_incoming"]
         )
-        self.postfix_reinject_port = int(params.get("postfix_reinject_port", "10025"))
+        self.postfix_reinject_port = int(params["postfix_reinject_port"])
         self.postfix_reinject_port_incoming = int(
-            params.get("postfix_reinject_port_incoming", "10026")
+            params["postfix_reinject_port_incoming"]
         )
         self.mtail_address = params.get("mtail_address")
         self.disable_ipv6 = params.get("disable_ipv6", "false").lower() == "true"

chatmaild/src/chatmaild/dictproxy.py

@@ -22,7 +22,7 @@ class DictProxy:
                 wfile.flush()
 
     def handle_dovecot_request(self, msg, transactions):
-        # see https://doc.dovecot.org/2.3/developer_manual/design/dict_protocol/#dovecot-dict-protocol
+        # see https://doc.dovecot.org/developer_manual/design/dict_protocol/#dovecot-dict-protocol
         short_command = msg[0]
         parts = msg[1:].split("\t")
 

chatmaild/src/chatmaild/doveauth.py

@@ -1,11 +1,8 @@
 import json
 import logging
 import os
-import re
 import sys
 
-import filelock
-
 try:
     import crypt_r
 except ImportError:
@@ -16,11 +13,10 @@ from .dictproxy import DictProxy
 from .migrate_db import migrate_from_db_to_maildir
 
 NOCREATE_FILE = "/etc/chatmail-nocreate"
-VALID_LOCALPART_RE = re.compile(r"^[a-z0-9._-]+$")
 
 
 def encrypt_password(password: str):
-    # https://doc.dovecot.org/2.3/configuration_manual/authentication/password_schemes/
+    # https://doc.dovecot.org/configuration_manual/authentication/password_schemes/
     passhash = crypt_r.crypt(password, crypt_r.METHOD_SHA512)
     return "{SHA512-CRYPT}" + passhash
 
@@ -56,10 +52,6 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
         )
         return False
 
-    if not VALID_LOCALPART_RE.match(localpart):
-        logging.warning("localpart %r contains invalid characters", localpart)
-        return False
-
     return True
 
 
@@ -148,13 +140,8 @@ class AuthDictProxy(DictProxy):
         if not is_allowed_to_create(self.config, addr, cleartext_password):
             return
 
-        lock = filelock.FileLock(str(user.password_path) + ".lock", timeout=5)
+        user.set_password(encrypt_password(cleartext_password))
-        with lock:
+        print(f"Created address: {addr}", file=sys.stderr)
-            userdata = user.get_userdb_dict()
-            if userdata:
-                return userdata
-            user.set_password(encrypt_password(cleartext_password))
-            print(f"Created address: {addr}", file=sys.stderr)
         return user.get_userdb_dict()
 
 

chatmaild/src/chatmaild/expire.py

@@ -144,7 +144,7 @@ class Expiry:
                 continue
             changed = True
         if changed:
-            self.remove_file(f"{mbox.basedir}/maildirsize")
+            self.remove_file("maildirsize")
 
     def get_summary(self):
         return (

chatmaild/src/chatmaild/filtermail.py

@@ -0,0 +1,378 @@
+#!/usr/bin/env python3
+import asyncio
+import base64
+import binascii
+import sys
+import time
+from email import policy
+from email.parser import BytesParser
+from email.utils import parseaddr
+from smtplib import SMTP as SMTPClient
+
+from aiosmtpd.controller import Controller
+from aiosmtpd.smtp import SMTP
+
+from .config import read_config
+
+ENCRYPTION_NEEDED_523 = "523 Encryption Needed: Invalid Unencrypted Mail"
+
+
+def check_openpgp_payload(payload: bytes):
+    """Checks the OpenPGP payload.
+
+    OpenPGP payload must consist only of PKESK and SKESK packets
+    terminated by a single SEIPD packet.
+
+    Returns True if OpenPGP payload is correct,
+    False otherwise.
+
+    May raise IndexError while trying to read OpenPGP packet header
+    if it is truncated.
+    """
+    i = 0
+    while i < len(payload):
+        # Only OpenPGP format is allowed.
+        if payload[i] & 0xC0 != 0xC0:
+            return False
+
+        packet_type_id = payload[i] & 0x3F
+        i += 1
+
+        while payload[i] >= 224 and payload[i] < 255:
+            # Partial body length.
+            partial_length = 1 << (payload[i] & 0x1F)
+            i += 1 + partial_length
+
+        if payload[i] < 192:
+            # One-octet length.
+            body_len = payload[i]
+            i += 1
+        elif payload[i] < 224:
+            # Two-octet length.
+            body_len = ((payload[i] - 192) << 8) + payload[i + 1] + 192
+            i += 2
+        elif payload[i] == 255:
+            # Five-octet length.
+            body_len = (
+                (payload[i + 1] << 24)
+                | (payload[i + 2] << 16)
+                | (payload[i + 3] << 8)
+                | payload[i + 4]
+            )
+            i += 5
+        else:
+            # Impossible, partial body length was processed above.
+            return False
+
+        i += body_len
+
+        if i == len(payload):
+            # Last packet should be
+            # Symmetrically Encrypted and Integrity Protected Data Packet (SEIPD)
+            #
+            # This is the only place where this function may return `True`.
+            return packet_type_id == 18
+        elif packet_type_id not in [1, 3]:
+            # All packets except the last one must be either
+            # Public-Key Encrypted Session Key Packet (PKESK)
+            # or
+            # Symmetric-Key Encrypted Session Key Packet (SKESK)
+            return False
+
+    return False
+
+
+def check_armored_payload(payload: str, outgoing: bool):
+    """Check the armored PGP message for invalid content.
+
+    :param payload: the armored PGP message
+    :param outgoing: whether the message is outgoing or incoming
+    :return: whether the message is a valid PGP message
+    """
+    prefix = "-----BEGIN PGP MESSAGE-----\r\n"
+    if not payload.startswith(prefix):
+        return False
+    payload = payload.removeprefix(prefix)
+
+    while payload.endswith("\r\n"):
+        payload = payload.removesuffix("\r\n")
+    suffix = "-----END PGP MESSAGE-----"
+    if not payload.endswith(suffix):
+        return False
+    payload = payload.removesuffix(suffix)
+
+    version_comment = "Version: "
+    if payload.startswith(version_comment):
+        if outgoing:  # Disallow comments in outgoing messages
+            return False
+        # Remove comments from incoming messages
+        payload = payload.partition("\r\n")[2]
+
+    while payload.startswith("\r\n"):
+        payload = payload.removeprefix("\r\n")
+
+    # Remove CRC24.
+    payload = payload.rpartition("=")[0]
+
+    try:
+        payload = base64.b64decode(payload)
+    except binascii.Error:
+        return False
+
+    try:
+        return check_openpgp_payload(payload)
+    except IndexError:
+        return False
+
+
+def is_securejoin(message):
+    if message.get("secure-join") not in ["vc-request", "vg-request"]:
+        return False
+    if not message.is_multipart():
+        return False
+    parts_count = 0
+    for part in message.iter_parts():
+        parts_count += 1
+        if parts_count > 1:
+            return False
+        if part.is_multipart():
+            return False
+        if part.get_content_type() != "text/plain":
+            return False
+
+        payload = part.get_payload().strip().lower()
+        if payload not in ("secure-join: vc-request", "secure-join: vg-request"):
+            return False
+    return True
+
+
+def check_encrypted(message, outgoing=True):
+    """Check that the message is an OpenPGP-encrypted message.
+
+    MIME structure of the message must correspond to <https://www.rfc-editor.org/rfc/rfc3156>.
+    """
+    if not message.is_multipart():
+        return False
+    if message.get_content_type() != "multipart/encrypted":
+        return False
+    parts_count = 0
+    for part in message.iter_parts():
+        # We explicitly check Content-Type of each part later,
+        # but this is to be absolutely sure `get_payload()` returns string and not list.
+        if part.is_multipart():
+            return False
+
+        if parts_count == 0:
+            if part.get_content_type() != "application/pgp-encrypted":
+                return False
+
+            payload = part.get_payload()
+            if payload.strip() != "Version: 1":
+                return False
+        elif parts_count == 1:
+            if part.get_content_type() != "application/octet-stream":
+                return False
+
+            if not check_armored_payload(part.get_payload(), outgoing=outgoing):
+                return False
+        else:
+            return False
+        parts_count += 1
+    return True
+
+
+async def asyncmain_beforequeue(config, mode):
+    if mode == "outgoing":
+        port = config.filtermail_smtp_port
+        handler = OutgoingBeforeQueueHandler(config)
+    else:
+        port = config.filtermail_smtp_port_incoming
+        handler = IncomingBeforeQueueHandler(config)
+    HackedController(
+        handler,
+        hostname="127.0.0.1",
+        port=port,
+        data_size_limit=config.max_message_size,
+    ).start()
+
+
+def recipient_matches_passthrough(recipient, passthrough_recipients):
+    for addr in passthrough_recipients:
+        if recipient == addr:
+            return True
+        if addr[0] == "@" and recipient.endswith(addr):
+            return True
+    return False
+
+
+class HackedController(Controller):
+    def factory(self):
+        return SMTPDiscardRCPTO_options(self.handler, **self.SMTP_kwargs)
+
+
+class SMTPDiscardRCPTO_options(SMTP):
+    def _getparams(self, params):
+        # Ignore RCPT TO parameters.
+        #
+        # Otherwise parameters such as `ORCPT=...`
+        # or `NOTIFY=DELAY,FAILURE` (generated by Stalwart)
+        # make aiosmtpd reject the message here:
+        # <https://github.com/aio-libs/aiosmtpd/blob/98f578389ae86e5345cc343fa4e5a17b21d9c96d/aiosmtpd/smtp.py#L1379-L1384>
+        return {}
+
+
+class OutgoingBeforeQueueHandler:
+    def __init__(self, config):
+        self.config = config
+        self.send_rate_limiter = SendRateLimiter()
+
+    async def handle_MAIL(self, server, session, envelope, address, mail_options):
+        log_info(f"handle_MAIL from {address}")
+        envelope.mail_from = address
+        max_sent = self.config.max_user_send_per_minute
+        if not self.send_rate_limiter.is_sending_allowed(address, max_sent):
+            return f"450 4.7.1: Too much mail from {address}"
+
+        parts = envelope.mail_from.split("@")
+        if len(parts) != 2:
+            return f"500 Invalid from address <{envelope.mail_from!r}>"
+
+        return "250 OK"
+
+    async def handle_DATA(self, server, session, envelope):
+        loop = asyncio.get_running_loop()
+        return await loop.run_in_executor(None, self.sync_handle_DATA, envelope)
+
+    def sync_handle_DATA(self, envelope):
+        log_info("handle_DATA before-queue")
+        error = self.check_DATA(envelope)
+        if error:
+            return error
+        log_info("re-injecting the mail that passed checks")
+        client = SMTPClient("localhost", self.config.postfix_reinject_port)
+        client.sendmail(
+            envelope.mail_from, envelope.rcpt_tos, envelope.original_content
+        )
+        return "250 OK"
+
+    def check_DATA(self, envelope):
+        """the central filtering function for e-mails."""
+        log_info(f"Processing DATA message from {envelope.mail_from}")
+
+        message = BytesParser(policy=policy.default).parsebytes(envelope.content)
+        mail_encrypted = check_encrypted(message, outgoing=True)
+
+        _, from_addr = parseaddr(message.get("from").strip())
+
+        if envelope.mail_from.lower() != from_addr.lower():
+            return f"500 Invalid FROM <{from_addr!r}> for <{envelope.mail_from!r}>"
+
+        if mail_encrypted or is_securejoin(message):
+            print("Outgoing: Filtering encrypted mail.", file=sys.stderr)
+            return
+
+        print("Outgoing: Filtering unencrypted mail.", file=sys.stderr)
+
+        if envelope.mail_from in self.config.passthrough_senders:
+            return
+
+        # allow self-sent Autocrypt Setup Message
+        if envelope.rcpt_tos == [from_addr]:
+            if message.get("subject") == "Autocrypt Setup Message":
+                if message.get_content_type() == "multipart/mixed":
+                    return
+
+        passthrough_recipients = self.config.passthrough_recipients
+
+        for recipient in envelope.rcpt_tos:
+            if recipient_matches_passthrough(recipient, passthrough_recipients):
+                continue
+
+            print("Rejected unencrypted mail.", file=sys.stderr)
+            return ENCRYPTION_NEEDED_523
+
+
+class IncomingBeforeQueueHandler:
+    def __init__(self, config):
+        self.config = config
+
+    async def handle_DATA(self, server, session, envelope):
+        loop = asyncio.get_running_loop()
+        return await loop.run_in_executor(None, self.sync_handle_DATA, envelope)
+
+    def sync_handle_DATA(self, envelope):
+        log_info("handle_DATA before-queue")
+        error = self.check_DATA(envelope)
+        if error:
+            return error
+        log_info("re-injecting the mail that passed checks")
+
+        client = SMTPClient(
+            "localhost",
+            self.config.postfix_reinject_port_incoming,
+        )
+        client.sendmail(
+            envelope.mail_from, envelope.rcpt_tos, envelope.original_content
+        )
+        return "250 OK"
+
+    def check_DATA(self, envelope):
+        """the central filtering function for e-mails."""
+        log_info(f"Processing DATA message from {envelope.mail_from}")
+
+        message = BytesParser(policy=policy.default).parsebytes(envelope.content)
+        mail_encrypted = check_encrypted(message, outgoing=False)
+
+        if mail_encrypted or is_securejoin(message):
+            print("Incoming: Filtering encrypted mail.", file=sys.stderr)
+            return
+
+        print("Incoming: Filtering unencrypted mail.", file=sys.stderr)
+
+        # we want cleartext mailer-daemon messages to pass through
+        # chatmail core will typically not display them as normal messages
+        if message.get("auto-submitted"):
+            _, from_addr = parseaddr(message.get("from").strip())
+            if from_addr.lower().startswith("mailer-daemon@"):
+                if message.get_content_type() == "multipart/report":
+                    return
+
+        for recipient in envelope.rcpt_tos:
+            user = self.config.get_user(recipient)
+            if user is None or user.is_incoming_cleartext_ok():
+                continue
+
+            print("Rejected unencrypted mail.", file=sys.stderr)
+            return ENCRYPTION_NEEDED_523
+
+
+class SendRateLimiter:
+    def __init__(self):
+        self.addr2timestamps = {}
+
+    def is_sending_allowed(self, mail_from, max_send_per_minute):
+        last = self.addr2timestamps.setdefault(mail_from, [])
+        now = time.time()
+        last[:] = [ts for ts in last if ts >= (now - 60)]
+        if len(last) <= max_send_per_minute:
+            last.append(now)
+            return True
+        return False
+
+
+def log_info(msg):
+    print(msg, file=sys.stderr)
+
+
+def main():
+    args = sys.argv[1:]
+    assert len(args) == 2
+    config = read_config(args[0])
+    mode = args[1]
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+    assert mode in ["incoming", "outgoing"]
+    task = asyncmain_beforequeue(config, mode)
+    loop.create_task(task)
+    log_info("entering serving loop")
+    loop.run_forever()

chatmaild/src/chatmaild/fsreport.py

@@ -68,7 +68,7 @@ class Report:
             for size in self.message_buckets:
                 for msg in mailbox.messages:
                     if msg.size >= size:
-                        if self.mdir and f"/{self.mdir}/" not in msg.path:
+                        if self.mdir and not msg.relpath.startswith(self.mdir):
                             continue
                         self.message_buckets[size] += msg.size
 

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -11,12 +11,9 @@ mail_domain = {mail_domain}
 # Restrictions on user addresses
 #
 
-# email sending rate per user and minute
+# how many mails a user can send out per minute
 max_user_send_per_minute = 60
 
-# per-user max burst size for sending rate limiting (GCRA bucket capacity)
-max_user_send_burst_size = 10
-
 # maximum mailbox size of a chatmail address
 max_mailbox_size = 500M
 

chatmaild/src/chatmaild/metadata.py

@@ -101,11 +101,7 @@ class MetadataDictProxy(DictProxy):
                 # Handle `GETMETADATA "" /shared/vendor/deltachat/irohrelay`
                 return f"O{self.iroh_relay}\n"
             elif keyname == "vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn":
-                try:
+                res = turn_credentials()
-                    res = turn_credentials()
-                except Exception:
-                    logging.exception("failed to get TURN credentials")
-                    return "N\n"
                 port = 3478
                 return f"O{self.turn_hostname}:{port}:{res}\n"
 

chatmaild/src/chatmaild/newemail.py

@@ -3,6 +3,7 @@
 """CGI script for creating new accounts."""
 
 import json
+import random
 import secrets
 import string
 
@@ -14,9 +15,7 @@ ALPHANUMERIC_PUNCT = string.ascii_letters + string.digits + string.punctuation
 
 
 def create_newemail_dict(config: Config):
-    user = "".join(
+    user = "".join(random.choices(ALPHANUMERIC, k=config.username_max_length))
-        secrets.choice(ALPHANUMERIC) for _ in range(config.username_max_length)
-    )
     password = "".join(
         secrets.choice(ALPHANUMERIC_PUNCT)
         for _ in range(config.password_min_length + 3)

chatmaild/src/chatmaild/tests/test_doveauth.py

@@ -120,60 +120,6 @@ def test_handle_dovecot_protocol_iterate(gencreds, example_config):
     assert not lines[2]
 
 
-def test_invalid_localpart_characters(make_config):
-    """Test that is_allowed_to_create rejects localparts with invalid characters."""
-    config = make_config("chat.example.org", {"username_min_length": "3"})
-    password = "zequ0Aimuchoodaechik"
-    domain = config.mail_domain
-
-    # valid localparts
-    assert is_allowed_to_create(config, f"abc123@{domain}", password)
-    assert is_allowed_to_create(config, f"a.b-c_d@{domain}", password)
-
-    # uppercase rejected
-    assert not is_allowed_to_create(config, f"Abc123@{domain}", password)
-    assert not is_allowed_to_create(config, f"ABCDEFG@{domain}", password)
-
-    # spaces and special chars rejected
-    assert not is_allowed_to_create(config, f"a b cde@{domain}", password)
-    assert not is_allowed_to_create(config, f"abc+def@{domain}", password)
-    assert not is_allowed_to_create(config, f"abc!def@{domain}", password)
-    assert not is_allowed_to_create(config, f"ab@cdef@{domain}", password)
-    assert not is_allowed_to_create(config, f"abc/def@{domain}", password)
-    assert not is_allowed_to_create(config, f"abc\\def@{domain}", password)
-
-
-def test_concurrent_creation_same_account(dictproxy):
-    """Test that concurrent creation of the same account doesn't corrupt password."""
-    addr = "racetest1@chat.example.org"
-    password = "zequ0Aimuchoodaechik"
-    num_threads = 10
-    results = queue.Queue()
-
-    def create():
-        try:
-            res = dictproxy.lookup_passdb(addr, password)
-            results.put(("ok", res))
-        except Exception:
-            results.put(("err", traceback.format_exc()))
-
-    threads = [threading.Thread(target=create, daemon=True) for _ in range(num_threads)]
-    for t in threads:
-        t.start()
-    for t in threads:
-        t.join(timeout=10)
-
-    passwords_seen = set()
-    for _ in range(num_threads):
-        status, res = results.get()
-        if status == "err":
-            pytest.fail(f"concurrent creation failed\n{res}")
-        passwords_seen.add(res["password"])
-
-    # all threads must see the same password hash
-    assert len(passwords_seen) == 1
-
-
 def test_50_concurrent_lookups_different_accounts(gencreds, dictproxy):
     num_threads = 50
     req_per_thread = 5

chatmaild/src/chatmaild/tests/test_expire.py

@@ -112,43 +112,6 @@ def test_report(mbox1, example_config):
     report_main(args)
 
 
-def test_report_mdir_filters_by_path(mbox1, example_config):
-    """Test that Report with mdir='cur' only counts messages in cur/ subdirectory."""
-    from chatmaild.fsreport import Report
-
-    now = datetime.utcnow().timestamp()
-
-    # Set password mtime to old enough so min_login_age check passes
-    password = Path(mbox1.basedir).joinpath("password")
-    old_time = now - 86400 * 10  # 10 days ago
-    os.utime(password, (old_time, old_time))
-
-    # Reload mailbox with updated mtime
-    from chatmaild.expire import MailboxStat
-
-    mbox = MailboxStat(mbox1.basedir)
-
-    # Report without mdir — should count all messages
-    rep_all = Report(now=now, min_login_age=1, mdir=None)
-    rep_all.process_mailbox_stat(mbox)
-    total_all = rep_all.message_buckets[0]
-
-    # Report with mdir='cur' — should only count cur/ messages
-    rep_cur = Report(now=now, min_login_age=1, mdir="cur")
-    rep_cur.process_mailbox_stat(mbox)
-    total_cur = rep_cur.message_buckets[0]
-
-    # Report with mdir='new' — should only count new/ messages
-    rep_new = Report(now=now, min_login_age=1, mdir="new")
-    rep_new.process_mailbox_stat(mbox)
-    total_new = rep_new.message_buckets[0]
-
-    # cur has 500-byte msg, new has 600-byte msg (from fill_mbox)
-    assert total_cur == 500
-    assert total_new == 600
-    assert total_all == 500 + 600
-
-
 def test_expiry_cli_basic(example_config, mbox1):
     args = (str(example_config._inipath),)
     expiry_main(args)

chatmaild/src/chatmaild/tests/test_filtermail.py

@@ -0,0 +1,361 @@
+import pytest
+
+from chatmaild.filtermail import (
+    IncomingBeforeQueueHandler,
+    OutgoingBeforeQueueHandler,
+    SendRateLimiter,
+    check_armored_payload,
+    check_encrypted,
+    is_securejoin,
+)
+
+
+@pytest.fixture
+def maildomain():
+    # let's not depend on a real chatmail instance for the offline tests below
+    return "chatmail.example.org"
+
+
+@pytest.fixture
+def handler(make_config, maildomain):
+    config = make_config(maildomain)
+    return OutgoingBeforeQueueHandler(config)
+
+
+@pytest.fixture
+def inhandler(make_config, maildomain):
+    config = make_config(maildomain)
+    return IncomingBeforeQueueHandler(config)
+
+
+def test_reject_forged_from(maildata, gencreds, handler):
+    class env:
+        mail_from = gencreds()[0]
+        rcpt_tos = [gencreds()[0]]
+
+    # test that the filter lets good mail through
+    to_addr = gencreds()[0]
+    env.content = maildata(
+        "encrypted.eml", from_addr=env.mail_from, to_addr=to_addr
+    ).as_bytes()
+
+    assert not handler.check_DATA(envelope=env)
+
+    # test that the filter rejects forged mail
+    env.content = maildata(
+        "encrypted.eml", from_addr="forged@c3.testrun.org", to_addr=to_addr
+    ).as_bytes()
+    error = handler.check_DATA(envelope=env)
+    assert "500" in error
+
+
+def test_filtermail_no_encryption_detection(maildata):
+    msg = maildata(
+        "plain.eml", from_addr="some@example.org", to_addr="other@example.org"
+    )
+    assert not check_encrypted(msg)
+
+    # https://xkcd.com/1181/
+    msg = maildata(
+        "fake-encrypted.eml", from_addr="some@example.org", to_addr="other@example.org"
+    )
+    assert not check_encrypted(msg)
+
+
+def test_filtermail_securejoin_detection(maildata):
+    msg = maildata(
+        "securejoin-vc.eml", from_addr="some@example.org", to_addr="other@example.org"
+    )
+    assert is_securejoin(msg)
+
+    msg = maildata(
+        "securejoin-vc-fake.eml",
+        from_addr="some@example.org",
+        to_addr="other@example.org",
+    )
+    assert not is_securejoin(msg)
+
+
+def test_filtermail_encryption_detection(maildata):
+    msg = maildata(
+        "encrypted.eml",
+        from_addr="1@example.org",
+        to_addr="2@example.org",
+        subject="Subject does not matter, will be replaced anyway",
+    )
+    assert check_encrypted(msg)
+
+
+def test_filtermail_no_literal_packets(maildata):
+    """Test that literal OpenPGP packet is not considered an encrypted mail."""
+    msg = maildata("literal.eml", from_addr="1@example.org", to_addr="2@example.org")
+    assert not check_encrypted(msg)
+
+
+def test_filtermail_unencrypted_mdn(maildata, gencreds):
+    """Unencrypted MDNs should not pass."""
+    from_addr = gencreds()[0]
+    to_addr = gencreds()[0] + ".other"
+    msg = maildata("mdn.eml", from_addr=from_addr, to_addr=to_addr)
+
+    assert not check_encrypted(msg)
+
+
+def test_send_rate_limiter():
+    limiter = SendRateLimiter()
+    for i in range(100):
+        if limiter.is_sending_allowed("some@example.org", 10):
+            if i <= 10:
+                continue
+            pytest.fail("limiter didn't work")
+        else:
+            assert i == 11
+            break
+
+
+def test_cleartext_excempt_privacy(maildata, gencreds, handler):
+    from_addr = gencreds()[0]
+    to_addr = "privacy@testrun.org"
+    handler.config.passthrough_recipients = [to_addr]
+    false_to = "privacy@something.org"
+
+    msg = maildata("plain.eml", from_addr=from_addr, to_addr=to_addr)
+
+    class env:
+        mail_from = from_addr
+        rcpt_tos = [to_addr]
+        content = msg.as_bytes()
+
+    # assert that None/no error is returned
+    assert not handler.check_DATA(envelope=env)
+
+    class env2:
+        mail_from = from_addr
+        rcpt_tos = [to_addr, false_to]
+        content = msg.as_bytes()
+
+    assert "523" in handler.check_DATA(envelope=env2)
+
+
+def test_cleartext_self_send_autocrypt_setup_message(maildata, gencreds, handler):
+    from_addr = gencreds()[0]
+    to_addr = from_addr
+
+    msg = maildata("asm.eml", from_addr=from_addr, to_addr=to_addr)
+
+    class env:
+        mail_from = from_addr
+        rcpt_tos = [to_addr]
+        content = msg.as_bytes()
+
+    assert not handler.check_DATA(envelope=env)
+
+
+def test_cleartext_send_fails(maildata, gencreds, handler):
+    from_addr = gencreds()[0]
+    to_addr = gencreds()[0]
+
+    msg = maildata("plain.eml", from_addr=from_addr, to_addr=to_addr)
+
+    class env:
+        mail_from = from_addr
+        rcpt_tos = [to_addr]
+        content = msg.as_bytes()
+
+    res = handler.check_DATA(envelope=env)
+    assert "523 Encryption Needed" in res
+
+
+def test_cleartext_incoming_fails(maildata, gencreds, inhandler):
+    from_addr = gencreds()[0]
+    to_addr, password = gencreds()
+
+    msg = maildata("plain.eml", from_addr=from_addr, to_addr=to_addr)
+
+    class env:
+        mail_from = from_addr
+        rcpt_tos = [to_addr]
+        content = msg.as_bytes()
+
+    user = inhandler.config.get_user(to_addr)
+    user.set_password(password)
+    res = inhandler.check_DATA(envelope=env)
+    assert "523 Encryption Needed" in res
+
+    user.allow_incoming_cleartext()
+    assert not inhandler.check_DATA(envelope=env)
+
+
+def test_cleartext_incoming_mailer_daemon(maildata, gencreds, inhandler):
+    from_addr = "mailer-daemon@example.org"
+    to_addr = gencreds()[0]
+
+    msg = maildata("mailer-daemon.eml", from_addr=from_addr, to_addr=to_addr)
+
+    class env:
+        mail_from = from_addr
+        rcpt_tos = [to_addr]
+        content = msg.as_bytes()
+
+    assert not inhandler.check_DATA(envelope=env)
+
+
+def test_cleartext_passthrough_domains(maildata, gencreds, handler):
+    from_addr = gencreds()[0]
+    to_addr = "privacy@x.y.z"
+    handler.config.passthrough_recipients = ["@x.y.z"]
+    false_to = "something@x.y"
+
+    msg = maildata("plain.eml", from_addr=from_addr, to_addr=to_addr)
+
+    class env:
+        mail_from = from_addr
+        rcpt_tos = [to_addr]
+        content = msg.as_bytes()
+
+    # assert that None/no error is returned
+    assert not handler.check_DATA(envelope=env)
+
+    class env2:
+        mail_from = from_addr
+        rcpt_tos = [to_addr, false_to]
+        content = msg.as_bytes()
+
+    assert "523" in handler.check_DATA(envelope=env2)
+
+
+def test_cleartext_passthrough_senders(gencreds, handler, maildata):
+    acc1 = gencreds()[0]
+    to_addr = "recipient@something.org"
+    handler.config.passthrough_senders = [acc1]
+
+    msg = maildata("plain.eml", from_addr=acc1, to_addr=to_addr)
+
+    class env:
+        mail_from = acc1
+        rcpt_tos = to_addr
+        content = msg.as_bytes()
+
+    # assert that None/no error is returned
+    assert not handler.check_DATA(envelope=env)
+
+
+def test_check_armored_payload():
+    prefix = "-----BEGIN PGP MESSAGE-----\r\n"
+    comment = "Version: ProtonMail\r\n"
+    payload = """\r
+wU4DSqFx0d1yqAoSAQdAYkX/ZN/Az4B0k7X47zKyWrXxlDEdS3WOy0Yf2+GJTFgg\r
+Zk5ql0mLG8Ze+ZifCS0XMO4otlemSyJ0K1ZPdFMGzUDBTgNqzkFabxXoXRIBB0AM\r
+755wlX41X6Ay3KhnwBq7yEqSykVH6F3x11iHPKraLCAGZoaS8bKKNy/zg5slda1X\r
+pt14b4aC1VwtSnYhcRRELNLD/wE2TFif+g7poMmFY50VyMPLYjVP96Z5QCT4+z4H\r
+Ikh/pRRN8S3JNMrRJHc6prooSJmLcx47Y5un7VFy390MsJ+LiUJuQMDdYWRAinfs\r
+Ebm89Ezjm7F03qbFPXE0X4ZNzVXS/eKO0uhJQdiov/vmbn41rNtHmNpqjaO0vi5+\r
+sS9tR7yDUrIXiCUCN78eBLVioxtktsPZm5cDORbQWzv+7nmCEz9/JowCUcBVdCGn\r
+1ofOaH82JCAX/cRx08pLaDNj6iolVBsi56Dd+2bGxJOZOG2AMcEyz0pXY0dOAJCD\r
+iUThcQeGIdRnU3j8UBcnIEsjLu2+C+rrwMZQESMWKnJ0rnqTk0pK5kXScr6F/L0L\r
+UE49ccIexNm3xZvYr5drszr6wz3Tv5fdue87P4etBt90gF/Vzknck+g1LLlkzZkp\r
+d8dI0k2tOSPjUbDPnSy1x+X73WGpPZmj0kWT+RGvq0nH6UkJj3AQTG2qf1T8jK+3\r
+rTp3LR9vDkMwDjX4R8SA9c0wdnUzzr79OYQC9lTnzcx+fM6BBmgQ2GrS33jaFLp7\r
+L6/DFpCl5zhnPjM/2dKvMkw/Kd6XS/vjwsO405FQdjSDiQEEAZA+ZvAfcjdccbbU\r
+yCO+x0QNdeBsufDVnh3xvzuWy4CICdTQT4s1AWRPCzjOj+SGmx5WqCLWfsd8Ma0+\r
+w/C7SfTYu1FDQILLM+llpq1M/9GPley4QZ8JQjo262AyPXsPF/OW48uuZz0Db1xT\r
+Yh4iHBztj4VSdy7l2+IyaIf7cnL4EEBFxv/MwmVDXvDlxyvfAfIsd3D9SvJESzKZ\r
+VWDYwaocgeCN+ojKu1p885lu1EfRbX3fr3YO02K5/c2JYDkc0Py0W3wUP/J1XUax\r
+pbKpzwlkxEgtmzsGqsOfMJqBV3TNDrOA2uBsa+uBqP5MGYLZ49S/4v/bW9I01Cr1\r
+D2ZkV510Y1Vgo66WlP8mRqOTyt/5WRhPD+MxXdk67BNN/PmO6tMlVoJDuk+XwWPR\r
+t2TvNaND/yabT9eYI55Og4fzKD6RIjouUX8DvKLkm+7aXxVs2uuLQ3Jco3O82z55\r
+dbShU1jYsrw9oouXUz06MHPbkdhNbF/2hfhZ2qA31sNeovJw65iUv7sDKX3LVWgJ\r
+10jlywcDwqlU8CO7WC9lGixYTbnOkYZpXCGEl8e6Jbs79l42YFo4ogYpFK1NXFhV\r
+kOXRmDf/wmfj+c/ld3L2PkvwlgofhCudOQknZbo3ub1gjiTn7L+lMGHIj/3suMIl\r
+ID4EUxAXScIM1ZEz2fjtW5jATlqYcLjLTbf/olw6HFyPNH+9IssqXeZNKnGwPUB9\r
+3lTXsg0tpzl+x7F/2WjEw1DSNhjC0KnHt1vEYNMkUGDGFdN9y3ERLqX/FIgiASUb\r
+bTvAVupnAK3raBezGmhrs6LsQtLS9P0VvQiLU3uDhMqw8Z4SISLpcD+NnVBHzQqm\r
+6W5Qn/8xsCL6av18yUVTi2G3igt3QCNoYx9evt2ZcIkNoyyagUVjfZe5GHXh8Dnz\r
+GaBXW/hg3HlXLRGaQu4RYCzBMJILcO25OhZOg6jbkCLiEexQlm2e9krB5cXR49Al\r
+UN4fiB0KR9JyG2ayUdNJVkXZSZLnHyRgiaadlpUo16LVvw==\r
+=b5Kp\r
+-----END PGP MESSAGE-----\r
+\r
+\r
+"""
+
+    commented_payload = prefix + comment + payload
+    assert check_armored_payload(commented_payload, outgoing=False) == True
+    assert check_armored_payload(commented_payload, outgoing=True) == False
+
+    payload = prefix + payload
+    assert check_armored_payload(payload, outgoing=False) == True
+    assert check_armored_payload(payload, outgoing=True) == True
+
+    payload = payload.removesuffix("\r\n")
+    assert check_armored_payload(payload, outgoing=False) == True
+    assert check_armored_payload(payload, outgoing=True) == True
+
+    payload = payload.removesuffix("\r\n")
+    assert check_armored_payload(payload, outgoing=False) == True
+    assert check_armored_payload(payload, outgoing=True) == True
+
+    payload = payload.removesuffix("\r\n")
+    assert check_armored_payload(payload, outgoing=False) == True
+    assert check_armored_payload(payload, outgoing=True) == True
+
+    payload = """-----BEGIN PGP MESSAGE-----\r
+\r
+HELLOWORLD
+-----END PGP MESSAGE-----\r
+\r
+"""
+    assert check_armored_payload(payload, outgoing=False) == False
+    assert check_armored_payload(payload, outgoing=True) == False
+
+    payload = """-----BEGIN PGP MESSAGE-----\r
+\r
+=njUN
+-----END PGP MESSAGE-----\r
+\r
+"""
+    assert check_armored_payload(payload, outgoing=False) == False
+    assert check_armored_payload(payload, outgoing=True) == False
+
+    # Test payload using partial body length
+    # as generated by GopenPGP.
+    payload = """-----BEGIN PGP MESSAGE-----\r
+\r
+wV4DdCVjRfOT3TQSAQdAY5+pjT6mlCxPGdR3be4w7oJJRUGIPI/Vnh+mJxGSm34w\r
+LNlVc89S1g22uQYFif2sUJsQWbpoHpNkuWpkSgOaHmNvrZiY/YU5iv+cZ3LbmtUG\r
+0uoBisSHh9O1c+5sYZSbrvYZ1NOwlD7Fv/U5/Mw4E5+CjxfdgNGp5o3DDddzPK78\r
+jseDhdSXxnaiIJC93hxNX6R1RPt3G2gukyzx69wciPQShcF8zf3W3o75Ed7B8etV\r
+QEeB16xzdFhKa9JxdjTu3osgCs21IO7wpcFkjc7nZzlW6jPnELJJaNmv4yOOCjMp\r
+6YAkaN/BkL+jHTznHDuDsT5ilnTXpwHDU1Cm9PIx/KFcNCQnIB+2DcdIHPHUH1ci\r
+jvqoeXAVWjKXEjS7PqPFuP/xGbrWG2ugs+toXJOKbgRkExvKs1dwPFKrgghvCVbW\r
+AcKejQKAPArLwpkA7aD875TZQShvGt74fNs45XBlGOYOnNOAJ1KAmzrXLIDViyyB\r
+kDsmTBk785xofuCkjBpXSe6vsMprPzCteDfaUibh8FHeJjucxPerwuOPEmnogNaf\r
+YyL4+iy8H8I9/p7pmUqILprxTG0jTOtlk0bTVzeiF56W1xbtSEMuOo4oFbQTyOM2\r
+bKXaYo774Jm+rRtKAnnI2dtf9RpK19cog6YNzfYjesLKbXDsPZbN5rmwyFiCvvxC\r
+kQ6JLob+B2fPdY2gzy7LypxktS8Zi1HJcWDHJGVmQodaDLqKUObb4M26bXDe6oxI\r
+NS8PJz5exVbM3KhZnUOEn6PJRBBf5a/ZqxlhZPcQo/oBuhKpBRpO5kSDwPIUByu3\r
+UlXLSkpMqe9pUarAOEuQjfl2RVY7U+RrQYp4YP5keMO+i8NCefAFbowTTufO1JIq\r
+2nVgCi/QVnxZyEc9OYt/8AE3g4cdojE+vsSDifZLSWYIetpfrohHv3dT3StD1QRG\r
+0QE6qq6oKpg/IL0cjvuX4c7a7bslv2fXp8t75y37RU6253qdIebhxc/cRhPbc/yu\r
+p0YLyD4SrvKTLP2ZV95jT4IPEpqm4AN3QmiOzdtqR2gLyb62L8QfqI/FdwsIiRiM\r
+hqydwoqt/lfSqG1WKPh+6EkMkH+TDiCC1BQdbN1MNcyUtcjb35PR2c8Ld2TF3guA\r
+jLIqMt/Vb7hBoMb2FcsOYY25ka9oV62OwgKWLXnFzk+modMR5fzb4kxVVAYEqP+D\r
+T5KO1Vs76v1fyPGOq6BbBCvLwTqe/e6IZInJles4v5jrhnLcGKmNGivCUDe6X6NY\r
+UKNt5RsZllwDQpaAb5dMNhyrk8SgIE7TBI7rvqIdUCE52Vy+0JDxFg5olRpFUfO6\r
+/MyTW3Yo/ekk/npHr7iYYqJTCc21bDGLWQcIo/XO7WPxrKNWGBNPFnkRdw0MaKr4\r
++cEM3V8NFnSEpC12xA+RX/CezuJtwXZK5MpG76eYqMO6qyC+c25YcFecEufDZDxx\r
+ZLqRszVRyxyWPtk/oIeQK2v9wOqY6N9/ff01gHz69vqYqN5bUw/QKZsmx1zW+gPw\r
+6x2tDK2BHeYl182gCbhlKISRFwCtbjqZSkiKWao/VtygHkw0fK34avJuyQ/X9YaN\r
+BRy+7Lf3VA53pnB5WJ1xwRXN8VDvmZeXzv2krHveCMemj0OjnRoCLu117xN0A5m9\r
+Fm/RoDix5PolDHtWTtr2m1n2hp2LHnj8at9lFEd0SKhAYHVL9KjzycwWODZRXt+x\r
+zGDDuooEeTvdY5NLyKcl4gETz1ZP4Ez5jGGjhPSwSpq1mU7UaJ9ZXXdr4KHyifW6\r
+ggNzNsGhXTap7IWZpTtqXABydfiBshmH2NjqtNDwBweJVSgP10+r0WhMWlaZs6xl\r
+V3o5yskJt6GlkwpJxZrTvN6Tiww/eW7HFV6NGf7IRSWY5tJc/iA7/92tOmkdvJ1q\r
+myLbG7cJB787QjplEyVe2P/JBO6xYvbkJLf9Q+HaviTO25rugRSrYsoKMDfO8VlQ\r
+1CcnTPVtApPZJEQzAWJEgVAM8uIlkqWJJMgyWT34sTkdBeCUFGloXQFs9Yxd0AGf\r
+/zHEkYZSTKpVSvAIGu4=\r
+=6iHb\r
+-----END PGP MESSAGE-----\r
+"""
+    assert check_armored_payload(payload, outgoing=False) == True
+    assert check_armored_payload(payload, outgoing=True) == True

chatmaild/src/chatmaild/tests/test_metadata.py

@@ -314,51 +314,6 @@ def test_persistent_queue_items(tmp_path, testaddr, token):
     assert not queue_item < item2 and not item2 < queue_item
 
 
-def test_turn_credentials_exception_returns_N(notifier, metadata, monkeypatch):
-    """Test that turn_credentials() failure returns N\\n instead of crashing."""
-    import chatmaild.metadata
-
-    dictproxy = MetadataDictProxy(
-        notifier=notifier,
-        metadata=metadata,
-        turn_hostname="turn.example.org",
-    )
-
-    def mock_turn_credentials():
-        raise ConnectionRefusedError("socket not available")
-
-    monkeypatch.setattr(chatmaild.metadata, "turn_credentials", mock_turn_credentials)
-
-    transactions = {}
-    res = dictproxy.handle_dovecot_request(
-        "Lshared/0123/vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn"
-        "\tuser@example.org",
-        transactions,
-    )
-    assert res == "N\n"
-
-
-def test_turn_credentials_success(notifier, metadata, monkeypatch):
-    """Test that valid turn_credentials() returns TURN URI."""
-    import chatmaild.metadata
-
-    dictproxy = MetadataDictProxy(
-        notifier=notifier,
-        metadata=metadata,
-        turn_hostname="turn.example.org",
-    )
-
-    monkeypatch.setattr(chatmaild.metadata, "turn_credentials", lambda: "user:pass")
-
-    transactions = {}
-    res = dictproxy.handle_dovecot_request(
-        "Lshared/0123/vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn"
-        "\tuser@example.org",
-        transactions,
-    )
-    assert res == "Oturn.example.org:3478:user:pass\n"
-
-
 def test_iroh_relay(dictproxy):
     rfile = io.BytesIO(
         b"\n".join(

chatmaild/src/chatmaild/tests/test_turnserver.py

@@ -1,73 +0,0 @@
-import socket
-import threading
-import time
-from unittest.mock import patch
-
-import pytest
-
-from chatmaild.turnserver import turn_credentials
-
-SOCKET_PATH = "/run/chatmail-turn/turn.socket"
-
-
-@pytest.fixture
-def turn_socket(tmp_path):
-    """Create a real Unix socket server at a temp path."""
-    sock_path = str(tmp_path / "turn.socket")
-    server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-    server.bind(sock_path)
-    server.listen(1)
-    yield sock_path, server
-    server.close()
-
-
-def _call_turn_credentials(sock_path):
-    """Call turn_credentials but connect to sock_path instead of hardcoded path."""
-    original_connect = socket.socket.connect
-
-    def patched_connect(self, address):
-        if address == SOCKET_PATH:
-            address = sock_path
-        return original_connect(self, address)
-
-    with patch.object(socket.socket, "connect", patched_connect):
-        return turn_credentials()
-
-
-def test_turn_credentials_timeout(turn_socket):
-    """Server accepts but never responds — must raise socket.timeout."""
-    sock_path, server = turn_socket
-
-    def accept_and_hang():
-        conn, _ = server.accept()
-        time.sleep(30)
-        conn.close()
-
-    t = threading.Thread(target=accept_and_hang, daemon=True)
-    t.start()
-
-    with pytest.raises(socket.timeout):
-        _call_turn_credentials(sock_path)
-
-
-def test_turn_credentials_connection_refused(tmp_path):
-    """Socket file doesn't exist — must raise ConnectionRefusedError or FileNotFoundError."""
-    missing = str(tmp_path / "nonexistent.socket")
-    with pytest.raises((ConnectionRefusedError, FileNotFoundError)):
-        _call_turn_credentials(missing)
-
-
-def test_turn_credentials_success(turn_socket):
-    """Server responds with credentials — must return stripped string."""
-    sock_path, server = turn_socket
-
-    def respond():
-        conn, _ = server.accept()
-        conn.sendall(b"testuser:testpass\n")
-        conn.close()
-
-    t = threading.Thread(target=respond, daemon=True)
-    t.start()
-
-    result = _call_turn_credentials(sock_path)
-    assert result == "testuser:testpass"

chatmaild/src/chatmaild/turnserver.py

@@ -4,7 +4,6 @@ import socket
 
 def turn_credentials() -> str:
     with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client_socket:
-        client_socket.settimeout(5)
         client_socket.connect("/run/chatmail-turn/turn.socket")
         with client_socket.makefile("rb") as file:
             return file.readline().decode("utf-8").strip()

cmdeploy/src/cmdeploy/acmetool/__init__.py

@@ -67,7 +67,7 @@ class AcmetoolDeployer(Deployer):
         )
         files.template(
             src=importlib.resources.files(__package__).joinpath("desired.yaml.j2"),
-            dest=f"/var/lib/acme/desired/{self.domains[0]}",  # 0 is mailhost TLD
+            dest=f"/var/lib/acme/desired/{self.domains[0]}", # 0 is mailhost TLD
             user="root",
             group="root",
             mode="644",

cmdeploy/src/cmdeploy/basedeploy.py

@@ -17,8 +17,9 @@ def configure_remote_units(mail_domain, units) -> None:
 
     # install systemd units
     for fn in units:
+        execpath = fn if fn != "filtermail-incoming" else "filtermail"
         params = dict(
-            execpath=f"{remote_venv_dir}/bin/{fn}",
+            execpath=f"{remote_venv_dir}/bin/{execpath}",
             config_path=remote_chatmail_inipath,
             remote_venv_dir=remote_venv_dir,
             mail_domain=mail_domain,

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -71,11 +71,6 @@ def run_cmd_options(parser):
         action="store_true",
         help="install/upgrade the server, but disable postfix & dovecot for now",
     )
-    parser.add_argument(
-        "--website-only",
-        action="store_true",
-        help="only update/deploy the website, skipping full server upgrade/deployment, useful when you only changed/updated the web pages and don't need to re-run a full server upgrade",
-    )
     parser.add_argument(
         "--skip-dns-check",
         dest="dns_check_disabled",
@@ -98,7 +93,6 @@ def run_cmd(args, out):
 
     env = os.environ.copy()
     env["CHATMAIL_INI"] = args.inipath
-    env["CHATMAIL_WEBSITE_ONLY"] = "True" if args.website_only else ""
     env["CHATMAIL_DISABLE_MAIL"] = "True" if args.disable_mail else ""
     env["CHATMAIL_REQUIRE_IROH"] = "True" if require_iroh else ""
     deploy_path = importlib.resources.files(__package__).joinpath("run.py").resolve()
@@ -113,15 +107,19 @@ def run_cmd(args, out):
         return 1
 
     try:
-        out.check_call(cmd, env=env)
+        retcode = out.check_call(cmd, env=env)
-        if args.website_only:
+        if retcode == 0:
-            out.green("Website deployment completed.")
-        else:
             out.green("Deploy completed, call `cmdeploy dns` next.")
-        return 0
+        elif not remote_data["acme_account_url"]:
+            out.red("Deploy completed but letsencrypt not configured")
+            out.red("Run 'cmdeploy run' again")
+            retcode = 0
+        else:
+            out.red("Deploy failed")
     except subprocess.CalledProcessError:
         out.red("Deploy failed")
-        return 1
+        retcode = 1
+    return retcode
 
 
 def dns_cmd_options(parser):

cmdeploy/src/cmdeploy/deployers.py

@@ -26,7 +26,6 @@ from .basedeploy import (
     get_resource,
 )
 from .dovecot.deployer import DovecotDeployer
-from .filtermail.deployer import FiltermailDeployer
 from .mtail.deployer import MtailDeployer
 from .nginx.deployer import NginxDeployer
 from .opendkim.deployer import OpendkimDeployer
@@ -141,10 +140,6 @@ def _configure_remote_venv_with_chatmaild(config) -> None:
 
 
 class UnboundDeployer(Deployer):
-    def __init__(self, config):
-        self.config = config
-        self.need_restart = False
-
     def install(self):
         # Run local DNS resolver `unbound`.
         # `resolvconf` takes care of setting up /etc/resolv.conf
@@ -181,27 +176,6 @@ class UnboundDeployer(Deployer):
                 "unbound-anchor -a /var/lib/unbound/root.key || true",
             ],
         )
-        if self.config.disable_ipv6:
-            files.directory(
-                path="/etc/unbound/unbound.conf.d",
-                present=True,
-                user="root",
-                group="root",
-                mode="755",
-            )
-            conf = files.put(
-                src=get_resource("unbound/unbound.conf.j2"),
-                dest="/etc/unbound/unbound.conf.d/chatmail.conf",
-                user="root",
-                group="root",
-                mode="644",
-            )
-        else:
-            conf = files.file(
-                path="/etc/unbound/unbound.conf.d/chatmail.conf",
-                present=False,
-            )
-        self.need_restart |= conf.changed
 
     def activate(self):
         server.shell(
@@ -216,7 +190,6 @@ class UnboundDeployer(Deployer):
             service="unbound.service",
             running=True,
             enabled=True,
-            restarted=self.need_restart,
         )
 
 
@@ -264,9 +237,6 @@ class WebsiteDeployer(Deployer):
             # if www_folder is a hugo page, build it
             if build_dir:
                 www_path = build_webpages(src_dir, build_dir, self.config)
-                if www_path is None:
-                    logger.warning("Web page build failed, skipping website deployment")
-                    return
             # if it is not a hugo page, upload it as is
             files.rsync(
                 f"{www_path}/", "/var/www/html", flags=["-avz", "--chown=www-data"]
@@ -446,6 +416,8 @@ class ChatmailVenvDeployer(Deployer):
     def __init__(self, config):
         self.config = config
         self.units = (
+            "filtermail",
+            "filtermail-incoming",
             "chatmail-metadata",
             "lastlogin",
             "chatmail-expire",
@@ -530,34 +502,28 @@ class GithashDeployer(Deployer):
         except Exception:
             git_diff = ""
         files.put(
-            name="Upload chatmail relay git commit hash",
+            name="Upload chatmail relay git commiit hash",
             src=StringIO(git_hash + git_diff),
             dest="/etc/chatmail-version",
             mode="700",
         )
 
 
-def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -> None:
+def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
     """Deploy a chat-mail instance.
 
     :param config_path: path to chatmail.ini
     :param disable_mail: whether to disable postfix & dovecot
-    :param website_only: if True, only deploy the website
     """
     config = read_config(config_path)
     check_config(config)
     mail_domain = config.mail_domain
 
-    if website_only:
-        Deployment().perform_stages([WebsiteDeployer(config)])
-        return
-
     if host.get_fact(Port, port=53) != "unbound":
         files.line(
             name="Add 9.9.9.9 to resolv.conf",
             path="/etc/resolv.conf",
-            # Guard against resolv.conf missing a trailing newline (SolusVM bug).
+            line="nameserver 9.9.9.9",
-            line="\nnameserver 9.9.9.9",
         )
 
     port_services = [
@@ -570,8 +536,6 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
         (["master", "smtpd"], 587),
         (["imap-login", "dovecot"], 993),
         ("iroh-relay", 3340),
-        ("mtail", 3903),
-        ("dovecot-stats", 3904),
         ("nginx", 8443),
         (["master", "smtpd"], config.postfix_reinject_port),
         (["master", "smtpd"], config.postfix_reinject_port_incoming),
@@ -593,9 +557,8 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
     all_deployers = [
         ChatmailDeployer(mail_domain),
         LegacyRemoveDeployer(),
-        FiltermailDeployer(),
         JournaldDeployer(),
-        UnboundDeployer(config),
+        UnboundDeployer(),
         TurnDeployer(mail_domain),
         IrohDeployer(config.enable_iroh_relay),
         AcmetoolDeployer(config.acme_email, tls_domains),

cmdeploy/src/cmdeploy/dovecot/auth.conf

@@ -4,7 +4,7 @@ iterate_prefix = userdb/
 
 default_pass_scheme = plain
 # %E escapes characters " (double quote), ' (single quote) and \ (backslash) with \ (backslash).
-# See <https://doc.dovecot.org/2.3/configuration_manual/config_file/config_variables/#modifiers>
+# See <https://doc.dovecot.org/configuration_manual/config_file/config_variables/#modifiers>
 # for documentation.
 #
 # We escape user-provided input and use double quote as a separator.

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -37,7 +37,7 @@ class DovecotDeployer(Deployer):
         restart = False if self.disable_mail else self.need_restart
 
         systemd.service(
-            name="Disable dovecot for now"
+            name="disable dovecot for now"
             if self.disable_mail
             else "Start and enable Dovecot",
             service="dovecot.service",
@@ -116,7 +116,7 @@ def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
     )
     need_restart |= lua_push_notification_script.changed
 
-    # as per https://doc.dovecot.org/2.3/configuration_manual/os/
+    # as per https://doc.dovecot.org/configuration_manual/os/
     # it is recommended to set the following inotify limits
     for name in ("max_user_instances", "max_user_watches"):
         key = f"fs.inotify.{name}"
@@ -145,11 +145,4 @@ def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
     )
     daemon_reload |= restart_conf.changed
 
-    # Validate dovecot configuration before restart
-    if need_restart:
-        server.shell(
-            name="Validate dovecot configuration",
-            commands=["doveconf -n >/dev/null"],
-        )
-
     return need_restart, daemon_reload

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -1,7 +1,7 @@
 ## Dovecot configuration file
 
 {% if disable_ipv6 %}
-listen = 0.0.0.0
+listen = *
 {% endif %}
 
 protocols = imap lmtp
@@ -26,7 +26,7 @@ default_client_limit = 20000
 # Increase number of logged in IMAP connections.
 # Each connection is handled by a separate `imap` process.
 # `imap` process should have `client_limit=1` as described in
-# <https://doc.dovecot.org/2.3/configuration_manual/service_configuration/#service-limits>
+# <https://doc.dovecot.org/configuration_manual/service_configuration/#service-limits>
 # so each logged in IMAP session will need its own `imap` process.
 #
 # If this limit is reached,
@@ -44,11 +44,11 @@ mail_server_comment = Chatmail server
 
 # `zlib` enables compressing messages stored in the maildir.
 # See
-# <https://doc.dovecot.org/2.3/configuration_manual/zlib_plugin/>
+# <https://doc.dovecot.org/configuration_manual/zlib_plugin/>
 # for documentation.
 #
 # quota plugin documentation:
-# <https://doc.dovecot.org/2.3/configuration_manual/quota_plugin/>
+# <https://doc.dovecot.org/configuration_manual/quota_plugin/>
 mail_plugins = zlib quota
 
 imap_capability = +XDELTAPUSH XCHATMAIL
@@ -125,13 +125,13 @@ plugin {
 
 protocol lmtp {
   # notify plugin is a dependency of push_notification plugin:
-  # <https://doc.dovecot.org/2.3/settings/plugin/notify-plugin/>
+  # <https://doc.dovecot.org/settings/plugin/notify-plugin/>
   #
   # push_notification plugin documentation:
-  # <https://doc.dovecot.org/2.3/configuration_manual/push_notification/>
+  # <https://doc.dovecot.org/configuration_manual/push_notification/>
   #
   # mail_lua and push_notification_lua are needed for Lua push notification handler.
-  # <https://doc.dovecot.org/2.3/configuration_manual/push_notification/#configuration>
+  # <https://doc.dovecot.org/configuration_manual/push_notification/#configuration>
   mail_plugins = $mail_plugins mail_lua notify push_notification push_notification_lua
 }
 
@@ -154,7 +154,7 @@ plugin {
 
 # push_notification configuration
 plugin {
-  # <https://doc.dovecot.org/2.3/configuration_manual/push_notification/#lua-lua>
+  # <https://doc.dovecot.org/configuration_manual/push_notification/#lua-lua>
   push_notification_driver = lua:file=/etc/dovecot/push_notification.lua
 }
 
@@ -168,8 +168,6 @@ service lmtp {
   }
 }
 
-lmtp_add_received_header = no
-
 service auth {
   unix_listener /var/spool/postfix/private/auth {
     mode = 0660
@@ -279,156 +277,3 @@ service imap-hibernate {
   }
 }
 {% endif %}
-
-{% if config.mtail_address %}
-#
-# Dovecot Statistics
-#
-# OpenMetrics endpoint at http://{{- config.mtail_address}}:3904/metrics
-service stats {
-  inet_listener http {
-    port = 3904
-    address = {{- config.mtail_address}}
-  }
-}
-
-# IMAP Command Metrics
-# - Bytes in/out for compression efficiency analysis
-# - Lock wait time for contention debugging
-# - Grouped by command name and reply state
-metric imap_command {
-  filter = event=imap_command_finished
-  fields = bytes_in bytes_out lock_wait_usecs running_usecs
-  group_by = cmd_name tagged_reply_state
-}
-
-# Duration buckets for latency histograms (base 10: 10us, 100us, 1ms, 10ms, 100ms, 1s, 10s, 100s)
-metric imap_command_duration {
-  filter = event=imap_command_finished
-  group_by = cmd_name duration:exponential:1:8:10
-}
-
-# Slow command outliers (>1 second = 1000000 usecs)
-# Useful for alerting without high cardinality
-metric imap_command_slow {
-  filter = event=imap_command_finished AND duration>1000000 AND NOT cmd_name=IDLE
-  group_by = cmd_name
-}
-
-# IDLE-specific Metrics
-
-metric imap_idle {
-  filter = event=imap_command_finished AND cmd_name=IDLE
-  fields = bytes_in bytes_out running_usecs
-  group_by = tagged_reply_state
-}
-
-metric imap_idle_duration {
-  filter = event=imap_command_finished AND cmd_name=IDLE
-  # Base 10: 100ms to 27h (covers short wakeups to long idle sessions)
-  group_by = duration:exponential:5:11:10
-}
-
-metric imap_idle_commands {
-  filter = event=imap_command_finished AND cmd_name=IDLE
-  group_by = tagged_reply_state
-}
-
-metric imap_idle_failed {
-  filter = event=imap_command_finished AND cmd_name=IDLE AND NOT tagged_reply_state=OK
-}
-
-# Hibernation Metrics (requires imap_hibernate_timeout)
-
-metric imap_hibernated {
-  filter = event=imap_client_hibernated
-}
-
-metric imap_hibernated_failed {
-  filter = event=imap_client_hibernated AND error=*
-}
-
-metric imap_unhibernated {
-  filter = event=imap_client_unhibernated
-  fields = hibernation_usecs
-}
-
-metric imap_unhibernated_reason {
-  filter = event=imap_client_unhibernated
-  group_by = reason
-  fields = hibernation_usecs
-}
-
-metric imap_unhibernated_reason_sleep {
-  filter = event=imap_client_unhibernated
-  group_by = reason hibernation_usecs:exponential:4:8:10
-}
-
-metric imap_unhibernated_failed {
-  filter = event=imap_client_unhibernated AND error=*
-}
-
-# Hibernation duration buckets (how long clients stayed hibernated)
-# Base 10: 100ms to 27h
-metric imap_hibernation_duration {
-  filter = event=imap_client_unhibernated
-  group_by = reason duration:exponential:5:11:10
-}
-
-# Authentication / Login Metrics
-
-metric auth_request {
-  filter = event=auth_request_finished
-  group_by = success
-}
-
-metric auth_request_duration {
-  filter = event=auth_request_finished
-  group_by = success duration:exponential:2:6:10
-}
-
-metric auth_failed {
-  filter = event=auth_request_finished AND success=no
-}
-
-# Passdb cache effectiveness
-metric auth_passdb {
-  filter = event=auth_passdb_request_finished
-  group_by = result cache
-}
-
-# Master login (post-auth userdb lookup)
-metric auth_master_login {
-  filter = event=auth_master_client_login_finished
-}
-
-metric auth_master_login_failed {
-  filter = event=auth_master_client_login_finished AND error=*
-}
-
-# Mail Delivery (LMTP) - affects IDLE wakeup latency
-
-metric mail_delivery {
-  filter = event=mail_delivery_finished
-}
-
-metric mail_delivery_duration {
-  filter = event=mail_delivery_finished
-  group_by = duration:exponential:3:7:10
-}
-
-metric mail_delivery_failed {
-  filter = event=mail_delivery_finished AND error=*
-}
-
-# Connection Events
-
-metric client_connected {
-  filter = event=client_connection_connected AND category="service:imap"
-}
-
-metric client_disconnected {
-  filter = event=client_connection_disconnected AND category="service:imap"
-  fields = bytes_in bytes_out
-}
-{% endif %}

cmdeploy/src/cmdeploy/filtermail/deployer.py

@@ -1,52 +0,0 @@
-from pyinfra import facts, host
-from pyinfra.operations import files, systemd
-
-from cmdeploy.basedeploy import Deployer, get_resource
-
-
-class FiltermailDeployer(Deployer):
-    services = ["filtermail", "filtermail-incoming"]
-    bin_path = "/usr/local/bin/filtermail"
-    config_path = "/usr/local/lib/chatmaild/chatmail.ini"
-
-    def __init__(self):
-        self.need_restart = False
-
-    def install(self):
-        arch = host.get_fact(facts.server.Arch)
-        url = f"https://github.com/chatmail/filtermail/releases/download/v0.2.0/filtermail-{arch}-musl"
-        sha256sum = {
-            "x86_64": "1e5bbb646582cb16740c6dfbbca39edba492b78cc96ec9fa2528c612bb504edd",
-            "aarch64": "3564fba8605f8f9adfeefff3f4580533205da043f47c5968d0d10db17e50f44e",
-        }[arch]
-        self.need_restart |= files.download(
-            name="Download filtermail",
-            src=url,
-            sha256sum=sha256sum,
-            dest=self.bin_path,
-            mode="755",
-        ).changed
-
-    def configure(self):
-        for service in self.services:
-            self.need_restart |= files.template(
-                src=get_resource(f"filtermail/{service}.service.j2"),
-                dest=f"/etc/systemd/system/{service}.service",
-                user="root",
-                group="root",
-                mode="644",
-                bin_path=self.bin_path,
-                config_path=self.config_path,
-            ).changed
-
-    def activate(self):
-        for service in self.services:
-            systemd.service(
-                name=f"Start and enable {service}",
-                service=f"{service}.service",
-                running=True,
-                enabled=True,
-                restarted=self.need_restart,
-                daemon_reload=True,
-            )
-        self.need_restart = False

cmdeploy/src/cmdeploy/mtail/delivered_mail.mtail

@@ -44,37 +44,21 @@ counter warning_count
 }
 
 
-counter filtered_outgoing_mail_count
+counter filtered_mail_count
 
-counter outgoing_encrypted_mail_count
+counter encrypted_mail_count
-/Outgoing: Filtering encrypted mail\./ {
+/Filtering encrypted mail\./ {
-  outgoing_encrypted_mail_count++
+  encrypted_mail_count++
-  filtered_outgoing_mail_count++
+  filtered_mail_count++
 }
 
-counter outgoing_unencrypted_mail_count
+counter unencrypted_mail_count
-/Outgoing: Filtering unencrypted mail\./ {
+/Filtering unencrypted mail\./ {
-  outgoing_unencrypted_mail_count++
+  unencrypted_mail_count++
-  filtered_outgoing_mail_count++
+  filtered_mail_count++
 }
 
-
-counter filtered_incoming_mail_count
-
-counter incoming_encrypted_mail_count
-/Incoming: Filtering encrypted mail\./ {
-  incoming_encrypted_mail_count++
-  filtered_incoming_mail_count++
-}
-
-counter incoming_unencrypted_mail_count
-/Incoming: Filtering unencrypted mail\./ {
-  incoming_unencrypted_mail_count++
-  filtered_incoming_mail_count++
-}
-
-
 counter rejected_unencrypted_mail_count
-/Rejected unencrypted mail/ {
+/Rejected unencrypted mail\./ {
   rejected_unencrypted_mail_count++
 }

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -29,6 +29,7 @@ stream {
             default 127.0.0.1:8443;
             ~\bsmtp\b 127.0.0.1:465;
             ~\bimap\b 127.0.0.1:993;
+            ~\bssh\b 127.0.0.1:22;
         }
 
         server {
@@ -51,7 +52,7 @@ http {
 	include /etc/nginx/mime.types;
 	default_type application/octet-stream;
 
-	ssl_protocols TLSv1.2 TLSv1.3;
+	ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
 	ssl_prefer_server_ciphers on;
 	ssl_certificate /var/lib/acme/live/{{ config.domain_name }}/fullchain;
 	ssl_certificate_key /var/lib/acme/live/{{ config.domain_name }}/privkey;

cmdeploy/src/cmdeploy/postfix/deployer.py

@@ -1,4 +1,4 @@
-from pyinfra.operations import apt, files, server, systemd
+from pyinfra.operations import apt, files, systemd
 
 from cmdeploy.basedeploy import Deployer, get_resource
 
@@ -52,15 +52,6 @@ class PostfixDeployer(Deployer):
         )
         need_restart |= header_cleanup.changed
 
-        lmtp_header_cleanup = files.put(
-            src=get_resource("postfix/lmtp_header_cleanup"),
-            dest="/etc/postfix/lmtp_header_cleanup",
-            user="root",
-            group="root",
-            mode="644",
-        )
-        need_restart |= lmtp_header_cleanup.changed
-
         # Login map that 1:1 maps email address to login.
         login_map = files.put(
             src=get_resource("postfix/login_map"),
@@ -74,19 +65,9 @@ class PostfixDeployer(Deployer):
         restart_conf = files.put(
             name="postfix: restart automatically on failure",
             src=get_resource("service/10_restart.conf"),
-            dest="/etc/systemd/system/postfix@.service.d/10_restart.conf",
+            dest="/etc/systemd/system/dovecot.service.d/10_restart.conf",
         )
         self.daemon_reload = restart_conf.changed
-
-        # Validate postfix configuration before restart
-        if need_restart:
-            server.shell(
-                name="Validate postfix configuration",
-                # Extract stderr and quit with error if non-zero
-                commands=[
-                    """bash -c 'w=$(postconf 2>&1 >/dev/null); [[ -z "$w" ]] || { echo "$w"; false; }'"""
-                ],
-            )
         self.need_restart = need_restart
 
     def activate(self):

cmdeploy/src/cmdeploy/postfix/lmtp_header_cleanup

@@ -1,2 +0,0 @@
-/^DKIM-Signature:/            IGNORE
-/^Authentication-Results:/            IGNORE

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -64,11 +64,7 @@ alias_database = hash:/etc/aliases
 mydestination =
 
 relayhost = 
-{% if disable_ipv6 %}
-mynetworks = 127.0.0.0/8
-{% else %}
 mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128
-{% endif %}
 mailbox_size_limit = 0
 message_size_limit = {{config.max_message_size}}
 recipient_delimiter = +
@@ -81,7 +77,6 @@ inet_protocols = all
 
 virtual_transport = lmtp:unix:private/dovecot-lmtp
 virtual_mailbox_domains = {{ config.mail_domain }}
-lmtp_header_checks = regexp:/etc/postfix/lmtp_header_cleanup
 
 mua_client_restrictions = permit_sasl_authenticated, reject
 mua_sender_restrictions = reject_sender_login_mismatch, permit_sasl_authenticated, reject

cmdeploy/src/cmdeploy/remote/rdns.py

@@ -53,7 +53,7 @@ def get_dkim_entry(mail_domain, pre_command, dkim_selector):
             print=log_progress,
         )
     except CalledProcessError:
-        return None, None
+        return
     dkim_value_raw = f"v=DKIM1;k=rsa;p={dkim_pubkey};s=email;t=s"
     dkim_value = '" "'.join(re.findall(".{1,255}", dkim_value_raw))
     web_dkim_value = "".join(re.findall(".{1,255}", dkim_value_raw))

cmdeploy/src/cmdeploy/remote/rshell.py

@@ -40,5 +40,5 @@ def dovecot_recalc_quota(user):
     #
     for line in output.split("\n"):
         parts = line.split()
-        if len(parts) >= 6 and parts[2] == "STORAGE":
+        if parts[2] == "STORAGE":
             return dict(value=int(parts[3]), limit=int(parts[4]), percent=int(parts[5]))

cmdeploy/src/cmdeploy/run.py

@@ -14,9 +14,8 @@ def main():
         importlib.resources.files("cmdeploy").joinpath("../../../chatmail.ini"),
     )
     disable_mail = bool(os.environ.get("CHATMAIL_DISABLE_MAIL"))
-    website_only = bool(os.environ.get("CHATMAIL_WEBSITE_ONLY"))
 
-    deploy_chatmail(config_path, disable_mail, website_only)
+    deploy_chatmail(config_path, disable_mail)
 
 
 if pyinfra.is_cli:

cmdeploy/src/cmdeploy/service/filtermail-incoming.service.f

@@ -2,10 +2,11 @@
 Description=Incoming Chatmail Postfix before queue filter 
 
 [Service]
-ExecStart={{ bin_path }} {{ config_path }} incoming
+ExecStart={execpath} {config_path} incoming
 Restart=always
 RestartSec=30
 User=vmail
 
 [Install]
 WantedBy=multi-user.target
+

cmdeploy/src/cmdeploy/service/filtermail.service.f

@@ -2,7 +2,7 @@
 Description=Outgoing Chatmail Postfix before queue filter
 
 [Service]
-ExecStart={{ bin_path }} {{ config_path }} outgoing
+ExecStart={execpath} {config_path} outgoing
 Restart=always
 RestartSec=30
 User=vmail

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -189,14 +189,12 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
     mail = maildata(
         "encrypted.eml", from_addr=user1.addr, to_addr=user2.addr
     ).as_string()
-
+    for i in range(chatmail_config.max_user_send_per_minute + 5):
-    start = time.time()
+        print("Sending mail", str(i))
-    for i in range(chatmail_config.max_user_send_per_minute * 3):
-        print("Sending mail", str(i + 1), "at", time.time() - start, "s.")
         try:
             user1.smtp.sendmail(user1.addr, [user2.addr], mail)
         except smtplib.SMTPException as e:
-            if i < chatmail_config.max_user_send_burst_size:
+            if i < chatmail_config.max_user_send_per_minute:
                 pytest.fail(f"rate limit was exceeded too early with msg {i}")
             outcome = e.recipients[user2.addr]
             assert outcome[0] == 450

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -17,7 +17,6 @@ def imap_mailbox(cmfactory):
     password = ac1.get_config("mail_pw")
     mailbox = imap_tools.MailBox(user.split("@")[1])
     mailbox.login(user, password)
-    mailbox.dc_ac = ac1
     return mailbox
 
 
@@ -122,28 +121,6 @@ class TestEndToEndDeltaChat:
         assert ch.id >= 10
         ac1._evtracker.wait_securejoin_inviter_progress(1000)
 
-    def test_dkim_header_stripped(self, cmfactory, maildomain2, lp, imap_mailbox):
-        """Test that if a DC address receives a message, it has no
-        DKIM-Signature and Authentication-Results headers."""
-        ac1 = cmfactory.new_online_configuring_account(cache=False)
-        cmfactory.switch_maildomain(maildomain2)
-        ac2 = cmfactory.new_online_configuring_account(cache=False)
-        cmfactory.bring_accounts_online()
-        chat = cmfactory.get_accepted_chat(ac1, imap_mailbox.dc_ac)
-        chat.send_text("message0")
-        chat2 = cmfactory.get_accepted_chat(ac2, imap_mailbox.dc_ac)
-        chat2.send_text("message1")
-
-        lp.sec("receive message with ac1...")
-        received = 0
-        while received < 2:
-            msgs = imap_mailbox.fetch()
-            for msg in msgs:
-                lp.sec(f"ac1 received msg from {msg.from_}")
-                received += 1
-                assert "authentication-results" not in msg.headers
-                assert "dkim-signature" not in msg.headers
-
     def test_read_receipts_between_instances(self, cmfactory, lp, maildomain2):
         ac1 = cmfactory.new_online_configuring_account(cache=False)
         cmfactory.switch_maildomain(maildomain2)

cmdeploy/src/cmdeploy/tests/test_dns.py

@@ -60,29 +60,6 @@ def mockdns(request, mockdns_base, mockdns_expected):
     return mockdns_base
 
 
-class TestGetDkimEntry:
-    def test_dkim_entry_returns_tuple_on_success(self, mockdns):
-        entry, web_entry = remote.rdns.get_dkim_entry(
-            "some.domain", "", dkim_selector="opendkim"
-        )
-        # May return None,None if openssl not available, but should never crash
-        if entry is not None:
-            assert "opendkim._domainkey.some.domain" in entry
-            assert "opendkim._domainkey.some.domain" in web_entry
-
-    def test_dkim_entry_returns_none_tuple_on_error(self, monkeypatch):
-        """CalledProcessError must return (None, None), not bare None."""
-        from subprocess import CalledProcessError
-
-        def failing_shell(command, fail_ok=False, print=print):
-            raise CalledProcessError(1, command)
-
-        monkeypatch.setattr(remote.rdns, "shell", failing_shell)
-        result = remote.rdns.get_dkim_entry("some.domain", "", dkim_selector="opendkim")
-        assert result == (None, None)
-        assert result[0] is None and result[1] is None
-
-
 class TestPerformInitialChecks:
     def test_perform_initial_checks_ok1(self, mockdns, mockdns_expected):
         remote_data = remote.rdns.perform_initial_checks("some.domain")

cmdeploy/src/cmdeploy/tests/test_rshell.py

@@ -1,68 +0,0 @@
-from unittest.mock import patch
-
-from cmdeploy.remote.rshell import dovecot_recalc_quota
-
-
-def test_dovecot_recalc_quota_normal_output():
-    """Normal doveadm output returns parsed dict."""
-    normal_output = (
-        "Quota name Type    Value  Limit  %\n"
-        "User quota STORAGE     5 102400  0\n"
-        "User quota MESSAGE     2      -  0\n"
-    )
-
-    with patch("cmdeploy.remote.rshell.shell", return_value=normal_output):
-        result = dovecot_recalc_quota("user@example.org")
-
-    # shell is called twice (recalc + get), patch returns same for both
-    assert result == {"value": 5, "limit": 102400, "percent": 0}
-
-
-def test_dovecot_recalc_quota_empty_output():
-    """Empty doveadm output (trailing newline) must not IndexError."""
-    call_count = [0]
-
-    def mock_shell(cmd):
-        call_count[0] += 1
-        if "recalc" in cmd:
-            return ""
-        # quota get returns only empty lines
-        return "\n\n"
-
-    with patch("cmdeploy.remote.rshell.shell", side_effect=mock_shell):
-        result = dovecot_recalc_quota("user@example.org")
-
-    assert result is None
-
-
-def test_dovecot_recalc_quota_malformed_output():
-    """Malformed output with too few columns must not crash."""
-    call_count = [0]
-
-    def mock_shell(cmd):
-        call_count[0] += 1
-        if "recalc" in cmd:
-            return ""
-        # partial line, fewer than 6 parts
-        return "Quota name\nUser quota STORAGE\n"
-
-    with patch("cmdeploy.remote.rshell.shell", side_effect=mock_shell):
-        result = dovecot_recalc_quota("user@example.org")
-
-    assert result is None
-
-
-def test_dovecot_recalc_quota_header_only():
-    """Only header line, no data rows."""
-    call_count = [0]
-
-    def mock_shell(cmd):
-        call_count[0] += 1
-        if "recalc" in cmd:
-            return ""
-        return "Quota name Type    Value  Limit  %\n"
-
-    with patch("cmdeploy.remote.rshell.shell", side_effect=mock_shell):
-        result = dovecot_recalc_quota("user@example.org")
-
-    assert result is None

cmdeploy/src/cmdeploy/unbound/unbound.conf.j2

@@ -1,4 +0,0 @@
-# Managed by cmdeploy: disable IPv6 in unbound.
-server:
-  interface: 127.0.0.1
-  do-ip6: no

doc/source/getting_started.rst

@@ -16,16 +16,15 @@ You will need the following:
 
 -  Control over a domain through a DNS provider of your choice.
 
--  A Debian 12 **deployment server** with reachable SMTP/SUBMISSIONS/IMAPS/HTTPS ports.
+-  A Debian 12 server with reachable SMTP/SUBMISSIONS/IMAPS/HTTPS ports.
    IPv6 is encouraged if available. Chatmail relay servers only require
    1GB RAM, one CPU, and perhaps 10GB storage for a few thousand active
    chatmail addresses.
 
--  A Linux or Unix **build machine** with key-based SSH access to the root
+-  Key-based SSH authentication to the root user. You must add a
-   user of the deployment server.
+   passphrase-protected private key to your local ssh-agent because you
-   You must add a passphrase-protected private key to your local ssh-agent because you
+   can’t type in your passphrase during deployment. (An ed25519 private
-   can’t type in your passphrase during deployment.
+   key is required due to an `upstream bug in
-   (An ed25519 private key is required due to an `upstream bug in
    paramiko <https://github.com/paramiko/paramiko/issues/2191>`_)
 
 
@@ -35,8 +34,7 @@ Setup with ``scripts/cmdeploy``
 We use ``chat.example.org`` as the chatmail domain in the following
 steps. Please substitute it with your own domain.
 
-1. Setup the initial DNS records for your deployment server.
+1. Setup the initial DNS records. The following is an example in the
-   The following is an example in the
    familiar BIND zone file format with a TTL of 1 hour (3600 seconds).
    Please substitute your domain and IP addresses.
 
@@ -56,20 +54,20 @@ steps. Please substitute it with your own domain.
        cd relay
        scripts/initenv.sh
 
-3. On your local build machine (PC), create a chatmail configuration file
+3. On your local PC, create chatmail configuration file
    ``chatmail.ini``:
 
    ::
 
        scripts/cmdeploy init chat.example.org  # <-- use your domain
 
-4. Verify that SSH root login to the deployment server server works:
+4. Verify that SSH root login to your remote server works:
 
    ::
 
        ssh root@chat.example.org  # <-- use your domain
 
-5. From your local build machine, setup and configure the remote deployment server:
+5. From your local PC, deploy the remote chatmail relay server:
 
    ::
 
@@ -83,7 +81,7 @@ steps. Please substitute it with your own domain.
 Other helpful commands
 ----------------------
 
-To check the status of your deployment server running the chatmail service:
+To check the status of your remotely running chatmail service:
 
 ::
 
@@ -160,7 +158,7 @@ Disable automatic address creation
 --------------------------------------------------------
 
 If you need to stop address creation, e.g. because some script is wildly
-creating addresses, login with ssh to the deployment machine and run:
+creating addresses, login with ssh and run:
 
 ::
 
@@ -169,23 +167,3 @@ creating addresses, login with ssh to the deployment machine and run:
 Chatmail address creation will be denied while this file is present.
 
 
-Migrating to a new build machine
-----------------------------------
-
-To move or add a build machine,
-clone the relay repository on the new build machine, and copy the ``chatmail.ini`` file from the old build machine.
-Make sure ``rsync`` is installed, then initialize the environment:
-
-::
-
-   ./scripts/initenv.sh
-
-Run safety checks before a new deployment:
-
-::
-
-   ./scripts/cmdeploy dns
-   ./scripts/cmdeploy status
-
-If you keep multiple build machines (ie laptop and desktop), keep ``chatmail.ini`` in sync between
-them.

doc/source/overview.rst

@@ -42,11 +42,6 @@ The deployed system components of a chatmail relay are:
 -  Dovecot_ is the Mail Delivery Agent (MDA) and
    stores messages for users until they download them
 
--  `filtermail <https://github.com/chatmail/filtermail>`_
-   prevents unencrypted email from leaving or entering the chatmail
-   service and is integrated into Postfix’s outbound and inbound mail
-   pipelines.
-
 -  Nginx_ shows the web page with privacy policy and additional information
 
 -  `acmetool <https://hlandau.github.io/acmetool/>`_ manages TLS
@@ -90,6 +85,11 @@ short overview of ``chatmaild`` services:
    <https://doc.dovecot.org/2.3/configuration_manual/authentication/dict/#complete-example-for-authenticating-via-a-unix-socket>`_
    to authenticate logins.
 
+-  `filtermail <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/filtermail.py>`_
+   prevents unencrypted email from leaving or entering the chatmail
+   service and is integrated into Postfix’s outbound and inbound mail
+   pipelines.
+
 -  `chatmail-metadata <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/metadata.py>`_
    is contacted by a `Dovecot lua
    script <https://github.com/chatmail/relay/blob/main/cmdeploy/src/cmdeploy/dovecot/push_notification.lua>`_

doc/source/related.rst

@@ -14,10 +14,10 @@ We know of three work-in-progress alternative implementation efforts:
    it to support all of the features and configuration settings required
    to operate as a chatmail relay.
 
--  `Madmail <https://github.com/omidz4t/madmail>`_: an
+-  `Maddy-Chatmail <https://github.com/sadraiiali/maddy_chatmail>`_: a
-   experimental fork of Maddy Mail Server <https://maddy.email/>`_ optimized
+   plugin for the `Maddy email server <https://maddy.email/>`_ which
-   for chatmail deployments.  It provides a single binary solution
+   aims to implement the chatmail relay features and configuration
-   for running a chatmail relay.
+   options.
 
 -  `Chatmail Cookbook <https://github.com/feld/chatmail-cookbook>`_:
    A Chef Cookbook implementing a relay server. The project follows the