postfix: do lmtp via local instead of virtual transport

Signed-off-by: Jagoda Ślązak <jslazak@jslazak.com>

.github/workflows/ci.yaml

@@ -14,7 +14,8 @@ jobs:
         # Otherwise `test_deployed_state` will be unhappy.
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-
+      - name: download filtermail
+        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.1.2/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
       - name: run chatmaild tests 
         working-directory: chatmaild
         run: pipx run tox

.github/workflows/test-and-deploy-ipv4only.yaml

@@ -74,6 +74,7 @@ jobs:
       - run: |
           cmdeploy init staging-ipv4.testrun.org
           sed -i 's#disable_ipv6 = False#disable_ipv6 = True#' chatmail.ini
+          sed -i 's/#\s*mtail_address/mtail_address/' chatmail.ini
 
       - run: cmdeploy run --verbose --skip-dns-check
 

.github/workflows/test-and-deploy.yaml

@@ -74,7 +74,9 @@ jobs:
       - name: run deploy-chatmail offline tests 
         run: pytest --pyargs cmdeploy 
 
-      - run: cmdeploy init staging2.testrun.org
+      - run: |
+          cmdeploy init staging2.testrun.org
+          sed -i 's/#\s*mtail_address/mtail_address/' chatmail.ini
 
       - run: cmdeploy run --verbose --skip-dns-check
 

README.md

@@ -1,20 +1,25 @@
 
-# Chatmail relays for end-to-end encrypted email
+# No-DNS Chatmail relay
 
-Chatmail relay servers are interoperable Mail Transport Agents (MTAs) designed for: 
+With this branch, you don't need DNS at all,
+just a VPS with an IPv4 address,
+let's take `77.42.80.106` as an example.
+First, choose a random domain name (it doesn't need working DNS)
+and create a chatmail.ini config file:
 
--  **Zero State:** no private data or metadata collected, messages are auto-deleted, low disk usage
+```
+cmdeploy init [77.42.80.106]
+```
 
--  **Instant/Realtime:** sub-second message delivery, realtime P2P
-   streaming, privacy-preserving Push Notifications for Apple, Google, and Huawei;
+Then, in `cmdeploy/src/cmdeploy/postfix/transport`,
+remove the line corresponding to your relay,
+and add other for relays you know.
+Now you can deploy the relay to your IP address:
 
--  **Security Enforcement**: only strict TLS, DKIM and OpenPGP with minimized metadata accepted
+```
+cmdeploy run --skip-dns-check --ssh-host 77.42.80.106
+```
 
--  **Reliable Federation and Decentralization:** No spam or IP reputation checks, federating
-   depends on established IETF standards and protocols.
-
-This repository contains everything needed to setup a ready-to-use chatmail relay on an ssh-reachable host. 
-For getting started and more information please refer to the web version of this repositories' documentation at
-
-[https://chatmail.at/doc/relay](https://chatmail.at/doc/relay)
+Finally, you can login with a `dclogin://` code like this, with the correct "domain name" and IP address:
 
+`dclogin:s0mer4nd0@[77.42.80.106]?p=w7i8da7h8uads92ycc2rufyl&v=1&ih=77.42.80.106&sh=77.42.80.106&sp=443&ip=443&ic=3&sc=3`

chatmaild/pyproject.toml

@@ -24,7 +24,6 @@ where = ['src']
 [project.scripts]
 doveauth = "chatmaild.doveauth:main"
 chatmail-metadata = "chatmaild.metadata:main"
-filtermail = "chatmaild.filtermail:main"
 chatmail-metrics = "chatmaild.metrics:main"
 chatmail-expire = "chatmaild.expire:main"
 chatmail-fsreport = "chatmaild.fsreport:main"

chatmaild/src/chatmaild/config.py

@@ -32,13 +32,13 @@ class Config:
         self.passthrough_senders = params["passthrough_senders"].split()
         self.passthrough_recipients = params["passthrough_recipients"].split()
         self.www_folder = params.get("www_folder", "")
-        self.filtermail_smtp_port = int(params["filtermail_smtp_port"])
+        self.filtermail_smtp_port = int(params.get("filtermail_smtp_port", "10080"))
         self.filtermail_smtp_port_incoming = int(
-            params["filtermail_smtp_port_incoming"]
+            params.get("filtermail_smtp_port_incoming", "10081")
         )
-        self.postfix_reinject_port = int(params["postfix_reinject_port"])
+        self.postfix_reinject_port = int(params.get("postfix_reinject_port", "10025"))
         self.postfix_reinject_port_incoming = int(
-            params["postfix_reinject_port_incoming"]
+            params.get("postfix_reinject_port_incoming", "10026")
         )
         self.mtail_address = params.get("mtail_address")
         self.disable_ipv6 = params.get("disable_ipv6", "false").lower() == "true"

chatmaild/src/chatmaild/dictproxy.py

@@ -22,7 +22,7 @@ class DictProxy:
                 wfile.flush()
 
     def handle_dovecot_request(self, msg, transactions):
-        # see https://doc.dovecot.org/developer_manual/design/dict_protocol/#dovecot-dict-protocol
+        # see https://doc.dovecot.org/2.3/developer_manual/design/dict_protocol/#dovecot-dict-protocol
         short_command = msg[0]
         parts = msg[1:].split("\t")
 

chatmaild/src/chatmaild/doveauth.py

@@ -16,7 +16,7 @@ NOCREATE_FILE = "/etc/chatmail-nocreate"
 
 
 def encrypt_password(password: str):
-    # https://doc.dovecot.org/configuration_manual/authentication/password_schemes/
+    # https://doc.dovecot.org/2.3/configuration_manual/authentication/password_schemes/
     passhash = crypt_r.crypt(password, crypt_r.METHOD_SHA512)
     return "{SHA512-CRYPT}" + passhash
 

chatmaild/src/chatmaild/expire.py

@@ -144,7 +144,7 @@ class Expiry:
                 continue
             changed = True
         if changed:
-            self.remove_file("maildirsize")
+            self.remove_file(f"{mbox.basedir}/maildirsize")
 
     def get_summary(self):
         return (

chatmaild/src/chatmaild/filtermail.py

@@ -1,378 +0,0 @@
-#!/usr/bin/env python3
-import asyncio
-import base64
-import binascii
-import sys
-import time
-from email import policy
-from email.parser import BytesParser
-from email.utils import parseaddr
-from smtplib import SMTP as SMTPClient
-
-from aiosmtpd.controller import Controller
-from aiosmtpd.smtp import SMTP
-
-from .config import read_config
-
-ENCRYPTION_NEEDED_523 = "523 Encryption Needed: Invalid Unencrypted Mail"
-
-
-def check_openpgp_payload(payload: bytes):
-    """Checks the OpenPGP payload.
-
-    OpenPGP payload must consist only of PKESK and SKESK packets
-    terminated by a single SEIPD packet.
-
-    Returns True if OpenPGP payload is correct,
-    False otherwise.
-
-    May raise IndexError while trying to read OpenPGP packet header
-    if it is truncated.
-    """
-    i = 0
-    while i < len(payload):
-        # Only OpenPGP format is allowed.
-        if payload[i] & 0xC0 != 0xC0:
-            return False
-
-        packet_type_id = payload[i] & 0x3F
-        i += 1
-
-        while payload[i] >= 224 and payload[i] < 255:
-            # Partial body length.
-            partial_length = 1 << (payload[i] & 0x1F)
-            i += 1 + partial_length
-
-        if payload[i] < 192:
-            # One-octet length.
-            body_len = payload[i]
-            i += 1
-        elif payload[i] < 224:
-            # Two-octet length.
-            body_len = ((payload[i] - 192) << 8) + payload[i + 1] + 192
-            i += 2
-        elif payload[i] == 255:
-            # Five-octet length.
-            body_len = (
-                (payload[i + 1] << 24)
-                | (payload[i + 2] << 16)
-                | (payload[i + 3] << 8)
-                | payload[i + 4]
-            )
-            i += 5
-        else:
-            # Impossible, partial body length was processed above.
-            return False
-
-        i += body_len
-
-        if i == len(payload):
-            # Last packet should be
-            # Symmetrically Encrypted and Integrity Protected Data Packet (SEIPD)
-            #
-            # This is the only place where this function may return `True`.
-            return packet_type_id == 18
-        elif packet_type_id not in [1, 3]:
-            # All packets except the last one must be either
-            # Public-Key Encrypted Session Key Packet (PKESK)
-            # or
-            # Symmetric-Key Encrypted Session Key Packet (SKESK)
-            return False
-
-    return False
-
-
-def check_armored_payload(payload: str, outgoing: bool):
-    """Check the armored PGP message for invalid content.
-
-    :param payload: the armored PGP message
-    :param outgoing: whether the message is outgoing or incoming
-    :return: whether the message is a valid PGP message
-    """
-    prefix = "-----BEGIN PGP MESSAGE-----\r\n"
-    if not payload.startswith(prefix):
-        return False
-    payload = payload.removeprefix(prefix)
-
-    while payload.endswith("\r\n"):
-        payload = payload.removesuffix("\r\n")
-    suffix = "-----END PGP MESSAGE-----"
-    if not payload.endswith(suffix):
-        return False
-    payload = payload.removesuffix(suffix)
-
-    version_comment = "Version: "
-    if payload.startswith(version_comment):
-        if outgoing:  # Disallow comments in outgoing messages
-            return False
-        # Remove comments from incoming messages
-        payload = payload.partition("\r\n")[2]
-
-    while payload.startswith("\r\n"):
-        payload = payload.removeprefix("\r\n")
-
-    # Remove CRC24.
-    payload = payload.rpartition("=")[0]
-
-    try:
-        payload = base64.b64decode(payload)
-    except binascii.Error:
-        return False
-
-    try:
-        return check_openpgp_payload(payload)
-    except IndexError:
-        return False
-
-
-def is_securejoin(message):
-    if message.get("secure-join") not in ["vc-request", "vg-request"]:
-        return False
-    if not message.is_multipart():
-        return False
-    parts_count = 0
-    for part in message.iter_parts():
-        parts_count += 1
-        if parts_count > 1:
-            return False
-        if part.is_multipart():
-            return False
-        if part.get_content_type() != "text/plain":
-            return False
-
-        payload = part.get_payload().strip().lower()
-        if payload not in ("secure-join: vc-request", "secure-join: vg-request"):
-            return False
-    return True
-
-
-def check_encrypted(message, outgoing=True):
-    """Check that the message is an OpenPGP-encrypted message.
-
-    MIME structure of the message must correspond to <https://www.rfc-editor.org/rfc/rfc3156>.
-    """
-    if not message.is_multipart():
-        return False
-    if message.get_content_type() != "multipart/encrypted":
-        return False
-    parts_count = 0
-    for part in message.iter_parts():
-        # We explicitly check Content-Type of each part later,
-        # but this is to be absolutely sure `get_payload()` returns string and not list.
-        if part.is_multipart():
-            return False
-
-        if parts_count == 0:
-            if part.get_content_type() != "application/pgp-encrypted":
-                return False
-
-            payload = part.get_payload()
-            if payload.strip() != "Version: 1":
-                return False
-        elif parts_count == 1:
-            if part.get_content_type() != "application/octet-stream":
-                return False
-
-            if not check_armored_payload(part.get_payload(), outgoing=outgoing):
-                return False
-        else:
-            return False
-        parts_count += 1
-    return True
-
-
-async def asyncmain_beforequeue(config, mode):
-    if mode == "outgoing":
-        port = config.filtermail_smtp_port
-        handler = OutgoingBeforeQueueHandler(config)
-    else:
-        port = config.filtermail_smtp_port_incoming
-        handler = IncomingBeforeQueueHandler(config)
-    HackedController(
-        handler,
-        hostname="127.0.0.1",
-        port=port,
-        data_size_limit=config.max_message_size,
-    ).start()
-
-
-def recipient_matches_passthrough(recipient, passthrough_recipients):
-    for addr in passthrough_recipients:
-        if recipient == addr:
-            return True
-        if addr[0] == "@" and recipient.endswith(addr):
-            return True
-    return False
-
-
-class HackedController(Controller):
-    def factory(self):
-        return SMTPDiscardRCPTO_options(self.handler, **self.SMTP_kwargs)
-
-
-class SMTPDiscardRCPTO_options(SMTP):
-    def _getparams(self, params):
-        # Ignore RCPT TO parameters.
-        #
-        # Otherwise parameters such as `ORCPT=...`
-        # or `NOTIFY=DELAY,FAILURE` (generated by Stalwart)
-        # make aiosmtpd reject the message here:
-        # <https://github.com/aio-libs/aiosmtpd/blob/98f578389ae86e5345cc343fa4e5a17b21d9c96d/aiosmtpd/smtp.py#L1379-L1384>
-        return {}
-
-
-class OutgoingBeforeQueueHandler:
-    def __init__(self, config):
-        self.config = config
-        self.send_rate_limiter = SendRateLimiter()
-
-    async def handle_MAIL(self, server, session, envelope, address, mail_options):
-        log_info(f"handle_MAIL from {address}")
-        envelope.mail_from = address
-        max_sent = self.config.max_user_send_per_minute
-        if not self.send_rate_limiter.is_sending_allowed(address, max_sent):
-            return f"450 4.7.1: Too much mail from {address}"
-
-        parts = envelope.mail_from.split("@")
-        if len(parts) != 2:
-            return f"500 Invalid from address <{envelope.mail_from!r}>"
-
-        return "250 OK"
-
-    async def handle_DATA(self, server, session, envelope):
-        loop = asyncio.get_running_loop()
-        return await loop.run_in_executor(None, self.sync_handle_DATA, envelope)
-
-    def sync_handle_DATA(self, envelope):
-        log_info("handle_DATA before-queue")
-        error = self.check_DATA(envelope)
-        if error:
-            return error
-        log_info("re-injecting the mail that passed checks")
-        client = SMTPClient("localhost", self.config.postfix_reinject_port)
-        client.sendmail(
-            envelope.mail_from, envelope.rcpt_tos, envelope.original_content
-        )
-        return "250 OK"
-
-    def check_DATA(self, envelope):
-        """the central filtering function for e-mails."""
-        log_info(f"Processing DATA message from {envelope.mail_from}")
-
-        message = BytesParser(policy=policy.default).parsebytes(envelope.content)
-        mail_encrypted = check_encrypted(message, outgoing=True)
-
-        _, from_addr = parseaddr(message.get("from").strip())
-
-        if envelope.mail_from.lower() != from_addr.lower():
-            return f"500 Invalid FROM <{from_addr!r}> for <{envelope.mail_from!r}>"
-
-        if mail_encrypted or is_securejoin(message):
-            print("Outgoing: Filtering encrypted mail.", file=sys.stderr)
-            return
-
-        print("Outgoing: Filtering unencrypted mail.", file=sys.stderr)
-
-        if envelope.mail_from in self.config.passthrough_senders:
-            return
-
-        # allow self-sent Autocrypt Setup Message
-        if envelope.rcpt_tos == [from_addr]:
-            if message.get("subject") == "Autocrypt Setup Message":
-                if message.get_content_type() == "multipart/mixed":
-                    return
-
-        passthrough_recipients = self.config.passthrough_recipients
-
-        for recipient in envelope.rcpt_tos:
-            if recipient_matches_passthrough(recipient, passthrough_recipients):
-                continue
-
-            print("Rejected unencrypted mail.", file=sys.stderr)
-            return ENCRYPTION_NEEDED_523
-
-
-class IncomingBeforeQueueHandler:
-    def __init__(self, config):
-        self.config = config
-
-    async def handle_DATA(self, server, session, envelope):
-        loop = asyncio.get_running_loop()
-        return await loop.run_in_executor(None, self.sync_handle_DATA, envelope)
-
-    def sync_handle_DATA(self, envelope):
-        log_info("handle_DATA before-queue")
-        error = self.check_DATA(envelope)
-        if error:
-            return error
-        log_info("re-injecting the mail that passed checks")
-
-        client = SMTPClient(
-            "localhost",
-            self.config.postfix_reinject_port_incoming,
-        )
-        client.sendmail(
-            envelope.mail_from, envelope.rcpt_tos, envelope.original_content
-        )
-        return "250 OK"
-
-    def check_DATA(self, envelope):
-        """the central filtering function for e-mails."""
-        log_info(f"Processing DATA message from {envelope.mail_from}")
-
-        message = BytesParser(policy=policy.default).parsebytes(envelope.content)
-        mail_encrypted = check_encrypted(message, outgoing=False)
-
-        if mail_encrypted or is_securejoin(message):
-            print("Incoming: Filtering encrypted mail.", file=sys.stderr)
-            return
-
-        print("Incoming: Filtering unencrypted mail.", file=sys.stderr)
-
-        # we want cleartext mailer-daemon messages to pass through
-        # chatmail core will typically not display them as normal messages
-        if message.get("auto-submitted"):
-            _, from_addr = parseaddr(message.get("from").strip())
-            if from_addr.lower().startswith("mailer-daemon@"):
-                if message.get_content_type() == "multipart/report":
-                    return
-
-        for recipient in envelope.rcpt_tos:
-            user = self.config.get_user(recipient)
-            if user is None or user.is_incoming_cleartext_ok():
-                continue
-
-            print("Rejected unencrypted mail.", file=sys.stderr)
-            return ENCRYPTION_NEEDED_523
-
-
-class SendRateLimiter:
-    def __init__(self):
-        self.addr2timestamps = {}
-
-    def is_sending_allowed(self, mail_from, max_send_per_minute):
-        last = self.addr2timestamps.setdefault(mail_from, [])
-        now = time.time()
-        last[:] = [ts for ts in last if ts >= (now - 60)]
-        if len(last) <= max_send_per_minute:
-            last.append(now)
-            return True
-        return False
-
-
-def log_info(msg):
-    print(msg, file=sys.stderr)
-
-
-def main():
-    args = sys.argv[1:]
-    assert len(args) == 2
-    config = read_config(args[0])
-    mode = args[1]
-    loop = asyncio.new_event_loop()
-    asyncio.set_event_loop(loop)
-    assert mode in ["incoming", "outgoing"]
-    task = asyncmain_beforequeue(config, mode)
-    loop.create_task(task)
-    log_info("entering serving loop")
-    loop.run_forever()

chatmaild/src/chatmaild/tests/test_filtermail.py

@@ -1,361 +0,0 @@
-import pytest
-
-from chatmaild.filtermail import (
-    IncomingBeforeQueueHandler,
-    OutgoingBeforeQueueHandler,
-    SendRateLimiter,
-    check_armored_payload,
-    check_encrypted,
-    is_securejoin,
-)
-
-
-@pytest.fixture
-def maildomain():
-    # let's not depend on a real chatmail instance for the offline tests below
-    return "chatmail.example.org"
-
-
-@pytest.fixture
-def handler(make_config, maildomain):
-    config = make_config(maildomain)
-    return OutgoingBeforeQueueHandler(config)
-
-
-@pytest.fixture
-def inhandler(make_config, maildomain):
-    config = make_config(maildomain)
-    return IncomingBeforeQueueHandler(config)
-
-
-def test_reject_forged_from(maildata, gencreds, handler):
-    class env:
-        mail_from = gencreds()[0]
-        rcpt_tos = [gencreds()[0]]
-
-    # test that the filter lets good mail through
-    to_addr = gencreds()[0]
-    env.content = maildata(
-        "encrypted.eml", from_addr=env.mail_from, to_addr=to_addr
-    ).as_bytes()
-
-    assert not handler.check_DATA(envelope=env)
-
-    # test that the filter rejects forged mail
-    env.content = maildata(
-        "encrypted.eml", from_addr="forged@c3.testrun.org", to_addr=to_addr
-    ).as_bytes()
-    error = handler.check_DATA(envelope=env)
-    assert "500" in error
-
-
-def test_filtermail_no_encryption_detection(maildata):
-    msg = maildata(
-        "plain.eml", from_addr="some@example.org", to_addr="other@example.org"
-    )
-    assert not check_encrypted(msg)
-
-    # https://xkcd.com/1181/
-    msg = maildata(
-        "fake-encrypted.eml", from_addr="some@example.org", to_addr="other@example.org"
-    )
-    assert not check_encrypted(msg)
-
-
-def test_filtermail_securejoin_detection(maildata):
-    msg = maildata(
-        "securejoin-vc.eml", from_addr="some@example.org", to_addr="other@example.org"
-    )
-    assert is_securejoin(msg)
-
-    msg = maildata(
-        "securejoin-vc-fake.eml",
-        from_addr="some@example.org",
-        to_addr="other@example.org",
-    )
-    assert not is_securejoin(msg)
-
-
-def test_filtermail_encryption_detection(maildata):
-    msg = maildata(
-        "encrypted.eml",
-        from_addr="1@example.org",
-        to_addr="2@example.org",
-        subject="Subject does not matter, will be replaced anyway",
-    )
-    assert check_encrypted(msg)
-
-
-def test_filtermail_no_literal_packets(maildata):
-    """Test that literal OpenPGP packet is not considered an encrypted mail."""
-    msg = maildata("literal.eml", from_addr="1@example.org", to_addr="2@example.org")
-    assert not check_encrypted(msg)
-
-
-def test_filtermail_unencrypted_mdn(maildata, gencreds):
-    """Unencrypted MDNs should not pass."""
-    from_addr = gencreds()[0]
-    to_addr = gencreds()[0] + ".other"
-    msg = maildata("mdn.eml", from_addr=from_addr, to_addr=to_addr)
-
-    assert not check_encrypted(msg)
-
-
-def test_send_rate_limiter():
-    limiter = SendRateLimiter()
-    for i in range(100):
-        if limiter.is_sending_allowed("some@example.org", 10):
-            if i <= 10:
-                continue
-            pytest.fail("limiter didn't work")
-        else:
-            assert i == 11
-            break
-
-
-def test_cleartext_excempt_privacy(maildata, gencreds, handler):
-    from_addr = gencreds()[0]
-    to_addr = "privacy@testrun.org"
-    handler.config.passthrough_recipients = [to_addr]
-    false_to = "privacy@something.org"
-
-    msg = maildata("plain.eml", from_addr=from_addr, to_addr=to_addr)
-
-    class env:
-        mail_from = from_addr
-        rcpt_tos = [to_addr]
-        content = msg.as_bytes()
-
-    # assert that None/no error is returned
-    assert not handler.check_DATA(envelope=env)
-
-    class env2:
-        mail_from = from_addr
-        rcpt_tos = [to_addr, false_to]
-        content = msg.as_bytes()
-
-    assert "523" in handler.check_DATA(envelope=env2)
-
-
-def test_cleartext_self_send_autocrypt_setup_message(maildata, gencreds, handler):
-    from_addr = gencreds()[0]
-    to_addr = from_addr
-
-    msg = maildata("asm.eml", from_addr=from_addr, to_addr=to_addr)
-
-    class env:
-        mail_from = from_addr
-        rcpt_tos = [to_addr]
-        content = msg.as_bytes()
-
-    assert not handler.check_DATA(envelope=env)
-
-
-def test_cleartext_send_fails(maildata, gencreds, handler):
-    from_addr = gencreds()[0]
-    to_addr = gencreds()[0]
-
-    msg = maildata("plain.eml", from_addr=from_addr, to_addr=to_addr)
-
-    class env:
-        mail_from = from_addr
-        rcpt_tos = [to_addr]
-        content = msg.as_bytes()
-
-    res = handler.check_DATA(envelope=env)
-    assert "523 Encryption Needed" in res
-
-
-def test_cleartext_incoming_fails(maildata, gencreds, inhandler):
-    from_addr = gencreds()[0]
-    to_addr, password = gencreds()
-
-    msg = maildata("plain.eml", from_addr=from_addr, to_addr=to_addr)
-
-    class env:
-        mail_from = from_addr
-        rcpt_tos = [to_addr]
-        content = msg.as_bytes()
-
-    user = inhandler.config.get_user(to_addr)
-    user.set_password(password)
-    res = inhandler.check_DATA(envelope=env)
-    assert "523 Encryption Needed" in res
-
-    user.allow_incoming_cleartext()
-    assert not inhandler.check_DATA(envelope=env)
-
-
-def test_cleartext_incoming_mailer_daemon(maildata, gencreds, inhandler):
-    from_addr = "mailer-daemon@example.org"
-    to_addr = gencreds()[0]
-
-    msg = maildata("mailer-daemon.eml", from_addr=from_addr, to_addr=to_addr)
-
-    class env:
-        mail_from = from_addr
-        rcpt_tos = [to_addr]
-        content = msg.as_bytes()
-
-    assert not inhandler.check_DATA(envelope=env)
-
-
-def test_cleartext_passthrough_domains(maildata, gencreds, handler):
-    from_addr = gencreds()[0]
-    to_addr = "privacy@x.y.z"
-    handler.config.passthrough_recipients = ["@x.y.z"]
-    false_to = "something@x.y"
-
-    msg = maildata("plain.eml", from_addr=from_addr, to_addr=to_addr)
-
-    class env:
-        mail_from = from_addr
-        rcpt_tos = [to_addr]
-        content = msg.as_bytes()
-
-    # assert that None/no error is returned
-    assert not handler.check_DATA(envelope=env)
-
-    class env2:
-        mail_from = from_addr
-        rcpt_tos = [to_addr, false_to]
-        content = msg.as_bytes()
-
-    assert "523" in handler.check_DATA(envelope=env2)
-
-
-def test_cleartext_passthrough_senders(gencreds, handler, maildata):
-    acc1 = gencreds()[0]
-    to_addr = "recipient@something.org"
-    handler.config.passthrough_senders = [acc1]
-
-    msg = maildata("plain.eml", from_addr=acc1, to_addr=to_addr)
-
-    class env:
-        mail_from = acc1
-        rcpt_tos = to_addr
-        content = msg.as_bytes()
-
-    # assert that None/no error is returned
-    assert not handler.check_DATA(envelope=env)
-
-
-def test_check_armored_payload():
-    prefix = "-----BEGIN PGP MESSAGE-----\r\n"
-    comment = "Version: ProtonMail\r\n"
-    payload = """\r
-wU4DSqFx0d1yqAoSAQdAYkX/ZN/Az4B0k7X47zKyWrXxlDEdS3WOy0Yf2+GJTFgg\r
-Zk5ql0mLG8Ze+ZifCS0XMO4otlemSyJ0K1ZPdFMGzUDBTgNqzkFabxXoXRIBB0AM\r
-755wlX41X6Ay3KhnwBq7yEqSykVH6F3x11iHPKraLCAGZoaS8bKKNy/zg5slda1X\r
-pt14b4aC1VwtSnYhcRRELNLD/wE2TFif+g7poMmFY50VyMPLYjVP96Z5QCT4+z4H\r
-Ikh/pRRN8S3JNMrRJHc6prooSJmLcx47Y5un7VFy390MsJ+LiUJuQMDdYWRAinfs\r
-Ebm89Ezjm7F03qbFPXE0X4ZNzVXS/eKO0uhJQdiov/vmbn41rNtHmNpqjaO0vi5+\r
-sS9tR7yDUrIXiCUCN78eBLVioxtktsPZm5cDORbQWzv+7nmCEz9/JowCUcBVdCGn\r
-1ofOaH82JCAX/cRx08pLaDNj6iolVBsi56Dd+2bGxJOZOG2AMcEyz0pXY0dOAJCD\r
-iUThcQeGIdRnU3j8UBcnIEsjLu2+C+rrwMZQESMWKnJ0rnqTk0pK5kXScr6F/L0L\r
-UE49ccIexNm3xZvYr5drszr6wz3Tv5fdue87P4etBt90gF/Vzknck+g1LLlkzZkp\r
-d8dI0k2tOSPjUbDPnSy1x+X73WGpPZmj0kWT+RGvq0nH6UkJj3AQTG2qf1T8jK+3\r
-rTp3LR9vDkMwDjX4R8SA9c0wdnUzzr79OYQC9lTnzcx+fM6BBmgQ2GrS33jaFLp7\r
-L6/DFpCl5zhnPjM/2dKvMkw/Kd6XS/vjwsO405FQdjSDiQEEAZA+ZvAfcjdccbbU\r
-yCO+x0QNdeBsufDVnh3xvzuWy4CICdTQT4s1AWRPCzjOj+SGmx5WqCLWfsd8Ma0+\r
-w/C7SfTYu1FDQILLM+llpq1M/9GPley4QZ8JQjo262AyPXsPF/OW48uuZz0Db1xT\r
-Yh4iHBztj4VSdy7l2+IyaIf7cnL4EEBFxv/MwmVDXvDlxyvfAfIsd3D9SvJESzKZ\r
-VWDYwaocgeCN+ojKu1p885lu1EfRbX3fr3YO02K5/c2JYDkc0Py0W3wUP/J1XUax\r
-pbKpzwlkxEgtmzsGqsOfMJqBV3TNDrOA2uBsa+uBqP5MGYLZ49S/4v/bW9I01Cr1\r
-D2ZkV510Y1Vgo66WlP8mRqOTyt/5WRhPD+MxXdk67BNN/PmO6tMlVoJDuk+XwWPR\r
-t2TvNaND/yabT9eYI55Og4fzKD6RIjouUX8DvKLkm+7aXxVs2uuLQ3Jco3O82z55\r
-dbShU1jYsrw9oouXUz06MHPbkdhNbF/2hfhZ2qA31sNeovJw65iUv7sDKX3LVWgJ\r
-10jlywcDwqlU8CO7WC9lGixYTbnOkYZpXCGEl8e6Jbs79l42YFo4ogYpFK1NXFhV\r
-kOXRmDf/wmfj+c/ld3L2PkvwlgofhCudOQknZbo3ub1gjiTn7L+lMGHIj/3suMIl\r
-ID4EUxAXScIM1ZEz2fjtW5jATlqYcLjLTbf/olw6HFyPNH+9IssqXeZNKnGwPUB9\r
-3lTXsg0tpzl+x7F/2WjEw1DSNhjC0KnHt1vEYNMkUGDGFdN9y3ERLqX/FIgiASUb\r
-bTvAVupnAK3raBezGmhrs6LsQtLS9P0VvQiLU3uDhMqw8Z4SISLpcD+NnVBHzQqm\r
-6W5Qn/8xsCL6av18yUVTi2G3igt3QCNoYx9evt2ZcIkNoyyagUVjfZe5GHXh8Dnz\r
-GaBXW/hg3HlXLRGaQu4RYCzBMJILcO25OhZOg6jbkCLiEexQlm2e9krB5cXR49Al\r
-UN4fiB0KR9JyG2ayUdNJVkXZSZLnHyRgiaadlpUo16LVvw==\r
-=b5Kp\r
------END PGP MESSAGE-----\r
-\r
-\r
-"""
-
-    commented_payload = prefix + comment + payload
-    assert check_armored_payload(commented_payload, outgoing=False) == True
-    assert check_armored_payload(commented_payload, outgoing=True) == False
-
-    payload = prefix + payload
-    assert check_armored_payload(payload, outgoing=False) == True
-    assert check_armored_payload(payload, outgoing=True) == True
-
-    payload = payload.removesuffix("\r\n")
-    assert check_armored_payload(payload, outgoing=False) == True
-    assert check_armored_payload(payload, outgoing=True) == True
-
-    payload = payload.removesuffix("\r\n")
-    assert check_armored_payload(payload, outgoing=False) == True
-    assert check_armored_payload(payload, outgoing=True) == True
-
-    payload = payload.removesuffix("\r\n")
-    assert check_armored_payload(payload, outgoing=False) == True
-    assert check_armored_payload(payload, outgoing=True) == True
-
-    payload = """-----BEGIN PGP MESSAGE-----\r
-\r
-HELLOWORLD
------END PGP MESSAGE-----\r
-\r
-"""
-    assert check_armored_payload(payload, outgoing=False) == False
-    assert check_armored_payload(payload, outgoing=True) == False
-
-    payload = """-----BEGIN PGP MESSAGE-----\r
-\r
-=njUN
------END PGP MESSAGE-----\r
-\r
-"""
-    assert check_armored_payload(payload, outgoing=False) == False
-    assert check_armored_payload(payload, outgoing=True) == False
-
-    # Test payload using partial body length
-    # as generated by GopenPGP.
-    payload = """-----BEGIN PGP MESSAGE-----\r
-\r
-wV4DdCVjRfOT3TQSAQdAY5+pjT6mlCxPGdR3be4w7oJJRUGIPI/Vnh+mJxGSm34w\r
-LNlVc89S1g22uQYFif2sUJsQWbpoHpNkuWpkSgOaHmNvrZiY/YU5iv+cZ3LbmtUG\r
-0uoBisSHh9O1c+5sYZSbrvYZ1NOwlD7Fv/U5/Mw4E5+CjxfdgNGp5o3DDddzPK78\r
-jseDhdSXxnaiIJC93hxNX6R1RPt3G2gukyzx69wciPQShcF8zf3W3o75Ed7B8etV\r
-QEeB16xzdFhKa9JxdjTu3osgCs21IO7wpcFkjc7nZzlW6jPnELJJaNmv4yOOCjMp\r
-6YAkaN/BkL+jHTznHDuDsT5ilnTXpwHDU1Cm9PIx/KFcNCQnIB+2DcdIHPHUH1ci\r
-jvqoeXAVWjKXEjS7PqPFuP/xGbrWG2ugs+toXJOKbgRkExvKs1dwPFKrgghvCVbW\r
-AcKejQKAPArLwpkA7aD875TZQShvGt74fNs45XBlGOYOnNOAJ1KAmzrXLIDViyyB\r
-kDsmTBk785xofuCkjBpXSe6vsMprPzCteDfaUibh8FHeJjucxPerwuOPEmnogNaf\r
-YyL4+iy8H8I9/p7pmUqILprxTG0jTOtlk0bTVzeiF56W1xbtSEMuOo4oFbQTyOM2\r
-bKXaYo774Jm+rRtKAnnI2dtf9RpK19cog6YNzfYjesLKbXDsPZbN5rmwyFiCvvxC\r
-kQ6JLob+B2fPdY2gzy7LypxktS8Zi1HJcWDHJGVmQodaDLqKUObb4M26bXDe6oxI\r
-NS8PJz5exVbM3KhZnUOEn6PJRBBf5a/ZqxlhZPcQo/oBuhKpBRpO5kSDwPIUByu3\r
-UlXLSkpMqe9pUarAOEuQjfl2RVY7U+RrQYp4YP5keMO+i8NCefAFbowTTufO1JIq\r
-2nVgCi/QVnxZyEc9OYt/8AE3g4cdojE+vsSDifZLSWYIetpfrohHv3dT3StD1QRG\r
-0QE6qq6oKpg/IL0cjvuX4c7a7bslv2fXp8t75y37RU6253qdIebhxc/cRhPbc/yu\r
-p0YLyD4SrvKTLP2ZV95jT4IPEpqm4AN3QmiOzdtqR2gLyb62L8QfqI/FdwsIiRiM\r
-hqydwoqt/lfSqG1WKPh+6EkMkH+TDiCC1BQdbN1MNcyUtcjb35PR2c8Ld2TF3guA\r
-jLIqMt/Vb7hBoMb2FcsOYY25ka9oV62OwgKWLXnFzk+modMR5fzb4kxVVAYEqP+D\r
-T5KO1Vs76v1fyPGOq6BbBCvLwTqe/e6IZInJles4v5jrhnLcGKmNGivCUDe6X6NY\r
-UKNt5RsZllwDQpaAb5dMNhyrk8SgIE7TBI7rvqIdUCE52Vy+0JDxFg5olRpFUfO6\r
-/MyTW3Yo/ekk/npHr7iYYqJTCc21bDGLWQcIo/XO7WPxrKNWGBNPFnkRdw0MaKr4\r
-+cEM3V8NFnSEpC12xA+RX/CezuJtwXZK5MpG76eYqMO6qyC+c25YcFecEufDZDxx\r
-ZLqRszVRyxyWPtk/oIeQK2v9wOqY6N9/ff01gHz69vqYqN5bUw/QKZsmx1zW+gPw\r
-6x2tDK2BHeYl182gCbhlKISRFwCtbjqZSkiKWao/VtygHkw0fK34avJuyQ/X9YaN\r
-BRy+7Lf3VA53pnB5WJ1xwRXN8VDvmZeXzv2krHveCMemj0OjnRoCLu117xN0A5m9\r
-Fm/RoDix5PolDHtWTtr2m1n2hp2LHnj8at9lFEd0SKhAYHVL9KjzycwWODZRXt+x\r
-zGDDuooEeTvdY5NLyKcl4gETz1ZP4Ez5jGGjhPSwSpq1mU7UaJ9ZXXdr4KHyifW6\r
-ggNzNsGhXTap7IWZpTtqXABydfiBshmH2NjqtNDwBweJVSgP10+r0WhMWlaZs6xl\r
-V3o5yskJt6GlkwpJxZrTvN6Tiww/eW7HFV6NGf7IRSWY5tJc/iA7/92tOmkdvJ1q\r
-myLbG7cJB787QjplEyVe2P/JBO6xYvbkJLf9Q+HaviTO25rugRSrYsoKMDfO8VlQ\r
-1CcnTPVtApPZJEQzAWJEgVAM8uIlkqWJJMgyWT34sTkdBeCUFGloXQFs9Yxd0AGf\r
-/zHEkYZSTKpVSvAIGu4=\r
-=6iHb\r
------END PGP MESSAGE-----\r
-"""
-    assert check_armored_payload(payload, outgoing=False) == True
-    assert check_armored_payload(payload, outgoing=True) == True

cmdeploy/src/cmdeploy/basedeploy.py

@@ -17,9 +17,8 @@ def configure_remote_units(mail_domain, units) -> None:
 
     # install systemd units
     for fn in units:
-        execpath = fn if fn != "filtermail-incoming" else "filtermail"
         params = dict(
-            execpath=f"{remote_venv_dir}/bin/{execpath}",
+            execpath=f"{remote_venv_dir}/bin/{fn}",
             config_path=remote_chatmail_inipath,
             remote_venv_dir=remote_venv_dir,
             mail_domain=mail_domain,

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -71,6 +71,11 @@ def run_cmd_options(parser):
         action="store_true",
         help="install/upgrade the server, but disable postfix & dovecot for now",
     )
+    parser.add_argument(
+        "--website-only",
+        action="store_true",
+        help="only update/deploy the website, skipping full server upgrade/deployment, useful when you only changed/updated the web pages and don't need to re-run a full server upgrade",
+    )
     parser.add_argument(
         "--skip-dns-check",
         dest="dns_check_disabled",
@@ -84,6 +89,7 @@ def run_cmd(args, out):
     """Deploy chatmail services on the remote server."""
 
     ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
+    ssh_host = ssh_host.strip("[").strip("]")
     sshexec = get_sshexec(ssh_host)
     require_iroh = args.config.enable_iroh_relay
     if not args.dns_check_disabled:
@@ -93,6 +99,7 @@ def run_cmd(args, out):
 
     env = os.environ.copy()
     env["CHATMAIL_INI"] = args.inipath
+    env["CHATMAIL_WEBSITE_ONLY"] = "True" if args.website_only else ""
     env["CHATMAIL_DISABLE_MAIL"] = "True" if args.disable_mail else ""
     env["CHATMAIL_REQUIRE_IROH"] = "True" if require_iroh else ""
     deploy_path = importlib.resources.files(__package__).joinpath("run.py").resolve()
@@ -108,7 +115,12 @@ def run_cmd(args, out):
 
     try:
         retcode = out.check_call(cmd, env=env)
-        if retcode == 0:
+        if args.website_only:
+            if retcode == 0:
+                out.green("Website deployment completed.")
+            else:
+                out.red("Website deployment failed.")
+        elif retcode == 0:
             out.green("Deploy completed, call `cmdeploy dns` next.")
         elif not remote_data["acme_account_url"]:
             out.red("Deploy completed but letsencrypt not configured")

cmdeploy/src/cmdeploy/deployers.py

@@ -17,7 +17,6 @@ from pyinfra.operations import apt, files, pip, server, systemd
 
 from cmdeploy.cmdeploy import Out
 
-from .acmetool import AcmetoolDeployer
 from .basedeploy import (
     Deployer,
     Deployment,
@@ -26,6 +25,7 @@ from .basedeploy import (
     get_resource,
 )
 from .dovecot.deployer import DovecotDeployer
+from .filtermail.deployer import FiltermailDeployer
 from .mtail.deployer import MtailDeployer
 from .nginx.deployer import NginxDeployer
 from .opendkim.deployer import OpendkimDeployer
@@ -416,8 +416,6 @@ class ChatmailVenvDeployer(Deployer):
     def __init__(self, config):
         self.config = config
         self.units = (
-            "filtermail",
-            "filtermail-incoming",
             "chatmail-metadata",
             "lastlogin",
             "chatmail-expire",
@@ -502,23 +500,28 @@ class GithashDeployer(Deployer):
         except Exception:
             git_diff = ""
         files.put(
-            name="Upload chatmail relay git commiit hash",
+            name="Upload chatmail relay git commit hash",
             src=StringIO(git_hash + git_diff),
             dest="/etc/chatmail-version",
             mode="700",
         )
 
 
-def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
+def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -> None:
     """Deploy a chat-mail instance.
 
     :param config_path: path to chatmail.ini
     :param disable_mail: whether to disable postfix & dovecot
+    :param website_only: if True, only deploy the website
     """
     config = read_config(config_path)
     check_config(config)
     mail_domain = config.mail_domain
 
+    if website_only:
+        Deployment().perform_stages([WebsiteDeployer(config)])
+        return
+
     if host.get_fact(Port, port=53) != "unbound":
         files.line(
             name="Add 9.9.9.9 to resolv.conf",
@@ -529,13 +532,14 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
     port_services = [
         (["master", "smtpd"], 25),
         ("unbound", 53),
-        ("acmetool", 80),
         (["imap-login", "dovecot"], 143),
         ("nginx", 443),
         (["master", "smtpd"], 465),
         (["master", "smtpd"], 587),
         (["imap-login", "dovecot"], 993),
         ("iroh-relay", 3340),
+        ("mtail", 3903),
+        ("dovecot-stats", 3904),
         ("nginx", 8443),
         (["master", "smtpd"], config.postfix_reinject_port),
         (["master", "smtpd"], config.postfix_reinject_port_incoming),
@@ -557,11 +561,11 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
     all_deployers = [
         ChatmailDeployer(mail_domain),
         LegacyRemoveDeployer(),
+        FiltermailDeployer(),
         JournaldDeployer(),
         UnboundDeployer(),
         TurnDeployer(mail_domain),
         IrohDeployer(config.enable_iroh_relay),
-        AcmetoolDeployer(config.acme_email, tls_domains),
         WebsiteDeployer(config),
         ChatmailVenvDeployer(config),
         MtastsDeployer(),

cmdeploy/src/cmdeploy/dovecot/auth.conf

@@ -4,7 +4,7 @@ iterate_prefix = userdb/
 
 default_pass_scheme = plain
 # %E escapes characters " (double quote), ' (single quote) and \ (backslash) with \ (backslash).
-# See <https://doc.dovecot.org/configuration_manual/config_file/config_variables/#modifiers>
+# See <https://doc.dovecot.org/2.3/configuration_manual/config_file/config_variables/#modifiers>
 # for documentation.
 #
 # We escape user-provided input and use double quote as a separator.

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -37,9 +37,7 @@ class DovecotDeployer(Deployer):
         restart = False if self.disable_mail else self.need_restart
 
         systemd.service(
-            name="disable dovecot for now"
-            if self.disable_mail
-            else "Start and enable Dovecot",
+            name="Disable dovecot for now" if self.disable_mail else "Start and enable Dovecot",
             service="dovecot.service",
             running=False if self.disable_mail else True,
             enabled=False if self.disable_mail else True,
@@ -116,7 +114,7 @@ def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
     )
     need_restart |= lua_push_notification_script.changed
 
-    # as per https://doc.dovecot.org/configuration_manual/os/
+    # as per https://doc.dovecot.org/2.3/configuration_manual/os/
     # it is recommended to set the following inotify limits
     for name in ("max_user_instances", "max_user_watches"):
         key = f"fs.inotify.{name}"
@@ -145,4 +143,11 @@ def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
     )
     daemon_reload |= restart_conf.changed
 
+    # Validate dovecot configuration before restart
+    if need_restart:
+        server.shell(
+            name="Validate dovecot configuration",
+            commands=["doveconf -n >/dev/null"],
+        )
+
     return need_restart, daemon_reload

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -7,6 +7,7 @@ listen = *
 protocols = imap lmtp
 
 auth_mechanisms = plain
+auth_username_chars = abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890.-_@[]
 
 {% if debug == true %}
 auth_verbose = yes
@@ -26,7 +27,7 @@ default_client_limit = 20000
 # Increase number of logged in IMAP connections.
 # Each connection is handled by a separate `imap` process.
 # `imap` process should have `client_limit=1` as described in
-# <https://doc.dovecot.org/configuration_manual/service_configuration/#service-limits>
+# <https://doc.dovecot.org/2.3/configuration_manual/service_configuration/#service-limits>
 # so each logged in IMAP session will need its own `imap` process.
 #
 # If this limit is reached,
@@ -44,11 +45,11 @@ mail_server_comment = Chatmail server
 
 # `zlib` enables compressing messages stored in the maildir.
 # See
-# <https://doc.dovecot.org/configuration_manual/zlib_plugin/>
+# <https://doc.dovecot.org/2.3/configuration_manual/zlib_plugin/>
 # for documentation.
 #
 # quota plugin documentation:
-# <https://doc.dovecot.org/configuration_manual/quota_plugin/>
+# <https://doc.dovecot.org/2.3/configuration_manual/quota_plugin/>
 mail_plugins = zlib quota
 
 imap_capability = +XDELTAPUSH XCHATMAIL
@@ -125,13 +126,13 @@ plugin {
 
 protocol lmtp {
   # notify plugin is a dependency of push_notification plugin:
-  # <https://doc.dovecot.org/settings/plugin/notify-plugin/>
+  # <https://doc.dovecot.org/2.3/settings/plugin/notify-plugin/>
   #
   # push_notification plugin documentation:
-  # <https://doc.dovecot.org/configuration_manual/push_notification/>
+  # <https://doc.dovecot.org/2.3/configuration_manual/push_notification/>
   #
   # mail_lua and push_notification_lua are needed for Lua push notification handler.
-  # <https://doc.dovecot.org/configuration_manual/push_notification/#configuration>
+  # <https://doc.dovecot.org/2.3/configuration_manual/push_notification/#configuration>
   mail_plugins = $mail_plugins mail_lua notify push_notification push_notification_lua
 }
 
@@ -154,7 +155,7 @@ plugin {
 
 # push_notification configuration
 plugin {
-  # <https://doc.dovecot.org/configuration_manual/push_notification/#lua-lua>
+  # <https://doc.dovecot.org/2.3/configuration_manual/push_notification/#lua-lua>
   push_notification_driver = lua:file=/etc/dovecot/push_notification.lua
 }
 
@@ -168,6 +169,8 @@ service lmtp {
   }
 }
 
+lmtp_add_received_header = no
+
 service auth {
   unix_listener /var/spool/postfix/private/auth {
     mode = 0660
@@ -226,8 +229,8 @@ service anvil {
 }
 
 ssl = required
-ssl_cert = </var/lib/acme/live/{{ config.mail_domain }}/fullchain
-ssl_key = </var/lib/acme/live/{{ config.mail_domain }}/privkey
+ssl_cert = </etc/ssl/certs/ssl-cert-snakeoil.pem
+ssl_key = </etc/ssl/private/ssl-cert-snakeoil.key
 ssl_dh = </usr/share/dovecot/dh.pem
 ssl_min_protocol = TLSv1.3
 ssl_prefer_server_ciphers = yes
@@ -277,3 +280,156 @@ service imap-hibernate {
   }
 }
 {% endif %}
+
+{% if config.mtail_address %}
+#
+# Dovecot Statistics
+#
+# OpenMetrics endpoint at http://{{- config.mtail_address}}:3904/metrics
+service stats {
+  inet_listener http {
+    port = 3904
+    address = {{- config.mtail_address}}
+  }
+}
+
+# IMAP Command Metrics
+# - Bytes in/out for compression efficiency analysis
+# - Lock wait time for contention debugging
+# - Grouped by command name and reply state
+metric imap_command {
+  filter = event=imap_command_finished
+  fields = bytes_in bytes_out lock_wait_usecs running_usecs
+  group_by = cmd_name tagged_reply_state
+}
+
+# Duration buckets for latency histograms (base 10: 10us, 100us, 1ms, 10ms, 100ms, 1s, 10s, 100s)
+metric imap_command_duration {
+  filter = event=imap_command_finished
+  group_by = cmd_name duration:exponential:1:8:10
+}
+
+# Slow command outliers (>1 second = 1000000 usecs)
+# Useful for alerting without high cardinality
+metric imap_command_slow {
+  filter = event=imap_command_finished AND duration>1000000 AND NOT cmd_name=IDLE
+  group_by = cmd_name
+}
+
+# IDLE-specific Metrics
+
+metric imap_idle {
+  filter = event=imap_command_finished AND cmd_name=IDLE
+  fields = bytes_in bytes_out running_usecs
+  group_by = tagged_reply_state
+}
+
+metric imap_idle_duration {
+  filter = event=imap_command_finished AND cmd_name=IDLE
+  # Base 10: 100ms to 27h (covers short wakeups to long idle sessions)
+  group_by = duration:exponential:5:11:10
+}
+
+metric imap_idle_commands {
+  filter = event=imap_command_finished AND cmd_name=IDLE
+  group_by = tagged_reply_state
+}
+
+metric imap_idle_failed {
+  filter = event=imap_command_finished AND cmd_name=IDLE AND NOT tagged_reply_state=OK
+}
+
+# Hibernation Metrics (requires imap_hibernate_timeout)
+
+metric imap_hibernated {
+  filter = event=imap_client_hibernated
+}
+
+metric imap_hibernated_failed {
+  filter = event=imap_client_hibernated AND error=*
+}
+
+metric imap_unhibernated {
+  filter = event=imap_client_unhibernated
+  fields = hibernation_usecs
+}
+
+metric imap_unhibernated_reason {
+  filter = event=imap_client_unhibernated
+  group_by = reason
+  fields = hibernation_usecs
+}
+
+metric imap_unhibernated_reason_sleep {
+  filter = event=imap_client_unhibernated
+  group_by = reason hibernation_usecs:exponential:4:8:10
+}
+
+metric imap_unhibernated_failed {
+  filter = event=imap_client_unhibernated AND error=*
+}
+
+# Hibernation duration buckets (how long clients stayed hibernated)
+# Base 10: 100ms to 27h
+metric imap_hibernation_duration {
+  filter = event=imap_client_unhibernated
+  group_by = reason duration:exponential:5:11:10
+}
+
+# Authentication / Login Metrics
+
+metric auth_request {
+  filter = event=auth_request_finished
+  group_by = success
+}
+
+metric auth_request_duration {
+  filter = event=auth_request_finished
+  group_by = success duration:exponential:2:6:10
+}
+
+metric auth_failed {
+  filter = event=auth_request_finished AND success=no
+}
+
+# Passdb cache effectiveness
+metric auth_passdb {
+  filter = event=auth_passdb_request_finished
+  group_by = result cache
+}
+
+# Master login (post-auth userdb lookup)
+metric auth_master_login {
+  filter = event=auth_master_client_login_finished
+}
+
+metric auth_master_login_failed {
+  filter = event=auth_master_client_login_finished AND error=*
+}
+
+# Mail Delivery (LMTP) - affects IDLE wakeup latency
+
+metric mail_delivery {
+  filter = event=mail_delivery_finished
+}
+
+metric mail_delivery_duration {
+  filter = event=mail_delivery_finished
+  group_by = duration:exponential:3:7:10
+}
+
+metric mail_delivery_failed {
+  filter = event=mail_delivery_finished AND error=*
+}
+
+# Connection Events
+
+metric client_connected {
+  filter = event=client_connection_connected AND category="service:imap"
+}
+
+metric client_disconnected {
+  filter = event=client_connection_disconnected AND category="service:imap"
+  fields = bytes_in bytes_out
+}
+{% endif %}

cmdeploy/src/cmdeploy/filtermail/deployer.py

@@ -0,0 +1,52 @@
+from pyinfra import facts, host
+from pyinfra.operations import files, systemd
+
+from cmdeploy.basedeploy import Deployer, get_resource
+
+
+class FiltermailDeployer(Deployer):
+    services = ["filtermail", "filtermail-incoming"]
+    bin_path = "/usr/local/bin/filtermail"
+    config_path = "/usr/local/lib/chatmaild/chatmail.ini"
+
+    def __init__(self):
+        self.need_restart = False
+
+    def install(self):
+        arch = host.get_fact(facts.server.Arch)
+        url = f"https://github.com/chatmail/filtermail/releases/download/v0.1.2/filtermail-{arch}"
+        sha256sum = {
+            "x86_64": "de7de6e011ffc06881d3a05fc9788e327ba2389219e77280ace38b429e11a5ce",
+            "aarch64": "a78fcdfb81eb3d9c8a8b6f84f6c0a75519b8be01aa25bd4617d72aae543992b4",
+        }[arch]
+        self.need_restart |= files.download(
+            name="Download filtermail",
+            src=url,
+            sha256sum=sha256sum,
+            dest=self.bin_path,
+            mode="755",
+        ).changed
+
+    def configure(self):
+        for service in self.services:
+            self.need_restart |= files.template(
+                src=get_resource(f"filtermail/{service}.service.j2"),
+                dest=f"/etc/systemd/system/{service}.service",
+                user="root",
+                group="root",
+                mode="644",
+                bin_path=self.bin_path,
+                config_path=self.config_path,
+            ).changed
+
+    def activate(self):
+        for service in self.services:
+            systemd.service(
+                name=f"Start and enable {service}",
+                service=f"{service}.service",
+                running=True,
+                enabled=True,
+                restarted=self.need_restart,
+                daemon_reload=True,
+            )
+        self.need_restart = False

cmdeploy/src/cmdeploy/filtermail/filtermail-incoming.service.j2

@@ -2,11 +2,10 @@
 Description=Incoming Chatmail Postfix before queue filter 
 
 [Service]
-ExecStart={execpath} {config_path} incoming
+ExecStart={{ bin_path }} {{ config_path }} incoming
 Restart=always
 RestartSec=30
 User=vmail
 
 [Install]
 WantedBy=multi-user.target
-

cmdeploy/src/cmdeploy/filtermail/filtermail.service.j2

@@ -2,7 +2,7 @@
 Description=Outgoing Chatmail Postfix before queue filter
 
 [Service]
-ExecStart={execpath} {config_path} outgoing
+ExecStart={{ bin_path }} {{ config_path }} outgoing
 Restart=always
 RestartSec=30
 User=vmail

cmdeploy/src/cmdeploy/mtail/delivered_mail.mtail

@@ -44,21 +44,37 @@ counter warning_count
 }
 
 
-counter filtered_mail_count
+counter filtered_outgoing_mail_count
 
-counter encrypted_mail_count
-/Filtering encrypted mail\./ {
-  encrypted_mail_count++
-  filtered_mail_count++
+counter outgoing_encrypted_mail_count
+/Outgoing: Filtering encrypted mail\./ {
+  outgoing_encrypted_mail_count++
+  filtered_outgoing_mail_count++
 }
 
-counter unencrypted_mail_count
-/Filtering unencrypted mail\./ {
-  unencrypted_mail_count++
-  filtered_mail_count++
+counter outgoing_unencrypted_mail_count
+/Outgoing: Filtering unencrypted mail\./ {
+  outgoing_unencrypted_mail_count++
+  filtered_outgoing_mail_count++
 }
 
+
+counter filtered_incoming_mail_count
+
+counter incoming_encrypted_mail_count
+/Incoming: Filtering encrypted mail\./ {
+  incoming_encrypted_mail_count++
+  filtered_incoming_mail_count++
+}
+
+counter incoming_unencrypted_mail_count
+/Incoming: Filtering unencrypted mail\./ {
+  incoming_unencrypted_mail_count++
+  filtered_incoming_mail_count++
+}
+
+
 counter rejected_unencrypted_mail_count
-/Rejected unencrypted mail\./ {
+/Rejected unencrypted mail/ {
   rejected_unencrypted_mail_count++
 }

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -53,8 +53,8 @@ http {
 
 	ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
 	ssl_prefer_server_ciphers on;
-	ssl_certificate /var/lib/acme/live/{{ config.domain_name }}/fullchain;
-	ssl_certificate_key /var/lib/acme/live/{{ config.domain_name }}/privkey;
+	ssl_certificate /etc/ssl/certs/ssl-cert-snakeoil.pem;
+	ssl_certificate_key /etc/ssl/private/ssl-cert-snakeoil.key;
 
 	gzip on;
 

cmdeploy/src/cmdeploy/postfix/deployer.py

@@ -1,4 +1,4 @@
-from pyinfra.operations import apt, files, systemd
+from pyinfra.operations import apt, files, server, systemd
 
 from cmdeploy.basedeploy import Deployer, get_resource
 
@@ -52,6 +52,27 @@ class PostfixDeployer(Deployer):
         )
         need_restart |= header_cleanup.changed
 
+        lmtp_header_cleanup = files.put(
+            src=get_resource("postfix/lmtp_header_cleanup"),
+            dest="/etc/postfix/lmtp_header_cleanup",
+            user="root",
+            group="root",
+            mode="644",
+        )
+        need_restart |= lmtp_header_cleanup.changed
+        # Transport map that discards messages to nine.testrun.org
+        transport_map = files.put(
+            src=get_resource("postfix/transport"),
+            dest="/etc/postfix/transport",
+            user="root",
+            group="root",
+            mode="644",
+        )
+        need_restart |= transport_map.changed
+        if transport_map.changed:
+            server.shell(
+                commands=["postmap /etc/postfix/transport"],
+            )
         # Login map that 1:1 maps email address to login.
         login_map = files.put(
             src=get_resource("postfix/login_map"),
@@ -65,9 +86,17 @@ class PostfixDeployer(Deployer):
         restart_conf = files.put(
             name="postfix: restart automatically on failure",
             src=get_resource("service/10_restart.conf"),
-            dest="/etc/systemd/system/dovecot.service.d/10_restart.conf",
+            dest="/etc/systemd/system/postfix@.service.d/10_restart.conf",
         )
         self.daemon_reload = restart_conf.changed
+
+        # Validate postfix configuration before restart
+        if need_restart:
+            server.shell(
+                name="Validate postfix configuration",
+                # Extract stderr and quit with error if non-zero
+                commands=["""bash -c 'w=$(postconf 2>&1 >/dev/null); [[ -z "$w" ]] || { echo "$w"; false; }'"""],
+            )
         self.need_restart = need_restart
 
     def activate(self):

cmdeploy/src/cmdeploy/postfix/lmtp_header_cleanup

@@ -0,0 +1,2 @@
+/^DKIM-Signature:/            IGNORE
+/^Authentication-Results:/            IGNORE

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -15,12 +15,12 @@ readme_directory = no
 compatibility_level = 3.6
 
 # TLS parameters
-smtpd_tls_cert_file=/var/lib/acme/live/{{ config.mail_domain }}/fullchain
-smtpd_tls_key_file=/var/lib/acme/live/{{ config.mail_domain }}/privkey
+smtpd_tls_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem
+smtpd_tls_key_file=/etc/ssl/private/ssl-cert-snakeoil.key
 smtpd_tls_security_level=may
 
 smtp_tls_CApath=/etc/ssl/certs
-smtp_tls_security_level=verify
+smtp_tls_security_level=encrypt
 # Send SNI extension when connecting to other servers.
 # <https://www.postfix.org/postconf.5.html#smtp_tls_servername>
 smtp_tls_servername = hostname
@@ -54,14 +54,15 @@ smtpd_tls_exclude_ciphers = aNULL, RC4, MD5, DES
 tls_preempt_cipherlist = yes
 
 smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination
-myhostname = {{ config.mail_domain }}
 alias_maps = hash:/etc/aliases
 alias_database = hash:/etc/aliases
 
 # Postfix does not deliver mail for any domain by itself.
 # Primary domain is listed in `virtual_mailbox_domains` instead
 # and handed over to Dovecot.
-mydestination =
+mydestination = {{ config.mail_domain }}
+local_transport = lmtp:unix:private/dovecot-lmtp
+local_recipient_maps =
 
 relayhost = 
 mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128
@@ -75,13 +76,15 @@ inet_protocols = ipv4
 inet_protocols = all
 {% endif %}
 
-virtual_transport = lmtp:unix:private/dovecot-lmtp
-virtual_mailbox_domains = {{ config.mail_domain }}
+lmtp_header_checks = regexp:/etc/postfix/lmtp_header_cleanup
 
 mua_client_restrictions = permit_sasl_authenticated, reject
 mua_sender_restrictions = reject_sender_login_mismatch, permit_sasl_authenticated, reject
 mua_helo_restrictions = permit_mynetworks, reject_invalid_helo_hostname, reject_non_fqdn_helo_hostname, permit
 
+# Discard messages to nine.testrun.org
+transport_maps = hash:/etc/postfix/transport
+
 # 1:1 map MAIL FROM to SASL login name.
 smtpd_sender_login_maps = regexp:/etc/postfix/login_map
 

cmdeploy/src/cmdeploy/postfix/master.cf.j2

@@ -86,7 +86,7 @@ filter    unix -        n       n       -       -       lmtp
 # Local SMTP server for reinjecting incoming filtered mail 
 127.0.0.1:{{ config.postfix_reinject_port_incoming }} inet  n       -       n       -       100      smtpd
   -o syslog_name=postfix/reinject_incoming
-  -o smtpd_milters=unix:opendkim/opendkim.sock
+#  -o smtpd_milters=unix:opendkim/opendkim.sock
 
 # Cleanup `Received` headers for authenticated mail
 # to avoid leaking client IP.

cmdeploy/src/cmdeploy/postfix/transport

@@ -0,0 +1,2 @@
+nine.testrun.org discard:
+* :

cmdeploy/src/cmdeploy/run.py

@@ -14,8 +14,9 @@ def main():
         importlib.resources.files("cmdeploy").joinpath("../../../chatmail.ini"),
     )
     disable_mail = bool(os.environ.get("CHATMAIL_DISABLE_MAIL"))
+    website_only = bool(os.environ.get("CHATMAIL_WEBSITE_ONLY"))
 
-    deploy_chatmail(config_path, disable_mail)
+    deploy_chatmail(config_path, disable_mail, website_only)
 
 
 if pyinfra.is_cli:

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -189,17 +189,23 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
     mail = maildata(
         "encrypted.eml", from_addr=user1.addr, to_addr=user2.addr
     ).as_string()
-    for i in range(chatmail_config.max_user_send_per_minute + 5):
+
+    timestamps = []
+    i = 0
+    while len(timestamps) <= chatmail_config.max_user_send_per_minute * 1.7:
         print("Sending mail", str(i))
+        i += 1
         try:
             user1.smtp.sendmail(user1.addr, [user2.addr], mail)
+            timestamps.append(time.time())
         except smtplib.SMTPException as e:
-            if i < chatmail_config.max_user_send_per_minute:
+            if len(timestamps) < chatmail_config.max_user_send_per_minute:
                 pytest.fail(f"rate limit was exceeded too early with msg {i}")
             outcome = e.recipients[user2.addr]
             assert outcome[0] == 450
             assert b"4.7.1: Too much mail from" in outcome[1]
             return
+        timestamps[:] = [ts for ts in timestamps if ts >= (time.time() - 60)]
     pytest.fail("Rate limit was not exceeded")
 
 

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -17,6 +17,7 @@ def imap_mailbox(cmfactory):
     password = ac1.get_config("mail_pw")
     mailbox = imap_tools.MailBox(user.split("@")[1])
     mailbox.login(user, password)
+    mailbox.dc_ac = ac1
     return mailbox
 
 
@@ -121,6 +122,28 @@ class TestEndToEndDeltaChat:
         assert ch.id >= 10
         ac1._evtracker.wait_securejoin_inviter_progress(1000)
 
+    def test_dkim_header_stripped(self, cmfactory, maildomain2, lp, imap_mailbox):
+        """Test that if a DC address receives a message, it has no
+        DKIM-Signature and Authentication-Results headers."""
+        ac1 = cmfactory.new_online_configuring_account(cache=False)
+        cmfactory.switch_maildomain(maildomain2)
+        ac2 = cmfactory.new_online_configuring_account(cache=False)
+        cmfactory.bring_accounts_online()
+        chat = cmfactory.get_accepted_chat(ac1, imap_mailbox.dc_ac)
+        chat.send_text("message0")
+        chat2 = cmfactory.get_accepted_chat(ac2, imap_mailbox.dc_ac)
+        chat2.send_text("message1")
+
+        lp.sec("receive message with ac1...")
+        received = 0
+        while received < 2:
+            msgs = imap_mailbox.fetch()
+            for msg in msgs:
+                lp.sec(f"ac1 received msg from {msg.from_}")
+                received += 1
+                assert "authentication-results" not in msg.headers
+                assert "dkim-signature" not in msg.headers
+
     def test_read_receipts_between_instances(self, cmfactory, lp, maildomain2):
         ac1 = cmfactory.new_online_configuring_account(cache=False)
         cmfactory.switch_maildomain(maildomain2)