Switch from BLF-CRYPT to SHA512-CRYPT

Add time as a prefix.

README.md

@@ -4,7 +4,60 @@ This package deploys Postfix and Dovecot servers, including OpenDKIM for DKIM si
 
 Postfix uses Dovecot for authentication as described in <https://www.postfix.org/SASL_README.html#server_dovecot>
 
-## Ports
+## Getting started 
+
+prepare:
+
+    pip install -e chatmail-infra
+
+
+then run with pyinfra command line tool: 
+
+    CHATMAIL_DOMAIN=c1.testrun.org pyinfra --ssh-user root c1.testrun.org deploy.py
+
+
+## Structure (wip)
+```
+
+# package doveauth tool and deploy chatmail server to a envvar-specified ssh-reachable host 
+deploy.py 
+
+# chatmail pyinfra deploy package 
+chatmail-pyinfra 
+    pyproject.toml
+    chatmail/__init__ ...
+
+# doveauth tool used by dovecot's auth mechanism on the host system 
+doveauth
+    README.md
+    pyproject.toml
+    doveauth.py
+    test_doveauth.py
+
+# lmtp server to block (outgoing) unencrypted messages 
+filtermail 
+    README.md
+    pyproject.toml
+    .... 
+
+# online tests (after deploy)
+
+online-tests  # runnable via pytest 
+
+
+
+# scripts for setup/development/deployment 
+
+scripts/
+    init.sh  # create venv/other perequires
+    deploy.sh  # run pyinfra based deploy of everything
+    test.sh # run all local and online tests 
+
+```
+
+## Dovecot/Postfix configuration
+
+### Ports
 
 Postfix listens on ports 25 (smtp) and 587 (submission) and 465 (submissions).
 Dovecot listens on ports 143(imap) and 993 (imaps).
@@ -12,9 +65,3 @@ Dovecot listens on ports 143(imap) and 993 (imaps).
 ## DNS
 
 For DKIM you must add a DNS entry as in /etc/opendkim/selector.txt (where selector is the opendkim_selector configured in the chatmail inventory).
-
-## Run with pyinfra
-
-```
-CHATMAIL_DOMAIN=c1.testrun.org pyinfra c1.testrun.org deploy.py
-```

c1-inv.py

@@ -1,11 +0,0 @@
-
-chatmail = [
-    (
-        "c1.testrun.org",
-        {
-            "ssh_user": "root",
-            "domain": "c1.testrun.org",
-            "dkim_selector": "2023",
-        },
-    ),
-]

chatmaild/README.md

@@ -0,0 +1,5 @@
+# chatmaild
+
+chatmaild provides dovecot autentication
+to create dovecot users on login
+and mail filtering.

chatmaild/pyproject.toml

@@ -0,0 +1,35 @@
+[build-system]
+requires = ["setuptools>=45"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "chatmaild"
+version = "0.1"
+dependencies = [
+  "aiosmtpd"
+]
+
+[project.scripts]
+doveauth = "chatmaild.doveauth:main"
+doveauth-dictproxy = "chatmaild.dictproxy:main"
+filtermail = "chatmaild.filtermail:main"
+
+[tool.pytest.ini_options]
+addopts = "-v -ra --strict-markers"
+
+[tool.tox]
+legacy_tox_ini = """
+[tox]
+isolated_build = true
+envlist = lint
+
+[testenv:lint]
+skipdist = True
+skip_install = True
+deps =
+  ruff
+  black
+commands =
+  black --quiet --check --diff src/
+  ruff src/
+"""

chatmaild/src/chatmaild/database.py

@@ -0,0 +1,140 @@
+import sqlite3
+import contextlib
+import time
+from pathlib import Path
+
+
+class DBError(Exception):
+    """error during an operation on the database."""
+
+
+class Connection:
+    def __init__(self, sqlconn, write):
+        self._sqlconn = sqlconn
+        self._write = write
+
+    def close(self):
+        self._sqlconn.close()
+
+    def commit(self):
+        self._sqlconn.commit()
+
+    def rollback(self):
+        self._sqlconn.rollback()
+
+    def execute(self, query, params=()):
+        cur = self.cursor()
+        try:
+            cur.execute(query, params)
+        except sqlite3.IntegrityError as e:
+            raise DBError(e)
+        return cur
+
+    def cursor(self):
+        return self._sqlconn.cursor()
+
+    def create_user(self, addr: str, password: str):
+        """Create a row in the users table."""
+        self.execute("PRAGMA foreign_keys=on")
+        q = """INSERT INTO users (addr, password, last_login)
+               VALUES (?, ?, ?)"""
+        self.execute(q, (addr, password, int(time.time())))
+
+    def get_user(self, addr: str) -> {}:
+        """Get a row from the users table."""
+        q = "SELECT addr, password, last_login from users WHERE addr = ?"
+        row = self._sqlconn.execute(q, (addr,)).fetchone()
+        result = {}
+        if row:
+            result = dict(
+                user=row[0],
+                password=row[1],
+                last_login=row[2],
+            )
+        return result
+
+
+class Database:
+    def __init__(self, path: str):
+        self.path = Path(path)
+        self.ensure_tables()
+
+    def _get_connection(
+        self, write=False, transaction=False, closing=False
+    ) -> Connection:
+        # we let the database serialize all writers at connection time
+        # to play it very safe (we don't have massive amounts of writes).
+        mode = "ro"
+        if write:
+            mode = "rw"
+        if not self.path.exists():
+            mode = "rwc"
+        uri = "file:%s?mode=%s" % (self.path, mode)
+        sqlconn = sqlite3.connect(
+            uri,
+            timeout=60,
+            isolation_level=None if transaction else "DEFERRED",
+            uri=True,
+        )
+
+        # Enable Write-Ahead Logging to avoid readers blocking writers and vice versa.
+        if write:
+            sqlconn.execute("PRAGMA journal_mode=wal")
+
+        if transaction:
+            start_time = time.time()
+            while 1:
+                try:
+                    sqlconn.execute("begin immediate")
+                    break
+                except sqlite3.OperationalError:
+                    # another thread may be writing, give it a chance to finish
+                    time.sleep(0.1)
+                    if time.time() - start_time > 5:
+                        # if it takes this long, something is wrong
+                        raise
+        conn = Connection(sqlconn, write=write)
+        if closing:
+            conn = contextlib.closing(conn)
+        return conn
+
+    @contextlib.contextmanager
+    def write_transaction(self):
+        conn = self._get_connection(closing=False, write=True, transaction=True)
+        try:
+            yield conn
+        except Exception:
+            conn.rollback()
+            conn.close()
+            raise
+        else:
+            conn.commit()
+            conn.close()
+
+    def read_connection(self, closing=True) -> Connection:
+        return self._get_connection(closing=closing, write=False)
+
+    def get_schema_version(self) -> int:
+        with self.read_connection() as conn:
+            dbversion = conn.execute("PRAGMA user_version").fetchone()[0]
+        return dbversion
+
+    CURRENT_DBVERSION = 1
+
+    def ensure_tables(self):
+        with self.write_transaction() as conn:
+            if self.get_schema_version() > 1:
+                raise DBError(
+                    "version is %s; downgrading schema is not supported"
+                    % (self.get_schema_version(),)
+                )
+            conn.execute(
+                """
+                CREATE TABLE IF NOT EXISTS users (
+                    addr TEXT PRIMARY KEY,
+                    password TEXT,
+                    last_login INTEGER
+                )
+            """,
+            )
+            conn.execute("PRAGMA user_version=%s" % (self.CURRENT_DBVERSION,))

chatmaild/src/chatmaild/dictproxy.py

@@ -0,0 +1,115 @@
+import os
+import sys
+import json
+from socketserver import (
+    UnixStreamServer,
+    StreamRequestHandler,
+    ThreadingMixIn,
+)
+import pwd
+import subprocess
+
+from .database import Database
+
+
+def encrypt_password(password: str):
+    password = password.encode("ascii")
+    # https://doc.dovecot.org/configuration_manual/authentication/password_schemes/
+    process = subprocess.Popen(
+        ["doveadm", "pw", "-s", "SHA512-CRYPT"],
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+    )
+    stdout_data, _stderr_data = process.communicate(
+        input=password + b"\n" + password + b"\n"
+    )
+    return stdout_data.decode("ascii").strip()
+
+
+def create_user(db, user, password):
+    with db.write_transaction() as conn:
+        conn.create_user(user, password)
+    return dict(home=f"/home/vmail/{user}", uid="vmail", gid="vmail", password=password)
+
+
+def get_user_data(db, user):
+    with db.read_connection() as conn:
+        result = conn.get_user(user)
+    if result:
+        result["uid"] = "vmail"
+        result["gid"] = "vmail"
+    return result
+
+
+def lookup_userdb(db, user):
+    return get_user_data(db, user)
+
+
+def lookup_passdb(db, user, password):
+    userdata = get_user_data(db, user)
+    if not userdata:
+        return create_user(db, user, encrypt_password(password))
+    userdata["password"] = userdata["password"].strip()
+    return userdata
+
+
+def handle_dovecot_request(msg, db):
+    print(f"received msg: {msg!r}", file=sys.stderr)
+    short_command = msg[0]
+    if short_command == "L":  # LOOKUP
+        parts = msg[1:].split("\t")
+        keyname, user = parts[:2]
+        namespace, type, *args = keyname.split("/")
+        reply_command = "F"
+        res = ""
+        if namespace == "shared":
+            if type == "userdb":
+                res = lookup_userdb(db, user)
+                if res:
+                    reply_command = "O"
+                else:
+                    reply_command = "N"
+            elif type == "passdb":
+                res = lookup_passdb(db, user, password=args[0])
+                if res:
+                    reply_command = "O"
+                else:
+                    reply_command = "N"
+        print(f"res: {res!r}", file=sys.stderr)
+        json_res = json.dumps(res) if res else ""
+        return f"{reply_command}{json_res}\n"
+    return None
+
+
+class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
+    pass
+
+
+def main():
+    socket = sys.argv[1]
+    passwd_entry = pwd.getpwnam(sys.argv[2])
+    db = Database(sys.argv[3])
+
+    class Handler(StreamRequestHandler):
+        def handle(self):
+            while True:
+                msg = self.rfile.readline().strip().decode()
+                if not msg:
+                    continue
+                res = handle_dovecot_request(msg, db)
+                if res:
+                    print(f"sending result: {res!r}", file=sys.stderr)
+                    self.wfile.write(res.encode("ascii"))
+                    self.wfile.flush()
+
+    try:
+        os.unlink(socket)
+    except FileNotFoundError:
+        pass
+
+    with ThreadedUnixStreamServer(socket, Handler) as server:
+        os.chown(socket, uid=passwd_entry.pw_uid, gid=passwd_entry.pw_gid)
+        try:
+            server.serve_forever()
+        except KeyboardInterrupt:
+            pass

chatmaild/src/chatmaild/doveauth-dictproxy.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=Dict authentication proxy for dovecot
+
+[Service]
+ExecStart=/usr/local/bin/doveauth-dictproxy /run/dovecot/doveauth.socket vmail /home/vmail/passdb.sqlite
+Restart=always
+RestartSec=30
+
+[Install]
+WantedBy=multi-user.target

chatmaild/src/chatmaild/doveauth.py

@@ -2,37 +2,40 @@
 import base64
 import sys
 
-
-def get_user_data(user):
-    if user == "link2xt@c1.testrun.org":
-        return dict(
-            uid="vmail",
-            gid="vmail",
-            password="Ahyei6ie",
-        )
-    return {}
+from .database import Database
 
 
-def create_user(user, password):
+def get_user_data(db, user):
+    with db.read_connection() as conn:
+        result = conn.get_user(user)
+    if result:
+        result["uid"] = "vmail"
+        result["gid"] = "vmail"
+    return result
+
+
+def create_user(db, user, password):
+    with db.write_transaction() as conn:
+        conn.create_user(user, password)
     return dict(home=f"/home/vmail/{user}", uid="vmail", gid="vmail", password=password)
 
 
-def verify_user(user, password):
-    userdata = get_user_data(user)
+def verify_user(db, user, password):
+    userdata = get_user_data(db, user)
     if userdata:
         if userdata.get("password") == password:
             userdata["status"] = "ok"
         else:
             userdata["status"] = "fail"
     else:
-        userdata = create_user(user, password)
+        userdata = create_user(db, user, password)
         userdata["status"] = "ok"
 
     return userdata
 
 
-def lookup_user(user):
-    userdata = get_user_data(user)
+def lookup_user(db, user):
+    userdata = get_user_data(db, user)
     if userdata:
         userdata["status"] = "ok"
     else:
@@ -45,13 +48,18 @@ def dump_result(res):
         print(f"{key}={value}")
 
 
-if __name__ == "__main__":
+def main():
+    db = Database("/home/vmail/passdb.sqlite")
     if sys.argv[1] == "hexauth":
         login = base64.b16decode(sys.argv[2]).decode()
         password = base64.b16decode(sys.argv[3]).decode()
-        res = verify_user(login, password)
+        res = verify_user(db, login, password)
         dump_result(res)
     elif sys.argv[1] == "hexlookup":
         login = base64.b16decode(sys.argv[2]).decode()
-        res = lookup_user(login)
+        res = lookup_user(db, login)
         dump_result(res)
+
+
+if __name__ == "__main__":
+    main()

chatmaild/src/chatmaild/filtermail.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+import asyncio
+import logging
+from email.parser import BytesParser
+from email import policy
+
+from aiosmtpd.lmtp import LMTP
+from aiosmtpd.controller import UnixSocketController
+from smtplib import SMTP as SMTPClient
+
+
+def check_encrypted(content):
+    """Check that the message is an OpenPGP-encrypted message."""
+    message = BytesParser(policy=policy.default).parsebytes(content)
+    if not message.is_multipart():
+        return False
+    if message.get("subject") != "...":
+        return False
+    if message.get_content_type() != "multipart/encrypted":
+        return False
+    parts_count = 0
+    for part in message.iter_parts():
+        if parts_count == 0:
+            if part.get_content_type() != "application/pgp-encrypted":
+                return False
+        elif parts_count == 1:
+            if part.get_content_type() != "application/octet-stream":
+                return False
+        else:
+            return False
+        parts_count += 1
+    return True
+
+
+class ExampleController(UnixSocketController):
+    def factory(self):
+        return LMTP(self.handler, **self.SMTP_kwargs)
+
+
+class ExampleHandler:
+    async def handle_RCPT(self, server, session, envelope, address, rcpt_options):
+        envelope.rcpt_tos.append(address)
+        return "250 OK"
+
+    async def handle_DATA(self, server, session, envelope):
+        logging.info("Processing DATA message from %s", envelope.mail_from)
+
+        valid_recipients = []
+
+        mail_encrypted = check_encrypted(envelope.content)
+
+        res = []
+        for recipient in envelope.rcpt_tos:
+            my_local_domain = envelope.mail_from.split("@")
+            if len(my_local_domain) != 2:
+                res += [f"500 Invalid from address <{envelope.mail_from}>"]
+                continue
+
+            if envelope.mail_from == recipient:
+                # Always allow sending emails to self.
+                valid_recipients += [recipient]
+                res += ["250 OK"]
+                continue
+
+            recipient_local_domain = recipient.split("@")
+            if len(recipient_local_domain) != 2:
+                res += [f"500 Invalid address <{recipient}>"]
+                continue
+
+            is_outgoing = recipient_local_domain[1] != my_local_domain[1]
+            if is_outgoing and not mail_encrypted:
+                res += ["500 Outgoing mail must be encrypted"]
+                continue
+
+            valid_recipients += [recipient]
+            res += ["250 OK"]
+
+        # Reinject the mail back into Postfix.
+        if valid_recipients:
+            logging.info("Reinjecting the mail")
+            client = SMTPClient("localhost", "10026")
+            client.sendmail(envelope.mail_from, valid_recipients, envelope.content)
+
+        return "\r\n".join(res)
+
+
+async def asyncmain(loop):
+    controller = ExampleController(
+        ExampleHandler(), unix_socket="/var/spool/postfix/private/filtermail"
+    )
+    controller.start()
+
+
+def main():
+    logging.basicConfig(level=logging.INFO)
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+    loop.create_task(asyncmain(loop=loop))
+    loop.run_forever()
+
+
+if __name__ == "__main__":
+    main()

chatmaild/src/chatmaild/filtermail.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=Email filter for chatmail servers
+
+[Service]
+ExecStart=/usr/local/bin/filtermail
+Restart=always
+RestartSec=30
+
+[Install]
+WantedBy=multi-user.target

chatmaild/src/chatmaild/test_doveauth.py

@@ -0,0 +1,36 @@
+import pytest
+
+from .dictproxy import get_user_data
+from .doveauth import verify_user
+from .database import Database, DBError
+
+
+@pytest.fixture()
+def db(tmpdir):
+    db_path = tmpdir / "passdb.sqlite"
+    print("database path:", db_path)
+    return Database(db_path)
+
+
+def test_basic(db):
+    verify_user(db, "link2xt@c1.testrun.org", "asdf")
+    data = get_user_data(db, "link2xt@c1.testrun.org")
+    assert data
+
+
+def test_verify_or_create(db):
+    res = verify_user(db, "newuser1@something.org", "kajdlkajsldk12l3kj1983")
+    assert res["status"] == "ok"
+    res = verify_user(db, "newuser1@something.org", "kajdlqweqwe")
+    assert res["status"] == "fail"
+
+
+def test_db_version(db):
+    assert db.get_schema_version() == 1
+
+
+def test_too_high_db_version(db):
+    with db.write_transaction() as conn:
+        conn.execute("PRAGMA user_version=%s;" % (999,))
+    with pytest.raises(DBError):
+        db.ensure_tables()

chatmaild/src/chatmaild/test_filtermail.py

@@ -0,0 +1,286 @@
+import pytest
+
+from .filtermail import check_encrypted
+
+
+def test_filtermail():
+    assert not check_encrypted(b"foo")
+
+    assert not check_encrypted(
+        "\r\n".join(
+            [
+                "Subject: =?utf-8?q?Message_from_foobar=40c2=2Etestrun=2Eorg?=",
+                "Chat-Disposition-Notification-To: foobar@c2.testrun.org",
+                "Chat-User-Avatar: 0",
+                "From: <foobar@c2.testrun.org>",
+                "To: <barbaz@c2.testrun.org>",
+                "Date: Sun, 15 Oct 2023 16:41:44 +0000",
+                "Message-ID: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>",
+                "References: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>",
+                "Chat-Version: 1.0",
+                "Autocrypt: addr=foobar@c2.testrun.org; prefer-encrypt=mutual;",
+                "\tkeydata=xjMEZSrw3hYJKwYBBAHaRw8BAQdAiEKNQFU28c6qsx4vo/JHdt73RXdjMOmByf/XsGiJ7m",
+                "\tnNFzxmb29iYXJAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUq8N4CGwMECwkIBwYVCAkKCwID",
+                "\tFgIBFiEEGil0OvTIa6RngmCLUYNnEa9leJAACgkQUYNnEa9leJCX3gEAhm0MehE5byBBU1avPczr/I",
+                "\tHjNLht7Qf6++mAhlJmtDcA/0C8VYJhsUpmiDjuZaMDWNv4FO2BJG6LH7gSm6n7ClMJzjgEZSrw3hIK",
+                "\tKwYBBAGXVQEFAQEHQAxGG/QW0owCfMp1A+vXEMwgzWcBpNFr58kX2eXuPpM6AwEIB8J4BBgWCAAgBQ",
+                "\tJlKvDeAhsMFiEEGil0OvTIa6RngmCLUYNnEa9leJAACgkQUYNnEa9leJDg1gEAwLf8KDoAAKyYgjyI",
+                "\tvYvO9VEgBni1C4Xx1VjcaEmlDK8BALoFuUCK+enw76TtDcAUKhlhUiM6SDRExkS4Nskp/BcK",
+                "MIME-Version: 1.0",
+                "Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no",
+                "",
+                "Hi!",
+                "",
+                "",
+            ]
+        ).encode()
+    )
+
+    assert not check_encrypted(
+        "\r\n".join(
+            [
+                "Subject: =?utf-8?q?Message_from_foobar=40c2=2Etestrun=2Eorg?=",
+                "Chat-Disposition-Notification-To: foobar@c2.testrun.org",
+                "Chat-User-Avatar: 0",
+                "From: <foobar@c2.testrun.org>",
+                "To: <barbaz@c2.testrun.org>",
+                "Date: Sun, 15 Oct 2023 16:41:44 +0000",
+                "Message-ID: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>",
+                "References: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>",
+                "Chat-Version: 1.0",
+                "Autocrypt: addr=foobar@c2.testrun.org; prefer-encrypt=mutual;",
+                "\tkeydata=xjMEZSrw3hYJKwYBBAHaRw8BAQdAiEKNQFU28c6qsx4vo/JHdt73RXdjMOmByf/XsGiJ7m",
+                "\tnNFzxmb29iYXJAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUq8N4CGwMECwkIBwYVCAkKCwID",
+                "\tFgIBFiEEGil0OvTIa6RngmCLUYNnEa9leJAACgkQUYNnEa9leJCX3gEAhm0MehE5byBBU1avPczr/I",
+                "\tHjNLht7Qf6++mAhlJmtDcA/0C8VYJhsUpmiDjuZaMDWNv4FO2BJG6LH7gSm6n7ClMJzjgEZSrw3hIK",
+                "\tKwYBBAGXVQEFAQEHQAxGG/QW0owCfMp1A+vXEMwgzWcBpNFr58kX2eXuPpM6AwEIB8J4BBgWCAAgBQ",
+                "\tJlKvDeAhsMFiEEGil0OvTIa6RngmCLUYNnEa9leJAACgkQUYNnEa9leJDg1gEAwLf8KDoAAKyYgjyI",
+                "\tvYvO9VEgBni1C4Xx1VjcaEmlDK8BALoFuUCK+enw76TtDcAUKhlhUiM6SDRExkS4Nskp/BcK",
+                "MIME-Version: 1.0",
+                "Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no",
+                "",
+                "Hi!",
+                "",
+                "",
+            ]
+        ).encode()
+    )
+
+    # https://xkcd.com/1181/
+    assert not check_encrypted(
+        "\r\n".join(
+            [
+                "Subject: =?utf-8?q?Message_from_foobar=40c2=2Etestrun=2Eorg?=",
+                "Chat-Disposition-Notification-To: foobar@c2.testrun.org",
+                "Chat-User-Avatar: 0",
+                "From: <foobar@c2.testrun.org>",
+                "To: <barbaz@c2.testrun.org>",
+                "Date: Sun, 15 Oct 2023 16:41:44 +0000",
+                "Message-ID: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>",
+                "References: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>",
+                "Chat-Version: 1.0",
+                "Autocrypt: addr=foobar@c2.testrun.org; prefer-encrypt=mutual;",
+                "\tkeydata=xjMEZSrw3hYJKwYBBAHaRw8BAQdAiEKNQFU28c6qsx4vo/JHdt73RXdjMOmByf/XsGiJ7m",
+                "\tnNFzxmb29iYXJAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUq8N4CGwMECwkIBwYVCAkKCwID",
+                "\tFgIBFiEEGil0OvTIa6RngmCLUYNnEa9leJAACgkQUYNnEa9leJCX3gEAhm0MehE5byBBU1avPczr/I",
+                "\tHjNLht7Qf6++mAhlJmtDcA/0C8VYJhsUpmiDjuZaMDWNv4FO2BJG6LH7gSm6n7ClMJzjgEZSrw3hIK",
+                "\tKwYBBAGXVQEFAQEHQAxGG/QW0owCfMp1A+vXEMwgzWcBpNFr58kX2eXuPpM6AwEIB8J4BBgWCAAgBQ",
+                "\tJlKvDeAhsMFiEEGil0OvTIa6RngmCLUYNnEa9leJAACgkQUYNnEa9leJDg1gEAwLf8KDoAAKyYgjyI",
+                "\tvYvO9VEgBni1C4Xx1VjcaEmlDK8BALoFuUCK+enw76TtDcAUKhlhUiM6SDRExkS4Nskp/BcK",
+                "MIME-Version: 1.0",
+                "Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no",
+                "",
+                "-----BEGIN PGP MESSAGE-----",
+                "Hi!",
+                "-----END PGP MESSAGE-----",
+                "",
+                "",
+            ]
+        ).encode()
+    )
+
+    assert check_encrypted(
+        "\r\n".join(
+            [
+                "Subject: ...",
+                "From: <barbaz@c2.testrun.org>",
+                "To: <foobar@c2.testrun.org>",
+                "Date: Sun, 15 Oct 2023 16:43:21 +0000",
+                "Message-ID: <Mr.UVyJWZmkCKM.hGzNc6glBE_@c2.testrun.org>",
+                "In-Reply-To: <Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>",
+                "References: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>",
+                "\t<Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>",
+                "Chat-Version: 1.0",
+                "Autocrypt: addr=barbaz@c2.testrun.org; prefer-encrypt=mutual;",
+                "\tkeydata=xjMEZSwWjhYJKwYBBAHaRw8BAQdAQBEhqeJh0GueHB6kF/DUQqYCxARNBVokg/AzT+7LqH",
+                "\trNFzxiYXJiYXpAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUsFo4CGwMECwkIBwYVCAkKCwID",
+                "\tFgIBFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX9A4AEAnHWHp49eBCMHK5t66gYPiW",
+                "\tXQuB1mwUjzGfYWB+0RXUoA/0xcQ3FbUNlGKW7Blp6eMFfViv6Mv2d3kNSXACB6nmcMzjgEZSwWjhIK",
+                "\tKwYBBAGXVQEFAQEHQBpY5L2M1XHo0uxf8SX1wNLBp/OVvidoWHQF2Jz+kJsUAwEIB8J4BBgWCAAgBQ",
+                "\tJlLBaOAhsMFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX/INgEA37AJaNvruYsJVanP",
+                "\tIXnYw4CKd55UAwl8Zcy+M2diAbkA/0fHHcGV4r78hpbbL1Os52DPOdqYQRauIeJUeG+G6bQO",
+                "MIME-Version: 1.0",
+                'Content-Type: multipart/encrypted; protocol="application/pgp-encrypted";',
+                '\tboundary="YFrteb74qSXmggbOxZL9dRnhymywAi"',
+                "",
+                "",
+                "--YFrteb74qSXmggbOxZL9dRnhymywAi",
+                "Content-Description: PGP/MIME version identification",
+                "Content-Type: application/pgp-encrypted",
+                "",
+                "Version: 1",
+                "",
+                "",
+                "--YFrteb74qSXmggbOxZL9dRnhymywAi",
+                "Content-Description: OpenPGP encrypted message",
+                'Content-Disposition: inline; filename="encrypted.asc";',
+                'Content-Type: application/octet-stream; name="encrypted.asc"',
+                "",
+                "-----BEGIN PGP MESSAGE-----",
+                "",
+                "wU4DhW3gBZ/VvCYSAQdA8bMs2spwbKdGjVsL1ByPkNrqD7frpB73maeL6I6SzDYg",
+                "O5G53tv339RdKq3WRcCtEEvxjHlUx2XNwXzC04BpmfvBTgNfPUyLDzjXnxIBB0Ae",
+                "8ymwGvXMCCimHXN0Dg8Ui62KOi03h0UgheoHWovJSCDF4CKre/xtFr3nL7lq/PKI",
+                "JsjVNz7/RK9FSXF6WwfONtLCyQGEuVAsB/KXfCBEyfKhaMwGHvhujRidGW5uV1no",
+                "lMGl3ODmo29Lgeu2uSE7EpJRZoe6hU6ddmBkqxax61ZtkaFlGFFpdo2K8balNNdz",
+                "ZsJ/9mmI9x3oOJ4/l1nhQbUO9ADbs7gJhFdV5Qkp30b5fCI7bU+aoe1ccBbLe/WM",
+                "YUty1PqcuQT7XjA+XmYuL261tvW8pBetT+i33/E2d8PzzYt2IuK9qeevyS+yxdwA",
+                "kfwejFWzzsUlJaDxs1x4XOxkMgSj+jo+g12dFOb7fyClsAnq23iDb8AuaT/BScAI",
+                "+lO+gher69+6LmM7VGHLG5k762J1jTaQCaKt1s8TAWV99Eo4491vL6fyvk3l/Cfg",
+                "RXSwiWFgj19Pn0Rq7CD9v22UE2vdUMBTcV4aw79mClk1YQ23jbF0y5DCjPdJ62Zo",
+                "tskBgFt3NoWV80jZ76zIBLrrjLwCCll8JjJtFwSkt2GX5RFBsVa4A8IDht9RtEk7",
+                "rrHgbSZQfkauEi/mH3/6CDZoLqSHudUZ7d4MaJwun1TkFYGe2ORwGJd4OBj3oGJp",
+                "H8YBwCpk///L/fKjX0Gg3M8nrpM4wrRFhPKidAgO/kcm25X4+ZHlVkWBTCt5RWKI",
+                "fHh6oLDZCqCfcgMkE1KKmwfIHaUkhq5BPRigwy6i5dh1DM4+1UCLh3dxzVbqE9b9",
+                "61NB19nXdRtDA2sOUnj9ve6m/wEPyCb6/zBQZqvCBYb1/AjdXpUrFT+DbpfyxaXN",
+                "XfhDVb5mNqNM/IVj0V5fvTc6vOfYbzQtPm10H+FdWWfb+rJRfyC3MA2w2IqstFe3",
+                "w3bu2iE6CQvSqRvge+ZqLKt/NqYwOURiUmpuklbl3kPJ97+mfKWoiqk8Iz1VY+bb",
+                "NMUC7aoGv+jcoj+WS6PYO8N6BeRVUUB3ZJSf8nzjgxm1/BcM+UD3BPrlhT11ODRs",
+                "baifGbprMWwt3dhb8cQgRT8GPdpO1OsDkzL6iikMjLHWWiA99GV6ruiHsIPw6boW",
+                "A6/uSOskbDHOROotKmddGTBd0iiHXAoQsJFt1ZjUkt6EHrgWs+GAvrvKpXs1mrz8",
+                "uj3GwEFrHS+Xuf2UDgpszYT3hI2cL/kUtGakVR7m7vVMZqXBUbZdGAEb1PZNPwsI",
+                "E4aMK02+EVB+tSN4Fzj99N2YD0inVYt+oPjr2tHhUS6aSGBNS/48Ki47DOg4Sxkn",
+                "lkOWnEbCD+XTnbDd",
+                "=agR5",
+                "-----END PGP MESSAGE-----",
+                "",
+                "",
+                "--YFrteb74qSXmggbOxZL9dRnhymywAi--",
+                "",
+                "",
+            ]
+        ).encode()
+    )
+
+    assert not check_encrypted(
+        "\r\n".join(
+            [
+                "Subject: Buy Penis Enlargement at www.malicious-domain.com",
+                "From: <barbaz@c2.testrun.org>",
+                "To: <foobar@c2.testrun.org>",
+                "Date: Sun, 15 Oct 2023 16:43:21 +0000",
+                "Message-ID: <Mr.UVyJWZmkCKM.hGzNc6glBE_@c2.testrun.org>",
+                "In-Reply-To: <Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>",
+                "References: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>",
+                "\t<Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>",
+                "Chat-Version: 1.0",
+                "Autocrypt: addr=barbaz@c2.testrun.org; prefer-encrypt=mutual;",
+                "\tkeydata=xjMEZSwWjhYJKwYBBAHaRw8BAQdAQBEhqeJh0GueHB6kF/DUQqYCxARNBVokg/AzT+7LqH",
+                "\trNFzxiYXJiYXpAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUsFo4CGwMECwkIBwYVCAkKCwID",
+                "\tFgIBFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX9A4AEAnHWHp49eBCMHK5t66gYPiW",
+                "\tXQuB1mwUjzGfYWB+0RXUoA/0xcQ3FbUNlGKW7Blp6eMFfViv6Mv2d3kNSXACB6nmcMzjgEZSwWjhIK",
+                "\tKwYBBAGXVQEFAQEHQBpY5L2M1XHo0uxf8SX1wNLBp/OVvidoWHQF2Jz+kJsUAwEIB8J4BBgWCAAgBQ",
+                "\tJlLBaOAhsMFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX/INgEA37AJaNvruYsJVanP",
+                "\tIXnYw4CKd55UAwl8Zcy+M2diAbkA/0fHHcGV4r78hpbbL1Os52DPOdqYQRauIeJUeG+G6bQO",
+                "MIME-Version: 1.0",
+                'Content-Type: multipart/encrypted; protocol="application/pgp-encrypted";',
+                '\tboundary="YFrteb74qSXmggbOxZL9dRnhymywAi"',
+                "",
+                "",
+                "--YFrteb74qSXmggbOxZL9dRnhymywAi",
+                "Content-Description: PGP/MIME version identification",
+                "Content-Type: application/pgp-encrypted",
+                "",
+                "Version: 1",
+                "",
+                "",
+                "--YFrteb74qSXmggbOxZL9dRnhymywAi",
+                "Content-Description: OpenPGP encrypted message",
+                'Content-Disposition: inline; filename="encrypted.asc";',
+                'Content-Type: application/octet-stream; name="encrypted.asc"',
+                "",
+                "-----BEGIN PGP MESSAGE-----",
+                "",
+                "wU4DhW3gBZ/VvCYSAQdA8bMs2spwbKdGjVsL1ByPkNrqD7frpB73maeL6I6SzDYg",
+                "O5G53tv339RdKq3WRcCtEEvxjHlUx2XNwXzC04BpmfvBTgNfPUyLDzjXnxIBB0Ae",
+                "8ymwGvXMCCimHXN0Dg8Ui62KOi03h0UgheoHWovJSCDF4CKre/xtFr3nL7lq/PKI",
+                "JsjVNz7/RK9FSXF6WwfONtLCyQGEuVAsB/KXfCBEyfKhaMwGHvhujRidGW5uV1no",
+                "lMGl3ODmo29Lgeu2uSE7EpJRZoe6hU6ddmBkqxax61ZtkaFlGFFpdo2K8balNNdz",
+                "ZsJ/9mmI9x3oOJ4/l1nhQbUO9ADbs7gJhFdV5Qkp30b5fCI7bU+aoe1ccBbLe/WM",
+                "YUty1PqcuQT7XjA+XmYuL261tvW8pBetT+i33/E2d8PzzYt2IuK9qeevyS+yxdwA",
+                "kfwejFWzzsUlJaDxs1x4XOxkMgSj+jo+g12dFOb7fyClsAnq23iDb8AuaT/BScAI",
+                "+lO+gher69+6LmM7VGHLG5k762J1jTaQCaKt1s8TAWV99Eo4491vL6fyvk3l/Cfg",
+                "RXSwiWFgj19Pn0Rq7CD9v22UE2vdUMBTcV4aw79mClk1YQ23jbF0y5DCjPdJ62Zo",
+                "tskBgFt3NoWV80jZ76zIBLrrjLwCCll8JjJtFwSkt2GX5RFBsVa4A8IDht9RtEk7",
+                "rrHgbSZQfkauEi/mH3/6CDZoLqSHudUZ7d4MaJwun1TkFYGe2ORwGJd4OBj3oGJp",
+                "H8YBwCpk///L/fKjX0Gg3M8nrpM4wrRFhPKidAgO/kcm25X4+ZHlVkWBTCt5RWKI",
+                "fHh6oLDZCqCfcgMkE1KKmwfIHaUkhq5BPRigwy6i5dh1DM4+1UCLh3dxzVbqE9b9",
+                "61NB19nXdRtDA2sOUnj9ve6m/wEPyCb6/zBQZqvCBYb1/AjdXpUrFT+DbpfyxaXN",
+                "XfhDVb5mNqNM/IVj0V5fvTc6vOfYbzQtPm10H+FdWWfb+rJRfyC3MA2w2IqstFe3",
+                "w3bu2iE6CQvSqRvge+ZqLKt/NqYwOURiUmpuklbl3kPJ97+mfKWoiqk8Iz1VY+bb",
+                "NMUC7aoGv+jcoj+WS6PYO8N6BeRVUUB3ZJSf8nzjgxm1/BcM+UD3BPrlhT11ODRs",
+                "baifGbprMWwt3dhb8cQgRT8GPdpO1OsDkzL6iikMjLHWWiA99GV6ruiHsIPw6boW",
+                "A6/uSOskbDHOROotKmddGTBd0iiHXAoQsJFt1ZjUkt6EHrgWs+GAvrvKpXs1mrz8",
+                "uj3GwEFrHS+Xuf2UDgpszYT3hI2cL/kUtGakVR7m7vVMZqXBUbZdGAEb1PZNPwsI",
+                "E4aMK02+EVB+tSN4Fzj99N2YD0inVYt+oPjr2tHhUS6aSGBNS/48Ki47DOg4Sxkn",
+                "lkOWnEbCD+XTnbDd",
+                "=agR5",
+                "-----END PGP MESSAGE-----",
+                "",
+                "",
+                "--YFrteb74qSXmggbOxZL9dRnhymywAi--",
+                "",
+                "",
+            ]
+        ).encode()
+    )
+
+    assert not check_encrypted(
+        "\r\n".join(
+            [
+                "Subject: Message opened",
+                "From: <barbaz@c2.testrun.org>",
+                "To: <foobar@c2.testrun.org>",
+                "Date: Sun, 15 Oct 2023 16:43:25 +0000",
+                "Message-ID: <Mr.78MWtlV7RAi.goCFzBhCYfy@c2.testrun.org>",
+                "Auto-Submitted: auto-replied",
+                "Chat-Version: 1.0",
+                "MIME-Version: 1.0",
+                "Content-Type: multipart/report; report-type=disposition-notification;",
+                '\tboundary="Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi"',
+                "",
+                "",
+                "--Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi",
+                "Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no",
+                "",
+                'The "Hi!" message you sent was displayed on the screen of the recipient.',
+                "",
+                "This is no guarantee the content was read.",
+                "",
+                "",
+                "--Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi",
+                "Content-Type: message/disposition-notification",
+                "",
+                "Reporting-UA: Delta Chat 1.124.1",
+                "Original-Recipient: rfc822;barbaz@c2.testrun.org",
+                "Final-Recipient: rfc822;barbaz@c2.testrun.org",
+                "Original-Message-ID: <Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>",
+                "Disposition: manual-action/MDN-sent-automatically; displayed",
+                "",
+                "",
+                "--Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi--",
+                "",
+                "",
+            ]
+        ).encode()
+    )

deploy-chatmail/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=45"]
 build-backend = "setuptools.build_meta"
 
 [project]
-name = "chatmail"
+name = "deploy-chatmail"
 version = "0.1"
 dependencies = [
   "pyinfra",

deploy-chatmail/src/deploy_chatmail/__init__.py

@@ -2,6 +2,7 @@
 Chat Mail pyinfra deploy.
 """
 import importlib.resources
+from pathlib import Path
 
 from pyinfra import host, logger
 from pyinfra.operations import apt, files, server, systemd, python
@@ -9,17 +10,67 @@ from pyinfra.facts.files import File
 from .acmetool import deploy_acmetool
 
 
-def _install_chatctl() -> None:
-    """Setup chatctl."""
-    files.put(
-        src=importlib.resources.files(__package__)
-        .joinpath("dovecot/doveauth.py")
-        .open("rb"),
-        dest="/home/vmail/chatctl",
-        user="vmail",
-        group="vmail",
-        mode="755",
+def _install_chatmaild() -> None:
+    chatmaild_filename = "chatmaild-0.1.tar.gz"
+    chatmaild_path = importlib.resources.files(__package__).joinpath(
+        f"../../../dist/{chatmaild_filename}"
     )
+    remote_path = f"/tmp/{chatmaild_filename}"
+    if Path(str(chatmaild_path)).exists():
+        files.put(
+            name="Upload chatmaild source package",
+            src=chatmaild_path.open("rb"),
+            dest=remote_path,
+        )
+
+        apt.packages(
+            name="apt install python3-aiosmtpd",
+            packages=["python3-aiosmtpd", "python3-pip"],
+        )
+
+        # --no-deps because aiosmtplib is installed with `apt`.
+        server.shell(
+            name="install chatmaild with pip",
+            commands=[f"pip install --break-system-packages {remote_path}"],
+        )
+
+        files.put(
+            name="upload doveauth-dictproxy.service",
+            src=importlib.resources.files("chatmaild")
+            .joinpath("doveauth-dictproxy.service")
+            .open("rb"),
+            dest="/etc/systemd/system/doveauth-dictproxy.service",
+            user="root",
+            group="root",
+            mode="644",
+        )
+        systemd.service(
+            name="Setup doveauth-dictproxy service",
+            service="doveauth-dictproxy.service",
+            running=True,
+            enabled=True,
+            restarted=True,
+            daemon_reload=True,
+        )
+
+        files.put(
+            name="upload filtermail.service",
+            src=importlib.resources.files("chatmaild")
+            .joinpath("filtermail.service")
+            .open("rb"),
+            dest="/etc/systemd/system/filtermail.service",
+            user="root",
+            group="root",
+            mode="644",
+        )
+        systemd.service(
+            name="Setup filtermail service",
+            service="filtermail.service",
+            running=True,
+            enabled=True,
+            restarted=True,
+            daemon_reload=True,
+        )
 
 
 def _configure_opendkim(domain: str, dkim_selector: str) -> bool:
@@ -100,16 +151,14 @@ def _configure_dovecot(mail_server: str) -> bool:
         config={"hostname": mail_server},
     )
     need_restart |= main_config.changed
-
-    # luarocks install http lpeg_patterns fifo
-    auth_script = files.put(
-        src=importlib.resources.files(__package__).joinpath("dovecot/doveauth.lua"),
-        dest="/etc/dovecot/doveauth.lua",
+    auth_config = files.put(
+        src=importlib.resources.files(__package__).joinpath("dovecot/auth.conf"),
+        dest="/etc/dovecot/auth.conf",
         user="root",
         group="root",
         mode="644",
     )
-    need_restart |= auth_script.changed
+    need_restart |= auth_config.changed
 
     return need_restart
 
@@ -122,7 +171,7 @@ def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> N
     :param dkim_selector:
     """
 
-    apt.update(name="apt update")
+    apt.update(name="apt update", cache_time=24 * 3600)
     server.group(name="Create vmail group", group="vmail", system=True)
     server.user(name="Create vmail user", user="vmail", group="vmail", system=True)
 
@@ -144,11 +193,7 @@ def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> N
 
     apt.packages(
         name="Install Dovecot",
-        packages=[
-            "dovecot-imapd",
-            "dovecot-lmtpd",
-            "dovecot-auth-lua",
-        ],
+        packages=["dovecot-imapd", "dovecot-lmtpd"],
     )
 
     apt.packages(
@@ -159,7 +204,7 @@ def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> N
         ],
     )
 
-    _install_chatctl()
+    _install_chatmaild()
     dovecot_need_restart = _configure_dovecot(mail_server)
     postfix_need_restart = _configure_postfix(mail_domain)
     opendkim_need_restart = _configure_opendkim(mail_domain, dkim_selector)

deploy-chatmail/src/deploy_chatmail/dovecot/auth.conf

@@ -0,0 +1,5 @@
+uri = proxy:/run/dovecot/doveauth.socket:auth
+iterate_disable = yes
+default_pass_scheme = plain
+password_key = passdb/%w/%u
+user_key = userdb/%u

deploy-chatmail/src/deploy_chatmail/dovecot/dovecot.conf.j2

@@ -8,17 +8,17 @@ auth_verbose = yes
 auth_debug = yes
 auth_debug_passwords = yes
 auth_verbose_passwords = plain
+auth_cache_size = 100M
 
 # Authentication for system users.
 passdb {
-  driver = lua
-  args = file=/etc/dovecot/doveauth.lua
+  driver = dict
+  args = /etc/dovecot/auth.conf
 }
 userdb {
-  driver = lua
-  args = file=/etc/dovecot/doveauth.lua
+  driver = dict
+  args = /etc/dovecot/auth.conf
 }
-
 ##
 ## Mailbox locations and namespaces
 ##
@@ -88,7 +88,7 @@ service auth {
 service auth-worker {
   # Default is root.
   # Drop privileges we don't need.
-  user = $default_internal_user
+  user = vmail
 }
 
 ssl = required

deploy-chatmail/src/deploy_chatmail/postfix/master.cf

@@ -28,6 +28,7 @@ submission inet n       -       y       -       -       smtpd
   -o smtpd_recipient_restrictions=
   -o smtpd_relay_restrictions=permit_sasl_authenticated,reject
   -o milter_macro_daemon_name=ORIGINATING
+  -o content_filter=filter:unix:private/filtemail
 smtps     inet  n       -       y       -       -       smtpd
   -o syslog_name=postfix/smtps
   -o smtpd_tls_wrappermode=yes
@@ -42,6 +43,7 @@ smtps     inet  n       -       y       -       -       smtpd
   -o smtpd_recipient_restrictions=
   -o smtpd_relay_restrictions=permit_sasl_authenticated,reject
   -o milter_macro_daemon_name=ORIGINATING
+  -o content_filter=filter:unix:private/filtermail
 #628       inet  n       -       y       -       -       qmqpd
 pickup    unix  n       -       y       60      1       pickup
 cleanup   unix  n       -       y       -       0       cleanup
@@ -70,3 +72,7 @@ lmtp      unix  -       -       y       -       -       lmtp
 anvil     unix  -       -       y       -       1       anvil
 scache    unix  -       -       y       -       1       scache
 postlog   unix-dgram n  -       n       -       1       postlogd
+filter    unix -        n       n       -       -       lmtp
+# Local SMTP server for reinjecting filered mail.
+localhost:10026 inet  n       -       n       -       10      smtpd
+        -o content_filter=

deploy.py

@@ -1,6 +1,6 @@
 import os
-from pyinfra import host, facts
-from chatmail import deploy_chatmail
+import pyinfra
+from deploy_chatmail import deploy_chatmail
 
 
 def main():
@@ -15,4 +15,5 @@ def main():
     deploy_chatmail(mail_domain, mail_server, dkim_selector)
 
 
-main()
+if pyinfra.is_cli:
+    main()

online-tests/conftest.py

@@ -0,0 +1,106 @@
+import os
+import io
+import imaplib
+import smtplib
+import itertools
+import pytest
+import time
+
+
+@pytest.fixture
+def maildomain():
+    return os.environ.get("CHATMAIL_DOMAIN", "c1.testrun.org")
+
+
+@pytest.fixture
+def imap(maildomain):
+    return ImapConn(maildomain)
+
+
+class ImapConn:
+    def __init__(self, host):
+        self.host = host
+
+    def connect(self):
+        print(f"imap-connect {self.host}")
+        self.conn = imaplib.IMAP4_SSL(self.host)
+
+    def login(self, user, password):
+        print(f"imap-login {user!r} {password!r}")
+        self.conn.login(user, password)
+
+
+@pytest.fixture
+def smtp(maildomain):
+    return SmtpConn(maildomain)
+
+
+class SmtpConn:
+    def __init__(self, host):
+        self.host = host
+
+    def connect(self):
+        print(f"smtp-connect {self.host}")
+        self.conn = smtplib.SMTP_SSL(self.host)
+
+    def login(self, user, password):
+        print(f"smtp-login {user!r} {password!r}")
+        self.conn.login(user, password)
+
+
+@pytest.fixture
+def gencreds(maildomain):
+    prefix = str(time.time())
+    count = itertools.count()
+
+    def gen():
+        while 1:
+            num = next(count)
+            yield f"user{prefix}_{num}@{maildomain}", f"password{prefix}_{num}"
+
+    return lambda: next(gen())
+
+
+#
+# Delta Chat testplugin re-use
+# use the cmfactory fixture to get chatmail instance accounts
+#
+
+
+class ChatmailTestProcess:
+    """Provider for chatmail instance accounts as used by deltachat.testplugin.acfactory"""
+
+    def __init__(self, pytestconfig, maildomain, gencreds):
+        self.pytestconfig = pytestconfig
+        self.maildomain = maildomain
+        self.gencreds = gencreds
+        self._addr2files = {}
+
+    def get_liveconfig_producer(self):
+        while 1:
+            user, password = self.gencreds()
+            config = {"addr": user, "mail_pw": password}
+            yield config
+
+    def cache_maybe_retrieve_configured_db_files(self, cache_addr, db_target_path):
+        pass
+
+    def cache_maybe_store_configured_db_files(self, acc):
+        pass
+
+
+@pytest.fixture
+def cmfactory(request, maildomain, gencreds, tmpdir, data):
+    # cloned from deltachat.testplugin.amfactory
+    pytest.importorskip("deltachat")
+    from deltachat.testplugin import ACFactory
+
+    testproc = ChatmailTestProcess(request.config, maildomain, gencreds)
+    am = ACFactory(request=request, tmpdir=tmpdir, testprocess=testproc, data=data)
+    yield am
+    if hasattr(request.node, "rep_call") and request.node.rep_call.failed:
+        if testprocess.pytestconfig.getoption("--extra-info"):
+            logfile = io.StringIO()
+            am.dump_imap_summary(logfile=logfile)
+            print(logfile.getvalue())
+            # request.node.add_report_section("call", "imap-server-state", s)

online-tests/pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+addopts = -vrsx

online-tests/test_0_login.py

@@ -0,0 +1,55 @@
+import pytest
+import imaplib
+import smtplib
+
+
+class TestDovecot:
+    def test_login_ok(self, imap, gencreds):
+        user, password = gencreds()
+        imap.connect()
+        imap.login(user, password)
+        # verify it works on another connection
+        imap.connect()
+        imap.login(user, password)
+
+    def test_login_same_password(self, imap, gencreds):
+        """Test two different users logging in with the same password.
+
+        This ensures that authentication process does not confuse the users
+        by using only the password hash as a key.
+        """
+        user1, password1 = gencreds()
+        user2, _password2 = gencreds()
+        imap.connect()
+        imap.login(user1, password1)
+        imap.connect()
+        imap.login(user2, password1)
+
+    def test_login_fail(self, imap, gencreds):
+        user, password = gencreds()
+        imap.connect()
+        imap.login(user, password)
+        imap.connect()
+        with pytest.raises(imaplib.IMAP4.error) as excinfo:
+            imap.login(user, password + "wrong")
+        assert "AUTHENTICATIONFAILED" in str(excinfo)
+
+
+class TestPostfix:
+    def test_login_ok(self, smtp, gencreds):
+        user, password = gencreds()
+        smtp.connect()
+        smtp.login(user, password)
+        # verify it works on another connection
+        smtp.connect()
+        smtp.login(user, password)
+
+    def test_login_fail(self, smtp, gencreds):
+        user, password = gencreds()
+        smtp.connect()
+        smtp.login(user, password)
+        smtp.connect()
+        with pytest.raises(smtplib.SMTPAuthenticationError) as excinfo:
+            smtp.login(user, password + "wrong")
+        assert excinfo.value.smtp_code == 535
+        assert "authentication failed" in str(excinfo)

online-tests/test_1_deltachat.py

@@ -0,0 +1,11 @@
+class TestMailSending:
+    def test_one_on_one(self, cmfactory, lp):
+        ac1, ac2 = cmfactory.get_online_accounts(2)
+        chat = cmfactory.get_accepted_chat(ac1, ac2)
+
+        lp.sec("ac1: prepare and send text message to ac2")
+        msg1 = chat.send_text("message0")
+
+        lp.sec("wait for ac2 to receive message")
+        msg2 = ac2._evtracker.wait_next_incoming_message()
+        assert msg2.text == "message0"

plan.txt

@@ -0,0 +1,74 @@
+# Chat-mail server development (up until Oct 18th)
+
+## Dovecot goals/steps
+
+2. (holger) per-user storage quota (adaptive)
+   a) define a static 100MB per-user quota
+
+3. automatic expiry of messages older than M days
+   - delete unconditionally messages older than 40 days
+
+4. limit: max-connections per account
+
+
+## Filtermail
+
+- (alex, Only allow (outgoing) mails if secure-join or autocrypt-pgp-encrypted format.
+  TODO: mime-parse mails and check/add tests
+
+
+## nami: send out rate limit / rspamd
+
+- basic outgoing send rate/limits (depending on "account-rating")
+  use rspamd in a minimal way, check support dkim-signing
+  (including an online test exceeding rate limit)
+
+
+## doveauth questions/futures
+
+- bcrypt-password scheme is slow: require long passwords, use faster hashing
+
+- define user-name and password policies, and implement them
+  (be very restrictive at the beginning, we can relax later)
+
+- password is part of the dictproxy-lookup key, is it safe to use auth-caching?
+
+
+## How to limit creation of accounts?
+
+attack: a 3-line bash script to fill the chatmail db with millions of unused accouts
+
+- make it computationally expensive (somehow try to except our tests from it)
+  1st pass instant onboarding: create userid + cheap password -- if it fails then
+  2nd pass instant onboarding: create userdid + comput. expensive password
+
+- probably also do firewall: limit number of new tcp-connections per IP address per duration
+
+
+## Open/deferred questions
+
+- automatic expiry of users that haven't logged in for N days
+  Is it neccessary?  If all messages are gone, does the existence of
+  an e-mail address bother anybody?
+
+
+## web page for chat-mail servers?
+
+- documentation for users, privacy policy etc.
+  (probably also with provider-messages ...)
+
+
+## online tests (first with plain python/pytest)
+
+- write tests for dovecot login (exists)
+- write tests for postfix logins (exists)
+- write A<>B send/receive tests (exists)
+
+
+## Delta Chat
+
+1. qr code that defines access to a chatmail instance (like mailadm but without http etc.)
+
+2. support for creating username/password and verifying login works
+
+

scripts/deploy.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+: ${CHATMAIL_DOMAIN:=c1.testrun.org}
+export CHATMAIL_DOMAIN
+
+venv/bin/python3 -m build -n --sdist chatmaild --outdir dist
+
+deploy-chatmail/venv/bin/pyinfra --ssh-user root "$CHATMAIL_DOMAIN" deploy.py
+
+rm -r dist/

scripts/get_imap_capabilities.py

@@ -0,0 +1,14 @@
+import os
+import time
+import imaplib
+
+domain = os.environ.get("CHATMAIL_DOMAIN", "c3.testrun.org")
+
+print("connecting")
+conn = imaplib.IMAP4_SSL(domain)
+print("logging in")
+conn.login(f"measure{time.time()}", "pass")
+status, res = conn.capability()
+for capa in sorted(res[0].decode().split()):
+    print(capa)
+

scripts/init.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+set -e
+python3 -m venv deploy-chatmail/venv
+deploy-chatmail/venv/bin/pip install pyinfra pytest
+deploy-chatmail/venv/bin/pip install -e deploy-chatmail
+deploy-chatmail/venv/bin/pip install -e chatmaild
+
+python3 -m venv chatmaild/venv
+chatmaild/venv/bin/pip install pytest
+chatmaild/venv/bin/pip install -e chatmaild
+
+python3 -m venv online-tests/venv
+online-tests/venv/bin/pip install pytest pytest-timeout pdbpp deltachat
+
+python3 -m venv venv
+venv/bin/pip install build
+venv/bin/pip install 'setuptools>=68'

scripts/measure_tls_and_logins.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+import os
+import time
+import imaplib
+
+domain = os.environ.get("CHATMAIL_DOMAIN", "c3.testrun.org")
+
+NUM_CONNECTIONS=10
+
+conns = []
+
+start = time.time()
+for i in range(NUM_CONNECTIONS):
+    print(f"opening connection {i} to {domain}")
+    conn = imaplib.IMAP4_SSL(domain)
+    conns.append(conn)
+
+tlsdone = time.time()
+duration = tlsdone-start
+print(f"{duration}: TLS connections opening TLS connections")
+
+for i, conn in enumerate(conns):
+    print(f"logging into connection {i}")
+    conn.login(f"measure{i}", "pass")
+
+logindone = time.time()
+duration = logindone - tlsdone
+print(f"{duration}: LOGINS done")

scripts/remote-deploy.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+set -e
+
+: ${CHATMAIL_DOMAIN:=c1.testrun.org}
+: ${CHATMAIL_SSH_HOST:=$CHATMAIL_DOMAIN}
+
+rsync -avz . "root@$CHATMAIL_SSH_HOST:/root/chatmail" --exclude='/.git' --filter="dir-merge,- .gitignore"
+ssh "root@$CHATMAIL_SSH_HOST" "cd /root/chatmail; apt install -y python3-venv; python3 -m venv venv; venv/bin/pip install pyinfra build; venv/bin/python3 -m build -n --sdist chatmaild --outdir dist; venv/bin/pip install -e ./deploy-chatmail -e ./chatmaild; export CHATMAIL_DOMAIN=$CHATMAIL_DOMAIN; venv/bin/pyinfra @local deploy.py"

scripts/test.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -e
+pushd chatmaild/src/chatmaild
+../../venv/bin/pytest
+popd
+
+online-tests/venv/bin/pytest online-tests/ -vrx --durations=5

src/chatmail/dovecot/doveauth.lua

@@ -1,57 +0,0 @@
-
--- Escape shell argument by hex encoding it and wrapping in quotes.
-function escape(data)
-   b16 = data:gsub(".", function(char) return string.format("%2X", char:byte()) end)
-   return ("'"..b16.."'")
-end
-
--- call out to python program to actually manage authentication for dovecot
-
-function chatctl_verify(user, password)
-    local cmd = "python3 /home/vmail/chatctl hexauth "..escape(user).." "..escape(password)
-    print("executing: "..cmd)
-    local handle = io.popen(cmd)
-    local result = handle:read("*a")
-    handle:close()
-    return split_chatctl(result)
-end
-
-function chatctl_lookup(user) 
-    assert(user)
-    local handle = io.popen("python3 /home/vmail/chatctl hexlookup "..escape(user))
-    local result = handle:read("*a")
-    handle:close()
-    return split_chatctl(result)
-end
-
-function get_extra_dovecot_output(res)
-    return {home=res.home, uid=res.uid, gid=res.gid}
-end
-
-
-function auth_password_verify(request, password)
-    local res = chatctl_verify(request.user, password)
-    -- request:log_error("auth_password_verify "..request.user.." "..password)
-    if res.status == "ok" then 
-        local extra = get_extra_dovecot_output(res)
-        return dovecot.auth.PASSDB_RESULT_OK, get_extra_dovecot_output(res)
-    end
-    return dovecot.auth.PASSDB_RESULT_PASSWORD_MISMATCH, ""
-end
-
-
-function auth_userdb_lookup(request)
-    local res = chatctl_lookup(request.user) 
-    if res.status == "ok" then
-        return dovecot.auth.USERDB_RESULT_OK, get_extra_dovecot_output(res)
-    end
-    return dovecot.auth.USERDB_RESULT_USER_UNKNOWN, "no such user"
-end
-
-function split_chatctl(output) 
-    local ret = {}
-    for key, value in output:gmatch "(%w+)%s*=%s*(%w+)" do
-        ret[key] = value
-    end
-    return ret
-end

src/chatmail/dovecot/test_doveauth.lua

@@ -1,78 +0,0 @@
-
-require "doveauth"
-
--- simulate dovecot defined result codes 
-
-dovecot = {
-    auth = {
-        PASSDB_RESULT_OK="PASSWORD-OK",
-        PASSDB_RESULT_PASSWORD_MISMATCH="PASSWORD-MISMATCH",
-        USERDB_RESULT_OK="USERDB-OK",
-        USERDB_RESULT_USER_UNKNOWN="USERDB-UNKNOWN"
-    }
-}
-
-
--- Tests for testing the lua<->python interaction 
-
-function test_password_verify_ok(user, password) 
-    local res, extra = auth_password_verify({user=user}, password)
-    assert(res==dovecot.auth.PASSDB_RESULT_OK)
-    assert(extra.uid == "vmail")
-    assert(extra.gid == "vmail")
-    -- assert(extra.homedir == "/home/vmail/link2xt")
-    print("OK test_password_verify_ok "..user.." "..password)
-end
-
-function test_password_verify_mismatch(user, password) 
-    local res = auth_password_verify({user=user}, password)
-    assert(res == dovecot.auth.PASSDB_RESULT_PASSWORD_MISMATCH)
-    print("OK test_password_verify_mismatch "..user.." "..password)
-end
-
-function test_userdb_lookup_ok(user)
-    local res, extra = auth_userdb_lookup({user=user})
-    assert(extra.uid == "vmail")
-    assert(extra.gid == "vmail")
-    assert(res == dovecot.auth.USERDB_RESULT_OK)
-    print("OK test_userdb_lookup_ok "..user)
-end
-
-function test_userdb_lookup_mismatch(user)
-    local res, extra = auth_userdb_lookup({user=user})
-    assert(res == dovecot.auth.USERDB_RESULT_USER_UNKNOWN)
-    print("OK test_userdb_lookup_mismatch "..user)
-end
-
-function test_passdb_lookup_ok(user)
-    local res, extra = auth_passdb_lookup({user=user})
-    assert(extra.uid == "vmail")
-    assert(extra.gid == "vmail")
-    assert(res == dovecot.auth.PASSDB_RESULT_OK)
-    print("OK test_passdb_lookup_ok "..user)
-end
-
-function test_passdb_lookup_mismatch(user)
-    local res, extra = auth_passdb_lookup({user=user})
-    assert(res == dovecot.auth.PASSDB_RESULT_USER_UNKNOWN)
-    print("OK test_passdb_lookup_mismatch "..user)
-end
-
-function test_split_chatctl()
-    local res = split_chatctl("a=3 b=4\nc=5")
-    assert(res["a"] == "3")
-    assert(res["b"] == "4")
-    assert(res["c"] == "5")
-    print("OK test_split_chatctl")
-end 
-
-test_split_chatctl()
-test_password_verify_ok("link2xt@c1.testrun.org", "Ahyei6ie")
-test_password_verify_mismatch("link2xt@c1.testrun.org", "Aqwlek")
-test_userdb_lookup_ok("link2xt@c1.testrun.org")
-test_userdb_lookup_mismatch("wlekqjlew@xyz.org")
-
--- probably not needed by dovecot? 
--- test_passdb_lookup_ok("link2xt@c1.testrun.org")
--- test_passdb_lookup_mismatch("llqkwjelqwe@xyz.org")
-

src/chatmail/dovecot/test_doveauth.py

@@ -1,23 +0,0 @@
-import subprocess
-import pytest
-
-from doveauth import get_user_data, verify_user
-
-
-def test_basic():
-    data = get_user_data("link2xt@c1.testrun.org")
-    assert data
-
-
-@pytest.mark.xfail(reason="no persistence yet")
-def test_verify_or_create():
-    res = verify_user("newuser1@something.org", "kajdlkajsldk12l3kj1983")
-    assert res["status"] == "ok"
-    res = verify_user("newuser1@something.org", "kajdlqweqwe")
-    assert res["status"] == "fail"
-
-
-def test_lua_integration(request):
-    p = request.fspath.dirpath("test_doveauth.lua")
-    proc = subprocess.run(["lua", str(p)])
-    assert proc.returncode == 0

tests/test_online_login.py

@@ -1,28 +0,0 @@
-import pytest
-import imaplib
-
-
-@pytest.fixture
-def conn():
-    return connect("c1.testrun.org")
-
-
-def login(conn, user, password):
-    print("trying to login", user, password)
-    conn.login(user, password)
-
-
-def connect(host):
-    print(f"connecting to {host}")
-    conn = imaplib.IMAP4_SSL(host)
-    return conn
-
-
-def test_login_ok(conn):
-    login(conn, "link2xt@c1.testrun.org", "Ahyei6ie")
-
-
-def test_login_fail(conn):
-    with pytest.raises(imaplib.IMAP4.error) as excinfo:
-        login(conn, "link2xt@c1.testrun.org", "qweqwe")
-    assert "AUTHENTICATIONFAILED" in str(excinfo)