plan: persistence is achieved

.gitignore

@@ -10,6 +10,7 @@ __pycache__/
 # Distribution / packaging
 .Python
 build/
+doveauth/dist/
 develop-eggs/
 dist/
 downloads/

README.md

@@ -1,40 +1,58 @@
 # Chat Mail server configuration
 
-This repository setups a ready-to-go chatmail instance
-comprised of a minimal setup of the battle-tested 
-[postfix smtp server](https://www.postfix.org) and [dovecot imap server](https://www.dovecot.org).
+This package deploys Postfix and Dovecot servers, including OpenDKIM for DKIM signing.
+
+Postfix uses Dovecot for authentication as described in <https://www.postfix.org/SASL_README.html#server_dovecot>
 
 ## Getting started 
 
-1. prepare your local system:
+prepare:
 
-    scripts/init.sh 
-
-2. set environment variable to the chatmail domain you want to setup:
-
-    export CHATMAIL_DOMAIN=c1.testrun.org   # replace with your host
-
-3. run the deploy of the chat mail instance: 
-
-    scripts/deploy.sh 
+    pip install -e chatmail-infra
 
 
-## Running tests and benchmarks (offline and online) 
+then run with pyinfra command line tool: 
 
-1. Set `CHATMAIL_SSH` so that `ssh root@$CHATMAIL_SSH` allows 
-   to login to the chatmail instance server. 
+    CHATMAIL_DOMAIN=c1.testrun.org pyinfra --ssh-user root c1.testrun.org deploy.py
 
-2. To run local and online tests: 
 
-    scripts/test.sh 
+## Structure (wip)
+```
 
-3. To run benchmarks against your chatmail instance: 
+# package doveauth tool and deploy chatmail server to a envvar-specified ssh-reachable host 
+deploy.py 
 
-    scripts/bench.sh 
+# chatmail pyinfra deploy package 
+chatmail-pyinfra 
+    pyproject.toml
+    chatmail/__init__ ...
 
-## Running tests (offline and online) 
+    # tests against the deployed system 
+    tests/test_online_test.py
+
+# doveauth tool used by dovecot's auth mechanism on the host system 
+doveauth
+    README.md
+    pyproject.toml
+    doveauth.py
+    doveauth.lua
+    test_doveauth.py
+
+# lmtp server to block (outgoing) unencrypted messages 
+filtermail 
+    README.md
+    pyproject.toml
+    .... 
+
+
+# scripts for setup/development/deployment 
+
+scripts/
+    init.sh  # create venv/other perequires
+    deploy.sh  # run pyinfra based deploy of everything
 
 ```
+
 ## Dovecot/Postfix configuration
 
 ### Ports
@@ -44,16 +62,4 @@ Dovecot listens on ports 143(imap) and 993 (imaps).
 
 ## DNS
 
-For DKIM you must add a DNS entry as found in /etc/opendkim/selector.txt on your chatmail instance.
-The above `scripts/deploy.sh` prints out the DKIM selector and DNS entry you
-need to setup with your DNS provider. 
-
-## Emergency Commands
-
-If you need to stop account creation,
-e.g. because some script is wildly creating accounts,
-just run `touch /tmp/nocreate`.
-You can remove the file
-as soon as the attacker was banned
-by different means.
-
+For DKIM you must add a DNS entry as in /etc/opendkim/selector.txt (where selector is the opendkim_selector configured in the chatmail inventory).

chatmail-pyinfra/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=45"]
 build-backend = "setuptools.build_meta"
 
 [project]
-name = "deploy-chatmail"
+name = "chatmail"
 version = "0.1"
 dependencies = [
   "pyinfra",

chatmail-pyinfra/src/chatmail/__init__.py

@@ -10,68 +10,30 @@ from pyinfra.facts.files import File
 from .acmetool import deploy_acmetool
 
 
-def _install_chatmaild() -> None:
-    chatmaild_filename = "chatmaild-0.1.tar.gz"
-    chatmaild_path = importlib.resources.files(__package__).joinpath(
-        f"../../../dist/{chatmaild_filename}"
+def _install_doveauth() -> None:
+    """Setup chatctl."""
+    doveauth_filename = "doveauth-0.1.tar.gz"
+    doveauth_path = importlib.resources.files(__package__).joinpath(
+        f"../../../doveauth/dist/{doveauth_filename}"
     )
-    remote_path = f"/tmp/{chatmaild_filename}"
-    if Path(str(chatmaild_path)).exists():
+    remote_path = f"/tmp/{doveauth_filename}"
+    if Path(str(doveauth_path)).exists():
         files.put(
-            name="Upload chatmaild source package",
-            src=chatmaild_path.open("rb"),
+            name="upload local doveauth build",
+            src=doveauth_path.open("rb"),
             dest=remote_path,
         )
-
         apt.packages(
-            name="apt install python3-aiosmtpd",
-            packages=["python3-aiosmtpd", "python3-pip"],
+            name="apt install python3-pip",
+            packages="python3-pip",
         )
-
-        # --no-deps because aiosmtplib is installed with `apt`.
+        # Maybe if we introduce dependencies to the doveauth package at some point, we should not install doveauth
+        # system-wide anymore. For now it's fine though.
         server.shell(
-            name="install chatmaild with pip",
+            name="install local doveauth build with pip",
             commands=[f"pip install --break-system-packages {remote_path}"],
         )
 
-        files.put(
-            name="upload doveauth-dictproxy.service",
-            src=importlib.resources.files("chatmaild")
-            .joinpath("doveauth-dictproxy.service")
-            .open("rb"),
-            dest="/etc/systemd/system/doveauth-dictproxy.service",
-            user="root",
-            group="root",
-            mode="644",
-        )
-        systemd.service(
-            name="Setup doveauth-dictproxy service",
-            service="doveauth-dictproxy.service",
-            running=True,
-            enabled=True,
-            restarted=True,
-            daemon_reload=True,
-        )
-
-        files.put(
-            name="upload filtermail.service",
-            src=importlib.resources.files("chatmaild")
-            .joinpath("filtermail.service")
-            .open("rb"),
-            dest="/etc/systemd/system/filtermail.service",
-            user="root",
-            group="root",
-            mode="644",
-        )
-        systemd.service(
-            name="Setup filtermail service",
-            service="filtermail.service",
-            running=True,
-            enabled=True,
-            restarted=True,
-            daemon_reload=True,
-        )
-
 
 def _configure_opendkim(domain: str, dkim_selector: str) -> bool:
     """Configures OpenDKIM"""
@@ -110,7 +72,7 @@ def _configure_opendkim(domain: str, dkim_selector: str) -> bool:
     return need_restart
 
 
-def _configure_postfix(domain: str, debug: bool = False) -> bool:
+def _configure_postfix(domain: str) -> bool:
     """Configures Postfix SMTP server."""
     need_restart = False
 
@@ -124,20 +86,21 @@ def _configure_postfix(domain: str, debug: bool = False) -> bool:
     )
     need_restart |= main_config.changed
 
-    master_config = files.template(
-        src=importlib.resources.files(__package__).joinpath("postfix/master.cf.j2"),
+    master_config = files.put(
+        src=importlib.resources.files(__package__)
+        .joinpath("postfix/master.cf")
+        .open("rb"),
         dest="/etc/postfix/master.cf",
         user="root",
         group="root",
         mode="644",
-        debug=debug,
     )
     need_restart |= master_config.changed
 
     return need_restart
 
 
-def _configure_dovecot(mail_server: str, debug: bool = False) -> bool:
+def _configure_dovecot(mail_server: str) -> bool:
     """Configures Dovecot IMAP server."""
     need_restart = False
 
@@ -148,27 +111,18 @@ def _configure_dovecot(mail_server: str, debug: bool = False) -> bool:
         group="root",
         mode="644",
         config={"hostname": mail_server},
-        debug=debug,
     )
     need_restart |= main_config.changed
-    auth_config = files.put(
-        src=importlib.resources.files(__package__).joinpath("dovecot/auth.conf"),
-        dest="/etc/dovecot/auth.conf",
-        user="root",
-        group="root",
-        mode="644",
-    )
-    need_restart |= auth_config.changed
 
-    files.put(
-        src=importlib.resources.files(__package__)
-        .joinpath("dovecot/expunge.cron")
-        .open("rb"),
-        dest="/etc/cron.d/expunge",
+    # luarocks install http lpeg_patterns fifo
+    auth_script = files.put(
+        src=importlib.resources.files("doveauth").joinpath("doveauth.lua"),
+        dest="/etc/dovecot/doveauth.lua",
         user="root",
         group="root",
         mode="644",
     )
+    need_restart |= auth_script.changed
 
     return need_restart
 
@@ -181,7 +135,7 @@ def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> N
     :param dkim_selector:
     """
 
-    apt.update(name="apt update", cache_time=24 * 3600)
+    apt.update(name="apt update")
     server.group(name="Create vmail group", group="vmail", system=True)
     server.user(name="Create vmail user", user="vmail", group="vmail", system=True)
 
@@ -203,7 +157,11 @@ def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> N
 
     apt.packages(
         name="Install Dovecot",
-        packages=["dovecot-imapd", "dovecot-lmtpd"],
+        packages=[
+            "dovecot-imapd",
+            "dovecot-lmtpd",
+            "dovecot-auth-lua",
+        ],
     )
 
     apt.packages(
@@ -214,10 +172,9 @@ def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> N
         ],
     )
 
-    _install_chatmaild()
-    debug = False
-    dovecot_need_restart = _configure_dovecot(mail_server, debug=debug)
-    postfix_need_restart = _configure_postfix(mail_domain, debug=debug)
+    _install_doveauth()
+    dovecot_need_restart = _configure_dovecot(mail_server)
+    postfix_need_restart = _configure_postfix(mail_domain)
     opendkim_need_restart = _configure_opendkim(mail_domain, dkim_selector)
 
     systemd.service(
@@ -244,13 +201,6 @@ def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> N
         restarted=dovecot_need_restart,
     )
 
-    # This file is used by auth proxy.
-    # https://wiki.debian.org/EtcMailName
-    server.shell(
-        name="Setup /etc/mailname",
-        commands=[f"echo {mail_domain} >/etc/mailname; chmod 644 /etc/mailname"],
-    )
-
     def callback():
         result = server.shell(
             commands=[

chatmail-pyinfra/src/chatmail/acmetool/__init__.py

@@ -50,8 +50,8 @@ def deploy_acmetool(nginx_hook=False, email="", domains=[]):
     systemd.service(
         name="Setup acmetool-redirector service",
         service="acmetool-redirector.service",
-        running=False,
-        enabled=False,
+        running=True,
+        enabled=True,
         restarted=service_file.changed,
     )
 

chatmail-pyinfra/src/chatmail/dovecot/dovecot.conf.j2

@@ -4,33 +4,21 @@ protocols = imap lmtp
 
 auth_mechanisms = plain
 
-{% if debug == true %}
 auth_verbose = yes
 auth_debug = yes
 auth_debug_passwords = yes
 auth_verbose_passwords = plain
-auth_cache_size = 100M
-mail_debug = yes
-{% endif %}
-
-
-mail_plugins = quota
-
-# these are the capabilities Delta Chat cares about actually 
-# so let's keep the network overhead per login small
-# https://github.com/deltachat/deltachat-core-rust/blob/master/src/imap/capabilities.rs
-imap_capability = IMAP4rev1 IDLE MOVE QUOTA CONDSTORE
-
 
 # Authentication for system users.
 passdb {
-  driver = dict
-  args = /etc/dovecot/auth.conf
+  driver = lua
+  args = file=/etc/dovecot/doveauth.lua
 }
 userdb {
-  driver = dict
-  args = /etc/dovecot/auth.conf
+  driver = lua
+  args = file=/etc/dovecot/doveauth.lua
 }
+
 ##
 ## Mailbox locations and namespaces
 ##
@@ -72,28 +60,13 @@ mail_privileged_group = vmail
 # Enable IMAP COMPRESS (RFC 4978).
 # <https://datatracker.ietf.org/doc/html/rfc4978.html>
 protocol imap {
-  mail_plugins = $mail_plugins imap_zlib imap_quota
-}
-
-protocol lmtp {
-  mail_plugins = $mail_plugins quota
+  mail_plugins = $mail_plugins imap_zlib
 }
 
 plugin {
   imap_compress_deflate_level = 6
 }
 
-plugin {
-  # for now we define static quota-rules for all users 
-  quota = maildir:User quota
-  quota_rule = *:storage=100M
-  quota_max_mail_size=30M
-  quota_grace = 0
-  # quota_over_flag_value = TRUE
-}
-
-
-
 service lmtp {
   user=vmail
 

chatmail-pyinfra/src/chatmail/postfix/main.cf.j2

@@ -37,8 +37,6 @@ mydestination =
 relayhost = 
 mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128
 mailbox_size_limit = 0
-# maximum 30MB sized messages
-message_size_limit = 31457280
 recipient_delimiter = +
 inet_interfaces = all
 inet_protocols = all

chatmail-pyinfra/src/chatmail/postfix/master.cf

@@ -9,11 +9,7 @@
 # service type  private unpriv  chroot  wakeup  maxproc command + args
 #               (yes)   (yes)   (no)    (never) (100)
 # ==========================================================================
-{% if debug == true %}
-smtp      inet  n       -       y       -       -       smtpd -v
-{% else %}
-smtp      inet  n       -       y       -       -       smtpd 
-{% endif %}
+smtp      inet  n       -       y       -       -       smtpd
 #smtp      inet  n       -       y       -       1       postscreen
 #smtpd     pass  -       -       y       -       -       smtpd
 #dnsblog   unix  -       -       y       -       0       dnsblog
@@ -32,7 +28,6 @@ submission inet n       -       y       -       -       smtpd
   -o smtpd_recipient_restrictions=
   -o smtpd_relay_restrictions=permit_sasl_authenticated,reject
   -o milter_macro_daemon_name=ORIGINATING
-  -o content_filter=filter:unix:private/filtermail
 smtps     inet  n       -       y       -       -       smtpd
   -o syslog_name=postfix/smtps
   -o smtpd_tls_wrappermode=yes
@@ -47,7 +42,6 @@ smtps     inet  n       -       y       -       -       smtpd
   -o smtpd_recipient_restrictions=
   -o smtpd_relay_restrictions=permit_sasl_authenticated,reject
   -o milter_macro_daemon_name=ORIGINATING
-  -o content_filter=filter:unix:private/filtermail
 #628       inet  n       -       y       -       -       qmqpd
 pickup    unix  n       -       y       60      1       pickup
 cleanup   unix  n       -       y       -       0       cleanup
@@ -70,11 +64,9 @@ showq     unix  n       -       y       -       -       showq
 error     unix  -       -       y       -       -       error
 retry     unix  -       -       y       -       -       error
 discard   unix  -       -       y       -       -       discard
+local     unix  -       n       n       -       -       local
+virtual   unix  -       n       n       -       -       virtual
 lmtp      unix  -       -       y       -       -       lmtp
 anvil     unix  -       -       y       -       1       anvil
 scache    unix  -       -       y       -       1       scache
 postlog   unix-dgram n  -       n       -       1       postlogd
-filter    unix -        n       n       -       -       lmtp
-# Local SMTP server for reinjecting filered mail.
-localhost:10026 inet  n       -       n       -       10      smtpd
-        -o content_filter=

chatmail-pyinfra/tests/test_online_login.py

@@ -0,0 +1,28 @@
+import pytest
+import imaplib
+
+
+@pytest.fixture
+def conn():
+    return connect("c1.testrun.org")
+
+
+def login(conn, user, password):
+    print("trying to login", user, password)
+    conn.login(user, password)
+
+
+def connect(host):
+    print(f"connecting to {host}")
+    conn = imaplib.IMAP4_SSL(host)
+    return conn
+
+
+def test_login_ok(conn):
+    login(conn, "link2xt@c1.testrun.org", "Ahyei6ie")
+
+
+def test_login_fail(conn):
+    with pytest.raises(imaplib.IMAP4.error) as excinfo:
+        login(conn, "link2xt@c1.testrun.org", "qweqwe")
+    assert "AUTHENTICATIONFAILED" in str(excinfo)

chatmaild/README.md

@@ -1,5 +0,0 @@
-# chatmaild
-
-chatmaild provides dovecot autentication
-to create dovecot users on login
-and mail filtering.

chatmaild/src/chatmaild/dictproxy.py

@@ -1,125 +0,0 @@
-import logging
-import os
-import sys
-import json
-from socketserver import (
-    UnixStreamServer,
-    StreamRequestHandler,
-    ThreadingMixIn,
-)
-import pwd
-import subprocess
-
-from .database import Database
-
-NOCREATE_FILE = "/etc/chatmail-nocreate"
-
-
-def encrypt_password(password: str):
-    password = password.encode("ascii")
-    # https://doc.dovecot.org/configuration_manual/authentication/password_schemes/
-    process = subprocess.Popen(
-        ["doveadm", "pw", "-s", "SHA512-CRYPT"],
-        stdin=subprocess.PIPE,
-        stdout=subprocess.PIPE,
-    )
-    stdout_data, _stderr_data = process.communicate(
-        input=password + b"\n" + password + b"\n"
-    )
-    return stdout_data.decode("ascii").strip()
-
-
-def create_user(db, user, password):
-    if os.path.exists(NOCREATE_FILE):
-        logging.warning(f"Didn't create account: {NOCREATE_FILE} exists. Delete the file to enable account creation.")
-        return
-    with db.write_transaction() as conn:
-        conn.create_user(user, password)
-    return dict(home=f"/home/vmail/{user}", uid="vmail", gid="vmail", password=password)
-
-
-def get_user_data(db, user):
-    with db.read_connection() as conn:
-        result = conn.get_user(user)
-    if result:
-        result["uid"] = "vmail"
-        result["gid"] = "vmail"
-    return result
-
-
-def lookup_userdb(db, user):
-    return get_user_data(db, user)
-
-
-def lookup_passdb(db, user, password):
-    userdata = get_user_data(db, user)
-    if not userdata:
-        return create_user(db, user, encrypt_password(password))
-    userdata["password"] = userdata["password"].strip()
-    return userdata
-
-
-def handle_dovecot_request(msg, db, mail_domain):
-    print(f"received msg: {msg!r}", file=sys.stderr)
-    short_command = msg[0]
-    if short_command == "L":  # LOOKUP
-        parts = msg[1:].split("\t")
-        keyname, user = parts[:2]
-        namespace, type, *args = keyname.split("/")
-        reply_command = "F"
-        res = ""
-        if namespace == "shared":
-            if type == "userdb":
-                if user.endswith(f"@{mail_domain}"):
-                    res = lookup_userdb(db, user)
-                if res:
-                    reply_command = "O"
-                else:
-                    reply_command = "N"
-            elif type == "passdb":
-                if user.endswith(f"@{mail_domain}"):
-                    res = lookup_passdb(db, user, password=args[0])
-                if res:
-                    reply_command = "O"
-                else:
-                    reply_command = "N"
-        print(f"res: {res!r}", file=sys.stderr)
-        json_res = json.dumps(res) if res else ""
-        return f"{reply_command}{json_res}\n"
-    return None
-
-
-class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
-    pass
-
-
-def main():
-    socket = sys.argv[1]
-    passwd_entry = pwd.getpwnam(sys.argv[2])
-    db = Database(sys.argv[3])
-    with open("/etc/mailname", "r") as fp:
-        mail_domain = fp.read().strip()
-
-    class Handler(StreamRequestHandler):
-        def handle(self):
-            while True:
-                msg = self.rfile.readline().strip().decode()
-                if not msg:
-                    break
-                res = handle_dovecot_request(msg, db, mail_domain)
-                if res:
-                    print(f"sending result: {res!r}", file=sys.stderr)
-                    self.wfile.write(res.encode("ascii"))
-                    self.wfile.flush()
-
-    try:
-        os.unlink(socket)
-    except FileNotFoundError:
-        pass
-
-    with ThreadedUnixStreamServer(socket, Handler) as server:
-        os.chown(socket, uid=passwd_entry.pw_uid, gid=passwd_entry.pw_gid)
-        try:
-            server.serve_forever()
-        except KeyboardInterrupt:
-            pass

chatmaild/src/chatmaild/doveauth-dictproxy.service

@@ -1,10 +0,0 @@
-[Unit]
-Description=Dict authentication proxy for dovecot
-
-[Service]
-ExecStart=/usr/local/bin/doveauth-dictproxy /run/dovecot/doveauth.socket vmail /home/vmail/passdb.sqlite
-Restart=always
-RestartSec=30
-
-[Install]
-WantedBy=multi-user.target

chatmaild/src/chatmaild/filtermail.py

@@ -1,109 +0,0 @@
-#!/usr/bin/env python3
-import asyncio
-import logging
-from email.parser import BytesParser
-from email import policy
-
-from aiosmtpd.lmtp import LMTP
-from aiosmtpd.controller import UnixSocketController
-from smtplib import SMTP as SMTPClient
-
-
-def check_encrypted(message):
-    """Check that the message is an OpenPGP-encrypted message."""
-    if not message.is_multipart():
-        return False
-    if message.get("subject") != "...":
-        return False
-    if message.get_content_type() != "multipart/encrypted":
-        return False
-    parts_count = 0
-    for part in message.iter_parts():
-        if parts_count == 0:
-            if part.get_content_type() != "application/pgp-encrypted":
-                return False
-        elif parts_count == 1:
-            if part.get_content_type() != "application/octet-stream":
-                return False
-        else:
-            return False
-        parts_count += 1
-    return True
-
-
-class ExampleController(UnixSocketController):
-    def factory(self):
-        return LMTP(self.handler, **self.SMTP_kwargs)
-
-
-class ExampleHandler:
-    async def handle_RCPT(self, server, session, envelope, address, rcpt_options):
-        envelope.rcpt_tos.append(address)
-        return "250 OK"
-
-    async def handle_DATA(self, server, session, envelope):
-        logging.info("Processing DATA message from %s", envelope.mail_from)
-
-        valid_recipients = []
-
-        message = BytesParser(policy=policy.default).parsebytes(envelope.content)
-        mail_encrypted = check_encrypted(message)
-
-        res = []
-        for recipient in envelope.rcpt_tos:
-            my_local_domain = envelope.mail_from.split("@")
-            if len(my_local_domain) != 2:
-                res += [f"500 Invalid from address <{envelope.mail_from}>"]
-                continue
-
-            if envelope.mail_from == recipient:
-                # Always allow sending emails to self.
-                valid_recipients += [recipient]
-                res += ["250 OK"]
-                continue
-
-            recipient_local_domain = recipient.split("@")
-            if len(recipient_local_domain) != 2:
-                res += [f"500 Invalid address <{recipient}>"]
-                continue
-
-            is_outgoing = recipient_local_domain[1] != my_local_domain[1]
-
-            if (
-                is_outgoing
-                and not mail_encrypted
-                and message.get("secure-join") != "vc-request"
-                and message.get("secure-join") != "vg-request"
-            ):
-                res += ["500 Outgoing mail must be encrypted"]
-                continue
-
-            valid_recipients += [recipient]
-            res += ["250 OK"]
-
-        # Reinject the mail back into Postfix.
-        if valid_recipients:
-            logging.info("Reinjecting the mail")
-            client = SMTPClient("localhost", "10026")
-            client.sendmail(envelope.mail_from, valid_recipients, envelope.content)
-
-        return "\r\n".join(res)
-
-
-async def asyncmain(loop):
-    controller = ExampleController(
-        ExampleHandler(), unix_socket="/var/spool/postfix/private/filtermail"
-    )
-    controller.start()
-
-
-def main():
-    logging.basicConfig(level=logging.INFO)
-    loop = asyncio.new_event_loop()
-    asyncio.set_event_loop(loop)
-    loop.create_task(asyncmain(loop=loop))
-    loop.run_forever()
-
-
-if __name__ == "__main__":
-    main()

chatmaild/src/chatmaild/filtermail.service

@@ -1,10 +0,0 @@
-[Unit]
-Description=Email filter for chatmail servers
-
-[Service]
-ExecStart=/usr/local/bin/filtermail
-Restart=always
-RestartSec=30
-
-[Install]
-WantedBy=multi-user.target

chatmaild/src/chatmaild/test_doveauth.py

@@ -1,53 +0,0 @@
-import os
-
-import pytest
-
-import chatmaild.dictproxy
-from .dictproxy import get_user_data, lookup_passdb
-from .database import Database, DBError
-
-
-@pytest.fixture()
-def db(tmpdir):
-    db_path = tmpdir / "passdb.sqlite"
-    print("database path:", db_path)
-    return Database(db_path)
-
-
-def test_basic(db):
-    chatmaild.dictproxy.NOCREATE_FILE = "/tmp/nocreate"
-    if os.path.exists(chatmaild.dictproxy.NOCREATE_FILE):
-        os.remove(chatmaild.dictproxy.NOCREATE_FILE)
-    lookup_passdb(db, "link2xt@c1.testrun.org", "asdf")
-    data = get_user_data(db, "link2xt@c1.testrun.org")
-    assert data
-
-
-def test_dont_overwrite_password_on_wrong_login(db):
-    """Test that logging in with a different password doesn't create a new user"""
-    res = lookup_passdb(db, "newuser1@something.org", "kajdlkajsldk12l3kj1983")
-    assert res["password"]
-    res2 = lookup_passdb(db, "newuser1@something.org", "kajdlqweqwe")
-    # this function always returns a password hash, which is actually compared by dovecot.
-    assert res["password"] == res2["password"]
-
-
-def test_nocreate_file(db):
-    chatmaild.dictproxy.NOCREATE_FILE = "/tmp/nocreate"
-    with open(chatmaild.dictproxy.NOCREATE_FILE, "w+") as f:
-        f.write("")
-    assert os.path.exists(chatmaild.dictproxy.NOCREATE_FILE)
-    lookup_passdb(db, "newuser1@something.org", "kajdlqweqwe")
-    assert not get_user_data(db, "newuser1@something.org")
-    os.remove(chatmaild.dictproxy.NOCREATE_FILE)
-
-
-def test_db_version(db):
-    assert db.get_schema_version() == 1
-
-
-def test_too_high_db_version(db):
-    with db.write_transaction() as conn:
-        conn.execute("PRAGMA user_version=%s;" % (999,))
-    with pytest.raises(DBError):
-        db.ensure_tables()

chatmaild/src/chatmaild/test_filtermail.py

@@ -1,290 +0,0 @@
-from .filtermail import check_encrypted
-from email.parser import BytesParser
-from email import policy
-
-
-def test_filtermail():
-    def check_encrypted_bstr(content):
-        message = BytesParser(policy=policy.default).parsebytes(content)
-        return check_encrypted(message)
-
-    assert not check_encrypted_bstr(b"foo")
-
-    assert not check_encrypted_bstr(
-        "\r\n".join(
-            [
-                "Subject: =?utf-8?q?Message_from_foobar=40c2=2Etestrun=2Eorg?=",
-                "Chat-Disposition-Notification-To: foobar@c2.testrun.org",
-                "Chat-User-Avatar: 0",
-                "From: <foobar@c2.testrun.org>",
-                "To: <barbaz@c2.testrun.org>",
-                "Date: Sun, 15 Oct 2023 16:41:44 +0000",
-                "Message-ID: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>",
-                "References: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>",
-                "Chat-Version: 1.0",
-                "Autocrypt: addr=foobar@c2.testrun.org; prefer-encrypt=mutual;",
-                "\tkeydata=xjMEZSrw3hYJKwYBBAHaRw8BAQdAiEKNQFU28c6qsx4vo/JHdt73RXdjMOmByf/XsGiJ7m",
-                "\tnNFzxmb29iYXJAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUq8N4CGwMECwkIBwYVCAkKCwID",
-                "\tFgIBFiEEGil0OvTIa6RngmCLUYNnEa9leJAACgkQUYNnEa9leJCX3gEAhm0MehE5byBBU1avPczr/I",
-                "\tHjNLht7Qf6++mAhlJmtDcA/0C8VYJhsUpmiDjuZaMDWNv4FO2BJG6LH7gSm6n7ClMJzjgEZSrw3hIK",
-                "\tKwYBBAGXVQEFAQEHQAxGG/QW0owCfMp1A+vXEMwgzWcBpNFr58kX2eXuPpM6AwEIB8J4BBgWCAAgBQ",
-                "\tJlKvDeAhsMFiEEGil0OvTIa6RngmCLUYNnEa9leJAACgkQUYNnEa9leJDg1gEAwLf8KDoAAKyYgjyI",
-                "\tvYvO9VEgBni1C4Xx1VjcaEmlDK8BALoFuUCK+enw76TtDcAUKhlhUiM6SDRExkS4Nskp/BcK",
-                "MIME-Version: 1.0",
-                "Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no",
-                "",
-                "Hi!",
-                "",
-                "",
-            ]
-        ).encode()
-    )
-
-    assert not check_encrypted_bstr(
-        "\r\n".join(
-            [
-                "Subject: =?utf-8?q?Message_from_foobar=40c2=2Etestrun=2Eorg?=",
-                "Chat-Disposition-Notification-To: foobar@c2.testrun.org",
-                "Chat-User-Avatar: 0",
-                "From: <foobar@c2.testrun.org>",
-                "To: <barbaz@c2.testrun.org>",
-                "Date: Sun, 15 Oct 2023 16:41:44 +0000",
-                "Message-ID: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>",
-                "References: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>",
-                "Chat-Version: 1.0",
-                "Autocrypt: addr=foobar@c2.testrun.org; prefer-encrypt=mutual;",
-                "\tkeydata=xjMEZSrw3hYJKwYBBAHaRw8BAQdAiEKNQFU28c6qsx4vo/JHdt73RXdjMOmByf/XsGiJ7m",
-                "\tnNFzxmb29iYXJAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUq8N4CGwMECwkIBwYVCAkKCwID",
-                "\tFgIBFiEEGil0OvTIa6RngmCLUYNnEa9leJAACgkQUYNnEa9leJCX3gEAhm0MehE5byBBU1avPczr/I",
-                "\tHjNLht7Qf6++mAhlJmtDcA/0C8VYJhsUpmiDjuZaMDWNv4FO2BJG6LH7gSm6n7ClMJzjgEZSrw3hIK",
-                "\tKwYBBAGXVQEFAQEHQAxGG/QW0owCfMp1A+vXEMwgzWcBpNFr58kX2eXuPpM6AwEIB8J4BBgWCAAgBQ",
-                "\tJlKvDeAhsMFiEEGil0OvTIa6RngmCLUYNnEa9leJAACgkQUYNnEa9leJDg1gEAwLf8KDoAAKyYgjyI",
-                "\tvYvO9VEgBni1C4Xx1VjcaEmlDK8BALoFuUCK+enw76TtDcAUKhlhUiM6SDRExkS4Nskp/BcK",
-                "MIME-Version: 1.0",
-                "Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no",
-                "",
-                "Hi!",
-                "",
-                "",
-            ]
-        ).encode()
-    )
-
-    # https://xkcd.com/1181/
-    assert not check_encrypted_bstr(
-        "\r\n".join(
-            [
-                "Subject: =?utf-8?q?Message_from_foobar=40c2=2Etestrun=2Eorg?=",
-                "Chat-Disposition-Notification-To: foobar@c2.testrun.org",
-                "Chat-User-Avatar: 0",
-                "From: <foobar@c2.testrun.org>",
-                "To: <barbaz@c2.testrun.org>",
-                "Date: Sun, 15 Oct 2023 16:41:44 +0000",
-                "Message-ID: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>",
-                "References: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>",
-                "Chat-Version: 1.0",
-                "Autocrypt: addr=foobar@c2.testrun.org; prefer-encrypt=mutual;",
-                "\tkeydata=xjMEZSrw3hYJKwYBBAHaRw8BAQdAiEKNQFU28c6qsx4vo/JHdt73RXdjMOmByf/XsGiJ7m",
-                "\tnNFzxmb29iYXJAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUq8N4CGwMECwkIBwYVCAkKCwID",
-                "\tFgIBFiEEGil0OvTIa6RngmCLUYNnEa9leJAACgkQUYNnEa9leJCX3gEAhm0MehE5byBBU1avPczr/I",
-                "\tHjNLht7Qf6++mAhlJmtDcA/0C8VYJhsUpmiDjuZaMDWNv4FO2BJG6LH7gSm6n7ClMJzjgEZSrw3hIK",
-                "\tKwYBBAGXVQEFAQEHQAxGG/QW0owCfMp1A+vXEMwgzWcBpNFr58kX2eXuPpM6AwEIB8J4BBgWCAAgBQ",
-                "\tJlKvDeAhsMFiEEGil0OvTIa6RngmCLUYNnEa9leJAACgkQUYNnEa9leJDg1gEAwLf8KDoAAKyYgjyI",
-                "\tvYvO9VEgBni1C4Xx1VjcaEmlDK8BALoFuUCK+enw76TtDcAUKhlhUiM6SDRExkS4Nskp/BcK",
-                "MIME-Version: 1.0",
-                "Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no",
-                "",
-                "-----BEGIN PGP MESSAGE-----",
-                "Hi!",
-                "-----END PGP MESSAGE-----",
-                "",
-                "",
-            ]
-        ).encode()
-    )
-
-    assert check_encrypted_bstr(
-        "\r\n".join(
-            [
-                "Subject: ...",
-                "From: <barbaz@c2.testrun.org>",
-                "To: <foobar@c2.testrun.org>",
-                "Date: Sun, 15 Oct 2023 16:43:21 +0000",
-                "Message-ID: <Mr.UVyJWZmkCKM.hGzNc6glBE_@c2.testrun.org>",
-                "In-Reply-To: <Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>",
-                "References: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>",
-                "\t<Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>",
-                "Chat-Version: 1.0",
-                "Autocrypt: addr=barbaz@c2.testrun.org; prefer-encrypt=mutual;",
-                "\tkeydata=xjMEZSwWjhYJKwYBBAHaRw8BAQdAQBEhqeJh0GueHB6kF/DUQqYCxARNBVokg/AzT+7LqH",
-                "\trNFzxiYXJiYXpAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUsFo4CGwMECwkIBwYVCAkKCwID",
-                "\tFgIBFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX9A4AEAnHWHp49eBCMHK5t66gYPiW",
-                "\tXQuB1mwUjzGfYWB+0RXUoA/0xcQ3FbUNlGKW7Blp6eMFfViv6Mv2d3kNSXACB6nmcMzjgEZSwWjhIK",
-                "\tKwYBBAGXVQEFAQEHQBpY5L2M1XHo0uxf8SX1wNLBp/OVvidoWHQF2Jz+kJsUAwEIB8J4BBgWCAAgBQ",
-                "\tJlLBaOAhsMFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX/INgEA37AJaNvruYsJVanP",
-                "\tIXnYw4CKd55UAwl8Zcy+M2diAbkA/0fHHcGV4r78hpbbL1Os52DPOdqYQRauIeJUeG+G6bQO",
-                "MIME-Version: 1.0",
-                'Content-Type: multipart/encrypted; protocol="application/pgp-encrypted";',
-                '\tboundary="YFrteb74qSXmggbOxZL9dRnhymywAi"',
-                "",
-                "",
-                "--YFrteb74qSXmggbOxZL9dRnhymywAi",
-                "Content-Description: PGP/MIME version identification",
-                "Content-Type: application/pgp-encrypted",
-                "",
-                "Version: 1",
-                "",
-                "",
-                "--YFrteb74qSXmggbOxZL9dRnhymywAi",
-                "Content-Description: OpenPGP encrypted message",
-                'Content-Disposition: inline; filename="encrypted.asc";',
-                'Content-Type: application/octet-stream; name="encrypted.asc"',
-                "",
-                "-----BEGIN PGP MESSAGE-----",
-                "",
-                "wU4DhW3gBZ/VvCYSAQdA8bMs2spwbKdGjVsL1ByPkNrqD7frpB73maeL6I6SzDYg",
-                "O5G53tv339RdKq3WRcCtEEvxjHlUx2XNwXzC04BpmfvBTgNfPUyLDzjXnxIBB0Ae",
-                "8ymwGvXMCCimHXN0Dg8Ui62KOi03h0UgheoHWovJSCDF4CKre/xtFr3nL7lq/PKI",
-                "JsjVNz7/RK9FSXF6WwfONtLCyQGEuVAsB/KXfCBEyfKhaMwGHvhujRidGW5uV1no",
-                "lMGl3ODmo29Lgeu2uSE7EpJRZoe6hU6ddmBkqxax61ZtkaFlGFFpdo2K8balNNdz",
-                "ZsJ/9mmI9x3oOJ4/l1nhQbUO9ADbs7gJhFdV5Qkp30b5fCI7bU+aoe1ccBbLe/WM",
-                "YUty1PqcuQT7XjA+XmYuL261tvW8pBetT+i33/E2d8PzzYt2IuK9qeevyS+yxdwA",
-                "kfwejFWzzsUlJaDxs1x4XOxkMgSj+jo+g12dFOb7fyClsAnq23iDb8AuaT/BScAI",
-                "+lO+gher69+6LmM7VGHLG5k762J1jTaQCaKt1s8TAWV99Eo4491vL6fyvk3l/Cfg",
-                "RXSwiWFgj19Pn0Rq7CD9v22UE2vdUMBTcV4aw79mClk1YQ23jbF0y5DCjPdJ62Zo",
-                "tskBgFt3NoWV80jZ76zIBLrrjLwCCll8JjJtFwSkt2GX5RFBsVa4A8IDht9RtEk7",
-                "rrHgbSZQfkauEi/mH3/6CDZoLqSHudUZ7d4MaJwun1TkFYGe2ORwGJd4OBj3oGJp",
-                "H8YBwCpk///L/fKjX0Gg3M8nrpM4wrRFhPKidAgO/kcm25X4+ZHlVkWBTCt5RWKI",
-                "fHh6oLDZCqCfcgMkE1KKmwfIHaUkhq5BPRigwy6i5dh1DM4+1UCLh3dxzVbqE9b9",
-                "61NB19nXdRtDA2sOUnj9ve6m/wEPyCb6/zBQZqvCBYb1/AjdXpUrFT+DbpfyxaXN",
-                "XfhDVb5mNqNM/IVj0V5fvTc6vOfYbzQtPm10H+FdWWfb+rJRfyC3MA2w2IqstFe3",
-                "w3bu2iE6CQvSqRvge+ZqLKt/NqYwOURiUmpuklbl3kPJ97+mfKWoiqk8Iz1VY+bb",
-                "NMUC7aoGv+jcoj+WS6PYO8N6BeRVUUB3ZJSf8nzjgxm1/BcM+UD3BPrlhT11ODRs",
-                "baifGbprMWwt3dhb8cQgRT8GPdpO1OsDkzL6iikMjLHWWiA99GV6ruiHsIPw6boW",
-                "A6/uSOskbDHOROotKmddGTBd0iiHXAoQsJFt1ZjUkt6EHrgWs+GAvrvKpXs1mrz8",
-                "uj3GwEFrHS+Xuf2UDgpszYT3hI2cL/kUtGakVR7m7vVMZqXBUbZdGAEb1PZNPwsI",
-                "E4aMK02+EVB+tSN4Fzj99N2YD0inVYt+oPjr2tHhUS6aSGBNS/48Ki47DOg4Sxkn",
-                "lkOWnEbCD+XTnbDd",
-                "=agR5",
-                "-----END PGP MESSAGE-----",
-                "",
-                "",
-                "--YFrteb74qSXmggbOxZL9dRnhymywAi--",
-                "",
-                "",
-            ]
-        ).encode()
-    )
-
-    assert not check_encrypted_bstr(
-        "\r\n".join(
-            [
-                "Subject: Buy Penis Enlargement at www.malicious-domain.com",
-                "From: <barbaz@c2.testrun.org>",
-                "To: <foobar@c2.testrun.org>",
-                "Date: Sun, 15 Oct 2023 16:43:21 +0000",
-                "Message-ID: <Mr.UVyJWZmkCKM.hGzNc6glBE_@c2.testrun.org>",
-                "In-Reply-To: <Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>",
-                "References: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>",
-                "\t<Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>",
-                "Chat-Version: 1.0",
-                "Autocrypt: addr=barbaz@c2.testrun.org; prefer-encrypt=mutual;",
-                "\tkeydata=xjMEZSwWjhYJKwYBBAHaRw8BAQdAQBEhqeJh0GueHB6kF/DUQqYCxARNBVokg/AzT+7LqH",
-                "\trNFzxiYXJiYXpAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUsFo4CGwMECwkIBwYVCAkKCwID",
-                "\tFgIBFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX9A4AEAnHWHp49eBCMHK5t66gYPiW",
-                "\tXQuB1mwUjzGfYWB+0RXUoA/0xcQ3FbUNlGKW7Blp6eMFfViv6Mv2d3kNSXACB6nmcMzjgEZSwWjhIK",
-                "\tKwYBBAGXVQEFAQEHQBpY5L2M1XHo0uxf8SX1wNLBp/OVvidoWHQF2Jz+kJsUAwEIB8J4BBgWCAAgBQ",
-                "\tJlLBaOAhsMFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX/INgEA37AJaNvruYsJVanP",
-                "\tIXnYw4CKd55UAwl8Zcy+M2diAbkA/0fHHcGV4r78hpbbL1Os52DPOdqYQRauIeJUeG+G6bQO",
-                "MIME-Version: 1.0",
-                'Content-Type: multipart/encrypted; protocol="application/pgp-encrypted";',
-                '\tboundary="YFrteb74qSXmggbOxZL9dRnhymywAi"',
-                "",
-                "",
-                "--YFrteb74qSXmggbOxZL9dRnhymywAi",
-                "Content-Description: PGP/MIME version identification",
-                "Content-Type: application/pgp-encrypted",
-                "",
-                "Version: 1",
-                "",
-                "",
-                "--YFrteb74qSXmggbOxZL9dRnhymywAi",
-                "Content-Description: OpenPGP encrypted message",
-                'Content-Disposition: inline; filename="encrypted.asc";',
-                'Content-Type: application/octet-stream; name="encrypted.asc"',
-                "",
-                "-----BEGIN PGP MESSAGE-----",
-                "",
-                "wU4DhW3gBZ/VvCYSAQdA8bMs2spwbKdGjVsL1ByPkNrqD7frpB73maeL6I6SzDYg",
-                "O5G53tv339RdKq3WRcCtEEvxjHlUx2XNwXzC04BpmfvBTgNfPUyLDzjXnxIBB0Ae",
-                "8ymwGvXMCCimHXN0Dg8Ui62KOi03h0UgheoHWovJSCDF4CKre/xtFr3nL7lq/PKI",
-                "JsjVNz7/RK9FSXF6WwfONtLCyQGEuVAsB/KXfCBEyfKhaMwGHvhujRidGW5uV1no",
-                "lMGl3ODmo29Lgeu2uSE7EpJRZoe6hU6ddmBkqxax61ZtkaFlGFFpdo2K8balNNdz",
-                "ZsJ/9mmI9x3oOJ4/l1nhQbUO9ADbs7gJhFdV5Qkp30b5fCI7bU+aoe1ccBbLe/WM",
-                "YUty1PqcuQT7XjA+XmYuL261tvW8pBetT+i33/E2d8PzzYt2IuK9qeevyS+yxdwA",
-                "kfwejFWzzsUlJaDxs1x4XOxkMgSj+jo+g12dFOb7fyClsAnq23iDb8AuaT/BScAI",
-                "+lO+gher69+6LmM7VGHLG5k762J1jTaQCaKt1s8TAWV99Eo4491vL6fyvk3l/Cfg",
-                "RXSwiWFgj19Pn0Rq7CD9v22UE2vdUMBTcV4aw79mClk1YQ23jbF0y5DCjPdJ62Zo",
-                "tskBgFt3NoWV80jZ76zIBLrrjLwCCll8JjJtFwSkt2GX5RFBsVa4A8IDht9RtEk7",
-                "rrHgbSZQfkauEi/mH3/6CDZoLqSHudUZ7d4MaJwun1TkFYGe2ORwGJd4OBj3oGJp",
-                "H8YBwCpk///L/fKjX0Gg3M8nrpM4wrRFhPKidAgO/kcm25X4+ZHlVkWBTCt5RWKI",
-                "fHh6oLDZCqCfcgMkE1KKmwfIHaUkhq5BPRigwy6i5dh1DM4+1UCLh3dxzVbqE9b9",
-                "61NB19nXdRtDA2sOUnj9ve6m/wEPyCb6/zBQZqvCBYb1/AjdXpUrFT+DbpfyxaXN",
-                "XfhDVb5mNqNM/IVj0V5fvTc6vOfYbzQtPm10H+FdWWfb+rJRfyC3MA2w2IqstFe3",
-                "w3bu2iE6CQvSqRvge+ZqLKt/NqYwOURiUmpuklbl3kPJ97+mfKWoiqk8Iz1VY+bb",
-                "NMUC7aoGv+jcoj+WS6PYO8N6BeRVUUB3ZJSf8nzjgxm1/BcM+UD3BPrlhT11ODRs",
-                "baifGbprMWwt3dhb8cQgRT8GPdpO1OsDkzL6iikMjLHWWiA99GV6ruiHsIPw6boW",
-                "A6/uSOskbDHOROotKmddGTBd0iiHXAoQsJFt1ZjUkt6EHrgWs+GAvrvKpXs1mrz8",
-                "uj3GwEFrHS+Xuf2UDgpszYT3hI2cL/kUtGakVR7m7vVMZqXBUbZdGAEb1PZNPwsI",
-                "E4aMK02+EVB+tSN4Fzj99N2YD0inVYt+oPjr2tHhUS6aSGBNS/48Ki47DOg4Sxkn",
-                "lkOWnEbCD+XTnbDd",
-                "=agR5",
-                "-----END PGP MESSAGE-----",
-                "",
-                "",
-                "--YFrteb74qSXmggbOxZL9dRnhymywAi--",
-                "",
-                "",
-            ]
-        ).encode()
-    )
-
-    assert not check_encrypted_bstr(
-        "\r\n".join(
-            [
-                "Subject: Message opened",
-                "From: <barbaz@c2.testrun.org>",
-                "To: <foobar@c2.testrun.org>",
-                "Date: Sun, 15 Oct 2023 16:43:25 +0000",
-                "Message-ID: <Mr.78MWtlV7RAi.goCFzBhCYfy@c2.testrun.org>",
-                "Auto-Submitted: auto-replied",
-                "Chat-Version: 1.0",
-                "MIME-Version: 1.0",
-                "Content-Type: multipart/report; report-type=disposition-notification;",
-                '\tboundary="Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi"',
-                "",
-                "",
-                "--Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi",
-                "Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no",
-                "",
-                'The "Hi!" message you sent was displayed on the screen of the recipient.',
-                "",
-                "This is no guarantee the content was read.",
-                "",
-                "",
-                "--Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi",
-                "Content-Type: message/disposition-notification",
-                "",
-                "Reporting-UA: Delta Chat 1.124.1",
-                "Original-Recipient: rfc822;barbaz@c2.testrun.org",
-                "Final-Recipient: rfc822;barbaz@c2.testrun.org",
-                "Original-Message-ID: <Mr.MvmCz-GQbi_.6FGRkhDf05c@c2.testrun.org>",
-                "Disposition: manual-action/MDN-sent-automatically; displayed",
-                "",
-                "",
-                "--Gl92xgZjOShJ5PGHntqYkoo2OK2Dvi--",
-                "",
-                "",
-            ]
-        ).encode()
-    )

deploy-chatmail/src/deploy_chatmail/dovecot/auth.conf

@@ -1,5 +0,0 @@
-uri = proxy:/run/dovecot/doveauth.socket:auth
-iterate_disable = yes
-default_pass_scheme = plain
-password_key = passdb/%w/%u
-user_key = userdb/%u

deploy-chatmail/src/deploy_chatmail/dovecot/expunge.cron

@@ -1,4 +0,0 @@
-2 0 * * * dovecot doveadm expunge -A SEEN BEFORE 40d INBOX
-2 0 * * * dovecot doveadm expunge -A SEEN BEFORE 40d Deltachat
-2 0 * * * dovecot doveadm expunge -A SEEN BEFORE 40d Trash
-2 30 * * * dovecot doveadm purge -A

deploy.py

@@ -1,6 +1,6 @@
 import os
 import pyinfra
-from deploy_chatmail import deploy_chatmail
+from chatmail import deploy_chatmail
 
 
 def main():

doveauth/README.md

@@ -0,0 +1,7 @@
+# doveauth
+
+doveauth is a python tool
+to create dovecot users on login.
+It is called by the
+[dovecot lua authentication module](https://doc.dovecot.org/configuration_manual/authentication/lua_based_authentication/)
+

doveauth/pyproject.toml

@@ -3,15 +3,11 @@ requires = ["setuptools>=45"]
 build-backend = "setuptools.build_meta"
 
 [project]
-name = "chatmaild"
+name = "doveauth"
 version = "0.1"
-dependencies = [
-  "aiosmtpd"
-]
 
 [project.scripts]
-doveauth-dictproxy = "chatmaild.dictproxy:main"
-filtermail = "chatmaild.filtermail:main"
+doveauth = "doveauth.doveauth:main"
 
 [tool.pytest.ini_options]
 addopts = "-v -ra --strict-markers"

doveauth/src/doveauth/database.py

@@ -35,7 +35,7 @@ class Connection:
 
     def create_user(self, addr: str, password: str):
         """Create a row in the users table."""
-        self.execute("PRAGMA foreign_keys=on")
+        self.execute("PRAGMA foreign_keys=on;")
         q = """INSERT INTO users (addr, password, last_login)
                VALUES (?, ?, ?)"""
         self.execute(q, (addr, password, int(time.time())))
@@ -53,15 +53,30 @@ class Connection:
             )
         return result
 
+    def set_config(self, name: str, value: str) -> str:
+        ok = [
+            "dbversion",
+        ]
+        assert name in ok, name
+        q = "INSERT OR REPLACE INTO config (key, value) VALUES (?, ?)"
+        self.cursor().execute(q, (name, value)).fetchone()
+        return value
+
+    def get_config(self, key: str) -> str:
+        q = "SELECT key, value from config WHERE name = ?"
+        c = self._sqlconn.cursor()
+        try:
+            return c.execute(q, key).fetchone()
+        except sqlite3.OperationalError:
+            return None
+
 
 class Database:
     def __init__(self, path: str):
         self.path = Path(path)
         self.ensure_tables()
 
-    def _get_connection(
-        self, write=False, transaction=False, closing=False
-    ) -> Connection:
+    def _get_connection(self, write=False, transaction=False, closing=False) -> Connection:
         # we let the database serialize all writers at connection time
         # to play it very safe (we don't have massive amounts of writes).
         mode = "ro"
@@ -114,20 +129,10 @@ class Database:
     def read_connection(self, closing=True) -> Connection:
         return self._get_connection(closing=closing, write=False)
 
-    def get_schema_version(self) -> int:
-        with self.read_connection() as conn:
-            dbversion = conn.execute("PRAGMA user_version").fetchone()[0]
-        return dbversion
-
     CURRENT_DBVERSION = 1
 
     def ensure_tables(self):
         with self.write_transaction() as conn:
-            if self.get_schema_version() > 1:
-                raise DBError(
-                    "version is %s; downgrading schema is not supported"
-                    % (self.get_schema_version(),)
-                )
             conn.execute(
                 """
                 CREATE TABLE IF NOT EXISTS users (
@@ -137,4 +142,12 @@ class Database:
                 )
             """,
             )
-            conn.execute("PRAGMA user_version=%s" % (self.CURRENT_DBVERSION,))
+            conn.execute(
+                """
+                CREATE TABLE IF NOT EXISTS config (
+                    key TEXT PRIMARY KEY,
+                    value TEXT
+                )
+            """,
+            )
+            conn.set_config("dbversion", self.CURRENT_DBVERSION)

doveauth/src/doveauth/doveauth.lua

@@ -0,0 +1,59 @@
+
+-- Escape shell argument by hex encoding it and wrapping in quotes.
+function escape(data)
+   b16 = data:gsub(".", function(char) return string.format("%2X", char:byte()) end)
+   return ("'"..b16.."'")
+end
+
+-- call out to python program to actually manage authentication for dovecot
+
+function chatctl_verify(user, password)
+    local cmd = "doveauth hexauth "..escape(user).." "..escape(password)
+    print("executing: "..cmd)
+    local handle = io.popen(cmd)
+    local result = handle:read("*a")
+    handle:close()
+    return split_chatctl(result)
+end
+
+function chatctl_lookup(user) 
+    local cmd = "doveauth hexlookup "..escape(user)
+    assert(user)
+    print("executing: "..cmd)
+    local handle = io.popen(cmd)
+    local result = handle:read("*a")
+    handle:close()
+    return split_chatctl(result)
+end
+
+function get_extra_dovecot_output(res)
+    return {home=res.home, uid=res.uid, gid=res.gid}
+end
+
+
+function auth_password_verify(request, password)
+    local res = chatctl_verify(request.user, password)
+    -- request:log_error("auth_password_verify "..request.user.." "..password)
+    if res.status == "ok" then 
+        local extra = get_extra_dovecot_output(res)
+        return dovecot.auth.PASSDB_RESULT_OK, get_extra_dovecot_output(res)
+    end
+    return dovecot.auth.PASSDB_RESULT_PASSWORD_MISMATCH, ""
+end
+
+
+function auth_userdb_lookup(request)
+    local res = chatctl_lookup(request.user) 
+    if res.status == "ok" then
+        return dovecot.auth.USERDB_RESULT_OK, get_extra_dovecot_output(res)
+    end
+    return dovecot.auth.USERDB_RESULT_USER_UNKNOWN, "no such user"
+end
+
+function split_chatctl(output) 
+    local ret = {}
+    for key, value in output:gmatch "(%w+)%s*=%s*(%w+)" do
+        ret[key] = value
+    end
+    return ret
+end

doveauth/src/doveauth/doveauth.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+import base64
+import sys
+
+from .database import Database
+
+
+def get_user_data(db, user):
+    with db.read_connection() as conn:
+        result = conn.get_user(user)
+    if result:
+        result['uid'] = "vmail"
+        result['gid'] = "vmail"
+    return result
+
+
+def create_user(db, user, password):
+    with db.write_transaction() as conn:
+        conn.create_user(user, password)
+    return dict(home=f"/home/vmail/{user}", uid="vmail", gid="vmail", password=password)
+
+
+def verify_user(db, user, password):
+    userdata = get_user_data(db, user)
+    if userdata:
+        if userdata.get("password") == password:
+            userdata["status"] = "ok"
+        else:
+            userdata["status"] = "fail"
+    else:
+        userdata = create_user(db, user, password)
+        userdata["status"] = "ok"
+
+    return userdata
+
+
+def lookup_user(db, user):
+    userdata = get_user_data(db, user)
+    if userdata:
+        userdata["status"] = "ok"
+    else:
+        userdata["status"] = "fail"
+    return userdata
+
+
+def dump_result(res):
+    for key, value in res.items():
+        print(f"{key}={value}")
+
+
+def main():
+    db = Database("/home/vmail/passdb.sqlite")
+    if sys.argv[1] == "hexauth":
+        login = base64.b16decode(sys.argv[2]).decode()
+        password = base64.b16decode(sys.argv[3]).decode()
+        res = verify_user(db, login, password)
+        dump_result(res)
+    elif sys.argv[1] == "hexlookup":
+        login = base64.b16decode(sys.argv[2]).decode()
+        res = lookup_user(db, login)
+        dump_result(res)
+
+
+if __name__ == "__main__":
+    main()

doveauth/src/doveauth/test_doveauth.lua

@@ -0,0 +1,78 @@
+
+require "doveauth"
+
+-- simulate dovecot defined result codes 
+
+dovecot = {
+    auth = {
+        PASSDB_RESULT_OK="PASSWORD-OK",
+        PASSDB_RESULT_PASSWORD_MISMATCH="PASSWORD-MISMATCH",
+        USERDB_RESULT_OK="USERDB-OK",
+        USERDB_RESULT_USER_UNKNOWN="USERDB-UNKNOWN"
+    }
+}
+
+
+-- Tests for testing the lua<->python interaction 
+
+function test_password_verify_ok(user, password) 
+    local res, extra = auth_password_verify({user=user}, password)
+    assert(res==dovecot.auth.PASSDB_RESULT_OK)
+    assert(extra.uid == "vmail")
+    assert(extra.gid == "vmail")
+    -- assert(extra.homedir == "/home/vmail/link2xt")
+    print("OK test_password_verify_ok "..user.." "..password)
+end
+
+function test_password_verify_mismatch(user, password) 
+    local res = auth_password_verify({user=user}, password)
+    assert(res == dovecot.auth.PASSDB_RESULT_PASSWORD_MISMATCH)
+    print("OK test_password_verify_mismatch "..user.." "..password)
+end
+
+function test_userdb_lookup_ok(user)
+    local res, extra = auth_userdb_lookup({user=user})
+    assert(extra.uid == "vmail")
+    assert(extra.gid == "vmail")
+    assert(res == dovecot.auth.USERDB_RESULT_OK)
+    print("OK test_userdb_lookup_ok "..user)
+end
+
+function test_userdb_lookup_mismatch(user)
+    local res, extra = auth_userdb_lookup({user=user})
+    assert(res == dovecot.auth.USERDB_RESULT_USER_UNKNOWN)
+    print("OK test_userdb_lookup_mismatch "..user)
+end
+
+function test_passdb_lookup_ok(user)
+    local res, extra = auth_passdb_lookup({user=user})
+    assert(extra.uid == "vmail")
+    assert(extra.gid == "vmail")
+    assert(res == dovecot.auth.PASSDB_RESULT_OK)
+    print("OK test_passdb_lookup_ok "..user)
+end
+
+function test_passdb_lookup_mismatch(user)
+    local res, extra = auth_passdb_lookup({user=user})
+    assert(res == dovecot.auth.PASSDB_RESULT_USER_UNKNOWN)
+    print("OK test_passdb_lookup_mismatch "..user)
+end
+
+function test_split_chatctl()
+    local res = split_chatctl("a=3 b=4\nc=5")
+    assert(res["a"] == "3")
+    assert(res["b"] == "4")
+    assert(res["c"] == "5")
+    print("OK test_split_chatctl")
+end 
+
+test_split_chatctl()
+test_password_verify_ok("link2xt@c1.testrun.org", "Ahyei6ie")
+test_password_verify_mismatch("link2xt@c1.testrun.org", "Aqwlek")
+test_userdb_lookup_ok("link2xt@c1.testrun.org")
+test_userdb_lookup_mismatch("wlekqjlew@xyz.org")
+
+-- probably not needed by dovecot? 
+-- test_passdb_lookup_ok("link2xt@c1.testrun.org")
+-- test_passdb_lookup_mismatch("llqkwjelqwe@xyz.org")
+

doveauth/src/doveauth/test_doveauth.py

@@ -0,0 +1,25 @@
+import subprocess
+import pytest
+
+from doveauth.doveauth import get_user_data, verify_user, Database
+
+
+def test_basic(tmpdir):
+    db = Database(tmpdir / "passdb.sqlite")
+    verify_user(db, "link2xt@c1.testrun.org", "asdf")
+    data = get_user_data(db, "link2xt@c1.testrun.org")
+    assert data
+
+
+def test_verify_or_create(tmpdir):
+    db = Database(tmpdir / "passdb.sqlite")
+    res = verify_user(db, "newuser1@something.org", "kajdlkajsldk12l3kj1983")
+    assert res["status"] == "ok"
+    res = verify_user(db, "newuser1@something.org", "kajdlqweqwe")
+    assert res["status"] == "fail"
+
+
+def test_lua_integration(request):
+    p = request.fspath.dirpath("test_doveauth.lua")
+    proc = subprocess.run(["lua", str(p)])
+    assert proc.returncode == 0

online-tests/benchmark.py

@@ -1,34 +0,0 @@
-def test_tls_serialized_connect(benchmark, imap_or_smtp):
-    def connect():
-        imap_or_smtp.connect()
-
-    benchmark(connect)
-
-
-def test_login(benchmark, imap_or_smtp, gencreds):
-    cls = imap_or_smtp.__class__
-    conns = []
-    for i in range(20):
-        conn = cls(imap_or_smtp.host)
-        conn.connect()
-        conns.append(conn)
-
-    def login():
-        conn = conns.pop()
-        conn.login(*gencreds())
-
-    benchmark(login)
-
-
-def test_send_and_receive_10(benchmark, cmfactory, lp):
-    """send many messages between two accounts"""
-    ac1, ac2 = cmfactory.get_online_accounts(2)
-    chat = cmfactory.get_accepted_chat(ac1, ac2)
-
-    def send_10_receive_all():
-        for i in range(10):
-            chat.send_text(f"hello {i}")
-        for i in range(10):
-            ac2.wait_next_incoming_message()
-
-    benchmark(send_10_receive_all)

online-tests/conftest.py

@@ -1,201 +0,0 @@
-import os
-import io
-import random
-import subprocess
-import imaplib
-import smtplib
-import itertools
-import pytest
-
-
-def pytest_addoption(parser):
-    parser.addoption(
-        "--slow", action="store_true", default=False, help="also run slow tests"
-    )
-
-
-def pytest_runtest_setup(item):
-    markers = list(item.iter_markers(name="slow"))
-    if markers:
-        if not item.config.getoption("--slow"):
-            pytest.skip("skipping slow test, use --slow to run")
-
-
-@pytest.fixture
-def maildomain():
-    domain = os.environ.get("CHATMAIL_DOMAIN")
-    if not domain:
-        pytest.skip("set CHATMAIL_DOMAIN to a ssh-reachable chatmail instance")
-    return domain
-
-
-@pytest.fixture
-def sshdomain(maildomain):
-    return os.environ.get("CHATMAIL_SSH", maildomain)
-
-
-@pytest.fixture
-def maildomain2():
-    domain = os.environ.get("CHATMAIL_DOMAIN2")
-    if not domain:
-        pytest.skip("set CHATMAIL_DOMAIN2 to a ssh-reachable chatmail instance")
-    return domain
-
-
-@pytest.fixture
-def sshdomain2(maildomain2):
-    return os.environ.get("CHATMAIL_SSH2", maildomain2)
-
-
-def pytest_report_header():
-    domain = os.environ.get("CHATMAIL_DOMAIN")
-    if domain:
-        text = f"chatmail test instance: {domain}"
-        return ["-" * len(text), text, "-" * len(text)]
-
-
-@pytest.fixture
-def imap(maildomain):
-    return ImapConn(maildomain)
-
-
-class ImapConn:
-    AuthError = imaplib.IMAP4.error
-    logcmd = "journalctl -f -u dovecot"
-    name = "dovecot"
-
-    def __init__(self, host):
-        self.host = host
-
-    def connect(self):
-        print(f"imap-connect {self.host}")
-        self.conn = imaplib.IMAP4_SSL(self.host)
-
-    def login(self, user, password):
-        print(f"imap-login {user!r} {password!r}")
-        self.conn.login(user, password)
-
-
-@pytest.fixture
-def smtp(maildomain):
-    return SmtpConn(maildomain)
-
-
-class SmtpConn:
-    AuthError = smtplib.SMTPAuthenticationError
-    logcmd = "journalctl -f -t postfix/smtpd -t postfix/smtp -t postfix/lmtp"
-    name = "postfix"
-
-    def __init__(self, host):
-        self.host = host
-
-    def connect(self):
-        print(f"smtp-connect {self.host}")
-        self.conn = smtplib.SMTP_SSL(self.host)
-
-    def login(self, user, password):
-        print(f"smtp-login {user!r} {password!r}")
-        self.conn.login(user, password)
-
-
-@pytest.fixture(params=["imap", "smtp"])
-def imap_or_smtp(request):
-    return request.getfixturevalue(request.param)
-
-
-@pytest.fixture
-def gencreds(maildomain):
-    count = itertools.count()
-    next(count)
-
-    def gen(domain=None):
-        domain = domain if domain else maildomain
-        while 1:
-            num = next(count)
-            alphanumeric = "abcdefghijklmnopqrstuvwxyz1234567890"
-            user = "".join(random.choices(alphanumeric, k=10))
-            user = f"ac{num}_{user}"
-            password = "".join(random.choices(alphanumeric, k=10))
-            yield f"{user}@{domain}", f"{password}"
-
-    return lambda domain=None: next(gen(domain))
-
-
-#
-# Delta Chat testplugin re-use
-# use the cmfactory fixture to get chatmail instance accounts
-#
-
-
-class ChatmailTestProcess:
-    """Provider for chatmail instance accounts as used by deltachat.testplugin.acfactory"""
-
-    def __init__(self, pytestconfig, maildomain, gencreds):
-        self.pytestconfig = pytestconfig
-        self.maildomain = maildomain
-        assert "." in self.maildomain, maildomain
-        self.gencreds = gencreds
-        self._addr2files = {}
-
-    def get_liveconfig_producer(self):
-        while 1:
-            user, password = self.gencreds(self.maildomain)
-            config = {
-                "addr": user,
-                "mail_pw": password,
-            }
-            # speed up account configuration
-            config["mail_server"] = self.maildomain
-            config["send_server"] = self.maildomain
-            yield config
-
-    def cache_maybe_retrieve_configured_db_files(self, cache_addr, db_target_path):
-        pass
-
-    def cache_maybe_store_configured_db_files(self, acc):
-        pass
-
-
-@pytest.fixture
-def cmfactory(request, gencreds, tmpdir, data, maildomain):
-    # cloned from deltachat.testplugin.amfactory
-    pytest.importorskip("deltachat")
-    from deltachat.testplugin import ACFactory
-
-    testproc = ChatmailTestProcess(request.config, maildomain, gencreds)
-    am = ACFactory(request=request, tmpdir=tmpdir, testprocess=testproc, data=data)
-
-    # nb. a bit hacky
-    # would probably be better if deltachat's test machinery grows native support
-    def switch_maildomain(maildomain2):
-        am.testprocess.maildomain = maildomain2
-
-    am.switch_maildomain = switch_maildomain
-
-    yield am
-    if hasattr(request.node, "rep_call") and request.node.rep_call.failed:
-        if testproc.pytestconfig.getoption("--extra-info"):
-            logfile = io.StringIO()
-            am.dump_imap_summary(logfile=logfile)
-            print(logfile.getvalue())
-            # request.node.add_report_section("call", "imap-server-state", s)
-
-
-@pytest.fixture
-def remote(sshdomain):
-    return Remote(sshdomain)
-
-
-class Remote:
-    def __init__(self, sshdomain):
-        self.sshdomain = sshdomain
-
-    def iter_output(self, logcmd=""):
-        getjournal = f"journalctl -f" if not logcmd else logcmd
-        self.popen = subprocess.Popen(
-            ["ssh", f"root@{self.sshdomain}", getjournal],
-            stdout=subprocess.PIPE,
-        )
-        while 1:
-            line = self.popen.stdout.readline()
-            yield line.decode().strip().lower()

online-tests/pytest.ini

@@ -1,3 +0,0 @@
-[pytest]
-addopts = -vrsx --strict-markers
-markers = slow: mark test as slow (requires --slow option to run)

online-tests/test_0_basic.py

@@ -1,15 +0,0 @@
-def test_remote(remote, imap_or_smtp):
-    lineproducer = remote.iter_output(imap_or_smtp.logcmd)
-    imap_or_smtp.connect()
-    assert imap_or_smtp.name in next(lineproducer)
-
-
-def test_use_two_chatmailservers(cmfactory, maildomain2):
-    ac1 = cmfactory.new_online_configuring_account(cache=False)
-    cmfactory.switch_maildomain(maildomain2)
-    ac2 = cmfactory.new_online_configuring_account(cache=False)
-    cmfactory.bring_accounts_online()
-    cmfactory.get_accepted_chat(ac1, ac2)
-    domain1 = ac1.get_config("addr").split("@")[1]
-    domain2 = ac2.get_config("addr").split("@")[1]
-    assert domain1 != domain2

online-tests/test_0_login.py

@@ -1,36 +0,0 @@
-import pytest
-
-
-def test_login_basic_functioning(imap_or_smtp, gencreds, lp):
-    """Test a) that an initial login creates a user automatically
-    and b) verify we can also login a second time with the same password
-    and c) that using a different password fails the login."""
-    user, password = gencreds()
-    lp.sec(f"login first time with {user} {password}")
-    imap_or_smtp.connect()
-    imap_or_smtp.login(user, password)
-    lp.indent("success")
-
-    lp.sec(f"reconnect and login second time {user} {password}")
-    imap_or_smtp.connect()
-    imap_or_smtp.login(user, password)
-    imap_or_smtp.connect()
-    lp.sec("success")
-
-    lp.sec(f"reconnect and verify wrong password fails {user} ")
-    imap_or_smtp.connect()
-    with pytest.raises(imap_or_smtp.AuthError):
-        imap_or_smtp.login(user, password + "wrong")
-
-
-def test_login_same_password(imap_or_smtp, gencreds):
-    """Test two different users logging in with the same password
-    to ensure that authentication process does not confuse the users
-    by using only the password hash as a key.
-    """
-    user1, password1 = gencreds()
-    user2, _ = gencreds()
-    imap_or_smtp.connect()
-    imap_or_smtp.login(user1, password1)
-    imap_or_smtp.connect()
-    imap_or_smtp.login(user2, password1)

online-tests/test_1_deltachat.py

@@ -1,83 +0,0 @@
-import random
-import pytest
-
-
-class TestEndToEndDeltaChat:
-    "Tests that use Delta Chat accounts on the chat mail instance."
-
-    def test_one_on_one(self, cmfactory, lp):
-        """Test that a DC account can send a message to a second DC account
-        on the same chat-mail instance."""
-        ac1, ac2 = cmfactory.get_online_accounts(2)
-        chat = cmfactory.get_accepted_chat(ac1, ac2)
-
-        lp.sec("ac1: prepare and send text message to ac2")
-        chat.send_text("message0")
-
-        lp.sec("wait for ac2 to receive message")
-        msg2 = ac2._evtracker.wait_next_incoming_message()
-        assert msg2.text == "message0"
-
-    @pytest.mark.slow
-    def test_exceed_quota(self, cmfactory, lp, tmpdir, remote):
-        """This is a very slow test as it needs to upload >100MB of mail data
-        before quota is exceeded, and thus depends on the speed of the upload.
-        """
-        ac1, ac2 = cmfactory.get_online_accounts(2)
-        chat = cmfactory.get_accepted_chat(ac1, ac2)
-
-        quota = 1024 * 1024 * 100
-        attachsize = 1 * 1024 * 1024
-        num_to_send = quota // attachsize + 2
-        lp.sec(f"ac1: send {num_to_send} large files to ac2")
-        lp.indent(f"per-user quota is assumed to be: {quota/(1024*1024)}MB")
-        alphanumeric = "abcdefghijklmnopqrstuvwxyz1234567890"
-        msgs = []
-        for i in range(num_to_send):
-            attachment = tmpdir / f"attachment{i}"
-            data = "".join(random.choice(alphanumeric) for i in range(1024))
-            with open(attachment, "w+") as f:
-                for j in range(attachsize // len(data)):
-                    f.write(data)
-
-            msg = chat.send_file(str(attachment))
-            msgs.append(msg)
-            lp.indent(f"Sent out msg {i}, size {attachsize/(1024*1024)}MB")
-
-        lp.sec("ac2: check messages are arriving until quota is reached")
-
-        addr = ac2.get_config("addr").lower()
-        saved_ok = 0
-        for line in remote.iter_output("journalctl -f -u dovecot"):
-            if addr not in line:
-                # print(line)
-                continue
-            if "quota" in line:
-                if "quota exceeded" in line:
-                    if saved_ok < num_to_send // 2:
-                        pytest.fail(
-                            f"quota exceeded too early: after {saved_ok} messages already"
-                        )
-                    lp.indent("good, message sending failed because quota was exceeded")
-                    return
-            if "saved mail to inbox" in line:
-                saved_ok += 1
-                print(f"{saved_ok}: {line}")
-                if saved_ok >= num_to_send:
-                    break
-
-        pytest.fail("sending succeeded although messages should exceed quota")
-
-    def test_securejoin(self, cmfactory, lp, maildomain2):
-        ac1 = cmfactory.new_online_configuring_account(cache=False)
-        cmfactory.switch_maildomain(maildomain2)
-        ac2 = cmfactory.new_online_configuring_account(cache=False)
-        cmfactory.bring_accounts_online()
-
-        lp.sec("ac1: create QR code and let ac2 scan it, starting the securejoin")
-        qr = ac1.get_setup_contact_qr()
-
-        lp.sec("ac2: start QR-code based setup contact protocol")
-        ch = ac2.qr_setup_contact(qr)
-        assert ch.id >= 10
-        ac1._evtracker.wait_securejoin_inviter_progress(1000)

plan.txt

@@ -2,58 +2,30 @@
 
 ## Dovecot goals/steps
 
-- automatic expiry of messages older than M days
-   - also expunge unread messages
+1. create-user-on-login ("doveauth")
 
-- limit: configure max-connections per account
+2. per-user quota (adaptive)
+
+3. automatic expiry of messages older than M days
+
+4. automatic expiry of users that haven't logged in for N days
 
 
-## nami: send out rate limit / rspamd
+## Postfix goals/steps
 
-- basic outgoing send rate/limits (depending on "account-rating")
-  use rspamd in a minimal way, check support dkim-signing
-  (including an online test exceeding rate limit)
+1. block all outgoing mails with our own LMTP program
 
+2. only allow (outgoing) mails if secure-join or autocrypt-pgp-encrypted format
+   (probably via an lmtp service)
 
-## doveauth questions/futures
-
-- bcrypt-password scheme is slow: require long passwords, use faster hashing
-
-- define user-name and password policies, and implement them
-  (be very restrictive at the beginning, we can relax later)
-
-- password is part of the dictproxy-lookup key, is it safe to use auth-caching?
-
-
-## How to limit creation of accounts?
-
-attack: a 3-line bash script to fill the chatmail db with millions of unused accouts
-
-- make it computationally expensive (somehow try to except our tests from it)
-  1st pass instant onboarding: create userid + cheap password -- if it fails then
-  2nd pass instant onboarding: create userdid + comput. expensive password
-
-- probably also do firewall: limit number of new tcp-connections per IP address per duration
-
-
-## Open/deferred questions
-
-- automatic expiry of users that haven't logged in for N days
-  Is it neccessary?  If all messages are gone, does the existence of
-  an e-mail address bother anybody?
-
-
-## web page for chat-mail servers?
-
-- documentation for users, privacy policy etc.
-  (probably also with provider-messages ...)
+3. basic outgoing send rate/limits (depending on "account-rating")
 
 
 ## online tests (first with plain python/pytest)
 
 - write tests for dovecot login (exists)
-- write tests for postfix logins (exists)
-- write A<>B send/receive tests (exists)
+- write tests for postfix logins
+- write A<>B send/receive tests
 
 
 ## Delta Chat

scripts/bench.sh

@@ -1,4 +0,0 @@
-#!/bin/bash
-set -e
-
-online-tests/venv/bin/pytest online-tests/benchmark.py -vrx 

scripts/deploy.sh

@@ -1,10 +1,7 @@
 #!/usr/bin/env bash
 : ${CHATMAIL_DOMAIN:=c1.testrun.org}
 export CHATMAIL_DOMAIN
-
-chatmaild/venv/bin/python3 -m build -n --sdist chatmaild --outdir dist
-
-deploy-chatmail/venv/bin/pyinfra --ssh-user root "$CHATMAIL_DOMAIN" \
-    deploy-chatmail/src/deploy_chatmail/deploy.py
-
+cd doveauth
+venv/bin/python3 -m build
+../chatmail-pyinfra/venv/bin/pyinfra --ssh-user root "$CHATMAIL_DOMAIN" ../deploy.py
 rm -r dist/

scripts/get_imap_capabilities.py

@@ -1,14 +0,0 @@
-import os
-import time
-import imaplib
-
-domain = os.environ.get("CHATMAIL_DOMAIN", "c3.testrun.org")
-
-print("connecting")
-conn = imaplib.IMAP4_SSL(domain)
-print("logging in")
-conn.login(f"imapcapa", "pass")
-status, res = conn.capability()
-for capa in sorted(res[0].decode().split()):
-    print(capa)
-

scripts/init.sh

@@ -1,14 +1,8 @@
 #!/bin/sh
-set -e
-python3 -m venv deploy-chatmail/venv
-deploy-chatmail/venv/bin/pip install pyinfra pytest
-deploy-chatmail/venv/bin/pip install -e deploy-chatmail
-deploy-chatmail/venv/bin/pip install -e chatmaild
-
-python3 -m venv chatmaild/venv
-sudo apt install -y dovecot-core && sudo systemctl disable --now dovecot
-chatmaild/venv/bin/pip install --upgrade pytest build 'setuptools>=68'
-chatmaild/venv/bin/pip install -e chatmaild
-
-python3 -m venv online-tests/venv
-online-tests/venv/bin/pip install pytest pytest-timeout pdbpp deltachat pytest-benchmark
+python3 -m venv chatmail-pyinfra/venv
+chatmail-pyinfra/venv/bin/pip install pyinfra pytest
+chatmail-pyinfra/venv/bin/pip install -e chatmail-pyinfra
+chatmail-pyinfra/venv/bin/pip install -e doveauth
+python3 -m venv doveauth/venv
+doveauth/venv/bin/pip install pytest build
+doveauth/venv/bin/pip install -e doveauth

scripts/measure_tls_and_logins.py

@@ -1,28 +0,0 @@
-#!/usr/bin/env python3
-import os
-import time
-import imaplib
-
-domain = os.environ.get("CHATMAIL_DOMAIN", "c3.testrun.org")
-
-NUM_CONNECTIONS=10
-
-conns = []
-
-start = time.time()
-for i in range(NUM_CONNECTIONS):
-    print(f"opening connection {i} to {domain}")
-    conn = imaplib.IMAP4_SSL(domain)
-    conns.append(conn)
-
-tlsdone = time.time()
-duration = tlsdone-start
-print(f"{duration}: TLS connections opening TLS connections")
-
-for i, conn in enumerate(conns):
-    print(f"logging into connection {i}")
-    conn.login(f"measure{i}", "pass")
-
-logindone = time.time()
-duration = logindone - tlsdone
-print(f"{duration}: LOGINS done")

scripts/remote-deploy.sh

@@ -1,8 +0,0 @@
-#!/bin/sh
-set -e
-
-: ${CHATMAIL_DOMAIN:=c1.testrun.org}
-: ${CHATMAIL_SSH:=$CHATMAIL_DOMAIN}
-
-rsync -avz . "root@$CHATMAIL_SSH:/root/chatmail" --exclude='/.git' --filter="dir-merge,- .gitignore"
-ssh "root@$CHATMAIL_SSH" "cd /root/chatmail; apt install -y python3-venv; python3 -m venv venv; venv/bin/pip install pyinfra build; venv/bin/python3 -m build -n --sdist chatmaild --outdir dist; venv/bin/pip install -e ./deploy-chatmail -e ./chatmaild; export CHATMAIL_DOMAIN=$CHATMAIL_DOMAIN; venv/bin/pyinfra @local deploy.py"

scripts/test.sh

@@ -1,3 +1,4 @@
-#!/bin/bash
-chatmaild/venv/bin/pytest chatmaild/ $@
-online-tests/venv/bin/pytest online-tests/ -vrx --durations=5 $@
+#!/bin/sh
+chatmail-pyinfra/venv/bin/pytest chatmail-pyinfra/tests
+cd doveauth/src/doveauth
+../../venv/bin/pytest