Prioritize port 443

Port 443 has more chance to work
in networks where ports other than 80 and 443
are dropped.
Otherwise user has to wait for other ports
to time out before trying port 443.

.github/workflows/staging-ipv4.testrun.org-default.zone

@@ -0,0 +1,20 @@
+;; Zone file for staging-ipv4.testrun.org
+
+$ORIGIN staging-ipv4.testrun.org.
+$TTL 300
+
+@  IN  SOA     ns.testrun.org. root.nine.testrun.org (
+  2023010101 ; Serial
+  7200       ; Refresh
+  3600       ; Retry
+  1209600    ; Expire
+  3600       ; Negative response caching TTL
+)
+
+;; Nameservers.
+@  IN NS ns.testrun.org.
+
+;; DNS records.
+@       IN      A       37.27.95.249
+mta-sts.staging-ipv4.testrun.org.    CNAME   staging-ipv4.testrun.org.
+www.staging-ipv4.testrun.org.        CNAME   staging-ipv4.testrun.org.

.github/workflows/test-and-deploy-ipv4only.yaml

@@ -0,0 +1,98 @@
+name: deploy on staging-ipv4.testrun.org, and run tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths-ignore:
+      - 'scripts/**'
+      - '**/README.md'
+      - 'CHANGELOG.md'
+      - 'LICENSE'
+
+jobs:
+  deploy:
+    name: deploy on staging-ipv4.testrun.org, and run tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    concurrency:
+      group: ci-ipv4-${{ github.workflow }}-${{ github.ref }}
+      cancel-in-progress: ${{ !contains(github.ref, '$GITHUB_REF') }}
+    steps:
+      - uses: jsok/serialize-workflow-action@v1
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions/checkout@v4
+
+      - name: prepare SSH
+        run: |
+          mkdir ~/.ssh
+          echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
+          chmod 600 ~/.ssh/id_ed25519
+          ssh-keyscan staging-ipv4.testrun.org > ~/.ssh/known_hosts
+          # save previous acme & dkim state
+          rsync -avz root@staging-ipv4.testrun.org:/var/lib/acme acme-ipv4 || true
+          rsync -avz root@staging-ipv4.testrun.org:/etc/dkimkeys dkimkeys-ipv4 || true
+          # store previous acme & dkim state on ns.testrun.org, if it contains useful certs
+          if [ -f dkimkeys-ipv4/dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys-ipv4 root@ns.testrun.org:/tmp/ || true; fi
+          if [ "$(ls -A acme-ipv4/acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme-ipv4 root@ns.testrun.org:/tmp/ || true; fi
+          # make sure CAA record isn't set
+          ssh -o StrictHostKeyChecking=accept-new root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging-ipv4.testrun.org.zone
+          ssh root@ns.testrun.org systemctl reload nsd
+
+      - name: rebuild staging-ipv4.testrun.org to have a clean VPS
+        run: |
+            curl -X POST \
+            -H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
+            -H "Content-Type: application/json" \
+            -d '{"image":"debian-12"}' \
+            "https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_SERVER_ID }}/actions/rebuild"
+
+      - run: scripts/initenv.sh
+
+      - name: append venv/bin to PATH
+        run: echo venv/bin >>$GITHUB_PATH
+
+      - name: upload TLS cert after rebuilding
+        run: |
+          echo " --- wait until staging-ipv4.testrun.org VPS is rebuilt --- "
+          rm ~/.ssh/known_hosts
+          while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u ; do sleep 1 ; done
+          ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u
+          # download acme & dkim state from ns.testrun.org
+          rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme-ipv4 acme-restore || true
+          rsync -avz root@ns.testrun.org:/tmp/dkimkeys-ipv4 dkimkeys-restore || true
+          # restore acme & dkim state to staging2.testrun.org
+          rsync -avz acme-restore/acme-ipv4/acme root@staging-ipv4.testrun.org:/var/lib/acme || true
+          rsync -avz dkimkeys-restore/dkimkeys-ipv4/dkimkeys root@staging-ipv4.testrun.org:/etc/dkimkeys || true
+          ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org chown root:root -R /var/lib/acme || true
+
+      - name: run formatting checks 
+        run: cmdeploy fmt -v 
+
+      - name: run deploy-chatmail offline tests 
+        run: pytest --pyargs cmdeploy 
+
+      - run: |
+          cmdeploy init staging-ipv4.testrun.org
+          sed -i 's#disable_ipv6 = False#disable_ipv6 = True#' chatmail.ini
+
+      - run: cmdeploy run
+
+      - name: set DNS entries
+        run: |
+          ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
+          cmdeploy dns --zonefile staging-generated.zone
+          cat staging-generated.zone >> .github/workflows/staging-ipv4.testrun.org-default.zone
+          cat .github/workflows/staging-ipv4.testrun.org-default.zone
+          scp .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
+          ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
+          ssh root@ns.testrun.org systemctl reload nsd
+
+      - name: cmdeploy test
+        run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow
+
+      - name: cmdeploy dns (try 3 times)
+        run: cmdeploy dns || cmdeploy dns || cmdeploy dns
+

CHANGELOG.md

@@ -2,18 +2,61 @@
 
 ## untagged
 
-- BREAKING: new required chatmail.ini values:
+- avoid nginx listening on ipv6 if v6 is dsiabled 
+  ([#402](https://github.com/deltachat/chatmail/pull/402))
 
-  mailboxes_dir = /home/vmail/mail/{mail_domain}
+- trigger "apt upgrade" during "cmdeploy run" 
-  passdb = /home/vmail/passdb.sqlite
+  ([#398](https://github.com/deltachat/chatmail/pull/398))
 
-  reducing hardcoding these two paths all over the files, also improving testability. 
+- drop hispanilandia passthrough address
+  ([#401](https://github.com/deltachat/chatmail/pull/401))
+
+
+## 1.4.1 2024-07-31
+
+- fix metadata dictproxy which would confuse transactions
+  resulting in missed notifications and other issues. 
+  ([#393](https://github.com/deltachat/chatmail/pull/393))
+  ([#394](https://github.com/deltachat/chatmail/pull/394))
+
+- add optional "imap_rawlog" config option. If true, 
+  .in/.out files are created in user home dirs 
+  containing the imap protocol messages. 
+  ([#389](https://github.com/deltachat/chatmail/pull/389))
+
+## 1.4.0 2024-07-28
+
+- Add `disable_ipv6` config option to chatmail.ini.
+  Required if the server doesn't have IPv6 connectivity.
+  ([#312](https://github.com/deltachat/chatmail/pull/312))
+
+- allow current K9/Thunderbird-mail releases to send encrypted messages
+  outside by accepting their localized "encrypted subject" strings. 
+  ([#370](https://github.com/deltachat/chatmail/pull/370))
+
+- Migrate and remove sqlite database in favor of password/lastlogin tracking 
+  in a user's maildir.  
+  ([#379](https://github.com/deltachat/chatmail/pull/379))
+
+- Require pyinfra V3 installed on the client side,
+  run `./scripts/initenv.sh` to upgrade locally.
+  ([#378](https://github.com/deltachat/chatmail/pull/378))
+
+- don't hardcode "/home/vmail" paths but rather set them 
+  once in the config object and use it everywhere else, 
+  thereby also improving testability.  
   ([#351](https://github.com/deltachat/chatmail/pull/351))
+  temporarily introduced obligatory "passdb_path" and "mailboxes_dir" 
+  settings but they were removed/obsoleted in 
+  ([#380](https://github.com/deltachat/chatmail/pull/380))
 
 - BREAKING: new required chatmail.ini value 'delete_inactive_users_after = 100'
   which removes users from database and mails after 100 days without any login. 
   ([#350](https://github.com/deltachat/chatmail/pull/350))
 
+- Refine DNS checking to distinguish between "required" and "recommended" settings 
+  ([#372](https://github.com/deltachat/chatmail/pull/372))
+
 - reload nginx in the acmetool cronjob
   ([#360](https://github.com/deltachat/chatmail/pull/360))
 

chatmaild/pyproject.toml

@@ -27,6 +27,7 @@ filtermail = "chatmaild.filtermail:main"
 echobot = "chatmaild.echo:main"
 chatmail-metrics = "chatmaild.metrics:main"
 delete_inactive_users = "chatmaild.delete_inactive_users:main"
+lastlogin = "chatmaild.lastlogin:main"
 
 [project.entry-points.pytest11]
 "chatmaild.testplugin" = "chatmaild.tests.plugin"

chatmaild/src/chatmaild/common_encrypted_subjects.py

@@ -0,0 +1,59 @@
+"""Generated from deltachat, draft-ietf-lamps-header-protection, and
+encrypted_subject localizations in
+https://github.com/thunderbird/thunderbird-android/
+
+"""
+
+common_encrypted_subjects = {
+    "...",
+    "[...]",
+    "암호화된 메시지",
+    "Ĉifrita mesaĝo",
+    "Courriel chiffré",
+    "Dulrituð skilaboð",
+    "Encrypted Message",
+    "Fersifere berjocht",
+    "Kemennadenn enrineget",
+    "Krüptitud kiri",
+    "Krypterat meddelande",
+    "Krypteret besked",
+    "Kryptert melding",
+    "Mensagem criptografada",
+    "Mensagem encriptada",
+    "Mensaje cifrado",
+    "Mensaxe cifrada",
+    "Mesaj Criptat",
+    "Mesazh i Fshehtëzuar",
+    "Messaggio criptato",
+    "Messaghju cifratu",
+    "Missatge encriptat",
+    "Neges wedi'i Hamgryptio",
+    "Pesan terenkripsi",
+    "Salattu viesti",
+    "Şifreli İleti",
+    "Šifrēta ziņa",
+    "Šifrirana poruka",
+    "Šifrirano sporočilo",
+    "Šifruotas laiškas",
+    "Tin nhắn được mã hóa",
+    "Titkosított üzenet",
+    "Verschlüsselte Nachricht",
+    "Versleuteld bericht",
+    "Zašifrovaná zpráva",
+    "Zaszyfrowana wiadomość",
+    "Zifratu mezua",
+    "Κρυπτογραφημένο μήνυμα",
+    "Зашифроване повідомлення",
+    "Зашифрованное сообщение",
+    "Зашыфраваны ліст",
+    "Криптирано съобщение",
+    "Шифрована порука",
+    "დაშიფრული წერილი",
+    "הודעה מוצפנת",
+    "پیام رمزنگاری‌شده",
+    "رسالة مشفّرة",
+    "എൻക്രിപ്റ്റുചെയ്‌ത സന്ദേശം",
+    "加密邮件",
+    "已加密的訊息",
+    "暗号化されたメッセージ",
+}

chatmaild/src/chatmaild/config.py

@@ -2,6 +2,10 @@ from pathlib import Path
 
 import iniconfig
 
+from chatmaild.user import User
+
+echobot_password_path = Path("/run/echobot/password")
+
 
 def read_config(inipath):
     assert Path(inipath).exists(), inipath
@@ -16,6 +20,7 @@ class Config:
         self.mail_domain = params["mail_domain"]
         self.max_user_send_per_minute = int(params["max_user_send_per_minute"])
         self.max_mailbox_size = params["max_mailbox_size"]
+        self.max_message_size = int(params.get("max_message_size", "31457280"))
         self.delete_mails_after = params["delete_mails_after"]
         self.delete_inactive_users_after = int(params["delete_inactive_users_after"])
         self.username_min_length = int(params["username_min_length"])
@@ -23,25 +28,37 @@ class Config:
         self.password_min_length = int(params["password_min_length"])
         self.passthrough_senders = params["passthrough_senders"].split()
         self.passthrough_recipients = params["passthrough_recipients"].split()
-        self.mailboxes_dir = Path(params["mailboxes_dir"].strip())
-        self.passdb_path = Path(params["passdb_path"].strip())
         self.filtermail_smtp_port = int(params["filtermail_smtp_port"])
         self.postfix_reinject_port = int(params["postfix_reinject_port"])
+        self.disable_ipv6 = params.get("disable_ipv6", "false").lower() == "true"
+        self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true"
         self.iroh_relay = params.get("iroh_relay")
         self.privacy_postal = params.get("privacy_postal")
         self.privacy_mail = params.get("privacy_mail")
         self.privacy_pdo = params.get("privacy_pdo")
         self.privacy_supervisor = params.get("privacy_supervisor")
 
+        # deprecated option
+        mbdir = params.get("mailboxes_dir", f"/home/vmail/mail/{self.mail_domain}")
+        self.mailboxes_dir = Path(mbdir.strip())
+
+        # old unused option (except for first migration from sqlite to maildir store)
+        self.passdb_path = Path(params.get("passdb_path", "/home/vmail/passdb.sqlite"))
+
     def _getbytefile(self):
         return open(self._inipath, "rb")
 
-    def get_user_maildir(self, addr):
+    def get_user(self, addr):
-        if addr and addr != "." and "/" not in addr:
+        if not addr or "@" not in addr or "/" in addr:
-            res = self.mailboxes_dir.joinpath(addr).resolve()
+            raise ValueError(f"invalid address {addr!r}")
-            if res.is_relative_to(self.mailboxes_dir):
+
-                return res
+        maildir = self.mailboxes_dir.joinpath(addr)
-        raise ValueError(f"invalid address {addr!r}")
+        if addr.startswith("echo@"):
+            password_path = echobot_password_path
+        else:
+            password_path = maildir.joinpath("password")
+
+        return User(maildir, addr, password_path, uid="vmail", gid="vmail")
 
 
 def write_initial_config(inipath, mail_domain, overrides):
@@ -54,14 +71,19 @@ def write_initial_config(inipath, mail_domain, overrides):
 
     # apply config overrides
     new_lines = []
+    extra = overrides.copy()
     for line in content.split("\n"):
         new_line = line.strip()
         if new_line and new_line[0] not in "#[":
             name, value = map(str.strip, new_line.split("=", maxsplit=1))
-            value = overrides.get(name, value)
+            value = extra.pop(name, value)
             new_line = f"{name} = {value}"
         new_lines.append(new_line)
 
+    for name, value in extra.items():
+        new_line = f"{name} = {value}"
+        new_lines.append(new_line)
+
     content = "\n".join(new_lines)
 
     # apply testrun privacy overrides

chatmaild/src/chatmaild/database.py

@@ -1,133 +0,0 @@
-import contextlib
-import sqlite3
-import time
-from pathlib import Path
-
-
-class DBError(Exception):
-    """error during an operation on the database."""
-
-
-class Connection:
-    def __init__(self, sqlconn, write):
-        self._sqlconn = sqlconn
-        self._write = write
-
-    def close(self):
-        self._sqlconn.close()
-
-    def commit(self):
-        self._sqlconn.commit()
-
-    def rollback(self):
-        self._sqlconn.rollback()
-
-    def execute(self, query, params=()):
-        cur = self.cursor()
-        try:
-            cur.execute(query, params)
-        except sqlite3.IntegrityError as e:
-            raise DBError(e)
-        return cur
-
-    def cursor(self):
-        return self._sqlconn.cursor()
-
-    def get_user(self, addr: str) -> {}:
-        """Get a row from the users table."""
-        q = "SELECT addr, password, last_login from users WHERE addr = ?"
-        row = self._sqlconn.execute(q, (addr,)).fetchone()
-        result = {}
-        if row:
-            result = dict(
-                user=row[0],
-                password=row[1],
-                last_login=row[2],
-            )
-        return result
-
-
-class Database:
-    def __init__(self, path: str):
-        self.path = Path(path)
-        self.ensure_tables()
-
-    def _get_connection(
-        self, write=False, transaction=False, closing=False
-    ) -> Connection:
-        # we let the database serialize all writers at connection time
-        # to play it very safe (we don't have massive amounts of writes).
-        mode = "ro"
-        if write:
-            mode = "rw"
-        if not self.path.exists():
-            mode = "rwc"
-        uri = "file:%s?mode=%s" % (self.path, mode)
-        sqlconn = sqlite3.connect(
-            uri,
-            timeout=60,
-            isolation_level=None if transaction else "DEFERRED",
-            uri=True,
-        )
-
-        # Enable Write-Ahead Logging to avoid readers blocking writers and vice versa.
-        if write:
-            sqlconn.execute("PRAGMA journal_mode=wal")
-
-        if transaction:
-            start_time = time.time()
-            while 1:
-                try:
-                    sqlconn.execute("begin immediate")
-                    break
-                except sqlite3.OperationalError:
-                    # another thread may be writing, give it a chance to finish
-                    time.sleep(0.1)
-                    if time.time() - start_time > 5:
-                        # if it takes this long, something is wrong
-                        raise
-        conn = Connection(sqlconn, write=write)
-        if closing:
-            conn = contextlib.closing(conn)
-        return conn
-
-    @contextlib.contextmanager
-    def write_transaction(self):
-        conn = self._get_connection(closing=False, write=True, transaction=True)
-        try:
-            yield conn
-        except Exception:
-            conn.rollback()
-            conn.close()
-            raise
-        else:
-            conn.commit()
-            conn.close()
-
-    def read_connection(self, closing=True) -> Connection:
-        return self._get_connection(closing=closing, write=False)
-
-    def get_schema_version(self) -> int:
-        with self.read_connection() as conn:
-            dbversion = conn.execute("PRAGMA user_version").fetchone()[0]
-        return dbversion
-
-    CURRENT_DBVERSION = 1
-
-    def ensure_tables(self):
-        with self.write_transaction() as conn:
-            if self.get_schema_version() > 1:
-                raise DBError(
-                    "version is %s; downgrading schema is not supported"
-                    % (self.get_schema_version(),)
-                )
-            conn.execute(
-                """
-                CREATE TABLE IF NOT EXISTS users (
-                    addr TEXT PRIMARY KEY,
-                    password TEXT,
-                    last_login INTEGER
-                )
-            """,
-            )
-            conn.execute("PRAGMA user_version=%s" % (self.CURRENT_DBVERSION,))

chatmaild/src/chatmaild/delete_inactive_users.py

@@ -2,32 +2,30 @@
 Remove inactive users
 """
 
+import os
 import shutil
 import sys
 import time
 
 from .config import read_config
-from .database import Database
-from .doveauth import iter_userdb_lastlogin_before
 
 
-def delete_inactive_users(db, config, CHUNK=100):
+def delete_inactive_users(config):
     cutoff_date = time.time() - config.delete_inactive_users_after * 86400
+    for addr in os.listdir(config.mailboxes_dir):
+        try:
+            user = config.get_user(addr)
+        except ValueError:
+            continue
 
-    old_users = iter_userdb_lastlogin_before(db, cutoff_date)
+        read_timestamp = user.get_last_login_timestamp()
-    chunks = (old_users[i : i + CHUNK] for i in range(0, len(old_users), CHUNK))
+        if read_timestamp and read_timestamp < cutoff_date:
-    for sublist in chunks:
+            path = config.mailboxes_dir.joinpath(addr)
-        for user in sublist:
+            assert path == user.maildir
-            user_mail_dir = config.get_user_maildir(user)
+            shutil.rmtree(path, ignore_errors=True)
-            shutil.rmtree(user_mail_dir, ignore_errors=True)
-
-        with db.write_transaction() as conn:
-            for user in sublist:
-                conn.execute("DELETE FROM users WHERE addr = ?", (user,))
 
 
 def main():
     (cfgpath,) = sys.argv[1:]
     config = read_config(cfgpath)
-    db = Database(config.passdb_path)
+    delete_inactive_users(config)
-    delete_inactive_users(db, config)

chatmaild/src/chatmaild/dictproxy.py

@@ -0,0 +1,94 @@
+import logging
+import os
+from socketserver import StreamRequestHandler, ThreadingUnixStreamServer
+
+
+class DictProxy:
+    def loop_forever(self, rfile, wfile):
+        # Transaction storage is local to each handler loop.
+        # Dovecot reuses transaction IDs across connections,
+        # starting transaction with the name `1`
+        # on two different connections to the same proxy sometimes.
+        transactions = {}
+
+        while True:
+            msg = rfile.readline().strip().decode()
+            if not msg:
+                break
+
+            res = self.handle_dovecot_request(msg, transactions)
+            if res:
+                wfile.write(res.encode("ascii"))
+                wfile.flush()
+
+    def handle_dovecot_request(self, msg, transactions):
+        # see https://doc.dovecot.org/developer_manual/design/dict_protocol/#dovecot-dict-protocol
+        short_command = msg[0]
+        parts = msg[1:].split("\t")
+
+        if short_command == "L":
+            return self.handle_lookup(parts)
+        elif short_command == "I":
+            return self.handle_iterate(parts)
+        elif short_command == "H":
+            return  # no version checking
+
+        if short_command not in ("BSC"):
+            logging.warning(f"unknown dictproxy request: {msg!r}")
+            return
+
+        transaction_id = parts[0]
+
+        if short_command == "B":
+            return self.handle_begin_transaction(transaction_id, parts, transactions)
+        elif short_command == "C":
+            return self.handle_commit_transaction(transaction_id, parts, transactions)
+        elif short_command == "S":
+            addr = transactions[transaction_id]["addr"]
+            if not self.handle_set(addr, parts):
+                transactions[transaction_id]["res"] = "F\n"
+                logging.error(f"dictproxy-set failed for {addr!r}: {msg!r}")
+
+    def handle_lookup(self, parts):
+        logging.warning(f"lookup ignored: {parts!r}")
+        return "N\n"
+
+    def handle_iterate(self, parts):
+        # Empty line means ITER_FINISHED.
+        # If we don't return empty line Dovecot will timeout.
+        return "\n"
+
+    def handle_begin_transaction(self, transaction_id, parts, transactions):
+        addr = parts[1]
+        transactions[transaction_id] = dict(addr=addr, res="O\n")
+
+    def handle_set(self, addr, parts):
+        # For documentation on key structure see
+        # https://github.com/dovecot/core/blob/main/src/lib-storage/mailbox-attribute.h
+        return False
+
+    def handle_commit_transaction(self, transaction_id, parts, transactions):
+        # return whatever "set" command(s) set as result.
+        return transactions.pop(transaction_id)["res"]
+
+    def serve_forever_from_socket(self, socket):
+        dictproxy = self
+
+        class Handler(StreamRequestHandler):
+            def handle(self):
+                try:
+                    dictproxy.loop_forever(self.rfile, self.wfile)
+                except Exception:
+                    logging.exception("Exception in the handler")
+                    raise
+
+        try:
+            os.unlink(socket)
+        except FileNotFoundError:
+            pass
+
+        with ThreadingUnixStreamServer(socket, Handler) as server:
+            try:
+                server.serve_forever()
+            except KeyboardInterrupt:
+                pass

chatmaild/src/chatmaild/doveauth.py

@@ -3,24 +3,14 @@ import json
 import logging
 import os
 import sys
-import time
-from pathlib import Path
-from socketserver import (
-    StreamRequestHandler,
-    ThreadingMixIn,
-    UnixStreamServer,
-)
 
 from .config import Config, read_config
-from .database import Database
+from .dictproxy import DictProxy
+from .migrate_db import migrate_from_db_to_maildir
 
 NOCREATE_FILE = "/etc/chatmail-nocreate"
 
 
-class UnknownCommand(ValueError):
-    """dictproxy handler received an unkown command"""
-
-
 def encrypt_password(password: str):
     # https://doc.dovecot.org/configuration_manual/authentication/password_schemes/
     passhash = crypt.crypt(password, crypt.METHOD_SHA512)
@@ -65,93 +55,6 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
     return True
 
 
-def get_user_data(db, config: Config, user):
-    if user == f"echo@{config.mail_domain}":
-        return dict(
-            home=str(config.get_user_maildir(user)),
-            uid="vmail",
-            gid="vmail",
-        )
-
-    with db.read_connection() as conn:
-        result = conn.get_user(user)
-    if result:
-        result["home"] = str(config.get_user_maildir(user))
-        result["uid"] = "vmail"
-        result["gid"] = "vmail"
-    return result
-
-
-def lookup_userdb(db, config: Config, user):
-    return get_user_data(db, config, user)
-
-
-def lookup_passdb(db, config: Config, user, cleartext_password, last_login=None):
-    if user == f"echo@{config.mail_domain}":
-        # Echobot writes password it wants to log in with into /run/echobot/password
-        try:
-            password = Path("/run/echobot/password").read_text()
-        except Exception:
-            logging.exception("Exception when trying to read /run/echobot/password")
-            return None
-
-        return dict(
-            home=str(config.get_user_maildir(user)),
-            uid="vmail",
-            gid="vmail",
-            password=encrypt_password(password),
-        )
-
-    if last_login is None:
-        last_login = time.time()
-    last_login = int(last_login)
-
-    with db.write_transaction() as conn:
-        userdata = conn.get_user(user)
-        if userdata:
-            # Update last login time.
-            conn.execute(
-                "UPDATE users SET last_login=? WHERE addr=?", (last_login, user)
-            )
-
-            userdata["home"] = str(config.get_user_maildir(user))
-            userdata["uid"] = "vmail"
-            userdata["gid"] = "vmail"
-            return userdata
-        if not is_allowed_to_create(config, user, cleartext_password):
-            return
-
-        encrypted_password = encrypt_password(cleartext_password)
-        q = """INSERT INTO users (addr, password, last_login)
-               VALUES (?, ?, ?)"""
-        conn.execute(q, (user, encrypted_password, last_login))
-        print(f"Created address: {user}", file=sys.stderr)
-        return dict(
-            home=str(config.get_user_maildir(user)),
-            uid="vmail",
-            gid="vmail",
-            password=encrypted_password,
-        )
-
-
-def iter_userdb(db) -> list:
-    """Get a list of all user addresses."""
-    with db.read_connection() as conn:
-        rows = conn.execute(
-            "SELECT addr from users",
-        ).fetchall()
-    return [x[0] for x in rows]
-
-
-def iter_userdb_lastlogin_before(db, cutoff_date):
-    """Get a list of users where last login was before cutoff_date."""
-    with db.read_connection() as conn:
-        rows = conn.execute(
-            "SELECT addr FROM users WHERE last_login < ?", (cutoff_date,)
-        ).fetchall()
-    return [x[0] for x in rows]
-
-
 def split_and_unescape(s):
     """Split strings using double quote as a separator and backslash as escape character
     into parts."""
@@ -178,15 +81,12 @@ def split_and_unescape(s):
     yield out
 
 
-def handle_dovecot_request(msg, db, config: Config):
+class AuthDictProxy(DictProxy):
-    # see https://doc.dovecot.org/3.0/developer_manual/design/dict_protocol/
+    def __init__(self, config):
-    short_command = msg[0]
+        super().__init__()
-    if short_command == "H":  # HELLO
+        self.config = config
-        # we don't do any checking on versions and just return
-        return
-    elif short_command == "L":  # LOOKUP
-        parts = msg[1:].split("\t")
 
+    def handle_lookup(self, parts):
         # Dovecot <2.3.17 has only one part,
         # do not attempt to read any other parts for compatibility.
         keyname = parts[0]
@@ -194,13 +94,14 @@ def handle_dovecot_request(msg, db, config: Config):
         namespace, type, args = keyname.split("/", 2)
         args = list(split_and_unescape(args))
 
+        config = self.config
         reply_command = "F"
         res = ""
         if namespace == "shared":
             if type == "userdb":
                 user = args[0]
                 if user.endswith(f"@{config.mail_domain}"):
-                    res = lookup_userdb(db, config, user)
+                    res = self.lookup_userdb(user)
                 if res:
                     reply_command = "O"
                 else:
@@ -208,62 +109,48 @@ def handle_dovecot_request(msg, db, config: Config):
             elif type == "passdb":
                 user = args[1]
                 if user.endswith(f"@{config.mail_domain}"):
-                    res = lookup_passdb(db, config, user, cleartext_password=args[0])
+                    res = self.lookup_passdb(user, cleartext_password=args[0])
                 if res:
                     reply_command = "O"
                 else:
                     reply_command = "N"
         json_res = json.dumps(res) if res else ""
         return f"{reply_command}{json_res}\n"
-    elif short_command == "I":  # ITERATE
+
+    def handle_iterate(self, parts):
         # example: I0\t0\tshared/userdb/
-        parts = msg[1:].split("\t")
         if parts[2] == "shared/userdb/":
-            result = "".join(f"Oshared/userdb/{user}\t\n" for user in iter_userdb(db))
+            result = "".join(
+                f"Oshared/userdb/{user}\t\n" for user in self.iter_userdb()
+            )
             return f"{result}\n"
 
-    raise UnknownCommand(msg)
+    def iter_userdb(self) -> list:
+        """Get a list of all user addresses."""
+        return [x for x in os.listdir(self.config.mailboxes_dir) if "@" in x]
 
+    def lookup_userdb(self, addr):
+        return self.config.get_user(addr).get_userdb_dict()
 
-def handle_dovecot_protocol(rfile, wfile, db: Database, config: Config):
+    def lookup_passdb(self, addr, cleartext_password):
-    while True:
+        user = self.config.get_user(addr)
-        msg = rfile.readline().strip().decode()
+        userdata = user.get_userdb_dict()
-        if not msg:
+        if userdata:
-            break
+            return userdata
-        try:
+        if not is_allowed_to_create(self.config, addr, cleartext_password):
-            res = handle_dovecot_request(msg, db, config)
+            return
-        except UnknownCommand:
-            logging.warning("unknown command: %r", msg)
-        else:
-            if res:
-                wfile.write(res.encode("ascii"))
-                wfile.flush()
 
-
+        user.set_password(encrypt_password(cleartext_password))
-class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
+        print(f"Created address: {addr}", file=sys.stderr)
-    request_queue_size = 100
+        return user.get_userdb_dict()
 
 
 def main():
     socket, cfgpath = sys.argv[1:]
     config = read_config(cfgpath)
-    db = Database(config.passdb_path)
 
-    class Handler(StreamRequestHandler):
+    migrate_from_db_to_maildir(config)
-        def handle(self):
-            try:
-                handle_dovecot_protocol(self.rfile, self.wfile, db, config)
-            except Exception:
-                logging.exception("Exception in the handler")
-                raise
 
-    try:
+    dictproxy = AuthDictProxy(config=config)
-        os.unlink(socket)
-    except FileNotFoundError:
-        pass
 
-    with ThreadedUnixStreamServer(socket, Handler) as server:
+    dictproxy.serve_forever_from_socket(socket)
-        try:
-            server.serve_forever()
-        except KeyboardInterrupt:
-            pass

chatmaild/src/chatmaild/echo.py

@@ -8,11 +8,11 @@ import logging
 import os
 import subprocess
 import sys
-from pathlib import Path
 
 from deltachat_rpc_client import Bot, DeltaChat, EventType, Rpc, events
 
-from chatmaild.config import read_config
+from chatmaild.config import echobot_password_path, read_config
+from chatmaild.doveauth import encrypt_password
 from chatmaild.newemail import create_newemail_dict
 
 hooks = events.HookCollection()
@@ -21,9 +21,9 @@ hooks = events.HookCollection()
 @hooks.on(events.RawEvent)
 def log_event(event):
     if event.kind == EventType.INFO:
-        logging.info("%s", event.msg)
+        logging.info(event.msg)
     elif event.kind == EventType.WARNING:
-        logging.warning("%s", event.msg)
+        logging.warning(event.msg)
 
 
 @hooks.on(events.RawEvent(EventType.ERROR))
@@ -45,7 +45,7 @@ def on_group_image_changed(event):
 
 @hooks.on(events.GroupNameChanged)
 def on_group_name_changed(event):
-    logging.info("group name changed, old name: %s", event.old_name)
+    logging.info(f"group name changed, old name: {event.old_name}")
 
 
 @hooks.on(events.NewMessage(func=lambda e: not e.command))
@@ -72,7 +72,7 @@ def main():
     with Rpc() as rpc:
         deltachat = DeltaChat(rpc)
         system_info = deltachat.get_system_info()
-        logging.info("Running deltachat core %s", system_info.deltachat_core_version)
+        logging.info(f"Running deltachat core {system_info.deltachat_core_version}")
 
         accounts = deltachat.get_all_accounts()
         account = accounts[0] if accounts else deltachat.add_account()
@@ -80,23 +80,23 @@ def main():
         bot = Bot(account, hooks)
 
         config = read_config(sys.argv[1])
+        addr = "echo@" + config.mail_domain
 
         # Create password file
         if bot.is_configured():
             password = bot.account.get_config("mail_pw")
         else:
             password = create_newemail_dict(config)["password"]
-        Path("/run/echobot/password").write_text(password)
 
+        echobot_password_path.write_text(encrypt_password(password))
         # Give the user which doveauth runs as access to the password file.
-        subprocess.run(
+        subprocess.check_call(
-            ["/usr/bin/setfacl", "-m", "user:vmail:r", "/run/echobot/password"],
+            ["/usr/bin/setfacl", "-m", "user:vmail:r", echobot_password_path],
-            check=True,
         )
 
         if not bot.is_configured():
-            email = "echo@" + config.mail_domain
+            bot.configure(addr, password)
-            bot.configure(email, password)
+
         bot.run_forever()
 
 

chatmaild/src/chatmaild/filedict.py

@@ -2,6 +2,7 @@ import json
 import logging
 import os
 from contextlib import contextmanager
+from random import randint
 
 import filelock
 
@@ -32,5 +33,12 @@ class FileDict:
         except FileNotFoundError:
             return {}
         except Exception:
-            logging.warning("corrupt serialization state at: %r", self.path)
+            logging.warning(f"corrupt serialization state at: {self.path!r}")
             return {}
+
+
+def write_bytes_atomic(path, content):
+    rint = randint(0, 10000000)
+    tmp = path.with_name(path.name + f".tmp-{rint}")
+    tmp.write_bytes(content)
+    os.rename(tmp, path)

chatmaild/src/chatmaild/filtermail.py

@@ -12,6 +12,7 @@ from smtplib import SMTP as SMTPClient
 
 from aiosmtpd.controller import Controller
 
+from .common_encrypted_subjects import common_encrypted_subjects
 from .config import read_config
 
 
@@ -111,7 +112,7 @@ def check_encrypted(message):
     """
     if not message.is_multipart():
         return False
-    if message.get("subject") not in {"...", "[...]"}:
+    if message.get("subject") not in common_encrypted_subjects:
         return False
     if message.get_content_type() != "multipart/encrypted":
         return False
@@ -152,7 +153,7 @@ class BeforeQueueHandler:
         self.send_rate_limiter = SendRateLimiter()
 
     async def handle_MAIL(self, server, session, envelope, address, mail_options):
-        logging.info("handle_MAIL from %s", address)
+        logging.info(f"handle_MAIL from {address}")
         envelope.mail_from = address
         max_sent = self.config.max_user_send_per_minute
         if not self.send_rate_limiter.is_sending_allowed(address, max_sent):
@@ -176,13 +177,13 @@ class BeforeQueueHandler:
 
     def check_DATA(self, envelope):
         """the central filtering function for e-mails."""
-        logging.info("Processing DATA message from %s", envelope.mail_from)
+        logging.info(f"Processing DATA message from {envelope.mail_from}")
 
         message = BytesParser(policy=policy.default).parsebytes(envelope.content)
         mail_encrypted = check_encrypted(message)
 
         _, from_addr = parseaddr(message.get("from").strip())
-        logging.info("mime-from: %s envelope-from: %r", from_addr, envelope.mail_from)
+        logging.info(f"mime-from: {from_addr} envelope-from: {envelope.mail_from!r}")
         if envelope.mail_from.lower() != from_addr.lower():
             return f"500 Invalid FROM <{from_addr!r}> for <{envelope.mail_from!r}>"
 

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -17,6 +17,9 @@ max_user_send_per_minute = 60
 # maximum mailbox size of a chatmail address
 max_mailbox_size = 100M
 
+# maximum message size for an e-mail in bytes
+max_message_size = 31457280
+
 # days after which mails are unconditionally deleted
 delete_mails_after = 20
 
@@ -36,24 +39,33 @@ password_min_length = 9
 passthrough_senders =
 
 # list of e-mail recipients for which to accept outbound un-encrypted mails
-passthrough_recipients = xstore@testrun.org groupsbot@hispanilandia.net
+# (space-separated)
+passthrough_recipients = xstore@testrun.org 
 
 #
 # Deployment Details
 #
 
-# Directory where user mailboxes are stored
-mailboxes_dir = /home/vmail/mail/{mail_domain}
-
-# user address sqlite database path
-passdb_path = /home/vmail/passdb.sqlite
-
 # where the filtermail SMTP service listens
 filtermail_smtp_port = 10080
 
 # postfix accepts on the localhost reinject SMTP port
 postfix_reinject_port = 10025
 
+# if set to "True" IPv6 is disabled
+disable_ipv6 = False
+
+#
+# Debugging options 
+#
+
+# set to True if you want to track imap protocol execution
+# in per-maildir ".in/.out" files. 
+# Note that you need to manually cleanup these files
+# so use this option with caution on production servers. 
+imap_rawlog = false 
+
+
 #
 # Privacy Policy
 #

chatmaild/src/chatmaild/ini/override-testrun.ini

@@ -1,7 +1,7 @@
 
 [privacy]
 
-passthrough_recipients = privacy@testrun.org xstore@testrun.org groupsbot@hispanilandia.net
+passthrough_recipients = privacy@testrun.org xstore@testrun.org
 
 privacy_postal =
     Merlinux GmbH, Represented by the managing director H. Krekel,

chatmaild/src/chatmaild/lastlogin.py

@@ -0,0 +1,31 @@
+import sys
+
+from .config import read_config
+from .dictproxy import DictProxy
+
+
+class LastLoginDictProxy(DictProxy):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+
+    def handle_set(self, addr, parts):
+        keyname = parts[1].split("/")
+        value = parts[2] if len(parts) > 2 else ""
+        if keyname[0] == "shared" and keyname[1] == "last-login":
+            if addr.startswith("echo@"):
+                return True
+            addr = keyname[2]
+            timestamp = int(value)
+            user = self.config.get_user(addr)
+            user.set_last_login_timestamp(timestamp)
+            return True
+
+        return False
+
+
+def main():
+    socket, config_path = sys.argv[1:]
+    config = read_config(config_path)
+    dictproxy = LastLoginDictProxy(config=config)
+    dictproxy.serve_forever_from_socket(socket)

chatmaild/src/chatmaild/metadata.py

@@ -1,24 +1,11 @@
 import logging
-import os
 import sys
-from socketserver import (
-    StreamRequestHandler,
-    ThreadingMixIn,
-    UnixStreamServer,
-)
 
 from .config import read_config
+from .dictproxy import DictProxy
 from .filedict import FileDict
 from .notifier import Notifier
 
-DICTPROXY_HELLO_CHAR = "H"
-DICTPROXY_LOOKUP_CHAR = "L"
-DICTPROXY_ITERATE_CHAR = "I"
-DICTPROXY_BEGIN_TRANSACTION_CHAR = "B"
-DICTPROXY_SET_CHAR = "S"
-DICTPROXY_COMMIT_TRANSACTION_CHAR = "C"
-DICTPROXY_TRANSACTION_CHARS = "BSC"
-
 
 class Metadata:
     # each SETMETADATA on this key appends to a list of unique device tokens
@@ -48,82 +35,46 @@ class Metadata:
         return mdict.get(self.DEVICETOKEN_KEY, [])
 
 
-def handle_dovecot_protocol(rfile, wfile, notifier, metadata, iroh_relay=None):
+class MetadataDictProxy(DictProxy):
-    transactions = {}
+    def __init__(self, notifier, metadata, iroh_relay=None):
-    while True:
+        super().__init__()
-        msg = rfile.readline().strip().decode()
+        self.notifier = notifier
-        if not msg:
+        self.metadata = metadata
-            break
+        self.iroh_relay = iroh_relay
 
-        res = handle_dovecot_request(msg, transactions, notifier, metadata, iroh_relay)
+    def handle_lookup(self, parts):
-        if res:
-            wfile.write(res.encode("ascii"))
-            wfile.flush()
-
-
-def handle_dovecot_request(msg, transactions, notifier, metadata, iroh_relay=None):
-    # see https://doc.dovecot.org/3.0/developer_manual/design/dict_protocol/
-    short_command = msg[0]
-    parts = msg[1:].split("\t")
-    if short_command == DICTPROXY_LOOKUP_CHAR:
         # Lpriv/43f5f508a7ea0366dff30200c15250e3/devicetoken\tlkj123poi@c2.testrun.org
         keyparts = parts[0].split("/", 2)
         if keyparts[0] == "priv":
             keyname = keyparts[2]
             addr = parts[1]
-            if keyname == metadata.DEVICETOKEN_KEY:
+            if keyname == self.metadata.DEVICETOKEN_KEY:
-                res = " ".join(metadata.get_tokens_for_addr(addr))
+                res = " ".join(self.metadata.get_tokens_for_addr(addr))
                 return f"O{res}\n"
         elif keyparts[0] == "shared":
             keyname = keyparts[2]
             if (
                 keyname == "vendor/vendor.dovecot/pvt/server/vendor/deltachat/irohrelay"
-                and iroh_relay
+                and self.iroh_relay
             ):
                 # Handle `GETMETADATA "" /shared/vendor/deltachat/irohrelay`
-                return f"O{iroh_relay}\n"
+                return f"O{self.iroh_relay}\n"
-        logging.warning("lookup ignored: %r", msg)
+        logging.warning(f"lookup ignored: {parts!r}")
         return "N\n"
-    elif short_command == DICTPROXY_ITERATE_CHAR:
-        # Empty line means ITER_FINISHED.
-        # If we don't return empty line Dovecot will timeout.
-        return "\n"
-    elif short_command == DICTPROXY_HELLO_CHAR:
-        return  # no version checking
 
-    if short_command not in (DICTPROXY_TRANSACTION_CHARS):
+    def handle_set(self, addr, parts):
-        logging.warning("unknown dictproxy request: %r", msg)
-        return
-
-    transaction_id = parts[0]
-
-    if short_command == DICTPROXY_BEGIN_TRANSACTION_CHAR:
-        addr = parts[1]
-        transactions[transaction_id] = dict(addr=addr, res="O\n")
-    elif short_command == DICTPROXY_COMMIT_TRANSACTION_CHAR:
-        # each set devicetoken operation persists directly
-        # and does not wait until a "commit" comes
-        # because our dovecot config does not involve
-        # multiple set-operations in a single commit
-        return transactions.pop(transaction_id)["res"]
-    elif short_command == DICTPROXY_SET_CHAR:
         # For documentation on key structure see
         # https://github.com/dovecot/core/blob/main/src/lib-storage/mailbox-attribute.h
-
         keyname = parts[1].split("/")
         value = parts[2] if len(parts) > 2 else ""
-        addr = transactions[transaction_id]["addr"]
+        if keyname[0] == "priv" and keyname[2] == self.metadata.DEVICETOKEN_KEY:
-        if keyname[0] == "priv" and keyname[2] == metadata.DEVICETOKEN_KEY:
+            self.metadata.add_token_to_addr(addr, value)
-            metadata.add_token_to_addr(addr, value)
+            return True
         elif keyname[0] == "priv" and keyname[2] == "messagenew":
-            notifier.new_message_for_addr(addr, metadata)
+            self.notifier.new_message_for_addr(addr, self.metadata)
-        else:
+            return True
-            # Transaction failed.
-            transactions[transaction_id]["res"] = "F\n"
 
-
+        return False
-class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
-    request_queue_size = 100
 
 
 def main():
@@ -143,23 +94,8 @@ def main():
     notifier = Notifier(queue_dir)
     notifier.start_notification_threads(metadata.remove_token_from_addr)
 
-    class Handler(StreamRequestHandler):
+    dictproxy = MetadataDictProxy(
-        def handle(self):
+        notifier=notifier, metadata=metadata, iroh_relay=iroh_relay
-            try:
+    )
-                handle_dovecot_protocol(
-                    self.rfile, self.wfile, notifier, metadata, iroh_relay
-                )
-            except Exception:
-                logging.exception("Exception in the dovecot dictproxy handler")
-                raise
 
-    try:
+    dictproxy.serve_forever_from_socket(socket)
-        os.unlink(socket)
-    except FileNotFoundError:
-        pass
-
-    with ThreadedUnixStreamServer(socket, Handler) as server:
-        try:
-            server.serve_forever()
-        except KeyboardInterrupt:
-            pass

chatmaild/src/chatmaild/migrate_db.py

@@ -0,0 +1,63 @@
+"""
+migration code from old sqlite databases into per-maildir "password" files
+where mtime reflects and is updated to be the "last-login" time.
+"""
+
+import logging
+import os
+import sqlite3
+import sys
+
+from chatmaild.config import read_config
+
+
+def get_all_rows(path):
+    assert path.exists()
+    uri = f"file:{path}?mode=ro"
+    sqlconn = sqlite3.connect(uri, timeout=60, isolation_level="DEFERRED", uri=True)
+    cur = sqlconn.cursor()
+    cur.execute("SELECT * from users")
+    rows = cur.fetchall()
+    sqlconn.close()
+    return rows
+
+
+def migrate_from_db_to_maildir(config, chunking=10000):
+    path = config.passdb_path
+    if not path.exists():
+        return
+
+    all_rows = get_all_rows(path)
+
+    # don't transfer special/CI accounts
+    rows = [row for row in all_rows if row[0][:3] not in ("ci-", "ac_")]
+
+    logging.info(f"ignoring {len(all_rows)-len(rows)} CI accounts")
+    logging.info(f"migrating {len(rows)} sqlite database passwords to user dirs")
+
+    for i, row in enumerate(rows):
+        addr = row[0]
+        enc_password = row[1]
+        user = config.get_user(addr)
+        user.set_password(enc_password)
+
+        if len(row) == 3 and row[2]:
+            timestamp = int(row[2])
+            user.set_last_login_timestamp(timestamp)
+
+        if i > 0 and i % chunking == 0:
+            logging.info(f"migration-progress: {i} passwords transferred")
+
+    logging.info("migration: all passwords migrated")
+    oldpath = config.passdb_path.with_suffix(config.passdb_path.suffix + ".old")
+    os.rename(config.passdb_path, oldpath)
+    for path in config.passdb_path.parent.iterdir():
+        if path.name.startswith(config.passdb_path.name + "-"):
+            path.unlink()
+    logging.info(f"migration: moved database to {oldpath!r}")
+
+
+if __name__ == "__main__":
+    config = read_config(sys.argv[1])
+    logging.basicConfig(level=logging.INFO)
+    migrate_from_db_to_maildir(config)

chatmaild/src/chatmaild/notifier.py

@@ -92,7 +92,7 @@ class Notifier:
     def requeue_persistent_queue_items(self):
         for queue_path in self.queue_dir.iterdir():
             if queue_path.name.endswith(".tmp"):
-                logging.warning("removing spurious queue item: %r", queue_path)
+                logging.warning(f"removing spurious queue item: {queue_path!r}")
                 queue_path.unlink()
                 continue
             queue_item = PersistentQueueItem.read_from_path(queue_path)
@@ -104,7 +104,7 @@ class Notifier:
         deadline = queue_item.start_ts + self.DROP_DEADLINE
         if retry_num >= len(self.retry_queues) or when > deadline:
             queue_item.delete()
-            logging.error("notification exceeded deadline: %r", queue_item.token)
+            logging.error(f"notification exceeded deadline: {queue_item.token!r}")
             return
 
         self.retry_queues[retry_num].put((when, queue_item))
@@ -162,5 +162,5 @@ class NotifyThread(Thread):
                 queue_item.delete()
                 return
 
-        logging.warning("Notification request failed: %r", res)
+        logging.warning(f"Notification request failed: {res!r}")
         self.notifier.queue_for_retry(queue_item, retry_num=self.retry_num + 1)

chatmaild/src/chatmaild/tests/plugin.py

@@ -7,19 +7,19 @@ from email.parser import BytesParser
 from pathlib import Path
 
 import pytest
+
 from chatmaild.config import read_config, write_initial_config
-from chatmaild.database import Database
 
 
 @pytest.fixture
 def make_config(tmp_path):
     inipath = tmp_path.joinpath("chatmail.ini")
 
-    def make_conf(mail_domain):
+    def make_conf(mail_domain, settings=None):
         basedir = tmp_path.joinpath(f"vmail/{mail_domain}")
         basedir.mkdir(parents=True, exist_ok=True)
-        passdb = tmp_path.joinpath("vmail/passdb.sqlite")
+        overrides = settings.copy() if settings else {}
-        overrides = dict(mailboxes_dir=str(basedir), passdb_path=str(passdb))
+        overrides["mailboxes_dir"] = str(basedir)
         write_initial_config(inipath, mail_domain, overrides=overrides)
         return read_config(inipath)
 
@@ -36,6 +36,11 @@ def maildomain(example_config):
     return example_config.mail_domain
 
 
+@pytest.fixture
+def testaddr(maildomain):
+    return f"user.name@{maildomain}"
+
+
 @pytest.fixture
 def gencreds(maildomain):
     count = itertools.count()
@@ -54,13 +59,6 @@ def gencreds(maildomain):
     return lambda domain=None: next(gen(domain))
 
 
-@pytest.fixture()
-def db(tmpdir):
-    db_path = tmpdir / "passdb.sqlite"
-    print("database path:", db_path)
-    return Database(db_path)
-
-
 @pytest.fixture
 def maildata(request):
     try:
@@ -79,3 +77,22 @@ def maildata(request):
         return BytesParser(policy=policy.default).parsebytes(text.encode())
 
     return maildata
+
+
+@pytest.fixture
+def mockout():
+    class MockOut:
+        captured_red = []
+        captured_green = []
+        captured_plain = []
+
+        def red(self, msg):
+            self.captured_red.append(msg)
+
+        def green(self, msg):
+            self.captured_green.append(msg)
+
+        def __call__(self, msg):
+            self.captured_plain.append(msg)
+
+    return MockOut()

chatmaild/src/chatmaild/tests/test_config.py

@@ -1,4 +1,5 @@
 import pytest
+
 from chatmaild.config import read_config
 
 
@@ -38,24 +39,28 @@ def test_config_userstate_paths(make_config, tmp_path):
     mailboxes_dir = config.mailboxes_dir
     passdb_path = config.passdb_path
     assert mailboxes_dir.name == "something.testrun.org"
-    assert passdb_path.name == "passdb.sqlite"
+    assert str(passdb_path) == "/home/vmail/passdb.sqlite"
-    assert passdb_path.is_relative_to(tmp_path)
     assert config.mail_domain == "something.testrun.org"
-    path = config.get_user_maildir("user1@something.testrun.org")
+    path = config.get_user("user1@something.testrun.org").maildir
     assert not path.exists()
     assert path == mailboxes_dir.joinpath("user1@something.testrun.org")
 
     with pytest.raises(ValueError):
-        config.get_user_maildir("")
+        config.get_user("")
 
     with pytest.raises(ValueError):
-        config.get_user_maildir(None)
+        config.get_user(None)
 
     with pytest.raises(ValueError):
-        config.get_user_maildir("../some@something.testrun.org")
+        config.get_user("../some@something.testrun.org").maildir
 
     with pytest.raises(ValueError):
-        config.get_user_maildir("..")
+        config.get_user("..").maildir
 
     with pytest.raises(ValueError):
-        config.get_user_maildir(".")
+        config.get_user(".")
+
+
+def test_config_max_message_size(make_config, tmp_path):
+    config = make_config("something.testrun.org", dict(max_message_size="10000"))
+    assert config.max_message_size == 10000

chatmaild/src/chatmaild/tests/test_delete_inactive_users.py

@@ -1,27 +1,37 @@
 import time
 
 from chatmaild.delete_inactive_users import delete_inactive_users
-from chatmaild.doveauth import lookup_passdb
+from chatmaild.doveauth import AuthDictProxy
 
 
-def test_remove_stale_users(db, example_config):
+def test_login_timestamps(example_config):
+    testaddr = "someuser@chat.example.org"
+    user = example_config.get_user(testaddr)
+
+    # password file needs to be set because it's mtime tracks last-login time
+    user.set_password("1l2k3j1l2k3j123")
+    for i in range(10):
+        user.set_last_login_timestamp(86400 * 4 + i)
+        assert user.get_last_login_timestamp() == 86400 * 4
+
+
+def test_delete_inactive_users(example_config):
     new = time.time()
     old = new - (example_config.delete_inactive_users_after * 86400) - 1
+    dictproxy = AuthDictProxy(example_config)
 
     def create_user(addr, last_login):
-        lookup_passdb(db, example_config, addr, "q9mr3faue", last_login=last_login)
+        dictproxy.lookup_passdb(addr, "q9mr3faue")
-        md = example_config.get_user_maildir(addr)
+        user = example_config.get_user(addr)
-        md.mkdir(parents=True)
+        user.maildir.joinpath("cur").mkdir()
-        md.joinpath("cur").mkdir()
+        user.maildir.joinpath("cur", "something").mkdir()
-        md.joinpath("cur", "something").mkdir()
+        user.set_last_login_timestamp(timestamp=last_login)
 
     # create some stale and some new accounts
     to_remove = []
     for i in range(150):
         addr = f"oldold{i:03}@chat.example.org"
         create_user(addr, last_login=old)
-        with db.read_connection() as conn:
-            assert conn.get_user(addr)
         to_remove.append(addr)
 
     remain = []
@@ -33,19 +43,17 @@ def test_remove_stale_users(db, example_config):
     # check pre and post-conditions for delete_inactive_users()
 
     for addr in to_remove:
-        assert example_config.get_user_maildir(addr).exists()
+        assert example_config.get_user(addr).maildir.exists()
 
-    delete_inactive_users(db, example_config)
+    delete_inactive_users(example_config)
 
     for p in example_config.mailboxes_dir.iterdir():
         assert not p.name.startswith("old")
 
     for addr in to_remove:
-        assert not example_config.get_user_maildir(addr).exists()
+        assert not example_config.get_user(addr).maildir.exists()
-        with db.read_connection() as conn:
-            assert not conn.get_user(addr)
 
     for addr in remain:
-        assert example_config.get_user_maildir(addr).exists()
+        userdir = example_config.get_user(addr).maildir
-        with db.read_connection() as conn:
+        assert userdir.exists()
-            assert conn.get_user(addr)
+        assert userdir.joinpath("password").read_text()

chatmaild/src/chatmaild/tests/test_doveauth.py

@@ -4,58 +4,39 @@ import queue
 import threading
 import traceback
 
-import chatmaild.doveauth
 import pytest
-from chatmaild.database import DBError
+
+import chatmaild.doveauth
 from chatmaild.doveauth import (
-    get_user_data,
+    AuthDictProxy,
-    handle_dovecot_protocol,
-    handle_dovecot_request,
     is_allowed_to_create,
-    iter_userdb,
-    iter_userdb_lastlogin_before,
-    lookup_passdb,
 )
 from chatmaild.newemail import create_newemail_dict
 
 
-def test_basic(db, example_config):
+@pytest.fixture
-    lookup_passdb(db, example_config, "asdf12345@chat.example.org", "q9mr3faue")
+def dictproxy(example_config):
-    data = get_user_data(db, example_config, "asdf12345@chat.example.org")
+    return AuthDictProxy(config=example_config)
+
+
+def test_basic(dictproxy, gencreds):
+    addr, password = gencreds()
+    dictproxy.lookup_passdb(addr, password)
+    data = dictproxy.lookup_userdb(addr)
     assert data
-    data2 = lookup_passdb(
+    data2 = dictproxy.lookup_passdb(addr, password)
-        db, example_config, "asdf12345@chat.example.org", "q9mr3jewvadsfaue"
-    )
     assert data == data2
 
 
-def test_iterate_addresses(db, example_config):
+def test_iterate_addresses(dictproxy):
     addresses = []
 
     for i in range(10):
         addresses.append(f"asdf1234{i}@chat.example.org")
-        lookup_passdb(db, example_config, addresses[-1], "q9mr3faue")
+        dictproxy.lookup_passdb(addresses[-1], "q9mr3faue")
-    res = iter_userdb(db)
-    assert res == addresses
 
-
+    res = dictproxy.iter_userdb()
-def test_iterate_addresses_lastlogin_before(db, example_config):
+    assert set(res) == set(addresses)
-    addresses = []
-
-    cutoff_date = 1000
-    for i in range(10):
-        addr = f"oldold{i:03}@chat.example.org"
-        lookup_passdb(
-            db, example_config, addr, "q9mr3faue", last_login=cutoff_date - 10
-        )
-        addresses.append(addr)
-
-    for i in range(5):
-        addr = f"newnew{i:03}@chat.example.org"
-        lookup_passdb(db, example_config, addr, "q9mr3faue", last_login=cutoff_date + i)
-
-    res = iter_userdb_lastlogin_before(db, cutoff_date)
-    assert sorted(res) == sorted(addresses)
 
 
 def test_invalid_username_length(example_config):
@@ -72,45 +53,33 @@ def test_invalid_username_length(example_config):
     )
 
 
-def test_dont_overwrite_password_on_wrong_login(db, example_config):
+def test_dont_overwrite_password_on_wrong_login(dictproxy):
     """Test that logging in with a different password doesn't create a new user"""
-    res = lookup_passdb(
+    res = dictproxy.lookup_passdb(
-        db, example_config, "newuser12@chat.example.org", "kajdlkajsldk12l3kj1983"
+        "newuser12@chat.example.org", "kajdlkajsldk12l3kj1983"
     )
     assert res["password"]
-    res2 = lookup_passdb(db, example_config, "newuser12@chat.example.org", "kajdslqwe")
+    res2 = dictproxy.lookup_passdb("newuser12@chat.example.org", "kajdslqwe")
     # this function always returns a password hash, which is actually compared by dovecot.
     assert res["password"] == res2["password"]
 
 
-def test_nocreate_file(db, monkeypatch, tmpdir, example_config):
+def test_nocreate_file(monkeypatch, tmpdir, dictproxy):
     p = tmpdir.join("nocreate")
     p.write("")
     monkeypatch.setattr(chatmaild.doveauth, "NOCREATE_FILE", str(p))
-    lookup_passdb(
+    dictproxy.lookup_passdb("newuser12@chat.example.org", "zequ0Aimuchoodaechik")
-        db, example_config, "newuser12@chat.example.org", "zequ0Aimuchoodaechik"
+    assert not dictproxy.lookup_userdb("newuser12@chat.example.org")
-    )
-    assert not get_user_data(db, example_config, "newuser12@chat.example.org")
 
 
-def test_db_version(db):
+def test_handle_dovecot_request(dictproxy):
-    assert db.get_schema_version() == 1
+    transactions = {}
-
-
-def test_too_high_db_version(db):
-    with db.write_transaction() as conn:
-        conn.execute("PRAGMA user_version=%s;" % (999,))
-    with pytest.raises(DBError):
-        db.ensure_tables()
-
-
-def test_handle_dovecot_request(db, example_config):
     # Test that password can contain ", ', \ and /
     msg = (
         'Lshared/passdb/laksjdlaksjdlak\\\\sjdlk\\"12j\\\'3l1/k2j3123"'
         "some42123@chat.example.org\tsome42123@chat.example.org"
     )
-    res = handle_dovecot_request(msg, db, example_config)
+    res = dictproxy.handle_dovecot_request(msg, transactions)
     assert res
     assert res[0] == "O" and res.endswith("\n")
     userdata = json.loads(res[1:].strip())
@@ -119,45 +88,48 @@ def test_handle_dovecot_request(db, example_config):
     assert userdata["password"].startswith("{SHA512-CRYPT}")
 
 
-def test_handle_dovecot_protocol_hello_is_skipped(db, example_config, caplog):
+def test_handle_dovecot_protocol_hello_is_skipped(example_config, caplog):
+    dictproxy = AuthDictProxy(config=example_config)
     rfile = io.BytesIO(b"H3\t2\t0\t\tauth\n")
     wfile = io.BytesIO()
-    handle_dovecot_protocol(rfile, wfile, db, example_config)
+    dictproxy.loop_forever(rfile, wfile)
     assert wfile.getvalue() == b""
     assert not caplog.messages
 
 
-def test_handle_dovecot_protocol(db, example_config):
+def test_handle_dovecot_protocol_user_not_exists(example_config):
+    dictproxy = AuthDictProxy(config=example_config)
     rfile = io.BytesIO(
         b"H3\t2\t0\t\tauth\nLshared/userdb/foobar@chat.example.org\tfoobar@chat.example.org\n"
     )
     wfile = io.BytesIO()
-    handle_dovecot_protocol(rfile, wfile, db, example_config)
+    dictproxy.loop_forever(rfile, wfile)
     assert wfile.getvalue() == b"N\n"
 
 
-def test_handle_dovecot_protocol_iterate(db, gencreds, example_config):
+def test_handle_dovecot_protocol_iterate(gencreds, example_config):
-    lookup_passdb(db, example_config, "asdf00000@chat.example.org", "q9mr3faue")
+    dictproxy = AuthDictProxy(config=example_config)
-    lookup_passdb(db, example_config, "asdf11111@chat.example.org", "q9mr3faue")
+    dictproxy.lookup_passdb("asdf00000@chat.example.org", "q9mr3faue")
+    dictproxy.lookup_passdb("asdf11111@chat.example.org", "q9mr3faue")
     rfile = io.BytesIO(b"H3\t2\t0\t\tauth\nI0\t0\tshared/userdb/")
     wfile = io.BytesIO()
-    handle_dovecot_protocol(rfile, wfile, db, example_config)
+    dictproxy.loop_forever(rfile, wfile)
     lines = wfile.getvalue().decode("ascii").split("\n")
-    assert lines[0] == "Oshared/userdb/asdf00000@chat.example.org\t"
+    assert "Oshared/userdb/asdf00000@chat.example.org\t" in lines
-    assert lines[1] == "Oshared/userdb/asdf11111@chat.example.org\t"
+    assert "Oshared/userdb/asdf11111@chat.example.org\t" in lines
     assert not lines[2]
 
 
-def test_50_concurrent_lookups_different_accounts(db, gencreds, example_config):
+def test_50_concurrent_lookups_different_accounts(gencreds, dictproxy):
     num_threads = 50
     req_per_thread = 5
     results = queue.Queue()
 
-    def lookup(db):
+    def lookup():
         for i in range(req_per_thread):
             addr, password = gencreds()
             try:
-                lookup_passdb(db, example_config, addr, password)
+                dictproxy.lookup_passdb(addr, password)
             except Exception:
                 results.put(traceback.format_exc())
             else:
@@ -165,7 +137,7 @@ def test_50_concurrent_lookups_different_accounts(db, gencreds, example_config):
 
     threads = []
     for i in range(num_threads):
-        thread = threading.Thread(target=lookup, args=(db,), daemon=True)
+        thread = threading.Thread(target=lookup, daemon=True)
         threads.append(thread)
 
     print(f"created {num_threads} threads, starting them and waiting for results")

chatmaild/src/chatmaild/tests/test_filedict.py

@@ -1,4 +1,6 @@
-from chatmaild.filedict import FileDict
+import threading
+
+from chatmaild.filedict import FileDict, write_bytes_atomic
 
 
 def test_basic(tmp_path):
@@ -17,3 +19,21 @@ def test_bad_marshal_file(tmp_path, caplog):
     fdict1.path.write_bytes(b"l12k3l12k3l")
     assert fdict1.read() == {}
     assert "corrupt" in caplog.records[0].msg
+
+
+def test_write_bytes_atomic_concurrent(tmp_path):
+    p = tmp_path.joinpath("somefile.ext")
+    write_bytes_atomic(p, b"hello")
+
+    threads = []
+    for i in range(30):
+        content = f"hello{i}".encode("ascii")
+        t = threading.Thread(target=lambda: write_bytes_atomic(p, content))
+        t.start()
+        threads.append(t)
+
+    for t in threads:
+        t.join()
+
+    assert p.read_text().strip() != "hello"
+    assert len(list(p.parent.iterdir())) == 1

chatmaild/src/chatmaild/tests/test_filtermail.py

@@ -1,9 +1,11 @@
 import pytest
+
 from chatmaild.filtermail import (
     BeforeQueueHandler,
     SendRateLimiter,
     check_armored_payload,
     check_encrypted,
+    common_encrypted_subjects,
 )
 
 
@@ -54,7 +56,7 @@ def test_filtermail_no_encryption_detection(maildata):
 
 
 def test_filtermail_encryption_detection(maildata):
-    for subject in ("...", "[...]"):
+    for subject in common_encrypted_subjects:
         msg = maildata(
             "encrypted.eml",
             from_addr="1@example.org",

chatmaild/src/chatmaild/tests/test_lastlogin.py

@@ -0,0 +1,64 @@
+import time
+
+from chatmaild.doveauth import AuthDictProxy
+from chatmaild.lastlogin import (
+    LastLoginDictProxy,
+)
+
+
+def test_handle_dovecot_request_last_login(testaddr, example_config):
+    dictproxy = LastLoginDictProxy(config=example_config)
+
+    authproxy = AuthDictProxy(config=example_config)
+    authproxy.lookup_passdb(testaddr, "1l2k3j1l2k3jl123")
+
+    dictproxy_transactions = {}
+
+    # Begin transaction
+    tx = "1111"
+    msg = f"B{tx}\t{testaddr}"
+    res = dictproxy.handle_dovecot_request(msg, dictproxy_transactions)
+    assert not res
+    assert dictproxy_transactions == {tx: dict(addr=testaddr, res="O\n")}
+
+    # set last-login info for user
+    user = dictproxy.config.get_user(testaddr)
+    timestamp = int(time.time())
+    msg = f"S{tx}\tshared/last-login/{testaddr}\t{timestamp}"
+    res = dictproxy.handle_dovecot_request(msg, dictproxy_transactions)
+    assert not res
+    assert len(dictproxy_transactions) == 1
+    read_timestamp = user.get_last_login_timestamp()
+    assert read_timestamp == timestamp // 86400 * 86400
+
+    # finish transaction
+    msg = f"C{tx}"
+    res = dictproxy.handle_dovecot_request(msg, dictproxy_transactions)
+    assert res == "O\n"
+    assert len(dictproxy_transactions) == 0
+
+
+def test_handle_dovecot_request_last_login_echobot(example_config):
+    dictproxy = LastLoginDictProxy(config=example_config)
+
+    authproxy = AuthDictProxy(config=example_config)
+    testaddr = f"echo@{example_config.mail_domain}"
+    authproxy.lookup_passdb(testaddr, "ignore")
+    user = dictproxy.config.get_user(testaddr)
+
+    transactions = {}
+
+    # set last-login info for user
+    tx = "1111"
+    msg = f"B{tx}\t{testaddr}"
+    res = dictproxy.handle_dovecot_request(msg, transactions)
+    assert not res
+    assert transactions == {tx: dict(addr=testaddr, res="O\n")}
+
+    timestamp = int(time.time())
+    msg = f"S{tx}\tshared/last-login/{testaddr}\t{timestamp}"
+    res = dictproxy.handle_dovecot_request(msg, transactions)
+    assert not res
+    assert len(transactions) == 1
+    read_timestamp = user.get_last_login_timestamp()
+    assert read_timestamp is None

chatmaild/src/chatmaild/tests/test_metadata.py

@@ -3,10 +3,10 @@ import time
 
 import pytest
 import requests
+
 from chatmaild.metadata import (
     Metadata,
-    handle_dovecot_protocol,
+    MetadataDictProxy,
-    handle_dovecot_request,
 )
 from chatmaild.notifier import (
     Notifier,
@@ -30,8 +30,8 @@ def metadata(tmp_path):
 
 
 @pytest.fixture
-def testaddr():
+def dictproxy(notifier, metadata):
-    return "user.name@example.org"
+    return MetadataDictProxy(notifier=notifier, metadata=metadata)
 
 
 @pytest.fixture
@@ -88,51 +88,51 @@ def test_notifier_remove_without_set(metadata, testaddr):
     assert not metadata.get_tokens_for_addr(testaddr)
 
 
-def test_handle_dovecot_request_lookup_fails(notifier, metadata, testaddr):
+def test_handle_dovecot_request_lookup_fails(dictproxy, testaddr):
-    res = handle_dovecot_request(
+    transactions = {}
-        f"Lpriv/123/chatmail\t{testaddr}", {}, notifier, metadata
+    res = dictproxy.handle_dovecot_request(
+        f"Lpriv/123/chatmail\t{testaddr}", transactions
     )
     assert res == "N\n"
 
 
-def test_handle_dovecot_request_happy_path(notifier, metadata, testaddr, token):
+def test_handle_dovecot_request_happy_path(dictproxy, testaddr, token):
+    metadata = dictproxy.metadata
     transactions = {}
+    notifier = dictproxy.notifier
 
     # set device token in a transaction
     tx = "1111"
     msg = f"B{tx}\t{testaddr}"
-    res = handle_dovecot_request(msg, transactions, notifier, metadata)
+    res = dictproxy.handle_dovecot_request(msg, transactions)
     assert not res and not metadata.get_tokens_for_addr(testaddr)
     assert transactions == {tx: dict(addr=testaddr, res="O\n")}
 
     msg = f"S{tx}\tpriv/guid00/devicetoken\t{token}"
-    res = handle_dovecot_request(msg, transactions, notifier, metadata)
+    res = dictproxy.handle_dovecot_request(msg, transactions)
     assert not res
     assert len(transactions) == 1
     assert metadata.get_tokens_for_addr(testaddr) == [token]
 
     msg = f"C{tx}"
-    res = handle_dovecot_request(msg, transactions, notifier, metadata)
+    res = dictproxy.handle_dovecot_request(msg, transactions)
     assert res == "O\n"
     assert len(transactions) == 0
     assert metadata.get_tokens_for_addr(testaddr) == [token]
 
     # trigger notification for incoming message
     tx2 = "2222"
-    assert (
+    assert dictproxy.handle_dovecot_request(f"B{tx2}\t{testaddr}", transactions) is None
-        handle_dovecot_request(f"B{tx2}\t{testaddr}", transactions, notifier, metadata)
-        is None
-    )
     msg = f"S{tx2}\tpriv/guid00/messagenew"
-    assert handle_dovecot_request(msg, transactions, notifier, metadata) is None
+    assert dictproxy.handle_dovecot_request(msg, transactions) is None
     queue_item = notifier.retry_queues[0].get()[1]
     assert queue_item.token == token
-    assert handle_dovecot_request(f"C{tx2}", transactions, notifier, metadata) == "O\n"
+    assert dictproxy.handle_dovecot_request(f"C{tx2}", transactions) == "O\n"
     assert not transactions
     assert queue_item.path.exists()
 
 
-def test_handle_dovecot_protocol_set_devicetoken(metadata, notifier):
+def test_handle_dovecot_protocol_set_devicetoken(dictproxy):
     rfile = io.BytesIO(
         b"\n".join(
             [
@@ -144,12 +144,12 @@ def test_handle_dovecot_protocol_set_devicetoken(metadata, notifier):
         )
     )
     wfile = io.BytesIO()
-    handle_dovecot_protocol(rfile, wfile, notifier, metadata)
+    dictproxy.loop_forever(rfile, wfile)
     assert wfile.getvalue() == b"O\n"
-    assert metadata.get_tokens_for_addr("user@example.org") == ["01234"]
+    assert dictproxy.metadata.get_tokens_for_addr("user@example.org") == ["01234"]
 
 
-def test_handle_dovecot_protocol_set_get_devicetoken(metadata, notifier):
+def test_handle_dovecot_protocol_set_get_devicetoken(dictproxy):
     rfile = io.BytesIO(
         b"\n".join(
             [
@@ -161,19 +161,19 @@ def test_handle_dovecot_protocol_set_get_devicetoken(metadata, notifier):
         )
     )
     wfile = io.BytesIO()
-    handle_dovecot_protocol(rfile, wfile, notifier, metadata)
+    dictproxy.loop_forever(rfile, wfile)
-    assert metadata.get_tokens_for_addr("user@example.org") == ["01234"]
+    assert dictproxy.metadata.get_tokens_for_addr("user@example.org") == ["01234"]
     assert wfile.getvalue() == b"O\n"
 
     rfile = io.BytesIO(
         b"\n".join([b"HELLO", b"Lpriv/0123/devicetoken\tuser@example.org"])
     )
     wfile = io.BytesIO()
-    handle_dovecot_protocol(rfile, wfile, notifier, metadata)
+    dictproxy.loop_forever(rfile, wfile)
     assert wfile.getvalue() == b"O01234\n"
 
 
-def test_handle_dovecot_protocol_iterate(metadata, notifier):
+def test_handle_dovecot_protocol_iterate(dictproxy):
     rfile = io.BytesIO(
         b"\n".join(
             [
@@ -183,7 +183,7 @@ def test_handle_dovecot_protocol_iterate(metadata, notifier):
         )
     )
     wfile = io.BytesIO()
-    handle_dovecot_protocol(rfile, wfile, notifier, metadata)
+    dictproxy.loop_forever(rfile, wfile)
     assert wfile.getvalue() == b"\n"
 
 
@@ -298,7 +298,7 @@ def test_persistent_queue_items(tmp_path, testaddr, token):
     assert not queue_item < item2 and not item2 < queue_item
 
 
-def test_iroh_relay(metadata):
+def test_iroh_relay(dictproxy):
     rfile = io.BytesIO(
         b"\n".join(
             [
@@ -308,5 +308,6 @@ def test_iroh_relay(metadata):
         )
     )
     wfile = io.BytesIO()
-    handle_dovecot_protocol(rfile, wfile, notifier, metadata, "https://example.org/")
+    dictproxy.iroh_relay = "https://example.org/"
+    dictproxy.loop_forever(rfile, wfile)
     assert wfile.getvalue() == b"Ohttps://example.org/\n"

chatmaild/src/chatmaild/tests/test_migrate_db.py

@@ -0,0 +1,67 @@
+import sqlite3
+
+from chatmaild.migrate_db import migrate_from_db_to_maildir
+
+
+def test_migration_not_exists(tmp_path, example_config):
+    example_config.passdb_path = tmp_path.joinpath("sqlite")
+
+
+def test_migration(tmp_path, example_config, caplog):
+    passdb_path = tmp_path.joinpath("passdb.sqlite")
+    uri = f"file:{passdb_path}?mode=rwc"
+    sqlconn = sqlite3.connect(uri, timeout=60, uri=True)
+    sqlconn.execute(
+        """
+        CREATE TABLE users (
+            addr TEXT PRIMARY KEY,
+            password TEXT,
+            last_login INTEGER
+        )
+    """
+    )
+    all = {}
+
+    for i in range(500):
+        values = (f"somsom{i:03}@example.org", f"passwo{i:03}", i * 86400)
+        sqlconn.execute(
+            """
+            INSERT INTO users (addr, password, last_login)
+            VALUES (?, ?, ?)""",
+            values,
+        )
+        all[values[0]] = values[1:]
+
+    for i in range(500):
+        values = (f"pompom{i:03}@example.org", f"wopass{i:03}", "")
+        sqlconn.execute(
+            """
+            INSERT INTO users (addr, password, last_login)
+            VALUES (?, ?, ?)""",
+            values,
+        )
+        all[values[0]] = values[1:]
+
+    sqlconn.commit()
+    sqlconn.close()
+
+    assert passdb_path.stat().st_size > 10000
+
+    example_config.passdb_path = passdb_path
+
+    assert not caplog.records
+
+    migrate_from_db_to_maildir(example_config, chunking=500)
+    assert len(caplog.records) > 3
+
+    for path in example_config.mailboxes_dir.iterdir():
+        if "@" not in path.name:
+            continue
+        password, last_login = all.pop(path.name)
+        user = example_config.get_user(path.name)
+        if last_login:
+            assert user.get_last_login_timestamp() == last_login
+        assert password == user.get_userdb_dict()["password"]
+
+    assert not all
+    assert not example_config.passdb_path.exists()

chatmaild/src/chatmaild/tests/test_user.py

@@ -0,0 +1,42 @@
+def test_login_timestamp(testaddr, example_config):
+    user = example_config.get_user(testaddr)
+    user.set_password("someeqkjwelkqwjleqwe")
+    user.set_last_login_timestamp(100000)
+    assert user.get_last_login_timestamp() == 86400
+
+    user.set_last_login_timestamp(200000)
+    assert user.get_last_login_timestamp() == 86400 * 2
+
+
+def test_get_user_dict_not_set(testaddr, example_config, caplog):
+    user = example_config.get_user(testaddr)
+    assert not caplog.records
+    assert user.get_userdb_dict() == {}
+    assert len(caplog.records) == 0
+
+    user.set_password("")
+    assert user.get_userdb_dict() == {}
+    assert len(caplog.records) == 1
+
+
+def test_get_user_dict(make_config, tmp_path):
+    config = make_config("something.testrun.org")
+    addr = "user1@something.org"
+    user = config.get_user(addr)
+    enc_password = "l1k2j31lk2j3l1k23j123"
+    user.set_password(enc_password)
+    data = user.get_userdb_dict()
+    assert addr in str(data["home"])
+    assert data["uid"] == "vmail"
+    assert data["gid"] == "vmail"
+    assert data["password"] == enc_password
+
+
+def test_no_mailboxes_dir(testaddr, example_config, tmp_path):
+    p = tmp_path.joinpath("a", "mailboxes")
+    example_config.mailboxes_dir = p
+
+    user = example_config.get_user(testaddr)
+    user.set_password("someeqkjwelkqwjleqwe")
+    user.set_last_login_timestamp(100000)
+    assert user.get_last_login_timestamp() == 86400

chatmaild/src/chatmaild/user.py

@@ -0,0 +1,74 @@
+import logging
+import os
+
+from chatmaild.filedict import write_bytes_atomic
+
+
+def get_daytimestamp(timestamp) -> int:
+    return int(timestamp) // 86400 * 86400
+
+
+class User:
+    def __init__(self, maildir, addr, password_path, uid, gid):
+        self.maildir = maildir
+        self.addr = addr
+        self.password_path = password_path
+        self.uid = uid
+        self.gid = gid
+
+    @property
+    def can_track(self):
+        return "@" in self.addr and not self.addr.startswith("echo@")
+
+    def get_userdb_dict(self):
+        """Return a non-empty dovecot 'userdb' style dict
+        if the user has an existing non-empty password"""
+        try:
+            pw = self.password_path.read_text()
+        except FileNotFoundError:
+            return {}
+
+        if not pw:
+            logging.error(f"password is empty for: {self.addr}")
+            return {}
+
+        home = str(self.maildir)
+        return dict(addr=self.addr, home=home, uid=self.uid, gid=self.gid, password=pw)
+
+    def set_password(self, enc_password):
+        """Set the specified password for this user.
+
+        This method can be called concurrently
+        but there is no guarantee which of the password-set calls will win.
+        """
+        self.maildir.mkdir(exist_ok=True, parents=True)
+        password = enc_password.encode("ascii")
+
+        try:
+            write_bytes_atomic(self.password_path, password)
+        except PermissionError:
+            if not self.addr.startswith("echo@"):
+                logging.error(f"could not write password for: {self.addr}")
+                raise
+
+    def set_last_login_timestamp(self, timestamp):
+        """Track login time with daily granularity
+        to minimize touching files and to minimize metadata leakage."""
+        if not self.can_track:
+            return
+        try:
+            mtime = int(os.stat(self.password_path).st_mtime)
+        except FileNotFoundError:
+            logging.error(f"Can not get last login timestamp for {self.addr}")
+            return
+
+        timestamp = get_daytimestamp(timestamp)
+        if mtime != timestamp:
+            os.utime(self.password_path, (timestamp, timestamp))
+
+    def get_last_login_timestamp(self):
+        if self.can_track:
+            try:
+                return int(self.password_path.stat().st_mtime)
+            except FileNotFoundError:
+                pass

cmdeploy/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 name = "cmdeploy"
 version = "0.2"
 dependencies = [
-  "pyinfra",
+  "pyinfra>=3",
   "pillow",
   "qrcode",
   "markdown",

cmdeploy/src/cmdeploy/__init__.py

@@ -103,6 +103,7 @@ def _install_remote_venv_with_chatmaild(config) -> None:
         "filtermail",
         "echobot",
         "chatmail-metadata",
+        "lastlogin",
     ):
         params = dict(
             execpath=f"{remote_venv_dir}/bin/{fn}",
@@ -267,6 +268,7 @@ def _configure_postfix(config: Config, debug: bool = False) -> bool:
         group="root",
         mode="644",
         config=config,
+        disable_ipv6=config.disable_ipv6,
     )
     need_restart |= main_config.changed
 
@@ -317,6 +319,7 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
         mode="644",
         config=config,
         debug=debug,
+        disable_ipv6=config.disable_ipv6,
     )
     need_restart |= main_config.changed
     auth_config = files.put(
@@ -361,7 +364,7 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
     return need_restart
 
 
-def _configure_nginx(domain: str, debug: bool = False) -> bool:
+def _configure_nginx(config: Config, debug: bool = False) -> bool:
     """Configures nginx HTTP server."""
     need_restart = False
 
@@ -371,7 +374,8 @@ def _configure_nginx(domain: str, debug: bool = False) -> bool:
         user="root",
         group="root",
         mode="644",
-        config={"domain_name": domain},
+        config={"domain_name": config.mail_domain},
+        disable_ipv6=config.disable_ipv6,
     )
     need_restart |= main_config.changed
 
@@ -381,7 +385,7 @@ def _configure_nginx(domain: str, debug: bool = False) -> bool:
         user="root",
         group="root",
         mode="644",
-        config={"domain_name": domain},
+        config={"domain_name": config.mail_domain},
     )
     need_restart |= autoconfig.changed
 
@@ -391,7 +395,7 @@ def _configure_nginx(domain: str, debug: bool = False) -> bool:
         user="root",
         group="root",
         mode="644",
-        config={"domain_name": domain},
+        config={"domain_name": config.mail_domain},
     )
     need_restart |= mta_sts_config.changed
 
@@ -450,6 +454,7 @@ def deploy_chatmail(config_path: Path) -> None:
 
     server.group(name="Create vmail group", group="vmail", system=True)
     server.user(name="Create vmail user", user="vmail", group="vmail", system=True)
+    server.user(name="Create filtermail user", user="filtermail", system=True)
     server.group(name="Create opendkim group", group="opendkim", system=True)
     server.user(
         name="Create opendkim user",
@@ -484,6 +489,7 @@ def deploy_chatmail(config_path: Path) -> None:
     )
 
     apt.update(name="apt update", cache_time=24 * 3600)
+    apt.upgrade(name="upgrade apt packages", auto_remove=True)
 
     apt.packages(
         name="Install rsync",
@@ -554,7 +560,7 @@ def deploy_chatmail(config_path: Path) -> None:
     dovecot_need_restart = _configure_dovecot(config, debug=debug)
     postfix_need_restart = _configure_postfix(config, debug=debug)
     mta_sts_need_restart = _install_mta_sts_daemon()
-    nginx_need_restart = _configure_nginx(mail_domain)
+    nginx_need_restart = _configure_nginx(config)
 
     _remove_rspamd()
     opendkim_need_restart = _configure_opendkim(mail_domain, "opendkim")

cmdeploy/src/cmdeploy/chatmail.zone.f

@@ -1,15 +0,0 @@
-{chatmail_domain}.                   A     {ipv4}
-{chatmail_domain}.                   AAAA  {ipv6}
-{chatmail_domain}.                   MX 10 {chatmail_domain}.
-_submission._tcp.{chatmail_domain}.  SRV 0 1 587 {chatmail_domain}.
-_submissions._tcp.{chatmail_domain}. SRV 0 1 465 {chatmail_domain}.
-_imap._tcp.{chatmail_domain}.        SRV 0 1 143 {chatmail_domain}.
-_imaps._tcp.{chatmail_domain}.       SRV 0 1 993 {chatmail_domain}.
-{chatmail_domain}.                   CAA 128 issue "letsencrypt.org;accounturi={acme_account_url}"
-{chatmail_domain}.                   TXT "v=spf1 a:{chatmail_domain} ~all"
-_dmarc.{chatmail_domain}.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
-_mta-sts.{chatmail_domain}.          TXT "v=STSv1; id={sts_id}"
-mta-sts.{chatmail_domain}.           CNAME {chatmail_domain}.
-www.{chatmail_domain}.               CNAME {chatmail_domain}.
-{dkim_entry}
-_adsp._domainkey.{chatmail_domain}.  TXT "dkim=discardable"

cmdeploy/src/cmdeploy/chatmail.zone.j2

@@ -0,0 +1,30 @@
+;
+; Required DNS entries for chatmail servers 
+;
+{% if A %}
+{{ mail_domain }}.                   A     {{ A }}
+{% endif %}
+{% if AAAA %}
+{{ mail_domain }}.                   AAAA  {{ AAAA }}
+{% endif %}
+{{ mail_domain }}.                   MX 10 {{ mail_domain }}.
+_mta-sts.{{ mail_domain }}.          TXT "v=STSv1; id={{ sts_id }}"
+mta-sts.{{ mail_domain }}.           CNAME {{ mail_domain }}.
+www.{{ mail_domain }}.               CNAME {{ mail_domain }}.
+{{ dkim_entry }}
+
+;
+; Recommended DNS entries for interoperability and security-hardening
+;
+{{ mail_domain }}.                   TXT "v=spf1 a:{{ mail_domain }} ~all"
+_dmarc.{{ mail_domain }}.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
+
+{% if acme_account_url %}
+{{ mail_domain }}.                   CAA 128 issue "letsencrypt.org;accounturi={{ acme_account_url }}"
+{% endif %}
+_adsp._domainkey.{{ mail_domain }}.  TXT "dkim=discardable"
+
+_submission._tcp.{{ mail_domain }}.  SRV 0 1 587 {{ mail_domain }}.
+_submissions._tcp.{{ mail_domain }}. SRV 0 1 465 {{ mail_domain }}.
+_imap._tcp.{{ mail_domain }}.        SRV 0 1 143 {{ mail_domain }}.
+_imaps._tcp.{{ mail_domain }}.       SRV 0 1 993 {{ mail_domain }}.

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -7,16 +7,18 @@ import argparse
 import importlib.resources
 import importlib.util
 import os
+import pathlib
 import shutil
 import subprocess
 import sys
 from pathlib import Path
 
+import pyinfra
 from chatmaild.config import read_config, write_initial_config
+from packaging import version
 from termcolor import colored
 
-from . import remote_funcs
+from . import dns, remote_funcs
-from .dns import show_dns
 from .sshexec import SSHExec
 
 #
@@ -54,20 +56,27 @@ def run_cmd_options(parser):
 
 def run_cmd(args, out):
     """Deploy chatmail services on the remote server."""
-    retcode, remote_data = show_dns(args, out)
+
+    sshexec = args.get_sshexec()
+    remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
+    if not dns.check_initial_remote_data(remote_data, print=out.red):
+        return 1
 
     env = os.environ.copy()
     env["CHATMAIL_INI"] = args.inipath
     deploy_path = importlib.resources.files(__package__).joinpath("deploy.py").resolve()
     pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
-    cmd = f"{pyinf} --ssh-user root {args.config.mail_domain} {deploy_path}"
+    cmd = f"{pyinf} --ssh-user root {args.config.mail_domain} {deploy_path} -y"
+    if version.parse(pyinfra.__version__) < version.parse("3"):
+        out.red("Please re-run scripts/initenv.sh to update pyinfra to version 3.")
+        return 1
 
-    out.check_call(cmd, env=env)
+    retcode = out.check_call(cmd, env=env)
     if retcode == 0:
-        out.green("Deploy completed, call `cmdeploy test` next.")
+        out.green("Deploy completed, call `cmdeploy dns` next.")
     elif not remote_data["acme_account_url"]:
         out.red("Deploy completed but letsencrypt not configured")
-        out.red("Run 'cmdeploy dns' or 'cmdeploy run' again")
+        out.red("Run 'cmdeploy run' again")
         retcode = 0
     else:
         out.red("Deploy failed")
@@ -78,13 +87,37 @@ def dns_cmd_options(parser):
     parser.add_argument(
         "--zonefile",
         dest="zonefile",
-        help="print the whole zonefile for deploying directly",
+        type=pathlib.Path,
+        default=None,
+        help="write out a zonefile",
     )
 
 
 def dns_cmd(args, out):
     """Check DNS entries and optionally generate dns zone file."""
-    retcode, remote_data = show_dns(args, out)
+    sshexec = args.get_sshexec()
+    remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
+    if not remote_data:
+        return 1
+
+    if not remote_data["acme_account_url"]:
+        out.red("could not get letsencrypt account url, please run 'cmdeploy run'")
+        return 1
+
+    if not remote_data["dkim_entry"]:
+        out.red("could not determine dkim_entry, please run 'cmdeploy run'")
+        return 1
+
+    zonefile = dns.get_filled_zone_file(remote_data)
+
+    if args.zonefile:
+        args.zonefile.write_text(zonefile)
+        out.green(f"DNS records successfully written to: {args.zonefile}")
+        return 0
+
+    retcode = dns.check_full_zone(
+        sshexec, remote_data=remote_data, zonefile=zonefile, out=out
+    )
     return retcode
 
 
@@ -278,9 +311,9 @@ def main(args=None):
     if not hasattr(args, "func"):
         return parser.parse_args(["-h"])
 
-    def get_sshexec(log=None):
+    def get_sshexec():
         print(f"[ssh] login to {args.config.mail_domain}")
-        return SSHExec(args.config.mail_domain, remote_funcs, log=log)
+        return SSHExec(args.config.mail_domain, remote_funcs, verbose=args.verbose)
 
     args.get_sshexec = get_sshexec
 
@@ -301,7 +334,6 @@ def main(args=None):
         if res is None:
             res = 0
         return res
-
     except KeyboardInterrupt:
         out.red("KeyboardInterrupt")
         sys.exit(130)

cmdeploy/src/cmdeploy/dns.py

@@ -1,59 +1,60 @@
 import datetime
 import importlib
-import sys
+
+from jinja2 import Template
 
 from . import remote_funcs
 
 
-def show_dns(args, out) -> int:
+def get_initial_remote_data(sshexec, mail_domain):
+    return sshexec.logged(
+        call=remote_funcs.perform_initial_checks, kwargs=dict(mail_domain=mail_domain)
+    )
+
+
+def check_initial_remote_data(remote_data, print=print):
+    mail_domain = remote_data["mail_domain"]
+    if not remote_data["A"] and not remote_data["AAAA"]:
+        print(f"Missing A and/or AAAA DNS records for {mail_domain}!")
+    elif not remote_data["MTA_STS"]:
+        print("Missing MTA-STS CNAME record:")
+        print(f"mta-sts.{mail_domain}.   CNAME  {mail_domain}")
+    else:
+        return remote_data
+
+
+def get_filled_zone_file(remote_data):
+    sts_id = remote_data.get("sts_id")
+    if not sts_id:
+        sts_id = datetime.datetime.now().strftime("%Y%m%d%H%M")
+
+    template = importlib.resources.files(__package__).joinpath("chatmail.zone.j2")
+    content = template.read_text()
+    zonefile = Template(content).render(**remote_data)
+    lines = [x.strip() for x in zonefile.split("\n") if x.strip()]
+    lines.append("")
+    zonefile = "\n".join(lines)
+    return zonefile
+
+
+def check_full_zone(sshexec, remote_data, out, zonefile) -> int:
     """Check existing DNS records, optionally write them to zone file
     and return (exitcode, remote_data) tuple."""
-    template = importlib.resources.files(__package__).joinpath("chatmail.zone.f")
-    mail_domain = args.config.mail_domain
 
-    def log_progress(data):
+    required_diff, recommended_diff = sshexec.logged(
-        sys.stdout.write(".")
+        remote_funcs.check_zonefile, kwargs=dict(zonefile=zonefile)
-        sys.stdout.flush()
+    )
 
-    sshexec = args.get_sshexec(log=print if args.verbose else log_progress)
+    if required_diff:
-    print("Checking DNS entries ", end="\n" if args.verbose else "")
+        out.red("Please set required DNS entries at your DNS provider:\n")
+        for line in required_diff:
+            out(line)
+        return 1
+    elif recommended_diff:
+        out("WARNING: these recommended DNS entries are not set:\n")
+        for line in recommended_diff:
+            out(line)
+        return 0
 
-    remote_data = sshexec(remote_funcs.perform_initial_checks, mail_domain=mail_domain)
+    out.green("Great! All your DNS entries are verified and correct.")
-
+    return 0
-    assert remote_data["ipv4"] or remote_data["ipv6"]
-
-    with open(template, "r") as f:
-        zonefile = f.read().format(
-            acme_account_url=remote_data["acme_account_url"],
-            dkim_entry=remote_data["dkim_entry"],
-            ipv6=remote_data["ipv6"],
-            ipv4=remote_data["ipv4"],
-            sts_id=datetime.datetime.now().strftime("%Y%m%d%H%M"),
-            chatmail_domain=args.config.mail_domain,
-        )
-
-    to_print = sshexec(remote_funcs.check_zonefile, zonefile=zonefile)
-    if not args.verbose:
-        print()
-
-    if getattr(args, "zonefile", None):
-        with open(args.zonefile, "w+") as zf:
-            zf.write(zonefile)
-        out.green(f"DNS records successfully written to: {args.zonefile}")
-        return 0, remote_data
-
-    if to_print:
-        to_print.insert(
-            0, "You should configure the following entries at your DNS provider:\n"
-        )
-        to_print.append(
-            "\nIf you already configured the DNS entries, "
-            "wait a bit until the DNS entries propagate to the Internet."
-        )
-        out.red("\n".join(to_print))
-        exit_code = 1
-    else:
-        out.green("Great! All your DNS entries are verified and correct.")
-        exit_code = 0
-
-    return exit_code, remote_data

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -1,5 +1,9 @@
 ## Dovecot configuration file
 
+{% if disable_ipv6 %}
+listen = *
+{% endif %}
+
 protocols = imap lmtp
 
 auth_mechanisms = plain
@@ -106,10 +110,16 @@ mail_attribute_dict = proxy:/run/chatmail-metadata/metadata.socket:metadata
 # `imap_zlib` enables IMAP COMPRESS (RFC 4978).
 # <https://datatracker.ietf.org/doc/html/rfc4978.html>
 protocol imap {
-  mail_plugins = $mail_plugins imap_zlib imap_quota
+  mail_plugins = $mail_plugins imap_zlib imap_quota last_login
   imap_metadata = yes
 }
 
+plugin {
+  last_login_dict = proxy:/run/chatmail-lastlogin/lastlogin.socket:lastlogin
+  #last_login_key = last-login/%u # default
+  last_login_precision = s
+}
+
 protocol lmtp {
   # notify plugin is a dependency of push_notification plugin:
   # <https://doc.dovecot.org/settings/plugin/notify-plugin/>
@@ -193,3 +203,24 @@ ssl_key = </var/lib/acme/live/{{ config.mail_domain }}/privkey
 ssl_dh = </usr/share/dovecot/dh.pem
 ssl_min_protocol = TLSv1.2
 ssl_prefer_server_ciphers = yes
+
+
+{% if config.imap_rawlog %}
+service postlogin {
+  executable = script-login -d rawlog
+  unix_listener postlogin {
+ }  
+} 
+service imap {
+  executable = imap postlogin 
+} 
+  
+protocol imap { 
+  #rawlog_dir = /tmp/rawlog/%u
+  # Put .in and .out imap protocol logging files into per-user homedir 
+  # You can use a command like this to combine into one protocol stream:
+  # sort -sn <(sed 's/ / C: /' *.in) <(sed 's/ / S: /' cat *.out)
+
+  rawlog_dir = %h 
+} 
+{% endif %}

cmdeploy/src/cmdeploy/nginx/autoconfig.xml.j2

@@ -5,6 +5,13 @@
     <domain>{{ config.domain_name }}</domain>
     <displayName>{{ config.domain_name }} chatmail</displayName>
     <displayShortName>{{ config.domain_name }}</displayShortName>
+    <incomingServer type="imap">
+      <hostname>{{ config.domain_name }}</hostname>
+      <port>443</port>
+      <socketType>SSL</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+    </incomingServer>
     <incomingServer type="imap">
       <hostname>{{ config.domain_name }}</hostname>
       <port>993</port>
@@ -19,13 +26,13 @@
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
     </incomingServer>
-    <incomingServer type="imap">
+    <outgoingServer type="smtp">
       <hostname>{{ config.domain_name }}</hostname>
       <port>443</port>
       <socketType>SSL</socketType>
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
-    </incomingServer>
+    </outgoingServer>
     <outgoingServer type="smtp">
       <hostname>{{ config.domain_name }}</hostname>
       <port>465</port>
@@ -40,12 +47,5 @@
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
     </outgoingServer>
-    <outgoingServer type="smtp">
-      <hostname>{{ config.domain_name }}</hostname>
-      <port>443</port>
-      <socketType>SSL</socketType>
-      <authentication>password-cleartext</authentication>
-      <username>%EMAILADDRESS%</username>
-    </outgoingServer>
   </emailProvider>
 </clientConfig>

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -13,13 +13,15 @@ events {
 stream {
         map $ssl_preread_alpn_protocols $proxy {
             default 127.0.0.1:8443;
-            ~\bsmtp\b 127.0.0.1:submissions;
+            ~\bsmtp\b 127.0.0.1:465;
-            ~\bimap\b 127.0.0.1:imaps;
+            ~\bimap\b 127.0.0.1:993;
         }
 
         server {
                 listen 443;
+                {% if not disable_ipv6 %}
                 listen [::]:443;
+                {% endif %}
                 proxy_pass $proxy;
                 ssl_preread on;
         }
@@ -43,8 +45,11 @@ http {
 	gzip on;
 
 	server {
+  
 		listen 8443 ssl default_server;
+		{% if not disable_ipv6 %}
 		listen [::]:8443 ssl default_server;
+		{% endif %}
 
 		root /var/www/html;
 
@@ -96,7 +101,9 @@ http {
 	# Redirect www. to non-www
 	server {
 		listen 8443 ssl;
+		{% if not disable_ipv6 %}
 		listen [::]:8443 ssl;
+		{% endif %}
 		server_name www.{{ config.domain_name }};
 		return 301 $scheme://{{ config.domain_name }}$request_uri;
 		access_log syslog:server=unix:/dev/log,facility=local7;

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -62,11 +62,14 @@ mydestination =
 relayhost = 
 mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128
 mailbox_size_limit = 0
-# maximum 30MB sized messages
+message_size_limit = {{config.max_message_size}}
-message_size_limit = 31457280
 recipient_delimiter = +
 inet_interfaces = all
+{% if disable_ipv6 %}
+inet_protocols = ipv4
+{% else %}
 inet_protocols = all
+{% endif %}
 
 virtual_transport = lmtp:unix:private/dovecot-lmtp
 virtual_mailbox_domains = {{ config.mail_domain }}

cmdeploy/src/cmdeploy/remote_funcs.py

@@ -1,21 +1,22 @@
 """
-Functions to be executed on an ssh-connected host.
+Pure python functions which execute remotely in a system Python interpreter.
 
-All functions of this module need to work with Python builtin types
+All functions of this module
-and standard library dependencies only.
 
-When a remote function executes remotely, it runs in a system python interpreter
+- need to get and and return Python builtin data types only,
-without any installed dependencies.
 
+- can only use standard library dependencies,
+
+- can freely call each other.
 """
 
 import re
-import socket
+import traceback
 from subprocess import CalledProcessError, check_output
 
 
 def shell(command, fail_ok=False):
-    log(f"$ {command}")
+    print(f"$ {command}")
     try:
         return check_output(command, shell=True).decode().rstrip()
     except CalledProcessError:
@@ -30,76 +31,105 @@ def get_systemd_running():
 
 
 def perform_initial_checks(mail_domain):
-    res = {}
+    """Collecting initial DNS settings."""
+    assert mail_domain
+    A = query_dns("A", mail_domain)
+    AAAA = query_dns("AAAA", mail_domain)
+    MTA_STS = query_dns("CNAME", f"mta-sts.{mail_domain}")
+
+    res = dict(mail_domain=mail_domain, A=A, AAAA=AAAA, MTA_STS=MTA_STS)
+    if not MTA_STS or (not A and not AAAA):
+        return res
 
     res["acme_account_url"] = shell("acmetool account-url", fail_ok=True)
     if not shell("dig", fail_ok=True):
         shell("apt-get install -y dnsutils")
     shell(f"unbound-control flush_zone {mail_domain}", fail_ok=True)
-
     res["dkim_entry"] = get_dkim_entry(mail_domain, dkim_selector="opendkim")
-    res["ipv4"] = get_ip_address(socket.AF_INET)
+
-    res["ipv6"] = get_ip_address(socket.AF_INET6)
+    # parse out sts-id if exists, example: "v=STSv1; id=2090123"
+    parts = query_dns("TXT", f"_mta-sts.{mail_domain}").split("id=")
+    res["sts_id"] = parts[1].rstrip('"') if len(parts) == 2 else ""
     return res
 
 
 def get_dkim_entry(mail_domain, dkim_selector):
-    dkim_pubkey = shell(
+    try:
-        f"openssl rsa -in /etc/dkimkeys/{dkim_selector}.private "
+        dkim_pubkey = shell(
-        "-pubout 2>/dev/null | awk '/-/{next}{printf(\"%s\",$0)}'"
+            f"openssl rsa -in /etc/dkimkeys/{dkim_selector}.private "
-    )
+            "-pubout 2>/dev/null | awk '/-/{next}{printf(\"%s\",$0)}'"
+        )
+    except CalledProcessError:
+        return
     dkim_value_raw = f"v=DKIM1;k=rsa;p={dkim_pubkey};s=email;t=s"
     dkim_value = '" "'.join(re.findall(".{1,255}", dkim_value_raw))
     return f'{dkim_selector}._domainkey.{mail_domain}. TXT "{dkim_value}"'
 
 
-def get_ip_address(typ):
-    sock = socket.socket(typ, socket.SOCK_DGRAM)
-    sock.settimeout(0)
-    sock.connect(("notifications.delta.chat", 1))
-    return sock.getsockname()[0]
-
-
 def query_dns(typ, domain):
     res = shell(f"dig -r -q {domain} -t {typ} +short")
-    return set(filter(None, res.split("\n")))
+    print(res)
+    if res:
+        return res.split("\n")[0]
+    return ""
 
 
 def check_zonefile(zonefile):
-    diff = []
+    """Check expected zone file entries."""
+    required = True
+    required_diff = []
+    recommended_diff = []
 
     for zf_line in zonefile.splitlines():
+        if "; Recommended" in zf_line:
+            required = False
+            continue
+        if not zf_line.strip() or zf_line.startswith(";"):
+            continue
+        print(f"dns-checking {zf_line!r}")
         zf_domain, zf_typ, zf_value = zf_line.split(maxsplit=2)
         zf_domain = zf_domain.rstrip(".")
         zf_value = zf_value.strip()
-        query_values = query_dns(zf_typ, zf_domain)
+        query_value = query_dns(zf_typ, zf_domain)
-        if zf_value in query_values:
+        if zf_value != query_value:
-            continue
+            assert zf_typ in ("A", "AAAA", "CNAME", "CAA", "SRV", "MX", "TXT"), zf_line
+            if required:
+                required_diff.append(zf_line)
+            else:
+                recommended_diff.append(zf_line)
 
-        if zf_typ == "CAA" and zf_value.endswith('accounturi="'):
+    return required_diff, recommended_diff
-            # this is an initial run where acmetool did not work yet
-            continue
 
-        if query_values and zf_typ == "TXT" and zf_domain.startswith("_mta-sts."):
-            (query_value,) = query_values
-            if query_value.split("id=")[0] == zf_value.split("id=")[0]:
-                continue
 
-        assert zf_typ in ("A", "AAAA", "CNAME", "CAA", "SRV", "MX", "TXT"), zf_line
+## Function Execution server
-        diff.append(zf_line)
 
-    return diff
+
+def _run_loop(cmd_channel):
+    while 1:
+        cmd = cmd_channel.receive()
+        if cmd is None:
+            break
+
+        cmd_channel.send(_handle_one_request(cmd))
+
+
+def _handle_one_request(cmd):
+    func_name, kwargs = cmd
+    try:
+        res = globals()[func_name](**kwargs)
+        return ("finish", res)
+    except:
+        data = traceback.format_exc()
+        return ("error", data)
 
 
 # check if this module is executed remotely
 # and setup a simple serialized function-execution loop
 
 if __name__ == "__channelexec__":
+    channel = channel  # noqa (channel object gets injected)
 
-    def log(item):
+    # enable simple "print" logging for anyone changing this module
-        channel.send(("log", item))  # noqa
+    globals()["print"] = lambda x="": channel.send(("log", x))
 
-    while 1:
+    _run_loop(channel)
-        func_name, kwargs = channel.receive()  # noqa
-        res = globals()[func_name](**kwargs)  # noqa
-        channel.send(("finish", res))  # noqa

cmdeploy/src/cmdeploy/service/filtermail.service.f

@@ -5,6 +5,7 @@ Description=Chatmail Postfix before queue filter
 ExecStart={execpath} {config_path}
 Restart=always
 RestartSec=30
+User=filtermail
 
 [Install]
 WantedBy=multi-user.target

cmdeploy/src/cmdeploy/service/lastlogin.service.f

@@ -0,0 +1,12 @@
+[Unit]
+Description=Dict proxy for last-login tracking
+
+[Service]
+ExecStart={execpath} /run/chatmail-lastlogin/lastlogin.socket {config_path}
+Restart=always
+RestartSec=30
+User=vmail
+RuntimeDirectory=chatmail-lastlogin
+
+[Install]
+WantedBy=multi-user.target

cmdeploy/src/cmdeploy/sshexec.py

@@ -1,20 +1,52 @@
+import sys
+
 import execnet
 
 
+class FuncError(Exception):
+    pass
+
+
+def print_stderr(item="", end="\n"):
+    print(item, file=sys.stderr, end=end)
+
+
 class SSHExec:
     RemoteError = execnet.RemoteError
+    FuncError = FuncError
 
-    def __init__(self, host, remote_funcs, log=None, python="python3", timeout=60):
+    def __init__(self, host, remote_funcs, verbose=False, python="python3", timeout=60):
         self.gateway = execnet.makegateway(f"ssh=root@{host}//python={python}")
         self._remote_cmdloop_channel = self.gateway.remote_exec(remote_funcs)
-        self.log = log
         self.timeout = timeout
+        self.verbose = verbose
 
-    def __call__(self, func, **kwargs):
+    def __call__(self, call, kwargs=None, log_callback=None):
-        self._remote_cmdloop_channel.send((func.__name__, kwargs))
+        if kwargs is None:
+            kwargs = {}
+        self._remote_cmdloop_channel.send((call.__name__, kwargs))
         while 1:
             code, data = self._remote_cmdloop_channel.receive(timeout=self.timeout)
-            if code == "log" and self.log:
+            if log_callback is not None and code == "log":
-                self.log(data)
+                log_callback(data)
             elif code == "finish":
                 return data
+            elif code == "error":
+                raise self.FuncError(data)
+
+    def logged(self, call, kwargs):
+        def log_progress(data):
+            sys.stderr.write(".")
+            sys.stderr.flush()
+
+        title = call.__doc__
+        if not title:
+            title = call.__name__
+        if self.verbose:
+            print_stderr("[ssh] " + title)
+            return self(call, kwargs, log_callback=print_stderr)
+        else:
+            print_stderr(title, end="")
+            res = self(call, kwargs, log_callback=log_progress)
+            print_stderr()
+            return res

cmdeploy/src/cmdeploy/tests/data/zftest.zone

@@ -0,0 +1,17 @@
+; Required DNS entries for chatmail servers
+zftest.testrun.org.                   A     135.181.204.127
+zftest.testrun.org.                   AAAA  2a01:4f9:c012:52f4::1
+zftest.testrun.org.                   MX 10 zftest.testrun.org.
+_mta-sts.zftest.testrun.org.          TXT "v=STSv1; id=202403211706"
+mta-sts.zftest.testrun.org.           CNAME zftest.testrun.org.
+www.zftest.testrun.org.               CNAME zftest.testrun.org.
+opendkim._domainkey.zftest.testrun.org. TXT "v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoYt82CVUyz2ouaqjX2kB+5J80knAyoOU3MGU5aWppmwUwwTvj/oSTSpkc5JMtVTRmKKr8NUDWAL1Yw7dfGqqPHdHfwwjS3BIvDzYx+hzgtz62RnfNgV+/2MAoNpfX7cAFIHdRzEHNtwugc3RDLquqPoupAE3Y2YRw2T5zG5fILh4vwIcJZL5Uq6B92j8wwJqOex" "33n+vm1NKQ9rxo/UsHAmZlJzpooXcG/4igTBxJyJlamVSRR6N7Nul1v//YJb7J6v2o0iPHW6uE0StzKaPPNC2IVosSRFbD9H2oqppltptFSNPlI0E+t0JBWHem6YK7xcugiO3ImMCaaU8g6Jt/wIDAQAB;s=email;t=s"
+; Recommended DNS entries
+_submission._tcp.zftest.testrun.org.  SRV 0 1 587 zftest.testrun.org.
+_submissions._tcp.zftest.testrun.org. SRV 0 1 465 zftest.testrun.org.
+_imap._tcp.zftest.testrun.org.        SRV 0 1 143 zftest.testrun.org.
+_imaps._tcp.zftest.testrun.org.       SRV 0 1 993 zftest.testrun.org.
+zftest.testrun.org.                   CAA 128 issue "letsencrypt.org;accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/1371472956"
+zftest.testrun.org.                   TXT "v=spf1 a:zftest.testrun.org ~all"
+_dmarc.zftest.testrun.org.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
+_adsp._domainkey.zftest.testrun.org.  TXT "dkim=discardable"

cmdeploy/src/cmdeploy/tests/online/benchmark.py

@@ -41,9 +41,9 @@ class TestDC:
 
         def dc_ping_pong():
             chat.send_text("ping")
-            msg = ac2.wait_next_incoming_message()
+            msg = ac2._evtracker.wait_next_incoming_message()
             msg.chat.send_text("pong")
-            ac1.wait_next_incoming_message()
+            ac1._evtracker.wait_next_incoming_message()
 
         benchmark(dc_ping_pong, 5)
 
@@ -55,6 +55,6 @@ class TestDC:
             for i in range(10):
                 chat.send_text(f"hello {i}")
             for i in range(10):
-                ac2.wait_next_incoming_message()
+                ac2._evtracker.wait_next_incoming_message()
 
         benchmark(dc_send_10_receive_10, 5)

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -7,18 +7,50 @@ from cmdeploy.sshexec import SSHExec
 
 
 class TestSSHExecutor:
-    @pytest.fixture
+    @pytest.fixture(scope="class")
     def sshexec(self, sshdomain):
         return SSHExec(sshdomain, remote_funcs)
 
     def test_ls(self, sshexec):
-        out = sshexec(remote_funcs.shell, command="ls")
+        out = sshexec(call=remote_funcs.shell, kwargs=dict(command="ls"))
-        out2 = sshexec(remote_funcs.shell, command="ls")
+        out2 = sshexec(call=remote_funcs.shell, kwargs=dict(command="ls"))
         assert out == out2
 
     def test_perform_initial(self, sshexec, maildomain):
-        res = sshexec(remote_funcs.perform_initial_checks, mail_domain=maildomain)
+        res = sshexec(
-        assert res["ipv4"] or res["ipv6"]
+            remote_funcs.perform_initial_checks, kwargs=dict(mail_domain=maildomain)
+        )
+        assert res["A"] or res["AAAA"]
+
+    def test_logged(self, sshexec, maildomain, capsys):
+        sshexec.logged(
+            remote_funcs.perform_initial_checks, kwargs=dict(mail_domain=maildomain)
+        )
+        out, err = capsys.readouterr()
+        assert err.startswith("Collecting")
+        assert err.endswith("....\n")
+        assert err.count("\n") == 1
+
+        sshexec.verbose = True
+        sshexec.logged(
+            remote_funcs.perform_initial_checks, kwargs=dict(mail_domain=maildomain)
+        )
+        out, err = capsys.readouterr()
+        lines = err.split("\n")
+        assert len(lines) > 4
+        assert remote_funcs.perform_initial_checks.__doc__ in lines[0]
+
+    def test_exception(self, sshexec, capsys):
+        try:
+            sshexec.logged(
+                remote_funcs.perform_initial_checks,
+                kwargs=dict(mail_domain=None),
+            )
+        except sshexec.FuncError as e:
+            assert "remote_funcs.py" in str(e)
+            assert "AssertionError" in str(e)
+        else:
+            pytest.fail("didn't raise exception")
 
 
 def test_remote(remote, imap_or_smtp):

cmdeploy/src/cmdeploy/tests/plugin.py

@@ -10,7 +10,6 @@ from pathlib import Path
 
 import pytest
 from chatmaild.config import read_config
-from chatmaild.database import Database
 
 conftestdir = Path(__file__).parent
 
@@ -35,7 +34,7 @@ def pytest_runtest_setup(item):
             pytest.skip("skipping slow test, use --slow to run")
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
 def chatmail_config(pytestconfig):
     current = basedir = Path().resolve()
     while 1:
@@ -49,12 +48,12 @@ def chatmail_config(pytestconfig):
     pytest.skip(f"no chatmail.ini file found in {basedir} or parent dirs")
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
 def maildomain(chatmail_config):
     return chatmail_config.mail_domain
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
 def sshdomain(maildomain):
     return os.environ.get("CHATMAIL_SSH", maildomain)
 
@@ -79,6 +78,17 @@ def pytest_report_header():
         return ["-" * len(text), text, "-" * len(text)]
 
 
+@pytest.fixture
+def cm_data(request):
+    datadir = request.fspath.dirpath("data")
+
+    class CMData:
+        def get(self, name):
+            return datadir.join(name).read()
+
+    return CMData()
+
+
 @pytest.fixture
 def benchmark(request):
     def bench(func, num, name=None, reportfunc=None):
@@ -251,13 +261,6 @@ def gencreds(chatmail_config):
     return lambda domain=None: next(gen(domain))
 
 
-@pytest.fixture()
-def db(tmpdir):
-    db_path = tmpdir / "passdb.sqlite"
-    print("database path:", db_path)
-    return Database(db_path)
-
-
 #
 # Delta Chat testplugin re-use
 # use the cmfactory fixture to get chatmail instance accounts

cmdeploy/src/cmdeploy/tests/test_dns.py

@@ -0,0 +1,117 @@
+import pytest
+
+from cmdeploy import remote_funcs
+from cmdeploy.dns import check_full_zone, check_initial_remote_data
+
+
+@pytest.fixture
+def mockdns_base(monkeypatch):
+    qdict = {}
+
+    def query_dns(typ, domain):
+        try:
+            return qdict[typ][domain]
+        except KeyError:
+            return ""
+
+    monkeypatch.setattr(remote_funcs, query_dns.__name__, query_dns)
+    return qdict
+
+
+@pytest.fixture
+def mockdns(mockdns_base):
+    mockdns_base.update(
+        {
+            "A": {"some.domain": "1.1.1.1"},
+            "AAAA": {"some.domain": "fde5:cd7a:9e1c:3240:5a99:936f:cdac:53ae"},
+            "CNAME": {"mta-sts.some.domain": "some.domain"},
+        }
+    )
+    return mockdns_base
+
+
+class TestPerformInitialChecks:
+    def test_perform_initial_checks_ok1(self, mockdns):
+        remote_data = remote_funcs.perform_initial_checks("some.domain")
+        assert len(remote_data) == 7
+
+    @pytest.mark.parametrize("drop", ["A", "AAAA"])
+    def test_perform_initial_checks_with_one_of_A_AAAA(self, mockdns, drop):
+        del mockdns[drop]
+        remote_data = remote_funcs.perform_initial_checks("some.domain")
+        assert len(remote_data) == 7
+        assert not remote_data[drop]
+
+        l = []
+        res = check_initial_remote_data(remote_data, print=l.append)
+        assert res
+        assert not l
+
+    def test_perform_initial_checks_no_mta_sts(self, mockdns):
+        del mockdns["CNAME"]
+        remote_data = remote_funcs.perform_initial_checks("some.domain")
+        assert len(remote_data) == 4
+        assert not remote_data["MTA_STS"]
+
+        l = []
+        res = check_initial_remote_data(remote_data, print=l.append)
+        assert not res
+        assert len(l) == 2
+
+
+def parse_zonefile_into_dict(zonefile, mockdns_base, only_required=False):
+    for zf_line in zonefile.split("\n"):
+        if zf_line.startswith("#"):
+            if "Recommended" in zf_line and only_required:
+                return
+            continue
+        if not zf_line.strip():
+            continue
+        zf_domain, zf_typ, zf_value = zf_line.split(maxsplit=2)
+        zf_domain = zf_domain.rstrip(".")
+        zf_value = zf_value.strip()
+        mockdns_base.setdefault(zf_typ, {})[zf_domain] = zf_value
+
+
+class MockSSHExec:
+    def logged(self, func, kwargs):
+        return func(**kwargs)
+
+    def call(self, func, kwargs):
+        return func(**kwargs)
+
+
+class TestZonefileChecks:
+    def test_check_zonefile_all_ok(self, cm_data, mockdns_base):
+        zonefile = cm_data.get("zftest.zone")
+        parse_zonefile_into_dict(zonefile, mockdns_base)
+        required_diff, recommended_diff = remote_funcs.check_zonefile(zonefile)
+        assert not required_diff and not recommended_diff
+
+    def test_check_zonefile_recommended_not_set(self, cm_data, mockdns_base):
+        zonefile = cm_data.get("zftest.zone")
+        zonefile_mocked = zonefile.split("; Recommended")[0]
+        parse_zonefile_into_dict(zonefile_mocked, mockdns_base)
+        required_diff, recommended_diff = remote_funcs.check_zonefile(zonefile)
+        assert not required_diff
+        assert len(recommended_diff) == 8
+
+    def test_check_zonefile_output_required_fine(self, cm_data, mockdns_base, mockout):
+        zonefile = cm_data.get("zftest.zone")
+        zonefile_mocked = zonefile.split("; Recommended")[0]
+        parse_zonefile_into_dict(zonefile_mocked, mockdns_base, only_required=True)
+        mssh = MockSSHExec()
+        res = check_full_zone(mssh, mockdns_base, out=mockout, zonefile=zonefile)
+        assert res == 0
+        assert "WARNING" in mockout.captured_plain[0]
+        assert len(mockout.captured_plain) == 9
+
+    def test_check_zonefile_output_full(self, cm_data, mockdns_base, mockout):
+        zonefile = cm_data.get("zftest.zone")
+        parse_zonefile_into_dict(zonefile, mockdns_base)
+        mssh = MockSSHExec()
+        res = check_full_zone(mssh, mockdns_base, out=mockout, zonefile=zonefile)
+        assert res == 0
+        assert not mockout.captured_red
+        assert "correct" in mockout.captured_green[0]
+        assert not mockout.captured_red

www/src/index.md

@@ -11,7 +11,11 @@ for Delta Chat users.  For details how it avoids storing personal information
 please see our [privacy policy](privacy.html). 
 {% endif %}
 
-👉 **Tap** or scan this QR code to get a `@{{config.mail_domain}}` chat profile
+<a class="cta-button" href="DCACCOUNT:https://{{ config.mail_domain }}/new">Get a {{config.mail_domain}} chat profile</a>
+
+If you are viewing this page on a different device
+without a Delta Chat app,
+you can also **scan this QR code** with Delta Chat:
 
 <a href="DCACCOUNT:https://{{ config.mail_domain }}/new">
     <img width=300 style="float: none;" src="qr-chatmail-invite-{{config.mail_domain}}.png" /></a>

www/src/info.md

@@ -8,11 +8,9 @@ for the usage in chats, especially DeltaChat.
 
 ### Choosing a chatmail address instead of using a random one
 
-In the Delta Chat account setup 
+In the Delta Chat account setup you may tap `Create a profile` then `Use other server` and choose `Classic e-mail login`. Here fill the two fields like this: 
-you may tap `I already have a profile`
-and fill the two fields like this: 
 
-- `Address`: invent a word with
+- `E-Mail Address`: invent a word with
 {% if username_min_length == username_max_length %}
   *exactly* {{ username_min_length }}
 {% else %}
@@ -26,7 +24,7 @@ and fill the two fields like this:
   characters
   and append `@{{config.mail_domain}}` to it.
 
-- `Password`: invent at least {{ password_min_length }} characters.
+- `Existing Password`: invent at least {{ password_min_length }} characters.
 
 If the e-mail address is not yet taken, you'll get that account. 
 The first login sets your password. 

www/src/main.css

@@ -72,3 +72,15 @@ code {
   color: red;
   font-weight: bold;
 }
+
+.cta-button, .cta-button:hover, .cta-button:visited {
+    border: 1.5px solid #a4c2d0;
+    border-radius: 5px;
+    padding: 10px;
+    display: inline-block;
+    margin: 10px 0;
+
+    background: linear-gradient(120deg, #77888f, #364e59);
+    color: white !important;
+    font-weight: bold;
+}