add DNS tests, make remote ssh-exec errors show locally, cleanup ssh-bootstrap

- don't try to guess IP addresses but insist on A and AAAA records
- try to allow ipv4 or ipv6 only zones
- move chatmail.zone generation to jinja so we can have conditionals

.github/workflows/staging-ipv4.testrun.org-default.zone

@@ -1,20 +0,0 @@
-;; Zone file for staging-ipv4.testrun.org
-
-$ORIGIN staging-ipv4.testrun.org.
-$TTL 300
-
-@  IN  SOA     ns.testrun.org. root.nine.testrun.org (
-  2023010101 ; Serial
-  7200       ; Refresh
-  3600       ; Retry
-  1209600    ; Expire
-  3600       ; Negative response caching TTL
-)
-
-;; Nameservers.
-@  IN NS ns.testrun.org.
-
-;; DNS records.
-@       IN      A       37.27.95.249
-mta-sts.staging-ipv4.testrun.org.    CNAME   staging-ipv4.testrun.org.
-www.staging-ipv4.testrun.org.        CNAME   staging-ipv4.testrun.org.

.github/workflows/test-and-deploy-ipv4only.yaml

@@ -1,98 +0,0 @@
-name: deploy on staging-ipv4.testrun.org, and run tests
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    paths-ignore:
-      - 'scripts/**'
-      - '**/README.md'
-      - 'CHANGELOG.md'
-      - 'LICENSE'
-
-jobs:
-  deploy:
-    name: deploy on staging-ipv4.testrun.org, and run tests
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    concurrency:
-      group: ci-ipv4-${{ github.workflow }}-${{ github.ref }}
-      cancel-in-progress: ${{ !contains(github.ref, '$GITHUB_REF') }}
-    steps:
-      - uses: jsok/serialize-workflow-action@v1
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: actions/checkout@v4
-
-      - name: prepare SSH
-        run: |
-          mkdir ~/.ssh
-          echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
-          chmod 600 ~/.ssh/id_ed25519
-          ssh-keyscan staging-ipv4.testrun.org > ~/.ssh/known_hosts
-          # save previous acme & dkim state
-          rsync -avz root@staging-ipv4.testrun.org:/var/lib/acme acme-ipv4 || true
-          rsync -avz root@staging-ipv4.testrun.org:/etc/dkimkeys dkimkeys-ipv4 || true
-          # store previous acme & dkim state on ns.testrun.org, if it contains useful certs
-          if [ -f dkimkeys-ipv4/dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys-ipv4 root@ns.testrun.org:/tmp/ || true; fi
-          if [ "$(ls -A acme-ipv4/acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme-ipv4 root@ns.testrun.org:/tmp/ || true; fi
-          # make sure CAA record isn't set
-          ssh -o StrictHostKeyChecking=accept-new root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging-ipv4.testrun.org.zone
-          ssh root@ns.testrun.org systemctl reload nsd
-
-      - name: rebuild staging-ipv4.testrun.org to have a clean VPS
-        run: |
-            curl -X POST \
-            -H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
-            -H "Content-Type: application/json" \
-            -d '{"image":"debian-12"}' \
-            "https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_SERVER_ID }}/actions/rebuild"
-
-      - run: scripts/initenv.sh
-
-      - name: append venv/bin to PATH
-        run: echo venv/bin >>$GITHUB_PATH
-
-      - name: upload TLS cert after rebuilding
-        run: |
-          echo " --- wait until staging-ipv4.testrun.org VPS is rebuilt --- "
-          rm ~/.ssh/known_hosts
-          while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u ; do sleep 1 ; done
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org id -u
-          # download acme & dkim state from ns.testrun.org
-          rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme-ipv4 acme-restore || true
-          rsync -avz root@ns.testrun.org:/tmp/dkimkeys-ipv4 dkimkeys-restore || true
-          # restore acme & dkim state to staging2.testrun.org
-          rsync -avz acme-restore/acme-ipv4/acme root@staging-ipv4.testrun.org:/var/lib/acme || true
-          rsync -avz dkimkeys-restore/dkimkeys-ipv4/dkimkeys root@staging-ipv4.testrun.org:/etc/dkimkeys || true
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org chown root:root -R /var/lib/acme || true
-
-      - name: run formatting checks 
-        run: cmdeploy fmt -v 
-
-      - name: run deploy-chatmail offline tests 
-        run: pytest --pyargs cmdeploy 
-
-      - run: |
-          cmdeploy init staging-ipv4.testrun.org
-          sed -i 's#disable_ipv6 = False#disable_ipv6 = True#' chatmail.ini
-
-      - run: cmdeploy run
-
-      - name: set DNS entries
-        run: |
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
-          cmdeploy dns --zonefile staging-generated.zone
-          cat staging-generated.zone >> .github/workflows/staging-ipv4.testrun.org-default.zone
-          cat .github/workflows/staging-ipv4.testrun.org-default.zone
-          scp .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
-          ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
-          ssh root@ns.testrun.org systemctl reload nsd
-
-      - name: cmdeploy test
-        run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow
-
-      - name: cmdeploy dns (try 3 times)
-        run: cmdeploy dns || cmdeploy dns || cmdeploy dns
-

CHANGELOG.md

@@ -2,76 +2,18 @@
 
 ## untagged
 
-- fix checking for required DNS records
+- BREAKING: new required chatmail.ini values:
-  ([#412](https://github.com/deltachat/chatmail/pull/412))
 
-- add a paragraph about "account deletion" to info page 
+  mailboxes_dir = /home/vmail/mail/{mail_domain}
-  ([#405](https://github.com/deltachat/chatmail/pull/405))
+  passdb = /home/vmail/passdb.sqlite
   
-- avoid nginx listening on ipv6 if v6 is dsiabled 
+  reducing hardcoding these two paths all over the files, also improving testability. 
-  ([#402](https://github.com/deltachat/chatmail/pull/402))
-
-- refactor ssh-based execution to allow organizing remote functions in
-  modules. 
-  ([#396](https://github.com/deltachat/chatmail/pull/396))
-
-- trigger "apt upgrade" during "cmdeploy run" 
-  ([#398](https://github.com/deltachat/chatmail/pull/398))
-
-- drop hispanilandia passthrough address
-  ([#401](https://github.com/deltachat/chatmail/pull/401))
-
-- set CAA record flags to 0
-
-- add IMAP capabilities instead of overwriting them
-  ([#413](https://github.com/deltachat/chatmail/pull/413))
-
-
-## 1.4.1 2024-07-31
-
-- fix metadata dictproxy which would confuse transactions
-  resulting in missed notifications and other issues. 
-  ([#393](https://github.com/deltachat/chatmail/pull/393))
-  ([#394](https://github.com/deltachat/chatmail/pull/394))
-
-- add optional "imap_rawlog" config option. If true, 
-  .in/.out files are created in user home dirs 
-  containing the imap protocol messages. 
-  ([#389](https://github.com/deltachat/chatmail/pull/389))
-
-## 1.4.0 2024-07-28
-
-- Add `disable_ipv6` config option to chatmail.ini.
-  Required if the server doesn't have IPv6 connectivity.
-  ([#312](https://github.com/deltachat/chatmail/pull/312))
-
-- allow current K9/Thunderbird-mail releases to send encrypted messages
-  outside by accepting their localized "encrypted subject" strings. 
-  ([#370](https://github.com/deltachat/chatmail/pull/370))
-
-- Migrate and remove sqlite database in favor of password/lastlogin tracking 
-  in a user's maildir.  
-  ([#379](https://github.com/deltachat/chatmail/pull/379))
-
-- Require pyinfra V3 installed on the client side,
-  run `./scripts/initenv.sh` to upgrade locally.
-  ([#378](https://github.com/deltachat/chatmail/pull/378))
-
-- don't hardcode "/home/vmail" paths but rather set them 
-  once in the config object and use it everywhere else, 
-  thereby also improving testability.  
   ([#351](https://github.com/deltachat/chatmail/pull/351))
-  temporarily introduced obligatory "passdb_path" and "mailboxes_dir" 
-  settings but they were removed/obsoleted in 
-  ([#380](https://github.com/deltachat/chatmail/pull/380))
 
 - BREAKING: new required chatmail.ini value 'delete_inactive_users_after = 100'
   which removes users from database and mails after 100 days without any login. 
   ([#350](https://github.com/deltachat/chatmail/pull/350))
 
-- Refine DNS checking to distinguish between "required" and "recommended" settings 
-  ([#372](https://github.com/deltachat/chatmail/pull/372))
-
 - reload nginx in the acmetool cronjob
   ([#360](https://github.com/deltachat/chatmail/pull/360))
 

README.md

@@ -34,8 +34,8 @@ Please substitute it with your own domain.
     scripts/cmdeploy init chat.example.org  # <-- use your domain 
    ```
 
-3. Point your domain to the server's IP address,
+3. Setup first DNS records for your chatmail domain, 
-   if you haven't done so already.
+   according to the hints provided by `cmdeploy init`.
    Verify that SSH root login works:
 
    ```
@@ -47,8 +47,7 @@ Please substitute it with your own domain.
    ```
     scripts/cmdeploy run
    ```
-   This script will check that you have all necessary DNS records.
+   This script will also show you additional DNS records
-   If DNS records are missing, it will recommend
    which you should configure at your DNS provider
    (it can take some time until they are public).
 
@@ -60,7 +59,7 @@ To check the status of your remotely running chatmail service:
 scripts/cmdeploy status
 ```
 
-To display and check all recommended DNS records:
+To check whether your DNS records are correct:
 
 ```
 scripts/cmdeploy dns
@@ -187,113 +186,3 @@ to MAIL FROM with
 and rejects incorrectly authenticated emails with [`reject_sender_login_mismatch`](reject_sender_login_mismatch) policy.
 `From:` header must correspond to envelope MAIL FROM,
 this is ensured by `filtermail` proxy.
-
-## Migrating chatmail server to a new host
-
-If you want to migrate your chatmail server to a new host,
-follow these steps:
-
-1. Block all ports except 80 and 22 with firewall on a new server.
-
-   To do this, add the following config to `/etc/nftables.conf`:
-```
-#!/usr/sbin/nft -f
-
-flush ruleset
-
-table inet filter {
-    chain input {
-        type filter hook input priority filter; policy drop;
-
-        # Accept ICMP.
-        # It is especially important to accept ICMPv6 ND messages,
-        # otherwise IPv6 connectivity breaks.
-        icmp type { echo-request } accept
-        icmpv6 type { echo-request, nd-neighbor-solicit, nd-router-advert, nd-neighbor-advert } accept
-
-        tcp dport { ssh, http } accept
-
-        ct state established accept
-    }
-    chain forward {
-        type filter hook forward priority filter;
-    }
-    chain output {
-        type filter hook output priority filter;
-    }
-}
-```
-   Then execute `nft -f /etc/nftables.conf` as root.
-
-   This will ensure users will not connect to the new server
-   and mails will not be delivered to the new server
-   before you finish the setup.
-
-   Port 22 is needed for SSH access
-   and port 80 is needed to get a TLS certificate.
-   They are not used by Delta Chat
-   or by other email servers trying to deliver the messages.
-
-2. Point DNS to the new IP addresses.
-
-   You can already remove the old IP addresses from DNS.
-   Existing Delta Chat users will still be able to connect
-   to the old server, send and receive messages,
-   but new users will fail to create new profiles
-   with your chatmail server.
-
-3. Setup the new server with `cmdeploy`.
-
-   This step is similar to initial setup.
-   However, because ports Delta Chat uses are blocked,
-   new server will not become usable immediately.
-   If other servers try to deliver messages to your new server they will fail,
-   but normally email servers will retry delivering messages
-   for at least a week, so messages will not be lost.
-
-4. Firewall all ports except `ssh` (22) on the old server.
-   Existing users will not be able to connect from now on
-   and no more messages will be delivered to your old chatmail server.
-
-   Blocking users from connecting to the new server
-   until mailboxes are migrated is needed to avoid UID validity change.
-   If Delta Chat connects to the new server before it is fully set up,
-   it will lose track of the IMAP message UID
-   and miss messages that arrived during migration.
-
-   Same for SMTP port 25, you want it blocked during migration so no new mails arrive
-   while the server is moving.
-
-5. Use `rsync -avz` over SSH to copy /home/vmail/mail from the old server to the new one
-   preserving file permissions and timestamps.
-
-6. Unblock ports used by Delta Chat and SMTP message exchange.
-   For that you can modify `/etc/nftables.conf` as follows:
-```
-#!/usr/sbin/nft -f
-
-flush ruleset
-
-table inet filter {
-    chain input {
-        type filter hook input priority filter; policy drop;
-
-        # Accept ICMP.
-        # It is especially important to accept ICMPv6 ND messages,
-        # otherwise IPv6 connectivity breaks.
-        icmp type { echo-request } accept
-        icmpv6 type { echo-request, nd-neighbor-solicit, nd-router-advert, nd-neighbor-advert } accept
-
-        tcp dport { ssh, smtp, http, https, imap, imaps, submission, submissions } accept
-
-        ct state established accept
-    }
-    chain forward {
-        type filter hook forward priority filter;
-    }
-    chain output {
-        type filter hook output priority filter;
-    }
-}
-```
-Execute `nft -f /etc/nftables.conf` as root to apply the changes.

chatmaild/pyproject.toml

@@ -27,7 +27,6 @@ filtermail = "chatmaild.filtermail:main"
 echobot = "chatmaild.echo:main"
 chatmail-metrics = "chatmaild.metrics:main"
 delete_inactive_users = "chatmaild.delete_inactive_users:main"
-lastlogin = "chatmaild.lastlogin:main"
 
 [project.entry-points.pytest11]
 "chatmaild.testplugin" = "chatmaild.tests.plugin"

chatmaild/src/chatmaild/__init__.py

@@ -1 +0,0 @@
-

chatmaild/src/chatmaild/common_encrypted_subjects.py

@@ -1,59 +0,0 @@
-"""Generated from deltachat, draft-ietf-lamps-header-protection, and
-encrypted_subject localizations in
-https://github.com/thunderbird/thunderbird-android/
-
-"""
-
-common_encrypted_subjects = {
-    "...",
-    "[...]",
-    "암호화된 메시지",
-    "Ĉifrita mesaĝo",
-    "Courriel chiffré",
-    "Dulrituð skilaboð",
-    "Encrypted Message",
-    "Fersifere berjocht",
-    "Kemennadenn enrineget",
-    "Krüptitud kiri",
-    "Krypterat meddelande",
-    "Krypteret besked",
-    "Kryptert melding",
-    "Mensagem criptografada",
-    "Mensagem encriptada",
-    "Mensaje cifrado",
-    "Mensaxe cifrada",
-    "Mesaj Criptat",
-    "Mesazh i Fshehtëzuar",
-    "Messaggio criptato",
-    "Messaghju cifratu",
-    "Missatge encriptat",
-    "Neges wedi'i Hamgryptio",
-    "Pesan terenkripsi",
-    "Salattu viesti",
-    "Şifreli İleti",
-    "Šifrēta ziņa",
-    "Šifrirana poruka",
-    "Šifrirano sporočilo",
-    "Šifruotas laiškas",
-    "Tin nhắn được mã hóa",
-    "Titkosított üzenet",
-    "Verschlüsselte Nachricht",
-    "Versleuteld bericht",
-    "Zašifrovaná zpráva",
-    "Zaszyfrowana wiadomość",
-    "Zifratu mezua",
-    "Κρυπτογραφημένο μήνυμα",
-    "Зашифроване повідомлення",
-    "Зашифрованное сообщение",
-    "Зашыфраваны ліст",
-    "Криптирано съобщение",
-    "Шифрована порука",
-    "დაშიფრული წერილი",
-    "הודעה מוצפנת",
-    "پیام رمزنگاری‌شده",
-    "رسالة مشفّرة",
-    "എൻക്രിപ്റ്റുചെയ്‌ത സന്ദേശം",
-    "加密邮件",
-    "已加密的訊息",
-    "暗号化されたメッセージ",
-}

chatmaild/src/chatmaild/config.py

@@ -2,10 +2,6 @@ from pathlib import Path
 
 import iniconfig
 
-from chatmaild.user import User
-
-echobot_password_path = Path("/run/echobot/password")
-
 
 def read_config(inipath):
     assert Path(inipath).exists(), inipath
@@ -20,7 +16,6 @@ class Config:
         self.mail_domain = params["mail_domain"]
         self.max_user_send_per_minute = int(params["max_user_send_per_minute"])
         self.max_mailbox_size = params["max_mailbox_size"]
-        self.max_message_size = int(params.get("max_message_size", "31457280"))
         self.delete_mails_after = params["delete_mails_after"]
         self.delete_inactive_users_after = int(params["delete_inactive_users_after"])
         self.username_min_length = int(params["username_min_length"])
@@ -28,37 +23,25 @@ class Config:
         self.password_min_length = int(params["password_min_length"])
         self.passthrough_senders = params["passthrough_senders"].split()
         self.passthrough_recipients = params["passthrough_recipients"].split()
+        self.mailboxes_dir = Path(params["mailboxes_dir"].strip())
+        self.passdb_path = Path(params["passdb_path"].strip())
         self.filtermail_smtp_port = int(params["filtermail_smtp_port"])
         self.postfix_reinject_port = int(params["postfix_reinject_port"])
-        self.disable_ipv6 = params.get("disable_ipv6", "false").lower() == "true"
-        self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true"
         self.iroh_relay = params.get("iroh_relay")
         self.privacy_postal = params.get("privacy_postal")
         self.privacy_mail = params.get("privacy_mail")
         self.privacy_pdo = params.get("privacy_pdo")
         self.privacy_supervisor = params.get("privacy_supervisor")
 
-        # deprecated option
-        mbdir = params.get("mailboxes_dir", f"/home/vmail/mail/{self.mail_domain}")
-        self.mailboxes_dir = Path(mbdir.strip())
-
-        # old unused option (except for first migration from sqlite to maildir store)
-        self.passdb_path = Path(params.get("passdb_path", "/home/vmail/passdb.sqlite"))
-
     def _getbytefile(self):
         return open(self._inipath, "rb")
 
-    def get_user(self, addr):
+    def get_user_maildir(self, addr):
-        if not addr or "@" not in addr or "/" in addr:
+        if addr and addr != "." and "/" not in addr:
-            raise ValueError(f"invalid address {addr!r}")
+            res = self.mailboxes_dir.joinpath(addr).resolve()
-
+            if res.is_relative_to(self.mailboxes_dir):
-        maildir = self.mailboxes_dir.joinpath(addr)
+                return res
-        if addr.startswith("echo@"):
+        raise ValueError(f"invalid address {addr!r}")
-            password_path = echobot_password_path
-        else:
-            password_path = maildir.joinpath("password")
-
-        return User(maildir, addr, password_path, uid="vmail", gid="vmail")
 
 
 def write_initial_config(inipath, mail_domain, overrides):
@@ -71,19 +54,14 @@ def write_initial_config(inipath, mail_domain, overrides):
 
     # apply config overrides
     new_lines = []
-    extra = overrides.copy()
     for line in content.split("\n"):
         new_line = line.strip()
         if new_line and new_line[0] not in "#[":
             name, value = map(str.strip, new_line.split("=", maxsplit=1))
-            value = extra.pop(name, value)
+            value = overrides.get(name, value)
             new_line = f"{name} = {value}"
         new_lines.append(new_line)
 
-    for name, value in extra.items():
-        new_line = f"{name} = {value}"
-        new_lines.append(new_line)
-
     content = "\n".join(new_lines)
 
     # apply testrun privacy overrides

chatmaild/src/chatmaild/database.py

@@ -0,0 +1,133 @@
+import contextlib
+import sqlite3
+import time
+from pathlib import Path
+
+
+class DBError(Exception):
+    """error during an operation on the database."""
+
+
+class Connection:
+    def __init__(self, sqlconn, write):
+        self._sqlconn = sqlconn
+        self._write = write
+
+    def close(self):
+        self._sqlconn.close()
+
+    def commit(self):
+        self._sqlconn.commit()
+
+    def rollback(self):
+        self._sqlconn.rollback()
+
+    def execute(self, query, params=()):
+        cur = self.cursor()
+        try:
+            cur.execute(query, params)
+        except sqlite3.IntegrityError as e:
+            raise DBError(e)
+        return cur
+
+    def cursor(self):
+        return self._sqlconn.cursor()
+
+    def get_user(self, addr: str) -> {}:
+        """Get a row from the users table."""
+        q = "SELECT addr, password, last_login from users WHERE addr = ?"
+        row = self._sqlconn.execute(q, (addr,)).fetchone()
+        result = {}
+        if row:
+            result = dict(
+                user=row[0],
+                password=row[1],
+                last_login=row[2],
+            )
+        return result
+
+
+class Database:
+    def __init__(self, path: str):
+        self.path = Path(path)
+        self.ensure_tables()
+
+    def _get_connection(
+        self, write=False, transaction=False, closing=False
+    ) -> Connection:
+        # we let the database serialize all writers at connection time
+        # to play it very safe (we don't have massive amounts of writes).
+        mode = "ro"
+        if write:
+            mode = "rw"
+        if not self.path.exists():
+            mode = "rwc"
+        uri = "file:%s?mode=%s" % (self.path, mode)
+        sqlconn = sqlite3.connect(
+            uri,
+            timeout=60,
+            isolation_level=None if transaction else "DEFERRED",
+            uri=True,
+        )
+
+        # Enable Write-Ahead Logging to avoid readers blocking writers and vice versa.
+        if write:
+            sqlconn.execute("PRAGMA journal_mode=wal")
+
+        if transaction:
+            start_time = time.time()
+            while 1:
+                try:
+                    sqlconn.execute("begin immediate")
+                    break
+                except sqlite3.OperationalError:
+                    # another thread may be writing, give it a chance to finish
+                    time.sleep(0.1)
+                    if time.time() - start_time > 5:
+                        # if it takes this long, something is wrong
+                        raise
+        conn = Connection(sqlconn, write=write)
+        if closing:
+            conn = contextlib.closing(conn)
+        return conn
+
+    @contextlib.contextmanager
+    def write_transaction(self):
+        conn = self._get_connection(closing=False, write=True, transaction=True)
+        try:
+            yield conn
+        except Exception:
+            conn.rollback()
+            conn.close()
+            raise
+        else:
+            conn.commit()
+            conn.close()
+
+    def read_connection(self, closing=True) -> Connection:
+        return self._get_connection(closing=closing, write=False)
+
+    def get_schema_version(self) -> int:
+        with self.read_connection() as conn:
+            dbversion = conn.execute("PRAGMA user_version").fetchone()[0]
+        return dbversion
+
+    CURRENT_DBVERSION = 1
+
+    def ensure_tables(self):
+        with self.write_transaction() as conn:
+            if self.get_schema_version() > 1:
+                raise DBError(
+                    "version is %s; downgrading schema is not supported"
+                    % (self.get_schema_version(),)
+                )
+            conn.execute(
+                """
+                CREATE TABLE IF NOT EXISTS users (
+                    addr TEXT PRIMARY KEY,
+                    password TEXT,
+                    last_login INTEGER
+                )
+            """,
+            )
+            conn.execute("PRAGMA user_version=%s" % (self.CURRENT_DBVERSION,))

chatmaild/src/chatmaild/delete_inactive_users.py

@@ -2,30 +2,32 @@
 Remove inactive users
 """
 
-import os
 import shutil
 import sys
 import time
 
 from .config import read_config
+from .database import Database
+from .doveauth import iter_userdb_lastlogin_before
 
 
-def delete_inactive_users(config):
+def delete_inactive_users(db, config, CHUNK=100):
     cutoff_date = time.time() - config.delete_inactive_users_after * 86400
-    for addr in os.listdir(config.mailboxes_dir):
-        try:
-            user = config.get_user(addr)
-        except ValueError:
-            continue
 
-        read_timestamp = user.get_last_login_timestamp()
+    old_users = iter_userdb_lastlogin_before(db, cutoff_date)
-        if read_timestamp and read_timestamp < cutoff_date:
+    chunks = (old_users[i : i + CHUNK] for i in range(0, len(old_users), CHUNK))
-            path = config.mailboxes_dir.joinpath(addr)
+    for sublist in chunks:
-            assert path == user.maildir
+        for user in sublist:
-            shutil.rmtree(path, ignore_errors=True)
+            user_mail_dir = config.get_user_maildir(user)
+            shutil.rmtree(user_mail_dir, ignore_errors=True)
+
+        with db.write_transaction() as conn:
+            for user in sublist:
+                conn.execute("DELETE FROM users WHERE addr = ?", (user,))
 
 
 def main():
     (cfgpath,) = sys.argv[1:]
     config = read_config(cfgpath)
-    delete_inactive_users(config)
+    db = Database(config.passdb_path)
+    delete_inactive_users(db, config)

chatmaild/src/chatmaild/dictproxy.py

@@ -1,94 +0,0 @@
-import logging
-import os
-from socketserver import StreamRequestHandler, ThreadingUnixStreamServer
-
-
-class DictProxy:
-    def loop_forever(self, rfile, wfile):
-        # Transaction storage is local to each handler loop.
-        # Dovecot reuses transaction IDs across connections,
-        # starting transaction with the name `1`
-        # on two different connections to the same proxy sometimes.
-        transactions = {}
-
-        while True:
-            msg = rfile.readline().strip().decode()
-            if not msg:
-                break
-
-            res = self.handle_dovecot_request(msg, transactions)
-            if res:
-                wfile.write(res.encode("ascii"))
-                wfile.flush()
-
-    def handle_dovecot_request(self, msg, transactions):
-        # see https://doc.dovecot.org/developer_manual/design/dict_protocol/#dovecot-dict-protocol
-        short_command = msg[0]
-        parts = msg[1:].split("\t")
-
-        if short_command == "L":
-            return self.handle_lookup(parts)
-        elif short_command == "I":
-            return self.handle_iterate(parts)
-        elif short_command == "H":
-            return  # no version checking
-
-        if short_command not in ("BSC"):
-            logging.warning(f"unknown dictproxy request: {msg!r}")
-            return
-
-        transaction_id = parts[0]
-
-        if short_command == "B":
-            return self.handle_begin_transaction(transaction_id, parts, transactions)
-        elif short_command == "C":
-            return self.handle_commit_transaction(transaction_id, parts, transactions)
-        elif short_command == "S":
-            addr = transactions[transaction_id]["addr"]
-            if not self.handle_set(addr, parts):
-                transactions[transaction_id]["res"] = "F\n"
-                logging.error(f"dictproxy-set failed for {addr!r}: {msg!r}")
-
-    def handle_lookup(self, parts):
-        logging.warning(f"lookup ignored: {parts!r}")
-        return "N\n"
-
-    def handle_iterate(self, parts):
-        # Empty line means ITER_FINISHED.
-        # If we don't return empty line Dovecot will timeout.
-        return "\n"
-
-    def handle_begin_transaction(self, transaction_id, parts, transactions):
-        addr = parts[1]
-        transactions[transaction_id] = dict(addr=addr, res="O\n")
-
-    def handle_set(self, addr, parts):
-        # For documentation on key structure see
-        # https://github.com/dovecot/core/blob/main/src/lib-storage/mailbox-attribute.h
-        return False
-
-    def handle_commit_transaction(self, transaction_id, parts, transactions):
-        # return whatever "set" command(s) set as result.
-        return transactions.pop(transaction_id)["res"]
-
-    def serve_forever_from_socket(self, socket):
-        dictproxy = self
-
-        class Handler(StreamRequestHandler):
-            def handle(self):
-                try:
-                    dictproxy.loop_forever(self.rfile, self.wfile)
-                except Exception:
-                    logging.exception("Exception in the handler")
-                    raise
-
-        try:
-            os.unlink(socket)
-        except FileNotFoundError:
-            pass
-
-        with ThreadingUnixStreamServer(socket, Handler) as server:
-            try:
-                server.serve_forever()
-            except KeyboardInterrupt:
-                pass

chatmaild/src/chatmaild/doveauth.py

@@ -3,14 +3,24 @@ import json
 import logging
 import os
 import sys
+import time
+from pathlib import Path
+from socketserver import (
+    StreamRequestHandler,
+    ThreadingMixIn,
+    UnixStreamServer,
+)
 
 from .config import Config, read_config
-from .dictproxy import DictProxy
+from .database import Database
-from .migrate_db import migrate_from_db_to_maildir
 
 NOCREATE_FILE = "/etc/chatmail-nocreate"
 
 
+class UnknownCommand(ValueError):
+    """dictproxy handler received an unkown command"""
+
+
 def encrypt_password(password: str):
     # https://doc.dovecot.org/configuration_manual/authentication/password_schemes/
     passhash = crypt.crypt(password, crypt.METHOD_SHA512)
@@ -55,6 +65,93 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
     return True
 
 
+def get_user_data(db, config: Config, user):
+    if user == f"echo@{config.mail_domain}":
+        return dict(
+            home=str(config.get_user_maildir(user)),
+            uid="vmail",
+            gid="vmail",
+        )
+
+    with db.read_connection() as conn:
+        result = conn.get_user(user)
+    if result:
+        result["home"] = str(config.get_user_maildir(user))
+        result["uid"] = "vmail"
+        result["gid"] = "vmail"
+    return result
+
+
+def lookup_userdb(db, config: Config, user):
+    return get_user_data(db, config, user)
+
+
+def lookup_passdb(db, config: Config, user, cleartext_password, last_login=None):
+    if user == f"echo@{config.mail_domain}":
+        # Echobot writes password it wants to log in with into /run/echobot/password
+        try:
+            password = Path("/run/echobot/password").read_text()
+        except Exception:
+            logging.exception("Exception when trying to read /run/echobot/password")
+            return None
+
+        return dict(
+            home=str(config.get_user_maildir(user)),
+            uid="vmail",
+            gid="vmail",
+            password=encrypt_password(password),
+        )
+
+    if last_login is None:
+        last_login = time.time()
+    last_login = int(last_login)
+
+    with db.write_transaction() as conn:
+        userdata = conn.get_user(user)
+        if userdata:
+            # Update last login time.
+            conn.execute(
+                "UPDATE users SET last_login=? WHERE addr=?", (last_login, user)
+            )
+
+            userdata["home"] = str(config.get_user_maildir(user))
+            userdata["uid"] = "vmail"
+            userdata["gid"] = "vmail"
+            return userdata
+        if not is_allowed_to_create(config, user, cleartext_password):
+            return
+
+        encrypted_password = encrypt_password(cleartext_password)
+        q = """INSERT INTO users (addr, password, last_login)
+               VALUES (?, ?, ?)"""
+        conn.execute(q, (user, encrypted_password, last_login))
+        print(f"Created address: {user}", file=sys.stderr)
+        return dict(
+            home=str(config.get_user_maildir(user)),
+            uid="vmail",
+            gid="vmail",
+            password=encrypted_password,
+        )
+
+
+def iter_userdb(db) -> list:
+    """Get a list of all user addresses."""
+    with db.read_connection() as conn:
+        rows = conn.execute(
+            "SELECT addr from users",
+        ).fetchall()
+    return [x[0] for x in rows]
+
+
+def iter_userdb_lastlogin_before(db, cutoff_date):
+    """Get a list of users where last login was before cutoff_date."""
+    with db.read_connection() as conn:
+        rows = conn.execute(
+            "SELECT addr FROM users WHERE last_login < ?", (cutoff_date,)
+        ).fetchall()
+    return [x[0] for x in rows]
+
+
 def split_and_unescape(s):
     """Split strings using double quote as a separator and backslash as escape character
     into parts."""
@@ -81,12 +178,15 @@ def split_and_unescape(s):
     yield out
 
 
-class AuthDictProxy(DictProxy):
+def handle_dovecot_request(msg, db, config: Config):
-    def __init__(self, config):
+    # see https://doc.dovecot.org/3.0/developer_manual/design/dict_protocol/
-        super().__init__()
+    short_command = msg[0]
-        self.config = config
+    if short_command == "H":  # HELLO
+        # we don't do any checking on versions and just return
+        return
+    elif short_command == "L":  # LOOKUP
+        parts = msg[1:].split("\t")
 
-    def handle_lookup(self, parts):
         # Dovecot <2.3.17 has only one part,
         # do not attempt to read any other parts for compatibility.
         keyname = parts[0]
@@ -94,14 +194,13 @@ class AuthDictProxy(DictProxy):
         namespace, type, args = keyname.split("/", 2)
         args = list(split_and_unescape(args))
 
-        config = self.config
         reply_command = "F"
         res = ""
         if namespace == "shared":
             if type == "userdb":
                 user = args[0]
                 if user.endswith(f"@{config.mail_domain}"):
-                    res = self.lookup_userdb(user)
+                    res = lookup_userdb(db, config, user)
                 if res:
                     reply_command = "O"
                 else:
@@ -109,48 +208,62 @@ class AuthDictProxy(DictProxy):
             elif type == "passdb":
                 user = args[1]
                 if user.endswith(f"@{config.mail_domain}"):
-                    res = self.lookup_passdb(user, cleartext_password=args[0])
+                    res = lookup_passdb(db, config, user, cleartext_password=args[0])
                 if res:
                     reply_command = "O"
                 else:
                     reply_command = "N"
         json_res = json.dumps(res) if res else ""
         return f"{reply_command}{json_res}\n"
-
+    elif short_command == "I":  # ITERATE
-    def handle_iterate(self, parts):
         # example: I0\t0\tshared/userdb/
+        parts = msg[1:].split("\t")
         if parts[2] == "shared/userdb/":
-            result = "".join(
+            result = "".join(f"Oshared/userdb/{user}\t\n" for user in iter_userdb(db))
-                f"Oshared/userdb/{user}\t\n" for user in self.iter_userdb()
-            )
             return f"{result}\n"
 
-    def iter_userdb(self) -> list:
+    raise UnknownCommand(msg)
-        """Get a list of all user addresses."""
-        return [x for x in os.listdir(self.config.mailboxes_dir) if "@" in x]
 
-    def lookup_userdb(self, addr):
-        return self.config.get_user(addr).get_userdb_dict()
 
-    def lookup_passdb(self, addr, cleartext_password):
+def handle_dovecot_protocol(rfile, wfile, db: Database, config: Config):
-        user = self.config.get_user(addr)
+    while True:
-        userdata = user.get_userdb_dict()
+        msg = rfile.readline().strip().decode()
-        if userdata:
+        if not msg:
-            return userdata
+            break
-        if not is_allowed_to_create(self.config, addr, cleartext_password):
+        try:
-            return
+            res = handle_dovecot_request(msg, db, config)
+        except UnknownCommand:
+            logging.warning("unknown command: %r", msg)
+        else:
+            if res:
+                wfile.write(res.encode("ascii"))
+                wfile.flush()
 
-        user.set_password(encrypt_password(cleartext_password))
+
-        print(f"Created address: {addr}", file=sys.stderr)
+class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
-        return user.get_userdb_dict()
+    request_queue_size = 100
 
 
 def main():
     socket, cfgpath = sys.argv[1:]
     config = read_config(cfgpath)
+    db = Database(config.passdb_path)
 
-    migrate_from_db_to_maildir(config)
+    class Handler(StreamRequestHandler):
+        def handle(self):
+            try:
+                handle_dovecot_protocol(self.rfile, self.wfile, db, config)
+            except Exception:
+                logging.exception("Exception in the handler")
+                raise
 
-    dictproxy = AuthDictProxy(config=config)
+    try:
+        os.unlink(socket)
+    except FileNotFoundError:
+        pass
 
-    dictproxy.serve_forever_from_socket(socket)
+    with ThreadedUnixStreamServer(socket, Handler) as server:
+        try:
+            server.serve_forever()
+        except KeyboardInterrupt:
+            pass

chatmaild/src/chatmaild/echo.py

@@ -8,11 +8,11 @@ import logging
 import os
 import subprocess
 import sys
+from pathlib import Path
 
 from deltachat_rpc_client import Bot, DeltaChat, EventType, Rpc, events
 
-from chatmaild.config import echobot_password_path, read_config
+from chatmaild.config import read_config
-from chatmaild.doveauth import encrypt_password
 from chatmaild.newemail import create_newemail_dict
 
 hooks = events.HookCollection()
@@ -21,9 +21,9 @@ hooks = events.HookCollection()
 @hooks.on(events.RawEvent)
 def log_event(event):
     if event.kind == EventType.INFO:
-        logging.info(event.msg)
+        logging.info("%s", event.msg)
     elif event.kind == EventType.WARNING:
-        logging.warning(event.msg)
+        logging.warning("%s", event.msg)
 
 
 @hooks.on(events.RawEvent(EventType.ERROR))
@@ -45,7 +45,7 @@ def on_group_image_changed(event):
 
 @hooks.on(events.GroupNameChanged)
 def on_group_name_changed(event):
-    logging.info(f"group name changed, old name: {event.old_name}")
+    logging.info("group name changed, old name: %s", event.old_name)
 
 
 @hooks.on(events.NewMessage(func=lambda e: not e.command))
@@ -72,7 +72,7 @@ def main():
     with Rpc() as rpc:
         deltachat = DeltaChat(rpc)
         system_info = deltachat.get_system_info()
-        logging.info(f"Running deltachat core {system_info.deltachat_core_version}")
+        logging.info("Running deltachat core %s", system_info.deltachat_core_version)
 
         accounts = deltachat.get_all_accounts()
         account = accounts[0] if accounts else deltachat.add_account()
@@ -80,23 +80,23 @@ def main():
         bot = Bot(account, hooks)
 
         config = read_config(sys.argv[1])
-        addr = "echo@" + config.mail_domain
 
         # Create password file
         if bot.is_configured():
             password = bot.account.get_config("mail_pw")
         else:
             password = create_newemail_dict(config)["password"]
+        Path("/run/echobot/password").write_text(password)
 
-        echobot_password_path.write_text(encrypt_password(password))
         # Give the user which doveauth runs as access to the password file.
-        subprocess.check_call(
+        subprocess.run(
-            ["/usr/bin/setfacl", "-m", "user:vmail:r", echobot_password_path],
+            ["/usr/bin/setfacl", "-m", "user:vmail:r", "/run/echobot/password"],
+            check=True,
         )
 
         if not bot.is_configured():
-            bot.configure(addr, password)
+            email = "echo@" + config.mail_domain
-
+            bot.configure(email, password)
         bot.run_forever()
 
 

chatmaild/src/chatmaild/filedict.py

@@ -2,7 +2,6 @@ import json
 import logging
 import os
 from contextlib import contextmanager
-from random import randint
 
 import filelock
 
@@ -33,12 +32,5 @@ class FileDict:
         except FileNotFoundError:
             return {}
         except Exception:
-            logging.warning(f"corrupt serialization state at: {self.path!r}")
+            logging.warning("corrupt serialization state at: %r", self.path)
             return {}
-
-
-def write_bytes_atomic(path, content):
-    rint = randint(0, 10000000)
-    tmp = path.with_name(path.name + f".tmp-{rint}")
-    tmp.write_bytes(content)
-    os.rename(tmp, path)

chatmaild/src/chatmaild/filtermail.py

@@ -12,7 +12,6 @@ from smtplib import SMTP as SMTPClient
 
 from aiosmtpd.controller import Controller
 
-from .common_encrypted_subjects import common_encrypted_subjects
 from .config import read_config
 
 
@@ -112,7 +111,7 @@ def check_encrypted(message):
     """
     if not message.is_multipart():
         return False
-    if message.get("subject") not in common_encrypted_subjects:
+    if message.get("subject") not in {"...", "[...]"}:
         return False
     if message.get_content_type() != "multipart/encrypted":
         return False
@@ -153,7 +152,7 @@ class BeforeQueueHandler:
         self.send_rate_limiter = SendRateLimiter()
 
     async def handle_MAIL(self, server, session, envelope, address, mail_options):
-        logging.info(f"handle_MAIL from {address}")
+        logging.info("handle_MAIL from %s", address)
         envelope.mail_from = address
         max_sent = self.config.max_user_send_per_minute
         if not self.send_rate_limiter.is_sending_allowed(address, max_sent):
@@ -177,13 +176,13 @@ class BeforeQueueHandler:
 
     def check_DATA(self, envelope):
         """the central filtering function for e-mails."""
-        logging.info(f"Processing DATA message from {envelope.mail_from}")
+        logging.info("Processing DATA message from %s", envelope.mail_from)
 
         message = BytesParser(policy=policy.default).parsebytes(envelope.content)
         mail_encrypted = check_encrypted(message)
 
         _, from_addr = parseaddr(message.get("from").strip())
-        logging.info(f"mime-from: {from_addr} envelope-from: {envelope.mail_from!r}")
+        logging.info("mime-from: %s envelope-from: %r", from_addr, envelope.mail_from)
         if envelope.mail_from.lower() != from_addr.lower():
             return f"500 Invalid FROM <{from_addr!r}> for <{envelope.mail_from!r}>"
 

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -17,14 +17,11 @@ max_user_send_per_minute = 60
 # maximum mailbox size of a chatmail address
 max_mailbox_size = 100M
 
-# maximum message size for an e-mail in bytes
-max_message_size = 31457280
-
 # days after which mails are unconditionally deleted
 delete_mails_after = 20
 
-# days after which users without a successful login are deleted (database and mails)
+# days after which users without a login are deleted (database and mails)
-delete_inactive_users_after = 90
+delete_inactive_users_after = 100
 
 # minimum length a username must have
 username_min_length = 9
@@ -39,33 +36,24 @@ password_min_length = 9
 passthrough_senders =
 
 # list of e-mail recipients for which to accept outbound un-encrypted mails
-# (space-separated)
+passthrough_recipients = xstore@testrun.org groupsbot@hispanilandia.net
-passthrough_recipients = xstore@testrun.org 
 
 #
 # Deployment Details
 #
 
+# Directory where user mailboxes are stored
+mailboxes_dir = /home/vmail/mail/{mail_domain}
+
+# user address sqlite database path
+passdb_path = /home/vmail/passdb.sqlite
+
 # where the filtermail SMTP service listens
 filtermail_smtp_port = 10080
 
 # postfix accepts on the localhost reinject SMTP port
 postfix_reinject_port = 10025
 
-# if set to "True" IPv6 is disabled
-disable_ipv6 = False
-
-#
-# Debugging options 
-#
-
-# set to True if you want to track imap protocol execution
-# in per-maildir ".in/.out" files. 
-# Note that you need to manually cleanup these files
-# so use this option with caution on production servers. 
-imap_rawlog = false 
-
-
 #
 # Privacy Policy
 #

chatmaild/src/chatmaild/ini/override-testrun.ini

@@ -1,7 +1,7 @@
 
 [privacy]
 
-passthrough_recipients = privacy@testrun.org xstore@testrun.org
+passthrough_recipients = privacy@testrun.org xstore@testrun.org groupsbot@hispanilandia.net
 
 privacy_postal =
     Merlinux GmbH, Represented by the managing director H. Krekel,

chatmaild/src/chatmaild/lastlogin.py

@@ -1,31 +0,0 @@
-import sys
-
-from .config import read_config
-from .dictproxy import DictProxy
-
-
-class LastLoginDictProxy(DictProxy):
-    def __init__(self, config):
-        super().__init__()
-        self.config = config
-
-    def handle_set(self, addr, parts):
-        keyname = parts[1].split("/")
-        value = parts[2] if len(parts) > 2 else ""
-        if keyname[0] == "shared" and keyname[1] == "last-login":
-            if addr.startswith("echo@"):
-                return True
-            addr = keyname[2]
-            timestamp = int(value)
-            user = self.config.get_user(addr)
-            user.set_last_login_timestamp(timestamp)
-            return True
-
-        return False
-
-
-def main():
-    socket, config_path = sys.argv[1:]
-    config = read_config(config_path)
-    dictproxy = LastLoginDictProxy(config=config)
-    dictproxy.serve_forever_from_socket(socket)

chatmaild/src/chatmaild/metadata.py

@@ -1,11 +1,24 @@
 import logging
+import os
 import sys
+from socketserver import (
+    StreamRequestHandler,
+    ThreadingMixIn,
+    UnixStreamServer,
+)
 
 from .config import read_config
-from .dictproxy import DictProxy
 from .filedict import FileDict
 from .notifier import Notifier
 
+DICTPROXY_HELLO_CHAR = "H"
+DICTPROXY_LOOKUP_CHAR = "L"
+DICTPROXY_ITERATE_CHAR = "I"
+DICTPROXY_BEGIN_TRANSACTION_CHAR = "B"
+DICTPROXY_SET_CHAR = "S"
+DICTPROXY_COMMIT_TRANSACTION_CHAR = "C"
+DICTPROXY_TRANSACTION_CHARS = "BSC"
+
 
 class Metadata:
     # each SETMETADATA on this key appends to a list of unique device tokens
@@ -35,46 +48,82 @@ class Metadata:
         return mdict.get(self.DEVICETOKEN_KEY, [])
 
 
-class MetadataDictProxy(DictProxy):
+def handle_dovecot_protocol(rfile, wfile, notifier, metadata, iroh_relay=None):
-    def __init__(self, notifier, metadata, iroh_relay=None):
+    transactions = {}
-        super().__init__()
+    while True:
-        self.notifier = notifier
+        msg = rfile.readline().strip().decode()
-        self.metadata = metadata
+        if not msg:
-        self.iroh_relay = iroh_relay
+            break
 
-    def handle_lookup(self, parts):
+        res = handle_dovecot_request(msg, transactions, notifier, metadata, iroh_relay)
+        if res:
+            wfile.write(res.encode("ascii"))
+            wfile.flush()
+
+
+def handle_dovecot_request(msg, transactions, notifier, metadata, iroh_relay=None):
+    # see https://doc.dovecot.org/3.0/developer_manual/design/dict_protocol/
+    short_command = msg[0]
+    parts = msg[1:].split("\t")
+    if short_command == DICTPROXY_LOOKUP_CHAR:
         # Lpriv/43f5f508a7ea0366dff30200c15250e3/devicetoken\tlkj123poi@c2.testrun.org
         keyparts = parts[0].split("/", 2)
         if keyparts[0] == "priv":
             keyname = keyparts[2]
             addr = parts[1]
-            if keyname == self.metadata.DEVICETOKEN_KEY:
+            if keyname == metadata.DEVICETOKEN_KEY:
-                res = " ".join(self.metadata.get_tokens_for_addr(addr))
+                res = " ".join(metadata.get_tokens_for_addr(addr))
                 return f"O{res}\n"
         elif keyparts[0] == "shared":
             keyname = keyparts[2]
             if (
                 keyname == "vendor/vendor.dovecot/pvt/server/vendor/deltachat/irohrelay"
-                and self.iroh_relay
+                and iroh_relay
             ):
                 # Handle `GETMETADATA "" /shared/vendor/deltachat/irohrelay`
-                return f"O{self.iroh_relay}\n"
+                return f"O{iroh_relay}\n"
-        logging.warning(f"lookup ignored: {parts!r}")
+        logging.warning("lookup ignored: %r", msg)
         return "N\n"
+    elif short_command == DICTPROXY_ITERATE_CHAR:
+        # Empty line means ITER_FINISHED.
+        # If we don't return empty line Dovecot will timeout.
+        return "\n"
+    elif short_command == DICTPROXY_HELLO_CHAR:
+        return  # no version checking
 
-    def handle_set(self, addr, parts):
+    if short_command not in (DICTPROXY_TRANSACTION_CHARS):
+        logging.warning("unknown dictproxy request: %r", msg)
+        return
+
+    transaction_id = parts[0]
+
+    if short_command == DICTPROXY_BEGIN_TRANSACTION_CHAR:
+        addr = parts[1]
+        transactions[transaction_id] = dict(addr=addr, res="O\n")
+    elif short_command == DICTPROXY_COMMIT_TRANSACTION_CHAR:
+        # each set devicetoken operation persists directly
+        # and does not wait until a "commit" comes
+        # because our dovecot config does not involve
+        # multiple set-operations in a single commit
+        return transactions.pop(transaction_id)["res"]
+    elif short_command == DICTPROXY_SET_CHAR:
         # For documentation on key structure see
         # https://github.com/dovecot/core/blob/main/src/lib-storage/mailbox-attribute.h
+
         keyname = parts[1].split("/")
         value = parts[2] if len(parts) > 2 else ""
-        if keyname[0] == "priv" and keyname[2] == self.metadata.DEVICETOKEN_KEY:
+        addr = transactions[transaction_id]["addr"]
-            self.metadata.add_token_to_addr(addr, value)
+        if keyname[0] == "priv" and keyname[2] == metadata.DEVICETOKEN_KEY:
-            return True
+            metadata.add_token_to_addr(addr, value)
         elif keyname[0] == "priv" and keyname[2] == "messagenew":
-            self.notifier.new_message_for_addr(addr, self.metadata)
+            notifier.new_message_for_addr(addr, metadata)
-            return True
+        else:
+            # Transaction failed.
+            transactions[transaction_id]["res"] = "F\n"
 
-        return False
+
+class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
+    request_queue_size = 100
 
 
 def main():
@@ -94,8 +143,23 @@ def main():
     notifier = Notifier(queue_dir)
     notifier.start_notification_threads(metadata.remove_token_from_addr)
 
-    dictproxy = MetadataDictProxy(
+    class Handler(StreamRequestHandler):
-        notifier=notifier, metadata=metadata, iroh_relay=iroh_relay
+        def handle(self):
-    )
+            try:
+                handle_dovecot_protocol(
+                    self.rfile, self.wfile, notifier, metadata, iroh_relay
+                )
+            except Exception:
+                logging.exception("Exception in the dovecot dictproxy handler")
+                raise
 
-    dictproxy.serve_forever_from_socket(socket)
+    try:
+        os.unlink(socket)
+    except FileNotFoundError:
+        pass
+
+    with ThreadedUnixStreamServer(socket, Handler) as server:
+        try:
+            server.serve_forever()
+        except KeyboardInterrupt:
+            pass

chatmaild/src/chatmaild/migrate_db.py

@@ -1,63 +0,0 @@
-"""
-migration code from old sqlite databases into per-maildir "password" files
-where mtime reflects and is updated to be the "last-login" time.
-"""
-
-import logging
-import os
-import sqlite3
-import sys
-
-from chatmaild.config import read_config
-
-
-def get_all_rows(path):
-    assert path.exists()
-    uri = f"file:{path}?mode=ro"
-    sqlconn = sqlite3.connect(uri, timeout=60, isolation_level="DEFERRED", uri=True)
-    cur = sqlconn.cursor()
-    cur.execute("SELECT * from users")
-    rows = cur.fetchall()
-    sqlconn.close()
-    return rows
-
-
-def migrate_from_db_to_maildir(config, chunking=10000):
-    path = config.passdb_path
-    if not path.exists():
-        return
-
-    all_rows = get_all_rows(path)
-
-    # don't transfer special/CI accounts
-    rows = [row for row in all_rows if row[0][:3] not in ("ci-", "ac_")]
-
-    logging.info(f"ignoring {len(all_rows)-len(rows)} CI accounts")
-    logging.info(f"migrating {len(rows)} sqlite database passwords to user dirs")
-
-    for i, row in enumerate(rows):
-        addr = row[0]
-        enc_password = row[1]
-        user = config.get_user(addr)
-        user.set_password(enc_password)
-
-        if len(row) == 3 and row[2]:
-            timestamp = int(row[2])
-            user.set_last_login_timestamp(timestamp)
-
-        if i > 0 and i % chunking == 0:
-            logging.info(f"migration-progress: {i} passwords transferred")
-
-    logging.info("migration: all passwords migrated")
-    oldpath = config.passdb_path.with_suffix(config.passdb_path.suffix + ".old")
-    os.rename(config.passdb_path, oldpath)
-    for path in config.passdb_path.parent.iterdir():
-        if path.name.startswith(config.passdb_path.name + "-"):
-            path.unlink()
-    logging.info(f"migration: moved database to {oldpath!r}")
-
-
-if __name__ == "__main__":
-    config = read_config(sys.argv[1])
-    logging.basicConfig(level=logging.INFO)
-    migrate_from_db_to_maildir(config)

chatmaild/src/chatmaild/notifier.py

@@ -92,7 +92,7 @@ class Notifier:
     def requeue_persistent_queue_items(self):
         for queue_path in self.queue_dir.iterdir():
             if queue_path.name.endswith(".tmp"):
-                logging.warning(f"removing spurious queue item: {queue_path!r}")
+                logging.warning("removing spurious queue item: %r", queue_path)
                 queue_path.unlink()
                 continue
             queue_item = PersistentQueueItem.read_from_path(queue_path)
@@ -104,7 +104,7 @@ class Notifier:
         deadline = queue_item.start_ts + self.DROP_DEADLINE
         if retry_num >= len(self.retry_queues) or when > deadline:
             queue_item.delete()
-            logging.error(f"notification exceeded deadline: {queue_item.token!r}")
+            logging.error("notification exceeded deadline: %r", queue_item.token)
             return
 
         self.retry_queues[retry_num].put((when, queue_item))
@@ -162,5 +162,5 @@ class NotifyThread(Thread):
                 queue_item.delete()
                 return
 
-        logging.warning(f"Notification request failed: {res!r}")
+        logging.warning("Notification request failed: %r", res)
         self.notifier.queue_for_retry(queue_item, retry_num=self.retry_num + 1)

chatmaild/src/chatmaild/tests/plugin.py

@@ -7,19 +7,19 @@ from email.parser import BytesParser
 from pathlib import Path
 
 import pytest
-
 from chatmaild.config import read_config, write_initial_config
+from chatmaild.database import Database
 
 
 @pytest.fixture
 def make_config(tmp_path):
     inipath = tmp_path.joinpath("chatmail.ini")
 
-    def make_conf(mail_domain, settings=None):
+    def make_conf(mail_domain):
         basedir = tmp_path.joinpath(f"vmail/{mail_domain}")
         basedir.mkdir(parents=True, exist_ok=True)
-        overrides = settings.copy() if settings else {}
+        passdb = tmp_path.joinpath("vmail/passdb.sqlite")
-        overrides["mailboxes_dir"] = str(basedir)
+        overrides = dict(mailboxes_dir=str(basedir), passdb_path=str(passdb))
         write_initial_config(inipath, mail_domain, overrides=overrides)
         return read_config(inipath)
 
@@ -36,11 +36,6 @@ def maildomain(example_config):
     return example_config.mail_domain
 
 
-@pytest.fixture
-def testaddr(maildomain):
-    return f"user.name@{maildomain}"
-
-
 @pytest.fixture
 def gencreds(maildomain):
     count = itertools.count()
@@ -59,6 +54,13 @@ def gencreds(maildomain):
     return lambda domain=None: next(gen(domain))
 
 
+@pytest.fixture()
+def db(tmpdir):
+    db_path = tmpdir / "passdb.sqlite"
+    print("database path:", db_path)
+    return Database(db_path)
+
+
 @pytest.fixture
 def maildata(request):
     try:
@@ -77,22 +79,3 @@ def maildata(request):
         return BytesParser(policy=policy.default).parsebytes(text.encode())
 
     return maildata
-
-
-@pytest.fixture
-def mockout():
-    class MockOut:
-        captured_red = []
-        captured_green = []
-        captured_plain = []
-
-        def red(self, msg):
-            self.captured_red.append(msg)
-
-        def green(self, msg):
-            self.captured_green.append(msg)
-
-        def __call__(self, msg):
-            self.captured_plain.append(msg)
-
-    return MockOut()

chatmaild/src/chatmaild/tests/test_config.py

@@ -1,5 +1,4 @@
 import pytest
-
 from chatmaild.config import read_config
 
 
@@ -39,28 +38,24 @@ def test_config_userstate_paths(make_config, tmp_path):
     mailboxes_dir = config.mailboxes_dir
     passdb_path = config.passdb_path
     assert mailboxes_dir.name == "something.testrun.org"
-    assert str(passdb_path) == "/home/vmail/passdb.sqlite"
+    assert passdb_path.name == "passdb.sqlite"
+    assert passdb_path.is_relative_to(tmp_path)
     assert config.mail_domain == "something.testrun.org"
-    path = config.get_user("user1@something.testrun.org").maildir
+    path = config.get_user_maildir("user1@something.testrun.org")
     assert not path.exists()
     assert path == mailboxes_dir.joinpath("user1@something.testrun.org")
 
     with pytest.raises(ValueError):
-        config.get_user("")
+        config.get_user_maildir("")
 
     with pytest.raises(ValueError):
-        config.get_user(None)
+        config.get_user_maildir(None)
 
     with pytest.raises(ValueError):
-        config.get_user("../some@something.testrun.org").maildir
+        config.get_user_maildir("../some@something.testrun.org")
 
     with pytest.raises(ValueError):
-        config.get_user("..").maildir
+        config.get_user_maildir("..")
 
     with pytest.raises(ValueError):
-        config.get_user(".")
+        config.get_user_maildir(".")
-
-
-def test_config_max_message_size(make_config, tmp_path):
-    config = make_config("something.testrun.org", dict(max_message_size="10000"))
-    assert config.max_message_size == 10000

chatmaild/src/chatmaild/tests/test_delete_inactive_users.py

@@ -1,37 +1,27 @@
 import time
 
 from chatmaild.delete_inactive_users import delete_inactive_users
-from chatmaild.doveauth import AuthDictProxy
+from chatmaild.doveauth import lookup_passdb
 
 
-def test_login_timestamps(example_config):
+def test_remove_stale_users(db, example_config):
-    testaddr = "someuser@chat.example.org"
-    user = example_config.get_user(testaddr)
-
-    # password file needs to be set because it's mtime tracks last-login time
-    user.set_password("1l2k3j1l2k3j123")
-    for i in range(10):
-        user.set_last_login_timestamp(86400 * 4 + i)
-        assert user.get_last_login_timestamp() == 86400 * 4
-
-
-def test_delete_inactive_users(example_config):
     new = time.time()
     old = new - (example_config.delete_inactive_users_after * 86400) - 1
-    dictproxy = AuthDictProxy(example_config)
 
     def create_user(addr, last_login):
-        dictproxy.lookup_passdb(addr, "q9mr3faue")
+        lookup_passdb(db, example_config, addr, "q9mr3faue", last_login=last_login)
-        user = example_config.get_user(addr)
+        md = example_config.get_user_maildir(addr)
-        user.maildir.joinpath("cur").mkdir()
+        md.mkdir(parents=True)
-        user.maildir.joinpath("cur", "something").mkdir()
+        md.joinpath("cur").mkdir()
-        user.set_last_login_timestamp(timestamp=last_login)
+        md.joinpath("cur", "something").mkdir()
 
     # create some stale and some new accounts
     to_remove = []
     for i in range(150):
         addr = f"oldold{i:03}@chat.example.org"
         create_user(addr, last_login=old)
+        with db.read_connection() as conn:
+            assert conn.get_user(addr)
         to_remove.append(addr)
 
     remain = []
@@ -43,17 +33,19 @@ def test_delete_inactive_users(example_config):
     # check pre and post-conditions for delete_inactive_users()
 
     for addr in to_remove:
-        assert example_config.get_user(addr).maildir.exists()
+        assert example_config.get_user_maildir(addr).exists()
 
-    delete_inactive_users(example_config)
+    delete_inactive_users(db, example_config)
 
     for p in example_config.mailboxes_dir.iterdir():
         assert not p.name.startswith("old")
 
     for addr in to_remove:
-        assert not example_config.get_user(addr).maildir.exists()
+        assert not example_config.get_user_maildir(addr).exists()
+        with db.read_connection() as conn:
+            assert not conn.get_user(addr)
 
     for addr in remain:
-        userdir = example_config.get_user(addr).maildir
+        assert example_config.get_user_maildir(addr).exists()
-        assert userdir.exists()
+        with db.read_connection() as conn:
-        assert userdir.joinpath("password").read_text()
+            assert conn.get_user(addr)

chatmaild/src/chatmaild/tests/test_doveauth.py

@@ -4,39 +4,58 @@ import queue
 import threading
 import traceback
 
-import pytest
-
 import chatmaild.doveauth
+import pytest
+from chatmaild.database import DBError
 from chatmaild.doveauth import (
-    AuthDictProxy,
+    get_user_data,
+    handle_dovecot_protocol,
+    handle_dovecot_request,
     is_allowed_to_create,
+    iter_userdb,
+    iter_userdb_lastlogin_before,
+    lookup_passdb,
 )
 from chatmaild.newemail import create_newemail_dict
 
 
-@pytest.fixture
+def test_basic(db, example_config):
-def dictproxy(example_config):
+    lookup_passdb(db, example_config, "asdf12345@chat.example.org", "q9mr3faue")
-    return AuthDictProxy(config=example_config)
+    data = get_user_data(db, example_config, "asdf12345@chat.example.org")
-
-
-def test_basic(dictproxy, gencreds):
-    addr, password = gencreds()
-    dictproxy.lookup_passdb(addr, password)
-    data = dictproxy.lookup_userdb(addr)
     assert data
-    data2 = dictproxy.lookup_passdb(addr, password)
+    data2 = lookup_passdb(
+        db, example_config, "asdf12345@chat.example.org", "q9mr3jewvadsfaue"
+    )
     assert data == data2
 
 
-def test_iterate_addresses(dictproxy):
+def test_iterate_addresses(db, example_config):
     addresses = []
 
     for i in range(10):
         addresses.append(f"asdf1234{i}@chat.example.org")
-        dictproxy.lookup_passdb(addresses[-1], "q9mr3faue")
+        lookup_passdb(db, example_config, addresses[-1], "q9mr3faue")
+    res = iter_userdb(db)
+    assert res == addresses
 
-    res = dictproxy.iter_userdb()
+
-    assert set(res) == set(addresses)
+def test_iterate_addresses_lastlogin_before(db, example_config):
+    addresses = []
+
+    cutoff_date = 1000
+    for i in range(10):
+        addr = f"oldold{i:03}@chat.example.org"
+        lookup_passdb(
+            db, example_config, addr, "q9mr3faue", last_login=cutoff_date - 10
+        )
+        addresses.append(addr)
+
+    for i in range(5):
+        addr = f"newnew{i:03}@chat.example.org"
+        lookup_passdb(db, example_config, addr, "q9mr3faue", last_login=cutoff_date + i)
+
+    res = iter_userdb_lastlogin_before(db, cutoff_date)
+    assert sorted(res) == sorted(addresses)
 
 
 def test_invalid_username_length(example_config):
@@ -53,33 +72,45 @@ def test_invalid_username_length(example_config):
     )
 
 
-def test_dont_overwrite_password_on_wrong_login(dictproxy):
+def test_dont_overwrite_password_on_wrong_login(db, example_config):
     """Test that logging in with a different password doesn't create a new user"""
-    res = dictproxy.lookup_passdb(
+    res = lookup_passdb(
-        "newuser12@chat.example.org", "kajdlkajsldk12l3kj1983"
+        db, example_config, "newuser12@chat.example.org", "kajdlkajsldk12l3kj1983"
     )
     assert res["password"]
-    res2 = dictproxy.lookup_passdb("newuser12@chat.example.org", "kajdslqwe")
+    res2 = lookup_passdb(db, example_config, "newuser12@chat.example.org", "kajdslqwe")
     # this function always returns a password hash, which is actually compared by dovecot.
     assert res["password"] == res2["password"]
 
 
-def test_nocreate_file(monkeypatch, tmpdir, dictproxy):
+def test_nocreate_file(db, monkeypatch, tmpdir, example_config):
     p = tmpdir.join("nocreate")
     p.write("")
     monkeypatch.setattr(chatmaild.doveauth, "NOCREATE_FILE", str(p))
-    dictproxy.lookup_passdb("newuser12@chat.example.org", "zequ0Aimuchoodaechik")
+    lookup_passdb(
-    assert not dictproxy.lookup_userdb("newuser12@chat.example.org")
+        db, example_config, "newuser12@chat.example.org", "zequ0Aimuchoodaechik"
+    )
+    assert not get_user_data(db, example_config, "newuser12@chat.example.org")
 
 
-def test_handle_dovecot_request(dictproxy):
+def test_db_version(db):
-    transactions = {}
+    assert db.get_schema_version() == 1
+
+
+def test_too_high_db_version(db):
+    with db.write_transaction() as conn:
+        conn.execute("PRAGMA user_version=%s;" % (999,))
+    with pytest.raises(DBError):
+        db.ensure_tables()
+
+
+def test_handle_dovecot_request(db, example_config):
     # Test that password can contain ", ', \ and /
     msg = (
         'Lshared/passdb/laksjdlaksjdlak\\\\sjdlk\\"12j\\\'3l1/k2j3123"'
         "some42123@chat.example.org\tsome42123@chat.example.org"
     )
-    res = dictproxy.handle_dovecot_request(msg, transactions)
+    res = handle_dovecot_request(msg, db, example_config)
     assert res
     assert res[0] == "O" and res.endswith("\n")
     userdata = json.loads(res[1:].strip())
@@ -88,48 +119,45 @@ def test_handle_dovecot_request(dictproxy):
     assert userdata["password"].startswith("{SHA512-CRYPT}")
 
 
-def test_handle_dovecot_protocol_hello_is_skipped(example_config, caplog):
+def test_handle_dovecot_protocol_hello_is_skipped(db, example_config, caplog):
-    dictproxy = AuthDictProxy(config=example_config)
     rfile = io.BytesIO(b"H3\t2\t0\t\tauth\n")
     wfile = io.BytesIO()
-    dictproxy.loop_forever(rfile, wfile)
+    handle_dovecot_protocol(rfile, wfile, db, example_config)
     assert wfile.getvalue() == b""
     assert not caplog.messages
 
 
-def test_handle_dovecot_protocol_user_not_exists(example_config):
+def test_handle_dovecot_protocol(db, example_config):
-    dictproxy = AuthDictProxy(config=example_config)
     rfile = io.BytesIO(
         b"H3\t2\t0\t\tauth\nLshared/userdb/foobar@chat.example.org\tfoobar@chat.example.org\n"
     )
     wfile = io.BytesIO()
-    dictproxy.loop_forever(rfile, wfile)
+    handle_dovecot_protocol(rfile, wfile, db, example_config)
     assert wfile.getvalue() == b"N\n"
 
 
-def test_handle_dovecot_protocol_iterate(gencreds, example_config):
+def test_handle_dovecot_protocol_iterate(db, gencreds, example_config):
-    dictproxy = AuthDictProxy(config=example_config)
+    lookup_passdb(db, example_config, "asdf00000@chat.example.org", "q9mr3faue")
-    dictproxy.lookup_passdb("asdf00000@chat.example.org", "q9mr3faue")
+    lookup_passdb(db, example_config, "asdf11111@chat.example.org", "q9mr3faue")
-    dictproxy.lookup_passdb("asdf11111@chat.example.org", "q9mr3faue")
     rfile = io.BytesIO(b"H3\t2\t0\t\tauth\nI0\t0\tshared/userdb/")
     wfile = io.BytesIO()
-    dictproxy.loop_forever(rfile, wfile)
+    handle_dovecot_protocol(rfile, wfile, db, example_config)
     lines = wfile.getvalue().decode("ascii").split("\n")
-    assert "Oshared/userdb/asdf00000@chat.example.org\t" in lines
+    assert lines[0] == "Oshared/userdb/asdf00000@chat.example.org\t"
-    assert "Oshared/userdb/asdf11111@chat.example.org\t" in lines
+    assert lines[1] == "Oshared/userdb/asdf11111@chat.example.org\t"
     assert not lines[2]
 
 
-def test_50_concurrent_lookups_different_accounts(gencreds, dictproxy):
+def test_50_concurrent_lookups_different_accounts(db, gencreds, example_config):
     num_threads = 50
     req_per_thread = 5
     results = queue.Queue()
 
-    def lookup():
+    def lookup(db):
         for i in range(req_per_thread):
             addr, password = gencreds()
             try:
-                dictproxy.lookup_passdb(addr, password)
+                lookup_passdb(db, example_config, addr, password)
             except Exception:
                 results.put(traceback.format_exc())
             else:
@@ -137,7 +165,7 @@ def test_50_concurrent_lookups_different_accounts(gencreds, dictproxy):
 
     threads = []
     for i in range(num_threads):
-        thread = threading.Thread(target=lookup, daemon=True)
+        thread = threading.Thread(target=lookup, args=(db,), daemon=True)
         threads.append(thread)
 
     print(f"created {num_threads} threads, starting them and waiting for results")

chatmaild/src/chatmaild/tests/test_filedict.py

@@ -1,6 +1,4 @@
-import threading
+from chatmaild.filedict import FileDict
-
-from chatmaild.filedict import FileDict, write_bytes_atomic
 
 
 def test_basic(tmp_path):
@@ -19,21 +17,3 @@ def test_bad_marshal_file(tmp_path, caplog):
     fdict1.path.write_bytes(b"l12k3l12k3l")
     assert fdict1.read() == {}
     assert "corrupt" in caplog.records[0].msg
-
-
-def test_write_bytes_atomic_concurrent(tmp_path):
-    p = tmp_path.joinpath("somefile.ext")
-    write_bytes_atomic(p, b"hello")
-
-    threads = []
-    for i in range(30):
-        content = f"hello{i}".encode("ascii")
-        t = threading.Thread(target=lambda: write_bytes_atomic(p, content))
-        t.start()
-        threads.append(t)
-
-    for t in threads:
-        t.join()
-
-    assert p.read_text().strip() != "hello"
-    assert len(list(p.parent.iterdir())) == 1

chatmaild/src/chatmaild/tests/test_filtermail.py

@@ -1,11 +1,9 @@
 import pytest
-
 from chatmaild.filtermail import (
     BeforeQueueHandler,
     SendRateLimiter,
     check_armored_payload,
     check_encrypted,
-    common_encrypted_subjects,
 )
 
 
@@ -56,7 +54,7 @@ def test_filtermail_no_encryption_detection(maildata):
 
 
 def test_filtermail_encryption_detection(maildata):
-    for subject in common_encrypted_subjects:
+    for subject in ("...", "[...]"):
         msg = maildata(
             "encrypted.eml",
             from_addr="1@example.org",

chatmaild/src/chatmaild/tests/test_lastlogin.py

@@ -1,64 +0,0 @@
-import time
-
-from chatmaild.doveauth import AuthDictProxy
-from chatmaild.lastlogin import (
-    LastLoginDictProxy,
-)
-
-
-def test_handle_dovecot_request_last_login(testaddr, example_config):
-    dictproxy = LastLoginDictProxy(config=example_config)
-
-    authproxy = AuthDictProxy(config=example_config)
-    authproxy.lookup_passdb(testaddr, "1l2k3j1l2k3jl123")
-
-    dictproxy_transactions = {}
-
-    # Begin transaction
-    tx = "1111"
-    msg = f"B{tx}\t{testaddr}"
-    res = dictproxy.handle_dovecot_request(msg, dictproxy_transactions)
-    assert not res
-    assert dictproxy_transactions == {tx: dict(addr=testaddr, res="O\n")}
-
-    # set last-login info for user
-    user = dictproxy.config.get_user(testaddr)
-    timestamp = int(time.time())
-    msg = f"S{tx}\tshared/last-login/{testaddr}\t{timestamp}"
-    res = dictproxy.handle_dovecot_request(msg, dictproxy_transactions)
-    assert not res
-    assert len(dictproxy_transactions) == 1
-    read_timestamp = user.get_last_login_timestamp()
-    assert read_timestamp == timestamp // 86400 * 86400
-
-    # finish transaction
-    msg = f"C{tx}"
-    res = dictproxy.handle_dovecot_request(msg, dictproxy_transactions)
-    assert res == "O\n"
-    assert len(dictproxy_transactions) == 0
-
-
-def test_handle_dovecot_request_last_login_echobot(example_config):
-    dictproxy = LastLoginDictProxy(config=example_config)
-
-    authproxy = AuthDictProxy(config=example_config)
-    testaddr = f"echo@{example_config.mail_domain}"
-    authproxy.lookup_passdb(testaddr, "ignore")
-    user = dictproxy.config.get_user(testaddr)
-
-    transactions = {}
-
-    # set last-login info for user
-    tx = "1111"
-    msg = f"B{tx}\t{testaddr}"
-    res = dictproxy.handle_dovecot_request(msg, transactions)
-    assert not res
-    assert transactions == {tx: dict(addr=testaddr, res="O\n")}
-
-    timestamp = int(time.time())
-    msg = f"S{tx}\tshared/last-login/{testaddr}\t{timestamp}"
-    res = dictproxy.handle_dovecot_request(msg, transactions)
-    assert not res
-    assert len(transactions) == 1
-    read_timestamp = user.get_last_login_timestamp()
-    assert read_timestamp is None

chatmaild/src/chatmaild/tests/test_metadata.py

@@ -3,10 +3,10 @@ import time
 
 import pytest
 import requests
-
 from chatmaild.metadata import (
     Metadata,
-    MetadataDictProxy,
+    handle_dovecot_protocol,
+    handle_dovecot_request,
 )
 from chatmaild.notifier import (
     Notifier,
@@ -30,8 +30,8 @@ def metadata(tmp_path):
 
 
 @pytest.fixture
-def dictproxy(notifier, metadata):
+def testaddr():
-    return MetadataDictProxy(notifier=notifier, metadata=metadata)
+    return "user.name@example.org"
 
 
 @pytest.fixture
@@ -88,51 +88,51 @@ def test_notifier_remove_without_set(metadata, testaddr):
     assert not metadata.get_tokens_for_addr(testaddr)
 
 
-def test_handle_dovecot_request_lookup_fails(dictproxy, testaddr):
+def test_handle_dovecot_request_lookup_fails(notifier, metadata, testaddr):
-    transactions = {}
+    res = handle_dovecot_request(
-    res = dictproxy.handle_dovecot_request(
+        f"Lpriv/123/chatmail\t{testaddr}", {}, notifier, metadata
-        f"Lpriv/123/chatmail\t{testaddr}", transactions
     )
     assert res == "N\n"
 
 
-def test_handle_dovecot_request_happy_path(dictproxy, testaddr, token):
+def test_handle_dovecot_request_happy_path(notifier, metadata, testaddr, token):
-    metadata = dictproxy.metadata
     transactions = {}
-    notifier = dictproxy.notifier
 
     # set device token in a transaction
     tx = "1111"
     msg = f"B{tx}\t{testaddr}"
-    res = dictproxy.handle_dovecot_request(msg, transactions)
+    res = handle_dovecot_request(msg, transactions, notifier, metadata)
     assert not res and not metadata.get_tokens_for_addr(testaddr)
     assert transactions == {tx: dict(addr=testaddr, res="O\n")}
 
     msg = f"S{tx}\tpriv/guid00/devicetoken\t{token}"
-    res = dictproxy.handle_dovecot_request(msg, transactions)
+    res = handle_dovecot_request(msg, transactions, notifier, metadata)
     assert not res
     assert len(transactions) == 1
     assert metadata.get_tokens_for_addr(testaddr) == [token]
 
     msg = f"C{tx}"
-    res = dictproxy.handle_dovecot_request(msg, transactions)
+    res = handle_dovecot_request(msg, transactions, notifier, metadata)
     assert res == "O\n"
     assert len(transactions) == 0
     assert metadata.get_tokens_for_addr(testaddr) == [token]
 
     # trigger notification for incoming message
     tx2 = "2222"
-    assert dictproxy.handle_dovecot_request(f"B{tx2}\t{testaddr}", transactions) is None
+    assert (
+        handle_dovecot_request(f"B{tx2}\t{testaddr}", transactions, notifier, metadata)
+        is None
+    )
     msg = f"S{tx2}\tpriv/guid00/messagenew"
-    assert dictproxy.handle_dovecot_request(msg, transactions) is None
+    assert handle_dovecot_request(msg, transactions, notifier, metadata) is None
     queue_item = notifier.retry_queues[0].get()[1]
     assert queue_item.token == token
-    assert dictproxy.handle_dovecot_request(f"C{tx2}", transactions) == "O\n"
+    assert handle_dovecot_request(f"C{tx2}", transactions, notifier, metadata) == "O\n"
     assert not transactions
     assert queue_item.path.exists()
 
 
-def test_handle_dovecot_protocol_set_devicetoken(dictproxy):
+def test_handle_dovecot_protocol_set_devicetoken(metadata, notifier):
     rfile = io.BytesIO(
         b"\n".join(
             [
@@ -144,12 +144,12 @@ def test_handle_dovecot_protocol_set_devicetoken(dictproxy):
         )
     )
     wfile = io.BytesIO()
-    dictproxy.loop_forever(rfile, wfile)
+    handle_dovecot_protocol(rfile, wfile, notifier, metadata)
     assert wfile.getvalue() == b"O\n"
-    assert dictproxy.metadata.get_tokens_for_addr("user@example.org") == ["01234"]
+    assert metadata.get_tokens_for_addr("user@example.org") == ["01234"]
 
 
-def test_handle_dovecot_protocol_set_get_devicetoken(dictproxy):
+def test_handle_dovecot_protocol_set_get_devicetoken(metadata, notifier):
     rfile = io.BytesIO(
         b"\n".join(
             [
@@ -161,19 +161,19 @@ def test_handle_dovecot_protocol_set_get_devicetoken(dictproxy):
         )
     )
     wfile = io.BytesIO()
-    dictproxy.loop_forever(rfile, wfile)
+    handle_dovecot_protocol(rfile, wfile, notifier, metadata)
-    assert dictproxy.metadata.get_tokens_for_addr("user@example.org") == ["01234"]
+    assert metadata.get_tokens_for_addr("user@example.org") == ["01234"]
     assert wfile.getvalue() == b"O\n"
 
     rfile = io.BytesIO(
         b"\n".join([b"HELLO", b"Lpriv/0123/devicetoken\tuser@example.org"])
     )
     wfile = io.BytesIO()
-    dictproxy.loop_forever(rfile, wfile)
+    handle_dovecot_protocol(rfile, wfile, notifier, metadata)
     assert wfile.getvalue() == b"O01234\n"
 
 
-def test_handle_dovecot_protocol_iterate(dictproxy):
+def test_handle_dovecot_protocol_iterate(metadata, notifier):
     rfile = io.BytesIO(
         b"\n".join(
             [
@@ -183,7 +183,7 @@ def test_handle_dovecot_protocol_iterate(dictproxy):
         )
     )
     wfile = io.BytesIO()
-    dictproxy.loop_forever(rfile, wfile)
+    handle_dovecot_protocol(rfile, wfile, notifier, metadata)
     assert wfile.getvalue() == b"\n"
 
 
@@ -298,7 +298,7 @@ def test_persistent_queue_items(tmp_path, testaddr, token):
     assert not queue_item < item2 and not item2 < queue_item
 
 
-def test_iroh_relay(dictproxy):
+def test_iroh_relay(metadata):
     rfile = io.BytesIO(
         b"\n".join(
             [
@@ -308,6 +308,5 @@ def test_iroh_relay(dictproxy):
         )
     )
     wfile = io.BytesIO()
-    dictproxy.iroh_relay = "https://example.org/"
+    handle_dovecot_protocol(rfile, wfile, notifier, metadata, "https://example.org/")
-    dictproxy.loop_forever(rfile, wfile)
     assert wfile.getvalue() == b"Ohttps://example.org/\n"

chatmaild/src/chatmaild/tests/test_migrate_db.py

@@ -1,67 +0,0 @@
-import sqlite3
-
-from chatmaild.migrate_db import migrate_from_db_to_maildir
-
-
-def test_migration_not_exists(tmp_path, example_config):
-    example_config.passdb_path = tmp_path.joinpath("sqlite")
-
-
-def test_migration(tmp_path, example_config, caplog):
-    passdb_path = tmp_path.joinpath("passdb.sqlite")
-    uri = f"file:{passdb_path}?mode=rwc"
-    sqlconn = sqlite3.connect(uri, timeout=60, uri=True)
-    sqlconn.execute(
-        """
-        CREATE TABLE users (
-            addr TEXT PRIMARY KEY,
-            password TEXT,
-            last_login INTEGER
-        )
-    """
-    )
-    all = {}
-
-    for i in range(500):
-        values = (f"somsom{i:03}@example.org", f"passwo{i:03}", i * 86400)
-        sqlconn.execute(
-            """
-            INSERT INTO users (addr, password, last_login)
-            VALUES (?, ?, ?)""",
-            values,
-        )
-        all[values[0]] = values[1:]
-
-    for i in range(500):
-        values = (f"pompom{i:03}@example.org", f"wopass{i:03}", "")
-        sqlconn.execute(
-            """
-            INSERT INTO users (addr, password, last_login)
-            VALUES (?, ?, ?)""",
-            values,
-        )
-        all[values[0]] = values[1:]
-
-    sqlconn.commit()
-    sqlconn.close()
-
-    assert passdb_path.stat().st_size > 10000
-
-    example_config.passdb_path = passdb_path
-
-    assert not caplog.records
-
-    migrate_from_db_to_maildir(example_config, chunking=500)
-    assert len(caplog.records) > 3
-
-    for path in example_config.mailboxes_dir.iterdir():
-        if "@" not in path.name:
-            continue
-        password, last_login = all.pop(path.name)
-        user = example_config.get_user(path.name)
-        if last_login:
-            assert user.get_last_login_timestamp() == last_login
-        assert password == user.get_userdb_dict()["password"]
-
-    assert not all
-    assert not example_config.passdb_path.exists()

chatmaild/src/chatmaild/tests/test_user.py

@@ -1,42 +0,0 @@
-def test_login_timestamp(testaddr, example_config):
-    user = example_config.get_user(testaddr)
-    user.set_password("someeqkjwelkqwjleqwe")
-    user.set_last_login_timestamp(100000)
-    assert user.get_last_login_timestamp() == 86400
-
-    user.set_last_login_timestamp(200000)
-    assert user.get_last_login_timestamp() == 86400 * 2
-
-
-def test_get_user_dict_not_set(testaddr, example_config, caplog):
-    user = example_config.get_user(testaddr)
-    assert not caplog.records
-    assert user.get_userdb_dict() == {}
-    assert len(caplog.records) == 0
-
-    user.set_password("")
-    assert user.get_userdb_dict() == {}
-    assert len(caplog.records) == 1
-
-
-def test_get_user_dict(make_config, tmp_path):
-    config = make_config("something.testrun.org")
-    addr = "user1@something.org"
-    user = config.get_user(addr)
-    enc_password = "l1k2j31lk2j3l1k23j123"
-    user.set_password(enc_password)
-    data = user.get_userdb_dict()
-    assert addr in str(data["home"])
-    assert data["uid"] == "vmail"
-    assert data["gid"] == "vmail"
-    assert data["password"] == enc_password
-
-
-def test_no_mailboxes_dir(testaddr, example_config, tmp_path):
-    p = tmp_path.joinpath("a", "mailboxes")
-    example_config.mailboxes_dir = p
-
-    user = example_config.get_user(testaddr)
-    user.set_password("someeqkjwelkqwjleqwe")
-    user.set_last_login_timestamp(100000)
-    assert user.get_last_login_timestamp() == 86400

chatmaild/src/chatmaild/user.py

@@ -1,74 +0,0 @@
-import logging
-import os
-
-from chatmaild.filedict import write_bytes_atomic
-
-
-def get_daytimestamp(timestamp) -> int:
-    return int(timestamp) // 86400 * 86400
-
-
-class User:
-    def __init__(self, maildir, addr, password_path, uid, gid):
-        self.maildir = maildir
-        self.addr = addr
-        self.password_path = password_path
-        self.uid = uid
-        self.gid = gid
-
-    @property
-    def can_track(self):
-        return "@" in self.addr and not self.addr.startswith("echo@")
-
-    def get_userdb_dict(self):
-        """Return a non-empty dovecot 'userdb' style dict
-        if the user has an existing non-empty password"""
-        try:
-            pw = self.password_path.read_text()
-        except FileNotFoundError:
-            return {}
-
-        if not pw:
-            logging.error(f"password is empty for: {self.addr}")
-            return {}
-
-        home = str(self.maildir)
-        return dict(addr=self.addr, home=home, uid=self.uid, gid=self.gid, password=pw)
-
-    def set_password(self, enc_password):
-        """Set the specified password for this user.
-
-        This method can be called concurrently
-        but there is no guarantee which of the password-set calls will win.
-        """
-        self.maildir.mkdir(exist_ok=True, parents=True)
-        password = enc_password.encode("ascii")
-
-        try:
-            write_bytes_atomic(self.password_path, password)
-        except PermissionError:
-            if not self.addr.startswith("echo@"):
-                logging.error(f"could not write password for: {self.addr}")
-                raise
-
-    def set_last_login_timestamp(self, timestamp):
-        """Track login time with daily granularity
-        to minimize touching files and to minimize metadata leakage."""
-        if not self.can_track:
-            return
-        try:
-            mtime = int(os.stat(self.password_path).st_mtime)
-        except FileNotFoundError:
-            logging.error(f"Can not get last login timestamp for {self.addr}")
-            return
-
-        timestamp = get_daytimestamp(timestamp)
-        if mtime != timestamp:
-            os.utime(self.password_path, (timestamp, timestamp))
-
-    def get_last_login_timestamp(self):
-        if self.can_track:
-            try:
-                return int(self.password_path.stat().st_mtime)
-            except FileNotFoundError:
-                pass

cmdeploy/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 name = "cmdeploy"
 version = "0.2"
 dependencies = [
-  "pyinfra>=3",
+  "pyinfra",
   "pillow",
   "qrcode",
   "markdown",

cmdeploy/src/cmdeploy/__init__.py

@@ -103,7 +103,6 @@ def _install_remote_venv_with_chatmaild(config) -> None:
         "filtermail",
         "echobot",
         "chatmail-metadata",
-        "lastlogin",
     ):
         params = dict(
             execpath=f"{remote_venv_dir}/bin/{fn}",
@@ -268,7 +267,6 @@ def _configure_postfix(config: Config, debug: bool = False) -> bool:
         group="root",
         mode="644",
         config=config,
-        disable_ipv6=config.disable_ipv6,
     )
     need_restart |= main_config.changed
 
@@ -319,7 +317,6 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
         mode="644",
         config=config,
         debug=debug,
-        disable_ipv6=config.disable_ipv6,
     )
     need_restart |= main_config.changed
     auth_config = files.put(
@@ -364,7 +361,7 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
     return need_restart
 
 
-def _configure_nginx(config: Config, debug: bool = False) -> bool:
+def _configure_nginx(domain: str, debug: bool = False) -> bool:
     """Configures nginx HTTP server."""
     need_restart = False
 
@@ -374,8 +371,7 @@ def _configure_nginx(config: Config, debug: bool = False) -> bool:
         user="root",
         group="root",
         mode="644",
-        config={"domain_name": config.mail_domain},
+        config={"domain_name": domain},
-        disable_ipv6=config.disable_ipv6,
     )
     need_restart |= main_config.changed
 
@@ -385,7 +381,7 @@ def _configure_nginx(config: Config, debug: bool = False) -> bool:
         user="root",
         group="root",
         mode="644",
-        config={"domain_name": config.mail_domain},
+        config={"domain_name": domain},
     )
     need_restart |= autoconfig.changed
 
@@ -395,7 +391,7 @@ def _configure_nginx(config: Config, debug: bool = False) -> bool:
         user="root",
         group="root",
         mode="644",
-        config={"domain_name": config.mail_domain},
+        config={"domain_name": domain},
     )
     need_restart |= mta_sts_config.changed
 
@@ -454,7 +450,6 @@ def deploy_chatmail(config_path: Path) -> None:
 
     server.group(name="Create vmail group", group="vmail", system=True)
     server.user(name="Create vmail user", user="vmail", group="vmail", system=True)
-    server.user(name="Create filtermail user", user="filtermail", system=True)
     server.group(name="Create opendkim group", group="opendkim", system=True)
     server.user(
         name="Create opendkim user",
@@ -489,7 +484,6 @@ def deploy_chatmail(config_path: Path) -> None:
     )
 
     apt.update(name="apt update", cache_time=24 * 3600)
-    apt.upgrade(name="upgrade apt packages", auto_remove=True)
 
     apt.packages(
         name="Install rsync",
@@ -560,7 +554,7 @@ def deploy_chatmail(config_path: Path) -> None:
     dovecot_need_restart = _configure_dovecot(config, debug=debug)
     postfix_need_restart = _configure_postfix(config, debug=debug)
     mta_sts_need_restart = _install_mta_sts_daemon()
-    nginx_need_restart = _configure_nginx(config)
+    nginx_need_restart = _configure_nginx(mail_domain)
 
     _remove_rspamd()
     opendkim_need_restart = _configure_opendkim(mail_domain, "opendkim")

cmdeploy/src/cmdeploy/chatmail.zone.j2

@@ -1,30 +1,21 @@
-;
+{% if ipv4 %}
-; Required DNS entries for chatmail servers 
+{{ chatmail_domain }}.                   A     {{ ipv4 }}
-;
-{% if A %}
-{{ mail_domain }}.                   A     {{ A }}
 {% endif %}
-{% if AAAA %}
+{% if ipv6 %}
-{{ mail_domain }}.                   AAAA  {{ AAAA }}
+{{ chatmail_domain }}.                   AAAA  {{ ipv6 }}
 {% endif %}
-{{ mail_domain }}.                   MX 10 {{ mail_domain }}.
+{{ chatmail_domain }}.                   MX 10 {{ chatmail_domain }}.
-_mta-sts.{{ mail_domain }}.          TXT "v=STSv1; id={{ sts_id }}"
+_submission._tcp.{{ chatmail_domain }}.  SRV 0 1 587 {{ chatmail_domain }}.
-mta-sts.{{ mail_domain }}.           CNAME {{ mail_domain }}.
+_submissions._tcp.{{ chatmail_domain }}. SRV 0 1 465 {{ chatmail_domain }}.
-www.{{ mail_domain }}.               CNAME {{ mail_domain }}.
+_imap._tcp.{{ chatmail_domain }}.        SRV 0 1 143 {{ chatmail_domain }}.
-{{ dkim_entry }}
+_imaps._tcp.{{ chatmail_domain }}.       SRV 0 1 993 {{ chatmail_domain }}.
-
-;
-; Recommended DNS entries for interoperability and security-hardening
-;
-{{ mail_domain }}.                   TXT "v=spf1 a:{{ mail_domain }} ~all"
-_dmarc.{{ mail_domain }}.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
-
 {% if acme_account_url %}
-{{ mail_domain }}.                   CAA 0 issue "letsencrypt.org;accounturi={{ acme_account_url }}"
+{{ chatmail_domain }}.                   CAA 128 issue "letsencrypt.org;accounturi={{ acme_account_url }}"
 {% endif %}
-_adsp._domainkey.{{ mail_domain }}.  TXT "dkim=discardable"
+{{ chatmail_domain }}.                   TXT "v=spf1 a:{{ chatmail_domain }} ~all"
-
+_dmarc.{{ chatmail_domain }}.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
-_submission._tcp.{{ mail_domain }}.  SRV 0 1 587 {{ mail_domain }}.
+_mta-sts.{{ chatmail_domain }}.          TXT "v=STSv1; id={{ sts_id }}"
-_submissions._tcp.{{ mail_domain }}. SRV 0 1 465 {{ mail_domain }}.
+mta-sts.{{ chatmail_domain }}.           CNAME {{ chatmail_domain }}.
-_imap._tcp.{{ mail_domain }}.        SRV 0 1 143 {{ mail_domain }}.
+www.{{ chatmail_domain }}.               CNAME {{ chatmail_domain }}.
-_imaps._tcp.{{ mail_domain }}.       SRV 0 1 993 {{ mail_domain }}.
+{{ dkim_entry }}
+_adsp._domainkey.{{ chatmail_domain }}.  TXT "dkim=discardable"

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -7,18 +7,15 @@ import argparse
 import importlib.resources
 import importlib.util
 import os
-import pathlib
 import shutil
 import subprocess
 import sys
 from pathlib import Path
 
-import pyinfra
 from chatmaild.config import read_config, write_initial_config
-from packaging import version
 from termcolor import colored
 
-from . import dns, remote
+from . import dns, remote_funcs
 from .sshexec import SSHExec
 
 #
@@ -57,8 +54,7 @@ def run_cmd_options(parser):
 def run_cmd(args, out):
     """Deploy chatmail services on the remote server."""
 
-    sshexec = args.get_sshexec()
+    remote_data = dns.get_initial_remote_data(args, out)
-    remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
     if not dns.check_initial_remote_data(remote_data, print=out.red):
         return 1
 
@@ -66,10 +62,7 @@ def run_cmd(args, out):
     env["CHATMAIL_INI"] = args.inipath
     deploy_path = importlib.resources.files(__package__).joinpath("deploy.py").resolve()
     pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
-    cmd = f"{pyinf} --ssh-user root {args.config.mail_domain} {deploy_path} -y"
+    cmd = f"{pyinf} --ssh-user root {args.config.mail_domain} {deploy_path}"
-    if version.parse(pyinfra.__version__) < version.parse("3"):
-        out.red("Please re-run scripts/initenv.sh to update pyinfra to version 3.")
-        return 1
 
     retcode = out.check_call(cmd, env=env)
     if retcode == 0:
@@ -87,37 +80,16 @@ def dns_cmd_options(parser):
     parser.add_argument(
         "--zonefile",
         dest="zonefile",
-        type=pathlib.Path,
+        help="print the whole zonefile for deploying directly",
-        default=None,
-        help="write out a zonefile",
     )
 
 
 def dns_cmd(args, out):
     """Check DNS entries and optionally generate dns zone file."""
-    sshexec = args.get_sshexec()
+    remote_data = dns.get_initial_remote_data(args, out)
-    remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
     if not remote_data:
         return 1
-
+    retcode = dns.show_dns(args, out, remote_data)
-    if not remote_data["acme_account_url"]:
-        out.red("could not get letsencrypt account url, please run 'cmdeploy run'")
-        return 1
-
-    if not remote_data["dkim_entry"]:
-        out.red("could not determine dkim_entry, please run 'cmdeploy run'")
-        return 1
-
-    zonefile = dns.get_filled_zone_file(remote_data)
-
-    if args.zonefile:
-        args.zonefile.write_text(zonefile)
-        out.green(f"DNS records successfully written to: {args.zonefile}")
-        return 0
-
-    retcode = dns.check_full_zone(
-        sshexec, remote_data=remote_data, zonefile=zonefile, out=out
-    )
     return retcode
 
 
@@ -132,7 +104,7 @@ def status_cmd(args, out):
     else:
         out.red("no privacy settings")
 
-    for line in sshexec(remote.rshell.get_systemd_running):
+    for line in sshexec(remote_funcs.get_systemd_running):
         print(line)
 
 
@@ -311,9 +283,14 @@ def main(args=None):
     if not hasattr(args, "func"):
         return parser.parse_args(["-h"])
 
+    ssh_cache = []
+
     def get_sshexec():
-        print(f"[ssh] login to {args.config.mail_domain}")
+        if not ssh_cache:
-        return SSHExec(args.config.mail_domain, verbose=args.verbose)
+            print(f"[ssh] login to {args.config.mail_domain}")
+            ssh = SSHExec(args.config.mail_domain, remote_funcs, verbose=args.verbose)
+            ssh_cache.append(ssh)
+        return ssh_cache[0]
 
     args.get_sshexec = get_sshexec
 

cmdeploy/src/cmdeploy/dns.py

@@ -3,62 +3,75 @@ import importlib
 
 from jinja2 import Template
 
-from . import remote
+from . import remote_funcs
 
 
-def get_initial_remote_data(sshexec, mail_domain):
+def get_initial_remote_data(args, out):
+    sshexec = args.get_sshexec()
+    mail_domain = args.config.mail_domain
     return sshexec.logged(
-        call=remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=mail_domain)
+        call=remote_funcs.perform_initial_checks, kwargs=dict(mail_domain=mail_domain)
     )
 
 
 def check_initial_remote_data(remote_data, print=print):
     mail_domain = remote_data["mail_domain"]
     if not remote_data["A"] and not remote_data["AAAA"]:
-        print(f"Missing A and/or AAAA DNS records for {mail_domain}!")
+        print("Missing A and/or AAAA DNS records for {mail_domain}!")
-    elif remote_data["MTA_STS"] != f"{mail_domain}.":
+    elif not remote_data["MTA_STS"]:
         print("Missing MTA-STS CNAME record:")
-        print(f"mta-sts.{mail_domain}.   CNAME  {mail_domain}.")
+        print(f"mta-sts.{mail_domain}.   CNAME  {mail_domain}")
-    elif remote_data["WWW"] != f"{mail_domain}.":
-        print("Missing www CNAME record:")
-        print(f"www.{mail_domain}.   CNAME  {mail_domain}.")
     else:
         return remote_data
 
 
-def get_filled_zone_file(remote_data):
+def show_dns(args, out, remote_data) -> int:
+    """Check existing DNS records, optionally write them to zone file
+    and return (exitcode, remote_data) tuple."""
+
+    sshexec = args.get_sshexec()
+
+    if not remote_data["acme_account_url"]:
+        out.red("could not get letsencrypt account url, please run 'cmdeploy run'")
+        return 1
+
+    if not remote_data["dkim_entry"]:
+        out.red("could not determine dkim_entry, please run 'cmdeploy run'")
+        return 1
+
     sts_id = remote_data.get("sts_id")
     if not sts_id:
         sts_id = datetime.datetime.now().strftime("%Y%m%d%H%M")
 
     template = importlib.resources.files(__package__).joinpath("chatmail.zone.j2")
     content = template.read_text()
-    zonefile = Template(content).render(**remote_data)
+    zonefile = Template(content).render(
+        acme_account_url=remote_data.get("acme_account_url"),
+        dkim_entry=remote_data["dkim_entry"],
+        ipv4=remote_data["A"],
+        ipv6=remote_data["AAAA"],
+        sts_id=sts_id,
+        chatmail_domain=args.config.mail_domain,
+    )
     lines = [x.strip() for x in zonefile.split("\n") if x.strip()]
     lines.append("")
     zonefile = "\n".join(lines)
-    return zonefile
 
-
+    diff_records = sshexec.logged(
-def check_full_zone(sshexec, remote_data, out, zonefile) -> int:
+        remote_funcs.check_zonefile, kwargs=dict(zonefile=zonefile)
-    """Check existing DNS records, optionally write them to zone file
-    and return (exitcode, remote_data) tuple."""
-
-    required_diff, recommended_diff = sshexec.logged(
-        remote.rdns.check_zonefile,
-        kwargs=dict(zonefile=zonefile, mail_domain=remote_data["mail_domain"]),
     )
 
-    if required_diff:
+    if getattr(args, "zonefile", None):
-        out.red("Please set required DNS entries at your DNS provider:\n")
+        with open(args.zonefile, "w+") as zf:
-        for line in required_diff:
+            zf.write(zonefile)
-            out(line)
+        out.green(f"DNS records successfully written to: {args.zonefile}")
-        return 1
-    elif recommended_diff:
-        out("WARNING: these recommended DNS entries are not set:\n")
-        for line in recommended_diff:
-            out(line)
         return 0
 
-    out.green("Great! All your DNS entries are verified and correct.")
+    if diff_records:
-    return 0
+        out.red("Please set the following DNS entries at your DNS provider:\n")
+        for line in diff_records:
+            out(line)
+        return 1
+    else:
+        out.green("Great! All your DNS entries are verified and correct.")
+        return 0

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -1,9 +1,5 @@
 ## Dovecot configuration file
 
-{% if disable_ipv6 %}
-listen = *
-{% endif %}
-
 protocols = imap lmtp
 
 auth_mechanisms = plain
@@ -51,7 +47,10 @@ mail_server_comment = Chatmail server
 # <https://doc.dovecot.org/configuration_manual/quota_plugin/>
 mail_plugins = zlib quota
 
-imap_capability = +XDELTAPUSH XCHATMAIL
+# these are the capabilities Delta Chat cares about actually 
+# so let's keep the network overhead per login small
+# https://github.com/deltachat/deltachat-core-rust/blob/master/src/imap/capabilities.rs
+imap_capability = IMAP4rev1 IDLE MOVE QUOTA CONDSTORE NOTIFY METADATA XDELTAPUSH XCHATMAIL
 
 
 # Authentication for system users.
@@ -107,16 +106,10 @@ mail_attribute_dict = proxy:/run/chatmail-metadata/metadata.socket:metadata
 # `imap_zlib` enables IMAP COMPRESS (RFC 4978).
 # <https://datatracker.ietf.org/doc/html/rfc4978.html>
 protocol imap {
-  mail_plugins = $mail_plugins imap_zlib imap_quota last_login
+  mail_plugins = $mail_plugins imap_zlib imap_quota
   imap_metadata = yes
 }
 
-plugin {
-  last_login_dict = proxy:/run/chatmail-lastlogin/lastlogin.socket:lastlogin
-  #last_login_key = last-login/%u # default
-  last_login_precision = s
-}
-
 protocol lmtp {
   # notify plugin is a dependency of push_notification plugin:
   # <https://doc.dovecot.org/settings/plugin/notify-plugin/>
@@ -200,24 +193,3 @@ ssl_key = </var/lib/acme/live/{{ config.mail_domain }}/privkey
 ssl_dh = </usr/share/dovecot/dh.pem
 ssl_min_protocol = TLSv1.2
 ssl_prefer_server_ciphers = yes
-
-
-{% if config.imap_rawlog %}
-service postlogin {
-  executable = script-login -d rawlog
-  unix_listener postlogin {
- }  
-} 
-service imap {
-  executable = imap postlogin 
-} 
-  
-protocol imap { 
-  #rawlog_dir = /tmp/rawlog/%u
-  # Put .in and .out imap protocol logging files into per-user homedir 
-  # You can use a command like this to combine into one protocol stream:
-  # sort -sn <(sed 's/ / C: /' *.in) <(sed 's/ / S: /' cat *.out)
-
-  rawlog_dir = %h 
-} 
-{% endif %}

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -13,15 +13,13 @@ events {
 stream {
         map $ssl_preread_alpn_protocols $proxy {
             default 127.0.0.1:8443;
-            ~\bsmtp\b 127.0.0.1:465;
+            ~\bsmtp\b 127.0.0.1:submissions;
-            ~\bimap\b 127.0.0.1:993;
+            ~\bimap\b 127.0.0.1:imaps;
         }
 
         server {
                 listen 443;
-                {% if not disable_ipv6 %}
                 listen [::]:443;
-                {% endif %}
                 proxy_pass $proxy;
                 ssl_preread on;
         }
@@ -45,11 +43,8 @@ http {
 	gzip on;
 
 	server {
-  
 		listen 8443 ssl default_server;
-		{% if not disable_ipv6 %}
 		listen [::]:8443 ssl default_server;
-		{% endif %}
 
 		root /var/www/html;
 
@@ -101,9 +96,7 @@ http {
 	# Redirect www. to non-www
 	server {
 		listen 8443 ssl;
-		{% if not disable_ipv6 %}
 		listen [::]:8443 ssl;
-		{% endif %}
 		server_name www.{{ config.domain_name }};
 		return 301 $scheme://{{ config.domain_name }}$request_uri;
 		access_log syslog:server=unix:/dev/log,facility=local7;

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -62,14 +62,11 @@ mydestination =
 relayhost = 
 mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128
 mailbox_size_limit = 0
-message_size_limit = {{config.max_message_size}}
+# maximum 30MB sized messages
+message_size_limit = 31457280
 recipient_delimiter = +
 inet_interfaces = all
-{% if disable_ipv6 %}
-inet_protocols = ipv4
-{% else %}
 inet_protocols = all
-{% endif %}
 
 virtual_transport = lmtp:unix:private/dovecot-lmtp
 virtual_mailbox_domains = {{ config.mail_domain }}

cmdeploy/src/cmdeploy/remote/__init__.py

@@ -1,12 +0,0 @@
-"""
-
-The 'cmdeploy.remote' sub package contains modules with remotely executing functions.
-
-Its "_sshexec_bootstrap" module is executed remotely through `SSHExec`
-and its main() loop there stays connected via a command channel,
-ready to receive function invocations ("command") and return results.
-"""
-
-from . import rdns, rshell
-
-__all__ = ["rdns", "rshell"]

cmdeploy/src/cmdeploy/remote/_sshexec_bootstrap.py

@@ -1,30 +0,0 @@
-import builtins
-import importlib
-import traceback
-
-## Function Execution server
-
-
-def _run_loop(cmd_channel):
-    while cmd := cmd_channel.receive():
-        cmd_channel.send(_handle_one_request(cmd))
-
-
-def _handle_one_request(cmd):
-    pymod_path, func_name, kwargs = cmd
-    try:
-        mod = importlib.import_module(pymod_path)
-        func = getattr(mod, func_name)
-        res = func(**kwargs)
-        return ("finish", res)
-    except:
-        data = traceback.format_exc()
-        return ("error", data)
-
-
-def main(channel):
-    # enable simple "print" logging
-
-    builtins.print = lambda x="": channel.send(("log", x))
-
-    _run_loop(channel)

cmdeploy/src/cmdeploy/remote/rshell.py

@@ -1,16 +0,0 @@
-from subprocess import CalledProcessError, check_output
-
-
-def shell(command, fail_ok=False):
-    print(f"$ {command}")
-    try:
-        return check_output(command, shell=True).decode().rstrip()
-    except CalledProcessError:
-        if not fail_ok:
-            raise
-        return ""
-
-
-def get_systemd_running():
-    lines = shell("systemctl --type=service --state=running").split("\n")
-    return [line for line in lines if line.startswith("  ")]

cmdeploy/src/cmdeploy/remote_funcs.py

@@ -11,26 +11,40 @@ All functions of this module
 """
 
 import re
+import traceback
+from subprocess import CalledProcessError, check_output
 
-from .rshell import CalledProcessError, shell
+
+def shell(command, fail_ok=False):
+    print(f"$ {command}")
+    try:
+        return check_output(command, shell=True).decode().rstrip()
+    except CalledProcessError:
+        if not fail_ok:
+            raise
+        return ""
+
+
+def get_systemd_running():
+    lines = shell("systemctl --type=service --state=running").split("\n")
+    return [line for line in lines if line.startswith("  ")]
 
 
 def perform_initial_checks(mail_domain):
-    """Collecting initial DNS settings."""
+    """Collecting initial DNS zone content."""
     assert mail_domain
-    if not shell("dig", fail_ok=True):
-        shell("apt-get install -y dnsutils")
-    shell(f"unbound-control flush_zone {mail_domain}", fail_ok=True)
     A = query_dns("A", mail_domain)
     AAAA = query_dns("AAAA", mail_domain)
     MTA_STS = query_dns("CNAME", f"mta-sts.{mail_domain}")
-    WWW = query_dns("CNAME", f"www.{mail_domain}")
 
-    res = dict(mail_domain=mail_domain, A=A, AAAA=AAAA, MTA_STS=MTA_STS, WWW=WWW)
+    res = dict(mail_domain=mail_domain, A=A, AAAA=AAAA, MTA_STS=MTA_STS)
-    if not MTA_STS or not WWW or (not A and not AAAA):
+    if not MTA_STS or (not A and not AAAA):
         return res
 
     res["acme_account_url"] = shell("acmetool account-url", fail_ok=True)
+    if not shell("dig", fail_ok=True):
+        shell("apt-get install -y dnsutils")
+    shell(f"unbound-control flush_zone {mail_domain}", fail_ok=True)
     res["dkim_entry"] = get_dkim_entry(mail_domain, dkim_selector="opendkim")
 
     # parse out sts-id if exists, example: "v=STSv1; id=2090123"
@@ -60,19 +74,11 @@ def query_dns(typ, domain):
     return ""
 
 
-def check_zonefile(zonefile, mail_domain):
+def check_zonefile(zonefile):
     """Check expected zone file entries."""
-    shell(f"unbound-control flush_zone {mail_domain}", fail_ok=True)
+    diff = []
-    required = True
-    required_diff = []
-    recommended_diff = []
 
     for zf_line in zonefile.splitlines():
-        if "; Recommended" in zf_line:
-            required = False
-            continue
-        if not zf_line.strip() or zf_line.startswith(";"):
-            continue
         print(f"dns-checking {zf_line!r}")
         zf_domain, zf_typ, zf_value = zf_line.split(maxsplit=2)
         zf_domain = zf_domain.rstrip(".")
@@ -80,9 +86,40 @@ def check_zonefile(zonefile, mail_domain):
         query_value = query_dns(zf_typ, zf_domain)
         if zf_value != query_value:
             assert zf_typ in ("A", "AAAA", "CNAME", "CAA", "SRV", "MX", "TXT"), zf_line
-            if required:
+            diff.append(zf_line)
-                required_diff.append(zf_line)
-            else:
-                recommended_diff.append(zf_line)
 
-    return required_diff, recommended_diff
+    return diff
+
+
+## Function Execution server
+
+
+def _run_loop(cmd_channel):
+    while 1:
+        cmd = cmd_channel.receive()
+        if cmd is None:
+            break
+
+        cmd_channel.send(_handle_one_request(cmd))
+
+
+def _handle_one_request(cmd):
+    func_name, kwargs = cmd
+    try:
+        res = globals()[func_name](**kwargs)
+        return ("finish", res)
+    except:
+        data = traceback.format_exc()
+        return ("error", data)
+
+
+# check if this module is executed remotely
+# and setup a simple serialized function-execution loop
+
+if __name__ == "__channelexec__":
+    channel = channel  # noqa (channel object gets injected)
+
+    # enable simple "print" debugging for anyone changing this module
+    globals()["print"] = lambda x="": channel.send(("log", x))
+
+    _run_loop(channel)

cmdeploy/src/cmdeploy/service/filtermail.service.f

@@ -5,7 +5,6 @@ Description=Chatmail Postfix before queue filter
 ExecStart={execpath} {config_path}
 Restart=always
 RestartSec=30
-User=filtermail
 
 [Install]
 WantedBy=multi-user.target

cmdeploy/src/cmdeploy/service/lastlogin.service.f

@@ -1,12 +0,0 @@
-[Unit]
-Description=Dict proxy for last-login tracking
-
-[Service]
-ExecStart={execpath} /run/chatmail-lastlogin/lastlogin.socket {config_path}
-Restart=always
-RestartSec=30
-User=vmail
-RuntimeDirectory=chatmail-lastlogin
-
-[Install]
-WantedBy=multi-user.target

cmdeploy/src/cmdeploy/sshexec.py

@@ -1,65 +1,26 @@
-import inspect
-import os
 import sys
-from queue import Queue
 
 import execnet
 
-from . import remote
-
 
 class FuncError(Exception):
     pass
 
 
-def bootstrap_remote(gateway, remote=remote):
-    """Return a command channel which can execute remote functions."""
-    source_init_path = inspect.getfile(remote)
-    basedir = os.path.dirname(source_init_path)
-    name = os.path.basename(basedir)
-
-    # rsync sourcedir to remote host
-    remote_pkg_path = f"/root/from-cmdeploy/{name}"
-    q = Queue()
-    finish = lambda: q.put(None)
-    rsync = execnet.RSync(sourcedir=basedir, verbose=False)
-    rsync.add_target(gateway, remote_pkg_path, finishedcallback=finish, delete=True)
-    rsync.send()
-    q.get()
-
-    # start sshexec bootstrap and return its command channel
-    remote_sys_path = os.path.dirname(remote_pkg_path)
-    channel = gateway.remote_exec(
-        f"""
-        import sys
-        sys.path.insert(0, {remote_sys_path!r})
-        from remote._sshexec_bootstrap import main
-        main(channel)
-    """
-    )
-    return channel
-
-
-def print_stderr(item="", end="\n"):
-    print(item, file=sys.stderr, end=end)
-
-
 class SSHExec:
     RemoteError = execnet.RemoteError
     FuncError = FuncError
 
-    def __init__(self, host, verbose=False, python="python3", timeout=60):
+    def __init__(self, host, remote_funcs, verbose=False, python="python3", timeout=60):
         self.gateway = execnet.makegateway(f"ssh=root@{host}//python={python}")
-        self._remote_cmdloop_channel = bootstrap_remote(self.gateway, remote)
+        self._remote_cmdloop_channel = self.gateway.remote_exec(remote_funcs)
         self.timeout = timeout
         self.verbose = verbose
 
     def __call__(self, call, kwargs=None, log_callback=None):
         if kwargs is None:
             kwargs = {}
-        assert call.__module__.startswith("cmdeploy.remote")
+        self._remote_cmdloop_channel.send((call.__name__, kwargs))
-        modname = call.__module__.replace("cmdeploy.", "")
-        self._remote_cmdloop_channel.send((modname, call.__name__, kwargs))
         while 1:
             code, data = self._remote_cmdloop_channel.receive(timeout=self.timeout)
             if log_callback is not None and code == "log":
@@ -71,17 +32,17 @@ class SSHExec:
 
     def logged(self, call, kwargs):
         def log_progress(data):
-            sys.stderr.write(".")
+            sys.stdout.write(".")
-            sys.stderr.flush()
+            sys.stdout.flush()
 
         title = call.__doc__
         if not title:
             title = call.__name__
         if self.verbose:
-            print_stderr("[ssh] " + title)
+            print("[ssh] " + title)
-            return self(call, kwargs, log_callback=print_stderr)
+            return self(call, kwargs, log_callback=print)
         else:
-            print_stderr(title, end="")
+            print(title, end="")
             res = self(call, kwargs, log_callback=log_progress)
-            print_stderr()
+            print()
             return res

cmdeploy/src/cmdeploy/tests/data/zftest.zone

@@ -1,17 +0,0 @@
-; Required DNS entries for chatmail servers
-zftest.testrun.org.                   A     135.181.204.127
-zftest.testrun.org.                   AAAA  2a01:4f9:c012:52f4::1
-zftest.testrun.org.                   MX 10 zftest.testrun.org.
-_mta-sts.zftest.testrun.org.          TXT "v=STSv1; id=202403211706"
-mta-sts.zftest.testrun.org.           CNAME zftest.testrun.org.
-www.zftest.testrun.org.               CNAME zftest.testrun.org.
-opendkim._domainkey.zftest.testrun.org. TXT "v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoYt82CVUyz2ouaqjX2kB+5J80knAyoOU3MGU5aWppmwUwwTvj/oSTSpkc5JMtVTRmKKr8NUDWAL1Yw7dfGqqPHdHfwwjS3BIvDzYx+hzgtz62RnfNgV+/2MAoNpfX7cAFIHdRzEHNtwugc3RDLquqPoupAE3Y2YRw2T5zG5fILh4vwIcJZL5Uq6B92j8wwJqOex" "33n+vm1NKQ9rxo/UsHAmZlJzpooXcG/4igTBxJyJlamVSRR6N7Nul1v//YJb7J6v2o0iPHW6uE0StzKaPPNC2IVosSRFbD9H2oqppltptFSNPlI0E+t0JBWHem6YK7xcugiO3ImMCaaU8g6Jt/wIDAQAB;s=email;t=s"
-; Recommended DNS entries
-_submission._tcp.zftest.testrun.org.  SRV 0 1 587 zftest.testrun.org.
-_submissions._tcp.zftest.testrun.org. SRV 0 1 465 zftest.testrun.org.
-_imap._tcp.zftest.testrun.org.        SRV 0 1 143 zftest.testrun.org.
-_imaps._tcp.zftest.testrun.org.       SRV 0 1 993 zftest.testrun.org.
-zftest.testrun.org.                   CAA 0 issue "letsencrypt.org;accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/1371472956"
-zftest.testrun.org.                   TXT "v=spf1 a:zftest.testrun.org ~all"
-_dmarc.zftest.testrun.org.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
-_adsp._domainkey.zftest.testrun.org.  TXT "dkim=discardable"

cmdeploy/src/cmdeploy/tests/online/benchmark.py

@@ -41,9 +41,9 @@ class TestDC:
 
         def dc_ping_pong():
             chat.send_text("ping")
-            msg = ac2._evtracker.wait_next_incoming_message()
+            msg = ac2.wait_next_incoming_message()
             msg.chat.send_text("pong")
-            ac1._evtracker.wait_next_incoming_message()
+            ac1.wait_next_incoming_message()
 
         benchmark(dc_ping_pong, 5)
 
@@ -55,6 +55,6 @@ class TestDC:
             for i in range(10):
                 chat.send_text(f"hello {i}")
             for i in range(10):
-                ac2._evtracker.wait_next_incoming_message()
+                ac2.wait_next_incoming_message()
 
         benchmark(dc_send_10_receive_10, 5)

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -2,52 +2,52 @@ import smtplib
 
 import pytest
 
-from cmdeploy import remote
+from cmdeploy import remote_funcs
 from cmdeploy.sshexec import SSHExec
 
 
 class TestSSHExecutor:
     @pytest.fixture(scope="class")
     def sshexec(self, sshdomain):
-        return SSHExec(sshdomain)
+        return SSHExec(sshdomain, remote_funcs)
 
     def test_ls(self, sshexec):
-        out = sshexec(call=remote.rdns.shell, kwargs=dict(command="ls"))
+        out = sshexec(call=remote_funcs.shell, kwargs=dict(command="ls"))
-        out2 = sshexec(call=remote.rdns.shell, kwargs=dict(command="ls"))
+        out2 = sshexec(call=remote_funcs.shell, kwargs=dict(command="ls"))
         assert out == out2
 
     def test_perform_initial(self, sshexec, maildomain):
         res = sshexec(
-            remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain)
+            remote_funcs.perform_initial_checks, kwargs=dict(mail_domain=maildomain)
         )
         assert res["A"] or res["AAAA"]
 
     def test_logged(self, sshexec, maildomain, capsys):
         sshexec.logged(
-            remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain)
+            remote_funcs.perform_initial_checks, kwargs=dict(mail_domain=maildomain)
         )
         out, err = capsys.readouterr()
-        assert err.startswith("Collecting")
+        assert out.startswith("Collecting")
-        assert err.endswith("....\n")
+        assert out.endswith("....\n")
-        assert err.count("\n") == 1
+        assert out.count("\n") == 1
 
         sshexec.verbose = True
         sshexec.logged(
-            remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain)
+            remote_funcs.perform_initial_checks, kwargs=dict(mail_domain=maildomain)
         )
         out, err = capsys.readouterr()
-        lines = err.split("\n")
+        lines = out.split("\n")
         assert len(lines) > 4
-        assert remote.rdns.perform_initial_checks.__doc__ in lines[0]
+        assert remote_funcs.perform_initial_checks.__doc__ in lines[0]
 
     def test_exception(self, sshexec, capsys):
         try:
             sshexec.logged(
-                remote.rdns.perform_initial_checks,
+                remote_funcs.perform_initial_checks,
                 kwargs=dict(mail_domain=None),
             )
         except sshexec.FuncError as e:
-            assert "rdns.py" in str(e)
+            assert "remote_funcs.py" in str(e)
             assert "AssertionError" in str(e)
         else:
             pytest.fail("didn't raise exception")

cmdeploy/src/cmdeploy/tests/plugin.py

@@ -10,6 +10,7 @@ from pathlib import Path
 
 import pytest
 from chatmaild.config import read_config
+from chatmaild.database import Database
 
 conftestdir = Path(__file__).parent
 
@@ -78,17 +79,6 @@ def pytest_report_header():
         return ["-" * len(text), text, "-" * len(text)]
 
 
-@pytest.fixture
-def cm_data(request):
-    datadir = request.fspath.dirpath("data")
-
-    class CMData:
-        def get(self, name):
-            return datadir.join(name).read()
-
-    return CMData()
-
-
 @pytest.fixture
 def benchmark(request):
     def bench(func, num, name=None, reportfunc=None):
@@ -261,6 +251,13 @@ def gencreds(chatmail_config):
     return lambda domain=None: next(gen(domain))
 
 
+@pytest.fixture()
+def db(tmpdir):
+    db_path = tmpdir / "passdb.sqlite"
+    print("database path:", db_path)
+    return Database(db_path)
+
+
 #
 # Delta Chat testplugin re-use
 # use the cmfactory fixture to get chatmail instance accounts

cmdeploy/src/cmdeploy/tests/test_dns.py

@@ -1,50 +1,36 @@
 import pytest
 
-from cmdeploy import remote
+from cmdeploy import remote_funcs
-from cmdeploy.dns import check_full_zone, check_initial_remote_data
+from cmdeploy.dns import check_initial_remote_data
-
-
-@pytest.fixture
-def mockdns_base(monkeypatch):
-    qdict = {}
-
-    def query_dns(typ, domain):
-        try:
-            return qdict[typ][domain]
-        except KeyError:
-            return ""
-
-    monkeypatch.setattr(remote.rdns, query_dns.__name__, query_dns)
-    return qdict
-
-
-@pytest.fixture
-def mockdns(mockdns_base):
-    mockdns_base.update(
-        {
-            "A": {"some.domain": "1.1.1.1"},
-            "AAAA": {"some.domain": "fde5:cd7a:9e1c:3240:5a99:936f:cdac:53ae"},
-            "CNAME": {
-                "mta-sts.some.domain": "some.domain.",
-                "www.some.domain": "some.domain.",
-            },
-        }
-    )
-    return mockdns_base
 
 
 class TestPerformInitialChecks:
+    @pytest.fixture
+    def mockdns(self, monkeypatch):
+        qdict = {
+            "A": {"some.domain": "1.1.1.1"},
+            "AAAA": {"some.domain": "fde5:cd7a:9e1c:3240:5a99:936f:cdac:53ae"},
+            "CNAME": {"mta-sts.some.domain": "some.domain"},
+        }.copy()
+
+        def query_dns(typ, domain):
+            try:
+                return qdict[typ][domain]
+            except KeyError:
+                return ""
+
+        monkeypatch.setattr(remote_funcs, query_dns.__name__, query_dns)
+        return qdict
+
     def test_perform_initial_checks_ok1(self, mockdns):
-        remote_data = remote.rdns.perform_initial_checks("some.domain")
+        remote_data = remote_funcs.perform_initial_checks("some.domain")
-        assert remote_data["A"] == mockdns["A"]["some.domain"]
+        assert len(remote_data) == 7
-        assert remote_data["AAAA"] == mockdns["AAAA"]["some.domain"]
-        assert remote_data["MTA_STS"] == mockdns["CNAME"]["mta-sts.some.domain"]
-        assert remote_data["WWW"] == mockdns["CNAME"]["www.some.domain"]
 
     @pytest.mark.parametrize("drop", ["A", "AAAA"])
     def test_perform_initial_checks_with_one_of_A_AAAA(self, mockdns, drop):
         del mockdns[drop]
-        remote_data = remote.rdns.perform_initial_checks("some.domain")
+        remote_data = remote_funcs.perform_initial_checks("some.domain")
+        assert len(remote_data) == 7
         assert not remote_data[drop]
 
         l = []
@@ -53,75 +39,12 @@ class TestPerformInitialChecks:
         assert not l
 
     def test_perform_initial_checks_no_mta_sts(self, mockdns):
-        del mockdns["CNAME"]["mta-sts.some.domain"]
+        del mockdns["CNAME"]
-        remote_data = remote.rdns.perform_initial_checks("some.domain")
+        remote_data = remote_funcs.perform_initial_checks("some.domain")
+        assert len(remote_data) == 4
         assert not remote_data["MTA_STS"]
 
         l = []
         res = check_initial_remote_data(remote_data, print=l.append)
         assert not res
         assert len(l) == 2
-
-
-def parse_zonefile_into_dict(zonefile, mockdns_base, only_required=False):
-    for zf_line in zonefile.split("\n"):
-        if zf_line.startswith("#"):
-            if "Recommended" in zf_line and only_required:
-                return
-            continue
-        if not zf_line.strip():
-            continue
-        zf_domain, zf_typ, zf_value = zf_line.split(maxsplit=2)
-        zf_domain = zf_domain.rstrip(".")
-        zf_value = zf_value.strip()
-        mockdns_base.setdefault(zf_typ, {})[zf_domain] = zf_value
-
-
-class MockSSHExec:
-    def logged(self, func, kwargs):
-        return func(**kwargs)
-
-    def call(self, func, kwargs):
-        return func(**kwargs)
-
-
-class TestZonefileChecks:
-    def test_check_zonefile_all_ok(self, cm_data, mockdns_base):
-        zonefile = cm_data.get("zftest.zone")
-        parse_zonefile_into_dict(zonefile, mockdns_base)
-        required_diff, recommended_diff = remote.rdns.check_zonefile(
-            zonefile, "some.domain"
-        )
-        assert not required_diff and not recommended_diff
-
-    def test_check_zonefile_recommended_not_set(self, cm_data, mockdns_base):
-        zonefile = cm_data.get("zftest.zone")
-        zonefile_mocked = zonefile.split("; Recommended")[0]
-        parse_zonefile_into_dict(zonefile_mocked, mockdns_base)
-        required_diff, recommended_diff = remote.rdns.check_zonefile(
-            zonefile, "some.domain"
-        )
-        assert not required_diff
-        assert len(recommended_diff) == 8
-
-    def test_check_zonefile_output_required_fine(self, cm_data, mockdns_base, mockout):
-        zonefile = cm_data.get("zftest.zone")
-        zonefile_mocked = zonefile.split("; Recommended")[0]
-        parse_zonefile_into_dict(zonefile_mocked, mockdns_base, only_required=True)
-        mssh = MockSSHExec()
-        mockdns_base["mail_domain"] = "some.domain"
-        res = check_full_zone(mssh, mockdns_base, out=mockout, zonefile=zonefile)
-        assert res == 0
-        assert "WARNING" in mockout.captured_plain[0]
-        assert len(mockout.captured_plain) == 9
-
-    def test_check_zonefile_output_full(self, cm_data, mockdns_base, mockout):
-        zonefile = cm_data.get("zftest.zone")
-        parse_zonefile_into_dict(zonefile, mockdns_base)
-        mssh = MockSSHExec()
-        mockdns_base["mail_domain"] = "some.domain"
-        res = check_full_zone(mssh, mockdns_base, out=mockout, zonefile=zonefile)
-        assert res == 0
-        assert not mockout.captured_red
-        assert "correct" in mockout.captured_green[0]
-        assert not mockout.captured_red

www/src/index.md

@@ -11,11 +11,7 @@ for Delta Chat users.  For details how it avoids storing personal information
 please see our [privacy policy](privacy.html). 
 {% endif %}
 
-<a class="cta-button" href="DCACCOUNT:https://{{ config.mail_domain }}/new">Get a {{config.mail_domain}} chat profile</a>
+👉 **Tap** or scan this QR code to get a `@{{config.mail_domain}}` chat profile
-
-If you are viewing this page on a different device
-without a Delta Chat app,
-you can also **scan this QR code** with Delta Chat:
 
 <a href="DCACCOUNT:https://{{ config.mail_domain }}/new">
     <img width=300 style="float: none;" src="qr-chatmail-invite-{{config.mail_domain}}.png" /></a>

www/src/info.md

@@ -8,9 +8,11 @@ for the usage in chats, especially DeltaChat.
 
 ### Choosing a chatmail address instead of using a random one
 
-In the Delta Chat account setup you may tap `Create a profile` then `Use other server` and choose `Classic e-mail login`. Here fill the two fields like this: 
+In the Delta Chat account setup 
+you may tap `I already have a profile`
+and fill the two fields like this: 
 
-- `E-Mail Address`: invent a word with
+- `Address`: invent a word with
 {% if username_min_length == username_max_length %}
   *exactly* {{ username_min_length }}
 {% else %}
@@ -24,7 +26,7 @@ In the Delta Chat account setup you may tap `Create a profile` then `Use other s
   characters
   and append `@{{config.mail_domain}}` to it.
 
-- `Existing Password`: invent at least {{ password_min_length }} characters.
+- `Password`: invent at least {{ password_min_length }} characters.
 
 If the e-mail address is not yet taken, you'll get that account. 
 The first login sets your password. 
@@ -43,20 +45,6 @@ The first login sets your password.
 - You can store up to [{{ config.max_mailbox_size }} messages on the server](https://delta.chat/en/help#what-happens-if-i-turn-on-delete-old-messages-from-server).
 
 
-### <a name="account-deletion"></a> Account deletion 
-
-If you remove a {{ config.mail_domain }} profile from within the Delta Chat app, 
-then the according account on the server, along with all associated data,
-is automatically deleted {{ config.delete_inactive_users_after }} days afterwards. 
-
-If you use multiple devices 
-then you need to remove the according chat profile from each device
-in order for all account data to be removed on the server side. 
-
-If you have any further questions or requests regarding account deletion
-please send a message from your account to {{ config.privacy_mail }}. 
-
-
 ### Who are the operators? Which software is running? 
 
 This chatmail provider is run by a small voluntary group of devs and sysadmins,

www/src/main.css

@@ -72,15 +72,3 @@ code {
   color: red;
   font-weight: bold;
 }
-
-.cta-button, .cta-button:hover, .cta-button:visited {
-    border: 1.5px solid #a4c2d0;
-    border-radius: 5px;
-    padding: 10px;
-    display: inline-block;
-    margin: 10px 0;
-
-    background: linear-gradient(120deg, #77888f, #364e59);
-    color: white !important;
-    font-weight: bold;
-}