apply cmdeploy fmt linting, no content changes

This change reverts 06560dd071

Main reason for using the same address for sending
as the one used in DNS is to pass FCrDNS
(forward-confirmed reverse DNS) checks:
IP address used by SMTP client should resolve
to the domain which in turn resolves to the same IP.
chatmail relays don't do check reverse DNS
for incoming connections,
but other email servers may do and reject email
if the check does not pass.

Most chatmail relays only have one IP address per address family,
so this configuration does not change anything.

For chatmail relays that have multiple addresses
and only publishing one IP to DNS,
source address used for outgoing SMTP connections
should be the public IP.

This can be ensured by configuring the source
address in the routing table,
e.g. with the `src` argument
to `ip route add/change/replace` command.

Solving this by binding SMTP client address
on the application level prevents chatmail relays
from configuring alternative routes.

Besides, some chatmail relays are NATed
and NAT is responsible for translating the address to the public one,
in which case using `smtp_bind_address_enforce`
will result in unnecessarily deferring all mails.

CHANGELOG.md

@@ -1,24 +1,5 @@
 # Changelog for chatmail deployment 
 
-## Unreleased
-
-### Features
-
-- Automated per-user quota-keeping.
-  Replace daily timer-based message expire script
-  with Dovecot quota-warning-triggered cleanup (`chatmail-quota-expire`).
-  When a user reaches 90% of their mailbox quota
-  Dovecot calls the new script which removes the largest and oldest messages
-  until usage drops below 80%.
-  The daily `chatmail-expire` timer now only handles deletion
-  of inactive user mailboxes.
-
-  After upgrading, run the following once to clean up
-  mailboxes that are already over quota::
-
-      /usr/local/lib/chatmaild/venv/bin/chatmail-quota-expire \
-          400 /home/vmail/mail/YOURDOMAIN --sweep
-
 ## 1.9.0 2025-12-18
 
 ### Documentation

chatmaild/pyproject.toml

@@ -21,8 +21,8 @@ where = ['src']
 [project.scripts]
 doveauth = "chatmaild.doveauth:main"
 chatmail-metadata = "chatmaild.metadata:main"
-chatmail-expire = "chatmaild.expire_inactive_users:main"
-chatmail-quota-expire = "chatmaild.quota_expire:main"
+chatmail-expire = "chatmaild.expire:daily_expire_main"
+chatmail-quota-expire = "chatmaild.expire:quota_expire_main"
 chatmail-fsreport = "chatmaild.fsreport:main"
 lastlogin = "chatmaild.lastlogin:main"
 turnserver = "chatmaild.turnserver:main"

chatmaild/src/chatmaild/config.py

@@ -1,4 +1,3 @@
-import os
 from pathlib import Path
 
 import iniconfig
@@ -25,6 +24,8 @@ class Config:
         self.max_user_send_burst_size = int(params.get("max_user_send_burst_size", 10))
         self.max_mailbox_size = params["max_mailbox_size"]
         self.max_message_size = int(params.get("max_message_size", "31457280"))
+        self.delete_mails_after = params["delete_mails_after"]
+        self.delete_large_after = params["delete_large_after"]
         self.delete_inactive_users_after = int(params["delete_inactive_users_after"])
         self.username_min_length = int(params["username_min_length"])
         self.username_max_length = int(params["username_max_length"])
@@ -45,8 +46,6 @@ class Config:
         )
         self.mtail_address = params.get("mtail_address")
         self.disable_ipv6 = params.get("disable_ipv6", "false").lower() == "true"
-        self.addr_v4 = os.environ.get("CHATMAIL_ADDR_V4", "")
-        self.addr_v6 = os.environ.get("CHATMAIL_ADDR_V6", "")
         self.acme_email = params.get("acme_email", "")
         self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true"
         self.imap_compress = params.get("imap_compress", "false").lower() == "true"
@@ -113,7 +112,7 @@ class Config:
 
 def parse_size_mb(limit):
     """Parse a size string like ``500M`` or ``2G`` and return megabytes."""
-    value = limit.strip().upper().rstrip("B")
+    value = limit.strip().upper().removesuffix("B")
     if value.endswith("G"):
         return int(value[:-1]) * 1024
     if value.endswith("M"):

chatmaild/src/chatmaild/expire.py

@@ -4,17 +4,26 @@ Expire old messages and addresses.
 """
 
 import os
+import re
 import shutil
 import sys
 import time
 from argparse import ArgumentParser
 from collections import namedtuple
 from datetime import datetime
+from pathlib import Path
 from stat import S_ISREG
 
 from chatmaild.config import read_config
 
 FileEntry = namedtuple("FileEntry", ("path", "mtime", "size"))
+QuotaFileEntry = namedtuple("QuotaFileEntry", ("mtime", "quota_size", "path"))
+
+# Quota cleanup factor of max_mailbox_size. The mailbox is reset to this size.
+QUOTA_CLEANUP_FACTOR = 0.7
+
+# e.g. "cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S"
+_dovecot_fn_rex = re.compile(r".+/(\d+)\..+,S=(\d+)")
 
 
 def iter_mailboxes(basedir, maxnum):
@@ -74,6 +83,42 @@ class MailboxStat:
         self.extrafiles.sort(key=lambda x: -x.size)
 
 
+def parse_dovecot_filename(relpath):
+    m = _dovecot_fn_rex.match(relpath)
+    if not m:
+        return None
+    return QuotaFileEntry(int(m.group(1)), int(m.group(2)), relpath)
+
+
+def scan_mailbox_messages(mbox):
+    messages = []
+    for sub in ("cur", "new"):
+        for name in os_listdir_if_exists(mbox / sub):
+            if entry := parse_dovecot_filename(f"{sub}/{name}"):
+                messages.append(entry)
+    return messages
+
+
+def expire_to_target(mbox, target_bytes):
+    messages = scan_mailbox_messages(mbox)
+    total_size = sum(m.quota_size for m in messages)
+    # Keep recent 24 hours of messages protected from expiry because
+    # likely something is wrong with interactions on that address
+    # and quota-full signal can help the address owner's device to notice it
+    undeletable_messages_cutoff = time.time() - (3600 * 24)
+    removed = 0
+    for entry in sorted(messages):
+        if total_size <= target_bytes:
+            break
+        if entry.mtime > undeletable_messages_cutoff:
+            break
+        (mbox / entry.path).unlink(missing_ok=True)
+        total_size -= entry.quota_size
+        removed += 1
+
+    return removed
+
+
 def print_info(msg):
     print(msg, file=sys.stderr)
 
@@ -115,8 +160,11 @@ class Expiry:
         cutoff_without_login = (
             self.now - int(self.config.delete_inactive_users_after) * 86400
         )
+        cutoff_mails = self.now - int(self.config.delete_mails_after) * 86400
+        cutoff_large_mails = self.now - int(self.config.delete_large_after) * 86400
 
         self.all_mboxes += 1
+        changed = False
         if mbox.last_login and mbox.last_login < cutoff_without_login:
             self.remove_mailbox(mbox.basedir)
             return
@@ -128,17 +176,45 @@ class Expiry:
                 print_info(f"checking mailbox {date.strftime('%b %d')} {mboxname}")
             else:
                 print_info(f"checking mailbox (no last_login) {mboxname}")
+        self.all_files += len(mbox.messages)
+        for message in mbox.messages:
+            if message.mtime < cutoff_mails:
+                self.remove_file(message.path, mtime=message.mtime)
+            elif message.size > 200000 and message.mtime < cutoff_large_mails:
+                # we only remove noticed large files (not unnoticed ones in new/)
+                parts = message.path.split("/")
+                if len(parts) >= 2 and parts[-2] == "cur":
+                    self.remove_file(message.path, mtime=message.mtime)
+            else:
+                continue
+            changed = True
+
+        target_bytes = (
+            self.config.max_mailbox_size_mb * 1024 * 1024 * QUOTA_CLEANUP_FACTOR
+        )
+        removed = expire_to_target(Path(mbox.basedir), target_bytes)
+        if removed:
+            changed = True
+            self.del_files += removed
+            if self.verbose:
+                print_info(
+                    f"quota-expire: removed {removed} message(s) from {mboxname}"
+                )
+
+        if changed:
+            self.remove_file(f"{mbox.basedir}/maildirsize")
 
     def get_summary(self):
         return (
             f"Removed {self.del_mboxes} out of {self.all_mboxes} mailboxes "
+            f"and {self.del_files} out of {self.all_files} files in existing mailboxes "
             f"in {time.time() - self.start:2.2f} seconds"
         )
 
 
-def main(args=None):
+def daily_expire_main(args=None):
     """Expire mailboxes and messages according to chatmail config"""
-    parser = ArgumentParser(description=main.__doc__)
+    parser = ArgumentParser(description=daily_expire_main.__doc__)
     ini = "/usr/local/lib/chatmaild/chatmail.ini"
     parser.add_argument(
         "chatmail_ini",
@@ -184,5 +260,33 @@ def main(args=None):
     print(exp.get_summary())
 
 
-if __name__ == "__main__":
-    main(sys.argv[1:])
+def quota_expire_main(args=None):
+    """Remove mailbox messages to stay within a megabyte target.
+
+    This entry point is called by dovecot when a quota threshold is passed.
+    """
+
+    parser = ArgumentParser(description=quota_expire_main.__doc__)
+    parser.add_argument(
+        "target_mb",
+        type=int,
+        help="target mailbox size in megabytes",
+    )
+    parser.add_argument(
+        "mailbox_path",
+        type=Path,
+        help="path to a user mailbox",
+    )
+    args = parser.parse_args(args)
+
+    target_bytes = args.target_mb * 1024 * 1024
+
+    removed_count = expire_to_target(args.mailbox_path, target_bytes)
+    if removed_count:
+        (args.mailbox_path / "maildirsize").unlink(missing_ok=True)
+        print(
+            f"quota-expire: removed {removed_count} message(s)"
+            f" from {args.mailbox_path.name}",
+            file=sys.stderr,
+        )
+    return 0

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -18,11 +18,18 @@ max_user_send_per_minute = 60
 max_user_send_burst_size = 10
 
 # maximum mailbox size of a chatmail address
+# Oldest messages will be removed automatically, so mailboxes never run full.
 max_mailbox_size = 500M
 
 # maximum message size for an e-mail in bytes
 max_message_size = 31457280
 
+# days after which mails are unconditionally deleted
+delete_mails_after = 20
+
+# days after which large messages (>200k) are unconditionally deleted
+delete_large_after = 7
+
 # days after which users without a successful login are deleted (database and mails)
 delete_inactive_users_after = 90
 

chatmaild/src/chatmaild/quota_expire.py

@@ -1,152 +0,0 @@
-"""
-Remove messages from a mailbox to meet a size target.
-
-Dovecot calls this script when a user's quota is near its limit.
-Files are scored by ``size * age`` so that large, old messages
-are removed first.
-
-Usage::
-
-    quota_expire <target_mb> <mailbox_path>
-
-"""
-
-import os
-import sys
-import time
-from argparse import ArgumentParser
-from collections import namedtuple
-from stat import S_ISREG
-
-FileEntry = namedtuple("FileEntry", ("path", "mtime", "size"))
-
-
-def _get_file_entry(path):
-    try:
-        st = os.stat(path)
-    except FileNotFoundError:
-        return None
-    if not S_ISREG(st.st_mode):
-        return None
-    return FileEntry(path, st.st_mtime, st.st_size)
-
-
-def _listdir(path):
-    try:
-        return os.listdir(path)
-    except FileNotFoundError:
-        return []
-
-
-def scan_mailbox_messages(mailbox_dir):
-    messages = []
-    for sub in ("cur", "new", "tmp"):
-        subdir = f"{mailbox_dir}/{sub}"
-        for name in _listdir(subdir):
-            entry = _get_file_entry(f"{subdir}/{name}")
-            if entry is not None:
-                messages.append(entry)
-    return messages
-
-
-def _remove_stale_caches(mailbox_dir):
-    for name in ("maildirsize", "dovecot.index.cache"):
-        try:
-            os.unlink(f"{mailbox_dir}/{name}")
-        except FileNotFoundError:
-            pass
-
-
-def expire_to_target(mailbox_dir, target_bytes, now=None):
-    """Remove highest-scored files until total size <= *target_bytes*.
-
-    Returns the list of removed file paths.
-    """
-    if now is None:
-        now = time.time()
-
-    messages = scan_mailbox_messages(mailbox_dir)
-    total_size = sum(m.size for m in messages)
-
-    if total_size <= target_bytes:
-        return []
-
-    # Score: large and old files get the highest score.
-    scored = sorted(
-        messages,
-        key=lambda m: m.size * (now - m.mtime),
-        reverse=True,
-    )
-
-    removed = []
-    for entry in scored:
-        if total_size <= target_bytes:
-            break
-        try:
-            os.unlink(entry.path)
-        except FileNotFoundError:
-            continue
-        total_size -= entry.size
-        removed.append(entry.path)
-
-    if removed:
-        _remove_stale_caches(mailbox_dir)
-
-    return removed
-
-
-def main(args=None):
-    """Remove mailbox messages to stay within a megabyte target."""
-    parser = ArgumentParser(description=main.__doc__)
-    parser.add_argument(
-        "target_mb",
-        type=int,
-        help="target mailbox size in megabytes",
-    )
-    parser.add_argument(
-        "mailbox_path",
-        help="path to a user mailbox, or with --sweep the mailboxes directory",
-    )
-    parser.add_argument(
-        "--sweep",
-        action="store_true",
-        help="sweep all mailboxes under mailbox_path",
-    )
-    args = parser.parse_args(args)
-
-    target_bytes = args.target_mb * 1024 * 1024
-
-    if args.sweep:
-        return _sweep(args.mailbox_path, target_bytes)
-
-    removed = expire_to_target(args.mailbox_path, target_bytes)
-    if removed:
-        print(
-            f"removed {len(removed)} file(s) from {args.mailbox_path}"
-            f" to reach {args.target_mb} MB target",
-            file=sys.stderr,
-        )
-    return 0
-
-
-def _sweep(mailboxes_dir, target_bytes):
-    try:
-        names = os.listdir(mailboxes_dir)
-    except FileNotFoundError:
-        print(f"directory not found: {mailboxes_dir}", file=sys.stderr)
-        return 1
-    for name in sorted(names):
-        if "@" not in name:
-            continue
-        mbox = f"{mailboxes_dir}/{name}"
-        removed = expire_to_target(mbox, target_bytes)
-        if removed:
-            print(
-                f"removed {len(removed)} file(s) from {name}",
-                file=sys.stderr,
-            )
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

chatmaild/src/chatmaild/tests/test_config.py

@@ -1,6 +1,6 @@
 import pytest
 
-from chatmaild.config import read_config
+from chatmaild.config import parse_size_mb, read_config
 
 
 def test_read_config_basic(example_config):
@@ -34,6 +34,8 @@ def test_read_config_testrun(make_config):
     assert config.postfix_reinject_port == 10025
     assert config.max_user_send_per_minute == 60
     assert config.max_mailbox_size == "500M"
+    assert config.delete_mails_after == "20"
+    assert config.delete_large_after == "7"
     assert config.username_min_length == 9
     assert config.username_max_length == 9
     assert config.password_min_length == 9
@@ -119,3 +121,17 @@ def test_config_tls_external_bad_format(make_config):
                 "tls_external_cert_and_key": "/only/one/path.pem",
             },
         )
+
+
+def test_parse_size_mb():
+    assert parse_size_mb("500M") == 500
+    assert parse_size_mb("2G") == 2048
+    assert parse_size_mb("  1g  ") == 1024
+    assert parse_size_mb("100MB") == 100
+    assert parse_size_mb("256") == 256
+
+
+def test_max_mailbox_size_mb(make_config):
+    config = make_config("chat.example.org")
+    assert config.max_mailbox_size == "500M"
+    assert config.max_mailbox_size_mb == 500

chatmaild/src/chatmaild/tests/test_delete_inactive_users.py

@@ -1,7 +1,7 @@
 import time
 
 from chatmaild.doveauth import AuthDictProxy
-from chatmaild.expire import main as main_expire
+from chatmaild.expire import daily_expire_main as main_expire
 
 
 def test_login_timestamps(example_config):

chatmaild/src/chatmaild/tests/test_expire.py

@@ -1,6 +1,9 @@
+import itertools
 import os
 import random
+import time
 from datetime import datetime
+from fnmatch import fnmatch
 from pathlib import Path
 
 import pytest
@@ -8,13 +11,19 @@ import pytest
 from chatmaild.expire import (
     FileEntry,
     MailboxStat,
+    expire_to_target,
     get_file_entry,
     iter_mailboxes,
     os_listdir_if_exists,
+    parse_dovecot_filename,
+    quota_expire_main,
+    scan_mailbox_messages,
 )
-from chatmaild.expire import main as expiry_main
+from chatmaild.expire import daily_expire_main as expiry_main
 from chatmaild.fsreport import main as report_main
 
+MB = 1024 * 1024
+
 
 def fill_mbox(folderdir):
     password = folderdir.joinpath("password")
@@ -153,6 +162,35 @@ def test_expiry_cli_basic(example_config, mbox1):
     expiry_main(args)
 
 
+def test_expiry_cli_old_files(capsys, example_config, mbox1):
+    relpaths_old = ["cur/msg_old1", "cur/msg_old1"]
+    cutoff_days = int(example_config.delete_mails_after) + 1
+    create_new_messages(mbox1.basedir, relpaths_old, size=1000, days=cutoff_days)
+
+    relpaths_large = ["cur/msg_old_large1", "new/msg_old_large2"]
+    cutoff_days = int(example_config.delete_large_after) + 1
+    create_new_messages(
+        mbox1.basedir, relpaths_large, size=1000 * 300, days=cutoff_days
+    )
+
+    create_new_messages(mbox1.basedir, ["cur/shouldstay"], size=1000 * 300, days=1)
+
+    args = str(example_config._inipath), "--remove", "-v"
+    expiry_main(args)
+    out, err = capsys.readouterr()
+
+    allpaths = relpaths_old + relpaths_large + ["maildirsize"]
+    for path in allpaths:
+        for line in err.split("\n"):
+            if fnmatch(line, f"removing*{path}"):
+                break
+        else:
+            if path != "new/msg_old_large2":
+                pytest.fail(f"failed to remove {path}\n{err}")
+
+    assert "shouldstay" not in err
+
+
 def test_get_file_entry(tmp_path):
     assert get_file_entry(str(tmp_path.joinpath("123123"))) is None
     p = tmp_path.joinpath("x")
@@ -166,3 +204,51 @@ def test_os_listdir_if_exists(tmp_path):
     tmp_path.joinpath("x").write_text("hello")
     assert len(os_listdir_if_exists(str(tmp_path))) == 1
     assert len(os_listdir_if_exists(str(tmp_path.joinpath("123123")))) == 0
+
+
+# --- quota expire tests ---
+
+_msg_counter = itertools.count(1)
+
+
+def _create_message(basedir, sub, size, days_old=0, disk_size=None):
+    seq = next(_msg_counter)
+    mtime = int(time.time() - days_old * 86400)
+    name = f"{mtime}.M1P1Q{seq}.hostname,S={size},W={size}:2,S"
+    path = basedir / sub / name
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_bytes(b"x" * (disk_size if disk_size is not None else size))
+    os.utime(path, (mtime, mtime))
+    return path
+
+
+def test_parse_dovecot_filename():
+    e = parse_dovecot_filename("cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S")
+    assert e.path == "cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S"
+    assert e.mtime == 1775324677
+    assert e.quota_size == 3235
+    assert parse_dovecot_filename("cur/msg_without_structure") is None
+
+
+def test_expire_to_target(tmp_path):
+    _create_message(tmp_path, "cur", MB, days_old=10, disk_size=100)
+    _create_message(tmp_path, "new", MB, days_old=5)
+    _create_message(tmp_path, "cur", MB, days_old=0)  # undeletable (<1 hour)
+    assert len(scan_mailbox_messages(tmp_path)) == 3
+    # removes oldest first, uses S= size not disk size
+    removed = expire_to_target(tmp_path, MB)
+    assert removed == 2
+    msgs = scan_mailbox_messages(tmp_path)
+    assert len(msgs) == 1
+    # the surviving message is the fresh undeletable one
+    assert msgs[0].mtime > time.time() - 3600
+
+
+def test_quota_expire_main(tmp_path, capsys):
+    mbox = tmp_path / "user@example.org"
+    _create_message(mbox, "cur", 2 * MB, days_old=5)
+    (mbox / "maildirsize").write_text("x")
+    quota_expire_main([str(1), str(mbox)])
+    _, err = capsys.readouterr()
+    assert "quota-expire: removed 1 message(s) from user@example.org" in err
+    assert not (mbox / "maildirsize").exists()

chatmaild/src/chatmaild/tests/test_quota_expire.py

@@ -1,91 +0,0 @@
-import os
-import time
-
-from chatmaild.quota_expire import expire_to_target, scan_mailbox_messages
-
-MB = 1024 * 1024
-
-
-def _create_message(basedir, relpath, size, days_old=0):
-    path = basedir / relpath
-    path.parent.mkdir(parents=True, exist_ok=True)
-    path.write_bytes(b"x" * size)
-    mtime = time.time() - days_old * 86400
-    os.utime(path, (mtime, mtime))
-    return path
-
-
-def test_scan_cur_new_tmp(tmp_path):
-    _create_message(tmp_path, "cur/msg1", 100)
-    _create_message(tmp_path, "new/msg2", 200)
-    _create_message(tmp_path, "tmp/msg3", 300)
-    messages = scan_mailbox_messages(str(tmp_path))
-    assert len(messages) == 3
-    sizes = sorted(m.size for m in messages)
-    assert sizes == [100, 200, 300]
-
-
-def test_scan_ignores_subfolders(tmp_path):
-    _create_message(tmp_path, "cur/a", 10)
-    _create_message(tmp_path, ".DeltaChat/cur/b", 20)
-    assert len(scan_mailbox_messages(str(tmp_path))) == 1
-
-
-def test_scan_empty(tmp_path):
-    assert scan_mailbox_messages(str(tmp_path)) == []
-    assert scan_mailbox_messages(str(tmp_path / "nope")) == []
-
-
-def test_noop_under_limit(tmp_path):
-    _create_message(tmp_path, "cur/msg1", MB)
-    assert expire_to_target(str(tmp_path), 2 * MB) == []
-    assert (tmp_path / "cur" / "msg1").exists()
-
-
-def test_removes_to_target(tmp_path):
-    now = time.time()
-    for i in range(15):
-        _create_message(tmp_path, f"cur/msg{i:02d}", MB, days_old=i + 1)
-    removed = expire_to_target(str(tmp_path), 10 * MB, now=now)
-    assert len(removed) == 5
-    assert len(scan_mailbox_messages(str(tmp_path))) == 10
-
-
-def test_scoring_prefers_large_old(tmp_path):
-    now = time.time()
-    _create_message(tmp_path, "cur/large_old", 2 * MB, days_old=30)
-    _create_message(tmp_path, "cur/small_new", MB, days_old=1)
-    removed = expire_to_target(str(tmp_path), 2 * MB, now=now)
-    assert len(removed) == 1
-    assert "large_old" in removed[0]
-
-
-def test_scoring_large_new_beats_small_old(tmp_path):
-    now = time.time()
-    _create_message(tmp_path, "cur/big_new", 10 * MB, days_old=1)
-    _create_message(tmp_path, "cur/small_old", MB, days_old=5)
-    # big_new score: 10MB * 1d = 10  vs  small_old score: 1MB * 5d = 5
-    removed = expire_to_target(str(tmp_path), 10 * MB, now=now)
-    assert len(removed) == 1
-    assert "big_new" in removed[0]
-
-
-def test_exact_limit(tmp_path):
-    _create_message(tmp_path, "cur/msg1", 5 * MB)
-    assert expire_to_target(str(tmp_path), 5 * MB) == []
-
-
-def test_removes_stale_caches(tmp_path):
-    _create_message(tmp_path, "cur/msg1", 2 * MB, days_old=5)
-    (tmp_path / "maildirsize").write_text("x")
-    (tmp_path / "dovecot.index.cache").write_text("x")
-    expire_to_target(str(tmp_path), MB)
-    assert not (tmp_path / "maildirsize").exists()
-    assert not (tmp_path / "dovecot.index.cache").exists()
-
-
-def test_no_cache_removal_when_under_limit(tmp_path):
-    _create_message(tmp_path, "cur/msg1", MB)
-    (tmp_path / "maildirsize").write_text("x")
-    expire_to_target(str(tmp_path), 2 * MB)
-    assert (tmp_path / "maildirsize").exists()

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -93,7 +93,9 @@ def run_cmd(args, out):
     strict_tls = args.config.tls_cert_mode == "acme"
     if not args.dns_check_disabled:
         remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
-        if not dns.check_initial_remote_data(remote_data, strict_tls=strict_tls, print=out.red):
+        if not dns.check_initial_remote_data(
+            remote_data, strict_tls=strict_tls, print=out.red
+        ):
             return 1
 
     env = os.environ.copy()
@@ -101,9 +103,6 @@ def run_cmd(args, out):
     env["CHATMAIL_WEBSITE_ONLY"] = "True" if args.website_only else ""
     env["CHATMAIL_DISABLE_MAIL"] = "True" if args.disable_mail else ""
     env["CHATMAIL_REQUIRE_IROH"] = "True" if require_iroh else ""
-    if not args.dns_check_disabled:
-        env["CHATMAIL_ADDR_V4"] = remote_data.get("A") or ""
-        env["CHATMAIL_ADDR_V6"] = remote_data.get("AAAA") or ""
     deploy_path = importlib.resources.files(__package__).joinpath("run.py").resolve()
     pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
 
@@ -119,7 +118,11 @@ def run_cmd(args, out):
         out.check_call(cmd, env=env)
         if args.website_only:
             out.green("Website deployment completed.")
-        elif not args.dns_check_disabled and strict_tls and not remote_data["acme_account_url"]:
+        elif (
+            not args.dns_check_disabled
+            and strict_tls
+            and not remote_data["acme_account_url"]
+        ):
             out.red("Deploy completed but letsencrypt not configured")
             out.red("Run 'cmdeploy run' again")
         else:
@@ -194,12 +197,6 @@ def status_cmd(args, out):
 
 
 def test_cmd_options(parser):
-    parser.add_argument(
-        "--slow",
-        dest="slow",
-        action="store_true",
-        help="also run slow tests",
-    )
     add_ssh_host_option(parser)
 
 
@@ -221,8 +218,6 @@ def test_cmd(args, out):
         "-v",
         "--durations=5",
     ]
-    if args.slow:
-        pytest_args.append("--slow")
     ret = out.run_ret(pytest_args, env=env)
     return ret
 

cmdeploy/src/cmdeploy/deployers.py

@@ -150,18 +150,40 @@ class UnboundDeployer(Deployer):
         self.need_restart = False
 
     def install(self):
-        # Run local DNS resolver `unbound`. `resolvconf` takes care of
-        # setting up /etc/resolv.conf to use 127.0.0.1 as the resolver.
-
         # On an IPv4-only system, if unbound is started but not configured,
         # it causes subsequent steps to fail to resolve hosts.
         with blocked_service_startup():
             apt.packages(
                 name="Install unbound",
-                packages=["unbound", "unbound-anchor", "dnsutils", "resolvconf"],
+                packages=["unbound", "unbound-anchor", "dnsutils"],
             )
 
     def configure(self):
+        # Remove dynamic resolver managers that compete for /etc/resolv.conf.
+        apt.packages(
+            name="Purge resolvconf",
+            packages=["resolvconf"],
+            present=False,
+            extra_uninstall_args="--purge",
+        )
+        # systemd-resolved can't be purged due to dependencies; stop and mask.
+        server.shell(
+            name="Stop and mask systemd-resolved",
+            commands=[
+                "systemctl stop systemd-resolved.service || true",
+                "systemctl mask systemd-resolved.service",
+            ],
+        )
+        # Configure unbound resolver with Quad9 fallback and a trailing newline
+        # (SolusVM bug).
+        files.put(
+            name="Write static resolv.conf",
+            src=BytesIO(b"nameserver 127.0.0.1\nnameserver 9.9.9.9\n"),
+            dest="/etc/resolv.conf",
+            user="root",
+            group="root",
+            mode="644",
+        )
         server.shell(
             name="Generate root keys for validating DNSSEC",
             commands=[
@@ -568,20 +590,18 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
         Deployment().perform_stages([WebsiteDeployer(config)])
         return
 
-    if host.get_fact(Port, port=53) != "unbound":
-        files.line(
-            name="Add 9.9.9.9 to resolv.conf",
-            path="/etc/resolv.conf",
-            # Guard against resolv.conf missing a trailing newline (SolusVM bug).
-            line="\nnameserver 9.9.9.9",
-        )
-
     # Check if mtail_address interface is available (if configured)
-    if config.mtail_address and config.mtail_address not in ('127.0.0.1', '::1', 'localhost'):
+    if config.mtail_address and config.mtail_address not in (
+        "127.0.0.1",
+        "::1",
+        "localhost",
+    ):
         ipv4_addrs = host.get_fact(hardware.Ipv4Addrs)
         all_addresses = [addr for addrs in ipv4_addrs.values() for addr in addrs]
         if config.mtail_address not in all_addresses:
-            Out().red(f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n")
+            Out().red(
+                f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n"
+            )
             exit(1)
 
     if not is_in_container():

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -20,12 +20,30 @@ DOVECOT_ARCHIVE_VERSION = "2.3.21+dfsg1-3"
 DOVECOT_PACKAGE_VERSION = f"1:{DOVECOT_ARCHIVE_VERSION}"
 
 DOVECOT_SHA256 = {
-    ("core", "amd64"): "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d",
-    ("core", "arm64"): "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9",
-    ("imapd", "amd64"): "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86",
-    ("imapd", "arm64"): "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f",
-    ("lmtpd", "amd64"): "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab",
-    ("lmtpd", "arm64"): "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f",
+    (
+        "core",
+        "amd64",
+    ): "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d",
+    (
+        "core",
+        "arm64",
+    ): "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9",
+    (
+        "imapd",
+        "amd64",
+    ): "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86",
+    (
+        "imapd",
+        "arm64",
+    ): "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f",
+    (
+        "lmtpd",
+        "amd64",
+    ): "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab",
+    (
+        "lmtpd",
+        "arm64",
+    ): "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f",
 }
 
 
@@ -61,11 +79,7 @@ class DovecotDeployer(Deployer):
                 self.need_restart = True
         files.put(
             name="Pin dovecot packages to block Debian dist-upgrades",
-            src=io.StringIO(
-                "Package: dovecot-*\n"
-                "Pin: version *\n"
-                "Pin-Priority: -1\n"
-            ),
+            src=io.StringIO("Package: dovecot-*\nPin: version *\nPin-Priority: -1\n"),
             dest="/etc/apt/preferences.d/pin-dovecot",
             user="root",
             group="root",
@@ -84,7 +98,7 @@ class DovecotDeployer(Deployer):
         if not self.disable_mail and not self.need_restart:
             stale = host.get_fact(
                 Command,
-                'pid=$(systemctl show -p MainPID --value dovecot.service 2>/dev/null);'
+                "pid=$(systemctl show -p MainPID --value dovecot.service 2>/dev/null);"
                 ' [ "${pid:-0}" != "0" ] && readlink "/proc/$pid/exe" 2>/dev/null | grep -q "(deleted)"'
                 " && echo STALE || true",
             )
@@ -186,7 +200,7 @@ def _configure_dovecot(config: Config, debug: bool = False) -> tuple[bool, bool]
     can_modify = not is_in_container()
     for name in ("max_user_instances", "max_user_watches"):
         key = f"fs.inotify.{name}"
-        value = host.get_fact(Sysctl)[key]
+        value = host.get_fact(Sysctl).get(key, 0)
         if value > 65534:
             continue
         if not can_modify:

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -149,21 +149,25 @@ plugin {
 }
 
 plugin {
-  # for now we define static quota-rules for all users
   quota = maildir:User quota
-  quota_rule = *:storage={{ config.max_mailbox_size }}
   quota_max_mail_size={{ config.max_message_size }}
   quota_grace = 0
 
-  # When a user reaches 90% quota, run chatmail-quota-expire
-  # to remove large/old messages until usage is below 80%.
-  quota_warning = storage=90%% quota-warning {{ config.max_mailbox_size_mb * 80 // 100 }} {{ config.mailboxes_dir }}/%u
+  quota_rule = *:storage={{ config.max_mailbox_size_mb }}M
+
+  # Trigger at 75%% of quota, expire oldest messages down to 70%%.
+  # The percentages are chosen to prevent current Delta Chat users
+  # from seeing "quota warnings" which trigger at 80% and 95%.
+
+  quota_warning = storage=75%% quota-warning {{ config.max_mailbox_size_mb * 70 // 100 }} {{ config.mailboxes_dir }}/%u
 }
 
 service quota-warning {
   executable = script /usr/local/lib/chatmaild/venv/bin/chatmail-quota-expire
   user = vmail
   unix_listener quota-warning {
+     user = vmail
+     mode = 0600
   }
 }
 

cmdeploy/src/cmdeploy/mtail/delivered_mail.mtail

@@ -78,3 +78,11 @@ counter rejected_unencrypted_mail_count
 /Rejected unencrypted mail/ {
   rejected_unencrypted_mail_count++
 }
+
+counter quota_expire_runs
+counter quota_expire_removed_files
+
+/quota-expire: removed (?P<count>\d+) message\(s\)/ {
+  quota_expire_runs++
+  quota_expire_removed_files += $count
+}

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -69,15 +69,6 @@ mynetworks = 127.0.0.0/8
 {% else %}
 mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128
 {% endif %}
-{% if config.addr_v4 %}
-smtp_bind_address = {{ config.addr_v4 }}
-{% endif %}
-{% if config.addr_v6 %}
-smtp_bind_address6 = {{ config.addr_v6 }}
-{% endif %}
-{% if config.addr_v4 or config.addr_v6 %}
-smtp_bind_address_enforce = yes
-{% endif %}
 mailbox_size_limit = 0
 message_size_limit = {{config.max_message_size}}
 recipient_delimiter = +

cmdeploy/src/cmdeploy/selfsigned/deployer.py

@@ -12,15 +12,27 @@ def openssl_selfsigned_args(domain, cert_path, key_path, days=36500):
     ``www.<domain>`` and ``mta-sts.<domain>``.
     """
     return [
-        "openssl", "req", "-x509",
-        "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:P-256",
-        "-noenc", "-days", str(days),
-        "-keyout", str(key_path),
-        "-out", str(cert_path),
-        "-subj", f"/CN={domain}",
+        "openssl",
+        "req",
+        "-x509",
+        "-newkey",
+        "ec",
+        "-pkeyopt",
+        "ec_paramgen_curve:P-256",
+        "-noenc",
+        "-days",
+        str(days),
+        "-keyout",
+        str(key_path),
+        "-out",
+        str(cert_path),
+        "-subj",
+        f"/CN={domain}",
         # Mark as end-entity cert so it cannot be used as a CA to sign others.
-        "-addext", "basicConstraints=critical,CA:FALSE",
-        "-addext", "extendedKeyUsage=serverAuth,clientAuth",
+        "-addext",
+        "basicConstraints=critical,CA:FALSE",
+        "-addext",
+        "extendedKeyUsage=serverAuth,clientAuth",
         "-addext",
         f"subjectAltName=DNS:{domain},DNS:www.{domain},DNS:mta-sts.{domain}",
     ]
@@ -42,7 +54,9 @@ class SelfSignedTlsDeployer(Deployer):
 
     def configure(self):
         args = openssl_selfsigned_args(
-            self.mail_domain, self.cert_path, self.key_path,
+            self.mail_domain,
+            self.cert_path,
+            self.key_path,
         )
         cmd = shlex.join(args)
         server.shell(

cmdeploy/src/cmdeploy/tests/online/test_0_qr.py

@@ -30,12 +30,15 @@ def test_newemail_configure(maildomain, rpc, chatmail_config):
             # set_config_from_qr, so fetch credentials via requests instead
             res = requests.post(f"https://{maildomain}/new", verify=False)
             data = res.json()
-            rpc.add_or_update_transport(account_id, {
-                "addr": data["email"],
-                "password": data["password"],
-                "imapServer": maildomain,
-                "smtpServer": maildomain,
-                "certificateChecks": "acceptInvalidCertificates",
-            })
+            rpc.add_or_update_transport(
+                account_id,
+                {
+                    "addr": data["email"],
+                    "password": data["password"],
+                    "imapServer": maildomain,
+                    "smtpServer": maildomain,
+                    "certificateChecks": "acceptInvalidCertificates",
+                },
+            )
         else:
             rpc.add_transport_from_qr(account_id, url)

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -221,7 +221,6 @@ def test_rewrite_subject(cmsetup, maildata):
     assert "Subject: Unencrypted subject" not in rcvd_msg
 
 
-@pytest.mark.slow
 def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
     """Test that the per-account send-mail limit is exceeded."""
     user1, user2 = cmsetup.gen_users(2)
@@ -244,6 +243,23 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
     pytest.fail("Rate limit was not exceeded")
 
 
+def test_expunged(remote, chatmail_config):
+    outdated_days = int(chatmail_config.delete_mails_after) + 1
+    find_cmds = [
+        f"find {chatmail_config.mailboxes_dir} -path '*/cur/*' -mtime +{outdated_days} -type f",
+        f"find {chatmail_config.mailboxes_dir} -path '*/.*/cur/*' -mtime +{outdated_days} -type f",
+        f"find {chatmail_config.mailboxes_dir} -path '*/new/*' -mtime +{outdated_days} -type f",
+        f"find {chatmail_config.mailboxes_dir} -path '*/.*/new/*' -mtime +{outdated_days} -type f",
+        f"find {chatmail_config.mailboxes_dir} -path '*/tmp/*' -mtime +{outdated_days} -type f",
+        f"find {chatmail_config.mailboxes_dir} -path '*/.*/tmp/*' -mtime +{outdated_days} -type f",
+    ]
+    outdated_days = int(chatmail_config.delete_large_after) + 1
+    find_cmds.append(
+        f"find {chatmail_config.mailboxes_dir} -path '*/cur/*' -mtime +{outdated_days} -size +200k -type f"
+    )
+    for cmd in find_cmds:
+        for line in remote.iter_output(cmd):
+            assert not line
 
 
 def test_deployed_state(remote):

cmdeploy/src/cmdeploy/tests/plugin.py

@@ -23,12 +23,6 @@ def _is_ip(domain):
         return False
 
 
-def pytest_addoption(parser):
-    parser.addoption(
-        "--slow", action="store_true", default=False, help="also run slow tests"
-    )
-
-
 def pytest_configure(config):
     config._benchresults = {}
     config.addinivalue_line(
@@ -36,13 +30,6 @@ def pytest_configure(config):
     )
 
 
-def pytest_runtest_setup(item):
-    markers = list(item.iter_markers(name="slow"))
-    if markers:
-        if not item.config.getoption("--slow"):
-            pytest.skip("skipping slow test, use --slow to run")
-
-
 def _get_chatmail_config():
     inipath = os.environ.get("CHATMAIL_INI")
     if inipath:
@@ -430,9 +417,12 @@ class Remote:
         getjournal = "journalctl -f" if not logcmd else logcmd
         print(self.sshdomain)
         match self.sshdomain:
-            case "@local": command = []
-            case "localhost": command = []
-            case _: command = ["ssh", f"root@{self.sshdomain}"]
+            case "@local":
+                command = []
+            case "localhost":
+                command = []
+            case _:
+                command = ["ssh", f"root@{self.sshdomain}"]
         [command.append(arg) for arg in getjournal.split()]
         popen = subprocess.Popen(
             command,

cmdeploy/src/cmdeploy/tests/test_dovecot_deployer.py

@@ -23,8 +23,7 @@ def make_host(*fact_pairs):
         if cls not in facts:
             registered = ", ".join(c.__name__ for c in facts)
             raise LookupError(
-                f"unexpected get_fact({cls.__name__}); "
-                f"only registered: {registered}"
+                f"unexpected get_fact({cls.__name__}); only registered: {registered}"
             )
         return facts[cls]
 

doc/README.md

@@ -4,12 +4,14 @@
 
 You can use the `make` command and `make html` to build web pages. 
 
-You need a Python environment where the following install was excuted: 
-
-    pip install furo sphinx-autobuild
+You need a Python environment with `sphinx` and other
+dependencies, you can create it by running `scripts/initenv.sh`
+from the repository root.
 
 To develop/change documentation, you can then do: 
 
+    . venv/bin/activate
+    cd doc
     make auto 
 
 A page will open at https://127.0.0.1:8000/ serving the docs and it will 

doc/source/index.rst

@@ -16,5 +16,6 @@ Contributions and feedback welcome through the https://github.com/chatmail/relay
     proxy
     migrate
     overview
+    reverse_dns
     related
     faq

doc/source/overview.rst

@@ -102,14 +102,12 @@ short overview of ``chatmaild`` services:
    Apple/Google/Huawei.
 
 -  `chatmail-expire <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/expire.py>`_
-   deletes entire mailboxes of users who have not logged in
-   for longer than ``delete_inactive_users_after`` days.
+   deletes old messages, large messages, and entire mailboxes
+   of users who have not logged in for longer than
+   ``delete_inactive_users_after`` days.
 
--  `chatmail-quota-expire <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/quota_expire.py>`_
-   is called by Dovecot's ``quota_warning`` mechanism when a
-   user reaches 90% of their mailbox quota.
-   It removes the largest and oldest messages
-   until usage drops below 80% of the quota.
+-  ``chatmail-quota-expire`` is called by Dovecot's ``quota_warning`` mechanism
+   and will automatically remove oldest messages to keep mailboxes well under ``max_mailbox_size``.
 
 -  `lastlogin <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/lastlogin.py>`_
    is contacted by Dovecot when a user logs in and stores the date of
@@ -145,7 +143,7 @@ Chatmail relay dependency diagram
         certs-nginx[("`TLS certs
         /var/lib/acme`")] --> nginx-internal;
         systemd-timer --- acmetool;
-        systemd-timer --- chatmail-expire-inactive;
+        systemd-timer --- chatmail-expire-daily;
         systemd-timer --- chatmail-fsreport-daily;
         acmetool --> certs[("`TLS certs
         /var/lib/acme`")];
@@ -166,7 +164,7 @@ Chatmail relay dependency diagram
         chatmail-quota-expire --- maildir;
         lastlogin --- maildir;
         doveauth --- maildir;
-        chatmail-expire-inactive --- maildir;
+        chatmail-expire-daily --- maildir;
         chatmail-fsreport-daily --- maildir;
         chatmail-metadata --- iroh-relay;
         chatmail-metadata --- |encrypted device token| notifications.delta.chat;

doc/source/reverse_dns.rst

@@ -0,0 +1,64 @@
+Configuring reverse DNS
+=======================
+
+Some email servers reject the emails
+if they don't pass `FCrDNS`_ check, also known as `iprev`_ check.
+
+.. _FCrDNS: https://en.wikipedia.org/wiki/Forward-confirmed_reverse_DNS
+.. _iprev: https://datatracker.ietf.org/doc/html/rfc8601#section-3
+
+Passing the check requires that the IP address that email is sent from
+should have a ``PTR`` record pointing to the domain name of the server,
+and domain name record should have an ``A/AAAA`` record
+pointing to the IP address.
+
+Modern email relies on DKIM and SPF for authentication,
+while iprev check exists for
+`historical reasons <https://datatracker.ietf.org/doc/html/draft-ietf-dnsop-reverse-mapping-considerations-06#section-2.1>`_.
+Chatmail relays don't resolve ``PTR`` records,
+so you can ignore this section if configuring ``PTR`` records
+is difficult and federation with legacy email servers that don't accept
+valid DKIM signature for authentication is not important.
+
+Multi-homed setups
+------------------
+
+If you have a server with multiple IP addresses,
+also known as multi-homed setup,
+and don't publish all IP addresses in DNS,
+you need to make sure you are using
+the published address when making outgoing connections.
+
+For example, your server may have a static IP
+address, and a so-called Floating IP or Virtual IP
+that can be moved between servers in case of
+migration or for failover.
+By using Floating IP you can avoid downtime
+and keep the IP address reputation
+for destinatinons that rely on IP reputation and IP blocklists.
+In this case you will only publish
+the Floating IP to DNS and only use the static IP
+to SSH into the server.
+
+If you have such setup, make sure that
+you not only set ``PTR`` records for the Floating IP,
+but make outgoing connections using the Floating IP.
+Otherwise reverse DNS check succeed,
+but forward check making sure your domain name points
+to the IP address will fail.
+Such setup is indistinguishable from someone
+setting IP address ``PTR`` with the domain they don't own
+and as a result don't succeed.
+
+On Linux you can configure source IP address with ``ip route`` command,
+for example:
+::
+
+  ip route change default via <default-gateway> dev eth0 src <source-address>
+
+Make sure to persist the change after verifying it is working.
+You can check what your outgoing IP address is
+with ``curl icanhazip.com``.
+Check both the IPv4 and IPv6 addresses.
+For IPv4 address use ``curl ipv4.icanhazip.com`` or ``curl -4 icanhazip.com``
+and similarly for IPv6 if you have it.