From 4512d1e735e21080f7debb2ef33703681ee079bc Mon Sep 17 00:00:00 2001 From: holger krekel Date: Sat, 18 Apr 2026 17:08:21 +0200 Subject: [PATCH] feat: add quota-triggered per-user mailbox cleanup Inflate the Dovecot-visible quota to 140% of the configured max_mailbox_size so that Delta Chat clients (which warn at 80% of IMAP-reported quota) never show quota warnings. A quota_warning at 72% of the inflated limit triggers chatmail-quota-expire, which trims the mailbox to 80% of the configured limit. Existing over-quota mailboxes start receiving mail again immediately after deploy without any manual operator action needed. --- CHANGELOG.md | 15 ++++ chatmaild/pyproject.toml | 1 + chatmaild/src/chatmaild/config.py | 15 ++++ chatmaild/src/chatmaild/quota_expire.py | 90 +++++++++++++++++++ chatmaild/src/chatmaild/tests/test_config.py | 16 +++- .../src/chatmaild/tests/test_quota_expire.py | 73 +++++++++++++++ cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 | 20 ++++- .../src/cmdeploy/mtail/delivered_mail.mtail | 8 ++ .../cmdeploy/tests/online/test_2_deltachat.py | 5 +- doc/source/overview.rst | 13 ++- 10 files changed, 249 insertions(+), 7 deletions(-) create mode 100644 chatmaild/src/chatmaild/quota_expire.py create mode 100644 chatmaild/src/chatmaild/tests/test_quota_expire.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 91ba8794..a77b5066 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog for chatmail deployment +## Unreleased + +### Features + +- Add per-user quota-triggered cleanup (`chatmail-quota-expire`). + When a mailbox exceeds the configured ``max_mailbox_size``, + Dovecot runs the new script which removes the oldest + messages until usage drops to a safe level. + No operator action is required after upgrading; + existing over-quota mailboxes start receiving mail + again immediately and are cleaned up automatically. + The daily `chatmail-expire` timer continues to handle + deletion of old messages, large messages, + and inactive user mailboxes. + ## 1.9.0 2025-12-18 ### Documentation diff --git a/chatmaild/pyproject.toml b/chatmaild/pyproject.toml index a29e1094..3a4167e9 100644 --- a/chatmaild/pyproject.toml +++ b/chatmaild/pyproject.toml @@ -22,6 +22,7 @@ where = ['src'] doveauth = "chatmaild.doveauth:main" chatmail-metadata = "chatmaild.metadata:main" chatmail-expire = "chatmaild.expire:main" +chatmail-quota-expire = "chatmaild.quota_expire:main" chatmail-fsreport = "chatmaild.fsreport:main" lastlogin = "chatmaild.lastlogin:main" turnserver = "chatmaild.turnserver:main" diff --git a/chatmaild/src/chatmaild/config.py b/chatmaild/src/chatmaild/config.py index 0a7fcb55..2b869267 100644 --- a/chatmaild/src/chatmaild/config.py +++ b/chatmaild/src/chatmaild/config.py @@ -95,6 +95,11 @@ class Config: # old unused option (except for first migration from sqlite to maildir store) self.passdb_path = Path(params.get("passdb_path", "/home/vmail/passdb.sqlite")) + @property + def max_mailbox_size_mb(self): + """Return max_mailbox_size as an integer in megabytes.""" + return parse_size_mb(self.max_mailbox_size) + def _getbytefile(self): return open(self._inipath, "rb") @@ -108,6 +113,16 @@ class Config: return User(maildir, addr, password_path, uid="vmail", gid="vmail") +def parse_size_mb(limit): + """Parse a size string like ``500M`` or ``2G`` and return megabytes.""" + value = limit.strip().upper().rstrip("B") + if value.endswith("G"): + return int(value[:-1]) * 1024 + if value.endswith("M"): + return int(value[:-1]) + return int(value) + + def write_initial_config(inipath, mail_domain, overrides): """Write out default config file, using the specified config value overrides.""" content = get_default_config_content(mail_domain, **overrides) diff --git a/chatmaild/src/chatmaild/quota_expire.py b/chatmaild/src/chatmaild/quota_expire.py new file mode 100644 index 00000000..3b935e54 --- /dev/null +++ b/chatmaild/src/chatmaild/quota_expire.py @@ -0,0 +1,90 @@ +""" +Quota-triggered per-user mailbox cleanup. + +Dovecot calls this script via ``quota_warning`` +when a user crosses the quota threshold. +The script removes oldest messages first +to keep the mailbox under a specified target size. + +Usage:: + + chatmail-quota-expire + +""" + +import sys +from argparse import ArgumentParser +from pathlib import Path + +from chatmaild.expire import get_file_entry, os_listdir_if_exists + + +def scan_mailbox_messages(mailbox_dir): + """Collect FileEntry items from top-level cur/new/tmp only.""" + mbox = Path(mailbox_dir) + messages = [] + for sub in ("cur", "new", "tmp"): + for name in os_listdir_if_exists(mbox / sub): + if entry := get_file_entry(str(mbox / sub / name)): + messages.append(entry) + return messages + + +def expire_to_target(mailbox_dir, target_bytes): + """Remove oldest files until total size <= *target_bytes*. + + Returns ``(removed_count, cache_bytes)`` where *cache_bytes* + is the size of the deleted ``dovecot.index.cache`` file + (0 when the file did not exist). + """ + mbox = Path(mailbox_dir) + messages = scan_mailbox_messages(mbox) + total_size = sum(m.size for m in messages) + removed = 0 + for count, entry in enumerate(sorted(messages, key=lambda m: m.mtime), 1): + if total_size <= target_bytes: + break + Path(entry.path).unlink(missing_ok=True) + total_size -= entry.size + removed = count + + (mbox / "maildirsize").unlink(missing_ok=True) + cache = mbox / "dovecot.index.cache" + try: + cache_bytes = cache.stat().st_size + except FileNotFoundError: + cache_bytes = 0 + cache.unlink(missing_ok=True) + return removed, cache_bytes + + +def main(args=None): + """Remove mailbox messages to stay within a megabyte target.""" + parser = ArgumentParser(description=main.__doc__) + parser.add_argument( + "target_mb", + type=int, + help="target mailbox size in megabytes", + ) + parser.add_argument( + "mailbox_path", + help="path to a user mailbox", + ) + args = parser.parse_args(args) + + target_bytes = args.target_mb * 1024 * 1024 + + removed_count, cache_bytes = expire_to_target(args.mailbox_path, target_bytes) + if removed_count: + user = Path(args.mailbox_path).name + cache_mb = cache_bytes / 1024 / 1024 + print( + f"quota-expire: removed {removed_count} message(s) from {user}" + f" cache={cache_mb:.1f}MB", + file=sys.stderr, + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/chatmaild/src/chatmaild/tests/test_config.py b/chatmaild/src/chatmaild/tests/test_config.py index b459ec6c..bccea318 100644 --- a/chatmaild/src/chatmaild/tests/test_config.py +++ b/chatmaild/src/chatmaild/tests/test_config.py @@ -1,6 +1,6 @@ import pytest -from chatmaild.config import read_config +from chatmaild.config import parse_size_mb, read_config def test_read_config_basic(example_config): @@ -121,3 +121,17 @@ def test_config_tls_external_bad_format(make_config): "tls_external_cert_and_key": "/only/one/path.pem", }, ) + + +def test_parse_size_mb(): + assert parse_size_mb("500M") == 500 + assert parse_size_mb("2G") == 2048 + assert parse_size_mb(" 1g ") == 1024 + assert parse_size_mb("100MB") == 100 + assert parse_size_mb("256") == 256 + + +def test_max_mailbox_size_mb(make_config): + config = make_config("chat.example.org") + assert config.max_mailbox_size == "500M" + assert config.max_mailbox_size_mb == 500 diff --git a/chatmaild/src/chatmaild/tests/test_quota_expire.py b/chatmaild/src/chatmaild/tests/test_quota_expire.py new file mode 100644 index 00000000..8f946c88 --- /dev/null +++ b/chatmaild/src/chatmaild/tests/test_quota_expire.py @@ -0,0 +1,73 @@ +import os +import time + +from chatmaild.quota_expire import expire_to_target, main, scan_mailbox_messages + +MB = 1024 * 1024 + + +def _create_message(basedir, relpath, size, days_old=0): + path = basedir / relpath + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"x" * size) + mtime = time.time() - days_old * 86400 + os.utime(path, (mtime, mtime)) + return path + + +def test_scan_cur_new_tmp(tmp_path): + _create_message(tmp_path, "cur/msg1", 100) + _create_message(tmp_path, "new/msg2", 200) + _create_message(tmp_path, "tmp/msg3", 300) + assert len(scan_mailbox_messages(str(tmp_path))) == 3 + + +def test_scan_ignores_subfolders(tmp_path): + _create_message(tmp_path, "cur/a", 10) + _create_message(tmp_path, ".DeltaChat/cur/b", 20) + assert len(scan_mailbox_messages(str(tmp_path))) == 1 + + +def test_removes_to_target(tmp_path): + for i in range(15): + _create_message(tmp_path, f"cur/msg{i:02d}", MB, days_old=i + 1) + removed, _ = expire_to_target(str(tmp_path), 10 * MB) + assert removed == 5 + assert len(scan_mailbox_messages(str(tmp_path))) == 10 + + +def test_removes_oldest_first(tmp_path): + _create_message(tmp_path, "cur/old_small", MB, days_old=30) + _create_message(tmp_path, "cur/new_huge", 10 * MB, days_old=1) + # the 10MB file is kept, the 1MB file is removed because it's older + removed, _ = expire_to_target(str(tmp_path), 10 * MB) + assert removed == 1 + assert not (tmp_path / "cur/old_small").exists() + assert (tmp_path / "cur/new_huge").exists() + + +def test_exact_limit(tmp_path): + _create_message(tmp_path, "cur/msg1", 5 * MB) + removed, _ = expire_to_target(str(tmp_path), 5 * MB) + assert removed == 0 + + +def test_removes_stale_caches(tmp_path): + _create_message(tmp_path, "cur/msg1", 2 * MB, days_old=5) + (tmp_path / "maildirsize").write_text("x") + (tmp_path / "dovecot.index.cache").write_bytes(b"y" * 4096) + removed, cache_bytes = expire_to_target(str(tmp_path), MB) + assert removed == 1 + assert cache_bytes == 4096 + assert not (tmp_path / "maildirsize").exists() + assert not (tmp_path / "dovecot.index.cache").exists() + + +def test_logging_output_is_mtail_compatible(tmp_path, capsys): + mbox = tmp_path / "user@example.org" + _create_message(mbox, "cur/msg1", 2 * MB, days_old=5) + (mbox / "dovecot.index.cache").write_bytes(b"c" * 2 * MB) + main([str(1), str(mbox)]) + _, err = capsys.readouterr() + assert "quota-expire: removed 1 message(s) from user@example.org" in err + assert "cache=2.0MB" in err diff --git a/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 b/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 index bd1ce053..55a84390 100644 --- a/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 +++ b/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 @@ -149,12 +149,26 @@ plugin { } plugin { - # for now we define static quota-rules for all users quota = maildir:User quota - quota_rule = *:storage={{ config.max_mailbox_size }} quota_max_mail_size={{ config.max_message_size }} quota_grace = 0 - # quota_over_flag_value = TRUE + + # Inflate the dovecot-visible quota so that Delta Chat clients + # (which warn at 80% of the IMAP-reported limit) never see + # quota warnings -- expire kicks in well before that point. + quota_rule = *:storage={{ config.max_mailbox_size_mb * 140 // 100 }}M + + # Trigger when usage reaches the configured max_mailbox_size + # (72% of inflated = ~100% of configured), then expire oldest + # messages down to 80% of the configured max_mailbox_size. + quota_warning = storage=72%% quota-warning {{ config.max_mailbox_size_mb * 80 // 100 }} {{ config.mailboxes_dir }}/%u +} + +service quota-warning { + executable = script /usr/local/lib/chatmaild/venv/bin/chatmail-quota-expire + user = vmail + unix_listener quota-warning { + } } # push_notification configuration diff --git a/cmdeploy/src/cmdeploy/mtail/delivered_mail.mtail b/cmdeploy/src/cmdeploy/mtail/delivered_mail.mtail index 21280c79..58de2e23 100644 --- a/cmdeploy/src/cmdeploy/mtail/delivered_mail.mtail +++ b/cmdeploy/src/cmdeploy/mtail/delivered_mail.mtail @@ -78,3 +78,11 @@ counter rejected_unencrypted_mail_count /Rejected unencrypted mail/ { rejected_unencrypted_mail_count++ } + +counter quota_expire_runs +counter quota_expire_removed_files + +/quota-expire: removed (?P\d+) message\(s\)/ { + quota_expire_runs++ + quota_expire_removed_files += $count +} diff --git a/cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py b/cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py index 947c34f9..b5fd6355 100644 --- a/cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py +++ b/cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py @@ -88,9 +88,12 @@ class TestEndToEndDeltaChat: return int(float(number) * units[unit]) quota = parse_size_limit(chatmail_config.max_mailbox_size) + # Dovecot quota is inflated to 140% of the configured limit + # so that quota-expire keeps users below the warning threshold. + dovecot_quota = quota * 140 // 100 lp.sec(f"filling remote inbox for {user}") - fn = f"7743102289.M843172P2484002.c20,S={quota},W=2398:2," + fn = f"7743102289.M843172P2484002.c20,S={dovecot_quota},W=2398:2," path = chatmail_config.mailboxes_dir.joinpath(user, "cur", fn) sshexec = get_sshexec(sshdomain) sshexec(call=rshell.write_numbytes, kwargs=dict(path=str(path), num=120)) diff --git a/doc/source/overview.rst b/doc/source/overview.rst index fd2c9401..4c747c6f 100644 --- a/doc/source/overview.rst +++ b/doc/source/overview.rst @@ -102,8 +102,15 @@ short overview of ``chatmaild`` services: Apple/Google/Huawei. - `chatmail-expire `_ - deletes users if they have not logged in for a longer while. - The timeframe can be configured in ``chatmail.ini``. + deletes old messages, large messages, and entire mailboxes + of users who have not logged in for longer than + ``delete_inactive_users_after`` days. + +- ``chatmail-quota-expire`` + is called by Dovecot's ``quota_warning`` mechanism when a + mailbox exceeds ``max_mailbox_size``. + It removes the oldest messages + until usage drops to a safe level. - `lastlogin `_ is contacted by Dovecot when a user logs in and stores the date of @@ -156,6 +163,8 @@ Chatmail relay dependency diagram /home/vmail/.../user"]; dovecot --- |lastlogin.socket|lastlogin; dovecot --- chatmail-metadata; + dovecot --- |quota-warning|chatmail-quota-expire; + chatmail-quota-expire --- maildir; lastlogin --- maildir; doveauth --- maildir; chatmail-expire-daily --- maildir;