From df3c460f3840e6c7f10ed6af6fa01db8cb52743a Mon Sep 17 00:00:00 2001 From: holger krekel Date: Sat, 18 Apr 2026 17:08:21 +0200 Subject: [PATCH] feat: automatic oldest-first message removal from mailboxes to always stay under max_mailbox_size Both dovecot-quota-threshold triggers and the daily expiry routine will now expunge oldest messages from mailboxes automatically when the mailbox reaches 75% of max_mailbox_size. Delta Chat users should not see any warnings (at 80/95 percent) or bounce messages, and existing over-quota mailboxes should start receiving mails again. --- chatmaild/pyproject.toml | 3 +- chatmaild/src/chatmaild/config.py | 15 ++++ chatmaild/src/chatmaild/expire.py | 88 ++++++++++++++++++- chatmaild/src/chatmaild/ini/chatmail.ini.f | 1 + chatmaild/src/chatmaild/tests/test_config.py | 16 +++- .../tests/test_delete_inactive_users.py | 2 +- chatmaild/src/chatmaild/tests/test_expire.py | 52 ++++++++++- cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 | 20 ++++- .../src/cmdeploy/mtail/delivered_mail.mtail | 8 ++ doc/source/overview.rst | 10 ++- 10 files changed, 202 insertions(+), 13 deletions(-) diff --git a/chatmaild/pyproject.toml b/chatmaild/pyproject.toml index a29e1094..01ef93d2 100644 --- a/chatmaild/pyproject.toml +++ b/chatmaild/pyproject.toml @@ -21,7 +21,8 @@ where = ['src'] [project.scripts] doveauth = "chatmaild.doveauth:main" chatmail-metadata = "chatmaild.metadata:main" -chatmail-expire = "chatmaild.expire:main" +chatmail-expire = "chatmaild.expire:daily_expire_main" +chatmail-quota-expire = "chatmaild.expire:quota_expire_main" chatmail-fsreport = "chatmaild.fsreport:main" lastlogin = "chatmaild.lastlogin:main" turnserver = "chatmaild.turnserver:main" diff --git a/chatmaild/src/chatmaild/config.py b/chatmaild/src/chatmaild/config.py index 0a7fcb55..43339fb3 100644 --- a/chatmaild/src/chatmaild/config.py +++ b/chatmaild/src/chatmaild/config.py @@ -95,6 +95,11 @@ class Config: # old unused option (except for first migration from sqlite to maildir store) self.passdb_path = Path(params.get("passdb_path", "/home/vmail/passdb.sqlite")) + @property + def max_mailbox_size_mb(self): + """Return max_mailbox_size as an integer in megabytes.""" + return parse_size_mb(self.max_mailbox_size) + def _getbytefile(self): return open(self._inipath, "rb") @@ -108,6 +113,16 @@ class Config: return User(maildir, addr, password_path, uid="vmail", gid="vmail") +def parse_size_mb(limit): + """Parse a size string like ``500M`` or ``2G`` and return megabytes.""" + value = limit.strip().upper().removesuffix("B") + if value.endswith("G"): + return int(value[:-1]) * 1024 + if value.endswith("M"): + return int(value[:-1]) + return int(value) + + def write_initial_config(inipath, mail_domain, overrides): """Write out default config file, using the specified config value overrides.""" content = get_default_config_content(mail_domain, **overrides) diff --git a/chatmaild/src/chatmaild/expire.py b/chatmaild/src/chatmaild/expire.py index 3074073a..d0634aa3 100644 --- a/chatmaild/src/chatmaild/expire.py +++ b/chatmaild/src/chatmaild/expire.py @@ -4,17 +4,26 @@ Expire old messages and addresses. """ import os +import re import shutil import sys import time from argparse import ArgumentParser from collections import namedtuple from datetime import datetime +from pathlib import Path from stat import S_ISREG from chatmaild.config import read_config FileEntry = namedtuple("FileEntry", ("path", "mtime", "size")) +QuotaFileEntry = namedtuple("QuotaFileEntry", ("mtime", "quota_size", "path")) + +# Quota cleanup factor of max_mailbox_size. The mailbox is reset to this size. +QUOTA_CLEANUP_FACTOR = 0.7 + +# e.g. "cur/1775324677.M448978P3029757.nine,S=3235,W=3305:2,S" +_dovecot_fn_rex = re.compile(r".+/(\d+)\..+,S=(\d+)") def iter_mailboxes(basedir, maxnum): @@ -74,6 +83,36 @@ class MailboxStat: self.extrafiles.sort(key=lambda x: -x.size) +def parse_dovecot_filename(relpath): + m = _dovecot_fn_rex.match(relpath) + if not m: + return None + return QuotaFileEntry(int(m.group(1)), int(m.group(2)), relpath) + + +def scan_mailbox_messages(mbox): + messages = [] + for sub in ("cur", "new", "tmp"): + for name in os_listdir_if_exists(mbox / sub): + if entry := parse_dovecot_filename(f"{sub}/{name}"): + messages.append(entry) + return messages + + +def expire_to_target(mbox, target_bytes): + messages = scan_mailbox_messages(mbox) + total_size = sum(m.quota_size for m in messages) + removed = 0 + for entry in sorted(messages): + if total_size <= target_bytes: + break + (mbox / entry.path).unlink(missing_ok=True) + total_size -= entry.quota_size + removed += 1 + + return removed + + def print_info(msg): print(msg, file=sys.stderr) @@ -143,6 +182,19 @@ class Expiry: else: continue changed = True + + target_bytes = ( + self.config.max_mailbox_size_mb * 1024 * 1024 * QUOTA_CLEANUP_FACTOR + ) + removed = expire_to_target(Path(mbox.basedir), target_bytes) + if removed: + changed = True + self.del_files += removed + if self.verbose: + print_info( + f"quota-expire: removed {removed} message(s) from {mboxname}" + ) + if changed: self.remove_file(f"{mbox.basedir}/maildirsize") @@ -154,9 +206,9 @@ class Expiry: ) -def main(args=None): +def daily_expire_main(args=None): """Expire mailboxes and messages according to chatmail config""" - parser = ArgumentParser(description=main.__doc__) + parser = ArgumentParser(description=daily_expire_main.__doc__) ini = "/usr/local/lib/chatmaild/chatmail.ini" parser.add_argument( "chatmail_ini", @@ -202,5 +254,33 @@ def main(args=None): print(exp.get_summary()) -if __name__ == "__main__": - main(sys.argv[1:]) +def quota_expire_main(args=None): + """Remove mailbox messages to stay within a megabyte target. + + This entry point is called by dovecot when a quota threshold is passed. + """ + + parser = ArgumentParser(description=quota_expire_main.__doc__) + parser.add_argument( + "target_mb", + type=int, + help="target mailbox size in megabytes", + ) + parser.add_argument( + "mailbox_path", + type=Path, + help="path to a user mailbox", + ) + args = parser.parse_args(args) + + target_bytes = args.target_mb * 1024 * 1024 + + removed_count = expire_to_target(args.mailbox_path, target_bytes) + if removed_count: + (args.mailbox_path / "maildirsize").unlink(missing_ok=True) + print( + f"quota-expire: removed {removed_count} message(s)" + f" from {args.mailbox_path.name}", + file=sys.stderr, + ) + return 0 diff --git a/chatmaild/src/chatmaild/ini/chatmail.ini.f b/chatmaild/src/chatmaild/ini/chatmail.ini.f index 353a9669..5d3cbe0d 100644 --- a/chatmaild/src/chatmaild/ini/chatmail.ini.f +++ b/chatmaild/src/chatmaild/ini/chatmail.ini.f @@ -18,6 +18,7 @@ max_user_send_per_minute = 60 max_user_send_burst_size = 10 # maximum mailbox size of a chatmail address +# Oldest messages will be removed automatically, so mailboxes never run full. max_mailbox_size = 500M # maximum message size for an e-mail in bytes diff --git a/chatmaild/src/chatmaild/tests/test_config.py b/chatmaild/src/chatmaild/tests/test_config.py index b459ec6c..bccea318 100644 --- a/chatmaild/src/chatmaild/tests/test_config.py +++ b/chatmaild/src/chatmaild/tests/test_config.py @@ -1,6 +1,6 @@ import pytest -from chatmaild.config import read_config +from chatmaild.config import parse_size_mb, read_config def test_read_config_basic(example_config): @@ -121,3 +121,17 @@ def test_config_tls_external_bad_format(make_config): "tls_external_cert_and_key": "/only/one/path.pem", }, ) + + +def test_parse_size_mb(): + assert parse_size_mb("500M") == 500 + assert parse_size_mb("2G") == 2048 + assert parse_size_mb(" 1g ") == 1024 + assert parse_size_mb("100MB") == 100 + assert parse_size_mb("256") == 256 + + +def test_max_mailbox_size_mb(make_config): + config = make_config("chat.example.org") + assert config.max_mailbox_size == "500M" + assert config.max_mailbox_size_mb == 500 diff --git a/chatmaild/src/chatmaild/tests/test_delete_inactive_users.py b/chatmaild/src/chatmaild/tests/test_delete_inactive_users.py index 5e662e4e..6a60362a 100644 --- a/chatmaild/src/chatmaild/tests/test_delete_inactive_users.py +++ b/chatmaild/src/chatmaild/tests/test_delete_inactive_users.py @@ -1,7 +1,7 @@ import time from chatmaild.doveauth import AuthDictProxy -from chatmaild.expire import main as main_expire +from chatmaild.expire import daily_expire_main as main_expire def test_login_timestamps(example_config): diff --git a/chatmaild/src/chatmaild/tests/test_expire.py b/chatmaild/src/chatmaild/tests/test_expire.py index 70355ffb..e53c8798 100644 --- a/chatmaild/src/chatmaild/tests/test_expire.py +++ b/chatmaild/src/chatmaild/tests/test_expire.py @@ -1,5 +1,6 @@ import os import random +import time from datetime import datetime from fnmatch import fnmatch from pathlib import Path @@ -9,13 +10,19 @@ import pytest from chatmaild.expire import ( FileEntry, MailboxStat, + expire_to_target, get_file_entry, iter_mailboxes, os_listdir_if_exists, + parse_dovecot_filename, + quota_expire_main, + scan_mailbox_messages, ) -from chatmaild.expire import main as expiry_main +from chatmaild.expire import daily_expire_main as expiry_main from chatmaild.fsreport import main as report_main +MB = 1024 * 1024 + def fill_mbox(folderdir): password = folderdir.joinpath("password") @@ -196,3 +203,46 @@ def test_os_listdir_if_exists(tmp_path): tmp_path.joinpath("x").write_text("hello") assert len(os_listdir_if_exists(str(tmp_path))) == 1 assert len(os_listdir_if_exists(str(tmp_path.joinpath("123123")))) == 0 + + +# --- quota expire tests --- + + +def _create_message(basedir, sub, size, days_old=0, disk_size=None): + mtime = int(time.time() - days_old * 86400) + name = f"{mtime}.M1P1.host,S={size},W={size}:2,S" + path = basedir / sub / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"x" * (disk_size if disk_size is not None else size)) + os.utime(path, (mtime, mtime)) + return path + + +def test_parse_dovecot_filename(): + e = parse_dovecot_filename("cur/1775324677.M448978P3029757.nine,S=3235,W=3305:2,S") + assert e.path == "cur/1775324677.M448978P3029757.nine,S=3235,W=3305:2,S" + assert e.mtime == 1775324677 + assert e.quota_size == 3235 + assert parse_dovecot_filename("cur/msg_without_structure") is None + + +def test_expire_to_target(tmp_path): + # scans cur, new, tmp + _create_message(tmp_path, "cur", MB, days_old=10, disk_size=100) + _create_message(tmp_path, "new", MB, days_old=5) + _create_message(tmp_path, "tmp", MB, days_old=1) + assert len(scan_mailbox_messages(tmp_path)) == 3 + # removes oldest first, uses S= size not disk size + removed = expire_to_target(tmp_path, 2 * MB) + assert removed == 1 + assert len(scan_mailbox_messages(tmp_path)) == 2 + + +def test_quota_expire_main(tmp_path, capsys): + mbox = tmp_path / "user@example.org" + _create_message(mbox, "cur", 2 * MB, days_old=5) + (mbox / "maildirsize").write_text("x") + quota_expire_main([str(1), str(mbox)]) + _, err = capsys.readouterr() + assert "quota-expire: removed 1 message(s) from user@example.org" in err + assert not (mbox / "maildirsize").exists() diff --git a/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 b/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 index bd1ce053..036bd79f 100644 --- a/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 +++ b/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 @@ -149,12 +149,26 @@ plugin { } plugin { - # for now we define static quota-rules for all users quota = maildir:User quota - quota_rule = *:storage={{ config.max_mailbox_size }} quota_max_mail_size={{ config.max_message_size }} quota_grace = 0 - # quota_over_flag_value = TRUE + + quota_rule = *:storage={{ config.max_mailbox_size_mb }}M + + # Trigger at 75%% of quota, expire oldest messages down to 70%%. + # The percentages are chosen to prevent current Delta Chat users + # from seeing "quota warnings" which trigger at 80% and 95%. + + quota_warning = storage=75%% quota-warning {{ config.max_mailbox_size_mb * 70 // 100 }} {{ config.mailboxes_dir }}/%u +} + +service quota-warning { + executable = script /usr/local/lib/chatmaild/venv/bin/chatmail-quota-expire + user = vmail + unix_listener quota-warning { + user = vmail + mode = 0600 + } } # push_notification configuration diff --git a/cmdeploy/src/cmdeploy/mtail/delivered_mail.mtail b/cmdeploy/src/cmdeploy/mtail/delivered_mail.mtail index 21280c79..58de2e23 100644 --- a/cmdeploy/src/cmdeploy/mtail/delivered_mail.mtail +++ b/cmdeploy/src/cmdeploy/mtail/delivered_mail.mtail @@ -78,3 +78,11 @@ counter rejected_unencrypted_mail_count /Rejected unencrypted mail/ { rejected_unencrypted_mail_count++ } + +counter quota_expire_runs +counter quota_expire_removed_files + +/quota-expire: removed (?P\d+) message\(s\)/ { + quota_expire_runs++ + quota_expire_removed_files += $count +} diff --git a/doc/source/overview.rst b/doc/source/overview.rst index fd2c9401..8c1608c2 100644 --- a/doc/source/overview.rst +++ b/doc/source/overview.rst @@ -102,8 +102,12 @@ short overview of ``chatmaild`` services: Apple/Google/Huawei. - `chatmail-expire `_ - deletes users if they have not logged in for a longer while. - The timeframe can be configured in ``chatmail.ini``. + deletes old messages, large messages, and entire mailboxes + of users who have not logged in for longer than + ``delete_inactive_users_after`` days. + +- ``chatmail-quota-expire`` is called by Dovecot's ``quota_warning`` mechanism + and will automatically remove oldest messages to keep mailboxes well under ``max_mailbox_size``. - `lastlogin `_ is contacted by Dovecot when a user logs in and stores the date of @@ -156,6 +160,8 @@ Chatmail relay dependency diagram /home/vmail/.../user"]; dovecot --- |lastlogin.socket|lastlogin; dovecot --- chatmail-metadata; + dovecot --- |quota-warning|chatmail-quota-expire; + chatmail-quota-expire --- maildir; lastlogin --- maildir; doveauth --- maildir; chatmail-expire-daily --- maildir;