diff --git a/CHANGELOG.md b/CHANGELOG.md index 91ba8794..a1ebcc9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Changelog for chatmail deployment +## Unreleased + +### Features + +- Automated per-user quota-keeping. + Replace daily timer-based message expire script + with Dovecot quota-warning-triggered cleanup (`chatmail-quota-expire`). + When a user reaches 90% of their mailbox quota + Dovecot calls the new script which removes the largest and oldest messages + until usage drops below 80%. + The daily `chatmail-expire` timer now only handles deletion + of inactive user mailboxes. + + After upgrading, run the following once to clean up + mailboxes that are already over quota:: + + /usr/local/lib/chatmaild/venv/bin/chatmail-quota-expire \ + 400 /home/vmail/mail/YOURDOMAIN --sweep + ## 1.9.0 2025-12-18 ### Documentation diff --git a/chatmaild/pyproject.toml b/chatmaild/pyproject.toml index a29e1094..9c9c2100 100644 --- a/chatmaild/pyproject.toml +++ b/chatmaild/pyproject.toml @@ -21,7 +21,8 @@ where = ['src'] [project.scripts] doveauth = "chatmaild.doveauth:main" chatmail-metadata = "chatmaild.metadata:main" -chatmail-expire = "chatmaild.expire:main" +chatmail-expire = "chatmaild.expire_inactive_users:main" +chatmail-quota-expire = "chatmaild.quota_expire:main" chatmail-fsreport = "chatmaild.fsreport:main" lastlogin = "chatmaild.lastlogin:main" turnserver = "chatmaild.turnserver:main" diff --git a/chatmaild/src/chatmaild/config.py b/chatmaild/src/chatmaild/config.py index 0a7fcb55..a07de18f 100644 --- a/chatmaild/src/chatmaild/config.py +++ b/chatmaild/src/chatmaild/config.py @@ -25,8 +25,6 @@ class Config: self.max_user_send_burst_size = int(params.get("max_user_send_burst_size", 10)) self.max_mailbox_size = params["max_mailbox_size"] self.max_message_size = int(params.get("max_message_size", "31457280")) - self.delete_mails_after = params["delete_mails_after"] - self.delete_large_after = params["delete_large_after"] self.delete_inactive_users_after = int(params["delete_inactive_users_after"]) self.username_min_length = int(params["username_min_length"]) self.username_max_length = int(params["username_max_length"]) @@ -95,6 +93,11 @@ class Config: # old unused option (except for first migration from sqlite to maildir store) self.passdb_path = Path(params.get("passdb_path", "/home/vmail/passdb.sqlite")) + @property + def max_mailbox_size_mb(self): + """Return max_mailbox_size as an integer in megabytes.""" + return parse_size_mb(self.max_mailbox_size) + def _getbytefile(self): return open(self._inipath, "rb") @@ -108,6 +111,16 @@ class Config: return User(maildir, addr, password_path, uid="vmail", gid="vmail") +def parse_size_mb(limit): + """Parse a size string like ``500M`` or ``2G`` and return megabytes.""" + value = limit.strip().upper().rstrip("B") + if value.endswith("G"): + return int(value[:-1]) * 1024 + if value.endswith("M"): + return int(value[:-1]) + return int(value) + + def write_initial_config(inipath, mail_domain, overrides): """Write out default config file, using the specified config value overrides.""" content = get_default_config_content(mail_domain, **overrides) diff --git a/chatmaild/src/chatmaild/expire.py b/chatmaild/src/chatmaild/expire.py index 3074073a..63424a8a 100644 --- a/chatmaild/src/chatmaild/expire.py +++ b/chatmaild/src/chatmaild/expire.py @@ -115,11 +115,8 @@ class Expiry: cutoff_without_login = ( self.now - int(self.config.delete_inactive_users_after) * 86400 ) - cutoff_mails = self.now - int(self.config.delete_mails_after) * 86400 - cutoff_large_mails = self.now - int(self.config.delete_large_after) * 86400 self.all_mboxes += 1 - changed = False if mbox.last_login and mbox.last_login < cutoff_without_login: self.remove_mailbox(mbox.basedir) return @@ -131,25 +128,10 @@ class Expiry: print_info(f"checking mailbox {date.strftime('%b %d')} {mboxname}") else: print_info(f"checking mailbox (no last_login) {mboxname}") - self.all_files += len(mbox.messages) - for message in mbox.messages: - if message.mtime < cutoff_mails: - self.remove_file(message.path, mtime=message.mtime) - elif message.size > 200000 and message.mtime < cutoff_large_mails: - # we only remove noticed large files (not unnoticed ones in new/) - parts = message.path.split("/") - if len(parts) >= 2 and parts[-2] == "cur": - self.remove_file(message.path, mtime=message.mtime) - else: - continue - changed = True - if changed: - self.remove_file(f"{mbox.basedir}/maildirsize") def get_summary(self): return ( f"Removed {self.del_mboxes} out of {self.all_mboxes} mailboxes " - f"and {self.del_files} out of {self.all_files} files in existing mailboxes " f"in {time.time() - self.start:2.2f} seconds" ) diff --git a/chatmaild/src/chatmaild/ini/chatmail.ini.f b/chatmaild/src/chatmaild/ini/chatmail.ini.f index 353a9669..f1f83a96 100644 --- a/chatmaild/src/chatmaild/ini/chatmail.ini.f +++ b/chatmaild/src/chatmaild/ini/chatmail.ini.f @@ -23,12 +23,6 @@ max_mailbox_size = 500M # maximum message size for an e-mail in bytes max_message_size = 31457280 -# days after which mails are unconditionally deleted -delete_mails_after = 20 - -# days after which large messages (>200k) are unconditionally deleted -delete_large_after = 7 - # days after which users without a successful login are deleted (database and mails) delete_inactive_users_after = 90 diff --git a/chatmaild/src/chatmaild/quota_expire.py b/chatmaild/src/chatmaild/quota_expire.py new file mode 100644 index 00000000..2976390e --- /dev/null +++ b/chatmaild/src/chatmaild/quota_expire.py @@ -0,0 +1,152 @@ +""" +Remove messages from a mailbox to meet a size target. + +Dovecot calls this script when a user's quota is near its limit. +Files are scored by ``size * age`` so that large, old messages +are removed first. + +Usage:: + + quota_expire + +""" + +import os +import sys +import time +from argparse import ArgumentParser +from collections import namedtuple +from stat import S_ISREG + +FileEntry = namedtuple("FileEntry", ("path", "mtime", "size")) + + +def _get_file_entry(path): + try: + st = os.stat(path) + except FileNotFoundError: + return None + if not S_ISREG(st.st_mode): + return None + return FileEntry(path, st.st_mtime, st.st_size) + + +def _listdir(path): + try: + return os.listdir(path) + except FileNotFoundError: + return [] + + +def scan_mailbox_messages(mailbox_dir): + messages = [] + for sub in ("cur", "new", "tmp"): + subdir = f"{mailbox_dir}/{sub}" + for name in _listdir(subdir): + entry = _get_file_entry(f"{subdir}/{name}") + if entry is not None: + messages.append(entry) + return messages + + +def _remove_stale_caches(mailbox_dir): + for name in ("maildirsize", "dovecot.index.cache"): + try: + os.unlink(f"{mailbox_dir}/{name}") + except FileNotFoundError: + pass + + +def expire_to_target(mailbox_dir, target_bytes, now=None): + """Remove highest-scored files until total size <= *target_bytes*. + + Returns the list of removed file paths. + """ + if now is None: + now = time.time() + + messages = scan_mailbox_messages(mailbox_dir) + total_size = sum(m.size for m in messages) + + if total_size <= target_bytes: + return [] + + # Score: large and old files get the highest score. + scored = sorted( + messages, + key=lambda m: m.size * (now - m.mtime), + reverse=True, + ) + + removed = [] + for entry in scored: + if total_size <= target_bytes: + break + try: + os.unlink(entry.path) + except FileNotFoundError: + continue + total_size -= entry.size + removed.append(entry.path) + + if removed: + _remove_stale_caches(mailbox_dir) + + return removed + + +def main(args=None): + """Remove mailbox messages to stay within a megabyte target.""" + parser = ArgumentParser(description=main.__doc__) + parser.add_argument( + "target_mb", + type=int, + help="target mailbox size in megabytes", + ) + parser.add_argument( + "mailbox_path", + help="path to a user mailbox, or with --sweep the mailboxes directory", + ) + parser.add_argument( + "--sweep", + action="store_true", + help="sweep all mailboxes under mailbox_path", + ) + args = parser.parse_args(args) + + target_bytes = args.target_mb * 1024 * 1024 + + if args.sweep: + return _sweep(args.mailbox_path, target_bytes) + + removed = expire_to_target(args.mailbox_path, target_bytes) + if removed: + print( + f"removed {len(removed)} file(s) from {args.mailbox_path}" + f" to reach {args.target_mb} MB target", + file=sys.stderr, + ) + return 0 + + +def _sweep(mailboxes_dir, target_bytes): + try: + names = os.listdir(mailboxes_dir) + except FileNotFoundError: + print(f"directory not found: {mailboxes_dir}", file=sys.stderr) + return 1 + for name in sorted(names): + if "@" not in name: + continue + mbox = f"{mailboxes_dir}/{name}" + removed = expire_to_target(mbox, target_bytes) + if removed: + print( + f"removed {len(removed)} file(s) from {name}", + file=sys.stderr, + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/chatmaild/src/chatmaild/tests/test_config.py b/chatmaild/src/chatmaild/tests/test_config.py index b459ec6c..5602b9b1 100644 --- a/chatmaild/src/chatmaild/tests/test_config.py +++ b/chatmaild/src/chatmaild/tests/test_config.py @@ -34,8 +34,6 @@ def test_read_config_testrun(make_config): assert config.postfix_reinject_port == 10025 assert config.max_user_send_per_minute == 60 assert config.max_mailbox_size == "500M" - assert config.delete_mails_after == "20" - assert config.delete_large_after == "7" assert config.username_min_length == 9 assert config.username_max_length == 9 assert config.password_min_length == 9 diff --git a/chatmaild/src/chatmaild/tests/test_expire.py b/chatmaild/src/chatmaild/tests/test_expire.py index 70355ffb..697f3ffc 100644 --- a/chatmaild/src/chatmaild/tests/test_expire.py +++ b/chatmaild/src/chatmaild/tests/test_expire.py @@ -1,7 +1,6 @@ import os import random from datetime import datetime -from fnmatch import fnmatch from pathlib import Path import pytest @@ -154,35 +153,6 @@ def test_expiry_cli_basic(example_config, mbox1): expiry_main(args) -def test_expiry_cli_old_files(capsys, example_config, mbox1): - relpaths_old = ["cur/msg_old1", "cur/msg_old1"] - cutoff_days = int(example_config.delete_mails_after) + 1 - create_new_messages(mbox1.basedir, relpaths_old, size=1000, days=cutoff_days) - - relpaths_large = ["cur/msg_old_large1", "new/msg_old_large2"] - cutoff_days = int(example_config.delete_large_after) + 1 - create_new_messages( - mbox1.basedir, relpaths_large, size=1000 * 300, days=cutoff_days - ) - - create_new_messages(mbox1.basedir, ["cur/shouldstay"], size=1000 * 300, days=1) - - args = str(example_config._inipath), "--remove", "-v" - expiry_main(args) - out, err = capsys.readouterr() - - allpaths = relpaths_old + relpaths_large + ["maildirsize"] - for path in allpaths: - for line in err.split("\n"): - if fnmatch(line, f"removing*{path}"): - break - else: - if path != "new/msg_old_large2": - pytest.fail(f"failed to remove {path}\n{err}") - - assert "shouldstay" not in err - - def test_get_file_entry(tmp_path): assert get_file_entry(str(tmp_path.joinpath("123123"))) is None p = tmp_path.joinpath("x") diff --git a/chatmaild/src/chatmaild/tests/test_quota_expire.py b/chatmaild/src/chatmaild/tests/test_quota_expire.py new file mode 100644 index 00000000..9fbf06db --- /dev/null +++ b/chatmaild/src/chatmaild/tests/test_quota_expire.py @@ -0,0 +1,91 @@ +import os +import time + +from chatmaild.quota_expire import expire_to_target, scan_mailbox_messages + +MB = 1024 * 1024 + + +def _create_message(basedir, relpath, size, days_old=0): + path = basedir / relpath + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"x" * size) + mtime = time.time() - days_old * 86400 + os.utime(path, (mtime, mtime)) + return path + + +def test_scan_cur_new_tmp(tmp_path): + _create_message(tmp_path, "cur/msg1", 100) + _create_message(tmp_path, "new/msg2", 200) + _create_message(tmp_path, "tmp/msg3", 300) + messages = scan_mailbox_messages(str(tmp_path)) + assert len(messages) == 3 + sizes = sorted(m.size for m in messages) + assert sizes == [100, 200, 300] + + +def test_scan_ignores_subfolders(tmp_path): + _create_message(tmp_path, "cur/a", 10) + _create_message(tmp_path, ".DeltaChat/cur/b", 20) + assert len(scan_mailbox_messages(str(tmp_path))) == 1 + + +def test_scan_empty(tmp_path): + assert scan_mailbox_messages(str(tmp_path)) == [] + assert scan_mailbox_messages(str(tmp_path / "nope")) == [] + + +def test_noop_under_limit(tmp_path): + _create_message(tmp_path, "cur/msg1", MB) + assert expire_to_target(str(tmp_path), 2 * MB) == [] + assert (tmp_path / "cur" / "msg1").exists() + + +def test_removes_to_target(tmp_path): + now = time.time() + for i in range(15): + _create_message(tmp_path, f"cur/msg{i:02d}", MB, days_old=i + 1) + removed = expire_to_target(str(tmp_path), 10 * MB, now=now) + assert len(removed) == 5 + assert len(scan_mailbox_messages(str(tmp_path))) == 10 + + +def test_scoring_prefers_large_old(tmp_path): + now = time.time() + _create_message(tmp_path, "cur/large_old", 2 * MB, days_old=30) + _create_message(tmp_path, "cur/small_new", MB, days_old=1) + removed = expire_to_target(str(tmp_path), 2 * MB, now=now) + assert len(removed) == 1 + assert "large_old" in removed[0] + + +def test_scoring_large_new_beats_small_old(tmp_path): + now = time.time() + _create_message(tmp_path, "cur/big_new", 10 * MB, days_old=1) + _create_message(tmp_path, "cur/small_old", MB, days_old=5) + # big_new score: 10MB * 1d = 10 vs small_old score: 1MB * 5d = 5 + removed = expire_to_target(str(tmp_path), 10 * MB, now=now) + assert len(removed) == 1 + assert "big_new" in removed[0] + + +def test_exact_limit(tmp_path): + _create_message(tmp_path, "cur/msg1", 5 * MB) + assert expire_to_target(str(tmp_path), 5 * MB) == [] + + +def test_removes_stale_caches(tmp_path): + _create_message(tmp_path, "cur/msg1", 2 * MB, days_old=5) + (tmp_path / "maildirsize").write_text("x") + (tmp_path / "dovecot.index.cache").write_text("x") + expire_to_target(str(tmp_path), MB) + assert not (tmp_path / "maildirsize").exists() + assert not (tmp_path / "dovecot.index.cache").exists() + + +def test_no_cache_removal_when_under_limit(tmp_path): + _create_message(tmp_path, "cur/msg1", MB) + (tmp_path / "maildirsize").write_text("x") + expire_to_target(str(tmp_path), 2 * MB) + assert (tmp_path / "maildirsize").exists() diff --git a/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 b/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 index bd1ce053..b3985bf7 100644 --- a/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 +++ b/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 @@ -149,12 +149,22 @@ plugin { } plugin { - # for now we define static quota-rules for all users + # for now we define static quota-rules for all users quota = maildir:User quota quota_rule = *:storage={{ config.max_mailbox_size }} quota_max_mail_size={{ config.max_message_size }} quota_grace = 0 - # quota_over_flag_value = TRUE + + # When a user reaches 90% quota, run chatmail-quota-expire + # to remove large/old messages until usage is below 80%. + quota_warning = storage=90%% quota-warning {{ config.max_mailbox_size_mb * 80 // 100 }} {{ config.mailboxes_dir }}/%u +} + +service quota-warning { + executable = script /usr/local/lib/chatmaild/venv/bin/chatmail-quota-expire + user = vmail + unix_listener quota-warning { + } } # push_notification configuration diff --git a/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py b/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py index 1a3fa43a..36afa643 100644 --- a/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py +++ b/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py @@ -244,24 +244,6 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config): pytest.fail("Rate limit was not exceeded") -@pytest.mark.slow -def test_expunged(remote, chatmail_config): - outdated_days = int(chatmail_config.delete_mails_after) + 1 - find_cmds = [ - f"find {chatmail_config.mailboxes_dir} -path '*/cur/*' -mtime +{outdated_days} -type f", - f"find {chatmail_config.mailboxes_dir} -path '*/.*/cur/*' -mtime +{outdated_days} -type f", - f"find {chatmail_config.mailboxes_dir} -path '*/new/*' -mtime +{outdated_days} -type f", - f"find {chatmail_config.mailboxes_dir} -path '*/.*/new/*' -mtime +{outdated_days} -type f", - f"find {chatmail_config.mailboxes_dir} -path '*/tmp/*' -mtime +{outdated_days} -type f", - f"find {chatmail_config.mailboxes_dir} -path '*/.*/tmp/*' -mtime +{outdated_days} -type f", - ] - outdated_days = int(chatmail_config.delete_large_after) + 1 - find_cmds.append( - f"find {chatmail_config.mailboxes_dir} -path '*/cur/*' -mtime +{outdated_days} -size +200k -type f" - ) - for cmd in find_cmds: - for line in remote.iter_output(cmd): - assert not line def test_deployed_state(remote): diff --git a/doc/source/overview.rst b/doc/source/overview.rst index fd2c9401..a10b243c 100644 --- a/doc/source/overview.rst +++ b/doc/source/overview.rst @@ -102,8 +102,14 @@ short overview of ``chatmaild`` services: Apple/Google/Huawei. - `chatmail-expire `_ - deletes users if they have not logged in for a longer while. - The timeframe can be configured in ``chatmail.ini``. + deletes entire mailboxes of users who have not logged in + for longer than ``delete_inactive_users_after`` days. + +- `chatmail-quota-expire `_ + is called by Dovecot's ``quota_warning`` mechanism when a + user reaches 90% of their mailbox quota. + It removes the largest and oldest messages + until usage drops below 80% of the quota. - `lastlogin `_ is contacted by Dovecot when a user logs in and stores the date of @@ -139,7 +145,7 @@ Chatmail relay dependency diagram certs-nginx[("`TLS certs /var/lib/acme`")] --> nginx-internal; systemd-timer --- acmetool; - systemd-timer --- chatmail-expire-daily; + systemd-timer --- chatmail-expire-inactive; systemd-timer --- chatmail-fsreport-daily; acmetool --> certs[("`TLS certs /var/lib/acme`")]; @@ -156,9 +162,11 @@ Chatmail relay dependency diagram /home/vmail/.../user"]; dovecot --- |lastlogin.socket|lastlogin; dovecot --- chatmail-metadata; + dovecot --- |quota-warning|chatmail-quota-expire; + chatmail-quota-expire --- maildir; lastlogin --- maildir; doveauth --- maildir; - chatmail-expire-daily --- maildir; + chatmail-expire-inactive --- maildir; chatmail-fsreport-daily --- maildir; chatmail-metadata --- iroh-relay; chatmail-metadata --- |encrypted device token| notifications.delta.chat;