Compare commits

..

1 Commits

Author SHA1 Message Date
holger krekel
4512d1e735 feat: add quota-triggered per-user mailbox cleanup
Inflate the Dovecot-visible quota to 140% of the configured
max_mailbox_size so that Delta Chat clients (which warn at
80% of IMAP-reported quota) never show quota warnings.
A quota_warning at 72% of the inflated limit triggers
chatmail-quota-expire, which trims the mailbox to 80% of the
configured limit.  Existing over-quota mailboxes start
receiving mail again immediately after deploy
without any manual operator action needed.
2026-04-18 23:08:50 +02:00
14 changed files with 196 additions and 174 deletions

View File

@@ -4,20 +4,16 @@
### Features
- Automated per-user quota-keeping.
Replace daily timer-based message expire script
with Dovecot quota-warning-triggered cleanup (`chatmail-quota-expire`).
When a user reaches 90% of their mailbox quota
Dovecot calls the new script which removes the largest and oldest messages
until usage drops below 80%.
The daily `chatmail-expire` timer now only handles deletion
of inactive user mailboxes.
After upgrading, run the following once to clean up
mailboxes that are already over quota::
/usr/local/lib/chatmaild/venv/bin/chatmail-quota-expire \
400 /home/vmail/mail/YOURDOMAIN --sweep
- Add per-user quota-triggered cleanup (`chatmail-quota-expire`).
When a mailbox exceeds the configured ``max_mailbox_size``,
Dovecot runs the new script which removes the oldest
messages until usage drops to a safe level.
No operator action is required after upgrading;
existing over-quota mailboxes start receiving mail
again immediately and are cleaned up automatically.
The daily `chatmail-expire` timer continues to handle
deletion of old messages, large messages,
and inactive user mailboxes.
## 1.9.0 2025-12-18

View File

@@ -21,7 +21,7 @@ where = ['src']
[project.scripts]
doveauth = "chatmaild.doveauth:main"
chatmail-metadata = "chatmaild.metadata:main"
chatmail-expire = "chatmaild.expire_inactive_users:main"
chatmail-expire = "chatmaild.expire:main"
chatmail-quota-expire = "chatmaild.quota_expire:main"
chatmail-fsreport = "chatmaild.fsreport:main"
lastlogin = "chatmaild.lastlogin:main"

View File

@@ -25,6 +25,8 @@ class Config:
self.max_user_send_burst_size = int(params.get("max_user_send_burst_size", 10))
self.max_mailbox_size = params["max_mailbox_size"]
self.max_message_size = int(params.get("max_message_size", "31457280"))
self.delete_mails_after = params["delete_mails_after"]
self.delete_large_after = params["delete_large_after"]
self.delete_inactive_users_after = int(params["delete_inactive_users_after"])
self.username_min_length = int(params["username_min_length"])
self.username_max_length = int(params["username_max_length"])

View File

@@ -115,8 +115,11 @@ class Expiry:
cutoff_without_login = (
self.now - int(self.config.delete_inactive_users_after) * 86400
)
cutoff_mails = self.now - int(self.config.delete_mails_after) * 86400
cutoff_large_mails = self.now - int(self.config.delete_large_after) * 86400
self.all_mboxes += 1
changed = False
if mbox.last_login and mbox.last_login < cutoff_without_login:
self.remove_mailbox(mbox.basedir)
return
@@ -128,10 +131,25 @@ class Expiry:
print_info(f"checking mailbox {date.strftime('%b %d')} {mboxname}")
else:
print_info(f"checking mailbox (no last_login) {mboxname}")
self.all_files += len(mbox.messages)
for message in mbox.messages:
if message.mtime < cutoff_mails:
self.remove_file(message.path, mtime=message.mtime)
elif message.size > 200000 and message.mtime < cutoff_large_mails:
# we only remove noticed large files (not unnoticed ones in new/)
parts = message.path.split("/")
if len(parts) >= 2 and parts[-2] == "cur":
self.remove_file(message.path, mtime=message.mtime)
else:
continue
changed = True
if changed:
self.remove_file(f"{mbox.basedir}/maildirsize")
def get_summary(self):
return (
f"Removed {self.del_mboxes} out of {self.all_mboxes} mailboxes "
f"and {self.del_files} out of {self.all_files} files in existing mailboxes "
f"in {time.time() - self.start:2.2f} seconds"
)

View File

@@ -23,6 +23,12 @@ max_mailbox_size = 500M
# maximum message size for an e-mail in bytes
max_message_size = 31457280
# days after which mails are unconditionally deleted
delete_mails_after = 20
# days after which large messages (>200k) are unconditionally deleted
delete_large_after = 7
# days after which users without a successful login are deleted (database and mails)
delete_inactive_users_after = 90

View File

@@ -1,98 +1,61 @@
"""
Remove messages from a mailbox to meet a size target.
Quota-triggered per-user mailbox cleanup.
Dovecot calls this script when a user's quota is near its limit.
Files are scored by ``size * age`` so that large, old messages
are removed first.
Dovecot calls this script via ``quota_warning``
when a user crosses the quota threshold.
The script removes oldest messages first
to keep the mailbox under a specified target size.
Usage::
quota_expire <target_mb> <mailbox_path>
chatmail-quota-expire <target_mb> <mailbox_path>
"""
import os
import sys
import time
from argparse import ArgumentParser
from collections import namedtuple
from stat import S_ISREG
from pathlib import Path
FileEntry = namedtuple("FileEntry", ("path", "mtime", "size"))
def _get_file_entry(path):
try:
st = os.stat(path)
except FileNotFoundError:
return None
if not S_ISREG(st.st_mode):
return None
return FileEntry(path, st.st_mtime, st.st_size)
def _listdir(path):
try:
return os.listdir(path)
except FileNotFoundError:
return []
from chatmaild.expire import get_file_entry, os_listdir_if_exists
def scan_mailbox_messages(mailbox_dir):
"""Collect FileEntry items from top-level cur/new/tmp only."""
mbox = Path(mailbox_dir)
messages = []
for sub in ("cur", "new", "tmp"):
subdir = f"{mailbox_dir}/{sub}"
for name in _listdir(subdir):
entry = _get_file_entry(f"{subdir}/{name}")
if entry is not None:
for name in os_listdir_if_exists(mbox / sub):
if entry := get_file_entry(str(mbox / sub / name)):
messages.append(entry)
return messages
def _remove_stale_caches(mailbox_dir):
for name in ("maildirsize", "dovecot.index.cache"):
try:
os.unlink(f"{mailbox_dir}/{name}")
except FileNotFoundError:
pass
def expire_to_target(mailbox_dir, target_bytes):
"""Remove oldest files until total size <= *target_bytes*.
def expire_to_target(mailbox_dir, target_bytes, now=None):
"""Remove highest-scored files until total size <= *target_bytes*.
Returns the list of removed file paths.
Returns ``(removed_count, cache_bytes)`` where *cache_bytes*
is the size of the deleted ``dovecot.index.cache`` file
(0 when the file did not exist).
"""
if now is None:
now = time.time()
messages = scan_mailbox_messages(mailbox_dir)
mbox = Path(mailbox_dir)
messages = scan_mailbox_messages(mbox)
total_size = sum(m.size for m in messages)
if total_size <= target_bytes:
return []
# Score: large and old files get the highest score.
scored = sorted(
messages,
key=lambda m: m.size * (now - m.mtime),
reverse=True,
)
removed = []
for entry in scored:
removed = 0
for count, entry in enumerate(sorted(messages, key=lambda m: m.mtime), 1):
if total_size <= target_bytes:
break
try:
os.unlink(entry.path)
except FileNotFoundError:
continue
Path(entry.path).unlink(missing_ok=True)
total_size -= entry.size
removed.append(entry.path)
removed = count
if removed:
_remove_stale_caches(mailbox_dir)
return removed
(mbox / "maildirsize").unlink(missing_ok=True)
cache = mbox / "dovecot.index.cache"
try:
cache_bytes = cache.stat().st_size
except FileNotFoundError:
cache_bytes = 0
cache.unlink(missing_ok=True)
return removed, cache_bytes
def main(args=None):
@@ -105,48 +68,23 @@ def main(args=None):
)
parser.add_argument(
"mailbox_path",
help="path to a user mailbox, or with --sweep the mailboxes directory",
)
parser.add_argument(
"--sweep",
action="store_true",
help="sweep all mailboxes under mailbox_path",
help="path to a user mailbox",
)
args = parser.parse_args(args)
target_bytes = args.target_mb * 1024 * 1024
if args.sweep:
return _sweep(args.mailbox_path, target_bytes)
removed = expire_to_target(args.mailbox_path, target_bytes)
if removed:
removed_count, cache_bytes = expire_to_target(args.mailbox_path, target_bytes)
if removed_count:
user = Path(args.mailbox_path).name
cache_mb = cache_bytes / 1024 / 1024
print(
f"removed {len(removed)} file(s) from {args.mailbox_path}"
f" to reach {args.target_mb} MB target",
f"quota-expire: removed {removed_count} message(s) from {user}"
f" cache={cache_mb:.1f}MB",
file=sys.stderr,
)
return 0
def _sweep(mailboxes_dir, target_bytes):
try:
names = os.listdir(mailboxes_dir)
except FileNotFoundError:
print(f"directory not found: {mailboxes_dir}", file=sys.stderr)
return 1
for name in sorted(names):
if "@" not in name:
continue
mbox = f"{mailboxes_dir}/{name}"
removed = expire_to_target(mbox, target_bytes)
if removed:
print(
f"removed {len(removed)} file(s) from {name}",
file=sys.stderr,
)
return 0
if __name__ == "__main__":
sys.exit(main())

View File

@@ -1,6 +1,6 @@
import pytest
from chatmaild.config import read_config
from chatmaild.config import parse_size_mb, read_config
def test_read_config_basic(example_config):
@@ -34,6 +34,8 @@ def test_read_config_testrun(make_config):
assert config.postfix_reinject_port == 10025
assert config.max_user_send_per_minute == 60
assert config.max_mailbox_size == "500M"
assert config.delete_mails_after == "20"
assert config.delete_large_after == "7"
assert config.username_min_length == 9
assert config.username_max_length == 9
assert config.password_min_length == 9
@@ -119,3 +121,17 @@ def test_config_tls_external_bad_format(make_config):
"tls_external_cert_and_key": "/only/one/path.pem",
},
)
def test_parse_size_mb():
assert parse_size_mb("500M") == 500
assert parse_size_mb("2G") == 2048
assert parse_size_mb(" 1g ") == 1024
assert parse_size_mb("100MB") == 100
assert parse_size_mb("256") == 256
def test_max_mailbox_size_mb(make_config):
config = make_config("chat.example.org")
assert config.max_mailbox_size == "500M"
assert config.max_mailbox_size_mb == 500

View File

@@ -1,6 +1,7 @@
import os
import random
from datetime import datetime
from fnmatch import fnmatch
from pathlib import Path
import pytest
@@ -153,6 +154,35 @@ def test_expiry_cli_basic(example_config, mbox1):
expiry_main(args)
def test_expiry_cli_old_files(capsys, example_config, mbox1):
relpaths_old = ["cur/msg_old1", "cur/msg_old1"]
cutoff_days = int(example_config.delete_mails_after) + 1
create_new_messages(mbox1.basedir, relpaths_old, size=1000, days=cutoff_days)
relpaths_large = ["cur/msg_old_large1", "new/msg_old_large2"]
cutoff_days = int(example_config.delete_large_after) + 1
create_new_messages(
mbox1.basedir, relpaths_large, size=1000 * 300, days=cutoff_days
)
create_new_messages(mbox1.basedir, ["cur/shouldstay"], size=1000 * 300, days=1)
args = str(example_config._inipath), "--remove", "-v"
expiry_main(args)
out, err = capsys.readouterr()
allpaths = relpaths_old + relpaths_large + ["maildirsize"]
for path in allpaths:
for line in err.split("\n"):
if fnmatch(line, f"removing*{path}"):
break
else:
if path != "new/msg_old_large2":
pytest.fail(f"failed to remove {path}\n{err}")
assert "shouldstay" not in err
def test_get_file_entry(tmp_path):
assert get_file_entry(str(tmp_path.joinpath("123123"))) is None
p = tmp_path.joinpath("x")

View File

@@ -1,7 +1,7 @@
import os
import time
from chatmaild.quota_expire import expire_to_target, scan_mailbox_messages
from chatmaild.quota_expire import expire_to_target, main, scan_mailbox_messages
MB = 1024 * 1024
@@ -19,10 +19,7 @@ def test_scan_cur_new_tmp(tmp_path):
_create_message(tmp_path, "cur/msg1", 100)
_create_message(tmp_path, "new/msg2", 200)
_create_message(tmp_path, "tmp/msg3", 300)
messages = scan_mailbox_messages(str(tmp_path))
assert len(messages) == 3
sizes = sorted(m.size for m in messages)
assert sizes == [100, 200, 300]
assert len(scan_mailbox_messages(str(tmp_path))) == 3
def test_scan_ignores_subfolders(tmp_path):
@@ -31,61 +28,46 @@ def test_scan_ignores_subfolders(tmp_path):
assert len(scan_mailbox_messages(str(tmp_path))) == 1
def test_scan_empty(tmp_path):
assert scan_mailbox_messages(str(tmp_path)) == []
assert scan_mailbox_messages(str(tmp_path / "nope")) == []
def test_noop_under_limit(tmp_path):
_create_message(tmp_path, "cur/msg1", MB)
assert expire_to_target(str(tmp_path), 2 * MB) == []
assert (tmp_path / "cur" / "msg1").exists()
def test_removes_to_target(tmp_path):
now = time.time()
for i in range(15):
_create_message(tmp_path, f"cur/msg{i:02d}", MB, days_old=i + 1)
removed = expire_to_target(str(tmp_path), 10 * MB, now=now)
assert len(removed) == 5
removed, _ = expire_to_target(str(tmp_path), 10 * MB)
assert removed == 5
assert len(scan_mailbox_messages(str(tmp_path))) == 10
def test_scoring_prefers_large_old(tmp_path):
now = time.time()
_create_message(tmp_path, "cur/large_old", 2 * MB, days_old=30)
_create_message(tmp_path, "cur/small_new", MB, days_old=1)
removed = expire_to_target(str(tmp_path), 2 * MB, now=now)
assert len(removed) == 1
assert "large_old" in removed[0]
def test_scoring_large_new_beats_small_old(tmp_path):
now = time.time()
_create_message(tmp_path, "cur/big_new", 10 * MB, days_old=1)
_create_message(tmp_path, "cur/small_old", MB, days_old=5)
# big_new score: 10MB * 1d = 10 vs small_old score: 1MB * 5d = 5
removed = expire_to_target(str(tmp_path), 10 * MB, now=now)
assert len(removed) == 1
assert "big_new" in removed[0]
def test_removes_oldest_first(tmp_path):
_create_message(tmp_path, "cur/old_small", MB, days_old=30)
_create_message(tmp_path, "cur/new_huge", 10 * MB, days_old=1)
# the 10MB file is kept, the 1MB file is removed because it's older
removed, _ = expire_to_target(str(tmp_path), 10 * MB)
assert removed == 1
assert not (tmp_path / "cur/old_small").exists()
assert (tmp_path / "cur/new_huge").exists()
def test_exact_limit(tmp_path):
_create_message(tmp_path, "cur/msg1", 5 * MB)
assert expire_to_target(str(tmp_path), 5 * MB) == []
removed, _ = expire_to_target(str(tmp_path), 5 * MB)
assert removed == 0
def test_removes_stale_caches(tmp_path):
_create_message(tmp_path, "cur/msg1", 2 * MB, days_old=5)
(tmp_path / "maildirsize").write_text("x")
(tmp_path / "dovecot.index.cache").write_text("x")
expire_to_target(str(tmp_path), MB)
(tmp_path / "dovecot.index.cache").write_bytes(b"y" * 4096)
removed, cache_bytes = expire_to_target(str(tmp_path), MB)
assert removed == 1
assert cache_bytes == 4096
assert not (tmp_path / "maildirsize").exists()
assert not (tmp_path / "dovecot.index.cache").exists()
def test_no_cache_removal_when_under_limit(tmp_path):
_create_message(tmp_path, "cur/msg1", MB)
(tmp_path / "maildirsize").write_text("x")
expire_to_target(str(tmp_path), 2 * MB)
assert (tmp_path / "maildirsize").exists()
def test_logging_output_is_mtail_compatible(tmp_path, capsys):
mbox = tmp_path / "user@example.org"
_create_message(mbox, "cur/msg1", 2 * MB, days_old=5)
(mbox / "dovecot.index.cache").write_bytes(b"c" * 2 * MB)
main([str(1), str(mbox)])
_, err = capsys.readouterr()
assert "quota-expire: removed 1 message(s) from user@example.org" in err
assert "cache=2.0MB" in err

View File

@@ -149,15 +149,19 @@ plugin {
}
plugin {
# for now we define static quota-rules for all users
quota = maildir:User quota
quota_rule = *:storage={{ config.max_mailbox_size }}
quota_max_mail_size={{ config.max_message_size }}
quota_grace = 0
# When a user reaches 90% quota, run chatmail-quota-expire
# to remove large/old messages until usage is below 80%.
quota_warning = storage=90%% quota-warning {{ config.max_mailbox_size_mb * 80 // 100 }} {{ config.mailboxes_dir }}/%u
# Inflate the dovecot-visible quota so that Delta Chat clients
# (which warn at 80% of the IMAP-reported limit) never see
# quota warnings -- expire kicks in well before that point.
quota_rule = *:storage={{ config.max_mailbox_size_mb * 140 // 100 }}M
# Trigger when usage reaches the configured max_mailbox_size
# (72% of inflated = ~100% of configured), then expire oldest
# messages down to 80% of the configured max_mailbox_size.
quota_warning = storage=72%% quota-warning {{ config.max_mailbox_size_mb * 80 // 100 }} {{ config.mailboxes_dir }}/%u
}
service quota-warning {

View File

@@ -78,3 +78,11 @@ counter rejected_unencrypted_mail_count
/Rejected unencrypted mail/ {
rejected_unencrypted_mail_count++
}
counter quota_expire_runs
counter quota_expire_removed_files
/quota-expire: removed (?P<count>\d+) message\(s\)/ {
quota_expire_runs++
quota_expire_removed_files += $count
}

View File

@@ -244,6 +244,24 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
pytest.fail("Rate limit was not exceeded")
@pytest.mark.slow
def test_expunged(remote, chatmail_config):
outdated_days = int(chatmail_config.delete_mails_after) + 1
find_cmds = [
f"find {chatmail_config.mailboxes_dir} -path '*/cur/*' -mtime +{outdated_days} -type f",
f"find {chatmail_config.mailboxes_dir} -path '*/.*/cur/*' -mtime +{outdated_days} -type f",
f"find {chatmail_config.mailboxes_dir} -path '*/new/*' -mtime +{outdated_days} -type f",
f"find {chatmail_config.mailboxes_dir} -path '*/.*/new/*' -mtime +{outdated_days} -type f",
f"find {chatmail_config.mailboxes_dir} -path '*/tmp/*' -mtime +{outdated_days} -type f",
f"find {chatmail_config.mailboxes_dir} -path '*/.*/tmp/*' -mtime +{outdated_days} -type f",
]
outdated_days = int(chatmail_config.delete_large_after) + 1
find_cmds.append(
f"find {chatmail_config.mailboxes_dir} -path '*/cur/*' -mtime +{outdated_days} -size +200k -type f"
)
for cmd in find_cmds:
for line in remote.iter_output(cmd):
assert not line
def test_deployed_state(remote):

View File

@@ -88,9 +88,12 @@ class TestEndToEndDeltaChat:
return int(float(number) * units[unit])
quota = parse_size_limit(chatmail_config.max_mailbox_size)
# Dovecot quota is inflated to 140% of the configured limit
# so that quota-expire keeps users below the warning threshold.
dovecot_quota = quota * 140 // 100
lp.sec(f"filling remote inbox for {user}")
fn = f"7743102289.M843172P2484002.c20,S={quota},W=2398:2,"
fn = f"7743102289.M843172P2484002.c20,S={dovecot_quota},W=2398:2,"
path = chatmail_config.mailboxes_dir.joinpath(user, "cur", fn)
sshexec = get_sshexec(sshdomain)
sshexec(call=rshell.write_numbytes, kwargs=dict(path=str(path), num=120))

View File

@@ -102,14 +102,15 @@ short overview of ``chatmaild`` services:
Apple/Google/Huawei.
- `chatmail-expire <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/expire.py>`_
deletes entire mailboxes of users who have not logged in
for longer than ``delete_inactive_users_after`` days.
deletes old messages, large messages, and entire mailboxes
of users who have not logged in for longer than
``delete_inactive_users_after`` days.
- `chatmail-quota-expire <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/quota_expire.py>`_
- ``chatmail-quota-expire``
is called by Dovecot's ``quota_warning`` mechanism when a
user reaches 90% of their mailbox quota.
It removes the largest and oldest messages
until usage drops below 80% of the quota.
mailbox exceeds ``max_mailbox_size``.
It removes the oldest messages
until usage drops to a safe level.
- `lastlogin <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/lastlogin.py>`_
is contacted by Dovecot when a user logs in and stores the date of
@@ -145,7 +146,7 @@ Chatmail relay dependency diagram
certs-nginx[("`TLS certs
/var/lib/acme`")] --> nginx-internal;
systemd-timer --- acmetool;
systemd-timer --- chatmail-expire-inactive;
systemd-timer --- chatmail-expire-daily;
systemd-timer --- chatmail-fsreport-daily;
acmetool --> certs[("`TLS certs
/var/lib/acme`")];
@@ -166,7 +167,7 @@ Chatmail relay dependency diagram
chatmail-quota-expire --- maildir;
lastlogin --- maildir;
doveauth --- maildir;
chatmail-expire-inactive --- maildir;
chatmail-expire-daily --- maildir;
chatmail-fsreport-daily --- maildir;
chatmail-metadata --- iroh-relay;
chatmail-metadata --- |encrypted device token| notifications.delta.chat;