Compare commits

..

1 Commits

Author SHA1 Message Date
holger krekel
4512d1e735 feat: add quota-triggered per-user mailbox cleanup
Inflate the Dovecot-visible quota to 140% of the configured
max_mailbox_size so that Delta Chat clients (which warn at
80% of IMAP-reported quota) never show quota warnings.
A quota_warning at 72% of the inflated limit triggers
chatmail-quota-expire, which trims the mailbox to 80% of the
configured limit.  Existing over-quota mailboxes start
receiving mail again immediately after deploy
without any manual operator action needed.
2026-04-18 23:08:50 +02:00
17 changed files with 240 additions and 226 deletions

View File

@@ -1,5 +1,20 @@
# Changelog for chatmail deployment
## Unreleased
### Features
- Add per-user quota-triggered cleanup (`chatmail-quota-expire`).
When a mailbox exceeds the configured ``max_mailbox_size``,
Dovecot runs the new script which removes the oldest
messages until usage drops to a safe level.
No operator action is required after upgrading;
existing over-quota mailboxes start receiving mail
again immediately and are cleaned up automatically.
The daily `chatmail-expire` timer continues to handle
deletion of old messages, large messages,
and inactive user mailboxes.
## 1.9.0 2025-12-18
### Documentation

View File

@@ -21,8 +21,8 @@ where = ['src']
[project.scripts]
doveauth = "chatmaild.doveauth:main"
chatmail-metadata = "chatmaild.metadata:main"
chatmail-expire = "chatmaild.expire:daily_expire_main"
chatmail-quota-expire = "chatmaild.expire:quota_expire_main"
chatmail-expire = "chatmaild.expire:main"
chatmail-quota-expire = "chatmaild.quota_expire:main"
chatmail-fsreport = "chatmaild.fsreport:main"
lastlogin = "chatmaild.lastlogin:main"
turnserver = "chatmaild.turnserver:main"

View File

@@ -115,7 +115,7 @@ class Config:
def parse_size_mb(limit):
"""Parse a size string like ``500M`` or ``2G`` and return megabytes."""
value = limit.strip().upper().removesuffix("B")
value = limit.strip().upper().rstrip("B")
if value.endswith("G"):
return int(value[:-1]) * 1024
if value.endswith("M"):

View File

@@ -4,26 +4,17 @@ Expire old messages and addresses.
"""
import os
import re
import shutil
import sys
import time
from argparse import ArgumentParser
from collections import namedtuple
from datetime import datetime
from pathlib import Path
from stat import S_ISREG
from chatmaild.config import read_config
FileEntry = namedtuple("FileEntry", ("path", "mtime", "size"))
QuotaFileEntry = namedtuple("QuotaFileEntry", ("mtime", "quota_size", "path"))
# Quota cleanup factor of max_mailbox_size. The mailbox is reset to this size.
QUOTA_CLEANUP_FACTOR = 0.7
# e.g. "cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S"
_dovecot_fn_rex = re.compile(r".+/(\d+)\..+,S=(\d+)")
def iter_mailboxes(basedir, maxnum):
@@ -83,54 +74,6 @@ class MailboxStat:
self.extrafiles.sort(key=lambda x: -x.size)
def parse_dovecot_filename(relpath):
m = _dovecot_fn_rex.match(relpath)
if not m:
return None
return QuotaFileEntry(int(m.group(1)), int(m.group(2)), relpath)
def scan_mailbox_messages(mbox):
messages = []
for sub in ("cur", "new"):
for name in os_listdir_if_exists(mbox / sub):
if entry := parse_dovecot_filename(f"{sub}/{name}"):
messages.append(entry)
return messages
# Within this window, large messages are deleted before small ones.
DELETE_LARGE_FIRST_DAYS = 7
def expire_to_target(mbox, target_bytes):
cutoff = time.time() - DELETE_LARGE_FIRST_DAYS * 86400
messages = scan_mailbox_messages(mbox)
def sort_key(msg):
# prio 0: Older than cutoff -> remove oldest first
if msg.mtime < cutoff:
return (0, msg.mtime)
# prio 1: more recent than cutoff, large -> remove largest first
if msg.quota_size > 200000:
return (1, -msg.quota_size, msg.mtime)
# prio 2: more recent than cutoff, small -> remove oldest first
return (2, msg.mtime)
total_size = sum(m.quota_size for m in messages)
removed = 0
for entry in sorted(messages, key=sort_key):
if total_size <= target_bytes:
break
(mbox / entry.path).unlink(missing_ok=True)
total_size -= entry.quota_size
removed += 1
return removed
def print_info(msg):
print(msg, file=sys.stderr)
@@ -200,19 +143,6 @@ class Expiry:
else:
continue
changed = True
target_bytes = (
self.config.max_mailbox_size_mb * 1024 * 1024 * QUOTA_CLEANUP_FACTOR
)
removed = expire_to_target(Path(mbox.basedir), target_bytes)
if removed:
changed = True
self.del_files += removed
if self.verbose:
print_info(
f"quota-expire: removed {removed} message(s) from {mboxname}"
)
if changed:
self.remove_file(f"{mbox.basedir}/maildirsize")
@@ -224,9 +154,9 @@ class Expiry:
)
def daily_expire_main(args=None):
def main(args=None):
"""Expire mailboxes and messages according to chatmail config"""
parser = ArgumentParser(description=daily_expire_main.__doc__)
parser = ArgumentParser(description=main.__doc__)
ini = "/usr/local/lib/chatmaild/chatmail.ini"
parser.add_argument(
"chatmail_ini",
@@ -272,33 +202,5 @@ def daily_expire_main(args=None):
print(exp.get_summary())
def quota_expire_main(args=None):
"""Remove mailbox messages to stay within a megabyte target.
This entry point is called by dovecot when a quota threshold is passed.
"""
parser = ArgumentParser(description=quota_expire_main.__doc__)
parser.add_argument(
"target_mb",
type=int,
help="target mailbox size in megabytes",
)
parser.add_argument(
"mailbox_path",
type=Path,
help="path to a user mailbox",
)
args = parser.parse_args(args)
target_bytes = args.target_mb * 1024 * 1024
removed_count = expire_to_target(args.mailbox_path, target_bytes)
if removed_count:
(args.mailbox_path / "maildirsize").unlink(missing_ok=True)
print(
f"quota-expire: removed {removed_count} message(s)"
f" from {args.mailbox_path.name}",
file=sys.stderr,
)
return 0
if __name__ == "__main__":
main(sys.argv[1:])

View File

@@ -18,7 +18,6 @@ max_user_send_per_minute = 60
max_user_send_burst_size = 10
# maximum mailbox size of a chatmail address
# Oldest messages will be removed automatically, so mailboxes never run full.
max_mailbox_size = 500M
# maximum message size for an e-mail in bytes

View File

@@ -0,0 +1,90 @@
"""
Quota-triggered per-user mailbox cleanup.
Dovecot calls this script via ``quota_warning``
when a user crosses the quota threshold.
The script removes oldest messages first
to keep the mailbox under a specified target size.
Usage::
chatmail-quota-expire <target_mb> <mailbox_path>
"""
import sys
from argparse import ArgumentParser
from pathlib import Path
from chatmaild.expire import get_file_entry, os_listdir_if_exists
def scan_mailbox_messages(mailbox_dir):
"""Collect FileEntry items from top-level cur/new/tmp only."""
mbox = Path(mailbox_dir)
messages = []
for sub in ("cur", "new", "tmp"):
for name in os_listdir_if_exists(mbox / sub):
if entry := get_file_entry(str(mbox / sub / name)):
messages.append(entry)
return messages
def expire_to_target(mailbox_dir, target_bytes):
"""Remove oldest files until total size <= *target_bytes*.
Returns ``(removed_count, cache_bytes)`` where *cache_bytes*
is the size of the deleted ``dovecot.index.cache`` file
(0 when the file did not exist).
"""
mbox = Path(mailbox_dir)
messages = scan_mailbox_messages(mbox)
total_size = sum(m.size for m in messages)
removed = 0
for count, entry in enumerate(sorted(messages, key=lambda m: m.mtime), 1):
if total_size <= target_bytes:
break
Path(entry.path).unlink(missing_ok=True)
total_size -= entry.size
removed = count
(mbox / "maildirsize").unlink(missing_ok=True)
cache = mbox / "dovecot.index.cache"
try:
cache_bytes = cache.stat().st_size
except FileNotFoundError:
cache_bytes = 0
cache.unlink(missing_ok=True)
return removed, cache_bytes
def main(args=None):
"""Remove mailbox messages to stay within a megabyte target."""
parser = ArgumentParser(description=main.__doc__)
parser.add_argument(
"target_mb",
type=int,
help="target mailbox size in megabytes",
)
parser.add_argument(
"mailbox_path",
help="path to a user mailbox",
)
args = parser.parse_args(args)
target_bytes = args.target_mb * 1024 * 1024
removed_count, cache_bytes = expire_to_target(args.mailbox_path, target_bytes)
if removed_count:
user = Path(args.mailbox_path).name
cache_mb = cache_bytes / 1024 / 1024
print(
f"quota-expire: removed {removed_count} message(s) from {user}"
f" cache={cache_mb:.1f}MB",
file=sys.stderr,
)
return 0
if __name__ == "__main__":
sys.exit(main())

View File

@@ -1,7 +1,7 @@
import time
from chatmaild.doveauth import AuthDictProxy
from chatmaild.expire import daily_expire_main as main_expire
from chatmaild.expire import main as main_expire
def test_login_timestamps(example_config):

View File

@@ -1,8 +1,5 @@
import itertools
import os
import random
import shutil
import time
from datetime import datetime
from fnmatch import fnmatch
from pathlib import Path
@@ -12,19 +9,13 @@ import pytest
from chatmaild.expire import (
FileEntry,
MailboxStat,
expire_to_target,
get_file_entry,
iter_mailboxes,
os_listdir_if_exists,
parse_dovecot_filename,
quota_expire_main,
scan_mailbox_messages,
)
from chatmaild.expire import daily_expire_main as expiry_main
from chatmaild.expire import main as expiry_main
from chatmaild.fsreport import main as report_main
MB = 1024 * 1024
def fill_mbox(folderdir):
password = folderdir.joinpath("password")
@@ -205,74 +196,3 @@ def test_os_listdir_if_exists(tmp_path):
tmp_path.joinpath("x").write_text("hello")
assert len(os_listdir_if_exists(str(tmp_path))) == 1
assert len(os_listdir_if_exists(str(tmp_path.joinpath("123123")))) == 0
# --- quota expire tests ---
_msg_counter = itertools.count(1)
def _create_message(basedir, sub, size, days_old=0, disk_size=None):
seq = next(_msg_counter)
mtime = int(time.time() - days_old * 86400)
name = f"{mtime}.M1P1Q{seq}.hostname,S={size},W={size}:2,S"
path = basedir / sub / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(b"x" * (disk_size if disk_size is not None else size))
os.utime(path, (mtime, mtime))
return path
def test_parse_dovecot_filename():
e = parse_dovecot_filename("cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S")
assert e.path == "cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S"
assert e.mtime == 1775324677
assert e.quota_size == 3235
assert parse_dovecot_filename("cur/msg_without_structure") is None
def test_expire_to_target(tmp_path):
_create_message(tmp_path, "cur", MB, days_old=10, disk_size=100)
_create_message(tmp_path, "new", MB, days_old=5)
assert len(scan_mailbox_messages(tmp_path)) == 2
# removes oldest first, uses S= size not disk size
removed = expire_to_target(tmp_path, MB)
assert removed == 1
assert len(scan_mailbox_messages(tmp_path)) == 1
def test_expire_to_target_prioritization(tmp_path):
def create_messages():
for sub in ("cur", "new"):
if (tmp_path / sub).exists():
shutil.rmtree(tmp_path / sub)
# prio 0: older than 7 days
_create_message(tmp_path, "cur", 5 * MB, days_old=10)
# prio 1: last 7 days, large (>200KB)
_create_message(tmp_path, "cur", 5 * MB, days_old=1)
# prio 2: last 7 days, small
_create_message(tmp_path, "cur", 1000, days_old=2)
# Shrink to 6MB: only the old message (prio 0) is removed.
create_messages()
assert expire_to_target(tmp_path, 6 * MB) == 1
msgs = scan_mailbox_messages(tmp_path)
assert len(msgs) == 2
assert all(m.mtime > time.time() - 7 * 86400 for m in msgs)
# Shrink to 1KB: old and recent-large removed, small survives.
create_messages()
assert expire_to_target(tmp_path, 1024) == 2
msgs = scan_mailbox_messages(tmp_path)
assert len(msgs) == 1
assert msgs[0].quota_size == 1000
def test_quota_expire_main(tmp_path, capsys):
mbox = tmp_path / "user@example.org"
_create_message(mbox, "cur", 2 * MB, days_old=5)
(mbox / "maildirsize").write_text("x")
quota_expire_main([str(1), str(mbox)])
_, err = capsys.readouterr()
assert "quota-expire: removed 1 message(s) from user@example.org" in err
assert not (mbox / "maildirsize").exists()

View File

@@ -0,0 +1,73 @@
import os
import time
from chatmaild.quota_expire import expire_to_target, main, scan_mailbox_messages
MB = 1024 * 1024
def _create_message(basedir, relpath, size, days_old=0):
path = basedir / relpath
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(b"x" * size)
mtime = time.time() - days_old * 86400
os.utime(path, (mtime, mtime))
return path
def test_scan_cur_new_tmp(tmp_path):
_create_message(tmp_path, "cur/msg1", 100)
_create_message(tmp_path, "new/msg2", 200)
_create_message(tmp_path, "tmp/msg3", 300)
assert len(scan_mailbox_messages(str(tmp_path))) == 3
def test_scan_ignores_subfolders(tmp_path):
_create_message(tmp_path, "cur/a", 10)
_create_message(tmp_path, ".DeltaChat/cur/b", 20)
assert len(scan_mailbox_messages(str(tmp_path))) == 1
def test_removes_to_target(tmp_path):
for i in range(15):
_create_message(tmp_path, f"cur/msg{i:02d}", MB, days_old=i + 1)
removed, _ = expire_to_target(str(tmp_path), 10 * MB)
assert removed == 5
assert len(scan_mailbox_messages(str(tmp_path))) == 10
def test_removes_oldest_first(tmp_path):
_create_message(tmp_path, "cur/old_small", MB, days_old=30)
_create_message(tmp_path, "cur/new_huge", 10 * MB, days_old=1)
# the 10MB file is kept, the 1MB file is removed because it's older
removed, _ = expire_to_target(str(tmp_path), 10 * MB)
assert removed == 1
assert not (tmp_path / "cur/old_small").exists()
assert (tmp_path / "cur/new_huge").exists()
def test_exact_limit(tmp_path):
_create_message(tmp_path, "cur/msg1", 5 * MB)
removed, _ = expire_to_target(str(tmp_path), 5 * MB)
assert removed == 0
def test_removes_stale_caches(tmp_path):
_create_message(tmp_path, "cur/msg1", 2 * MB, days_old=5)
(tmp_path / "maildirsize").write_text("x")
(tmp_path / "dovecot.index.cache").write_bytes(b"y" * 4096)
removed, cache_bytes = expire_to_target(str(tmp_path), MB)
assert removed == 1
assert cache_bytes == 4096
assert not (tmp_path / "maildirsize").exists()
assert not (tmp_path / "dovecot.index.cache").exists()
def test_logging_output_is_mtail_compatible(tmp_path, capsys):
mbox = tmp_path / "user@example.org"
_create_message(mbox, "cur/msg1", 2 * MB, days_old=5)
(mbox / "dovecot.index.cache").write_bytes(b"c" * 2 * MB)
main([str(1), str(mbox)])
_, err = capsys.readouterr()
assert "quota-expire: removed 1 message(s) from user@example.org" in err
assert "cache=2.0MB" in err

View File

@@ -194,6 +194,12 @@ def status_cmd(args, out):
def test_cmd_options(parser):
parser.add_argument(
"--slow",
dest="slow",
action="store_true",
help="also run slow tests",
)
add_ssh_host_option(parser)
@@ -215,6 +221,8 @@ def test_cmd(args, out):
"-v",
"--durations=5",
]
if args.slow:
pytest_args.append("--slow")
ret = out.run_ret(pytest_args, env=env)
return ret

View File

@@ -150,40 +150,18 @@ class UnboundDeployer(Deployer):
self.need_restart = False
def install(self):
# Run local DNS resolver `unbound`. `resolvconf` takes care of
# setting up /etc/resolv.conf to use 127.0.0.1 as the resolver.
# On an IPv4-only system, if unbound is started but not configured,
# it causes subsequent steps to fail to resolve hosts.
with blocked_service_startup():
apt.packages(
name="Install unbound",
packages=["unbound", "unbound-anchor", "dnsutils"],
packages=["unbound", "unbound-anchor", "dnsutils", "resolvconf"],
)
def configure(self):
# Remove dynamic resolver managers that compete for /etc/resolv.conf.
apt.packages(
name="Purge resolvconf",
packages=["resolvconf"],
present=False,
extra_uninstall_args="--purge",
)
# systemd-resolved can't be purged due to dependencies; stop and mask.
server.shell(
name="Stop and mask systemd-resolved",
commands=[
"systemctl stop systemd-resolved.service || true",
"systemctl mask systemd-resolved.service",
],
)
# Configure unbound resolver with Quad9 fallback and a trailing newline
# (SolusVM bug).
files.put(
name="Write static resolv.conf",
src=BytesIO(b"nameserver 127.0.0.1\nnameserver 9.9.9.9\n"),
dest="/etc/resolv.conf",
user="root",
group="root",
mode="644",
)
server.shell(
name="Generate root keys for validating DNSSEC",
commands=[
@@ -590,6 +568,14 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
Deployment().perform_stages([WebsiteDeployer(config)])
return
if host.get_fact(Port, port=53) != "unbound":
files.line(
name="Add 9.9.9.9 to resolv.conf",
path="/etc/resolv.conf",
# Guard against resolv.conf missing a trailing newline (SolusVM bug).
line="\nnameserver 9.9.9.9",
)
# Check if mtail_address interface is available (if configured)
if config.mtail_address and config.mtail_address not in ('127.0.0.1', '::1', 'localhost'):
ipv4_addrs = host.get_fact(hardware.Ipv4Addrs)

View File

@@ -186,7 +186,7 @@ def _configure_dovecot(config: Config, debug: bool = False) -> tuple[bool, bool]
can_modify = not is_in_container()
for name in ("max_user_instances", "max_user_watches"):
key = f"fs.inotify.{name}"
value = host.get_fact(Sysctl).get(key, 0)
value = host.get_fact(Sysctl)[key]
if value > 65534:
continue
if not can_modify:

View File

@@ -153,21 +153,21 @@ plugin {
quota_max_mail_size={{ config.max_message_size }}
quota_grace = 0
quota_rule = *:storage={{ config.max_mailbox_size_mb }}M
# Inflate the dovecot-visible quota so that Delta Chat clients
# (which warn at 80% of the IMAP-reported limit) never see
# quota warnings -- expire kicks in well before that point.
quota_rule = *:storage={{ config.max_mailbox_size_mb * 140 // 100 }}M
# Trigger at 75%% of quota, expire oldest messages down to 70%%.
# The percentages are chosen to prevent current Delta Chat users
# from seeing "quota warnings" which trigger at 80% and 95%.
quota_warning = storage=75%% quota-warning {{ config.max_mailbox_size_mb * 70 // 100 }} {{ config.mailboxes_dir }}/%u
# Trigger when usage reaches the configured max_mailbox_size
# (72% of inflated = ~100% of configured), then expire oldest
# messages down to 80% of the configured max_mailbox_size.
quota_warning = storage=72%% quota-warning {{ config.max_mailbox_size_mb * 80 // 100 }} {{ config.mailboxes_dir }}/%u
}
service quota-warning {
executable = script /usr/local/lib/chatmaild/venv/bin/chatmail-quota-expire
user = vmail
unix_listener quota-warning {
user = vmail
mode = 0600
}
}

View File

@@ -221,6 +221,7 @@ def test_rewrite_subject(cmsetup, maildata):
assert "Subject: Unencrypted subject" not in rcvd_msg
@pytest.mark.slow
def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
"""Test that the per-account send-mail limit is exceeded."""
user1, user2 = cmsetup.gen_users(2)
@@ -243,6 +244,7 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
pytest.fail("Rate limit was not exceeded")
@pytest.mark.slow
def test_expunged(remote, chatmail_config):
outdated_days = int(chatmail_config.delete_mails_after) + 1
find_cmds = [

View File

@@ -88,9 +88,12 @@ class TestEndToEndDeltaChat:
return int(float(number) * units[unit])
quota = parse_size_limit(chatmail_config.max_mailbox_size)
# Dovecot quota is inflated to 140% of the configured limit
# so that quota-expire keeps users below the warning threshold.
dovecot_quota = quota * 140 // 100
lp.sec(f"filling remote inbox for {user}")
fn = f"7743102289.M843172P2484002.c20,S={quota},W=2398:2,"
fn = f"7743102289.M843172P2484002.c20,S={dovecot_quota},W=2398:2,"
path = chatmail_config.mailboxes_dir.joinpath(user, "cur", fn)
sshexec = get_sshexec(sshdomain)
sshexec(call=rshell.write_numbytes, kwargs=dict(path=str(path), num=120))

View File

@@ -23,6 +23,12 @@ def _is_ip(domain):
return False
def pytest_addoption(parser):
parser.addoption(
"--slow", action="store_true", default=False, help="also run slow tests"
)
def pytest_configure(config):
config._benchresults = {}
config.addinivalue_line(
@@ -30,6 +36,13 @@ def pytest_configure(config):
)
def pytest_runtest_setup(item):
markers = list(item.iter_markers(name="slow"))
if markers:
if not item.config.getoption("--slow"):
pytest.skip("skipping slow test, use --slow to run")
def _get_chatmail_config():
inipath = os.environ.get("CHATMAIL_INI")
if inipath:

View File

@@ -106,8 +106,11 @@ short overview of ``chatmaild`` services:
of users who have not logged in for longer than
``delete_inactive_users_after`` days.
- ``chatmail-quota-expire`` is called by Dovecot's ``quota_warning`` mechanism
and will automatically remove oldest messages to keep mailboxes well under ``max_mailbox_size``.
- ``chatmail-quota-expire``
is called by Dovecot's ``quota_warning`` mechanism when a
mailbox exceeds ``max_mailbox_size``.
It removes the oldest messages
until usage drops to a safe level.
- `lastlogin <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/lastlogin.py>`_
is contacted by Dovecot when a user logs in and stores the date of