feat: add quota-triggered per-user mailbox cleanup

Inflate the Dovecot-visible quota to 140% of the configured
max_mailbox_size so that Delta Chat clients (which warn at
80% of IMAP-reported quota) never show quota warnings.
A quota_warning at 72% of the inflated limit triggers
chatmail-quota-expire, which trims the mailbox to 80% of the
configured limit.  Existing over-quota mailboxes start
receiving mail again immediately after deploy
without any manual operator action needed.

CHANGELOG.md

@@ -1,5 +1,20 @@
 # Changelog for chatmail deployment 
 
+## Unreleased
+
+### Features
+
+- Add per-user quota-triggered cleanup (`chatmail-quota-expire`).
+  When a mailbox exceeds the configured ``max_mailbox_size``,
+  Dovecot runs the new script which removes the oldest
+  messages until usage drops to a safe level.
+  No operator action is required after upgrading;
+  existing over-quota mailboxes start receiving mail
+  again immediately and are cleaned up automatically.
+  The daily `chatmail-expire` timer continues to handle
+  deletion of old messages, large messages,
+  and inactive user mailboxes.
+
 ## 1.9.0 2025-12-18
 
 ### Documentation

chatmaild/pyproject.toml

@@ -22,6 +22,7 @@ where = ['src']
 doveauth = "chatmaild.doveauth:main"
 chatmail-metadata = "chatmaild.metadata:main"
 chatmail-expire = "chatmaild.expire:main"
+chatmail-quota-expire = "chatmaild.quota_expire:main"
 chatmail-fsreport = "chatmaild.fsreport:main"
 lastlogin = "chatmaild.lastlogin:main"
 turnserver = "chatmaild.turnserver:main"

chatmaild/src/chatmaild/config.py

@@ -95,6 +95,11 @@ class Config:
         # old unused option (except for first migration from sqlite to maildir store)
         self.passdb_path = Path(params.get("passdb_path", "/home/vmail/passdb.sqlite"))
 
+    @property
+    def max_mailbox_size_mb(self):
+        """Return max_mailbox_size as an integer in megabytes."""
+        return parse_size_mb(self.max_mailbox_size)
+
     def _getbytefile(self):
         return open(self._inipath, "rb")
 
@@ -108,6 +113,16 @@ class Config:
         return User(maildir, addr, password_path, uid="vmail", gid="vmail")
 
 
+def parse_size_mb(limit):
+    """Parse a size string like ``500M`` or ``2G`` and return megabytes."""
+    value = limit.strip().upper().rstrip("B")
+    if value.endswith("G"):
+        return int(value[:-1]) * 1024
+    if value.endswith("M"):
+        return int(value[:-1])
+    return int(value)
+
+
 def write_initial_config(inipath, mail_domain, overrides):
     """Write out default config file, using the specified config value overrides."""
     content = get_default_config_content(mail_domain, **overrides)

chatmaild/src/chatmaild/quota_expire.py

@@ -0,0 +1,90 @@
+"""
+Quota-triggered per-user mailbox cleanup.
+
+Dovecot calls this script via ``quota_warning``
+when a user crosses the quota threshold.
+The script removes oldest messages first
+to keep the mailbox under a specified target size.
+
+Usage::
+
+    chatmail-quota-expire <target_mb> <mailbox_path>
+
+"""
+
+import sys
+from argparse import ArgumentParser
+from pathlib import Path
+
+from chatmaild.expire import get_file_entry, os_listdir_if_exists
+
+
+def scan_mailbox_messages(mailbox_dir):
+    """Collect FileEntry items from top-level cur/new/tmp only."""
+    mbox = Path(mailbox_dir)
+    messages = []
+    for sub in ("cur", "new", "tmp"):
+        for name in os_listdir_if_exists(mbox / sub):
+            if entry := get_file_entry(str(mbox / sub / name)):
+                messages.append(entry)
+    return messages
+
+
+def expire_to_target(mailbox_dir, target_bytes):
+    """Remove oldest files until total size <= *target_bytes*.
+
+    Returns ``(removed_count, cache_bytes)`` where *cache_bytes*
+    is the size of the deleted ``dovecot.index.cache`` file
+    (0 when the file did not exist).
+    """
+    mbox = Path(mailbox_dir)
+    messages = scan_mailbox_messages(mbox)
+    total_size = sum(m.size for m in messages)
+    removed = 0
+    for count, entry in enumerate(sorted(messages, key=lambda m: m.mtime), 1):
+        if total_size <= target_bytes:
+            break
+        Path(entry.path).unlink(missing_ok=True)
+        total_size -= entry.size
+        removed = count
+
+    (mbox / "maildirsize").unlink(missing_ok=True)
+    cache = mbox / "dovecot.index.cache"
+    try:
+        cache_bytes = cache.stat().st_size
+    except FileNotFoundError:
+        cache_bytes = 0
+    cache.unlink(missing_ok=True)
+    return removed, cache_bytes
+
+
+def main(args=None):
+    """Remove mailbox messages to stay within a megabyte target."""
+    parser = ArgumentParser(description=main.__doc__)
+    parser.add_argument(
+        "target_mb",
+        type=int,
+        help="target mailbox size in megabytes",
+    )
+    parser.add_argument(
+        "mailbox_path",
+        help="path to a user mailbox",
+    )
+    args = parser.parse_args(args)
+
+    target_bytes = args.target_mb * 1024 * 1024
+
+    removed_count, cache_bytes = expire_to_target(args.mailbox_path, target_bytes)
+    if removed_count:
+        user = Path(args.mailbox_path).name
+        cache_mb = cache_bytes / 1024 / 1024
+        print(
+            f"quota-expire: removed {removed_count} message(s) from {user}"
+            f" cache={cache_mb:.1f}MB",
+            file=sys.stderr,
+        )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

chatmaild/src/chatmaild/tests/test_config.py

@@ -1,6 +1,6 @@
 import pytest
 
-from chatmaild.config import read_config
+from chatmaild.config import parse_size_mb, read_config
 
 
 def test_read_config_basic(example_config):
@@ -121,3 +121,17 @@ def test_config_tls_external_bad_format(make_config):
                 "tls_external_cert_and_key": "/only/one/path.pem",
             },
         )
+
+
+def test_parse_size_mb():
+    assert parse_size_mb("500M") == 500
+    assert parse_size_mb("2G") == 2048
+    assert parse_size_mb("  1g  ") == 1024
+    assert parse_size_mb("100MB") == 100
+    assert parse_size_mb("256") == 256
+
+
+def test_max_mailbox_size_mb(make_config):
+    config = make_config("chat.example.org")
+    assert config.max_mailbox_size == "500M"
+    assert config.max_mailbox_size_mb == 500

chatmaild/src/chatmaild/tests/test_quota_expire.py

@@ -0,0 +1,73 @@
+import os
+import time
+
+from chatmaild.quota_expire import expire_to_target, main, scan_mailbox_messages
+
+MB = 1024 * 1024
+
+
+def _create_message(basedir, relpath, size, days_old=0):
+    path = basedir / relpath
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_bytes(b"x" * size)
+    mtime = time.time() - days_old * 86400
+    os.utime(path, (mtime, mtime))
+    return path
+
+
+def test_scan_cur_new_tmp(tmp_path):
+    _create_message(tmp_path, "cur/msg1", 100)
+    _create_message(tmp_path, "new/msg2", 200)
+    _create_message(tmp_path, "tmp/msg3", 300)
+    assert len(scan_mailbox_messages(str(tmp_path))) == 3
+
+
+def test_scan_ignores_subfolders(tmp_path):
+    _create_message(tmp_path, "cur/a", 10)
+    _create_message(tmp_path, ".DeltaChat/cur/b", 20)
+    assert len(scan_mailbox_messages(str(tmp_path))) == 1
+
+
+def test_removes_to_target(tmp_path):
+    for i in range(15):
+        _create_message(tmp_path, f"cur/msg{i:02d}", MB, days_old=i + 1)
+    removed, _ = expire_to_target(str(tmp_path), 10 * MB)
+    assert removed == 5
+    assert len(scan_mailbox_messages(str(tmp_path))) == 10
+
+
+def test_removes_oldest_first(tmp_path):
+    _create_message(tmp_path, "cur/old_small", MB, days_old=30)
+    _create_message(tmp_path, "cur/new_huge", 10 * MB, days_old=1)
+    # the 10MB file is kept, the 1MB file is removed because it's older
+    removed, _ = expire_to_target(str(tmp_path), 10 * MB)
+    assert removed == 1
+    assert not (tmp_path / "cur/old_small").exists()
+    assert (tmp_path / "cur/new_huge").exists()
+
+
+def test_exact_limit(tmp_path):
+    _create_message(tmp_path, "cur/msg1", 5 * MB)
+    removed, _ = expire_to_target(str(tmp_path), 5 * MB)
+    assert removed == 0
+
+
+def test_removes_stale_caches(tmp_path):
+    _create_message(tmp_path, "cur/msg1", 2 * MB, days_old=5)
+    (tmp_path / "maildirsize").write_text("x")
+    (tmp_path / "dovecot.index.cache").write_bytes(b"y" * 4096)
+    removed, cache_bytes = expire_to_target(str(tmp_path), MB)
+    assert removed == 1
+    assert cache_bytes == 4096
+    assert not (tmp_path / "maildirsize").exists()
+    assert not (tmp_path / "dovecot.index.cache").exists()
+
+
+def test_logging_output_is_mtail_compatible(tmp_path, capsys):
+    mbox = tmp_path / "user@example.org"
+    _create_message(mbox, "cur/msg1", 2 * MB, days_old=5)
+    (mbox / "dovecot.index.cache").write_bytes(b"c" * 2 * MB)
+    main([str(1), str(mbox)])
+    _, err = capsys.readouterr()
+    assert "quota-expire: removed 1 message(s) from user@example.org" in err
+    assert "cache=2.0MB" in err

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -149,12 +149,26 @@ plugin {
 }
 
 plugin {
-  # for now we define static quota-rules for all users 
   quota = maildir:User quota
-  quota_rule = *:storage={{ config.max_mailbox_size }}
   quota_max_mail_size={{ config.max_message_size }}
   quota_grace = 0
-  # quota_over_flag_value = TRUE
+
+  # Inflate the dovecot-visible quota so that Delta Chat clients
+  # (which warn at 80% of the IMAP-reported limit) never see
+  # quota warnings -- expire kicks in well before that point.
+  quota_rule = *:storage={{ config.max_mailbox_size_mb * 140 // 100 }}M
+
+  # Trigger when usage reaches the configured max_mailbox_size
+  # (72% of inflated = ~100% of configured), then expire oldest
+  # messages down to 80% of the configured max_mailbox_size.
+  quota_warning = storage=72%% quota-warning {{ config.max_mailbox_size_mb * 80 // 100 }} {{ config.mailboxes_dir }}/%u
+}
+
+service quota-warning {
+  executable = script /usr/local/lib/chatmaild/venv/bin/chatmail-quota-expire
+  user = vmail
+  unix_listener quota-warning {
+  }
 }
 
 # push_notification configuration

cmdeploy/src/cmdeploy/mtail/delivered_mail.mtail

@@ -78,3 +78,11 @@ counter rejected_unencrypted_mail_count
 /Rejected unencrypted mail/ {
   rejected_unencrypted_mail_count++
 }
+
+counter quota_expire_runs
+counter quota_expire_removed_files
+
+/quota-expire: removed (?P<count>\d+) message\(s\)/ {
+  quota_expire_runs++
+  quota_expire_removed_files += $count
+}

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -88,9 +88,12 @@ class TestEndToEndDeltaChat:
             return int(float(number) * units[unit])
 
         quota = parse_size_limit(chatmail_config.max_mailbox_size)
+        # Dovecot quota is inflated to 140% of the configured limit
+        # so that quota-expire keeps users below the warning threshold.
+        dovecot_quota = quota * 140 // 100
 
         lp.sec(f"filling remote inbox for {user}")
-        fn = f"7743102289.M843172P2484002.c20,S={quota},W=2398:2,"
+        fn = f"7743102289.M843172P2484002.c20,S={dovecot_quota},W=2398:2,"
         path = chatmail_config.mailboxes_dir.joinpath(user, "cur", fn)
         sshexec = get_sshexec(sshdomain)
         sshexec(call=rshell.write_numbytes, kwargs=dict(path=str(path), num=120))

doc/source/overview.rst

@@ -102,8 +102,15 @@ short overview of ``chatmaild`` services:
    Apple/Google/Huawei.
 
 -  `chatmail-expire <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/expire.py>`_
-   deletes users if they have not logged in for a longer while.
+   deletes old messages, large messages, and entire mailboxes
-   The timeframe can be configured in ``chatmail.ini``.
+   of users who have not logged in for longer than
+   ``delete_inactive_users_after`` days.
+
+-  ``chatmail-quota-expire``
+   is called by Dovecot's ``quota_warning`` mechanism when a
+   mailbox exceeds ``max_mailbox_size``.
+   It removes the oldest messages
+   until usage drops to a safe level.
 
 -  `lastlogin <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/lastlogin.py>`_
    is contacted by Dovecot when a user logs in and stores the date of
@@ -156,6 +163,8 @@ Chatmail relay dependency diagram
         /home/vmail/.../user"];
         dovecot --- |lastlogin.socket|lastlogin;
         dovecot --- chatmail-metadata;
+        dovecot --- |quota-warning|chatmail-quota-expire;
+        chatmail-quota-expire --- maildir;
         lastlogin --- maildir;
         doveauth --- maildir;
         chatmail-expire-daily --- maildir;