feat: Automatic per-user quota-preservation.

Replace daily timer-based message expire script
with Dovecot quota-warning-triggered cleanup.
When a user reaches 90% of their mailbox quota
Dovecot calls the new script which removes the largest and oldest messages
until usage drops below 80%.

The daily `chatmail-expire` service now only
handles deletion of inactive user mailboxes.

CHANGELOG.md

@@ -1,5 +1,24 @@
 # Changelog for chatmail deployment 
 
+## Unreleased
+
+### Features
+
+- Automated per-user quota-keeping.
+  Replace daily timer-based message expire script
+  with Dovecot quota-warning-triggered cleanup (`chatmail-quota-expire`).
+  When a user reaches 90% of their mailbox quota
+  Dovecot calls the new script which removes the largest and oldest messages
+  until usage drops below 80%.
+  The daily `chatmail-expire` timer now only handles deletion
+  of inactive user mailboxes.
+
+  After upgrading, run the following once to clean up
+  mailboxes that are already over quota::
+
+      /usr/local/lib/chatmaild/venv/bin/chatmail-quota-expire \
+          400 /home/vmail/mail/YOURDOMAIN --sweep
+
 ## 1.9.0 2025-12-18
 
 ### Documentation

chatmaild/pyproject.toml

@@ -21,7 +21,8 @@ where = ['src']
 [project.scripts]
 doveauth = "chatmaild.doveauth:main"
 chatmail-metadata = "chatmaild.metadata:main"
-chatmail-expire = "chatmaild.expire:main"
+chatmail-expire = "chatmaild.expire_inactive_users:main"
+chatmail-quota-expire = "chatmaild.quota_expire:main"
 chatmail-fsreport = "chatmaild.fsreport:main"
 lastlogin = "chatmaild.lastlogin:main"
 turnserver = "chatmaild.turnserver:main"

chatmaild/src/chatmaild/config.py

@@ -25,8 +25,6 @@ class Config:
         self.max_user_send_burst_size = int(params.get("max_user_send_burst_size", 10))
         self.max_mailbox_size = params["max_mailbox_size"]
         self.max_message_size = int(params.get("max_message_size", "31457280"))
-        self.delete_mails_after = params["delete_mails_after"]
-        self.delete_large_after = params["delete_large_after"]
         self.delete_inactive_users_after = int(params["delete_inactive_users_after"])
         self.username_min_length = int(params["username_min_length"])
         self.username_max_length = int(params["username_max_length"])
@@ -95,6 +93,11 @@ class Config:
         # old unused option (except for first migration from sqlite to maildir store)
         self.passdb_path = Path(params.get("passdb_path", "/home/vmail/passdb.sqlite"))
 
+    @property
+    def max_mailbox_size_mb(self):
+        """Return max_mailbox_size as an integer in megabytes."""
+        return parse_size_mb(self.max_mailbox_size)
+
     def _getbytefile(self):
         return open(self._inipath, "rb")
 
@@ -108,6 +111,16 @@ class Config:
         return User(maildir, addr, password_path, uid="vmail", gid="vmail")
 
 
+def parse_size_mb(limit):
+    """Parse a size string like ``500M`` or ``2G`` and return megabytes."""
+    value = limit.strip().upper().rstrip("B")
+    if value.endswith("G"):
+        return int(value[:-1]) * 1024
+    if value.endswith("M"):
+        return int(value[:-1])
+    return int(value)
+
+
 def write_initial_config(inipath, mail_domain, overrides):
     """Write out default config file, using the specified config value overrides."""
     content = get_default_config_content(mail_domain, **overrides)

chatmaild/src/chatmaild/expire.py

@@ -115,11 +115,8 @@ class Expiry:
         cutoff_without_login = (
             self.now - int(self.config.delete_inactive_users_after) * 86400
         )
-        cutoff_mails = self.now - int(self.config.delete_mails_after) * 86400
-        cutoff_large_mails = self.now - int(self.config.delete_large_after) * 86400
 
         self.all_mboxes += 1
-        changed = False
         if mbox.last_login and mbox.last_login < cutoff_without_login:
             self.remove_mailbox(mbox.basedir)
             return
@@ -131,25 +128,10 @@ class Expiry:
                 print_info(f"checking mailbox {date.strftime('%b %d')} {mboxname}")
             else:
                 print_info(f"checking mailbox (no last_login) {mboxname}")
-        self.all_files += len(mbox.messages)
-        for message in mbox.messages:
-            if message.mtime < cutoff_mails:
-                self.remove_file(message.path, mtime=message.mtime)
-            elif message.size > 200000 and message.mtime < cutoff_large_mails:
-                # we only remove noticed large files (not unnoticed ones in new/)
-                parts = message.path.split("/")
-                if len(parts) >= 2 and parts[-2] == "cur":
-                    self.remove_file(message.path, mtime=message.mtime)
-            else:
-                continue
-            changed = True
-        if changed:
-            self.remove_file(f"{mbox.basedir}/maildirsize")
 
     def get_summary(self):
         return (
             f"Removed {self.del_mboxes} out of {self.all_mboxes} mailboxes "
-            f"and {self.del_files} out of {self.all_files} files in existing mailboxes "
             f"in {time.time() - self.start:2.2f} seconds"
         )
 

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -23,12 +23,6 @@ max_mailbox_size = 500M
 # maximum message size for an e-mail in bytes
 max_message_size = 31457280
 
-# days after which mails are unconditionally deleted
-delete_mails_after = 20
-
-# days after which large messages (>200k) are unconditionally deleted
-delete_large_after = 7
-
 # days after which users without a successful login are deleted (database and mails)
 delete_inactive_users_after = 90
 

chatmaild/src/chatmaild/quota_expire.py

@@ -0,0 +1,152 @@
+"""
+Remove messages from a mailbox to meet a size target.
+
+Dovecot calls this script when a user's quota is near its limit.
+Files are scored by ``size * age`` so that large, old messages
+are removed first.
+
+Usage::
+
+    quota_expire <target_mb> <mailbox_path>
+
+"""
+
+import os
+import sys
+import time
+from argparse import ArgumentParser
+from collections import namedtuple
+from stat import S_ISREG
+
+FileEntry = namedtuple("FileEntry", ("path", "mtime", "size"))
+
+
+def _get_file_entry(path):
+    try:
+        st = os.stat(path)
+    except FileNotFoundError:
+        return None
+    if not S_ISREG(st.st_mode):
+        return None
+    return FileEntry(path, st.st_mtime, st.st_size)
+
+
+def _listdir(path):
+    try:
+        return os.listdir(path)
+    except FileNotFoundError:
+        return []
+
+
+def scan_mailbox_messages(mailbox_dir):
+    messages = []
+    for sub in ("cur", "new", "tmp"):
+        subdir = f"{mailbox_dir}/{sub}"
+        for name in _listdir(subdir):
+            entry = _get_file_entry(f"{subdir}/{name}")
+            if entry is not None:
+                messages.append(entry)
+    return messages
+
+
+def _remove_stale_caches(mailbox_dir):
+    for name in ("maildirsize", "dovecot.index.cache"):
+        try:
+            os.unlink(f"{mailbox_dir}/{name}")
+        except FileNotFoundError:
+            pass
+
+
+def expire_to_target(mailbox_dir, target_bytes, now=None):
+    """Remove highest-scored files until total size <= *target_bytes*.
+
+    Returns the list of removed file paths.
+    """
+    if now is None:
+        now = time.time()
+
+    messages = scan_mailbox_messages(mailbox_dir)
+    total_size = sum(m.size for m in messages)
+
+    if total_size <= target_bytes:
+        return []
+
+    # Score: large and old files get the highest score.
+    scored = sorted(
+        messages,
+        key=lambda m: m.size * (now - m.mtime),
+        reverse=True,
+    )
+
+    removed = []
+    for entry in scored:
+        if total_size <= target_bytes:
+            break
+        try:
+            os.unlink(entry.path)
+        except FileNotFoundError:
+            continue
+        total_size -= entry.size
+        removed.append(entry.path)
+
+    if removed:
+        _remove_stale_caches(mailbox_dir)
+
+    return removed
+
+
+def main(args=None):
+    """Remove mailbox messages to stay within a megabyte target."""
+    parser = ArgumentParser(description=main.__doc__)
+    parser.add_argument(
+        "target_mb",
+        type=int,
+        help="target mailbox size in megabytes",
+    )
+    parser.add_argument(
+        "mailbox_path",
+        help="path to a user mailbox, or with --sweep the mailboxes directory",
+    )
+    parser.add_argument(
+        "--sweep",
+        action="store_true",
+        help="sweep all mailboxes under mailbox_path",
+    )
+    args = parser.parse_args(args)
+
+    target_bytes = args.target_mb * 1024 * 1024
+
+    if args.sweep:
+        return _sweep(args.mailbox_path, target_bytes)
+
+    removed = expire_to_target(args.mailbox_path, target_bytes)
+    if removed:
+        print(
+            f"removed {len(removed)} file(s) from {args.mailbox_path}"
+            f" to reach {args.target_mb} MB target",
+            file=sys.stderr,
+        )
+    return 0
+
+
+def _sweep(mailboxes_dir, target_bytes):
+    try:
+        names = os.listdir(mailboxes_dir)
+    except FileNotFoundError:
+        print(f"directory not found: {mailboxes_dir}", file=sys.stderr)
+        return 1
+    for name in sorted(names):
+        if "@" not in name:
+            continue
+        mbox = f"{mailboxes_dir}/{name}"
+        removed = expire_to_target(mbox, target_bytes)
+        if removed:
+            print(
+                f"removed {len(removed)} file(s) from {name}",
+                file=sys.stderr,
+            )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

chatmaild/src/chatmaild/tests/test_config.py

@@ -34,8 +34,6 @@ def test_read_config_testrun(make_config):
     assert config.postfix_reinject_port == 10025
     assert config.max_user_send_per_minute == 60
     assert config.max_mailbox_size == "500M"
-    assert config.delete_mails_after == "20"
-    assert config.delete_large_after == "7"
     assert config.username_min_length == 9
     assert config.username_max_length == 9
     assert config.password_min_length == 9

chatmaild/src/chatmaild/tests/test_expire.py

@@ -1,7 +1,6 @@
 import os
 import random
 from datetime import datetime
-from fnmatch import fnmatch
 from pathlib import Path
 
 import pytest
@@ -154,35 +153,6 @@ def test_expiry_cli_basic(example_config, mbox1):
     expiry_main(args)
 
 
-def test_expiry_cli_old_files(capsys, example_config, mbox1):
-    relpaths_old = ["cur/msg_old1", "cur/msg_old1"]
-    cutoff_days = int(example_config.delete_mails_after) + 1
-    create_new_messages(mbox1.basedir, relpaths_old, size=1000, days=cutoff_days)
-
-    relpaths_large = ["cur/msg_old_large1", "new/msg_old_large2"]
-    cutoff_days = int(example_config.delete_large_after) + 1
-    create_new_messages(
-        mbox1.basedir, relpaths_large, size=1000 * 300, days=cutoff_days
-    )
-
-    create_new_messages(mbox1.basedir, ["cur/shouldstay"], size=1000 * 300, days=1)
-
-    args = str(example_config._inipath), "--remove", "-v"
-    expiry_main(args)
-    out, err = capsys.readouterr()
-
-    allpaths = relpaths_old + relpaths_large + ["maildirsize"]
-    for path in allpaths:
-        for line in err.split("\n"):
-            if fnmatch(line, f"removing*{path}"):
-                break
-        else:
-            if path != "new/msg_old_large2":
-                pytest.fail(f"failed to remove {path}\n{err}")
-
-    assert "shouldstay" not in err
-
-
 def test_get_file_entry(tmp_path):
     assert get_file_entry(str(tmp_path.joinpath("123123"))) is None
     p = tmp_path.joinpath("x")

chatmaild/src/chatmaild/tests/test_quota_expire.py

@@ -0,0 +1,91 @@
+import os
+import time
+
+from chatmaild.quota_expire import expire_to_target, scan_mailbox_messages
+
+MB = 1024 * 1024
+
+
+def _create_message(basedir, relpath, size, days_old=0):
+    path = basedir / relpath
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_bytes(b"x" * size)
+    mtime = time.time() - days_old * 86400
+    os.utime(path, (mtime, mtime))
+    return path
+
+
+def test_scan_cur_new_tmp(tmp_path):
+    _create_message(tmp_path, "cur/msg1", 100)
+    _create_message(tmp_path, "new/msg2", 200)
+    _create_message(tmp_path, "tmp/msg3", 300)
+    messages = scan_mailbox_messages(str(tmp_path))
+    assert len(messages) == 3
+    sizes = sorted(m.size for m in messages)
+    assert sizes == [100, 200, 300]
+
+
+def test_scan_ignores_subfolders(tmp_path):
+    _create_message(tmp_path, "cur/a", 10)
+    _create_message(tmp_path, ".DeltaChat/cur/b", 20)
+    assert len(scan_mailbox_messages(str(tmp_path))) == 1
+
+
+def test_scan_empty(tmp_path):
+    assert scan_mailbox_messages(str(tmp_path)) == []
+    assert scan_mailbox_messages(str(tmp_path / "nope")) == []
+
+
+def test_noop_under_limit(tmp_path):
+    _create_message(tmp_path, "cur/msg1", MB)
+    assert expire_to_target(str(tmp_path), 2 * MB) == []
+    assert (tmp_path / "cur" / "msg1").exists()
+
+
+def test_removes_to_target(tmp_path):
+    now = time.time()
+    for i in range(15):
+        _create_message(tmp_path, f"cur/msg{i:02d}", MB, days_old=i + 1)
+    removed = expire_to_target(str(tmp_path), 10 * MB, now=now)
+    assert len(removed) == 5
+    assert len(scan_mailbox_messages(str(tmp_path))) == 10
+
+
+def test_scoring_prefers_large_old(tmp_path):
+    now = time.time()
+    _create_message(tmp_path, "cur/large_old", 2 * MB, days_old=30)
+    _create_message(tmp_path, "cur/small_new", MB, days_old=1)
+    removed = expire_to_target(str(tmp_path), 2 * MB, now=now)
+    assert len(removed) == 1
+    assert "large_old" in removed[0]
+
+
+def test_scoring_large_new_beats_small_old(tmp_path):
+    now = time.time()
+    _create_message(tmp_path, "cur/big_new", 10 * MB, days_old=1)
+    _create_message(tmp_path, "cur/small_old", MB, days_old=5)
+    # big_new score: 10MB * 1d = 10  vs  small_old score: 1MB * 5d = 5
+    removed = expire_to_target(str(tmp_path), 10 * MB, now=now)
+    assert len(removed) == 1
+    assert "big_new" in removed[0]
+
+
+def test_exact_limit(tmp_path):
+    _create_message(tmp_path, "cur/msg1", 5 * MB)
+    assert expire_to_target(str(tmp_path), 5 * MB) == []
+
+
+def test_removes_stale_caches(tmp_path):
+    _create_message(tmp_path, "cur/msg1", 2 * MB, days_old=5)
+    (tmp_path / "maildirsize").write_text("x")
+    (tmp_path / "dovecot.index.cache").write_text("x")
+    expire_to_target(str(tmp_path), MB)
+    assert not (tmp_path / "maildirsize").exists()
+    assert not (tmp_path / "dovecot.index.cache").exists()
+
+
+def test_no_cache_removal_when_under_limit(tmp_path):
+    _create_message(tmp_path, "cur/msg1", MB)
+    (tmp_path / "maildirsize").write_text("x")
+    expire_to_target(str(tmp_path), 2 * MB)
+    assert (tmp_path / "maildirsize").exists()

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -149,12 +149,22 @@ plugin {
 }
 
 plugin {
-  # for now we define static quota-rules for all users 
+  # for now we define static quota-rules for all users
   quota = maildir:User quota
   quota_rule = *:storage={{ config.max_mailbox_size }}
   quota_max_mail_size={{ config.max_message_size }}
   quota_grace = 0
-  # quota_over_flag_value = TRUE
+
+  # When a user reaches 90% quota, run chatmail-quota-expire
+  # to remove large/old messages until usage is below 80%.
+  quota_warning = storage=90%% quota-warning {{ config.max_mailbox_size_mb * 80 // 100 }} {{ config.mailboxes_dir }}/%u
+}
+
+service quota-warning {
+  executable = script /usr/local/lib/chatmaild/venv/bin/chatmail-quota-expire
+  user = vmail
+  unix_listener quota-warning {
+  }
 }
 
 # push_notification configuration

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -244,24 +244,6 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
     pytest.fail("Rate limit was not exceeded")
 
 
-@pytest.mark.slow
-def test_expunged(remote, chatmail_config):
-    outdated_days = int(chatmail_config.delete_mails_after) + 1
-    find_cmds = [
-        f"find {chatmail_config.mailboxes_dir} -path '*/cur/*' -mtime +{outdated_days} -type f",
-        f"find {chatmail_config.mailboxes_dir} -path '*/.*/cur/*' -mtime +{outdated_days} -type f",
-        f"find {chatmail_config.mailboxes_dir} -path '*/new/*' -mtime +{outdated_days} -type f",
-        f"find {chatmail_config.mailboxes_dir} -path '*/.*/new/*' -mtime +{outdated_days} -type f",
-        f"find {chatmail_config.mailboxes_dir} -path '*/tmp/*' -mtime +{outdated_days} -type f",
-        f"find {chatmail_config.mailboxes_dir} -path '*/.*/tmp/*' -mtime +{outdated_days} -type f",
-    ]
-    outdated_days = int(chatmail_config.delete_large_after) + 1
-    find_cmds.append(
-        f"find {chatmail_config.mailboxes_dir} -path '*/cur/*' -mtime +{outdated_days} -size +200k -type f"
-    )
-    for cmd in find_cmds:
-        for line in remote.iter_output(cmd):
-            assert not line
 
 
 def test_deployed_state(remote):

doc/source/overview.rst

@@ -102,8 +102,14 @@ short overview of ``chatmaild`` services:
    Apple/Google/Huawei.
 
 -  `chatmail-expire <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/expire.py>`_
-   deletes users if they have not logged in for a longer while.
-   The timeframe can be configured in ``chatmail.ini``.
+   deletes entire mailboxes of users who have not logged in
+   for longer than ``delete_inactive_users_after`` days.
+
+-  `chatmail-quota-expire <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/quota_expire.py>`_
+   is called by Dovecot's ``quota_warning`` mechanism when a
+   user reaches 90% of their mailbox quota.
+   It removes the largest and oldest messages
+   until usage drops below 80% of the quota.
 
 -  `lastlogin <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/lastlogin.py>`_
    is contacted by Dovecot when a user logs in and stores the date of
@@ -139,7 +145,7 @@ Chatmail relay dependency diagram
         certs-nginx[("`TLS certs
         /var/lib/acme`")] --> nginx-internal;
         systemd-timer --- acmetool;
-        systemd-timer --- chatmail-expire-daily;
+        systemd-timer --- chatmail-expire-inactive;
         systemd-timer --- chatmail-fsreport-daily;
         acmetool --> certs[("`TLS certs
         /var/lib/acme`")];
@@ -156,9 +162,11 @@ Chatmail relay dependency diagram
         /home/vmail/.../user"];
         dovecot --- |lastlogin.socket|lastlogin;
         dovecot --- chatmail-metadata;
+        dovecot --- |quota-warning|chatmail-quota-expire;
+        chatmail-quota-expire --- maildir;
         lastlogin --- maildir;
         doveauth --- maildir;
-        chatmail-expire-daily --- maildir;
+        chatmail-expire-inactive --- maildir;
         chatmail-fsreport-daily --- maildir;
         chatmail-metadata --- iroh-relay;
         chatmail-metadata --- |encrypted device token| notifications.delta.chat;