mirror of
https://github.com/chatmail/relay.git
synced 2026-05-23 14:28:02 +00:00
Compare commits
1 Commits
quota_expi
...
add-quota-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4512d1e735 |
24
CHANGELOG.md
24
CHANGELOG.md
@@ -4,20 +4,16 @@
|
|||||||
|
|
||||||
### Features
|
### Features
|
||||||
|
|
||||||
- Automated per-user quota-keeping.
|
- Add per-user quota-triggered cleanup (`chatmail-quota-expire`).
|
||||||
Replace daily timer-based message expire script
|
When a mailbox exceeds the configured ``max_mailbox_size``,
|
||||||
with Dovecot quota-warning-triggered cleanup (`chatmail-quota-expire`).
|
Dovecot runs the new script which removes the oldest
|
||||||
When a user reaches 90% of their mailbox quota
|
messages until usage drops to a safe level.
|
||||||
Dovecot calls the new script which removes the largest and oldest messages
|
No operator action is required after upgrading;
|
||||||
until usage drops below 80%.
|
existing over-quota mailboxes start receiving mail
|
||||||
The daily `chatmail-expire` timer now only handles deletion
|
again immediately and are cleaned up automatically.
|
||||||
of inactive user mailboxes.
|
The daily `chatmail-expire` timer continues to handle
|
||||||
|
deletion of old messages, large messages,
|
||||||
After upgrading, run the following once to clean up
|
and inactive user mailboxes.
|
||||||
mailboxes that are already over quota::
|
|
||||||
|
|
||||||
/usr/local/lib/chatmaild/venv/bin/chatmail-quota-expire \
|
|
||||||
400 /home/vmail/mail/YOURDOMAIN --sweep
|
|
||||||
|
|
||||||
## 1.9.0 2025-12-18
|
## 1.9.0 2025-12-18
|
||||||
|
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ where = ['src']
|
|||||||
[project.scripts]
|
[project.scripts]
|
||||||
doveauth = "chatmaild.doveauth:main"
|
doveauth = "chatmaild.doveauth:main"
|
||||||
chatmail-metadata = "chatmaild.metadata:main"
|
chatmail-metadata = "chatmaild.metadata:main"
|
||||||
chatmail-expire = "chatmaild.expire_inactive_users:main"
|
chatmail-expire = "chatmaild.expire:main"
|
||||||
chatmail-quota-expire = "chatmaild.quota_expire:main"
|
chatmail-quota-expire = "chatmaild.quota_expire:main"
|
||||||
chatmail-fsreport = "chatmaild.fsreport:main"
|
chatmail-fsreport = "chatmaild.fsreport:main"
|
||||||
lastlogin = "chatmaild.lastlogin:main"
|
lastlogin = "chatmaild.lastlogin:main"
|
||||||
|
|||||||
@@ -25,6 +25,8 @@ class Config:
|
|||||||
self.max_user_send_burst_size = int(params.get("max_user_send_burst_size", 10))
|
self.max_user_send_burst_size = int(params.get("max_user_send_burst_size", 10))
|
||||||
self.max_mailbox_size = params["max_mailbox_size"]
|
self.max_mailbox_size = params["max_mailbox_size"]
|
||||||
self.max_message_size = int(params.get("max_message_size", "31457280"))
|
self.max_message_size = int(params.get("max_message_size", "31457280"))
|
||||||
|
self.delete_mails_after = params["delete_mails_after"]
|
||||||
|
self.delete_large_after = params["delete_large_after"]
|
||||||
self.delete_inactive_users_after = int(params["delete_inactive_users_after"])
|
self.delete_inactive_users_after = int(params["delete_inactive_users_after"])
|
||||||
self.username_min_length = int(params["username_min_length"])
|
self.username_min_length = int(params["username_min_length"])
|
||||||
self.username_max_length = int(params["username_max_length"])
|
self.username_max_length = int(params["username_max_length"])
|
||||||
|
|||||||
@@ -115,8 +115,11 @@ class Expiry:
|
|||||||
cutoff_without_login = (
|
cutoff_without_login = (
|
||||||
self.now - int(self.config.delete_inactive_users_after) * 86400
|
self.now - int(self.config.delete_inactive_users_after) * 86400
|
||||||
)
|
)
|
||||||
|
cutoff_mails = self.now - int(self.config.delete_mails_after) * 86400
|
||||||
|
cutoff_large_mails = self.now - int(self.config.delete_large_after) * 86400
|
||||||
|
|
||||||
self.all_mboxes += 1
|
self.all_mboxes += 1
|
||||||
|
changed = False
|
||||||
if mbox.last_login and mbox.last_login < cutoff_without_login:
|
if mbox.last_login and mbox.last_login < cutoff_without_login:
|
||||||
self.remove_mailbox(mbox.basedir)
|
self.remove_mailbox(mbox.basedir)
|
||||||
return
|
return
|
||||||
@@ -128,10 +131,25 @@ class Expiry:
|
|||||||
print_info(f"checking mailbox {date.strftime('%b %d')} {mboxname}")
|
print_info(f"checking mailbox {date.strftime('%b %d')} {mboxname}")
|
||||||
else:
|
else:
|
||||||
print_info(f"checking mailbox (no last_login) {mboxname}")
|
print_info(f"checking mailbox (no last_login) {mboxname}")
|
||||||
|
self.all_files += len(mbox.messages)
|
||||||
|
for message in mbox.messages:
|
||||||
|
if message.mtime < cutoff_mails:
|
||||||
|
self.remove_file(message.path, mtime=message.mtime)
|
||||||
|
elif message.size > 200000 and message.mtime < cutoff_large_mails:
|
||||||
|
# we only remove noticed large files (not unnoticed ones in new/)
|
||||||
|
parts = message.path.split("/")
|
||||||
|
if len(parts) >= 2 and parts[-2] == "cur":
|
||||||
|
self.remove_file(message.path, mtime=message.mtime)
|
||||||
|
else:
|
||||||
|
continue
|
||||||
|
changed = True
|
||||||
|
if changed:
|
||||||
|
self.remove_file(f"{mbox.basedir}/maildirsize")
|
||||||
|
|
||||||
def get_summary(self):
|
def get_summary(self):
|
||||||
return (
|
return (
|
||||||
f"Removed {self.del_mboxes} out of {self.all_mboxes} mailboxes "
|
f"Removed {self.del_mboxes} out of {self.all_mboxes} mailboxes "
|
||||||
|
f"and {self.del_files} out of {self.all_files} files in existing mailboxes "
|
||||||
f"in {time.time() - self.start:2.2f} seconds"
|
f"in {time.time() - self.start:2.2f} seconds"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -23,6 +23,12 @@ max_mailbox_size = 500M
|
|||||||
# maximum message size for an e-mail in bytes
|
# maximum message size for an e-mail in bytes
|
||||||
max_message_size = 31457280
|
max_message_size = 31457280
|
||||||
|
|
||||||
|
# days after which mails are unconditionally deleted
|
||||||
|
delete_mails_after = 20
|
||||||
|
|
||||||
|
# days after which large messages (>200k) are unconditionally deleted
|
||||||
|
delete_large_after = 7
|
||||||
|
|
||||||
# days after which users without a successful login are deleted (database and mails)
|
# days after which users without a successful login are deleted (database and mails)
|
||||||
delete_inactive_users_after = 90
|
delete_inactive_users_after = 90
|
||||||
|
|
||||||
|
|||||||
@@ -1,98 +1,61 @@
|
|||||||
"""
|
"""
|
||||||
Remove messages from a mailbox to meet a size target.
|
Quota-triggered per-user mailbox cleanup.
|
||||||
|
|
||||||
Dovecot calls this script when a user's quota is near its limit.
|
Dovecot calls this script via ``quota_warning``
|
||||||
Files are scored by ``size * age`` so that large, old messages
|
when a user crosses the quota threshold.
|
||||||
are removed first.
|
The script removes oldest messages first
|
||||||
|
to keep the mailbox under a specified target size.
|
||||||
|
|
||||||
Usage::
|
Usage::
|
||||||
|
|
||||||
quota_expire <target_mb> <mailbox_path>
|
chatmail-quota-expire <target_mb> <mailbox_path>
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import os
|
|
||||||
import sys
|
import sys
|
||||||
import time
|
|
||||||
from argparse import ArgumentParser
|
from argparse import ArgumentParser
|
||||||
from collections import namedtuple
|
from pathlib import Path
|
||||||
from stat import S_ISREG
|
|
||||||
|
|
||||||
FileEntry = namedtuple("FileEntry", ("path", "mtime", "size"))
|
from chatmaild.expire import get_file_entry, os_listdir_if_exists
|
||||||
|
|
||||||
|
|
||||||
def _get_file_entry(path):
|
|
||||||
try:
|
|
||||||
st = os.stat(path)
|
|
||||||
except FileNotFoundError:
|
|
||||||
return None
|
|
||||||
if not S_ISREG(st.st_mode):
|
|
||||||
return None
|
|
||||||
return FileEntry(path, st.st_mtime, st.st_size)
|
|
||||||
|
|
||||||
|
|
||||||
def _listdir(path):
|
|
||||||
try:
|
|
||||||
return os.listdir(path)
|
|
||||||
except FileNotFoundError:
|
|
||||||
return []
|
|
||||||
|
|
||||||
|
|
||||||
def scan_mailbox_messages(mailbox_dir):
|
def scan_mailbox_messages(mailbox_dir):
|
||||||
|
"""Collect FileEntry items from top-level cur/new/tmp only."""
|
||||||
|
mbox = Path(mailbox_dir)
|
||||||
messages = []
|
messages = []
|
||||||
for sub in ("cur", "new", "tmp"):
|
for sub in ("cur", "new", "tmp"):
|
||||||
subdir = f"{mailbox_dir}/{sub}"
|
for name in os_listdir_if_exists(mbox / sub):
|
||||||
for name in _listdir(subdir):
|
if entry := get_file_entry(str(mbox / sub / name)):
|
||||||
entry = _get_file_entry(f"{subdir}/{name}")
|
|
||||||
if entry is not None:
|
|
||||||
messages.append(entry)
|
messages.append(entry)
|
||||||
return messages
|
return messages
|
||||||
|
|
||||||
|
|
||||||
def _remove_stale_caches(mailbox_dir):
|
def expire_to_target(mailbox_dir, target_bytes):
|
||||||
for name in ("maildirsize", "dovecot.index.cache"):
|
"""Remove oldest files until total size <= *target_bytes*.
|
||||||
try:
|
|
||||||
os.unlink(f"{mailbox_dir}/{name}")
|
|
||||||
except FileNotFoundError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
Returns ``(removed_count, cache_bytes)`` where *cache_bytes*
|
||||||
def expire_to_target(mailbox_dir, target_bytes, now=None):
|
is the size of the deleted ``dovecot.index.cache`` file
|
||||||
"""Remove highest-scored files until total size <= *target_bytes*.
|
(0 when the file did not exist).
|
||||||
|
|
||||||
Returns the list of removed file paths.
|
|
||||||
"""
|
"""
|
||||||
if now is None:
|
mbox = Path(mailbox_dir)
|
||||||
now = time.time()
|
messages = scan_mailbox_messages(mbox)
|
||||||
|
|
||||||
messages = scan_mailbox_messages(mailbox_dir)
|
|
||||||
total_size = sum(m.size for m in messages)
|
total_size = sum(m.size for m in messages)
|
||||||
|
removed = 0
|
||||||
if total_size <= target_bytes:
|
for count, entry in enumerate(sorted(messages, key=lambda m: m.mtime), 1):
|
||||||
return []
|
|
||||||
|
|
||||||
# Score: large and old files get the highest score.
|
|
||||||
scored = sorted(
|
|
||||||
messages,
|
|
||||||
key=lambda m: m.size * (now - m.mtime),
|
|
||||||
reverse=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
removed = []
|
|
||||||
for entry in scored:
|
|
||||||
if total_size <= target_bytes:
|
if total_size <= target_bytes:
|
||||||
break
|
break
|
||||||
try:
|
Path(entry.path).unlink(missing_ok=True)
|
||||||
os.unlink(entry.path)
|
|
||||||
except FileNotFoundError:
|
|
||||||
continue
|
|
||||||
total_size -= entry.size
|
total_size -= entry.size
|
||||||
removed.append(entry.path)
|
removed = count
|
||||||
|
|
||||||
if removed:
|
(mbox / "maildirsize").unlink(missing_ok=True)
|
||||||
_remove_stale_caches(mailbox_dir)
|
cache = mbox / "dovecot.index.cache"
|
||||||
|
try:
|
||||||
return removed
|
cache_bytes = cache.stat().st_size
|
||||||
|
except FileNotFoundError:
|
||||||
|
cache_bytes = 0
|
||||||
|
cache.unlink(missing_ok=True)
|
||||||
|
return removed, cache_bytes
|
||||||
|
|
||||||
|
|
||||||
def main(args=None):
|
def main(args=None):
|
||||||
@@ -105,48 +68,23 @@ def main(args=None):
|
|||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"mailbox_path",
|
"mailbox_path",
|
||||||
help="path to a user mailbox, or with --sweep the mailboxes directory",
|
help="path to a user mailbox",
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--sweep",
|
|
||||||
action="store_true",
|
|
||||||
help="sweep all mailboxes under mailbox_path",
|
|
||||||
)
|
)
|
||||||
args = parser.parse_args(args)
|
args = parser.parse_args(args)
|
||||||
|
|
||||||
target_bytes = args.target_mb * 1024 * 1024
|
target_bytes = args.target_mb * 1024 * 1024
|
||||||
|
|
||||||
if args.sweep:
|
removed_count, cache_bytes = expire_to_target(args.mailbox_path, target_bytes)
|
||||||
return _sweep(args.mailbox_path, target_bytes)
|
if removed_count:
|
||||||
|
user = Path(args.mailbox_path).name
|
||||||
removed = expire_to_target(args.mailbox_path, target_bytes)
|
cache_mb = cache_bytes / 1024 / 1024
|
||||||
if removed:
|
|
||||||
print(
|
print(
|
||||||
f"removed {len(removed)} file(s) from {args.mailbox_path}"
|
f"quota-expire: removed {removed_count} message(s) from {user}"
|
||||||
f" to reach {args.target_mb} MB target",
|
f" cache={cache_mb:.1f}MB",
|
||||||
file=sys.stderr,
|
file=sys.stderr,
|
||||||
)
|
)
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
def _sweep(mailboxes_dir, target_bytes):
|
|
||||||
try:
|
|
||||||
names = os.listdir(mailboxes_dir)
|
|
||||||
except FileNotFoundError:
|
|
||||||
print(f"directory not found: {mailboxes_dir}", file=sys.stderr)
|
|
||||||
return 1
|
|
||||||
for name in sorted(names):
|
|
||||||
if "@" not in name:
|
|
||||||
continue
|
|
||||||
mbox = f"{mailboxes_dir}/{name}"
|
|
||||||
removed = expire_to_target(mbox, target_bytes)
|
|
||||||
if removed:
|
|
||||||
print(
|
|
||||||
f"removed {len(removed)} file(s) from {name}",
|
|
||||||
file=sys.stderr,
|
|
||||||
)
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
sys.exit(main())
|
sys.exit(main())
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from chatmaild.config import read_config
|
from chatmaild.config import parse_size_mb, read_config
|
||||||
|
|
||||||
|
|
||||||
def test_read_config_basic(example_config):
|
def test_read_config_basic(example_config):
|
||||||
@@ -34,6 +34,8 @@ def test_read_config_testrun(make_config):
|
|||||||
assert config.postfix_reinject_port == 10025
|
assert config.postfix_reinject_port == 10025
|
||||||
assert config.max_user_send_per_minute == 60
|
assert config.max_user_send_per_minute == 60
|
||||||
assert config.max_mailbox_size == "500M"
|
assert config.max_mailbox_size == "500M"
|
||||||
|
assert config.delete_mails_after == "20"
|
||||||
|
assert config.delete_large_after == "7"
|
||||||
assert config.username_min_length == 9
|
assert config.username_min_length == 9
|
||||||
assert config.username_max_length == 9
|
assert config.username_max_length == 9
|
||||||
assert config.password_min_length == 9
|
assert config.password_min_length == 9
|
||||||
@@ -119,3 +121,17 @@ def test_config_tls_external_bad_format(make_config):
|
|||||||
"tls_external_cert_and_key": "/only/one/path.pem",
|
"tls_external_cert_and_key": "/only/one/path.pem",
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_size_mb():
|
||||||
|
assert parse_size_mb("500M") == 500
|
||||||
|
assert parse_size_mb("2G") == 2048
|
||||||
|
assert parse_size_mb(" 1g ") == 1024
|
||||||
|
assert parse_size_mb("100MB") == 100
|
||||||
|
assert parse_size_mb("256") == 256
|
||||||
|
|
||||||
|
|
||||||
|
def test_max_mailbox_size_mb(make_config):
|
||||||
|
config = make_config("chat.example.org")
|
||||||
|
assert config.max_mailbox_size == "500M"
|
||||||
|
assert config.max_mailbox_size_mb == 500
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import os
|
import os
|
||||||
import random
|
import random
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
from fnmatch import fnmatch
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
@@ -153,6 +154,35 @@ def test_expiry_cli_basic(example_config, mbox1):
|
|||||||
expiry_main(args)
|
expiry_main(args)
|
||||||
|
|
||||||
|
|
||||||
|
def test_expiry_cli_old_files(capsys, example_config, mbox1):
|
||||||
|
relpaths_old = ["cur/msg_old1", "cur/msg_old1"]
|
||||||
|
cutoff_days = int(example_config.delete_mails_after) + 1
|
||||||
|
create_new_messages(mbox1.basedir, relpaths_old, size=1000, days=cutoff_days)
|
||||||
|
|
||||||
|
relpaths_large = ["cur/msg_old_large1", "new/msg_old_large2"]
|
||||||
|
cutoff_days = int(example_config.delete_large_after) + 1
|
||||||
|
create_new_messages(
|
||||||
|
mbox1.basedir, relpaths_large, size=1000 * 300, days=cutoff_days
|
||||||
|
)
|
||||||
|
|
||||||
|
create_new_messages(mbox1.basedir, ["cur/shouldstay"], size=1000 * 300, days=1)
|
||||||
|
|
||||||
|
args = str(example_config._inipath), "--remove", "-v"
|
||||||
|
expiry_main(args)
|
||||||
|
out, err = capsys.readouterr()
|
||||||
|
|
||||||
|
allpaths = relpaths_old + relpaths_large + ["maildirsize"]
|
||||||
|
for path in allpaths:
|
||||||
|
for line in err.split("\n"):
|
||||||
|
if fnmatch(line, f"removing*{path}"):
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
if path != "new/msg_old_large2":
|
||||||
|
pytest.fail(f"failed to remove {path}\n{err}")
|
||||||
|
|
||||||
|
assert "shouldstay" not in err
|
||||||
|
|
||||||
|
|
||||||
def test_get_file_entry(tmp_path):
|
def test_get_file_entry(tmp_path):
|
||||||
assert get_file_entry(str(tmp_path.joinpath("123123"))) is None
|
assert get_file_entry(str(tmp_path.joinpath("123123"))) is None
|
||||||
p = tmp_path.joinpath("x")
|
p = tmp_path.joinpath("x")
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import os
|
import os
|
||||||
import time
|
import time
|
||||||
|
|
||||||
from chatmaild.quota_expire import expire_to_target, scan_mailbox_messages
|
from chatmaild.quota_expire import expire_to_target, main, scan_mailbox_messages
|
||||||
|
|
||||||
MB = 1024 * 1024
|
MB = 1024 * 1024
|
||||||
|
|
||||||
@@ -19,10 +19,7 @@ def test_scan_cur_new_tmp(tmp_path):
|
|||||||
_create_message(tmp_path, "cur/msg1", 100)
|
_create_message(tmp_path, "cur/msg1", 100)
|
||||||
_create_message(tmp_path, "new/msg2", 200)
|
_create_message(tmp_path, "new/msg2", 200)
|
||||||
_create_message(tmp_path, "tmp/msg3", 300)
|
_create_message(tmp_path, "tmp/msg3", 300)
|
||||||
messages = scan_mailbox_messages(str(tmp_path))
|
assert len(scan_mailbox_messages(str(tmp_path))) == 3
|
||||||
assert len(messages) == 3
|
|
||||||
sizes = sorted(m.size for m in messages)
|
|
||||||
assert sizes == [100, 200, 300]
|
|
||||||
|
|
||||||
|
|
||||||
def test_scan_ignores_subfolders(tmp_path):
|
def test_scan_ignores_subfolders(tmp_path):
|
||||||
@@ -31,61 +28,46 @@ def test_scan_ignores_subfolders(tmp_path):
|
|||||||
assert len(scan_mailbox_messages(str(tmp_path))) == 1
|
assert len(scan_mailbox_messages(str(tmp_path))) == 1
|
||||||
|
|
||||||
|
|
||||||
def test_scan_empty(tmp_path):
|
|
||||||
assert scan_mailbox_messages(str(tmp_path)) == []
|
|
||||||
assert scan_mailbox_messages(str(tmp_path / "nope")) == []
|
|
||||||
|
|
||||||
|
|
||||||
def test_noop_under_limit(tmp_path):
|
|
||||||
_create_message(tmp_path, "cur/msg1", MB)
|
|
||||||
assert expire_to_target(str(tmp_path), 2 * MB) == []
|
|
||||||
assert (tmp_path / "cur" / "msg1").exists()
|
|
||||||
|
|
||||||
|
|
||||||
def test_removes_to_target(tmp_path):
|
def test_removes_to_target(tmp_path):
|
||||||
now = time.time()
|
|
||||||
for i in range(15):
|
for i in range(15):
|
||||||
_create_message(tmp_path, f"cur/msg{i:02d}", MB, days_old=i + 1)
|
_create_message(tmp_path, f"cur/msg{i:02d}", MB, days_old=i + 1)
|
||||||
removed = expire_to_target(str(tmp_path), 10 * MB, now=now)
|
removed, _ = expire_to_target(str(tmp_path), 10 * MB)
|
||||||
assert len(removed) == 5
|
assert removed == 5
|
||||||
assert len(scan_mailbox_messages(str(tmp_path))) == 10
|
assert len(scan_mailbox_messages(str(tmp_path))) == 10
|
||||||
|
|
||||||
|
|
||||||
def test_scoring_prefers_large_old(tmp_path):
|
def test_removes_oldest_first(tmp_path):
|
||||||
now = time.time()
|
_create_message(tmp_path, "cur/old_small", MB, days_old=30)
|
||||||
_create_message(tmp_path, "cur/large_old", 2 * MB, days_old=30)
|
_create_message(tmp_path, "cur/new_huge", 10 * MB, days_old=1)
|
||||||
_create_message(tmp_path, "cur/small_new", MB, days_old=1)
|
# the 10MB file is kept, the 1MB file is removed because it's older
|
||||||
removed = expire_to_target(str(tmp_path), 2 * MB, now=now)
|
removed, _ = expire_to_target(str(tmp_path), 10 * MB)
|
||||||
assert len(removed) == 1
|
assert removed == 1
|
||||||
assert "large_old" in removed[0]
|
assert not (tmp_path / "cur/old_small").exists()
|
||||||
|
assert (tmp_path / "cur/new_huge").exists()
|
||||||
|
|
||||||
def test_scoring_large_new_beats_small_old(tmp_path):
|
|
||||||
now = time.time()
|
|
||||||
_create_message(tmp_path, "cur/big_new", 10 * MB, days_old=1)
|
|
||||||
_create_message(tmp_path, "cur/small_old", MB, days_old=5)
|
|
||||||
# big_new score: 10MB * 1d = 10 vs small_old score: 1MB * 5d = 5
|
|
||||||
removed = expire_to_target(str(tmp_path), 10 * MB, now=now)
|
|
||||||
assert len(removed) == 1
|
|
||||||
assert "big_new" in removed[0]
|
|
||||||
|
|
||||||
|
|
||||||
def test_exact_limit(tmp_path):
|
def test_exact_limit(tmp_path):
|
||||||
_create_message(tmp_path, "cur/msg1", 5 * MB)
|
_create_message(tmp_path, "cur/msg1", 5 * MB)
|
||||||
assert expire_to_target(str(tmp_path), 5 * MB) == []
|
removed, _ = expire_to_target(str(tmp_path), 5 * MB)
|
||||||
|
assert removed == 0
|
||||||
|
|
||||||
|
|
||||||
def test_removes_stale_caches(tmp_path):
|
def test_removes_stale_caches(tmp_path):
|
||||||
_create_message(tmp_path, "cur/msg1", 2 * MB, days_old=5)
|
_create_message(tmp_path, "cur/msg1", 2 * MB, days_old=5)
|
||||||
(tmp_path / "maildirsize").write_text("x")
|
(tmp_path / "maildirsize").write_text("x")
|
||||||
(tmp_path / "dovecot.index.cache").write_text("x")
|
(tmp_path / "dovecot.index.cache").write_bytes(b"y" * 4096)
|
||||||
expire_to_target(str(tmp_path), MB)
|
removed, cache_bytes = expire_to_target(str(tmp_path), MB)
|
||||||
|
assert removed == 1
|
||||||
|
assert cache_bytes == 4096
|
||||||
assert not (tmp_path / "maildirsize").exists()
|
assert not (tmp_path / "maildirsize").exists()
|
||||||
assert not (tmp_path / "dovecot.index.cache").exists()
|
assert not (tmp_path / "dovecot.index.cache").exists()
|
||||||
|
|
||||||
|
|
||||||
def test_no_cache_removal_when_under_limit(tmp_path):
|
def test_logging_output_is_mtail_compatible(tmp_path, capsys):
|
||||||
_create_message(tmp_path, "cur/msg1", MB)
|
mbox = tmp_path / "user@example.org"
|
||||||
(tmp_path / "maildirsize").write_text("x")
|
_create_message(mbox, "cur/msg1", 2 * MB, days_old=5)
|
||||||
expire_to_target(str(tmp_path), 2 * MB)
|
(mbox / "dovecot.index.cache").write_bytes(b"c" * 2 * MB)
|
||||||
assert (tmp_path / "maildirsize").exists()
|
main([str(1), str(mbox)])
|
||||||
|
_, err = capsys.readouterr()
|
||||||
|
assert "quota-expire: removed 1 message(s) from user@example.org" in err
|
||||||
|
assert "cache=2.0MB" in err
|
||||||
|
|||||||
@@ -149,15 +149,19 @@ plugin {
|
|||||||
}
|
}
|
||||||
|
|
||||||
plugin {
|
plugin {
|
||||||
# for now we define static quota-rules for all users
|
|
||||||
quota = maildir:User quota
|
quota = maildir:User quota
|
||||||
quota_rule = *:storage={{ config.max_mailbox_size }}
|
|
||||||
quota_max_mail_size={{ config.max_message_size }}
|
quota_max_mail_size={{ config.max_message_size }}
|
||||||
quota_grace = 0
|
quota_grace = 0
|
||||||
|
|
||||||
# When a user reaches 90% quota, run chatmail-quota-expire
|
# Inflate the dovecot-visible quota so that Delta Chat clients
|
||||||
# to remove large/old messages until usage is below 80%.
|
# (which warn at 80% of the IMAP-reported limit) never see
|
||||||
quota_warning = storage=90%% quota-warning {{ config.max_mailbox_size_mb * 80 // 100 }} {{ config.mailboxes_dir }}/%u
|
# quota warnings -- expire kicks in well before that point.
|
||||||
|
quota_rule = *:storage={{ config.max_mailbox_size_mb * 140 // 100 }}M
|
||||||
|
|
||||||
|
# Trigger when usage reaches the configured max_mailbox_size
|
||||||
|
# (72% of inflated = ~100% of configured), then expire oldest
|
||||||
|
# messages down to 80% of the configured max_mailbox_size.
|
||||||
|
quota_warning = storage=72%% quota-warning {{ config.max_mailbox_size_mb * 80 // 100 }} {{ config.mailboxes_dir }}/%u
|
||||||
}
|
}
|
||||||
|
|
||||||
service quota-warning {
|
service quota-warning {
|
||||||
|
|||||||
@@ -78,3 +78,11 @@ counter rejected_unencrypted_mail_count
|
|||||||
/Rejected unencrypted mail/ {
|
/Rejected unencrypted mail/ {
|
||||||
rejected_unencrypted_mail_count++
|
rejected_unencrypted_mail_count++
|
||||||
}
|
}
|
||||||
|
|
||||||
|
counter quota_expire_runs
|
||||||
|
counter quota_expire_removed_files
|
||||||
|
|
||||||
|
/quota-expire: removed (?P<count>\d+) message\(s\)/ {
|
||||||
|
quota_expire_runs++
|
||||||
|
quota_expire_removed_files += $count
|
||||||
|
}
|
||||||
|
|||||||
@@ -244,6 +244,24 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
|
|||||||
pytest.fail("Rate limit was not exceeded")
|
pytest.fail("Rate limit was not exceeded")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.slow
|
||||||
|
def test_expunged(remote, chatmail_config):
|
||||||
|
outdated_days = int(chatmail_config.delete_mails_after) + 1
|
||||||
|
find_cmds = [
|
||||||
|
f"find {chatmail_config.mailboxes_dir} -path '*/cur/*' -mtime +{outdated_days} -type f",
|
||||||
|
f"find {chatmail_config.mailboxes_dir} -path '*/.*/cur/*' -mtime +{outdated_days} -type f",
|
||||||
|
f"find {chatmail_config.mailboxes_dir} -path '*/new/*' -mtime +{outdated_days} -type f",
|
||||||
|
f"find {chatmail_config.mailboxes_dir} -path '*/.*/new/*' -mtime +{outdated_days} -type f",
|
||||||
|
f"find {chatmail_config.mailboxes_dir} -path '*/tmp/*' -mtime +{outdated_days} -type f",
|
||||||
|
f"find {chatmail_config.mailboxes_dir} -path '*/.*/tmp/*' -mtime +{outdated_days} -type f",
|
||||||
|
]
|
||||||
|
outdated_days = int(chatmail_config.delete_large_after) + 1
|
||||||
|
find_cmds.append(
|
||||||
|
f"find {chatmail_config.mailboxes_dir} -path '*/cur/*' -mtime +{outdated_days} -size +200k -type f"
|
||||||
|
)
|
||||||
|
for cmd in find_cmds:
|
||||||
|
for line in remote.iter_output(cmd):
|
||||||
|
assert not line
|
||||||
|
|
||||||
|
|
||||||
def test_deployed_state(remote):
|
def test_deployed_state(remote):
|
||||||
|
|||||||
@@ -88,9 +88,12 @@ class TestEndToEndDeltaChat:
|
|||||||
return int(float(number) * units[unit])
|
return int(float(number) * units[unit])
|
||||||
|
|
||||||
quota = parse_size_limit(chatmail_config.max_mailbox_size)
|
quota = parse_size_limit(chatmail_config.max_mailbox_size)
|
||||||
|
# Dovecot quota is inflated to 140% of the configured limit
|
||||||
|
# so that quota-expire keeps users below the warning threshold.
|
||||||
|
dovecot_quota = quota * 140 // 100
|
||||||
|
|
||||||
lp.sec(f"filling remote inbox for {user}")
|
lp.sec(f"filling remote inbox for {user}")
|
||||||
fn = f"7743102289.M843172P2484002.c20,S={quota},W=2398:2,"
|
fn = f"7743102289.M843172P2484002.c20,S={dovecot_quota},W=2398:2,"
|
||||||
path = chatmail_config.mailboxes_dir.joinpath(user, "cur", fn)
|
path = chatmail_config.mailboxes_dir.joinpath(user, "cur", fn)
|
||||||
sshexec = get_sshexec(sshdomain)
|
sshexec = get_sshexec(sshdomain)
|
||||||
sshexec(call=rshell.write_numbytes, kwargs=dict(path=str(path), num=120))
|
sshexec(call=rshell.write_numbytes, kwargs=dict(path=str(path), num=120))
|
||||||
|
|||||||
@@ -102,14 +102,15 @@ short overview of ``chatmaild`` services:
|
|||||||
Apple/Google/Huawei.
|
Apple/Google/Huawei.
|
||||||
|
|
||||||
- `chatmail-expire <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/expire.py>`_
|
- `chatmail-expire <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/expire.py>`_
|
||||||
deletes entire mailboxes of users who have not logged in
|
deletes old messages, large messages, and entire mailboxes
|
||||||
for longer than ``delete_inactive_users_after`` days.
|
of users who have not logged in for longer than
|
||||||
|
``delete_inactive_users_after`` days.
|
||||||
|
|
||||||
- `chatmail-quota-expire <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/quota_expire.py>`_
|
- ``chatmail-quota-expire``
|
||||||
is called by Dovecot's ``quota_warning`` mechanism when a
|
is called by Dovecot's ``quota_warning`` mechanism when a
|
||||||
user reaches 90% of their mailbox quota.
|
mailbox exceeds ``max_mailbox_size``.
|
||||||
It removes the largest and oldest messages
|
It removes the oldest messages
|
||||||
until usage drops below 80% of the quota.
|
until usage drops to a safe level.
|
||||||
|
|
||||||
- `lastlogin <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/lastlogin.py>`_
|
- `lastlogin <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/lastlogin.py>`_
|
||||||
is contacted by Dovecot when a user logs in and stores the date of
|
is contacted by Dovecot when a user logs in and stores the date of
|
||||||
@@ -145,7 +146,7 @@ Chatmail relay dependency diagram
|
|||||||
certs-nginx[("`TLS certs
|
certs-nginx[("`TLS certs
|
||||||
/var/lib/acme`")] --> nginx-internal;
|
/var/lib/acme`")] --> nginx-internal;
|
||||||
systemd-timer --- acmetool;
|
systemd-timer --- acmetool;
|
||||||
systemd-timer --- chatmail-expire-inactive;
|
systemd-timer --- chatmail-expire-daily;
|
||||||
systemd-timer --- chatmail-fsreport-daily;
|
systemd-timer --- chatmail-fsreport-daily;
|
||||||
acmetool --> certs[("`TLS certs
|
acmetool --> certs[("`TLS certs
|
||||||
/var/lib/acme`")];
|
/var/lib/acme`")];
|
||||||
@@ -166,7 +167,7 @@ Chatmail relay dependency diagram
|
|||||||
chatmail-quota-expire --- maildir;
|
chatmail-quota-expire --- maildir;
|
||||||
lastlogin --- maildir;
|
lastlogin --- maildir;
|
||||||
doveauth --- maildir;
|
doveauth --- maildir;
|
||||||
chatmail-expire-inactive --- maildir;
|
chatmail-expire-daily --- maildir;
|
||||||
chatmail-fsreport-daily --- maildir;
|
chatmail-fsreport-daily --- maildir;
|
||||||
chatmail-metadata --- iroh-relay;
|
chatmail-metadata --- iroh-relay;
|
||||||
chatmail-metadata --- |encrypted device token| notifications.delta.chat;
|
chatmail-metadata --- |encrypted device token| notifications.delta.chat;
|
||||||
|
|||||||
Reference in New Issue
Block a user