mirror of
https://github.com/chatmail/relay.git
synced 2026-05-21 13:28:05 +00:00
Compare commits
1 Commits
three-tier
...
add-quota-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4512d1e735 |
15
CHANGELOG.md
15
CHANGELOG.md
@@ -1,5 +1,20 @@
|
|||||||
# Changelog for chatmail deployment
|
# Changelog for chatmail deployment
|
||||||
|
|
||||||
|
## Unreleased
|
||||||
|
|
||||||
|
### Features
|
||||||
|
|
||||||
|
- Add per-user quota-triggered cleanup (`chatmail-quota-expire`).
|
||||||
|
When a mailbox exceeds the configured ``max_mailbox_size``,
|
||||||
|
Dovecot runs the new script which removes the oldest
|
||||||
|
messages until usage drops to a safe level.
|
||||||
|
No operator action is required after upgrading;
|
||||||
|
existing over-quota mailboxes start receiving mail
|
||||||
|
again immediately and are cleaned up automatically.
|
||||||
|
The daily `chatmail-expire` timer continues to handle
|
||||||
|
deletion of old messages, large messages,
|
||||||
|
and inactive user mailboxes.
|
||||||
|
|
||||||
## 1.9.0 2025-12-18
|
## 1.9.0 2025-12-18
|
||||||
|
|
||||||
### Documentation
|
### Documentation
|
||||||
|
|||||||
@@ -21,8 +21,8 @@ where = ['src']
|
|||||||
[project.scripts]
|
[project.scripts]
|
||||||
doveauth = "chatmaild.doveauth:main"
|
doveauth = "chatmaild.doveauth:main"
|
||||||
chatmail-metadata = "chatmaild.metadata:main"
|
chatmail-metadata = "chatmaild.metadata:main"
|
||||||
chatmail-expire = "chatmaild.expire:daily_expire_main"
|
chatmail-expire = "chatmaild.expire:main"
|
||||||
chatmail-quota-expire = "chatmaild.expire:quota_expire_main"
|
chatmail-quota-expire = "chatmaild.quota_expire:main"
|
||||||
chatmail-fsreport = "chatmaild.fsreport:main"
|
chatmail-fsreport = "chatmaild.fsreport:main"
|
||||||
lastlogin = "chatmaild.lastlogin:main"
|
lastlogin = "chatmaild.lastlogin:main"
|
||||||
turnserver = "chatmaild.turnserver:main"
|
turnserver = "chatmaild.turnserver:main"
|
||||||
|
|||||||
@@ -115,7 +115,7 @@ class Config:
|
|||||||
|
|
||||||
def parse_size_mb(limit):
|
def parse_size_mb(limit):
|
||||||
"""Parse a size string like ``500M`` or ``2G`` and return megabytes."""
|
"""Parse a size string like ``500M`` or ``2G`` and return megabytes."""
|
||||||
value = limit.strip().upper().removesuffix("B")
|
value = limit.strip().upper().rstrip("B")
|
||||||
if value.endswith("G"):
|
if value.endswith("G"):
|
||||||
return int(value[:-1]) * 1024
|
return int(value[:-1]) * 1024
|
||||||
if value.endswith("M"):
|
if value.endswith("M"):
|
||||||
|
|||||||
@@ -4,26 +4,17 @@ Expire old messages and addresses.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import re
|
|
||||||
import shutil
|
import shutil
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
from argparse import ArgumentParser
|
from argparse import ArgumentParser
|
||||||
from collections import namedtuple
|
from collections import namedtuple
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from pathlib import Path
|
|
||||||
from stat import S_ISREG
|
from stat import S_ISREG
|
||||||
|
|
||||||
from chatmaild.config import read_config
|
from chatmaild.config import read_config
|
||||||
|
|
||||||
FileEntry = namedtuple("FileEntry", ("path", "mtime", "size"))
|
FileEntry = namedtuple("FileEntry", ("path", "mtime", "size"))
|
||||||
QuotaFileEntry = namedtuple("QuotaFileEntry", ("mtime", "quota_size", "path"))
|
|
||||||
|
|
||||||
# Quota cleanup factor of max_mailbox_size. The mailbox is reset to this size.
|
|
||||||
QUOTA_CLEANUP_FACTOR = 0.7
|
|
||||||
|
|
||||||
# e.g. "cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S"
|
|
||||||
_dovecot_fn_rex = re.compile(r".+/(\d+)\..+,S=(\d+)")
|
|
||||||
|
|
||||||
|
|
||||||
def iter_mailboxes(basedir, maxnum):
|
def iter_mailboxes(basedir, maxnum):
|
||||||
@@ -83,54 +74,6 @@ class MailboxStat:
|
|||||||
self.extrafiles.sort(key=lambda x: -x.size)
|
self.extrafiles.sort(key=lambda x: -x.size)
|
||||||
|
|
||||||
|
|
||||||
def parse_dovecot_filename(relpath):
|
|
||||||
m = _dovecot_fn_rex.match(relpath)
|
|
||||||
if not m:
|
|
||||||
return None
|
|
||||||
return QuotaFileEntry(int(m.group(1)), int(m.group(2)), relpath)
|
|
||||||
|
|
||||||
|
|
||||||
def scan_mailbox_messages(mbox):
|
|
||||||
messages = []
|
|
||||||
for sub in ("cur", "new"):
|
|
||||||
for name in os_listdir_if_exists(mbox / sub):
|
|
||||||
if entry := parse_dovecot_filename(f"{sub}/{name}"):
|
|
||||||
messages.append(entry)
|
|
||||||
return messages
|
|
||||||
|
|
||||||
|
|
||||||
# Within this window, large messages are deleted before small ones.
|
|
||||||
DELETE_LARGE_FIRST_DAYS = 7
|
|
||||||
|
|
||||||
|
|
||||||
def expire_to_target(mbox, target_bytes):
|
|
||||||
cutoff = time.time() - DELETE_LARGE_FIRST_DAYS * 86400
|
|
||||||
messages = scan_mailbox_messages(mbox)
|
|
||||||
|
|
||||||
def sort_key(msg):
|
|
||||||
# prio 0: Older than cutoff -> remove oldest first
|
|
||||||
if msg.mtime < cutoff:
|
|
||||||
return (0, msg.mtime)
|
|
||||||
|
|
||||||
# prio 1: more recent than cutoff, large -> remove largest first
|
|
||||||
if msg.quota_size > 200000:
|
|
||||||
return (1, -msg.quota_size, msg.mtime)
|
|
||||||
|
|
||||||
# prio 2: more recent than cutoff, small -> remove oldest first
|
|
||||||
return (2, msg.mtime)
|
|
||||||
|
|
||||||
total_size = sum(m.quota_size for m in messages)
|
|
||||||
removed = 0
|
|
||||||
for entry in sorted(messages, key=sort_key):
|
|
||||||
if total_size <= target_bytes:
|
|
||||||
break
|
|
||||||
(mbox / entry.path).unlink(missing_ok=True)
|
|
||||||
total_size -= entry.quota_size
|
|
||||||
removed += 1
|
|
||||||
|
|
||||||
return removed
|
|
||||||
|
|
||||||
|
|
||||||
def print_info(msg):
|
def print_info(msg):
|
||||||
print(msg, file=sys.stderr)
|
print(msg, file=sys.stderr)
|
||||||
|
|
||||||
@@ -200,19 +143,6 @@ class Expiry:
|
|||||||
else:
|
else:
|
||||||
continue
|
continue
|
||||||
changed = True
|
changed = True
|
||||||
|
|
||||||
target_bytes = (
|
|
||||||
self.config.max_mailbox_size_mb * 1024 * 1024 * QUOTA_CLEANUP_FACTOR
|
|
||||||
)
|
|
||||||
removed = expire_to_target(Path(mbox.basedir), target_bytes)
|
|
||||||
if removed:
|
|
||||||
changed = True
|
|
||||||
self.del_files += removed
|
|
||||||
if self.verbose:
|
|
||||||
print_info(
|
|
||||||
f"quota-expire: removed {removed} message(s) from {mboxname}"
|
|
||||||
)
|
|
||||||
|
|
||||||
if changed:
|
if changed:
|
||||||
self.remove_file(f"{mbox.basedir}/maildirsize")
|
self.remove_file(f"{mbox.basedir}/maildirsize")
|
||||||
|
|
||||||
@@ -224,9 +154,9 @@ class Expiry:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def daily_expire_main(args=None):
|
def main(args=None):
|
||||||
"""Expire mailboxes and messages according to chatmail config"""
|
"""Expire mailboxes and messages according to chatmail config"""
|
||||||
parser = ArgumentParser(description=daily_expire_main.__doc__)
|
parser = ArgumentParser(description=main.__doc__)
|
||||||
ini = "/usr/local/lib/chatmaild/chatmail.ini"
|
ini = "/usr/local/lib/chatmaild/chatmail.ini"
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"chatmail_ini",
|
"chatmail_ini",
|
||||||
@@ -272,33 +202,5 @@ def daily_expire_main(args=None):
|
|||||||
print(exp.get_summary())
|
print(exp.get_summary())
|
||||||
|
|
||||||
|
|
||||||
def quota_expire_main(args=None):
|
if __name__ == "__main__":
|
||||||
"""Remove mailbox messages to stay within a megabyte target.
|
main(sys.argv[1:])
|
||||||
|
|
||||||
This entry point is called by dovecot when a quota threshold is passed.
|
|
||||||
"""
|
|
||||||
|
|
||||||
parser = ArgumentParser(description=quota_expire_main.__doc__)
|
|
||||||
parser.add_argument(
|
|
||||||
"target_mb",
|
|
||||||
type=int,
|
|
||||||
help="target mailbox size in megabytes",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"mailbox_path",
|
|
||||||
type=Path,
|
|
||||||
help="path to a user mailbox",
|
|
||||||
)
|
|
||||||
args = parser.parse_args(args)
|
|
||||||
|
|
||||||
target_bytes = args.target_mb * 1024 * 1024
|
|
||||||
|
|
||||||
removed_count = expire_to_target(args.mailbox_path, target_bytes)
|
|
||||||
if removed_count:
|
|
||||||
(args.mailbox_path / "maildirsize").unlink(missing_ok=True)
|
|
||||||
print(
|
|
||||||
f"quota-expire: removed {removed_count} message(s)"
|
|
||||||
f" from {args.mailbox_path.name}",
|
|
||||||
file=sys.stderr,
|
|
||||||
)
|
|
||||||
return 0
|
|
||||||
|
|||||||
@@ -18,7 +18,6 @@ max_user_send_per_minute = 60
|
|||||||
max_user_send_burst_size = 10
|
max_user_send_burst_size = 10
|
||||||
|
|
||||||
# maximum mailbox size of a chatmail address
|
# maximum mailbox size of a chatmail address
|
||||||
# Oldest messages will be removed automatically, so mailboxes never run full.
|
|
||||||
max_mailbox_size = 500M
|
max_mailbox_size = 500M
|
||||||
|
|
||||||
# maximum message size for an e-mail in bytes
|
# maximum message size for an e-mail in bytes
|
||||||
|
|||||||
90
chatmaild/src/chatmaild/quota_expire.py
Normal file
90
chatmaild/src/chatmaild/quota_expire.py
Normal file
@@ -0,0 +1,90 @@
|
|||||||
|
"""
|
||||||
|
Quota-triggered per-user mailbox cleanup.
|
||||||
|
|
||||||
|
Dovecot calls this script via ``quota_warning``
|
||||||
|
when a user crosses the quota threshold.
|
||||||
|
The script removes oldest messages first
|
||||||
|
to keep the mailbox under a specified target size.
|
||||||
|
|
||||||
|
Usage::
|
||||||
|
|
||||||
|
chatmail-quota-expire <target_mb> <mailbox_path>
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sys
|
||||||
|
from argparse import ArgumentParser
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from chatmaild.expire import get_file_entry, os_listdir_if_exists
|
||||||
|
|
||||||
|
|
||||||
|
def scan_mailbox_messages(mailbox_dir):
|
||||||
|
"""Collect FileEntry items from top-level cur/new/tmp only."""
|
||||||
|
mbox = Path(mailbox_dir)
|
||||||
|
messages = []
|
||||||
|
for sub in ("cur", "new", "tmp"):
|
||||||
|
for name in os_listdir_if_exists(mbox / sub):
|
||||||
|
if entry := get_file_entry(str(mbox / sub / name)):
|
||||||
|
messages.append(entry)
|
||||||
|
return messages
|
||||||
|
|
||||||
|
|
||||||
|
def expire_to_target(mailbox_dir, target_bytes):
|
||||||
|
"""Remove oldest files until total size <= *target_bytes*.
|
||||||
|
|
||||||
|
Returns ``(removed_count, cache_bytes)`` where *cache_bytes*
|
||||||
|
is the size of the deleted ``dovecot.index.cache`` file
|
||||||
|
(0 when the file did not exist).
|
||||||
|
"""
|
||||||
|
mbox = Path(mailbox_dir)
|
||||||
|
messages = scan_mailbox_messages(mbox)
|
||||||
|
total_size = sum(m.size for m in messages)
|
||||||
|
removed = 0
|
||||||
|
for count, entry in enumerate(sorted(messages, key=lambda m: m.mtime), 1):
|
||||||
|
if total_size <= target_bytes:
|
||||||
|
break
|
||||||
|
Path(entry.path).unlink(missing_ok=True)
|
||||||
|
total_size -= entry.size
|
||||||
|
removed = count
|
||||||
|
|
||||||
|
(mbox / "maildirsize").unlink(missing_ok=True)
|
||||||
|
cache = mbox / "dovecot.index.cache"
|
||||||
|
try:
|
||||||
|
cache_bytes = cache.stat().st_size
|
||||||
|
except FileNotFoundError:
|
||||||
|
cache_bytes = 0
|
||||||
|
cache.unlink(missing_ok=True)
|
||||||
|
return removed, cache_bytes
|
||||||
|
|
||||||
|
|
||||||
|
def main(args=None):
|
||||||
|
"""Remove mailbox messages to stay within a megabyte target."""
|
||||||
|
parser = ArgumentParser(description=main.__doc__)
|
||||||
|
parser.add_argument(
|
||||||
|
"target_mb",
|
||||||
|
type=int,
|
||||||
|
help="target mailbox size in megabytes",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"mailbox_path",
|
||||||
|
help="path to a user mailbox",
|
||||||
|
)
|
||||||
|
args = parser.parse_args(args)
|
||||||
|
|
||||||
|
target_bytes = args.target_mb * 1024 * 1024
|
||||||
|
|
||||||
|
removed_count, cache_bytes = expire_to_target(args.mailbox_path, target_bytes)
|
||||||
|
if removed_count:
|
||||||
|
user = Path(args.mailbox_path).name
|
||||||
|
cache_mb = cache_bytes / 1024 / 1024
|
||||||
|
print(
|
||||||
|
f"quota-expire: removed {removed_count} message(s) from {user}"
|
||||||
|
f" cache={cache_mb:.1f}MB",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
import time
|
import time
|
||||||
|
|
||||||
from chatmaild.doveauth import AuthDictProxy
|
from chatmaild.doveauth import AuthDictProxy
|
||||||
from chatmaild.expire import daily_expire_main as main_expire
|
from chatmaild.expire import main as main_expire
|
||||||
|
|
||||||
|
|
||||||
def test_login_timestamps(example_config):
|
def test_login_timestamps(example_config):
|
||||||
|
|||||||
@@ -1,8 +1,5 @@
|
|||||||
import itertools
|
|
||||||
import os
|
import os
|
||||||
import random
|
import random
|
||||||
import shutil
|
|
||||||
import time
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from fnmatch import fnmatch
|
from fnmatch import fnmatch
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -12,19 +9,13 @@ import pytest
|
|||||||
from chatmaild.expire import (
|
from chatmaild.expire import (
|
||||||
FileEntry,
|
FileEntry,
|
||||||
MailboxStat,
|
MailboxStat,
|
||||||
expire_to_target,
|
|
||||||
get_file_entry,
|
get_file_entry,
|
||||||
iter_mailboxes,
|
iter_mailboxes,
|
||||||
os_listdir_if_exists,
|
os_listdir_if_exists,
|
||||||
parse_dovecot_filename,
|
|
||||||
quota_expire_main,
|
|
||||||
scan_mailbox_messages,
|
|
||||||
)
|
)
|
||||||
from chatmaild.expire import daily_expire_main as expiry_main
|
from chatmaild.expire import main as expiry_main
|
||||||
from chatmaild.fsreport import main as report_main
|
from chatmaild.fsreport import main as report_main
|
||||||
|
|
||||||
MB = 1024 * 1024
|
|
||||||
|
|
||||||
|
|
||||||
def fill_mbox(folderdir):
|
def fill_mbox(folderdir):
|
||||||
password = folderdir.joinpath("password")
|
password = folderdir.joinpath("password")
|
||||||
@@ -205,74 +196,3 @@ def test_os_listdir_if_exists(tmp_path):
|
|||||||
tmp_path.joinpath("x").write_text("hello")
|
tmp_path.joinpath("x").write_text("hello")
|
||||||
assert len(os_listdir_if_exists(str(tmp_path))) == 1
|
assert len(os_listdir_if_exists(str(tmp_path))) == 1
|
||||||
assert len(os_listdir_if_exists(str(tmp_path.joinpath("123123")))) == 0
|
assert len(os_listdir_if_exists(str(tmp_path.joinpath("123123")))) == 0
|
||||||
|
|
||||||
|
|
||||||
# --- quota expire tests ---
|
|
||||||
|
|
||||||
_msg_counter = itertools.count(1)
|
|
||||||
|
|
||||||
|
|
||||||
def _create_message(basedir, sub, size, days_old=0, disk_size=None):
|
|
||||||
seq = next(_msg_counter)
|
|
||||||
mtime = int(time.time() - days_old * 86400)
|
|
||||||
name = f"{mtime}.M1P1Q{seq}.hostname,S={size},W={size}:2,S"
|
|
||||||
path = basedir / sub / name
|
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
path.write_bytes(b"x" * (disk_size if disk_size is not None else size))
|
|
||||||
os.utime(path, (mtime, mtime))
|
|
||||||
return path
|
|
||||||
|
|
||||||
|
|
||||||
def test_parse_dovecot_filename():
|
|
||||||
e = parse_dovecot_filename("cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S")
|
|
||||||
assert e.path == "cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S"
|
|
||||||
assert e.mtime == 1775324677
|
|
||||||
assert e.quota_size == 3235
|
|
||||||
assert parse_dovecot_filename("cur/msg_without_structure") is None
|
|
||||||
|
|
||||||
|
|
||||||
def test_expire_to_target(tmp_path):
|
|
||||||
_create_message(tmp_path, "cur", MB, days_old=10, disk_size=100)
|
|
||||||
_create_message(tmp_path, "new", MB, days_old=5)
|
|
||||||
assert len(scan_mailbox_messages(tmp_path)) == 2
|
|
||||||
# removes oldest first, uses S= size not disk size
|
|
||||||
removed = expire_to_target(tmp_path, MB)
|
|
||||||
assert removed == 1
|
|
||||||
assert len(scan_mailbox_messages(tmp_path)) == 1
|
|
||||||
|
|
||||||
|
|
||||||
def test_expire_to_target_prioritization(tmp_path):
|
|
||||||
def create_messages():
|
|
||||||
for sub in ("cur", "new"):
|
|
||||||
if (tmp_path / sub).exists():
|
|
||||||
shutil.rmtree(tmp_path / sub)
|
|
||||||
# prio 0: older than 7 days
|
|
||||||
_create_message(tmp_path, "cur", 5 * MB, days_old=10)
|
|
||||||
# prio 1: last 7 days, large (>200KB)
|
|
||||||
_create_message(tmp_path, "cur", 5 * MB, days_old=1)
|
|
||||||
# prio 2: last 7 days, small
|
|
||||||
_create_message(tmp_path, "cur", 1000, days_old=2)
|
|
||||||
|
|
||||||
# Shrink to 6MB: only the old message (prio 0) is removed.
|
|
||||||
create_messages()
|
|
||||||
assert expire_to_target(tmp_path, 6 * MB) == 1
|
|
||||||
msgs = scan_mailbox_messages(tmp_path)
|
|
||||||
assert len(msgs) == 2
|
|
||||||
assert all(m.mtime > time.time() - 7 * 86400 for m in msgs)
|
|
||||||
|
|
||||||
# Shrink to 1KB: old and recent-large removed, small survives.
|
|
||||||
create_messages()
|
|
||||||
assert expire_to_target(tmp_path, 1024) == 2
|
|
||||||
msgs = scan_mailbox_messages(tmp_path)
|
|
||||||
assert len(msgs) == 1
|
|
||||||
assert msgs[0].quota_size == 1000
|
|
||||||
|
|
||||||
|
|
||||||
def test_quota_expire_main(tmp_path, capsys):
|
|
||||||
mbox = tmp_path / "user@example.org"
|
|
||||||
_create_message(mbox, "cur", 2 * MB, days_old=5)
|
|
||||||
(mbox / "maildirsize").write_text("x")
|
|
||||||
quota_expire_main([str(1), str(mbox)])
|
|
||||||
_, err = capsys.readouterr()
|
|
||||||
assert "quota-expire: removed 1 message(s) from user@example.org" in err
|
|
||||||
assert not (mbox / "maildirsize").exists()
|
|
||||||
|
|||||||
73
chatmaild/src/chatmaild/tests/test_quota_expire.py
Normal file
73
chatmaild/src/chatmaild/tests/test_quota_expire.py
Normal file
@@ -0,0 +1,73 @@
|
|||||||
|
import os
|
||||||
|
import time
|
||||||
|
|
||||||
|
from chatmaild.quota_expire import expire_to_target, main, scan_mailbox_messages
|
||||||
|
|
||||||
|
MB = 1024 * 1024
|
||||||
|
|
||||||
|
|
||||||
|
def _create_message(basedir, relpath, size, days_old=0):
|
||||||
|
path = basedir / relpath
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
path.write_bytes(b"x" * size)
|
||||||
|
mtime = time.time() - days_old * 86400
|
||||||
|
os.utime(path, (mtime, mtime))
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def test_scan_cur_new_tmp(tmp_path):
|
||||||
|
_create_message(tmp_path, "cur/msg1", 100)
|
||||||
|
_create_message(tmp_path, "new/msg2", 200)
|
||||||
|
_create_message(tmp_path, "tmp/msg3", 300)
|
||||||
|
assert len(scan_mailbox_messages(str(tmp_path))) == 3
|
||||||
|
|
||||||
|
|
||||||
|
def test_scan_ignores_subfolders(tmp_path):
|
||||||
|
_create_message(tmp_path, "cur/a", 10)
|
||||||
|
_create_message(tmp_path, ".DeltaChat/cur/b", 20)
|
||||||
|
assert len(scan_mailbox_messages(str(tmp_path))) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_removes_to_target(tmp_path):
|
||||||
|
for i in range(15):
|
||||||
|
_create_message(tmp_path, f"cur/msg{i:02d}", MB, days_old=i + 1)
|
||||||
|
removed, _ = expire_to_target(str(tmp_path), 10 * MB)
|
||||||
|
assert removed == 5
|
||||||
|
assert len(scan_mailbox_messages(str(tmp_path))) == 10
|
||||||
|
|
||||||
|
|
||||||
|
def test_removes_oldest_first(tmp_path):
|
||||||
|
_create_message(tmp_path, "cur/old_small", MB, days_old=30)
|
||||||
|
_create_message(tmp_path, "cur/new_huge", 10 * MB, days_old=1)
|
||||||
|
# the 10MB file is kept, the 1MB file is removed because it's older
|
||||||
|
removed, _ = expire_to_target(str(tmp_path), 10 * MB)
|
||||||
|
assert removed == 1
|
||||||
|
assert not (tmp_path / "cur/old_small").exists()
|
||||||
|
assert (tmp_path / "cur/new_huge").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_exact_limit(tmp_path):
|
||||||
|
_create_message(tmp_path, "cur/msg1", 5 * MB)
|
||||||
|
removed, _ = expire_to_target(str(tmp_path), 5 * MB)
|
||||||
|
assert removed == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_removes_stale_caches(tmp_path):
|
||||||
|
_create_message(tmp_path, "cur/msg1", 2 * MB, days_old=5)
|
||||||
|
(tmp_path / "maildirsize").write_text("x")
|
||||||
|
(tmp_path / "dovecot.index.cache").write_bytes(b"y" * 4096)
|
||||||
|
removed, cache_bytes = expire_to_target(str(tmp_path), MB)
|
||||||
|
assert removed == 1
|
||||||
|
assert cache_bytes == 4096
|
||||||
|
assert not (tmp_path / "maildirsize").exists()
|
||||||
|
assert not (tmp_path / "dovecot.index.cache").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_logging_output_is_mtail_compatible(tmp_path, capsys):
|
||||||
|
mbox = tmp_path / "user@example.org"
|
||||||
|
_create_message(mbox, "cur/msg1", 2 * MB, days_old=5)
|
||||||
|
(mbox / "dovecot.index.cache").write_bytes(b"c" * 2 * MB)
|
||||||
|
main([str(1), str(mbox)])
|
||||||
|
_, err = capsys.readouterr()
|
||||||
|
assert "quota-expire: removed 1 message(s) from user@example.org" in err
|
||||||
|
assert "cache=2.0MB" in err
|
||||||
@@ -194,6 +194,12 @@ def status_cmd(args, out):
|
|||||||
|
|
||||||
|
|
||||||
def test_cmd_options(parser):
|
def test_cmd_options(parser):
|
||||||
|
parser.add_argument(
|
||||||
|
"--slow",
|
||||||
|
dest="slow",
|
||||||
|
action="store_true",
|
||||||
|
help="also run slow tests",
|
||||||
|
)
|
||||||
add_ssh_host_option(parser)
|
add_ssh_host_option(parser)
|
||||||
|
|
||||||
|
|
||||||
@@ -215,6 +221,8 @@ def test_cmd(args, out):
|
|||||||
"-v",
|
"-v",
|
||||||
"--durations=5",
|
"--durations=5",
|
||||||
]
|
]
|
||||||
|
if args.slow:
|
||||||
|
pytest_args.append("--slow")
|
||||||
ret = out.run_ret(pytest_args, env=env)
|
ret = out.run_ret(pytest_args, env=env)
|
||||||
return ret
|
return ret
|
||||||
|
|
||||||
|
|||||||
@@ -150,40 +150,18 @@ class UnboundDeployer(Deployer):
|
|||||||
self.need_restart = False
|
self.need_restart = False
|
||||||
|
|
||||||
def install(self):
|
def install(self):
|
||||||
|
# Run local DNS resolver `unbound`. `resolvconf` takes care of
|
||||||
|
# setting up /etc/resolv.conf to use 127.0.0.1 as the resolver.
|
||||||
|
|
||||||
# On an IPv4-only system, if unbound is started but not configured,
|
# On an IPv4-only system, if unbound is started but not configured,
|
||||||
# it causes subsequent steps to fail to resolve hosts.
|
# it causes subsequent steps to fail to resolve hosts.
|
||||||
with blocked_service_startup():
|
with blocked_service_startup():
|
||||||
apt.packages(
|
apt.packages(
|
||||||
name="Install unbound",
|
name="Install unbound",
|
||||||
packages=["unbound", "unbound-anchor", "dnsutils"],
|
packages=["unbound", "unbound-anchor", "dnsutils", "resolvconf"],
|
||||||
)
|
)
|
||||||
|
|
||||||
def configure(self):
|
def configure(self):
|
||||||
# Remove dynamic resolver managers that compete for /etc/resolv.conf.
|
|
||||||
apt.packages(
|
|
||||||
name="Purge resolvconf",
|
|
||||||
packages=["resolvconf"],
|
|
||||||
present=False,
|
|
||||||
extra_uninstall_args="--purge",
|
|
||||||
)
|
|
||||||
# systemd-resolved can't be purged due to dependencies; stop and mask.
|
|
||||||
server.shell(
|
|
||||||
name="Stop and mask systemd-resolved",
|
|
||||||
commands=[
|
|
||||||
"systemctl stop systemd-resolved.service || true",
|
|
||||||
"systemctl mask systemd-resolved.service",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
# Configure unbound resolver with Quad9 fallback and a trailing newline
|
|
||||||
# (SolusVM bug).
|
|
||||||
files.put(
|
|
||||||
name="Write static resolv.conf",
|
|
||||||
src=BytesIO(b"nameserver 127.0.0.1\nnameserver 9.9.9.9\n"),
|
|
||||||
dest="/etc/resolv.conf",
|
|
||||||
user="root",
|
|
||||||
group="root",
|
|
||||||
mode="644",
|
|
||||||
)
|
|
||||||
server.shell(
|
server.shell(
|
||||||
name="Generate root keys for validating DNSSEC",
|
name="Generate root keys for validating DNSSEC",
|
||||||
commands=[
|
commands=[
|
||||||
@@ -590,6 +568,14 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
|
|||||||
Deployment().perform_stages([WebsiteDeployer(config)])
|
Deployment().perform_stages([WebsiteDeployer(config)])
|
||||||
return
|
return
|
||||||
|
|
||||||
|
if host.get_fact(Port, port=53) != "unbound":
|
||||||
|
files.line(
|
||||||
|
name="Add 9.9.9.9 to resolv.conf",
|
||||||
|
path="/etc/resolv.conf",
|
||||||
|
# Guard against resolv.conf missing a trailing newline (SolusVM bug).
|
||||||
|
line="\nnameserver 9.9.9.9",
|
||||||
|
)
|
||||||
|
|
||||||
# Check if mtail_address interface is available (if configured)
|
# Check if mtail_address interface is available (if configured)
|
||||||
if config.mtail_address and config.mtail_address not in ('127.0.0.1', '::1', 'localhost'):
|
if config.mtail_address and config.mtail_address not in ('127.0.0.1', '::1', 'localhost'):
|
||||||
ipv4_addrs = host.get_fact(hardware.Ipv4Addrs)
|
ipv4_addrs = host.get_fact(hardware.Ipv4Addrs)
|
||||||
|
|||||||
@@ -186,7 +186,7 @@ def _configure_dovecot(config: Config, debug: bool = False) -> tuple[bool, bool]
|
|||||||
can_modify = not is_in_container()
|
can_modify = not is_in_container()
|
||||||
for name in ("max_user_instances", "max_user_watches"):
|
for name in ("max_user_instances", "max_user_watches"):
|
||||||
key = f"fs.inotify.{name}"
|
key = f"fs.inotify.{name}"
|
||||||
value = host.get_fact(Sysctl).get(key, 0)
|
value = host.get_fact(Sysctl)[key]
|
||||||
if value > 65534:
|
if value > 65534:
|
||||||
continue
|
continue
|
||||||
if not can_modify:
|
if not can_modify:
|
||||||
|
|||||||
@@ -153,21 +153,21 @@ plugin {
|
|||||||
quota_max_mail_size={{ config.max_message_size }}
|
quota_max_mail_size={{ config.max_message_size }}
|
||||||
quota_grace = 0
|
quota_grace = 0
|
||||||
|
|
||||||
quota_rule = *:storage={{ config.max_mailbox_size_mb }}M
|
# Inflate the dovecot-visible quota so that Delta Chat clients
|
||||||
|
# (which warn at 80% of the IMAP-reported limit) never see
|
||||||
|
# quota warnings -- expire kicks in well before that point.
|
||||||
|
quota_rule = *:storage={{ config.max_mailbox_size_mb * 140 // 100 }}M
|
||||||
|
|
||||||
# Trigger at 75%% of quota, expire oldest messages down to 70%%.
|
# Trigger when usage reaches the configured max_mailbox_size
|
||||||
# The percentages are chosen to prevent current Delta Chat users
|
# (72% of inflated = ~100% of configured), then expire oldest
|
||||||
# from seeing "quota warnings" which trigger at 80% and 95%.
|
# messages down to 80% of the configured max_mailbox_size.
|
||||||
|
quota_warning = storage=72%% quota-warning {{ config.max_mailbox_size_mb * 80 // 100 }} {{ config.mailboxes_dir }}/%u
|
||||||
quota_warning = storage=75%% quota-warning {{ config.max_mailbox_size_mb * 70 // 100 }} {{ config.mailboxes_dir }}/%u
|
|
||||||
}
|
}
|
||||||
|
|
||||||
service quota-warning {
|
service quota-warning {
|
||||||
executable = script /usr/local/lib/chatmaild/venv/bin/chatmail-quota-expire
|
executable = script /usr/local/lib/chatmaild/venv/bin/chatmail-quota-expire
|
||||||
user = vmail
|
user = vmail
|
||||||
unix_listener quota-warning {
|
unix_listener quota-warning {
|
||||||
user = vmail
|
|
||||||
mode = 0600
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -221,6 +221,7 @@ def test_rewrite_subject(cmsetup, maildata):
|
|||||||
assert "Subject: Unencrypted subject" not in rcvd_msg
|
assert "Subject: Unencrypted subject" not in rcvd_msg
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.slow
|
||||||
def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
|
def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
|
||||||
"""Test that the per-account send-mail limit is exceeded."""
|
"""Test that the per-account send-mail limit is exceeded."""
|
||||||
user1, user2 = cmsetup.gen_users(2)
|
user1, user2 = cmsetup.gen_users(2)
|
||||||
@@ -243,6 +244,7 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
|
|||||||
pytest.fail("Rate limit was not exceeded")
|
pytest.fail("Rate limit was not exceeded")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.slow
|
||||||
def test_expunged(remote, chatmail_config):
|
def test_expunged(remote, chatmail_config):
|
||||||
outdated_days = int(chatmail_config.delete_mails_after) + 1
|
outdated_days = int(chatmail_config.delete_mails_after) + 1
|
||||||
find_cmds = [
|
find_cmds = [
|
||||||
|
|||||||
@@ -88,9 +88,12 @@ class TestEndToEndDeltaChat:
|
|||||||
return int(float(number) * units[unit])
|
return int(float(number) * units[unit])
|
||||||
|
|
||||||
quota = parse_size_limit(chatmail_config.max_mailbox_size)
|
quota = parse_size_limit(chatmail_config.max_mailbox_size)
|
||||||
|
# Dovecot quota is inflated to 140% of the configured limit
|
||||||
|
# so that quota-expire keeps users below the warning threshold.
|
||||||
|
dovecot_quota = quota * 140 // 100
|
||||||
|
|
||||||
lp.sec(f"filling remote inbox for {user}")
|
lp.sec(f"filling remote inbox for {user}")
|
||||||
fn = f"7743102289.M843172P2484002.c20,S={quota},W=2398:2,"
|
fn = f"7743102289.M843172P2484002.c20,S={dovecot_quota},W=2398:2,"
|
||||||
path = chatmail_config.mailboxes_dir.joinpath(user, "cur", fn)
|
path = chatmail_config.mailboxes_dir.joinpath(user, "cur", fn)
|
||||||
sshexec = get_sshexec(sshdomain)
|
sshexec = get_sshexec(sshdomain)
|
||||||
sshexec(call=rshell.write_numbytes, kwargs=dict(path=str(path), num=120))
|
sshexec(call=rshell.write_numbytes, kwargs=dict(path=str(path), num=120))
|
||||||
|
|||||||
@@ -23,6 +23,12 @@ def _is_ip(domain):
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def pytest_addoption(parser):
|
||||||
|
parser.addoption(
|
||||||
|
"--slow", action="store_true", default=False, help="also run slow tests"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def pytest_configure(config):
|
def pytest_configure(config):
|
||||||
config._benchresults = {}
|
config._benchresults = {}
|
||||||
config.addinivalue_line(
|
config.addinivalue_line(
|
||||||
@@ -30,6 +36,13 @@ def pytest_configure(config):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def pytest_runtest_setup(item):
|
||||||
|
markers = list(item.iter_markers(name="slow"))
|
||||||
|
if markers:
|
||||||
|
if not item.config.getoption("--slow"):
|
||||||
|
pytest.skip("skipping slow test, use --slow to run")
|
||||||
|
|
||||||
|
|
||||||
def _get_chatmail_config():
|
def _get_chatmail_config():
|
||||||
inipath = os.environ.get("CHATMAIL_INI")
|
inipath = os.environ.get("CHATMAIL_INI")
|
||||||
if inipath:
|
if inipath:
|
||||||
|
|||||||
@@ -106,8 +106,11 @@ short overview of ``chatmaild`` services:
|
|||||||
of users who have not logged in for longer than
|
of users who have not logged in for longer than
|
||||||
``delete_inactive_users_after`` days.
|
``delete_inactive_users_after`` days.
|
||||||
|
|
||||||
- ``chatmail-quota-expire`` is called by Dovecot's ``quota_warning`` mechanism
|
- ``chatmail-quota-expire``
|
||||||
and will automatically remove oldest messages to keep mailboxes well under ``max_mailbox_size``.
|
is called by Dovecot's ``quota_warning`` mechanism when a
|
||||||
|
mailbox exceeds ``max_mailbox_size``.
|
||||||
|
It removes the oldest messages
|
||||||
|
until usage drops to a safe level.
|
||||||
|
|
||||||
- `lastlogin <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/lastlogin.py>`_
|
- `lastlogin <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/lastlogin.py>`_
|
||||||
is contacted by Dovecot when a user logs in and stores the date of
|
is contacted by Dovecot when a user logs in and stores the date of
|
||||||
|
|||||||
Reference in New Issue
Block a user