mirror of
https://github.com/chatmail/relay.git
synced 2026-09-03 05:53:15 +00:00
holger krekel
ae6b89f936
datetime.now(timezone.utc).timestamp() is actually 7200 seconds in the past. The Python deprecation warning of utcnow() warning actually pointed there.
303 lines
9.4 KiB
Python
303 lines
9.4 KiB
Python
"""
|
|
Expire old messages and addresses.
|
|
|
|
"""
|
|
|
|
import os
|
|
import re
|
|
import shutil
|
|
import sys
|
|
import time
|
|
from argparse import ArgumentParser
|
|
from collections import namedtuple
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from stat import S_ISREG
|
|
|
|
from chatmaild.config import read_config
|
|
|
|
FileEntry = namedtuple("FileEntry", ("path", "mtime", "size"))
|
|
QuotaFileEntry = namedtuple("QuotaFileEntry", ("mtime", "quota_size", "path"))
|
|
|
|
# Quota cleanup factor of max_mailbox_size. The mailbox is reset to this size.
|
|
QUOTA_CLEANUP_FACTOR = 0.7
|
|
|
|
# e.g. "cur/1775324677.M448978P3029757.exam,S=3235,W=3305:2,S"
|
|
_dovecot_fn_rex = re.compile(r".+/(\d+)\..+,S=(\d+)")
|
|
|
|
|
|
def iter_mailboxes(basedir, maxnum):
|
|
if not os.path.exists(basedir):
|
|
print_info(f"no mailboxes found at: {basedir}")
|
|
return
|
|
|
|
for name in os_listdir_if_exists(basedir)[:maxnum]:
|
|
if "@" in name:
|
|
yield MailboxStat(basedir + "/" + name)
|
|
|
|
|
|
def get_file_entry(path):
|
|
"""return a FileEntry or None if the path does not exist or is not a regular file."""
|
|
try:
|
|
st = os.stat(path)
|
|
except FileNotFoundError:
|
|
return None
|
|
if not S_ISREG(st.st_mode):
|
|
return None
|
|
return FileEntry(path, st.st_mtime, st.st_size)
|
|
|
|
|
|
def os_listdir_if_exists(path):
|
|
"""return a list of names obtained from os.listdir or an empty list if the path does not exist."""
|
|
try:
|
|
return os.listdir(path)
|
|
except FileNotFoundError:
|
|
return []
|
|
|
|
|
|
class MailboxStat:
|
|
last_login = None
|
|
|
|
def __init__(self, basedir):
|
|
self.basedir = str(basedir)
|
|
self.messages = []
|
|
self.extrafiles = []
|
|
self.scandir(self.basedir)
|
|
|
|
def scandir(self, folderdir):
|
|
for name in os_listdir_if_exists(folderdir):
|
|
path = f"{folderdir}/{name}"
|
|
if name in ("cur", "new", "tmp"):
|
|
for msg_name in os_listdir_if_exists(path):
|
|
entry = get_file_entry(f"{path}/{msg_name}")
|
|
if entry is not None:
|
|
self.messages.append(entry)
|
|
elif os.path.isdir(path):
|
|
self.scandir(path)
|
|
else:
|
|
entry = get_file_entry(path)
|
|
if entry is not None:
|
|
self.extrafiles.append(entry)
|
|
if name == "password":
|
|
self.last_login = entry.mtime
|
|
self.extrafiles.sort(key=lambda x: -x.size)
|
|
|
|
|
|
def parse_dovecot_filename(relpath):
|
|
m = _dovecot_fn_rex.match(relpath)
|
|
if not m:
|
|
return None
|
|
return QuotaFileEntry(int(m.group(1)), int(m.group(2)), relpath)
|
|
|
|
|
|
def scan_mailbox_messages(mbox):
|
|
messages = []
|
|
for sub in ("cur", "new"):
|
|
for name in os_listdir_if_exists(mbox / sub):
|
|
if entry := parse_dovecot_filename(f"{sub}/{name}"):
|
|
messages.append(entry)
|
|
return messages
|
|
|
|
|
|
def expire_to_target(mbox, target_bytes):
|
|
messages = scan_mailbox_messages(mbox)
|
|
total_size = sum(m.quota_size for m in messages)
|
|
# Keep recent 24 hours of messages protected from expiry because
|
|
# likely something is wrong with interactions on that address
|
|
# and quota-full signal can help the address owner's device to notice it
|
|
undeletable_messages_cutoff = time.time() - (3600 * 24)
|
|
removed = 0
|
|
for entry in sorted(messages):
|
|
if total_size <= target_bytes:
|
|
break
|
|
if entry.mtime > undeletable_messages_cutoff:
|
|
break
|
|
(mbox / entry.path).unlink(missing_ok=True)
|
|
total_size -= entry.quota_size
|
|
removed += 1
|
|
|
|
return removed
|
|
|
|
|
|
def print_info(msg):
|
|
print(msg, file=sys.stderr)
|
|
|
|
|
|
class Expiry:
|
|
def __init__(self, config, dry, now, verbose):
|
|
self.config = config
|
|
self.dry = dry
|
|
self.now = now
|
|
self.verbose = verbose
|
|
self.del_mboxes = 0
|
|
self.all_mboxes = 0
|
|
self.del_files = 0
|
|
self.all_files = 0
|
|
self.start = time.time()
|
|
|
|
def remove_mailbox(self, mboxdir):
|
|
if self.verbose:
|
|
print_info(f"removing {mboxdir}")
|
|
if not self.dry:
|
|
shutil.rmtree(mboxdir)
|
|
self.del_mboxes += 1
|
|
|
|
def remove_file(self, path, mtime=None):
|
|
if self.verbose:
|
|
if mtime is not None:
|
|
date = datetime.fromtimestamp(mtime).strftime("%b %d")
|
|
print_info(f"removing {date} {path}")
|
|
else:
|
|
print_info(f"removing {path}")
|
|
if not self.dry:
|
|
try:
|
|
os.unlink(path)
|
|
except FileNotFoundError:
|
|
print_info(f"file not found/vanished {path}")
|
|
self.del_files += 1
|
|
|
|
def process_mailbox_stat(self, mbox):
|
|
cutoff_without_login = (
|
|
self.now - int(self.config.delete_inactive_users_after) * 86400
|
|
)
|
|
cutoff_mails = self.now - int(self.config.delete_mails_after) * 86400
|
|
cutoff_large_mails = self.now - int(self.config.delete_large_after) * 86400
|
|
|
|
self.all_mboxes += 1
|
|
changed = False
|
|
if mbox.last_login and mbox.last_login < cutoff_without_login:
|
|
self.remove_mailbox(mbox.basedir)
|
|
return
|
|
elif mbox.last_login is None:
|
|
try:
|
|
if not self.dry:
|
|
os.rmdir(mbox.basedir)
|
|
self.del_mboxes += 1
|
|
except OSError:
|
|
print_info(
|
|
f"Skipped deleting {mbox.basedir}, doesn't have last_login but isn't empty"
|
|
)
|
|
return
|
|
|
|
mboxname = os.path.basename(mbox.basedir)
|
|
if self.verbose:
|
|
date = datetime.fromtimestamp(mbox.last_login) if mbox.last_login else None
|
|
if date:
|
|
print_info(f"checking mailbox {date.strftime('%b %d')} {mboxname}")
|
|
else:
|
|
print_info(f"checking mailbox (no last_login) {mboxname}")
|
|
self.all_files += len(mbox.messages)
|
|
for message in mbox.messages:
|
|
if message.mtime < cutoff_mails:
|
|
self.remove_file(message.path, mtime=message.mtime)
|
|
elif message.size > 200000 and message.mtime < cutoff_large_mails:
|
|
# we only remove noticed large files (not unnoticed ones in new/)
|
|
parts = message.path.split("/")
|
|
if len(parts) >= 2 and parts[-2] == "cur":
|
|
self.remove_file(message.path, mtime=message.mtime)
|
|
else:
|
|
continue
|
|
changed = True
|
|
|
|
target_bytes = (
|
|
self.config.max_mailbox_size_mb * 1024 * 1024 * QUOTA_CLEANUP_FACTOR
|
|
)
|
|
removed = expire_to_target(Path(mbox.basedir), target_bytes)
|
|
if removed:
|
|
changed = True
|
|
self.del_files += removed
|
|
if self.verbose:
|
|
print_info(
|
|
f"quota-expire: removed {removed} message(s) from {mboxname}"
|
|
)
|
|
|
|
if changed:
|
|
self.remove_file(f"{mbox.basedir}/maildirsize")
|
|
|
|
def get_summary(self):
|
|
return (
|
|
f"Removed {self.del_mboxes} out of {self.all_mboxes} mailboxes "
|
|
f"and {self.del_files} out of {self.all_files} files in existing mailboxes "
|
|
f"in {time.time() - self.start:2.2f} seconds"
|
|
)
|
|
|
|
|
|
def daily_expire_main(args=None):
|
|
"""Expire mailboxes and messages according to chatmail config"""
|
|
parser = ArgumentParser(description=daily_expire_main.__doc__)
|
|
ini = "/usr/local/lib/chatmaild/chatmail.ini"
|
|
parser.add_argument(
|
|
"chatmail_ini",
|
|
action="store",
|
|
nargs="?",
|
|
help=f"path pointing to chatmail.ini file, default: {ini}",
|
|
default=ini,
|
|
)
|
|
parser.add_argument(
|
|
"--days", action="store", help="assume date to be days older than now"
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--maxnum",
|
|
default=None,
|
|
action="store",
|
|
help="maximum number of mailboxes to iterate on",
|
|
)
|
|
parser.add_argument(
|
|
"-v",
|
|
dest="verbose",
|
|
action="store_true",
|
|
help="print out removed files and mailboxes",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--remove",
|
|
dest="remove",
|
|
action="store_true",
|
|
help="actually remove all expired files and dirs",
|
|
)
|
|
args = parser.parse_args(args)
|
|
|
|
config = read_config(args.chatmail_ini)
|
|
now = time.time()
|
|
if args.days:
|
|
now = now - 86400 * int(args.days)
|
|
|
|
maxnum = int(args.maxnum) if args.maxnum else None
|
|
exp = Expiry(config, dry=not args.remove, now=now, verbose=args.verbose)
|
|
for mailbox in iter_mailboxes(str(config.mailboxes_dir), maxnum=maxnum):
|
|
exp.process_mailbox_stat(mailbox)
|
|
print(exp.get_summary())
|
|
|
|
|
|
def quota_expire_main(args=None):
|
|
"""Remove mailbox messages to stay within a megabyte target.
|
|
|
|
This entry point is called by dovecot when a quota threshold is passed.
|
|
"""
|
|
|
|
parser = ArgumentParser(description=quota_expire_main.__doc__)
|
|
parser.add_argument(
|
|
"target_mb",
|
|
type=int,
|
|
help="target mailbox size in megabytes",
|
|
)
|
|
parser.add_argument(
|
|
"mailbox_path",
|
|
type=Path,
|
|
help="path to a user mailbox",
|
|
)
|
|
args = parser.parse_args(args)
|
|
|
|
target_bytes = args.target_mb * 1024 * 1024
|
|
|
|
removed_count = expire_to_target(args.mailbox_path, target_bytes)
|
|
if removed_count:
|
|
(args.mailbox_path / "maildirsize").unlink(missing_ok=True)
|
|
print(
|
|
f"quota-expire: removed {removed_count} message(s)"
|
|
f" from {args.mailbox_path.name}",
|
|
file=sys.stderr,
|
|
)
|
|
return 0
|