From b13929119bb91bca2ae178983e6cc97284a6c0d8 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Thu, 11 Sep 2025 15:32:30 +0200 Subject: [PATCH] do all expunging in python --- chatmaild/src/chatmaild/expire.py | 96 ++++++++++++------- .../tests/test_delete_inactive_users.py | 2 +- cmdeploy/src/cmdeploy/dovecot/expunge.cron.j2 | 18 +--- 3 files changed, 66 insertions(+), 50 deletions(-) diff --git a/chatmaild/src/chatmaild/expire.py b/chatmaild/src/chatmaild/expire.py index c0e85085..01b375f0 100644 --- a/chatmaild/src/chatmaild/expire.py +++ b/chatmaild/src/chatmaild/expire.py @@ -1,25 +1,16 @@ -import sys import os import shutil -import logging +import sys import time -from stat import S_ISREG -from pathlib import Path -from datetime import datetime from collections import namedtuple +from datetime import datetime +from stat import S_ISREG + +from chatmaild.config import read_config # delete already seen big mails after 7 days, in the INBOX # 2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/cur/*' -mtime +{{ config.delete_large_after }} -size +200k -type f -delete # # delete all mails after {{ config.delete_mails_after }} days, in the Inbox -# 2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete -## or in any IMAP subfolder -# 2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/.*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete -## even if they are unseen -# 2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete -# 2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/.*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete -## or only temporary (but then they shouldn't be around after {{ config.delete_mails_after }} days anyway). -# 2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete -# 2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/.*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete # 3 0 * * * vmail find {{ config.mailboxes_dir }} -name 'maildirsize' -type f -delete @@ -46,16 +37,14 @@ def M(size): return f"{int(size/1000000):6.0f}M" -now = datetime.utcnow().timestamp() - - class Stats: - def __init__(self, basedir): + def __init__(self, basedir, maxnum=None): self.basedir = str(basedir) self.mailboxes = [] + self.maxnum = maxnum - def iter_mailboxes(self, maxnum=None): - for mailbox in os.listdir(self.basedir)[:maxnum]: + def iter_mailboxes(self): + for mailbox in os.listdir(self.basedir)[: self.maxnum]: if "@" in mailbox: mailboxdir = joinpath(self.basedir, mailbox) self.mailboxes.append(MailboxStat(mailboxdir)) @@ -99,6 +88,11 @@ class MailboxStat: def get_extra_files(self): return list(self.extrafiles) + def get_file_entry(self, name): + for entry in self.extrafiles: + if name == entry.relapth: + return entry + class XXXStats: def __init__(self): @@ -123,6 +117,8 @@ class XXXStats: self.sum_extra += size def dump_summary(self): + now = datetime.utcnow().timestamp() + print(f"size of everything: {M(self.sum_extra + self.sum_all_messages)}") print(f"size all messages: {M(self.sum_all_messages)}") percent = self.sum_extra / (self.sum_extra + self.sum_all_messages) * 100 @@ -174,21 +170,55 @@ class XXXStats: print(f"daily active: {K(daily_active)} {p(daily_active)}") -def run_expire(config, basedir): - stat = Stats(basedir) - stat.iter_mailboxes() - cutoff_date = time.time() - config.delete_inactive_users_after * 86400 +def run_expire(config, basedir, dry=False, maxnum=None): + now = time.time() + + stat = Stats(basedir, maxnum=maxnum) + stat.iter_mailboxes() + cutoff_date_without_login = now - int(config.delete_inactive_users_after) * 86400 + cutoff_date_mails = now - int(config.delete_mails_after) * 86400 + cutoff_date_large_mails = now - int(config.delete_large_after) * 86400 + + def rmtree(path): + if dry: + print("would remove mailbox", path) + else: + shutil.rmtree(path, ignore_errors=True) + + def unlink(mailboxdir, message): + if dry: + relpath = os.path.basename(mailboxdir) + message.relpath + print(f"would remove {D(message.mtime)} {K(message.size)} {relpath}") + else: + os.unlink(path) - num = 0 for mbox in stat.mailboxes: - if mbox.last_login < cutoff_date: - logging.info("removing outdated mailbox %s", mbox.mailboxdir) - shutil.rmtree(mbox.mailboxdir, ignore_errors=True) - num += 1 - print(f"expired {num} mailboxes") + changed = False + if mbox.last_login and mbox.last_login < cutoff_date_without_login: + rmtree(mbox.mailboxdir) + continue + for message in mbox.messages: + path = joinpath(mbox.mailboxdir, message.relpath) + if message.mtime < cutoff_date_mails: + unlink(mbox.mailboxdir, message) + elif message.size > 200000 and message.mtime < cutoff_date_large_mails: + unlink(mbox.mailboxdir, message) + else: + continue + changed = True + if changed and not dry: + p = joinpath(mbox.mailboxdir, "maildirsize") + try: + os.unlink(p) + except FileNotFoundError: + pass + + +def main(): + cfgpath, basedir, maxnum = sys.argv[1:] + config = read_config(cfgpath) + run_expire(config, basedir, dry=True, maxnum=int(maxnum)) if __name__ == "__main__": - cfgpath, basedir = sys.argv[1:] - config = read_config(cfgpath) - run_expire(config, basedir) + main() diff --git a/chatmaild/src/chatmaild/tests/test_delete_inactive_users.py b/chatmaild/src/chatmaild/tests/test_delete_inactive_users.py index 3eb21d12..a7c74a9e 100644 --- a/chatmaild/src/chatmaild/tests/test_delete_inactive_users.py +++ b/chatmaild/src/chatmaild/tests/test_delete_inactive_users.py @@ -1,7 +1,7 @@ import time -from chatmaild.expire import run_expire from chatmaild.doveauth import AuthDictProxy +from chatmaild.expire import run_expire def test_login_timestamps(example_config): diff --git a/cmdeploy/src/cmdeploy/dovecot/expunge.cron.j2 b/cmdeploy/src/cmdeploy/dovecot/expunge.cron.j2 index a9f10e8e..dc9af7e0 100644 --- a/cmdeploy/src/cmdeploy/dovecot/expunge.cron.j2 +++ b/cmdeploy/src/cmdeploy/dovecot/expunge.cron.j2 @@ -1,16 +1,2 @@ -# delete already seen big mails after 7 days, in the INBOX -2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/cur/*' -mtime +{{ config.delete_large_after }} -size +200k -type f -delete -# delete all mails after {{ config.delete_mails_after }} days, in the Inbox -2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete -# or in any IMAP subfolder -2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/.*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete -# even if they are unseen -2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete -2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/.*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete -# or only temporary (but then they shouldn't be around after {{ config.delete_mails_after }} days anyway). -2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete -2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/.*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete -3 0 * * * vmail find {{ config.mailboxes_dir }} -name 'maildirsize' -type f -delete - -# ported -4 0 * * * vmail /usr/local/lib/chatmaild/venv/bin/expire /usr/local/lib/chatmaild/chatmail.ini {config.mailboxes_dir} +# expire mailboxes and old or too large messages +2 0 * * * vmail /usr/local/lib/chatmaild/venv/bin/expire /usr/local/lib/chatmaild/chatmail.ini {config.mailboxes_dir}