expire: remove large dovecot index and index cache files

chatmaild/src/chatmaild/expire.py

@@ -17,14 +17,14 @@ from chatmaild.config import read_config
 FileEntry = namedtuple("FileEntry", ("path", "mtime", "size"))
 
 
-def iter_mailboxes(basedir, maxnum):
+def iter_mailboxes(basedir, maxnum, tmpfs_index):
     if not os.path.exists(basedir):
         print_info(f"no mailboxes found at: {basedir}")
         return
 
     for name in os_listdir_if_exists(basedir)[:maxnum]:
         if "@" in name:
-            yield MailboxStat(basedir + "/" + name, name)
+            yield MailboxStat(basedir + "/" + name, name, tmpfs_index)
 
 
 def get_file_entry(path):
@@ -49,12 +49,14 @@ def os_listdir_if_exists(path):
 class MailboxStat:
     last_login = None
 
-    def __init__(self, basedir, name):
+    def __init__(self, basedir, name, tmpfs_index):
         self.basedir = str(basedir)
         self.name = name
         self.messages = []
         self.extrafiles = []
         self.scandir(self.basedir)
+        if tmpfs_index:
+            self.scandir("/dev/shm/" + name)
 
     def scandir(self, folderdir):
         for name in os_listdir_if_exists(folderdir):
@@ -148,6 +150,9 @@ class Expiry:
             changed = True
         if changed:
             self.remove_file(f"{mbox.basedir}/maildirsize")
+        for file in mbox.extrafiles:
+            if "dovecot.index" in file.path.split("/")[-1] and file.size > 500 * 1024:
+                self.remove_file(file.path)
 
     def get_summary(self):
         return (
@@ -200,7 +205,9 @@ def main(args=None):
 
     maxnum = int(args.maxnum) if args.maxnum else None
     exp = Expiry(config, dry=not args.remove, now=now, verbose=args.verbose)
-    for mailbox in iter_mailboxes(str(config.mailboxes_dir), maxnum=maxnum):
+    for mailbox in iter_mailboxes(
+        str(config.mailboxes_dir), maxnum, config.tmpfs_index
+    ):
         exp.process_mailbox_stat(mailbox)
     print(exp.get_summary())
 

chatmaild/src/chatmaild/fsreport.py

@@ -159,7 +159,7 @@ def main(args=None):
 
     maxnum = int(args.maxnum) if args.maxnum else None
     rep = Report(now=now, min_login_age=int(args.min_login_age), mdir=args.mdir)
-    for mbox in iter_mailboxes(str(config.mailboxes_dir), maxnum=maxnum):
+    for mbox in iter_mailboxes(str(config.mailboxes_dir), maxnum, config.tmpfs_index):
         rep.process_mailbox_stat(mbox)
     rep.dump_summary()
 

chatmaild/src/chatmaild/tests/test_expire.py

@@ -47,7 +47,7 @@ def mbox1(example_config):
     mboxdir = example_config.mailboxes_dir.joinpath(addr)
     mboxdir.mkdir()
     fill_mbox(mboxdir)
-    return MailboxStat(mboxdir, addr)
+    return MailboxStat(mboxdir, addr, False)
 
 
 def test_deltachat_folder(example_config):
@@ -58,7 +58,7 @@ def test_deltachat_folder(example_config):
     mbox2dir = mboxdir.joinpath(".DeltaChat")
     mbox2dir.mkdir()
     fill_mbox(mbox2dir)
-    mb = MailboxStat(mboxdir, addr)
+    mb = MailboxStat(mboxdir, addr, False)
     assert len(mb.messages) == 2
 
 
@@ -71,7 +71,11 @@ def test_filentry_ordering(tmp_path):
 
 
 def test_no_mailbxoes(tmp_path, capsys):
-    assert [] == list(iter_mailboxes(str(tmp_path.joinpath("notexists")), maxnum=10))
+    assert [] == list(
+        iter_mailboxes(
+            str(tmp_path.joinpath("notexists")), maxnum=10, tmpfs_index=False
+        )
+    )
     out, err = capsys.readouterr()
     assert "no mailboxes" in err
 
@@ -88,13 +92,13 @@ def test_stats_mailbox(mbox1):
 
     create_new_messages(mbox1.basedir, ["large-extra"], size=1000)
     create_new_messages(mbox1.basedir, ["index-something"], size=3)
-    mbox2 = MailboxStat(mbox1.basedir, mbox1.name)
+    mbox2 = MailboxStat(mbox1.basedir, mbox1.name, False)
     assert len(mbox2.extrafiles) == 5
     assert mbox2.extrafiles[0].size == 1000
 
     # cope well with mailbox dirs that have no password (for whatever reason)
     Path(mbox1.basedir).joinpath("password").unlink()
-    mbox3 = MailboxStat(mbox1.basedir, mbox1.name)
+    mbox3 = MailboxStat(mbox1.basedir, mbox1.name, False)
     assert mbox3.last_login is None
 
 

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -74,12 +74,6 @@ mail_location = maildir:{{ config.mailboxes_dir }}/%u:INDEX=/dev/shm/%u
 mail_location = maildir:{{ config.mailboxes_dir }}/%u
 {% endif %}
 
-# index/cache files are not very useful for chatmail relay operations 
-# but it's not clear how to disable them completely. 
-# According to https://doc.dovecot.org/2.3/settings/advanced/#core_setting-mail_cache_max_size
-# if the cache file becomes larger than the specified size, it is truncated by dovecot 
-mail_cache_max_size = 500K
-
 namespace inbox {
   inbox = yes