Remove acmetool cronjob

We already require that outgoing connections
use STARTTLS so other servers need a valid TLS
certificate to accept messages from us.
It is then very unlikely that they cannot use TLS
to send messages to us.

Conversely, if they only can send messages to use without TLS,
it likely does not have STARTLS on its port 25
and then we don't want to accept messages from them
because we will likely not be able to reply.

.github/ISSUE_TEMPLATE/config.yml

@@ -1,5 +1 @@
 blank_issues_enabled: true
-contact_links:
-  - name: Mutual Help Chat Group
-    url: https://i.delta.chat/#6CBFF8FFD505C0FDEA20A66674F2916EA8FBEE99&a=invitebot%40nine.testrun.org&g=Chatmail%20Mutual%20Help&x=7sFF7Ik50pWv6J1z7RVC5527&i=X69wTFfvCfs3d-JzqP0kVA3i&s=ibp-447dU-wUq-52QanwAtWc
-    about: If you have troubles setting up the relay server, feel free to ask here.

ARCHITECTURE.md

@@ -19,7 +19,6 @@ graph LR;
     /var/lib/acme`")] --> nginx-internal;
     cron --- chatmail-metrics;
     cron --- acmetool;
-    cron --- expunge;
     chatmail-metrics --- website;
     acmetool --> certs[("`TLS certs
     /var/lib/acme`")];
@@ -35,7 +34,8 @@ graph LR;
     dovecot --- users;
     dovecot --- |metadata.socket|chatmail-metadata;
     doveauth --- users;
-    expunge --- users;
+    chatmail-expire-daily --- users;
+    chatmail-fsreport-daily --- users;
     chatmail-metadata --- iroh-relay;
     certs-nginx --> postfix;
     certs-nginx --> dovecot;

CHANGELOG.md

@@ -2,6 +2,15 @@
 
 ## untagged
 
+- acmetool: use ECDSA keys instead of RSA
+  ([#689](https://github.com/chatmail/relay/pull/689))
+
+- Require TLS 1.2 for outgoing SMTP connections
+  ([#685](https://github.com/chatmail/relay/pull/685))
+
+- require STARTTLS for incoming port 25 connections
+  ([#684](https://github.com/chatmail/relay/pull/684))
+
 - filtermail: run CPU-intensive handle_DATA in a thread pool executor
   ([#676](https://github.com/chatmail/relay/pull/676))
 
@@ -21,7 +30,7 @@
   ([#650](https://github.com/chatmail/relay/pull/650))
 
 - filtermail: accept mails from Protonmail
-  ([#616](https://github.com/chatmail/relay/pull/655))
+  ([#616](https://github.com/chatmail/relay/pull/616))
 
 - Ignore all RCPT TO: parameters
   ([#651](https://github.com/chatmail/relay/pull/651))
@@ -50,6 +59,13 @@
 - Add `--skip-dns-check` argument to `cmdeploy run` command, which disables DNS record checking before installation.
   ([#661](https://github.com/chatmail/relay/pull/661))
 
+- Rework expiry of message files and mailboxes in Python 
+  to only do a single iteration over sometimes millions of messages
+  instead of doing "find" commands that iterate 9 times over the messages. 
+  Provide an "fsreport" CLI for more fine grained analysis of message files. 
+  ([#637](https://github.com/chatmail/relay/pull/637))
+
+
 ## 1.7.0 2025-09-11
 
 - Make www upload path configurable

README.md

@@ -180,6 +180,10 @@ The components of chatmail are:
 - [Iroh relay](https://www.iroh.computer/docs/concepts/relay) 
   which helps client devices to establish Peer-to-Peer connections 
 
+- [TURN](https://github.com/chatmail/chatmail-turn)
+  to enable relay users to start webRTC calls
+  even if a p2p connection can't be established
+
 - and the chatmaild services, explained in the next section:
 
 ### chatmaild
@@ -304,6 +308,8 @@ Chatmail address creation will be denied while this file is present.
 [Nginx](https://www.nginx.com/) listens on port 8443 (HTTPS-ALT) and 443 (HTTPS).
 Port 443 multiplexes HTTPS, IMAP and SMTP using ALPN to redirect connections to ports 8443, 465 or 993.
 [acmetool](https://hlandau.github.io/acmetool/) listens on port 80 (HTTP).
+[chatmail-turn](https://github.com/chatmail/chatmail-turn) listens on UDP port 3478 (STUN/TURN),
+and temporarily opens UDP ports when users request them. UDP port range is not restricted, any free port may be allocated.
 
 chatmail-core based apps will, however, discover all ports and configurations
 automatically by reading the [autoconfig XML file](https://www.ietf.org/archive/id/draft-bucksch-autoconfig-00.html) from the chatmail relay server.

chatmaild/pyproject.toml

@@ -27,7 +27,8 @@ chatmail-metadata = "chatmaild.metadata:main"
 filtermail = "chatmaild.filtermail:main"
 echobot = "chatmaild.echo:main"
 chatmail-metrics = "chatmaild.metrics:main"
-delete_inactive_users = "chatmaild.delete_inactive_users:main"
+chatmail-expire = "chatmaild.expire:main"
+chatmail-fsreport = "chatmaild.fsreport:main"
 lastlogin = "chatmaild.lastlogin:main"
 turnserver = "chatmaild.turnserver:main"
 

chatmaild/src/chatmaild/delete_inactive_users.py

@@ -1,31 +0,0 @@
-"""
-Remove inactive users
-"""
-
-import os
-import shutil
-import sys
-import time
-
-from .config import read_config
-
-
-def delete_inactive_users(config):
-    cutoff_date = time.time() - config.delete_inactive_users_after * 86400
-    for addr in os.listdir(config.mailboxes_dir):
-        try:
-            user = config.get_user(addr)
-        except ValueError:
-            continue
-
-        read_timestamp = user.get_last_login_timestamp()
-        if read_timestamp and read_timestamp < cutoff_date:
-            path = config.mailboxes_dir.joinpath(addr)
-            assert path == user.maildir
-            shutil.rmtree(path, ignore_errors=True)
-
-
-def main():
-    (cfgpath,) = sys.argv[1:]
-    config = read_config(cfgpath)
-    delete_inactive_users(config)

chatmaild/src/chatmaild/expire.py

@@ -0,0 +1,218 @@
+"""
+Expire old messages and addresses.
+
+"""
+
+import os
+import shutil
+import sys
+import time
+from argparse import ArgumentParser
+from collections import namedtuple
+from datetime import datetime
+from stat import S_ISREG
+
+from chatmaild.config import read_config
+
+FileEntry = namedtuple("FileEntry", ("relpath", "mtime", "size"))
+
+
+def iter_mailboxes(basedir, maxnum):
+    if not os.path.exists(basedir):
+        print_info(f"no mailboxes found at: {basedir}")
+        return
+
+    for name in os_listdir_if_exists(basedir)[:maxnum]:
+        if "@" in name:
+            yield MailboxStat(basedir + "/" + name)
+
+
+def get_file_entry(path):
+    """return a FileEntry or None if the path does not exist or is not a regular file."""
+    try:
+        st = os.stat(path)
+    except FileNotFoundError:
+        return None
+    if not S_ISREG(st.st_mode):
+        return None
+    return FileEntry(path, st.st_mtime, st.st_size)
+
+
+def os_listdir_if_exists(path):
+    """return a list of names obtained from os.listdir or an empty list if the path does not exist."""
+    try:
+        return os.listdir(path)
+    except FileNotFoundError:
+        return []
+
+
+class MailboxStat:
+    last_login = None
+
+    def __init__(self, basedir):
+        self.basedir = str(basedir)
+        # all detected messages in cur/new/tmp folders
+        self.messages = []
+
+        # all detected files in mailbox top dir
+        self.extrafiles = []
+
+        # scan all relevant files (without recursion)
+        old_cwd = os.getcwd()
+        try:
+            os.chdir(self.basedir)
+        except FileNotFoundError:
+            return
+        for name in os_listdir_if_exists("."):
+            if name in ("cur", "new", "tmp"):
+                for msg_name in os_listdir_if_exists(name):
+                    entry = get_file_entry(f"{name}/{msg_name}")
+                    if entry is not None:
+                        self.messages.append(entry)
+
+            else:
+                entry = get_file_entry(name)
+                if entry is not None:
+                    self.extrafiles.append(entry)
+                    if name == "password":
+                        self.last_login = entry.mtime
+        self.extrafiles.sort(key=lambda x: -x.size)
+        os.chdir(old_cwd)
+
+
+def print_info(msg):
+    print(msg, file=sys.stderr)
+
+
+class Expiry:
+    def __init__(self, config, dry, now, verbose):
+        self.config = config
+        self.dry = dry
+        self.now = now
+        self.verbose = verbose
+        self.del_mboxes = 0
+        self.all_mboxes = 0
+        self.del_files = 0
+        self.all_files = 0
+        self.start = time.time()
+
+    def remove_mailbox(self, mboxdir):
+        if self.verbose:
+            print_info(f"removing {mboxdir}")
+        if not self.dry:
+            shutil.rmtree(mboxdir)
+        self.del_mboxes += 1
+
+    def remove_file(self, path, mtime=None):
+        if self.verbose:
+            if mtime is not None:
+                date = datetime.fromtimestamp(mtime).strftime("%b %d")
+                print_info(f"removing {date} {path}")
+            else:
+                print_info(f"removing {path}")
+        if not self.dry:
+            try:
+                os.unlink(path)
+            except FileNotFoundError:
+                print_info(f"file not found/vanished {path}")
+        self.del_files += 1
+
+    def process_mailbox_stat(self, mbox):
+        cutoff_without_login = (
+            self.now - int(self.config.delete_inactive_users_after) * 86400
+        )
+        cutoff_mails = self.now - int(self.config.delete_mails_after) * 86400
+        cutoff_large_mails = self.now - int(self.config.delete_large_after) * 86400
+
+        self.all_mboxes += 1
+        changed = False
+        if mbox.last_login and mbox.last_login < cutoff_without_login:
+            self.remove_mailbox(mbox.basedir)
+            return
+
+        # all to-be-removed files are relative to the mailbox basedir
+        try:
+            os.chdir(mbox.basedir)
+        except FileNotFoundError:
+            print_info(f"mailbox not found/vanished {mbox.basedir}")
+            return
+
+        mboxname = os.path.basename(mbox.basedir)
+        if self.verbose:
+            date = datetime.fromtimestamp(mbox.last_login) if mbox.last_login else None
+            if date:
+                print_info(f"checking mailbox {date.strftime('%b %d')} {mboxname}")
+            else:
+                print_info(f"checking mailbox (no last_login) {mboxname}")
+        self.all_files += len(mbox.messages)
+        for message in mbox.messages:
+            if message.mtime < cutoff_mails:
+                self.remove_file(message.relpath, mtime=message.mtime)
+            elif message.size > 200000 and message.mtime < cutoff_large_mails:
+                # we only remove noticed large files (not unnoticed ones in new/)
+                if message.relpath.startswith("cur/"):
+                    self.remove_file(message.relpath, mtime=message.mtime)
+            else:
+                continue
+            changed = True
+        if changed:
+            self.remove_file("maildirsize")
+
+    def get_summary(self):
+        return (
+            f"Removed {self.del_mboxes} out of {self.all_mboxes} mailboxes "
+            f"and {self.del_files} out of {self.all_files} files in existing mailboxes "
+            f"in {time.time() - self.start:2.2f} seconds"
+        )
+
+
+def main(args=None):
+    """Expire mailboxes and messages according to chatmail config"""
+    parser = ArgumentParser(description=main.__doc__)
+    ini = "/usr/local/lib/chatmaild/chatmail.ini"
+    parser.add_argument(
+        "chatmail_ini",
+        action="store",
+        nargs="?",
+        help=f"path pointing to chatmail.ini file, default: {ini}",
+        default=ini,
+    )
+    parser.add_argument(
+        "--days", action="store", help="assume date to be days older than now"
+    )
+
+    parser.add_argument(
+        "--maxnum",
+        default=None,
+        action="store",
+        help="maximum number of mailboxes to iterate on",
+    )
+    parser.add_argument(
+        "-v",
+        dest="verbose",
+        action="store_true",
+        help="print out removed files and mailboxes",
+    )
+
+    parser.add_argument(
+        "--remove",
+        dest="remove",
+        action="store_true",
+        help="actually remove all expired files and dirs",
+    )
+    args = parser.parse_args(args)
+
+    config = read_config(args.chatmail_ini)
+    now = datetime.utcnow().timestamp()
+    if args.days:
+        now = now - 86400 * int(args.days)
+
+    maxnum = int(args.maxnum) if args.maxnum else None
+    exp = Expiry(config, dry=not args.remove, now=now, verbose=args.verbose)
+    for mailbox in iter_mailboxes(str(config.mailboxes_dir), maxnum=maxnum):
+        exp.process_mailbox_stat(mailbox)
+    print(exp.get_summary())
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

chatmaild/src/chatmaild/fsreport.py

@@ -0,0 +1,168 @@
+"""
+command line tool to analyze mailbox message storage
+
+example invocation:
+
+    python -m chatmaild.fsreport /path/to/chatmail.ini
+
+to show storage summaries for all "cur" folders
+
+    python -m chatmaild.fsreport /path/to/chatmail.ini --mdir cur
+
+to show storage summaries only for first 1000 mailboxes
+
+    python -m chatmaild.fsreport /path/to/chatmail.ini --maxnum 1000
+
+"""
+
+import os
+from argparse import ArgumentParser
+from datetime import datetime
+
+from chatmaild.config import read_config
+from chatmaild.expire import iter_mailboxes
+
+DAYSECONDS = 24 * 60 * 60
+MONTHSECONDS = DAYSECONDS * 30
+
+
+def HSize(size: int):
+    """Format a size integer as a Human-readable string Kilobyte, Megabyte or Gigabyte"""
+    if size < 10000:
+        return f"{size / 1000:5.2f}K"
+    if size < 1000 * 1000:
+        return f"{size / 1000:5.0f}K"
+    if size < 1000 * 1000 * 1000:
+        return f"{int(size / 1000000):5.0f}M"
+    return f"{size / 1000000000:5.2f}G"
+
+
+class Report:
+    def __init__(self, now, min_login_age, mdir):
+        self.size_extra = 0
+        self.size_messages = 0
+        self.now = now
+        self.min_login_age = min_login_age
+        self.mdir = mdir
+
+        self.num_ci_logins = self.num_all_logins = 0
+        self.login_buckets = {x: 0 for x in (1, 10, 30, 40, 80, 100, 150)}
+
+        self.message_buckets = {x: 0 for x in (0, 160000, 500000, 2000000)}
+
+    def process_mailbox_stat(self, mailbox):
+        # categorize login times
+        last_login = mailbox.last_login
+        if last_login:
+            self.num_all_logins += 1
+            if os.path.basename(mailbox.basedir)[:3] == "ci-":
+                self.num_ci_logins += 1
+            else:
+                for days in self.login_buckets:
+                    if last_login >= self.now - days * DAYSECONDS:
+                        self.login_buckets[days] += 1
+
+        cutoff_login_date = self.now - self.min_login_age * DAYSECONDS
+        if last_login and last_login <= cutoff_login_date:
+            # categorize message sizes
+            for size in self.message_buckets:
+                for msg in mailbox.messages:
+                    if msg.size >= size:
+                        if self.mdir and not msg.relpath.startswith(self.mdir):
+                            continue
+                        self.message_buckets[size] += msg.size
+
+        self.size_messages += sum(entry.size for entry in mailbox.messages)
+        self.size_extra += sum(entry.size for entry in mailbox.extrafiles)
+
+    def dump_summary(self):
+        all_messages = self.size_messages
+        print()
+        print("## Mailbox storage use analysis")
+        print(f"Mailbox data total size: {HSize(self.size_extra + all_messages)}")
+        print(f"Messages total size    : {HSize(all_messages)}")
+        try:
+            percent = self.size_extra / (self.size_extra + all_messages) * 100
+        except ZeroDivisionError:
+            percent = 100
+        print(f"Extra files : {HSize(self.size_extra)} ({percent:.2f}%)")
+
+        print()
+        if self.min_login_age:
+            print(f"### Message storage for {self.min_login_age} days old logins")
+
+        pref = f"[{self.mdir}] " if self.mdir else ""
+        for minsize, sumsize in self.message_buckets.items():
+            percent = (sumsize / all_messages * 100) if all_messages else 0
+            print(
+                f"{pref}larger than {HSize(minsize)}: {HSize(sumsize)} ({percent:.2f}%)"
+            )
+
+        user_logins = self.num_all_logins - self.num_ci_logins
+
+        def p(num):
+            return f"({num / user_logins * 100:2.2f}%)" if user_logins else "100%"
+
+        print()
+        print(f"## Login stats, from date reference {datetime.fromtimestamp(self.now)}")
+        print(f"all:     {HSize(self.num_all_logins)}")
+        print(f"non-ci:  {HSize(user_logins)}")
+        print(f"ci:      {HSize(self.num_ci_logins)}")
+        for days, active in self.login_buckets.items():
+            print(f"last {days:3} days: {HSize(active)} {p(active)}")
+
+
+def main(args=None):
+    """Report about filesystem storage usage of all mailboxes and messages"""
+    parser = ArgumentParser(description=main.__doc__)
+    ini = "/usr/local/lib/chatmaild/chatmail.ini"
+    parser.add_argument(
+        "chatmail_ini",
+        action="store",
+        nargs="?",
+        help=f"path pointing to chatmail.ini file, default: {ini}",
+        default=ini,
+    )
+    parser.add_argument(
+        "--days",
+        default=0,
+        action="store",
+        help="assume date to be days older than now",
+    )
+    parser.add_argument(
+        "--min-login-age",
+        default=0,
+        dest="min_login_age",
+        action="store",
+        help="only sum up message size if last login is at least min-login-age days old",
+    )
+    parser.add_argument(
+        "--mdir",
+        action="store",
+        help="only consider 'cur' or 'new' or 'tmp' messages for summary",
+    )
+
+    parser.add_argument(
+        "--maxnum",
+        default=None,
+        action="store",
+        help="maximum number of mailboxes to iterate on",
+    )
+
+    args = parser.parse_args(args)
+
+    config = read_config(args.chatmail_ini)
+
+    now = datetime.utcnow().timestamp()
+    if args.days:
+        now = now - 86400 * int(args.days)
+
+    maxnum = int(args.maxnum) if args.maxnum else None
+    rep = Report(now=now, min_login_age=int(args.min_login_age), mdir=args.mdir)
+    for mbox in iter_mailboxes(str(config.mailboxes_dir), maxnum=maxnum):
+        rep.process_mailbox_stat(mbox)
+    rep.dump_summary()
+
+
+if __name__ == "__main__":
+    main()

chatmaild/src/chatmaild/tests/test_delete_inactive_users.py

@@ -1,7 +1,7 @@
 import time
 
-from chatmaild.delete_inactive_users import delete_inactive_users
 from chatmaild.doveauth import AuthDictProxy
+from chatmaild.expire import main as main_expire
 
 
 def test_login_timestamps(example_config):
@@ -45,7 +45,12 @@ def test_delete_inactive_users(example_config):
     for addr in to_remove:
         assert example_config.get_user(addr).maildir.exists()
 
-    delete_inactive_users(example_config)
+    main_expire(
+        args=[
+            "--remove",
+            str(example_config._inipath),
+        ]
+    )
 
     for p in example_config.mailboxes_dir.iterdir():
         assert not p.name.startswith("old")

chatmaild/src/chatmaild/tests/test_expire.py

@@ -0,0 +1,150 @@
+import os
+import random
+from datetime import datetime
+from fnmatch import fnmatch
+from pathlib import Path
+
+import pytest
+
+from chatmaild.expire import (
+    FileEntry,
+    MailboxStat,
+    get_file_entry,
+    iter_mailboxes,
+    os_listdir_if_exists,
+)
+from chatmaild.expire import main as expiry_main
+from chatmaild.fsreport import main as report_main
+
+
+def fill_mbox(basedir):
+    basedir1 = basedir.joinpath("mailbox1@example.org")
+    basedir1.mkdir()
+    password = basedir1.joinpath("password")
+    password.write_text("xxx")
+    basedir1.joinpath("maildirsize").write_text("xxx")
+
+    garbagedir = basedir1.joinpath("garbagedir")
+    garbagedir.mkdir()
+
+    create_new_messages(basedir1, ["cur/msg1"], size=500)
+    create_new_messages(basedir1, ["new/msg2"], size=600)
+    return basedir1
+
+
+def create_new_messages(basedir, relpaths, size=1000, days=0):
+    now = datetime.utcnow().timestamp()
+
+    for relpath in relpaths:
+        msg_path = Path(basedir).joinpath(relpath)
+        msg_path.parent.mkdir(parents=True, exist_ok=True)
+        msg_path.write_text("x" * size)
+        # accessed now, modified N days ago
+        os.utime(msg_path, (now, now - days * 86400))
+
+
+@pytest.fixture
+def mbox1(example_config):
+    basedir1 = fill_mbox(example_config.mailboxes_dir)
+    return MailboxStat(basedir1)
+
+
+def test_filentry_ordering(tmp_path):
+    l = [FileEntry(f"x{i}", size=i + 10, mtime=1000 - i) for i in range(10)]
+    sorted = list(l)
+    random.shuffle(l)
+    l.sort(key=lambda x: x.size)
+    assert l == sorted
+
+
+def test_no_mailbxoes(tmp_path, capsys):
+    assert [] == list(iter_mailboxes(str(tmp_path.joinpath("notexists")), maxnum=10))
+    out, err = capsys.readouterr()
+    assert "no mailboxes" in err
+
+
+def test_stats_mailbox(mbox1):
+    password = Path(mbox1.basedir).joinpath("password")
+    assert mbox1.last_login == password.stat().st_mtime
+    assert len(mbox1.messages) == 2
+
+    msgs = list(sorted(mbox1.messages, key=lambda x: x.size))
+    assert len(msgs) == 2
+    assert msgs[0].size == 500  # cur
+    assert msgs[1].size == 600  # new
+
+    create_new_messages(mbox1.basedir, ["large-extra"], size=1000)
+    create_new_messages(mbox1.basedir, ["index-something"], size=3)
+    mbox2 = MailboxStat(mbox1.basedir)
+    assert len(mbox2.extrafiles) == 4
+    assert mbox2.extrafiles[0].size == 1000
+
+    # cope well with mailbox dirs that have no password (for whatever reason)
+    Path(mbox1.basedir).joinpath("password").unlink()
+    mbox3 = MailboxStat(mbox1.basedir)
+    assert mbox3.last_login is None
+
+
+def test_report_no_mailboxes(example_config):
+    args = (str(example_config._inipath),)
+    report_main(args)
+
+
+def test_report(mbox1, example_config):
+    args = (str(example_config._inipath),)
+    report_main(args)
+    args = list(args) + "--days 1".split()
+    report_main(args)
+    args = list(args) + "--min-login-age 1".split()
+    report_main(args)
+    args = list(args) + "--mdir cur".split()
+    report_main(args)
+
+
+def test_expiry_cli_basic(example_config, mbox1):
+    args = (str(example_config._inipath),)
+    expiry_main(args)
+
+
+def test_expiry_cli_old_files(capsys, example_config, mbox1):
+    relpaths_old = ["cur/msg_old1", "cur/msg_old1"]
+    cutoff_days = int(example_config.delete_mails_after) + 1
+    create_new_messages(mbox1.basedir, relpaths_old, size=1000, days=cutoff_days)
+
+    relpaths_large = ["cur/msg_old_large1", "new/msg_old_large2"]
+    cutoff_days = int(example_config.delete_large_after) + 1
+    create_new_messages(
+        mbox1.basedir, relpaths_large, size=1000 * 300, days=cutoff_days
+    )
+
+    create_new_messages(mbox1.basedir, ["cur/shouldstay"], size=1000 * 300, days=1)
+
+    args = str(example_config._inipath), "--remove", "-v"
+    expiry_main(args)
+    out, err = capsys.readouterr()
+
+    allpaths = relpaths_old + relpaths_large + ["maildirsize"]
+    for path in allpaths:
+        for line in err.split("\n"):
+            if fnmatch(line, f"removing*{path}"):
+                break
+        else:
+            if path != "new/msg_old_large2":
+                pytest.fail(f"failed to remove {path}\n{err}")
+
+    assert "shouldstay" not in err
+
+
+def test_get_file_entry(tmp_path):
+    assert get_file_entry(str(tmp_path.joinpath("123123"))) is None
+    p = tmp_path.joinpath("x")
+    p.write_text("hello")
+    entry = get_file_entry(str(p))
+    assert entry.size == 5
+    assert entry.mtime
+
+
+def test_os_listdir_if_exists(tmp_path):
+    tmp_path.joinpath("x").write_text("hello")
+    assert len(os_listdir_if_exists(str(tmp_path))) == 1
+    assert len(os_listdir_if_exists(str(tmp_path.joinpath("123123")))) == 0

cmdeploy/src/cmdeploy/__init__.py

@@ -129,6 +129,10 @@ def _install_remote_venv_with_chatmaild(config) -> None:
         "chatmail-metadata",
         "lastlogin",
         "turnserver",
+        "chatmail-expire",
+        "chatmail-expire.timer",
+        "chatmail-fsreport",
+        "chatmail-fsreport.timer",
     ):
         execpath = fn if fn != "filtermail-incoming" else "filtermail"
         params = dict(
@@ -137,27 +141,34 @@ def _install_remote_venv_with_chatmaild(config) -> None:
             remote_venv_dir=remote_venv_dir,
             mail_domain=config.mail_domain,
         )
-        source_path = importlib.resources.files(__package__).joinpath(
-            "service", f"{fn}.service.f"
-        )
+
+        basename = fn if "." in fn else f"{fn}.service"
+
+        source_path = importlib.resources.files(__package__).joinpath("service", f"{basename}.f")
         content = source_path.read_text().format(**params).encode()
 
         files.put(
-            name=f"Upload {fn}.service",
+            name=f"Upload {basename}",
             src=io.BytesIO(content),
-            dest=f"/etc/systemd/system/{fn}.service",
+            dest=f"/etc/systemd/system/{basename}",
             **root_owned,
         )
+        if fn == "chatmail-expire" or fn == "chatmail-fsreport":
+            # don't auto-start but let the corresponding timer trigger execution
+            enabled = False
+        else:
+            enabled = True
         systemd.service(
-            name=f"Setup {fn} service",
-            service=f"{fn}.service",
-            running=True,
-            enabled=True,
-            restarted=True,
+            name=f"Setup {basename}",
+            service=basename,
+            running=enabled,
+            enabled=enabled,
+            restarted=enabled,
             daemon_reload=True,
         )
 
 
+
 def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
     """Configures OpenDKIM"""
     need_restart = False
@@ -327,9 +338,9 @@ def _install_dovecot_package(package: str, arch: str):
 
     match (package, arch):
         case ("core", "amd64"):
-            sha256 = "43f593332e22ac7701c62d58b575d2ca409e0f64857a2803be886c22860f5587"
+            sha256 = "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d"
         case ("core", "arm64"):
-            sha256 = "4d21eba1a83f51c100f08f2e49f0c9f8f52f721ebc34f75018e043306da993a7"
+            sha256 = "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9"
         case ("imapd", "amd64"):
             sha256 = "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86"
         case ("imapd", "arm64"):
@@ -387,13 +398,11 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
     )
     need_restart |= lua_push_notification_script.changed
 
-    files.template(
-        src=importlib.resources.files(__package__).joinpath("dovecot/expunge.cron.j2"),
-        dest="/etc/cron.d/expunge",
-        user="root",
-        group="root",
-        mode="644",
-        config=config,
+    # remove historic expunge script
+    # which is now implemented through a systemd chatmail-expire service/timer
+    files.file(
+        path="/etc/cron.d/expunge",
+        present=False,
     )
 
     # as per https://doc.dovecot.org/configuration_manual/os/

cmdeploy/src/cmdeploy/acmetool/__init__.py

@@ -10,12 +10,9 @@ def deploy_acmetool(email="", domains=[]):
         packages=["acmetool"],
     )
 
-    files.put(
-        src=importlib.resources.files(__package__).joinpath("acmetool.cron").open("rb"),
-        dest="/etc/cron.d/acmetool",
-        user="root",
-        group="root",
-        mode="644",
+    files.file(
+        path="/etc/cron.d/acmetool",
+        present=False,
     )
 
     files.put(

cmdeploy/src/cmdeploy/acmetool/acmetool.cron

@@ -1,4 +0,0 @@
-SHELL=/bin/sh
-PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
-MAILTO=root
-20 16 * * * root /usr/bin/acmetool --batch reconcile && systemctl reload dovecot && systemctl reload postfix && systemctl reload nginx

cmdeploy/src/cmdeploy/acmetool/target.yaml.j2

@@ -1,7 +1,8 @@
 request:
   provider: https://acme-v02.api.letsencrypt.org/directory
   key:
-    type: rsa
+    type: ecdsa
+    ecdsa-curve: nistp256
   challenge:
     webroot-paths:
     - /var/www/html/.well-known/acme-challenge

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -70,6 +70,12 @@ userdb {
 # Mailboxes are stored in the "mail" directory of the vmail user home.
 mail_location = maildir:{{ config.mailboxes_dir }}/%u
 
+# index/cache files are not very useful for chatmail relay operations 
+# but it's not clear how to disable them completely. 
+# According to https://doc.dovecot.org/2.3/settings/advanced/#core_setting-mail_cache_max_size
+# if the cache file becomes larger than the specified size, it is truncated by dovecot 
+mail_cache_max_size = 500K
+
 namespace inbox {
   inbox = yes
 

cmdeploy/src/cmdeploy/dovecot/expunge.cron.j2

@@ -1,14 +0,0 @@
-# delete already seen big mails after 7 days, in the INBOX
-2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/cur/*' -mtime +{{ config.delete_large_after }} -size +200k -type f -delete
-# delete all mails after {{ config.delete_mails_after }} days, in the Inbox
-2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-# or in any IMAP subfolder
-2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/.*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-# even if they are unseen
-2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/.*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-# or only temporary (but then they shouldn't be around after {{ config.delete_mails_after }} days anyway).
-2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-2 0 * * * vmail find {{ config.mailboxes_dir }} -path '*/.*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-3 0 * * * vmail find {{ config.mailboxes_dir }} -name 'maildirsize' -type f -delete
-4 0 * * * vmail /usr/local/lib/chatmaild/venv/bin/delete_inactive_users /usr/local/lib/chatmaild/chatmail.ini 

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -26,6 +26,7 @@ smtp_tls_security_level=verify
 smtp_tls_servername = hostname
 smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
 smtp_tls_policy_maps = inline:{nauta.cu=may}
+smtp_tls_protocols = >=TLSv1.2
 smtpd_tls_protocols = >=TLSv1.2
 
 # Disable anonymous cipher suites

cmdeploy/src/cmdeploy/postfix/master.cf.j2

@@ -14,6 +14,7 @@ smtp      inet  n       -       y       -       -       smtpd -v
 {%- else %}
 smtp      inet  n       -       y       -       -       smtpd 
 {%- endif %}
+  -o smtpd_tls_security_level=encrypt
   -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port_incoming }}
 submission inet n       -       y       -       5000    smtpd
   -o syslog_name=postfix/submission

cmdeploy/src/cmdeploy/service/chatmail-expire.service.f

@@ -0,0 +1,9 @@
+[Unit]
+Description=chatmail mail storage expiration job
+After=network.target
+
+[Service]
+Type=oneshot
+User=vmail
+ExecStart=/usr/local/lib/chatmaild/venv/bin/chatmail-expire /usr/local/lib/chatmaild/chatmail.ini -v --remove
+

cmdeploy/src/cmdeploy/service/chatmail-expire.timer.f

@@ -0,0 +1,8 @@
+[Unit]
+Description=Run Daily chatmail-expire job
+
+[Timer]
+OnCalendar=*-*-* 00:02:00
+
+[Install]
+WantedBy=timers.target

cmdeploy/src/cmdeploy/service/chatmail-fsreport.service.f

@@ -0,0 +1,9 @@
+[Unit]
+Description=chatmail file system storage reporting job 
+After=network.target
+
+[Service]
+Type=oneshot
+User=vmail
+ExecStart=/usr/local/lib/chatmaild/venv/bin/chatmail-fsreport /usr/local/lib/chatmaild/chatmail.ini 
+

cmdeploy/src/cmdeploy/service/chatmail-fsreport.timer.f

@@ -0,0 +1,9 @@
+[Unit]
+Description=Run Daily Chatmail fsreport Job
+
+[Timer]
+OnCalendar=*-*-* 08:02:00
+Persistent=true
+
+[Install]
+WantedBy=timers.target

cmdeploy/src/cmdeploy/tests/online/test_0_login.py

@@ -1,5 +1,5 @@
 import queue
-import socket
+import smtplib
 import threading
 
 import pytest
@@ -91,25 +91,23 @@ def test_concurrent_logins_same_account(
 
 def test_no_vrfy(chatmail_config):
     domain = chatmail_config.mail_domain
-    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    sock.settimeout(10)
-    try:
-        sock.connect((domain, 25))
-    except socket.timeout:
-        pytest.skip(f"port 25 not reachable for {domain}")
-    banner = sock.recv(1024)
-    print(banner)
-    sock.send(b"VRFY wrongaddress@%s\r\n" % (chatmail_config.mail_domain.encode(),))
-    result = sock.recv(1024)
+
+    s = smtplib.SMTP(domain)
+    s.starttls()
+
+    s.putcmd("vrfy", f"wrongaddress@{chatmail_config.mail_domain}")
+    result = s.getreply()
     print(result)
-    sock.send(b"VRFY echo@%s\r\n" % (chatmail_config.mail_domain.encode(),))
-    result2 = sock.recv(1024)
+    s.putcmd("vrfy", f"echo@{chatmail_config.mail_domain}")
+    result2 = s.getreply()
     print(result2)
-    assert result[0:10] == result2[0:10]
-    sock.send(b"VRFY wrongaddress\r\n")
-    result = sock.recv(1024)
+    assert result[0] == result2[0] == 252
+    assert result[1][0:6] == result2[1][0:6] == b"2.0.0 "
+    s.putcmd("vrfy", "wrongaddress")
+    result = s.getreply()
     print(result)
-    sock.send(b"VRFY echo\r\n")
-    result2 = sock.recv(1024)
+    s.putcmd("vrfy", "echo")
+    result2 = s.getreply()
     print(result2)
-    assert result[0:10] == result2[0:10] == b"252 2.0.0 "
+    assert result[0] == result2[0] == 252
+    assert result[1][0:6] == result2[1][0:6] == b"2.0.0 "

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -143,6 +143,7 @@ def test_reject_missing_dkim(cmsetup, maildata, from_addr):
         "encrypted.eml", from_addr=from_addr, to_addr=recipient.addr
     ).as_string()
     conn = smtplib.SMTP(cmsetup.maildomain, 25, timeout=10)
+    conn.starttls()
 
     with conn as s:
         with pytest.raises(smtplib.SMTPDataError, match="No valid DKIM signature"):