config: fix overriding chatmail.ini params

chatmaild/src/chatmaild/config.py

@@ -46,7 +46,6 @@ class Config:
         self.acme_email = params.get("acme_email", "")
         self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true"
         self.imap_compress = params.get("imap_compress", "false").lower() == "true"
-        self.turn_socket_path = params.get("turn_socket_path", "/run/chatmail-turn/turn.socket")
         if "iroh_relay" not in params:
             self.iroh_relay = "https://" + params["mail_domain"]
             self.enable_iroh_relay = True
@@ -57,6 +56,7 @@ class Config:
         self.privacy_mail = params.get("privacy_mail")
         self.privacy_pdo = params.get("privacy_pdo")
         self.privacy_supervisor = params.get("privacy_supervisor")
+        self.tmpfs_index = params.get("tmpfs_index", "false").lower() == "true"
 
         # deprecated option
         mbdir = params.get("mailboxes_dir", f"/home/vmail/mail/{self.mail_domain}")
@@ -112,10 +112,10 @@ def get_default_config_content(mail_domain, **overrides):
 
     if mail_domain.endswith(".testrun.org"):
         override_inipath = inidir.joinpath("override-testrun.ini")
-        privacy = iniconfig.IniConfig(override_inipath)["privacy"]
+        params = iniconfig.IniConfig(override_inipath)["params"]
         lines = []
         for line in content.split("\n"):
-            for key, value in privacy.items():
+            for key, value in params.items():
                 value_lines = value.format(mail_domain=mail_domain).strip().split("\n")
                 if not line.startswith(f"{key} =") or not value_lines:
                     continue

chatmaild/src/chatmaild/expire.py

@@ -17,14 +17,14 @@ from chatmaild.config import read_config
 FileEntry = namedtuple("FileEntry", ("path", "mtime", "size"))
 
 
-def iter_mailboxes(basedir, maxnum):
+def iter_mailboxes(basedir, maxnum, tmpfs_index):
     if not os.path.exists(basedir):
         print_info(f"no mailboxes found at: {basedir}")
         return
 
     for name in os_listdir_if_exists(basedir)[:maxnum]:
         if "@" in name:
-            yield MailboxStat(basedir + "/" + name)
+            yield MailboxStat(basedir + "/" + name, name, tmpfs_index)
 
 
 def get_file_entry(path):
@@ -49,11 +49,14 @@ def os_listdir_if_exists(path):
 class MailboxStat:
     last_login = None
 
-    def __init__(self, basedir):
+    def __init__(self, basedir, name, tmpfs_index):
         self.basedir = str(basedir)
+        self.name = name
         self.messages = []
         self.extrafiles = []
         self.scandir(self.basedir)
+        if tmpfs_index:
+            self.scandir("/dev/shm/" + name)
 
     def scandir(self, folderdir):
         for name in os_listdir_if_exists(folderdir):
@@ -90,11 +93,13 @@ class Expiry:
         self.all_files = 0
         self.start = time.time()
 
-    def remove_mailbox(self, mboxdir):
+    def remove_mailbox(self, mboxdir, name):
         if self.verbose:
             print_info(f"removing {mboxdir}")
         if not self.dry:
             shutil.rmtree(mboxdir)
+            if self.config.tmpfs_index:
+                shutil.rmtree("/dev/shm/" + name)
         self.del_mboxes += 1
 
     def remove_file(self, path, mtime=None):
@@ -121,7 +126,7 @@ class Expiry:
         self.all_mboxes += 1
         changed = False
         if mbox.last_login and mbox.last_login < cutoff_without_login:
-            self.remove_mailbox(mbox.basedir)
+            self.remove_mailbox(mbox.basedir, mbox.name)
             return
 
         mboxname = os.path.basename(mbox.basedir)
@@ -145,6 +150,9 @@ class Expiry:
             changed = True
         if changed:
             self.remove_file(f"{mbox.basedir}/maildirsize")
+        for file in mbox.extrafiles:
+            if "dovecot.index" in file.path.split("/")[-1] and file.size > 500 * 1024:
+                self.remove_file(file.path)
 
     def get_summary(self):
         return (
@@ -197,7 +205,9 @@ def main(args=None):
 
     maxnum = int(args.maxnum) if args.maxnum else None
     exp = Expiry(config, dry=not args.remove, now=now, verbose=args.verbose)
-    for mailbox in iter_mailboxes(str(config.mailboxes_dir), maxnum=maxnum):
+    for mailbox in iter_mailboxes(
+        str(config.mailboxes_dir), maxnum, config.tmpfs_index
+    ):
         exp.process_mailbox_stat(mailbox)
     print(exp.get_summary())
 

chatmaild/src/chatmaild/fsreport.py

@@ -159,7 +159,7 @@ def main(args=None):
 
     maxnum = int(args.maxnum) if args.maxnum else None
     rep = Report(now=now, min_login_age=int(args.min_login_age), mdir=args.mdir)
-    for mbox in iter_mailboxes(str(config.mailboxes_dir), maxnum=maxnum):
+    for mbox in iter_mailboxes(str(config.mailboxes_dir), maxnum, config.tmpfs_index):
         rep.process_mailbox_stat(mbox)
     rep.dump_summary()
 

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -48,6 +48,9 @@ passthrough_senders =
 # (space-separated, item may start with "@" to whitelist whole recipient domains)
 passthrough_recipients =
 
+# store index files in tmpfs (good for disk size and I/O, bad for ram)
+tmpfs_index = false
+
 # path to www directory - documented here: https://chatmail.at/doc/relay/getting_started.html#custom-web-pages
 #www_folder = www
 
@@ -55,10 +58,7 @@ passthrough_recipients =
 # Deployment Details
 #
 
-# Path to the TURN server Unix socket
+# SMTP outgoing filtermail and reinjection 
-turn_socket_path = /run/chatmail-turn/turn.socket
-
-# SMTP outgoing filtermail and reinjection
 filtermail_smtp_port = 10080
 postfix_reinject_port = 10025
 

chatmaild/src/chatmaild/ini/override-testrun.ini

@@ -1,5 +1,6 @@
+[params]
 
-[privacy]
+tmpfs_index = true
 
 passthrough_recipients = privacy@testrun.org echo@{mail_domain}
 

chatmaild/src/chatmaild/metadata.py

@@ -76,13 +76,12 @@ class Metadata:
 
 
 class MetadataDictProxy(DictProxy):
-    def __init__(self, notifier, metadata, iroh_relay=None, turn_hostname=None, config=None):
+    def __init__(self, notifier, metadata, iroh_relay=None, turn_hostname=None):
         super().__init__()
         self.notifier = notifier
         self.metadata = metadata
         self.iroh_relay = iroh_relay
         self.turn_hostname = turn_hostname
-        self.config = config
 
     def handle_lookup(self, parts):
         # Lpriv/43f5f508a7ea0366dff30200c15250e3/devicetoken\tlkj123poi@c2.testrun.org
@@ -102,7 +101,7 @@ class MetadataDictProxy(DictProxy):
                 # Handle `GETMETADATA "" /shared/vendor/deltachat/irohrelay`
                 return f"O{self.iroh_relay}\n"
             elif keyname == "vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn":
-                res = turn_credentials(self.config)
+                res = turn_credentials()
                 port = 3478
                 return f"O{self.turn_hostname}:{port}:{res}\n"
 
@@ -147,7 +146,6 @@ def main():
         metadata=metadata,
         iroh_relay=iroh_relay,
         turn_hostname=mail_domain,
-        config=config,
     )
 
     dictproxy.serve_forever_from_socket(socket)

chatmaild/src/chatmaild/tests/test_expire.py

@@ -43,20 +43,22 @@ def create_new_messages(basedir, relpaths, size=1000, days=0):
 
 @pytest.fixture
 def mbox1(example_config):
-    mboxdir = example_config.mailboxes_dir.joinpath("mailbox1@example.org")
+    addr = "mailbox1@example.org"
+    mboxdir = example_config.mailboxes_dir.joinpath(addr)
     mboxdir.mkdir()
     fill_mbox(mboxdir)
-    return MailboxStat(mboxdir)
+    return MailboxStat(mboxdir, addr, False)
 
 
 def test_deltachat_folder(example_config):
     """Test old setups that might have a .DeltaChat folder where messages also need to get removed."""
-    mboxdir = example_config.mailboxes_dir.joinpath("mailbox1@example.org")
+    addr = "mailbox1@example.org"
+    mboxdir = example_config.mailboxes_dir.joinpath(addr)
     mboxdir.mkdir()
     mbox2dir = mboxdir.joinpath(".DeltaChat")
     mbox2dir.mkdir()
     fill_mbox(mbox2dir)
-    mb = MailboxStat(mboxdir)
+    mb = MailboxStat(mboxdir, addr, False)
     assert len(mb.messages) == 2
 
 
@@ -69,7 +71,11 @@ def test_filentry_ordering(tmp_path):
 
 
 def test_no_mailbxoes(tmp_path, capsys):
-    assert [] == list(iter_mailboxes(str(tmp_path.joinpath("notexists")), maxnum=10))
+    assert [] == list(
+        iter_mailboxes(
+            str(tmp_path.joinpath("notexists")), maxnum=10, tmpfs_index=False
+        )
+    )
     out, err = capsys.readouterr()
     assert "no mailboxes" in err
 
@@ -86,13 +92,13 @@ def test_stats_mailbox(mbox1):
 
     create_new_messages(mbox1.basedir, ["large-extra"], size=1000)
     create_new_messages(mbox1.basedir, ["index-something"], size=3)
-    mbox2 = MailboxStat(mbox1.basedir)
+    mbox2 = MailboxStat(mbox1.basedir, mbox1.name, False)
     assert len(mbox2.extrafiles) == 5
     assert mbox2.extrafiles[0].size == 1000
 
     # cope well with mailbox dirs that have no password (for whatever reason)
     Path(mbox1.basedir).joinpath("password").unlink()
-    mbox3 = MailboxStat(mbox1.basedir)
+    mbox3 = MailboxStat(mbox1.basedir, mbox1.name, False)
     assert mbox3.last_login is None
 
 

chatmaild/src/chatmaild/tests/test_turnserver.py

@@ -1,120 +0,0 @@
-"""Tests for turnserver functionality, particularly metadata integration."""
-
-import socket
-import tempfile
-import threading
-from pathlib import Path
-
-from chatmaild.config import read_config, write_initial_config
-from chatmaild.metadata import MetadataDictProxy, Metadata
-from chatmaild.notifier import Notifier
-from chatmaild.turnserver import turn_credentials
-
-
-def test_turn_credentials_function_with_custom_socket():
-    """Test that turn_credentials function works with a custom socket path from config."""
-    # Create a temporary directory and socket file
-    temp_dir = Path(tempfile.mkdtemp())
-    temp_socket_path = temp_dir / "test_turn.socket"
-
-    # Create a mock TURN credentials server
-    def mock_server():
-        server_sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-        server_sock.bind(str(temp_socket_path))
-        server_sock.listen(1)
-
-        # Accept connection and send mock credentials
-        conn, addr = server_sock.accept()
-        with conn:
-            conn.send(b"mock_turn_credentials_abc123\n")
-        server_sock.close()
-
-    # Start server in a background thread
-    server_thread = threading.Thread(target=mock_server, daemon=True)
-    server_thread.start()
-
-    # Create a config with custom socket path
-    config_path = temp_dir / "chatmail.ini"
-    write_initial_config(config_path, "test.example.org", {
-        "turn_socket_path": str(temp_socket_path)
-    })
-    config = read_config(config_path)
-
-    # Allow time for server to start
-    import time
-    time.sleep(0.01)
-
-    # Test that turn_credentials can connect using the config
-    credentials = turn_credentials(config)
-    assert credentials == "mock_turn_credentials_abc123"
-
-    server_thread.join(timeout=1)  # Clean up thread
-
-
-def test_metadata_turn_lookup_integration(tmp_path):
-    """Test that metadata service properly handles TURN metadata lookups."""
-    # Create mock config with custom turn socket path
-    config_path = tmp_path / "chatmail.ini"
-    socket_path = tmp_path / "test_turn.socket"
-    write_initial_config(config_path, "example.org", {
-        "turn_socket_path": str(socket_path)
-    })
-    config = read_config(config_path)
-
-    # Create mock TURN server to return credentials
-    def mock_turn_server():
-        import os
-        os.makedirs(socket_path.parent, exist_ok=True)  # Ensure parent directory exists
-
-        server_sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-        server_sock.bind(str(socket_path))
-        server_sock.listen(1)
-
-        # Accept connection and send mock credentials
-        conn, addr = server_sock.accept()
-        with conn:
-            conn.send(b"test_creds_12345\n")
-        server_sock.close()
-
-    server_thread = threading.Thread(target=mock_turn_server, daemon=True)
-    server_thread.start()
-
-    import time
-    time.sleep(0.01)  # Allow server to start
-
-    # Create a MetadataDictProxy with config
-    queue_dir = tmp_path / "queue"
-    queue_dir.mkdir()
-    notifier = Notifier(queue_dir)
-    metadata = Metadata(tmp_path / "vmail")
-
-    dict_proxy = MetadataDictProxy(
-        notifier=notifier,
-        metadata=metadata,
-        iroh_relay="https://example.org",
-        turn_hostname="example.org",
-        config=config
-    )
-
-    # Simulate a lookup for TURN credentials using the correct format
-    # Input: "shared/0123/vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn"
-    # After parts[0].split("/", 2):
-    # - keyparts[0] = "shared"
-    # - keyparts[1] = "0123"
-    # - keyparts[2] = "vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn"
-    # So keyname = keyparts[2] should match "vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn"
-    parts = [
-        "shared/0123/vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn",
-        "dummy@user.org"
-    ]
-
-    # Call handle_lookup directly
-    result = dict_proxy.handle_lookup(parts)
-
-    # Verify the response format is correct for TURN credentials
-    assert result.startswith("O")  # Output response starts with 'O'
-    assert ":3478:" in result  # Contains port 3478
-    assert "test_creds_12345" in result  # Contains credentials returned by mock server
-    assert "example.org:3478:test_creds_12345" in result
-
-    server_thread.join(timeout=1)  # Clean up thread

chatmaild/src/chatmaild/turnserver.py

@@ -2,8 +2,8 @@
 import socket
 
 
-def turn_credentials(config) -> str:
+def turn_credentials() -> str:
     with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client_socket:
-        client_socket.connect(config.turn_socket_path)
+        client_socket.connect("/run/chatmail-turn/turn.socket")
         with client_socket.makefile("rb") as file:
             return file.readline().decode("utf-8").strip()

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -68,13 +68,11 @@ userdb {
 ##
 
 # Mailboxes are stored in the "mail" directory of the vmail user home.
+{% if config.tmpfs_index %}
+mail_location = maildir:{{ config.mailboxes_dir }}/%u:INDEX=/dev/shm/%u
+{% else %}
 mail_location = maildir:{{ config.mailboxes_dir }}/%u
-
+{% endif %}
-# index/cache files are not very useful for chatmail relay operations 
-# but it's not clear how to disable them completely. 
-# According to https://doc.dovecot.org/2.3/settings/advanced/#core_setting-mail_cache_max_size
-# if the cache file becomes larger than the specified size, it is truncated by dovecot 
-mail_cache_max_size = 500K
 
 namespace inbox {
   inbox = yes