config: fix overriding chatmail.ini params

fix: remediates issue with improper concat on resolver injection

.github/workflows/ci.yaml

@@ -15,7 +15,7 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: download filtermail
-        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.1.2/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
+        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.2.0/filtermail-x86_64-musl -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
       - name: run chatmaild tests 
         working-directory: chatmaild
         run: pipx run tox

chatmaild/src/chatmaild/config.py

@@ -20,7 +20,8 @@ class Config:
     def __init__(self, inipath, params):
         self._inipath = inipath
         self.mail_domain = params["mail_domain"]
-        self.max_user_send_per_minute = int(params["max_user_send_per_minute"])
+        self.max_user_send_per_minute = int(params.get("max_user_send_per_minute", 60))
+        self.max_user_send_burst_size = int(params.get("max_user_send_burst_size", 10))
         self.max_mailbox_size = params["max_mailbox_size"]
         self.max_message_size = int(params.get("max_message_size", "31457280"))
         self.delete_mails_after = params["delete_mails_after"]
@@ -55,6 +56,7 @@ class Config:
         self.privacy_mail = params.get("privacy_mail")
         self.privacy_pdo = params.get("privacy_pdo")
         self.privacy_supervisor = params.get("privacy_supervisor")
+        self.tmpfs_index = params.get("tmpfs_index", "false").lower() == "true"
 
         # deprecated option
         mbdir = params.get("mailboxes_dir", f"/home/vmail/mail/{self.mail_domain}")
@@ -110,10 +112,10 @@ def get_default_config_content(mail_domain, **overrides):
 
     if mail_domain.endswith(".testrun.org"):
         override_inipath = inidir.joinpath("override-testrun.ini")
-        privacy = iniconfig.IniConfig(override_inipath)["privacy"]
+        params = iniconfig.IniConfig(override_inipath)["params"]
         lines = []
         for line in content.split("\n"):
-            for key, value in privacy.items():
+            for key, value in params.items():
                 value_lines = value.format(mail_domain=mail_domain).strip().split("\n")
                 if not line.startswith(f"{key} =") or not value_lines:
                     continue

chatmaild/src/chatmaild/expire.py

@@ -17,14 +17,14 @@ from chatmaild.config import read_config
 FileEntry = namedtuple("FileEntry", ("path", "mtime", "size"))
 
 
-def iter_mailboxes(basedir, maxnum):
+def iter_mailboxes(basedir, maxnum, tmpfs_index):
     if not os.path.exists(basedir):
         print_info(f"no mailboxes found at: {basedir}")
         return
 
     for name in os_listdir_if_exists(basedir)[:maxnum]:
         if "@" in name:
-            yield MailboxStat(basedir + "/" + name)
+            yield MailboxStat(basedir + "/" + name, name, tmpfs_index)
 
 
 def get_file_entry(path):
@@ -49,11 +49,14 @@ def os_listdir_if_exists(path):
 class MailboxStat:
     last_login = None
 
-    def __init__(self, basedir):
+    def __init__(self, basedir, name, tmpfs_index):
         self.basedir = str(basedir)
+        self.name = name
         self.messages = []
         self.extrafiles = []
         self.scandir(self.basedir)
+        if tmpfs_index:
+            self.scandir("/dev/shm/" + name)
 
     def scandir(self, folderdir):
         for name in os_listdir_if_exists(folderdir):
@@ -90,11 +93,13 @@ class Expiry:
         self.all_files = 0
         self.start = time.time()
 
-    def remove_mailbox(self, mboxdir):
+    def remove_mailbox(self, mboxdir, name):
         if self.verbose:
             print_info(f"removing {mboxdir}")
         if not self.dry:
             shutil.rmtree(mboxdir)
+            if self.config.tmpfs_index:
+                shutil.rmtree("/dev/shm/" + name)
         self.del_mboxes += 1
 
     def remove_file(self, path, mtime=None):
@@ -121,7 +126,7 @@ class Expiry:
         self.all_mboxes += 1
         changed = False
         if mbox.last_login and mbox.last_login < cutoff_without_login:
-            self.remove_mailbox(mbox.basedir)
+            self.remove_mailbox(mbox.basedir, mbox.name)
             return
 
         mboxname = os.path.basename(mbox.basedir)
@@ -145,6 +150,9 @@ class Expiry:
             changed = True
         if changed:
             self.remove_file(f"{mbox.basedir}/maildirsize")
+        for file in mbox.extrafiles:
+            if "dovecot.index" in file.path.split("/")[-1] and file.size > 500 * 1024:
+                self.remove_file(file.path)
 
     def get_summary(self):
         return (
@@ -197,7 +205,9 @@ def main(args=None):
 
     maxnum = int(args.maxnum) if args.maxnum else None
     exp = Expiry(config, dry=not args.remove, now=now, verbose=args.verbose)
-    for mailbox in iter_mailboxes(str(config.mailboxes_dir), maxnum=maxnum):
+    for mailbox in iter_mailboxes(
+        str(config.mailboxes_dir), maxnum, config.tmpfs_index
+    ):
         exp.process_mailbox_stat(mailbox)
     print(exp.get_summary())
 

chatmaild/src/chatmaild/fsreport.py

@@ -159,7 +159,7 @@ def main(args=None):
 
     maxnum = int(args.maxnum) if args.maxnum else None
     rep = Report(now=now, min_login_age=int(args.min_login_age), mdir=args.mdir)
-    for mbox in iter_mailboxes(str(config.mailboxes_dir), maxnum=maxnum):
+    for mbox in iter_mailboxes(str(config.mailboxes_dir), maxnum, config.tmpfs_index):
         rep.process_mailbox_stat(mbox)
     rep.dump_summary()
 

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -11,9 +11,12 @@ mail_domain = {mail_domain}
 # Restrictions on user addresses
 #
 
-# how many mails a user can send out per minute
+# email sending rate per user and minute
 max_user_send_per_minute = 60
 
+# per-user max burst size for sending rate limiting (GCRA bucket capacity)
+max_user_send_burst_size = 10
+
 # maximum mailbox size of a chatmail address
 max_mailbox_size = 500M
 
@@ -45,6 +48,9 @@ passthrough_senders =
 # (space-separated, item may start with "@" to whitelist whole recipient domains)
 passthrough_recipients =
 
+# store index files in tmpfs (good for disk size and I/O, bad for ram)
+tmpfs_index = false
+
 # path to www directory - documented here: https://chatmail.at/doc/relay/getting_started.html#custom-web-pages
 #www_folder = www
 

chatmaild/src/chatmaild/ini/override-testrun.ini

@@ -1,5 +1,6 @@
+[params]
 
-[privacy]
+tmpfs_index = true
 
 passthrough_recipients = privacy@testrun.org echo@{mail_domain}
 

chatmaild/src/chatmaild/tests/test_expire.py

@@ -43,20 +43,22 @@ def create_new_messages(basedir, relpaths, size=1000, days=0):
 
 @pytest.fixture
 def mbox1(example_config):
-    mboxdir = example_config.mailboxes_dir.joinpath("mailbox1@example.org")
+    addr = "mailbox1@example.org"
+    mboxdir = example_config.mailboxes_dir.joinpath(addr)
     mboxdir.mkdir()
     fill_mbox(mboxdir)
-    return MailboxStat(mboxdir)
+    return MailboxStat(mboxdir, addr, False)
 
 
 def test_deltachat_folder(example_config):
     """Test old setups that might have a .DeltaChat folder where messages also need to get removed."""
-    mboxdir = example_config.mailboxes_dir.joinpath("mailbox1@example.org")
+    addr = "mailbox1@example.org"
+    mboxdir = example_config.mailboxes_dir.joinpath(addr)
     mboxdir.mkdir()
     mbox2dir = mboxdir.joinpath(".DeltaChat")
     mbox2dir.mkdir()
     fill_mbox(mbox2dir)
-    mb = MailboxStat(mboxdir)
+    mb = MailboxStat(mboxdir, addr, False)
     assert len(mb.messages) == 2
 
 
@@ -69,7 +71,11 @@ def test_filentry_ordering(tmp_path):
 
 
 def test_no_mailbxoes(tmp_path, capsys):
-    assert [] == list(iter_mailboxes(str(tmp_path.joinpath("notexists")), maxnum=10))
+    assert [] == list(
+        iter_mailboxes(
+            str(tmp_path.joinpath("notexists")), maxnum=10, tmpfs_index=False
+        )
+    )
     out, err = capsys.readouterr()
     assert "no mailboxes" in err
 
@@ -86,13 +92,13 @@ def test_stats_mailbox(mbox1):
 
     create_new_messages(mbox1.basedir, ["large-extra"], size=1000)
     create_new_messages(mbox1.basedir, ["index-something"], size=3)
-    mbox2 = MailboxStat(mbox1.basedir)
+    mbox2 = MailboxStat(mbox1.basedir, mbox1.name, False)
     assert len(mbox2.extrafiles) == 5
     assert mbox2.extrafiles[0].size == 1000
 
     # cope well with mailbox dirs that have no password (for whatever reason)
     Path(mbox1.basedir).joinpath("password").unlink()
-    mbox3 = MailboxStat(mbox1.basedir)
+    mbox3 = MailboxStat(mbox1.basedir, mbox1.name, False)
     assert mbox3.last_login is None
 
 

cmdeploy/src/cmdeploy/deployers.py

@@ -141,6 +141,10 @@ def _configure_remote_venv_with_chatmaild(config) -> None:
 
 
 class UnboundDeployer(Deployer):
+    def __init__(self, config):
+        self.config = config
+        self.need_restart = False
+
     def install(self):
         # Run local DNS resolver `unbound`.
         # `resolvconf` takes care of setting up /etc/resolv.conf
@@ -177,6 +181,27 @@ class UnboundDeployer(Deployer):
                 "unbound-anchor -a /var/lib/unbound/root.key || true",
             ],
         )
+        if self.config.disable_ipv6:
+            files.directory(
+                path="/etc/unbound/unbound.conf.d",
+                present=True,
+                user="root",
+                group="root",
+                mode="755",
+            )
+            conf = files.put(
+                src=get_resource("unbound/unbound.conf.j2"),
+                dest="/etc/unbound/unbound.conf.d/chatmail.conf",
+                user="root",
+                group="root",
+                mode="644",
+            )
+        else:
+            conf = files.file(
+                path="/etc/unbound/unbound.conf.d/chatmail.conf",
+                present=False,
+            )
+        self.need_restart |= conf.changed
 
     def activate(self):
         server.shell(
@@ -191,6 +216,7 @@ class UnboundDeployer(Deployer):
             service="unbound.service",
             running=True,
             enabled=True,
+            restarted=self.need_restart,
         )
 
 
@@ -527,7 +553,8 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
         files.line(
             name="Add 9.9.9.9 to resolv.conf",
             path="/etc/resolv.conf",
-            line="nameserver 9.9.9.9",
+            # Guard against resolv.conf missing a trailing newline (SolusVM bug).
+            line="\nnameserver 9.9.9.9",
         )
 
     port_services = [
@@ -565,7 +592,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
         LegacyRemoveDeployer(),
         FiltermailDeployer(),
         JournaldDeployer(),
-        UnboundDeployer(),
+        UnboundDeployer(config),
         TurnDeployer(mail_domain),
         IrohDeployer(config.enable_iroh_relay),
         AcmetoolDeployer(config.acme_email, tls_domains),

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -1,7 +1,7 @@
 ## Dovecot configuration file
 
 {% if disable_ipv6 %}
-listen = *
+listen = 0.0.0.0
 {% endif %}
 
 protocols = imap lmtp
@@ -68,13 +68,11 @@ userdb {
 ##
 
 # Mailboxes are stored in the "mail" directory of the vmail user home.
+{% if config.tmpfs_index %}
+mail_location = maildir:{{ config.mailboxes_dir }}/%u:INDEX=/dev/shm/%u
+{% else %}
 mail_location = maildir:{{ config.mailboxes_dir }}/%u
-
+{% endif %}
-# index/cache files are not very useful for chatmail relay operations 
-# but it's not clear how to disable them completely. 
-# According to https://doc.dovecot.org/2.3/settings/advanced/#core_setting-mail_cache_max_size
-# if the cache file becomes larger than the specified size, it is truncated by dovecot 
-mail_cache_max_size = 500K
 
 namespace inbox {
   inbox = yes

cmdeploy/src/cmdeploy/filtermail/deployer.py

@@ -14,10 +14,10 @@ class FiltermailDeployer(Deployer):
 
     def install(self):
         arch = host.get_fact(facts.server.Arch)
-        url = f"https://github.com/chatmail/filtermail/releases/download/v0.1.2/filtermail-{arch}"
+        url = f"https://github.com/chatmail/filtermail/releases/download/v0.2.0/filtermail-{arch}-musl"
         sha256sum = {
-            "x86_64": "de7de6e011ffc06881d3a05fc9788e327ba2389219e77280ace38b429e11a5ce",
+            "x86_64": "1e5bbb646582cb16740c6dfbbca39edba492b78cc96ec9fa2528c612bb504edd",
-            "aarch64": "a78fcdfb81eb3d9c8a8b6f84f6c0a75519b8be01aa25bd4617d72aae543992b4",
+            "aarch64": "3564fba8605f8f9adfeefff3f4580533205da043f47c5968d0d10db17e50f44e",
         }[arch]
         self.need_restart |= files.download(
             name="Download filtermail",

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -64,7 +64,11 @@ alias_database = hash:/etc/aliases
 mydestination =
 
 relayhost = 
+{% if disable_ipv6 %}
+mynetworks = 127.0.0.0/8
+{% else %}
 mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128
+{% endif %}
 mailbox_size_limit = 0
 message_size_limit = {{config.max_message_size}}
 recipient_delimiter = +

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -190,22 +190,18 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
         "encrypted.eml", from_addr=user1.addr, to_addr=user2.addr
     ).as_string()
 
-    timestamps = []
+    start = time.time()
-    i = 0
+    for i in range(chatmail_config.max_user_send_per_minute * 3):
-    while len(timestamps) <= chatmail_config.max_user_send_per_minute * 1.7:
+        print("Sending mail", str(i + 1), "at", time.time() - start, "s.")
-        print("Sending mail", str(i))
-        i += 1
         try:
             user1.smtp.sendmail(user1.addr, [user2.addr], mail)
-            timestamps.append(time.time())
         except smtplib.SMTPException as e:
-            if len(timestamps) < chatmail_config.max_user_send_per_minute:
+            if i < chatmail_config.max_user_send_burst_size:
                 pytest.fail(f"rate limit was exceeded too early with msg {i}")
             outcome = e.recipients[user2.addr]
             assert outcome[0] == 450
             assert b"4.7.1: Too much mail from" in outcome[1]
             return
-        timestamps[:] = [ts for ts in timestamps if ts >= (time.time() - 60)]
     pytest.fail("Rate limit was not exceeded")
 
 

cmdeploy/src/cmdeploy/unbound/unbound.conf.j2

@@ -0,0 +1,4 @@
+# Managed by cmdeploy: disable IPv6 in unbound.
+server:
+  interface: 127.0.0.1
+  do-ip6: no

doc/source/overview.rst

@@ -42,6 +42,11 @@ The deployed system components of a chatmail relay are:
 -  Dovecot_ is the Mail Delivery Agent (MDA) and
    stores messages for users until they download them
 
+-  `filtermail <https://github.com/chatmail/filtermail>`_
+   prevents unencrypted email from leaving or entering the chatmail
+   service and is integrated into Postfix’s outbound and inbound mail
+   pipelines.
+
 -  Nginx_ shows the web page with privacy policy and additional information
 
 -  `acmetool <https://hlandau.github.io/acmetool/>`_ manages TLS
@@ -85,11 +90,6 @@ short overview of ``chatmaild`` services:
    <https://doc.dovecot.org/2.3/configuration_manual/authentication/dict/#complete-example-for-authenticating-via-a-unix-socket>`_
    to authenticate logins.
 
--  `filtermail <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/filtermail.py>`_
-   prevents unencrypted email from leaving or entering the chatmail
-   service and is integrated into Postfix’s outbound and inbound mail
-   pipelines.
-
 -  `chatmail-metadata <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/metadata.py>`_
    is contacted by a `Dovecot lua
    script <https://github.com/chatmail/relay/blob/main/cmdeploy/src/cmdeploy/dovecot/push_notification.lua>`_