feat: randomize SMTP + IMAP ports

This reverts commit 921080125f.

.github/workflows/ci-no-dns.yaml

@@ -20,9 +20,9 @@ concurrency:
 jobs:
   no-dns:
     name: LXC deploy and test
-    uses: chatmail/cmlxc/.github/workflows/lxc-test.yml@v0.14.6
+    uses: chatmail/cmlxc/.github/workflows/lxc-test.yml@main
     with:
-      cmlxc_version: v0.14.6
+      cmlxc_version: main
       cmlxc_commands: |
         cmlxc init 
         # single cmdeploy relay test

.github/workflows/ci.yaml

@@ -29,7 +29,7 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha }}
           persist-credentials: false
       - name: download filtermail
-        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.6.6/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
+        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.7.0/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
       - name: run chatmaild tests 
         working-directory: chatmaild
         run: pipx run tox
@@ -57,9 +57,9 @@ jobs:
 
   lxc-test:
     name: LXC deploy and test
-    uses: chatmail/cmlxc/.github/workflows/lxc-test.yml@v0.14.6
+    uses: chatmail/cmlxc/.github/workflows/lxc-test.yml@main
     with:
-      cmlxc_version: v0.14.6
+      cmlxc_version: main
       cmlxc_commands: |
         cmlxc init 
         # single cmdeploy relay test

chatmaild/pyproject.toml

@@ -10,7 +10,6 @@ dependencies = [
   "filelock",
   "requests",
   "crypt-r >= 3.13.1 ; python_version >= '3.11'",
-  "domain-validator",
 ]
 
 [tool.setuptools]

chatmaild/src/chatmaild/config.py

@@ -1,8 +1,8 @@
 import ipaddress
 from pathlib import Path
+from random import randint
 
 import iniconfig
-from domain_validator import DomainValidator
 
 from chatmaild.user import User
 
@@ -25,7 +25,6 @@ class Config:
             self.mail_domain = f"[{raw_domain}]"
             self.postfix_myhostname = ipaddress.IPv4Address(raw_domain).reverse_pointer
         else:
-            DomainValidator().validate_domain_re(raw_domain)
             self.ipv4_relay = None
             self.mail_domain = raw_domain
             self.postfix_myhostname = raw_domain
@@ -43,6 +42,11 @@ class Config:
         self.username_max_length = int(params.pop("username_max_length", 9))
         self.password_min_length = int(params.pop("password_min_length", 9))
         self.www_folder = params.pop("www_folder", "")
+
+        self.imap_port = int(params.pop("imap_port", 143))
+        self.imaps_port = int(params.pop("imaps_port", 993))
+        self.smtp_port = int(params.pop("smtp_port", 587))
+        self.smtps_port = int(params.pop("smtps_port", 465))
         self.filtermail_smtp_port = int(params.pop("filtermail_smtp_port", "10080"))
         self.filtermail_smtp_port_incoming = int(
             params.pop("filtermail_smtp_port_incoming", "10081")
@@ -140,8 +144,15 @@ def parse_size_mb(limit):
 
 def write_initial_config(inipath, mail_domain, overrides):
     """Write out default config file, using the specified config value overrides."""
-    content = get_default_config_content(mail_domain, **overrides)
-    inipath.write_text(content)
+    content = get_default_config_content(mail_domain, **overrides).splitlines()
+    used_ports = [25, 53, 80, 143, 402, 443, 465, 587, 993, 3340, 3903, 3904, 8443, 10080, 10081, 10082, 10083, 10025, 10026]
+    for config_key in ["smtp_port", "imap_port", "smtps_port", "imaps_port"]:
+        value = randint(1, 65536)
+        while value in used_ports:
+            value = randint(65535)
+        used_ports.append(value)
+        content.append(f"{config_key} = {value}")
+    inipath.write_text("\n".join(content))
 
 
 def get_default_config_content(mail_domain, **overrides):

chatmaild/src/chatmaild/expire.py

@@ -168,6 +168,16 @@ class Expiry:
         if mbox.last_login and mbox.last_login < cutoff_without_login:
             self.remove_mailbox(mbox.basedir)
             return
+        elif mbox.last_login is None:
+            try:
+                if not self.dry:
+                    os.rmdir(mbox.basedir)
+                    self.del_mboxes += 1
+            except OSError:
+                print_info(
+                    f"Skipped deleting {mbox.basedir}, doesn't have last_login but isn't empty"
+                )
+            return
 
         mboxname = os.path.basename(mbox.basedir)
         if self.verbose:

chatmaild/src/chatmaild/tests/test_expire.py

@@ -1,6 +1,7 @@
 import itertools
 import os
 import random
+import shutil
 import time
 from datetime import datetime
 from fnmatch import fnmatch
@@ -9,6 +10,7 @@ from pathlib import Path
 import pytest
 
 from chatmaild.expire import (
+    Expiry,
     FileEntry,
     MailboxStat,
     expire_to_target,
@@ -104,6 +106,32 @@ def test_stats_mailbox(mbox1):
     assert mbox3.last_login is None
 
 
+def test_mbox_without_password(mbox1, example_config, capsys):
+    password = Path(mbox1.basedir).joinpath("password")
+    os.remove(password)
+    mbox_rescan = MailboxStat(mbox1.basedir)
+    assert mbox_rescan.last_login is None
+
+    exp = Expiry(
+        example_config, dry=False, now=datetime.now().timestamp(), verbose=False
+    )
+    exp.process_mailbox_stat(mbox_rescan)
+    out, err = capsys.readouterr()
+    assert "doesn't have last_login but isn't empty" in err
+    assert os.path.isdir(mbox_rescan.basedir)
+
+    for entry in os.scandir(mbox_rescan.basedir):
+        if os.path.isdir(entry):
+            shutil.rmtree(entry)
+        else:
+            os.remove(entry)
+
+    exp.process_mailbox_stat(mbox_rescan)
+    out, err = capsys.readouterr()
+    assert "doesn't have last_login but isn't empty" not in err
+    assert not os.path.isdir(mbox_rescan.basedir)
+
+
 def test_report_no_mailboxes(example_config):
     args = (str(example_config._inipath),)
     report_main(args)

cmdeploy/src/cmdeploy/acmetool/response-file.yaml.j2

@@ -1,2 +1,2 @@
 "acme-enter-email": "{{ email }}"
-"acme-agreement:https://letsencrypt.org/documents/LE-SA-v1.6-August-18-2025.pdf": true
+"acme-agreement:https://letsencrypt.org/documents/LE-SA-v1.7-June-04-2026.pdf": true

cmdeploy/src/cmdeploy/deployers.py

@@ -171,16 +171,14 @@ class UnboundDeployer(Deployer):
                 "unbound-anchor -a /var/lib/unbound/root.key || true",
             ],
         )
-        if self.config.disable_ipv6:
-            self.ensure_directory(
-                path="/etc/unbound/unbound.conf.d",
-            )
-            self.put_template(
-                "unbound/unbound.conf.j2",
-                "/etc/unbound/unbound.conf.d/chatmail.conf",
-            )
-        else:
-            self.remove_file("/etc/unbound/unbound.conf.d/chatmail.conf")
+        self.ensure_directory(
+            path="/etc/unbound/unbound.conf.d",
+        )
+        self.put_template(
+            "unbound/unbound.conf.j2",
+            "/etc/unbound/unbound.conf.d/chatmail.conf",
+            disable_ipv6=self.config.disable_ipv6,
+        )
 
     def activate(self):
         server.shell(
@@ -497,15 +495,15 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
         if config.tls_cert_mode == "acme":
             port_services.append(("acmetool", 402))
         port_services += [
-            (["imap-login", "dovecot"], 143),
+            (["imap-login", "dovecot"], config.imap_port),
             # acmetool previously listened on port 80,
             # so don't complain during upgrade that moved it to port 402
             # and gave the port to nginx.
             (["acmetool", "nginx"], 80),
             ("nginx", 443),
-            (["master", "smtpd"], 465),
-            (["master", "smtpd"], 587),
-            (["imap-login", "dovecot"], 993),
+            (["master", "smtpd"], config.smtp_port),
+            (["master", "smtpd"], config.smtps_port),
+            (["imap-login", "dovecot"], config.imaps_port),
             ("iroh-relay", 3340),
             ("mtail", 3903),
             ("stats", 3904),
@@ -514,6 +512,8 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
             (["master", "smtpd"], config.postfix_reinject_port_incoming),
             ("filtermail", config.filtermail_smtp_port),
             ("filtermail", config.filtermail_smtp_port_incoming),
+            ("filtermail", config.filtermail_http_port_incoming),
+            ("filtermail", config.filtermail_lmtp_port_transport),
         ]
         for service, port in port_services:
             print(f"Checking if port {port} is available for {service}...")

cmdeploy/src/cmdeploy/filtermail/deployer.py

@@ -20,10 +20,10 @@ class FiltermailDeployer(Deployer):
             return
 
         arch = host.get_fact(facts.server.Arch)
-        url = f"https://github.com/chatmail/filtermail/releases/download/v0.6.6/filtermail-{arch}"
+        url = f"https://github.com/chatmail/filtermail/releases/download/v0.7.0/filtermail-{arch}"
         sha256sum = {
-            "x86_64": "05c7e7ac244606c2eeb275f2d282ffdbc2403e0169f1cdd3110ffcebdb994a92",
-            "aarch64": "8cf8bbda4d907beca547b365cc7e6753532a74b1712492d0d2f3d2d8a553fb3d",
+            "x86_64": "451f295a85b3b12dbb0f89e18ec319f742ee46dec218f20f7923bfb017a248bd",
+            "aarch64": "6833061b2a2028264fdeb32f0a6123e1ff73de57dace125364016300b748452e",
         }[arch]
         self.download_executable(url, self.bin_path, sha256sum)
 

cmdeploy/src/cmdeploy/nginx/autoconfig.xml.j2

@@ -7,14 +7,14 @@
     <displayShortName>{{ config.mail_domain }}</displayShortName>
     <incomingServer type="imap">
       <hostname>{{ config.mail_domain }}</hostname>
-      <port>993</port>
+      <port>{{ config.imaps_port }}</port>
       <socketType>SSL</socketType>
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
     </incomingServer>
     <incomingServer type="imap">
       <hostname>{{ config.mail_domain }}</hostname>
-      <port>143</port>
+      <port>{{ config.imap_port }}</port>
       <socketType>STARTTLS</socketType>
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
@@ -28,14 +28,14 @@
     </incomingServer>
     <outgoingServer type="smtp">
       <hostname>{{ config.mail_domain }}</hostname>
-      <port>465</port>
+      <port>{{ config.smtps_port }}</port>
       <socketType>SSL</socketType>
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
     </outgoingServer>
     <outgoingServer type="smtp">
       <hostname>{{ config.mail_domain }}</hostname>
-      <port>587</port>
+      <port>{{ config.smtp_port }}</port>
       <socketType>STARTTLS</socketType>
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -31,6 +31,26 @@ stream {
             ~\bimap\b 127.0.0.1:993;
         }
 
+        server {
+                listen {{ config.smtp_port }};
+                proxy_pass 127.0.0.1:587;
+        }
+
+        server {
+                listen {{ config.imap_port }};
+                proxy_pass 127.0.0.1:143;
+        }
+
+        server {
+                listen {{ config.smtps_port }};
+                proxy_pass 127.0.0.1:465;
+        }
+
+        server {
+                listen {{ config.imaps_port }};
+                proxy_pass 127.0.0.1:993;
+        }
+
         server {
                 listen 443;
                 {% if not disable_ipv6 %}

cmdeploy/src/cmdeploy/tests/plugin.py

@@ -10,13 +10,11 @@ from pathlib import Path
 
 import pytest
 from chatmaild.config import is_valid_ipv4, read_config
-from domain_validator import DomainValidator
 
 
 def format_mail_domain(raw_domain: str) -> str:
     if is_valid_ipv4(raw_domain):
         return f"[{raw_domain}]"
-    DomainValidator().validate_domain_re(raw_domain)
     return raw_domain
 
 
@@ -349,9 +347,9 @@ class ChatmailACFactory:
                 qr = (
                     f"dclogin:{addr}"
                     f"?p={password}&v=1"
-                    f"&ih={domain}&ip=993"
-                    f"&sh={domain}&sp=465"
-                    f"&ic=3&ss=default"
+                    f"&ih={domain}&ip=993&is=ssl"
+                    f"&sh={domain}&sp=465&ss=ssl"
+                    f"&ic=3"
                 )
                 future = account.add_transport_from_qr.future(qr)
             else:
@@ -362,7 +360,7 @@ class ChatmailACFactory:
 
             # ensure messages stay in INBOX so that they can be
             # concurrently fetched via extra IMAP connections during tests
-            account.set_config("delete_server_after", "10")
+            account.set_config("bcc_self", "1")
             accounts.append(account)
 
         for future in futures:

cmdeploy/src/cmdeploy/unbound/unbound.conf.j2

@@ -1,4 +1,7 @@
-# Managed by cmdeploy: disable IPv6 in unbound.
+# Managed by cmdeploy
 server:
+{% if disable_ipv6 %}
   interface: 127.0.0.1
   do-ip6: no
+{% endif %}
+  cache-max-negative-ttl: 0

doc/source/overview.rst

@@ -156,6 +156,7 @@ Chatmail relay dependency diagram
         postfix --- |10083|filtermail-transport;
         filtermail-outgoing --- |10025 reinject|postfix;
         filtermail-incoming --- |10026 reinject|postfix;
+        postfix --- |milter opendkim.sock|OpenDKIM
         dovecot --- |doveauth.socket|doveauth;
         dovecot --- |message delivery|maildir["maildir
         /home/vmail/.../user"];
@@ -179,26 +180,66 @@ Chatmail relay dependency diagram
         style nginx-right fill:#f66;
         style postfix fill:#f66;
         style dovecot fill:#f66;
+        style OpenDKIM fill:#f66;
         style notification-proxy fill:#f66;
 
-Message between users on the same relay
----------------------------------------
+Accepting and delivering mail
+-----------------------------
 
 .. mermaid::
-    :caption: This diagram shows the path a non-federated message takes.
+    :caption: This diagram shows all the paths a message can take.
 
-    graph LR;
-        sender --> |465|smtps/smtpd;
-        sender --> |587|submission/smtpd;
-        smtps/smtpd --> |10080|filtermail;
-        submission/smtpd --> |10080|filtermail;
-        filtermail --> |10025|smtpd_reinject;
-        smtpd_reinject --> cleanup;
-        cleanup --> qmgr;
-        qmgr --> smtpd_accepts_message;
-        qmgr --> |lmtp|dovecot;
-        dovecot --> recipient;
-        dovecot --> sender's_other_devices;
+    flowchart LR
+        subgraph chatmail relay
+            subgraph postfix
+                qmgr .-> lmtp-filtermail["lmtp/lmtp-filtermail (default_transport)"]
+                qmgr .-> lmtp["lmtp (local_transport)"]
+                lmtp --> cleanup["cleanup (lmtp_header_cleanup)"]
+                bounce
+                smtpd-submission["smtpd/submission"]
+                smtpd-smtps["smtpd/smtps"]
+                smtpd-reinject-outgoing["smtpd/reinject-outgoing"] --> authclean["cleanup/authclean (submission_header_cleanup)"]
+                authclean --> qmgr
+                smtpd-smtp["smtpd/smtp"]
+                smtpd-reinject-incoming["smtpd/reinject-incoming"] --> qmgr
+            end
+            lmtp-filtermail --LMTP inet:10083--> filtermail-transport
+            cleanup --LMTP unix:private/dovecot-lmtp --> dovecot
+            dovecot --> maildir
+            smtpd-submission --SMTP inet:10080--> filtermail-outgoing
+            smtpd-smtps --SMTP inet:10080--> filtermail-outgoing
+            filtermail-outgoing --SMTP inet:10025--> smtpd-reinject-outgoing
+            open-dkim["OpenDKIM (signing only)"] <--milter unix:opendkim/opendkim.sock--> smtpd-reinject-outgoing
+            bounce <--milter unix:opendkim/opendkim.sock--> open-dkim
+            bounce --> qmgr
+            nginx
+            smtpd-smtp -.SMTP inet:10081.-> filtermail-incoming
+            nginx -.HTTP inet:10082.-> filtermail-incoming
+            filtermail-incoming --SMTP inet:10026--> smtpd-reinject-incoming
+        end
+        filtermail-transport -.SMTP inet:25.-> mta1[Remote relay]
+        filtermail-transport -.HTTPS /mxdeliv.-> mta1
+        client[Client] -.SMTP inet:587.-> smtpd-submission
+        client -.SMTP inet:465.-> smtpd-smtps
+        client -.SMTP inet:443.-> nginx
+        nginx -.SMTP inet:465.-> smtpd-smtps
+        mta2[Remote relay] -.SMTP inet:25.-> smtpd-smtp
+        mta2 -.HTTPS /mxdeliv.-> nginx
+        style postfix fill:#363
+        style qmgr fill:#252
+        style authclean fill:#252
+        style cleanup fill:#252
+        style lmtp-filtermail fill:#252
+        style lmtp fill:#252
+        style bounce fill:#252
+        style smtpd-submission fill:#252
+        style smtpd-smtps fill:#252
+        style smtpd-reinject-outgoing fill:#252
+        style smtpd-reinject-incoming fill:#252
+        style smtpd-smtp fill:#252
+        style filtermail-outgoing fill:#225
+        style filtermail-incoming fill:#225
+        style filtermail-transport fill:#225
 
 Operational details of a chatmail relay
 ----------------------------------------