feat: add tool to analyze deferred queue

It prints all destinations with the number of recipients
and all the reasons. Operator can then try
to fix the problems for destinations,
e.g. by manually adding reverse proxy
addresses to /etc/hosts for failing domains
or routing IP addresses to another interface.

.github/workflows/ci-no-dns.yaml

@@ -1,40 +0,0 @@
-name: No-DNS
-
-on:
-  # Triggers when a PR is merged into main or a direct push occurs
-  push:
-    branches: [ "main" ]
-    
-  # Triggers for any PR (and its subsequent commits) targeting the main branch
-  pull_request:
-    branches: [ "main" ]
-
-permissions: {}
-
-# Newest push wins: Prevents multiple runs from clashing and wasting runner efforts
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-
-jobs:
-  no-dns:
-    name: LXC deploy and test
-    uses: chatmail/cmlxc/.github/workflows/lxc-test.yml@v0.14.6
-    with:
-      cmlxc_version: v0.14.6
-      cmlxc_commands: |
-        cmlxc init 
-        # single cmdeploy relay test
-        cmlxc -v deploy-cmdeploy --source ./repo --type ipv4 cm0
-        cmlxc -v test-cmdeploy cm0
-
-        # cross cmdeploy relay test (two ipv4 relays)
-        cmlxc -v deploy-cmdeploy --source ./repo --ipv4-only --type ipv4 cm1
-        cmlxc -v test-cmdeploy cm0 cm1
-
-        # cross cmdeploy/madmail relay tests
-        cmlxc -v deploy-madmail mad0
-        cmlxc -v test-cmdeploy cm0 mad0
-        cmlxc -v test-mini mad0 cm0
-        cmlxc -v test-mini cm0 mad0

.github/workflows/ci.yaml

@@ -1,4 +1,4 @@
-name: CI
+name: Run unit-tests and container-based deploy+test verification
 
 on:
   # Triggers when a PR is merged into main or a direct push occurs
@@ -29,7 +29,7 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha }}
           persist-credentials: false
       - name: download filtermail
-        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.6.6/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
+        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.6.4/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
       - name: run chatmaild tests 
         working-directory: chatmaild
         run: pipx run tox
@@ -57,9 +57,9 @@ jobs:
 
   lxc-test:
     name: LXC deploy and test
-    uses: chatmail/cmlxc/.github/workflows/lxc-test.yml@v0.14.6
+    uses: chatmail/cmlxc/.github/workflows/lxc-test.yml@v0.13.5
     with:
-      cmlxc_version: v0.14.6
+      cmlxc_version: v0.13.5
       cmlxc_commands: |
         cmlxc init 
         # single cmdeploy relay test
@@ -76,4 +76,3 @@ jobs:
         cmlxc -v test-cmdeploy cm0 mad0
         cmlxc -v test-mini cm0 mad0
         cmlxc -v test-mini mad0 cm0
-

chatmaild/pyproject.toml

@@ -10,7 +10,6 @@ dependencies = [
   "filelock",
   "requests",
   "crypt-r >= 3.13.1 ; python_version >= '3.11'",
-  "domain-validator",
 ]
 
 [tool.setuptools]
@@ -25,6 +24,7 @@ chatmail-metadata = "chatmaild.metadata:main"
 chatmail-expire = "chatmaild.expire:daily_expire_main"
 chatmail-quota-expire = "chatmaild.expire:quota_expire_main"
 chatmail-fsreport = "chatmaild.fsreport:main"
+chatmail-deferred = "chatmaild.deferred:main"
 lastlogin = "chatmaild.lastlogin:main"
 turnserver = "chatmaild.turnserver:main"
 

chatmaild/src/chatmaild/config.py

@@ -1,8 +1,6 @@
-import ipaddress
 from pathlib import Path
 
 import iniconfig
-from domain_validator import DomainValidator
 
 from chatmaild.user import User
 
@@ -21,19 +19,7 @@ def read_config(inipath):
 class Config:
     def __init__(self, inipath, params):
         self._inipath = inipath
-        raw_domain = params["mail_domain"]
+        self.mail_domain = params["mail_domain"]
-        self.mail_domain_bare = raw_domain
-
-        if is_valid_ipv4(raw_domain):
-            self.ipv4_relay = raw_domain
-            self.mail_domain = f"[{raw_domain}]"
-            self.postfix_myhostname = ipaddress.IPv4Address(raw_domain).reverse_pointer
-        else:
-            DomainValidator().validate_domain_re(raw_domain)
-            self.ipv4_relay = None
-            self.mail_domain = raw_domain
-            self.postfix_myhostname = raw_domain
-
         self.max_user_send_per_minute = int(params.get("max_user_send_per_minute", 60))
         self.max_user_send_burst_size = int(params.get("max_user_send_burst_size", 10))
         self.max_mailbox_size = params["max_mailbox_size"]
@@ -67,7 +53,7 @@ class Config:
         self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true"
         self.imap_compress = params.get("imap_compress", "false").lower() == "true"
         if "iroh_relay" not in params:
-            self.iroh_relay = "https://" + raw_domain
+            self.iroh_relay = "https://" + params["mail_domain"]
             self.enable_iroh_relay = True
         else:
             self.iroh_relay = params["iroh_relay"].strip()
@@ -93,17 +79,17 @@ class Config:
                 )
             self.tls_cert_mode = "external"
             self.tls_cert_path, self.tls_key_path = parts
-        elif raw_domain.startswith("_") or self.ipv4_relay:
+        elif self.mail_domain.startswith("_"):
             self.tls_cert_mode = "self"
             self.tls_cert_path = "/etc/ssl/certs/mailserver.pem"
             self.tls_key_path = "/etc/ssl/private/mailserver.key"
         else:
             self.tls_cert_mode = "acme"
-            self.tls_cert_path = f"/var/lib/acme/live/{raw_domain}/fullchain"
+            self.tls_cert_path = f"/var/lib/acme/live/{self.mail_domain}/fullchain"
-            self.tls_key_path = f"/var/lib/acme/live/{raw_domain}/privkey"
+            self.tls_key_path = f"/var/lib/acme/live/{self.mail_domain}/privkey"
 
         # deprecated option
-        mbdir = params.get("mailboxes_dir", f"/home/vmail/mail/{raw_domain}")
+        mbdir = params.get("mailboxes_dir", f"/home/vmail/mail/{self.mail_domain}")
         self.mailboxes_dir = Path(mbdir.strip())
 
         # old unused option (except for first migration from sqlite to maildir store)
@@ -189,12 +175,3 @@ def get_default_config_content(mail_domain, **overrides):
                 lines.append(line)
         content = "\n".join(lines)
     return content
-
-
-def is_valid_ipv4(address: str) -> bool:
-    """Check if a mail_domain is an IPv4 address."""
-    try:
-        ipaddress.IPv4Address(address)
-        return True
-    except ValueError:
-        return False

chatmaild/src/chatmaild/deferred.py

@@ -0,0 +1,37 @@
+"""
+Analyze deferred mails and print most common failing destinations.
+
+Example:
+
+    python -m chatmaild.deferred
+"""
+
+import json
+import subprocess
+from collections import Counter, defaultdict
+
+
+def main():
+    p = subprocess.Popen(["postqueue", "-j"], text=True, stdout=subprocess.PIPE)
+    domain_reasons = defaultdict(Counter)
+    domain_total = Counter()
+
+    for line in p.stdout:
+        item = json.loads(line)
+        if item["queue_name"] != "deferred":
+            continue
+
+        for recipient in item["recipients"]:
+            _, domain = recipient["address"].rsplit("@", 1)
+            reason = recipient["delay_reason"].removeprefix("host 127.0.0.1[127.0.0.1] said: ")
+            domain_total[domain] += 1
+            domain_reasons[domain][reason] += 1
+
+    for domain, total in reversed(domain_total.most_common()):
+        print(f"{domain} ({total} recipients)")
+        for reason, count in domain_reasons[domain].most_common():
+            print(f"  {count}: {reason}")
+
+
+if __name__ == "__main__":
+    main()

chatmaild/src/chatmaild/metadata.py

@@ -88,27 +88,29 @@ class MetadataDictProxy(DictProxy):
 
     def handle_lookup(self, parts):
         # Lpriv/43f5f508a7ea0366dff30200c15250e3/devicetoken\tlkj123poi@c2.testrun.org
-        match parts[0].split("/", 2):
+        keyparts = parts[0].split("/", 2)
-            case ["priv", _, keyname] if keyname == self.metadata.DEVICETOKEN_KEY:
+        if keyparts[0] == "priv":
-                addr = parts[1]
+            keyname = keyparts[2]
+            addr = parts[1]
+            if keyname == self.metadata.DEVICETOKEN_KEY:
                 res = " ".join(self.metadata.get_tokens_for_addr(addr))
                 return f"O{res}\n"
-            case ["shared", _, keyname]:
+        elif keyparts[0] == "shared":
-                prefix = "vendor/vendor.dovecot/pvt/server/vendor/deltachat/"
+            keyname = keyparts[2]
-                if keyname.startswith(prefix):
+            if (
-                    match keyname[len(prefix) :]:
+                keyname == "vendor/vendor.dovecot/pvt/server/vendor/deltachat/irohrelay"
-                        case "irohrelay" if self.iroh_relay:
+                and self.iroh_relay
-                            return f"O{self.iroh_relay}\n"
+            ):
-                        case "turn":
+                # Handle `GETMETADATA "" /shared/vendor/deltachat/irohrelay`
-                            try:
+                return f"O{self.iroh_relay}\n"
-                                res = turn_credentials()
+            elif keyname == "vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn":
-                            except Exception:
+                try:
-                                logging.exception("failed to get TURN credentials")
+                    res = turn_credentials()
-                                return "N\n"
+                except Exception:
-                            return f"O{self.turn_hostname}:3478:{res}\n"
+                    logging.exception("failed to get TURN credentials")
-                        case "maxsmtprecipients":
+                    return "N\n"
-                            # postfix default  (see "postconf smtpd_recipient_limit")
+                port = 3478
-                            return "O1000\n"
+                return f"O{self.turn_hostname}:{port}:{res}\n"
 
         logging.warning(f"lookup ignored: {parts!r}")
         return "N\n"
@@ -118,13 +120,12 @@ class MetadataDictProxy(DictProxy):
         # https://github.com/dovecot/core/blob/main/src/lib-storage/mailbox-attribute.h
         keyname = parts[1].split("/")
         value = parts[2] if len(parts) > 2 else ""
-        match keyname:
+        if keyname[0] == "priv" and keyname[2] == self.metadata.DEVICETOKEN_KEY:
-            case ["priv", _, key] if key == self.metadata.DEVICETOKEN_KEY:
+            self.metadata.add_token_to_addr(addr, value)
-                self.metadata.add_token_to_addr(addr, value)
+            return True
-                return True
+        elif keyname[0] == "priv" and keyname[2] == "messagenew":
-            case ["priv", _, "messagenew"]:
+            self.notifier.new_message_for_addr(addr, self.metadata)
-                self.notifier.new_message_for_addr(addr, self.metadata)
+            return True
-                return True
 
         return False
 

chatmaild/src/chatmaild/newemail.py

@@ -2,6 +2,7 @@
 
 """CGI script for creating new accounts."""
 
+import ipaddress
 import json
 import secrets
 import string
@@ -14,6 +15,16 @@ ALPHANUMERIC = string.ascii_lowercase + string.digits
 ALPHANUMERIC_PUNCT = string.ascii_letters + string.digits + string.punctuation
 
 
+def wrap_ip(host):
+    if host.startswith("[") and host.endswith("]"):
+        return host
+    try:
+        ipaddress.ip_address(host)
+        return f"[{host}]"
+    except ValueError:
+        return host
+
+
 def create_newemail_dict(config: Config):
     user = "".join(
         secrets.choice(ALPHANUMERIC) for _ in range(config.username_max_length)
@@ -22,22 +33,16 @@ def create_newemail_dict(config: Config):
         secrets.choice(ALPHANUMERIC_PUNCT)
         for _ in range(config.password_min_length + 3)
     )
-    return dict(email=f"{user}@{config.mail_domain}", password=f"{password}")
+    return dict(email=f"{user}@{wrap_ip(config.mail_domain)}", password=f"{password}")
 
 
-def create_dclogin_url(config, email, password):
+def create_dclogin_url(email, password):
     """Build a dclogin: URL with credentials and self-signed cert acceptance.
 
     Uses ic=3 (AcceptInvalidCertificates) so chatmail clients
     can connect to servers with self-signed TLS certificates.
     """
-    if config.ipv4_relay:
+    return f"dclogin:{quote(email, safe='@')}?p={quote(password, safe='')}&v=1&ic=3"
-        imap_host = "&ih=" + config.ipv4_relay
-        smtp_host = "&sh=" + config.ipv4_relay
-    else:
-        imap_host = ""
-        smtp_host = ""
-    return f"dclogin:{quote(email, safe='@[]')}?p={quote(password, safe='')}&v=1{imap_host}{smtp_host}&ic=3"
 
 
 def print_new_account():
@@ -46,9 +51,7 @@ def print_new_account():
 
     result = dict(email=creds["email"], password=creds["password"])
     if config.tls_cert_mode == "self":
-        result["dclogin_url"] = create_dclogin_url(
+        result["dclogin_url"] = create_dclogin_url(creds["email"], creds["password"])
-            config, creds["email"], creds["password"]
-        )
 
     print("Content-Type: application/json")
     print("")

chatmaild/src/chatmaild/tests/plugin.py

@@ -31,11 +31,6 @@ def example_config(make_config):
     return make_config("chat.example.org")
 
 
-@pytest.fixture
-def ipv4_config(make_config):
-    return make_config("1.3.3.7")
-
-
 @pytest.fixture
 def maildomain(example_config):
     return example_config.mail_domain

chatmaild/src/chatmaild/tests/test_config.py

@@ -1,10 +1,6 @@
 import pytest
 
-from chatmaild.config import (
+from chatmaild.config import parse_size_mb, read_config
-    is_valid_ipv4,
-    parse_size_mb,
-    read_config,
-)
 
 
 def test_read_config_basic(example_config):
@@ -17,12 +13,6 @@ def test_read_config_basic(example_config):
     example_config = read_config(inipath)
     assert example_config.max_user_send_per_minute == 37
     assert example_config.mail_domain == "chat.example.org"
-    assert example_config.ipv4_relay is None
-
-
-def test_read_config_ipv4(ipv4_config):
-    assert ipv4_config.ipv4_relay == "1.3.3.7"
-    assert ipv4_config.mail_domain == "[1.3.3.7]"
 
 
 def test_read_config_basic_using_defaults(tmp_path, maildomain):
@@ -145,17 +135,3 @@ def test_max_mailbox_size_mb(make_config):
     config = make_config("chat.example.org")
     assert config.max_mailbox_size == "500M"
     assert config.max_mailbox_size_mb == 500
-
-
-@pytest.mark.parametrize(
-    ["input", "result"],
-    [
-        ("example.org", False),
-        ("1.3.3.7", True),
-        ("fe::1", False),
-        ("ad.1e.dag.adf", False),
-        ("12394142", False),
-    ],
-)
-def test_is_valid_ipv4(input, result):
-    assert result == is_valid_ipv4(input)

chatmaild/src/chatmaild/tests/test_metadata.py

@@ -360,8 +360,15 @@ def test_turn_credentials_success(notifier, metadata, monkeypatch):
 
 
 def test_iroh_relay(dictproxy):
-    key = b"Lshared/0123/vendor/vendor.dovecot/pvt/server/vendor/deltachat/irohrelay\tuser@example.org"
+    rfile = io.BytesIO(
-    rfile, wfile = io.BytesIO(b"H\n" + key), io.BytesIO()
+        b"\n".join(
+            [
+                b"H",
+                b"Lshared/0123/vendor/vendor.dovecot/pvt/server/vendor/deltachat/irohrelay\tuser@example.org",
+            ]
+        )
+    )
+    wfile = io.BytesIO()
     dictproxy.iroh_relay = "https://example.org/"
     dictproxy.loop_forever(rfile, wfile)
     assert wfile.getvalue() == b"Ohttps://example.org/\n"
@@ -376,23 +383,3 @@ def test_legacy_token_migration(metadata, testaddr):
     tokens = mdict[metadata.DEVICETOKEN_KEY]
     assert isinstance(tokens, dict)
     assert "oldtoken1" in tokens and "oldtoken2" in tokens
-
-
-@pytest.mark.parametrize(
-    "suffix, expected",
-    [
-        (b"vendor/deltachat/maxsmtprecipients", b"O1000\n"),
-        (b"wrong/prefix/key", b"N\n"),
-        (b"vendor/deltachat/unknown", b"N\n"),
-    ],
-    ids=["maxsmtprecipients", "prefix_mismatch", "unknown_name"],
-)
-def test_shared_lookup(dictproxy, suffix, expected):
-    key = (
-        b"Lshared/0123/vendor/vendor.dovecot/pvt/server/"
-        + suffix
-        + b"\tuser@example.org"
-    )
-    rfile, wfile = io.BytesIO(b"H\n" + key), io.BytesIO()
-    dictproxy.loop_forever(rfile, wfile)
-    assert wfile.getvalue() == expected

chatmaild/src/chatmaild/tests/test_newmail.py

@@ -19,35 +19,24 @@ def test_create_newemail_dict(example_config):
     assert ac1["password"] != ac2["password"]
 
 
-def test_create_newemail_dict_ip(ipv4_config):
+def test_create_newemail_dict_ip(make_config):
-    ac = create_newemail_dict(ipv4_config)
+    config = make_config("1.2.3.4")
-    assert ac["email"].endswith("@[1.3.3.7]")
+    ac = create_newemail_dict(config)
+    assert ac["email"].endswith("@[1.2.3.4]")
 
 
-def test_create_dclogin_url(example_config):
+def test_create_dclogin_url():
-    addr = "user@example.org"
+    url = create_dclogin_url("user@example.org", "p@ss w+rd")
-    password = "p@ss w+rd"
-    url = create_dclogin_url(example_config, addr, password)
     assert url.startswith("dclogin:")
     assert "v=1" in url
     assert "ic=3" in url
 
-    assert addr in url
+    assert "user@example.org" in url
     # password special chars must be encoded
     assert "p%40ss" in url
     assert "w%2Brd" in url
 
 
-def test_create_dclogin_url_ipv4(ipv4_config):
-    addr = "user@[1.3.3.7]"
-    password = "p@ss w+rd"
-    url = create_dclogin_url(ipv4_config, addr, password)
-    assert url.startswith("dclogin:")
-    assert "v=1" in url
-    assert "ic=3" in url
-    assert addr in url
-
-
 def test_print_new_account(capsys, monkeypatch, maildomain, tmpdir, example_config):
     monkeypatch.setattr(chatmaild.newemail, "CONFIG_PATH", str(example_config._inipath))
     print_new_account()

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -87,12 +87,10 @@ def run_cmd_options(parser):
 def run_cmd(args, out):
     """Deploy chatmail services on the remote server."""
 
-    ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain_bare
+    ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
     sshexec = get_sshexec(ssh_host)
     require_iroh = args.config.enable_iroh_relay
     strict_tls = args.config.tls_cert_mode == "acme"
-    if args.config.ipv4_relay:
-        args.dns_check_disabled = True
     if not args.dns_check_disabled:
         remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
         if not dns.check_initial_remote_data(remote_data, strict_tls=strict_tls, print=out.red):
@@ -121,8 +119,6 @@ def run_cmd(args, out):
         elif not args.dns_check_disabled and strict_tls and not remote_data["acme_account_url"]:
             out.red("Deploy completed but letsencrypt not configured")
             out.red("Run 'cmdeploy run' again")
-        elif args.config.ipv4_relay:
-            out.green("Deploy completed.")
         else:
             out.green("Deploy completed, call `cmdeploy dns` next.")
         return 0
@@ -144,10 +140,6 @@ def dns_cmd_options(parser):
 
 def dns_cmd(args, out):
     """Check DNS entries and optionally generate dns zone file."""
-    if args.config.ipv4_relay:
-        ipv4 = args.config.ipv4_relay
-        print(f"[WARNING] {ipv4} is not a domain, skipping DNS checks.")
-        return 0
     ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
     sshexec = get_sshexec(ssh_host, verbose=args.verbose)
     tls_cert_mode = args.config.tls_cert_mode
@@ -185,7 +177,7 @@ def status_cmd_options(parser):
 def status_cmd(args, out):
     """Display status for online chatmail instance."""
 
-    ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain_bare
+    ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
     sshexec = get_sshexec(ssh_host, verbose=args.verbose)
 
     out.green(f"chatmail domain: {args.config.mail_domain}")

cmdeploy/src/cmdeploy/deployers.py

@@ -370,7 +370,7 @@ class ChatmailVenvDeployer(Deployer):
 
     def configure(self):
         _configure_remote_venv_with_chatmaild(self, self.config)
-        configure_remote_units(self, self.config.mail_domain_bare, self.units)
+        configure_remote_units(self, self.config.mail_domain, self.units)
 
     def activate(self):
         activate_remote_units(self, self.units)
@@ -469,7 +469,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
     """
     config = read_config(config_path)
     check_config(config)
-    bare_host = config.mail_domain_bare
+    mail_domain = config.mail_domain
 
     if website_only:
         Deployment().perform_stages([WebsiteDeployer(config)])
@@ -526,7 +526,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
                     )
                     exit(1)
 
-    tls_deployer = get_tls_deployer(config, bare_host)
+    tls_deployer = get_tls_deployer(config, mail_domain)
 
     all_deployers = [
         ChatmailDeployer(config),
@@ -534,13 +534,13 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
         FiltermailDeployer(),
         JournaldDeployer(),
         UnboundDeployer(config),
-        TurnDeployer(bare_host),
+        TurnDeployer(mail_domain),
         IrohDeployer(config.enable_iroh_relay),
         tls_deployer,
         WebsiteDeployer(config),
         ChatmailVenvDeployer(config),
         MtastsDeployer(),
-        *([] if config.ipv4_relay else [OpendkimDeployer(bare_host)]),
+        OpendkimDeployer(mail_domain),
         # Dovecot should be started before Postfix
         # because it creates authentication socket
         # required by Postfix.

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -68,7 +68,7 @@ class DovecotDeployer(Deployer):
         )
 
     def configure(self):
-        configure_remote_units(self, self.config.mail_domain_bare, self.units)
+        configure_remote_units(self, self.config.mail_domain, self.units)
         _configure_dovecot(self, self.config)
 
     def activate(self):

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -7,7 +7,6 @@ listen = 0.0.0.0
 protocols = imap lmtp
 
 auth_mechanisms = plain
-auth_username_chars = abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890.-_@[]
 
 {% if debug == true %}
 auth_verbose = yes

cmdeploy/src/cmdeploy/external/deployer.py

@@ -1,3 +1,4 @@
+
 from pyinfra import host
 from pyinfra.facts.files import File
 
@@ -20,8 +21,8 @@ class ExternalTlsDeployer(Deployer):
     def configure(self):
         # Verify cert and key exist on the remote host using pyinfra facts.
         for path in (self.cert_path, self.key_path):
-            if host.get_fact(File, path=path) is None:
+                if host.get_fact(File, path=path) is None:
-                raise Exception(f"External TLS file not found on server: {path}")
+                    raise Exception(f"External TLS file not found on server: {path}")
 
         self.ensure_systemd_unit(
             "external/tls-cert-reload.path.j2",
@@ -39,3 +40,5 @@ class ExternalTlsDeployer(Deployer):
             running=True,
             enabled=True,
         )
+
+

cmdeploy/src/cmdeploy/filtermail/deployer.py

@@ -20,10 +20,10 @@ class FiltermailDeployer(Deployer):
             return
 
         arch = host.get_fact(facts.server.Arch)
-        url = f"https://github.com/chatmail/filtermail/releases/download/v0.6.6/filtermail-{arch}"
+        url = f"https://github.com/chatmail/filtermail/releases/download/v0.6.4/filtermail-{arch}"
         sha256sum = {
-            "x86_64": "05c7e7ac244606c2eeb275f2d282ffdbc2403e0169f1cdd3110ffcebdb994a92",
+            "x86_64": "5295115952c72e4c4ec3c85546e094b4155a4c702c82bd71fcdcb744dc73adf6",
-            "aarch64": "8cf8bbda4d907beca547b365cc7e6753532a74b1712492d0d2f3d2d8a553fb3d",
+            "aarch64": "6892244f17b8f26ccb465766e96028e7222b3c8adefca9fc6bfe9ff332ca8dff",
         }[arch]
         self.download_executable(url, self.bin_path, sha256sum)
 

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -54,17 +54,14 @@ smtpd_tls_exclude_ciphers = aNULL, RC4, MD5, DES
 tls_preempt_cipherlist = yes
 
 smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination
-myhostname = {{ config.postfix_myhostname }}
+myhostname = {{ config.mail_domain }}
 alias_maps = hash:/etc/aliases
 alias_database = hash:/etc/aliases
 
-# When postfix receives mail for $mydestination,
+# Postfix does not deliver mail for any domain by itself.
-# it hands it over to dovecot via $local_transport.
+# Primary domain is listed in `virtual_mailbox_domains` instead
-# Note: IP literals must be handled via local delivery / mydestination.
+# and handed over to Dovecot.
-mydestination = {{ config.mail_domain }}
+mydestination =
-local_transport = lmtp:unix:private/dovecot-lmtp
-# postfix doesn't check whether local users exist or not:
-local_recipient_maps =
 
 relayhost = 
 {% if disable_ipv6 %}
@@ -82,6 +79,8 @@ inet_protocols = ipv4
 inet_protocols = all
 {% endif %}
 
+virtual_transport = lmtp:unix:private/dovecot-lmtp
+virtual_mailbox_domains = {{ config.mail_domain }}
 lmtp_header_checks = regexp:/etc/postfix/lmtp_header_cleanup
 
 mua_client_restrictions = permit_sasl_authenticated, reject
@@ -103,11 +102,3 @@ smtpd_peername_lookup = no
 default_transport = lmtp-filtermail:inet:[127.0.0.1]:{{ config.filtermail_lmtp_port_transport }}
 lmtp-filtermail_initial_destination_concurrency=10000
 lmtp-filtermail_destination_concurrency_limit=10000
-
-{% if not config.ipv4_relay %}
-# DKIM-sign locally generated mail (bounces, DSNs).
-# These bypass smtpd, so they need explicit milter configuration.
-non_smtpd_milters = unix:opendkim/opendkim.sock
-internal_mail_filter_classes = bounce
-milter_macro_daemon_name = ORIGINATING
-{% endif %}

cmdeploy/src/cmdeploy/postfix/master.cf.j2

@@ -80,9 +80,8 @@ filter    unix -        n       n       -       -       lmtp
 127.0.0.1:{{ config.postfix_reinject_port }} inet  n       -       n       -       100      smtpd
   -o syslog_name=postfix/reinject
   -o milter_macro_daemon_name=ORIGINATING
+  -o smtpd_milters=unix:opendkim/opendkim.sock
   -o cleanup_service_name=authclean
-{% if not config.ipv4_relay %}  -o smtpd_milters=unix:opendkim/opendkim.sock
-{% endif %}
 
 # Local SMTP server for reinjecting incoming filtered mail 
 127.0.0.1:{{ config.postfix_reinject_port_incoming }} inet  n       -       n       -       100      smtpd

cmdeploy/src/cmdeploy/tests/online/test_0_login.py

@@ -12,7 +12,7 @@ def test_init(tmp_path, maildomain):
     inipath = tmp_path.joinpath("chatmail.ini")
     main(["init", "--config", str(inipath), maildomain])
     config = read_config(inipath)
-    assert config.mail_domain_bare == maildomain
+    assert config.mail_domain == maildomain
 
 
 def test_capabilities(imap):
@@ -89,11 +89,12 @@ def test_concurrent_logins_same_account(
         assert login_results.get()
 
 
-def test_no_vrfy(cmfactory, chatmail_config, maildomain):
+def test_no_vrfy(cmfactory, chatmail_config):
     ac = cmfactory.get_online_account()
     addr = ac.get_config("addr")
+    domain = chatmail_config.mail_domain
 
-    s = smtplib.SMTP(maildomain)
+    s = smtplib.SMTP(domain)
     s.starttls()
 
     s.putcmd("vrfy", f"wrongaddress@{chatmail_config.mail_domain}")

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -5,7 +5,6 @@ import subprocess
 import time
 
 import pytest
-from chatmaild.config import is_valid_ipv4
 
 from cmdeploy import remote
 from cmdeploy.cmdeploy import get_sshexec
@@ -22,8 +21,6 @@ class TestSSHExecutor:
         assert out == out2
 
     def test_perform_initial(self, sshexec, maildomain):
-        if is_valid_ipv4(maildomain):
-            pytest.skip(f"{maildomain} is not a domain")
         res = sshexec(
             remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain)
         )
@@ -64,10 +61,8 @@ class TestSSHExecutor:
         else:
             pytest.fail("didn't raise exception")
 
-    def test_opendkim_restarted(self, sshexec, maildomain):
+    def test_opendkim_restarted(self, sshexec):
         """check that opendkim is not running for longer than a day."""
-        if is_valid_ipv4(maildomain):
-            pytest.skip(f"{maildomain} is an IPv4 relay, opendkim is not installed")
         cmd = "systemctl show opendkim --timestamp=utc --property=ActiveEnterTimestamp"
         out = sshexec(call=remote.rshell.shell, kwargs=dict(command=cmd))
         datestring = out.split("=")[1]
@@ -194,34 +189,6 @@ def test_reject_missing_dkim(cmsetup, maildata, from_addr):
             s.sendmail(from_addr=from_addr, to_addrs=recipient.addr, msg=msg)
 
 
-def test_bounces_are_dkim_signed(cmsetup, cmsetup2, maildata, maildomain):
-    # we send a message to non-existant user and expect a bounce message
-    # which will only get through if the bounce message was DKIM-signed
-
-    if is_valid_ipv4(maildomain):
-        pytest.skip("DKIM is not configured on IPv4-only relays")
-
-    sender = cmsetup2.gen_users(1)[0]
-    nonexistent = f"nosuchuser_test42@{cmsetup.maildomain}"
-
-    msg = maildata(
-        "encrypted.eml",
-        from_addr=sender.addr,
-        to_addr=nonexistent,
-    ).as_string()
-    sender.smtp.sendmail(sender.addr, [nonexistent], msg)
-
-    def bounce_in_inbox():
-        messages = sender.imap.fetch_all_messages()
-        for m in messages:
-            if "mail delivery" in m.lower() or "undelivered" in m.lower():
-                return m
-        raise ValueError("bounce not yet in inbox")
-
-    bounce = try_n_times(30, bounce_in_inbox)
-    assert "nosuchuser_test42" in bounce
-
-
 def try_n_times(n, f):
     for _ in range(n - 1):
         try:
@@ -323,6 +290,4 @@ def test_nginx_access_log_only_defined_once(sshdomain):
         kwargs=dict(command="nginx -T 2>/dev/null"),
     )
     access_logs = [l for l in conf.splitlines() if l.strip().startswith("access_log")]
-    assert len(access_logs) == 1, (
+    assert len(access_logs) == 1, f"expected 1 access_log, found {len(access_logs)}: {access_logs}"
-        f"expected 1 access_log, found {len(access_logs)}: {access_logs}"
-    )

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -15,7 +15,7 @@ def imap_mailbox(cmfactory, ssl_context):
     (ac1,) = cmfactory.get_online_accounts(1)
     user = ac1.get_config("addr")
     password = ac1.get_config("mail_pw")
-    host = user.split("@")[1].strip("[").strip("]")
+    host = user.split("@")[1]
     mailbox = imap_tools.MailBox(host, ssl_context=ssl_context)
     mailbox.login(user, password)
     mailbox.dc_ac = ac1
@@ -178,7 +178,7 @@ def test_hide_senders_ip_address(cmfactory, ssl_context):
     chat.send_text("testing submission header cleanup")
     user2.wait_for_incoming_msg()
     addr = user2.get_config("addr")
-    host = addr.split("@")[1].strip("[").strip("]")
+    host = addr.split("@")[1]
     pw = user2.get_config("mail_pw")
     mailbox = imap_tools.MailBox(host, ssl_context=ssl_context)
     mailbox.login(addr, pw)

cmdeploy/src/cmdeploy/tests/plugin.py

@@ -1,4 +1,5 @@
 import imaplib
+import ipaddress
 import itertools
 import os
 import random
@@ -9,20 +10,19 @@ import time
 from pathlib import Path
 
 import pytest
-from chatmaild.config import is_valid_ipv4, read_config
+from chatmaild.config import read_config
-from domain_validator import DomainValidator
-
-
-def format_mail_domain(raw_domain: str) -> str:
-    if is_valid_ipv4(raw_domain):
-        return f"[{raw_domain}]"
-    DomainValidator().validate_domain_re(raw_domain)
-    return raw_domain
-
 
 conftestdir = Path(__file__).parent
 
 
+def _is_ip(domain):
+    try:
+        ipaddress.ip_address(domain)
+        return True
+    except ValueError:
+        return False
+
+
 def pytest_configure(config):
     config._benchresults = {}
     config.addinivalue_line(
@@ -58,7 +58,7 @@ def chatmail_config(pytestconfig):
 
 @pytest.fixture(scope="session")
 def maildomain(chatmail_config):
-    return chatmail_config.mail_domain_bare
+    return chatmail_config.mail_domain
 
 
 @pytest.fixture(scope="session")
@@ -278,6 +278,7 @@ def gencreds(chatmail_config):
 
     def gen(domain=None):
         domain = domain if domain else chatmail_config.mail_domain
+        addr_domain = f"[{domain}]" if _is_ip(domain) else domain
         while 1:
             num = next(count)
             alphanumeric = "abcdefghijklmnopqrstuvwxyz1234567890"
@@ -291,7 +292,7 @@ def gencreds(chatmail_config):
             password = "".join(
                 random.choices(alphanumeric, k=chatmail_config.password_min_length)
             )
-            yield f"{user}@{domain}", f"{password}"
+            yield f"{user}@{addr_domain}", f"{password}"
 
     return lambda domain=None: next(gen(domain))
 
@@ -316,8 +317,7 @@ class ChatmailACFactory:
 
     def _make_transport(self, domain):
         """Build a transport config dict for the given domain."""
-        domain_deliverable = format_mail_domain(domain)
+        addr, password = self.gencreds(domain)
-        addr, password = self.gencreds(domain_deliverable)
         transport = {
             "addr": addr,
             "password": password,
@@ -326,7 +326,7 @@ class ChatmailACFactory:
             "imapServer": domain,
             "smtpServer": domain,
         }
-        if domain.startswith("_") or is_valid_ipv4(domain):
+        if self.chatmail_config.tls_cert_mode == "self":
             transport["certificateChecks"] = "acceptInvalidCertificates"
         return transport
 
@@ -341,9 +341,8 @@ class ChatmailACFactory:
         accounts = []
         for _ in range(num):
             account = self.dc.add_account()
-            domain_deliverable = format_mail_domain(domain)
+            addr, password = self.gencreds(domain)
-            addr, password = self.gencreds(domain_deliverable)
+            if _is_ip(domain):
-            if is_valid_ipv4(domain):
                 # Use DCLOGIN scheme with explicit server hosts,
                 # matching how madmail presents its addresses to users.
                 qr = (
@@ -417,10 +416,10 @@ class Remote:
     def iter_output(self, logcmd="", ready=None):
         getjournal = "journalctl -f" if not logcmd else logcmd
         print(self.sshdomain)
-        if self.sshdomain in ("@local", "localhost"):
+        match self.sshdomain:
-            command = []
+            case "@local": command = []
-        else:
+            case "localhost": command = []
-            command = ["ssh", f"root@{self.sshdomain}"]
+            case _: command = ["ssh", f"root@{self.sshdomain}"]
         [command.append(arg) for arg in getjournal.split()]
         popen = subprocess.Popen(
             command,
@@ -467,11 +466,6 @@ def cmsetup(maildomain, gencreds, ssl_context):
     return CMSetup(maildomain, gencreds, ssl_context)
 
 
-@pytest.fixture
-def cmsetup2(maildomain2, gencreds, ssl_context):
-    return CMSetup(maildomain2, gencreds, ssl_context)
-
-
 class CMSetup:
     def __init__(self, maildomain, gencreds, ssl_context):
         self.maildomain = maildomain
@@ -482,7 +476,7 @@ class CMSetup:
         print(f"Creating {num} online users")
         users = []
         for i in range(num):
-            addr, password = self.gencreds(format_mail_domain(self.maildomain))
+            addr, password = self.gencreds()
             user = CMUser(self.maildomain, addr, password, self.ssl_context)
             assert user.smtp
             users.append(user)

cmdeploy/src/cmdeploy/tests/test_cmdeploy.py

@@ -39,14 +39,6 @@ class TestCmdline:
         out, err = capsys.readouterr()
         assert "deleting config file" in out.lower()
 
-    def test_dns_skip_on_ip(self, capsys, tmp_path, monkeypatch):
-        monkeypatch.delenv("CHATMAIL_INI", raising=False)
-        inipath = tmp_path / "chatmail.ini"
-        assert main(["init", "--config", str(inipath), "1.3.3.7"]) == 0
-        assert main(["dns", "--config", str(inipath)]) == 0
-        out, err = capsys.readouterr()
-        assert out == "[WARNING] 1.3.3.7 is not a domain, skipping DNS checks.\n"
-
 
 def test_www_folder(example_config, tmp_path):
     reporoot = importlib.resources.files(__package__).joinpath("../../../../").resolve()

cmdeploy/src/cmdeploy/tests/test_dovecot_deployer.py

@@ -23,7 +23,8 @@ def make_host(*fact_pairs):
         if cls not in facts:
             registered = ", ".join(c.__name__ for c in facts)
             raise LookupError(
-                f"unexpected get_fact({cls.__name__}); only registered: {registered}"
+                f"unexpected get_fact({cls.__name__}); "
+                f"only registered: {registered}"
             )
         return facts[cls]
 

doc/source/faq.rst

@@ -15,7 +15,6 @@ goes beyond what classic email servers offer:
    streaming, privacy-preserving Push Notifications for Apple, Google, and `Ubuntu Touch <https://docs.ubports.com/en/latest/appdev/guides/pushnotifications.html>`_;
 
 -  **Security Enforcement**: only strict TLS, DKIM and OpenPGP with minimized metadata accepted
-   (DKIM is not enforced on :ref:`IP-only relays <iponly>`)
 
 -  **Reliable Federation and Decentralization:** No spam or IP reputation checks, federating
    depends on established IETF standards and protocols.
@@ -48,28 +47,6 @@ Dovecot, and are configured to run unattended without much maintenance
 effort. Chatmail relays happily run on low-end hardware like a Raspberry
 Pi.
 
-.. _upgrade:
-
-How can I upgrade my chatmail relay?
-------------------------------------
-
-To upgrade to the latest ``main`` branch,
-``cd`` into your local checkout of `https://github.com/chatmail/relay/`_
-and run the following commands:
-
-   ::
-
-       git pull origin main --rebase --autostash
-       scripts/cmdeploy run
-
-If you don't want the latest development version,
-but a specific tagged release like `1.10.0 <https://github.com/chatmail/relay/releases/tag/1.10.0>`_,
-run ``git pull origin 1.10.0`` instead.
-
-If you made local changes for your setup,
-they will be reapplied as long as they don't conflict with the upgrade.
-If a conflict arises, ``git status`` will tell you how to resolve it.
-
 
 How trustable are chatmail relays?
 ----------------------------------

doc/source/getting_started.rst

@@ -14,6 +14,8 @@ Minimal requirements and prerequisites
 
 You will need the following:
 
+-  Control over a domain through a DNS provider of your choice.
+
 -  A Debian 12 **deployment server** with reachable SMTP/SUBMISSIONS/IMAPS/HTTPS ports.
    IPv6 is encouraged if available. Chatmail relay servers only require
    1GB RAM, one CPU, and perhaps 10GB storage for a few thousand active
@@ -26,11 +28,6 @@ You will need the following:
    (An ed25519 private key is required due to an `upstream bug in
    paramiko <https://github.com/paramiko/paramiko/issues/2191>`_)
 
--  Control over a domain through a DNS provider of your choice
-   (there is experimental support for :ref:`IP-only relays <iponly>`).
-
-
-.. _setup:
 
 Setup with ``scripts/cmdeploy``
 -------------------------------------
@@ -101,15 +98,6 @@ steps. Please substitute it with your own domain.
    configure at your DNS provider (it can take some time until they are
    public).
 
-
-
-Docker installation
--------------------
-
-There is experimental support for running chatmail via Docker.
-A monolithic image based on the above cmdeploy method is available `through a separate repository <https://github.com/chatmail/docker/pkgs/container/docker>`_.
-See the `chatmail/docker README <https://github.com/chatmail/docker>`_ for full setup instructions.
-
 Other helpful commands
 ----------------------
 

doc/source/index.rst

@@ -19,4 +19,3 @@ Contributions and feedback welcome through the https://github.com/chatmail/relay
     reverse_dns
     related
     faq
-    iponly

doc/source/iponly.rst

@@ -1,40 +0,0 @@
-.. _iponly:
-
-Hosting without DNS records
-===========================
-
-.. note::
-
-   This option is experimental and might change without notice.
-
-In case you don't have a domain,
-for example in a local network,
-you can run a chatmail relay with only an IPv4 address as well.
-
-To deploy a relay without a domain,
-run ``cmdeploy init`` with only the IPv4 address
-during the :ref:`installation steps <setup>`,
-for example ``cmdeploy init 13.12.23.42``.
-
-Drawbacks
----------
-
-- your transport encryption will only use self-signed TLS certificates,
-  which are vulnerable against MITM attacks.
-  the chatmail core's end-to-end encryption should suffice in most scenarios though.
-
-- your messages will not be DKIM-signed;
-  experimentally, most chatmail relays accept non-DKIM-signed messages from IP-only relays,
-  but some relays might not accept messages from yours.
-
-
-Email addresses
----------------
-
-When running without a domain,
-your chatmail addresses will use the IPv4 address
-in brackets as the domain part,
-for example ``user@[13.12.23.42]``.
-This is a valid email address format
-according to :rfc:`5321`.
-

doc/source/overview.rst

@@ -265,8 +265,7 @@ from the chatmail relay server.
 Email domain authentication (DKIM)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Chatmail relays enforce :rfc:`DKIM <6376>` to authenticate incoming emails
+Chatmail relays enforce :rfc:`DKIM <6376>` to authenticate incoming emails.
-(except for :ref:`IP-only relays <iponly>`).
 Incoming emails must have a valid DKIM signature with
 Signing Domain Identifier (SDID, ``d=`` parameter in the DKIM-Signature
 header) equal to the ``From:`` header domain. This property is checked