fix: DNS check timeout with IPv6-broken authoritative NS

Add IPv4 fallback for authoritative NS queries in query_dns().
When dig @ns returns no useful answer (e.g. IPv6 transport to
the NS times out), retry with -4 to force IPv4. Also limits
timeout and tries (+timeout=10 +tries=2) to avoid hanging.

Fixes #851

.github/workflows/ci.yaml

@@ -29,7 +29,7 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha }}
           persist-credentials: false
       - name: download filtermail
-        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.6.4/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
+        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.6.1/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
       - name: run chatmaild tests 
         working-directory: chatmaild
         run: pipx run tox

chatmaild/pyproject.toml

@@ -24,7 +24,6 @@ chatmail-metadata = "chatmaild.metadata:main"
 chatmail-expire = "chatmaild.expire:daily_expire_main"
 chatmail-quota-expire = "chatmaild.expire:quota_expire_main"
 chatmail-fsreport = "chatmaild.fsreport:main"
-chatmail-deferred = "chatmaild.deferred:main"
 lastlogin = "chatmaild.lastlogin:main"
 turnserver = "chatmaild.turnserver:main"
 

chatmaild/src/chatmaild/config.py

@@ -40,9 +40,6 @@ class Config:
         self.filtermail_http_port_incoming = int(
             params.get("filtermail_http_port_incoming", "10082")
         )
-        self.filtermail_lmtp_port_transport = int(
-            params.get("filtermail_lmtp_port_transport", "10083")
-        )
         self.postfix_reinject_port = int(params.get("postfix_reinject_port", "10025"))
         self.postfix_reinject_port_incoming = int(
             params.get("postfix_reinject_port_incoming", "10026")

chatmaild/src/chatmaild/deferred.py

@@ -1,37 +0,0 @@
-"""
-Analyze deferred mails and print most common failing destinations.
-
-Example:
-
-    python -m chatmaild.deferred
-"""
-
-import json
-import subprocess
-from collections import Counter, defaultdict
-
-
-def main():
-    p = subprocess.Popen(["postqueue", "-j"], text=True, stdout=subprocess.PIPE)
-    domain_reasons = defaultdict(Counter)
-    domain_total = Counter()
-
-    for line in p.stdout:
-        item = json.loads(line)
-        if item["queue_name"] != "deferred":
-            continue
-
-        for recipient in item["recipients"]:
-            _, domain = recipient["address"].rsplit("@", 1)
-            reason = recipient["delay_reason"]
-            domain_total[domain] += 1
-            domain_reasons[domain][reason] += 1
-
-    for domain, total in reversed(domain_total.most_common()):
-        print(f"{domain} ({total} recipients)")
-        for reason, count in domain_reasons[domain].most_common():
-            print(f"  {count}: {reason}")
-
-
-if __name__ == "__main__":
-    main()

cmdeploy/src/cmdeploy/filtermail/deployer.py

@@ -7,7 +7,7 @@ from cmdeploy.basedeploy import Deployer, get_resource
 
 
 class FiltermailDeployer(Deployer):
-    services = ["filtermail", "filtermail-incoming", "filtermail-transport"]
+    services = ["filtermail", "filtermail-incoming"]
     bin_path = "/usr/local/bin/filtermail"
     config_path = "/usr/local/lib/chatmaild/chatmail.ini"
 
@@ -26,10 +26,10 @@ class FiltermailDeployer(Deployer):
             return
 
         arch = host.get_fact(facts.server.Arch)
-        url = f"https://github.com/chatmail/filtermail/releases/download/v0.6.4/filtermail-{arch}"
+        url = f"https://github.com/chatmail/filtermail/releases/download/v0.6.1/filtermail-{arch}"
         sha256sum = {
-            "x86_64": "5295115952c72e4c4ec3c85546e094b4155a4c702c82bd71fcdcb744dc73adf6",
-            "aarch64": "6892244f17b8f26ccb465766e96028e7222b3c8adefca9fc6bfe9ff332ca8dff",
+            "x86_64": "48b3fb80c092d00b9b0a0ef77a8673496da3b9aed5ec1851e1df936d5589d62f",
+            "aarch64": "c65bd5f45df187d3d65d6965a285583a3be0f44a6916ff12909ff9a8d702c22e",
         }[arch]
         self.need_restart |= files.download(
             name="Download filtermail",

cmdeploy/src/cmdeploy/filtermail/filtermail-transport.service.j2

@@ -1,11 +0,0 @@
-[Unit]
-Description=Chatmail transport service
-
-[Service]
-ExecStart={{ bin_path }} {{ config_path }} transport
-Restart=always
-RestartSec=30
-User=vmail
-
-[Install]
-WantedBy=multi-user.target

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -73,7 +73,7 @@ http {
 
 		access_log syslog:server=unix:/dev/log,facility=local7;
 
-		location /mxdeliv {
+		location /mxdeliv/ {
 		    proxy_pass http://127.0.0.1:{{ config.filtermail_http_port_incoming }};
 		}
 

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -79,6 +79,22 @@ inet_protocols = ipv4
 inet_protocols = all
 {% endif %}
 
+# Postfix does not try IPv4 and IPv6 connections
+# concurrently as of version 3.7.11.
+#
+# When relay has both A (IPv4) and AAAA (IPv6) records,
+# but broken IPv6 connectivity,
+# every second message is delayed by the connection timeout
+# <https://www.postfix.org/postconf.5.html#smtp_connect_timeout>
+# which defaults to 30 seconds. Reducing timeouts is not a solution
+# as this will result in a failure to connect to slow servers.
+#
+# As a workaround we always prefer IPv4 when it is available.
+# 
+# The setting is documented at
+# <https://www.postfix.org/postconf.5.html#smtp_address_preference>
+smtp_address_preference=ipv4
+
 virtual_transport = lmtp:unix:private/dovecot-lmtp
 virtual_mailbox_domains = {{ config.mail_domain }}
 lmtp_header_checks = regexp:/etc/postfix/lmtp_header_cleanup
@@ -93,10 +109,3 @@ smtpd_sender_login_maps = regexp:/etc/postfix/login_map
 # Do not lookup SMTP client hostnames to reduce delays
 # and avoid unnecessary DNS requests.
 smtpd_peername_lookup = no
-
-# Use filtermail-transport to relay messages.
-# We can't force postfix to split messages per destination,
-# when specifying a custom next-hop,
-# so instead this is handled in filtermail.
-# We use LMTP instead SMTP so we can communicate per-recipient errors back to postfix.
-default_transport = lmtp-filtermail:inet:[127.0.0.1]:{{ config.filtermail_lmtp_port_transport }}

cmdeploy/src/cmdeploy/postfix/master.cf.j2

@@ -100,8 +100,3 @@ filter    unix -        n       n       -       -       lmtp
 # cannot send unprotected Subject.
 authclean unix  n       -       -       -       0       cleanup
   -o header_checks=regexp:/etc/postfix/submission_header_cleanup
-
-lmtp-filtermail unix  -       -       y       -       -       lmtp
-  -o syslog_name=postfix/lmtp-filtermail
-  -o lmtp_header_checks=
-  -o lmtp_tls_security_level=none

cmdeploy/src/cmdeploy/remote/rdns.py

@@ -64,11 +64,13 @@ def get_dkim_entry(mail_domain, pre_command, dkim_selector):
     )
 
 
-def query_dns(typ, domain):
+def query_dns(typ, domain, shell_exec=None):
+    if shell_exec is None:
+        shell_exec = shell
     # Get autoritative nameserver from the SOA record.
     soa_answers = [
         x.split()
-        for x in shell(
+        for x in shell_exec(
             f"dig -r -q {domain} -t SOA +noall +authority +answer", print=log_progress
         ).split("\n")
     ]
@@ -78,8 +80,27 @@ def query_dns(typ, domain):
     ns = soa[0][4]
 
     # Query authoritative nameserver directly to bypass DNS cache.
-    res = shell(f"dig @{ns} -r -q {domain} -t {typ} +short", print=log_progress)
-    return next((line for line in res.split("\n") if not line.startswith(";")), "")
+    return _dig_authoritative(ns, domain, typ, shell_exec=shell_exec)
+
+
+def _parse_dig_result(output):
+    """Return first non-comment, non-empty line from dig output, or empty string."""
+    lines = [line for line in output.split("\n") if not line.startswith(";")]
+    return next((line for line in lines if line.strip()), "")
+
+
+def _dig_authoritative(ns, domain, typ, shell_exec=None):
+    """Query authoritative NS, falling back to IPv4-only if default fails."""
+    if shell_exec is None:
+        shell_exec = shell
+
+    # limit timeout and tries to not hang on a broken default NS
+    cmd = f"dig @{ns} -r -q {domain} -t {typ} +short +timeout=10 +tries=2"
+    result = _parse_dig_result(shell_exec(cmd, print=log_progress))
+    if result:
+        return result
+    # Fallback: force IPv4 transport (handles broken IPv6 to NS)
+    return _parse_dig_result(shell_exec(cmd + " -4", print=log_progress))
 
 
 def check_zonefile(zonefile, verbose=True):

cmdeploy/src/cmdeploy/tests/test_dns.py

@@ -214,3 +214,59 @@ class TestZonefileChecks:
         assert not mockout.captured_red
         assert "correct" in mockout.captured_green[0]
         assert not mockout.captured_red
+
+
+class TestDigAuthoritative:
+    @pytest.fixture
+    def dig_auth(self):
+        """Helper that calls _dig_authoritative with a recording shell function."""
+        calls = []
+
+        def run(first_result, ipv4_result=None):
+            def shell(cmd, print=print):
+                calls.append(cmd)
+                if "-4" in cmd:
+                    return ipv4_result or ""
+                return first_result
+
+            result = remote.rdns._dig_authoritative(
+                "ns1.example.", "example.com", "A", shell_exec=shell
+            )
+            return result, calls
+
+        return run
+
+    def test_ipv4_fallback_on_error(self, dig_auth):
+        """Fallback to -4 when first query returns only error lines."""
+        result, calls = dig_auth(
+            first_result=(
+                ";; communications error to 2a01:4f8:10a:1044::80#53: timed out\n"
+                ";; communications error to 2a01:4f8:10a:1044::80#53: timed out\n"
+            ),
+            ipv4_result="1.2.3.4",
+        )
+        assert len(calls) == 2
+        assert "-4" not in calls[0]
+        assert "-4" in calls[1]
+        assert result == "1.2.3.4"
+
+    def test_no_fallback_on_success(self, dig_auth):
+        """No -4 fallback when first query returns a valid answer."""
+        result, calls = dig_auth(first_result="1.2.3.4")
+        assert result == "1.2.3.4"
+        assert len(calls) == 1
+
+    def test_fallback_with_mixed_error_lines(self, dig_auth):
+        """Fallback triggers when output has only error lines mixed with empty."""
+        result, calls = dig_auth(
+            first_result=(
+                ";; communications error to 2a01:4f8::80#53: timed out\n"
+                "\n"
+                ";; communications error to 2a01:4f8::80#53: timed out\n"
+            ),
+            ipv4_result=";; some warning\n1.2.3.4",
+        )
+        assert len(calls) == 2
+        assert result == "1.2.3.4"
+
+

cmdeploy/src/cmdeploy/tests/test_helpers.py

@@ -1,10 +1,11 @@
-from pathlib import Path
+import importlib.resources
 
 from cmdeploy.www import build_webpages
 
 
 def test_build_webpages(tmp_path, make_config):
-    src_dir = (Path(__file__).resolve() / "../../../../../www/src").resolve()
+    pkgroot = importlib.resources.files("cmdeploy")
+    src_dir = pkgroot.joinpath("../../../www/src").resolve()
     assert src_dir.exists(), src_dir
     config = make_config("chat.example.org")
     build_dir = tmp_path.joinpath("build")

cmdeploy/src/cmdeploy/www.py

@@ -1,4 +1,5 @@
 import hashlib
+import importlib.resources
 import re
 import time
 import traceback
@@ -36,7 +37,7 @@ def prepare_template(source):
 
 
 def get_paths(config) -> (Path, Path, Path):
-    reporoot = (Path(__file__).resolve() / "../../../../").resolve()
+    reporoot = importlib.resources.files(__package__).joinpath("../../../").resolve()
     www_path = Path(config.www_folder)
     # if www_folder was not set, use default directory
     if config.www_folder == "":
@@ -132,7 +133,8 @@ def find_merge_conflict(src_dir) -> Path:
 
 
 def main():
-    reporoot = (Path(__file__).resolve() / "../../../../").resolve()
+    path = importlib.resources.files(__package__)
+    reporoot = path.joinpath("../../../").resolve()
     inipath = reporoot.joinpath("chatmail.ini")
     config = read_config(inipath)
     config.webdev = True

doc/source/overview.rst

@@ -153,7 +153,6 @@ Chatmail relay dependency diagram
         autoconfig.xml --- dovecot;
         postfix --- |10080|filtermail-outgoing;
         postfix --- |10081|filtermail-incoming;
-        postfix --- |10083|filtermail-transport;
         filtermail-outgoing --- |10025 reinject|postfix;
         filtermail-incoming --- |10026 reinject|postfix;
         dovecot --- |doveauth.socket|doveauth;
@@ -296,7 +295,9 @@ ensured by ``filtermail`` proxy.
 TLS requirements
 ~~~~~~~~~~~~~~~~
 
-Filtermail (used for delivery) requires a valid TLS.
+Postfix is configured to require valid TLS by setting
+`smtp_tls_security_level <https://www.postfix.org/postconf.5.html#smtp_tls_security_level>`_
+to ``verify``.
 
 You can test it by resolving ``MX`` records of your relay domain and
 then connecting to MX relays (e.g ``mx.example.org``) with