feat: add tool to analyze deferred queue

It prints all destinations with the number of recipients
and all the reasons. Operator can then try
to fix the problems for destinations,
e.g. by manually adding reverse proxy
addresses to /etc/hosts for failing domains
or routing IP addresses to another interface.

.github/workflows/ci.yaml

@@ -29,7 +29,7 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha }}
           persist-credentials: false
       - name: download filtermail
-        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.6.1/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
+        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.6.4/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
       - name: run chatmaild tests 
         working-directory: chatmaild
         run: pipx run tox

chatmaild/pyproject.toml

@@ -24,6 +24,7 @@ chatmail-metadata = "chatmaild.metadata:main"
 chatmail-expire = "chatmaild.expire:daily_expire_main"
 chatmail-quota-expire = "chatmaild.expire:quota_expire_main"
 chatmail-fsreport = "chatmaild.fsreport:main"
+chatmail-deferred = "chatmaild.deferred:main"
 lastlogin = "chatmaild.lastlogin:main"
 turnserver = "chatmaild.turnserver:main"
 

chatmaild/src/chatmaild/config.py

@@ -40,6 +40,9 @@ class Config:
         self.filtermail_http_port_incoming = int(
             params.get("filtermail_http_port_incoming", "10082")
         )
+        self.filtermail_lmtp_port_transport = int(
+            params.get("filtermail_lmtp_port_transport", "10083")
+        )
         self.postfix_reinject_port = int(params.get("postfix_reinject_port", "10025"))
         self.postfix_reinject_port_incoming = int(
             params.get("postfix_reinject_port_incoming", "10026")

chatmaild/src/chatmaild/deferred.py

@@ -0,0 +1,37 @@
+"""
+Analyze deferred mails and print most common failing destinations.
+
+Example:
+
+    python -m chatmaild.deferred
+"""
+
+import json
+import subprocess
+from collections import Counter, defaultdict
+
+
+def main():
+    p = subprocess.Popen(["postqueue", "-j"], text=True, stdout=subprocess.PIPE)
+    domain_reasons = defaultdict(Counter)
+    domain_total = Counter()
+
+    for line in p.stdout:
+        item = json.loads(line)
+        if item["queue_name"] != "deferred":
+            continue
+
+        for recipient in item["recipients"]:
+            _, domain = recipient["address"].rsplit("@", 1)
+            reason = recipient["delay_reason"]
+            domain_total[domain] += 1
+            domain_reasons[domain][reason] += 1
+
+    for domain, total in reversed(domain_total.most_common()):
+        print(f"{domain} ({total} recipients)")
+        for reason, count in domain_reasons[domain].most_common():
+            print(f"  {count}: {reason}")
+
+
+if __name__ == "__main__":
+    main()

cmdeploy/src/cmdeploy/filtermail/deployer.py

@@ -7,7 +7,7 @@ from cmdeploy.basedeploy import Deployer, get_resource
 
 
 class FiltermailDeployer(Deployer):
-    services = ["filtermail", "filtermail-incoming"]
+    services = ["filtermail", "filtermail-incoming", "filtermail-transport"]
     bin_path = "/usr/local/bin/filtermail"
     config_path = "/usr/local/lib/chatmaild/chatmail.ini"
 
@@ -26,10 +26,10 @@ class FiltermailDeployer(Deployer):
             return
 
         arch = host.get_fact(facts.server.Arch)
-        url = f"https://github.com/chatmail/filtermail/releases/download/v0.6.1/filtermail-{arch}"
+        url = f"https://github.com/chatmail/filtermail/releases/download/v0.6.4/filtermail-{arch}"
         sha256sum = {
-            "x86_64": "48b3fb80c092d00b9b0a0ef77a8673496da3b9aed5ec1851e1df936d5589d62f",
-            "aarch64": "c65bd5f45df187d3d65d6965a285583a3be0f44a6916ff12909ff9a8d702c22e",
+            "x86_64": "5295115952c72e4c4ec3c85546e094b4155a4c702c82bd71fcdcb744dc73adf6",
+            "aarch64": "6892244f17b8f26ccb465766e96028e7222b3c8adefca9fc6bfe9ff332ca8dff",
         }[arch]
         self.need_restart |= files.download(
             name="Download filtermail",

cmdeploy/src/cmdeploy/filtermail/filtermail-transport.service.j2

@@ -0,0 +1,11 @@
+[Unit]
+Description=Chatmail transport service
+
+[Service]
+ExecStart={{ bin_path }} {{ config_path }} transport
+Restart=always
+RestartSec=30
+User=vmail
+
+[Install]
+WantedBy=multi-user.target

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -73,7 +73,7 @@ http {
 
 		access_log syslog:server=unix:/dev/log,facility=local7;
 
-		location /mxdeliv/ {
+		location /mxdeliv {
 		    proxy_pass http://127.0.0.1:{{ config.filtermail_http_port_incoming }};
 		}
 

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -79,22 +79,6 @@ inet_protocols = ipv4
 inet_protocols = all
 {% endif %}
 
-# Postfix does not try IPv4 and IPv6 connections
-# concurrently as of version 3.7.11.
-#
-# When relay has both A (IPv4) and AAAA (IPv6) records,
-# but broken IPv6 connectivity,
-# every second message is delayed by the connection timeout
-# <https://www.postfix.org/postconf.5.html#smtp_connect_timeout>
-# which defaults to 30 seconds. Reducing timeouts is not a solution
-# as this will result in a failure to connect to slow servers.
-#
-# As a workaround we always prefer IPv4 when it is available.
-# 
-# The setting is documented at
-# <https://www.postfix.org/postconf.5.html#smtp_address_preference>
-smtp_address_preference=ipv4
-
 virtual_transport = lmtp:unix:private/dovecot-lmtp
 virtual_mailbox_domains = {{ config.mail_domain }}
 lmtp_header_checks = regexp:/etc/postfix/lmtp_header_cleanup
@@ -109,3 +93,10 @@ smtpd_sender_login_maps = regexp:/etc/postfix/login_map
 # Do not lookup SMTP client hostnames to reduce delays
 # and avoid unnecessary DNS requests.
 smtpd_peername_lookup = no
+
+# Use filtermail-transport to relay messages.
+# We can't force postfix to split messages per destination,
+# when specifying a custom next-hop,
+# so instead this is handled in filtermail.
+# We use LMTP instead SMTP so we can communicate per-recipient errors back to postfix.
+default_transport = lmtp-filtermail:inet:[127.0.0.1]:{{ config.filtermail_lmtp_port_transport }}

cmdeploy/src/cmdeploy/postfix/master.cf.j2

@@ -100,3 +100,8 @@ filter    unix -        n       n       -       -       lmtp
 # cannot send unprotected Subject.
 authclean unix  n       -       -       -       0       cleanup
   -o header_checks=regexp:/etc/postfix/submission_header_cleanup
+
+lmtp-filtermail unix  -       -       y       -       -       lmtp
+  -o syslog_name=postfix/lmtp-filtermail
+  -o lmtp_header_checks=
+  -o lmtp_tls_security_level=none

cmdeploy/src/cmdeploy/remote/rdns.py

@@ -64,13 +64,11 @@ def get_dkim_entry(mail_domain, pre_command, dkim_selector):
     )
 
 
-def query_dns(typ, domain, shell_exec=None):
-    if shell_exec is None:
-        shell_exec = shell
+def query_dns(typ, domain):
     # Get autoritative nameserver from the SOA record.
     soa_answers = [
         x.split()
-        for x in shell_exec(
+        for x in shell(
             f"dig -r -q {domain} -t SOA +noall +authority +answer", print=log_progress
         ).split("\n")
     ]
@@ -80,27 +78,8 @@ def query_dns(typ, domain, shell_exec=None):
     ns = soa[0][4]
 
     # Query authoritative nameserver directly to bypass DNS cache.
-    return _dig_authoritative(ns, domain, typ, shell_exec=shell_exec)
-
-
-def _parse_dig_result(output):
-    """Return first non-comment, non-empty line from dig output, or empty string."""
-    lines = [line for line in output.split("\n") if not line.startswith(";")]
-    return next((line for line in lines if line.strip()), "")
-
-
-def _dig_authoritative(ns, domain, typ, shell_exec=None):
-    """Query authoritative NS, falling back to IPv4-only if default fails."""
-    if shell_exec is None:
-        shell_exec = shell
-
-    # limit timeout and tries to not hang on a broken default NS
-    cmd = f"dig @{ns} -r -q {domain} -t {typ} +short +timeout=10 +tries=2"
-    result = _parse_dig_result(shell_exec(cmd, print=log_progress))
-    if result:
-        return result
-    # Fallback: force IPv4 transport (handles broken IPv6 to NS)
-    return _parse_dig_result(shell_exec(cmd + " -4", print=log_progress))
+    res = shell(f"dig @{ns} -r -q {domain} -t {typ} +short", print=log_progress)
+    return next((line for line in res.split("\n") if not line.startswith(";")), "")
 
 
 def check_zonefile(zonefile, verbose=True):

cmdeploy/src/cmdeploy/tests/test_dns.py

@@ -214,59 +214,3 @@ class TestZonefileChecks:
         assert not mockout.captured_red
         assert "correct" in mockout.captured_green[0]
         assert not mockout.captured_red
-
-
-class TestDigAuthoritative:
-    @pytest.fixture
-    def dig_auth(self):
-        """Helper that calls _dig_authoritative with a recording shell function."""
-        calls = []
-
-        def run(first_result, ipv4_result=None):
-            def shell(cmd, print=print):
-                calls.append(cmd)
-                if "-4" in cmd:
-                    return ipv4_result or ""
-                return first_result
-
-            result = remote.rdns._dig_authoritative(
-                "ns1.example.", "example.com", "A", shell_exec=shell
-            )
-            return result, calls
-
-        return run
-
-    def test_ipv4_fallback_on_error(self, dig_auth):
-        """Fallback to -4 when first query returns only error lines."""
-        result, calls = dig_auth(
-            first_result=(
-                ";; communications error to 2a01:4f8:10a:1044::80#53: timed out\n"
-                ";; communications error to 2a01:4f8:10a:1044::80#53: timed out\n"
-            ),
-            ipv4_result="1.2.3.4",
-        )
-        assert len(calls) == 2
-        assert "-4" not in calls[0]
-        assert "-4" in calls[1]
-        assert result == "1.2.3.4"
-
-    def test_no_fallback_on_success(self, dig_auth):
-        """No -4 fallback when first query returns a valid answer."""
-        result, calls = dig_auth(first_result="1.2.3.4")
-        assert result == "1.2.3.4"
-        assert len(calls) == 1
-
-    def test_fallback_with_mixed_error_lines(self, dig_auth):
-        """Fallback triggers when output has only error lines mixed with empty."""
-        result, calls = dig_auth(
-            first_result=(
-                ";; communications error to 2a01:4f8::80#53: timed out\n"
-                "\n"
-                ";; communications error to 2a01:4f8::80#53: timed out\n"
-            ),
-            ipv4_result=";; some warning\n1.2.3.4",
-        )
-        assert len(calls) == 2
-        assert result == "1.2.3.4"
-
-

cmdeploy/src/cmdeploy/tests/test_helpers.py

@@ -1,11 +1,10 @@
-import importlib.resources
+from pathlib import Path
 
 from cmdeploy.www import build_webpages
 
 
 def test_build_webpages(tmp_path, make_config):
-    pkgroot = importlib.resources.files("cmdeploy")
-    src_dir = pkgroot.joinpath("../../../www/src").resolve()
+    src_dir = (Path(__file__).resolve() / "../../../../../www/src").resolve()
     assert src_dir.exists(), src_dir
     config = make_config("chat.example.org")
     build_dir = tmp_path.joinpath("build")

cmdeploy/src/cmdeploy/www.py

@@ -1,5 +1,4 @@
 import hashlib
-import importlib.resources
 import re
 import time
 import traceback
@@ -37,7 +36,7 @@ def prepare_template(source):
 
 
 def get_paths(config) -> (Path, Path, Path):
-    reporoot = importlib.resources.files(__package__).joinpath("../../../").resolve()
+    reporoot = (Path(__file__).resolve() / "../../../../").resolve()
     www_path = Path(config.www_folder)
     # if www_folder was not set, use default directory
     if config.www_folder == "":
@@ -133,8 +132,7 @@ def find_merge_conflict(src_dir) -> Path:
 
 
 def main():
-    path = importlib.resources.files(__package__)
-    reporoot = path.joinpath("../../../").resolve()
+    reporoot = (Path(__file__).resolve() / "../../../../").resolve()
     inipath = reporoot.joinpath("chatmail.ini")
     config = read_config(inipath)
     config.webdev = True

doc/source/overview.rst

@@ -153,6 +153,7 @@ Chatmail relay dependency diagram
         autoconfig.xml --- dovecot;
         postfix --- |10080|filtermail-outgoing;
         postfix --- |10081|filtermail-incoming;
+        postfix --- |10083|filtermail-transport;
         filtermail-outgoing --- |10025 reinject|postfix;
         filtermail-incoming --- |10026 reinject|postfix;
         dovecot --- |doveauth.socket|doveauth;
@@ -295,9 +296,7 @@ ensured by ``filtermail`` proxy.
 TLS requirements
 ~~~~~~~~~~~~~~~~
 
-Postfix is configured to require valid TLS by setting
-`smtp_tls_security_level <https://www.postfix.org/postconf.5.html#smtp_tls_security_level>`_
-to ``verify``.
+Filtermail (used for delivery) requires a valid TLS.
 
 You can test it by resolving ``MX`` records of your relay domain and
 then connecting to MX relays (e.g ``mx.example.org``) with