feat: add tool to analyze deferred queue

It prints all destinations with the number of recipients
and all the reasons. Operator can then try
to fix the problems for destinations,
e.g. by manually adding reverse proxy
addresses to /etc/hosts for failing domains
or routing IP addresses to another interface.

.github/workflows/ci.yaml

@@ -29,7 +29,7 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha }}
           persist-credentials: false
       - name: download filtermail
-        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.6.1/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
+        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.6.4/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
       - name: run chatmaild tests 
         working-directory: chatmaild
         run: pipx run tox

chatmaild/pyproject.toml

@@ -24,6 +24,7 @@ chatmail-metadata = "chatmaild.metadata:main"
 chatmail-expire = "chatmaild.expire:daily_expire_main"
 chatmail-quota-expire = "chatmaild.expire:quota_expire_main"
 chatmail-fsreport = "chatmaild.fsreport:main"
+chatmail-deferred = "chatmaild.deferred:main"
 lastlogin = "chatmaild.lastlogin:main"
 turnserver = "chatmaild.turnserver:main"
 

chatmaild/src/chatmaild/config.py

@@ -40,6 +40,9 @@ class Config:
         self.filtermail_http_port_incoming = int(
             params.get("filtermail_http_port_incoming", "10082")
         )
+        self.filtermail_lmtp_port_transport = int(
+            params.get("filtermail_lmtp_port_transport", "10083")
+        )
         self.postfix_reinject_port = int(params.get("postfix_reinject_port", "10025"))
         self.postfix_reinject_port_incoming = int(
             params.get("postfix_reinject_port_incoming", "10026")

chatmaild/src/chatmaild/deferred.py

@@ -0,0 +1,37 @@
+"""
+Analyze deferred mails and print most common failing destinations.
+
+Example:
+
+    python -m chatmaild.deferred
+"""
+
+import json
+import subprocess
+from collections import Counter, defaultdict
+
+
+def main():
+    p = subprocess.Popen(["postqueue", "-j"], text=True, stdout=subprocess.PIPE)
+    domain_reasons = defaultdict(Counter)
+    domain_total = Counter()
+
+    for line in p.stdout:
+        item = json.loads(line)
+        if item["queue_name"] != "deferred":
+            continue
+
+        for recipient in item["recipients"]:
+            _, domain = recipient["address"].rsplit("@", 1)
+            reason = recipient["delay_reason"]
+            domain_total[domain] += 1
+            domain_reasons[domain][reason] += 1
+
+    for domain, total in reversed(domain_total.most_common()):
+        print(f"{domain} ({total} recipients)")
+        for reason, count in domain_reasons[domain].most_common():
+            print(f"  {count}: {reason}")
+
+
+if __name__ == "__main__":
+    main()

cmdeploy/src/cmdeploy/filtermail/deployer.py

@@ -7,7 +7,7 @@ from cmdeploy.basedeploy import Deployer, get_resource
 
 
 class FiltermailDeployer(Deployer):
-    services = ["filtermail", "filtermail-incoming"]
+    services = ["filtermail", "filtermail-incoming", "filtermail-transport"]
     bin_path = "/usr/local/bin/filtermail"
     config_path = "/usr/local/lib/chatmaild/chatmail.ini"
 
@@ -26,10 +26,10 @@ class FiltermailDeployer(Deployer):
             return
 
         arch = host.get_fact(facts.server.Arch)
-        url = f"https://github.com/chatmail/filtermail/releases/download/v0.6.1/filtermail-{arch}"
+        url = f"https://github.com/chatmail/filtermail/releases/download/v0.6.4/filtermail-{arch}"
         sha256sum = {
-            "x86_64": "48b3fb80c092d00b9b0a0ef77a8673496da3b9aed5ec1851e1df936d5589d62f",
+            "x86_64": "5295115952c72e4c4ec3c85546e094b4155a4c702c82bd71fcdcb744dc73adf6",
-            "aarch64": "c65bd5f45df187d3d65d6965a285583a3be0f44a6916ff12909ff9a8d702c22e",
+            "aarch64": "6892244f17b8f26ccb465766e96028e7222b3c8adefca9fc6bfe9ff332ca8dff",
         }[arch]
         self.need_restart |= files.download(
             name="Download filtermail",

cmdeploy/src/cmdeploy/filtermail/filtermail-transport.service.j2

@@ -0,0 +1,11 @@
+[Unit]
+Description=Chatmail transport service
+
+[Service]
+ExecStart={{ bin_path }} {{ config_path }} transport
+Restart=always
+RestartSec=30
+User=vmail
+
+[Install]
+WantedBy=multi-user.target

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -73,7 +73,7 @@ http {
 
 		access_log syslog:server=unix:/dev/log,facility=local7;
 
-		location /mxdeliv/ {
+		location /mxdeliv {
 		    proxy_pass http://127.0.0.1:{{ config.filtermail_http_port_incoming }};
 		}
 

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -79,22 +79,6 @@ inet_protocols = ipv4
 inet_protocols = all
 {% endif %}
 
-# Postfix does not try IPv4 and IPv6 connections
-# concurrently as of version 3.7.11.
-#
-# When relay has both A (IPv4) and AAAA (IPv6) records,
-# but broken IPv6 connectivity,
-# every second message is delayed by the connection timeout
-# <https://www.postfix.org/postconf.5.html#smtp_connect_timeout>
-# which defaults to 30 seconds. Reducing timeouts is not a solution
-# as this will result in a failure to connect to slow servers.
-#
-# As a workaround we always prefer IPv4 when it is available.
-# 
-# The setting is documented at
-# <https://www.postfix.org/postconf.5.html#smtp_address_preference>
-smtp_address_preference=ipv4
-
 virtual_transport = lmtp:unix:private/dovecot-lmtp
 virtual_mailbox_domains = {{ config.mail_domain }}
 lmtp_header_checks = regexp:/etc/postfix/lmtp_header_cleanup
@@ -109,3 +93,10 @@ smtpd_sender_login_maps = regexp:/etc/postfix/login_map
 # Do not lookup SMTP client hostnames to reduce delays
 # and avoid unnecessary DNS requests.
 smtpd_peername_lookup = no
+
+# Use filtermail-transport to relay messages.
+# We can't force postfix to split messages per destination,
+# when specifying a custom next-hop,
+# so instead this is handled in filtermail.
+# We use LMTP instead SMTP so we can communicate per-recipient errors back to postfix.
+default_transport = lmtp-filtermail:inet:[127.0.0.1]:{{ config.filtermail_lmtp_port_transport }}

cmdeploy/src/cmdeploy/postfix/master.cf.j2

@@ -100,3 +100,8 @@ filter    unix -        n       n       -       -       lmtp
 # cannot send unprotected Subject.
 authclean unix  n       -       -       -       0       cleanup
   -o header_checks=regexp:/etc/postfix/submission_header_cleanup
+
+lmtp-filtermail unix  -       -       y       -       -       lmtp
+  -o syslog_name=postfix/lmtp-filtermail
+  -o lmtp_header_checks=
+  -o lmtp_tls_security_level=none

cmdeploy/src/cmdeploy/tests/test_helpers.py

@@ -1,11 +1,10 @@
-import importlib.resources
+from pathlib import Path
 
 from cmdeploy.www import build_webpages
 
 
 def test_build_webpages(tmp_path, make_config):
-    pkgroot = importlib.resources.files("cmdeploy")
+    src_dir = (Path(__file__).resolve() / "../../../../../www/src").resolve()
-    src_dir = pkgroot.joinpath("../../../www/src").resolve()
     assert src_dir.exists(), src_dir
     config = make_config("chat.example.org")
     build_dir = tmp_path.joinpath("build")

cmdeploy/src/cmdeploy/www.py

@@ -1,5 +1,4 @@
 import hashlib
-import importlib.resources
 import re
 import time
 import traceback
@@ -37,7 +36,7 @@ def prepare_template(source):
 
 
 def get_paths(config) -> (Path, Path, Path):
-    reporoot = importlib.resources.files(__package__).joinpath("../../../").resolve()
+    reporoot = (Path(__file__).resolve() / "../../../../").resolve()
     www_path = Path(config.www_folder)
     # if www_folder was not set, use default directory
     if config.www_folder == "":
@@ -133,8 +132,7 @@ def find_merge_conflict(src_dir) -> Path:
 
 
 def main():
-    path = importlib.resources.files(__package__)
+    reporoot = (Path(__file__).resolve() / "../../../../").resolve()
-    reporoot = path.joinpath("../../../").resolve()
     inipath = reporoot.joinpath("chatmail.ini")
     config = read_config(inipath)
     config.webdev = True

doc/source/overview.rst

@@ -153,6 +153,7 @@ Chatmail relay dependency diagram
         autoconfig.xml --- dovecot;
         postfix --- |10080|filtermail-outgoing;
         postfix --- |10081|filtermail-incoming;
+        postfix --- |10083|filtermail-transport;
         filtermail-outgoing --- |10025 reinject|postfix;
         filtermail-incoming --- |10026 reinject|postfix;
         dovecot --- |doveauth.socket|doveauth;
@@ -295,9 +296,7 @@ ensured by ``filtermail`` proxy.
 TLS requirements
 ~~~~~~~~~~~~~~~~
 
-Postfix is configured to require valid TLS by setting
+Filtermail (used for delivery) requires a valid TLS.
-`smtp_tls_security_level <https://www.postfix.org/postconf.5.html#smtp_tls_security_level>`_
-to ``verify``.
 
 You can test it by resolving ``MX`` records of your relay domain and
 then connecting to MX relays (e.g ``mx.example.org``) with