feat(mtail): validate programs during deploy

Compile /etc/mtail before activate() restarts the service, to catch errors
early.

APPVERSIONS.json

@@ -1,15 +0,0 @@
-{
-  "clients": [
-    {
-      "clientId": "deltachat",
-      "sources": [
-        {
-          "sourceId": "gplay",
-          "versionInteger": 754,
-          "versionString": "2.57.0",
-          "downloadUrl": "https://github.com/deltachat/deltachat-android/releases/download/v2.57.0/deltachat-gplay-release-2.57.0.apk"
-        }
-      ]
-    }
-  ]
-}

CONTRIBUTING.md

@@ -5,6 +5,3 @@ We use [git-cliff] to generate the changelog from commit messages before the rel
 
 [Conventional Commits]: https://www.conventionalcommits.org/
 [git-cliff]: https://git-cliff.org/
-
-To update client app version information,
-edit [APPVERSIONS.json](APPVERSIONS.json).

chatmaild/src/chatmaild/config.py

@@ -63,9 +63,6 @@ class Config:
         self.turn_socket_path = params.pop(
             "turn_socket_path", "/run/chatmail-turn/turn.socket"
         )
-        self.appversions_path = Path(
-            params.pop("appversions_path", "/usr/local/lib/chatmaild/appversions.json")
-        )
         iroh_relay = params.pop("iroh_relay", None)
         if iroh_relay is None:
             self.iroh_relay = "https://" + raw_domain

chatmaild/src/chatmaild/metadata.py

@@ -1,4 +1,3 @@
-import json
 import logging
 import socket
 import sys
@@ -19,18 +18,6 @@ def turn_credentials(turn_socket_path):
             return file.readline().decode("utf-8").strip()
 
 
-def read_appversions(path):
-    try:
-        data = json.loads(path.read_bytes())
-    except FileNotFoundError:
-        return None
-    except (OSError, ValueError):
-        logging.exception(f"failed to read {path}")
-        return None
-    # the dict protocol is line-based, keep the value single-line
-    return json.dumps(data, separators=(",", ":"))
-
-
 def _is_valid_token_timestamp(timestamp, now):
     # Token if invalid after 90 days
     # or if the timestamp is in the future.
@@ -107,7 +94,6 @@ class MetadataDictProxy(DictProxy):
         iroh_relay=None,
         turn_hostname=None,
         turn_socket_path=None,
-        appversions_path=None,
     ):
         super().__init__()
         self.notifier = notifier
@@ -115,7 +101,6 @@ class MetadataDictProxy(DictProxy):
         self.iroh_relay = iroh_relay
         self.turn_hostname = turn_hostname
         self.turn_socket_path = turn_socket_path
-        self.appversions_path = appversions_path
 
     def handle_lookup(self, parts):
         # Lpriv/43f5f508a7ea0366dff30200c15250e3/devicetoken\tlkj123poi@c2.testrun.org
@@ -140,9 +125,6 @@ class MetadataDictProxy(DictProxy):
                         case "maxsmtprecipients":
                             # postfix default  (see "postconf smtpd_recipient_limit")
                             return "O1000\n"
-                        case "appversions" if self.appversions_path:
-                            value = read_appversions(self.appversions_path)
-                            return f"O{value}\n" if value else "N\n"
 
         logging.warning(f"lookup ignored: {parts!r}")
         return "N\n"
@@ -188,7 +170,6 @@ def main():
         iroh_relay=iroh_relay,
         turn_hostname=mail_domain,
         turn_socket_path=socket_path,
-        appversions_path=config.appversions_path,
     )
 
     dictproxy.serve_forever_from_socket(socket)

chatmaild/src/chatmaild/tests/test_config.py

@@ -47,9 +47,6 @@ def test_read_config_basic_using_defaults(tmp_path, maildomain):
     assert example_config.password_min_length == 9
     assert example_config.max_imap_connections == 10000
     assert example_config.max_smtp_connections == 1000
-    assert str(example_config.appversions_path) == (
-        "/usr/local/lib/chatmaild/appversions.json"
-    )
     assert example_config._unused_keys == []
 
 

chatmaild/src/chatmaild/tests/test_metadata.py

@@ -7,7 +7,6 @@ import requests
 from chatmaild.metadata import (
     Metadata,
     MetadataDictProxy,
-    read_appversions,
 )
 from chatmaild.notifier import (
     Notifier,
@@ -370,17 +369,6 @@ def test_iroh_relay(dictproxy):
     assert wfile.getvalue() == b"Ohttps://example.org/\n"
 
 
-def test_read_appversions(tmp_path):
-    path = tmp_path.joinpath("appversions.json")
-    assert read_appversions(path) is None
-
-    path.write_text('{\n  "clients": []\n}')
-    assert read_appversions(path) == '{"clients":[]}'
-
-    path.write_text("bad json")
-    assert read_appversions(path) is None
-
-
 def test_legacy_token_migration(metadata, testaddr):
     with metadata.get_metadata_dict(testaddr).modify() as data:
         data[metadata.DEVICETOKEN_KEY] = ["oldtoken1", "oldtoken2"]

cmdeploy/src/cmdeploy/deployers.py

@@ -35,7 +35,7 @@ from .nginx.deployer import NginxDeployer
 from .opendkim.deployer import OpendkimDeployer
 from .postfix.deployer import PostfixDeployer
 from .selfsigned.deployer import SelfSignedTlsDeployer
-from .www import build_webpages, find_merge_conflict, get_paths, get_reporoot
+from .www import build_webpages, find_merge_conflict, get_paths
 
 
 class Port(FactBase):
@@ -126,11 +126,6 @@ def _configure_remote_venv_with_chatmaild(deployer, config) -> None:
         dest=remote_chatmail_inipath,
     )
 
-    deployer.put_file(
-        src=get_reporoot().joinpath("APPVERSIONS.json").open("rb"),
-        dest=str(config.appversions_path),
-    )
-
     deployer.remove_file("/etc/cron.d/chatmail-metrics")
     deployer.remove_file("/var/www/html/metrics")
 

cmdeploy/src/cmdeploy/mtail/deployer.py

@@ -1,5 +1,5 @@
 from pyinfra import facts, host
-from pyinfra.operations import apt
+from pyinfra.operations import apt, server
 
 from cmdeploy.basedeploy import Deployer
 
@@ -37,7 +37,20 @@ class MtailDeployer(Deployer):
             address=self.mtail_address or "127.0.0.1",
             port=3903,
         )
-        self.put_file("mtail/delivered_mail.mtail", "/etc/mtail/delivered_mail.mtail")
+        if self.mtail_address:
+            self.put_file("mtail/delivered_mail.mtail", "/etc/mtail/delivered_mail.mtail")
+            self.put_file("mtail/filtermail.mtail", "/etc/mtail/filtermail.mtail")
+            if self.need_restart:
+                # Check if all installed mtail rules compile or fail early
+                # --one_shot to exit, --port 0 to not clash with running mtail.
+                server.shell(
+                    name="Validate mtail programs",
+                    commands=[
+                        "timeout 30 /usr/local/bin/mtail --compile_only --one_shot"
+                        " --progs /etc/mtail --logs /dev/null"
+                        " --address 127.0.0.1 --port 0"
+                    ],
+                )
 
     def activate(self):
         active = bool(self.mtail_address)

cmdeploy/src/cmdeploy/mtail/filtermail.mtail

@@ -0,0 +1,101 @@
+# filtermail.mtail: filtermail process event counters
+
+#
+# Counters
+#
+
+# Connection errors in the transport path
+# Error::Io / Error::ConnectionFailed / Error::Tls: the reason string filtermail strips
+counter filtermail_transport_error_total by reason
+
+# Unexpected transport errors: DNS failure, HTTP error, config error.
+counter filtermail_transport_unexpected_total
+
+# Silent drops: inbound
+# "unencrypted" overlaps with rejected_unencrypted_mail_count in delivered_mail.mtail.
+counter filtermail_inbound_drop_total by reason
+
+# Silent drops: outbound
+# "unencrypted" overlaps with rejected_unencrypted_mail_count in delivered_mail.mtail;
+# "sender_disabled" and "all_recipients_disabled" are new.
+counter filtermail_outbound_drop_total by reason
+
+# Reinject failures
+counter filtermail_reinject_error_total by direction
+
+# SMTP-level connection errors (client dropped mid-session)
+counter filtermail_smtp_error_total by reason
+
+#
+# filtermail::transport
+#
+
+/filtermail\[\d+\]: \[WARN\s+filtermail::transport::worker\] Connection error relaying to mail server \S+: / {
+    /timed out/ {
+        filtermail_transport_error_total["connection_timeout"]++
+    } otherwise {
+    /Failed to connect to any of the following addresses/ {
+        filtermail_transport_error_total["connection_refused"]++
+    } otherwise {
+    /Connection refused/ {
+        filtermail_transport_error_total["connection_refused"]++
+    } otherwise {
+    /invalid certificate/ {
+        filtermail_transport_error_total["tls_cert_not_verified"]++
+    } otherwise {
+    /peer sent fatal alert|handshake/ {
+        filtermail_transport_error_total["tls_handshake"]++
+    } otherwise {
+    /close_notify|connection closed/ {
+        filtermail_transport_error_total["lost_connection"]++
+    } otherwise {
+        filtermail_transport_error_total["other"]++
+    }}}}}}}
+
+/filtermail\[\d+\]: \[WARN\s+filtermail::transport::worker\] Unexpected error while delivering/ {
+    filtermail_transport_unexpected_total++
+}
+
+#
+# filtermail::smtp_server
+#
+
+/filtermail(?:-incoming)?\[\d+\]: \[WARN\s+filtermail::smtp_server\] Unexpected EoF while receiving DATA/ {
+    filtermail_smtp_error_total["unexpected_eof"]++
+}
+
+/filtermail(?:-incoming)?\[\d+\]: \[WARN\s+filtermail::smtp_server\] Malformed DATA line without CRLF/ {
+    filtermail_smtp_error_total["malformed_data_line"]++
+}
+
+#
+# filtermail::outbound
+#
+
+/filtermail\[\d+\]: \[WARN\s+filtermail::outbound\] Rejected unencrypted mail/ {
+    filtermail_outbound_drop_total["unencrypted"]++
+}
+
+/filtermail\[\d+\]: \[WARN\s+filtermail::outbound\] Dropping mail; Sender .* is disabled/ {
+    filtermail_outbound_drop_total["sender_disabled"]++
+}
+
+/filtermail\[\d+\]: \[WARN\s+filtermail::outbound\] Dropping mail; All recipients disabled/ {
+    filtermail_outbound_drop_total["all_recipients_disabled"]++
+}
+
+/filtermail\[\d+\]: \[WARN\s+filtermail::outbound\] Failed to re.inject mail/ {
+    filtermail_reinject_error_total["outbound"]++
+}
+
+#
+# filtermail::inbound
+#
+
+/filtermail(?:-incoming)?\[\d+\]: \[WARN\s+filtermail::inbound\] Rejected unencrypted mail/ {
+    filtermail_inbound_drop_total["unencrypted"]++
+}
+
+/filtermail(?:-incoming)?\[\d+\]: \[WARN\s+filtermail::inbound\] Failed to re.inject mail/ {
+    filtermail_reinject_error_total["inbound"]++
+}

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -51,16 +51,6 @@ class TestMetadataTokens:
         assert res == b"1111 2222"
         assert b"Getmetadata completed" in client.readline()
 
-    def test_get_appversions(self, imap_mailbox):
-        "get app version information shipped with the relay"
-        client = imap_mailbox.client
-        client.send(b'a01 GETMETADATA "" /shared/vendor/deltachat/appversions\n')
-        res = client.readline()
-        assert res[:1] == b"*"
-        res = client.readline().strip().rstrip(b")")
-        assert b'"clients":' in res
-        assert b"Getmetadata completed" in client.readline()
-
 
 class TestEndToEndDeltaChat:
     "Tests that use Delta Chat accounts on the chat mail instance."

cmdeploy/src/cmdeploy/tests/test_appversions.py

@@ -1,20 +0,0 @@
-import json
-
-from cmdeploy.www import get_reporoot
-
-ALLOWED_URL_PREFIXES = (
-    "https://github.com/deltachat/",
-    "https://download.delta.chat/",
-)
-
-
-def test_appversions_schema():
-    data = json.loads(get_reporoot().joinpath("APPVERSIONS.json").read_text())
-    assert data["clients"]
-    for client in data["clients"]:
-        assert isinstance(client["clientId"], str)
-        for source in client["sources"]:
-            assert isinstance(source["sourceId"], str)
-            assert isinstance(source["versionInteger"], int)
-            assert isinstance(source["versionString"], str)
-            assert source["downloadUrl"].startswith(ALLOWED_URL_PREFIXES)

cmdeploy/src/cmdeploy/www.py

@@ -35,12 +35,8 @@ def prepare_template(source):
     return render_vars, page_layout
 
 
-def get_reporoot() -> Path:
-    return (Path(__file__).resolve() / "../../../../").resolve()
-
-
 def get_paths(config) -> (Path, Path, Path):
-    reporoot = get_reporoot()
+    reporoot = (Path(__file__).resolve() / "../../../../").resolve()
     www_path = Path(config.www_folder)
     # if www_folder was not set, use default directory
     if config.www_folder == "":

doc/source/overview.rst

@@ -249,28 +249,6 @@ Fresh chatmail addresses have a mailbox directory that contains:
    directories will typically be empty unless the user of that address
    hasn’t been online for a while.
 
-App version information (experimental)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-A chatmail relay ships the repository's
-`APPVERSIONS.json <https://github.com/chatmail/relay/blob/main/APPVERSIONS.json>`_
-and serves its content under the IMAP METADATA key
-``/shared/vendor/deltachat/appversions``.
-Chat apps installed outside of app stores read this key
-to learn about updates and where to download them.
-The mechanism is experimental and may change.
-
-The file travels with the normal deploy:
-update the repository checkout and run ``cmdeploy run``.
-Local modifications of ``APPVERSIONS.json`` are deployed as-is,
-so you can serve your own app version information,
-including links to app downloads.
-
-Note that as of August 2026, only Delta Chat Android is beginning
-to support discovering app versions from relays.
-Consumers of relay-provided app version information
-need to verify themselves that downloaded app files are valid.
-
 Active ports
 ~~~~~~~~~~~~