feat: serve an APPVERSIONS.json index file to clients via IMAP metadata

This is designed to help implement self-updating APKs (and later other clients),
see counterpart https://github.com/chatmail/core/pull/8557

APPVERSIONS.json

@@ -0,0 +1,15 @@
+{
+  "clients": [
+    {
+      "clientId": "deltachat",
+      "sources": [
+        {
+          "sourceId": "gplay",
+          "versionInteger": 754,
+          "versionString": "2.57.0",
+          "downloadUrl": "https://github.com/deltachat/deltachat-android/releases/download/v2.57.0/deltachat-gplay-release-2.57.0.apk"
+        }
+      ]
+    }
+  ]
+}

CONTRIBUTING.md

@@ -5,3 +5,6 @@ We use [git-cliff] to generate the changelog from commit messages before the rel
 
 [Conventional Commits]: https://www.conventionalcommits.org/
 [git-cliff]: https://git-cliff.org/
+
+To update client app version information,
+edit [APPVERSIONS.json](APPVERSIONS.json).

chatmaild/src/chatmaild/config.py

@@ -63,6 +63,9 @@ class Config:
         self.turn_socket_path = params.pop(
             "turn_socket_path", "/run/chatmail-turn/turn.socket"
         )
+        self.appversions_path = Path(
+            params.pop("appversions_path", "/usr/local/lib/chatmaild/appversions.json")
+        )
         iroh_relay = params.pop("iroh_relay", None)
         if iroh_relay is None:
             self.iroh_relay = "https://" + raw_domain

chatmaild/src/chatmaild/metadata.py

@@ -1,3 +1,4 @@
+import json
 import logging
 import socket
 import sys
@@ -18,6 +19,18 @@ def turn_credentials(turn_socket_path):
             return file.readline().decode("utf-8").strip()
 
 
+def read_appversions(path):
+    try:
+        data = json.loads(path.read_bytes())
+    except FileNotFoundError:
+        return None
+    except (OSError, ValueError):
+        logging.exception(f"failed to read {path}")
+        return None
+    # the dict protocol is line-based, keep the value single-line
+    return json.dumps(data, separators=(",", ":"))
+
+
 def _is_valid_token_timestamp(timestamp, now):
     # Token if invalid after 90 days
     # or if the timestamp is in the future.
@@ -94,6 +107,7 @@ class MetadataDictProxy(DictProxy):
         iroh_relay=None,
         turn_hostname=None,
         turn_socket_path=None,
+        appversions_path=None,
     ):
         super().__init__()
         self.notifier = notifier
@@ -101,6 +115,7 @@ class MetadataDictProxy(DictProxy):
         self.iroh_relay = iroh_relay
         self.turn_hostname = turn_hostname
         self.turn_socket_path = turn_socket_path
+        self.appversions_path = appversions_path
 
     def handle_lookup(self, parts):
         # Lpriv/43f5f508a7ea0366dff30200c15250e3/devicetoken\tlkj123poi@c2.testrun.org
@@ -125,6 +140,9 @@ class MetadataDictProxy(DictProxy):
                         case "maxsmtprecipients":
                             # postfix default  (see "postconf smtpd_recipient_limit")
                             return "O1000\n"
+                        case "appversions" if self.appversions_path:
+                            value = read_appversions(self.appversions_path)
+                            return f"O{value}\n" if value else "N\n"
 
         logging.warning(f"lookup ignored: {parts!r}")
         return "N\n"
@@ -170,6 +188,7 @@ def main():
         iroh_relay=iroh_relay,
         turn_hostname=mail_domain,
         turn_socket_path=socket_path,
+        appversions_path=config.appversions_path,
     )
 
     dictproxy.serve_forever_from_socket(socket)

chatmaild/src/chatmaild/tests/test_config.py

@@ -47,6 +47,9 @@ def test_read_config_basic_using_defaults(tmp_path, maildomain):
     assert example_config.password_min_length == 9
     assert example_config.max_imap_connections == 10000
     assert example_config.max_smtp_connections == 1000
+    assert str(example_config.appversions_path) == (
+        "/usr/local/lib/chatmaild/appversions.json"
+    )
     assert example_config._unused_keys == []
 
 

chatmaild/src/chatmaild/tests/test_metadata.py

@@ -7,6 +7,7 @@ import requests
 from chatmaild.metadata import (
     Metadata,
     MetadataDictProxy,
+    read_appversions,
 )
 from chatmaild.notifier import (
     Notifier,
@@ -369,6 +370,17 @@ def test_iroh_relay(dictproxy):
     assert wfile.getvalue() == b"Ohttps://example.org/\n"
 
 
+def test_read_appversions(tmp_path):
+    path = tmp_path.joinpath("appversions.json")
+    assert read_appversions(path) is None
+
+    path.write_text('{\n  "clients": []\n}')
+    assert read_appversions(path) == '{"clients":[]}'
+
+    path.write_text("bad json")
+    assert read_appversions(path) is None
+
+
 def test_legacy_token_migration(metadata, testaddr):
     with metadata.get_metadata_dict(testaddr).modify() as data:
         data[metadata.DEVICETOKEN_KEY] = ["oldtoken1", "oldtoken2"]

cmdeploy/src/cmdeploy/deployers.py

@@ -35,7 +35,7 @@ from .nginx.deployer import NginxDeployer
 from .opendkim.deployer import OpendkimDeployer
 from .postfix.deployer import PostfixDeployer
 from .selfsigned.deployer import SelfSignedTlsDeployer
-from .www import build_webpages, find_merge_conflict, get_paths
+from .www import build_webpages, find_merge_conflict, get_paths, get_reporoot
 
 
 class Port(FactBase):
@@ -126,6 +126,11 @@ def _configure_remote_venv_with_chatmaild(deployer, config) -> None:
         dest=remote_chatmail_inipath,
     )
 
+    deployer.put_file(
+        src=get_reporoot().joinpath("APPVERSIONS.json").open("rb"),
+        dest=str(config.appversions_path),
+    )
+
     deployer.remove_file("/etc/cron.d/chatmail-metrics")
     deployer.remove_file("/var/www/html/metrics")
 

cmdeploy/src/cmdeploy/mtail/deployer.py

@@ -1,5 +1,5 @@
 from pyinfra import facts, host
-from pyinfra.operations import apt, server
+from pyinfra.operations import apt
 
 from cmdeploy.basedeploy import Deployer
 
@@ -37,20 +37,7 @@ class MtailDeployer(Deployer):
             address=self.mtail_address or "127.0.0.1",
             port=3903,
         )
-        if self.mtail_address:
-            self.put_file("mtail/delivered_mail.mtail", "/etc/mtail/delivered_mail.mtail")
-            self.put_file("mtail/filtermail.mtail", "/etc/mtail/filtermail.mtail")
-            if self.need_restart:
-                # Check if all installed mtail rules compile or fail early
-                # --one_shot to exit, --port 0 to not clash with running mtail.
-                server.shell(
-                    name="Validate mtail programs",
-                    commands=[
-                        "timeout 30 /usr/local/bin/mtail --compile_only --one_shot"
-                        " --progs /etc/mtail --logs /dev/null"
-                        " --address 127.0.0.1 --port 0"
-                    ],
-                )
+        self.put_file("mtail/delivered_mail.mtail", "/etc/mtail/delivered_mail.mtail")
 
     def activate(self):
         active = bool(self.mtail_address)

cmdeploy/src/cmdeploy/mtail/filtermail.mtail

@@ -1,101 +0,0 @@
-# filtermail.mtail: filtermail process event counters
-
-#
-# Counters
-#
-
-# Connection errors in the transport path
-# Error::Io / Error::ConnectionFailed / Error::Tls: the reason string filtermail strips
-counter filtermail_transport_error_total by reason
-
-# Unexpected transport errors: DNS failure, HTTP error, config error.
-counter filtermail_transport_unexpected_total
-
-# Silent drops: inbound
-# "unencrypted" overlaps with rejected_unencrypted_mail_count in delivered_mail.mtail.
-counter filtermail_inbound_drop_total by reason
-
-# Silent drops: outbound
-# "unencrypted" overlaps with rejected_unencrypted_mail_count in delivered_mail.mtail;
-# "sender_disabled" and "all_recipients_disabled" are new.
-counter filtermail_outbound_drop_total by reason
-
-# Reinject failures
-counter filtermail_reinject_error_total by direction
-
-# SMTP-level connection errors (client dropped mid-session)
-counter filtermail_smtp_error_total by reason
-
-#
-# filtermail::transport
-#
-
-/filtermail\[\d+\]: \[WARN\s+filtermail::transport::worker\] Connection error relaying to mail server \S+: / {
-    /timed out/ {
-        filtermail_transport_error_total["connection_timeout"]++
-    } otherwise {
-    /Failed to connect to any of the following addresses/ {
-        filtermail_transport_error_total["connection_refused"]++
-    } otherwise {
-    /Connection refused/ {
-        filtermail_transport_error_total["connection_refused"]++
-    } otherwise {
-    /invalid certificate/ {
-        filtermail_transport_error_total["tls_cert_not_verified"]++
-    } otherwise {
-    /peer sent fatal alert|handshake/ {
-        filtermail_transport_error_total["tls_handshake"]++
-    } otherwise {
-    /close_notify|connection closed/ {
-        filtermail_transport_error_total["lost_connection"]++
-    } otherwise {
-        filtermail_transport_error_total["other"]++
-    }}}}}}}
-
-/filtermail\[\d+\]: \[WARN\s+filtermail::transport::worker\] Unexpected error while delivering/ {
-    filtermail_transport_unexpected_total++
-}
-
-#
-# filtermail::smtp_server
-#
-
-/filtermail(?:-incoming)?\[\d+\]: \[WARN\s+filtermail::smtp_server\] Unexpected EoF while receiving DATA/ {
-    filtermail_smtp_error_total["unexpected_eof"]++
-}
-
-/filtermail(?:-incoming)?\[\d+\]: \[WARN\s+filtermail::smtp_server\] Malformed DATA line without CRLF/ {
-    filtermail_smtp_error_total["malformed_data_line"]++
-}
-
-#
-# filtermail::outbound
-#
-
-/filtermail\[\d+\]: \[WARN\s+filtermail::outbound\] Rejected unencrypted mail/ {
-    filtermail_outbound_drop_total["unencrypted"]++
-}
-
-/filtermail\[\d+\]: \[WARN\s+filtermail::outbound\] Dropping mail; Sender .* is disabled/ {
-    filtermail_outbound_drop_total["sender_disabled"]++
-}
-
-/filtermail\[\d+\]: \[WARN\s+filtermail::outbound\] Dropping mail; All recipients disabled/ {
-    filtermail_outbound_drop_total["all_recipients_disabled"]++
-}
-
-/filtermail\[\d+\]: \[WARN\s+filtermail::outbound\] Failed to re.inject mail/ {
-    filtermail_reinject_error_total["outbound"]++
-}
-
-#
-# filtermail::inbound
-#
-
-/filtermail(?:-incoming)?\[\d+\]: \[WARN\s+filtermail::inbound\] Rejected unencrypted mail/ {
-    filtermail_inbound_drop_total["unencrypted"]++
-}
-
-/filtermail(?:-incoming)?\[\d+\]: \[WARN\s+filtermail::inbound\] Failed to re.inject mail/ {
-    filtermail_reinject_error_total["inbound"]++
-}

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -51,6 +51,16 @@ class TestMetadataTokens:
         assert res == b"1111 2222"
         assert b"Getmetadata completed" in client.readline()
 
+    def test_get_appversions(self, imap_mailbox):
+        "get app version information shipped with the relay"
+        client = imap_mailbox.client
+        client.send(b'a01 GETMETADATA "" /shared/vendor/deltachat/appversions\n')
+        res = client.readline()
+        assert res[:1] == b"*"
+        res = client.readline().strip().rstrip(b")")
+        assert b'"clients":' in res
+        assert b"Getmetadata completed" in client.readline()
+
 
 class TestEndToEndDeltaChat:
     "Tests that use Delta Chat accounts on the chat mail instance."

cmdeploy/src/cmdeploy/tests/test_appversions.py

@@ -0,0 +1,20 @@
+import json
+
+from cmdeploy.www import get_reporoot
+
+ALLOWED_URL_PREFIXES = (
+    "https://github.com/deltachat/",
+    "https://download.delta.chat/",
+)
+
+
+def test_appversions_schema():
+    data = json.loads(get_reporoot().joinpath("APPVERSIONS.json").read_text())
+    assert data["clients"]
+    for client in data["clients"]:
+        assert isinstance(client["clientId"], str)
+        for source in client["sources"]:
+            assert isinstance(source["sourceId"], str)
+            assert isinstance(source["versionInteger"], int)
+            assert isinstance(source["versionString"], str)
+            assert source["downloadUrl"].startswith(ALLOWED_URL_PREFIXES)

cmdeploy/src/cmdeploy/www.py

@@ -35,8 +35,12 @@ def prepare_template(source):
     return render_vars, page_layout
 
 
+def get_reporoot() -> Path:
+    return (Path(__file__).resolve() / "../../../../").resolve()
+
+
 def get_paths(config) -> (Path, Path, Path):
-    reporoot = (Path(__file__).resolve() / "../../../../").resolve()
+    reporoot = get_reporoot()
     www_path = Path(config.www_folder)
     # if www_folder was not set, use default directory
     if config.www_folder == "":

doc/source/overview.rst

@@ -249,6 +249,28 @@ Fresh chatmail addresses have a mailbox directory that contains:
    directories will typically be empty unless the user of that address
    hasn’t been online for a while.
 
+App version information (experimental)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A chatmail relay ships the repository's
+`APPVERSIONS.json <https://github.com/chatmail/relay/blob/main/APPVERSIONS.json>`_
+and serves its content under the IMAP METADATA key
+``/shared/vendor/deltachat/appversions``.
+Chat apps installed outside of app stores read this key
+to learn about updates and where to download them.
+The mechanism is experimental and may change.
+
+The file travels with the normal deploy:
+update the repository checkout and run ``cmdeploy run``.
+Local modifications of ``APPVERSIONS.json`` are deployed as-is,
+so you can serve your own app version information,
+including links to app downloads.
+
+Note that as of August 2026, only Delta Chat Android is beginning
+to support discovering app versions from relays.
+Consumers of relay-provided app version information
+need to verify themselves that downloaded app files are valid.
+
 Active ports
 ~~~~~~~~~~~~