Setup mtail (#388)

Co-authored-by: holger krekel <holger@merlinux.eu>

CHANGELOG.md

@@ -2,6 +2,16 @@
 
 ## untagged
 
+- add mtail support (new optional `mail_address` ini value)
+  This defines the address on which [`mtail`](https://google.github.io/mtail/)
+  exposes its metrics collected from the logs.
+  If you want to collect the metrics with Prometheus,
+  setup a private network (e.g. WireGuard interface)
+  and assign an IP address from this network to the host.
+  If you do not plan to collect metrics,
+  keep this setting unset.
+  ([#388](https://github.com/deltachat/chatmail/pull/388))
+
 - fix checking for required DNS records
   ([#412](https://github.com/deltachat/chatmail/pull/412))
 

chatmaild/src/chatmaild/config.py

@@ -30,6 +30,7 @@ class Config:
         self.passthrough_recipients = params["passthrough_recipients"].split()
         self.filtermail_smtp_port = int(params["filtermail_smtp_port"])
         self.postfix_reinject_port = int(params["postfix_reinject_port"])
+        self.mtail_address = params.get("mtail_address")
         self.disable_ipv6 = params.get("disable_ipv6", "false").lower() == "true"
         self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true"
         self.iroh_relay = params.get("iroh_relay")

chatmaild/src/chatmaild/filtermail.py

@@ -183,15 +183,29 @@ class BeforeQueueHandler:
         mail_encrypted = check_encrypted(message)
 
         _, from_addr = parseaddr(message.get("from").strip())
+        envelope_from_domain = from_addr.split("@").pop()
+
         logging.info(f"mime-from: {from_addr} envelope-from: {envelope.mail_from!r}")
         if envelope.mail_from.lower() != from_addr.lower():
             return f"500 Invalid FROM <{from_addr!r}> for <{envelope.mail_from!r}>"
 
+        if mail_encrypted:
+            print("Filtering encrypted mail.", file=sys.stderr)
+        else:
+            print("Filtering unencrypted mail.", file=sys.stderr)
+
         if envelope.mail_from in self.config.passthrough_senders:
             return
 
         passthrough_recipients = self.config.passthrough_recipients
-        envelope_from_domain = from_addr.split("@").pop()
+
+        is_securejoin = message.get("secure-join") in [
+            "vc-request",
+            "vg-request",
+        ]
+        if is_securejoin:
+            return
+
         for recipient in envelope.rcpt_tos:
             if envelope.mail_from == recipient:
                 # Always allow sending emails to self.
@@ -205,12 +219,8 @@ class BeforeQueueHandler:
 
             is_outgoing = recipient_domain != envelope_from_domain
             if is_outgoing and not mail_encrypted:
-                is_securejoin = message.get("secure-join") in [
-                    "vc-request",
-                    "vg-request",
-                ]
-                if not is_securejoin:
-                    return f"500 Invalid unencrypted mail to <{recipient}>"
+                print("Rejected unencrypted mail.", file=sys.stderr)
+                return f"500 Invalid unencrypted mail to <{recipient}>"
 
 
 class SendRateLimiter:

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -55,6 +55,22 @@ postfix_reinject_port = 10025
 # if set to "True" IPv6 is disabled
 disable_ipv6 = False
 
+# Address on which `mtail` listens,
+# e.g. 127.0.0.1 or some private network
+# address like 192.168.10.1.
+# You can point Prometheus
+# or some other OpenMetrics-compatible
+# collector to
+# http://{{mtail_address}}:3903/metrics
+# and display collected metrics with Grafana.
+#
+# WARNING: do not expose this service
+# to the public IP address.
+#
+# `mtail is not running if the setting is not set.
+
+# mtail_address = 127.0.0.1
+
 #
 # Debugging options 
 #

cmdeploy/src/cmdeploy/__init__.py

@@ -441,6 +441,44 @@ def check_config(config):
     return config
 
 
+def deploy_mtail(config):
+    apt.packages(
+        name="Install mtail",
+        packages=["mtail"],
+    )
+
+    # Using our own systemd unit instead of `/usr/lib/systemd/system/mtail.service`.
+    # This allows to read from journalctl instead of log files.
+    files.template(
+        src=importlib.resources.files(__package__).joinpath("mtail/mtail.service.j2"),
+        dest="/etc/systemd/system/mtail.service",
+        user="root",
+        group="root",
+        mode="644",
+        address=config.mtail_address or "127.0.0.1",
+        port=3903,
+    )
+
+    mtail_conf = files.put(
+        name="Mtail configuration",
+        src=importlib.resources.files(__package__).joinpath(
+            "mtail/delivered_mail.mtail"
+        ),
+        dest="/etc/mtail/delivered_mail.mtail",
+        user="root",
+        group="root",
+        mode="644",
+    )
+
+    systemd.service(
+        name="Start and enable mtail",
+        service="mtail.service",
+        running=bool(config.mtail_address),
+        enabled=bool(config.mtail_address),
+        restarted=mtail_conf.changed,
+    )
+
+
 def deploy_chatmail(config_path: Path) -> None:
     """Deploy a chat-mail instance.
 
@@ -636,3 +674,5 @@ def deploy_chatmail(config_path: Path) -> None:
         name="Ensure cron is installed",
         packages=["cron"],
     )
+
+    deploy_mtail(config)

cmdeploy/src/cmdeploy/mtail/delivered_mail.mtail

@@ -0,0 +1,64 @@
+counter delivered_mail
+/saved mail to INBOX$/ {
+  delivered_mail++
+}
+
+counter quota_exceeded
+/Quota exceeded \(mailbox for user is full\)$/ {
+  quota_exceeded++
+}
+
+# Essentially the number of outgoing messages.
+counter dkim_signed
+/DKIM-Signature field added/ {
+  dkim_signed++
+}
+
+counter created_accounts
+counter created_ci_accounts
+counter created_nonci_accounts
+
+/: Created address: (?P<addr>.*)$/ {
+  created_accounts++
+
+  $addr =~ /ci-/ {
+    created_ci_accounts++
+  } else {
+    created_nonci_accounts++
+  }
+}
+
+counter postfix_timeouts
+/timeout after DATA/ {
+  postfix_timeouts++
+}
+
+counter postfix_noqueue
+/postfix\/.*NOQUEUE/ {
+  postfix_noqueue++
+}
+
+counter warning_count
+/warning/ {
+  warning_count++
+}
+
+
+counter filtered_mail_count
+
+counter encrypted_mail_count
+/Filtering encrypted mail\./ {
+  encrypted_mail_count++
+  filtered_mail_count++
+}
+
+counter unencrypted_mail_count
+/Filtering unencrypted mail\./ {
+  unencrypted_mail_count++
+  filtered_mail_count++
+}
+
+counter rejected_unencrypted_mail_count
+/Rejected unencrypted mail\./ {
+  rejected_unencrypted_mail_count++
+}

cmdeploy/src/cmdeploy/mtail/mtail.service.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=mtail
+
+[Service]
+Type=simple
+ExecStart=/bin/sh -c "journalctl -f -o short-iso -n 0 | /usr/bin/mtail --address={{ address }} --port={{ port }} --progs /etc/mtail --logtostderr --logs -"
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target