feat(mtail): validate programs during deploy

Compile /etc/mtail before activate() restarts the service, to catch errors
early.

cmdeploy/src/cmdeploy/mtail/deployer.py

@@ -1,5 +1,5 @@
 from pyinfra import facts, host
-from pyinfra.operations import apt
+from pyinfra.operations import apt, server
 
 from cmdeploy.basedeploy import Deployer
 
@@ -37,7 +37,20 @@ class MtailDeployer(Deployer):
             address=self.mtail_address or "127.0.0.1",
             port=3903,
         )
-        self.put_file("mtail/delivered_mail.mtail", "/etc/mtail/delivered_mail.mtail")
+        if self.mtail_address:
+            self.put_file("mtail/delivered_mail.mtail", "/etc/mtail/delivered_mail.mtail")
+            self.put_file("mtail/filtermail.mtail", "/etc/mtail/filtermail.mtail")
+            if self.need_restart:
+                # Check if all installed mtail rules compile or fail early
+                # --one_shot to exit, --port 0 to not clash with running mtail.
+                server.shell(
+                    name="Validate mtail programs",
+                    commands=[
+                        "timeout 30 /usr/local/bin/mtail --compile_only --one_shot"
+                        " --progs /etc/mtail --logs /dev/null"
+                        " --address 127.0.0.1 --port 0"
+                    ],
+                )
 
     def activate(self):
         active = bool(self.mtail_address)

cmdeploy/src/cmdeploy/mtail/filtermail.mtail

@@ -0,0 +1,101 @@
+# filtermail.mtail: filtermail process event counters
+
+#
+# Counters
+#
+
+# Connection errors in the transport path
+# Error::Io / Error::ConnectionFailed / Error::Tls: the reason string filtermail strips
+counter filtermail_transport_error_total by reason
+
+# Unexpected transport errors: DNS failure, HTTP error, config error.
+counter filtermail_transport_unexpected_total
+
+# Silent drops: inbound
+# "unencrypted" overlaps with rejected_unencrypted_mail_count in delivered_mail.mtail.
+counter filtermail_inbound_drop_total by reason
+
+# Silent drops: outbound
+# "unencrypted" overlaps with rejected_unencrypted_mail_count in delivered_mail.mtail;
+# "sender_disabled" and "all_recipients_disabled" are new.
+counter filtermail_outbound_drop_total by reason
+
+# Reinject failures
+counter filtermail_reinject_error_total by direction
+
+# SMTP-level connection errors (client dropped mid-session)
+counter filtermail_smtp_error_total by reason
+
+#
+# filtermail::transport
+#
+
+/filtermail\[\d+\]: \[WARN\s+filtermail::transport::worker\] Connection error relaying to mail server \S+: / {
+    /timed out/ {
+        filtermail_transport_error_total["connection_timeout"]++
+    } otherwise {
+    /Failed to connect to any of the following addresses/ {
+        filtermail_transport_error_total["connection_refused"]++
+    } otherwise {
+    /Connection refused/ {
+        filtermail_transport_error_total["connection_refused"]++
+    } otherwise {
+    /invalid certificate/ {
+        filtermail_transport_error_total["tls_cert_not_verified"]++
+    } otherwise {
+    /peer sent fatal alert|handshake/ {
+        filtermail_transport_error_total["tls_handshake"]++
+    } otherwise {
+    /close_notify|connection closed/ {
+        filtermail_transport_error_total["lost_connection"]++
+    } otherwise {
+        filtermail_transport_error_total["other"]++
+    }}}}}}}
+
+/filtermail\[\d+\]: \[WARN\s+filtermail::transport::worker\] Unexpected error while delivering/ {
+    filtermail_transport_unexpected_total++
+}
+
+#
+# filtermail::smtp_server
+#
+
+/filtermail(?:-incoming)?\[\d+\]: \[WARN\s+filtermail::smtp_server\] Unexpected EoF while receiving DATA/ {
+    filtermail_smtp_error_total["unexpected_eof"]++
+}
+
+/filtermail(?:-incoming)?\[\d+\]: \[WARN\s+filtermail::smtp_server\] Malformed DATA line without CRLF/ {
+    filtermail_smtp_error_total["malformed_data_line"]++
+}
+
+#
+# filtermail::outbound
+#
+
+/filtermail\[\d+\]: \[WARN\s+filtermail::outbound\] Rejected unencrypted mail/ {
+    filtermail_outbound_drop_total["unencrypted"]++
+}
+
+/filtermail\[\d+\]: \[WARN\s+filtermail::outbound\] Dropping mail; Sender .* is disabled/ {
+    filtermail_outbound_drop_total["sender_disabled"]++
+}
+
+/filtermail\[\d+\]: \[WARN\s+filtermail::outbound\] Dropping mail; All recipients disabled/ {
+    filtermail_outbound_drop_total["all_recipients_disabled"]++
+}
+
+/filtermail\[\d+\]: \[WARN\s+filtermail::outbound\] Failed to re.inject mail/ {
+    filtermail_reinject_error_total["outbound"]++
+}
+
+#
+# filtermail::inbound
+#
+
+/filtermail(?:-incoming)?\[\d+\]: \[WARN\s+filtermail::inbound\] Rejected unencrypted mail/ {
+    filtermail_inbound_drop_total["unencrypted"]++
+}
+
+/filtermail(?:-incoming)?\[\d+\]: \[WARN\s+filtermail::inbound\] Failed to re.inject mail/ {
+    filtermail_reinject_error_total["inbound"]++
+}