Also preserve the Chat-Is-Post-Message header

This will remove all headers possible during LMTP delivery.

From header: required or clients do not process the message correctly

Message-Id header: required for clients to know which messages have been
downloaded

.github/workflows/test-and-deploy-ipv4only.yaml

@@ -74,7 +74,6 @@ jobs:
       - run: |
           cmdeploy init staging-ipv4.testrun.org
           sed -i 's#disable_ipv6 = False#disable_ipv6 = True#' chatmail.ini
-          sed -i 's/#\s*mtail_address/mtail_address/' chatmail.ini
 
       - run: cmdeploy run --verbose --skip-dns-check
 

.github/workflows/test-and-deploy.yaml

@@ -74,9 +74,7 @@ jobs:
       - name: run deploy-chatmail offline tests 
         run: pytest --pyargs cmdeploy 
 
-      - run: |
-          cmdeploy init staging2.testrun.org
-          sed -i 's/#\s*mtail_address/mtail_address/' chatmail.ini
+      - run: cmdeploy init staging2.testrun.org
 
       - run: cmdeploy run --verbose --skip-dns-check
 

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -37,7 +37,9 @@ class DovecotDeployer(Deployer):
         restart = False if self.disable_mail else self.need_restart
 
         systemd.service(
-            name="Disable dovecot for now" if self.disable_mail else "Start and enable Dovecot",
+            name="disable dovecot for now"
+            if self.disable_mail
+            else "Start and enable Dovecot",
             service="dovecot.service",
             running=False if self.disable_mail else True,
             enabled=False if self.disable_mail else True,
@@ -143,11 +145,4 @@ def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
     )
     daemon_reload |= restart_conf.changed
 
-    # Validate dovecot configuration before restart
-    if need_restart:
-        server.shell(
-            name="Validate dovecot configuration",
-            commands=["doveconf -n >/dev/null"],
-        )
-
     return need_restart, daemon_reload

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -168,8 +168,6 @@ service lmtp {
   }
 }
 
-lmtp_add_received_header = no
-
 service auth {
   unix_listener /var/spool/postfix/private/auth {
     mode = 0660
@@ -315,8 +313,7 @@ metric imap_command_slow {
   group_by = cmd_name
 }
 
-# IDLE-specific Metrics
-
+# IDLE-specific metrics
 metric imap_idle {
   filter = event=imap_command_finished AND cmd_name=IDLE
   fields = bytes_in bytes_out running_usecs
@@ -329,19 +326,11 @@ metric imap_idle_duration {
   group_by = duration:exponential:5:11:10
 }
 
-metric imap_idle_commands {
-  filter = event=imap_command_finished AND cmd_name=IDLE
-  group_by = tagged_reply_state
-}
-
-metric imap_idle_failed {
-  filter = event=imap_command_finished AND cmd_name=IDLE AND NOT tagged_reply_state=OK
-}
-
-# Hibernation Metrics (requires imap_hibernate_timeout)
-
+# Hibernation Metrics (requires imap_hibernate_timeout to be set)
 metric imap_hibernated {
   filter = event=imap_client_hibernated
+  # error field present = failure
+  group_by = mailbox
 }
 
 metric imap_hibernated_failed {
@@ -351,24 +340,14 @@ metric imap_hibernated_failed {
 metric imap_unhibernated {
   filter = event=imap_client_unhibernated
   fields = hibernation_usecs
-}
-
-metric imap_unhibernated_reason {
-  filter = event=imap_client_unhibernated
   group_by = reason
-  fields = hibernation_usecs
-}
-
-metric imap_unhibernated_reason_sleep {
-  filter = event=imap_client_unhibernated
-  group_by = reason hibernation_usecs:exponential:4:8:10
 }
 
 metric imap_unhibernated_failed {
   filter = event=imap_client_unhibernated AND error=*
 }
 
-# Hibernation duration buckets (how long clients stayed hibernated)
+# Hibernation duration buckets (how long clients actually stayed hibernated)
 # Base 10: 100ms to 27h
 metric imap_hibernation_duration {
   filter = event=imap_client_unhibernated
@@ -376,7 +355,6 @@ metric imap_hibernation_duration {
 }
 
 # Authentication / Login Metrics
-
 metric auth_request {
   filter = event=auth_request_finished
   group_by = success
@@ -407,7 +385,6 @@ metric auth_master_login_failed {
 }
 
 # Mail Delivery (LMTP) - affects IDLE wakeup latency
-
 metric mail_delivery {
   filter = event=mail_delivery_finished
 }
@@ -422,13 +399,12 @@ metric mail_delivery_failed {
 }
 
 # Connection Events
-
 metric client_connected {
-  filter = event=client_connection_connected AND category="service:imap"
+  filter = event=client_connection_connected AND category=service:imap
 }
 
 metric client_disconnected {
-  filter = event=client_connection_disconnected AND category="service:imap"
+  filter = event=client_connection_disconnected AND category=service:imap
   fields = bytes_in bytes_out
 }
 {% endif %}

cmdeploy/src/cmdeploy/mtail/mtail.service.j2

@@ -3,7 +3,7 @@ Description=mtail
 
 [Service]
 Type=simple
-ExecStart=/bin/sh -c "journalctl -f -o short-iso -n 0 | /usr/local/bin/mtail --address={{ address }} --port={{ port }} --progs /etc/mtail --logtostderr --logs /dev/stdin"
+ExecStart=/bin/sh -c "journalctl -f -o short-iso -n 0 | /usr/local/bin/mtail --address={{ address }} --port={{ port }} --progs /etc/mtail --logtostderr --logs -"
 Restart=on-failure
 
 [Install]

cmdeploy/src/cmdeploy/postfix/deployer.py

@@ -1,4 +1,4 @@
-from pyinfra.operations import apt, files, server, systemd
+from pyinfra.operations import apt, files, systemd
 
 from cmdeploy.basedeploy import Deployer, get_resource
 
@@ -77,14 +77,6 @@ class PostfixDeployer(Deployer):
             dest="/etc/systemd/system/postfix@.service.d/10_restart.conf",
         )
         self.daemon_reload = restart_conf.changed
-
-        # Validate postfix configuration before restart
-        if need_restart:
-            server.shell(
-                name="Validate postfix configuration",
-                # Extract stderr and quit with error if non-zero
-                commands=["""bash -c 'w=$(postconf 2>&1 >/dev/null); [[ -z "$w" ]] || { echo "$w"; false; }'"""],
-            )
         self.need_restart = need_restart
 
     def activate(self):

cmdeploy/src/cmdeploy/postfix/lmtp_header_cleanup

@@ -1,2 +1,4 @@
-/^DKIM-Signature:/            IGNORE
-/^Authentication-Results:/            IGNORE
+/^From:/            DUNNO
+/^Message-Id:/      DUNNO
+/^Chat-Is-Post-Message:/      DUNNO
+/.*/                IGNORE