opendkim/final.lua: also strip Authentication-Results Header

Refactor DKIM-Header-Pruning into a  "if valid-else" construct with a single "return nil" at the end

.github/workflows/test-and-deploy-ipv4only.yaml

@@ -19,13 +19,8 @@ jobs:
     environment:
       name: staging-ipv4.testrun.org
       url: https://staging-ipv4.testrun.org/
-    concurrency:
-      group: ci-ipv4-${{ github.workflow }}-${{ github.ref }}
-      cancel-in-progress: ${{ !contains(github.ref, '$GITHUB_REF') }}
+    concurrency: staging-ipv4.testrun.org
     steps:
-      - uses: jsok/serialize-workflow-action@515cd04c46d7ea7435c4a22a3b4419127afdefe9
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions/checkout@v4
 
       - name: prepare SSH

.github/workflows/test-and-deploy.yaml

@@ -19,13 +19,8 @@ jobs:
     environment:
       name: staging2.testrun.org
       url: https://staging2.testrun.org/
-    concurrency:
-      group: ci-${{ github.workflow }}-${{ github.ref }}
-      cancel-in-progress: ${{ !contains(github.ref, '$GITHUB_REF') }}
+    concurrency: staging2.testrun.org
     steps:
-      - uses: jsok/serialize-workflow-action@515cd04c46d7ea7435c4a22a3b4419127afdefe9
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions/checkout@v4
 
       - name: prepare SSH

chatmaild/src/chatmaild/filtermail.py

@@ -307,12 +307,9 @@ class IncomingBeforeQueueHandler:
             return error
         log_info("re-injecting the mail that passed checks")
 
-        # the smtp daemon on reinject_port_incoming gives it to dkim milter
-        # which looks at source address to determine whether to verify or sign
         client = SMTPClient(
             "localhost",
             self.config.postfix_reinject_port_incoming,
-            source_address=("127.0.0.2", 0),
         )
         client.sendmail(
             envelope.mail_from, envelope.rcpt_tos, envelope.original_content

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -15,7 +15,7 @@ mail_domain = {mail_domain}
 max_user_send_per_minute = 60
 
 # maximum mailbox size of a chatmail address
-max_mailbox_size = 100M
+max_mailbox_size = 500M
 
 # maximum message size for an e-mail in bytes
 max_message_size = 31457280

chatmaild/src/chatmaild/tests/test_config.py

@@ -33,7 +33,7 @@ def test_read_config_testrun(make_config):
     assert config.filtermail_smtp_port == 10080
     assert config.postfix_reinject_port == 10025
     assert config.max_user_send_per_minute == 60
-    assert config.max_mailbox_size == "100M"
+    assert config.max_mailbox_size == "500M"
     assert config.delete_mails_after == "20"
     assert config.delete_large_after == "7"
     assert config.username_min_length == 9

cmdeploy/src/cmdeploy/opendkim/final.lua

@@ -1,4 +1,5 @@
-if odkim.internal_ip(ctx) == 1 then
+mtaname = odkim.get_mtasymbol(ctx, "{daemon_name}")
+if mtaname == "ORIGINATING" then
 	-- Outgoing message will be signed,
 	-- no need to look for signatures.
 	return nil
@@ -33,6 +34,15 @@ if valid then
 	for i = nsigs, 1, -1 do
 		odkim.del_header(ctx, "DKIM-Signature", i)
 	end
+
+	-- Delete first and presumably only occurence
+        odkim.del_header(ctx, "Authentication-Results", 0)  
+else
+	odkim.set_reply(ctx, "554", "5.7.1", "No valid DKIM signature found")
+	-- Delete in reverse order to avoid index shifting.
+	for i = nsigs, 1, -1 do
+		odkim.del_header(ctx, "DKIM-Signature", i)
+	end
 else
 	odkim.set_reply(ctx, "554", "5.7.1", error_msg)
 	odkim.set_result(ctx, SMFIS_REJECT)

cmdeploy/src/cmdeploy/opendkim/opendkim.conf

@@ -65,3 +65,9 @@ PidFile			/run/opendkim/opendkim.pid
 # The trust anchor enables DNSSEC. In Debian, the trust anchor file is provided
 # by the package dns-root-data.
 TrustAnchorFile		/usr/share/dns/root.key
+
+# Sign messages when `-o milter_macro_daemon_name=ORIGINATING` is set.
+MTA ORIGINATING
+
+# No hosts are treated as internal, ORIGINATING daemon name should be set explicitly.
+InternalHosts -

cmdeploy/src/cmdeploy/postfix/master.cf.j2

@@ -31,7 +31,6 @@ submission inet n       -       y       -       5000    smtpd
   -o smtpd_sender_restrictions=$mua_sender_restrictions
   -o smtpd_recipient_restrictions=
   -o smtpd_relay_restrictions=permit_sasl_authenticated,reject
-  -o milter_macro_daemon_name=ORIGINATING
   -o smtpd_client_connection_count_limit=1000
   -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
 smtps     inet  n       -       y       -       5000    smtpd
@@ -49,7 +48,6 @@ smtps     inet  n       -       y       -       5000    smtpd
   -o smtpd_recipient_restrictions=
   -o smtpd_relay_restrictions=permit_sasl_authenticated,reject
   -o smtpd_client_connection_count_limit=1000
-  -o milter_macro_daemon_name=ORIGINATING
   -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
 #628       inet  n       -       y       -       -       qmqpd
 pickup    unix  n       -       y       60      1       pickup
@@ -81,6 +79,7 @@ filter    unix -        n       n       -       -       lmtp
 # Local SMTP server for reinjecting outgoing filtered mail.
 127.0.0.1:{{ config.postfix_reinject_port }} inet  n       -       n       -       100      smtpd
   -o syslog_name=postfix/reinject
+  -o milter_macro_daemon_name=ORIGINATING
   -o smtpd_milters=unix:opendkim/opendkim.sock
   -o cleanup_service_name=authclean
 

doc/source/getting_started.rst

@@ -16,15 +16,16 @@ You will need the following:
 
 -  Control over a domain through a DNS provider of your choice.
 
--  A Debian 12 server with reachable SMTP/SUBMISSIONS/IMAPS/HTTPS ports.
+-  A Debian 12 **deployment server** with reachable SMTP/SUBMISSIONS/IMAPS/HTTPS ports.
    IPv6 is encouraged if available. Chatmail relay servers only require
    1GB RAM, one CPU, and perhaps 10GB storage for a few thousand active
    chatmail addresses.
 
--  Key-based SSH authentication to the root user. You must add a
-   passphrase-protected private key to your local ssh-agent because you
-   can’t type in your passphrase during deployment. (An ed25519 private
-   key is required due to an `upstream bug in
+-  A Linux or Unix **build machine** with key-based SSH access to the root
+   user of the deployment server.
+   You must add a passphrase-protected private key to your local ssh-agent because you
+   can’t type in your passphrase during deployment.
+   (An ed25519 private key is required due to an `upstream bug in
    paramiko <https://github.com/paramiko/paramiko/issues/2191>`_)
 
 
@@ -34,16 +35,17 @@ Setup with ``scripts/cmdeploy``
 We use ``chat.example.org`` as the chatmail domain in the following
 steps. Please substitute it with your own domain.
 
-1. Setup the initial DNS records. The following is an example in the
+1. Setup the initial DNS records for your deployment server.
+   The following is an example in the
    familiar BIND zone file format with a TTL of 1 hour (3600 seconds).
    Please substitute your domain and IP addresses.
 
    ::
 
-       chat.example.com. 3600 IN A 198.51.100.5
-       chat.example.com. 3600 IN AAAA 2001:db8::5
-       www.chat.example.com. 3600 IN CNAME chat.example.com.
-       mta-sts.chat.example.com. 3600 IN CNAME chat.example.com.
+       chat.example.org. 3600 IN A 198.51.100.5
+       chat.example.org. 3600 IN AAAA 2001:db8::5
+       www.chat.example.org. 3600 IN CNAME chat.example.org.
+       mta-sts.chat.example.org. 3600 IN CNAME chat.example.org.
 
 2. On your local PC, clone the repository and bootstrap the Python
    virtualenv.
@@ -54,20 +56,20 @@ steps. Please substitute it with your own domain.
        cd relay
        scripts/initenv.sh
 
-3. On your local PC, create chatmail configuration file
+3. On your local build machine (PC), create a chatmail configuration file
    ``chatmail.ini``:
 
    ::
 
        scripts/cmdeploy init chat.example.org  # <-- use your domain
 
-4. Verify that SSH root login to your remote server works:
+4. Verify that SSH root login to the deployment server server works:
 
    ::
 
        ssh root@chat.example.org  # <-- use your domain
 
-5. From your local PC, deploy the remote chatmail relay server:
+5. From your local build machine, setup and configure the remote deployment server:
 
    ::
 
@@ -81,7 +83,7 @@ steps. Please substitute it with your own domain.
 Other helpful commands
 ----------------------
 
-To check the status of your remotely running chatmail service:
+To check the status of your deployment server running the chatmail service:
 
 ::
 
@@ -158,7 +160,7 @@ Disable automatic address creation
 --------------------------------------------------------
 
 If you need to stop address creation, e.g. because some script is wildly
-creating addresses, login with ssh and run:
+creating addresses, login with ssh to the deployment machine and run:
 
 ::
 
@@ -167,3 +169,23 @@ creating addresses, login with ssh and run:
 Chatmail address creation will be denied while this file is present.
 
 
+Migrating to a new build machine
+----------------------------------
+
+To move or add a build machine,
+clone the relay repository on the new build machine, and copy the ``chatmail.ini`` file from the old build machine.
+Make sure ``rsync`` is installed, then initialize the environment:
+
+::
+
+   ./scripts/initenv.sh
+
+Run safety checks before a new deployment:
+
+::
+
+   ./scripts/cmdeploy dns
+   ./scripts/cmdeploy status
+
+If you keep multiple build machines (ie laptop and desktop), keep ``chatmail.ini`` in sync between
+them.

doc/source/migrate.rst

@@ -1,72 +1,98 @@
 
-Migrating to a new host
------------------------
+Migrating to a new machine
+===========================
 
-If you want to migrate chatmail relay from an old machine to a new
-machine, you can use these steps. They were tested with a Linux laptop;
-you might need to adjust some of the steps to your environment.
+This migration tutorial provides a step-wise approach
+to safely migrate a chatmail relay from one remote machine to another.
 
-Let’s assume that your ``mail_domain`` is ``mail.example.org``, all
-involved machines run Debian 12, your old site’s IP address is
-``13.37.13.37``, and your new site’s IP address is ``13.12.23.42``.
+Preliminary notes and assumptions
+---------------------------------
 
-Note, you should lower the TTLs of your DNS records to a value such as
-300 (5 minutes) so the migration happens as smoothly as possible.
+- If the migration is a planned move,
+  it's recommended to lower the Time To Live (TTL) of your DNS records to a value such as 300 (5 minutes),
+  at best much earlier than the actual planned migration.
+  This speeds up propagation of DNS changes in the Internet after the migration is complete.
 
-During the guide you might get a warning about changed SSH Host keys; in
-this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
+- The migration steps were tested with a Linux laptop; you might need to adjust some of the steps to your local environment.
 
-1. First, disable mail services on the old site.
+- Your ``mail_domain`` is ``mail.example.org``.
+
+- All remote machines run Debian 12.
+
+- The old site’s IP version 4 address is ``$OLD_IP4``.
+
+- The new site’s IP addresses are ``$NEW_IP4`` and ``$NEW_IPV6``.
+
+
+The six steps to migrate
+------------------------
+
+Note that during some of the following steps you might get a warning about changed SSH Host keys;
+in this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
+
+
+1. **Initially transfer mailboxes from old to new site.**
+
+   Login to old site, forwarding your ssh-agent with ``ssh -A``
+   to allow using ssh to directly copy files from old to new site.
+   ::
+
+       ssh -A root@$OLD_IP4
+       tar c /home/vmail/mail | ssh root@$NEW_IP4 "tar x -C /"
+
+
+2. **Pre-configure the new site but keep it inactive until step 6**
+   ::
+
+       CMDEPLOY_STAGES=install,configure scripts/cmdeploy run --ssh-host $NEW_IP4
+
+
+3. **It's getting serious: disable mail services on the old site.**
+   Users will not be able to send or receive messages until all steps are completed.
+   Other relays and mail servers will retry delivering messages from time to time,
+   so nothing is lost for users.
 
    ::
 
-       cmdeploy run --disable-mail --ssh-host 13.37.13.37
+       scripts/cmdeploy run --disable-mail --ssh-host $OLD_IP4
 
-   Now your users will notice the migration and will not be able to send
-   or receive messages until the migration is completed.
 
-2. Now we want to copy ``/home/vmail``, ``/var/lib/acme``,
-   ``/etc/dkimkeys``, and ``/var/spool/postfix`` to
-   the new site. Login to the old site while forwarding your SSH agent
-   so you can copy directly from the old to the new site with your SSH
-   key:
-
-   ::
-
-       ssh -A root@13.37.13.37
-       tar c - /home/vmail/mail /var/lib/acme /etc/dkimkeys /var/spool/postfix | ssh root@13.12.23.42 "tar x -C /"
-
-   This transfers all addresses, the TLS certificate,
-   and DKIM keys (so DKIM DNS record remains valid).
-   It also preserves the Postfix mail spool so any messages
-   pending delivery will still be delivered.
-
-3. Install chatmail on the new machine:
-
-   ::
-
-       cmdeploy run --disable-mail --ssh-host 13.12.23.42
-
-   Postfix and Dovecot are disabled for now; we will enable them later.
-   We first need to make the new site fully operational.
-
-4. On the new site, run the following to ensure the ownership is correct
-   in case UIDs/GIDs changed:
+4. **Final synchronization of TLS/DKIM secrets, mail queues and mailboxes.**
+   Again we use ssh-agent forwarding (``-A``) to allow transfering all important data directly
+   from the old to the new site.
+   ::
+
+       ssh -A root@$OLD_IP4
+       tar c /var/lib/acme /etc/dkimkeys /var/spool/postfix | ssh root@$NEW_IP4 "tar x -C /"
+       rsync -azH /home/vmail/mail root@$NEW_IP4:/home/vmail/
+
+   Login to the new site and ensure file ownerships are correctly set:
 
    ::
 
+       ssh root@$NEW_IP4
        chown root: -R /var/lib/acme
        chown opendkim: -R /etc/dkimkeys
        chown vmail: -R /home/vmail/mail
 
-5. Now, update DNS entries.
 
-   If other MTAs try to deliver messages to your chatmail domain they
-   may fail intermittently, as DNS catches up with the new site settings
-   but normally will retry delivering messages for at least a week, so
-   messages will not be lost.
+5. **Update the DNS entries to point to the new site.**
+   You only need to change the ``A`` and ``AAAA`` records, for example:
 
-6. Finally, you can execute ``cmdeploy run --ssh-host 13.12.23.42`` to
-   turn on chatmail on the new relay. Your users will be able to use the
-   chatmail relay as soon as the DNS changes have propagated. Voilà!
+   ::
+
+       mail.example.org.    IN A    $NEW_IP4
+       mail.example.org.    IN AAAA $NEW_IP6
+
+
+6. **Activate chatmail relay on new site.**
+
+   ::
+
+        CMDEPLOY_STAGES=activate scripts/cmdeploy run --ssh-host $NEW_IP4
+
+   Voilà!
+   Users will be able to use the relay as soon as the DNS changes have propagated.
+   If you have lowered the Time-to-Live for DNS records in step 1,
+   better use a higher value again (between 14400 and 86400 seconds) once you are sure everything works.
 

doc/source/overview.rst

@@ -272,8 +272,8 @@ by OpenDKIM screen policy script before validating the signatures. This
 corresponds to strict :rfc:`DMARC <7489>` alignment (``adkim=s``).
 If there is no valid DKIM signature on the incoming email, the
 sender receives a “5.7.1 No valid DKIM signature found” error.
-After validating the DKIM signature,
-the `final.lua` script strips all ``OpenDKIM:`` headers to reduce message size on disc.
+After validating the DKIM signature, 
+the `final.lua` script strips all ``OpenDKIM:`` headers to reduce message size on disc. 
 
 Note that chatmail relays
 

doc/source/related.rst

@@ -14,10 +14,10 @@ We know of three work-in-progress alternative implementation efforts:
    it to support all of the features and configuration settings required
    to operate as a chatmail relay.
 
--  `Maddy-Chatmail <https://github.com/sadraiiali/maddy_chatmail>`_: a
-   plugin for the `Maddy email server <https://maddy.email/>`_ which
-   aims to implement the chatmail relay features and configuration
-   options.
+-  `Madmail <https://github.com/omidz4t/madmail>`_: an
+   experimental fork of Maddy Mail Server <https://maddy.email/>`_ optimized
+   for chatmail deployments.  It provides a single binary solution
+   for running a chatmail relay.
 
 -  `Chatmail Cookbook <https://github.com/feld/chatmail-cookbook>`_:
    A Chef Cookbook implementing a relay server. The project follows the