fix tar commands

.github/workflows/test-and-deploy-ipv4only.yaml

@@ -19,8 +19,13 @@ jobs:
     environment:
       name: staging-ipv4.testrun.org
       url: https://staging-ipv4.testrun.org/
-    concurrency: staging-ipv4.testrun.org
+    concurrency:
+      group: ci-ipv4-${{ github.workflow }}-${{ github.ref }}
+      cancel-in-progress: ${{ !contains(github.ref, '$GITHUB_REF') }}
     steps:
+      - uses: jsok/serialize-workflow-action@515cd04c46d7ea7435c4a22a3b4419127afdefe9
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions/checkout@v4
 
       - name: prepare SSH
@@ -88,7 +93,7 @@ jobs:
           ssh root@ns.testrun.org systemctl reload nsd
 
       - name: cmdeploy test
-        run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
+        run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow
 
       - name: cmdeploy dns
         run: cmdeploy dns -v

.github/workflows/test-and-deploy.yaml

@@ -19,8 +19,13 @@ jobs:
     environment:
       name: staging2.testrun.org
       url: https://staging2.testrun.org/
-    concurrency: staging2.testrun.org
+    concurrency:
+      group: ci-${{ github.workflow }}-${{ github.ref }}
+      cancel-in-progress: ${{ !contains(github.ref, '$GITHUB_REF') }}
     steps:
+      - uses: jsok/serialize-workflow-action@515cd04c46d7ea7435c4a22a3b4419127afdefe9
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions/checkout@v4
 
       - name: prepare SSH
@@ -89,7 +94,7 @@ jobs:
           ssh root@ns.testrun.org systemctl reload nsd
 
       - name: cmdeploy test
-        run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
+        run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow
 
       - name: cmdeploy dns
         run: cmdeploy dns -v

CHANGELOG.md

@@ -1,31 +1,5 @@
 # Changelog for chatmail deployment 
 
-## 1.9.0 2025-12-18
-
-### Documentation
-
-- Add RELEASE.md and CONTRIBUTING.md
-- README update, mention Chatmail Cookbook project
-
-### Bug Fixes
-
-- Expire messages also from IMAP subfolders
-- Use absolute path instead of relative path in message expiration script
-- Restart Postfix and Dovecot automatically on failure
-- acmetool: Use a fixed name and `reconcile` instead of `want`
-
-### Features
-
-- Report DKIM error code in SMTP response
-- Remove development notice from the web pages
-
-### Miscellaneous Tasks
-
-- Update the heading in the CHANGELOG.md
-- Setup git-cliff
-- Run tests against ci-chatmail.testrun.org instead of nine.testrun.org
-- Cleanup remaining echobot code, remove echobot user from deployment and passthrough recipients
-
 ## 1.8.0 2025-12-12
 
 - Add imap_compress option to chatmail.ini

chatmaild/src/chatmaild/expire.py

@@ -14,7 +14,7 @@ from stat import S_ISREG
 
 from chatmaild.config import read_config
 
-FileEntry = namedtuple("FileEntry", ("path", "mtime", "size"))
+FileEntry = namedtuple("FileEntry", ("relpath", "mtime", "size"))
 
 
 def iter_mailboxes(basedir, maxnum):
@@ -51,27 +51,33 @@ class MailboxStat:
 
     def __init__(self, basedir):
         self.basedir = str(basedir)
+        # all detected messages in cur/new/tmp folders
         self.messages = []
-        self.extrafiles = []
-        self.scandir(self.basedir)
 
-    def scandir(self, folderdir):
+        # all detected files in mailbox top dir
-        for name in os_listdir_if_exists(folderdir):
+        self.extrafiles = []
-            path = f"{folderdir}/{name}"
+
+        # scan all relevant files (without recursion)
+        old_cwd = os.getcwd()
+        try:
+            os.chdir(self.basedir)
+        except FileNotFoundError:
+            return
+        for name in os_listdir_if_exists("."):
             if name in ("cur", "new", "tmp"):
-                for msg_name in os_listdir_if_exists(path):
+                for msg_name in os_listdir_if_exists(name):
-                    entry = get_file_entry(f"{path}/{msg_name}")
+                    entry = get_file_entry(f"{name}/{msg_name}")
                     if entry is not None:
                         self.messages.append(entry)
-            elif os.path.isdir(path):
+
-                self.scandir(path)
             else:
-                entry = get_file_entry(path)
+                entry = get_file_entry(name)
                 if entry is not None:
                     self.extrafiles.append(entry)
                     if name == "password":
                         self.last_login = entry.mtime
         self.extrafiles.sort(key=lambda x: -x.size)
+        os.chdir(old_cwd)
 
 
 def print_info(msg):
@@ -124,6 +130,13 @@ class Expiry:
             self.remove_mailbox(mbox.basedir)
             return
 
+        # all to-be-removed files are relative to the mailbox basedir
+        try:
+            os.chdir(mbox.basedir)
+        except FileNotFoundError:
+            print_info(f"mailbox not found/vanished {mbox.basedir}")
+            return
+
         mboxname = os.path.basename(mbox.basedir)
         if self.verbose:
             date = datetime.fromtimestamp(mbox.last_login) if mbox.last_login else None
@@ -134,12 +147,11 @@ class Expiry:
         self.all_files += len(mbox.messages)
         for message in mbox.messages:
             if message.mtime < cutoff_mails:
-                self.remove_file(message.path, mtime=message.mtime)
+                self.remove_file(message.relpath, mtime=message.mtime)
             elif message.size > 200000 and message.mtime < cutoff_large_mails:
                 # we only remove noticed large files (not unnoticed ones in new/)
-                parts = message.path.split("/")
+                if message.relpath.startswith("cur/"):
-                if len(parts) >= 2 and parts[-2] == "cur":
+                    self.remove_file(message.relpath, mtime=message.mtime)
-                    self.remove_file(message.path, mtime=message.mtime)
             else:
                 continue
             changed = True

chatmaild/src/chatmaild/filtermail.py

@@ -307,9 +307,12 @@ class IncomingBeforeQueueHandler:
             return error
         log_info("re-injecting the mail that passed checks")
 
+        # the smtp daemon on reinject_port_incoming gives it to dkim milter
+        # which looks at source address to determine whether to verify or sign
         client = SMTPClient(
             "localhost",
             self.config.postfix_reinject_port_incoming,
+            source_address=("127.0.0.2", 0),
         )
         client.sendmail(
             envelope.mail_from, envelope.rcpt_tos, envelope.original_content

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -15,7 +15,7 @@ mail_domain = {mail_domain}
 max_user_send_per_minute = 60
 
 # maximum mailbox size of a chatmail address
-max_mailbox_size = 500M
+max_mailbox_size = 100M
 
 # maximum message size for an e-mail in bytes
 max_message_size = 31457280

chatmaild/src/chatmaild/tests/test_config.py

@@ -33,7 +33,7 @@ def test_read_config_testrun(make_config):
     assert config.filtermail_smtp_port == 10080
     assert config.postfix_reinject_port == 10025
     assert config.max_user_send_per_minute == 60
-    assert config.max_mailbox_size == "500M"
+    assert config.max_mailbox_size == "100M"
     assert config.delete_mails_after == "20"
     assert config.delete_large_after == "7"
     assert config.username_min_length == 9

chatmaild/src/chatmaild/tests/test_expire.py

@@ -17,17 +17,19 @@ from chatmaild.expire import main as expiry_main
 from chatmaild.fsreport import main as report_main
 
 
-def fill_mbox(folderdir):
+def fill_mbox(basedir):
-    password = folderdir.joinpath("password")
+    basedir1 = basedir.joinpath("mailbox1@example.org")
+    basedir1.mkdir()
+    password = basedir1.joinpath("password")
     password.write_text("xxx")
-    folderdir.joinpath("maildirsize").write_text("xxx")
+    basedir1.joinpath("maildirsize").write_text("xxx")
 
-    garbagedir = folderdir.joinpath("garbagedir")
+    garbagedir = basedir1.joinpath("garbagedir")
     garbagedir.mkdir()
-    garbagedir.joinpath("bimbum").write_text("hello")
 
-    create_new_messages(folderdir, ["cur/msg1"], size=500)
+    create_new_messages(basedir1, ["cur/msg1"], size=500)
-    create_new_messages(folderdir, ["new/msg2"], size=600)
+    create_new_messages(basedir1, ["new/msg2"], size=600)
+    return basedir1
 
 
 def create_new_messages(basedir, relpaths, size=1000, days=0):
@@ -43,21 +45,8 @@ def create_new_messages(basedir, relpaths, size=1000, days=0):
 
 @pytest.fixture
 def mbox1(example_config):
-    mboxdir = example_config.mailboxes_dir.joinpath("mailbox1@example.org")
+    basedir1 = fill_mbox(example_config.mailboxes_dir)
-    mboxdir.mkdir()
+    return MailboxStat(basedir1)
-    fill_mbox(mboxdir)
-    return MailboxStat(mboxdir)
-
-
-def test_deltachat_folder(example_config):
-    """Test old setups that might have a .DeltaChat folder where messages also need to get removed."""
-    mboxdir = example_config.mailboxes_dir.joinpath("mailbox1@example.org")
-    mboxdir.mkdir()
-    mbox2dir = mboxdir.joinpath(".DeltaChat")
-    mbox2dir.mkdir()
-    fill_mbox(mbox2dir)
-    mb = MailboxStat(mboxdir)
-    assert len(mb.messages) == 2
 
 
 def test_filentry_ordering(tmp_path):
@@ -87,7 +76,7 @@ def test_stats_mailbox(mbox1):
     create_new_messages(mbox1.basedir, ["large-extra"], size=1000)
     create_new_messages(mbox1.basedir, ["index-something"], size=3)
     mbox2 = MailboxStat(mbox1.basedir)
-    assert len(mbox2.extrafiles) == 5
+    assert len(mbox2.extrafiles) == 4
     assert mbox2.extrafiles[0].size == 1000
 
     # cope well with mailbox dirs that have no password (for whatever reason)

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -29,7 +29,6 @@ stream {
             default 127.0.0.1:8443;
             ~\bsmtp\b 127.0.0.1:465;
             ~\bimap\b 127.0.0.1:993;
-            ~\bssh\b 127.0.0.1:22;
         }
 
         server {

cmdeploy/src/cmdeploy/opendkim/final.lua

@@ -1,5 +1,4 @@
-mtaname = odkim.get_mtasymbol(ctx, "{daemon_name}")
+if odkim.internal_ip(ctx) == 1 then
-if mtaname == "ORIGINATING" then
 	-- Outgoing message will be signed,
 	-- no need to look for signatures.
 	return nil

cmdeploy/src/cmdeploy/opendkim/opendkim.conf

@@ -65,9 +65,3 @@ PidFile			/run/opendkim/opendkim.pid
 # The trust anchor enables DNSSEC. In Debian, the trust anchor file is provided
 # by the package dns-root-data.
 TrustAnchorFile		/usr/share/dns/root.key
-
-# Sign messages when `-o milter_macro_daemon_name=ORIGINATING` is set.
-MTA ORIGINATING
-
-# No hosts are treated as internal, ORIGINATING daemon name should be set explicitly.
-InternalHosts -

cmdeploy/src/cmdeploy/postfix/master.cf.j2

@@ -31,6 +31,7 @@ submission inet n       -       y       -       5000    smtpd
   -o smtpd_sender_restrictions=$mua_sender_restrictions
   -o smtpd_recipient_restrictions=
   -o smtpd_relay_restrictions=permit_sasl_authenticated,reject
+  -o milter_macro_daemon_name=ORIGINATING
   -o smtpd_client_connection_count_limit=1000
   -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
 smtps     inet  n       -       y       -       5000    smtpd
@@ -48,6 +49,7 @@ smtps     inet  n       -       y       -       5000    smtpd
   -o smtpd_recipient_restrictions=
   -o smtpd_relay_restrictions=permit_sasl_authenticated,reject
   -o smtpd_client_connection_count_limit=1000
+  -o milter_macro_daemon_name=ORIGINATING
   -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
 #628       inet  n       -       y       -       -       qmqpd
 pickup    unix  n       -       y       60      1       pickup
@@ -79,7 +81,6 @@ filter    unix -        n       n       -       -       lmtp
 # Local SMTP server for reinjecting outgoing filtered mail.
 127.0.0.1:{{ config.postfix_reinject_port }} inet  n       -       n       -       100      smtpd
   -o syslog_name=postfix/reinject
-  -o milter_macro_daemon_name=ORIGINATING
   -o smtpd_milters=unix:opendkim/opendkim.sock
   -o cleanup_service_name=authclean
 

doc/source/getting_started.rst

@@ -40,10 +40,10 @@ steps. Please substitute it with your own domain.
 
    ::
 
-       chat.example.org. 3600 IN A 198.51.100.5
+       chat.example.com. 3600 IN A 198.51.100.5
-       chat.example.org. 3600 IN AAAA 2001:db8::5
+       chat.example.com. 3600 IN AAAA 2001:db8::5
-       www.chat.example.org. 3600 IN CNAME chat.example.org.
+       www.chat.example.com. 3600 IN CNAME chat.example.com.
-       mta-sts.chat.example.org. 3600 IN CNAME chat.example.org.
+       mta-sts.chat.example.com. 3600 IN CNAME chat.example.com.
 
 2. On your local PC, clone the repository and bootstrap the Python
    virtualenv.

doc/source/migrate.rst

@@ -1,72 +1,77 @@
 
-Migrating to a new machine
+Migrating to a new host
-===========================
+-----------------------
 
-This migration tutorial provides a step-wise approach
+If you want to migrate chatmail relay from an old machine to a new
-to safely migrate a chatmail relay from one remote machine to another.
+machine, you can use these steps. They were tested with a Linux laptop;
+you might need to adjust some of the steps to your environment.
 
-Preliminary notes and assumptions
+Let’s assume that your ``mail_domain`` is ``mail.example.org``, all
----------------------------------
+involved machines run Debian 12, your old site’s IP version 4 address is
+``$OLD_IP4``, and your new site’s IP4 address is ``$NEW_IP4``.
 
-- If the migration is a planned move,
+First of all, you should lower the Time To Live (TTL) of your DNS records
-  it's recommended to lower the Time To Live (TTL) of your DNS records to a value such as 300 (5 minutes),
+to a value such as 300 (5 minutes).
-  at best much earlier than the actual planned migration.
+Short TTL values allow to change DNS records during the migration more timely.
-  This speeds up propagation of DNS changes in the Internet after the migration is complete.
 
-- The migration steps were tested with a Linux laptop; you might need to adjust some of the steps to your local environment.
+During the guide you might get a warning about changed SSH Host keys; in
+this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
 
-- Your ``mail_domain`` is ``mail.example.org``.
+1. First, to make the downtime during the migration shorter,
+   let's transfer the current state of the mailboxes.
+   Login to your old machine (while forwarding your ssh-agent with ``ssh -A``)
+   so you can copy directly from the old to the new site with your SSH
+   key:
 
-- All remote machines run Debian 12.
-
-- The old site’s IP version 4 address is ``$OLD_IP4``.
-
-- The new site’s IP addresses are ``$NEW_IP4`` and ``$NEW_IPV6``.
-
-
-The six steps to migrate
-------------------------
-
-Note that during some of the following steps you might get a warning about changed SSH Host keys;
-in this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
-
-
-1. **Initially transfer mailboxes from old to new site.**
-
-   Login to old site, forwarding your ssh-agent with ``ssh -A``
-   to allow using ssh to directly copy files from old to new site.
    ::
 
        ssh -A root@$OLD_IP4
        tar c /home/vmail/mail | ssh root@$NEW_IP4 "tar x -C /"
 
+   This saves us time during the downtime,
+   at least the mailboxes are there already.
+   They contain user passwords, encrypted push notification tokens,
+   messages which might not have been fetched by all devices of the user yet,
+   and dovecot indexes which track the state of the mailbox.
 
-2. **Pre-configure the new site but keep it inactive until step 6**
+2. Then, from your local machine, install chatmail on the new machine, but don't activate it yet:
-   ::
-
-       CMDEPLOY_STAGES=install,configure scripts/cmdeploy run --ssh-host $NEW_IP4
-
-
-3. **It's getting serious: disable mail services on the old site.**
-   Users will not be able to send or receive messages until all steps are completed.
-   Other relays and mail servers will retry delivering messages from time to time,
-   so nothing is lost for users.
 
    ::
 
-       scripts/cmdeploy run --disable-mail --ssh-host $OLD_IP4
+       CMDEPLOY_STAGES=install,configure cmdeploy run --ssh-host $NEW_IP4
 
+   The services are disabled for now; we will enable them later.
+   We first need to make the new site fully operational.
+
+3. Now it's getting serious: disable the mail services on the old site.
+
+   ::
+
+       cmdeploy run --disable-mail --ssh-host $OLD_IP4
+
+   Your users will start to notice the migration and will not be able to send
+   or receive messages until the migration is completed.
+   Other relays and mail servers will wait with delivering messages
+   until your relay is reachable again.
+
+4. Now we want to copy ``/home/vmail``, ``/var/lib/acme``,
+   ``/etc/dkimkeys``, and ``/var/spool/postfix`` to
+   the new site. Let's forward the SSH agent again to copy the files directly.
+   This time, we copy ``/home/vmail/mail`` with rsync to only copy the recent changes:
 
-4. **Final synchronization of TLS/DKIM secrets, mail queues and mailboxes.**
-   Again we use ssh-agent forwarding (``-A``) to allow transfering all important data directly
-   from the old to the new site.
    ::
 
        ssh -A root@$OLD_IP4
        tar c /var/lib/acme /etc/dkimkeys /var/spool/postfix | ssh root@$NEW_IP4 "tar x -C /"
        rsync -azH /home/vmail/mail root@$NEW_IP4:/home/vmail/
 
-   Login to the new site and ensure file ownerships are correctly set:
+   This transfers all messages which have not been fetched yet, the TLS certificate,
+   and DKIM keys (so DKIM DNS record remains valid).
+   It also preserves the Postfix mail spool so any messages
+   pending delivery will still be delivered.
+
+5. Now login to the new site and run the following to ensure the ownership is correct
+   in case UIDs/GIDs changed:
 
    ::
 
@@ -75,8 +80,7 @@ in this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
        chown opendkim: -R /etc/dkimkeys
        chown vmail: -R /home/vmail/mail
 
-
+6. Now, update the DNS entries.
-5. **Update the DNS entries to point to the new site.**
    You only need to change the ``A`` and ``AAAA`` records, for example:
 
    ::
@@ -84,15 +88,7 @@ in this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
        mail.example.org.    IN A    $NEW_IP4
        mail.example.org.    IN AAAA $NEW_IP6
 
-
+7. Finally, you can execute ``CMDEPLOY_STAGES=activate cmdeploy run --ssh-host $NEW_IP4`` to
-6. **Activate chatmail relay on new site.**
+   turn on chatmail on the new relay. Your users will be able to use the
-
+   chatmail relay as soon as the DNS changes have propagated. Voilà!
-   ::
-
-        CMDEPLOY_STAGES=activate scripts/cmdeploy run --ssh-host $NEW_IP4
-
-   Voilà!
-   Users will be able to use the relay as soon as the DNS changes have propagated.
-   If you have lowered the Time-to-Live for DNS records in step 1,
-   better use a higher value again (between 14400 and 86400 seconds) once you are sure everything works.