chore(release): prepare for 1.9.0

.github/workflows/test-and-deploy-ipv4only.yaml

@@ -93,7 +93,7 @@ jobs:
           ssh root@ns.testrun.org systemctl reload nsd
 
       - name: cmdeploy test
-        run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow
+        run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
 
       - name: cmdeploy dns
         run: cmdeploy dns -v

.github/workflows/test-and-deploy.yaml

@@ -94,7 +94,7 @@ jobs:
           ssh root@ns.testrun.org systemctl reload nsd
 
       - name: cmdeploy test
-        run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow
+        run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
 
       - name: cmdeploy dns
         run: cmdeploy dns -v

CHANGELOG.md

@@ -1,5 +1,31 @@
 # Changelog for chatmail deployment 
 
+## 1.9.0 2025-12-18
+
+### Documentation
+
+- Add RELEASE.md and CONTRIBUTING.md
+- README update, mention Chatmail Cookbook project
+
+### Bug Fixes
+
+- Expire messages also from IMAP subfolders
+- Use absolute path instead of relative path in message expiration script
+- Restart Postfix and Dovecot automatically on failure
+- acmetool: Use a fixed name and `reconcile` instead of `want`
+
+### Features
+
+- Report DKIM error code in SMTP response
+- Remove development notice from the web pages
+
+### Miscellaneous Tasks
+
+- Update the heading in the CHANGELOG.md
+- Setup git-cliff
+- Run tests against ci-chatmail.testrun.org instead of nine.testrun.org
+- Cleanup remaining echobot code, remove echobot user from deployment and passthrough recipients
+
 ## 1.8.0 2025-12-12
 
 - Add imap_compress option to chatmail.ini

chatmaild/src/chatmaild/expire.py

@@ -14,7 +14,7 @@ from stat import S_ISREG
 
 from chatmaild.config import read_config
 
-FileEntry = namedtuple("FileEntry", ("relpath", "mtime", "size"))
+FileEntry = namedtuple("FileEntry", ("path", "mtime", "size"))
 
 
 def iter_mailboxes(basedir, maxnum):
@@ -51,33 +51,27 @@ class MailboxStat:
 
     def __init__(self, basedir):
         self.basedir = str(basedir)
-        # all detected messages in cur/new/tmp folders
         self.messages = []
-
-        # all detected files in mailbox top dir
         self.extrafiles = []
+        self.scandir(self.basedir)
 
-        # scan all relevant files (without recursion)
-        old_cwd = os.getcwd()
-        try:
-            os.chdir(self.basedir)
-        except FileNotFoundError:
-            return
-        for name in os_listdir_if_exists("."):
+    def scandir(self, folderdir):
+        for name in os_listdir_if_exists(folderdir):
+            path = f"{folderdir}/{name}"
             if name in ("cur", "new", "tmp"):
-                for msg_name in os_listdir_if_exists(name):
-                    entry = get_file_entry(f"{name}/{msg_name}")
+                for msg_name in os_listdir_if_exists(path):
+                    entry = get_file_entry(f"{path}/{msg_name}")
                     if entry is not None:
                         self.messages.append(entry)
-
+            elif os.path.isdir(path):
+                self.scandir(path)
             else:
-                entry = get_file_entry(name)
+                entry = get_file_entry(path)
                 if entry is not None:
                     self.extrafiles.append(entry)
                     if name == "password":
                         self.last_login = entry.mtime
         self.extrafiles.sort(key=lambda x: -x.size)
-        os.chdir(old_cwd)
 
 
 def print_info(msg):
@@ -130,13 +124,6 @@ class Expiry:
             self.remove_mailbox(mbox.basedir)
             return
 
-        # all to-be-removed files are relative to the mailbox basedir
-        try:
-            os.chdir(mbox.basedir)
-        except FileNotFoundError:
-            print_info(f"mailbox not found/vanished {mbox.basedir}")
-            return
-
         mboxname = os.path.basename(mbox.basedir)
         if self.verbose:
             date = datetime.fromtimestamp(mbox.last_login) if mbox.last_login else None
@@ -147,11 +134,12 @@ class Expiry:
         self.all_files += len(mbox.messages)
         for message in mbox.messages:
             if message.mtime < cutoff_mails:
-                self.remove_file(message.relpath, mtime=message.mtime)
+                self.remove_file(message.path, mtime=message.mtime)
             elif message.size > 200000 and message.mtime < cutoff_large_mails:
                 # we only remove noticed large files (not unnoticed ones in new/)
-                if message.relpath.startswith("cur/"):
-                    self.remove_file(message.relpath, mtime=message.mtime)
+                parts = message.path.split("/")
+                if len(parts) >= 2 and parts[-2] == "cur":
+                    self.remove_file(message.path, mtime=message.mtime)
             else:
                 continue
             changed = True

chatmaild/src/chatmaild/tests/test_expire.py

@@ -17,19 +17,17 @@ from chatmaild.expire import main as expiry_main
 from chatmaild.fsreport import main as report_main
 
 
-def fill_mbox(basedir):
-    basedir1 = basedir.joinpath("mailbox1@example.org")
-    basedir1.mkdir()
-    password = basedir1.joinpath("password")
+def fill_mbox(folderdir):
+    password = folderdir.joinpath("password")
     password.write_text("xxx")
-    basedir1.joinpath("maildirsize").write_text("xxx")
+    folderdir.joinpath("maildirsize").write_text("xxx")
 
-    garbagedir = basedir1.joinpath("garbagedir")
+    garbagedir = folderdir.joinpath("garbagedir")
     garbagedir.mkdir()
+    garbagedir.joinpath("bimbum").write_text("hello")
 
-    create_new_messages(basedir1, ["cur/msg1"], size=500)
-    create_new_messages(basedir1, ["new/msg2"], size=600)
-    return basedir1
+    create_new_messages(folderdir, ["cur/msg1"], size=500)
+    create_new_messages(folderdir, ["new/msg2"], size=600)
 
 
 def create_new_messages(basedir, relpaths, size=1000, days=0):
@@ -45,8 +43,21 @@ def create_new_messages(basedir, relpaths, size=1000, days=0):
 
 @pytest.fixture
 def mbox1(example_config):
-    basedir1 = fill_mbox(example_config.mailboxes_dir)
-    return MailboxStat(basedir1)
+    mboxdir = example_config.mailboxes_dir.joinpath("mailbox1@example.org")
+    mboxdir.mkdir()
+    fill_mbox(mboxdir)
+    return MailboxStat(mboxdir)
+
+
+def test_deltachat_folder(example_config):
+    """Test old setups that might have a .DeltaChat folder where messages also need to get removed."""
+    mboxdir = example_config.mailboxes_dir.joinpath("mailbox1@example.org")
+    mboxdir.mkdir()
+    mbox2dir = mboxdir.joinpath(".DeltaChat")
+    mbox2dir.mkdir()
+    fill_mbox(mbox2dir)
+    mb = MailboxStat(mboxdir)
+    assert len(mb.messages) == 2
 
 
 def test_filentry_ordering(tmp_path):
@@ -76,7 +87,7 @@ def test_stats_mailbox(mbox1):
     create_new_messages(mbox1.basedir, ["large-extra"], size=1000)
     create_new_messages(mbox1.basedir, ["index-something"], size=3)
     mbox2 = MailboxStat(mbox1.basedir)
-    assert len(mbox2.extrafiles) == 4
+    assert len(mbox2.extrafiles) == 5
     assert mbox2.extrafiles[0].size == 1000
 
     # cope well with mailbox dirs that have no password (for whatever reason)

doc/source/migrate.rst

@@ -7,88 +7,66 @@ machine, you can use these steps. They were tested with a Linux laptop;
 you might need to adjust some of the steps to your environment.
 
 Let’s assume that your ``mail_domain`` is ``mail.example.org``, all
-involved machines run Debian 12, your old site’s IP version 4 address is
-``$OLD_IP4``, and your new site’s IP4 address is ``$NEW_IP4``.
+involved machines run Debian 12, your old site’s IP address is
+``13.37.13.37``, and your new site’s IP address is ``13.12.23.42``.
 
-First of all, you should lower the Time To Live (TTL) of your DNS records
-to a value such as 300 (5 minutes).
-Short TTL values allow to change DNS records during the migration more timely.
+Note, you should lower the TTLs of your DNS records to a value such as
+300 (5 minutes) so the migration happens as smoothly as possible.
 
 During the guide you might get a warning about changed SSH Host keys; in
 this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
 
-1. First, to make the downtime during the migration shorter,
-   let's transfer the current state of the mailboxes.
-   Login to your old machine (while forwarding your ssh-agent with ``ssh -A``)
+1. First, disable mail services on the old site.
+
+   ::
+
+       cmdeploy run --disable-mail --ssh-host 13.37.13.37
+
+   Now your users will notice the migration and will not be able to send
+   or receive messages until the migration is completed.
+
+2. Now we want to copy ``/home/vmail``, ``/var/lib/acme``,
+   ``/etc/dkimkeys``, and ``/var/spool/postfix`` to
+   the new site. Login to the old site while forwarding your SSH agent
    so you can copy directly from the old to the new site with your SSH
    key:
 
    ::
 
-       ssh -A root@$OLD_IP4
-       tar c /home/vmail/mail | ssh root@$NEW_IP4 "tar x -C /"
+       ssh -A root@13.37.13.37
+       tar c - /home/vmail/mail /var/lib/acme /etc/dkimkeys /var/spool/postfix | ssh root@13.12.23.42 "tar x -C /"
 
-   This saves us time during the downtime,
-   at least the mailboxes are there already.
-   They contain user passwords, encrypted push notification tokens,
-   messages which might not have been fetched by all devices of the user yet,
-   and dovecot indexes which track the state of the mailbox.
-
-2. Then, from your local machine, install chatmail on the new machine, but don't activate it yet:
-
-   ::
-
-       CMDEPLOY_STAGES=install,configure cmdeploy run --ssh-host $NEW_IP4
-
-   The services are disabled for now; we will enable them later.
-   We first need to make the new site fully operational.
-
-3. Now it's getting serious: disable the mail services on the old site.
-
-   ::
-
-       cmdeploy run --disable-mail --ssh-host $OLD_IP4
-
-   Your users will start to notice the migration and will not be able to send
-   or receive messages until the migration is completed.
-   Other relays and mail servers will wait with delivering messages
-   until your relay is reachable again.
-
-4. Now we want to copy ``/home/vmail``, ``/var/lib/acme``,
-   ``/etc/dkimkeys``, and ``/var/spool/postfix`` to
-   the new site. Let's forward the SSH agent again to copy the files directly.
-   This time, we copy ``/home/vmail/mail`` with rsync to only copy the recent changes:
-
-   ::
-
-       ssh -A root@$OLD_IP4
-       tar c /var/lib/acme /etc/dkimkeys /var/spool/postfix | ssh root@$NEW_IP4 "tar x -C /"
-       rsync -azH /home/vmail/mail root@$NEW_IP4:/home/vmail/
-
-   This transfers all messages which have not been fetched yet, the TLS certificate,
+   This transfers all addresses, the TLS certificate,
    and DKIM keys (so DKIM DNS record remains valid).
    It also preserves the Postfix mail spool so any messages
    pending delivery will still be delivered.
 
-5. Now login to the new site and run the following to ensure the ownership is correct
+3. Install chatmail on the new machine:
+
+   ::
+
+       cmdeploy run --disable-mail --ssh-host 13.12.23.42
+
+   Postfix and Dovecot are disabled for now; we will enable them later.
+   We first need to make the new site fully operational.
+
+4. On the new site, run the following to ensure the ownership is correct
    in case UIDs/GIDs changed:
 
    ::
 
-       ssh root@$NEW_IP4
        chown root: -R /var/lib/acme
        chown opendkim: -R /etc/dkimkeys
        chown vmail: -R /home/vmail/mail
 
-6. Now, update the DNS entries.
-   You only need to change the ``A`` and ``AAAA`` records, for example:
+5. Now, update DNS entries.
 
-   ::
+   If other MTAs try to deliver messages to your chatmail domain they
+   may fail intermittently, as DNS catches up with the new site settings
+   but normally will retry delivering messages for at least a week, so
+   messages will not be lost.
 
-       mail.example.org.    IN A    $NEW_IP4
-       mail.example.org.    IN AAAA $NEW_IP6
-
-7. Finally, you can execute ``CMDEPLOY_STAGES=activate cmdeploy run --ssh-host $NEW_IP4`` to
+6. Finally, you can execute ``cmdeploy run --ssh-host 13.12.23.42`` to
    turn on chatmail on the new relay. Your users will be able to use the
    chatmail relay as soon as the DNS changes have propagated. Voilà!