ci: delete old files when backing up acmetool state and DKIM keys

.github/workflows/test-and-deploy-ipv4only.yaml

@@ -35,11 +35,11 @@ jobs:
           chmod 600 ~/.ssh/id_ed25519
           ssh-keyscan staging-ipv4.testrun.org > ~/.ssh/known_hosts
           # save previous acme & dkim state
-          rsync -avz root@staging-ipv4.testrun.org:/var/lib/acme acme-ipv4 || true
+          rsync -avz --delete root@staging-ipv4.testrun.org:/var/lib/acme acme-ipv4
-          rsync -avz root@staging-ipv4.testrun.org:/etc/dkimkeys dkimkeys-ipv4 || true
+          rsync -avz --delete root@staging-ipv4.testrun.org:/etc/dkimkeys dkimkeys-ipv4
           # store previous acme & dkim state on ns.testrun.org, if it contains useful certs
-          if [ -f dkimkeys-ipv4/dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys-ipv4 root@ns.testrun.org:/tmp/ || true; fi
+          test -f dkimkeys-ipv4/dkimkeys/opendkim.private && rsync -avz --delete -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys-ipv4 root@ns.testrun.org:/tmp/
-          if [ "$(ls -A acme-ipv4/acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme-ipv4 root@ns.testrun.org:/tmp/ || true; fi
+          test -n "$(ls -A acme-ipv4/acme/certs)" && rsync -avz --delete -e "ssh -o StrictHostKeyChecking=accept-new" acme-ipv4 root@ns.testrun.org:/tmp/
           # make sure CAA record isn't set
           scp -o StrictHostKeyChecking=accept-new .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
           ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging-ipv4.testrun.org.zone
@@ -93,7 +93,7 @@ jobs:
           ssh root@ns.testrun.org systemctl reload nsd
 
       - name: cmdeploy test
-        run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
+        run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow
 
       - name: cmdeploy dns
         run: cmdeploy dns -v

.github/workflows/test-and-deploy.yaml

@@ -35,11 +35,11 @@ jobs:
           chmod 600 ~/.ssh/id_ed25519
           ssh-keyscan staging2.testrun.org > ~/.ssh/known_hosts
           # save previous acme & dkim state
-          rsync -avz root@staging2.testrun.org:/var/lib/acme . || true
+          rsync -avz --delete root@staging2.testrun.org:/var/lib/acme .
-          rsync -avz root@staging2.testrun.org:/etc/dkimkeys . || true
+          rsync -avz --delete root@staging2.testrun.org:/etc/dkimkeys .
           # store previous acme & dkim state on ns.testrun.org, if it contains useful certs
-          if [ -f dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys root@ns.testrun.org:/tmp/ || true; fi
+          test -f dkimkeys/opendkim.private && rsync -avz --delete -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys root@ns.testrun.org:/tmp/
-          if [ "$(ls -A acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme root@ns.testrun.org:/tmp/ || true; fi
+          test -n "$(ls -A acme/certs)" && rsync -avz --delete -e "ssh -o StrictHostKeyChecking=accept-new" acme root@ns.testrun.org:/tmp/
           # make sure CAA record isn't set
           scp -o StrictHostKeyChecking=accept-new .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
           ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging2.testrun.org.zone
@@ -94,7 +94,7 @@ jobs:
           ssh root@ns.testrun.org systemctl reload nsd
 
       - name: cmdeploy test
-        run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
+        run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow
 
       - name: cmdeploy dns
         run: cmdeploy dns -v

CHANGELOG.md

@@ -1,32 +1,6 @@
 # Changelog for chatmail deployment 
 
-## 1.9.0 2025-12-18
+## untagged
-
-### Documentation
-
-- Add RELEASE.md and CONTRIBUTING.md
-- README update, mention Chatmail Cookbook project
-
-### Bug Fixes
-
-- Expire messages also from IMAP subfolders
-- Use absolute path instead of relative path in message expiration script
-- Restart Postfix and Dovecot automatically on failure
-- acmetool: Use a fixed name and `reconcile` instead of `want`
-
-### Features
-
-- Report DKIM error code in SMTP response
-- Remove development notice from the web pages
-
-### Miscellaneous Tasks
-
-- Update the heading in the CHANGELOG.md
-- Setup git-cliff
-- Run tests against ci-chatmail.testrun.org instead of nine.testrun.org
-- Cleanup remaining echobot code, remove echobot user from deployment and passthrough recipients
-
-## 1.8.0 2025-12-12
 
 - Add imap_compress option to chatmail.ini
   ([#760](https://github.com/chatmail/relay/pull/760))

CONTRIBUTING.md

@@ -1,7 +0,0 @@
-# Contributing to the chatmail relay
-
-Commit messages follow the [Conventional Commits] notation.
-We use [git-cliff] to generate the changelog from commit messages before the release.
-
-[Conventional Commits]: https://www.conventionalcommits.org/
-[git-cliff]: https://git-cliff.org/

RELEASE.md

@@ -1,15 +0,0 @@
-# Releasing a new version of chatmail relay
-
-For example, to release version 1.9.0 of chatmail relay, do the following steps.
-
-1. Update the changelog: `git cliff --unreleased --tag 1.9.0 --prepend CHANGELOG.md` or `git cliff -u -t 1.9.0 -p CHANGELOG.md`.
-
-2. Open the changelog in the editor, edit it if required.
-
-3. Commit the changes to the changelog with a commit message `chore(release): prepare for 1.9.0`.
-
-3. Tag the release: `git tag --annotate 1.9.0`.
-
-4. Push the release tag: `git push origin 1.9.0`.
-
-5. Create a GitHub release: `gh release create 1.9.0`.

chatmaild/src/chatmaild/expire.py

@@ -14,7 +14,7 @@ from stat import S_ISREG
 
 from chatmaild.config import read_config
 
-FileEntry = namedtuple("FileEntry", ("path", "mtime", "size"))
+FileEntry = namedtuple("FileEntry", ("relpath", "mtime", "size"))
 
 
 def iter_mailboxes(basedir, maxnum):
@@ -51,27 +51,33 @@ class MailboxStat:
 
     def __init__(self, basedir):
         self.basedir = str(basedir)
+        # all detected messages in cur/new/tmp folders
         self.messages = []
-        self.extrafiles = []
-        self.scandir(self.basedir)
 
-    def scandir(self, folderdir):
+        # all detected files in mailbox top dir
-        for name in os_listdir_if_exists(folderdir):
+        self.extrafiles = []
-            path = f"{folderdir}/{name}"
+
+        # scan all relevant files (without recursion)
+        old_cwd = os.getcwd()
+        try:
+            os.chdir(self.basedir)
+        except FileNotFoundError:
+            return
+        for name in os_listdir_if_exists("."):
             if name in ("cur", "new", "tmp"):
-                for msg_name in os_listdir_if_exists(path):
+                for msg_name in os_listdir_if_exists(name):
-                    entry = get_file_entry(f"{path}/{msg_name}")
+                    entry = get_file_entry(f"{name}/{msg_name}")
                     if entry is not None:
                         self.messages.append(entry)
-            elif os.path.isdir(path):
+
-                self.scandir(path)
             else:
-                entry = get_file_entry(path)
+                entry = get_file_entry(name)
                 if entry is not None:
                     self.extrafiles.append(entry)
                     if name == "password":
                         self.last_login = entry.mtime
         self.extrafiles.sort(key=lambda x: -x.size)
+        os.chdir(old_cwd)
 
 
 def print_info(msg):
@@ -124,6 +130,13 @@ class Expiry:
             self.remove_mailbox(mbox.basedir)
             return
 
+        # all to-be-removed files are relative to the mailbox basedir
+        try:
+            os.chdir(mbox.basedir)
+        except FileNotFoundError:
+            print_info(f"mailbox not found/vanished {mbox.basedir}")
+            return
+
         mboxname = os.path.basename(mbox.basedir)
         if self.verbose:
             date = datetime.fromtimestamp(mbox.last_login) if mbox.last_login else None
@@ -134,12 +147,11 @@ class Expiry:
         self.all_files += len(mbox.messages)
         for message in mbox.messages:
             if message.mtime < cutoff_mails:
-                self.remove_file(message.path, mtime=message.mtime)
+                self.remove_file(message.relpath, mtime=message.mtime)
             elif message.size > 200000 and message.mtime < cutoff_large_mails:
                 # we only remove noticed large files (not unnoticed ones in new/)
-                parts = message.path.split("/")
+                if message.relpath.startswith("cur/"):
-                if len(parts) >= 2 and parts[-2] == "cur":
+                    self.remove_file(message.relpath, mtime=message.mtime)
-                    self.remove_file(message.path, mtime=message.mtime)
             else:
                 continue
             changed = True

chatmaild/src/chatmaild/tests/test_expire.py

@@ -17,17 +17,19 @@ from chatmaild.expire import main as expiry_main
 from chatmaild.fsreport import main as report_main
 
 
-def fill_mbox(folderdir):
+def fill_mbox(basedir):
-    password = folderdir.joinpath("password")
+    basedir1 = basedir.joinpath("mailbox1@example.org")
+    basedir1.mkdir()
+    password = basedir1.joinpath("password")
     password.write_text("xxx")
-    folderdir.joinpath("maildirsize").write_text("xxx")
+    basedir1.joinpath("maildirsize").write_text("xxx")
 
-    garbagedir = folderdir.joinpath("garbagedir")
+    garbagedir = basedir1.joinpath("garbagedir")
     garbagedir.mkdir()
-    garbagedir.joinpath("bimbum").write_text("hello")
 
-    create_new_messages(folderdir, ["cur/msg1"], size=500)
+    create_new_messages(basedir1, ["cur/msg1"], size=500)
-    create_new_messages(folderdir, ["new/msg2"], size=600)
+    create_new_messages(basedir1, ["new/msg2"], size=600)
+    return basedir1
 
 
 def create_new_messages(basedir, relpaths, size=1000, days=0):
@@ -43,21 +45,8 @@ def create_new_messages(basedir, relpaths, size=1000, days=0):
 
 @pytest.fixture
 def mbox1(example_config):
-    mboxdir = example_config.mailboxes_dir.joinpath("mailbox1@example.org")
+    basedir1 = fill_mbox(example_config.mailboxes_dir)
-    mboxdir.mkdir()
+    return MailboxStat(basedir1)
-    fill_mbox(mboxdir)
-    return MailboxStat(mboxdir)
-
-
-def test_deltachat_folder(example_config):
-    """Test old setups that might have a .DeltaChat folder where messages also need to get removed."""
-    mboxdir = example_config.mailboxes_dir.joinpath("mailbox1@example.org")
-    mboxdir.mkdir()
-    mbox2dir = mboxdir.joinpath(".DeltaChat")
-    mbox2dir.mkdir()
-    fill_mbox(mbox2dir)
-    mb = MailboxStat(mboxdir)
-    assert len(mb.messages) == 2
 
 
 def test_filentry_ordering(tmp_path):
@@ -87,7 +76,7 @@ def test_stats_mailbox(mbox1):
     create_new_messages(mbox1.basedir, ["large-extra"], size=1000)
     create_new_messages(mbox1.basedir, ["index-something"], size=3)
     mbox2 = MailboxStat(mbox1.basedir)
-    assert len(mbox2.extrafiles) == 5
+    assert len(mbox2.extrafiles) == 4
     assert mbox2.extrafiles[0].size == 1000
 
     # cope well with mailbox dirs that have no password (for whatever reason)

cliff.toml

@@ -1,94 +0,0 @@
-# git-cliff ~ configuration file
-# https://git-cliff.org/docs/configuration
-
-
-[changelog]
-# A Tera template to be rendered for each release in the changelog.
-# See https://keats.github.io/tera/docs/#introduction
-body = """
-{% if version %}\
-    ## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }}
-{% else %}\
-    ## [unreleased]
-{% endif %}\
-{% for group, commits in commits | group_by(attribute="group") %}
-    ### {{ group | striptags | trim | upper_first }}
-    {% for commit in commits %}
-        - {% if commit.scope %}*({{ commit.scope }})* {% endif %}\
-            {% if commit.breaking %}[**breaking**] {% endif %}\
-            {{ commit.message | upper_first }}\
-    {% endfor %}
-{% endfor %}
-"""
-# Remove leading and trailing whitespaces from the changelog's body.
-trim = true
-# Render body even when there are no releases to process.
-render_always = true
-# An array of regex based postprocessors to modify the changelog.
-postprocessors = [
-    # Replace the placeholder <REPO> with a URL.
-    #{ pattern = '<REPO>', replace = "https://github.com/orhun/git-cliff" },
-]
-# render body even when there are no releases to process
-# render_always = true
-# output file path
-# output = "test.md"
-
-[git]
-# Parse commits according to the conventional commits specification.
-# See https://www.conventionalcommits.org
-conventional_commits = true
-# Exclude commits that do not match the conventional commits specification.
-filter_unconventional = true
-# Require all commits to be conventional.
-# Takes precedence over filter_unconventional.
-require_conventional = false
-# Split commits on newlines, treating each line as an individual commit.
-split_commits = false
-# An array of regex based parsers to modify commit messages prior to further processing.
-commit_preprocessors = [
-    # Replace issue numbers with link templates to be updated in `changelog.postprocessors`.
-    #{ pattern = '\((\w+\s)?#([0-9]+)\)', replace = "([#${2}](<REPO>/issues/${2}))"},
-    # Check spelling of the commit message using https://github.com/crate-ci/typos.
-    # If the spelling is incorrect, it will be fixed automatically.
-    #{ pattern = '.*', replace_command = 'typos --write-changes -' },
-]
-# Prevent commits that are breaking from being excluded by commit parsers.
-protect_breaking_commits = false
-# An array of regex based parsers for extracting data from the commit message.
-# Assigns commits to groups.
-# Optionally sets the commit's scope and can decide to exclude commits from further processing.
-commit_parsers = [
-    { message = "^feat", group = "Features" },
-    { message = "^fix", group = "Bug Fixes" },
-    { message = "^docs", group = "Documentation" },
-    { message = "^perf", group = "Performance" },
-    { message = "^refactor", group = "Refactor" },
-    { message = "^style", group = "Styling" },
-    { message = "^test", group = "Testing" },
-    { message = "^chore\\(release\\): prepare for", skip = true },
-    { message = "^chore\\(deps.*\\)", skip = true },
-    { message = "^chore\\(pr\\)", skip = true },
-    { message = "^chore\\(pull\\)", skip = true },
-    { message = "^chore|^ci", group = "Miscellaneous Tasks" },
-    { body = ".*security", group = "Security" },
-    { message = "^revert", group = "Revert" },
-    { message = ".*", group = "Other" },
-]
-# Exclude commits that are not matched by any commit parser.
-filter_commits = false
-# Fail on a commit that is not matched by any commit parser.
-fail_on_unmatched_commit = false
-# An array of link parsers for extracting external references, and turning them into URLs, using regex.
-link_parsers = []
-# Include only the tags that belong to the current branch.
-use_branch_tags = false
-# Order releases topologically instead of chronologically.
-topo_order = false
-# Order commits topologically instead of chronologically.
-topo_order_commits = true
-# Order of commits in each group/release within the changelog.
-# Allowed values: newest, oldest
-sort_commits = "oldest"
-# Process submodules commits
-recurse_submodules = false

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -13,8 +13,6 @@ from cmdeploy.basedeploy import (
 
 
 class DovecotDeployer(Deployer):
-    daemon_reload = False
-
     def __init__(self, config, disable_mail):
         self.config = config
         self.disable_mail = disable_mail
@@ -29,7 +27,7 @@ class DovecotDeployer(Deployer):
 
     def configure(self):
         configure_remote_units(self.config.mail_domain, self.units)
-        self.need_restart, self.daemon_reload = _configure_dovecot(self.config)
+        self.need_restart = _configure_dovecot(self.config)
 
     def activate(self):
         activate_remote_units(self.units)
@@ -44,7 +42,6 @@ class DovecotDeployer(Deployer):
             running=False if self.disable_mail else True,
             enabled=False if self.disable_mail else True,
             restarted=restart,
-            daemon_reload=self.daemon_reload,
         )
         self.need_restart = False
 
@@ -83,10 +80,9 @@ def _install_dovecot_package(package: str, arch: str):
     apt.deb(name=f"Install dovecot-{package}", src=deb_filename)
 
 
-def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
+def _configure_dovecot(config: Config, debug: bool = False) -> bool:
     """Configures Dovecot IMAP server."""
     need_restart = False
-    daemon_reload = False
 
     main_config = files.template(
         src=get_resource("dovecot/dovecot.conf.j2"),
@@ -138,11 +134,4 @@ def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
     )
     need_restart |= timezone_env.changed
 
-    restart_conf = files.put(
+    return need_restart
-        name="dovecot: restart automatically on failure",
-        src=get_resource("service/10_restart.conf"),
-        dest="/etc/systemd/system/dovecot.service.d/10_restart.conf",
-    )
-    daemon_reload |= restart_conf.changed
-
-    return need_restart, daemon_reload

cmdeploy/src/cmdeploy/postfix/deployer.py

@@ -5,7 +5,6 @@ from cmdeploy.basedeploy import Deployer, get_resource
 
 class PostfixDeployer(Deployer):
     required_users = [("postfix", None, ["opendkim"])]
-    daemon_reload = False
 
     def __init__(self, config, disable_mail):
         self.config = config
@@ -61,13 +60,6 @@ class PostfixDeployer(Deployer):
             mode="644",
         )
         need_restart |= login_map.changed
-
-        restart_conf = files.put(
-            name="postfix: restart automatically on failure",
-            src=get_resource("service/10_restart.conf"),
-            dest="/etc/systemd/system/dovecot.service.d/10_restart.conf",
-        )
-        self.daemon_reload = restart_conf.changed
         self.need_restart = need_restart
 
     def activate(self):
@@ -81,6 +73,5 @@ class PostfixDeployer(Deployer):
             running=False if self.disable_mail else True,
             enabled=False if self.disable_mail else True,
             restarted=restart,
-            daemon_reload=self.daemon_reload,
         )
         self.need_restart = False

cmdeploy/src/cmdeploy/service/10_restart.conf

@@ -1,3 +0,0 @@
-[Service]
-Restart=always
-RestartSec=30