chore(release): prepare for 1.9.0

I am running git-cliff 2.11.0.
Ran `git-cliff --init` to generate `cliff.toml`.
Removed emojis, replaced `doc` with `docs` to match chatmail core
convention.

.github/workflows/docs-preview.yaml

@@ -11,6 +11,9 @@ jobs:
   scripts:
     name: build 
     runs-on: ubuntu-latest
+    environment:
+      name: 'staging.chatmail.at/doc/relay/'
+      url: https://staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}
     steps:
       - uses: actions/checkout@v4
 
@@ -44,36 +47,6 @@ jobs:
           chmod 600 "$HOME/.ssh/key"
           rsync -rILvh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/doc/build/ "${{ secrets.USERNAME }}@chatmail.at:/var/www/html/staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}/"
 
-      - name: "Post links to details"
-        id: details
-        if: steps.prepare.outputs.uploadtoserver
-        run: |
-          # URLs for API connection and uploads
-          export GITHUB_API_URL="https://api.github.com/repos/chatmail/relay/statuses/${{ github.event.after }}"
-          export PREVIEW_LINK="https://staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}/"
-          export STATUS_DATA="{\"state\": \"success\", \
-                               \"description\": \"Preview the changed documentation here:\", \
-                               \"context\": \"Documentation Preview\", \
-                               \"target_url\": \"${PREVIEW_LINK}\"}"
-          curl -X POST --header "Accept: application/vnd.github+json" --header "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" --url "$GITHUB_API_URL" --header "content-type: application/json" --data "$STATUS_DATA"
-
-          #check if comment already exists, if not post it
-          export GITHUB_API_URL="https://api.github.com/repos/chatmail/relay/issues/${{ steps.prepare.outputs.prid }}/comments"
-          export RESPONSE=$(curl -L --header "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" --url "$GITHUB_API_URL" --header "content-type: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28")
-          echo $RESPONSE > response
-          grep -v '"Check out the page preview at https://staging.chatmail.at/doc/relay' response && echo "comment=true" >> $GITHUB_OUTPUT || true
-      - name: "Post link to comments"
-        if: steps.details.outputs.comment
-        uses: actions/github-script@v7
-        with:
-          script: |
-            github.rest.issues.createComment({
-              issue_number: context.issue.number,
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              body: "Check out the page preview at https://staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}/"
-            })
-
       - name: check links 
         working-directory: doc
         run: sphinx-build --builder linkcheck source build

.github/workflows/docs.yaml

@@ -14,6 +14,9 @@ jobs:
   scripts:
     name: build 
     runs-on: ubuntu-latest
+    environment: 
+      name: 'chatmail.at/doc/relay/'
+      url: https://chatmail.at/doc/relay/
     steps:
       - uses: actions/checkout@v4
 

.github/workflows/test-and-deploy-ipv4only.yaml

@@ -16,6 +16,9 @@ jobs:
     name: deploy on staging-ipv4.testrun.org, and run tests
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    environment:
+      name: staging-ipv4.testrun.org
+      url: https://staging-ipv4.testrun.org/
     concurrency:
       group: ci-ipv4-${{ github.workflow }}-${{ github.ref }}
       cancel-in-progress: ${{ !contains(github.ref, '$GITHUB_REF') }}
@@ -90,7 +93,7 @@ jobs:
           ssh root@ns.testrun.org systemctl reload nsd
 
       - name: cmdeploy test
-        run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow
+        run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
 
       - name: cmdeploy dns
         run: cmdeploy dns -v

.github/workflows/test-and-deploy.yaml

@@ -16,6 +16,9 @@ jobs:
     name: deploy on staging2.testrun.org, and run tests
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    environment:
+      name: staging2.testrun.org
+      url: https://staging2.testrun.org/
     concurrency:
       group: ci-${{ github.workflow }}-${{ github.ref }}
       cancel-in-progress: ${{ !contains(github.ref, '$GITHUB_REF') }}
@@ -70,6 +73,9 @@ jobs:
           rsync -avz dkimkeys-restore/dkimkeys root@staging2.testrun.org:/etc/ || true
           ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org chown root:root -R /var/lib/acme || true
 
+      - name: add hpk42 key to staging server
+        run: ssh root@staging2.testrun.org 'curl -s https://github.com/hpk42.keys >> .ssh/authorized_keys'
+
       - name: run deploy-chatmail offline tests 
         run: pytest --pyargs cmdeploy 
 
@@ -88,7 +94,7 @@ jobs:
           ssh root@ns.testrun.org systemctl reload nsd
 
       - name: cmdeploy test
-        run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow
+        run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
 
       - name: cmdeploy dns
         run: cmdeploy dns -v

CHANGELOG.md

@@ -1,6 +1,50 @@
 # Changelog for chatmail deployment 
 
-## untagged
+## 1.9.0 2025-12-18
+
+### Documentation
+
+- Add RELEASE.md and CONTRIBUTING.md
+- README update, mention Chatmail Cookbook project
+
+### Bug Fixes
+
+- Expire messages also from IMAP subfolders
+- Use absolute path instead of relative path in message expiration script
+- Restart Postfix and Dovecot automatically on failure
+- acmetool: Use a fixed name and `reconcile` instead of `want`
+
+### Features
+
+- Report DKIM error code in SMTP response
+- Remove development notice from the web pages
+
+### Miscellaneous Tasks
+
+- Update the heading in the CHANGELOG.md
+- Setup git-cliff
+- Run tests against ci-chatmail.testrun.org instead of nine.testrun.org
+- Cleanup remaining echobot code, remove echobot user from deployment and passthrough recipients
+
+## 1.8.0 2025-12-12
+
+- Add imap_compress option to chatmail.ini
+  ([#760](https://github.com/chatmail/relay/pull/760))
+
+- Remove echobot from relays 
+  ([#753](https://github.com/chatmail/relay/pull/753))
+
+- Fix `cmdeploy webdev`
+  ([#743](https://github.com/chatmail/relay/pull/743))
+
+- Add robots.txt to exclude all web crawlers
+  ([#732](https://github.com/chatmail/relay/pull/732))
+
+- acmetool: accept new Let's Encrypt ToS: https://letsencrypt.org/documents/LE-SA-v1.6-August-18-2025.pdf
+  ([#729](https://github.com/chatmail/relay/pull/729))
+
+- Organized cmdeploy into install, configure, and activate stages
+  ([#695](https://github.com/chatmail/relay/pull/695))
 
 - docs: move readme.md docs to sphinx documentation rendered at https://chatmail.at/doc/relay 
   ([#711](https://github.com/chatmail/relay/pull/711))
@@ -18,10 +62,10 @@
   ([#689](https://github.com/chatmail/relay/pull/689))
 
 - Require TLS 1.2 for outgoing SMTP connections
-  ([#685](https://github.com/chatmail/relay/pull/685))
+  ([#685](https://github.com/chatmail/relay/pull/685), [#730](https://github.com/chatmail/relay/pull/730))
 
 - require STARTTLS for incoming port 25 connections
-  ([#684](https://github.com/chatmail/relay/pull/684))
+  ([#684](https://github.com/chatmail/relay/pull/684), [#730](https://github.com/chatmail/relay/pull/730))
 
 - filtermail: run CPU-intensive handle_DATA in a thread pool executor
   ([#676](https://github.com/chatmail/relay/pull/676))

CONTRIBUTING.md

@@ -0,0 +1,7 @@
+# Contributing to the chatmail relay
+
+Commit messages follow the [Conventional Commits] notation.
+We use [git-cliff] to generate the changelog from commit messages before the release.
+
+[Conventional Commits]: https://www.conventionalcommits.org/
+[git-cliff]: https://git-cliff.org/

RELEASE.md

@@ -0,0 +1,15 @@
+# Releasing a new version of chatmail relay
+
+For example, to release version 1.9.0 of chatmail relay, do the following steps.
+
+1. Update the changelog: `git cliff --unreleased --tag 1.9.0 --prepend CHANGELOG.md` or `git cliff -u -t 1.9.0 -p CHANGELOG.md`.
+
+2. Open the changelog in the editor, edit it if required.
+
+3. Commit the changes to the changelog with a commit message `chore(release): prepare for 1.9.0`.
+
+3. Tag the release: `git tag --annotate 1.9.0`.
+
+4. Push the release tag: `git push origin 1.9.0`.
+
+5. Create a GitHub release: `gh release create 1.9.0`.

chatmaild/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "chatmaild"
-version = "0.2"
+version = "0.3"
 dependencies = [
   "aiosmtpd",
   "iniconfig",
@@ -25,7 +25,6 @@ where = ['src']
 doveauth = "chatmaild.doveauth:main"
 chatmail-metadata = "chatmaild.metadata:main"
 filtermail = "chatmaild.filtermail:main"
-echobot = "chatmaild.echo:main"
 chatmail-metrics = "chatmaild.metrics:main"
 chatmail-expire = "chatmaild.expire:main"
 chatmail-fsreport = "chatmaild.fsreport:main"
@@ -73,5 +72,6 @@ commands =
 deps = pytest 
        pdbpp 
        pytest-localserver
+       execnet
 commands = pytest -v -rsXx {posargs}
 """

chatmaild/src/chatmaild/config.py

@@ -4,8 +4,6 @@ import iniconfig
 
 from chatmaild.user import User
 
-echobot_password_path = Path("/run/echobot/password")
-
 
 def read_config(inipath):
     assert Path(inipath).exists(), inipath
@@ -46,6 +44,7 @@ class Config:
         self.disable_ipv6 = params.get("disable_ipv6", "false").lower() == "true"
         self.acme_email = params.get("acme_email", "")
         self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true"
+        self.imap_compress = params.get("imap_compress", "false").lower() == "true"
         if "iroh_relay" not in params:
             self.iroh_relay = "https://" + params["mail_domain"]
             self.enable_iroh_relay = True
@@ -72,10 +71,7 @@ class Config:
             raise ValueError(f"invalid address {addr!r}")
 
         maildir = self.mailboxes_dir.joinpath(addr)
-        if addr.startswith("echo@"):
-            password_path = echobot_password_path
-        else:
-            password_path = maildir.joinpath("password")
+        password_path = maildir.joinpath("password")
 
         return User(maildir, addr, password_path, uid="vmail", gid="vmail")
 

chatmaild/src/chatmaild/doveauth.py

@@ -40,10 +40,6 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
         return False
     localpart, domain = parts
 
-    if localpart == "echo":
-        # echobot account should not be created in the database
-        return False
-
     if (
         len(localpart) > config.username_max_length
         or len(localpart) < config.username_min_length

chatmaild/src/chatmaild/echo.py

@@ -1,109 +0,0 @@
-#!/usr/bin/env python3
-"""Advanced echo bot example.
-
-it will echo back any message that has non-empty text and also supports the /help command.
-"""
-
-import logging
-import os
-import subprocess
-import sys
-from pathlib import Path
-
-from deltachat_rpc_client import Bot, DeltaChat, EventType, Rpc, events
-
-from chatmaild.config import echobot_password_path, read_config
-from chatmaild.doveauth import encrypt_password
-from chatmaild.newemail import create_newemail_dict
-
-hooks = events.HookCollection()
-
-
-@hooks.on(events.RawEvent)
-def log_event(event):
-    if event.kind == EventType.INFO:
-        logging.info(event.msg)
-    elif event.kind == EventType.WARNING:
-        logging.warning(event.msg)
-
-
-@hooks.on(events.RawEvent(EventType.ERROR))
-def log_error(event):
-    logging.error("%s", event.msg)
-
-
-@hooks.on(events.MemberListChanged)
-def on_memberlist_changed(event):
-    logging.info(
-        "member %s was %s", event.member, "added" if event.member_added else "removed"
-    )
-
-
-@hooks.on(events.GroupImageChanged)
-def on_group_image_changed(event):
-    logging.info("group image %s", "deleted" if event.image_deleted else "changed")
-
-
-@hooks.on(events.GroupNameChanged)
-def on_group_name_changed(event):
-    logging.info(f"group name changed, old name: {event.old_name}")
-
-
-@hooks.on(events.NewMessage(func=lambda e: not e.command))
-def echo(event):
-    snapshot = event.message_snapshot
-    if snapshot.is_info:
-        # Ignore info messages
-        return
-    if snapshot.text or snapshot.file:
-        snapshot.chat.send_message(text=snapshot.text, file=snapshot.file)
-
-
-@hooks.on(events.NewMessage(command="/help"))
-def help_command(event):
-    snapshot = event.message_snapshot
-    snapshot.chat.send_text("Send me any message and I will echo it back")
-
-
-def main():
-    logging.basicConfig(level=logging.INFO)
-    path = os.environ.get("PATH")
-    venv_path = sys.argv[0].strip("echobot")
-    os.environ["PATH"] = path + ":" + venv_path
-    with Rpc() as rpc:
-        deltachat = DeltaChat(rpc)
-        system_info = deltachat.get_system_info()
-        logging.info(f"Running deltachat core {system_info.deltachat_core_version}")
-
-        accounts = deltachat.get_all_accounts()
-        account = accounts[0] if accounts else deltachat.add_account()
-
-        bot = Bot(account, hooks)
-
-        config = read_config(sys.argv[1])
-        addr = "echo@" + config.mail_domain
-
-        # Create password file
-        if bot.is_configured():
-            password = bot.account.get_config("mail_pw")
-        else:
-            password = create_newemail_dict(config)["password"]
-
-        echobot_password_path.write_text(encrypt_password(password))
-        # Give the user which doveauth runs as access to the password file.
-        subprocess.check_call(
-            ["/usr/bin/setfacl", "-m", "user:vmail:r", echobot_password_path],
-        )
-
-        if not bot.is_configured():
-            bot.configure(addr, password)
-
-        # write invite link to working directory
-        invitelink = bot.account.get_qr_code()
-        Path("invite-link.txt").write_text(invitelink)
-
-        bot.run_forever()
-
-
-if __name__ == "__main__":
-    main()

chatmaild/src/chatmaild/expire.py

@@ -14,7 +14,7 @@ from stat import S_ISREG
 
 from chatmaild.config import read_config
 
-FileEntry = namedtuple("FileEntry", ("relpath", "mtime", "size"))
+FileEntry = namedtuple("FileEntry", ("path", "mtime", "size"))
 
 
 def iter_mailboxes(basedir, maxnum):
@@ -51,33 +51,27 @@ class MailboxStat:
 
     def __init__(self, basedir):
         self.basedir = str(basedir)
-        # all detected messages in cur/new/tmp folders
         self.messages = []
-
-        # all detected files in mailbox top dir
         self.extrafiles = []
+        self.scandir(self.basedir)
 
-        # scan all relevant files (without recursion)
-        old_cwd = os.getcwd()
-        try:
-            os.chdir(self.basedir)
-        except FileNotFoundError:
-            return
-        for name in os_listdir_if_exists("."):
+    def scandir(self, folderdir):
+        for name in os_listdir_if_exists(folderdir):
+            path = f"{folderdir}/{name}"
             if name in ("cur", "new", "tmp"):
-                for msg_name in os_listdir_if_exists(name):
-                    entry = get_file_entry(f"{name}/{msg_name}")
+                for msg_name in os_listdir_if_exists(path):
+                    entry = get_file_entry(f"{path}/{msg_name}")
                     if entry is not None:
                         self.messages.append(entry)
-
+            elif os.path.isdir(path):
+                self.scandir(path)
             else:
-                entry = get_file_entry(name)
+                entry = get_file_entry(path)
                 if entry is not None:
                     self.extrafiles.append(entry)
                     if name == "password":
                         self.last_login = entry.mtime
         self.extrafiles.sort(key=lambda x: -x.size)
-        os.chdir(old_cwd)
 
 
 def print_info(msg):
@@ -130,13 +124,6 @@ class Expiry:
             self.remove_mailbox(mbox.basedir)
             return
 
-        # all to-be-removed files are relative to the mailbox basedir
-        try:
-            os.chdir(mbox.basedir)
-        except FileNotFoundError:
-            print_info(f"mailbox not found/vanished {mbox.basedir}")
-            return
-
         mboxname = os.path.basename(mbox.basedir)
         if self.verbose:
             date = datetime.fromtimestamp(mbox.last_login) if mbox.last_login else None
@@ -147,11 +134,12 @@ class Expiry:
         self.all_files += len(mbox.messages)
         for message in mbox.messages:
             if message.mtime < cutoff_mails:
-                self.remove_file(message.relpath, mtime=message.mtime)
+                self.remove_file(message.path, mtime=message.mtime)
             elif message.size > 200000 and message.mtime < cutoff_large_mails:
                 # we only remove noticed large files (not unnoticed ones in new/)
-                if message.relpath.startswith("cur/"):
-                    self.remove_file(message.relpath, mtime=message.mtime)
+                parts = message.path.split("/")
+                if len(parts) >= 2 and parts[-2] == "cur":
+                    self.remove_file(message.path, mtime=message.mtime)
             else:
                 continue
             changed = True

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -43,9 +43,9 @@ passthrough_senders =
 
 # list of e-mail recipients for which to accept outbound un-encrypted mails
 # (space-separated, item may start with "@" to whitelist whole recipient domains)
-passthrough_recipients = echo@{mail_domain}
+passthrough_recipients =
 
-# path to www directory - documented here: https://github.com/chatmail/relay/#custom-web-pages
+# path to www directory - documented here: https://chatmail.at/doc/relay/getting_started.html#custom-web-pages
 #www_folder = www
 
 #
@@ -99,6 +99,12 @@ acme_email =
 # so use this option with caution on production servers. 
 imap_rawlog = false 
 
+# set to true if you want to enable the IMAP COMPRESS Extension,
+# which allows IMAP connections to be efficiently compressed.
+# WARNING: Enabling this makes it impossible to hibernate IMAP
+# processes which will result in much higher memory/RAM usage.
+imap_compress = false
+
 
 #
 # Privacy Policy

chatmaild/src/chatmaild/lastlogin.py

@@ -13,8 +13,6 @@ class LastLoginDictProxy(DictProxy):
         keyname = parts[1].split("/")
         value = parts[2] if len(parts) > 2 else ""
         if keyname[0] == "shared" and keyname[1] == "last-login":
-            if addr.startswith("echo@"):
-                return True
             addr = keyname[2]
             timestamp = int(value)
             user = self.config.get_user(addr)

chatmaild/src/chatmaild/tests/test_expire.py

@@ -17,19 +17,17 @@ from chatmaild.expire import main as expiry_main
 from chatmaild.fsreport import main as report_main
 
 
-def fill_mbox(basedir):
-    basedir1 = basedir.joinpath("mailbox1@example.org")
-    basedir1.mkdir()
-    password = basedir1.joinpath("password")
+def fill_mbox(folderdir):
+    password = folderdir.joinpath("password")
     password.write_text("xxx")
-    basedir1.joinpath("maildirsize").write_text("xxx")
+    folderdir.joinpath("maildirsize").write_text("xxx")
 
-    garbagedir = basedir1.joinpath("garbagedir")
+    garbagedir = folderdir.joinpath("garbagedir")
     garbagedir.mkdir()
+    garbagedir.joinpath("bimbum").write_text("hello")
 
-    create_new_messages(basedir1, ["cur/msg1"], size=500)
-    create_new_messages(basedir1, ["new/msg2"], size=600)
-    return basedir1
+    create_new_messages(folderdir, ["cur/msg1"], size=500)
+    create_new_messages(folderdir, ["new/msg2"], size=600)
 
 
 def create_new_messages(basedir, relpaths, size=1000, days=0):
@@ -45,8 +43,21 @@ def create_new_messages(basedir, relpaths, size=1000, days=0):
 
 @pytest.fixture
 def mbox1(example_config):
-    basedir1 = fill_mbox(example_config.mailboxes_dir)
-    return MailboxStat(basedir1)
+    mboxdir = example_config.mailboxes_dir.joinpath("mailbox1@example.org")
+    mboxdir.mkdir()
+    fill_mbox(mboxdir)
+    return MailboxStat(mboxdir)
+
+
+def test_deltachat_folder(example_config):
+    """Test old setups that might have a .DeltaChat folder where messages also need to get removed."""
+    mboxdir = example_config.mailboxes_dir.joinpath("mailbox1@example.org")
+    mboxdir.mkdir()
+    mbox2dir = mboxdir.joinpath(".DeltaChat")
+    mbox2dir.mkdir()
+    fill_mbox(mbox2dir)
+    mb = MailboxStat(mboxdir)
+    assert len(mb.messages) == 2
 
 
 def test_filentry_ordering(tmp_path):
@@ -76,7 +87,7 @@ def test_stats_mailbox(mbox1):
     create_new_messages(mbox1.basedir, ["large-extra"], size=1000)
     create_new_messages(mbox1.basedir, ["index-something"], size=3)
     mbox2 = MailboxStat(mbox1.basedir)
-    assert len(mbox2.extrafiles) == 4
+    assert len(mbox2.extrafiles) == 5
     assert mbox2.extrafiles[0].size == 1000
 
     # cope well with mailbox dirs that have no password (for whatever reason)

chatmaild/src/chatmaild/tests/test_lastlogin.py

@@ -36,29 +36,3 @@ def test_handle_dovecot_request_last_login(testaddr, example_config):
     res = dictproxy.handle_dovecot_request(msg, dictproxy_transactions)
     assert res == "O\n"
     assert len(dictproxy_transactions) == 0
-
-
-def test_handle_dovecot_request_last_login_echobot(example_config):
-    dictproxy = LastLoginDictProxy(config=example_config)
-
-    authproxy = AuthDictProxy(config=example_config)
-    testaddr = f"echo@{example_config.mail_domain}"
-    authproxy.lookup_passdb(testaddr, "ignore")
-    user = dictproxy.config.get_user(testaddr)
-
-    transactions = {}
-
-    # set last-login info for user
-    tx = "1111"
-    msg = f"B{tx}\t{testaddr}"
-    res = dictproxy.handle_dovecot_request(msg, transactions)
-    assert not res
-    assert transactions == {tx: dict(addr=testaddr, res="O\n")}
-
-    timestamp = int(time.time())
-    msg = f"S{tx}\tshared/last-login/{testaddr}\t{timestamp}"
-    res = dictproxy.handle_dovecot_request(msg, transactions)
-    assert not res
-    assert len(transactions) == 1
-    read_timestamp = user.get_last_login_timestamp()
-    assert read_timestamp is None

chatmaild/src/chatmaild/user.py

@@ -19,7 +19,7 @@ class User:
 
     @property
     def can_track(self):
-        return "@" in self.addr and not self.addr.startswith("echo@")
+        return "@" in self.addr
 
     def get_userdb_dict(self):
         """Return a non-empty dovecot 'userdb' style dict
@@ -55,11 +55,9 @@ class User:
         try:
             write_bytes_atomic(self.password_path, password)
         except PermissionError:
-            if not self.addr.startswith("echo@"):
-                logging.error(f"could not write password for: {self.addr}")
-                raise
-        if not self.addr.startswith("echo@"):
-            self.enforce_E2EE_path.touch()
+            logging.error(f"could not write password for: {self.addr}")
+            raise
+        self.enforce_E2EE_path.touch()
 
     def set_last_login_timestamp(self, timestamp):
         """Track login time with daily granularity

cliff.toml

@@ -0,0 +1,94 @@
+# git-cliff ~ configuration file
+# https://git-cliff.org/docs/configuration
+
+
+[changelog]
+# A Tera template to be rendered for each release in the changelog.
+# See https://keats.github.io/tera/docs/#introduction
+body = """
+{% if version %}\
+    ## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }}
+{% else %}\
+    ## [unreleased]
+{% endif %}\
+{% for group, commits in commits | group_by(attribute="group") %}
+    ### {{ group | striptags | trim | upper_first }}
+    {% for commit in commits %}
+        - {% if commit.scope %}*({{ commit.scope }})* {% endif %}\
+            {% if commit.breaking %}[**breaking**] {% endif %}\
+            {{ commit.message | upper_first }}\
+    {% endfor %}
+{% endfor %}
+"""
+# Remove leading and trailing whitespaces from the changelog's body.
+trim = true
+# Render body even when there are no releases to process.
+render_always = true
+# An array of regex based postprocessors to modify the changelog.
+postprocessors = [
+    # Replace the placeholder <REPO> with a URL.
+    #{ pattern = '<REPO>', replace = "https://github.com/orhun/git-cliff" },
+]
+# render body even when there are no releases to process
+# render_always = true
+# output file path
+# output = "test.md"
+
+[git]
+# Parse commits according to the conventional commits specification.
+# See https://www.conventionalcommits.org
+conventional_commits = true
+# Exclude commits that do not match the conventional commits specification.
+filter_unconventional = true
+# Require all commits to be conventional.
+# Takes precedence over filter_unconventional.
+require_conventional = false
+# Split commits on newlines, treating each line as an individual commit.
+split_commits = false
+# An array of regex based parsers to modify commit messages prior to further processing.
+commit_preprocessors = [
+    # Replace issue numbers with link templates to be updated in `changelog.postprocessors`.
+    #{ pattern = '\((\w+\s)?#([0-9]+)\)', replace = "([#${2}](<REPO>/issues/${2}))"},
+    # Check spelling of the commit message using https://github.com/crate-ci/typos.
+    # If the spelling is incorrect, it will be fixed automatically.
+    #{ pattern = '.*', replace_command = 'typos --write-changes -' },
+]
+# Prevent commits that are breaking from being excluded by commit parsers.
+protect_breaking_commits = false
+# An array of regex based parsers for extracting data from the commit message.
+# Assigns commits to groups.
+# Optionally sets the commit's scope and can decide to exclude commits from further processing.
+commit_parsers = [
+    { message = "^feat", group = "Features" },
+    { message = "^fix", group = "Bug Fixes" },
+    { message = "^docs", group = "Documentation" },
+    { message = "^perf", group = "Performance" },
+    { message = "^refactor", group = "Refactor" },
+    { message = "^style", group = "Styling" },
+    { message = "^test", group = "Testing" },
+    { message = "^chore\\(release\\): prepare for", skip = true },
+    { message = "^chore\\(deps.*\\)", skip = true },
+    { message = "^chore\\(pr\\)", skip = true },
+    { message = "^chore\\(pull\\)", skip = true },
+    { message = "^chore|^ci", group = "Miscellaneous Tasks" },
+    { body = ".*security", group = "Security" },
+    { message = "^revert", group = "Revert" },
+    { message = ".*", group = "Other" },
+]
+# Exclude commits that are not matched by any commit parser.
+filter_commits = false
+# Fail on a commit that is not matched by any commit parser.
+fail_on_unmatched_commit = false
+# An array of link parsers for extracting external references, and turning them into URLs, using regex.
+link_parsers = []
+# Include only the tags that belong to the current branch.
+use_branch_tags = false
+# Order releases topologically instead of chronologically.
+topo_order = false
+# Order commits topologically instead of chronologically.
+topo_order_commits = true
+# Order of commits in each group/release within the changelog.
+# Allowed values: newest, oldest
+sort_commits = "oldest"
+# Process submodules commits
+recurse_submodules = false

cmdeploy/src/cmdeploy/__init__.py

@@ -1,941 +0,0 @@
-"""
-Chat Mail pyinfra deploy.
-"""
-
-import importlib.resources
-import io
-import shutil
-import subprocess
-import sys
-from io import StringIO
-from pathlib import Path
-
-from chatmaild.config import Config, read_config
-from pyinfra import facts, host, logger
-from pyinfra.api import FactBase
-from pyinfra.facts.files import File, Sha256File
-from pyinfra.facts.server import Sysctl
-from pyinfra.facts.systemd import SystemdEnabled
-from pyinfra.operations import apt, files, pip, server, systemd
-
-from .acmetool import deploy_acmetool
-
-
-class Port(FactBase):
-    """
-    Returns the process occuping a port.
-    """
-
-    def command(self, port: int) -> str:
-        return (
-            "ss -lptn 'src :%d' | awk 'NR>1 {print $6,$7}' | sed 's/users:((\"//;s/\".*//'"
-            % (port,)
-        )
-
-    def process(self, output: [str]) -> str:
-        return output[0]
-
-
-def _build_chatmaild(dist_dir) -> None:
-    dist_dir = Path(dist_dir).resolve()
-    if dist_dir.exists():
-        shutil.rmtree(dist_dir)
-    dist_dir.mkdir()
-    subprocess.check_output(
-        [sys.executable, "-m", "build", "-n"]
-        + ["--sdist", "chatmaild", "--outdir", str(dist_dir)]
-    )
-    entries = list(dist_dir.iterdir())
-    assert len(entries) == 1
-    return entries[0]
-
-
-def remove_legacy_artifacts():
-    # disable legacy doveauth-dictproxy.service
-    if host.get_fact(SystemdEnabled).get("doveauth-dictproxy.service"):
-        systemd.service(
-            name="Disable legacy doveauth-dictproxy.service",
-            service="doveauth-dictproxy.service",
-            running=False,
-            enabled=False,
-        )
-
-
-def _install_remote_venv_with_chatmaild(config) -> None:
-    remove_legacy_artifacts()
-    dist_file = _build_chatmaild(dist_dir=Path("chatmaild/dist"))
-    remote_base_dir = "/usr/local/lib/chatmaild"
-    remote_dist_file = f"{remote_base_dir}/dist/{dist_file.name}"
-    remote_venv_dir = f"{remote_base_dir}/venv"
-    remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini"
-    root_owned = dict(user="root", group="root", mode="644")
-
-    apt.packages(
-        name="apt install python3-virtualenv",
-        packages=["python3-virtualenv"],
-    )
-
-    files.put(
-        name="Upload chatmaild source package",
-        src=dist_file.open("rb"),
-        dest=remote_dist_file,
-        create_remote_dir=True,
-        **root_owned,
-    )
-
-    files.put(
-        name=f"Upload {remote_chatmail_inipath}",
-        src=config._getbytefile(),
-        dest=remote_chatmail_inipath,
-        **root_owned,
-    )
-
-    pip.virtualenv(
-        name=f"chatmaild virtualenv {remote_venv_dir}",
-        path=remote_venv_dir,
-        always_copy=True,
-    )
-
-    apt.packages(
-        name="install gcc and headers to build crypt_r source package",
-        packages=["gcc", "python3-dev"],
-    )
-
-    server.shell(
-        name=f"forced pip-install {dist_file.name}",
-        commands=[
-            f"{remote_venv_dir}/bin/pip install --force-reinstall {remote_dist_file}"
-        ],
-    )
-
-    files.template(
-        src=importlib.resources.files(__package__).joinpath("metrics.cron.j2"),
-        dest="/etc/cron.d/chatmail-metrics",
-        user="root",
-        group="root",
-        mode="644",
-        config={
-            "mailboxes_dir": config.mailboxes_dir,
-            "execpath": f"{remote_venv_dir}/bin/chatmail-metrics",
-        },
-    )
-
-    # install systemd units
-    for fn in (
-        "doveauth",
-        "filtermail",
-        "filtermail-incoming",
-        "echobot",
-        "chatmail-metadata",
-        "lastlogin",
-        "turnserver",
-        "chatmail-expire",
-        "chatmail-expire.timer",
-        "chatmail-fsreport",
-        "chatmail-fsreport.timer",
-    ):
-        execpath = fn if fn != "filtermail-incoming" else "filtermail"
-        params = dict(
-            execpath=f"{remote_venv_dir}/bin/{execpath}",
-            config_path=remote_chatmail_inipath,
-            remote_venv_dir=remote_venv_dir,
-            mail_domain=config.mail_domain,
-        )
-
-        basename = fn if "." in fn else f"{fn}.service"
-
-        source_path = importlib.resources.files(__package__).joinpath("service", f"{basename}.f")
-        content = source_path.read_text().format(**params).encode()
-
-        files.put(
-            name=f"Upload {basename}",
-            src=io.BytesIO(content),
-            dest=f"/etc/systemd/system/{basename}",
-            **root_owned,
-        )
-        if fn == "chatmail-expire" or fn == "chatmail-fsreport":
-            # don't auto-start but let the corresponding timer trigger execution
-            enabled = False
-        else:
-            enabled = True
-        systemd.service(
-            name=f"Setup {basename}",
-            service=basename,
-            running=enabled,
-            enabled=enabled,
-            restarted=enabled,
-            daemon_reload=True,
-        )
-
-
-
-def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
-    """Configures OpenDKIM"""
-    need_restart = False
-
-    main_config = files.template(
-        src=importlib.resources.files(__package__).joinpath("opendkim/opendkim.conf"),
-        dest="/etc/opendkim.conf",
-        user="root",
-        group="root",
-        mode="644",
-        config={"domain_name": domain, "opendkim_selector": dkim_selector},
-    )
-    need_restart |= main_config.changed
-
-    screen_script = files.put(
-        src=importlib.resources.files(__package__).joinpath("opendkim/screen.lua"),
-        dest="/etc/opendkim/screen.lua",
-        user="root",
-        group="root",
-        mode="644",
-    )
-    need_restart |= screen_script.changed
-
-    final_script = files.put(
-        src=importlib.resources.files(__package__).joinpath("opendkim/final.lua"),
-        dest="/etc/opendkim/final.lua",
-        user="root",
-        group="root",
-        mode="644",
-    )
-    need_restart |= final_script.changed
-
-    files.directory(
-        name="Add opendkim directory to /etc",
-        path="/etc/opendkim",
-        user="opendkim",
-        group="opendkim",
-        mode="750",
-        present=True,
-    )
-
-    keytable = files.template(
-        src=importlib.resources.files(__package__).joinpath("opendkim/KeyTable"),
-        dest="/etc/dkimkeys/KeyTable",
-        user="opendkim",
-        group="opendkim",
-        mode="644",
-        config={"domain_name": domain, "opendkim_selector": dkim_selector},
-    )
-    need_restart |= keytable.changed
-
-    signing_table = files.template(
-        src=importlib.resources.files(__package__).joinpath("opendkim/SigningTable"),
-        dest="/etc/dkimkeys/SigningTable",
-        user="opendkim",
-        group="opendkim",
-        mode="644",
-        config={"domain_name": domain, "opendkim_selector": dkim_selector},
-    )
-    need_restart |= signing_table.changed
-    files.directory(
-        name="Add opendkim socket directory to /var/spool/postfix",
-        path="/var/spool/postfix/opendkim",
-        user="opendkim",
-        group="opendkim",
-        mode="750",
-        present=True,
-    )
-
-    apt.packages(
-        name="apt install opendkim opendkim-tools",
-        packages=["opendkim", "opendkim-tools"],
-    )
-
-    if not host.get_fact(File, f"/etc/dkimkeys/{dkim_selector}.private"):
-        server.shell(
-            name="Generate OpenDKIM domain keys",
-            commands=[
-                f"/usr/sbin/opendkim-genkey -D /etc/dkimkeys -d {domain} -s {dkim_selector}"
-            ],
-            _use_su_login=True,
-            _su_user="opendkim",
-        )
-
-    service_file = files.put(
-        name="Configure opendkim to restart once a day",
-        src=importlib.resources.files(__package__).joinpath("opendkim/systemd.conf"),
-        dest="/etc/systemd/system/opendkim.service.d/10-prevent-memory-leak.conf",
-    )
-    need_restart |= service_file.changed
-
-    return need_restart
-
-
-def _uninstall_mta_sts_daemon() -> None:
-    # Remove configuration.
-    files.file("/etc/mta-sts-daemon.yml", present=False)
-
-    files.directory("/usr/local/lib/postfix-mta-sts-resolver", present=False)
-
-    files.file("/etc/systemd/system/mta-sts-daemon.service", present=False)
-
-    systemd.service(
-        name="Stop MTA-STS daemon",
-        service="mta-sts-daemon.service",
-        daemon_reload=True,
-        running=False,
-        enabled=False,
-    )
-
-
-def _configure_postfix(config: Config, debug: bool = False) -> bool:
-    """Configures Postfix SMTP server."""
-    need_restart = False
-
-    main_config = files.template(
-        src=importlib.resources.files(__package__).joinpath("postfix/main.cf.j2"),
-        dest="/etc/postfix/main.cf",
-        user="root",
-        group="root",
-        mode="644",
-        config=config,
-        disable_ipv6=config.disable_ipv6,
-    )
-    need_restart |= main_config.changed
-
-    master_config = files.template(
-        src=importlib.resources.files(__package__).joinpath("postfix/master.cf.j2"),
-        dest="/etc/postfix/master.cf",
-        user="root",
-        group="root",
-        mode="644",
-        debug=debug,
-        config=config,
-    )
-    need_restart |= master_config.changed
-
-    header_cleanup = files.put(
-        src=importlib.resources.files(__package__).joinpath(
-            "postfix/submission_header_cleanup"
-        ),
-        dest="/etc/postfix/submission_header_cleanup",
-        user="root",
-        group="root",
-        mode="644",
-    )
-    need_restart |= header_cleanup.changed
-
-    # Login map that 1:1 maps email address to login.
-    login_map = files.put(
-        src=importlib.resources.files(__package__).joinpath("postfix/login_map"),
-        dest="/etc/postfix/login_map",
-        user="root",
-        group="root",
-        mode="644",
-    )
-    need_restart |= login_map.changed
-
-    return need_restart
-
-
-def _install_dovecot_package(package: str, arch: str):
-    arch = "amd64" if arch == "x86_64" else arch
-    arch = "arm64" if arch == "aarch64" else arch
-    url = f"https://download.delta.chat/dovecot/dovecot-{package}_2.3.21%2Bdfsg1-3_{arch}.deb"
-    deb_filename = "/root/" + url.split("/")[-1]
-
-    match (package, arch):
-        case ("core", "amd64"):
-            sha256 = "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d"
-        case ("core", "arm64"):
-            sha256 = "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9"
-        case ("imapd", "amd64"):
-            sha256 = "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86"
-        case ("imapd", "arm64"):
-            sha256 = "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f"
-        case ("lmtpd", "amd64"):
-            sha256 = "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab"
-        case ("lmtpd", "arm64"):
-            sha256 = "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f"
-        case _:
-            apt.packages(packages=[f"dovecot-{package}"])
-            return
-
-    files.download(
-        name=f"Download dovecot-{package}",
-        src=url,
-        dest=deb_filename,
-        sha256sum=sha256,
-        cache_time=60 * 60 * 24 * 365 * 10,  # never redownload the package
-    )
-
-    apt.deb(name=f"Install dovecot-{package}", src=deb_filename)
-
-
-def _configure_dovecot(config: Config, debug: bool = False) -> bool:
-    """Configures Dovecot IMAP server."""
-    need_restart = False
-
-    main_config = files.template(
-        src=importlib.resources.files(__package__).joinpath("dovecot/dovecot.conf.j2"),
-        dest="/etc/dovecot/dovecot.conf",
-        user="root",
-        group="root",
-        mode="644",
-        config=config,
-        debug=debug,
-        disable_ipv6=config.disable_ipv6,
-    )
-    need_restart |= main_config.changed
-    auth_config = files.put(
-        src=importlib.resources.files(__package__).joinpath("dovecot/auth.conf"),
-        dest="/etc/dovecot/auth.conf",
-        user="root",
-        group="root",
-        mode="644",
-    )
-    need_restart |= auth_config.changed
-    lua_push_notification_script = files.put(
-        src=importlib.resources.files(__package__).joinpath(
-            "dovecot/push_notification.lua"
-        ),
-        dest="/etc/dovecot/push_notification.lua",
-        user="root",
-        group="root",
-        mode="644",
-    )
-    need_restart |= lua_push_notification_script.changed
-
-    # remove historic expunge script
-    # which is now implemented through a systemd chatmail-expire service/timer
-    files.file(
-        path="/etc/cron.d/expunge",
-        present=False,
-    )
-
-    # as per https://doc.dovecot.org/configuration_manual/os/
-    # it is recommended to set the following inotify limits
-    for name in ("max_user_instances", "max_user_watches"):
-        key = f"fs.inotify.{name}"
-        if host.get_fact(Sysctl)[key] > 65535:
-            # Skip updating limits if already sufficient
-            # (enables running in incus containers where sysctl readonly)
-            continue
-        server.sysctl(
-            name=f"Change {key}",
-            key=key,
-            value=65535,
-            persist=True,
-        )
-
-    timezone_env = files.line(
-        name="Set TZ environment variable",
-        path="/etc/environment",
-        line="TZ=:/etc/localtime",
-    )
-    need_restart |= timezone_env.changed
-
-    return need_restart
-
-
-def _configure_nginx(config: Config, debug: bool = False) -> bool:
-    """Configures nginx HTTP server."""
-    need_restart = False
-
-    main_config = files.template(
-        src=importlib.resources.files(__package__).joinpath("nginx/nginx.conf.j2"),
-        dest="/etc/nginx/nginx.conf",
-        user="root",
-        group="root",
-        mode="644",
-        config={"domain_name": config.mail_domain},
-        disable_ipv6=config.disable_ipv6,
-    )
-    need_restart |= main_config.changed
-
-    autoconfig = files.template(
-        src=importlib.resources.files(__package__).joinpath("nginx/autoconfig.xml.j2"),
-        dest="/var/www/html/.well-known/autoconfig/mail/config-v1.1.xml",
-        user="root",
-        group="root",
-        mode="644",
-        config={"domain_name": config.mail_domain},
-    )
-    need_restart |= autoconfig.changed
-
-    mta_sts_config = files.template(
-        src=importlib.resources.files(__package__).joinpath("nginx/mta-sts.txt.j2"),
-        dest="/var/www/html/.well-known/mta-sts.txt",
-        user="root",
-        group="root",
-        mode="644",
-        config={"domain_name": config.mail_domain},
-    )
-    need_restart |= mta_sts_config.changed
-
-    # install CGI newemail script
-    #
-    cgi_dir = "/usr/lib/cgi-bin"
-    files.directory(
-        name=f"Ensure {cgi_dir} exists",
-        path=cgi_dir,
-        user="root",
-        group="root",
-    )
-
-    files.put(
-        name="Upload cgi newemail.py script",
-        src=importlib.resources.files("chatmaild").joinpath("newemail.py").open("rb"),
-        dest=f"{cgi_dir}/newemail.py",
-        user="root",
-        group="root",
-        mode="755",
-    )
-
-    return need_restart
-
-
-def _remove_rspamd() -> None:
-    """Remove rspamd"""
-    apt.packages(name="Remove rspamd", packages="rspamd", present=False)
-
-
-def check_config(config):
-    mail_domain = config.mail_domain
-    if mail_domain != "testrun.org" and not mail_domain.endswith(".testrun.org"):
-        blocked_words = "merlinux schmieder testrun.org".split()
-        for key in config.__dict__:
-            value = config.__dict__[key]
-            if key.startswith("privacy") and any(
-                x in str(value) for x in blocked_words
-            ):
-                raise ValueError(
-                    f"please set your own privacy contacts/addresses in {config._inipath}"
-                )
-    return config
-
-
-def deploy_turn_server(config):
-    (url, sha256sum) = {
-        "x86_64": (
-            "https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-x86_64-linux",
-            "841e527c15fdc2940b0469e206188ea8f0af48533be12ecb8098520f813d41e4",
-        ),
-        "aarch64": (
-            "https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-aarch64-linux",
-            "a5fc2d06d937b56a34e098d2cd72a82d3e89967518d159bf246dc69b65e81b42",
-        ),
-    }[host.get_fact(facts.server.Arch)]
-
-    need_restart = False
-
-    existing_sha256sum = host.get_fact(Sha256File, "/usr/local/bin/chatmail-turn")
-    if existing_sha256sum != sha256sum:
-        server.shell(
-            name="Download chatmail-turn",
-            commands=[
-                f"(curl -L {url} >/usr/local/bin/chatmail-turn.new && (echo '{sha256sum} /usr/local/bin/chatmail-turn.new' | sha256sum -c) && mv /usr/local/bin/chatmail-turn.new /usr/local/bin/chatmail-turn)",
-                "chmod 755 /usr/local/bin/chatmail-turn",
-            ],
-        )
-        need_restart = True
-
-    source_path = importlib.resources.files(__package__).joinpath(
-        "service", "turnserver.service.f"
-    )
-    content = source_path.read_text().format(mail_domain=config.mail_domain).encode()
-
-    systemd_unit = files.put(
-        name="Upload turnserver.service",
-        src=io.BytesIO(content),
-        dest="/etc/systemd/system/turnserver.service",
-        user="root",
-        group="root",
-        mode="644",
-    )
-    need_restart |= systemd_unit.changed
-
-    systemd.service(
-        name="Setup turnserver service",
-        service="turnserver.service",
-        running=True,
-        enabled=True,
-        restarted=need_restart,
-        daemon_reload=systemd_unit.changed,
-    )
-
-
-def deploy_mtail(config):
-    # Uninstall mtail package, we are going to install a static binary.
-    apt.packages(name="Uninstall mtail", packages=["mtail"], present=False)
-
-    (url, sha256sum) = {
-        "x86_64": (
-            "https://github.com/google/mtail/releases/download/v3.0.8/mtail_3.0.8_linux_amd64.tar.gz",
-            "123c2ee5f48c3eff12ebccee38befd2233d715da736000ccde49e3d5607724e4",
-        ),
-        "aarch64": (
-            "https://github.com/google/mtail/releases/download/v3.0.8/mtail_3.0.8_linux_arm64.tar.gz",
-            "aa04811c0929b6754408676de520e050c45dddeb3401881888a092c9aea89cae",
-        ),
-    }[host.get_fact(facts.server.Arch)]
-
-    server.shell(
-        name="Download mtail",
-        commands=[
-            f"(echo '{sha256sum} /usr/local/bin/mtail' | sha256sum -c) || (curl -L {url} | gunzip | tar -x -f - mtail -O >/usr/local/bin/mtail.new && mv /usr/local/bin/mtail.new /usr/local/bin/mtail)",
-            "chmod 755 /usr/local/bin/mtail",
-        ],
-    )
-
-    # Using our own systemd unit instead of `/usr/lib/systemd/system/mtail.service`.
-    # This allows to read from journalctl instead of log files.
-    files.template(
-        src=importlib.resources.files(__package__).joinpath("mtail/mtail.service.j2"),
-        dest="/etc/systemd/system/mtail.service",
-        user="root",
-        group="root",
-        mode="644",
-        address=config.mtail_address or "127.0.0.1",
-        port=3903,
-    )
-
-    mtail_conf = files.put(
-        name="Mtail configuration",
-        src=importlib.resources.files(__package__).joinpath(
-            "mtail/delivered_mail.mtail"
-        ),
-        dest="/etc/mtail/delivered_mail.mtail",
-        user="root",
-        group="root",
-        mode="644",
-    )
-
-    systemd.service(
-        name="Start and enable mtail",
-        service="mtail.service",
-        running=bool(config.mtail_address),
-        enabled=bool(config.mtail_address),
-        restarted=mtail_conf.changed,
-    )
-
-
-def deploy_iroh_relay(config) -> None:
-    (url, sha256sum) = {
-        "x86_64": (
-            "https://github.com/n0-computer/iroh/releases/download/v0.35.0/iroh-relay-v0.35.0-x86_64-unknown-linux-musl.tar.gz",
-            "45c81199dbd70f8c4c30fef7f3b9727ca6e3cea8f2831333eeaf8aa71bf0fac1",
-        ),
-        "aarch64": (
-            "https://github.com/n0-computer/iroh/releases/download/v0.35.0/iroh-relay-v0.35.0-aarch64-unknown-linux-musl.tar.gz",
-            "f8ef27631fac213b3ef668d02acd5b3e215292746a3fc71d90c63115446008b1",
-        ),
-    }[host.get_fact(facts.server.Arch)]
-
-    apt.packages(
-        name="Install curl",
-        packages=["curl"],
-    )
-
-    need_restart = False
-
-    existing_sha256sum = host.get_fact(Sha256File, "/usr/local/bin/iroh-relay")
-    if existing_sha256sum != sha256sum:
-        server.shell(
-            name="Download iroh-relay",
-            commands=[
-                f"(curl -L {url} | gunzip | tar -x -f - ./iroh-relay -O >/usr/local/bin/iroh-relay.new && (echo '{sha256sum} /usr/local/bin/iroh-relay.new' | sha256sum -c) && mv /usr/local/bin/iroh-relay.new /usr/local/bin/iroh-relay)",
-                "chmod 755 /usr/local/bin/iroh-relay",
-            ],
-        )
-        need_restart = True
-
-    systemd_unit = files.put(
-        name="Upload iroh-relay systemd unit",
-        src=importlib.resources.files(__package__).joinpath("iroh-relay.service"),
-        dest="/etc/systemd/system/iroh-relay.service",
-        user="root",
-        group="root",
-        mode="644",
-    )
-    need_restart |= systemd_unit.changed
-
-    iroh_config = files.put(
-        name="Upload iroh-relay config",
-        src=importlib.resources.files(__package__).joinpath("iroh-relay.toml"),
-        dest="/etc/iroh-relay.toml",
-        user="root",
-        group="root",
-        mode="644",
-    )
-    need_restart |= iroh_config.changed
-
-    systemd.service(
-        name="Start and enable iroh-relay",
-        service="iroh-relay.service",
-        running=True,
-        enabled=config.enable_iroh_relay,
-        restarted=need_restart,
-    )
-
-
-def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
-    """Deploy a chat-mail instance.
-
-    :param config_path: path to chatmail.ini
-    :param disable_mail: whether to disable postfix & dovecot
-    """
-    config = read_config(config_path)
-    check_config(config)
-    mail_domain = config.mail_domain
-
-    from .www import build_webpages, find_merge_conflict, get_paths
-
-    server.group(name="Create vmail group", group="vmail", system=True)
-    server.user(name="Create vmail user", user="vmail", group="vmail", system=True)
-    server.group(name="Create opendkim group", group="opendkim", system=True)
-    server.user(
-        name="Create opendkim user",
-        user="opendkim",
-        groups=["opendkim"],
-        system=True,
-    )
-    server.user(
-        name="Add postfix user to opendkim group for socket access",
-        user="postfix",
-        groups=["opendkim"],
-        system=True,
-    )
-    server.user(name="Create echobot user", user="echobot", system=True)
-    server.user(name="Create iroh user", user="iroh", system=True)
-
-    # Add our OBS repository for dovecot_no_delay
-    files.put(
-        name="Add Deltachat OBS GPG key to apt keyring",
-        src=importlib.resources.files(__package__).joinpath("obs-home-deltachat.gpg"),
-        dest="/etc/apt/keyrings/obs-home-deltachat.gpg",
-        user="root",
-        group="root",
-        mode="644",
-    )
-
-    files.line(
-        name="Add DeltaChat OBS home repository to sources.list",
-        path="/etc/apt/sources.list",
-        line="deb [signed-by=/etc/apt/keyrings/obs-home-deltachat.gpg] https://download.opensuse.org/repositories/home:/deltachat/Debian_12/ ./",
-        escape_regex_characters=True,
-        present=False,
-    )
-
-    if host.get_fact(Port, port=53) != "unbound":
-        files.line(
-            name="Add 9.9.9.9 to resolv.conf",
-            path="/etc/resolv.conf",
-            line="nameserver 9.9.9.9",
-        )
-    apt.update(name="apt update", cache_time=24 * 3600)
-    apt.upgrade(name="upgrade apt packages", auto_remove=True)
-
-    apt.packages(
-        name="Install rsync",
-        packages=["rsync"],
-    )
-
-    deploy_turn_server(config)
-
-    # Run local DNS resolver `unbound`.
-    # `resolvconf` takes care of setting up /etc/resolv.conf
-    # to use 127.0.0.1 as the resolver.
-    from cmdeploy.cmdeploy import Out
-
-    port_services = [
-        (["master", "smtpd"], 25),
-        ("unbound", 53),
-        ("acmetool", 80),
-        (["imap-login", "dovecot"], 143),
-        ("nginx", 443),
-        (["master", "smtpd"], 465),
-        (["master", "smtpd"], 587),
-        (["imap-login", "dovecot"], 993),
-        ("iroh-relay", 3340),
-        ("nginx", 8443),
-        (["master", "smtpd"], config.postfix_reinject_port),
-        (["master", "smtpd"], config.postfix_reinject_port_incoming),
-        ("filtermail", config.filtermail_smtp_port),
-        ("filtermail", config.filtermail_smtp_port_incoming),
-    ]
-    for service, port in port_services:
-        print(f"Checking if port {port} is available for {service}...")
-        running_service = host.get_fact(Port, port=port)
-        if running_service:
-            if running_service not in service:
-                Out().red(
-                    f"Deploy failed: port {port} is occupied by: {running_service}"
-                )
-                exit(1)
-
-    apt.packages(
-        name="Install unbound",
-        packages=["unbound", "unbound-anchor", "dnsutils"],
-    )
-    server.shell(
-        name="Generate root keys for validating DNSSEC",
-        commands=[
-            "unbound-anchor -a /var/lib/unbound/root.key || true",
-            "systemctl reset-failed unbound.service",
-        ],
-    )
-    systemd.service(
-        name="Start and enable unbound",
-        service="unbound.service",
-        running=True,
-        enabled=True,
-    )
-
-    deploy_iroh_relay(config)
-
-    # Deploy acmetool to have TLS certificates.
-    tls_domains = [mail_domain, f"mta-sts.{mail_domain}", f"www.{mail_domain}"]
-    deploy_acmetool(
-        email=config.acme_email,
-        domains=tls_domains,
-    )
-
-    apt.packages(
-        # required for setfacl for echobot
-        name="Install acl",
-        packages="acl",
-    )
-
-    apt.packages(
-        name="Install Postfix",
-        packages="postfix",
-    )
-
-    if not "dovecot.service" in host.get_fact(SystemdEnabled):
-        _install_dovecot_package("core", host.get_fact(facts.server.Arch))
-        _install_dovecot_package("imapd", host.get_fact(facts.server.Arch))
-        _install_dovecot_package("lmtpd", host.get_fact(facts.server.Arch))
-
-    apt.packages(
-        name="Install nginx",
-        packages=["nginx", "libnginx-mod-stream"],
-    )
-
-    apt.packages(
-        name="Install fcgiwrap",
-        packages=["fcgiwrap"],
-    )
-
-    www_path, src_dir, build_dir = get_paths(config)
-    # if www_folder was set to a non-existing folder, skip upload
-    if not www_path.is_dir():
-        logger.warning("Building web pages is disabled in chatmail.ini, skipping")
-    elif (path := find_merge_conflict(src_dir)) is not None:
-        logger.warning(f"Merge conflict found in {path}, skipping website deployment. Fix merge conflict if you want to upload your web page.")
-    else:
-        # if www_folder is a hugo page, build it
-        if build_dir:
-            www_path = build_webpages(src_dir, build_dir, config)
-        # if it is not a hugo page, upload it as is
-        files.rsync(f"{www_path}/", "/var/www/html", flags=["-avz", "--chown=www-data"])
-
-    _install_remote_venv_with_chatmaild(config)
-    debug = False
-    dovecot_need_restart = _configure_dovecot(config, debug=debug)
-    postfix_need_restart = _configure_postfix(config, debug=debug)
-    nginx_need_restart = _configure_nginx(config)
-    _uninstall_mta_sts_daemon()
-
-    _remove_rspamd()
-    opendkim_need_restart = _configure_opendkim(mail_domain, "opendkim")
-
-    systemd.service(
-        name="Start and enable OpenDKIM",
-        service="opendkim.service",
-        running=True,
-        enabled=True,
-        daemon_reload=opendkim_need_restart,
-        restarted=opendkim_need_restart,
-    )
-
-    # Dovecot should be started before Postfix
-    # because it creates authentication socket
-    # required by Postfix.
-    systemd.service(
-        name="disable dovecot for now" if disable_mail else "Start and enable Dovecot",
-        service="dovecot.service",
-        running=False if disable_mail else True,
-        enabled=False if disable_mail else True,
-        restarted=dovecot_need_restart if not disable_mail else False,
-    )
-
-    systemd.service(
-        name="disable postfix for now" if disable_mail else "Start and enable Postfix",
-        service="postfix.service",
-        running=False if disable_mail else True,
-        enabled=False if disable_mail else True,
-        restarted=postfix_need_restart if not disable_mail else False,
-    )
-
-    systemd.service(
-        name="Start and enable nginx",
-        service="nginx.service",
-        running=True,
-        enabled=True,
-        restarted=nginx_need_restart,
-    )
-
-    systemd.service(
-        name="Start and enable fcgiwrap",
-        service="fcgiwrap.service",
-        running=True,
-        enabled=True,
-    )
-
-    systemd.service(
-        name="Restart echobot if postfix and dovecot were just started",
-        service="echobot.service",
-        restarted=postfix_need_restart and dovecot_need_restart,
-    )
-
-    # This file is used by auth proxy.
-    # https://wiki.debian.org/EtcMailName
-    server.shell(
-        name="Setup /etc/mailname",
-        commands=[f"echo {mail_domain} >/etc/mailname; chmod 644 /etc/mailname"],
-    )
-
-    journald_conf = files.put(
-        name="Configure journald",
-        src=importlib.resources.files(__package__).joinpath("journald.conf"),
-        dest="/etc/systemd/journald.conf",
-        user="root",
-        group="root",
-        mode="644",
-    )
-    systemd.service(
-        name="Start and enable journald",
-        service="systemd-journald.service",
-        running=True,
-        enabled=True,
-        restarted=journald_conf.changed,
-    )
-    files.directory(
-        name="Ensure old logs on disk are deleted",
-        path="/var/log/journal/",
-        present=False,
-    )
-
-    apt.packages(
-        name="Ensure cron is installed",
-        packages=["cron"],
-    )
-    try:
-        git_hash = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode()
-    except Exception:
-        git_hash = "unknown\n"
-    try:
-        git_diff = subprocess.check_output(["git", "diff"]).decode()
-    except Exception:
-        git_diff = ""
-    files.put(
-        name="Upload chatmail relay git commiit hash",
-        src=StringIO(git_hash + git_diff),
-        dest="/etc/chatmail-version",
-        mode="700",
-    )
-
-    deploy_mtail(config)

cmdeploy/src/cmdeploy/acmetool/__init__.py

@@ -2,104 +2,140 @@ import importlib.resources
 
 from pyinfra.operations import apt, files, server, systemd
 
+from ..basedeploy import Deployer
 
-def deploy_acmetool(email="", domains=[]):
-    """Deploy acmetool."""
-    apt.packages(
-        name="Install acmetool",
-        packages=["acmetool"],
-    )
 
-    files.file(
-        name="Remove old acmetool cronjob, it is replaced with systemd timer.",
-        path="/etc/cron.d/acmetool",
-        present=False,
-    )
+class AcmetoolDeployer(Deployer):
+    def __init__(self, email, domains):
+        self.domains = domains
+        self.email = email
+        self.need_restart_redirector = False
+        self.need_restart_reconcile_service = False
+        self.need_restart_reconcile_timer = False
 
-    files.put(
-        name="Install acmetool hook.",
-        src=importlib.resources.files(__package__).joinpath("acmetool.hook").open("rb"),
-        dest="/etc/acme/hooks/nginx",
-        user="root",
-        group="root",
-        mode="755",
-    )
-    files.file(
-        name="Remove acmetool hook from the wrong location where it was previously installed.",
-        path="/usr/lib/acme/hooks/nginx",
-        present=False,
-    )
+    def install(self):
+        apt.packages(
+            name="Install acmetool",
+            packages=["acmetool"],
+        )
 
-    files.template(
-        src=importlib.resources.files(__package__).joinpath("response-file.yaml.j2"),
-        dest="/var/lib/acme/conf/responses",
-        user="root",
-        group="root",
-        mode="644",
-        email=email,
-    )
+        files.file(
+            name="Remove old acmetool cronjob, it is replaced with systemd timer.",
+            path="/etc/cron.d/acmetool",
+            present=False,
+        )
 
-    files.template(
-        src=importlib.resources.files(__package__).joinpath("target.yaml.j2"),
-        dest="/var/lib/acme/conf/target",
-        user="root",
-        group="root",
-        mode="644",
-    )
+        files.put(
+            name="Install acmetool hook.",
+            src=importlib.resources.files(__package__)
+            .joinpath("acmetool.hook")
+            .open("rb"),
+            dest="/etc/acme/hooks/nginx",
+            user="root",
+            group="root",
+            mode="755",
+        )
+        files.file(
+            name="Remove acmetool hook from the wrong location where it was previously installed.",
+            path="/usr/lib/acme/hooks/nginx",
+            present=False,
+        )
 
-    service_file = files.put(
-        src=importlib.resources.files(__package__).joinpath(
-            "acmetool-redirector.service"
-        ),
-        dest="/etc/systemd/system/acmetool-redirector.service",
-        user="root",
-        group="root",
-        mode="644",
-    )
+    def configure(self):
+        files.template(
+            src=importlib.resources.files(__package__).joinpath(
+                "response-file.yaml.j2"
+            ),
+            dest="/var/lib/acme/conf/responses",
+            user="root",
+            group="root",
+            mode="644",
+            email=self.email,
+        )
 
-    systemd.service(
-        name="Setup acmetool-redirector service",
-        service="acmetool-redirector.service",
-        running=True,
-        enabled=True,
-        restarted=service_file.changed,
-    )
+        files.template(
+            src=importlib.resources.files(__package__).joinpath("target.yaml.j2"),
+            dest="/var/lib/acme/conf/target",
+            user="root",
+            group="root",
+            mode="644",
+        )
 
-    reconcile_service_file = files.put(
-        src=importlib.resources.files(__package__).joinpath(
-            "acmetool-reconcile.service"
-        ),
-        dest="/etc/systemd/system/acmetool-reconcile.service",
-        user="root",
-        group="root",
-        mode="644",
-    )
+        server.shell(
+            name=f"Remove old acmetool desired files for {self.domains[0]}",
+            commands=[f"rm -f /var/lib/acme/desired/{self.domains[0]}-*"],
+        )
+        files.template(
+            src=importlib.resources.files(__package__).joinpath("desired.yaml.j2"),
+            dest=f"/var/lib/acme/desired/{self.domains[0]}", # 0 is mailhost TLD
+            user="root",
+            group="root",
+            mode="644",
+            domains=self.domains,
+        )
 
-    systemd.service(
-        name="Setup acmetool-reconcile service",
-        service="acmetool-reconcile.service",
-        running=False,
-        enabled=False,
-        daemon_reload=reconcile_service_file.changed,
-    )
+        service_file = files.put(
+            src=importlib.resources.files(__package__).joinpath(
+                "acmetool-redirector.service"
+            ),
+            dest="/etc/systemd/system/acmetool-redirector.service",
+            user="root",
+            group="root",
+            mode="644",
+        )
+        self.need_restart_redirector = service_file.changed
 
-    reconcile_timer_file = files.put(
-        src=importlib.resources.files(__package__).joinpath("acmetool-reconcile.timer"),
-        dest="/etc/systemd/system/acmetool-reconcile.timer",
-        user="root",
-        group="root",
-        mode="644",
-    )
+        reconcile_service_file = files.put(
+            src=importlib.resources.files(__package__).joinpath(
+                "acmetool-reconcile.service"
+            ),
+            dest="/etc/systemd/system/acmetool-reconcile.service",
+            user="root",
+            group="root",
+            mode="644",
+        )
+        self.need_restart_reconcile_service = reconcile_service_file.changed
 
-    systemd.service(
-        name="Setup acmetool-reconcile timer",
-        service="acmetool-reconcile.timer",
-        running=True,
-        enabled=True,
-        daemon_reload=reconcile_timer_file.changed,
-    )
+        reconcile_timer_file = files.put(
+            src=importlib.resources.files(__package__).joinpath(
+                "acmetool-reconcile.timer"
+            ),
+            dest="/etc/systemd/system/acmetool-reconcile.timer",
+            user="root",
+            group="root",
+            mode="644",
+        )
+        self.need_restart_reconcile_timer = reconcile_timer_file.changed
 
-    server.shell(
-        name=f"Request certificate for: {', '.join(domains)}",
-        commands=[f"acmetool want --xlog.severity=debug {' '.join(domains)}"],
-    )
+    def activate(self):
+        systemd.service(
+            name="Setup acmetool-redirector service",
+            service="acmetool-redirector.service",
+            running=True,
+            enabled=True,
+            restarted=self.need_restart_redirector,
+        )
+        self.need_restart_redirector = False
+
+        systemd.service(
+            name="Setup acmetool-reconcile service",
+            service="acmetool-reconcile.service",
+            running=False,
+            enabled=False,
+            daemon_reload=self.need_restart_reconcile_service,
+        )
+        self.need_restart_reconcile_service = False
+
+        systemd.service(
+            name="Setup acmetool-reconcile timer",
+            service="acmetool-reconcile.timer",
+            running=True,
+            enabled=True,
+            daemon_reload=self.need_restart_reconcile_timer,
+        )
+        self.need_restart_reconcile_timer = False
+
+        server.shell(
+            name=f"Reconcile certificates for: {', '.join(self.domains)}",
+            commands=["acmetool --batch --xlog.severity=debug reconcile"],
+        )

cmdeploy/src/cmdeploy/acmetool/desired.yaml.j2

@@ -0,0 +1,6 @@
+satisfy:
+  names:
+{%- for domain in domains %}
+  - {{ domain }}
+{%- endfor %}
+

cmdeploy/src/cmdeploy/acmetool/response-file.yaml.j2

@@ -1,2 +1,2 @@
 "acme-enter-email": "{{ email }}"
-"acme-agreement:https://letsencrypt.org/documents/LE-SA-v1.5-February-24-2025.pdf": true
+"acme-agreement:https://letsencrypt.org/documents/LE-SA-v1.6-August-18-2025.pdf": true

cmdeploy/src/cmdeploy/basedeploy.py

@@ -0,0 +1,112 @@
+import importlib.resources
+import io
+import os
+
+from pyinfra.operations import files, server, systemd
+
+
+def get_resource(arg, pkg=__package__):
+    return importlib.resources.files(pkg).joinpath(arg)
+
+
+def configure_remote_units(mail_domain, units) -> None:
+    remote_base_dir = "/usr/local/lib/chatmaild"
+    remote_venv_dir = f"{remote_base_dir}/venv"
+    remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini"
+    root_owned = dict(user="root", group="root", mode="644")
+
+    # install systemd units
+    for fn in units:
+        execpath = fn if fn != "filtermail-incoming" else "filtermail"
+        params = dict(
+            execpath=f"{remote_venv_dir}/bin/{execpath}",
+            config_path=remote_chatmail_inipath,
+            remote_venv_dir=remote_venv_dir,
+            mail_domain=mail_domain,
+        )
+
+        basename = fn if "." in fn else f"{fn}.service"
+
+        source_path = get_resource(f"service/{basename}.f")
+        content = source_path.read_text().format(**params).encode()
+
+        files.put(
+            name=f"Upload {basename}",
+            src=io.BytesIO(content),
+            dest=f"/etc/systemd/system/{basename}",
+            **root_owned,
+        )
+
+
+def activate_remote_units(units) -> None:
+    # activate systemd units
+    for fn in units:
+        basename = fn if "." in fn else f"{fn}.service"
+
+        if fn == "chatmail-expire" or fn == "chatmail-fsreport":
+            # don't auto-start but let the corresponding timer trigger execution
+            enabled = False
+        else:
+            enabled = True
+        systemd.service(
+            name=f"Setup {basename}",
+            service=basename,
+            running=enabled,
+            enabled=enabled,
+            restarted=enabled,
+            daemon_reload=True,
+        )
+
+
+class Deployment:
+    def install(self, deployer):
+        # optional 'required_users' contains a list of (user, group, secondary-group-list) tuples.
+        # If the group is None, no group is created corresponding to that user.
+        # If the secondary group list is not None, all listed groups are created as well.
+        required_users = getattr(deployer, "required_users", [])
+        for user, group, groups in required_users:
+            if group is not None:
+                server.group(
+                    name="Create {} group".format(group), group=group, system=True
+                )
+            if groups is not None:
+                for group2 in groups:
+                    server.group(
+                        name="Create {} group".format(group2), group=group2, system=True
+                    )
+            server.user(
+                name="Create {} user".format(user),
+                user=user,
+                group=group,
+                groups=groups,
+                system=True,
+            )
+
+        deployer.install()
+
+    def configure(self, deployer):
+        deployer.configure()
+
+    def activate(self, deployer):
+        deployer.activate()
+
+    def perform_stages(self, deployers):
+        default_stages = "install,configure,activate"
+        stages = os.getenv("CMDEPLOY_STAGES", default_stages).split(",")
+
+        for stage in stages:
+            for deployer in deployers:
+                getattr(self, stage)(deployer)
+
+
+class Deployer:
+    need_restart = False
+
+    def install(self):
+        pass
+
+    def configure(self):
+        pass
+
+    def activate(self):
+        pass

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -19,7 +19,7 @@ from packaging import version
 from termcolor import colored
 
 from . import dns, remote
-from .sshexec import SSHExec, LocalExec
+from .sshexec import LocalExec, SSHExec
 
 #
 # cmdeploy sub commands and options
@@ -95,7 +95,7 @@ def run_cmd(args, out):
     env["CHATMAIL_INI"] = args.inipath
     env["CHATMAIL_DISABLE_MAIL"] = "True" if args.disable_mail else ""
     env["CHATMAIL_REQUIRE_IROH"] = "True" if require_iroh else ""
-    deploy_path = importlib.resources.files(__package__).joinpath("deploy.py").resolve()
+    deploy_path = importlib.resources.files(__package__).joinpath("run.py").resolve()
     pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
 
     cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y"
@@ -109,15 +109,6 @@ def run_cmd(args, out):
     try:
         retcode = out.check_call(cmd, env=env)
         if retcode == 0:
-            if not args.disable_mail:
-                print("\nYou can try out the relay by talking to this echo bot: ")
-                sshexec = SSHExec(args.config.mail_domain, verbose=args.verbose)
-                print(
-                    sshexec(
-                        call=remote.rshell.shell,
-                        kwargs=dict(command="cat /var/lib/echobot/invite-link.txt"),
-                    )
-                )
             out.green("Deploy completed, call `cmdeploy dns` next.")
         elif not remote_data["acme_account_url"]:
             out.red("Deploy completed but letsencrypt not configured")
@@ -238,7 +229,12 @@ def fmt_cmd_options(parser):
 def fmt_cmd(args, out):
     """Run formattting fixes on all chatmail source code."""
 
-    sources = [str(importlib.resources.files(x)) for x in ("chatmaild", "cmdeploy")]
+    chatmaild_dir = importlib.resources.files("chatmaild").resolve()
+    cmdeploy_dir = chatmaild_dir.joinpath(
+        "..", "..", "..", "cmdeploy", "src", "cmdeploy"
+    ).resolve()
+    sources = [str(chatmaild_dir), str(cmdeploy_dir)]
+
     format_args = [shutil.which("ruff"), "format"]
     check_args = [shutil.which("ruff"), "check"]
 
@@ -309,7 +305,7 @@ def add_ssh_host_option(parser):
         "--ssh-host",
         dest="ssh_host",
         help="Run commands on 'localhost', via '@docker', or on a specific SSH host "
-             "instead of chatmail.ini's mail_domain.",
+        "instead of chatmail.ini's mail_domain.",
     )
 
 

cmdeploy/src/cmdeploy/deployers.py

@@ -0,0 +1,580 @@
+"""
+Chat Mail pyinfra deploy.
+"""
+
+import shutil
+import subprocess
+import sys
+from io import StringIO
+from pathlib import Path
+
+from chatmaild.config import read_config
+from pyinfra import facts, host, logger
+from pyinfra.api import FactBase
+from pyinfra.facts.files import Sha256File
+from pyinfra.facts.systemd import SystemdEnabled
+from pyinfra.operations import apt, files, pip, server, systemd
+
+from cmdeploy.cmdeploy import Out
+
+from .acmetool import AcmetoolDeployer
+from .basedeploy import (
+    Deployer,
+    Deployment,
+    activate_remote_units,
+    configure_remote_units,
+    get_resource,
+)
+from .dovecot.deployer import DovecotDeployer
+from .mtail.deployer import MtailDeployer
+from .nginx.deployer import NginxDeployer
+from .opendkim.deployer import OpendkimDeployer
+from .postfix.deployer import PostfixDeployer
+from .www import build_webpages, find_merge_conflict, get_paths
+
+
+class Port(FactBase):
+    """
+    Returns the process occuping a port.
+    """
+
+    def command(self, port: int) -> str:
+        return (
+            "ss -lptn 'src :%d' | awk 'NR>1 {print $6,$7}' | sed 's/users:((\"//;s/\".*//'"
+            % (port,)
+        )
+
+    def process(self, output: [str]) -> str:
+        return output[0]
+
+
+def _build_chatmaild(dist_dir) -> None:
+    dist_dir = Path(dist_dir).resolve()
+    if dist_dir.exists():
+        shutil.rmtree(dist_dir)
+    dist_dir.mkdir()
+    subprocess.check_output(
+        [sys.executable, "-m", "build", "-n"]
+        + ["--sdist", "chatmaild", "--outdir", str(dist_dir)]
+    )
+    entries = list(dist_dir.iterdir())
+    assert len(entries) == 1
+    return entries[0]
+
+
+def remove_legacy_artifacts():
+    # disable legacy doveauth-dictproxy.service
+    if host.get_fact(SystemdEnabled).get("doveauth-dictproxy.service"):
+        systemd.service(
+            name="Disable legacy doveauth-dictproxy.service",
+            service="doveauth-dictproxy.service",
+            running=False,
+            enabled=False,
+        )
+
+
+def _install_remote_venv_with_chatmaild() -> None:
+    remove_legacy_artifacts()
+    dist_file = _build_chatmaild(dist_dir=Path("chatmaild/dist"))
+    remote_base_dir = "/usr/local/lib/chatmaild"
+    remote_dist_file = f"{remote_base_dir}/dist/{dist_file.name}"
+    remote_venv_dir = f"{remote_base_dir}/venv"
+    root_owned = dict(user="root", group="root", mode="644")
+
+    apt.packages(
+        name="apt install python3-virtualenv",
+        packages=["python3-virtualenv"],
+    )
+
+    files.put(
+        name="Upload chatmaild source package",
+        src=dist_file.open("rb"),
+        dest=remote_dist_file,
+        create_remote_dir=True,
+        **root_owned,
+    )
+
+    pip.virtualenv(
+        name=f"chatmaild virtualenv {remote_venv_dir}",
+        path=remote_venv_dir,
+        always_copy=True,
+    )
+
+    apt.packages(
+        name="install gcc and headers to build crypt_r source package",
+        packages=["gcc", "python3-dev"],
+    )
+
+    server.shell(
+        name=f"forced pip-install {dist_file.name}",
+        commands=[
+            f"{remote_venv_dir}/bin/pip install --force-reinstall {remote_dist_file}"
+        ],
+    )
+
+
+def _configure_remote_venv_with_chatmaild(config) -> None:
+    remote_base_dir = "/usr/local/lib/chatmaild"
+    remote_venv_dir = f"{remote_base_dir}/venv"
+    remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini"
+    root_owned = dict(user="root", group="root", mode="644")
+
+    files.put(
+        name=f"Upload {remote_chatmail_inipath}",
+        src=config._getbytefile(),
+        dest=remote_chatmail_inipath,
+        **root_owned,
+    )
+
+    files.template(
+        src=get_resource("metrics.cron.j2"),
+        dest="/etc/cron.d/chatmail-metrics",
+        user="root",
+        group="root",
+        mode="644",
+        config={
+            "mailboxes_dir": config.mailboxes_dir,
+            "execpath": f"{remote_venv_dir}/bin/chatmail-metrics",
+        },
+    )
+
+
+class UnboundDeployer(Deployer):
+    def install(self):
+        # Run local DNS resolver `unbound`.
+        # `resolvconf` takes care of setting up /etc/resolv.conf
+        # to use 127.0.0.1 as the resolver.
+
+        #
+        # On an IPv4-only system, if unbound is started but not
+        # configured, it causes subsequent steps to fail to resolve hosts.
+        # Here, we use policy-rc.d to prevent unbound from starting up
+        # on initial install.  Later, we will configure it and start it.
+        #
+        # For documentation about policy-rc.d, see:
+        # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
+        #
+        files.put(
+            src=get_resource("policy-rc.d"),
+            dest="/usr/sbin/policy-rc.d",
+            user="root",
+            group="root",
+            mode="755",
+        )
+
+        apt.packages(
+            name="Install unbound",
+            packages=["unbound", "unbound-anchor", "dnsutils"],
+        )
+
+        files.file("/usr/sbin/policy-rc.d", present=False)
+
+    def configure(self):
+        server.shell(
+            name="Generate root keys for validating DNSSEC",
+            commands=[
+                "unbound-anchor -a /var/lib/unbound/root.key || true",
+            ],
+        )
+
+    def activate(self):
+        server.shell(
+            name="Generate root keys for validating DNSSEC",
+            commands=[
+                "systemctl reset-failed unbound.service",
+            ],
+        )
+
+        systemd.service(
+            name="Start and enable unbound",
+            service="unbound.service",
+            running=True,
+            enabled=True,
+        )
+
+
+class MtastsDeployer(Deployer):
+    def configure(self):
+        # Remove configuration.
+        files.file("/etc/mta-sts-daemon.yml", present=False)
+        files.directory("/usr/local/lib/postfix-mta-sts-resolver", present=False)
+        files.file("/etc/systemd/system/mta-sts-daemon.service", present=False)
+
+    def activate(self):
+        systemd.service(
+            name="Stop MTA-STS daemon",
+            service="mta-sts-daemon.service",
+            daemon_reload=True,
+            running=False,
+            enabled=False,
+        )
+
+
+class WebsiteDeployer(Deployer):
+    def __init__(self, config):
+        self.config = config
+
+    def install(self):
+        files.directory(
+            name="Ensure /var/www exists",
+            path="/var/www",
+            user="root",
+            group="root",
+            mode="755",
+            present=True,
+        )
+
+    def configure(self):
+        www_path, src_dir, build_dir = get_paths(self.config)
+        # if www_folder was set to a non-existing folder, skip upload
+        if not www_path.is_dir():
+            logger.warning("Building web pages is disabled in chatmail.ini, skipping")
+        elif (path := find_merge_conflict(src_dir)) is not None:
+            logger.warning(
+                f"Merge conflict found in {path}, skipping website deployment. Fix merge conflict if you want to upload your web page."
+            )
+        else:
+            # if www_folder is a hugo page, build it
+            if build_dir:
+                www_path = build_webpages(src_dir, build_dir, self.config)
+            # if it is not a hugo page, upload it as is
+            files.rsync(
+                f"{www_path}/", "/var/www/html", flags=["-avz", "--chown=www-data"]
+            )
+
+
+class LegacyRemoveDeployer(Deployer):
+    def install(self):
+        apt.packages(name="Remove rspamd", packages="rspamd", present=False)
+
+        # remove historic expunge script
+        # which is now implemented through a systemd timer (chatmail-expire)
+        files.file(
+            path="/etc/cron.d/expunge",
+            present=False,
+        )
+
+        # Remove OBS repository key that is no longer used.
+        files.file("/etc/apt/keyrings/obs-home-deltachat.gpg", present=False)
+        files.line(
+            name="Remove DeltaChat OBS home repository from sources.list",
+            path="/etc/apt/sources.list",
+            line="deb [signed-by=/etc/apt/keyrings/obs-home-deltachat.gpg] https://download.opensuse.org/repositories/home:/deltachat/Debian_12/ ./",
+            escape_regex_characters=True,
+            present=False,
+        )
+
+        # prior relay versions used filelogging
+        files.directory(
+            name="Ensure old logs on disk are deleted",
+            path="/var/log/journal/",
+            present=False,
+        )
+        # remove echobot if it is still running
+        if host.get_fact(SystemdEnabled).get("echobot.service"):
+            systemd.service(
+                name="Disable echobot.service",
+                service="echobot.service",
+                running=False,
+                enabled=False,
+            )
+
+
+def check_config(config):
+    mail_domain = config.mail_domain
+    if mail_domain != "testrun.org" and not mail_domain.endswith(".testrun.org"):
+        blocked_words = "merlinux schmieder testrun.org".split()
+        for key in config.__dict__:
+            value = config.__dict__[key]
+            if key.startswith("privacy") and any(
+                x in str(value) for x in blocked_words
+            ):
+                raise ValueError(
+                    f"please set your own privacy contacts/addresses in {config._inipath}"
+                )
+    return config
+
+
+class TurnDeployer(Deployer):
+    def __init__(self, mail_domain):
+        self.mail_domain = mail_domain
+        self.units = ["turnserver"]
+
+    def install(self):
+        (url, sha256sum) = {
+            "x86_64": (
+                "https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-x86_64-linux",
+                "841e527c15fdc2940b0469e206188ea8f0af48533be12ecb8098520f813d41e4",
+            ),
+            "aarch64": (
+                "https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-aarch64-linux",
+                "a5fc2d06d937b56a34e098d2cd72a82d3e89967518d159bf246dc69b65e81b42",
+            ),
+        }[host.get_fact(facts.server.Arch)]
+
+        existing_sha256sum = host.get_fact(Sha256File, "/usr/local/bin/chatmail-turn")
+        if existing_sha256sum != sha256sum:
+            server.shell(
+                name="Download chatmail-turn",
+                commands=[
+                    f"(curl -L {url} >/usr/local/bin/chatmail-turn.new && (echo '{sha256sum} /usr/local/bin/chatmail-turn.new' | sha256sum -c) && mv /usr/local/bin/chatmail-turn.new /usr/local/bin/chatmail-turn)",
+                    "chmod 755 /usr/local/bin/chatmail-turn",
+                ],
+            )
+
+    def configure(self):
+        configure_remote_units(self.mail_domain, self.units)
+
+    def activate(self):
+        activate_remote_units(self.units)
+
+
+class IrohDeployer(Deployer):
+    def __init__(self, enable_iroh_relay):
+        self.enable_iroh_relay = enable_iroh_relay
+
+    def install(self):
+        (url, sha256sum) = {
+            "x86_64": (
+                "https://github.com/n0-computer/iroh/releases/download/v0.35.0/iroh-relay-v0.35.0-x86_64-unknown-linux-musl.tar.gz",
+                "45c81199dbd70f8c4c30fef7f3b9727ca6e3cea8f2831333eeaf8aa71bf0fac1",
+            ),
+            "aarch64": (
+                "https://github.com/n0-computer/iroh/releases/download/v0.35.0/iroh-relay-v0.35.0-aarch64-unknown-linux-musl.tar.gz",
+                "f8ef27631fac213b3ef668d02acd5b3e215292746a3fc71d90c63115446008b1",
+            ),
+        }[host.get_fact(facts.server.Arch)]
+
+        existing_sha256sum = host.get_fact(Sha256File, "/usr/local/bin/iroh-relay")
+        if existing_sha256sum != sha256sum:
+            server.shell(
+                name="Download iroh-relay",
+                commands=[
+                    f"(curl -L {url} | gunzip | tar -x -f - ./iroh-relay -O >/usr/local/bin/iroh-relay.new && (echo '{sha256sum} /usr/local/bin/iroh-relay.new' | sha256sum -c) && mv /usr/local/bin/iroh-relay.new /usr/local/bin/iroh-relay)",
+                    "chmod 755 /usr/local/bin/iroh-relay",
+                ],
+            )
+
+            self.need_restart = True
+
+    def configure(self):
+        systemd_unit = files.put(
+            name="Upload iroh-relay systemd unit",
+            src=get_resource("iroh-relay.service"),
+            dest="/etc/systemd/system/iroh-relay.service",
+            user="root",
+            group="root",
+            mode="644",
+        )
+        self.need_restart |= systemd_unit.changed
+
+        iroh_config = files.put(
+            name="Upload iroh-relay config",
+            src=get_resource("iroh-relay.toml"),
+            dest="/etc/iroh-relay.toml",
+            user="root",
+            group="root",
+            mode="644",
+        )
+        self.need_restart |= iroh_config.changed
+
+    def activate(self):
+        systemd.service(
+            name="Start and enable iroh-relay",
+            service="iroh-relay.service",
+            running=True,
+            enabled=self.enable_iroh_relay,
+            restarted=self.need_restart,
+        )
+        self.need_restart = False
+
+
+class JournaldDeployer(Deployer):
+    def configure(self):
+        journald_conf = files.put(
+            name="Configure journald",
+            src=get_resource("journald.conf"),
+            dest="/etc/systemd/journald.conf",
+            user="root",
+            group="root",
+            mode="644",
+        )
+        self.need_restart = journald_conf.changed
+
+    def activate(self):
+        systemd.service(
+            name="Start and enable journald",
+            service="systemd-journald.service",
+            running=True,
+            enabled=True,
+            restarted=self.need_restart,
+        )
+        self.need_restart = False
+
+
+class ChatmailVenvDeployer(Deployer):
+    def __init__(self, config):
+        self.config = config
+        self.units = (
+            "filtermail",
+            "filtermail-incoming",
+            "chatmail-metadata",
+            "lastlogin",
+            "chatmail-expire",
+            "chatmail-expire.timer",
+            "chatmail-fsreport",
+            "chatmail-fsreport.timer",
+        )
+
+    def install(self):
+        _install_remote_venv_with_chatmaild()
+
+    def configure(self):
+        _configure_remote_venv_with_chatmaild(self.config)
+        configure_remote_units(self.config.mail_domain, self.units)
+
+    def activate(self):
+        activate_remote_units(self.units)
+
+
+class ChatmailDeployer(Deployer):
+    required_users = [
+        ("vmail", "vmail", None),
+        ("iroh", None, None),
+    ]
+
+    def __init__(self, mail_domain):
+        self.mail_domain = mail_domain
+
+    def install(self):
+        apt.update(name="apt update", cache_time=24 * 3600)
+        apt.upgrade(name="upgrade apt packages", auto_remove=True)
+
+        apt.packages(
+            name="Install curl",
+            packages=["curl"],
+        )
+
+        apt.packages(
+            name="Install rsync",
+            packages=["rsync"],
+        )
+        apt.packages(
+            name="Ensure cron is installed",
+            packages=["cron"],
+        )
+
+    def configure(self):
+        # This file is used by auth proxy.
+        # https://wiki.debian.org/EtcMailName
+        server.shell(
+            name="Setup /etc/mailname",
+            commands=[
+                f"echo {self.mail_domain} >/etc/mailname; chmod 644 /etc/mailname"
+            ],
+        )
+
+
+class FcgiwrapDeployer(Deployer):
+    def install(self):
+        apt.packages(
+            name="Install fcgiwrap",
+            packages=["fcgiwrap"],
+        )
+
+    def activate(self):
+        systemd.service(
+            name="Start and enable fcgiwrap",
+            service="fcgiwrap.service",
+            running=True,
+            enabled=True,
+        )
+
+
+class GithashDeployer(Deployer):
+    def activate(self):
+        try:
+            git_hash = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode()
+        except Exception:
+            git_hash = "unknown\n"
+        try:
+            git_diff = subprocess.check_output(["git", "diff"]).decode()
+        except Exception:
+            git_diff = ""
+        files.put(
+            name="Upload chatmail relay git commiit hash",
+            src=StringIO(git_hash + git_diff),
+            dest="/etc/chatmail-version",
+            mode="700",
+        )
+
+
+def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
+    """Deploy a chat-mail instance.
+
+    :param config_path: path to chatmail.ini
+    :param disable_mail: whether to disable postfix & dovecot
+    """
+    config = read_config(config_path)
+    check_config(config)
+    mail_domain = config.mail_domain
+
+    if host.get_fact(Port, port=53) != "unbound":
+        files.line(
+            name="Add 9.9.9.9 to resolv.conf",
+            path="/etc/resolv.conf",
+            line="nameserver 9.9.9.9",
+        )
+
+    port_services = [
+        (["master", "smtpd"], 25),
+        ("unbound", 53),
+        ("acmetool", 80),
+        (["imap-login", "dovecot"], 143),
+        ("nginx", 443),
+        (["master", "smtpd"], 465),
+        (["master", "smtpd"], 587),
+        (["imap-login", "dovecot"], 993),
+        ("iroh-relay", 3340),
+        ("nginx", 8443),
+        (["master", "smtpd"], config.postfix_reinject_port),
+        (["master", "smtpd"], config.postfix_reinject_port_incoming),
+        ("filtermail", config.filtermail_smtp_port),
+        ("filtermail", config.filtermail_smtp_port_incoming),
+    ]
+    for service, port in port_services:
+        print(f"Checking if port {port} is available for {service}...")
+        running_service = host.get_fact(Port, port=port)
+        if running_service:
+            if running_service not in service:
+                Out().red(
+                    f"Deploy failed: port {port} is occupied by: {running_service}"
+                )
+                exit(1)
+
+    tls_domains = [mail_domain, f"mta-sts.{mail_domain}", f"www.{mail_domain}"]
+
+    all_deployers = [
+        ChatmailDeployer(mail_domain),
+        LegacyRemoveDeployer(),
+        JournaldDeployer(),
+        UnboundDeployer(),
+        TurnDeployer(mail_domain),
+        IrohDeployer(config.enable_iroh_relay),
+        AcmetoolDeployer(config.acme_email, tls_domains),
+        WebsiteDeployer(config),
+        ChatmailVenvDeployer(config),
+        MtastsDeployer(),
+        OpendkimDeployer(mail_domain),
+        # Dovecot should be started before Postfix
+        # because it creates authentication socket
+        # required by Postfix.
+        DovecotDeployer(config, disable_mail),
+        PostfixDeployer(config, disable_mail),
+        FcgiwrapDeployer(),
+        NginxDeployer(config),
+        MtailDeployer(config.mtail_address),
+        GithashDeployer(),
+    ]
+
+    Deployment().perform_stages(all_deployers)

cmdeploy/src/cmdeploy/dns.py

@@ -45,7 +45,8 @@ def check_full_zone(sshexec, remote_data, out, zonefile) -> int:
     and return (exitcode, remote_data) tuple."""
 
     required_diff, recommended_diff = sshexec.logged(
-        remote.rdns.check_zonefile, kwargs=dict(zonefile=zonefile, verbose=False),
+        remote.rdns.check_zonefile,
+        kwargs=dict(zonefile=zonefile, verbose=False),
     )
 
     returncode = 0

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -0,0 +1,148 @@
+from chatmaild.config import Config
+from pyinfra import host
+from pyinfra.facts.server import Arch, Sysctl
+from pyinfra.facts.systemd import SystemdEnabled
+from pyinfra.operations import apt, files, server, systemd
+
+from cmdeploy.basedeploy import (
+    Deployer,
+    activate_remote_units,
+    configure_remote_units,
+    get_resource,
+)
+
+
+class DovecotDeployer(Deployer):
+    daemon_reload = False
+
+    def __init__(self, config, disable_mail):
+        self.config = config
+        self.disable_mail = disable_mail
+        self.units = ["doveauth"]
+
+    def install(self):
+        arch = host.get_fact(Arch)
+        if not "dovecot.service" in host.get_fact(SystemdEnabled):
+            _install_dovecot_package("core", arch)
+            _install_dovecot_package("imapd", arch)
+            _install_dovecot_package("lmtpd", arch)
+
+    def configure(self):
+        configure_remote_units(self.config.mail_domain, self.units)
+        self.need_restart, self.daemon_reload = _configure_dovecot(self.config)
+
+    def activate(self):
+        activate_remote_units(self.units)
+
+        restart = False if self.disable_mail else self.need_restart
+
+        systemd.service(
+            name="disable dovecot for now"
+            if self.disable_mail
+            else "Start and enable Dovecot",
+            service="dovecot.service",
+            running=False if self.disable_mail else True,
+            enabled=False if self.disable_mail else True,
+            restarted=restart,
+            daemon_reload=self.daemon_reload,
+        )
+        self.need_restart = False
+
+
+def _install_dovecot_package(package: str, arch: str):
+    arch = "amd64" if arch == "x86_64" else arch
+    arch = "arm64" if arch == "aarch64" else arch
+    url = f"https://download.delta.chat/dovecot/dovecot-{package}_2.3.21%2Bdfsg1-3_{arch}.deb"
+    deb_filename = "/root/" + url.split("/")[-1]
+
+    match (package, arch):
+        case ("core", "amd64"):
+            sha256 = "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d"
+        case ("core", "arm64"):
+            sha256 = "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9"
+        case ("imapd", "amd64"):
+            sha256 = "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86"
+        case ("imapd", "arm64"):
+            sha256 = "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f"
+        case ("lmtpd", "amd64"):
+            sha256 = "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab"
+        case ("lmtpd", "arm64"):
+            sha256 = "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f"
+        case _:
+            apt.packages(packages=[f"dovecot-{package}"])
+            return
+
+    files.download(
+        name=f"Download dovecot-{package}",
+        src=url,
+        dest=deb_filename,
+        sha256sum=sha256,
+        cache_time=60 * 60 * 24 * 365 * 10,  # never redownload the package
+    )
+
+    apt.deb(name=f"Install dovecot-{package}", src=deb_filename)
+
+
+def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
+    """Configures Dovecot IMAP server."""
+    need_restart = False
+    daemon_reload = False
+
+    main_config = files.template(
+        src=get_resource("dovecot/dovecot.conf.j2"),
+        dest="/etc/dovecot/dovecot.conf",
+        user="root",
+        group="root",
+        mode="644",
+        config=config,
+        debug=debug,
+        disable_ipv6=config.disable_ipv6,
+    )
+    need_restart |= main_config.changed
+    auth_config = files.put(
+        src=get_resource("dovecot/auth.conf"),
+        dest="/etc/dovecot/auth.conf",
+        user="root",
+        group="root",
+        mode="644",
+    )
+    need_restart |= auth_config.changed
+    lua_push_notification_script = files.put(
+        src=get_resource("dovecot/push_notification.lua"),
+        dest="/etc/dovecot/push_notification.lua",
+        user="root",
+        group="root",
+        mode="644",
+    )
+    need_restart |= lua_push_notification_script.changed
+
+    # as per https://doc.dovecot.org/configuration_manual/os/
+    # it is recommended to set the following inotify limits
+    for name in ("max_user_instances", "max_user_watches"):
+        key = f"fs.inotify.{name}"
+        if host.get_fact(Sysctl)[key] > 65535:
+            # Skip updating limits if already sufficient
+            # (enables running in incus containers where sysctl readonly)
+            continue
+        server.sysctl(
+            name=f"Change {key}",
+            key=key,
+            value=65535,
+            persist=True,
+        )
+
+    timezone_env = files.line(
+        name="Set TZ environment variable",
+        path="/etc/environment",
+        line="TZ=:/etc/localtime",
+    )
+    need_restart |= timezone_env.changed
+
+    restart_conf = files.put(
+        name="dovecot: restart automatically on failure",
+        src=get_resource("service/10_restart.conf"),
+        dest="/etc/systemd/system/dovecot.service.d/10_restart.conf",
+    )
+    daemon_reload |= restart_conf.changed
+
+    return need_restart, daemon_reload

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -113,7 +113,7 @@ mail_attribute_dict = proxy:/run/chatmail-metadata/metadata.socket:metadata
 # `imap_zlib` enables IMAP COMPRESS (RFC 4978).
 # <https://datatracker.ietf.org/doc/html/rfc4978.html>
 protocol imap {
-  mail_plugins = $mail_plugins imap_zlib imap_quota last_login
+  mail_plugins = $mail_plugins imap_quota last_login {% if config.imap_compress %}imap_zlib{% endif %}
   imap_metadata = yes
 }
 
@@ -252,3 +252,28 @@ protocol imap {
   rawlog_dir = %h 
 } 
 {% endif %}
+
+{% if not config.imap_compress %}
+# Hibernate IDLE users to save memory and CPU resources
+# NOTE: this will have no effect if imap_zlib plugin is used
+imap_hibernate_timeout = 30s
+service imap {
+  # Note that this change will allow any process running as
+  # $default_internal_user (dovecot) to access mails as any other user.
+  # This may be insecure in some installations, which is why this isn't
+  # done by default.
+  unix_listener imap-master {
+    user = $default_internal_user
+  }
+}
+# The following is the default already in v2.3.1+:
+service imap {
+  extra_groups = $default_internal_group
+}
+service imap-hibernate {
+  unix_listener imap-hibernate {
+    mode = 0660
+    group = $default_internal_group
+  }
+}
+{% endif %}

cmdeploy/src/cmdeploy/mtail/deployer.py

@@ -0,0 +1,68 @@
+from pyinfra import facts, host
+from pyinfra.operations import apt, files, server, systemd
+
+from cmdeploy.basedeploy import (
+    Deployer,
+    get_resource,
+)
+
+
+class MtailDeployer(Deployer):
+    def __init__(self, mtail_address):
+        self.mtail_address = mtail_address
+
+    def install(self):
+        # Uninstall mtail package to install a static binary.
+        apt.packages(name="Uninstall mtail", packages=["mtail"], present=False)
+
+        (url, sha256sum) = {
+            "x86_64": (
+                "https://github.com/google/mtail/releases/download/v3.0.8/mtail_3.0.8_linux_amd64.tar.gz",
+                "123c2ee5f48c3eff12ebccee38befd2233d715da736000ccde49e3d5607724e4",
+            ),
+            "aarch64": (
+                "https://github.com/google/mtail/releases/download/v3.0.8/mtail_3.0.8_linux_arm64.tar.gz",
+                "aa04811c0929b6754408676de520e050c45dddeb3401881888a092c9aea89cae",
+            ),
+        }[host.get_fact(facts.server.Arch)]
+
+        server.shell(
+            name="Download mtail",
+            commands=[
+                f"(echo '{sha256sum} /usr/local/bin/mtail' | sha256sum -c) || (curl -L {url} | gunzip | tar -x -f - mtail -O >/usr/local/bin/mtail.new && mv /usr/local/bin/mtail.new /usr/local/bin/mtail)",
+                "chmod 755 /usr/local/bin/mtail",
+            ],
+        )
+
+    def configure(self):
+        # Using our own systemd unit instead of `/usr/lib/systemd/system/mtail.service`.
+        # This allows to read from journalctl instead of log files.
+        files.template(
+            src=get_resource("mtail/mtail.service.j2"),
+            dest="/etc/systemd/system/mtail.service",
+            user="root",
+            group="root",
+            mode="644",
+            address=self.mtail_address or "127.0.0.1",
+            port=3903,
+        )
+
+        mtail_conf = files.put(
+            name="Mtail configuration",
+            src=get_resource("mtail/delivered_mail.mtail"),
+            dest="/etc/mtail/delivered_mail.mtail",
+            user="root",
+            group="root",
+            mode="644",
+        )
+        self.need_restart = mtail_conf.changed
+
+    def activate(self):
+        systemd.service(
+            name="Start and enable mtail",
+            service="mtail.service",
+            running=bool(self.mtail_address),
+            enabled=bool(self.mtail_address),
+            restarted=self.need_restart,
+        )
+        self.need_restart = False

cmdeploy/src/cmdeploy/nginx/deployer.py

@@ -0,0 +1,117 @@
+from chatmaild.config import Config
+from pyinfra.operations import apt, files, systemd
+
+from cmdeploy.basedeploy import (
+    Deployer,
+    get_resource,
+)
+
+
+class NginxDeployer(Deployer):
+    def __init__(self, config):
+        self.config = config
+
+    def install(self):
+        #
+        # If we allow nginx to start up on install, it will grab port
+        # 80, which then will block acmetool from listening on the port.
+        # That in turn prevents getting certificates, which then causes
+        # an error when we try to start nginx on the custom config
+        # that leaves port 80 open but also requires certificates to
+        # be present.  To avoid getting into that interlocking mess,
+        # we use policy-rc.d to prevent nginx from starting up when it
+        # is installed.
+        #
+        # This approach allows us to avoid performing any explicit
+        # systemd operations during the install stage (as opposed to
+        # allowing it to start and then forcing it to stop), which allows
+        # the install stage to run in non-systemd environments like a
+        # container image build.
+        #
+        # For documentation about policy-rc.d, see:
+        # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
+        #
+        files.put(
+            src=get_resource("policy-rc.d"),
+            dest="/usr/sbin/policy-rc.d",
+            user="root",
+            group="root",
+            mode="755",
+        )
+
+        apt.packages(
+            name="Install nginx",
+            packages=["nginx", "libnginx-mod-stream"],
+        )
+
+        files.file("/usr/sbin/policy-rc.d", present=False)
+
+    def configure(self):
+        self.need_restart = _configure_nginx(self.config)
+
+    def activate(self):
+        systemd.service(
+            name="Start and enable nginx",
+            service="nginx.service",
+            running=True,
+            enabled=True,
+            restarted=self.need_restart,
+        )
+        self.need_restart = False
+
+
+def _configure_nginx(config: Config, debug: bool = False) -> bool:
+    """Configures nginx HTTP server."""
+    need_restart = False
+
+    main_config = files.template(
+        src=get_resource("nginx/nginx.conf.j2"),
+        dest="/etc/nginx/nginx.conf",
+        user="root",
+        group="root",
+        mode="644",
+        config={"domain_name": config.mail_domain},
+        disable_ipv6=config.disable_ipv6,
+    )
+    need_restart |= main_config.changed
+
+    autoconfig = files.template(
+        src=get_resource("nginx/autoconfig.xml.j2"),
+        dest="/var/www/html/.well-known/autoconfig/mail/config-v1.1.xml",
+        user="root",
+        group="root",
+        mode="644",
+        config={"domain_name": config.mail_domain},
+    )
+    need_restart |= autoconfig.changed
+
+    mta_sts_config = files.template(
+        src=get_resource("nginx/mta-sts.txt.j2"),
+        dest="/var/www/html/.well-known/mta-sts.txt",
+        user="root",
+        group="root",
+        mode="644",
+        config={"domain_name": config.mail_domain},
+    )
+    need_restart |= mta_sts_config.changed
+
+    # install CGI newemail script
+    #
+    cgi_dir = "/usr/lib/cgi-bin"
+    files.directory(
+        name=f"Ensure {cgi_dir} exists",
+        path=cgi_dir,
+        user="root",
+        group="root",
+    )
+
+    files.put(
+        name="Upload cgi newemail.py script",
+        src=get_resource("newemail.py", pkg="chatmaild").open("rb"),
+        dest=f"{cgi_dir}/newemail.py",
+        user="root",
+        group="root",
+        mode="755",
+    )
+
+    return need_restart

cmdeploy/src/cmdeploy/opendkim/deployer.py

@@ -0,0 +1,123 @@
+"""
+Installs OpenDKIM
+"""
+
+from pyinfra import host
+from pyinfra.facts.files import File
+from pyinfra.operations import apt, files, server, systemd
+
+from cmdeploy.basedeploy import Deployer, get_resource
+
+
+class OpendkimDeployer(Deployer):
+    required_users = [("opendkim", None, ["opendkim"])]
+
+    def __init__(self, mail_domain):
+        self.mail_domain = mail_domain
+
+    def install(self):
+        apt.packages(
+            name="apt install opendkim opendkim-tools",
+            packages=["opendkim", "opendkim-tools"],
+        )
+
+    def configure(self):
+        domain = self.mail_domain
+        dkim_selector = "opendkim"
+        """Configures OpenDKIM"""
+        need_restart = False
+
+        main_config = files.template(
+            src=get_resource("opendkim/opendkim.conf"),
+            dest="/etc/opendkim.conf",
+            user="root",
+            group="root",
+            mode="644",
+            config={"domain_name": domain, "opendkim_selector": dkim_selector},
+        )
+        need_restart |= main_config.changed
+
+        screen_script = files.put(
+            src=get_resource("opendkim/screen.lua"),
+            dest="/etc/opendkim/screen.lua",
+            user="root",
+            group="root",
+            mode="644",
+        )
+        need_restart |= screen_script.changed
+
+        final_script = files.put(
+            src=get_resource("opendkim/final.lua"),
+            dest="/etc/opendkim/final.lua",
+            user="root",
+            group="root",
+            mode="644",
+        )
+        need_restart |= final_script.changed
+
+        files.directory(
+            name="Add opendkim directory to /etc",
+            path="/etc/opendkim",
+            user="opendkim",
+            group="opendkim",
+            mode="750",
+            present=True,
+        )
+
+        keytable = files.template(
+            src=get_resource("opendkim/KeyTable"),
+            dest="/etc/dkimkeys/KeyTable",
+            user="opendkim",
+            group="opendkim",
+            mode="644",
+            config={"domain_name": domain, "opendkim_selector": dkim_selector},
+        )
+        need_restart |= keytable.changed
+
+        signing_table = files.template(
+            src=get_resource("opendkim/SigningTable"),
+            dest="/etc/dkimkeys/SigningTable",
+            user="opendkim",
+            group="opendkim",
+            mode="644",
+            config={"domain_name": domain, "opendkim_selector": dkim_selector},
+        )
+        need_restart |= signing_table.changed
+        files.directory(
+            name="Add opendkim socket directory to /var/spool/postfix",
+            path="/var/spool/postfix/opendkim",
+            user="opendkim",
+            group="opendkim",
+            mode="750",
+            present=True,
+        )
+
+        if not host.get_fact(File, f"/etc/dkimkeys/{dkim_selector}.private"):
+            server.shell(
+                name="Generate OpenDKIM domain keys",
+                commands=[
+                    f"/usr/sbin/opendkim-genkey -D /etc/dkimkeys -d {domain} -s {dkim_selector}"
+                ],
+                _use_su_login=True,
+                _su_user="opendkim",
+            )
+
+        service_file = files.put(
+            name="Configure opendkim to restart once a day",
+            src=get_resource("opendkim/systemd.conf"),
+            dest="/etc/systemd/system/opendkim.service.d/10-prevent-memory-leak.conf",
+        )
+        need_restart |= service_file.changed
+
+        self.need_restart = need_restart
+
+    def activate(self):
+        systemd.service(
+            name="Start and enable OpenDKIM",
+            service="opendkim.service",
+            running=True,
+            enabled=True,
+            daemon_reload=self.need_restart,
+            restarted=self.need_restart,
+        )
+        self.need_restart = False

cmdeploy/src/cmdeploy/opendkim/final.lua

@@ -9,9 +9,11 @@ if nsigs == nil then
 	return nil
 end
 
+local valid = false
+local error_msg = "No valid DKIM signature found."
 for i = 1, nsigs do
-        sig = odkim.get_sighandle(ctx, i - 1)
-        sigres = odkim.sig_result(sig)
+	sig = odkim.get_sighandle(ctx, i - 1)
+	sigres = odkim.sig_result(sig)
 
 	-- All signatures that do not correspond to From: 
 	-- were ignored in screen.lua and return sigres -1.
@@ -19,10 +21,21 @@ for i = 1, nsigs do
 	-- Any valid signature that was not ignored like this
 	-- means the message is acceptable.
 	if sigres == 0 then
-		return nil
-	end	
+		valid = true
+    else
+        error_msg = "DKIM signature is invalid, error code " .. tostring(sigres) .. ", search https://github.com/trusteddomainproject/OpenDKIM/blob/master/libopendkim/dkim.h#L108"
+	end
+end
+
+if valid then
+	-- Strip all DKIM-Signature headers after successful validation
+	-- Delete in reverse order to avoid index shifting.
+	for i = nsigs, 1, -1 do
+		odkim.del_header(ctx, "DKIM-Signature", i)
+	end
+else
+	odkim.set_reply(ctx, "554", "5.7.1", error_msg)
+	odkim.set_result(ctx, SMFIS_REJECT)
 end
 
-odkim.set_reply(ctx, "554", "5.7.1", "No valid DKIM signature found")
-odkim.set_result(ctx, SMFIS_REJECT)
 return nil

cmdeploy/src/cmdeploy/policy-rc.d

@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "All runlevel operations denied by policy" >&2
+exit 101

cmdeploy/src/cmdeploy/postfix/deployer.py

@@ -0,0 +1,86 @@
+from pyinfra.operations import apt, files, systemd
+
+from cmdeploy.basedeploy import Deployer, get_resource
+
+
+class PostfixDeployer(Deployer):
+    required_users = [("postfix", None, ["opendkim"])]
+    daemon_reload = False
+
+    def __init__(self, config, disable_mail):
+        self.config = config
+        self.disable_mail = disable_mail
+
+    def install(self):
+        apt.packages(
+            name="Install Postfix",
+            packages="postfix",
+        )
+
+    def configure(self):
+        config = self.config
+        need_restart = False
+
+        main_config = files.template(
+            src=get_resource("postfix/main.cf.j2"),
+            dest="/etc/postfix/main.cf",
+            user="root",
+            group="root",
+            mode="644",
+            config=config,
+            disable_ipv6=config.disable_ipv6,
+        )
+        need_restart |= main_config.changed
+
+        master_config = files.template(
+            src=get_resource("postfix/master.cf.j2"),
+            dest="/etc/postfix/master.cf",
+            user="root",
+            group="root",
+            mode="644",
+            debug=False,
+            config=config,
+        )
+        need_restart |= master_config.changed
+
+        header_cleanup = files.put(
+            src=get_resource("postfix/submission_header_cleanup"),
+            dest="/etc/postfix/submission_header_cleanup",
+            user="root",
+            group="root",
+            mode="644",
+        )
+        need_restart |= header_cleanup.changed
+
+        # Login map that 1:1 maps email address to login.
+        login_map = files.put(
+            src=get_resource("postfix/login_map"),
+            dest="/etc/postfix/login_map",
+            user="root",
+            group="root",
+            mode="644",
+        )
+        need_restart |= login_map.changed
+
+        restart_conf = files.put(
+            name="postfix: restart automatically on failure",
+            src=get_resource("service/10_restart.conf"),
+            dest="/etc/systemd/system/dovecot.service.d/10_restart.conf",
+        )
+        self.daemon_reload = restart_conf.changed
+        self.need_restart = need_restart
+
+    def activate(self):
+        restart = False if self.disable_mail else self.need_restart
+
+        systemd.service(
+            name="disable postfix for now"
+            if self.disable_mail
+            else "Start and enable Postfix",
+            service="postfix.service",
+            running=False if self.disable_mail else True,
+            enabled=False if self.disable_mail else True,
+            restarted=restart,
+            daemon_reload=self.daemon_reload,
+        )
+        self.need_restart = False

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -27,7 +27,7 @@ smtp_tls_servername = hostname
 smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
 smtp_tls_policy_maps = inline:{nauta.cu=may}
 smtp_tls_protocols = >=TLSv1.2
-smtpd_tls_protocols = >=TLSv1.2
+smtp_tls_mandatory_protocols = >=TLSv1.2
 
 # Disable anonymous cipher suites
 # and known insecure algorithms.

cmdeploy/src/cmdeploy/postfix/master.cf.j2

@@ -15,6 +15,7 @@ smtp      inet  n       -       y       -       -       smtpd -v
 smtp      inet  n       -       y       -       -       smtpd 
 {%- endif %}
   -o smtpd_tls_security_level=encrypt
+  -o smtpd_tls_mandatory_protocols=>=TLSv1.2
   -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port_incoming }}
 submission inet n       -       y       -       5000    smtpd
   -o syslog_name=postfix/submission

cmdeploy/src/cmdeploy/remote/rdns.py

@@ -12,7 +12,7 @@ All functions of this module
 
 import re
 
-from .rshell import CalledProcessError, shell, log_progress
+from .rshell import CalledProcessError, log_progress, shell
 
 
 def perform_initial_checks(mail_domain, pre_command=""):
@@ -26,7 +26,9 @@ def perform_initial_checks(mail_domain, pre_command=""):
     WWW = query_dns("CNAME", f"www.{mail_domain}")
 
     res = dict(mail_domain=mail_domain, A=A, AAAA=AAAA, MTA_STS=MTA_STS, WWW=WWW)
-    res["acme_account_url"] = shell(pre_command + "acmetool account-url", fail_ok=True, print=log_progress)
+    res["acme_account_url"] = shell(
+        pre_command + "acmetool account-url", fail_ok=True, print=log_progress
+    )
     res["dkim_entry"], res["web_dkim_entry"] = get_dkim_entry(
         mail_domain, pre_command, dkim_selector="opendkim"
     )
@@ -35,7 +37,10 @@ def perform_initial_checks(mail_domain, pre_command=""):
         return res
 
     # parse out sts-id if exists, example: "v=STSv1; id=2090123"
-    parts = query_dns("TXT", f"_mta-sts.{mail_domain}").split("id=")
+    mta_sts_txt = query_dns("TXT", f"_mta-sts.{mail_domain}")
+    if not mta_sts_txt:
+        return res
+    parts = mta_sts_txt.split("id=")
     res["sts_id"] = parts[1].rstrip('"') if len(parts) == 2 else ""
     return res
 
@@ -45,7 +50,7 @@ def get_dkim_entry(mail_domain, pre_command, dkim_selector):
         dkim_pubkey = shell(
             f"{pre_command}openssl rsa -in /etc/dkimkeys/{dkim_selector}.private "
             "-pubout 2>/dev/null | awk '/-/{next}{printf(\"%s\",$0)}'",
-            print=log_progress
+            print=log_progress,
         )
     except CalledProcessError:
         return
@@ -62,9 +67,9 @@ def query_dns(typ, domain):
     # Get autoritative nameserver from the SOA record.
     soa_answers = [
         x.split()
-        for x in shell(f"dig -r -q {domain} -t SOA +noall +authority +answer", print=log_progress).split(
-            "\n"
-        )
+        for x in shell(
+            f"dig -r -q {domain} -t SOA +noall +authority +answer", print=log_progress
+        ).split("\n")
     ]
     soa = [a for a in soa_answers if len(a) >= 3 and a[3] == "SOA"]
     if not soa:
@@ -73,7 +78,7 @@ def query_dns(typ, domain):
 
     # Query authoritative nameserver directly to bypass DNS cache.
     res = shell(f"dig @{ns} -r -q {domain} -t {typ} +short", print=log_progress)
-    return next((line for line in res.split("\n") if not line.startswith(';')), '')
+    return next((line for line in res.split("\n") if not line.startswith(";")), "")
 
 
 def check_zonefile(zonefile, verbose=True):

cmdeploy/src/cmdeploy/remote/rshell.py

@@ -1,5 +1,4 @@
 import sys
-
 from subprocess import DEVNULL, CalledProcessError, check_output
 
 

cmdeploy/src/cmdeploy/run.py

@@ -3,7 +3,9 @@ import os
 
 import pyinfra
 
-from cmdeploy import deploy_chatmail
+# pyinfra runs this module as a python file and not as a module so
+# import paths must be absolute
+from cmdeploy.deployers import deploy_chatmail
 
 
 def main():

cmdeploy/src/cmdeploy/service/10_restart.conf

@@ -0,0 +1,3 @@
+[Service]
+Restart=always
+RestartSec=30

cmdeploy/src/cmdeploy/service/echobot.service.f

@@ -1,67 +0,0 @@
-[Unit]
-Description=Chatmail echo bot for testing it works
-
-[Service]
-ExecStart={execpath} {config_path}
-Environment="PATH={remote_venv_dir}:$PATH"
-Restart=always
-RestartSec=30
-
-User=echobot
-Group=echobot
-
-# Create /var/lib/echobot
-StateDirectory=echobot
-
-# Create /run/echobot
-#
-# echobot stores /run/echobot/password
-# with a password there, which doveauth then reads.
-RuntimeDirectory=echobot
-
-WorkingDirectory=/var/lib/echobot
-
-# Apply security restrictions suggested by
-#   systemd-analyze security echobot.service
-CapabilityBoundingSet=
-LockPersonality=true
-MemoryDenyWriteExecute=true
-NoNewPrivileges=true
-PrivateDevices=true
-PrivateMounts=true
-PrivateTmp=true
-
-# We need to know about doveauth user to give it access to /run/echobot/password
-PrivateUsers=false
-
-ProtectClock=true
-ProtectControlGroups=true
-ProtectHostname=true
-ProtectKernelLogs=true
-ProtectKernelModules=true
-ProtectKernelTunables=true
-ProtectProc=noaccess
-
-# Should be "strict", but we currently write /accounts folder in a protected path
-ProtectSystem=full
-
-RemoveIPC=true
-RestrictAddressFamilies=AF_INET AF_INET6
-RestrictNamespaces=true
-RestrictRealtime=true
-RestrictSUIDSGID=true
-SystemCallArchitectures=native
-SystemCallFilter=~@clock
-SystemCallFilter=~@cpu-emulation
-SystemCallFilter=~@debug
-SystemCallFilter=~@module
-SystemCallFilter=~@mount
-SystemCallFilter=~@obsolete
-SystemCallFilter=~@raw-io
-SystemCallFilter=~@reboot
-SystemCallFilter=~@resources
-SystemCallFilter=~@swap
-UMask=0077
-
-[Install]
-WantedBy=multi-user.target

cmdeploy/src/cmdeploy/sshexec.py

@@ -93,7 +93,7 @@ class LocalExec:
         where = "locally"
         if self.docker:
             if call == remote.rdns.perform_initial_checks:
-                kwargs['pre_command'] = "docker exec chatmail "
+                kwargs["pre_command"] = "docker exec chatmail "
                 where = "in docker"
         if self.verbose:
             print(f"Running {where}: {call.__name__}(**{kwargs})")

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -1,5 +1,4 @@
 import datetime
-import os
 import smtplib
 import socket
 import subprocess
@@ -8,7 +7,6 @@ import time
 import pytest
 
 from cmdeploy import remote
-from cmdeploy.cmdeploy import main
 from cmdeploy.sshexec import SSHExec
 
 
@@ -35,7 +33,7 @@ class TestSSHExecutor:
         out, err = capsys.readouterr()
         assert err.startswith("Collecting")
         # XXX could not figure out how capturing can be made to work properly
-        #assert err.endswith("....\n")
+        # assert err.endswith("....\n")
         assert err.count("\n") == 1
 
         sshexec.verbose = True
@@ -45,7 +43,7 @@ class TestSSHExecutor:
         out, err = capsys.readouterr()
         lines = err.split("\n")
         # XXX could not figure out how capturing can be made to work properly
-        #assert len(lines) > 4
+        # assert len(lines) > 4
         assert remote.rdns.perform_initial_checks.__doc__ in lines[0]
 
     def test_exception(self, sshexec, capsys):
@@ -70,47 +68,6 @@ class TestSSHExecutor:
         assert (now - since_date).total_seconds() < 60 * 60 * 51
 
 
-def test_status_cmd(chatmail_config, capsys, request):
-    os.chdir(request.config.invocation_params.dir)
-    assert main(["status"]) == 0
-    status_out = capsys.readouterr()
-    print(status_out.out)
-
-    services = [
-        "acmetool-redirector",
-        "chatmail-metadata",
-        "doveauth",
-        "dovecot",
-        "echobot",
-        "fcgiwrap",
-        "filtermail-incoming",
-        "filtermail",
-        "lastlogin",
-        "nginx",
-        "opendkim",
-        "postfix@-",
-        "systemd-journald",
-        "turnserver",
-        "unbound",
-    ]
-    not_running = []
-    for service in services:
-        active = False
-        for line in status_out:
-            if service in line:
-                active = True
-                if not "loaded" in line:
-                    active = False
-                if not "active" in line:
-                    active = False
-                if not "running" in line:
-                    active = False
-                break
-        if not active:
-            not_running.append(service)
-    assert not_running == []
-
-
 def test_timezone_env(remote):
     for line in remote.iter_output("env"):
         print(line)

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -160,22 +160,3 @@ def test_hide_senders_ip_address(cmfactory):
     user2.direct_imap.select_folder("Inbox")
     msg = user2.direct_imap.get_all_messages()[0]
     assert public_ip not in msg.obj.as_string()
-
-
-def test_echobot(cmfactory, chatmail_config, lp, sshdomain):
-    ac = cmfactory.get_online_accounts(1)[0]
-
-    # establish contact with echobot
-    sshexec = SSHExec(sshdomain)
-    command = "cat /var/lib/echobot/invite-link.txt"
-    echo_invite_link = sshexec(call=rshell.shell, kwargs=dict(command=command))
-    chat = ac.qr_setup_contact(echo_invite_link)
-    ac._evtracker.wait_securejoin_joiner_progress(1000)
-
-    # send message and check it gets replied back
-    lp.sec("Send message to echobot")
-    text = "hi, I hope you text me back"
-    chat.send_text(text)
-    lp.sec("Wait for reply from echobot")
-    reply = ac._evtracker.wait_next_incoming_message()
-    assert reply.text == text

cmdeploy/src/cmdeploy/tests/online/test_3_status.py

@@ -0,0 +1,49 @@
+import os
+
+from cmdeploy.cmdeploy import main
+
+
+def test_status_cmd(chatmail_config, capsys, request):
+    os.chdir(request.config.invocation_params.dir)
+    assert main(["status"]) == 0
+    status_out = capsys.readouterr()
+    print(status_out.out)
+
+    assert len(status_out.out.splitlines()) > 5
+
+    """
+    don't test actual server state:
+
+    services = [
+        "acmetool-redirector",
+        "chatmail-metadata",
+        "doveauth",
+        "dovecot",
+        "fcgiwrap",
+        "filtermail-incoming",
+        "filtermail",
+        "lastlogin",
+        "nginx",
+        "opendkim",
+        "postfix@-",
+        "systemd-journald",
+        "turnserver",
+        "unbound",
+    ]
+    not_running = []
+    for service in services:
+        active = False
+        for line in status_out:
+            if service in line:
+                active = True
+                if not "loaded" in line:
+                    active = False
+                if not "active" in line:
+                    active = False
+                if not "running" in line:
+                    active = False
+                break
+        if not active:
+            not_running.append(service)
+    assert not_running == []
+    """

cmdeploy/src/cmdeploy/tests/test_dns.py

@@ -65,7 +65,9 @@ class TestPerformInitialChecks:
         remote_data = remote.rdns.perform_initial_checks("some.domain")
         assert remote_data["A"] == mockdns_expected["A"]["some.domain"]
         assert remote_data["AAAA"] == mockdns_expected["AAAA"]["some.domain"]
-        assert remote_data["MTA_STS"] == mockdns_expected["CNAME"]["mta-sts.some.domain"]
+        assert (
+            remote_data["MTA_STS"] == mockdns_expected["CNAME"]["mta-sts.some.domain"]
+        )
         assert remote_data["WWW"] == mockdns_expected["CNAME"]["www.some.domain"]
 
     @pytest.mark.parametrize("drop", ["A", "AAAA"])

cmdeploy/src/cmdeploy/www.py

@@ -1,10 +1,10 @@
 import hashlib
 import importlib.resources
+import re
 import time
 import traceback
 import webbrowser
 from pathlib import Path
-import re
 
 import markdown
 from chatmaild.config import read_config
@@ -12,8 +12,9 @@ from jinja2 import Template
 
 from .genqr import gen_qr_png_data
 
-
-_MERGE_CONFLICT_RE = re.compile(r"^<<<<<<<.+^=======.+^>>>>>>>", re.DOTALL | re.MULTILINE)
+_MERGE_CONFLICT_RE = re.compile(
+    r"^<<<<<<<.+^=======.+^>>>>>>>", re.DOTALL | re.MULTILINE
+)
 
 
 def snapshot_dir_stats(somedir):
@@ -139,34 +140,34 @@ def main():
     config.webdev = True
     assert config.mail_domain
 
-    # start web page generation, open a browser and wait for changes
     www_path, src_path, build_dir = get_paths(config)
     build_dir = build_webpages(src_path, build_dir, config)
     index_path = build_dir.joinpath("index.html")
     webbrowser.open(str(index_path))
-    stats = snapshot_dir_stats(src_path)
+
     print(f"\nOpened URL: file://{index_path.resolve()}\n")
-    print(f"watching {src_path} directory for changes")
+    print(f"Watching {src_path} directory for changes...")
 
+    stats = snapshot_dir_stats(src_path)
     changenum = 0
-    count = 0
+    debounce_time = 0.5  # wait 0.5s after detecting a change
+
     while True:
+        time.sleep(1)
         newstats = snapshot_dir_stats(src_path)
-        if newstats == stats and count % 60 != 0:
-            count += 1
-            time.sleep(1.0)
-            continue
 
-        for key in newstats:
-            if stats[key] != newstats[key]:
-                print(f"*** CHANGED: {key}")
-                changenum += 1
+        if newstats != stats:
+            changed_files = [f for f in newstats if stats.get(f) != newstats[f]]
+            for f in changed_files:
+                print(f"*** CHANGED: {f}")
 
-        stats = newstats
-        build_webpages(src_path, build_dir, config)
-        print(f"[{changenum}] regenerated web pages at: {index_path}")
-        print(f"URL: file://{index_path.resolve()}\n\n")
-        count = 0
+            stats = newstats
+            changenum += 1
+            build_webpages(src_path, build_dir, config)
+            print(f"[{changenum}] regenerated web pages at: {index_path}")
+            print(f"URL: file://{index_path.resolve()}\n\n")
+
+            time.sleep(debounce_time)  # simple debounce
 
 
 if __name__ == "__main__":

doc/README.md

@@ -6,7 +6,7 @@ You can use the `make` command and `make html` to build web pages.
 
 You need a Python environment where the following install was excuted: 
 
-    pip install sphinx-build furo sphinx-autobuild 
+    pip install furo sphinx-autobuild
 
 To develop/change documentation, you can then do: 
 

doc/source/migrate.rst

@@ -26,7 +26,7 @@ this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
    or receive messages until the migration is completed.
 
 2. Now we want to copy ``/home/vmail``, ``/var/lib/acme``,
-   ``/etc/dkimkeys``, ``/run/echobot``, and ``/var/spool/postfix`` to
+   ``/etc/dkimkeys``, and ``/var/spool/postfix`` to
    the new site. Login to the old site while forwarding your SSH agent
    so you can copy directly from the old to the new site with your SSH
    key:
@@ -34,11 +34,11 @@ this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
    ::
 
        ssh -A root@13.37.13.37
-       tar c - /home/vmail/mail /var/lib/acme /etc/dkimkeys /run/echobot /var/spool/postfix | ssh root@13.12.23.42 "tar x -C /"
+       tar c - /home/vmail/mail /var/lib/acme /etc/dkimkeys /var/spool/postfix | ssh root@13.12.23.42 "tar x -C /"
 
-   This transfers all addresses, the TLS certificate, DKIM keys (so DKIM
-   DNS record remains valid), and the echobot’s password so it continues
-   to function. It also preserves the Postfix mail spool so any messages
+   This transfers all addresses, the TLS certificate,
+   and DKIM keys (so DKIM DNS record remains valid).
+   It also preserves the Postfix mail spool so any messages
    pending delivery will still be delivered.
 
 3. Install chatmail on the new machine:
@@ -58,7 +58,6 @@ this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
        chown root: -R /var/lib/acme
        chown opendkim: -R /etc/dkimkeys
        chown vmail: -R /home/vmail/mail
-       chown echobot: -R /run/echobot
 
 5. Now, update DNS entries.
 

doc/source/overview.rst

@@ -109,10 +109,6 @@ short overview of ``chatmaild`` services:
    is contacted by Dovecot when a user logs in and stores the date of
    the login.
 
--  `echobot <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/echo.py>`_
-   is a small bot for test purposes. It simply echoes back messages from
-   users.
-
 -  `metrics <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/metrics.py>`_
    collects some metrics and displays them at
    ``https://example.org/metrics``.
@@ -126,14 +122,13 @@ web page. Edit them before deploying to make your chatmail relay
 stand out.
 
 
-Component dependency diagram
---------------------------------------
+Chatmail relay dependency diagram
+---------------------------------
 
 .. mermaid::
    :caption: This diagram shows relay components and dependencies/communication paths.
 
     graph LR;
-        cmdeploy --- sshd;
         letsencrypt --- |80|acmetool-redirector;
         acmetool-redirector --- |443|nginx-right(["`nginx
         (external)`"]);
@@ -147,33 +142,62 @@ Component dependency diagram
         nginx-internal --- autoconfig.xml;
         certs-nginx[("`TLS certs
         /var/lib/acme`")] --> nginx-internal;
-        cron --- chatmail-metrics;
-        cron --- acmetool;
+        systemd-timer --- chatmail-metrics;
+        systemd-timer --- acmetool;
+        systemd-timer --- chatmail-expire-daily;
+        systemd-timer --- chatmail-fsreport-daily;
         chatmail-metrics --- website;
         acmetool --> certs[("`TLS certs
         /var/lib/acme`")];
         nginx-external --- |993|dovecot;
+        postfix --- |SASL|dovecot;
         autoconfig.xml --- postfix;
         autoconfig.xml --- dovecot;
-        postfix --- echobot;
-        postfix --- |10080,10081|filtermail;
-        postfix --- users["`User data
-        home/vmail/mail`"];
-        postfix --- |doveauth.socket|doveauth;
+        postfix --- |10080|filtermail-outgoing;
+        postfix --- |10081|filtermail-incoming;
+        filtermail-outgoing --- |10025 reinject|postfix;
+        filtermail-incoming --- |10026 reinject|postfix;
         dovecot --- |doveauth.socket|doveauth;
-        dovecot --- users;
-        dovecot --- |metadata.socket|chatmail-metadata;
-        doveauth --- users;
-        chatmail-expire-daily --- users;
-        chatmail-fsreport-daily --- users;
+        dovecot --- |message delivery|maildir["maildir
+        /home/vmail/.../user"];
+        dovecot --- |lastlogin.socket|lastlogin;
+        dovecot --- chatmail-metadata;
+        lastlogin --- maildir;
+        doveauth --- maildir;
+        chatmail-expire-daily --- maildir;
+        chatmail-fsreport-daily --- maildir;
         chatmail-metadata --- iroh-relay;
+        chatmail-metadata --- |encrypted device token| notifications.delta.chat;
         certs-nginx --> postfix;
         certs-nginx --> dovecot;
         style certs fill:#ff6;
+        style website fill:#ff6;
+        style maildir fill:#ff6;
         style certs-nginx fill:#ff6;
-        style nginx-external fill:#fc9;
-        style nginx-right fill:#fc9;
+        style nginx-external fill:#f66;
+        style nginx-right fill:#f66;
+        style postfix fill:#f66;
+        style dovecot fill:#f66;
+        style notification-proxy fill:#f66;
 
+Message between users on the same relay
+---------------------------------------
+
+.. mermaid::
+    :caption: This diagram shows the path a non-federated message takes.
+
+    graph LR;
+        sender --> |465|smtps/smtpd;
+        sender --> |587|submission/smtpd;
+        smtps/smtpd --> |10080|filtermail;
+        submission/smtpd --> |10080|filtermail;
+        filtermail --> |10025|smtpd_reinject;
+        smtpd_reinject --> cleanup;
+        cleanup --> qmgr;
+        qmgr --> smtpd_accepts_message;
+        qmgr --> |lmtp|dovecot;
+        dovecot --> recipient;
+        dovecot --> sender's_other_devices;
 
 Operational details of a chatmail relay
 ----------------------------------------
@@ -245,9 +269,11 @@ Incoming emails must have a valid DKIM signature with
 Signing Domain Identifier (SDID, ``d=`` parameter in the DKIM-Signature
 header) equal to the ``From:`` header domain. This property is checked
 by OpenDKIM screen policy script before validating the signatures. This
-correpsonds to strict :rfc:`DMARC <7489>` alignment (``adkim=s``).
+corresponds to strict :rfc:`DMARC <7489>` alignment (``adkim=s``).
 If there is no valid DKIM signature on the incoming email, the
 sender receives a “5.7.1 No valid DKIM signature found” error.
+After validating the DKIM signature,
+the `final.lua` script strips all ``OpenDKIM:`` headers to reduce message size on disc.
 
 Note that chatmail relays
 
@@ -297,3 +323,48 @@ actually it is a problem with your TLS certificate.
 .. _nginx: https://nginx.org
 .. _pyinfra: https://pyinfra.com
 
+
+Architecture of cmdeploy
+------------------------
+
+cmdeploy is a Python program that uses the pyinfra library to deploy
+chatmail relays, with all the necessary software, configuration, and
+services.  The deployment process performs three primary types of
+operation:
+
+1. Installation of software, universal across all deployments.
+2. Configuration of software, with deploy-specific variations.
+3. Activation of services.
+
+The process is implemented through a family of "deployer" objects
+which all derive from a common ``Deployer`` base class, defined in
+cmdeploy/src/cmdeploy/deployer.py.  Each object provides
+implementation methods for the three stages -- install, configure, and
+activate.  The top-level procedure in ``deploy_chatmail()`` calls
+these methods for all the deployer objects, via the
+``Deployment.perform_stages()`` method, also defined in deployer.py.
+This first calls all the install methods, then the configure methods,
+then the activate methods.
+
+The ``Deployment`` class also implements support for a CMDEPLOY_STAGES
+environment variable, which allows limiting the process to specific
+stages.  Note that some deployers are stateful between the stages
+(this is one reason why they are implemented as objects), and that
+state will not get propagated between stages when run in separate
+invocations of cmdeploy.  This environment variable is intended for
+use in future revisions to support building Docker images with
+software pre-installed, and configuration of containers at run time
+from environment variables.
+
+The, ``install()`` methods for the deployer classes should use 'self'
+as little as possible, preferably not at all.  In particular,
+``install()`` methods should never depend on "config" data, such as
+the config dictionary in ``self.config`` or specific values like
+``self.mail_domain``.  This ensures that these methods can be used to
+perform generic installation operations that are applicable across
+multiple relay deployments, and therefore can be called in the process
+of building a general-purpose container image.
+
+Operations that start services for systemd-based deployments should
+only be called from the ``activate_impl()`` methods.  These methods
+will not be called in non-systemd container environments.

doc/source/proxy.rst

@@ -22,7 +22,12 @@ Note that your chatmail relay still needs to be able to make outgoing
 connections on port 25 to send messages outside.
 
 To setup a reverse proxy (or rather Destination NAT, DNAT) for your
-chatmail relay, put the following configuration in
+chatmail relay, follow these instructions:
+
+Linux
+^^^^^
+
+Put the following configuration in
 ``/etc/nftables.conf``:
 
 ::
@@ -110,5 +115,61 @@ Uncomment in ``/etc/sysctl.conf`` the following two lines:
 Then reboot the relay or do ``sysctl -p`` and
 ``nft -f /etc/nftables.conf``.
 
-Once proxy relay is set up, you can add its IP address to the DNS.
+FreeBSD / pf
+^^^^^^^^^^^^
 
+Put the following configuration in
+``/etc/pf.conf``:
+
+::
+
+    ext_if = "em0"
+    forward_ports = "{ 25, 80, 143, 443, 465, 587, 993 }"
+    chatmail_ipv4 = "AAA.BBB.CCC.DDD"
+    icmp_types = "{ echoreq, echorep, unreach, timex }"
+    chatmail_ipv6 = "XXX::1"
+    icmp6_types = "{ echorep, echoreq, neighbradv, neighbrsol, routeradv, routersol, unreach, toobig, timex }"
+
+    set skip on lo0
+
+    nat on $ext_if inet from any to any -> ($ext_if:0)
+    nat on $ext_if inet6 from any to any -> ($ext_if:0)
+
+    # Define the redirect rules
+    rdr on $ext_if inet proto tcp from any to ($ext_if:0) port $forward_ports -> $chatmail_ipv4
+    rdr on $ext_if inet6 proto tcp from any to ($ext_if:0) port $forward_ports -> $chatmail_ipv6
+
+    # Accept the incoming traffic to the specified ports we will NAT redirect
+    pass in quick on $ext_if inet proto tcp from any to any port $forward_ports flags S/SA modulate state
+    pass in quick on $ext_if inet6 proto tcp from any to any port $forward_ports flags S/SA modulate state
+
+    # Allow incoming SSH for host mgmt
+    pass in quick on $ext_if proto tcp from any to ($ext_if) port 22 flags S/SA modulate state
+
+    # Allow ICMP
+    pass in quick on $ext_if inet proto icmp all icmp-type $icmp_types keep state
+    pass in quick on $ext_if inet6 proto ipv6-icmp all icmp6-type $icmp6_types keep state
+
+    # Allow traffic from anyone to go through the NAT
+    pass on $ext_if inet proto tcp from any to $chatmail_ipv4 flags S/SA modulate state
+    pass on $ext_if inet6 proto tcp from any to $chatmail_ipv6 flags S/SA modulate state
+
+    # Default allow out
+    pass out quick on $ext_if from any to any
+
+    # Default block
+    block drop in log all
+
+Insert into ``/etc/sysctl.conf.local`` the following two lines:
+
+::
+
+    net.inet.ip.forwarding=1
+    net.inet6.ip6.forwarding=1
+
+Activate the sysctls with ``service sysctl onestart``.
+Enable the pf firewall with ``service pf enable``.
+Apply the firewall rules with ``service pf start`` or ``pfctl -f /etc/pf.conf``.
+Note, enabling the firewall may interrupt your SSH session, but you can reconnect.
+
+Once proxy relay is set up, you can add its IP address to the DNS.

doc/source/related.rst

@@ -7,7 +7,7 @@ Active development takes place in the `chatmail/relay github repository <https:/
 You can check out the `'chatmail' tag in the support.delta.chat forum <https://support.delta.chat/tag/chatmail>`_
 and ask to get added to a non-public support chat for debugging issues.
 
-We know of two work-in-progress alternative implementation efforts:
+We know of three work-in-progress alternative implementation efforts:
 
 -  `Mox <https://github.com/mjl-/mox>`_: A Golang email server. `Work
    is in progress <https://github.com/mjl-/mox/issues/251>`_ to modify
@@ -18,3 +18,10 @@ We know of two work-in-progress alternative implementation efforts:
    plugin for the `Maddy email server <https://maddy.email/>`_ which
    aims to implement the chatmail relay features and configuration
    options.
+
+-  `Chatmail Cookbook <https://github.com/feld/chatmail-cookbook>`_:
+   A Chef Cookbook implementing a relay server. The project follows the
+   official relay server software and configurations converted to a Chef
+   Cookbook with only minor differences. The cookbook uses DNS-01 for
+   certificate validation and additionally supports FreeBSD. It does not
+   require a Chef server to use.

www/src/index.md

@@ -23,7 +23,3 @@ you can also **scan this QR code** with Delta Chat:
 🐣 **Choose** your Avatar and Name
 
 💬 **Start** chatting with any Delta Chat contacts using [QR invite codes](https://delta.chat/en/help#howtoe2ee)
-
-{% if config.mail_domain != "nine.testrun.org" %}
-<div class="experimental">Note: this is only a temporary development chatmail service</div>
-{% endif %}

www/src/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /