opendkim/final.lua: also strip Authentication-Results Header

Refactor DKIM-Header-Pruning into a  "if valid-else" construct with a single "return nil" at the end

.github/workflows/docs-preview.yaml

@@ -11,6 +11,9 @@ jobs:
   scripts:
     name: build 
     runs-on: ubuntu-latest
+    environment:
+      name: 'staging.chatmail.at/doc/relay/'
+      url: https://staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}
     steps:
       - uses: actions/checkout@v4
 
@@ -44,36 +47,6 @@ jobs:
           chmod 600 "$HOME/.ssh/key"
           rsync -rILvh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/doc/build/ "${{ secrets.USERNAME }}@chatmail.at:/var/www/html/staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}/"
 
-      - name: "Post links to details"
-        id: details
-        if: steps.prepare.outputs.uploadtoserver
-        run: |
-          # URLs for API connection and uploads
-          export GITHUB_API_URL="https://api.github.com/repos/chatmail/relay/statuses/${{ github.event.after }}"
-          export PREVIEW_LINK="https://staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}/"
-          export STATUS_DATA="{\"state\": \"success\", \
-                               \"description\": \"Preview the changed documentation here:\", \
-                               \"context\": \"Documentation Preview\", \
-                               \"target_url\": \"${PREVIEW_LINK}\"}"
-          curl -X POST --header "Accept: application/vnd.github+json" --header "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" --url "$GITHUB_API_URL" --header "content-type: application/json" --data "$STATUS_DATA"
-
-          #check if comment already exists, if not post it
-          export GITHUB_API_URL="https://api.github.com/repos/chatmail/relay/issues/${{ steps.prepare.outputs.prid }}/comments"
-          export RESPONSE=$(curl -L --header "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" --url "$GITHUB_API_URL" --header "content-type: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28")
-          echo $RESPONSE > response
-          grep -v '"Check out the page preview at https://staging.chatmail.at/doc/relay' response && echo "comment=true" >> $GITHUB_OUTPUT || true
-      - name: "Post link to comments"
-        if: steps.details.outputs.comment
-        uses: actions/github-script@v7
-        with:
-          script: |
-            github.rest.issues.createComment({
-              issue_number: context.issue.number,
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              body: "Check out the page preview at https://staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}/"
-            })
-
       - name: check links 
         working-directory: doc
         run: sphinx-build --builder linkcheck source build

.github/workflows/docs.yaml

@@ -14,6 +14,9 @@ jobs:
   scripts:
     name: build 
     runs-on: ubuntu-latest
+    environment: 
+      name: 'chatmail.at/doc/relay/'
+      url: https://chatmail.at/doc/relay/
     steps:
       - uses: actions/checkout@v4
 

.github/workflows/test-and-deploy-ipv4only.yaml

@@ -16,13 +16,11 @@ jobs:
     name: deploy on staging-ipv4.testrun.org, and run tests
     runs-on: ubuntu-latest
     timeout-minutes: 30
-    concurrency:
-      group: ci-ipv4-${{ github.workflow }}-${{ github.ref }}
-      cancel-in-progress: ${{ !contains(github.ref, '$GITHUB_REF') }}
+    environment:
+      name: staging-ipv4.testrun.org
+      url: https://staging-ipv4.testrun.org/
+    concurrency: staging-ipv4.testrun.org
     steps:
-      - uses: jsok/serialize-workflow-action@515cd04c46d7ea7435c4a22a3b4419127afdefe9
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions/checkout@v4
 
       - name: prepare SSH
@@ -90,7 +88,7 @@ jobs:
           ssh root@ns.testrun.org systemctl reload nsd
 
       - name: cmdeploy test
-        run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow
+        run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
 
       - name: cmdeploy dns
         run: cmdeploy dns -v

.github/workflows/test-and-deploy.yaml

@@ -16,13 +16,11 @@ jobs:
     name: deploy on staging2.testrun.org, and run tests
     runs-on: ubuntu-latest
     timeout-minutes: 30
-    concurrency:
-      group: ci-${{ github.workflow }}-${{ github.ref }}
-      cancel-in-progress: ${{ !contains(github.ref, '$GITHUB_REF') }}
+    environment:
+      name: staging2.testrun.org
+      url: https://staging2.testrun.org/
+    concurrency: staging2.testrun.org
     steps:
-      - uses: jsok/serialize-workflow-action@515cd04c46d7ea7435c4a22a3b4419127afdefe9
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions/checkout@v4
 
       - name: prepare SSH
@@ -70,6 +68,9 @@ jobs:
           rsync -avz dkimkeys-restore/dkimkeys root@staging2.testrun.org:/etc/ || true
           ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org chown root:root -R /var/lib/acme || true
 
+      - name: add hpk42 key to staging server
+        run: ssh root@staging2.testrun.org 'curl -s https://github.com/hpk42.keys >> .ssh/authorized_keys'
+
       - name: run deploy-chatmail offline tests 
         run: pytest --pyargs cmdeploy 
 
@@ -88,7 +89,7 @@ jobs:
           ssh root@ns.testrun.org systemctl reload nsd
 
       - name: cmdeploy test
-        run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow
+        run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
 
       - name: cmdeploy dns
         run: cmdeploy dns -v

CHANGELOG.md

@@ -1,6 +1,41 @@
 # Changelog for chatmail deployment 
 
-## untagged
+## 1.9.0 2025-12-18
+
+### Documentation
+
+- Add RELEASE.md and CONTRIBUTING.md
+- README update, mention Chatmail Cookbook project
+
+### Bug Fixes
+
+- Expire messages also from IMAP subfolders
+- Use absolute path instead of relative path in message expiration script
+- Restart Postfix and Dovecot automatically on failure
+- acmetool: Use a fixed name and `reconcile` instead of `want`
+
+### Features
+
+- Report DKIM error code in SMTP response
+- Remove development notice from the web pages
+
+### Miscellaneous Tasks
+
+- Update the heading in the CHANGELOG.md
+- Setup git-cliff
+- Run tests against ci-chatmail.testrun.org instead of nine.testrun.org
+- Cleanup remaining echobot code, remove echobot user from deployment and passthrough recipients
+
+## 1.8.0 2025-12-12
+
+- Add imap_compress option to chatmail.ini
+  ([#760](https://github.com/chatmail/relay/pull/760))
+
+- Remove echobot from relays 
+  ([#753](https://github.com/chatmail/relay/pull/753))
+
+- Fix `cmdeploy webdev`
+  ([#743](https://github.com/chatmail/relay/pull/743))
 
 - Add robots.txt to exclude all web crawlers
   ([#732](https://github.com/chatmail/relay/pull/732))

CONTRIBUTING.md

@@ -0,0 +1,7 @@
+# Contributing to the chatmail relay
+
+Commit messages follow the [Conventional Commits] notation.
+We use [git-cliff] to generate the changelog from commit messages before the release.
+
+[Conventional Commits]: https://www.conventionalcommits.org/
+[git-cliff]: https://git-cliff.org/

RELEASE.md

@@ -0,0 +1,15 @@
+# Releasing a new version of chatmail relay
+
+For example, to release version 1.9.0 of chatmail relay, do the following steps.
+
+1. Update the changelog: `git cliff --unreleased --tag 1.9.0 --prepend CHANGELOG.md` or `git cliff -u -t 1.9.0 -p CHANGELOG.md`.
+
+2. Open the changelog in the editor, edit it if required.
+
+3. Commit the changes to the changelog with a commit message `chore(release): prepare for 1.9.0`.
+
+3. Tag the release: `git tag --annotate 1.9.0`.
+
+4. Push the release tag: `git push origin 1.9.0`.
+
+5. Create a GitHub release: `gh release create 1.9.0`.

chatmaild/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "chatmaild"
-version = "0.2"
+version = "0.3"
 dependencies = [
   "aiosmtpd",
   "iniconfig",
@@ -25,7 +25,6 @@ where = ['src']
 doveauth = "chatmaild.doveauth:main"
 chatmail-metadata = "chatmaild.metadata:main"
 filtermail = "chatmaild.filtermail:main"
-echobot = "chatmaild.echo:main"
 chatmail-metrics = "chatmaild.metrics:main"
 chatmail-expire = "chatmaild.expire:main"
 chatmail-fsreport = "chatmaild.fsreport:main"
@@ -73,5 +72,6 @@ commands =
 deps = pytest 
        pdbpp 
        pytest-localserver
+       execnet
 commands = pytest -v -rsXx {posargs}
 """

chatmaild/src/chatmaild/config.py

@@ -4,8 +4,6 @@ import iniconfig
 
 from chatmaild.user import User
 
-echobot_password_path = Path("/run/echobot/password")
-
 
 def read_config(inipath):
     assert Path(inipath).exists(), inipath
@@ -46,6 +44,7 @@ class Config:
         self.disable_ipv6 = params.get("disable_ipv6", "false").lower() == "true"
         self.acme_email = params.get("acme_email", "")
         self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true"
+        self.imap_compress = params.get("imap_compress", "false").lower() == "true"
         if "iroh_relay" not in params:
             self.iroh_relay = "https://" + params["mail_domain"]
             self.enable_iroh_relay = True
@@ -72,10 +71,7 @@ class Config:
             raise ValueError(f"invalid address {addr!r}")
 
         maildir = self.mailboxes_dir.joinpath(addr)
-        if addr.startswith("echo@"):
-            password_path = echobot_password_path
-        else:
-            password_path = maildir.joinpath("password")
+        password_path = maildir.joinpath("password")
 
         return User(maildir, addr, password_path, uid="vmail", gid="vmail")
 

chatmaild/src/chatmaild/doveauth.py

@@ -40,10 +40,6 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
         return False
     localpart, domain = parts
 
-    if localpart == "echo":
-        # echobot account should not be created in the database
-        return False
-
     if (
         len(localpart) > config.username_max_length
         or len(localpart) < config.username_min_length

chatmaild/src/chatmaild/echo.py

@@ -1,109 +0,0 @@
-#!/usr/bin/env python3
-"""Advanced echo bot example.
-
-it will echo back any message that has non-empty text and also supports the /help command.
-"""
-
-import logging
-import os
-import subprocess
-import sys
-from pathlib import Path
-
-from deltachat_rpc_client import Bot, DeltaChat, EventType, Rpc, events
-
-from chatmaild.config import echobot_password_path, read_config
-from chatmaild.doveauth import encrypt_password
-from chatmaild.newemail import create_newemail_dict
-
-hooks = events.HookCollection()
-
-
-@hooks.on(events.RawEvent)
-def log_event(event):
-    if event.kind == EventType.INFO:
-        logging.info(event.msg)
-    elif event.kind == EventType.WARNING:
-        logging.warning(event.msg)
-
-
-@hooks.on(events.RawEvent(EventType.ERROR))
-def log_error(event):
-    logging.error("%s", event.msg)
-
-
-@hooks.on(events.MemberListChanged)
-def on_memberlist_changed(event):
-    logging.info(
-        "member %s was %s", event.member, "added" if event.member_added else "removed"
-    )
-
-
-@hooks.on(events.GroupImageChanged)
-def on_group_image_changed(event):
-    logging.info("group image %s", "deleted" if event.image_deleted else "changed")
-
-
-@hooks.on(events.GroupNameChanged)
-def on_group_name_changed(event):
-    logging.info(f"group name changed, old name: {event.old_name}")
-
-
-@hooks.on(events.NewMessage(func=lambda e: not e.command))
-def echo(event):
-    snapshot = event.message_snapshot
-    if snapshot.is_info:
-        # Ignore info messages
-        return
-    if snapshot.text or snapshot.file:
-        snapshot.chat.send_message(text=snapshot.text, file=snapshot.file)
-
-
-@hooks.on(events.NewMessage(command="/help"))
-def help_command(event):
-    snapshot = event.message_snapshot
-    snapshot.chat.send_text("Send me any message and I will echo it back")
-
-
-def main():
-    logging.basicConfig(level=logging.INFO)
-    path = os.environ.get("PATH")
-    venv_path = sys.argv[0].strip("echobot")
-    os.environ["PATH"] = path + ":" + venv_path
-    with Rpc() as rpc:
-        deltachat = DeltaChat(rpc)
-        system_info = deltachat.get_system_info()
-        logging.info(f"Running deltachat core {system_info.deltachat_core_version}")
-
-        accounts = deltachat.get_all_accounts()
-        account = accounts[0] if accounts else deltachat.add_account()
-
-        bot = Bot(account, hooks)
-
-        config = read_config(sys.argv[1])
-        addr = "echo@" + config.mail_domain
-
-        # Create password file
-        if bot.is_configured():
-            password = bot.account.get_config("mail_pw")
-        else:
-            password = create_newemail_dict(config)["password"]
-
-        echobot_password_path.write_text(encrypt_password(password))
-        # Give the user which doveauth runs as access to the password file.
-        subprocess.check_call(
-            ["/usr/bin/setfacl", "-m", "user:vmail:r", echobot_password_path],
-        )
-
-        if not bot.is_configured():
-            bot.configure(addr, password)
-
-        # write invite link to working directory
-        invitelink = bot.account.get_qr_code()
-        Path("invite-link.txt").write_text(invitelink)
-
-        bot.run_forever()
-
-
-if __name__ == "__main__":
-    main()

chatmaild/src/chatmaild/expire.py

@@ -14,7 +14,7 @@ from stat import S_ISREG
 
 from chatmaild.config import read_config
 
-FileEntry = namedtuple("FileEntry", ("relpath", "mtime", "size"))
+FileEntry = namedtuple("FileEntry", ("path", "mtime", "size"))
 
 
 def iter_mailboxes(basedir, maxnum):
@@ -51,33 +51,27 @@ class MailboxStat:
 
     def __init__(self, basedir):
         self.basedir = str(basedir)
-        # all detected messages in cur/new/tmp folders
         self.messages = []
-
-        # all detected files in mailbox top dir
         self.extrafiles = []
+        self.scandir(self.basedir)
 
-        # scan all relevant files (without recursion)
-        old_cwd = os.getcwd()
-        try:
-            os.chdir(self.basedir)
-        except FileNotFoundError:
-            return
-        for name in os_listdir_if_exists("."):
+    def scandir(self, folderdir):
+        for name in os_listdir_if_exists(folderdir):
+            path = f"{folderdir}/{name}"
             if name in ("cur", "new", "tmp"):
-                for msg_name in os_listdir_if_exists(name):
-                    entry = get_file_entry(f"{name}/{msg_name}")
+                for msg_name in os_listdir_if_exists(path):
+                    entry = get_file_entry(f"{path}/{msg_name}")
                     if entry is not None:
                         self.messages.append(entry)
-
+            elif os.path.isdir(path):
+                self.scandir(path)
             else:
-                entry = get_file_entry(name)
+                entry = get_file_entry(path)
                 if entry is not None:
                     self.extrafiles.append(entry)
                     if name == "password":
                         self.last_login = entry.mtime
         self.extrafiles.sort(key=lambda x: -x.size)
-        os.chdir(old_cwd)
 
 
 def print_info(msg):
@@ -130,13 +124,6 @@ class Expiry:
             self.remove_mailbox(mbox.basedir)
             return
 
-        # all to-be-removed files are relative to the mailbox basedir
-        try:
-            os.chdir(mbox.basedir)
-        except FileNotFoundError:
-            print_info(f"mailbox not found/vanished {mbox.basedir}")
-            return
-
         mboxname = os.path.basename(mbox.basedir)
         if self.verbose:
             date = datetime.fromtimestamp(mbox.last_login) if mbox.last_login else None
@@ -147,11 +134,12 @@ class Expiry:
         self.all_files += len(mbox.messages)
         for message in mbox.messages:
             if message.mtime < cutoff_mails:
-                self.remove_file(message.relpath, mtime=message.mtime)
+                self.remove_file(message.path, mtime=message.mtime)
             elif message.size > 200000 and message.mtime < cutoff_large_mails:
                 # we only remove noticed large files (not unnoticed ones in new/)
-                if message.relpath.startswith("cur/"):
-                    self.remove_file(message.relpath, mtime=message.mtime)
+                parts = message.path.split("/")
+                if len(parts) >= 2 and parts[-2] == "cur":
+                    self.remove_file(message.path, mtime=message.mtime)
             else:
                 continue
             changed = True

chatmaild/src/chatmaild/filtermail.py

@@ -307,12 +307,9 @@ class IncomingBeforeQueueHandler:
             return error
         log_info("re-injecting the mail that passed checks")
 
-        # the smtp daemon on reinject_port_incoming gives it to dkim milter
-        # which looks at source address to determine whether to verify or sign
         client = SMTPClient(
             "localhost",
             self.config.postfix_reinject_port_incoming,
-            source_address=("127.0.0.2", 0),
         )
         client.sendmail(
             envelope.mail_from, envelope.rcpt_tos, envelope.original_content

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -15,7 +15,7 @@ mail_domain = {mail_domain}
 max_user_send_per_minute = 60
 
 # maximum mailbox size of a chatmail address
-max_mailbox_size = 100M
+max_mailbox_size = 500M
 
 # maximum message size for an e-mail in bytes
 max_message_size = 31457280
@@ -43,9 +43,9 @@ passthrough_senders =
 
 # list of e-mail recipients for which to accept outbound un-encrypted mails
 # (space-separated, item may start with "@" to whitelist whole recipient domains)
-passthrough_recipients = echo@{mail_domain}
+passthrough_recipients =
 
-# path to www directory - documented here: https://github.com/chatmail/relay/#custom-web-pages
+# path to www directory - documented here: https://chatmail.at/doc/relay/getting_started.html#custom-web-pages
 #www_folder = www
 
 #
@@ -99,6 +99,12 @@ acme_email =
 # so use this option with caution on production servers. 
 imap_rawlog = false 
 
+# set to true if you want to enable the IMAP COMPRESS Extension,
+# which allows IMAP connections to be efficiently compressed.
+# WARNING: Enabling this makes it impossible to hibernate IMAP
+# processes which will result in much higher memory/RAM usage.
+imap_compress = false
+
 
 #
 # Privacy Policy

chatmaild/src/chatmaild/lastlogin.py

@@ -13,8 +13,6 @@ class LastLoginDictProxy(DictProxy):
         keyname = parts[1].split("/")
         value = parts[2] if len(parts) > 2 else ""
         if keyname[0] == "shared" and keyname[1] == "last-login":
-            if addr.startswith("echo@"):
-                return True
             addr = keyname[2]
             timestamp = int(value)
             user = self.config.get_user(addr)

chatmaild/src/chatmaild/tests/test_config.py

@@ -33,7 +33,7 @@ def test_read_config_testrun(make_config):
     assert config.filtermail_smtp_port == 10080
     assert config.postfix_reinject_port == 10025
     assert config.max_user_send_per_minute == 60
-    assert config.max_mailbox_size == "100M"
+    assert config.max_mailbox_size == "500M"
     assert config.delete_mails_after == "20"
     assert config.delete_large_after == "7"
     assert config.username_min_length == 9

chatmaild/src/chatmaild/tests/test_expire.py

@@ -17,19 +17,17 @@ from chatmaild.expire import main as expiry_main
 from chatmaild.fsreport import main as report_main
 
 
-def fill_mbox(basedir):
-    basedir1 = basedir.joinpath("mailbox1@example.org")
-    basedir1.mkdir()
-    password = basedir1.joinpath("password")
+def fill_mbox(folderdir):
+    password = folderdir.joinpath("password")
     password.write_text("xxx")
-    basedir1.joinpath("maildirsize").write_text("xxx")
+    folderdir.joinpath("maildirsize").write_text("xxx")
 
-    garbagedir = basedir1.joinpath("garbagedir")
+    garbagedir = folderdir.joinpath("garbagedir")
     garbagedir.mkdir()
+    garbagedir.joinpath("bimbum").write_text("hello")
 
-    create_new_messages(basedir1, ["cur/msg1"], size=500)
-    create_new_messages(basedir1, ["new/msg2"], size=600)
-    return basedir1
+    create_new_messages(folderdir, ["cur/msg1"], size=500)
+    create_new_messages(folderdir, ["new/msg2"], size=600)
 
 
 def create_new_messages(basedir, relpaths, size=1000, days=0):
@@ -45,8 +43,21 @@ def create_new_messages(basedir, relpaths, size=1000, days=0):
 
 @pytest.fixture
 def mbox1(example_config):
-    basedir1 = fill_mbox(example_config.mailboxes_dir)
-    return MailboxStat(basedir1)
+    mboxdir = example_config.mailboxes_dir.joinpath("mailbox1@example.org")
+    mboxdir.mkdir()
+    fill_mbox(mboxdir)
+    return MailboxStat(mboxdir)
+
+
+def test_deltachat_folder(example_config):
+    """Test old setups that might have a .DeltaChat folder where messages also need to get removed."""
+    mboxdir = example_config.mailboxes_dir.joinpath("mailbox1@example.org")
+    mboxdir.mkdir()
+    mbox2dir = mboxdir.joinpath(".DeltaChat")
+    mbox2dir.mkdir()
+    fill_mbox(mbox2dir)
+    mb = MailboxStat(mboxdir)
+    assert len(mb.messages) == 2
 
 
 def test_filentry_ordering(tmp_path):
@@ -76,7 +87,7 @@ def test_stats_mailbox(mbox1):
     create_new_messages(mbox1.basedir, ["large-extra"], size=1000)
     create_new_messages(mbox1.basedir, ["index-something"], size=3)
     mbox2 = MailboxStat(mbox1.basedir)
-    assert len(mbox2.extrafiles) == 4
+    assert len(mbox2.extrafiles) == 5
     assert mbox2.extrafiles[0].size == 1000
 
     # cope well with mailbox dirs that have no password (for whatever reason)

chatmaild/src/chatmaild/tests/test_lastlogin.py

@@ -36,29 +36,3 @@ def test_handle_dovecot_request_last_login(testaddr, example_config):
     res = dictproxy.handle_dovecot_request(msg, dictproxy_transactions)
     assert res == "O\n"
     assert len(dictproxy_transactions) == 0
-
-
-def test_handle_dovecot_request_last_login_echobot(example_config):
-    dictproxy = LastLoginDictProxy(config=example_config)
-
-    authproxy = AuthDictProxy(config=example_config)
-    testaddr = f"echo@{example_config.mail_domain}"
-    authproxy.lookup_passdb(testaddr, "ignore")
-    user = dictproxy.config.get_user(testaddr)
-
-    transactions = {}
-
-    # set last-login info for user
-    tx = "1111"
-    msg = f"B{tx}\t{testaddr}"
-    res = dictproxy.handle_dovecot_request(msg, transactions)
-    assert not res
-    assert transactions == {tx: dict(addr=testaddr, res="O\n")}
-
-    timestamp = int(time.time())
-    msg = f"S{tx}\tshared/last-login/{testaddr}\t{timestamp}"
-    res = dictproxy.handle_dovecot_request(msg, transactions)
-    assert not res
-    assert len(transactions) == 1
-    read_timestamp = user.get_last_login_timestamp()
-    assert read_timestamp is None

chatmaild/src/chatmaild/user.py

@@ -19,7 +19,7 @@ class User:
 
     @property
     def can_track(self):
-        return "@" in self.addr and not self.addr.startswith("echo@")
+        return "@" in self.addr
 
     def get_userdb_dict(self):
         """Return a non-empty dovecot 'userdb' style dict
@@ -55,11 +55,9 @@ class User:
         try:
             write_bytes_atomic(self.password_path, password)
         except PermissionError:
-            if not self.addr.startswith("echo@"):
-                logging.error(f"could not write password for: {self.addr}")
-                raise
-        if not self.addr.startswith("echo@"):
-            self.enforce_E2EE_path.touch()
+            logging.error(f"could not write password for: {self.addr}")
+            raise
+        self.enforce_E2EE_path.touch()
 
     def set_last_login_timestamp(self, timestamp):
         """Track login time with daily granularity

cliff.toml

@@ -0,0 +1,94 @@
+# git-cliff ~ configuration file
+# https://git-cliff.org/docs/configuration
+
+
+[changelog]
+# A Tera template to be rendered for each release in the changelog.
+# See https://keats.github.io/tera/docs/#introduction
+body = """
+{% if version %}\
+    ## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }}
+{% else %}\
+    ## [unreleased]
+{% endif %}\
+{% for group, commits in commits | group_by(attribute="group") %}
+    ### {{ group | striptags | trim | upper_first }}
+    {% for commit in commits %}
+        - {% if commit.scope %}*({{ commit.scope }})* {% endif %}\
+            {% if commit.breaking %}[**breaking**] {% endif %}\
+            {{ commit.message | upper_first }}\
+    {% endfor %}
+{% endfor %}
+"""
+# Remove leading and trailing whitespaces from the changelog's body.
+trim = true
+# Render body even when there are no releases to process.
+render_always = true
+# An array of regex based postprocessors to modify the changelog.
+postprocessors = [
+    # Replace the placeholder <REPO> with a URL.
+    #{ pattern = '<REPO>', replace = "https://github.com/orhun/git-cliff" },
+]
+# render body even when there are no releases to process
+# render_always = true
+# output file path
+# output = "test.md"
+
+[git]
+# Parse commits according to the conventional commits specification.
+# See https://www.conventionalcommits.org
+conventional_commits = true
+# Exclude commits that do not match the conventional commits specification.
+filter_unconventional = true
+# Require all commits to be conventional.
+# Takes precedence over filter_unconventional.
+require_conventional = false
+# Split commits on newlines, treating each line as an individual commit.
+split_commits = false
+# An array of regex based parsers to modify commit messages prior to further processing.
+commit_preprocessors = [
+    # Replace issue numbers with link templates to be updated in `changelog.postprocessors`.
+    #{ pattern = '\((\w+\s)?#([0-9]+)\)', replace = "([#${2}](<REPO>/issues/${2}))"},
+    # Check spelling of the commit message using https://github.com/crate-ci/typos.
+    # If the spelling is incorrect, it will be fixed automatically.
+    #{ pattern = '.*', replace_command = 'typos --write-changes -' },
+]
+# Prevent commits that are breaking from being excluded by commit parsers.
+protect_breaking_commits = false
+# An array of regex based parsers for extracting data from the commit message.
+# Assigns commits to groups.
+# Optionally sets the commit's scope and can decide to exclude commits from further processing.
+commit_parsers = [
+    { message = "^feat", group = "Features" },
+    { message = "^fix", group = "Bug Fixes" },
+    { message = "^docs", group = "Documentation" },
+    { message = "^perf", group = "Performance" },
+    { message = "^refactor", group = "Refactor" },
+    { message = "^style", group = "Styling" },
+    { message = "^test", group = "Testing" },
+    { message = "^chore\\(release\\): prepare for", skip = true },
+    { message = "^chore\\(deps.*\\)", skip = true },
+    { message = "^chore\\(pr\\)", skip = true },
+    { message = "^chore\\(pull\\)", skip = true },
+    { message = "^chore|^ci", group = "Miscellaneous Tasks" },
+    { body = ".*security", group = "Security" },
+    { message = "^revert", group = "Revert" },
+    { message = ".*", group = "Other" },
+]
+# Exclude commits that are not matched by any commit parser.
+filter_commits = false
+# Fail on a commit that is not matched by any commit parser.
+fail_on_unmatched_commit = false
+# An array of link parsers for extracting external references, and turning them into URLs, using regex.
+link_parsers = []
+# Include only the tags that belong to the current branch.
+use_branch_tags = false
+# Order releases topologically instead of chronologically.
+topo_order = false
+# Order commits topologically instead of chronologically.
+topo_order_commits = true
+# Order of commits in each group/release within the changelog.
+# Allowed values: newest, oldest
+sort_commits = "oldest"
+# Process submodules commits
+recurse_submodules = false

cmdeploy/src/cmdeploy/acmetool/__init__.py

@@ -61,6 +61,19 @@ class AcmetoolDeployer(Deployer):
             mode="644",
         )
 
+        server.shell(
+            name=f"Remove old acmetool desired files for {self.domains[0]}",
+            commands=[f"rm -f /var/lib/acme/desired/{self.domains[0]}-*"],
+        )
+        files.template(
+            src=importlib.resources.files(__package__).joinpath("desired.yaml.j2"),
+            dest=f"/var/lib/acme/desired/{self.domains[0]}", # 0 is mailhost TLD
+            user="root",
+            group="root",
+            mode="644",
+            domains=self.domains,
+        )
+
         service_file = files.put(
             src=importlib.resources.files(__package__).joinpath(
                 "acmetool-redirector.service"
@@ -123,6 +136,6 @@ class AcmetoolDeployer(Deployer):
         self.need_restart_reconcile_timer = False
 
         server.shell(
-            name=f"Request certificate for: {', '.join(self.domains)}",
-            commands=[f"acmetool want --xlog.severity=debug {' '.join(self.domains)}"],
+            name=f"Reconcile certificates for: {', '.join(self.domains)}",
+            commands=["acmetool --batch --xlog.severity=debug reconcile"],
         )

cmdeploy/src/cmdeploy/acmetool/desired.yaml.j2

@@ -0,0 +1,6 @@
+satisfy:
+  names:
+{%- for domain in domains %}
+  - {{ domain }}
+{%- endfor %}
+

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -109,15 +109,6 @@ def run_cmd(args, out):
     try:
         retcode = out.check_call(cmd, env=env)
         if retcode == 0:
-            if not args.disable_mail:
-                print("\nYou can try out the relay by talking to this echo bot: ")
-                sshexec = SSHExec(args.config.mail_domain, verbose=args.verbose)
-                print(
-                    sshexec(
-                        call=remote.rshell.shell,
-                        kwargs=dict(command="cat /var/lib/echobot/invite-link.txt"),
-                    )
-                )
             out.green("Deploy completed, call `cmdeploy dns` next.")
         elif not remote_data["acme_account_url"]:
             out.red("Deploy completed but letsencrypt not configured")

cmdeploy/src/cmdeploy/deployers.py

@@ -270,6 +270,14 @@ class LegacyRemoveDeployer(Deployer):
             path="/var/log/journal/",
             present=False,
         )
+        # remove echobot if it is still running
+        if host.get_fact(SystemdEnabled).get("echobot.service"):
+            systemd.service(
+                name="Disable echobot.service",
+                service="echobot.service",
+                running=False,
+                enabled=False,
+            )
 
 
 def check_config(config):
@@ -404,30 +412,6 @@ class JournaldDeployer(Deployer):
         self.need_restart = False
 
 
-class EchobotDeployer(Deployer):
-    #
-    # This deployer depends on the dovecot and postfix deployers because
-    # it needs to base its decision of whether to restart the service on
-    # whether those two services were restarted.
-    #
-    def __init__(self, mail_domain):
-        self.mail_domain = mail_domain
-        self.units = ["echobot"]
-
-    def install(self):
-        apt.packages(
-            # required for setfacl for echobot
-            name="Install acl",
-            packages="acl",
-        )
-
-    def configure(self):
-        configure_remote_units(self.mail_domain, self.units)
-
-    def activate(self):
-        activate_remote_units(self.units)
-
-
 class ChatmailVenvDeployer(Deployer):
     def __init__(self, config):
         self.config = config
@@ -456,7 +440,6 @@ class ChatmailVenvDeployer(Deployer):
 class ChatmailDeployer(Deployer):
     required_users = [
         ("vmail", "vmail", None),
-        ("echobot", None, None),
         ("iroh", None, None),
     ]
 
@@ -590,7 +573,6 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
         PostfixDeployer(config, disable_mail),
         FcgiwrapDeployer(),
         NginxDeployer(config),
-        EchobotDeployer(mail_domain),
         MtailDeployer(config.mtail_address),
         GithashDeployer(),
     ]

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -13,6 +13,8 @@ from cmdeploy.basedeploy import (
 
 
 class DovecotDeployer(Deployer):
+    daemon_reload = False
+
     def __init__(self, config, disable_mail):
         self.config = config
         self.disable_mail = disable_mail
@@ -27,7 +29,7 @@ class DovecotDeployer(Deployer):
 
     def configure(self):
         configure_remote_units(self.config.mail_domain, self.units)
-        self.need_restart = _configure_dovecot(self.config)
+        self.need_restart, self.daemon_reload = _configure_dovecot(self.config)
 
     def activate(self):
         activate_remote_units(self.units)
@@ -42,6 +44,7 @@ class DovecotDeployer(Deployer):
             running=False if self.disable_mail else True,
             enabled=False if self.disable_mail else True,
             restarted=restart,
+            daemon_reload=self.daemon_reload,
         )
         self.need_restart = False
 
@@ -80,9 +83,10 @@ def _install_dovecot_package(package: str, arch: str):
     apt.deb(name=f"Install dovecot-{package}", src=deb_filename)
 
 
-def _configure_dovecot(config: Config, debug: bool = False) -> bool:
+def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
     """Configures Dovecot IMAP server."""
     need_restart = False
+    daemon_reload = False
 
     main_config = files.template(
         src=get_resource("dovecot/dovecot.conf.j2"),
@@ -134,4 +138,11 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
     )
     need_restart |= timezone_env.changed
 
-    return need_restart
+    restart_conf = files.put(
+        name="dovecot: restart automatically on failure",
+        src=get_resource("service/10_restart.conf"),
+        dest="/etc/systemd/system/dovecot.service.d/10_restart.conf",
+    )
+    daemon_reload |= restart_conf.changed
+
+    return need_restart, daemon_reload

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -113,7 +113,7 @@ mail_attribute_dict = proxy:/run/chatmail-metadata/metadata.socket:metadata
 # `imap_zlib` enables IMAP COMPRESS (RFC 4978).
 # <https://datatracker.ietf.org/doc/html/rfc4978.html>
 protocol imap {
-  mail_plugins = $mail_plugins imap_zlib imap_quota last_login
+  mail_plugins = $mail_plugins imap_quota last_login {% if config.imap_compress %}imap_zlib{% endif %}
   imap_metadata = yes
 }
 
@@ -252,3 +252,28 @@ protocol imap {
   rawlog_dir = %h 
 } 
 {% endif %}
+
+{% if not config.imap_compress %}
+# Hibernate IDLE users to save memory and CPU resources
+# NOTE: this will have no effect if imap_zlib plugin is used
+imap_hibernate_timeout = 30s
+service imap {
+  # Note that this change will allow any process running as
+  # $default_internal_user (dovecot) to access mails as any other user.
+  # This may be insecure in some installations, which is why this isn't
+  # done by default.
+  unix_listener imap-master {
+    user = $default_internal_user
+  }
+}
+# The following is the default already in v2.3.1+:
+service imap {
+  extra_groups = $default_internal_group
+}
+service imap-hibernate {
+  unix_listener imap-hibernate {
+    mode = 0660
+    group = $default_internal_group
+  }
+}
+{% endif %}

cmdeploy/src/cmdeploy/opendkim/final.lua

@@ -1,4 +1,5 @@
-if odkim.internal_ip(ctx) == 1 then
+mtaname = odkim.get_mtasymbol(ctx, "{daemon_name}")
+if mtaname == "ORIGINATING" then
 	-- Outgoing message will be signed,
 	-- no need to look for signatures.
 	return nil
@@ -10,6 +11,7 @@ if nsigs == nil then
 end
 
 local valid = false
+local error_msg = "No valid DKIM signature found."
 for i = 1, nsigs do
 	sig = odkim.get_sighandle(ctx, i - 1)
 	sigres = odkim.sig_result(sig)
@@ -21,6 +23,8 @@ for i = 1, nsigs do
 	-- means the message is acceptable.
 	if sigres == 0 then
 		valid = true
+    else
+        error_msg = "DKIM signature is invalid, error code " .. tostring(sigres) .. ", search https://github.com/trusteddomainproject/OpenDKIM/blob/master/libopendkim/dkim.h#L108"
 	end
 end
 
@@ -30,8 +34,17 @@ if valid then
 	for i = nsigs, 1, -1 do
 		odkim.del_header(ctx, "DKIM-Signature", i)
 	end
+
+	-- Delete first and presumably only occurence
+        odkim.del_header(ctx, "Authentication-Results", 0)  
 else
 	odkim.set_reply(ctx, "554", "5.7.1", "No valid DKIM signature found")
+	-- Delete in reverse order to avoid index shifting.
+	for i = nsigs, 1, -1 do
+		odkim.del_header(ctx, "DKIM-Signature", i)
+	end
+else
+	odkim.set_reply(ctx, "554", "5.7.1", error_msg)
 	odkim.set_result(ctx, SMFIS_REJECT)
 end
 

cmdeploy/src/cmdeploy/opendkim/opendkim.conf

@@ -65,3 +65,9 @@ PidFile			/run/opendkim/opendkim.pid
 # The trust anchor enables DNSSEC. In Debian, the trust anchor file is provided
 # by the package dns-root-data.
 TrustAnchorFile		/usr/share/dns/root.key
+
+# Sign messages when `-o milter_macro_daemon_name=ORIGINATING` is set.
+MTA ORIGINATING
+
+# No hosts are treated as internal, ORIGINATING daemon name should be set explicitly.
+InternalHosts -

cmdeploy/src/cmdeploy/postfix/deployer.py

@@ -5,6 +5,7 @@ from cmdeploy.basedeploy import Deployer, get_resource
 
 class PostfixDeployer(Deployer):
     required_users = [("postfix", None, ["opendkim"])]
+    daemon_reload = False
 
     def __init__(self, config, disable_mail):
         self.config = config
@@ -60,6 +61,13 @@ class PostfixDeployer(Deployer):
             mode="644",
         )
         need_restart |= login_map.changed
+
+        restart_conf = files.put(
+            name="postfix: restart automatically on failure",
+            src=get_resource("service/10_restart.conf"),
+            dest="/etc/systemd/system/dovecot.service.d/10_restart.conf",
+        )
+        self.daemon_reload = restart_conf.changed
         self.need_restart = need_restart
 
     def activate(self):
@@ -73,5 +81,6 @@ class PostfixDeployer(Deployer):
             running=False if self.disable_mail else True,
             enabled=False if self.disable_mail else True,
             restarted=restart,
+            daemon_reload=self.daemon_reload,
         )
         self.need_restart = False

cmdeploy/src/cmdeploy/postfix/master.cf.j2

@@ -31,7 +31,6 @@ submission inet n       -       y       -       5000    smtpd
   -o smtpd_sender_restrictions=$mua_sender_restrictions
   -o smtpd_recipient_restrictions=
   -o smtpd_relay_restrictions=permit_sasl_authenticated,reject
-  -o milter_macro_daemon_name=ORIGINATING
   -o smtpd_client_connection_count_limit=1000
   -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
 smtps     inet  n       -       y       -       5000    smtpd
@@ -49,7 +48,6 @@ smtps     inet  n       -       y       -       5000    smtpd
   -o smtpd_recipient_restrictions=
   -o smtpd_relay_restrictions=permit_sasl_authenticated,reject
   -o smtpd_client_connection_count_limit=1000
-  -o milter_macro_daemon_name=ORIGINATING
   -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
 #628       inet  n       -       y       -       -       qmqpd
 pickup    unix  n       -       y       60      1       pickup
@@ -81,6 +79,7 @@ filter    unix -        n       n       -       -       lmtp
 # Local SMTP server for reinjecting outgoing filtered mail.
 127.0.0.1:{{ config.postfix_reinject_port }} inet  n       -       n       -       100      smtpd
   -o syslog_name=postfix/reinject
+  -o milter_macro_daemon_name=ORIGINATING
   -o smtpd_milters=unix:opendkim/opendkim.sock
   -o cleanup_service_name=authclean
 

cmdeploy/src/cmdeploy/remote/rdns.py

@@ -37,7 +37,10 @@ def perform_initial_checks(mail_domain, pre_command=""):
         return res
 
     # parse out sts-id if exists, example: "v=STSv1; id=2090123"
-    parts = query_dns("TXT", f"_mta-sts.{mail_domain}").split("id=")
+    mta_sts_txt = query_dns("TXT", f"_mta-sts.{mail_domain}")
+    if not mta_sts_txt:
+        return res
+    parts = mta_sts_txt.split("id=")
     res["sts_id"] = parts[1].rstrip('"') if len(parts) == 2 else ""
     return res
 

cmdeploy/src/cmdeploy/service/10_restart.conf

@@ -0,0 +1,3 @@
+[Service]
+Restart=always
+RestartSec=30

cmdeploy/src/cmdeploy/service/echobot.service.f

@@ -1,67 +0,0 @@
-[Unit]
-Description=Chatmail echo bot for testing it works
-
-[Service]
-ExecStart={execpath} {config_path}
-Environment="PATH={remote_venv_dir}:$PATH"
-Restart=always
-RestartSec=30
-
-User=echobot
-Group=echobot
-
-# Create /var/lib/echobot
-StateDirectory=echobot
-
-# Create /run/echobot
-#
-# echobot stores /run/echobot/password
-# with a password there, which doveauth then reads.
-RuntimeDirectory=echobot
-
-WorkingDirectory=/var/lib/echobot
-
-# Apply security restrictions suggested by
-#   systemd-analyze security echobot.service
-CapabilityBoundingSet=
-LockPersonality=true
-MemoryDenyWriteExecute=true
-NoNewPrivileges=true
-PrivateDevices=true
-PrivateMounts=true
-PrivateTmp=true
-
-# We need to know about doveauth user to give it access to /run/echobot/password
-PrivateUsers=false
-
-ProtectClock=true
-ProtectControlGroups=true
-ProtectHostname=true
-ProtectKernelLogs=true
-ProtectKernelModules=true
-ProtectKernelTunables=true
-ProtectProc=noaccess
-
-# Should be "strict", but we currently write /accounts folder in a protected path
-ProtectSystem=full
-
-RemoveIPC=true
-RestrictAddressFamilies=AF_INET AF_INET6
-RestrictNamespaces=true
-RestrictRealtime=true
-RestrictSUIDSGID=true
-SystemCallArchitectures=native
-SystemCallFilter=~@clock
-SystemCallFilter=~@cpu-emulation
-SystemCallFilter=~@debug
-SystemCallFilter=~@module
-SystemCallFilter=~@mount
-SystemCallFilter=~@obsolete
-SystemCallFilter=~@raw-io
-SystemCallFilter=~@reboot
-SystemCallFilter=~@resources
-SystemCallFilter=~@swap
-UMask=0077
-
-[Install]
-WantedBy=multi-user.target

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -1,5 +1,4 @@
 import datetime
-import os
 import smtplib
 import socket
 import subprocess
@@ -8,7 +7,6 @@ import time
 import pytest
 
 from cmdeploy import remote
-from cmdeploy.cmdeploy import main
 from cmdeploy.sshexec import SSHExec
 
 
@@ -70,47 +68,6 @@ class TestSSHExecutor:
         assert (now - since_date).total_seconds() < 60 * 60 * 51
 
 
-def test_status_cmd(chatmail_config, capsys, request):
-    os.chdir(request.config.invocation_params.dir)
-    assert main(["status"]) == 0
-    status_out = capsys.readouterr()
-    print(status_out.out)
-
-    services = [
-        "acmetool-redirector",
-        "chatmail-metadata",
-        "doveauth",
-        "dovecot",
-        "echobot",
-        "fcgiwrap",
-        "filtermail-incoming",
-        "filtermail",
-        "lastlogin",
-        "nginx",
-        "opendkim",
-        "postfix@-",
-        "systemd-journald",
-        "turnserver",
-        "unbound",
-    ]
-    not_running = []
-    for service in services:
-        active = False
-        for line in status_out:
-            if service in line:
-                active = True
-                if not "loaded" in line:
-                    active = False
-                if not "active" in line:
-                    active = False
-                if not "running" in line:
-                    active = False
-                break
-        if not active:
-            not_running.append(service)
-    assert not_running == []
-
-
 def test_timezone_env(remote):
     for line in remote.iter_output("env"):
         print(line)

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -160,22 +160,3 @@ def test_hide_senders_ip_address(cmfactory):
     user2.direct_imap.select_folder("Inbox")
     msg = user2.direct_imap.get_all_messages()[0]
     assert public_ip not in msg.obj.as_string()
-
-
-def test_echobot(cmfactory, chatmail_config, lp, sshdomain):
-    ac = cmfactory.get_online_accounts(1)[0]
-
-    # establish contact with echobot
-    sshexec = SSHExec(sshdomain)
-    command = "cat /var/lib/echobot/invite-link.txt"
-    echo_invite_link = sshexec(call=rshell.shell, kwargs=dict(command=command))
-    chat = ac.qr_setup_contact(echo_invite_link)
-    ac._evtracker.wait_securejoin_joiner_progress(1000)
-
-    # send message and check it gets replied back
-    lp.sec("Send message to echobot")
-    text = "hi, I hope you text me back"
-    chat.send_text(text)
-    lp.sec("Wait for reply from echobot")
-    reply = ac._evtracker.wait_next_incoming_message()
-    assert reply.text == text

cmdeploy/src/cmdeploy/tests/online/test_3_status.py

@@ -0,0 +1,49 @@
+import os
+
+from cmdeploy.cmdeploy import main
+
+
+def test_status_cmd(chatmail_config, capsys, request):
+    os.chdir(request.config.invocation_params.dir)
+    assert main(["status"]) == 0
+    status_out = capsys.readouterr()
+    print(status_out.out)
+
+    assert len(status_out.out.splitlines()) > 5
+
+    """
+    don't test actual server state:
+
+    services = [
+        "acmetool-redirector",
+        "chatmail-metadata",
+        "doveauth",
+        "dovecot",
+        "fcgiwrap",
+        "filtermail-incoming",
+        "filtermail",
+        "lastlogin",
+        "nginx",
+        "opendkim",
+        "postfix@-",
+        "systemd-journald",
+        "turnserver",
+        "unbound",
+    ]
+    not_running = []
+    for service in services:
+        active = False
+        for line in status_out:
+            if service in line:
+                active = True
+                if not "loaded" in line:
+                    active = False
+                if not "active" in line:
+                    active = False
+                if not "running" in line:
+                    active = False
+                break
+        if not active:
+            not_running.append(service)
+    assert not_running == []
+    """

cmdeploy/src/cmdeploy/www.py

@@ -166,7 +166,7 @@ def main():
             build_webpages(src_path, build_dir, config)
             print(f"[{changenum}] regenerated web pages at: {index_path}")
             print(f"URL: file://{index_path.resolve()}\n\n")
-            
+
             time.sleep(debounce_time)  # simple debounce
 
 

doc/README.md

@@ -6,7 +6,7 @@ You can use the `make` command and `make html` to build web pages.
 
 You need a Python environment where the following install was excuted: 
 
-    pip install sphinx-build furo sphinx-autobuild 
+    pip install furo sphinx-autobuild
 
 To develop/change documentation, you can then do: 
 

doc/source/getting_started.rst

@@ -16,15 +16,16 @@ You will need the following:
 
 -  Control over a domain through a DNS provider of your choice.
 
--  A Debian 12 server with reachable SMTP/SUBMISSIONS/IMAPS/HTTPS ports.
+-  A Debian 12 **deployment server** with reachable SMTP/SUBMISSIONS/IMAPS/HTTPS ports.
    IPv6 is encouraged if available. Chatmail relay servers only require
    1GB RAM, one CPU, and perhaps 10GB storage for a few thousand active
    chatmail addresses.
 
--  Key-based SSH authentication to the root user. You must add a
-   passphrase-protected private key to your local ssh-agent because you
-   can’t type in your passphrase during deployment. (An ed25519 private
-   key is required due to an `upstream bug in
+-  A Linux or Unix **build machine** with key-based SSH access to the root
+   user of the deployment server.
+   You must add a passphrase-protected private key to your local ssh-agent because you
+   can’t type in your passphrase during deployment.
+   (An ed25519 private key is required due to an `upstream bug in
    paramiko <https://github.com/paramiko/paramiko/issues/2191>`_)
 
 
@@ -34,16 +35,17 @@ Setup with ``scripts/cmdeploy``
 We use ``chat.example.org`` as the chatmail domain in the following
 steps. Please substitute it with your own domain.
 
-1. Setup the initial DNS records. The following is an example in the
+1. Setup the initial DNS records for your deployment server.
+   The following is an example in the
    familiar BIND zone file format with a TTL of 1 hour (3600 seconds).
    Please substitute your domain and IP addresses.
 
    ::
 
-       chat.example.com. 3600 IN A 198.51.100.5
-       chat.example.com. 3600 IN AAAA 2001:db8::5
-       www.chat.example.com. 3600 IN CNAME chat.example.com.
-       mta-sts.chat.example.com. 3600 IN CNAME chat.example.com.
+       chat.example.org. 3600 IN A 198.51.100.5
+       chat.example.org. 3600 IN AAAA 2001:db8::5
+       www.chat.example.org. 3600 IN CNAME chat.example.org.
+       mta-sts.chat.example.org. 3600 IN CNAME chat.example.org.
 
 2. On your local PC, clone the repository and bootstrap the Python
    virtualenv.
@@ -54,20 +56,20 @@ steps. Please substitute it with your own domain.
        cd relay
        scripts/initenv.sh
 
-3. On your local PC, create chatmail configuration file
+3. On your local build machine (PC), create a chatmail configuration file
    ``chatmail.ini``:
 
    ::
 
        scripts/cmdeploy init chat.example.org  # <-- use your domain
 
-4. Verify that SSH root login to your remote server works:
+4. Verify that SSH root login to the deployment server server works:
 
    ::
 
        ssh root@chat.example.org  # <-- use your domain
 
-5. From your local PC, deploy the remote chatmail relay server:
+5. From your local build machine, setup and configure the remote deployment server:
 
    ::
 
@@ -81,7 +83,7 @@ steps. Please substitute it with your own domain.
 Other helpful commands
 ----------------------
 
-To check the status of your remotely running chatmail service:
+To check the status of your deployment server running the chatmail service:
 
 ::
 
@@ -158,7 +160,7 @@ Disable automatic address creation
 --------------------------------------------------------
 
 If you need to stop address creation, e.g. because some script is wildly
-creating addresses, login with ssh and run:
+creating addresses, login with ssh to the deployment machine and run:
 
 ::
 
@@ -167,3 +169,23 @@ creating addresses, login with ssh and run:
 Chatmail address creation will be denied while this file is present.
 
 
+Migrating to a new build machine
+----------------------------------
+
+To move or add a build machine,
+clone the relay repository on the new build machine, and copy the ``chatmail.ini`` file from the old build machine.
+Make sure ``rsync`` is installed, then initialize the environment:
+
+::
+
+   ./scripts/initenv.sh
+
+Run safety checks before a new deployment:
+
+::
+
+   ./scripts/cmdeploy dns
+   ./scripts/cmdeploy status
+
+If you keep multiple build machines (ie laptop and desktop), keep ``chatmail.ini`` in sync between
+them.

doc/source/migrate.rst

@@ -1,73 +1,98 @@
 
-Migrating to a new host
------------------------
+Migrating to a new machine
+===========================
 
-If you want to migrate chatmail relay from an old machine to a new
-machine, you can use these steps. They were tested with a Linux laptop;
-you might need to adjust some of the steps to your environment.
+This migration tutorial provides a step-wise approach
+to safely migrate a chatmail relay from one remote machine to another.
 
-Let’s assume that your ``mail_domain`` is ``mail.example.org``, all
-involved machines run Debian 12, your old site’s IP address is
-``13.37.13.37``, and your new site’s IP address is ``13.12.23.42``.
+Preliminary notes and assumptions
+---------------------------------
 
-Note, you should lower the TTLs of your DNS records to a value such as
-300 (5 minutes) so the migration happens as smoothly as possible.
+- If the migration is a planned move,
+  it's recommended to lower the Time To Live (TTL) of your DNS records to a value such as 300 (5 minutes),
+  at best much earlier than the actual planned migration.
+  This speeds up propagation of DNS changes in the Internet after the migration is complete.
 
-During the guide you might get a warning about changed SSH Host keys; in
-this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
+- The migration steps were tested with a Linux laptop; you might need to adjust some of the steps to your local environment.
 
-1. First, disable mail services on the old site.
+- Your ``mail_domain`` is ``mail.example.org``.
+
+- All remote machines run Debian 12.
+
+- The old site’s IP version 4 address is ``$OLD_IP4``.
+
+- The new site’s IP addresses are ``$NEW_IP4`` and ``$NEW_IPV6``.
+
+
+The six steps to migrate
+------------------------
+
+Note that during some of the following steps you might get a warning about changed SSH Host keys;
+in this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
+
+
+1. **Initially transfer mailboxes from old to new site.**
+
+   Login to old site, forwarding your ssh-agent with ``ssh -A``
+   to allow using ssh to directly copy files from old to new site.
+   ::
+
+       ssh -A root@$OLD_IP4
+       tar c /home/vmail/mail | ssh root@$NEW_IP4 "tar x -C /"
+
+
+2. **Pre-configure the new site but keep it inactive until step 6**
+   ::
+
+       CMDEPLOY_STAGES=install,configure scripts/cmdeploy run --ssh-host $NEW_IP4
+
+
+3. **It's getting serious: disable mail services on the old site.**
+   Users will not be able to send or receive messages until all steps are completed.
+   Other relays and mail servers will retry delivering messages from time to time,
+   so nothing is lost for users.
 
    ::
 
-       cmdeploy run --disable-mail --ssh-host 13.37.13.37
+       scripts/cmdeploy run --disable-mail --ssh-host $OLD_IP4
 
-   Now your users will notice the migration and will not be able to send
-   or receive messages until the migration is completed.
 
-2. Now we want to copy ``/home/vmail``, ``/var/lib/acme``,
-   ``/etc/dkimkeys``, ``/run/echobot``, and ``/var/spool/postfix`` to
-   the new site. Login to the old site while forwarding your SSH agent
-   so you can copy directly from the old to the new site with your SSH
-   key:
-
-   ::
-
-       ssh -A root@13.37.13.37
-       tar c - /home/vmail/mail /var/lib/acme /etc/dkimkeys /run/echobot /var/spool/postfix | ssh root@13.12.23.42 "tar x -C /"
-
-   This transfers all addresses, the TLS certificate, DKIM keys (so DKIM
-   DNS record remains valid), and the echobot’s password so it continues
-   to function. It also preserves the Postfix mail spool so any messages
-   pending delivery will still be delivered.
-
-3. Install chatmail on the new machine:
-
-   ::
-
-       cmdeploy run --disable-mail --ssh-host 13.12.23.42
-
-   Postfix and Dovecot are disabled for now; we will enable them later.
-   We first need to make the new site fully operational.
-
-4. On the new site, run the following to ensure the ownership is correct
-   in case UIDs/GIDs changed:
+4. **Final synchronization of TLS/DKIM secrets, mail queues and mailboxes.**
+   Again we use ssh-agent forwarding (``-A``) to allow transfering all important data directly
+   from the old to the new site.
+   ::
+
+       ssh -A root@$OLD_IP4
+       tar c /var/lib/acme /etc/dkimkeys /var/spool/postfix | ssh root@$NEW_IP4 "tar x -C /"
+       rsync -azH /home/vmail/mail root@$NEW_IP4:/home/vmail/
+
+   Login to the new site and ensure file ownerships are correctly set:
 
    ::
 
+       ssh root@$NEW_IP4
        chown root: -R /var/lib/acme
        chown opendkim: -R /etc/dkimkeys
        chown vmail: -R /home/vmail/mail
-       chown echobot: -R /run/echobot
 
-5. Now, update DNS entries.
 
-   If other MTAs try to deliver messages to your chatmail domain they
-   may fail intermittently, as DNS catches up with the new site settings
-   but normally will retry delivering messages for at least a week, so
-   messages will not be lost.
+5. **Update the DNS entries to point to the new site.**
+   You only need to change the ``A`` and ``AAAA`` records, for example:
 
-6. Finally, you can execute ``cmdeploy run --ssh-host 13.12.23.42`` to
-   turn on chatmail on the new relay. Your users will be able to use the
-   chatmail relay as soon as the DNS changes have propagated. Voilà!
+   ::
+
+       mail.example.org.    IN A    $NEW_IP4
+       mail.example.org.    IN AAAA $NEW_IP6
+
+
+6. **Activate chatmail relay on new site.**
+
+   ::
+
+        CMDEPLOY_STAGES=activate scripts/cmdeploy run --ssh-host $NEW_IP4
+
+   Voilà!
+   Users will be able to use the relay as soon as the DNS changes have propagated.
+   If you have lowered the Time-to-Live for DNS records in step 1,
+   better use a higher value again (between 14400 and 86400 seconds) once you are sure everything works.
 

doc/source/overview.rst

@@ -109,10 +109,6 @@ short overview of ``chatmaild`` services:
    is contacted by Dovecot when a user logs in and stores the date of
    the login.
 
--  `echobot <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/echo.py>`_
-   is a small bot for test purposes. It simply echoes back messages from
-   users.
-
 -  `metrics <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/metrics.py>`_
    collects some metrics and displays them at
    ``https://example.org/metrics``.

doc/source/related.rst

@@ -7,14 +7,21 @@ Active development takes place in the `chatmail/relay github repository <https:/
 You can check out the `'chatmail' tag in the support.delta.chat forum <https://support.delta.chat/tag/chatmail>`_
 and ask to get added to a non-public support chat for debugging issues.
 
-We know of two work-in-progress alternative implementation efforts:
+We know of three work-in-progress alternative implementation efforts:
 
 -  `Mox <https://github.com/mjl-/mox>`_: A Golang email server. `Work
    is in progress <https://github.com/mjl-/mox/issues/251>`_ to modify
    it to support all of the features and configuration settings required
    to operate as a chatmail relay.
 
--  `Maddy-Chatmail <https://github.com/sadraiiali/maddy_chatmail>`_: a
-   plugin for the `Maddy email server <https://maddy.email/>`_ which
-   aims to implement the chatmail relay features and configuration
-   options.
+-  `Madmail <https://github.com/omidz4t/madmail>`_: an
+   experimental fork of Maddy Mail Server <https://maddy.email/>`_ optimized
+   for chatmail deployments.  It provides a single binary solution
+   for running a chatmail relay.
+
+-  `Chatmail Cookbook <https://github.com/feld/chatmail-cookbook>`_:
+   A Chef Cookbook implementing a relay server. The project follows the
+   official relay server software and configurations converted to a Chef
+   Cookbook with only minor differences. The cookbook uses DNS-01 for
+   certificate validation and additionally supports FreeBSD. It does not
+   require a Chef server to use.

www/src/index.md

@@ -23,7 +23,3 @@ you can also **scan this QR code** with Delta Chat:
 🐣 **Choose** your Avatar and Name
 
 💬 **Start** chatting with any Delta Chat contacts using [QR invite codes](https://delta.chat/en/help#howtoe2ee)
-
-{% if config.mail_domain != "nine.testrun.org" %}
-<div class="experimental">Note: this is only a temporary development chatmail service</div>
-{% endif %}