fix tar commands

.github/workflows/test-and-deploy-ipv4only.yaml

@@ -19,8 +19,13 @@ jobs:
     environment:
       name: staging-ipv4.testrun.org
       url: https://staging-ipv4.testrun.org/
-    concurrency: staging-ipv4.testrun.org
+    concurrency:
+      group: ci-ipv4-${{ github.workflow }}-${{ github.ref }}
+      cancel-in-progress: ${{ !contains(github.ref, '$GITHUB_REF') }}
     steps:
+      - uses: jsok/serialize-workflow-action@515cd04c46d7ea7435c4a22a3b4419127afdefe9
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions/checkout@v4
 
       - name: prepare SSH
@@ -74,7 +79,6 @@ jobs:
       - run: |
           cmdeploy init staging-ipv4.testrun.org
           sed -i 's#disable_ipv6 = False#disable_ipv6 = True#' chatmail.ini
-          sed -i 's/#\s*mtail_address/mtail_address/' chatmail.ini
 
       - run: cmdeploy run --verbose --skip-dns-check
 
@@ -89,7 +93,7 @@ jobs:
           ssh root@ns.testrun.org systemctl reload nsd
 
       - name: cmdeploy test
-        run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
+        run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow
 
       - name: cmdeploy dns
         run: cmdeploy dns -v

.github/workflows/test-and-deploy.yaml

@@ -19,8 +19,13 @@ jobs:
     environment:
       name: staging2.testrun.org
       url: https://staging2.testrun.org/
-    concurrency: staging2.testrun.org
+    concurrency:
+      group: ci-${{ github.workflow }}-${{ github.ref }}
+      cancel-in-progress: ${{ !contains(github.ref, '$GITHUB_REF') }}
     steps:
+      - uses: jsok/serialize-workflow-action@515cd04c46d7ea7435c4a22a3b4419127afdefe9
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions/checkout@v4
 
       - name: prepare SSH
@@ -74,9 +79,7 @@ jobs:
       - name: run deploy-chatmail offline tests 
         run: pytest --pyargs cmdeploy 
 
-      - run: |
-          cmdeploy init staging2.testrun.org
-          sed -i 's/#\s*mtail_address/mtail_address/' chatmail.ini
+      - run: cmdeploy init staging2.testrun.org
 
       - run: cmdeploy run --verbose --skip-dns-check
 
@@ -91,7 +94,7 @@ jobs:
           ssh root@ns.testrun.org systemctl reload nsd
 
       - name: cmdeploy test
-        run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
+        run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow
 
       - name: cmdeploy dns
         run: cmdeploy dns -v

CHANGELOG.md

@@ -1,31 +1,5 @@
 # Changelog for chatmail deployment 
 
-## 1.9.0 2025-12-18
-
-### Documentation
-
-- Add RELEASE.md and CONTRIBUTING.md
-- README update, mention Chatmail Cookbook project
-
-### Bug Fixes
-
-- Expire messages also from IMAP subfolders
-- Use absolute path instead of relative path in message expiration script
-- Restart Postfix and Dovecot automatically on failure
-- acmetool: Use a fixed name and `reconcile` instead of `want`
-
-### Features
-
-- Report DKIM error code in SMTP response
-- Remove development notice from the web pages
-
-### Miscellaneous Tasks
-
-- Update the heading in the CHANGELOG.md
-- Setup git-cliff
-- Run tests against ci-chatmail.testrun.org instead of nine.testrun.org
-- Cleanup remaining echobot code, remove echobot user from deployment and passthrough recipients
-
 ## 1.8.0 2025-12-12
 
 - Add imap_compress option to chatmail.ini

chatmaild/src/chatmaild/dictproxy.py

@@ -22,7 +22,7 @@ class DictProxy:
                 wfile.flush()
 
     def handle_dovecot_request(self, msg, transactions):
-        # see https://doc.dovecot.org/2.3/developer_manual/design/dict_protocol/#dovecot-dict-protocol
+        # see https://doc.dovecot.org/developer_manual/design/dict_protocol/#dovecot-dict-protocol
         short_command = msg[0]
         parts = msg[1:].split("\t")
 

chatmaild/src/chatmaild/doveauth.py

@@ -16,7 +16,7 @@ NOCREATE_FILE = "/etc/chatmail-nocreate"
 
 
 def encrypt_password(password: str):
-    # https://doc.dovecot.org/2.3/configuration_manual/authentication/password_schemes/
+    # https://doc.dovecot.org/configuration_manual/authentication/password_schemes/
     passhash = crypt_r.crypt(password, crypt_r.METHOD_SHA512)
     return "{SHA512-CRYPT}" + passhash
 

chatmaild/src/chatmaild/expire.py

@@ -14,7 +14,7 @@ from stat import S_ISREG
 
 from chatmaild.config import read_config
 
-FileEntry = namedtuple("FileEntry", ("path", "mtime", "size"))
+FileEntry = namedtuple("FileEntry", ("relpath", "mtime", "size"))
 
 
 def iter_mailboxes(basedir, maxnum):
@@ -51,27 +51,33 @@ class MailboxStat:
 
     def __init__(self, basedir):
         self.basedir = str(basedir)
+        # all detected messages in cur/new/tmp folders
         self.messages = []
-        self.extrafiles = []
-        self.scandir(self.basedir)
 
-    def scandir(self, folderdir):
-        for name in os_listdir_if_exists(folderdir):
-            path = f"{folderdir}/{name}"
+        # all detected files in mailbox top dir
+        self.extrafiles = []
+
+        # scan all relevant files (without recursion)
+        old_cwd = os.getcwd()
+        try:
+            os.chdir(self.basedir)
+        except FileNotFoundError:
+            return
+        for name in os_listdir_if_exists("."):
             if name in ("cur", "new", "tmp"):
-                for msg_name in os_listdir_if_exists(path):
-                    entry = get_file_entry(f"{path}/{msg_name}")
+                for msg_name in os_listdir_if_exists(name):
+                    entry = get_file_entry(f"{name}/{msg_name}")
                     if entry is not None:
                         self.messages.append(entry)
-            elif os.path.isdir(path):
-                self.scandir(path)
+
             else:
-                entry = get_file_entry(path)
+                entry = get_file_entry(name)
                 if entry is not None:
                     self.extrafiles.append(entry)
                     if name == "password":
                         self.last_login = entry.mtime
         self.extrafiles.sort(key=lambda x: -x.size)
+        os.chdir(old_cwd)
 
 
 def print_info(msg):
@@ -124,6 +130,13 @@ class Expiry:
             self.remove_mailbox(mbox.basedir)
             return
 
+        # all to-be-removed files are relative to the mailbox basedir
+        try:
+            os.chdir(mbox.basedir)
+        except FileNotFoundError:
+            print_info(f"mailbox not found/vanished {mbox.basedir}")
+            return
+
         mboxname = os.path.basename(mbox.basedir)
         if self.verbose:
             date = datetime.fromtimestamp(mbox.last_login) if mbox.last_login else None
@@ -134,17 +147,16 @@ class Expiry:
         self.all_files += len(mbox.messages)
         for message in mbox.messages:
             if message.mtime < cutoff_mails:
-                self.remove_file(message.path, mtime=message.mtime)
+                self.remove_file(message.relpath, mtime=message.mtime)
             elif message.size > 200000 and message.mtime < cutoff_large_mails:
                 # we only remove noticed large files (not unnoticed ones in new/)
-                parts = message.path.split("/")
-                if len(parts) >= 2 and parts[-2] == "cur":
-                    self.remove_file(message.path, mtime=message.mtime)
+                if message.relpath.startswith("cur/"):
+                    self.remove_file(message.relpath, mtime=message.mtime)
             else:
                 continue
             changed = True
         if changed:
-            self.remove_file(f"{mbox.basedir}/maildirsize")
+            self.remove_file("maildirsize")
 
     def get_summary(self):
         return (

chatmaild/src/chatmaild/filtermail.py

@@ -307,9 +307,12 @@ class IncomingBeforeQueueHandler:
             return error
         log_info("re-injecting the mail that passed checks")
 
+        # the smtp daemon on reinject_port_incoming gives it to dkim milter
+        # which looks at source address to determine whether to verify or sign
         client = SMTPClient(
             "localhost",
             self.config.postfix_reinject_port_incoming,
+            source_address=("127.0.0.2", 0),
         )
         client.sendmail(
             envelope.mail_from, envelope.rcpt_tos, envelope.original_content

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -15,7 +15,7 @@ mail_domain = {mail_domain}
 max_user_send_per_minute = 60
 
 # maximum mailbox size of a chatmail address
-max_mailbox_size = 500M
+max_mailbox_size = 100M
 
 # maximum message size for an e-mail in bytes
 max_message_size = 31457280

chatmaild/src/chatmaild/tests/test_config.py

@@ -33,7 +33,7 @@ def test_read_config_testrun(make_config):
     assert config.filtermail_smtp_port == 10080
     assert config.postfix_reinject_port == 10025
     assert config.max_user_send_per_minute == 60
-    assert config.max_mailbox_size == "500M"
+    assert config.max_mailbox_size == "100M"
     assert config.delete_mails_after == "20"
     assert config.delete_large_after == "7"
     assert config.username_min_length == 9

chatmaild/src/chatmaild/tests/test_expire.py

@@ -17,17 +17,19 @@ from chatmaild.expire import main as expiry_main
 from chatmaild.fsreport import main as report_main
 
 
-def fill_mbox(folderdir):
-    password = folderdir.joinpath("password")
+def fill_mbox(basedir):
+    basedir1 = basedir.joinpath("mailbox1@example.org")
+    basedir1.mkdir()
+    password = basedir1.joinpath("password")
     password.write_text("xxx")
-    folderdir.joinpath("maildirsize").write_text("xxx")
+    basedir1.joinpath("maildirsize").write_text("xxx")
 
-    garbagedir = folderdir.joinpath("garbagedir")
+    garbagedir = basedir1.joinpath("garbagedir")
     garbagedir.mkdir()
-    garbagedir.joinpath("bimbum").write_text("hello")
 
-    create_new_messages(folderdir, ["cur/msg1"], size=500)
-    create_new_messages(folderdir, ["new/msg2"], size=600)
+    create_new_messages(basedir1, ["cur/msg1"], size=500)
+    create_new_messages(basedir1, ["new/msg2"], size=600)
+    return basedir1
 
 
 def create_new_messages(basedir, relpaths, size=1000, days=0):
@@ -43,21 +45,8 @@ def create_new_messages(basedir, relpaths, size=1000, days=0):
 
 @pytest.fixture
 def mbox1(example_config):
-    mboxdir = example_config.mailboxes_dir.joinpath("mailbox1@example.org")
-    mboxdir.mkdir()
-    fill_mbox(mboxdir)
-    return MailboxStat(mboxdir)
-
-
-def test_deltachat_folder(example_config):
-    """Test old setups that might have a .DeltaChat folder where messages also need to get removed."""
-    mboxdir = example_config.mailboxes_dir.joinpath("mailbox1@example.org")
-    mboxdir.mkdir()
-    mbox2dir = mboxdir.joinpath(".DeltaChat")
-    mbox2dir.mkdir()
-    fill_mbox(mbox2dir)
-    mb = MailboxStat(mboxdir)
-    assert len(mb.messages) == 2
+    basedir1 = fill_mbox(example_config.mailboxes_dir)
+    return MailboxStat(basedir1)
 
 
 def test_filentry_ordering(tmp_path):
@@ -87,7 +76,7 @@ def test_stats_mailbox(mbox1):
     create_new_messages(mbox1.basedir, ["large-extra"], size=1000)
     create_new_messages(mbox1.basedir, ["index-something"], size=3)
     mbox2 = MailboxStat(mbox1.basedir)
-    assert len(mbox2.extrafiles) == 5
+    assert len(mbox2.extrafiles) == 4
     assert mbox2.extrafiles[0].size == 1000
 
     # cope well with mailbox dirs that have no password (for whatever reason)

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -71,11 +71,6 @@ def run_cmd_options(parser):
         action="store_true",
         help="install/upgrade the server, but disable postfix & dovecot for now",
     )
-    parser.add_argument(
-        "--website-only",
-        action="store_true",
-        help="only update/deploy the website, skipping full server upgrade/deployment, useful when you only changed/updated the web pages and don't need to re-run a full server upgrade",
-    )
     parser.add_argument(
         "--skip-dns-check",
         dest="dns_check_disabled",
@@ -98,7 +93,6 @@ def run_cmd(args, out):
 
     env = os.environ.copy()
     env["CHATMAIL_INI"] = args.inipath
-    env["CHATMAIL_WEBSITE_ONLY"] = "True" if args.website_only else ""
     env["CHATMAIL_DISABLE_MAIL"] = "True" if args.disable_mail else ""
     env["CHATMAIL_REQUIRE_IROH"] = "True" if require_iroh else ""
     deploy_path = importlib.resources.files(__package__).joinpath("run.py").resolve()
@@ -114,12 +108,7 @@ def run_cmd(args, out):
 
     try:
         retcode = out.check_call(cmd, env=env)
-        if args.website_only:
-            if retcode == 0:
-                out.green("Website deployment completed.")
-            else:
-                out.red("Website deployment failed.")
-        elif retcode == 0:
+        if retcode == 0:
             out.green("Deploy completed, call `cmdeploy dns` next.")
         elif not remote_data["acme_account_url"]:
             out.red("Deploy completed but letsencrypt not configured")

cmdeploy/src/cmdeploy/deployers.py

@@ -502,28 +502,23 @@ class GithashDeployer(Deployer):
         except Exception:
             git_diff = ""
         files.put(
-            name="Upload chatmail relay git commit hash",
+            name="Upload chatmail relay git commiit hash",
             src=StringIO(git_hash + git_diff),
             dest="/etc/chatmail-version",
             mode="700",
         )
 
 
-def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -> None:
+def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
     """Deploy a chat-mail instance.
 
     :param config_path: path to chatmail.ini
     :param disable_mail: whether to disable postfix & dovecot
-    :param website_only: if True, only deploy the website
     """
     config = read_config(config_path)
     check_config(config)
     mail_domain = config.mail_domain
 
-    if website_only:
-        Deployment().perform_stages([WebsiteDeployer(config)])
-        return
-
     if host.get_fact(Port, port=53) != "unbound":
         files.line(
             name="Add 9.9.9.9 to resolv.conf",
@@ -541,8 +536,6 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
         (["master", "smtpd"], 587),
         (["imap-login", "dovecot"], 993),
         ("iroh-relay", 3340),
-        ("mtail", 3903),
-        ("dovecot-stats", 3904),
         ("nginx", 8443),
         (["master", "smtpd"], config.postfix_reinject_port),
         (["master", "smtpd"], config.postfix_reinject_port_incoming),

cmdeploy/src/cmdeploy/dovecot/auth.conf

@@ -4,7 +4,7 @@ iterate_prefix = userdb/
 
 default_pass_scheme = plain
 # %E escapes characters " (double quote), ' (single quote) and \ (backslash) with \ (backslash).
-# See <https://doc.dovecot.org/2.3/configuration_manual/config_file/config_variables/#modifiers>
+# See <https://doc.dovecot.org/configuration_manual/config_file/config_variables/#modifiers>
 # for documentation.
 #
 # We escape user-provided input and use double quote as a separator.

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -37,7 +37,9 @@ class DovecotDeployer(Deployer):
         restart = False if self.disable_mail else self.need_restart
 
         systemd.service(
-            name="Disable dovecot for now" if self.disable_mail else "Start and enable Dovecot",
+            name="disable dovecot for now"
+            if self.disable_mail
+            else "Start and enable Dovecot",
             service="dovecot.service",
             running=False if self.disable_mail else True,
             enabled=False if self.disable_mail else True,
@@ -114,7 +116,7 @@ def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
     )
     need_restart |= lua_push_notification_script.changed
 
-    # as per https://doc.dovecot.org/2.3/configuration_manual/os/
+    # as per https://doc.dovecot.org/configuration_manual/os/
     # it is recommended to set the following inotify limits
     for name in ("max_user_instances", "max_user_watches"):
         key = f"fs.inotify.{name}"
@@ -143,11 +145,4 @@ def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
     )
     daemon_reload |= restart_conf.changed
 
-    # Validate dovecot configuration before restart
-    if need_restart:
-        server.shell(
-            name="Validate dovecot configuration",
-            commands=["doveconf -n >/dev/null"],
-        )
-
     return need_restart, daemon_reload

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -26,7 +26,7 @@ default_client_limit = 20000
 # Increase number of logged in IMAP connections.
 # Each connection is handled by a separate `imap` process.
 # `imap` process should have `client_limit=1` as described in
-# <https://doc.dovecot.org/2.3/configuration_manual/service_configuration/#service-limits>
+# <https://doc.dovecot.org/configuration_manual/service_configuration/#service-limits>
 # so each logged in IMAP session will need its own `imap` process.
 #
 # If this limit is reached,
@@ -44,11 +44,11 @@ mail_server_comment = Chatmail server
 
 # `zlib` enables compressing messages stored in the maildir.
 # See
-# <https://doc.dovecot.org/2.3/configuration_manual/zlib_plugin/>
+# <https://doc.dovecot.org/configuration_manual/zlib_plugin/>
 # for documentation.
 #
 # quota plugin documentation:
-# <https://doc.dovecot.org/2.3/configuration_manual/quota_plugin/>
+# <https://doc.dovecot.org/configuration_manual/quota_plugin/>
 mail_plugins = zlib quota
 
 imap_capability = +XDELTAPUSH XCHATMAIL
@@ -125,13 +125,13 @@ plugin {
 
 protocol lmtp {
   # notify plugin is a dependency of push_notification plugin:
-  # <https://doc.dovecot.org/2.3/settings/plugin/notify-plugin/>
+  # <https://doc.dovecot.org/settings/plugin/notify-plugin/>
   #
   # push_notification plugin documentation:
-  # <https://doc.dovecot.org/2.3/configuration_manual/push_notification/>
+  # <https://doc.dovecot.org/configuration_manual/push_notification/>
   #
   # mail_lua and push_notification_lua are needed for Lua push notification handler.
-  # <https://doc.dovecot.org/2.3/configuration_manual/push_notification/#configuration>
+  # <https://doc.dovecot.org/configuration_manual/push_notification/#configuration>
   mail_plugins = $mail_plugins mail_lua notify push_notification push_notification_lua
 }
 
@@ -154,7 +154,7 @@ plugin {
 
 # push_notification configuration
 plugin {
-  # <https://doc.dovecot.org/2.3/configuration_manual/push_notification/#lua-lua>
+  # <https://doc.dovecot.org/configuration_manual/push_notification/#lua-lua>
   push_notification_driver = lua:file=/etc/dovecot/push_notification.lua
 }
 
@@ -168,8 +168,6 @@ service lmtp {
   }
 }
 
-lmtp_add_received_header = no
-
 service auth {
   unix_listener /var/spool/postfix/private/auth {
     mode = 0660
@@ -279,156 +277,3 @@ service imap-hibernate {
   }
 }
 {% endif %}
-
-{% if config.mtail_address %}
-#
-# Dovecot Statistics
-#
-# OpenMetrics endpoint at http://{{- config.mtail_address}}:3904/metrics
-service stats {
-  inet_listener http {
-    port = 3904
-    address = {{- config.mtail_address}}
-  }
-}
-
-# IMAP Command Metrics
-# - Bytes in/out for compression efficiency analysis
-# - Lock wait time for contention debugging
-# - Grouped by command name and reply state
-metric imap_command {
-  filter = event=imap_command_finished
-  fields = bytes_in bytes_out lock_wait_usecs running_usecs
-  group_by = cmd_name tagged_reply_state
-}
-
-# Duration buckets for latency histograms (base 10: 10us, 100us, 1ms, 10ms, 100ms, 1s, 10s, 100s)
-metric imap_command_duration {
-  filter = event=imap_command_finished
-  group_by = cmd_name duration:exponential:1:8:10
-}
-
-# Slow command outliers (>1 second = 1000000 usecs)
-# Useful for alerting without high cardinality
-metric imap_command_slow {
-  filter = event=imap_command_finished AND duration>1000000 AND NOT cmd_name=IDLE
-  group_by = cmd_name
-}
-
-# IDLE-specific Metrics
-
-metric imap_idle {
-  filter = event=imap_command_finished AND cmd_name=IDLE
-  fields = bytes_in bytes_out running_usecs
-  group_by = tagged_reply_state
-}
-
-metric imap_idle_duration {
-  filter = event=imap_command_finished AND cmd_name=IDLE
-  # Base 10: 100ms to 27h (covers short wakeups to long idle sessions)
-  group_by = duration:exponential:5:11:10
-}
-
-metric imap_idle_commands {
-  filter = event=imap_command_finished AND cmd_name=IDLE
-  group_by = tagged_reply_state
-}
-
-metric imap_idle_failed {
-  filter = event=imap_command_finished AND cmd_name=IDLE AND NOT tagged_reply_state=OK
-}
-
-# Hibernation Metrics (requires imap_hibernate_timeout)
-
-metric imap_hibernated {
-  filter = event=imap_client_hibernated
-}
-
-metric imap_hibernated_failed {
-  filter = event=imap_client_hibernated AND error=*
-}
-
-metric imap_unhibernated {
-  filter = event=imap_client_unhibernated
-  fields = hibernation_usecs
-}
-
-metric imap_unhibernated_reason {
-  filter = event=imap_client_unhibernated
-  group_by = reason
-  fields = hibernation_usecs
-}
-
-metric imap_unhibernated_reason_sleep {
-  filter = event=imap_client_unhibernated
-  group_by = reason hibernation_usecs:exponential:4:8:10
-}
-
-metric imap_unhibernated_failed {
-  filter = event=imap_client_unhibernated AND error=*
-}
-
-# Hibernation duration buckets (how long clients stayed hibernated)
-# Base 10: 100ms to 27h
-metric imap_hibernation_duration {
-  filter = event=imap_client_unhibernated
-  group_by = reason duration:exponential:5:11:10
-}
-
-# Authentication / Login Metrics
-
-metric auth_request {
-  filter = event=auth_request_finished
-  group_by = success
-}
-
-metric auth_request_duration {
-  filter = event=auth_request_finished
-  group_by = success duration:exponential:2:6:10
-}
-
-metric auth_failed {
-  filter = event=auth_request_finished AND success=no
-}
-
-# Passdb cache effectiveness
-metric auth_passdb {
-  filter = event=auth_passdb_request_finished
-  group_by = result cache
-}
-
-# Master login (post-auth userdb lookup)
-metric auth_master_login {
-  filter = event=auth_master_client_login_finished
-}
-
-metric auth_master_login_failed {
-  filter = event=auth_master_client_login_finished AND error=*
-}
-
-# Mail Delivery (LMTP) - affects IDLE wakeup latency
-
-metric mail_delivery {
-  filter = event=mail_delivery_finished
-}
-
-metric mail_delivery_duration {
-  filter = event=mail_delivery_finished
-  group_by = duration:exponential:3:7:10
-}
-
-metric mail_delivery_failed {
-  filter = event=mail_delivery_finished AND error=*
-}
-
-# Connection Events
-
-metric client_connected {
-  filter = event=client_connection_connected AND category="service:imap"
-}
-
-metric client_disconnected {
-  filter = event=client_connection_disconnected AND category="service:imap"
-  fields = bytes_in bytes_out
-}
-{% endif %}

cmdeploy/src/cmdeploy/mtail/mtail.service.j2

@@ -3,7 +3,7 @@ Description=mtail
 
 [Service]
 Type=simple
-ExecStart=/bin/sh -c "journalctl -f -o short-iso -n 0 | /usr/local/bin/mtail --address={{ address }} --port={{ port }} --progs /etc/mtail --logtostderr --logs /dev/stdin"
+ExecStart=/bin/sh -c "journalctl -f -o short-iso -n 0 | /usr/local/bin/mtail --address={{ address }} --port={{ port }} --progs /etc/mtail --logtostderr --logs -"
 Restart=on-failure
 
 [Install]

cmdeploy/src/cmdeploy/opendkim/final.lua

@@ -1,5 +1,4 @@
-mtaname = odkim.get_mtasymbol(ctx, "{daemon_name}")
-if mtaname == "ORIGINATING" then
+if odkim.internal_ip(ctx) == 1 then
 	-- Outgoing message will be signed,
 	-- no need to look for signatures.
 	return nil

cmdeploy/src/cmdeploy/opendkim/opendkim.conf

@@ -65,9 +65,3 @@ PidFile			/run/opendkim/opendkim.pid
 # The trust anchor enables DNSSEC. In Debian, the trust anchor file is provided
 # by the package dns-root-data.
 TrustAnchorFile		/usr/share/dns/root.key
-
-# Sign messages when `-o milter_macro_daemon_name=ORIGINATING` is set.
-MTA ORIGINATING
-
-# No hosts are treated as internal, ORIGINATING daemon name should be set explicitly.
-InternalHosts -

cmdeploy/src/cmdeploy/postfix/deployer.py

@@ -1,4 +1,4 @@
-from pyinfra.operations import apt, files, server, systemd
+from pyinfra.operations import apt, files, systemd
 
 from cmdeploy.basedeploy import Deployer, get_resource
 
@@ -52,15 +52,6 @@ class PostfixDeployer(Deployer):
         )
         need_restart |= header_cleanup.changed
 
-        lmtp_header_cleanup = files.put(
-            src=get_resource("postfix/lmtp_header_cleanup"),
-            dest="/etc/postfix/lmtp_header_cleanup",
-            user="root",
-            group="root",
-            mode="644",
-        )
-        need_restart |= lmtp_header_cleanup.changed
-
         # Login map that 1:1 maps email address to login.
         login_map = files.put(
             src=get_resource("postfix/login_map"),
@@ -74,17 +65,9 @@ class PostfixDeployer(Deployer):
         restart_conf = files.put(
             name="postfix: restart automatically on failure",
             src=get_resource("service/10_restart.conf"),
-            dest="/etc/systemd/system/postfix@.service.d/10_restart.conf",
+            dest="/etc/systemd/system/dovecot.service.d/10_restart.conf",
         )
         self.daemon_reload = restart_conf.changed
-
-        # Validate postfix configuration before restart
-        if need_restart:
-            server.shell(
-                name="Validate postfix configuration",
-                # Extract stderr and quit with error if non-zero
-                commands=["""bash -c 'w=$(postconf 2>&1 >/dev/null); [[ -z "$w" ]] || { echo "$w"; false; }'"""],
-            )
         self.need_restart = need_restart
 
     def activate(self):

cmdeploy/src/cmdeploy/postfix/lmtp_header_cleanup

@@ -1,2 +0,0 @@
-/^DKIM-Signature:/            IGNORE
-/^Authentication-Results:/            IGNORE

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -77,7 +77,6 @@ inet_protocols = all
 
 virtual_transport = lmtp:unix:private/dovecot-lmtp
 virtual_mailbox_domains = {{ config.mail_domain }}
-lmtp_header_checks = regexp:/etc/postfix/lmtp_header_cleanup
 
 mua_client_restrictions = permit_sasl_authenticated, reject
 mua_sender_restrictions = reject_sender_login_mismatch, permit_sasl_authenticated, reject

cmdeploy/src/cmdeploy/postfix/master.cf.j2

@@ -31,6 +31,7 @@ submission inet n       -       y       -       5000    smtpd
   -o smtpd_sender_restrictions=$mua_sender_restrictions
   -o smtpd_recipient_restrictions=
   -o smtpd_relay_restrictions=permit_sasl_authenticated,reject
+  -o milter_macro_daemon_name=ORIGINATING
   -o smtpd_client_connection_count_limit=1000
   -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
 smtps     inet  n       -       y       -       5000    smtpd
@@ -48,6 +49,7 @@ smtps     inet  n       -       y       -       5000    smtpd
   -o smtpd_recipient_restrictions=
   -o smtpd_relay_restrictions=permit_sasl_authenticated,reject
   -o smtpd_client_connection_count_limit=1000
+  -o milter_macro_daemon_name=ORIGINATING
   -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
 #628       inet  n       -       y       -       -       qmqpd
 pickup    unix  n       -       y       60      1       pickup
@@ -79,7 +81,6 @@ filter    unix -        n       n       -       -       lmtp
 # Local SMTP server for reinjecting outgoing filtered mail.
 127.0.0.1:{{ config.postfix_reinject_port }} inet  n       -       n       -       100      smtpd
   -o syslog_name=postfix/reinject
-  -o milter_macro_daemon_name=ORIGINATING
   -o smtpd_milters=unix:opendkim/opendkim.sock
   -o cleanup_service_name=authclean
 

cmdeploy/src/cmdeploy/run.py

@@ -14,9 +14,8 @@ def main():
         importlib.resources.files("cmdeploy").joinpath("../../../chatmail.ini"),
     )
     disable_mail = bool(os.environ.get("CHATMAIL_DISABLE_MAIL"))
-    website_only = bool(os.environ.get("CHATMAIL_WEBSITE_ONLY"))
 
-    deploy_chatmail(config_path, disable_mail, website_only)
+    deploy_chatmail(config_path, disable_mail)
 
 
 if pyinfra.is_cli:

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -17,7 +17,6 @@ def imap_mailbox(cmfactory):
     password = ac1.get_config("mail_pw")
     mailbox = imap_tools.MailBox(user.split("@")[1])
     mailbox.login(user, password)
-    mailbox.dc_ac = ac1
     return mailbox
 
 
@@ -122,28 +121,6 @@ class TestEndToEndDeltaChat:
         assert ch.id >= 10
         ac1._evtracker.wait_securejoin_inviter_progress(1000)
 
-    def test_dkim_header_stripped(self, cmfactory, maildomain2, lp, imap_mailbox):
-        """Test that if a DC address receives a message, it has no
-        DKIM-Signature and Authentication-Results headers."""
-        ac1 = cmfactory.new_online_configuring_account(cache=False)
-        cmfactory.switch_maildomain(maildomain2)
-        ac2 = cmfactory.new_online_configuring_account(cache=False)
-        cmfactory.bring_accounts_online()
-        chat = cmfactory.get_accepted_chat(ac1, imap_mailbox.dc_ac)
-        chat.send_text("message0")
-        chat2 = cmfactory.get_accepted_chat(ac2, imap_mailbox.dc_ac)
-        chat2.send_text("message1")
-
-        lp.sec("receive message with ac1...")
-        received = 0
-        while received < 2:
-            msgs = imap_mailbox.fetch()
-            for msg in msgs:
-                lp.sec(f"ac1 received msg from {msg.from_}")
-                received += 1
-                assert "authentication-results" not in msg.headers
-                assert "dkim-signature" not in msg.headers
-
     def test_read_receipts_between_instances(self, cmfactory, lp, maildomain2):
         ac1 = cmfactory.new_online_configuring_account(cache=False)
         cmfactory.switch_maildomain(maildomain2)

doc/source/getting_started.rst

@@ -16,16 +16,15 @@ You will need the following:
 
 -  Control over a domain through a DNS provider of your choice.
 
--  A Debian 12 **deployment server** with reachable SMTP/SUBMISSIONS/IMAPS/HTTPS ports.
+-  A Debian 12 server with reachable SMTP/SUBMISSIONS/IMAPS/HTTPS ports.
    IPv6 is encouraged if available. Chatmail relay servers only require
    1GB RAM, one CPU, and perhaps 10GB storage for a few thousand active
    chatmail addresses.
 
--  A Linux or Unix **build machine** with key-based SSH access to the root
-   user of the deployment server.
-   You must add a passphrase-protected private key to your local ssh-agent because you
-   can’t type in your passphrase during deployment.
-   (An ed25519 private key is required due to an `upstream bug in
+-  Key-based SSH authentication to the root user. You must add a
+   passphrase-protected private key to your local ssh-agent because you
+   can’t type in your passphrase during deployment. (An ed25519 private
+   key is required due to an `upstream bug in
    paramiko <https://github.com/paramiko/paramiko/issues/2191>`_)
 
 
@@ -35,17 +34,16 @@ Setup with ``scripts/cmdeploy``
 We use ``chat.example.org`` as the chatmail domain in the following
 steps. Please substitute it with your own domain.
 
-1. Setup the initial DNS records for your deployment server.
-   The following is an example in the
+1. Setup the initial DNS records. The following is an example in the
    familiar BIND zone file format with a TTL of 1 hour (3600 seconds).
    Please substitute your domain and IP addresses.
 
    ::
 
-       chat.example.org. 3600 IN A 198.51.100.5
-       chat.example.org. 3600 IN AAAA 2001:db8::5
-       www.chat.example.org. 3600 IN CNAME chat.example.org.
-       mta-sts.chat.example.org. 3600 IN CNAME chat.example.org.
+       chat.example.com. 3600 IN A 198.51.100.5
+       chat.example.com. 3600 IN AAAA 2001:db8::5
+       www.chat.example.com. 3600 IN CNAME chat.example.com.
+       mta-sts.chat.example.com. 3600 IN CNAME chat.example.com.
 
 2. On your local PC, clone the repository and bootstrap the Python
    virtualenv.
@@ -56,20 +54,20 @@ steps. Please substitute it with your own domain.
        cd relay
        scripts/initenv.sh
 
-3. On your local build machine (PC), create a chatmail configuration file
+3. On your local PC, create chatmail configuration file
    ``chatmail.ini``:
 
    ::
 
        scripts/cmdeploy init chat.example.org  # <-- use your domain
 
-4. Verify that SSH root login to the deployment server server works:
+4. Verify that SSH root login to your remote server works:
 
    ::
 
        ssh root@chat.example.org  # <-- use your domain
 
-5. From your local build machine, setup and configure the remote deployment server:
+5. From your local PC, deploy the remote chatmail relay server:
 
    ::
 
@@ -83,7 +81,7 @@ steps. Please substitute it with your own domain.
 Other helpful commands
 ----------------------
 
-To check the status of your deployment server running the chatmail service:
+To check the status of your remotely running chatmail service:
 
 ::
 
@@ -160,7 +158,7 @@ Disable automatic address creation
 --------------------------------------------------------
 
 If you need to stop address creation, e.g. because some script is wildly
-creating addresses, login with ssh to the deployment machine and run:
+creating addresses, login with ssh and run:
 
 ::
 
@@ -169,23 +167,3 @@ creating addresses, login with ssh to the deployment machine and run:
 Chatmail address creation will be denied while this file is present.
 
 
-Migrating to a new build machine
-----------------------------------
-
-To move or add a build machine,
-clone the relay repository on the new build machine, and copy the ``chatmail.ini`` file from the old build machine.
-Make sure ``rsync`` is installed, then initialize the environment:
-
-::
-
-   ./scripts/initenv.sh
-
-Run safety checks before a new deployment:
-
-::
-
-   ./scripts/cmdeploy dns
-   ./scripts/cmdeploy status
-
-If you keep multiple build machines (ie laptop and desktop), keep ``chatmail.ini`` in sync between
-them.

doc/source/migrate.rst

@@ -1,72 +1,77 @@
 
-Migrating to a new machine
-===========================
+Migrating to a new host
+-----------------------
 
-This migration tutorial provides a step-wise approach
-to safely migrate a chatmail relay from one remote machine to another.
+If you want to migrate chatmail relay from an old machine to a new
+machine, you can use these steps. They were tested with a Linux laptop;
+you might need to adjust some of the steps to your environment.
 
-Preliminary notes and assumptions
----------------------------------
+Let’s assume that your ``mail_domain`` is ``mail.example.org``, all
+involved machines run Debian 12, your old site’s IP version 4 address is
+``$OLD_IP4``, and your new site’s IP4 address is ``$NEW_IP4``.
 
-- If the migration is a planned move,
-  it's recommended to lower the Time To Live (TTL) of your DNS records to a value such as 300 (5 minutes),
-  at best much earlier than the actual planned migration.
-  This speeds up propagation of DNS changes in the Internet after the migration is complete.
+First of all, you should lower the Time To Live (TTL) of your DNS records
+to a value such as 300 (5 minutes).
+Short TTL values allow to change DNS records during the migration more timely.
 
-- The migration steps were tested with a Linux laptop; you might need to adjust some of the steps to your local environment.
+During the guide you might get a warning about changed SSH Host keys; in
+this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
 
-- Your ``mail_domain`` is ``mail.example.org``.
+1. First, to make the downtime during the migration shorter,
+   let's transfer the current state of the mailboxes.
+   Login to your old machine (while forwarding your ssh-agent with ``ssh -A``)
+   so you can copy directly from the old to the new site with your SSH
+   key:
 
-- All remote machines run Debian 12.
-
-- The old site’s IP version 4 address is ``$OLD_IP4``.
-
-- The new site’s IP addresses are ``$NEW_IP4`` and ``$NEW_IPV6``.
-
-
-The six steps to migrate
-------------------------
-
-Note that during some of the following steps you might get a warning about changed SSH Host keys;
-in this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
-
-
-1. **Initially transfer mailboxes from old to new site.**
-
-   Login to old site, forwarding your ssh-agent with ``ssh -A``
-   to allow using ssh to directly copy files from old to new site.
    ::
 
        ssh -A root@$OLD_IP4
        tar c /home/vmail/mail | ssh root@$NEW_IP4 "tar x -C /"
 
+   This saves us time during the downtime,
+   at least the mailboxes are there already.
+   They contain user passwords, encrypted push notification tokens,
+   messages which might not have been fetched by all devices of the user yet,
+   and dovecot indexes which track the state of the mailbox.
 
-2. **Pre-configure the new site but keep it inactive until step 6**
-   ::
-
-       CMDEPLOY_STAGES=install,configure scripts/cmdeploy run --ssh-host $NEW_IP4
-
-
-3. **It's getting serious: disable mail services on the old site.**
-   Users will not be able to send or receive messages until all steps are completed.
-   Other relays and mail servers will retry delivering messages from time to time,
-   so nothing is lost for users.
+2. Then, from your local machine, install chatmail on the new machine, but don't activate it yet:
 
    ::
 
-       scripts/cmdeploy run --disable-mail --ssh-host $OLD_IP4
+       CMDEPLOY_STAGES=install,configure cmdeploy run --ssh-host $NEW_IP4
 
+   The services are disabled for now; we will enable them later.
+   We first need to make the new site fully operational.
+
+3. Now it's getting serious: disable the mail services on the old site.
+
+   ::
+
+       cmdeploy run --disable-mail --ssh-host $OLD_IP4
+
+   Your users will start to notice the migration and will not be able to send
+   or receive messages until the migration is completed.
+   Other relays and mail servers will wait with delivering messages
+   until your relay is reachable again.
+
+4. Now we want to copy ``/home/vmail``, ``/var/lib/acme``,
+   ``/etc/dkimkeys``, and ``/var/spool/postfix`` to
+   the new site. Let's forward the SSH agent again to copy the files directly.
+   This time, we copy ``/home/vmail/mail`` with rsync to only copy the recent changes:
 
-4. **Final synchronization of TLS/DKIM secrets, mail queues and mailboxes.**
-   Again we use ssh-agent forwarding (``-A``) to allow transfering all important data directly
-   from the old to the new site.
    ::
 
        ssh -A root@$OLD_IP4
        tar c /var/lib/acme /etc/dkimkeys /var/spool/postfix | ssh root@$NEW_IP4 "tar x -C /"
        rsync -azH /home/vmail/mail root@$NEW_IP4:/home/vmail/
 
-   Login to the new site and ensure file ownerships are correctly set:
+   This transfers all messages which have not been fetched yet, the TLS certificate,
+   and DKIM keys (so DKIM DNS record remains valid).
+   It also preserves the Postfix mail spool so any messages
+   pending delivery will still be delivered.
+
+5. Now login to the new site and run the following to ensure the ownership is correct
+   in case UIDs/GIDs changed:
 
    ::
 
@@ -75,8 +80,7 @@ in this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
        chown opendkim: -R /etc/dkimkeys
        chown vmail: -R /home/vmail/mail
 
-
-5. **Update the DNS entries to point to the new site.**
+6. Now, update the DNS entries.
    You only need to change the ``A`` and ``AAAA`` records, for example:
 
    ::
@@ -84,15 +88,7 @@ in this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
        mail.example.org.    IN A    $NEW_IP4
        mail.example.org.    IN AAAA $NEW_IP6
 
-
-6. **Activate chatmail relay on new site.**
-
-   ::
-
-        CMDEPLOY_STAGES=activate scripts/cmdeploy run --ssh-host $NEW_IP4
-
-   Voilà!
-   Users will be able to use the relay as soon as the DNS changes have propagated.
-   If you have lowered the Time-to-Live for DNS records in step 1,
-   better use a higher value again (between 14400 and 86400 seconds) once you are sure everything works.
+7. Finally, you can execute ``CMDEPLOY_STAGES=activate cmdeploy run --ssh-host $NEW_IP4`` to
+   turn on chatmail on the new relay. Your users will be able to use the
+   chatmail relay as soon as the DNS changes have propagated. Voilà!
 

doc/source/related.rst

@@ -14,10 +14,10 @@ We know of three work-in-progress alternative implementation efforts:
    it to support all of the features and configuration settings required
    to operate as a chatmail relay.
 
--  `Madmail <https://github.com/omidz4t/madmail>`_: an
-   experimental fork of Maddy Mail Server <https://maddy.email/>`_ optimized
-   for chatmail deployments.  It provides a single binary solution
-   for running a chatmail relay.
+-  `Maddy-Chatmail <https://github.com/sadraiiali/maddy_chatmail>`_: a
+   plugin for the `Maddy email server <https://maddy.email/>`_ which
+   aims to implement the chatmail relay features and configuration
+   options.
 
 -  `Chatmail Cookbook <https://github.com/feld/chatmail-cookbook>`_:
    A Chef Cookbook implementing a relay server. The project follows the