doc: add diagram for receiving from external SMTP servers

Co-authored-by: missytake <missytake@systemli.org>

cmdeploy/src/cmdeploy/acmetool/__init__.py

@@ -2,7 +2,7 @@ import importlib.resources
 
 from pyinfra.operations import apt, files, server, systemd
 
-from ..deployer import Deployer
+from ..basedeploy import Deployer
 
 
 class AcmetoolDeployer(Deployer):
@@ -27,7 +27,9 @@ class AcmetoolDeployer(Deployer):
 
         files.put(
             name="Install acmetool hook.",
-            src=importlib.resources.files(__package__).joinpath("acmetool.hook").open("rb"),
+            src=importlib.resources.files(__package__)
+            .joinpath("acmetool.hook")
+            .open("rb"),
             dest="/etc/acme/hooks/nginx",
             user="root",
             group="root",
@@ -41,7 +43,9 @@ class AcmetoolDeployer(Deployer):
 
     def configure(self):
         files.template(
-            src=importlib.resources.files(__package__).joinpath("response-file.yaml.j2"),
+            src=importlib.resources.files(__package__).joinpath(
+                "response-file.yaml.j2"
+            ),
             dest="/var/lib/acme/conf/responses",
             user="root",
             group="root",
@@ -80,7 +84,9 @@ class AcmetoolDeployer(Deployer):
         self.need_restart_reconcile_service = reconcile_service_file.changed
 
         reconcile_timer_file = files.put(
-            src=importlib.resources.files(__package__).joinpath("acmetool-reconcile.timer"),
+            src=importlib.resources.files(__package__).joinpath(
+                "acmetool-reconcile.timer"
+            ),
             dest="/etc/systemd/system/acmetool-reconcile.timer",
             user="root",
             group="root",

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -19,7 +19,7 @@ from packaging import version
 from termcolor import colored
 
 from . import dns, remote
-from .sshexec import SSHExec, LocalExec
+from .sshexec import LocalExec, SSHExec
 
 #
 # cmdeploy sub commands and options
@@ -95,7 +95,7 @@ def run_cmd(args, out):
     env["CHATMAIL_INI"] = args.inipath
     env["CHATMAIL_DISABLE_MAIL"] = "True" if args.disable_mail else ""
     env["CHATMAIL_REQUIRE_IROH"] = "True" if require_iroh else ""
-    deploy_path = importlib.resources.files(__package__).joinpath("deploy.py").resolve()
+    deploy_path = importlib.resources.files(__package__).joinpath("run.py").resolve()
     pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
 
     cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y"
@@ -238,7 +238,12 @@ def fmt_cmd_options(parser):
 def fmt_cmd(args, out):
     """Run formattting fixes on all chatmail source code."""
 
-    sources = [str(importlib.resources.files(x)) for x in ("chatmaild", "cmdeploy")]
+    chatmaild_dir = importlib.resources.files("chatmaild").resolve()
+    cmdeploy_dir = chatmaild_dir.joinpath(
+        "..", "..", "..", "cmdeploy", "src", "cmdeploy"
+    ).resolve()
+    sources = [str(chatmaild_dir), str(cmdeploy_dir)]
+
     format_args = [shutil.which("ruff"), "format"]
     check_args = [shutil.which("ruff"), "check"]
 
@@ -309,7 +314,7 @@ def add_ssh_host_option(parser):
         "--ssh-host",
         dest="ssh_host",
         help="Run commands on 'localhost', via '@docker', or on a specific SSH host "
-             "instead of chatmail.ini's mail_domain.",
+        "instead of chatmail.ini's mail_domain.",
     )
 
 

cmdeploy/src/cmdeploy/deployers.py

@@ -11,7 +11,6 @@ from io import StringIO
 from pathlib import Path
 
 from chatmaild.config import Config, read_config
-from cmdeploy.cmdeploy import Out
 from pyinfra import facts, host, logger
 from pyinfra.api import FactBase
 from pyinfra.facts.files import File, Sha256File
@@ -19,8 +18,10 @@ from pyinfra.facts.server import Sysctl
 from pyinfra.facts.systemd import SystemdEnabled
 from pyinfra.operations import apt, files, pip, server, systemd
 
+from cmdeploy.cmdeploy import Out
+
 from .acmetool import AcmetoolDeployer
-from .deployer import Deployer, Deployment
+from .basedeploy import Deployer, Deployment
 from .www import build_webpages, find_merge_conflict, get_paths
 
 
@@ -39,6 +40,10 @@ class Port(FactBase):
         return output[0]
 
 
+def get_resource(arg, pkg=__package__):
+    return importlib.resources.files(pkg).joinpath(arg)
+
+
 def _build_chatmaild(dist_dir) -> None:
     dist_dir = Path(dist_dir).resolve()
     if dist_dir.exists():
@@ -118,7 +123,7 @@ def _configure_remote_venv_with_chatmaild(config) -> None:
     )
 
     files.template(
-        src=importlib.resources.files(__package__).joinpath("metrics.cron.j2"),
+        src=get_resource("metrics.cron.j2"),
         dest="/etc/cron.d/chatmail-metrics",
         user="root",
         group="root",
@@ -148,7 +153,7 @@ def _configure_remote_units(mail_domain, units) -> None:
 
         basename = fn if "." in fn else f"{fn}.service"
 
-        source_path = importlib.resources.files(__package__).joinpath("service", f"{basename}.f")
+        source_path = get_resource(f"service/{basename}.f")
         content = source_path.read_text().format(**params).encode()
 
         files.put(
@@ -179,13 +184,12 @@ def _activate_remote_units(units) -> None:
         )
 
 
-
 def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
     """Configures OpenDKIM"""
     need_restart = False
 
     main_config = files.template(
-        src=importlib.resources.files(__package__).joinpath("opendkim/opendkim.conf"),
+        src=get_resource("opendkim/opendkim.conf"),
         dest="/etc/opendkim.conf",
         user="root",
         group="root",
@@ -195,7 +199,7 @@ def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
     need_restart |= main_config.changed
 
     screen_script = files.put(
-        src=importlib.resources.files(__package__).joinpath("opendkim/screen.lua"),
+        src=get_resource("opendkim/screen.lua"),
         dest="/etc/opendkim/screen.lua",
         user="root",
         group="root",
@@ -204,7 +208,7 @@ def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
     need_restart |= screen_script.changed
 
     final_script = files.put(
-        src=importlib.resources.files(__package__).joinpath("opendkim/final.lua"),
+        src=get_resource("opendkim/final.lua"),
         dest="/etc/opendkim/final.lua",
         user="root",
         group="root",
@@ -222,7 +226,7 @@ def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
     )
 
     keytable = files.template(
-        src=importlib.resources.files(__package__).joinpath("opendkim/KeyTable"),
+        src=get_resource("opendkim/KeyTable"),
         dest="/etc/dkimkeys/KeyTable",
         user="opendkim",
         group="opendkim",
@@ -232,7 +236,7 @@ def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
     need_restart |= keytable.changed
 
     signing_table = files.template(
-        src=importlib.resources.files(__package__).joinpath("opendkim/SigningTable"),
+        src=get_resource("opendkim/SigningTable"),
         dest="/etc/dkimkeys/SigningTable",
         user="opendkim",
         group="opendkim",
@@ -261,7 +265,7 @@ def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
 
     service_file = files.put(
         name="Configure opendkim to restart once a day",
-        src=importlib.resources.files(__package__).joinpath("opendkim/systemd.conf"),
+        src=get_resource("opendkim/systemd.conf"),
         dest="/etc/systemd/system/opendkim.service.d/10-prevent-memory-leak.conf",
     )
     need_restart |= service_file.changed
@@ -312,7 +316,7 @@ class UnboundDeployer(Deployer):
         # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
         #
         files.put(
-            src=importlib.resources.files(__package__).joinpath("policy-rc.d"),
+            src=get_resource("policy-rc.d"),
             dest="/usr/sbin/policy-rc.d",
             user="root",
             group="root",
@@ -372,7 +376,7 @@ def _configure_postfix(config: Config, debug: bool = False) -> bool:
     need_restart = False
 
     main_config = files.template(
-        src=importlib.resources.files(__package__).joinpath("postfix/main.cf.j2"),
+        src=get_resource("postfix/main.cf.j2"),
         dest="/etc/postfix/main.cf",
         user="root",
         group="root",
@@ -383,7 +387,7 @@ def _configure_postfix(config: Config, debug: bool = False) -> bool:
     need_restart |= main_config.changed
 
     master_config = files.template(
-        src=importlib.resources.files(__package__).joinpath("postfix/master.cf.j2"),
+        src=get_resource("postfix/master.cf.j2"),
         dest="/etc/postfix/master.cf",
         user="root",
         group="root",
@@ -394,9 +398,7 @@ def _configure_postfix(config: Config, debug: bool = False) -> bool:
     need_restart |= master_config.changed
 
     header_cleanup = files.put(
-        src=importlib.resources.files(__package__).joinpath(
+        src=get_resource("postfix/submission_header_cleanup"),
-            "postfix/submission_header_cleanup"
-        ),
         dest="/etc/postfix/submission_header_cleanup",
         user="root",
         group="root",
@@ -406,7 +408,7 @@ def _configure_postfix(config: Config, debug: bool = False) -> bool:
 
     # Login map that 1:1 maps email address to login.
     login_map = files.put(
-        src=importlib.resources.files(__package__).joinpath("postfix/login_map"),
+        src=get_resource("postfix/login_map"),
         dest="/etc/postfix/login_map",
         user="root",
         group="root",
@@ -437,7 +439,9 @@ class PostfixDeployer(Deployer):
         restart = False if self.disable_mail else self.need_restart
 
         systemd.service(
-            name="disable postfix for now" if self.disable_mail else "Start and enable Postfix",
+            name="disable postfix for now"
+            if self.disable_mail
+            else "Start and enable Postfix",
             service="postfix.service",
             running=False if self.disable_mail else True,
             enabled=False if self.disable_mail else True,
@@ -485,7 +489,7 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
     need_restart = False
 
     main_config = files.template(
-        src=importlib.resources.files(__package__).joinpath("dovecot/dovecot.conf.j2"),
+        src=get_resource("dovecot/dovecot.conf.j2"),
         dest="/etc/dovecot/dovecot.conf",
         user="root",
         group="root",
@@ -496,7 +500,7 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
     )
     need_restart |= main_config.changed
     auth_config = files.put(
-        src=importlib.resources.files(__package__).joinpath("dovecot/auth.conf"),
+        src=get_resource("dovecot/auth.conf"),
         dest="/etc/dovecot/auth.conf",
         user="root",
         group="root",
@@ -504,9 +508,7 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
     )
     need_restart |= auth_config.changed
     lua_push_notification_script = files.put(
-        src=importlib.resources.files(__package__).joinpath(
+        src=get_resource("dovecot/push_notification.lua"),
-            "dovecot/push_notification.lua"
-        ),
         dest="/etc/dovecot/push_notification.lua",
         user="root",
         group="root",
@@ -514,13 +516,6 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
     )
     need_restart |= lua_push_notification_script.changed
 
-    # remove historic expunge script
-    # which is now implemented through a systemd chatmail-expire service/timer
-    files.file(
-        path="/etc/cron.d/expunge",
-        present=False,
-    )
-
     # as per https://doc.dovecot.org/configuration_manual/os/
     # it is recommended to set the following inotify limits
     for name in ("max_user_instances", "max_user_watches"):
@@ -569,7 +564,9 @@ class DovecotDeployer(Deployer):
         restart = False if self.disable_mail else self.need_restart
 
         systemd.service(
-            name="disable dovecot for now" if self.disable_mail else "Start and enable Dovecot",
+            name="disable dovecot for now"
+            if self.disable_mail
+            else "Start and enable Dovecot",
             service="dovecot.service",
             running=False if self.disable_mail else True,
             enabled=False if self.disable_mail else True,
@@ -583,7 +580,7 @@ def _configure_nginx(config: Config, debug: bool = False) -> bool:
     need_restart = False
 
     main_config = files.template(
-        src=importlib.resources.files(__package__).joinpath("nginx/nginx.conf.j2"),
+        src=get_resource("nginx/nginx.conf.j2"),
         dest="/etc/nginx/nginx.conf",
         user="root",
         group="root",
@@ -594,7 +591,7 @@ def _configure_nginx(config: Config, debug: bool = False) -> bool:
     need_restart |= main_config.changed
 
     autoconfig = files.template(
-        src=importlib.resources.files(__package__).joinpath("nginx/autoconfig.xml.j2"),
+        src=get_resource("nginx/autoconfig.xml.j2"),
         dest="/var/www/html/.well-known/autoconfig/mail/config-v1.1.xml",
         user="root",
         group="root",
@@ -604,7 +601,7 @@ def _configure_nginx(config: Config, debug: bool = False) -> bool:
     need_restart |= autoconfig.changed
 
     mta_sts_config = files.template(
-        src=importlib.resources.files(__package__).joinpath("nginx/mta-sts.txt.j2"),
+        src=get_resource("nginx/mta-sts.txt.j2"),
         dest="/var/www/html/.well-known/mta-sts.txt",
         user="root",
         group="root",
@@ -625,7 +622,7 @@ def _configure_nginx(config: Config, debug: bool = False) -> bool:
 
     files.put(
         name="Upload cgi newemail.py script",
-        src=importlib.resources.files("chatmaild").joinpath("newemail.py").open("rb"),
+        src=get_resource("newemail.py", pkg="chatmaild").open("rb"),
         dest=f"{cgi_dir}/newemail.py",
         user="root",
         group="root",
@@ -660,7 +657,7 @@ class NginxDeployer(Deployer):
         # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
         #
         files.put(
-            src=importlib.resources.files(__package__).joinpath("policy-rc.d"),
+            src=get_resource("policy-rc.d"),
             dest="/usr/sbin/policy-rc.d",
             user="root",
             group="root",
@@ -708,7 +705,9 @@ class WebsiteDeployer(Deployer):
         if not www_path.is_dir():
             logger.warning("Building web pages is disabled in chatmail.ini, skipping")
         elif (path := find_merge_conflict(src_dir)) is not None:
-            logger.warning(f"Merge conflict found in {path}, skipping website deployment. Fix merge conflict if you want to upload your web page.")
+            logger.warning(
+                f"Merge conflict found in {path}, skipping website deployment. Fix merge conflict if you want to upload your web page."
+            )
         else:
             # if www_folder is a hugo page, build it
             if build_dir:
@@ -719,10 +718,34 @@ class WebsiteDeployer(Deployer):
             )
 
 
-class RspamdDeployer(Deployer):
+class LegacyRemoveDeployer(Deployer):
     def install(self):
         apt.packages(name="Remove rspamd", packages="rspamd", present=False)
 
+        # remove historic expunge script
+        # which is now implemented through a systemd timer (chatmail-expire)
+        files.file(
+            path="/etc/cron.d/expunge",
+            present=False,
+        )
+
+        # Remove OBS repository key that is no longer used.
+        files.file("/etc/apt/keyrings/obs-home-deltachat.gpg", present=False)
+        files.line(
+            name="Remove DeltaChat OBS home repository from sources.list",
+            path="/etc/apt/sources.list",
+            line="deb [signed-by=/etc/apt/keyrings/obs-home-deltachat.gpg] https://download.opensuse.org/repositories/home:/deltachat/Debian_12/ ./",
+            escape_regex_characters=True,
+            present=False,
+        )
+
+        # prior relay versions used filelogging
+        files.directory(
+            name="Ensure old logs on disk are deleted",
+            path="/var/log/journal/",
+            present=False,
+        )
+
 
 def check_config(config):
     mail_domain = config.mail_domain
@@ -778,7 +801,7 @@ class MtailDeployer(Deployer):
         self.mtail_address = mtail_address
 
     def install(self):
-        # Uninstall mtail package, we are going to install a static binary.
+        # Uninstall mtail package to install a static binary.
         apt.packages(name="Uninstall mtail", packages=["mtail"], present=False)
 
         (url, sha256sum) = {
@@ -804,9 +827,7 @@ class MtailDeployer(Deployer):
         # Using our own systemd unit instead of `/usr/lib/systemd/system/mtail.service`.
         # This allows to read from journalctl instead of log files.
         files.template(
-            src=importlib.resources.files(__package__).joinpath(
+            src=get_resource("mtail/mtail.service.j2"),
-                "mtail/mtail.service.j2"
-            ),
             dest="/etc/systemd/system/mtail.service",
             user="root",
             group="root",
@@ -817,9 +838,7 @@ class MtailDeployer(Deployer):
 
         mtail_conf = files.put(
             name="Mtail configuration",
-            src=importlib.resources.files(__package__).joinpath(
+            src=get_resource("mtail/delivered_mail.mtail"),
-                "mtail/delivered_mail.mtail"
-            ),
             dest="/etc/mtail/delivered_mail.mtail",
             user="root",
             group="root",
@@ -869,7 +888,7 @@ class IrohDeployer(Deployer):
     def configure(self):
         systemd_unit = files.put(
             name="Upload iroh-relay systemd unit",
-            src=importlib.resources.files(__package__).joinpath("iroh-relay.service"),
+            src=get_resource("iroh-relay.service"),
             dest="/etc/systemd/system/iroh-relay.service",
             user="root",
             group="root",
@@ -879,7 +898,7 @@ class IrohDeployer(Deployer):
 
         iroh_config = files.put(
             name="Upload iroh-relay config",
-            src=importlib.resources.files(__package__).joinpath("iroh-relay.toml"),
+            src=get_resource("iroh-relay.toml"),
             dest="/etc/iroh-relay.toml",
             user="root",
             group="root",
@@ -902,7 +921,7 @@ class JournaldDeployer(Deployer):
     def configure(self):
         journald_conf = files.put(
             name="Configure journald",
-            src=importlib.resources.files(__package__).joinpath("journald.conf"),
+            src=get_resource("journald.conf"),
             dest="/etc/systemd/journald.conf",
             user="root",
             group="root",
@@ -981,17 +1000,6 @@ class ChatmailDeployer(Deployer):
         self.mail_domain = mail_domain
 
     def install(self):
-        # Remove OBS repository key that is no longer used.
-        files.file("/etc/apt/keyrings/obs-home-deltachat.gpg", present=False)
-
-        files.line(
-            name="Remove DeltaChat OBS home repository from sources.list",
-            path="/etc/apt/sources.list",
-            line="deb [signed-by=/etc/apt/keyrings/obs-home-deltachat.gpg] https://download.opensuse.org/repositories/home:/deltachat/Debian_12/ ./",
-            escape_regex_characters=True,
-            present=False,
-        )
-
         apt.update(name="apt update", cache_time=24 * 3600)
         apt.upgrade(name="upgrade apt packages", auto_remove=True)
 
@@ -1101,17 +1109,16 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
 
     all_deployers = [
         ChatmailDeployer(mail_domain),
+        LegacyRemoveDeployer(),
         JournaldDeployer(),
         UnboundDeployer(),
         TurnDeployer(mail_domain),
         IrohDeployer(config.enable_iroh_relay),
         AcmetoolDeployer(config.acme_email, tls_domains),
-
         WebsiteDeployer(config),
         ChatmailVenvDeployer(config),
         MtastsDeployer(),
         OpendkimDeployer(mail_domain),
-
         # Dovecot should be started before Postfix
         # because it creates authentication socket
         # required by Postfix.
@@ -1119,7 +1126,6 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
         PostfixDeployer(config, disable_mail),
         FcgiwrapDeployer(),
         NginxDeployer(config),
-        RspamdDeployer(),
         EchobotDeployer(mail_domain),
         MtailDeployer(config.mtail_address),
         GithashDeployer(),
@@ -1127,8 +1133,3 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
 
     Deployment().perform_stages(all_deployers)
 
-    files.directory(
-        name="Ensure old logs on disk are deleted",
-        path="/var/log/journal/",
-        present=False,
-    )

cmdeploy/src/cmdeploy/dns.py

@@ -45,7 +45,8 @@ def check_full_zone(sshexec, remote_data, out, zonefile) -> int:
     and return (exitcode, remote_data) tuple."""
 
     required_diff, recommended_diff = sshexec.logged(
-        remote.rdns.check_zonefile, kwargs=dict(zonefile=zonefile, verbose=False),
+        remote.rdns.check_zonefile,
+        kwargs=dict(zonefile=zonefile, verbose=False),
     )
 
     returncode = 0

cmdeploy/src/cmdeploy/remote/rdns.py

@@ -12,7 +12,7 @@ All functions of this module
 
 import re
 
-from .rshell import CalledProcessError, shell, log_progress
+from .rshell import CalledProcessError, log_progress, shell
 
 
 def perform_initial_checks(mail_domain, pre_command=""):
@@ -26,7 +26,9 @@ def perform_initial_checks(mail_domain, pre_command=""):
     WWW = query_dns("CNAME", f"www.{mail_domain}")
 
     res = dict(mail_domain=mail_domain, A=A, AAAA=AAAA, MTA_STS=MTA_STS, WWW=WWW)
-    res["acme_account_url"] = shell(pre_command + "acmetool account-url", fail_ok=True, print=log_progress)
+    res["acme_account_url"] = shell(
+        pre_command + "acmetool account-url", fail_ok=True, print=log_progress
+    )
     res["dkim_entry"], res["web_dkim_entry"] = get_dkim_entry(
         mail_domain, pre_command, dkim_selector="opendkim"
     )
@@ -45,7 +47,7 @@ def get_dkim_entry(mail_domain, pre_command, dkim_selector):
         dkim_pubkey = shell(
             f"{pre_command}openssl rsa -in /etc/dkimkeys/{dkim_selector}.private "
             "-pubout 2>/dev/null | awk '/-/{next}{printf(\"%s\",$0)}'",
-            print=log_progress
+            print=log_progress,
         )
     except CalledProcessError:
         return
@@ -62,9 +64,9 @@ def query_dns(typ, domain):
     # Get autoritative nameserver from the SOA record.
     soa_answers = [
         x.split()
-        for x in shell(f"dig -r -q {domain} -t SOA +noall +authority +answer", print=log_progress).split(
+        for x in shell(
-            "\n"
+            f"dig -r -q {domain} -t SOA +noall +authority +answer", print=log_progress
-        )
+        ).split("\n")
     ]
     soa = [a for a in soa_answers if len(a) >= 3 and a[3] == "SOA"]
     if not soa:
@@ -73,7 +75,7 @@ def query_dns(typ, domain):
 
     # Query authoritative nameserver directly to bypass DNS cache.
     res = shell(f"dig @{ns} -r -q {domain} -t {typ} +short", print=log_progress)
-    return next((line for line in res.split("\n") if not line.startswith(';')), '')
+    return next((line for line in res.split("\n") if not line.startswith(";")), "")
 
 
 def check_zonefile(zonefile, verbose=True):

cmdeploy/src/cmdeploy/remote/rshell.py

@@ -1,5 +1,4 @@
 import sys
-
 from subprocess import DEVNULL, CalledProcessError, check_output
 
 

cmdeploy/src/cmdeploy/run.py

@@ -3,7 +3,9 @@ import os
 
 import pyinfra
 
-from cmdeploy import deploy_chatmail
+# pyinfra runs this module as a python file and not as a module so
+# import paths must be absolute
+from cmdeploy.deployers import deploy_chatmail
 
 
 def main():

cmdeploy/src/cmdeploy/sshexec.py

@@ -93,7 +93,7 @@ class LocalExec:
         where = "locally"
         if self.docker:
             if call == remote.rdns.perform_initial_checks:
-                kwargs['pre_command'] = "docker exec chatmail "
+                kwargs["pre_command"] = "docker exec chatmail "
                 where = "in docker"
         if self.verbose:
             print(f"Running {where}: {call.__name__}(**{kwargs})")

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -35,7 +35,7 @@ class TestSSHExecutor:
         out, err = capsys.readouterr()
         assert err.startswith("Collecting")
         # XXX could not figure out how capturing can be made to work properly
-        #assert err.endswith("....\n")
+        # assert err.endswith("....\n")
         assert err.count("\n") == 1
 
         sshexec.verbose = True
@@ -45,7 +45,7 @@ class TestSSHExecutor:
         out, err = capsys.readouterr()
         lines = err.split("\n")
         # XXX could not figure out how capturing can be made to work properly
-        #assert len(lines) > 4
+        # assert len(lines) > 4
         assert remote.rdns.perform_initial_checks.__doc__ in lines[0]
 
     def test_exception(self, sshexec, capsys):

cmdeploy/src/cmdeploy/tests/test_dns.py

@@ -65,7 +65,9 @@ class TestPerformInitialChecks:
         remote_data = remote.rdns.perform_initial_checks("some.domain")
         assert remote_data["A"] == mockdns_expected["A"]["some.domain"]
         assert remote_data["AAAA"] == mockdns_expected["AAAA"]["some.domain"]
-        assert remote_data["MTA_STS"] == mockdns_expected["CNAME"]["mta-sts.some.domain"]
+        assert (
+            remote_data["MTA_STS"] == mockdns_expected["CNAME"]["mta-sts.some.domain"]
+        )
         assert remote_data["WWW"] == mockdns_expected["CNAME"]["www.some.domain"]
 
     @pytest.mark.parametrize("drop", ["A", "AAAA"])

cmdeploy/src/cmdeploy/www.py

@@ -1,10 +1,10 @@
 import hashlib
 import importlib.resources
+import re
 import time
 import traceback
 import webbrowser
 from pathlib import Path
-import re
 
 import markdown
 from chatmaild.config import read_config
@@ -12,8 +12,9 @@ from jinja2 import Template
 
 from .genqr import gen_qr_png_data
 
-
+_MERGE_CONFLICT_RE = re.compile(
-_MERGE_CONFLICT_RE = re.compile(r"^<<<<<<<.+^=======.+^>>>>>>>", re.DOTALL | re.MULTILINE)
+    r"^<<<<<<<.+^=======.+^>>>>>>>", re.DOTALL | re.MULTILINE
+)
 
 
 def snapshot_dir_stats(somedir):

doc/source/overview.rst

@@ -126,14 +126,13 @@ web page. Edit them before deploying to make your chatmail relay
 stand out.
 
 
-Component dependency diagram
+Chatmail relay dependency diagram
---------------------------------------
+---------------------------------
 
 .. mermaid::
    :caption: This diagram shows relay components and dependencies/communication paths.
 
     graph LR;
-        cmdeploy --- sshd;
         letsencrypt --- |80|acmetool-redirector;
         acmetool-redirector --- |443|nginx-right(["`nginx
         (external)`"]);
@@ -147,33 +146,100 @@ Component dependency diagram
         nginx-internal --- autoconfig.xml;
         certs-nginx[("`TLS certs
         /var/lib/acme`")] --> nginx-internal;
-        cron --- chatmail-metrics;
+        systemd-timer --- chatmail-metrics;
-        cron --- acmetool;
+        systemd-timer --- acmetool;
+        systemd-timer --- chatmail-expire-daily;
+        systemd-timer --- chatmail-fsreport-daily;
         chatmail-metrics --- website;
         acmetool --> certs[("`TLS certs
         /var/lib/acme`")];
         nginx-external --- |993|dovecot;
+        postfix --- |SASL|dovecot;
         autoconfig.xml --- postfix;
         autoconfig.xml --- dovecot;
-        postfix --- echobot;
+        postfix --- |10080|filtermail-outgoing;
-        postfix --- |10080,10081|filtermail;
+        postfix --- |10081|filtermail-incoming;
-        postfix --- users["`User data
+        filtermail-outgoing --- |10025 reinject|postfix;
-        home/vmail/mail`"];
+        filtermail-incoming --- |10026 reinject|postfix;
-        postfix --- |doveauth.socket|doveauth;
         dovecot --- |doveauth.socket|doveauth;
-        dovecot --- users;
+        dovecot --- |message delivery|maildir["maildir
-        dovecot --- |metadata.socket|chatmail-metadata;
+        /home/vmail/.../user"];
-        doveauth --- users;
+        dovecot --- |lastlogin.socket|lastlogin;
-        chatmail-expire-daily --- users;
+        dovecot --- chatmail-metadata;
-        chatmail-fsreport-daily --- users;
+        lastlogin --- maildir;
+        doveauth --- maildir;
+        chatmail-expire-daily --- maildir;
+        chatmail-fsreport-daily --- maildir;
         chatmail-metadata --- iroh-relay;
+        chatmail-metadata --- |encrypted device token| notifications.delta.chat;
         certs-nginx --> postfix;
         certs-nginx --> dovecot;
         style certs fill:#ff6;
+        style website fill:#ff6;
+        style maildir fill:#ff6;
         style certs-nginx fill:#ff6;
-        style nginx-external fill:#fc9;
+        style nginx-external fill:#f66;
-        style nginx-right fill:#fc9;
+        style nginx-right fill:#f66;
+        style postfix fill:#f66;
+        style dovecot fill:#f66;
+        style notification-proxy fill:#f66;
 
+Message between users on the same relay
+---------------------------------------
+
+.. mermaid::
+    :caption: This diagram shows the path a non-federated message takes.
+
+    graph LR;
+        sender --> |465|smtps/smtpd;
+        sender --> |587|submission/smtpd;
+        smtps/smtpd --> |10080|filtermail;
+        submission/smtpd --> |10080|filtermail;
+        filtermail --> |10025|smtpd_reinject;
+        smtpd_reinject --> cleanup;
+        cleanup --> qmgr;
+        qmgr --> smtpd_accepts_message;
+        qmgr --> |lmtp|dovecot;
+        dovecot --> recipient;
+        dovecot --> sender's_other_devices;
+
+Message to an external address
+------------------------------
+
+.. mermaid::
+    :caption: This diagram shows the path a federated message takes.
+
+    graph LR;
+        sender --> |465|smtps/smtpd;
+        sender --> |587|submission/smtpd;
+        smtps/smtpd --> |10080|filtermail;
+        submission/smtpd --> |10080|filtermail;
+        filtermail --> |10025|smtpd_reinject;
+        smtpd_reinject --> opendkim;
+        opendkim --> smtpd_reinject;
+        smtpd_reinject --> cleanup;
+        cleanup --> qmgr;
+        qmgr --> external_smtp_server;
+        qmgr --> |lmtp|dovecot;
+        external_smtp_server --> recipient;
+        dovecot --> senders_other_devices;
+
+Message from an external address
+--------------------------------
+
+.. mermaid::
+    :caption: This diagram shows the path a federated message takes.
+
+    graph LR;
+        external_smtp_server --> |25|smtpd;
+        smtps/smtpd --> |10081|filtermail-incoming;
+        filtermail-incoming --> |10026|smtpd_reinject_incoming;
+        smtpd_reinject_incoming --> opendkim;
+        opendkim --> smtpd_reinject_incoming;
+        smtpd_reinject_incoming --> cleanup;
+        cleanup --> qmgr;
+        qmgr --> |lmtp|dovecot;
+        dovecot --> recipient;
 
 Operational details of a chatmail relay
 ----------------------------------------