docs: Remove section about use of objects

- Revised cmdeploy documentation in doc/source/overview.rst to reflect
  the recent revisions to the Deployer interface.

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## untagged
 
+- Organized cmdeploy into install, configure, and activate stages
+  ([#695](https://github.com/chatmail/relay/pull/695))
+
 - docs: move readme.md docs to sphinx documentation rendered at https://chatmail.at/doc/relay 
   ([#711](https://github.com/chatmail/relay/pull/711))
 

cmdeploy/src/cmdeploy/__init__.py

@@ -11,6 +11,7 @@ from io import StringIO
 from pathlib import Path
 
 from chatmaild.config import Config, read_config
+from cmdeploy.cmdeploy import Out
 from pyinfra import facts, host, logger
 from pyinfra.api import FactBase
 from pyinfra.facts.files import File, Sha256File
@@ -18,7 +19,9 @@ from pyinfra.facts.server import Sysctl
 from pyinfra.facts.systemd import SystemdEnabled
 from pyinfra.operations import apt, files, pip, server, systemd
 
-from .acmetool import deploy_acmetool
+from .acmetool import AcmetoolDeployer
+from .deployer import Deployer, Deployment
+from .www import build_webpages, find_merge_conflict, get_paths
 
 
 class Port(FactBase):
@@ -61,13 +64,12 @@ def remove_legacy_artifacts():
         )
 
 
-def _install_remote_venv_with_chatmaild(config) -> None:
+def _install_remote_venv_with_chatmaild() -> None:
     remove_legacy_artifacts()
     dist_file = _build_chatmaild(dist_dir=Path("chatmaild/dist"))
     remote_base_dir = "/usr/local/lib/chatmaild"
     remote_dist_file = f"{remote_base_dir}/dist/{dist_file.name}"
     remote_venv_dir = f"{remote_base_dir}/venv"
-    remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini"
     root_owned = dict(user="root", group="root", mode="644")
 
     apt.packages(
@@ -83,13 +85,6 @@ def _install_remote_venv_with_chatmaild(config) -> None:
         **root_owned,
     )
 
-    files.put(
-        name=f"Upload {remote_chatmail_inipath}",
-        src=config._getbytefile(),
-        dest=remote_chatmail_inipath,
-        **root_owned,
-    )
-
     pip.virtualenv(
         name=f"chatmaild virtualenv {remote_venv_dir}",
         path=remote_venv_dir,
@@ -108,6 +103,20 @@ def _install_remote_venv_with_chatmaild(config) -> None:
         ],
     )
 
+
+def _configure_remote_venv_with_chatmaild(config) -> None:
+    remote_base_dir = "/usr/local/lib/chatmaild"
+    remote_venv_dir = f"{remote_base_dir}/venv"
+    remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini"
+    root_owned = dict(user="root", group="root", mode="644")
+
+    files.put(
+        name=f"Upload {remote_chatmail_inipath}",
+        src=config._getbytefile(),
+        dest=remote_chatmail_inipath,
+        **root_owned,
+    )
+
     files.template(
         src=importlib.resources.files(__package__).joinpath("metrics.cron.j2"),
         dest="/etc/cron.d/chatmail-metrics",
@@ -120,26 +129,21 @@ def _install_remote_venv_with_chatmaild(config) -> None:
         },
     )
 
+
+def _configure_remote_units(mail_domain, units) -> None:
+    remote_base_dir = "/usr/local/lib/chatmaild"
+    remote_venv_dir = f"{remote_base_dir}/venv"
+    remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini"
+    root_owned = dict(user="root", group="root", mode="644")
+
     # install systemd units
-    for fn in (
+    for fn in units:
-        "doveauth",
-        "filtermail",
-        "filtermail-incoming",
-        "echobot",
-        "chatmail-metadata",
-        "lastlogin",
-        "turnserver",
-        "chatmail-expire",
-        "chatmail-expire.timer",
-        "chatmail-fsreport",
-        "chatmail-fsreport.timer",
-    ):
         execpath = fn if fn != "filtermail-incoming" else "filtermail"
         params = dict(
             execpath=f"{remote_venv_dir}/bin/{execpath}",
             config_path=remote_chatmail_inipath,
             remote_venv_dir=remote_venv_dir,
-            mail_domain=config.mail_domain,
+            mail_domain=mail_domain,
         )
 
         basename = fn if "." in fn else f"{fn}.service"
@@ -153,6 +157,13 @@ def _install_remote_venv_with_chatmaild(config) -> None:
             dest=f"/etc/systemd/system/{basename}",
             **root_owned,
         )
+
+
+def _activate_remote_units(units) -> None:
+    # activate systemd units
+    for fn in units:
+        basename = fn if "." in fn else f"{fn}.service"
+
         if fn == "chatmail-expire" or fn == "chatmail-fsreport":
             # don't auto-start but let the corresponding timer trigger execution
             enabled = False
@@ -238,11 +249,6 @@ def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
         present=True,
     )
 
-    apt.packages(
-        name="apt install opendkim opendkim-tools",
-        packages=["opendkim", "opendkim-tools"],
-    )
-
     if not host.get_fact(File, f"/etc/dkimkeys/{dkim_selector}.private"):
         server.shell(
             name="Generate OpenDKIM domain keys",
@@ -263,14 +269,95 @@ def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
     return need_restart
 
 
-def _uninstall_mta_sts_daemon() -> None:
+class OpendkimDeployer(Deployer):
+    required_users = [("opendkim", None, ["opendkim"])]
+
+    def __init__(self, mail_domain):
+        self.mail_domain = mail_domain
+
+    def install(self):
+        apt.packages(
+            name="apt install opendkim opendkim-tools",
+            packages=["opendkim", "opendkim-tools"],
+        )
+
+    def configure(self):
+        self.need_restart = _configure_opendkim(self.mail_domain, "opendkim")
+
+    def activate(self):
+        systemd.service(
+            name="Start and enable OpenDKIM",
+            service="opendkim.service",
+            running=True,
+            enabled=True,
+            daemon_reload=self.need_restart,
+            restarted=self.need_restart,
+        )
+        self.need_restart = False
+
+
+class UnboundDeployer(Deployer):
+    def install(self):
+        # Run local DNS resolver `unbound`.
+        # `resolvconf` takes care of setting up /etc/resolv.conf
+        # to use 127.0.0.1 as the resolver.
+
+        #
+        # On an IPv4-only system, if unbound is started but not
+        # configured, it causes subsequent steps to fail to resolve hosts.
+        # Here, we use policy-rc.d to prevent unbound from starting up
+        # on initial install.  Later, we will configure it and start it.
+        #
+        # For documentation about policy-rc.d, see:
+        # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
+        #
+        files.put(
+            src=importlib.resources.files(__package__).joinpath("policy-rc.d"),
+            dest="/usr/sbin/policy-rc.d",
+            user="root",
+            group="root",
+            mode="755",
+        )
+
+        apt.packages(
+            name="Install unbound",
+            packages=["unbound", "unbound-anchor", "dnsutils"],
+        )
+
+        files.file("/usr/sbin/policy-rc.d", present=False)
+
+    def configure(self):
+        server.shell(
+            name="Generate root keys for validating DNSSEC",
+            commands=[
+                "unbound-anchor -a /var/lib/unbound/root.key || true",
+            ],
+        )
+
+    def activate(self):
+        server.shell(
+            name="Generate root keys for validating DNSSEC",
+            commands=[
+                "systemctl reset-failed unbound.service",
+            ],
+        )
+
+        systemd.service(
+            name="Start and enable unbound",
+            service="unbound.service",
+            running=True,
+            enabled=True,
+        )
+
+
+class MtastsDeployer(Deployer):
+    def configure(self):
         # Remove configuration.
         files.file("/etc/mta-sts-daemon.yml", present=False)
-
         files.directory("/usr/local/lib/postfix-mta-sts-resolver", present=False)
-
         files.file("/etc/systemd/system/mta-sts-daemon.service", present=False)
 
+    def activate(self):
         systemd.service(
             name="Stop MTA-STS daemon",
             service="mta-sts-daemon.service",
@@ -330,6 +417,35 @@ def _configure_postfix(config: Config, debug: bool = False) -> bool:
     return need_restart
 
 
+class PostfixDeployer(Deployer):
+    required_users = [("postfix", None, ["opendkim"])]
+
+    def __init__(self, config, disable_mail):
+        self.config = config
+        self.disable_mail = disable_mail
+
+    def install(self):
+        apt.packages(
+            name="Install Postfix",
+            packages="postfix",
+        )
+
+    def configure(self):
+        self.need_restart = _configure_postfix(self.config)
+
+    def activate(self):
+        restart = False if self.disable_mail else self.need_restart
+
+        systemd.service(
+            name="disable postfix for now" if self.disable_mail else "Start and enable Postfix",
+            service="postfix.service",
+            running=False if self.disable_mail else True,
+            enabled=False if self.disable_mail else True,
+            restarted=restart,
+        )
+        self.need_restart = False
+
+
 def _install_dovecot_package(package: str, arch: str):
     arch = "amd64" if arch == "x86_64" else arch
     arch = "arm64" if arch == "aarch64" else arch
@@ -430,6 +546,38 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
     return need_restart
 
 
+class DovecotDeployer(Deployer):
+    def __init__(self, config, disable_mail):
+        self.config = config
+        self.disable_mail = disable_mail
+        self.units = ["doveauth"]
+
+    def install(self):
+        arch = host.get_fact(facts.server.Arch)
+        if not "dovecot.service" in host.get_fact(SystemdEnabled):
+            _install_dovecot_package("core", arch)
+            _install_dovecot_package("imapd", arch)
+            _install_dovecot_package("lmtpd", arch)
+
+    def configure(self):
+        _configure_remote_units(self.config.mail_domain, self.units)
+        self.need_restart = _configure_dovecot(self.config)
+
+    def activate(self):
+        _activate_remote_units(self.units)
+
+        restart = False if self.disable_mail else self.need_restart
+
+        systemd.service(
+            name="disable dovecot for now" if self.disable_mail else "Start and enable Dovecot",
+            service="dovecot.service",
+            running=False if self.disable_mail else True,
+            enabled=False if self.disable_mail else True,
+            restarted=restart,
+        )
+        self.need_restart = False
+
+
 def _configure_nginx(config: Config, debug: bool = False) -> bool:
     """Configures nginx HTTP server."""
     need_restart = False
@@ -487,8 +635,92 @@ def _configure_nginx(config: Config, debug: bool = False) -> bool:
     return need_restart
 
 
-def _remove_rspamd() -> None:
+class NginxDeployer(Deployer):
-    """Remove rspamd"""
+    def __init__(self, config):
+        self.config = config
+
+    def install(self):
+        #
+        # If we allow nginx to start up on install, it will grab port
+        # 80, which then will block acmetool from listening on the port.
+        # That in turn prevents getting certificates, which then causes
+        # an error when we try to start nginx on the custom config
+        # that leaves port 80 open but also requires certificates to
+        # be present.  To avoid getting into that interlocking mess,
+        # we use policy-rc.d to prevent nginx from starting up when it
+        # is installed.
+        #
+        # This approach allows us to avoid performing any explicit
+        # systemd operations during the install stage (as opposed to
+        # allowing it to start and then forcing it to stop), which allows
+        # the install stage to run in non-systemd environments like a
+        # container image build.
+        #
+        # For documentation about policy-rc.d, see:
+        # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
+        #
+        files.put(
+            src=importlib.resources.files(__package__).joinpath("policy-rc.d"),
+            dest="/usr/sbin/policy-rc.d",
+            user="root",
+            group="root",
+            mode="755",
+        )
+
+        apt.packages(
+            name="Install nginx",
+            packages=["nginx", "libnginx-mod-stream"],
+        )
+
+        files.file("/usr/sbin/policy-rc.d", present=False)
+
+    def configure(self):
+        self.need_restart = _configure_nginx(self.config)
+
+    def activate(self):
+        systemd.service(
+            name="Start and enable nginx",
+            service="nginx.service",
+            running=True,
+            enabled=True,
+            restarted=self.need_restart,
+        )
+        self.need_restart = False
+
+
+class WebsiteDeployer(Deployer):
+    def __init__(self, config):
+        self.config = config
+
+    def install(self):
+        files.directory(
+            name="Ensure /var/www exists",
+            path="/var/www",
+            user="root",
+            group="root",
+            mode="755",
+            present=True,
+        )
+
+    def configure(self):
+        www_path, src_dir, build_dir = get_paths(self.config)
+        # if www_folder was set to a non-existing folder, skip upload
+        if not www_path.is_dir():
+            logger.warning("Building web pages is disabled in chatmail.ini, skipping")
+        elif (path := find_merge_conflict(src_dir)) is not None:
+            logger.warning(f"Merge conflict found in {path}, skipping website deployment. Fix merge conflict if you want to upload your web page.")
+        else:
+            # if www_folder is a hugo page, build it
+            if build_dir:
+                www_path = build_webpages(src_dir, build_dir, self.config)
+            # if it is not a hugo page, upload it as is
+            files.rsync(
+                f"{www_path}/", "/var/www/html", flags=["-avz", "--chown=www-data"]
+            )
+
+
+class RspamdDeployer(Deployer):
+    def install(self):
         apt.packages(name="Remove rspamd", packages="rspamd", present=False)
 
 
@@ -507,7 +739,12 @@ def check_config(config):
     return config
 
 
-def deploy_turn_server(config):
+class TurnDeployer(Deployer):
+    def __init__(self, mail_domain):
+        self.mail_domain = mail_domain
+        self.units = ["turnserver"]
+
+    def install(self):
         (url, sha256sum) = {
             "x86_64": (
                 "https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-x86_64-linux",
@@ -519,8 +756,6 @@ def deploy_turn_server(config):
             ),
         }[host.get_fact(facts.server.Arch)]
 
-    need_restart = False
-
         existing_sha256sum = host.get_fact(Sha256File, "/usr/local/bin/chatmail-turn")
         if existing_sha256sum != sha256sum:
             server.shell(
@@ -530,34 +765,19 @@ def deploy_turn_server(config):
                     "chmod 755 /usr/local/bin/chatmail-turn",
                 ],
             )
-        need_restart = True
 
-    source_path = importlib.resources.files(__package__).joinpath(
+    def configure(self):
-        "service", "turnserver.service.f"
+        _configure_remote_units(self.mail_domain, self.units)
-    )
-    content = source_path.read_text().format(mail_domain=config.mail_domain).encode()
 
-    systemd_unit = files.put(
+    def activate(self):
-        name="Upload turnserver.service",
+        _activate_remote_units(self.units)
-        src=io.BytesIO(content),
-        dest="/etc/systemd/system/turnserver.service",
-        user="root",
-        group="root",
-        mode="644",
-    )
-    need_restart |= systemd_unit.changed
-
-    systemd.service(
-        name="Setup turnserver service",
-        service="turnserver.service",
-        running=True,
-        enabled=True,
-        restarted=need_restart,
-        daemon_reload=systemd_unit.changed,
-    )
 
 
-def deploy_mtail(config):
+class MtailDeployer(Deployer):
+    def __init__(self, mtail_address):
+        self.mtail_address = mtail_address
+
+    def install(self):
         # Uninstall mtail package, we are going to install a static binary.
         apt.packages(name="Uninstall mtail", packages=["mtail"], present=False)
 
@@ -580,15 +800,18 @@ def deploy_mtail(config):
             ],
         )
 
+    def configure(self):
         # Using our own systemd unit instead of `/usr/lib/systemd/system/mtail.service`.
         # This allows to read from journalctl instead of log files.
         files.template(
-        src=importlib.resources.files(__package__).joinpath("mtail/mtail.service.j2"),
+            src=importlib.resources.files(__package__).joinpath(
+                "mtail/mtail.service.j2"
+            ),
             dest="/etc/systemd/system/mtail.service",
             user="root",
             group="root",
             mode="644",
-        address=config.mtail_address or "127.0.0.1",
+            address=self.mtail_address or "127.0.0.1",
             port=3903,
         )
 
@@ -602,17 +825,24 @@ def deploy_mtail(config):
             group="root",
             mode="644",
         )
+        self.need_restart = mtail_conf.changed
 
+    def activate(self):
         systemd.service(
             name="Start and enable mtail",
             service="mtail.service",
-        running=bool(config.mtail_address),
+            running=bool(self.mtail_address),
-        enabled=bool(config.mtail_address),
+            enabled=bool(self.mtail_address),
-        restarted=mtail_conf.changed,
+            restarted=self.need_restart,
         )
+        self.need_restart = False
 
 
-def deploy_iroh_relay(config) -> None:
+class IrohDeployer(Deployer):
+    def __init__(self, enable_iroh_relay):
+        self.enable_iroh_relay = enable_iroh_relay
+
+    def install(self):
         (url, sha256sum) = {
             "x86_64": (
                 "https://github.com/n0-computer/iroh/releases/download/v0.35.0/iroh-relay-v0.35.0-x86_64-unknown-linux-musl.tar.gz",
@@ -624,13 +854,6 @@ def deploy_iroh_relay(config) -> None:
             ),
         }[host.get_fact(facts.server.Arch)]
 
-    apt.packages(
-        name="Install curl",
-        packages=["curl"],
-    )
-
-    need_restart = False
-
         existing_sha256sum = host.get_fact(Sha256File, "/usr/local/bin/iroh-relay")
         if existing_sha256sum != sha256sum:
             server.shell(
@@ -640,8 +863,10 @@ def deploy_iroh_relay(config) -> None:
                     "chmod 755 /usr/local/bin/iroh-relay",
                 ],
             )
-        need_restart = True
 
+            self.need_restart = True
+
+    def configure(self):
         systemd_unit = files.put(
             name="Upload iroh-relay systemd unit",
             src=importlib.resources.files(__package__).joinpath("iroh-relay.service"),
@@ -650,7 +875,7 @@ def deploy_iroh_relay(config) -> None:
             group="root",
             mode="644",
         )
-    need_restart |= systemd_unit.changed
+        self.need_restart |= systemd_unit.changed
 
         iroh_config = files.put(
             name="Upload iroh-relay config",
@@ -660,14 +885,172 @@ def deploy_iroh_relay(config) -> None:
             group="root",
             mode="644",
         )
-    need_restart |= iroh_config.changed
+        self.need_restart |= iroh_config.changed
 
+    def activate(self):
         systemd.service(
             name="Start and enable iroh-relay",
             service="iroh-relay.service",
             running=True,
-        enabled=config.enable_iroh_relay,
+            enabled=self.enable_iroh_relay,
-        restarted=need_restart,
+            restarted=self.need_restart,
+        )
+        self.need_restart = False
+
+
+class JournaldDeployer(Deployer):
+    def configure(self):
+        journald_conf = files.put(
+            name="Configure journald",
+            src=importlib.resources.files(__package__).joinpath("journald.conf"),
+            dest="/etc/systemd/journald.conf",
+            user="root",
+            group="root",
+            mode="644",
+        )
+        self.need_restart = journald_conf.changed
+
+    def activate(self):
+        systemd.service(
+            name="Start and enable journald",
+            service="systemd-journald.service",
+            running=True,
+            enabled=True,
+            restarted=self.need_restart,
+        )
+        self.need_restart = False
+
+
+class EchobotDeployer(Deployer):
+    #
+    # This deployer depends on the dovecot and postfix deployers because
+    # it needs to base its decision of whether to restart the service on
+    # whether those two services were restarted.
+    #
+    def __init__(self, mail_domain):
+        self.mail_domain = mail_domain
+        self.units = ["echobot"]
+
+    def install(self):
+        apt.packages(
+            # required for setfacl for echobot
+            name="Install acl",
+            packages="acl",
+        )
+
+    def configure(self):
+        _configure_remote_units(self.mail_domain, self.units)
+
+    def activate(self):
+        _activate_remote_units(self.units)
+
+
+class ChatmailVenvDeployer(Deployer):
+    def __init__(self, config):
+        self.config = config
+        self.units = (
+            "filtermail",
+            "filtermail-incoming",
+            "chatmail-metadata",
+            "lastlogin",
+            "chatmail-expire",
+            "chatmail-expire.timer",
+            "chatmail-fsreport",
+            "chatmail-fsreport.timer",
+        )
+
+    def install(self):
+        _install_remote_venv_with_chatmaild()
+
+    def configure(self):
+        _configure_remote_venv_with_chatmaild(self.config)
+        _configure_remote_units(self.config.mail_domain, self.units)
+
+    def activate(self):
+        _activate_remote_units(self.units)
+
+
+class ChatmailDeployer(Deployer):
+    required_users = [
+        ("vmail", "vmail", None),
+        ("echobot", None, None),
+        ("iroh", None, None),
+    ]
+
+    def __init__(self, mail_domain):
+        self.mail_domain = mail_domain
+
+    def install(self):
+        # Remove OBS repository key that is no longer used.
+        files.file("/etc/apt/keyrings/obs-home-deltachat.gpg", present=False)
+
+        files.line(
+            name="Remove DeltaChat OBS home repository from sources.list",
+            path="/etc/apt/sources.list",
+            line="deb [signed-by=/etc/apt/keyrings/obs-home-deltachat.gpg] https://download.opensuse.org/repositories/home:/deltachat/Debian_12/ ./",
+            escape_regex_characters=True,
+            present=False,
+        )
+
+        apt.update(name="apt update", cache_time=24 * 3600)
+        apt.upgrade(name="upgrade apt packages", auto_remove=True)
+
+        apt.packages(
+            name="Install curl",
+            packages=["curl"],
+        )
+
+        apt.packages(
+            name="Install rsync",
+            packages=["rsync"],
+        )
+        apt.packages(
+            name="Ensure cron is installed",
+            packages=["cron"],
+        )
+
+    def configure(self):
+        # This file is used by auth proxy.
+        # https://wiki.debian.org/EtcMailName
+        server.shell(
+            name="Setup /etc/mailname",
+            commands=[
+                f"echo {self.mail_domain} >/etc/mailname; chmod 644 /etc/mailname"
+            ],
+        )
+
+
+class FcgiwrapDeployer(Deployer):
+    def install(self):
+        apt.packages(
+            name="Install fcgiwrap",
+            packages=["fcgiwrap"],
+        )
+
+    def activate(self):
+        systemd.service(
+            name="Start and enable fcgiwrap",
+            service="fcgiwrap.service",
+            running=True,
+            enabled=True,
+        )
+
+
+class GithashDeployer(Deployer):
+    def activate(self):
+        try:
+            git_hash = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode()
+        except Exception:
+            git_hash = "unknown\n"
+        try:
+            git_diff = subprocess.check_output(["git", "diff"]).decode()
+        except Exception:
+            git_diff = ""
+        files.put(
+            name="Upload chatmail relay git commiit hash",
+            src=StringIO(git_hash + git_diff),
+            dest="/etc/chatmail-version",
+            mode="700",
         )
 
 
@@ -681,64 +1064,12 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
     check_config(config)
     mail_domain = config.mail_domain
 
-    from .www import build_webpages, find_merge_conflict, get_paths
-
-    server.group(name="Create vmail group", group="vmail", system=True)
-    server.user(name="Create vmail user", user="vmail", group="vmail", system=True)
-    server.group(name="Create opendkim group", group="opendkim", system=True)
-    server.user(
-        name="Create opendkim user",
-        user="opendkim",
-        groups=["opendkim"],
-        system=True,
-    )
-    server.user(
-        name="Add postfix user to opendkim group for socket access",
-        user="postfix",
-        groups=["opendkim"],
-        system=True,
-    )
-    server.user(name="Create echobot user", user="echobot", system=True)
-    server.user(name="Create iroh user", user="iroh", system=True)
-
-    # Add our OBS repository for dovecot_no_delay
-    files.put(
-        name="Add Deltachat OBS GPG key to apt keyring",
-        src=importlib.resources.files(__package__).joinpath("obs-home-deltachat.gpg"),
-        dest="/etc/apt/keyrings/obs-home-deltachat.gpg",
-        user="root",
-        group="root",
-        mode="644",
-    )
-
-    files.line(
-        name="Add DeltaChat OBS home repository to sources.list",
-        path="/etc/apt/sources.list",
-        line="deb [signed-by=/etc/apt/keyrings/obs-home-deltachat.gpg] https://download.opensuse.org/repositories/home:/deltachat/Debian_12/ ./",
-        escape_regex_characters=True,
-        present=False,
-    )
-
     if host.get_fact(Port, port=53) != "unbound":
         files.line(
             name="Add 9.9.9.9 to resolv.conf",
             path="/etc/resolv.conf",
             line="nameserver 9.9.9.9",
         )
-    apt.update(name="apt update", cache_time=24 * 3600)
-    apt.upgrade(name="upgrade apt packages", auto_remove=True)
-
-    apt.packages(
-        name="Install rsync",
-        packages=["rsync"],
-    )
-
-    deploy_turn_server(config)
-
-    # Run local DNS resolver `unbound`.
-    # `resolvconf` takes care of setting up /etc/resolv.conf
-    # to use 127.0.0.1 as the resolver.
-    from cmdeploy.cmdeploy import Out
 
     port_services = [
         (["master", "smtpd"], 25),
@@ -766,176 +1097,38 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
                 )
                 exit(1)
 
-    apt.packages(
-        name="Install unbound",
-        packages=["unbound", "unbound-anchor", "dnsutils"],
-    )
-    server.shell(
-        name="Generate root keys for validating DNSSEC",
-        commands=[
-            "unbound-anchor -a /var/lib/unbound/root.key || true",
-            "systemctl reset-failed unbound.service",
-        ],
-    )
-    systemd.service(
-        name="Start and enable unbound",
-        service="unbound.service",
-        running=True,
-        enabled=True,
-    )
-
-    deploy_iroh_relay(config)
-
-    # Deploy acmetool to have TLS certificates.
     tls_domains = [mail_domain, f"mta-sts.{mail_domain}", f"www.{mail_domain}"]
-    deploy_acmetool(
-        email=config.acme_email,
-        domains=tls_domains,
-    )
 
-    apt.packages(
+    all_deployers = [
-        # required for setfacl for echobot
+        ChatmailDeployer(mail_domain),
-        name="Install acl",
+        JournaldDeployer(),
-        packages="acl",
+        UnboundDeployer(),
-    )
+        TurnDeployer(mail_domain),
+        IrohDeployer(config.enable_iroh_relay),
+        AcmetoolDeployer(config.acme_email, tls_domains),
 
-    apt.packages(
+        WebsiteDeployer(config),
-        name="Install Postfix",
+        ChatmailVenvDeployer(config),
-        packages="postfix",
+        MtastsDeployer(),
-    )
+        OpendkimDeployer(mail_domain),
-
-    if not "dovecot.service" in host.get_fact(SystemdEnabled):
-        _install_dovecot_package("core", host.get_fact(facts.server.Arch))
-        _install_dovecot_package("imapd", host.get_fact(facts.server.Arch))
-        _install_dovecot_package("lmtpd", host.get_fact(facts.server.Arch))
-
-    apt.packages(
-        name="Install nginx",
-        packages=["nginx", "libnginx-mod-stream"],
-    )
-
-    apt.packages(
-        name="Install fcgiwrap",
-        packages=["fcgiwrap"],
-    )
-
-    www_path, src_dir, build_dir = get_paths(config)
-    # if www_folder was set to a non-existing folder, skip upload
-    if not www_path.is_dir():
-        logger.warning("Building web pages is disabled in chatmail.ini, skipping")
-    elif (path := find_merge_conflict(src_dir)) is not None:
-        logger.warning(f"Merge conflict found in {path}, skipping website deployment. Fix merge conflict if you want to upload your web page.")
-    else:
-        # if www_folder is a hugo page, build it
-        if build_dir:
-            www_path = build_webpages(src_dir, build_dir, config)
-        # if it is not a hugo page, upload it as is
-        files.rsync(f"{www_path}/", "/var/www/html", flags=["-avz", "--chown=www-data"])
-
-    _install_remote_venv_with_chatmaild(config)
-    debug = False
-    dovecot_need_restart = _configure_dovecot(config, debug=debug)
-    postfix_need_restart = _configure_postfix(config, debug=debug)
-    nginx_need_restart = _configure_nginx(config)
-    _uninstall_mta_sts_daemon()
-
-    _remove_rspamd()
-    opendkim_need_restart = _configure_opendkim(mail_domain, "opendkim")
-
-    systemd.service(
-        name="Start and enable OpenDKIM",
-        service="opendkim.service",
-        running=True,
-        enabled=True,
-        daemon_reload=opendkim_need_restart,
-        restarted=opendkim_need_restart,
-    )
 
         # Dovecot should be started before Postfix
         # because it creates authentication socket
         # required by Postfix.
-    systemd.service(
+        DovecotDeployer(config, disable_mail),
-        name="disable dovecot for now" if disable_mail else "Start and enable Dovecot",
+        PostfixDeployer(config, disable_mail),
-        service="dovecot.service",
+        FcgiwrapDeployer(),
-        running=False if disable_mail else True,
+        NginxDeployer(config),
-        enabled=False if disable_mail else True,
+        RspamdDeployer(),
-        restarted=dovecot_need_restart if not disable_mail else False,
+        EchobotDeployer(mail_domain),
-    )
+        MtailDeployer(config.mtail_address),
+        GithashDeployer(),
+    ]
 
-    systemd.service(
+    Deployment().perform_stages(all_deployers)
-        name="disable postfix for now" if disable_mail else "Start and enable Postfix",
-        service="postfix.service",
-        running=False if disable_mail else True,
-        enabled=False if disable_mail else True,
-        restarted=postfix_need_restart if not disable_mail else False,
-    )
 
-    systemd.service(
-        name="Start and enable nginx",
-        service="nginx.service",
-        running=True,
-        enabled=True,
-        restarted=nginx_need_restart,
-    )
-
-    systemd.service(
-        name="Start and enable fcgiwrap",
-        service="fcgiwrap.service",
-        running=True,
-        enabled=True,
-    )
-
-    systemd.service(
-        name="Restart echobot if postfix and dovecot were just started",
-        service="echobot.service",
-        restarted=postfix_need_restart and dovecot_need_restart,
-    )
-
-    # This file is used by auth proxy.
-    # https://wiki.debian.org/EtcMailName
-    server.shell(
-        name="Setup /etc/mailname",
-        commands=[f"echo {mail_domain} >/etc/mailname; chmod 644 /etc/mailname"],
-    )
-
-    journald_conf = files.put(
-        name="Configure journald",
-        src=importlib.resources.files(__package__).joinpath("journald.conf"),
-        dest="/etc/systemd/journald.conf",
-        user="root",
-        group="root",
-        mode="644",
-    )
-    systemd.service(
-        name="Start and enable journald",
-        service="systemd-journald.service",
-        running=True,
-        enabled=True,
-        restarted=journald_conf.changed,
-    )
     files.directory(
         name="Ensure old logs on disk are deleted",
         path="/var/log/journal/",
         present=False,
     )
-
-    apt.packages(
-        name="Ensure cron is installed",
-        packages=["cron"],
-    )
-    try:
-        git_hash = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode()
-    except Exception:
-        git_hash = "unknown\n"
-    try:
-        git_diff = subprocess.check_output(["git", "diff"]).decode()
-    except Exception:
-        git_diff = ""
-    files.put(
-        name="Upload chatmail relay git commiit hash",
-        src=StringIO(git_hash + git_diff),
-        dest="/etc/chatmail-version",
-        mode="700",
-    )
-
-    deploy_mtail(config)

cmdeploy/src/cmdeploy/acmetool/__init__.py

@@ -2,9 +2,18 @@ import importlib.resources
 
 from pyinfra.operations import apt, files, server, systemd
 
+from ..deployer import Deployer
 
-def deploy_acmetool(email="", domains=[]):
+
-    """Deploy acmetool."""
+class AcmetoolDeployer(Deployer):
+    def __init__(self, email, domains):
+        self.domains = domains
+        self.email = email
+        self.need_restart_redirector = False
+        self.need_restart_reconcile_service = False
+        self.need_restart_reconcile_timer = False
+
+    def install(self):
         apt.packages(
             name="Install acmetool",
             packages=["acmetool"],
@@ -30,13 +39,14 @@ def deploy_acmetool(email="", domains=[]):
             present=False,
         )
 
+    def configure(self):
         files.template(
             src=importlib.resources.files(__package__).joinpath("response-file.yaml.j2"),
             dest="/var/lib/acme/conf/responses",
             user="root",
             group="root",
             mode="644",
-        email=email,
+            email=self.email,
         )
 
         files.template(
@@ -56,14 +66,7 @@ def deploy_acmetool(email="", domains=[]):
             group="root",
             mode="644",
         )
-
+        self.need_restart_redirector = service_file.changed
-    systemd.service(
-        name="Setup acmetool-redirector service",
-        service="acmetool-redirector.service",
-        running=True,
-        enabled=True,
-        restarted=service_file.changed,
-    )
 
         reconcile_service_file = files.put(
             src=importlib.resources.files(__package__).joinpath(
@@ -74,14 +77,7 @@ def deploy_acmetool(email="", domains=[]):
             group="root",
             mode="644",
         )
-
+        self.need_restart_reconcile_service = reconcile_service_file.changed
-    systemd.service(
-        name="Setup acmetool-reconcile service",
-        service="acmetool-reconcile.service",
-        running=False,
-        enabled=False,
-        daemon_reload=reconcile_service_file.changed,
-    )
 
         reconcile_timer_file = files.put(
             src=importlib.resources.files(__package__).joinpath("acmetool-reconcile.timer"),
@@ -90,16 +86,37 @@ def deploy_acmetool(email="", domains=[]):
             group="root",
             mode="644",
         )
+        self.need_restart_reconcile_timer = reconcile_timer_file.changed
+
+    def activate(self):
+        systemd.service(
+            name="Setup acmetool-redirector service",
+            service="acmetool-redirector.service",
+            running=True,
+            enabled=True,
+            restarted=self.need_restart_redirector,
+        )
+        self.need_restart_redirector = False
+
+        systemd.service(
+            name="Setup acmetool-reconcile service",
+            service="acmetool-reconcile.service",
+            running=False,
+            enabled=False,
+            daemon_reload=self.need_restart_reconcile_service,
+        )
+        self.need_restart_reconcile_service = False
 
         systemd.service(
             name="Setup acmetool-reconcile timer",
             service="acmetool-reconcile.timer",
             running=True,
             enabled=True,
-        daemon_reload=reconcile_timer_file.changed,
+            daemon_reload=self.need_restart_reconcile_timer,
         )
+        self.need_restart_reconcile_timer = False
 
         server.shell(
-        name=f"Request certificate for: {', '.join(domains)}",
+            name=f"Request certificate for: {', '.join(self.domains)}",
-        commands=[f"acmetool want --xlog.severity=debug {' '.join(domains)}"],
+            commands=[f"acmetool want --xlog.severity=debug {' '.join(self.domains)}"],
         )

cmdeploy/src/cmdeploy/deployer.py

@@ -0,0 +1,57 @@
+import os
+
+from pyinfra.operations import server
+
+
+class Deployment:
+    def install(self, deployer):
+        # optional 'required_users' contains a list of (user, group, secondary-group-list) tuples.
+        # If the group is None, no group is created corresponding to that user.
+        # If the secondary group list is not None, all listed groups are created as well.
+        required_users = getattr(deployer, "required_users", [])
+        for user, group, groups in required_users:
+            if group is not None:
+                server.group(
+                    name="Create {} group".format(group), group=group, system=True
+                )
+            if groups is not None:
+                for group2 in groups:
+                    server.group(
+                        name="Create {} group".format(group2), group=group2, system=True
+                    )
+            server.user(
+                name="Create {} user".format(user),
+                user=user,
+                group=group,
+                groups=groups,
+                system=True,
+            )
+
+        deployer.install()
+
+    def configure(self, deployer):
+        deployer.configure()
+
+    def activate(self, deployer):
+        deployer.activate()
+
+    def perform_stages(self, deployers):
+        default_stages = "install,configure,activate"
+        stages = os.getenv("CMDEPLOY_STAGES", default_stages).split(",")
+
+        for stage in stages:
+            for deployer in deployers:
+                getattr(self, stage)(deployer)
+
+
+class Deployer:
+    need_restart = False
+
+    def install(self):
+        pass
+
+    def configure(self):
+        pass
+
+    def activate(self):
+        pass

cmdeploy/src/cmdeploy/policy-rc.d

@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "All runlevel operations denied by policy" >&2
+exit 101

doc/source/overview.rst

@@ -297,3 +297,48 @@ actually it is a problem with your TLS certificate.
 .. _nginx: https://nginx.org
 .. _pyinfra: https://pyinfra.com
 
+
+Architecture of cmdeploy
+------------------------
+
+cmdeploy is a Python program that uses the pyinfra library to deploy
+chatmail relays, with all the necessary software, configuration, and
+services.  The deployment process performs three primary types of
+operation:
+
+1. Installation of software, universal across all deployments.
+2. Configuration of software, with deploy-specific variations.
+3. Activation of services.
+
+The process is implemented through a family of "deployer" objects
+which all derive from a common ``Deployer`` base class, defined in
+cmdeploy/src/cmdeploy/deployer.py.  Each object provides
+implementation methods for the three stages -- install, configure, and
+activate.  The top-level procedure in ``deploy_chatmail()`` calls
+these methods for all the deployer objects, via the
+``Deployment.perform_stages()`` method, also defined in deployer.py.
+This first calls all the install methods, then the configure methods,
+then the activate methods.
+
+The ``Deployment`` class also implements support for a CMDEPLOY_STAGES
+environment variable, which allows limiting the process to specific
+stages.  Note that some deployers are stateful between the stages
+(this is one reason why they are implemented as objects), and that
+state will not get propagated between stages when run in separate
+invocations of cmdeploy.  This environment variable is intended for
+use in future revisions to support building Docker images with
+software pre-installed, and configuration of containers at run time
+from environment variables.
+
+The, ``install()`` methods for the deployer classes should use 'self'
+as little as possible, preferably not at all.  In particular,
+``install()`` methods should never depend on "config" data, such as
+the config dictionary in ``self.config`` or specific values like
+``self.mail_domain``.  This ensures that these methods can be used to
+perform generic installation operations that are applicable across
+multiple relay deployments, and therefore can be called in the process
+of building a general-purpose container image.
+
+Operations that start services for systemd-based deployments should
+only be called from the ``activate_impl()`` methods.  These methods
+will not be called in non-systemd container environments.