diff --git a/CHANGELOG.md b/CHANGELOG.md index 12c08ce5..3c64d2c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## untagged +- Organized cmdeploy into install, configure, and activate stages + ([#695](https://github.com/chatmail/relay/pull/695)) + - docs: move readme.md docs to sphinx documentation rendered at https://chatmail.at/doc/relay ([#711](https://github.com/chatmail/relay/pull/711)) diff --git a/cmdeploy/src/cmdeploy/__init__.py b/cmdeploy/src/cmdeploy/__init__.py index 50977905..fe88115c 100644 --- a/cmdeploy/src/cmdeploy/__init__.py +++ b/cmdeploy/src/cmdeploy/__init__.py @@ -11,6 +11,7 @@ from io import StringIO from pathlib import Path from chatmaild.config import Config, read_config +from cmdeploy.cmdeploy import Out from pyinfra import facts, host, logger from pyinfra.api import FactBase from pyinfra.facts.files import File, Sha256File @@ -18,7 +19,9 @@ from pyinfra.facts.server import Sysctl from pyinfra.facts.systemd import SystemdEnabled from pyinfra.operations import apt, files, pip, server, systemd -from .acmetool import deploy_acmetool +from .acmetool import AcmetoolDeployer +from .deployer import Deployer, Deployment +from .www import build_webpages, find_merge_conflict, get_paths class Port(FactBase): @@ -61,13 +64,12 @@ def remove_legacy_artifacts(): ) -def _install_remote_venv_with_chatmaild(config) -> None: +def _install_remote_venv_with_chatmaild() -> None: remove_legacy_artifacts() dist_file = _build_chatmaild(dist_dir=Path("chatmaild/dist")) remote_base_dir = "/usr/local/lib/chatmaild" remote_dist_file = f"{remote_base_dir}/dist/{dist_file.name}" remote_venv_dir = f"{remote_base_dir}/venv" - remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini" root_owned = dict(user="root", group="root", mode="644") apt.packages( @@ -83,13 +85,6 @@ def _install_remote_venv_with_chatmaild(config) -> None: **root_owned, ) - files.put( - name=f"Upload {remote_chatmail_inipath}", - src=config._getbytefile(), - dest=remote_chatmail_inipath, - **root_owned, - ) - pip.virtualenv( name=f"chatmaild virtualenv {remote_venv_dir}", path=remote_venv_dir, @@ -108,6 +103,20 @@ def _install_remote_venv_with_chatmaild(config) -> None: ], ) + +def _configure_remote_venv_with_chatmaild(config) -> None: + remote_base_dir = "/usr/local/lib/chatmaild" + remote_venv_dir = f"{remote_base_dir}/venv" + remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini" + root_owned = dict(user="root", group="root", mode="644") + + files.put( + name=f"Upload {remote_chatmail_inipath}", + src=config._getbytefile(), + dest=remote_chatmail_inipath, + **root_owned, + ) + files.template( src=importlib.resources.files(__package__).joinpath("metrics.cron.j2"), dest="/etc/cron.d/chatmail-metrics", @@ -120,26 +129,21 @@ def _install_remote_venv_with_chatmaild(config) -> None: }, ) + +def _configure_remote_units(mail_domain, units) -> None: + remote_base_dir = "/usr/local/lib/chatmaild" + remote_venv_dir = f"{remote_base_dir}/venv" + remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini" + root_owned = dict(user="root", group="root", mode="644") + # install systemd units - for fn in ( - "doveauth", - "filtermail", - "filtermail-incoming", - "echobot", - "chatmail-metadata", - "lastlogin", - "turnserver", - "chatmail-expire", - "chatmail-expire.timer", - "chatmail-fsreport", - "chatmail-fsreport.timer", - ): + for fn in units: execpath = fn if fn != "filtermail-incoming" else "filtermail" params = dict( execpath=f"{remote_venv_dir}/bin/{execpath}", config_path=remote_chatmail_inipath, remote_venv_dir=remote_venv_dir, - mail_domain=config.mail_domain, + mail_domain=mail_domain, ) basename = fn if "." in fn else f"{fn}.service" @@ -153,6 +157,13 @@ def _install_remote_venv_with_chatmaild(config) -> None: dest=f"/etc/systemd/system/{basename}", **root_owned, ) + + +def _activate_remote_units(units) -> None: + # activate systemd units + for fn in units: + basename = fn if "." in fn else f"{fn}.service" + if fn == "chatmail-expire" or fn == "chatmail-fsreport": # don't auto-start but let the corresponding timer trigger execution enabled = False @@ -238,11 +249,6 @@ def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool: present=True, ) - apt.packages( - name="apt install opendkim opendkim-tools", - packages=["opendkim", "opendkim-tools"], - ) - if not host.get_fact(File, f"/etc/dkimkeys/{dkim_selector}.private"): server.shell( name="Generate OpenDKIM domain keys", @@ -263,21 +269,102 @@ def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool: return need_restart -def _uninstall_mta_sts_daemon() -> None: - # Remove configuration. - files.file("/etc/mta-sts-daemon.yml", present=False) +class OpendkimDeployer(Deployer): + required_users = [("opendkim", None, ["opendkim"])] - files.directory("/usr/local/lib/postfix-mta-sts-resolver", present=False) + def __init__(self, mail_domain): + self.mail_domain = mail_domain - files.file("/etc/systemd/system/mta-sts-daemon.service", present=False) + def install(self): + apt.packages( + name="apt install opendkim opendkim-tools", + packages=["opendkim", "opendkim-tools"], + ) - systemd.service( - name="Stop MTA-STS daemon", - service="mta-sts-daemon.service", - daemon_reload=True, - running=False, - enabled=False, - ) + def configure(self): + self.need_restart = _configure_opendkim(self.mail_domain, "opendkim") + + def activate(self): + systemd.service( + name="Start and enable OpenDKIM", + service="opendkim.service", + running=True, + enabled=True, + daemon_reload=self.need_restart, + restarted=self.need_restart, + ) + self.need_restart = False + + +class UnboundDeployer(Deployer): + def install(self): + # Run local DNS resolver `unbound`. + # `resolvconf` takes care of setting up /etc/resolv.conf + # to use 127.0.0.1 as the resolver. + + # + # On an IPv4-only system, if unbound is started but not + # configured, it causes subsequent steps to fail to resolve hosts. + # Here, we use policy-rc.d to prevent unbound from starting up + # on initial install. Later, we will configure it and start it. + # + # For documentation about policy-rc.d, see: + # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt + # + files.put( + src=importlib.resources.files(__package__).joinpath("policy-rc.d"), + dest="/usr/sbin/policy-rc.d", + user="root", + group="root", + mode="755", + ) + + apt.packages( + name="Install unbound", + packages=["unbound", "unbound-anchor", "dnsutils"], + ) + + files.file("/usr/sbin/policy-rc.d", present=False) + + def configure(self): + server.shell( + name="Generate root keys for validating DNSSEC", + commands=[ + "unbound-anchor -a /var/lib/unbound/root.key || true", + ], + ) + + def activate(self): + server.shell( + name="Generate root keys for validating DNSSEC", + commands=[ + "systemctl reset-failed unbound.service", + ], + ) + + systemd.service( + name="Start and enable unbound", + service="unbound.service", + running=True, + enabled=True, + ) + + +class MtastsDeployer(Deployer): + def configure(self): + # Remove configuration. + files.file("/etc/mta-sts-daemon.yml", present=False) + files.directory("/usr/local/lib/postfix-mta-sts-resolver", present=False) + files.file("/etc/systemd/system/mta-sts-daemon.service", present=False) + + def activate(self): + systemd.service( + name="Stop MTA-STS daemon", + service="mta-sts-daemon.service", + daemon_reload=True, + running=False, + enabled=False, + ) def _configure_postfix(config: Config, debug: bool = False) -> bool: @@ -330,6 +417,35 @@ def _configure_postfix(config: Config, debug: bool = False) -> bool: return need_restart +class PostfixDeployer(Deployer): + required_users = [("postfix", None, ["opendkim"])] + + def __init__(self, config, disable_mail): + self.config = config + self.disable_mail = disable_mail + + def install(self): + apt.packages( + name="Install Postfix", + packages="postfix", + ) + + def configure(self): + self.need_restart = _configure_postfix(self.config) + + def activate(self): + restart = False if self.disable_mail else self.need_restart + + systemd.service( + name="disable postfix for now" if self.disable_mail else "Start and enable Postfix", + service="postfix.service", + running=False if self.disable_mail else True, + enabled=False if self.disable_mail else True, + restarted=restart, + ) + self.need_restart = False + + def _install_dovecot_package(package: str, arch: str): arch = "amd64" if arch == "x86_64" else arch arch = "arm64" if arch == "aarch64" else arch @@ -430,6 +546,38 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool: return need_restart +class DovecotDeployer(Deployer): + def __init__(self, config, disable_mail): + self.config = config + self.disable_mail = disable_mail + self.units = ["doveauth"] + + def install(self): + arch = host.get_fact(facts.server.Arch) + if not "dovecot.service" in host.get_fact(SystemdEnabled): + _install_dovecot_package("core", arch) + _install_dovecot_package("imapd", arch) + _install_dovecot_package("lmtpd", arch) + + def configure(self): + _configure_remote_units(self.config.mail_domain, self.units) + self.need_restart = _configure_dovecot(self.config) + + def activate(self): + _activate_remote_units(self.units) + + restart = False if self.disable_mail else self.need_restart + + systemd.service( + name="disable dovecot for now" if self.disable_mail else "Start and enable Dovecot", + service="dovecot.service", + running=False if self.disable_mail else True, + enabled=False if self.disable_mail else True, + restarted=restart, + ) + self.need_restart = False + + def _configure_nginx(config: Config, debug: bool = False) -> bool: """Configures nginx HTTP server.""" need_restart = False @@ -487,9 +635,93 @@ def _configure_nginx(config: Config, debug: bool = False) -> bool: return need_restart -def _remove_rspamd() -> None: - """Remove rspamd""" - apt.packages(name="Remove rspamd", packages="rspamd", present=False) +class NginxDeployer(Deployer): + def __init__(self, config): + self.config = config + + def install(self): + # + # If we allow nginx to start up on install, it will grab port + # 80, which then will block acmetool from listening on the port. + # That in turn prevents getting certificates, which then causes + # an error when we try to start nginx on the custom config + # that leaves port 80 open but also requires certificates to + # be present. To avoid getting into that interlocking mess, + # we use policy-rc.d to prevent nginx from starting up when it + # is installed. + # + # This approach allows us to avoid performing any explicit + # systemd operations during the install stage (as opposed to + # allowing it to start and then forcing it to stop), which allows + # the install stage to run in non-systemd environments like a + # container image build. + # + # For documentation about policy-rc.d, see: + # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt + # + files.put( + src=importlib.resources.files(__package__).joinpath("policy-rc.d"), + dest="/usr/sbin/policy-rc.d", + user="root", + group="root", + mode="755", + ) + + apt.packages( + name="Install nginx", + packages=["nginx", "libnginx-mod-stream"], + ) + + files.file("/usr/sbin/policy-rc.d", present=False) + + def configure(self): + self.need_restart = _configure_nginx(self.config) + + def activate(self): + systemd.service( + name="Start and enable nginx", + service="nginx.service", + running=True, + enabled=True, + restarted=self.need_restart, + ) + self.need_restart = False + + +class WebsiteDeployer(Deployer): + def __init__(self, config): + self.config = config + + def install(self): + files.directory( + name="Ensure /var/www exists", + path="/var/www", + user="root", + group="root", + mode="755", + present=True, + ) + + def configure(self): + www_path, src_dir, build_dir = get_paths(self.config) + # if www_folder was set to a non-existing folder, skip upload + if not www_path.is_dir(): + logger.warning("Building web pages is disabled in chatmail.ini, skipping") + elif (path := find_merge_conflict(src_dir)) is not None: + logger.warning(f"Merge conflict found in {path}, skipping website deployment. Fix merge conflict if you want to upload your web page.") + else: + # if www_folder is a hugo page, build it + if build_dir: + www_path = build_webpages(src_dir, build_dir, self.config) + # if it is not a hugo page, upload it as is + files.rsync( + f"{www_path}/", "/var/www/html", flags=["-avz", "--chown=www-data"] + ) + + +class RspamdDeployer(Deployer): + def install(self): + apt.packages(name="Remove rspamd", packages="rspamd", present=False) def check_config(config): @@ -507,168 +739,319 @@ def check_config(config): return config -def deploy_turn_server(config): - (url, sha256sum) = { - "x86_64": ( - "https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-x86_64-linux", - "841e527c15fdc2940b0469e206188ea8f0af48533be12ecb8098520f813d41e4", - ), - "aarch64": ( - "https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-aarch64-linux", - "a5fc2d06d937b56a34e098d2cd72a82d3e89967518d159bf246dc69b65e81b42", - ), - }[host.get_fact(facts.server.Arch)] +class TurnDeployer(Deployer): + def __init__(self, mail_domain): + self.mail_domain = mail_domain + self.units = ["turnserver"] - need_restart = False + def install(self): + (url, sha256sum) = { + "x86_64": ( + "https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-x86_64-linux", + "841e527c15fdc2940b0469e206188ea8f0af48533be12ecb8098520f813d41e4", + ), + "aarch64": ( + "https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-aarch64-linux", + "a5fc2d06d937b56a34e098d2cd72a82d3e89967518d159bf246dc69b65e81b42", + ), + }[host.get_fact(facts.server.Arch)] + + existing_sha256sum = host.get_fact(Sha256File, "/usr/local/bin/chatmail-turn") + if existing_sha256sum != sha256sum: + server.shell( + name="Download chatmail-turn", + commands=[ + f"(curl -L {url} >/usr/local/bin/chatmail-turn.new && (echo '{sha256sum} /usr/local/bin/chatmail-turn.new' | sha256sum -c) && mv /usr/local/bin/chatmail-turn.new /usr/local/bin/chatmail-turn)", + "chmod 755 /usr/local/bin/chatmail-turn", + ], + ) + + def configure(self): + _configure_remote_units(self.mail_domain, self.units) + + def activate(self): + _activate_remote_units(self.units) + + +class MtailDeployer(Deployer): + def __init__(self, mtail_address): + self.mtail_address = mtail_address + + def install(self): + # Uninstall mtail package, we are going to install a static binary. + apt.packages(name="Uninstall mtail", packages=["mtail"], present=False) + + (url, sha256sum) = { + "x86_64": ( + "https://github.com/google/mtail/releases/download/v3.0.8/mtail_3.0.8_linux_amd64.tar.gz", + "123c2ee5f48c3eff12ebccee38befd2233d715da736000ccde49e3d5607724e4", + ), + "aarch64": ( + "https://github.com/google/mtail/releases/download/v3.0.8/mtail_3.0.8_linux_arm64.tar.gz", + "aa04811c0929b6754408676de520e050c45dddeb3401881888a092c9aea89cae", + ), + }[host.get_fact(facts.server.Arch)] - existing_sha256sum = host.get_fact(Sha256File, "/usr/local/bin/chatmail-turn") - if existing_sha256sum != sha256sum: server.shell( - name="Download chatmail-turn", + name="Download mtail", commands=[ - f"(curl -L {url} >/usr/local/bin/chatmail-turn.new && (echo '{sha256sum} /usr/local/bin/chatmail-turn.new' | sha256sum -c) && mv /usr/local/bin/chatmail-turn.new /usr/local/bin/chatmail-turn)", - "chmod 755 /usr/local/bin/chatmail-turn", + f"(echo '{sha256sum} /usr/local/bin/mtail' | sha256sum -c) || (curl -L {url} | gunzip | tar -x -f - mtail -O >/usr/local/bin/mtail.new && mv /usr/local/bin/mtail.new /usr/local/bin/mtail)", + "chmod 755 /usr/local/bin/mtail", ], ) - need_restart = True - source_path = importlib.resources.files(__package__).joinpath( - "service", "turnserver.service.f" - ) - content = source_path.read_text().format(mail_domain=config.mail_domain).encode() + def configure(self): + # Using our own systemd unit instead of `/usr/lib/systemd/system/mtail.service`. + # This allows to read from journalctl instead of log files. + files.template( + src=importlib.resources.files(__package__).joinpath( + "mtail/mtail.service.j2" + ), + dest="/etc/systemd/system/mtail.service", + user="root", + group="root", + mode="644", + address=self.mtail_address or "127.0.0.1", + port=3903, + ) - systemd_unit = files.put( - name="Upload turnserver.service", - src=io.BytesIO(content), - dest="/etc/systemd/system/turnserver.service", - user="root", - group="root", - mode="644", - ) - need_restart |= systemd_unit.changed + mtail_conf = files.put( + name="Mtail configuration", + src=importlib.resources.files(__package__).joinpath( + "mtail/delivered_mail.mtail" + ), + dest="/etc/mtail/delivered_mail.mtail", + user="root", + group="root", + mode="644", + ) + self.need_restart = mtail_conf.changed - systemd.service( - name="Setup turnserver service", - service="turnserver.service", - running=True, - enabled=True, - restarted=need_restart, - daemon_reload=systemd_unit.changed, - ) + def activate(self): + systemd.service( + name="Start and enable mtail", + service="mtail.service", + running=bool(self.mtail_address), + enabled=bool(self.mtail_address), + restarted=self.need_restart, + ) + self.need_restart = False -def deploy_mtail(config): - # Uninstall mtail package, we are going to install a static binary. - apt.packages(name="Uninstall mtail", packages=["mtail"], present=False) +class IrohDeployer(Deployer): + def __init__(self, enable_iroh_relay): + self.enable_iroh_relay = enable_iroh_relay - (url, sha256sum) = { - "x86_64": ( - "https://github.com/google/mtail/releases/download/v3.0.8/mtail_3.0.8_linux_amd64.tar.gz", - "123c2ee5f48c3eff12ebccee38befd2233d715da736000ccde49e3d5607724e4", - ), - "aarch64": ( - "https://github.com/google/mtail/releases/download/v3.0.8/mtail_3.0.8_linux_arm64.tar.gz", - "aa04811c0929b6754408676de520e050c45dddeb3401881888a092c9aea89cae", - ), - }[host.get_fact(facts.server.Arch)] + def install(self): + (url, sha256sum) = { + "x86_64": ( + "https://github.com/n0-computer/iroh/releases/download/v0.35.0/iroh-relay-v0.35.0-x86_64-unknown-linux-musl.tar.gz", + "45c81199dbd70f8c4c30fef7f3b9727ca6e3cea8f2831333eeaf8aa71bf0fac1", + ), + "aarch64": ( + "https://github.com/n0-computer/iroh/releases/download/v0.35.0/iroh-relay-v0.35.0-aarch64-unknown-linux-musl.tar.gz", + "f8ef27631fac213b3ef668d02acd5b3e215292746a3fc71d90c63115446008b1", + ), + }[host.get_fact(facts.server.Arch)] - server.shell( - name="Download mtail", - commands=[ - f"(echo '{sha256sum} /usr/local/bin/mtail' | sha256sum -c) || (curl -L {url} | gunzip | tar -x -f - mtail -O >/usr/local/bin/mtail.new && mv /usr/local/bin/mtail.new /usr/local/bin/mtail)", - "chmod 755 /usr/local/bin/mtail", - ], - ) + existing_sha256sum = host.get_fact(Sha256File, "/usr/local/bin/iroh-relay") + if existing_sha256sum != sha256sum: + server.shell( + name="Download iroh-relay", + commands=[ + f"(curl -L {url} | gunzip | tar -x -f - ./iroh-relay -O >/usr/local/bin/iroh-relay.new && (echo '{sha256sum} /usr/local/bin/iroh-relay.new' | sha256sum -c) && mv /usr/local/bin/iroh-relay.new /usr/local/bin/iroh-relay)", + "chmod 755 /usr/local/bin/iroh-relay", + ], + ) - # Using our own systemd unit instead of `/usr/lib/systemd/system/mtail.service`. - # This allows to read from journalctl instead of log files. - files.template( - src=importlib.resources.files(__package__).joinpath("mtail/mtail.service.j2"), - dest="/etc/systemd/system/mtail.service", - user="root", - group="root", - mode="644", - address=config.mtail_address or "127.0.0.1", - port=3903, - ) + self.need_restart = True - mtail_conf = files.put( - name="Mtail configuration", - src=importlib.resources.files(__package__).joinpath( - "mtail/delivered_mail.mtail" - ), - dest="/etc/mtail/delivered_mail.mtail", - user="root", - group="root", - mode="644", - ) + def configure(self): + systemd_unit = files.put( + name="Upload iroh-relay systemd unit", + src=importlib.resources.files(__package__).joinpath("iroh-relay.service"), + dest="/etc/systemd/system/iroh-relay.service", + user="root", + group="root", + mode="644", + ) + self.need_restart |= systemd_unit.changed - systemd.service( - name="Start and enable mtail", - service="mtail.service", - running=bool(config.mtail_address), - enabled=bool(config.mtail_address), - restarted=mtail_conf.changed, - ) + iroh_config = files.put( + name="Upload iroh-relay config", + src=importlib.resources.files(__package__).joinpath("iroh-relay.toml"), + dest="/etc/iroh-relay.toml", + user="root", + group="root", + mode="644", + ) + self.need_restart |= iroh_config.changed + + def activate(self): + systemd.service( + name="Start and enable iroh-relay", + service="iroh-relay.service", + running=True, + enabled=self.enable_iroh_relay, + restarted=self.need_restart, + ) + self.need_restart = False -def deploy_iroh_relay(config) -> None: - (url, sha256sum) = { - "x86_64": ( - "https://github.com/n0-computer/iroh/releases/download/v0.35.0/iroh-relay-v0.35.0-x86_64-unknown-linux-musl.tar.gz", - "45c81199dbd70f8c4c30fef7f3b9727ca6e3cea8f2831333eeaf8aa71bf0fac1", - ), - "aarch64": ( - "https://github.com/n0-computer/iroh/releases/download/v0.35.0/iroh-relay-v0.35.0-aarch64-unknown-linux-musl.tar.gz", - "f8ef27631fac213b3ef668d02acd5b3e215292746a3fc71d90c63115446008b1", - ), - }[host.get_fact(facts.server.Arch)] +class JournaldDeployer(Deployer): + def configure(self): + journald_conf = files.put( + name="Configure journald", + src=importlib.resources.files(__package__).joinpath("journald.conf"), + dest="/etc/systemd/journald.conf", + user="root", + group="root", + mode="644", + ) + self.need_restart = journald_conf.changed - apt.packages( - name="Install curl", - packages=["curl"], - ) + def activate(self): + systemd.service( + name="Start and enable journald", + service="systemd-journald.service", + running=True, + enabled=True, + restarted=self.need_restart, + ) + self.need_restart = False - need_restart = False - existing_sha256sum = host.get_fact(Sha256File, "/usr/local/bin/iroh-relay") - if existing_sha256sum != sha256sum: +class EchobotDeployer(Deployer): + # + # This deployer depends on the dovecot and postfix deployers because + # it needs to base its decision of whether to restart the service on + # whether those two services were restarted. + # + def __init__(self, mail_domain): + self.mail_domain = mail_domain + self.units = ["echobot"] + + def install(self): + apt.packages( + # required for setfacl for echobot + name="Install acl", + packages="acl", + ) + + def configure(self): + _configure_remote_units(self.mail_domain, self.units) + + def activate(self): + _activate_remote_units(self.units) + + +class ChatmailVenvDeployer(Deployer): + def __init__(self, config): + self.config = config + self.units = ( + "filtermail", + "filtermail-incoming", + "chatmail-metadata", + "lastlogin", + "chatmail-expire", + "chatmail-expire.timer", + "chatmail-fsreport", + "chatmail-fsreport.timer", + ) + + def install(self): + _install_remote_venv_with_chatmaild() + + def configure(self): + _configure_remote_venv_with_chatmaild(self.config) + _configure_remote_units(self.config.mail_domain, self.units) + + def activate(self): + _activate_remote_units(self.units) + + +class ChatmailDeployer(Deployer): + required_users = [ + ("vmail", "vmail", None), + ("echobot", None, None), + ("iroh", None, None), + ] + + def __init__(self, mail_domain): + self.mail_domain = mail_domain + + def install(self): + # Remove OBS repository key that is no longer used. + files.file("/etc/apt/keyrings/obs-home-deltachat.gpg", present=False) + + files.line( + name="Remove DeltaChat OBS home repository from sources.list", + path="/etc/apt/sources.list", + line="deb [signed-by=/etc/apt/keyrings/obs-home-deltachat.gpg] https://download.opensuse.org/repositories/home:/deltachat/Debian_12/ ./", + escape_regex_characters=True, + present=False, + ) + + apt.update(name="apt update", cache_time=24 * 3600) + apt.upgrade(name="upgrade apt packages", auto_remove=True) + + apt.packages( + name="Install curl", + packages=["curl"], + ) + + apt.packages( + name="Install rsync", + packages=["rsync"], + ) + apt.packages( + name="Ensure cron is installed", + packages=["cron"], + ) + + def configure(self): + # This file is used by auth proxy. + # https://wiki.debian.org/EtcMailName server.shell( - name="Download iroh-relay", + name="Setup /etc/mailname", commands=[ - f"(curl -L {url} | gunzip | tar -x -f - ./iroh-relay -O >/usr/local/bin/iroh-relay.new && (echo '{sha256sum} /usr/local/bin/iroh-relay.new' | sha256sum -c) && mv /usr/local/bin/iroh-relay.new /usr/local/bin/iroh-relay)", - "chmod 755 /usr/local/bin/iroh-relay", + f"echo {self.mail_domain} >/etc/mailname; chmod 644 /etc/mailname" ], ) - need_restart = True - systemd_unit = files.put( - name="Upload iroh-relay systemd unit", - src=importlib.resources.files(__package__).joinpath("iroh-relay.service"), - dest="/etc/systemd/system/iroh-relay.service", - user="root", - group="root", - mode="644", - ) - need_restart |= systemd_unit.changed - iroh_config = files.put( - name="Upload iroh-relay config", - src=importlib.resources.files(__package__).joinpath("iroh-relay.toml"), - dest="/etc/iroh-relay.toml", - user="root", - group="root", - mode="644", - ) - need_restart |= iroh_config.changed +class FcgiwrapDeployer(Deployer): + def install(self): + apt.packages( + name="Install fcgiwrap", + packages=["fcgiwrap"], + ) - systemd.service( - name="Start and enable iroh-relay", - service="iroh-relay.service", - running=True, - enabled=config.enable_iroh_relay, - restarted=need_restart, - ) + def activate(self): + systemd.service( + name="Start and enable fcgiwrap", + service="fcgiwrap.service", + running=True, + enabled=True, + ) + + +class GithashDeployer(Deployer): + def activate(self): + try: + git_hash = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode() + except Exception: + git_hash = "unknown\n" + try: + git_diff = subprocess.check_output(["git", "diff"]).decode() + except Exception: + git_diff = "" + files.put( + name="Upload chatmail relay git commiit hash", + src=StringIO(git_hash + git_diff), + dest="/etc/chatmail-version", + mode="700", + ) def deploy_chatmail(config_path: Path, disable_mail: bool) -> None: @@ -681,64 +1064,12 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None: check_config(config) mail_domain = config.mail_domain - from .www import build_webpages, find_merge_conflict, get_paths - - server.group(name="Create vmail group", group="vmail", system=True) - server.user(name="Create vmail user", user="vmail", group="vmail", system=True) - server.group(name="Create opendkim group", group="opendkim", system=True) - server.user( - name="Create opendkim user", - user="opendkim", - groups=["opendkim"], - system=True, - ) - server.user( - name="Add postfix user to opendkim group for socket access", - user="postfix", - groups=["opendkim"], - system=True, - ) - server.user(name="Create echobot user", user="echobot", system=True) - server.user(name="Create iroh user", user="iroh", system=True) - - # Add our OBS repository for dovecot_no_delay - files.put( - name="Add Deltachat OBS GPG key to apt keyring", - src=importlib.resources.files(__package__).joinpath("obs-home-deltachat.gpg"), - dest="/etc/apt/keyrings/obs-home-deltachat.gpg", - user="root", - group="root", - mode="644", - ) - - files.line( - name="Add DeltaChat OBS home repository to sources.list", - path="/etc/apt/sources.list", - line="deb [signed-by=/etc/apt/keyrings/obs-home-deltachat.gpg] https://download.opensuse.org/repositories/home:/deltachat/Debian_12/ ./", - escape_regex_characters=True, - present=False, - ) - if host.get_fact(Port, port=53) != "unbound": files.line( name="Add 9.9.9.9 to resolv.conf", path="/etc/resolv.conf", line="nameserver 9.9.9.9", ) - apt.update(name="apt update", cache_time=24 * 3600) - apt.upgrade(name="upgrade apt packages", auto_remove=True) - - apt.packages( - name="Install rsync", - packages=["rsync"], - ) - - deploy_turn_server(config) - - # Run local DNS resolver `unbound`. - # `resolvconf` takes care of setting up /etc/resolv.conf - # to use 127.0.0.1 as the resolver. - from cmdeploy.cmdeploy import Out port_services = [ (["master", "smtpd"], 25), @@ -766,176 +1097,38 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None: ) exit(1) - apt.packages( - name="Install unbound", - packages=["unbound", "unbound-anchor", "dnsutils"], - ) - server.shell( - name="Generate root keys for validating DNSSEC", - commands=[ - "unbound-anchor -a /var/lib/unbound/root.key || true", - "systemctl reset-failed unbound.service", - ], - ) - systemd.service( - name="Start and enable unbound", - service="unbound.service", - running=True, - enabled=True, - ) - - deploy_iroh_relay(config) - - # Deploy acmetool to have TLS certificates. tls_domains = [mail_domain, f"mta-sts.{mail_domain}", f"www.{mail_domain}"] - deploy_acmetool( - email=config.acme_email, - domains=tls_domains, - ) - apt.packages( - # required for setfacl for echobot - name="Install acl", - packages="acl", - ) + all_deployers = [ + ChatmailDeployer(mail_domain), + JournaldDeployer(), + UnboundDeployer(), + TurnDeployer(mail_domain), + IrohDeployer(config.enable_iroh_relay), + AcmetoolDeployer(config.acme_email, tls_domains), - apt.packages( - name="Install Postfix", - packages="postfix", - ) + WebsiteDeployer(config), + ChatmailVenvDeployer(config), + MtastsDeployer(), + OpendkimDeployer(mail_domain), - if not "dovecot.service" in host.get_fact(SystemdEnabled): - _install_dovecot_package("core", host.get_fact(facts.server.Arch)) - _install_dovecot_package("imapd", host.get_fact(facts.server.Arch)) - _install_dovecot_package("lmtpd", host.get_fact(facts.server.Arch)) + # Dovecot should be started before Postfix + # because it creates authentication socket + # required by Postfix. + DovecotDeployer(config, disable_mail), + PostfixDeployer(config, disable_mail), + FcgiwrapDeployer(), + NginxDeployer(config), + RspamdDeployer(), + EchobotDeployer(mail_domain), + MtailDeployer(config.mtail_address), + GithashDeployer(), + ] - apt.packages( - name="Install nginx", - packages=["nginx", "libnginx-mod-stream"], - ) + Deployment().perform_stages(all_deployers) - apt.packages( - name="Install fcgiwrap", - packages=["fcgiwrap"], - ) - - www_path, src_dir, build_dir = get_paths(config) - # if www_folder was set to a non-existing folder, skip upload - if not www_path.is_dir(): - logger.warning("Building web pages is disabled in chatmail.ini, skipping") - elif (path := find_merge_conflict(src_dir)) is not None: - logger.warning(f"Merge conflict found in {path}, skipping website deployment. Fix merge conflict if you want to upload your web page.") - else: - # if www_folder is a hugo page, build it - if build_dir: - www_path = build_webpages(src_dir, build_dir, config) - # if it is not a hugo page, upload it as is - files.rsync(f"{www_path}/", "/var/www/html", flags=["-avz", "--chown=www-data"]) - - _install_remote_venv_with_chatmaild(config) - debug = False - dovecot_need_restart = _configure_dovecot(config, debug=debug) - postfix_need_restart = _configure_postfix(config, debug=debug) - nginx_need_restart = _configure_nginx(config) - _uninstall_mta_sts_daemon() - - _remove_rspamd() - opendkim_need_restart = _configure_opendkim(mail_domain, "opendkim") - - systemd.service( - name="Start and enable OpenDKIM", - service="opendkim.service", - running=True, - enabled=True, - daemon_reload=opendkim_need_restart, - restarted=opendkim_need_restart, - ) - - # Dovecot should be started before Postfix - # because it creates authentication socket - # required by Postfix. - systemd.service( - name="disable dovecot for now" if disable_mail else "Start and enable Dovecot", - service="dovecot.service", - running=False if disable_mail else True, - enabled=False if disable_mail else True, - restarted=dovecot_need_restart if not disable_mail else False, - ) - - systemd.service( - name="disable postfix for now" if disable_mail else "Start and enable Postfix", - service="postfix.service", - running=False if disable_mail else True, - enabled=False if disable_mail else True, - restarted=postfix_need_restart if not disable_mail else False, - ) - - systemd.service( - name="Start and enable nginx", - service="nginx.service", - running=True, - enabled=True, - restarted=nginx_need_restart, - ) - - systemd.service( - name="Start and enable fcgiwrap", - service="fcgiwrap.service", - running=True, - enabled=True, - ) - - systemd.service( - name="Restart echobot if postfix and dovecot were just started", - service="echobot.service", - restarted=postfix_need_restart and dovecot_need_restart, - ) - - # This file is used by auth proxy. - # https://wiki.debian.org/EtcMailName - server.shell( - name="Setup /etc/mailname", - commands=[f"echo {mail_domain} >/etc/mailname; chmod 644 /etc/mailname"], - ) - - journald_conf = files.put( - name="Configure journald", - src=importlib.resources.files(__package__).joinpath("journald.conf"), - dest="/etc/systemd/journald.conf", - user="root", - group="root", - mode="644", - ) - systemd.service( - name="Start and enable journald", - service="systemd-journald.service", - running=True, - enabled=True, - restarted=journald_conf.changed, - ) files.directory( name="Ensure old logs on disk are deleted", path="/var/log/journal/", present=False, ) - - apt.packages( - name="Ensure cron is installed", - packages=["cron"], - ) - try: - git_hash = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode() - except Exception: - git_hash = "unknown\n" - try: - git_diff = subprocess.check_output(["git", "diff"]).decode() - except Exception: - git_diff = "" - files.put( - name="Upload chatmail relay git commiit hash", - src=StringIO(git_hash + git_diff), - dest="/etc/chatmail-version", - mode="700", - ) - - deploy_mtail(config) diff --git a/cmdeploy/src/cmdeploy/acmetool/__init__.py b/cmdeploy/src/cmdeploy/acmetool/__init__.py index 26b0f4a2..495abe96 100644 --- a/cmdeploy/src/cmdeploy/acmetool/__init__.py +++ b/cmdeploy/src/cmdeploy/acmetool/__init__.py @@ -2,104 +2,121 @@ import importlib.resources from pyinfra.operations import apt, files, server, systemd +from ..deployer import Deployer -def deploy_acmetool(email="", domains=[]): - """Deploy acmetool.""" - apt.packages( - name="Install acmetool", - packages=["acmetool"], - ) - files.file( - name="Remove old acmetool cronjob, it is replaced with systemd timer.", - path="/etc/cron.d/acmetool", - present=False, - ) +class AcmetoolDeployer(Deployer): + def __init__(self, email, domains): + self.domains = domains + self.email = email + self.need_restart_redirector = False + self.need_restart_reconcile_service = False + self.need_restart_reconcile_timer = False - files.put( - name="Install acmetool hook.", - src=importlib.resources.files(__package__).joinpath("acmetool.hook").open("rb"), - dest="/etc/acme/hooks/nginx", - user="root", - group="root", - mode="755", - ) - files.file( - name="Remove acmetool hook from the wrong location where it was previously installed.", - path="/usr/lib/acme/hooks/nginx", - present=False, - ) + def install(self): + apt.packages( + name="Install acmetool", + packages=["acmetool"], + ) - files.template( - src=importlib.resources.files(__package__).joinpath("response-file.yaml.j2"), - dest="/var/lib/acme/conf/responses", - user="root", - group="root", - mode="644", - email=email, - ) + files.file( + name="Remove old acmetool cronjob, it is replaced with systemd timer.", + path="/etc/cron.d/acmetool", + present=False, + ) - files.template( - src=importlib.resources.files(__package__).joinpath("target.yaml.j2"), - dest="/var/lib/acme/conf/target", - user="root", - group="root", - mode="644", - ) + files.put( + name="Install acmetool hook.", + src=importlib.resources.files(__package__).joinpath("acmetool.hook").open("rb"), + dest="/etc/acme/hooks/nginx", + user="root", + group="root", + mode="755", + ) + files.file( + name="Remove acmetool hook from the wrong location where it was previously installed.", + path="/usr/lib/acme/hooks/nginx", + present=False, + ) - service_file = files.put( - src=importlib.resources.files(__package__).joinpath( - "acmetool-redirector.service" - ), - dest="/etc/systemd/system/acmetool-redirector.service", - user="root", - group="root", - mode="644", - ) + def configure(self): + files.template( + src=importlib.resources.files(__package__).joinpath("response-file.yaml.j2"), + dest="/var/lib/acme/conf/responses", + user="root", + group="root", + mode="644", + email=self.email, + ) - systemd.service( - name="Setup acmetool-redirector service", - service="acmetool-redirector.service", - running=True, - enabled=True, - restarted=service_file.changed, - ) + files.template( + src=importlib.resources.files(__package__).joinpath("target.yaml.j2"), + dest="/var/lib/acme/conf/target", + user="root", + group="root", + mode="644", + ) - reconcile_service_file = files.put( - src=importlib.resources.files(__package__).joinpath( - "acmetool-reconcile.service" - ), - dest="/etc/systemd/system/acmetool-reconcile.service", - user="root", - group="root", - mode="644", - ) + service_file = files.put( + src=importlib.resources.files(__package__).joinpath( + "acmetool-redirector.service" + ), + dest="/etc/systemd/system/acmetool-redirector.service", + user="root", + group="root", + mode="644", + ) + self.need_restart_redirector = service_file.changed - systemd.service( - name="Setup acmetool-reconcile service", - service="acmetool-reconcile.service", - running=False, - enabled=False, - daemon_reload=reconcile_service_file.changed, - ) + reconcile_service_file = files.put( + src=importlib.resources.files(__package__).joinpath( + "acmetool-reconcile.service" + ), + dest="/etc/systemd/system/acmetool-reconcile.service", + user="root", + group="root", + mode="644", + ) + self.need_restart_reconcile_service = reconcile_service_file.changed - reconcile_timer_file = files.put( - src=importlib.resources.files(__package__).joinpath("acmetool-reconcile.timer"), - dest="/etc/systemd/system/acmetool-reconcile.timer", - user="root", - group="root", - mode="644", - ) + reconcile_timer_file = files.put( + src=importlib.resources.files(__package__).joinpath("acmetool-reconcile.timer"), + dest="/etc/systemd/system/acmetool-reconcile.timer", + user="root", + group="root", + mode="644", + ) + self.need_restart_reconcile_timer = reconcile_timer_file.changed - systemd.service( - name="Setup acmetool-reconcile timer", - service="acmetool-reconcile.timer", - running=True, - enabled=True, - daemon_reload=reconcile_timer_file.changed, - ) + def activate(self): + systemd.service( + name="Setup acmetool-redirector service", + service="acmetool-redirector.service", + running=True, + enabled=True, + restarted=self.need_restart_redirector, + ) + self.need_restart_redirector = False - server.shell( - name=f"Request certificate for: {', '.join(domains)}", - commands=[f"acmetool want --xlog.severity=debug {' '.join(domains)}"], - ) + systemd.service( + name="Setup acmetool-reconcile service", + service="acmetool-reconcile.service", + running=False, + enabled=False, + daemon_reload=self.need_restart_reconcile_service, + ) + self.need_restart_reconcile_service = False + + systemd.service( + name="Setup acmetool-reconcile timer", + service="acmetool-reconcile.timer", + running=True, + enabled=True, + daemon_reload=self.need_restart_reconcile_timer, + ) + self.need_restart_reconcile_timer = False + + server.shell( + name=f"Request certificate for: {', '.join(self.domains)}", + commands=[f"acmetool want --xlog.severity=debug {' '.join(self.domains)}"], + ) diff --git a/cmdeploy/src/cmdeploy/deployer.py b/cmdeploy/src/cmdeploy/deployer.py new file mode 100644 index 00000000..a55eb4ef --- /dev/null +++ b/cmdeploy/src/cmdeploy/deployer.py @@ -0,0 +1,57 @@ +import os + +from pyinfra.operations import server + + +class Deployment: + def install(self, deployer): + # optional 'required_users' contains a list of (user, group, secondary-group-list) tuples. + # If the group is None, no group is created corresponding to that user. + # If the secondary group list is not None, all listed groups are created as well. + required_users = getattr(deployer, "required_users", []) + for user, group, groups in required_users: + if group is not None: + server.group( + name="Create {} group".format(group), group=group, system=True + ) + if groups is not None: + for group2 in groups: + server.group( + name="Create {} group".format(group2), group=group2, system=True + ) + server.user( + name="Create {} user".format(user), + user=user, + group=group, + groups=groups, + system=True, + ) + + deployer.install() + + def configure(self, deployer): + deployer.configure() + + def activate(self, deployer): + deployer.activate() + + def perform_stages(self, deployers): + default_stages = "install,configure,activate" + stages = os.getenv("CMDEPLOY_STAGES", default_stages).split(",") + + for stage in stages: + for deployer in deployers: + getattr(self, stage)(deployer) + + +class Deployer: + need_restart = False + + def install(self): + pass + + def configure(self): + pass + + def activate(self): + pass diff --git a/cmdeploy/src/cmdeploy/obs-home-deltachat.gpg b/cmdeploy/src/cmdeploy/obs-home-deltachat.gpg deleted file mode 100644 index 7b7c710e..00000000 Binary files a/cmdeploy/src/cmdeploy/obs-home-deltachat.gpg and /dev/null differ diff --git a/cmdeploy/src/cmdeploy/policy-rc.d b/cmdeploy/src/cmdeploy/policy-rc.d new file mode 100755 index 00000000..ec21cabf --- /dev/null +++ b/cmdeploy/src/cmdeploy/policy-rc.d @@ -0,0 +1,3 @@ +#!/bin/sh +echo "All runlevel operations denied by policy" >&2 +exit 101 diff --git a/doc/source/overview.rst b/doc/source/overview.rst index c04394ba..4b4a044f 100644 --- a/doc/source/overview.rst +++ b/doc/source/overview.rst @@ -297,3 +297,48 @@ actually it is a problem with your TLS certificate. .. _nginx: https://nginx.org .. _pyinfra: https://pyinfra.com + +Architecture of cmdeploy +------------------------ + +cmdeploy is a Python program that uses the pyinfra library to deploy +chatmail relays, with all the necessary software, configuration, and +services. The deployment process performs three primary types of +operation: + +1. Installation of software, universal across all deployments. +2. Configuration of software, with deploy-specific variations. +3. Activation of services. + +The process is implemented through a family of "deployer" objects +which all derive from a common ``Deployer`` base class, defined in +cmdeploy/src/cmdeploy/deployer.py. Each object provides +implementation methods for the three stages -- install, configure, and +activate. The top-level procedure in ``deploy_chatmail()`` calls +these methods for all the deployer objects, via the +``Deployment.perform_stages()`` method, also defined in deployer.py. +This first calls all the install methods, then the configure methods, +then the activate methods. + +The ``Deployment`` class also implements support for a CMDEPLOY_STAGES +environment variable, which allows limiting the process to specific +stages. Note that some deployers are stateful between the stages +(this is one reason why they are implemented as objects), and that +state will not get propagated between stages when run in separate +invocations of cmdeploy. This environment variable is intended for +use in future revisions to support building Docker images with +software pre-installed, and configuration of containers at run time +from environment variables. + +The, ``install()`` methods for the deployer classes should use 'self' +as little as possible, preferably not at all. In particular, +``install()`` methods should never depend on "config" data, such as +the config dictionary in ``self.config`` or specific values like +``self.mail_domain``. This ensures that these methods can be used to +perform generic installation operations that are applicable across +multiple relay deployments, and therefore can be called in the process +of building a general-purpose container image. + +Operations that start services for systemd-based deployments should +only be called from the ``activate_impl()`` methods. These methods +will not be called in non-systemd container environments.