""" Chat Mail pyinfra deploy. """ import shutil import subprocess import sys from io import StringIO from pathlib import Path from chatmaild.config import read_config from pyinfra import facts, host, logger from pyinfra.api import FactBase from pyinfra.facts.files import Sha256File from pyinfra.facts.systemd import SystemdEnabled from pyinfra.operations import apt, files, pip, server, systemd from cmdeploy.cmdeploy import Out from .basedeploy import ( Deployer, Deployment, activate_remote_units, configure_remote_units, get_resource, ) from .dovecot.deployer import DovecotDeployer from .filtermail.deployer import FiltermailDeployer from .mtail.deployer import MtailDeployer from .nginx.deployer import NginxDeployer from .opendkim.deployer import OpendkimDeployer from .postfix.deployer import PostfixDeployer from .www import build_webpages, find_merge_conflict, get_paths class Port(FactBase): """ Returns the process occuping a port. """ def command(self, port: int) -> str: return ( "ss -lptn 'src :%d' | awk 'NR>1 {print $6,$7}' | sed 's/users:((\"//;s/\".*//'" % (port,) ) def process(self, output: [str]) -> str: return output[0] def _build_chatmaild(dist_dir) -> None: dist_dir = Path(dist_dir).resolve() if dist_dir.exists(): shutil.rmtree(dist_dir) dist_dir.mkdir() subprocess.check_output( [sys.executable, "-m", "build", "-n"] + ["--sdist", "chatmaild", "--outdir", str(dist_dir)] ) entries = list(dist_dir.iterdir()) assert len(entries) == 1 return entries[0] def remove_legacy_artifacts(): # disable legacy doveauth-dictproxy.service if host.get_fact(SystemdEnabled).get("doveauth-dictproxy.service"): systemd.service( name="Disable legacy doveauth-dictproxy.service", service="doveauth-dictproxy.service", running=False, enabled=False, ) def _install_remote_venv_with_chatmaild() -> None: remove_legacy_artifacts() dist_file = _build_chatmaild(dist_dir=Path("chatmaild/dist")) remote_base_dir = "/usr/local/lib/chatmaild" remote_dist_file = f"{remote_base_dir}/dist/{dist_file.name}" remote_venv_dir = f"{remote_base_dir}/venv" root_owned = dict(user="root", group="root", mode="644") apt.packages( name="apt install python3-virtualenv", packages=["python3-virtualenv"], ) files.put( name="Upload chatmaild source package", src=dist_file.open("rb"), dest=remote_dist_file, create_remote_dir=True, **root_owned, ) pip.virtualenv( name=f"chatmaild virtualenv {remote_venv_dir}", path=remote_venv_dir, always_copy=True, ) apt.packages( name="install gcc and headers to build crypt_r source package", packages=["gcc", "python3-dev"], ) server.shell( name=f"forced pip-install {dist_file.name}", commands=[ f"{remote_venv_dir}/bin/pip install --force-reinstall {remote_dist_file}" ], ) def _configure_remote_venv_with_chatmaild(config) -> None: remote_base_dir = "/usr/local/lib/chatmaild" remote_venv_dir = f"{remote_base_dir}/venv" remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini" root_owned = dict(user="root", group="root", mode="644") files.put( name=f"Upload {remote_chatmail_inipath}", src=config._getbytefile(), dest=remote_chatmail_inipath, **root_owned, ) files.template( src=get_resource("metrics.cron.j2"), dest="/etc/cron.d/chatmail-metrics", user="root", group="root", mode="644", config={ "mailboxes_dir": config.mailboxes_dir, "execpath": f"{remote_venv_dir}/bin/chatmail-metrics", }, ) class UnboundDeployer(Deployer): def install(self): # Run local DNS resolver `unbound`. # `resolvconf` takes care of setting up /etc/resolv.conf # to use 127.0.0.1 as the resolver. # # On an IPv4-only system, if unbound is started but not # configured, it causes subsequent steps to fail to resolve hosts. # Here, we use policy-rc.d to prevent unbound from starting up # on initial install. Later, we will configure it and start it. # # For documentation about policy-rc.d, see: # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt # files.put( src=get_resource("policy-rc.d"), dest="/usr/sbin/policy-rc.d", user="root", group="root", mode="755", ) apt.packages( name="Install unbound", packages=["unbound", "unbound-anchor", "dnsutils"], ) files.file("/usr/sbin/policy-rc.d", present=False) def configure(self): server.shell( name="Generate root keys for validating DNSSEC", commands=[ "unbound-anchor -a /var/lib/unbound/root.key || true", ], ) def activate(self): server.shell( name="Generate root keys for validating DNSSEC", commands=[ "systemctl reset-failed unbound.service", ], ) systemd.service( name="Start and enable unbound", service="unbound.service", running=True, enabled=True, ) class MtastsDeployer(Deployer): def configure(self): # Remove configuration. files.file("/etc/mta-sts-daemon.yml", present=False) files.directory("/usr/local/lib/postfix-mta-sts-resolver", present=False) files.file("/etc/systemd/system/mta-sts-daemon.service", present=False) def activate(self): systemd.service( name="Stop MTA-STS daemon", service="mta-sts-daemon.service", daemon_reload=True, running=False, enabled=False, ) class WebsiteDeployer(Deployer): def __init__(self, config): self.config = config def install(self): files.directory( name="Ensure /var/www exists", path="/var/www", user="root", group="root", mode="755", present=True, ) def configure(self): www_path, src_dir, build_dir = get_paths(self.config) # if www_folder was set to a non-existing folder, skip upload if not www_path.is_dir(): logger.warning("Building web pages is disabled in chatmail.ini, skipping") elif (path := find_merge_conflict(src_dir)) is not None: logger.warning( f"Merge conflict found in {path}, skipping website deployment. Fix merge conflict if you want to upload your web page." ) else: # if www_folder is a hugo page, build it if build_dir: www_path = build_webpages(src_dir, build_dir, self.config) # if it is not a hugo page, upload it as is files.rsync( f"{www_path}/", "/var/www/html", flags=["-avz", "--chown=www-data"] ) class LegacyRemoveDeployer(Deployer): def install(self): apt.packages(name="Remove rspamd", packages="rspamd", present=False) # remove historic expunge script # which is now implemented through a systemd timer (chatmail-expire) files.file( path="/etc/cron.d/expunge", present=False, ) # Remove OBS repository key that is no longer used. files.file("/etc/apt/keyrings/obs-home-deltachat.gpg", present=False) files.line( name="Remove DeltaChat OBS home repository from sources.list", path="/etc/apt/sources.list", line="deb [signed-by=/etc/apt/keyrings/obs-home-deltachat.gpg] https://download.opensuse.org/repositories/home:/deltachat/Debian_12/ ./", escape_regex_characters=True, present=False, ) # prior relay versions used filelogging files.directory( name="Ensure old logs on disk are deleted", path="/var/log/journal/", present=False, ) # remove echobot if it is still running if host.get_fact(SystemdEnabled).get("echobot.service"): systemd.service( name="Disable echobot.service", service="echobot.service", running=False, enabled=False, ) def check_config(config): mail_domain = config.mail_domain if mail_domain != "testrun.org" and not mail_domain.endswith(".testrun.org"): blocked_words = "merlinux schmieder testrun.org".split() for key in config.__dict__: value = config.__dict__[key] if key.startswith("privacy") and any( x in str(value) for x in blocked_words ): raise ValueError( f"please set your own privacy contacts/addresses in {config._inipath}" ) return config class TurnDeployer(Deployer): def __init__(self, mail_domain): self.mail_domain = mail_domain self.units = ["turnserver"] def install(self): (url, sha256sum) = { "x86_64": ( "https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-x86_64-linux", "841e527c15fdc2940b0469e206188ea8f0af48533be12ecb8098520f813d41e4", ), "aarch64": ( "https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-aarch64-linux", "a5fc2d06d937b56a34e098d2cd72a82d3e89967518d159bf246dc69b65e81b42", ), }[host.get_fact(facts.server.Arch)] existing_sha256sum = host.get_fact(Sha256File, "/usr/local/bin/chatmail-turn") if existing_sha256sum != sha256sum: server.shell( name="Download chatmail-turn", commands=[ f"(curl -L {url} >/usr/local/bin/chatmail-turn.new && (echo '{sha256sum} /usr/local/bin/chatmail-turn.new' | sha256sum -c) && mv /usr/local/bin/chatmail-turn.new /usr/local/bin/chatmail-turn)", "chmod 755 /usr/local/bin/chatmail-turn", ], ) def configure(self): configure_remote_units(self.mail_domain, self.units) def activate(self): activate_remote_units(self.units) class IrohDeployer(Deployer): def __init__(self, enable_iroh_relay): self.enable_iroh_relay = enable_iroh_relay def install(self): (url, sha256sum) = { "x86_64": ( "https://github.com/n0-computer/iroh/releases/download/v0.35.0/iroh-relay-v0.35.0-x86_64-unknown-linux-musl.tar.gz", "45c81199dbd70f8c4c30fef7f3b9727ca6e3cea8f2831333eeaf8aa71bf0fac1", ), "aarch64": ( "https://github.com/n0-computer/iroh/releases/download/v0.35.0/iroh-relay-v0.35.0-aarch64-unknown-linux-musl.tar.gz", "f8ef27631fac213b3ef668d02acd5b3e215292746a3fc71d90c63115446008b1", ), }[host.get_fact(facts.server.Arch)] existing_sha256sum = host.get_fact(Sha256File, "/usr/local/bin/iroh-relay") if existing_sha256sum != sha256sum: server.shell( name="Download iroh-relay", commands=[ f"(curl -L {url} | gunzip | tar -x -f - ./iroh-relay -O >/usr/local/bin/iroh-relay.new && (echo '{sha256sum} /usr/local/bin/iroh-relay.new' | sha256sum -c) && mv /usr/local/bin/iroh-relay.new /usr/local/bin/iroh-relay)", "chmod 755 /usr/local/bin/iroh-relay", ], ) self.need_restart = True def configure(self): systemd_unit = files.put( name="Upload iroh-relay systemd unit", src=get_resource("iroh-relay.service"), dest="/etc/systemd/system/iroh-relay.service", user="root", group="root", mode="644", ) self.need_restart |= systemd_unit.changed iroh_config = files.put( name="Upload iroh-relay config", src=get_resource("iroh-relay.toml"), dest="/etc/iroh-relay.toml", user="root", group="root", mode="644", ) self.need_restart |= iroh_config.changed def activate(self): systemd.service( name="Start and enable iroh-relay", service="iroh-relay.service", running=True, enabled=self.enable_iroh_relay, restarted=self.need_restart, ) self.need_restart = False class JournaldDeployer(Deployer): def configure(self): journald_conf = files.put( name="Configure journald", src=get_resource("journald.conf"), dest="/etc/systemd/journald.conf", user="root", group="root", mode="644", ) self.need_restart = journald_conf.changed def activate(self): systemd.service( name="Start and enable journald", service="systemd-journald.service", running=True, enabled=True, restarted=self.need_restart, ) self.need_restart = False class ChatmailVenvDeployer(Deployer): def __init__(self, config): self.config = config self.units = ( "chatmail-metadata", "lastlogin", "chatmail-expire", "chatmail-expire.timer", "chatmail-fsreport", "chatmail-fsreport.timer", ) def install(self): _install_remote_venv_with_chatmaild() def configure(self): _configure_remote_venv_with_chatmaild(self.config) configure_remote_units(self.config.mail_domain, self.units) def activate(self): activate_remote_units(self.units) class ChatmailDeployer(Deployer): required_users = [ ("vmail", "vmail", None), ("iroh", None, None), ] def __init__(self, mail_domain): self.mail_domain = mail_domain def install(self): apt.update(name="apt update", cache_time=24 * 3600) apt.upgrade(name="upgrade apt packages", auto_remove=True) apt.packages( name="Install curl", packages=["curl"], ) apt.packages( name="Install rsync", packages=["rsync"], ) apt.packages( name="Ensure cron is installed", packages=["cron"], ) def configure(self): # This file is used by auth proxy. # https://wiki.debian.org/EtcMailName server.shell( name="Setup /etc/mailname", commands=[ f"echo {self.mail_domain} >/etc/mailname; chmod 644 /etc/mailname" ], ) class FcgiwrapDeployer(Deployer): def install(self): apt.packages( name="Install fcgiwrap", packages=["fcgiwrap"], ) def activate(self): systemd.service( name="Start and enable fcgiwrap", service="fcgiwrap.service", running=True, enabled=True, ) class GithashDeployer(Deployer): def activate(self): try: git_hash = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode() except Exception: git_hash = "unknown\n" try: git_diff = subprocess.check_output(["git", "diff"]).decode() except Exception: git_diff = "" files.put( name="Upload chatmail relay git commit hash", src=StringIO(git_hash + git_diff), dest="/etc/chatmail-version", mode="700", ) def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -> None: """Deploy a chat-mail instance. :param config_path: path to chatmail.ini :param disable_mail: whether to disable postfix & dovecot :param website_only: if True, only deploy the website """ config = read_config(config_path) check_config(config) mail_domain = config.mail_domain if website_only: Deployment().perform_stages([WebsiteDeployer(config)]) return if host.get_fact(Port, port=53) != "unbound": files.line( name="Add 9.9.9.9 to resolv.conf", path="/etc/resolv.conf", line="nameserver 9.9.9.9", ) port_services = [ (["master", "smtpd"], 25), ("unbound", 53), (["imap-login", "dovecot"], 143), ("nginx", 443), (["master", "smtpd"], 465), (["master", "smtpd"], 587), (["imap-login", "dovecot"], 993), ("iroh-relay", 3340), ("mtail", 3903), ("dovecot-stats", 3904), ("nginx", 8443), (["master", "smtpd"], config.postfix_reinject_port), (["master", "smtpd"], config.postfix_reinject_port_incoming), ("filtermail", config.filtermail_smtp_port), ("filtermail", config.filtermail_smtp_port_incoming), ] for service, port in port_services: print(f"Checking if port {port} is available for {service}...") running_service = host.get_fact(Port, port=port) if running_service: if running_service not in service: Out().red( f"Deploy failed: port {port} is occupied by: {running_service}" ) exit(1) tls_domains = [mail_domain, f"mta-sts.{mail_domain}", f"www.{mail_domain}"] all_deployers = [ ChatmailDeployer(mail_domain), LegacyRemoveDeployer(), FiltermailDeployer(), JournaldDeployer(), UnboundDeployer(), TurnDeployer(mail_domain), IrohDeployer(config.enable_iroh_relay), WebsiteDeployer(config), ChatmailVenvDeployer(config), MtastsDeployer(), OpendkimDeployer(mail_domain), # Dovecot should be started before Postfix # because it creates authentication socket # required by Postfix. DovecotDeployer(config, disable_mail), PostfixDeployer(config, disable_mail), FcgiwrapDeployer(), NginxDeployer(config), MtailDeployer(config.mtail_address), GithashDeployer(), ] Deployment().perform_stages(all_deployers)