refactor: introduce automated change-tracking across deployers

This commit is contained in:
holger krekel
2026-04-30 17:43:19 +02:00
parent fb64be97b5
commit a1f64ebd96
15 changed files with 446 additions and 655 deletions

View File

@@ -48,6 +48,8 @@ def test_migration(tmp_path, example_config, caplog):
assert passdb_path.stat().st_size > 10000
example_config.passdb_path = passdb_path
# ensure logging.info records are captured regardless of global configuration
caplog.set_level("INFO")
assert not caplog.records

View File

@@ -1,6 +1,4 @@
import importlib.resources
from pyinfra.operations import apt, files, server, systemd
from pyinfra.operations import apt, server
from ..basedeploy import Deployer
@@ -9,9 +7,6 @@ class AcmetoolDeployer(Deployer):
def __init__(self, email, domains):
self.domains = domains
self.email = email
self.need_restart_redirector = False
self.need_restart_reconcile_service = False
self.need_restart_reconcile_timer = False
def install(self):
apt.packages(
@@ -19,121 +14,41 @@ class AcmetoolDeployer(Deployer):
packages=["acmetool"],
)
files.file(
name="Remove old acmetool cronjob, it is replaced with systemd timer.",
path="/etc/cron.d/acmetool",
present=False,
)
self.remove_file("/etc/cron.d/acmetool")
files.put(
name="Install acmetool hook.",
src=importlib.resources.files(__package__)
.joinpath("acmetool.hook")
.open("rb"),
dest="/etc/acme/hooks/nginx",
user="root",
group="root",
mode="755",
)
files.file(
name="Remove acmetool hook from the wrong location where it was previously installed.",
path="/usr/lib/acme/hooks/nginx",
present=False,
)
self.put_executable("acmetool/acmetool.hook", "/etc/acme/hooks/nginx")
self.remove_file("/usr/lib/acme/hooks/nginx")
def configure(self):
files.template(
src=importlib.resources.files(__package__).joinpath(
"response-file.yaml.j2"
),
dest="/var/lib/acme/conf/responses",
user="root",
group="root",
mode="644",
self.put_template(
"acmetool/response-file.yaml.j2",
"/var/lib/acme/conf/responses",
email=self.email,
)
files.template(
src=importlib.resources.files(__package__).joinpath("target.yaml.j2"),
dest="/var/lib/acme/conf/target",
user="root",
group="root",
mode="644",
self.put_template(
"acmetool/target.yaml.j2",
"/var/lib/acme/conf/target",
)
server.shell(
name=f"Remove old acmetool desired files for {self.domains[0]}",
commands=[f"rm -f /var/lib/acme/desired/{self.domains[0]}-*"],
)
files.template(
src=importlib.resources.files(__package__).joinpath("desired.yaml.j2"),
dest=f"/var/lib/acme/desired/{self.domains[0]}", # 0 is mailhost TLD
user="root",
group="root",
mode="644",
self.put_template(
"acmetool/desired.yaml.j2",
f"/var/lib/acme/desired/{self.domains[0]}",
domains=self.domains,
)
service_file = files.put(
src=importlib.resources.files(__package__).joinpath(
"acmetool-redirector.service"
),
dest="/etc/systemd/system/acmetool-redirector.service",
user="root",
group="root",
mode="644",
)
self.need_restart_redirector = service_file.changed
reconcile_service_file = files.put(
src=importlib.resources.files(__package__).joinpath(
"acmetool-reconcile.service"
),
dest="/etc/systemd/system/acmetool-reconcile.service",
user="root",
group="root",
mode="644",
)
self.need_restart_reconcile_service = reconcile_service_file.changed
reconcile_timer_file = files.put(
src=importlib.resources.files(__package__).joinpath(
"acmetool-reconcile.timer"
),
dest="/etc/systemd/system/acmetool-reconcile.timer",
user="root",
group="root",
mode="644",
)
self.need_restart_reconcile_timer = reconcile_timer_file.changed
self.ensure_systemd_unit("acmetool/acmetool-redirector.service")
self.ensure_systemd_unit("acmetool/acmetool-reconcile.service")
self.ensure_systemd_unit("acmetool/acmetool-reconcile.timer")
def activate(self):
systemd.service(
name="Setup acmetool-redirector service",
service="acmetool-redirector.service",
running=True,
enabled=True,
restarted=self.need_restart_redirector,
)
self.need_restart_redirector = False
systemd.service(
name="Setup acmetool-reconcile service",
service="acmetool-reconcile.service",
running=False,
enabled=False,
daemon_reload=self.need_restart_reconcile_service,
)
self.need_restart_reconcile_service = False
systemd.service(
name="Setup acmetool-reconcile timer",
service="acmetool-reconcile.timer",
running=True,
enabled=True,
daemon_reload=self.need_restart_reconcile_timer,
)
self.need_restart_reconcile_timer = False
self.ensure_service("acmetool-redirector.service")
self.ensure_service("acmetool-reconcile.service", running=False, enabled=False)
self.ensure_service("acmetool-reconcile.timer")
server.shell(
name=f"Reconcile certificates for: {', '.join(self.domains)}",

View File

@@ -4,6 +4,7 @@ import os
from contextlib import contextmanager
from pyinfra import host
from pyinfra.facts.files import Sha256File
from pyinfra.facts.server import Command
from pyinfra.operations import files, server, systemd
@@ -50,11 +51,10 @@ def get_resource(arg, pkg=__package__):
return importlib.resources.files(pkg).joinpath(arg)
def configure_remote_units(mail_domain, units) -> None:
def configure_remote_units(deployer, mail_domain, units) -> None:
remote_base_dir = "/usr/local/lib/chatmaild"
remote_venv_dir = f"{remote_base_dir}/venv"
remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini"
root_owned = dict(user="root", group="root", mode="644")
# install systemd units
for fn in units:
@@ -70,15 +70,13 @@ def configure_remote_units(mail_domain, units) -> None:
source_path = get_resource(f"service/{basename}.f")
content = source_path.read_text().format(**params).encode()
files.put(
name=f"Upload {basename}",
deployer.put_file(
src=io.BytesIO(content),
dest=f"/etc/systemd/system/{basename}",
**root_owned,
)
def activate_remote_units(units) -> None:
def activate_remote_units(deployer, units) -> None:
# activate systemd units
for fn in units:
basename = fn if "." in fn else f"{fn}.service"
@@ -88,14 +86,8 @@ def activate_remote_units(units) -> None:
enabled = False
else:
enabled = True
systemd.service(
name=f"Setup {basename}",
service=basename,
running=enabled,
enabled=enabled,
restarted=enabled,
daemon_reload=True,
)
deployer.ensure_service(basename, running=enabled, enabled=enabled)
class Deployment:
@@ -141,6 +133,7 @@ class Deployment:
class Deployer:
need_restart = False
daemon_reload = False
def install(self):
pass
@@ -150,3 +143,113 @@ class Deployer:
def activate(self):
pass
def ensure_service(self, service, running=True, enabled=True):
if running:
verb = "Start and enable"
else:
verb = "Stop"
systemd.service(
name=f"{verb} {service}",
service=service,
running=running,
enabled=enabled,
restarted=self.need_restart if running else False,
daemon_reload=self.daemon_reload,
)
self.daemon_reload = False
def ensure_systemd_unit(self, src, **kwargs):
dest_name = src.split("/")[-1].replace(".j2", "")
dest = f"/etc/systemd/system/{dest_name}"
if src.endswith(".j2"):
return self.put_template(src, dest, **kwargs)
return self.put_file(src, dest)
def put_file(self, src, dest, mode="644"):
if isinstance(src, str):
src = get_resource(src)
res = files.put(
name=f"Upload {dest}",
src=src,
dest=dest,
user="root",
group="root",
mode=mode,
)
return self._update_restart_signals(dest, res)
def put_executable(self, src, dest):
return self.put_file(src, dest, mode="755")
def put_template(self, src, dest, owner="root", **kwargs):
if isinstance(src, str):
src = get_resource(src)
res = files.template(
name=f"Upload {dest}",
src=src,
dest=dest,
user=owner,
group=owner,
mode="644",
**kwargs,
)
return self._update_restart_signals(dest, res)
def remove_file(self, dest):
res = files.file(name=f"Remove {dest}", path=dest, present=False)
return self._update_restart_signals(dest, res)
def ensure_line(self, path, line, **kwargs):
name = kwargs.pop("name", f"Ensure line in {path}")
res = files.line(name=name, path=path, line=line, **kwargs)
return self._update_restart_signals(path, res)
def ensure_directory(self, path, owner="root", mode="755", **kwargs):
name = kwargs.pop("name", f"Ensure directory {path}")
res = files.directory(
name=name,
path=path,
user=owner,
group=owner,
mode=mode,
present=True,
**kwargs,
)
return self._update_restart_signals(path, res)
def remove_directory(self, path, **kwargs):
name = kwargs.pop("name", f"Remove directory {path}")
res = files.directory(name=name, path=path, present=False, **kwargs)
return self._update_restart_signals(path, res)
def download_executable(self, url, dest, sha256sum, extract=None):
existing = host.get_fact(Sha256File, dest)
if existing == sha256sum:
return
tmp = f"{dest}.new"
if extract:
dl_cmd = f"curl -fSL {url} | {extract} >{tmp}"
else:
dl_cmd = f"curl -fSL {url} -o {tmp}"
server.shell(
name=f"Download {dest}",
commands=[
f"({dl_cmd}"
f" && echo '{sha256sum} {tmp}' | sha256sum -c"
f" && mv {tmp} {dest})",
f"chmod 755 {dest}",
],
)
self.need_restart = True
def _update_restart_signals(self, path, res):
if res.changed:
self.need_restart = True
if str(path).startswith("/etc/systemd/system/"):
self.daemon_reload = True
return res

View File

@@ -12,7 +12,6 @@ from chatmaild.config import read_config
from pyinfra import facts, host, logger
from pyinfra.api import FactBase
from pyinfra.facts import hardware
from pyinfra.facts.files import Sha256File
from pyinfra.facts.systemd import SystemdEnabled
from pyinfra.operations import apt, files, pip, server, systemd
@@ -25,7 +24,6 @@ from .basedeploy import (
activate_remote_units,
blocked_service_startup,
configure_remote_units,
get_resource,
has_systemd,
is_in_container,
)
@@ -82,25 +80,22 @@ def remove_legacy_artifacts():
)
def _install_remote_venv_with_chatmaild() -> None:
def _install_remote_venv_with_chatmaild(deployer) -> None:
remove_legacy_artifacts()
dist_file = _build_chatmaild(dist_dir=Path("chatmaild/dist"))
remote_base_dir = "/usr/local/lib/chatmaild"
remote_dist_file = f"{remote_base_dir}/dist/{dist_file.name}"
remote_venv_dir = f"{remote_base_dir}/venv"
root_owned = dict(user="root", group="root", mode="644")
apt.packages(
name="apt install python3-virtualenv",
packages=["python3-virtualenv"],
)
files.put(
name="Upload chatmaild source package",
deployer.ensure_directory(f"{remote_base_dir}/dist")
deployer.put_file(
src=dist_file.open("rb"),
dest=remote_dist_file,
create_remote_dir=True,
**root_owned,
)
pip.virtualenv(
@@ -122,32 +117,22 @@ def _install_remote_venv_with_chatmaild() -> None:
)
def _configure_remote_venv_with_chatmaild(config) -> None:
def _configure_remote_venv_with_chatmaild(deployer, config) -> None:
remote_base_dir = "/usr/local/lib/chatmaild"
remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini"
root_owned = dict(user="root", group="root", mode="644")
files.put(
name=f"Upload {remote_chatmail_inipath}",
deployer.put_file(
src=config._getbytefile(),
dest=remote_chatmail_inipath,
**root_owned,
)
files.file(
path="/etc/cron.d/chatmail-metrics",
present=False,
)
files.file(
path="/var/www/html/metrics",
present=False,
)
deployer.remove_file("/etc/cron.d/chatmail-metrics")
deployer.remove_file("/var/www/html/metrics")
class UnboundDeployer(Deployer):
def __init__(self, config):
self.config = config
self.need_restart = False
def install(self):
# On an IPv4-only system, if unbound is started but not configured,
@@ -176,13 +161,9 @@ class UnboundDeployer(Deployer):
)
# Configure unbound resolver with Quad9 fallback and a trailing newline
# (SolusVM bug).
files.put(
name="Write static resolv.conf",
self.put_file(
src=BytesIO(b"nameserver 127.0.0.1\nnameserver 9.9.9.9\n"),
dest="/etc/resolv.conf",
user="root",
group="root",
mode="644",
)
server.shell(
name="Generate root keys for validating DNSSEC",
@@ -191,26 +172,15 @@ class UnboundDeployer(Deployer):
],
)
if self.config.disable_ipv6:
files.directory(
self.ensure_directory(
path="/etc/unbound/unbound.conf.d",
present=True,
user="root",
group="root",
mode="755",
)
conf = files.put(
src=get_resource("unbound/unbound.conf.j2"),
dest="/etc/unbound/unbound.conf.d/chatmail.conf",
user="root",
group="root",
mode="644",
self.put_template(
"unbound/unbound.conf.j2",
"/etc/unbound/unbound.conf.d/chatmail.conf",
)
else:
conf = files.file(
path="/etc/unbound/unbound.conf.d/chatmail.conf",
present=False,
)
self.need_restart |= conf.changed
self.remove_file("/etc/unbound/unbound.conf.d/chatmail.conf")
def activate(self):
server.shell(
@@ -220,27 +190,19 @@ class UnboundDeployer(Deployer):
],
)
systemd.service(
name="Start and enable unbound",
service="unbound.service",
running=True,
enabled=True,
restarted=self.need_restart,
)
self.ensure_service("unbound.service")
class MtastsDeployer(Deployer):
def configure(self):
# Remove configuration.
files.file("/etc/mta-sts-daemon.yml", present=False)
files.directory("/usr/local/lib/postfix-mta-sts-resolver", present=False)
files.file("/etc/systemd/system/mta-sts-daemon.service", present=False)
self.remove_file("/etc/mta-sts-daemon.yml")
self.remove_directory("/usr/local/lib/postfix-mta-sts-resolver")
self.remove_file("/etc/systemd/system/mta-sts-daemon.service")
def activate(self):
systemd.service(
name="Stop MTA-STS daemon",
service="mta-sts-daemon.service",
daemon_reload=True,
self.ensure_service(
"mta-sts-daemon.service",
running=False,
enabled=False,
)
@@ -251,14 +213,7 @@ class WebsiteDeployer(Deployer):
self.config = config
def install(self):
files.directory(
name="Ensure /var/www exists",
path="/var/www",
user="root",
group="root",
mode="755",
present=True,
)
self.ensure_directory("/var/www")
def configure(self):
www_path, src_dir, build_dir = get_paths(self.config)
@@ -288,15 +243,11 @@ class LegacyRemoveDeployer(Deployer):
# remove historic expunge script
# which is now implemented through a systemd timer (chatmail-expire)
files.file(
path="/etc/cron.d/expunge",
present=False,
)
self.remove_file("/etc/cron.d/expunge")
# Remove OBS repository key that is no longer used.
files.file("/etc/apt/keyrings/obs-home-deltachat.gpg", present=False)
files.line(
name="Remove DeltaChat OBS home repository from sources.list",
self.remove_file("/etc/apt/keyrings/obs-home-deltachat.gpg")
self.ensure_line(
path="/etc/apt/sources.list",
line="deb [signed-by=/etc/apt/keyrings/obs-home-deltachat.gpg] https://download.opensuse.org/repositories/home:/deltachat/Debian_12/ ./",
escape_regex_characters=True,
@@ -304,11 +255,7 @@ class LegacyRemoveDeployer(Deployer):
)
# prior relay versions used filelogging
files.directory(
name="Ensure old logs on disk are deleted",
path="/var/log/journal/",
present=False,
)
self.remove_directory("/var/log/journal/")
# remove echobot if it is still running
if has_systemd() and host.get_fact(SystemdEnabled).get("echobot.service"):
systemd.service(
@@ -350,22 +297,13 @@ class TurnDeployer(Deployer):
"0fb3e792419494e21ecad536464929dba706bb2c88884ed8f1788141d26fc756",
),
}[host.get_fact(facts.server.Arch)]
existing_sha256sum = host.get_fact(Sha256File, "/usr/local/bin/chatmail-turn")
if existing_sha256sum != sha256sum:
server.shell(
name="Download chatmail-turn",
commands=[
f"(curl -L {url} >/usr/local/bin/chatmail-turn.new && (echo '{sha256sum} /usr/local/bin/chatmail-turn.new' | sha256sum -c) && mv /usr/local/bin/chatmail-turn.new /usr/local/bin/chatmail-turn)",
"chmod 755 /usr/local/bin/chatmail-turn",
],
)
self.download_executable(url, "/usr/local/bin/chatmail-turn", sha256sum)
def configure(self):
configure_remote_units(self.mail_domain, self.units)
configure_remote_units(self, self.mail_domain, self.units)
def activate(self):
activate_remote_units(self.units)
activate_remote_units(self, self.units)
class IrohDeployer(Deployer):
@@ -383,72 +321,30 @@ class IrohDeployer(Deployer):
"f8ef27631fac213b3ef668d02acd5b3e215292746a3fc71d90c63115446008b1",
),
}[host.get_fact(facts.server.Arch)]
existing_sha256sum = host.get_fact(Sha256File, "/usr/local/bin/iroh-relay")
if existing_sha256sum != sha256sum:
server.shell(
name="Download iroh-relay",
commands=[
f"(curl -L {url} | gunzip | tar -x -f - ./iroh-relay -O >/usr/local/bin/iroh-relay.new && (echo '{sha256sum} /usr/local/bin/iroh-relay.new' | sha256sum -c) && mv /usr/local/bin/iroh-relay.new /usr/local/bin/iroh-relay)",
"chmod 755 /usr/local/bin/iroh-relay",
],
self.download_executable(
url,
"/usr/local/bin/iroh-relay",
sha256sum,
extract="gunzip | tar -xf - ./iroh-relay -O",
)
self.need_restart = True
def configure(self):
systemd_unit = files.put(
name="Upload iroh-relay systemd unit",
src=get_resource("iroh-relay.service"),
dest="/etc/systemd/system/iroh-relay.service",
user="root",
group="root",
mode="644",
)
self.need_restart |= systemd_unit.changed
iroh_config = files.put(
name="Upload iroh-relay config",
src=get_resource("iroh-relay.toml"),
dest="/etc/iroh-relay.toml",
user="root",
group="root",
mode="644",
)
self.need_restart |= iroh_config.changed
self.ensure_systemd_unit("iroh-relay.service")
self.put_file("iroh-relay.toml", "/etc/iroh-relay.toml")
def activate(self):
systemd.service(
name="Start and enable iroh-relay",
service="iroh-relay.service",
running=True,
self.ensure_service(
"iroh-relay.service",
enabled=self.enable_iroh_relay,
restarted=self.need_restart,
)
self.need_restart = False
class JournaldDeployer(Deployer):
def configure(self):
journald_conf = files.put(
name="Configure journald",
src=get_resource("journald.conf"),
dest="/etc/systemd/journald.conf",
user="root",
group="root",
mode="644",
)
self.need_restart = journald_conf.changed
self.put_file("journald.conf", "/etc/systemd/journald.conf")
def activate(self):
systemd.service(
name="Start and enable journald",
service="systemd-journald.service",
running=True,
enabled=True,
restarted=self.need_restart,
)
self.need_restart = False
self.ensure_service("systemd-journald.service")
class ChatmailVenvDeployer(Deployer):
@@ -464,14 +360,14 @@ class ChatmailVenvDeployer(Deployer):
)
def install(self):
_install_remote_venv_with_chatmaild()
_install_remote_venv_with_chatmaild(self)
def configure(self):
_configure_remote_venv_with_chatmaild(self.config)
configure_remote_units(self.config.mail_domain, self.units)
_configure_remote_venv_with_chatmaild(self, self.config)
configure_remote_units(self, self.config.mail_domain, self.units)
def activate(self):
activate_remote_units(self.units)
activate_remote_units(self, self.units)
class ChatmailDeployer(Deployer):
@@ -485,13 +381,9 @@ class ChatmailDeployer(Deployer):
self.mail_domain = config.mail_domain
def install(self):
files.put(
name="Disable installing recommended packages globally",
self.put_file(
src=BytesIO(b'APT::Install-Recommends "false";\n'),
dest="/etc/apt/apt.conf.d/00InstallRecommends",
user="root",
group="root",
mode="644",
)
apt.update(name="apt update", cache_time=24 * 3600)
apt.upgrade(name="upgrade apt packages", auto_remove=True)
@@ -508,13 +400,10 @@ class ChatmailDeployer(Deployer):
def configure(self):
# metadata crashes if the mailboxes dir does not exist
files.directory(
name="Ensure vmail mailbox directory exists",
path=str(self.config.mailboxes_dir),
user="vmail",
group="vmail",
self.ensure_directory(
str(self.config.mailboxes_dir),
owner="vmail",
mode="700",
present=True,
)
# This file is used by auth proxy.
@@ -535,12 +424,7 @@ class FcgiwrapDeployer(Deployer):
)
def activate(self):
systemd.service(
name="Start and enable fcgiwrap",
service="fcgiwrap.service",
running=True,
enabled=True,
)
self.ensure_service("fcgiwrap.service")
class GithashDeployer(Deployer):
@@ -553,12 +437,7 @@ class GithashDeployer(Deployer):
git_diff = subprocess.check_output(["git", "diff"]).decode()
except Exception:
git_diff = ""
files.put(
name="Upload chatmail relay git commit hash",
src=StringIO(git_hash + git_diff),
dest="/etc/chatmail-version",
mode="700",
)
self.put_file(src=StringIO(git_hash + git_diff), dest="/etc/chatmail-version")
def get_tls_deployer(config, mail_domain):
@@ -591,11 +470,17 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
return
# Check if mtail_address interface is available (if configured)
if config.mtail_address and config.mtail_address not in ('127.0.0.1', '::1', 'localhost'):
if config.mtail_address and config.mtail_address not in (
"127.0.0.1",
"::1",
"localhost",
):
ipv4_addrs = host.get_fact(hardware.Ipv4Addrs)
all_addresses = [addr for addrs in ipv4_addrs.values() for addr in addrs]
if config.mtail_address not in all_addresses:
Out().red(f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n")
Out().red(
f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n"
)
exit(1)
if not is_in_container():

View File

@@ -5,14 +5,13 @@ from chatmaild.config import Config
from pyinfra import host
from pyinfra.facts.deb import DebPackages
from pyinfra.facts.server import Arch, Command, Sysctl
from pyinfra.operations import apt, files, server, systemd
from pyinfra.operations import apt, files, server
from cmdeploy.basedeploy import (
Deployer,
activate_remote_units,
blocked_service_startup,
configure_remote_units,
get_resource,
is_in_container,
)
@@ -59,26 +58,21 @@ class DovecotDeployer(Deployer):
],
)
self.need_restart = True
files.put(
name="Pin dovecot packages to block Debian dist-upgrades",
self.put_file(
src=io.StringIO(
"Package: dovecot-*\n"
"Pin: version *\n"
"Pin-Priority: -1\n"
),
dest="/etc/apt/preferences.d/pin-dovecot",
user="root",
group="root",
mode="644",
)
def configure(self):
configure_remote_units(self.config.mail_domain, self.units)
config_restart, self.daemon_reload = _configure_dovecot(self.config)
self.need_restart |= config_restart
configure_remote_units(self, self.config.mail_domain, self.units)
_configure_dovecot(self, self.config)
def activate(self):
activate_remote_units(self.units)
activate_remote_units(self, self.units)
# Detect stale binary: package installed but service still runs old (deleted) binary.
if not self.disable_mail and not self.need_restart:
@@ -91,19 +85,12 @@ class DovecotDeployer(Deployer):
if stale == "STALE":
self.need_restart = True
restart = False if self.disable_mail else self.need_restart
systemd.service(
name="Disable dovecot for now"
if self.disable_mail
else "Start and enable Dovecot",
service="dovecot.service",
running=False if self.disable_mail else True,
enabled=False if self.disable_mail else True,
restarted=restart,
daemon_reload=self.daemon_reload,
active = not self.disable_mail
self.ensure_service(
"dovecot.service",
running=active,
enabled=active,
)
self.need_restart = False
def _pick_url(primary, fallback):
@@ -147,39 +134,19 @@ def _download_dovecot_package(package: str, arch: str) -> tuple[str | None, bool
return deb_filename, True
def _configure_dovecot(config: Config, debug: bool = False) -> tuple[bool, bool]:
def _configure_dovecot(deployer, config: Config, debug: bool = False):
"""Configures Dovecot IMAP server."""
need_restart = False
daemon_reload = False
main_config = files.template(
src=get_resource("dovecot/dovecot.conf.j2"),
dest="/etc/dovecot/dovecot.conf",
user="root",
group="root",
mode="644",
deployer.put_template(
"dovecot/dovecot.conf.j2",
"/etc/dovecot/dovecot.conf",
config=config,
debug=debug,
disable_ipv6=config.disable_ipv6,
)
need_restart |= main_config.changed
auth_config = files.put(
src=get_resource("dovecot/auth.conf"),
dest="/etc/dovecot/auth.conf",
user="root",
group="root",
mode="644",
deployer.put_file("dovecot/auth.conf", "/etc/dovecot/auth.conf")
deployer.put_file(
"dovecot/push_notification.lua", "/etc/dovecot/push_notification.lua"
)
need_restart |= auth_config.changed
lua_push_notification_script = files.put(
src=get_resource("dovecot/push_notification.lua"),
dest="/etc/dovecot/push_notification.lua",
user="root",
group="root",
mode="644",
)
need_restart |= lua_push_notification_script.changed
# as per https://doc.dovecot.org/2.3/configuration_manual/os/
# it is recommended to set the following inotify limits
@@ -203,25 +170,20 @@ def _configure_dovecot(config: Config, debug: bool = False) -> tuple[bool, bool]
persist=True,
)
timezone_env = files.line(
deployer.ensure_line(
name="Set TZ environment variable",
path="/etc/environment",
line="TZ=:/etc/localtime",
)
need_restart |= timezone_env.changed
restart_conf = files.put(
name="dovecot: restart automatically on failure",
src=get_resource("service/10_restart.conf"),
dest="/etc/systemd/system/dovecot.service.d/10_restart.conf",
deployer.put_file(
"service/10_restart_on_failure.conf",
"/etc/systemd/system/dovecot.service.d/10_restart.conf",
)
daemon_reload |= restart_conf.changed
# Validate dovecot configuration before restart
if need_restart:
if deployer.need_restart:
server.shell(
name="Validate dovecot configuration",
commands=["doveconf -n >/dev/null"],
)
return need_restart, daemon_reload

View File

@@ -1,10 +1,8 @@
import io
from pyinfra import host
from pyinfra.facts.files import File
from pyinfra.operations import files, systemd
from cmdeploy.basedeploy import Deployer, get_resource
from ..basedeploy import Deployer
class ExternalTlsDeployer(Deployer):
@@ -23,45 +21,24 @@ class ExternalTlsDeployer(Deployer):
def configure(self):
# Verify cert and key exist on the remote host using pyinfra facts.
for path in (self.cert_path, self.key_path):
info = host.get_fact(File, path=path)
if info is None:
if host.get_fact(File, path=path) is None:
raise Exception(f"External TLS file not found on server: {path}")
# Deploy the .path unit (templated with the cert path).
# pkg=__package__ is required here because the resource files
# live in cmdeploy.external, not the default cmdeploy package.
source = get_resource("tls-cert-reload.path.f", pkg=__package__)
content = source.read_text().format(cert_path=self.cert_path).encode()
path_unit = files.put(
name="Upload tls-cert-reload.path",
src=io.BytesIO(content),
dest="/etc/systemd/system/tls-cert-reload.path",
user="root",
group="root",
mode="644",
self.ensure_systemd_unit(
"external/tls-cert-reload.path.j2",
cert_path=self.cert_path,
)
service_unit = files.put(
name="Upload tls-cert-reload.service",
src=get_resource("tls-cert-reload.service", pkg=__package__),
dest="/etc/systemd/system/tls-cert-reload.service",
user="root",
group="root",
mode="644",
self.ensure_systemd_unit(
"external/tls-cert-reload.service",
)
if path_unit.changed or service_unit.changed:
self.need_restart = True
def activate(self):
systemd.service(
name="Enable tls-cert-reload path watcher",
service="tls-cert-reload.path",
running=True,
enabled=True,
restarted=self.need_restart,
daemon_reload=self.need_restart,
)
# No explicit reload needed here: dovecot/nginx read the cert
# on startup, and the .path watcher handles live changes.
self.ensure_service(
"tls-cert-reload.path",
running=True,
enabled=True,
)

View File

@@ -9,7 +9,7 @@
Description=Watch TLS certificate for changes
[Path]
PathChanged={cert_path}
PathChanged={{ cert_path }}
[Install]
WantedBy=multi-user.target

View File

@@ -1,9 +1,8 @@
import os
from pyinfra import facts, host
from pyinfra.operations import files, systemd
from cmdeploy.basedeploy import Deployer, get_resource
from cmdeploy.basedeploy import Deployer
class FiltermailDeployer(Deployer):
@@ -11,18 +10,13 @@ class FiltermailDeployer(Deployer):
bin_path = "/usr/local/bin/filtermail"
config_path = "/usr/local/lib/chatmaild/chatmail.ini"
def __init__(self):
self.need_restart = False
def install(self):
local_bin = os.environ.get("CHATMAIL_FILTERMAIL_BINARY")
if local_bin:
self.need_restart |= files.put(
name="Upload locally built filtermail",
self.put_executable(
src=local_bin,
dest=self.bin_path,
mode="755",
).changed
)
return
arch = host.get_fact(facts.server.Arch)
@@ -31,34 +25,16 @@ class FiltermailDeployer(Deployer):
"x86_64": "5295115952c72e4c4ec3c85546e094b4155a4c702c82bd71fcdcb744dc73adf6",
"aarch64": "6892244f17b8f26ccb465766e96028e7222b3c8adefca9fc6bfe9ff332ca8dff",
}[arch]
self.need_restart |= files.download(
name="Download filtermail",
src=url,
sha256sum=sha256sum,
dest=self.bin_path,
mode="755",
).changed
self.download_executable(url, self.bin_path, sha256sum)
def configure(self):
for service in self.services:
self.need_restart |= files.template(
src=get_resource(f"filtermail/{service}.service.j2"),
dest=f"/etc/systemd/system/{service}.service",
user="root",
group="root",
mode="644",
self.ensure_systemd_unit(
f"filtermail/{service}.service.j2",
bin_path=self.bin_path,
config_path=self.config_path,
).changed
)
def activate(self):
for service in self.services:
systemd.service(
name=f"Start and enable {service}",
service=f"{service}.service",
running=True,
enabled=True,
restarted=self.need_restart,
daemon_reload=True,
)
self.need_restart = False
self.ensure_service(f"{service}.service")

View File

@@ -1,10 +1,7 @@
from pyinfra import facts, host
from pyinfra.operations import apt, files, server, systemd
from pyinfra.operations import apt
from cmdeploy.basedeploy import (
Deployer,
get_resource,
)
from cmdeploy.basedeploy import Deployer
class MtailDeployer(Deployer):
@@ -18,53 +15,30 @@ class MtailDeployer(Deployer):
(url, sha256sum) = {
"x86_64": (
"https://github.com/google/mtail/releases/download/v3.0.8/mtail_3.0.8_linux_amd64.tar.gz",
"123c2ee5f48c3eff12ebccee38befd2233d715da736000ccde49e3d5607724e4",
"d55cb601049c5e61eabab29998dbbcea95d480e5448544f9470337ba2eea882e",
),
"aarch64": (
"https://github.com/google/mtail/releases/download/v3.0.8/mtail_3.0.8_linux_arm64.tar.gz",
"aa04811c0929b6754408676de520e050c45dddeb3401881888a092c9aea89cae",
"f748db8ad2a1e0b63684d4c8868cf6a373a20f7e6922e5ece601fff0ee00eb1a",
),
}[host.get_fact(facts.server.Arch)]
server.shell(
name="Download mtail",
commands=[
f"(echo '{sha256sum} /usr/local/bin/mtail' | sha256sum -c) || (curl -L {url} | gunzip | tar -x -f - mtail -O >/usr/local/bin/mtail.new && mv /usr/local/bin/mtail.new /usr/local/bin/mtail)",
"chmod 755 /usr/local/bin/mtail",
],
self.download_executable(
url,
"/usr/local/bin/mtail",
sha256sum,
extract="gunzip | tar -xf - mtail -O",
)
def configure(self):
# Using our own systemd unit instead of `/usr/lib/systemd/system/mtail.service`.
# This allows to read from journalctl instead of log files.
unit = files.template(
src=get_resource("mtail/mtail.service.j2"),
dest="/etc/systemd/system/mtail.service",
user="root",
group="root",
mode="644",
self.ensure_systemd_unit(
"mtail/mtail.service.j2",
address=self.mtail_address or "127.0.0.1",
port=3903,
)
mtail_conf = files.put(
name="Mtail configuration",
src=get_resource("mtail/delivered_mail.mtail"),
dest="/etc/mtail/delivered_mail.mtail",
user="root",
group="root",
mode="644",
)
self.unit_changed = unit.changed
self.need_restart = unit.changed or mtail_conf.changed
self.put_file("mtail/delivered_mail.mtail", "/etc/mtail/delivered_mail.mtail")
def activate(self):
systemd.service(
name="Start and enable mtail",
service="mtail.service",
running=bool(self.mtail_address),
enabled=bool(self.mtail_address),
restarted=self.need_restart,
daemon_reload=self.unit_changed,
)
self.need_restart = False
active = bool(self.mtail_address)
self.ensure_service("mtail.service", running=active, enabled=active)

View File

@@ -1,5 +1,5 @@
from chatmaild.config import Config
from pyinfra.operations import apt, files, systemd
from pyinfra.operations import apt
from cmdeploy.basedeploy import (
Deployer,
@@ -31,87 +31,50 @@ class NginxDeployer(Deployer):
# For documentation about policy-rc.d, see:
# https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
#
files.put(
src=get_resource("policy-rc.d"),
dest="/usr/sbin/policy-rc.d",
user="root",
group="root",
mode="755",
)
self.put_executable(src="policy-rc.d", dest="/usr/sbin/policy-rc.d")
apt.packages(
name="Install nginx",
packages=["nginx", "libnginx-mod-stream"],
)
files.file("/usr/sbin/policy-rc.d", present=False)
self.remove_file("/usr/sbin/policy-rc.d")
def configure(self):
self.need_restart = _configure_nginx(self.config)
_configure_nginx(self, self.config)
def activate(self):
systemd.service(
name="Start and enable nginx",
service="nginx.service",
running=True,
enabled=True,
restarted=self.need_restart,
)
self.need_restart = False
self.ensure_service("nginx.service")
def _configure_nginx(config: Config, debug: bool = False) -> bool:
def _configure_nginx(deployer, config: Config, debug: bool = False):
"""Configures nginx HTTP server."""
need_restart = False
main_config = files.template(
src=get_resource("nginx/nginx.conf.j2"),
dest="/etc/nginx/nginx.conf",
user="root",
group="root",
mode="644",
deployer.put_template(
"nginx/nginx.conf.j2",
"/etc/nginx/nginx.conf",
config=config,
disable_ipv6=config.disable_ipv6,
)
need_restart |= main_config.changed
autoconfig = files.template(
src=get_resource("nginx/autoconfig.xml.j2"),
dest="/var/www/html/.well-known/autoconfig/mail/config-v1.1.xml",
user="root",
group="root",
mode="644",
deployer.put_template(
"nginx/autoconfig.xml.j2",
"/var/www/html/.well-known/autoconfig/mail/config-v1.1.xml",
config=config,
)
need_restart |= autoconfig.changed
mta_sts_config = files.template(
src=get_resource("nginx/mta-sts.txt.j2"),
dest="/var/www/html/.well-known/mta-sts.txt",
user="root",
group="root",
mode="644",
deployer.put_template(
"nginx/mta-sts.txt.j2",
"/var/www/html/.well-known/mta-sts.txt",
config=config,
)
need_restart |= mta_sts_config.changed
# install CGI newemail script
#
cgi_dir = "/usr/lib/cgi-bin"
files.directory(
name=f"Ensure {cgi_dir} exists",
path=cgi_dir,
user="root",
group="root",
)
deployer.ensure_directory(cgi_dir)
files.put(
name="Upload cgi newemail.py script",
deployer.put_executable(
src=get_resource("newemail.py", pkg="chatmaild").open("rb"),
dest=f"{cgi_dir}/newemail.py",
user="root",
group="root",
mode="755",
)
return need_restart

View File

@@ -4,9 +4,9 @@ Installs OpenDKIM
from pyinfra import host
from pyinfra.facts.files import File
from pyinfra.operations import apt, files, server, systemd
from pyinfra.operations import apt, files, server
from cmdeploy.basedeploy import Deployer, get_resource
from cmdeploy.basedeploy import Deployer
class OpendkimDeployer(Deployer):
@@ -25,65 +25,39 @@ class OpendkimDeployer(Deployer):
domain = self.mail_domain
dkim_selector = "opendkim"
"""Configures OpenDKIM"""
need_restart = False
main_config = files.template(
src=get_resource("opendkim/opendkim.conf"),
dest="/etc/opendkim.conf",
user="root",
group="root",
mode="644",
self.put_template(
"opendkim/opendkim.conf",
"/etc/opendkim.conf",
config={"domain_name": domain, "opendkim_selector": dkim_selector},
)
need_restart |= main_config.changed
screen_script = files.file(
path="/etc/opendkim/screen.lua",
present=False,
)
need_restart |= screen_script.changed
self.remove_file("/etc/opendkim/screen.lua")
self.remove_file("/etc/opendkim/final.lua")
final_script = files.file(
path="/etc/opendkim/final.lua",
present=False,
)
need_restart |= final_script.changed
files.directory(
name="Add opendkim directory to /etc",
path="/etc/opendkim",
user="opendkim",
group="opendkim",
self.ensure_directory(
"/etc/opendkim",
owner="opendkim",
mode="750",
present=True,
)
keytable = files.template(
src=get_resource("opendkim/KeyTable"),
dest="/etc/dkimkeys/KeyTable",
user="opendkim",
group="opendkim",
mode="644",
self.put_template(
"opendkim/KeyTable",
"/etc/dkimkeys/KeyTable",
owner="opendkim",
config={"domain_name": domain, "opendkim_selector": dkim_selector},
)
need_restart |= keytable.changed
signing_table = files.template(
src=get_resource("opendkim/SigningTable"),
dest="/etc/dkimkeys/SigningTable",
user="opendkim",
group="opendkim",
mode="644",
self.put_template(
"opendkim/SigningTable",
"/etc/dkimkeys/SigningTable",
owner="opendkim",
config={"domain_name": domain, "opendkim_selector": dkim_selector},
)
need_restart |= signing_table.changed
files.directory(
name="Add opendkim socket directory to /var/spool/postfix",
path="/var/spool/postfix/opendkim",
user="opendkim",
group="opendkim",
self.ensure_directory(
"/var/spool/postfix/opendkim",
owner="opendkim",
mode="750",
present=True,
)
if not host.get_fact(File, f"/etc/dkimkeys/{dkim_selector}.private"):
@@ -96,12 +70,10 @@ class OpendkimDeployer(Deployer):
_su_user="opendkim",
)
service_file = files.put(
name="Configure opendkim to restart once a day",
src=get_resource("opendkim/systemd.conf"),
dest="/etc/systemd/system/opendkim.service.d/10-prevent-memory-leak.conf",
self.put_file(
"opendkim/systemd.conf",
"/etc/systemd/system/opendkim.service.d/10-prevent-memory-leak.conf",
)
need_restart |= service_file.changed
files.file(
name="chown opendkim: /etc/dkimkeys/opendkim.private",
@@ -110,15 +82,5 @@ class OpendkimDeployer(Deployer):
group="opendkim",
)
self.need_restart = need_restart
def activate(self):
systemd.service(
name="Start and enable OpenDKIM",
service="opendkim.service",
running=True,
enabled=True,
daemon_reload=self.need_restart,
restarted=self.need_restart,
)
self.need_restart = False
self.ensure_service("opendkim.service")

View File

@@ -1,11 +1,10 @@
from pyinfra.operations import apt, files, server, systemd
from pyinfra.operations import apt, server
from cmdeploy.basedeploy import Deployer, get_resource
from cmdeploy.basedeploy import Deployer
class PostfixDeployer(Deployer):
required_users = [("postfix", None, ["opendkim"])]
daemon_reload = False
def __init__(self, config, disable_mail):
self.config = config
@@ -19,81 +18,46 @@ class PostfixDeployer(Deployer):
def configure(self):
config = self.config
need_restart = False
main_config = files.template(
src=get_resource("postfix/main.cf.j2"),
dest="/etc/postfix/main.cf",
user="root",
group="root",
mode="644",
self.put_template(
"postfix/main.cf.j2",
"/etc/postfix/main.cf",
config=config,
disable_ipv6=config.disable_ipv6,
)
need_restart |= main_config.changed
master_config = files.template(
src=get_resource("postfix/master.cf.j2"),
dest="/etc/postfix/master.cf",
user="root",
group="root",
mode="644",
self.put_template(
"postfix/master.cf.j2",
"/etc/postfix/master.cf",
debug=False,
config=config,
)
need_restart |= master_config.changed
header_cleanup = files.put(
src=get_resource("postfix/submission_header_cleanup"),
dest="/etc/postfix/submission_header_cleanup",
user="root",
group="root",
mode="644",
self.put_file(
"postfix/submission_header_cleanup",
"/etc/postfix/submission_header_cleanup",
)
need_restart |= header_cleanup.changed
self.put_file("postfix/lmtp_header_cleanup", "/etc/postfix/lmtp_header_cleanup")
lmtp_header_cleanup = files.put(
src=get_resource("postfix/lmtp_header_cleanup"),
dest="/etc/postfix/lmtp_header_cleanup",
user="root",
group="root",
mode="644",
res = self.put_file(
"postfix/smtp_tls_policy_map", "/etc/postfix/smtp_tls_policy_map"
)
need_restart |= lmtp_header_cleanup.changed
tls_policy_map = files.put(
name="Upload SMTP TLS Policy that accepts self-signed certificates for IP-only hosts",
src=get_resource("postfix/smtp_tls_policy_map"),
dest="/etc/postfix/smtp_tls_policy_map",
user="root",
group="root",
mode="644",
)
need_restart |= tls_policy_map.changed
if tls_policy_map.changed:
tls_policy_changed = res.changed
if tls_policy_changed:
server.shell(
commands=["postmap /etc/postfix/smtp_tls_policy_map"],
)
# Login map that 1:1 maps email address to login.
login_map = files.put(
src=get_resource("postfix/login_map"),
dest="/etc/postfix/login_map",
user="root",
group="root",
mode="644",
)
need_restart |= login_map.changed
self.put_file("postfix/login_map", "/etc/postfix/login_map")
restart_conf = files.put(
name="postfix: restart automatically on failure",
src=get_resource("service/10_restart.conf"),
dest="/etc/systemd/system/postfix@.service.d/10_restart.conf",
self.put_file(
"service/10_restart_on_failure.conf",
"/etc/systemd/system/postfix@.service.d/10_restart.conf",
)
self.daemon_reload = restart_conf.changed
# Validate postfix configuration before restart
if need_restart:
if self.need_restart:
server.shell(
name="Validate postfix configuration",
# Extract stderr and quit with error if non-zero
@@ -101,19 +65,11 @@ class PostfixDeployer(Deployer):
"""bash -c 'w=$(postconf 2>&1 >/dev/null); [[ -z "$w" ]] || { echo "$w"; false; }'"""
],
)
self.need_restart = need_restart
def activate(self):
restart = False if self.disable_mail else self.need_restart
systemd.service(
name="disable postfix for now"
if self.disable_mail
else "Start and enable Postfix",
service="postfix.service",
running=False if self.disable_mail else True,
enabled=False if self.disable_mail else True,
restarted=restart,
daemon_reload=self.daemon_reload,
active = not self.disable_mail
self.ensure_service(
"postfix.service",
running=active,
enabled=active,
)
self.need_restart = False

View File

@@ -1,8 +1,8 @@
import shlex
from pyinfra.operations import apt, server
from pyinfra.operations import server
from cmdeploy.basedeploy import Deployer
from ..basedeploy import Deployer
def openssl_selfsigned_args(domain, cert_path, key_path, days=36500):
@@ -34,11 +34,7 @@ class SelfSignedTlsDeployer(Deployer):
self.cert_path = "/etc/ssl/certs/mailserver.pem"
self.key_path = "/etc/ssl/private/mailserver.key"
def install(self):
apt.packages(
name="Install openssl",
packages=["openssl"],
)
def configure(self):
args = openssl_selfsigned_args(
@@ -52,3 +48,5 @@ class SelfSignedTlsDeployer(Deployer):
def activate(self):
pass

View File

@@ -0,0 +1,118 @@
from unittest.mock import MagicMock, patch
from cmdeploy.basedeploy import Deployer
def test_put_file_restart_and_reload():
deployer = Deployer()
mock_res = MagicMock()
mock_res.changed = True
with patch("cmdeploy.basedeploy.files.put", return_value=mock_res):
deployer.put_file("foo.conf", "/etc/foo.conf")
assert deployer.need_restart is True
assert deployer.daemon_reload is False
deployer = Deployer()
deployer.put_file("test.service", "/etc/systemd/system/test.service")
assert deployer.need_restart is True
assert deployer.daemon_reload is True
def test_remove_file():
deployer = Deployer()
mock_res = MagicMock()
mock_res.changed = True
with patch("cmdeploy.basedeploy.files.file", return_value=mock_res) as mock_file:
deployer.remove_file("/etc/foo.conf")
mock_file.assert_called_once_with(
name="Remove /etc/foo.conf", path="/etc/foo.conf", present=False
)
assert deployer.need_restart is True
def test_ensure_systemd_unit():
deployer = Deployer()
mock_res = MagicMock()
mock_res.changed = True
# Plain service file
with patch("cmdeploy.basedeploy.files.put", return_value=mock_res) as mock_put:
deployer.ensure_systemd_unit("iroh-relay.service")
assert (
mock_put.call_args.kwargs["dest"]
== "/etc/systemd/system/iroh-relay.service"
)
assert deployer.need_restart is True
assert deployer.daemon_reload is True
deployer = Deployer()
# Template (.j2) dispatches to put_template and strips .j2 suffix
with patch("cmdeploy.basedeploy.files.template", return_value=mock_res) as mock_tpl:
deployer.ensure_systemd_unit(
"filtermail/chatmaild.service.j2",
bin_path="/usr/local/bin/filtermail",
)
assert (
mock_tpl.call_args.kwargs["dest"] == "/etc/systemd/system/chatmaild.service"
)
deployer = Deployer()
# Explicit dest_name override
with patch("cmdeploy.basedeploy.files.put", return_value=mock_res) as mock_put:
deployer.ensure_systemd_unit(
"acmetool/acmetool-reconcile.timer",
dest_name="acmetool-reconcile.timer",
)
assert (
mock_put.call_args.kwargs["dest"]
== "/etc/systemd/system/acmetool-reconcile.timer"
)
def test_ensure_service():
with patch("cmdeploy.basedeploy.systemd.service") as mock_svc:
deployer = Deployer()
deployer.need_restart = True
deployer.daemon_reload = True
deployer.ensure_service("nginx.service")
mock_svc.assert_called_once_with(
name="Start and enable nginx.service",
service="nginx.service",
running=True,
enabled=True,
restarted=True,
daemon_reload=True,
)
# daemon_reload is cleared to avoid multiple systemctl daemon-reload calls
# need_restart is kept to ensure all subsequent services also restart
assert deployer.need_restart is True
assert deployer.daemon_reload is False
with patch("cmdeploy.basedeploy.systemd.service") as mock_svc:
# Stopping suppresses restarted even when need_restart is True
deployer = Deployer()
deployer.need_restart = True
deployer.daemon_reload = True
deployer.ensure_service(
"mta-sts-daemon.service",
running=False,
enabled=False,
)
assert mock_svc.call_args.kwargs["restarted"] is False
assert deployer.need_restart is True
with patch("cmdeploy.basedeploy.systemd.service") as mock_svc:
# Multiple calls: daemon_reload resets after first, need_restart persists
deployer = Deployer()
deployer.need_restart = True
deployer.daemon_reload = True
deployer.ensure_service("chatmaild.service")
deployer.ensure_service("chatmaild-metadata.service")
second_call = mock_svc.call_args_list[1]
assert second_call.kwargs["restarted"] is True
assert second_call.kwargs["daemon_reload"] is False