docs: Remove section about use of objects

- Revised cmdeploy documentation in doc/source/overview.rst to reflect
  the recent revisions to the Deployer interface.

CHANGELOG.md

@@ -2,12 +2,6 @@
 
 ## untagged
 
-- Add robots.txt to exclude all web crawlers
-  ([#732](https://github.com/chatmail/relay/pull/732))
-
-- acmetool: accept new Let's Encrypt ToS: https://letsencrypt.org/documents/LE-SA-v1.6-August-18-2025.pdf
-  ([#729](https://github.com/chatmail/relay/pull/729))
-
 - Organized cmdeploy into install, configure, and activate stages
   ([#695](https://github.com/chatmail/relay/pull/695))
 
@@ -27,10 +21,10 @@
   ([#689](https://github.com/chatmail/relay/pull/689))
 
 - Require TLS 1.2 for outgoing SMTP connections
-  ([#685](https://github.com/chatmail/relay/pull/685), [#730](https://github.com/chatmail/relay/pull/730))
+  ([#685](https://github.com/chatmail/relay/pull/685))
 
 - require STARTTLS for incoming port 25 connections
-  ([#684](https://github.com/chatmail/relay/pull/684), [#730](https://github.com/chatmail/relay/pull/730))
+  ([#684](https://github.com/chatmail/relay/pull/684))
 
 - filtermail: run CPU-intensive handle_DATA in a thread pool executor
   ([#676](https://github.com/chatmail/relay/pull/676))

chatmaild/pyproject.toml

@@ -73,6 +73,5 @@ commands =
 deps = pytest 
        pdbpp 
        pytest-localserver
-       execnet
 commands = pytest -v -rsXx {posargs}
 """

cmdeploy/src/cmdeploy/acmetool/__init__.py

@@ -2,7 +2,7 @@ import importlib.resources
 
 from pyinfra.operations import apt, files, server, systemd
 
-from ..basedeploy import Deployer
+from ..deployer import Deployer
 
 
 class AcmetoolDeployer(Deployer):
@@ -27,9 +27,7 @@ class AcmetoolDeployer(Deployer):
 
         files.put(
             name="Install acmetool hook.",
-            src=importlib.resources.files(__package__)
+            src=importlib.resources.files(__package__).joinpath("acmetool.hook").open("rb"),
-            .joinpath("acmetool.hook")
-            .open("rb"),
             dest="/etc/acme/hooks/nginx",
             user="root",
             group="root",
@@ -43,9 +41,7 @@ class AcmetoolDeployer(Deployer):
 
     def configure(self):
         files.template(
-            src=importlib.resources.files(__package__).joinpath(
+            src=importlib.resources.files(__package__).joinpath("response-file.yaml.j2"),
-                "response-file.yaml.j2"
-            ),
             dest="/var/lib/acme/conf/responses",
             user="root",
             group="root",
@@ -84,9 +80,7 @@ class AcmetoolDeployer(Deployer):
         self.need_restart_reconcile_service = reconcile_service_file.changed
 
         reconcile_timer_file = files.put(
-            src=importlib.resources.files(__package__).joinpath(
+            src=importlib.resources.files(__package__).joinpath("acmetool-reconcile.timer"),
-                "acmetool-reconcile.timer"
-            ),
             dest="/etc/systemd/system/acmetool-reconcile.timer",
             user="root",
             group="root",

cmdeploy/src/cmdeploy/acmetool/response-file.yaml.j2

@@ -1,2 +1,2 @@
 "acme-enter-email": "{{ email }}"
-"acme-agreement:https://letsencrypt.org/documents/LE-SA-v1.6-August-18-2025.pdf": true
+"acme-agreement:https://letsencrypt.org/documents/LE-SA-v1.5-February-24-2025.pdf": true

cmdeploy/src/cmdeploy/basedeploy.py

@@ -1,112 +0,0 @@
-import importlib.resources
-import io
-import os
-
-from pyinfra.operations import files, server, systemd
-
-
-def get_resource(arg, pkg=__package__):
-    return importlib.resources.files(pkg).joinpath(arg)
-
-
-def configure_remote_units(mail_domain, units) -> None:
-    remote_base_dir = "/usr/local/lib/chatmaild"
-    remote_venv_dir = f"{remote_base_dir}/venv"
-    remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini"
-    root_owned = dict(user="root", group="root", mode="644")
-
-    # install systemd units
-    for fn in units:
-        execpath = fn if fn != "filtermail-incoming" else "filtermail"
-        params = dict(
-            execpath=f"{remote_venv_dir}/bin/{execpath}",
-            config_path=remote_chatmail_inipath,
-            remote_venv_dir=remote_venv_dir,
-            mail_domain=mail_domain,
-        )
-
-        basename = fn if "." in fn else f"{fn}.service"
-
-        source_path = get_resource(f"service/{basename}.f")
-        content = source_path.read_text().format(**params).encode()
-
-        files.put(
-            name=f"Upload {basename}",
-            src=io.BytesIO(content),
-            dest=f"/etc/systemd/system/{basename}",
-            **root_owned,
-        )
-
-
-def activate_remote_units(units) -> None:
-    # activate systemd units
-    for fn in units:
-        basename = fn if "." in fn else f"{fn}.service"
-
-        if fn == "chatmail-expire" or fn == "chatmail-fsreport":
-            # don't auto-start but let the corresponding timer trigger execution
-            enabled = False
-        else:
-            enabled = True
-        systemd.service(
-            name=f"Setup {basename}",
-            service=basename,
-            running=enabled,
-            enabled=enabled,
-            restarted=enabled,
-            daemon_reload=True,
-        )
-
-
-class Deployment:
-    def install(self, deployer):
-        # optional 'required_users' contains a list of (user, group, secondary-group-list) tuples.
-        # If the group is None, no group is created corresponding to that user.
-        # If the secondary group list is not None, all listed groups are created as well.
-        required_users = getattr(deployer, "required_users", [])
-        for user, group, groups in required_users:
-            if group is not None:
-                server.group(
-                    name="Create {} group".format(group), group=group, system=True
-                )
-            if groups is not None:
-                for group2 in groups:
-                    server.group(
-                        name="Create {} group".format(group2), group=group2, system=True
-                    )
-            server.user(
-                name="Create {} user".format(user),
-                user=user,
-                group=group,
-                groups=groups,
-                system=True,
-            )
-
-        deployer.install()
-
-    def configure(self, deployer):
-        deployer.configure()
-
-    def activate(self, deployer):
-        deployer.activate()
-
-    def perform_stages(self, deployers):
-        default_stages = "install,configure,activate"
-        stages = os.getenv("CMDEPLOY_STAGES", default_stages).split(",")
-
-        for stage in stages:
-            for deployer in deployers:
-                getattr(self, stage)(deployer)
-
-
-class Deployer:
-    need_restart = False
-
-    def install(self):
-        pass
-
-    def configure(self):
-        pass
-
-    def activate(self):
-        pass

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -19,7 +19,7 @@ from packaging import version
 from termcolor import colored
 
 from . import dns, remote
-from .sshexec import LocalExec, SSHExec
+from .sshexec import SSHExec, LocalExec
 
 #
 # cmdeploy sub commands and options
@@ -95,7 +95,7 @@ def run_cmd(args, out):
     env["CHATMAIL_INI"] = args.inipath
     env["CHATMAIL_DISABLE_MAIL"] = "True" if args.disable_mail else ""
     env["CHATMAIL_REQUIRE_IROH"] = "True" if require_iroh else ""
-    deploy_path = importlib.resources.files(__package__).joinpath("run.py").resolve()
+    deploy_path = importlib.resources.files(__package__).joinpath("deploy.py").resolve()
     pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
 
     cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y"
@@ -238,12 +238,7 @@ def fmt_cmd_options(parser):
 def fmt_cmd(args, out):
     """Run formattting fixes on all chatmail source code."""
 
-    chatmaild_dir = importlib.resources.files("chatmaild").resolve()
+    sources = [str(importlib.resources.files(x)) for x in ("chatmaild", "cmdeploy")]
-    cmdeploy_dir = chatmaild_dir.joinpath(
-        "..", "..", "..", "cmdeploy", "src", "cmdeploy"
-    ).resolve()
-    sources = [str(chatmaild_dir), str(cmdeploy_dir)]
-
     format_args = [shutil.which("ruff"), "format"]
     check_args = [shutil.which("ruff"), "check"]
 
@@ -314,7 +309,7 @@ def add_ssh_host_option(parser):
         "--ssh-host",
         dest="ssh_host",
         help="Run commands on 'localhost', via '@docker', or on a specific SSH host "
-        "instead of chatmail.ini's mail_domain.",
+             "instead of chatmail.ini's mail_domain.",
     )
 
 

cmdeploy/src/cmdeploy/deploy.py

@@ -3,9 +3,7 @@ import os
 
 import pyinfra
 
-# pyinfra runs this module as a python file and not as a module so
+from cmdeploy import deploy_chatmail
-# import paths must be absolute
-from cmdeploy.deployers import deploy_chatmail
 
 
 def main():

cmdeploy/src/cmdeploy/deployer.py

@@ -0,0 +1,57 @@
+import os
+
+from pyinfra.operations import server
+
+
+class Deployment:
+    def install(self, deployer):
+        # optional 'required_users' contains a list of (user, group, secondary-group-list) tuples.
+        # If the group is None, no group is created corresponding to that user.
+        # If the secondary group list is not None, all listed groups are created as well.
+        required_users = getattr(deployer, "required_users", [])
+        for user, group, groups in required_users:
+            if group is not None:
+                server.group(
+                    name="Create {} group".format(group), group=group, system=True
+                )
+            if groups is not None:
+                for group2 in groups:
+                    server.group(
+                        name="Create {} group".format(group2), group=group2, system=True
+                    )
+            server.user(
+                name="Create {} user".format(user),
+                user=user,
+                group=group,
+                groups=groups,
+                system=True,
+            )
+
+        deployer.install()
+
+    def configure(self, deployer):
+        deployer.configure()
+
+    def activate(self, deployer):
+        deployer.activate()
+
+    def perform_stages(self, deployers):
+        default_stages = "install,configure,activate"
+        stages = os.getenv("CMDEPLOY_STAGES", default_stages).split(",")
+
+        for stage in stages:
+            for deployer in deployers:
+                getattr(self, stage)(deployer)
+
+
+class Deployer:
+    need_restart = False
+
+    def install(self):
+        pass
+
+    def configure(self):
+        pass
+
+    def activate(self):
+        pass

cmdeploy/src/cmdeploy/deployers.py

@@ -1,598 +0,0 @@
-"""
-Chat Mail pyinfra deploy.
-"""
-
-import shutil
-import subprocess
-import sys
-from io import StringIO
-from pathlib import Path
-
-from chatmaild.config import read_config
-from pyinfra import facts, host, logger
-from pyinfra.api import FactBase
-from pyinfra.facts.files import Sha256File
-from pyinfra.facts.systemd import SystemdEnabled
-from pyinfra.operations import apt, files, pip, server, systemd
-
-from cmdeploy.cmdeploy import Out
-
-from .acmetool import AcmetoolDeployer
-from .basedeploy import (
-    Deployer,
-    Deployment,
-    activate_remote_units,
-    configure_remote_units,
-    get_resource,
-)
-from .dovecot.deployer import DovecotDeployer
-from .mtail.deployer import MtailDeployer
-from .nginx.deployer import NginxDeployer
-from .opendkim.deployer import OpendkimDeployer
-from .postfix.deployer import PostfixDeployer
-from .www import build_webpages, find_merge_conflict, get_paths
-
-
-class Port(FactBase):
-    """
-    Returns the process occuping a port.
-    """
-
-    def command(self, port: int) -> str:
-        return (
-            "ss -lptn 'src :%d' | awk 'NR>1 {print $6,$7}' | sed 's/users:((\"//;s/\".*//'"
-            % (port,)
-        )
-
-    def process(self, output: [str]) -> str:
-        return output[0]
-
-
-def _build_chatmaild(dist_dir) -> None:
-    dist_dir = Path(dist_dir).resolve()
-    if dist_dir.exists():
-        shutil.rmtree(dist_dir)
-    dist_dir.mkdir()
-    subprocess.check_output(
-        [sys.executable, "-m", "build", "-n"]
-        + ["--sdist", "chatmaild", "--outdir", str(dist_dir)]
-    )
-    entries = list(dist_dir.iterdir())
-    assert len(entries) == 1
-    return entries[0]
-
-
-def remove_legacy_artifacts():
-    # disable legacy doveauth-dictproxy.service
-    if host.get_fact(SystemdEnabled).get("doveauth-dictproxy.service"):
-        systemd.service(
-            name="Disable legacy doveauth-dictproxy.service",
-            service="doveauth-dictproxy.service",
-            running=False,
-            enabled=False,
-        )
-
-
-def _install_remote_venv_with_chatmaild() -> None:
-    remove_legacy_artifacts()
-    dist_file = _build_chatmaild(dist_dir=Path("chatmaild/dist"))
-    remote_base_dir = "/usr/local/lib/chatmaild"
-    remote_dist_file = f"{remote_base_dir}/dist/{dist_file.name}"
-    remote_venv_dir = f"{remote_base_dir}/venv"
-    root_owned = dict(user="root", group="root", mode="644")
-
-    apt.packages(
-        name="apt install python3-virtualenv",
-        packages=["python3-virtualenv"],
-    )
-
-    files.put(
-        name="Upload chatmaild source package",
-        src=dist_file.open("rb"),
-        dest=remote_dist_file,
-        create_remote_dir=True,
-        **root_owned,
-    )
-
-    pip.virtualenv(
-        name=f"chatmaild virtualenv {remote_venv_dir}",
-        path=remote_venv_dir,
-        always_copy=True,
-    )
-
-    apt.packages(
-        name="install gcc and headers to build crypt_r source package",
-        packages=["gcc", "python3-dev"],
-    )
-
-    server.shell(
-        name=f"forced pip-install {dist_file.name}",
-        commands=[
-            f"{remote_venv_dir}/bin/pip install --force-reinstall {remote_dist_file}"
-        ],
-    )
-
-
-def _configure_remote_venv_with_chatmaild(config) -> None:
-    remote_base_dir = "/usr/local/lib/chatmaild"
-    remote_venv_dir = f"{remote_base_dir}/venv"
-    remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini"
-    root_owned = dict(user="root", group="root", mode="644")
-
-    files.put(
-        name=f"Upload {remote_chatmail_inipath}",
-        src=config._getbytefile(),
-        dest=remote_chatmail_inipath,
-        **root_owned,
-    )
-
-    files.template(
-        src=get_resource("metrics.cron.j2"),
-        dest="/etc/cron.d/chatmail-metrics",
-        user="root",
-        group="root",
-        mode="644",
-        config={
-            "mailboxes_dir": config.mailboxes_dir,
-            "execpath": f"{remote_venv_dir}/bin/chatmail-metrics",
-        },
-    )
-
-
-class UnboundDeployer(Deployer):
-    def install(self):
-        # Run local DNS resolver `unbound`.
-        # `resolvconf` takes care of setting up /etc/resolv.conf
-        # to use 127.0.0.1 as the resolver.
-
-        #
-        # On an IPv4-only system, if unbound is started but not
-        # configured, it causes subsequent steps to fail to resolve hosts.
-        # Here, we use policy-rc.d to prevent unbound from starting up
-        # on initial install.  Later, we will configure it and start it.
-        #
-        # For documentation about policy-rc.d, see:
-        # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
-        #
-        files.put(
-            src=get_resource("policy-rc.d"),
-            dest="/usr/sbin/policy-rc.d",
-            user="root",
-            group="root",
-            mode="755",
-        )
-
-        apt.packages(
-            name="Install unbound",
-            packages=["unbound", "unbound-anchor", "dnsutils"],
-        )
-
-        files.file("/usr/sbin/policy-rc.d", present=False)
-
-    def configure(self):
-        server.shell(
-            name="Generate root keys for validating DNSSEC",
-            commands=[
-                "unbound-anchor -a /var/lib/unbound/root.key || true",
-            ],
-        )
-
-    def activate(self):
-        server.shell(
-            name="Generate root keys for validating DNSSEC",
-            commands=[
-                "systemctl reset-failed unbound.service",
-            ],
-        )
-
-        systemd.service(
-            name="Start and enable unbound",
-            service="unbound.service",
-            running=True,
-            enabled=True,
-        )
-
-
-class MtastsDeployer(Deployer):
-    def configure(self):
-        # Remove configuration.
-        files.file("/etc/mta-sts-daemon.yml", present=False)
-        files.directory("/usr/local/lib/postfix-mta-sts-resolver", present=False)
-        files.file("/etc/systemd/system/mta-sts-daemon.service", present=False)
-
-    def activate(self):
-        systemd.service(
-            name="Stop MTA-STS daemon",
-            service="mta-sts-daemon.service",
-            daemon_reload=True,
-            running=False,
-            enabled=False,
-        )
-
-
-class WebsiteDeployer(Deployer):
-    def __init__(self, config):
-        self.config = config
-
-    def install(self):
-        files.directory(
-            name="Ensure /var/www exists",
-            path="/var/www",
-            user="root",
-            group="root",
-            mode="755",
-            present=True,
-        )
-
-    def configure(self):
-        www_path, src_dir, build_dir = get_paths(self.config)
-        # if www_folder was set to a non-existing folder, skip upload
-        if not www_path.is_dir():
-            logger.warning("Building web pages is disabled in chatmail.ini, skipping")
-        elif (path := find_merge_conflict(src_dir)) is not None:
-            logger.warning(
-                f"Merge conflict found in {path}, skipping website deployment. Fix merge conflict if you want to upload your web page."
-            )
-        else:
-            # if www_folder is a hugo page, build it
-            if build_dir:
-                www_path = build_webpages(src_dir, build_dir, self.config)
-            # if it is not a hugo page, upload it as is
-            files.rsync(
-                f"{www_path}/", "/var/www/html", flags=["-avz", "--chown=www-data"]
-            )
-
-
-class LegacyRemoveDeployer(Deployer):
-    def install(self):
-        apt.packages(name="Remove rspamd", packages="rspamd", present=False)
-
-        # remove historic expunge script
-        # which is now implemented through a systemd timer (chatmail-expire)
-        files.file(
-            path="/etc/cron.d/expunge",
-            present=False,
-        )
-
-        # Remove OBS repository key that is no longer used.
-        files.file("/etc/apt/keyrings/obs-home-deltachat.gpg", present=False)
-        files.line(
-            name="Remove DeltaChat OBS home repository from sources.list",
-            path="/etc/apt/sources.list",
-            line="deb [signed-by=/etc/apt/keyrings/obs-home-deltachat.gpg] https://download.opensuse.org/repositories/home:/deltachat/Debian_12/ ./",
-            escape_regex_characters=True,
-            present=False,
-        )
-
-        # prior relay versions used filelogging
-        files.directory(
-            name="Ensure old logs on disk are deleted",
-            path="/var/log/journal/",
-            present=False,
-        )
-
-
-def check_config(config):
-    mail_domain = config.mail_domain
-    if mail_domain != "testrun.org" and not mail_domain.endswith(".testrun.org"):
-        blocked_words = "merlinux schmieder testrun.org".split()
-        for key in config.__dict__:
-            value = config.__dict__[key]
-            if key.startswith("privacy") and any(
-                x in str(value) for x in blocked_words
-            ):
-                raise ValueError(
-                    f"please set your own privacy contacts/addresses in {config._inipath}"
-                )
-    return config
-
-
-class TurnDeployer(Deployer):
-    def __init__(self, mail_domain):
-        self.mail_domain = mail_domain
-        self.units = ["turnserver"]
-
-    def install(self):
-        (url, sha256sum) = {
-            "x86_64": (
-                "https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-x86_64-linux",
-                "841e527c15fdc2940b0469e206188ea8f0af48533be12ecb8098520f813d41e4",
-            ),
-            "aarch64": (
-                "https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-aarch64-linux",
-                "a5fc2d06d937b56a34e098d2cd72a82d3e89967518d159bf246dc69b65e81b42",
-            ),
-        }[host.get_fact(facts.server.Arch)]
-
-        existing_sha256sum = host.get_fact(Sha256File, "/usr/local/bin/chatmail-turn")
-        if existing_sha256sum != sha256sum:
-            server.shell(
-                name="Download chatmail-turn",
-                commands=[
-                    f"(curl -L {url} >/usr/local/bin/chatmail-turn.new && (echo '{sha256sum} /usr/local/bin/chatmail-turn.new' | sha256sum -c) && mv /usr/local/bin/chatmail-turn.new /usr/local/bin/chatmail-turn)",
-                    "chmod 755 /usr/local/bin/chatmail-turn",
-                ],
-            )
-
-    def configure(self):
-        configure_remote_units(self.mail_domain, self.units)
-
-    def activate(self):
-        activate_remote_units(self.units)
-
-
-class IrohDeployer(Deployer):
-    def __init__(self, enable_iroh_relay):
-        self.enable_iroh_relay = enable_iroh_relay
-
-    def install(self):
-        (url, sha256sum) = {
-            "x86_64": (
-                "https://github.com/n0-computer/iroh/releases/download/v0.35.0/iroh-relay-v0.35.0-x86_64-unknown-linux-musl.tar.gz",
-                "45c81199dbd70f8c4c30fef7f3b9727ca6e3cea8f2831333eeaf8aa71bf0fac1",
-            ),
-            "aarch64": (
-                "https://github.com/n0-computer/iroh/releases/download/v0.35.0/iroh-relay-v0.35.0-aarch64-unknown-linux-musl.tar.gz",
-                "f8ef27631fac213b3ef668d02acd5b3e215292746a3fc71d90c63115446008b1",
-            ),
-        }[host.get_fact(facts.server.Arch)]
-
-        existing_sha256sum = host.get_fact(Sha256File, "/usr/local/bin/iroh-relay")
-        if existing_sha256sum != sha256sum:
-            server.shell(
-                name="Download iroh-relay",
-                commands=[
-                    f"(curl -L {url} | gunzip | tar -x -f - ./iroh-relay -O >/usr/local/bin/iroh-relay.new && (echo '{sha256sum} /usr/local/bin/iroh-relay.new' | sha256sum -c) && mv /usr/local/bin/iroh-relay.new /usr/local/bin/iroh-relay)",
-                    "chmod 755 /usr/local/bin/iroh-relay",
-                ],
-            )
-
-            self.need_restart = True
-
-    def configure(self):
-        systemd_unit = files.put(
-            name="Upload iroh-relay systemd unit",
-            src=get_resource("iroh-relay.service"),
-            dest="/etc/systemd/system/iroh-relay.service",
-            user="root",
-            group="root",
-            mode="644",
-        )
-        self.need_restart |= systemd_unit.changed
-
-        iroh_config = files.put(
-            name="Upload iroh-relay config",
-            src=get_resource("iroh-relay.toml"),
-            dest="/etc/iroh-relay.toml",
-            user="root",
-            group="root",
-            mode="644",
-        )
-        self.need_restart |= iroh_config.changed
-
-    def activate(self):
-        systemd.service(
-            name="Start and enable iroh-relay",
-            service="iroh-relay.service",
-            running=True,
-            enabled=self.enable_iroh_relay,
-            restarted=self.need_restart,
-        )
-        self.need_restart = False
-
-
-class JournaldDeployer(Deployer):
-    def configure(self):
-        journald_conf = files.put(
-            name="Configure journald",
-            src=get_resource("journald.conf"),
-            dest="/etc/systemd/journald.conf",
-            user="root",
-            group="root",
-            mode="644",
-        )
-        self.need_restart = journald_conf.changed
-
-    def activate(self):
-        systemd.service(
-            name="Start and enable journald",
-            service="systemd-journald.service",
-            running=True,
-            enabled=True,
-            restarted=self.need_restart,
-        )
-        self.need_restart = False
-
-
-class EchobotDeployer(Deployer):
-    #
-    # This deployer depends on the dovecot and postfix deployers because
-    # it needs to base its decision of whether to restart the service on
-    # whether those two services were restarted.
-    #
-    def __init__(self, mail_domain):
-        self.mail_domain = mail_domain
-        self.units = ["echobot"]
-
-    def install(self):
-        apt.packages(
-            # required for setfacl for echobot
-            name="Install acl",
-            packages="acl",
-        )
-
-    def configure(self):
-        configure_remote_units(self.mail_domain, self.units)
-
-    def activate(self):
-        activate_remote_units(self.units)
-
-
-class ChatmailVenvDeployer(Deployer):
-    def __init__(self, config):
-        self.config = config
-        self.units = (
-            "filtermail",
-            "filtermail-incoming",
-            "chatmail-metadata",
-            "lastlogin",
-            "chatmail-expire",
-            "chatmail-expire.timer",
-            "chatmail-fsreport",
-            "chatmail-fsreport.timer",
-        )
-
-    def install(self):
-        _install_remote_venv_with_chatmaild()
-
-    def configure(self):
-        _configure_remote_venv_with_chatmaild(self.config)
-        configure_remote_units(self.config.mail_domain, self.units)
-
-    def activate(self):
-        activate_remote_units(self.units)
-
-
-class ChatmailDeployer(Deployer):
-    required_users = [
-        ("vmail", "vmail", None),
-        ("echobot", None, None),
-        ("iroh", None, None),
-    ]
-
-    def __init__(self, mail_domain):
-        self.mail_domain = mail_domain
-
-    def install(self):
-        apt.update(name="apt update", cache_time=24 * 3600)
-        apt.upgrade(name="upgrade apt packages", auto_remove=True)
-
-        apt.packages(
-            name="Install curl",
-            packages=["curl"],
-        )
-
-        apt.packages(
-            name="Install rsync",
-            packages=["rsync"],
-        )
-        apt.packages(
-            name="Ensure cron is installed",
-            packages=["cron"],
-        )
-
-    def configure(self):
-        # This file is used by auth proxy.
-        # https://wiki.debian.org/EtcMailName
-        server.shell(
-            name="Setup /etc/mailname",
-            commands=[
-                f"echo {self.mail_domain} >/etc/mailname; chmod 644 /etc/mailname"
-            ],
-        )
-
-
-class FcgiwrapDeployer(Deployer):
-    def install(self):
-        apt.packages(
-            name="Install fcgiwrap",
-            packages=["fcgiwrap"],
-        )
-
-    def activate(self):
-        systemd.service(
-            name="Start and enable fcgiwrap",
-            service="fcgiwrap.service",
-            running=True,
-            enabled=True,
-        )
-
-
-class GithashDeployer(Deployer):
-    def activate(self):
-        try:
-            git_hash = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode()
-        except Exception:
-            git_hash = "unknown\n"
-        try:
-            git_diff = subprocess.check_output(["git", "diff"]).decode()
-        except Exception:
-            git_diff = ""
-        files.put(
-            name="Upload chatmail relay git commiit hash",
-            src=StringIO(git_hash + git_diff),
-            dest="/etc/chatmail-version",
-            mode="700",
-        )
-
-
-def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
-    """Deploy a chat-mail instance.
-
-    :param config_path: path to chatmail.ini
-    :param disable_mail: whether to disable postfix & dovecot
-    """
-    config = read_config(config_path)
-    check_config(config)
-    mail_domain = config.mail_domain
-
-    if host.get_fact(Port, port=53) != "unbound":
-        files.line(
-            name="Add 9.9.9.9 to resolv.conf",
-            path="/etc/resolv.conf",
-            line="nameserver 9.9.9.9",
-        )
-
-    port_services = [
-        (["master", "smtpd"], 25),
-        ("unbound", 53),
-        ("acmetool", 80),
-        (["imap-login", "dovecot"], 143),
-        ("nginx", 443),
-        (["master", "smtpd"], 465),
-        (["master", "smtpd"], 587),
-        (["imap-login", "dovecot"], 993),
-        ("iroh-relay", 3340),
-        ("nginx", 8443),
-        (["master", "smtpd"], config.postfix_reinject_port),
-        (["master", "smtpd"], config.postfix_reinject_port_incoming),
-        ("filtermail", config.filtermail_smtp_port),
-        ("filtermail", config.filtermail_smtp_port_incoming),
-    ]
-    for service, port in port_services:
-        print(f"Checking if port {port} is available for {service}...")
-        running_service = host.get_fact(Port, port=port)
-        if running_service:
-            if running_service not in service:
-                Out().red(
-                    f"Deploy failed: port {port} is occupied by: {running_service}"
-                )
-                exit(1)
-
-    tls_domains = [mail_domain, f"mta-sts.{mail_domain}", f"www.{mail_domain}"]
-
-    all_deployers = [
-        ChatmailDeployer(mail_domain),
-        LegacyRemoveDeployer(),
-        JournaldDeployer(),
-        UnboundDeployer(),
-        TurnDeployer(mail_domain),
-        IrohDeployer(config.enable_iroh_relay),
-        AcmetoolDeployer(config.acme_email, tls_domains),
-        WebsiteDeployer(config),
-        ChatmailVenvDeployer(config),
-        MtastsDeployer(),
-        OpendkimDeployer(mail_domain),
-        # Dovecot should be started before Postfix
-        # because it creates authentication socket
-        # required by Postfix.
-        DovecotDeployer(config, disable_mail),
-        PostfixDeployer(config, disable_mail),
-        FcgiwrapDeployer(),
-        NginxDeployer(config),
-        EchobotDeployer(mail_domain),
-        MtailDeployer(config.mtail_address),
-        GithashDeployer(),
-    ]
-
-    Deployment().perform_stages(all_deployers)

cmdeploy/src/cmdeploy/dns.py

@@ -45,8 +45,7 @@ def check_full_zone(sshexec, remote_data, out, zonefile) -> int:
     and return (exitcode, remote_data) tuple."""
 
     required_diff, recommended_diff = sshexec.logged(
-        remote.rdns.check_zonefile,
+        remote.rdns.check_zonefile, kwargs=dict(zonefile=zonefile, verbose=False),
-        kwargs=dict(zonefile=zonefile, verbose=False),
     )
 
     returncode = 0

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -1,137 +0,0 @@
-from chatmaild.config import Config
-from pyinfra import host
-from pyinfra.facts.server import Arch, Sysctl
-from pyinfra.facts.systemd import SystemdEnabled
-from pyinfra.operations import apt, files, server, systemd
-
-from cmdeploy.basedeploy import (
-    Deployer,
-    activate_remote_units,
-    configure_remote_units,
-    get_resource,
-)
-
-
-class DovecotDeployer(Deployer):
-    def __init__(self, config, disable_mail):
-        self.config = config
-        self.disable_mail = disable_mail
-        self.units = ["doveauth"]
-
-    def install(self):
-        arch = host.get_fact(Arch)
-        if not "dovecot.service" in host.get_fact(SystemdEnabled):
-            _install_dovecot_package("core", arch)
-            _install_dovecot_package("imapd", arch)
-            _install_dovecot_package("lmtpd", arch)
-
-    def configure(self):
-        configure_remote_units(self.config.mail_domain, self.units)
-        self.need_restart = _configure_dovecot(self.config)
-
-    def activate(self):
-        activate_remote_units(self.units)
-
-        restart = False if self.disable_mail else self.need_restart
-
-        systemd.service(
-            name="disable dovecot for now"
-            if self.disable_mail
-            else "Start and enable Dovecot",
-            service="dovecot.service",
-            running=False if self.disable_mail else True,
-            enabled=False if self.disable_mail else True,
-            restarted=restart,
-        )
-        self.need_restart = False
-
-
-def _install_dovecot_package(package: str, arch: str):
-    arch = "amd64" if arch == "x86_64" else arch
-    arch = "arm64" if arch == "aarch64" else arch
-    url = f"https://download.delta.chat/dovecot/dovecot-{package}_2.3.21%2Bdfsg1-3_{arch}.deb"
-    deb_filename = "/root/" + url.split("/")[-1]
-
-    match (package, arch):
-        case ("core", "amd64"):
-            sha256 = "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d"
-        case ("core", "arm64"):
-            sha256 = "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9"
-        case ("imapd", "amd64"):
-            sha256 = "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86"
-        case ("imapd", "arm64"):
-            sha256 = "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f"
-        case ("lmtpd", "amd64"):
-            sha256 = "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab"
-        case ("lmtpd", "arm64"):
-            sha256 = "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f"
-        case _:
-            apt.packages(packages=[f"dovecot-{package}"])
-            return
-
-    files.download(
-        name=f"Download dovecot-{package}",
-        src=url,
-        dest=deb_filename,
-        sha256sum=sha256,
-        cache_time=60 * 60 * 24 * 365 * 10,  # never redownload the package
-    )
-
-    apt.deb(name=f"Install dovecot-{package}", src=deb_filename)
-
-
-def _configure_dovecot(config: Config, debug: bool = False) -> bool:
-    """Configures Dovecot IMAP server."""
-    need_restart = False
-
-    main_config = files.template(
-        src=get_resource("dovecot/dovecot.conf.j2"),
-        dest="/etc/dovecot/dovecot.conf",
-        user="root",
-        group="root",
-        mode="644",
-        config=config,
-        debug=debug,
-        disable_ipv6=config.disable_ipv6,
-    )
-    need_restart |= main_config.changed
-    auth_config = files.put(
-        src=get_resource("dovecot/auth.conf"),
-        dest="/etc/dovecot/auth.conf",
-        user="root",
-        group="root",
-        mode="644",
-    )
-    need_restart |= auth_config.changed
-    lua_push_notification_script = files.put(
-        src=get_resource("dovecot/push_notification.lua"),
-        dest="/etc/dovecot/push_notification.lua",
-        user="root",
-        group="root",
-        mode="644",
-    )
-    need_restart |= lua_push_notification_script.changed
-
-    # as per https://doc.dovecot.org/configuration_manual/os/
-    # it is recommended to set the following inotify limits
-    for name in ("max_user_instances", "max_user_watches"):
-        key = f"fs.inotify.{name}"
-        if host.get_fact(Sysctl)[key] > 65535:
-            # Skip updating limits if already sufficient
-            # (enables running in incus containers where sysctl readonly)
-            continue
-        server.sysctl(
-            name=f"Change {key}",
-            key=key,
-            value=65535,
-            persist=True,
-        )
-
-    timezone_env = files.line(
-        name="Set TZ environment variable",
-        path="/etc/environment",
-        line="TZ=:/etc/localtime",
-    )
-    need_restart |= timezone_env.changed
-
-    return need_restart

cmdeploy/src/cmdeploy/mtail/deployer.py

@@ -1,68 +0,0 @@
-from pyinfra import facts, host
-from pyinfra.operations import apt, files, server, systemd
-
-from cmdeploy.basedeploy import (
-    Deployer,
-    get_resource,
-)
-
-
-class MtailDeployer(Deployer):
-    def __init__(self, mtail_address):
-        self.mtail_address = mtail_address
-
-    def install(self):
-        # Uninstall mtail package to install a static binary.
-        apt.packages(name="Uninstall mtail", packages=["mtail"], present=False)
-
-        (url, sha256sum) = {
-            "x86_64": (
-                "https://github.com/google/mtail/releases/download/v3.0.8/mtail_3.0.8_linux_amd64.tar.gz",
-                "123c2ee5f48c3eff12ebccee38befd2233d715da736000ccde49e3d5607724e4",
-            ),
-            "aarch64": (
-                "https://github.com/google/mtail/releases/download/v3.0.8/mtail_3.0.8_linux_arm64.tar.gz",
-                "aa04811c0929b6754408676de520e050c45dddeb3401881888a092c9aea89cae",
-            ),
-        }[host.get_fact(facts.server.Arch)]
-
-        server.shell(
-            name="Download mtail",
-            commands=[
-                f"(echo '{sha256sum} /usr/local/bin/mtail' | sha256sum -c) || (curl -L {url} | gunzip | tar -x -f - mtail -O >/usr/local/bin/mtail.new && mv /usr/local/bin/mtail.new /usr/local/bin/mtail)",
-                "chmod 755 /usr/local/bin/mtail",
-            ],
-        )
-
-    def configure(self):
-        # Using our own systemd unit instead of `/usr/lib/systemd/system/mtail.service`.
-        # This allows to read from journalctl instead of log files.
-        files.template(
-            src=get_resource("mtail/mtail.service.j2"),
-            dest="/etc/systemd/system/mtail.service",
-            user="root",
-            group="root",
-            mode="644",
-            address=self.mtail_address or "127.0.0.1",
-            port=3903,
-        )
-
-        mtail_conf = files.put(
-            name="Mtail configuration",
-            src=get_resource("mtail/delivered_mail.mtail"),
-            dest="/etc/mtail/delivered_mail.mtail",
-            user="root",
-            group="root",
-            mode="644",
-        )
-        self.need_restart = mtail_conf.changed
-
-    def activate(self):
-        systemd.service(
-            name="Start and enable mtail",
-            service="mtail.service",
-            running=bool(self.mtail_address),
-            enabled=bool(self.mtail_address),
-            restarted=self.need_restart,
-        )
-        self.need_restart = False

cmdeploy/src/cmdeploy/nginx/deployer.py

@@ -1,117 +0,0 @@
-from chatmaild.config import Config
-from pyinfra.operations import apt, files, systemd
-
-from cmdeploy.basedeploy import (
-    Deployer,
-    get_resource,
-)
-
-
-class NginxDeployer(Deployer):
-    def __init__(self, config):
-        self.config = config
-
-    def install(self):
-        #
-        # If we allow nginx to start up on install, it will grab port
-        # 80, which then will block acmetool from listening on the port.
-        # That in turn prevents getting certificates, which then causes
-        # an error when we try to start nginx on the custom config
-        # that leaves port 80 open but also requires certificates to
-        # be present.  To avoid getting into that interlocking mess,
-        # we use policy-rc.d to prevent nginx from starting up when it
-        # is installed.
-        #
-        # This approach allows us to avoid performing any explicit
-        # systemd operations during the install stage (as opposed to
-        # allowing it to start and then forcing it to stop), which allows
-        # the install stage to run in non-systemd environments like a
-        # container image build.
-        #
-        # For documentation about policy-rc.d, see:
-        # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
-        #
-        files.put(
-            src=get_resource("policy-rc.d"),
-            dest="/usr/sbin/policy-rc.d",
-            user="root",
-            group="root",
-            mode="755",
-        )
-
-        apt.packages(
-            name="Install nginx",
-            packages=["nginx", "libnginx-mod-stream"],
-        )
-
-        files.file("/usr/sbin/policy-rc.d", present=False)
-
-    def configure(self):
-        self.need_restart = _configure_nginx(self.config)
-
-    def activate(self):
-        systemd.service(
-            name="Start and enable nginx",
-            service="nginx.service",
-            running=True,
-            enabled=True,
-            restarted=self.need_restart,
-        )
-        self.need_restart = False
-
-
-def _configure_nginx(config: Config, debug: bool = False) -> bool:
-    """Configures nginx HTTP server."""
-    need_restart = False
-
-    main_config = files.template(
-        src=get_resource("nginx/nginx.conf.j2"),
-        dest="/etc/nginx/nginx.conf",
-        user="root",
-        group="root",
-        mode="644",
-        config={"domain_name": config.mail_domain},
-        disable_ipv6=config.disable_ipv6,
-    )
-    need_restart |= main_config.changed
-
-    autoconfig = files.template(
-        src=get_resource("nginx/autoconfig.xml.j2"),
-        dest="/var/www/html/.well-known/autoconfig/mail/config-v1.1.xml",
-        user="root",
-        group="root",
-        mode="644",
-        config={"domain_name": config.mail_domain},
-    )
-    need_restart |= autoconfig.changed
-
-    mta_sts_config = files.template(
-        src=get_resource("nginx/mta-sts.txt.j2"),
-        dest="/var/www/html/.well-known/mta-sts.txt",
-        user="root",
-        group="root",
-        mode="644",
-        config={"domain_name": config.mail_domain},
-    )
-    need_restart |= mta_sts_config.changed
-
-    # install CGI newemail script
-    #
-    cgi_dir = "/usr/lib/cgi-bin"
-    files.directory(
-        name=f"Ensure {cgi_dir} exists",
-        path=cgi_dir,
-        user="root",
-        group="root",
-    )
-
-    files.put(
-        name="Upload cgi newemail.py script",
-        src=get_resource("newemail.py", pkg="chatmaild").open("rb"),
-        dest=f"{cgi_dir}/newemail.py",
-        user="root",
-        group="root",
-        mode="755",
-    )
-
-    return need_restart

cmdeploy/src/cmdeploy/opendkim/deployer.py

@@ -1,123 +0,0 @@
-"""
-Installs OpenDKIM
-"""
-
-from pyinfra import host
-from pyinfra.facts.files import File
-from pyinfra.operations import apt, files, server, systemd
-
-from cmdeploy.basedeploy import Deployer, get_resource
-
-
-class OpendkimDeployer(Deployer):
-    required_users = [("opendkim", None, ["opendkim"])]
-
-    def __init__(self, mail_domain):
-        self.mail_domain = mail_domain
-
-    def install(self):
-        apt.packages(
-            name="apt install opendkim opendkim-tools",
-            packages=["opendkim", "opendkim-tools"],
-        )
-
-    def configure(self):
-        domain = self.mail_domain
-        dkim_selector = "opendkim"
-        """Configures OpenDKIM"""
-        need_restart = False
-
-        main_config = files.template(
-            src=get_resource("opendkim/opendkim.conf"),
-            dest="/etc/opendkim.conf",
-            user="root",
-            group="root",
-            mode="644",
-            config={"domain_name": domain, "opendkim_selector": dkim_selector},
-        )
-        need_restart |= main_config.changed
-
-        screen_script = files.put(
-            src=get_resource("opendkim/screen.lua"),
-            dest="/etc/opendkim/screen.lua",
-            user="root",
-            group="root",
-            mode="644",
-        )
-        need_restart |= screen_script.changed
-
-        final_script = files.put(
-            src=get_resource("opendkim/final.lua"),
-            dest="/etc/opendkim/final.lua",
-            user="root",
-            group="root",
-            mode="644",
-        )
-        need_restart |= final_script.changed
-
-        files.directory(
-            name="Add opendkim directory to /etc",
-            path="/etc/opendkim",
-            user="opendkim",
-            group="opendkim",
-            mode="750",
-            present=True,
-        )
-
-        keytable = files.template(
-            src=get_resource("opendkim/KeyTable"),
-            dest="/etc/dkimkeys/KeyTable",
-            user="opendkim",
-            group="opendkim",
-            mode="644",
-            config={"domain_name": domain, "opendkim_selector": dkim_selector},
-        )
-        need_restart |= keytable.changed
-
-        signing_table = files.template(
-            src=get_resource("opendkim/SigningTable"),
-            dest="/etc/dkimkeys/SigningTable",
-            user="opendkim",
-            group="opendkim",
-            mode="644",
-            config={"domain_name": domain, "opendkim_selector": dkim_selector},
-        )
-        need_restart |= signing_table.changed
-        files.directory(
-            name="Add opendkim socket directory to /var/spool/postfix",
-            path="/var/spool/postfix/opendkim",
-            user="opendkim",
-            group="opendkim",
-            mode="750",
-            present=True,
-        )
-
-        if not host.get_fact(File, f"/etc/dkimkeys/{dkim_selector}.private"):
-            server.shell(
-                name="Generate OpenDKIM domain keys",
-                commands=[
-                    f"/usr/sbin/opendkim-genkey -D /etc/dkimkeys -d {domain} -s {dkim_selector}"
-                ],
-                _use_su_login=True,
-                _su_user="opendkim",
-            )
-
-        service_file = files.put(
-            name="Configure opendkim to restart once a day",
-            src=get_resource("opendkim/systemd.conf"),
-            dest="/etc/systemd/system/opendkim.service.d/10-prevent-memory-leak.conf",
-        )
-        need_restart |= service_file.changed
-
-        self.need_restart = need_restart
-
-    def activate(self):
-        systemd.service(
-            name="Start and enable OpenDKIM",
-            service="opendkim.service",
-            running=True,
-            enabled=True,
-            daemon_reload=self.need_restart,
-            restarted=self.need_restart,
-        )
-        self.need_restart = False

cmdeploy/src/cmdeploy/opendkim/final.lua

@@ -9,10 +9,9 @@ if nsigs == nil then
 	return nil
 end
 
-local valid = false
 for i = 1, nsigs do
-	sig = odkim.get_sighandle(ctx, i - 1)
+        sig = odkim.get_sighandle(ctx, i - 1)
-	sigres = odkim.sig_result(sig)
+        sigres = odkim.sig_result(sig)
 
 	-- All signatures that do not correspond to From: 
 	-- were ignored in screen.lua and return sigres -1.
@@ -20,19 +19,10 @@ for i = 1, nsigs do
 	-- Any valid signature that was not ignored like this
 	-- means the message is acceptable.
 	if sigres == 0 then
-		valid = true
+		return nil
-	end
+	end	
-end
-
-if valid then
-	-- Strip all DKIM-Signature headers after successful validation
-	-- Delete in reverse order to avoid index shifting.
-	for i = nsigs, 1, -1 do
-		odkim.del_header(ctx, "DKIM-Signature", i)
-	end
-else
-	odkim.set_reply(ctx, "554", "5.7.1", "No valid DKIM signature found")
-	odkim.set_result(ctx, SMFIS_REJECT)
 end
 
+odkim.set_reply(ctx, "554", "5.7.1", "No valid DKIM signature found")
+odkim.set_result(ctx, SMFIS_REJECT)
 return nil

cmdeploy/src/cmdeploy/postfix/deployer.py

@@ -1,77 +0,0 @@
-from pyinfra.operations import apt, files, systemd
-
-from cmdeploy.basedeploy import Deployer, get_resource
-
-
-class PostfixDeployer(Deployer):
-    required_users = [("postfix", None, ["opendkim"])]
-
-    def __init__(self, config, disable_mail):
-        self.config = config
-        self.disable_mail = disable_mail
-
-    def install(self):
-        apt.packages(
-            name="Install Postfix",
-            packages="postfix",
-        )
-
-    def configure(self):
-        config = self.config
-        need_restart = False
-
-        main_config = files.template(
-            src=get_resource("postfix/main.cf.j2"),
-            dest="/etc/postfix/main.cf",
-            user="root",
-            group="root",
-            mode="644",
-            config=config,
-            disable_ipv6=config.disable_ipv6,
-        )
-        need_restart |= main_config.changed
-
-        master_config = files.template(
-            src=get_resource("postfix/master.cf.j2"),
-            dest="/etc/postfix/master.cf",
-            user="root",
-            group="root",
-            mode="644",
-            debug=False,
-            config=config,
-        )
-        need_restart |= master_config.changed
-
-        header_cleanup = files.put(
-            src=get_resource("postfix/submission_header_cleanup"),
-            dest="/etc/postfix/submission_header_cleanup",
-            user="root",
-            group="root",
-            mode="644",
-        )
-        need_restart |= header_cleanup.changed
-
-        # Login map that 1:1 maps email address to login.
-        login_map = files.put(
-            src=get_resource("postfix/login_map"),
-            dest="/etc/postfix/login_map",
-            user="root",
-            group="root",
-            mode="644",
-        )
-        need_restart |= login_map.changed
-        self.need_restart = need_restart
-
-    def activate(self):
-        restart = False if self.disable_mail else self.need_restart
-
-        systemd.service(
-            name="disable postfix for now"
-            if self.disable_mail
-            else "Start and enable Postfix",
-            service="postfix.service",
-            running=False if self.disable_mail else True,
-            enabled=False if self.disable_mail else True,
-            restarted=restart,
-        )
-        self.need_restart = False

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -27,7 +27,7 @@ smtp_tls_servername = hostname
 smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
 smtp_tls_policy_maps = inline:{nauta.cu=may}
 smtp_tls_protocols = >=TLSv1.2
-smtp_tls_mandatory_protocols = >=TLSv1.2
+smtpd_tls_protocols = >=TLSv1.2
 
 # Disable anonymous cipher suites
 # and known insecure algorithms.

cmdeploy/src/cmdeploy/postfix/master.cf.j2

@@ -15,7 +15,6 @@ smtp      inet  n       -       y       -       -       smtpd -v
 smtp      inet  n       -       y       -       -       smtpd 
 {%- endif %}
   -o smtpd_tls_security_level=encrypt
-  -o smtpd_tls_mandatory_protocols=>=TLSv1.2
   -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port_incoming }}
 submission inet n       -       y       -       5000    smtpd
   -o syslog_name=postfix/submission

cmdeploy/src/cmdeploy/remote/rdns.py

@@ -12,7 +12,7 @@ All functions of this module
 
 import re
 
-from .rshell import CalledProcessError, log_progress, shell
+from .rshell import CalledProcessError, shell, log_progress
 
 
 def perform_initial_checks(mail_domain, pre_command=""):
@@ -26,9 +26,7 @@ def perform_initial_checks(mail_domain, pre_command=""):
     WWW = query_dns("CNAME", f"www.{mail_domain}")
 
     res = dict(mail_domain=mail_domain, A=A, AAAA=AAAA, MTA_STS=MTA_STS, WWW=WWW)
-    res["acme_account_url"] = shell(
+    res["acme_account_url"] = shell(pre_command + "acmetool account-url", fail_ok=True, print=log_progress)
-        pre_command + "acmetool account-url", fail_ok=True, print=log_progress
-    )
     res["dkim_entry"], res["web_dkim_entry"] = get_dkim_entry(
         mail_domain, pre_command, dkim_selector="opendkim"
     )
@@ -47,7 +45,7 @@ def get_dkim_entry(mail_domain, pre_command, dkim_selector):
         dkim_pubkey = shell(
             f"{pre_command}openssl rsa -in /etc/dkimkeys/{dkim_selector}.private "
             "-pubout 2>/dev/null | awk '/-/{next}{printf(\"%s\",$0)}'",
-            print=log_progress,
+            print=log_progress
         )
     except CalledProcessError:
         return
@@ -64,9 +62,9 @@ def query_dns(typ, domain):
     # Get autoritative nameserver from the SOA record.
     soa_answers = [
         x.split()
-        for x in shell(
+        for x in shell(f"dig -r -q {domain} -t SOA +noall +authority +answer", print=log_progress).split(
-            f"dig -r -q {domain} -t SOA +noall +authority +answer", print=log_progress
+            "\n"
-        ).split("\n")
+        )
     ]
     soa = [a for a in soa_answers if len(a) >= 3 and a[3] == "SOA"]
     if not soa:
@@ -75,7 +73,7 @@ def query_dns(typ, domain):
 
     # Query authoritative nameserver directly to bypass DNS cache.
     res = shell(f"dig @{ns} -r -q {domain} -t {typ} +short", print=log_progress)
-    return next((line for line in res.split("\n") if not line.startswith(";")), "")
+    return next((line for line in res.split("\n") if not line.startswith(';')), '')
 
 
 def check_zonefile(zonefile, verbose=True):

cmdeploy/src/cmdeploy/remote/rshell.py

@@ -1,4 +1,5 @@
 import sys
+
 from subprocess import DEVNULL, CalledProcessError, check_output
 
 

cmdeploy/src/cmdeploy/sshexec.py

@@ -93,7 +93,7 @@ class LocalExec:
         where = "locally"
         if self.docker:
             if call == remote.rdns.perform_initial_checks:
-                kwargs["pre_command"] = "docker exec chatmail "
+                kwargs['pre_command'] = "docker exec chatmail "
                 where = "in docker"
         if self.verbose:
             print(f"Running {where}: {call.__name__}(**{kwargs})")

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -35,7 +35,7 @@ class TestSSHExecutor:
         out, err = capsys.readouterr()
         assert err.startswith("Collecting")
         # XXX could not figure out how capturing can be made to work properly
-        # assert err.endswith("....\n")
+        #assert err.endswith("....\n")
         assert err.count("\n") == 1
 
         sshexec.verbose = True
@@ -45,7 +45,7 @@ class TestSSHExecutor:
         out, err = capsys.readouterr()
         lines = err.split("\n")
         # XXX could not figure out how capturing can be made to work properly
-        # assert len(lines) > 4
+        #assert len(lines) > 4
         assert remote.rdns.perform_initial_checks.__doc__ in lines[0]
 
     def test_exception(self, sshexec, capsys):

cmdeploy/src/cmdeploy/tests/test_dns.py

@@ -65,9 +65,7 @@ class TestPerformInitialChecks:
         remote_data = remote.rdns.perform_initial_checks("some.domain")
         assert remote_data["A"] == mockdns_expected["A"]["some.domain"]
         assert remote_data["AAAA"] == mockdns_expected["AAAA"]["some.domain"]
-        assert (
+        assert remote_data["MTA_STS"] == mockdns_expected["CNAME"]["mta-sts.some.domain"]
-            remote_data["MTA_STS"] == mockdns_expected["CNAME"]["mta-sts.some.domain"]
-        )
         assert remote_data["WWW"] == mockdns_expected["CNAME"]["www.some.domain"]
 
     @pytest.mark.parametrize("drop", ["A", "AAAA"])

cmdeploy/src/cmdeploy/www.py

@@ -1,10 +1,10 @@
 import hashlib
 import importlib.resources
-import re
 import time
 import traceback
 import webbrowser
 from pathlib import Path
+import re
 
 import markdown
 from chatmaild.config import read_config
@@ -12,9 +12,8 @@ from jinja2 import Template
 
 from .genqr import gen_qr_png_data
 
-_MERGE_CONFLICT_RE = re.compile(
+
-    r"^<<<<<<<.+^=======.+^>>>>>>>", re.DOTALL | re.MULTILINE
+_MERGE_CONFLICT_RE = re.compile(r"^<<<<<<<.+^=======.+^>>>>>>>", re.DOTALL | re.MULTILINE)
-)
 
 
 def snapshot_dir_stats(somedir):
@@ -140,34 +139,34 @@ def main():
     config.webdev = True
     assert config.mail_domain
 
+    # start web page generation, open a browser and wait for changes
     www_path, src_path, build_dir = get_paths(config)
     build_dir = build_webpages(src_path, build_dir, config)
     index_path = build_dir.joinpath("index.html")
     webbrowser.open(str(index_path))
-
-    print(f"\nOpened URL: file://{index_path.resolve()}\n")
-    print(f"Watching {src_path} directory for changes...")
-
     stats = snapshot_dir_stats(src_path)
+    print(f"\nOpened URL: file://{index_path.resolve()}\n")
+    print(f"watching {src_path} directory for changes")
+
     changenum = 0
-    debounce_time = 0.5  # wait 0.5s after detecting a change
+    count = 0
-
     while True:
-        time.sleep(1)
         newstats = snapshot_dir_stats(src_path)
+        if newstats == stats and count % 60 != 0:
+            count += 1
+            time.sleep(1.0)
+            continue
 
-        if newstats != stats:
+        for key in newstats:
-            changed_files = [f for f in newstats if stats.get(f) != newstats[f]]
+            if stats[key] != newstats[key]:
-            for f in changed_files:
+                print(f"*** CHANGED: {key}")
-                print(f"*** CHANGED: {f}")
+                changenum += 1
 
-            stats = newstats
+        stats = newstats
-            changenum += 1
+        build_webpages(src_path, build_dir, config)
-            build_webpages(src_path, build_dir, config)
+        print(f"[{changenum}] regenerated web pages at: {index_path}")
-            print(f"[{changenum}] regenerated web pages at: {index_path}")
+        print(f"URL: file://{index_path.resolve()}\n\n")
-            print(f"URL: file://{index_path.resolve()}\n\n")
+        count = 0
-            
-            time.sleep(debounce_time)  # simple debounce
 
 
 if __name__ == "__main__":

doc/source/overview.rst

@@ -126,13 +126,14 @@ web page. Edit them before deploying to make your chatmail relay
 stand out.
 
 
-Chatmail relay dependency diagram
+Component dependency diagram
----------------------------------
+--------------------------------------
 
 .. mermaid::
    :caption: This diagram shows relay components and dependencies/communication paths.
 
     graph LR;
+        cmdeploy --- sshd;
         letsencrypt --- |80|acmetool-redirector;
         acmetool-redirector --- |443|nginx-right(["`nginx
         (external)`"]);
@@ -146,62 +147,33 @@ Chatmail relay dependency diagram
         nginx-internal --- autoconfig.xml;
         certs-nginx[("`TLS certs
         /var/lib/acme`")] --> nginx-internal;
-        systemd-timer --- chatmail-metrics;
+        cron --- chatmail-metrics;
-        systemd-timer --- acmetool;
+        cron --- acmetool;
-        systemd-timer --- chatmail-expire-daily;
-        systemd-timer --- chatmail-fsreport-daily;
         chatmail-metrics --- website;
         acmetool --> certs[("`TLS certs
         /var/lib/acme`")];
         nginx-external --- |993|dovecot;
-        postfix --- |SASL|dovecot;
         autoconfig.xml --- postfix;
         autoconfig.xml --- dovecot;
-        postfix --- |10080|filtermail-outgoing;
+        postfix --- echobot;
-        postfix --- |10081|filtermail-incoming;
+        postfix --- |10080,10081|filtermail;
-        filtermail-outgoing --- |10025 reinject|postfix;
+        postfix --- users["`User data
-        filtermail-incoming --- |10026 reinject|postfix;
+        home/vmail/mail`"];
+        postfix --- |doveauth.socket|doveauth;
         dovecot --- |doveauth.socket|doveauth;
-        dovecot --- |message delivery|maildir["maildir
+        dovecot --- users;
-        /home/vmail/.../user"];
+        dovecot --- |metadata.socket|chatmail-metadata;
-        dovecot --- |lastlogin.socket|lastlogin;
+        doveauth --- users;
-        dovecot --- chatmail-metadata;
+        chatmail-expire-daily --- users;
-        lastlogin --- maildir;
+        chatmail-fsreport-daily --- users;
-        doveauth --- maildir;
-        chatmail-expire-daily --- maildir;
-        chatmail-fsreport-daily --- maildir;
         chatmail-metadata --- iroh-relay;
-        chatmail-metadata --- |encrypted device token| notifications.delta.chat;
         certs-nginx --> postfix;
         certs-nginx --> dovecot;
         style certs fill:#ff6;
-        style website fill:#ff6;
-        style maildir fill:#ff6;
         style certs-nginx fill:#ff6;
-        style nginx-external fill:#f66;
+        style nginx-external fill:#fc9;
-        style nginx-right fill:#f66;
+        style nginx-right fill:#fc9;
-        style postfix fill:#f66;
-        style dovecot fill:#f66;
-        style notification-proxy fill:#f66;
 
-Message between users on the same relay
----------------------------------------
-
-.. mermaid::
-    :caption: This diagram shows the path a non-federated message takes.
-
-    graph LR;
-        sender --> |465|smtps/smtpd;
-        sender --> |587|submission/smtpd;
-        smtps/smtpd --> |10080|filtermail;
-        submission/smtpd --> |10080|filtermail;
-        filtermail --> |10025|smtpd_reinject;
-        smtpd_reinject --> cleanup;
-        cleanup --> qmgr;
-        qmgr --> smtpd_accepts_message;
-        qmgr --> |lmtp|dovecot;
-        dovecot --> recipient;
-        dovecot --> sender's_other_devices;
 
 Operational details of a chatmail relay
 ----------------------------------------
@@ -273,11 +245,9 @@ Incoming emails must have a valid DKIM signature with
 Signing Domain Identifier (SDID, ``d=`` parameter in the DKIM-Signature
 header) equal to the ``From:`` header domain. This property is checked
 by OpenDKIM screen policy script before validating the signatures. This
-corresponds to strict :rfc:`DMARC <7489>` alignment (``adkim=s``).
+correpsonds to strict :rfc:`DMARC <7489>` alignment (``adkim=s``).
 If there is no valid DKIM signature on the incoming email, the
 sender receives a “5.7.1 No valid DKIM signature found” error.
-After validating the DKIM signature, 
-the `final.lua` script strips all ``OpenDKIM:`` headers to reduce message size on disc. 
 
 Note that chatmail relays
 

doc/source/proxy.rst

@@ -22,12 +22,7 @@ Note that your chatmail relay still needs to be able to make outgoing
 connections on port 25 to send messages outside.
 
 To setup a reverse proxy (or rather Destination NAT, DNAT) for your
-chatmail relay, follow these instructions:
+chatmail relay, put the following configuration in
-
-Linux
-^^^^^
-
-Put the following configuration in
 ``/etc/nftables.conf``:
 
 ::
@@ -115,61 +110,5 @@ Uncomment in ``/etc/sysctl.conf`` the following two lines:
 Then reboot the relay or do ``sysctl -p`` and
 ``nft -f /etc/nftables.conf``.
 
-FreeBSD / pf
-^^^^^^^^^^^^
-
-Put the following configuration in
-``/etc/pf.conf``:
-
-::
-
-    ext_if = "em0"
-    forward_ports = "{ 25, 80, 143, 443, 465, 587, 993 }"
-    chatmail_ipv4 = "AAA.BBB.CCC.DDD"
-    icmp_types = "{ echoreq, echorep, unreach, timex }"
-    chatmail_ipv6 = "XXX::1"
-    icmp6_types = "{ echorep, echoreq, neighbradv, neighbrsol, routeradv, routersol, unreach, toobig, timex }"
-
-    set skip on lo0
-
-    nat on $ext_if inet from any to any -> ($ext_if:0)
-    nat on $ext_if inet6 from any to any -> ($ext_if:0)
-
-    # Define the redirect rules
-    rdr on $ext_if inet proto tcp from any to ($ext_if:0) port $forward_ports -> $chatmail_ipv4
-    rdr on $ext_if inet6 proto tcp from any to ($ext_if:0) port $forward_ports -> $chatmail_ipv6
-
-    # Accept the incoming traffic to the specified ports we will NAT redirect
-    pass in quick on $ext_if inet proto tcp from any to any port $forward_ports flags S/SA modulate state
-    pass in quick on $ext_if inet6 proto tcp from any to any port $forward_ports flags S/SA modulate state
-
-    # Allow incoming SSH for host mgmt
-    pass in quick on $ext_if proto tcp from any to ($ext_if) port 22 flags S/SA modulate state
-
-    # Allow ICMP
-    pass in quick on $ext_if inet proto icmp all icmp-type $icmp_types keep state
-    pass in quick on $ext_if inet6 proto ipv6-icmp all icmp6-type $icmp6_types keep state
-
-    # Allow traffic from anyone to go through the NAT
-    pass on $ext_if inet proto tcp from any to $chatmail_ipv4 flags S/SA modulate state
-    pass on $ext_if inet6 proto tcp from any to $chatmail_ipv6 flags S/SA modulate state
-
-    # Default allow out
-    pass out quick on $ext_if from any to any
-
-    # Default block
-    block drop in log all
-
-Insert into ``/etc/sysctl.conf.local`` the following two lines:
-
-::
-
-    net.inet.ip.forwarding=1
-    net.inet6.ip6.forwarding=1
-
-Activate the sysctls with ``service sysctl onestart``.
-Enable the pf firewall with ``service pf enable``.
-Apply the firewall rules with ``service pf start`` or ``pfctl -f /etc/pf.conf``.
-Note, enabling the firewall may interrupt your SSH session, but you can reconnect.
-
 Once proxy relay is set up, you can add its IP address to the DNS.
+

www/src/robots.txt

@@ -1,2 +0,0 @@
-User-agent: *
-Disallow: /