feat: add tool to analyze deferred queue

It prints all destinations with the number of recipients
and all the reasons. Operator can then try
to fix the problems for destinations,
e.g. by manually adding reverse proxy
addresses to /etc/hosts for failing domains
or routing IP addresses to another interface.

.github/workflows/ci-no-dns.yaml

@@ -1,35 +0,0 @@
-name: No-DNS
-
-on:
-  # Triggers when a PR is merged into main or a direct push occurs
-  push:
-    branches: [ "main" ]
-    
-  # Triggers for any PR (and its subsequent commits) targeting the main branch
-  pull_request:
-    branches: [ "main" ]
-
-# Newest push wins: Prevents multiple runs from clashing and wasting runner efforts
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-
-jobs:
-  no-dns:
-    name: LXC deploy and test
-    uses: chatmail/cmlxc/.github/workflows/lxc-test.yml@d39fe34c39cee6d760c3479325e8dc82b66a8928
-    with:
-      cmlxc_commands: |
-        cmlxc init 
-        # single cmdeploy relay test
-        cmlxc -v deploy-cmdeploy --source ./repo --ipv4-only --no-dns cm0
-        cmlxc -v test-cmdeploy --no-dns cm0
-
-        # cross cmdeploy relay test
-        cmlxc -v deploy-cmdeploy --source ./repo cm1
-        cmlxc -v test-cmdeploy --no-dns cm0 cm1
-
-        # cross cmdeploy/madmail relay tests
-        cmlxc -v deploy-madmail mad0
-        cmlxc -v test-cmdeploy --no-dns cm0 mad0

.github/workflows/ci.yaml

@@ -1,4 +1,4 @@
-name: CI
+name: Run unit-tests and container-based deploy+test verification
 
 on:
   # Triggers when a PR is merged into main or a direct push occurs
@@ -57,8 +57,9 @@ jobs:
 
   lxc-test:
     name: LXC deploy and test
-    uses: chatmail/cmlxc/.github/workflows/lxc-test.yml@d39fe34c39cee6d760c3479325e8dc82b66a8928
+    uses: chatmail/cmlxc/.github/workflows/lxc-test.yml@v0.13.5
     with:
+      cmlxc_version: v0.13.5
       cmlxc_commands: |
         cmlxc init 
         # single cmdeploy relay test

chatmaild/pyproject.toml

@@ -10,7 +10,6 @@ dependencies = [
   "filelock",
   "requests",
   "crypt-r >= 3.13.1 ; python_version >= '3.11'",
-  "domain-validator",
 ]
 
 [tool.setuptools]
@@ -25,6 +24,7 @@ chatmail-metadata = "chatmaild.metadata:main"
 chatmail-expire = "chatmaild.expire:daily_expire_main"
 chatmail-quota-expire = "chatmaild.expire:quota_expire_main"
 chatmail-fsreport = "chatmaild.fsreport:main"
+chatmail-deferred = "chatmaild.deferred:main"
 lastlogin = "chatmaild.lastlogin:main"
 turnserver = "chatmaild.turnserver:main"
 

chatmaild/src/chatmaild/config.py

@@ -1,8 +1,6 @@
-import ipaddress
 from pathlib import Path
 
 import iniconfig
-from domain_validator import DomainValidator
 
 from chatmaild.user import User
 
@@ -21,19 +19,7 @@ def read_config(inipath):
 class Config:
     def __init__(self, inipath, params):
         self._inipath = inipath
-        raw_domain = params["mail_domain"]
+        self.mail_domain = params["mail_domain"]
-        self.mail_domain_bare = raw_domain
-
-        if is_valid_ipv4(raw_domain):
-            self.ipv4_relay = raw_domain
-            self.mail_domain = f"[{raw_domain}]"
-            self.postfix_myhostname = ipaddress.IPv4Address(raw_domain).reverse_pointer
-        else:
-            DomainValidator().validate_domain_re(raw_domain)
-            self.ipv4_relay = None
-            self.mail_domain = raw_domain
-            self.postfix_myhostname = raw_domain
-
         self.max_user_send_per_minute = int(params.get("max_user_send_per_minute", 60))
         self.max_user_send_burst_size = int(params.get("max_user_send_burst_size", 10))
         self.max_mailbox_size = params["max_mailbox_size"]
@@ -67,7 +53,7 @@ class Config:
         self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true"
         self.imap_compress = params.get("imap_compress", "false").lower() == "true"
         if "iroh_relay" not in params:
-            self.iroh_relay = "https://" + raw_domain
+            self.iroh_relay = "https://" + params["mail_domain"]
             self.enable_iroh_relay = True
         else:
             self.iroh_relay = params["iroh_relay"].strip()
@@ -93,17 +79,17 @@ class Config:
                 )
             self.tls_cert_mode = "external"
             self.tls_cert_path, self.tls_key_path = parts
-        elif raw_domain.startswith("_") or self.ipv4_relay:
+        elif self.mail_domain.startswith("_"):
             self.tls_cert_mode = "self"
             self.tls_cert_path = "/etc/ssl/certs/mailserver.pem"
             self.tls_key_path = "/etc/ssl/private/mailserver.key"
         else:
             self.tls_cert_mode = "acme"
-            self.tls_cert_path = f"/var/lib/acme/live/{raw_domain}/fullchain"
+            self.tls_cert_path = f"/var/lib/acme/live/{self.mail_domain}/fullchain"
-            self.tls_key_path = f"/var/lib/acme/live/{raw_domain}/privkey"
+            self.tls_key_path = f"/var/lib/acme/live/{self.mail_domain}/privkey"
 
         # deprecated option
-        mbdir = params.get("mailboxes_dir", f"/home/vmail/mail/{raw_domain}")
+        mbdir = params.get("mailboxes_dir", f"/home/vmail/mail/{self.mail_domain}")
         self.mailboxes_dir = Path(mbdir.strip())
 
         # old unused option (except for first migration from sqlite to maildir store)
@@ -189,27 +175,3 @@ def get_default_config_content(mail_domain, **overrides):
                 lines.append(line)
         content = "\n".join(lines)
     return content
-
-
-def is_valid_ipv4(address: str) -> bool:
-    """Check if a mail_domain is an IPv4 address."""
-    try:
-        ipaddress.IPv4Address(address)
-        return True
-    except ValueError:
-        return False
-
-
-
-def format_arpa_address(address: str) -> str:
-    if is_valid_ipv4(address):
-        return ipaddress.IPv4Address(address).reverse_pointer
-    DomainValidator().validate_domain_re(address)
-    return address
-
-
-def format_mail_domain(raw_domain: str) -> str:
-    if is_valid_ipv4(raw_domain):
-        return f"[{raw_domain}]"
-    DomainValidator().validate_domain_re(raw_domain)
-    return raw_domain

chatmaild/src/chatmaild/deferred.py

@@ -0,0 +1,37 @@
+"""
+Analyze deferred mails and print most common failing destinations.
+
+Example:
+
+    python -m chatmaild.deferred
+"""
+
+import json
+import subprocess
+from collections import Counter, defaultdict
+
+
+def main():
+    p = subprocess.Popen(["postqueue", "-j"], text=True, stdout=subprocess.PIPE)
+    domain_reasons = defaultdict(Counter)
+    domain_total = Counter()
+
+    for line in p.stdout:
+        item = json.loads(line)
+        if item["queue_name"] != "deferred":
+            continue
+
+        for recipient in item["recipients"]:
+            _, domain = recipient["address"].rsplit("@", 1)
+            reason = recipient["delay_reason"].removeprefix("host 127.0.0.1[127.0.0.1] said: ")
+            domain_total[domain] += 1
+            domain_reasons[domain][reason] += 1
+
+    for domain, total in reversed(domain_total.most_common()):
+        print(f"{domain} ({total} recipients)")
+        for reason, count in domain_reasons[domain].most_common():
+            print(f"  {count}: {reason}")
+
+
+if __name__ == "__main__":
+    main()

chatmaild/src/chatmaild/metadata.py

@@ -70,6 +70,9 @@ class Metadata:
                 # Some tokens have expired, remove them.
                 with self._modify_tokens(addr) as _tokens:
                     pass
+        elif isinstance(tokens, list):
+            with self._modify_tokens(addr) as tokens:
+                token_list = list(tokens.keys())
         else:
             token_list = []
         return token_list

chatmaild/src/chatmaild/newemail.py

@@ -2,6 +2,7 @@
 
 """CGI script for creating new accounts."""
 
+import ipaddress
 import json
 import secrets
 import string
@@ -14,6 +15,16 @@ ALPHANUMERIC = string.ascii_lowercase + string.digits
 ALPHANUMERIC_PUNCT = string.ascii_letters + string.digits + string.punctuation
 
 
+def wrap_ip(host):
+    if host.startswith("[") and host.endswith("]"):
+        return host
+    try:
+        ipaddress.ip_address(host)
+        return f"[{host}]"
+    except ValueError:
+        return host
+
+
 def create_newemail_dict(config: Config):
     user = "".join(
         secrets.choice(ALPHANUMERIC) for _ in range(config.username_max_length)
@@ -22,22 +33,16 @@ def create_newemail_dict(config: Config):
         secrets.choice(ALPHANUMERIC_PUNCT)
         for _ in range(config.password_min_length + 3)
     )
-    return dict(email=f"{user}@{config.mail_domain}", password=f"{password}")
+    return dict(email=f"{user}@{wrap_ip(config.mail_domain)}", password=f"{password}")
 
 
-def create_dclogin_url(config, email, password):
+def create_dclogin_url(email, password):
     """Build a dclogin: URL with credentials and self-signed cert acceptance.
 
     Uses ic=3 (AcceptInvalidCertificates) so chatmail clients
     can connect to servers with self-signed TLS certificates.
     """
-    if config.ipv4_relay:
+    return f"dclogin:{quote(email, safe='@')}?p={quote(password, safe='')}&v=1&ic=3"
-        imap_host = "&ih=" + config.ipv4_relay
-        smtp_host = "&sh=" + config.ipv4_relay
-    else:
-        imap_host = ""
-        smtp_host = ""
-    return f"dclogin:{quote(email, safe='@[]')}?p={quote(password, safe='')}&v=1{imap_host}{smtp_host}&ic=3"
 
 
 def print_new_account():
@@ -46,9 +51,7 @@ def print_new_account():
 
     result = dict(email=creds["email"], password=creds["password"])
     if config.tls_cert_mode == "self":
-        result["dclogin_url"] = create_dclogin_url(
+        result["dclogin_url"] = create_dclogin_url(creds["email"], creds["password"])
-            config, creds["email"], creds["password"]
-        )
 
     print("Content-Type: application/json")
     print("")

chatmaild/src/chatmaild/tests/plugin.py

@@ -31,11 +31,6 @@ def example_config(make_config):
     return make_config("chat.example.org")
 
 
-@pytest.fixture
-def ipv4_config(make_config):
-    return make_config("1.3.3.7")
-
-
 @pytest.fixture
 def maildomain(example_config):
     return example_config.mail_domain

chatmaild/src/chatmaild/tests/test_config.py

@@ -1,14 +1,6 @@
-from contextlib import nullcontext as does_not_raise
-
 import pytest
 
-from chatmaild.config import (
+from chatmaild.config import parse_size_mb, read_config
-    format_arpa_address,
-    format_mail_domain,
-    is_valid_ipv4,
-    parse_size_mb,
-    read_config,
-)
 
 
 def test_read_config_basic(example_config):
@@ -21,12 +13,6 @@ def test_read_config_basic(example_config):
     example_config = read_config(inipath)
     assert example_config.max_user_send_per_minute == 37
     assert example_config.mail_domain == "chat.example.org"
-    assert example_config.ipv4_relay is None
-
-
-def test_read_config_ipv4(ipv4_config):
-    assert ipv4_config.ipv4_relay == "1.3.3.7"
-    assert ipv4_config.mail_domain == "[1.3.3.7]"
 
 
 def test_read_config_basic_using_defaults(tmp_path, maildomain):
@@ -149,45 +135,3 @@ def test_max_mailbox_size_mb(make_config):
     config = make_config("chat.example.org")
     assert config.max_mailbox_size == "500M"
     assert config.max_mailbox_size_mb == 500
-
-
-@pytest.mark.parametrize(
-    ["input", "result"],
-    [
-        ("example.org", False),
-        ("1.3.3.7", True),
-        ("fe::1", False),
-        ("ad.1e.dag.adf", False),
-        ("12394142", False),
-    ],
-)
-def test_is_valid_ipv4(input, result):
-    assert result == is_valid_ipv4(input)
-
-
-@pytest.mark.parametrize(
-    ["input", "result", "exception"],
-    [
-        ("example.org", "example.org", does_not_raise()),
-        ("1.3.3.7", "7.3.3.1.in-addr.arpa", does_not_raise()),
-        ("fe::1", None, pytest.raises(ValueError)),
-        ("12394142", None, pytest.raises(ValueError)),
-    ],
-)
-def test_format_arpa_address(input, result, exception):
-    with exception:
-        assert result == format_arpa_address(input)
-
-
-@pytest.mark.parametrize(
-    ["input", "result", "exception"],
-    [
-        ("example.org", "example.org", does_not_raise()),
-        ("1.3.3.7", "[1.3.3.7]", does_not_raise()),
-        ("fe::1", None, pytest.raises(ValueError)),
-        ("12394142", None, pytest.raises(ValueError)),
-    ],
-)
-def test_format_mail_domain(input, result, exception):
-    with exception:
-        assert result == format_mail_domain(input)

chatmaild/src/chatmaild/tests/test_metadata.py

@@ -372,3 +372,14 @@ def test_iroh_relay(dictproxy):
     dictproxy.iroh_relay = "https://example.org/"
     dictproxy.loop_forever(rfile, wfile)
     assert wfile.getvalue() == b"Ohttps://example.org/\n"
+
+
+def test_legacy_token_migration(metadata, testaddr):
+    with metadata.get_metadata_dict(testaddr).modify() as data:
+        data[metadata.DEVICETOKEN_KEY] = ["oldtoken1", "oldtoken2"]
+
+    assert metadata.get_tokens_for_addr(testaddr) == ["oldtoken1", "oldtoken2"]
+    mdict = metadata.get_metadata_dict(testaddr).read()
+    tokens = mdict[metadata.DEVICETOKEN_KEY]
+    assert isinstance(tokens, dict)
+    assert "oldtoken1" in tokens and "oldtoken2" in tokens

chatmaild/src/chatmaild/tests/test_migrate_db.py

@@ -48,6 +48,8 @@ def test_migration(tmp_path, example_config, caplog):
     assert passdb_path.stat().st_size > 10000
 
     example_config.passdb_path = passdb_path
+    # ensure logging.info records are captured regardless of global configuration
+    caplog.set_level("INFO")
 
     assert not caplog.records
 

chatmaild/src/chatmaild/tests/test_newmail.py

@@ -19,35 +19,24 @@ def test_create_newemail_dict(example_config):
     assert ac1["password"] != ac2["password"]
 
 
-def test_create_newemail_dict_ip(ipv4_config):
+def test_create_newemail_dict_ip(make_config):
-    ac = create_newemail_dict(ipv4_config)
+    config = make_config("1.2.3.4")
-    assert ac["email"].endswith("@[1.3.3.7]")
+    ac = create_newemail_dict(config)
+    assert ac["email"].endswith("@[1.2.3.4]")
 
 
-def test_create_dclogin_url(example_config):
+def test_create_dclogin_url():
-    addr = "user@example.org"
+    url = create_dclogin_url("user@example.org", "p@ss w+rd")
-    password = "p@ss w+rd"
-    url = create_dclogin_url(example_config, addr, password)
     assert url.startswith("dclogin:")
     assert "v=1" in url
     assert "ic=3" in url
 
-    assert addr in url
+    assert "user@example.org" in url
     # password special chars must be encoded
     assert "p%40ss" in url
     assert "w%2Brd" in url
 
 
-def test_create_dclogin_url_ipv4(ipv4_config):
-    addr = "user@[1.3.3.7]"
-    password = "p@ss w+rd"
-    url = create_dclogin_url(ipv4_config, addr, password)
-    assert url.startswith("dclogin:")
-    assert "v=1" in url
-    assert "ic=3" in url
-    assert addr in url
-
-
 def test_print_new_account(capsys, monkeypatch, maildomain, tmpdir, example_config):
     monkeypatch.setattr(chatmaild.newemail, "CONFIG_PATH", str(example_config._inipath))
     print_new_account()

cmdeploy/src/cmdeploy/acmetool/__init__.py

@@ -1,6 +1,4 @@
-import importlib.resources
+from pyinfra.operations import apt, server
-
-from pyinfra.operations import apt, files, server, systemd
 
 from ..basedeploy import Deployer
 
@@ -9,9 +7,6 @@ class AcmetoolDeployer(Deployer):
     def __init__(self, email, domains):
         self.domains = domains
         self.email = email
-        self.need_restart_redirector = False
-        self.need_restart_reconcile_service = False
-        self.need_restart_reconcile_timer = False
 
     def install(self):
         apt.packages(
@@ -19,121 +14,41 @@ class AcmetoolDeployer(Deployer):
             packages=["acmetool"],
         )
 
-        files.file(
+        self.remove_file("/etc/cron.d/acmetool")
-            name="Remove old acmetool cronjob, it is replaced with systemd timer.",
-            path="/etc/cron.d/acmetool",
-            present=False,
-        )
 
-        files.put(
+        self.put_executable("acmetool/acmetool.hook", "/etc/acme/hooks/nginx")
-            name="Install acmetool hook.",
+        self.remove_file("/usr/lib/acme/hooks/nginx")
-            src=importlib.resources.files(__package__)
-            .joinpath("acmetool.hook")
-            .open("rb"),
-            dest="/etc/acme/hooks/nginx",
-            user="root",
-            group="root",
-            mode="755",
-        )
-        files.file(
-            name="Remove acmetool hook from the wrong location where it was previously installed.",
-            path="/usr/lib/acme/hooks/nginx",
-            present=False,
-        )
 
     def configure(self):
-        files.template(
+        self.put_template(
-            src=importlib.resources.files(__package__).joinpath(
+            "acmetool/response-file.yaml.j2",
-                "response-file.yaml.j2"
+            "/var/lib/acme/conf/responses",
-            ),
-            dest="/var/lib/acme/conf/responses",
-            user="root",
-            group="root",
-            mode="644",
             email=self.email,
         )
 
-        files.template(
+        self.put_template(
-            src=importlib.resources.files(__package__).joinpath("target.yaml.j2"),
+            "acmetool/target.yaml.j2",
-            dest="/var/lib/acme/conf/target",
+            "/var/lib/acme/conf/target",
-            user="root",
-            group="root",
-            mode="644",
         )
 
         server.shell(
             name=f"Remove old acmetool desired files for {self.domains[0]}",
             commands=[f"rm -f /var/lib/acme/desired/{self.domains[0]}-*"],
         )
-        files.template(
+        self.put_template(
-            src=importlib.resources.files(__package__).joinpath("desired.yaml.j2"),
+            "acmetool/desired.yaml.j2",
-            dest=f"/var/lib/acme/desired/{self.domains[0]}",  # 0 is mailhost TLD
+            f"/var/lib/acme/desired/{self.domains[0]}",
-            user="root",
-            group="root",
-            mode="644",
             domains=self.domains,
         )
 
-        service_file = files.put(
+        self.ensure_systemd_unit("acmetool/acmetool-redirector.service")
-            src=importlib.resources.files(__package__).joinpath(
+        self.ensure_systemd_unit("acmetool/acmetool-reconcile.service")
-                "acmetool-redirector.service"
+        self.ensure_systemd_unit("acmetool/acmetool-reconcile.timer")
-            ),
-            dest="/etc/systemd/system/acmetool-redirector.service",
-            user="root",
-            group="root",
-            mode="644",
-        )
-        self.need_restart_redirector = service_file.changed
-
-        reconcile_service_file = files.put(
-            src=importlib.resources.files(__package__).joinpath(
-                "acmetool-reconcile.service"
-            ),
-            dest="/etc/systemd/system/acmetool-reconcile.service",
-            user="root",
-            group="root",
-            mode="644",
-        )
-        self.need_restart_reconcile_service = reconcile_service_file.changed
-
-        reconcile_timer_file = files.put(
-            src=importlib.resources.files(__package__).joinpath(
-                "acmetool-reconcile.timer"
-            ),
-            dest="/etc/systemd/system/acmetool-reconcile.timer",
-            user="root",
-            group="root",
-            mode="644",
-        )
-        self.need_restart_reconcile_timer = reconcile_timer_file.changed
 
     def activate(self):
-        systemd.service(
+        self.ensure_service("acmetool-redirector.service")
-            name="Setup acmetool-redirector service",
+        self.ensure_service("acmetool-reconcile.service", running=False, enabled=False)
-            service="acmetool-redirector.service",
+        self.ensure_service("acmetool-reconcile.timer")
-            running=True,
-            enabled=True,
-            restarted=self.need_restart_redirector,
-        )
-        self.need_restart_redirector = False
-
-        systemd.service(
-            name="Setup acmetool-reconcile service",
-            service="acmetool-reconcile.service",
-            running=False,
-            enabled=False,
-            daemon_reload=self.need_restart_reconcile_service,
-        )
-        self.need_restart_reconcile_service = False
-
-        systemd.service(
-            name="Setup acmetool-reconcile timer",
-            service="acmetool-reconcile.timer",
-            running=True,
-            enabled=True,
-            daemon_reload=self.need_restart_reconcile_timer,
-        )
-        self.need_restart_reconcile_timer = False
 
         server.shell(
             name=f"Reconcile certificates for: {', '.join(self.domains)}",

cmdeploy/src/cmdeploy/basedeploy.py

@@ -4,6 +4,7 @@ import os
 from contextlib import contextmanager
 
 from pyinfra import host
+from pyinfra.facts.files import Sha256File
 from pyinfra.facts.server import Command
 from pyinfra.operations import files, server, systemd
 
@@ -50,11 +51,10 @@ def get_resource(arg, pkg=__package__):
     return importlib.resources.files(pkg).joinpath(arg)
 
 
-def configure_remote_units(mail_domain, units) -> None:
+def configure_remote_units(deployer, mail_domain, units) -> None:
     remote_base_dir = "/usr/local/lib/chatmaild"
     remote_venv_dir = f"{remote_base_dir}/venv"
     remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini"
-    root_owned = dict(user="root", group="root", mode="644")
 
     # install systemd units
     for fn in units:
@@ -70,15 +70,13 @@ def configure_remote_units(mail_domain, units) -> None:
         source_path = get_resource(f"service/{basename}.f")
         content = source_path.read_text().format(**params).encode()
 
-        files.put(
+        deployer.put_file(
-            name=f"Upload {basename}",
             src=io.BytesIO(content),
             dest=f"/etc/systemd/system/{basename}",
-            **root_owned,
         )
 
 
-def activate_remote_units(units) -> None:
+def activate_remote_units(deployer, units) -> None:
     # activate systemd units
     for fn in units:
         basename = fn if "." in fn else f"{fn}.service"
@@ -88,14 +86,8 @@ def activate_remote_units(units) -> None:
             enabled = False
         else:
             enabled = True
-        systemd.service(
+
-            name=f"Setup {basename}",
+        deployer.ensure_service(basename, running=enabled, enabled=enabled)
-            service=basename,
-            running=enabled,
-            enabled=enabled,
-            restarted=enabled,
-            daemon_reload=True,
-        )
 
 
 class Deployment:
@@ -141,6 +133,7 @@ class Deployment:
 
 class Deployer:
     need_restart = False
+    daemon_reload = False
 
     def install(self):
         pass
@@ -150,3 +143,113 @@ class Deployer:
 
     def activate(self):
         pass
+
+    def ensure_service(self, service, running=True, enabled=True):
+        if running:
+            verb = "Start and enable"
+        else:
+            verb = "Stop"
+        systemd.service(
+            name=f"{verb} {service}",
+            service=service,
+            running=running,
+            enabled=enabled,
+            restarted=self.need_restart if running else False,
+            daemon_reload=self.daemon_reload,
+        )
+        self.daemon_reload = False
+
+    def ensure_systemd_unit(self, src, **kwargs):
+        dest_name = src.split("/")[-1].replace(".j2", "")
+        dest = f"/etc/systemd/system/{dest_name}"
+        if src.endswith(".j2"):
+            return self.put_template(src, dest, **kwargs)
+        return self.put_file(src, dest)
+
+    def put_file(self, src, dest, mode="644"):
+        if isinstance(src, str):
+            src = get_resource(src)
+        res = files.put(
+            name=f"Upload {dest}",
+            src=src,
+            dest=dest,
+            user="root",
+            group="root",
+            mode=mode,
+        )
+
+        return self._update_restart_signals(dest, res)
+
+    def put_executable(self, src, dest):
+        return self.put_file(src, dest, mode="755")
+
+    def put_template(self, src, dest, owner="root", **kwargs):
+        if isinstance(src, str):
+            src = get_resource(src)
+        res = files.template(
+            name=f"Upload {dest}",
+            src=src,
+            dest=dest,
+            user=owner,
+            group=owner,
+            mode="644",
+            **kwargs,
+        )
+
+        return self._update_restart_signals(dest, res)
+
+    def remove_file(self, dest):
+        res = files.file(name=f"Remove {dest}", path=dest, present=False)
+        return self._update_restart_signals(dest, res)
+
+    def ensure_line(self, path, line, **kwargs):
+        name = kwargs.pop("name", f"Ensure line in {path}")
+        res = files.line(name=name, path=path, line=line, **kwargs)
+        return self._update_restart_signals(path, res)
+
+    def ensure_directory(self, path, owner="root", mode="755", **kwargs):
+        name = kwargs.pop("name", f"Ensure directory {path}")
+        res = files.directory(
+            name=name,
+            path=path,
+            user=owner,
+            group=owner,
+            mode=mode,
+            present=True,
+            **kwargs,
+        )
+        return self._update_restart_signals(path, res)
+
+    def remove_directory(self, path, **kwargs):
+        name = kwargs.pop("name", f"Remove directory {path}")
+        res = files.directory(name=name, path=path, present=False, **kwargs)
+        return self._update_restart_signals(path, res)
+
+    def download_executable(self, url, dest, sha256sum, extract=None):
+        existing = host.get_fact(Sha256File, dest)
+        if existing == sha256sum:
+            return
+
+        tmp = f"{dest}.new"
+        if extract:
+            dl_cmd = f"curl -fSL {url} | {extract} >{tmp}"
+        else:
+            dl_cmd = f"curl -fSL {url} -o {tmp}"
+
+        server.shell(
+            name=f"Download {dest}",
+            commands=[
+                f"({dl_cmd}"
+                f" && echo '{sha256sum}  {tmp}' | sha256sum -c"
+                f" && mv {tmp} {dest})",
+                f"chmod 755 {dest}",
+            ],
+        )
+        self.need_restart = True
+
+    def _update_restart_signals(self, path, res):
+        if res.changed:
+            self.need_restart = True
+            if str(path).startswith("/etc/systemd/system/"):
+                self.daemon_reload = True
+        return res

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -87,12 +87,10 @@ def run_cmd_options(parser):
 def run_cmd(args, out):
     """Deploy chatmail services on the remote server."""
 
-    ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain_bare
+    ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
     sshexec = get_sshexec(ssh_host)
     require_iroh = args.config.enable_iroh_relay
     strict_tls = args.config.tls_cert_mode == "acme"
-    if args.config.ipv4_relay:
-        args.dns_check_disabled = True
     if not args.dns_check_disabled:
         remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
         if not dns.check_initial_remote_data(remote_data, strict_tls=strict_tls, print=out.red):
@@ -121,8 +119,6 @@ def run_cmd(args, out):
         elif not args.dns_check_disabled and strict_tls and not remote_data["acme_account_url"]:
             out.red("Deploy completed but letsencrypt not configured")
             out.red("Run 'cmdeploy run' again")
-        elif args.config.ipv4_relay:
-            out.green("Deploy completed.")
         else:
             out.green("Deploy completed, call `cmdeploy dns` next.")
         return 0
@@ -144,10 +140,6 @@ def dns_cmd_options(parser):
 
 def dns_cmd(args, out):
     """Check DNS entries and optionally generate dns zone file."""
-    if args.config.ipv4_relay:
-        ipv4 = args.config.ipv4_relay
-        print(f"[WARNING] {ipv4} is not a domain, skipping DNS checks.")
-        return 0
     ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
     sshexec = get_sshexec(ssh_host, verbose=args.verbose)
     tls_cert_mode = args.config.tls_cert_mode
@@ -185,7 +177,7 @@ def status_cmd_options(parser):
 def status_cmd(args, out):
     """Display status for online chatmail instance."""
 
-    ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain_bare
+    ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
     sshexec = get_sshexec(ssh_host, verbose=args.verbose)
 
     out.green(f"chatmail domain: {args.config.mail_domain}")

cmdeploy/src/cmdeploy/deployers.py

@@ -12,7 +12,6 @@ from chatmaild.config import read_config
 from pyinfra import facts, host, logger
 from pyinfra.api import FactBase
 from pyinfra.facts import hardware
-from pyinfra.facts.files import Sha256File
 from pyinfra.facts.systemd import SystemdEnabled
 from pyinfra.operations import apt, files, pip, server, systemd
 
@@ -25,7 +24,6 @@ from .basedeploy import (
     activate_remote_units,
     blocked_service_startup,
     configure_remote_units,
-    get_resource,
     has_systemd,
     is_in_container,
 )
@@ -82,25 +80,22 @@ def remove_legacy_artifacts():
         )
 
 
-def _install_remote_venv_with_chatmaild() -> None:
+def _install_remote_venv_with_chatmaild(deployer) -> None:
     remove_legacy_artifacts()
     dist_file = _build_chatmaild(dist_dir=Path("chatmaild/dist"))
     remote_base_dir = "/usr/local/lib/chatmaild"
     remote_dist_file = f"{remote_base_dir}/dist/{dist_file.name}"
     remote_venv_dir = f"{remote_base_dir}/venv"
-    root_owned = dict(user="root", group="root", mode="644")
 
     apt.packages(
         name="apt install python3-virtualenv",
         packages=["python3-virtualenv"],
     )
 
-    files.put(
+    deployer.ensure_directory(f"{remote_base_dir}/dist")
-        name="Upload chatmaild source package",
+    deployer.put_file(
         src=dist_file.open("rb"),
         dest=remote_dist_file,
-        create_remote_dir=True,
-        **root_owned,
     )
 
     pip.virtualenv(
@@ -122,32 +117,22 @@ def _install_remote_venv_with_chatmaild() -> None:
     )
 
 
-def _configure_remote_venv_with_chatmaild(config) -> None:
+def _configure_remote_venv_with_chatmaild(deployer, config) -> None:
     remote_base_dir = "/usr/local/lib/chatmaild"
     remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini"
-    root_owned = dict(user="root", group="root", mode="644")
 
-    files.put(
+    deployer.put_file(
-        name=f"Upload {remote_chatmail_inipath}",
         src=config._getbytefile(),
         dest=remote_chatmail_inipath,
-        **root_owned,
     )
 
-    files.file(
+    deployer.remove_file("/etc/cron.d/chatmail-metrics")
-        path="/etc/cron.d/chatmail-metrics",
+    deployer.remove_file("/var/www/html/metrics")
-        present=False,
-    )
-    files.file(
-        path="/var/www/html/metrics",
-        present=False,
-    )
 
 
 class UnboundDeployer(Deployer):
     def __init__(self, config):
         self.config = config
-        self.need_restart = False
 
     def install(self):
         # On an IPv4-only system, if unbound is started but not configured,
@@ -176,13 +161,9 @@ class UnboundDeployer(Deployer):
         )
         # Configure unbound resolver with Quad9 fallback and a trailing newline
         # (SolusVM bug).
-        files.put(
+        self.put_file(
-            name="Write static resolv.conf",
             src=BytesIO(b"nameserver 127.0.0.1\nnameserver 9.9.9.9\n"),
             dest="/etc/resolv.conf",
-            user="root",
-            group="root",
-            mode="644",
         )
         server.shell(
             name="Generate root keys for validating DNSSEC",
@@ -191,26 +172,15 @@ class UnboundDeployer(Deployer):
             ],
         )
         if self.config.disable_ipv6:
-            files.directory(
+            self.ensure_directory(
                 path="/etc/unbound/unbound.conf.d",
-                present=True,
-                user="root",
-                group="root",
-                mode="755",
             )
-            conf = files.put(
+            self.put_template(
-                src=get_resource("unbound/unbound.conf.j2"),
+                "unbound/unbound.conf.j2",
-                dest="/etc/unbound/unbound.conf.d/chatmail.conf",
+                "/etc/unbound/unbound.conf.d/chatmail.conf",
-                user="root",
-                group="root",
-                mode="644",
             )
         else:
-            conf = files.file(
+            self.remove_file("/etc/unbound/unbound.conf.d/chatmail.conf")
-                path="/etc/unbound/unbound.conf.d/chatmail.conf",
-                present=False,
-            )
-        self.need_restart |= conf.changed
 
     def activate(self):
         server.shell(
@@ -220,27 +190,25 @@ class UnboundDeployer(Deployer):
             ],
         )
 
-        systemd.service(
+        self.ensure_service("unbound.service")
-            name="Start and enable unbound",
+
-            service="unbound.service",
+        self.ensure_service(
-            running=True,
+            "unbound-resolvconf.service",
-            enabled=True,
+            running=False,
-            restarted=self.need_restart,
+            enabled=False,
         )
 
 
 class MtastsDeployer(Deployer):
     def configure(self):
         # Remove configuration.
-        files.file("/etc/mta-sts-daemon.yml", present=False)
+        self.remove_file("/etc/mta-sts-daemon.yml")
-        files.directory("/usr/local/lib/postfix-mta-sts-resolver", present=False)
+        self.remove_directory("/usr/local/lib/postfix-mta-sts-resolver")
-        files.file("/etc/systemd/system/mta-sts-daemon.service", present=False)
+        self.remove_file("/etc/systemd/system/mta-sts-daemon.service")
 
     def activate(self):
-        systemd.service(
+        self.ensure_service(
-            name="Stop MTA-STS daemon",
+            "mta-sts-daemon.service",
-            service="mta-sts-daemon.service",
-            daemon_reload=True,
             running=False,
             enabled=False,
         )
@@ -251,14 +219,7 @@ class WebsiteDeployer(Deployer):
         self.config = config
 
     def install(self):
-        files.directory(
+        self.ensure_directory("/var/www")
-            name="Ensure /var/www exists",
-            path="/var/www",
-            user="root",
-            group="root",
-            mode="755",
-            present=True,
-        )
 
     def configure(self):
         www_path, src_dir, build_dir = get_paths(self.config)
@@ -288,15 +249,11 @@ class LegacyRemoveDeployer(Deployer):
 
         # remove historic expunge script
         # which is now implemented through a systemd timer (chatmail-expire)
-        files.file(
+        self.remove_file("/etc/cron.d/expunge")
-            path="/etc/cron.d/expunge",
-            present=False,
-        )
 
         # Remove OBS repository key that is no longer used.
-        files.file("/etc/apt/keyrings/obs-home-deltachat.gpg", present=False)
+        self.remove_file("/etc/apt/keyrings/obs-home-deltachat.gpg")
-        files.line(
+        self.ensure_line(
-            name="Remove DeltaChat OBS home repository from sources.list",
             path="/etc/apt/sources.list",
             line="deb [signed-by=/etc/apt/keyrings/obs-home-deltachat.gpg] https://download.opensuse.org/repositories/home:/deltachat/Debian_12/ ./",
             escape_regex_characters=True,
@@ -304,11 +261,7 @@ class LegacyRemoveDeployer(Deployer):
         )
 
         # prior relay versions used filelogging
-        files.directory(
+        self.remove_directory("/var/log/journal/")
-            name="Ensure old logs on disk are deleted",
-            path="/var/log/journal/",
-            present=False,
-        )
         # remove echobot if it is still running
         if has_systemd() and host.get_fact(SystemdEnabled).get("echobot.service"):
             systemd.service(
@@ -350,22 +303,13 @@ class TurnDeployer(Deployer):
                 "0fb3e792419494e21ecad536464929dba706bb2c88884ed8f1788141d26fc756",
             ),
         }[host.get_fact(facts.server.Arch)]
-
+        self.download_executable(url, "/usr/local/bin/chatmail-turn", sha256sum)
-        existing_sha256sum = host.get_fact(Sha256File, "/usr/local/bin/chatmail-turn")
-        if existing_sha256sum != sha256sum:
-            server.shell(
-                name="Download chatmail-turn",
-                commands=[
-                    f"(curl -L {url} >/usr/local/bin/chatmail-turn.new && (echo '{sha256sum} /usr/local/bin/chatmail-turn.new' | sha256sum -c) && mv /usr/local/bin/chatmail-turn.new /usr/local/bin/chatmail-turn)",
-                    "chmod 755 /usr/local/bin/chatmail-turn",
-                ],
-            )
 
     def configure(self):
-        configure_remote_units(self.mail_domain, self.units)
+        configure_remote_units(self, self.mail_domain, self.units)
 
     def activate(self):
-        activate_remote_units(self.units)
+        activate_remote_units(self, self.units)
 
 
 class IrohDeployer(Deployer):
@@ -383,72 +327,30 @@ class IrohDeployer(Deployer):
                 "f8ef27631fac213b3ef668d02acd5b3e215292746a3fc71d90c63115446008b1",
             ),
         }[host.get_fact(facts.server.Arch)]
-
+        self.download_executable(
-        existing_sha256sum = host.get_fact(Sha256File, "/usr/local/bin/iroh-relay")
+            url,
-        if existing_sha256sum != sha256sum:
+            "/usr/local/bin/iroh-relay",
-            server.shell(
+            sha256sum,
-                name="Download iroh-relay",
+            extract="gunzip | tar -xf - ./iroh-relay -O",
-                commands=[
+        )
-                    f"(curl -L {url} | gunzip | tar -x -f - ./iroh-relay -O >/usr/local/bin/iroh-relay.new && (echo '{sha256sum} /usr/local/bin/iroh-relay.new' | sha256sum -c) && mv /usr/local/bin/iroh-relay.new /usr/local/bin/iroh-relay)",
-                    "chmod 755 /usr/local/bin/iroh-relay",
-                ],
-            )
-
-            self.need_restart = True
 
     def configure(self):
-        systemd_unit = files.put(
+        self.ensure_systemd_unit("iroh-relay.service")
-            name="Upload iroh-relay systemd unit",
+        self.put_file("iroh-relay.toml", "/etc/iroh-relay.toml")
-            src=get_resource("iroh-relay.service"),
-            dest="/etc/systemd/system/iroh-relay.service",
-            user="root",
-            group="root",
-            mode="644",
-        )
-        self.need_restart |= systemd_unit.changed
-
-        iroh_config = files.put(
-            name="Upload iroh-relay config",
-            src=get_resource("iroh-relay.toml"),
-            dest="/etc/iroh-relay.toml",
-            user="root",
-            group="root",
-            mode="644",
-        )
-        self.need_restart |= iroh_config.changed
 
     def activate(self):
-        systemd.service(
+        self.ensure_service(
-            name="Start and enable iroh-relay",
+            "iroh-relay.service",
-            service="iroh-relay.service",
-            running=True,
             enabled=self.enable_iroh_relay,
-            restarted=self.need_restart,
         )
-        self.need_restart = False
 
 
 class JournaldDeployer(Deployer):
     def configure(self):
-        journald_conf = files.put(
+        self.put_file("journald.conf", "/etc/systemd/journald.conf")
-            name="Configure journald",
-            src=get_resource("journald.conf"),
-            dest="/etc/systemd/journald.conf",
-            user="root",
-            group="root",
-            mode="644",
-        )
-        self.need_restart = journald_conf.changed
 
     def activate(self):
-        systemd.service(
+        self.ensure_service("systemd-journald.service")
-            name="Start and enable journald",
-            service="systemd-journald.service",
-            running=True,
-            enabled=True,
-            restarted=self.need_restart,
-        )
-        self.need_restart = False
 
 
 class ChatmailVenvDeployer(Deployer):
@@ -464,14 +366,14 @@ class ChatmailVenvDeployer(Deployer):
         )
 
     def install(self):
-        _install_remote_venv_with_chatmaild()
+        _install_remote_venv_with_chatmaild(self)
 
     def configure(self):
-        _configure_remote_venv_with_chatmaild(self.config)
+        _configure_remote_venv_with_chatmaild(self, self.config)
-        configure_remote_units(self.config.mail_domain_bare, self.units)
+        configure_remote_units(self, self.config.mail_domain, self.units)
 
     def activate(self):
-        activate_remote_units(self.units)
+        activate_remote_units(self, self.units)
 
 
 class ChatmailDeployer(Deployer):
@@ -485,13 +387,9 @@ class ChatmailDeployer(Deployer):
         self.mail_domain = config.mail_domain
 
     def install(self):
-        files.put(
+        self.put_file(
-            name="Disable installing recommended packages globally",
             src=BytesIO(b'APT::Install-Recommends "false";\n'),
             dest="/etc/apt/apt.conf.d/00InstallRecommends",
-            user="root",
-            group="root",
-            mode="644",
         )
         apt.update(name="apt update", cache_time=24 * 3600)
         apt.upgrade(name="upgrade apt packages", auto_remove=True)
@@ -508,13 +406,10 @@ class ChatmailDeployer(Deployer):
 
     def configure(self):
         # metadata crashes if the mailboxes dir does not exist
-        files.directory(
+        self.ensure_directory(
-            name="Ensure vmail mailbox directory exists",
+            str(self.config.mailboxes_dir),
-            path=str(self.config.mailboxes_dir),
+            owner="vmail",
-            user="vmail",
-            group="vmail",
             mode="700",
-            present=True,
         )
 
         # This file is used by auth proxy.
@@ -535,12 +430,7 @@ class FcgiwrapDeployer(Deployer):
         )
 
     def activate(self):
-        systemd.service(
+        self.ensure_service("fcgiwrap.service")
-            name="Start and enable fcgiwrap",
-            service="fcgiwrap.service",
-            running=True,
-            enabled=True,
-        )
 
 
 class GithashDeployer(Deployer):
@@ -553,12 +443,7 @@ class GithashDeployer(Deployer):
             git_diff = subprocess.check_output(["git", "diff"]).decode()
         except Exception:
             git_diff = ""
-        files.put(
+        self.put_file(src=StringIO(git_hash + git_diff), dest="/etc/chatmail-version")
-            name="Upload chatmail relay git commit hash",
-            src=StringIO(git_hash + git_diff),
-            dest="/etc/chatmail-version",
-            mode="700",
-        )
 
 
 def get_tls_deployer(config, mail_domain):
@@ -584,18 +469,24 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
     """
     config = read_config(config_path)
     check_config(config)
-    bare_host = config.mail_domain_bare
+    mail_domain = config.mail_domain
 
     if website_only:
         Deployment().perform_stages([WebsiteDeployer(config)])
         return
 
     # Check if mtail_address interface is available (if configured)
-    if config.mtail_address and config.mtail_address not in ('127.0.0.1', '::1', 'localhost'):
+    if config.mtail_address and config.mtail_address not in (
+        "127.0.0.1",
+        "::1",
+        "localhost",
+    ):
         ipv4_addrs = host.get_fact(hardware.Ipv4Addrs)
         all_addresses = [addr for addrs in ipv4_addrs.values() for addr in addrs]
         if config.mtail_address not in all_addresses:
-            Out().red(f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n")
+            Out().red(
+                f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n"
+            )
             exit(1)
 
     if not is_in_container():
@@ -635,7 +526,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
                     )
                     exit(1)
 
-    tls_deployer = get_tls_deployer(config, bare_host)
+    tls_deployer = get_tls_deployer(config, mail_domain)
 
     all_deployers = [
         ChatmailDeployer(config),
@@ -643,13 +534,13 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
         FiltermailDeployer(),
         JournaldDeployer(),
         UnboundDeployer(config),
-        TurnDeployer(bare_host),
+        TurnDeployer(mail_domain),
         IrohDeployer(config.enable_iroh_relay),
         tls_deployer,
         WebsiteDeployer(config),
         ChatmailVenvDeployer(config),
         MtastsDeployer(),
-        OpendkimDeployer(config.mail_domain),
+        OpendkimDeployer(mail_domain),
         # Dovecot should be started before Postfix
         # because it creates authentication socket
         # required by Postfix.

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -5,14 +5,13 @@ from chatmaild.config import Config
 from pyinfra import host
 from pyinfra.facts.deb import DebPackages
 from pyinfra.facts.server import Arch, Command, Sysctl
-from pyinfra.operations import apt, files, server, systemd
+from pyinfra.operations import apt, files, server
 
 from cmdeploy.basedeploy import (
     Deployer,
     activate_remote_units,
     blocked_service_startup,
     configure_remote_units,
-    get_resource,
     is_in_container,
 )
 
@@ -59,26 +58,21 @@ class DovecotDeployer(Deployer):
                     ],
                 )
                 self.need_restart = True
-        files.put(
+        self.put_file(
-            name="Pin dovecot packages to block Debian dist-upgrades",
             src=io.StringIO(
                 "Package: dovecot-*\n"
                 "Pin: version *\n"
                 "Pin-Priority: -1\n"
             ),
             dest="/etc/apt/preferences.d/pin-dovecot",
-            user="root",
-            group="root",
-            mode="644",
         )
 
     def configure(self):
-        configure_remote_units(self.config.mail_domain_bare, self.units)
+        configure_remote_units(self, self.config.mail_domain, self.units)
-        config_restart, self.daemon_reload = _configure_dovecot(self.config)
+        _configure_dovecot(self, self.config)
-        self.need_restart |= config_restart
 
     def activate(self):
-        activate_remote_units(self.units)
+        activate_remote_units(self, self.units)
 
         # Detect stale binary: package installed but service still runs old (deleted) binary.
         if not self.disable_mail and not self.need_restart:
@@ -91,19 +85,12 @@ class DovecotDeployer(Deployer):
             if stale == "STALE":
                 self.need_restart = True
 
-        restart = False if self.disable_mail else self.need_restart
+        active = not self.disable_mail
-
+        self.ensure_service(
-        systemd.service(
+            "dovecot.service",
-            name="Disable dovecot for now"
+            running=active,
-            if self.disable_mail
+            enabled=active,
-            else "Start and enable Dovecot",
-            service="dovecot.service",
-            running=False if self.disable_mail else True,
-            enabled=False if self.disable_mail else True,
-            restarted=restart,
-            daemon_reload=self.daemon_reload,
         )
-        self.need_restart = False
 
 
 def _pick_url(primary, fallback):
@@ -147,39 +134,19 @@ def _download_dovecot_package(package: str, arch: str) -> tuple[str | None, bool
 
     return deb_filename, True
 
-
+def _configure_dovecot(deployer, config: Config, debug: bool = False):
-def _configure_dovecot(config: Config, debug: bool = False) -> tuple[bool, bool]:
     """Configures Dovecot IMAP server."""
-    need_restart = False
+    deployer.put_template(
-    daemon_reload = False
+        "dovecot/dovecot.conf.j2",
-
+        "/etc/dovecot/dovecot.conf",
-    main_config = files.template(
-        src=get_resource("dovecot/dovecot.conf.j2"),
-        dest="/etc/dovecot/dovecot.conf",
-        user="root",
-        group="root",
-        mode="644",
         config=config,
         debug=debug,
         disable_ipv6=config.disable_ipv6,
     )
-    need_restart |= main_config.changed
+    deployer.put_file("dovecot/auth.conf", "/etc/dovecot/auth.conf")
-    auth_config = files.put(
+    deployer.put_file(
-        src=get_resource("dovecot/auth.conf"),
+        "dovecot/push_notification.lua", "/etc/dovecot/push_notification.lua"
-        dest="/etc/dovecot/auth.conf",
-        user="root",
-        group="root",
-        mode="644",
     )
-    need_restart |= auth_config.changed
-    lua_push_notification_script = files.put(
-        src=get_resource("dovecot/push_notification.lua"),
-        dest="/etc/dovecot/push_notification.lua",
-        user="root",
-        group="root",
-        mode="644",
-    )
-    need_restart |= lua_push_notification_script.changed
 
     # as per https://doc.dovecot.org/2.3/configuration_manual/os/
     # it is recommended to set the following inotify limits
@@ -203,25 +170,20 @@ def _configure_dovecot(config: Config, debug: bool = False) -> tuple[bool, bool]
             persist=True,
         )
 
-    timezone_env = files.line(
+    deployer.ensure_line(
         name="Set TZ environment variable",
         path="/etc/environment",
         line="TZ=:/etc/localtime",
     )
-    need_restart |= timezone_env.changed
 
-    restart_conf = files.put(
+    deployer.put_file(
-        name="dovecot: restart automatically on failure",
+        "service/10_restart_on_failure.conf",
-        src=get_resource("service/10_restart.conf"),
+        "/etc/systemd/system/dovecot.service.d/10_restart.conf",
-        dest="/etc/systemd/system/dovecot.service.d/10_restart.conf",
     )
-    daemon_reload |= restart_conf.changed
 
     # Validate dovecot configuration before restart
-    if need_restart:
+    if deployer.need_restart:
         server.shell(
             name="Validate dovecot configuration",
             commands=["doveconf -n >/dev/null"],
         )
-
-    return need_restart, daemon_reload

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -7,7 +7,6 @@ listen = 0.0.0.0
 protocols = imap lmtp
 
 auth_mechanisms = plain
-auth_username_chars = abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890.-_@[]
 
 {% if debug == true %}
 auth_verbose = yes

cmdeploy/src/cmdeploy/external/deployer.py

@@ -1,10 +1,8 @@
-import io
 
 from pyinfra import host
 from pyinfra.facts.files import File
-from pyinfra.operations import files, systemd
 
-from cmdeploy.basedeploy import Deployer, get_resource
+from ..basedeploy import Deployer
 
 
 class ExternalTlsDeployer(Deployer):
@@ -23,45 +21,24 @@ class ExternalTlsDeployer(Deployer):
     def configure(self):
         # Verify cert and key exist on the remote host using pyinfra facts.
         for path in (self.cert_path, self.key_path):
-            info = host.get_fact(File, path=path)
+                if host.get_fact(File, path=path) is None:
-            if info is None:
+                    raise Exception(f"External TLS file not found on server: {path}")
-                raise Exception(f"External TLS file not found on server: {path}")
 
-        # Deploy the .path unit (templated with the cert path).
+        self.ensure_systemd_unit(
-        # pkg=__package__ is required here because the resource files
+            "external/tls-cert-reload.path.j2",
-        # live in cmdeploy.external, not the default cmdeploy package.
+            cert_path=self.cert_path,
-        source = get_resource("tls-cert-reload.path.f", pkg=__package__)
-        content = source.read_text().format(cert_path=self.cert_path).encode()
-
-        path_unit = files.put(
-            name="Upload tls-cert-reload.path",
-            src=io.BytesIO(content),
-            dest="/etc/systemd/system/tls-cert-reload.path",
-            user="root",
-            group="root",
-            mode="644",
         )
-
+        self.ensure_systemd_unit(
-        service_unit = files.put(
+            "external/tls-cert-reload.service",
-            name="Upload tls-cert-reload.service",
-            src=get_resource("tls-cert-reload.service", pkg=__package__),
-            dest="/etc/systemd/system/tls-cert-reload.service",
-            user="root",
-            group="root",
-            mode="644",
         )
 
-        if path_unit.changed or service_unit.changed:
-            self.need_restart = True
-
     def activate(self):
-        systemd.service(
-            name="Enable tls-cert-reload path watcher",
-            service="tls-cert-reload.path",
-            running=True,
-            enabled=True,
-            restarted=self.need_restart,
-            daemon_reload=self.need_restart,
-        )
         # No explicit reload needed here: dovecot/nginx read the cert
         # on startup, and the .path watcher handles live changes.
+        self.ensure_service(
+            "tls-cert-reload.path",
+            running=True,
+            enabled=True,
+        )
+
+

cmdeploy/src/cmdeploy/external/tls-cert-reload.path.j2

@@ -9,7 +9,7 @@
 Description=Watch TLS certificate for changes
 
 [Path]
-PathChanged={cert_path}
+PathChanged={{ cert_path }}
 
 [Install]
 WantedBy=multi-user.target

cmdeploy/src/cmdeploy/filtermail/deployer.py

@@ -1,9 +1,8 @@
 import os
 
 from pyinfra import facts, host
-from pyinfra.operations import files, systemd
 
-from cmdeploy.basedeploy import Deployer, get_resource
+from cmdeploy.basedeploy import Deployer
 
 
 class FiltermailDeployer(Deployer):
@@ -11,18 +10,13 @@ class FiltermailDeployer(Deployer):
     bin_path = "/usr/local/bin/filtermail"
     config_path = "/usr/local/lib/chatmaild/chatmail.ini"
 
-    def __init__(self):
-        self.need_restart = False
-
     def install(self):
         local_bin = os.environ.get("CHATMAIL_FILTERMAIL_BINARY")
         if local_bin:
-            self.need_restart |= files.put(
+            self.put_executable(
-                name="Upload locally built filtermail",
                 src=local_bin,
                 dest=self.bin_path,
-                mode="755",
+            )
-            ).changed
             return
 
         arch = host.get_fact(facts.server.Arch)
@@ -31,34 +25,16 @@ class FiltermailDeployer(Deployer):
             "x86_64": "5295115952c72e4c4ec3c85546e094b4155a4c702c82bd71fcdcb744dc73adf6",
             "aarch64": "6892244f17b8f26ccb465766e96028e7222b3c8adefca9fc6bfe9ff332ca8dff",
         }[arch]
-        self.need_restart |= files.download(
+        self.download_executable(url, self.bin_path, sha256sum)
-            name="Download filtermail",
-            src=url,
-            sha256sum=sha256sum,
-            dest=self.bin_path,
-            mode="755",
-        ).changed
 
     def configure(self):
         for service in self.services:
-            self.need_restart |= files.template(
+            self.ensure_systemd_unit(
-                src=get_resource(f"filtermail/{service}.service.j2"),
+                f"filtermail/{service}.service.j2",
-                dest=f"/etc/systemd/system/{service}.service",
-                user="root",
-                group="root",
-                mode="644",
                 bin_path=self.bin_path,
                 config_path=self.config_path,
-            ).changed
+            )
 
     def activate(self):
         for service in self.services:
-            systemd.service(
+            self.ensure_service(f"{service}.service")
-                name=f"Start and enable {service}",
-                service=f"{service}.service",
-                running=True,
-                enabled=True,
-                restarted=self.need_restart,
-                daemon_reload=True,
-            )
-        self.need_restart = False

cmdeploy/src/cmdeploy/mtail/deployer.py

@@ -1,10 +1,7 @@
 from pyinfra import facts, host
-from pyinfra.operations import apt, files, server, systemd
+from pyinfra.operations import apt
 
-from cmdeploy.basedeploy import (
+from cmdeploy.basedeploy import Deployer
-    Deployer,
-    get_resource,
-)
 
 
 class MtailDeployer(Deployer):
@@ -18,51 +15,30 @@ class MtailDeployer(Deployer):
         (url, sha256sum) = {
             "x86_64": (
                 "https://github.com/google/mtail/releases/download/v3.0.8/mtail_3.0.8_linux_amd64.tar.gz",
-                "123c2ee5f48c3eff12ebccee38befd2233d715da736000ccde49e3d5607724e4",
+                "d55cb601049c5e61eabab29998dbbcea95d480e5448544f9470337ba2eea882e",
             ),
             "aarch64": (
                 "https://github.com/google/mtail/releases/download/v3.0.8/mtail_3.0.8_linux_arm64.tar.gz",
-                "aa04811c0929b6754408676de520e050c45dddeb3401881888a092c9aea89cae",
+                "f748db8ad2a1e0b63684d4c8868cf6a373a20f7e6922e5ece601fff0ee00eb1a",
             ),
         }[host.get_fact(facts.server.Arch)]
-
+        self.download_executable(
-        server.shell(
+            url,
-            name="Download mtail",
+            "/usr/local/bin/mtail",
-            commands=[
+            sha256sum,
-                f"(echo '{sha256sum} /usr/local/bin/mtail' | sha256sum -c) || (curl -L {url} | gunzip | tar -x -f - mtail -O >/usr/local/bin/mtail.new && mv /usr/local/bin/mtail.new /usr/local/bin/mtail)",
+            extract="gunzip | tar -xf - mtail -O",
-                "chmod 755 /usr/local/bin/mtail",
-            ],
         )
 
     def configure(self):
         # Using our own systemd unit instead of `/usr/lib/systemd/system/mtail.service`.
         # This allows to read from journalctl instead of log files.
-        files.template(
+        self.ensure_systemd_unit(
-            src=get_resource("mtail/mtail.service.j2"),
+            "mtail/mtail.service.j2",
-            dest="/etc/systemd/system/mtail.service",
-            user="root",
-            group="root",
-            mode="644",
             address=self.mtail_address or "127.0.0.1",
             port=3903,
         )
-
+        self.put_file("mtail/delivered_mail.mtail", "/etc/mtail/delivered_mail.mtail")
-        mtail_conf = files.put(
-            name="Mtail configuration",
-            src=get_resource("mtail/delivered_mail.mtail"),
-            dest="/etc/mtail/delivered_mail.mtail",
-            user="root",
-            group="root",
-            mode="644",
-        )
-        self.need_restart = mtail_conf.changed
 
     def activate(self):
-        systemd.service(
+        active = bool(self.mtail_address)
-            name="Start and enable mtail",
+        self.ensure_service("mtail.service", running=active, enabled=active)
-            service="mtail.service",
-            running=bool(self.mtail_address),
-            enabled=bool(self.mtail_address),
-            restarted=self.need_restart,
-        )
-        self.need_restart = False

cmdeploy/src/cmdeploy/mtail/mtail.service.j2

@@ -1,11 +1,13 @@
 [Unit]
 Description=mtail
-After=multi-user.target
+After=network-online.target
+Wants=network-online.target
 
 [Service]
 Type=simple
 ExecStart=/bin/sh -c "journalctl -f -o short-iso -n 0 | /usr/local/bin/mtail --address={{ address }} --port={{ port }} --progs /etc/mtail --logtostderr --logs -"
 Restart=on-failure
+RestartSec=2s
 
 [Install]
 WantedBy=multi-user.target

cmdeploy/src/cmdeploy/nginx/deployer.py

@@ -1,5 +1,5 @@
 from chatmaild.config import Config
-from pyinfra.operations import apt, files, systemd
+from pyinfra.operations import apt
 
 from cmdeploy.basedeploy import (
     Deployer,
@@ -31,87 +31,50 @@ class NginxDeployer(Deployer):
         # For documentation about policy-rc.d, see:
         # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
         #
-        files.put(
+        self.put_executable(src="policy-rc.d", dest="/usr/sbin/policy-rc.d")
-            src=get_resource("policy-rc.d"),
-            dest="/usr/sbin/policy-rc.d",
-            user="root",
-            group="root",
-            mode="755",
-        )
 
         apt.packages(
             name="Install nginx",
             packages=["nginx", "libnginx-mod-stream"],
         )
 
-        files.file("/usr/sbin/policy-rc.d", present=False)
+        self.remove_file("/usr/sbin/policy-rc.d")
 
     def configure(self):
-        self.need_restart = _configure_nginx(self.config)
+        _configure_nginx(self, self.config)
 
     def activate(self):
-        systemd.service(
+        self.ensure_service("nginx.service")
-            name="Start and enable nginx",
-            service="nginx.service",
-            running=True,
-            enabled=True,
-            restarted=self.need_restart,
-        )
-        self.need_restart = False
 
 
-def _configure_nginx(config: Config, debug: bool = False) -> bool:
+def _configure_nginx(deployer, config: Config, debug: bool = False):
     """Configures nginx HTTP server."""
-    need_restart = False
 
-    main_config = files.template(
+    deployer.put_template(
-        src=get_resource("nginx/nginx.conf.j2"),
+        "nginx/nginx.conf.j2",
-        dest="/etc/nginx/nginx.conf",
+        "/etc/nginx/nginx.conf",
-        user="root",
-        group="root",
-        mode="644",
         config=config,
         disable_ipv6=config.disable_ipv6,
     )
-    need_restart |= main_config.changed
 
-    autoconfig = files.template(
+    deployer.put_template(
-        src=get_resource("nginx/autoconfig.xml.j2"),
+        "nginx/autoconfig.xml.j2",
-        dest="/var/www/html/.well-known/autoconfig/mail/config-v1.1.xml",
+        "/var/www/html/.well-known/autoconfig/mail/config-v1.1.xml",
-        user="root",
-        group="root",
-        mode="644",
         config=config,
     )
-    need_restart |= autoconfig.changed
 
-    mta_sts_config = files.template(
+    deployer.put_template(
-        src=get_resource("nginx/mta-sts.txt.j2"),
+        "nginx/mta-sts.txt.j2",
-        dest="/var/www/html/.well-known/mta-sts.txt",
+        "/var/www/html/.well-known/mta-sts.txt",
-        user="root",
-        group="root",
-        mode="644",
         config=config,
     )
-    need_restart |= mta_sts_config.changed
 
     # install CGI newemail script
     #
     cgi_dir = "/usr/lib/cgi-bin"
-    files.directory(
+    deployer.ensure_directory(cgi_dir)
-        name=f"Ensure {cgi_dir} exists",
-        path=cgi_dir,
-        user="root",
-        group="root",
-    )
 
-    files.put(
+    deployer.put_executable(
-        name="Upload cgi newemail.py script",
         src=get_resource("newemail.py", pkg="chatmaild").open("rb"),
         dest=f"{cgi_dir}/newemail.py",
-        user="root",
-        group="root",
-        mode="755",
     )
-
-    return need_restart

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -42,6 +42,9 @@ stream {
 }
 
 http {
+	# access_log setting is inherited by all server sections
+	access_log syslog:server=unix:/dev/log,facility=local7;
+
 {% if config.tls_cert_mode == "self" %}
 	limit_req_zone $binary_remote_addr zone=newaccount:10m rate=2r/s;
 {% endif %}
@@ -69,9 +72,7 @@ http {
 
 		index index.html index.htm;
 
-		server_name {{ config.mail_domain }} www.{{ config.mail_domain }} mta-sts.{{ config.mail_domain }};
+		server_name {{ config.mail_domain }} mta-sts.{{ config.mail_domain }};
-
-		access_log syslog:server=unix:/dev/log,facility=local7;
 
 		location /mxdeliv {
 		    proxy_pass http://127.0.0.1:{{ config.filtermail_http_port_incoming }};
@@ -143,7 +144,6 @@ http {
 		listen 127.0.0.1:8443 ssl;
 		server_name www.{{ config.mail_domain }};
 		return 301 $scheme://{{ config.mail_domain }}$request_uri;
-		access_log syslog:server=unix:/dev/log,facility=local7;
 	}
 
 	server {

cmdeploy/src/cmdeploy/opendkim/deployer.py

@@ -4,9 +4,9 @@ Installs OpenDKIM
 
 from pyinfra import host
 from pyinfra.facts.files import File
-from pyinfra.operations import apt, files, server, systemd
+from pyinfra.operations import apt, files, server
 
-from cmdeploy.basedeploy import Deployer, get_resource
+from cmdeploy.basedeploy import Deployer
 
 
 class OpendkimDeployer(Deployer):
@@ -25,65 +25,39 @@ class OpendkimDeployer(Deployer):
         domain = self.mail_domain
         dkim_selector = "opendkim"
         """Configures OpenDKIM"""
-        need_restart = False
 
-        main_config = files.template(
+        self.put_template(
-            src=get_resource("opendkim/opendkim.conf"),
+            "opendkim/opendkim.conf",
-            dest="/etc/opendkim.conf",
+            "/etc/opendkim.conf",
-            user="root",
-            group="root",
-            mode="644",
             config={"domain_name": domain, "opendkim_selector": dkim_selector},
         )
-        need_restart |= main_config.changed
 
-        screen_script = files.file(
+        self.remove_file("/etc/opendkim/screen.lua")
-            path="/etc/opendkim/screen.lua",
+        self.remove_file("/etc/opendkim/final.lua")
-            present=False,
-        )
-        need_restart |= screen_script.changed
 
-        final_script = files.file(
+        self.ensure_directory(
-            path="/etc/opendkim/final.lua",
+            "/etc/opendkim",
-            present=False,
+            owner="opendkim",
-        )
-        need_restart |= final_script.changed
-
-        files.directory(
-            name="Add opendkim directory to /etc",
-            path="/etc/opendkim",
-            user="opendkim",
-            group="opendkim",
             mode="750",
-            present=True,
         )
 
-        keytable = files.template(
+        self.put_template(
-            src=get_resource("opendkim/KeyTable"),
+            "opendkim/KeyTable",
-            dest="/etc/dkimkeys/KeyTable",
+            "/etc/dkimkeys/KeyTable",
-            user="opendkim",
+            owner="opendkim",
-            group="opendkim",
-            mode="644",
             config={"domain_name": domain, "opendkim_selector": dkim_selector},
         )
-        need_restart |= keytable.changed
 
-        signing_table = files.template(
+        self.put_template(
-            src=get_resource("opendkim/SigningTable"),
+            "opendkim/SigningTable",
-            dest="/etc/dkimkeys/SigningTable",
+            "/etc/dkimkeys/SigningTable",
-            user="opendkim",
+            owner="opendkim",
-            group="opendkim",
-            mode="644",
             config={"domain_name": domain, "opendkim_selector": dkim_selector},
         )
-        need_restart |= signing_table.changed
+        self.ensure_directory(
-        files.directory(
+            "/var/spool/postfix/opendkim",
-            name="Add opendkim socket directory to /var/spool/postfix",
+            owner="opendkim",
-            path="/var/spool/postfix/opendkim",
-            user="opendkim",
-            group="opendkim",
             mode="750",
-            present=True,
         )
 
         if not host.get_fact(File, f"/etc/dkimkeys/{dkim_selector}.private"):
@@ -96,12 +70,10 @@ class OpendkimDeployer(Deployer):
                 _su_user="opendkim",
             )
 
-        service_file = files.put(
+        self.put_file(
-            name="Configure opendkim to restart once a day",
+            "opendkim/systemd.conf",
-            src=get_resource("opendkim/systemd.conf"),
+            "/etc/systemd/system/opendkim.service.d/10-prevent-memory-leak.conf",
-            dest="/etc/systemd/system/opendkim.service.d/10-prevent-memory-leak.conf",
         )
-        need_restart |= service_file.changed
 
         files.file(
             name="chown opendkim: /etc/dkimkeys/opendkim.private",
@@ -110,15 +82,5 @@ class OpendkimDeployer(Deployer):
             group="opendkim",
         )
 
-        self.need_restart = need_restart
-
     def activate(self):
-        systemd.service(
+        self.ensure_service("opendkim.service")
-            name="Start and enable OpenDKIM",
-            service="opendkim.service",
-            running=True,
-            enabled=True,
-            daemon_reload=self.need_restart,
-            restarted=self.need_restart,
-        )
-        self.need_restart = False

cmdeploy/src/cmdeploy/postfix/deployer.py

@@ -1,11 +1,10 @@
-from pyinfra.operations import apt, files, server, systemd
+from pyinfra.operations import apt, server
 
-from cmdeploy.basedeploy import Deployer, get_resource
+from cmdeploy.basedeploy import Deployer
 
 
 class PostfixDeployer(Deployer):
     required_users = [("postfix", None, ["opendkim"])]
-    daemon_reload = False
 
     def __init__(self, config, disable_mail):
         self.config = config
@@ -19,81 +18,46 @@ class PostfixDeployer(Deployer):
 
     def configure(self):
         config = self.config
-        need_restart = False
 
-        main_config = files.template(
+        self.put_template(
-            src=get_resource("postfix/main.cf.j2"),
+            "postfix/main.cf.j2",
-            dest="/etc/postfix/main.cf",
+            "/etc/postfix/main.cf",
-            user="root",
-            group="root",
-            mode="644",
             config=config,
             disable_ipv6=config.disable_ipv6,
         )
-        need_restart |= main_config.changed
 
-        master_config = files.template(
+        self.put_template(
-            src=get_resource("postfix/master.cf.j2"),
+            "postfix/master.cf.j2",
-            dest="/etc/postfix/master.cf",
+            "/etc/postfix/master.cf",
-            user="root",
-            group="root",
-            mode="644",
             debug=False,
             config=config,
         )
-        need_restart |= master_config.changed
 
-        header_cleanup = files.put(
+        self.put_file(
-            src=get_resource("postfix/submission_header_cleanup"),
+            "postfix/submission_header_cleanup",
-            dest="/etc/postfix/submission_header_cleanup",
+            "/etc/postfix/submission_header_cleanup",
-            user="root",
-            group="root",
-            mode="644",
         )
-        need_restart |= header_cleanup.changed
+        self.put_file("postfix/lmtp_header_cleanup", "/etc/postfix/lmtp_header_cleanup")
 
-        lmtp_header_cleanup = files.put(
+        res = self.put_file(
-            src=get_resource("postfix/lmtp_header_cleanup"),
+            "postfix/smtp_tls_policy_map", "/etc/postfix/smtp_tls_policy_map"
-            dest="/etc/postfix/lmtp_header_cleanup",
-            user="root",
-            group="root",
-            mode="644",
         )
-        need_restart |= lmtp_header_cleanup.changed
+        tls_policy_changed = res.changed
-
+        if tls_policy_changed:
-        tls_policy_map = files.put(
-            name="Upload SMTP TLS Policy that accepts self-signed certificates for IP-only hosts",
-            src=get_resource("postfix/smtp_tls_policy_map"),
-            dest="/etc/postfix/smtp_tls_policy_map",
-            user="root",
-            group="root",
-            mode="644",
-        )
-        need_restart |= tls_policy_map.changed
-        if tls_policy_map.changed:
             server.shell(
                 commands=["postmap /etc/postfix/smtp_tls_policy_map"],
             )
 
         # Login map that 1:1 maps email address to login.
-        login_map = files.put(
+        self.put_file("postfix/login_map", "/etc/postfix/login_map")
-            src=get_resource("postfix/login_map"),
-            dest="/etc/postfix/login_map",
-            user="root",
-            group="root",
-            mode="644",
-        )
-        need_restart |= login_map.changed
 
-        restart_conf = files.put(
+        self.put_file(
-            name="postfix: restart automatically on failure",
+            "service/10_restart_on_failure.conf",
-            src=get_resource("service/10_restart.conf"),
+            "/etc/systemd/system/postfix@.service.d/10_restart.conf",
-            dest="/etc/systemd/system/postfix@.service.d/10_restart.conf",
         )
-        self.daemon_reload = restart_conf.changed
 
         # Validate postfix configuration before restart
-        if need_restart:
+        if self.need_restart:
             server.shell(
                 name="Validate postfix configuration",
                 # Extract stderr and quit with error if non-zero
@@ -101,19 +65,11 @@ class PostfixDeployer(Deployer):
                     """bash -c 'w=$(postconf 2>&1 >/dev/null); [[ -z "$w" ]] || { echo "$w"; false; }'"""
                 ],
             )
-        self.need_restart = need_restart
 
     def activate(self):
-        restart = False if self.disable_mail else self.need_restart
+        active = not self.disable_mail
-
+        self.ensure_service(
-        systemd.service(
+            "postfix.service",
-            name="disable postfix for now"
+            running=active,
-            if self.disable_mail
+            enabled=active,
-            else "Start and enable Postfix",
-            service="postfix.service",
-            running=False if self.disable_mail else True,
-            enabled=False if self.disable_mail else True,
-            restarted=restart,
-            daemon_reload=self.daemon_reload,
         )
-        self.need_restart = False

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -54,16 +54,14 @@ smtpd_tls_exclude_ciphers = aNULL, RC4, MD5, DES
 tls_preempt_cipherlist = yes
 
 smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination
-myhostname = {{ config.postfix_myhostname }}
+myhostname = {{ config.mail_domain }}
 alias_maps = hash:/etc/aliases
 alias_database = hash:/etc/aliases
 
-# When postfix receives mail for $mydestination,
+# Postfix does not deliver mail for any domain by itself.
-# it hands it over to dovecot via $local_transport.
+# Primary domain is listed in `virtual_mailbox_domains` instead
-mydestination = {{ config.mail_domain }}
+# and handed over to Dovecot.
-local_transport = lmtp:unix:private/dovecot-lmtp
+mydestination =
-# postfix doesn't check whether local users exist or not:
-local_recipient_maps =
 
 relayhost = 
 {% if disable_ipv6 %}
@@ -81,6 +79,8 @@ inet_protocols = ipv4
 inet_protocols = all
 {% endif %}
 
+virtual_transport = lmtp:unix:private/dovecot-lmtp
+virtual_mailbox_domains = {{ config.mail_domain }}
 lmtp_header_checks = regexp:/etc/postfix/lmtp_header_cleanup
 
 mua_client_restrictions = permit_sasl_authenticated, reject

cmdeploy/src/cmdeploy/postfix/master.cf.j2

@@ -80,9 +80,8 @@ filter    unix -        n       n       -       -       lmtp
 127.0.0.1:{{ config.postfix_reinject_port }} inet  n       -       n       -       100      smtpd
   -o syslog_name=postfix/reinject
   -o milter_macro_daemon_name=ORIGINATING
+  -o smtpd_milters=unix:opendkim/opendkim.sock
   -o cleanup_service_name=authclean
-{% if not config.ipv4_relay %}  -o smtpd_milters=unix:opendkim/opendkim.sock
-{% endif %}
 
 # Local SMTP server for reinjecting incoming filtered mail 
 127.0.0.1:{{ config.postfix_reinject_port_incoming }} inet  n       -       n       -       100      smtpd

cmdeploy/src/cmdeploy/remote/rdns.py

@@ -64,21 +64,25 @@ def get_dkim_entry(mail_domain, pre_command, dkim_selector):
     )
 
 
-def query_dns(typ, domain):
+def get_authoritative_ns(domain):
-    # Get autoritative nameserver from the SOA record.
+    ns_replies = [
-    soa_answers = [
         x.split()
         for x in shell(
-            f"dig -r -q {domain} -t SOA +noall +authority +answer", print=log_progress
+            f"dig -r -q {domain} -t NS +noall +authority +answer", print=log_progress
         ).split("\n")
     ]
-    soa = [a for a in soa_answers if len(a) >= 3 and a[3] == "SOA"]
+    filtered_replies = [a for a in ns_replies if len(a) >= 5 and a[3] == "NS"]
-    if not soa:
+    if not filtered_replies:
         return
-    ns = soa[0][4]
+    return filtered_replies[0][4]
+
+
+def query_dns(typ, domain):
+    ns = get_authoritative_ns(domain)
 
     # Query authoritative nameserver directly to bypass DNS cache.
-    res = shell(f"dig @{ns} -r -q {domain} -t {typ} +short", print=log_progress)
+    direct_ns = f"@{ns}" if ns else ""
+    res = shell(f"dig {direct_ns} -r -q {domain} -t {typ} +short", print=log_progress)
     return next((line for line in res.split("\n") if not line.startswith(";")), "")
 
 

cmdeploy/src/cmdeploy/selfsigned/deployer.py

@@ -1,8 +1,8 @@
 import shlex
 
-from pyinfra.operations import apt, server
+from pyinfra.operations import server
 
-from cmdeploy.basedeploy import Deployer
+from ..basedeploy import Deployer
 
 
 def openssl_selfsigned_args(domain, cert_path, key_path, days=36500):
@@ -34,11 +34,7 @@ class SelfSignedTlsDeployer(Deployer):
         self.cert_path = "/etc/ssl/certs/mailserver.pem"
         self.key_path = "/etc/ssl/private/mailserver.key"
 
-    def install(self):
+
-        apt.packages(
-            name="Install openssl",
-            packages=["openssl"],
-        )
 
     def configure(self):
         args = openssl_selfsigned_args(
@@ -52,3 +48,5 @@ class SelfSignedTlsDeployer(Deployer):
 
     def activate(self):
         pass
+
+

cmdeploy/src/cmdeploy/tests/online/test_0_login.py

@@ -89,11 +89,12 @@ def test_concurrent_logins_same_account(
         assert login_results.get()
 
 
-def test_no_vrfy(cmfactory, chatmail_config, maildomain):
+def test_no_vrfy(cmfactory, chatmail_config):
     ac = cmfactory.get_online_account()
     addr = ac.get_config("addr")
+    domain = chatmail_config.mail_domain
 
-    s = smtplib.SMTP(maildomain)
+    s = smtplib.SMTP(domain)
     s.starttls()
 
     s.putcmd("vrfy", f"wrongaddress@{chatmail_config.mail_domain}")

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -8,7 +8,6 @@ import pytest
 
 from cmdeploy import remote
 from cmdeploy.cmdeploy import get_sshexec
-from chatmaild.config import is_valid_ipv4
 
 
 class TestSSHExecutor:
@@ -22,8 +21,6 @@ class TestSSHExecutor:
         assert out == out2
 
     def test_perform_initial(self, sshexec, maildomain):
-        if is_valid_ipv4(maildomain):
-            pytest.skip(f"{maildomain} is not a domain")
         res = sshexec(
             remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain)
         )
@@ -284,3 +281,13 @@ def test_deployed_state(remote):
     # assert len(git_status) == len(remote_version)  # for some reason, we only get 11 lines from remote.iter_output()
     for i in range(len(remote_version)):
         assert git_status[i] == remote_version[i], "You have undeployed changes."
+
+
+def test_nginx_access_log_only_defined_once(sshdomain):
+    sshexec = get_sshexec(sshdomain)
+    conf = sshexec(
+        call=remote.rshell.shell,
+        kwargs=dict(command="nginx -T 2>/dev/null"),
+    )
+    access_logs = [l for l in conf.splitlines() if l.strip().startswith("access_log")]
+    assert len(access_logs) == 1, f"expected 1 access_log, found {len(access_logs)}: {access_logs}"

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -15,7 +15,7 @@ def imap_mailbox(cmfactory, ssl_context):
     (ac1,) = cmfactory.get_online_accounts(1)
     user = ac1.get_config("addr")
     password = ac1.get_config("mail_pw")
-    host = user.split("@")[1].strip("[").strip("]")
+    host = user.split("@")[1]
     mailbox = imap_tools.MailBox(host, ssl_context=ssl_context)
     mailbox.login(user, password)
     mailbox.dc_ac = ac1
@@ -178,7 +178,7 @@ def test_hide_senders_ip_address(cmfactory, ssl_context):
     chat.send_text("testing submission header cleanup")
     user2.wait_for_incoming_msg()
     addr = user2.get_config("addr")
-    host = addr.split("@")[1].strip("[").strip("]")
+    host = addr.split("@")[1]
     pw = user2.get_config("mail_pw")
     mailbox = imap_tools.MailBox(host, ssl_context=ssl_context)
     mailbox.login(addr, pw)

cmdeploy/src/cmdeploy/tests/plugin.py

@@ -10,8 +10,7 @@ import time
 from pathlib import Path
 
 import pytest
-from chatmaild.config import read_config, format_mail_domain, is_valid_ipv4
+from chatmaild.config import read_config
-
 
 conftestdir = Path(__file__).parent
 
@@ -59,12 +58,7 @@ def chatmail_config(pytestconfig):
 
 @pytest.fixture(scope="session")
 def maildomain(chatmail_config):
-    return chatmail_config.mail_domain_bare
+    return chatmail_config.mail_domain
-
-
-@pytest.fixture(scope="session")
-def maildomain_deliverable(maildomain):
-    return format_mail_domain(maildomain)
 
 
 @pytest.fixture(scope="session")
@@ -284,6 +278,7 @@ def gencreds(chatmail_config):
 
     def gen(domain=None):
         domain = domain if domain else chatmail_config.mail_domain
+        addr_domain = f"[{domain}]" if _is_ip(domain) else domain
         while 1:
             num = next(count)
             alphanumeric = "abcdefghijklmnopqrstuvwxyz1234567890"
@@ -297,7 +292,7 @@ def gencreds(chatmail_config):
             password = "".join(
                 random.choices(alphanumeric, k=chatmail_config.password_min_length)
             )
-            yield f"{user}@{domain}", f"{password}"
+            yield f"{user}@{addr_domain}", f"{password}"
 
     return lambda domain=None: next(gen(domain))
 
@@ -322,8 +317,7 @@ class ChatmailACFactory:
 
     def _make_transport(self, domain):
         """Build a transport config dict for the given domain."""
-        domain_deliverable = format_mail_domain(domain)
+        addr, password = self.gencreds(domain)
-        addr, password = self.gencreds(domain_deliverable)
         transport = {
             "addr": addr,
             "password": password,
@@ -332,7 +326,7 @@ class ChatmailACFactory:
             "imapServer": domain,
             "smtpServer": domain,
         }
-        if domain.startswith("_") or is_valid_ipv4(domain):
+        if self.chatmail_config.tls_cert_mode == "self":
             transport["certificateChecks"] = "acceptInvalidCertificates"
         return transport
 
@@ -347,8 +341,7 @@ class ChatmailACFactory:
         accounts = []
         for _ in range(num):
             account = self.dc.add_account()
-            domain_deliverable = format_mail_domain(domain)
+            addr, password = self.gencreds(domain)
-            addr, password = self.gencreds(domain_deliverable)
             if _is_ip(domain):
                 # Use DCLOGIN scheme with explicit server hosts,
                 # matching how madmail presents its addresses to users.

cmdeploy/src/cmdeploy/tests/test_basedeploy.py

@@ -0,0 +1,118 @@
+from unittest.mock import MagicMock, patch
+
+from cmdeploy.basedeploy import Deployer
+
+
+def test_put_file_restart_and_reload():
+    deployer = Deployer()
+    mock_res = MagicMock()
+    mock_res.changed = True
+
+    with patch("cmdeploy.basedeploy.files.put", return_value=mock_res):
+        deployer.put_file("foo.conf", "/etc/foo.conf")
+        assert deployer.need_restart is True
+        assert deployer.daemon_reload is False
+
+        deployer = Deployer()
+
+        deployer.put_file("test.service", "/etc/systemd/system/test.service")
+        assert deployer.need_restart is True
+        assert deployer.daemon_reload is True
+
+
+def test_remove_file():
+    deployer = Deployer()
+    mock_res = MagicMock()
+    mock_res.changed = True
+
+    with patch("cmdeploy.basedeploy.files.file", return_value=mock_res) as mock_file:
+        deployer.remove_file("/etc/foo.conf")
+        mock_file.assert_called_once_with(
+            name="Remove /etc/foo.conf", path="/etc/foo.conf", present=False
+        )
+        assert deployer.need_restart is True
+
+
+def test_ensure_systemd_unit():
+    deployer = Deployer()
+    mock_res = MagicMock()
+    mock_res.changed = True
+
+    # Plain service file
+    with patch("cmdeploy.basedeploy.files.put", return_value=mock_res) as mock_put:
+        deployer.ensure_systemd_unit("iroh-relay.service")
+        assert (
+            mock_put.call_args.kwargs["dest"]
+            == "/etc/systemd/system/iroh-relay.service"
+        )
+        assert deployer.need_restart is True
+        assert deployer.daemon_reload is True
+
+    deployer = Deployer()
+
+    # Template (.j2) dispatches to put_template and strips .j2 suffix
+    with patch("cmdeploy.basedeploy.files.template", return_value=mock_res) as mock_tpl:
+        deployer.ensure_systemd_unit(
+            "filtermail/chatmaild.service.j2",
+            bin_path="/usr/local/bin/filtermail",
+        )
+        assert (
+            mock_tpl.call_args.kwargs["dest"] == "/etc/systemd/system/chatmaild.service"
+        )
+
+    deployer = Deployer()
+
+    # Explicit dest_name override
+    with patch("cmdeploy.basedeploy.files.put", return_value=mock_res) as mock_put:
+        deployer.ensure_systemd_unit(
+            "acmetool/acmetool-reconcile.timer",
+            dest_name="acmetool-reconcile.timer",
+        )
+        assert (
+            mock_put.call_args.kwargs["dest"]
+            == "/etc/systemd/system/acmetool-reconcile.timer"
+        )
+
+
+def test_ensure_service():
+    with patch("cmdeploy.basedeploy.systemd.service") as mock_svc:
+        deployer = Deployer()
+        deployer.need_restart = True
+        deployer.daemon_reload = True
+        deployer.ensure_service("nginx.service")
+        mock_svc.assert_called_once_with(
+            name="Start and enable nginx.service",
+            service="nginx.service",
+            running=True,
+            enabled=True,
+            restarted=True,
+            daemon_reload=True,
+        )
+        # daemon_reload is cleared to avoid multiple systemctl daemon-reload calls
+        # need_restart is kept to ensure all subsequent services also restart
+        assert deployer.need_restart is True
+        assert deployer.daemon_reload is False
+
+    with patch("cmdeploy.basedeploy.systemd.service") as mock_svc:
+        # Stopping suppresses restarted even when need_restart is True
+        deployer = Deployer()
+        deployer.need_restart = True
+        deployer.daemon_reload = True
+        deployer.ensure_service(
+            "mta-sts-daemon.service",
+            running=False,
+            enabled=False,
+        )
+        assert mock_svc.call_args.kwargs["restarted"] is False
+        assert deployer.need_restart is True
+
+    with patch("cmdeploy.basedeploy.systemd.service") as mock_svc:
+        # Multiple calls: daemon_reload resets after first, need_restart persists
+        deployer = Deployer()
+        deployer.need_restart = True
+        deployer.daemon_reload = True
+        deployer.ensure_service("chatmaild.service")
+        deployer.ensure_service("chatmaild-metadata.service")
+        second_call = mock_svc.call_args_list[1]
+        assert second_call.kwargs["restarted"] is True
+        assert second_call.kwargs["daemon_reload"] is False

cmdeploy/src/cmdeploy/tests/test_cmdeploy.py

@@ -39,14 +39,6 @@ class TestCmdline:
         out, err = capsys.readouterr()
         assert "deleting config file" in out.lower()
 
-    def test_dns_skip_on_ip(self, capsys, tmp_path, monkeypatch):
-        monkeypatch.delenv("CHATMAIL_INI", raising=False)
-        inipath = tmp_path / "chatmail.ini"
-        assert main(["init", "--config", str(inipath), "1.3.3.7"]) == 0
-        assert main(["dns", "--config", str(inipath)]) == 0
-        out, err = capsys.readouterr()
-        assert out == "[WARNING] 1.3.3.7 is not a domain, skipping DNS checks.\n"
-
 
 def test_www_folder(example_config, tmp_path):
     reporoot = importlib.resources.files(__package__).joinpath("../../../../").resolve()

cmdeploy/src/cmdeploy/tests/test_dns.py

@@ -4,6 +4,7 @@ import pytest
 
 from cmdeploy import remote
 from cmdeploy.dns import check_full_zone, check_initial_remote_data, parse_zone_records
+from cmdeploy.remote.rdns import get_authoritative_ns
 
 
 @pytest.fixture
@@ -14,11 +15,15 @@ def mockdns_base(monkeypatch):
         if command.startswith("dig"):
             if command == "dig":
                 return "."
-            if "SOA" in command:
+            if "with.public.soa" in command and "NS" in command:
+                return "domain.with.public.soa. 2419 IN NS ns1.first-ns.de."
+            if "with.hidden.soa" in command and "NS" in command:
                 return (
-                    "delta.chat. 21600 IN SOA ns1.first-ns.de. dns.hetzner.com."
+                    "domain.with.hidden.soa. 2137 IN NS ns1.desec.io.\n"
-                    " 2025102800 14400 1800 604800 3600"
+                    "domain.with.hidden.soa. 2137 IN NS ns2.desec.org."
                 )
+            if "NS" in command:
+                return "delta.chat. 21600 IN NS ns1.first-ns.de."
             command_chunks = command.split()
             domain, typ = command_chunks[4], command_chunks[6]
             try:
@@ -125,6 +130,17 @@ class TestPerformInitialChecks:
         assert not l
 
 
+@pytest.mark.parametrize(
+    ("domain", "ns"),
+    [
+        ("domain.with.public.soa", "ns1.first-ns.de."),
+        ("domain.with.hidden.soa", "ns1.desec.io."),
+    ],
+)
+def test_get_authoritative_ns(domain, ns, mockdns):
+    assert get_authoritative_ns(domain) == ns
+
+
 def test_parse_zone_records():
     text = """
     ; This is a comment

doc/source/getting_started.rst

@@ -14,6 +14,8 @@ Minimal requirements and prerequisites
 
 You will need the following:
 
+-  Control over a domain through a DNS provider of your choice.
+
 -  A Debian 12 **deployment server** with reachable SMTP/SUBMISSIONS/IMAPS/HTTPS ports.
    IPv6 is encouraged if available. Chatmail relay servers only require
    1GB RAM, one CPU, and perhaps 10GB storage for a few thousand active
@@ -26,11 +28,6 @@ You will need the following:
    (An ed25519 private key is required due to an `upstream bug in
    paramiko <https://github.com/paramiko/paramiko/issues/2191>`_)
 
--  Control over a domain through a DNS provider of your choice
-   (there is experimental support for :ref:`DNS-less relays <iponly>`).
-
-
-.. _setup:
 
 Setup with ``scripts/cmdeploy``
 -------------------------------------

doc/source/index.rst

@@ -19,4 +19,3 @@ Contributions and feedback welcome through the https://github.com/chatmail/relay
     reverse_dns
     related
     faq
-    iponly

doc/source/iponly.rst

@@ -1,29 +0,0 @@
-.. _iponly:
-
-Hosting without DNS records
-===========================
-
-.. note::
-
-   This option is experimental and might change without notice.
-
-In case you don't have a domain,
-for example in a local network,
-you can run a chatmail relay with only an IPv4 address as well.
-
-To deploy a relay without a domain,
-run ``cmdeploy init`` with only the IPv4 address
-during the :ref:`installation steps <setup>`,
-for example ``cmdeploy init 13.12.23.42``.
-
-Drawbacks
----------
-
-- your transport encryption will only use self-signed TLS certificates,
-  which are vulnerable against MITM attacks.
-  the chatmail core's end-to-end encryption should suffice in most scenarios though.
-
-- your messages will not be DKIM-signed;
-  experimentally, most chatmail relays accept non-DKIM-signed messages from IPv4-only relays,
-  but some relays might not accept messages from yours.
-