Prepare for pyinfra 3

chatmaild/src/chatmaild/tests/plugin.py

@@ -1,6 +1,4 @@
 import random
-from pathlib import Path
-import os
 import importlib.resources
 import itertools
 from email.parser import BytesParser
@@ -59,12 +57,7 @@ def db(tmpdir):
 
 @pytest.fixture
 def maildata(request):
-    try:
+    datadir = importlib.resources.files(__package__).joinpath("mail-data")
-        datadir = importlib.resources.files(__package__).joinpath("mail-data")
-    except TypeError:
-        # in python3.9 or lower, the above doesn't work, so we get datadir this way:
-        datadir = Path(os.getcwd()).joinpath("chatmaild/src/chatmaild/tests/mail-data")
-
     assert datadir.exists(), datadir
 
     def maildata(name, from_addr, to_addr):

cmdeploy/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 name = "cmdeploy"
 version = "0.2"
 dependencies = [
-  "pyinfra",
+  "pyinfra==3.0b0",
   "pillow",
   "qrcode",
   "markdown",

cmdeploy/src/cmdeploy/__init__.py

@@ -10,13 +10,15 @@ import io
 from pathlib import Path
 
 from pyinfra import host
-from pyinfra.operations import apt, files, server, systemd, pip
+from pyinfra.operations import apt, files, server, systemd, pip, util
 from pyinfra.facts.files import File
 from pyinfra.facts.systemd import SystemdEnabled
 from .acmetool import deploy_acmetool
 
 from chatmaild.config import read_config, Config
 
+from typing import Callable
+
 
 def _build_chatmaild(dist_dir) -> None:
     dist_dir = Path(dist_dir).resolve()
@@ -127,10 +129,8 @@ def _install_remote_venv_with_chatmaild(config) -> None:
         )
 
 
-def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
+def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> Callable[[], bool]:
     """Configures OpenDKIM"""
-    need_restart = False
-
     server.group(name="Create opendkim group", group="opendkim", system=True)
     server.user(
         name="Create opendkim user",
@@ -153,7 +153,6 @@ def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
         mode="644",
         config={"domain_name": domain, "opendkim_selector": dkim_selector},
     )
-    need_restart |= main_config.changed
 
     screen_script = files.put(
         src=importlib.resources.files(__package__).joinpath("opendkim/screen.lua"),
@@ -162,7 +161,6 @@ def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
         group="root",
         mode="644",
     )
-    need_restart |= screen_script.changed
 
     final_script = files.put(
         src=importlib.resources.files(__package__).joinpath("opendkim/final.lua"),
@@ -171,7 +169,6 @@ def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
         group="root",
         mode="644",
     )
-    need_restart |= final_script.changed
 
     files.directory(
         name="Add opendkim directory to /etc",
@@ -190,7 +187,6 @@ def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
         mode="644",
         config={"domain_name": domain, "opendkim_selector": dkim_selector},
     )
-    need_restart |= keytable.changed
 
     signing_table = files.template(
         src=importlib.resources.files(__package__).joinpath("opendkim/SigningTable"),
@@ -200,7 +196,7 @@ def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
         mode="644",
         config={"domain_name": domain, "opendkim_selector": dkim_selector},
     )
-    need_restart |= signing_table.changed
+
     files.directory(
         name="Add opendkim socket directory to /var/spool/postfix",
         path="/var/spool/postfix/opendkim",
@@ -225,10 +221,12 @@ def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
             _sudo_user="opendkim",
         )
 
-    return need_restart
+    return util.any_changed(
+        main_config, screen_script, final_script, keytable, signing_table
+    )
 
 
-def _install_mta_sts_daemon() -> bool:
+def _install_mta_sts_daemon() -> Callable[[], bool]:
     need_restart = False
 
     config = files.put(
@@ -241,7 +239,6 @@ def _install_mta_sts_daemon() -> bool:
         group="root",
         mode="644",
     )
-    need_restart |= config.changed
 
     server.shell(
         name="install postfix-mta-sts-resolver with pip",
@@ -261,15 +258,12 @@ def _install_mta_sts_daemon() -> bool:
         group="root",
         mode="644",
     )
-    need_restart |= systemd_unit.changed
 
-    return need_restart
+    return util.any_changed(config, systemd_unit)
 
 
-def _configure_postfix(config: Config, debug: bool = False) -> bool:
+def _configure_postfix(config: Config, debug: bool = False) -> Callable[[], bool]:
     """Configures Postfix SMTP server."""
-    need_restart = False
-
     main_config = files.template(
         src=importlib.resources.files(__package__).joinpath("postfix/main.cf.j2"),
         dest="/etc/postfix/main.cf",
@@ -278,7 +272,6 @@ def _configure_postfix(config: Config, debug: bool = False) -> bool:
         mode="644",
         config=config,
     )
-    need_restart |= main_config.changed
 
     master_config = files.template(
         src=importlib.resources.files(__package__).joinpath("postfix/master.cf.j2"),
@@ -289,7 +282,6 @@ def _configure_postfix(config: Config, debug: bool = False) -> bool:
         debug=debug,
         config=config,
     )
-    need_restart |= master_config.changed
 
     header_cleanup = files.put(
         src=importlib.resources.files(__package__).joinpath(
@@ -300,7 +292,6 @@ def _configure_postfix(config: Config, debug: bool = False) -> bool:
         group="root",
         mode="644",
     )
-    need_restart |= header_cleanup.changed
 
     # Login map that 1:1 maps email address to login.
     login_map = files.put(
@@ -310,15 +301,12 @@ def _configure_postfix(config: Config, debug: bool = False) -> bool:
         group="root",
         mode="644",
     )
-    need_restart |= login_map.changed
 
-    return need_restart
+    return util.any_changed(main_config, master_config, header_cleanup, login_map)
 
 
-def _configure_dovecot(config: Config, debug: bool = False) -> bool:
+def _configure_dovecot(config: Config, debug: bool = False) -> Callable[[], bool]:
     """Configures Dovecot IMAP server."""
-    need_restart = False
-
     main_config = files.template(
         src=importlib.resources.files(__package__).joinpath("dovecot/dovecot.conf.j2"),
         dest="/etc/dovecot/dovecot.conf",
@@ -328,7 +316,6 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
         config=config,
         debug=debug,
     )
-    need_restart |= main_config.changed
     auth_config = files.put(
         src=importlib.resources.files(__package__).joinpath("dovecot/auth.conf"),
         dest="/etc/dovecot/auth.conf",
@@ -336,7 +323,6 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
         group="root",
         mode="644",
     )
-    need_restart |= auth_config.changed
 
     files.template(
         src=importlib.resources.files(__package__).joinpath("dovecot/expunge.cron.j2"),
@@ -358,13 +344,11 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
             persist=True,
         )
 
-    return need_restart
+    return util.any_changed(main_config, auth_config)
 
 
-def _configure_nginx(domain: str, debug: bool = False) -> bool:
+def _configure_nginx(domain: str, debug: bool = False) -> Callable[[], bool]:
     """Configures nginx HTTP server."""
-    need_restart = False
-
     main_config = files.template(
         src=importlib.resources.files(__package__).joinpath("nginx/nginx.conf.j2"),
         dest="/etc/nginx/nginx.conf",
@@ -373,7 +357,6 @@ def _configure_nginx(domain: str, debug: bool = False) -> bool:
         mode="644",
         config={"domain_name": domain},
     )
-    need_restart |= main_config.changed
 
     autoconfig = files.template(
         src=importlib.resources.files(__package__).joinpath("nginx/autoconfig.xml.j2"),
@@ -383,7 +366,6 @@ def _configure_nginx(domain: str, debug: bool = False) -> bool:
         mode="644",
         config={"domain_name": domain},
     )
-    need_restart |= autoconfig.changed
 
     mta_sts_config = files.template(
         src=importlib.resources.files(__package__).joinpath("nginx/mta-sts.txt.j2"),
@@ -393,7 +375,6 @@ def _configure_nginx(domain: str, debug: bool = False) -> bool:
         mode="644",
         config={"domain_name": domain},
     )
-    need_restart |= mta_sts_config.changed
 
     # install CGI newemail script
     #
@@ -414,7 +395,7 @@ def _configure_nginx(domain: str, debug: bool = False) -> bool:
         mode="755",
     )
 
-    return need_restart
+    return util.any_changed(main_config, autoconfig, mta_sts_config)
 
 
 def _remove_rspamd() -> None:
@@ -518,7 +499,12 @@ def deploy_chatmail(config_path: Path) -> None:
         service="opendkim.service",
         running=True,
         enabled=True,
-        restarted=opendkim_need_restart,
+    )
+    systemd.service(
+        name="Restart OpenDKIM",
+        service="opendkim.service",
+        restarted=True,
+        _if=opendkim_need_restart,
     )
 
     systemd.service(
@@ -527,7 +513,12 @@ def deploy_chatmail(config_path: Path) -> None:
         daemon_reload=True,
         running=True,
         enabled=True,
-        restarted=mta_sts_need_restart,
+    )
+    systemd.service(
+        name="Restart MTA-STS daemon",
+        service="mta-sts-daemon.service",
+        restarted=True,
+        _if=mta_sts_need_restart,
     )
 
     systemd.service(
@@ -535,7 +526,12 @@ def deploy_chatmail(config_path: Path) -> None:
         service="postfix.service",
         running=True,
         enabled=True,
-        restarted=postfix_need_restart,
+    )
+    systemd.service(
+        name="Restart Postfix",
+        service="postfix.service",
+        restarted=True,
+        _if=postfix_need_restart,
     )
 
     systemd.service(
@@ -543,7 +539,12 @@ def deploy_chatmail(config_path: Path) -> None:
         service="dovecot.service",
         running=True,
         enabled=True,
-        restarted=dovecot_need_restart,
+    )
+    systemd.service(
+        name="Restart Dovecot",
+        service="dovecot.service",
+        restarted=True,
+        _if=dovecot_need_restart,
     )
 
     systemd.service(
@@ -551,7 +552,12 @@ def deploy_chatmail(config_path: Path) -> None:
         service="nginx.service",
         running=True,
         enabled=True,
-        restarted=nginx_need_restart,
+    )
+    systemd.service(
+        name="Restart nginx",
+        service="nginx.service",
+        restarted=True,
+        _if=nginx_need_restart,
     )
 
     # This file is used by auth proxy.

cmdeploy/src/cmdeploy/acmetool/__init__.py

@@ -69,7 +69,12 @@ def deploy_acmetool(nginx_hook=False, email="", domains=[]):
         service="acmetool-redirector.service",
         running=True,
         enabled=True,
-        restarted=service_file.changed,
+    )
+    systemd.service(
+        name="Restart acmetool-redirector service",
+        service="acmetool-redirector.service",
+        restarted=True,
+        _if=service_file.did_change,
     )
 
     server.shell(

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -2,7 +2,6 @@
 Provides the `cmdeploy` entry point function,
 along with command line option and subcommand parsing.
 """
-
 import argparse
 import shutil
 import subprocess

cmdeploy/src/cmdeploy/dns.py

@@ -5,8 +5,6 @@ import importlib
 import subprocess
 import datetime
 
-from typing import Optional
-
 
 class DNS:
     def __init__(self, out, mail_domain):
@@ -36,11 +34,12 @@ class DNS:
         cmd = "ip a | grep inet6 | grep 'scope global' | sed -e 's#/64 scope global##' | sed -e 's#inet6##'"
         return self.shell(cmd).strip()
 
-    def get(self, typ: str, domain: str) -> str:
+    def get(self, typ: str, domain: str) -> str | None:
-        """Get a DNS entry or empty string if there is none."""
+        """Get a DNS entry"""
         dig_result = self.shell(f"dig -r -q {domain} -t {typ} +short")
         line = dig_result.partition("\n")[0]
-        return line
+        if line:
+            return line
 
     def check_ptr_record(self, ip: str, mail_domain) -> bool:
         """Check the PTR record for an IPv4 or IPv6 address."""
@@ -55,25 +54,22 @@ def show_dns(args, out) -> int:
     ssh = f"ssh root@{mail_domain}"
     dns = DNS(out, mail_domain)
 
+    def read_dkim_entries(entry):
+        lines = []
+        for line in entry.split("\n"):
+            if line.startswith(";") or not line.strip():
+                continue
+            line = line.replace("\t", " ")
+            lines.append(line)
+        return "\n".join(lines)
+
     print("Checking your DKIM keys and DNS entries...")
     try:
         acme_account_url = out.shell_output(f"{ssh} -- acmetool account-url")
     except subprocess.CalledProcessError:
         print("Please run `cmdeploy run` first.")
         return 1
-
+    dkim_entry = read_dkim_entries(out.shell_output(f"{ssh} -- opendkim-genzone -F"))
-    dkim_selector = "opendkim"
-    dkim_pubkey = out.shell_output(
-        ssh + f" -- openssl rsa -in /etc/dkimkeys/{dkim_selector}.private"
-        " -pubout 2>/dev/null | awk '/-/{next}{printf(\"%s\",$0)}'"
-    )
-    dkim_entry_value = f"v=DKIM1;k=rsa;p={dkim_pubkey};s=email;t=s"
-    dkim_entry_str = ""
-    while len(dkim_entry_value) >= 255:
-        dkim_entry_str += '"' + dkim_entry_value[:255] + '" '
-        dkim_entry_value = dkim_entry_value[255:]
-    dkim_entry_str += '"' + dkim_entry_value + '"'
-    dkim_entry = f"{dkim_selector}._domainkey.{mail_domain}. TXT {dkim_entry_str}"
 
     ipv6 = dns.get_ipv6()
     reverse_ipv6 = dns.check_ptr_record(ipv6, mail_domain)
@@ -101,6 +97,7 @@ def show_dns(args, out) -> int:
             return 0
         except TypeError:
             pass
+        started_dkim_parsing = False
         for line in zonefile.splitlines():
             line = line.format(
                 acme_account_url=acme_account_url,
@@ -127,23 +124,28 @@ def show_dns(args, out) -> int:
                 current = dns.get("SRV", domain[:-1])
                 if current != f"{prio} {weight} {port} {value}":
                     to_print.append(line)
-            if " TXT " in line:
+            if "  TXT " in line:
                 domain, value = line.split(" TXT ")
                 current = dns.get("TXT", domain.strip()[:-1])
                 if domain.startswith("_mta-sts."):
                     if current:
                         if current.split("id=")[0] == value.split("id=")[0]:
                             continue
-
+                if current != value:
-                # TXT records longer than 255 bytes
-                # are split into multiple <character-string>s.
-                # This typically happens with DKIM record
-                # which contains long RSA key.
-                #
-                # Removing `" "` before comparison
-                # to get back a single string.
-                if current.replace('" "', "") != value.replace('" "', ""):
                     to_print.append(line)
+            if "IN TXT ( " in line:
+                started_dkim_parsing = True
+                dkim_lines = [line]
+            if started_dkim_parsing and line.startswith('"'):
+                dkim_lines.append(" " + line)
+        domain, data = "\n".join(dkim_lines).split(" IN TXT ")
+        current = dns.get("TXT", domain.strip()[:-1])
+        if current:
+            current = "( %s )" % (current.replace('" "', '"\n "'))
+            if current.replace(";", "\\;") != data:
+                to_print.append(dkim_entry)
+        else:
+            to_print.append(dkim_entry)
 
     exit_code = 0
     if to_print:

cmdeploy/src/cmdeploy/dovecot/expunge.cron.j2

@@ -1,10 +1,10 @@
 # delete all mails after {{ config.delete_mails_after }} days, in the Inbox
-2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }} -path '*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 # or in any IMAP subfolder
-2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 # even if they are unseen
-2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }} -path '*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 # or only temporary (but then they shouldn't be around after {{ config.delete_mails_after }} days anyway).
-2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }} -path '*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete

cmdeploy/src/cmdeploy/tests/online/test_0_login.py

@@ -1,7 +1,6 @@
 import pytest
 import threading
 import queue
-import socket
 
 from chatmaild.config import read_config
 from cmdeploy.cmdeploy import main
@@ -79,24 +78,3 @@ def test_concurrent_logins_same_account(
 
     for _ in conns:
         assert login_results.get()
-
-
-def test_no_vrfy(chatmail_config):
-    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    sock.connect((chatmail_config.mail_domain, 25))
-    banner = sock.recv(1024)
-    print(banner)
-    sock.send(b"VRFY wrongaddress@%s\r\n" % (chatmail_config.mail_domain.encode(),))
-    result = sock.recv(1024)
-    print(result)
-    sock.send(b"VRFY echo@%s\r\n" % (chatmail_config.mail_domain.encode(),))
-    result2 = sock.recv(1024)
-    print(result2)
-    assert result[0:10] == result2[0:10]
-    sock.send(b"VRFY wrongaddress\r\n")
-    result = sock.recv(1024)
-    print(result)
-    sock.send(b"VRFY echo\r\n")
-    result2 = sock.recv(1024)
-    print(result2)
-    assert result[0:10] == result2[0:10] == b"252 2.0.0 "

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -83,18 +83,3 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
             assert b"4.7.1: Too much mail from" in outcome[1]
             return
     pytest.fail("Rate limit was not exceeded")
-
-
-def test_expunged(remote, chatmail_config):
-    outdated_days = int(chatmail_config.delete_mails_after) + 1
-    find_cmds = [
-        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/cur/*' -mtime +{outdated_days} -type f",
-        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/.*/cur/*' -mtime +{outdated_days} -type f",
-        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/new/*' -mtime +{outdated_days} -type f",
-        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/.*/new/*' -mtime +{outdated_days} -type f",
-        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/tmp/*' -mtime +{outdated_days} -type f",
-        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/.*/tmp/*' -mtime +{outdated_days} -type f",
-    ]
-    for cmd in find_cmds:
-        for line in remote.iter_output(cmd):
-            assert not line