feat: faster and simpler DNS checks, better ip-address determination (#346)

* drastically reduce round-trips for dns checks, and do it during 'run' and 'dns' sub commands 
* provide progress-dots for dns checks and "--verbose" for seeing what is executed remotely 
* introduce ssh-mediated remote python function execution mechanism

.github/workflows/test-and-deploy.yaml

@@ -72,12 +72,12 @@ jobs:
 
       - run: cmdeploy init staging2.testrun.org
 
-      - run: cmdeploy run
+      - run: cmdeploy run --verbose
 
       - name: set DNS entries
         run: |
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
-          cmdeploy dns --zonefile staging-generated.zone
+          ssh -o StrictHostKeyChecking=accept-new root@staging2.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
+          cmdeploy dns --zonefile staging-generated.zone --verbose
           cat staging-generated.zone >> .github/workflows/staging.testrun.org-default.zone
           cat .github/workflows/staging.testrun.org-default.zone
           scp .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
@@ -88,5 +88,5 @@ jobs:
         run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow
 
       - name: cmdeploy dns (try 3 times)
-        run: cmdeploy dns || cmdeploy dns || cmdeploy dns
+        run: cmdeploy dns -v || cmdeploy dns -v || cmdeploy dns -v
 

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## untagged
 
+- Make DNS-checking faster and more interactive, run it fully during "cmdeploy run",
+  also introducing a generic mechanism for rapid remote ssh-based python function execution. 
+  ([#346](https://github.com/deltachat/chatmail/pull/346))
+
 - Don't fix file owner ship of /home/vmail 
   ([#345](https://github.com/deltachat/chatmail/pull/345))
 

cmdeploy/pyproject.toml

@@ -18,6 +18,7 @@ dependencies = [
   "ruff",
   "pytest",
   "pytest-xdist", 
+  "execnet",
   "imap_tools",
 ]
 

cmdeploy/src/cmdeploy/__init__.py

@@ -630,5 +630,3 @@ def deploy_chatmail(config_path: Path) -> None:
         name="Ensure cron is installed",
         packages=["cron"],
     )
-
-

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -15,7 +15,9 @@ from pathlib import Path
 from chatmaild.config import read_config, write_initial_config
 from termcolor import colored
 
-from cmdeploy.dns import check_necessary_dns, show_dns
+from . import remote_funcs
+from .dns import show_dns
+from .sshexec import SSHExec
 
 #
 # cmdeploy sub commands and options
@@ -51,12 +53,7 @@ def run_cmd_options(parser):
 
 def run_cmd(args, out):
     """Deploy chatmail services on the remote server."""
-    mail_domain = args.config.mail_domain
-    if not check_necessary_dns(
-        out,
-        mail_domain,
-    ):
-        sys.exit(1)
+    retcode, remote_data = show_dns(args, out)
 
     env = os.environ.copy()
     env["CHATMAIL_INI"] = args.inipath
@@ -65,7 +62,15 @@ def run_cmd(args, out):
     cmd = f"{pyinf} --ssh-user root {args.config.mail_domain} {deploy_path}"
 
     out.check_call(cmd, env=env)
-    print("Deploy completed, call `cmdeploy dns` next.")
+    if retcode == 0:
+        out.green("Deploy completed, call `cmdeploy test` next.")
+    elif not remote_data["acme_account_url"]:
+        out.red("Deploy completed but letsencrypt not configured")
+        out.red("Run 'cmdeploy dns' or 'cmdeploy run' again")
+        retcode = 0
+    else:
+        out.red("Deploy failed")
+    return retcode
 
 
 def dns_cmd_options(parser):
@@ -77,15 +82,15 @@ def dns_cmd_options(parser):
 
 
 def dns_cmd(args, out):
-    """Generate dns zone file."""
-    exit_code = show_dns(args, out)
-    exit(exit_code)
+    """Check DNS entries and optionally generate dns zone file."""
+    retcode, remote_data = show_dns(args, out)
+    return retcode
 
 
 def status_cmd(args, out):
     """Display status for online chatmail instance."""
 
-    ssh = f"ssh root@{args.config.mail_domain}"
+    sshexec = args.get_sshexec()
 
     out.green(f"chatmail domain: {args.config.mail_domain}")
     if args.config.privacy_mail:
@@ -93,10 +98,8 @@ def status_cmd(args, out):
     else:
         out.red("no privacy settings")
 
-    s1 = "systemctl --type=service --state=running"
-    for line in out.shell_output(f"{ssh} -- {s1}").split("\n"):
-        if line.startswith("  "):
-            print(line)
+    for line in sshexec(remote_funcs.get_systemd_running):
+        print(line)
 
 
 def test_cmd_options(parser):
@@ -135,14 +138,6 @@ def test_cmd(args, out):
 
 
 def fmt_cmd_options(parser):
-    parser.add_argument(
-        "--verbose",
-        "-v",
-        dest="verbose",
-        action="store_true",
-        help="provide information on invocations",
-    )
-
     parser.add_argument(
         "--check",
         "-c",
@@ -172,7 +167,6 @@ def fmt_cmd(args, out):
 
     out.check_call(" ".join(format_args), quiet=not args.verbose)
     out.check_call(" ".join(check_args), quiet=not args.verbose)
-    return 0
 
 
 def bench_cmd(args, out):
@@ -208,16 +202,6 @@ class Out:
         color = "red" if red else ("green" if green else None)
         print(colored(msg, color), file=file)
 
-    def shell_output(self, arg, no_print=False, timeout=10):
-        if not no_print:
-            self(f"[$ {arg}]", file=sys.stderr)
-            output = subprocess.STDOUT
-        else:
-            output = subprocess.DEVNULL
-        return subprocess.check_output(
-            arg, shell=True, timeout=timeout, stderr=output
-        ).decode()
-
     def check_call(self, arg, env=None, quiet=False):
         if not quiet:
             self(f"[$ {arg}]", file=sys.stderr)
@@ -240,6 +224,14 @@ def add_config_option(parser):
         type=Path,
         help="path to the chatmail.ini file",
     )
+    parser.add_argument(
+        "--verbose",
+        "-v",
+        dest="verbose",
+        action="store_true",
+        default=False,
+        help="provide verbose logging",
+    )
 
 
 def add_subcommand(subparsers, func):
@@ -279,11 +271,18 @@ def get_parser():
 
 
 def main(args=None):
-    """Provide main entry point for 'xdcget' CLI invocation."""
+    """Provide main entry point for 'cmdeploy' CLI invocation."""
     parser = get_parser()
     args = parser.parse_args(args=args)
     if not hasattr(args, "func"):
         return parser.parse_args(["-h"])
+
+    def get_sshexec(log=None):
+        print(f"[ssh] login to {args.config.mail_domain}")
+        return SSHExec(args.config.mail_domain, remote_funcs, log=log)
+
+    args.get_sshexec = get_sshexec
+
     out = Out()
     kwargs = {}
     if args.func.__name__ not in ("init_cmd", "fmt_cmd"):

cmdeploy/src/cmdeploy/dns.py

@@ -1,209 +1,71 @@
 import datetime
 import importlib
-import subprocess
 import sys
 
-import requests
-
-
-class DNS:
-    def __init__(self, out, mail_domain):
-        self.session = requests.Session()
-        self.out = out
-        self.ssh = f"ssh root@{mail_domain} -- "
-        self.out.shell_output(
-            f"{ self.ssh }'apt-get update && apt-get install -y dnsutils'",
-            timeout=60,
-            no_print=True,
-        )
-        try:
-            self.shell(f"unbound-control flush_zone {mail_domain}")
-        except subprocess.CalledProcessError:
-            pass
-
-    def shell(self, cmd):
-        try:
-            return self.out.shell_output(f"{self.ssh}{cmd}", no_print=True)
-        except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
-            if "exit status 255" in str(e) or "timed out" in str(e):
-                self.out.red(f"Error: can't reach the server with: {self.ssh[:-4]}")
-                sys.exit(1)
-            else:
-                raise
-
-    def get_ipv4(self):
-        cmd = "ip a | grep 'inet ' | grep 'scope global' | grep -oE '[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}' | head -1"
-        return self.shell(cmd).strip()
-
-    def get_ipv6(self):
-        cmd = "ip a | grep inet6 | grep 'scope global' | sed -e 's#/64 scope global##' | sed -e 's#inet6##'"
-        return self.shell(cmd).strip()
-
-    def get(self, typ: str, domain: str) -> str:
-        """Get a DNS entry or empty string if there is none."""
-        dig_result = self.shell(f"dig -r -q {domain} -t {typ} +short")
-        line = dig_result.partition("\n")[0]
-        return line
-
-    def check_ptr_record(self, ip: str, mail_domain) -> bool:
-        """Check the PTR record for an IPv4 or IPv6 address."""
-        result = self.shell(f"dig -r -x {ip} +short").rstrip()
-        return result == f"{mail_domain}."
+from . import remote_funcs
 
 
 def show_dns(args, out) -> int:
-    """Check existing DNS records, optionally write them to zone file, return exit code 0 or 1."""
+    """Check existing DNS records, optionally write them to zone file
+    and return (exitcode, remote_data) tuple."""
     template = importlib.resources.files(__package__).joinpath("chatmail.zone.f")
     mail_domain = args.config.mail_domain
-    ssh = f"ssh root@{mail_domain}"
-    dns = DNS(out, mail_domain)
 
-    print("Checking your DKIM keys and DNS entries...")
-    try:
-        acme_account_url = out.shell_output(f"{ssh} -- acmetool account-url")
-    except subprocess.CalledProcessError:
-        print("Please run `cmdeploy run` first.")
-        return 1
+    def log_progress(data):
+        sys.stdout.write(".")
+        sys.stdout.flush()
 
-    dkim_selector = "opendkim"
-    dkim_pubkey = out.shell_output(
-        ssh + f" -- openssl rsa -in /etc/dkimkeys/{dkim_selector}.private"
-        " -pubout 2>/dev/null | awk '/-/{next}{printf(\"%s\",$0)}'"
-    )
-    dkim_entry_value = f"v=DKIM1;k=rsa;p={dkim_pubkey};s=email;t=s"
-    dkim_entry_str = ""
-    while len(dkim_entry_value) >= 255:
-        dkim_entry_str += '"' + dkim_entry_value[:255] + '" '
-        dkim_entry_value = dkim_entry_value[255:]
-    dkim_entry_str += '"' + dkim_entry_value + '"'
-    dkim_entry = f"{dkim_selector}._domainkey.{mail_domain}. TXT {dkim_entry_str}"
+    sshexec = args.get_sshexec(log=print if args.verbose else log_progress)
+    print("Checking DNS entries ", end="\n" if args.verbose else "")
 
-    ipv6 = dns.get_ipv6()
-    reverse_ipv6 = dns.check_ptr_record(ipv6, mail_domain)
-    ipv4 = dns.get_ipv4()
-    reverse_ipv4 = dns.check_ptr_record(ipv4, mail_domain)
-    to_print = []
+    remote_data = sshexec(remote_funcs.perform_initial_checks, mail_domain=mail_domain)
+
+    assert remote_data["ipv4"] or remote_data["ipv6"]
 
     with open(template, "r") as f:
-        zonefile = (
-            f.read()
-            .format(
-                acme_account_url=acme_account_url,
-                sts_id=datetime.datetime.now().strftime("%Y%m%d%H%M"),
-                chatmail_domain=args.config.mail_domain,
-                dkim_entry=dkim_entry,
-                ipv6=ipv6,
-                ipv4=ipv4,
-            )
-            .strip()
+        zonefile = f.read().format(
+            acme_account_url=remote_data["acme_account_url"],
+            dkim_entry=remote_data["dkim_entry"],
+            ipv6=remote_data["ipv6"],
+            ipv4=remote_data["ipv4"],
+            sts_id=datetime.datetime.now().strftime("%Y%m%d%H%M"),
+            chatmail_domain=args.config.mail_domain,
         )
-        try:
-            with open(args.zonefile, "w+") as zf:
-                zf.write(zonefile)
-                print(f"DNS records successfully written to: {args.zonefile}")
-            return 0
-        except TypeError:
-            pass
-        for raw_line in zonefile.splitlines():
-            line = raw_line.format(
-                acme_account_url=acme_account_url,
-                sts_id=datetime.datetime.now().strftime("%Y%m%d%H%M"),
-                chatmail_domain=args.config.mail_domain,
-                dkim_entry=dkim_entry,
-                ipv6=ipv6,
-            ).strip()
-            for typ in ["A", "AAAA", "CNAME", "CAA"]:
-                if f" {typ} " in line:
-                    domain, value = line.split(f" {typ} ")
-                    current = dns.get(typ, domain.strip()[:-1])
-                    if current != value.strip():
-                        to_print.append(line)
-            if " MX " in line:
-                domain, typ, prio, value = line.split()
-                current = dns.get(typ, domain[:-1])
-                if not current:
-                    to_print.append(line)
-                elif current.split()[1] != value:
-                    print(line.replace(prio, str(int(current[0]) + 1)))
-            if " SRV " in line:
-                domain, typ, prio, weight, port, value = line.split()
-                current = dns.get("SRV", domain[:-1])
-                if current != f"{prio} {weight} {port} {value}":
-                    to_print.append(line)
-            if " TXT " in line:
-                domain, value = line.split(" TXT ")
-                current = dns.get("TXT", domain.strip()[:-1])
-                if domain.startswith("_mta-sts."):
-                    if current:
-                        if current.split("id=")[0] == value.split("id=")[0]:
-                            continue
 
-                # TXT records longer than 255 bytes
-                # are split into multiple <character-string>s.
-                # This typically happens with DKIM record
-                # which contains long RSA key.
-                #
-                # Removing `" "` before comparison
-                # to get back a single string.
-                if current.replace('" "', "") != value.replace('" "', ""):
-                    to_print.append(line)
+    to_print = sshexec(remote_funcs.check_zonefile, zonefile=zonefile)
+    if not args.verbose:
+        print()
+
+    if getattr(args, "zonefile", None):
+        with open(args.zonefile, "w+") as zf:
+            zf.write(zonefile)
+        out.green(f"DNS records successfully written to: {args.zonefile}")
+        return 0, remote_data
 
-    exit_code = 0
     if to_print:
         to_print.insert(
-            0, "You should configure the following DNS entries at your provider:\n"
+            0, "You should configure the following entries at your DNS provider:\n"
         )
         to_print.append(
             "\nIf you already configured the DNS entries, wait a bit until the DNS entries propagate to the Internet."
         )
-        print("\n".join(to_print))
+        out.red("\n".join(to_print))
         exit_code = 1
     else:
-        out.green("Great! All your DNS entries are correct.")
+        out.green("Great! All your DNS entries are verified and correct.")
+        exit_code = 0
 
     to_print = []
-    if not reverse_ipv4:
-        to_print.append(f"\tIPv4:\t{ipv4}\t{args.config.mail_domain}")
-    if not reverse_ipv6:
-        to_print.append(f"\tIPv6:\t{ipv6}\t{args.config.mail_domain}")
+    if not remote_data["reverse_ipv4"]:
+        to_print.append(f"\tIPv4:\t{remote_data['ipv4']}\t{args.config.mail_domain}")
+    if not remote_data["reverse_ipv6"]:
+        to_print.append(f"\tIPv6:\t{remote_data['ipv6']}\t{args.config.mail_domain}")
     if len(to_print) > 0:
-        if len(to_print) == 1:
-            warning = "You should add the following PTR/reverse DNS entry:"
-        else:
-            warning = "You should add the following PTR/reverse DNS entries:"
-        out.red(warning)
+        out.red("You need to set the following PTR/reverse DNS data:")
         for entry in to_print:
             print(entry)
-        print(
+        out.red(
             "You can do so at your hosting provider (maybe this isn't your DNS provider)."
         )
-        exit_code = 1
-    return exit_code
 
-
-def check_necessary_dns(out, mail_domain):
-    """Check whether $mail_domain and mta-sts.$mail_domain resolve."""
-    print("Checking necessary DNS records... ")
-    dns = DNS(out, mail_domain)
-    ipv4 = dns.get("A", mail_domain)
-    ipv6 = dns.get("AAAA", mail_domain)
-    mta_entry = dns.get("CNAME", "mta-sts." + mail_domain)
-    www_entry = dns.get("CNAME", "www." + mail_domain)
-    to_print = []
-    if not (ipv4 or ipv6):
-        to_print.append(f"\t{mail_domain}.\t\t\tA<your server's IPv4 address>")
-    if mta_entry != mail_domain + ".":
-        to_print.append(f"\tmta-sts.{mail_domain}.\tCNAME\t{mail_domain}.")
-    if www_entry != mail_domain + ".":
-        to_print.append(f"\twww.{mail_domain}.\tCNAME\t{mail_domain}.")
-    if to_print:
-        to_print.insert(
-            0,
-            "\nFor chatmail to work, you need to configure this at your DNS provider:\n",
-        )
-        for line in to_print:
-            print(line)
-        print()
-    else:
-        dns.out.green("All necessary DNS records seem to be set.")
-        return True
+    return exit_code, remote_data

cmdeploy/src/cmdeploy/remote_funcs.py

@@ -0,0 +1,109 @@
+"""
+Functions to be executed on an ssh-connected host.
+
+All functions of this module need to work with Python builtin types
+and standard library dependencies only.
+
+When a remote function executes remotely, it runs in a system python interpreter
+without any installed dependencies.
+
+"""
+
+import re
+import socket
+from subprocess import CalledProcessError, check_output
+
+
+def shell(command, fail_ok=False):
+    log(f"$ {command}")
+    try:
+        return check_output(command, shell=True).decode().rstrip()
+    except CalledProcessError:
+        if not fail_ok:
+            raise
+        return ""
+
+
+def get_systemd_running():
+    lines = shell("systemctl --type=service --state=running").split("\n")
+    return [line for line in lines if line.startswith("  ")]
+
+
+def perform_initial_checks(mail_domain):
+    res = {}
+
+    res["acme_account_url"] = shell("acmetool account-url", fail_ok=True)
+    if not shell("dig", fail_ok=True):
+        shell("apt-get install -y dnsutils")
+    shell(f"unbound-control flush_zone {mail_domain}", fail_ok=True)
+
+    res["dkim_entry"] = get_dkim_entry(mail_domain, dkim_selector="opendkim")
+
+    ipv4, reverse_ipv4 = get_ip_address_and_reverse(socket.AF_INET)
+    ipv6, reverse_ipv6 = get_ip_address_and_reverse(socket.AF_INET6)
+    res.update(dict(ipv4=ipv4, reverse_ipv4=reverse_ipv4))
+    res.update(dict(ipv6=ipv6, reverse_ipv6=reverse_ipv6))
+    return res
+
+
+def get_dkim_entry(mail_domain, dkim_selector):
+    dkim_pubkey = shell(
+        f"openssl rsa -in /etc/dkimkeys/{dkim_selector}.private "
+        "-pubout 2>/dev/null | awk '/-/{next}{printf(\"%s\",$0)}'"
+    )
+    dkim_value_raw = f"v=DKIM1;k=rsa;p={dkim_pubkey};s=email;t=s"
+    dkim_value = '" "'.join(re.findall(".{1,255}", dkim_value_raw))
+    return f'{dkim_selector}._domainkey.{mail_domain}. TXT "{dkim_value}"'
+
+
+def get_ip_address_and_reverse(typ):
+    sock = socket.socket(typ, socket.SOCK_DGRAM)
+    sock.settimeout(0)
+    sock.connect(("notifications.delta.chat", 1))
+    ip = sock.getsockname()[0]
+    return ip, shell(f"dig -r -x {ip} +short").rstrip(".")
+
+
+def query_dns(typ, domain):
+    res = shell(f"dig -r -q {domain} -t {typ} +short")
+    return set(filter(None, res.split("\n")))
+
+
+def check_zonefile(zonefile):
+    diff = []
+
+    for zf_line in zonefile.splitlines():
+        zf_domain, zf_typ, zf_value = zf_line.split(maxsplit=2)
+        zf_domain = zf_domain.rstrip(".")
+        zf_value = zf_value.strip()
+        query_values = query_dns(zf_typ, zf_domain)
+        if zf_value in query_values:
+            continue
+
+        if zf_typ == "CAA" and zf_value.endswith('accounturi="'):
+            # this is an initial run where acmetool did not work yet
+            continue
+
+        if query_values and zf_typ == "TXT" and zf_domain.startswith("_mta-sts."):
+            (query_value,) = query_values
+            if query_value.split("id=")[0] == zf_value.split("id=")[0]:
+                continue
+
+        assert zf_typ in ("A", "AAAA", "CNAME", "CAA", "SRV", "MX", "TXT"), zf_line
+        diff.append(zf_line)
+
+    return diff
+
+
+# check if this module is executed remotely
+# and setup a simple serialized function-execution loop
+
+if __name__ == "__channelexec__":
+
+    def log(item):
+        channel.send(("log", item))  # noqa
+
+    while 1:
+        func_name, kwargs = channel.receive()  # noqa
+        res = globals()[func_name](**kwargs)  # noqa
+        channel.send(("finish", res))  # noqa

cmdeploy/src/cmdeploy/sshexec.py

@@ -0,0 +1,20 @@
+import execnet
+
+
+class SSHExec:
+    RemoteError = execnet.RemoteError
+
+    def __init__(self, host, remote_funcs, log=None, python="python3", timeout=60):
+        self.gateway = execnet.makegateway(f"ssh=root@{host}//python={python}")
+        self._remote_cmdloop_channel = self.gateway.remote_exec(remote_funcs)
+        self.log = log
+        self.timeout = timeout
+
+    def __call__(self, func, **kwargs):
+        self._remote_cmdloop_channel.send((func.__name__, kwargs))
+        while 1:
+            code, data = self._remote_cmdloop_channel.receive(timeout=self.timeout)
+            if code == "log" and self.log:
+                self.log(data)
+            elif code == "finish":
+                return data

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -2,6 +2,24 @@ import smtplib
 
 import pytest
 
+from cmdeploy import remote_funcs
+from cmdeploy.sshexec import SSHExec
+
+
+class TestSSHExecutor:
+    @pytest.fixture
+    def sshexec(self, sshdomain):
+        return SSHExec(sshdomain, remote_funcs)
+
+    def test_ls(self, sshexec):
+        out = sshexec(remote_funcs.shell, command="ls")
+        out2 = sshexec(remote_funcs.shell, command="ls")
+        assert out == out2
+
+    def test_perform_initial(self, sshexec, maildomain):
+        res = sshexec(remote_funcs.perform_initial_checks, mail_domain=maildomain)
+        assert res["ipv4"] or res["ipv6"]
+
 
 def test_remote(remote, imap_or_smtp):
     lineproducer = remote.iter_output(imap_or_smtp.logcmd)