Document ports 80 and 443 and add more hyperlinks

This reverts commit c1d3de926e.

.github/workflows/ci.yaml

@@ -33,8 +33,5 @@ jobs:
       - name: run deploy-chatmail offline tests 
         run: pytest --pyargs cmdeploy 
 
-      - name: initialize with chatmail domain
-        run: cmdeploy init chat.example.org  
-
       # all other cmdeploy commands require a staging server 
       # see https://github.com/deltachat/chatmail/issues/100

README.md

@@ -15,8 +15,8 @@ after which the initially specified password is required for using them.
 
 ## Deploying your own chatmail server 
 
-We subsequently use `CHATMAIL_DOMAIN` as a placeholder for your fully qualified 
+We use `chat.example.org` as the chatmail domain in the following steps. 
-DNS domain name (FQDN), for example `chat.example.org`.
+Please substitute it with your own domain. 
 
 1. Install the `cmdeploy` command in a virtualenv
 
@@ -29,15 +29,15 @@ DNS domain name (FQDN), for example `chat.example.org`.
 2. Create chatmail configuration file `chatmail.ini`:
 
    ```
-    scripts/cmdeploy init CHATMAIL_DOMAIN
+    scripts/cmdeploy init chat.example.org  # <-- use your domain 
    ```
 
-3. Setup first DNS records for your `CHATMAIL_DOMAIN`,
+3. Setup first DNS records for your chatmail domain, 
    according to the hints provided by `cmdeploy init`.
    Verify that SSH root login works:
 
    ```
-    ssh root@CHATMAIL_DOMAIN
+    ssh root@chat.example.org   # <-- use your domain 
    ```
 
 4. Deploy to the remote chatmail server:
@@ -45,13 +45,9 @@ DNS domain name (FQDN), for example `chat.example.org`.
    ```
     scripts/cmdeploy run
    ```
-
+   This script will also show you additional DNS records
-5. To output a DNS zone file from which you can transfer DNS records 
+   which you should configure at your DNS provider
-   to your DNS provider:
+   (it can take some time until they are public).
-
-   ```
-    scripts/cmdeploy dns
-   ```
 
 ### Other helpful commands:
 
@@ -61,6 +57,12 @@ To check the status of your remotely running chatmail service:
 scripts/cmdeploy status
 ```
 
+To check whether your DNS records are correct:
+
+```
+scripts/cmdeploy dns
+```
+
 To test whether your chatmail service is working correctly:
 
 ```
@@ -75,7 +77,7 @@ scripts/cmdeploy bench
 
 ## Overview of this repository
 
-This repository drives the development of "chatmail instances",
+This repository drives the development of chatmail services, 
 comprised of minimal setups of
 
 - [postfix smtp server](https://www.postfix.org)
@@ -91,7 +93,7 @@ as well as custom services that are integrated with these two:
   to send mails for them.
 
 - `chatmaild/src/chatmaild/filtermail.py` prevents
-  unencrypted e-mail from leaving the chatmail instance
+  unencrypted e-mail from leaving the chatmail service
   and is integrated into postfix's outbound mail pipelines.
 
 There is also the `cmdeploy/src/cmdeploy/cmdeploy.py` command line tool
@@ -104,7 +106,7 @@ to automatically install all chatmail components on a server.
 ### Home page and getting started for users
 
 `cmdeploy run` also creates default static Web pages and deploys them
-to an nginx web server under `https://CHATMAIL_DOMAIN`.
+to a nginx web server with: 
 
 - a default `index.html` along with a QR code that users can click to
   create accounts on your chatmail provider,
@@ -149,10 +151,12 @@ While this file is present, account creation will be blocked.
 
 ### Ports
 
-Postfix listens on ports 25 (smtp) and 587 (submission) and 465 (submissions).
+[Postfix](http://www.postfix.org/) listens on ports 25 (smtp) and 587 (submission) and 465 (submissions).
-Dovecot listens on ports 143(imap) and 993 (imaps).
+[Dovecot](https://www.dovecot.org/) listens on ports 143 (imap) and 993 (imaps).
+[nginx](https://www.nginx.com/) listens on port 443 (https).
+[acmetool](https://hlandau.github.io/acmetool/) listens on port 80 (http).
 
 Delta Chat apps will, however, discover all ports and configurations
-automatically by reading the `autoconfig.xml` file from the chatmail service.
+automatically by reading the [autoconfig XML file](https://web.archive.org/web/20210624004729/https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration) from the chatmail service.
 
 

chatmaild/src/chatmaild/doveauth.py

@@ -102,12 +102,17 @@ def handle_dovecot_request(msg, db, config: Config):
     short_command = msg[0]
     if short_command == "L":  # LOOKUP
         parts = msg[1:].split("\t")
-        keyname, user = parts[:2]
+
+        # Dovecot <2.3.17 has only one part,
+        # do not attempt to read any other parts for compatibility.
+        keyname = parts[0]
+
         namespace, type, *args = keyname.split("/")
         reply_command = "F"
         res = ""
         if namespace == "shared":
             if type == "userdb":
+                user = args[0]
                 if user.endswith(f"@{config.mail_domain}"):
                     res = lookup_userdb(db, user)
                 if res:
@@ -115,6 +120,7 @@ def handle_dovecot_request(msg, db, config: Config):
                 else:
                     reply_command = "N"
             elif type == "passdb":
+                user = args[1]
                 if user.endswith(f"@{config.mail_domain}"):
                     res = lookup_passdb(db, config, user, cleartext_password=args[0])
                 if res:

chatmaild/src/chatmaild/echo.py

@@ -6,7 +6,6 @@ it will echo back any message that has non-empty text and also supports the /hel
 import logging
 import os
 import sys
-from threading import Thread
 
 from deltachat_rpc_client import Bot, DeltaChat, EventType, Rpc, events
 
@@ -76,10 +75,7 @@ def main():
             config = read_config(sys.argv[1])
             password = create_newemail_dict(config).get("password")
             email = "echo@" + config.mail_domain
-            configure_thread = Thread(
+            bot.configure(email, password)
-                target=bot.configure, kwargs={"email": email, "password": password}
-            )
-            configure_thread.start()
         bot.run_forever()
 
 

cmdeploy/src/cmdeploy/__init__.py

@@ -399,7 +399,11 @@ def deploy_chatmail(config_path: Path) -> None:
     # to use 127.0.0.1 as the resolver.
     apt.packages(
         name="Install unbound",
-        packages="unbound",
+        packages=["unbound", "unbound-anchor", "dnsutils"],
+    )
+    server.shell(
+        name="Generate root keys for validating DNSSEC",
+        commands=["unbound-anchor -a /var/lib/unbound/root.key || true"],
     )
     systemd.service(
         name="Start and enable unbound",

cmdeploy/src/cmdeploy/acmetool/__init__.py

@@ -1,6 +1,6 @@
 import importlib.resources
 
-from pyinfra.operations import apt, files, server
+from pyinfra.operations import apt, files, systemd, server
 
 
 def deploy_acmetool(nginx_hook=False, email="", domains=[]):
@@ -46,6 +46,23 @@ def deploy_acmetool(nginx_hook=False, email="", domains=[]):
         mode="644",
     )
 
+    service_file = files.put(
+        src=importlib.resources.files(__package__).joinpath(
+            "acmetool-redirector.service"
+        ),
+        dest="/etc/systemd/system/acmetool-redirector.service",
+        user="root",
+        group="root",
+        mode="644",
+    )
+    systemd.service(
+        name="Setup acmetool-redirector service",
+        service="acmetool-redirector.service",
+        running=True,
+        enabled=True,
+        restarted=service_file.changed,
+    )
+
     server.shell(
         name=f"Request certificate for: { ', '.join(domains) }",
         commands=[f"acmetool want { ' '.join(domains)}"],

cmdeploy/src/cmdeploy/acmetool/acmetool-redirector.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=acmetool HTTP redirector
+
+[Service]
+Type=notify
+ExecStart=/usr/bin/acmetool redirector --service.uid=daemon
+Restart=always
+RestartSec=30
+
+[Install]
+WantedBy=multi-user.target

cmdeploy/src/cmdeploy/chatmail.zone.f

@@ -1,12 +1,14 @@
+{chatmail_domain}.                   A     {ipv4}
+{chatmail_domain}.                   AAAA  {ipv6}
 {chatmail_domain}.                   MX 10 {chatmail_domain}.
 _submission._tcp.{chatmail_domain}.  SRV 0 1 587 {chatmail_domain}.
 _submissions._tcp.{chatmail_domain}. SRV 0 1 465 {chatmail_domain}.
 _imap._tcp.{chatmail_domain}.        SRV 0 1 143 {chatmail_domain}.
 _imaps._tcp.{chatmail_domain}.       SRV 0 1 993 {chatmail_domain}.
-{chatmail_domain}.                IN CAA 128 issue "letsencrypt.org;accounturi={acme_account_url}"
+{chatmail_domain}.                   CAA 128 issue "letsencrypt.org;accounturi={acme_account_url}"
 {chatmail_domain}.                   TXT "v=spf1 a:{chatmail_domain} -all"
 _dmarc.{chatmail_domain}.            TXT "v=DMARC1;p=reject;rua=mailto:{email};ruf=mailto:{email};fo=1;adkim=r;aspf=r"
-_mta-sts.{chatmail_domain}.       IN TXT "v=STSv1; id={sts_id}"
+_mta-sts.{chatmail_domain}.          TXT "v=STSv1; id={sts_id}"
-mta-sts.{chatmail_domain}.        IN CNAME {chatmail_domain}.
+mta-sts.{chatmail_domain}.           CNAME {chatmail_domain}.
-_smtp._tls.{chatmail_domain}.     IN TXT "v=TLSRPTv1;rua=mailto:{email}"
+_smtp._tls.{chatmail_domain}.        TXT "v=TLSRPTv1;rua=mailto:{email}"
 {dkim_entry}

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -3,7 +3,6 @@ Provides the `cmdeploy` entry point function,
 along with command line option and subcommand parsing.
 """
 import argparse
-import datetime
 import shutil
 import subprocess
 import importlib.resources
@@ -15,6 +14,7 @@ from pathlib import Path
 
 from termcolor import colored
 from chatmaild.config import read_config, write_initial_config
+from cmdeploy.dns import show_dns, check_necessary_dns
 
 
 #
@@ -32,11 +32,16 @@ def init_cmd_options(parser):
 
 def init_cmd(args, out):
     """Initialize chatmail config file."""
+    mail_domain = args.chatmail_domain
     if args.inipath.exists():
-        out.red(f"Path exists, not modifying: {args.inipath}")
+        print(f"Path exists, not modifying: {args.inipath}")
-        raise SystemExit(1)
+    else:
-    write_initial_config(args.inipath, args.chatmail_domain)
+        write_initial_config(args.inipath, mail_domain)
-    out.green(f"created config file for {args.chatmail_domain} in {args.inipath}")
+        out.green(f"created config file for {mail_domain} in {args.inipath}")
+    check_necessary_dns(
+        out,
+        mail_domain,
+    )
 
 
 def run_cmd_options(parser):
@@ -50,47 +55,34 @@ def run_cmd_options(parser):
 
 def run_cmd(args, out):
     """Deploy chatmail services on the remote server."""
+    mail_domain = args.config.mail_domain
+    if not check_necessary_dns(
+        out,
+        mail_domain,
+    ):
+        sys.exit(1)
 
     env = os.environ.copy()
     env["CHATMAIL_INI"] = args.inipath
     deploy_path = importlib.resources.files(__package__).joinpath("deploy.py").resolve()
     pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
     cmd = f"{pyinf} --ssh-user root {args.config.mail_domain} {deploy_path}"
+
     out.check_call(cmd, env=env)
+    print("Deploy completed, call `cmdeploy dns` next.")
+
+
+def dns_cmd_options(parser):
+    parser.add_argument(
+        "--zonefile",
+        dest="zonefile",
+        help="print the whole zonefile for deploying directly",
+    )
 
 
 def dns_cmd(args, out):
     """Generate dns zone file."""
-    template = importlib.resources.files(__package__).joinpath("chatmail.zone.f")
+    show_dns(args, out)
-    ssh = f"ssh root@{args.config.mail_domain}"
-
-    def read_dkim_entries(entry):
-        lines = []
-        for line in entry.split("\n"):
-            if line.startswith(";") or not line.strip():
-                continue
-            line = line.replace("\t", " ")
-            lines.append(line)
-        return "\n".join(lines)
-
-    acme_account_url = out.shell_output(f"{ssh} -- acmetool account-url")
-    dkim_entry = read_dkim_entries(out.shell_output(f"{ssh} -- opendkim-genzone -F"))
-
-    out(
-        f"[writing {args.config.mail_domain} zone data (using space as separator) to stdout output]",
-        green=True,
-    )
-    print(
-        template.read_text()
-        .format(
-            acme_account_url=acme_account_url,
-            email=f"root@{args.config.mail_domain}",
-            sts_id=datetime.datetime.now().strftime("%Y%m%d%H%M"),
-            chatmail_domain=args.config.mail_domain,
-            dkim_entry=dkim_entry,
-        )
-        .strip()
-    )
 
 
 def status_cmd(args, out):
@@ -219,9 +211,15 @@ class Out:
         color = "red" if red else ("green" if green else None)
         print(colored(msg, color), file=file)
 
-    def shell_output(self, arg):
+    def shell_output(self, arg, no_print=False, timeout=10):
+        if not no_print:
             self(f"[$ {arg}]", file=sys.stderr)
-        return subprocess.check_output(arg, shell=True).decode()
+            output = subprocess.STDOUT
+        else:
+            output = subprocess.DEVNULL
+        return subprocess.check_output(
+            arg, shell=True, timeout=timeout, stderr=output
+        ).decode()
 
     def check_call(self, arg, env=None, quiet=False):
         if not quiet:

cmdeploy/src/cmdeploy/dns.py

@@ -0,0 +1,211 @@
+import sys
+
+import requests
+import importlib
+import subprocess
+import datetime
+from ipaddress import ip_address
+
+
+class DNS:
+    def __init__(self, out, mail_domain):
+        self.session = requests.Session()
+        self.out = out
+        self.ssh = f"ssh root@{mail_domain} -- "
+        try:
+            self.shell(f"unbound-control flush_zone {mail_domain}")
+        except subprocess.CalledProcessError:
+            pass
+
+    def shell(self, cmd):
+        try:
+            return self.out.shell_output(f"{self.ssh}{cmd}", no_print=True)
+        except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
+            if "exit status 255" in str(e) or "timed out" in str(e):
+                self.out.red(f"Error: can't reach the server with: {self.ssh[:-4]}")
+                sys.exit(1)
+            else:
+                raise
+
+    def get_ipv4(self):
+        cmd = "ip a | grep 'inet ' | grep 'scope global' | grep -oE '[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}' | head -1"
+        return self.shell(cmd).strip()
+
+    def get_ipv6(self):
+        cmd = "ip a | grep inet6 | grep 'scope global' | sed -e 's#/64 scope global##' | sed -e 's#inet6##'"
+        return self.shell(cmd).strip()
+
+    def get(self, typ: str, domain: str) -> str | None:
+        """Get a DNS entry"""
+        dig_result = self.shell(f"dig {typ} {domain}")
+        line_num = 0
+        for line in dig_result.splitlines():
+            line_num += 1
+            if line.strip() == ";; ANSWER SECTION:":
+                return dig_result.splitlines()[line_num].split("\t")[-1]
+
+    def check_ptr_record(self, ip: str, mail_domain) -> str:
+        """Check the PTR record for an IPv4 or IPv6 address."""
+        result = self.get("-x", ip)
+        if result:
+            if ip_address(ip).version == 6:
+                result = result.split()[-1]
+            if result[:-1] == mail_domain:
+                return result
+
+
+def show_dns(args, out):
+    template = importlib.resources.files(__package__).joinpath("chatmail.zone.f")
+    mail_domain = args.config.mail_domain
+    ssh = f"ssh root@{mail_domain}"
+    dns = DNS(out, mail_domain)
+
+    def read_dkim_entries(entry):
+        lines = []
+        for line in entry.split("\n"):
+            if line.startswith(";") or not line.strip():
+                continue
+            line = line.replace("\t", " ")
+            lines.append(line)
+        return "\n".join(lines)
+
+    print("Checking your DKIM keys and DNS entries...")
+    try:
+        acme_account_url = out.shell_output(f"{ssh} -- acmetool account-url")
+    except subprocess.CalledProcessError:
+        print("Please run `cmdeploy run` first.")
+        return
+    dkim_entry = read_dkim_entries(out.shell_output(f"{ssh} -- opendkim-genzone -F"))
+
+    ipv6 = dns.get_ipv6()
+    reverse_ipv6 = dns.check_ptr_record(ipv6, mail_domain)
+    ipv4 = dns.get_ipv4()
+    reverse_ipv4 = dns.check_ptr_record(ipv4, mail_domain)
+    to_print = []
+
+    with open(template, "r") as f:
+        zonefile = (
+            f.read()
+            .format(
+                acme_account_url=acme_account_url,
+                email=f"root@{args.config.mail_domain}",
+                sts_id=datetime.datetime.now().strftime("%Y%m%d%H%M"),
+                chatmail_domain=args.config.mail_domain,
+                dkim_entry=dkim_entry,
+                ipv6=ipv6,
+                ipv4=ipv4,
+            )
+            .strip()
+        )
+        try:
+            with open(args.zonefile, "w+") as zf:
+                zf.write(zonefile)
+                print(f"DNS records successfully written to: {args.zonefile}")
+            return
+        except TypeError:
+            pass
+        started_dkim_parsing = False
+        for line in zonefile.splitlines():
+            line = line.format(
+                acme_account_url=acme_account_url,
+                email=f"root@{args.config.mail_domain}",
+                sts_id=datetime.datetime.now().strftime("%Y%m%d%H%M"),
+                chatmail_domain=args.config.mail_domain,
+                dkim_entry=dkim_entry,
+                ipv6=ipv6,
+            ).strip()
+            for typ in ["A", "AAAA", "CNAME", "CAA"]:
+                if f" {typ} " in line:
+                    domain, value = line.split(f" {typ} ")
+                    current = dns.get(typ, domain.strip()[:-1])
+                    if current != value.strip():
+                        to_print.append(line)
+            if " MX " in line:
+                domain, typ, prio, value = line.split()
+                current = dns.get(typ, domain[:-1])
+                if not current:
+                    to_print.append(line)
+                elif current.split()[1] != value:
+                    print(line.replace(prio, str(int(current[0]) + 1)))
+            if " SRV " in line:
+                domain, typ, prio, weight, port, value = line.split()
+                current = dns.get("SRV", domain[:-1])
+                if current != f"{prio} {weight} {port} {value}":
+                    to_print.append(line)
+            if "  TXT " in line:
+                domain, value = line.split(" TXT ")
+                current = dns.get("TXT", domain.strip()[:-1])
+                if domain.startswith("_mta-sts."):
+                    if current:
+                        if current.split("id=")[0] == value.split("id=")[0]:
+                            continue
+                if current != value:
+                    to_print.append(line)
+            if " IN TXT ( " in line:
+                started_dkim_parsing = True
+                dkim_lines = [line]
+            if started_dkim_parsing and line.startswith('"'):
+                dkim_lines.append(" " + line)
+        domain, data = "\n".join(dkim_lines).split(" IN TXT ")
+        current = dns.get("TXT", domain.strip()[:-1])
+        if current:
+            current = "( %s )" % (current.replace('" "', '"\n "'))
+            if current.replace(";", "\\;") != data:
+                to_print.append(dkim_entry)
+        else:
+            to_print.append(dkim_entry)
+
+    if to_print:
+        to_print.insert(
+            0, "You should configure the following DNS entries at your provider:\n"
+        )
+        to_print.append(
+            "\nIf you already configured the DNS entries, wait a bit until the DNS entries propagate to the Internet."
+        )
+        print("\n".join(to_print))
+    else:
+        out.green("Great! All your DNS entries are correct.")
+
+    to_print = []
+    if not reverse_ipv4:
+        to_print.append(f"\tIPv4:\t{ipv4}\t{args.config.mail_domain}")
+    if not reverse_ipv6:
+        to_print.append(f"\tIPv6:\t{ipv6}\t{args.config.mail_domain}")
+    if len(to_print) > 0:
+        if len(to_print) == 1:
+            warning = "You should add the following PTR/reverse DNS entry:"
+        else:
+            warning = "You should add the following PTR/reverse DNS entries:"
+        out.red(warning)
+        for entry in to_print:
+            print(entry)
+        print(
+            "You can do so at your hosting provider (maybe this isn't your DNS provider)."
+        )
+
+
+def check_necessary_dns(out, mail_domain):
+    """Check whether $mail_domain and mta-sts.$mail_domain resolve."""
+    dns = DNS(out, mail_domain)
+    ipv4 = dns.get("A", mail_domain)
+    ipv6 = dns.get("AAAA", mail_domain)
+    mta_entry = dns.get("CNAME", "mta-sts." + mail_domain)
+    mta_ip = dns.get("A", mta_entry)
+    if not mta_ip:
+        mta_ip = dns.get("AAAA", mta_entry)
+    to_print = []
+    if not (ipv4 or ipv6):
+        to_print.append(f"\t{mail_domain}.\t\t\tA<your server's IPv4 address>")
+    if not mta_ip or not (mta_ip == ipv4 or mta_ip == ipv6):
+        to_print.append(f"\tmta-sts.{mail_domain}.\tCNAME\t{mail_domain}.")
+    if to_print:
+        to_print.insert(
+            0,
+            "\nFor chatmail to work, you need to configure this at your DNS provider:\n",
+        )
+        for line in to_print:
+            print(line)
+        print()
+    else:
+        dns.out.green("\nAll necessary DNS entries seem to be set.")
+        return True

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -48,12 +48,4 @@ http {
         # add cgi-bin support
         include /usr/share/doc/fcgiwrap/examples/nginx.conf;
 	}
-    server {
-        listen 80 default_server;
-        listen [::]:80 default_server;
-        server_name _;
-
-        return 301 https://$host$request_uri;
-    }
 }
-

cmdeploy/src/cmdeploy/tests/online/test_0_login.py

@@ -2,6 +2,16 @@ import pytest
 import threading
 import queue
 
+from chatmaild.config import read_config
+from cmdeploy.cmdeploy import main
+
+
+def test_init(tmp_path, maildomain):
+    inipath = tmp_path.joinpath("chatmail.ini")
+    main(["init", "--config", str(inipath), maildomain])
+    config = read_config(inipath)
+    assert config.mail_domain == maildomain
+
 
 def test_login_basic_functioning(imap_or_smtp, gencreds, lp):
     """Test a) that an initial login creates a user automatically

cmdeploy/src/cmdeploy/tests/test_cmdeploy.py

@@ -2,7 +2,6 @@ import os
 
 import pytest
 from cmdeploy.cmdeploy import get_parser, main
-from chatmaild.config import read_config
 
 
 @pytest.fixture(autouse=True)
@@ -21,12 +20,7 @@ class TestCmdline:
         run = parser.parse_args(["run"])
         assert init and run
 
-    def test_init(self, tmp_path):
+    @pytest.mark.xfail(reason="init doesn't exit anymore, check for CLI output instead")
-        main(["init", "chat.example.org"])
-        inipath = tmp_path.joinpath("chatmail.ini")
-        config = read_config(inipath)
-        assert config.mail_domain == "chat.example.org"
-
     def test_init_not_overwrite(self):
         main(["init", "chat.example.org"])
         with pytest.raises(SystemExit):