feat: create accounts without HTTP request

This is supported since chatmail core version 2.23.0.

.github/workflows/ci.yaml

@@ -15,7 +15,7 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: download filtermail
-        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.2.0/filtermail-x86_64-musl -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
+        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.3.0/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
       - name: run chatmaild tests 
         working-directory: chatmaild
         run: pipx run tox

.github/workflows/test-and-deploy-ipv4only.yaml

@@ -75,8 +75,7 @@ jobs:
           cmdeploy init staging-ipv4.testrun.org
           sed -i 's#disable_ipv6 = False#disable_ipv6 = True#' chatmail.ini
           sed -i 's/#\s*mtail_address/mtail_address/' chatmail.ini
-
-      - run: cmdeploy run --verbose --skip-dns-check
+          cmdeploy run --verbose --skip-dns-check
 
       - name: set DNS entries
         run: |

chatmaild/src/chatmaild/config.py

@@ -1,3 +1,4 @@
+import os
 from pathlib import Path
 
 import iniconfig
@@ -43,6 +44,8 @@ class Config:
         )
         self.mtail_address = params.get("mtail_address")
         self.disable_ipv6 = params.get("disable_ipv6", "false").lower() == "true"
+        self.addr_v4 = os.environ.get("CHATMAIL_ADDR_V4", "")
+        self.addr_v6 = os.environ.get("CHATMAIL_ADDR_V6", "")
         self.acme_email = params.get("acme_email", "")
         self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true"
         self.imap_compress = params.get("imap_compress", "false").lower() == "true"
@@ -57,6 +60,18 @@ class Config:
         self.privacy_pdo = params.get("privacy_pdo")
         self.privacy_supervisor = params.get("privacy_supervisor")
 
+        # TLS certificate management: derived from the domain name.
+        # Domains starting with "_" use self-signed certificates
+        # All other domains use ACME.
+        if self.mail_domain.startswith("_"):
+            self.tls_cert_mode = "self"
+            self.tls_cert_path = "/etc/ssl/certs/mailserver.pem"
+            self.tls_key_path = "/etc/ssl/private/mailserver.key"
+        else:
+            self.tls_cert_mode = "acme"
+            self.tls_cert_path = f"/var/lib/acme/live/{self.mail_domain}/fullchain"
+            self.tls_key_path = f"/var/lib/acme/live/{self.mail_domain}/privkey"
+
         # deprecated option
         mbdir = params.get("mailboxes_dir", f"/home/vmail/mail/{self.mail_domain}")
         self.mailboxes_dir = Path(mbdir.strip())

chatmaild/src/chatmaild/doveauth.py

@@ -1,11 +1,8 @@
 import json
 import logging
 import os
-import re
 import sys
 
-import filelock
-
 try:
     import crypt_r
 except ImportError:
@@ -16,7 +13,6 @@ from .dictproxy import DictProxy
 from .migrate_db import migrate_from_db_to_maildir
 
 NOCREATE_FILE = "/etc/chatmail-nocreate"
-VALID_LOCALPART_RE = re.compile(r"^[a-z0-9._-]+$")
 
 
 def encrypt_password(password: str):
@@ -56,10 +52,6 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
         )
         return False
 
-    if not VALID_LOCALPART_RE.match(localpart):
-        logging.warning("localpart %r contains invalid characters", localpart)
-        return False
-
     return True
 
 
@@ -148,13 +140,8 @@ class AuthDictProxy(DictProxy):
         if not is_allowed_to_create(self.config, addr, cleartext_password):
             return
 
-        lock = filelock.FileLock(str(user.password_path) + ".lock", timeout=5)
-        with lock:
-            userdata = user.get_userdb_dict()
-            if userdata:
-                return userdata
-            user.set_password(encrypt_password(cleartext_password))
-            print(f"Created address: {addr}", file=sys.stderr)
+        user.set_password(encrypt_password(cleartext_password))
+        print(f"Created address: {addr}", file=sys.stderr)
         return user.get_userdb_dict()
 
 

chatmaild/src/chatmaild/fsreport.py

@@ -68,7 +68,7 @@ class Report:
             for size in self.message_buckets:
                 for msg in mailbox.messages:
                     if msg.size >= size:
-                        if self.mdir and f"/{self.mdir}/" not in msg.path:
+                        if self.mdir and not msg.relpath.startswith(self.mdir):
                             continue
                         self.message_buckets[size] += msg.size
 

chatmaild/src/chatmaild/metadata.py

@@ -101,11 +101,7 @@ class MetadataDictProxy(DictProxy):
                 # Handle `GETMETADATA "" /shared/vendor/deltachat/irohrelay`
                 return f"O{self.iroh_relay}\n"
             elif keyname == "vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn":
-                try:
-                    res = turn_credentials()
-                except Exception:
-                    logging.exception("failed to get TURN credentials")
-                    return "N\n"
+                res = turn_credentials()
                 port = 3478
                 return f"O{self.turn_hostname}:{port}:{res}\n"
 

chatmaild/src/chatmaild/newemail.py

@@ -3,8 +3,10 @@
 """CGI script for creating new accounts."""
 
 import json
+import random
 import secrets
 import string
+from urllib.parse import quote
 
 from chatmaild.config import Config, read_config
 
@@ -14,9 +16,7 @@ ALPHANUMERIC_PUNCT = string.ascii_letters + string.digits + string.punctuation
 
 
 def create_newemail_dict(config: Config):
-    user = "".join(
-        secrets.choice(ALPHANUMERIC) for _ in range(config.username_max_length)
-    )
+    user = "".join(random.choices(ALPHANUMERIC, k=config.username_max_length))
     password = "".join(
         secrets.choice(ALPHANUMERIC_PUNCT)
         for _ in range(config.password_min_length + 3)
@@ -24,13 +24,26 @@ def create_newemail_dict(config: Config):
     return dict(email=f"{user}@{config.mail_domain}", password=f"{password}")
 
 
+def create_dclogin_url(email, password):
+    """Build a dclogin: URL with credentials and self-signed cert acceptance.
+
+    Uses ic=3 (AcceptInvalidCertificates) so chatmail clients
+    can connect to servers with self-signed TLS certificates.
+    """
+    return f"dclogin:{quote(email, safe='@')}?p={quote(password, safe='')}&v=1&ic=3"
+
+
 def print_new_account():
     config = read_config(CONFIG_PATH)
     creds = create_newemail_dict(config)
 
+    result = dict(email=creds["email"], password=creds["password"])
+    if config.tls_cert_mode == "self":
+        result["dclogin_url"] = create_dclogin_url(creds["email"], creds["password"])
+
     print("Content-Type: application/json")
     print("")
-    print(json.dumps(creds))
+    print(json.dumps(result))
 
 
 if __name__ == "__main__":

chatmaild/src/chatmaild/tests/test_config.py

@@ -73,3 +73,17 @@ def test_config_userstate_paths(make_config, tmp_path):
 def test_config_max_message_size(make_config, tmp_path):
     config = make_config("something.testrun.org", dict(max_message_size="10000"))
     assert config.max_message_size == 10000
+
+
+def test_config_tls_default_acme(make_config):
+    config = make_config("chat.example.org")
+    assert config.tls_cert_mode == "acme"
+    assert config.tls_cert_path == "/var/lib/acme/live/chat.example.org/fullchain"
+    assert config.tls_key_path == "/var/lib/acme/live/chat.example.org/privkey"
+
+
+def test_config_tls_self(make_config):
+    config = make_config("_test.example.org")
+    assert config.tls_cert_mode == "self"
+    assert config.tls_cert_path == "/etc/ssl/certs/mailserver.pem"
+    assert config.tls_key_path == "/etc/ssl/private/mailserver.key"

chatmaild/src/chatmaild/tests/test_doveauth.py

@@ -120,60 +120,6 @@ def test_handle_dovecot_protocol_iterate(gencreds, example_config):
     assert not lines[2]
 
 
-def test_invalid_localpart_characters(make_config):
-    """Test that is_allowed_to_create rejects localparts with invalid characters."""
-    config = make_config("chat.example.org", {"username_min_length": "3"})
-    password = "zequ0Aimuchoodaechik"
-    domain = config.mail_domain
-
-    # valid localparts
-    assert is_allowed_to_create(config, f"abc123@{domain}", password)
-    assert is_allowed_to_create(config, f"a.b-c_d@{domain}", password)
-
-    # uppercase rejected
-    assert not is_allowed_to_create(config, f"Abc123@{domain}", password)
-    assert not is_allowed_to_create(config, f"ABCDEFG@{domain}", password)
-
-    # spaces and special chars rejected
-    assert not is_allowed_to_create(config, f"a b cde@{domain}", password)
-    assert not is_allowed_to_create(config, f"abc+def@{domain}", password)
-    assert not is_allowed_to_create(config, f"abc!def@{domain}", password)
-    assert not is_allowed_to_create(config, f"ab@cdef@{domain}", password)
-    assert not is_allowed_to_create(config, f"abc/def@{domain}", password)
-    assert not is_allowed_to_create(config, f"abc\\def@{domain}", password)
-
-
-def test_concurrent_creation_same_account(dictproxy):
-    """Test that concurrent creation of the same account doesn't corrupt password."""
-    addr = "racetest1@chat.example.org"
-    password = "zequ0Aimuchoodaechik"
-    num_threads = 10
-    results = queue.Queue()
-
-    def create():
-        try:
-            res = dictproxy.lookup_passdb(addr, password)
-            results.put(("ok", res))
-        except Exception:
-            results.put(("err", traceback.format_exc()))
-
-    threads = [threading.Thread(target=create, daemon=True) for _ in range(num_threads)]
-    for t in threads:
-        t.start()
-    for t in threads:
-        t.join(timeout=10)
-
-    passwords_seen = set()
-    for _ in range(num_threads):
-        status, res = results.get()
-        if status == "err":
-            pytest.fail(f"concurrent creation failed\n{res}")
-        passwords_seen.add(res["password"])
-
-    # all threads must see the same password hash
-    assert len(passwords_seen) == 1
-
-
 def test_50_concurrent_lookups_different_accounts(gencreds, dictproxy):
     num_threads = 50
     req_per_thread = 5

chatmaild/src/chatmaild/tests/test_expire.py

@@ -112,43 +112,6 @@ def test_report(mbox1, example_config):
     report_main(args)
 
 
-def test_report_mdir_filters_by_path(mbox1, example_config):
-    """Test that Report with mdir='cur' only counts messages in cur/ subdirectory."""
-    from chatmaild.fsreport import Report
-
-    now = datetime.utcnow().timestamp()
-
-    # Set password mtime to old enough so min_login_age check passes
-    password = Path(mbox1.basedir).joinpath("password")
-    old_time = now - 86400 * 10  # 10 days ago
-    os.utime(password, (old_time, old_time))
-
-    # Reload mailbox with updated mtime
-    from chatmaild.expire import MailboxStat
-
-    mbox = MailboxStat(mbox1.basedir)
-
-    # Report without mdir — should count all messages
-    rep_all = Report(now=now, min_login_age=1, mdir=None)
-    rep_all.process_mailbox_stat(mbox)
-    total_all = rep_all.message_buckets[0]
-
-    # Report with mdir='cur' — should only count cur/ messages
-    rep_cur = Report(now=now, min_login_age=1, mdir="cur")
-    rep_cur.process_mailbox_stat(mbox)
-    total_cur = rep_cur.message_buckets[0]
-
-    # Report with mdir='new' — should only count new/ messages
-    rep_new = Report(now=now, min_login_age=1, mdir="new")
-    rep_new.process_mailbox_stat(mbox)
-    total_new = rep_new.message_buckets[0]
-
-    # cur has 500-byte msg, new has 600-byte msg (from fill_mbox)
-    assert total_cur == 500
-    assert total_new == 600
-    assert total_all == 500 + 600
-
-
 def test_expiry_cli_basic(example_config, mbox1):
     args = (str(example_config._inipath),)
     expiry_main(args)

chatmaild/src/chatmaild/tests/test_filtermail_blackbox.py

@@ -1,9 +1,15 @@
+import shutil
 import smtplib
 import subprocess
 import sys
 
 import pytest
 
+pytestmark = pytest.mark.skipif(
+    shutil.which("filtermail") is None,
+    reason="filtermail binary not found",
+)
+
 
 @pytest.fixture
 def smtpserver():

chatmaild/src/chatmaild/tests/test_metadata.py

@@ -314,51 +314,6 @@ def test_persistent_queue_items(tmp_path, testaddr, token):
     assert not queue_item < item2 and not item2 < queue_item
 
 
-def test_turn_credentials_exception_returns_N(notifier, metadata, monkeypatch):
-    """Test that turn_credentials() failure returns N\\n instead of crashing."""
-    import chatmaild.metadata
-
-    dictproxy = MetadataDictProxy(
-        notifier=notifier,
-        metadata=metadata,
-        turn_hostname="turn.example.org",
-    )
-
-    def mock_turn_credentials():
-        raise ConnectionRefusedError("socket not available")
-
-    monkeypatch.setattr(chatmaild.metadata, "turn_credentials", mock_turn_credentials)
-
-    transactions = {}
-    res = dictproxy.handle_dovecot_request(
-        "Lshared/0123/vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn"
-        "\tuser@example.org",
-        transactions,
-    )
-    assert res == "N\n"
-
-
-def test_turn_credentials_success(notifier, metadata, monkeypatch):
-    """Test that valid turn_credentials() returns TURN URI."""
-    import chatmaild.metadata
-
-    dictproxy = MetadataDictProxy(
-        notifier=notifier,
-        metadata=metadata,
-        turn_hostname="turn.example.org",
-    )
-
-    monkeypatch.setattr(chatmaild.metadata, "turn_credentials", lambda: "user:pass")
-
-    transactions = {}
-    res = dictproxy.handle_dovecot_request(
-        "Lshared/0123/vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn"
-        "\tuser@example.org",
-        transactions,
-    )
-    assert res == "Oturn.example.org:3478:user:pass\n"
-
-
 def test_iroh_relay(dictproxy):
     rfile = io.BytesIO(
         b"\n".join(

chatmaild/src/chatmaild/tests/test_newmail.py

@@ -1,7 +1,11 @@
 import json
 
 import chatmaild
-from chatmaild.newemail import create_newemail_dict, print_new_account
+from chatmaild.newemail import (
+    create_dclogin_url,
+    create_newemail_dict,
+    print_new_account,
+)
 
 
 def test_create_newemail_dict(example_config):
@@ -15,6 +19,18 @@ def test_create_newemail_dict(example_config):
     assert ac1["password"] != ac2["password"]
 
 
+def test_create_dclogin_url():
+    url = create_dclogin_url("user@example.org", "p@ss w+rd")
+    assert url.startswith("dclogin:")
+    assert "v=1" in url
+    assert "ic=3" in url
+
+    assert "user@example.org" in url
+    # password special chars must be encoded
+    assert "p%40ss" in url
+    assert "w%2Brd" in url
+
+
 def test_print_new_account(capsys, monkeypatch, maildomain, tmpdir, example_config):
     monkeypatch.setattr(chatmaild.newemail, "CONFIG_PATH", str(example_config._inipath))
     print_new_account()
@@ -25,3 +41,20 @@ def test_print_new_account(capsys, monkeypatch, maildomain, tmpdir, example_conf
     dic = json.loads(lines[2])
     assert dic["email"].endswith(f"@{example_config.mail_domain}")
     assert len(dic["password"]) >= 10
+    # default tls_cert=acme should not include dclogin_url
+    assert "dclogin_url" not in dic
+
+
+def test_print_new_account_self_signed(capsys, monkeypatch, make_config):
+    config = make_config("_test.example.org")
+    monkeypatch.setattr(chatmaild.newemail, "CONFIG_PATH", str(config._inipath))
+    print_new_account()
+    out, err = capsys.readouterr()
+    lines = out.split("\n")
+    dic = json.loads(lines[2])
+    assert "dclogin_url" in dic
+    url = dic["dclogin_url"]
+    assert url.startswith("dclogin:")
+    assert "ic=3" in url
+
+    assert dic["email"].split("@")[0] in url

chatmaild/src/chatmaild/tests/test_turnserver.py

@@ -1,73 +0,0 @@
-import socket
-import threading
-import time
-from unittest.mock import patch
-
-import pytest
-
-from chatmaild.turnserver import turn_credentials
-
-SOCKET_PATH = "/run/chatmail-turn/turn.socket"
-
-
-@pytest.fixture
-def turn_socket(tmp_path):
-    """Create a real Unix socket server at a temp path."""
-    sock_path = str(tmp_path / "turn.socket")
-    server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-    server.bind(sock_path)
-    server.listen(1)
-    yield sock_path, server
-    server.close()
-
-
-def _call_turn_credentials(sock_path):
-    """Call turn_credentials but connect to sock_path instead of hardcoded path."""
-    original_connect = socket.socket.connect
-
-    def patched_connect(self, address):
-        if address == SOCKET_PATH:
-            address = sock_path
-        return original_connect(self, address)
-
-    with patch.object(socket.socket, "connect", patched_connect):
-        return turn_credentials()
-
-
-def test_turn_credentials_timeout(turn_socket):
-    """Server accepts but never responds — must raise socket.timeout."""
-    sock_path, server = turn_socket
-
-    def accept_and_hang():
-        conn, _ = server.accept()
-        time.sleep(30)
-        conn.close()
-
-    t = threading.Thread(target=accept_and_hang, daemon=True)
-    t.start()
-
-    with pytest.raises(socket.timeout):
-        _call_turn_credentials(sock_path)
-
-
-def test_turn_credentials_connection_refused(tmp_path):
-    """Socket file doesn't exist — must raise ConnectionRefusedError or FileNotFoundError."""
-    missing = str(tmp_path / "nonexistent.socket")
-    with pytest.raises((ConnectionRefusedError, FileNotFoundError)):
-        _call_turn_credentials(missing)
-
-
-def test_turn_credentials_success(turn_socket):
-    """Server responds with credentials — must return stripped string."""
-    sock_path, server = turn_socket
-
-    def respond():
-        conn, _ = server.accept()
-        conn.sendall(b"testuser:testpass\n")
-        conn.close()
-
-    t = threading.Thread(target=respond, daemon=True)
-    t.start()
-
-    result = _call_turn_credentials(sock_path)
-    assert result == "testuser:testpass"

chatmaild/src/chatmaild/turnserver.py

@@ -4,7 +4,6 @@ import socket
 
 def turn_credentials() -> str:
     with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client_socket:
-        client_socket.settimeout(5)
         client_socket.connect("/run/chatmail-turn/turn.socket")
         with client_socket.makefile("rb") as file:
             return file.readline().decode("utf-8").strip()

cmdeploy/src/cmdeploy/acmetool/__init__.py

@@ -67,7 +67,7 @@ class AcmetoolDeployer(Deployer):
         )
         files.template(
             src=importlib.resources.files(__package__).joinpath("desired.yaml.j2"),
-            dest=f"/var/lib/acme/desired/{self.domains[0]}",  # 0 is mailhost TLD
+            dest=f"/var/lib/acme/desired/{self.domains[0]}", # 0 is mailhost TLD
             user="root",
             group="root",
             mode="644",

cmdeploy/src/cmdeploy/chatmail.zone.j2

@@ -8,8 +8,10 @@
 {{ mail_domain }}.                   AAAA  {{ AAAA }}
 {% endif %}
 {{ mail_domain }}.                   MX 10 {{ mail_domain }}.
+{% if strict_tls %}
 _mta-sts.{{ mail_domain }}.          TXT "v=STSv1; id={{ sts_id }}"
 mta-sts.{{ mail_domain }}.           CNAME {{ mail_domain }}.
+{% endif %}
 www.{{ mail_domain }}.               CNAME {{ mail_domain }}.
 {{ dkim_entry }}
 

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -91,9 +91,10 @@ def run_cmd(args, out):
     ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
     sshexec = get_sshexec(ssh_host)
     require_iroh = args.config.enable_iroh_relay
+    strict_tls = args.config.tls_cert_mode == "acme"
     if not args.dns_check_disabled:
         remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
-        if not dns.check_initial_remote_data(remote_data, print=out.red):
+        if not dns.check_initial_remote_data(remote_data, strict_tls=strict_tls, print=out.red):
             return 1
 
     env = os.environ.copy()
@@ -101,6 +102,9 @@ def run_cmd(args, out):
     env["CHATMAIL_WEBSITE_ONLY"] = "True" if args.website_only else ""
     env["CHATMAIL_DISABLE_MAIL"] = "True" if args.disable_mail else ""
     env["CHATMAIL_REQUIRE_IROH"] = "True" if require_iroh else ""
+    if not args.dns_check_disabled:
+        env["CHATMAIL_ADDR_V4"] = remote_data.get("A") or ""
+        env["CHATMAIL_ADDR_V6"] = remote_data.get("AAAA") or ""
     deploy_path = importlib.resources.files(__package__).joinpath("run.py").resolve()
     pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
 
@@ -113,15 +117,24 @@ def run_cmd(args, out):
         return 1
 
     try:
-        out.check_call(cmd, env=env)
+        retcode = out.check_call(cmd, env=env)
         if args.website_only:
-            out.green("Website deployment completed.")
-        else:
+            if retcode == 0:
+                out.green("Website deployment completed.")
+            else:
+                out.red("Website deployment failed.")
+        elif retcode == 0:
             out.green("Deploy completed, call `cmdeploy dns` next.")
-        return 0
+        elif not args.dns_check_disabled and strict_tls and not remote_data["acme_account_url"]:
+            out.red("Deploy completed but letsencrypt not configured")
+            out.red("Run 'cmdeploy run' again")
+            retcode = 0
+        else:
+            out.red("Deploy failed")
     except subprocess.CalledProcessError:
         out.red("Deploy failed")
-        return 1
+        retcode = 1
+    return retcode
 
 
 def dns_cmd_options(parser):
@@ -139,11 +152,13 @@ def dns_cmd(args, out):
     """Check DNS entries and optionally generate dns zone file."""
     ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
     sshexec = get_sshexec(ssh_host, verbose=args.verbose)
+    tls_cert_mode = args.config.tls_cert_mode
+    strict_tls = tls_cert_mode == "acme"
     remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
-    if not remote_data:
+    if not dns.check_initial_remote_data(remote_data, strict_tls=strict_tls):
         return 1
 
-    if not remote_data["acme_account_url"]:
+    if strict_tls and not remote_data["acme_account_url"]:
         out.red("could not get letsencrypt account url, please run 'cmdeploy run'")
         return 1
 
@@ -151,6 +166,7 @@ def dns_cmd(args, out):
         out.red("could not determine dkim_entry, please run 'cmdeploy run'")
         return 1
 
+    remote_data["strict_tls"] = strict_tls
     zonefile = dns.get_filled_zone_file(remote_data)
 
     if args.zonefile:

cmdeploy/src/cmdeploy/deployers.py

@@ -10,6 +10,7 @@ from pathlib import Path
 
 from chatmaild.config import read_config
 from pyinfra import facts, host, logger
+from pyinfra.facts import hardware
 from pyinfra.api import FactBase
 from pyinfra.facts.files import Sha256File
 from pyinfra.facts.systemd import SystemdEnabled
@@ -18,6 +19,7 @@ from pyinfra.operations import apt, files, pip, server, systemd
 from cmdeploy.cmdeploy import Out
 
 from .acmetool import AcmetoolDeployer
+from .selfsigned.deployer import SelfSignedTlsDeployer
 from .basedeploy import (
     Deployer,
     Deployment,
@@ -36,7 +38,7 @@ from .www import build_webpages, find_merge_conflict, get_paths
 
 class Port(FactBase):
     """
-    Returns the process occuping a port.
+    Returns the process occupying a port.
     """
 
     def command(self, port: int) -> str:
@@ -264,9 +266,6 @@ class WebsiteDeployer(Deployer):
             # if www_folder is a hugo page, build it
             if build_dir:
                 www_path = build_webpages(src_dir, build_dir, self.config)
-                if www_path is None:
-                    logger.warning("Web page build failed, skipping website deployment")
-                    return
             # if it is not a hugo page, upload it as is
             files.rsync(
                 f"{www_path}/", "/var/www/html", flags=["-avz", "--chown=www-data"]
@@ -560,10 +559,21 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
             line="\nnameserver 9.9.9.9",
         )
 
+    # Check if mtail_address interface is available (if configured)
+    if config.mtail_address and config.mtail_address not in ('127.0.0.1', '::1', 'localhost'):
+        ipv4_addrs = host.get_fact(hardware.Ipv4Addrs)
+        all_addresses = [addr for addrs in ipv4_addrs.values() for addr in addrs]
+        if config.mtail_address not in all_addresses:
+            Out().red(f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n")
+            exit(1)
+
     port_services = [
         (["master", "smtpd"], 25),
         ("unbound", 53),
-        ("acmetool", 80),
+    ]
+    if config.tls_cert_mode == "acme":
+        port_services.append(("acmetool", 80))
+    port_services += [
         (["imap-login", "dovecot"], 143),
         ("nginx", 443),
         (["master", "smtpd"], 465),
@@ -571,7 +581,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
         (["imap-login", "dovecot"], 993),
         ("iroh-relay", 3340),
         ("mtail", 3903),
-        ("dovecot-stats", 3904),
+        ("stats", 3904),
         ("nginx", 8443),
         (["master", "smtpd"], config.postfix_reinject_port),
         (["master", "smtpd"], config.postfix_reinject_port_incoming),
@@ -581,8 +591,9 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
     for service, port in port_services:
         print(f"Checking if port {port} is available for {service}...")
         running_service = host.get_fact(Port, port=port)
+        services = [service] if isinstance(service, str) else service
         if running_service:
-            if running_service not in service:
+            if running_service not in services:
                 Out().red(
                     f"Deploy failed: port {port} is occupied by: {running_service}"
                 )
@@ -590,6 +601,11 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
 
     tls_domains = [mail_domain, f"mta-sts.{mail_domain}", f"www.{mail_domain}"]
 
+    if config.tls_cert_mode == "acme":
+        tls_deployer = AcmetoolDeployer(config.acme_email, tls_domains)
+    else:
+        tls_deployer = SelfSignedTlsDeployer(mail_domain)
+
     all_deployers = [
         ChatmailDeployer(mail_domain),
         LegacyRemoveDeployer(),
@@ -598,7 +614,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
         UnboundDeployer(config),
         TurnDeployer(mail_domain),
         IrohDeployer(config.enable_iroh_relay),
-        AcmetoolDeployer(config.acme_email, tls_domains),
+        tls_deployer,
         WebsiteDeployer(config),
         ChatmailVenvDeployer(config),
         MtastsDeployer(),

cmdeploy/src/cmdeploy/dns.py

@@ -12,14 +12,14 @@ def get_initial_remote_data(sshexec, mail_domain):
     )
 
 
-def check_initial_remote_data(remote_data, *, print=print):
+def check_initial_remote_data(remote_data, *, strict_tls=True, print=print):
     mail_domain = remote_data["mail_domain"]
     if not remote_data["A"] and not remote_data["AAAA"]:
         print(f"Missing A and/or AAAA DNS records for {mail_domain}!")
-    elif remote_data["MTA_STS"] != f"{mail_domain}.":
+    elif strict_tls and remote_data["MTA_STS"] != f"{mail_domain}.":
         print("Missing MTA-STS CNAME record:")
         print(f"mta-sts.{mail_domain}.   CNAME  {mail_domain}.")
-    elif remote_data["WWW"] != f"{mail_domain}.":
+    elif strict_tls and remote_data["WWW"] != f"{mail_domain}.":
         print("Missing www CNAME record:")
         print(f"www.{mail_domain}.   CNAME  {mail_domain}.")
     else:

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -37,9 +37,7 @@ class DovecotDeployer(Deployer):
         restart = False if self.disable_mail else self.need_restart
 
         systemd.service(
-            name="Disable dovecot for now"
-            if self.disable_mail
-            else "Start and enable Dovecot",
+            name="Disable dovecot for now" if self.disable_mail else "Start and enable Dovecot",
             service="dovecot.service",
             running=False if self.disable_mail else True,
             enabled=False if self.disable_mail else True,

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -228,8 +228,8 @@ service anvil {
 }
 
 ssl = required
-ssl_cert = </var/lib/acme/live/{{ config.mail_domain }}/fullchain
-ssl_key = </var/lib/acme/live/{{ config.mail_domain }}/privkey
+ssl_cert = <{{ config.tls_cert_path }}
+ssl_key = <{{ config.tls_key_path }}
 ssl_dh = </usr/share/dovecot/dh.pem
 ssl_min_protocol = TLSv1.3
 ssl_prefer_server_ciphers = yes

cmdeploy/src/cmdeploy/filtermail/deployer.py

@@ -14,10 +14,10 @@ class FiltermailDeployer(Deployer):
 
     def install(self):
         arch = host.get_fact(facts.server.Arch)
-        url = f"https://github.com/chatmail/filtermail/releases/download/v0.2.0/filtermail-{arch}-musl"
+        url = f"https://github.com/chatmail/filtermail/releases/download/v0.3.0/filtermail-{arch}"
         sha256sum = {
-            "x86_64": "1e5bbb646582cb16740c6dfbbca39edba492b78cc96ec9fa2528c612bb504edd",
-            "aarch64": "3564fba8605f8f9adfeefff3f4580533205da043f47c5968d0d10db17e50f44e",
+            "x86_64": "f14a31323ae2dad3b59d3fdafcde507521da2f951a9478cd1f2fe2b4463df71d",
+            "aarch64": "933770d75046c4fd7084ce8d43f905f8748333426ad839154f0fc654755ef09f",
         }[arch]
         self.need_restart |= files.download(
             name="Download filtermail",

cmdeploy/src/cmdeploy/genqr.py

@@ -7,7 +7,7 @@ from PIL import Image, ImageDraw, ImageFont
 
 
 def gen_qr_png_data(maildomain):
-    url = f"DCACCOUNT:https://{maildomain}/new"
+    url = f"DCACCOUNT:{maildomain}"
     image = gen_qr(maildomain, url)
     temp = io.BytesIO()
     image.save(temp, format="png")

cmdeploy/src/cmdeploy/nginx/autoconfig.xml.j2

@@ -1,47 +1,47 @@
 <?xml version="1.0" encoding="UTF-8"?>
 
 <clientConfig version="1.1">
-  <emailProvider id="{{ config.domain_name }}">
-    <domain>{{ config.domain_name }}</domain>
-    <displayName>{{ config.domain_name }} chatmail</displayName>
-    <displayShortName>{{ config.domain_name }}</displayShortName>
+  <emailProvider id="{{ config.mail_domain }}">
+    <domain>{{ config.mail_domain }}</domain>
+    <displayName>{{ config.mail_domain }} chatmail</displayName>
+    <displayShortName>{{ config.mail_domain }}</displayShortName>
     <incomingServer type="imap">
-      <hostname>{{ config.domain_name }}</hostname>
+      <hostname>{{ config.mail_domain }}</hostname>
       <port>993</port>
       <socketType>SSL</socketType>
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
     </incomingServer>
     <incomingServer type="imap">
-      <hostname>{{ config.domain_name }}</hostname>
+      <hostname>{{ config.mail_domain }}</hostname>
       <port>143</port>
       <socketType>STARTTLS</socketType>
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
     </incomingServer>
     <incomingServer type="imap">
-      <hostname>{{ config.domain_name }}</hostname>
+      <hostname>{{ config.mail_domain }}</hostname>
       <port>443</port>
       <socketType>SSL</socketType>
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
     </incomingServer>
     <outgoingServer type="smtp">
-      <hostname>{{ config.domain_name }}</hostname>
+      <hostname>{{ config.mail_domain }}</hostname>
       <port>465</port>
       <socketType>SSL</socketType>
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
     </outgoingServer>
     <outgoingServer type="smtp">
-      <hostname>{{ config.domain_name }}</hostname>
+      <hostname>{{ config.mail_domain }}</hostname>
       <port>587</port>
       <socketType>STARTTLS</socketType>
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
     </outgoingServer>
     <outgoingServer type="smtp">
-      <hostname>{{ config.domain_name }}</hostname>
+      <hostname>{{ config.mail_domain }}</hostname>
       <port>443</port>
       <socketType>SSL</socketType>
       <authentication>password-cleartext</authentication>

cmdeploy/src/cmdeploy/nginx/deployer.py

@@ -70,7 +70,7 @@ def _configure_nginx(config: Config, debug: bool = False) -> bool:
         user="root",
         group="root",
         mode="644",
-        config={"domain_name": config.mail_domain},
+        config=config,
         disable_ipv6=config.disable_ipv6,
     )
     need_restart |= main_config.changed
@@ -81,7 +81,7 @@ def _configure_nginx(config: Config, debug: bool = False) -> bool:
         user="root",
         group="root",
         mode="644",
-        config={"domain_name": config.mail_domain},
+        config=config,
     )
     need_restart |= autoconfig.changed
 
@@ -91,7 +91,7 @@ def _configure_nginx(config: Config, debug: bool = False) -> bool:
         user="root",
         group="root",
         mode="644",
-        config={"domain_name": config.mail_domain},
+        config=config,
     )
     need_restart |= mta_sts_config.changed
 

cmdeploy/src/cmdeploy/nginx/mta-sts.txt.j2

@@ -1,4 +1,4 @@
 version: STSv1
 mode: enforce
-mx: {{ config.domain_name }}
+mx: {{ config.mail_domain }}
 max_age: 2419200

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -42,6 +42,9 @@ stream {
 }
 
 http {
+{% if config.tls_cert_mode == "self" %}
+	limit_req_zone $binary_remote_addr zone=newaccount:10m rate=2r/s;
+{% endif %}
 	sendfile on;
 	tcp_nopush on;
 
@@ -51,10 +54,10 @@ http {
 	include /etc/nginx/mime.types;
 	default_type application/octet-stream;
 
-	ssl_protocols TLSv1.2 TLSv1.3;
+	ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
 	ssl_prefer_server_ciphers on;
-	ssl_certificate /var/lib/acme/live/{{ config.domain_name }}/fullchain;
-	ssl_certificate_key /var/lib/acme/live/{{ config.domain_name }}/privkey;
+	ssl_certificate {{ config.tls_cert_path }};
+	ssl_certificate_key {{ config.tls_key_path }};
 
 	gzip on;
 
@@ -66,7 +69,7 @@ http {
 
 		index index.html index.htm;
 
-		server_name {{ config.domain_name }} www.{{ config.domain_name }} mta-sts.{{ config.domain_name }};
+		server_name {{ config.mail_domain }} www.{{ config.mail_domain }} mta-sts.{{ config.mail_domain }};
 
 		access_log syslog:server=unix:/dev/log,facility=local7;
 
@@ -81,11 +84,15 @@ http {
 		}
 
 		location /new {
+{% if config.tls_cert_mode == "acme" %}
 			if ($request_method = GET) {
 				# Redirect to Delta Chat,
 				# which will in turn do a POST request.
-				return 301 dcaccount:https://{{ config.domain_name }}/new;
+				return 301 dcaccount:https://{{ config.mail_domain }}/new;
 			}
+{% else %}
+			limit_req zone=newaccount burst=5 nodelay;
+{% endif %}
 
 			fastcgi_pass unix:/run/fcgiwrap.socket;
 			include /etc/nginx/fastcgi_params;
@@ -99,9 +106,11 @@ http {
 		#
 		# Redirects are only for browsers.
 		location /cgi-bin/newemail.py {
+{% if config.tls_cert_mode == "acme" %}
 			if ($request_method = GET) {
-				return 301 dcaccount:https://{{ config.domain_name }}/new;
+				return 301 dcaccount:https://{{ config.mail_domain }}/new;
 			}
+{% endif %}
 
 			fastcgi_pass unix:/run/fcgiwrap.socket;
 			include /etc/nginx/fastcgi_params;
@@ -132,8 +141,8 @@ http {
 	# Redirect www. to non-www
 	server {
 		listen 127.0.0.1:8443 ssl;
-		server_name www.{{ config.domain_name }};
-		return 301 $scheme://{{ config.domain_name }}$request_uri;
+		server_name www.{{ config.mail_domain }};
+		return 301 $scheme://{{ config.mail_domain }}$request_uri;
 		access_log syslog:server=unix:/dev/log,facility=local7;
 	}
 }

cmdeploy/src/cmdeploy/postfix/deployer.py

@@ -61,6 +61,20 @@ class PostfixDeployer(Deployer):
         )
         need_restart |= lmtp_header_cleanup.changed
 
+        tls_policy_map = files.put(
+            name="Upload SMTP TLS Policy that accepts self-signed certificates for IP-only hosts",
+            src=get_resource("postfix/smtp_tls_policy_map"),
+            dest="/etc/postfix/smtp_tls_policy_map",
+            user="root",
+            group="root",
+            mode="644",
+        )
+        need_restart |= tls_policy_map.changed
+        if tls_policy_map.changed:
+            server.shell(
+                commands=["postmap /etc/postfix/smtp_tls_policy_map"],
+            )
+
         # Login map that 1:1 maps email address to login.
         login_map = files.put(
             src=get_resource("postfix/login_map"),
@@ -83,9 +97,7 @@ class PostfixDeployer(Deployer):
             server.shell(
                 name="Validate postfix configuration",
                 # Extract stderr and quit with error if non-zero
-                commands=[
-                    """bash -c 'w=$(postconf 2>&1 >/dev/null); [[ -z "$w" ]] || { echo "$w"; false; }'"""
-                ],
+                commands=["""bash -c 'w=$(postconf 2>&1 >/dev/null); [[ -z "$w" ]] || { echo "$w"; false; }'"""],
             )
         self.need_restart = need_restart
 

cmdeploy/src/cmdeploy/postfix/lmtp_header_cleanup

@@ -1,2 +1,3 @@
 /^DKIM-Signature:/            IGNORE
 /^Authentication-Results:/            IGNORE
+/^Received:/            IGNORE

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -15,17 +15,17 @@ readme_directory = no
 compatibility_level = 3.6
 
 # TLS parameters
-smtpd_tls_cert_file=/var/lib/acme/live/{{ config.mail_domain }}/fullchain
-smtpd_tls_key_file=/var/lib/acme/live/{{ config.mail_domain }}/privkey
+smtpd_tls_cert_file={{ config.tls_cert_path }}
+smtpd_tls_key_file={{ config.tls_key_path }}
 smtpd_tls_security_level=may
 
 smtp_tls_CApath=/etc/ssl/certs
-smtp_tls_security_level=verify
+smtp_tls_security_level={{ "verify" if config.tls_cert_mode == "acme" else "encrypt" }}
 # Send SNI extension when connecting to other servers.
 # <https://www.postfix.org/postconf.5.html#smtp_tls_servername>
 smtp_tls_servername = hostname
 smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
-smtp_tls_policy_maps = inline:{nauta.cu=may}
+smtp_tls_policy_maps = regexp:/etc/postfix/smtp_tls_policy_map
 smtp_tls_protocols = >=TLSv1.2
 smtp_tls_mandatory_protocols = >=TLSv1.2
 
@@ -69,6 +69,15 @@ mynetworks = 127.0.0.0/8
 {% else %}
 mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128
 {% endif %}
+{% if config.addr_v4 %}
+smtp_bind_address = {{ config.addr_v4 }}
+{% endif %}
+{% if config.addr_v6 %}
+smtp_bind_address6 = {{ config.addr_v6 }}
+{% endif %}
+{% if config.addr_v4 or config.addr_v6 %}
+smtp_bind_address_enforce = yes
+{% endif %}
 mailbox_size_limit = 0
 message_size_limit = {{config.max_message_size}}
 recipient_delimiter = +

cmdeploy/src/cmdeploy/postfix/smtp_tls_policy_map

@@ -0,0 +1,3 @@
+/^\[[^]]+\]$/ encrypt
+/^_/          encrypt
+/^nauta\.cu$/    may

cmdeploy/src/cmdeploy/remote/rdns.py

@@ -53,7 +53,7 @@ def get_dkim_entry(mail_domain, pre_command, dkim_selector):
             print=log_progress,
         )
     except CalledProcessError:
-        return None, None
+        return
     dkim_value_raw = f"v=DKIM1;k=rsa;p={dkim_pubkey};s=email;t=s"
     dkim_value = '" "'.join(re.findall(".{1,255}", dkim_value_raw))
     web_dkim_value = "".join(re.findall(".{1,255}", dkim_value_raw))

cmdeploy/src/cmdeploy/remote/rshell.py

@@ -40,5 +40,5 @@ def dovecot_recalc_quota(user):
     #
     for line in output.split("\n"):
         parts = line.split()
-        if len(parts) >= 6 and parts[2] == "STORAGE":
+        if parts[2] == "STORAGE":
             return dict(value=int(parts[3]), limit=int(parts[4]), percent=int(parts[5]))

cmdeploy/src/cmdeploy/selfsigned/deployer.py

@@ -0,0 +1,36 @@
+from pyinfra.operations import apt, files, server
+
+from cmdeploy.basedeploy import Deployer
+
+
+class SelfSignedTlsDeployer(Deployer):
+    """Generates a self-signed TLS certificate for all chatmail endpoints."""
+
+    def __init__(self, mail_domain):
+        self.mail_domain = mail_domain
+        self.cert_path = "/etc/ssl/certs/mailserver.pem"
+        self.key_path = "/etc/ssl/private/mailserver.key"
+
+    def install(self):
+        apt.packages(
+            name="Install openssl",
+            packages=["openssl"],
+        )
+
+    def configure(self):
+        server.shell(
+            name="Generate self-signed TLS certificate if not present",
+            commands=[
+                f"[ -f {self.cert_path} ] || openssl req -x509"
+                f" -newkey ec -pkeyopt ec_paramgen_curve:P-256"
+                f" -noenc -days 36500"
+                f" -keyout {self.key_path}"
+                f" -out {self.cert_path}"
+                f' -subj "/CN={self.mail_domain}"'
+                f' -addext "extendedKeyUsage=serverAuth,clientAuth"'
+                f' -addext "subjectAltName=DNS:{self.mail_domain},DNS:www.{self.mail_domain},DNS:mta-sts.{self.mail_domain}"',
+            ],
+        )
+
+    def activate(self):
+        pass

cmdeploy/src/cmdeploy/tests/online/test_0_qr.py

@@ -1,3 +1,4 @@
+import pytest
 import requests
 
 from cmdeploy.genqr import gen_qr_png_data
@@ -8,18 +9,33 @@ def test_gen_qr_png_data(maildomain):
     assert data
 
 
+@pytest.mark.filterwarnings("ignore::urllib3.exceptions.InsecureRequestWarning")
 def test_fastcgi_working(maildomain, chatmail_config):
     url = f"https://{maildomain}/new"
     print(url)
-    res = requests.post(url)
+    verify = chatmail_config.tls_cert_mode == "acme"
+    res = requests.post(url, verify=verify)
     assert maildomain in res.json().get("email")
     assert len(res.json().get("password")) > chatmail_config.password_min_length
 
 
-def test_newemail_configure(maildomain, rpc):
+@pytest.mark.filterwarnings("ignore::urllib3.exceptions.InsecureRequestWarning")
+def test_newemail_configure(maildomain, rpc, chatmail_config):
     """Test configuring accounts by scanning a QR code works."""
-    url = f"DCACCOUNT:https://{maildomain}/new"
+    url = f"DCACCOUNT:{maildomain}"
     for i in range(3):
         account_id = rpc.add_account()
-        rpc.set_config_from_qr(account_id, url)
-        rpc.configure(account_id)
+        if chatmail_config.tls_cert_mode == "self":
+            # deltachat core's rustls rejects self-signed HTTPS certs during
+            # set_config_from_qr, so fetch credentials via requests instead
+            res = requests.post(f"https://{maildomain}/new", verify=False)
+            data = res.json()
+            rpc.add_or_update_transport(account_id, {
+                "addr": data["email"],
+                "password": data["password"],
+                "imapServer": maildomain,
+                "smtpServer": maildomain,
+                "certificateChecks": "acceptInvalidCertificates",
+            })
+        else:
+            rpc.add_transport_from_qr(account_id, url)

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -11,11 +11,12 @@ from cmdeploy.sshexec import SSHExec
 
 
 @pytest.fixture
-def imap_mailbox(cmfactory):
+def imap_mailbox(cmfactory, ssl_context):
     (ac1,) = cmfactory.get_online_accounts(1)
     user = ac1.get_config("addr")
     password = ac1.get_config("mail_pw")
-    mailbox = imap_tools.MailBox(user.split("@")[1])
+    host = user.split("@")[1]
+    mailbox = imap_tools.MailBox(host, ssl_context=ssl_context)
     mailbox.login(user, password)
     mailbox.dc_ac = ac1
     return mailbox
@@ -171,7 +172,7 @@ class TestEndToEndDeltaChat:
             time.sleep(1)
 
 
-def test_hide_senders_ip_address(cmfactory):
+def test_hide_senders_ip_address(cmfactory, ssl_context):
     public_ip = requests.get("http://icanhazip.com").content.decode().strip()
     assert ipaddress.ip_address(public_ip)
 
@@ -180,6 +181,11 @@ def test_hide_senders_ip_address(cmfactory):
 
     chat.send_text("testing submission header cleanup")
     user2._evtracker.wait_next_incoming_message()
-    user2.direct_imap.select_folder("Inbox")
-    msg = user2.direct_imap.get_all_messages()[0]
-    assert public_ip not in msg.obj.as_string()
+    addr = user2.get_config("addr")
+    host = addr.split("@")[1]
+    pw = user2.get_config("mail_pw")
+    mailbox = imap_tools.MailBox(host, ssl_context=ssl_context)
+    mailbox.login(addr, pw)
+    msgs = list(mailbox.fetch(mark_seen=False))
+    assert msgs, "expected at least one message"
+    assert public_ip not in msgs[0].obj.as_string()

cmdeploy/src/cmdeploy/tests/plugin.py

@@ -4,6 +4,7 @@ import itertools
 import os
 import random
 import smtplib
+import ssl
 import subprocess
 import time
 from pathlib import Path
@@ -144,15 +145,25 @@ def pytest_terminal_summary(terminalreporter):
             tr.write_line(line)
 
 
-@pytest.fixture
-def imap(maildomain):
-    return ImapConn(maildomain)
+@pytest.fixture(scope="session")
+def ssl_context(chatmail_config):
+    if chatmail_config.tls_cert_mode == "self":
+        ctx = ssl.create_default_context()
+        ctx.check_hostname = False
+        ctx.verify_mode = ssl.CERT_NONE
+        return ctx
+    return None
 
 
 @pytest.fixture
-def make_imap_connection(maildomain):
+def imap(maildomain, ssl_context):
+    return ImapConn(maildomain, ssl_context=ssl_context)
+
+
+@pytest.fixture
+def make_imap_connection(maildomain, ssl_context):
     def make_imap_connection():
-        conn = ImapConn(maildomain)
+        conn = ImapConn(maildomain, ssl_context=ssl_context)
         conn.connect()
         return conn
 
@@ -164,12 +175,13 @@ class ImapConn:
     logcmd = "journalctl -f -u dovecot"
     name = "dovecot"
 
-    def __init__(self, host):
+    def __init__(self, host, ssl_context=None):
         self.host = host
+        self.ssl_context = ssl_context
 
     def connect(self):
         print(f"imap-connect {self.host}")
-        self.conn = imaplib.IMAP4_SSL(self.host)
+        self.conn = imaplib.IMAP4_SSL(self.host, ssl_context=self.ssl_context)
 
     def login(self, user, password):
         print(f"imap-login {user!r} {password!r}")
@@ -195,14 +207,14 @@ class ImapConn:
 
 
 @pytest.fixture
-def smtp(maildomain):
-    return SmtpConn(maildomain)
+def smtp(maildomain, ssl_context):
+    return SmtpConn(maildomain, ssl_context=ssl_context)
 
 
 @pytest.fixture
-def make_smtp_connection(maildomain):
+def make_smtp_connection(maildomain, ssl_context):
     def make_smtp_connection():
-        conn = SmtpConn(maildomain)
+        conn = SmtpConn(maildomain, ssl_context=ssl_context)
         conn.connect()
         return conn
 
@@ -214,12 +226,14 @@ class SmtpConn:
     logcmd = "journalctl -f -t postfix/smtpd -t postfix/smtp -t postfix/lmtp"
     name = "postfix"
 
-    def __init__(self, host):
+    def __init__(self, host, ssl_context=None):
         self.host = host
+        self.ssl_context = ssl_context
 
     def connect(self):
         print(f"smtp-connect {self.host}")
-        self.conn = smtplib.SMTP_SSL(self.host)
+        context = self.ssl_context or ssl.create_default_context()
+        self.conn = smtplib.SMTP_SSL(self.host, context=context)
 
     def login(self, user, password):
         print(f"smtp-login {user!r} {password!r}")
@@ -270,11 +284,12 @@ def gencreds(chatmail_config):
 class ChatmailTestProcess:
     """Provider for chatmail instance accounts as used by deltachat.testplugin.acfactory"""
 
-    def __init__(self, pytestconfig, maildomain, gencreds):
+    def __init__(self, pytestconfig, maildomain, gencreds, chatmail_config):
         self.pytestconfig = pytestconfig
         self.maildomain = maildomain
         assert "." in self.maildomain, maildomain
         self.gencreds = gencreds
+        self.chatmail_config = chatmail_config
         self._addr2files = {}
 
     def get_liveconfig_producer(self):
@@ -287,6 +302,9 @@ class ChatmailTestProcess:
             # speed up account configuration
             config["mail_server"] = self.maildomain
             config["send_server"] = self.maildomain
+            if self.chatmail_config.tls_cert_mode == "self":
+                # Accept self-signed TLS certificates
+                config["imap_certificate_checks"] = "3"
             yield config
 
     def cache_maybe_retrieve_configured_db_files(self, cache_addr, db_target_path):
@@ -297,12 +315,14 @@ class ChatmailTestProcess:
 
 
 @pytest.fixture
-def cmfactory(request, gencreds, tmpdir, maildomain):
+def cmfactory(request, gencreds, tmpdir, maildomain, chatmail_config):
     # cloned from deltachat.testplugin.amfactory
     pytest.importorskip("deltachat")
     from deltachat.testplugin import ACFactory
 
-    testproc = ChatmailTestProcess(request.config, maildomain, gencreds)
+    testproc = ChatmailTestProcess(
+        request.config, maildomain, gencreds, chatmail_config
+    )
 
     class Data:
         def read_path(self, path):
@@ -310,6 +330,10 @@ def cmfactory(request, gencreds, tmpdir, maildomain):
 
     am = ACFactory(request=request, tmpdir=tmpdir, testprocess=testproc, data=Data())
 
+    # Skip upstream's init_imap to prevent extra imap connections not
+    # needed for relay testing
+    am._acsetup.init_imap = lambda acc: None
+
     # nb. a bit hacky
     # would probably be better if deltachat's test machinery grows native support
     def switch_maildomain(maildomain2):
@@ -363,38 +387,40 @@ def lp(request):
 
 
 @pytest.fixture
-def cmsetup(maildomain, gencreds):
-    return CMSetup(maildomain, gencreds)
+def cmsetup(maildomain, gencreds, ssl_context):
+    return CMSetup(maildomain, gencreds, ssl_context)
 
 
 class CMSetup:
-    def __init__(self, maildomain, gencreds):
+    def __init__(self, maildomain, gencreds, ssl_context):
         self.maildomain = maildomain
         self.gencreds = gencreds
+        self.ssl_context = ssl_context
 
     def gen_users(self, num):
         print(f"Creating {num} online users")
         users = []
         for i in range(num):
             addr, password = self.gencreds()
-            user = CMUser(self.maildomain, addr, password)
+            user = CMUser(self.maildomain, addr, password, self.ssl_context)
             assert user.smtp
             users.append(user)
         return users
 
 
 class CMUser:
-    def __init__(self, maildomain, addr, password):
+    def __init__(self, maildomain, addr, password, ssl_context=None):
         self.maildomain = maildomain
         self.addr = addr
         self.password = password
+        self.ssl_context = ssl_context
         self._smtp = None
         self._imap = None
 
     @property
     def smtp(self):
         if not self._smtp:
-            handle = SmtpConn(self.maildomain)
+            handle = SmtpConn(self.maildomain, ssl_context=self.ssl_context)
             handle.connect()
             handle.login(self.addr, self.password)
             self._smtp = handle
@@ -403,7 +429,7 @@ class CMUser:
     @property
     def imap(self):
         if not self._imap:
-            imap = ImapConn(self.maildomain)
+            imap = ImapConn(self.maildomain, ssl_context=self.ssl_context)
             imap.connect()
             imap.login(self.addr, self.password)
             self._imap = imap

cmdeploy/src/cmdeploy/tests/test_dns.py

@@ -60,29 +60,6 @@ def mockdns(request, mockdns_base, mockdns_expected):
     return mockdns_base
 
 
-class TestGetDkimEntry:
-    def test_dkim_entry_returns_tuple_on_success(self, mockdns):
-        entry, web_entry = remote.rdns.get_dkim_entry(
-            "some.domain", "", dkim_selector="opendkim"
-        )
-        # May return None,None if openssl not available, but should never crash
-        if entry is not None:
-            assert "opendkim._domainkey.some.domain" in entry
-            assert "opendkim._domainkey.some.domain" in web_entry
-
-    def test_dkim_entry_returns_none_tuple_on_error(self, monkeypatch):
-        """CalledProcessError must return (None, None), not bare None."""
-        from subprocess import CalledProcessError
-
-        def failing_shell(command, fail_ok=False, print=print):
-            raise CalledProcessError(1, command)
-
-        monkeypatch.setattr(remote.rdns, "shell", failing_shell)
-        result = remote.rdns.get_dkim_entry("some.domain", "", dkim_selector="opendkim")
-        assert result == (None, None)
-        assert result[0] is None and result[1] is None
-
-
 class TestPerformInitialChecks:
     def test_perform_initial_checks_ok1(self, mockdns, mockdns_expected):
         remote_data = remote.rdns.perform_initial_checks("some.domain")
@@ -114,6 +91,16 @@ class TestPerformInitialChecks:
         assert not res
         assert len(l) == 2
 
+    def test_perform_initial_checks_no_mta_sts_self_signed(self, mockdns):
+        del mockdns["CNAME"]["mta-sts.some.domain"]
+        remote_data = remote.rdns.perform_initial_checks("some.domain")
+        assert not remote_data["MTA_STS"]
+
+        l = []
+        res = check_initial_remote_data(remote_data, strict_tls=False, print=l.append)
+        assert res
+        assert not l
+
 
 def parse_zonefile_into_dict(zonefile, mockdns_base, only_required=False):
     for zf_line in zonefile.split("\n"):

cmdeploy/src/cmdeploy/tests/test_rshell.py

@@ -1,68 +0,0 @@
-from unittest.mock import patch
-
-from cmdeploy.remote.rshell import dovecot_recalc_quota
-
-
-def test_dovecot_recalc_quota_normal_output():
-    """Normal doveadm output returns parsed dict."""
-    normal_output = (
-        "Quota name Type    Value  Limit  %\n"
-        "User quota STORAGE     5 102400  0\n"
-        "User quota MESSAGE     2      -  0\n"
-    )
-
-    with patch("cmdeploy.remote.rshell.shell", return_value=normal_output):
-        result = dovecot_recalc_quota("user@example.org")
-
-    # shell is called twice (recalc + get), patch returns same for both
-    assert result == {"value": 5, "limit": 102400, "percent": 0}
-
-
-def test_dovecot_recalc_quota_empty_output():
-    """Empty doveadm output (trailing newline) must not IndexError."""
-    call_count = [0]
-
-    def mock_shell(cmd):
-        call_count[0] += 1
-        if "recalc" in cmd:
-            return ""
-        # quota get returns only empty lines
-        return "\n\n"
-
-    with patch("cmdeploy.remote.rshell.shell", side_effect=mock_shell):
-        result = dovecot_recalc_quota("user@example.org")
-
-    assert result is None
-
-
-def test_dovecot_recalc_quota_malformed_output():
-    """Malformed output with too few columns must not crash."""
-    call_count = [0]
-
-    def mock_shell(cmd):
-        call_count[0] += 1
-        if "recalc" in cmd:
-            return ""
-        # partial line, fewer than 6 parts
-        return "Quota name\nUser quota STORAGE\n"
-
-    with patch("cmdeploy.remote.rshell.shell", side_effect=mock_shell):
-        result = dovecot_recalc_quota("user@example.org")
-
-    assert result is None
-
-
-def test_dovecot_recalc_quota_header_only():
-    """Only header line, no data rows."""
-    call_count = [0]
-
-    def mock_shell(cmd):
-        call_count[0] += 1
-        if "recalc" in cmd:
-            return ""
-        return "Quota name Type    Value  Limit  %\n"
-
-    with patch("cmdeploy.remote.rshell.shell", side_effect=mock_shell):
-        result = dovecot_recalc_quota("user@example.org")
-
-    assert result is None

doc/source/getting_started.rst

@@ -47,6 +47,14 @@ steps. Please substitute it with your own domain.
        www.chat.example.org. 3600 IN CNAME chat.example.org.
        mta-sts.chat.example.org. 3600 IN CNAME chat.example.org.
 
+   .. note::
+
+      For experimental deployments using self-signed certificates,
+      use a domain name starting with ``_``
+      (e.g. ``_chat.example.org``).
+      The ``mta-sts`` CNAME and ``_mta-sts`` TXT records
+      are not needed for such domains.
+
 2. On your local PC, clone the repository and bootstrap the Python
    virtualenv.
 
@@ -63,6 +71,16 @@ steps. Please substitute it with your own domain.
 
        scripts/cmdeploy init chat.example.org  # <-- use your domain
 
+   To use self-signed TLS certificates
+   instead of Let's Encrypt,
+   use a domain name starting with ``_``
+   (e.g. ``scripts/cmdeploy init _chat.example.org``).
+   Domains starting with ``_`` cannot obtain WebPKI certificates,
+   so self-signed mode is derived automatically.
+   This is useful for private or test deployments.
+   See the :doc:`overview`
+   for details on certificate provisioning.
+
 4. Verify that SSH root login to the deployment server server works:
 
    ::
@@ -169,6 +187,17 @@ creating addresses, login with ssh to the deployment machine and run:
 Chatmail address creation will be denied while this file is present.
 
 
+Running a relay with self-signed certificates
+----------------------------------------------
+
+Use a domain name starting with ``_`` (e.g. ``_chat.example.org``)
+to run a relay with self-signed certificates.
+Domains starting with ``_`` cannot obtain WebPKI certificates
+so the relay automatically uses self-signed certificates
+and all other relays will accept connections from it
+without requiring certificate verification.
+This is useful for experimental setups and testing.
+
 Migrating to a new build machine
 ----------------------------------
 

doc/source/overview.rst

@@ -297,8 +297,7 @@ TLS requirements
 
 Postfix is configured to require valid TLS by setting
 `smtp_tls_security_level <https://www.postfix.org/postconf.5.html#smtp_tls_security_level>`_
-to ``verify``. If emails don’t arrive at your chatmail relay server, the
-problem is likely that your relay does not have a valid TLS certificate.
+to ``verify``.
 
 You can test it by resolving ``MX`` records of your relay domain and
 then connecting to MX relays (e.g ``mx.example.org``) with
@@ -317,6 +316,14 @@ default Exim does not log sessions that are closed before sending the
 by Postfix, so you might think that connection is not established while
 actually it is a problem with your TLS certificate.
 
+If emails don’t arrive at your chatmail relay server, the
+problem is likely that your relay does not have a valid TLS certificate.
+
+Note that connections to relays with underscore-prefixed test domains
+(e.g. ``_chat.example.org``) use ``encrypt`` tls security level,
+because such domains cannot obtain valid Let's Encrypt certificates
+and run with self-signed certificates.
+
 
 .. _dovecot: https://dovecot.org
 .. _postfix: https://www.postfix.org

doc/source/related.rst

@@ -14,8 +14,8 @@ We know of three work-in-progress alternative implementation efforts:
    it to support all of the features and configuration settings required
    to operate as a chatmail relay.
 
--  `Madmail <https://github.com/omidz4t/madmail>`_: an
-   experimental fork of Maddy Mail Server <https://maddy.email/>`_ optimized
+-  `Madmail <https://github.com/themadorg/madmail>`_: an
+   experimental fork of `Maddy Mail Server <https://maddy.email/>`_, modified
    for chatmail deployments.  It provides a single binary solution
    for running a chatmail relay.
 

www/src/dclogin.js

@@ -0,0 +1,21 @@
+/* dclogin profile generator for self-signed chatmail relays.
+ * Fetches credentials from /new and generates a dclogin: QR code.
+ * Requires qrcode-svg.min.js to be loaded first.
+ */
+(function () {
+    function generateProfile() {
+        fetch('/new')
+            .then(function (r) { return r.json(); })
+            .then(function (data) {
+                var url = data.dclogin_url;
+                var link = document.getElementById('dclogin-link');
+                link.href = url;
+                var qrLink = document.getElementById('qr-link');
+                qrLink.href = url;
+                var qrCode = document.getElementById('qr-code');
+                var qr = new QRCode({ content: url, width: 300, height: 300, padding: 1, join: true });
+                qrCode.innerHTML = qr.svg();
+            });
+    }
+    generateProfile();
+})();

www/src/index.md

@@ -11,14 +11,27 @@ for Delta Chat users.  For details how it avoids storing personal information
 please see our [privacy policy](privacy.html). 
 {% endif %}
 
-<a class="cta-button" href="DCACCOUNT:https://{{ config.mail_domain }}/new">Get a {{config.mail_domain}} chat profile</a>
+{% if config.tls_cert_mode == "self" %}
+<a class="cta-button" id="dclogin-link" href="#">Get a {{config.mail_domain}} chat profile</a>
 
 If you are viewing this page on a different device
 without a Delta Chat app,
 you can also **scan this QR code** with Delta Chat:
 
-<a href="DCACCOUNT:https://{{ config.mail_domain }}/new">
+<a id="qr-link" href="#"><div id="qr-code"></div></a>
+
+<script src="qrcode-svg.min.js"></script>
+<script src="dclogin.js"></script>
+{% else %}
+<a class="cta-button" href="DCACCOUNT:{{ config.mail_domain }}">Get a {{config.mail_domain}} chat profile</a>
+
+If you are viewing this page on a different device
+without a Delta Chat app,
+you can also **scan this QR code** with Delta Chat:
+
+<a href="DCACCOUNT:{{ config.mail_domain }}">
     <img width=300 style="float: none;" src="qr-chatmail-invite-{{config.mail_domain}}.png" /></a>
+{% endif %}
 
 🐣 **Choose** your Avatar and Name