feat: metadata service: make turnserver socket path configurable

also add tests for the turnserver metadata

.github/workflows/ci.yaml

@@ -15,7 +15,7 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: download filtermail
-        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.3.0/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
+        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.2.0/filtermail-x86_64-musl -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
       - name: run chatmaild tests 
         working-directory: chatmaild
         run: pipx run tox

.github/workflows/test-and-deploy-ipv4only.yaml

@@ -71,35 +71,26 @@ jobs:
       - name: run deploy-chatmail offline tests 
         run: pytest --pyargs cmdeploy 
 
-      - name: setup dependencies
+      - run: |
-        run: |
+          cmdeploy init staging-ipv4.testrun.org
-          ssh root@staging-ipv4.testrun.org apt update
+          sed -i 's#disable_ipv6 = False#disable_ipv6 = True#' chatmail.ini
-          ssh root@staging-ipv4.testrun.org apt install -y git python3.11-venv python3-dev gcc
+          sed -i 's/#\s*mtail_address/mtail_address/' chatmail.ini
-          ssh root@staging-ipv4.testrun.org git clone https://github.com/chatmail/relay
-          ssh root@staging-ipv4.testrun.org "cd relay && git checkout " ${{ github.head_ref }}
-          ssh root@staging-ipv4.testrun.org "cd relay && scripts/initenv.sh"
 
-      - name: initialize config
+      - run: cmdeploy run --verbose --skip-dns-check
-        run: |
-          ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy init staging-ipv4.testrun.org"
-          ssh root@staging-ipv4.testrun.org "sed -i 's#disable_ipv6 = False#disable_ipv6 = True#' relay/chatmail.ini"
-          ssh root@staging-ipv4.testrun.org "sed -i 's/#\s*mtail_address/mtail_address/' relay/chatmail.ini"
-
-      - run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy run --verbose --skip-dns-check"
 
       - name: set DNS entries
         run: |
-          ssh root@staging-ipv4.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
+          ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
-          ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns --zonefile staging-generated.zone"
+          cmdeploy dns --zonefile staging-generated.zone
-          ssh root@staging-ipv4.testrun.org cat relay/staging-generated.zone >> .github/workflows/staging-ipv4.testrun.org-default.zone
+          cat staging-generated.zone >> .github/workflows/staging-ipv4.testrun.org-default.zone
           cat .github/workflows/staging-ipv4.testrun.org-default.zone
           scp .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
           ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
           ssh root@ns.testrun.org systemctl reload nsd
 
       - name: cmdeploy test
-        run: ssh root@staging-ipv4.testrun.org "cd relay && CHATMAIL_DOMAIN2=ci-chatmail.testrun.org scripts/cmdeploy test --slow"
+        run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
 
       - name: cmdeploy dns
-        run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns -v"
+        run: cmdeploy dns -v
 

.github/workflows/test-and-deploy.yaml

@@ -76,7 +76,6 @@ jobs:
 
       - run: |
           cmdeploy init staging2.testrun.org
-          sed -i 's/^ssh_host/#ssh_host/' chatmail.ini
           sed -i 's/#\s*mtail_address/mtail_address/' chatmail.ini
 
       - run: cmdeploy run --verbose --skip-dns-check

chatmaild/src/chatmaild/config.py

@@ -1,4 +1,3 @@
-import os
 from pathlib import Path
 
 import iniconfig
@@ -9,28 +8,30 @@ from chatmaild.user import User
 def read_config(inipath):
     assert Path(inipath).exists(), inipath
     cfg = iniconfig.IniConfig(inipath)
-    return Config(inipath, params=cfg.sections["params"])
+    params = cfg.sections["params"]
+    default_config_content = get_default_config_content(params["mail_domain"])
+    df_params = iniconfig.IniConfig("ini", data=default_config_content)["params"]
+    new_params = dict(df_params.items())
+    new_params.update(params)
+    return Config(inipath, params=new_params)
 
 
 class Config:
     def __init__(self, inipath, params):
         self._inipath = inipath
         self.mail_domain = params["mail_domain"]
-        self.ssh_host = params.get("ssh_host", self.mail_domain)
         self.max_user_send_per_minute = int(params.get("max_user_send_per_minute", 60))
         self.max_user_send_burst_size = int(params.get("max_user_send_burst_size", 10))
-        self.max_mailbox_size = params.get("max_mailbox_size", "500M")
+        self.max_mailbox_size = params["max_mailbox_size"]
-        self.max_message_size = int(params.get("max_message_size", 31457280))
+        self.max_message_size = int(params.get("max_message_size", "31457280"))
-        self.delete_mails_after = params.get("delete_mails_after", "20")
+        self.delete_mails_after = params["delete_mails_after"]
-        self.delete_large_after = params.get("delete_large_after", "7")
+        self.delete_large_after = params["delete_large_after"]
-        self.delete_inactive_users_after = int(
+        self.delete_inactive_users_after = int(params["delete_inactive_users_after"])
-            params.get("delete_inactive_users_after", 100)
+        self.username_min_length = int(params["username_min_length"])
-        )
+        self.username_max_length = int(params["username_max_length"])
-        self.username_min_length = int(params.get("username_min_length", 9))
+        self.password_min_length = int(params["password_min_length"])
-        self.username_max_length = int(params.get("username_max_length", 9))
+        self.passthrough_senders = params["passthrough_senders"].split()
-        self.password_min_length = int(params.get("password_min_length", 9))
+        self.passthrough_recipients = params["passthrough_recipients"].split()
-        self.passthrough_senders = params.get("passthrough_senders", "").split()
-        self.passthrough_recipients = params.get("passthrough_recipients", "").split()
         self.www_folder = params.get("www_folder", "")
         self.filtermail_smtp_port = int(params.get("filtermail_smtp_port", "10080"))
         self.filtermail_smtp_port_incoming = int(
@@ -42,11 +43,10 @@ class Config:
         )
         self.mtail_address = params.get("mtail_address")
         self.disable_ipv6 = params.get("disable_ipv6", "false").lower() == "true"
-        self.addr_v4 = os.environ.get("CHATMAIL_ADDR_V4", "")
-        self.addr_v6 = os.environ.get("CHATMAIL_ADDR_V6", "")
         self.acme_email = params.get("acme_email", "")
         self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true"
         self.imap_compress = params.get("imap_compress", "false").lower() == "true"
+        self.turn_socket_path = params.get("turn_socket_path", "/run/chatmail-turn/turn.socket")
         if "iroh_relay" not in params:
             self.iroh_relay = "https://" + params["mail_domain"]
             self.enable_iroh_relay = True
@@ -58,18 +58,6 @@ class Config:
         self.privacy_pdo = params.get("privacy_pdo")
         self.privacy_supervisor = params.get("privacy_supervisor")
 
-        # TLS certificate management: derived from the domain name.
-        # Domains starting with "_" use self-signed certificates
-        # All other domains use ACME.
-        if self.mail_domain.startswith("_"):
-            self.tls_cert_mode = "self"
-            self.tls_cert_path = "/etc/ssl/certs/mailserver.pem"
-            self.tls_key_path = "/etc/ssl/private/mailserver.key"
-        else:
-            self.tls_cert_mode = "acme"
-            self.tls_cert_path = f"/var/lib/acme/live/{self.mail_domain}/fullchain"
-            self.tls_key_path = f"/var/lib/acme/live/{self.mail_domain}/privkey"
-
         # deprecated option
         mbdir = params.get("mailboxes_dir", f"/home/vmail/mail/{self.mail_domain}")
         self.mailboxes_dir = Path(mbdir.strip())

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -3,9 +3,6 @@
 # mail domain (MUST be set to fully qualified chat mail domain)
 mail_domain = {mail_domain}
 
-# Where to deploy the relay - if unspecified, mail_domain will be used.
-ssh_host = localhost
-
 #
 # If you only do private test deploys, you don't need to modify any settings below
 #
@@ -58,7 +55,10 @@ passthrough_recipients =
 # Deployment Details
 #
 
-# SMTP outgoing filtermail and reinjection 
+# Path to the TURN server Unix socket
+turn_socket_path = /run/chatmail-turn/turn.socket
+
+# SMTP outgoing filtermail and reinjection
 filtermail_smtp_port = 10080
 postfix_reinject_port = 10025
 

chatmaild/src/chatmaild/metadata.py

@@ -76,12 +76,13 @@ class Metadata:
 
 
 class MetadataDictProxy(DictProxy):
-    def __init__(self, notifier, metadata, iroh_relay=None, turn_hostname=None):
+    def __init__(self, notifier, metadata, iroh_relay=None, turn_hostname=None, config=None):
         super().__init__()
         self.notifier = notifier
         self.metadata = metadata
         self.iroh_relay = iroh_relay
         self.turn_hostname = turn_hostname
+        self.config = config
 
     def handle_lookup(self, parts):
         # Lpriv/43f5f508a7ea0366dff30200c15250e3/devicetoken\tlkj123poi@c2.testrun.org
@@ -101,7 +102,7 @@ class MetadataDictProxy(DictProxy):
                 # Handle `GETMETADATA "" /shared/vendor/deltachat/irohrelay`
                 return f"O{self.iroh_relay}\n"
             elif keyname == "vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn":
-                res = turn_credentials()
+                res = turn_credentials(self.config)
                 port = 3478
                 return f"O{self.turn_hostname}:{port}:{res}\n"
 
@@ -146,6 +147,7 @@ def main():
         metadata=metadata,
         iroh_relay=iroh_relay,
         turn_hostname=mail_domain,
+        config=config,
     )
 
     dictproxy.serve_forever_from_socket(socket)

chatmaild/src/chatmaild/newemail.py

@@ -6,7 +6,6 @@ import json
 import random
 import secrets
 import string
-from urllib.parse import quote
 
 from chatmaild.config import Config, read_config
 
@@ -24,26 +23,13 @@ def create_newemail_dict(config: Config):
     return dict(email=f"{user}@{config.mail_domain}", password=f"{password}")
 
 
-def create_dclogin_url(email, password):
-    """Build a dclogin: URL with credentials and self-signed cert acceptance.
-
-    Uses ic=3 (AcceptInvalidCertificates) so chatmail clients
-    can connect to servers with self-signed TLS certificates.
-    """
-    return f"dclogin:{quote(email, safe='@')}?p={quote(password, safe='')}&v=1&ic=3"
-
-
 def print_new_account():
     config = read_config(CONFIG_PATH)
     creds = create_newemail_dict(config)
 
-    result = dict(email=creds["email"], password=creds["password"])
-    if config.tls_cert_mode == "self":
-        result["dclogin_url"] = create_dclogin_url(creds["email"], creds["password"])
-
     print("Content-Type: application/json")
     print("")
-    print(json.dumps(result))
+    print(json.dumps(creds))
 
 
 if __name__ == "__main__":

chatmaild/src/chatmaild/tests/test_config.py

@@ -73,17 +73,3 @@ def test_config_userstate_paths(make_config, tmp_path):
 def test_config_max_message_size(make_config, tmp_path):
     config = make_config("something.testrun.org", dict(max_message_size="10000"))
     assert config.max_message_size == 10000
-
-
-def test_config_tls_default_acme(make_config):
-    config = make_config("chat.example.org")
-    assert config.tls_cert_mode == "acme"
-    assert config.tls_cert_path == "/var/lib/acme/live/chat.example.org/fullchain"
-    assert config.tls_key_path == "/var/lib/acme/live/chat.example.org/privkey"
-
-
-def test_config_tls_self(make_config):
-    config = make_config("_test.example.org")
-    assert config.tls_cert_mode == "self"
-    assert config.tls_cert_path == "/etc/ssl/certs/mailserver.pem"
-    assert config.tls_key_path == "/etc/ssl/private/mailserver.key"

chatmaild/src/chatmaild/tests/test_filtermail_blackbox.py

@@ -1,15 +1,9 @@
-import shutil
 import smtplib
 import subprocess
 import sys
 
 import pytest
 
-pytestmark = pytest.mark.skipif(
-    shutil.which("filtermail") is None,
-    reason="filtermail binary not found",
-)
-
 
 @pytest.fixture
 def smtpserver():

chatmaild/src/chatmaild/tests/test_newmail.py

@@ -1,11 +1,7 @@
 import json
 
 import chatmaild
-from chatmaild.newemail import (
+from chatmaild.newemail import create_newemail_dict, print_new_account
-    create_dclogin_url,
-    create_newemail_dict,
-    print_new_account,
-)
 
 
 def test_create_newemail_dict(example_config):
@@ -19,18 +15,6 @@ def test_create_newemail_dict(example_config):
     assert ac1["password"] != ac2["password"]
 
 
-def test_create_dclogin_url():
-    url = create_dclogin_url("user@example.org", "p@ss w+rd")
-    assert url.startswith("dclogin:")
-    assert "v=1" in url
-    assert "ic=3" in url
-
-    assert "user@example.org" in url
-    # password special chars must be encoded
-    assert "p%40ss" in url
-    assert "w%2Brd" in url
-
-
 def test_print_new_account(capsys, monkeypatch, maildomain, tmpdir, example_config):
     monkeypatch.setattr(chatmaild.newemail, "CONFIG_PATH", str(example_config._inipath))
     print_new_account()
@@ -41,20 +25,3 @@ def test_print_new_account(capsys, monkeypatch, maildomain, tmpdir, example_conf
     dic = json.loads(lines[2])
     assert dic["email"].endswith(f"@{example_config.mail_domain}")
     assert len(dic["password"]) >= 10
-    # default tls_cert=acme should not include dclogin_url
-    assert "dclogin_url" not in dic
-
-
-def test_print_new_account_self_signed(capsys, monkeypatch, make_config):
-    config = make_config("_test.example.org")
-    monkeypatch.setattr(chatmaild.newemail, "CONFIG_PATH", str(config._inipath))
-    print_new_account()
-    out, err = capsys.readouterr()
-    lines = out.split("\n")
-    dic = json.loads(lines[2])
-    assert "dclogin_url" in dic
-    url = dic["dclogin_url"]
-    assert url.startswith("dclogin:")
-    assert "ic=3" in url
-
-    assert dic["email"].split("@")[0] in url

chatmaild/src/chatmaild/tests/test_turnserver.py

@@ -0,0 +1,120 @@
+"""Tests for turnserver functionality, particularly metadata integration."""
+
+import socket
+import tempfile
+import threading
+from pathlib import Path
+
+from chatmaild.config import read_config, write_initial_config
+from chatmaild.metadata import MetadataDictProxy, Metadata
+from chatmaild.notifier import Notifier
+from chatmaild.turnserver import turn_credentials
+
+
+def test_turn_credentials_function_with_custom_socket():
+    """Test that turn_credentials function works with a custom socket path from config."""
+    # Create a temporary directory and socket file
+    temp_dir = Path(tempfile.mkdtemp())
+    temp_socket_path = temp_dir / "test_turn.socket"
+
+    # Create a mock TURN credentials server
+    def mock_server():
+        server_sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        server_sock.bind(str(temp_socket_path))
+        server_sock.listen(1)
+
+        # Accept connection and send mock credentials
+        conn, addr = server_sock.accept()
+        with conn:
+            conn.send(b"mock_turn_credentials_abc123\n")
+        server_sock.close()
+
+    # Start server in a background thread
+    server_thread = threading.Thread(target=mock_server, daemon=True)
+    server_thread.start()
+
+    # Create a config with custom socket path
+    config_path = temp_dir / "chatmail.ini"
+    write_initial_config(config_path, "test.example.org", {
+        "turn_socket_path": str(temp_socket_path)
+    })
+    config = read_config(config_path)
+
+    # Allow time for server to start
+    import time
+    time.sleep(0.01)
+
+    # Test that turn_credentials can connect using the config
+    credentials = turn_credentials(config)
+    assert credentials == "mock_turn_credentials_abc123"
+
+    server_thread.join(timeout=1)  # Clean up thread
+
+
+def test_metadata_turn_lookup_integration(tmp_path):
+    """Test that metadata service properly handles TURN metadata lookups."""
+    # Create mock config with custom turn socket path
+    config_path = tmp_path / "chatmail.ini"
+    socket_path = tmp_path / "test_turn.socket"
+    write_initial_config(config_path, "example.org", {
+        "turn_socket_path": str(socket_path)
+    })
+    config = read_config(config_path)
+
+    # Create mock TURN server to return credentials
+    def mock_turn_server():
+        import os
+        os.makedirs(socket_path.parent, exist_ok=True)  # Ensure parent directory exists
+
+        server_sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        server_sock.bind(str(socket_path))
+        server_sock.listen(1)
+
+        # Accept connection and send mock credentials
+        conn, addr = server_sock.accept()
+        with conn:
+            conn.send(b"test_creds_12345\n")
+        server_sock.close()
+
+    server_thread = threading.Thread(target=mock_turn_server, daemon=True)
+    server_thread.start()
+
+    import time
+    time.sleep(0.01)  # Allow server to start
+
+    # Create a MetadataDictProxy with config
+    queue_dir = tmp_path / "queue"
+    queue_dir.mkdir()
+    notifier = Notifier(queue_dir)
+    metadata = Metadata(tmp_path / "vmail")
+
+    dict_proxy = MetadataDictProxy(
+        notifier=notifier,
+        metadata=metadata,
+        iroh_relay="https://example.org",
+        turn_hostname="example.org",
+        config=config
+    )
+
+    # Simulate a lookup for TURN credentials using the correct format
+    # Input: "shared/0123/vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn"
+    # After parts[0].split("/", 2):
+    # - keyparts[0] = "shared"
+    # - keyparts[1] = "0123"
+    # - keyparts[2] = "vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn"
+    # So keyname = keyparts[2] should match "vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn"
+    parts = [
+        "shared/0123/vendor/vendor.dovecot/pvt/server/vendor/deltachat/turn",
+        "dummy@user.org"
+    ]
+
+    # Call handle_lookup directly
+    result = dict_proxy.handle_lookup(parts)
+
+    # Verify the response format is correct for TURN credentials
+    assert result.startswith("O")  # Output response starts with 'O'
+    assert ":3478:" in result  # Contains port 3478
+    assert "test_creds_12345" in result  # Contains credentials returned by mock server
+    assert "example.org:3478:test_creds_12345" in result
+
+    server_thread.join(timeout=1)  # Clean up thread

chatmaild/src/chatmaild/turnserver.py

@@ -2,8 +2,8 @@
 import socket
 
 
-def turn_credentials() -> str:
+def turn_credentials(config) -> str:
     with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client_socket:
-        client_socket.connect("/run/chatmail-turn/turn.socket")
+        client_socket.connect(config.turn_socket_path)
         with client_socket.makefile("rb") as file:
             return file.readline().decode("utf-8").strip()

cmdeploy/src/cmdeploy/chatmail.zone.j2

@@ -8,10 +8,8 @@
 {{ mail_domain }}.                   AAAA  {{ AAAA }}
 {% endif %}
 {{ mail_domain }}.                   MX 10 {{ mail_domain }}.
-{% if strict_tls %}
 _mta-sts.{{ mail_domain }}.          TXT "v=STSv1; id={{ sts_id }}"
 mta-sts.{{ mail_domain }}.           CNAME {{ mail_domain }}.
-{% endif %}
 www.{{ mail_domain }}.               CNAME {{ mail_domain }}.
 {{ dkim_entry }}
 

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -88,13 +88,12 @@ def run_cmd_options(parser):
 def run_cmd(args, out):
     """Deploy chatmail services on the remote server."""
 
-    ssh_host = args.ssh_host if args.ssh_host else args.config.ssh_host
+    ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
     sshexec = get_sshexec(ssh_host)
     require_iroh = args.config.enable_iroh_relay
-    strict_tls = args.config.tls_cert_mode == "acme"
     if not args.dns_check_disabled:
         remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
-        if not dns.check_initial_remote_data(remote_data, strict_tls=strict_tls, print=out.red):
+        if not dns.check_initial_remote_data(remote_data, print=out.red):
             return 1
 
     env = os.environ.copy()
@@ -102,14 +101,11 @@ def run_cmd(args, out):
     env["CHATMAIL_WEBSITE_ONLY"] = "True" if args.website_only else ""
     env["CHATMAIL_DISABLE_MAIL"] = "True" if args.disable_mail else ""
     env["CHATMAIL_REQUIRE_IROH"] = "True" if require_iroh else ""
-    if not args.dns_check_disabled:
-        env["CHATMAIL_ADDR_V4"] = remote_data.get("A") or ""
-        env["CHATMAIL_ADDR_V6"] = remote_data.get("AAAA") or ""
     deploy_path = importlib.resources.files(__package__).joinpath("run.py").resolve()
     pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
 
     cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y"
-    if ssh_host in ["localhost", "@local", "@docker"]:
+    if ssh_host in ["localhost", "@docker"]:
         cmd = f"{pyinf} @local {deploy_path} -y"
 
     if version.parse(pyinfra.__version__) < version.parse("3"):
@@ -125,7 +121,7 @@ def run_cmd(args, out):
                 out.red("Website deployment failed.")
         elif retcode == 0:
             out.green("Deploy completed, call `cmdeploy dns` next.")
-        elif not args.dns_check_disabled and strict_tls and not remote_data["acme_account_url"]:
+        elif not remote_data["acme_account_url"]:
             out.red("Deploy completed but letsencrypt not configured")
             out.red("Run 'cmdeploy run' again")
             retcode = 0
@@ -150,15 +146,13 @@ def dns_cmd_options(parser):
 
 def dns_cmd(args, out):
     """Check DNS entries and optionally generate dns zone file."""
-    ssh_host = args.ssh_host if args.ssh_host else args.config.ssh_host
+    ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
     sshexec = get_sshexec(ssh_host, verbose=args.verbose)
-    tls_cert_mode = args.config.tls_cert_mode
-    strict_tls = tls_cert_mode == "acme"
     remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
-    if not dns.check_initial_remote_data(remote_data, strict_tls=strict_tls):
+    if not remote_data:
         return 1
 
-    if strict_tls and not remote_data["acme_account_url"]:
+    if not remote_data["acme_account_url"]:
         out.red("could not get letsencrypt account url, please run 'cmdeploy run'")
         return 1
 
@@ -166,7 +160,6 @@ def dns_cmd(args, out):
         out.red("could not determine dkim_entry, please run 'cmdeploy run'")
         return 1
 
-    remote_data["strict_tls"] = strict_tls
     zonefile = dns.get_filled_zone_file(remote_data)
 
     if args.zonefile:
@@ -187,7 +180,7 @@ def status_cmd_options(parser):
 def status_cmd(args, out):
     """Display status for online chatmail instance."""
 
-    ssh_host = args.ssh_host if args.ssh_host else args.config.ssh_host
+    ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
     sshexec = get_sshexec(ssh_host, verbose=args.verbose)
 
     out.green(f"chatmail domain: {args.config.mail_domain}")
@@ -207,7 +200,6 @@ def test_cmd_options(parser):
         action="store_true",
         help="also run slow tests",
     )
-    add_ssh_host_option(parser)
 
 
 def test_cmd(args, out):
@@ -219,9 +211,6 @@ def test_cmd(args, out):
     x = importlib.util.find_spec("deltachat")
     if x is None:
         out.check_call(f"{sys.executable} -m pip install deltachat")
-    env = os.environ.copy()
-    if args.ssh_host:
-        env["CHATMAIL_SSH"] = args.ssh_host
 
     pytest_path = shutil.which("pytest")
     pytest_args = [
@@ -235,7 +224,7 @@ def test_cmd(args, out):
     ]
     if args.slow:
         pytest_args.append("--slow")
-    ret = out.run_ret(pytest_args, env=env)
+    ret = out.run_ret(pytest_args)
     return ret
 
 

cmdeploy/src/cmdeploy/deployers.py

@@ -10,7 +10,6 @@ from pathlib import Path
 
 from chatmaild.config import read_config
 from pyinfra import facts, host, logger
-from pyinfra.facts import hardware
 from pyinfra.api import FactBase
 from pyinfra.facts.files import Sha256File
 from pyinfra.facts.systemd import SystemdEnabled
@@ -19,7 +18,6 @@ from pyinfra.operations import apt, files, pip, server, systemd
 from cmdeploy.cmdeploy import Out
 
 from .acmetool import AcmetoolDeployer
-from .selfsigned.deployer import SelfSignedTlsDeployer
 from .basedeploy import (
     Deployer,
     Deployment,
@@ -38,7 +36,7 @@ from .www import build_webpages, find_merge_conflict, get_paths
 
 class Port(FactBase):
     """
-    Returns the process occupying a port.
+    Returns the process occuping a port.
     """
 
     def command(self, port: int) -> str:
@@ -559,21 +557,10 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
             line="\nnameserver 9.9.9.9",
         )
 
-    # Check if mtail_address interface is available (if configured)
-    if config.mtail_address and config.mtail_address not in ('127.0.0.1', '::1', 'localhost'):
-        ipv4_addrs = host.get_fact(hardware.Ipv4Addrs)
-        all_addresses = [addr for addrs in ipv4_addrs.values() for addr in addrs]
-        if config.mtail_address not in all_addresses:
-            Out().red(f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n")
-            exit(1)
-
     port_services = [
         (["master", "smtpd"], 25),
         ("unbound", 53),
-    ]
+        ("acmetool", 80),
-    if config.tls_cert_mode == "acme":
-        port_services.append(("acmetool", 80))
-    port_services += [
         (["imap-login", "dovecot"], 143),
         ("nginx", 443),
         (["master", "smtpd"], 465),
@@ -581,7 +568,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
         (["imap-login", "dovecot"], 993),
         ("iroh-relay", 3340),
         ("mtail", 3903),
-        ("stats", 3904),
+        ("dovecot-stats", 3904),
         ("nginx", 8443),
         (["master", "smtpd"], config.postfix_reinject_port),
         (["master", "smtpd"], config.postfix_reinject_port_incoming),
@@ -591,9 +578,8 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
     for service, port in port_services:
         print(f"Checking if port {port} is available for {service}...")
         running_service = host.get_fact(Port, port=port)
-        services = [service] if isinstance(service, str) else service
         if running_service:
-            if running_service not in services:
+            if running_service not in service:
                 Out().red(
                     f"Deploy failed: port {port} is occupied by: {running_service}"
                 )
@@ -601,11 +587,6 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
 
     tls_domains = [mail_domain, f"mta-sts.{mail_domain}", f"www.{mail_domain}"]
 
-    if config.tls_cert_mode == "acme":
-        tls_deployer = AcmetoolDeployer(config.acme_email, tls_domains)
-    else:
-        tls_deployer = SelfSignedTlsDeployer(mail_domain)
-
     all_deployers = [
         ChatmailDeployer(mail_domain),
         LegacyRemoveDeployer(),
@@ -614,7 +595,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
         UnboundDeployer(config),
         TurnDeployer(mail_domain),
         IrohDeployer(config.enable_iroh_relay),
-        tls_deployer,
+        AcmetoolDeployer(config.acme_email, tls_domains),
         WebsiteDeployer(config),
         ChatmailVenvDeployer(config),
         MtastsDeployer(),

cmdeploy/src/cmdeploy/dns.py

@@ -12,14 +12,14 @@ def get_initial_remote_data(sshexec, mail_domain):
     )
 
 
-def check_initial_remote_data(remote_data, *, strict_tls=True, print=print):
+def check_initial_remote_data(remote_data, *, print=print):
     mail_domain = remote_data["mail_domain"]
     if not remote_data["A"] and not remote_data["AAAA"]:
         print(f"Missing A and/or AAAA DNS records for {mail_domain}!")
-    elif strict_tls and remote_data["MTA_STS"] != f"{mail_domain}.":
+    elif remote_data["MTA_STS"] != f"{mail_domain}.":
         print("Missing MTA-STS CNAME record:")
         print(f"mta-sts.{mail_domain}.   CNAME  {mail_domain}.")
-    elif strict_tls and remote_data["WWW"] != f"{mail_domain}.":
+    elif remote_data["WWW"] != f"{mail_domain}.":
         print("Missing www CNAME record:")
         print(f"www.{mail_domain}.   CNAME  {mail_domain}.")
     else:

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -22,7 +22,7 @@ class DovecotDeployer(Deployer):
 
     def install(self):
         arch = host.get_fact(Arch)
-        if not host.get_fact(SystemdEnabled).get("dovecot.service"):
+        if not "dovecot.service" in host.get_fact(SystemdEnabled):
             _install_dovecot_package("core", arch)
             _install_dovecot_package("imapd", arch)
             _install_dovecot_package("lmtpd", arch)

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -228,8 +228,8 @@ service anvil {
 }
 
 ssl = required
-ssl_cert = <{{ config.tls_cert_path }}
+ssl_cert = </var/lib/acme/live/{{ config.mail_domain }}/fullchain
-ssl_key = <{{ config.tls_key_path }}
+ssl_key = </var/lib/acme/live/{{ config.mail_domain }}/privkey
 ssl_dh = </usr/share/dovecot/dh.pem
 ssl_min_protocol = TLSv1.3
 ssl_prefer_server_ciphers = yes

cmdeploy/src/cmdeploy/filtermail/deployer.py

@@ -14,10 +14,10 @@ class FiltermailDeployer(Deployer):
 
     def install(self):
         arch = host.get_fact(facts.server.Arch)
-        url = f"https://github.com/chatmail/filtermail/releases/download/v0.3.0/filtermail-{arch}"
+        url = f"https://github.com/chatmail/filtermail/releases/download/v0.2.0/filtermail-{arch}-musl"
         sha256sum = {
-            "x86_64": "f14a31323ae2dad3b59d3fdafcde507521da2f951a9478cd1f2fe2b4463df71d",
+            "x86_64": "1e5bbb646582cb16740c6dfbbca39edba492b78cc96ec9fa2528c612bb504edd",
-            "aarch64": "933770d75046c4fd7084ce8d43f905f8748333426ad839154f0fc654755ef09f",
+            "aarch64": "3564fba8605f8f9adfeefff3f4580533205da043f47c5968d0d10db17e50f44e",
         }[arch]
         self.need_restart |= files.download(
             name="Download filtermail",

cmdeploy/src/cmdeploy/nginx/autoconfig.xml.j2

@@ -1,47 +1,47 @@
 <?xml version="1.0" encoding="UTF-8"?>
 
 <clientConfig version="1.1">
-  <emailProvider id="{{ config.mail_domain }}">
+  <emailProvider id="{{ config.domain_name }}">
-    <domain>{{ config.mail_domain }}</domain>
+    <domain>{{ config.domain_name }}</domain>
-    <displayName>{{ config.mail_domain }} chatmail</displayName>
+    <displayName>{{ config.domain_name }} chatmail</displayName>
-    <displayShortName>{{ config.mail_domain }}</displayShortName>
+    <displayShortName>{{ config.domain_name }}</displayShortName>
     <incomingServer type="imap">
-      <hostname>{{ config.mail_domain }}</hostname>
+      <hostname>{{ config.domain_name }}</hostname>
       <port>993</port>
       <socketType>SSL</socketType>
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
     </incomingServer>
     <incomingServer type="imap">
-      <hostname>{{ config.mail_domain }}</hostname>
+      <hostname>{{ config.domain_name }}</hostname>
       <port>143</port>
       <socketType>STARTTLS</socketType>
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
     </incomingServer>
     <incomingServer type="imap">
-      <hostname>{{ config.mail_domain }}</hostname>
+      <hostname>{{ config.domain_name }}</hostname>
       <port>443</port>
       <socketType>SSL</socketType>
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
     </incomingServer>
     <outgoingServer type="smtp">
-      <hostname>{{ config.mail_domain }}</hostname>
+      <hostname>{{ config.domain_name }}</hostname>
       <port>465</port>
       <socketType>SSL</socketType>
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
     </outgoingServer>
     <outgoingServer type="smtp">
-      <hostname>{{ config.mail_domain }}</hostname>
+      <hostname>{{ config.domain_name }}</hostname>
       <port>587</port>
       <socketType>STARTTLS</socketType>
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
     </outgoingServer>
     <outgoingServer type="smtp">
-      <hostname>{{ config.mail_domain }}</hostname>
+      <hostname>{{ config.domain_name }}</hostname>
       <port>443</port>
       <socketType>SSL</socketType>
       <authentication>password-cleartext</authentication>

cmdeploy/src/cmdeploy/nginx/deployer.py

@@ -70,7 +70,7 @@ def _configure_nginx(config: Config, debug: bool = False) -> bool:
         user="root",
         group="root",
         mode="644",
-        config=config,
+        config={"domain_name": config.mail_domain},
         disable_ipv6=config.disable_ipv6,
     )
     need_restart |= main_config.changed
@@ -81,7 +81,7 @@ def _configure_nginx(config: Config, debug: bool = False) -> bool:
         user="root",
         group="root",
         mode="644",
-        config=config,
+        config={"domain_name": config.mail_domain},
     )
     need_restart |= autoconfig.changed
 
@@ -91,7 +91,7 @@ def _configure_nginx(config: Config, debug: bool = False) -> bool:
         user="root",
         group="root",
         mode="644",
-        config=config,
+        config={"domain_name": config.mail_domain},
     )
     need_restart |= mta_sts_config.changed
 

cmdeploy/src/cmdeploy/nginx/mta-sts.txt.j2

@@ -1,4 +1,4 @@
 version: STSv1
 mode: enforce
-mx: {{ config.mail_domain }}
+mx: {{ config.domain_name }}
 max_age: 2419200

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -42,9 +42,6 @@ stream {
 }
 
 http {
-{% if config.tls_cert_mode == "self" %}
-	limit_req_zone $binary_remote_addr zone=newaccount:10m rate=2r/s;
-{% endif %}
 	sendfile on;
 	tcp_nopush on;
 
@@ -56,8 +53,8 @@ http {
 
 	ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
 	ssl_prefer_server_ciphers on;
-	ssl_certificate {{ config.tls_cert_path }};
+	ssl_certificate /var/lib/acme/live/{{ config.domain_name }}/fullchain;
-	ssl_certificate_key {{ config.tls_key_path }};
+	ssl_certificate_key /var/lib/acme/live/{{ config.domain_name }}/privkey;
 
 	gzip on;
 
@@ -69,7 +66,7 @@ http {
 
 		index index.html index.htm;
 
-		server_name {{ config.mail_domain }} www.{{ config.mail_domain }} mta-sts.{{ config.mail_domain }};
+		server_name {{ config.domain_name }} www.{{ config.domain_name }} mta-sts.{{ config.domain_name }};
 
 		access_log syslog:server=unix:/dev/log,facility=local7;
 
@@ -84,15 +81,11 @@ http {
 		}
 
 		location /new {
-{% if config.tls_cert_mode == "acme" %}
 			if ($request_method = GET) {
 				# Redirect to Delta Chat,
 				# which will in turn do a POST request.
-				return 301 dcaccount:https://{{ config.mail_domain }}/new;
+				return 301 dcaccount:https://{{ config.domain_name }}/new;
 			}
-{% else %}
-			limit_req zone=newaccount burst=5 nodelay;
-{% endif %}
 
 			fastcgi_pass unix:/run/fcgiwrap.socket;
 			include /etc/nginx/fastcgi_params;
@@ -106,11 +99,9 @@ http {
 		#
 		# Redirects are only for browsers.
 		location /cgi-bin/newemail.py {
-{% if config.tls_cert_mode == "acme" %}
 			if ($request_method = GET) {
-				return 301 dcaccount:https://{{ config.mail_domain }}/new;
+				return 301 dcaccount:https://{{ config.domain_name }}/new;
 			}
-{% endif %}
 
 			fastcgi_pass unix:/run/fcgiwrap.socket;
 			include /etc/nginx/fastcgi_params;
@@ -141,8 +132,8 @@ http {
 	# Redirect www. to non-www
 	server {
 		listen 127.0.0.1:8443 ssl;
-		server_name www.{{ config.mail_domain }};
+		server_name www.{{ config.domain_name }};
-		return 301 $scheme://{{ config.mail_domain }}$request_uri;
+		return 301 $scheme://{{ config.domain_name }}$request_uri;
 		access_log syslog:server=unix:/dev/log,facility=local7;
 	}
 }

cmdeploy/src/cmdeploy/postfix/deployer.py

@@ -61,20 +61,6 @@ class PostfixDeployer(Deployer):
         )
         need_restart |= lmtp_header_cleanup.changed
 
-        tls_policy_map = files.put(
-            name="Upload SMTP TLS Policy that accepts self-signed certificates for IP-only hosts",
-            src=get_resource("postfix/smtp_tls_policy_map"),
-            dest="/etc/postfix/smtp_tls_policy_map",
-            user="root",
-            group="root",
-            mode="644",
-        )
-        need_restart |= tls_policy_map.changed
-        if tls_policy_map.changed:
-            server.shell(
-                commands=["postmap /etc/postfix/smtp_tls_policy_map"],
-            )
-
         # Login map that 1:1 maps email address to login.
         login_map = files.put(
             src=get_resource("postfix/login_map"),

cmdeploy/src/cmdeploy/postfix/lmtp_header_cleanup

@@ -1,3 +1,2 @@
 /^DKIM-Signature:/            IGNORE
 /^Authentication-Results:/            IGNORE
-/^Received:/            IGNORE

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -15,17 +15,17 @@ readme_directory = no
 compatibility_level = 3.6
 
 # TLS parameters
-smtpd_tls_cert_file={{ config.tls_cert_path }}
+smtpd_tls_cert_file=/var/lib/acme/live/{{ config.mail_domain }}/fullchain
-smtpd_tls_key_file={{ config.tls_key_path }}
+smtpd_tls_key_file=/var/lib/acme/live/{{ config.mail_domain }}/privkey
 smtpd_tls_security_level=may
 
 smtp_tls_CApath=/etc/ssl/certs
-smtp_tls_security_level={{ "verify" if config.tls_cert_mode == "acme" else "encrypt" }}
+smtp_tls_security_level=verify
 # Send SNI extension when connecting to other servers.
 # <https://www.postfix.org/postconf.5.html#smtp_tls_servername>
 smtp_tls_servername = hostname
 smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
-smtp_tls_policy_maps = regexp:/etc/postfix/smtp_tls_policy_map
+smtp_tls_policy_maps = inline:{nauta.cu=may}
 smtp_tls_protocols = >=TLSv1.2
 smtp_tls_mandatory_protocols = >=TLSv1.2
 
@@ -69,15 +69,6 @@ mynetworks = 127.0.0.0/8
 {% else %}
 mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128
 {% endif %}
-{% if config.addr_v4 %}
-smtp_bind_address = {{ config.addr_v4 }}
-{% endif %}
-{% if config.addr_v6 %}
-smtp_bind_address6 = {{ config.addr_v6 }}
-{% endif %}
-{% if config.addr_v4 or config.addr_v6 %}
-smtp_bind_address_enforce = yes
-{% endif %}
 mailbox_size_limit = 0
 message_size_limit = {{config.max_message_size}}
 recipient_delimiter = +

cmdeploy/src/cmdeploy/postfix/smtp_tls_policy_map

@@ -1,3 +0,0 @@
-/^\[[^]]+\]$/ encrypt
-/^_/          encrypt
-/^nauta\.cu$/    may

cmdeploy/src/cmdeploy/selfsigned/deployer.py

@@ -1,36 +0,0 @@
-from pyinfra.operations import apt, files, server
-
-from cmdeploy.basedeploy import Deployer
-
-
-class SelfSignedTlsDeployer(Deployer):
-    """Generates a self-signed TLS certificate for all chatmail endpoints."""
-
-    def __init__(self, mail_domain):
-        self.mail_domain = mail_domain
-        self.cert_path = "/etc/ssl/certs/mailserver.pem"
-        self.key_path = "/etc/ssl/private/mailserver.key"
-
-    def install(self):
-        apt.packages(
-            name="Install openssl",
-            packages=["openssl"],
-        )
-
-    def configure(self):
-        server.shell(
-            name="Generate self-signed TLS certificate if not present",
-            commands=[
-                f"[ -f {self.cert_path} ] || openssl req -x509"
-                f" -newkey ec -pkeyopt ec_paramgen_curve:P-256"
-                f" -noenc -days 36500"
-                f" -keyout {self.key_path}"
-                f" -out {self.cert_path}"
-                f' -subj "/CN={self.mail_domain}"'
-                f' -addext "extendedKeyUsage=serverAuth,clientAuth"'
-                f' -addext "subjectAltName=DNS:{self.mail_domain},DNS:www.{self.mail_domain},DNS:mta-sts.{self.mail_domain}"',
-            ],
-        )
-
-    def activate(self):
-        pass

cmdeploy/src/cmdeploy/service/chatmail-metadata.service.f

@@ -4,7 +4,7 @@ Description=Chatmail dict proxy for IMAP METADATA
 [Service]
 ExecStart={execpath} /run/chatmail-metadata/metadata.socket {config_path}
 Restart=always
-RestartSec=5
+RestartSec=30
 User=vmail
 RuntimeDirectory=chatmail-metadata
 UMask=0077

cmdeploy/src/cmdeploy/sshexec.py

@@ -85,31 +85,16 @@ class SSHExec:
 
 
 class LocalExec:
-    FuncError = FuncError
-
     def __init__(self, verbose=False, docker=False):
         self.verbose = verbose
         self.docker = docker
 
-    def __call__(self, call, kwargs=None, log_callback=None):
-        if kwargs is None:
-            kwargs = {}
-        return call(**kwargs)
-
     def logged(self, call, kwargs: dict):
-        title = call.__doc__
-        if not title:
-            title = call.__name__
         where = "locally"
         if self.docker:
             if call == remote.rdns.perform_initial_checks:
                 kwargs["pre_command"] = "docker exec chatmail "
                 where = "in docker"
         if self.verbose:
-            print_stderr(f"Running {where}: {title}(**{kwargs})")
+            print(f"Running {where}: {call.__name__}(**{kwargs})")
-            return self(call, kwargs, log_callback=print_stderr)
+        return call(**kwargs)
-        else:
-            print_stderr(title, end="")
-            res = self(call, kwargs, log_callback=remote.rshell.log_progress)
-            print_stderr()
-            return res

cmdeploy/src/cmdeploy/tests/online/test_0_qr.py

@@ -1,4 +1,3 @@
-import pytest
 import requests
 
 from cmdeploy.genqr import gen_qr_png_data
@@ -9,33 +8,18 @@ def test_gen_qr_png_data(maildomain):
     assert data
 
 
-@pytest.mark.filterwarnings("ignore::urllib3.exceptions.InsecureRequestWarning")
 def test_fastcgi_working(maildomain, chatmail_config):
     url = f"https://{maildomain}/new"
     print(url)
-    verify = chatmail_config.tls_cert_mode == "acme"
+    res = requests.post(url)
-    res = requests.post(url, verify=verify)
     assert maildomain in res.json().get("email")
     assert len(res.json().get("password")) > chatmail_config.password_min_length
 
 
-@pytest.mark.filterwarnings("ignore::urllib3.exceptions.InsecureRequestWarning")
+def test_newemail_configure(maildomain, rpc):
-def test_newemail_configure(maildomain, rpc, chatmail_config):
     """Test configuring accounts by scanning a QR code works."""
     url = f"DCACCOUNT:https://{maildomain}/new"
     for i in range(3):
         account_id = rpc.add_account()
-        if chatmail_config.tls_cert_mode == "self":
+        rpc.set_config_from_qr(account_id, url)
-            # deltachat core's rustls rejects self-signed HTTPS certs during
+        rpc.configure(account_id)
-            # set_config_from_qr, so fetch credentials via requests instead
-            res = requests.post(f"https://{maildomain}/new", verify=False)
-            data = res.json()
-            rpc.add_or_update_transport(account_id, {
-                "addr": data["email"],
-                "password": data["password"],
-                "imapServer": maildomain,
-                "smtpServer": maildomain,
-                "certificateChecks": "acceptInvalidCertificates",
-            })
-        else:
-            rpc.add_transport_from_qr(account_id, url)

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -7,13 +7,13 @@ import time
 import pytest
 
 from cmdeploy import remote
-from cmdeploy.cmdeploy import get_sshexec
+from cmdeploy.sshexec import SSHExec
 
 
 class TestSSHExecutor:
     @pytest.fixture(scope="class")
     def sshexec(self, sshdomain):
-        return get_sshexec(sshdomain)
+        return SSHExec(sshdomain)
 
     def test_ls(self, sshexec):
         out = sshexec(call=remote.rdns.shell, kwargs=dict(command="ls"))
@@ -27,7 +27,6 @@ class TestSSHExecutor:
         assert res["A"] or res["AAAA"]
 
     def test_logged(self, sshexec, maildomain, capsys):
-        sshexec.verbose = False
         sshexec.logged(
             remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain)
         )
@@ -53,8 +52,6 @@ class TestSSHExecutor:
                 remote.rdns.perform_initial_checks,
                 kwargs=dict(mail_domain=None),
             )
-        except AssertionError:
-            pass
         except sshexec.FuncError as e:
             assert "rdns.py" in str(e)
             assert "AssertionError" in str(e)
@@ -221,7 +218,7 @@ def test_expunged(remote, chatmail_config):
     ]
     outdated_days = int(chatmail_config.delete_large_after) + 1
     find_cmds.append(
-        f"find {chatmail_config.mailboxes_dir} -path '*/cur/*' -mtime +{outdated_days} -size +200k -type f"
+        "find {chatmail_config.mailboxes_dir} -path '*/cur/*' -mtime +{outdated_days} -size +200k -type f"
     )
     for cmd in find_cmds:
         for line in remote.iter_output(cmd):

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -7,16 +7,15 @@ import pytest
 import requests
 
 from cmdeploy.remote import rshell
-from cmdeploy.cmdeploy import get_sshexec
+from cmdeploy.sshexec import SSHExec
 
 
 @pytest.fixture
-def imap_mailbox(cmfactory, ssl_context):
+def imap_mailbox(cmfactory):
     (ac1,) = cmfactory.get_online_accounts(1)
     user = ac1.get_config("addr")
     password = ac1.get_config("mail_pw")
-    host = user.split("@")[1]
+    mailbox = imap_tools.MailBox(user.split("@")[1])
-    mailbox = imap_tools.MailBox(host, ssl_context=ssl_context)
     mailbox.login(user, password)
     mailbox.dc_ac = ac1
     return mailbox
@@ -91,7 +90,7 @@ class TestEndToEndDeltaChat:
         lp.sec(f"filling remote inbox for {user}")
         fn = f"7743102289.M843172P2484002.c20,S={quota},W=2398:2,"
         path = chatmail_config.mailboxes_dir.joinpath(user, "cur", fn)
-        sshexec = get_sshexec(sshdomain)
+        sshexec = SSHExec(sshdomain)
         sshexec(call=rshell.write_numbytes, kwargs=dict(path=str(path), num=120))
         res = sshexec(call=rshell.dovecot_recalc_quota, kwargs=dict(user=user))
         assert res["percent"] >= 100
@@ -172,7 +171,7 @@ class TestEndToEndDeltaChat:
             time.sleep(1)
 
 
-def test_hide_senders_ip_address(cmfactory, ssl_context):
+def test_hide_senders_ip_address(cmfactory):
     public_ip = requests.get("http://icanhazip.com").content.decode().strip()
     assert ipaddress.ip_address(public_ip)
 
@@ -181,11 +180,6 @@ def test_hide_senders_ip_address(cmfactory, ssl_context):
 
     chat.send_text("testing submission header cleanup")
     user2._evtracker.wait_next_incoming_message()
-    addr = user2.get_config("addr")
+    user2.direct_imap.select_folder("Inbox")
-    host = addr.split("@")[1]
+    msg = user2.direct_imap.get_all_messages()[0]
-    pw = user2.get_config("mail_pw")
+    assert public_ip not in msg.obj.as_string()
-    mailbox = imap_tools.MailBox(host, ssl_context=ssl_context)
-    mailbox.login(addr, pw)
-    msgs = list(mailbox.fetch(mark_seen=False))
-    assert msgs, "expected at least one message"
-    assert public_ip not in msgs[0].obj.as_string()

cmdeploy/src/cmdeploy/tests/online/test_3_status.py

@@ -5,11 +5,7 @@ from cmdeploy.cmdeploy import main
 
 def test_status_cmd(chatmail_config, capsys, request):
     os.chdir(request.config.invocation_params.dir)
-    command = ["status"]
+    assert main(["status"]) == 0
-    if os.getenv("CHATMAIL_SSH"):
-        command.append("--ssh-host")
-        command.append(os.getenv("CHATMAIL_SSH"))
-    assert main(command) == 0
     status_out = capsys.readouterr()
     print(status_out.out)
 

cmdeploy/src/cmdeploy/tests/plugin.py

@@ -4,7 +4,6 @@ import itertools
 import os
 import random
 import smtplib
-import ssl
 import subprocess
 import time
 from pathlib import Path
@@ -55,8 +54,8 @@ def maildomain(chatmail_config):
 
 
 @pytest.fixture(scope="session")
-def sshdomain(chatmail_config):
+def sshdomain(maildomain):
-    return os.environ.get("CHATMAIL_SSH", chatmail_config.ssh_host)
+    return os.environ.get("CHATMAIL_SSH", maildomain)
 
 
 @pytest.fixture
@@ -145,25 +144,15 @@ def pytest_terminal_summary(terminalreporter):
             tr.write_line(line)
 
 
-@pytest.fixture(scope="session")
+@pytest.fixture
-def ssl_context(chatmail_config):
+def imap(maildomain):
-    if chatmail_config.tls_cert_mode == "self":
+    return ImapConn(maildomain)
-        ctx = ssl.create_default_context()
-        ctx.check_hostname = False
-        ctx.verify_mode = ssl.CERT_NONE
-        return ctx
-    return None
 
 
 @pytest.fixture
-def imap(maildomain, ssl_context):
+def make_imap_connection(maildomain):
-    return ImapConn(maildomain, ssl_context=ssl_context)
-
-
-@pytest.fixture
-def make_imap_connection(maildomain, ssl_context):
     def make_imap_connection():
-        conn = ImapConn(maildomain, ssl_context=ssl_context)
+        conn = ImapConn(maildomain)
         conn.connect()
         return conn
 
@@ -175,13 +164,12 @@ class ImapConn:
     logcmd = "journalctl -f -u dovecot"
     name = "dovecot"
 
-    def __init__(self, host, ssl_context=None):
+    def __init__(self, host):
         self.host = host
-        self.ssl_context = ssl_context
 
     def connect(self):
         print(f"imap-connect {self.host}")
-        self.conn = imaplib.IMAP4_SSL(self.host, ssl_context=self.ssl_context)
+        self.conn = imaplib.IMAP4_SSL(self.host)
 
     def login(self, user, password):
         print(f"imap-login {user!r} {password!r}")
@@ -207,14 +195,14 @@ class ImapConn:
 
 
 @pytest.fixture
-def smtp(maildomain, ssl_context):
+def smtp(maildomain):
-    return SmtpConn(maildomain, ssl_context=ssl_context)
+    return SmtpConn(maildomain)
 
 
 @pytest.fixture
-def make_smtp_connection(maildomain, ssl_context):
+def make_smtp_connection(maildomain):
     def make_smtp_connection():
-        conn = SmtpConn(maildomain, ssl_context=ssl_context)
+        conn = SmtpConn(maildomain)
         conn.connect()
         return conn
 
@@ -226,14 +214,12 @@ class SmtpConn:
     logcmd = "journalctl -f -t postfix/smtpd -t postfix/smtp -t postfix/lmtp"
     name = "postfix"
 
-    def __init__(self, host, ssl_context=None):
+    def __init__(self, host):
         self.host = host
-        self.ssl_context = ssl_context
 
     def connect(self):
         print(f"smtp-connect {self.host}")
-        context = self.ssl_context or ssl.create_default_context()
+        self.conn = smtplib.SMTP_SSL(self.host)
-        self.conn = smtplib.SMTP_SSL(self.host, context=context)
 
     def login(self, user, password):
         print(f"smtp-login {user!r} {password!r}")
@@ -284,12 +270,11 @@ def gencreds(chatmail_config):
 class ChatmailTestProcess:
     """Provider for chatmail instance accounts as used by deltachat.testplugin.acfactory"""
 
-    def __init__(self, pytestconfig, maildomain, gencreds, chatmail_config):
+    def __init__(self, pytestconfig, maildomain, gencreds):
         self.pytestconfig = pytestconfig
         self.maildomain = maildomain
         assert "." in self.maildomain, maildomain
         self.gencreds = gencreds
-        self.chatmail_config = chatmail_config
         self._addr2files = {}
 
     def get_liveconfig_producer(self):
@@ -302,9 +287,6 @@ class ChatmailTestProcess:
             # speed up account configuration
             config["mail_server"] = self.maildomain
             config["send_server"] = self.maildomain
-            if self.chatmail_config.tls_cert_mode == "self":
-                # Accept self-signed TLS certificates
-                config["imap_certificate_checks"] = "3"
             yield config
 
     def cache_maybe_retrieve_configured_db_files(self, cache_addr, db_target_path):
@@ -315,14 +297,12 @@ class ChatmailTestProcess:
 
 
 @pytest.fixture
-def cmfactory(request, gencreds, tmpdir, maildomain, chatmail_config):
+def cmfactory(request, gencreds, tmpdir, maildomain):
     # cloned from deltachat.testplugin.amfactory
     pytest.importorskip("deltachat")
     from deltachat.testplugin import ACFactory
 
-    testproc = ChatmailTestProcess(
+    testproc = ChatmailTestProcess(request.config, maildomain, gencreds)
-        request.config, maildomain, gencreds, chatmail_config
-    )
 
     class Data:
         def read_path(self, path):
@@ -330,10 +310,6 @@ def cmfactory(request, gencreds, tmpdir, maildomain, chatmail_config):
 
     am = ACFactory(request=request, tmpdir=tmpdir, testprocess=testproc, data=Data())
 
-    # Skip upstream's init_imap to prevent extra imap connections not
-    # needed for relay testing
-    am._acsetup.init_imap = lambda acc: None
-
     # nb. a bit hacky
     # would probably be better if deltachat's test machinery grows native support
     def switch_maildomain(maildomain2):
@@ -361,14 +337,8 @@ class Remote:
 
     def iter_output(self, logcmd=""):
         getjournal = "journalctl -f" if not logcmd else logcmd
-        print(self.sshdomain)
-        match self.sshdomain:
-            case "@local": command = []
-            case "localhost": command = []
-            case _: command = ["ssh", f"root@{self.sshdomain}"]
-        [command.append(arg) for arg in getjournal.split()]
         self.popen = subprocess.Popen(
-            command,
+            ["ssh", f"root@{self.sshdomain}", getjournal],
             stdout=subprocess.PIPE,
         )
         while 1:
@@ -393,40 +363,38 @@ def lp(request):
 
 
 @pytest.fixture
-def cmsetup(maildomain, gencreds, ssl_context):
+def cmsetup(maildomain, gencreds):
-    return CMSetup(maildomain, gencreds, ssl_context)
+    return CMSetup(maildomain, gencreds)
 
 
 class CMSetup:
-    def __init__(self, maildomain, gencreds, ssl_context):
+    def __init__(self, maildomain, gencreds):
         self.maildomain = maildomain
         self.gencreds = gencreds
-        self.ssl_context = ssl_context
 
     def gen_users(self, num):
         print(f"Creating {num} online users")
         users = []
         for i in range(num):
             addr, password = self.gencreds()
-            user = CMUser(self.maildomain, addr, password, self.ssl_context)
+            user = CMUser(self.maildomain, addr, password)
             assert user.smtp
             users.append(user)
         return users
 
 
 class CMUser:
-    def __init__(self, maildomain, addr, password, ssl_context=None):
+    def __init__(self, maildomain, addr, password):
         self.maildomain = maildomain
         self.addr = addr
         self.password = password
-        self.ssl_context = ssl_context
         self._smtp = None
         self._imap = None
 
     @property
     def smtp(self):
         if not self._smtp:
-            handle = SmtpConn(self.maildomain, ssl_context=self.ssl_context)
+            handle = SmtpConn(self.maildomain)
             handle.connect()
             handle.login(self.addr, self.password)
             self._smtp = handle
@@ -435,7 +403,7 @@ class CMUser:
     @property
     def imap(self):
         if not self._imap:
-            imap = ImapConn(self.maildomain, ssl_context=self.ssl_context)
+            imap = ImapConn(self.maildomain)
             imap.connect()
             imap.login(self.addr, self.password)
             self._imap = imap

cmdeploy/src/cmdeploy/tests/test_dns.py

@@ -91,16 +91,6 @@ class TestPerformInitialChecks:
         assert not res
         assert len(l) == 2
 
-    def test_perform_initial_checks_no_mta_sts_self_signed(self, mockdns):
-        del mockdns["CNAME"]["mta-sts.some.domain"]
-        remote_data = remote.rdns.perform_initial_checks("some.domain")
-        assert not remote_data["MTA_STS"]
-
-        l = []
-        res = check_initial_remote_data(remote_data, strict_tls=False, print=l.append)
-        assert res
-        assert not l
-
 
 def parse_zonefile_into_dict(zonefile, mockdns_base, only_required=False):
     for zf_line in zonefile.split("\n"):

doc/source/getting_started.rst

@@ -16,11 +16,18 @@ You will need the following:
 
 -  Control over a domain through a DNS provider of your choice.
 
--  A Debian 12 server with reachable SMTP/SUBMISSIONS/IMAPS/HTTPS ports.
+-  A Debian 12 **deployment server** with reachable SMTP/SUBMISSIONS/IMAPS/HTTPS ports.
    IPv6 is encouraged if available. Chatmail relay servers only require
    1GB RAM, one CPU, and perhaps 10GB storage for a few thousand active
    chatmail addresses.
 
+-  A Linux or Unix **build machine** with key-based SSH access to the root
+   user of the deployment server.
+   You must add a passphrase-protected private key to your local ssh-agent because you
+   can’t type in your passphrase during deployment.
+   (An ed25519 private key is required due to an `upstream bug in
+   paramiko <https://github.com/paramiko/paramiko/issues/2191>`_)
+
 
 Setup with ``scripts/cmdeploy``
 -------------------------------------
@@ -28,7 +35,7 @@ Setup with ``scripts/cmdeploy``
 We use ``chat.example.org`` as the chatmail domain in the following
 steps. Please substitute it with your own domain.
 
-1. Setup the initial DNS records for your relay.
+1. Setup the initial DNS records for your deployment server.
    The following is an example in the
    familiar BIND zone file format with a TTL of 1 hour (3600 seconds).
    Please substitute your domain and IP addresses.
@@ -40,44 +47,29 @@ steps. Please substitute it with your own domain.
        www.chat.example.org. 3600 IN CNAME chat.example.org.
        mta-sts.chat.example.org. 3600 IN CNAME chat.example.org.
 
-   .. note::
+2. On your local PC, clone the repository and bootstrap the Python
-
-      For experimental deployments using self-signed certificates,
-      use a domain name starting with ``_``
-      (e.g. ``_chat.example.org``).
-      The ``mta-sts`` CNAME and ``_mta-sts`` TXT records
-      are not needed for such domains.
-
-2. Login to the server with SSH, clone the repository and bootstrap the Python
    virtualenv.
 
    ::
 
-       ssh root@chat.example.org
        git clone https://github.com/chatmail/relay
        cd relay
        scripts/initenv.sh
 
-3. Then, create a chatmail configuration file
+3. On your local build machine (PC), create a chatmail configuration file
    ``chatmail.ini``:
 
    ::
 
        scripts/cmdeploy init chat.example.org  # <-- use your domain
 
-   .. note::
+4. Verify that SSH root login to the deployment server server works:
 
-   To use self-signed TLS certificates
+   ::
-   instead of Let's Encrypt,
-   use a domain name starting with ``_``
-   (e.g. ``scripts/cmdeploy init _chat.example.org``).
-   Domains starting with ``_`` cannot obtain WebPKI certificates,
-   so self-signed mode is derived automatically.
-   This is useful for private or test deployments.
-   See the :doc:`overview`
-   for details on certificate provisioning.
 
-4. Now run the deployment script to install the relay to the server:
+       ssh root@chat.example.org  # <-- use your domain
+
+5. From your local build machine, setup and configure the remote deployment server:
 
    ::
 
@@ -88,31 +80,26 @@ steps. Please substitute it with your own domain.
    configure at your DNS provider (it can take some time until they are
    public).
 
-Next Steps
+Other helpful commands
-----------
-
-Now you should display and check all recommended DNS records
-to enable federation with other relays:
-
-::
-
-   scripts/cmdeploy dns
-
-You should also test whether your chatmail service is working correctly:
-
-::
-
-   scripts/cmdeploy test
-
-Other Helpful Commands
 ----------------------
 
-To check the status of your chatmail relay:
+To check the status of your deployment server running the chatmail service:
 
 ::
 
    scripts/cmdeploy status
 
+To display and check all recommended DNS records:
+
+::
+
+   scripts/cmdeploy dns
+
+To test whether your chatmail service is working correctly:
+
+::
+
+   scripts/cmdeploy test
 
 To measure the performance of your chatmail service:
 
@@ -154,9 +141,8 @@ This starts a local live development cycle for chatmail web pages:
    directory and generating HTML files and copying assets to the
    ``www/build`` directory.
 
--  if you are running scripts/cmdeploy webdev on the relay itself,
+-  Starts a browser window automatically where you can “refresh” as
-   you need to configure a route in /etc/nginx/nginx.conf
+   needed.
-   to expose the build directory.
 
 Custom web pages
 ----------------
@@ -174,7 +160,7 @@ Disable automatic address creation
 --------------------------------------------------------
 
 If you need to stop address creation, e.g. because some script is wildly
-creating addresses, login with ssh to the relay and run:
+creating addresses, login with ssh to the deployment machine and run:
 
 ::
 
@@ -183,14 +169,23 @@ creating addresses, login with ssh to the relay and run:
 Chatmail address creation will be denied while this file is present.
 
 
-Running a relay with self-signed certificates
+Migrating to a new build machine
-----------------------------------------------
+----------------------------------
 
-Use a domain name starting with ``_`` (e.g. ``_chat.example.org``)
+To move or add a build machine,
-to run a relay with self-signed certificates.
+clone the relay repository on the new build machine, and copy the ``chatmail.ini`` file from the old build machine.
-Domains starting with ``_`` cannot obtain WebPKI certificates
+Make sure ``rsync`` is installed, then initialize the environment:
-so the relay automatically uses self-signed certificates
-and all other relays will accept connections from it
-without requiring certificate verification.
-This is useful for experimental setups and testing.
 
+::
+
+   ./scripts/initenv.sh
+
+Run safety checks before a new deployment:
+
+::
+
+   ./scripts/cmdeploy dns
+   ./scripts/cmdeploy status
+
+If you keep multiple build machines (ie laptop and desktop), keep ``chatmail.ini`` in sync between
+them.

doc/source/overview.rst

@@ -297,7 +297,8 @@ TLS requirements
 
 Postfix is configured to require valid TLS by setting
 `smtp_tls_security_level <https://www.postfix.org/postconf.5.html#smtp_tls_security_level>`_
-to ``verify``.
+to ``verify``. If emails don’t arrive at your chatmail relay server, the
+problem is likely that your relay does not have a valid TLS certificate.
 
 You can test it by resolving ``MX`` records of your relay domain and
 then connecting to MX relays (e.g ``mx.example.org``) with
@@ -316,14 +317,6 @@ default Exim does not log sessions that are closed before sending the
 by Postfix, so you might think that connection is not established while
 actually it is a problem with your TLS certificate.
 
-If emails don’t arrive at your chatmail relay server, the
-problem is likely that your relay does not have a valid TLS certificate.
-
-Note that connections to relays with underscore-prefixed test domains
-(e.g. ``_chat.example.org``) use ``encrypt`` tls security level,
-because such domains cannot obtain valid Let's Encrypt certificates
-and run with self-signed certificates.
-
 
 .. _dovecot: https://dovecot.org
 .. _postfix: https://www.postfix.org

doc/source/related.rst

@@ -14,8 +14,8 @@ We know of three work-in-progress alternative implementation efforts:
    it to support all of the features and configuration settings required
    to operate as a chatmail relay.
 
--  `Madmail <https://github.com/themadorg/madmail>`_: an
+-  `Madmail <https://github.com/omidz4t/madmail>`_: an
-   experimental fork of `Maddy Mail Server <https://maddy.email/>`_, modified
+   experimental fork of Maddy Mail Server <https://maddy.email/>`_ optimized
    for chatmail deployments.  It provides a single binary solution
    for running a chatmail relay.
 

www/src/dclogin.js

@@ -1,21 +0,0 @@
-/* dclogin profile generator for self-signed chatmail relays.
- * Fetches credentials from /new and generates a dclogin: QR code.
- * Requires qrcode-svg.min.js to be loaded first.
- */
-(function () {
-    function generateProfile() {
-        fetch('/new')
-            .then(function (r) { return r.json(); })
-            .then(function (data) {
-                var url = data.dclogin_url;
-                var link = document.getElementById('dclogin-link');
-                link.href = url;
-                var qrLink = document.getElementById('qr-link');
-                qrLink.href = url;
-                var qrCode = document.getElementById('qr-code');
-                var qr = new QRCode({ content: url, width: 300, height: 300, padding: 1, join: true });
-                qrCode.innerHTML = qr.svg();
-            });
-    }
-    generateProfile();
-})();

www/src/index.md

@@ -11,18 +11,6 @@ for Delta Chat users.  For details how it avoids storing personal information
 please see our [privacy policy](privacy.html). 
 {% endif %}
 
-{% if config.tls_cert_mode == "self" %}
-<a class="cta-button" id="dclogin-link" href="#">Get a {{config.mail_domain}} chat profile</a>
-
-If you are viewing this page on a different device
-without a Delta Chat app,
-you can also **scan this QR code** with Delta Chat:
-
-<a id="qr-link" href="#"><div id="qr-code"></div></a>
-
-<script src="qrcode-svg.min.js"></script>
-<script src="dclogin.js"></script>
-{% else %}
 <a class="cta-button" href="DCACCOUNT:https://{{ config.mail_domain }}/new">Get a {{config.mail_domain}} chat profile</a>
 
 If you are viewing this page on a different device
@@ -31,7 +19,6 @@ you can also **scan this QR code** with Delta Chat:
 
 <a href="DCACCOUNT:https://{{ config.mail_domain }}/new">
     <img width=300 style="float: none;" src="qr-chatmail-invite-{{config.mail_domain}}.png" /></a>
-{% endif %}
 
 🐣 **Choose** your Avatar and Name