docker: make compose work with cgroups (v2), conversion scripts/docs

opendkim-genkey was running unconditionally on every startup,
check if file exists and skip.

.dockerignore

@@ -0,0 +1,7 @@
+.git
+data/
+venv/
+__pycache__
+*.pyc
+*.orig
+.pytest_cache

.github/workflows/ci.yaml

@@ -15,7 +15,7 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: download filtermail
-        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.5.1/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
+        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.3.0/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
       - name: run chatmaild tests 
         working-directory: chatmaild
         run: pipx run tox

.github/workflows/test-and-deploy-ipv4only.yaml

@@ -71,35 +71,26 @@ jobs:
       - name: run deploy-chatmail offline tests 
         run: pytest --pyargs cmdeploy 
 
-      - name: setup dependencies
-        run: |
-          ssh root@staging-ipv4.testrun.org apt update
-          ssh root@staging-ipv4.testrun.org apt install -y git python3.11-venv python3-dev gcc
-          ssh root@staging-ipv4.testrun.org git clone https://github.com/chatmail/relay
-          ssh root@staging-ipv4.testrun.org "cd relay && git checkout " ${{ github.head_ref }}
-          ssh root@staging-ipv4.testrun.org "cd relay && scripts/initenv.sh"
+      - run: |
+          cmdeploy init staging-ipv4.testrun.org
+          sed -i 's#disable_ipv6 = False#disable_ipv6 = True#' chatmail.ini
+          sed -i 's/#\s*mtail_address/mtail_address/' chatmail.ini
 
-      - name: initialize config
-        run: |
-          ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy init staging-ipv4.testrun.org"
-          ssh root@staging-ipv4.testrun.org "sed -i 's#disable_ipv6 = False#disable_ipv6 = True#' relay/chatmail.ini"
-          ssh root@staging-ipv4.testrun.org "sed -i 's/#\s*mtail_address/mtail_address/' relay/chatmail.ini"
-
-      - run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy run --verbose --skip-dns-check --ssh-host localhost"
+      - run: cmdeploy run --verbose --skip-dns-check
 
       - name: set DNS entries
         run: |
-          ssh root@staging-ipv4.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
-          ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns --zonefile staging-generated.zone --ssh-host localhost"
-          ssh root@staging-ipv4.testrun.org cat relay/staging-generated.zone >> .github/workflows/staging-ipv4.testrun.org-default.zone
+          ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
+          cmdeploy dns --zonefile staging-generated.zone
+          cat staging-generated.zone >> .github/workflows/staging-ipv4.testrun.org-default.zone
           cat .github/workflows/staging-ipv4.testrun.org-default.zone
           scp .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
           ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
           ssh root@ns.testrun.org systemctl reload nsd
 
       - name: cmdeploy test
-        run: ssh root@staging-ipv4.testrun.org "cd relay && CHATMAIL_DOMAIN2=ci-chatmail.testrun.org scripts/cmdeploy test --slow --ssh-host localhost"
+        run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
 
       - name: cmdeploy dns
-        run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns -v --ssh-host localhost"
+        run: cmdeploy dns -v
 

.gitignore

@@ -4,7 +4,7 @@ __pycache__/
 *$py.class
 *.swp
 *qr-*.png
-chatmail*.ini
+chatmail.ini
 
 
 # C extensions
@@ -164,3 +164,8 @@ cython_debug/
 #.idea/
 
 chatmail.zone
+
+# docker
+/data/
+/custom/
+.env

CHANGELOG.md

@@ -121,6 +121,13 @@
   Provide an "fsreport" CLI for more fine grained analysis of message files. 
   ([#637](https://github.com/chatmail/relay/pull/637))
 
+- Add installation via docker compose (MVP 1). The instructions, known issues and limitations are located in `/docs` 
+  ([#614](https://github.com/chatmail/relay/pull/614))
+
+- Add configuration parameters
+  ([#614](https://github.com/chatmail/relay/pull/614)):
+  - `change_kernel_settings` - Whether to change kernel parameters during installation (default: `True`)
+  - `fs_inotify_max_user_instances_and_watchers` - Value for kernel parameters `fs.inotify.max_user_instances` and `fs.inotify.max_user_watches` (default: `65535`)
 
 ## 1.7.0 2025-09-11
 

chatmaild/src/chatmaild/config.py

@@ -47,6 +47,12 @@ class Config:
         self.addr_v4 = os.environ.get("CHATMAIL_ADDR_V4", "")
         self.addr_v6 = os.environ.get("CHATMAIL_ADDR_V6", "")
         self.acme_email = params.get("acme_email", "")
+        self.change_kernel_settings = (
+            params.get("change_kernel_settings", "true").lower() == "true"
+        )
+        self.fs_inotify_max_user_instances_and_watchers = int(
+            params["fs_inotify_max_user_instances_and_watchers"]
+        )
         self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true"
         self.imap_compress = params.get("imap_compress", "false").lower() == "true"
         if "iroh_relay" not in params:
@@ -60,31 +66,6 @@ class Config:
         self.privacy_pdo = params.get("privacy_pdo")
         self.privacy_supervisor = params.get("privacy_supervisor")
 
-        # TLS certificate management.
-        # If tls_external_cert_and_key is set, use externally managed certs.
-        # Otherwise derived from the domain name:
-        # - Domains starting with "_" use self-signed certificates
-        # - All other domains use ACME.
-        external = params.get("tls_external_cert_and_key", "").strip()
-
-        if external:
-            parts = external.split()
-            if len(parts) != 2:
-                raise ValueError(
-                    "tls_external_cert_and_key must have two space-separated"
-                    " paths: CERT_PATH KEY_PATH"
-                )
-            self.tls_cert_mode = "external"
-            self.tls_cert_path, self.tls_key_path = parts
-        elif self.mail_domain.startswith("_"):
-            self.tls_cert_mode = "self"
-            self.tls_cert_path = "/etc/ssl/certs/mailserver.pem"
-            self.tls_key_path = "/etc/ssl/private/mailserver.key"
-        else:
-            self.tls_cert_mode = "acme"
-            self.tls_cert_path = f"/var/lib/acme/live/{self.mail_domain}/fullchain"
-            self.tls_key_path = f"/var/lib/acme/live/{self.mail_domain}/privkey"
-
         # deprecated option
         mbdir = params.get("mailboxes_dir", f"/home/vmail/mail/{self.mail_domain}")
         self.mailboxes_dir = Path(mbdir.strip())

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -48,13 +48,6 @@ passthrough_senders =
 # (space-separated, item may start with "@" to whitelist whole recipient domains)
 passthrough_recipients =
 
-# Use externally managed TLS certificates instead of built-in acmetool.
-# Paths refer to files on the deployment server (not the build machine).
-# Both files must already exist before running cmdeploy.
-# Certificate renewal is your responsibility; changed files are
-# picked up automatically by all relay services.
-# tls_external_cert_and_key = /path/to/fullchain.pem /path/to/privkey.pem
-
 # path to www directory - documented here: https://chatmail.at/doc/relay/getting_started.html#custom-web-pages
 #www_folder = www
 
@@ -76,6 +69,16 @@ disable_ipv6 = False
 # Your email adress, which will be used in acmetool to manage Let's Encrypt SSL certificates
 acme_email = 
 
+#
+# Kernel settings
+#
+
+# if you set "True", the kernel settings will be configured according to the values below
+change_kernel_settings  = True
+
+# change fs.inotify.max_user_instances and fs.inotify.max_user_watches kernel settings
+fs_inotify_max_user_instances_and_watchers = 65535
+
 # Defaults to https://iroh.{{mail_domain}} and running `iroh-relay` on the chatmail
 # service.
 # If you set it to anything else, the service will be disabled

chatmaild/src/chatmaild/newemail.py

@@ -6,7 +6,6 @@ import json
 import random
 import secrets
 import string
-from urllib.parse import quote
 
 from chatmaild.config import Config, read_config
 
@@ -24,26 +23,13 @@ def create_newemail_dict(config: Config):
     return dict(email=f"{user}@{config.mail_domain}", password=f"{password}")
 
 
-def create_dclogin_url(email, password):
-    """Build a dclogin: URL with credentials and self-signed cert acceptance.
-
-    Uses ic=3 (AcceptInvalidCertificates) so chatmail clients
-    can connect to servers with self-signed TLS certificates.
-    """
-    return f"dclogin:{quote(email, safe='@')}?p={quote(password, safe='')}&v=1&ic=3"
-
-
 def print_new_account():
     config = read_config(CONFIG_PATH)
     creds = create_newemail_dict(config)
 
-    result = dict(email=creds["email"], password=creds["password"])
-    if config.tls_cert_mode == "self":
-        result["dclogin_url"] = create_dclogin_url(creds["email"], creds["password"])
-
     print("Content-Type: application/json")
     print("")
-    print(json.dumps(result))
+    print(json.dumps(creds))
 
 
 if __name__ == "__main__":

chatmaild/src/chatmaild/tests/test_config.py

@@ -73,51 +73,3 @@ def test_config_userstate_paths(make_config, tmp_path):
 def test_config_max_message_size(make_config, tmp_path):
     config = make_config("something.testrun.org", dict(max_message_size="10000"))
     assert config.max_message_size == 10000
-
-
-def test_config_tls_default_acme(make_config):
-    config = make_config("chat.example.org")
-    assert config.tls_cert_mode == "acme"
-    assert config.tls_cert_path == "/var/lib/acme/live/chat.example.org/fullchain"
-    assert config.tls_key_path == "/var/lib/acme/live/chat.example.org/privkey"
-
-
-def test_config_tls_self(make_config):
-    config = make_config("_test.example.org")
-    assert config.tls_cert_mode == "self"
-    assert config.tls_cert_path == "/etc/ssl/certs/mailserver.pem"
-    assert config.tls_key_path == "/etc/ssl/private/mailserver.key"
-
-
-def test_config_tls_external(make_config):
-    config = make_config(
-        "chat.example.org",
-        {
-            "tls_external_cert_and_key": "/custom/fullchain.pem /custom/privkey.pem",
-        },
-    )
-    assert config.tls_cert_mode == "external"
-    assert config.tls_cert_path == "/custom/fullchain.pem"
-    assert config.tls_key_path == "/custom/privkey.pem"
-
-
-def test_config_tls_external_overrides_underscore(make_config):
-    config = make_config(
-        "_test.example.org",
-        {
-            "tls_external_cert_and_key": "/certs/fullchain.pem /certs/privkey.pem",
-        },
-    )
-    assert config.tls_cert_mode == "external"
-    assert config.tls_cert_path == "/certs/fullchain.pem"
-    assert config.tls_key_path == "/certs/privkey.pem"
-
-
-def test_config_tls_external_bad_format(make_config):
-    with pytest.raises(ValueError, match="two space-separated"):
-        make_config(
-            "chat.example.org",
-            {
-                "tls_external_cert_and_key": "/only/one/path.pem",
-            },
-        )

chatmaild/src/chatmaild/tests/test_filtermail_blackbox.py

@@ -1,15 +1,9 @@
-import shutil
 import smtplib
 import subprocess
 import sys
 
 import pytest
 
-pytestmark = pytest.mark.skipif(
-    shutil.which("filtermail") is None,
-    reason="filtermail binary not found",
-)
-
 
 @pytest.fixture
 def smtpserver():
@@ -47,8 +41,6 @@ def test_one_mail(
     make_config, make_popen, smtpserver, maildata, filtermail_mode, monkeypatch
 ):
     monkeypatch.setenv("PYTHONUNBUFFERED", "1")
-    # DKIM is tested by cmdeploy tests.
-    monkeypatch.setenv("FILTERMAIL_SKIP_DKIM", "1")
     smtp_inject_port = 20025
     if filtermail_mode == "outgoing":
         settings = dict(
@@ -66,10 +58,6 @@ def test_one_mail(
 
     popen = make_popen(["filtermail", path, filtermail_mode])
     line = popen.stderr.readline().strip()
-
-    # skip a warning that FILTERMAIL_SKIP_DKIM shouldn't be used in prod
-    if b"DKIM verification DISABLED!" in line:
-        line = popen.stderr.readline().strip()
     if b"loop" not in line:
         print(line.decode("ascii"), file=sys.stderr)
         pytest.fail("starting filtermail failed")

chatmaild/src/chatmaild/tests/test_newmail.py

@@ -1,11 +1,7 @@
 import json
 
 import chatmaild
-from chatmaild.newemail import (
-    create_dclogin_url,
-    create_newemail_dict,
-    print_new_account,
-)
+from chatmaild.newemail import create_newemail_dict, print_new_account
 
 
 def test_create_newemail_dict(example_config):
@@ -19,18 +15,6 @@ def test_create_newemail_dict(example_config):
     assert ac1["password"] != ac2["password"]
 
 
-def test_create_dclogin_url():
-    url = create_dclogin_url("user@example.org", "p@ss w+rd")
-    assert url.startswith("dclogin:")
-    assert "v=1" in url
-    assert "ic=3" in url
-
-    assert "user@example.org" in url
-    # password special chars must be encoded
-    assert "p%40ss" in url
-    assert "w%2Brd" in url
-
-
 def test_print_new_account(capsys, monkeypatch, maildomain, tmpdir, example_config):
     monkeypatch.setattr(chatmaild.newemail, "CONFIG_PATH", str(example_config._inipath))
     print_new_account()
@@ -41,20 +25,3 @@ def test_print_new_account(capsys, monkeypatch, maildomain, tmpdir, example_conf
     dic = json.loads(lines[2])
     assert dic["email"].endswith(f"@{example_config.mail_domain}")
     assert len(dic["password"]) >= 10
-    # default tls_cert=acme should not include dclogin_url
-    assert "dclogin_url" not in dic
-
-
-def test_print_new_account_self_signed(capsys, monkeypatch, make_config):
-    config = make_config("_test.example.org")
-    monkeypatch.setattr(chatmaild.newemail, "CONFIG_PATH", str(config._inipath))
-    print_new_account()
-    out, err = capsys.readouterr()
-    lines = out.split("\n")
-    dic = json.loads(lines[2])
-    assert "dclogin_url" in dic
-    url = dic["dclogin_url"]
-    assert url.startswith("dclogin:")
-    assert "ic=3" in url
-
-    assert dic["email"].split("@")[0] in url

cmdeploy/pyproject.toml

@@ -20,7 +20,6 @@ dependencies = [
   "pytest-xdist", 
   "execnet",
   "imap_tools",
-  "deltachat-rpc-client",
 ]
 
 [project.scripts]

cmdeploy/src/cmdeploy/acmetool/acmetool-redirector.service

@@ -3,7 +3,7 @@ Description=acmetool HTTP redirector
 
 [Service]
 Type=notify
-ExecStart=/usr/bin/acmetool redirector --service.uid=daemon --bind=127.0.0.1:402
+ExecStart=/usr/bin/acmetool redirector --service.uid=daemon
 Restart=always
 RestartSec=30
 

cmdeploy/src/cmdeploy/chatmail.zone.j2

@@ -8,10 +8,8 @@
 {{ mail_domain }}.                   AAAA  {{ AAAA }}
 {% endif %}
 {{ mail_domain }}.                   MX 10 {{ mail_domain }}.
-{% if strict_tls %}
 _mta-sts.{{ mail_domain }}.          TXT "v=STSv1; id={{ sts_id }}"
 mta-sts.{{ mail_domain }}.           CNAME {{ mail_domain }}.
-{% endif %}
 www.{{ mail_domain }}.               CNAME {{ mail_domain }}.
 {{ dkim_entry }}
 

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -5,6 +5,7 @@ along with command line option and subcommand parsing.
 
 import argparse
 import importlib.resources
+import importlib.util
 import os
 import pathlib
 import shutil
@@ -90,10 +91,9 @@ def run_cmd(args, out):
     ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
     sshexec = get_sshexec(ssh_host)
     require_iroh = args.config.enable_iroh_relay
-    strict_tls = args.config.tls_cert_mode == "acme"
     if not args.dns_check_disabled:
         remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
-        if not dns.check_initial_remote_data(remote_data, strict_tls=strict_tls, print=out.red):
+        if not dns.check_initial_remote_data(remote_data, print=out.red):
             return 1
 
     env = os.environ.copy()
@@ -110,8 +110,7 @@ def run_cmd(args, out):
     cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y"
     if ssh_host in ["localhost", "@docker"]:
         if ssh_host == "@docker":
-            env["CHATMAIL_NOPORTCHECK"] = "True"
-            env["CHATMAIL_NOSYSCTL"] = "True"
+            env["CHATMAIL_DOCKER"] = "True"
         cmd = f"{pyinf} @local {deploy_path} -y"
 
     if version.parse(pyinfra.__version__) < version.parse("3"):
@@ -127,7 +126,7 @@ def run_cmd(args, out):
                 out.red("Website deployment failed.")
         elif retcode == 0:
             out.green("Deploy completed, call `cmdeploy dns` next.")
-        elif not args.dns_check_disabled and strict_tls and not remote_data["acme_account_url"]:
+        elif not args.dns_check_disabled and not remote_data["acme_account_url"]:
             out.red("Deploy completed but letsencrypt not configured")
             out.red("Run 'cmdeploy run' again")
             retcode = 0
@@ -154,13 +153,11 @@ def dns_cmd(args, out):
     """Check DNS entries and optionally generate dns zone file."""
     ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
     sshexec = get_sshexec(ssh_host, verbose=args.verbose)
-    tls_cert_mode = args.config.tls_cert_mode
-    strict_tls = tls_cert_mode == "acme"
     remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
-    if not dns.check_initial_remote_data(remote_data, strict_tls=strict_tls):
+    if not remote_data:
         return 1
 
-    if strict_tls and not remote_data["acme_account_url"]:
+    if not remote_data["acme_account_url"]:
         out.red("could not get letsencrypt account url, please run 'cmdeploy run'")
         return 1
 
@@ -168,7 +165,6 @@ def dns_cmd(args, out):
         out.red("could not determine dkim_entry, please run 'cmdeploy run'")
         return 1
 
-    remote_data["strict_tls"] = strict_tls
     zonefile = dns.get_filled_zone_file(remote_data)
 
     if args.zonefile:
@@ -209,15 +205,17 @@ def test_cmd_options(parser):
         action="store_true",
         help="also run slow tests",
     )
-    add_ssh_host_option(parser)
 
 
 def test_cmd(args, out):
-    """Run local and online tests for chatmail deployment."""
+    """Run local and online tests for chatmail deployment.
 
-    env = os.environ.copy()
-    if args.ssh_host:
-        env["CHATMAIL_SSH"] = args.ssh_host
+    This will automatically pip-install 'deltachat' if it's not available.
+    """
+
+    x = importlib.util.find_spec("deltachat")
+    if x is None:
+        out.check_call(f"{sys.executable} -m pip install deltachat")
 
     pytest_path = shutil.which("pytest")
     pytest_args = [
@@ -231,7 +229,7 @@ def test_cmd(args, out):
     ]
     if args.slow:
         pytest_args.append("--slow")
-    ret = out.run_ret(pytest_args, env=env)
+    ret = out.run_ret(pytest_args)
     return ret
 
 
@@ -332,7 +330,7 @@ def add_config_option(parser):
         "--config",
         dest="inipath",
         action="store",
-        default=Path(os.environ.get("CHATMAIL_INI", "chatmail.ini")),
+        default=Path("chatmail.ini"),
         type=Path,
         help="path to the chatmail.ini file",
     )

cmdeploy/src/cmdeploy/deployers.py

@@ -2,7 +2,6 @@
 Chat Mail pyinfra deploy.
 """
 
-import os
 import shutil
 import subprocess
 import sys
@@ -11,8 +10,8 @@ from pathlib import Path
 
 from chatmaild.config import read_config
 from pyinfra import facts, host, logger
-from pyinfra.api import FactBase
 from pyinfra.facts import hardware
+from pyinfra.api import FactBase
 from pyinfra.facts.files import Sha256File
 from pyinfra.facts.systemd import SystemdEnabled
 from pyinfra.operations import apt, files, pip, server, systemd
@@ -29,13 +28,11 @@ from .basedeploy import (
     has_systemd,
 )
 from .dovecot.deployer import DovecotDeployer
-from .external.deployer import ExternalTlsDeployer
 from .filtermail.deployer import FiltermailDeployer
 from .mtail.deployer import MtailDeployer
 from .nginx.deployer import NginxDeployer
 from .opendkim.deployer import OpendkimDeployer
 from .postfix.deployer import PostfixDeployer
-from .selfsigned.deployer import SelfSignedTlsDeployer
 from .www import build_webpages, find_merge_conflict, get_paths
 
 
@@ -541,26 +538,13 @@ class GithashDeployer(Deployer):
         )
 
 
-def get_tls_deployer(config, mail_domain):
-    """Select the appropriate TLS deployer based on config."""
-    tls_domains = [mail_domain, f"mta-sts.{mail_domain}", f"www.{mail_domain}"]
-
-    if config.tls_cert_mode == "acme":
-        return AcmetoolDeployer(config.acme_email, tls_domains)
-    elif config.tls_cert_mode == "self":
-        return SelfSignedTlsDeployer(mail_domain)
-    elif config.tls_cert_mode == "external":
-        return ExternalTlsDeployer(config.tls_cert_path, config.tls_key_path)
-    else:
-        raise ValueError(f"Unknown tls_cert_mode: {config.tls_cert_mode}")
-
-
-def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -> None:
+def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool, docker: bool) -> None:
     """Deploy a chat-mail instance.
 
     :param config_path: path to chatmail.ini
     :param disable_mail: whether to disable postfix & dovecot
     :param website_only: if True, only deploy the website
+    :param docker: whether it is running in a docker container
     """
     config = read_config(config_path)
     check_config(config)
@@ -586,19 +570,12 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
             Out().red(f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n")
             exit(1)
 
-    if not os.environ.get("CHATMAIL_NOPORTCHECK"):
+    if not docker:
         port_services = [
             (["master", "smtpd"], 25),
             ("unbound", 53),
-        ]
-        if config.tls_cert_mode == "acme":
-            port_services.append(("acmetool", 402))
-        port_services += [
+            ("acmetool", 80),
             (["imap-login", "dovecot"], 143),
-            # acmetool previously listened on port 80,
-            # so don't complain during upgrade that moved it to port 402
-            # and gave the port to nginx.
-            (["acmetool", "nginx"], 80),
             ("nginx", 443),
             (["master", "smtpd"], 465),
             (["master", "smtpd"], 587),
@@ -623,7 +600,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
                     )
                     exit(1)
 
-    tls_deployer = get_tls_deployer(config, mail_domain)
+    tls_domains = [mail_domain, f"mta-sts.{mail_domain}", f"www.{mail_domain}"]
 
     all_deployers = [
         ChatmailDeployer(mail_domain),
@@ -633,7 +610,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
         UnboundDeployer(config),
         TurnDeployer(mail_domain),
         IrohDeployer(config.enable_iroh_relay),
-        tls_deployer,
+        AcmetoolDeployer(config.acme_email, tls_domains),
         WebsiteDeployer(config),
         ChatmailVenvDeployer(config),
         MtastsDeployer(),

cmdeploy/src/cmdeploy/dns.py

@@ -12,14 +12,14 @@ def get_initial_remote_data(sshexec, mail_domain):
     )
 
 
-def check_initial_remote_data(remote_data, *, strict_tls=True, print=print):
+def check_initial_remote_data(remote_data, *, print=print):
     mail_domain = remote_data["mail_domain"]
     if not remote_data["A"] and not remote_data["AAAA"]:
         print(f"Missing A and/or AAAA DNS records for {mail_domain}!")
-    elif strict_tls and remote_data["MTA_STS"] != f"{mail_domain}.":
+    elif remote_data["MTA_STS"] != f"{mail_domain}.":
         print("Missing MTA-STS CNAME record:")
         print(f"mta-sts.{mail_domain}.   CNAME  {mail_domain}.")
-    elif strict_tls and remote_data["WWW"] != f"{mail_domain}.":
+    elif remote_data["WWW"] != f"{mail_domain}.":
         print("Missing www CNAME record:")
         print(f"www.{mail_domain}.   CNAME  {mail_domain}.")
     else:

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -1,5 +1,3 @@
-import os
-
 from chatmaild.config import Config
 from pyinfra import host
 from pyinfra.facts.server import Arch, Sysctl
@@ -120,7 +118,7 @@ def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
 
     # as per https://doc.dovecot.org/2.3/configuration_manual/os/
     # it is recommended to set the following inotify limits
-    if not os.environ.get("CHATMAIL_NOSYSCTL"):
+    if config.change_kernel_settings:
         for name in ("max_user_instances", "max_user_watches"):
             key = f"fs.inotify.{name}"
             if host.get_fact(Sysctl)[key] > 65535:

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -228,8 +228,8 @@ service anvil {
 }
 
 ssl = required
-ssl_cert = <{{ config.tls_cert_path }}
-ssl_key = <{{ config.tls_key_path }}
+ssl_cert = </var/lib/acme/live/{{ config.mail_domain }}/fullchain
+ssl_key = </var/lib/acme/live/{{ config.mail_domain }}/privkey
 ssl_dh = </usr/share/dovecot/dh.pem
 ssl_min_protocol = TLSv1.3
 ssl_prefer_server_ciphers = yes

cmdeploy/src/cmdeploy/external/deployer.py

@@ -1,67 +0,0 @@
-import io
-
-from pyinfra import host
-from pyinfra.facts.files import File
-from pyinfra.operations import files, systemd
-
-from cmdeploy.basedeploy import Deployer, get_resource
-
-
-class ExternalTlsDeployer(Deployer):
-    """Expects TLS certificates to be managed on the server.
-
-    Validates that the configured certificate and key files
-    exist on the remote host.  Installs a systemd path unit
-    that watches the certificate file and automatically
-    restarts/reloads affected services when it changes.
-    """
-
-    def __init__(self, cert_path, key_path):
-        self.cert_path = cert_path
-        self.key_path = key_path
-
-    def configure(self):
-        # Verify cert and key exist on the remote host using pyinfra facts.
-        for path in (self.cert_path, self.key_path):
-            info = host.get_fact(File, path=path)
-            if info is None:
-                raise Exception(f"External TLS file not found on server: {path}")
-
-        # Deploy the .path unit (templated with the cert path).
-        # pkg=__package__ is required here because the resource files
-        # live in cmdeploy.external, not the default cmdeploy package.
-        source = get_resource("tls-cert-reload.path.f", pkg=__package__)
-        content = source.read_text().format(cert_path=self.cert_path).encode()
-
-        path_unit = files.put(
-            name="Upload tls-cert-reload.path",
-            src=io.BytesIO(content),
-            dest="/etc/systemd/system/tls-cert-reload.path",
-            user="root",
-            group="root",
-            mode="644",
-        )
-
-        service_unit = files.put(
-            name="Upload tls-cert-reload.service",
-            src=get_resource("tls-cert-reload.service", pkg=__package__),
-            dest="/etc/systemd/system/tls-cert-reload.service",
-            user="root",
-            group="root",
-            mode="644",
-        )
-
-        if path_unit.changed or service_unit.changed:
-            self.need_restart = True
-
-    def activate(self):
-        systemd.service(
-            name="Enable tls-cert-reload path watcher",
-            service="tls-cert-reload.path",
-            running=True,
-            enabled=True,
-            restarted=self.need_restart,
-            daemon_reload=self.need_restart,
-        )
-        # No explicit reload needed here: dovecot/nginx read the cert
-        # on startup, and the .path watcher handles live changes.

cmdeploy/src/cmdeploy/external/tls-cert-reload.path.f

@@ -1,15 +0,0 @@
-# Watch the TLS certificate file for changes.
-# When the cert is updated (e.g. renewed by an external process),
-# this triggers tls-cert-reload.service to reload the affected services.
-#
-# NOTE: changes to the certificates are not detected if they cross bind-mount boundaries. 
-# After cert renewal, you must then trigger the reload explicitly:
-#   systemctl start tls-cert-reload.service
-[Unit]
-Description=Watch TLS certificate for changes
-
-[Path]
-PathChanged={cert_path}
-
-[Install]
-WantedBy=multi-user.target

cmdeploy/src/cmdeploy/external/tls-cert-reload.service

@@ -1,15 +0,0 @@
-# Reload services that cache the TLS certificate.
-#
-# dovecot: caches the cert at startup; reload re-reads SSL certs
-#          without dropping existing connections.
-# nginx:   caches the cert at startup; reload gracefully picks up
-#          the new cert for new connections.
-# postfix: reads the cert fresh on each TLS handshake,
-#          does NOT need a reload/restart.
-[Unit]
-Description=Reload TLS services after certificate change
-
-[Service]
-Type=oneshot
-ExecStart=/bin/systemctl try-reload-or-restart dovecot
-ExecStart=/bin/systemctl try-reload-or-restart nginx

cmdeploy/src/cmdeploy/filtermail/deployer.py

@@ -14,10 +14,10 @@ class FiltermailDeployer(Deployer):
 
     def install(self):
         arch = host.get_fact(facts.server.Arch)
-        url = f"https://github.com/chatmail/filtermail/releases/download/v0.5.1/filtermail-{arch}"
+        url = f"https://github.com/chatmail/filtermail/releases/download/v0.3.0/filtermail-{arch}"
         sha256sum = {
-            "x86_64": "adce2ddb461c5fd744df699f3b0b3c33b6d52413c641f18695b93826e5e0d234",
-            "aarch64": "b51cf4248c6c443308f21b1811da1cc919b98b719a2138f4b60940ea093a5422",
+            "x86_64": "f14a31323ae2dad3b59d3fdafcde507521da2f951a9478cd1f2fe2b4463df71d",
+            "aarch64": "933770d75046c4fd7084ce8d43f905f8748333426ad839154f0fc654755ef09f",
         }[arch]
         self.need_restart |= files.download(
             name="Download filtermail",

cmdeploy/src/cmdeploy/nginx/autoconfig.xml.j2

@@ -1,47 +1,47 @@
 <?xml version="1.0" encoding="UTF-8"?>
 
 <clientConfig version="1.1">
-  <emailProvider id="{{ config.mail_domain }}">
-    <domain>{{ config.mail_domain }}</domain>
-    <displayName>{{ config.mail_domain }} chatmail</displayName>
-    <displayShortName>{{ config.mail_domain }}</displayShortName>
+  <emailProvider id="{{ config.domain_name }}">
+    <domain>{{ config.domain_name }}</domain>
+    <displayName>{{ config.domain_name }} chatmail</displayName>
+    <displayShortName>{{ config.domain_name }}</displayShortName>
     <incomingServer type="imap">
-      <hostname>{{ config.mail_domain }}</hostname>
+      <hostname>{{ config.domain_name }}</hostname>
       <port>993</port>
       <socketType>SSL</socketType>
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
     </incomingServer>
     <incomingServer type="imap">
-      <hostname>{{ config.mail_domain }}</hostname>
+      <hostname>{{ config.domain_name }}</hostname>
       <port>143</port>
       <socketType>STARTTLS</socketType>
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
     </incomingServer>
     <incomingServer type="imap">
-      <hostname>{{ config.mail_domain }}</hostname>
+      <hostname>{{ config.domain_name }}</hostname>
       <port>443</port>
       <socketType>SSL</socketType>
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
     </incomingServer>
     <outgoingServer type="smtp">
-      <hostname>{{ config.mail_domain }}</hostname>
+      <hostname>{{ config.domain_name }}</hostname>
       <port>465</port>
       <socketType>SSL</socketType>
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
     </outgoingServer>
     <outgoingServer type="smtp">
-      <hostname>{{ config.mail_domain }}</hostname>
+      <hostname>{{ config.domain_name }}</hostname>
       <port>587</port>
       <socketType>STARTTLS</socketType>
       <authentication>password-cleartext</authentication>
       <username>%EMAILADDRESS%</username>
     </outgoingServer>
     <outgoingServer type="smtp">
-      <hostname>{{ config.mail_domain }}</hostname>
+      <hostname>{{ config.domain_name }}</hostname>
       <port>443</port>
       <socketType>SSL</socketType>
       <authentication>password-cleartext</authentication>

cmdeploy/src/cmdeploy/nginx/deployer.py

@@ -70,7 +70,7 @@ def _configure_nginx(config: Config, debug: bool = False) -> bool:
         user="root",
         group="root",
         mode="644",
-        config=config,
+        config={"domain_name": config.mail_domain},
         disable_ipv6=config.disable_ipv6,
     )
     need_restart |= main_config.changed
@@ -81,7 +81,7 @@ def _configure_nginx(config: Config, debug: bool = False) -> bool:
         user="root",
         group="root",
         mode="644",
-        config=config,
+        config={"domain_name": config.mail_domain},
     )
     need_restart |= autoconfig.changed
 
@@ -91,7 +91,7 @@ def _configure_nginx(config: Config, debug: bool = False) -> bool:
         user="root",
         group="root",
         mode="644",
-        config=config,
+        config={"domain_name": config.mail_domain},
     )
     need_restart |= mta_sts_config.changed
 

cmdeploy/src/cmdeploy/nginx/mta-sts.txt.j2

@@ -1,4 +1,4 @@
 version: STSv1
 mode: enforce
-mx: {{ config.mail_domain }}
+mx: {{ config.domain_name }}
 max_age: 2419200

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -42,9 +42,6 @@ stream {
 }
 
 http {
-{% if config.tls_cert_mode == "self" %}
-	limit_req_zone $binary_remote_addr zone=newaccount:10m rate=2r/s;
-{% endif %}
 	sendfile on;
 	tcp_nopush on;
 
@@ -56,8 +53,8 @@ http {
 
 	ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
 	ssl_prefer_server_ciphers on;
-	ssl_certificate {{ config.tls_cert_path }};
-	ssl_certificate_key {{ config.tls_key_path }};
+	ssl_certificate /var/lib/acme/live/{{ config.domain_name }}/fullchain;
+	ssl_certificate_key /var/lib/acme/live/{{ config.domain_name }}/privkey;
 
 	gzip on;
 
@@ -69,7 +66,7 @@ http {
 
 		index index.html index.htm;
 
-		server_name {{ config.mail_domain }} www.{{ config.mail_domain }} mta-sts.{{ config.mail_domain }};
+		server_name {{ config.domain_name }} www.{{ config.domain_name }} mta-sts.{{ config.domain_name }};
 
 		access_log syslog:server=unix:/dev/log,facility=local7;
 
@@ -84,15 +81,11 @@ http {
 		}
 
 		location /new {
-{% if config.tls_cert_mode != "self" %}
 			if ($request_method = GET) {
 				# Redirect to Delta Chat,
 				# which will in turn do a POST request.
-				return 301 dcaccount:https://{{ config.mail_domain }}/new;
+				return 301 dcaccount:https://{{ config.domain_name }}/new;
 			}
-{% else %}
-			limit_req zone=newaccount burst=5 nodelay;
-{% endif %}
 
 			fastcgi_pass unix:/run/fcgiwrap.socket;
 			include /etc/nginx/fastcgi_params;
@@ -106,11 +99,9 @@ http {
 		#
 		# Redirects are only for browsers.
 		location /cgi-bin/newemail.py {
-{% if config.tls_cert_mode != "self" %}
 			if ($request_method = GET) {
-				return 301 dcaccount:https://{{ config.mail_domain }}/new;
+				return 301 dcaccount:https://{{ config.domain_name }}/new;
 			}
-{% endif %}
 
 			fastcgi_pass unix:/run/fcgiwrap.socket;
 			include /etc/nginx/fastcgi_params;
@@ -141,29 +132,8 @@ http {
 	# Redirect www. to non-www
 	server {
 		listen 127.0.0.1:8443 ssl;
-		server_name www.{{ config.mail_domain }};
-		return 301 $scheme://{{ config.mail_domain }}$request_uri;
+		server_name www.{{ config.domain_name }};
+		return 301 $scheme://{{ config.domain_name }}$request_uri;
 		access_log syslog:server=unix:/dev/log,facility=local7;
 	}
-
-	server {
-		listen 80;
-		{% if not disable_ipv6 %}
-		listen [::]:80;
-		{% endif %}
-
-		{% if config.tls_cert_mode == "acme" %}
-		location /.well-known/acme-challenge/ {
-			proxy_pass http://acmetool;
-		}
-		{% endif %}
-
-		return 301 https://$host$request_uri;
-	}
-
-	{% if config.tls_cert_mode == "acme" %}
-	upstream acmetool {
-		server 127.0.0.1:402;
-	}
-	{% endif %}
 }

cmdeploy/src/cmdeploy/opendkim/deployer.py

@@ -37,15 +37,21 @@ class OpendkimDeployer(Deployer):
         )
         need_restart |= main_config.changed
 
-        screen_script = files.file(
-            path="/etc/opendkim/screen.lua",
-            present=False,
+        screen_script = files.put(
+            src=get_resource("opendkim/screen.lua"),
+            dest="/etc/opendkim/screen.lua",
+            user="root",
+            group="root",
+            mode="644",
         )
         need_restart |= screen_script.changed
 
-        final_script = files.file(
-            path="/etc/opendkim/final.lua",
-            present=False,
+        final_script = files.put(
+            src=get_resource("opendkim/final.lua"),
+            dest="/etc/opendkim/final.lua",
+            user="root",
+            group="root",
+            mode="644",
         )
         need_restart |= final_script.changed
 

cmdeploy/src/cmdeploy/opendkim/final.lua

@@ -0,0 +1,42 @@
+mtaname = odkim.get_mtasymbol(ctx, "{daemon_name}")
+if mtaname == "ORIGINATING" then
+	-- Outgoing message will be signed,
+	-- no need to look for signatures.
+	return nil
+end
+
+nsigs = odkim.get_sigcount(ctx)
+if nsigs == nil then
+	return nil
+end
+
+local valid = false
+local error_msg = "No valid DKIM signature found."
+for i = 1, nsigs do
+	sig = odkim.get_sighandle(ctx, i - 1)
+	sigres = odkim.sig_result(sig)
+
+	-- All signatures that do not correspond to From: 
+	-- were ignored in screen.lua and return sigres -1.
+	-- 
+	-- Any valid signature that was not ignored like this
+	-- means the message is acceptable.
+	if sigres == 0 then
+		valid = true
+    else
+        error_msg = "DKIM signature is invalid, error code " .. tostring(sigres) .. ", search https://github.com/trusteddomainproject/OpenDKIM/blob/master/libopendkim/dkim.h#L108"
+	end
+end
+
+if valid then
+	-- Strip all DKIM-Signature headers after successful validation
+	-- Delete in reverse order to avoid index shifting.
+	for i = nsigs, 1, -1 do
+		odkim.del_header(ctx, "DKIM-Signature", i)
+	end
+else
+	odkim.set_reply(ctx, "554", "5.7.1", error_msg)
+	odkim.set_result(ctx, SMFIS_REJECT)
+end
+
+return nil

cmdeploy/src/cmdeploy/opendkim/opendkim.conf

@@ -45,6 +45,12 @@ SignHeaders *,+autocrypt,+content-type
 # Default is empty.
 OversignHeaders from,reply-to,subject,date,to,cc,resent-date,resent-from,resent-sender,resent-to,resent-cc,in-reply-to,references,list-id,list-help,list-unsubscribe,list-subscribe,list-post,list-owner,list-archive,autocrypt
 
+# Script to ignore signatures that do not correspond to the From: domain.
+ScreenPolicyScript /etc/opendkim/screen.lua
+
+# Script to reject mails without a valid DKIM signature.
+FinalPolicyScript /etc/opendkim/final.lua
+
 # In Debian, opendkim runs as user "opendkim". A umask of 007 is required when
 # using a local socket with MTAs that access the socket as a non-privileged
 # user (for example, Postfix). You may need to add user "postfix" to group

cmdeploy/src/cmdeploy/opendkim/screen.lua

@@ -0,0 +1,21 @@
+-- Ignore signatures that do not correspond to the From: domain.
+
+from_domain = odkim.get_fromdomain(ctx)
+if from_domain == nil then
+	return nil
+end
+
+n = odkim.get_sigcount(ctx)
+if n == nil then
+	return nil
+end
+
+for i = 1, n do
+	sig = odkim.get_sighandle(ctx, i - 1)
+	sig_domain = odkim.sig_getdomain(sig)
+	if from_domain ~= sig_domain then
+		odkim.sig_ignore(sig)
+	end
+end
+
+return nil

cmdeploy/src/cmdeploy/postfix/lmtp_header_cleanup

@@ -1,3 +1,2 @@
 /^DKIM-Signature:/            IGNORE
 /^Authentication-Results:/            IGNORE
-/^Received:/            IGNORE

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -15,12 +15,12 @@ readme_directory = no
 compatibility_level = 3.6
 
 # TLS parameters
-smtpd_tls_cert_file={{ config.tls_cert_path }}
-smtpd_tls_key_file={{ config.tls_key_path }}
+smtpd_tls_cert_file=/var/lib/acme/live/{{ config.mail_domain }}/fullchain
+smtpd_tls_key_file=/var/lib/acme/live/{{ config.mail_domain }}/privkey
 smtpd_tls_security_level=may
 
 smtp_tls_CApath=/etc/ssl/certs
-smtp_tls_security_level={{ "verify" if config.tls_cert_mode == "acme" else "encrypt" }}
+smtp_tls_security_level=verify
 # Send SNI extension when connecting to other servers.
 # <https://www.postfix.org/postconf.5.html#smtp_tls_servername>
 smtp_tls_servername = hostname

cmdeploy/src/cmdeploy/postfix/master.cf.j2

@@ -86,6 +86,7 @@ filter    unix -        n       n       -       -       lmtp
 # Local SMTP server for reinjecting incoming filtered mail 
 127.0.0.1:{{ config.postfix_reinject_port_incoming }} inet  n       -       n       -       100      smtpd
   -o syslog_name=postfix/reinject_incoming
+  -o smtpd_milters=unix:opendkim/opendkim.sock
 
 # Cleanup `Received` headers for authenticated mail
 # to avoid leaking client IP.

cmdeploy/src/cmdeploy/postfix/smtp_tls_policy_map

@@ -1,3 +1,2 @@
 /^\[[^]]+\]$/ encrypt
-/^_/          encrypt
 /^nauta\.cu$/    may

cmdeploy/src/cmdeploy/run.py

@@ -15,8 +15,9 @@ def main():
     )
     disable_mail = bool(os.environ.get("CHATMAIL_DISABLE_MAIL"))
     website_only = bool(os.environ.get("CHATMAIL_WEBSITE_ONLY"))
+    docker = bool(os.environ.get("CHATMAIL_DOCKER"))
 
-    deploy_chatmail(config_path, disable_mail, website_only)
+    deploy_chatmail(config_path, disable_mail, website_only, docker)
 
 
 if pyinfra.is_cli:

cmdeploy/src/cmdeploy/selfsigned/deployer.py

@@ -1,52 +0,0 @@
-import shlex
-
-from pyinfra.operations import apt, server
-
-from cmdeploy.basedeploy import Deployer
-
-
-def openssl_selfsigned_args(domain, cert_path, key_path, days=36500):
-    """Return the openssl argument list for a self-signed certificate.
-
-    The certificate uses an EC P-256 key with SAN entries for *domain*,
-    ``www.<domain>`` and ``mta-sts.<domain>``.
-    """
-    return [
-        "openssl", "req", "-x509",
-        "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:P-256",
-        "-noenc", "-days", str(days),
-        "-keyout", str(key_path),
-        "-out", str(cert_path),
-        "-subj", f"/CN={domain}",
-        "-addext", "extendedKeyUsage=serverAuth,clientAuth",
-        "-addext",
-        f"subjectAltName=DNS:{domain},DNS:www.{domain},DNS:mta-sts.{domain}",
-    ]
-
-
-class SelfSignedTlsDeployer(Deployer):
-    """Generates a self-signed TLS certificate for all chatmail endpoints."""
-
-    def __init__(self, mail_domain):
-        self.mail_domain = mail_domain
-        self.cert_path = "/etc/ssl/certs/mailserver.pem"
-        self.key_path = "/etc/ssl/private/mailserver.key"
-
-    def install(self):
-        apt.packages(
-            name="Install openssl",
-            packages=["openssl"],
-        )
-
-    def configure(self):
-        args = openssl_selfsigned_args(
-            self.mail_domain, self.cert_path, self.key_path,
-        )
-        cmd = shlex.join(args)
-        server.shell(
-            name="Generate self-signed TLS certificate if not present",
-            commands=[f"[ -f {self.cert_path} ] || {cmd}"],
-        )
-
-    def activate(self):
-        pass

cmdeploy/src/cmdeploy/service/chatmail-metadata.service.f

@@ -4,7 +4,7 @@ Description=Chatmail dict proxy for IMAP METADATA
 [Service]
 ExecStart={execpath} /run/chatmail-metadata/metadata.socket {config_path}
 Restart=always
-RestartSec=5
+RestartSec=30
 User=vmail
 RuntimeDirectory=chatmail-metadata
 UMask=0077

cmdeploy/src/cmdeploy/sshexec.py

@@ -85,31 +85,16 @@ class SSHExec:
 
 
 class LocalExec:
-    FuncError = FuncError
-
     def __init__(self, verbose=False, docker=False):
         self.verbose = verbose
         self.docker = docker
 
-    def __call__(self, call, kwargs=None, log_callback=None):
-        if kwargs is None:
-            kwargs = {}
-        return call(**kwargs)
-
     def logged(self, call, kwargs: dict):
-        title = call.__doc__
-        if not title:
-            title = call.__name__
         where = "locally"
         if self.docker:
             if call == remote.rdns.perform_initial_checks:
                 kwargs["pre_command"] = "docker exec chatmail "
                 where = "in docker"
         if self.verbose:
-            print_stderr(f"Running {where}: {title}(**{kwargs})")
-            return self(call, kwargs, log_callback=print_stderr)
-        else:
-            print_stderr(title, end="")
-            res = self(call, kwargs, log_callback=remote.rshell.log_progress)
-            print_stderr()
-            return res
+            print(f"Running {where}: {call.__name__}(**{kwargs})")
+        return call(**kwargs)

cmdeploy/src/cmdeploy/tests/online/benchmark.py

@@ -1,4 +1,3 @@
-import time
 def test_tls_imap(benchmark, imap):
     def imap_connect():
         imap.connect()
@@ -42,9 +41,9 @@ class TestDC:
 
         def dc_ping_pong():
             chat.send_text("ping")
-            msg = ac2.wait_for_incoming_msg()
-            msg.get_snapshot().chat.send_text("pong")
-            ac1.wait_for_incoming_msg()
+            msg = ac2._evtracker.wait_next_incoming_message()
+            msg.chat.send_text("pong")
+            ac1._evtracker.wait_next_incoming_message()
 
         benchmark(dc_ping_pong, 5)
 
@@ -56,6 +55,6 @@ class TestDC:
             for i in range(10):
                 chat.send_text(f"hello {i}")
             for i in range(10):
-                ac2.wait_for_incoming_msg()
+                ac2._evtracker.wait_next_incoming_message()
 
-        benchmark(dc_send_10_receive_10, 5, cooldown="auto")
+        benchmark(dc_send_10_receive_10, 5)

cmdeploy/src/cmdeploy/tests/online/test_0_qr.py

@@ -1,4 +1,3 @@
-import pytest
 import requests
 
 from cmdeploy.genqr import gen_qr_png_data
@@ -9,33 +8,18 @@ def test_gen_qr_png_data(maildomain):
     assert data
 
 
-@pytest.mark.filterwarnings("ignore::urllib3.exceptions.InsecureRequestWarning")
 def test_fastcgi_working(maildomain, chatmail_config):
     url = f"https://{maildomain}/new"
     print(url)
-    verify = chatmail_config.tls_cert_mode == "acme"
-    res = requests.post(url, verify=verify)
+    res = requests.post(url)
     assert maildomain in res.json().get("email")
     assert len(res.json().get("password")) > chatmail_config.password_min_length
 
 
-@pytest.mark.filterwarnings("ignore::urllib3.exceptions.InsecureRequestWarning")
-def test_newemail_configure(maildomain, rpc, chatmail_config):
+def test_newemail_configure(maildomain, rpc):
     """Test configuring accounts by scanning a QR code works."""
     url = f"DCACCOUNT:https://{maildomain}/new"
     for i in range(3):
         account_id = rpc.add_account()
-        if chatmail_config.tls_cert_mode == "self":
-            # deltachat core's rustls rejects self-signed HTTPS certs during
-            # set_config_from_qr, so fetch credentials via requests instead
-            res = requests.post(f"https://{maildomain}/new", verify=False)
-            data = res.json()
-            rpc.add_or_update_transport(account_id, {
-                "addr": data["email"],
-                "password": data["password"],
-                "imapServer": maildomain,
-                "smtpServer": maildomain,
-                "certificateChecks": "acceptInvalidCertificates",
-            })
-        else:
-            rpc.add_transport_from_qr(account_id, url)
+        rpc.set_config_from_qr(account_id, url)
+        rpc.configure(account_id)

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -7,13 +7,13 @@ import time
 import pytest
 
 from cmdeploy import remote
-from cmdeploy.cmdeploy import get_sshexec
+from cmdeploy.sshexec import SSHExec
 
 
 class TestSSHExecutor:
     @pytest.fixture(scope="class")
     def sshexec(self, sshdomain):
-        return get_sshexec(sshdomain)
+        return SSHExec(sshdomain)
 
     def test_ls(self, sshexec):
         out = sshexec(call=remote.rdns.shell, kwargs=dict(command="ls"))
@@ -27,7 +27,6 @@ class TestSSHExecutor:
         assert res["A"] or res["AAAA"]
 
     def test_logged(self, sshexec, maildomain, capsys):
-        sshexec.verbose = False
         sshexec.logged(
             remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain)
         )
@@ -53,8 +52,6 @@ class TestSSHExecutor:
                 remote.rdns.perform_initial_checks,
                 kwargs=dict(mail_domain=None),
             )
-        except AssertionError:
-            pass
         except sshexec.FuncError as e:
             assert "rdns.py" in str(e)
             assert "AssertionError" in str(e)
@@ -86,8 +83,10 @@ def test_remote(remote, imap_or_smtp):
 
 
 def test_use_two_chatmailservers(cmfactory, maildomain2):
-    ac1 = cmfactory.get_online_account()
-    ac2 = cmfactory.get_online_account(domain=maildomain2)
+    ac1 = cmfactory.new_online_configuring_account(cache=False)
+    cmfactory.switch_maildomain(maildomain2)
+    ac2 = cmfactory.new_online_configuring_account(cache=False)
+    cmfactory.bring_accounts_online()
     cmfactory.get_accepted_chat(ac1, ac2)
     domain1 = ac1.get_config("addr").split("@")[1]
     domain2 = ac2.get_config("addr").split("@")[1]
@@ -147,7 +146,7 @@ def test_reject_missing_dkim(cmsetup, maildata, from_addr):
     conn.starttls()
 
     with conn as s:
-        with pytest.raises(smtplib.SMTPDataError, match="No DKIM signature found"):
+        with pytest.raises(smtplib.SMTPDataError, match="No valid DKIM signature"):
             s.sendmail(from_addr=from_addr, to_addrs=recipient.addr, msg=msg)
 
 
@@ -219,7 +218,7 @@ def test_expunged(remote, chatmail_config):
     ]
     outdated_days = int(chatmail_config.delete_large_after) + 1
     find_cmds.append(
-        f"find {chatmail_config.mailboxes_dir} -path '*/cur/*' -mtime +{outdated_days} -size +200k -type f"
+        "find {chatmail_config.mailboxes_dir} -path '*/cur/*' -mtime +{outdated_days} -size +200k -type f"
     )
     for cmd in find_cmds:
         for line in remote.iter_output(cmd):

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -7,16 +7,15 @@ import pytest
 import requests
 
 from cmdeploy.remote import rshell
-from cmdeploy.cmdeploy import get_sshexec
+from cmdeploy.sshexec import SSHExec
 
 
 @pytest.fixture
-def imap_mailbox(cmfactory, ssl_context):
+def imap_mailbox(cmfactory):
     (ac1,) = cmfactory.get_online_accounts(1)
     user = ac1.get_config("addr")
     password = ac1.get_config("mail_pw")
-    host = user.split("@")[1]
-    mailbox = imap_tools.MailBox(host, ssl_context=ssl_context)
+    mailbox = imap_tools.MailBox(user.split("@")[1])
     mailbox.login(user, password)
     mailbox.dc_ac = ac1
     return mailbox
@@ -27,7 +26,6 @@ class TestMetadataTokens:
 
     def test_set_get_metadata(self, imap_mailbox):
         "set and get metadata token for an account"
-        time.sleep(5)  # make sure Metadata service had a chance to restart
         client = imap_mailbox.client
         client.send(b'a01 SETMETADATA INBOX (/private/devicetoken "1111" )\n')
         res = client.readline()
@@ -63,8 +61,8 @@ class TestEndToEndDeltaChat:
         chat.send_text("message0")
 
         lp.sec("wait for ac2 to receive message")
-        msg2 = ac2.wait_for_incoming_msg()
-        assert msg2.get_snapshot().text == "message0"
+        msg2 = ac2._evtracker.wait_next_incoming_message()
+        assert msg2.text == "message0"
 
     def test_exceed_quota(
         self, cmfactory, lp, tmpdir, remote, chatmail_config, sshdomain
@@ -92,41 +90,45 @@ class TestEndToEndDeltaChat:
         lp.sec(f"filling remote inbox for {user}")
         fn = f"7743102289.M843172P2484002.c20,S={quota},W=2398:2,"
         path = chatmail_config.mailboxes_dir.joinpath(user, "cur", fn)
-        sshexec = get_sshexec(sshdomain)
+        sshexec = SSHExec(sshdomain)
         sshexec(call=rshell.write_numbytes, kwargs=dict(path=str(path), num=120))
         res = sshexec(call=rshell.dovecot_recalc_quota, kwargs=dict(user=user))
         assert res["percent"] >= 100
 
         lp.sec("ac2: check quota is triggered")
 
-        def send_hello():
-            chat.send_text("hello")
-
-        for line in remote.iter_output(
-            "journalctl -n1 -f -u dovecot", ready=send_hello
-        ):
+        starting = True
+        for line in remote.iter_output("journalctl -n0 -f -u dovecot"):
+            if starting:
+                chat.send_text("hello")
+                starting = False
             if user not in line:
+                # print(line)
                 continue
             if "quota exceeded" in line:
                 return
 
     def test_securejoin(self, cmfactory, lp, maildomain2):
-        ac1 = cmfactory.get_online_account()
-        ac2 = cmfactory.get_online_account(domain=maildomain2)
+        ac1 = cmfactory.new_online_configuring_account(cache=False)
+        cmfactory.switch_maildomain(maildomain2)
+        ac2 = cmfactory.new_online_configuring_account(cache=False)
+        cmfactory.bring_accounts_online()
 
         lp.sec("ac1: create QR code and let ac2 scan it, starting the securejoin")
-        qr = ac1.get_qr_code()
+        qr = ac1.get_setup_contact_qr()
 
         lp.sec("ac2: start QR-code based setup contact protocol")
-        ch = ac2.secure_join(qr)
+        ch = ac2.qr_setup_contact(qr)
         assert ch.id >= 10
-        ac1.wait_for_securejoin_inviter_success()
+        ac1._evtracker.wait_securejoin_inviter_progress(1000)
 
     def test_dkim_header_stripped(self, cmfactory, maildomain2, lp, imap_mailbox):
         """Test that if a DC address receives a message, it has no
         DKIM-Signature and Authentication-Results headers."""
-        ac1 = cmfactory.get_online_account()
-        ac2 = cmfactory.get_online_account(domain=maildomain2)
+        ac1 = cmfactory.new_online_configuring_account(cache=False)
+        cmfactory.switch_maildomain(maildomain2)
+        ac2 = cmfactory.new_online_configuring_account(cache=False)
+        cmfactory.bring_accounts_online()
         chat = cmfactory.get_accepted_chat(ac1, imap_mailbox.dc_ac)
         chat.send_text("message0")
         chat2 = cmfactory.get_accepted_chat(ac2, imap_mailbox.dc_ac)
@@ -143,32 +145,33 @@ class TestEndToEndDeltaChat:
                 assert "dkim-signature" not in msg.headers
 
     def test_read_receipts_between_instances(self, cmfactory, lp, maildomain2):
-        ac1 = cmfactory.get_online_account()
-        ac2 = cmfactory.get_online_account(domain=maildomain2)
+        ac1 = cmfactory.new_online_configuring_account(cache=False)
+        cmfactory.switch_maildomain(maildomain2)
+        ac2 = cmfactory.new_online_configuring_account(cache=False)
+        cmfactory.bring_accounts_online()
 
         lp.sec("setup encrypted comms between ac1 and ac2 on different instances")
-        qr = ac1.get_qr_code()
-        ch = ac2.secure_join(qr)
+        qr = ac1.get_setup_contact_qr()
+        ch = ac2.qr_setup_contact(qr)
         assert ch.id >= 10
-        ac1.wait_for_securejoin_inviter_success()
+        ac1._evtracker.wait_securejoin_inviter_progress(1000)
 
         lp.sec("ac1 sends a message and ac2 marks it as seen")
         chat = ac1.create_chat(ac2)
         msg = chat.send_text("hi")
-        m = ac2.wait_for_incoming_msg()
+        m = ac2._evtracker.wait_next_incoming_message()
         m.mark_seen()
         # we can only indirectly wait for mark-seen to cause an smtp-error
         lp.sec("try to wait for markseen to complete and check error states")
         deadline = time.time() + 3.1
         while time.time() < deadline:
-            m_snap = m.get_snapshot()
-            msgs = m_snap.chat.get_messages()
+            msgs = m.chat.get_messages()
             for msg in msgs:
-                assert "error" not in m.get_info()
+                assert "error" not in m.get_message_info()
             time.sleep(1)
 
 
-def test_hide_senders_ip_address(cmfactory, ssl_context):
+def test_hide_senders_ip_address(cmfactory):
     public_ip = requests.get("http://icanhazip.com").content.decode().strip()
     assert ipaddress.ip_address(public_ip)
 
@@ -176,12 +179,7 @@ def test_hide_senders_ip_address(cmfactory, ssl_context):
     chat = cmfactory.get_accepted_chat(user1, user2)
 
     chat.send_text("testing submission header cleanup")
-    user2.wait_for_incoming_msg()
-    addr = user2.get_config("addr")
-    host = addr.split("@")[1]
-    pw = user2.get_config("mail_pw")
-    mailbox = imap_tools.MailBox(host, ssl_context=ssl_context)
-    mailbox.login(addr, pw)
-    msgs = list(mailbox.fetch(mark_seen=False))
-    assert msgs, "expected at least one message"
-    assert public_ip not in msgs[0].obj.as_string()
+    user2._evtracker.wait_next_incoming_message()
+    user2.direct_imap.select_folder("Inbox")
+    msg = user2.direct_imap.get_all_messages()[0]
+    assert public_ip not in msg.obj.as_string()

cmdeploy/src/cmdeploy/tests/online/test_3_status.py

@@ -5,11 +5,7 @@ from cmdeploy.cmdeploy import main
 
 def test_status_cmd(chatmail_config, capsys, request):
     os.chdir(request.config.invocation_params.dir)
-    command = ["status"]
-    if os.getenv("CHATMAIL_SSH"):
-        command.append("--ssh-host")
-        command.append(os.getenv("CHATMAIL_SSH"))
-    assert main(command) == 0
+    assert main(["status"]) == 0
     status_out = capsys.readouterr()
     print(status_out.out)
 

cmdeploy/src/cmdeploy/tests/plugin.py

@@ -1,9 +1,9 @@
 import imaplib
+import io
 import itertools
 import os
 import random
 import smtplib
-import ssl
 import subprocess
 import time
 from pathlib import Path
@@ -34,24 +34,17 @@ def pytest_runtest_setup(item):
             pytest.skip("skipping slow test, use --slow to run")
 
 
-def _get_chatmail_config():
-    current = Path().resolve()
+@pytest.fixture(scope="session")
+def chatmail_config(pytestconfig):
+    current = basedir = Path().resolve()
     while 1:
         path = current.joinpath("chatmail.ini").resolve()
         if path.exists():
-            return read_config(path), path
+            return read_config(path)
         if current == current.parent:
             break
         current = current.parent
-    return None, None
 
-
-@pytest.fixture(scope="session")
-def chatmail_config(pytestconfig):
-    config, path = _get_chatmail_config()
-    if config:
-        return config
-    basedir = Path().resolve()
     pytest.skip(f"no chatmail.ini file found in {basedir} or parent dirs")
 
 
@@ -79,17 +72,10 @@ def sshdomain2(maildomain2):
 
 
 def pytest_report_header():
-    config, path = _get_chatmail_config()
-    domain2 = os.environ.get("CHATMAIL_DOMAIN2", "NOT SET")
-    domain = config.mail_domain if config else "NOT SET"
-    path = path if path else "NOT SET"
-
-    lines = [
-        f"chatmail.ini {domain} location: {path}",
-        f"chatmail2: {domain2}",
-    ]
-    sep = "-" * max(map(len, lines))
-    return [sep, *lines, sep]
+    domain = os.environ.get("CHATMAIL_DOMAIN")
+    if domain:
+        text = f"chatmail test instance: {domain}"
+        return ["-" * len(text), text, "-" * len(text)]
 
 
 @pytest.fixture
@@ -104,22 +90,15 @@ def cm_data(request):
 
 
 @pytest.fixture
-def benchmark(request, chatmail_config):
-    def bench(func, num, name=None, reportfunc=None, cooldown=0.0):
+def benchmark(request):
+    def bench(func, num, name=None, reportfunc=None):
         if name is None:
             name = func.__name__
-        if cooldown == "auto":
-            per_minute = max(chatmail_config.max_user_send_per_minute, 1)
-            cooldown = chatmail_config.max_user_send_burst_size * 60 / per_minute
-
         durations = []
         for i in range(num):
             now = time.time()
             func()
             durations.append(time.time() - now)
-            if cooldown > 0 and i + 1 < num:
-                # Keep post-run cooldown out of measured benchmark duration.
-                time.sleep(cooldown)
         durations.sort()
         request.config._benchresults[name] = (reportfunc, durations)
 
@@ -165,25 +144,15 @@ def pytest_terminal_summary(terminalreporter):
             tr.write_line(line)
 
 
-@pytest.fixture(scope="session")
-def ssl_context(chatmail_config):
-    if chatmail_config.tls_cert_mode == "self":
-        ctx = ssl.create_default_context()
-        ctx.check_hostname = False
-        ctx.verify_mode = ssl.CERT_NONE
-        return ctx
-    return None
+@pytest.fixture
+def imap(maildomain):
+    return ImapConn(maildomain)
 
 
 @pytest.fixture
-def imap(maildomain, ssl_context):
-    return ImapConn(maildomain, ssl_context=ssl_context)
-
-
-@pytest.fixture
-def make_imap_connection(maildomain, ssl_context):
+def make_imap_connection(maildomain):
     def make_imap_connection():
-        conn = ImapConn(maildomain, ssl_context=ssl_context)
+        conn = ImapConn(maildomain)
         conn.connect()
         return conn
 
@@ -195,13 +164,12 @@ class ImapConn:
     logcmd = "journalctl -f -u dovecot"
     name = "dovecot"
 
-    def __init__(self, host, ssl_context=None):
+    def __init__(self, host):
         self.host = host
-        self.ssl_context = ssl_context
 
     def connect(self):
         print(f"imap-connect {self.host}")
-        self.conn = imaplib.IMAP4_SSL(self.host, ssl_context=self.ssl_context)
+        self.conn = imaplib.IMAP4_SSL(self.host)
 
     def login(self, user, password):
         print(f"imap-login {user!r} {password!r}")
@@ -227,14 +195,14 @@ class ImapConn:
 
 
 @pytest.fixture
-def smtp(maildomain, ssl_context):
-    return SmtpConn(maildomain, ssl_context=ssl_context)
+def smtp(maildomain):
+    return SmtpConn(maildomain)
 
 
 @pytest.fixture
-def make_smtp_connection(maildomain, ssl_context):
+def make_smtp_connection(maildomain):
     def make_smtp_connection():
-        conn = SmtpConn(maildomain, ssl_context=ssl_context)
+        conn = SmtpConn(maildomain)
         conn.connect()
         return conn
 
@@ -246,14 +214,12 @@ class SmtpConn:
     logcmd = "journalctl -f -t postfix/smtpd -t postfix/smtp -t postfix/lmtp"
     name = "postfix"
 
-    def __init__(self, host, ssl_context=None):
+    def __init__(self, host):
         self.host = host
-        self.ssl_context = ssl_context
 
     def connect(self):
         print(f"smtp-connect {self.host}")
-        context = self.ssl_context or ssl.create_default_context()
-        self.conn = smtplib.SMTP_SSL(self.host, context=context)
+        self.conn = smtplib.SMTP_SSL(self.host)
 
     def login(self, user, password):
         print(f"smtp-login {user!r} {password!r}")
@@ -296,94 +262,68 @@ def gencreds(chatmail_config):
 
 
 #
-# Delta Chat RPC-based test support
+# Delta Chat testplugin re-use
 # use the cmfactory fixture to get chatmail instance accounts
 #
 
-from deltachat_rpc_client import DeltaChat, Rpc
 
+class ChatmailTestProcess:
+    """Provider for chatmail instance accounts as used by deltachat.testplugin.acfactory"""
 
-class ChatmailACFactory:
-    """RPC-based account factory for chatmail testing."""
-
-    def __init__(self, rpc, maildomain, gencreds, chatmail_config):
-        self.dc = DeltaChat(rpc)
-        self.rpc = rpc
-        self._maildomain = maildomain
+    def __init__(self, pytestconfig, maildomain, gencreds):
+        self.pytestconfig = pytestconfig
+        self.maildomain = maildomain
+        assert "." in self.maildomain, maildomain
         self.gencreds = gencreds
-        self.chatmail_config = chatmail_config
+        self._addr2files = {}
 
-    def _make_transport(self, domain):
-        """Build a transport config dict for the given domain."""
-        addr, password = self.gencreds(domain)
-        transport = {
-            "addr": addr,
-            "password": password,
-            # Setting server explicitly skips requesting autoconfig XML,
-            # see https://datatracker.ietf.org/doc/draft-ietf-mailmaint-autoconfig/
-            "imapServer": domain,
-            "smtpServer": domain,
-        }
-        if self.chatmail_config.tls_cert_mode == "self":
-            transport["certificateChecks"] = "acceptInvalidCertificates"
-        return transport
+    def get_liveconfig_producer(self):
+        while 1:
+            user, password = self.gencreds(self.maildomain)
+            config = {
+                "addr": user,
+                "mail_pw": password,
+            }
+            # speed up account configuration
+            config["mail_server"] = self.maildomain
+            config["send_server"] = self.maildomain
+            yield config
 
-    def get_online_account(self, domain=None):
-        """Create, configure and bring online a single account."""
-        return self.get_online_accounts(1, domain)[0]
+    def cache_maybe_retrieve_configured_db_files(self, cache_addr, db_target_path):
+        pass
 
-    def get_online_accounts(self, num, domain=None):
-        """Create multiple online accounts in parallel."""
-        domain = domain or self._maildomain
-        futures = []
-        accounts = []
-        for _ in range(num):
-            account = self.dc.add_account()
-            future = account.add_or_update_transport.future(
-                self._make_transport(domain)
-            )
-            futures.append(future)
-
-            # ensure messages stay in INBOX so that they can be
-            # concurrently fetched via extra IMAP connections during tests
-            account.set_config("delete_server_after", "10")
-            accounts.append(account)
-
-        for future in futures:
-            future()
-
-        for account in accounts:
-            account.bring_online()
-        return accounts
-
-    def get_accepted_chat(self, ac1, ac2):
-        """Create a 1:1 chat between ac1 and ac2 accepted on both sides."""
-        ac2.create_chat(ac1)
-        return ac1.create_chat(ac2)
-
-
-@pytest.fixture(scope="session")
-def rpc(tmp_path_factory):
-    """Start a deltachat-rpc-server process for the test session."""
-
-    # NB: accounts_dir must NOT already exist as directory --
-    # core-rust only creates accounts.toml if the dir doesn't exist yet.
-    accounts_dir = str(tmp_path_factory.mktemp("dc") / "accounts")
-    rpc = Rpc(accounts_dir=accounts_dir)
-    rpc.start()
-    yield rpc
-    rpc.close()
+    def cache_maybe_store_configured_db_files(self, acc):
+        pass
 
 
 @pytest.fixture
-def cmfactory(rpc, gencreds, maildomain, chatmail_config):
-    """Return a ChatmailACFactory for creating online Delta Chat accounts."""
-    return ChatmailACFactory(
-        rpc=rpc,
-        maildomain=maildomain,
-        gencreds=gencreds,
-        chatmail_config=chatmail_config,
-    )
+def cmfactory(request, gencreds, tmpdir, maildomain):
+    # cloned from deltachat.testplugin.amfactory
+    pytest.importorskip("deltachat")
+    from deltachat.testplugin import ACFactory
+
+    testproc = ChatmailTestProcess(request.config, maildomain, gencreds)
+
+    class Data:
+        def read_path(self, path):
+            return
+
+    am = ACFactory(request=request, tmpdir=tmpdir, testprocess=testproc, data=Data())
+
+    # nb. a bit hacky
+    # would probably be better if deltachat's test machinery grows native support
+    def switch_maildomain(maildomain2):
+        am.testprocess.maildomain = maildomain2
+
+    am.switch_maildomain = switch_maildomain
+
+    yield am
+    if hasattr(request.node, "rep_call") and request.node.rep_call.failed:
+        if testproc.pytestconfig.getoption("--extra-info"):
+            logfile = io.StringIO()
+            am.dump_imap_summary(logfile=logfile)
+            print(logfile.getvalue())
+            # request.node.add_report_section("call", "imap-server-state", s)
 
 
 @pytest.fixture
@@ -395,27 +335,19 @@ class Remote:
     def __init__(self, sshdomain):
         self.sshdomain = sshdomain
 
-    def iter_output(self, logcmd="", ready=None):
+    def iter_output(self, logcmd=""):
         getjournal = "journalctl -f" if not logcmd else logcmd
-        print(self.sshdomain)
-        match self.sshdomain:
-            case "@local": command = []
-            case "localhost": command = []
-            case _: command = ["ssh", f"root@{self.sshdomain}"]
-        [command.append(arg) for arg in getjournal.split()]
         self.popen = subprocess.Popen(
-            command,
+            ["ssh", f"root@{self.sshdomain}", getjournal],
             stdout=subprocess.PIPE,
         )
         while 1:
             line = self.popen.stdout.readline()
             res = line.decode().strip().lower()
-            if not res:
+            if res:
+                yield res
+            else:
                 break
-            if ready is not None:
-                ready()
-                ready = None
-            yield res
 
 
 @pytest.fixture
@@ -431,40 +363,38 @@ def lp(request):
 
 
 @pytest.fixture
-def cmsetup(maildomain, gencreds, ssl_context):
-    return CMSetup(maildomain, gencreds, ssl_context)
+def cmsetup(maildomain, gencreds):
+    return CMSetup(maildomain, gencreds)
 
 
 class CMSetup:
-    def __init__(self, maildomain, gencreds, ssl_context):
+    def __init__(self, maildomain, gencreds):
         self.maildomain = maildomain
         self.gencreds = gencreds
-        self.ssl_context = ssl_context
 
     def gen_users(self, num):
         print(f"Creating {num} online users")
         users = []
         for i in range(num):
             addr, password = self.gencreds()
-            user = CMUser(self.maildomain, addr, password, self.ssl_context)
+            user = CMUser(self.maildomain, addr, password)
             assert user.smtp
             users.append(user)
         return users
 
 
 class CMUser:
-    def __init__(self, maildomain, addr, password, ssl_context=None):
+    def __init__(self, maildomain, addr, password):
         self.maildomain = maildomain
         self.addr = addr
         self.password = password
-        self.ssl_context = ssl_context
         self._smtp = None
         self._imap = None
 
     @property
     def smtp(self):
         if not self._smtp:
-            handle = SmtpConn(self.maildomain, ssl_context=self.ssl_context)
+            handle = SmtpConn(self.maildomain)
             handle.connect()
             handle.login(self.addr, self.password)
             self._smtp = handle
@@ -473,7 +403,7 @@ class CMUser:
     @property
     def imap(self):
         if not self._imap:
-            imap = ImapConn(self.maildomain, ssl_context=self.ssl_context)
+            imap = ImapConn(self.maildomain)
             imap.connect()
             imap.login(self.addr, self.password)
             self._imap = imap

cmdeploy/src/cmdeploy/tests/test_dns.py

@@ -91,16 +91,6 @@ class TestPerformInitialChecks:
         assert not res
         assert len(l) == 2
 
-    def test_perform_initial_checks_no_mta_sts_self_signed(self, mockdns):
-        del mockdns["CNAME"]["mta-sts.some.domain"]
-        remote_data = remote.rdns.perform_initial_checks("some.domain")
-        assert not remote_data["MTA_STS"]
-
-        l = []
-        res = check_initial_remote_data(remote_data, strict_tls=False, print=l.append)
-        assert res
-        assert not l
-
 
 def parse_zonefile_into_dict(zonefile, mockdns_base, only_required=False):
     for zf_line in zonefile.split("\n"):

cmdeploy/src/cmdeploy/tests/test_external_tls.py

@@ -1,78 +0,0 @@
-"""Functional tests for tls_external_cert_and_key option."""
-
-import json
-
-import chatmaild.newemail
-import pytest
-from chatmaild.config import read_config, write_initial_config
-
-
-def make_external_config(tmp_path, cert_key=None):
-    inipath = tmp_path / "chatmail.ini"
-    overrides = {}
-    if cert_key is not None:
-        overrides["tls_external_cert_and_key"] = cert_key
-    write_initial_config(inipath, "chat.example.org", overrides=overrides)
-    return inipath
-
-
-def test_external_tls_config_reads_paths(tmp_path):
-    inipath = make_external_config(
-        tmp_path,
-        cert_key=(
-            "/etc/letsencrypt/live/chat.example.org/fullchain.pem"
-            " /etc/letsencrypt/live/chat.example.org/privkey.pem"
-        ),
-    )
-    config = read_config(inipath)
-    assert config.tls_cert_mode == "external"
-    assert (
-        config.tls_cert_path == "/etc/letsencrypt/live/chat.example.org/fullchain.pem"
-    )
-    assert config.tls_key_path == "/etc/letsencrypt/live/chat.example.org/privkey.pem"
-
-
-def test_external_tls_missing_option_uses_acme(tmp_path):
-    config = read_config(make_external_config(tmp_path))
-    assert config.tls_cert_mode == "acme"
-
-
-def test_external_tls_bad_format_raises(tmp_path):
-    inipath = make_external_config(tmp_path, cert_key="/only/one/path.pem")
-    with pytest.raises(ValueError, match="two space-separated"):
-        read_config(inipath)
-
-
-def test_external_tls_three_paths_raises(tmp_path):
-    inipath = make_external_config(tmp_path, cert_key="/a /b /c")
-    with pytest.raises(ValueError, match="two space-separated"):
-        read_config(inipath)
-
-
-def test_external_tls_no_dclogin_url(tmp_path, capsys, monkeypatch):
-    inipath = make_external_config(
-        tmp_path, cert_key="/certs/fullchain.pem /certs/privkey.pem"
-    )
-    monkeypatch.setattr(chatmaild.newemail, "CONFIG_PATH", str(inipath))
-    chatmaild.newemail.print_new_account()
-    out, _ = capsys.readouterr()
-    lines = out.split("\n")
-    dic = json.loads(lines[2])
-    assert "dclogin_url" not in dic
-
-
-def test_external_tls_selects_correct_deployer(tmp_path):
-    from cmdeploy.deployers import get_tls_deployer
-    from cmdeploy.external.deployer import ExternalTlsDeployer
-    from cmdeploy.selfsigned.deployer import SelfSignedTlsDeployer
-
-    inipath = make_external_config(
-        tmp_path, cert_key="/certs/fullchain.pem /certs/privkey.pem"
-    )
-    config = read_config(inipath)
-    deployer = get_tls_deployer(config, "chat.example.org")
-
-    assert isinstance(deployer, ExternalTlsDeployer)
-    assert not isinstance(deployer, SelfSignedTlsDeployer)
-    assert deployer.cert_path == "/certs/fullchain.pem"
-    assert deployer.key_path == "/certs/privkey.pem"

doc/source/getting_started.rst

@@ -47,14 +47,6 @@ steps. Please substitute it with your own domain.
        www.chat.example.org. 3600 IN CNAME chat.example.org.
        mta-sts.chat.example.org. 3600 IN CNAME chat.example.org.
 
-   .. note::
-
-      For experimental deployments using self-signed certificates,
-      use a domain name starting with ``_``
-      (e.g. ``_chat.example.org``).
-      The ``mta-sts`` CNAME and ``_mta-sts`` TXT records
-      are not needed for such domains.
-
 2. On your local PC, clone the repository and bootstrap the Python
    virtualenv.
 
@@ -71,16 +63,6 @@ steps. Please substitute it with your own domain.
 
        scripts/cmdeploy init chat.example.org  # <-- use your domain
 
-   To use self-signed TLS certificates
-   instead of Let's Encrypt,
-   use a domain name starting with ``_``
-   (e.g. ``scripts/cmdeploy init _chat.example.org``).
-   Domains starting with ``_`` cannot obtain WebPKI certificates,
-   so self-signed mode is derived automatically.
-   This is useful for private or test deployments.
-   See the :doc:`overview`
-   for details on certificate provisioning.
-
 4. Verify that SSH root login to the deployment server server works:
 
    ::
@@ -98,6 +80,13 @@ steps. Please substitute it with your own domain.
    configure at your DNS provider (it can take some time until they are
    public).
 
+Docker installation
+-------------------
+
+We have experimental support for `docker compose <https://github.com/chatmail/relay/blob/docker-rebase/docs/DOCKER_INSTALLATION_EN.md>`_,
+but it is not covered by automated tests yet,
+so don't expect everything to work.
+
 Other helpful commands
 ----------------------
 
@@ -187,55 +176,6 @@ creating addresses, login with ssh to the deployment machine and run:
 Chatmail address creation will be denied while this file is present.
 
 
-Running a relay with self-signed certificates
-----------------------------------------------
-
-Use a domain name starting with ``_`` (e.g. ``_chat.example.org``)
-to run a relay with self-signed certificates.
-Domains starting with ``_`` cannot obtain WebPKI certificates
-so the relay automatically uses self-signed certificates
-and all other relays will accept connections from it
-without requiring certificate verification.
-This is useful for experimental setups and testing.
-
-.. _external-tls:
-
-Running a relay with externally managed certificates
------------------------------------------------------
-
-If you already have a TLS certificate manager
-(e.g. Traefik, certbot, or another ACME client)
-running on the deployment server,
-you can configure the relay to use those certificates
-instead of the built-in ``acmetool``.
-
-Set the following in ``chatmail.ini``::
-
-    tls_external_cert_and_key = /path/to/fullchain.pem /path/to/privkey.pem
-
-The paths must point to certificate and key files
-on the deployment server.
-During ``cmdeploy run``, these paths are written into
-the Postfix, Dovecot, and Nginx configurations.
-No certificate files are transferred from the build machine —
-they must already exist on the server,
-managed by your external certificate tool.
-
-The deploy will verify that both files exist on the server.
-``acmetool`` is **not** installed or run in this mode.
-
-.. note::
-
-   You are responsible for certificate renewal.
-   When the certificate file changes on disk,
-   all relay services pick up the new certificate automatically
-   via a systemd path watcher installed during deploy.
-   The watcher uses inotify, which does not cross bind-mount boundaries.
-   If you use such a setup, you must trigger the reload explicitly after renewal::
-
-      systemctl start tls-cert-reload.service
-
-
 Migrating to a new build machine
 ----------------------------------
 

doc/source/overview.rst

@@ -297,7 +297,8 @@ TLS requirements
 
 Postfix is configured to require valid TLS by setting
 `smtp_tls_security_level <https://www.postfix.org/postconf.5.html#smtp_tls_security_level>`_
-to ``verify``.
+to ``verify``. If emails don’t arrive at your chatmail relay server, the
+problem is likely that your relay does not have a valid TLS certificate.
 
 You can test it by resolving ``MX`` records of your relay domain and
 then connecting to MX relays (e.g ``mx.example.org``) with
@@ -308,11 +309,6 @@ When providing a TLS certificate to your chatmail relay server, make
 sure to provide the full certificate chain and not just the last
 certificate.
 
-If you use an external certificate manager (e.g. Traefik or certbot),
-set ``tls_external_cert_and_key`` in ``chatmail.ini``
-to provide the certificate and key paths.
-See :ref:`external-tls` for details.
-
 If you are running an Exim server and don’t see incoming connections
 from a chatmail relay server in the logs, make sure ``smtp_no_mail`` log
 item is enabled in the config with ``log_selector = +smtp_no_mail``. By
@@ -321,14 +317,6 @@ default Exim does not log sessions that are closed before sending the
 by Postfix, so you might think that connection is not established while
 actually it is a problem with your TLS certificate.
 
-If emails don’t arrive at your chatmail relay server, the
-problem is likely that your relay does not have a valid TLS certificate.
-
-Note that connections to relays with underscore-prefixed test domains
-(e.g. ``_chat.example.org``) use ``encrypt`` tls security level,
-because such domains cannot obtain valid Let's Encrypt certificates
-and run with self-signed certificates.
-
 
 .. _dovecot: https://dovecot.org
 .. _postfix: https://www.postfix.org

docker-compose.yaml

@@ -0,0 +1,52 @@
+services:
+  chatmail:
+    build:
+      context: ./
+      dockerfile: docker/chatmail_relay.dockerfile
+    image: chatmail-relay:latest
+    restart: unless-stopped
+    container_name: chatmail
+    # Required for systemd — use only one of the following:
+    cgroup: host          # compose v2 only
+    # privileged: true    # compose v1 (not tested)
+    tty: true # required for logs
+    tmpfs: # required for systemd
+      - /tmp
+      - /run
+      - /run/lock
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "3"
+    environment:
+      CHANGE_KERNEL_SETTINGS: "False"
+      MAIL_DOMAIN: $MAIL_DOMAIN
+      ACME_EMAIL: $ACME_EMAIL
+      RECREATE_VENV: $RECREATE_VENV
+      MAX_MESSAGE_SIZE: $MAX_MESSAGE_SIZE
+      DEBUG_COMMANDS_ENABLED: $DEBUG_COMMANDS_ENABLED
+      FORCE_REINIT_INI_FILE: $FORCE_REINIT_INI_FILE
+      USE_FOREIGN_CERT_MANAGER: $USE_FOREIGN_CERT_MANAGER
+      ENABLE_CERTS_MONITORING: $ENABLE_CERTS_MONITORING
+      CERTS_MONITORING_TIMEOUT: $CERTS_MONITORING_TIMEOUT
+      IS_DEVELOPMENT_INSTANCE: $IS_DEVELOPMENT_INSTANCE
+      CMDEPLOY_STAGES: ${CMDEPLOY_STAGES:-}
+    network_mode: "host"
+    volumes:
+      ## system
+      - /sys/fs/cgroup:/sys/fs/cgroup:rw # required for systemd
+      - ./:/opt/chatmail
+
+      ## data
+      - ./data/chatmail:/home
+      - ./data/chatmail-dkimkeys:/etc/dkimkeys
+      - ./data/chatmail-acme:/var/lib/acme
+
+      ## custom resources
+      # - ./custom/www/src/index.md:/opt/chatmail/www/src/index.md
+
+      ## debug
+      # - ./docker/files/setup_chatmail_docker.sh:/setup_chatmail_docker.sh
+      # - ./docker/files/entrypoint.sh:/entrypoint.sh
+      # - ./docker/files/update_ini.sh:/update_ini.sh

docker/chatmail_relay.dockerfile

@@ -0,0 +1,100 @@
+FROM jrei/systemd-debian:12 AS base
+
+ENV LANG=en_US.UTF-8
+
+RUN echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/01norecommend && \
+    echo 'APT::Install-Suggests "0";' >> /etc/apt/apt.conf.d/01norecommend && \
+    apt-get update && \
+    apt-get install -y \
+        ca-certificates && \
+    DEBIAN_FRONTEND=noninteractive \
+    TZ=Europe/London \
+    apt-get install -y tzdata && \
+    apt-get install -y locales && \
+    sed -i -e "s/# $LANG.*/$LANG UTF-8/" /etc/locale.gen && \
+    dpkg-reconfigure --frontend=noninteractive locales && \
+    update-locale LANG=$LANG \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN apt-get update && \
+    apt-get install -y \
+        git \
+        python3 \
+        python3-venv \
+        python3-virtualenv \
+        gcc \
+        python3-dev \
+        opendkim \
+        opendkim-tools \
+        curl \
+        rsync \
+        unbound \
+        unbound-anchor \
+        dnsutils \
+        postfix \
+        acl \
+        nginx \
+        libnginx-mod-stream \
+        fcgiwrap \
+        cron \
+    && for pkg in core imapd lmtpd; do \
+      case "$pkg" in \
+        core) sha256="43f593332e22ac7701c62d58b575d2ca409e0f64857a2803be886c22860f5587" ;; \
+        imapd) sha256="8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86" ;; \
+        lmtpd) sha256="2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab" ;; \
+      esac; \
+      url="https://download.delta.chat/dovecot/dovecot-${pkg}_2.3.21%2Bdfsg1-3_amd64.deb"; \
+      file="/tmp/$(basename "$url")"; \
+      curl -fsSL "$url" -o "$file"; \
+      echo "$sha256  $file" | sha256sum -c -; \
+      apt-get install -y "$file"; \
+      rm -f "$file"; \
+    done \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /opt/chatmail
+
+# --- Build-time install stage ---
+# Bake the "install" deployer stage into the image; we can't use
+# scripts/initenv.sh because /opt/chatmail is empty at build time as 
+# source arrives at runtime via volume mount., so we use a throwaway venv.
+# On container start only "configure,activate" stages run.
+COPY . /tmp/chatmail-src/
+WORKDIR /tmp/chatmail-src
+
+# Dummy config — deploy_chatmail() needs a parseable ini to instantiate deployers
+RUN printf '[params]\nmail_domain = build.local\n' > /tmp/chatmail.ini
+
+# Do what initenv.sh would do without the docs
+RUN python3 -m venv /tmp/build-venv && \
+    /tmp/build-venv/bin/pip install --no-cache-dir \
+        -e chatmaild -e cmdeploy
+
+RUN CMDEPLOY_STAGES=install \
+    CHATMAIL_INI=/tmp/chatmail.ini \
+    CHATMAIL_DOCKER=True \
+    /tmp/build-venv/bin/pyinfra @local \
+        /tmp/chatmail-src/cmdeploy/src/cmdeploy/run.py -y
+
+RUN rm -rf /tmp/chatmail-src /tmp/build-venv /tmp/chatmail.ini
+
+WORKDIR /opt/chatmail
+# --- End build-time install stage ---
+
+ARG SETUP_CHATMAIL_SERVICE_PATH=/lib/systemd/system/setup_chatmail.service
+COPY ./docker/files/setup_chatmail.service "$SETUP_CHATMAIL_SERVICE_PATH"
+RUN ln -sf "$SETUP_CHATMAIL_SERVICE_PATH" "/etc/systemd/system/multi-user.target.wants/setup_chatmail.service"
+
+COPY --chmod=555 ./docker/files/setup_chatmail_docker.sh /setup_chatmail_docker.sh
+COPY --chmod=555 ./docker/files/update_ini.sh /update_ini.sh
+COPY --chmod=555 ./docker/files/entrypoint.sh /entrypoint.sh
+
+VOLUME ["/sys/fs/cgroup", "/home"]
+
+STOPSIGNAL SIGRTMIN+3
+
+ENTRYPOINT ["/entrypoint.sh"]
+
+CMD [   "--default-standard-output=journal+console", \
+        "--default-standard-error=journal+console" ]
+

docker/cm_ini_to_env.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""Convert a chatmail.ini to a Docker .env file.
+
+Usage: python docker/cm_ini_to_env.py [chatmail.ini] [.env]
+
+Reads the ini file, extracts all non-default key=value pairs,
+and writes them as UPPER_CASE env vars suitable for docker-compose.
+"""
+
+import configparser
+import sys
+from pathlib import Path
+
+# Keys that only make sense for bare-metal deploys or are handled
+# separately by the Docker setup and should not appear in .env.
+SKIP_KEYS = set()
+
+# Keys that exist in .env but have a different name than the ini key.
+# ini_key -> env_key
+RENAMES = {}
+
+
+def read_ini(path):
+    """Return dict of key=value from [params] section."""
+    cp = configparser.ConfigParser()
+    cp.read(path)
+    if not cp.has_section("params"):
+        sys.exit(f"Error: {path} has no [params] section")
+    return dict(cp.items("params"))
+
+
+def read_defaults():
+    """Return dict of default values from the ini template."""
+    template = Path(__file__).resolve().parent.parent / "chatmaild/src/chatmaild/ini/chatmail.ini.f"
+    if not template.exists():
+        return {}
+    cp = configparser.ConfigParser()
+    cp.read(template)
+    if not cp.has_section("params"):
+        return {}
+    defaults = {}
+    for key, value in cp.items("params"):
+        # Template placeholders like {mail_domain} aren't real defaults.
+        if "{" not in value:
+            defaults[key] = value
+    return defaults
+
+
+def ini_to_env(ini_path, only_non_default=True):
+    """Yield (ENV_KEY, value) pairs from an ini file."""
+    params = read_ini(ini_path)
+    defaults = read_defaults() if only_non_default else {}
+
+    for key, value in sorted(params.items()):
+        if key in SKIP_KEYS:
+            continue
+        if only_non_default and key in defaults and value.strip() == defaults[key].strip():
+            continue
+        env_key = RENAMES.get(key, key.upper())
+        yield env_key, value.strip()
+
+
+def main():
+    ini_path = sys.argv[1] if len(sys.argv) > 1 else "chatmail.ini"
+    env_path = sys.argv[2] if len(sys.argv) > 2 else None
+
+    if not Path(ini_path).exists():
+        sys.exit(f"Error: {ini_path} not found")
+
+    lines = []
+    for env_key, value in ini_to_env(ini_path):
+        lines.append(f'{env_key}="{value}"')
+
+    output = "\n".join(lines) + "\n"
+
+    if env_path:
+        Path(env_path).write_text(output)
+        print(f"Wrote {len(lines)} variables to {env_path}")
+    else:
+        print(output, end="")
+
+
+if __name__ == "__main__":
+    main()

docker/example.env

@@ -0,0 +1,11 @@
+MAIL_DOMAIN="chat.example.com"
+# ACME_EMAIL=""
+# RECREATE_VENV="false"
+# MAX_MESSAGE_SIZE="50M"
+# DEBUG_COMMANDS_ENABLED="true"
+# FORCE_REINIT_INI_FILE="true"
+# USE_FOREIGN_CERT_MANAGER="True"
+# ENABLE_CERTS_MONITORING="true"
+# CERTS_MONITORING_TIMEOUT=10
+# IS_DEVELOPMENT_INSTANCE="True"
+# CMDEPLOY_STAGES - default: "configure,activate". Set to "install,configure,activate" to force full reinstall.

docker/files/entrypoint.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -eo pipefail
+
+unlink /etc/nginx/sites-enabled/default || true
+
+SETUP_CHATMAIL_SERVICE_PATH="${SETUP_CHATMAIL_SERVICE_PATH:-/lib/systemd/system/setup_chatmail.service}"
+
+env_vars=$(printenv | cut -d= -f1 | xargs)
+sed -i "s|<envs_list>|$env_vars|g" $SETUP_CHATMAIL_SERVICE_PATH
+
+exec /lib/systemd/systemd $@

docker/files/setup_chatmail.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Run container setup commands
+After=multi-user.target
+ConditionPathExists=/setup_chatmail_docker.sh
+
+[Service]
+Type=oneshot
+ExecStart=/bin/bash /setup_chatmail_docker.sh
+RemainAfterExit=true
+WorkingDirectory=/opt/chatmail
+PassEnvironment=<envs_list>
+
+[Install]
+WantedBy=multi-user.target

docker/files/setup_chatmail_docker.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+
+set -eo pipefail
+export INI_FILE="${INI_FILE:-chatmail.ini}"
+export ENABLE_CERTS_MONITORING="${ENABLE_CERTS_MONITORING:-true}"
+export CERTS_MONITORING_TIMEOUT="${CERTS_MONITORING_TIMEOUT:-60}"
+export PATH_TO_SSL="${PATH_TO_SSL:-/var/lib/acme/live/${MAIL_DOMAIN}}"
+export CHANGE_KERNEL_SETTINGS=${CHANGE_KERNEL_SETTINGS:-"False"}
+export RECREATE_VENV=${RECREATE_VENV:-"false"}
+
+if [ -z "$MAIL_DOMAIN" ]; then
+    echo "ERROR: Environment variable 'MAIL_DOMAIN' must be set!" >&2
+    exit 1
+fi
+
+debug_commands() {
+    echo "Executing debug commands"
+    # git config --global --add safe.directory /opt/chatmail
+    # ./scripts/initenv.sh
+}
+
+calculate_hash() {
+    find "$PATH_TO_SSL" -type f -exec sha1sum {} \; | sort | sha1sum | awk '{print $1}'
+}
+
+monitor_certificates() {
+    if [ "$ENABLE_CERTS_MONITORING" != "true" ]; then
+        echo "Certs monitoring disabled."
+        exit 0
+    fi
+
+    current_hash=$(calculate_hash)
+    previous_hash=$current_hash
+
+    while true; do
+        current_hash=$(calculate_hash)
+        if [[ "$current_hash" != "$previous_hash" ]]; then
+        # TODO: add an option to restart at a specific time interval 
+            echo "[INFO] Certificate's folder hash was changed, reloading nginx, dovecot and postfix services."
+            systemctl reload nginx.service
+            systemctl reload dovecot.service
+            systemctl reload postfix.service
+            previous_hash=$current_hash
+        fi
+        sleep $CERTS_MONITORING_TIMEOUT
+    done
+}
+
+### MAIN
+
+if [ "$DEBUG_COMMANDS_ENABLED" = true ]; then
+    debug_commands
+fi
+
+if [ "$FORCE_REINIT_INI_FILE" = true ]; then 
+    INI_CMD_ARGS=--force
+fi
+
+if [ ! -f /etc/dkimkeys/opendkim.private ]; then
+    /usr/sbin/opendkim-genkey -D /etc/dkimkeys -d $MAIL_DOMAIN -s opendkim
+fi
+chown opendkim:opendkim /etc/dkimkeys/opendkim.private
+chown opendkim:opendkim /etc/dkimkeys/opendkim.txt
+
+# TODO: Move to debug_commands after git clone is moved to dockerfile. 
+git config --global --add safe.directory /opt/chatmail
+if [ "$RECREATE_VENV" = true ]; then
+    rm -rf venv
+fi
+# Skip venv creation if it already exists
+if [ ! -x venv/bin/python ] || [ ! -x venv/bin/cmdeploy ]; then
+    ./scripts/initenv.sh
+fi
+
+./scripts/cmdeploy init --config "${INI_FILE}" $INI_CMD_ARGS $MAIL_DOMAIN || true
+bash /update_ini.sh
+
+export CMDEPLOY_STAGES="${CMDEPLOY_STAGES:-configure,activate}"
+./scripts/cmdeploy run --ssh-host @docker
+
+echo "ForwardToConsole=yes" >> /etc/systemd/journald.conf
+systemctl restart systemd-journald
+
+monitor_certificates &

docker/files/update_ini.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+set -eo pipefail
+
+INI_FILE="${INI_FILE:-chatmail.ini}"
+
+if [ ! -f "$INI_FILE" ]; then
+    echo "Error: file $INI_FILE not found." >&2
+    exit 1
+fi
+
+TMP_FILE="$(mktemp)"
+
+convert_to_bytes() {
+    local value="$1"
+    if [[ "$value" =~ ^([0-9]+)([KkMmGgTt])$ ]]; then
+        local num="${BASH_REMATCH[1]}"
+        local unit="${BASH_REMATCH[2]}"
+        case "$unit" in
+            [Kk]) echo $((num * 1024)) ;;
+            [Mm]) echo $((num * 1024 * 1024)) ;;
+            [Gg]) echo $((num * 1024 * 1024 * 1024)) ;;
+            [Tt]) echo $((num * 1024 * 1024 * 1024 * 1024)) ;;
+        esac
+    elif [[ "$value" =~ ^[0-9]+$ ]]; then
+        echo "$value"
+    else
+        echo "Error: incorrect size format: $value." >&2
+        return 1
+    fi
+}
+
+process_specific_params() {
+    local key=$1
+    local value=$2
+    local destination_file=$3
+
+    if [[ "$key" == "max_message_size" ]]; then
+        converted=$(convert_to_bytes "$value") || exit 1
+        if grep -q -e "## .* = .* bytes" "$destination_file"; then 
+            sed "s|## .* = .* bytes|## $value = $converted bytes|g" "$destination_file";
+        else
+            echo "## $value = $converted bytes"  >> "$destination_file"
+        fi
+        echo "$key = $converted" >> "$destination_file"
+    else
+        echo "$key = $value" >> "$destination_file"
+    fi
+}
+
+while IFS= read -r line; do
+    if [[ "$line" =~ ^[[:space:]]*#.* || "$line" =~ ^[[:space:]]*$ ]]; then
+        echo "$line" >> "$TMP_FILE"
+        continue
+    fi
+
+    if [[ "$line" =~ ^([a-z0-9_]+)[[:space:]]*=[[:space:]]*(.*)$ ]]; then
+        key="${BASH_REMATCH[1]}"
+        current_value="${BASH_REMATCH[2]}"
+        env_var_name=$(echo "$key" | tr 'a-z' 'A-Z')
+        env_value="${!env_var_name}"
+
+        if [[ -n "$env_value" ]]; then
+            process_specific_params "$key" "$env_value" "$TMP_FILE"
+        else
+            echo "$line" >> "$TMP_FILE"
+        fi
+    else
+        echo "$line" >> "$TMP_FILE"
+    fi
+done < "$INI_FILE"
+
+PERMS=$(stat -c %a "$INI_FILE")
+OWNER=$(stat -c %u "$INI_FILE")
+GROUP=$(stat -c %g "$INI_FILE")
+
+chmod "$PERMS" "$TMP_FILE"
+chown "$OWNER":"$GROUP" "$TMP_FILE"
+
+mv "$TMP_FILE" "$INI_FILE"

docs/DOCKER_INSTALLATION_EN.md

@@ -0,0 +1,185 @@
+# Known issues and limitations
+
+- Requires cgroups v2 configured in the system. Operation with cgroups v1 has not been tested.
+- Yes, of course, using systemd inside a container is a hack, and it would be better to split it into several services, but since this is an MVP, it turned out to be easier to do it this way initially than to rewrite the entire deployment system.
+- The Docker image is only suitable for amd64. If you need to run it on a different architecture, try modifying the Dockerfile (specifically the part responsible for installing dovecot).
+
+# Docker installation
+This section provides instructions for installing Chatmail using Docker Compose.
+
+**Note:** Docker Compose v2 is required (`docker compose`, not `docker-compose`) for its support of the `cgroup: host` option in `docker-compose.yaml` is only supported by Compose v2.
+[see documentation](https://docs.docker.com/engine/install/debian/#install-using-the-repository)
+```shell
+apt install docker-ce docker-compose-plugin docker.io- docker-compose-
+```
+
+## Preliminary setup
+We use `chat.example.org` as the Chatmail domain in the following steps.
+Please substitute it with your own domain.
+
+1. Setup the initial DNS records.
+   The following is an example in the familiar BIND zone file format with
+   a TTL of 1 hour (3600 seconds).
+   Please substitute your domain and IP addresses.
+
+   ```
+    chat.example.com. 3600 IN A 198.51.100.5
+    chat.example.com. 3600 IN AAAA 2001:db8::5
+    www.chat.example.com. 3600 IN CNAME chat.example.com.
+    mta-sts.chat.example.com. 3600 IN CNAME chat.example.com.
+   ```
+
+2. clone the repository on your server.
+
+   ```shell
+    git clone https://github.com/chatmail/relay
+    cd relay
+   ```
+
+## Installation
+
+1. Configure kernel parameters because they cannot be changed inside the container, specifically `fs.inotify.max_user_instances` and `fs.inotify.max_user_watches`. Run the following:
+
+```shell
+echo "fs.inotify.max_user_instances=65536" | sudo tee -a /etc/sysctl.d/99-inotify.conf
+echo "fs.inotify.max_user_watches=65536" | sudo tee -a /etc/sysctl.d/99-inotify.conf
+sudo sysctl --system
+```
+
+2. Copy `./docker/example.env` and rename it to `.env`. This file stores variables used in `docker-compose.yaml`.
+
+```shell
+cp ./docker/example.env .env
+```
+
+3. Configure environment variables in the `.env` file. These variables are used in the `docker-compose.yaml` file to pass repeated values.
+   Below is the list of variables used during deployment:
+
+- `MAIL_DOMAIN` – The domain name of the future server. (required)
+- `DEBUG_COMMANDS_ENABLED` – Run debug commands before installation. (default: `false`)
+- `FORCE_REINIT_INI_FILE` – Recreate the ini configuration file on startup. (default: `false`)
+- `USE_FOREIGN_CERT_MANAGER` – Use a third-party certificate manager. (default: `false`)
+- `RECREATE_VENV` - Recreate the virtual environment (venv). If set to `true`, the environment will be recreated when the container starts, which will increase the startup time of the service but can help avoid certain errors. (default: `false`)
+- `INI_FILE` – Path to the ini configuration file. (default: `./chatmail.ini`)
+- `PATH_TO_SSL` – Path to where the certificates are stored. (default: `/var/lib/acme/live/${MAIL_DOMAIN}`)
+- `ENABLE_CERTS_MONITORING` – Enable certificate monitoring if `USE_FOREIGN_CERT_MANAGER=true`. If certificates change, services will be automatically restarted. (default: `false`)
+- `CERTS_MONITORING_TIMEOUT` – Interval in seconds to check if certificates have changed. (default: `'60'`)
+- `CMDEPLOY_STAGES` – Deployment stages to run on container start. (default: `"configure,activate"`). Set to `"install,configure,activate"` to force a full reinstall.
+
+You can also use any variables from the [ini configuration file](https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/ini/chatmail.ini.f); they must be in uppercase.
+
+4. Build the Docker image:
+
+```shell
+docker compose build chatmail
+```
+
+5. Start docker compose and wait for the installation to finish:
+
+```shell
+docker compose up -d # start service
+docker compose logs -f chatmail # view container logs, press CTRL+C to exit
+```
+
+### venv creation
+The first container start takes longer because it creates the cmdeploy Python virtualenv at `/opt/chatmail/venv` (persisted on the host via volume mount). Subsequent starts reuse the existing venv. Set `RECREATE_VENV=true` in `.env` to force a rebuild if needed.
+
+6. After installation is complete, you can open `https://<your_domain_name>` in your browser.
+
+## Using custom files
+
+When using Docker, you can apply modified configuration files to make the installation more personalized. This is usually needed for the `www/src` section so that the Chatmail landing page is customized to your taste, but it can be used for any other cases as well.
+
+To replace files correctly:
+
+1. Create the `./custom` directory. It is in `.gitignore`, so it won’t cause conflicts when updating.
+
+```shell
+mkdir -p ./custom
+```
+
+2. Modify the required file. For example, `index.md`:
+
+```shell
+mkdir -p ./custom/www/src
+nano ./custom/www/src/index.md
+```
+
+3. In `docker-compose.yaml`, add the file mount in the `volumes` section:
+
+```yaml
+services:
+  chatmail:
+    volumes:
+      ...
+      ## custom resources
+      - ./custom/www/src/index.md:/opt/chatmail/www/src/index.md
+```
+
+4. Restart the service:
+
+```shell
+docker compose down
+docker compose up -d
+```
+
+## Migrating from a bare-metal install
+
+If you have an existing bare-metal Chatmail installation and want to switch to Docker:
+
+1. Stop all existing services:
+
+```shell
+systemctl stop postfix dovecot doveauth nginx opendkim unbound acmetool-redirector \
+  filtermail filtermail-incoming chatmail-turn iroh-relay chatmail-metadata \
+  lastlogin mtail
+systemctl disable postfix dovecot doveauth nginx opendkim unbound acmetool-redirector \
+  filtermail filtermail-incoming chatmail-turn iroh-relay chatmail-metadata \
+  lastlogin mtail
+```
+
+2. Convert your existing `chatmail.ini` to the Docker `.env` format:
+
+```shell
+python3 docker/cm_ini_to_env.py /usr/local/lib/chatmaild/chatmail.ini .env
+```
+
+3. Copy persistent data into the `./data/` subdirectories:
+
+```shell
+mkdir -p data/chatmail-dkimkeys data/chatmail-acme data/chatmail
+
+# DKIM keys
+cp -a /etc/dkimkeys/* data/chatmail-dkimkeys/
+
+# ACME certificates and account
+rsync -a /var/lib/acme/ data/chatmail-acme/
+
+# Mail data
+rsync -a /home/ data/chatmail/
+```
+
+Alternatively, you can mount `/home/vmail` directly by changing the volume in `docker-compose.yaml`:
+
+```yaml
+- /home/vmail:/home/vmail
+```
+
+The three `./data/` subdirectories cover all persistent state. Everything else is regenerated by the `configure` and `activate` stages on container start.
+
+## Forcing a full reinstall
+
+The Docker image bakes the install stage (binary downloads, package setup, chatmaild venv) into the image at build time. On container start, only the `configure` and `activate` stages run by default.
+
+To force a full reinstall (e.g., after updating the source), either rebuild the image:
+
+```shell
+docker compose build chatmail
+docker compose up -d
+```
+
+Or override the stages at runtime without rebuilding:
+
+```shell
+CMDEPLOY_STAGES="install,configure,activate" docker compose up -d
+```

docs/DOCKER_INSTALLATION_RU.md

@@ -0,0 +1,174 @@
+# Известные проблемы и ограничения
+- Chatmail будет переустановлен при каждом запуске контейнера (при первом - долго, при последующих быстрее). Так устроен изначальный установщик, потому что он не был заточен под docker. В конце документации [представлено](#фиксирование-версии-chatmail) возможное решение
+- Требуется настроенный в системе cgroups v2. Работа с cgroups v1 не тестировалась.
+- Да, понятно дело что systemd использовать в контейнере костыль и надо это всё разнести на несколько сервисов, но это MVP и в первом приближении оказалось сделать проще так, чем переписывать всю систему развертывания.
+- docker образ подходит только для amd64, если нужно запустить на другой архитектуре, попробуйте изменить dockerfile (конкретно ту часть что ответсвенна за установку dovecot)
+
+# Docker installation
+Здесь представлена инструкция по установке chatmail с помощью docker-compose.
+
+## Предварительная настройка
+We use `chat.example.org` as the chatmail domain in the following steps.
+Please substitute it with your own domain. 
+
+1. Настройте начальные записи DNS.Ниже приведен пример в привычном формате файла зоны BIND сTTL 1 час (3600 секунд).
+   Замените домен и IP-адреса на свои.
+
+   ```
+    chat.example.com. 3600 IN A 198.51.100.5
+    chat.example.com. 3600 IN AAAA 2001:db8::5
+    www.chat.example.com. 3600 IN CNAME chat.example.com.
+    mta-sts.chat.example.com. 3600 IN CNAME chat.example.com.
+   ```
+
+2. Склонируйте репозиторий на свой сервер.
+
+   ```shell
+    git clone https://github.com/chatmail/relay
+    cd relay
+   ```
+
+## Installation
+
+1. Настроить параметры ядра, потому что внутри контейнера их нельзя изменить, а конкретно `fs.inotify.max_user_instances` и `fs.inotify.max_user_watches`. Для этого выполнить следующее:
+```shell
+echo "fs.inotify.max_user_instances=65536" | sudo tee -a /etc/sysctl.d/99-inotify.conf
+echo "fs.inotify.max_user_watches=65536" | sudo tee -a /etc/sysctl.d/99-inotify.conf
+sudo sysctl --system
+```
+
+2. Скопировать `./docker/example.env` и переименовать в `.env`. Здесь хранятся переменные, которые используются в `docker-compose.yaml`.
+```shell
+cp ./docker/example.env .env
+```
+
+3. Настроить переменные окружения в `.env` файле. Эти переменные используются в `docker-compose.yaml` файле, чтобы передавать повторяющиеся значения.
+   Ниже перечислен список переменных учавствующих при развертывании:
+
+- `MAIL_DOMAIN` - Доменное имя будущего сервера. (required)
+- `DEBUG_COMMANDS_ENABLED` - Выполнить debug команды перед установкой. (default: `false`)
+- `FORCE_REINIT_INI_FILE` - Пересоздавать ini файл конфигурации при запуске. (default: `false`)
+- `USE_FOREIGN_CERT_MANAGER` - Использовать сторонний менеджер сертификатов. (default: `false`)
+- `RECREATE_VENV` - Пересоздать виртуальное окружение (venv). Если выставлено `true`, то окружение будет пересоздано при запуске контейнера, из-за чего включение сервиса займет больше времени, но поможет избежать ряда ошибок. (default: `false`)
+- `INI_FILE` - путь к ini файлу конфигурации. (default: `./chatmail.ini`)
+- `PATH_TO_SSL` - Путь где располагаются сертификаты. (default: `/var/lib/acme/live/${MAIL_DOMAIN}`)
+- `ENABLE_CERTS_MONITORING` - Включить мониторинг сертификатов, если `USE_FOREIGN_CERT_MANAGER=true`.  Если сертфикаты изменятся сервисы будут автоматически перезапущены. (default: `false`)
+- `CERTS_MONITORING_TIMEOUT` - Раз во сколько секунд проверять что изменились сертификаты. (default: `'60'`)
+
+Также могут быть использованы все переменные из [ini файла конфигурации](https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/ini/chatmail.ini.f), они обязаны быть в uppercase формате. 
+
+4. Собрать docker образ
+```shell
+docker compose build chatmail
+```
+
+5. Запустить docker compose и дождаться завершения установки
+```shell
+docker compose up -d # запуск сервиса
+docker compose logs -f chatmail # просмотр логов контейнера. Для выхода нажать CTRL+C
+```
+
+6. По окончанию установки можно открыть в браузер `https://<your_domain_name>`
+
+## Использование кастомных файлов
+При использовании docker есть возможность использовать измененые файлы конфигурации, чтобы сделать установку более персонализированной. Обычно это требуется для секции `www/src`, чтобы ознакомительная страница Chatmail была сделана на ваш вкус. Но также это можно использовать и для любых других случаев.
+
+Для того чтобы корректно выполнить подмену файлов необходимо 
+1. создать каталог `./custom`, он находится в `.gitignore`, поэтому при обновлении не вызовет конфликтов.
+```shell
+mkdir -p ./custom
+```
+
+2. Изменить нужный файл. Для примера возьмем `index.md`
+```shell
+mkdir -p ./custom/www/src
+nano ./custom/www/src/index.md
+```
+
+3. В `docker-compose.yaml` добавить монтирование файла с помощью секции `volumes`
+```yaml
+services:
+  chatmail:
+    volumes:
+      ...
+      ## custom resources
+      - ./custom/www/src/index.md:/opt/chatmail/www/src/index.md
+```
+
+4. Перезапустить сервис
+```shell
+docker compose down
+docker compose up -d
+```
+
+## Фиксирование версии Chatmail
+> [!note]
+> Это опциональные шаги, их делать требуется только если вас не устраивает что сервис устанавливается каждый раз при запуске
+
+Поскольку в текущей версии docker chatmail сервис устанавливается каждый раз запуске контейнера, чтобы этого не происходило можно зафиксировать версию контейнера после установки. Делается это следующим образом:
+
+1. Зафиксировать текущее состояние сконфигурированного контейнера
+```shell
+docker container commit chatmail configured-chatmail:$(date +'%Y-%m-%d')
+docker image ls | grep configured-chatmail
+```
+
+2. Изменить entrypoint для контейнера в `docker-compose.yaml` на
+```yaml
+services:
+  chatmail:
+    image: <image name from step 1>
+    volumes:
+      ...
+      ## custom resources
+      - ./custom/setup_chatmail_docker.sh:/setup_chatmail_docker.sh
+```
+
+3. Создать файл `./custom/setup_chatmail_docker.sh` с новым файлом конфигурации
+```shell
+mkdir -p ./custom
+cat > ./custom/setup_chatmail_docker.sh << 'EOF'
+#!/bin/bash
+
+set -eo pipefail
+
+export ENABLE_CERTS_MONITORING="${ENABLE_CERTS_MONITORING:-true}"
+export CERTS_MONITORING_TIMEOUT="${CERTS_MONITORING_TIMEOUT:-60}"
+export PATH_TO_SSL="${PATH_TO_SSL:-/var/lib/acme/live/${MAIL_DOMAIN}}"
+
+calculate_hash() {
+    find "$PATH_TO_SSL" -type f -exec sha1sum {} \; | sort | sha1sum | awk '{print $1}'
+}
+
+monitor_certificates() {
+    if [ "$ENABLE_CERTS_MONITORING" != "true" ]; then
+        echo "Certs monitoring disabled."
+        exit 0
+    fi
+
+    current_hash=$(calculate_hash)
+    previous_hash=$current_hash
+
+    while true; do
+        current_hash=$(calculate_hash)
+        if [[ "$current_hash" != "$previous_hash" ]]; then
+        # TODO: add an option to restart at a specific time interval 
+            echo "[INFO] Certificate's folder hash was changed, reloading nginx, dovecot and postfix services."
+            systemctl reload nginx.service
+            systemctl reload dovecot.service
+            systemctl reload postfix.service
+            previous_hash=$current_hash
+        fi
+        sleep $CERTS_MONITORING_TIMEOUT
+    done
+}
+
+monitor_certificates &
+EOF
+```
+
+4. Перезапустить сервис
+```shell
+docker compose down
+docker compose up -d
+```

www/src/dclogin.js

@@ -1,21 +0,0 @@
-/* dclogin profile generator for self-signed chatmail relays.
- * Fetches credentials from /new and generates a dclogin: QR code.
- * Requires qrcode-svg.min.js to be loaded first.
- */
-(function () {
-    function generateProfile() {
-        fetch('/new')
-            .then(function (r) { return r.json(); })
-            .then(function (data) {
-                var url = data.dclogin_url;
-                var link = document.getElementById('dclogin-link');
-                link.href = url;
-                var qrLink = document.getElementById('qr-link');
-                qrLink.href = url;
-                var qrCode = document.getElementById('qr-code');
-                var qr = new QRCode({ content: url, width: 300, height: 300, padding: 1, join: true });
-                qrCode.innerHTML = qr.svg();
-            });
-    }
-    generateProfile();
-})();

www/src/index.md

@@ -11,18 +11,6 @@ for Delta Chat users.  For details how it avoids storing personal information
 please see our [privacy policy](privacy.html). 
 {% endif %}
 
-{% if config.tls_cert_mode == "self" %}
-<a class="cta-button" id="dclogin-link" href="#">Get a {{config.mail_domain}} chat profile</a>
-
-If you are viewing this page on a different device
-without a Delta Chat app,
-you can also **scan this QR code** with Delta Chat:
-
-<a id="qr-link" href="#"><div id="qr-code"></div></a>
-
-<script src="qrcode-svg.min.js"></script>
-<script src="dclogin.js"></script>
-{% else %}
 <a class="cta-button" href="DCACCOUNT:https://{{ config.mail_domain }}/new">Get a {{config.mail_domain}} chat profile</a>
 
 If you are viewing this page on a different device
@@ -31,7 +19,6 @@ you can also **scan this QR code** with Delta Chat:
 
 <a href="DCACCOUNT:https://{{ config.mail_domain }}/new">
     <img width=300 style="float: none;" src="qr-chatmail-invite-{{config.mail_domain}}.png" /></a>
-{% endif %}
 
 🐣 **Choose** your Avatar and Name