config: fix overriding chatmail.ini params

.github/workflows/ci.yaml

@@ -15,7 +15,7 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: download filtermail
-        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.3.0/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
+        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.2.0/filtermail-x86_64-musl -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
       - name: run chatmaild tests 
         working-directory: chatmaild
         run: pipx run tox

.github/workflows/test-and-deploy-ipv4only.yaml

@@ -71,35 +71,26 @@ jobs:
       - name: run deploy-chatmail offline tests 
         run: pytest --pyargs cmdeploy 
 
-      - name: setup dependencies
+      - run: |
-        run: |
+          cmdeploy init staging-ipv4.testrun.org
-          ssh root@staging-ipv4.testrun.org apt update
+          sed -i 's#disable_ipv6 = False#disable_ipv6 = True#' chatmail.ini
-          ssh root@staging-ipv4.testrun.org apt install -y git python3.11-venv python3-dev gcc
+          sed -i 's/#\s*mtail_address/mtail_address/' chatmail.ini
-          ssh root@staging-ipv4.testrun.org git clone https://github.com/chatmail/relay
-          ssh root@staging-ipv4.testrun.org "cd relay && git checkout " ${{ github.head_ref }}
-          ssh root@staging-ipv4.testrun.org "cd relay && scripts/initenv.sh"
 
-      - name: initialize config
+      - run: cmdeploy run --verbose --skip-dns-check
-        run: |
-          ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy init staging-ipv4.testrun.org"
-          ssh root@staging-ipv4.testrun.org "sed -i 's#disable_ipv6 = False#disable_ipv6 = True#' relay/chatmail.ini"
-          ssh root@staging-ipv4.testrun.org "sed -i 's/#\s*mtail_address/mtail_address/' relay/chatmail.ini"
-
-      - run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy run --verbose --skip-dns-check"
 
       - name: set DNS entries
         run: |
-          ssh root@staging-ipv4.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
+          ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
-          ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns --zonefile staging-generated.zone"
+          cmdeploy dns --zonefile staging-generated.zone
-          ssh root@staging-ipv4.testrun.org cat relay/staging-generated.zone >> .github/workflows/staging-ipv4.testrun.org-default.zone
+          cat staging-generated.zone >> .github/workflows/staging-ipv4.testrun.org-default.zone
           cat .github/workflows/staging-ipv4.testrun.org-default.zone
           scp .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
           ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
           ssh root@ns.testrun.org systemctl reload nsd
 
       - name: cmdeploy test
-        run: ssh root@staging-ipv4.testrun.org "cd relay && CHATMAIL_DOMAIN2=ci-chatmail.testrun.org scripts/cmdeploy test --slow"
+        run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
 
       - name: cmdeploy dns
-        run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns -v"
+        run: cmdeploy dns -v
 

.github/workflows/test-and-deploy.yaml

@@ -76,7 +76,6 @@ jobs:
 
       - run: |
           cmdeploy init staging2.testrun.org
-          sed -i 's/^ssh_host/#ssh_host/' chatmail.ini
           sed -i 's/#\s*mtail_address/mtail_address/' chatmail.ini
 
       - run: cmdeploy run --verbose --skip-dns-check

chatmaild/src/chatmaild/config.py

@@ -1,4 +1,3 @@
-import os
 from pathlib import Path
 
 import iniconfig
@@ -9,28 +8,30 @@ from chatmaild.user import User
 def read_config(inipath):
     assert Path(inipath).exists(), inipath
     cfg = iniconfig.IniConfig(inipath)
-    return Config(inipath, params=cfg.sections["params"])
+    params = cfg.sections["params"]
+    default_config_content = get_default_config_content(params["mail_domain"])
+    df_params = iniconfig.IniConfig("ini", data=default_config_content)["params"]
+    new_params = dict(df_params.items())
+    new_params.update(params)
+    return Config(inipath, params=new_params)
 
 
 class Config:
     def __init__(self, inipath, params):
         self._inipath = inipath
         self.mail_domain = params["mail_domain"]
-        self.ssh_host = params.get("ssh_host", self.mail_domain)
         self.max_user_send_per_minute = int(params.get("max_user_send_per_minute", 60))
         self.max_user_send_burst_size = int(params.get("max_user_send_burst_size", 10))
-        self.max_mailbox_size = params.get("max_mailbox_size", "500M")
+        self.max_mailbox_size = params["max_mailbox_size"]
-        self.max_message_size = int(params.get("max_message_size", 31457280))
+        self.max_message_size = int(params.get("max_message_size", "31457280"))
-        self.delete_mails_after = params.get("delete_mails_after", "20")
+        self.delete_mails_after = params["delete_mails_after"]
-        self.delete_large_after = params.get("delete_large_after", "7")
+        self.delete_large_after = params["delete_large_after"]
-        self.delete_inactive_users_after = int(
+        self.delete_inactive_users_after = int(params["delete_inactive_users_after"])
-            params.get("delete_inactive_users_after", 100)
+        self.username_min_length = int(params["username_min_length"])
-        )
+        self.username_max_length = int(params["username_max_length"])
-        self.username_min_length = int(params.get("username_min_length", 9))
+        self.password_min_length = int(params["password_min_length"])
-        self.username_max_length = int(params.get("username_max_length", 9))
+        self.passthrough_senders = params["passthrough_senders"].split()
-        self.password_min_length = int(params.get("password_min_length", 9))
+        self.passthrough_recipients = params["passthrough_recipients"].split()
-        self.passthrough_senders = params.get("passthrough_senders", "").split()
-        self.passthrough_recipients = params.get("passthrough_recipients", "").split()
         self.www_folder = params.get("www_folder", "")
         self.filtermail_smtp_port = int(params.get("filtermail_smtp_port", "10080"))
         self.filtermail_smtp_port_incoming = int(
@@ -42,8 +43,6 @@ class Config:
         )
         self.mtail_address = params.get("mtail_address")
         self.disable_ipv6 = params.get("disable_ipv6", "false").lower() == "true"
-        self.addr_v4 = os.environ.get("CHATMAIL_ADDR_V4", "")
-        self.addr_v6 = os.environ.get("CHATMAIL_ADDR_V6", "")
         self.acme_email = params.get("acme_email", "")
         self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true"
         self.imap_compress = params.get("imap_compress", "false").lower() == "true"
@@ -57,6 +56,7 @@ class Config:
         self.privacy_mail = params.get("privacy_mail")
         self.privacy_pdo = params.get("privacy_pdo")
         self.privacy_supervisor = params.get("privacy_supervisor")
+        self.tmpfs_index = params.get("tmpfs_index", "false").lower() == "true"
 
         # deprecated option
         mbdir = params.get("mailboxes_dir", f"/home/vmail/mail/{self.mail_domain}")
@@ -112,10 +112,10 @@ def get_default_config_content(mail_domain, **overrides):
 
     if mail_domain.endswith(".testrun.org"):
         override_inipath = inidir.joinpath("override-testrun.ini")
-        privacy = iniconfig.IniConfig(override_inipath)["privacy"]
+        params = iniconfig.IniConfig(override_inipath)["params"]
         lines = []
         for line in content.split("\n"):
-            for key, value in privacy.items():
+            for key, value in params.items():
                 value_lines = value.format(mail_domain=mail_domain).strip().split("\n")
                 if not line.startswith(f"{key} =") or not value_lines:
                     continue

chatmaild/src/chatmaild/expire.py

@@ -17,14 +17,14 @@ from chatmaild.config import read_config
 FileEntry = namedtuple("FileEntry", ("path", "mtime", "size"))
 
 
-def iter_mailboxes(basedir, maxnum):
+def iter_mailboxes(basedir, maxnum, tmpfs_index):
     if not os.path.exists(basedir):
         print_info(f"no mailboxes found at: {basedir}")
         return
 
     for name in os_listdir_if_exists(basedir)[:maxnum]:
         if "@" in name:
-            yield MailboxStat(basedir + "/" + name)
+            yield MailboxStat(basedir + "/" + name, name, tmpfs_index)
 
 
 def get_file_entry(path):
@@ -49,11 +49,14 @@ def os_listdir_if_exists(path):
 class MailboxStat:
     last_login = None
 
-    def __init__(self, basedir):
+    def __init__(self, basedir, name, tmpfs_index):
         self.basedir = str(basedir)
+        self.name = name
         self.messages = []
         self.extrafiles = []
         self.scandir(self.basedir)
+        if tmpfs_index:
+            self.scandir("/dev/shm/" + name)
 
     def scandir(self, folderdir):
         for name in os_listdir_if_exists(folderdir):
@@ -90,11 +93,13 @@ class Expiry:
         self.all_files = 0
         self.start = time.time()
 
-    def remove_mailbox(self, mboxdir):
+    def remove_mailbox(self, mboxdir, name):
         if self.verbose:
             print_info(f"removing {mboxdir}")
         if not self.dry:
             shutil.rmtree(mboxdir)
+            if self.config.tmpfs_index:
+                shutil.rmtree("/dev/shm/" + name)
         self.del_mboxes += 1
 
     def remove_file(self, path, mtime=None):
@@ -121,7 +126,7 @@ class Expiry:
         self.all_mboxes += 1
         changed = False
         if mbox.last_login and mbox.last_login < cutoff_without_login:
-            self.remove_mailbox(mbox.basedir)
+            self.remove_mailbox(mbox.basedir, mbox.name)
             return
 
         mboxname = os.path.basename(mbox.basedir)
@@ -145,6 +150,9 @@ class Expiry:
             changed = True
         if changed:
             self.remove_file(f"{mbox.basedir}/maildirsize")
+        for file in mbox.extrafiles:
+            if "dovecot.index" in file.path.split("/")[-1] and file.size > 500 * 1024:
+                self.remove_file(file.path)
 
     def get_summary(self):
         return (
@@ -197,7 +205,9 @@ def main(args=None):
 
     maxnum = int(args.maxnum) if args.maxnum else None
     exp = Expiry(config, dry=not args.remove, now=now, verbose=args.verbose)
-    for mailbox in iter_mailboxes(str(config.mailboxes_dir), maxnum=maxnum):
+    for mailbox in iter_mailboxes(
+        str(config.mailboxes_dir), maxnum, config.tmpfs_index
+    ):
         exp.process_mailbox_stat(mailbox)
     print(exp.get_summary())
 

chatmaild/src/chatmaild/fsreport.py

@@ -159,7 +159,7 @@ def main(args=None):
 
     maxnum = int(args.maxnum) if args.maxnum else None
     rep = Report(now=now, min_login_age=int(args.min_login_age), mdir=args.mdir)
-    for mbox in iter_mailboxes(str(config.mailboxes_dir), maxnum=maxnum):
+    for mbox in iter_mailboxes(str(config.mailboxes_dir), maxnum, config.tmpfs_index):
         rep.process_mailbox_stat(mbox)
     rep.dump_summary()
 

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -3,9 +3,6 @@
 # mail domain (MUST be set to fully qualified chat mail domain)
 mail_domain = {mail_domain}
 
-# Where to deploy the relay - if unspecified, mail_domain will be used.
-ssh_host = localhost
-
 #
 # If you only do private test deploys, you don't need to modify any settings below
 #
@@ -51,6 +48,9 @@ passthrough_senders =
 # (space-separated, item may start with "@" to whitelist whole recipient domains)
 passthrough_recipients =
 
+# store index files in tmpfs (good for disk size and I/O, bad for ram)
+tmpfs_index = false
+
 # path to www directory - documented here: https://chatmail.at/doc/relay/getting_started.html#custom-web-pages
 #www_folder = www
 

chatmaild/src/chatmaild/ini/override-testrun.ini

@@ -1,5 +1,6 @@
+[params]
 
-[privacy]
+tmpfs_index = true
 
 passthrough_recipients = privacy@testrun.org echo@{mail_domain}
 

chatmaild/src/chatmaild/tests/test_expire.py

@@ -43,20 +43,22 @@ def create_new_messages(basedir, relpaths, size=1000, days=0):
 
 @pytest.fixture
 def mbox1(example_config):
-    mboxdir = example_config.mailboxes_dir.joinpath("mailbox1@example.org")
+    addr = "mailbox1@example.org"
+    mboxdir = example_config.mailboxes_dir.joinpath(addr)
     mboxdir.mkdir()
     fill_mbox(mboxdir)
-    return MailboxStat(mboxdir)
+    return MailboxStat(mboxdir, addr, False)
 
 
 def test_deltachat_folder(example_config):
     """Test old setups that might have a .DeltaChat folder where messages also need to get removed."""
-    mboxdir = example_config.mailboxes_dir.joinpath("mailbox1@example.org")
+    addr = "mailbox1@example.org"
+    mboxdir = example_config.mailboxes_dir.joinpath(addr)
     mboxdir.mkdir()
     mbox2dir = mboxdir.joinpath(".DeltaChat")
     mbox2dir.mkdir()
     fill_mbox(mbox2dir)
-    mb = MailboxStat(mboxdir)
+    mb = MailboxStat(mboxdir, addr, False)
     assert len(mb.messages) == 2
 
 
@@ -69,7 +71,11 @@ def test_filentry_ordering(tmp_path):
 
 
 def test_no_mailbxoes(tmp_path, capsys):
-    assert [] == list(iter_mailboxes(str(tmp_path.joinpath("notexists")), maxnum=10))
+    assert [] == list(
+        iter_mailboxes(
+            str(tmp_path.joinpath("notexists")), maxnum=10, tmpfs_index=False
+        )
+    )
     out, err = capsys.readouterr()
     assert "no mailboxes" in err
 
@@ -86,13 +92,13 @@ def test_stats_mailbox(mbox1):
 
     create_new_messages(mbox1.basedir, ["large-extra"], size=1000)
     create_new_messages(mbox1.basedir, ["index-something"], size=3)
-    mbox2 = MailboxStat(mbox1.basedir)
+    mbox2 = MailboxStat(mbox1.basedir, mbox1.name, False)
     assert len(mbox2.extrafiles) == 5
     assert mbox2.extrafiles[0].size == 1000
 
     # cope well with mailbox dirs that have no password (for whatever reason)
     Path(mbox1.basedir).joinpath("password").unlink()
-    mbox3 = MailboxStat(mbox1.basedir)
+    mbox3 = MailboxStat(mbox1.basedir, mbox1.name, False)
     assert mbox3.last_login is None
 
 

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -88,7 +88,7 @@ def run_cmd_options(parser):
 def run_cmd(args, out):
     """Deploy chatmail services on the remote server."""
 
-    ssh_host = args.ssh_host if args.ssh_host else args.config.ssh_host
+    ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
     sshexec = get_sshexec(ssh_host)
     require_iroh = args.config.enable_iroh_relay
     if not args.dns_check_disabled:
@@ -101,14 +101,11 @@ def run_cmd(args, out):
     env["CHATMAIL_WEBSITE_ONLY"] = "True" if args.website_only else ""
     env["CHATMAIL_DISABLE_MAIL"] = "True" if args.disable_mail else ""
     env["CHATMAIL_REQUIRE_IROH"] = "True" if require_iroh else ""
-    if not args.dns_check_disabled:
-        env["CHATMAIL_ADDR_V4"] = remote_data.get("A") or ""
-        env["CHATMAIL_ADDR_V6"] = remote_data.get("AAAA") or ""
     deploy_path = importlib.resources.files(__package__).joinpath("run.py").resolve()
     pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
 
     cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y"
-    if ssh_host in ["localhost", "@local", "@docker"]:
+    if ssh_host in ["localhost", "@docker"]:
         cmd = f"{pyinf} @local {deploy_path} -y"
 
     if version.parse(pyinfra.__version__) < version.parse("3"):
@@ -124,7 +121,7 @@ def run_cmd(args, out):
                 out.red("Website deployment failed.")
         elif retcode == 0:
             out.green("Deploy completed, call `cmdeploy dns` next.")
-        elif not args.dns_check_disabled and not remote_data["acme_account_url"]:
+        elif not remote_data["acme_account_url"]:
             out.red("Deploy completed but letsencrypt not configured")
             out.red("Run 'cmdeploy run' again")
             retcode = 0
@@ -149,7 +146,7 @@ def dns_cmd_options(parser):
 
 def dns_cmd(args, out):
     """Check DNS entries and optionally generate dns zone file."""
-    ssh_host = args.ssh_host if args.ssh_host else args.config.ssh_host
+    ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
     sshexec = get_sshexec(ssh_host, verbose=args.verbose)
     remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
     if not remote_data:
@@ -183,7 +180,7 @@ def status_cmd_options(parser):
 def status_cmd(args, out):
     """Display status for online chatmail instance."""
 
-    ssh_host = args.ssh_host if args.ssh_host else args.config.ssh_host
+    ssh_host = args.ssh_host if args.ssh_host else args.config.mail_domain
     sshexec = get_sshexec(ssh_host, verbose=args.verbose)
 
     out.green(f"chatmail domain: {args.config.mail_domain}")
@@ -203,7 +200,6 @@ def test_cmd_options(parser):
         action="store_true",
         help="also run slow tests",
     )
-    add_ssh_host_option(parser)
 
 
 def test_cmd(args, out):
@@ -215,9 +211,6 @@ def test_cmd(args, out):
     x = importlib.util.find_spec("deltachat")
     if x is None:
         out.check_call(f"{sys.executable} -m pip install deltachat")
-    env = os.environ.copy()
-    if args.ssh_host:
-        env["CHATMAIL_SSH"] = args.ssh_host
 
     pytest_path = shutil.which("pytest")
     pytest_args = [
@@ -231,7 +224,7 @@ def test_cmd(args, out):
     ]
     if args.slow:
         pytest_args.append("--slow")
-    ret = out.run_ret(pytest_args, env=env)
+    ret = out.run_ret(pytest_args)
     return ret
 
 

cmdeploy/src/cmdeploy/deployers.py

@@ -10,7 +10,6 @@ from pathlib import Path
 
 from chatmaild.config import read_config
 from pyinfra import facts, host, logger
-from pyinfra.facts import hardware
 from pyinfra.api import FactBase
 from pyinfra.facts.files import Sha256File
 from pyinfra.facts.systemd import SystemdEnabled
@@ -37,7 +36,7 @@ from .www import build_webpages, find_merge_conflict, get_paths
 
 class Port(FactBase):
     """
-    Returns the process occupying a port.
+    Returns the process occuping a port.
     """
 
     def command(self, port: int) -> str:
@@ -558,14 +557,6 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
             line="\nnameserver 9.9.9.9",
         )
 
-    # Check if mtail_address interface is available (if configured)
-    if config.mtail_address and config.mtail_address not in ('127.0.0.1', '::1', 'localhost'):
-        ipv4_addrs = host.get_fact(hardware.Ipv4Addrs)
-        all_addresses = [addr for addrs in ipv4_addrs.values() for addr in addrs]
-        if config.mtail_address not in all_addresses:
-            Out().red(f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n")
-            exit(1)
-
     port_services = [
         (["master", "smtpd"], 25),
         ("unbound", 53),
@@ -577,7 +568,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
         (["imap-login", "dovecot"], 993),
         ("iroh-relay", 3340),
         ("mtail", 3903),
-        ("stats", 3904),
+        ("dovecot-stats", 3904),
         ("nginx", 8443),
         (["master", "smtpd"], config.postfix_reinject_port),
         (["master", "smtpd"], config.postfix_reinject_port_incoming),
@@ -587,9 +578,8 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
     for service, port in port_services:
         print(f"Checking if port {port} is available for {service}...")
         running_service = host.get_fact(Port, port=port)
-        services = [service] if isinstance(service, str) else service
         if running_service:
-            if running_service not in services:
+            if running_service not in service:
                 Out().red(
                     f"Deploy failed: port {port} is occupied by: {running_service}"
                 )

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -68,13 +68,11 @@ userdb {
 ##
 
 # Mailboxes are stored in the "mail" directory of the vmail user home.
+{% if config.tmpfs_index %}
+mail_location = maildir:{{ config.mailboxes_dir }}/%u:INDEX=/dev/shm/%u
+{% else %}
 mail_location = maildir:{{ config.mailboxes_dir }}/%u
-
+{% endif %}
-# index/cache files are not very useful for chatmail relay operations 
-# but it's not clear how to disable them completely. 
-# According to https://doc.dovecot.org/2.3/settings/advanced/#core_setting-mail_cache_max_size
-# if the cache file becomes larger than the specified size, it is truncated by dovecot 
-mail_cache_max_size = 500K
 
 namespace inbox {
   inbox = yes

cmdeploy/src/cmdeploy/filtermail/deployer.py

@@ -14,10 +14,10 @@ class FiltermailDeployer(Deployer):
 
     def install(self):
         arch = host.get_fact(facts.server.Arch)
-        url = f"https://github.com/chatmail/filtermail/releases/download/v0.3.0/filtermail-{arch}"
+        url = f"https://github.com/chatmail/filtermail/releases/download/v0.2.0/filtermail-{arch}-musl"
         sha256sum = {
-            "x86_64": "f14a31323ae2dad3b59d3fdafcde507521da2f951a9478cd1f2fe2b4463df71d",
+            "x86_64": "1e5bbb646582cb16740c6dfbbca39edba492b78cc96ec9fa2528c612bb504edd",
-            "aarch64": "933770d75046c4fd7084ce8d43f905f8748333426ad839154f0fc654755ef09f",
+            "aarch64": "3564fba8605f8f9adfeefff3f4580533205da043f47c5968d0d10db17e50f44e",
         }[arch]
         self.need_restart |= files.download(
             name="Download filtermail",

cmdeploy/src/cmdeploy/postfix/deployer.py

@@ -61,20 +61,6 @@ class PostfixDeployer(Deployer):
         )
         need_restart |= lmtp_header_cleanup.changed
 
-        tls_policy_map = files.put(
-            name="Upload SMTP TLS Policy that accepts self-signed certificates for IP-only hosts",
-            src=get_resource("postfix/smtp_tls_policy_map"),
-            dest="/etc/postfix/smtp_tls_policy_map",
-            user="root",
-            group="root",
-            mode="644",
-        )
-        need_restart |= tls_policy_map.changed
-        if tls_policy_map.changed:
-            server.shell(
-                commands=["postmap /etc/postfix/smtp_tls_policy_map"],
-            )
-
         # Login map that 1:1 maps email address to login.
         login_map = files.put(
             src=get_resource("postfix/login_map"),

cmdeploy/src/cmdeploy/postfix/lmtp_header_cleanup

@@ -1,3 +1,2 @@
 /^DKIM-Signature:/            IGNORE
 /^Authentication-Results:/            IGNORE
-/^Received:/            IGNORE

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -25,7 +25,7 @@ smtp_tls_security_level=verify
 # <https://www.postfix.org/postconf.5.html#smtp_tls_servername>
 smtp_tls_servername = hostname
 smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
-smtp_tls_policy_maps = regexp:/etc/postfix/smtp_tls_policy_map
+smtp_tls_policy_maps = inline:{nauta.cu=may}
 smtp_tls_protocols = >=TLSv1.2
 smtp_tls_mandatory_protocols = >=TLSv1.2
 
@@ -69,15 +69,6 @@ mynetworks = 127.0.0.0/8
 {% else %}
 mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128
 {% endif %}
-{% if config.addr_v4 %}
-smtp_bind_address = {{ config.addr_v4 }}
-{% endif %}
-{% if config.addr_v6 %}
-smtp_bind_address6 = {{ config.addr_v6 }}
-{% endif %}
-{% if config.addr_v4 or config.addr_v6 %}
-smtp_bind_address_enforce = yes
-{% endif %}
 mailbox_size_limit = 0
 message_size_limit = {{config.max_message_size}}
 recipient_delimiter = +

cmdeploy/src/cmdeploy/postfix/smtp_tls_policy_map

@@ -1,2 +0,0 @@
-/^\[[^]]+\]$/ encrypt
-/^nauta\.cu$/    may

cmdeploy/src/cmdeploy/sshexec.py

@@ -85,31 +85,16 @@ class SSHExec:
 
 
 class LocalExec:
-    FuncError = FuncError
-
     def __init__(self, verbose=False, docker=False):
         self.verbose = verbose
         self.docker = docker
 
-    def __call__(self, call, kwargs=None, log_callback=None):
-        if kwargs is None:
-            kwargs = {}
-        return call(**kwargs)
-
     def logged(self, call, kwargs: dict):
-        title = call.__doc__
-        if not title:
-            title = call.__name__
         where = "locally"
         if self.docker:
             if call == remote.rdns.perform_initial_checks:
                 kwargs["pre_command"] = "docker exec chatmail "
                 where = "in docker"
         if self.verbose:
-            print_stderr(f"Running {where}: {title}(**{kwargs})")
+            print(f"Running {where}: {call.__name__}(**{kwargs})")
-            return self(call, kwargs, log_callback=print_stderr)
+        return call(**kwargs)
-        else:
-            print_stderr(title, end="")
-            res = self(call, kwargs, log_callback=remote.rshell.log_progress)
-            print_stderr()
-            return res

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -7,13 +7,13 @@ import time
 import pytest
 
 from cmdeploy import remote
-from cmdeploy.cmdeploy import get_sshexec
+from cmdeploy.sshexec import SSHExec
 
 
 class TestSSHExecutor:
     @pytest.fixture(scope="class")
     def sshexec(self, sshdomain):
-        return get_sshexec(sshdomain)
+        return SSHExec(sshdomain)
 
     def test_ls(self, sshexec):
         out = sshexec(call=remote.rdns.shell, kwargs=dict(command="ls"))
@@ -27,7 +27,6 @@ class TestSSHExecutor:
         assert res["A"] or res["AAAA"]
 
     def test_logged(self, sshexec, maildomain, capsys):
-        sshexec.verbose = False
         sshexec.logged(
             remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain)
         )
@@ -53,8 +52,6 @@ class TestSSHExecutor:
                 remote.rdns.perform_initial_checks,
                 kwargs=dict(mail_domain=None),
             )
-        except AssertionError:
-            pass
         except sshexec.FuncError as e:
             assert "rdns.py" in str(e)
             assert "AssertionError" in str(e)

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -7,7 +7,7 @@ import pytest
 import requests
 
 from cmdeploy.remote import rshell
-from cmdeploy.cmdeploy import get_sshexec
+from cmdeploy.sshexec import SSHExec
 
 
 @pytest.fixture
@@ -90,7 +90,7 @@ class TestEndToEndDeltaChat:
         lp.sec(f"filling remote inbox for {user}")
         fn = f"7743102289.M843172P2484002.c20,S={quota},W=2398:2,"
         path = chatmail_config.mailboxes_dir.joinpath(user, "cur", fn)
-        sshexec = get_sshexec(sshdomain)
+        sshexec = SSHExec(sshdomain)
         sshexec(call=rshell.write_numbytes, kwargs=dict(path=str(path), num=120))
         res = sshexec(call=rshell.dovecot_recalc_quota, kwargs=dict(user=user))
         assert res["percent"] >= 100

cmdeploy/src/cmdeploy/tests/online/test_3_status.py

@@ -5,11 +5,7 @@ from cmdeploy.cmdeploy import main
 
 def test_status_cmd(chatmail_config, capsys, request):
     os.chdir(request.config.invocation_params.dir)
-    command = ["status"]
+    assert main(["status"]) == 0
-    if os.getenv("CHATMAIL_SSH"):
-        command.append("--ssh-host")
-        command.append(os.getenv("CHATMAIL_SSH"))
-    assert main(command) == 0
     status_out = capsys.readouterr()
     print(status_out.out)
 

cmdeploy/src/cmdeploy/tests/plugin.py

@@ -54,8 +54,8 @@ def maildomain(chatmail_config):
 
 
 @pytest.fixture(scope="session")
-def sshdomain(chatmail_config):
+def sshdomain(maildomain):
-    return os.environ.get("CHATMAIL_SSH", chatmail_config.ssh_host)
+    return os.environ.get("CHATMAIL_SSH", maildomain)
 
 
 @pytest.fixture
@@ -337,14 +337,8 @@ class Remote:
 
     def iter_output(self, logcmd=""):
         getjournal = "journalctl -f" if not logcmd else logcmd
-        print(self.sshdomain)
-        match self.sshdomain:
-            case "@local": command = []
-            case "localhost": command = []
-            case _: command = ["ssh", f"root@{self.sshdomain}"]
-        [command.append(arg) for arg in getjournal.split()]
         self.popen = subprocess.Popen(
-            command,
+            ["ssh", f"root@{self.sshdomain}", getjournal],
             stdout=subprocess.PIPE,
         )
         while 1:

doc/source/getting_started.rst

@@ -16,11 +16,18 @@ You will need the following:
 
 -  Control over a domain through a DNS provider of your choice.
 
--  A Debian 12 server with reachable SMTP/SUBMISSIONS/IMAPS/HTTPS ports.
+-  A Debian 12 **deployment server** with reachable SMTP/SUBMISSIONS/IMAPS/HTTPS ports.
    IPv6 is encouraged if available. Chatmail relay servers only require
    1GB RAM, one CPU, and perhaps 10GB storage for a few thousand active
    chatmail addresses.
 
+-  A Linux or Unix **build machine** with key-based SSH access to the root
+   user of the deployment server.
+   You must add a passphrase-protected private key to your local ssh-agent because you
+   can’t type in your passphrase during deployment.
+   (An ed25519 private key is required due to an `upstream bug in
+   paramiko <https://github.com/paramiko/paramiko/issues/2191>`_)
+
 
 Setup with ``scripts/cmdeploy``
 -------------------------------------
@@ -28,7 +35,7 @@ Setup with ``scripts/cmdeploy``
 We use ``chat.example.org`` as the chatmail domain in the following
 steps. Please substitute it with your own domain.
 
-1. Setup the initial DNS records for your relay.
+1. Setup the initial DNS records for your deployment server.
    The following is an example in the
    familiar BIND zone file format with a TTL of 1 hour (3600 seconds).
    Please substitute your domain and IP addresses.
@@ -40,24 +47,29 @@ steps. Please substitute it with your own domain.
        www.chat.example.org. 3600 IN CNAME chat.example.org.
        mta-sts.chat.example.org. 3600 IN CNAME chat.example.org.
 
-2. Login to the server with SSH, clone the repository and bootstrap the Python
+2. On your local PC, clone the repository and bootstrap the Python
    virtualenv.
 
    ::
 
-       ssh root@chat.example.org
        git clone https://github.com/chatmail/relay
        cd relay
        scripts/initenv.sh
 
-3. Then, create a chatmail configuration file
+3. On your local build machine (PC), create a chatmail configuration file
    ``chatmail.ini``:
 
    ::
 
        scripts/cmdeploy init chat.example.org  # <-- use your domain
 
-4. Now run the deployment script to install the relay to the server:
+4. Verify that SSH root login to the deployment server server works:
+
+   ::
+
+       ssh root@chat.example.org  # <-- use your domain
+
+5. From your local build machine, setup and configure the remote deployment server:
 
    ::
 
@@ -68,31 +80,26 @@ steps. Please substitute it with your own domain.
    configure at your DNS provider (it can take some time until they are
    public).
 
-Next Steps
+Other helpful commands
-----------
-
-Now you should display and check all recommended DNS records
-to enable federation with other relays:
-
-::
-
-   scripts/cmdeploy dns
-
-You should also test whether your chatmail service is working correctly:
-
-::
-
-   scripts/cmdeploy test
-
-Other Helpful Commands
 ----------------------
 
-To check the status of your chatmail relay:
+To check the status of your deployment server running the chatmail service:
 
 ::
 
    scripts/cmdeploy status
 
+To display and check all recommended DNS records:
+
+::
+
+   scripts/cmdeploy dns
+
+To test whether your chatmail service is working correctly:
+
+::
+
+   scripts/cmdeploy test
 
 To measure the performance of your chatmail service:
 
@@ -134,9 +141,8 @@ This starts a local live development cycle for chatmail web pages:
    directory and generating HTML files and copying assets to the
    ``www/build`` directory.
 
--  if you are running scripts/cmdeploy webdev on the relay itself,
+-  Starts a browser window automatically where you can “refresh” as
-   you need to configure a route in /etc/nginx/nginx.conf
+   needed.
-   to expose the build directory.
 
 Custom web pages
 ----------------
@@ -154,7 +160,7 @@ Disable automatic address creation
 --------------------------------------------------------
 
 If you need to stop address creation, e.g. because some script is wildly
-creating addresses, login with ssh to the relay and run:
+creating addresses, login with ssh to the deployment machine and run:
 
 ::
 
@@ -162,3 +168,24 @@ creating addresses, login with ssh to the relay and run:
 
 Chatmail address creation will be denied while this file is present.
 
+
+Migrating to a new build machine
+----------------------------------
+
+To move or add a build machine,
+clone the relay repository on the new build machine, and copy the ``chatmail.ini`` file from the old build machine.
+Make sure ``rsync`` is installed, then initialize the environment:
+
+::
+
+   ./scripts/initenv.sh
+
+Run safety checks before a new deployment:
+
+::
+
+   ./scripts/cmdeploy dns
+   ./scripts/cmdeploy status
+
+If you keep multiple build machines (ie laptop and desktop), keep ``chatmail.ini`` in sync between
+them.

doc/source/related.rst

@@ -14,8 +14,8 @@ We know of three work-in-progress alternative implementation efforts:
    it to support all of the features and configuration settings required
    to operate as a chatmail relay.
 
--  `Madmail <https://github.com/themadorg/madmail>`_: an
+-  `Madmail <https://github.com/omidz4t/madmail>`_: an
-   experimental fork of `Maddy Mail Server <https://maddy.email/>`_, modified
+   experimental fork of Maddy Mail Server <https://maddy.email/>`_ optimized
    for chatmail deployments.  It provides a single binary solution
    for running a chatmail relay.