config: comment out values in chatmail.ini.f, so defaults take precedence

* tests: fix test_remote[imap]
* cmdeploy: call LocalExec directly, not .logged()
* tests: fix TestSSHExecutor.test_logged
* tests: fix test_status_cmd with --ssh-host @local
* tests: fix test_logged with --ssh-host localhost
* tests: fix TestSSHExecutor::test_exception with --ssh-host localhost
* ci: deploy with --ssh-host localhost on staging-ipv4
* metadata: lower RestartSec

.github/workflows/test-and-deploy-ipv4only.yaml

@@ -71,25 +71,35 @@ jobs:
       - name: run deploy-chatmail offline tests 
         run: pytest --pyargs cmdeploy 
 
-      - run: |
-          cmdeploy init staging-ipv4.testrun.org
-          sed -i 's#disable_ipv6 = False#disable_ipv6 = True#' chatmail.ini
-          sed -i 's/#\s*mtail_address/mtail_address/' chatmail.ini
-          cmdeploy run --verbose --skip-dns-check
+      - name: setup dependencies
+        run: |
+          ssh root@staging-ipv4.testrun.org apt update
+          ssh root@staging-ipv4.testrun.org apt install -y git python3.11-venv python3-dev gcc
+          ssh root@staging-ipv4.testrun.org git clone https://github.com/chatmail/relay
+          ssh root@staging-ipv4.testrun.org "cd relay && git checkout " ${{ github.head_ref }}
+          ssh root@staging-ipv4.testrun.org "cd relay && scripts/initenv.sh"
+
+      - name: initialize config
+        run: |
+          ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy init staging-ipv4.testrun.org"
+          ssh root@staging-ipv4.testrun.org "sed -i 's#disable_ipv6 = False#disable_ipv6 = True#' relay/chatmail.ini"
+          ssh root@staging-ipv4.testrun.org "sed -i 's/#\s*mtail_address/mtail_address/' relay/chatmail.ini"
+
+      - run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy run --verbose --skip-dns-check --ssh-host localhost"
 
       - name: set DNS entries
         run: |
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
-          cmdeploy dns --zonefile staging-generated.zone
-          cat staging-generated.zone >> .github/workflows/staging-ipv4.testrun.org-default.zone
+          ssh root@staging-ipv4.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
+          ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns --zonefile staging-generated.zone --ssh-host localhost"
+          ssh root@staging-ipv4.testrun.org cat relay/staging-generated.zone >> .github/workflows/staging-ipv4.testrun.org-default.zone
           cat .github/workflows/staging-ipv4.testrun.org-default.zone
           scp .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
           ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone
           ssh root@ns.testrun.org systemctl reload nsd
 
       - name: cmdeploy test
-        run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
+        run: ssh root@staging-ipv4.testrun.org "cd relay && CHATMAIL_DOMAIN2=ci-chatmail.testrun.org scripts/cmdeploy test --slow --ssh-host localhost"
 
       - name: cmdeploy dns
-        run: cmdeploy dns -v
+        run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns -v --ssh-host localhost"
 

chatmaild/src/chatmaild/config.py

@@ -9,12 +9,7 @@ from chatmaild.user import User
 def read_config(inipath):
     assert Path(inipath).exists(), inipath
     cfg = iniconfig.IniConfig(inipath)
-    params = cfg.sections["params"]
-    default_config_content = get_default_config_content(params["mail_domain"])
-    df_params = iniconfig.IniConfig("ini", data=default_config_content)["params"]
-    new_params = dict(df_params.items())
-    new_params.update(params)
-    return Config(inipath, params=new_params)
+    return Config(inipath, params=cfg.sections["params"])
 
 
 class Config:
@@ -23,16 +18,18 @@ class Config:
         self.mail_domain = params["mail_domain"]
         self.max_user_send_per_minute = int(params.get("max_user_send_per_minute", 60))
         self.max_user_send_burst_size = int(params.get("max_user_send_burst_size", 10))
-        self.max_mailbox_size = params["max_mailbox_size"]
-        self.max_message_size = int(params.get("max_message_size", "31457280"))
-        self.delete_mails_after = params["delete_mails_after"]
-        self.delete_large_after = params["delete_large_after"]
-        self.delete_inactive_users_after = int(params["delete_inactive_users_after"])
-        self.username_min_length = int(params["username_min_length"])
-        self.username_max_length = int(params["username_max_length"])
-        self.password_min_length = int(params["password_min_length"])
-        self.passthrough_senders = params["passthrough_senders"].split()
-        self.passthrough_recipients = params["passthrough_recipients"].split()
+        self.max_mailbox_size = params.get("max_mailbox_size", "500M")
+        self.max_message_size = int(params.get("max_message_size", 31457280))
+        self.delete_mails_after = params.get("delete_mails_after", "20")
+        self.delete_large_after = params.get("delete_large_after", "7")
+        self.delete_inactive_users_after = int(
+            params.get("delete_inactive_users_after", 90)
+        )
+        self.username_min_length = int(params.get("username_min_length", 9))
+        self.username_max_length = int(params.get("username_max_length", 9))
+        self.password_min_length = int(params.get("password_min_length", 9))
+        self.passthrough_senders = params.get("passthrough_senders", "").split()
+        self.passthrough_recipients = params.get("passthrough_recipients", "").split()
         self.www_folder = params.get("www_folder", "")
         self.filtermail_smtp_port = int(params.get("filtermail_smtp_port", "10080"))
         self.filtermail_smtp_port_incoming = int(

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -12,41 +12,41 @@ mail_domain = {mail_domain}
 #
 
 # email sending rate per user and minute
-max_user_send_per_minute = 60
+#max_user_send_per_minute = 60
 
 # per-user max burst size for sending rate limiting (GCRA bucket capacity)
-max_user_send_burst_size = 10
+#max_user_send_burst_size = 10
 
 # maximum mailbox size of a chatmail address
-max_mailbox_size = 500M
+#max_mailbox_size = 500M
 
 # maximum message size for an e-mail in bytes
-max_message_size = 31457280
+#max_message_size = 31457280
 
 # days after which mails are unconditionally deleted
-delete_mails_after = 20
+#delete_mails_after = 20
 
 # days after which large messages (>200k) are unconditionally deleted
-delete_large_after = 7
+#delete_large_after = 7
 
 # days after which users without a successful login are deleted (database and mails)
-delete_inactive_users_after = 90
+#delete_inactive_users_after = 90
 
 # minimum length a username must have
-username_min_length = 9
+#username_min_length = 9
 
 # maximum length a username can have
-username_max_length = 9
+#username_max_length = 9
 
 # minimum length a password must have
-password_min_length = 9
+#password_min_length = 9
 
 # list of chatmail addresses which can send outbound un-encrypted mail
-passthrough_senders =
+#passthrough_senders =
 
 # list of e-mail recipients for which to accept outbound un-encrypted mails
 # (space-separated, item may start with "@" to whitelist whole recipient domains)
-passthrough_recipients =
+#passthrough_recipients =
 
 # path to www directory - documented here: https://chatmail.at/doc/relay/getting_started.html#custom-web-pages
 #www_folder = www
@@ -56,18 +56,18 @@ passthrough_recipients =
 #
 
 # SMTP outgoing filtermail and reinjection 
-filtermail_smtp_port = 10080
-postfix_reinject_port = 10025
+#filtermail_smtp_port = 10080
+#postfix_reinject_port = 10025
 
 # SMTP incoming filtermail and reinjection 
-filtermail_smtp_port_incoming = 10081
-postfix_reinject_port_incoming = 10026
+#filtermail_smtp_port_incoming = 10081
+#postfix_reinject_port_incoming = 10026
 
 # if set to "True" IPv6 is disabled
-disable_ipv6 = False
+#disable_ipv6 = False
 
 # Your email adress, which will be used in acmetool to manage Let's Encrypt SSL certificates
-acme_email = 
+#acme_email =
 
 # Defaults to https://iroh.{{mail_domain}} and running `iroh-relay` on the chatmail
 # service.
@@ -100,13 +100,13 @@ acme_email =
 # in per-maildir ".in/.out" files. 
 # Note that you need to manually cleanup these files
 # so use this option with caution on production servers. 
-imap_rawlog = false 
+#imap_rawlog = false
 
 # set to true if you want to enable the IMAP COMPRESS Extension,
 # which allows IMAP connections to be efficiently compressed.
 # WARNING: Enabling this makes it impossible to hibernate IMAP
 # processes which will result in much higher memory/RAM usage.
-imap_compress = false
+#imap_compress = false
 
 
 #

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -207,6 +207,7 @@ def test_cmd_options(parser):
         action="store_true",
         help="also run slow tests",
     )
+    add_ssh_host_option(parser)
 
 
 def test_cmd(args, out):
@@ -218,6 +219,9 @@ def test_cmd(args, out):
     x = importlib.util.find_spec("deltachat")
     if x is None:
         out.check_call(f"{sys.executable} -m pip install deltachat")
+    env = os.environ.copy()
+    if args.ssh_host:
+        env["CHATMAIL_SSH"] = args.ssh_host
 
     pytest_path = shutil.which("pytest")
     pytest_args = [
@@ -231,7 +235,7 @@ def test_cmd(args, out):
     ]
     if args.slow:
         pytest_args.append("--slow")
-    ret = out.run_ret(pytest_args)
+    ret = out.run_ret(pytest_args, env=env)
     return ret
 
 

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -22,7 +22,7 @@ class DovecotDeployer(Deployer):
 
     def install(self):
         arch = host.get_fact(Arch)
-        if not "dovecot.service" in host.get_fact(SystemdEnabled):
+        if not host.get_fact(SystemdEnabled).get("dovecot.service"):
             _install_dovecot_package("core", arch)
             _install_dovecot_package("imapd", arch)
             _install_dovecot_package("lmtpd", arch)

cmdeploy/src/cmdeploy/service/chatmail-metadata.service.f

@@ -4,7 +4,7 @@ Description=Chatmail dict proxy for IMAP METADATA
 [Service]
 ExecStart={execpath} /run/chatmail-metadata/metadata.socket {config_path}
 Restart=always
-RestartSec=30
+RestartSec=5
 User=vmail
 RuntimeDirectory=chatmail-metadata
 UMask=0077

cmdeploy/src/cmdeploy/sshexec.py

@@ -85,16 +85,31 @@ class SSHExec:
 
 
 class LocalExec:
+    FuncError = FuncError
+
     def __init__(self, verbose=False, docker=False):
         self.verbose = verbose
         self.docker = docker
 
+    def __call__(self, call, kwargs=None, log_callback=None):
+        if kwargs is None:
+            kwargs = {}
+        return call(**kwargs)
+
     def logged(self, call, kwargs: dict):
+        title = call.__doc__
+        if not title:
+            title = call.__name__
         where = "locally"
         if self.docker:
             if call == remote.rdns.perform_initial_checks:
                 kwargs["pre_command"] = "docker exec chatmail "
                 where = "in docker"
         if self.verbose:
-            print(f"Running {where}: {call.__name__}(**{kwargs})")
-        return call(**kwargs)
+            print_stderr(f"Running {where}: {title}(**{kwargs})")
+            return self(call, kwargs, log_callback=print_stderr)
+        else:
+            print_stderr(title, end="")
+            res = self(call, kwargs, log_callback=remote.rshell.log_progress)
+            print_stderr()
+            return res

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -7,13 +7,13 @@ import time
 import pytest
 
 from cmdeploy import remote
-from cmdeploy.sshexec import SSHExec
+from cmdeploy.cmdeploy import get_sshexec
 
 
 class TestSSHExecutor:
     @pytest.fixture(scope="class")
     def sshexec(self, sshdomain):
-        return SSHExec(sshdomain)
+        return get_sshexec(sshdomain)
 
     def test_ls(self, sshexec):
         out = sshexec(call=remote.rdns.shell, kwargs=dict(command="ls"))
@@ -27,6 +27,7 @@ class TestSSHExecutor:
         assert res["A"] or res["AAAA"]
 
     def test_logged(self, sshexec, maildomain, capsys):
+        sshexec.verbose = False
         sshexec.logged(
             remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain)
         )
@@ -52,6 +53,8 @@ class TestSSHExecutor:
                 remote.rdns.perform_initial_checks,
                 kwargs=dict(mail_domain=None),
             )
+        except AssertionError:
+            pass
         except sshexec.FuncError as e:
             assert "rdns.py" in str(e)
             assert "AssertionError" in str(e)
@@ -218,7 +221,7 @@ def test_expunged(remote, chatmail_config):
     ]
     outdated_days = int(chatmail_config.delete_large_after) + 1
     find_cmds.append(
-        "find {chatmail_config.mailboxes_dir} -path '*/cur/*' -mtime +{outdated_days} -size +200k -type f"
+        f"find {chatmail_config.mailboxes_dir} -path '*/cur/*' -mtime +{outdated_days} -size +200k -type f"
     )
     for cmd in find_cmds:
         for line in remote.iter_output(cmd):

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -7,7 +7,7 @@ import pytest
 import requests
 
 from cmdeploy.remote import rshell
-from cmdeploy.sshexec import SSHExec
+from cmdeploy.cmdeploy import get_sshexec
 
 
 @pytest.fixture
@@ -91,7 +91,7 @@ class TestEndToEndDeltaChat:
         lp.sec(f"filling remote inbox for {user}")
         fn = f"7743102289.M843172P2484002.c20,S={quota},W=2398:2,"
         path = chatmail_config.mailboxes_dir.joinpath(user, "cur", fn)
-        sshexec = SSHExec(sshdomain)
+        sshexec = get_sshexec(sshdomain)
         sshexec(call=rshell.write_numbytes, kwargs=dict(path=str(path), num=120))
         res = sshexec(call=rshell.dovecot_recalc_quota, kwargs=dict(user=user))
         assert res["percent"] >= 100

cmdeploy/src/cmdeploy/tests/online/test_3_status.py

@@ -5,7 +5,11 @@ from cmdeploy.cmdeploy import main
 
 def test_status_cmd(chatmail_config, capsys, request):
     os.chdir(request.config.invocation_params.dir)
-    assert main(["status"]) == 0
+    command = ["status"]
+    if os.getenv("CHATMAIL_SSH"):
+        command.append("--ssh-host")
+        command.append(os.getenv("CHATMAIL_SSH"))
+    assert main(command) == 0
     status_out = capsys.readouterr()
     print(status_out.out)
 

cmdeploy/src/cmdeploy/tests/plugin.py

@@ -361,8 +361,14 @@ class Remote:
 
     def iter_output(self, logcmd=""):
         getjournal = "journalctl -f" if not logcmd else logcmd
+        print(self.sshdomain)
+        match self.sshdomain:
+            case "@local": command = []
+            case "localhost": command = []
+            case _: command = ["ssh", f"root@{self.sshdomain}"]
+        [command.append(arg) for arg in getjournal.split()]
         self.popen = subprocess.Popen(
-            ["ssh", f"root@{self.sshdomain}", getjournal],
+            command,
             stdout=subprocess.PIPE,
         )
         while 1: