Revert "expanding ~ doesn't work with -i"

This reverts commit 4c62714fbb.

.github/workflows/ci.yaml

@@ -15,7 +15,7 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: download filtermail
-        run: curl -L https://kamiokan.de/bin/filtermail -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
+        run: curl -L https://github.com/chatmail/filtermail/releases/download/v0.5.2/filtermail-x86_64 -o /usr/local/bin/filtermail && chmod +x /usr/local/bin/filtermail
       - name: run chatmaild tests 
         working-directory: chatmaild
         run: pipx run tox

.github/workflows/test-and-deploy.yaml

@@ -1,4 +1,4 @@
-name: deploy on staging2.testrun.org, and run tests
+name: deploy on allocated VPS and run tests
 
 on:
   push:
@@ -12,86 +12,27 @@ on:
       - 'LICENSE'
 
 jobs:
-  deploy:
-    name: deploy on staging2.testrun.org, and run tests
+  deploy-and-test:
+    name: deploy on allocated VPS, and run tests
     runs-on: ubuntu-latest
-    timeout-minutes: 30
-    environment:
-      name: staging2.testrun.org
-      url: https://staging2.testrun.org/
-    concurrency: staging2.testrun.org
+    timeout-minutes: 15
     steps:
       - uses: actions/checkout@v4
+        with:
+          repository: chatmail/hetzner-relay
+          path: hetzner-relay
+
+      - uses: actions/checkout@v4
+        with:
+          path: relay
 
       - name: prepare SSH
         run: |
           mkdir ~/.ssh
-          echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
-          chmod 600 ~/.ssh/id_ed25519
-          ssh-keyscan staging2.testrun.org > ~/.ssh/known_hosts
-          # save previous acme & dkim state
-          rsync -avz root@staging2.testrun.org:/var/lib/acme . || true
-          rsync -avz root@staging2.testrun.org:/etc/dkimkeys . || true
-          # store previous acme & dkim state on ns.testrun.org, if it contains useful certs
-          if [ -f dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys root@ns.testrun.org:/tmp/ || true; fi
-          if [ "$(ls -A acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme root@ns.testrun.org:/tmp/ || true; fi
-          # make sure CAA record isn't set
-          scp -o StrictHostKeyChecking=accept-new .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
-          ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging2.testrun.org.zone
-          ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
-          ssh root@ns.testrun.org systemctl reload nsd
+          echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/staging.testrun.org
+          chmod 600 ~/.ssh/staging.testrun.org
 
-      - name: rebuild staging2.testrun.org to have a clean VPS
-        run: |
-            curl -X POST \
-            -H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
-            -H "Content-Type: application/json" \
-            -d '{"image":"debian-12"}' \
-            "https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_SERVER_ID }}/actions/rebuild"
+      - run: pip install hetzner-relay/
 
-      - run: scripts/initenv.sh
-
-      - name: append venv/bin to PATH
-        run: echo venv/bin >>$GITHUB_PATH
-
-      - name: upload TLS cert after rebuilding
-        run: |
-          echo " --- wait until staging2.testrun.org VPS is rebuilt --- "
-          rm ~/.ssh/known_hosts
-          while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org id -u ; do sleep 1 ; done
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org id -u
-          # download acme & dkim state from ns.testrun.org
-          rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme acme-restore || true
-          rsync -avz root@ns.testrun.org:/tmp/dkimkeys dkimkeys-restore || true
-          # restore acme & dkim state to staging2.testrun.org
-          rsync -avz acme-restore/acme root@staging2.testrun.org:/var/lib/ || true
-          rsync -avz dkimkeys-restore/dkimkeys root@staging2.testrun.org:/etc/ || true
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org chown root:root -R /var/lib/acme || true
-
-      - name: add hpk42 key to staging server
-        run: ssh root@staging2.testrun.org 'curl -s https://github.com/hpk42.keys >> .ssh/authorized_keys'
-
-      - name: run deploy-chatmail offline tests 
-        run: pytest --pyargs cmdeploy 
-
-      - run: |
-          cmdeploy init staging2.testrun.org
-          sed -i 's/#\s*mtail_address/mtail_address/' chatmail.ini
-
-      - run: cmdeploy run --verbose --skip-dns-check
-
-      - name: set DNS entries
-        run: |
-          cmdeploy dns --zonefile staging-generated.zone --verbose
-          cat staging-generated.zone >> .github/workflows/staging.testrun.org-default.zone
-          cat .github/workflows/staging.testrun.org-default.zone
-          scp .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
-          ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone
-          ssh root@ns.testrun.org systemctl reload nsd
-
-      - name: cmdeploy test
-        run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow
-
-      - name: cmdeploy dns
-        run: cmdeploy dns -v
+      - run: python3 hetzner-relay/main.py -i ~/.ssh/staging.testrun.org --dns --test --rebuild --run-id $GITHUB_RUN_ID --domain2 ci-chatmail.testrun.org --hetzner-api-token ${{ secrets.HETZNER_API_TOKEN }} relay/
 

chatmaild/pyproject.toml

@@ -6,7 +6,10 @@ build-backend = "setuptools.build_meta"
 name = "chatmaild"
 version = "0.3"
 dependencies = [
+  "aiosmtpd",
   "iniconfig",
+  "deltachat-rpc-server",
+  "deltachat-rpc-client",
   "filelock",
   "requests",
   "crypt-r >= 3.13.1 ; python_version >= '3.11'",
@@ -67,7 +70,6 @@ commands =
 deps = pytest 
        pdbpp 
        pytest-localserver
-       aiosmtpd
        execnet
 commands = pytest -v -rsXx {posargs}
 """

cmdeploy/pyproject.toml

@@ -10,6 +10,7 @@ dependencies = [
   "pillow",
   "qrcode",
   "markdown",
+  "pytest",
   "setuptools>=68",
   "termcolor",
   "build",
@@ -20,7 +21,6 @@ dependencies = [
   "execnet",
   "imap_tools",
   "deltachat-rpc-client",
-  "deltachat-rpc-server",
 ]
 
 [project.scripts]

cmdeploy/src/cmdeploy/basedeploy.py

@@ -1,7 +1,6 @@
 import importlib.resources
 import io
 import os
-from contextlib import contextmanager
 
 from pyinfra.operations import files, server, systemd
 
@@ -11,28 +10,6 @@ def has_systemd():
     return os.path.isdir("/run/systemd/system")
 
 
-@contextmanager
-def blocked_service_startup():
-    """Prevent services from auto-starting during package installation.
-
-    Installs a ``/usr/sbin/policy-rc.d`` that exits 101, blocking any
-    service from being started by the package manager.  This avoids bind
-    conflicts and CPU/RAM spikes during initial setup.  The file is removed
-    when the context exits.
-    """
-    # For documentation about policy-rc.d, see:
-    # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
-    files.put(
-        src=get_resource("policy-rc.d"),
-        dest="/usr/sbin/policy-rc.d",
-        user="root",
-        group="root",
-        mode="755",
-    )
-    yield
-    files.file("/usr/sbin/policy-rc.d", present=False)
-
-
 def get_resource(arg, pkg=__package__):
     return importlib.resources.files(pkg).joinpath(arg)
 

cmdeploy/src/cmdeploy/deployers.py

@@ -498,6 +498,10 @@ class ChatmailDeployer(Deployer):
             name="Install rsync",
             packages=["rsync"],
         )
+        apt.packages(
+            name="Ensure cron is installed",
+            packages=["cron"],
+        )
 
     def configure(self):
         # This file is used by auth proxy.

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -1,32 +1,20 @@
-import io
 import os
 import urllib.request
 
 from chatmaild.config import Config
 from pyinfra import host
-from pyinfra.facts.deb import DebPackages
 from pyinfra.facts.server import Arch, Sysctl
+from pyinfra.facts.systemd import SystemdEnabled
 from pyinfra.operations import apt, files, server, systemd
 
 from cmdeploy.basedeploy import (
     Deployer,
     activate_remote_units,
-    blocked_service_startup,
     configure_remote_units,
     get_resource,
+    has_systemd,
 )
 
-DOVECOT_VERSION = "2.3.21+dfsg1-3"
-
-DOVECOT_SHA256 = {
-    ("core", "amd64"): "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d",
-    ("core", "arm64"): "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9",
-    ("imapd", "amd64"): "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86",
-    ("imapd", "arm64"): "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f",
-    ("lmtpd", "amd64"): "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab",
-    ("lmtpd", "arm64"): "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f",
-}
-
 
 class DovecotDeployer(Deployer):
     daemon_reload = False
@@ -38,31 +26,11 @@ class DovecotDeployer(Deployer):
 
     def install(self):
         arch = host.get_fact(Arch)
-        with blocked_service_startup():
-            debs = []
-            for pkg in ("core", "imapd", "lmtpd"):
-                deb = _download_dovecot_package(pkg, arch)
-                if deb:
-                    debs.append(deb)
-            if debs:
-                deb_list = " ".join(debs)
-                server.shell(
-                    name="Install dovecot packages",
-                    commands=[
-                        f"dpkg --force-confdef --force-confold -i {deb_list} 2> /dev/null || true",
-                        "DEBIAN_FRONTEND=noninteractive apt-get -y --fix-broken install",
-                        f"dpkg --force-confdef --force-confold -i {deb_list}",
-                    ],
-                )
-        files.put(
-            name="Pin dovecot packages to block Debian dist-upgrades",
-            src=io.StringIO(
-                "Package: dovecot-*\n"
-                "Pin: version *\n"
-                "Pin-Priority: -1\n"
-            ),
-            dest="/etc/apt/preferences.d/pin-dovecot",
-        )
+        if has_systemd() and "dovecot.service" in host.get_fact(SystemdEnabled):
+            return  # already installed and running
+        _install_dovecot_package("core", arch)
+        _install_dovecot_package("imapd", arch)
+        _install_dovecot_package("lmtpd", arch)
 
     def configure(self):
         configure_remote_units(self.config.mail_domain, self.units)
@@ -95,37 +63,40 @@ def _pick_url(primary, fallback):
         return fallback
 
 
-def _download_dovecot_package(package: str, arch: str):
-    """Download a dovecot .deb if needed, return its path (or None)."""
+def _install_dovecot_package(package: str, arch: str):
     arch = "amd64" if arch == "x86_64" else arch
     arch = "arm64" if arch == "aarch64" else arch
-
-    pkg_name = f"dovecot-{package}"
-    sha256 = DOVECOT_SHA256.get((package, arch))
-    if sha256 is None:
-        apt.packages(packages=[pkg_name])
-        return None
-
-    installed_versions = host.get_fact(DebPackages).get(pkg_name, [])
-    if DOVECOT_VERSION in installed_versions:
-        return None
-
-    url_version = DOVECOT_VERSION.replace("+", "%2B")
-    deb_base = f"{pkg_name}_{url_version}_{arch}.deb"
-    primary_url = f"https://download.delta.chat/dovecot/{deb_base}"
-    fallback_url = f"https://github.com/chatmail/dovecot/releases/download/upstream%2F{url_version}/{deb_base}"
+    primary_url = f"https://download.delta.chat/dovecot/dovecot-{package}_2.3.21%2Bdfsg1-3_{arch}.deb"
+    fallback_url = f"https://github.com/chatmail/dovecot/releases/download/upstream%2F2.3.21%2Bdfsg1/dovecot-{package}_2.3.21%2Bdfsg1-3_{arch}.deb"
     url = _pick_url(primary_url, fallback_url)
-    deb_filename = f"/root/{deb_base}"
+    deb_filename = "/root/" + url.split("/")[-1]
+
+    match (package, arch):
+        case ("core", "amd64"):
+            sha256 = "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d"
+        case ("core", "arm64"):
+            sha256 = "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9"
+        case ("imapd", "amd64"):
+            sha256 = "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86"
+        case ("imapd", "arm64"):
+            sha256 = "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f"
+        case ("lmtpd", "amd64"):
+            sha256 = "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab"
+        case ("lmtpd", "arm64"):
+            sha256 = "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f"
+        case _:
+            apt.packages(packages=[f"dovecot-{package}"])
+            return
 
     files.download(
-        name=f"Download {pkg_name}",
+        name=f"Download dovecot-{package}",
         src=url,
         dest=deb_filename,
         sha256sum=sha256,
         cache_time=60 * 60 * 24 * 365 * 10,  # never redownload the package
     )
 
-    return deb_filename
+    apt.deb(name=f"Install dovecot-{package}", src=deb_filename)
 
 
 def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):

cmdeploy/src/cmdeploy/filtermail/deployer.py

@@ -14,9 +14,9 @@ class FiltermailDeployer(Deployer):
 
     def install(self):
         arch = host.get_fact(facts.server.Arch)
-        url = f"https://kamiokan.de/bin/filtermail"
+        url = f"https://github.com/chatmail/filtermail/releases/download/v0.5.2/filtermail-{arch}"
         sha256sum = {
-            "x86_64": "d64db7c295ba1c1c62ae592dd4ddbd179169ff7427382ce3f0d16ed2fb70d919",
+            "x86_64": "ce24ca0075aa445510291d775fb3aea8f4411818c7b885ae51a0fe18c5f789ce",
             "aarch64": "c5d783eefa5332db3d97a0e6a23917d72849e3eb45da3d16ce908a9b4e5a797d",
         }[arch]
         self.need_restart |= files.download(

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -20,7 +20,7 @@ smtpd_tls_key_file={{ config.tls_key_path }}
 smtpd_tls_security_level=may
 
 smtp_tls_CApath=/etc/ssl/certs
-smtp_tls_security_level=verify
+smtp_tls_security_level={{ "verify" if config.tls_cert_mode == "acme" else "encrypt" }}
 # Send SNI extension when connecting to other servers.
 # <https://www.postfix.org/postconf.5.html#smtp_tls_servername>
 smtp_tls_servername = hostname
@@ -88,22 +88,6 @@ inet_protocols = ipv4
 inet_protocols = all
 {% endif %}
 
-# Postfix does not try IPv4 and IPv6 connections
-# concurrently as of version 3.7.11.
-#
-# When relay has both A (IPv4) and AAAA (IPv6) records,
-# but broken IPv6 connectivity,
-# every second message is delayed by the connection timeout
-# <https://www.postfix.org/postconf.5.html#smtp_connect_timeout>
-# which defaults to 30 seconds. Reducing timeouts is not a solution
-# as this will result in a failure to connect to slow servers.
-#
-# As a workaround we always prefer IPv4 when it is available.
-# 
-# The setting is documented at
-# <https://www.postfix.org/postconf.5.html#smtp_address_preference>
-smtp_address_preference=ipv4
-
 virtual_transport = lmtp:unix:private/dovecot-lmtp
 virtual_mailbox_domains = {{ config.mail_domain }}
 lmtp_header_checks = regexp:/etc/postfix/lmtp_header_cleanup

cmdeploy/src/cmdeploy/tests/online/benchmark.py

@@ -1,3 +1,4 @@
+import time
 def test_tls_imap(benchmark, imap):
     def imap_connect():
         imap.connect()

cmdeploy/src/cmdeploy/tests/online/test_0_login.py

@@ -89,9 +89,7 @@ def test_concurrent_logins_same_account(
         assert login_results.get()
 
 
-def test_no_vrfy(cmfactory, chatmail_config):
-    ac = cmfactory.get_online_account()
-    addr = ac.get_config("addr")
+def test_no_vrfy(chatmail_config):
     domain = chatmail_config.mail_domain
 
     s = smtplib.SMTP(domain)
@@ -100,7 +98,7 @@ def test_no_vrfy(cmfactory, chatmail_config):
     s.putcmd("vrfy", f"wrongaddress@{chatmail_config.mail_domain}")
     result = s.getreply()
     print(result)
-    s.putcmd("vrfy", addr)
+    s.putcmd("vrfy", f"echo@{chatmail_config.mail_domain}")
     result2 = s.getreply()
     print(result2)
     assert result[0] == result2[0] == 252

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -6,8 +6,8 @@ import imap_tools
 import pytest
 import requests
 
-from cmdeploy.cmdeploy import get_sshexec
 from cmdeploy.remote import rshell
+from cmdeploy.cmdeploy import get_sshexec
 
 
 @pytest.fixture