Add execnet dependency

.github/workflows/docs-preview.yaml

@@ -11,9 +11,6 @@ jobs:
   scripts:
     name: build 
     runs-on: ubuntu-latest
-    environment:
-      name: 'staging.chatmail.at/doc/relay/'
-      url: https://staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}
     steps:
       - uses: actions/checkout@v4
 
@@ -47,6 +44,36 @@ jobs:
           chmod 600 "$HOME/.ssh/key"
           rsync -rILvh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/doc/build/ "${{ secrets.USERNAME }}@chatmail.at:/var/www/html/staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}/"
 
+      - name: "Post links to details"
+        id: details
+        if: steps.prepare.outputs.uploadtoserver
+        run: |
+          # URLs for API connection and uploads
+          export GITHUB_API_URL="https://api.github.com/repos/chatmail/relay/statuses/${{ github.event.after }}"
+          export PREVIEW_LINK="https://staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}/"
+          export STATUS_DATA="{\"state\": \"success\", \
+                               \"description\": \"Preview the changed documentation here:\", \
+                               \"context\": \"Documentation Preview\", \
+                               \"target_url\": \"${PREVIEW_LINK}\"}"
+          curl -X POST --header "Accept: application/vnd.github+json" --header "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" --url "$GITHUB_API_URL" --header "content-type: application/json" --data "$STATUS_DATA"
+
+          #check if comment already exists, if not post it
+          export GITHUB_API_URL="https://api.github.com/repos/chatmail/relay/issues/${{ steps.prepare.outputs.prid }}/comments"
+          export RESPONSE=$(curl -L --header "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" --url "$GITHUB_API_URL" --header "content-type: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28")
+          echo $RESPONSE > response
+          grep -v '"Check out the page preview at https://staging.chatmail.at/doc/relay' response && echo "comment=true" >> $GITHUB_OUTPUT || true
+      - name: "Post link to comments"
+        if: steps.details.outputs.comment
+        uses: actions/github-script@v7
+        with:
+          script: |
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: "Check out the page preview at https://staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}/"
+            })
+
       - name: check links 
         working-directory: doc
         run: sphinx-build --builder linkcheck source build

.github/workflows/docs.yaml

@@ -14,9 +14,6 @@ jobs:
   scripts:
     name: build 
     runs-on: ubuntu-latest
-    environment: 
-      name: 'chatmail.at/doc/relay/'
-      url: https://chatmail.at/doc/relay/
     steps:
       - uses: actions/checkout@v4
 

.github/workflows/test-and-deploy-ipv4only.yaml

@@ -16,9 +16,6 @@ jobs:
     name: deploy on staging-ipv4.testrun.org, and run tests
     runs-on: ubuntu-latest
     timeout-minutes: 30
-    environment:
-      name: staging-ipv4.testrun.org
-      url: https://staging-ipv4.testrun.org/
     concurrency:
       group: ci-ipv4-${{ github.workflow }}-${{ github.ref }}
       cancel-in-progress: ${{ !contains(github.ref, '$GITHUB_REF') }}
@@ -35,11 +32,11 @@ jobs:
           chmod 600 ~/.ssh/id_ed25519
           ssh-keyscan staging-ipv4.testrun.org > ~/.ssh/known_hosts
           # save previous acme & dkim state
-          rsync -avz --delete root@staging-ipv4.testrun.org:/var/lib/acme acme-ipv4
+          rsync -avz root@staging-ipv4.testrun.org:/var/lib/acme acme-ipv4 || true
-          rsync -avz --delete root@staging-ipv4.testrun.org:/etc/dkimkeys dkimkeys-ipv4
+          rsync -avz root@staging-ipv4.testrun.org:/etc/dkimkeys dkimkeys-ipv4 || true
           # store previous acme & dkim state on ns.testrun.org, if it contains useful certs
-          test -f dkimkeys-ipv4/dkimkeys/opendkim.private && rsync -avz --delete -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys-ipv4 root@ns.testrun.org:/tmp/
+          if [ -f dkimkeys-ipv4/dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys-ipv4 root@ns.testrun.org:/tmp/ || true; fi
-          test -n "$(ls -A acme-ipv4/acme/certs)" && rsync -avz --delete -e "ssh -o StrictHostKeyChecking=accept-new" acme-ipv4 root@ns.testrun.org:/tmp/
+          if [ "$(ls -A acme-ipv4/acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme-ipv4 root@ns.testrun.org:/tmp/ || true; fi
           # make sure CAA record isn't set
           scp -o StrictHostKeyChecking=accept-new .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone
           ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging-ipv4.testrun.org.zone

.github/workflows/test-and-deploy.yaml

@@ -16,9 +16,6 @@ jobs:
     name: deploy on staging2.testrun.org, and run tests
     runs-on: ubuntu-latest
     timeout-minutes: 30
-    environment:
-      name: staging2.testrun.org
-      url: https://staging2.testrun.org/
     concurrency:
       group: ci-${{ github.workflow }}-${{ github.ref }}
       cancel-in-progress: ${{ !contains(github.ref, '$GITHUB_REF') }}
@@ -35,11 +32,11 @@ jobs:
           chmod 600 ~/.ssh/id_ed25519
           ssh-keyscan staging2.testrun.org > ~/.ssh/known_hosts
           # save previous acme & dkim state
-          rsync -avz --delete root@staging2.testrun.org:/var/lib/acme .
+          rsync -avz root@staging2.testrun.org:/var/lib/acme . || true
-          rsync -avz --delete root@staging2.testrun.org:/etc/dkimkeys .
+          rsync -avz root@staging2.testrun.org:/etc/dkimkeys . || true
           # store previous acme & dkim state on ns.testrun.org, if it contains useful certs
-          test -f dkimkeys/opendkim.private && rsync -avz --delete -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys root@ns.testrun.org:/tmp/
+          if [ -f dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys root@ns.testrun.org:/tmp/ || true; fi
-          test -n "$(ls -A acme/certs)" && rsync -avz --delete -e "ssh -o StrictHostKeyChecking=accept-new" acme root@ns.testrun.org:/tmp/
+          if [ "$(ls -A acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme root@ns.testrun.org:/tmp/ || true; fi
           # make sure CAA record isn't set
           scp -o StrictHostKeyChecking=accept-new .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone
           ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging2.testrun.org.zone
@@ -73,9 +70,6 @@ jobs:
           rsync -avz dkimkeys-restore/dkimkeys root@staging2.testrun.org:/etc/ || true
           ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org chown root:root -R /var/lib/acme || true
 
-      - name: add hpk42 key to staging server
-        run: ssh root@staging2.testrun.org 'curl -s https://github.com/hpk42.keys >> .ssh/authorized_keys'
-
       - name: run deploy-chatmail offline tests 
         run: pytest --pyargs cmdeploy 
 

CHANGELOG.md

@@ -2,15 +2,6 @@
 
 ## untagged
 
-- Add imap_compress option to chatmail.ini
-  ([#760](https://github.com/chatmail/relay/pull/760))
-
-- Remove echobot from relays 
-  ([#753](https://github.com/chatmail/relay/pull/753))
-
-- Fix `cmdeploy webdev`
-  ([#743](https://github.com/chatmail/relay/pull/743))
-
 - Add robots.txt to exclude all web crawlers
   ([#732](https://github.com/chatmail/relay/pull/732))
 

chatmaild/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "chatmaild"
-version = "0.3"
+version = "0.2"
 dependencies = [
   "aiosmtpd",
   "iniconfig",
@@ -25,6 +25,7 @@ where = ['src']
 doveauth = "chatmaild.doveauth:main"
 chatmail-metadata = "chatmaild.metadata:main"
 filtermail = "chatmaild.filtermail:main"
+echobot = "chatmaild.echo:main"
 chatmail-metrics = "chatmaild.metrics:main"
 chatmail-expire = "chatmaild.expire:main"
 chatmail-fsreport = "chatmaild.fsreport:main"

chatmaild/src/chatmaild/config.py

@@ -4,6 +4,8 @@ import iniconfig
 
 from chatmaild.user import User
 
+echobot_password_path = Path("/run/echobot/password")
+
 
 def read_config(inipath):
     assert Path(inipath).exists(), inipath
@@ -44,7 +46,6 @@ class Config:
         self.disable_ipv6 = params.get("disable_ipv6", "false").lower() == "true"
         self.acme_email = params.get("acme_email", "")
         self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true"
-        self.imap_compress = params.get("imap_compress", "false").lower() == "true"
         if "iroh_relay" not in params:
             self.iroh_relay = "https://" + params["mail_domain"]
             self.enable_iroh_relay = True
@@ -71,7 +72,10 @@ class Config:
             raise ValueError(f"invalid address {addr!r}")
 
         maildir = self.mailboxes_dir.joinpath(addr)
-        password_path = maildir.joinpath("password")
+        if addr.startswith("echo@"):
+            password_path = echobot_password_path
+        else:
+            password_path = maildir.joinpath("password")
 
         return User(maildir, addr, password_path, uid="vmail", gid="vmail")
 

chatmaild/src/chatmaild/doveauth.py

@@ -40,6 +40,10 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
         return False
     localpart, domain = parts
 
+    if localpart == "echo":
+        # echobot account should not be created in the database
+        return False
+
     if (
         len(localpart) > config.username_max_length
         or len(localpart) < config.username_min_length

chatmaild/src/chatmaild/echo.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+"""Advanced echo bot example.
+
+it will echo back any message that has non-empty text and also supports the /help command.
+"""
+
+import logging
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+from deltachat_rpc_client import Bot, DeltaChat, EventType, Rpc, events
+
+from chatmaild.config import echobot_password_path, read_config
+from chatmaild.doveauth import encrypt_password
+from chatmaild.newemail import create_newemail_dict
+
+hooks = events.HookCollection()
+
+
+@hooks.on(events.RawEvent)
+def log_event(event):
+    if event.kind == EventType.INFO:
+        logging.info(event.msg)
+    elif event.kind == EventType.WARNING:
+        logging.warning(event.msg)
+
+
+@hooks.on(events.RawEvent(EventType.ERROR))
+def log_error(event):
+    logging.error("%s", event.msg)
+
+
+@hooks.on(events.MemberListChanged)
+def on_memberlist_changed(event):
+    logging.info(
+        "member %s was %s", event.member, "added" if event.member_added else "removed"
+    )
+
+
+@hooks.on(events.GroupImageChanged)
+def on_group_image_changed(event):
+    logging.info("group image %s", "deleted" if event.image_deleted else "changed")
+
+
+@hooks.on(events.GroupNameChanged)
+def on_group_name_changed(event):
+    logging.info(f"group name changed, old name: {event.old_name}")
+
+
+@hooks.on(events.NewMessage(func=lambda e: not e.command))
+def echo(event):
+    snapshot = event.message_snapshot
+    if snapshot.is_info:
+        # Ignore info messages
+        return
+    if snapshot.text or snapshot.file:
+        snapshot.chat.send_message(text=snapshot.text, file=snapshot.file)
+
+
+@hooks.on(events.NewMessage(command="/help"))
+def help_command(event):
+    snapshot = event.message_snapshot
+    snapshot.chat.send_text("Send me any message and I will echo it back")
+
+
+def main():
+    logging.basicConfig(level=logging.INFO)
+    path = os.environ.get("PATH")
+    venv_path = sys.argv[0].strip("echobot")
+    os.environ["PATH"] = path + ":" + venv_path
+    with Rpc() as rpc:
+        deltachat = DeltaChat(rpc)
+        system_info = deltachat.get_system_info()
+        logging.info(f"Running deltachat core {system_info.deltachat_core_version}")
+
+        accounts = deltachat.get_all_accounts()
+        account = accounts[0] if accounts else deltachat.add_account()
+
+        bot = Bot(account, hooks)
+
+        config = read_config(sys.argv[1])
+        addr = "echo@" + config.mail_domain
+
+        # Create password file
+        if bot.is_configured():
+            password = bot.account.get_config("mail_pw")
+        else:
+            password = create_newemail_dict(config)["password"]
+
+        echobot_password_path.write_text(encrypt_password(password))
+        # Give the user which doveauth runs as access to the password file.
+        subprocess.check_call(
+            ["/usr/bin/setfacl", "-m", "user:vmail:r", echobot_password_path],
+        )
+
+        if not bot.is_configured():
+            bot.configure(addr, password)
+
+        # write invite link to working directory
+        invitelink = bot.account.get_qr_code()
+        Path("invite-link.txt").write_text(invitelink)
+
+        bot.run_forever()
+
+
+if __name__ == "__main__":
+    main()

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -43,9 +43,9 @@ passthrough_senders =
 
 # list of e-mail recipients for which to accept outbound un-encrypted mails
 # (space-separated, item may start with "@" to whitelist whole recipient domains)
-passthrough_recipients =
+passthrough_recipients = echo@{mail_domain}
 
-# path to www directory - documented here: https://chatmail.at/doc/relay/getting_started.html#custom-web-pages
+# path to www directory - documented here: https://github.com/chatmail/relay/#custom-web-pages
 #www_folder = www
 
 #
@@ -99,12 +99,6 @@ acme_email =
 # so use this option with caution on production servers. 
 imap_rawlog = false 
 
-# set to true if you want to enable the IMAP COMPRESS Extension,
-# which allows IMAP connections to be efficiently compressed.
-# WARNING: Enabling this makes it impossible to hibernate IMAP
-# processes which will result in much higher memory/RAM usage.
-imap_compress = false
-
 
 #
 # Privacy Policy

chatmaild/src/chatmaild/lastlogin.py

@@ -13,6 +13,8 @@ class LastLoginDictProxy(DictProxy):
         keyname = parts[1].split("/")
         value = parts[2] if len(parts) > 2 else ""
         if keyname[0] == "shared" and keyname[1] == "last-login":
+            if addr.startswith("echo@"):
+                return True
             addr = keyname[2]
             timestamp = int(value)
             user = self.config.get_user(addr)

chatmaild/src/chatmaild/tests/test_lastlogin.py

@@ -36,3 +36,29 @@ def test_handle_dovecot_request_last_login(testaddr, example_config):
     res = dictproxy.handle_dovecot_request(msg, dictproxy_transactions)
     assert res == "O\n"
     assert len(dictproxy_transactions) == 0
+
+
+def test_handle_dovecot_request_last_login_echobot(example_config):
+    dictproxy = LastLoginDictProxy(config=example_config)
+
+    authproxy = AuthDictProxy(config=example_config)
+    testaddr = f"echo@{example_config.mail_domain}"
+    authproxy.lookup_passdb(testaddr, "ignore")
+    user = dictproxy.config.get_user(testaddr)
+
+    transactions = {}
+
+    # set last-login info for user
+    tx = "1111"
+    msg = f"B{tx}\t{testaddr}"
+    res = dictproxy.handle_dovecot_request(msg, transactions)
+    assert not res
+    assert transactions == {tx: dict(addr=testaddr, res="O\n")}
+
+    timestamp = int(time.time())
+    msg = f"S{tx}\tshared/last-login/{testaddr}\t{timestamp}"
+    res = dictproxy.handle_dovecot_request(msg, transactions)
+    assert not res
+    assert len(transactions) == 1
+    read_timestamp = user.get_last_login_timestamp()
+    assert read_timestamp is None

chatmaild/src/chatmaild/user.py

@@ -19,7 +19,7 @@ class User:
 
     @property
     def can_track(self):
-        return "@" in self.addr
+        return "@" in self.addr and not self.addr.startswith("echo@")
 
     def get_userdb_dict(self):
         """Return a non-empty dovecot 'userdb' style dict
@@ -55,9 +55,11 @@ class User:
         try:
             write_bytes_atomic(self.password_path, password)
         except PermissionError:
-            logging.error(f"could not write password for: {self.addr}")
+            if not self.addr.startswith("echo@"):
-            raise
+                logging.error(f"could not write password for: {self.addr}")
-        self.enforce_E2EE_path.touch()
+                raise
+        if not self.addr.startswith("echo@"):
+            self.enforce_E2EE_path.touch()
 
     def set_last_login_timestamp(self, timestamp):
         """Track login time with daily granularity

cmdeploy/src/cmdeploy/acmetool/__init__.py

@@ -61,19 +61,6 @@ class AcmetoolDeployer(Deployer):
             mode="644",
         )
 
-        server.shell(
-            name=f"Remove old acmetool desired files for {self.domains[0]}",
-            commands=[f"rm -f /var/lib/acme/desired/{self.domains[0]}-*"],
-        )
-        files.template(
-            src=importlib.resources.files(__package__).joinpath("desired.yaml.j2"),
-            dest=f"/var/lib/acme/desired/{self.domains[0]}", # 0 is mailhost TLD
-            user="root",
-            group="root",
-            mode="644",
-            domains=self.domains,
-        )
-
         service_file = files.put(
             src=importlib.resources.files(__package__).joinpath(
                 "acmetool-redirector.service"
@@ -136,6 +123,6 @@ class AcmetoolDeployer(Deployer):
         self.need_restart_reconcile_timer = False
 
         server.shell(
-            name=f"Reconcile certificates for: {', '.join(self.domains)}",
+            name=f"Request certificate for: {', '.join(self.domains)}",
-            commands=["acmetool --batch --xlog.severity=debug reconcile"],
+            commands=[f"acmetool want --xlog.severity=debug {' '.join(self.domains)}"],
         )

cmdeploy/src/cmdeploy/acmetool/desired.yaml.j2

@@ -1,6 +0,0 @@
-satisfy:
-  names:
-{%- for domain in domains %}
-  - {{ domain }}
-{%- endfor %}
-

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -109,6 +109,15 @@ def run_cmd(args, out):
     try:
         retcode = out.check_call(cmd, env=env)
         if retcode == 0:
+            if not args.disable_mail:
+                print("\nYou can try out the relay by talking to this echo bot: ")
+                sshexec = SSHExec(args.config.mail_domain, verbose=args.verbose)
+                print(
+                    sshexec(
+                        call=remote.rshell.shell,
+                        kwargs=dict(command="cat /var/lib/echobot/invite-link.txt"),
+                    )
+                )
             out.green("Deploy completed, call `cmdeploy dns` next.")
         elif not remote_data["acme_account_url"]:
             out.red("Deploy completed but letsencrypt not configured")

cmdeploy/src/cmdeploy/deployers.py

@@ -270,14 +270,6 @@ class LegacyRemoveDeployer(Deployer):
             path="/var/log/journal/",
             present=False,
         )
-        # remove echobot if it is still running
-        if host.get_fact(SystemdEnabled).get("echobot.service"):
-            systemd.service(
-                name="Disable echobot.service",
-                service="echobot.service",
-                running=False,
-                enabled=False,
-            )
 
 
 def check_config(config):
@@ -412,6 +404,30 @@ class JournaldDeployer(Deployer):
         self.need_restart = False
 
 
+class EchobotDeployer(Deployer):
+    #
+    # This deployer depends on the dovecot and postfix deployers because
+    # it needs to base its decision of whether to restart the service on
+    # whether those two services were restarted.
+    #
+    def __init__(self, mail_domain):
+        self.mail_domain = mail_domain
+        self.units = ["echobot"]
+
+    def install(self):
+        apt.packages(
+            # required for setfacl for echobot
+            name="Install acl",
+            packages="acl",
+        )
+
+    def configure(self):
+        configure_remote_units(self.mail_domain, self.units)
+
+    def activate(self):
+        activate_remote_units(self.units)
+
+
 class ChatmailVenvDeployer(Deployer):
     def __init__(self, config):
         self.config = config
@@ -440,6 +456,7 @@ class ChatmailVenvDeployer(Deployer):
 class ChatmailDeployer(Deployer):
     required_users = [
         ("vmail", "vmail", None),
+        ("echobot", None, None),
         ("iroh", None, None),
     ]
 
@@ -573,6 +590,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
         PostfixDeployer(config, disable_mail),
         FcgiwrapDeployer(),
         NginxDeployer(config),
+        EchobotDeployer(mail_domain),
         MtailDeployer(config.mtail_address),
         GithashDeployer(),
     ]

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -113,7 +113,7 @@ mail_attribute_dict = proxy:/run/chatmail-metadata/metadata.socket:metadata
 # `imap_zlib` enables IMAP COMPRESS (RFC 4978).
 # <https://datatracker.ietf.org/doc/html/rfc4978.html>
 protocol imap {
-  mail_plugins = $mail_plugins imap_quota last_login {% if config.imap_compress %}imap_zlib{% endif %}
+  mail_plugins = $mail_plugins imap_zlib imap_quota last_login
   imap_metadata = yes
 }
 
@@ -252,28 +252,3 @@ protocol imap {
   rawlog_dir = %h 
 } 
 {% endif %}
-
-{% if not config.imap_compress %}
-# Hibernate IDLE users to save memory and CPU resources
-# NOTE: this will have no effect if imap_zlib plugin is used
-imap_hibernate_timeout = 30s
-service imap {
-  # Note that this change will allow any process running as
-  # $default_internal_user (dovecot) to access mails as any other user.
-  # This may be insecure in some installations, which is why this isn't
-  # done by default.
-  unix_listener imap-master {
-    user = $default_internal_user
-  }
-}
-# The following is the default already in v2.3.1+:
-service imap {
-  extra_groups = $default_internal_group
-}
-service imap-hibernate {
-  unix_listener imap-hibernate {
-    mode = 0660
-    group = $default_internal_group
-  }
-}
-{% endif %}

cmdeploy/src/cmdeploy/opendkim/final.lua

@@ -10,7 +10,6 @@ if nsigs == nil then
 end
 
 local valid = false
-local error_msg = "No valid DKIM signature found."
 for i = 1, nsigs do
 	sig = odkim.get_sighandle(ctx, i - 1)
 	sigres = odkim.sig_result(sig)
@@ -22,8 +21,6 @@ for i = 1, nsigs do
 	-- means the message is acceptable.
 	if sigres == 0 then
 		valid = true
-    else
-        error_msg = "DKIM signature is invalid, error code " .. tostring(sigres) .. ", search https://github.com/trusteddomainproject/OpenDKIM/blob/master/libopendkim/dkim.h#L108"
 	end
 end
 
@@ -34,7 +31,7 @@ if valid then
 		odkim.del_header(ctx, "DKIM-Signature", i)
 	end
 else
-	odkim.set_reply(ctx, "554", "5.7.1", error_msg)
+	odkim.set_reply(ctx, "554", "5.7.1", "No valid DKIM signature found")
 	odkim.set_result(ctx, SMFIS_REJECT)
 end
 

cmdeploy/src/cmdeploy/remote/rdns.py

@@ -37,10 +37,7 @@ def perform_initial_checks(mail_domain, pre_command=""):
         return res
 
     # parse out sts-id if exists, example: "v=STSv1; id=2090123"
-    mta_sts_txt = query_dns("TXT", f"_mta-sts.{mail_domain}")
+    parts = query_dns("TXT", f"_mta-sts.{mail_domain}").split("id=")
-    if not mta_sts_txt:
-        return res
-    parts = mta_sts_txt.split("id=")
     res["sts_id"] = parts[1].rstrip('"') if len(parts) == 2 else ""
     return res
 

cmdeploy/src/cmdeploy/service/echobot.service.f

@@ -0,0 +1,67 @@
+[Unit]
+Description=Chatmail echo bot for testing it works
+
+[Service]
+ExecStart={execpath} {config_path}
+Environment="PATH={remote_venv_dir}:$PATH"
+Restart=always
+RestartSec=30
+
+User=echobot
+Group=echobot
+
+# Create /var/lib/echobot
+StateDirectory=echobot
+
+# Create /run/echobot
+#
+# echobot stores /run/echobot/password
+# with a password there, which doveauth then reads.
+RuntimeDirectory=echobot
+
+WorkingDirectory=/var/lib/echobot
+
+# Apply security restrictions suggested by
+#   systemd-analyze security echobot.service
+CapabilityBoundingSet=
+LockPersonality=true
+MemoryDenyWriteExecute=true
+NoNewPrivileges=true
+PrivateDevices=true
+PrivateMounts=true
+PrivateTmp=true
+
+# We need to know about doveauth user to give it access to /run/echobot/password
+PrivateUsers=false
+
+ProtectClock=true
+ProtectControlGroups=true
+ProtectHostname=true
+ProtectKernelLogs=true
+ProtectKernelModules=true
+ProtectKernelTunables=true
+ProtectProc=noaccess
+
+# Should be "strict", but we currently write /accounts folder in a protected path
+ProtectSystem=full
+
+RemoveIPC=true
+RestrictAddressFamilies=AF_INET AF_INET6
+RestrictNamespaces=true
+RestrictRealtime=true
+RestrictSUIDSGID=true
+SystemCallArchitectures=native
+SystemCallFilter=~@clock
+SystemCallFilter=~@cpu-emulation
+SystemCallFilter=~@debug
+SystemCallFilter=~@module
+SystemCallFilter=~@mount
+SystemCallFilter=~@obsolete
+SystemCallFilter=~@raw-io
+SystemCallFilter=~@reboot
+SystemCallFilter=~@resources
+SystemCallFilter=~@swap
+UMask=0077
+
+[Install]
+WantedBy=multi-user.target

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -1,4 +1,5 @@
 import datetime
+import os
 import smtplib
 import socket
 import subprocess
@@ -7,6 +8,7 @@ import time
 import pytest
 
 from cmdeploy import remote
+from cmdeploy.cmdeploy import main
 from cmdeploy.sshexec import SSHExec
 
 
@@ -68,6 +70,47 @@ class TestSSHExecutor:
         assert (now - since_date).total_seconds() < 60 * 60 * 51
 
 
+def test_status_cmd(chatmail_config, capsys, request):
+    os.chdir(request.config.invocation_params.dir)
+    assert main(["status"]) == 0
+    status_out = capsys.readouterr()
+    print(status_out.out)
+
+    services = [
+        "acmetool-redirector",
+        "chatmail-metadata",
+        "doveauth",
+        "dovecot",
+        "echobot",
+        "fcgiwrap",
+        "filtermail-incoming",
+        "filtermail",
+        "lastlogin",
+        "nginx",
+        "opendkim",
+        "postfix@-",
+        "systemd-journald",
+        "turnserver",
+        "unbound",
+    ]
+    not_running = []
+    for service in services:
+        active = False
+        for line in status_out:
+            if service in line:
+                active = True
+                if not "loaded" in line:
+                    active = False
+                if not "active" in line:
+                    active = False
+                if not "running" in line:
+                    active = False
+                break
+        if not active:
+            not_running.append(service)
+    assert not_running == []
+
+
 def test_timezone_env(remote):
     for line in remote.iter_output("env"):
         print(line)

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -160,3 +160,22 @@ def test_hide_senders_ip_address(cmfactory):
     user2.direct_imap.select_folder("Inbox")
     msg = user2.direct_imap.get_all_messages()[0]
     assert public_ip not in msg.obj.as_string()
+
+
+def test_echobot(cmfactory, chatmail_config, lp, sshdomain):
+    ac = cmfactory.get_online_accounts(1)[0]
+
+    # establish contact with echobot
+    sshexec = SSHExec(sshdomain)
+    command = "cat /var/lib/echobot/invite-link.txt"
+    echo_invite_link = sshexec(call=rshell.shell, kwargs=dict(command=command))
+    chat = ac.qr_setup_contact(echo_invite_link)
+    ac._evtracker.wait_securejoin_joiner_progress(1000)
+
+    # send message and check it gets replied back
+    lp.sec("Send message to echobot")
+    text = "hi, I hope you text me back"
+    chat.send_text(text)
+    lp.sec("Wait for reply from echobot")
+    reply = ac._evtracker.wait_next_incoming_message()
+    assert reply.text == text

cmdeploy/src/cmdeploy/tests/online/test_3_status.py

@@ -1,49 +0,0 @@
-import os
-
-from cmdeploy.cmdeploy import main
-
-
-def test_status_cmd(chatmail_config, capsys, request):
-    os.chdir(request.config.invocation_params.dir)
-    assert main(["status"]) == 0
-    status_out = capsys.readouterr()
-    print(status_out.out)
-
-    assert len(status_out.out.splitlines()) > 5
-
-    """
-    don't test actual server state:
-
-    services = [
-        "acmetool-redirector",
-        "chatmail-metadata",
-        "doveauth",
-        "dovecot",
-        "fcgiwrap",
-        "filtermail-incoming",
-        "filtermail",
-        "lastlogin",
-        "nginx",
-        "opendkim",
-        "postfix@-",
-        "systemd-journald",
-        "turnserver",
-        "unbound",
-    ]
-    not_running = []
-    for service in services:
-        active = False
-        for line in status_out:
-            if service in line:
-                active = True
-                if not "loaded" in line:
-                    active = False
-                if not "active" in line:
-                    active = False
-                if not "running" in line:
-                    active = False
-                break
-        if not active:
-            not_running.append(service)
-    assert not_running == []
-    """

cmdeploy/src/cmdeploy/www.py

@@ -166,7 +166,7 @@ def main():
             build_webpages(src_path, build_dir, config)
             print(f"[{changenum}] regenerated web pages at: {index_path}")
             print(f"URL: file://{index_path.resolve()}\n\n")
-
+            
             time.sleep(debounce_time)  # simple debounce
 
 

doc/README.md

@@ -6,7 +6,7 @@ You can use the `make` command and `make html` to build web pages.
 
 You need a Python environment where the following install was excuted: 
 
-    pip install furo sphinx-autobuild
+    pip install sphinx-build furo sphinx-autobuild 
 
 To develop/change documentation, you can then do: 
 

doc/source/migrate.rst

@@ -26,7 +26,7 @@ this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
    or receive messages until the migration is completed.
 
 2. Now we want to copy ``/home/vmail``, ``/var/lib/acme``,
-   ``/etc/dkimkeys``, and ``/var/spool/postfix`` to
+   ``/etc/dkimkeys``, ``/run/echobot``, and ``/var/spool/postfix`` to
    the new site. Login to the old site while forwarding your SSH agent
    so you can copy directly from the old to the new site with your SSH
    key:
@@ -34,11 +34,11 @@ this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
    ::
 
        ssh -A root@13.37.13.37
-       tar c - /home/vmail/mail /var/lib/acme /etc/dkimkeys /var/spool/postfix | ssh root@13.12.23.42 "tar x -C /"
+       tar c - /home/vmail/mail /var/lib/acme /etc/dkimkeys /run/echobot /var/spool/postfix | ssh root@13.12.23.42 "tar x -C /"
 
-   This transfers all addresses, the TLS certificate,
+   This transfers all addresses, the TLS certificate, DKIM keys (so DKIM
-   and DKIM keys (so DKIM DNS record remains valid).
+   DNS record remains valid), and the echobot’s password so it continues
-   It also preserves the Postfix mail spool so any messages
+   to function. It also preserves the Postfix mail spool so any messages
    pending delivery will still be delivered.
 
 3. Install chatmail on the new machine:
@@ -58,6 +58,7 @@ this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
        chown root: -R /var/lib/acme
        chown opendkim: -R /etc/dkimkeys
        chown vmail: -R /home/vmail/mail
+       chown echobot: -R /run/echobot
 
 5. Now, update DNS entries.
 

doc/source/overview.rst

@@ -109,6 +109,10 @@ short overview of ``chatmaild`` services:
    is contacted by Dovecot when a user logs in and stores the date of
    the login.
 
+-  `echobot <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/echo.py>`_
+   is a small bot for test purposes. It simply echoes back messages from
+   users.
+
 -  `metrics <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/metrics.py>`_
    collects some metrics and displays them at
    ``https://example.org/metrics``.
@@ -272,8 +276,8 @@ by OpenDKIM screen policy script before validating the signatures. This
 corresponds to strict :rfc:`DMARC <7489>` alignment (``adkim=s``).
 If there is no valid DKIM signature on the incoming email, the
 sender receives a “5.7.1 No valid DKIM signature found” error.
-After validating the DKIM signature,
+After validating the DKIM signature, 
-the `final.lua` script strips all ``OpenDKIM:`` headers to reduce message size on disc.
+the `final.lua` script strips all ``OpenDKIM:`` headers to reduce message size on disc. 
 
 Note that chatmail relays
 

doc/source/related.rst

@@ -7,7 +7,7 @@ Active development takes place in the `chatmail/relay github repository <https:/
 You can check out the `'chatmail' tag in the support.delta.chat forum <https://support.delta.chat/tag/chatmail>`_
 and ask to get added to a non-public support chat for debugging issues.
 
-We know of three work-in-progress alternative implementation efforts:
+We know of two work-in-progress alternative implementation efforts:
 
 -  `Mox <https://github.com/mjl-/mox>`_: A Golang email server. `Work
    is in progress <https://github.com/mjl-/mox/issues/251>`_ to modify
@@ -18,10 +18,3 @@ We know of three work-in-progress alternative implementation efforts:
    plugin for the `Maddy email server <https://maddy.email/>`_ which
    aims to implement the chatmail relay features and configuration
    options.
-
--  `Chatmail Cookbook <https://github.com/feld/chatmail-cookbook>`_:
-   A Chef Cookbook implementing a relay server. The project follows the
-   official relay server software and configurations converted to a Chef
-   Cookbook with only minor differences. The cookbook uses DNS-01 for
-   certificate validation and additionally supports FreeBSD. It does not
-   require a Chef server to use.

www/src/index.md

@@ -23,3 +23,7 @@ you can also **scan this QR code** with Delta Chat:
 🐣 **Choose** your Avatar and Name
 
 💬 **Start** chatting with any Delta Chat contacts using [QR invite codes](https://delta.chat/en/help#howtoe2ee)
+
+{% if config.mail_domain != "nine.testrun.org" %}
+<div class="experimental">Note: this is only a temporary development chatmail service</div>
+{% endif %}