Add execnet dependency

.github/workflows/docs-preview.yaml

@@ -11,9 +11,6 @@ jobs:
   scripts:
     name: build 
     runs-on: ubuntu-latest
-    environment:
-      name: 'staging.chatmail.at/doc/relay/'
-      url: https://staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}
     steps:
       - uses: actions/checkout@v4
 
@@ -47,6 +44,36 @@ jobs:
           chmod 600 "$HOME/.ssh/key"
           rsync -rILvh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/doc/build/ "${{ secrets.USERNAME }}@chatmail.at:/var/www/html/staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}/"
 
+      - name: "Post links to details"
+        id: details
+        if: steps.prepare.outputs.uploadtoserver
+        run: |
+          # URLs for API connection and uploads
+          export GITHUB_API_URL="https://api.github.com/repos/chatmail/relay/statuses/${{ github.event.after }}"
+          export PREVIEW_LINK="https://staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}/"
+          export STATUS_DATA="{\"state\": \"success\", \
+                               \"description\": \"Preview the changed documentation here:\", \
+                               \"context\": \"Documentation Preview\", \
+                               \"target_url\": \"${PREVIEW_LINK}\"}"
+          curl -X POST --header "Accept: application/vnd.github+json" --header "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" --url "$GITHUB_API_URL" --header "content-type: application/json" --data "$STATUS_DATA"
+
+          #check if comment already exists, if not post it
+          export GITHUB_API_URL="https://api.github.com/repos/chatmail/relay/issues/${{ steps.prepare.outputs.prid }}/comments"
+          export RESPONSE=$(curl -L --header "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" --url "$GITHUB_API_URL" --header "content-type: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28")
+          echo $RESPONSE > response
+          grep -v '"Check out the page preview at https://staging.chatmail.at/doc/relay' response && echo "comment=true" >> $GITHUB_OUTPUT || true
+      - name: "Post link to comments"
+        if: steps.details.outputs.comment
+        uses: actions/github-script@v7
+        with:
+          script: |
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: "Check out the page preview at https://staging.chatmail.at/doc/relay/${{ steps.prepare.outputs.prid }}/"
+            })
+
       - name: check links 
         working-directory: doc
         run: sphinx-build --builder linkcheck source build

.github/workflows/docs.yaml

@@ -14,9 +14,6 @@ jobs:
   scripts:
     name: build 
     runs-on: ubuntu-latest
-    environment: 
-      name: 'chatmail.at/doc/relay/'
-      url: https://chatmail.at/doc/relay/
     steps:
       - uses: actions/checkout@v4
 

.github/workflows/test-and-deploy-ipv4only.yaml

@@ -16,9 +16,6 @@ jobs:
     name: deploy on staging-ipv4.testrun.org, and run tests
     runs-on: ubuntu-latest
     timeout-minutes: 30
-    environment:
-      name: staging-ipv4.testrun.org
-      url: https://staging-ipv4.testrun.org/
     concurrency:
       group: ci-ipv4-${{ github.workflow }}-${{ github.ref }}
       cancel-in-progress: ${{ !contains(github.ref, '$GITHUB_REF') }}

.github/workflows/test-and-deploy.yaml

@@ -16,9 +16,6 @@ jobs:
     name: deploy on staging2.testrun.org, and run tests
     runs-on: ubuntu-latest
     timeout-minutes: 30
-    environment:
-      name: staging2.testrun.org
-      url: https://staging2.testrun.org/
     concurrency:
       group: ci-${{ github.workflow }}-${{ github.ref }}
       cancel-in-progress: ${{ !contains(github.ref, '$GITHUB_REF') }}
@@ -73,9 +70,6 @@ jobs:
           rsync -avz dkimkeys-restore/dkimkeys root@staging2.testrun.org:/etc/ || true
           ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org chown root:root -R /var/lib/acme || true
 
-      - name: add hpk42 key to staging server
-        run: ssh root@staging2.testrun.org 'curl -s https://github.com/hpk42.keys >> .ssh/authorized_keys'
-
       - name: run deploy-chatmail offline tests 
         run: pytest --pyargs cmdeploy 
 

CHANGELOG.md

@@ -2,9 +2,6 @@
 
 ## untagged
 
-- Remove echobot from relays 
-  ([#753](https://github.com/chatmail/relay/pull/753))
-
 - Add robots.txt to exclude all web crawlers
   ([#732](https://github.com/chatmail/relay/pull/732))
 

chatmaild/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "chatmaild"
-version = "0.3"
+version = "0.2"
 dependencies = [
   "aiosmtpd",
   "iniconfig",
@@ -25,6 +25,7 @@ where = ['src']
 doveauth = "chatmaild.doveauth:main"
 chatmail-metadata = "chatmaild.metadata:main"
 filtermail = "chatmaild.filtermail:main"
+echobot = "chatmaild.echo:main"
 chatmail-metrics = "chatmaild.metrics:main"
 chatmail-expire = "chatmaild.expire:main"
 chatmail-fsreport = "chatmaild.fsreport:main"

chatmaild/src/chatmaild/config.py

@@ -4,6 +4,8 @@ import iniconfig
 
 from chatmaild.user import User
 
+echobot_password_path = Path("/run/echobot/password")
+
 
 def read_config(inipath):
     assert Path(inipath).exists(), inipath
@@ -70,7 +72,10 @@ class Config:
             raise ValueError(f"invalid address {addr!r}")
 
         maildir = self.mailboxes_dir.joinpath(addr)
-        password_path = maildir.joinpath("password")
+        if addr.startswith("echo@"):
+            password_path = echobot_password_path
+        else:
+            password_path = maildir.joinpath("password")
 
         return User(maildir, addr, password_path, uid="vmail", gid="vmail")
 

chatmaild/src/chatmaild/doveauth.py

@@ -40,6 +40,10 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
         return False
     localpart, domain = parts
 
+    if localpart == "echo":
+        # echobot account should not be created in the database
+        return False
+
     if (
         len(localpart) > config.username_max_length
         or len(localpart) < config.username_min_length

chatmaild/src/chatmaild/echo.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+"""Advanced echo bot example.
+
+it will echo back any message that has non-empty text and also supports the /help command.
+"""
+
+import logging
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+from deltachat_rpc_client import Bot, DeltaChat, EventType, Rpc, events
+
+from chatmaild.config import echobot_password_path, read_config
+from chatmaild.doveauth import encrypt_password
+from chatmaild.newemail import create_newemail_dict
+
+hooks = events.HookCollection()
+
+
+@hooks.on(events.RawEvent)
+def log_event(event):
+    if event.kind == EventType.INFO:
+        logging.info(event.msg)
+    elif event.kind == EventType.WARNING:
+        logging.warning(event.msg)
+
+
+@hooks.on(events.RawEvent(EventType.ERROR))
+def log_error(event):
+    logging.error("%s", event.msg)
+
+
+@hooks.on(events.MemberListChanged)
+def on_memberlist_changed(event):
+    logging.info(
+        "member %s was %s", event.member, "added" if event.member_added else "removed"
+    )
+
+
+@hooks.on(events.GroupImageChanged)
+def on_group_image_changed(event):
+    logging.info("group image %s", "deleted" if event.image_deleted else "changed")
+
+
+@hooks.on(events.GroupNameChanged)
+def on_group_name_changed(event):
+    logging.info(f"group name changed, old name: {event.old_name}")
+
+
+@hooks.on(events.NewMessage(func=lambda e: not e.command))
+def echo(event):
+    snapshot = event.message_snapshot
+    if snapshot.is_info:
+        # Ignore info messages
+        return
+    if snapshot.text or snapshot.file:
+        snapshot.chat.send_message(text=snapshot.text, file=snapshot.file)
+
+
+@hooks.on(events.NewMessage(command="/help"))
+def help_command(event):
+    snapshot = event.message_snapshot
+    snapshot.chat.send_text("Send me any message and I will echo it back")
+
+
+def main():
+    logging.basicConfig(level=logging.INFO)
+    path = os.environ.get("PATH")
+    venv_path = sys.argv[0].strip("echobot")
+    os.environ["PATH"] = path + ":" + venv_path
+    with Rpc() as rpc:
+        deltachat = DeltaChat(rpc)
+        system_info = deltachat.get_system_info()
+        logging.info(f"Running deltachat core {system_info.deltachat_core_version}")
+
+        accounts = deltachat.get_all_accounts()
+        account = accounts[0] if accounts else deltachat.add_account()
+
+        bot = Bot(account, hooks)
+
+        config = read_config(sys.argv[1])
+        addr = "echo@" + config.mail_domain
+
+        # Create password file
+        if bot.is_configured():
+            password = bot.account.get_config("mail_pw")
+        else:
+            password = create_newemail_dict(config)["password"]
+
+        echobot_password_path.write_text(encrypt_password(password))
+        # Give the user which doveauth runs as access to the password file.
+        subprocess.check_call(
+            ["/usr/bin/setfacl", "-m", "user:vmail:r", echobot_password_path],
+        )
+
+        if not bot.is_configured():
+            bot.configure(addr, password)
+
+        # write invite link to working directory
+        invitelink = bot.account.get_qr_code()
+        Path("invite-link.txt").write_text(invitelink)
+
+        bot.run_forever()
+
+
+if __name__ == "__main__":
+    main()

chatmaild/src/chatmaild/newemail.py

@@ -20,7 +20,7 @@ def create_newemail_dict(config: Config):
         secrets.choice(ALPHANUMERIC_PUNCT)
         for _ in range(config.password_min_length + 3)
     )
-    return dict(email=f"{user}@bloc7.icu", password=f"{password}")
+    return dict(email=f"{user}@{config.mail_domain}", password=f"{password}")
 
 
 def print_new_account():

chatmaild/src/chatmaild/tests/test_lastlogin.py

@@ -36,3 +36,29 @@ def test_handle_dovecot_request_last_login(testaddr, example_config):
     res = dictproxy.handle_dovecot_request(msg, dictproxy_transactions)
     assert res == "O\n"
     assert len(dictproxy_transactions) == 0
+
+
+def test_handle_dovecot_request_last_login_echobot(example_config):
+    dictproxy = LastLoginDictProxy(config=example_config)
+
+    authproxy = AuthDictProxy(config=example_config)
+    testaddr = f"echo@{example_config.mail_domain}"
+    authproxy.lookup_passdb(testaddr, "ignore")
+    user = dictproxy.config.get_user(testaddr)
+
+    transactions = {}
+
+    # set last-login info for user
+    tx = "1111"
+    msg = f"B{tx}\t{testaddr}"
+    res = dictproxy.handle_dovecot_request(msg, transactions)
+    assert not res
+    assert transactions == {tx: dict(addr=testaddr, res="O\n")}
+
+    timestamp = int(time.time())
+    msg = f"S{tx}\tshared/last-login/{testaddr}\t{timestamp}"
+    res = dictproxy.handle_dovecot_request(msg, transactions)
+    assert not res
+    assert len(transactions) == 1
+    read_timestamp = user.get_last_login_timestamp()
+    assert read_timestamp is None

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -109,6 +109,15 @@ def run_cmd(args, out):
     try:
         retcode = out.check_call(cmd, env=env)
         if retcode == 0:
+            if not args.disable_mail:
+                print("\nYou can try out the relay by talking to this echo bot: ")
+                sshexec = SSHExec(args.config.mail_domain, verbose=args.verbose)
+                print(
+                    sshexec(
+                        call=remote.rshell.shell,
+                        kwargs=dict(command="cat /var/lib/echobot/invite-link.txt"),
+                    )
+                )
             out.green("Deploy completed, call `cmdeploy dns` next.")
         elif not remote_data["acme_account_url"]:
             out.red("Deploy completed but letsencrypt not configured")

cmdeploy/src/cmdeploy/deployers.py

@@ -270,14 +270,6 @@ class LegacyRemoveDeployer(Deployer):
             path="/var/log/journal/",
             present=False,
         )
-        # remove echobot if it is still running
-        if host.get_fact(SystemdEnabled).get("echobot.service"):
-            systemd.service(
-                name="Disable echobot.service",
-                service="echobot.service",
-                running=False,
-                enabled=False,
-            )
 
 
 def check_config(config):
@@ -412,6 +404,30 @@ class JournaldDeployer(Deployer):
         self.need_restart = False
 
 
+class EchobotDeployer(Deployer):
+    #
+    # This deployer depends on the dovecot and postfix deployers because
+    # it needs to base its decision of whether to restart the service on
+    # whether those two services were restarted.
+    #
+    def __init__(self, mail_domain):
+        self.mail_domain = mail_domain
+        self.units = ["echobot"]
+
+    def install(self):
+        apt.packages(
+            # required for setfacl for echobot
+            name="Install acl",
+            packages="acl",
+        )
+
+    def configure(self):
+        configure_remote_units(self.mail_domain, self.units)
+
+    def activate(self):
+        activate_remote_units(self.units)
+
+
 class ChatmailVenvDeployer(Deployer):
     def __init__(self, config):
         self.config = config
@@ -574,6 +590,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None:
         PostfixDeployer(config, disable_mail),
         FcgiwrapDeployer(),
         NginxDeployer(config),
+        EchobotDeployer(mail_domain),
         MtailDeployer(config.mtail_address),
         GithashDeployer(),
     ]

cmdeploy/src/cmdeploy/service/echobot.service.f

@@ -0,0 +1,67 @@
+[Unit]
+Description=Chatmail echo bot for testing it works
+
+[Service]
+ExecStart={execpath} {config_path}
+Environment="PATH={remote_venv_dir}:$PATH"
+Restart=always
+RestartSec=30
+
+User=echobot
+Group=echobot
+
+# Create /var/lib/echobot
+StateDirectory=echobot
+
+# Create /run/echobot
+#
+# echobot stores /run/echobot/password
+# with a password there, which doveauth then reads.
+RuntimeDirectory=echobot
+
+WorkingDirectory=/var/lib/echobot
+
+# Apply security restrictions suggested by
+#   systemd-analyze security echobot.service
+CapabilityBoundingSet=
+LockPersonality=true
+MemoryDenyWriteExecute=true
+NoNewPrivileges=true
+PrivateDevices=true
+PrivateMounts=true
+PrivateTmp=true
+
+# We need to know about doveauth user to give it access to /run/echobot/password
+PrivateUsers=false
+
+ProtectClock=true
+ProtectControlGroups=true
+ProtectHostname=true
+ProtectKernelLogs=true
+ProtectKernelModules=true
+ProtectKernelTunables=true
+ProtectProc=noaccess
+
+# Should be "strict", but we currently write /accounts folder in a protected path
+ProtectSystem=full
+
+RemoveIPC=true
+RestrictAddressFamilies=AF_INET AF_INET6
+RestrictNamespaces=true
+RestrictRealtime=true
+RestrictSUIDSGID=true
+SystemCallArchitectures=native
+SystemCallFilter=~@clock
+SystemCallFilter=~@cpu-emulation
+SystemCallFilter=~@debug
+SystemCallFilter=~@module
+SystemCallFilter=~@mount
+SystemCallFilter=~@obsolete
+SystemCallFilter=~@raw-io
+SystemCallFilter=~@reboot
+SystemCallFilter=~@resources
+SystemCallFilter=~@swap
+UMask=0077
+
+[Install]
+WantedBy=multi-user.target

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -81,6 +81,7 @@ def test_status_cmd(chatmail_config, capsys, request):
         "chatmail-metadata",
         "doveauth",
         "dovecot",
+        "echobot",
         "fcgiwrap",
         "filtermail-incoming",
         "filtermail",

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -160,3 +160,22 @@ def test_hide_senders_ip_address(cmfactory):
     user2.direct_imap.select_folder("Inbox")
     msg = user2.direct_imap.get_all_messages()[0]
     assert public_ip not in msg.obj.as_string()
+
+
+def test_echobot(cmfactory, chatmail_config, lp, sshdomain):
+    ac = cmfactory.get_online_accounts(1)[0]
+
+    # establish contact with echobot
+    sshexec = SSHExec(sshdomain)
+    command = "cat /var/lib/echobot/invite-link.txt"
+    echo_invite_link = sshexec(call=rshell.shell, kwargs=dict(command=command))
+    chat = ac.qr_setup_contact(echo_invite_link)
+    ac._evtracker.wait_securejoin_joiner_progress(1000)
+
+    # send message and check it gets replied back
+    lp.sec("Send message to echobot")
+    text = "hi, I hope you text me back"
+    chat.send_text(text)
+    lp.sec("Wait for reply from echobot")
+    reply = ac._evtracker.wait_next_incoming_message()
+    assert reply.text == text

doc/source/migrate.rst

@@ -26,7 +26,7 @@ this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
    or receive messages until the migration is completed.
 
 2. Now we want to copy ``/home/vmail``, ``/var/lib/acme``,
-   ``/etc/dkimkeys``, and ``/var/spool/postfix`` to
+   ``/etc/dkimkeys``, ``/run/echobot``, and ``/var/spool/postfix`` to
    the new site. Login to the old site while forwarding your SSH agent
    so you can copy directly from the old to the new site with your SSH
    key:
@@ -34,11 +34,11 @@ this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
    ::
 
        ssh -A root@13.37.13.37
-       tar c - /home/vmail/mail /var/lib/acme /etc/dkimkeys /var/spool/postfix | ssh root@13.12.23.42 "tar x -C /"
+       tar c - /home/vmail/mail /var/lib/acme /etc/dkimkeys /run/echobot /var/spool/postfix | ssh root@13.12.23.42 "tar x -C /"
 
-   This transfers all addresses, the TLS certificate,
+   This transfers all addresses, the TLS certificate, DKIM keys (so DKIM
-   and DKIM keys (so DKIM DNS record remains valid).
+   DNS record remains valid), and the echobot’s password so it continues
-   It also preserves the Postfix mail spool so any messages
+   to function. It also preserves the Postfix mail spool so any messages
    pending delivery will still be delivered.
 
 3. Install chatmail on the new machine:
@@ -58,6 +58,7 @@ this case, just run ``ssh-keygen -R "mail.example.org"`` as recommended.
        chown root: -R /var/lib/acme
        chown opendkim: -R /etc/dkimkeys
        chown vmail: -R /home/vmail/mail
+       chown echobot: -R /run/echobot
 
 5. Now, update DNS entries.
 

doc/source/overview.rst

@@ -109,6 +109,10 @@ short overview of ``chatmaild`` services:
    is contacted by Dovecot when a user logs in and stores the date of
    the login.
 
+-  `echobot <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/echo.py>`_
+   is a small bot for test purposes. It simply echoes back messages from
+   users.
+
 -  `metrics <https://github.com/chatmail/relay/blob/main/chatmaild/src/chatmaild/metrics.py>`_
    collects some metrics and displays them at
    ``https://example.org/metrics``.