tests: make maildata work with python3.9

I installed pyenv and then installed Python 3.9:
$ pyenv install 3.9
$ eval "$(pyenv init -)"
$ pyenv shell 3.9

In a clean repository I ran
$ scripts/cmdeploy init
$ scripts/cmdeploy run
$ scripts/cmdeploy dns
$ scripts/cmdeploy fmt

With the changes made all these commands work.

scripts/cmdeploy test fails some tests
using maildata fixture at
  importlib.resources.files(__package__).joinpath("mail-data")
line but this is not critical.

.github/workflows/test-and-deploy.yaml

@@ -4,17 +4,12 @@ on:
   push:
     branches:
       - main
-  pull_request:
-    paths-ignore:
-      - 'scripts/**'
+      - staging-ci
 
 jobs:
   deploy:
     name: deploy on staging.testrun.org, and run tests
     runs-on: ubuntu-latest
-    concurrency:
-      group: staging-deploy
-      cancel-in-progress: true
     steps:
       - uses: actions/checkout@v3
 
@@ -24,45 +19,44 @@ jobs:
           echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
           chmod 600 ~/.ssh/id_ed25519
           ssh-keyscan staging.testrun.org > ~/.ssh/known_hosts
-          rsync -avz root@staging.testrun.org:/var/lib/acme . || true
-          rsync -avz root@staging.testrun.org:/etc/dkimkeys . || true
+     #    rsync -avz root@staging.testrun.org:/var/lib/acme . || true
+     #    rsync -avz root@staging.testrun.org:/var/lib/rspamd/dkim . || true
 
-      - name: rebuild staging.testrun.org to have a clean VPS
-        run: |
-            curl -X POST \
-            -H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
-            -H "Content-Type: application/json" \
-            -d '{"image":"debian-12"}' \
-            "https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_SERVER_ID }}/actions/rebuild"
+     #- name: rebuild staging.testrun.org to have a clean VPS
+     #  run: |
+     #      curl -X POST \
+     #      -H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
+     #      -H "Content-Type: application/json" \
+     #      -d '{"image":"debian-12"}' \
+     #      "https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_SERVER_ID }}/actions/rebuild"
 
       - run: scripts/initenv.sh
 
       - name: append venv/bin to PATH
         run: echo venv/bin >>$GITHUB_PATH
 
-      - name: upload TLS cert after rebuilding
-        run: |
-          echo " --- wait until staging.testrun.org VPS is rebuilt --- "
-          rm ~/.ssh/known_hosts
-          while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u ; do sleep 1 ; done
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u
-          rsync -avz acme/ root@staging.testrun.org:/var/lib/acme || true
-          rsync -avz dkimkeys/ root@staging.testrun.org:/etc/dkimkeys || true
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org chown root:root -R /var/lib/acme
-
       - name: run formatting checks 
         run: cmdeploy fmt -v 
 
       - name: run deploy-chatmail offline tests 
         run: pytest --pyargs cmdeploy 
 
+     #- name: upload TLS cert after rebuilding
+     #  run: |
+     #    echo " --- wait until staging.testrun.org VPS is rebuilt --- "
+     #    rm ~/.ssh/known_hosts
+     #    while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u ; do sleep 1 ; done
+     #    ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u
+     #    rsync -avz acme root@staging.testrun.org:/var/lib/ || true
+     #    rsync -avz dkim root@staging.testrun.org:/var/lib/rspamd/ || true
+
       - run: cmdeploy init staging.testrun.org
 
       - run: cmdeploy run
 
       - name: set DNS entries
         run: |
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
+          #ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org chown _rspamd:_rspamd -R /var/lib/rspamd/dkim
           cmdeploy dns --zonefile staging-generated.zone
           cat staging-generated.zone >> .github/workflows/staging.testrun.org-default.zone
           cat .github/workflows/staging.testrun.org-default.zone

CHANGELOG.md

@@ -1,43 +0,0 @@
-# Changelog for chatmail deployment 
-
-## untagged 
-
-- Run chatmail-metadata and doveauth as vmail
-  ([#261](https://github.com/deltachat/chatmail/pull/261))
-
-- Apply systemd restrictions to echobot
-  ([#259](https://github.com/deltachat/chatmail/pull/259))
-
-- re-enable running the CI in pull requests, but not concurrently 
-  ([#258](https://github.com/deltachat/chatmail/pull/258))
-
-
-## 1.1.0 - 2024-03-28
-
-### The changelog starts to record changes from March 15th, 2024 
-
-- Move systemd unit templates to cmdeploy package 
-  ([#255](https://github.com/deltachat/chatmail/pull/255))
-
-- Persist push tokens and support multiple device per address 
-  ([#254](https://github.com/deltachat/chatmail/pull/254))
-
-- Avoid warning for regular doveauth protocol's hello message. 
-  ([#250](https://github.com/deltachat/chatmail/pull/250))
-
-- Fix various tests to pass again with "cmdeploy test". 
-  ([#245](https://github.com/deltachat/chatmail/pull/245),
-  [#242](https://github.com/deltachat/chatmail/pull/242)
-
-- Ensure lets-encrypt certificates are reloaded after renewal 
-  ([#244]) https://github.com/deltachat/chatmail/pull/244
-
-- Persist tokens to avoid iOS users loosing push-notifications when the
-  chatmail metadata service is restarted (happens regularly during deploys)
-  ([#238](https://github.com/deltachat/chatmail/pull/239)
-
-- Fix failing sieve-script compile errors on incoming messages
-  ([#237](https://github.com/deltachat/chatmail/pull/239)
-
-- Fix quota reporting after expunging of old mails
-  ([#233](https://github.com/deltachat/chatmail/pull/239)

chatmaild/MANIFEST.in

@@ -1,3 +1,4 @@
+include src/chatmaild/*.f 
 include src/chatmaild/ini/*.ini.f
 include src/chatmaild/ini/*.ini
 include src/chatmaild/tests/mail-data/*

chatmaild/pyproject.toml

@@ -10,8 +10,6 @@ dependencies = [
   "iniconfig",
   "deltachat-rpc-server",
   "deltachat-rpc-client",
-  "filelock",
-  "requests",
 ]
 
 [tool.setuptools]
@@ -22,7 +20,6 @@ where = ['src']
 
 [project.scripts]
 doveauth = "chatmaild.doveauth:main"
-chatmail-metadata = "chatmaild.metadata:main"
 filtermail = "chatmaild.filtermail:main"
 echobot = "chatmaild.echo:main"
 chatmail-metrics = "chatmaild.metrics:main"

chatmaild/src/chatmaild/doveauth.py

@@ -17,10 +17,6 @@ from .config import read_config, Config
 NOCREATE_FILE = "/etc/chatmail-nocreate"
 
 
-class UnknownCommand(ValueError):
-    """dictproxy handler received an unkown command"""
-
-
 def encrypt_password(password: str):
     # https://doc.dovecot.org/configuration_manual/authentication/password_schemes/
     passhash = crypt.crypt(password, crypt.METHOD_SHA512)
@@ -62,18 +58,17 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
     return True
 
 
-def get_user_data(db, config: Config, user):
+def get_user_data(db, user):
     with db.read_connection() as conn:
         result = conn.get_user(user)
     if result:
-        result["home"] = f"/home/vmail/mail/{config.mail_domain}/{user}"
         result["uid"] = "vmail"
         result["gid"] = "vmail"
     return result
 
 
-def lookup_userdb(db, config: Config, user):
-    return get_user_data(db, config, user)
+def lookup_userdb(db, user):
+    return get_user_data(db, user)
 
 
 def lookup_passdb(db, config: Config, user, cleartext_password):
@@ -85,7 +80,6 @@ def lookup_passdb(db, config: Config, user, cleartext_password):
                 "UPDATE users SET last_login=? WHERE addr=?", (int(time.time()), user)
             )
 
-            userdata["home"] = f"/home/vmail/mail/{config.mail_domain}/{user}"
             userdata["uid"] = "vmail"
             userdata["gid"] = "vmail"
             return userdata
@@ -131,12 +125,8 @@ def split_and_unescape(s):
 
 
 def handle_dovecot_request(msg, db, config: Config):
-    # see https://doc.dovecot.org/3.0/developer_manual/design/dict_protocol/
     short_command = msg[0]
-    if short_command == "H":  # HELLO
-        # we don't do any checking on versions and just return
-        return
-    elif short_command == "L":  # LOOKUP
+    if short_command == "L":  # LOOKUP
         parts = msg[1:].split("\t")
 
         # Dovecot <2.3.17 has only one part,
@@ -152,7 +142,7 @@ def handle_dovecot_request(msg, db, config: Config):
             if type == "userdb":
                 user = args[0]
                 if user.endswith(f"@{config.mail_domain}"):
-                    res = lookup_userdb(db, config, user)
+                    res = lookup_userdb(db, user)
                 if res:
                     reply_command = "O"
                 else:
@@ -167,7 +157,7 @@ def handle_dovecot_request(msg, db, config: Config):
                     reply_command = "N"
         json_res = json.dumps(res) if res else ""
         return f"{reply_command}{json_res}\n"
-    raise UnknownCommand(msg)
+    return None
 
 
 def handle_dovecot_protocol(rfile, wfile, db: Database, config: Config):
@@ -175,14 +165,12 @@ def handle_dovecot_protocol(rfile, wfile, db: Database, config: Config):
         msg = rfile.readline().strip().decode()
         if not msg:
             break
-        try:
-            res = handle_dovecot_request(msg, db, config)
-        except UnknownCommand:
-            logging.warning("unknown command: %r", msg)
+        res = handle_dovecot_request(msg, db, config)
+        if res:
+            wfile.write(res.encode("ascii"))
+            wfile.flush()
         else:
-            if res:
-                wfile.write(res.encode("ascii"))
-                wfile.flush()
+            logging.warning("request had no answer: %r", msg)
 
 
 class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):

chatmaild/src/chatmaild/doveauth.service.f

@@ -5,7 +5,6 @@ Description=Chatmail dict authentication proxy for dovecot
 ExecStart={execpath} /run/dovecot/doveauth.socket vmail /home/vmail/passdb.sqlite {config_path}
 Restart=always
 RestartSec=30
-User=vmail
 
 [Install]
 WantedBy=multi-user.target

chatmaild/src/chatmaild/echo.py

@@ -18,14 +18,14 @@ hooks = events.HookCollection()
 @hooks.on(events.RawEvent)
 def log_event(event):
     if event.kind == EventType.INFO:
-        logging.info("%s", event.msg)
+        logging.info(event.msg)
     elif event.kind == EventType.WARNING:
-        logging.warning("%s", event.msg)
+        logging.warning(event.msg)
 
 
 @hooks.on(events.RawEvent(EventType.ERROR))
 def log_error(event):
-    logging.error("%s", event.msg)
+    logging.error(event.msg)
 
 
 @hooks.on(events.MemberListChanged)
@@ -48,9 +48,6 @@ def on_group_name_changed(event):
 @hooks.on(events.NewMessage(func=lambda e: not e.command))
 def echo(event):
     snapshot = event.message_snapshot
-    if snapshot.is_info:
-        # Ignore info messages
-        return
     if snapshot.text or snapshot.file:
         snapshot.chat.send_message(text=snapshot.text, file=snapshot.file)
 
@@ -62,7 +59,6 @@ def help_command(event):
 
 
 def main():
-    logging.basicConfig(level=logging.INFO)
     path = os.environ.get("PATH")
     venv_path = sys.argv[0].strip("echobot")
     os.environ["PATH"] = path + ":" + venv_path
@@ -84,4 +80,5 @@ def main():
 
 
 if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
     main()

chatmaild/src/chatmaild/echobot.service.f

@@ -0,0 +1,11 @@
+[Unit]
+Description=Chatmail echo bot for testing it works
+
+[Service]
+ExecStart={execpath} {config_path}
+Environment="PATH={remote_venv_dir}:$PATH"
+Restart=always
+RestartSec=30
+
+[Install]
+WantedBy=multi-user.target

chatmaild/src/chatmaild/filedict.py

@@ -1,35 +0,0 @@
-import os
-import logging
-import json
-import filelock
-from contextlib import contextmanager
-
-
-class FileDict:
-    """Concurrency-safe multi-reader/single-writer persistent dict."""
-
-    def __init__(self, path):
-        self.path = path
-        self.lock_path = path.with_name(path.name + ".lock")
-
-    @contextmanager
-    def modify(self):
-        # the OS will release the lock if the process dies,
-        # and the contextmanager will otherwise guarantee release
-        with filelock.FileLock(self.lock_path):
-            data = self.read()
-            yield data
-            write_path = self.path.with_name(self.path.name + ".tmp")
-            with write_path.open("w") as f:
-                json.dump(data, f)
-            os.rename(write_path, self.path)
-
-    def read(self):
-        try:
-            with self.path.open("r") as f:
-                return json.load(f)
-        except FileNotFoundError:
-            return {}
-        except Exception:
-            logging.warning("corrupt serialization state at: %r", self.path)
-            return {}

chatmaild/src/chatmaild/filtermail.service.f

@@ -1,5 +1,5 @@
 [Unit]
-Description=Chatmail Postfix before queue filter
+Description=Chatmail Postfix BeforeQeue filter 
 
 [Service]
 ExecStart={execpath} {config_path}

chatmaild/src/chatmaild/metadata.py

@@ -1,199 +0,0 @@
-import pwd
-
-from pathlib import Path
-from threading import Thread, Event
-from socketserver import (
-    UnixStreamServer,
-    StreamRequestHandler,
-    ThreadingMixIn,
-)
-import sys
-import logging
-import os
-import requests
-
-from .filedict import FileDict
-
-
-DICTPROXY_HELLO_CHAR = "H"
-DICTPROXY_LOOKUP_CHAR = "L"
-DICTPROXY_ITERATE_CHAR = "I"
-DICTPROXY_BEGIN_TRANSACTION_CHAR = "B"
-DICTPROXY_SET_CHAR = "S"
-DICTPROXY_COMMIT_TRANSACTION_CHAR = "C"
-DICTPROXY_TRANSACTION_CHARS = "BSC"
-
-# each SETMETADATA on this key appends to a list of unique device tokens
-# which only ever get removed if the upstream indicates the token is invalid
-METADATA_TOKEN_KEY = "devicetoken"
-
-
-class Notifier:
-    def __init__(self, vmail_dir):
-        self.vmail_dir = vmail_dir
-        self.notification_dir = vmail_dir / "pending_notifications"
-        if not self.notification_dir.exists():
-            self.notification_dir.mkdir()
-        self.message_arrived_event = Event()
-
-    def get_metadata_dict(self, addr):
-        return FileDict(self.vmail_dir / addr / "metadata.json")
-
-    def add_token(self, addr, token):
-        with self.get_metadata_dict(addr).modify() as data:
-            tokens = data.get(METADATA_TOKEN_KEY)
-            if tokens is None:
-                data[METADATA_TOKEN_KEY] = [token]
-            elif token not in tokens:
-                tokens.append(token)
-
-    def remove_token(self, addr, token):
-        with self.get_metadata_dict(addr).modify() as data:
-            tokens = data.get(METADATA_TOKEN_KEY, [])
-            try:
-                tokens.remove(token)
-            except ValueError:
-                pass
-
-    def get_tokens(self, addr):
-        return self.get_metadata_dict(addr).read().get(METADATA_TOKEN_KEY, [])
-
-    def new_message_for_addr(self, addr):
-        self.notification_dir.joinpath(addr).touch()
-        self.message_arrived_event.set()
-
-    def thread_run_loop(self):
-        requests_session = requests.Session()
-        while 1:
-            self.message_arrived_event.wait()
-            self.message_arrived_event.clear()
-            self.thread_run_one(requests_session)
-
-    def thread_run_one(self, requests_session):
-        for addr_path in self.notification_dir.iterdir():
-            addr = addr_path.name
-            if "@" not in addr:
-                continue
-            for token in self.get_tokens(addr):
-                response = requests_session.post(
-                    "https://notifications.delta.chat/notify",
-                    data=token,
-                    timeout=60,
-                )
-                if response.status_code == 410:
-                    # 410 Gone status code
-                    # means the token is no longer valid.
-                    self.remove_token(addr, token)
-            addr_path.unlink()
-
-
-def handle_dovecot_protocol(rfile, wfile, notifier):
-    transactions = {}
-    while True:
-        msg = rfile.readline().strip().decode()
-        if not msg:
-            break
-
-        res = handle_dovecot_request(msg, transactions, notifier)
-        if res:
-            wfile.write(res.encode("ascii"))
-            wfile.flush()
-
-
-def handle_dovecot_request(msg, transactions, notifier):
-    # see https://doc.dovecot.org/3.0/developer_manual/design/dict_protocol/
-    short_command = msg[0]
-    parts = msg[1:].split("\t")
-    if short_command == DICTPROXY_LOOKUP_CHAR:
-        # Lpriv/43f5f508a7ea0366dff30200c15250e3/devicetoken\tlkj123poi@c2.testrun.org
-        keyparts = parts[0].split("/")
-        if keyparts[0] == "priv":
-            keyname = keyparts[2]
-            addr = parts[1]
-            if keyname == METADATA_TOKEN_KEY:
-                res = " ".join(notifier.get_tokens(addr))
-                return f"O{res}\n"
-        logging.warning("lookup ignored: %r", msg)
-        return "N\n"
-    elif short_command == DICTPROXY_ITERATE_CHAR:
-        # Empty line means ITER_FINISHED.
-        # If we don't return empty line Dovecot will timeout.
-        return "\n"
-    elif short_command == DICTPROXY_HELLO_CHAR:
-        return  # no version checking
-
-    if short_command not in (DICTPROXY_TRANSACTION_CHARS):
-        logging.warning("unknown dictproxy request: %r", msg)
-        return
-
-    transaction_id = parts[0]
-
-    if short_command == DICTPROXY_BEGIN_TRANSACTION_CHAR:
-        addr = parts[1]
-        transactions[transaction_id] = dict(addr=addr, res="O\n")
-    elif short_command == DICTPROXY_COMMIT_TRANSACTION_CHAR:
-        # each set devicetoken operation persists directly
-        # and does not wait until a "commit" comes
-        # because our dovecot config does not involve
-        # multiple set-operations in a single commit
-        return transactions.pop(transaction_id)["res"]
-    elif short_command == DICTPROXY_SET_CHAR:
-        # For documentation on key structure see
-        # https://github.com/dovecot/core/blob/main/src/lib-storage/mailbox-attribute.h
-
-        keyname = parts[1].split("/")
-        value = parts[2] if len(parts) > 2 else ""
-        addr = transactions[transaction_id]["addr"]
-        if keyname[0] == "priv" and keyname[2] == METADATA_TOKEN_KEY:
-            notifier.add_token(addr, value)
-        elif keyname[0] == "priv" and keyname[2] == "messagenew":
-            notifier.new_message_for_addr(addr)
-        else:
-            # Transaction failed.
-            transactions[transaction_id]["res"] = "F\n"
-
-
-class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
-    request_queue_size = 100
-
-
-def main():
-    socket, username, vmail_dir = sys.argv[1:]
-    passwd_entry = pwd.getpwnam(username)
-
-    vmail_dir = Path(vmail_dir)
-
-    if not vmail_dir.exists():
-        logging.error("vmail dir does not exist: %r", vmail_dir)
-        return 1
-
-    notifier = Notifier(vmail_dir)
-
-    class Handler(StreamRequestHandler):
-        def handle(self):
-            try:
-                handle_dovecot_protocol(self.rfile, self.wfile, notifier)
-            except Exception:
-                logging.exception("Exception in the dovecot dictproxy handler")
-                raise
-
-    try:
-        os.unlink(socket)
-    except FileNotFoundError:
-        pass
-
-    # start notifier thread for signalling new messages to
-    # Delta Chat notification server
-
-    t = Thread(target=notifier.thread_run_loop)
-    t.setDaemon(True)
-    t.start()
-    # let notifier thread run once for any pending notifications from last run
-    notifier.message_arrived_event.set()
-
-    with ThreadedUnixStreamServer(socket, Handler) as server:
-        os.chown(socket, uid=passwd_entry.pw_uid, gid=passwd_entry.pw_gid)
-        try:
-            server.serve_forever()
-        except KeyboardInterrupt:
-            pass

chatmaild/src/chatmaild/tests/test_doveauth.py

@@ -17,7 +17,7 @@ from chatmaild.database import DBError
 
 def test_basic(db, example_config):
     lookup_passdb(db, example_config, "asdf12345@chat.example.org", "q9mr3faue")
-    data = get_user_data(db, example_config, "asdf12345@chat.example.org")
+    data = get_user_data(db, "asdf12345@chat.example.org")
     assert data
     data2 = lookup_passdb(
         db, example_config, "asdf12345@chat.example.org", "q9mr3jewvadsfaue"
@@ -43,7 +43,7 @@ def test_nocreate_file(db, monkeypatch, tmpdir, example_config):
     lookup_passdb(
         db, example_config, "newuser12@chat.example.org", "zequ0Aimuchoodaechik"
     )
-    assert not get_user_data(db, example_config, "newuser12@chat.example.org")
+    assert not get_user_data(db, "newuser12@chat.example.org")
 
 
 def test_db_version(db):
@@ -75,14 +75,6 @@ def test_handle_dovecot_request(db, example_config):
     assert userdata["password"].startswith("{SHA512-CRYPT}")
 
 
-def test_handle_dovecot_protocol_hello_is_skipped(db, example_config, caplog):
-    rfile = io.BytesIO(b"H3\t2\t0\t\tauth\n")
-    wfile = io.BytesIO()
-    handle_dovecot_protocol(rfile, wfile, db, example_config)
-    assert wfile.getvalue() == b""
-    assert not caplog.messages
-
-
 def test_handle_dovecot_protocol(db, example_config):
     rfile = io.BytesIO(
         b"H3\t2\t0\t\tauth\nLshared/userdb/foobar@chat.example.org\tfoobar@chat.example.org\n"

chatmaild/src/chatmaild/tests/test_filedict.py

@@ -1,19 +0,0 @@
-from chatmaild.filedict import FileDict
-
-
-def test_basic(tmp_path):
-    fdict = FileDict(tmp_path.joinpath("metadata"))
-    assert fdict.read() == {}
-    with fdict.modify() as d:
-        d["devicetoken"] = [1, 2, 3]
-        d["456"] = 4.2
-    new = fdict.read()
-    assert new["devicetoken"] == [1, 2, 3]
-    assert new["456"] == 4.2
-
-
-def test_bad_marshal_file(tmp_path, caplog):
-    fdict1 = FileDict(tmp_path.joinpath("metadata"))
-    fdict1.path.write_bytes(b"l12k3l12k3l")
-    assert fdict1.read() == {}
-    assert "corrupt" in caplog.records[0].msg

chatmaild/src/chatmaild/tests/test_metadata.py

@@ -1,230 +0,0 @@
-import io
-import pytest
-
-from chatmaild.metadata import (
-    handle_dovecot_request,
-    handle_dovecot_protocol,
-    Notifier,
-)
-
-
-@pytest.fixture
-def notifier(tmp_path):
-    vmail_dir = tmp_path.joinpath("vmaildir")
-    vmail_dir.mkdir()
-    return Notifier(vmail_dir)
-
-
-@pytest.fixture
-def testaddr():
-    return "user.name@example.org"
-
-
-@pytest.fixture
-def testaddr2():
-    return "user2@example.org"
-
-
-def test_notifier_persistence(tmp_path, testaddr, testaddr2):
-    notifier1 = Notifier(tmp_path)
-    notifier2 = Notifier(tmp_path)
-    assert not notifier1.get_tokens(testaddr)
-    assert not notifier2.get_tokens(testaddr)
-
-    notifier1.add_token(testaddr, "01234")
-    notifier1.add_token(testaddr2, "456")
-    assert notifier2.get_tokens(testaddr) == ["01234"]
-    assert notifier2.get_tokens(testaddr2) == ["456"]
-    notifier2.remove_token(testaddr, "01234")
-    assert not notifier1.get_tokens(testaddr)
-    assert notifier1.get_tokens(testaddr2) == ["456"]
-
-
-def test_remove_nonexisting(tmp_path, testaddr):
-    notifier1 = Notifier(tmp_path)
-    notifier1.add_token(testaddr, "123")
-    notifier1.remove_token(testaddr, "1l23k1l2k3")
-    assert notifier1.get_tokens(testaddr) == ["123"]
-
-
-def test_notifier_delete_without_set(notifier, testaddr):
-    notifier.remove_token(testaddr, "123")
-    assert not notifier.get_tokens(testaddr)
-
-
-def test_handle_dovecot_request_lookup_fails(notifier, testaddr):
-    res = handle_dovecot_request(f"Lpriv/123/chatmail\t{testaddr}", {}, notifier)
-    assert res == "N\n"
-
-
-def test_handle_dovecot_request_happy_path(notifier, testaddr):
-    transactions = {}
-
-    # set device token in a transaction
-    tx = "1111"
-    msg = f"B{tx}\t{testaddr}"
-    res = handle_dovecot_request(msg, transactions, notifier)
-    assert not res and not notifier.get_tokens(testaddr)
-    assert transactions == {tx: dict(addr=testaddr, res="O\n")}
-
-    msg = f"S{tx}\tpriv/guid00/devicetoken\t01234"
-    res = handle_dovecot_request(msg, transactions, notifier)
-    assert not res
-    assert len(transactions) == 1
-    assert notifier.get_tokens(testaddr) == ["01234"]
-
-    msg = f"C{tx}"
-    res = handle_dovecot_request(msg, transactions, notifier)
-    assert res == "O\n"
-    assert len(transactions) == 0
-    assert notifier.get_tokens(testaddr) == ["01234"]
-
-    # trigger notification for incoming message
-    tx2 = "2222"
-    assert handle_dovecot_request(f"B{tx2}\t{testaddr}", transactions, notifier) is None
-    msg = f"S{tx2}\tpriv/guid00/messagenew"
-    assert handle_dovecot_request(msg, transactions, notifier) is None
-    assert notifier.message_arrived_event.is_set()
-    assert handle_dovecot_request(f"C{tx2}", transactions, notifier) == "O\n"
-    assert not transactions
-    assert notifier.notification_dir.joinpath(testaddr).exists()
-
-
-def test_handle_dovecot_protocol_set_devicetoken(notifier):
-    rfile = io.BytesIO(
-        b"\n".join(
-            [
-                b"HELLO",
-                b"Btx00\tuser@example.org",
-                b"Stx00\tpriv/guid00/devicetoken\t01234",
-                b"Ctx00",
-            ]
-        )
-    )
-    wfile = io.BytesIO()
-    handle_dovecot_protocol(rfile, wfile, notifier)
-    assert wfile.getvalue() == b"O\n"
-    assert notifier.get_tokens("user@example.org") == ["01234"]
-
-
-def test_handle_dovecot_protocol_set_get_devicetoken(notifier):
-    rfile = io.BytesIO(
-        b"\n".join(
-            [
-                b"HELLO",
-                b"Btx00\tuser@example.org",
-                b"Stx00\tpriv/guid00/devicetoken\t01234",
-                b"Ctx00",
-            ]
-        )
-    )
-    wfile = io.BytesIO()
-    handle_dovecot_protocol(rfile, wfile, notifier)
-    assert notifier.get_tokens("user@example.org") == ["01234"]
-    assert wfile.getvalue() == b"O\n"
-
-    rfile = io.BytesIO(
-        b"\n".join([b"HELLO", b"Lpriv/0123/devicetoken\tuser@example.org"])
-    )
-    wfile = io.BytesIO()
-    handle_dovecot_protocol(rfile, wfile, notifier)
-    assert wfile.getvalue() == b"O01234\n"
-
-
-def test_handle_dovecot_protocol_iterate(notifier):
-    rfile = io.BytesIO(
-        b"\n".join(
-            [
-                b"H",
-                b"I9\t0\tpriv/5cbe730f146fea6535be0d003dd4fc98/\tci-2dzsrs@nine.testrun.org",
-            ]
-        )
-    )
-    wfile = io.BytesIO()
-    handle_dovecot_protocol(rfile, wfile, notifier)
-    assert wfile.getvalue() == b"\n"
-
-
-def test_handle_dovecot_protocol_messagenew(notifier):
-    rfile = io.BytesIO(
-        b"\n".join(
-            [
-                b"HELLO",
-                b"Btx01\tuser@example.org",
-                b"Stx01\tpriv/guid00/messagenew",
-                b"Ctx01",
-            ]
-        )
-    )
-    wfile = io.BytesIO()
-    handle_dovecot_protocol(rfile, wfile, notifier)
-    assert wfile.getvalue() == b"O\n"
-    assert notifier.message_arrived_event.is_set()
-    assert notifier.notification_dir.joinpath("user@example.org").exists()
-
-
-def test_notifier_thread_run(notifier, testaddr):
-    requests = []
-
-    class ReqMock:
-        def post(self, url, data, timeout):
-            requests.append((url, data, timeout))
-
-            class Result:
-                status_code = 200
-
-            return Result()
-
-    notifier.add_token(testaddr, "01234")
-    notifier.new_message_for_addr(testaddr)
-    notifier.thread_run_one(ReqMock())
-    url, data, timeout = requests[0]
-    assert data == "01234"
-    assert notifier.get_tokens(testaddr) == ["01234"]
-
-
-def test_multi_device_notifier(notifier, testaddr):
-    requests = []
-
-    class ReqMock:
-        def post(self, url, data, timeout):
-            requests.append((url, data, timeout))
-
-            class Result:
-                status_code = 200
-
-            return Result()
-
-    notifier.add_token(testaddr, "01234")
-    notifier.add_token(testaddr, "56789")
-    notifier.new_message_for_addr(testaddr)
-    notifier.thread_run_one(ReqMock())
-    url, data, timeout = requests[0]
-    assert data == "01234"
-    url, data, timeout = requests[1]
-    assert data == "56789"
-    assert notifier.get_tokens(testaddr) == ["01234", "56789"]
-
-
-def test_notifier_thread_run_gone_removes_token(notifier, testaddr):
-    requests = []
-
-    class ReqMock:
-        def post(self, url, data, timeout):
-            requests.append((url, data, timeout))
-
-            class Result:
-                status_code = 410 if data == "01234" else 200
-
-            return Result()
-
-    notifier.add_token(testaddr, "01234")
-    notifier.new_message_for_addr(testaddr)
-    assert notifier.get_tokens(testaddr) == ["01234"]
-    notifier.add_token(testaddr, "45678")
-    notifier.thread_run_one(ReqMock())
-    url, data, timeout = requests[0]
-    assert data == "01234"
-    url, data, timeout = requests[1]
-    assert data == "45678"
-    assert notifier.get_tokens(testaddr) == ["45678"]

cmdeploy/pyproject.toml

@@ -19,7 +19,6 @@ dependencies = [
   "black",
   "pytest",
   "pytest-xdist", 
-  "imap_tools",
 ]
 
 [project.scripts]

cmdeploy/src/cmdeploy/__init__.py

@@ -102,17 +102,13 @@ def _install_remote_venv_with_chatmaild(config) -> None:
         "doveauth",
         "filtermail",
         "echobot",
-        "chatmail-metadata",
     ):
         params = dict(
             execpath=f"{remote_venv_dir}/bin/{fn}",
             config_path=remote_chatmail_inipath,
             remote_venv_dir=remote_venv_dir,
-            mail_domain=config.mail_domain,
-        )
-        source_path = importlib.resources.files(__package__).joinpath(
-            "service", f"{fn}.service.f"
         )
+        source_path = importlib.resources.files("chatmaild").joinpath(f"{fn}.service.f")
         content = source_path.read_text().format(**params).encode()
 
         files.put(
@@ -135,6 +131,20 @@ def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
     """Configures OpenDKIM"""
     need_restart = False
 
+    server.group(name="Create opendkim group", group="opendkim", system=True)
+    server.user(
+        name="Create opendkim user",
+        user="opendkim",
+        groups=["opendkim"],
+        system=True,
+    )
+    server.user(
+        name="Add postfix user to opendkim group for socket access",
+        user="postfix",
+        groups=["opendkim"],
+        system=True,
+    )
+
     main_config = files.template(
         src=importlib.resources.files(__package__).joinpath("opendkim/opendkim.conf"),
         dest="/etc/opendkim.conf",
@@ -327,30 +337,6 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
         mode="644",
     )
     need_restart |= auth_config.changed
-    lua_push_notification_script = files.put(
-        src=importlib.resources.files(__package__).joinpath(
-            "dovecot/push_notification.lua"
-        ),
-        dest="/etc/dovecot/push_notification.lua",
-        user="root",
-        group="root",
-        mode="644",
-    )
-    need_restart |= lua_push_notification_script.changed
-
-    sieve_script = files.put(
-        src=importlib.resources.files(__package__).joinpath("dovecot/default.sieve"),
-        dest="/etc/dovecot/default.sieve",
-        user="root",
-        group="root",
-        mode="644",
-    )
-    need_restart |= sieve_script.changed
-    if sieve_script.changed:
-        server.shell(
-            name="compile sieve script",
-            commands=["/usr/bin/sievec /etc/dovecot/default.sieve"],
-        )
 
     files.template(
         src=importlib.resources.files(__package__).joinpath("dovecot/expunge.cron.j2"),
@@ -440,11 +426,8 @@ def check_config(config):
     mail_domain = config.mail_domain
     if mail_domain != "testrun.org" and not mail_domain.endswith(".testrun.org"):
         blocked_words = "merlinux schmieder testrun.org".split()
-        for key in config.__dict__:
-            value = config.__dict__[key]
-            if key.startswith("privacy") and any(
-                x in str(value) for x in blocked_words
-            ):
+        for value in config.__dict__.values():
+            if any(x in str(value) for x in blocked_words):
                 raise ValueError(
                     f"please set your own privacy contacts/addresses in {config._inipath}"
                 )
@@ -462,33 +445,9 @@ def deploy_chatmail(config_path: Path) -> None:
 
     from .www import build_webpages
 
+    apt.update(name="apt update", cache_time=24 * 3600)
     server.group(name="Create vmail group", group="vmail", system=True)
     server.user(name="Create vmail user", user="vmail", group="vmail", system=True)
-    server.group(name="Create opendkim group", group="opendkim", system=True)
-    server.user(
-        name="Create opendkim user",
-        user="opendkim",
-        groups=["opendkim"],
-        system=True,
-    )
-    server.user(
-        name="Add postfix user to opendkim group for socket access",
-        user="postfix",
-        groups=["opendkim"],
-        system=True,
-    )
-
-    server.shell(
-        name="Fix file owner in /home/vmail",
-        commands=["test -d /home/vmail && chown -R vmail:vmail /home/vmail"],
-    )
-
-    apt.update(name="apt update", cache_time=24 * 3600)
-
-    apt.packages(
-        name="Install rsync",
-        packages=["rsync"],
-    )
 
     # Run local DNS resolver `unbound`.
     # `resolvconf` takes care of setting up /etc/resolv.conf
@@ -524,7 +483,7 @@ def deploy_chatmail(config_path: Path) -> None:
 
     apt.packages(
         name="Install Dovecot",
-        packages=["dovecot-imapd", "dovecot-lmtpd", "dovecot-sieve"],
+        packages=["dovecot-imapd", "dovecot-lmtpd"],
     )
 
     apt.packages(
@@ -571,17 +530,6 @@ def deploy_chatmail(config_path: Path) -> None:
         restarted=mta_sts_need_restart,
     )
 
-    # Dovecot should be started before Postfix
-    # because it creates authentication socket
-    # required by Postfix.
-    systemd.service(
-        name="Start and enable Dovecot",
-        service="dovecot.service",
-        running=True,
-        enabled=True,
-        restarted=dovecot_need_restart,
-    )
-
     systemd.service(
         name="Start and enable Postfix",
         service="postfix.service",
@@ -590,6 +538,14 @@ def deploy_chatmail(config_path: Path) -> None:
         restarted=postfix_need_restart,
     )
 
+    systemd.service(
+        name="Start and enable Dovecot",
+        service="dovecot.service",
+        running=True,
+        enabled=True,
+        restarted=dovecot_need_restart,
+    )
+
     systemd.service(
         name="Start and enable nginx",
         service="nginx.service",

cmdeploy/src/cmdeploy/acmetool/acmetool.cron

@@ -1,4 +1,4 @@
 SHELL=/bin/sh
 PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
 MAILTO=root
-20 16 * * * root /usr/bin/acmetool --batch reconcile && systemctl reload dovecot && systemctl reload postfix
+20 16 * * * root /usr/bin/acmetool --batch reconcile

cmdeploy/src/cmdeploy/chatmail.zone.f

@@ -6,7 +6,7 @@ _submissions._tcp.{chatmail_domain}. SRV 0 1 465 {chatmail_domain}.
 _imap._tcp.{chatmail_domain}.        SRV 0 1 143 {chatmail_domain}.
 _imaps._tcp.{chatmail_domain}.       SRV 0 1 993 {chatmail_domain}.
 {chatmail_domain}.                   CAA 128 issue "letsencrypt.org;accounturi={acme_account_url}"
-{chatmail_domain}.                   TXT "v=spf1 a:{chatmail_domain} ~all"
+{chatmail_domain}.                   TXT "v=spf1 a:{chatmail_domain} -all"
 _dmarc.{chatmail_domain}.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
 _mta-sts.{chatmail_domain}.          TXT "v=STSv1; id={sts_id}"
 mta-sts.{chatmail_domain}.           CNAME {chatmail_domain}.

cmdeploy/src/cmdeploy/dns.py

@@ -5,6 +5,8 @@ import importlib
 import subprocess
 import datetime
 
+from typing import Optional
+
 
 class DNS:
     def __init__(self, out, mail_domain):
@@ -34,11 +36,12 @@ class DNS:
         cmd = "ip a | grep inet6 | grep 'scope global' | sed -e 's#/64 scope global##' | sed -e 's#inet6##'"
         return self.shell(cmd).strip()
 
-    def get(self, typ: str, domain: str) -> str:
-        """Get a DNS entry or empty string if there is none."""
+    def get(self, typ: str, domain: str) -> Optional[str]:
+        """Get a DNS entry"""
         dig_result = self.shell(f"dig -r -q {domain} -t {typ} +short")
         line = dig_result.partition("\n")[0]
-        return line
+        if line:
+            return line
 
     def check_ptr_record(self, ip: str, mail_domain) -> bool:
         """Check the PTR record for an IPv4 or IPv6 address."""
@@ -53,25 +56,22 @@ def show_dns(args, out) -> int:
     ssh = f"ssh root@{mail_domain}"
     dns = DNS(out, mail_domain)
 
+    def read_dkim_entries(entry):
+        lines = []
+        for line in entry.split("\n"):
+            if line.startswith(";") or not line.strip():
+                continue
+            line = line.replace("\t", " ")
+            lines.append(line)
+        return "\n".join(lines)
+
     print("Checking your DKIM keys and DNS entries...")
     try:
         acme_account_url = out.shell_output(f"{ssh} -- acmetool account-url")
     except subprocess.CalledProcessError:
         print("Please run `cmdeploy run` first.")
         return 1
-
-    dkim_selector = "opendkim"
-    dkim_pubkey = out.shell_output(
-        ssh + f" -- openssl rsa -in /etc/dkimkeys/{dkim_selector}.private"
-        " -pubout 2>/dev/null | awk '/-/{next}{printf(\"%s\",$0)}'"
-    )
-    dkim_entry_value = f"v=DKIM1;k=rsa;p={dkim_pubkey};s=email;t=s"
-    dkim_entry_str = ""
-    while len(dkim_entry_value) >= 255:
-        dkim_entry_str += '"' + dkim_entry_value[:255] + '" '
-        dkim_entry_value = dkim_entry_value[255:]
-    dkim_entry_str += '"' + dkim_entry_value + '"'
-    dkim_entry = f"{dkim_selector}._domainkey.{mail_domain}. TXT {dkim_entry_str}"
+    dkim_entry = read_dkim_entries(out.shell_output(f"{ssh} -- opendkim-genzone -F"))
 
     ipv6 = dns.get_ipv6()
     reverse_ipv6 = dns.check_ptr_record(ipv6, mail_domain)
@@ -99,6 +99,7 @@ def show_dns(args, out) -> int:
             return 0
         except TypeError:
             pass
+        started_dkim_parsing = False
         for line in zonefile.splitlines():
             line = line.format(
                 acme_account_url=acme_account_url,
@@ -125,23 +126,28 @@ def show_dns(args, out) -> int:
                 current = dns.get("SRV", domain[:-1])
                 if current != f"{prio} {weight} {port} {value}":
                     to_print.append(line)
-            if " TXT " in line:
+            if "  TXT " in line:
                 domain, value = line.split(" TXT ")
                 current = dns.get("TXT", domain.strip()[:-1])
                 if domain.startswith("_mta-sts."):
                     if current:
                         if current.split("id=")[0] == value.split("id=")[0]:
                             continue
-
-                # TXT records longer than 255 bytes
-                # are split into multiple <character-string>s.
-                # This typically happens with DKIM record
-                # which contains long RSA key.
-                #
-                # Removing `" "` before comparison
-                # to get back a single string.
-                if current.replace('" "', "") != value.replace('" "', ""):
+                if current != value:
                     to_print.append(line)
+            if "IN TXT ( " in line:
+                started_dkim_parsing = True
+                dkim_lines = [line]
+            if started_dkim_parsing and line.startswith('"'):
+                dkim_lines.append(" " + line)
+        domain, data = "\n".join(dkim_lines).split(" IN TXT ")
+        current = dns.get("TXT", domain.strip()[:-1])
+        if current:
+            current = "( %s )" % (current.replace('" "', '"\n "'))
+            if current.replace(";", "\\;") != data:
+                to_print.append(dkim_entry)
+        else:
+            to_print.append(dkim_entry)
 
     exit_code = 0
     if to_print:

cmdeploy/src/cmdeploy/dovecot/default.sieve

@@ -1,7 +0,0 @@
-require ["imap4flags"];
-
-# flag the message so it doesn't cause a push notification
-
-if header :is ["Auto-Submitted"] ["auto-replied", "auto-generated"] {
-	addflag "$Auto";
-}

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -13,12 +13,6 @@ auth_cache_size = 100M
 mail_debug = yes
 {% endif %}
 
-# Prevent warnings similar to:
-#   config: Warning: service auth { client_limit=1000 } is lower than required under max. load (10200). Counted for protocol services with service_count != 1: service lmtp { process_limit=100 } + service imap-urlauth-login { process_limit=100 } + service imap-login { process_limit=10000 }
-#   config: Warning: service anvil { client_limit=1000 } is lower than required under max. load (10103). Counted with: service imap-urlauth-login { process_limit=100 } + service imap-login { process_limit=10000 } + service auth { process_limit=1 }
-#   master: Warning: service(stats): client_limit (1000) reached, client connections are being dropped
-default_client_limit = 20000
-
 mail_server_admin = mailto:root@{{ config.mail_domain }}
 mail_server_comment = Chatmail server
 
@@ -27,7 +21,7 @@ mail_plugins = quota
 # these are the capabilities Delta Chat cares about actually 
 # so let's keep the network overhead per login small
 # https://github.com/deltachat/deltachat-core-rust/blob/master/src/imap/capabilities.rs
-imap_capability = IMAP4rev1 IDLE MOVE QUOTA CONDSTORE NOTIFY METADATA XDELTAPUSH
+imap_capability = IMAP4rev1 IDLE MOVE QUOTA CONDSTORE NOTIFY METADATA
 
 
 # Authentication for system users.
@@ -77,9 +71,6 @@ mail_privileged_group = vmail
 ## Mail processes
 ##
 
-# Pass all IMAP METADATA requests to the server implementing Dovecot's dict protocol.
-mail_attribute_dict = proxy:/run/dovecot/metadata.socket:metadata
-
 # Enable IMAP COMPRESS (RFC 4978).
 # <https://datatracker.ietf.org/doc/html/rfc4978.html>
 protocol imap {
@@ -88,21 +79,7 @@ protocol imap {
 }
 
 protocol lmtp {
-  # quota plugin documentation:
-  # <https://doc.dovecot.org/configuration_manual/quota_plugin/>
-  #
-  # notify plugin is a dependency of push_notification plugin:
-  # <https://doc.dovecot.org/settings/plugin/notify-plugin/>
-  #
-  # push_notification plugin documentation:
-  # <https://doc.dovecot.org/configuration_manual/push_notification/>
-  #
-  # mail_lua and push_notification_lua are needed for Lua push notification handler.
-  # <https://doc.dovecot.org/configuration_manual/push_notification/#configuration>
-  #
-  # Sieve to mark messages that should not be notified as \Seen
-  # <https://doc.dovecot.org/configuration_manual/sieve/configuration/>
-  mail_plugins = $mail_plugins quota mail_lua notify push_notification push_notification_lua sieve
+  mail_plugins = $mail_plugins quota
 }
 
 plugin {
@@ -118,15 +95,7 @@ plugin {
   # quota_over_flag_value = TRUE
 }
 
-# push_notification configuration
-plugin {
-  # <https://doc.dovecot.org/configuration_manual/push_notification/#lua-lua>
-  push_notification_driver = lua:file=/etc/dovecot/push_notification.lua
-}
 
-plugin {
-  sieve_default = file:/etc/dovecot/default.sieve
-}
 
 service lmtp {
   user=vmail

cmdeploy/src/cmdeploy/dovecot/expunge.cron.j2

@@ -8,4 +8,3 @@
 # or only temporary (but then they shouldn't be around after {{ config.delete_mails_after }} days anyway).
 2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-3 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -name 'maildirsize' -type f -delete

cmdeploy/src/cmdeploy/dovecot/push_notification.lua

@@ -1,32 +0,0 @@
-function dovecot_lua_notify_begin_txn(user)
-  return user
-end
-
-function contains(v, needle)
-  for _, keyword in ipairs(v) do
-    if keyword == needle then
-      return true
-    end
-  end
-  return false
-end
-
-function dovecot_lua_notify_event_message_new(user, event)
-  local mbox = user:mailbox(event.mailbox)
-  mbox:sync()
-
-  if user.username ~= event.from_address then
-    -- Incoming message
-    if not contains(event.keywords, "$Auto") then
-      -- Not an Auto-Submitted message, notifying.
-
-      -- Notify METADATA server about new message.
-      mbox:metadata_set("/private/messagenew", "")
-    end
-  end
-
-  mbox:free()
-end
-
-function dovecot_lua_notify_end_txn(ctx, success)
-end

cmdeploy/src/cmdeploy/service/chatmail-metadata.service.f

@@ -1,47 +0,0 @@
-[Unit]
-Description=Chatmail dict proxy for IMAP METADATA
-
-[Service]
-ExecStart={execpath} /run/dovecot/metadata.socket vmail /home/vmail/mail/{mail_domain}
-Restart=always
-RestartSec=30
-User=vmail
-
-# Make `systemd-analyze security` happy.
-CapabilityBoundingSet=
-LockPersonality=true
-MemoryDenyWriteExecute=true
-NoNewPrivileges=true
-PrivateDevices=true
-PrivateMounts=true
-PrivateTmp=true
-PrivateUsers=true
-ProtectClock=true
-ProtectControlGroups=true
-ProtectHostname=true
-ProtectKernelLogs=true
-ProtectKernelModules=true
-ProtectKernelTunables=true
-ProtectProc=noaccess
-ProtectSystem=strict
-RemoveIPC=true
-RestrictAddressFamilies=AF_UNIX
-RestrictNamespaces=true
-RestrictRealtime=true
-RestrictSUIDSGID=true
-SystemCallArchitectures=native
-SystemCallFilter=~@clock
-SystemCallFilter=~@cpu-emulation
-SystemCallFilter=~@debug
-SystemCallFilter=~@module
-SystemCallFilter=~@mount
-SystemCallFilter=~@obsolete
-SystemCallFilter=~@privileged
-SystemCallFilter=~@raw-io
-SystemCallFilter=~@reboot
-SystemCallFilter=~@resources
-SystemCallFilter=~@swap
-UMask=0077
-
-[Install]
-WantedBy=multi-user.target

cmdeploy/src/cmdeploy/service/echobot.service.f

@@ -1,50 +0,0 @@
-[Unit]
-Description=Chatmail echo bot for testing it works
-
-[Service]
-ExecStart={execpath} {config_path}
-Environment="PATH={remote_venv_dir}:$PATH"
-Restart=always
-RestartSec=30
-
-# Apply security restrictions suggested by
-#   systemd-analyze security echobot.service
-CapabilityBoundingSet=
-LockPersonality=true
-MemoryDenyWriteExecute=true
-NoNewPrivileges=true
-PrivateDevices=true
-PrivateMounts=true
-PrivateTmp=true
-PrivateUsers=true
-ProtectClock=true
-ProtectControlGroups=true
-ProtectHostname=true
-ProtectKernelLogs=true
-ProtectKernelModules=true
-ProtectKernelTunables=true
-ProtectProc=noaccess
-
-# Should be "strict", but we currently write /accounts folder in a protected path
-ProtectSystem=full
-
-RemoveIPC=true
-RestrictAddressFamilies=AF_INET AF_INET6
-RestrictNamespaces=true
-RestrictRealtime=true
-RestrictSUIDSGID=true
-SystemCallArchitectures=native
-SystemCallFilter=~@clock
-SystemCallFilter=~@cpu-emulation
-SystemCallFilter=~@debug
-SystemCallFilter=~@module
-SystemCallFilter=~@mount
-SystemCallFilter=~@obsolete
-SystemCallFilter=~@raw-io
-SystemCallFilter=~@reboot
-SystemCallFilter=~@resources
-SystemCallFilter=~@swap
-UMask=0077
-
-[Install]
-WantedBy=multi-user.target

cmdeploy/src/cmdeploy/tests/online/test_0_login.py

@@ -1,7 +1,6 @@
 import pytest
 import threading
 import queue
-import socket
 
 from chatmaild.config import read_config
 from cmdeploy.cmdeploy import main
@@ -79,24 +78,3 @@ def test_concurrent_logins_same_account(
 
     for _ in conns:
         assert login_results.get()
-
-
-def test_no_vrfy(chatmail_config):
-    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    sock.connect((chatmail_config.mail_domain, 25))
-    banner = sock.recv(1024)
-    print(banner)
-    sock.send(b"VRFY wrongaddress@%s\r\n" % (chatmail_config.mail_domain.encode(),))
-    result = sock.recv(1024)
-    print(result)
-    sock.send(b"VRFY echo@%s\r\n" % (chatmail_config.mail_domain.encode(),))
-    result2 = sock.recv(1024)
-    print(result2)
-    assert result[0:10] == result2[0:10]
-    sock.send(b"VRFY wrongaddress\r\n")
-    result = sock.recv(1024)
-    print(result)
-    sock.send(b"VRFY echo\r\n")
-    result2 = sock.recv(1024)
-    print(result2)
-    assert result[0:10] == result2[0:10] == b"252 2.0.0 "

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -5,46 +5,6 @@ import random
 import pytest
 import requests
 import ipaddress
-import imap_tools
-
-
-@pytest.fixture
-def imap_mailbox(cmfactory):
-    (ac1,) = cmfactory.get_online_accounts(1)
-    user = ac1.get_config("addr")
-    password = ac1.get_config("mail_pw")
-    mailbox = imap_tools.MailBox(user.split("@")[1])
-    mailbox.login(user, password)
-    return mailbox
-
-
-class TestMetadataTokens:
-    "Tests that use Metadata extension for storing tokens"
-
-    def test_set_get_metadata(self, imap_mailbox):
-        "set and get metadata token for an account"
-        client = imap_mailbox.client
-        client.send(b'a01 SETMETADATA INBOX (/private/devicetoken "1111" )\n')
-        res = client.readline()
-        assert b"OK Setmetadata completed" in res
-
-        client.send(b"a02 GETMETADATA INBOX /private/devicetoken\n")
-        res = client.readline()
-        assert res[:1] == b"*"
-        res = client.readline().strip().rstrip(b")")
-        assert res == b"1111"
-        assert b"Getmetadata completed" in client.readline()
-
-        client.send(b'a01 SETMETADATA INBOX (/private/devicetoken "2222" )\n')
-        res = client.readline()
-        assert b"OK Setmetadata completed" in res
-
-        client.send(b"a02 GETMETADATA INBOX /private/devicetoken\n")
-        res = client.readline()
-        assert res[:1] == b"*"
-        res = client.readline().strip().rstrip(b")")
-        assert res == b"1111 2222"
-        assert b"Getmetadata completed" in client.readline()
 
 
 class TestEndToEndDeltaChat:
@@ -103,7 +63,7 @@ class TestEndToEndDeltaChat:
 
         addr = ac2.get_config("addr").lower()
         saved_ok = 0
-        for line in remote.iter_output("journalctl -n0 -f -u dovecot"):
+        for line in remote.iter_output("journalctl -f -u dovecot"):
             if addr not in line:
                 # print(line)
                 continue
@@ -115,10 +75,7 @@ class TestEndToEndDeltaChat:
                         )
                     lp.indent("good, message sending failed because quota was exceeded")
                     return
-            if (
-                "stored mail into mailbox 'inbox'" in line
-                or "saved mail to inbox" in line
-            ):
+            if "saved mail to inbox" in line:
                 saved_ok += 1
                 print(f"{saved_ok}: {line}")
                 if saved_ok >= num_to_send:
@@ -155,7 +112,7 @@ class TestEndToEndDeltaChat:
         lp.sec("ac1 sends a message and ac2 marks it as seen")
         chat = ac1.create_chat(ac2)
         msg = chat.send_text("hi")
-        m = ac2._evtracker.wait_next_incoming_message()
+        m = ac2.wait_next_incoming_message()
         m.mark_seen()
         # we can only indirectly wait for mark-seen to cause an smtp-error
         lp.sec("try to wait for markseen to complete and check error states")
@@ -175,7 +132,7 @@ def test_hide_senders_ip_address(cmfactory):
     chat = cmfactory.get_accepted_chat(user1, user2)
 
     chat.send_text("testing submission header cleanup")
-    user2._evtracker.wait_next_incoming_message()
+    user2.wait_next_incoming_message()
     user2.direct_imap.select_folder("Inbox")
     msg = user2.direct_imap.get_all_messages()[0]
     assert public_ip not in msg.obj.as_string()
@@ -189,5 +146,5 @@ def test_echobot(cmfactory, chatmail_config, lp):
     text = "hi, I hope you text me back"
     chat.send_text(text)
     lp.sec("Wait for reply from echobot")
-    reply = ac._evtracker.wait_next_incoming_message()
+    reply = ac.wait_next_incoming_message()
     assert reply.text == text