better preserve notification order, using a queue again

Otherwise packages may add user
without correct configuration such as groups
and the step adding user will be skipped.

.github/workflows/test-and-deploy.yaml

@@ -4,12 +4,17 @@ on:
   push:
     branches:
       - main
-      - staging-ci
+  pull_request:
+    paths-ignore:
+      - 'scripts/**'
 
 jobs:
   deploy:
     name: deploy on staging.testrun.org, and run tests
     runs-on: ubuntu-latest
+    concurrency:
+      group: staging-deploy
+      cancel-in-progress: true
     steps:
       - uses: actions/checkout@v3
 
@@ -19,44 +24,45 @@ jobs:
           echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
           chmod 600 ~/.ssh/id_ed25519
           ssh-keyscan staging.testrun.org > ~/.ssh/known_hosts
-     #    rsync -avz root@staging.testrun.org:/var/lib/acme . || true
-     #    rsync -avz root@staging.testrun.org:/var/lib/rspamd/dkim . || true
+          rsync -avz root@staging.testrun.org:/var/lib/acme . || true
+          rsync -avz root@staging.testrun.org:/etc/dkimkeys . || true
 
-     #- name: rebuild staging.testrun.org to have a clean VPS
-     #  run: |
-     #      curl -X POST \
-     #      -H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
-     #      -H "Content-Type: application/json" \
-     #      -d '{"image":"debian-12"}' \
-     #      "https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_SERVER_ID }}/actions/rebuild"
+      - name: rebuild staging.testrun.org to have a clean VPS
+        run: |
+            curl -X POST \
+            -H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
+            -H "Content-Type: application/json" \
+            -d '{"image":"debian-12"}' \
+            "https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_SERVER_ID }}/actions/rebuild"
 
       - run: scripts/initenv.sh
 
       - name: append venv/bin to PATH
         run: echo venv/bin >>$GITHUB_PATH
 
+      - name: upload TLS cert after rebuilding
+        run: |
+          echo " --- wait until staging.testrun.org VPS is rebuilt --- "
+          rm ~/.ssh/known_hosts
+          while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u ; do sleep 1 ; done
+          ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u
+          rsync -avz acme/ root@staging.testrun.org:/var/lib/acme || true
+          rsync -avz dkimkeys/ root@staging.testrun.org:/etc/dkimkeys || true
+          ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org chown root:root -R /var/lib/acme
+
       - name: run formatting checks 
         run: cmdeploy fmt -v 
 
       - name: run deploy-chatmail offline tests 
         run: pytest --pyargs cmdeploy 
 
-     #- name: upload TLS cert after rebuilding
-     #  run: |
-     #    echo " --- wait until staging.testrun.org VPS is rebuilt --- "
-     #    rm ~/.ssh/known_hosts
-     #    while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u ; do sleep 1 ; done
-     #    ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u
-     #    rsync -avz acme root@staging.testrun.org:/var/lib/ || true
-     #    rsync -avz dkim root@staging.testrun.org:/var/lib/rspamd/ || true
-
       - run: cmdeploy init staging.testrun.org
 
       - run: cmdeploy run
 
       - name: set DNS entries
         run: |
-          #ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org chown _rspamd:_rspamd -R /var/lib/rspamd/dkim
+          ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org chown opendkim:opendkim -R /etc/dkimkeys
           cmdeploy dns --zonefile staging-generated.zone
           cat staging-generated.zone >> .github/workflows/staging.testrun.org-default.zone
           cat .github/workflows/staging.testrun.org-default.zone

CHANGELOG.md

@@ -1,8 +1,20 @@
 # Changelog for chatmail deployment 
 
-## unreleased
+## untagged 
 
-### Changes since March 15th, 2024
+- better preserve notification order 
+  ([#263](https://github.com/deltachat/chatmail/pull/263))
+
+- re-enable running the CI in pull requests, but not concurrently 
+  ([#258](https://github.com/deltachat/chatmail/pull/258))
+
+
+## 1.1.0 - 2024-03-28
+
+### The changelog starts to record changes from March 15th, 2024 
+
+- Move systemd unit templates to cmdeploy package 
+  ([#255](https://github.com/deltachat/chatmail/pull/255))
 
 - Persist push tokens and support multiple device per address 
   ([#254](https://github.com/deltachat/chatmail/pull/254))

chatmaild/src/chatmaild/metadata.py

@@ -1,7 +1,8 @@
 import pwd
 
 from pathlib import Path
-from threading import Thread, Event
+from threading import Thread
+from queue import Queue
 from socketserver import (
     UnixStreamServer,
     StreamRequestHandler,
@@ -34,7 +35,7 @@ class Notifier:
         self.notification_dir = vmail_dir / "pending_notifications"
         if not self.notification_dir.exists():
             self.notification_dir.mkdir()
-        self.message_arrived_event = Event()
+        self.notification_queue = Queue()
 
     def get_metadata_dict(self, addr):
         return FileDict(self.vmail_dir / addr / "metadata.json")
@@ -60,31 +61,37 @@ class Notifier:
 
     def new_message_for_addr(self, addr):
         self.notification_dir.joinpath(addr).touch()
-        self.message_arrived_event.set()
+        self.notification_queue.put(addr)
 
     def thread_run_loop(self):
         requests_session = requests.Session()
+        # on startup deliver all persisted notifications from last process run
+        self.notification_queue.put(None)
         while 1:
-            self.message_arrived_event.wait()
-            self.message_arrived_event.clear()
             self.thread_run_one(requests_session)
 
     def thread_run_one(self, requests_session):
-        for addr_path in self.notification_dir.iterdir():
-            addr = addr_path.name
-            if "@" not in addr:
-                continue
-            for token in self.get_tokens(addr):
-                response = requests_session.post(
-                    "https://notifications.delta.chat/notify",
-                    data=token,
-                    timeout=60,
-                )
-                if response.status_code == 410:
-                    # 410 Gone status code
-                    # means the token is no longer valid.
-                    self.remove_token(addr, token)
-            addr_path.unlink()
+        addr = self.notification_queue.get()
+        if addr is None:
+            # startup, notify any "pending" notifications from last run
+            for addr_path in self.notification_dir.iterdir():
+                if "@" in addr_path.name:
+                    self.notify_tokens_for(requests_session, addr_path.name)
+        else:
+            self.notify_tokens_for(requests_session, addr)
+
+    def notify_tokens_for(self, requests_session, addr):
+        for token in self.get_tokens(addr):
+            response = requests_session.post(
+                "https://notifications.delta.chat/notify",
+                data=token,
+                timeout=60,
+            )
+            if response.status_code == 410:
+                # 410 Gone status code
+                # means the token is no longer valid.
+                self.remove_token(addr, token)
+        self.notification_dir.joinpath(addr).unlink(missing_ok=True)
 
 
 def handle_dovecot_protocol(rfile, wfile, notifier):

chatmaild/src/chatmaild/tests/test_metadata.py

@@ -84,7 +84,7 @@ def test_handle_dovecot_request_happy_path(notifier, testaddr):
     assert handle_dovecot_request(f"B{tx2}\t{testaddr}", transactions, notifier) is None
     msg = f"S{tx2}\tpriv/guid00/messagenew"
     assert handle_dovecot_request(msg, transactions, notifier) is None
-    assert notifier.message_arrived_event.is_set()
+    assert notifier.notification_queue.get() == testaddr
     assert handle_dovecot_request(f"C{tx2}", transactions, notifier) == "O\n"
     assert not transactions
     assert notifier.notification_dir.joinpath(testaddr).exists()
@@ -159,8 +159,8 @@ def test_handle_dovecot_protocol_messagenew(notifier):
     wfile = io.BytesIO()
     handle_dovecot_protocol(rfile, wfile, notifier)
     assert wfile.getvalue() == b"O\n"
-    assert notifier.message_arrived_event.is_set()
-    assert notifier.notification_dir.joinpath("user@example.org").exists()
+    addr = notifier.notification_queue.get()
+    assert notifier.notification_dir.joinpath(addr).exists()
 
 
 def test_notifier_thread_run(notifier, testaddr):

cmdeploy/src/cmdeploy/__init__.py

@@ -135,20 +135,6 @@ def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
     """Configures OpenDKIM"""
     need_restart = False
 
-    server.group(name="Create opendkim group", group="opendkim", system=True)
-    server.user(
-        name="Create opendkim user",
-        user="opendkim",
-        groups=["opendkim"],
-        system=True,
-    )
-    server.user(
-        name="Add postfix user to opendkim group for socket access",
-        user="postfix",
-        groups=["opendkim"],
-        system=True,
-    )
-
     main_config = files.template(
         src=importlib.resources.files(__package__).joinpath("opendkim/opendkim.conf"),
         dest="/etc/opendkim.conf",
@@ -476,9 +462,24 @@ def deploy_chatmail(config_path: Path) -> None:
 
     from .www import build_webpages
 
-    apt.update(name="apt update", cache_time=24 * 3600)
     server.group(name="Create vmail group", group="vmail", system=True)
     server.user(name="Create vmail user", user="vmail", group="vmail", system=True)
+    server.group(name="Create opendkim group", group="opendkim", system=True)
+    server.user(
+        name="Create opendkim user",
+        user="opendkim",
+        groups=["opendkim"],
+        system=True,
+    )
+    server.user(
+        name="Add postfix user to opendkim group for socket access",
+        user="postfix",
+        groups=["opendkim"],
+        system=True,
+    )
+
+    apt.update(name="apt update", cache_time=24 * 3600)
+
     apt.packages(
         name="Install rsync",
         packages=["rsync"],
@@ -565,14 +566,9 @@ def deploy_chatmail(config_path: Path) -> None:
         restarted=mta_sts_need_restart,
     )
 
-    systemd.service(
-        name="Start and enable Postfix",
-        service="postfix.service",
-        running=True,
-        enabled=True,
-        restarted=postfix_need_restart,
-    )
-
+    # Dovecot should be started before Postfix
+    # because it creates authentication socket
+    # required by Postfix.
     systemd.service(
         name="Start and enable Dovecot",
         service="dovecot.service",
@@ -581,6 +577,14 @@ def deploy_chatmail(config_path: Path) -> None:
         restarted=dovecot_need_restart,
     )
 
+    systemd.service(
+        name="Start and enable Postfix",
+        service="postfix.service",
+        running=True,
+        enabled=True,
+        restarted=postfix_need_restart,
+    )
+
     systemd.service(
         name="Start and enable nginx",
         service="nginx.service",