refinements and fixes

.github/workflows/test-and-deploy.yaml

@@ -7,9 +7,6 @@ on:
   pull_request:
     paths-ignore:
       - 'scripts/**'
-      - '**/README.md'
-      - 'CHANGELOG.md'
-      - 'LICENSE'
 
 jobs:
   deploy:
@@ -59,7 +56,7 @@ jobs:
           # restore acme & dkim state to staging.testrun.org
           rsync -avz acme-restore/acme/ root@staging.testrun.org:/var/lib/acme || true
           rsync -avz dkimkeys-restore/dkimkeys/ root@staging.testrun.org:/etc/dkimkeys || true
-          ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org chown root:root -R /var/lib/acme || true
+          ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org chown root:root -R /var/lib/acme
 
       - name: run formatting checks 
         run: cmdeploy fmt -v 

CHANGELOG.md

@@ -1,35 +1,6 @@
 # Changelog for chatmail deployment 
 
-## untagged
-
-- run metrics generation with systemd-timer instead of cron
-  ([#304](https://github.com/deltachat/chatmail/pull/304))
-
-- change default for delete_mails_after from 40 to 20 days
-  ([#300]https://github.com/deltachat/chatmail/pull/300)
-
-- fix writing of multiple obs repositories in `/etc/apt/sources.list`
-  ([#272](https://github.com/deltachat/chatmail/issues/272))
-
-- metadata: add support for `/shared/vendor/deltachat/irohrelay`
-  ([#284](https://github.com/deltachat/chatmail/pull/284))
-
-- Emit "XCHATMAIL" capability from IMAP server 
-  ([#278](https://github.com/deltachat/chatmail/pull/278))
-
-- Move echobot `into /var/lib/echobot`
-  ([#281](https://github.com/deltachat/chatmail/pull/281))
-
-- Accept Let's Encrypt's new Terms of Services
-  ([#275](https://github.com/deltachat/chatmail/pull/276))
-
-- Reload Dovecot and Postfix when TLS certificate updates
-  ([#271](https://github.com/deltachat/chatmail/pull/271))
-
-- Use forked version of dovecot without hardcoded delays
-  ([#270](https://github.com/deltachat/chatmail/pull/270))
-
-## 1.2.0 - 2024-04-04
+## untagged 
 
 - Install dig on the server to resolve DNS records
   ([#267](https://github.com/deltachat/chatmail/pull/267))

README.md

@@ -15,8 +15,6 @@ after which the initially specified password is required for using them.
 
 ## Deploying your own chatmail server 
 
-To deploy chatmail on your own server, you must have set-up ssh authentication and need to use an ed25519 key, due to an [upstream bug in paramiko](https://github.com/paramiko/paramiko/issues/2191). You also need to add your private key to the local ssh-agent, because you can't type in your password during deployment.
-
 We use `chat.example.org` as the chatmail domain in the following steps. 
 Please substitute it with your own domain. 
 
@@ -161,27 +159,4 @@ While this file is present, account creation will be blocked.
 Delta Chat apps will, however, discover all ports and configurations
 automatically by reading the [autoconfig XML file](https://www.ietf.org/archive/id/draft-bucksch-autoconfig-00.html) from the chatmail service.
 
-## Email authentication
 
-chatmail servers rely on [DKIM](https://www.rfc-editor.org/rfc/rfc6376)
-to authenticate incoming emails.
-Incoming emails must have a valid DKIM signature with
-Signing Domain Identifier (SDID, `d=` parameter in the DKIM-Signature header)
-equal to the `From:` header domain.
-This property is checked by OpenDKIM screen policy script
-before validating the signatures.
-This correpsonds to strict [DMARC](https://www.rfc-editor.org/rfc/rfc7489) alignment (`adkim=s`),
-but chatmail does not rely on DMARC and does not consult the sender policy published in DMARC records.
-Other legacy authentication mechanisms such as [iprev](https://www.rfc-editor.org/rfc/rfc8601#section-2.7.3)
-and [SPF](https://www.rfc-editor.org/rfc/rfc7208) are also not taken into account.
-If there is no valid DKIM signature on the incoming email,
-the sender receives a "5.7.1 No valid DKIM signature found" error.
-
-Outgoing emails must be sent over authenticated connection
-with envelope MAIL FROM (return path) corresponding to the login.
-This is ensured by Postfix which maps login username
-to MAIL FROM with
-[`smtpd_sender_login_maps`](https://www.postfix.org/postconf.5.html#smtpd_sender_login_maps)
-and rejects incorrectly authenticated emails with [`reject_sender_login_mismatch`](reject_sender_login_mismatch) policy.
-`From:` header must correspond to envelope MAIL FROM,
-this is ensured by `filtermail` proxy.

chatmaild/pyproject.toml

@@ -36,16 +36,6 @@ log_format = "%(asctime)s %(levelname)s %(message)s"
 log_date_format = "%Y-%m-%d %H:%M:%S"
 log_level = "INFO"
 
-[tool.ruff]
-lint.select = [
-  "F", # Pyflakes
-  "I", # isort
-
-  "PLC", # Pylint Convention
-  "PLE", # Pylint Error
-  "PLW", # Pylint Warning
-]
-
 [tool.tox]
 legacy_tox_ini = """
 [tox]
@@ -57,9 +47,10 @@ skipdist = True
 skip_install = True
 deps =
   ruff
+  black
 commands =
-  ruff format --quiet --diff src/
-  ruff check src/
+  black --quiet --check --diff src/
+  ruff src/
 
 [testenv]
 deps = pytest 

chatmaild/src/chatmaild/config.py

@@ -20,7 +20,6 @@ class Config:
         self.passthrough_recipients = params["passthrough_recipients"].split()
         self.filtermail_smtp_port = int(params["filtermail_smtp_port"])
         self.postfix_reinject_port = int(params["postfix_reinject_port"])
-        self.iroh_relay = params.get("iroh_relay")
         self.privacy_postal = params.get("privacy_postal")
         self.privacy_mail = params.get("privacy_mail")
         self.privacy_pdo = params.get("privacy_pdo")

chatmaild/src/chatmaild/database.py

@@ -1,5 +1,5 @@
-import contextlib
 import sqlite3
+import contextlib
 import time
 from pathlib import Path
 

chatmaild/src/chatmaild/doveauth.py

@@ -1,18 +1,17 @@
-import crypt
-import json
 import logging
 import os
-import sys
 import time
-from pathlib import Path
+import sys
+import json
+import crypt
 from socketserver import (
+    UnixStreamServer,
     StreamRequestHandler,
     ThreadingMixIn,
-    UnixStreamServer,
 )
 
-from .config import Config, read_config
 from .database import Database
+from .config import read_config, Config
 
 NOCREATE_FILE = "/etc/chatmail-nocreate"
 
@@ -46,32 +45,23 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
         return False
     localpart, domain = parts
 
-    if localpart == "echo":
-        # echobot account should not be created in the database
-        return False
-
     if (
         len(localpart) > config.username_max_length
         or len(localpart) < config.username_min_length
     ):
-        logging.warning(
-            "localpart %s has to be between %s and %s chars long",
-            localpart,
-            config.username_min_length,
-            config.username_max_length,
-        )
+        if localpart != "echo":
+            logging.warning(
+                "localpart %s has to be between %s and %s chars long",
+                localpart,
+                config.username_min_length,
+                config.username_max_length,
+            )
+            return False
 
     return True
 
 
 def get_user_data(db, config: Config, user):
-    if user == f"echo@{config.mail_domain}":
-        return dict(
-            home=f"/home/vmail/mail/{config.mail_domain}/echo@{config.mail_domain}",
-            uid="vmail",
-            gid="vmail",
-        )
-
     with db.read_connection() as conn:
         result = conn.get_user(user)
     if result:
@@ -86,21 +76,6 @@ def lookup_userdb(db, config: Config, user):
 
 
 def lookup_passdb(db, config: Config, user, cleartext_password):
-    if user == f"echo@{config.mail_domain}":
-        # Echobot writes password it wants to log in with into /run/echobot/password
-        try:
-            password = Path("/run/echobot/password").read_text()
-        except Exception:
-            logging.exception("Exception when trying to read /run/echobot/password")
-            return None
-
-        return dict(
-            home=f"/home/vmail/mail/{config.mail_domain}/echo@{config.mail_domain}",
-            uid="vmail",
-            gid="vmail",
-            password=encrypt_password(password),
-        )
-
     with db.write_transaction() as conn:
         userdata = conn.get_user(user)
         if userdata:

chatmaild/src/chatmaild/echo.py

@@ -3,17 +3,14 @@
 
 it will echo back any message that has non-empty text and also supports the /help command.
 """
-
 import logging
 import os
-import subprocess
 import sys
-from pathlib import Path
 
 from deltachat_rpc_client import Bot, DeltaChat, EventType, Rpc, events
 
-from chatmaild.config import read_config
 from chatmaild.newemail import create_newemail_dict
+from chatmaild.config import read_config
 
 hooks = events.HookCollection()
 
@@ -78,23 +75,9 @@ def main():
         account = accounts[0] if accounts else deltachat.add_account()
 
         bot = Bot(account, hooks)
-
-        config = read_config(sys.argv[1])
-
-        # Create password file
-        if bot.is_configured():
-            password = bot.account.get_config("mail_pw")
-        else:
-            password = create_newemail_dict(config)["password"]
-        Path("/run/echobot/password").write_text(password)
-
-        # Give the user which doveauth runs as access to the password file.
-        subprocess.run(
-            ["/usr/bin/setfacl", "-m", "user:vmail:r", "/run/echobot/password"],
-            check=True,
-        )
-
         if not bot.is_configured():
+            config = read_config(sys.argv[1])
+            password = create_newemail_dict(config).get("password")
             email = "echo@" + config.mail_domain
             bot.configure(email, password)
         bot.run_forever()

chatmaild/src/chatmaild/filedict.py

@@ -1,9 +1,8 @@
-import json
-import logging
 import os
-from contextlib import contextmanager
-
+import logging
+import json
 import filelock
+from contextlib import contextmanager
 
 
 class FileDict:

chatmaild/src/chatmaild/filtermail.py

@@ -1,14 +1,14 @@
 #!/usr/bin/env python3
 import asyncio
 import logging
-import sys
 import time
-from email import policy
+import sys
 from email.parser import BytesParser
+from email import policy
 from email.utils import parseaddr
-from smtplib import SMTP as SMTPClient
 
 from aiosmtpd.controller import Controller
+from smtplib import SMTP as SMTPClient
 
 from .config import read_config
 

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -18,7 +18,7 @@ max_user_send_per_minute = 60
 max_mailbox_size = 100M
 
 # days after which mails are unconditionally deleted
-delete_mails_after = 20
+delete_mails_after = 40
 
 # minimum length a username must have
 username_min_length = 9

chatmaild/src/chatmaild/metadata.py

@@ -1,17 +1,17 @@
-import logging
-import os
-import sys
 from pathlib import Path
 from socketserver import (
+    UnixStreamServer,
     StreamRequestHandler,
     ThreadingMixIn,
-    UnixStreamServer,
 )
+import sys
+import logging
+import os
 
-from .config import read_config
 from .filedict import FileDict
 from .notifier import Notifier
 
+
 DICTPROXY_HELLO_CHAR = "H"
 DICTPROXY_LOOKUP_CHAR = "L"
 DICTPROXY_ITERATE_CHAR = "I"
@@ -49,40 +49,32 @@ class Metadata:
         return mdict.get(self.DEVICETOKEN_KEY, [])
 
 
-def handle_dovecot_protocol(rfile, wfile, notifier, metadata, iroh_relay=None):
+def handle_dovecot_protocol(rfile, wfile, notifier, metadata):
     transactions = {}
     while True:
         msg = rfile.readline().strip().decode()
         if not msg:
             break
 
-        res = handle_dovecot_request(msg, transactions, notifier, metadata, iroh_relay)
+        res = handle_dovecot_request(msg, transactions, notifier, metadata)
         if res:
             wfile.write(res.encode("ascii"))
             wfile.flush()
 
 
-def handle_dovecot_request(msg, transactions, notifier, metadata, iroh_relay=None):
+def handle_dovecot_request(msg, transactions, notifier, metadata):
     # see https://doc.dovecot.org/3.0/developer_manual/design/dict_protocol/
     short_command = msg[0]
     parts = msg[1:].split("\t")
     if short_command == DICTPROXY_LOOKUP_CHAR:
         # Lpriv/43f5f508a7ea0366dff30200c15250e3/devicetoken\tlkj123poi@c2.testrun.org
-        keyparts = parts[0].split("/", 2)
+        keyparts = parts[0].split("/")
         if keyparts[0] == "priv":
             keyname = keyparts[2]
             addr = parts[1]
             if keyname == metadata.DEVICETOKEN_KEY:
                 res = " ".join(metadata.get_tokens_for_addr(addr))
                 return f"O{res}\n"
-        elif keyparts[0] == "shared":
-            keyname = keyparts[2]
-            if (
-                keyname == "vendor/vendor.dovecot/pvt/server/vendor/deltachat/irohrelay"
-                and iroh_relay
-            ):
-                # Handle `GETMETADATA "" /shared/vendor/deltachat/irohrelay`
-                return f"O{iroh_relay}\n"
         logging.warning("lookup ignored: %r", msg)
         return "N\n"
     elif short_command == DICTPROXY_ITERATE_CHAR:
@@ -128,10 +120,7 @@ class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
 
 
 def main():
-    socket, vmail_dir, config_path = sys.argv[1:]
-
-    config = read_config(config_path)
-    iroh_relay = config.iroh_relay
+    socket, vmail_dir = sys.argv[1:]
 
     vmail_dir = Path(vmail_dir)
     if not vmail_dir.exists():
@@ -147,9 +136,7 @@ def main():
     class Handler(StreamRequestHandler):
         def handle(self):
             try:
-                handle_dovecot_protocol(
-                    self.rfile, self.wfile, notifier, metadata, iroh_relay
-                )
+                handle_dovecot_protocol(self.rfile, self.wfile, notifier, metadata)
             except Exception:
                 logging.exception("Exception in the dovecot dictproxy handler")
                 raise

chatmaild/src/chatmaild/metrics.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
-import sys
-import time
 from pathlib import Path
+import time
+import sys
 
 
 def main(vmail_dir=None):

chatmaild/src/chatmaild/newemail.py

@@ -1,13 +1,13 @@
 #!/usr/local/lib/chatmaild/venv/bin/python3
 
-"""CGI script for creating new accounts."""
+""" CGI script for creating new accounts. """
 
 import json
 import random
 import secrets
 import string
 
-from chatmaild.config import Config, read_config
+from chatmaild.config import read_config, Config
 
 CONFIG_PATH = "/usr/local/lib/chatmaild/chatmail.ini"
 ALPHANUMERIC = string.ascii_lowercase + string.digits

chatmaild/src/chatmaild/notifier.py

@@ -25,16 +25,15 @@ The meaning and format of tokens is basically a matter of Delta-Chat Core and
 the `notification.delta.chat` service.
 """
 
-import logging
-import math
 import os
 import time
-from dataclasses import dataclass
+import math
+import logging
+from uuid import uuid4
+from threading import Thread
 from pathlib import Path
 from queue import PriorityQueue
-from threading import Thread
-from uuid import uuid4
-
+from dataclasses import dataclass
 import requests
 
 

chatmaild/src/chatmaild/tests/plugin.py

@@ -1,14 +1,14 @@
+import random
+from pathlib import Path
+import os
 import importlib.resources
 import itertools
-import os
-import random
-from email import policy
 from email.parser import BytesParser
-from pathlib import Path
-
+from email import policy
 import pytest
-from chatmaild.config import read_config, write_initial_config
+
 from chatmaild.database import Database
+from chatmaild.config import read_config, write_initial_config
 
 
 @pytest.fixture

chatmaild/src/chatmaild/tests/test_config.py

@@ -24,7 +24,7 @@ def test_read_config_testrun(make_config):
     assert config.postfix_reinject_port == 10025
     assert config.max_user_send_per_minute == 60
     assert config.max_mailbox_size == "100M"
-    assert config.delete_mails_after == "20"
+    assert config.delete_mails_after == "40"
     assert config.username_min_length == 9
     assert config.username_max_length == 9
     assert config.password_min_length == 9

chatmaild/src/chatmaild/tests/test_doveauth.py

@@ -1,18 +1,18 @@
 import io
 import json
+import pytest
 import queue
 import threading
 import traceback
 
 import chatmaild.doveauth
-import pytest
-from chatmaild.database import DBError
 from chatmaild.doveauth import (
     get_user_data,
-    handle_dovecot_protocol,
-    handle_dovecot_request,
     lookup_passdb,
+    handle_dovecot_request,
+    handle_dovecot_protocol,
 )
+from chatmaild.database import DBError
 
 
 def test_basic(db, example_config):

chatmaild/src/chatmaild/tests/test_filtermail.py

@@ -1,11 +1,12 @@
-import pytest
 from chatmaild.filtermail import (
+    check_encrypted,
     BeforeQueueHandler,
     SendRateLimiter,
-    check_encrypted,
     check_mdn,
 )
 
+import pytest
+
 
 @pytest.fixture
 def maildomain():

chatmaild/src/chatmaild/tests/test_metadata.py

@@ -1,12 +1,12 @@
 import io
-import time
-
 import pytest
 import requests
+import time
+
 from chatmaild.metadata import (
-    Metadata,
-    handle_dovecot_protocol,
     handle_dovecot_request,
+    handle_dovecot_protocol,
+    Metadata,
 )
 from chatmaild.notifier import (
     Notifier,
@@ -296,17 +296,3 @@ def test_persistent_queue_items(tmp_path, testaddr, token):
     item2.delete()
     assert not item2.path.exists()
     assert not queue_item < item2 and not item2 < queue_item
-
-
-def test_iroh_relay(metadata):
-    rfile = io.BytesIO(
-        b"\n".join(
-            [
-                b"H",
-                b"Lshared/0123/vendor/vendor.dovecot/pvt/server/vendor/deltachat/irohrelay\tuser@example.org",
-            ]
-        )
-    )
-    wfile = io.BytesIO()
-    handle_dovecot_protocol(rfile, wfile, notifier, metadata, "https://example.org/")
-    assert wfile.getvalue() == b"Ohttps://example.org/\n"

cmdeploy/pyproject.toml

@@ -16,6 +16,7 @@ dependencies = [
   "build",
   "tox",
   "ruff",
+  "black",
   "pytest",
   "pytest-xdist", 
   "imap_tools",
@@ -30,13 +31,3 @@ cmdeploy = "cmdeploy.cmdeploy:main"
 
 [tool.pytest.ini_options]
 addopts = "-v -ra --strict-markers"
-
-[tool.ruff]
-lint.select = [
-  "F", # Pyflakes
-  "I", # isort
-
-  "PLC", # Pylint Convention
-  "PLE", # Pylint Error
-  "PLW", # Pylint Warning
-]

cmdeploy/src/cmdeploy/__init__.py

@@ -2,22 +2,21 @@
 Chat Mail pyinfra deploy.
 """
 
-import importlib.resources
-import io
-import shutil
-import subprocess
 import sys
+import importlib.resources
+import subprocess
+import shutil
+import io
 from pathlib import Path
 
-from chatmaild.config import Config, read_config
 from pyinfra import host
+from pyinfra.operations import apt, files, server, systemd, pip
 from pyinfra.facts.files import File
 from pyinfra.facts.systemd import SystemdEnabled
-from pyinfra.operations import apt, files, pip, server, systemd
-
 from .acmetool import deploy_acmetool
 
-root_owned = dict(user="root", group="root", mode="644")
+from chatmaild.config import read_config, Config
+
 
 def _build_chatmaild(dist_dir) -> None:
     dist_dir = Path(dist_dir).resolve()
@@ -51,6 +50,7 @@ def _install_remote_venv_with_chatmaild(config) -> None:
     remote_dist_file = f"{remote_base_dir}/dist/{dist_file.name}"
     remote_venv_dir = f"{remote_base_dir}/venv"
     remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini"
+    root_owned = dict(user="root", group="root", mode="644")
 
     apt.packages(
         name="apt install python3-virtualenv",
@@ -85,19 +85,9 @@ def _install_remote_venv_with_chatmaild(config) -> None:
         ],
     )
 
-    # create metrics every 5 minutes via systemd
-
-    files.put(
-        name="Upload metrics.timer",
-        src=importlib.resources.files(__package__).joinpath("service/metrics.timer"),
-        dest=f"/etc/systemd/system/metrics.timer",
-        **root_owned,
-    )
-
     files.template(
-        name="upload metrics.service",
-        src=importlib.resources.files(__package__).joinpath("service/metrics.service.j2"),
-        dest="/etc/systemd/system/metrics.service",
+        src=importlib.resources.files(__package__).joinpath("metrics.cron.j2"),
+        dest="/etc/cron.d/chatmail-metrics",
         user="root",
         group="root",
         mode="644",
@@ -107,15 +97,6 @@ def _install_remote_venv_with_chatmaild(config) -> None:
         },
     )
 
-    systemd.service(
-        name=f"Setup metrics timer",
-        service="metrics.timer",
-        running=True,
-        enabled=True,
-        restarted=True,
-        daemon_reload=True,
-    )
-
     # install systemd units
     for fn in (
         "doveauth",
@@ -371,23 +352,6 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
             commands=["/usr/bin/sievec /etc/dovecot/default.sieve"],
         )
 
-    files.template(
-        src=importlib.resources.files(__package__).joinpath("service/expunge.service.j2"),
-        dest="/etc/systemd/system/expunge.service",
-        config={
-            "mail_domain": config.mail_domain,
-            "delete_mails_after": config.delete_mails_after,
-        },
-        **root_owned,
-    )
-
-    files.put(
-        name="Upload expunge.timer",
-        src=importlib.resources.files(__package__).joinpath("service/expunge.timer"),
-        dest=f"/etc/systemd/system/expunge.timer",
-        **root_owned,
-    )
-
     files.template(
         src=importlib.resources.files(__package__).joinpath("dovecot/expunge.cron.j2"),
         dest="/etc/cron.d/expunge",
@@ -513,31 +477,12 @@ def deploy_chatmail(config_path: Path) -> None:
         groups=["opendkim"],
         system=True,
     )
-    server.user(name="Create echobot user", user="echobot", system=True)
 
     server.shell(
         name="Fix file owner in /home/vmail",
         commands=["test -d /home/vmail && chown -R vmail:vmail /home/vmail"],
     )
 
-    # Add our OBS repository for dovecot_no_delay
-    files.put(
-        name="Add Deltachat OBS GPG key to apt keyring",
-        src=importlib.resources.files(__package__).joinpath("obs-home-deltachat.gpg"),
-        dest="/etc/apt/keyrings/obs-home-deltachat.gpg",
-        user="root",
-        group="root",
-        mode="644",
-    )
-
-    files.line(
-        name="Add DeltaChat OBS home repository to sources.list",
-        path="/etc/apt/sources.list",
-        line="deb [signed-by=/etc/apt/keyrings/obs-home-deltachat.gpg] https://download.opensuse.org/repositories/home:/deltachat/Debian_12/ ./",
-        escape_regex_characters=True,
-        ensure_newline=True,
-    )
-
     apt.update(name="apt update", cache_time=24 * 3600)
 
     apt.packages(
@@ -559,7 +504,6 @@ def deploy_chatmail(config_path: Path) -> None:
             "systemctl reset-failed unbound.service",
         ],
     )
-
     systemd.service(
         name="Start and enable unbound",
         service="unbound.service",
@@ -569,15 +513,10 @@ def deploy_chatmail(config_path: Path) -> None:
 
     # Deploy acmetool to have TLS certificates.
     deploy_acmetool(
+        nginx_hook=True,
         domains=[mail_domain, f"mta-sts.{mail_domain}", f"www.{mail_domain}"],
     )
 
-    apt.packages(
-        # required for setfacl for echobot
-        name="Install acl",
-        packages="acl",
-    )
-
     apt.packages(
         name="Install Postfix",
         packages="postfix",

cmdeploy/src/cmdeploy/acmetool/__init__.py

@@ -1,11 +1,11 @@
 import importlib.resources
 
+from pyinfra.operations import apt, files, systemd, server
 from pyinfra import host
 from pyinfra.facts.systemd import SystemdStatus
-from pyinfra.operations import apt, files, server, systemd
 
 
-def deploy_acmetool(email="", domains=[]):
+def deploy_acmetool(nginx_hook=False, email="", domains=[]):
     """Deploy acmetool."""
     apt.packages(
         name="Install acmetool",
@@ -20,13 +20,16 @@ def deploy_acmetool(email="", domains=[]):
         mode="644",
     )
 
-    files.put(
-        src=importlib.resources.files(__package__).joinpath("acmetool.hook").open("rb"),
-        dest="/usr/lib/acme/hooks/nginx",
-        user="root",
-        group="root",
-        mode="744",
-    )
+    if nginx_hook:
+        files.put(
+            src=importlib.resources.files(__package__)
+            .joinpath("acmetool.hook")
+            .open("rb"),
+            dest="/usr/lib/acme/hooks/nginx",
+            user="root",
+            group="root",
+            mode="744",
+        )
 
     files.template(
         src=importlib.resources.files(__package__).joinpath("response-file.yaml.j2"),
@@ -69,8 +72,7 @@ def deploy_acmetool(email="", domains=[]):
         restarted=service_file.changed,
     )
 
-    if str(host) != "staging.testrun.org":
-        server.shell(
-            name=f"Request certificate for: { ', '.join(domains) }",
-            commands=[f"acmetool want --xlog.severity=debug { ' '.join(domains)}"],
-        )
+    server.shell(
+        name=f"Request certificate for: { ', '.join(domains) }",
+        commands=[f"acmetool want { ' '.join(domains)}"],
+    )

cmdeploy/src/cmdeploy/acmetool/acmetool.hook

@@ -3,5 +3,3 @@ set -e
 EVENT_NAME="$1"
 [ "$EVENT_NAME" = "live-updated" ] || exit 42
 systemctl restart nginx.service
-systemctl reload dovecot.service
-systemctl reload postfix.service

cmdeploy/src/cmdeploy/acmetool/response-file.yaml.j2

@@ -1,2 +1,2 @@
 "acme-enter-email": "{{ email }}"
-"acme-agreement:https://letsencrypt.org/documents/LE-SA-v1.4-April-3-2024.pdf": true
+"acme-agreement:https://letsencrypt.org/documents/LE-SA-v1.3-September-21-2022.pdf": true

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -4,18 +4,19 @@ along with command line option and subcommand parsing.
 """
 
 import argparse
+import shutil
+import subprocess
 import importlib.resources
 import importlib.util
 import os
-import shutil
-import subprocess
 import sys
 from pathlib import Path
 
-from chatmaild.config import read_config, write_initial_config
-from termcolor import colored
 
-from cmdeploy.dns import check_necessary_dns, show_dns
+from termcolor import colored
+from chatmaild.config import read_config, write_initial_config
+from cmdeploy.dns import show_dns, check_necessary_dns
+
 
 #
 # cmdeploy sub commands and options
@@ -156,26 +157,26 @@ def fmt_cmd_options(parser):
 
 
 def fmt_cmd(args, out):
-    """Run formattting fixes on all chatmail source code."""
+    """Run formattting fixes (ruff and black) on all chatmail source code."""
 
     sources = [str(importlib.resources.files(x)) for x in ("chatmaild", "cmdeploy")]
-    format_args = [shutil.which("ruff"), "format"]
-    check_args = [shutil.which("ruff"), "check"]
+    black_args = [shutil.which("black")]
+    ruff_args = [shutil.which("ruff")]
 
     if args.check:
-        format_args.append("--diff")
+        black_args.append("--check")
     else:
-        check_args.append("--fix")
+        ruff_args.append("--fix")
 
     if not args.verbose:
-        check_args.append("--quiet")
-        format_args.append("--quiet")
+        black_args.append("-q")
+        ruff_args.append("-q")
 
-    format_args.extend(sources)
-    check_args.extend(sources)
+    black_args.extend(sources)
+    ruff_args.extend(sources)
 
-    out.check_call(" ".join(format_args), quiet=not args.verbose)
-    out.check_call(" ".join(check_args), quiet=not args.verbose)
+    out.check_call(" ".join(black_args), quiet=not args.verbose)
+    out.check_call(" ".join(ruff_args), quiet=not args.verbose)
     return 0
 
 
@@ -231,7 +232,7 @@ class Out:
         if not quiet:
             cmdstring = " ".join(args)
             self(f"[$ {cmdstring}]", file=sys.stderr)
-        proc = subprocess.run(args, env=env, check=False)
+        proc = subprocess.run(args, env=env)
         return proc.returncode
 
 

cmdeploy/src/cmdeploy/deploy.py

@@ -1,8 +1,6 @@
-import importlib.resources
 import os
-
+import importlib.resources
 import pyinfra
-
 from cmdeploy import deploy_chatmail
 
 

cmdeploy/src/cmdeploy/dns.py

@@ -1,9 +1,9 @@
-import datetime
-import importlib
-import subprocess
 import sys
 
 import requests
+import importlib
+import subprocess
+import datetime
 
 
 class DNS:
@@ -104,8 +104,8 @@ def show_dns(args, out) -> int:
             return 0
         except TypeError:
             pass
-        for raw_line in zonefile.splitlines():
-            line = raw_line.format(
+        for line in zonefile.splitlines():
+            line = line.format(
                 acme_account_url=acme_account_url,
                 sts_id=datetime.datetime.now().strftime("%Y%m%d%H%M"),
                 chatmail_domain=args.config.mail_domain,

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -27,7 +27,7 @@ mail_plugins = quota
 # these are the capabilities Delta Chat cares about actually 
 # so let's keep the network overhead per login small
 # https://github.com/deltachat/deltachat-core-rust/blob/master/src/imap/capabilities.rs
-imap_capability = IMAP4rev1 IDLE MOVE QUOTA CONDSTORE NOTIFY METADATA XDELTAPUSH XCHATMAIL
+imap_capability = IMAP4rev1 IDLE MOVE QUOTA CONDSTORE NOTIFY METADATA XDELTAPUSH
 
 
 # Authentication for system users.

cmdeploy/src/cmdeploy/genqr.py

@@ -1,9 +1,8 @@
 import importlib
-import io
-import os
-
 import qrcode
-from PIL import Image, ImageDraw, ImageFont
+import os
+from PIL import ImageFont, ImageDraw, Image
+import io
 
 
 def gen_qr_png_data(maildomain):

cmdeploy/src/cmdeploy/service/chatmail-metadata.service.f

@@ -2,7 +2,7 @@
 Description=Chatmail dict proxy for IMAP METADATA
 
 [Service]
-ExecStart={execpath} /run/chatmail-metadata/metadata.socket /home/vmail/mail/{mail_domain} {config_path}
+ExecStart={execpath} /run/chatmail-metadata/metadata.socket /home/vmail/mail/{mail_domain}
 Restart=always
 RestartSec=30
 User=vmail

cmdeploy/src/cmdeploy/service/echobot.service.f

@@ -7,20 +7,6 @@ Environment="PATH={remote_venv_dir}:$PATH"
 Restart=always
 RestartSec=30
 
-User=echobot
-Group=echobot
-
-# Create /var/lib/echobot
-StateDirectory=echobot
-
-# Create /run/echobot
-#
-# echobot stores /run/echobot/password
-# with a password there, which doveauth then reads.
-RuntimeDirectory=echobot
-
-WorkingDirectory=/var/lib/echobot
-
 # Apply security restrictions suggested by
 #   systemd-analyze security echobot.service
 CapabilityBoundingSet=
@@ -30,10 +16,7 @@ NoNewPrivileges=true
 PrivateDevices=true
 PrivateMounts=true
 PrivateTmp=true
-
-# We need to know about doveauth user to give it access to /run/echobot/password
-PrivateUsers=false
-
+PrivateUsers=true
 ProtectClock=true
 ProtectControlGroups=true
 ProtectHostname=true

cmdeploy/src/cmdeploy/service/expunge.service.j2

@@ -1,16 +0,0 @@
-[Unit]
-Description=Expunge old mails after {{ config.delete_mails_after }} days
-
-[Service]
-Type=oneshot
-# delete all mails after {{ config.delete_mails_after }} days, in the Inbox
-ExecStart=/home/vmail/mail/{{ config.mail_domain }} -path '*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-# or in any IMAP subfolder
-ExecStart=vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-# even if they are unseen
-ExecStart=vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-ExecStart=vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-# or only temporary (but then they shouldn't be around after {{ config.delete_mails_after }} days anyway).
-ExecStart=vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-ExecStart=vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-ExecStart=vmail find /home/vmail/mail/{{ config.mail_domain }} -name 'maildirsize' -type f -delete

cmdeploy/src/cmdeploy/service/expunge.timer

@@ -1,9 +0,0 @@
-[Unit]
-Description=Run expunge.service daily
-
-[Timer]
-OnCalendar=weekly
-Persistent=true
-
-[Install]
-WantedBy=timers.target

cmdeploy/src/cmdeploy/service/metrics.service.j2

@@ -1,5 +0,0 @@
-[Unit]
-Description=Generate metrics in /var/www/html/metrics
-
-[Service]
-ExecStart={{ config.execpath }} /home/vmail/mail/{{ config.mail_domain }} > /var/www/html/metrics

cmdeploy/src/cmdeploy/service/metrics.timer

@@ -1,9 +0,0 @@
-[Unit]
-Description=Run metrics.service every 5 minutes
-
-[Timer]
-OnBootSec=5min
-OnUnitActiveSec=5min
-
-[Install]
-WantedBy=timers.target

cmdeploy/src/cmdeploy/tests/online/test_0_login.py

@@ -1,10 +1,9 @@
+import pytest
+import threading
 import queue
 import socket
-import threading
 
-import pytest
 from chatmaild.config import read_config
-
 from cmdeploy.cmdeploy import main
 
 
@@ -15,13 +14,6 @@ def test_init(tmp_path, maildomain):
     assert config.mail_domain == maildomain
 
 
-def test_capabilities(imap):
-    imap.connect()
-    capas = imap.conn.capabilities
-    assert "XCHATMAIL" in capas
-    assert "XDELTAPUSH" in capas
-
-
 def test_login_basic_functioning(imap_or_smtp, gencreds, lp):
     """Test a) that an initial login creates a user automatically
     and b) verify we can also login a second time with the same password

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -1,5 +1,4 @@
 import smtplib
-
 import pytest
 
 

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -1,11 +1,11 @@
-import ipaddress
-import random
-import re
 import time
+import re
+import random
 
-import imap_tools
 import pytest
 import requests
+import ipaddress
+import imap_tools
 
 
 @pytest.fixture

cmdeploy/src/cmdeploy/tests/plugin.py

@@ -1,16 +1,17 @@
-import imaplib
-import io
-import itertools
 import os
-import random
-import smtplib
-import subprocess
+import io
 import time
+import random
+import subprocess
+import imaplib
+import smtplib
+import itertools
 from pathlib import Path
-
 import pytest
-from chatmaild.config import read_config
+
 from chatmaild.database import Database
+from chatmaild.config import read_config
+
 
 conftestdir = Path(__file__).parent
 

cmdeploy/src/cmdeploy/tests/test_cmdeploy.py

@@ -1,7 +1,6 @@
 import os
 
 import pytest
-
 from cmdeploy.cmdeploy import get_parser, main
 
 

cmdeploy/src/cmdeploy/www.py

@@ -1,14 +1,13 @@
-import hashlib
 import importlib.resources
+import webbrowser
+import hashlib
 import time
 import traceback
-import webbrowser
 
 import markdown
-from chatmaild.config import read_config
 from jinja2 import Template
-
 from .genqr import gen_qr_png_data
+from chatmaild.config import read_config
 
 
 def snapshot_dir_stats(somedir):
@@ -121,8 +120,7 @@ def main():
     print(f"watching {src_path} directory for changes")
 
     changenum = 0
-    count = 0
-    while True:
+    for count in range(0, 1000000):
         newstats = snapshot_dir_stats(src_path)
         if newstats == stats and count % 60 != 0:
             count += 1