Prepare for pyinfra 3

Make `find` look for accounts.

chatmaild/src/chatmaild/doveauth.py

@@ -160,6 +160,19 @@ def handle_dovecot_request(msg, db, config: Config):
     return None
 
 
+def handle_dovecot_protocol(rfile, wfile, db: Database, config: Config):
+    while True:
+        msg = rfile.readline().strip().decode()
+        if not msg:
+            break
+        res = handle_dovecot_request(msg, db, config)
+        if res:
+            wfile.write(res.encode("ascii"))
+            wfile.flush()
+        else:
+            logging.warning("request had no answer: %r", msg)
+
+
 class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
     request_queue_size = 100
 
@@ -173,16 +186,7 @@ def main():
     class Handler(StreamRequestHandler):
         def handle(self):
             try:
-                while True:
+                handle_dovecot_protocol(self.rfile, self.wfile, db, config)
-                    msg = self.rfile.readline().strip().decode()
-                    if not msg:
-                        break
-                    res = handle_dovecot_request(msg, db, config)
-                    if res:
-                        self.wfile.write(res.encode("ascii"))
-                        self.wfile.flush()
-                    else:
-                        logging.warn("request had no answer: %r", msg)
             except Exception:
                 logging.exception("Exception in the handler")
                 raise

chatmaild/src/chatmaild/tests/test_doveauth.py

@@ -1,11 +1,17 @@
+import io
 import json
 import pytest
-import threading
 import queue
+import threading
 import traceback
 
 import chatmaild.doveauth
-from chatmaild.doveauth import get_user_data, lookup_passdb, handle_dovecot_request
+from chatmaild.doveauth import (
+    get_user_data,
+    lookup_passdb,
+    handle_dovecot_request,
+    handle_dovecot_protocol,
+)
 from chatmaild.database import DBError
 
 
@@ -69,6 +75,15 @@ def test_handle_dovecot_request(db, example_config):
     assert userdata["password"].startswith("{SHA512-CRYPT}")
 
 
+def test_handle_dovecot_protocol(db, example_config):
+    rfile = io.BytesIO(
+        b"H3\t2\t0\t\tauth\nLshared/userdb/foobar@chat.example.org\tfoobar@chat.example.org\n"
+    )
+    wfile = io.BytesIO()
+    handle_dovecot_protocol(rfile, wfile, db, example_config)
+    assert wfile.getvalue() == b"N\n"
+
+
 def test_50_concurrent_lookups_different_accounts(db, gencreds, example_config):
     num_threads = 50
     req_per_thread = 5

cmdeploy/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 name = "cmdeploy"
 version = "0.2"
 dependencies = [
-  "pyinfra",
+  "pyinfra==3.0b0",
   "pillow",
   "qrcode",
   "markdown",

cmdeploy/src/cmdeploy/__init__.py

@@ -1,6 +1,7 @@
 """
 Chat Mail pyinfra deploy.
 """
+
 import sys
 import importlib.resources
 import subprocess
@@ -9,13 +10,15 @@ import io
 from pathlib import Path
 
 from pyinfra import host
-from pyinfra.operations import apt, files, server, systemd, pip
+from pyinfra.operations import apt, files, server, systemd, pip, util
 from pyinfra.facts.files import File
 from pyinfra.facts.systemd import SystemdEnabled
 from .acmetool import deploy_acmetool
 
 from chatmaild.config import read_config, Config
 
+from typing import Callable
+
 
 def _build_chatmaild(dist_dir) -> None:
     dist_dir = Path(dist_dir).resolve()
@@ -126,10 +129,8 @@ def _install_remote_venv_with_chatmaild(config) -> None:
         )
 
 
-def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
+def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> Callable[[], bool]:
     """Configures OpenDKIM"""
-    need_restart = False
-
     server.group(name="Create opendkim group", group="opendkim", system=True)
     server.user(
         name="Create opendkim user",
@@ -152,7 +153,6 @@ def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
         mode="644",
         config={"domain_name": domain, "opendkim_selector": dkim_selector},
     )
-    need_restart |= main_config.changed
 
     screen_script = files.put(
         src=importlib.resources.files(__package__).joinpath("opendkim/screen.lua"),
@@ -161,7 +161,6 @@ def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
         group="root",
         mode="644",
     )
-    need_restart |= screen_script.changed
 
     final_script = files.put(
         src=importlib.resources.files(__package__).joinpath("opendkim/final.lua"),
@@ -170,7 +169,6 @@ def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
         group="root",
         mode="644",
     )
-    need_restart |= final_script.changed
 
     files.directory(
         name="Add opendkim directory to /etc",
@@ -189,7 +187,6 @@ def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
         mode="644",
         config={"domain_name": domain, "opendkim_selector": dkim_selector},
     )
-    need_restart |= keytable.changed
 
     signing_table = files.template(
         src=importlib.resources.files(__package__).joinpath("opendkim/SigningTable"),
@@ -199,7 +196,7 @@ def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
         mode="644",
         config={"domain_name": domain, "opendkim_selector": dkim_selector},
     )
-    need_restart |= signing_table.changed
+
     files.directory(
         name="Add opendkim socket directory to /var/spool/postfix",
         path="/var/spool/postfix/opendkim",
@@ -224,10 +221,12 @@ def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
             _sudo_user="opendkim",
         )
 
-    return need_restart
+    return util.any_changed(
+        main_config, screen_script, final_script, keytable, signing_table
+    )
 
 
-def _install_mta_sts_daemon() -> bool:
+def _install_mta_sts_daemon() -> Callable[[], bool]:
     need_restart = False
 
     config = files.put(
@@ -240,7 +239,6 @@ def _install_mta_sts_daemon() -> bool:
         group="root",
         mode="644",
     )
-    need_restart |= config.changed
 
     server.shell(
         name="install postfix-mta-sts-resolver with pip",
@@ -260,15 +258,12 @@ def _install_mta_sts_daemon() -> bool:
         group="root",
         mode="644",
     )
-    need_restart |= systemd_unit.changed
 
-    return need_restart
+    return util.any_changed(config, systemd_unit)
 
 
-def _configure_postfix(config: Config, debug: bool = False) -> bool:
+def _configure_postfix(config: Config, debug: bool = False) -> Callable[[], bool]:
     """Configures Postfix SMTP server."""
-    need_restart = False
-
     main_config = files.template(
         src=importlib.resources.files(__package__).joinpath("postfix/main.cf.j2"),
         dest="/etc/postfix/main.cf",
@@ -277,7 +272,6 @@ def _configure_postfix(config: Config, debug: bool = False) -> bool:
         mode="644",
         config=config,
     )
-    need_restart |= main_config.changed
 
     master_config = files.template(
         src=importlib.resources.files(__package__).joinpath("postfix/master.cf.j2"),
@@ -288,7 +282,6 @@ def _configure_postfix(config: Config, debug: bool = False) -> bool:
         debug=debug,
         config=config,
     )
-    need_restart |= master_config.changed
 
     header_cleanup = files.put(
         src=importlib.resources.files(__package__).joinpath(
@@ -299,27 +292,21 @@ def _configure_postfix(config: Config, debug: bool = False) -> bool:
         group="root",
         mode="644",
     )
-    need_restart |= header_cleanup.changed
 
     # Login map that 1:1 maps email address to login.
     login_map = files.put(
-        src=importlib.resources.files(__package__).joinpath(
+        src=importlib.resources.files(__package__).joinpath("postfix/login_map"),
-            "postfix/login_map"
-        ),
         dest="/etc/postfix/login_map",
         user="root",
         group="root",
         mode="644",
     )
-    need_restart |= login_map.changed
 
-    return need_restart
+    return util.any_changed(main_config, master_config, header_cleanup, login_map)
 
 
-def _configure_dovecot(config: Config, debug: bool = False) -> bool:
+def _configure_dovecot(config: Config, debug: bool = False) -> Callable[[], bool]:
     """Configures Dovecot IMAP server."""
-    need_restart = False
-
     main_config = files.template(
         src=importlib.resources.files(__package__).joinpath("dovecot/dovecot.conf.j2"),
         dest="/etc/dovecot/dovecot.conf",
@@ -329,7 +316,6 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
         config=config,
         debug=debug,
     )
-    need_restart |= main_config.changed
     auth_config = files.put(
         src=importlib.resources.files(__package__).joinpath("dovecot/auth.conf"),
         dest="/etc/dovecot/auth.conf",
@@ -337,7 +323,6 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
         group="root",
         mode="644",
     )
-    need_restart |= auth_config.changed
 
     files.template(
         src=importlib.resources.files(__package__).joinpath("dovecot/expunge.cron.j2"),
@@ -359,13 +344,11 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
             persist=True,
         )
 
-    return need_restart
+    return util.any_changed(main_config, auth_config)
 
 
-def _configure_nginx(domain: str, debug: bool = False) -> bool:
+def _configure_nginx(domain: str, debug: bool = False) -> Callable[[], bool]:
     """Configures nginx HTTP server."""
-    need_restart = False
-
     main_config = files.template(
         src=importlib.resources.files(__package__).joinpath("nginx/nginx.conf.j2"),
         dest="/etc/nginx/nginx.conf",
@@ -374,7 +357,6 @@ def _configure_nginx(domain: str, debug: bool = False) -> bool:
         mode="644",
         config={"domain_name": domain},
     )
-    need_restart |= main_config.changed
 
     autoconfig = files.template(
         src=importlib.resources.files(__package__).joinpath("nginx/autoconfig.xml.j2"),
@@ -384,7 +366,6 @@ def _configure_nginx(domain: str, debug: bool = False) -> bool:
         mode="644",
         config={"domain_name": domain},
     )
-    need_restart |= autoconfig.changed
 
     mta_sts_config = files.template(
         src=importlib.resources.files(__package__).joinpath("nginx/mta-sts.txt.j2"),
@@ -394,7 +375,6 @@ def _configure_nginx(domain: str, debug: bool = False) -> bool:
         mode="644",
         config={"domain_name": domain},
     )
-    need_restart |= mta_sts_config.changed
 
     # install CGI newemail script
     #
@@ -415,7 +395,7 @@ def _configure_nginx(domain: str, debug: bool = False) -> bool:
         mode="755",
     )
 
-    return need_restart
+    return util.any_changed(main_config, autoconfig, mta_sts_config)
 
 
 def _remove_rspamd() -> None:
@@ -519,7 +499,12 @@ def deploy_chatmail(config_path: Path) -> None:
         service="opendkim.service",
         running=True,
         enabled=True,
-        restarted=opendkim_need_restart,
+    )
+    systemd.service(
+        name="Restart OpenDKIM",
+        service="opendkim.service",
+        restarted=True,
+        _if=opendkim_need_restart,
     )
 
     systemd.service(
@@ -528,7 +513,12 @@ def deploy_chatmail(config_path: Path) -> None:
         daemon_reload=True,
         running=True,
         enabled=True,
-        restarted=mta_sts_need_restart,
+    )
+    systemd.service(
+        name="Restart MTA-STS daemon",
+        service="mta-sts-daemon.service",
+        restarted=True,
+        _if=mta_sts_need_restart,
     )
 
     systemd.service(
@@ -536,7 +526,12 @@ def deploy_chatmail(config_path: Path) -> None:
         service="postfix.service",
         running=True,
         enabled=True,
-        restarted=postfix_need_restart,
+    )
+    systemd.service(
+        name="Restart Postfix",
+        service="postfix.service",
+        restarted=True,
+        _if=postfix_need_restart,
     )
 
     systemd.service(
@@ -544,7 +539,12 @@ def deploy_chatmail(config_path: Path) -> None:
         service="dovecot.service",
         running=True,
         enabled=True,
-        restarted=dovecot_need_restart,
+    )
+    systemd.service(
+        name="Restart Dovecot",
+        service="dovecot.service",
+        restarted=True,
+        _if=dovecot_need_restart,
     )
 
     systemd.service(
@@ -552,7 +552,12 @@ def deploy_chatmail(config_path: Path) -> None:
         service="nginx.service",
         running=True,
         enabled=True,
-        restarted=nginx_need_restart,
+    )
+    systemd.service(
+        name="Restart nginx",
+        service="nginx.service",
+        restarted=True,
+        _if=nginx_need_restart,
     )
 
     # This file is used by auth proxy.

cmdeploy/src/cmdeploy/acmetool/__init__.py

@@ -69,7 +69,12 @@ def deploy_acmetool(nginx_hook=False, email="", domains=[]):
         service="acmetool-redirector.service",
         running=True,
         enabled=True,
-        restarted=service_file.changed,
+    )
+    systemd.service(
+        name="Restart acmetool-redirector service",
+        service="acmetool-redirector.service",
+        restarted=True,
+        _if=service_file.did_change,
     )
 
     server.shell(

cmdeploy/src/cmdeploy/chatmail.zone.f

@@ -11,6 +11,5 @@ _dmarc.{chatmail_domain}.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
 _mta-sts.{chatmail_domain}.          TXT "v=STSv1; id={sts_id}"
 mta-sts.{chatmail_domain}.           CNAME {chatmail_domain}.
 www.{chatmail_domain}.               CNAME {chatmail_domain}.
-_smtp._tls.{chatmail_domain}.        TXT "v=TLSRPTv1;rua=mailto:{email}"
 {dkim_entry}
 _adsp._domainkey.{chatmail_domain}.  TXT "dkim=discardable"

cmdeploy/src/cmdeploy/dns.py

@@ -82,7 +82,6 @@ def show_dns(args, out) -> int:
             f.read()
             .format(
                 acme_account_url=acme_account_url,
-                email=f"root@{args.config.mail_domain}",
                 sts_id=datetime.datetime.now().strftime("%Y%m%d%H%M"),
                 chatmail_domain=args.config.mail_domain,
                 dkim_entry=dkim_entry,
@@ -102,7 +101,6 @@ def show_dns(args, out) -> int:
         for line in zonefile.splitlines():
             line = line.format(
                 acme_account_url=acme_account_url,
-                email=f"root@{args.config.mail_domain}",
                 sts_id=datetime.datetime.now().strftime("%Y%m%d%H%M"),
                 chatmail_domain=args.config.mail_domain,
                 dkim_entry=dkim_entry,

cmdeploy/src/cmdeploy/dovecot/expunge.cron.j2

@@ -1,10 +1,10 @@
 # delete all mails after {{ config.delete_mails_after }} days, in the Inbox
-2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }}/*/cur -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }} -path '*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 # or in any IMAP subfolder
-2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }}/*/.*/cur -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 # even if they are unseen
-2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }}/*/new -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }} -path '*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }}/*/.*/new -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 # or only temporary (but then they shouldn't be around after {{ config.delete_mails_after }} days anyway).
-2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }}/*/tmp -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }} -path '*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }}/*/.*/tmp -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -136,3 +136,15 @@ def test_hide_senders_ip_address(cmfactory):
     user2.direct_imap.select_folder("Inbox")
     msg = user2.direct_imap.get_all_messages()[0]
     assert public_ip not in msg.obj.as_string()
+
+
+def test_echobot(cmfactory, chatmail_config, lp):
+    ac = cmfactory.get_online_accounts(1)[0]
+
+    lp.sec(f"Send message to echo@{chatmail_config.mail_domain}")
+    chat = ac.create_chat(f"echo@{chatmail_config.mail_domain}")
+    text = "hi, I hope you text me back"
+    chat.send_text(text)
+    lp.sec("Wait for reply from echobot")
+    reply = ac.wait_next_incoming_message()
+    assert reply.text == text