tests: make maildata work with python3.9

I installed pyenv and then installed Python 3.9:
$ pyenv install 3.9
$ eval "$(pyenv init -)"
$ pyenv shell 3.9

In a clean repository I ran
$ scripts/cmdeploy init
$ scripts/cmdeploy run
$ scripts/cmdeploy dns
$ scripts/cmdeploy fmt

With the changes made all these commands work.

scripts/cmdeploy test fails some tests
using maildata fixture at
  importlib.resources.files(__package__).joinpath("mail-data")
line but this is not critical.

chatmaild/src/chatmaild/doveauth.py

@@ -160,6 +160,19 @@ def handle_dovecot_request(msg, db, config: Config):
     return None
 
 
+def handle_dovecot_protocol(rfile, wfile, db: Database, config: Config):
+    while True:
+        msg = rfile.readline().strip().decode()
+        if not msg:
+            break
+        res = handle_dovecot_request(msg, db, config)
+        if res:
+            wfile.write(res.encode("ascii"))
+            wfile.flush()
+        else:
+            logging.warning("request had no answer: %r", msg)
+
+
 class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
     request_queue_size = 100
 
@@ -173,16 +186,7 @@ def main():
     class Handler(StreamRequestHandler):
         def handle(self):
             try:
-                while True:
+                handle_dovecot_protocol(self.rfile, self.wfile, db, config)
-                    msg = self.rfile.readline().strip().decode()
-                    if not msg:
-                        break
-                    res = handle_dovecot_request(msg, db, config)
-                    if res:
-                        self.wfile.write(res.encode("ascii"))
-                        self.wfile.flush()
-                    else:
-                        logging.warn("request had no answer: %r", msg)
             except Exception:
                 logging.exception("Exception in the handler")
                 raise

chatmaild/src/chatmaild/tests/plugin.py

@@ -1,4 +1,6 @@
 import random
+from pathlib import Path
+import os
 import importlib.resources
 import itertools
 from email.parser import BytesParser
@@ -57,7 +59,12 @@ def db(tmpdir):
 
 @pytest.fixture
 def maildata(request):
-    datadir = importlib.resources.files(__package__).joinpath("mail-data")
+    try:
+        datadir = importlib.resources.files(__package__).joinpath("mail-data")
+    except TypeError:
+        # in python3.9 or lower, the above doesn't work, so we get datadir this way:
+        datadir = Path(os.getcwd()).joinpath("chatmaild/src/chatmaild/tests/mail-data")
+
     assert datadir.exists(), datadir
 
     def maildata(name, from_addr, to_addr):

chatmaild/src/chatmaild/tests/test_doveauth.py

@@ -1,11 +1,17 @@
+import io
 import json
 import pytest
-import threading
 import queue
+import threading
 import traceback
 
 import chatmaild.doveauth
-from chatmaild.doveauth import get_user_data, lookup_passdb, handle_dovecot_request
+from chatmaild.doveauth import (
+    get_user_data,
+    lookup_passdb,
+    handle_dovecot_request,
+    handle_dovecot_protocol,
+)
 from chatmaild.database import DBError
 
 
@@ -69,6 +75,15 @@ def test_handle_dovecot_request(db, example_config):
     assert userdata["password"].startswith("{SHA512-CRYPT}")
 
 
+def test_handle_dovecot_protocol(db, example_config):
+    rfile = io.BytesIO(
+        b"H3\t2\t0\t\tauth\nLshared/userdb/foobar@chat.example.org\tfoobar@chat.example.org\n"
+    )
+    wfile = io.BytesIO()
+    handle_dovecot_protocol(rfile, wfile, db, example_config)
+    assert wfile.getvalue() == b"N\n"
+
+
 def test_50_concurrent_lookups_different_accounts(db, gencreds, example_config):
     num_threads = 50
     req_per_thread = 5

cmdeploy/src/cmdeploy/__init__.py

@@ -1,6 +1,7 @@
 """
 Chat Mail pyinfra deploy.
 """
+
 import sys
 import importlib.resources
 import subprocess
@@ -303,9 +304,7 @@ def _configure_postfix(config: Config, debug: bool = False) -> bool:
 
     # Login map that 1:1 maps email address to login.
     login_map = files.put(
-        src=importlib.resources.files(__package__).joinpath(
+        src=importlib.resources.files(__package__).joinpath("postfix/login_map"),
-            "postfix/login_map"
-        ),
         dest="/etc/postfix/login_map",
         user="root",
         group="root",

cmdeploy/src/cmdeploy/chatmail.zone.f

@@ -11,6 +11,5 @@ _dmarc.{chatmail_domain}.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
 _mta-sts.{chatmail_domain}.          TXT "v=STSv1; id={sts_id}"
 mta-sts.{chatmail_domain}.           CNAME {chatmail_domain}.
 www.{chatmail_domain}.               CNAME {chatmail_domain}.
-_smtp._tls.{chatmail_domain}.        TXT "v=TLSRPTv1;rua=mailto:{email}"
 {dkim_entry}
 _adsp._domainkey.{chatmail_domain}.  TXT "dkim=discardable"

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -2,6 +2,7 @@
 Provides the `cmdeploy` entry point function,
 along with command line option and subcommand parsing.
 """
+
 import argparse
 import shutil
 import subprocess

cmdeploy/src/cmdeploy/dns.py

@@ -5,6 +5,8 @@ import importlib
 import subprocess
 import datetime
 
+from typing import Optional
+
 
 class DNS:
     def __init__(self, out, mail_domain):
@@ -34,7 +36,7 @@ class DNS:
         cmd = "ip a | grep inet6 | grep 'scope global' | sed -e 's#/64 scope global##' | sed -e 's#inet6##'"
         return self.shell(cmd).strip()
 
-    def get(self, typ: str, domain: str) -> str | None:
+    def get(self, typ: str, domain: str) -> Optional[str]:
         """Get a DNS entry"""
         dig_result = self.shell(f"dig -r -q {domain} -t {typ} +short")
         line = dig_result.partition("\n")[0]
@@ -82,7 +84,6 @@ def show_dns(args, out) -> int:
             f.read()
             .format(
                 acme_account_url=acme_account_url,
-                email=f"root@{args.config.mail_domain}",
                 sts_id=datetime.datetime.now().strftime("%Y%m%d%H%M"),
                 chatmail_domain=args.config.mail_domain,
                 dkim_entry=dkim_entry,
@@ -102,7 +103,6 @@ def show_dns(args, out) -> int:
         for line in zonefile.splitlines():
             line = line.format(
                 acme_account_url=acme_account_url,
-                email=f"root@{args.config.mail_domain}",
                 sts_id=datetime.datetime.now().strftime("%Y%m%d%H%M"),
                 chatmail_domain=args.config.mail_domain,
                 dkim_entry=dkim_entry,

cmdeploy/src/cmdeploy/dovecot/expunge.cron.j2

@@ -1,10 +1,10 @@
 # delete all mails after {{ config.delete_mails_after }} days, in the Inbox
-2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }}/*/cur -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 # or in any IMAP subfolder
-2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }}/*/.*/cur -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 # even if they are unseen
-2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }}/*/new -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }}/*/.*/new -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 # or only temporary (but then they shouldn't be around after {{ config.delete_mails_after }} days anyway).
-2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }}/*/tmp -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }}/*/.*/tmp -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -83,3 +83,18 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
             assert b"4.7.1: Too much mail from" in outcome[1]
             return
     pytest.fail("Rate limit was not exceeded")
+
+
+def test_expunged(remote, chatmail_config):
+    outdated_days = int(chatmail_config.delete_mails_after) + 1
+    find_cmds = [
+        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/cur/*' -mtime +{outdated_days} -type f",
+        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/.*/cur/*' -mtime +{outdated_days} -type f",
+        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/new/*' -mtime +{outdated_days} -type f",
+        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/.*/new/*' -mtime +{outdated_days} -type f",
+        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/tmp/*' -mtime +{outdated_days} -type f",
+        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/.*/tmp/*' -mtime +{outdated_days} -type f",
+    ]
+    for cmd in find_cmds:
+        for line in remote.iter_output(cmd):
+            assert not line

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -136,3 +136,15 @@ def test_hide_senders_ip_address(cmfactory):
     user2.direct_imap.select_folder("Inbox")
     msg = user2.direct_imap.get_all_messages()[0]
     assert public_ip not in msg.obj.as_string()
+
+
+def test_echobot(cmfactory, chatmail_config, lp):
+    ac = cmfactory.get_online_accounts(1)[0]
+
+    lp.sec(f"Send message to echo@{chatmail_config.mail_domain}")
+    chat = ac.create_chat(f"echo@{chatmail_config.mail_domain}")
+    text = "hi, I hope you text me back"
+    chat.send_text(text)
+    lp.sec("Wait for reply from echobot")
+    reply = ac.wait_next_incoming_message()
+    assert reply.text == text