fix bailout if there is no TXT entry

I installed pyenv and then installed Python 3.9:
$ pyenv install 3.9
$ eval "$(pyenv init -)"
$ pyenv shell 3.9

In a clean repository I ran
$ scripts/cmdeploy init
$ scripts/cmdeploy run
$ scripts/cmdeploy dns
$ scripts/cmdeploy fmt

With the changes made all these commands work.

scripts/cmdeploy test fails some tests
using maildata fixture at
  importlib.resources.files(__package__).joinpath("mail-data")
line but this is not critical.

chatmaild/src/chatmaild/doveauth.py

@@ -160,6 +160,19 @@ def handle_dovecot_request(msg, db, config: Config):
     return None
 
 
+def handle_dovecot_protocol(rfile, wfile, db: Database, config: Config):
+    while True:
+        msg = rfile.readline().strip().decode()
+        if not msg:
+            break
+        res = handle_dovecot_request(msg, db, config)
+        if res:
+            wfile.write(res.encode("ascii"))
+            wfile.flush()
+        else:
+            logging.warning("request had no answer: %r", msg)
+
+
 class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
     request_queue_size = 100
 
@@ -173,16 +186,7 @@ def main():
     class Handler(StreamRequestHandler):
         def handle(self):
             try:
-                while True:
+                handle_dovecot_protocol(self.rfile, self.wfile, db, config)
-                    msg = self.rfile.readline().strip().decode()
-                    if not msg:
-                        break
-                    res = handle_dovecot_request(msg, db, config)
-                    if res:
-                        self.wfile.write(res.encode("ascii"))
-                        self.wfile.flush()
-                    else:
-                        logging.warn("request had no answer: %r", msg)
             except Exception:
                 logging.exception("Exception in the handler")
                 raise

chatmaild/src/chatmaild/tests/plugin.py

@@ -1,4 +1,6 @@
 import random
+from pathlib import Path
+import os
 import importlib.resources
 import itertools
 from email.parser import BytesParser
@@ -57,7 +59,12 @@ def db(tmpdir):
 
 @pytest.fixture
 def maildata(request):
-    datadir = importlib.resources.files(__package__).joinpath("mail-data")
+    try:
+        datadir = importlib.resources.files(__package__).joinpath("mail-data")
+    except TypeError:
+        # in python3.9 or lower, the above doesn't work, so we get datadir this way:
+        datadir = Path(os.getcwd()).joinpath("chatmaild/src/chatmaild/tests/mail-data")
+
     assert datadir.exists(), datadir
 
     def maildata(name, from_addr, to_addr):

chatmaild/src/chatmaild/tests/test_doveauth.py

@@ -1,11 +1,17 @@
+import io
 import json
 import pytest
-import threading
 import queue
+import threading
 import traceback
 
 import chatmaild.doveauth
-from chatmaild.doveauth import get_user_data, lookup_passdb, handle_dovecot_request
+from chatmaild.doveauth import (
+    get_user_data,
+    lookup_passdb,
+    handle_dovecot_request,
+    handle_dovecot_protocol,
+)
 from chatmaild.database import DBError
 
 
@@ -69,6 +75,15 @@ def test_handle_dovecot_request(db, example_config):
     assert userdata["password"].startswith("{SHA512-CRYPT}")
 
 
+def test_handle_dovecot_protocol(db, example_config):
+    rfile = io.BytesIO(
+        b"H3\t2\t0\t\tauth\nLshared/userdb/foobar@chat.example.org\tfoobar@chat.example.org\n"
+    )
+    wfile = io.BytesIO()
+    handle_dovecot_protocol(rfile, wfile, db, example_config)
+    assert wfile.getvalue() == b"N\n"
+
+
 def test_50_concurrent_lookups_different_accounts(db, gencreds, example_config):
     num_threads = 50
     req_per_thread = 5

cmdeploy/src/cmdeploy/__init__.py

@@ -1,6 +1,7 @@
 """
 Chat Mail pyinfra deploy.
 """
+
 import sys
 import importlib.resources
 import subprocess
@@ -303,9 +304,7 @@ def _configure_postfix(config: Config, debug: bool = False) -> bool:
 
     # Login map that 1:1 maps email address to login.
     login_map = files.put(
-        src=importlib.resources.files(__package__).joinpath(
+        src=importlib.resources.files(__package__).joinpath("postfix/login_map"),
-            "postfix/login_map"
-        ),
         dest="/etc/postfix/login_map",
         user="root",
         group="root",

cmdeploy/src/cmdeploy/chatmail.zone.f

@@ -11,6 +11,5 @@ _dmarc.{chatmail_domain}.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
 _mta-sts.{chatmail_domain}.          TXT "v=STSv1; id={sts_id}"
 mta-sts.{chatmail_domain}.           CNAME {chatmail_domain}.
 www.{chatmail_domain}.               CNAME {chatmail_domain}.
-_smtp._tls.{chatmail_domain}.        TXT "v=TLSRPTv1;rua=mailto:{email}"
 {dkim_entry}
 _adsp._domainkey.{chatmail_domain}.  TXT "dkim=discardable"

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -2,6 +2,7 @@
 Provides the `cmdeploy` entry point function,
 along with command line option and subcommand parsing.
 """
+
 import argparse
 import shutil
 import subprocess

cmdeploy/src/cmdeploy/dns.py

@@ -5,6 +5,8 @@ import importlib
 import subprocess
 import datetime
 
+from typing import Optional
+
 
 class DNS:
     def __init__(self, out, mail_domain):
@@ -34,12 +36,11 @@ class DNS:
         cmd = "ip a | grep inet6 | grep 'scope global' | sed -e 's#/64 scope global##' | sed -e 's#inet6##'"
         return self.shell(cmd).strip()
 
-    def get(self, typ: str, domain: str) -> str | None:
+    def get(self, typ: str, domain: str) -> str:
-        """Get a DNS entry"""
+        """Get a DNS entry or empty string if there is none."""
         dig_result = self.shell(f"dig -r -q {domain} -t {typ} +short")
         line = dig_result.partition("\n")[0]
-        if line:
+        return line
-            return line
 
     def check_ptr_record(self, ip: str, mail_domain) -> bool:
         """Check the PTR record for an IPv4 or IPv6 address."""
@@ -54,22 +55,25 @@ def show_dns(args, out) -> int:
     ssh = f"ssh root@{mail_domain}"
     dns = DNS(out, mail_domain)
 
-    def read_dkim_entries(entry):
-        lines = []
-        for line in entry.split("\n"):
-            if line.startswith(";") or not line.strip():
-                continue
-            line = line.replace("\t", " ")
-            lines.append(line)
-        return "\n".join(lines)
-
     print("Checking your DKIM keys and DNS entries...")
     try:
         acme_account_url = out.shell_output(f"{ssh} -- acmetool account-url")
     except subprocess.CalledProcessError:
         print("Please run `cmdeploy run` first.")
         return 1
-    dkim_entry = read_dkim_entries(out.shell_output(f"{ssh} -- opendkim-genzone -F"))
+
+    dkim_selector = "opendkim"
+    dkim_pubkey = out.shell_output(
+        ssh + f" -- openssl rsa -in /etc/dkimkeys/{dkim_selector}.private"
+        " -pubout 2>/dev/null | awk '/-/{next}{printf(\"%s\",$0)}'"
+    )
+    dkim_entry_value = f"v=DKIM1;k=rsa;p={dkim_pubkey};s=email;t=s"
+    dkim_entry_str = ""
+    while len(dkim_entry_value) >= 255:
+        dkim_entry_str += '"' + dkim_entry_value[:255] + '" '
+        dkim_entry_value = dkim_entry_value[255:]
+    dkim_entry_str += '"' + dkim_entry_value + '"'
+    dkim_entry = f"{dkim_selector}._domainkey.{mail_domain}. TXT {dkim_entry_str}"
 
     ipv6 = dns.get_ipv6()
     reverse_ipv6 = dns.check_ptr_record(ipv6, mail_domain)
@@ -82,7 +86,6 @@ def show_dns(args, out) -> int:
             f.read()
             .format(
                 acme_account_url=acme_account_url,
-                email=f"root@{args.config.mail_domain}",
                 sts_id=datetime.datetime.now().strftime("%Y%m%d%H%M"),
                 chatmail_domain=args.config.mail_domain,
                 dkim_entry=dkim_entry,
@@ -98,11 +101,9 @@ def show_dns(args, out) -> int:
             return 0
         except TypeError:
             pass
-        started_dkim_parsing = False
         for line in zonefile.splitlines():
             line = line.format(
                 acme_account_url=acme_account_url,
-                email=f"root@{args.config.mail_domain}",
                 sts_id=datetime.datetime.now().strftime("%Y%m%d%H%M"),
                 chatmail_domain=args.config.mail_domain,
                 dkim_entry=dkim_entry,
@@ -126,28 +127,23 @@ def show_dns(args, out) -> int:
                 current = dns.get("SRV", domain[:-1])
                 if current != f"{prio} {weight} {port} {value}":
                     to_print.append(line)
-            if "  TXT " in line:
+            if " TXT " in line:
                 domain, value = line.split(" TXT ")
                 current = dns.get("TXT", domain.strip()[:-1])
                 if domain.startswith("_mta-sts."):
                     if current:
                         if current.split("id=")[0] == value.split("id=")[0]:
                             continue
-                if current != value:
+
+                # TXT records longer than 255 bytes
+                # are split into multiple <character-string>s.
+                # This typically happens with DKIM record
+                # which contains long RSA key.
+                #
+                # Removing `" "` before comparison
+                # to get back a single string.
+                if current.replace('" "', "") != value.replace('" "', ""):
                     to_print.append(line)
-            if "IN TXT ( " in line:
-                started_dkim_parsing = True
-                dkim_lines = [line]
-            if started_dkim_parsing and line.startswith('"'):
-                dkim_lines.append(" " + line)
-        domain, data = "\n".join(dkim_lines).split(" IN TXT ")
-        current = dns.get("TXT", domain.strip()[:-1])
-        if current:
-            current = "( %s )" % (current.replace('" "', '"\n "'))
-            if current.replace(";", "\\;") != data:
-                to_print.append(dkim_entry)
-        else:
-            to_print.append(dkim_entry)
 
     exit_code = 0
     if to_print:

cmdeploy/src/cmdeploy/dovecot/expunge.cron.j2

@@ -1,10 +1,10 @@
 # delete all mails after {{ config.delete_mails_after }} days, in the Inbox
-2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }}/*/cur -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 # or in any IMAP subfolder
-2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }}/*/.*/cur -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/cur/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 # even if they are unseen
-2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }}/*/new -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }}/*/.*/new -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/new/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 # or only temporary (but then they shouldn't be around after {{ config.delete_mails_after }} days anyway).
-2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }}/*/tmp -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
-2 0 * * * dovecot find /home/vmail/mail/{{ config.mail_domain }}/*/.*/tmp -mtime +{{ config.delete_mails_after }} -type f -delete
+2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -23,6 +23,31 @@ smtp_tls_CApath=/etc/ssl/certs
 smtp_tls_security_level=may
 smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
 smtp_tls_policy_maps = socketmap:inet:127.0.0.1:8461:postfix
+smtpd_tls_protocols = >=TLSv1.2
+
+# Disable anonymous cipher suites
+# and known insecure algorithms.
+#
+# Disabling anonymous ciphers
+# does not generally improve security
+# because clients that want to verify certificate
+# will not select them anyway,
+# but makes cipher suite list shorter and security scanners happy.
+# See <https://www.postfix.org/TLS_README.html> for discussion.
+#
+# Only ancient insecure ciphers should be disabled here
+# as MTA clients that do not support more secure cipher
+# likely do not support MTA-STS either and will
+# otherwise fall back to using plaintext connection.
+smtpd_tls_exclude_ciphers = aNULL, RC4, MD5, DES
+
+# Override client's preference order.
+# <https://www.postfix.org/postconf.5.html#tls_preempt_cipherlist>
+#
+# This is mostly to ensure cipher suites with forward secrecy
+# are preferred over non cipher suites without forward secrecy.
+# See <https://www.postfix.org/FORWARD_SECRECY_README.html#server_fs>.
+tls_preempt_cipherlist = yes
 
 smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination
 myhostname = {{ config.mail_domain }}

cmdeploy/src/cmdeploy/tests/online/test_0_login.py

@@ -1,6 +1,7 @@
 import pytest
 import threading
 import queue
+import socket
 
 from chatmaild.config import read_config
 from cmdeploy.cmdeploy import main
@@ -78,3 +79,24 @@ def test_concurrent_logins_same_account(
 
     for _ in conns:
         assert login_results.get()
+
+
+def test_no_vrfy(chatmail_config):
+    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    sock.connect((chatmail_config.mail_domain, 25))
+    banner = sock.recv(1024)
+    print(banner)
+    sock.send(b"VRFY wrongaddress@%s\r\n" % (chatmail_config.mail_domain.encode(),))
+    result = sock.recv(1024)
+    print(result)
+    sock.send(b"VRFY echo@%s\r\n" % (chatmail_config.mail_domain.encode(),))
+    result2 = sock.recv(1024)
+    print(result2)
+    assert result[0:10] == result2[0:10]
+    sock.send(b"VRFY wrongaddress\r\n")
+    result = sock.recv(1024)
+    print(result)
+    sock.send(b"VRFY echo\r\n")
+    result2 = sock.recv(1024)
+    print(result2)
+    assert result[0:10] == result2[0:10] == b"252 2.0.0 "

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -83,3 +83,18 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
             assert b"4.7.1: Too much mail from" in outcome[1]
             return
     pytest.fail("Rate limit was not exceeded")
+
+
+def test_expunged(remote, chatmail_config):
+    outdated_days = int(chatmail_config.delete_mails_after) + 1
+    find_cmds = [
+        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/cur/*' -mtime +{outdated_days} -type f",
+        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/.*/cur/*' -mtime +{outdated_days} -type f",
+        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/new/*' -mtime +{outdated_days} -type f",
+        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/.*/new/*' -mtime +{outdated_days} -type f",
+        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/tmp/*' -mtime +{outdated_days} -type f",
+        f"find /home/vmail/mail/{chatmail_config.mail_domain} -path '*/.*/tmp/*' -mtime +{outdated_days} -type f",
+    ]
+    for cmd in find_cmds:
+        for line in remote.iter_output(cmd):
+            assert not line

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -136,3 +136,15 @@ def test_hide_senders_ip_address(cmfactory):
     user2.direct_imap.select_folder("Inbox")
     msg = user2.direct_imap.get_all_messages()[0]
     assert public_ip not in msg.obj.as_string()
+
+
+def test_echobot(cmfactory, chatmail_config, lp):
+    ac = cmfactory.get_online_accounts(1)[0]
+
+    lp.sec(f"Send message to echo@{chatmail_config.mail_domain}")
+    chat = ac.create_chat(f"echo@{chatmail_config.mail_domain}")
+    text = "hi, I hope you text me back"
+    chat.send_text(text)
+    lp.sec("Wait for reply from echobot")
+    reply = ac.wait_next_incoming_message()
+    assert reply.text == text