various fixes

chatmail.ini

@@ -1,15 +1,19 @@
-[config]
 
+[params]
+max_user_send_per_minute = 60
+filtermail_smtp_port = 10080
+postfix_reinject_port = 10025
+
+[privacy:testrun]
+domain = *.testrun.org
 privacy_postal =
     Merlinux GmbH, Represented by the managing director H. Krekel,
     Reichgrafen Str. 20, 79102 Freiburg, Germany
 
 privacy_mail = delta-privacy@merlinux.eu
-
 privacy_pdo =
     Prof. Dr. Fabian Schmieder, lexICT UG (limited), Ostfeldstr. 49, 30559 Hannover.
     You can contact him at *delta-privacy@merlinux.eu* (Keyword: DPO)
-
 privacy_supervisor =
     State Commissioner for Data Protection and Freedom of Information of
     Baden-Württemberg in 70173 Stuttgart, Germany.

chatmaild/src/chatmaild/config.py

@@ -1,21 +1,19 @@
 
+
+from pathlib import Path
 from fnmatch import fnmatch
 import iniconfig
 
-
+system_mailname_path = Path("/etc/mailname")
-class Config:
-    def __init__(self, mailname, section):
-        self.mailname = mailname
-        self.privacy_postal = section.get("privacy_postal")
-        self.privacy_mail = section.get("privacy_mail")
-        self.privacy_pdo = section.get("privacy_pdo")
-        self.privacy_supervisor = section.get("privacy_supervisor")
-        self.has_privacy_policy = self.privacy_mail != None
 
 
-def read_config(inipath, mailname):
+def read_config(inipath, mailname=None):
+    if mailname is None:
+        with open(system_mailname_path) as f:
+            mailname = f.read().strip()
+
     ini = iniconfig.IniConfig(inipath)
-    privacy = None
+    privacy = {}
     for section in ini:
         if section.name.startswith("privacy:"):
             domain = section["domain"]
@@ -23,4 +21,20 @@ def read_config(inipath, mailname):
                 privacy = section
                 break
 
-    return Config(mailname, privacy or {})
+    return Config(inipath, mailname, privacy, params=ini.sections["params"])
+
+
+class Config:
+    def __init__(self, inipath, mailname, privacy, params):
+        self._inipath = inipath
+        self.mailname = mailname
+        self.privacy_postal = privacy.get("privacy_postal")
+        self.privacy_mail = privacy.get("privacy_mail")
+        self.privacy_pdo = privacy.get("privacy_pdo")
+        self.privacy_supervisor = privacy.get("privacy_supervisor")
+        self.max_user_send_per_minute = int(params["max_user_send_per_minute"])
+        self.filtermail_smtp_port = int(params["filtermail_smtp_port"])
+        self.postfix_reinject_port = int(params["postfix_reinject_port"])
+
+    def _getbytefile(self):
+        return open(self._inipath, "rb")

chatmaild/src/chatmaild/filtermail.py

@@ -11,6 +11,8 @@ from aiosmtpd.smtp import SMTP
 from aiosmtpd.controller import Controller
 from smtplib import SMTP as SMTPClient
 
+from .config import read_config
+
 
 def check_encrypted(message):
     """Check that the message is an OpenPGP-encrypted message."""
@@ -76,13 +78,15 @@ class SMTPController(Controller):
 
 
 class BeforeQueueHandler:
-    def __init__(self):
+    def __init__(self, config):
+        self.config = config
         self.send_rate_limiter = SendRateLimiter()
 
     async def handle_MAIL(self, server, session, envelope, address, mail_options):
         logging.info(f"handle_MAIL from {address}")
         envelope.mail_from = address
-        if not self.send_rate_limiter.is_sending_allowed(address):
+        max_sent = self.config.max_user_send_per_minute
+        if not self.send_rate_limiter.is_sending_allowed(address, max_sent):
             return f"450 4.7.1: Too much mail from {address}"
 
         parts = envelope.mail_from.split("@")
@@ -97,13 +101,14 @@ class BeforeQueueHandler:
         if error:
             return error
         logging.info("re-injecting the mail that passed checks")
-        client = SMTPClient("localhost", "10025")
+        client = SMTPClient("localhost", self.config.postfix_reinject_port)
         client.sendmail(envelope.mail_from, envelope.rcpt_tos, envelope.content)
         return "250 OK"
 
 
-async def asyncmain_beforequeue(port):
+async def asyncmain_beforequeue(config):
-    Controller(BeforeQueueHandler(), hostname="127.0.0.1", port=port).start()
+    port = config.filtermail_smtp_port
+    Controller(BeforeQueueHandler(config), hostname="127.0.0.1", port=port).start()
 
 
 def check_DATA(envelope):
@@ -142,16 +147,14 @@ def check_DATA(envelope):
 
 
 class SendRateLimiter:
-    MAX_USER_SEND_PER_MINUTE = 80
-
     def __init__(self):
         self.addr2timestamps = {}
 
-    def is_sending_allowed(self, mail_from):
+    def is_sending_allowed(self, mail_from, max_send_per_minute):
         last = self.addr2timestamps.setdefault(mail_from, [])
         now = time.time()
         last[:] = [ts for ts in last if ts >= (now - 60)]
-        if len(last) <= self.MAX_USER_SEND_PER_MINUTE:
+        if len(last) <= max_send_per_minute:
             last.append(now)
             return True
         return False
@@ -160,9 +163,10 @@ class SendRateLimiter:
 def main():
     args = sys.argv[1:]
     assert len(args) == 1
+    config = read_config(args[0])
     logging.basicConfig(level=logging.WARN)
     loop = asyncio.new_event_loop()
     asyncio.set_event_loop(loop)
-    task = asyncmain_beforequeue(port=int(args[0]))
+    task = asyncmain_beforequeue(config)
     loop.create_task(task)
     loop.run_forever()

chatmaild/src/chatmaild/filtermail.service.f

@@ -2,7 +2,7 @@
 Description=Chatmail Postfix BeforeQeue filter 
 
 [Service]
-ExecStart={execpath} 10080
+ExecStart={execpath} {config_path}
 Restart=always
 RestartSec=30
 

deploy-chatmail/src/deploy_chatmail/__init__.py

@@ -6,7 +6,6 @@ import importlib.resources
 import subprocess
 import shutil
 import io
-import configparser
 from pathlib import Path
 
 from pyinfra import host
@@ -15,6 +14,9 @@ from pyinfra.facts.files import File
 from pyinfra.facts.systemd import SystemdEnabled
 from .acmetool import deploy_acmetool
 
+import chatmaild.filtermail
+from chatmaild.config import read_config
+
 
 def _build_chatmaild(dist_dir) -> None:
     dist_dir = Path(dist_dir).resolve()
@@ -30,11 +32,24 @@ def _build_chatmaild(dist_dir) -> None:
     return entries[0]
 
 
-def _install_remote_venv_with_chatmaild() -> None:
+def remove_legacy_artifacts():
+    # disable legacy doveauth-dictproxy.service
+    if host.get_fact(SystemdEnabled).get("doveauth-dictproxy.service"):
+        systemd.service(
+            name="Disable legacy doveauth-dictproxy.service",
+            service="doveauth-dictproxy.service",
+            running=False,
+            enabled=False,
+        )
+
+
+def _install_remote_venv_with_chatmaild(config) -> None:
+    remove_legacy_artifacts()
     dist_file = _build_chatmaild(dist_dir=Path("chatmaild/dist"))
     remote_base_dir = "/usr/local/lib/chatmaild"
     remote_dist_file = f"{remote_base_dir}/dist/{dist_file.name}"
     remote_venv_dir = f"{remote_base_dir}/venv"
+    remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini"
     root_owned = dict(user="root", group="root", mode="644")
 
     apt.packages(
@@ -50,6 +65,13 @@ def _install_remote_venv_with_chatmaild() -> None:
         **root_owned,
     )
 
+    files.put(
+        name=f"Upload {remote_chatmail_inipath}",
+        src=config._getbytefile(),
+        dest=remote_chatmail_inipath,
+        **root_owned,
+    )
+
     pip.virtualenv(
         name=f"chatmaild virtualenv {remote_venv_dir}",
         path=remote_venv_dir,
@@ -63,24 +85,17 @@ def _install_remote_venv_with_chatmaild() -> None:
         ],
     )
 
-    # disable legacy doveauth-dictproxy.service
-    if host.get_fact(SystemdEnabled).get("doveauth-dictproxy.service"):
-        systemd.service(
-            name="Disable legacy doveauth-dictproxy.service",
-            service="doveauth-dictproxy.service",
-            running=False,
-            enabled=False,
-        )
-
     # install systemd units
-
     for fn in (
         "doveauth",
         "filtermail",
     ):
-        execpath = f"{remote_venv_dir}/bin/{fn}"
+        params = dict(
+            execpath=f"{remote_venv_dir}/bin/{fn}",
+            config_path=remote_chatmail_inipath,
+        )
         source_path = importlib.resources.files("chatmaild").joinpath(f"{fn}.service.f")
-        content = source_path.read_text().format(execpath=execpath).encode()
+        content = source_path.read_text().format(**params).encode()
 
         files.put(
             name=f"Upload {fn}.service",
@@ -201,7 +216,7 @@ def _install_mta_sts_daemon() -> bool:
     return need_restart
 
 
-def _configure_postfix(domain: str, debug: bool = False) -> bool:
+def _configure_postfix(config: chatmaild.config.Config, debug: bool = False) -> bool:
     """Configures Postfix SMTP server."""
     need_restart = False
 
@@ -211,7 +226,7 @@ def _configure_postfix(domain: str, debug: bool = False) -> bool:
         user="root",
         group="root",
         mode="644",
-        config={"domain_name": domain},
+        config=config,
     )
     need_restart |= main_config.changed
 
@@ -222,6 +237,7 @@ def _configure_postfix(domain: str, debug: bool = False) -> bool:
         group="root",
         mode="644",
         debug=debug,
+        config=config,
     )
     need_restart |= master_config.changed
 
@@ -331,19 +347,16 @@ def _configure_nginx(domain: str, debug: bool = False) -> bool:
     return need_restart
 
 
-def get_ini_settings(mail_domain, inipath):
+def check_config(config):
-    parser = configparser.ConfigParser()
+    mailname = config.mailname
-    parser.read(inipath)
+    if mailname != "testrun.org" and not mailname.endswith(".testrun.org"):
-    settings = {key: value.strip() for (key, value) in parser["config"].items()}
+        blocked_words = "merlinux schmieder testrun.org".split()
-    if mail_domain != "testrun.org" and not mail_domain.endswith(".testrun.org"):
+        for value in config.__dict__.values():
-        for value in settings.values():
+            if any(x in value for x in blocked_words):
-            value = value.lower()
-            if "merlinux" in value or "schmieder" in value or "@testrun.org" in value:
                 raise ValueError(
-                    f"please set your own privacy contacts/addresses in {inipath}"
+                    f"please set your own privacy contacts/addresses in {config._inipath}"
                 )
-    settings["mail_domain"] = mail_domain
+    return config
-    return settings
 
 
 def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> None:
@@ -400,7 +413,8 @@ def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> N
 
     pkg_root = importlib.resources.files(__package__)
     chatmail_ini = pkg_root.joinpath("../../../chatmail.ini").resolve()
-    config = get_ini_settings(mail_domain, chatmail_ini)
+    config = read_config(chatmail_ini, mailname=mail_domain)
+    check_config(config)
     www_path = pkg_root.joinpath("../../../www").resolve()
 
     build_dir = www_path.joinpath("build")
@@ -408,10 +422,10 @@ def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> N
     build_webpages(src_dir, build_dir, config)
     files.rsync(f"{build_dir}/", "/var/www/html", flags=["-avz"])
 
-    _install_remote_venv_with_chatmaild()
+    _install_remote_venv_with_chatmaild(config)
     debug = False
     dovecot_need_restart = _configure_dovecot(mail_server, debug=debug)
-    postfix_need_restart = _configure_postfix(mail_domain, debug=debug)
+    postfix_need_restart = _configure_postfix(config, debug=debug)
     opendkim_need_restart = _configure_opendkim(mail_domain, dkim_selector)
     mta_sts_need_restart = _install_mta_sts_daemon()
     nginx_need_restart = _configure_nginx(mail_domain)

deploy-chatmail/src/deploy_chatmail/postfix/main.cf.j2

@@ -1,4 +1,4 @@
-myorigin = {{ config.domain_name }}
+myorigin = {{ config.mailname }}
 
 smtpd_banner = $myhostname ESMTP $mail_name (Debian/GNU)
 biff = no
@@ -16,8 +16,8 @@ readme_directory = no
 compatibility_level = 2
 
 # TLS parameters
-smtpd_tls_cert_file=/var/lib/acme/live/{{ config.domain_name }}/fullchain
+smtpd_tls_cert_file=/var/lib/acme/live/{{ config.mailname }}/fullchain
-smtpd_tls_key_file=/var/lib/acme/live/{{ config.domain_name }}/privkey
+smtpd_tls_key_file=/var/lib/acme/live/{{ config.mailname }}/privkey
 smtpd_tls_security_level=may
 
 smtp_tls_CApath=/etc/ssl/certs
@@ -26,7 +26,7 @@ smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
 smtp_tls_policy_maps = socketmap:inet:127.0.0.1:8461:postfix
 
 smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination
-myhostname = {{ config.domain_name }}
+myhostname = {{ config.mailname }}
 alias_maps = hash:/etc/aliases
 alias_database = hash:/etc/aliases
 
@@ -45,7 +45,7 @@ inet_interfaces = all
 inet_protocols = all
 
 virtual_transport = lmtp:unix:private/dovecot-lmtp
-virtual_mailbox_domains = {{ config.domain_name }}
+virtual_mailbox_domains = {{ config.mailname }}
 
 smtpd_milters = unix:opendkim/opendkim.sock
 non_smtpd_milters = $smtpd_milters

deploy-chatmail/src/deploy_chatmail/postfix/master.cf.j2

@@ -33,7 +33,7 @@ submission inet n       -       y       -       -       smtpd
   -o smtpd_relay_restrictions=permit_sasl_authenticated,reject
   -o milter_macro_daemon_name=ORIGINATING
   -o smtpd_client_connection_count_limit=1000
-  -o smtpd_proxy_filter=127.0.0.1:10080
+  -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
 smtps     inet  n       -       y       -       -       smtpd
   -o syslog_name=postfix/smtps
   -o smtpd_tls_wrappermode=yes
@@ -49,7 +49,7 @@ smtps     inet  n       -       y       -       -       smtpd
   -o smtpd_relay_restrictions=permit_sasl_authenticated,reject
   -o smtpd_client_connection_count_limit=1000
   -o milter_macro_daemon_name=ORIGINATING
-  -o smtpd_proxy_filter=127.0.0.1:10080
+  -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
 #628       inet  n       -       y       -       -       qmqpd
 pickup    unix  n       -       y       60      1       pickup
 cleanup   unix  n       -       y       -       0       cleanup
@@ -78,5 +78,5 @@ scache    unix  -       -       y       -       1       scache
 postlog   unix-dgram n  -       n       -       1       postlogd
 filter    unix -        n       n       -       -       lmtp
 # Local SMTP server for reinjecting filered mail.
-localhost:10025 inet  n       -       n       -       10      smtpd
+localhost:{{ config.postfix_reinject_port }} inet  n       -       n       -       10      smtpd
   -o syslog_name=postfix/reinject

deploy-chatmail/src/deploy_chatmail/www.py

@@ -7,7 +7,7 @@ import traceback
 import markdown
 from jinja2 import Template
 from .genqr import gen_qr_png_data
-from deploy_chatmail import get_ini_settings
+from chatmaild.config import read_config
 
 
 def snapshot_dir_stats(somedir):
@@ -37,7 +37,7 @@ def build_webpages(src_dir, build_dir, config):
 
 
 def _build_webpages(src_dir, build_dir, config):
-    mail_domain = config["mail_domain"]
+    mail_domain = config.mailname
     assert src_dir.exists(), src_dir
     if not build_dir.exists():
         build_dir.mkdir()
@@ -70,7 +70,7 @@ def main():
     path = importlib.resources.files(__package__)
     reporoot = path.joinpath("../../../").resolve()
     inipath = reporoot.joinpath("chatmail.ini")
-    config = get_ini_settings(chatmail_domain, inipath)
+    config = read_config(inipath, mailname=chatmail_domain)
     config["webdev"] = True
     www_path = reporoot.joinpath("www")
     src_path = www_path.joinpath("src")

tests/chatmaild/test_config.py

@@ -1,21 +1,55 @@
 from chatmaild.config import read_config
+import chatmaild.config
 
 
-def test_read_config_no_privacy_policy(tmp_path, create_ini):
+def test_read_config_without_mailname(tmp_path, create_ini, monkeypatch):
+    mailname_path = tmp_path.joinpath("mailname")
+    mailname_path.write_text("something.example.org")
+    monkeypatch.setattr(chatmaild.config, "system_mailname_path", mailname_path)
+
     inipath = create_ini(
         """
+        [params]
+        max_user_send_per_minute = 40
+        filtermail_smtp_port = 9875
+        postfix_reinject_port = 9999
+    """
+    )
+    config = read_config(inipath)
+    assert config.mailname == "something.example.org"
+
+
+def test_read_config_without_privacy_policy(tmp_path, create_ini):
+    inipath = create_ini(
+        """
+        [params]
+        max_user_send_per_minute = 40
+        filtermail_smtp_port = 9875
+        postfix_reinject_port = 9999
+
         [privacy:testrun]
         domain = *.example.org
     """
     )
     config = read_config(inipath, "something.example.org")
     assert config.mailname == "something.example.org"
-    assert not config.has_privacy_policy
+    assert config.max_user_send_per_minute == 40
+    assert config.filtermail_smtp_port == 9875
+    assert config.postfix_reinject_port == 9999
+    assert not config.privacy_postal
+    assert not config.privacy_mail
+    assert not config.privacy_pdo
+    assert not config.privacy_supervisor
 
 
 def test_read_config(create_ini):
     inipath = create_ini(
         """
+        [params]
+        max_user_send_per_minute = 40
+        filtermail_smtp_port = 10080
+        postfix_reinject_port = 10025
+
         [privacy:testrun]
         domain = *.testrun.org
 
@@ -35,9 +69,10 @@ def test_read_config(create_ini):
     )
 
     config = read_config(inipath, "something.testrun.org")
-    assert config.has_privacy_policy
 
     assert config.mailname == "something.testrun.org"
+    assert config.filtermail_smtp_port == 10080
+    assert config.postfix_reinject_port == 10025
     assert config.privacy_postal == "Postal Ltd"
     assert config.privacy_mail == "privacy@merlinux.eu"
     lines = config.privacy_pdo.split("\n")

tests/chatmaild/test_doveauth.py

@@ -1,5 +1,4 @@
 import json
-import sys
 import pytest
 import threading
 import queue
@@ -7,7 +6,7 @@ import traceback
 
 import chatmaild.doveauth
 from chatmaild.doveauth import get_user_data, lookup_passdb, handle_dovecot_request
-from chatmaild.database import Database, DBError
+from chatmaild.database import DBError
 
 
 def test_basic(db):

tests/chatmaild/test_filtermail.py

@@ -1,4 +1,10 @@
-from chatmaild.filtermail import check_encrypted, check_DATA, SendRateLimiter, check_mdn, is_passthrough_recipient
+from chatmaild.filtermail import (
+    check_encrypted,
+    check_DATA,
+    SendRateLimiter,
+    check_mdn,
+    is_passthrough_recipient,
+)
 import pytest
 
 
@@ -73,12 +79,12 @@ def test_filtermail_to_multiple_recipients_no_mdn(maildata, gencreds):
 def test_send_rate_limiter():
     limiter = SendRateLimiter()
     for i in range(100):
-        if limiter.is_sending_allowed("some@example.org"):
+        if limiter.is_sending_allowed("some@example.org", 10):
-            if i <= SendRateLimiter.MAX_USER_SEND_PER_MINUTE:
+            if i <= 10:
                 continue
             pytest.fail("limiter didn't work")
         else:
-            assert i == SendRateLimiter.MAX_USER_SEND_PER_MINUTE + 1
+            assert i == 11
             break
 
 

tests/online/test_1_basic.py

@@ -54,7 +54,7 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata):
         try:
             user1.smtp.sendmail(user1.addr, [user2.addr], mail)
         except smtplib.SMTPException as e:
-            if i < 80:
+            if i < 60:
                 pytest.fail(f"rate limit was exceeded too early with msg {i}")
             outcome = e.recipients[user2.addr]
             assert outcome[0] == 450

tests/test_helpers.py

@@ -2,20 +2,25 @@ import textwrap
 import importlib.resources
 
 from deploy_chatmail.www import build_webpages
-from deploy_chatmail import get_ini_settings
+from chatmaild.config import read_config
 
 
-def create_ini(inipath):
+def create_ini(inipath, domain="example.org"):
     inipath.write_text(
         textwrap.dedent(
-            """\
+            f"""\
-        [config]
+        [params]
+        max_user_send_per_minute = 60
+        filtermail_smtp_port = 10080
+        postfix_reinject_port = 10025
 
+        [privacy:{domain}]
+        domain = example.org
         privacy_postal =
             address-line1
             address-line2
 
-        privacy_mail = privacy@example.org
+        privacy_mail = privacy@{domain}
 
         privacy_pdo =
             address-line3
@@ -30,18 +35,18 @@ def test_build_webpages(tmp_path):
     assert src_dir.exists(), src_dir
 
     inipath = tmp_path.joinpath("chatmail.ini")
-    create_ini(inipath)
+    create_ini(inipath, "example.org")
-    config = get_ini_settings("example.org", inipath)
+    config = read_config(inipath, "example.org")
     build_dir = tmp_path.joinpath("build")
     build_webpages(src_dir, build_dir, config)
 
 
 def test_get_settings(tmp_path):
     inipath = tmp_path.joinpath("chatmail.ini")
-    create_ini(inipath)
+    create_ini(inipath, "example.org")
 
-    d = get_ini_settings("x.testrun.org", inipath)
+    config = read_config(inipath, "example.org")
-    assert d["privacy_postal"] == "address-line1\naddress-line2"
+    assert config.privacy_postal == "address-line1\naddress-line2"
-    assert d["privacy_mail"] == "privacy@example.org"
+    assert config.privacy_mail == "privacy@example.org"
-    assert d["privacy_pdo"] == "address-line3"
+    assert config.privacy_pdo == "address-line3"
-    assert d["mail_domain"] == "x.testrun.org"
+    assert config.mailname == "example.org"