From b52a8c969f1acb7f9cc0afc879c92180dab00851 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Sat, 9 Dec 2023 00:18:03 +0100 Subject: [PATCH] various fixes --- chatmail.ini | 10 ++- chatmaild/src/chatmaild/config.py | 38 +++++++--- chatmaild/src/chatmaild/filtermail.py | 24 +++--- chatmaild/src/chatmaild/filtermail.service.f | 2 +- .../src/deploy_chatmail/__init__.py | 74 +++++++++++-------- .../src/deploy_chatmail/postfix/main.cf.j2 | 10 +-- .../src/deploy_chatmail/postfix/master.cf.j2 | 6 +- deploy-chatmail/src/deploy_chatmail/www.py | 6 +- tests/chatmaild/test_config.py | 41 +++++++++- tests/chatmaild/test_doveauth.py | 3 +- tests/chatmaild/test_filtermail.py | 14 +++- tests/online/test_1_basic.py | 2 +- tests/test_helpers.py | 31 ++++---- 13 files changed, 171 insertions(+), 90 deletions(-) diff --git a/chatmail.ini b/chatmail.ini index 2d9dee5f..4d7008a3 100644 --- a/chatmail.ini +++ b/chatmail.ini @@ -1,15 +1,19 @@ -[config] +[params] +max_user_send_per_minute = 60 +filtermail_smtp_port = 10080 +postfix_reinject_port = 10025 + +[privacy:testrun] +domain = *.testrun.org privacy_postal = Merlinux GmbH, Represented by the managing director H. Krekel, Reichgrafen Str. 20, 79102 Freiburg, Germany privacy_mail = delta-privacy@merlinux.eu - privacy_pdo = Prof. Dr. Fabian Schmieder, lexICT UG (limited), Ostfeldstr. 49, 30559 Hannover. You can contact him at *delta-privacy@merlinux.eu* (Keyword: DPO) - privacy_supervisor = State Commissioner for Data Protection and Freedom of Information of Baden-Württemberg in 70173 Stuttgart, Germany. diff --git a/chatmaild/src/chatmaild/config.py b/chatmaild/src/chatmaild/config.py index 1668715d..ba5548af 100644 --- a/chatmaild/src/chatmaild/config.py +++ b/chatmaild/src/chatmaild/config.py @@ -1,21 +1,19 @@ + +from pathlib import Path from fnmatch import fnmatch import iniconfig - -class Config: - def __init__(self, mailname, section): - self.mailname = mailname - self.privacy_postal = section.get("privacy_postal") - self.privacy_mail = section.get("privacy_mail") - self.privacy_pdo = section.get("privacy_pdo") - self.privacy_supervisor = section.get("privacy_supervisor") - self.has_privacy_policy = self.privacy_mail != None +system_mailname_path = Path("/etc/mailname") -def read_config(inipath, mailname): +def read_config(inipath, mailname=None): + if mailname is None: + with open(system_mailname_path) as f: + mailname = f.read().strip() + ini = iniconfig.IniConfig(inipath) - privacy = None + privacy = {} for section in ini: if section.name.startswith("privacy:"): domain = section["domain"] @@ -23,4 +21,20 @@ def read_config(inipath, mailname): privacy = section break - return Config(mailname, privacy or {}) + return Config(inipath, mailname, privacy, params=ini.sections["params"]) + + +class Config: + def __init__(self, inipath, mailname, privacy, params): + self._inipath = inipath + self.mailname = mailname + self.privacy_postal = privacy.get("privacy_postal") + self.privacy_mail = privacy.get("privacy_mail") + self.privacy_pdo = privacy.get("privacy_pdo") + self.privacy_supervisor = privacy.get("privacy_supervisor") + self.max_user_send_per_minute = int(params["max_user_send_per_minute"]) + self.filtermail_smtp_port = int(params["filtermail_smtp_port"]) + self.postfix_reinject_port = int(params["postfix_reinject_port"]) + + def _getbytefile(self): + return open(self._inipath, "rb") diff --git a/chatmaild/src/chatmaild/filtermail.py b/chatmaild/src/chatmaild/filtermail.py index 1bc23a52..a97a9b72 100644 --- a/chatmaild/src/chatmaild/filtermail.py +++ b/chatmaild/src/chatmaild/filtermail.py @@ -11,6 +11,8 @@ from aiosmtpd.smtp import SMTP from aiosmtpd.controller import Controller from smtplib import SMTP as SMTPClient +from .config import read_config + def check_encrypted(message): """Check that the message is an OpenPGP-encrypted message.""" @@ -76,13 +78,15 @@ class SMTPController(Controller): class BeforeQueueHandler: - def __init__(self): + def __init__(self, config): + self.config = config self.send_rate_limiter = SendRateLimiter() async def handle_MAIL(self, server, session, envelope, address, mail_options): logging.info(f"handle_MAIL from {address}") envelope.mail_from = address - if not self.send_rate_limiter.is_sending_allowed(address): + max_sent = self.config.max_user_send_per_minute + if not self.send_rate_limiter.is_sending_allowed(address, max_sent): return f"450 4.7.1: Too much mail from {address}" parts = envelope.mail_from.split("@") @@ -97,13 +101,14 @@ class BeforeQueueHandler: if error: return error logging.info("re-injecting the mail that passed checks") - client = SMTPClient("localhost", "10025") + client = SMTPClient("localhost", self.config.postfix_reinject_port) client.sendmail(envelope.mail_from, envelope.rcpt_tos, envelope.content) return "250 OK" -async def asyncmain_beforequeue(port): - Controller(BeforeQueueHandler(), hostname="127.0.0.1", port=port).start() +async def asyncmain_beforequeue(config): + port = config.filtermail_smtp_port + Controller(BeforeQueueHandler(config), hostname="127.0.0.1", port=port).start() def check_DATA(envelope): @@ -142,16 +147,14 @@ def check_DATA(envelope): class SendRateLimiter: - MAX_USER_SEND_PER_MINUTE = 80 - def __init__(self): self.addr2timestamps = {} - def is_sending_allowed(self, mail_from): + def is_sending_allowed(self, mail_from, max_send_per_minute): last = self.addr2timestamps.setdefault(mail_from, []) now = time.time() last[:] = [ts for ts in last if ts >= (now - 60)] - if len(last) <= self.MAX_USER_SEND_PER_MINUTE: + if len(last) <= max_send_per_minute: last.append(now) return True return False @@ -160,9 +163,10 @@ class SendRateLimiter: def main(): args = sys.argv[1:] assert len(args) == 1 + config = read_config(args[0]) logging.basicConfig(level=logging.WARN) loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - task = asyncmain_beforequeue(port=int(args[0])) + task = asyncmain_beforequeue(config) loop.create_task(task) loop.run_forever() diff --git a/chatmaild/src/chatmaild/filtermail.service.f b/chatmaild/src/chatmaild/filtermail.service.f index dd9d1290..4bc35d23 100644 --- a/chatmaild/src/chatmaild/filtermail.service.f +++ b/chatmaild/src/chatmaild/filtermail.service.f @@ -2,7 +2,7 @@ Description=Chatmail Postfix BeforeQeue filter [Service] -ExecStart={execpath} 10080 +ExecStart={execpath} {config_path} Restart=always RestartSec=30 diff --git a/deploy-chatmail/src/deploy_chatmail/__init__.py b/deploy-chatmail/src/deploy_chatmail/__init__.py index ab42a275..2541c9fb 100644 --- a/deploy-chatmail/src/deploy_chatmail/__init__.py +++ b/deploy-chatmail/src/deploy_chatmail/__init__.py @@ -6,7 +6,6 @@ import importlib.resources import subprocess import shutil import io -import configparser from pathlib import Path from pyinfra import host @@ -15,6 +14,9 @@ from pyinfra.facts.files import File from pyinfra.facts.systemd import SystemdEnabled from .acmetool import deploy_acmetool +import chatmaild.filtermail +from chatmaild.config import read_config + def _build_chatmaild(dist_dir) -> None: dist_dir = Path(dist_dir).resolve() @@ -30,11 +32,24 @@ def _build_chatmaild(dist_dir) -> None: return entries[0] -def _install_remote_venv_with_chatmaild() -> None: +def remove_legacy_artifacts(): + # disable legacy doveauth-dictproxy.service + if host.get_fact(SystemdEnabled).get("doveauth-dictproxy.service"): + systemd.service( + name="Disable legacy doveauth-dictproxy.service", + service="doveauth-dictproxy.service", + running=False, + enabled=False, + ) + + +def _install_remote_venv_with_chatmaild(config) -> None: + remove_legacy_artifacts() dist_file = _build_chatmaild(dist_dir=Path("chatmaild/dist")) remote_base_dir = "/usr/local/lib/chatmaild" remote_dist_file = f"{remote_base_dir}/dist/{dist_file.name}" remote_venv_dir = f"{remote_base_dir}/venv" + remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini" root_owned = dict(user="root", group="root", mode="644") apt.packages( @@ -50,6 +65,13 @@ def _install_remote_venv_with_chatmaild() -> None: **root_owned, ) + files.put( + name=f"Upload {remote_chatmail_inipath}", + src=config._getbytefile(), + dest=remote_chatmail_inipath, + **root_owned, + ) + pip.virtualenv( name=f"chatmaild virtualenv {remote_venv_dir}", path=remote_venv_dir, @@ -63,24 +85,17 @@ def _install_remote_venv_with_chatmaild() -> None: ], ) - # disable legacy doveauth-dictproxy.service - if host.get_fact(SystemdEnabled).get("doveauth-dictproxy.service"): - systemd.service( - name="Disable legacy doveauth-dictproxy.service", - service="doveauth-dictproxy.service", - running=False, - enabled=False, - ) - # install systemd units - for fn in ( "doveauth", "filtermail", ): - execpath = f"{remote_venv_dir}/bin/{fn}" + params = dict( + execpath=f"{remote_venv_dir}/bin/{fn}", + config_path=remote_chatmail_inipath, + ) source_path = importlib.resources.files("chatmaild").joinpath(f"{fn}.service.f") - content = source_path.read_text().format(execpath=execpath).encode() + content = source_path.read_text().format(**params).encode() files.put( name=f"Upload {fn}.service", @@ -201,7 +216,7 @@ def _install_mta_sts_daemon() -> bool: return need_restart -def _configure_postfix(domain: str, debug: bool = False) -> bool: +def _configure_postfix(config: chatmaild.config.Config, debug: bool = False) -> bool: """Configures Postfix SMTP server.""" need_restart = False @@ -211,7 +226,7 @@ def _configure_postfix(domain: str, debug: bool = False) -> bool: user="root", group="root", mode="644", - config={"domain_name": domain}, + config=config, ) need_restart |= main_config.changed @@ -222,6 +237,7 @@ def _configure_postfix(domain: str, debug: bool = False) -> bool: group="root", mode="644", debug=debug, + config=config, ) need_restart |= master_config.changed @@ -331,19 +347,16 @@ def _configure_nginx(domain: str, debug: bool = False) -> bool: return need_restart -def get_ini_settings(mail_domain, inipath): - parser = configparser.ConfigParser() - parser.read(inipath) - settings = {key: value.strip() for (key, value) in parser["config"].items()} - if mail_domain != "testrun.org" and not mail_domain.endswith(".testrun.org"): - for value in settings.values(): - value = value.lower() - if "merlinux" in value or "schmieder" in value or "@testrun.org" in value: +def check_config(config): + mailname = config.mailname + if mailname != "testrun.org" and not mailname.endswith(".testrun.org"): + blocked_words = "merlinux schmieder testrun.org".split() + for value in config.__dict__.values(): + if any(x in value for x in blocked_words): raise ValueError( - f"please set your own privacy contacts/addresses in {inipath}" + f"please set your own privacy contacts/addresses in {config._inipath}" ) - settings["mail_domain"] = mail_domain - return settings + return config def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> None: @@ -400,7 +413,8 @@ def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> N pkg_root = importlib.resources.files(__package__) chatmail_ini = pkg_root.joinpath("../../../chatmail.ini").resolve() - config = get_ini_settings(mail_domain, chatmail_ini) + config = read_config(chatmail_ini, mailname=mail_domain) + check_config(config) www_path = pkg_root.joinpath("../../../www").resolve() build_dir = www_path.joinpath("build") @@ -408,10 +422,10 @@ def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> N build_webpages(src_dir, build_dir, config) files.rsync(f"{build_dir}/", "/var/www/html", flags=["-avz"]) - _install_remote_venv_with_chatmaild() + _install_remote_venv_with_chatmaild(config) debug = False dovecot_need_restart = _configure_dovecot(mail_server, debug=debug) - postfix_need_restart = _configure_postfix(mail_domain, debug=debug) + postfix_need_restart = _configure_postfix(config, debug=debug) opendkim_need_restart = _configure_opendkim(mail_domain, dkim_selector) mta_sts_need_restart = _install_mta_sts_daemon() nginx_need_restart = _configure_nginx(mail_domain) diff --git a/deploy-chatmail/src/deploy_chatmail/postfix/main.cf.j2 b/deploy-chatmail/src/deploy_chatmail/postfix/main.cf.j2 index b9f2496f..8ca20acd 100644 --- a/deploy-chatmail/src/deploy_chatmail/postfix/main.cf.j2 +++ b/deploy-chatmail/src/deploy_chatmail/postfix/main.cf.j2 @@ -1,4 +1,4 @@ -myorigin = {{ config.domain_name }} +myorigin = {{ config.mailname }} smtpd_banner = $myhostname ESMTP $mail_name (Debian/GNU) biff = no @@ -16,8 +16,8 @@ readme_directory = no compatibility_level = 2 # TLS parameters -smtpd_tls_cert_file=/var/lib/acme/live/{{ config.domain_name }}/fullchain -smtpd_tls_key_file=/var/lib/acme/live/{{ config.domain_name }}/privkey +smtpd_tls_cert_file=/var/lib/acme/live/{{ config.mailname }}/fullchain +smtpd_tls_key_file=/var/lib/acme/live/{{ config.mailname }}/privkey smtpd_tls_security_level=may smtp_tls_CApath=/etc/ssl/certs @@ -26,7 +26,7 @@ smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache smtp_tls_policy_maps = socketmap:inet:127.0.0.1:8461:postfix smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination -myhostname = {{ config.domain_name }} +myhostname = {{ config.mailname }} alias_maps = hash:/etc/aliases alias_database = hash:/etc/aliases @@ -45,7 +45,7 @@ inet_interfaces = all inet_protocols = all virtual_transport = lmtp:unix:private/dovecot-lmtp -virtual_mailbox_domains = {{ config.domain_name }} +virtual_mailbox_domains = {{ config.mailname }} smtpd_milters = unix:opendkim/opendkim.sock non_smtpd_milters = $smtpd_milters diff --git a/deploy-chatmail/src/deploy_chatmail/postfix/master.cf.j2 b/deploy-chatmail/src/deploy_chatmail/postfix/master.cf.j2 index 1a57bfed..dd0cc0eb 100644 --- a/deploy-chatmail/src/deploy_chatmail/postfix/master.cf.j2 +++ b/deploy-chatmail/src/deploy_chatmail/postfix/master.cf.j2 @@ -33,7 +33,7 @@ submission inet n - y - - smtpd -o smtpd_relay_restrictions=permit_sasl_authenticated,reject -o milter_macro_daemon_name=ORIGINATING -o smtpd_client_connection_count_limit=1000 - -o smtpd_proxy_filter=127.0.0.1:10080 + -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }} smtps inet n - y - - smtpd -o syslog_name=postfix/smtps -o smtpd_tls_wrappermode=yes @@ -49,7 +49,7 @@ smtps inet n - y - - smtpd -o smtpd_relay_restrictions=permit_sasl_authenticated,reject -o smtpd_client_connection_count_limit=1000 -o milter_macro_daemon_name=ORIGINATING - -o smtpd_proxy_filter=127.0.0.1:10080 + -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }} #628 inet n - y - - qmqpd pickup unix n - y 60 1 pickup cleanup unix n - y - 0 cleanup @@ -78,5 +78,5 @@ scache unix - - y - 1 scache postlog unix-dgram n - n - 1 postlogd filter unix - n n - - lmtp # Local SMTP server for reinjecting filered mail. -localhost:10025 inet n - n - 10 smtpd +localhost:{{ config.postfix_reinject_port }} inet n - n - 10 smtpd -o syslog_name=postfix/reinject diff --git a/deploy-chatmail/src/deploy_chatmail/www.py b/deploy-chatmail/src/deploy_chatmail/www.py index d06aa709..fd3f4bfb 100644 --- a/deploy-chatmail/src/deploy_chatmail/www.py +++ b/deploy-chatmail/src/deploy_chatmail/www.py @@ -7,7 +7,7 @@ import traceback import markdown from jinja2 import Template from .genqr import gen_qr_png_data -from deploy_chatmail import get_ini_settings +from chatmaild.config import read_config def snapshot_dir_stats(somedir): @@ -37,7 +37,7 @@ def build_webpages(src_dir, build_dir, config): def _build_webpages(src_dir, build_dir, config): - mail_domain = config["mail_domain"] + mail_domain = config.mailname assert src_dir.exists(), src_dir if not build_dir.exists(): build_dir.mkdir() @@ -70,7 +70,7 @@ def main(): path = importlib.resources.files(__package__) reporoot = path.joinpath("../../../").resolve() inipath = reporoot.joinpath("chatmail.ini") - config = get_ini_settings(chatmail_domain, inipath) + config = read_config(inipath, mailname=chatmail_domain) config["webdev"] = True www_path = reporoot.joinpath("www") src_path = www_path.joinpath("src") diff --git a/tests/chatmaild/test_config.py b/tests/chatmaild/test_config.py index 6cbf9e6d..84ae1193 100644 --- a/tests/chatmaild/test_config.py +++ b/tests/chatmaild/test_config.py @@ -1,21 +1,55 @@ from chatmaild.config import read_config +import chatmaild.config -def test_read_config_no_privacy_policy(tmp_path, create_ini): +def test_read_config_without_mailname(tmp_path, create_ini, monkeypatch): + mailname_path = tmp_path.joinpath("mailname") + mailname_path.write_text("something.example.org") + monkeypatch.setattr(chatmaild.config, "system_mailname_path", mailname_path) + inipath = create_ini( """ + [params] + max_user_send_per_minute = 40 + filtermail_smtp_port = 9875 + postfix_reinject_port = 9999 + """ + ) + config = read_config(inipath) + assert config.mailname == "something.example.org" + + +def test_read_config_without_privacy_policy(tmp_path, create_ini): + inipath = create_ini( + """ + [params] + max_user_send_per_minute = 40 + filtermail_smtp_port = 9875 + postfix_reinject_port = 9999 + [privacy:testrun] domain = *.example.org """ ) config = read_config(inipath, "something.example.org") assert config.mailname == "something.example.org" - assert not config.has_privacy_policy + assert config.max_user_send_per_minute == 40 + assert config.filtermail_smtp_port == 9875 + assert config.postfix_reinject_port == 9999 + assert not config.privacy_postal + assert not config.privacy_mail + assert not config.privacy_pdo + assert not config.privacy_supervisor def test_read_config(create_ini): inipath = create_ini( """ + [params] + max_user_send_per_minute = 40 + filtermail_smtp_port = 10080 + postfix_reinject_port = 10025 + [privacy:testrun] domain = *.testrun.org @@ -35,9 +69,10 @@ def test_read_config(create_ini): ) config = read_config(inipath, "something.testrun.org") - assert config.has_privacy_policy assert config.mailname == "something.testrun.org" + assert config.filtermail_smtp_port == 10080 + assert config.postfix_reinject_port == 10025 assert config.privacy_postal == "Postal Ltd" assert config.privacy_mail == "privacy@merlinux.eu" lines = config.privacy_pdo.split("\n") diff --git a/tests/chatmaild/test_doveauth.py b/tests/chatmaild/test_doveauth.py index b51ebe6c..07446861 100644 --- a/tests/chatmaild/test_doveauth.py +++ b/tests/chatmaild/test_doveauth.py @@ -1,5 +1,4 @@ import json -import sys import pytest import threading import queue @@ -7,7 +6,7 @@ import traceback import chatmaild.doveauth from chatmaild.doveauth import get_user_data, lookup_passdb, handle_dovecot_request -from chatmaild.database import Database, DBError +from chatmaild.database import DBError def test_basic(db): diff --git a/tests/chatmaild/test_filtermail.py b/tests/chatmaild/test_filtermail.py index fba7a9f9..0dd071c2 100644 --- a/tests/chatmaild/test_filtermail.py +++ b/tests/chatmaild/test_filtermail.py @@ -1,4 +1,10 @@ -from chatmaild.filtermail import check_encrypted, check_DATA, SendRateLimiter, check_mdn, is_passthrough_recipient +from chatmaild.filtermail import ( + check_encrypted, + check_DATA, + SendRateLimiter, + check_mdn, + is_passthrough_recipient, +) import pytest @@ -73,12 +79,12 @@ def test_filtermail_to_multiple_recipients_no_mdn(maildata, gencreds): def test_send_rate_limiter(): limiter = SendRateLimiter() for i in range(100): - if limiter.is_sending_allowed("some@example.org"): - if i <= SendRateLimiter.MAX_USER_SEND_PER_MINUTE: + if limiter.is_sending_allowed("some@example.org", 10): + if i <= 10: continue pytest.fail("limiter didn't work") else: - assert i == SendRateLimiter.MAX_USER_SEND_PER_MINUTE + 1 + assert i == 11 break diff --git a/tests/online/test_1_basic.py b/tests/online/test_1_basic.py index b3583f99..31e877ad 100644 --- a/tests/online/test_1_basic.py +++ b/tests/online/test_1_basic.py @@ -54,7 +54,7 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata): try: user1.smtp.sendmail(user1.addr, [user2.addr], mail) except smtplib.SMTPException as e: - if i < 80: + if i < 60: pytest.fail(f"rate limit was exceeded too early with msg {i}") outcome = e.recipients[user2.addr] assert outcome[0] == 450 diff --git a/tests/test_helpers.py b/tests/test_helpers.py index bb11c1e2..6479a18a 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -2,20 +2,25 @@ import textwrap import importlib.resources from deploy_chatmail.www import build_webpages -from deploy_chatmail import get_ini_settings +from chatmaild.config import read_config -def create_ini(inipath): +def create_ini(inipath, domain="example.org"): inipath.write_text( textwrap.dedent( - """\ - [config] + f"""\ + [params] + max_user_send_per_minute = 60 + filtermail_smtp_port = 10080 + postfix_reinject_port = 10025 + [privacy:{domain}] + domain = example.org privacy_postal = address-line1 address-line2 - privacy_mail = privacy@example.org + privacy_mail = privacy@{domain} privacy_pdo = address-line3 @@ -30,18 +35,18 @@ def test_build_webpages(tmp_path): assert src_dir.exists(), src_dir inipath = tmp_path.joinpath("chatmail.ini") - create_ini(inipath) - config = get_ini_settings("example.org", inipath) + create_ini(inipath, "example.org") + config = read_config(inipath, "example.org") build_dir = tmp_path.joinpath("build") build_webpages(src_dir, build_dir, config) def test_get_settings(tmp_path): inipath = tmp_path.joinpath("chatmail.ini") - create_ini(inipath) + create_ini(inipath, "example.org") - d = get_ini_settings("x.testrun.org", inipath) - assert d["privacy_postal"] == "address-line1\naddress-line2" - assert d["privacy_mail"] == "privacy@example.org" - assert d["privacy_pdo"] == "address-line3" - assert d["mail_domain"] == "x.testrun.org" + config = read_config(inipath, "example.org") + assert config.privacy_postal == "address-line1\naddress-line2" + assert config.privacy_mail == "privacy@example.org" + assert config.privacy_pdo == "address-line3" + assert config.mailname == "example.org"