From 194030a4561032b0a96d199e815fdcc27ae42996 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Sat, 29 Mar 2025 21:22:26 +0100 Subject: [PATCH] enforce encryption for in-server mails (#535) * enforce encryption for in-server mails * make tests work with chatmail server only support e2ee internally * fix echobot test * simplify quota-exceeded test * work around rpc-server fixture changes --- CHANGELOG.md | 3 + chatmaild/src/chatmaild/echo.py | 5 ++ chatmaild/src/chatmaild/filtermail.py | 19 +---- chatmaild/src/chatmaild/tests/plugin.py | 3 +- .../src/chatmaild/tests/test_filtermail.py | 25 ++++-- cmdeploy/src/cmdeploy/remote/rshell.py | 19 +++++ .../src/cmdeploy/tests/online/test_1_basic.py | 3 +- .../cmdeploy/tests/online/test_2_deltachat.py | 84 ++++++++----------- cmdeploy/src/cmdeploy/tests/plugin.py | 8 +- 9 files changed, 94 insertions(+), 75 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9708827e..8c68ed23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## untagged +- Enforce end-to-end encryption between local addresses + ([#535](https://github.com/chatmail/server/pull/535)) + - Limit the bind for the HTTPS server on 8443 to 127.0.0.1 ([#522](https://github.com/chatmail/server/pull/522)) ([#532](https://github.com/chatmail/server/pull/532)) diff --git a/chatmaild/src/chatmaild/echo.py b/chatmaild/src/chatmaild/echo.py index c0f45fca..c31701fd 100644 --- a/chatmaild/src/chatmaild/echo.py +++ b/chatmaild/src/chatmaild/echo.py @@ -8,6 +8,7 @@ import logging import os import subprocess import sys +from pathlib import Path from deltachat_rpc_client import Bot, DeltaChat, EventType, Rpc, events @@ -97,6 +98,10 @@ def main(): if not bot.is_configured(): bot.configure(addr, password) + # write invite link to working directory + invitelink = bot.account.get_qr_code() + Path("invite-link.txt").write_text(invitelink) + bot.run_forever() diff --git a/chatmaild/src/chatmaild/filtermail.py b/chatmaild/src/chatmaild/filtermail.py index 3258b107..25d92487 100644 --- a/chatmaild/src/chatmaild/filtermail.py +++ b/chatmaild/src/chatmaild/filtermail.py @@ -209,7 +209,6 @@ class BeforeQueueHandler: mail_encrypted = check_encrypted(message) _, from_addr = parseaddr(message.get("from").strip()) - envelope_from_domain = from_addr.split("@").pop() logging.info(f"mime-from: {from_addr} envelope-from: {envelope.mail_from!r}") if envelope.mail_from.lower() != from_addr.lower(): @@ -223,26 +222,16 @@ class BeforeQueueHandler: if envelope.mail_from in self.config.passthrough_senders: return - passthrough_recipients = self.config.passthrough_recipients - if mail_encrypted or is_securejoin(message): return + passthrough_recipients = self.config.passthrough_recipients + for recipient in envelope.rcpt_tos: - if envelope.mail_from == recipient: - # Always allow sending emails to self. - continue if recipient_matches_passthrough(recipient, passthrough_recipients): continue - res = recipient.split("@") - if len(res) != 2: - return f"500 Invalid address <{recipient}>" - _recipient_addr, recipient_domain = res - - is_outgoing = recipient_domain != envelope_from_domain - if is_outgoing: - print("Rejected unencrypted mail.", file=sys.stderr) - return f"500 Invalid unencrypted mail to <{recipient}>" + print("Rejected unencrypted mail.", file=sys.stderr) + return "500 Invalid unencrypted mail" class SendRateLimiter: diff --git a/chatmaild/src/chatmaild/tests/plugin.py b/chatmaild/src/chatmaild/tests/plugin.py index e3938d04..b57418a3 100644 --- a/chatmaild/src/chatmaild/tests/plugin.py +++ b/chatmaild/src/chatmaild/tests/plugin.py @@ -72,9 +72,8 @@ def maildata(request): def maildata(name, from_addr, to_addr, subject="[...]"): # Using `.read_bytes().decode()` instead of `.read_text()` to preserve newlines. data = datadir.joinpath(name).read_bytes().decode() - text = data.format(from_addr=from_addr, to_addr=to_addr, subject=subject) - return BytesParser(policy=policy.default).parsebytes(text.encode()) + return BytesParser(policy=policy.SMTP).parsebytes(text.encode()) return maildata diff --git a/chatmaild/src/chatmaild/tests/test_filtermail.py b/chatmaild/src/chatmaild/tests/test_filtermail.py index 033f4c9c..efbfc17e 100644 --- a/chatmaild/src/chatmaild/tests/test_filtermail.py +++ b/chatmaild/src/chatmaild/tests/test_filtermail.py @@ -29,14 +29,14 @@ def test_reject_forged_from(maildata, gencreds, handler): # test that the filter lets good mail through to_addr = gencreds()[0] env.content = maildata( - "plain.eml", from_addr=env.mail_from, to_addr=to_addr + "encrypted.eml", from_addr=env.mail_from, to_addr=to_addr ).as_bytes() assert not handler.check_DATA(envelope=env) # test that the filter rejects forged mail env.content = maildata( - "plain.eml", from_addr="forged@c3.testrun.org", to_addr=to_addr + "encrypted.eml", from_addr="forged@c3.testrun.org", to_addr=to_addr ).as_bytes() error = handler.check_DATA(envelope=env) assert "500" in error @@ -106,7 +106,7 @@ def test_send_rate_limiter(): break -def test_excempt_privacy(maildata, gencreds, handler): +def test_cleartext_excempt_privacy(maildata, gencreds, handler): from_addr = gencreds()[0] to_addr = "privacy@testrun.org" handler.config.passthrough_recipients = [to_addr] @@ -130,7 +130,22 @@ def test_excempt_privacy(maildata, gencreds, handler): assert "500" in handler.check_DATA(envelope=env2) -def test_passthrough_domains(maildata, gencreds, handler): +def test_cleartext_self_send_fails(maildata, gencreds, handler): + from_addr = gencreds()[0] + to_addr = from_addr + + msg = maildata("plain.eml", from_addr=from_addr, to_addr=to_addr) + + class env: + mail_from = from_addr + rcpt_tos = [to_addr] + content = msg.as_bytes() + + res = handler.check_DATA(envelope=env) + assert "500 Invalid unencrypted" in res + + +def test_cleartext_passthrough_domains(maildata, gencreds, handler): from_addr = gencreds()[0] to_addr = "privacy@x.y.z" handler.config.passthrough_recipients = ["@x.y.z"] @@ -154,7 +169,7 @@ def test_passthrough_domains(maildata, gencreds, handler): assert "500" in handler.check_DATA(envelope=env2) -def test_passthrough_senders(gencreds, handler, maildata): +def test_cleartext_passthrough_senders(gencreds, handler, maildata): acc1 = gencreds()[0] to_addr = "recipient@something.org" handler.config.passthrough_senders = [acc1] diff --git a/cmdeploy/src/cmdeploy/remote/rshell.py b/cmdeploy/src/cmdeploy/remote/rshell.py index ffc1ab66..a07f539b 100644 --- a/cmdeploy/src/cmdeploy/remote/rshell.py +++ b/cmdeploy/src/cmdeploy/remote/rshell.py @@ -14,3 +14,22 @@ def shell(command, fail_ok=False): def get_systemd_running(): lines = shell("systemctl --type=service --state=running").split("\n") return [line for line in lines if line.startswith(" ")] + + +def write_numbytes(path, num): + with open(path, "w") as f: + f.write("x" * num) + + +def dovecot_recalc_quota(user): + shell(f"doveadm quota recalc -u {user}") + output = shell(f"doveadm quota get -u {user}") + # + # Quota name Type Value Limit % + # User quota STORAGE 5 102400 0 + # User quota MESSAGE 2 - 0 + # + for line in output.split("\n"): + parts = line.split() + if parts[2] == "STORAGE": + return dict(value=int(parts[3]), limit=int(parts[4]), percent=int(parts[5])) diff --git a/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py b/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py index e311bc0a..980d7069 100644 --- a/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py +++ b/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py @@ -116,9 +116,8 @@ def test_authenticated_from(cmsetup, maildata): @pytest.mark.parametrize("from_addr", ["fake@example.org", "fake@testrun.org"]) def test_reject_missing_dkim(cmsetup, maildata, from_addr): - """Test that emails with missing or wrong DMARC, DKIM, and SPF entries are rejected.""" recipient = cmsetup.gen_users(1)[0] - msg = maildata("plain.eml", from_addr=from_addr, to_addr=recipient.addr).as_string() + msg = maildata("encrypted.eml", from_addr=from_addr, to_addr=recipient.addr).as_string() with smtplib.SMTP(cmsetup.maildomain, 25) as s: with pytest.raises(smtplib.SMTPDataError, match="No valid DKIM signature"): s.sendmail(from_addr=from_addr, to_addrs=recipient.addr, msg=msg) diff --git a/cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py b/cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py index 466dae7b..cb31a471 100644 --- a/cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py +++ b/cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py @@ -1,5 +1,4 @@ import ipaddress -import random import re import time @@ -7,6 +6,9 @@ import imap_tools import pytest import requests +from cmdeploy.remote import rshell +from cmdeploy.sshexec import SSHExec + @pytest.fixture def imap_mailbox(cmfactory): @@ -54,22 +56,23 @@ class TestEndToEndDeltaChat: """Test that a DC account can send a message to a second DC account on the same chat-mail instance.""" ac1, ac2 = cmfactory.get_online_accounts(2) - chat = cmfactory.get_accepted_chat(ac1, ac2) - - lp.sec("ac1: prepare and send text message to ac2") + chat = cmfactory.get_protected_chat(ac1, ac2) chat.send_text("message0") lp.sec("wait for ac2 to receive message") msg2 = ac2._evtracker.wait_next_incoming_message() assert msg2.text == "message0" - @pytest.mark.slow - def test_exceed_quota(self, cmfactory, lp, tmpdir, remote, chatmail_config): + def test_exceed_quota( + self, cmfactory, lp, tmpdir, remote, chatmail_config, sshdomain + ): """This is a very slow test as it needs to upload >100MB of mail data before quota is exceeded, and thus depends on the speed of the upload. """ ac1, ac2 = cmfactory.get_online_accounts(2) - chat = cmfactory.get_accepted_chat(ac1, ac2) + chat = cmfactory.get_protected_chat(ac1, ac2) + + user = ac2.get_config("configured_addr") def parse_size_limit(limit: str) -> int: """Parse a size limit and return the number of bytes as integer. @@ -82,49 +85,27 @@ class TestEndToEndDeltaChat: return int(float(number) * units[unit]) quota = parse_size_limit(chatmail_config.max_mailbox_size) - attachsize = 1 * 1024 * 1024 - num_to_send = quota // attachsize + 2 - lp.sec(f"ac1: send {num_to_send} large files to ac2") - lp.indent(f"per-user quota is assumed to be: {quota / (1024 * 1024)}MB") - alphanumeric = "abcdefghijklmnopqrstuvwxyz1234567890" - msgs = [] - for i in range(num_to_send): - attachment = tmpdir / f"attachment{i}" - data = "".join(random.choice(alphanumeric) for i in range(1024)) - with open(attachment, "w+") as f: - for j in range(attachsize // len(data)): - f.write(data) - msg = chat.send_file(str(attachment)) - msgs.append(msg) - lp.indent(f"Sent out msg {i}, size {attachsize / (1024 * 1024)}MB") + lp.sec(f"filling remote inbox for {user}") + fn = f"7743102289.M843172P2484002.c20,S={quota},W=2398:2," + path = chatmail_config.mailboxes_dir.joinpath(user, "cur", fn) + sshexec = SSHExec(sshdomain) + sshexec(call=rshell.write_numbytes, kwargs=dict(path=str(path), num=120)) + res = sshexec(call=rshell.dovecot_recalc_quota, kwargs=dict(user=user)) + assert res["percent"] >= 100 - lp.sec("ac2: check messages are arriving until quota is reached") + lp.sec("ac2: check quota is triggered") - addr = ac2.get_config("addr").lower() - saved_ok = 0 + starting = True for line in remote.iter_output("journalctl -n0 -f -u dovecot"): - if addr not in line: + if starting: + chat.send_text("hello") + starting = False + if user not in line: # print(line) continue - if "quota" in line: - if "quota exceeded" in line: - if saved_ok < num_to_send // 2: - pytest.fail( - f"quota exceeded too early: after {saved_ok} messages already" - ) - lp.indent("good, message sending failed because quota was exceeded") - return - if ( - "stored mail into mailbox 'inbox'" in line - or "saved mail to inbox" in line - ): - saved_ok += 1 - print(f"{saved_ok}: {line}") - if saved_ok >= num_to_send: - break - - pytest.fail("sending succeeded although messages should exceed quota") + if "quota exceeded" in line: + return def test_securejoin(self, cmfactory, lp, maildomain2): ac1 = cmfactory.new_online_configuring_account(cache=False) @@ -172,7 +153,7 @@ def test_hide_senders_ip_address(cmfactory): assert ipaddress.ip_address(public_ip) user1, user2 = cmfactory.get_online_accounts(2) - chat = cmfactory.get_accepted_chat(user1, user2) + chat = cmfactory.get_protected_chat(user1, user2) chat.send_text("testing submission header cleanup") user2._evtracker.wait_next_incoming_message() @@ -181,11 +162,18 @@ def test_hide_senders_ip_address(cmfactory): assert public_ip not in msg.obj.as_string() -def test_echobot(cmfactory, chatmail_config, lp): +def test_echobot(cmfactory, chatmail_config, lp, sshdomain): ac = cmfactory.get_online_accounts(1)[0] - lp.sec(f"Send message to echo@{chatmail_config.mail_domain}") - chat = ac.create_chat(f"echo@{chatmail_config.mail_domain}") + # establish contact with echobot + sshexec = SSHExec(sshdomain) + command = "cat /var/lib/echobot/invite-link.txt" + echo_invite_link = sshexec(call=rshell.shell, kwargs=dict(command=command)) + chat = ac.qr_setup_contact(echo_invite_link) + ac._evtracker.wait_securejoin_joiner_progress(1000) + + # send message and check it gets replied back + lp.sec(f"Send message to echobot") text = "hi, I hope you text me back" chat.send_text(text) lp.sec("Wait for reply from echobot") diff --git a/cmdeploy/src/cmdeploy/tests/plugin.py b/cmdeploy/src/cmdeploy/tests/plugin.py index beae6e5a..b4f17fa7 100644 --- a/cmdeploy/src/cmdeploy/tests/plugin.py +++ b/cmdeploy/src/cmdeploy/tests/plugin.py @@ -302,10 +302,12 @@ def cmfactory(request, gencreds, tmpdir, maildomain): pytest.importorskip("deltachat") from deltachat.testplugin import ACFactory - data = request.getfixturevalue("data") - testproc = ChatmailTestProcess(request.config, maildomain, gencreds) - am = ACFactory(request=request, tmpdir=tmpdir, testprocess=testproc, data=data) + + class Data: + def read_path(self, path): + return + am = ACFactory(request=request, tmpdir=tmpdir, testprocess=testproc, data=Data()) # nb. a bit hacky # would probably be better if deltachat's test machinery grows native support