Compare commits

..

2 Commits

Author SHA1 Message Date
missytake
88963bf1ad tests: make maildata work with python3.9 2024-02-17 10:18:11 +01:00
link2xt
000fed7495 Fix Python 3.9 support
I installed pyenv and then installed Python 3.9:
$ pyenv install 3.9
$ eval "$(pyenv init -)"
$ pyenv shell 3.9

In a clean repository I ran
$ scripts/cmdeploy init
$ scripts/cmdeploy run
$ scripts/cmdeploy dns
$ scripts/cmdeploy fmt

With the changes made all these commands work.

scripts/cmdeploy test fails some tests
using maildata fixture at
  importlib.resources.files(__package__).joinpath("mail-data")
line but this is not critical.
2024-02-15 14:37:31 +00:00
14 changed files with 42 additions and 471 deletions

View File

@@ -10,7 +10,6 @@ dependencies = [
"iniconfig",
"deltachat-rpc-server",
"deltachat-rpc-client",
"requests",
]
[tool.setuptools]
@@ -21,7 +20,6 @@ where = ['src']
[project.scripts]
doveauth = "chatmaild.doveauth:main"
chatmail-metadata = "chatmaild.metadata:main"
filtermail = "chatmaild.filtermail:main"
echobot = "chatmaild.echo:main"
chatmail-metrics = "chatmaild.metrics:main"

View File

@@ -1,10 +0,0 @@
[Unit]
Description=Chatmail dict proxy for IMAP METADATA
[Service]
ExecStart={execpath} /run/dovecot/metadata.socket vmail {config_path}
Restart=always
RestartSec=30
[Install]
WantedBy=multi-user.target

View File

@@ -58,18 +58,17 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
return True
def get_user_data(db, config: Config, user):
def get_user_data(db, user):
with db.read_connection() as conn:
result = conn.get_user(user)
if result:
result["home"] = f"/home/vmail/mail/{config.mail_domain}/{user}"
result["uid"] = "vmail"
result["gid"] = "vmail"
return result
def lookup_userdb(db, config: Config, user):
return get_user_data(db, config, user)
def lookup_userdb(db, user):
return get_user_data(db, user)
def lookup_passdb(db, config: Config, user, cleartext_password):
@@ -81,7 +80,6 @@ def lookup_passdb(db, config: Config, user, cleartext_password):
"UPDATE users SET last_login=? WHERE addr=?", (int(time.time()), user)
)
userdata["home"] = f"/home/vmail/mail/{config.mail_domain}/{user}"
userdata["uid"] = "vmail"
userdata["gid"] = "vmail"
return userdata
@@ -144,7 +142,7 @@ def handle_dovecot_request(msg, db, config: Config):
if type == "userdb":
user = args[0]
if user.endswith(f"@{config.mail_domain}"):
res = lookup_userdb(db, config, user)
res = lookup_userdb(db, user)
if res:
reply_command = "O"
else:

View File

@@ -1,154 +0,0 @@
import pwd
from queue import Queue
from threading import Thread
from socketserver import (
UnixStreamServer,
StreamRequestHandler,
ThreadingMixIn,
)
from .config import read_config
import sys
import logging
import os
import requests
DICTPROXY_LOOKUP_CHAR = "L"
DICTPROXY_ITERATE_CHAR = "I"
DICTPROXY_SET_CHAR = "S"
DICTPROXY_BEGIN_TRANSACTION_CHAR = "B"
DICTPROXY_COMMIT_TRANSACTION_CHAR = "C"
DICTPROXY_TRANSACTION_CHARS = "SBC"
class Notifier:
def __init__(self):
self.guid2token = {}
self.to_notify_queue = Queue()
def set_token(self, guid, token):
self.guid2token[guid] = token
def new_message_for_guid(self, guid):
self.to_notify_queue.put(guid)
def thread_run_loop(self):
requests_session = requests.Session()
while 1:
self.thread_run_one(requests_session)
def thread_run_one(self, requests_session):
guid = self.to_notify_queue.get()
token = self.guid2token.get(guid)
if token:
response = requests_session.post(
"https://notifications.delta.chat/notify",
data=token,
timeout=60,
)
if response.status_code == 410:
# 410 Gone status code
# means the token is no longer valid.
del self.guid2token[guid]
def handle_dovecot_protocol(rfile, wfile, notifier):
# HELLO message, ignored.
msg = rfile.readline().strip().decode()
transactions = {}
while True:
msg = rfile.readline().strip().decode()
if not msg:
break
res = handle_dovecot_request(msg, transactions, notifier)
if res:
wfile.write(res.encode("ascii"))
wfile.flush()
def handle_dovecot_request(msg, transactions, notifier):
# see https://doc.dovecot.org/3.0/developer_manual/design/dict_protocol/
short_command = msg[0]
parts = msg[1:].split("\t")
if short_command == DICTPROXY_LOOKUP_CHAR:
return "N\n"
elif short_command == DICTPROXY_ITERATE_CHAR:
# Empty line means ITER_FINISHED.
# If we don't return empty line Dovecot will timeout.
return "\n"
if short_command not in (DICTPROXY_TRANSACTION_CHARS):
return
transaction_id = parts[0]
if short_command == DICTPROXY_BEGIN_TRANSACTION_CHAR:
transactions[transaction_id] = "O\n"
elif short_command == DICTPROXY_COMMIT_TRANSACTION_CHAR:
# returns whether it failed or succeeded.
return transactions.pop(transaction_id, "N\n")
elif short_command == DICTPROXY_SET_CHAR:
# See header of
# <https://github.com/dovecot/core/blob/5e7965632395793d9355eb906b173bf28d2a10ca/src/lib-storage/mailbox-attribute.h>
# for the documentation on the structure of the key.
# Request GETMETADATA "INBOX" /private/chatmail
# results in a query for
# priv/dd72550f05eadc65542a1200cac67ad7/chatmail
#
# Request GETMETADATA "" /private/chatmail
# results in
# priv/dd72550f05eadc65542a1200cac67ad7/vendor/vendor.dovecot/pvt/server/chatmail
keyname = parts[1].split("/")
value = parts[2] if len(parts) > 2 else ""
if keyname[0] == "priv" and keyname[2] == "devicetoken":
notifier.set_token(keyname[1], value)
elif keyname[0] == "priv" and keyname[2] == "messagenew":
notifier.new_message_for_guid(keyname[1])
else:
# Transaction failed.
transactions[transaction_id] = "F\n"
class ThreadedUnixStreamServer(ThreadingMixIn, UnixStreamServer):
request_queue_size = 100
def main():
socket, username, config = sys.argv[1:]
passwd_entry = pwd.getpwnam(username)
# XXX config is not currently used
config = read_config(config)
notifier = Notifier()
class Handler(StreamRequestHandler):
def handle(self):
try:
handle_dovecot_protocol(self.rfile, self.wfile, notifier)
except Exception:
logging.exception("Exception in the handler")
raise
try:
os.unlink(socket)
except FileNotFoundError:
pass
# start notifier thread for signalling new messages to
# Delta Chat notification server
t = Thread(target=notifier.thread_run_loop)
t.setDaemon(True)
t.start()
with ThreadedUnixStreamServer(socket, Handler) as server:
os.chown(socket, uid=passwd_entry.pw_uid, gid=passwd_entry.pw_gid)
try:
server.serve_forever()
except KeyboardInterrupt:
pass

View File

@@ -17,7 +17,7 @@ from chatmaild.database import DBError
def test_basic(db, example_config):
lookup_passdb(db, example_config, "asdf12345@chat.example.org", "q9mr3faue")
data = get_user_data(db, example_config, "asdf12345@chat.example.org")
data = get_user_data(db, "asdf12345@chat.example.org")
assert data
data2 = lookup_passdb(
db, example_config, "asdf12345@chat.example.org", "q9mr3jewvadsfaue"
@@ -43,7 +43,7 @@ def test_nocreate_file(db, monkeypatch, tmpdir, example_config):
lookup_passdb(
db, example_config, "newuser12@chat.example.org", "zequ0Aimuchoodaechik"
)
assert not get_user_data(db, example_config, "newuser12@chat.example.org")
assert not get_user_data(db, "newuser12@chat.example.org")
def test_db_version(db):

View File

@@ -1,147 +0,0 @@
import io
from chatmaild.metadata import (
handle_dovecot_request,
handle_dovecot_protocol,
Notifier,
)
def test_handle_dovecot_request_lookup_fails():
notifier = Notifier()
res = handle_dovecot_request("Lpriv/123/chatmail", {}, notifier)
assert res == "N\n"
def test_handle_dovecot_request_happy_path():
notifier = Notifier()
transactions = {}
# lookups return the same NOTFOUND result
res = handle_dovecot_request("Lpriv/123/chatmail", transactions, notifier)
assert res == "N\n"
assert not notifier.guid2token and not transactions
# set device token in a transaction
tx = "1111"
msg = f"B{tx}\tuser"
res = handle_dovecot_request(msg, transactions, notifier)
assert not res and not notifier.guid2token
assert transactions == {tx: "O\n"}
msg = f"S{tx}\tpriv/guid00/devicetoken\t01234"
res = handle_dovecot_request(msg, transactions, notifier)
assert not res
assert len(transactions) == 1
assert len(notifier.guid2token) == 1
assert notifier.guid2token["guid00"] == "01234"
msg = f"C{tx}"
res = handle_dovecot_request(msg, transactions, notifier)
assert res == "O\n"
assert len(transactions) == 0
assert notifier.guid2token["guid00"] == "01234"
# trigger notification for incoming message
assert handle_dovecot_request(f"B{tx}\tuser", transactions, notifier) is None
msg = f"S{tx}\tpriv/guid00/messagenew"
assert handle_dovecot_request(msg, transactions, notifier) is None
assert notifier.to_notify_queue.get() == "guid00"
assert notifier.to_notify_queue.qsize() == 0
assert handle_dovecot_request(f"C{tx}\tuser", transactions, notifier) == "O\n"
assert not transactions
def test_handle_dovecot_protocol_set_devicetoken():
rfile = io.BytesIO(
b"\n".join(
[
b"HELLO",
b"Btx00\tuser",
b"Stx00\tpriv/guid00/devicetoken\t01234",
b"Ctx00",
]
)
)
wfile = io.BytesIO()
notifier = Notifier()
handle_dovecot_protocol(rfile, wfile, notifier)
assert notifier.guid2token["guid00"] == "01234"
assert wfile.getvalue() == b"O\n"
def test_handle_dovecot_protocol_iterate():
rfile = io.BytesIO(
b"\n".join(
[
b"H",
b"I9\t0\tpriv/5cbe730f146fea6535be0d003dd4fc98/\tci-2dzsrs@nine.testrun.org",
]
)
)
wfile = io.BytesIO()
notifier = Notifier()
handle_dovecot_protocol(rfile, wfile, notifier)
assert wfile.getvalue() == b"\n"
def test_handle_dovecot_protocol_messagenew():
rfile = io.BytesIO(
b"\n".join(
[
b"HELLO",
b"Btx01\tuser",
b"Stx01\tpriv/guid00/messagenew",
b"Ctx01",
]
)
)
wfile = io.BytesIO()
notifier = Notifier()
handle_dovecot_protocol(rfile, wfile, notifier)
assert wfile.getvalue() == b"O\n"
assert notifier.to_notify_queue.get() == "guid00"
assert notifier.to_notify_queue.qsize() == 0
def test_notifier_thread_run():
requests = []
class ReqMock:
def post(self, url, data, timeout):
requests.append((url, data, timeout))
class Result:
status_code = 200
return Result()
notifier = Notifier()
notifier.set_token("guid00", "01234")
notifier.new_message_for_guid("guid00")
notifier.thread_run_one(ReqMock())
url, data, timeout = requests[0]
assert data == "01234"
assert len(notifier.guid2token) == 1
def test_notifier_thread_run_gone_removes_token():
requests = []
class ReqMock:
def post(self, url, data, timeout):
requests.append((url, data, timeout))
class Result:
status_code = 410
return Result()
notifier = Notifier()
notifier.set_token("guid00", "01234")
notifier.new_message_for_guid("guid00")
assert notifier.guid2token["guid00"] == "01234"
notifier.thread_run_one(ReqMock())
url, data, timeout = requests[0]
assert data == "01234"
assert len(notifier.guid2token) == 0

View File

@@ -102,7 +102,6 @@ def _install_remote_venv_with_chatmaild(config) -> None:
"doveauth",
"filtermail",
"echobot",
"chatmail-metadata",
):
params = dict(
execpath=f"{remote_venv_dir}/bin/{fn}",
@@ -338,27 +337,6 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
mode="644",
)
need_restart |= auth_config.changed
lua_push_notification_script = files.put(
src=importlib.resources.files(__package__).joinpath(
"dovecot/push_notification.lua"
),
dest="/etc/dovecot/push_notification.lua",
user="root",
group="root",
mode="644",
)
need_restart |= lua_push_notification_script.changed
sieve_script = files.put(
src=importlib.resources.files(__package__).joinpath(
"dovecot/default.sieve"
),
dest="/etc/dovecot/default.sieve",
user="root",
group="root",
mode="644",
)
need_restart |= sieve_script.changed
files.template(
src=importlib.resources.files(__package__).joinpath("dovecot/expunge.cron.j2"),
@@ -448,9 +426,8 @@ def check_config(config):
mail_domain = config.mail_domain
if mail_domain != "testrun.org" and not mail_domain.endswith(".testrun.org"):
blocked_words = "merlinux schmieder testrun.org".split()
for key in config.__dict__:
value = config.__dict__[key]
if key.startswith("privacy") and any(x in str(value) for x in blocked_words):
for value in config.__dict__.values():
if any(x in str(value) for x in blocked_words):
raise ValueError(
f"please set your own privacy contacts/addresses in {config._inipath}"
)
@@ -471,10 +448,6 @@ def deploy_chatmail(config_path: Path) -> None:
apt.update(name="apt update", cache_time=24 * 3600)
server.group(name="Create vmail group", group="vmail", system=True)
server.user(name="Create vmail user", user="vmail", group="vmail", system=True)
apt.packages(
name="Install rsync",
packages=["rsync"],
)
# Run local DNS resolver `unbound`.
# `resolvconf` takes care of setting up /etc/resolv.conf
@@ -510,7 +483,7 @@ def deploy_chatmail(config_path: Path) -> None:
apt.packages(
name="Install Dovecot",
packages=["dovecot-imapd", "dovecot-lmtpd", "dovecot-sieve"],
packages=["dovecot-imapd", "dovecot-lmtpd"],
)
apt.packages(

View File

@@ -6,7 +6,7 @@ _submissions._tcp.{chatmail_domain}. SRV 0 1 465 {chatmail_domain}.
_imap._tcp.{chatmail_domain}. SRV 0 1 143 {chatmail_domain}.
_imaps._tcp.{chatmail_domain}. SRV 0 1 993 {chatmail_domain}.
{chatmail_domain}. CAA 128 issue "letsencrypt.org;accounturi={acme_account_url}"
{chatmail_domain}. TXT "v=spf1 a:{chatmail_domain} ~all"
{chatmail_domain}. TXT "v=spf1 a:{chatmail_domain} -all"
_dmarc.{chatmail_domain}. TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
_mta-sts.{chatmail_domain}. TXT "v=STSv1; id={sts_id}"
mta-sts.{chatmail_domain}. CNAME {chatmail_domain}.

View File

@@ -36,11 +36,12 @@ class DNS:
cmd = "ip a | grep inet6 | grep 'scope global' | sed -e 's#/64 scope global##' | sed -e 's#inet6##'"
return self.shell(cmd).strip()
def get(self, typ: str, domain: str) -> str:
"""Get a DNS entry or empty string if there is none."""
def get(self, typ: str, domain: str) -> Optional[str]:
"""Get a DNS entry"""
dig_result = self.shell(f"dig -r -q {domain} -t {typ} +short")
line = dig_result.partition("\n")[0]
return line
if line:
return line
def check_ptr_record(self, ip: str, mail_domain) -> bool:
"""Check the PTR record for an IPv4 or IPv6 address."""
@@ -55,25 +56,22 @@ def show_dns(args, out) -> int:
ssh = f"ssh root@{mail_domain}"
dns = DNS(out, mail_domain)
def read_dkim_entries(entry):
lines = []
for line in entry.split("\n"):
if line.startswith(";") or not line.strip():
continue
line = line.replace("\t", " ")
lines.append(line)
return "\n".join(lines)
print("Checking your DKIM keys and DNS entries...")
try:
acme_account_url = out.shell_output(f"{ssh} -- acmetool account-url")
except subprocess.CalledProcessError:
print("Please run `cmdeploy run` first.")
return 1
dkim_selector = "opendkim"
dkim_pubkey = out.shell_output(
ssh + f" -- openssl rsa -in /etc/dkimkeys/{dkim_selector}.private"
" -pubout 2>/dev/null | awk '/-/{next}{printf(\"%s\",$0)}'"
)
dkim_entry_value = f"v=DKIM1;k=rsa;p={dkim_pubkey};s=email;t=s"
dkim_entry_str = ""
while len(dkim_entry_value) >= 255:
dkim_entry_str += '"' + dkim_entry_value[:255] + '" '
dkim_entry_value = dkim_entry_value[255:]
dkim_entry_str += '"' + dkim_entry_value + '"'
dkim_entry = f"{dkim_selector}._domainkey.{mail_domain}. TXT {dkim_entry_str}"
dkim_entry = read_dkim_entries(out.shell_output(f"{ssh} -- opendkim-genzone -F"))
ipv6 = dns.get_ipv6()
reverse_ipv6 = dns.check_ptr_record(ipv6, mail_domain)
@@ -101,6 +99,7 @@ def show_dns(args, out) -> int:
return 0
except TypeError:
pass
started_dkim_parsing = False
for line in zonefile.splitlines():
line = line.format(
acme_account_url=acme_account_url,
@@ -127,23 +126,28 @@ def show_dns(args, out) -> int:
current = dns.get("SRV", domain[:-1])
if current != f"{prio} {weight} {port} {value}":
to_print.append(line)
if " TXT " in line:
if " TXT " in line:
domain, value = line.split(" TXT ")
current = dns.get("TXT", domain.strip()[:-1])
if domain.startswith("_mta-sts."):
if current:
if current.split("id=")[0] == value.split("id=")[0]:
continue
# TXT records longer than 255 bytes
# are split into multiple <character-string>s.
# This typically happens with DKIM record
# which contains long RSA key.
#
# Removing `" "` before comparison
# to get back a single string.
if current.replace('" "', "") != value.replace('" "', ""):
if current != value:
to_print.append(line)
if "IN TXT ( " in line:
started_dkim_parsing = True
dkim_lines = [line]
if started_dkim_parsing and line.startswith('"'):
dkim_lines.append(" " + line)
domain, data = "\n".join(dkim_lines).split(" IN TXT ")
current = dns.get("TXT", domain.strip()[:-1])
if current:
current = "( %s )" % (current.replace('" "', '"\n "'))
if current.replace(";", "\\;") != data:
to_print.append(dkim_entry)
else:
to_print.append(dkim_entry)
exit_code = 0
if to_print:

View File

@@ -1,5 +0,0 @@
require ["imap4flags"];
if header :is ["Auto-Submitted"] ["auto-replied", "auto-generated"] {
addflag "$Auto";
}

View File

@@ -13,12 +13,6 @@ auth_cache_size = 100M
mail_debug = yes
{% endif %}
# Prevent warnings similar to:
# config: Warning: service auth { client_limit=1000 } is lower than required under max. load (10200). Counted for protocol services with service_count != 1: service lmtp { process_limit=100 } + service imap-urlauth-login { process_limit=100 } + service imap-login { process_limit=10000 }
# config: Warning: service anvil { client_limit=1000 } is lower than required under max. load (10103). Counted with: service imap-urlauth-login { process_limit=100 } + service imap-login { process_limit=10000 } + service auth { process_limit=1 }
# master: Warning: service(stats): client_limit (1000) reached, client connections are being dropped
default_client_limit = 20000
mail_server_admin = mailto:root@{{ config.mail_domain }}
mail_server_comment = Chatmail server
@@ -27,7 +21,7 @@ mail_plugins = quota
# these are the capabilities Delta Chat cares about actually
# so let's keep the network overhead per login small
# https://github.com/deltachat/deltachat-core-rust/blob/master/src/imap/capabilities.rs
imap_capability = IMAP4rev1 IDLE MOVE QUOTA CONDSTORE NOTIFY METADATA XDELTAPUSH
imap_capability = IMAP4rev1 IDLE MOVE QUOTA CONDSTORE NOTIFY METADATA
# Authentication for system users.
@@ -77,9 +71,6 @@ mail_privileged_group = vmail
## Mail processes
##
# Pass all IMAP METADATA requests to the server implementing Dovecot's dict protocol.
mail_attribute_dict = proxy:/run/dovecot/metadata.socket:metadata
# Enable IMAP COMPRESS (RFC 4978).
# <https://datatracker.ietf.org/doc/html/rfc4978.html>
protocol imap {
@@ -88,21 +79,7 @@ protocol imap {
}
protocol lmtp {
# quota plugin documentation:
# <https://doc.dovecot.org/configuration_manual/quota_plugin/>
#
# notify plugin is a dependency of push_notification plugin:
# <https://doc.dovecot.org/settings/plugin/notify-plugin/>
#
# push_notification plugin documentation:
# <https://doc.dovecot.org/configuration_manual/push_notification/>
#
# mail_lua and push_notification_lua are needed for Lua push notification handler.
# <https://doc.dovecot.org/configuration_manual/push_notification/#configuration>
#
# Sieve to mark messages that should not be notified as \Seen
# <https://doc.dovecot.org/configuration_manual/sieve/configuration/>
mail_plugins = $mail_plugins quota mail_lua notify push_notification push_notification_lua sieve
mail_plugins = $mail_plugins quota
}
plugin {
@@ -118,15 +95,7 @@ plugin {
# quota_over_flag_value = TRUE
}
# push_notification configuration
plugin {
# <https://doc.dovecot.org/configuration_manual/push_notification/#lua-lua>
push_notification_driver = lua:file=/etc/dovecot/push_notification.lua
}
plugin {
sieve_default = file:/etc/dovecot/default.sieve
}
service lmtp {
user=vmail

View File

@@ -8,4 +8,3 @@
# or only temporary (but then they shouldn't be around after {{ config.delete_mails_after }} days anyway).
2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
3 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -name 'maildirsize' -type f -delete

View File

@@ -1,32 +0,0 @@
function dovecot_lua_notify_begin_txn(user)
return user
end
function contains(v, needle)
for _, keyword in ipairs(v) do
if keyword == needle then
return true
end
end
return false
end
function dovecot_lua_notify_event_message_new(user, event)
local mbox = user:mailbox(event.mailbox)
mbox:sync()
if user.username ~= event.from_address then
-- Incoming message
if not contains(event.keywords, "$Auto") then
-- Not an Auto-Submitted message, notifying.
-- Notify METADATA server about new message.
mbox:metadata_set("/private/messagenew", "")
end
end
mbox:free()
end
function dovecot_lua_notify_end_txn(ctx, success)
end

View File

@@ -1,7 +1,6 @@
import pytest
import threading
import queue
import socket
from chatmaild.config import read_config
from cmdeploy.cmdeploy import main
@@ -79,24 +78,3 @@ def test_concurrent_logins_same_account(
for _ in conns:
assert login_results.get()
def test_no_vrfy(chatmail_config):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((chatmail_config.mail_domain, 25))
banner = sock.recv(1024)
print(banner)
sock.send(b"VRFY wrongaddress@%s\r\n" % (chatmail_config.mail_domain.encode(),))
result = sock.recv(1024)
print(result)
sock.send(b"VRFY echo@%s\r\n" % (chatmail_config.mail_domain.encode(),))
result2 = sock.recv(1024)
print(result2)
assert result[0:10] == result2[0:10]
sock.send(b"VRFY wrongaddress\r\n")
result = sock.recv(1024)
print(result)
sock.send(b"VRFY echo\r\n")
result2 = sock.recv(1024)
print(result2)
assert result[0:10] == result2[0:10] == b"252 2.0.0 "