From 2d0fc2e70e988263460105b5271588eaf992476f Mon Sep 17 00:00:00 2001 From: holger krekel Date: Mon, 24 Aug 2026 23:18:43 +0200 Subject: [PATCH] feat: move doveauth from dictproxy to lua/http 1. existing logins are now verified by lua only 2. non-existing logins are delegated to the new Python doveauth http /create endpoint Using Lua and http this way makes doveauth more compatible to dovecot 2.4 --- chatmaild/src/chatmaild/config.py | 1 + chatmaild/src/chatmaild/doveauth.py | 206 ++++++------ .../tests/test_delete_inactive_users.py | 6 +- .../src/chatmaild/tests/test_doveauth.py | 308 ++++++++++-------- .../src/chatmaild/tests/test_lastlogin.py | 6 +- .../src/chatmaild/tests/test_migrate_db.py | 2 +- chatmaild/src/chatmaild/tests/test_user.py | 17 +- chatmaild/src/chatmaild/user.py | 15 +- cmdeploy/pyproject.toml | 1 + cmdeploy/src/cmdeploy/dovecot/auth.conf | 12 - cmdeploy/src/cmdeploy/dovecot/auth.lua.j2 | 64 ++++ cmdeploy/src/cmdeploy/dovecot/deployer.py | 5 +- cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 | 8 +- cmdeploy/src/cmdeploy/pins.py | 2 + .../src/cmdeploy/service/doveauth.service.f | 7 +- cmdeploy/src/cmdeploy/tests/test_authlua.py | 206 ++++++++++++ .../cmdeploy/tests/test_dovecot_deployer.py | 10 +- doc/source/overview.rst | 13 +- 18 files changed, 593 insertions(+), 296 deletions(-) delete mode 100644 cmdeploy/src/cmdeploy/dovecot/auth.conf create mode 100644 cmdeploy/src/cmdeploy/dovecot/auth.lua.j2 create mode 100644 cmdeploy/src/cmdeploy/tests/test_authlua.py diff --git a/chatmaild/src/chatmaild/config.py b/chatmaild/src/chatmaild/config.py index 21f817e7..09085742 100644 --- a/chatmaild/src/chatmaild/config.py +++ b/chatmaild/src/chatmaild/config.py @@ -55,6 +55,7 @@ class Config: self.postfix_reinject_port_incoming = int( params.pop("postfix_reinject_port_incoming", "10026") ) + self.doveauth_http_port = int(params.pop("doveauth_http_port", "10084")) self.mtail_address = params.pop("mtail_address", None) self.disable_ipv6 = params.pop("disable_ipv6", "false").lower() == "true" self.acme_email = params.pop("acme_email", "") diff --git a/chatmaild/src/chatmaild/doveauth.py b/chatmaild/src/chatmaild/doveauth.py index eb0ea918..315a30f4 100644 --- a/chatmaild/src/chatmaild/doveauth.py +++ b/chatmaild/src/chatmaild/doveauth.py @@ -1,10 +1,17 @@ -import json +"""Create chatmail addresses on first login. + +Dovecot only asks us about addresses it does not already find in the mailbox: +the auth.lua we deploy with dovecot (cmdeploy/src/cmdeploy/dovecot/auth.lua.j2) +verifies existing users itself against a mailbox password file, +and HTTP-POSTs everything else to the /create endpoint implemented in this module. +""" + import logging import os import re import sys - -import filelock +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer try: import crypt_r @@ -12,7 +19,6 @@ except ImportError: import crypt as crypt_r from .config import Config, read_config -from .dictproxy import DictProxy from .migrate_db import migrate_from_db_to_maildir from .syslimits import has_sufficient_resources @@ -64,109 +70,117 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool: return True -def split_and_unescape(s): - """Split strings using double quote as a separator and backslash as escape character - into parts.""" - - out = "" - i = 0 - while i < len(s): - c = s[i] - if c == "\\": - # Skip escape character. - i += 1 - - # This will raise IndexError if there is no character - # after escape character. This is expected - # as this is an invalid input. - out += s[i] - elif c == '"': - # Separator - yield out - out = "" - else: - out += c - i += 1 - yield out +def verify_password(stored, cleartext_password) -> bool: + if stored.startswith("{"): + stored = stored.split("}", 1)[1] + return crypt_r.crypt(cleartext_password, stored) == stored -class AuthDictProxy(DictProxy): +class DoveAuth: def __init__(self, config): - super().__init__() self.config = config + self.creation_lock = threading.Lock() - def handle_lookup(self, parts): - # Dovecot <2.3.17 has only one part, - # do not attempt to read any other parts for compatibility. - keyname = parts[0] - - namespace, type, args = keyname.split("/", 2) - args = list(split_and_unescape(args)) - + def create_user(self, addr, cleartext_password) -> bool: + """Create the address, or verify the password if it exists already.""" config = self.config - reply_command = "F" - res = "" - if namespace == "shared": - if type == "userdb": - user = args[0] - if user.endswith(f"@{config.mail_domain}"): - res = self.lookup_userdb(user) - if res: - reply_command = "O" - else: - reply_command = "N" - elif type == "passdb": - user = args[1] - if user.endswith(f"@{config.mail_domain}"): - res = self.lookup_passdb(user, cleartext_password=args[0]) - if res: - reply_command = "O" - else: - reply_command = "N" - json_res = json.dumps(res) if res else "" - return f"{reply_command}{json_res}\n" - - def handle_iterate(self, parts): - # example: I0\t0\tshared/userdb/ - if parts[2] == "shared/userdb/": - result = "".join( - f"Oshared/userdb/{user}\t\n" for user in self.iter_userdb() - ) - return f"{result}\n" - - def iter_userdb(self) -> list: - """Get a list of all user addresses.""" - return [x for x in os.listdir(self.config.mailboxes_dir) if "@" in x] - - def lookup_userdb(self, addr): - return self.config.get_user(addr).get_userdb_dict() - - def lookup_passdb(self, addr, cleartext_password): - user = self.config.get_user(addr) - userdata = user.get_userdb_dict() - if userdata: - return userdata - if not is_allowed_to_create(self.config, addr, cleartext_password): - return - if not has_sufficient_resources(self.config): - return - - lock = filelock.FileLock(str(user.password_path) + ".lock", timeout=5) - with lock: - userdata = user.get_userdb_dict() - if userdata: - return userdata + if not addr.endswith(f"@{config.mail_domain}"): + logging.warning("address not in mail domain: %r", addr) + return False + try: + user = config.get_user(addr) + except ValueError: + logging.warning("invalid address: %r", addr) + return False + with self.creation_lock: + passhash = user.get_password_hash() + if passhash is not None: + # a concurrent first login may have just created the address + return verify_password(passhash, cleartext_password) + if not is_allowed_to_create(config, addr, cleartext_password): + return False + if not has_sufficient_resources(config): + return False user.set_password(encrypt_password(cleartext_password)) - print(f"Created address: {addr}", file=sys.stderr) - return user.get_userdb_dict() + # mtail counts created_accounts off this exact line + print(f"Created address: {addr}", file=sys.stderr) + return True + + +class CreateHandler(BaseHTTPRequestHandler): + """Answer POST /create requests from dovecot's auth.lua, body `addr\\tpassword`. + + The body must be UTF-8 and only the first tab separates the fields, + so a password may itself contain tabs. + Any non-UTF8 or \\0 bytes in the body fail the request. + + Addresses are ASCII: dovecot refuses any login name outside its + auth_username_chars before auth.lua ever sees it. + + Dovecot hands auth.lua the exact password bytes the client sent; + decoding and re-encoding UTF-8 is byte-identical, + so dovecot's password_verify later recomputes the same hash crypt() stores here. + """ + + protocol_version = "HTTP/1.1" # dovecot's HTTP client reuses connections + + max_body_len = 512 # an address and a password + + def do_POST(self): + if self.path != "/create": + self.reply(404) + return + length = self.body_length() + if length is None: + self.reply(400) + return + body = self.rfile.read(length) + try: + addr, _, password = body.decode("utf-8").partition("\t") + except UnicodeDecodeError: + self.reply(400) + return + if "\0" in addr or "\0" in password: + self.reply(400) + return + self.reply(200 if self.server.doveauth.create_user(addr, password) else 403) + + def body_length(self): + try: + length = int(self.headers["Content-Length"]) + except (TypeError, ValueError): + return None + return length if 0 <= length <= self.max_body_len else None + + def reply(self, status): + self.send_response(status) + self.send_header("Content-Length", "0") + if status != 200: + # Just close on any failure, as body might not be fully read. + # It's anyway cheap to re-establish http localhost without TLS. + self.send_header("Connection", "close") + self.end_headers() + + def log_message(self, format, *args): + # the per-request access log would only duplicate our own stderr lines + pass + + +class DoveAuthServer(ThreadingHTTPServer): + # a burst of first-time logins (e.g. from CI) must not overflow + # the accept queue, see https://github.com/chatmail/relay/issues/436 + request_queue_size = 1000 + + def __init__(self, config, port): + super().__init__(("127.0.0.1", port), CreateHandler) + self.doveauth = DoveAuth(config) def main(): - socket, cfgpath = sys.argv[1:] + (cfgpath,) = sys.argv[1:] config = read_config(cfgpath) migrate_from_db_to_maildir(config) - dictproxy = AuthDictProxy(config=config) - - dictproxy.serve_forever_from_socket(socket) + server = DoveAuthServer(config, config.doveauth_http_port) + server.serve_forever() diff --git a/chatmaild/src/chatmaild/tests/test_delete_inactive_users.py b/chatmaild/src/chatmaild/tests/test_delete_inactive_users.py index 6a60362a..ecc3325a 100644 --- a/chatmaild/src/chatmaild/tests/test_delete_inactive_users.py +++ b/chatmaild/src/chatmaild/tests/test_delete_inactive_users.py @@ -1,6 +1,6 @@ import time -from chatmaild.doveauth import AuthDictProxy +from chatmaild.doveauth import DoveAuth from chatmaild.expire import daily_expire_main as main_expire @@ -18,10 +18,10 @@ def test_login_timestamps(example_config): def test_delete_inactive_users(example_config): new = time.time() old = new - (example_config.delete_inactive_users_after * 86400) - 1 - dictproxy = AuthDictProxy(example_config) + doveauth = DoveAuth(example_config) def create_user(addr, last_login): - dictproxy.lookup_passdb(addr, "q9mr3faue") + doveauth.create_user(addr, "q9mr3faue") user = example_config.get_user(addr) user.maildir.joinpath("cur").mkdir() user.maildir.joinpath("cur", "something").mkdir() diff --git a/chatmaild/src/chatmaild/tests/test_doveauth.py b/chatmaild/src/chatmaild/tests/test_doveauth.py index c2057196..94e99d6f 100644 --- a/chatmaild/src/chatmaild/tests/test_doveauth.py +++ b/chatmaild/src/chatmaild/tests/test_doveauth.py @@ -1,42 +1,37 @@ -import io -import json -import queue +import http.client import threading -import traceback +from concurrent.futures import ThreadPoolExecutor import pytest import chatmaild.doveauth from chatmaild.doveauth import ( - AuthDictProxy, + CreateHandler, + DoveAuth, + DoveAuthServer, is_allowed_to_create, ) from chatmaild.newemail import create_newemail_dict @pytest.fixture -def dictproxy(example_config): - return AuthDictProxy(config=example_config) +def doveauth(example_config): + return DoveAuth(example_config) -def test_basic(dictproxy, example_gencreds): +def stored_hash(config, addr): + return config.get_user(addr).get_password_hash() + + +def test_basic(doveauth, example_config, example_gencreds): addr, password = example_gencreds() - dictproxy.lookup_passdb(addr, password) - data = dictproxy.lookup_userdb(addr) - assert data - data2 = dictproxy.lookup_passdb(addr, password) - assert data == data2 + assert doveauth.create_user(addr, password) + passhash = stored_hash(example_config, addr) + assert passhash.startswith("{SHA512-CRYPT}") - -def test_iterate_addresses(dictproxy): - addresses = [] - - for i in range(10): - addresses.append(f"asdf1234{i}@chat.example.org") - dictproxy.lookup_passdb(addresses[-1], "q9mr3faue") - - res = dictproxy.iter_userdb() - assert set(res) == set(addresses) + # a second login verifies against the stored hash and rewrites nothing + assert doveauth.create_user(addr, password) + assert stored_hash(example_config, addr) == passhash def test_invalid_username_length(example_config): @@ -53,75 +48,32 @@ def test_invalid_username_length(example_config): ) -def test_dont_overwrite_password_on_wrong_login(dictproxy): - """Test that logging in with a different password doesn't create a new user""" - res = dictproxy.lookup_passdb( - "newuser12@chat.example.org", "kajdlkajsldk12l3kj1983" - ) - assert res["password"] - res2 = dictproxy.lookup_passdb("newuser12@chat.example.org", "kajdslqwe") - # this function always returns a password hash, which is actually compared by dovecot. - assert res["password"] == res2["password"] +def test_dont_overwrite_password_on_wrong_login(doveauth, example_config): + addr = "newuser12@chat.example.org" + assert doveauth.create_user(addr, "kajdlkajsldk12l3kj1983") + passhash = stored_hash(example_config, addr) + + assert not doveauth.create_user(addr, "kajdslqwe") + assert stored_hash(example_config, addr) == passhash + + assert doveauth.create_user(addr, "kajdlkajsldk12l3kj1983") + assert stored_hash(example_config, addr) == passhash -def test_nocreate_file(monkeypatch, tmpdir, dictproxy): +def test_foreign_domain_is_refused(doveauth): + assert not doveauth.create_user("newuser12@evil.example.org", "qlwkejqlwe12") + + +def test_nocreate_file(monkeypatch, tmpdir, doveauth, example_config): p = tmpdir.join("nocreate") p.write("") monkeypatch.setattr(chatmaild.doveauth, "NOCREATE_FILE", str(p)) - dictproxy.lookup_passdb("newuser12@chat.example.org", "zequ0Aimuchoodaechik") - assert not dictproxy.lookup_userdb("newuser12@chat.example.org") - - -def test_handle_dovecot_request(dictproxy): - transactions = {} - # Test that password can contain ", ', \ and / - msg = ( - 'Lshared/passdb/laksjdlaksjdlak\\\\sjdlk\\"12j\\\'3l1/k2j3123"' - "some42123@chat.example.org\tsome42123@chat.example.org" - ) - res = dictproxy.handle_dovecot_request(msg, transactions) - assert res - assert res[0] == "O" and res.endswith("\n") - userdata = json.loads(res[1:].strip()) - assert userdata["home"].endswith("chat.example.org/some42123@chat.example.org") - assert userdata["uid"] == userdata["gid"] == "vmail" - assert userdata["password"].startswith("{SHA512-CRYPT}") - - -def test_handle_dovecot_protocol_hello_is_skipped(example_config, caplog): - dictproxy = AuthDictProxy(config=example_config) - rfile = io.BytesIO(b"H3\t2\t0\t\tauth\n") - wfile = io.BytesIO() - dictproxy.loop_forever(rfile, wfile) - assert wfile.getvalue() == b"" - assert not caplog.messages - - -def test_handle_dovecot_protocol_user_not_exists(example_config): - dictproxy = AuthDictProxy(config=example_config) - rfile = io.BytesIO( - b"H3\t2\t0\t\tauth\nLshared/userdb/foobar@chat.example.org\tfoobar@chat.example.org\n" - ) - wfile = io.BytesIO() - dictproxy.loop_forever(rfile, wfile) - assert wfile.getvalue() == b"N\n" - - -def test_handle_dovecot_protocol_iterate(example_config): - dictproxy = AuthDictProxy(config=example_config) - dictproxy.lookup_passdb("asdf00000@chat.example.org", "q9mr3faue") - dictproxy.lookup_passdb("asdf11111@chat.example.org", "q9mr3faue") - rfile = io.BytesIO(b"H3\t2\t0\t\tauth\nI0\t0\tshared/userdb/") - wfile = io.BytesIO() - dictproxy.loop_forever(rfile, wfile) - lines = wfile.getvalue().decode("ascii").split("\n") - assert "Oshared/userdb/asdf00000@chat.example.org\t" in lines - assert "Oshared/userdb/asdf11111@chat.example.org\t" in lines - assert not lines[2] + addr = "newuser12@chat.example.org" + assert not doveauth.create_user(addr, "zequ0Aimuchoodaechik") + assert stored_hash(example_config, addr) is None def test_invalid_localpart_characters(make_config): - """Test that is_allowed_to_create rejects localparts with invalid characters.""" config = make_config("chat.example.org", {"username_min_length": "3"}) password = "zequ0Aimuchoodaechik" domain = config.mail_domain @@ -141,78 +93,150 @@ def test_invalid_localpart_characters(make_config): assert not is_allowed_to_create(config, f"ab@cdef@{domain}", password) assert not is_allowed_to_create(config, f"abc/def@{domain}", password) assert not is_allowed_to_create(config, f"abc\\def@{domain}", password) + assert not is_allowed_to_create(config, f"üser123@{domain}", password) -def test_concurrent_creation_same_account(dictproxy): - """Test that concurrent creation of the same account doesn't corrupt password.""" +def test_concurrent_creation_same_account(doveauth, example_config, capsys): addr = "racetest1@chat.example.org" password = "zequ0Aimuchoodaechik" - num_threads = 10 - results = queue.Queue() - def create(): - try: - res = dictproxy.lookup_passdb(addr, password) - results.put(("ok", res)) - except Exception: - results.put(("err", traceback.format_exc())) - - threads = [threading.Thread(target=create, daemon=True) for _ in range(num_threads)] - for t in threads: - t.start() - for t in threads: - t.join(timeout=10) - - passwords_seen = set() - for _ in range(num_threads): - status, res = results.get() - if status == "err": - pytest.fail(f"concurrent creation failed\n{res}") - passwords_seen.add(res["password"]) + def create(_): + ok = doveauth.create_user(addr, password) + return ok, stored_hash(example_config, addr) + with ThreadPoolExecutor(10) as pool: + results = list(pool.map(create, range(10))) + assert all(ok for ok, _ in results) # all threads must see the same password hash - assert len(passwords_seen) == 1 - - -def test_50_concurrent_lookups_different_accounts(example_gencreds, dictproxy): - num_threads = 50 - req_per_thread = 5 - results = queue.Queue() - - def lookup(): - for i in range(req_per_thread): - addr, password = example_gencreds() - try: - dictproxy.lookup_passdb(addr, password) - except Exception: - results.put(traceback.format_exc()) - else: - results.put(None) - - threads = [] - for i in range(num_threads): - thread = threading.Thread(target=lookup, daemon=True) - threads.append(thread) - - print(f"created {num_threads} threads, starting them and waiting for results") - for thread in threads: - thread.start() - - for i in range(num_threads * req_per_thread): - res = results.get() - if res is not None: - pytest.fail(f"concurrent lookup failed\n{res}") + assert len({passhash for _, passhash in results}) == 1 + assert capsys.readouterr().err.count("Created address:") == 1 def test_insufficient_resources_block_creation_not_existing_logins( - dictproxy, example_gencreds, monkeypatch + doveauth, example_gencreds, monkeypatch ): addr, password = example_gencreds() - assert dictproxy.lookup_passdb(addr, password) + assert doveauth.create_user(addr, password) monkeypatch.setattr( chatmaild.doveauth, "has_sufficient_resources", lambda config: False ) newaddr, newpassword = example_gencreds() - assert not dictproxy.lookup_passdb(newaddr, newpassword) - assert dictproxy.lookup_passdb(addr, password) + assert not doveauth.create_user(newaddr, newpassword) + assert doveauth.create_user(addr, password) + + +class TestHttpPost: + @pytest.fixture + def doveauth_server(self, example_config): + server = DoveAuthServer(example_config, port=0) + threading.Thread(target=server.serve_forever, daemon=True).start() + yield f"127.0.0.1:{server.server_address[1]}" + server.shutdown() + server.server_close() + + @pytest.fixture + def post(self, doveauth_server): + def post(path, data): + conn = http.client.HTTPConnection(doveauth_server, timeout=10) + try: + return self.post_on(conn, path, data).status + finally: + conn.close() + + return post + + @pytest.fixture + def connection(self, doveauth_server): + """One kept-alive connection, which is all dovecot's HTTP client opens.""" + conn = http.client.HTTPConnection(doveauth_server, timeout=10) + yield conn + conn.close() + + @staticmethod + def post_on(conn, path, data): + conn.request("POST", path, body=data) + resp = conn.getresponse() + resp.read() + return resp + + def test_create_and_verify(self, post, example_config, example_gencreds): + addr, password = example_gencreds() + assert post("/create", f"{addr}\t{password}".encode()) == 200 + assert stored_hash(example_config, addr).startswith("{SHA512-CRYPT}") + + # second login with the same password verifies, a wrong one is refused + assert post("/create", f"{addr}\t{password}".encode()) == 200 + assert post("/create", f"{addr}\twrong{password}".encode()) == 403 + + def test_password_special_chars_survive_transport(self, post, example_gencreds): + addr, _ = example_gencreds() + password = "laksjdlaksjdlak\\sjdlk\"12j'3l1/k2\tj3123" + body = f"{addr}\t{password}".encode() + assert post("/create", body) == 200 + assert post("/create", body) == 200 + assert post("/create", f"{addr}\totherpassword1".encode()) == 403 + + def test_password_must_be_utf8(self, post, example_gencreds): + addr, _ = example_gencreds() + assert post("/create", f"{addr}\tpässwort12".encode()) == 200 + assert post("/create", addr.encode() + b"\tp\xe4sswort12") == 400 + + def test_nul_is_refused_before_crypt_sees_it(self, post, example_gencreds): + addr, _ = example_gencreds() + assert post("/create", f"{addr}\tpass\0word12".encode()) == 400 + assert ( + post("/create", "us\0er12345@chat.example.org\tlongenough1".encode()) == 400 + ) + + def test_refused_creation(self, post, example_gencreds): + addr, _ = example_gencreds() + assert post("/create", f"{addr}\tshort".encode()) == 403 + assert post("/create", b"not-an-address\tlongenoughpassword") == 403 + body = "bürger123@chat.example.org\tlongenoughpw".encode() + assert post("/create", body) == 403 + assert post("/create", b"") == 403 + + def test_body_length_limit(self, post, example_gencreds): + addr, _ = example_gencreds() + fill = CreateHandler.max_body_len - len(addr) - len("\t") + body = f"{addr}\t{'x' * fill}".encode() + assert len(body) == CreateHandler.max_body_len + assert post("/create", body) == 200 + + body = f"{addr}\t{'x' * (fill + 1)}".encode() + assert len(body) == CreateHandler.max_body_len + 1 + assert post("/create", body) == 400 + + def test_connection_is_reused_across_200_replies( + self, connection, example_gencreds + ): + addr, password = example_gencreds() + body = f"{addr}\t{password}".encode() + # create, then verify the same password, on one connection + for _ in range(2): + resp = self.post_on(connection, "/create", body) + assert (resp.status, resp.will_close) == (200, False) + + @pytest.mark.parametrize( + "path,data,status", + [ + ("/other", b"not read", 404), + ("/create", b"x" * (CreateHandler.max_body_len + 1), 400), + ("/create", b"not-an-address\tlongenoughpassword", 403), + ], + ) + def test_error_replies_close_the_connection(self, connection, path, data, status): + resp = self.post_on(connection, path, data) + assert (resp.status, resp.will_close) == (status, True) + + @pytest.mark.parametrize("content_length", [None, "-1", "notanumber", "999999"]) + def test_bad_content_length(self, connection, content_length): + # the fixture timeout turns a server that waits for the body into a failure + connection.putrequest("POST", "/create", skip_accept_encoding=True) + if content_length is not None: + connection.putheader("Content-Length", content_length) + connection.endheaders() + resp = connection.getresponse() + assert resp.status == 400 + assert resp.will_close diff --git a/chatmaild/src/chatmaild/tests/test_lastlogin.py b/chatmaild/src/chatmaild/tests/test_lastlogin.py index a7e9e348..e4e8e81b 100644 --- a/chatmaild/src/chatmaild/tests/test_lastlogin.py +++ b/chatmaild/src/chatmaild/tests/test_lastlogin.py @@ -1,6 +1,6 @@ import time -from chatmaild.doveauth import AuthDictProxy +from chatmaild.doveauth import DoveAuth from chatmaild.lastlogin import ( LastLoginDictProxy, ) @@ -9,8 +9,8 @@ from chatmaild.lastlogin import ( def test_handle_dovecot_request_last_login(testaddr, example_config): dictproxy = LastLoginDictProxy(config=example_config) - authproxy = AuthDictProxy(config=example_config) - authproxy.lookup_passdb(testaddr, "1l2k3j1l2k3jl123") + doveauth = DoveAuth(example_config) + doveauth.create_user(testaddr, "1l2k3j1l2k3jl123") dictproxy_transactions = {} diff --git a/chatmaild/src/chatmaild/tests/test_migrate_db.py b/chatmaild/src/chatmaild/tests/test_migrate_db.py index 7313939a..1763a559 100644 --- a/chatmaild/src/chatmaild/tests/test_migrate_db.py +++ b/chatmaild/src/chatmaild/tests/test_migrate_db.py @@ -63,7 +63,7 @@ def test_migration(tmp_path, example_config, caplog): user = example_config.get_user(path.name) if last_login: assert user.get_last_login_timestamp() == last_login - assert password == user.get_userdb_dict()["password"] + assert password == user.get_password_hash() assert not all assert not example_config.passdb_path.exists() diff --git a/chatmaild/src/chatmaild/tests/test_user.py b/chatmaild/src/chatmaild/tests/test_user.py index 3e7b5063..aca1a057 100644 --- a/chatmaild/src/chatmaild/tests/test_user.py +++ b/chatmaild/src/chatmaild/tests/test_user.py @@ -8,28 +8,23 @@ def test_login_timestamp(testaddr, example_config): assert user.get_last_login_timestamp() == 86400 * 2 -def test_get_user_dict_not_set(testaddr, example_config, caplog): +def test_get_password_hash_not_set(testaddr, example_config, caplog): user = example_config.get_user(testaddr) assert not caplog.records - assert user.get_userdb_dict() == {} + assert user.get_password_hash() is None assert len(caplog.records) == 0 user.set_password("") - assert user.get_userdb_dict() == {} + assert user.get_password_hash() is None assert len(caplog.records) == 1 -def test_get_user_dict(make_config, tmp_path): +def test_get_password_hash(make_config, tmp_path): config = make_config("something.testrun.org") - addr = "user1@something.org" - user = config.get_user(addr) + user = config.get_user("user1@something.org") enc_password = "l1k2j31lk2j3l1k23j123" user.set_password(enc_password) - data = user.get_userdb_dict() - assert addr in str(data["home"]) - assert data["uid"] == "vmail" - assert data["gid"] == "vmail" - assert data["password"] == enc_password + assert user.get_password_hash() == enc_password def test_no_mailboxes_dir(testaddr, example_config, tmp_path): diff --git a/chatmaild/src/chatmaild/user.py b/chatmaild/src/chatmaild/user.py index 3a86958e..27c0fe83 100644 --- a/chatmaild/src/chatmaild/user.py +++ b/chatmaild/src/chatmaild/user.py @@ -21,20 +21,17 @@ class User: def can_track(self): return "@" in self.addr - def get_userdb_dict(self): - """Return a non-empty dovecot 'userdb' style dict - if the user has an existing non-empty password""" + def get_password_hash(self): try: - pw = self.password_path.read_text() + passhash = self.password_path.read_text() except FileNotFoundError: - return {} + return None - if not pw: + if not passhash: logging.error(f"password is empty for: {self.addr}") - return {} + return None - home = str(self.maildir) - return dict(addr=self.addr, home=home, uid=self.uid, gid=self.gid, password=pw) + return passhash def is_incoming_cleartext_ok(self): return not self.enforce_E2EE_path.exists() diff --git a/cmdeploy/pyproject.toml b/cmdeploy/pyproject.toml index 272cb45a..a372ccaf 100644 --- a/cmdeploy/pyproject.toml +++ b/cmdeploy/pyproject.toml @@ -19,6 +19,7 @@ dependencies = [ "pytest-xdist", "execnet", "imap_tools", + "jinja2", "lupa", "deltachat-rpc-client", "deltachat-rpc-server", diff --git a/cmdeploy/src/cmdeploy/dovecot/auth.conf b/cmdeploy/src/cmdeploy/dovecot/auth.conf deleted file mode 100644 index fe40a9b0..00000000 --- a/cmdeploy/src/cmdeploy/dovecot/auth.conf +++ /dev/null @@ -1,12 +0,0 @@ -uri = proxy:/run/doveauth/doveauth.socket:auth -iterate_disable = no -iterate_prefix = userdb/ - -default_pass_scheme = plain -# %E escapes characters " (double quote), ' (single quote) and \ (backslash) with \ (backslash). -# See -# for documentation. -# -# We escape user-provided input and use double quote as a separator. -password_key = passdb/%Ew"%Eu -user_key = userdb/%Eu diff --git a/cmdeploy/src/cmdeploy/dovecot/auth.lua.j2 b/cmdeploy/src/cmdeploy/dovecot/auth.lua.j2 new file mode 100644 index 00000000..753c63cb --- /dev/null +++ b/cmdeploy/src/cmdeploy/dovecot/auth.lua.j2 @@ -0,0 +1,64 @@ +-- Existing addresses are served from the maildir directly. +-- Unknown ones are offered to doveauth, which owns the creation policy. +local mailboxes_dir = "{{ config.mailboxes_dir }}" +local domain_suffix = "@{{ config.mail_domain }}" +local create_url = "http://127.0.0.1:{{ config.doveauth_http_port }}/create" + +local http_client + +local function is_ours(user) + return user:sub(-#domain_suffix) == domain_suffix + and not user:find("/", 1, true) +end + +local function password_hash(user) + local fh = io.open(mailboxes_dir .. "/" .. user .. "/password", "r") + if not fh then + return nil + end + local hash, rest = fh:read("l", "a") + fh:close() + if hash == nil or hash == "" or rest ~= "" then + return nil + end + return hash +end + +local function userdb_fields(user) + return {home = mailboxes_dir .. "/" .. user, uid = "vmail", gid = "vmail"} +end + +local function create(user, password) + local request = http_client:request({url = create_url, method = "POST"}) + request:set_payload(user .. "\t" .. password) + return request:submit():status() == 200 +end + +-- Entry points called by dovecot + +function script_init() + http_client = dovecot.http.client({request_timeout_msecs = 5000, max_attempts = 1}) + return 0 +end + +function auth_userdb_lookup(req) + if not is_ours(req.user) or password_hash(req.user) == nil then + return dovecot.auth.USERDB_RESULT_USER_UNKNOWN, {} + end + return dovecot.auth.USERDB_RESULT_OK, userdb_fields(req.user) +end + +function auth_password_verify(req, password) + if not is_ours(req.user) then + return dovecot.auth.PASSDB_RESULT_USER_UNKNOWN, {} + end + local hash = password_hash(req.user) + if hash == nil then + if not create(req.user, password) then + return dovecot.auth.PASSDB_RESULT_USER_UNKNOWN, {} + end + elseif req:password_verify(hash, password) ~= 1 then + return dovecot.auth.PASSDB_RESULT_PASSWORD_MISMATCH, {} + end + return dovecot.auth.PASSDB_RESULT_OK, userdb_fields(req.user) +end diff --git a/cmdeploy/src/cmdeploy/dovecot/deployer.py b/cmdeploy/src/cmdeploy/dovecot/deployer.py index 98588439..a4e0d5a7 100644 --- a/cmdeploy/src/cmdeploy/dovecot/deployer.py +++ b/cmdeploy/src/cmdeploy/dovecot/deployer.py @@ -31,7 +31,7 @@ class DovecotDeployer(Deployer): arch = host.get_fact(Arch) with blocked_service_startup(): debs = [] - for pkg in ("core", "imapd", "lmtpd"): + for pkg in ("core", "imapd", "lmtpd", "auth-lua"): deb, changed = _download_dovecot_package(pkg, arch) self.need_restart |= changed if deb: @@ -134,7 +134,8 @@ def _configure_dovecot(deployer, config: Config, debug: bool = False): debug=debug, disable_ipv6=config.disable_ipv6, ) - deployer.put_file("dovecot/auth.conf", "/etc/dovecot/auth.conf") + deployer.put_template("dovecot/auth.lua.j2", "/etc/dovecot/auth.lua", config=config) + deployer.remove_file("/etc/dovecot/auth.conf") deployer.put_file( "dovecot/push_notification.lua", "/etc/dovecot/push_notification.lua" ) diff --git a/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 b/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 index c9977683..19b08a64 100644 --- a/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 +++ b/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 @@ -61,12 +61,12 @@ imap_capability = +XDELTAPUSH XCHATMAIL # Authentication for system users. passdb { - driver = dict - args = /etc/dovecot/auth.conf + driver = lua + args = file=/etc/dovecot/auth.lua blocking=yes } userdb { - driver = dict - args = /etc/dovecot/auth.conf + driver = lua + args = file=/etc/dovecot/auth.lua blocking=yes } ## ## Mailbox locations and namespaces diff --git a/cmdeploy/src/cmdeploy/pins.py b/cmdeploy/src/cmdeploy/pins.py index a555ae38..20ae826e 100644 --- a/cmdeploy/src/cmdeploy/pins.py +++ b/cmdeploy/src/cmdeploy/pins.py @@ -36,6 +36,8 @@ DOVECOT_SHA256 = { ("imapd", "arm64"): "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f", ("lmtpd", "amd64"): "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab", ("lmtpd", "arm64"): "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f", + ("auth-lua", "amd64"): "d724f37712faba52e177153114af1831e54da555c57c6474c05f96f176176ce4", + ("auth-lua", "arm64"): "7272768e20de148c35891d99cd60205acbe7e732b056caf0a83e8194d49136e6", } TURN_VERSION = "v0.4" TURN_ARTIFACTS = { diff --git a/cmdeploy/src/cmdeploy/service/doveauth.service.f b/cmdeploy/src/cmdeploy/service/doveauth.service.f index 9d858960..68574167 100644 --- a/cmdeploy/src/cmdeploy/service/doveauth.service.f +++ b/cmdeploy/src/cmdeploy/service/doveauth.service.f @@ -1,12 +1,11 @@ [Unit] -Description=Chatmail dict authentication proxy for dovecot +Description=Chatmail HTTP authentication service for dovecot [Service] -ExecStart={execpath} /run/doveauth/doveauth.socket {config_path} +ExecStart={execpath} {config_path} Restart=always -RestartSec=30 +RestartSec=5 User=vmail -RuntimeDirectory=doveauth UMask=0077 [Install] diff --git a/cmdeploy/src/cmdeploy/tests/test_authlua.py b/cmdeploy/src/cmdeploy/tests/test_authlua.py new file mode 100644 index 00000000..e79972e6 --- /dev/null +++ b/cmdeploy/src/cmdeploy/tests/test_authlua.py @@ -0,0 +1,206 @@ +"""Test auth.lua script against mocked dovecot auth API.""" + +import jinja2 +import pytest +from chatmaild.doveauth import encrypt_password, verify_password + +from cmdeploy.basedeploy import get_resource + +USER1 = "user12345@chat.example.org" +USER2 = "newuser12@chat.example.org" +OK, UNKNOWN, MISMATCH = 1, -2, -3 + +DOVECOT_MOCKS = """ +create_status = 200 + +dovecot = { + auth = { + PASSDB_RESULT_OK = OK, + PASSDB_RESULT_USER_UNKNOWN = UNKNOWN, + PASSDB_RESULT_PASSWORD_MISMATCH = MISMATCH, + USERDB_RESULT_OK = OK, + USERDB_RESULT_USER_UNKNOWN = UNKNOWN, + }, + http = { + client = function(options) + client_options = options + return {request = function(_, options) + create_request = options + return { + set_payload = function(_, payload) create_payload = payload end, + submit = function() + return {status = function() return create_status end} + end, + } + end} + end, + }, +} +""" + + +def load_authlua(lua, config): + lua.g.OK, lua.g.UNKNOWN, lua.g.MISMATCH = OK, UNKNOWN, MISMATCH + lua.rt.execute(DOVECOT_MOCKS) + template = jinja2.Template(get_resource("dovecot/auth.lua.j2").read_text()) + lua.rt.execute(template.render(config=config)) + assert lua.g.script_init() == 0 + return lua + + +@pytest.fixture +def authlua(lua, example_config): + return load_authlua(lua, example_config) + + +@pytest.fixture +def request_for(lua): + def request_for(addr): + def password_verify(_self, hashed, plain): + return 1 if verify_password(hashed, plain) else 0 + + return lua.table(user=addr, password_verify=password_verify) + + return request_for + + +@pytest.fixture +def create_user(example_config): + def create_user(addr, password): + example_config.get_user(addr).set_password(encrypt_password(password)) + + return create_user + + +@pytest.fixture +def write_password_file(example_config): + def write_password_file(addr, content): + maildir = example_config.mailboxes_dir / addr + maildir.mkdir(parents=True, exist_ok=True) + maildir.joinpath("password").write_text(content) + + return write_password_file + + +def test_http_client_uses_dovecot_setting_names(authlua): + """dovecot's lua http binding silently ignores keys it does not know.""" + assert dict(authlua.g.client_options) == { + "request_timeout_msecs": 5000, + "max_attempts": 1, + } + + +def test_existing_address_correct_password(authlua, request_for, create_user): + create_user(USER1, "correctgoose") + res, fields = authlua.g.auth_password_verify(request_for(USER1), "correctgoose") + assert res == OK + assert fields["uid"] == fields["gid"] == "vmail" + assert fields["home"].endswith(USER1) + assert authlua.g.create_payload is None + + +def test_existing_address_wrong_password(authlua, request_for, create_user): + create_user(USER1, "correctgoose") + res, _ = authlua.g.auth_password_verify(request_for(USER1), "wronghorse") + assert res == MISMATCH + + +def test_foreign_domain_is_refused_without_calling_out( + authlua, request_for, create_user +): + create_user("user12345@evil.example.org", "correctgoose") + request = request_for("user12345@evil.example.org") + res, _ = authlua.g.auth_password_verify(request, "correctgoose") + assert res == UNKNOWN + assert authlua.g.auth_userdb_lookup(request)[0] == UNKNOWN + assert authlua.g.create_payload is None + + +def test_name_shorter_than_the_domain_is_refused(authlua, request_for): + for name in ("x", "", "chat.example.org"): + res, _ = authlua.g.auth_password_verify(request_for(name), "correctgoose") + assert res == UNKNOWN + assert authlua.g.auth_userdb_lookup(request_for(name))[0] == UNKNOWN + assert authlua.g.create_payload is None + + +def test_slash_in_username_is_refused(authlua, request_for): + request = request_for("../../etc/shadow@chat.example.org") + res, _ = authlua.g.auth_password_verify(request, "somepassword") + assert res == UNKNOWN + assert authlua.g.auth_userdb_lookup(request)[0] == UNKNOWN + assert authlua.g.create_payload is None + + +def test_localpart_policy_is_left_to_doveauth(authlua, request_for): + authlua.g.create_status = 403 + res, _ = authlua.g.auth_password_verify(request_for("@chat.example.org"), "somepw") + assert res == UNKNOWN + assert authlua.g.create_payload == "@chat.example.org\tsomepw" + + +def test_unknown_address_is_created_via_endpoint(authlua, request_for): + res, fields = authlua.g.auth_password_verify(request_for(USER2), "brandnewpass") + assert res == OK + assert fields["home"].endswith(USER2) + assert authlua.g.create_payload == f"{USER2}\tbrandnewpass" + assert authlua.g.create_request["url"] == "http://127.0.0.1:10084/create" + + +# a policy refusal and a doveauth that is down are both fail-closed +@pytest.mark.parametrize("status", [403, 0]) +def test_creation_that_is_not_answered_with_200_is_user_unknown( + authlua, request_for, status +): + authlua.g.create_status = status + res, _ = authlua.g.auth_password_verify(request_for(USER2), "brandnewpass") + assert res == UNKNOWN + + +def test_userdb_unknown_before_creation_ok_after(authlua, request_for, create_user): + request = request_for(USER1) + res, _ = authlua.g.auth_userdb_lookup(request) + assert res == UNKNOWN + # a userdb lookup must never create anything + assert authlua.g.create_payload is None + + create_user(USER1, "correctgoose") + res, fields = authlua.g.auth_userdb_lookup(request) + assert res == OK + assert fields["home"].endswith(USER1) + assert fields["uid"] == fields["gid"] == "vmail" + + +def test_empty_password_file_is_unknown(authlua, request_for, write_password_file): + write_password_file(USER1, "") + assert authlua.g.auth_userdb_lookup(request_for(USER1))[0] == UNKNOWN + + write_password_file(USER1, "\n") + assert authlua.g.auth_userdb_lookup(request_for(USER1))[0] == UNKNOWN + + +def test_password_file_format_checks(authlua, request_for, write_password_file): + write_password_file(USER1, encrypt_password("correctgoose") + "\n") + res, _ = authlua.g.auth_password_verify(request_for(USER1), "correctgoose") + assert res == OK + assert authlua.g.auth_userdb_lookup(request_for(USER1))[0] == OK + + passhash = encrypt_password("correctgoose") + write_password_file(USER1, passhash + "\ntrailing junk") + authlua.g.create_status = 403 + res, _ = authlua.g.auth_password_verify(request_for(USER1), "correctgoose") + assert res == UNKNOWN + assert authlua.g.auth_userdb_lookup(request_for(USER1))[0] == UNKNOWN + + +def test_ipv4_relay_uses_bracketed_domain(lua, ipv4_config, request_for): + # mail_domain is "[1.3.3.7]" here, and is_ours must not read it as a pattern + authlua = load_authlua(lua, ipv4_config) + addr = f"user12345@{ipv4_config.mail_domain}" + ipv4_config.get_user(addr).set_password(encrypt_password("correctgoose")) + + res, fields = authlua.g.auth_password_verify(request_for(addr), "correctgoose") + assert res == OK + assert fields["home"].endswith(addr) + assert authlua.g.auth_userdb_lookup(request_for(addr))[0] == OK + assert authlua.g.auth_userdb_lookup(request_for(USER1))[0] == UNKNOWN diff --git a/cmdeploy/src/cmdeploy/tests/test_dovecot_deployer.py b/cmdeploy/src/cmdeploy/tests/test_dovecot_deployer.py index 0eabbd5d..4d064c0d 100644 --- a/cmdeploy/src/cmdeploy/tests/test_dovecot_deployer.py +++ b/cmdeploy/src/cmdeploy/tests/test_dovecot_deployer.py @@ -136,6 +136,7 @@ def test_install_skips_dpkg_path_when_epoch_matched_packages_present( "dovecot-core": [dovecot_deployer.DOVECOT_PACKAGE_VERSION], "dovecot-imapd": [dovecot_deployer.DOVECOT_PACKAGE_VERSION], "dovecot-lmtpd": [dovecot_deployer.DOVECOT_PACKAGE_VERSION], + "dovecot-auth-lua": [dovecot_deployer.DOVECOT_PACKAGE_VERSION], }, ), (dovecot_deployer.Arch, "x86_64"), @@ -180,9 +181,12 @@ def test_install_unsupported_arch_falls_back_to_apt( deployer.install() actual_pkgs = [c["packages"] for c in apt_calls] - assert actual_pkgs == [["dovecot-core"], ["dovecot-imapd"], ["dovecot-lmtpd"]], ( - f"expected apt install of core/imapd/lmtpd, got {actual_pkgs}" - ) + assert actual_pkgs == [ + ["dovecot-core"], + ["dovecot-imapd"], + ["dovecot-lmtpd"], + ["dovecot-auth-lua"], + ], f"expected apt install of core/imapd/lmtpd/auth-lua, got {actual_pkgs}" assert track_shell == [], "should not run dpkg for unsupported arch" assert deployer.need_restart is True, ( "need_restart should be True when apt installed a package" diff --git a/doc/source/overview.rst b/doc/source/overview.rst index 6363a27a..264eb805 100644 --- a/doc/source/overview.rst +++ b/doc/source/overview.rst @@ -84,11 +84,12 @@ and only relaying OpenPGP end-to-end messages encrypted messages. A short overview of ``chatmaild`` services: - :repofile:`doveauth ` - implements create-on-login address semantics and is used by Dovecot - during IMAP login and by Postfix during SMTP/SUBMISSION login which - in turn uses `Dovecot SASL - `_ - to authenticate logins. + implements create-on-login address semantics. + Dovecot authenticates IMAP logins, and Postfix SMTP/SUBMISSION logins through `Dovecot SASL + `_, + from an :repofile:`auth.lua ` script + that reads the maildir directly. Only addresses which do not exist yet + are passed on to doveauth, which owns the creation policy. - :repofile:`chatmail-metadata ` is contacted by a @@ -155,7 +156,7 @@ Chatmail relay dependency diagram filtermail-outgoing --- |10025 reinject|postfix; filtermail-incoming --- |10026 reinject|postfix; postfix --- |milter opendkim.sock|OpenDKIM - dovecot --- |doveauth.socket|doveauth; + dovecot --- |10084 create|doveauth; dovecot --- |message delivery|maildir["maildir /home/vmail/.../user"]; dovecot --- |lastlogin.socket|lastlogin;