feat: move doveauth from dictproxy to lua/http

1. existing logins are now verified by lua only

2. non-existing logins are delegated to the new Python doveauth http /create endpoint

Using Lua and http this way makes doveauth more compatible to dovecot 2.4
This commit is contained in:
holger krekel
2026-09-01 22:56:07 +02:00
parent 051f831518
commit 2d0fc2e70e
18 changed files with 593 additions and 296 deletions
+1
View File
@@ -55,6 +55,7 @@ class Config:
self.postfix_reinject_port_incoming = int(
params.pop("postfix_reinject_port_incoming", "10026")
)
self.doveauth_http_port = int(params.pop("doveauth_http_port", "10084"))
self.mtail_address = params.pop("mtail_address", None)
self.disable_ipv6 = params.pop("disable_ipv6", "false").lower() == "true"
self.acme_email = params.pop("acme_email", "")
+110 -96
View File
@@ -1,10 +1,17 @@
import json
"""Create chatmail addresses on first login.
Dovecot only asks us about addresses it does not already find in the mailbox:
the auth.lua we deploy with dovecot (cmdeploy/src/cmdeploy/dovecot/auth.lua.j2)
verifies existing users itself against a mailbox password file,
and HTTP-POSTs everything else to the /create endpoint implemented in this module.
"""
import logging
import os
import re
import sys
import filelock
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
try:
import crypt_r
@@ -12,7 +19,6 @@ except ImportError:
import crypt as crypt_r
from .config import Config, read_config
from .dictproxy import DictProxy
from .migrate_db import migrate_from_db_to_maildir
from .syslimits import has_sufficient_resources
@@ -64,109 +70,117 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
return True
def split_and_unescape(s):
"""Split strings using double quote as a separator and backslash as escape character
into parts."""
out = ""
i = 0
while i < len(s):
c = s[i]
if c == "\\":
# Skip escape character.
i += 1
# This will raise IndexError if there is no character
# after escape character. This is expected
# as this is an invalid input.
out += s[i]
elif c == '"':
# Separator
yield out
out = ""
else:
out += c
i += 1
yield out
def verify_password(stored, cleartext_password) -> bool:
if stored.startswith("{"):
stored = stored.split("}", 1)[1]
return crypt_r.crypt(cleartext_password, stored) == stored
class AuthDictProxy(DictProxy):
class DoveAuth:
def __init__(self, config):
super().__init__()
self.config = config
self.creation_lock = threading.Lock()
def handle_lookup(self, parts):
# Dovecot <2.3.17 has only one part,
# do not attempt to read any other parts for compatibility.
keyname = parts[0]
namespace, type, args = keyname.split("/", 2)
args = list(split_and_unescape(args))
def create_user(self, addr, cleartext_password) -> bool:
"""Create the address, or verify the password if it exists already."""
config = self.config
reply_command = "F"
res = ""
if namespace == "shared":
if type == "userdb":
user = args[0]
if user.endswith(f"@{config.mail_domain}"):
res = self.lookup_userdb(user)
if res:
reply_command = "O"
else:
reply_command = "N"
elif type == "passdb":
user = args[1]
if user.endswith(f"@{config.mail_domain}"):
res = self.lookup_passdb(user, cleartext_password=args[0])
if res:
reply_command = "O"
else:
reply_command = "N"
json_res = json.dumps(res) if res else ""
return f"{reply_command}{json_res}\n"
def handle_iterate(self, parts):
# example: I0\t0\tshared/userdb/
if parts[2] == "shared/userdb/":
result = "".join(
f"Oshared/userdb/{user}\t\n" for user in self.iter_userdb()
)
return f"{result}\n"
def iter_userdb(self) -> list:
"""Get a list of all user addresses."""
return [x for x in os.listdir(self.config.mailboxes_dir) if "@" in x]
def lookup_userdb(self, addr):
return self.config.get_user(addr).get_userdb_dict()
def lookup_passdb(self, addr, cleartext_password):
user = self.config.get_user(addr)
userdata = user.get_userdb_dict()
if userdata:
return userdata
if not is_allowed_to_create(self.config, addr, cleartext_password):
return
if not has_sufficient_resources(self.config):
return
lock = filelock.FileLock(str(user.password_path) + ".lock", timeout=5)
with lock:
userdata = user.get_userdb_dict()
if userdata:
return userdata
if not addr.endswith(f"@{config.mail_domain}"):
logging.warning("address not in mail domain: %r", addr)
return False
try:
user = config.get_user(addr)
except ValueError:
logging.warning("invalid address: %r", addr)
return False
with self.creation_lock:
passhash = user.get_password_hash()
if passhash is not None:
# a concurrent first login may have just created the address
return verify_password(passhash, cleartext_password)
if not is_allowed_to_create(config, addr, cleartext_password):
return False
if not has_sufficient_resources(config):
return False
user.set_password(encrypt_password(cleartext_password))
print(f"Created address: {addr}", file=sys.stderr)
return user.get_userdb_dict()
# mtail counts created_accounts off this exact line
print(f"Created address: {addr}", file=sys.stderr)
return True
class CreateHandler(BaseHTTPRequestHandler):
"""Answer POST /create requests from dovecot's auth.lua, body `addr\\tpassword`.
The body must be UTF-8 and only the first tab separates the fields,
so a password may itself contain tabs.
Any non-UTF8 or \\0 bytes in the body fail the request.
Addresses are ASCII: dovecot refuses any login name outside its
auth_username_chars before auth.lua ever sees it.
Dovecot hands auth.lua the exact password bytes the client sent;
decoding and re-encoding UTF-8 is byte-identical,
so dovecot's password_verify later recomputes the same hash crypt() stores here.
"""
protocol_version = "HTTP/1.1" # dovecot's HTTP client reuses connections
max_body_len = 512 # an address and a password
def do_POST(self):
if self.path != "/create":
self.reply(404)
return
length = self.body_length()
if length is None:
self.reply(400)
return
body = self.rfile.read(length)
try:
addr, _, password = body.decode("utf-8").partition("\t")
except UnicodeDecodeError:
self.reply(400)
return
if "\0" in addr or "\0" in password:
self.reply(400)
return
self.reply(200 if self.server.doveauth.create_user(addr, password) else 403)
def body_length(self):
try:
length = int(self.headers["Content-Length"])
except (TypeError, ValueError):
return None
return length if 0 <= length <= self.max_body_len else None
def reply(self, status):
self.send_response(status)
self.send_header("Content-Length", "0")
if status != 200:
# Just close on any failure, as body might not be fully read.
# It's anyway cheap to re-establish http localhost without TLS.
self.send_header("Connection", "close")
self.end_headers()
def log_message(self, format, *args):
# the per-request access log would only duplicate our own stderr lines
pass
class DoveAuthServer(ThreadingHTTPServer):
# a burst of first-time logins (e.g. from CI) must not overflow
# the accept queue, see https://github.com/chatmail/relay/issues/436
request_queue_size = 1000
def __init__(self, config, port):
super().__init__(("127.0.0.1", port), CreateHandler)
self.doveauth = DoveAuth(config)
def main():
socket, cfgpath = sys.argv[1:]
(cfgpath,) = sys.argv[1:]
config = read_config(cfgpath)
migrate_from_db_to_maildir(config)
dictproxy = AuthDictProxy(config=config)
dictproxy.serve_forever_from_socket(socket)
server = DoveAuthServer(config, config.doveauth_http_port)
server.serve_forever()
@@ -1,6 +1,6 @@
import time
from chatmaild.doveauth import AuthDictProxy
from chatmaild.doveauth import DoveAuth
from chatmaild.expire import daily_expire_main as main_expire
@@ -18,10 +18,10 @@ def test_login_timestamps(example_config):
def test_delete_inactive_users(example_config):
new = time.time()
old = new - (example_config.delete_inactive_users_after * 86400) - 1
dictproxy = AuthDictProxy(example_config)
doveauth = DoveAuth(example_config)
def create_user(addr, last_login):
dictproxy.lookup_passdb(addr, "q9mr3faue")
doveauth.create_user(addr, "q9mr3faue")
user = example_config.get_user(addr)
user.maildir.joinpath("cur").mkdir()
user.maildir.joinpath("cur", "something").mkdir()
+166 -142
View File
@@ -1,42 +1,37 @@
import io
import json
import queue
import http.client
import threading
import traceback
from concurrent.futures import ThreadPoolExecutor
import pytest
import chatmaild.doveauth
from chatmaild.doveauth import (
AuthDictProxy,
CreateHandler,
DoveAuth,
DoveAuthServer,
is_allowed_to_create,
)
from chatmaild.newemail import create_newemail_dict
@pytest.fixture
def dictproxy(example_config):
return AuthDictProxy(config=example_config)
def doveauth(example_config):
return DoveAuth(example_config)
def test_basic(dictproxy, example_gencreds):
def stored_hash(config, addr):
return config.get_user(addr).get_password_hash()
def test_basic(doveauth, example_config, example_gencreds):
addr, password = example_gencreds()
dictproxy.lookup_passdb(addr, password)
data = dictproxy.lookup_userdb(addr)
assert data
data2 = dictproxy.lookup_passdb(addr, password)
assert data == data2
assert doveauth.create_user(addr, password)
passhash = stored_hash(example_config, addr)
assert passhash.startswith("{SHA512-CRYPT}")
def test_iterate_addresses(dictproxy):
addresses = []
for i in range(10):
addresses.append(f"asdf1234{i}@chat.example.org")
dictproxy.lookup_passdb(addresses[-1], "q9mr3faue")
res = dictproxy.iter_userdb()
assert set(res) == set(addresses)
# a second login verifies against the stored hash and rewrites nothing
assert doveauth.create_user(addr, password)
assert stored_hash(example_config, addr) == passhash
def test_invalid_username_length(example_config):
@@ -53,75 +48,32 @@ def test_invalid_username_length(example_config):
)
def test_dont_overwrite_password_on_wrong_login(dictproxy):
"""Test that logging in with a different password doesn't create a new user"""
res = dictproxy.lookup_passdb(
"newuser12@chat.example.org", "kajdlkajsldk12l3kj1983"
)
assert res["password"]
res2 = dictproxy.lookup_passdb("newuser12@chat.example.org", "kajdslqwe")
# this function always returns a password hash, which is actually compared by dovecot.
assert res["password"] == res2["password"]
def test_dont_overwrite_password_on_wrong_login(doveauth, example_config):
addr = "newuser12@chat.example.org"
assert doveauth.create_user(addr, "kajdlkajsldk12l3kj1983")
passhash = stored_hash(example_config, addr)
assert not doveauth.create_user(addr, "kajdslqwe")
assert stored_hash(example_config, addr) == passhash
assert doveauth.create_user(addr, "kajdlkajsldk12l3kj1983")
assert stored_hash(example_config, addr) == passhash
def test_nocreate_file(monkeypatch, tmpdir, dictproxy):
def test_foreign_domain_is_refused(doveauth):
assert not doveauth.create_user("newuser12@evil.example.org", "qlwkejqlwe12")
def test_nocreate_file(monkeypatch, tmpdir, doveauth, example_config):
p = tmpdir.join("nocreate")
p.write("")
monkeypatch.setattr(chatmaild.doveauth, "NOCREATE_FILE", str(p))
dictproxy.lookup_passdb("newuser12@chat.example.org", "zequ0Aimuchoodaechik")
assert not dictproxy.lookup_userdb("newuser12@chat.example.org")
def test_handle_dovecot_request(dictproxy):
transactions = {}
# Test that password can contain ", ', \ and /
msg = (
'Lshared/passdb/laksjdlaksjdlak\\\\sjdlk\\"12j\\\'3l1/k2j3123"'
"some42123@chat.example.org\tsome42123@chat.example.org"
)
res = dictproxy.handle_dovecot_request(msg, transactions)
assert res
assert res[0] == "O" and res.endswith("\n")
userdata = json.loads(res[1:].strip())
assert userdata["home"].endswith("chat.example.org/some42123@chat.example.org")
assert userdata["uid"] == userdata["gid"] == "vmail"
assert userdata["password"].startswith("{SHA512-CRYPT}")
def test_handle_dovecot_protocol_hello_is_skipped(example_config, caplog):
dictproxy = AuthDictProxy(config=example_config)
rfile = io.BytesIO(b"H3\t2\t0\t\tauth\n")
wfile = io.BytesIO()
dictproxy.loop_forever(rfile, wfile)
assert wfile.getvalue() == b""
assert not caplog.messages
def test_handle_dovecot_protocol_user_not_exists(example_config):
dictproxy = AuthDictProxy(config=example_config)
rfile = io.BytesIO(
b"H3\t2\t0\t\tauth\nLshared/userdb/foobar@chat.example.org\tfoobar@chat.example.org\n"
)
wfile = io.BytesIO()
dictproxy.loop_forever(rfile, wfile)
assert wfile.getvalue() == b"N\n"
def test_handle_dovecot_protocol_iterate(example_config):
dictproxy = AuthDictProxy(config=example_config)
dictproxy.lookup_passdb("asdf00000@chat.example.org", "q9mr3faue")
dictproxy.lookup_passdb("asdf11111@chat.example.org", "q9mr3faue")
rfile = io.BytesIO(b"H3\t2\t0\t\tauth\nI0\t0\tshared/userdb/")
wfile = io.BytesIO()
dictproxy.loop_forever(rfile, wfile)
lines = wfile.getvalue().decode("ascii").split("\n")
assert "Oshared/userdb/asdf00000@chat.example.org\t" in lines
assert "Oshared/userdb/asdf11111@chat.example.org\t" in lines
assert not lines[2]
addr = "newuser12@chat.example.org"
assert not doveauth.create_user(addr, "zequ0Aimuchoodaechik")
assert stored_hash(example_config, addr) is None
def test_invalid_localpart_characters(make_config):
"""Test that is_allowed_to_create rejects localparts with invalid characters."""
config = make_config("chat.example.org", {"username_min_length": "3"})
password = "zequ0Aimuchoodaechik"
domain = config.mail_domain
@@ -141,78 +93,150 @@ def test_invalid_localpart_characters(make_config):
assert not is_allowed_to_create(config, f"ab@cdef@{domain}", password)
assert not is_allowed_to_create(config, f"abc/def@{domain}", password)
assert not is_allowed_to_create(config, f"abc\\def@{domain}", password)
assert not is_allowed_to_create(config, f"üser123@{domain}", password)
def test_concurrent_creation_same_account(dictproxy):
"""Test that concurrent creation of the same account doesn't corrupt password."""
def test_concurrent_creation_same_account(doveauth, example_config, capsys):
addr = "racetest1@chat.example.org"
password = "zequ0Aimuchoodaechik"
num_threads = 10
results = queue.Queue()
def create():
try:
res = dictproxy.lookup_passdb(addr, password)
results.put(("ok", res))
except Exception:
results.put(("err", traceback.format_exc()))
threads = [threading.Thread(target=create, daemon=True) for _ in range(num_threads)]
for t in threads:
t.start()
for t in threads:
t.join(timeout=10)
passwords_seen = set()
for _ in range(num_threads):
status, res = results.get()
if status == "err":
pytest.fail(f"concurrent creation failed\n{res}")
passwords_seen.add(res["password"])
def create(_):
ok = doveauth.create_user(addr, password)
return ok, stored_hash(example_config, addr)
with ThreadPoolExecutor(10) as pool:
results = list(pool.map(create, range(10)))
assert all(ok for ok, _ in results)
# all threads must see the same password hash
assert len(passwords_seen) == 1
def test_50_concurrent_lookups_different_accounts(example_gencreds, dictproxy):
num_threads = 50
req_per_thread = 5
results = queue.Queue()
def lookup():
for i in range(req_per_thread):
addr, password = example_gencreds()
try:
dictproxy.lookup_passdb(addr, password)
except Exception:
results.put(traceback.format_exc())
else:
results.put(None)
threads = []
for i in range(num_threads):
thread = threading.Thread(target=lookup, daemon=True)
threads.append(thread)
print(f"created {num_threads} threads, starting them and waiting for results")
for thread in threads:
thread.start()
for i in range(num_threads * req_per_thread):
res = results.get()
if res is not None:
pytest.fail(f"concurrent lookup failed\n{res}")
assert len({passhash for _, passhash in results}) == 1
assert capsys.readouterr().err.count("Created address:") == 1
def test_insufficient_resources_block_creation_not_existing_logins(
dictproxy, example_gencreds, monkeypatch
doveauth, example_gencreds, monkeypatch
):
addr, password = example_gencreds()
assert dictproxy.lookup_passdb(addr, password)
assert doveauth.create_user(addr, password)
monkeypatch.setattr(
chatmaild.doveauth, "has_sufficient_resources", lambda config: False
)
newaddr, newpassword = example_gencreds()
assert not dictproxy.lookup_passdb(newaddr, newpassword)
assert dictproxy.lookup_passdb(addr, password)
assert not doveauth.create_user(newaddr, newpassword)
assert doveauth.create_user(addr, password)
class TestHttpPost:
@pytest.fixture
def doveauth_server(self, example_config):
server = DoveAuthServer(example_config, port=0)
threading.Thread(target=server.serve_forever, daemon=True).start()
yield f"127.0.0.1:{server.server_address[1]}"
server.shutdown()
server.server_close()
@pytest.fixture
def post(self, doveauth_server):
def post(path, data):
conn = http.client.HTTPConnection(doveauth_server, timeout=10)
try:
return self.post_on(conn, path, data).status
finally:
conn.close()
return post
@pytest.fixture
def connection(self, doveauth_server):
"""One kept-alive connection, which is all dovecot's HTTP client opens."""
conn = http.client.HTTPConnection(doveauth_server, timeout=10)
yield conn
conn.close()
@staticmethod
def post_on(conn, path, data):
conn.request("POST", path, body=data)
resp = conn.getresponse()
resp.read()
return resp
def test_create_and_verify(self, post, example_config, example_gencreds):
addr, password = example_gencreds()
assert post("/create", f"{addr}\t{password}".encode()) == 200
assert stored_hash(example_config, addr).startswith("{SHA512-CRYPT}")
# second login with the same password verifies, a wrong one is refused
assert post("/create", f"{addr}\t{password}".encode()) == 200
assert post("/create", f"{addr}\twrong{password}".encode()) == 403
def test_password_special_chars_survive_transport(self, post, example_gencreds):
addr, _ = example_gencreds()
password = "laksjdlaksjdlak\\sjdlk\"12j'3l1/k2\tj3123"
body = f"{addr}\t{password}".encode()
assert post("/create", body) == 200
assert post("/create", body) == 200
assert post("/create", f"{addr}\totherpassword1".encode()) == 403
def test_password_must_be_utf8(self, post, example_gencreds):
addr, _ = example_gencreds()
assert post("/create", f"{addr}\tpässwort12".encode()) == 200
assert post("/create", addr.encode() + b"\tp\xe4sswort12") == 400
def test_nul_is_refused_before_crypt_sees_it(self, post, example_gencreds):
addr, _ = example_gencreds()
assert post("/create", f"{addr}\tpass\0word12".encode()) == 400
assert (
post("/create", "us\0er12345@chat.example.org\tlongenough1".encode()) == 400
)
def test_refused_creation(self, post, example_gencreds):
addr, _ = example_gencreds()
assert post("/create", f"{addr}\tshort".encode()) == 403
assert post("/create", b"not-an-address\tlongenoughpassword") == 403
body = "bürger123@chat.example.org\tlongenoughpw".encode()
assert post("/create", body) == 403
assert post("/create", b"") == 403
def test_body_length_limit(self, post, example_gencreds):
addr, _ = example_gencreds()
fill = CreateHandler.max_body_len - len(addr) - len("\t")
body = f"{addr}\t{'x' * fill}".encode()
assert len(body) == CreateHandler.max_body_len
assert post("/create", body) == 200
body = f"{addr}\t{'x' * (fill + 1)}".encode()
assert len(body) == CreateHandler.max_body_len + 1
assert post("/create", body) == 400
def test_connection_is_reused_across_200_replies(
self, connection, example_gencreds
):
addr, password = example_gencreds()
body = f"{addr}\t{password}".encode()
# create, then verify the same password, on one connection
for _ in range(2):
resp = self.post_on(connection, "/create", body)
assert (resp.status, resp.will_close) == (200, False)
@pytest.mark.parametrize(
"path,data,status",
[
("/other", b"not read", 404),
("/create", b"x" * (CreateHandler.max_body_len + 1), 400),
("/create", b"not-an-address\tlongenoughpassword", 403),
],
)
def test_error_replies_close_the_connection(self, connection, path, data, status):
resp = self.post_on(connection, path, data)
assert (resp.status, resp.will_close) == (status, True)
@pytest.mark.parametrize("content_length", [None, "-1", "notanumber", "999999"])
def test_bad_content_length(self, connection, content_length):
# the fixture timeout turns a server that waits for the body into a failure
connection.putrequest("POST", "/create", skip_accept_encoding=True)
if content_length is not None:
connection.putheader("Content-Length", content_length)
connection.endheaders()
resp = connection.getresponse()
assert resp.status == 400
assert resp.will_close
@@ -1,6 +1,6 @@
import time
from chatmaild.doveauth import AuthDictProxy
from chatmaild.doveauth import DoveAuth
from chatmaild.lastlogin import (
LastLoginDictProxy,
)
@@ -9,8 +9,8 @@ from chatmaild.lastlogin import (
def test_handle_dovecot_request_last_login(testaddr, example_config):
dictproxy = LastLoginDictProxy(config=example_config)
authproxy = AuthDictProxy(config=example_config)
authproxy.lookup_passdb(testaddr, "1l2k3j1l2k3jl123")
doveauth = DoveAuth(example_config)
doveauth.create_user(testaddr, "1l2k3j1l2k3jl123")
dictproxy_transactions = {}
@@ -63,7 +63,7 @@ def test_migration(tmp_path, example_config, caplog):
user = example_config.get_user(path.name)
if last_login:
assert user.get_last_login_timestamp() == last_login
assert password == user.get_userdb_dict()["password"]
assert password == user.get_password_hash()
assert not all
assert not example_config.passdb_path.exists()
+6 -11
View File
@@ -8,28 +8,23 @@ def test_login_timestamp(testaddr, example_config):
assert user.get_last_login_timestamp() == 86400 * 2
def test_get_user_dict_not_set(testaddr, example_config, caplog):
def test_get_password_hash_not_set(testaddr, example_config, caplog):
user = example_config.get_user(testaddr)
assert not caplog.records
assert user.get_userdb_dict() == {}
assert user.get_password_hash() is None
assert len(caplog.records) == 0
user.set_password("")
assert user.get_userdb_dict() == {}
assert user.get_password_hash() is None
assert len(caplog.records) == 1
def test_get_user_dict(make_config, tmp_path):
def test_get_password_hash(make_config, tmp_path):
config = make_config("something.testrun.org")
addr = "user1@something.org"
user = config.get_user(addr)
user = config.get_user("user1@something.org")
enc_password = "l1k2j31lk2j3l1k23j123"
user.set_password(enc_password)
data = user.get_userdb_dict()
assert addr in str(data["home"])
assert data["uid"] == "vmail"
assert data["gid"] == "vmail"
assert data["password"] == enc_password
assert user.get_password_hash() == enc_password
def test_no_mailboxes_dir(testaddr, example_config, tmp_path):
+6 -9
View File
@@ -21,20 +21,17 @@ class User:
def can_track(self):
return "@" in self.addr
def get_userdb_dict(self):
"""Return a non-empty dovecot 'userdb' style dict
if the user has an existing non-empty password"""
def get_password_hash(self):
try:
pw = self.password_path.read_text()
passhash = self.password_path.read_text()
except FileNotFoundError:
return {}
return None
if not pw:
if not passhash:
logging.error(f"password is empty for: {self.addr}")
return {}
return None
home = str(self.maildir)
return dict(addr=self.addr, home=home, uid=self.uid, gid=self.gid, password=pw)
return passhash
def is_incoming_cleartext_ok(self):
return not self.enforce_E2EE_path.exists()