feat: move doveauth from dictproxy to lua/http

1. existing logins are now verified by lua only

2. non-existing logins are delegated to the new Python doveauth http /create endpoint

Using Lua and http this way makes doveauth more compatible to dovecot 2.4

cmdeploy/src/cmdeploy/dovecot/auth.conf

@@ -1,12 +0,0 @@
-uri = proxy:/run/doveauth/doveauth.socket:auth
-iterate_disable = no
-iterate_prefix = userdb/
-
-default_pass_scheme = plain
-# %E escapes characters " (double quote), ' (single quote) and \ (backslash) with \ (backslash).
-# See <https://doc.dovecot.org/2.3/configuration_manual/config_file/config_variables/#modifiers>
-# for documentation.
-#
-# We escape user-provided input and use double quote as a separator.
-password_key = passdb/%Ew"%Eu
-user_key = userdb/%Eu

cmdeploy/src/cmdeploy/dovecot/auth.lua.j2

@@ -0,0 +1,64 @@
+-- Existing addresses are served from the maildir directly.
+-- Unknown ones are offered to doveauth, which owns the creation policy.
+local mailboxes_dir = "{{ config.mailboxes_dir }}"
+local domain_suffix = "@{{ config.mail_domain }}"
+local create_url = "http://127.0.0.1:{{ config.doveauth_http_port }}/create"
+
+local http_client
+
+local function is_ours(user)
+  return user:sub(-#domain_suffix) == domain_suffix
+     and not user:find("/", 1, true)
+end
+
+local function password_hash(user)
+  local fh = io.open(mailboxes_dir .. "/" .. user .. "/password", "r")
+  if not fh then
+    return nil
+  end
+  local hash, rest = fh:read("l", "a")
+  fh:close()
+  if hash == nil or hash == "" or rest ~= "" then
+    return nil
+  end
+  return hash
+end
+
+local function userdb_fields(user)
+  return {home = mailboxes_dir .. "/" .. user, uid = "vmail", gid = "vmail"}
+end
+
+local function create(user, password)
+  local request = http_client:request({url = create_url, method = "POST"})
+  request:set_payload(user .. "\t" .. password)
+  return request:submit():status() == 200
+end
+
+-- Entry points called by dovecot
+
+function script_init()
+  http_client = dovecot.http.client({request_timeout_msecs = 5000, max_attempts = 1})
+  return 0
+end
+
+function auth_userdb_lookup(req)
+  if not is_ours(req.user) or password_hash(req.user) == nil then
+    return dovecot.auth.USERDB_RESULT_USER_UNKNOWN, {}
+  end
+  return dovecot.auth.USERDB_RESULT_OK, userdb_fields(req.user)
+end
+
+function auth_password_verify(req, password)
+  if not is_ours(req.user) then
+    return dovecot.auth.PASSDB_RESULT_USER_UNKNOWN, {}
+  end
+  local hash = password_hash(req.user)
+  if hash == nil then
+    if not create(req.user, password) then
+      return dovecot.auth.PASSDB_RESULT_USER_UNKNOWN, {}
+    end
+  elseif req:password_verify(hash, password) ~= 1 then
+    return dovecot.auth.PASSDB_RESULT_PASSWORD_MISMATCH, {}
+  end
+  return dovecot.auth.PASSDB_RESULT_OK, userdb_fields(req.user)
+end

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -31,7 +31,7 @@ class DovecotDeployer(Deployer):
         arch = host.get_fact(Arch)
         with blocked_service_startup():
             debs = []
-            for pkg in ("core", "imapd", "lmtpd"):
+            for pkg in ("core", "imapd", "lmtpd", "auth-lua"):
                 deb, changed = _download_dovecot_package(pkg, arch)
                 self.need_restart |= changed
                 if deb:
@@ -134,7 +134,8 @@ def _configure_dovecot(deployer, config: Config, debug: bool = False):
         debug=debug,
         disable_ipv6=config.disable_ipv6,
     )
-    deployer.put_file("dovecot/auth.conf", "/etc/dovecot/auth.conf")
+    deployer.put_template("dovecot/auth.lua.j2", "/etc/dovecot/auth.lua", config=config)
+    deployer.remove_file("/etc/dovecot/auth.conf")
     deployer.put_file(
         "dovecot/push_notification.lua", "/etc/dovecot/push_notification.lua"
     )

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -61,12 +61,12 @@ imap_capability = +XDELTAPUSH XCHATMAIL
 
 # Authentication for system users.
 passdb {
-  driver = dict
-  args = /etc/dovecot/auth.conf
+  driver = lua
+  args = file=/etc/dovecot/auth.lua blocking=yes
 }
 userdb {
-  driver = dict
-  args = /etc/dovecot/auth.conf
+  driver = lua
+  args = file=/etc/dovecot/auth.lua blocking=yes
 }
 ##
 ## Mailbox locations and namespaces

cmdeploy/src/cmdeploy/pins.py

@@ -36,6 +36,8 @@ DOVECOT_SHA256 = {
     ("imapd", "arm64"): "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f",
     ("lmtpd", "amd64"): "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab",
     ("lmtpd", "arm64"): "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f",
+    ("auth-lua", "amd64"): "d724f37712faba52e177153114af1831e54da555c57c6474c05f96f176176ce4",
+    ("auth-lua", "arm64"): "7272768e20de148c35891d99cd60205acbe7e732b056caf0a83e8194d49136e6",
 }
 TURN_VERSION = "v0.4"
 TURN_ARTIFACTS = {

cmdeploy/src/cmdeploy/service/doveauth.service.f

@@ -1,12 +1,11 @@
 [Unit]
-Description=Chatmail dict authentication proxy for dovecot
+Description=Chatmail HTTP authentication service for dovecot
 
 [Service]
-ExecStart={execpath} /run/doveauth/doveauth.socket {config_path}
+ExecStart={execpath} {config_path}
 Restart=always
-RestartSec=30
+RestartSec=5
 User=vmail
-RuntimeDirectory=doveauth
 UMask=0077
 
 [Install]

cmdeploy/src/cmdeploy/tests/test_authlua.py

@@ -0,0 +1,206 @@
+"""Test auth.lua script against mocked dovecot auth API."""
+
+import jinja2
+import pytest
+from chatmaild.doveauth import encrypt_password, verify_password
+
+from cmdeploy.basedeploy import get_resource
+
+USER1 = "user12345@chat.example.org"
+USER2 = "newuser12@chat.example.org"
+OK, UNKNOWN, MISMATCH = 1, -2, -3
+
+DOVECOT_MOCKS = """
+create_status = 200
+
+dovecot = {
+  auth = {
+    PASSDB_RESULT_OK = OK,
+    PASSDB_RESULT_USER_UNKNOWN = UNKNOWN,
+    PASSDB_RESULT_PASSWORD_MISMATCH = MISMATCH,
+    USERDB_RESULT_OK = OK,
+    USERDB_RESULT_USER_UNKNOWN = UNKNOWN,
+  },
+  http = {
+    client = function(options)
+      client_options = options
+      return {request = function(_, options)
+        create_request = options
+        return {
+          set_payload = function(_, payload) create_payload = payload end,
+          submit = function()
+            return {status = function() return create_status end}
+          end,
+        }
+      end}
+    end,
+  },
+}
+"""
+
+
+def load_authlua(lua, config):
+    lua.g.OK, lua.g.UNKNOWN, lua.g.MISMATCH = OK, UNKNOWN, MISMATCH
+    lua.rt.execute(DOVECOT_MOCKS)
+    template = jinja2.Template(get_resource("dovecot/auth.lua.j2").read_text())
+    lua.rt.execute(template.render(config=config))
+    assert lua.g.script_init() == 0
+    return lua
+
+
+@pytest.fixture
+def authlua(lua, example_config):
+    return load_authlua(lua, example_config)
+
+
+@pytest.fixture
+def request_for(lua):
+    def request_for(addr):
+        def password_verify(_self, hashed, plain):
+            return 1 if verify_password(hashed, plain) else 0
+
+        return lua.table(user=addr, password_verify=password_verify)
+
+    return request_for
+
+
+@pytest.fixture
+def create_user(example_config):
+    def create_user(addr, password):
+        example_config.get_user(addr).set_password(encrypt_password(password))
+
+    return create_user
+
+
+@pytest.fixture
+def write_password_file(example_config):
+    def write_password_file(addr, content):
+        maildir = example_config.mailboxes_dir / addr
+        maildir.mkdir(parents=True, exist_ok=True)
+        maildir.joinpath("password").write_text(content)
+
+    return write_password_file
+
+
+def test_http_client_uses_dovecot_setting_names(authlua):
+    """dovecot's lua http binding silently ignores keys it does not know."""
+    assert dict(authlua.g.client_options) == {
+        "request_timeout_msecs": 5000,
+        "max_attempts": 1,
+    }
+
+
+def test_existing_address_correct_password(authlua, request_for, create_user):
+    create_user(USER1, "correctgoose")
+    res, fields = authlua.g.auth_password_verify(request_for(USER1), "correctgoose")
+    assert res == OK
+    assert fields["uid"] == fields["gid"] == "vmail"
+    assert fields["home"].endswith(USER1)
+    assert authlua.g.create_payload is None
+
+
+def test_existing_address_wrong_password(authlua, request_for, create_user):
+    create_user(USER1, "correctgoose")
+    res, _ = authlua.g.auth_password_verify(request_for(USER1), "wronghorse")
+    assert res == MISMATCH
+
+
+def test_foreign_domain_is_refused_without_calling_out(
+    authlua, request_for, create_user
+):
+    create_user("user12345@evil.example.org", "correctgoose")
+    request = request_for("user12345@evil.example.org")
+    res, _ = authlua.g.auth_password_verify(request, "correctgoose")
+    assert res == UNKNOWN
+    assert authlua.g.auth_userdb_lookup(request)[0] == UNKNOWN
+    assert authlua.g.create_payload is None
+
+
+def test_name_shorter_than_the_domain_is_refused(authlua, request_for):
+    for name in ("x", "", "chat.example.org"):
+        res, _ = authlua.g.auth_password_verify(request_for(name), "correctgoose")
+        assert res == UNKNOWN
+        assert authlua.g.auth_userdb_lookup(request_for(name))[0] == UNKNOWN
+    assert authlua.g.create_payload is None
+
+
+def test_slash_in_username_is_refused(authlua, request_for):
+    request = request_for("../../etc/shadow@chat.example.org")
+    res, _ = authlua.g.auth_password_verify(request, "somepassword")
+    assert res == UNKNOWN
+    assert authlua.g.auth_userdb_lookup(request)[0] == UNKNOWN
+    assert authlua.g.create_payload is None
+
+
+def test_localpart_policy_is_left_to_doveauth(authlua, request_for):
+    authlua.g.create_status = 403
+    res, _ = authlua.g.auth_password_verify(request_for("@chat.example.org"), "somepw")
+    assert res == UNKNOWN
+    assert authlua.g.create_payload == "@chat.example.org\tsomepw"
+
+
+def test_unknown_address_is_created_via_endpoint(authlua, request_for):
+    res, fields = authlua.g.auth_password_verify(request_for(USER2), "brandnewpass")
+    assert res == OK
+    assert fields["home"].endswith(USER2)
+    assert authlua.g.create_payload == f"{USER2}\tbrandnewpass"
+    assert authlua.g.create_request["url"] == "http://127.0.0.1:10084/create"
+
+
+# a policy refusal and a doveauth that is down are both fail-closed
+@pytest.mark.parametrize("status", [403, 0])
+def test_creation_that_is_not_answered_with_200_is_user_unknown(
+    authlua, request_for, status
+):
+    authlua.g.create_status = status
+    res, _ = authlua.g.auth_password_verify(request_for(USER2), "brandnewpass")
+    assert res == UNKNOWN
+
+
+def test_userdb_unknown_before_creation_ok_after(authlua, request_for, create_user):
+    request = request_for(USER1)
+    res, _ = authlua.g.auth_userdb_lookup(request)
+    assert res == UNKNOWN
+    # a userdb lookup must never create anything
+    assert authlua.g.create_payload is None
+
+    create_user(USER1, "correctgoose")
+    res, fields = authlua.g.auth_userdb_lookup(request)
+    assert res == OK
+    assert fields["home"].endswith(USER1)
+    assert fields["uid"] == fields["gid"] == "vmail"
+
+
+def test_empty_password_file_is_unknown(authlua, request_for, write_password_file):
+    write_password_file(USER1, "")
+    assert authlua.g.auth_userdb_lookup(request_for(USER1))[0] == UNKNOWN
+
+    write_password_file(USER1, "\n")
+    assert authlua.g.auth_userdb_lookup(request_for(USER1))[0] == UNKNOWN
+
+
+def test_password_file_format_checks(authlua, request_for, write_password_file):
+    write_password_file(USER1, encrypt_password("correctgoose") + "\n")
+    res, _ = authlua.g.auth_password_verify(request_for(USER1), "correctgoose")
+    assert res == OK
+    assert authlua.g.auth_userdb_lookup(request_for(USER1))[0] == OK
+
+    passhash = encrypt_password("correctgoose")
+    write_password_file(USER1, passhash + "\ntrailing junk")
+    authlua.g.create_status = 403
+    res, _ = authlua.g.auth_password_verify(request_for(USER1), "correctgoose")
+    assert res == UNKNOWN
+    assert authlua.g.auth_userdb_lookup(request_for(USER1))[0] == UNKNOWN
+
+
+def test_ipv4_relay_uses_bracketed_domain(lua, ipv4_config, request_for):
+    # mail_domain is "[1.3.3.7]" here, and is_ours must not read it as a pattern
+    authlua = load_authlua(lua, ipv4_config)
+    addr = f"user12345@{ipv4_config.mail_domain}"
+    ipv4_config.get_user(addr).set_password(encrypt_password("correctgoose"))
+
+    res, fields = authlua.g.auth_password_verify(request_for(addr), "correctgoose")
+    assert res == OK
+    assert fields["home"].endswith(addr)
+    assert authlua.g.auth_userdb_lookup(request_for(addr))[0] == OK
+    assert authlua.g.auth_userdb_lookup(request_for(USER1))[0] == UNKNOWN

cmdeploy/src/cmdeploy/tests/test_dovecot_deployer.py

@@ -136,6 +136,7 @@ def test_install_skips_dpkg_path_when_epoch_matched_packages_present(
                     "dovecot-core": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
                     "dovecot-imapd": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
                     "dovecot-lmtpd": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
+                    "dovecot-auth-lua": [dovecot_deployer.DOVECOT_PACKAGE_VERSION],
                 },
             ),
             (dovecot_deployer.Arch, "x86_64"),
@@ -180,9 +181,12 @@ def test_install_unsupported_arch_falls_back_to_apt(
     deployer.install()
 
     actual_pkgs = [c["packages"] for c in apt_calls]
-    assert actual_pkgs == [["dovecot-core"], ["dovecot-imapd"], ["dovecot-lmtpd"]], (
-        f"expected apt install of core/imapd/lmtpd, got {actual_pkgs}"
-    )
+    assert actual_pkgs == [
+        ["dovecot-core"],
+        ["dovecot-imapd"],
+        ["dovecot-lmtpd"],
+        ["dovecot-auth-lua"],
+    ], f"expected apt install of core/imapd/lmtpd/auth-lua, got {actual_pkgs}"
     assert track_shell == [], "should not run dpkg for unsupported arch"
     assert deployer.need_restart is True, (
         "need_restart should be True when apt installed a package"