some fixes to try running on old python versions

README.md

@@ -4,58 +4,7 @@ This package deploys Postfix and Dovecot servers, including OpenDKIM for DKIM si
 
 Postfix uses Dovecot for authentication as described in <https://www.postfix.org/SASL_README.html#server_dovecot>
 
-## Getting started 
+## Ports
-
-prepare:
-
-    pip install -e chatmail-infra
-
-
-then run with pyinfra command line tool: 
-
-    CHATMAIL_DOMAIN=c1.testrun.org pyinfra --ssh-user root c1.testrun.org deploy.py
-
-
-## Structure (wip)
-```
-
-# package doveauth tool and deploy chatmail server to a envvar-specified ssh-reachable host 
-deploy.py 
-
-# chatmail pyinfra deploy package 
-chatmail-pyinfra 
-    pyproject.toml
-    chatmail/__init__ ...
-
-    # tests against the deployed system 
-    tests/test_online_test.py
-
-# doveauth tool used by dovecot's auth mechanism on the host system 
-doveauth
-    README.md
-    pyproject.toml
-    doveauth.py
-    doveauth.lua
-    test_doveauth.py
-
-# lmtp server to block (outgoing) unencrypted messages 
-filtermail 
-    README.md
-    pyproject.toml
-    .... 
-
-
-# scripts for setup/development/deployment 
-
-scripts/
-    init.sh  # create venv/other perequires
-    deploy.sh  # run pyinfra based deploy of everything
-
-```
-
-## Dovecot/Postfix configuration
-
-### Ports
 
 Postfix listens on ports 25 (smtp) and 587 (submission) and 465 (submissions).
 Dovecot listens on ports 143(imap) and 993 (imaps).
@@ -63,3 +12,9 @@ Dovecot listens on ports 143(imap) and 993 (imaps).
 ## DNS
 
 For DKIM you must add a DNS entry as in /etc/opendkim/selector.txt (where selector is the opendkim_selector configured in the chatmail inventory).
+
+## Run with pyinfra
+
+```
+CHATMAIL_DOMAIN=c1.testrun.org pyinfra c1.testrun.org deploy.py
+```

chatmail-pyinfra/pyproject.toml

@@ -1,30 +0,0 @@
-[build-system]
-requires = ["setuptools>=45"]
-build-backend = "setuptools.build_meta"
-
-[project]
-name = "chatmail"
-version = "0.1"
-dependencies = [
-  "pyinfra",
-]
-
-[tool.pytest.ini_options]
-addopts = "-v -ra --strict-markers"
-
-[tool.tox]
-legacy_tox_ini = """
-[tox]
-isolated_build = true
-envlist = lint
-
-[testenv:lint]
-skipdist = True
-skip_install = True
-deps =
-  ruff
-  black
-commands =
-  black --quiet --check --diff src/
-  ruff src/
-"""

chatmail-pyinfra/src/chatmail/dovecot/doveauth.lua

@@ -1,57 +0,0 @@
-
--- Escape shell argument by hex encoding it and wrapping in quotes.
-function escape(data)
-   b16 = data:gsub(".", function(char) return string.format("%2X", char:byte()) end)
-   return ("'"..b16.."'")
-end
-
--- call out to python program to actually manage authentication for dovecot
-
-function chatctl_verify(user, password)
-    local cmd = "python3 /home/vmail/chatctl hexauth "..escape(user).." "..escape(password)
-    print("executing: "..cmd)
-    local handle = io.popen(cmd)
-    local result = handle:read("*a")
-    handle:close()
-    return split_chatctl(result)
-end
-
-function chatctl_lookup(user) 
-    assert(user)
-    local handle = io.popen("python3 /home/vmail/chatctl hexlookup "..escape(user))
-    local result = handle:read("*a")
-    handle:close()
-    return split_chatctl(result)
-end
-
-function get_extra_dovecot_output(res)
-    return {home=res.home, uid=res.uid, gid=res.gid}
-end
-
-
-function auth_password_verify(request, password)
-    local res = chatctl_verify(request.user, password)
-    -- request:log_error("auth_password_verify "..request.user.." "..password)
-    if res.status == "ok" then 
-        local extra = get_extra_dovecot_output(res)
-        return dovecot.auth.PASSDB_RESULT_OK, get_extra_dovecot_output(res)
-    end
-    return dovecot.auth.PASSDB_RESULT_PASSWORD_MISMATCH, ""
-end
-
-
-function auth_userdb_lookup(request)
-    local res = chatctl_lookup(request.user) 
-    if res.status == "ok" then
-        return dovecot.auth.USERDB_RESULT_OK, get_extra_dovecot_output(res)
-    end
-    return dovecot.auth.USERDB_RESULT_USER_UNKNOWN, "no such user"
-end
-
-function split_chatctl(output) 
-    local ret = {}
-    for key, value in output:gmatch "(%w+)%s*=%s*(%w+)" do
-        ret[key] = value
-    end
-    return ret
-end

chatmail-pyinfra/src/chatmail/dovecot/doveauth.py

@@ -1,57 +0,0 @@
-#!/usr/bin/env python3
-import base64
-import sys
-
-
-def get_user_data(user):
-    if user == "link2xt@c1.testrun.org":
-        return dict(
-            uid="vmail",
-            gid="vmail",
-            password="Ahyei6ie",
-        )
-    return {}
-
-
-def create_user(user, password):
-    return dict(home=f"/home/vmail/{user}", uid="vmail", gid="vmail", password=password)
-
-
-def verify_user(user, password):
-    userdata = get_user_data(user)
-    if userdata:
-        if userdata.get("password") == password:
-            userdata["status"] = "ok"
-        else:
-            userdata["status"] = "fail"
-    else:
-        userdata = create_user(user, password)
-        userdata["status"] = "ok"
-
-    return userdata
-
-
-def lookup_user(user):
-    userdata = get_user_data(user)
-    if userdata:
-        userdata["status"] = "ok"
-    else:
-        userdata["status"] = "fail"
-    return userdata
-
-
-def dump_result(res):
-    for key, value in res.items():
-        print(f"{key}={value}")
-
-
-if __name__ == "__main__":
-    if sys.argv[1] == "hexauth":
-        login = base64.b16decode(sys.argv[2]).decode()
-        password = base64.b16decode(sys.argv[3]).decode()
-        res = verify_user(login, password)
-        dump_result(res)
-    elif sys.argv[1] == "hexlookup":
-        login = base64.b16decode(sys.argv[2]).decode()
-        res = lookup_user(login)
-        dump_result(res)

chatmail-pyinfra/src/chatmail/dovecot/test_doveauth.lua

@@ -1,78 +0,0 @@
-
-require "doveauth"
-
--- simulate dovecot defined result codes 
-
-dovecot = {
-    auth = {
-        PASSDB_RESULT_OK="PASSWORD-OK",
-        PASSDB_RESULT_PASSWORD_MISMATCH="PASSWORD-MISMATCH",
-        USERDB_RESULT_OK="USERDB-OK",
-        USERDB_RESULT_USER_UNKNOWN="USERDB-UNKNOWN"
-    }
-}
-
-
--- Tests for testing the lua<->python interaction 
-
-function test_password_verify_ok(user, password) 
-    local res, extra = auth_password_verify({user=user}, password)
-    assert(res==dovecot.auth.PASSDB_RESULT_OK)
-    assert(extra.uid == "vmail")
-    assert(extra.gid == "vmail")
-    -- assert(extra.homedir == "/home/vmail/link2xt")
-    print("OK test_password_verify_ok "..user.." "..password)
-end
-
-function test_password_verify_mismatch(user, password) 
-    local res = auth_password_verify({user=user}, password)
-    assert(res == dovecot.auth.PASSDB_RESULT_PASSWORD_MISMATCH)
-    print("OK test_password_verify_mismatch "..user.." "..password)
-end
-
-function test_userdb_lookup_ok(user)
-    local res, extra = auth_userdb_lookup({user=user})
-    assert(extra.uid == "vmail")
-    assert(extra.gid == "vmail")
-    assert(res == dovecot.auth.USERDB_RESULT_OK)
-    print("OK test_userdb_lookup_ok "..user)
-end
-
-function test_userdb_lookup_mismatch(user)
-    local res, extra = auth_userdb_lookup({user=user})
-    assert(res == dovecot.auth.USERDB_RESULT_USER_UNKNOWN)
-    print("OK test_userdb_lookup_mismatch "..user)
-end
-
-function test_passdb_lookup_ok(user)
-    local res, extra = auth_passdb_lookup({user=user})
-    assert(extra.uid == "vmail")
-    assert(extra.gid == "vmail")
-    assert(res == dovecot.auth.PASSDB_RESULT_OK)
-    print("OK test_passdb_lookup_ok "..user)
-end
-
-function test_passdb_lookup_mismatch(user)
-    local res, extra = auth_passdb_lookup({user=user})
-    assert(res == dovecot.auth.PASSDB_RESULT_USER_UNKNOWN)
-    print("OK test_passdb_lookup_mismatch "..user)
-end
-
-function test_split_chatctl()
-    local res = split_chatctl("a=3 b=4\nc=5")
-    assert(res["a"] == "3")
-    assert(res["b"] == "4")
-    assert(res["c"] == "5")
-    print("OK test_split_chatctl")
-end 
-
-test_split_chatctl()
-test_password_verify_ok("link2xt@c1.testrun.org", "Ahyei6ie")
-test_password_verify_mismatch("link2xt@c1.testrun.org", "Aqwlek")
-test_userdb_lookup_ok("link2xt@c1.testrun.org")
-test_userdb_lookup_mismatch("wlekqjlew@xyz.org")
-
--- probably not needed by dovecot? 
--- test_passdb_lookup_ok("link2xt@c1.testrun.org")
--- test_passdb_lookup_mismatch("llqkwjelqwe@xyz.org")
-

chatmail-pyinfra/src/chatmail/dovecot/test_doveauth.py

@@ -1,23 +0,0 @@
-import subprocess
-import pytest
-
-from doveauth import get_user_data, verify_user
-
-
-def test_basic():
-    data = get_user_data("link2xt@c1.testrun.org")
-    assert data
-
-
-@pytest.mark.xfail(reason="no persistence yet")
-def test_verify_or_create():
-    res = verify_user("newuser1@something.org", "kajdlkajsldk12l3kj1983")
-    assert res["status"] == "ok"
-    res = verify_user("newuser1@something.org", "kajdlqweqwe")
-    assert res["status"] == "fail"
-
-
-def test_lua_integration(request):
-    p = request.fspath.dirpath("test_doveauth.lua")
-    proc = subprocess.run(["lua", str(p)])
-    assert proc.returncode == 0

chatmail-pyinfra/tests/test_online_login.py

@@ -1,28 +0,0 @@
-import pytest
-import imaplib
-
-
-@pytest.fixture
-def conn():
-    return connect("c1.testrun.org")
-
-
-def login(conn, user, password):
-    print("trying to login", user, password)
-    conn.login(user, password)
-
-
-def connect(host):
-    print(f"connecting to {host}")
-    conn = imaplib.IMAP4_SSL(host)
-    return conn
-
-
-def test_login_ok(conn):
-    login(conn, "link2xt@c1.testrun.org", "Ahyei6ie")
-
-
-def test_login_fail(conn):
-    with pytest.raises(imaplib.IMAP4.error) as excinfo:
-        login(conn, "link2xt@c1.testrun.org", "qweqwe")
-    assert "AUTHENTICATIONFAILED" in str(excinfo)

deploy.py

@@ -8,11 +8,7 @@ def main():
     mail_server = os.getenv("CHATMAIL_SERVER", mail_domain)
     dkim_selector = os.getenv("CHATMAIL_DKIM_SELECTOR", "2023")
 
-    assert mail_domain
-    assert mail_server
-    assert dkim_selector
-
     deploy_chatmail(mail_domain, mail_server, dkim_selector)
 
 
-main()
+main()

hpk-inv.py

@@ -0,0 +1,11 @@
+
+chatmail = [
+    (
+        "c1.testrun.org",
+        {
+            "ssh_user": "root",
+            "domain": "c1.testrun.org",
+            "dkim_selector": "2023",
+        },
+    ),
+]

plan.txt

@@ -1,39 +0,0 @@
-# Chat-mail server development (up until Oct 18th)
-
-## Dovecot goals/steps
-
-1. create-user-on-login ("doveauth")
-  - repackage so that "doveauth" does not come from a hard-coded path
-  - persistence of accounts
-
-2. per-user quota (adaptive)
-
-3. automatic expiry of messages older than M days
-
-4. automatic expiry of users that haven't logged in for N days
-
-
-## Postfix goals/steps
-
-1. block all outgoing mails with our own LMTP program
-
-2. only allow (outgoing) mails if secure-join or autocrypt-pgp-encrypted format
-   (probably via an lmtp service)
-
-3. basic outgoing send rate/limits (depending on "account-rating")
-
-
-## online tests (first with plain python/pytest)
-
-- write tests for dovecot login (exists)
-- write tests for postfix logins
-- write A<>B send/receive tests
-
-
-## Delta Chat
-
-1. qr code that defines access to a chatmail instance (like mailadm but without http etc.)
-
-2. support for creating username/password and verifying login works
-
-

pyproject.toml

@@ -0,0 +1,10 @@
+[build-system]
+requires = ["setuptools>=45"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "chatmail"
+version = "0.1"
+dependencies = [
+  "pyinfra",
+]

scripts/deploy.sh

@@ -1,3 +0,0 @@
-#!/usr/bin/env bash
-export CHATMAIL_DOMAIN="${1:-c1.testrun.org}"
-chatmail-pyinfra/venv/bin/pyinfra --ssh-user root "$CHATMAIL_DOMAIN" deploy.py

scripts/init.sh

@@ -1,4 +0,0 @@
-#!/bin/sh
-python3 -m venv chatmail-pyinfra/venv
-chatmail-pyinfra/venv/bin/pip install pyinfra
-chatmail-pyinfra/venv/bin/pip install -e chatmail-pyinfra

src/chatmail/__init__.py

@@ -2,6 +2,7 @@
 Chat Mail pyinfra deploy.
 """
 import importlib.resources
+from io import StringIO
 
 from pyinfra import host, logger
 from pyinfra.operations import apt, files, server, systemd, python
@@ -13,7 +14,7 @@ def _install_chatctl() -> None:
     """Setup chatctl."""
     files.put(
         src=importlib.resources.files(__package__)
-        .joinpath("dovecot/doveauth.py")
+        .joinpath("chatctl/chatctl.py")
         .open("rb"),
         dest="/home/vmail/chatctl",
         user="vmail",
@@ -103,8 +104,8 @@ def _configure_dovecot(mail_server: str) -> bool:
 
     # luarocks install http lpeg_patterns fifo
     auth_script = files.put(
-        src=importlib.resources.files(__package__).joinpath("dovecot/doveauth.lua"),
+        src=importlib.resources.files(__package__).joinpath("dovecot/auth.lua"),
-        dest="/etc/dovecot/doveauth.lua",
+        dest="/etc/dovecot/auth.lua",
         user="root",
         group="root",
         mode="644",
@@ -117,7 +118,7 @@ def _configure_dovecot(mail_server: str) -> bool:
 def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> None:
     """Deploy a chat-mail instance.
 
-    :param mail_domain: domain part of your future email addresses
+    :param mail_domain: the domain part of your future email addresses, so "example.org" in user@example.org
     :param mail_server: the DNS name under which your mail server is reachable
     :param dkim_selector:
     """

src/chatmail/acmetool/__init__.py

@@ -1,8 +1,16 @@
-import importlib.resources
+from pathlib import Path
 
 from pyinfra.operations import apt, files, systemd, server
 
 
+def openfile(basename):
+    # on newer python versions:
+    # importlib.resources.files(__package__).joinpath(basename).open("rb")
+    # but here we use a way supported on old pythons
+    dirpath = Path(__path__[0])
+    return dirpath.joinpath(basename).open("rb")
+
+
 def deploy_acmetool(nginx_hook=False, email="", domains=[]):
     """Deploy acmetool."""
     apt.packages(
@@ -11,7 +19,7 @@ def deploy_acmetool(nginx_hook=False, email="", domains=[]):
     )
 
     files.put(
-        src=importlib.resources.files(__package__).joinpath("acmetool.cron").open("rb"),
+        src=openfile("acmetool.cron"),
         dest="/etc/cron.d/acmetool",
         user="root",
         group="root",
@@ -20,9 +28,7 @@ def deploy_acmetool(nginx_hook=False, email="", domains=[]):
 
     if nginx_hook:
         files.put(
-            src=importlib.resources.files(__package__)
+            src=openfile("acmetool.hook"),
-            .joinpath("acmetool.hook")
-            .open("rb"),
             dest="/usr/lib/acme/hooks/nginx",
             user="root",
             group="root",
@@ -30,7 +36,7 @@ def deploy_acmetool(nginx_hook=False, email="", domains=[]):
         )
 
     files.template(
-        src=importlib.resources.files(__package__).joinpath("response-file.yaml.j2"),
+        src=openfile("response-file.yaml.j2"),
         dest="/var/lib/acme/conf/responses",
         user="root",
         group="root",
@@ -39,9 +45,7 @@ def deploy_acmetool(nginx_hook=False, email="", domains=[]):
     )
 
     service_file = files.put(
-        src=importlib.resources.files(__package__)
+        src=openfile("acmetool-redirector.service"),
-        .joinpath("acmetool-redirector.service")
-        .open("rb"),
         dest="/etc/systemd/system/acmetool-redirector.service",
         user="root",
         group="root",

src/chatmail/chatctl/chatctl.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+import base64
+import sys
+
+if sys.argv[1] == "hexauth":
+    login = base64.b16decode(sys.argv[2])
+    password = base64.b16decode(sys.argv[3])
+    if login == b"link2xt@instant2.testrun.org" and password == b"Ahyei6ie":
+        sys.exit(0)
+    else:
+        sys.exit(1)
+elif sys.argv[1] == "hexlookup":
+    login = base64.b16decode(sys.argv[2])
+    if login == b"link2xt@instant2.testrun.org":
+        sys.exit(0)
+    else:
+        sys.exit(1)

src/chatmail/dovecot/auth.lua

@@ -0,0 +1,35 @@
+-- Lua based authentication script for Dovecot.
+--
+-- It calls external chatctl command to answer requests.
+
+-- Hexadecimal aka base16 encoding.
+function hex(data)
+   return (data:gsub(".", function(char) return string.format("%2X", char:byte()) end))
+end
+
+-- Escape shell argument by hex encoding it and wrapping in quotes.
+function escape(data)
+   return ("'"..hex(data).."'")
+end
+
+function auth_password_verify(request, password)
+  if os.execute("/home/vmail/chatctl hexauth "..escape(request.user).." "..escape(password)) then
+    return dovecot.auth.PASSDB_RESULT_OK, {}
+  end
+  return dovecot.auth.PASSDB_RESULT_PASSWORD_MISMATCH, ""
+end
+
+function auth_passdb_lookup(request)
+  if os.execute("/home/vmail/chatctl hexlookup "..escape(request.user)) then
+    return dovecot.auth.PASSDB_RESULT_OK, {}
+  end
+  return dovecot.auth.PASSDB_RESULT_USER_UNKNOWN, "no such user"
+end
+
+function auth_userdb_lookup(request)
+  if os.execute("/home/vmail/chatctl hexlookup "..escape(request.user)) then
+    return dovecot.auth.USERDB_RESULT_OK, "uid=vmail gid=vmail"
+  end
+
+  return dovecot.auth.USERDB_RESULT_USER_UNKNOWN, "no such user"
+end

src/chatmail/dovecot/dovecot.conf.j2

@@ -4,19 +4,14 @@ protocols = imap lmtp
 
 auth_mechanisms = plain
 
-auth_verbose = yes
-auth_debug = yes
-auth_debug_passwords = yes
-auth_verbose_passwords = plain
-
 # Authentication for system users.
 passdb {
   driver = lua
-  args = file=/etc/dovecot/doveauth.lua
+  args = file=/etc/dovecot/auth.lua
 }
 userdb {
   driver = lua
-  args = file=/etc/dovecot/doveauth.lua
+  args = file=/etc/dovecot/auth.lua
 }
 
 ##

tox.ini

@@ -0,0 +1,13 @@
+[tox]
+isolated_build = true
+envlist = lint
+
+[testenv:lint]
+skipdist = True
+skip_install = True
+deps =
+  ruff
+  black
+commands =
+  black --quiet --check --diff src/
+  ruff src/