Add init.sh and deploy.sh scripts

.gitignore

@@ -10,7 +10,6 @@ __pycache__/
 # Distribution / packaging
 .Python
 build/
-doveauth/dist/
 develop-eggs/
 dist/
 downloads/

README.md

@@ -4,61 +4,7 @@ This package deploys Postfix and Dovecot servers, including OpenDKIM for DKIM si
 
 Postfix uses Dovecot for authentication as described in <https://www.postfix.org/SASL_README.html#server_dovecot>
 
-## Getting started 
-
-prepare:
-
-    pip install -e chatmail-infra
-
-
-then run with pyinfra command line tool: 
-
-    CHATMAIL_DOMAIN=c1.testrun.org pyinfra --ssh-user root c1.testrun.org deploy.py
-
-
-## Structure (wip)
-```
-
-# package doveauth tool and deploy chatmail server to a envvar-specified ssh-reachable host 
-deploy.py 
-
-# chatmail pyinfra deploy package 
-chatmail-pyinfra 
-    pyproject.toml
-    chatmail/__init__ ...
-
-# doveauth tool used by dovecot's auth mechanism on the host system 
-doveauth
-    README.md
-    pyproject.toml
-    doveauth.py
-    doveauth.lua
-    test_doveauth.py
-
-# lmtp server to block (outgoing) unencrypted messages 
-filtermail 
-    README.md
-    pyproject.toml
-    .... 
-
-# online tests (after deploy)
-
-online-tests  # runnable via pytest 
-
-
-
-# scripts for setup/development/deployment 
-
-scripts/
-    init.sh  # create venv/other perequires
-    deploy.sh  # run pyinfra based deploy of everything
-    test.sh # run all local and online tests 
-
-```
-
-## Dovecot/Postfix configuration
-
-### Ports
+## Ports
 
 Postfix listens on ports 25 (smtp) and 587 (submission) and 465 (submissions).
 Dovecot listens on ports 143(imap) and 993 (imaps).
@@ -66,3 +12,9 @@ Dovecot listens on ports 143(imap) and 993 (imaps).
 ## DNS
 
 For DKIM you must add a DNS entry as in /etc/opendkim/selector.txt (where selector is the opendkim_selector configured in the chatmail inventory).
+
+## Run with pyinfra
+
+```
+CHATMAIL_DOMAIN=c1.testrun.org pyinfra --ssh-user root c1.testrun.org deploy.py
+```

deploy.py

@@ -1,5 +1,5 @@
 import os
-import pyinfra
+from pyinfra import host, facts
 from chatmail import deploy_chatmail
 
 
@@ -15,5 +15,4 @@ def main():
     deploy_chatmail(mail_domain, mail_server, dkim_selector)
 
 
-if pyinfra.is_cli:
-    main()
+main()

doveauth/README.md

@@ -1,7 +0,0 @@
-# doveauth
-
-doveauth is a python tool
-to create dovecot users on login.
-It is called by the
-[dovecot lua authentication module](https://doc.dovecot.org/configuration_manual/authentication/lua_based_authentication/)
-

doveauth/pyproject.toml

@@ -1,30 +0,0 @@
-[build-system]
-requires = ["setuptools>=45"]
-build-backend = "setuptools.build_meta"
-
-[project]
-name = "doveauth"
-version = "0.1"
-
-[project.scripts]
-doveauth = "doveauth.doveauth:main"
-
-[tool.pytest.ini_options]
-addopts = "-v -ra --strict-markers"
-
-[tool.tox]
-legacy_tox_ini = """
-[tox]
-isolated_build = true
-envlist = lint
-
-[testenv:lint]
-skipdist = True
-skip_install = True
-deps =
-  ruff
-  black
-commands =
-  black --quiet --check --diff src/
-  ruff src/
-"""

doveauth/src/doveauth/database.py

@@ -1,153 +0,0 @@
-import sqlite3
-import contextlib
-import time
-from pathlib import Path
-
-
-class DBError(Exception):
-    """error during an operation on the database."""
-
-
-class Connection:
-    def __init__(self, sqlconn, write):
-        self._sqlconn = sqlconn
-        self._write = write
-
-    def close(self):
-        self._sqlconn.close()
-
-    def commit(self):
-        self._sqlconn.commit()
-
-    def rollback(self):
-        self._sqlconn.rollback()
-
-    def execute(self, query, params=()):
-        cur = self.cursor()
-        try:
-            cur.execute(query, params)
-        except sqlite3.IntegrityError as e:
-            raise DBError(e)
-        return cur
-
-    def cursor(self):
-        return self._sqlconn.cursor()
-
-    def create_user(self, addr: str, password: str):
-        """Create a row in the users table."""
-        self.execute("PRAGMA foreign_keys=on;")
-        q = """INSERT INTO users (addr, password, last_login)
-               VALUES (?, ?, ?)"""
-        self.execute(q, (addr, password, int(time.time())))
-
-    def get_user(self, addr: str) -> {}:
-        """Get a row from the users table."""
-        q = "SELECT addr, password, last_login from users WHERE addr = ?"
-        row = self._sqlconn.execute(q, (addr,)).fetchone()
-        result = {}
-        if row:
-            result = dict(
-                user=row[0],
-                password=row[1],
-                last_login=row[2],
-            )
-        return result
-
-    def set_config(self, name: str, value: str) -> str:
-        ok = [
-            "dbversion",
-        ]
-        assert name in ok, name
-        q = "INSERT OR REPLACE INTO config (key, value) VALUES (?, ?)"
-        self.cursor().execute(q, (name, value)).fetchone()
-        return value
-
-    def get_config(self, key: str) -> str:
-        q = "SELECT key, value from config WHERE name = ?"
-        c = self._sqlconn.cursor()
-        try:
-            return c.execute(q, key).fetchone()
-        except sqlite3.OperationalError:
-            return None
-
-
-class Database:
-    def __init__(self, path: str):
-        self.path = Path(path)
-        self.ensure_tables()
-
-    def _get_connection(self, write=False, transaction=False, closing=False) -> Connection:
-        # we let the database serialize all writers at connection time
-        # to play it very safe (we don't have massive amounts of writes).
-        mode = "ro"
-        if write:
-            mode = "rw"
-        if not self.path.exists():
-            mode = "rwc"
-        uri = "file:%s?mode=%s" % (self.path, mode)
-        sqlconn = sqlite3.connect(
-            uri,
-            timeout=60,
-            isolation_level=None if transaction else "DEFERRED",
-            uri=True,
-        )
-
-        # Enable Write-Ahead Logging to avoid readers blocking writers and vice versa.
-        if write:
-            sqlconn.execute("PRAGMA journal_mode=wal")
-
-        if transaction:
-            start_time = time.time()
-            while 1:
-                try:
-                    sqlconn.execute("begin immediate")
-                    break
-                except sqlite3.OperationalError:
-                    # another thread may be writing, give it a chance to finish
-                    time.sleep(0.1)
-                    if time.time() - start_time > 5:
-                        # if it takes this long, something is wrong
-                        raise
-        conn = Connection(sqlconn, write=write)
-        if closing:
-            conn = contextlib.closing(conn)
-        return conn
-
-    @contextlib.contextmanager
-    def write_transaction(self):
-        conn = self._get_connection(closing=False, write=True, transaction=True)
-        try:
-            yield conn
-        except Exception:
-            conn.rollback()
-            conn.close()
-            raise
-        else:
-            conn.commit()
-            conn.close()
-
-    def read_connection(self, closing=True) -> Connection:
-        return self._get_connection(closing=closing, write=False)
-
-    CURRENT_DBVERSION = 1
-
-    def ensure_tables(self):
-        with self.write_transaction() as conn:
-            conn.execute(
-                """
-                CREATE TABLE IF NOT EXISTS users (
-                    addr TEXT PRIMARY KEY,
-                    password TEXT,
-                    last_login INTEGER
-                )
-            """,
-            )
-            conn.execute(
-                """
-                CREATE TABLE IF NOT EXISTS config (
-                    key TEXT PRIMARY KEY,
-                    value TEXT
-                )
-            """,
-            )
-            conn.set_config("dbversion", self.CURRENT_DBVERSION)

doveauth/src/doveauth/test_doveauth.py

@@ -1,25 +0,0 @@
-import subprocess
-import pytest
-
-from doveauth.doveauth import get_user_data, verify_user, Database
-
-
-def test_basic(tmpdir):
-    db = Database(tmpdir / "passdb.sqlite")
-    verify_user(db, "link2xt@c1.testrun.org", "asdf")
-    data = get_user_data(db, "link2xt@c1.testrun.org")
-    assert data
-
-
-def test_verify_or_create(tmpdir):
-    db = Database(tmpdir / "passdb.sqlite")
-    res = verify_user(db, "newuser1@something.org", "kajdlkajsldk12l3kj1983")
-    assert res["status"] == "ok"
-    res = verify_user(db, "newuser1@something.org", "kajdlqweqwe")
-    assert res["status"] == "fail"
-
-
-def test_lua_integration(request):
-    p = request.fspath.dirpath("test_doveauth.lua")
-    proc = subprocess.run(["lua", str(p)])
-    assert proc.returncode == 0

online-tests/conftest.py

@@ -1,101 +0,0 @@
-import os
-import io
-import imaplib
-import smtplib
-import itertools
-import pytest
-
-
-@pytest.fixture
-def maildomain():
-    return os.environ.get("CHATMAIL_DOMAIN", "c1.testrun.org")
-
-
-@pytest.fixture
-def imap(maildomain):
-    return ImapConn(maildomain)
-
-
-class ImapConn:
-    def __init__(self, host):
-        self.host = host
-
-    def connect(self):
-        print(f"imap-connect {self.host}")
-        self.conn = imaplib.IMAP4_SSL(self.host)
-
-    def login(self, user, password):
-        print(f"imap-login {user!r} {password!r}")
-        self.conn.login(user, password)
-
-
-@pytest.fixture
-def smtp(maildomain):
-    return SmtpConn(maildomain)
-
-
-class SmtpConn:
-    def __init__(self, host):
-        self.host = host
-
-    def connect(self):
-        print(f"smtp-connect {self.host}")
-        self.conn = smtplib.SMTP_SSL(self.host)
-
-    def login(self, user, password):
-        print(f"smtp-login {user!r} {password!r}")
-        self.conn.login(user, password)
-
-
-@pytest.fixture
-def gencreds(maildomain):
-    count = itertools.count()
-
-    def gen():
-        while 1:
-            num = next(count)
-            yield f"user{num}@{maildomain}", f"password{num}"
-
-    return lambda: next(gen())
-
-
-#
-# Delta Chat testplugin re-use
-# use the cmfactory fixture to get chatmail instance accounts
-#
-
-class ChatmailTestProcess:
-    """Provider for chatmail instance accounts as used by deltachat.testplugin.acfactory """
-    def __init__(self, pytestconfig, maildomain, gencreds):
-        self.pytestconfig = pytestconfig
-        self.maildomain = maildomain
-        self.gencreds = gencreds
-        self._addr2files = {}
-
-    def get_liveconfig_producer(self):
-        while 1:
-            user, password = self.gencreds()
-            config = {"addr": user, "mail_pw": password}
-            yield config
-
-    def cache_maybe_retrieve_configured_db_files(self, cache_addr, db_target_path):
-        pass
-
-    def cache_maybe_store_configured_db_files(self, acc):
-        pass
-
-
-@pytest.fixture
-def cmfactory(request, maildomain, gencreds, tmpdir, data):
-    # cloned from deltachat.testplugin.amfactory
-    pytest.importorskip("deltachat")
-    from deltachat.testplugin import ACFactory
-    testproc = ChatmailTestProcess(request.config, maildomain, gencreds)
-    am = ACFactory(request=request, tmpdir=tmpdir, testprocess=testproc, data=data)
-    yield am
-    if hasattr(request.node, "rep_call") and request.node.rep_call.failed:
-        if testprocess.pytestconfig.getoption("--extra-info"):
-            logfile = io.StringIO()
-            am.dump_imap_summary(logfile=logfile)
-            print(logfile.getvalue())
-            # request.node.add_report_section("call", "imap-server-state", s)

online-tests/test_deltachat.py

@@ -1,13 +0,0 @@
-
-
-class TestMailSending:
-    def test_one_on_one(self, cmfactory, lp):
-        ac1, ac2 = cmfactory.get_online_accounts(2)
-        chat = cmfactory.get_accepted_chat(ac1, ac2)
-
-        lp.sec("ac1: prepare and send text message to ac2")
-        msg1 = chat.send_text("message0")
-
-        lp.sec("wait for ac2 to receive message")
-        msg2 = ac2._evtracker.wait_next_incoming_message()
-        assert msg2.text == "message0"

online-tests/test_login.py

@@ -1,42 +0,0 @@
-import pytest
-import imaplib
-import smtplib
-
-
-class TestDovecot:
-    def test_login_ok(self, imap, gencreds):
-        user, password = gencreds()
-        imap.connect()
-        imap.login(user, password)
-        # verify it works on another connection
-        imap.connect()
-        imap.login(user, password)
-
-    def test_login_fail(self, imap, gencreds):
-        user, password = gencreds()
-        imap.connect()
-        imap.login(user, password)
-        imap.connect()
-        with pytest.raises(imaplib.IMAP4.error) as excinfo:
-            imap.login(user, password + "wrong")
-        assert "AUTHENTICATIONFAILED" in str(excinfo)
-
-
-class TestPostfix:
-    def test_login_ok(self, smtp, gencreds):
-        user, password = gencreds()
-        smtp.connect()
-        smtp.login(user, password)
-        # verify it works on another connection
-        smtp.connect()
-        smtp.login(user, password)
-
-    def test_login_fail(self, smtp, gencreds):
-        user, password = gencreds()
-        smtp.connect()
-        smtp.login(user, password)
-        smtp.connect()
-        with pytest.raises(smtplib.SMTPAuthenticationError) as excinfo:
-            smtp.login(user, password + "wrong")
-        assert excinfo.value.smtp_code == 535
-        assert "authentication failed" in str(excinfo)

plan.txt

@@ -1,37 +0,0 @@
-# Chat-mail server development (up until Oct 18th)
-
-## Dovecot goals/steps
-
-1. create-user-on-login ("doveauth")
-
-2. per-user quota (adaptive)
-
-3. automatic expiry of messages older than M days
-
-4. automatic expiry of users that haven't logged in for N days
-
-
-## Postfix goals/steps
-
-1. block all outgoing mails with our own LMTP program
-
-2. only allow (outgoing) mails if secure-join or autocrypt-pgp-encrypted format
-   (probably via an lmtp service)
-
-3. basic outgoing send rate/limits (depending on "account-rating")
-
-
-## online tests (first with plain python/pytest)
-
-- write tests for dovecot login (exists)
-- write tests for postfix logins
-- write A<>B send/receive tests
-
-
-## Delta Chat
-
-1. qr code that defines access to a chatmail instance (like mailadm but without http etc.)
-
-2. support for creating username/password and verifying login works
-
-

scripts/deploy.sh

@@ -1,7 +1,3 @@
 #!/usr/bin/env bash
-: ${CHATMAIL_DOMAIN:=c1.testrun.org}
-export CHATMAIL_DOMAIN
-cd doveauth
-venv/bin/python3 -m build
-../chatmail-pyinfra/venv/bin/pyinfra --ssh-user root "$CHATMAIL_DOMAIN" ../deploy.py
-rm -r dist/
+export CHATMAIL_DOMAIN="${1:-c1.testrun.org}"
+venv/bin/pyinfra --ssh-user root "$CHATMAIL_DOMAIN" deploy.py

scripts/init.sh

@@ -1,12 +1,4 @@
 #!/bin/sh
-python3 -m venv chatmail-pyinfra/venv
-chatmail-pyinfra/venv/bin/pip install pyinfra pytest
-chatmail-pyinfra/venv/bin/pip install -e chatmail-pyinfra
-chatmail-pyinfra/venv/bin/pip install -e doveauth
-
-python3 -m venv doveauth/venv
-doveauth/venv/bin/pip install pytest build
-doveauth/venv/bin/pip install -e doveauth
-
-python3 -m venv online-tests/venv
-online-tests/venv/bin/pip install pytest pytest-timeout pdbpp deltachat
+python3 -m venv venv
+venv/bin/pip install pyinfra
+venv/bin/pip install -e .

scripts/test.sh

@@ -1,6 +0,0 @@
-#!/bin/bash
-pushd doveauth/src/doveauth
-../../venv/bin/pytest
-popd
-
-online-tests/venv/bin/pytest online-tests/ -vrx --durations=5

src/chatmail/__init__.py

@@ -2,7 +2,6 @@
 Chat Mail pyinfra deploy.
 """
 import importlib.resources
-from pathlib import Path
 
 from pyinfra import host, logger
 from pyinfra.operations import apt, files, server, systemd, python
@@ -10,29 +9,17 @@ from pyinfra.facts.files import File
 from .acmetool import deploy_acmetool
 
 
-def _install_doveauth() -> None:
+def _install_chatctl() -> None:
     """Setup chatctl."""
-    doveauth_filename = "doveauth-0.1.tar.gz"
-    doveauth_path = importlib.resources.files(__package__).joinpath(
-        f"../../../doveauth/dist/{doveauth_filename}"
+    files.put(
+        src=importlib.resources.files(__package__)
+        .joinpath("dovecot/doveauth.py")
+        .open("rb"),
+        dest="/home/vmail/chatctl",
+        user="vmail",
+        group="vmail",
+        mode="755",
     )
-    remote_path = f"/tmp/{doveauth_filename}"
-    if Path(str(doveauth_path)).exists():
-        files.put(
-            name="upload local doveauth build",
-            src=doveauth_path.open("rb"),
-            dest=remote_path,
-        )
-        apt.packages(
-            name="apt install python3-pip",
-            packages="python3-pip",
-        )
-        # Maybe if we introduce dependencies to the doveauth package at some point, we should not install doveauth
-        # system-wide anymore. For now it's fine though.
-        server.shell(
-            name="install local doveauth build with pip",
-            commands=[f"pip install --break-system-packages {remote_path}"],
-        )
 
 
 def _configure_opendkim(domain: str, dkim_selector: str) -> bool:
@@ -116,7 +103,7 @@ def _configure_dovecot(mail_server: str) -> bool:
 
     # luarocks install http lpeg_patterns fifo
     auth_script = files.put(
-        src=importlib.resources.files("doveauth").joinpath("doveauth.lua"),
+        src=importlib.resources.files(__package__).joinpath("dovecot/doveauth.lua"),
         dest="/etc/dovecot/doveauth.lua",
         user="root",
         group="root",
@@ -172,7 +159,7 @@ def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> N
         ],
     )
 
-    _install_doveauth()
+    _install_chatctl()
     dovecot_need_restart = _configure_dovecot(mail_server)
     postfix_need_restart = _configure_postfix(mail_domain)
     opendkim_need_restart = _configure_opendkim(mail_domain, dkim_selector)

src/chatmail/dovecot/doveauth.lua

@@ -8,7 +8,7 @@ end
 -- call out to python program to actually manage authentication for dovecot
 
 function chatctl_verify(user, password)
-    local cmd = "doveauth hexauth "..escape(user).." "..escape(password)
+    local cmd = "python3 /home/vmail/chatctl hexauth "..escape(user).." "..escape(password)
     print("executing: "..cmd)
     local handle = io.popen(cmd)
     local result = handle:read("*a")
@@ -17,10 +17,8 @@ function chatctl_verify(user, password)
 end
 
 function chatctl_lookup(user) 
-    local cmd = "doveauth hexlookup "..escape(user)
     assert(user)
-    print("executing: "..cmd)
-    local handle = io.popen(cmd)
+    local handle = io.popen("python3 /home/vmail/chatctl hexlookup "..escape(user))
     local result = handle:read("*a")
     handle:close()
     return split_chatctl(result)

src/chatmail/dovecot/doveauth.py

@@ -2,40 +2,37 @@
 import base64
 import sys
 
-from .database import Database
+
+def get_user_data(user):
+    if user == "link2xt@c1.testrun.org":
+        return dict(
+            uid="vmail",
+            gid="vmail",
+            password="Ahyei6ie",
+        )
+    return {}
 
 
-def get_user_data(db, user):
-    with db.read_connection() as conn:
-        result = conn.get_user(user)
-    if result:
-        result['uid'] = "vmail"
-        result['gid'] = "vmail"
-    return result
-
-
-def create_user(db, user, password):
-    with db.write_transaction() as conn:
-        conn.create_user(user, password)
+def create_user(user, password):
     return dict(home=f"/home/vmail/{user}", uid="vmail", gid="vmail", password=password)
 
 
-def verify_user(db, user, password):
-    userdata = get_user_data(db, user)
+def verify_user(user, password):
+    userdata = get_user_data(user)
     if userdata:
         if userdata.get("password") == password:
             userdata["status"] = "ok"
         else:
             userdata["status"] = "fail"
     else:
-        userdata = create_user(db, user, password)
+        userdata = create_user(user, password)
         userdata["status"] = "ok"
 
     return userdata
 
 
-def lookup_user(db, user):
-    userdata = get_user_data(db, user)
+def lookup_user(user):
+    userdata = get_user_data(user)
     if userdata:
         userdata["status"] = "ok"
     else:
@@ -48,18 +45,13 @@ def dump_result(res):
         print(f"{key}={value}")
 
 
-def main():
-    db = Database("/home/vmail/passdb.sqlite")
+if __name__ == "__main__":
     if sys.argv[1] == "hexauth":
         login = base64.b16decode(sys.argv[2]).decode()
         password = base64.b16decode(sys.argv[3]).decode()
-        res = verify_user(db, login, password)
+        res = verify_user(login, password)
         dump_result(res)
     elif sys.argv[1] == "hexlookup":
         login = base64.b16decode(sys.argv[2]).decode()
-        res = lookup_user(db, login)
+        res = lookup_user(login)
         dump_result(res)
-
-
-if __name__ == "__main__":
-    main()

src/chatmail/dovecot/dovecot.conf.j2

@@ -88,7 +88,7 @@ service auth {
 service auth-worker {
   # Default is root.
   # Drop privileges we don't need.
-  user = vmail
+  user = $default_internal_user
 }
 
 ssl = required

src/chatmail/dovecot/test_doveauth.py

@@ -0,0 +1,23 @@
+import subprocess
+import pytest
+
+from doveauth import get_user_data, verify_user
+
+
+def test_basic():
+    data = get_user_data("link2xt@c1.testrun.org")
+    assert data
+
+
+@pytest.mark.xfail(reason="no persistence yet")
+def test_verify_or_create():
+    res = verify_user("newuser1@something.org", "kajdlkajsldk12l3kj1983")
+    assert res["status"] == "ok"
+    res = verify_user("newuser1@something.org", "kajdlqweqwe")
+    assert res["status"] == "fail"
+
+
+def test_lua_integration(request):
+    p = request.fspath.dirpath("test_doveauth.lua")
+    proc = subprocess.run(["lua", str(p)])
+    assert proc.returncode == 0

tests/test_online_login.py

@@ -0,0 +1,28 @@
+import pytest
+import imaplib
+
+
+@pytest.fixture
+def conn():
+    return connect("c1.testrun.org")
+
+
+def login(conn, user, password):
+    print("trying to login", user, password)
+    conn.login(user, password)
+
+
+def connect(host):
+    print(f"connecting to {host}")
+    conn = imaplib.IMAP4_SSL(host)
+    return conn
+
+
+def test_login_ok(conn):
+    login(conn, "link2xt@c1.testrun.org", "Ahyei6ie")
+
+
+def test_login_fail(conn):
+    with pytest.raises(imaplib.IMAP4.error) as excinfo:
+        login(conn, "link2xt@c1.testrun.org", "qweqwe")
+    assert "AUTHENTICATIONFAILED" in str(excinfo)