deploy chatmaild in a virtualenv to make it easier to add dependencies

.github/workflows/ci.yaml

@@ -6,35 +6,26 @@ on:
 
 jobs:
   tox:
-    name: isolated chatmaild tests
+    name: chatmail tests
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-
       - name: run chatmaild tests 
         working-directory: chatmaild
         run: pipx run tox
+      - name: run deploy-chatmail offline tests 
+        working-directory: deploy-chatmail
+        run: pipx run tox
+      - name: run deploy-chatmail offline tests 
+        working-directory: deploy-chatmail
+        run: pipx run tox
 
   scripts:
-    name: deploy-chatmail tests 
+    name: chatmail script invocations 
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-
-      - name: initenv 
-        run: scripts/initenv.sh
-
-      - name: append venv/bin to PATH
-        run: echo venv/bin >>$GITHUB_PATH
-
-      - name: run formatting checks 
-        run: cmdeploy fmt -v 
-
-      - name: run deploy-chatmail offline tests 
-        run: pytest --pyargs cmdeploy 
-
-      - name: initialize with chatmail domain
-        run: cmdeploy init chat.example.org  
-
-      # all other cmdeploy commands require a staging server 
-      # see https://github.com/deltachat/chatmail/issues/100
+      - name: run init.sh 
+        run: ./scripts/init.sh 
+      - name: run test.sh 
+        run: ./scripts/test.sh 

.gitignore

@@ -3,8 +3,10 @@ __pycache__/
 *.py[cod]
 *$py.class
 *.swp
+www/privacy.html*
+www/index.html*
+www/info.html*
 *qr-*.png
-chatmail.ini
 
 
 # C extensions

README.md

@@ -1,9 +1,9 @@
 
 <img width="800px" src="www/src/collage-top.png"/>
 
-# Chatmail services optimized for Delta Chat apps 
+# Chatmail instances optimized for Delta Chat apps 
 
-This repository helps to setup a ready-to-use chatmail server
+This repository helps to setup a ready-to-use chatmail instance
 comprised of a minimal setup of the battle-tested 
 [postfix smtp](https://www.postfix.org) and [dovecot imap](https://www.dovecot.org) services. 
 
@@ -13,83 +13,33 @@ for use by [Delta Chat apps](https://delta.chat).
 Chatmail accounts are automatically created by a first login, 
 after which the initially specified password is required for using them. 
 
-## Deploying your own chatmail server 
+## Getting Started deploying your own chatmail instance
 
-We subsequently use `CHATMAIL_DOMAIN` as a placeholder for your fully qualified 
-DNS domain name (FQDN), for example `chat.example.org`.
+1. Prepare your local (presumably Linux) system:
 
-1. Setup DNS `A` and `AAAA` records for your `CHATMAIL_DOMAIN`. 
-   Verify that DNS is set and SSH root login works:
+    scripts/init.sh 
 
-   ```
-        ssh root@CHATMAIL_DOMAIN 
-   ```
+2. Setup a domain with `A` and `AAAA` records for your chatmail server.
 
-2. Install the `cmdeploy` command in a virtualenv 
+3. Set environment variable to the chatmail domain you want to setup:
 
-   ```
-    source scripts/initenv.sh
-   ```
-  
-3. Create chatmail configuration file `chatmail.ini`:
+    export CHATMAIL_DOMAIN=c1.testrun.org   # replace with your host
 
-   ```
-    cmdeploy init CHATMAIL_DOMAIN
-   ```
+4. Fill in privacy contact data into the `chatmail.ini` file
 
-4. Deploy to the remote chatmail server:
+5. Deploy the chat mail instance to your chatmail server: 
 
-   ```
-    cmdeploy run 
-   ```
+    scripts/deploy.sh 
 
-5. To output a DNS zone file from which you can transfer DNS records 
-   to your DNS provider:
+   This script remotely sets up packages and configures the chatmail provider. 
 
-   ```
-    cmdeploy dns
-   ```
-
-6. To check status of your remotely running chatmail service: 
-
-   ```
-    cmdeploy status
-   ```
-
-7. To test your chatmail service: 
-
-   ```
-    cmdeploy test
-   ```
-
-8. To benchmark your chatmail service: 
-
-   ```
-    cmdeploy bench
-   ```
-
-### Refining the web pages 
-
-
-``` 
-    cmdeploy webdev 
-```
-
-This starts a local live development cycle for chatmail Web pages: 
-
-- uses the `www/src/page-layout.html` file for producing static 
-  HTML pages from `www/src/*.md` files
-
-- continously builds the web presence reading files from `www/src` directory
-  and generating html files and copying assets to the `www/build` directory. 
-
-- Starts a browser window automatically where you can "refresh" as needed. 
+6. Run `scripts/generate-dns-zone.sh` and 
+   transfer the generated DNS records at your DNS provider
 
 
 ### Home page and getting started for users 
 
-`cmdeploy run` sets up mail services,
-and also creates default static Web pages and deploys them: 
+The `deploy.sh` script deploys 
 
 - a default `index.html` along with a QR code that users can click to 
   create accounts on your chatmail provider,
@@ -98,10 +48,31 @@ and also creates default static Web pages and deploys them:
 
 - a default `policy.html` that is linked from the home page. 
 
-All `.html` files are generated 
-by the according markdown `.md` file in the `www/src` directory.
+All files are generated by the according markdown `.md` file in the `www` directory.
 
 
+### Refining the web pages 
+
+The `scripts/webdev.sh` script supports live development of the chatmail web presence:
+
+``` 
+    scripts/init.sh  # to locally initialize python virtual environments etc. 
+    scripts/webdev.sh
+```
+
+- uses the `www/src/page-layout.html` file for producing html documents
+  from `www/src/*.md` files. 
+
+- continously builds the web presence reading files from `www/src` directory
+  and generating html files and copying assets to the `www/build` directory. 
+
+- Starts a browser window automatically where you can "refresh" as needed. 
+
+Note that this script is not needed for running `scripts/deploy.sh" 
+which deploys the whole chatmail setup remotely. 
+The code that generates the web pages is identical
+which means that `webdev.sh` gives a pretty good preview. 
+
 ### Ports
 
 Postfix listens on ports 25 (smtp) and 587 (submission) and 465 (submissions).

chatmail.ini

@@ -1,16 +1,15 @@
-
-[privacy]
-
-passthrough_recipients = privacy@testrun.org
+[config]
 
 privacy_postal =
     Merlinux GmbH, Represented by the managing director H. Krekel,
     Reichgrafen Str. 20, 79102 Freiburg, Germany
 
-privacy_mail = privacy@testrun.org
+privacy_mail = delta-privacy@merlinux.eu
+
 privacy_pdo =
     Prof. Dr. Fabian Schmieder, lexICT UG (limited), Ostfeldstr. 49, 30559 Hannover.
     You can contact him at *delta-privacy@merlinux.eu* (Keyword: DPO)
+
 privacy_supervisor =
     State Commissioner for Data Protection and Freedom of Information of
     Baden-Württemberg in 70173 Stuttgart, Germany.

chatmaild/MANIFEST.in

@@ -1,4 +0,0 @@
-include src/chatmaild/*.f 
-include src/chatmaild/ini/*.ini.f
-include src/chatmaild/ini/*.ini
-include src/chatmaild/tests/mail-data/*

chatmaild/pyproject.toml

@@ -1,28 +1,18 @@
 [build-system]
-requires = ["setuptools>=61"]
+requires = ["setuptools>=45"]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "chatmaild"
-version = "0.2"
+version = "0.1"
 dependencies = [
   "aiosmtpd",
-  "iniconfig",
 ]
 
-[tool.setuptools]
-include-package-data = true
-
-[tool.setuptools.packages.find]
-where = ['src']
-
 [project.scripts]
 doveauth = "chatmaild.doveauth:main"
 filtermail = "chatmaild.filtermail:main"
 
-[project.entry-points.pytest11]
-"chatmaild.testplugin" = "chatmaild.tests.plugin"
-
 [tool.pytest.ini_options]
 addopts = "-v -ra --strict-markers"
 log_format = "%(asctime)s %(levelname)s %(message)s"
@@ -46,7 +36,8 @@ commands =
   ruff src/
 
 [testenv]
+passenv = CHATMAIL_DOMAIN
 deps = pytest 
        pdbpp 
-commands = pytest -v -rsXx {posargs}
+commands = pytest -v -rsXx {posargs: ../tests/chatmaild}
 """

chatmaild/src/chatmaild/config.py

@@ -1,53 +0,0 @@
-import iniconfig
-
-
-def read_config(inipath):
-    cfg = iniconfig.IniConfig(inipath)
-    return Config(inipath, params=cfg.sections["params"])
-
-
-class Config:
-    def __init__(self, inipath, params):
-        self._inipath = inipath
-        self.mail_domain = params["mail_domain"]
-        self.max_user_send_per_minute = int(params["max_user_send_per_minute"])
-        self.filtermail_smtp_port = int(params["filtermail_smtp_port"])
-        self.postfix_reinject_port = int(params["postfix_reinject_port"])
-        self.passthrough_recipients = params["passthrough_recipients"].split()
-        self.privacy_postal = params.get("privacy_postal")
-        self.privacy_mail = params.get("privacy_mail")
-        self.privacy_pdo = params.get("privacy_pdo")
-        self.privacy_supervisor = params.get("privacy_supervisor")
-
-    def _getbytefile(self):
-        return open(self._inipath, "rb")
-
-
-def write_initial_config(inipath, mail_domain):
-    from importlib.resources import files
-
-    inidir = files(__package__).joinpath("ini")
-    content = (
-        inidir.joinpath("chatmail.ini.f").read_text().format(mail_domain=mail_domain)
-    )
-    if mail_domain.endswith(".testrun.org"):
-        override_inipath = inidir.joinpath("override-testrun.ini")
-        privacy = iniconfig.IniConfig(override_inipath)["privacy"]
-        lines = []
-        for line in content.split("\n"):
-            for key, value in privacy.items():
-                value_lines = value.strip().split("\n")
-                if not line.startswith(f"{key} =") or not value_lines:
-                    continue
-                if len(value_lines) == 1:
-                    lines.append(f"{key} = {value}")
-                else:
-                    lines.append(f"{key} =")
-                    for vl in value_lines:
-                        lines.append(f"    {vl}")
-                break
-            else:
-                lines.append(line)
-        content = "\n".join(lines)
-
-    inipath.write_text(content)

chatmaild/src/chatmaild/doveauth.service.f

@@ -1,5 +1,5 @@
 [Unit]
-Description=Chatmail dict authentication proxy for dovecot
+Description=Dict authentication proxy for dovecot
 
 [Service]
 ExecStart={execpath} /run/dovecot/doveauth.socket vmail /home/vmail/passdb.sqlite

chatmaild/src/chatmaild/filtermail.py

@@ -7,11 +7,10 @@ from email.parser import BytesParser
 from email import policy
 from email.utils import parseaddr
 
+from aiosmtpd.smtp import SMTP
 from aiosmtpd.controller import Controller
 from smtplib import SMTP as SMTPClient
 
-from .config import read_config
-
 
 def check_encrypted(message):
     """Check that the message is an OpenPGP-encrypted message."""
@@ -35,6 +34,14 @@ def check_encrypted(message):
     return True
 
 
+def is_passthrough_recipient(recipient):
+    """Check whether a recipient is configured as passthrough."""
+    passthroughlist = ["privacy@testrun.org"]
+    if recipient in passthroughlist:
+        return True
+    return False
+
+
 def check_mdn(message, envelope):
     if len(envelope.rcpt_tos) != 1:
         return False
@@ -63,21 +70,19 @@ def check_mdn(message, envelope):
     return True
 
 
-async def asyncmain_beforequeue(config):
-    port = config.filtermail_smtp_port
-    Controller(BeforeQueueHandler(config), hostname="127.0.0.1", port=port).start()
+class SMTPController(Controller):
+    def factory(self):
+        return SMTP(self.handler, **self.SMTP_kwargs)
 
 
 class BeforeQueueHandler:
-    def __init__(self, config):
-        self.config = config
+    def __init__(self):
         self.send_rate_limiter = SendRateLimiter()
 
     async def handle_MAIL(self, server, session, envelope, address, mail_options):
         logging.info(f"handle_MAIL from {address}")
         envelope.mail_from = address
-        max_sent = self.config.max_user_send_per_minute
-        if not self.send_rate_limiter.is_sending_allowed(address, max_sent):
+        if not self.send_rate_limiter.is_sending_allowed(address):
             return f"450 4.7.1: Too much mail from {address}"
 
         parts = envelope.mail_from.split("@")
@@ -88,61 +93,65 @@ class BeforeQueueHandler:
 
     async def handle_DATA(self, server, session, envelope):
         logging.info("handle_DATA before-queue")
-        error = self.check_DATA(envelope)
+        error = check_DATA(envelope)
         if error:
             return error
         logging.info("re-injecting the mail that passed checks")
-        client = SMTPClient("localhost", self.config.postfix_reinject_port)
+        client = SMTPClient("localhost", "10025")
         client.sendmail(envelope.mail_from, envelope.rcpt_tos, envelope.content)
         return "250 OK"
 
-    def check_DATA(self, envelope):
-        """the central filtering function for e-mails."""
-        logging.info(f"Processing DATA message from {envelope.mail_from}")
 
-        message = BytesParser(policy=policy.default).parsebytes(envelope.content)
-        mail_encrypted = check_encrypted(message)
+async def asyncmain_beforequeue(port):
+    Controller(BeforeQueueHandler(), hostname="127.0.0.1", port=port).start()
 
-        _, from_addr = parseaddr(message.get("from").strip())
-        logging.info(f"mime-from: {from_addr} envelope-from: {envelope.mail_from!r}")
-        if envelope.mail_from.lower() != from_addr.lower():
-            return f"500 Invalid FROM <{from_addr!r}> for <{envelope.mail_from!r}>"
 
-        if not mail_encrypted and check_mdn(message, envelope):
-            return
+def check_DATA(envelope):
+    """the central filtering function for e-mails."""
+    logging.info(f"Processing DATA message from {envelope.mail_from}")
 
-        passthrough_recipients = self.config.passthrough_recipients
-        envelope_from_domain = from_addr.split("@").pop()
-        for recipient in envelope.rcpt_tos:
-            if envelope.mail_from == recipient:
-                # Always allow sending emails to self.
-                continue
-            if recipient in passthrough_recipients:
-                continue
-            res = recipient.split("@")
-            if len(res) != 2:
-                return f"500 Invalid address <{recipient}>"
-            _recipient_addr, recipient_domain = res
+    message = BytesParser(policy=policy.default).parsebytes(envelope.content)
+    mail_encrypted = check_encrypted(message)
 
-            is_outgoing = recipient_domain != envelope_from_domain
-            if is_outgoing and not mail_encrypted:
-                is_securejoin = message.get("secure-join") in [
-                    "vc-request",
-                    "vg-request",
-                ]
-                if not is_securejoin:
-                    return f"500 Invalid unencrypted mail to <{recipient}>"
+    _, from_addr = parseaddr(message.get("from").strip())
+    logging.info(f"mime-from: {from_addr} envelope-from: {envelope.mail_from!r}")
+    if envelope.mail_from.lower() != from_addr.lower():
+        return f"500 Invalid FROM <{from_addr!r}> for <{envelope.mail_from!r}>"
+
+    if not mail_encrypted and check_mdn(message, envelope):
+        return
+
+    envelope_from_domain = from_addr.split("@").pop()
+    for recipient in envelope.rcpt_tos:
+        if envelope.mail_from == recipient:
+            # Always allow sending emails to self.
+            continue
+        if is_passthrough_recipient(recipient):
+            # Always allow recipients marked as passthrough
+            continue
+        res = recipient.split("@")
+        if len(res) != 2:
+            return f"500 Invalid address <{recipient}>"
+        _recipient_addr, recipient_domain = res
+
+        is_outgoing = recipient_domain != envelope_from_domain
+        if is_outgoing and not mail_encrypted:
+            is_securejoin = message.get("secure-join") in ["vc-request", "vg-request"]
+            if not is_securejoin:
+                return f"500 Invalid unencrypted mail to <{recipient}>"
 
 
 class SendRateLimiter:
+    MAX_USER_SEND_PER_MINUTE = 80
+
     def __init__(self):
         self.addr2timestamps = {}
 
-    def is_sending_allowed(self, mail_from, max_send_per_minute):
+    def is_sending_allowed(self, mail_from):
         last = self.addr2timestamps.setdefault(mail_from, [])
         now = time.time()
         last[:] = [ts for ts in last if ts >= (now - 60)]
-        if len(last) <= max_send_per_minute:
+        if len(last) <= self.MAX_USER_SEND_PER_MINUTE:
             last.append(now)
             return True
         return False
@@ -151,10 +160,9 @@ class SendRateLimiter:
 def main():
     args = sys.argv[1:]
     assert len(args) == 1
-    config = read_config(args[0])
     logging.basicConfig(level=logging.WARN)
     loop = asyncio.new_event_loop()
     asyncio.set_event_loop(loop)
-    task = asyncmain_beforequeue(config)
+    task = asyncmain_beforequeue(port=int(args[0]))
     loop.create_task(task)
     loop.run_forever()

chatmaild/src/chatmaild/filtermail.service.f

@@ -2,7 +2,7 @@
 Description=Chatmail Postfix BeforeQeue filter 
 
 [Service]
-ExecStart={execpath} {config_path}
+ExecStart={execpath} 10080
 Restart=always
 RestartSec=30
 

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -1,33 +0,0 @@
-[params]
-
-# mail domain (MUST be set to fully qualified chat mail domain)
-mail_domain = {mail_domain}
-
-#
-# If you only do private test deploys, you don't need to modify any settings below
-#
-
-# how many mails a user can send out per minute
-max_user_send_per_minute = 60
-
-# list of e-mail recipients for which to accept outbound un-encrypted mails
-passthrough_recipients =
-
-# where the filtermail SMTP service listens
-filtermail_smtp_port = 10080
-
-# postfix accepts on the localhost reinject SMTP port
-postfix_reinject_port = 10025
-
-# postal address of privacy contact
-privacy_postal =
-
-# email address of privacy contact
-privacy_mail =
-
-# postal address of the privacy data officer
-privacy_pdo =
-
-# postal address of the privacy supervisor
-privacy_supervisor =
-

chatmaild/src/chatmaild/newemail.py

@@ -5,7 +5,7 @@
 import json
 import random
 
-mail_domain_path = "/etc/mailname"
+mailname_path = "/etc/mailname"
 
 
 def create_newemail_dict(domain):
@@ -16,7 +16,7 @@ def create_newemail_dict(domain):
 
 
 def print_new_account():
-    domain = open(mail_domain_path).read().strip()
+    domain = open(mailname_path).read().strip()
     creds = create_newemail_dict(domain=domain)
 
     print("Content-Type: application/json")

chatmaild/src/chatmaild/tests/plugin.py

@@ -1,68 +0,0 @@
-import random
-import importlib.resources
-import itertools
-from email.parser import BytesParser
-from email import policy
-import pytest
-
-from chatmaild.database import Database
-from chatmaild.config import read_config, write_initial_config
-
-
-@pytest.fixture
-def make_config(tmp_path):
-    inipath = tmp_path.joinpath("chatmail.ini")
-
-    def make_conf(mail_domain):
-        write_initial_config(inipath, mail_domain=mail_domain)
-        return read_config(inipath)
-
-    return make_conf
-
-
-@pytest.fixture
-def example_config(make_config):
-    return make_config("chat.example.org")
-
-
-@pytest.fixture
-def maildomain(example_config):
-    return example_config.mail_domain
-
-
-@pytest.fixture
-def gencreds(maildomain):
-    count = itertools.count()
-    next(count)
-
-    def gen(domain=None):
-        domain = domain if domain else maildomain
-        while 1:
-            num = next(count)
-            alphanumeric = "abcdefghijklmnopqrstuvwxyz1234567890"
-            user = "".join(random.choices(alphanumeric, k=10))
-            user = f"ac{num}_{user}"[:9]
-            password = "".join(random.choices(alphanumeric, k=12))
-            yield f"{user}@{domain}", f"{password}"
-
-    return lambda domain=None: next(gen(domain))
-
-
-@pytest.fixture()
-def db(tmpdir):
-    db_path = tmpdir / "passdb.sqlite"
-    print("database path:", db_path)
-    return Database(db_path)
-
-
-@pytest.fixture
-def maildata(request):
-    datadir = importlib.resources.files(__package__).joinpath("mail-data")
-    assert datadir.exists(), datadir
-
-    def maildata(name, from_addr, to_addr):
-        data = datadir.joinpath(name).read_text()
-        text = data.format(from_addr=from_addr, to_addr=to_addr)
-        return BytesParser(policy=policy.default).parsebytes(text.encode())
-
-    return maildata

chatmaild/src/chatmaild/tests/test_config.py

@@ -1,27 +0,0 @@
-from chatmaild.config import read_config
-
-
-def test_read_config_basic(make_config):
-    config = make_config("chat.example.org")
-    assert config.mail_domain == "chat.example.org"
-    assert not config.privacy_supervisor and not config.privacy_mail
-    assert not config.privacy_pdo and not config.privacy_postal
-
-    inipath = config._inipath
-    inipath.write_text(inipath.read_text().replace("60", "37"))
-    config = read_config(inipath)
-    assert config.max_user_send_per_minute == 37
-    assert config.mail_domain == "chat.example.org"
-
-
-def test_read_config_testrun(make_config):
-    config = make_config("something.testrun.org")
-    assert config.mail_domain == "something.testrun.org"
-    assert len(config.privacy_postal.split("\n")) > 1
-    assert len(config.privacy_supervisor.split("\n")) > 1
-    assert len(config.privacy_pdo.split("\n")) > 1
-    assert config.privacy_mail == "privacy@testrun.org"
-    assert config.filtermail_smtp_port == 10080
-    assert config.postfix_reinject_port == 10025
-    assert config.max_user_send_per_minute == 60
-    assert config.passthrough_recipients

cmdeploy/pyproject.toml

@@ -1,32 +0,0 @@
-[build-system]
-requires = ["setuptools>=68"]
-build-backend = "setuptools.build_meta"
-
-[project]
-name = "cmdeploy"
-version = "0.2"
-dependencies = [
-  "pyinfra",
-  "pillow",
-  "qrcode",
-  "markdown",
-  "pytest",
-  "setuptools>=68",
-  "termcolor",
-  "build",
-  "tox",
-  "ruff",
-  "black",
-  "pytest",
-  "pytest-xdist", 
-]
-
-[project.scripts]
-cmdeploy = "cmdeploy.cmdeploy:main"
-
-[project.entry-points.pytest11]
-"chatmaild.testplugin" = "chatmaild.tests.plugin"
-"cmdeploy.testplugin" = "cmdeploy.tests.plugin"
-
-[tool.pytest.ini_options]
-addopts = "-v -ra --strict-markers"

cmdeploy/src/cmdeploy/chatmail.zone.f

@@ -1,12 +0,0 @@
-{chatmail_domain}.                   MX 10 {chatmail_domain}.
-_submission._tcp.{chatmail_domain}.  SRV 0 1 587 {chatmail_domain}.
-_submissions._tcp.{chatmail_domain}. SRV 0 1 465 {chatmail_domain}.
-_imap._tcp.{chatmail_domain}.        SRV 0 1 143 {chatmail_domain}.
-_imaps._tcp.{chatmail_domain}.       SRV 0 1 993 {chatmail_domain}.
-{chatmail_domain}.                IN CAA 128 issue "letsencrypt.org;accounturi={acme_account_url}"
-{chatmail_domain}.                   TXT "v=spf1 a:{chatmail_domain} -all"
-_dmarc.{chatmail_domain}.            TXT "v=DMARC1;p=reject;rua=mailto:{email};ruf=mailto:{email};fo=1;adkim=r;aspf=r"
-_mta-sts.{chatmail_domain}.       IN TXT "v=STSv1; id={sts_id}"
-mta-sts.{chatmail_domain}.        IN CNAME {chatmail_domain}.
-_smtp._tls.{chatmail_domain}.     IN TXT "v=TLSRPTv1;rua=mailto:{email}"
-{dkim_entry}

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -1,298 +0,0 @@
-"""
-Provides the `cmdeploy` entry point function,
-along with command line option and subcommand parsing.
-"""
-import argparse
-import datetime
-import shutil
-import subprocess
-import importlib.resources
-import importlib.util
-import os
-import sys
-from pathlib import Path
-
-
-from termcolor import colored
-from chatmaild.config import read_config, write_initial_config
-
-
-#
-# cmdeploy sub commands and options
-#
-
-
-def init_cmd_options(parser):
-    parser.add_argument(
-        "chatmail_domain",
-        action="store",
-        help="fully qualified DNS domain name for your chatmail instance",
-    )
-
-
-def init_cmd(args, out):
-    """Initialize chatmail config file."""
-    if args.inipath.exists():
-        out.red(f"Path exists, not modifying: {args.inipath}")
-        raise SystemExit(1)
-    write_initial_config(args.inipath, args.chatmail_domain)
-    out.green(f"created config file for {args.chatmail_domain} in {args.inipath}")
-
-
-def run_cmd_options(parser):
-    parser.add_argument(
-        "--dry-run",
-        dest="dry_run",
-        action="store_true",
-        help="don't actually modify the server",
-    )
-
-
-def run_cmd(args, out):
-    """Deploy chatmail services on the remote server."""
-
-    env = os.environ.copy()
-    env["CHATMAIL_DOMAIN"] = args.config.mail_domain
-    deploy_path = "cmdeploy/src/cmdeploy/deploy.py"
-    pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
-    cmd = f"{pyinf} --ssh-user root {args.config.mail_domain} {deploy_path}"
-    out.check_call(cmd, env=env)
-
-
-def dns_cmd(args, out):
-    """Generate dns zone file."""
-    template = importlib.resources.files(__package__).joinpath("chatmail.zone.f")
-    ssh = f"ssh root@{args.config.mail_domain}"
-
-    def read_dkim_entries(entry):
-        lines = []
-        for line in entry.split("\n"):
-            if line.startswith(";") or not line.strip():
-                continue
-            line = line.replace("\t", " ")
-            lines.append(line)
-        return "\n".join(lines)
-
-    acme_account_url = out.shell_output(f"{ssh} -- acmetool account-url")
-    dkim_entry = read_dkim_entries(out.shell_output(f"{ssh} -- opendkim-genzone -F"))
-
-    out(
-        f"[writing {args.config.mail_domain} zone data (using space as separator) to stdout output]",
-        green=True,
-    )
-    print(
-        template.read_text()
-        .format(
-            acme_account_url=acme_account_url,
-            email=f"root@{args.config.mail_domain}",
-            sts_id=datetime.datetime.now().strftime("%Y%m%d%H%M"),
-            chatmail_domain=args.config.mail_domain,
-            dkim_entry=dkim_entry,
-        )
-        .strip()
-    )
-
-
-def status_cmd(args, out):
-    """Display status for online chatmail instance."""
-
-    ssh = f"ssh root@{args.config.mail_domain}"
-
-    out.green(f"chatmail domain: {args.config.mail_domain}")
-    if args.config.privacy_mail:
-        out.green("privacy settings: present")
-    else:
-        out.red("no privacy settings")
-
-    s1 = "systemctl --type=service --state=running"
-    for line in out.shell_output(f"{ssh} -- {s1}").split("\n"):
-        if line.startswith("  "):
-            print(line)
-
-
-def test_cmd(args, out):
-    """Run local and online tests for chatmail deployment.
-
-    This will automatically pip-install 'deltachat' if it's not available.
-    """
-
-    x = importlib.util.find_spec("deltachat")
-    if x is None:
-        out.check_call(f"{sys.executable} -m pip install deltachat")
-
-    pytest_path = shutil.which("pytest")
-    ret = out.run_ret(
-        [pytest_path, "cmdeploy/src/", "-n4", "-rs", "-x", "-vrx", "--durations=5"]
-    )
-    return ret
-
-
-def fmt_cmd_options(parser):
-    parser.add_argument(
-        "--verbose",
-        "-v",
-        dest="verbose",
-        action="store_true",
-        help="provide information on invocations",
-    )
-
-    parser.add_argument(
-        "--check",
-        "-c",
-        action="store_true",
-        help="only check but don't fix problems",
-    )
-
-
-def fmt_cmd(args, out):
-    """Run formattting fixes (fuff and black) on all chatmail source code."""
-
-    sources = [str(importlib.resources.files(x)) for x in ("chatmaild", "cmdeploy")]
-    black_args = [shutil.which("black")]
-    ruff_args = [shutil.which("ruff")]
-
-    if args.check:
-        black_args.append("--check")
-    else:
-        ruff_args.append("--fix")
-
-    if not args.verbose:
-        black_args.append("-q")
-        ruff_args.append("-q")
-
-    black_args.extend(sources)
-    ruff_args.extend(sources)
-
-    out.check_call(" ".join(black_args), quiet=not args.verbose)
-    out.check_call(" ".join(ruff_args), quiet=not args.verbose)
-    return 0
-
-
-def bench_cmd(args, out):
-    """Run benchmarks against an online chatmail instance."""
-    args = ["pytest", "--pyargs", "cmdeploy.tests.online.benchmark", "-vrx"]
-    cmdstring = " ".join(args)
-    out.green(f"[$ {cmdstring}]")
-    subprocess.check_call(args)
-
-
-def webdev_cmd(args, out):
-    """Run local web development loop for static web pages."""
-    from .www import main
-
-    main()
-
-
-#
-# Parsing command line options and starting commands
-#
-
-
-class Out:
-    """Convenience output printer providing coloring."""
-
-    def red(self, msg, file=sys.stderr):
-        print(colored(msg, "red"), file=file)
-
-    def green(self, msg, file=sys.stderr):
-        print(colored(msg, "green"), file=file)
-
-    def __call__(self, msg, red=False, green=False, file=sys.stdout):
-        color = "red" if red else ("green" if green else None)
-        print(colored(msg, color), file=file)
-
-    def shell_output(self, arg):
-        self(f"[$ {arg}]", file=sys.stderr)
-        return subprocess.check_output(arg, shell=True).decode()
-
-    def check_call(self, arg, env=None, quiet=False):
-        if not quiet:
-            self(f"[$ {arg}]", file=sys.stderr)
-        return subprocess.check_call(arg, shell=True, env=env)
-
-    def run_ret(self, args, env=None, quiet=False):
-        if not quiet:
-            cmdstring = " ".join(args)
-            self(f"[$ {cmdstring}]", file=sys.stderr)
-        proc = subprocess.run(args, env=env)
-        return proc.returncode
-
-
-def add_config_option(parser):
-    parser.add_argument(
-        "--config",
-        dest="inipath",
-        action="store",
-        default=Path("chatmail.ini"),
-        type=Path,
-        help="path to the chatmail.ini file",
-    )
-
-
-def add_subcommand(subparsers, func):
-    name = func.__name__
-    assert name.endswith("_cmd")
-    name = name[:-4]
-    doc = func.__doc__.strip()
-    help = doc.split("\n")[0].strip(".")
-    p = subparsers.add_parser(name, description=doc, help=help)
-    p.set_defaults(func=func)
-    add_config_option(p)
-    return p
-
-
-description = """
-Setup your chatmail server configuration and
-deploy it via SSH to your remote location.
-"""
-
-
-def get_parser():
-    """Return an ArgumentParser for the 'cmdeploy' CLI"""
-
-    parser = argparse.ArgumentParser(description=description.strip())
-    subparsers = parser.add_subparsers(title="subcommands")
-
-    # find all subcommands in the module namespace
-    glob = globals()
-    for name, func in glob.items():
-        if name.endswith("_cmd"):
-            subparser = add_subcommand(subparsers, func)
-            addopts = glob.get(name + "_options")
-            if addopts is not None:
-                addopts(subparser)
-
-    return parser
-
-
-def main(args=None):
-    """Provide main entry point for 'xdcget' CLI invocation."""
-    parser = get_parser()
-    args = parser.parse_args(args=args)
-    if not hasattr(args, "func"):
-        return parser.parse_args(["-h"])
-    out = Out()
-    kwargs = {}
-    if args.func.__name__ not in ("init_cmd", "fmt_cmd"):
-        if not args.inipath.exists():
-            out.red(f"expecting {args.inipath} to exist, run init first?")
-            raise SystemExit(1)
-        try:
-            args.config = read_config(args.inipath)
-        except Exception as ex:
-            out.red(ex)
-            raise SystemExit(1)
-
-    try:
-        res = args.func(args, out, **kwargs)
-        if res is None:
-            res = 0
-        return res
-
-    except KeyboardInterrupt:
-        out.red("KeyboardInterrupt")
-        sys.exit(130)
-
-
-if __name__ == "__main__":
-    main()

cmdeploy/src/cmdeploy/tests/test_cmdeploy.py

@@ -1,33 +0,0 @@
-import os
-
-import pytest
-from cmdeploy.cmdeploy import get_parser, main
-from chatmaild.config import read_config
-
-
-@pytest.fixture(autouse=True)
-def _chdir(tmp_path):
-    old = os.getcwd()
-    os.chdir(tmp_path)
-    yield
-    os.chdir(old)
-
-
-class TestCmdline:
-    def test_parser(self, capsys):
-        parser = get_parser()
-        parser.parse_args([])
-        init = parser.parse_args(["init", "chat.example.org"])
-        run = parser.parse_args(["run"])
-        assert init and run
-
-    def test_init(self, tmp_path):
-        main(["init", "chat.example.org"])
-        inipath = tmp_path.joinpath("chatmail.ini")
-        config = read_config(inipath)
-        assert config.mail_domain == "chat.example.org"
-
-    def test_init_not_overwrite(self):
-        main(["init", "chat.example.org"])
-        with pytest.raises(SystemExit):
-            main(["init", "chat.example.org"])

cmdeploy/src/cmdeploy/tests/test_helpers.py

@@ -1,13 +0,0 @@
-import importlib.resources
-
-from cmdeploy.www import build_webpages
-
-
-def test_build_webpages(tmp_path, make_config):
-    pkgroot = importlib.resources.files("cmdeploy")
-    src_dir = pkgroot.joinpath("../../../www/src").resolve()
-    assert src_dir.exists(), src_dir
-    config = make_config("chat.example.org")
-    build_dir = tmp_path.joinpath("build")
-    build_webpages(src_dir, build_dir, config)
-    assert len([x for x in build_dir.iterdir() if x.suffix == ".html"]) >= 3

deploy-chatmail/pyproject.toml

@@ -0,0 +1,33 @@
+[build-system]
+requires = ["setuptools>=45"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "deploy-chatmail"
+version = "0.1"
+dependencies = [
+  "pyinfra",
+  "pillow",
+  "qrcode",
+  "markdown",
+]
+
+[tool.pytest.ini_options]
+addopts = "-v -ra --strict-markers"
+
+[tool.tox]
+legacy_tox_ini = """
+[tox]
+isolated_build = true
+envlist = lint
+
+[testenv:lint]
+skipdist = True
+skip_install = True
+deps =
+  ruff
+  black
+commands =
+  black --quiet --check --diff src/
+  ruff src/
+"""

deploy-chatmail/src/deploy_chatmail/__init__.py

@@ -6,6 +6,7 @@ import importlib.resources
 import subprocess
 import shutil
 import io
+import configparser
 from pathlib import Path
 
 from pyinfra import host
@@ -14,8 +15,6 @@ from pyinfra.facts.files import File
 from pyinfra.facts.systemd import SystemdEnabled
 from .acmetool import deploy_acmetool
 
-from chatmaild.config import read_config, Config
-
 
 def _build_chatmaild(dist_dir) -> None:
     dist_dir = Path(dist_dir).resolve()
@@ -31,24 +30,11 @@ def _build_chatmaild(dist_dir) -> None:
     return entries[0]
 
 
-def remove_legacy_artifacts():
-    # disable legacy doveauth-dictproxy.service
-    if host.get_fact(SystemdEnabled).get("doveauth-dictproxy.service"):
-        systemd.service(
-            name="Disable legacy doveauth-dictproxy.service",
-            service="doveauth-dictproxy.service",
-            running=False,
-            enabled=False,
-        )
-
-
-def _install_remote_venv_with_chatmaild(config) -> None:
-    remove_legacy_artifacts()
+def _install_remote_venv_with_chatmaild() -> None:
     dist_file = _build_chatmaild(dist_dir=Path("chatmaild/dist"))
     remote_base_dir = "/usr/local/lib/chatmaild"
     remote_dist_file = f"{remote_base_dir}/dist/{dist_file.name}"
     remote_venv_dir = f"{remote_base_dir}/venv"
-    remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini"
     root_owned = dict(user="root", group="root", mode="644")
 
     apt.packages(
@@ -64,13 +50,6 @@ def _install_remote_venv_with_chatmaild(config) -> None:
         **root_owned,
     )
 
-    files.put(
-        name=f"Upload {remote_chatmail_inipath}",
-        src=config._getbytefile(),
-        dest=remote_chatmail_inipath,
-        **root_owned,
-    )
-
     pip.virtualenv(
         name=f"chatmaild virtualenv {remote_venv_dir}",
         path=remote_venv_dir,
@@ -84,17 +63,24 @@ def _install_remote_venv_with_chatmaild(config) -> None:
         ],
     )
 
+    # disable legacy doveauth-dictproxy.service
+    if host.get_fact(SystemdEnabled).get("doveauth-dictproxy.service"):
+        systemd.service(
+            name="Disable legacy doveauth-dictproxy.service",
+            service="doveauth-dictproxy.service",
+            running=False,
+            enabled=False,
+        )
+
     # install systemd units
+
     for fn in (
         "doveauth",
         "filtermail",
     ):
-        params = dict(
-            execpath=f"{remote_venv_dir}/bin/{fn}",
-            config_path=remote_chatmail_inipath,
-        )
+        execpath = f"{remote_venv_dir}/bin/{fn}"
         source_path = importlib.resources.files("chatmaild").joinpath(f"{fn}.service.f")
-        content = source_path.read_text().format(**params).encode()
+        content = source_path.read_text().format(execpath=execpath).encode()
 
         files.put(
             name=f"Upload {fn}.service",
@@ -195,7 +181,7 @@ def _install_mta_sts_daemon() -> bool:
     server.shell(
         name="install postfix-mta-sts-resolver with pip",
         commands=[
-            "python3 -m virtualenv /usr/local/lib/postfix-mta-sts-resolver",
+            "python3 -m venv /usr/local/lib/postfix-mta-sts-resolver",
             "/usr/local/lib/postfix-mta-sts-resolver/bin/pip install postfix-mta-sts-resolver",
         ],
     )
@@ -215,7 +201,7 @@ def _install_mta_sts_daemon() -> bool:
     return need_restart
 
 
-def _configure_postfix(config: Config, debug: bool = False) -> bool:
+def _configure_postfix(domain: str, debug: bool = False) -> bool:
     """Configures Postfix SMTP server."""
     need_restart = False
 
@@ -225,7 +211,7 @@ def _configure_postfix(config: Config, debug: bool = False) -> bool:
         user="root",
         group="root",
         mode="644",
-        config=config,
+        config={"domain_name": domain},
     )
     need_restart |= main_config.changed
 
@@ -236,7 +222,6 @@ def _configure_postfix(config: Config, debug: bool = False) -> bool:
         group="root",
         mode="644",
         debug=debug,
-        config=config,
     )
     need_restart |= master_config.changed
 
@@ -346,16 +331,19 @@ def _configure_nginx(domain: str, debug: bool = False) -> bool:
     return need_restart
 
 
-def check_config(config):
-    mail_domain = config.mail_domain
+def get_ini_settings(mail_domain, inipath):
+    parser = configparser.ConfigParser()
+    parser.read(inipath)
+    settings = {key: value.strip() for (key, value) in parser["config"].items()}
     if mail_domain != "testrun.org" and not mail_domain.endswith(".testrun.org"):
-        blocked_words = "merlinux schmieder testrun.org".split()
-        for value in config.__dict__.values():
-            if any(x in value for x in blocked_words):
+        for value in settings.values():
+            value = value.lower()
+            if "merlinux" in value or "schmieder" in value or "@testrun.org" in value:
                 raise ValueError(
-                    f"please set your own privacy contacts/addresses in {config._inipath}"
+                    f"please set your own privacy contacts/addresses in {inipath}"
                 )
-    return config
+    settings["mail_domain"] = mail_domain
+    return settings
 
 
 def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> None:
@@ -412,8 +400,7 @@ def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> N
 
     pkg_root = importlib.resources.files(__package__)
     chatmail_ini = pkg_root.joinpath("../../../chatmail.ini").resolve()
-    config = read_config(chatmail_ini)
-    check_config(config)
+    config = get_ini_settings(mail_domain, chatmail_ini)
     www_path = pkg_root.joinpath("../../../www").resolve()
 
     build_dir = www_path.joinpath("build")
@@ -421,10 +408,10 @@ def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> N
     build_webpages(src_dir, build_dir, config)
     files.rsync(f"{build_dir}/", "/var/www/html", flags=["-avz"])
 
-    _install_remote_venv_with_chatmaild(config)
+    _install_remote_venv_with_chatmaild()
     debug = False
     dovecot_need_restart = _configure_dovecot(mail_server, debug=debug)
-    postfix_need_restart = _configure_postfix(config, debug=debug)
+    postfix_need_restart = _configure_postfix(mail_domain, debug=debug)
     opendkim_need_restart = _configure_opendkim(mail_domain, dkim_selector)
     mta_sts_need_restart = _install_mta_sts_daemon()
     nginx_need_restart = _configure_nginx(mail_domain)

deploy-chatmail/src/deploy_chatmail/deploy.py

@@ -1,6 +1,6 @@
 import os
 import pyinfra
-from cmdeploy import deploy_chatmail
+from deploy_chatmail import deploy_chatmail
 
 
 def main():

deploy-chatmail/src/deploy_chatmail/postfix/main.cf.j2

@@ -1,4 +1,4 @@
-myorigin = {{ config.mail_domain }}
+myorigin = {{ config.domain_name }}
 
 smtpd_banner = $myhostname ESMTP $mail_name (Debian/GNU)
 biff = no
@@ -16,8 +16,8 @@ readme_directory = no
 compatibility_level = 2
 
 # TLS parameters
-smtpd_tls_cert_file=/var/lib/acme/live/{{ config.mail_domain }}/fullchain
-smtpd_tls_key_file=/var/lib/acme/live/{{ config.mail_domain }}/privkey
+smtpd_tls_cert_file=/var/lib/acme/live/{{ config.domain_name }}/fullchain
+smtpd_tls_key_file=/var/lib/acme/live/{{ config.domain_name }}/privkey
 smtpd_tls_security_level=may
 
 smtp_tls_CApath=/etc/ssl/certs
@@ -26,7 +26,7 @@ smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
 smtp_tls_policy_maps = socketmap:inet:127.0.0.1:8461:postfix
 
 smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination
-myhostname = {{ config.mail_domain }}
+myhostname = {{ config.domain_name }}
 alias_maps = hash:/etc/aliases
 alias_database = hash:/etc/aliases
 
@@ -45,7 +45,7 @@ inet_interfaces = all
 inet_protocols = all
 
 virtual_transport = lmtp:unix:private/dovecot-lmtp
-virtual_mailbox_domains = {{ config.mail_domain }}
+virtual_mailbox_domains = {{ config.domain_name }}
 
 smtpd_milters = unix:opendkim/opendkim.sock
 non_smtpd_milters = $smtpd_milters

deploy-chatmail/src/deploy_chatmail/postfix/master.cf.j2

@@ -33,7 +33,7 @@ submission inet n       -       y       -       -       smtpd
   -o smtpd_relay_restrictions=permit_sasl_authenticated,reject
   -o milter_macro_daemon_name=ORIGINATING
   -o smtpd_client_connection_count_limit=1000
-  -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
+  -o smtpd_proxy_filter=127.0.0.1:10080
 smtps     inet  n       -       y       -       -       smtpd
   -o syslog_name=postfix/smtps
   -o smtpd_tls_wrappermode=yes
@@ -49,7 +49,7 @@ smtps     inet  n       -       y       -       -       smtpd
   -o smtpd_relay_restrictions=permit_sasl_authenticated,reject
   -o smtpd_client_connection_count_limit=1000
   -o milter_macro_daemon_name=ORIGINATING
-  -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
+  -o smtpd_proxy_filter=127.0.0.1:10080
 #628       inet  n       -       y       -       -       qmqpd
 pickup    unix  n       -       y       60      1       pickup
 cleanup   unix  n       -       y       -       0       cleanup
@@ -78,5 +78,5 @@ scache    unix  -       -       y       -       1       scache
 postlog   unix-dgram n  -       n       -       1       postlogd
 filter    unix -        n       n       -       -       lmtp
 # Local SMTP server for reinjecting filered mail.
-localhost:{{ config.postfix_reinject_port }} inet  n       -       n       -       10      smtpd
+localhost:10025 inet  n       -       n       -       10      smtpd
   -o syslog_name=postfix/reinject

deploy-chatmail/src/deploy_chatmail/www.py

@@ -7,7 +7,7 @@ import traceback
 import markdown
 from jinja2 import Template
 from .genqr import gen_qr_png_data
-from chatmaild.config import read_config
+from deploy_chatmail import get_ini_settings
 
 
 def snapshot_dir_stats(somedir):
@@ -37,7 +37,7 @@ def build_webpages(src_dir, build_dir, config):
 
 
 def _build_webpages(src_dir, build_dir, config):
-    mail_domain = config.mail_domain
+    mail_domain = config["mail_domain"]
     assert src_dir.exists(), src_dir
     if not build_dir.exists():
         build_dir.mkdir()
@@ -66,12 +66,12 @@ def _build_webpages(src_dir, build_dir, config):
 
 
 def main():
+    chatmail_domain = "example.testrun.org"
     path = importlib.resources.files(__package__)
     reporoot = path.joinpath("../../../").resolve()
     inipath = reporoot.joinpath("chatmail.ini")
-    config = read_config(inipath)
-    config.webdev = True
-    assert config.mail_domain
+    config = get_ini_settings(chatmail_domain, inipath)
+    config["webdev"] = True
     www_path = reporoot.joinpath("www")
     src_path = www_path.joinpath("src")
     stats = None

plan.txt

@@ -0,0 +1,65 @@
+# Chat-mail server development (up until Oct 18th)
+
+## Dovecot goals/steps
+
+- automatic expiry of messages older than M days
+   - also expunge unread messages
+
+- limit: configure max-connections per account
+
+
+## nami: send out rate limit / rspamd
+
+- basic outgoing send rate/limits (depending on "account-rating")
+  use rspamd in a minimal way, check support dkim-signing
+  (including an online test exceeding rate limit)
+
+
+## doveauth questions/futures
+
+- bcrypt-password scheme is slow: require long passwords, use faster hashing
+
+- define user-name and password policies, and implement them
+  (be very restrictive at the beginning, we can relax later)
+
+- password is part of the dictproxy-lookup key, is it safe to use auth-caching?
+
+
+## How to limit creation of accounts?
+
+attack: a 3-line bash script to fill the chatmail db with millions of unused accouts
+
+- make it computationally expensive (somehow try to except our tests from it)
+  1st pass instant onboarding: create userid + cheap password -- if it fails then
+  2nd pass instant onboarding: create userdid + comput. expensive password
+
+- probably also do firewall: limit number of new tcp-connections per IP address per duration
+
+
+## Open/deferred questions
+
+- automatic expiry of users that haven't logged in for N days
+  Is it neccessary?  If all messages are gone, does the existence of
+  an e-mail address bother anybody?
+
+
+## web page for chat-mail servers?
+
+- documentation for users, privacy policy etc.
+  (probably also with provider-messages ...)
+
+
+## online tests (first with plain python/pytest)
+
+- write tests for dovecot login (exists)
+- write tests for postfix logins (exists)
+- write A<>B send/receive tests (exists)
+
+
+## Delta Chat
+
+1. qr code that defines access to a chatmail instance (like mailadm but without http etc.)
+
+2. support for creating username/password and verifying login works
+
+

scripts/bench.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+
+venv/bin/pytest tests/online/benchmark.py -vrx

scripts/deploy.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+echo -----------------------------------------
+echo deploying to $CHATMAIL_DOMAIN 
+echo -----------------------------------------
+
+venv/bin/pyinfra --ssh-user root "$CHATMAIL_DOMAIN" \
+    deploy-chatmail/src/deploy_chatmail/deploy.py

scripts/generate-dns-zone.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+: ${CHATMAIL_DOMAIN:=c1.testrun.org}
+: ${CHATMAIL_SSH:=$CHATMAIL_DOMAIN}
+
+set -e
+SSH="ssh root@$CHATMAIL_SSH"
+EMAIL="root@$CHATMAIL_DOMAIN"
+ACME_ACCOUNT_URL="$($SSH -- acmetool account-url)"
+
+cat <<EOF
+$CHATMAIL_DOMAIN. MX 10 $CHATMAIL_DOMAIN.
+$CHATMAIL_DOMAIN. TXT "v=spf1 a:$CHATMAIL_DOMAIN -all"
+_dmarc.$CHATMAIL_DOMAIN. TXT "v=DMARC1;p=reject;rua=mailto:$EMAIL;ruf=mailto:$EMAIL;fo=1;adkim=r;aspf=r"
+_submission._tcp.$CHATMAIL_DOMAIN.  SRV 0 1 587 $CHATMAIL_DOMAIN.
+_submissions._tcp.$CHATMAIL_DOMAIN. SRV 0 1 465 $CHATMAIL_DOMAIN.
+_imap._tcp.$CHATMAIL_DOMAIN.        SRV 0 1 143 $CHATMAIL_DOMAIN.
+_imaps._tcp.$CHATMAIL_DOMAIN.       SRV 0 1 993 $CHATMAIL_DOMAIN.
+$CHATMAIL_DOMAIN. IN CAA 128 issue "letsencrypt.org;accounturi=$ACME_ACCOUNT_URL"
+_mta-sts.$CHATMAIL_DOMAIN. IN TXT "v=STSv1; id=$(date -u '+%Y%m%d%H%M')"
+mta-sts.$CHATMAIL_DOMAIN. IN CNAME $CHATMAIL_DOMAIN.
+_smtp._tls.$CHATMAIL_DOMAIN. IN TXT "v=TLSRPTv1;rua=mailto:$EMAIL"
+EOF
+$SSH opendkim-genzone -F | sed 's/^;.*$//;/^$/d'

scripts/get_imap_capabilities.py

@@ -0,0 +1,14 @@
+import os
+import time
+import imaplib
+
+domain = os.environ.get("CHATMAIL_DOMAIN", "c3.testrun.org")
+
+print("connecting")
+conn = imaplib.IMAP4_SSL(domain)
+print("logging in")
+conn.login(f"imapcapa", "pass")
+status, res = conn.capability()
+for capa in sorted(res[0].decode().split()):
+    print(capa)
+

scripts/init.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+set -e
+python3 -m venv venv
+pip=venv/bin/pip
+
+$pip install pyinfra pytest build 'setuptools>=68' tox 
+$pip install -e deploy-chatmail 
+$pip install -e chatmaild 

scripts/initenv.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-set -e
-python3 -m venv venv
-
-venv/bin/pip install -e chatmaild 
-venv/bin/pip install -e cmdeploy
-
-source venv/bin/activate
-echo activated 'venv' python virtualenv environment containing "cmdeploy" tool

scripts/measure_tls_and_logins.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+import os
+import time
+import imaplib
+
+domain = os.environ.get("CHATMAIL_DOMAIN", "c3.testrun.org")
+
+NUM_CONNECTIONS=10
+
+conns = []
+
+start = time.time()
+for i in range(NUM_CONNECTIONS):
+    print(f"opening connection {i} to {domain}")
+    conn = imaplib.IMAP4_SSL(domain)
+    conns.append(conn)
+
+tlsdone = time.time()
+duration = tlsdone-start
+print(f"{duration}: TLS connections opening TLS connections")
+
+for i, conn in enumerate(conns):
+    print(f"logging into connection {i}")
+    conn.login(f"measure{i}", "pass")
+
+logindone = time.time()
+duration = logindone - tlsdone
+print(f"{duration}: LOGINS done")

scripts/remote-deploy.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+set -e
+
+: ${CHATMAIL_DOMAIN:=c1.testrun.org}
+: ${CHATMAIL_SSH:=$CHATMAIL_DOMAIN}
+
+rsync -avz . "root@$CHATMAIL_SSH:/root/chatmail" --exclude='/.git' --filter="dir-merge,- .gitignore"
+ssh "root@$CHATMAIL_SSH" "cd /root/chatmail; apt install -y python3-venv; python3 -m venv venv; venv/bin/pip install pyinfra build; venv/bin/python3 -m build -n --sdist chatmaild --outdir dist; venv/bin/pip install -e ./deploy-chatmail -e ./chatmaild; export CHATMAIL_DOMAIN=$CHATMAIL_DOMAIN; venv/bin/pyinfra @local deploy.py"

scripts/test.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+venv/bin/tox -c chatmaild
+venv/bin/tox -c deploy-chatmail
+venv/bin/pytest tests/online -rs -vrx --durations=5 $@

scripts/webdev.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+echo -----------------------------------------
+echo starting local webdev 
+echo -----------------------------------------
+
+venv/bin/python3 -m deploy_chatmail.www 
+
+

tests/chatmaild/test_doveauth.py

@@ -1,4 +1,5 @@
 import json
+import sys
 import pytest
 import threading
 import queue
@@ -6,7 +7,7 @@ import traceback
 
 import chatmaild.doveauth
 from chatmaild.doveauth import get_user_data, lookup_passdb, handle_dovecot_request
-from chatmaild.database import DBError
+from chatmaild.database import Database, DBError
 
 
 def test_basic(db):
@@ -59,8 +60,8 @@ def test_handle_dovecot_request(db):
     assert userdata["password"].startswith("{SHA512-CRYPT}")
 
 
-def test_50_concurrent_lookups_different_accounts(db, gencreds):
-    num_threads = 50
+def test_100_concurrent_lookups_different_accounts(db, gencreds):
+    num_threads = 100
     req_per_thread = 5
     results = queue.Queue()
 

tests/chatmaild/test_filtermail.py

@@ -1,10 +1,4 @@
-from chatmaild.filtermail import (
-    check_encrypted,
-    BeforeQueueHandler,
-    SendRateLimiter,
-    check_mdn,
-)
-
+from chatmaild.filtermail import check_encrypted, check_DATA, SendRateLimiter, check_mdn, is_passthrough_recipient
 import pytest
 
 
@@ -14,48 +8,32 @@ def maildomain():
     return "chatmail.example.org"
 
 
-@pytest.fixture
-def handler(make_config, maildomain):
-    config = make_config(maildomain)
-    return BeforeQueueHandler(config)
-
-
-def test_reject_forged_from(maildata, gencreds, handler):
+def test_reject_forged_from(maildata, gencreds):
     class env:
         mail_from = gencreds()[0]
         rcpt_tos = [gencreds()[0]]
 
     # test that the filter lets good mail through
-    to_addr = gencreds()[0]
-    env.content = maildata(
-        "plain.eml", from_addr=env.mail_from, to_addr=to_addr
-    ).as_bytes()
-
-    assert not handler.check_DATA(envelope=env)
+    env.content = maildata("plain.eml", from_addr=env.mail_from).as_bytes()
+    assert not check_DATA(envelope=env)
 
     # test that the filter rejects forged mail
-    env.content = maildata(
-        "plain.eml", from_addr="forged@c3.testrun.org", to_addr=to_addr
-    ).as_bytes()
-    error = handler.check_DATA(envelope=env)
+    env.content = maildata("plain.eml", from_addr="forged@c3.testrun.org").as_bytes()
+    error = check_DATA(envelope=env)
     assert "500" in error
 
 
 def test_filtermail_no_encryption_detection(maildata):
-    msg = maildata(
-        "plain.eml", from_addr="some@example.org", to_addr="other@example.org"
-    )
+    msg = maildata("plain.eml")
     assert not check_encrypted(msg)
 
     # https://xkcd.com/1181/
-    msg = maildata(
-        "fake-encrypted.eml", from_addr="some@example.org", to_addr="other@example.org"
-    )
+    msg = maildata("fake-encrypted.eml")
     assert not check_encrypted(msg)
 
 
 def test_filtermail_encryption_detection(maildata):
-    msg = maildata("encrypted.eml", from_addr="1@example.org", to_addr="2@example.org")
+    msg = maildata("encrypted.eml")
     assert check_encrypted(msg)
 
     # if the subject is not "..." it is not considered ac-encrypted
@@ -63,7 +41,7 @@ def test_filtermail_encryption_detection(maildata):
     assert not check_encrypted(msg)
 
 
-def test_filtermail_is_mdn(maildata, gencreds, handler):
+def test_filtermail_is_mdn(maildata, gencreds):
     from_addr = gencreds()[0]
     to_addr = gencreds()[0] + ".other"
     msg = maildata("mdn.eml", from_addr, to_addr)
@@ -75,8 +53,7 @@ def test_filtermail_is_mdn(maildata, gencreds, handler):
 
     assert check_mdn(msg, env)
     print(msg.as_string())
-
-    assert not handler.check_DATA(env)
+    assert not check_DATA(env)
 
 
 def test_filtermail_to_multiple_recipients_no_mdn(maildata, gencreds):
@@ -96,20 +73,21 @@ def test_filtermail_to_multiple_recipients_no_mdn(maildata, gencreds):
 def test_send_rate_limiter():
     limiter = SendRateLimiter()
     for i in range(100):
-        if limiter.is_sending_allowed("some@example.org", 10):
-            if i <= 10:
+        if limiter.is_sending_allowed("some@example.org"):
+            if i <= SendRateLimiter.MAX_USER_SEND_PER_MINUTE:
                 continue
             pytest.fail("limiter didn't work")
         else:
-            assert i == 11
+            assert i == SendRateLimiter.MAX_USER_SEND_PER_MINUTE + 1
             break
 
 
-def test_excempt_privacy(maildata, gencreds, handler):
+def test_excempt_privacy(maildata, gencreds):
     from_addr = gencreds()[0]
     to_addr = "privacy@testrun.org"
-    handler.config.passthrough_recipients = [to_addr]
-    false_to = "privacy@something.org"
+    false_to = "privacy@tstrn.org"
+    false_to2 = "prvcy@testrun.org"
+    assert is_passthrough_recipient(to_addr)
 
     msg = maildata("plain.eml", from_addr, to_addr)
 
@@ -119,11 +97,11 @@ def test_excempt_privacy(maildata, gencreds, handler):
         content = msg.as_bytes()
 
     # assert that None/no error is returned
-    assert not handler.check_DATA(envelope=env)
+    assert not check_DATA(envelope=env)
 
     class env2:
         mail_from = from_addr
-        rcpt_tos = [to_addr, false_to]
+        rcpt_tos = [to_addr, false_to, false_to2]
         content = msg.as_bytes()
 
-    assert "500" in handler.check_DATA(envelope=env2)
+    assert "500" in check_DATA(envelope=env2)

tests/chatmaild/test_newmail.py

@@ -18,7 +18,7 @@ def test_create_newemail_dict():
 def test_print_new_account(capsys, monkeypatch, maildomain, tmpdir):
     p = tmpdir.join("mailname")
     p.write(maildomain)
-    monkeypatch.setattr(chatmaild.newemail, "mail_domain_path", str(p))
+    monkeypatch.setattr(chatmaild.newemail, "mailname_path", str(p))
     print_new_account()
     out, err = capsys.readouterr()
     lines = out.split("\n")

tests/conftest.py

@@ -6,11 +6,12 @@ import subprocess
 import imaplib
 import smtplib
 import itertools
+from email.parser import BytesParser
+from email import policy
 from pathlib import Path
 import pytest
 
 from chatmaild.database import Database
-from chatmaild.config import read_config
 
 
 conftestdir = Path(__file__).parent
@@ -37,22 +38,11 @@ def pytest_runtest_setup(item):
 
 
 @pytest.fixture
-def chatmail_config(pytestconfig):
-    current = basedir = Path().resolve()
-    while 1:
-        path = current.joinpath("chatmail.ini").resolve()
-        if path.exists():
-            return read_config(path)
-        if current == current.parent:
-            break
-        current = current.parent
-
-    pytest.skip(f"no chatmail.ini file found in {basedir} or parent dirs")
-
-
-@pytest.fixture
-def maildomain(chatmail_config):
-    return chatmail_config.mail_domain
+def maildomain():
+    domain = os.environ.get("CHATMAIL_DOMAIN")
+    if not domain:
+        pytest.skip("set CHATMAIL_DOMAIN to a ssh-reachable chatmail instance")
+    return domain
 
 
 @pytest.fixture
@@ -350,6 +340,22 @@ def lp(request):
     return LP()
 
 
+@pytest.fixture
+def maildata(request, gencreds):
+    datadir = conftestdir.joinpath("mail-data")
+
+    def maildata(name, from_addr=None, to_addr=None):
+        if from_addr is None:
+            from_addr = gencreds()[0]
+        if to_addr is None:
+            to_addr = gencreds()[0]
+        data = datadir.joinpath(name).read_text()
+        text = data.format(from_addr=from_addr, to_addr=to_addr)
+        return BytesParser(policy=policy.default).parsebytes(text.encode())
+
+    return maildata
+
+
 @pytest.fixture
 def cmsetup(maildomain, gencreds):
     return CMSetup(maildomain, gencreds)

tests/online/test_0_qr.py

@@ -1,4 +1,4 @@
-from cmdeploy.genqr import gen_qr_png_data
+from deploy_chatmail.genqr import gen_qr_png_data
 
 
 def test_gen_qr_png_data(maildomain):

tests/online/test_1_basic.py

@@ -20,7 +20,7 @@ def test_use_two_chatmailservers(cmfactory, maildomain2):
 
 
 @pytest.mark.parametrize("forgeaddr", ["internal", "someone@example.org"])
-def test_reject_forged_from(cmsetup, maildata, gencreds, lp, forgeaddr):
+def test_reject_forged_from(cmsetup, maildata, lp, forgeaddr):
     user1, user3 = cmsetup.gen_users(2)
 
     lp.sec("send encrypted message with forged from")
@@ -54,7 +54,7 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata):
         try:
             user1.smtp.sendmail(user1.addr, [user2.addr], mail)
         except smtplib.SMTPException as e:
-            if i < 60:
+            if i < 80:
                 pytest.fail(f"rate limit was exceeded too early with msg {i}")
             outcome = e.recipients[user2.addr]
             assert outcome[0] == 450

tests/online/test_2_deltachat.py

@@ -91,9 +91,9 @@ class TestEndToEndDeltaChat:
 
         lp.sec("setup encrypted comms between ac1 and ac2 on different instances")
         qr = ac1.get_setup_contact_qr()
-        ch = ac2.qr_setup_contact(qr)
-        assert ch.id >= 10
-        ac1._evtracker.wait_securejoin_inviter_progress(1000)
+        ac2.qr_setup_contact(qr)
+        msg = ac2.wait_next_incoming_message()
+        assert "verified" in msg.text
 
         lp.sec("ac1 sends a message and ac2 marks it as seen")
         chat = ac1.create_chat(ac2)

tests/test_helpers.py

@@ -0,0 +1,47 @@
+import textwrap
+import importlib.resources
+
+from deploy_chatmail.www import build_webpages
+from deploy_chatmail import get_ini_settings
+
+
+def create_ini(inipath):
+    inipath.write_text(
+        textwrap.dedent(
+            """\
+        [config]
+
+        privacy_postal =
+            address-line1
+            address-line2
+
+        privacy_mail = privacy@example.org
+
+        privacy_pdo =
+            address-line3
+    """
+        )
+    )
+
+
+def test_build_webpages(tmp_path):
+    pkgroot = importlib.resources.files("deploy_chatmail")
+    src_dir = pkgroot.joinpath("../../../www/src").resolve()
+    assert src_dir.exists(), src_dir
+
+    inipath = tmp_path.joinpath("chatmail.ini")
+    create_ini(inipath)
+    config = get_ini_settings("example.org", inipath)
+    build_dir = tmp_path.joinpath("build")
+    build_webpages(src_dir, build_dir, config)
+
+
+def test_get_settings(tmp_path):
+    inipath = tmp_path.joinpath("chatmail.ini")
+    create_ini(inipath)
+
+    d = get_ini_settings("x.testrun.org", inipath)
+    assert d["privacy_postal"] == "address-line1\naddress-line2"
+    assert d["privacy_mail"] == "privacy@example.org"
+    assert d["privacy_pdo"] == "address-line3"
+    assert d["mail_domain"] == "x.testrun.org"