make 'experimental' more outstanding

* create a wwwdev.sh entry point for developing the web part

* rename script

* fix README

* add a note

* don't depend on deltachat python package

* avoid bailing out on jinja2 errors, and provide file-url for instant clickability

* in webdev mode make page auto-refresh every 3 seconds

.github/workflows/ci.yaml

@@ -6,26 +6,35 @@ on:
 
 jobs:
   tox:
-    name: chatmail tests
+    name: isolated chatmaild tests
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+
       - name: run chatmaild tests 
         working-directory: chatmaild
         run: pipx run tox
-      - name: run deploy-chatmail offline tests 
-        working-directory: deploy-chatmail
-        run: pipx run tox
-      - name: run deploy-chatmail offline tests 
-        working-directory: deploy-chatmail
-        run: pipx run tox
 
   scripts:
-    name: chatmail script invocations 
+    name: deploy-chatmail tests 
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: run init.sh 
-        run: ./scripts/init.sh 
-      - name: run test.sh 
-        run: ./scripts/test.sh 
+
+      - name: initenv 
+        run: scripts/initenv.sh
+
+      - name: append venv/bin to PATH
+        run: echo venv/bin >>$GITHUB_PATH
+
+      - name: run formatting checks 
+        run: cmdeploy fmt -v 
+
+      - name: run deploy-chatmail offline tests 
+        run: pytest --pyargs cmdeploy 
+
+      - name: initialize with chatmail domain
+        run: cmdeploy init chat.example.org  
+
+      # all other cmdeploy commands require a staging server 
+      # see https://github.com/deltachat/chatmail/issues/100

.gitignore

@@ -3,10 +3,8 @@ __pycache__/
 *.py[cod]
 *$py.class
 *.swp
-www/privacy.html*
-www/index.html*
-www/info.html*
 *qr-*.png
+chatmail.ini
 
 
 # C extensions

README.md

@@ -1,9 +1,9 @@
 
-<img width="800px" src="www/collage-top.png"/>
+<img width="800px" src="www/src/collage-top.png"/>
 
-# Chatmail instances optimized for Delta Chat apps 
+# Chatmail services optimized for Delta Chat apps 
 
-This repository helps to setup a ready-to-use chatmail instance
+This repository helps to setup a ready-to-use chatmail server
 comprised of a minimal setup of the battle-tested 
 [postfix smtp](https://www.postfix.org) and [dovecot imap](https://www.dovecot.org) services. 
 
@@ -13,33 +13,83 @@ for use by [Delta Chat apps](https://delta.chat).
 Chatmail accounts are automatically created by a first login, 
 after which the initially specified password is required for using them. 
 
-## Getting Started deploying your own chatmail instance
+## Deploying your own chatmail server 
 
-1. Prepare your local (presumably Linux) system:
+We subsequently use `CHATMAIL_DOMAIN` as a placeholder for your fully qualified 
+DNS domain name (FQDN), for example `chat.example.org`.
 
-    scripts/init.sh 
+1. Setup DNS `A` and `AAAA` records for your `CHATMAIL_DOMAIN`. 
+   Verify that DNS is set and SSH root login works:
 
-2. Setup a domain with `A` and `AAAA` records for your chatmail server.
+   ```
+        ssh root@CHATMAIL_DOMAIN 
+   ```
 
-3. Set environment variable to the chatmail domain you want to setup:
+2. Install the `cmdeploy` command in a virtualenv 
 
-    export CHATMAIL_DOMAIN=c1.testrun.org   # replace with your host
+   ```
+    source scripts/initenv.sh
+   ```
+  
+3. Create chatmail configuration file `chatmail.ini`:
 
-4. Fill in privacy contact data into the `chatmail.ini` file
+   ```
+    cmdeploy init CHATMAIL_DOMAIN
+   ```
 
-5. Deploy the chat mail instance to your chatmail server: 
+4. Deploy to the remote chatmail server:
 
-    scripts/deploy.sh 
+   ```
+    cmdeploy run 
+   ```
 
-   This script remotely sets up packages and configures the chatmail provider. 
+5. To output a DNS zone file from which you can transfer DNS records 
+   to your DNS provider:
 
-6. Run `scripts/generate-dns-zone.sh` and 
-   transfer the generated DNS records at your DNS provider
+   ```
+    cmdeploy dns
+   ```
+
+6. To check status of your remotely running chatmail service: 
+
+   ```
+    cmdeploy status
+   ```
+
+7. To test your chatmail service: 
+
+   ```
+    cmdeploy test
+   ```
+
+8. To benchmark your chatmail service: 
+
+   ```
+    cmdeploy bench
+   ```
+
+### Refining the web pages 
+
+
+``` 
+    cmdeploy webdev 
+```
+
+This starts a local live development cycle for chatmail Web pages: 
+
+- uses the `www/src/page-layout.html` file for producing static 
+  HTML pages from `www/src/*.md` files
+
+- continously builds the web presence reading files from `www/src` directory
+  and generating html files and copying assets to the `www/build` directory. 
+
+- Starts a browser window automatically where you can "refresh" as needed. 
 
 
 ### Home page and getting started for users 
 
-The `deploy.sh` script deploys 
+`cmdeploy run` sets up mail services,
+and also creates default static Web pages and deploys them: 
 
 - a default `index.html` along with a QR code that users can click to 
   create accounts on your chatmail provider,
@@ -48,7 +98,8 @@ The `deploy.sh` script deploys
 
 - a default `policy.html` that is linked from the home page. 
 
-All files are generated by the according markdown `.md` file in the `www` directory.
+All `.html` files are generated 
+by the according markdown `.md` file in the `www/src` directory.
 
 
 ### Ports

chatmaild/MANIFEST.in

@@ -0,0 +1,4 @@
+include src/chatmaild/*.f 
+include src/chatmaild/ini/*.ini.f
+include src/chatmaild/ini/*.ini
+include src/chatmaild/tests/mail-data/*

chatmaild/pyproject.toml

@@ -1,18 +1,28 @@
 [build-system]
-requires = ["setuptools>=45"]
+requires = ["setuptools>=61"]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "chatmaild"
-version = "0.1"
+version = "0.2"
 dependencies = [
   "aiosmtpd",
+  "iniconfig",
 ]
 
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.packages.find]
+where = ['src']
+
 [project.scripts]
 doveauth = "chatmaild.doveauth:main"
 filtermail = "chatmaild.filtermail:main"
 
+[project.entry-points.pytest11]
+"chatmaild.testplugin" = "chatmaild.tests.plugin"
+
 [tool.pytest.ini_options]
 addopts = "-v -ra --strict-markers"
 log_format = "%(asctime)s %(levelname)s %(message)s"
@@ -36,8 +46,7 @@ commands =
   ruff src/
 
 [testenv]
-passenv = CHATMAIL_DOMAIN
 deps = pytest 
        pdbpp 
-commands = pytest -v -rsXx {posargs: ../tests/chatmaild}
+commands = pytest -v -rsXx {posargs}
 """

chatmaild/src/chatmaild/config.py

@@ -0,0 +1,53 @@
+import iniconfig
+
+
+def read_config(inipath):
+    cfg = iniconfig.IniConfig(inipath)
+    return Config(inipath, params=cfg.sections["params"])
+
+
+class Config:
+    def __init__(self, inipath, params):
+        self._inipath = inipath
+        self.mail_domain = params["mail_domain"]
+        self.max_user_send_per_minute = int(params["max_user_send_per_minute"])
+        self.filtermail_smtp_port = int(params["filtermail_smtp_port"])
+        self.postfix_reinject_port = int(params["postfix_reinject_port"])
+        self.passthrough_recipients = params["passthrough_recipients"].split()
+        self.privacy_postal = params.get("privacy_postal")
+        self.privacy_mail = params.get("privacy_mail")
+        self.privacy_pdo = params.get("privacy_pdo")
+        self.privacy_supervisor = params.get("privacy_supervisor")
+
+    def _getbytefile(self):
+        return open(self._inipath, "rb")
+
+
+def write_initial_config(inipath, mail_domain):
+    from importlib.resources import files
+
+    inidir = files(__package__).joinpath("ini")
+    content = (
+        inidir.joinpath("chatmail.ini.f").read_text().format(mail_domain=mail_domain)
+    )
+    if mail_domain.endswith(".testrun.org"):
+        override_inipath = inidir.joinpath("override-testrun.ini")
+        privacy = iniconfig.IniConfig(override_inipath)["privacy"]
+        lines = []
+        for line in content.split("\n"):
+            for key, value in privacy.items():
+                value_lines = value.strip().split("\n")
+                if not line.startswith(f"{key} =") or not value_lines:
+                    continue
+                if len(value_lines) == 1:
+                    lines.append(f"{key} = {value}")
+                else:
+                    lines.append(f"{key} =")
+                    for vl in value_lines:
+                        lines.append(f"    {vl}")
+                break
+            else:
+                lines.append(line)
+        content = "\n".join(lines)
+
+    inipath.write_text(content)

chatmaild/src/chatmaild/doveauth.service

@@ -1,10 +0,0 @@
-[Unit]
-Description=Dict authentication proxy for dovecot
-
-[Service]
-ExecStart=/usr/local/bin/doveauth /run/dovecot/doveauth.socket vmail /home/vmail/passdb.sqlite
-Restart=always
-RestartSec=30
-
-[Install]
-WantedBy=multi-user.target

chatmaild/src/chatmaild/doveauth.service.f

@@ -0,0 +1,10 @@
+[Unit]
+Description=Chatmail dict authentication proxy for dovecot
+
+[Service]
+ExecStart={execpath} /run/dovecot/doveauth.socket vmail /home/vmail/passdb.sqlite
+Restart=always
+RestartSec=30
+
+[Install]
+WantedBy=multi-user.target

chatmaild/src/chatmaild/filtermail.py

@@ -7,10 +7,11 @@ from email.parser import BytesParser
 from email import policy
 from email.utils import parseaddr
 
-from aiosmtpd.smtp import SMTP
 from aiosmtpd.controller import Controller
 from smtplib import SMTP as SMTPClient
 
+from .config import read_config
+
 
 def check_encrypted(message):
     """Check that the message is an OpenPGP-encrypted message."""
@@ -62,19 +63,21 @@ def check_mdn(message, envelope):
     return True
 
 
-class SMTPController(Controller):
-    def factory(self):
-        return SMTP(self.handler, **self.SMTP_kwargs)
+async def asyncmain_beforequeue(config):
+    port = config.filtermail_smtp_port
+    Controller(BeforeQueueHandler(config), hostname="127.0.0.1", port=port).start()
 
 
 class BeforeQueueHandler:
-    def __init__(self):
+    def __init__(self, config):
+        self.config = config
         self.send_rate_limiter = SendRateLimiter()
 
     async def handle_MAIL(self, server, session, envelope, address, mail_options):
         logging.info(f"handle_MAIL from {address}")
         envelope.mail_from = address
-        if not self.send_rate_limiter.is_sending_allowed(address):
+        max_sent = self.config.max_user_send_per_minute
+        if not self.send_rate_limiter.is_sending_allowed(address, max_sent):
             return f"450 4.7.1: Too much mail from {address}"
 
         parts = envelope.mail_from.split("@")
@@ -85,62 +88,61 @@ class BeforeQueueHandler:
 
     async def handle_DATA(self, server, session, envelope):
         logging.info("handle_DATA before-queue")
-        error = check_DATA(envelope)
+        error = self.check_DATA(envelope)
         if error:
             return error
         logging.info("re-injecting the mail that passed checks")
-        client = SMTPClient("localhost", "10025")
+        client = SMTPClient("localhost", self.config.postfix_reinject_port)
         client.sendmail(envelope.mail_from, envelope.rcpt_tos, envelope.content)
         return "250 OK"
 
+    def check_DATA(self, envelope):
+        """the central filtering function for e-mails."""
+        logging.info(f"Processing DATA message from {envelope.mail_from}")
 
-async def asyncmain_beforequeue(port):
-    Controller(BeforeQueueHandler(), hostname="127.0.0.1", port=port).start()
+        message = BytesParser(policy=policy.default).parsebytes(envelope.content)
+        mail_encrypted = check_encrypted(message)
 
+        _, from_addr = parseaddr(message.get("from").strip())
+        logging.info(f"mime-from: {from_addr} envelope-from: {envelope.mail_from!r}")
+        if envelope.mail_from.lower() != from_addr.lower():
+            return f"500 Invalid FROM <{from_addr!r}> for <{envelope.mail_from!r}>"
 
-def check_DATA(envelope):
-    """the central filtering function for e-mails."""
-    logging.info(f"Processing DATA message from {envelope.mail_from}")
+        if not mail_encrypted and check_mdn(message, envelope):
+            return
 
-    message = BytesParser(policy=policy.default).parsebytes(envelope.content)
-    mail_encrypted = check_encrypted(message)
+        passthrough_recipients = self.config.passthrough_recipients
+        envelope_from_domain = from_addr.split("@").pop()
+        for recipient in envelope.rcpt_tos:
+            if envelope.mail_from == recipient:
+                # Always allow sending emails to self.
+                continue
+            if recipient in passthrough_recipients:
+                continue
+            res = recipient.split("@")
+            if len(res) != 2:
+                return f"500 Invalid address <{recipient}>"
+            _recipient_addr, recipient_domain = res
 
-    _, from_addr = parseaddr(message.get("from").strip())
-    logging.info(f"mime-from: {from_addr} envelope-from: {envelope.mail_from!r}")
-    if envelope.mail_from.lower() != from_addr.lower():
-        return f"500 Invalid FROM <{from_addr!r}> for <{envelope.mail_from!r}>"
-
-    if not mail_encrypted and check_mdn(message, envelope):
-        return
-
-    envelope_from_domain = from_addr.split("@").pop()
-    for recipient in envelope.rcpt_tos:
-        if envelope.mail_from == recipient:
-            # Always allow sending emails to self.
-            continue
-        res = recipient.split("@")
-        if len(res) != 2:
-            return f"500 Invalid address <{recipient}>"
-        _recipient_addr, recipient_domain = res
-
-        is_outgoing = recipient_domain != envelope_from_domain
-        if is_outgoing and not mail_encrypted:
-            is_securejoin = message.get("secure-join") in ["vc-request", "vg-request"]
-            if not is_securejoin:
-                return f"500 Invalid unencrypted mail to <{recipient}>"
+            is_outgoing = recipient_domain != envelope_from_domain
+            if is_outgoing and not mail_encrypted:
+                is_securejoin = message.get("secure-join") in [
+                    "vc-request",
+                    "vg-request",
+                ]
+                if not is_securejoin:
+                    return f"500 Invalid unencrypted mail to <{recipient}>"
 
 
 class SendRateLimiter:
-    MAX_USER_SEND_PER_MINUTE = 80
-
     def __init__(self):
         self.addr2timestamps = {}
 
-    def is_sending_allowed(self, mail_from):
+    def is_sending_allowed(self, mail_from, max_send_per_minute):
         last = self.addr2timestamps.setdefault(mail_from, [])
         now = time.time()
         last[:] = [ts for ts in last if ts >= (now - 60)]
-        if len(last) <= self.MAX_USER_SEND_PER_MINUTE:
+        if len(last) <= max_send_per_minute:
             last.append(now)
             return True
         return False
@@ -149,9 +151,10 @@ class SendRateLimiter:
 def main():
     args = sys.argv[1:]
     assert len(args) == 1
+    config = read_config(args[0])
     logging.basicConfig(level=logging.WARN)
     loop = asyncio.new_event_loop()
     asyncio.set_event_loop(loop)
-    task = asyncmain_beforequeue(port=int(args[0]))
+    task = asyncmain_beforequeue(config)
     loop.create_task(task)
     loop.run_forever()

chatmaild/src/chatmaild/filtermail.service.f

@@ -2,7 +2,7 @@
 Description=Chatmail Postfix BeforeQeue filter 
 
 [Service]
-ExecStart=/usr/local/bin/filtermail 10080
+ExecStart={execpath} {config_path}
 Restart=always
 RestartSec=30
 

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -0,0 +1,33 @@
+[params]
+
+# mail domain (MUST be set to fully qualified chat mail domain)
+mail_domain = {mail_domain}
+
+#
+# If you only do private test deploys, you don't need to modify any settings below
+#
+
+# how many mails a user can send out per minute
+max_user_send_per_minute = 60
+
+# list of e-mail recipients for which to accept outbound un-encrypted mails
+passthrough_recipients =
+
+# where the filtermail SMTP service listens
+filtermail_smtp_port = 10080
+
+# postfix accepts on the localhost reinject SMTP port
+postfix_reinject_port = 10025
+
+# postal address of privacy contact
+privacy_postal =
+
+# email address of privacy contact
+privacy_mail =
+
+# postal address of the privacy data officer
+privacy_pdo =
+
+# postal address of the privacy supervisor
+privacy_supervisor =
+

chatmaild/src/chatmaild/ini/override-testrun.ini

@@ -1,15 +1,16 @@
-[config]
+
+[privacy]
+
+passthrough_recipients = privacy@testrun.org
 
 privacy_postal =
     Merlinux GmbH, Represented by the managing director H. Krekel,
     Reichgrafen Str. 20, 79102 Freiburg, Germany
 
-privacy_mail = delta-privacy@merlinux.eu
-
+privacy_mail = privacy@testrun.org
 privacy_pdo =
     Prof. Dr. Fabian Schmieder, lexICT UG (limited), Ostfeldstr. 49, 30559 Hannover.
     You can contact him at *delta-privacy@merlinux.eu* (Keyword: DPO)
-
 privacy_supervisor =
     State Commissioner for Data Protection and Freedom of Information of
     Baden-Württemberg in 70173 Stuttgart, Germany.

chatmaild/src/chatmaild/newemail.py

@@ -5,7 +5,7 @@
 import json
 import random
 
-mailname_path = "/etc/mailname"
+mail_domain_path = "/etc/mailname"
 
 
 def create_newemail_dict(domain):
@@ -16,7 +16,7 @@ def create_newemail_dict(domain):
 
 
 def print_new_account():
-    domain = open(mailname_path).read().strip()
+    domain = open(mail_domain_path).read().strip()
     creds = create_newemail_dict(domain=domain)
 
     print("Content-Type: application/json")

chatmaild/src/chatmaild/tests/plugin.py

@@ -0,0 +1,68 @@
+import random
+import importlib.resources
+import itertools
+from email.parser import BytesParser
+from email import policy
+import pytest
+
+from chatmaild.database import Database
+from chatmaild.config import read_config, write_initial_config
+
+
+@pytest.fixture
+def make_config(tmp_path):
+    inipath = tmp_path.joinpath("chatmail.ini")
+
+    def make_conf(mail_domain):
+        write_initial_config(inipath, mail_domain=mail_domain)
+        return read_config(inipath)
+
+    return make_conf
+
+
+@pytest.fixture
+def example_config(make_config):
+    return make_config("chat.example.org")
+
+
+@pytest.fixture
+def maildomain(example_config):
+    return example_config.mail_domain
+
+
+@pytest.fixture
+def gencreds(maildomain):
+    count = itertools.count()
+    next(count)
+
+    def gen(domain=None):
+        domain = domain if domain else maildomain
+        while 1:
+            num = next(count)
+            alphanumeric = "abcdefghijklmnopqrstuvwxyz1234567890"
+            user = "".join(random.choices(alphanumeric, k=10))
+            user = f"ac{num}_{user}"[:9]
+            password = "".join(random.choices(alphanumeric, k=12))
+            yield f"{user}@{domain}", f"{password}"
+
+    return lambda domain=None: next(gen(domain))
+
+
+@pytest.fixture()
+def db(tmpdir):
+    db_path = tmpdir / "passdb.sqlite"
+    print("database path:", db_path)
+    return Database(db_path)
+
+
+@pytest.fixture
+def maildata(request):
+    datadir = importlib.resources.files(__package__).joinpath("mail-data")
+    assert datadir.exists(), datadir
+
+    def maildata(name, from_addr, to_addr):
+        data = datadir.joinpath(name).read_text()
+        text = data.format(from_addr=from_addr, to_addr=to_addr)
+        return BytesParser(policy=policy.default).parsebytes(text.encode())
+
+    return maildata

chatmaild/src/chatmaild/tests/test_config.py

@@ -0,0 +1,27 @@
+from chatmaild.config import read_config
+
+
+def test_read_config_basic(make_config):
+    config = make_config("chat.example.org")
+    assert config.mail_domain == "chat.example.org"
+    assert not config.privacy_supervisor and not config.privacy_mail
+    assert not config.privacy_pdo and not config.privacy_postal
+
+    inipath = config._inipath
+    inipath.write_text(inipath.read_text().replace("60", "37"))
+    config = read_config(inipath)
+    assert config.max_user_send_per_minute == 37
+    assert config.mail_domain == "chat.example.org"
+
+
+def test_read_config_testrun(make_config):
+    config = make_config("something.testrun.org")
+    assert config.mail_domain == "something.testrun.org"
+    assert len(config.privacy_postal.split("\n")) > 1
+    assert len(config.privacy_supervisor.split("\n")) > 1
+    assert len(config.privacy_pdo.split("\n")) > 1
+    assert config.privacy_mail == "privacy@testrun.org"
+    assert config.filtermail_smtp_port == 10080
+    assert config.postfix_reinject_port == 10025
+    assert config.max_user_send_per_minute == 60
+    assert config.passthrough_recipients

chatmaild/src/chatmaild/tests/test_doveauth.py

@@ -1,5 +1,4 @@
 import json
-import sys
 import pytest
 import threading
 import queue
@@ -7,7 +6,7 @@ import traceback
 
 import chatmaild.doveauth
 from chatmaild.doveauth import get_user_data, lookup_passdb, handle_dovecot_request
-from chatmaild.database import Database, DBError
+from chatmaild.database import DBError
 
 
 def test_basic(db):
@@ -60,8 +59,8 @@ def test_handle_dovecot_request(db):
     assert userdata["password"].startswith("{SHA512-CRYPT}")
 
 
-def test_100_concurrent_lookups_different_accounts(db, gencreds):
-    num_threads = 100
+def test_50_concurrent_lookups_different_accounts(db, gencreds):
+    num_threads = 50
     req_per_thread = 5
     results = queue.Queue()
 

chatmaild/src/chatmaild/tests/test_filtermail.py

@@ -0,0 +1,129 @@
+from chatmaild.filtermail import (
+    check_encrypted,
+    BeforeQueueHandler,
+    SendRateLimiter,
+    check_mdn,
+)
+
+import pytest
+
+
+@pytest.fixture
+def maildomain():
+    # let's not depend on a real chatmail instance for the offline tests below
+    return "chatmail.example.org"
+
+
+@pytest.fixture
+def handler(make_config, maildomain):
+    config = make_config(maildomain)
+    return BeforeQueueHandler(config)
+
+
+def test_reject_forged_from(maildata, gencreds, handler):
+    class env:
+        mail_from = gencreds()[0]
+        rcpt_tos = [gencreds()[0]]
+
+    # test that the filter lets good mail through
+    to_addr = gencreds()[0]
+    env.content = maildata(
+        "plain.eml", from_addr=env.mail_from, to_addr=to_addr
+    ).as_bytes()
+
+    assert not handler.check_DATA(envelope=env)
+
+    # test that the filter rejects forged mail
+    env.content = maildata(
+        "plain.eml", from_addr="forged@c3.testrun.org", to_addr=to_addr
+    ).as_bytes()
+    error = handler.check_DATA(envelope=env)
+    assert "500" in error
+
+
+def test_filtermail_no_encryption_detection(maildata):
+    msg = maildata(
+        "plain.eml", from_addr="some@example.org", to_addr="other@example.org"
+    )
+    assert not check_encrypted(msg)
+
+    # https://xkcd.com/1181/
+    msg = maildata(
+        "fake-encrypted.eml", from_addr="some@example.org", to_addr="other@example.org"
+    )
+    assert not check_encrypted(msg)
+
+
+def test_filtermail_encryption_detection(maildata):
+    msg = maildata("encrypted.eml", from_addr="1@example.org", to_addr="2@example.org")
+    assert check_encrypted(msg)
+
+    # if the subject is not "..." it is not considered ac-encrypted
+    msg.replace_header("Subject", "Click this link")
+    assert not check_encrypted(msg)
+
+
+def test_filtermail_is_mdn(maildata, gencreds, handler):
+    from_addr = gencreds()[0]
+    to_addr = gencreds()[0] + ".other"
+    msg = maildata("mdn.eml", from_addr, to_addr)
+
+    class env:
+        mail_from = from_addr
+        rcpt_tos = [to_addr]
+        content = msg.as_bytes()
+
+    assert check_mdn(msg, env)
+    print(msg.as_string())
+
+    assert not handler.check_DATA(env)
+
+
+def test_filtermail_to_multiple_recipients_no_mdn(maildata, gencreds):
+    from_addr = gencreds()[0]
+    to_addr = gencreds()[0] + ".other"
+    thirdaddr = gencreds()[0]
+    msg = maildata("mdn.eml", from_addr, to_addr)
+
+    class env:
+        mail_from = from_addr
+        rcpt_tos = [to_addr, thirdaddr]
+        content = msg.as_bytes()
+
+    assert not check_mdn(msg, env)
+
+
+def test_send_rate_limiter():
+    limiter = SendRateLimiter()
+    for i in range(100):
+        if limiter.is_sending_allowed("some@example.org", 10):
+            if i <= 10:
+                continue
+            pytest.fail("limiter didn't work")
+        else:
+            assert i == 11
+            break
+
+
+def test_excempt_privacy(maildata, gencreds, handler):
+    from_addr = gencreds()[0]
+    to_addr = "privacy@testrun.org"
+    handler.config.passthrough_recipients = [to_addr]
+    false_to = "privacy@something.org"
+
+    msg = maildata("plain.eml", from_addr, to_addr)
+
+    class env:
+        mail_from = from_addr
+        rcpt_tos = [to_addr]
+        content = msg.as_bytes()
+
+    # assert that None/no error is returned
+    assert not handler.check_DATA(envelope=env)
+
+    class env2:
+        mail_from = from_addr
+        rcpt_tos = [to_addr, false_to]
+        content = msg.as_bytes()
+
+    assert "500" in handler.check_DATA(envelope=env2)

chatmaild/src/chatmaild/tests/test_newmail.py

@@ -3,6 +3,7 @@ import json
 import chatmaild
 from chatmaild.newemail import create_newemail_dict, print_new_account
 
+
 def test_create_newemail_dict():
     ac1 = create_newemail_dict(domain="example.org")
     assert "@" in ac1["email"]
@@ -17,7 +18,7 @@ def test_create_newemail_dict():
 def test_print_new_account(capsys, monkeypatch, maildomain, tmpdir):
     p = tmpdir.join("mailname")
     p.write(maildomain)
-    monkeypatch.setattr(chatmaild.newemail, "mailname_path", str(p))
+    monkeypatch.setattr(chatmaild.newemail, "mail_domain_path", str(p))
     print_new_account()
     out, err = capsys.readouterr()
     lines = out.split("\n")

cmdeploy/pyproject.toml

@@ -0,0 +1,32 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "cmdeploy"
+version = "0.2"
+dependencies = [
+  "pyinfra",
+  "pillow",
+  "qrcode",
+  "markdown",
+  "pytest",
+  "setuptools>=68",
+  "termcolor",
+  "build",
+  "tox",
+  "ruff",
+  "black",
+  "pytest",
+  "pytest-xdist", 
+]
+
+[project.scripts]
+cmdeploy = "cmdeploy.cmdeploy:main"
+
+[project.entry-points.pytest11]
+"chatmaild.testplugin" = "chatmaild.tests.plugin"
+"cmdeploy.testplugin" = "cmdeploy.tests.plugin"
+
+[tool.pytest.ini_options]
+addopts = "-v -ra --strict-markers"

cmdeploy/src/cmdeploy/__init__.py

@@ -1,81 +1,116 @@
 """
 Chat Mail pyinfra deploy.
 """
+import sys
 import importlib.resources
-import configparser
-import textwrap
+import subprocess
+import shutil
+import io
 from pathlib import Path
 
 from pyinfra import host
-from pyinfra.operations import apt, files, server, systemd
+from pyinfra.operations import apt, files, server, systemd, pip
 from pyinfra.facts.files import File
 from pyinfra.facts.systemd import SystemdEnabled
 from .acmetool import deploy_acmetool
-import markdown
-from jinja2 import Template
+
+from chatmaild.config import read_config, Config
 
 
-from .genqr import gen_qr_png_data
-
-
-def _install_chatmaild() -> None:
-    chatmaild_filename = "chatmaild-0.1.tar.gz"
-    chatmaild_path = importlib.resources.files(__package__).joinpath(
-        f"../../../dist/{chatmaild_filename}"
+def _build_chatmaild(dist_dir) -> None:
+    dist_dir = Path(dist_dir).resolve()
+    if dist_dir.exists():
+        shutil.rmtree(dist_dir)
+    dist_dir.mkdir()
+    subprocess.check_output(
+        [sys.executable, "-m", "build", "-n"]
+        + ["--sdist", "chatmaild", "--outdir", str(dist_dir)]
     )
-    remote_path = f"/tmp/{chatmaild_filename}"
-    if Path(str(chatmaild_path)).exists():
+    entries = list(dist_dir.iterdir())
+    assert len(entries) == 1
+    return entries[0]
+
+
+def remove_legacy_artifacts():
+    # disable legacy doveauth-dictproxy.service
+    if host.get_fact(SystemdEnabled).get("doveauth-dictproxy.service"):
+        systemd.service(
+            name="Disable legacy doveauth-dictproxy.service",
+            service="doveauth-dictproxy.service",
+            running=False,
+            enabled=False,
+        )
+
+
+def _install_remote_venv_with_chatmaild(config) -> None:
+    remove_legacy_artifacts()
+    dist_file = _build_chatmaild(dist_dir=Path("chatmaild/dist"))
+    remote_base_dir = "/usr/local/lib/chatmaild"
+    remote_dist_file = f"{remote_base_dir}/dist/{dist_file.name}"
+    remote_venv_dir = f"{remote_base_dir}/venv"
+    remote_chatmail_inipath = f"{remote_base_dir}/chatmail.ini"
+    root_owned = dict(user="root", group="root", mode="644")
+
+    apt.packages(
+        name="apt install python3-virtualenv",
+        packages=["python3-virtualenv"],
+    )
+
+    files.put(
+        name="Upload chatmaild source package",
+        src=dist_file.open("rb"),
+        dest=remote_dist_file,
+        create_remote_dir=True,
+        **root_owned,
+    )
+
+    files.put(
+        name=f"Upload {remote_chatmail_inipath}",
+        src=config._getbytefile(),
+        dest=remote_chatmail_inipath,
+        **root_owned,
+    )
+
+    pip.virtualenv(
+        name=f"chatmaild virtualenv {remote_venv_dir}",
+        path=remote_venv_dir,
+        always_copy=True,
+    )
+
+    server.shell(
+        name=f"forced pip-install {dist_file.name}",
+        commands=[
+            f"{remote_venv_dir}/bin/pip install --force-reinstall {remote_dist_file}"
+        ],
+    )
+
+    # install systemd units
+    for fn in (
+        "doveauth",
+        "filtermail",
+    ):
+        params = dict(
+            execpath=f"{remote_venv_dir}/bin/{fn}",
+            config_path=remote_chatmail_inipath,
+        )
+        source_path = importlib.resources.files("chatmaild").joinpath(f"{fn}.service.f")
+        content = source_path.read_text().format(**params).encode()
+
         files.put(
-            name="Upload chatmaild source package",
-            src=chatmaild_path.open("rb"),
-            dest=remote_path,
+            name=f"Upload {fn}.service",
+            src=io.BytesIO(content),
+            dest=f"/etc/systemd/system/{fn}.service",
+            **root_owned,
         )
-
-        apt.packages(
-            name="apt install python3-aiosmtpd python3-pip python3-venv",
-            packages=["python3-aiosmtpd", "python3-pip", "python3-venv"],
+        systemd.service(
+            name=f"Setup {fn} service",
+            service=f"{fn}.service",
+            running=True,
+            enabled=True,
+            restarted=True,
+            daemon_reload=True,
         )
 
-        # --no-deps because aiosmtplib is installed with `apt`.
-        server.shell(
-            name="install chatmaild with pip",
-            commands=[f"pip install --break-system-packages {remote_path}"],
-        )
-
-        # disable legacy doveauth-dictproxy.service
-        if host.get_fact(SystemdEnabled).get("doveauth-dictproxy.service"):
-            systemd.service(
-                name="Disable legacy doveauth-dictproxy.service",
-                service="doveauth-dictproxy.service",
-                running=False,
-                enabled=False,
-            )
-
-        # install systemd units
-
-        for fn in (
-            "doveauth",
-            "filtermail",
-        ):
-            files.put(
-                name=f"Upload {fn}.service",
-                src=importlib.resources.files("chatmaild")
-                .joinpath(f"{fn}.service")
-                .open("rb"),
-                dest=f"/etc/systemd/system/{fn}.service",
-                user="root",
-                group="root",
-                mode="644",
-            )
-            systemd.service(
-                name=f"Setup {fn} service",
-                service=f"{fn}.service",
-                running=True,
-                enabled=True,
-                restarted=True,
-                daemon_reload=True,
-            )
-
 
 def _configure_opendkim(domain: str, dkim_selector: str) -> bool:
     """Configures OpenDKIM"""
@@ -160,7 +195,7 @@ def _install_mta_sts_daemon() -> bool:
     server.shell(
         name="install postfix-mta-sts-resolver with pip",
         commands=[
-            "python3 -m venv /usr/local/lib/postfix-mta-sts-resolver",
+            "python3 -m virtualenv /usr/local/lib/postfix-mta-sts-resolver",
             "/usr/local/lib/postfix-mta-sts-resolver/bin/pip install postfix-mta-sts-resolver",
         ],
     )
@@ -180,7 +215,7 @@ def _install_mta_sts_daemon() -> bool:
     return need_restart
 
 
-def _configure_postfix(domain: str, debug: bool = False) -> bool:
+def _configure_postfix(config: Config, debug: bool = False) -> bool:
     """Configures Postfix SMTP server."""
     need_restart = False
 
@@ -190,7 +225,7 @@ def _configure_postfix(domain: str, debug: bool = False) -> bool:
         user="root",
         group="root",
         mode="644",
-        config={"domain_name": domain},
+        config=config,
     )
     need_restart |= main_config.changed
 
@@ -201,6 +236,7 @@ def _configure_postfix(domain: str, debug: bool = False) -> bool:
         group="root",
         mode="644",
         debug=debug,
+        config=config,
     )
     need_restart |= master_config.changed
 
@@ -310,80 +346,16 @@ def _configure_nginx(domain: str, debug: bool = False) -> bool:
     return need_restart
 
 
-def get_ini_settings(mail_domain, inipath):
-    parser = configparser.ConfigParser()
-    parser.read(inipath)
-    settings = {key: value.strip() for (key, value) in parser["config"].items()}
+def check_config(config):
+    mail_domain = config.mail_domain
     if mail_domain != "testrun.org" and not mail_domain.endswith(".testrun.org"):
-        for value in settings.values():
-            value = value.lower()
-            if "merlinux" in value or "schmieder" in value or "@testrun.org" in value:
+        blocked_words = "merlinux schmieder testrun.org".split()
+        for value in config.__dict__.values():
+            if any(x in value for x in blocked_words):
                 raise ValueError(
-                    f"please set your own privacy contacts/addresses in {inipath}"
+                    f"please set your own privacy contacts/addresses in {config._inipath}"
                 )
-    settings["mail_domain"] = mail_domain
-    return settings
-
-
-def build_htmlj2_from_markdown(source):
-    assert source.exists(), source
-    template_content = open(source).read()
-    if source.stem == "privacy":
-        title = "privacy {{ config.mail_domain }}"
-    elif source.stem == "index":
-        title = "home {{ config.mail_domain }}"
-    elif source.stem == "info":
-        title = "info {{ config.mail_domain }}"
-
-    html = markdown.markdown(template_content)
-    html = (
-        textwrap.dedent(
-            f"""\
-        <!DOCTYPE html>
-        <html>
-        <head>
-            <meta charset="utf-8" />
-            <title>{title}</title>
-            <link rel="stylesheet" href="./water.css">
-        </head>
-        <body>
-    """
-        )
-        + html
-        + "\n"
-        + textwrap.dedent(
-            """\
-        <footer>
-        <a href="index.html">home</a> |
-        <a href="info.html">more info</a> |
-        <a href="privacy.html">privacy</a> |
-        <a href="https://github.com/deltachat/chatmail">-> public development </a>
-        </footer>
-        </body>"""
-        )
-    )
-
-    target_path = source.with_name(source.stem + ".html.j2")
-    with open(target_path, "w") as f:
-        f.write(html)
-    print(f"wrote {target_path}")
-    return target_path
-
-
-def build_webpages(www_path, config):
-    mail_domain = config["mail_domain"]
-    qr_data = gen_qr_png_data(mail_domain).read()
-    www_path.joinpath(f"qr-chatmail-invite-{mail_domain}.png").write_bytes(qr_data)
-
-    for path in www_path.iterdir():
-        if path.suffix == ".md":
-            path = build_htmlj2_from_markdown(path)
-
-        if path.suffix == ".j2":
-            target = path.with_name(path.name[:-3])
-            template = Template(path.read_text())
-            with target.open("w") as f:
-                f.write(template.render(config=config))
+    return config
 
 
 def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> None:
@@ -393,6 +365,7 @@ def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> N
     :param mail_server: the DNS name under which your mail server is reachable
     :param dkim_selector:
     """
+    from .www import build_webpages
 
     apt.update(name="apt update", cache_time=24 * 3600)
     server.group(name="Create vmail group", group="vmail", system=True)
@@ -439,16 +412,19 @@ def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> N
 
     pkg_root = importlib.resources.files(__package__)
     chatmail_ini = pkg_root.joinpath("../../../chatmail.ini").resolve()
-    config = get_ini_settings(mail_domain, chatmail_ini)
+    config = read_config(chatmail_ini)
+    check_config(config)
     www_path = pkg_root.joinpath("../../../www").resolve()
 
-    build_webpages(www_path, config)
-    files.rsync(f"{www_path}/", "/var/www/html", flags=["-avz"])
+    build_dir = www_path.joinpath("build")
+    src_dir = www_path.joinpath("src")
+    build_webpages(src_dir, build_dir, config)
+    files.rsync(f"{build_dir}/", "/var/www/html", flags=["-avz"])
 
-    _install_chatmaild()
+    _install_remote_venv_with_chatmaild(config)
     debug = False
     dovecot_need_restart = _configure_dovecot(mail_server, debug=debug)
-    postfix_need_restart = _configure_postfix(mail_domain, debug=debug)
+    postfix_need_restart = _configure_postfix(config, debug=debug)
     opendkim_need_restart = _configure_opendkim(mail_domain, dkim_selector)
     mta_sts_need_restart = _install_mta_sts_daemon()
     nginx_need_restart = _configure_nginx(mail_domain)

cmdeploy/src/cmdeploy/chatmail.zone.f

@@ -0,0 +1,12 @@
+{chatmail_domain}.                   MX 10 {chatmail_domain}.
+_submission._tcp.{chatmail_domain}.  SRV 0 1 587 {chatmail_domain}.
+_submissions._tcp.{chatmail_domain}. SRV 0 1 465 {chatmail_domain}.
+_imap._tcp.{chatmail_domain}.        SRV 0 1 143 {chatmail_domain}.
+_imaps._tcp.{chatmail_domain}.       SRV 0 1 993 {chatmail_domain}.
+{chatmail_domain}.                IN CAA 128 issue "letsencrypt.org;accounturi={acme_account_url}"
+{chatmail_domain}.                   TXT "v=spf1 a:{chatmail_domain} -all"
+_dmarc.{chatmail_domain}.            TXT "v=DMARC1;p=reject;rua=mailto:{email};ruf=mailto:{email};fo=1;adkim=r;aspf=r"
+_mta-sts.{chatmail_domain}.       IN TXT "v=STSv1; id={sts_id}"
+mta-sts.{chatmail_domain}.        IN CNAME {chatmail_domain}.
+_smtp._tls.{chatmail_domain}.     IN TXT "v=TLSRPTv1;rua=mailto:{email}"
+{dkim_entry}

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -0,0 +1,298 @@
+"""
+Provides the `cmdeploy` entry point function,
+along with command line option and subcommand parsing.
+"""
+import argparse
+import datetime
+import shutil
+import subprocess
+import importlib.resources
+import importlib.util
+import os
+import sys
+from pathlib import Path
+
+
+from termcolor import colored
+from chatmaild.config import read_config, write_initial_config
+
+
+#
+# cmdeploy sub commands and options
+#
+
+
+def init_cmd_options(parser):
+    parser.add_argument(
+        "chatmail_domain",
+        action="store",
+        help="fully qualified DNS domain name for your chatmail instance",
+    )
+
+
+def init_cmd(args, out):
+    """Initialize chatmail config file."""
+    if args.inipath.exists():
+        out.red(f"Path exists, not modifying: {args.inipath}")
+        raise SystemExit(1)
+    write_initial_config(args.inipath, args.chatmail_domain)
+    out.green(f"created config file for {args.chatmail_domain} in {args.inipath}")
+
+
+def run_cmd_options(parser):
+    parser.add_argument(
+        "--dry-run",
+        dest="dry_run",
+        action="store_true",
+        help="don't actually modify the server",
+    )
+
+
+def run_cmd(args, out):
+    """Deploy chatmail services on the remote server."""
+
+    env = os.environ.copy()
+    env["CHATMAIL_DOMAIN"] = args.config.mail_domain
+    deploy_path = "cmdeploy/src/cmdeploy/deploy.py"
+    pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
+    cmd = f"{pyinf} --ssh-user root {args.config.mail_domain} {deploy_path}"
+    out.check_call(cmd, env=env)
+
+
+def dns_cmd(args, out):
+    """Generate dns zone file."""
+    template = importlib.resources.files(__package__).joinpath("chatmail.zone.f")
+    ssh = f"ssh root@{args.config.mail_domain}"
+
+    def read_dkim_entries(entry):
+        lines = []
+        for line in entry.split("\n"):
+            if line.startswith(";") or not line.strip():
+                continue
+            line = line.replace("\t", " ")
+            lines.append(line)
+        return "\n".join(lines)
+
+    acme_account_url = out.shell_output(f"{ssh} -- acmetool account-url")
+    dkim_entry = read_dkim_entries(out.shell_output(f"{ssh} -- opendkim-genzone -F"))
+
+    out(
+        f"[writing {args.config.mail_domain} zone data (using space as separator) to stdout output]",
+        green=True,
+    )
+    print(
+        template.read_text()
+        .format(
+            acme_account_url=acme_account_url,
+            email=f"root@{args.config.mail_domain}",
+            sts_id=datetime.datetime.now().strftime("%Y%m%d%H%M"),
+            chatmail_domain=args.config.mail_domain,
+            dkim_entry=dkim_entry,
+        )
+        .strip()
+    )
+
+
+def status_cmd(args, out):
+    """Display status for online chatmail instance."""
+
+    ssh = f"ssh root@{args.config.mail_domain}"
+
+    out.green(f"chatmail domain: {args.config.mail_domain}")
+    if args.config.privacy_mail:
+        out.green("privacy settings: present")
+    else:
+        out.red("no privacy settings")
+
+    s1 = "systemctl --type=service --state=running"
+    for line in out.shell_output(f"{ssh} -- {s1}").split("\n"):
+        if line.startswith("  "):
+            print(line)
+
+
+def test_cmd(args, out):
+    """Run local and online tests for chatmail deployment.
+
+    This will automatically pip-install 'deltachat' if it's not available.
+    """
+
+    x = importlib.util.find_spec("deltachat")
+    if x is None:
+        out.check_call(f"{sys.executable} -m pip install deltachat")
+
+    pytest_path = shutil.which("pytest")
+    ret = out.run_ret(
+        [pytest_path, "cmdeploy/src/", "-n4", "-rs", "-x", "-vrx", "--durations=5"]
+    )
+    return ret
+
+
+def fmt_cmd_options(parser):
+    parser.add_argument(
+        "--verbose",
+        "-v",
+        dest="verbose",
+        action="store_true",
+        help="provide information on invocations",
+    )
+
+    parser.add_argument(
+        "--check",
+        "-c",
+        action="store_true",
+        help="only check but don't fix problems",
+    )
+
+
+def fmt_cmd(args, out):
+    """Run formattting fixes (fuff and black) on all chatmail source code."""
+
+    sources = [str(importlib.resources.files(x)) for x in ("chatmaild", "cmdeploy")]
+    black_args = [shutil.which("black")]
+    ruff_args = [shutil.which("ruff")]
+
+    if args.check:
+        black_args.append("--check")
+    else:
+        ruff_args.append("--fix")
+
+    if not args.verbose:
+        black_args.append("-q")
+        ruff_args.append("-q")
+
+    black_args.extend(sources)
+    ruff_args.extend(sources)
+
+    out.check_call(" ".join(black_args), quiet=not args.verbose)
+    out.check_call(" ".join(ruff_args), quiet=not args.verbose)
+    return 0
+
+
+def bench_cmd(args, out):
+    """Run benchmarks against an online chatmail instance."""
+    args = ["pytest", "--pyargs", "cmdeploy.tests.online.benchmark", "-vrx"]
+    cmdstring = " ".join(args)
+    out.green(f"[$ {cmdstring}]")
+    subprocess.check_call(args)
+
+
+def webdev_cmd(args, out):
+    """Run local web development loop for static web pages."""
+    from .www import main
+
+    main()
+
+
+#
+# Parsing command line options and starting commands
+#
+
+
+class Out:
+    """Convenience output printer providing coloring."""
+
+    def red(self, msg, file=sys.stderr):
+        print(colored(msg, "red"), file=file)
+
+    def green(self, msg, file=sys.stderr):
+        print(colored(msg, "green"), file=file)
+
+    def __call__(self, msg, red=False, green=False, file=sys.stdout):
+        color = "red" if red else ("green" if green else None)
+        print(colored(msg, color), file=file)
+
+    def shell_output(self, arg):
+        self(f"[$ {arg}]", file=sys.stderr)
+        return subprocess.check_output(arg, shell=True).decode()
+
+    def check_call(self, arg, env=None, quiet=False):
+        if not quiet:
+            self(f"[$ {arg}]", file=sys.stderr)
+        return subprocess.check_call(arg, shell=True, env=env)
+
+    def run_ret(self, args, env=None, quiet=False):
+        if not quiet:
+            cmdstring = " ".join(args)
+            self(f"[$ {cmdstring}]", file=sys.stderr)
+        proc = subprocess.run(args, env=env)
+        return proc.returncode
+
+
+def add_config_option(parser):
+    parser.add_argument(
+        "--config",
+        dest="inipath",
+        action="store",
+        default=Path("chatmail.ini"),
+        type=Path,
+        help="path to the chatmail.ini file",
+    )
+
+
+def add_subcommand(subparsers, func):
+    name = func.__name__
+    assert name.endswith("_cmd")
+    name = name[:-4]
+    doc = func.__doc__.strip()
+    help = doc.split("\n")[0].strip(".")
+    p = subparsers.add_parser(name, description=doc, help=help)
+    p.set_defaults(func=func)
+    add_config_option(p)
+    return p
+
+
+description = """
+Setup your chatmail server configuration and
+deploy it via SSH to your remote location.
+"""
+
+
+def get_parser():
+    """Return an ArgumentParser for the 'cmdeploy' CLI"""
+
+    parser = argparse.ArgumentParser(description=description.strip())
+    subparsers = parser.add_subparsers(title="subcommands")
+
+    # find all subcommands in the module namespace
+    glob = globals()
+    for name, func in glob.items():
+        if name.endswith("_cmd"):
+            subparser = add_subcommand(subparsers, func)
+            addopts = glob.get(name + "_options")
+            if addopts is not None:
+                addopts(subparser)
+
+    return parser
+
+
+def main(args=None):
+    """Provide main entry point for 'xdcget' CLI invocation."""
+    parser = get_parser()
+    args = parser.parse_args(args=args)
+    if not hasattr(args, "func"):
+        return parser.parse_args(["-h"])
+    out = Out()
+    kwargs = {}
+    if args.func.__name__ not in ("init_cmd", "fmt_cmd"):
+        if not args.inipath.exists():
+            out.red(f"expecting {args.inipath} to exist, run init first?")
+            raise SystemExit(1)
+        try:
+            args.config = read_config(args.inipath)
+        except Exception as ex:
+            out.red(ex)
+            raise SystemExit(1)
+
+    try:
+        res = args.func(args, out, **kwargs)
+        if res is None:
+            res = 0
+        return res
+
+    except KeyboardInterrupt:
+        out.red("KeyboardInterrupt")
+        sys.exit(130)
+
+
+if __name__ == "__main__":
+    main()

cmdeploy/src/cmdeploy/deploy.py

@@ -1,6 +1,6 @@
 import os
 import pyinfra
-from deploy_chatmail import deploy_chatmail
+from cmdeploy import deploy_chatmail
 
 
 def main():

cmdeploy/src/cmdeploy/genqr.py

@@ -49,7 +49,7 @@ def gen_qr(maildomain, url):
     size = width = 384
     qr_padding = 6
     text_height = font_size * num_lines
-    height = size + text_height + qr_padding * 2
+    height = size + text_height
 
     image = Image.new("RGBA", (width, height), "white")
     qr_final_size = width - (qr_padding * 2)

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -1,4 +1,4 @@
-myorigin = {{ config.domain_name }}
+myorigin = {{ config.mail_domain }}
 
 smtpd_banner = $myhostname ESMTP $mail_name (Debian/GNU)
 biff = no
@@ -16,8 +16,8 @@ readme_directory = no
 compatibility_level = 2
 
 # TLS parameters
-smtpd_tls_cert_file=/var/lib/acme/live/{{ config.domain_name }}/fullchain
-smtpd_tls_key_file=/var/lib/acme/live/{{ config.domain_name }}/privkey
+smtpd_tls_cert_file=/var/lib/acme/live/{{ config.mail_domain }}/fullchain
+smtpd_tls_key_file=/var/lib/acme/live/{{ config.mail_domain }}/privkey
 smtpd_tls_security_level=may
 
 smtp_tls_CApath=/etc/ssl/certs
@@ -26,7 +26,7 @@ smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
 smtp_tls_policy_maps = socketmap:inet:127.0.0.1:8461:postfix
 
 smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination
-myhostname = {{ config.domain_name }}
+myhostname = {{ config.mail_domain }}
 alias_maps = hash:/etc/aliases
 alias_database = hash:/etc/aliases
 
@@ -45,7 +45,7 @@ inet_interfaces = all
 inet_protocols = all
 
 virtual_transport = lmtp:unix:private/dovecot-lmtp
-virtual_mailbox_domains = {{ config.domain_name }}
+virtual_mailbox_domains = {{ config.mail_domain }}
 
 smtpd_milters = unix:opendkim/opendkim.sock
 non_smtpd_milters = $smtpd_milters

cmdeploy/src/cmdeploy/postfix/master.cf.j2

@@ -33,7 +33,7 @@ submission inet n       -       y       -       -       smtpd
   -o smtpd_relay_restrictions=permit_sasl_authenticated,reject
   -o milter_macro_daemon_name=ORIGINATING
   -o smtpd_client_connection_count_limit=1000
-  -o smtpd_proxy_filter=127.0.0.1:10080
+  -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
 smtps     inet  n       -       y       -       -       smtpd
   -o syslog_name=postfix/smtps
   -o smtpd_tls_wrappermode=yes
@@ -49,7 +49,7 @@ smtps     inet  n       -       y       -       -       smtpd
   -o smtpd_relay_restrictions=permit_sasl_authenticated,reject
   -o smtpd_client_connection_count_limit=1000
   -o milter_macro_daemon_name=ORIGINATING
-  -o smtpd_proxy_filter=127.0.0.1:10080
+  -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
 #628       inet  n       -       y       -       -       qmqpd
 pickup    unix  n       -       y       60      1       pickup
 cleanup   unix  n       -       y       -       0       cleanup
@@ -78,5 +78,5 @@ scache    unix  -       -       y       -       1       scache
 postlog   unix-dgram n  -       n       -       1       postlogd
 filter    unix -        n       n       -       -       lmtp
 # Local SMTP server for reinjecting filered mail.
-localhost:10025 inet  n       -       n       -       10      smtpd
+localhost:{{ config.postfix_reinject_port }} inet  n       -       n       -       10      smtpd
   -o syslog_name=postfix/reinject

cmdeploy/src/cmdeploy/tests/online/test_0_qr.py

@@ -1,6 +1,4 @@
-
-
-from deploy_chatmail.genqr import gen_qr_png_data
+from cmdeploy.genqr import gen_qr_png_data
 
 
 def test_gen_qr_png_data(maildomain):

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -20,7 +20,7 @@ def test_use_two_chatmailservers(cmfactory, maildomain2):
 
 
 @pytest.mark.parametrize("forgeaddr", ["internal", "someone@example.org"])
-def test_reject_forged_from(cmsetup, maildata, lp, forgeaddr):
+def test_reject_forged_from(cmsetup, maildata, gencreds, lp, forgeaddr):
     user1, user3 = cmsetup.gen_users(2)
 
     lp.sec("send encrypted message with forged from")
@@ -54,7 +54,7 @@ def test_exceed_rate_limit(cmsetup, gencreds, maildata):
         try:
             user1.smtp.sendmail(user1.addr, [user2.addr], mail)
         except smtplib.SMTPException as e:
-            if i < 80:
+            if i < 60:
                 pytest.fail(f"rate limit was exceeded too early with msg {i}")
             outcome = e.recipients[user2.addr]
             assert outcome[0] == 450

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -91,9 +91,9 @@ class TestEndToEndDeltaChat:
 
         lp.sec("setup encrypted comms between ac1 and ac2 on different instances")
         qr = ac1.get_setup_contact_qr()
-        ac2.qr_setup_contact(qr)
-        msg = ac2.wait_next_incoming_message()
-        assert "verified" in msg.text
+        ch = ac2.qr_setup_contact(qr)
+        assert ch.id >= 10
+        ac1._evtracker.wait_securejoin_inviter_progress(1000)
 
         lp.sec("ac1 sends a message and ac2 marks it as seen")
         chat = ac1.create_chat(ac2)

cmdeploy/src/cmdeploy/tests/plugin.py

@@ -6,12 +6,11 @@ import subprocess
 import imaplib
 import smtplib
 import itertools
-from email.parser import BytesParser
-from email import policy
 from pathlib import Path
 import pytest
 
 from chatmaild.database import Database
+from chatmaild.config import read_config
 
 
 conftestdir = Path(__file__).parent
@@ -38,11 +37,22 @@ def pytest_runtest_setup(item):
 
 
 @pytest.fixture
-def maildomain():
-    domain = os.environ.get("CHATMAIL_DOMAIN")
-    if not domain:
-        pytest.skip("set CHATMAIL_DOMAIN to a ssh-reachable chatmail instance")
-    return domain
+def chatmail_config(pytestconfig):
+    current = basedir = Path().resolve()
+    while 1:
+        path = current.joinpath("chatmail.ini").resolve()
+        if path.exists():
+            return read_config(path)
+        if current == current.parent:
+            break
+        current = current.parent
+
+    pytest.skip(f"no chatmail.ini file found in {basedir} or parent dirs")
+
+
+@pytest.fixture
+def maildomain(chatmail_config):
+    return chatmail_config.mail_domain
 
 
 @pytest.fixture
@@ -278,11 +288,13 @@ class ChatmailTestProcess:
 
 
 @pytest.fixture
-def cmfactory(request, gencreds, tmpdir, data, maildomain):
+def cmfactory(request, gencreds, tmpdir, maildomain):
     # cloned from deltachat.testplugin.amfactory
     pytest.importorskip("deltachat")
     from deltachat.testplugin import ACFactory
 
+    data = request.getfixturevalue("data")
+
     testproc = ChatmailTestProcess(request.config, maildomain, gencreds)
     am = ACFactory(request=request, tmpdir=tmpdir, testprocess=testproc, data=data)
 
@@ -327,19 +339,15 @@ class Remote:
 
 
 @pytest.fixture
-def maildata(request, gencreds):
-    datadir = conftestdir.joinpath("mail-data")
+def lp(request):
+    class LP:
+        def sec(self, msg):
+            print(f"---- {msg} ----")
 
-    def maildata(name, from_addr=None, to_addr=None):
-        if from_addr is None:
-            from_addr = gencreds()[0]
-        if to_addr is None:
-            to_addr = gencreds()[0]
-        data = datadir.joinpath(name).read_text()
-        text = data.format(from_addr=from_addr, to_addr=to_addr)
-        return BytesParser(policy=policy.default).parsebytes(text.encode())
+        def indent(self, msg):
+            print(f"     {msg}")
 
-    return maildata
+    return LP()
 
 
 @pytest.fixture

cmdeploy/src/cmdeploy/tests/test_cmdeploy.py

@@ -0,0 +1,33 @@
+import os
+
+import pytest
+from cmdeploy.cmdeploy import get_parser, main
+from chatmaild.config import read_config
+
+
+@pytest.fixture(autouse=True)
+def _chdir(tmp_path):
+    old = os.getcwd()
+    os.chdir(tmp_path)
+    yield
+    os.chdir(old)
+
+
+class TestCmdline:
+    def test_parser(self, capsys):
+        parser = get_parser()
+        parser.parse_args([])
+        init = parser.parse_args(["init", "chat.example.org"])
+        run = parser.parse_args(["run"])
+        assert init and run
+
+    def test_init(self, tmp_path):
+        main(["init", "chat.example.org"])
+        inipath = tmp_path.joinpath("chatmail.ini")
+        config = read_config(inipath)
+        assert config.mail_domain == "chat.example.org"
+
+    def test_init_not_overwrite(self):
+        main(["init", "chat.example.org"])
+        with pytest.raises(SystemExit):
+            main(["init", "chat.example.org"])

cmdeploy/src/cmdeploy/tests/test_helpers.py

@@ -0,0 +1,13 @@
+import importlib.resources
+
+from cmdeploy.www import build_webpages
+
+
+def test_build_webpages(tmp_path, make_config):
+    pkgroot = importlib.resources.files("cmdeploy")
+    src_dir = pkgroot.joinpath("../../../www/src").resolve()
+    assert src_dir.exists(), src_dir
+    config = make_config("chat.example.org")
+    build_dir = tmp_path.joinpath("build")
+    build_webpages(src_dir, build_dir, config)
+    assert len([x for x in build_dir.iterdir() if x.suffix == ".html"]) >= 3

cmdeploy/src/cmdeploy/www.py

@@ -0,0 +1,110 @@
+import importlib.resources
+import webbrowser
+import hashlib
+import time
+import traceback
+
+import markdown
+from jinja2 import Template
+from .genqr import gen_qr_png_data
+from chatmaild.config import read_config
+
+
+def snapshot_dir_stats(somedir):
+    d = {}
+    for path in somedir.iterdir():
+        if path.is_file() and path.name[0] != "." and path.suffix != ".swp":
+            mtime = path.stat().st_mtime
+            hash = hashlib.md5(path.read_bytes()).hexdigest()
+            d[path] = (mtime, hash)
+    return d
+
+
+def prepare_template(source):
+    assert source.exists(), source
+    render_vars = {}
+    render_vars["pagename"] = "home" if source.stem == "index" else source.stem
+    render_vars["markdown_html"] = markdown.markdown(source.read_text())
+    page_layout = source.with_name("page-layout.html").read_text()
+    return render_vars, page_layout
+
+
+def build_webpages(src_dir, build_dir, config):
+    try:
+        _build_webpages(src_dir, build_dir, config)
+    except Exception:
+        print(traceback.format_exc())
+
+
+def _build_webpages(src_dir, build_dir, config):
+    mail_domain = config.mail_domain
+    assert src_dir.exists(), src_dir
+    if not build_dir.exists():
+        build_dir.mkdir()
+
+    qr_path = build_dir.joinpath(f"qr-chatmail-invite-{mail_domain}.png")
+    qr_path.write_bytes(gen_qr_png_data(mail_domain).read())
+
+    for path in src_dir.iterdir():
+        if path.suffix == ".md":
+            render_vars, content = prepare_template(path)
+            target = build_dir.joinpath(path.stem + ".html")
+
+            # recursive jinja2 rendering
+            while 1:
+                new = Template(content).render(config=config, **render_vars)
+                if new == content:
+                    break
+                content = new
+
+            with target.open("w") as f:
+                f.write(content)
+        elif path.name != "page-layout.html":
+            target = build_dir.joinpath(path.name)
+            target.write_bytes(path.read_bytes())
+    return build_dir
+
+
+def main():
+    path = importlib.resources.files(__package__)
+    reporoot = path.joinpath("../../../").resolve()
+    inipath = reporoot.joinpath("chatmail.ini")
+    config = read_config(inipath)
+    config.webdev = True
+    assert config.mail_domain
+    www_path = reporoot.joinpath("www")
+    src_path = www_path.joinpath("src")
+    stats = None
+    build_dir = www_path.joinpath("build")
+    src_dir = www_path.joinpath("src")
+    index_path = build_dir.joinpath("index.html")
+
+    # start web page generation, open a browser and wait for changes
+    build_webpages(src_dir, build_dir, config)
+    webbrowser.open(str(index_path))
+    stats = snapshot_dir_stats(src_path)
+    print(f"\nOpened URL: file://{index_path.resolve()}\n")
+    print(f"watching {src_path} directory for changes")
+
+    changenum = 0
+    for count in range(0, 1000000):
+        newstats = snapshot_dir_stats(src_path)
+        if newstats == stats and count % 60 != 0:
+            count += 1
+            time.sleep(1.0)
+            continue
+
+        for key in newstats:
+            if stats[key] != newstats[key]:
+                print(f"*** CHANGED: {key}")
+                changenum += 1
+
+        stats = newstats
+        build_webpages(src_dir, build_dir, config)
+        print(f"[{changenum}] regenerated web pages at: {index_path}")
+        print(f"URL: file://{index_path.resolve()}\n\n")
+        count = 0
+
+
+if __name__ == "__main__":
+    main()

deploy-chatmail/pyproject.toml

@@ -1,33 +0,0 @@
-[build-system]
-requires = ["setuptools>=45"]
-build-backend = "setuptools.build_meta"
-
-[project]
-name = "deploy-chatmail"
-version = "0.1"
-dependencies = [
-  "pyinfra",
-  "pillow",
-  "qrcode",
-  "markdown",
-]
-
-[tool.pytest.ini_options]
-addopts = "-v -ra --strict-markers"
-
-[tool.tox]
-legacy_tox_ini = """
-[tox]
-isolated_build = true
-envlist = lint
-
-[testenv:lint]
-skipdist = True
-skip_install = True
-deps =
-  ruff
-  black
-commands =
-  black --quiet --check --diff src/
-  ruff src/
-"""

plan.txt

@@ -1,65 +0,0 @@
-# Chat-mail server development (up until Oct 18th)
-
-## Dovecot goals/steps
-
-- automatic expiry of messages older than M days
-   - also expunge unread messages
-
-- limit: configure max-connections per account
-
-
-## nami: send out rate limit / rspamd
-
-- basic outgoing send rate/limits (depending on "account-rating")
-  use rspamd in a minimal way, check support dkim-signing
-  (including an online test exceeding rate limit)
-
-
-## doveauth questions/futures
-
-- bcrypt-password scheme is slow: require long passwords, use faster hashing
-
-- define user-name and password policies, and implement them
-  (be very restrictive at the beginning, we can relax later)
-
-- password is part of the dictproxy-lookup key, is it safe to use auth-caching?
-
-
-## How to limit creation of accounts?
-
-attack: a 3-line bash script to fill the chatmail db with millions of unused accouts
-
-- make it computationally expensive (somehow try to except our tests from it)
-  1st pass instant onboarding: create userid + cheap password -- if it fails then
-  2nd pass instant onboarding: create userdid + comput. expensive password
-
-- probably also do firewall: limit number of new tcp-connections per IP address per duration
-
-
-## Open/deferred questions
-
-- automatic expiry of users that haven't logged in for N days
-  Is it neccessary?  If all messages are gone, does the existence of
-  an e-mail address bother anybody?
-
-
-## web page for chat-mail servers?
-
-- documentation for users, privacy policy etc.
-  (probably also with provider-messages ...)
-
-
-## online tests (first with plain python/pytest)
-
-- write tests for dovecot login (exists)
-- write tests for postfix logins (exists)
-- write A<>B send/receive tests (exists)
-
-
-## Delta Chat
-
-1. qr code that defines access to a chatmail instance (like mailadm but without http etc.)
-
-2. support for creating username/password and verifying login works
-
-

scripts/bench.sh

@@ -1,4 +0,0 @@
-#!/bin/bash
-set -e
-
-venv/bin/pytest tests/online/benchmark.py -vrx

scripts/deploy.sh

@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-
-echo -----------------------------------------
-echo deploying to $CHATMAIL_DOMAIN 
-echo -----------------------------------------
-
-echo WARNING: in five seconds deploy to $CHATMAIL_DOMAIN starts
-sleep 5
-
-venv/bin/python3 -m build -n --sdist chatmaild --outdir dist
-
-venv/bin/pyinfra --ssh-user root "$CHATMAIL_DOMAIN" \
-    deploy-chatmail/src/deploy_chatmail/deploy.py
-
-rm -r dist/

scripts/generate-dns-zone.sh

@@ -1,23 +0,0 @@
-#!/bin/sh
-: ${CHATMAIL_DOMAIN:=c1.testrun.org}
-: ${CHATMAIL_SSH:=$CHATMAIL_DOMAIN}
-
-set -e
-SSH="ssh root@$CHATMAIL_SSH"
-EMAIL="root@$CHATMAIL_DOMAIN"
-ACME_ACCOUNT_URL="$($SSH -- acmetool account-url)"
-
-cat <<EOF
-$CHATMAIL_DOMAIN. MX 10 $CHATMAIL_DOMAIN.
-$CHATMAIL_DOMAIN. TXT "v=spf1 a:$CHATMAIL_DOMAIN -all"
-_dmarc.$CHATMAIL_DOMAIN. TXT "v=DMARC1;p=reject;rua=mailto:$EMAIL;ruf=mailto:$EMAIL;fo=1;adkim=r;aspf=r"
-_submission._tcp.$CHATMAIL_DOMAIN.  SRV 0 1 587 $CHATMAIL_DOMAIN.
-_submissions._tcp.$CHATMAIL_DOMAIN. SRV 0 1 465 $CHATMAIL_DOMAIN.
-_imap._tcp.$CHATMAIL_DOMAIN.        SRV 0 1 143 $CHATMAIL_DOMAIN.
-_imaps._tcp.$CHATMAIL_DOMAIN.       SRV 0 1 993 $CHATMAIL_DOMAIN.
-$CHATMAIL_DOMAIN. IN CAA 128 issue "letsencrypt.org;accounturi=$ACME_ACCOUNT_URL"
-_mta-sts.$CHATMAIL_DOMAIN. IN TXT "v=STSv1; id=$(date -u '+%Y%m%d%H%M')"
-mta-sts.$CHATMAIL_DOMAIN. IN CNAME $CHATMAIL_DOMAIN.
-_smtp._tls.$CHATMAIL_DOMAIN. IN TXT "v=TLSRPTv1;rua=mailto:$EMAIL"
-EOF
-$SSH opendkim-genzone -F | sed 's/^;.*$//;/^$/d'

scripts/get_imap_capabilities.py

@@ -1,14 +0,0 @@
-import os
-import time
-import imaplib
-
-domain = os.environ.get("CHATMAIL_DOMAIN", "c3.testrun.org")
-
-print("connecting")
-conn = imaplib.IMAP4_SSL(domain)
-print("logging in")
-conn.login(f"imapcapa", "pass")
-status, res = conn.capability()
-for capa in sorted(res[0].decode().split()):
-    print(capa)
-

scripts/init.sh

@@ -1,8 +0,0 @@
-#!/bin/sh
-set -e
-python3 -m venv venv
-pip=venv/bin/pip
-
-$pip install pyinfra pytest build 'setuptools>=68' tox deltachat
-$pip install -e deploy-chatmail 
-$pip install -e chatmaild 

scripts/initenv.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+python3 -m venv venv
+
+venv/bin/pip install -e chatmaild 
+venv/bin/pip install -e cmdeploy
+
+source venv/bin/activate
+echo activated 'venv' python virtualenv environment containing "cmdeploy" tool

scripts/measure_tls_and_logins.py

@@ -1,28 +0,0 @@
-#!/usr/bin/env python3
-import os
-import time
-import imaplib
-
-domain = os.environ.get("CHATMAIL_DOMAIN", "c3.testrun.org")
-
-NUM_CONNECTIONS=10
-
-conns = []
-
-start = time.time()
-for i in range(NUM_CONNECTIONS):
-    print(f"opening connection {i} to {domain}")
-    conn = imaplib.IMAP4_SSL(domain)
-    conns.append(conn)
-
-tlsdone = time.time()
-duration = tlsdone-start
-print(f"{duration}: TLS connections opening TLS connections")
-
-for i, conn in enumerate(conns):
-    print(f"logging into connection {i}")
-    conn.login(f"measure{i}", "pass")
-
-logindone = time.time()
-duration = logindone - tlsdone
-print(f"{duration}: LOGINS done")

scripts/remote-deploy.sh

@@ -1,8 +0,0 @@
-#!/bin/sh
-set -e
-
-: ${CHATMAIL_DOMAIN:=c1.testrun.org}
-: ${CHATMAIL_SSH:=$CHATMAIL_DOMAIN}
-
-rsync -avz . "root@$CHATMAIL_SSH:/root/chatmail" --exclude='/.git' --filter="dir-merge,- .gitignore"
-ssh "root@$CHATMAIL_SSH" "cd /root/chatmail; apt install -y python3-venv; python3 -m venv venv; venv/bin/pip install pyinfra build; venv/bin/python3 -m build -n --sdist chatmaild --outdir dist; venv/bin/pip install -e ./deploy-chatmail -e ./chatmaild; export CHATMAIL_DOMAIN=$CHATMAIL_DOMAIN; venv/bin/pyinfra @local deploy.py"

scripts/test.sh

@@ -1,4 +0,0 @@
-#!/bin/bash
-venv/bin/tox -c chatmaild
-venv/bin/tox -c deploy-chatmail
-venv/bin/pytest tests/online -rs -vrx --durations=5 $@

tests/chatmaild/test_filtermail.py

@@ -1,82 +0,0 @@
-from chatmaild.filtermail import check_encrypted, check_DATA, SendRateLimiter, check_mdn
-import pytest
-
-
-@pytest.fixture
-def maildomain():
-    # let's not depend on a real chatmail instance for the offline tests below
-    return "chatmail.example.org"
-
-
-def test_reject_forged_from(maildata, gencreds):
-    class env:
-        mail_from = gencreds()[0]
-        rcpt_tos = [gencreds()[0]]
-
-    # test that the filter lets good mail through
-    env.content = maildata("plain.eml", from_addr=env.mail_from).as_bytes()
-    assert not check_DATA(envelope=env)
-
-    # test that the filter rejects forged mail
-    env.content = maildata("plain.eml", from_addr="forged@c3.testrun.org").as_bytes()
-    error = check_DATA(envelope=env)
-    assert "500" in error
-
-
-def test_filtermail_no_encryption_detection(maildata):
-    msg = maildata("plain.eml")
-    assert not check_encrypted(msg)
-
-    # https://xkcd.com/1181/
-    msg = maildata("fake-encrypted.eml")
-    assert not check_encrypted(msg)
-
-
-def test_filtermail_encryption_detection(maildata):
-    msg = maildata("encrypted.eml")
-    assert check_encrypted(msg)
-
-    # if the subject is not "..." it is not considered ac-encrypted
-    msg.replace_header("Subject", "Click this link")
-    assert not check_encrypted(msg)
-
-
-def test_filtermail_is_mdn(maildata, gencreds):
-    from_addr = gencreds()[0]
-    to_addr = gencreds()[0] + ".other"
-    msg = maildata("mdn.eml", from_addr, to_addr)
-
-    class env:
-        mail_from = from_addr
-        rcpt_tos = [to_addr]
-        content = msg.as_bytes()
-
-    assert check_mdn(msg, env)
-    print(msg.as_string())
-    assert not check_DATA(env)
-
-
-def test_filtermail_to_multiple_recipients_no_mdn(maildata, gencreds):
-    from_addr = gencreds()[0]
-    to_addr = gencreds()[0] + ".other"
-    thirdaddr = gencreds()[0]
-    msg = maildata("mdn.eml", from_addr, to_addr)
-
-    class env:
-        mail_from = from_addr
-        rcpt_tos = [to_addr, thirdaddr]
-        content = msg.as_bytes()
-
-    assert not check_mdn(msg, env)
-
-
-def test_send_rate_limiter():
-    limiter = SendRateLimiter()
-    for i in range(100):
-        if limiter.is_sending_allowed("some@example.org"):
-            if i <= SendRateLimiter.MAX_USER_SEND_PER_MINUTE:
-                continue
-            pytest.fail("limiter didn't work")
-        else:
-            assert i == SendRateLimiter.MAX_USER_SEND_PER_MINUTE + 1
-            break

tests/test_helpers.py

@@ -1,39 +0,0 @@
-import textwrap
-
-from deploy_chatmail import build_htmlj2_from_markdown, get_ini_settings
-
-
-def test_markdown(tmp_path):
-    path = tmp_path.joinpath("privacy.md")
-    path.write_text("# privacy policy")
-    build_htmlj2_from_markdown(path)
-    output = path.with_name("privacy.html.j2")
-    assert output.exists()
-    print(output.read_text())
-
-
-def test_get_settings(tmp_path):
-    inipath = tmp_path.joinpath("chatmail.ini")
-    inipath.write_text(
-        textwrap.dedent(
-            """\
-        [config]
-
-        privacy_postal =
-            address-line1
-            address-line2
-
-        privacy_mail = privacy@example.org
-
-        privacy_pdo =
-            address-line3
-    """
-        )
-    )
-    d = get_ini_settings("x.testrun.org", inipath)
-    assert d["privacy_postal"] == "address-line1\naddress-line2"
-    assert d["privacy_mail"] == "privacy@example.org"
-    assert d["privacy_pdo"] == "address-line3"
-    assert d["mail_domain"] == "x.testrun.org"
-
-

www/src/index.md

@@ -1,5 +1,5 @@
 
-<img width="800px" src="collage-top.png"/>
+<img class="banner" src="collage-top.png"/>
 
 ## Dear [Delta Chat](https://get.delta.chat) users and newcomers, 
 
@@ -14,6 +14,7 @@ Welcome to instant, interoperable and [privacy-preserving](privacy.html) messagi
 
 💬 **Start** chatting with any Delta Chat contacts using [QR invite codes](https://delta.chat/en/help#howtoe2ee)
 
+<div class="experimental">Note: this is an experimental service</div>
+
 
-## ⚡ Note: this is an experimental service ⚡
 

www/src/info.md

@@ -1,5 +1,5 @@
 
-<img width="800px" src="collage-info.png"/>
+<img class="banner" src="collage-info.png"/>
 
 ## More information 
 

www/src/logo.svg

@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   width="145"
+   height="145"
+   version="1.1"
+   id="svg4"
+   sodipodi:docname="At_sign.svg"
+   inkscape:version="1.2.2 (b0a84865, 2022-12-01)"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <defs
+     id="defs8" />
+  <sodipodi:namedview
+     id="namedview6"
+     pagecolor="#ffffff"
+     bordercolor="#000000"
+     borderopacity="0.25"
+     inkscape:showpageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     inkscape:deskcolor="#d1d1d1"
+     showgrid="false"
+     inkscape:zoom="3.0241379"
+     inkscape:cx="67.622577"
+     inkscape:cy="72.913341"
+     inkscape:window-width="1390"
+     inkscape:window-height="1027"
+     inkscape:window-x="55"
+     inkscape:window-y="25"
+     inkscape:window-maximized="0"
+     inkscape:current-layer="svg4" />
+  <g
+     aria-label="@"
+     id="text2"
+     style="font-size:144px;font-family:Arial">
+    <path
+       d="m 79.927878,94.422406 c -2.704286,3.120332 -5.741407,5.637394 -9.111364,7.551194 -3.328352,1.87221 -6.677506,2.80831 -10.047463,2.80831 -3.702792,0 -7.301573,-1.08172 -10.796342,-3.24515 -3.49477,-2.163426 -6.344671,-5.491779 -8.549704,-9.985058 -2.163429,-4.493275 -3.245144,-9.423397 -3.245144,-14.790365 0,-6.615099 1.684978,-13.230199 5.054935,-19.845299 3.411561,-6.656705 7.634407,-11.649233 12.66854,-14.977585 5.034133,-3.328352 9.92265,-4.992528 14.665552,-4.992528 3.619583,0 7.072748,0.956901 10.359496,2.870704 3.286748,1.872198 6.115847,4.742902 8.487297,8.612111 l 2.121825,-9.673023 h 11.170784 l -8.986557,41.87483 c -1.248129,5.824616 -1.872194,9.048957 -1.872194,9.673023 0,1.123319 0.416044,2.101022 1.248132,2.93311 0.873692,0.790484 1.913802,1.185726 3.120332,1.185726 2.20503,0 5.096537,-1.268934 8.674517,-3.806803 4.7429,-3.328352 8.4873,-7.780023 11.23319,-13.355013 2.78749,-5.616594 4.18124,-11.399606 4.18124,-17.349035 0,-6.947935 -1.78899,-13.438222 -5.36697,-19.47086 -3.53637,-6.032638 -8.84094,-10.858749 -15.913687,-14.478332 -7.03114,-3.619583 -14.811161,-5.429374 -23.340064,-5.429374 -9.73543,0 -18.638772,2.288242 -26.710026,6.864726 -8.029649,4.534879 -14.27031,11.06677 -18.721981,19.595673 -4.410066,8.487298 -6.615099,17.598662 -6.615099,27.334092 0,10.193078 2.205033,18.971607 6.615099,26.33559 2.290454,3.78888 -7.136335,18.96983 -3.810585,21.73443 3.138096,2.60861 18.971963,-7.14297 23.031819,-5.44631 8.404089,3.53637 17.702673,5.30456 27.895752,5.30456 10.90035,0 20.032515,-1.83059 27.396492,-5.49178 7.36399,-3.66119 12.87657,-8.11286 16.53776,-13.35501 l 9.29559,4 c -2.12183,4.36846 -3.76221,4.82013 -8.92116,9.35501 -5.15895,4.53488 -11.2956,8.11286 -18.40995,10.73393 -7.114346,2.66268 -15.684851,3.99402 -25.711512,3.99402 -9.236177,0 -17.76508,-1.18572 -25.586707,-3.55717 -7.780023,-2.37145 -29.296198,9.26152 -34.78798,4.47701 -5.49178,-4.7429 5.248856,-25.42482 2.461361,-31.62388 -3.49477,-7.863231 -5.242155,-16.350531 -5.242155,-25.461894 0,-10.151474 2.08022,-19.824498 6.240661,-29.019071 5.075736,-11.274793 12.273297,-19.907706 21.592683,-25.898739 9.360991,-5.991034 20.69819,-8.986551 34.011599,-8.986551 10.317891,0 19.574873,2.121824 27.77093,6.365473 8.23767,4.202045 14.72796,10.484309 19.47086,18.846794 4.03563,7.197561 6.05344,15.019189 6.05344,23.464883 0,12.065277 -4.24365,22.77841 -12.73094,32.1394 -7.572,8.404095 -15.85128,12.606135 -24.837827,12.606135 -2.870704,0 -5.200551,-0.43684 -6.98954,-1.31053 -1.747385,-0.8737 -3.037121,-2.12183 -3.869209,-3.744402 -0.540857,-1.040114 -0.936099,-2.829105 -1.185726,-5.366972 z M 49.723082,77.510217 c 0,5.699803 1.352143,10.130671 4.05643,13.292606 2.704286,3.161935 5.803814,4.742902 9.298583,4.742902 2.329847,0 4.784506,-0.686473 7.363979,-2.059418 2.579473,-1.41455 5.034133,-3.49477 7.363979,-6.240661 2.371451,-2.74589 4.306056,-6.219857 5.803815,-10.421902 1.497759,-4.243649 2.246638,-8.487298 2.246638,-12.730947 0,-5.658198 -1.41455,-10.047462 -4.243649,-13.167793 -2.787495,-3.12033 -6.199056,-4.680495 -10.234683,-4.680495 -2.662682,0 -5.179749,0.686473 -7.5512,2.059418 -2.329846,1.331341 -4.597286,3.494769 -6.802319,6.490286 -2.205033,2.995517 -3.97322,6.635903 -5.304561,10.921156 -1.331341,4.285253 -1.997012,8.216869 -1.997012,11.794848 z"
+       id="path347"
+       style="stroke-width:0.887561"
+       sodipodi:nodetypes="ccsscscsscccccscsccsccsccscscssccscscccsccsccscscccssscccscscsss" />
+  </g>
+</svg>

www/src/main.css

@@ -0,0 +1,31 @@
+
+#menu {
+  display: flex;
+  flex-wrap: wrap;
+  padding: 0;
+}
+
+#menu li {
+  display: inline-block;
+  padding-right: 0.5em;
+}
+
+#domain {
+  margin-left: auto;
+}
+
+#domain a {
+  color: #888;
+}
+
+.banner {
+  width: 100%;
+}
+
+.experimental {
+  margin: 3em 0;
+  padding: 1em;
+  border: 4px dashed red;
+  color: red;
+  font-weight: bold;
+}

www/src/page-layout.html

@@ -0,0 +1,27 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8" />
+    {% if config.webdev %}
+    <meta http-equiv="refresh" content="3">
+    {% endif %}
+    <title>{{ config.mail_domain }} {{ pagename }}</title>
+    <link rel="stylesheet" href="./water.css">
+    <link rel="stylesheet" href="./main.css">
+    <link rel="icon" href="/logo.svg">
+    <link rel=”mask-icon” href=”/logo.svg” color=”#000000">
+</head>
+<body>
+
+<ul id="menu">
+    <li><a href="index.html">home</a></li>
+    <li><a href="info.html">info</a></li>
+    <li><a href="privacy.html">privacy</a></li>
+    <li><a href="https://github.com/deltachat/chatmail">public code ↗</a></li>
+    <li id="domain"><a href="index.html">{{ config.mail_domain }}</a></li>
+</ul>
+
+{{ markdown_html }}
+
+</body>
+</html>

www/src/privacy.md

@@ -1,4 +1,4 @@
-<img width="800px" src="collage-privacy.png"/>
+<img class="banner" src="collage-privacy.png"/>
 
 # Privacy Policy for {{ config.mail_domain }}