deploy chatmaild in a virtualenv to make it easier to add dependencies

* create a wwwdev.sh entry point for developing the web part

* rename script

* fix README

* add a note

* don't depend on deltachat python package

* avoid bailing out on jinja2 errors, and provide file-url for instant clickability

* in webdev mode make page auto-refresh every 3 seconds

README.md

@@ -1,5 +1,5 @@
 
-<img width="800px" src="www/collage-top.png"/>
+<img width="800px" src="www/src/collage-top.png"/>
 
 # Chatmail instances optimized for Delta Chat apps 
 
@@ -51,6 +51,28 @@ The `deploy.sh` script deploys
 All files are generated by the according markdown `.md` file in the `www` directory.
 
 
+### Refining the web pages 
+
+The `scripts/webdev.sh` script supports live development of the chatmail web presence:
+
+``` 
+    scripts/init.sh  # to locally initialize python virtual environments etc. 
+    scripts/webdev.sh
+```
+
+- uses the `www/src/page-layout.html` file for producing html documents
+  from `www/src/*.md` files. 
+
+- continously builds the web presence reading files from `www/src` directory
+  and generating html files and copying assets to the `www/build` directory. 
+
+- Starts a browser window automatically where you can "refresh" as needed. 
+
+Note that this script is not needed for running `scripts/deploy.sh" 
+which deploys the whole chatmail setup remotely. 
+The code that generates the web pages is identical
+which means that `webdev.sh` gives a pretty good preview. 
+
 ### Ports
 
 Postfix listens on ports 25 (smtp) and 587 (submission) and 465 (submissions).

chatmaild/src/chatmaild/doveauth.service.f

@@ -2,7 +2,7 @@
 Description=Dict authentication proxy for dovecot
 
 [Service]
-ExecStart=/usr/local/bin/doveauth /run/dovecot/doveauth.socket vmail /home/vmail/passdb.sqlite
+ExecStart={execpath} /run/dovecot/doveauth.socket vmail /home/vmail/passdb.sqlite
 Restart=always
 RestartSec=30
 

chatmaild/src/chatmaild/filtermail.py

@@ -34,6 +34,14 @@ def check_encrypted(message):
     return True
 
 
+def is_passthrough_recipient(recipient):
+    """Check whether a recipient is configured as passthrough."""
+    passthroughlist = ["privacy@testrun.org"]
+    if recipient in passthroughlist:
+        return True
+    return False
+
+
 def check_mdn(message, envelope):
     if len(envelope.rcpt_tos) != 1:
         return False
@@ -118,6 +126,9 @@ def check_DATA(envelope):
         if envelope.mail_from == recipient:
             # Always allow sending emails to self.
             continue
+        if is_passthrough_recipient(recipient):
+            # Always allow recipients marked as passthrough
+            continue
         res = recipient.split("@")
         if len(res) != 2:
             return f"500 Invalid address <{recipient}>"

chatmaild/src/chatmaild/filtermail.service.f

@@ -2,7 +2,7 @@
 Description=Chatmail Postfix BeforeQeue filter 
 
 [Service]
-ExecStart=/usr/local/bin/filtermail 10080
+ExecStart={execpath} 10080
 Restart=always
 RestartSec=30
 

deploy-chatmail/src/deploy_chatmail/__init__.py

@@ -1,81 +1,102 @@
 """
 Chat Mail pyinfra deploy.
 """
+import sys
 import importlib.resources
+import subprocess
+import shutil
+import io
 import configparser
-import textwrap
 from pathlib import Path
 
 from pyinfra import host
-from pyinfra.operations import apt, files, server, systemd
+from pyinfra.operations import apt, files, server, systemd, pip
 from pyinfra.facts.files import File
 from pyinfra.facts.systemd import SystemdEnabled
 from .acmetool import deploy_acmetool
-import markdown
-from jinja2 import Template
 
 
-from .genqr import gen_qr_png_data
+def _build_chatmaild(dist_dir) -> None:
-
+    dist_dir = Path(dist_dir).resolve()
-
+    if dist_dir.exists():
-def _install_chatmaild() -> None:
+        shutil.rmtree(dist_dir)
-    chatmaild_filename = "chatmaild-0.1.tar.gz"
+    dist_dir.mkdir()
-    chatmaild_path = importlib.resources.files(__package__).joinpath(
+    subprocess.check_output(
-        f"../../../dist/{chatmaild_filename}"
+        [sys.executable, "-m", "build", "-n"]
+        + ["--sdist", "chatmaild", "--outdir", str(dist_dir)]
     )
-    remote_path = f"/tmp/{chatmaild_filename}"
+    entries = list(dist_dir.iterdir())
-    if Path(str(chatmaild_path)).exists():
+    assert len(entries) == 1
+    return entries[0]
+
+
+def _install_remote_venv_with_chatmaild() -> None:
+    dist_file = _build_chatmaild(dist_dir=Path("chatmaild/dist"))
+    remote_base_dir = "/usr/local/lib/chatmaild"
+    remote_dist_file = f"{remote_base_dir}/dist/{dist_file.name}"
+    remote_venv_dir = f"{remote_base_dir}/venv"
+    root_owned = dict(user="root", group="root", mode="644")
+
+    apt.packages(
+        name="apt install python3-virtualenv",
+        packages=["python3-virtualenv"],
+    )
+
+    files.put(
+        name="Upload chatmaild source package",
+        src=dist_file.open("rb"),
+        dest=remote_dist_file,
+        create_remote_dir=True,
+        **root_owned,
+    )
+
+    pip.virtualenv(
+        name=f"chatmaild virtualenv {remote_venv_dir}",
+        path=remote_venv_dir,
+        always_copy=True,
+    )
+
+    server.shell(
+        name=f"forced pip-install {dist_file.name}",
+        commands=[
+            f"{remote_venv_dir}/bin/pip install --force-reinstall {remote_dist_file}"
+        ],
+    )
+
+    # disable legacy doveauth-dictproxy.service
+    if host.get_fact(SystemdEnabled).get("doveauth-dictproxy.service"):
+        systemd.service(
+            name="Disable legacy doveauth-dictproxy.service",
+            service="doveauth-dictproxy.service",
+            running=False,
+            enabled=False,
+        )
+
+    # install systemd units
+
+    for fn in (
+        "doveauth",
+        "filtermail",
+    ):
+        execpath = f"{remote_venv_dir}/bin/{fn}"
+        source_path = importlib.resources.files("chatmaild").joinpath(f"{fn}.service.f")
+        content = source_path.read_text().format(execpath=execpath).encode()
+
         files.put(
-            name="Upload chatmaild source package",
+            name=f"Upload {fn}.service",
-            src=chatmaild_path.open("rb"),
+            src=io.BytesIO(content),
-            dest=remote_path,
+            dest=f"/etc/systemd/system/{fn}.service",
+            **root_owned,
         )
-
+        systemd.service(
-        apt.packages(
+            name=f"Setup {fn} service",
-            name="apt install python3-aiosmtpd python3-pip python3-venv",
+            service=f"{fn}.service",
-            packages=["python3-aiosmtpd", "python3-pip", "python3-venv"],
+            running=True,
+            enabled=True,
+            restarted=True,
+            daemon_reload=True,
         )
 
-        # --no-deps because aiosmtplib is installed with `apt`.
-        server.shell(
-            name="install chatmaild with pip",
-            commands=[f"pip install --break-system-packages {remote_path}"],
-        )
-
-        # disable legacy doveauth-dictproxy.service
-        if host.get_fact(SystemdEnabled).get("doveauth-dictproxy.service"):
-            systemd.service(
-                name="Disable legacy doveauth-dictproxy.service",
-                service="doveauth-dictproxy.service",
-                running=False,
-                enabled=False,
-            )
-
-        # install systemd units
-
-        for fn in (
-            "doveauth",
-            "filtermail",
-        ):
-            files.put(
-                name=f"Upload {fn}.service",
-                src=importlib.resources.files("chatmaild")
-                .joinpath(f"{fn}.service")
-                .open("rb"),
-                dest=f"/etc/systemd/system/{fn}.service",
-                user="root",
-                group="root",
-                mode="644",
-            )
-            systemd.service(
-                name=f"Setup {fn} service",
-                service=f"{fn}.service",
-                running=True,
-                enabled=True,
-                restarted=True,
-                daemon_reload=True,
-            )
-
 
 def _configure_opendkim(domain: str, dkim_selector: str) -> bool:
     """Configures OpenDKIM"""
@@ -325,67 +346,6 @@ def get_ini_settings(mail_domain, inipath):
     return settings
 
 
-def build_htmlj2_from_markdown(source):
-    assert source.exists(), source
-    template_content = open(source).read()
-    if source.stem == "privacy":
-        title = "privacy {{ config.mail_domain }}"
-    elif source.stem == "index":
-        title = "home {{ config.mail_domain }}"
-    elif source.stem == "info":
-        title = "info {{ config.mail_domain }}"
-
-    html = markdown.markdown(template_content)
-    html = (
-        textwrap.dedent(
-            f"""\
-        <!DOCTYPE html>
-        <html>
-        <head>
-            <meta charset="utf-8" />
-            <title>{title}</title>
-            <link rel="stylesheet" href="./water.css">
-        </head>
-        <body>
-    """
-        )
-        + html
-        + "\n"
-        + textwrap.dedent(
-            """\
-        <footer>
-        <a href="index.html">home</a> |
-        <a href="info.html">more info</a> |
-        <a href="privacy.html">privacy</a> |
-        <a href="https://github.com/deltachat/chatmail">-> public development </a>
-        </footer>
-        </body>"""
-        )
-    )
-
-    target_path = source.with_name(source.stem + ".html.j2")
-    with open(target_path, "w") as f:
-        f.write(html)
-    print(f"wrote {target_path}")
-    return target_path
-
-
-def build_webpages(www_path, config):
-    mail_domain = config["mail_domain"]
-    qr_data = gen_qr_png_data(mail_domain).read()
-    www_path.joinpath(f"qr-chatmail-invite-{mail_domain}.png").write_bytes(qr_data)
-
-    for path in www_path.iterdir():
-        if path.suffix == ".md":
-            path = build_htmlj2_from_markdown(path)
-
-        if path.suffix == ".j2":
-            target = path.with_name(path.name[:-3])
-            template = Template(path.read_text())
-            with target.open("w") as f:
-                f.write(template.render(config=config))
-
-
 def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> None:
     """Deploy a chat-mail instance.
 
@@ -393,6 +353,7 @@ def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> N
     :param mail_server: the DNS name under which your mail server is reachable
     :param dkim_selector:
     """
+    from .www import build_webpages
 
     apt.update(name="apt update", cache_time=24 * 3600)
     server.group(name="Create vmail group", group="vmail", system=True)
@@ -442,10 +403,12 @@ def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> N
     config = get_ini_settings(mail_domain, chatmail_ini)
     www_path = pkg_root.joinpath("../../../www").resolve()
 
-    build_webpages(www_path, config)
+    build_dir = www_path.joinpath("build")
-    files.rsync(f"{www_path}/", "/var/www/html", flags=["-avz"])
+    src_dir = www_path.joinpath("src")
+    build_webpages(src_dir, build_dir, config)
+    files.rsync(f"{build_dir}/", "/var/www/html", flags=["-avz"])
 
-    _install_chatmaild()
+    _install_remote_venv_with_chatmaild()
     debug = False
     dovecot_need_restart = _configure_dovecot(mail_server, debug=debug)
     postfix_need_restart = _configure_postfix(mail_domain, debug=debug)

deploy-chatmail/src/deploy_chatmail/genqr.py

@@ -49,7 +49,7 @@ def gen_qr(maildomain, url):
     size = width = 384
     qr_padding = 6
     text_height = font_size * num_lines
-    height = size + text_height + qr_padding * 2
+    height = size + text_height
 
     image = Image.new("RGBA", (width, height), "white")
     qr_final_size = width - (qr_padding * 2)

deploy-chatmail/src/deploy_chatmail/www.py

@@ -0,0 +1,110 @@
+import importlib.resources
+import webbrowser
+import hashlib
+import time
+import traceback
+
+import markdown
+from jinja2 import Template
+from .genqr import gen_qr_png_data
+from deploy_chatmail import get_ini_settings
+
+
+def snapshot_dir_stats(somedir):
+    d = {}
+    for path in somedir.iterdir():
+        if path.is_file() and path.name[0] != "." and path.suffix != ".swp":
+            mtime = path.stat().st_mtime
+            hash = hashlib.md5(path.read_bytes()).hexdigest()
+            d[path] = (mtime, hash)
+    return d
+
+
+def prepare_template(source):
+    assert source.exists(), source
+    render_vars = {}
+    render_vars["pagename"] = "home" if source.stem == "index" else source.stem
+    render_vars["markdown_html"] = markdown.markdown(source.read_text())
+    page_layout = source.with_name("page-layout.html").read_text()
+    return render_vars, page_layout
+
+
+def build_webpages(src_dir, build_dir, config):
+    try:
+        _build_webpages(src_dir, build_dir, config)
+    except Exception:
+        print(traceback.format_exc())
+
+
+def _build_webpages(src_dir, build_dir, config):
+    mail_domain = config["mail_domain"]
+    assert src_dir.exists(), src_dir
+    if not build_dir.exists():
+        build_dir.mkdir()
+
+    qr_path = build_dir.joinpath(f"qr-chatmail-invite-{mail_domain}.png")
+    qr_path.write_bytes(gen_qr_png_data(mail_domain).read())
+
+    for path in src_dir.iterdir():
+        if path.suffix == ".md":
+            render_vars, content = prepare_template(path)
+            target = build_dir.joinpath(path.stem + ".html")
+
+            # recursive jinja2 rendering
+            while 1:
+                new = Template(content).render(config=config, **render_vars)
+                if new == content:
+                    break
+                content = new
+
+            with target.open("w") as f:
+                f.write(content)
+        elif path.name != "page-layout.html":
+            target = build_dir.joinpath(path.name)
+            target.write_bytes(path.read_bytes())
+    return build_dir
+
+
+def main():
+    chatmail_domain = "example.testrun.org"
+    path = importlib.resources.files(__package__)
+    reporoot = path.joinpath("../../../").resolve()
+    inipath = reporoot.joinpath("chatmail.ini")
+    config = get_ini_settings(chatmail_domain, inipath)
+    config["webdev"] = True
+    www_path = reporoot.joinpath("www")
+    src_path = www_path.joinpath("src")
+    stats = None
+    build_dir = www_path.joinpath("build")
+    src_dir = www_path.joinpath("src")
+    index_path = build_dir.joinpath("index.html")
+
+    # start web page generation, open a browser and wait for changes
+    build_webpages(src_dir, build_dir, config)
+    webbrowser.open(str(index_path))
+    stats = snapshot_dir_stats(src_path)
+    print(f"\nOpened URL: file://{index_path.resolve()}\n")
+    print(f"watching {src_path} directory for changes")
+
+    changenum = 0
+    for count in range(0, 1000000):
+        newstats = snapshot_dir_stats(src_path)
+        if newstats == stats and count % 60 != 0:
+            count += 1
+            time.sleep(1.0)
+            continue
+
+        for key in newstats:
+            if stats[key] != newstats[key]:
+                print(f"*** CHANGED: {key}")
+                changenum += 1
+
+        stats = newstats
+        build_webpages(src_dir, build_dir, config)
+        print(f"[{changenum}] regenerated web pages at: {index_path}")
+        print(f"URL: file://{index_path.resolve()}\n\n")
+        count = 0
+
+
+if __name__ == "__main__":
+    main()

scripts/deploy.sh

@@ -4,12 +4,5 @@ echo -----------------------------------------
 echo deploying to $CHATMAIL_DOMAIN 
 echo -----------------------------------------
 
-echo WARNING: in five seconds deploy to $CHATMAIL_DOMAIN starts
-sleep 5
-
-venv/bin/python3 -m build -n --sdist chatmaild --outdir dist
-
 venv/bin/pyinfra --ssh-user root "$CHATMAIL_DOMAIN" \
     deploy-chatmail/src/deploy_chatmail/deploy.py
-
-rm -r dist/

scripts/init.sh

@@ -3,6 +3,6 @@ set -e
 python3 -m venv venv
 pip=venv/bin/pip
 
-$pip install pyinfra pytest build 'setuptools>=68' tox deltachat
+$pip install pyinfra pytest build 'setuptools>=68' tox 
 $pip install -e deploy-chatmail 
 $pip install -e chatmaild 

scripts/webdev.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+echo -----------------------------------------
+echo starting local webdev 
+echo -----------------------------------------
+
+venv/bin/python3 -m deploy_chatmail.www 
+
+

tests/chatmaild/test_filtermail.py

@@ -1,4 +1,4 @@
-from chatmaild.filtermail import check_encrypted, check_DATA, SendRateLimiter, check_mdn
+from chatmaild.filtermail import check_encrypted, check_DATA, SendRateLimiter, check_mdn, is_passthrough_recipient
 import pytest
 
 
@@ -80,3 +80,28 @@ def test_send_rate_limiter():
         else:
             assert i == SendRateLimiter.MAX_USER_SEND_PER_MINUTE + 1
             break
+
+
+def test_excempt_privacy(maildata, gencreds):
+    from_addr = gencreds()[0]
+    to_addr = "privacy@testrun.org"
+    false_to = "privacy@tstrn.org"
+    false_to2 = "prvcy@testrun.org"
+    assert is_passthrough_recipient(to_addr)
+
+    msg = maildata("plain.eml", from_addr, to_addr)
+
+    class env:
+        mail_from = from_addr
+        rcpt_tos = [to_addr]
+        content = msg.as_bytes()
+
+    # assert that None/no error is returned
+    assert not check_DATA(envelope=env)
+
+    class env2:
+        mail_from = from_addr
+        rcpt_tos = [to_addr, false_to, false_to2]
+        content = msg.as_bytes()
+
+    assert "500" in check_DATA(envelope=env2)

tests/chatmaild/test_newmail.py

@@ -3,6 +3,7 @@ import json
 import chatmaild
 from chatmaild.newemail import create_newemail_dict, print_new_account
 
+
 def test_create_newemail_dict():
     ac1 = create_newemail_dict(domain="example.org")
     assert "@" in ac1["email"]

tests/conftest.py

@@ -278,11 +278,13 @@ class ChatmailTestProcess:
 
 
 @pytest.fixture
-def cmfactory(request, gencreds, tmpdir, data, maildomain):
+def cmfactory(request, gencreds, tmpdir, maildomain):
     # cloned from deltachat.testplugin.amfactory
     pytest.importorskip("deltachat")
     from deltachat.testplugin import ACFactory
 
+    data = request.getfixturevalue("data")
+
     testproc = ChatmailTestProcess(request.config, maildomain, gencreds)
     am = ACFactory(request=request, tmpdir=tmpdir, testprocess=testproc, data=data)
 
@@ -326,6 +328,18 @@ class Remote:
                 break
 
 
+@pytest.fixture
+def lp(request):
+    class LP:
+        def sec(self, msg):
+            print(f"---- {msg} ----")
+
+        def indent(self, msg):
+            print(f"     {msg}")
+
+    return LP()
+
+
 @pytest.fixture
 def maildata(request, gencreds):
     datadir = conftestdir.joinpath("mail-data")

tests/online/test_0_qr.py

@@ -1,5 +1,3 @@
-
-
 from deploy_chatmail.genqr import gen_qr_png_data
 
 

tests/test_helpers.py

@@ -1,19 +1,11 @@
 import textwrap
+import importlib.resources
 
-from deploy_chatmail import build_htmlj2_from_markdown, get_ini_settings
+from deploy_chatmail.www import build_webpages
+from deploy_chatmail import get_ini_settings
 
 
-def test_markdown(tmp_path):
+def create_ini(inipath):
-    path = tmp_path.joinpath("privacy.md")
-    path.write_text("# privacy policy")
-    build_htmlj2_from_markdown(path)
-    output = path.with_name("privacy.html.j2")
-    assert output.exists()
-    print(output.read_text())
-
-
-def test_get_settings(tmp_path):
-    inipath = tmp_path.joinpath("chatmail.ini")
     inipath.write_text(
         textwrap.dedent(
             """\
@@ -30,10 +22,26 @@ def test_get_settings(tmp_path):
     """
         )
     )
+
+
+def test_build_webpages(tmp_path):
+    pkgroot = importlib.resources.files("deploy_chatmail")
+    src_dir = pkgroot.joinpath("../../../www/src").resolve()
+    assert src_dir.exists(), src_dir
+
+    inipath = tmp_path.joinpath("chatmail.ini")
+    create_ini(inipath)
+    config = get_ini_settings("example.org", inipath)
+    build_dir = tmp_path.joinpath("build")
+    build_webpages(src_dir, build_dir, config)
+
+
+def test_get_settings(tmp_path):
+    inipath = tmp_path.joinpath("chatmail.ini")
+    create_ini(inipath)
+
     d = get_ini_settings("x.testrun.org", inipath)
     assert d["privacy_postal"] == "address-line1\naddress-line2"
     assert d["privacy_mail"] == "privacy@example.org"
     assert d["privacy_pdo"] == "address-line3"
     assert d["mail_domain"] == "x.testrun.org"
-
-

www/src/page-layout.html

@@ -0,0 +1,20 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8" />
+    {% if config.webdev %}
+    <meta http-equiv="refresh" content="3">
+    {% endif %}
+    <title>{{ config.mail_domain }} {{ pagename }}</title>
+    <link rel="stylesheet" href="./water.css">
+</head>
+<body>
+{{ markdown_html }}
+<footer>
+    <a href="index.html">home</a> |
+    <a href="info.html">more info</a> |
+    <a href="privacy.html">privacy</a> |
+    <a href="https://github.com/deltachat/chatmail">-> public development </a>
+</footer>
+</body>
+</html>