in webdev mode make page auto-refresh every 3 seconds

avoid bailing out on jinja2 errors, and provide file-url for instant clickability
don't depend on deltachat python package
2026-05-10 16:04:37 +00:00 · 2023-12-08 14:31:03 +01:00 · 2023-12-08 14:02:51 +01:00 · 2023-12-08 12:15:39 +01:00 · 2023-12-07 17:41:25 +01:00 · 2023-12-07 17:39:03 +01:00
18 changed files with 206 additions and 90 deletions
--- a/README.md
+++ b/README.md
@@ -51,6 +51,28 @@ The `deploy.sh` script deploys
 All files are generated by the according markdown `.md` file in the `www` directory.


+### Refining the web pages 
+
+The `scripts/webdev.sh` script supports live development of the chatmail web presence:
+
+``` 
+    scripts/init.sh  # to locally initialize python virtual environments etc. 
+    scripts/webdev.sh
+```
+
+- uses the `www/src/page-layout.html` file for producing html documents
+  from `www/src/*.md` files. 
+
+- continously builds the web presence reading files from `www/src` directory
+  and generating html files and copying assets to the `www/build` directory. 
+
+- Starts a browser window automatically where you can "refresh" as needed. 
+
+Note that this script is not needed for running `scripts/deploy.sh" 
+which deploys the whole chatmail setup remotely. 
+The code that generates the web pages is identical
+which means that `webdev.sh` gives a pretty good preview. 
+
 ### Ports

 Postfix listens on ports 25 (smtp) and 587 (submission) and 465 (submissions).
--- a/deploy-chatmail/src/deploy_chatmail/init.py
+++ b/deploy-chatmail/src/deploy_chatmail/init.py
@@ -3,7 +3,6 @@ Chat Mail pyinfra deploy.
 """
 import importlib.resources
 import configparser
-import textwrap
 from pathlib import Path

 from pyinfra import host
@@ -11,11 +10,6 @@ from pyinfra.operations import apt, files, server, systemd
 from pyinfra.facts.files import File
 from pyinfra.facts.systemd import SystemdEnabled
 from .acmetool import deploy_acmetool
-import markdown
-from jinja2 import Template
-
-
-from .genqr import gen_qr_png_data


 def _install_chatmaild() -> None:
@@ -325,67 +319,6 @@ def get_ini_settings(mail_domain, inipath):
    return settings


-def build_htmlj2_from_markdown(source):
-    assert source.exists(), source
-    template_content = open(source).read()
-    if source.stem == "privacy":
-        title = "privacy {{ config.mail_domain }}"
-    elif source.stem == "index":
-        title = "home {{ config.mail_domain }}"
-    elif source.stem == "info":
-        title = "info {{ config.mail_domain }}"
-
-    html = markdown.markdown(template_content)
-    html = (
-        textwrap.dedent(
-            f"""\
-        <!DOCTYPE html>
-        <html>
-        <head>
-            <meta charset="utf-8" />
-            <title>{title}</title>
-            <link rel="stylesheet" href="./water.css">
-        </head>
-        <body>
-    """
-        )
-        + html
-        + "\n"
-        + textwrap.dedent(
-            """\
-        <footer>
-        <a href="index.html">home</a> |
-        <a href="info.html">more info</a> |
-        <a href="privacy.html">privacy</a> |
-        <a href="https://github.com/deltachat/chatmail">-> public development </a>
-        </footer>
-        </body>"""
-        )
-    )
-
-    target_path = source.with_name(source.stem + ".html.j2")
-    with open(target_path, "w") as f:
-        f.write(html)
-    print(f"wrote {target_path}")
-    return target_path
-
-
-def build_webpages(www_path, config):
-    mail_domain = config["mail_domain"]
-    qr_data = gen_qr_png_data(mail_domain).read()
-    www_path.joinpath(f"qr-chatmail-invite-{mail_domain}.png").write_bytes(qr_data)
-
-    for path in www_path.iterdir():
-        if path.suffix == ".md":
-            path = build_htmlj2_from_markdown(path)
-
-        if path.suffix == ".j2":
-            target = path.with_name(path.name[:-3])
-            template = Template(path.read_text())
-            with target.open("w") as f:
-                f.write(template.render(config=config))
-
-
 def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> None:
    """Deploy a chat-mail instance.

@@ -393,6 +326,7 @@ def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> N
    :param mail_server: the DNS name under which your mail server is reachable
    :param dkim_selector:
    """
+    from .www import build_webpages

    apt.update(name="apt update", cache_time=24 * 3600)
    server.group(name="Create vmail group", group="vmail", system=True)
@@ -442,8 +376,10 @@ def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> N
    config = get_ini_settings(mail_domain, chatmail_ini)
    www_path = pkg_root.joinpath("../../../www").resolve()

-    build_webpages(www_path, config)
-    files.rsync(f"{www_path}/", "/var/www/html", flags=["-avz"])
+    build_dir = www_path.joinpath("build")
+    src_dir = www_path.joinpath("src")
+    build_webpages(src_dir, build_dir, config)
+    files.rsync(f"{build_dir}/", "/var/www/html", flags=["-avz"])

    _install_chatmaild()
    debug = False
--- a/deploy-chatmail/src/deploy_chatmail/genqr.py
+++ b/deploy-chatmail/src/deploy_chatmail/genqr.py
@@ -49,10 +49,10 @@ def gen_qr(maildomain, url):
    size = width = 384
    qr_padding = 6
    text_height = font_size * num_lines
-    height = size + text_height + qr_padding * 2
+    height = size + text_height

    image = Image.new("RGBA", (width, height), "white")
-    qr_final_size = width - (qr_padding * 2)
+    qr_final_size = width

    if num_lines:
        draw = ImageDraw.Draw(image)
--- a/deploy-chatmail/src/deploy_chatmail/www.py
+++ b/deploy-chatmail/src/deploy_chatmail/www.py
@@ -0,0 +1,110 @@
+import importlib.resources
+import webbrowser
+import hashlib
+import time
+import traceback
+
+import markdown
+from jinja2 import Template
+from .genqr import gen_qr_png_data
+from deploy_chatmail import get_ini_settings
+
+
+def snapshot_dir_stats(somedir):
+    d = {}
+    for path in somedir.iterdir():
+        if path.is_file() and path.name[0] != "." and path.suffix != ".swp":
+            mtime = path.stat().st_mtime
+            hash = hashlib.md5(path.read_bytes()).hexdigest()
+            d[path] = (mtime, hash)
+    return d
+
+
+def prepare_template(source):
+    assert source.exists(), source
+    render_vars = {}
+    render_vars["pagename"] = "home" if source.stem == "index" else source.stem
+    render_vars["markdown_html"] = markdown.markdown(source.read_text())
+    page_layout = source.with_name("page-layout.html").read_text()
+    return render_vars, page_layout
+
+
+def build_webpages(src_dir, build_dir, config):
+    try:
+        _build_webpages(src_dir, build_dir, config)
+    except Exception:
+        print(traceback.format_exc())
+
+
+def _build_webpages(src_dir, build_dir, config):
+    mail_domain = config["mail_domain"]
+    assert src_dir.exists(), src_dir
+    if not build_dir.exists():
+        build_dir.mkdir()
+
+    qr_path = build_dir.joinpath(f"qr-chatmail-invite-{mail_domain}.png")
+    qr_path.write_bytes(gen_qr_png_data(mail_domain).read())
+
+    for path in src_dir.iterdir():
+        if path.suffix == ".md":
+            render_vars, content = prepare_template(path)
+            target = build_dir.joinpath(path.stem + ".html")
+
+            # recursive jinja2 rendering
+            while 1:
+                new = Template(content).render(config=config, **render_vars)
+                if new == content:
+                    break
+                content = new
+
+            with target.open("w") as f:
+                f.write(content)
+        elif path.name != "page-layout.html":
+            target = build_dir.joinpath(path.name)
+            target.write_bytes(path.read_bytes())
+    return build_dir
+
+
+def main():
+    chatmail_domain = "example.testrun.org"
+    path = importlib.resources.files(__package__)
+    reporoot = path.joinpath("../../../").resolve()
+    inipath = reporoot.joinpath("chatmail.ini")
+    config = get_ini_settings(chatmail_domain, inipath)
+    config["webdev"] = True
+    www_path = reporoot.joinpath("www")
+    src_path = www_path.joinpath("src")
+    stats = None
+    build_dir = www_path.joinpath("build")
+    src_dir = www_path.joinpath("src")
+    index_path = build_dir.joinpath("index.html")
+
+    # start web page generation, open a browser and wait for changes
+    build_webpages(src_dir, build_dir, config)
+    webbrowser.open(str(index_path))
+    stats = snapshot_dir_stats(src_path)
+    print(f"\nOpened URL: file://{index_path.resolve()}\n")
+    print(f"watching {src_path} directory for changes")
+
+    changenum = 0
+    for count in range(0, 1000000):
+        newstats = snapshot_dir_stats(src_path)
+        if newstats == stats and count % 60 != 0:
+            count += 1
+            time.sleep(1.0)
+            continue
+
+        for key in newstats:
+            if stats[key] != newstats[key]:
+                print(f"*** CHANGED: {key}")
+                changenum += 1
+
+        stats = newstats
+        build_webpages(src_dir, build_dir, config)
+        print(f"[{changenum}] regenerated web pages at: {index_path}")
+        print(f"URL: file://{index_path.resolve()}\n\n")
+        count = 0
+
+
+if __name__ == "__main__":
+    main()
--- a/scripts/init.sh
+++ b/scripts/init.sh
@@ -3,6 +3,6 @@ set -e
 python3 -m venv venv
 pip=venv/bin/pip

-$pip install pyinfra pytest build 'setuptools>=68' tox deltachat
+$pip install pyinfra pytest build 'setuptools>=68' tox 
 $pip install -e deploy-chatmail 
 $pip install -e chatmaild 
--- a/scripts/webdev.sh
+++ b/scripts/webdev.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+echo -----------------------------------------
+echo starting local webdev 
+echo -----------------------------------------
+
+venv/bin/python3 -m deploy_chatmail.www 
+
+
--- a/tests/chatmaild/test_newmail.py
+++ b/tests/chatmaild/test_newmail.py
@@ -3,6 +3,7 @@ import json
 import chatmaild
 from chatmaild.newemail import create_newemail_dict, print_new_account

+
 def test_create_newemail_dict():
    ac1 = create_newemail_dict(domain="example.org")
    assert "@" in ac1["email"]
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -278,13 +278,13 @@ class ChatmailTestProcess:


@pytest.fixture
-def cmfactory(request, gencreds, tmpdir, data, maildomain):
+def cmfactory(request, gencreds, tmpdir, maildomain):
    # cloned from deltachat.testplugin.amfactory
    pytest.importorskip("deltachat")
    from deltachat.testplugin import ACFactory

    testproc = ChatmailTestProcess(request.config, maildomain, gencreds)
-    am = ACFactory(request=request, tmpdir=tmpdir, testprocess=testproc, data=data)
+    am = ACFactory(request=request, tmpdir=tmpdir, testprocess=testproc, data=None)

    # nb. a bit hacky
    # would probably be better if deltachat's test machinery grows native support
@@ -326,6 +326,18 @@ class Remote:
                break


+@pytest.fixture
+def lp(request):
+    class LP:
+        def sec(self, msg):
+            print(f"---- {msg} ----")
+
+        def indent(self, msg):
+            print(f"     {msg}")
+
+    return LP()
+
+
@pytest.fixture
 def maildata(request, gencreds):
    datadir = conftestdir.joinpath("mail-data")
--- a/tests/online/test_0_qr.py
+++ b/tests/online/test_0_qr.py
@@ -1,5 +1,3 @@
-
-
 from deploy_chatmail.genqr import gen_qr_png_data


--- a/tests/test_helpers.py
+++ b/tests/test_helpers.py
@@ -1,19 +1,11 @@
 import textwrap
+import importlib.resources

-from deploy_chatmail import build_htmlj2_from_markdown, get_ini_settings
+from deploy_chatmail.www import build_webpages
+from deploy_chatmail import get_ini_settings


-def test_markdown(tmp_path):
-    path = tmp_path.joinpath("privacy.md")
-    path.write_text("# privacy policy")
-    build_htmlj2_from_markdown(path)
-    output = path.with_name("privacy.html.j2")
-    assert output.exists()
-    print(output.read_text())
-
-
-def test_get_settings(tmp_path):
-    inipath = tmp_path.joinpath("chatmail.ini")
+def create_ini(inipath):
    inipath.write_text(
        textwrap.dedent(
            """\
@@ -30,10 +22,26 @@ def test_get_settings(tmp_path):
    """
        )
    )
+
+
+def test_build_webpages(tmp_path):
+    pkgroot = importlib.resources.files("deploy_chatmail")
+    src_dir = pkgroot.joinpath("../../../www/src").resolve()
+    assert src_dir.exists(), src_dir
+
+    inipath = tmp_path.joinpath("chatmail.ini")
+    create_ini(inipath)
+    config = get_ini_settings("example.org", inipath)
+    build_dir = tmp_path.joinpath("build")
+    build_webpages(src_dir, build_dir, config)
+
+
+def test_get_settings(tmp_path):
+    inipath = tmp_path.joinpath("chatmail.ini")
+    create_ini(inipath)
+
    d = get_ini_settings("x.testrun.org", inipath)
    assert d["privacy_postal"] == "address-line1\naddress-line2"
    assert d["privacy_mail"] == "privacy@example.org"
    assert d["privacy_pdo"] == "address-line3"
    assert d["mail_domain"] == "x.testrun.org"
-
-
--- a/www/src/collage-info.png
+++ b/www/src/collage-info.png
--- a/www/src/collage-privacy.png
+++ b/www/src/collage-privacy.png
--- a/www/src/collage-top.png
+++ b/www/src/collage-top.png
--- a/www/src/index.md
+++ b/www/src/index.md
--- a/www/src/info.md
+++ b/www/src/info.md
--- a/www/src/page-layout.html
+++ b/www/src/page-layout.html
@@ -0,0 +1,20 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8" />
+    {% if config.webdev %}
+    <meta http-equiv="refresh" content="3">
+    {% endif %}
+    <title>{{ config.mail_domain }} {{ pagename }}</title>
+    <link rel="stylesheet" href="./water.css">
+</head>
+<body>
+{{ markdown_html }}
+<footer>
+    <a href="index.html">home</a> |
+    <a href="info.html">more info</a> |
+    <a href="privacy.html">privacy</a> |
+    <a href="https://github.com/deltachat/chatmail">-> public development </a>
+</footer>
+</body>
+</html>
--- a/www/src/privacy.md
+++ b/www/src/privacy.md
--- a/www/src/water.css
+++ b/www/src/water.css
Author	SHA1	Message	Date
holger krekel	9531d62f57	in webdev mode make page auto-refresh every 3 seconds	2023-12-08 14:31:03 +01:00
holger krekel	931305d454	avoid bailing out on jinja2 errors, and provide file-url for instant clickability	2023-12-08 14:02:51 +01:00
holger krekel	648174c226	don't depend on deltachat python package	2023-12-08 12:15:39 +01:00
holger krekel	2b5cdd143e	add a note	2023-12-07 17:41:25 +01:00
holger krekel	f506a62196	fix README	2023-12-07 17:39:03 +01:00
holger krekel	e8107c6854	rename script	2023-12-07 17:34:43 +01:00
holger krekel	858e079418	create a wwwdev.sh entry point for developing the web part	2023-12-07 17:34:43 +01:00
				`@@ -1,5 +1,3 @@`


				`from deploy_chatmail.genqr import gen_qr_png_data`