diff --git a/README.md b/README.md index f7418fb6..918f9b77 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,28 @@ The `deploy.sh` script deploys All files are generated by the according markdown `.md` file in the `www` directory. +### Refining the web pages + +The `scripts/webdev.sh` script supports live development of the chatmail web presence: + +``` + scripts/init.sh # to locally initialize python virtual environments etc. + scripts/webdev.sh +``` + +- uses the `www/src/page-layout.html` file for producing html documents + from `www/src/*.md` files. + +- continously builds the web presence reading files from `www/src` directory + and generating html files and copying assets to the `www/build` directory. + +- Starts a browser window automatically where you can "refresh" as needed. + +Note that this script is not needed for running `scripts/deploy.sh" +which deploys the whole chatmail setup remotely. +The code that generates the web pages is identical +which means that `webdev.sh` gives a pretty good preview. + ### Ports Postfix listens on ports 25 (smtp) and 587 (submission) and 465 (submissions). diff --git a/deploy-chatmail/src/deploy_chatmail/__init__.py b/deploy-chatmail/src/deploy_chatmail/__init__.py index 197526bf..20dc0dfd 100644 --- a/deploy-chatmail/src/deploy_chatmail/__init__.py +++ b/deploy-chatmail/src/deploy_chatmail/__init__.py @@ -3,7 +3,6 @@ Chat Mail pyinfra deploy. """ import importlib.resources import configparser -import textwrap from pathlib import Path from pyinfra import host @@ -11,11 +10,6 @@ from pyinfra.operations import apt, files, server, systemd from pyinfra.facts.files import File from pyinfra.facts.systemd import SystemdEnabled from .acmetool import deploy_acmetool -import markdown -from jinja2 import Template - - -from .genqr import gen_qr_png_data def _install_chatmaild() -> None: @@ -325,67 +319,6 @@ def get_ini_settings(mail_domain, inipath): return settings -def build_htmlj2_from_markdown(source): - assert source.exists(), source - template_content = open(source).read() - if source.stem == "privacy": - title = "privacy {{ config.mail_domain }}" - elif source.stem == "index": - title = "home {{ config.mail_domain }}" - elif source.stem == "info": - title = "info {{ config.mail_domain }}" - - html = markdown.markdown(template_content) - html = ( - textwrap.dedent( - f"""\ - - - - - {title} - - - - """ - ) - + html - + "\n" - + textwrap.dedent( - """\ - - """ - ) - ) - - target_path = source.with_name(source.stem + ".html.j2") - with open(target_path, "w") as f: - f.write(html) - print(f"wrote {target_path}") - return target_path - - -def build_webpages(www_path, config): - mail_domain = config["mail_domain"] - qr_data = gen_qr_png_data(mail_domain).read() - www_path.joinpath(f"qr-chatmail-invite-{mail_domain}.png").write_bytes(qr_data) - - for path in www_path.iterdir(): - if path.suffix == ".md": - path = build_htmlj2_from_markdown(path) - - if path.suffix == ".j2": - target = path.with_name(path.name[:-3]) - template = Template(path.read_text()) - with target.open("w") as f: - f.write(template.render(config=config)) - - def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> None: """Deploy a chat-mail instance. @@ -393,6 +326,7 @@ def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> N :param mail_server: the DNS name under which your mail server is reachable :param dkim_selector: """ + from .www import build_webpages apt.update(name="apt update", cache_time=24 * 3600) server.group(name="Create vmail group", group="vmail", system=True) @@ -442,8 +376,10 @@ def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> N config = get_ini_settings(mail_domain, chatmail_ini) www_path = pkg_root.joinpath("../../../www").resolve() - build_webpages(www_path, config) - files.rsync(f"{www_path}/", "/var/www/html", flags=["-avz"]) + build_dir = www_path.joinpath("build") + src_dir = www_path.joinpath("src") + build_webpages(src_dir, build_dir, config) + files.rsync(f"{build_dir}/", "/var/www/html", flags=["-avz"]) _install_chatmaild() debug = False diff --git a/deploy-chatmail/src/deploy_chatmail/genqr.py b/deploy-chatmail/src/deploy_chatmail/genqr.py index 8ae7fd24..21419859 100644 --- a/deploy-chatmail/src/deploy_chatmail/genqr.py +++ b/deploy-chatmail/src/deploy_chatmail/genqr.py @@ -49,10 +49,10 @@ def gen_qr(maildomain, url): size = width = 384 qr_padding = 6 text_height = font_size * num_lines - height = size + text_height + qr_padding * 2 + height = size + text_height image = Image.new("RGBA", (width, height), "white") - qr_final_size = width - (qr_padding * 2) + qr_final_size = width if num_lines: draw = ImageDraw.Draw(image) diff --git a/deploy-chatmail/src/deploy_chatmail/www.py b/deploy-chatmail/src/deploy_chatmail/www.py new file mode 100644 index 00000000..d06aa709 --- /dev/null +++ b/deploy-chatmail/src/deploy_chatmail/www.py @@ -0,0 +1,110 @@ +import importlib.resources +import webbrowser +import hashlib +import time +import traceback + +import markdown +from jinja2 import Template +from .genqr import gen_qr_png_data +from deploy_chatmail import get_ini_settings + + +def snapshot_dir_stats(somedir): + d = {} + for path in somedir.iterdir(): + if path.is_file() and path.name[0] != "." and path.suffix != ".swp": + mtime = path.stat().st_mtime + hash = hashlib.md5(path.read_bytes()).hexdigest() + d[path] = (mtime, hash) + return d + + +def prepare_template(source): + assert source.exists(), source + render_vars = {} + render_vars["pagename"] = "home" if source.stem == "index" else source.stem + render_vars["markdown_html"] = markdown.markdown(source.read_text()) + page_layout = source.with_name("page-layout.html").read_text() + return render_vars, page_layout + + +def build_webpages(src_dir, build_dir, config): + try: + _build_webpages(src_dir, build_dir, config) + except Exception: + print(traceback.format_exc()) + + +def _build_webpages(src_dir, build_dir, config): + mail_domain = config["mail_domain"] + assert src_dir.exists(), src_dir + if not build_dir.exists(): + build_dir.mkdir() + + qr_path = build_dir.joinpath(f"qr-chatmail-invite-{mail_domain}.png") + qr_path.write_bytes(gen_qr_png_data(mail_domain).read()) + + for path in src_dir.iterdir(): + if path.suffix == ".md": + render_vars, content = prepare_template(path) + target = build_dir.joinpath(path.stem + ".html") + + # recursive jinja2 rendering + while 1: + new = Template(content).render(config=config, **render_vars) + if new == content: + break + content = new + + with target.open("w") as f: + f.write(content) + elif path.name != "page-layout.html": + target = build_dir.joinpath(path.name) + target.write_bytes(path.read_bytes()) + return build_dir + + +def main(): + chatmail_domain = "example.testrun.org" + path = importlib.resources.files(__package__) + reporoot = path.joinpath("../../../").resolve() + inipath = reporoot.joinpath("chatmail.ini") + config = get_ini_settings(chatmail_domain, inipath) + config["webdev"] = True + www_path = reporoot.joinpath("www") + src_path = www_path.joinpath("src") + stats = None + build_dir = www_path.joinpath("build") + src_dir = www_path.joinpath("src") + index_path = build_dir.joinpath("index.html") + + # start web page generation, open a browser and wait for changes + build_webpages(src_dir, build_dir, config) + webbrowser.open(str(index_path)) + stats = snapshot_dir_stats(src_path) + print(f"\nOpened URL: file://{index_path.resolve()}\n") + print(f"watching {src_path} directory for changes") + + changenum = 0 + for count in range(0, 1000000): + newstats = snapshot_dir_stats(src_path) + if newstats == stats and count % 60 != 0: + count += 1 + time.sleep(1.0) + continue + + for key in newstats: + if stats[key] != newstats[key]: + print(f"*** CHANGED: {key}") + changenum += 1 + + stats = newstats + build_webpages(src_dir, build_dir, config) + print(f"[{changenum}] regenerated web pages at: {index_path}") + print(f"URL: file://{index_path.resolve()}\n\n") + count = 0 + + +if __name__ == "__main__": + main() diff --git a/scripts/init.sh b/scripts/init.sh index 7ba6c13d..3e19cf36 100755 --- a/scripts/init.sh +++ b/scripts/init.sh @@ -3,6 +3,6 @@ set -e python3 -m venv venv pip=venv/bin/pip -$pip install pyinfra pytest build 'setuptools>=68' tox deltachat +$pip install pyinfra pytest build 'setuptools>=68' tox $pip install -e deploy-chatmail $pip install -e chatmaild diff --git a/scripts/webdev.sh b/scripts/webdev.sh new file mode 100755 index 00000000..73e3f7d7 --- /dev/null +++ b/scripts/webdev.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +echo ----------------------------------------- +echo starting local webdev +echo ----------------------------------------- + +venv/bin/python3 -m deploy_chatmail.www + + diff --git a/tests/chatmaild/test_newmail.py b/tests/chatmaild/test_newmail.py index 5b6dad1f..8d13f981 100644 --- a/tests/chatmaild/test_newmail.py +++ b/tests/chatmaild/test_newmail.py @@ -3,6 +3,7 @@ import json import chatmaild from chatmaild.newemail import create_newemail_dict, print_new_account + def test_create_newemail_dict(): ac1 = create_newemail_dict(domain="example.org") assert "@" in ac1["email"] diff --git a/tests/conftest.py b/tests/conftest.py index 00a7853a..536902cd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -278,13 +278,13 @@ class ChatmailTestProcess: @pytest.fixture -def cmfactory(request, gencreds, tmpdir, data, maildomain): +def cmfactory(request, gencreds, tmpdir, maildomain): # cloned from deltachat.testplugin.amfactory pytest.importorskip("deltachat") from deltachat.testplugin import ACFactory testproc = ChatmailTestProcess(request.config, maildomain, gencreds) - am = ACFactory(request=request, tmpdir=tmpdir, testprocess=testproc, data=data) + am = ACFactory(request=request, tmpdir=tmpdir, testprocess=testproc, data=None) # nb. a bit hacky # would probably be better if deltachat's test machinery grows native support @@ -326,6 +326,18 @@ class Remote: break +@pytest.fixture +def lp(request): + class LP: + def sec(self, msg): + print(f"---- {msg} ----") + + def indent(self, msg): + print(f" {msg}") + + return LP() + + @pytest.fixture def maildata(request, gencreds): datadir = conftestdir.joinpath("mail-data") diff --git a/tests/online/test_0_qr.py b/tests/online/test_0_qr.py index 2f4c5327..5c1a5865 100644 --- a/tests/online/test_0_qr.py +++ b/tests/online/test_0_qr.py @@ -1,5 +1,3 @@ - - from deploy_chatmail.genqr import gen_qr_png_data diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 683ed435..bb11c1e2 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -1,19 +1,11 @@ import textwrap +import importlib.resources -from deploy_chatmail import build_htmlj2_from_markdown, get_ini_settings +from deploy_chatmail.www import build_webpages +from deploy_chatmail import get_ini_settings -def test_markdown(tmp_path): - path = tmp_path.joinpath("privacy.md") - path.write_text("# privacy policy") - build_htmlj2_from_markdown(path) - output = path.with_name("privacy.html.j2") - assert output.exists() - print(output.read_text()) - - -def test_get_settings(tmp_path): - inipath = tmp_path.joinpath("chatmail.ini") +def create_ini(inipath): inipath.write_text( textwrap.dedent( """\ @@ -30,10 +22,26 @@ def test_get_settings(tmp_path): """ ) ) + + +def test_build_webpages(tmp_path): + pkgroot = importlib.resources.files("deploy_chatmail") + src_dir = pkgroot.joinpath("../../../www/src").resolve() + assert src_dir.exists(), src_dir + + inipath = tmp_path.joinpath("chatmail.ini") + create_ini(inipath) + config = get_ini_settings("example.org", inipath) + build_dir = tmp_path.joinpath("build") + build_webpages(src_dir, build_dir, config) + + +def test_get_settings(tmp_path): + inipath = tmp_path.joinpath("chatmail.ini") + create_ini(inipath) + d = get_ini_settings("x.testrun.org", inipath) assert d["privacy_postal"] == "address-line1\naddress-line2" assert d["privacy_mail"] == "privacy@example.org" assert d["privacy_pdo"] == "address-line3" assert d["mail_domain"] == "x.testrun.org" - - diff --git a/www/collage-info.png b/www/src/collage-info.png similarity index 100% rename from www/collage-info.png rename to www/src/collage-info.png diff --git a/www/collage-privacy.png b/www/src/collage-privacy.png similarity index 100% rename from www/collage-privacy.png rename to www/src/collage-privacy.png diff --git a/www/collage-top.png b/www/src/collage-top.png similarity index 100% rename from www/collage-top.png rename to www/src/collage-top.png diff --git a/www/index.md b/www/src/index.md similarity index 100% rename from www/index.md rename to www/src/index.md diff --git a/www/info.md b/www/src/info.md similarity index 100% rename from www/info.md rename to www/src/info.md diff --git a/www/src/page-layout.html b/www/src/page-layout.html new file mode 100644 index 00000000..d999da75 --- /dev/null +++ b/www/src/page-layout.html @@ -0,0 +1,20 @@ + + + + + {% if config.webdev %} + + {% endif %} + {{ config.mail_domain }} {{ pagename }} + + + +{{ markdown_html }} + + + diff --git a/www/privacy.md b/www/src/privacy.md similarity index 100% rename from www/privacy.md rename to www/src/privacy.md diff --git a/www/water.css b/www/src/water.css similarity index 100% rename from www/water.css rename to www/src/water.css