be fine with 9 chars for password already

README.md

@@ -1,5 +1,5 @@
 
-<img width="800px" src="www/src/collage-top.png"/>
+<img width="800px" src="www/collage-top.png"/>
 
 # Chatmail instances optimized for Delta Chat apps 
 
@@ -51,28 +51,6 @@ The `deploy.sh` script deploys
 All files are generated by the according markdown `.md` file in the `www` directory.
 
 
-### Refining the web pages 
-
-The `scripts/webdev.sh` script supports live development of the chatmail web presence:
-
-``` 
-    scripts/init.sh  # to locally initialize python virtual environments etc. 
-    scripts/webdev.sh
-```
-
-- uses the `www/src/page-layout.html` file for producing html documents
-  from `www/src/*.md` files. 
-
-- continously builds the web presence reading files from `www/src` directory
-  and generating html files and copying assets to the `www/build` directory. 
-
-- Starts a browser window automatically where you can "refresh" as needed. 
-
-Note that this script is not needed for running `scripts/deploy.sh" 
-which deploys the whole chatmail setup remotely. 
-The code that generates the web pages is identical
-which means that `webdev.sh` gives a pretty good preview. 
-
 ### Ports
 
 Postfix listens on ports 25 (smtp) and 587 (submission) and 465 (submissions).

chatmaild/src/chatmaild/doveauth.service

@@ -2,7 +2,7 @@
 Description=Dict authentication proxy for dovecot
 
 [Service]
-ExecStart={execpath} /run/dovecot/doveauth.socket vmail /home/vmail/passdb.sqlite
+ExecStart=/usr/local/bin/doveauth /run/dovecot/doveauth.socket vmail /home/vmail/passdb.sqlite
 Restart=always
 RestartSec=30
 

chatmaild/src/chatmaild/filtermail.py

@@ -34,14 +34,6 @@ def check_encrypted(message):
     return True
 
 
-def is_passthrough_recipient(recipient):
-    """Check whether a recipient is configured as passthrough."""
-    passthroughlist = ["privacy@testrun.org"]
-    if recipient in passthroughlist:
-        return True
-    return False
-
-
 def check_mdn(message, envelope):
     if len(envelope.rcpt_tos) != 1:
         return False
@@ -126,9 +118,6 @@ def check_DATA(envelope):
         if envelope.mail_from == recipient:
             # Always allow sending emails to self.
             continue
-        if is_passthrough_recipient(recipient):
-            # Always allow recipients marked as passthrough
-            continue
         res = recipient.split("@")
         if len(res) != 2:
             return f"500 Invalid address <{recipient}>"

chatmaild/src/chatmaild/filtermail.service

@@ -2,7 +2,7 @@
 Description=Chatmail Postfix BeforeQeue filter 
 
 [Service]
-ExecStart={execpath} 10080
+ExecStart=/usr/local/bin/filtermail 10080
 Restart=always
 RestartSec=30
 

deploy-chatmail/src/deploy_chatmail/__init__.py

@@ -1,102 +1,81 @@
 """
 Chat Mail pyinfra deploy.
 """
-import sys
 import importlib.resources
-import subprocess
-import shutil
-import io
 import configparser
+import textwrap
 from pathlib import Path
 
 from pyinfra import host
-from pyinfra.operations import apt, files, server, systemd, pip
+from pyinfra.operations import apt, files, server, systemd
 from pyinfra.facts.files import File
 from pyinfra.facts.systemd import SystemdEnabled
 from .acmetool import deploy_acmetool
+import markdown
+from jinja2 import Template
 
 
-def _build_chatmaild(dist_dir) -> None:
-    dist_dir = Path(dist_dir).resolve()
-    if dist_dir.exists():
-        shutil.rmtree(dist_dir)
-    dist_dir.mkdir()
-    subprocess.check_output(
-        [sys.executable, "-m", "build", "-n"]
-        + ["--sdist", "chatmaild", "--outdir", str(dist_dir)]
+from .genqr import gen_qr_png_data
+
+
+def _install_chatmaild() -> None:
+    chatmaild_filename = "chatmaild-0.1.tar.gz"
+    chatmaild_path = importlib.resources.files(__package__).joinpath(
+        f"../../../dist/{chatmaild_filename}"
     )
-    entries = list(dist_dir.iterdir())
-    assert len(entries) == 1
-    return entries[0]
-
-
-def _install_remote_venv_with_chatmaild() -> None:
-    dist_file = _build_chatmaild(dist_dir=Path("chatmaild/dist"))
-    remote_base_dir = "/usr/local/lib/chatmaild"
-    remote_dist_file = f"{remote_base_dir}/dist/{dist_file.name}"
-    remote_venv_dir = f"{remote_base_dir}/venv"
-    root_owned = dict(user="root", group="root", mode="644")
-
-    apt.packages(
-        name="apt install python3-virtualenv",
-        packages=["python3-virtualenv"],
-    )
-
-    files.put(
-        name="Upload chatmaild source package",
-        src=dist_file.open("rb"),
-        dest=remote_dist_file,
-        create_remote_dir=True,
-        **root_owned,
-    )
-
-    pip.virtualenv(
-        name=f"chatmaild virtualenv {remote_venv_dir}",
-        path=remote_venv_dir,
-        always_copy=True,
-    )
-
-    server.shell(
-        name=f"forced pip-install {dist_file.name}",
-        commands=[
-            f"{remote_venv_dir}/bin/pip install --force-reinstall {remote_dist_file}"
-        ],
-    )
-
-    # disable legacy doveauth-dictproxy.service
-    if host.get_fact(SystemdEnabled).get("doveauth-dictproxy.service"):
-        systemd.service(
-            name="Disable legacy doveauth-dictproxy.service",
-            service="doveauth-dictproxy.service",
-            running=False,
-            enabled=False,
-        )
-
-    # install systemd units
-
-    for fn in (
-        "doveauth",
-        "filtermail",
-    ):
-        execpath = f"{remote_venv_dir}/bin/{fn}"
-        source_path = importlib.resources.files("chatmaild").joinpath(f"{fn}.service.f")
-        content = source_path.read_text().format(execpath=execpath).encode()
-
+    remote_path = f"/tmp/{chatmaild_filename}"
+    if Path(str(chatmaild_path)).exists():
         files.put(
-            name=f"Upload {fn}.service",
-            src=io.BytesIO(content),
-            dest=f"/etc/systemd/system/{fn}.service",
-            **root_owned,
+            name="Upload chatmaild source package",
+            src=chatmaild_path.open("rb"),
+            dest=remote_path,
         )
-        systemd.service(
-            name=f"Setup {fn} service",
-            service=f"{fn}.service",
-            running=True,
-            enabled=True,
-            restarted=True,
-            daemon_reload=True,
+
+        apt.packages(
+            name="apt install python3-aiosmtpd python3-pip python3-venv",
+            packages=["python3-aiosmtpd", "python3-pip", "python3-venv"],
         )
 
+        # --no-deps because aiosmtplib is installed with `apt`.
+        server.shell(
+            name="install chatmaild with pip",
+            commands=[f"pip install --break-system-packages {remote_path}"],
+        )
+
+        # disable legacy doveauth-dictproxy.service
+        if host.get_fact(SystemdEnabled).get("doveauth-dictproxy.service"):
+            systemd.service(
+                name="Disable legacy doveauth-dictproxy.service",
+                service="doveauth-dictproxy.service",
+                running=False,
+                enabled=False,
+            )
+
+        # install systemd units
+
+        for fn in (
+            "doveauth",
+            "filtermail",
+        ):
+            files.put(
+                name=f"Upload {fn}.service",
+                src=importlib.resources.files("chatmaild")
+                .joinpath(f"{fn}.service")
+                .open("rb"),
+                dest=f"/etc/systemd/system/{fn}.service",
+                user="root",
+                group="root",
+                mode="644",
+            )
+            systemd.service(
+                name=f"Setup {fn} service",
+                service=f"{fn}.service",
+                running=True,
+                enabled=True,
+                restarted=True,
+                daemon_reload=True,
+            )
+
 
 def _configure_opendkim(domain: str, dkim_selector: str) -> bool:
     """Configures OpenDKIM"""
@@ -346,6 +325,67 @@ def get_ini_settings(mail_domain, inipath):
     return settings
 
 
+def build_htmlj2_from_markdown(source):
+    assert source.exists(), source
+    template_content = open(source).read()
+    if source.stem == "privacy":
+        title = "privacy {{ config.mail_domain }}"
+    elif source.stem == "index":
+        title = "home {{ config.mail_domain }}"
+    elif source.stem == "info":
+        title = "info {{ config.mail_domain }}"
+
+    html = markdown.markdown(template_content)
+    html = (
+        textwrap.dedent(
+            f"""\
+        <!DOCTYPE html>
+        <html>
+        <head>
+            <meta charset="utf-8" />
+            <title>{title}</title>
+            <link rel="stylesheet" href="./water.css">
+        </head>
+        <body>
+    """
+        )
+        + html
+        + "\n"
+        + textwrap.dedent(
+            """\
+        <footer>
+        <a href="index.html">home</a> |
+        <a href="info.html">more info</a> |
+        <a href="privacy.html">privacy</a> |
+        <a href="https://github.com/deltachat/chatmail">-> public development </a>
+        </footer>
+        </body>"""
+        )
+    )
+
+    target_path = source.with_name(source.stem + ".html.j2")
+    with open(target_path, "w") as f:
+        f.write(html)
+    print(f"wrote {target_path}")
+    return target_path
+
+
+def build_webpages(www_path, config):
+    mail_domain = config["mail_domain"]
+    qr_data = gen_qr_png_data(mail_domain).read()
+    www_path.joinpath(f"qr-chatmail-invite-{mail_domain}.png").write_bytes(qr_data)
+
+    for path in www_path.iterdir():
+        if path.suffix == ".md":
+            path = build_htmlj2_from_markdown(path)
+
+        if path.suffix == ".j2":
+            target = path.with_name(path.name[:-3])
+            template = Template(path.read_text())
+            with target.open("w") as f:
+                f.write(template.render(config=config))
+
+
 def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> None:
     """Deploy a chat-mail instance.
 
@@ -353,7 +393,6 @@ def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> N
     :param mail_server: the DNS name under which your mail server is reachable
     :param dkim_selector:
     """
-    from .www import build_webpages
 
     apt.update(name="apt update", cache_time=24 * 3600)
     server.group(name="Create vmail group", group="vmail", system=True)
@@ -403,12 +442,10 @@ def deploy_chatmail(mail_domain: str, mail_server: str, dkim_selector: str) -> N
     config = get_ini_settings(mail_domain, chatmail_ini)
     www_path = pkg_root.joinpath("../../../www").resolve()
 
-    build_dir = www_path.joinpath("build")
-    src_dir = www_path.joinpath("src")
-    build_webpages(src_dir, build_dir, config)
-    files.rsync(f"{build_dir}/", "/var/www/html", flags=["-avz"])
+    build_webpages(www_path, config)
+    files.rsync(f"{www_path}/", "/var/www/html", flags=["-avz"])
 
-    _install_remote_venv_with_chatmaild()
+    _install_chatmaild()
     debug = False
     dovecot_need_restart = _configure_dovecot(mail_server, debug=debug)
     postfix_need_restart = _configure_postfix(mail_domain, debug=debug)

deploy-chatmail/src/deploy_chatmail/genqr.py

@@ -49,7 +49,7 @@ def gen_qr(maildomain, url):
     size = width = 384
     qr_padding = 6
     text_height = font_size * num_lines
-    height = size + text_height
+    height = size + text_height + qr_padding * 2
 
     image = Image.new("RGBA", (width, height), "white")
     qr_final_size = width - (qr_padding * 2)

deploy-chatmail/src/deploy_chatmail/www.py

@@ -1,110 +0,0 @@
-import importlib.resources
-import webbrowser
-import hashlib
-import time
-import traceback
-
-import markdown
-from jinja2 import Template
-from .genqr import gen_qr_png_data
-from deploy_chatmail import get_ini_settings
-
-
-def snapshot_dir_stats(somedir):
-    d = {}
-    for path in somedir.iterdir():
-        if path.is_file() and path.name[0] != "." and path.suffix != ".swp":
-            mtime = path.stat().st_mtime
-            hash = hashlib.md5(path.read_bytes()).hexdigest()
-            d[path] = (mtime, hash)
-    return d
-
-
-def prepare_template(source):
-    assert source.exists(), source
-    render_vars = {}
-    render_vars["pagename"] = "home" if source.stem == "index" else source.stem
-    render_vars["markdown_html"] = markdown.markdown(source.read_text())
-    page_layout = source.with_name("page-layout.html").read_text()
-    return render_vars, page_layout
-
-
-def build_webpages(src_dir, build_dir, config):
-    try:
-        _build_webpages(src_dir, build_dir, config)
-    except Exception:
-        print(traceback.format_exc())
-
-
-def _build_webpages(src_dir, build_dir, config):
-    mail_domain = config["mail_domain"]
-    assert src_dir.exists(), src_dir
-    if not build_dir.exists():
-        build_dir.mkdir()
-
-    qr_path = build_dir.joinpath(f"qr-chatmail-invite-{mail_domain}.png")
-    qr_path.write_bytes(gen_qr_png_data(mail_domain).read())
-
-    for path in src_dir.iterdir():
-        if path.suffix == ".md":
-            render_vars, content = prepare_template(path)
-            target = build_dir.joinpath(path.stem + ".html")
-
-            # recursive jinja2 rendering
-            while 1:
-                new = Template(content).render(config=config, **render_vars)
-                if new == content:
-                    break
-                content = new
-
-            with target.open("w") as f:
-                f.write(content)
-        elif path.name != "page-layout.html":
-            target = build_dir.joinpath(path.name)
-            target.write_bytes(path.read_bytes())
-    return build_dir
-
-
-def main():
-    chatmail_domain = "example.testrun.org"
-    path = importlib.resources.files(__package__)
-    reporoot = path.joinpath("../../../").resolve()
-    inipath = reporoot.joinpath("chatmail.ini")
-    config = get_ini_settings(chatmail_domain, inipath)
-    config["webdev"] = True
-    www_path = reporoot.joinpath("www")
-    src_path = www_path.joinpath("src")
-    stats = None
-    build_dir = www_path.joinpath("build")
-    src_dir = www_path.joinpath("src")
-    index_path = build_dir.joinpath("index.html")
-
-    # start web page generation, open a browser and wait for changes
-    build_webpages(src_dir, build_dir, config)
-    webbrowser.open(str(index_path))
-    stats = snapshot_dir_stats(src_path)
-    print(f"\nOpened URL: file://{index_path.resolve()}\n")
-    print(f"watching {src_path} directory for changes")
-
-    changenum = 0
-    for count in range(0, 1000000):
-        newstats = snapshot_dir_stats(src_path)
-        if newstats == stats and count % 60 != 0:
-            count += 1
-            time.sleep(1.0)
-            continue
-
-        for key in newstats:
-            if stats[key] != newstats[key]:
-                print(f"*** CHANGED: {key}")
-                changenum += 1
-
-        stats = newstats
-        build_webpages(src_dir, build_dir, config)
-        print(f"[{changenum}] regenerated web pages at: {index_path}")
-        print(f"URL: file://{index_path.resolve()}\n\n")
-        count = 0
-
-
-if __name__ == "__main__":
-    main()

scripts/deploy.sh

@@ -4,5 +4,12 @@ echo -----------------------------------------
 echo deploying to $CHATMAIL_DOMAIN 
 echo -----------------------------------------
 
+echo WARNING: in five seconds deploy to $CHATMAIL_DOMAIN starts
+sleep 5
+
+venv/bin/python3 -m build -n --sdist chatmaild --outdir dist
+
 venv/bin/pyinfra --ssh-user root "$CHATMAIL_DOMAIN" \
     deploy-chatmail/src/deploy_chatmail/deploy.py
+
+rm -r dist/

scripts/init.sh

@@ -3,6 +3,6 @@ set -e
 python3 -m venv venv
 pip=venv/bin/pip
 
-$pip install pyinfra pytest build 'setuptools>=68' tox 
+$pip install pyinfra pytest build 'setuptools>=68' tox deltachat
 $pip install -e deploy-chatmail 
 $pip install -e chatmaild 

scripts/webdev.sh

@@ -1,9 +0,0 @@
-#!/usr/bin/env bash
-
-echo -----------------------------------------
-echo starting local webdev 
-echo -----------------------------------------
-
-venv/bin/python3 -m deploy_chatmail.www 
-
-

tests/chatmaild/test_filtermail.py

@@ -1,4 +1,4 @@
-from chatmaild.filtermail import check_encrypted, check_DATA, SendRateLimiter, check_mdn, is_passthrough_recipient
+from chatmaild.filtermail import check_encrypted, check_DATA, SendRateLimiter, check_mdn
 import pytest
 
 
@@ -80,28 +80,3 @@ def test_send_rate_limiter():
         else:
             assert i == SendRateLimiter.MAX_USER_SEND_PER_MINUTE + 1
             break
-
-
-def test_excempt_privacy(maildata, gencreds):
-    from_addr = gencreds()[0]
-    to_addr = "privacy@testrun.org"
-    false_to = "privacy@tstrn.org"
-    false_to2 = "prvcy@testrun.org"
-    assert is_passthrough_recipient(to_addr)
-
-    msg = maildata("plain.eml", from_addr, to_addr)
-
-    class env:
-        mail_from = from_addr
-        rcpt_tos = [to_addr]
-        content = msg.as_bytes()
-
-    # assert that None/no error is returned
-    assert not check_DATA(envelope=env)
-
-    class env2:
-        mail_from = from_addr
-        rcpt_tos = [to_addr, false_to, false_to2]
-        content = msg.as_bytes()
-
-    assert "500" in check_DATA(envelope=env2)

tests/chatmaild/test_newmail.py

@@ -3,7 +3,6 @@ import json
 import chatmaild
 from chatmaild.newemail import create_newemail_dict, print_new_account
 
-
 def test_create_newemail_dict():
     ac1 = create_newemail_dict(domain="example.org")
     assert "@" in ac1["email"]

tests/conftest.py

@@ -278,13 +278,11 @@ class ChatmailTestProcess:
 
 
 @pytest.fixture
-def cmfactory(request, gencreds, tmpdir, maildomain):
+def cmfactory(request, gencreds, tmpdir, data, maildomain):
     # cloned from deltachat.testplugin.amfactory
     pytest.importorskip("deltachat")
     from deltachat.testplugin import ACFactory
 
-    data = request.getfixturevalue("data")
-
     testproc = ChatmailTestProcess(request.config, maildomain, gencreds)
     am = ACFactory(request=request, tmpdir=tmpdir, testprocess=testproc, data=data)
 
@@ -328,18 +326,6 @@ class Remote:
                 break
 
 
-@pytest.fixture
-def lp(request):
-    class LP:
-        def sec(self, msg):
-            print(f"---- {msg} ----")
-
-        def indent(self, msg):
-            print(f"     {msg}")
-
-    return LP()
-
-
 @pytest.fixture
 def maildata(request, gencreds):
     datadir = conftestdir.joinpath("mail-data")

tests/online/test_0_qr.py

@@ -1,3 +1,5 @@
+
+
 from deploy_chatmail.genqr import gen_qr_png_data
 
 

tests/test_helpers.py

@@ -1,11 +1,19 @@
 import textwrap
-import importlib.resources
 
-from deploy_chatmail.www import build_webpages
-from deploy_chatmail import get_ini_settings
+from deploy_chatmail import build_htmlj2_from_markdown, get_ini_settings
 
 
-def create_ini(inipath):
+def test_markdown(tmp_path):
+    path = tmp_path.joinpath("privacy.md")
+    path.write_text("# privacy policy")
+    build_htmlj2_from_markdown(path)
+    output = path.with_name("privacy.html.j2")
+    assert output.exists()
+    print(output.read_text())
+
+
+def test_get_settings(tmp_path):
+    inipath = tmp_path.joinpath("chatmail.ini")
     inipath.write_text(
         textwrap.dedent(
             """\
@@ -22,26 +30,10 @@ def create_ini(inipath):
     """
         )
     )
-
-
-def test_build_webpages(tmp_path):
-    pkgroot = importlib.resources.files("deploy_chatmail")
-    src_dir = pkgroot.joinpath("../../../www/src").resolve()
-    assert src_dir.exists(), src_dir
-
-    inipath = tmp_path.joinpath("chatmail.ini")
-    create_ini(inipath)
-    config = get_ini_settings("example.org", inipath)
-    build_dir = tmp_path.joinpath("build")
-    build_webpages(src_dir, build_dir, config)
-
-
-def test_get_settings(tmp_path):
-    inipath = tmp_path.joinpath("chatmail.ini")
-    create_ini(inipath)
-
     d = get_ini_settings("x.testrun.org", inipath)
     assert d["privacy_postal"] == "address-line1\naddress-line2"
     assert d["privacy_mail"] == "privacy@example.org"
     assert d["privacy_pdo"] == "address-line3"
     assert d["mail_domain"] == "x.testrun.org"
+
+

www/src/page-layout.html

@@ -1,20 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-    <meta charset="utf-8" />
-    {% if config.webdev %}
-    <meta http-equiv="refresh" content="3">
-    {% endif %}
-    <title>{{ config.mail_domain }} {{ pagename }}</title>
-    <link rel="stylesheet" href="./water.css">
-</head>
-<body>
-{{ markdown_html }}
-<footer>
-    <a href="index.html">home</a> |
-    <a href="info.html">more info</a> |
-    <a href="privacy.html">privacy</a> |
-    <a href="https://github.com/deltachat/chatmail">-> public development </a>
-</footer>
-</body>
-</html>