Update pip and setuptools in scripts/initenv.sh

This is to support Debian 11 which ships setuptools
that do not support `-e` without setup.py

.github/workflows/staging.testrun.org-default.zone

@@ -0,0 +1,20 @@
+;; Zone file for staging.testrun.org
+
+$ORIGIN staging.testrun.org.
+$TTL 300
+
+@  IN  SOA     ns.testrun.org. root.nine.testrun.org (
+  2023010101 ; Serial
+  7200       ; Refresh
+  3600       ; Retry
+  1209600    ; Expire
+  3600       ; Negative response caching TTL
+)
+
+;; Nameservers.
+@  IN NS ns.testrun.org.
+
+;; DNS records.
+@       IN      A       37.27.37.98
+mta-sts.staging.testrun.org.    CNAME   staging.testrun.org.
+www.staging.testrun.org.        CNAME   staging.testrun.org.

.github/workflows/test-and-deploy.yaml

@@ -0,0 +1,72 @@
+name: deploy on staging.testrun.org, and run tests
+
+on:
+  push:
+    branches:
+      - main
+      - staging-ci
+
+jobs:
+  deploy:
+    name: deploy on staging.testrun.org, and run tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: prepare SSH
+        run: |
+          mkdir ~/.ssh
+          echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
+          chmod 600 ~/.ssh/id_ed25519
+          ssh-keyscan staging.testrun.org > ~/.ssh/known_hosts
+     #    rsync -avz root@staging.testrun.org:/var/lib/acme . || true
+     #    rsync -avz root@staging.testrun.org:/var/lib/rspamd/dkim . || true
+
+     #- name: rebuild staging.testrun.org to have a clean VPS
+     #  run: |
+     #      curl -X POST \
+     #      -H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
+     #      -H "Content-Type: application/json" \
+     #      -d '{"image":"debian-12"}' \
+     #      "https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_SERVER_ID }}/actions/rebuild"
+
+      - run: scripts/initenv.sh
+
+      - name: append venv/bin to PATH
+        run: echo venv/bin >>$GITHUB_PATH
+
+      - name: run formatting checks 
+        run: cmdeploy fmt -v 
+
+      - name: run deploy-chatmail offline tests 
+        run: pytest --pyargs cmdeploy 
+
+     #- name: upload TLS cert after rebuilding
+     #  run: |
+     #    echo " --- wait until staging.testrun.org VPS is rebuilt --- "
+     #    rm ~/.ssh/known_hosts
+     #    while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u ; do sleep 1 ; done
+     #    ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u
+     #    rsync -avz acme root@staging.testrun.org:/var/lib/ || true
+     #    rsync -avz dkim root@staging.testrun.org:/var/lib/rspamd/ || true
+
+      - run: cmdeploy init staging.testrun.org
+
+      - run: cmdeploy run
+
+      - name: set DNS entries
+        run: |
+          #ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org chown _rspamd:_rspamd -R /var/lib/rspamd/dkim
+          cmdeploy dns --zonefile staging-generated.zone
+          cat staging-generated.zone >> .github/workflows/staging.testrun.org-default.zone
+          cat .github/workflows/staging.testrun.org-default.zone
+          scp -o StrictHostKeyChecking=accept-new .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging.testrun.org.zone
+          ssh root@ns.testrun.org nsd-checkzone staging.testrun.org /etc/nsd/staging.testrun.org.zone
+          ssh root@ns.testrun.org systemctl reload nsd
+
+      - name: cmdeploy test
+        run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow
+
+      - name: cmdeploy dns (try 3 times)
+        run: cmdeploy dns || cmdeploy dns || cmdeploy dns
+

README.md

@@ -157,6 +157,6 @@ While this file is present, account creation will be blocked.
 [acmetool](https://hlandau.github.io/acmetool/) listens on port 80 (http).
 
 Delta Chat apps will, however, discover all ports and configurations
-automatically by reading the [autoconfig XML file](https://web.archive.org/web/20210624004729/https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration) from the chatmail service.
+automatically by reading the [autoconfig XML file](https://www.ietf.org/archive/id/draft-bucksch-autoconfig-00.html) from the chatmail service.
 
 

chatmaild/src/chatmaild/doveauth.py

@@ -91,7 +91,7 @@ def lookup_passdb(db, config: Config, user, cleartext_password):
                VALUES (?, ?, ?)"""
         conn.execute(q, (user, encrypted_password, int(time.time())))
         return dict(
-            home=f"/home/vmail/{user}",
+            home=f"/home/vmail/mail/{config.mail_domain}/{user}",
             uid="vmail",
             gid="vmail",
             password=encrypted_password,

chatmaild/src/chatmaild/tests/mail-data/plain.eml

@@ -7,7 +7,7 @@ Date: Sun, 15 Oct 2023 16:41:44 +0000
 Message-ID: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>
 References: <Mr.3gckbNy5bch.uK3Hd2Ws6-w@c2.testrun.org>
 Chat-Version: 1.0
-Autocrypt: addr=foobar@c2.testrun.org; prefer-encrypt=mutual;
+Autocrypt: addr={from_addr}; prefer-encrypt=mutual;
 	keydata=xjMEZSrw3hYJKwYBBAHaRw8BAQdAiEKNQFU28c6qsx4vo/JHdt73RXdjMOmByf/XsGiJ7m
 	nNFzxmb29iYXJAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUq8N4CGwMECwkIBwYVCAkKCwID
 	FgIBFiEEGil0OvTIa6RngmCLUYNnEa9leJAACgkQUYNnEa9leJCX3gEAhm0MehE5byBBU1avPczr/I
@@ -20,4 +20,4 @@ Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
 
 Hi!
 
-
+

chatmaild/src/chatmaild/tests/test_doveauth.py

@@ -61,7 +61,10 @@ def test_handle_dovecot_request(db, example_config):
     assert res
     assert res[0] == "O" and res.endswith("\n")
     userdata = json.loads(res[1:].strip())
-    assert userdata["home"] == "/home/vmail/some42123@chat.example.org"
+    assert (
+        userdata["home"]
+        == "/home/vmail/mail/chat.example.org/some42123@chat.example.org"
+    )
     assert userdata["uid"] == userdata["gid"] == "vmail"
     assert userdata["password"].startswith("{SHA512-CRYPT}")
 

cmdeploy/src/cmdeploy/__init__.py

@@ -126,71 +126,6 @@ def _install_remote_venv_with_chatmaild(config) -> None:
         )
 
 
-def _configure_opendkim(domain: str, dkim_selector: str = "dkim") -> bool:
-    """Configures OpenDKIM"""
-    need_restart = False
-
-    main_config = files.template(
-        src=importlib.resources.files(__package__).joinpath("opendkim/opendkim.conf"),
-        dest="/etc/opendkim.conf",
-        user="root",
-        group="root",
-        mode="644",
-        config={"domain_name": domain, "opendkim_selector": dkim_selector},
-    )
-    need_restart |= main_config.changed
-
-    files.directory(
-        name="Add opendkim directory to /etc",
-        path="/etc/opendkim",
-        user="opendkim",
-        group="opendkim",
-        mode="750",
-        present=True,
-    )
-
-    keytable = files.template(
-        src=importlib.resources.files(__package__).joinpath("opendkim/KeyTable"),
-        dest="/etc/dkimkeys/KeyTable",
-        user="opendkim",
-        group="opendkim",
-        mode="644",
-        config={"domain_name": domain, "opendkim_selector": dkim_selector},
-    )
-    need_restart |= keytable.changed
-
-    signing_table = files.template(
-        src=importlib.resources.files(__package__).joinpath("opendkim/SigningTable"),
-        dest="/etc/dkimkeys/SigningTable",
-        user="opendkim",
-        group="opendkim",
-        mode="644",
-        config={"domain_name": domain, "opendkim_selector": dkim_selector},
-    )
-    need_restart |= signing_table.changed
-
-    files.directory(
-        name="Add opendkim socket directory to /var/spool/postfix",
-        path="/var/spool/postfix/opendkim",
-        user="opendkim",
-        group="opendkim",
-        mode="750",
-        present=True,
-    )
-
-    if not host.get_fact(File, f"/etc/dkimkeys/{dkim_selector}.private"):
-        server.shell(
-            name="Generate OpenDKIM domain keys",
-            commands=[
-                f"opendkim-genkey -D /etc/dkimkeys -d {domain} -s {dkim_selector}"
-            ],
-            _sudo=True,
-            _sudo_user="opendkim",
-        )
-
-    return need_restart
-
-
 def _install_mta_sts_daemon() -> bool:
     need_restart = False
 
@@ -254,6 +189,17 @@ def _configure_postfix(config: Config, debug: bool = False) -> bool:
     )
     need_restart |= master_config.changed
 
+    header_cleanup = files.put(
+        src=importlib.resources.files(__package__).joinpath(
+            "postfix/submission_header_cleanup"
+        ),
+        dest="/etc/postfix/submission_header_cleanup",
+        user="root",
+        group="root",
+        mode="644",
+    )
+    need_restart |= header_cleanup.changed
+
     return need_restart
 
 
@@ -359,6 +305,107 @@ def _configure_nginx(domain: str, debug: bool = False) -> bool:
     return need_restart
 
 
+def remove_opendkim() -> None:
+    """Remove OpenDKIM, deprecated"""
+    files.file(
+        name="Remove legacy opendkim.conf",
+        path="/etc/opendkim.conf",
+        present=False,
+    )
+
+    files.directory(
+        name="Remove legacy opendkim socket directory from /var/spool/postfix",
+        path="/var/spool/postfix/opendkim",
+        present=False,
+    )
+
+    apt.packages(name="Remove openDKIM", packages="opendkim", present=False)
+
+
+def _configure_rspamd(dkim_selector: str, mail_domain: str) -> bool:
+    """Configures rspamd for Rate Limiting."""
+    need_restart = False
+
+    apt.packages(
+        name="apt install rspamd",
+        packages="rspamd",
+    )
+
+    for module in ["phishing", "rbl", "hfilter", "ratelimit"]:
+        disabled_module_conf = files.put(
+            name=f"disable {module} rspamd plugin",
+            src=importlib.resources.files(__package__).joinpath("rspamd/disabled.conf"),
+            dest=f"/etc/rspamd/local.d/{module}.conf",
+            user="root",
+            group="root",
+            mode="644",
+        )
+        need_restart |= disabled_module_conf.changed
+
+    options_inc = files.put(
+        name="disable fuzzy checks",
+        src=importlib.resources.files(__package__).joinpath("rspamd/options.inc"),
+        dest="/etc/rspamd/local.d/options.inc",
+        user="root",
+        group="root",
+        mode="644",
+    )
+    need_restart |= options_inc.changed
+
+    # https://rspamd.com/doc/modules/force_actions.html
+    force_actions_conf = files.put(
+        name="Set up rules to reject on DKIM, SPF and DMARC fails",
+        src=importlib.resources.files(__package__).joinpath(
+            "rspamd/force_actions.conf"
+        ),
+        dest="/etc/rspamd/local.d/force_actions.conf",
+        user="root",
+        group="root",
+        mode="644",
+    )
+    need_restart |= force_actions_conf.changed
+
+    dkim_directory = "/var/lib/rspamd/dkim/"
+    dkim_key_path = f"{dkim_directory}{mail_domain}.{dkim_selector}.key"
+    dkim_dns_file = f"{dkim_directory}{mail_domain}.{dkim_selector}.zone"
+
+    dkim_config = files.template(
+        src=importlib.resources.files(__package__).joinpath(
+            "rspamd/dkim_signing.conf.j2"
+        ),
+        dest="/etc/rspamd/local.d/dkim_signing.conf",
+        user="root",
+        group="root",
+        mode="644",
+        config={
+            "dkim_selector": str(dkim_selector),
+            "mail_domain": mail_domain,
+            "dkim_key_path": dkim_key_path,
+        },
+    )
+    need_restart |= dkim_config.changed
+
+    files.directory(
+        name="ensure DKIM key directory exists",
+        path=dkim_directory,
+        present=True,
+        user="_rspamd",
+        group="_rspamd",
+    )
+
+    if not host.get_fact(File, dkim_key_path):
+        server.shell(
+            name="Generate DKIM domain keys with rspamd",
+            commands=[
+                f"rspamadm dkim_keygen -b 2048 -s {dkim_selector} -d {mail_domain} -k {dkim_key_path} > {dkim_dns_file}"
+            ],
+            _sudo=True,
+            _sudo_user="_rspamd",
+        )
+
+    return need_restart
+
+
 def check_config(config):
     mail_domain = config.mail_domain
     if mail_domain != "testrun.org" and not mail_domain.endswith(".testrun.org"):
@@ -386,14 +433,6 @@ def deploy_chatmail(config_path: Path) -> None:
     server.group(name="Create vmail group", group="vmail", system=True)
     server.user(name="Create vmail user", user="vmail", group="vmail", system=True)
 
-    server.group(name="Create opendkim group", group="opendkim", system=True)
-    server.user(
-        name="Add postfix user to opendkim group for socket access",
-        user="postfix",
-        groups=["opendkim"],
-        system=True,
-    )
-
     # Run local DNS resolver `unbound`.
     # `resolvconf` takes care of setting up /etc/resolv.conf
     # to use 127.0.0.1 as the resolver.
@@ -403,7 +442,10 @@ def deploy_chatmail(config_path: Path) -> None:
     )
     server.shell(
         name="Generate root keys for validating DNSSEC",
-        commands=["unbound-anchor -a /var/lib/unbound/root.key || true"],
+        commands=[
+            "unbound-anchor -a /var/lib/unbound/root.key || true",
+            "systemctl reset-failed unbound.service",
+        ],
     )
     systemd.service(
         name="Start and enable unbound",
@@ -413,7 +455,10 @@ def deploy_chatmail(config_path: Path) -> None:
     )
 
     # Deploy acmetool to have TLS certificates.
-    deploy_acmetool(nginx_hook=True, domains=[mail_domain, f"mta-sts.{mail_domain}"])
+    deploy_acmetool(
+        nginx_hook=True,
+        domains=[mail_domain, f"mta-sts.{mail_domain}", f"www.{mail_domain}"],
+    )
 
     apt.packages(
         name="Install Postfix",
@@ -425,14 +470,6 @@ def deploy_chatmail(config_path: Path) -> None:
         packages=["dovecot-imapd", "dovecot-lmtpd"],
     )
 
-    apt.packages(
-        name="Install OpenDKIM",
-        packages=[
-            "opendkim",
-            "opendkim-tools",
-        ],
-    )
-
     apt.packages(
         name="Install nginx",
         packages=["nginx"],
@@ -454,16 +491,18 @@ def deploy_chatmail(config_path: Path) -> None:
     debug = False
     dovecot_need_restart = _configure_dovecot(config, debug=debug)
     postfix_need_restart = _configure_postfix(config, debug=debug)
-    opendkim_need_restart = _configure_opendkim(mail_domain)
     mta_sts_need_restart = _install_mta_sts_daemon()
     nginx_need_restart = _configure_nginx(mail_domain)
 
+    remove_opendkim()
+    rspamd_need_restart = _configure_rspamd("dkim", mail_domain)
+
     systemd.service(
-        name="Start and enable OpenDKIM",
-        service="opendkim.service",
+        name="Start and enable rspamd",
+        service="rspamd.service",
         running=True,
         enabled=True,
-        restarted=opendkim_need_restart,
+        restarted=rspamd_need_restart,
     )
 
     systemd.service(

cmdeploy/src/cmdeploy/chatmail.zone.f

@@ -7,8 +7,9 @@ _imap._tcp.{chatmail_domain}.        SRV 0 1 143 {chatmail_domain}.
 _imaps._tcp.{chatmail_domain}.       SRV 0 1 993 {chatmail_domain}.
 {chatmail_domain}.                   CAA 128 issue "letsencrypt.org;accounturi={acme_account_url}"
 {chatmail_domain}.                   TXT "v=spf1 a:{chatmail_domain} -all"
-_dmarc.{chatmail_domain}.            TXT "v=DMARC1;p=reject;rua=mailto:{email};ruf=mailto:{email};fo=1;adkim=r;aspf=r"
+_dmarc.{chatmail_domain}.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
 _mta-sts.{chatmail_domain}.          TXT "v=STSv1; id={sts_id}"
 mta-sts.{chatmail_domain}.           CNAME {chatmail_domain}.
+www.{chatmail_domain}.               CNAME {chatmail_domain}.
 _smtp._tls.{chatmail_domain}.        TXT "v=TLSRPTv1;rua=mailto:{email}"
 {dkim_entry}

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -82,7 +82,8 @@ def dns_cmd_options(parser):
 
 def dns_cmd(args, out):
     """Generate dns zone file."""
-    show_dns(args, out)
+    exit_code = show_dns(args, out)
+    exit(exit_code)
 
 
 def status_cmd(args, out):

cmdeploy/src/cmdeploy/dns.py

@@ -4,7 +4,6 @@ import requests
 import importlib
 import subprocess
 import datetime
-from ipaddress import ip_address
 
 
 class DNS:
@@ -48,7 +47,8 @@ class DNS:
         return result == f"{mail_domain}."
 
 
-def show_dns(args, out):
+def show_dns(args, out) -> int:
+    """Check existing DNS records, optionally write them to zone file, return exit code 0 or 1."""
     template = importlib.resources.files(__package__).joinpath("chatmail.zone.f")
     mail_domain = args.config.mail_domain
     ssh = f"ssh root@{mail_domain}"
@@ -61,6 +61,9 @@ def show_dns(args, out):
                 continue
             line = line.replace("\t", " ")
             lines.append(line)
+        lines[0] = f"dkim._domainkey.{mail_domain}. IN TXT " + lines[0].strip(
+            "dkim._domainkey IN TXT "
+        )
         return "\n".join(lines)
 
     print("Checking your DKIM keys and DNS entries...")
@@ -68,8 +71,10 @@ def show_dns(args, out):
         acme_account_url = out.shell_output(f"{ssh} -- acmetool account-url")
     except subprocess.CalledProcessError:
         print("Please run `cmdeploy run` first.")
-        return
-    dkim_entry = read_dkim_entries(out.shell_output(f"{ssh} -- opendkim-genzone -F"))
+        return 1
+    dkim_entry = read_dkim_entries(
+        out.shell_output(f"{ssh} -- cat /var/lib/rspamd/dkim/{mail_domain}.dkim.zone")
+    )
 
     ipv6 = dns.get_ipv6()
     reverse_ipv6 = dns.check_ptr_record(ipv6, mail_domain)
@@ -95,7 +100,7 @@ def show_dns(args, out):
             with open(args.zonefile, "w+") as zf:
                 zf.write(zonefile)
                 print(f"DNS records successfully written to: {args.zonefile}")
-            return
+            return 0
         except TypeError:
             pass
         started_dkim_parsing = False
@@ -143,12 +148,13 @@ def show_dns(args, out):
         domain, data = "\n".join(dkim_lines).split(" IN TXT ")
         current = dns.get("TXT", domain.strip()[:-1])
         if current:
-            current = "( %s )" % (current.replace('" "', '"\n "'))
-            if current.replace(";", "\\;") != data:
+            current = "( %s" % (current.replace('" "', '"\n "'))
+            if current != data:
                 to_print.append(dkim_entry)
         else:
             to_print.append(dkim_entry)
 
+    exit_code = 0
     if to_print:
         to_print.insert(
             0, "You should configure the following DNS entries at your provider:\n"
@@ -157,6 +163,7 @@ def show_dns(args, out):
             "\nIf you already configured the DNS entries, wait a bit until the DNS entries propagate to the Internet."
         )
         print("\n".join(to_print))
+        exit_code = 1
     else:
         out.green("Great! All your DNS entries are correct.")
 
@@ -176,6 +183,8 @@ def show_dns(args, out):
         print(
             "You can do so at your hosting provider (maybe this isn't your DNS provider)."
         )
+        exit_code = 1
+    return exit_code
 
 
 def check_necessary_dns(out, mail_domain):
@@ -184,14 +193,14 @@ def check_necessary_dns(out, mail_domain):
     ipv4 = dns.get("A", mail_domain)
     ipv6 = dns.get("AAAA", mail_domain)
     mta_entry = dns.get("CNAME", "mta-sts." + mail_domain)
-    mta_ip = dns.get("A", mta_entry)
-    if not mta_ip:
-        mta_ip = dns.get("AAAA", mta_entry)
+    www_entry = dns.get("CNAME", "www." + mail_domain)
     to_print = []
     if not (ipv4 or ipv6):
         to_print.append(f"\t{mail_domain}.\t\t\tA<your server's IPv4 address>")
-    if not mta_ip or not (mta_ip == ipv4 or mta_ip == ipv6):
+    if mta_entry != mail_domain + ".":
         to_print.append(f"\tmta-sts.{mail_domain}.\tCNAME\t{mail_domain}.")
+    if www_entry != mail_domain + ".":
+        to_print.append(f"\twww.{mail_domain}.\tCNAME\t{mail_domain}.")
     if to_print:
         to_print.insert(
             0,

cmdeploy/src/cmdeploy/genqr.py

@@ -6,7 +6,7 @@ import io
 
 
 def gen_qr_png_data(maildomain):
-    url = f"DCACCOUNT:https://{maildomain}/cgi-bin/newemail.py"
+    url = f"DCACCOUNT:https://{maildomain}/new"
     image = gen_qr(maildomain, url)
     temp = io.BytesIO()
     image.save(temp, format="png")

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -41,11 +41,33 @@ http {
 			try_files $uri $uri/ =404;
 		}
 
-                location /metrics {
-                        default_type text/plain;
-                }
+		location /metrics {
+			default_type text/plain;
+		}
 
-        # add cgi-bin support
-        include /usr/share/doc/fcgiwrap/examples/nginx.conf;
+		location /new {
+			if ($request_method = GET) {
+				# Redirect to Delta Chat,
+				# which will in turn do a POST request.
+				return 301 dcaccount:https://{{ config.domain_name }}/new;
+			}
+
+			fastcgi_pass unix:/run/fcgiwrap.socket;
+			include /etc/nginx/fastcgi_params;
+			fastcgi_param SCRIPT_FILENAME /usr/lib/cgi-bin/newemail.py;
+		}
+
+		# Old URL for compatibility with e.g. printed QR codes.
+		location /cgi-bin/newemail.py {
+			return 301 /new;
+		}
+	}
+
+	# Redirect www. to non-www
+	server {
+		listen 443 ssl;
+		listen [::]:443 ssl;
+		server_name www.{{ config.domain_name }};
+		return 301 $scheme://{{ config.domain_name }}$request_uri;
 	}
 }

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -46,5 +46,7 @@ inet_protocols = all
 virtual_transport = lmtp:unix:private/dovecot-lmtp
 virtual_mailbox_domains = {{ config.mail_domain }}
 
-smtpd_milters = unix:opendkim/opendkim.sock
+smtpd_milters = inet:127.0.0.1:11332
 non_smtpd_milters = $smtpd_milters
+
+header_checks = regexp:/etc/postfix/submission_header_cleanup

cmdeploy/src/cmdeploy/postfix/submission_header_cleanup

@@ -0,0 +1,4 @@
+/^Received:/            IGNORE
+/^X-Originating-IP:/    IGNORE
+/^X-Mailer:/            IGNORE
+/^User-Agent:/          IGNORE

cmdeploy/src/cmdeploy/rspamd/disabled.conf

@@ -0,0 +1 @@
+enabled = false;

cmdeploy/src/cmdeploy/rspamd/dkim_signing.conf.j2

@@ -0,0 +1,10 @@
+selector = {{ config.dkim_selector }}
+use_esld = false  # don't cut c1.testrun.org down to testrun.org
+domain = {
+    {{ config.mail_domain }} {
+        selectors [
+            selector = {{ config.dkim_selector }}
+            path = {{ config.dkim_key_path }}
+        ]
+    }
+}

cmdeploy/src/cmdeploy/rspamd/force_actions.conf

@@ -0,0 +1,60 @@
+rules {
+    ## Reject on missing or invalid DKIM signatures.
+    ##
+    ## We require DKIM signature on incoming mails regardless of DMARC policy.
+
+    # R_DKIM_REJECT: DKIM reject inserted by `dkim` module.
+    REJECT_INVALID_DKIM {
+        action = "reject";
+        expression = "R_DKIM_REJECT";
+        message = "Rejected due to invalid DKIM signature";
+    }
+
+    # R_DKIM_PERMFAIL: permanent failure inserted by `dkim` module e.g. no DKIM DNS record found.
+    REJECT_PERMFAIL_DKIM {
+        action = "reject";
+        expression = "R_DKIM_PERMFAIL";
+        message = "Rejected due to missing DKIM DNS entry";
+    }
+
+    # No DKIM signature (R_DKIM_NA symbol inserted by `dkim` module).
+    REJECT_MISSING_DKIM {
+        action = "reject";
+        expression = "R_DKIM_NA";
+        message = "Rejected due to missing DKIM signature";
+    }
+
+
+    ## Reject on SPF failure.
+
+    # - SPF failure (R_SPF_FAIL)
+    # - SPF permanent failure, e.g. failed to resolve DNS record referenced from SPF (R_SPF_PERMFAIL)
+    REJECT_SPF {
+        action = "reject";
+        expression = "R_SPF_FAIL | R_SPF_PERMFAIL";
+        message = "Rejected due to failed SPF check";
+    }
+
+    # Reject on DMARC policy check failure.
+    REJECT_DMARC {
+        action = "reject";
+        expression = "DMARC_POLICY_REJECT";
+        message = "Rejected due to DMARC policy";
+    }
+
+
+    # Do not reject if:
+    # - R_DKIM_TEMPFAIL, it is a DNS resolution failure
+    #   and we do not want to lose messages because of faulty network.
+    #
+    # - R_SPF_SOFTFAIL
+    # - R_SPF_NEUTRAL
+    # - R_SPF_DNSFAIL
+    # - R_SPF_NA
+    #
+    # - DMARC_DNSFAIL
+    # - DMARC_NA
+    # - DMARC_POLICY_SOFTFAIL
+    # - DMARC_POLICY_QUARANTINE
+    # - DMARC_BAD_POLICY
+}

cmdeploy/src/cmdeploy/rspamd/options.inc

@@ -0,0 +1 @@
+filters = "dkim";

cmdeploy/src/cmdeploy/tests/online/test_0_qr.py

@@ -9,7 +9,7 @@ def test_gen_qr_png_data(maildomain):
 
 
 def test_fastcgi_working(maildomain, chatmail_config):
-    url = f"https://{maildomain}/cgi-bin/newemail.py"
+    url = f"https://{maildomain}/new"
     print(url)
     res = requests.post(url)
     assert maildomain in res.json().get("email")
@@ -18,7 +18,7 @@ def test_fastcgi_working(maildomain, chatmail_config):
 
 def test_newemail_configure(maildomain, rpc):
     """Test configuring accounts by scanning a QR code works."""
-    url = f"DCACCOUNT:https://{maildomain}/cgi-bin/newemail.py"
+    url = f"DCACCOUNT:https://{maildomain}/new"
     for i in range(3):
         account_id = rpc.add_account()
         rpc.set_config_from_qr(account_id, url)

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -42,6 +42,16 @@ def test_reject_forged_from(cmsetup, maildata, gencreds, lp, forgeaddr):
     assert "500" in str(e.value)
 
 
+@pytest.mark.parametrize("from_addr", ["fake@example.org", "fake@testrun.org"])
+def test_reject_missing_dkim(cmsetup, maildata, from_addr):
+    """Test that emails with missing or wrong DMARC, DKIM, and SPF entries are rejected."""
+    recipient = cmsetup.gen_users(1)[0]
+    msg = maildata("plain.eml", from_addr=from_addr, to_addr=recipient.addr).as_string()
+    with smtplib.SMTP(cmsetup.maildomain, 25) as s:
+        with pytest.raises(smtplib.SMTPDataError, match="missing DKIM signature"):
+            s.sendmail(from_addr=from_addr, to_addrs=recipient.addr, msg=msg)
+
+
 @pytest.mark.slow
 def test_exceed_rate_limit(cmsetup, gencreds, maildata, chatmail_config):
     """Test that the per-account send-mail limit is exceeded."""

cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py

@@ -1,7 +1,10 @@
 import time
 import re
 import random
+
 import pytest
+import requests
+import ipaddress
 
 
 class TestEndToEndDeltaChat:
@@ -119,3 +122,17 @@ class TestEndToEndDeltaChat:
             for msg in msgs:
                 assert "error" not in m.get_message_info()
             time.sleep(1)
+
+
+def test_hide_senders_ip_address(cmfactory):
+    public_ip = requests.get("http://icanhazip.com").content.decode().strip()
+    assert ipaddress.ip_address(public_ip)
+
+    user1, user2 = cmfactory.get_online_accounts(2)
+    chat = cmfactory.get_accepted_chat(user1, user2)
+
+    chat.send_text("testing submission header cleanup")
+    user2.wait_next_incoming_message()
+    user2.direct_imap.select_folder("Inbox")
+    msg = user2.direct_imap.get_all_messages()[0]
+    assert public_ip not in msg.obj.as_string()

scripts/initenv.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 set -e
-python3 -m venv venv
+python3 -m venv --upgrade-deps venv
 
 venv/bin/pip install -e chatmaild 
 venv/bin/pip install -e cmdeploy

www/src/index.md

@@ -7,7 +7,7 @@ Welcome to instant, interoperable and [privacy-preserving](privacy.html) messagi
 
 👉 **Tap** or scan this QR code to get a random `@{{config.mail_domain}}` e-mail address
 
-<a href="DCACCOUNT:https://{{ config.mail_domain }}/cgi-bin/newemail.py">
+<a href="DCACCOUNT:https://{{ config.mail_domain }}/new">
     <img width=300 style="float: none;" src="qr-chatmail-invite-{{config.mail_domain}}.png" /></a>
 
 🐣 **Choose** your Avatar and Name

www/src/info.md

@@ -3,7 +3,7 @@
 
 ## More information 
 
-`nine.testrun.org` provides a low-maintenance, resource efficient and 
+{{ config.mail_domain }} provides a low-maintenance, resource efficient and 
 interoperable e-mail service for everyone. What's behind a `chatmail` is 
 effectively a normal e-mail address just like any other but optimized 
 for the usage in chats, especially DeltaChat.