Nicer /new URL for new accounts and redirect GET requests

If user types in https://nine.testrun.org/new manually
in the browser, at least Firefox and Brave suggest
to open the app after following the redirect.

.github/workflows/staging.testrun.org-default.zone

@@ -1,20 +0,0 @@
-;; Zone file for staging.testrun.org
-
-$ORIGIN staging.testrun.org.
-$TTL 300
-
-@  IN  SOA     ns.testrun.org. root.nine.testrun.org (
-  2023010101 ; Serial
-  7200       ; Refresh
-  3600       ; Retry
-  1209600    ; Expire
-  3600       ; Negative response caching TTL
-)
-
-;; Nameservers.
-@  IN NS ns.testrun.org.
-
-;; DNS records.
-@       IN      A       37.27.37.98
-mta-sts.staging.testrun.org.    CNAME   staging.testrun.org.
-www.staging.testrun.org.        CNAME   staging.testrun.org.

.github/workflows/test-and-deploy.yaml

@@ -1,73 +0,0 @@
-name: deploy on staging.testrun.org, and run tests
-
-on:
-  push:
-    branches:
-      - main
-      - staging-ci
-      - generate-dkim-selector
-
-jobs:
-  deploy:
-    name: deploy on staging.testrun.org, and run tests
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: prepare SSH
-        run: |
-          mkdir ~/.ssh
-          echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
-          chmod 600 ~/.ssh/id_ed25519
-          ssh-keyscan staging.testrun.org > ~/.ssh/known_hosts
-     #    rsync -avz root@staging.testrun.org:/var/lib/acme . || true
-     #    rsync -avz root@staging.testrun.org:/var/lib/rspamd/dkim . || true
-
-     #- name: rebuild staging.testrun.org to have a clean VPS
-     #  run: |
-     #      curl -X POST \
-     #      -H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
-     #      -H "Content-Type: application/json" \
-     #      -d '{"image":"debian-12"}' \
-     #      "https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_SERVER_ID }}/actions/rebuild"
-
-      - run: scripts/initenv.sh
-
-      - name: append venv/bin to PATH
-        run: echo venv/bin >>$GITHUB_PATH
-
-      - name: run formatting checks 
-        run: cmdeploy fmt -v 
-
-      - name: run deploy-chatmail offline tests 
-        run: pytest --pyargs cmdeploy 
-
-     #- name: upload TLS cert after rebuilding
-     #  run: |
-     #    echo " --- wait until staging.testrun.org VPS is rebuilt --- "
-     #    rm ~/.ssh/known_hosts
-     #    while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u ; do sleep 1 ; done
-     #    ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u
-     #    rsync -avz acme root@staging.testrun.org:/var/lib/ || true
-     #    rsync -avz dkim root@staging.testrun.org:/var/lib/rspamd/ || true
-
-      - run: cmdeploy init staging.testrun.org
-
-      - run: cmdeploy run
-
-      - name: set DNS entries
-        run: |
-          #ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org chown _rspamd:_rspamd -R /var/lib/rspamd/dkim
-          cmdeploy dns --zonefile staging-generated.zone
-          cat staging-generated.zone >> .github/workflows/staging.testrun.org-default.zone
-          cat .github/workflows/staging.testrun.org-default.zone
-          scp -o StrictHostKeyChecking=accept-new .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging.testrun.org.zone
-          ssh root@ns.testrun.org nsd-checkzone staging.testrun.org /etc/nsd/staging.testrun.org.zone
-          ssh root@ns.testrun.org systemctl reload nsd
-
-      - name: cmdeploy test
-        run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow
-
-      - name: cmdeploy dns (try 3 times)
-        run: cmdeploy dns || cmdeploy dns || cmdeploy dns
-

chatmaild/src/chatmaild/config.py

@@ -1,5 +1,4 @@
 import iniconfig
-from datetime import datetime
 
 
 def read_config(inipath):
@@ -25,7 +24,6 @@ class Config:
         self.privacy_mail = params.get("privacy_mail")
         self.privacy_pdo = params.get("privacy_pdo")
         self.privacy_supervisor = params.get("privacy_supervisor")
-        self.dkim_selector = params.get("dkim_selector")
 
     def _getbytefile(self):
         return open(self._inipath, "rb")
@@ -35,16 +33,8 @@ def write_initial_config(inipath, mail_domain):
     from importlib.resources import files
 
     inidir = files(__package__).joinpath("ini")
-    selector = "dkim"
-    if mail_domain == "staging.testrun.org":
-        selector = datetime.now().strftime("%Y%m%d%H%M")
     content = (
-        inidir.joinpath("chatmail.ini.f")
-        .read_text()
-        .format(
-            mail_domain=mail_domain,
-            dkim_selector=selector,
-        )
+        inidir.joinpath("chatmail.ini.f").read_text().format(mail_domain=mail_domain)
     )
     if mail_domain.endswith(".testrun.org"):
         override_inipath = inidir.joinpath("override-testrun.ini")

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -33,9 +33,7 @@ password_min_length = 9
 passthrough_senders =
 
 # list of e-mail recipients for which to accept outbound un-encrypted mails
-passthrough_recipients = xstore@testrun.org groupsbot@hispanilandia.net
-
-dkim_selector = {dkim_selector}
+passthrough_recipients =
 
 #
 # Deployment Details

chatmaild/src/chatmaild/ini/override-testrun.ini

@@ -1,7 +1,7 @@
 
 [privacy]
 
-passthrough_recipients = privacy@testrun.org xstore@testrun.org groupsbot@hispanilandia.net
+passthrough_recipients = privacy@testrun.org
 
 privacy_postal =
     Merlinux GmbH, Represented by the managing director H. Krekel,

chatmaild/src/chatmaild/tests/test_config.py

@@ -28,5 +28,5 @@ def test_read_config_testrun(make_config):
     assert config.username_min_length == 9
     assert config.username_max_length == 9
     assert config.password_min_length == 9
-    assert "privacy@testrun.org" in config.passthrough_recipients
+    assert config.passthrough_recipients == ["privacy@testrun.org"]
     assert config.passthrough_senders == []

cmdeploy/src/cmdeploy/__init__.py

@@ -442,10 +442,7 @@ def deploy_chatmail(config_path: Path) -> None:
     )
     server.shell(
         name="Generate root keys for validating DNSSEC",
-        commands=[
-            "unbound-anchor -a /var/lib/unbound/root.key || true",
-            "systemctl reset-failed unbound.service",
-        ],
+        commands=["unbound-anchor -a /var/lib/unbound/root.key || true"],
     )
     systemd.service(
         name="Start and enable unbound",
@@ -495,7 +492,7 @@ def deploy_chatmail(config_path: Path) -> None:
     nginx_need_restart = _configure_nginx(mail_domain)
 
     remove_opendkim()
-    rspamd_need_restart = _configure_rspamd(config.dkim_selector, mail_domain)
+    rspamd_need_restart = _configure_rspamd("dkim", mail_domain)
 
     systemd.service(
         name="Start and enable rspamd",

cmdeploy/src/cmdeploy/chatmail.zone.f

@@ -7,7 +7,7 @@ _imap._tcp.{chatmail_domain}.        SRV 0 1 143 {chatmail_domain}.
 _imaps._tcp.{chatmail_domain}.       SRV 0 1 993 {chatmail_domain}.
 {chatmail_domain}.                   CAA 128 issue "letsencrypt.org;accounturi={acme_account_url}"
 {chatmail_domain}.                   TXT "v=spf1 a:{chatmail_domain} -all"
-_dmarc.{chatmail_domain}.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
+_dmarc.{chatmail_domain}.            TXT "v=DMARC1;p=reject;rua=mailto:{email};ruf=mailto:{email};fo=1;adkim=s;aspf=s"
 _mta-sts.{chatmail_domain}.          TXT "v=STSv1; id={sts_id}"
 mta-sts.{chatmail_domain}.           CNAME {chatmail_domain}.
 www.{chatmail_domain}.               CNAME {chatmail_domain}.

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -82,8 +82,7 @@ def dns_cmd_options(parser):
 
 def dns_cmd(args, out):
     """Generate dns zone file."""
-    exit_code = show_dns(args, out)
-    exit(exit_code)
+    show_dns(args, out)
 
 
 def status_cmd(args, out):

cmdeploy/src/cmdeploy/dns.py

@@ -47,11 +47,9 @@ class DNS:
         return result == f"{mail_domain}."
 
 
-def show_dns(args, out) -> int:
-    """Check existing DNS records, optionally write them to zone file, return exit code 0 or 1."""
+def show_dns(args, out):
     template = importlib.resources.files(__package__).joinpath("chatmail.zone.f")
     mail_domain = args.config.mail_domain
-    selector = args.config.dkim_selector
     ssh = f"ssh root@{mail_domain}"
     dns = DNS(out, mail_domain)
 
@@ -62,8 +60,8 @@ def show_dns(args, out) -> int:
                 continue
             line = line.replace("\t", " ")
             lines.append(line)
-        lines[0] = f"{selector}._domainkey.{mail_domain}. IN TXT " + lines[0].strip(
-            f"{selector}._domainkey IN TXT "
+        lines[0] = f"dkim._domainkey.{mail_domain}. IN TXT " + lines[0].strip(
+            "dkim._domainkey IN TXT "
         )
         return "\n".join(lines)
 
@@ -72,11 +70,9 @@ def show_dns(args, out) -> int:
         acme_account_url = out.shell_output(f"{ssh} -- acmetool account-url")
     except subprocess.CalledProcessError:
         print("Please run `cmdeploy run` first.")
-        return 1
+        return
     dkim_entry = read_dkim_entries(
-        out.shell_output(
-            f"{ssh} -- cat /var/lib/rspamd/dkim/{mail_domain}.{selector}.zone"
-        )
+        out.shell_output(f"{ssh} -- cat /var/lib/rspamd/dkim/{mail_domain}.dkim.zone")
     )
 
     ipv6 = dns.get_ipv6()
@@ -103,7 +99,7 @@ def show_dns(args, out) -> int:
             with open(args.zonefile, "w+") as zf:
                 zf.write(zonefile)
                 print(f"DNS records successfully written to: {args.zonefile}")
-            return 0
+            return
         except TypeError:
             pass
         started_dkim_parsing = False
@@ -157,7 +153,6 @@ def show_dns(args, out) -> int:
         else:
             to_print.append(dkim_entry)
 
-    exit_code = 0
     if to_print:
         to_print.insert(
             0, "You should configure the following DNS entries at your provider:\n"
@@ -166,7 +161,6 @@ def show_dns(args, out) -> int:
             "\nIf you already configured the DNS entries, wait a bit until the DNS entries propagate to the Internet."
         )
         print("\n".join(to_print))
-        exit_code = 1
     else:
         out.green("Great! All your DNS entries are correct.")
 
@@ -186,8 +180,6 @@ def show_dns(args, out) -> int:
         print(
             "You can do so at your hosting provider (maybe this isn't your DNS provider)."
         )
-        exit_code = 1
-    return exit_code
 
 
 def check_necessary_dns(out, mail_domain):

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -13,15 +13,13 @@ auth_cache_size = 100M
 mail_debug = yes
 {% endif %}
 
-mail_server_admin = mailto:root@{{ config.mail_domain }}
-mail_server_comment = Chatmail server
 
 mail_plugins = quota
 
 # these are the capabilities Delta Chat cares about actually 
 # so let's keep the network overhead per login small
 # https://github.com/deltachat/deltachat-core-rust/blob/master/src/imap/capabilities.rs
-imap_capability = IMAP4rev1 IDLE MOVE QUOTA CONDSTORE NOTIFY METADATA
+imap_capability = IMAP4rev1 IDLE MOVE QUOTA CONDSTORE NOTIFY
 
 
 # Authentication for system users.
@@ -75,7 +73,6 @@ mail_privileged_group = vmail
 # <https://datatracker.ietf.org/doc/html/rfc4978.html>
 protocol imap {
   mail_plugins = $mail_plugins imap_zlib imap_quota
-  imap_metadata = yes
 }
 
 protocol lmtp {

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -58,19 +58,8 @@ http {
 		}
 
 		# Old URL for compatibility with e.g. printed QR codes.
-		#
-		# Copy-paste instead of redirect to /new
-		# because Delta Chat core does not follow redirects.
-		#
-		# Redirects are only for browsers.
 		location /cgi-bin/newemail.py {
-			if ($request_method = GET) {
-				return 301 dcaccount:https://{{ config.domain_name }}/new;
-			}
-
-			fastcgi_pass unix:/run/fcgiwrap.socket;
-			include /etc/nginx/fastcgi_params;
-			fastcgi_param SCRIPT_FILENAME /usr/lib/cgi-bin/newemail.py;
+			return 301 /new;
 		}
 	}
 

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -48,3 +48,5 @@ virtual_mailbox_domains = {{ config.mail_domain }}
 
 smtpd_milters = inet:127.0.0.1:11332
 non_smtpd_milters = $smtpd_milters
+
+header_checks = regexp:/etc/postfix/submission_header_cleanup

cmdeploy/src/cmdeploy/postfix/master.cf.j2

@@ -14,6 +14,10 @@ smtp      inet  n       -       y       -       -       smtpd -v
 {% else %}
 smtp      inet  n       -       y       -       -       smtpd 
 {% endif %}
+#smtp      inet  n       -       y       -       1       postscreen
+#smtpd     pass  -       -       y       -       -       smtpd
+#dnsblog   unix  -       -       y       -       0       dnsblog
+#tlsproxy  unix  -       -       y       -       0       tlsproxy
 submission inet n       -       y       -       -       smtpd
   -o syslog_name=postfix/submission
   -o smtpd_tls_security_level=encrypt
@@ -30,7 +34,6 @@ submission inet n       -       y       -       -       smtpd
   -o milter_macro_daemon_name=ORIGINATING
   -o smtpd_client_connection_count_limit=1000
   -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
-  -o cleanup_service_name=authclean
 smtps     inet  n       -       y       -       -       smtpd
   -o syslog_name=postfix/smtps
   -o smtpd_tls_wrappermode=yes
@@ -47,7 +50,6 @@ smtps     inet  n       -       y       -       -       smtpd
   -o smtpd_client_connection_count_limit=1000
   -o milter_macro_daemon_name=ORIGINATING
   -o smtpd_proxy_filter=127.0.0.1:{{ config.filtermail_smtp_port }}
-  -o cleanup_service_name=authclean
 #628       inet  n       -       y       -       -       qmqpd
 pickup    unix  n       -       y       60      1       pickup
 cleanup   unix  n       -       y       -       0       cleanup
@@ -78,13 +80,3 @@ filter    unix -        n       n       -       -       lmtp
 # Local SMTP server for reinjecting filered mail.
 localhost:{{ config.postfix_reinject_port }} inet  n       -       n       -       10      smtpd
   -o syslog_name=postfix/reinject
-  -o cleanup_service_name=authclean
-
-# Cleanup `Received` headers for authenticated mail
-# to avoid leaking client IP.
-#
-# We do not do this for received mails
-# as this will break DKIM signatures
-# if `Received` header is signed.
-authclean unix  n       -       -       -       0       cleanup
-  -o header_checks=regexp:/etc/postfix/submission_header_cleanup

cmdeploy/src/cmdeploy/rspamd/dkim_signing.conf.j2

@@ -1,9 +1,9 @@
-selector = "{{ config.dkim_selector }}"
+selector = {{ config.dkim_selector }}
 use_esld = false  # don't cut c1.testrun.org down to testrun.org
 domain = {
     {{ config.mail_domain }} {
         selectors [
-            selector = "{{ config.dkim_selector }}"
+            selector = {{ config.dkim_selector }}
             path = {{ config.dkim_key_path }}
         ]
     }

cmdeploy/src/cmdeploy/tests/online/test_0_qr.py

@@ -9,7 +9,7 @@ def test_gen_qr_png_data(maildomain):
 
 
 def test_fastcgi_working(maildomain, chatmail_config):
-    url = f"https://{maildomain}/new"
+    url = f"https://{maildomain}/cgi-bin/newemail.py"
     print(url)
     res = requests.post(url)
     assert maildomain in res.json().get("email")
@@ -18,7 +18,7 @@ def test_fastcgi_working(maildomain, chatmail_config):
 
 def test_newemail_configure(maildomain, rpc):
     """Test configuring accounts by scanning a QR code works."""
-    url = f"DCACCOUNT:https://{maildomain}/new"
+    url = f"DCACCOUNT:https://{maildomain}/cgi-bin/newemail.py"
     for i in range(3):
         account_id = rpc.add_account()
         rpc.set_config_from_qr(account_id, url)

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -48,7 +48,7 @@ def test_reject_missing_dkim(cmsetup, maildata, from_addr):
     recipient = cmsetup.gen_users(1)[0]
     msg = maildata("plain.eml", from_addr=from_addr, to_addr=recipient.addr).as_string()
     with smtplib.SMTP(cmsetup.maildomain, 25) as s:
-        with pytest.raises(smtplib.SMTPDataError, match="missing DKIM signature"):
+        with pytest.raises(smtplib.SMTPDataError, match="Spam message rejected"):
             s.sendmail(from_addr=from_addr, to_addrs=recipient.addr, msg=msg)
 
 

scripts/initenv.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 set -e
-python3 -m venv --upgrade-deps venv
+python3 -m venv venv
 
 venv/bin/pip install -e chatmaild 
 venv/bin/pip install -e cmdeploy