Dovecot: setup METADATA

There is no dictionary to set additional attributes,
but admin email can already be retrieved:

? GETMETADATA "" (/shared/admin)
* METADATA "" (/shared/admin {27}
mailto:root@c20.testrun.org)
? OK Getmetadata completed (0.001 + 0.000 secs).

.github/workflows/staging.testrun.org-default.zone

@@ -0,0 +1,20 @@
+;; Zone file for staging.testrun.org
+
+$ORIGIN staging.testrun.org.
+$TTL 300
+
+@  IN  SOA     ns.testrun.org. root.nine.testrun.org (
+  2023010101 ; Serial
+  7200       ; Refresh
+  3600       ; Retry
+  1209600    ; Expire
+  3600       ; Negative response caching TTL
+)
+
+;; Nameservers.
+@  IN NS ns.testrun.org.
+
+;; DNS records.
+@       IN      A       37.27.37.98
+mta-sts.staging.testrun.org.    CNAME   staging.testrun.org.
+www.staging.testrun.org.        CNAME   staging.testrun.org.

.github/workflows/test-and-deploy.yaml

@@ -0,0 +1,72 @@
+name: deploy on staging.testrun.org, and run tests
+
+on:
+  push:
+    branches:
+      - main
+      - staging-ci
+
+jobs:
+  deploy:
+    name: deploy on staging.testrun.org, and run tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: prepare SSH
+        run: |
+          mkdir ~/.ssh
+          echo "${{ secrets.STAGING_SSH_KEY }}" >> ~/.ssh/id_ed25519
+          chmod 600 ~/.ssh/id_ed25519
+          ssh-keyscan staging.testrun.org > ~/.ssh/known_hosts
+     #    rsync -avz root@staging.testrun.org:/var/lib/acme . || true
+     #    rsync -avz root@staging.testrun.org:/var/lib/rspamd/dkim . || true
+
+     #- name: rebuild staging.testrun.org to have a clean VPS
+     #  run: |
+     #      curl -X POST \
+     #      -H "Authorization: Bearer ${{ secrets.HETZNER_API_TOKEN }}" \
+     #      -H "Content-Type: application/json" \
+     #      -d '{"image":"debian-12"}' \
+     #      "https://api.hetzner.cloud/v1/servers/${{ secrets.STAGING_SERVER_ID }}/actions/rebuild"
+
+      - run: scripts/initenv.sh
+
+      - name: append venv/bin to PATH
+        run: echo venv/bin >>$GITHUB_PATH
+
+      - name: run formatting checks 
+        run: cmdeploy fmt -v 
+
+      - name: run deploy-chatmail offline tests 
+        run: pytest --pyargs cmdeploy 
+
+     #- name: upload TLS cert after rebuilding
+     #  run: |
+     #    echo " --- wait until staging.testrun.org VPS is rebuilt --- "
+     #    rm ~/.ssh/known_hosts
+     #    while ! ssh -o ConnectTimeout=180 -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u ; do sleep 1 ; done
+     #    ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org id -u
+     #    rsync -avz acme root@staging.testrun.org:/var/lib/ || true
+     #    rsync -avz dkim root@staging.testrun.org:/var/lib/rspamd/ || true
+
+      - run: cmdeploy init staging.testrun.org
+
+      - run: cmdeploy run
+
+      - name: set DNS entries
+        run: |
+          #ssh -o StrictHostKeyChecking=accept-new -v root@staging.testrun.org chown _rspamd:_rspamd -R /var/lib/rspamd/dkim
+          cmdeploy dns --zonefile staging-generated.zone
+          cat staging-generated.zone >> .github/workflows/staging.testrun.org-default.zone
+          cat .github/workflows/staging.testrun.org-default.zone
+          scp -o StrictHostKeyChecking=accept-new .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging.testrun.org.zone
+          ssh root@ns.testrun.org nsd-checkzone staging.testrun.org /etc/nsd/staging.testrun.org.zone
+          ssh root@ns.testrun.org systemctl reload nsd
+
+      - name: cmdeploy test
+        run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow
+
+      - name: cmdeploy dns (try 3 times)
+        run: cmdeploy dns || cmdeploy dns || cmdeploy dns
+

chatmaild/pyproject.toml

@@ -10,10 +10,6 @@ dependencies = [
   "iniconfig",
   "deltachat-rpc-server",
   "deltachat-rpc-client",
-  "ConfigArgParse",
-  "deltachat",
-  "setuptools>=60",
-  "setuptools-scm>=8",
 ]
 
 [tool.setuptools]
@@ -26,7 +22,6 @@ where = ['src']
 doveauth = "chatmaild.doveauth:main"
 filtermail = "chatmaild.filtermail:main"
 echobot = "chatmaild.echo:main"
-greeterbot = "chatmaild.greeterbot:main"
 chatmail-metrics = "chatmaild.metrics:main"
 
 [project.entry-points.pytest11]

chatmaild/src/chatmaild/database.py

@@ -46,17 +46,11 @@ class Connection:
             )
         return result
 
-    def get_user_list(self) -> set[str]:
-        """Get a set of all users."""
-        q = "SELECT addr from users"
-        return set([tup[0] for tup in self._sqlconn.execute(q).fetchall()])
-
 
 class Database:
-    def __init__(self, path: str, read_only=False):
+    def __init__(self, path: str):
         self.path = Path(path)
-        if not read_only:
-            self.ensure_tables()
+        self.ensure_tables()
 
     def _get_connection(
         self, write=False, transaction=False, closing=False

chatmaild/src/chatmaild/doveauth.py

@@ -46,7 +46,7 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
         len(localpart) > config.username_max_length
         or len(localpart) < config.username_min_length
     ):
-        if localpart not in ("echo", "hello"):
+        if localpart != "echo":
             logging.warning(
                 "localpart %s has to be between %s and %s chars long",
                 localpart,

chatmaild/src/chatmaild/greeterbot.py

@@ -1,132 +0,0 @@
-import time
-
-import deltachat
-from deltachat.tracker import ConfigureFailed
-from time import sleep
-import tempfile
-import os
-import configargparse
-import pkg_resources
-import secrets
-
-from chatmaild.database import Database
-from chatmaild.config import read_config
-from chatmaild.newemail import ALPHANUMERIC_PUNCT, CONFIG_PATH
-
-PASSDB_PATH = "/home/vmail/passdb.sqlite"
-
-
-def setup_account(data_dir: str, debug: bool) -> deltachat.Account:
-    """Create a deltachat account with a given addr/password combination.
-
-    :param data_dir: the directory where the data(base) is stored.
-    :param debug: whether to show log messages for the account.
-    :return: the deltachat account object.
-    """
-    chatmail_config = read_config(CONFIG_PATH)
-    addr = "hello@" + chatmail_config.mail_domain
-
-    try:
-        os.mkdir(os.path.join(data_dir, addr))
-    except FileExistsError:
-        pass
-    db_path = os.path.join(data_dir, addr, "db.sqlite")
-
-    ac = deltachat.Account(db_path)
-    if debug:
-        ac.add_account_plugin(deltachat.events.FFIEventLogger(ac))
-
-    ac.set_config("mvbox_move", "0")
-    ac.set_config("sentbox_watch", "0")
-    ac.set_config("bot", "1")
-    ac.set_config("mdns_enabled", "0")
-
-    if not ac.is_configured():
-        cleartext_password = "".join(
-            secrets.choice(ALPHANUMERIC_PUNCT)
-            for _ in range(chatmail_config.password_min_length + 3)
-        )
-        ac.set_config("mail_pw", cleartext_password)
-        ac.set_config("addr", addr)
-
-        configtracker = ac.configure()
-        try:
-            configtracker.wait_finish()
-        except ConfigureFailed:
-            print(
-                "configuration setup failed for %s with password:\n%s"
-                % (ac.get_config("addr"), ac.get_config("mail_pw"))
-            )
-            raise
-
-    ac.start_io()
-    avatar = pkg_resources.resource_filename(__name__, "avatar.jpg")
-    ac.set_avatar(avatar)
-    ac.set_config("displayname", f"Hello at {chatmail_config.mail_domain}!")
-    return ac
-
-
-class GreetBot:
-    def __init__(self, passdb, account):
-        self.db = Database(passdb, read_only=True)
-        self.account = account
-        self.domain = account.get_config("addr").split("@")[1]
-        with self.db.read_connection() as conn:
-            self.existing_users = conn.get_user_list()
-
-    def greet_users(self):
-        with self.db.read_connection() as conn:
-            users = conn.get_user_list()
-        new_users = users.difference(self.existing_users)
-        self.existing_users = users
-        time.sleep(20)  # wait until Delta is configured on the user side
-        for user in new_users:
-            for ci_prefix in ["ac1_", "ac2_", "ac3_", "ac4_", "ac5_", "ci-"]:
-                if user.startswith(ci_prefix):
-                    continue
-            if user not in [c.addr for c in self.account.get_contacts()]:
-                print("Inviting", user)
-                contact = self.account.create_contact(user)
-                chat = contact.create_chat()
-                chat.send_text(
-                    "Welcome to %s! Here you can try out Delta Chat." % (self.domain,)
-                )
-                chat.send_text(
-                    "I prepared some webxdc apps for you, if you are interested:"
-                )
-                chat.send_file(pkg_resources.resource_filename(__name__, "editor.xdc"))
-                chat.send_file(
-                    pkg_resources.resource_filename(__name__, "tower-builder.xdc")
-                )
-                chat.send_text(
-                    "You can visit https://webxdc.org/apps to discover more apps! "
-                    "Some of these games you can also play with friends, directly in the chat."
-                )
-
-
-def main():
-    args = configargparse.ArgumentParser()
-    args.add_argument("--db_path", help="location of the Delta Chat database")
-    args.add_argument(
-        "--passdb", default=PASSDB_PATH, help="location of the chatmail passdb"
-    )
-    args.add_argument("--show-ffi", action="store_true", help="print Delta Chat log")
-    ops = args.parse_args()
-
-    # ensuring account data directory
-    if ops.db_path is None:
-        tempdir = tempfile.TemporaryDirectory(prefix="hellobot")
-        ops.db_path = tempdir.name
-    elif not os.path.exists(ops.db_path):
-        os.mkdir(ops.db_path)
-
-    ac = setup_account(ops.db_path, ops.show_ffi)
-    greeter = GreetBot(ops.passdb, ac)
-    print("waiting for new chatmail users...")
-    while 1:
-        greeter.greet_users()
-        sleep(5)
-
-
-if __name__ == "__main__":
-    main()

chatmaild/src/chatmaild/greeterbot.service.f

@@ -1,11 +0,0 @@
-[Unit]
-Description=Chatmail greeterbot, a Delta Chat bot to greet new users
-
-[Service]
-ExecStart={execpath} --passdb {passdb_path} --db_path /home/vmail/greeterbot/ --show-ffi
-User=vmail
-Restart=always
-RestartSec=30
-
-[Install]
-WantedBy=multi-user.target

cmdeploy/src/cmdeploy/__init__.py

@@ -101,13 +101,11 @@ def _install_remote_venv_with_chatmaild(config) -> None:
         "doveauth",
         "filtermail",
         "echobot",
-        "greeterbot",
     ):
         params = dict(
             execpath=f"{remote_venv_dir}/bin/{fn}",
             config_path=remote_chatmail_inipath,
             remote_venv_dir=remote_venv_dir,
-            passdb_path="/home/vmail/passdb.sqlite",
         )
         source_path = importlib.resources.files("chatmaild").joinpath(f"{fn}.service.f")
         content = source_path.read_text().format(**params).encode()
@@ -444,7 +442,10 @@ def deploy_chatmail(config_path: Path) -> None:
     )
     server.shell(
         name="Generate root keys for validating DNSSEC",
-        commands=["unbound-anchor -a /var/lib/unbound/root.key || true"],
+        commands=[
+            "unbound-anchor -a /var/lib/unbound/root.key || true",
+            "systemctl reset-failed unbound.service",
+        ],
     )
     systemd.service(
         name="Start and enable unbound",

cmdeploy/src/cmdeploy/chatmail.zone.f

@@ -7,7 +7,7 @@ _imap._tcp.{chatmail_domain}.        SRV 0 1 143 {chatmail_domain}.
 _imaps._tcp.{chatmail_domain}.       SRV 0 1 993 {chatmail_domain}.
 {chatmail_domain}.                   CAA 128 issue "letsencrypt.org;accounturi={acme_account_url}"
 {chatmail_domain}.                   TXT "v=spf1 a:{chatmail_domain} -all"
-_dmarc.{chatmail_domain}.            TXT "v=DMARC1;p=reject;rua=mailto:{email};ruf=mailto:{email};fo=1;adkim=s;aspf=s"
+_dmarc.{chatmail_domain}.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
 _mta-sts.{chatmail_domain}.          TXT "v=STSv1; id={sts_id}"
 mta-sts.{chatmail_domain}.           CNAME {chatmail_domain}.
 www.{chatmail_domain}.               CNAME {chatmail_domain}.

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -82,7 +82,8 @@ def dns_cmd_options(parser):
 
 def dns_cmd(args, out):
     """Generate dns zone file."""
-    show_dns(args, out)
+    exit_code = show_dns(args, out)
+    exit(exit_code)
 
 
 def status_cmd(args, out):

cmdeploy/src/cmdeploy/dns.py

@@ -47,7 +47,8 @@ class DNS:
         return result == f"{mail_domain}."
 
 
-def show_dns(args, out):
+def show_dns(args, out) -> int:
+    """Check existing DNS records, optionally write them to zone file, return exit code 0 or 1."""
     template = importlib.resources.files(__package__).joinpath("chatmail.zone.f")
     mail_domain = args.config.mail_domain
     ssh = f"ssh root@{mail_domain}"
@@ -70,7 +71,7 @@ def show_dns(args, out):
         acme_account_url = out.shell_output(f"{ssh} -- acmetool account-url")
     except subprocess.CalledProcessError:
         print("Please run `cmdeploy run` first.")
-        return
+        return 1
     dkim_entry = read_dkim_entries(
         out.shell_output(f"{ssh} -- cat /var/lib/rspamd/dkim/{mail_domain}.dkim.zone")
     )
@@ -99,7 +100,7 @@ def show_dns(args, out):
             with open(args.zonefile, "w+") as zf:
                 zf.write(zonefile)
                 print(f"DNS records successfully written to: {args.zonefile}")
-            return
+            return 0
         except TypeError:
             pass
         started_dkim_parsing = False
@@ -153,6 +154,7 @@ def show_dns(args, out):
         else:
             to_print.append(dkim_entry)
 
+    exit_code = 0
     if to_print:
         to_print.insert(
             0, "You should configure the following DNS entries at your provider:\n"
@@ -161,6 +163,7 @@ def show_dns(args, out):
             "\nIf you already configured the DNS entries, wait a bit until the DNS entries propagate to the Internet."
         )
         print("\n".join(to_print))
+        exit_code = 1
     else:
         out.green("Great! All your DNS entries are correct.")
 
@@ -180,6 +183,8 @@ def show_dns(args, out):
         print(
             "You can do so at your hosting provider (maybe this isn't your DNS provider)."
         )
+        exit_code = 1
+    return exit_code
 
 
 def check_necessary_dns(out, mail_domain):

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -13,13 +13,15 @@ auth_cache_size = 100M
 mail_debug = yes
 {% endif %}
 
+mail_server_admin = mailto:root@{{ config.mail_domain }}
+mail_server_comment = Chatmail server
 
 mail_plugins = quota
 
 # these are the capabilities Delta Chat cares about actually 
 # so let's keep the network overhead per login small
 # https://github.com/deltachat/deltachat-core-rust/blob/master/src/imap/capabilities.rs
-imap_capability = IMAP4rev1 IDLE MOVE QUOTA CONDSTORE NOTIFY
+imap_capability = IMAP4rev1 IDLE MOVE QUOTA CONDSTORE NOTIFY METADATA
 
 
 # Authentication for system users.
@@ -73,6 +75,7 @@ mail_privileged_group = vmail
 # <https://datatracker.ietf.org/doc/html/rfc4978.html>
 protocol imap {
   mail_plugins = $mail_plugins imap_zlib imap_quota
+  imap_metadata = yes
 }
 
 protocol lmtp {

cmdeploy/src/cmdeploy/genqr.py

@@ -6,7 +6,7 @@ import io
 
 
 def gen_qr_png_data(maildomain):
-    url = f"DCACCOUNT:https://{maildomain}/cgi-bin/newemail.py"
+    url = f"DCACCOUNT:https://{maildomain}/new"
     image = gen_qr(maildomain, url)
     temp = io.BytesIO()
     image.save(temp, format="png")

cmdeploy/src/cmdeploy/nginx/nginx.conf.j2

@@ -45,8 +45,22 @@ http {
 			default_type text/plain;
 		}
 
-		# add cgi-bin support
-		include /usr/share/doc/fcgiwrap/examples/nginx.conf;
+		location /new {
+			if ($request_method = GET) {
+				# Redirect to Delta Chat,
+				# which will in turn do a POST request.
+				return 301 dcaccount:https://{{ config.domain_name }}/new;
+			}
+
+			fastcgi_pass unix:/run/fcgiwrap.socket;
+			include /etc/nginx/fastcgi_params;
+			fastcgi_param SCRIPT_FILENAME /usr/lib/cgi-bin/newemail.py;
+		}
+
+		# Old URL for compatibility with e.g. printed QR codes.
+		location /cgi-bin/newemail.py {
+			return 301 /new;
+		}
 	}
 
 	# Redirect www. to non-www

cmdeploy/src/cmdeploy/rspamd/force_actions.conf

@@ -1,30 +1,60 @@
 rules {
-    REJECT_DKIM_SPF {
+    ## Reject on missing or invalid DKIM signatures.
+    ##
+    ## We require DKIM signature on incoming mails regardless of DMARC policy.
+
+    # R_DKIM_REJECT: DKIM reject inserted by `dkim` module.
+    REJECT_INVALID_DKIM {
         action = "reject";
-        # Reject if
-        #   - R_DKIM_RJECT: DKIM reject inserted by `dkim` module.
-        #   - R_DKIM_PERMFAIL: permanent failure inserted by `dkim` module e.g. no DKIM DNS record found.
-        #   - No DKIM signing (R_DKIM_NA symbol inserted by `dkim` module)
-        #
-        #   - SPF failure (R_SPF_FAIL)
-        #   - SPF permanent failure, e.g. failed to resolve DNS record referenced from SPF (R_SPF_PERMFAIL)
-        #
-        #   - DMARC policy failure (DMARC_POLICY_REJECT)
-        #
-        # Do not reject if:
-        #   - R_DKIM_TEMPFAIL, it is a DNS resolution failure
-        #     and we do not want to lose messages because of faulty network.
-        #
-        #   - R_SPF_SOFTFAIL
-        #   - R_SPF_NEUTRAL
-        #   - R_SPF_DNSFAIL
-        #   - R_SPF_NA
-        #
-        #   - DMARC_DNSFAIL
-        #   - DMARC_NA
-        #   - DMARC_POLICY_SOFTFAIL
-        #   - DMARC_POLICY_QUARANTINE
-        #   - DMARC_BAD_POLICY
-        expression = "R_DKIM_REJECT | R_DKIM_PERMFAIL | R_DKIM_NA | R_SPF_FAIL | R_SPF_PERMFAIL | DMARC_POLICY_REJECT";
+        expression = "R_DKIM_REJECT";
+        message = "Rejected due to invalid DKIM signature";
     }
+
+    # R_DKIM_PERMFAIL: permanent failure inserted by `dkim` module e.g. no DKIM DNS record found.
+    REJECT_PERMFAIL_DKIM {
+        action = "reject";
+        expression = "R_DKIM_PERMFAIL";
+        message = "Rejected due to missing DKIM DNS entry";
+    }
+
+    # No DKIM signature (R_DKIM_NA symbol inserted by `dkim` module).
+    REJECT_MISSING_DKIM {
+        action = "reject";
+        expression = "R_DKIM_NA";
+        message = "Rejected due to missing DKIM signature";
+    }
+
+
+    ## Reject on SPF failure.
+
+    # - SPF failure (R_SPF_FAIL)
+    # - SPF permanent failure, e.g. failed to resolve DNS record referenced from SPF (R_SPF_PERMFAIL)
+    REJECT_SPF {
+        action = "reject";
+        expression = "R_SPF_FAIL | R_SPF_PERMFAIL";
+        message = "Rejected due to failed SPF check";
+    }
+
+    # Reject on DMARC policy check failure.
+    REJECT_DMARC {
+        action = "reject";
+        expression = "DMARC_POLICY_REJECT";
+        message = "Rejected due to DMARC policy";
+    }
+
+
+    # Do not reject if:
+    # - R_DKIM_TEMPFAIL, it is a DNS resolution failure
+    #   and we do not want to lose messages because of faulty network.
+    #
+    # - R_SPF_SOFTFAIL
+    # - R_SPF_NEUTRAL
+    # - R_SPF_DNSFAIL
+    # - R_SPF_NA
+    #
+    # - DMARC_DNSFAIL
+    # - DMARC_NA
+    # - DMARC_POLICY_SOFTFAIL
+    # - DMARC_POLICY_QUARANTINE
+    # - DMARC_BAD_POLICY
 }

cmdeploy/src/cmdeploy/tests/online/test_0_qr.py

@@ -9,7 +9,7 @@ def test_gen_qr_png_data(maildomain):
 
 
 def test_fastcgi_working(maildomain, chatmail_config):
-    url = f"https://{maildomain}/cgi-bin/newemail.py"
+    url = f"https://{maildomain}/new"
     print(url)
     res = requests.post(url)
     assert maildomain in res.json().get("email")
@@ -18,7 +18,7 @@ def test_fastcgi_working(maildomain, chatmail_config):
 
 def test_newemail_configure(maildomain, rpc):
     """Test configuring accounts by scanning a QR code works."""
-    url = f"DCACCOUNT:https://{maildomain}/cgi-bin/newemail.py"
+    url = f"DCACCOUNT:https://{maildomain}/new"
     for i in range(3):
         account_id = rpc.add_account()
         rpc.set_config_from_qr(account_id, url)

cmdeploy/src/cmdeploy/tests/online/test_1_basic.py

@@ -48,7 +48,7 @@ def test_reject_missing_dkim(cmsetup, maildata, from_addr):
     recipient = cmsetup.gen_users(1)[0]
     msg = maildata("plain.eml", from_addr=from_addr, to_addr=recipient.addr).as_string()
     with smtplib.SMTP(cmsetup.maildomain, 25) as s:
-        with pytest.raises(smtplib.SMTPDataError, match="Spam message rejected"):
+        with pytest.raises(smtplib.SMTPDataError, match="missing DKIM signature"):
             s.sendmail(from_addr=from_addr, to_addrs=recipient.addr, msg=msg)
 
 

www/src/index.md

@@ -7,7 +7,7 @@ Welcome to instant, interoperable and [privacy-preserving](privacy.html) messagi
 
 👉 **Tap** or scan this QR code to get a random `@{{config.mail_domain}}` e-mail address
 
-<a href="DCACCOUNT:https://{{ config.mail_domain }}/cgi-bin/newemail.py">
+<a href="DCACCOUNT:https://{{ config.mail_domain }}/new">
     <img width=300 style="float: none;" src="qr-chatmail-invite-{{config.mail_domain}}.png" /></a>
 
 🐣 **Choose** your Avatar and Name