Add a script for creating DNS records in Cloudflare

also activates actual deletion (after quite some dry test runs on nine)

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## untagged
 
+- QoL: Add a script for creating DNS records in Cloudflare 
+  ([#692](https://github.com/chatmail/relay/pull/692))
+
+- Require TLS 1.2 for outgoing SMTP connections
+  ([#685](https://github.com/chatmail/relay/pull/685))
+
 - filtermail: run CPU-intensive handle_DATA in a thread pool executor
   ([#676](https://github.com/chatmail/relay/pull/676))
 

README.md

@@ -69,6 +69,20 @@ Please substitute it with your own domain.
     mta-sts.chat.example.com. 3600 IN CNAME chat.example.com.
    ```
 
+> [!note]
+> If you use Cloudflare as your DNS server, you can use a script that will automatically create all the necessary DNS records!
+> To do this, you need to [create an API token](https://dash.cloudflare.com/profile/api-tokens) 
+> and execute the following commands in the console after you clone the repository (step 2):
+> ```bash
+> CLOUDFLARE_API_KEY="dsfkljhfkjldwsnfkjldsnf" # REPLACE TO YOURS
+> ZONE_ID="sdkjbfbnjkdsbfjkdsbkjfbds" # REPLACE TO YOURS
+> CHATMAIL_FULL_DNS_NAME="chat.example.com" # REPLACE TO YOURS
+> CHATMAIL_PUBLIC_IP="198.51.100.5" # REPLACE TO YOURS
+> # IPV6_ENABLED="true" # (optional) by default 'false'
+> # CHATMAIL_PUBLIC_IPv6="2001:db8::5" # (optional) REPLACE TO YOURS
+> ./scripts/create_cloudflare_records.sh
+> ```
+
 2. On your local PC, clone the repository and bootstrap the Python virtualenv.
 
    ```

chatmaild/src/chatmaild/expire.py

@@ -22,11 +22,30 @@ def iter_mailboxes(basedir, maxnum):
         print_info(f"no mailboxes found at: {basedir}")
         return
 
-    for name in os.listdir(basedir)[:maxnum]:
+    for name in os_listdir_if_exists(basedir)[:maxnum]:
         if "@" in name:
             yield MailboxStat(basedir + "/" + name)
 
 
+def get_file_entry(path):
+    """return a FileEntry or None if the path does not exist or is not a regular file."""
+    try:
+        st = os.stat(path)
+    except FileNotFoundError:
+        return None
+    if not S_ISREG(st.st_mode):
+        return None
+    return FileEntry(path, st.st_mtime, st.st_size)
+
+
+def os_listdir_if_exists(path):
+    """return a list of names obtained from os.listdir or an empty list if the path does not exist."""
+    try:
+        return os.listdir(path)
+    except FileNotFoundError:
+        return []
+
+
 class MailboxStat:
     last_login = None
 
@@ -40,19 +59,23 @@ class MailboxStat:
 
         # scan all relevant files (without recursion)
         old_cwd = os.getcwd()
-        os.chdir(self.basedir)
-        for name in os.listdir("."):
+        try:
+            os.chdir(self.basedir)
+        except FileNotFoundError:
+            return
+        for name in os_listdir_if_exists("."):
             if name in ("cur", "new", "tmp"):
-                for msg_name in os.listdir(name):
-                    relpath = name + "/" + msg_name
-                    st = os.stat(relpath)
-                    self.messages.append(FileEntry(relpath, st.st_mtime, st.st_size))
+                for msg_name in os_listdir_if_exists(name):
+                    entry = get_file_entry(f"{name}/{msg_name}")
+                    if entry is not None:
+                        self.messages.append(entry)
+
             else:
-                st = os.stat(name)
-                if S_ISREG(st.st_mode):
-                    self.extrafiles.append(FileEntry(name, st.st_mtime, st.st_size))
+                entry = get_file_entry(name)
+                if entry is not None:
+                    self.extrafiles.append(entry)
                     if name == "password":
-                        self.last_login = st.st_mtime
+                        self.last_login = entry.mtime
         self.extrafiles.sort(key=lambda x: -x.size)
         os.chdir(old_cwd)
 
@@ -80,9 +103,13 @@ class Expiry:
             shutil.rmtree(mboxdir)
         self.del_mboxes += 1
 
-    def remove_file(self, path):
+    def remove_file(self, path, mtime=None):
         if self.verbose:
-            print_info(f"removing {path}")
+            if mtime is not None:
+                date = datetime.fromtimestamp(mtime).strftime("%b %d")
+                print_info(f"removing {date} {path}")
+            else:
+                print_info(f"removing {path}")
         if not self.dry:
             try:
                 os.unlink(path)
@@ -104,18 +131,27 @@ class Expiry:
             return
 
         # all to-be-removed files are relative to the mailbox basedir
-        os.chdir(mbox.basedir)
+        try:
+            os.chdir(mbox.basedir)
+        except FileNotFoundError:
+            print_info(f"mailbox not found/vanished {mbox.basedir}")
+            return
+
         mboxname = os.path.basename(mbox.basedir)
         if self.verbose:
-            print_info(f"checking for mailbox messages in: {mboxname}")
+            date = datetime.fromtimestamp(mbox.last_login) if mbox.last_login else None
+            if date:
+                print_info(f"checking mailbox {date.strftime('%b %d')} {mboxname}")
+            else:
+                print_info(f"checking mailbox (no last_login) {mboxname}")
         self.all_files += len(mbox.messages)
         for message in mbox.messages:
             if message.mtime < cutoff_mails:
-                self.remove_file(message.relpath)
+                self.remove_file(message.relpath, mtime=message.mtime)
             elif message.size > 200000 and message.mtime < cutoff_large_mails:
                 # we only remove noticed large files (not unnoticed ones in new/)
                 if message.relpath.startswith("cur/"):
-                    self.remove_file(message.relpath)
+                    self.remove_file(message.relpath, mtime=message.mtime)
             else:
                 continue
             changed = True

chatmaild/src/chatmaild/tests/test_expire.py

@@ -6,7 +6,13 @@ from pathlib import Path
 
 import pytest
 
-from chatmaild.expire import FileEntry, MailboxStat, iter_mailboxes
+from chatmaild.expire import (
+    FileEntry,
+    MailboxStat,
+    get_file_entry,
+    iter_mailboxes,
+    os_listdir_if_exists,
+)
 from chatmaild.expire import main as expiry_main
 from chatmaild.fsreport import main as report_main
 
@@ -127,3 +133,18 @@ def test_expiry_cli_old_files(capsys, example_config, mbox1):
                 pytest.fail(f"failed to remove {path}\n{err}")
 
     assert "shouldstay" not in err
+
+
+def test_get_file_entry(tmp_path):
+    assert get_file_entry(str(tmp_path.joinpath("123123"))) is None
+    p = tmp_path.joinpath("x")
+    p.write_text("hello")
+    entry = get_file_entry(str(p))
+    assert entry.size == 5
+    assert entry.mtime
+
+
+def test_os_listdir_if_exists(tmp_path):
+    tmp_path.joinpath("x").write_text("hello")
+    assert len(os_listdir_if_exists(str(tmp_path))) == 1
+    assert len(os_listdir_if_exists(str(tmp_path.joinpath("123123")))) == 0

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -26,6 +26,7 @@ smtp_tls_security_level=verify
 smtp_tls_servername = hostname
 smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
 smtp_tls_policy_maps = inline:{nauta.cu=may}
+smtp_tls_protocols = >=TLSv1.2
 smtpd_tls_protocols = >=TLSv1.2
 
 # Disable anonymous cipher suites

scripts/create_cloudflare_records.sh

@@ -0,0 +1,173 @@
+#!/bin/bash
+# go to https://dash.cloudflare.com/profile/api-tokens
+# "create token" -> "Edit zone DNS"
+## optionaly: rename token
+## set your zone 
+# "continue to summary" -> "create token"
+# copy your created token
+
+CLOUDFLARE_API_KEY=${CLOUDFLARE_API_KEY}
+ZONE_ID=${ZONE_ID}
+
+CHATMAIL_FULL_DNS_NAME=${CHATMAIL_FULL_DNS_NAME}
+CHATMAIL_PUBLIC_IP=${CHATMAIL_PUBLIC_IP}
+
+IPV6_ENABLED=${IPV6_ENABLED:-false}
+CHATMAIL_PUBLIC_IPv6=${CHATMAIL_PUBLIC_IPv6}
+
+#####################
+# why 'proxied' is 'false'?
+# I suppose that if Cloudflare is blocked in a country, clients cannot use Deltachat without a VPN.
+#####################
+PROXIED=${PROXIED:-"false"}
+
+check_variables() {
+    required_vars=(
+    CLOUDFLARE_API_KEY
+    ZONE_ID
+    CHATMAIL_FULL_DNS_NAME
+    CHATMAIL_PUBLIC_IP
+    )
+
+    missing_vars=()
+
+    for var in "${required_vars[@]}"; do
+    if [ -z "${!var}" ]; then
+        missing_vars+=("$var")
+    fi
+    done
+
+    if [ ${#missing_vars[@]} -ne 0 ]; then
+    echo "❌ Error: this variables not set or empty:"
+    for var in "${missing_vars[@]}"; do
+        echo "  - $var"
+    done
+    echo "Please execute command 'export var_name=\"var_value\"' and restart script."
+    exit 1
+    fi
+}
+
+
+create_record() {
+    local data=$1
+    curl https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records \
+    -H 'Content-Type: application/json' \
+    -H "Authorization: Bearer ${CLOUDFLARE_API_KEY}" \
+    -d "$1"
+}
+
+generate_post_data_a_aaaa_record()
+{
+    local name=$1
+    local type=${2:-"A"}
+    cat <<EOF
+{
+    "name": "${name}",
+    "ttl": 3600,
+    "type": "${type}",
+    "comment": "",
+    "content": "${CHATMAIL_PUBLIC_IP}",
+    "proxied": ${PROXIED}
+}
+EOF
+}
+
+generate_post_data_cname_record()
+{
+    local name=$1
+    cat <<EOF
+{
+    "name": "${name}",
+    "ttl": 3600,
+    "type": "CNAME",
+    "comment": "",
+    "content": "${CHATMAIL_FULL_DNS_NAME}",
+    "proxied": ${PROXIED}
+}
+EOF
+}
+
+generate_post_data_mx_record()
+{
+    local name=$1
+    cat <<EOF
+{
+    "name": "${name}",
+    "ttl": 120,
+    "type": "MX",
+    "comment": "",
+    "content": "${CHATMAIL_FULL_DNS_NAME}",
+    "priority": 10,
+    "proxied": ${PROXIED}
+}
+EOF
+}
+
+generate_post_data_txt_record()
+{
+    local name=$1
+    local content=$2
+    cat <<EOF
+{
+    "name": "${name}",
+    "ttl": 120,
+    "type": "TXT",
+    "comment": "",
+    "content": "$content",
+    "proxied": ${PROXIED}
+}
+EOF
+}
+
+generate_post_data_srv_record()
+{
+    local name=$1
+    local port=$2
+    cat <<EOF
+{
+    "name": "${name}",
+    "ttl": 120,
+    "type": "SRV",
+    "comment": "",
+    "data": {
+        "port": $port,
+        "priority": 0,
+        "target": "${CHATMAIL_FULL_DNS_NAME}",
+        "weight": 1
+    },
+    "proxied": ${PROXIED}
+}
+EOF
+}
+
+check_variables
+
+# A records
+create_record "$(generate_post_data_a_record "$CHATMAIL_FULL_DNS_NAME" "A")"
+create_record "$(generate_post_data_a_record "*.$CHATMAIL_FULL_DNS_NAME" "A")"
+
+# AAAA records
+if [ $IPV6_ENABLED = true ]; then # note: I don't have an IPv6 address, so this part hasn't been tested!
+    create_record "$(generate_post_data_a_record "$CHATMAIL_FULL_DNS_NAME" "AAAA")"
+    # create_record "$(generate_post_data_a_record "*.$CHATMAIL_FULL_DNS_NAME" "AAAA")"
+fi
+
+# CNAME records
+create_record "$(generate_post_data_cname_record "mta-sts.$CHATMAIL_FULL_DNS_NAME")"
+create_record "$(generate_post_data_cname_record "www.$CHATMAIL_FULL_DNS_NAME")"
+
+# MX records
+create_record "$(generate_post_data_mx_record "$CHATMAIL_FULL_DNS_NAME")"
+
+# TXT records
+create_record "$(generate_post_data_txt_record "$CHATMAIL_FULL_DNS_NAME" '\"v=spf1 a ~all\"')"
+create_record "$(generate_post_data_txt_record "_dmarc.$CHATMAIL_FULL_DNS_NAME" '\"v=DMARC1;p=reject;adkim=s;aspf=s\"')"
+create_record "$(generate_post_data_txt_record "_adsp._domainkey.$CHATMAIL_FULL_DNS_NAME" '\"dkim=discardable\"')"
+create_record "$(generate_post_data_txt_record "opendkim._domainkey.$CHATMAIL_FULL_DNS_NAME" '\"v=DKIM1;k=rsa;p=;s=email;t=s\"')"
+create_record "$(generate_post_data_txt_record "_mta-sts.$CHATMAIL_FULL_DNS_NAME" '\"v=STSv1; id='"$(date +"%Y%m%d%H%M")"'\"')"
+
+# SRV records
+create_record "$(generate_post_data_srv_record "_imap._tcp.$CHATMAIL_FULL_DNS_NAME" "143")"
+create_record "$(generate_post_data_srv_record "_imaps._tcp.$CHATMAIL_FULL_DNS_NAME" "993")"
+create_record "$(generate_post_data_srv_record "_submission._tcp.$CHATMAIL_FULL_DNS_NAME" "587")"
+create_record "$(generate_post_data_srv_record "_submissions._tcp.$CHATMAIL_FULL_DNS_NAME" "465")"