fixup after testing on nine:

- don't remove large files already after 7 days if they are in the "new/" folder
- report which mailbox is being checked so that "journalctl -u
  chatmail-expire.service" provides sufficient output for checking
- don't trigger expiry or fsreport services during cmdeploy-run but run it from timer only

CHANGELOG.md

@@ -2,12 +2,6 @@
 
 ## untagged
 
-- QoL: Add a script for creating DNS records in Cloudflare 
-  ([#692](https://github.com/chatmail/relay/pull/692))
-
-- Require TLS 1.2 for outgoing SMTP connections
-  ([#685](https://github.com/chatmail/relay/pull/685))
-
 - filtermail: run CPU-intensive handle_DATA in a thread pool executor
   ([#676](https://github.com/chatmail/relay/pull/676))
 

README.md

@@ -69,20 +69,6 @@ Please substitute it with your own domain.
     mta-sts.chat.example.com. 3600 IN CNAME chat.example.com.
    ```
 
-> [!note]
-> If you use Cloudflare as your DNS server, you can use a script that will automatically create all the necessary DNS records!
-> To do this, you need to [create an API token](https://dash.cloudflare.com/profile/api-tokens) 
-> and execute the following commands in the console after you clone the repository (step 2):
-> ```bash
-> CLOUDFLARE_API_KEY="dsfkljhfkjldwsnfkjldsnf" # REPLACE TO YOURS
-> ZONE_ID="sdkjbfbnjkdsbfjkdsbkjfbds" # REPLACE TO YOURS
-> CHATMAIL_FULL_DNS_NAME="chat.example.com" # REPLACE TO YOURS
-> CHATMAIL_PUBLIC_IP="198.51.100.5" # REPLACE TO YOURS
-> # IPV6_ENABLED="true" # (optional) by default 'false'
-> # CHATMAIL_PUBLIC_IPv6="2001:db8::5" # (optional) REPLACE TO YOURS
-> ./scripts/create_cloudflare_records.sh
-> ```
-
 2. On your local PC, clone the repository and bootstrap the Python virtualenv.
 
    ```

chatmaild/src/chatmaild/expire.py

@@ -22,30 +22,11 @@ def iter_mailboxes(basedir, maxnum):
         print_info(f"no mailboxes found at: {basedir}")
         return
 
-    for name in os_listdir_if_exists(basedir)[:maxnum]:
+    for name in os.listdir(basedir)[:maxnum]:
         if "@" in name:
             yield MailboxStat(basedir + "/" + name)
 
 
-def get_file_entry(path):
-    """return a FileEntry or None if the path does not exist or is not a regular file."""
-    try:
-        st = os.stat(path)
-    except FileNotFoundError:
-        return None
-    if not S_ISREG(st.st_mode):
-        return None
-    return FileEntry(path, st.st_mtime, st.st_size)
-
-
-def os_listdir_if_exists(path):
-    """return a list of names obtained from os.listdir or an empty list if the path does not exist."""
-    try:
-        return os.listdir(path)
-    except FileNotFoundError:
-        return []
-
-
 class MailboxStat:
     last_login = None
 
@@ -59,23 +40,19 @@ class MailboxStat:
 
         # scan all relevant files (without recursion)
         old_cwd = os.getcwd()
-        try:
-            os.chdir(self.basedir)
-        except FileNotFoundError:
-            return
-        for name in os_listdir_if_exists("."):
+        os.chdir(self.basedir)
+        for name in os.listdir("."):
             if name in ("cur", "new", "tmp"):
-                for msg_name in os_listdir_if_exists(name):
-                    entry = get_file_entry(f"{name}/{msg_name}")
-                    if entry is not None:
-                        self.messages.append(entry)
-
+                for msg_name in os.listdir(name):
+                    relpath = name + "/" + msg_name
+                    st = os.stat(relpath)
+                    self.messages.append(FileEntry(relpath, st.st_mtime, st.st_size))
             else:
-                entry = get_file_entry(name)
-                if entry is not None:
-                    self.extrafiles.append(entry)
+                st = os.stat(name)
+                if S_ISREG(st.st_mode):
+                    self.extrafiles.append(FileEntry(name, st.st_mtime, st.st_size))
                     if name == "password":
-                        self.last_login = entry.mtime
+                        self.last_login = st.st_mtime
         self.extrafiles.sort(key=lambda x: -x.size)
         os.chdir(old_cwd)
 
@@ -103,13 +80,9 @@ class Expiry:
             shutil.rmtree(mboxdir)
         self.del_mboxes += 1
 
-    def remove_file(self, path, mtime=None):
+    def remove_file(self, path):
         if self.verbose:
-            if mtime is not None:
-                date = datetime.fromtimestamp(mtime).strftime("%b %d")
-                print_info(f"removing {date} {path}")
-            else:
-                print_info(f"removing {path}")
+            print_info(f"removing {path}")
         if not self.dry:
             try:
                 os.unlink(path)
@@ -131,27 +104,18 @@ class Expiry:
             return
 
         # all to-be-removed files are relative to the mailbox basedir
-        try:
-            os.chdir(mbox.basedir)
-        except FileNotFoundError:
-            print_info(f"mailbox not found/vanished {mbox.basedir}")
-            return
-
+        os.chdir(mbox.basedir)
         mboxname = os.path.basename(mbox.basedir)
         if self.verbose:
-            date = datetime.fromtimestamp(mbox.last_login) if mbox.last_login else None
-            if date:
-                print_info(f"checking mailbox {date.strftime('%b %d')} {mboxname}")
-            else:
-                print_info(f"checking mailbox (no last_login) {mboxname}")
+            print_info(f"checking for mailbox messages in: {mboxname}")
         self.all_files += len(mbox.messages)
         for message in mbox.messages:
             if message.mtime < cutoff_mails:
-                self.remove_file(message.relpath, mtime=message.mtime)
+                self.remove_file(message.relpath)
             elif message.size > 200000 and message.mtime < cutoff_large_mails:
                 # we only remove noticed large files (not unnoticed ones in new/)
                 if message.relpath.startswith("cur/"):
-                    self.remove_file(message.relpath, mtime=message.mtime)
+                    self.remove_file(message.relpath)
             else:
                 continue
             changed = True

chatmaild/src/chatmaild/tests/test_expire.py

@@ -6,13 +6,7 @@ from pathlib import Path
 
 import pytest
 
-from chatmaild.expire import (
-    FileEntry,
-    MailboxStat,
-    get_file_entry,
-    iter_mailboxes,
-    os_listdir_if_exists,
-)
+from chatmaild.expire import FileEntry, MailboxStat, iter_mailboxes
 from chatmaild.expire import main as expiry_main
 from chatmaild.fsreport import main as report_main
 
@@ -133,18 +127,3 @@ def test_expiry_cli_old_files(capsys, example_config, mbox1):
                 pytest.fail(f"failed to remove {path}\n{err}")
 
     assert "shouldstay" not in err
-
-
-def test_get_file_entry(tmp_path):
-    assert get_file_entry(str(tmp_path.joinpath("123123"))) is None
-    p = tmp_path.joinpath("x")
-    p.write_text("hello")
-    entry = get_file_entry(str(p))
-    assert entry.size == 5
-    assert entry.mtime
-
-
-def test_os_listdir_if_exists(tmp_path):
-    tmp_path.joinpath("x").write_text("hello")
-    assert len(os_listdir_if_exists(str(tmp_path))) == 1
-    assert len(os_listdir_if_exists(str(tmp_path.joinpath("123123")))) == 0

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -26,7 +26,6 @@ smtp_tls_security_level=verify
 smtp_tls_servername = hostname
 smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
 smtp_tls_policy_maps = inline:{nauta.cu=may}
-smtp_tls_protocols = >=TLSv1.2
 smtpd_tls_protocols = >=TLSv1.2
 
 # Disable anonymous cipher suites

scripts/create_cloudflare_records.sh

@@ -1,173 +0,0 @@
-#!/bin/bash
-# go to https://dash.cloudflare.com/profile/api-tokens
-# "create token" -> "Edit zone DNS"
-## optionaly: rename token
-## set your zone 
-# "continue to summary" -> "create token"
-# copy your created token
-
-CLOUDFLARE_API_KEY=${CLOUDFLARE_API_KEY}
-ZONE_ID=${ZONE_ID}
-
-CHATMAIL_FULL_DNS_NAME=${CHATMAIL_FULL_DNS_NAME}
-CHATMAIL_PUBLIC_IP=${CHATMAIL_PUBLIC_IP}
-
-IPV6_ENABLED=${IPV6_ENABLED:-false}
-CHATMAIL_PUBLIC_IPv6=${CHATMAIL_PUBLIC_IPv6}
-
-#####################
-# why 'proxied' is 'false'?
-# I suppose that if Cloudflare is blocked in a country, clients cannot use Deltachat without a VPN.
-#####################
-PROXIED=${PROXIED:-"false"}
-
-check_variables() {
-    required_vars=(
-    CLOUDFLARE_API_KEY
-    ZONE_ID
-    CHATMAIL_FULL_DNS_NAME
-    CHATMAIL_PUBLIC_IP
-    )
-
-    missing_vars=()
-
-    for var in "${required_vars[@]}"; do
-    if [ -z "${!var}" ]; then
-        missing_vars+=("$var")
-    fi
-    done
-
-    if [ ${#missing_vars[@]} -ne 0 ]; then
-    echo "❌ Error: this variables not set or empty:"
-    for var in "${missing_vars[@]}"; do
-        echo "  - $var"
-    done
-    echo "Please execute command 'export var_name=\"var_value\"' and restart script."
-    exit 1
-    fi
-}
-
-
-create_record() {
-    local data=$1
-    curl https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records \
-    -H 'Content-Type: application/json' \
-    -H "Authorization: Bearer ${CLOUDFLARE_API_KEY}" \
-    -d "$1"
-}
-
-generate_post_data_a_aaaa_record()
-{
-    local name=$1
-    local type=${2:-"A"}
-    cat <<EOF
-{
-    "name": "${name}",
-    "ttl": 3600,
-    "type": "${type}",
-    "comment": "",
-    "content": "${CHATMAIL_PUBLIC_IP}",
-    "proxied": ${PROXIED}
-}
-EOF
-}
-
-generate_post_data_cname_record()
-{
-    local name=$1
-    cat <<EOF
-{
-    "name": "${name}",
-    "ttl": 3600,
-    "type": "CNAME",
-    "comment": "",
-    "content": "${CHATMAIL_FULL_DNS_NAME}",
-    "proxied": ${PROXIED}
-}
-EOF
-}
-
-generate_post_data_mx_record()
-{
-    local name=$1
-    cat <<EOF
-{
-    "name": "${name}",
-    "ttl": 120,
-    "type": "MX",
-    "comment": "",
-    "content": "${CHATMAIL_FULL_DNS_NAME}",
-    "priority": 10,
-    "proxied": ${PROXIED}
-}
-EOF
-}
-
-generate_post_data_txt_record()
-{
-    local name=$1
-    local content=$2
-    cat <<EOF
-{
-    "name": "${name}",
-    "ttl": 120,
-    "type": "TXT",
-    "comment": "",
-    "content": "$content",
-    "proxied": ${PROXIED}
-}
-EOF
-}
-
-generate_post_data_srv_record()
-{
-    local name=$1
-    local port=$2
-    cat <<EOF
-{
-    "name": "${name}",
-    "ttl": 120,
-    "type": "SRV",
-    "comment": "",
-    "data": {
-        "port": $port,
-        "priority": 0,
-        "target": "${CHATMAIL_FULL_DNS_NAME}",
-        "weight": 1
-    },
-    "proxied": ${PROXIED}
-}
-EOF
-}
-
-check_variables
-
-# A records
-create_record "$(generate_post_data_a_record "$CHATMAIL_FULL_DNS_NAME" "A")"
-create_record "$(generate_post_data_a_record "*.$CHATMAIL_FULL_DNS_NAME" "A")"
-
-# AAAA records
-if [ $IPV6_ENABLED = true ]; then # note: I don't have an IPv6 address, so this part hasn't been tested!
-    create_record "$(generate_post_data_a_record "$CHATMAIL_FULL_DNS_NAME" "AAAA")"
-    # create_record "$(generate_post_data_a_record "*.$CHATMAIL_FULL_DNS_NAME" "AAAA")"
-fi
-
-# CNAME records
-create_record "$(generate_post_data_cname_record "mta-sts.$CHATMAIL_FULL_DNS_NAME")"
-create_record "$(generate_post_data_cname_record "www.$CHATMAIL_FULL_DNS_NAME")"
-
-# MX records
-create_record "$(generate_post_data_mx_record "$CHATMAIL_FULL_DNS_NAME")"
-
-# TXT records
-create_record "$(generate_post_data_txt_record "$CHATMAIL_FULL_DNS_NAME" '\"v=spf1 a ~all\"')"
-create_record "$(generate_post_data_txt_record "_dmarc.$CHATMAIL_FULL_DNS_NAME" '\"v=DMARC1;p=reject;adkim=s;aspf=s\"')"
-create_record "$(generate_post_data_txt_record "_adsp._domainkey.$CHATMAIL_FULL_DNS_NAME" '\"dkim=discardable\"')"
-create_record "$(generate_post_data_txt_record "opendkim._domainkey.$CHATMAIL_FULL_DNS_NAME" '\"v=DKIM1;k=rsa;p=;s=email;t=s\"')"
-create_record "$(generate_post_data_txt_record "_mta-sts.$CHATMAIL_FULL_DNS_NAME" '\"v=STSv1; id='"$(date +"%Y%m%d%H%M")"'\"')"
-
-# SRV records
-create_record "$(generate_post_data_srv_record "_imap._tcp.$CHATMAIL_FULL_DNS_NAME" "143")"
-create_record "$(generate_post_data_srv_record "_imaps._tcp.$CHATMAIL_FULL_DNS_NAME" "993")"
-create_record "$(generate_post_data_srv_record "_submission._tcp.$CHATMAIL_FULL_DNS_NAME" "587")"
-create_record "$(generate_post_data_srv_record "_submissions._tcp.$CHATMAIL_FULL_DNS_NAME" "465")"