for last 7 days of messages remove large messages first, then by age

Both dovecot-quota-threshold triggers and the daily expiry routine
will now expunge oldest messages from mailboxes automatically
when the mailbox reaches 75% of max_mailbox_size.
Delta Chat users should not see any warnings (at 80/95 percent) or bounce messages,
and existing over-quota mailboxes should start receiving mails again.

chatmaild/src/chatmaild/config.py

@@ -1,3 +1,4 @@
+import os
 from pathlib import Path
 
 import iniconfig
@@ -46,6 +47,8 @@ class Config:
         )
         self.mtail_address = params.get("mtail_address")
         self.disable_ipv6 = params.get("disable_ipv6", "false").lower() == "true"
+        self.addr_v4 = os.environ.get("CHATMAIL_ADDR_V4", "")
+        self.addr_v6 = os.environ.get("CHATMAIL_ADDR_V6", "")
         self.acme_email = params.get("acme_email", "")
         self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true"
         self.imap_compress = params.get("imap_compress", "false").lower() == "true"

chatmaild/src/chatmaild/expire.py

@@ -99,19 +99,31 @@ def scan_mailbox_messages(mbox):
     return messages
 
 
+# Within this window, large messages are deleted before small ones.
+DELETE_LARGE_FIRST_DAYS = 7
+
+
 def expire_to_target(mbox, target_bytes):
+    cutoff = time.time() - DELETE_LARGE_FIRST_DAYS * 86400
     messages = scan_mailbox_messages(mbox)
+
+    def sort_key(msg):
+        # prio 0: Older than cutoff -> remove oldest first
+        if msg.mtime < cutoff:
+            return (0, msg.mtime)
+
+        # prio 1: more recent than cutoff, large -> remove largest first
+        if msg.quota_size > 200000:
+            return (1, -msg.quota_size, msg.mtime)
+
+        # prio 2: more recent than cutoff, small -> remove oldest first
+        return (2, msg.mtime)
+
     total_size = sum(m.quota_size for m in messages)
-    # Keep recent 24 hours of messages protected from expiry because
-    # likely something is wrong with interactions on that address
-    # and quota-full signal can help the address owner's device to notice it
-    undeletable_messages_cutoff = time.time() - (3600 * 24)
     removed = 0
-    for entry in sorted(messages):
+    for entry in sorted(messages, key=sort_key):
         if total_size <= target_bytes:
             break
-        if entry.mtime > undeletable_messages_cutoff:
-            break
         (mbox / entry.path).unlink(missing_ok=True)
         total_size -= entry.quota_size
         removed += 1

chatmaild/src/chatmaild/tests/test_expire.py

@@ -1,6 +1,7 @@
 import itertools
 import os
 import random
+import shutil
 import time
 from datetime import datetime
 from fnmatch import fnmatch
@@ -233,15 +234,38 @@ def test_parse_dovecot_filename():
 def test_expire_to_target(tmp_path):
     _create_message(tmp_path, "cur", MB, days_old=10, disk_size=100)
     _create_message(tmp_path, "new", MB, days_old=5)
-    _create_message(tmp_path, "cur", MB, days_old=0)  # undeletable (<1 hour)
+    assert len(scan_mailbox_messages(tmp_path)) == 2
-    assert len(scan_mailbox_messages(tmp_path)) == 3
     # removes oldest first, uses S= size not disk size
     removed = expire_to_target(tmp_path, MB)
-    assert removed == 2
+    assert removed == 1
+    assert len(scan_mailbox_messages(tmp_path)) == 1
+
+
+def test_expire_to_target_prioritization(tmp_path):
+    def create_messages():
+        for sub in ("cur", "new"):
+            if (tmp_path / sub).exists():
+                shutil.rmtree(tmp_path / sub)
+        # prio 0: older than 7 days
+        _create_message(tmp_path, "cur", 5 * MB, days_old=10)
+        # prio 1: last 7 days, large (>200KB)
+        _create_message(tmp_path, "cur", 5 * MB, days_old=1)
+        # prio 2: last 7 days, small
+        _create_message(tmp_path, "cur", 1000, days_old=2)
+
+    # Shrink to 6MB: only the old message (prio 0) is removed.
+    create_messages()
+    assert expire_to_target(tmp_path, 6 * MB) == 1
+    msgs = scan_mailbox_messages(tmp_path)
+    assert len(msgs) == 2
+    assert all(m.mtime > time.time() - 7 * 86400 for m in msgs)
+
+    # Shrink to 1KB: old and recent-large removed, small survives.
+    create_messages()
+    assert expire_to_target(tmp_path, 1024) == 2
     msgs = scan_mailbox_messages(tmp_path)
     assert len(msgs) == 1
-    # the surviving message is the fresh undeletable one
+    assert msgs[0].quota_size == 1000
-    assert msgs[0].mtime > time.time() - 3600
 
 
 def test_quota_expire_main(tmp_path, capsys):

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -93,9 +93,7 @@ def run_cmd(args, out):
     strict_tls = args.config.tls_cert_mode == "acme"
     if not args.dns_check_disabled:
         remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain)
-        if not dns.check_initial_remote_data(
+        if not dns.check_initial_remote_data(remote_data, strict_tls=strict_tls, print=out.red):
-            remote_data, strict_tls=strict_tls, print=out.red
-        ):
             return 1
 
     env = os.environ.copy()
@@ -103,6 +101,9 @@ def run_cmd(args, out):
     env["CHATMAIL_WEBSITE_ONLY"] = "True" if args.website_only else ""
     env["CHATMAIL_DISABLE_MAIL"] = "True" if args.disable_mail else ""
     env["CHATMAIL_REQUIRE_IROH"] = "True" if require_iroh else ""
+    if not args.dns_check_disabled:
+        env["CHATMAIL_ADDR_V4"] = remote_data.get("A") or ""
+        env["CHATMAIL_ADDR_V6"] = remote_data.get("AAAA") or ""
     deploy_path = importlib.resources.files(__package__).joinpath("run.py").resolve()
     pyinf = "pyinfra --dry" if args.dry_run else "pyinfra"
 
@@ -118,11 +119,7 @@ def run_cmd(args, out):
         out.check_call(cmd, env=env)
         if args.website_only:
             out.green("Website deployment completed.")
-        elif (
+        elif not args.dns_check_disabled and strict_tls and not remote_data["acme_account_url"]:
-            not args.dns_check_disabled
-            and strict_tls
-            and not remote_data["acme_account_url"]
-        ):
             out.red("Deploy completed but letsencrypt not configured")
             out.red("Run 'cmdeploy run' again")
         else:

cmdeploy/src/cmdeploy/deployers.py

@@ -591,17 +591,11 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
         return
 
     # Check if mtail_address interface is available (if configured)
-    if config.mtail_address and config.mtail_address not in (
+    if config.mtail_address and config.mtail_address not in ('127.0.0.1', '::1', 'localhost'):
-        "127.0.0.1",
-        "::1",
-        "localhost",
-    ):
         ipv4_addrs = host.get_fact(hardware.Ipv4Addrs)
         all_addresses = [addr for addrs in ipv4_addrs.values() for addr in addrs]
         if config.mtail_address not in all_addresses:
-            Out().red(
+            Out().red(f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n")
-                f"Deploy failed: mtail_address {config.mtail_address} is not available (VPN up?).\n"
-            )
             exit(1)
 
     if not is_in_container():

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -20,30 +20,12 @@ DOVECOT_ARCHIVE_VERSION = "2.3.21+dfsg1-3"
 DOVECOT_PACKAGE_VERSION = f"1:{DOVECOT_ARCHIVE_VERSION}"
 
 DOVECOT_SHA256 = {
-    (
+    ("core", "amd64"): "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d",
-        "core",
+    ("core", "arm64"): "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9",
-        "amd64",
+    ("imapd", "amd64"): "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86",
-    ): "dd060706f52a306fa863d874717210b9fe10536c824afe1790eec247ded5b27d",
+    ("imapd", "arm64"): "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f",
-    (
+    ("lmtpd", "amd64"): "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab",
-        "core",
+    ("lmtpd", "arm64"): "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f",
-        "arm64",
-    ): "e7548e8a82929722e973629ecc40fcfa886894cef3db88f23535149e7f730dc9",
-    (
-        "imapd",
-        "amd64",
-    ): "8d8dc6fc00bbb6cdb25d345844f41ce2f1c53f764b79a838eb2a03103eebfa86",
-    (
-        "imapd",
-        "arm64",
-    ): "178fa877ddd5df9930e8308b518f4b07df10e759050725f8217a0c1fb3fd707f",
-    (
-        "lmtpd",
-        "amd64",
-    ): "2f69ba5e35363de50962d42cccbfe4ed8495265044e244007d7ccddad77513ab",
-    (
-        "lmtpd",
-        "arm64",
-    ): "89f52fb36524f5877a177dff4a713ba771fd3f91f22ed0af7238d495e143b38f",
 }
 
 
@@ -79,7 +61,11 @@ class DovecotDeployer(Deployer):
                 self.need_restart = True
         files.put(
             name="Pin dovecot packages to block Debian dist-upgrades",
-            src=io.StringIO("Package: dovecot-*\nPin: version *\nPin-Priority: -1\n"),
+            src=io.StringIO(
+                "Package: dovecot-*\n"
+                "Pin: version *\n"
+                "Pin-Priority: -1\n"
+            ),
             dest="/etc/apt/preferences.d/pin-dovecot",
             user="root",
             group="root",
@@ -98,7 +84,7 @@ class DovecotDeployer(Deployer):
         if not self.disable_mail and not self.need_restart:
             stale = host.get_fact(
                 Command,
-                "pid=$(systemctl show -p MainPID --value dovecot.service 2>/dev/null);"
+                'pid=$(systemctl show -p MainPID --value dovecot.service 2>/dev/null);'
                 ' [ "${pid:-0}" != "0" ] && readlink "/proc/$pid/exe" 2>/dev/null | grep -q "(deleted)"'
                 " && echo STALE || true",
             )

cmdeploy/src/cmdeploy/postfix/main.cf.j2

@@ -69,6 +69,15 @@ mynetworks = 127.0.0.0/8
 {% else %}
 mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128
 {% endif %}
+{% if config.addr_v4 %}
+smtp_bind_address = {{ config.addr_v4 }}
+{% endif %}
+{% if config.addr_v6 %}
+smtp_bind_address6 = {{ config.addr_v6 }}
+{% endif %}
+{% if config.addr_v4 or config.addr_v6 %}
+smtp_bind_address_enforce = yes
+{% endif %}
 mailbox_size_limit = 0
 message_size_limit = {{config.max_message_size}}
 recipient_delimiter = +

cmdeploy/src/cmdeploy/selfsigned/deployer.py

@@ -12,27 +12,15 @@ def openssl_selfsigned_args(domain, cert_path, key_path, days=36500):
     ``www.<domain>`` and ``mta-sts.<domain>``.
     """
     return [
-        "openssl",
+        "openssl", "req", "-x509",
-        "req",
+        "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:P-256",
-        "-x509",
+        "-noenc", "-days", str(days),
-        "-newkey",
+        "-keyout", str(key_path),
-        "ec",
+        "-out", str(cert_path),
-        "-pkeyopt",
+        "-subj", f"/CN={domain}",
-        "ec_paramgen_curve:P-256",
-        "-noenc",
-        "-days",
-        str(days),
-        "-keyout",
-        str(key_path),
-        "-out",
-        str(cert_path),
-        "-subj",
-        f"/CN={domain}",
         # Mark as end-entity cert so it cannot be used as a CA to sign others.
-        "-addext",
+        "-addext", "basicConstraints=critical,CA:FALSE",
-        "basicConstraints=critical,CA:FALSE",
+        "-addext", "extendedKeyUsage=serverAuth,clientAuth",
-        "-addext",
-        "extendedKeyUsage=serverAuth,clientAuth",
         "-addext",
         f"subjectAltName=DNS:{domain},DNS:www.{domain},DNS:mta-sts.{domain}",
     ]
@@ -54,9 +42,7 @@ class SelfSignedTlsDeployer(Deployer):
 
     def configure(self):
         args = openssl_selfsigned_args(
-            self.mail_domain,
+            self.mail_domain, self.cert_path, self.key_path,
-            self.cert_path,
-            self.key_path,
         )
         cmd = shlex.join(args)
         server.shell(

cmdeploy/src/cmdeploy/tests/online/test_0_qr.py

@@ -30,15 +30,12 @@ def test_newemail_configure(maildomain, rpc, chatmail_config):
             # set_config_from_qr, so fetch credentials via requests instead
             res = requests.post(f"https://{maildomain}/new", verify=False)
             data = res.json()
-            rpc.add_or_update_transport(
+            rpc.add_or_update_transport(account_id, {
-                account_id,
+                "addr": data["email"],
-                {
+                "password": data["password"],
-                    "addr": data["email"],
+                "imapServer": maildomain,
-                    "password": data["password"],
+                "smtpServer": maildomain,
-                    "imapServer": maildomain,
+                "certificateChecks": "acceptInvalidCertificates",
-                    "smtpServer": maildomain,
+            })
-                    "certificateChecks": "acceptInvalidCertificates",
-                },
-            )
         else:
             rpc.add_transport_from_qr(account_id, url)

cmdeploy/src/cmdeploy/tests/plugin.py

@@ -417,12 +417,9 @@ class Remote:
         getjournal = "journalctl -f" if not logcmd else logcmd
         print(self.sshdomain)
         match self.sshdomain:
-            case "@local":
+            case "@local": command = []
-                command = []
+            case "localhost": command = []
-            case "localhost":
+            case _: command = ["ssh", f"root@{self.sshdomain}"]
-                command = []
-            case _:
-                command = ["ssh", f"root@{self.sshdomain}"]
         [command.append(arg) for arg in getjournal.split()]
         popen = subprocess.Popen(
             command,

cmdeploy/src/cmdeploy/tests/test_dovecot_deployer.py

@@ -23,7 +23,8 @@ def make_host(*fact_pairs):
         if cls not in facts:
             registered = ", ".join(c.__name__ for c in facts)
             raise LookupError(
-                f"unexpected get_fact({cls.__name__}); only registered: {registered}"
+                f"unexpected get_fact({cls.__name__}); "
+                f"only registered: {registered}"
             )
         return facts[cls]
 

doc/README.md

@@ -4,14 +4,12 @@
 
 You can use the `make` command and `make html` to build web pages. 
 
-You need a Python environment with `sphinx` and other
+You need a Python environment where the following install was excuted: 
-dependencies, you can create it by running `scripts/initenv.sh`
+
-from the repository root.
+    pip install furo sphinx-autobuild
 
 To develop/change documentation, you can then do: 
 
-    . venv/bin/activate
-    cd doc
     make auto 
 
 A page will open at https://127.0.0.1:8000/ serving the docs and it will 

doc/source/index.rst

@@ -16,6 +16,5 @@ Contributions and feedback welcome through the https://github.com/chatmail/relay
     proxy
     migrate
     overview
-    reverse_dns
     related
     faq

doc/source/reverse_dns.rst

@@ -1,64 +0,0 @@
-Configuring reverse DNS
-=======================
-
-Some email servers reject the emails
-if they don't pass `FCrDNS`_ check, also known as `iprev`_ check.
-
-.. _FCrDNS: https://en.wikipedia.org/wiki/Forward-confirmed_reverse_DNS
-.. _iprev: https://datatracker.ietf.org/doc/html/rfc8601#section-3
-
-Passing the check requires that the IP address that email is sent from
-should have a ``PTR`` record pointing to the domain name of the server,
-and domain name record should have an ``A/AAAA`` record
-pointing to the IP address.
-
-Modern email relies on DKIM and SPF for authentication,
-while iprev check exists for
-`historical reasons <https://datatracker.ietf.org/doc/html/draft-ietf-dnsop-reverse-mapping-considerations-06#section-2.1>`_.
-Chatmail relays don't resolve ``PTR`` records,
-so you can ignore this section if configuring ``PTR`` records
-is difficult and federation with legacy email servers that don't accept
-valid DKIM signature for authentication is not important.
-
-Multi-homed setups
-------------------
-
-If you have a server with multiple IP addresses,
-also known as multi-homed setup,
-and don't publish all IP addresses in DNS,
-you need to make sure you are using
-the published address when making outgoing connections.
-
-For example, your server may have a static IP
-address, and a so-called Floating IP or Virtual IP
-that can be moved between servers in case of
-migration or for failover.
-By using Floating IP you can avoid downtime
-and keep the IP address reputation
-for destinatinons that rely on IP reputation and IP blocklists.
-In this case you will only publish
-the Floating IP to DNS and only use the static IP
-to SSH into the server.
-
-If you have such setup, make sure that
-you not only set ``PTR`` records for the Floating IP,
-but make outgoing connections using the Floating IP.
-Otherwise reverse DNS check succeed,
-but forward check making sure your domain name points
-to the IP address will fail.
-Such setup is indistinguishable from someone
-setting IP address ``PTR`` with the domain they don't own
-and as a result don't succeed.
-
-On Linux you can configure source IP address with ``ip route`` command,
-for example:
-::
-
-  ip route change default via <default-gateway> dev eth0 src <source-address>
-
-Make sure to persist the change after verifying it is working.
-You can check what your outgoing IP address is
-with ``curl icanhazip.com``.
-Check both the IPv4 and IPv6 addresses.
-For IPv4 address use ``curl ipv4.icanhazip.com`` or ``curl -4 icanhazip.com``
-and similarly for IPv6 if you have it.