Add script to remove old seen messages

Apparently this causes outlook.com messages to be rejected
even though they don't use `l=` tag.

CHANGELOG.md

@@ -2,8 +2,8 @@
 
 ## untagged
 
-- Reject DKIM signatures that do not cover the whole message body.
-  ([#321](https://github.com/deltachat/chatmail/pull/321))
+- Test and fix for attempts to create inadmissible accounts 
+  ([#333](https://github.com/deltachat/chatmail/pull/321))
 
 - check that OpenPGP has only PKESK, SKESK and SEIPD packets
   ([#323](https://github.com/deltachat/chatmail/pull/323),
@@ -12,6 +12,15 @@
 - improve filtermail checks for encrypted messages and drop support for unencrypted MDNs
   ([#320](https://github.com/deltachat/chatmail/pull/320))
 
+- replace `bash` with `/bin/sh`
+  ([#334](https://github.com/deltachat/chatmail/pull/334))
+
+- Increase number of logged in IMAP sessions to 50000
+  ([#335](https://github.com/deltachat/chatmail/pull/335))
+
+- filtermail: do not allow ASCII armor without actual payload
+  ([#325](https://github.com/deltachat/chatmail/pull/325))
+
 ## 1.3.0 - 2024-06-06
 
 - don't check necessary DNS records on cmdeploy init anymore

chatmaild/pyproject.toml

@@ -26,7 +26,6 @@ chatmail-metadata = "chatmaild.metadata:main"
 filtermail = "chatmaild.filtermail:main"
 echobot = "chatmaild.echo:main"
 chatmail-metrics = "chatmaild.metrics:main"
-rm_accounts = "chatmaild.rm_accounts:main"
 
 [project.entry-points.pytest11]
 "chatmaild.testplugin" = "chatmaild.tests.plugin"

chatmaild/src/chatmaild/config.py

@@ -13,7 +13,6 @@ class Config:
         self.max_user_send_per_minute = int(params["max_user_send_per_minute"])
         self.max_mailbox_size = params["max_mailbox_size"]
         self.delete_mails_after = params["delete_mails_after"]
-        self.delete_accounts_after = int(params["delete_accounts_after"])
         self.username_min_length = int(params["username_min_length"])
         self.username_max_length = int(params["username_max_length"])
         self.password_min_length = int(params["password_min_length"])

chatmaild/src/chatmaild/doveauth.py

@@ -60,6 +60,7 @@ def is_allowed_to_create(config: Config, user, cleartext_password) -> bool:
             config.username_min_length,
             config.username_max_length,
         )
+        return False
 
     return True
 
@@ -106,8 +107,7 @@ def lookup_passdb(db, config: Config, user, cleartext_password):
         if userdata:
             # Update last login time.
             conn.execute(
-                "UPDATE users SET last_login=? WHERE addr=?",
-                (int(time.time() // 86400), user),
+                "UPDATE users SET last_login=? WHERE addr=?", (int(time.time()), user)
             )
 
             userdata["home"] = f"/home/vmail/mail/{config.mail_domain}/{user}"

chatmaild/src/chatmaild/filtermail.py

@@ -70,6 +70,9 @@ def check_openpgp_payload(payload: bytes):
             # Symmetric-Key Encrypted Session Key Packet (SKESK)
             return False
 
+    if i == 0:
+        return False
+
     if i > len(payload):
         # Payload is truncated.
         return False

chatmaild/src/chatmaild/ini/chatmail.ini.f

@@ -20,9 +20,6 @@ max_mailbox_size = 100M
 # days after which mails are unconditionally deleted
 delete_mails_after = 20
 
-# days after which accounts are deleted if nobody logged in
-delete_accounts_after = 25
-
 # minimum length a username must have
 username_min_length = 9
 

chatmaild/src/chatmaild/rm_accounts.py

@@ -1,37 +0,0 @@
-import sys
-import time
-import shutil
-from pathlib import Path
-
-from .config import read_config
-from .database import Database
-
-
-def remove_users(db: Database, cutoff_date: int):
-    with db.write_transaction() as conn:
-        delete_query = "DELETE FROM users WHERE last_login <?"
-        conn.execute(delete_query, (cutoff_date))
-
-
-def remove_user_data(db: Database, cutoff_date: int, vmail_basedir: Path):
-    """Collects all users where last_login < cutoff_date and deletes corresponding directories."""
-
-    with db.write_transaction() as conn:
-        select_query = "SELECT user FROM users WHERE last_login <?"
-        cursor = conn.execute(select_query, (cutoff_date,))
-        usernames = cursor.fetchall()
-
-        for username in usernames:
-            user_dir = vmail_basedir / username[0]
-            if user_dir.exists() and user_dir.is_dir():
-                shutil.rmtree(user_dir, ignore_errors=True)
-                print(f"Deleted directory: {user_dir}")
-
-
-def main():
-    db = Database(sys.argv[2])
-    config = read_config(sys.argv[3])
-    today = int(time.time() // 86400)
-
-    cutoff_date = today - config.delete_accounts_after
-    remove_user_data(db, cutoff_date)

chatmaild/src/chatmaild/tests/test_doveauth.py

@@ -11,8 +11,10 @@ from chatmaild.doveauth import (
     get_user_data,
     handle_dovecot_protocol,
     handle_dovecot_request,
+    is_allowed_to_create,
     lookup_passdb,
 )
+from chatmaild.newemail import create_newemail_dict
 
 
 def test_basic(db, example_config):
@@ -25,6 +27,20 @@ def test_basic(db, example_config):
     assert data == data2
 
 
+def test_invalid_username_length(example_config):
+    config = example_config
+    config.username_min_length = 6
+    config.username_max_length = 10
+    password = create_newemail_dict(config)["password"]
+    assert not is_allowed_to_create(config, f"a1234@{config.mail_domain}", password)
+    assert is_allowed_to_create(config, f"012345@{config.mail_domain}", password)
+    assert is_allowed_to_create(config, f"0123456@{config.mail_domain}", password)
+    assert is_allowed_to_create(config, f"0123456789@{config.mail_domain}", password)
+    assert not is_allowed_to_create(
+        config, f"0123456789x@{config.mail_domain}", password
+    )
+
+
 def test_dont_overwrite_password_on_wrong_login(db, example_config):
     """Test that logging in with a different password doesn't create a new user"""
     res = lookup_passdb(

chatmaild/src/chatmaild/tests/test_filtermail.py

@@ -167,3 +167,19 @@ UN4fiB0KR9JyG2ayUdNJVkXZSZLnHyRgiaadlpUo16LVvw==\r
 """
 
     assert check_armored_payload(payload) == True
+
+    payload = """-----BEGIN PGP MESSAGE-----\r
+\r
+HELLOWORLD
+-----END PGP MESSAGE-----\r
+\r
+"""
+    assert check_armored_payload(payload) == False
+
+    payload = """-----BEGIN PGP MESSAGE-----\r
+\r
+=njUN
+-----END PGP MESSAGE-----\r
+\r
+"""
+    assert check_armored_payload(payload) == False

cmdeploy/src/cmdeploy/__init__.py

@@ -361,6 +361,14 @@ def _configure_dovecot(config: Config, debug: bool = False) -> bool:
         config=config,
     )
 
+    files.put(
+            src=importlib.resources.files(__package__).joinpath("dovecot/remove-seen.py"),
+            dest="/usr/local/bin/remove-seen.py",
+            user="root",
+            group="root",
+            mode="755"
+    )
+
     # as per https://doc.dovecot.org/configuration_manual/os/
     # it is recommended to set the following inotify limits
     for name in ("max_user_instances", "max_user_watches"):
@@ -649,3 +657,5 @@ def deploy_chatmail(config_path: Path) -> None:
         name="Ensure cron is installed",
         packages=["cron"],
     )
+
+

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -19,6 +19,22 @@ mail_debug = yes
 #   master: Warning: service(stats): client_limit (1000) reached, client connections are being dropped
 default_client_limit = 20000
 
+# Increase number of logged in IMAP connections.
+# Each connection is handled by a separate `imap` process.
+# `imap` process should have `client_limit=1` as described in
+# <https://doc.dovecot.org/configuration_manual/service_configuration/#service-limits>
+# so each logged in IMAP session will need its own `imap` process.
+#
+# If this limit is reached,
+# users will fail to LOGIN as `imap-login` process
+# will accept them logging in but fail to transfer logged in
+# connection to `imap` process until someone logs out and
+# the following warning will be logged:
+#   Warning: service(imap): process_limit (1024) reached, client connections are being dropped
+service imap {
+  process_limit = 50000
+}
+
 mail_server_admin = mailto:root@{{ config.mail_domain }}
 mail_server_comment = Chatmail server
 

cmdeploy/src/cmdeploy/dovecot/expunge.cron.j2

@@ -9,3 +9,4 @@
 2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 2 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -path '*/.*/tmp/*' -mtime +{{ config.delete_mails_after }} -type f -delete
 3 0 * * * vmail find /home/vmail/mail/{{ config.mail_domain }} -name 'maildirsize' -type f -delete
+4 0 * * * vmail /usr/local/bin/remove-seen.py /home/vmail/mail/{{ config.mail_domain }}

cmdeploy/src/cmdeploy/dovecot/remove-seen.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+"""Remove seen messages that are older than two days
+if maildir has more than 80 MB of messages."""
+import sys
+import time
+from pathlib import Path
+
+
+def getdirsize(path):
+    return sum(f.stat().st_size for f in path.glob("**/*") if f.is_file())
+
+
+def parse_dovecot_seen(path):
+    return "S" in path.name.split(":2,")[-1]
+
+
+def main():
+    now = time.time()
+
+    mailhome = Path(sys.argv[1])
+
+    for p in mailhome.iterdir():
+        dirsize = getdirsize(p / "cur") + getdirsize(p / "new")
+        if dirsize < 80000000:
+            continue
+
+        removed_bytes = 0
+        for mailpath in (p / "cur").iterdir():
+            seen = parse_dovecot_seen(mailpath)
+            stat = mailpath.stat()
+            size = stat.st_size
+            if seen and now > stat.st_mtime + 2 * 24 * 3600:
+                removed_bytes += size
+                mailpath.unlink(missing_ok=True)
+
+        if removed_bytes > 0:
+            (p / "maildirsize").unlink(missing_ok=True)
+
+
+if __name__ == "__main__":
+    main()

cmdeploy/src/cmdeploy/opendkim/final.lua

@@ -19,11 +19,7 @@ for i = 1, nsigs do
 	-- Any valid signature that was not ignored like this
 	-- means the message is acceptable.
 	if sigres == 0 then
-		-- Do not accept the signature if it does not cover the whole body
-		-- of the message by using `l=` tag.
-		if odkim.sig_canonlength(ctx, sig) < odkim.sig_bodylength(ctx, sig) then
-			return nil
-		end
+		return nil
 	end	
 end
 

scripts/cmdeploy

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/sh
 #
 # Wrapper for cmdelpoy to run it in activated virtualenv.
 set -e

scripts/dovecot/build-obs.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
 
 # Install dependencies
 echo "Installing dependencies for this script:"

scripts/initenv.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
 set -e
 python3 -m venv --upgrade-deps venv
 

www/src/privacy.md

@@ -77,7 +77,7 @@ we process the following data and details:
 - Users can retrieve or delete all stored messages 
   without intervention from the operators using standard IMAP client tools.
 
-### 3.1 Account setup
+### 2.1 Account setup
 
 Creating an account happens in one of two ways on our mail servers: 
 
@@ -98,7 +98,7 @@ Art. 6 (1) lit. b GDPR,
 as you have a usage contract with us
 by using our services.
 
-## 3.2 Processing of E-Mail-Messages
+### 2.2 Processing of E-Mail-Messages
 
 In addition,
 we will process data
@@ -124,7 +124,7 @@ Therefore, limits are enforced:
 
 - message size limits
 
-- any other limit neccessary for the whole server to function in a healthy way
+- any other limit necessary for the whole server to function in a healthy way
   and to prevent abuse.
 
 The processing and use of the above permissions