delete users from mailboxes_dir

chatmaild/src/chatmaild/delete_inactive_users.py

@@ -2,28 +2,65 @@
 Remove inactive users
 """
 
+import os
 import shutil
 import sys
 import time
+from pathlib import Path
 
 from .config import read_config
 from .database import Database
-from .doveauth import iter_userdb_lastlogin_before
+
+LAST_LOGIN = "last-login"
 
 
-def delete_inactive_users(db, config, CHUNK=100):
+def write_last_login_to_userdir(userdir, timestamp):
+    target = userdir.joinpath(LAST_LOGIN)
+    timestamp = int(timestamp // 86400 * 86400)
+    try:
+        st = target.stat()
+    except FileNotFoundError:
+        # only happens on initial login
+        userdir.mkdir(exist_ok=True)
+        target.write_text("")
+        os.utime(target, (timestamp, timestamp))
+    else:
+        if st.st_mtime < timestamp:
+            os.utime(target, (timestamp, timestamp))
+
+
+def get_last_login_from_userdir(userdir):
+    target = userdir.joinpath(LAST_LOGIN)
+    try:
+        return int(target.stat().st_mtime)
+    except FileNotFoundError:
+        target.write_text("")
+        timestamp = int(time.time() // 86400 * 86400)
+        os.utime(target, (timestamp, timestamp))
+        return timestamp
+
+
+def delete_inactive_users(db, config, chunksize=100):
     cutoff_date = time.time() - config.delete_inactive_users_after * 86400
+    pending = []
 
-    old_users = iter_userdb_lastlogin_before(db, cutoff_date)
-    chunks = (old_users[i : i + CHUNK] for i in range(0, len(old_users), CHUNK))
-    for sublist in chunks:
-        for user in sublist:
-            user_mail_dir = config.get_user_maildir(user)
-            shutil.rmtree(user_mail_dir, ignore_errors=True)
+    def remove(userdir):
+        shutil.rmtree(userdir, ignore_errors=True)
+        pending.append(userdir.name)
+        if len(pending) > chunksize:
+            clear_pending()
 
+    def clear_pending():
         with db.write_transaction() as conn:
-            for user in sublist:
+            for user in pending:
                 conn.execute("DELETE FROM users WHERE addr = ?", (user,))
+        pending[:] = []
+
+    for userdir in Path(config.mailboxes_dir).iterdir():
+        if get_last_login_from_userdir(userdir) < cutoff_date:
+            remove(userdir)
+
+    clear_pending()
 
 
 def main():

chatmaild/src/chatmaild/doveauth.py

@@ -123,18 +123,7 @@ def lookup_passdb(db, config: Config, user, cleartext_password):
 def iter_userdb(db) -> list:
     """Get a list of all user addresses."""
     with db.read_connection() as conn:
-        rows = conn.execute(
-            "SELECT addr from users",
-        ).fetchall()
-    return [x[0] for x in rows]
-
-
-def iter_userdb_lastlogin_before(db, cutoff_date):
-    """Get a list of users where last login was before cutoff_date."""
-    with db.read_connection() as conn:
-        rows = conn.execute(
-            "SELECT addr FROM users WHERE last_login < ?", (cutoff_date,)
-        ).fetchall()
+        rows = conn.execute("SELECT addr from users").fetchall()
     return [x[0] for x in rows]
 
 

chatmaild/src/chatmaild/metadata.py

@@ -18,16 +18,6 @@ class Metadata:
     def get_metadata_dict(self, addr):
         return FileDict(self.vmail_dir / addr / "metadata.json")
 
-    def write_login_timestamp(self, addr, timestamp):
-        # day resolution is enough for timestamp
-        timestamp = int(timestamp) // 86400 * 86400
-        target_file = self.vmail_dir.joinpath(addr, "last-login")
-        try:
-            target_file.write_text(str(timestamp))
-        except FileNotFoundError:
-            target_file.parent.mkdir()
-            target_file.write_text(str(timestamp))
-
     def add_token_to_addr(self, addr, token):
         with self.get_metadata_dict(addr).modify() as data:
             tokens = data.setdefault(self.DEVICETOKEN_KEY, [])

chatmaild/src/chatmaild/tests/test_delete_inactive_users.py

@@ -1,19 +1,37 @@
 import time
 
-from chatmaild.delete_inactive_users import delete_inactive_users
+from chatmaild.delete_inactive_users import (
+    delete_inactive_users,
+    get_last_login_from_userdir,
+    write_last_login_to_userdir,
+)
 from chatmaild.doveauth import lookup_passdb
 
 
-def test_remove_stale_users(db, example_config):
+def test_login_timestamps(tmp_path):
+    userdir = tmp_path.joinpath("someuser")
+    userdir.mkdir()
+    write_last_login_to_userdir(userdir, timestamp=100000)
+    assert get_last_login_from_userdir(userdir) == 86400
+
+    write_last_login_to_userdir(userdir, timestamp=200000)
+    assert get_last_login_from_userdir(userdir) == 86400 * 2
+
+    write_last_login_to_userdir(userdir, timestamp=200000)
+    assert get_last_login_from_userdir(userdir) == 86400 * 2
+
+
+def test_delete_inactive_users(db, example_config):
     new = time.time()
     old = new - (example_config.delete_inactive_users_after * 86400) - 1
 
     def create_user(addr, last_login):
-        lookup_passdb(db, example_config, addr, "q9mr3faue", last_login=last_login)
+        lookup_passdb(db, example_config, addr, "q9mr3faue")
         md = example_config.get_user_maildir(addr)
         md.mkdir(parents=True)
         md.joinpath("cur").mkdir()
         md.joinpath("cur", "something").mkdir()
+        write_last_login_to_userdir(md, timestamp=last_login)
 
     # create some stale and some new accounts
     to_remove = []

chatmaild/src/chatmaild/tests/test_metadata.py

@@ -3,6 +3,7 @@ import time
 
 import pytest
 import requests
+from chatmaild.delete_inactive_users import get_last_login_from_userdir
 from chatmaild.metadata import (
     Metadata,
     MetadataDictProxy,
@@ -92,10 +93,10 @@ def test_notifier_remove_without_set(metadata, testaddr):
     assert not metadata.get_tokens_for_addr(testaddr)
 
 
-<<<<<<< HEAD
 def test_handle_dovecot_request_lookup_fails(dictproxy, testaddr):
     res = dictproxy.handle_dovecot_request(f"Lpriv/123/chatmail\t{testaddr}")
-=======
+    assert res == "N\n"
+
 def test_metadata_login_timestamp(metadata, testaddr):
     timestamp = metadata.vmail_dir.joinpath(testaddr).mkdir()
     metadata.write_login_timestamp(testaddr, timestamp=100000)
@@ -107,14 +108,6 @@ def test_metadata_login_timestamp(metadata, testaddr):
     assert int(timestamp) == 86400 * 2
 
 
-def test_handle_dovecot_request_lookup_fails(notifier, metadata, testaddr):
-    res = handle_dovecot_request(
-        f"Lpriv/123/chatmail\t{testaddr}", {}, notifier, metadata
-    )
->>>>>>> 317d30f (write last login differently)
-    assert res == "N\n"
-
-
 def test_handle_dovecot_request_happy_path(dictproxy, testaddr, token):
     metadata = dictproxy.metadata
     transactions = dictproxy.transactions
@@ -151,37 +144,33 @@ def test_handle_dovecot_request_happy_path(dictproxy, testaddr, token):
     assert queue_item.path.exists()
 
 
-<<<<<<< HEAD
-def test_handle_dovecot_protocol_set_devicetoken(dictproxy):
-=======
 def test_handle_dovecot_request_last_login(notifier, metadata, testaddr, token):
-    transactions = {}
+    dictproxy = MetadataDictProxy(notifier=notifier, metadata=metadata)
 
     userdir = metadata.vmail_dir.joinpath(testaddr)
 
     # set last-login info for user
     tx = "1111"
     msg = f"B{tx}\t{testaddr}"
-    res = handle_dovecot_request(msg, transactions, notifier, metadata)
+    res = dictproxy.handle_dovecot_request(msg)
     assert not res
-    assert transactions == {tx: dict(addr=testaddr, res="O\n")}
+    assert dictproxy.transactions == {tx: dict(addr=testaddr, res="O\n")}
 
     timestamp = int(time.time())
     msg = f"S{tx}\tshared/last-login/{testaddr}\t{timestamp}"
-    res = handle_dovecot_request(msg, transactions, notifier, metadata)
+    res = dictproxy.handle_dovecot_request(msg)
     assert not res
-    assert len(transactions) == 1
-    read_timestamp = int(userdir.joinpath("last-login").read_text())
+    assert len(dictproxy.transactions) == 1
+    read_timestamp = get_last_login_from_userdir(userdir)
     assert read_timestamp == timestamp // 86400 * 86400
 
     msg = f"C{tx}"
-    res = handle_dovecot_request(msg, transactions, notifier, metadata)
+    res = dictproxy.handle_dovecot_request(msg)
     assert res == "O\n"
-    assert len(transactions) == 0
+    assert len(dictproxy.transactions) == 0
 
 
-def test_handle_dovecot_protocol_set_devicetoken(metadata, notifier):
->>>>>>> 317d30f (write last login differently)
+def test_handle_dovecot_protocol_set_devicetoken(dictproxy):
     rfile = io.BytesIO(
         b"\n".join(
             [