From 4dbb19db466f7767f7352866f9a445cfae0b5419 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Wed, 10 Jul 2024 02:42:33 +0200 Subject: [PATCH] delete users from mailboxes_dir --- .../src/chatmaild/delete_inactive_users.py | 55 ++++++++++++++++--- chatmaild/src/chatmaild/doveauth.py | 13 +---- chatmaild/src/chatmaild/metadata.py | 10 ---- .../tests/test_delete_inactive_users.py | 24 +++++++- .../src/chatmaild/tests/test_metadata.py | 35 ++++-------- 5 files changed, 80 insertions(+), 57 deletions(-) diff --git a/chatmaild/src/chatmaild/delete_inactive_users.py b/chatmaild/src/chatmaild/delete_inactive_users.py index 4d23d2e6..9a9a81e9 100644 --- a/chatmaild/src/chatmaild/delete_inactive_users.py +++ b/chatmaild/src/chatmaild/delete_inactive_users.py @@ -2,28 +2,65 @@ Remove inactive users """ +import os import shutil import sys import time +from pathlib import Path from .config import read_config from .database import Database -from .doveauth import iter_userdb_lastlogin_before + +LAST_LOGIN = "last-login" -def delete_inactive_users(db, config, CHUNK=100): +def write_last_login_to_userdir(userdir, timestamp): + target = userdir.joinpath(LAST_LOGIN) + timestamp = int(timestamp // 86400 * 86400) + try: + st = target.stat() + except FileNotFoundError: + # only happens on initial login + userdir.mkdir(exist_ok=True) + target.write_text("") + os.utime(target, (timestamp, timestamp)) + else: + if st.st_mtime < timestamp: + os.utime(target, (timestamp, timestamp)) + + +def get_last_login_from_userdir(userdir): + target = userdir.joinpath(LAST_LOGIN) + try: + return int(target.stat().st_mtime) + except FileNotFoundError: + target.write_text("") + timestamp = int(time.time() // 86400 * 86400) + os.utime(target, (timestamp, timestamp)) + return timestamp + + +def delete_inactive_users(db, config, chunksize=100): cutoff_date = time.time() - config.delete_inactive_users_after * 86400 + pending = [] - old_users = iter_userdb_lastlogin_before(db, cutoff_date) - chunks = (old_users[i : i + CHUNK] for i in range(0, len(old_users), CHUNK)) - for sublist in chunks: - for user in sublist: - user_mail_dir = config.get_user_maildir(user) - shutil.rmtree(user_mail_dir, ignore_errors=True) + def remove(userdir): + shutil.rmtree(userdir, ignore_errors=True) + pending.append(userdir.name) + if len(pending) > chunksize: + clear_pending() + def clear_pending(): with db.write_transaction() as conn: - for user in sublist: + for user in pending: conn.execute("DELETE FROM users WHERE addr = ?", (user,)) + pending[:] = [] + + for userdir in Path(config.mailboxes_dir).iterdir(): + if get_last_login_from_userdir(userdir) < cutoff_date: + remove(userdir) + + clear_pending() def main(): diff --git a/chatmaild/src/chatmaild/doveauth.py b/chatmaild/src/chatmaild/doveauth.py index 594822d9..a759047c 100644 --- a/chatmaild/src/chatmaild/doveauth.py +++ b/chatmaild/src/chatmaild/doveauth.py @@ -123,18 +123,7 @@ def lookup_passdb(db, config: Config, user, cleartext_password): def iter_userdb(db) -> list: """Get a list of all user addresses.""" with db.read_connection() as conn: - rows = conn.execute( - "SELECT addr from users", - ).fetchall() - return [x[0] for x in rows] - - -def iter_userdb_lastlogin_before(db, cutoff_date): - """Get a list of users where last login was before cutoff_date.""" - with db.read_connection() as conn: - rows = conn.execute( - "SELECT addr FROM users WHERE last_login < ?", (cutoff_date,) - ).fetchall() + rows = conn.execute("SELECT addr from users").fetchall() return [x[0] for x in rows] diff --git a/chatmaild/src/chatmaild/metadata.py b/chatmaild/src/chatmaild/metadata.py index 40ebafec..5d886cb8 100644 --- a/chatmaild/src/chatmaild/metadata.py +++ b/chatmaild/src/chatmaild/metadata.py @@ -18,16 +18,6 @@ class Metadata: def get_metadata_dict(self, addr): return FileDict(self.vmail_dir / addr / "metadata.json") - def write_login_timestamp(self, addr, timestamp): - # day resolution is enough for timestamp - timestamp = int(timestamp) // 86400 * 86400 - target_file = self.vmail_dir.joinpath(addr, "last-login") - try: - target_file.write_text(str(timestamp)) - except FileNotFoundError: - target_file.parent.mkdir() - target_file.write_text(str(timestamp)) - def add_token_to_addr(self, addr, token): with self.get_metadata_dict(addr).modify() as data: tokens = data.setdefault(self.DEVICETOKEN_KEY, []) diff --git a/chatmaild/src/chatmaild/tests/test_delete_inactive_users.py b/chatmaild/src/chatmaild/tests/test_delete_inactive_users.py index 0cdc163f..2a7334e2 100644 --- a/chatmaild/src/chatmaild/tests/test_delete_inactive_users.py +++ b/chatmaild/src/chatmaild/tests/test_delete_inactive_users.py @@ -1,19 +1,37 @@ import time -from chatmaild.delete_inactive_users import delete_inactive_users +from chatmaild.delete_inactive_users import ( + delete_inactive_users, + get_last_login_from_userdir, + write_last_login_to_userdir, +) from chatmaild.doveauth import lookup_passdb -def test_remove_stale_users(db, example_config): +def test_login_timestamps(tmp_path): + userdir = tmp_path.joinpath("someuser") + userdir.mkdir() + write_last_login_to_userdir(userdir, timestamp=100000) + assert get_last_login_from_userdir(userdir) == 86400 + + write_last_login_to_userdir(userdir, timestamp=200000) + assert get_last_login_from_userdir(userdir) == 86400 * 2 + + write_last_login_to_userdir(userdir, timestamp=200000) + assert get_last_login_from_userdir(userdir) == 86400 * 2 + + +def test_delete_inactive_users(db, example_config): new = time.time() old = new - (example_config.delete_inactive_users_after * 86400) - 1 def create_user(addr, last_login): - lookup_passdb(db, example_config, addr, "q9mr3faue", last_login=last_login) + lookup_passdb(db, example_config, addr, "q9mr3faue") md = example_config.get_user_maildir(addr) md.mkdir(parents=True) md.joinpath("cur").mkdir() md.joinpath("cur", "something").mkdir() + write_last_login_to_userdir(md, timestamp=last_login) # create some stale and some new accounts to_remove = [] diff --git a/chatmaild/src/chatmaild/tests/test_metadata.py b/chatmaild/src/chatmaild/tests/test_metadata.py index de5fa7a3..544b1e65 100644 --- a/chatmaild/src/chatmaild/tests/test_metadata.py +++ b/chatmaild/src/chatmaild/tests/test_metadata.py @@ -3,6 +3,7 @@ import time import pytest import requests +from chatmaild.delete_inactive_users import get_last_login_from_userdir from chatmaild.metadata import ( Metadata, MetadataDictProxy, @@ -92,10 +93,10 @@ def test_notifier_remove_without_set(metadata, testaddr): assert not metadata.get_tokens_for_addr(testaddr) -<<<<<<< HEAD def test_handle_dovecot_request_lookup_fails(dictproxy, testaddr): res = dictproxy.handle_dovecot_request(f"Lpriv/123/chatmail\t{testaddr}") -======= + assert res == "N\n" + def test_metadata_login_timestamp(metadata, testaddr): timestamp = metadata.vmail_dir.joinpath(testaddr).mkdir() metadata.write_login_timestamp(testaddr, timestamp=100000) @@ -107,14 +108,6 @@ def test_metadata_login_timestamp(metadata, testaddr): assert int(timestamp) == 86400 * 2 -def test_handle_dovecot_request_lookup_fails(notifier, metadata, testaddr): - res = handle_dovecot_request( - f"Lpriv/123/chatmail\t{testaddr}", {}, notifier, metadata - ) ->>>>>>> 317d30f (write last login differently) - assert res == "N\n" - - def test_handle_dovecot_request_happy_path(dictproxy, testaddr, token): metadata = dictproxy.metadata transactions = dictproxy.transactions @@ -151,37 +144,33 @@ def test_handle_dovecot_request_happy_path(dictproxy, testaddr, token): assert queue_item.path.exists() -<<<<<<< HEAD -def test_handle_dovecot_protocol_set_devicetoken(dictproxy): -======= def test_handle_dovecot_request_last_login(notifier, metadata, testaddr, token): - transactions = {} + dictproxy = MetadataDictProxy(notifier=notifier, metadata=metadata) userdir = metadata.vmail_dir.joinpath(testaddr) # set last-login info for user tx = "1111" msg = f"B{tx}\t{testaddr}" - res = handle_dovecot_request(msg, transactions, notifier, metadata) + res = dictproxy.handle_dovecot_request(msg) assert not res - assert transactions == {tx: dict(addr=testaddr, res="O\n")} + assert dictproxy.transactions == {tx: dict(addr=testaddr, res="O\n")} timestamp = int(time.time()) msg = f"S{tx}\tshared/last-login/{testaddr}\t{timestamp}" - res = handle_dovecot_request(msg, transactions, notifier, metadata) + res = dictproxy.handle_dovecot_request(msg) assert not res - assert len(transactions) == 1 - read_timestamp = int(userdir.joinpath("last-login").read_text()) + assert len(dictproxy.transactions) == 1 + read_timestamp = get_last_login_from_userdir(userdir) assert read_timestamp == timestamp // 86400 * 86400 msg = f"C{tx}" - res = handle_dovecot_request(msg, transactions, notifier, metadata) + res = dictproxy.handle_dovecot_request(msg) assert res == "O\n" - assert len(transactions) == 0 + assert len(dictproxy.transactions) == 0 -def test_handle_dovecot_protocol_set_devicetoken(metadata, notifier): ->>>>>>> 317d30f (write last login differently) +def test_handle_dovecot_protocol_set_devicetoken(dictproxy): rfile = io.BytesIO( b"\n".join( [