From 353d3bfb3f46a2ca605998d4a8e4918439639dae Mon Sep 17 00:00:00 2001 From: holger krekel Date: Sun, 21 Jul 2024 18:31:34 +0200 Subject: [PATCH] introduce last-login proxy --- CHANGELOG.md | 4 ++ chatmaild/pyproject.toml | 1 + .../src/chatmaild/delete_inactive_users.py | 39 +---------- chatmaild/src/chatmaild/lastlogin.py | 67 +++++++++++++++++++ .../tests/test_delete_inactive_users.py | 7 +- .../src/chatmaild/tests/test_lastlogin.py | 50 ++++++++++++++ .../src/chatmaild/tests/test_metadata.py | 37 ---------- cmdeploy/src/cmdeploy/__init__.py | 1 + cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 | 2 +- .../src/cmdeploy/service/lastlogin.service.f | 12 ++++ 10 files changed, 140 insertions(+), 80 deletions(-) create mode 100644 chatmaild/src/chatmaild/lastlogin.py create mode 100644 chatmaild/src/chatmaild/tests/test_lastlogin.py create mode 100644 cmdeploy/src/cmdeploy/service/lastlogin.service.f diff --git a/CHANGELOG.md b/CHANGELOG.md index ea792758..1dfcae38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,10 @@ which removes users from database and mails after 100 days without any login. ([#350](https://github.com/deltachat/chatmail/pull/350)) +- Fix and refine "last-login" tracking which now happens via a dedicated + dovecot dictproxy with state kept in "$USERDIR/last-login" files. + ([#354](https://github.com/deltachat/chatmail/pull/354)) + - Refine DNS checking to distinguish between "required" and "recommended" settings ([#372](https://github.com/deltachat/chatmail/pull/372)) diff --git a/chatmaild/pyproject.toml b/chatmaild/pyproject.toml index 406e0d0f..1fa8a774 100644 --- a/chatmaild/pyproject.toml +++ b/chatmaild/pyproject.toml @@ -27,6 +27,7 @@ filtermail = "chatmaild.filtermail:main" echobot = "chatmaild.echo:main" chatmail-metrics = "chatmaild.metrics:main" delete_inactive_users = "chatmaild.delete_inactive_users:main" +lastlogin = "chatmaild.lastlogin:main" [project.entry-points.pytest11] "chatmaild.testplugin" = "chatmaild.tests.plugin" diff --git a/chatmaild/src/chatmaild/delete_inactive_users.py b/chatmaild/src/chatmaild/delete_inactive_users.py index 3c59037e..ad94ab48 100644 --- a/chatmaild/src/chatmaild/delete_inactive_users.py +++ b/chatmaild/src/chatmaild/delete_inactive_users.py @@ -2,48 +2,13 @@ Remove inactive users """ -import os import shutil import sys import time -from pathlib import Path from .config import read_config from .database import Database - -LAST_LOGIN = "last-login" - - -def get_daytimestamp(timestamp) -> int: - return int(timestamp) // 86400 * 86400 - - -def write_last_login_to_userdir(userdir, timestamp): - target = userdir.joinpath(LAST_LOGIN) - timestamp = get_daytimestamp(timestamp) - try: - st = target.stat() - except FileNotFoundError: - # only happens on initial login - userdir.mkdir(exist_ok=True) - target.touch() - os.utime(target, (timestamp, timestamp)) - else: - if st.st_mtime < timestamp: - os.utime(target, (timestamp, timestamp)) - - -def get_last_login_from_userdir(userdir) -> int: - target = userdir.joinpath(LAST_LOGIN) - try: - return int(target.stat().st_mtime) - except FileNotFoundError: - # during migration many directories will not have last-login file - # so we write it here to the current time - target.touch() - timestamp = get_daytimestamp(time.time()) - os.utime(target, (timestamp, timestamp)) - return timestamp +from .lastlogin import get_last_login_from_userdir def delete_inactive_users(db, config, chunksize=100): @@ -62,7 +27,7 @@ def delete_inactive_users(db, config, chunksize=100): conn.execute("DELETE FROM users WHERE addr = ?", (user,)) pending.clear() - for userdir in Path(config.mailboxes_dir).iterdir(): + for userdir in config.mailboxes_dir.iterdir(): if get_last_login_from_userdir(userdir) < cutoff_date: remove(userdir) diff --git a/chatmaild/src/chatmaild/lastlogin.py b/chatmaild/src/chatmaild/lastlogin.py new file mode 100644 index 00000000..fe1a2fa6 --- /dev/null +++ b/chatmaild/src/chatmaild/lastlogin.py @@ -0,0 +1,67 @@ +import os +import sys +import time + +from .config import read_config +from .dictproxy import DictProxy + +# this file's mtime reflects the last login-time for a user +LAST_LOGIN = "last-login" + + +def get_daytimestamp(timestamp) -> int: + return int(timestamp) // 86400 * 86400 + + +def write_last_login_to_userdir(userdir, timestamp): + target = userdir.joinpath(LAST_LOGIN) + timestamp = get_daytimestamp(timestamp) + try: + st = target.stat() + except FileNotFoundError: + # only happens on initial login + userdir.mkdir(exist_ok=True) + target.touch() + os.utime(target, (timestamp, timestamp)) + else: + if st.st_mtime < timestamp: + os.utime(target, (timestamp, timestamp)) + + +def get_last_login_from_userdir(userdir) -> int: + target = userdir.joinpath(LAST_LOGIN) + try: + return int(target.stat().st_mtime) + except FileNotFoundError: + # during migration many directories will not have last-login file + # so we write it here to the current time + target.touch() + timestamp = get_daytimestamp(time.time()) + os.utime(target, (timestamp, timestamp)) + return timestamp + + +class LastLoginDictProxy(DictProxy): + def __init__(self, config): + super().__init__() + self.config = config + + def handle_set(self, transaction_id, parts): + keyname = parts[1].split("/") + value = parts[2] if len(parts) > 2 else "" + addr = self.transactions[transaction_id]["addr"] + if keyname[0] == "shared" and keyname[1] == "last-login": + addr = keyname[2] + timestamp = int(value) + userdir = self.config.get_user_maildir(addr) + write_last_login_to_userdir(userdir, timestamp) + else: + # Transaction failed. + self.transactions[transaction_id]["res"] = "F\n" + + +def main(): + socket, config_path = sys.argv[1:] + config = read_config(config_path) + dictproxy = LastLoginDictProxy(config=config) + dictproxy.serve_forever_from_socket(socket) diff --git a/chatmaild/src/chatmaild/tests/test_delete_inactive_users.py b/chatmaild/src/chatmaild/tests/test_delete_inactive_users.py index 2a7334e2..33e2b69c 100644 --- a/chatmaild/src/chatmaild/tests/test_delete_inactive_users.py +++ b/chatmaild/src/chatmaild/tests/test_delete_inactive_users.py @@ -1,11 +1,8 @@ import time -from chatmaild.delete_inactive_users import ( - delete_inactive_users, - get_last_login_from_userdir, - write_last_login_to_userdir, -) +from chatmaild.delete_inactive_users import delete_inactive_users from chatmaild.doveauth import lookup_passdb +from chatmaild.lastlogin import get_last_login_from_userdir, write_last_login_to_userdir def test_login_timestamps(tmp_path): diff --git a/chatmaild/src/chatmaild/tests/test_lastlogin.py b/chatmaild/src/chatmaild/tests/test_lastlogin.py new file mode 100644 index 00000000..3d02f8a6 --- /dev/null +++ b/chatmaild/src/chatmaild/tests/test_lastlogin.py @@ -0,0 +1,50 @@ +import time + +import pytest +from chatmaild.lastlogin import ( + LastLoginDictProxy, + get_last_login_from_userdir, + write_last_login_to_userdir, +) + + +@pytest.fixture +def testaddr(): + return "user.name@example.org" + + +def test_handle_dovecot_request_last_login(testaddr, example_config): + dictproxy = LastLoginDictProxy(config=example_config) + + userdir = dictproxy.config.get_user_maildir(testaddr) + + # set last-login info for user + tx = "1111" + msg = f"B{tx}\t{testaddr}" + res = dictproxy.handle_dovecot_request(msg) + assert not res + assert dictproxy.transactions == {tx: dict(addr=testaddr, res="O\n")} + + timestamp = int(time.time()) + msg = f"S{tx}\tshared/last-login/{testaddr}\t{timestamp}" + res = dictproxy.handle_dovecot_request(msg) + assert not res + assert len(dictproxy.transactions) == 1 + read_timestamp = get_last_login_from_userdir(userdir) + assert read_timestamp == timestamp // 86400 * 86400 + + msg = f"C{tx}" + res = dictproxy.handle_dovecot_request(msg) + assert res == "O\n" + assert len(dictproxy.transactions) == 0 + + +def test_login_timestamp(testaddr, example_config): + dictproxy = LastLoginDictProxy(config=example_config) + userdir = dictproxy.config.get_user_maildir(testaddr) + userdir.mkdir() + write_last_login_to_userdir(userdir, timestamp=100000) + assert get_last_login_from_userdir(userdir) == 86400 + + write_last_login_to_userdir(userdir, timestamp=200000) + assert get_last_login_from_userdir(userdir) == 86400 * 2 diff --git a/chatmaild/src/chatmaild/tests/test_metadata.py b/chatmaild/src/chatmaild/tests/test_metadata.py index 544b1e65..7a0573d2 100644 --- a/chatmaild/src/chatmaild/tests/test_metadata.py +++ b/chatmaild/src/chatmaild/tests/test_metadata.py @@ -3,7 +3,6 @@ import time import pytest import requests -from chatmaild.delete_inactive_users import get_last_login_from_userdir from chatmaild.metadata import ( Metadata, MetadataDictProxy, @@ -97,16 +96,6 @@ def test_handle_dovecot_request_lookup_fails(dictproxy, testaddr): res = dictproxy.handle_dovecot_request(f"Lpriv/123/chatmail\t{testaddr}") assert res == "N\n" -def test_metadata_login_timestamp(metadata, testaddr): - timestamp = metadata.vmail_dir.joinpath(testaddr).mkdir() - metadata.write_login_timestamp(testaddr, timestamp=100000) - timestamp = metadata.vmail_dir.joinpath(testaddr, "last-login").read_text() - assert int(timestamp) == 86400 - - metadata.write_login_timestamp(testaddr, timestamp=200000) - timestamp = metadata.vmail_dir.joinpath(testaddr, "last-login").read_text() - assert int(timestamp) == 86400 * 2 - def test_handle_dovecot_request_happy_path(dictproxy, testaddr, token): metadata = dictproxy.metadata @@ -144,32 +133,6 @@ def test_handle_dovecot_request_happy_path(dictproxy, testaddr, token): assert queue_item.path.exists() -def test_handle_dovecot_request_last_login(notifier, metadata, testaddr, token): - dictproxy = MetadataDictProxy(notifier=notifier, metadata=metadata) - - userdir = metadata.vmail_dir.joinpath(testaddr) - - # set last-login info for user - tx = "1111" - msg = f"B{tx}\t{testaddr}" - res = dictproxy.handle_dovecot_request(msg) - assert not res - assert dictproxy.transactions == {tx: dict(addr=testaddr, res="O\n")} - - timestamp = int(time.time()) - msg = f"S{tx}\tshared/last-login/{testaddr}\t{timestamp}" - res = dictproxy.handle_dovecot_request(msg) - assert not res - assert len(dictproxy.transactions) == 1 - read_timestamp = get_last_login_from_userdir(userdir) - assert read_timestamp == timestamp // 86400 * 86400 - - msg = f"C{tx}" - res = dictproxy.handle_dovecot_request(msg) - assert res == "O\n" - assert len(dictproxy.transactions) == 0 - - def test_handle_dovecot_protocol_set_devicetoken(dictproxy): rfile = io.BytesIO( b"\n".join( diff --git a/cmdeploy/src/cmdeploy/__init__.py b/cmdeploy/src/cmdeploy/__init__.py index c7485c23..3d0a03bc 100644 --- a/cmdeploy/src/cmdeploy/__init__.py +++ b/cmdeploy/src/cmdeploy/__init__.py @@ -103,6 +103,7 @@ def _install_remote_venv_with_chatmaild(config) -> None: "filtermail", "echobot", "chatmail-metadata", + "lastlogin", ): params = dict( execpath=f"{remote_venv_dir}/bin/{fn}", diff --git a/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 b/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 index 581ce9a6..ae9d74c6 100644 --- a/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 +++ b/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 @@ -111,7 +111,7 @@ protocol imap { } plugin { - last_login_dict = proxy:/run/chatmail-metadata/metadata.socket:metadata + last_login_dict = proxy:/run/chatmail-lastlogin/lastlogin.socket:lastlogin #last_login_key = last-login/%u # default last_login_precision = s } diff --git a/cmdeploy/src/cmdeploy/service/lastlogin.service.f b/cmdeploy/src/cmdeploy/service/lastlogin.service.f new file mode 100644 index 00000000..d1f9b3e4 --- /dev/null +++ b/cmdeploy/src/cmdeploy/service/lastlogin.service.f @@ -0,0 +1,12 @@ +[Unit] +Description=Dict proxy for last-login tracking + +[Service] +ExecStart={execpath} /run/chatmail-lastlogin/lastlogin.socket {config_path} +Restart=always +RestartSec=30 +User=vmail +RuntimeDirectory=chatmail-lastlogin + +[Install] +WantedBy=multi-user.target