introduce last-login proxy

CHANGELOG.md

@@ -14,6 +14,10 @@
   which removes users from database and mails after 100 days without any login. 
   ([#350](https://github.com/deltachat/chatmail/pull/350))
 
+- Fix and refine "last-login" tracking which now happens via a dedicated
+  dovecot dictproxy with state kept in "$USERDIR/last-login" files. 
+  ([#354](https://github.com/deltachat/chatmail/pull/354))
+
 - Refine DNS checking to distinguish between "required" and "recommended" settings 
   ([#372](https://github.com/deltachat/chatmail/pull/372))
 

chatmaild/pyproject.toml

@@ -27,6 +27,7 @@ filtermail = "chatmaild.filtermail:main"
 echobot = "chatmaild.echo:main"
 chatmail-metrics = "chatmaild.metrics:main"
 delete_inactive_users = "chatmaild.delete_inactive_users:main"
+lastlogin = "chatmaild.lastlogin:main"
 
 [project.entry-points.pytest11]
 "chatmaild.testplugin" = "chatmaild.tests.plugin"

chatmaild/src/chatmaild/delete_inactive_users.py

@@ -2,48 +2,13 @@
 Remove inactive users
 """
 
-import os
 import shutil
 import sys
 import time
-from pathlib import Path
 
 from .config import read_config
 from .database import Database
-
+from .lastlogin import get_last_login_from_userdir
-LAST_LOGIN = "last-login"
-
-
-def get_daytimestamp(timestamp) -> int:
-    return int(timestamp) // 86400 * 86400
-
-
-def write_last_login_to_userdir(userdir, timestamp):
-    target = userdir.joinpath(LAST_LOGIN)
-    timestamp = get_daytimestamp(timestamp)
-    try:
-        st = target.stat()
-    except FileNotFoundError:
-        # only happens on initial login
-        userdir.mkdir(exist_ok=True)
-        target.touch()
-        os.utime(target, (timestamp, timestamp))
-    else:
-        if st.st_mtime < timestamp:
-            os.utime(target, (timestamp, timestamp))
-
-
-def get_last_login_from_userdir(userdir) -> int:
-    target = userdir.joinpath(LAST_LOGIN)
-    try:
-        return int(target.stat().st_mtime)
-    except FileNotFoundError:
-        # during migration many directories will not have last-login file
-        # so we write it here to the current time
-        target.touch()
-        timestamp = get_daytimestamp(time.time())
-        os.utime(target, (timestamp, timestamp))
-        return timestamp
 
 
 def delete_inactive_users(db, config, chunksize=100):
@@ -62,7 +27,7 @@ def delete_inactive_users(db, config, chunksize=100):
                 conn.execute("DELETE FROM users WHERE addr = ?", (user,))
         pending.clear()
 
-    for userdir in Path(config.mailboxes_dir).iterdir():
+    for userdir in config.mailboxes_dir.iterdir():
         if get_last_login_from_userdir(userdir) < cutoff_date:
             remove(userdir)
 

chatmaild/src/chatmaild/lastlogin.py

@@ -0,0 +1,67 @@
+import os
+import sys
+import time
+
+from .config import read_config
+from .dictproxy import DictProxy
+
+# this file's mtime reflects the last login-time for a user
+LAST_LOGIN = "last-login"
+
+
+def get_daytimestamp(timestamp) -> int:
+    return int(timestamp) // 86400 * 86400
+
+
+def write_last_login_to_userdir(userdir, timestamp):
+    target = userdir.joinpath(LAST_LOGIN)
+    timestamp = get_daytimestamp(timestamp)
+    try:
+        st = target.stat()
+    except FileNotFoundError:
+        # only happens on initial login
+        userdir.mkdir(exist_ok=True)
+        target.touch()
+        os.utime(target, (timestamp, timestamp))
+    else:
+        if st.st_mtime < timestamp:
+            os.utime(target, (timestamp, timestamp))
+
+
+def get_last_login_from_userdir(userdir) -> int:
+    target = userdir.joinpath(LAST_LOGIN)
+    try:
+        return int(target.stat().st_mtime)
+    except FileNotFoundError:
+        # during migration many directories will not have last-login file
+        # so we write it here to the current time
+        target.touch()
+        timestamp = get_daytimestamp(time.time())
+        os.utime(target, (timestamp, timestamp))
+        return timestamp
+
+
+class LastLoginDictProxy(DictProxy):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+
+    def handle_set(self, transaction_id, parts):
+        keyname = parts[1].split("/")
+        value = parts[2] if len(parts) > 2 else ""
+        addr = self.transactions[transaction_id]["addr"]
+        if keyname[0] == "shared" and keyname[1] == "last-login":
+            addr = keyname[2]
+            timestamp = int(value)
+            userdir = self.config.get_user_maildir(addr)
+            write_last_login_to_userdir(userdir, timestamp)
+        else:
+            # Transaction failed.
+            self.transactions[transaction_id]["res"] = "F\n"
+
+
+def main():
+    socket, config_path = sys.argv[1:]
+    config = read_config(config_path)
+    dictproxy = LastLoginDictProxy(config=config)
+    dictproxy.serve_forever_from_socket(socket)

chatmaild/src/chatmaild/tests/test_delete_inactive_users.py

@@ -1,11 +1,8 @@
 import time
 
-from chatmaild.delete_inactive_users import (
+from chatmaild.delete_inactive_users import delete_inactive_users
-    delete_inactive_users,
-    get_last_login_from_userdir,
-    write_last_login_to_userdir,
-)
 from chatmaild.doveauth import lookup_passdb
+from chatmaild.lastlogin import get_last_login_from_userdir, write_last_login_to_userdir
 
 
 def test_login_timestamps(tmp_path):

chatmaild/src/chatmaild/tests/test_lastlogin.py

@@ -0,0 +1,50 @@
+import time
+
+import pytest
+from chatmaild.lastlogin import (
+    LastLoginDictProxy,
+    get_last_login_from_userdir,
+    write_last_login_to_userdir,
+)
+
+
+@pytest.fixture
+def testaddr():
+    return "user.name@example.org"
+
+
+def test_handle_dovecot_request_last_login(testaddr, example_config):
+    dictproxy = LastLoginDictProxy(config=example_config)
+
+    userdir = dictproxy.config.get_user_maildir(testaddr)
+
+    # set last-login info for user
+    tx = "1111"
+    msg = f"B{tx}\t{testaddr}"
+    res = dictproxy.handle_dovecot_request(msg)
+    assert not res
+    assert dictproxy.transactions == {tx: dict(addr=testaddr, res="O\n")}
+
+    timestamp = int(time.time())
+    msg = f"S{tx}\tshared/last-login/{testaddr}\t{timestamp}"
+    res = dictproxy.handle_dovecot_request(msg)
+    assert not res
+    assert len(dictproxy.transactions) == 1
+    read_timestamp = get_last_login_from_userdir(userdir)
+    assert read_timestamp == timestamp // 86400 * 86400
+
+    msg = f"C{tx}"
+    res = dictproxy.handle_dovecot_request(msg)
+    assert res == "O\n"
+    assert len(dictproxy.transactions) == 0
+
+
+def test_login_timestamp(testaddr, example_config):
+    dictproxy = LastLoginDictProxy(config=example_config)
+    userdir = dictproxy.config.get_user_maildir(testaddr)
+    userdir.mkdir()
+    write_last_login_to_userdir(userdir, timestamp=100000)
+    assert get_last_login_from_userdir(userdir) == 86400
+
+    write_last_login_to_userdir(userdir, timestamp=200000)
+    assert get_last_login_from_userdir(userdir) == 86400 * 2

chatmaild/src/chatmaild/tests/test_metadata.py

@@ -3,7 +3,6 @@ import time
 
 import pytest
 import requests
-from chatmaild.delete_inactive_users import get_last_login_from_userdir
 from chatmaild.metadata import (
     Metadata,
     MetadataDictProxy,
@@ -97,16 +96,6 @@ def test_handle_dovecot_request_lookup_fails(dictproxy, testaddr):
     res = dictproxy.handle_dovecot_request(f"Lpriv/123/chatmail\t{testaddr}")
     assert res == "N\n"
 
-def test_metadata_login_timestamp(metadata, testaddr):
-    timestamp = metadata.vmail_dir.joinpath(testaddr).mkdir()
-    metadata.write_login_timestamp(testaddr, timestamp=100000)
-    timestamp = metadata.vmail_dir.joinpath(testaddr, "last-login").read_text()
-    assert int(timestamp) == 86400
-
-    metadata.write_login_timestamp(testaddr, timestamp=200000)
-    timestamp = metadata.vmail_dir.joinpath(testaddr, "last-login").read_text()
-    assert int(timestamp) == 86400 * 2
-
 
 def test_handle_dovecot_request_happy_path(dictproxy, testaddr, token):
     metadata = dictproxy.metadata
@@ -144,32 +133,6 @@ def test_handle_dovecot_request_happy_path(dictproxy, testaddr, token):
     assert queue_item.path.exists()
 
 
-def test_handle_dovecot_request_last_login(notifier, metadata, testaddr, token):
-    dictproxy = MetadataDictProxy(notifier=notifier, metadata=metadata)
-
-    userdir = metadata.vmail_dir.joinpath(testaddr)
-
-    # set last-login info for user
-    tx = "1111"
-    msg = f"B{tx}\t{testaddr}"
-    res = dictproxy.handle_dovecot_request(msg)
-    assert not res
-    assert dictproxy.transactions == {tx: dict(addr=testaddr, res="O\n")}
-
-    timestamp = int(time.time())
-    msg = f"S{tx}\tshared/last-login/{testaddr}\t{timestamp}"
-    res = dictproxy.handle_dovecot_request(msg)
-    assert not res
-    assert len(dictproxy.transactions) == 1
-    read_timestamp = get_last_login_from_userdir(userdir)
-    assert read_timestamp == timestamp // 86400 * 86400
-
-    msg = f"C{tx}"
-    res = dictproxy.handle_dovecot_request(msg)
-    assert res == "O\n"
-    assert len(dictproxy.transactions) == 0
-
-
 def test_handle_dovecot_protocol_set_devicetoken(dictproxy):
     rfile = io.BytesIO(
         b"\n".join(

cmdeploy/src/cmdeploy/__init__.py

@@ -103,6 +103,7 @@ def _install_remote_venv_with_chatmaild(config) -> None:
         "filtermail",
         "echobot",
         "chatmail-metadata",
+        "lastlogin",
     ):
         params = dict(
             execpath=f"{remote_venv_dir}/bin/{fn}",

cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2

@@ -111,7 +111,7 @@ protocol imap {
 }
 
 plugin {
-  last_login_dict = proxy:/run/chatmail-metadata/metadata.socket:metadata
+  last_login_dict = proxy:/run/chatmail-lastlogin/lastlogin.socket:lastlogin
   #last_login_key = last-login/%u # default
   last_login_precision = s
 }

cmdeploy/src/cmdeploy/service/lastlogin.service.f

@@ -0,0 +1,12 @@
+[Unit]
+Description=Dict proxy for last-login tracking
+
+[Service]
+ExecStart={execpath} /run/chatmail-lastlogin/lastlogin.socket {config_path}
+Restart=always
+RestartSec=30
+User=vmail
+RuntimeDirectory=chatmail-lastlogin
+
+[Install]
+WantedBy=multi-user.target