feat: implement Phase 3 scaling

- Replace skip-based pagination with cursor-based pagination (timestamp|_id cursors)
- Add Prometheus /metrics endpoint with request latency, fetch volume, and error counters
- Implement incremental fetch watermarking per source (watermarks collection in MongoDB)
- Add Graph change notification webhook endpoint (/api/webhooks/graph)
- Add correlation ID middleware for distributed tracing (x-request-id header)
- Update frontend to use cursor-based pagination with Prev/Next navigation
- Update tests for cursor pagination, metrics, webhooks, and watermark mocking

backend/routes/events.py

@@ -1,3 +1,4 @@
+import base64
 import re
 
 from auth import require_auth
@@ -8,6 +9,20 @@ from models.api import FilterOptionsResponse, PaginatedEventResponse
 router = APIRouter(dependencies=[Depends(require_auth)])
 
 
+def _encode_cursor(timestamp: str, oid: str) -> str:
+    payload = f"{timestamp}|{oid}"
+    return base64.b64encode(payload.encode()).decode()
+
+
+def _decode_cursor(cursor: str) -> tuple[str, str]:
+    try:
+        payload = base64.b64decode(cursor.encode()).decode()
+        timestamp, oid = payload.split("|", 1)
+        return timestamp, oid
+    except Exception as exc:
+        raise HTTPException(status_code=400, detail="Invalid cursor") from exc
+
+
 @router.get("/events", response_model=PaginatedEventResponse)
 def list_events(
     service: str | None = None,
@@ -17,7 +32,7 @@ def list_events(
     start: str | None = None,
     end: str | None = None,
     search: str | None = None,
-    page: int = Query(default=1, ge=1),
+    cursor: str | None = None,
     page_size: int = Query(default=50, ge=1, le=500),
 ):
     filters = []
@@ -61,26 +76,47 @@ def list_events(
             }
         )
 
+    if cursor:
+        try:
+            cursor_ts, cursor_oid = _decode_cursor(cursor)
+        except HTTPException:
+            raise
+        filters.append(
+            {
+                "$or": [
+                    {"timestamp": {"$lt": cursor_ts}},
+                    {"timestamp": cursor_ts, "_id": {"$lt": cursor_oid}},
+                ]
+            }
+        )
+
     query = {"$and": filters} if filters else {}
 
     safe_page_size = max(1, min(page_size, 500))
-    safe_page = max(1, page)
-    skip = (safe_page - 1) * safe_page_size
 
     try:
-        total = events_collection.count_documents(query)
-        cursor = events_collection.find(query).sort("timestamp", -1).skip(skip).limit(safe_page_size)
-        events = list(cursor)
+        total = events_collection.count_documents(query) if not cursor else -1
+        cursor_query = (
+            events_collection.find(query)
+            .sort([("timestamp", -1), ("_id", -1)])
+            .limit(safe_page_size)
+        )
+        events = list(cursor_query)
     except Exception as exc:
         raise HTTPException(status_code=500, detail=f"Failed to query events: {exc}") from exc
 
+    next_cursor = None
+    if len(events) == safe_page_size:
+        last = events[-1]
+        next_cursor = _encode_cursor(last["timestamp"], str(last["_id"]))
+
     for e in events:
         e["_id"] = str(e["_id"])
     return {
         "items": events,
         "total": total,
-        "page": safe_page,
         "page_size": safe_page_size,
+        "next_cursor": next_cursor,
     }