feat: implement Phase 3 scaling

- Replace skip-based pagination with cursor-based pagination (timestamp|_id cursors)
- Add Prometheus /metrics endpoint with request latency, fetch volume, and error counters
- Implement incremental fetch watermarking per source (watermarks collection in MongoDB)
- Add Graph change notification webhook endpoint (/api/webhooks/graph)
- Add correlation ID middleware for distributed tracing (x-request-id header)
- Update frontend to use cursor-based pagination with Prev/Next navigation
- Update tests for cursor pagination, metrics, webhooks, and watermark mocking

backend/tests/conftest.py

@@ -12,13 +12,22 @@ def mock_events_collection():
 
 
 @pytest.fixture(scope="function")
-def client(mock_events_collection, monkeypatch):
-    # Patch the collection in all modules that import it before the app is imported
+def mock_watermarks_collection():
+    client = mongomock.MongoClient()
+    db = client["micro_soc"]
+    coll = db["watermarks"]
+    return coll
+
+
+@pytest.fixture(scope="function")
+def client(mock_events_collection, mock_watermarks_collection, monkeypatch):
     monkeypatch.setattr("database.events_collection", mock_events_collection)
     monkeypatch.setattr("routes.fetch.events_collection", mock_events_collection)
     monkeypatch.setattr("routes.events.events_collection", mock_events_collection)
+    monkeypatch.setattr("watermark.watermarks_collection", mock_watermarks_collection)
+    monkeypatch.setattr("routes.fetch.get_watermark", lambda source: None)
+    monkeypatch.setattr("routes.fetch.set_watermark", lambda source, ts: None)
     monkeypatch.setattr("auth.AUTH_ENABLED", False)
-    # Patch health check db.command so it doesn't need a real MongoDB server
     monkeypatch.setattr("database.db.command", lambda cmd: {"ok": 1} if cmd == "ping" else {})
 
     from main import app

backend/tests/test_api.py

@@ -9,15 +9,21 @@ def test_health(client):
     assert data["database"] == "connected"
 
 
+def test_metrics(client):
+    response = client.get("/metrics")
+    assert response.status_code == 200
+    assert "aoc_request_duration_seconds" in response.text
+
+
 def test_list_events_empty(client):
     response = client.get("/api/events")
     assert response.status_code == 200
     data = response.json()
     assert data["items"] == []
-    assert data["total"] == 0
+    assert data["next_cursor"] is None
 
 
-def test_list_events_pagination(client, mock_events_collection):
+def test_list_events_cursor_pagination(client, mock_events_collection):
     for i in range(5):
         mock_events_collection.insert_one({
             "id": f"evt-{i}",
@@ -28,13 +34,18 @@ def test_list_events_pagination(client, mock_events_collection):
             "actor_display": f"Actor {i}",
             "raw_text": "",
         })
-    response = client.get("/api/events?page=1&page_size=2")
+    response = client.get("/api/events?page_size=2")
     assert response.status_code == 200
     data = response.json()
-    assert data["total"] == 5
     assert len(data["items"]) == 2
-    assert data["page"] == 1
-    assert data["page_size"] == 2
+    assert data["next_cursor"] is not None
+
+    # Follow cursor
+    response2 = client.get(f"/api/events?page_size=2&cursor={data['next_cursor']}")
+    assert response2.status_code == 200
+    data2 = response2.json()
+    assert len(data2["items"]) == 2
+    assert data2["next_cursor"] is not None
 
 
 def test_list_events_filter_by_service(client, mock_events_collection):
@@ -59,7 +70,7 @@ def test_list_events_filter_by_service(client, mock_events_collection):
     response = client.get("/api/events?service=Exchange")
     assert response.status_code == 200
     data = response.json()
-    assert data["total"] == 1
+    assert len(data["items"]) == 1
     assert data["items"][0]["service"] == "Exchange"
 
 
@@ -96,3 +107,26 @@ def test_fetch_audit_logs_validation(client):
     assert response.status_code == 422
     response = client.get("/api/fetch-audit-logs?hours=721")
     assert response.status_code == 422
+
+
+def test_graph_webhook_validation(client):
+    token = "test-validation-token-123"
+    response = client.post("/api/webhooks/graph?validationToken=" + token)
+    assert response.status_code == 200
+    assert response.text == token
+    assert response.headers["content-type"] == "text/plain; charset=utf-8"
+
+
+def test_graph_webhook_notification(client):
+    payload = {
+        "value": [
+            {
+                "changeType": "updated",
+                "resource": "auditLogs/directoryAudits",
+                "clientState": "secret",
+            }
+        ]
+    }
+    response = client.post("/api/webhooks/graph", json=payload)
+    assert response.status_code == 200
+    assert response.json()["status"] == "accepted"