feat: implement Phase 3 scaling

- Replace skip-based pagination with cursor-based pagination (timestamp|_id cursors)
- Add Prometheus /metrics endpoint with request latency, fetch volume, and error counters
- Implement incremental fetch watermarking per source (watermarks collection in MongoDB)
- Add Graph change notification webhook endpoint (/api/webhooks/graph)
- Add correlation ID middleware for distributed tracing (x-request-id header)
- Update frontend to use cursor-based pagination with Prev/Next navigation
- Update tests for cursor pagination, metrics, webhooks, and watermark mocking

backend/tests/test_api.py

@@ -9,15 +9,21 @@ def test_health(client):
     assert data["database"] == "connected"
 
 
+def test_metrics(client):
+    response = client.get("/metrics")
+    assert response.status_code == 200
+    assert "aoc_request_duration_seconds" in response.text
+
+
 def test_list_events_empty(client):
     response = client.get("/api/events")
     assert response.status_code == 200
     data = response.json()
     assert data["items"] == []
-    assert data["total"] == 0
+    assert data["next_cursor"] is None
 
 
-def test_list_events_pagination(client, mock_events_collection):
+def test_list_events_cursor_pagination(client, mock_events_collection):
     for i in range(5):
         mock_events_collection.insert_one({
             "id": f"evt-{i}",
@@ -28,13 +34,18 @@ def test_list_events_pagination(client, mock_events_collection):
             "actor_display": f"Actor {i}",
             "raw_text": "",
         })
-    response = client.get("/api/events?page=1&page_size=2")
+    response = client.get("/api/events?page_size=2")
     assert response.status_code == 200
     data = response.json()
-    assert data["total"] == 5
     assert len(data["items"]) == 2
-    assert data["page"] == 1
-    assert data["page_size"] == 2
+    assert data["next_cursor"] is not None
+
+    # Follow cursor
+    response2 = client.get(f"/api/events?page_size=2&cursor={data['next_cursor']}")
+    assert response2.status_code == 200
+    data2 = response2.json()
+    assert len(data2["items"]) == 2
+    assert data2["next_cursor"] is not None
 
 
 def test_list_events_filter_by_service(client, mock_events_collection):
@@ -59,7 +70,7 @@ def test_list_events_filter_by_service(client, mock_events_collection):
     response = client.get("/api/events?service=Exchange")
     assert response.status_code == 200
     data = response.json()
-    assert data["total"] == 1
+    assert len(data["items"]) == 1
     assert data["items"][0]["service"] == "Exchange"
 
 
@@ -96,3 +107,26 @@ def test_fetch_audit_logs_validation(client):
     assert response.status_code == 422
     response = client.get("/api/fetch-audit-logs?hours=721")
     assert response.status_code == 422
+
+
+def test_graph_webhook_validation(client):
+    token = "test-validation-token-123"
+    response = client.post("/api/webhooks/graph?validationToken=" + token)
+    assert response.status_code == 200
+    assert response.text == token
+    assert response.headers["content-type"] == "text/plain; charset=utf-8"
+
+
+def test_graph_webhook_notification(client):
+    payload = {
+        "value": [
+            {
+                "changeType": "updated",
+                "resource": "auditLogs/directoryAudits",
+                "clientState": "secret",
+            }
+        ]
+    }
+    response = client.post("/api/webhooks/graph", json=payload)
+    assert response.status_code == 200
+    assert response.json()["status"] == "accepted"