feat: implement Phase 3 scaling

- Replace skip-based pagination with cursor-based pagination (timestamp|_id cursors)
- Add Prometheus /metrics endpoint with request latency, fetch volume, and error counters
- Implement incremental fetch watermarking per source (watermarks collection in MongoDB)
- Add Graph change notification webhook endpoint (/api/webhooks/graph)
- Add correlation ID middleware for distributed tracing (x-request-id header)
- Update frontend to use cursor-based pagination with Prev/Next navigation
- Update tests for cursor pagination, metrics, webhooks, and watermark mocking

backend/sources/intune_audit.py

@@ -4,13 +4,13 @@ from graph.auth import get_access_token
 from utils.http import get_with_retry
 
 
-def fetch_intune_audit(hours: int = 24, max_pages: int = 50) -> list[dict]:
+def fetch_intune_audit(hours: int = 24, since: str | None = None, max_pages: int = 50) -> list[dict]:
     """
     Fetch Intune audit events via Microsoft Graph.
     Requires Intune audit permissions (e.g., DeviceManagementConfiguration.Read.All).
     """
     token = get_access_token()
-    start_time = (datetime.utcnow() - timedelta(hours=hours)).isoformat() + "Z"
+    start_time = since or (datetime.utcnow() - timedelta(hours=hours)).isoformat() + "Z"
     url = (
         "https://graph.microsoft.com/v1.0/deviceManagement/auditEvents"
         f"?$filter=activityDateTime ge {start_time}"

backend/sources/unified_audit.py

@@ -11,10 +11,13 @@ AUDIT_CONTENT_TYPES = {
 }
 
 
-def _time_window(hours: int):
+def _time_window(hours: int, since: str | None = None):
     end = datetime.utcnow()
-    start = end - timedelta(hours=hours)
-    # Activity API expects UTC ISO without Z
+    if since:
+        # Office 365 API expects format without Z
+        start = datetime.fromisoformat(since.replace("Z", "+00:00")).replace(tzinfo=None)
+    else:
+        start = end - timedelta(hours=hours)
     return start.strftime("%Y-%m-%dT%H:%M:%S"), end.strftime("%Y-%m-%dT%H:%M:%S")
 
 
@@ -26,8 +29,8 @@ def _ensure_subscription(content_type: str, token: str, tenant_id: str):
         post_with_retry(url, params=params, headers=headers, timeout=10)
 
 
-def _list_content(content_type: str, token: str, tenant_id: str, hours: int) -> list[dict]:
-    start, end = _time_window(hours)
+def _list_content(content_type: str, token: str, tenant_id: str, hours: int, since: str | None = None) -> list[dict]:
+    start, end = _time_window(hours, since)
     url = f"https://manage.office.com/api/v1.0/{tenant_id}/activity/feed/subscriptions/content"
     params = {"contentType": content_type, "startTime": start, "endTime": end}
     headers = {"Authorization": f"Bearer {token}"}
@@ -56,7 +59,7 @@ def _download_content(content_uri: str, token: str) -> list[dict]:
         raise RuntimeError(f"Failed to download audit content: {exc}") from exc
 
 
-def fetch_unified_audit(hours: int = 24, max_files: int = 50) -> list[dict]:
+def fetch_unified_audit(hours: int = 24, since: str | None = None, max_files: int = 50) -> list[dict]:
     """
     Fetch unified audit logs (Exchange, SharePoint, Teams policy changes via Audit.General)
     using the Office 365 Management Activity API.
@@ -69,7 +72,7 @@ def fetch_unified_audit(hours: int = 24, max_files: int = 50) -> list[dict]:
 
     for content_type in AUDIT_CONTENT_TYPES:
         _ensure_subscription(content_type, token, TENANT_ID)
-        contents = _list_content(content_type, token, TENANT_ID, hours)
+        contents = _list_content(content_type, token, TENANT_ID, hours, since)
         for item in contents[:max_files]:
             content_uri = item.get("contentUri")
             if not content_uri: