feat: implement Phase 3 scaling

- Replace skip-based pagination with cursor-based pagination (timestamp|_id cursors)
- Add Prometheus /metrics endpoint with request latency, fetch volume, and error counters
- Implement incremental fetch watermarking per source (watermarks collection in MongoDB)
- Add Graph change notification webhook endpoint (/api/webhooks/graph)
- Add correlation ID middleware for distributed tracing (x-request-id header)
- Update frontend to use cursor-based pagination with Prev/Next navigation
- Update tests for cursor pagination, metrics, webhooks, and watermark mocking

backend/main.py

@@ -1,18 +1,23 @@
 import asyncio
 import logging
+import time
 from contextlib import suppress
 from pathlib import Path
 
 import structlog
 from config import CORS_ORIGINS, ENABLE_PERIODIC_FETCH, FETCH_INTERVAL_MINUTES
 from database import setup_indexes
-from fastapi import FastAPI, HTTPException
+from fastapi import FastAPI, HTTPException, Request
 from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import Response
 from fastapi.staticfiles import StaticFiles
+from metrics import observe_request, prometheus_metrics
+from middleware import CorrelationIdMiddleware
 from routes.config import router as config_router
 from routes.events import router as events_router
 from routes.fetch import router as fetch_router
 from routes.fetch import run_fetch
+from routes.webhooks import router as webhooks_router
 
 
 def configure_logging():
@@ -41,6 +46,7 @@ logger = structlog.get_logger("aoc.fetcher")
 
 app = FastAPI()
 
+app.add_middleware(CorrelationIdMiddleware)
 app.add_middleware(
     CORSMiddleware,
     allow_origins=CORS_ORIGINS,
@@ -49,9 +55,21 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
+
+@app.middleware("http")
+async def prometheus_middleware(request: Request, call_next):
+    start = time.time()
+    response = await call_next(request)
+    duration = time.time() - start
+    path = getattr(request.scope.get("route"), "path", request.url.path)
+    observe_request(request.method, path, response.status_code, duration)
+    return response
+
+
 app.include_router(fetch_router, prefix="/api")
 app.include_router(events_router, prefix="/api")
 app.include_router(config_router, prefix="/api")
+app.include_router(webhooks_router, prefix="/api")
 
 
 @app.get("/health")
@@ -65,6 +83,11 @@ async def health_check():
         raise HTTPException(status_code=503, detail="Database unavailable") from exc
 
 
+@app.get("/metrics")
+async def metrics():
+    return Response(content=prometheus_metrics(), media_type="text/plain")
+
+
 frontend_dir = Path(__file__).parent / "frontend"
 app.mount("/", StaticFiles(directory=frontend_dir, html=True), name="frontend")