- Replace skip-based pagination with cursor-based pagination (timestamp|_id cursors) - Add Prometheus /metrics endpoint with request latency, fetch volume, and error counters - Implement incremental fetch watermarking per source (watermarks collection in MongoDB) - Add Graph change notification webhook endpoint (/api/webhooks/graph) - Add correlation ID middleware for distributed tracing (x-request-id header) - Update frontend to use cursor-based pagination with Prev/Next navigation - Update tests for cursor pagination, metrics, webhooks, and watermark mocking
This commit is contained in:
18
backend/watermark.py
Normal file
18
backend/watermark.py
Normal file
@@ -0,0 +1,18 @@
|
||||
from database import db
|
||||
|
||||
watermarks_collection = db["watermarks"]
|
||||
|
||||
|
||||
def get_watermark(source: str) -> str | None:
|
||||
"""Return the ISO timestamp of the last successful fetch for a source."""
|
||||
doc = watermarks_collection.find_one({"source": source})
|
||||
return doc.get("last_fetch_time") if doc else None
|
||||
|
||||
|
||||
def set_watermark(source: str, timestamp: str):
|
||||
"""Persist the latest successful fetch timestamp for a source."""
|
||||
watermarks_collection.update_one(
|
||||
{"source": source},
|
||||
{"$set": {"last_fetch_time": timestamp}},
|
||||
upsert=True,
|
||||
)
|
||||
Reference in New Issue
Block a user