diff --git a/.env.example b/.env.example index ecc06e9..43392b4 100644 --- a/.env.example +++ b/.env.example @@ -50,8 +50,10 @@ LLM_MAX_EVENTS=200 LLM_TIMEOUT_SECONDS=30 LLM_API_VERSION= -# Optional: privacy / service-level access control -# Comma-separated list of services considered privacy-sensitive (hidden from users without PRIVACY_SERVICE_ROLES) +# Optional: privacy / access control +# Hide entire services from users without PRIVACY_SERVICE_ROLES # PRIVACY_SERVICES=Exchange,Teams -# Comma-separated list of Entra roles that can access privacy-sensitive services +# Hide specific operations across all services from users without PRIVACY_SERVICE_ROLES +# PRIVACY_SENSITIVE_OPERATIONS=MailItemsAccessed,Search-Mailbox,Send,ChatMessageRead +# Comma-separated list of Entra roles that can access privacy-sensitive data # PRIVACY_SERVICE_ROLES=SecurityAdministrator,ComplianceAdministrator diff --git a/backend/config.py b/backend/config.py index a0c6bda..9343b00 100644 --- a/backend/config.py +++ b/backend/config.py @@ -51,9 +51,10 @@ class Settings(BaseSettings): LLM_TIMEOUT_SECONDS: int = 30 LLM_API_VERSION: str = "" # e.g. 2025-01-01-preview for Azure OpenAI - # Privacy / Service-level access control - # Services listed here are hidden from users who don't have PRIVACY_SERVICE_ROLES + # Privacy / access control + # Entire services can be hidden, or specific operations can be gated. PRIVACY_SERVICES: str = "" # comma-separated, e.g. "Exchange,Teams" + PRIVACY_SENSITIVE_OPERATIONS: str = "" # comma-separated, e.g. "MailItemsAccessed,Search-Mailbox,Send" PRIVACY_SERVICE_ROLES: str = "" # comma-separated, e.g. "SecurityAdministrator,ComplianceAdministrator" @@ -92,4 +93,5 @@ LLM_TIMEOUT_SECONDS = _settings.LLM_TIMEOUT_SECONDS LLM_API_VERSION = _settings.LLM_API_VERSION PRIVACY_SERVICES = {s.strip() for s in _settings.PRIVACY_SERVICES.split(",") if s.strip()} +PRIVACY_SENSITIVE_OPERATIONS = {o.strip() for o in _settings.PRIVACY_SENSITIVE_OPERATIONS.split(",") if o.strip()} PRIVACY_SERVICE_ROLES = {r.strip() for r in _settings.PRIVACY_SERVICE_ROLES.split(",") if r.strip()} diff --git a/backend/routes/ask.py b/backend/routes/ask.py index e7b15ed..d52fd8c 100644 --- a/backend/routes/ask.py +++ b/backend/routes/ask.py @@ -13,6 +13,7 @@ from config import ( LLM_MAX_EVENTS, LLM_MODEL, LLM_TIMEOUT_SECONDS, + PRIVACY_SENSITIVE_OPERATIONS, PRIVACY_SERVICES, ) from database import events_collection @@ -596,7 +597,9 @@ async def explain_event(event_id: str, user: dict = Depends(require_auth)): if not event: raise HTTPException(status_code=404, detail="Event not found") - if event.get("service") in PRIVACY_SERVICES and not user_can_access_privacy_services(user): + if ( + event.get("service") in PRIVACY_SERVICES or event.get("operation") in PRIVACY_SENSITIVE_OPERATIONS + ) and not user_can_access_privacy_services(user): raise HTTPException(status_code=403, detail="Access to this event is restricted") event.pop("_id", None) @@ -689,7 +692,8 @@ async def ask_question(body: AskRequest, user: dict = Depends(require_auth)): # ----------------------------------------------------------------------- # Build and run query # ----------------------------------------------------------------------- - privacy_excluded = [] if user_can_access_privacy_services(user) else list(PRIVACY_SERVICES) + privacy_excluded_services = [] if user_can_access_privacy_services(user) else list(PRIVACY_SERVICES) + privacy_excluded_ops = [] if user_can_access_privacy_services(user) else list(PRIVACY_SENSITIVE_OPERATIONS) query = _build_event_query( entity, start, @@ -701,8 +705,13 @@ async def ask_question(body: AskRequest, user: dict = Depends(require_auth)): include_tags=body.include_tags, exclude_tags=body.exclude_tags, ) - if privacy_excluded: - query["$and"] = query.get("$and", []) + [{"service": {"$nin": privacy_excluded}}] + extra_filters = [] + if privacy_excluded_services: + extra_filters.append({"service": {"$nin": privacy_excluded_services}}) + if privacy_excluded_ops: + extra_filters.append({"operation": {"$nin": privacy_excluded_ops}}) + if extra_filters: + query["$and"] = query.get("$and", []) + extra_filters try: total = events_collection.count_documents(query) diff --git a/backend/routes/events.py b/backend/routes/events.py index 8909b33..43bdbd3 100644 --- a/backend/routes/events.py +++ b/backend/routes/events.py @@ -5,7 +5,7 @@ from datetime import UTC, datetime from audit_trail import log_action from auth import require_auth, user_can_access_privacy_services from bson import ObjectId -from config import PRIVACY_SERVICES +from config import PRIVACY_SENSITIVE_OPERATIONS, PRIVACY_SERVICES from database import events_collection from fastapi import APIRouter, Depends, HTTPException, Query from models.api import ( @@ -45,7 +45,7 @@ def _build_query( cursor: str | None = None, include_tags: list[str] | None = None, exclude_tags: list[str] | None = None, - exclude_services: list[str] | None = None, + exclude_operations: list[str] | None = None, ) -> dict: filters = [] @@ -53,8 +53,8 @@ def _build_query( filters.append({"service": service}) if services: filters.append({"service": {"$in": services}}) - if exclude_services: - filters.append({"service": {"$nin": exclude_services}}) + if exclude_operations: + filters.append({"operation": {"$nin": exclude_operations}}) if actor: actor_safe = re.escape(actor) filters.append( @@ -129,7 +129,8 @@ def list_events( exclude_tags: list[str] | None = Query(default=None), user: dict = Depends(require_auth), ): - privacy_excluded = [] if user_can_access_privacy_services(user) else list(PRIVACY_SERVICES) + privacy_excluded_services = [] if user_can_access_privacy_services(user) else list(PRIVACY_SERVICES) + privacy_excluded_ops = [] if user_can_access_privacy_services(user) else list(PRIVACY_SENSITIVE_OPERATIONS) query = _build_query( service=service, services=services, @@ -142,8 +143,13 @@ def list_events( cursor=cursor, include_tags=include_tags, exclude_tags=exclude_tags, - exclude_services=privacy_excluded, + exclude_operations=privacy_excluded_ops, ) + if privacy_excluded_services: + query = query if query else {} + if "$and" not in query: + query = {"$and": [query]} if query else {"$and": []} + query["$and"].append({"service": {"$nin": privacy_excluded_services}}) safe_page_size = max(1, min(page_size, 500)) @@ -208,7 +214,8 @@ def bulk_tags( exclude_tags: list[str] | None = Query(default=None), user: dict = Depends(require_auth), ): - privacy_excluded = [] if user_can_access_privacy_services(user) else list(PRIVACY_SERVICES) + privacy_excluded_services = [] if user_can_access_privacy_services(user) else list(PRIVACY_SERVICES) + privacy_excluded_ops = [] if user_can_access_privacy_services(user) else list(PRIVACY_SENSITIVE_OPERATIONS) query = _build_query( service=service, services=services, @@ -220,8 +227,13 @@ def bulk_tags( search=search, include_tags=include_tags, exclude_tags=exclude_tags, - exclude_services=privacy_excluded, + exclude_operations=privacy_excluded_ops, ) + if privacy_excluded_services: + query = query if query else {} + if "$and" not in query: + query = {"$and": [query]} if query else {"$and": []} + query["$and"].append({"service": {"$nin": privacy_excluded_services}}) tags = [t.strip() for t in body.tags if t.strip()] if not tags: raise HTTPException(status_code=400, detail="No tags provided") @@ -260,6 +272,7 @@ def filter_options( if not user_can_access_privacy_services(user): services = [s for s in services if s not in PRIVACY_SERVICES] + operations = [o for o in operations if o not in PRIVACY_SENSITIVE_OPERATIONS] return { "services": services, diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index b67b02c..482a62b 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -34,8 +34,11 @@ def client(mock_events_collection, mock_watermarks_collection, monkeypatch): monkeypatch.setattr("auth.AUTH_ENABLED", False) monkeypatch.setattr("routes.mcp.AUTH_ENABLED", False) monkeypatch.setattr("config.PRIVACY_SERVICES", set()) + monkeypatch.setattr("config.PRIVACY_SENSITIVE_OPERATIONS", set()) monkeypatch.setattr("routes.events.PRIVACY_SERVICES", set()) + monkeypatch.setattr("routes.events.PRIVACY_SENSITIVE_OPERATIONS", set()) monkeypatch.setattr("routes.ask.PRIVACY_SERVICES", set()) + monkeypatch.setattr("routes.ask.PRIVACY_SENSITIVE_OPERATIONS", set()) monkeypatch.setattr("database.db.command", lambda cmd: {"ok": 1} if cmd == "ping" else {}) # Mock audit trail and rules collections so tests don't wait on real MongoDB diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index df93e61..1d0f89f 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -149,19 +149,19 @@ def test_saved_searches_create_validation(client, monkeypatch): assert response.status_code == 400 -def test_privacy_filtering_events(client, mock_events_collection, monkeypatch): - monkeypatch.setattr("config.PRIVACY_SERVICES", {"Exchange", "Teams"}) - monkeypatch.setattr("routes.events.PRIVACY_SERVICES", {"Exchange", "Teams"}) +def test_privacy_filtering_events_by_operation(client, mock_events_collection, monkeypatch): + monkeypatch.setattr("config.PRIVACY_SENSITIVE_OPERATIONS", {"MailItemsAccessed", "Send"}) + monkeypatch.setattr("routes.events.PRIVACY_SENSITIVE_OPERATIONS", {"MailItemsAccessed", "Send"}) monkeypatch.setattr("auth.PRIVACY_SERVICE_ROLES", {"SecurityAdmin"}) monkeypatch.setattr("auth.user_can_access_privacy_services", lambda claims: False) monkeypatch.setattr("routes.events.user_can_access_privacy_services", lambda claims: False) mock_events_collection.insert_one( { - "id": "evt-dir", + "id": "evt-safe", "timestamp": datetime.now(UTC).isoformat(), - "service": "Directory", - "operation": "Add user", + "service": "Exchange", + "operation": "Add-MailboxPermission", "result": "success", "actor_display": "Alice", "raw_text": "", @@ -169,7 +169,7 @@ def test_privacy_filtering_events(client, mock_events_collection, monkeypatch): ) mock_events_collection.insert_one( { - "id": "evt-exc", + "id": "evt-priv", "timestamp": datetime.now(UTC).isoformat(), "service": "Exchange", "operation": "Send", @@ -183,33 +183,58 @@ def test_privacy_filtering_events(client, mock_events_collection, monkeypatch): assert response.status_code == 200 data = response.json() ids = [e["id"] for e in data["items"]] - assert "evt-dir" in ids - assert "evt-exc" not in ids + assert "evt-safe" in ids + assert "evt-priv" not in ids -def test_privacy_filter_options(client, mock_events_collection, monkeypatch): - monkeypatch.setattr("config.PRIVACY_SERVICES", {"Exchange"}) - monkeypatch.setattr("routes.events.PRIVACY_SERVICES", {"Exchange"}) +def test_privacy_filter_options_shows_service_hides_ops(client, mock_events_collection, monkeypatch): + monkeypatch.setattr("config.PRIVACY_SENSITIVE_OPERATIONS", {"MailItemsAccessed"}) + monkeypatch.setattr("routes.events.PRIVACY_SENSITIVE_OPERATIONS", {"MailItemsAccessed"}) monkeypatch.setattr("auth.PRIVACY_SERVICE_ROLES", {"SecurityAdmin"}) monkeypatch.setattr("auth.user_can_access_privacy_services", lambda claims: False) monkeypatch.setattr("routes.events.user_can_access_privacy_services", lambda claims: False) + mock_events_collection.insert_one( + { + "id": "evt-1", + "timestamp": datetime.now(UTC).isoformat(), + "service": "Exchange", + "operation": "MailItemsAccessed", + "result": "success", + "actor_display": "Alice", + "raw_text": "", + } + ) + mock_events_collection.insert_one( + { + "id": "evt-2", + "timestamp": datetime.now(UTC).isoformat(), + "service": "Exchange", + "operation": "Add-MailboxPermission", + "result": "success", + "actor_display": "Bob", + "raw_text": "", + } + ) + response = client.get("/api/filter-options") assert response.status_code == 200 data = response.json() - assert "Exchange" not in data["services"] + assert "Exchange" in data["services"] + assert "MailItemsAccessed" not in data["operations"] + assert "Add-MailboxPermission" in data["operations"] -def test_privacy_explain_forbidden(client, mock_events_collection, monkeypatch): - monkeypatch.setattr("config.PRIVACY_SERVICES", {"Exchange"}) - monkeypatch.setattr("routes.ask.PRIVACY_SERVICES", {"Exchange"}) +def test_privacy_explain_forbidden_by_operation(client, mock_events_collection, monkeypatch): + monkeypatch.setattr("config.PRIVACY_SENSITIVE_OPERATIONS", {"Send"}) + monkeypatch.setattr("routes.ask.PRIVACY_SENSITIVE_OPERATIONS", {"Send"}) monkeypatch.setattr("auth.PRIVACY_SERVICE_ROLES", {"SecurityAdmin"}) monkeypatch.setattr("auth.user_can_access_privacy_services", lambda claims: False) monkeypatch.setattr("routes.ask.user_can_access_privacy_services", lambda claims: False) mock_events_collection.insert_one( { - "id": "evt-exc2", + "id": "evt-send", "timestamp": datetime.now(UTC).isoformat(), "service": "Exchange", "operation": "Send", @@ -218,7 +243,7 @@ def test_privacy_explain_forbidden(client, mock_events_collection, monkeypatch): "raw_text": "", } ) - response = client.post("/api/events/evt-exc2/explain") + response = client.post("/api/events/evt-send/explain") assert response.status_code == 403