Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 47e0dfc2ca | |||
| 2fffe3aec2 | |||
| b2f4cabef4 | |||
| e069869a94 | |||
| fb2386e190 |
@@ -49,3 +49,11 @@ LLM_MODEL=gpt-4o-mini
|
|||||||
LLM_MAX_EVENTS=200
|
LLM_MAX_EVENTS=200
|
||||||
LLM_TIMEOUT_SECONDS=30
|
LLM_TIMEOUT_SECONDS=30
|
||||||
LLM_API_VERSION=
|
LLM_API_VERSION=
|
||||||
|
|
||||||
|
# Optional: privacy / access control
|
||||||
|
# Hide entire services from users without PRIVACY_SERVICE_ROLES
|
||||||
|
# PRIVACY_SERVICES=Exchange,Teams
|
||||||
|
# Hide specific operations across all services from users without PRIVACY_SERVICE_ROLES
|
||||||
|
# PRIVACY_SENSITIVE_OPERATIONS=MailItemsAccessed,Search-Mailbox,Send,ChatMessageRead
|
||||||
|
# Comma-separated list of Entra roles that can access privacy-sensitive data
|
||||||
|
# PRIVACY_SERVICE_ROLES=SecurityAdministrator,ComplianceAdministrator
|
||||||
|
|||||||
@@ -8,6 +8,8 @@ from config import (
|
|||||||
AUTH_CLIENT_ID,
|
AUTH_CLIENT_ID,
|
||||||
AUTH_ENABLED,
|
AUTH_ENABLED,
|
||||||
AUTH_TENANT_ID,
|
AUTH_TENANT_ID,
|
||||||
|
PRIVACY_SERVICE_ROLES,
|
||||||
|
PRIVACY_SERVICES,
|
||||||
)
|
)
|
||||||
from fastapi import Header, HTTPException
|
from fastapi import Header, HTTPException
|
||||||
from jwt import ExpiredSignatureError, InvalidTokenError, decode
|
from jwt import ExpiredSignatureError, InvalidTokenError, decode
|
||||||
@@ -82,6 +84,14 @@ def _decode_token(token: str, jwks):
|
|||||||
raise HTTPException(status_code=401, detail=f"Invalid token ({type(exc).__name__})") from None
|
raise HTTPException(status_code=401, detail=f"Invalid token ({type(exc).__name__})") from None
|
||||||
|
|
||||||
|
|
||||||
|
def user_can_access_privacy_services(claims: dict) -> bool:
|
||||||
|
"""Check if the user has roles that grant access to privacy-sensitive services."""
|
||||||
|
if not PRIVACY_SERVICES or not PRIVACY_SERVICE_ROLES:
|
||||||
|
return True
|
||||||
|
user_roles = set(claims.get("roles", []) or claims.get("role", []) or [])
|
||||||
|
return bool(user_roles.intersection(PRIVACY_SERVICE_ROLES))
|
||||||
|
|
||||||
|
|
||||||
def require_auth(authorization: str | None = Header(None)):
|
def require_auth(authorization: str | None = Header(None)):
|
||||||
if not AUTH_ENABLED:
|
if not AUTH_ENABLED:
|
||||||
return {"sub": "anonymous"}
|
return {"sub": "anonymous"}
|
||||||
|
|||||||
@@ -51,6 +51,12 @@ class Settings(BaseSettings):
|
|||||||
LLM_TIMEOUT_SECONDS: int = 30
|
LLM_TIMEOUT_SECONDS: int = 30
|
||||||
LLM_API_VERSION: str = "" # e.g. 2025-01-01-preview for Azure OpenAI
|
LLM_API_VERSION: str = "" # e.g. 2025-01-01-preview for Azure OpenAI
|
||||||
|
|
||||||
|
# Privacy / access control
|
||||||
|
# Entire services can be hidden, or specific operations can be gated.
|
||||||
|
PRIVACY_SERVICES: str = "" # comma-separated, e.g. "Exchange,Teams"
|
||||||
|
PRIVACY_SENSITIVE_OPERATIONS: str = "" # comma-separated, e.g. "MailItemsAccessed,Search-Mailbox,Send"
|
||||||
|
PRIVACY_SERVICE_ROLES: str = "" # comma-separated, e.g. "SecurityAdministrator,ComplianceAdministrator"
|
||||||
|
|
||||||
|
|
||||||
_settings = Settings()
|
_settings = Settings()
|
||||||
|
|
||||||
@@ -85,3 +91,7 @@ LLM_MODEL = _settings.LLM_MODEL
|
|||||||
LLM_MAX_EVENTS = _settings.LLM_MAX_EVENTS
|
LLM_MAX_EVENTS = _settings.LLM_MAX_EVENTS
|
||||||
LLM_TIMEOUT_SECONDS = _settings.LLM_TIMEOUT_SECONDS
|
LLM_TIMEOUT_SECONDS = _settings.LLM_TIMEOUT_SECONDS
|
||||||
LLM_API_VERSION = _settings.LLM_API_VERSION
|
LLM_API_VERSION = _settings.LLM_API_VERSION
|
||||||
|
|
||||||
|
PRIVACY_SERVICES = {s.strip() for s in _settings.PRIVACY_SERVICES.split(",") if s.strip()}
|
||||||
|
PRIVACY_SENSITIVE_OPERATIONS = {o.strip() for o in _settings.PRIVACY_SENSITIVE_OPERATIONS.split(",") if o.strip()}
|
||||||
|
PRIVACY_SERVICE_ROLES = {r.strip() for r in _settings.PRIVACY_SERVICE_ROLES.split(",") if r.strip()}
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ from pymongo import ASCENDING, DESCENDING, TEXT, MongoClient
|
|||||||
client = MongoClient(MONGO_URI or "mongodb://localhost:27017")
|
client = MongoClient(MONGO_URI or "mongodb://localhost:27017")
|
||||||
db = client[DB_NAME]
|
db = client[DB_NAME]
|
||||||
events_collection = db["events"]
|
events_collection = db["events"]
|
||||||
|
saved_searches_collection = db["saved_searches"]
|
||||||
logger = structlog.get_logger("aoc.database")
|
logger = structlog.get_logger("aoc.database")
|
||||||
|
|
||||||
|
|
||||||
@@ -20,6 +21,7 @@ def setup_indexes(max_retries: int = 5, delay: float = 2.0):
|
|||||||
events_collection.create_index([("timestamp", DESCENDING)])
|
events_collection.create_index([("timestamp", DESCENDING)])
|
||||||
events_collection.create_index([("service", ASCENDING), ("timestamp", DESCENDING)])
|
events_collection.create_index([("service", ASCENDING), ("timestamp", DESCENDING)])
|
||||||
events_collection.create_index("id")
|
events_collection.create_index("id")
|
||||||
|
saved_searches_collection.create_index([("created_by", ASCENDING), ("created_at", DESCENDING)])
|
||||||
events_collection.create_index(
|
events_collection.create_index(
|
||||||
[("actor_display", TEXT), ("raw_text", TEXT), ("operation", TEXT)],
|
[("actor_display", TEXT), ("raw_text", TEXT), ("operation", TEXT)],
|
||||||
name="text_search_index",
|
name="text_search_index",
|
||||||
|
|||||||
@@ -112,11 +112,23 @@
|
|||||||
<div class="actions">
|
<div class="actions">
|
||||||
<button type="submit">Apply filters</button>
|
<button type="submit">Apply filters</button>
|
||||||
<button type="button" id="clearBtn" class="ghost" @click="clearFilters()">Clear</button>
|
<button type="button" id="clearBtn" class="ghost" @click="clearFilters()">Clear</button>
|
||||||
|
<button type="button" class="ghost" @click="saveCurrentFilters()">Save filters</button>
|
||||||
<button type="button" class="ghost" @click="bulkTagMatching()">Bulk tag matching</button>
|
<button type="button" class="ghost" @click="bulkTagMatching()">Bulk tag matching</button>
|
||||||
<button type="button" class="ghost" @click="exportJSON()">Export JSON</button>
|
<button type="button" class="ghost" @click="exportJSON()">Export JSON</button>
|
||||||
<button type="button" class="ghost" @click="exportCSV()">Export CSV</button>
|
<button type="button" class="ghost" @click="exportCSV()">Export CSV</button>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="filter-row" x-show="savedSearches.length">
|
||||||
|
<div class="saved-searches">
|
||||||
|
<span>Saved:</span>
|
||||||
|
<template x-for="ss in savedSearches" :key="ss.id">
|
||||||
|
<span class="pill pill--tag" style="cursor:pointer;" @click="applySavedSearch(ss)">
|
||||||
|
<span x-text="ss.name"></span>
|
||||||
|
<button type="button" class="link" style="margin-left:4px;" @click.stop="deleteSavedSearch(ss.id)">×</button>
|
||||||
|
</span>
|
||||||
|
</template>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
</form>
|
</form>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
@@ -255,6 +267,7 @@
|
|||||||
actor: '', selectedServices: [], search: '', operation: '', result: '', start: '', end: '', limit: 100, includeTags: '', excludeTags: '',
|
actor: '', selectedServices: [], search: '', operation: '', result: '', start: '', end: '', limit: 100, includeTags: '', excludeTags: '',
|
||||||
},
|
},
|
||||||
options: { actors: [], services: [], operations: [], results: [] },
|
options: { actors: [], services: [], operations: [], results: [] },
|
||||||
|
savedSearches: [],
|
||||||
appVersion: '',
|
appVersion: '',
|
||||||
aiFeaturesEnabled: true,
|
aiFeaturesEnabled: true,
|
||||||
askQuestionText: '',
|
askQuestionText: '',
|
||||||
@@ -271,6 +284,7 @@
|
|||||||
this.loadSavedFilters();
|
this.loadSavedFilters();
|
||||||
if (!this.authConfig?.auth_enabled || this.accessToken) {
|
if (!this.authConfig?.auth_enabled || this.accessToken) {
|
||||||
await this.loadFilterOptions();
|
await this.loadFilterOptions();
|
||||||
|
await this.loadSavedSearches();
|
||||||
await this.loadSourceHealth();
|
await this.loadSourceHealth();
|
||||||
await this.loadEvents();
|
await this.loadEvents();
|
||||||
}
|
}
|
||||||
@@ -508,7 +522,7 @@
|
|||||||
const saved = localStorage.getItem('aoc_filters');
|
const saved = localStorage.getItem('aoc_filters');
|
||||||
if (!saved && this.options.services.length) {
|
if (!saved && this.options.services.length) {
|
||||||
// Default: exclude noisy high-volume services
|
// Default: exclude noisy high-volume services
|
||||||
const noisy = ['Exchange', 'SharePoint'];
|
const noisy = ['Exchange', 'SharePoint', 'Teams'];
|
||||||
this.filters.selectedServices = this.options.services.filter((s) => !noisy.includes(s));
|
this.filters.selectedServices = this.options.services.filter((s) => !noisy.includes(s));
|
||||||
} else if (saved) {
|
} else if (saved) {
|
||||||
try {
|
try {
|
||||||
@@ -529,6 +543,59 @@
|
|||||||
} catch {}
|
} catch {}
|
||||||
},
|
},
|
||||||
|
|
||||||
|
async loadSavedSearches() {
|
||||||
|
try {
|
||||||
|
const res = await fetch('/api/saved-searches', { headers: this.authHeader() });
|
||||||
|
if (!res.ok) return;
|
||||||
|
this.savedSearches = await res.json();
|
||||||
|
} catch {}
|
||||||
|
},
|
||||||
|
|
||||||
|
async saveCurrentFilters() {
|
||||||
|
const name = prompt('Name this saved filter:');
|
||||||
|
if (!name || !name.trim()) return;
|
||||||
|
try {
|
||||||
|
const res = await fetch('/api/saved-searches', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json', ...this.authHeader() },
|
||||||
|
body: JSON.stringify({ name: name.trim(), filters: { ...this.filters } }),
|
||||||
|
});
|
||||||
|
if (!res.ok) throw new Error(await res.text());
|
||||||
|
const created = await res.json();
|
||||||
|
this.savedSearches.unshift(created);
|
||||||
|
this.statusText = 'Filters saved.';
|
||||||
|
setTimeout(() => { if (this.statusText === 'Filters saved.') this.statusText = ''; }, 2000);
|
||||||
|
} catch (err) {
|
||||||
|
this.statusText = err.message || 'Failed to save filters.';
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
applySavedSearch(ss) {
|
||||||
|
if (!ss || !ss.filters) return;
|
||||||
|
const fields = ['actor', 'selectedServices', 'search', 'operation', 'result', 'start', 'end', 'limit', 'includeTags', 'excludeTags'];
|
||||||
|
fields.forEach((f) => {
|
||||||
|
if (ss.filters[f] !== undefined) this.filters[f] = ss.filters[f];
|
||||||
|
});
|
||||||
|
// Validate selectedServices against current options
|
||||||
|
this.filters.selectedServices = this.filters.selectedServices.filter((s) => this.options.services.includes(s));
|
||||||
|
this.resetPagination();
|
||||||
|
this.loadEvents();
|
||||||
|
},
|
||||||
|
|
||||||
|
async deleteSavedSearch(id) {
|
||||||
|
if (!confirm('Delete this saved search?')) return;
|
||||||
|
try {
|
||||||
|
const res = await fetch(`/api/saved-searches/${id}`, {
|
||||||
|
method: 'DELETE',
|
||||||
|
headers: this.authHeader(),
|
||||||
|
});
|
||||||
|
if (!res.ok) throw new Error(await res.text());
|
||||||
|
this.savedSearches = this.savedSearches.filter((s) => s.id !== id);
|
||||||
|
} catch (err) {
|
||||||
|
this.statusText = err.message || 'Failed to delete saved search.';
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
resetPagination() {
|
resetPagination() {
|
||||||
this.cursorStack = [];
|
this.cursorStack = [];
|
||||||
this.nextCursor = null;
|
this.nextCursor = null;
|
||||||
@@ -550,7 +617,7 @@
|
|||||||
},
|
},
|
||||||
|
|
||||||
clearFilters() {
|
clearFilters() {
|
||||||
const noisy = ['Exchange', 'SharePoint'];
|
const noisy = ['Exchange', 'SharePoint', 'Teams'];
|
||||||
this.filters = { actor: '', selectedServices: this.options.services.filter((s) => !noisy.includes(s)), search: '', operation: '', result: '', start: '', end: '', limit: 100, includeTags: '', excludeTags: '' };
|
this.filters = { actor: '', selectedServices: this.options.services.filter((s) => !noisy.includes(s)), search: '', operation: '', result: '', start: '', end: '', limit: 100, includeTags: '', excludeTags: '' };
|
||||||
this.saveFilters();
|
this.saveFilters();
|
||||||
this.resetPagination();
|
this.resetPagination();
|
||||||
|
|||||||
@@ -370,6 +370,14 @@ input {
|
|||||||
align-items: center;
|
align-items: center;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.saved-searches {
|
||||||
|
display: flex;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
gap: 8px;
|
||||||
|
align-items: center;
|
||||||
|
font-size: 13px;
|
||||||
|
}
|
||||||
|
|
||||||
.modal__explanation {
|
.modal__explanation {
|
||||||
background: rgba(255, 255, 255, 0.03);
|
background: rgba(255, 255, 255, 0.03);
|
||||||
border: 1px solid var(--border);
|
border: 1px solid var(--border);
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ from routes.fetch import router as fetch_router
|
|||||||
from routes.fetch import run_fetch
|
from routes.fetch import run_fetch
|
||||||
from routes.health import router as health_router
|
from routes.health import router as health_router
|
||||||
from routes.rules import router as rules_router
|
from routes.rules import router as rules_router
|
||||||
|
from routes.saved_searches import router as saved_searches_router
|
||||||
from routes.webhooks import router as webhooks_router
|
from routes.webhooks import router as webhooks_router
|
||||||
|
|
||||||
|
|
||||||
@@ -119,6 +120,7 @@ if AI_FEATURES_ENABLED:
|
|||||||
from routes.mcp import mcp_asgi
|
from routes.mcp import mcp_asgi
|
||||||
|
|
||||||
app.mount("/mcp", mcp_asgi)
|
app.mount("/mcp", mcp_asgi)
|
||||||
|
app.include_router(saved_searches_router, prefix="/api")
|
||||||
app.include_router(rules_router, prefix="/api")
|
app.include_router(rules_router, prefix="/api")
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,11 +1,21 @@
|
|||||||
|
import asyncio
|
||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
from datetime import UTC, datetime, timedelta
|
from datetime import UTC, datetime, timedelta
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
import structlog
|
import structlog
|
||||||
from auth import require_auth
|
from auth import require_auth, user_can_access_privacy_services
|
||||||
from config import LLM_API_KEY, LLM_API_VERSION, LLM_BASE_URL, LLM_MAX_EVENTS, LLM_MODEL, LLM_TIMEOUT_SECONDS
|
from config import (
|
||||||
|
LLM_API_KEY,
|
||||||
|
LLM_API_VERSION,
|
||||||
|
LLM_BASE_URL,
|
||||||
|
LLM_MAX_EVENTS,
|
||||||
|
LLM_MODEL,
|
||||||
|
LLM_TIMEOUT_SECONDS,
|
||||||
|
PRIVACY_SENSITIVE_OPERATIONS,
|
||||||
|
PRIVACY_SERVICES,
|
||||||
|
)
|
||||||
from database import events_collection
|
from database import events_collection
|
||||||
from fastapi import APIRouter, Depends, HTTPException
|
from fastapi import APIRouter, Depends, HTTPException
|
||||||
from models.api import AskRequest, AskResponse
|
from models.api import AskRequest, AskResponse
|
||||||
@@ -49,7 +59,7 @@ _SERVICE_INTENTS = {
|
|||||||
|
|
||||||
# Services that are extremely noisy for typical admin questions.
|
# Services that are extremely noisy for typical admin questions.
|
||||||
# We exclude them by default on broad questions unless the user explicitly mentions them.
|
# We exclude them by default on broad questions unless the user explicitly mentions them.
|
||||||
_NOISY_SERVICES = {"Exchange", "SharePoint"}
|
_NOISY_SERVICES = {"Exchange", "SharePoint", "Teams"}
|
||||||
|
|
||||||
# Services that are generally admin-relevant and kept by default.
|
# Services that are generally admin-relevant and kept by default.
|
||||||
_DEFAULT_ADMIN_SERVICES = {
|
_DEFAULT_ADMIN_SERVICES = {
|
||||||
@@ -471,11 +481,73 @@ Do not invent facts that are not in the data.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
_GUID_RE = re.compile(r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$")
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_guids(obj: dict | list | str) -> set[str]:
|
||||||
|
"""Recursively extract UUID-like strings from a JSON structure."""
|
||||||
|
guids = set()
|
||||||
|
if isinstance(obj, dict):
|
||||||
|
for k, v in obj.items():
|
||||||
|
if k.lower() in ("id", "groupid", "userid", "targetid") and isinstance(v, str) and _GUID_RE.match(v):
|
||||||
|
guids.add(v)
|
||||||
|
guids.update(_extract_guids(v))
|
||||||
|
elif isinstance(obj, list):
|
||||||
|
for item in obj:
|
||||||
|
guids.update(_extract_guids(item))
|
||||||
|
elif isinstance(obj, str) and _GUID_RE.match(obj):
|
||||||
|
guids.add(obj)
|
||||||
|
return guids
|
||||||
|
|
||||||
|
|
||||||
|
async def _resolve_guids_for_event(event: dict) -> dict[str, str]:
|
||||||
|
"""Try to resolve GUIDs in an event to human-readable names via Graph API."""
|
||||||
|
raw = event.get("raw") or {}
|
||||||
|
guids = _extract_guids(raw)
|
||||||
|
# Also include any GUIDs in targetResources that might not have displayName
|
||||||
|
for tr in raw.get("targetResources") or []:
|
||||||
|
tid = tr.get("id")
|
||||||
|
if tid and _GUID_RE.match(tid):
|
||||||
|
guids.add(tid)
|
||||||
|
for tr in raw.get("modifiedProperties") or []:
|
||||||
|
for key in ("oldValue", "newValue"):
|
||||||
|
val = tr.get(key)
|
||||||
|
if val and _GUID_RE.match(val):
|
||||||
|
guids.add(val)
|
||||||
|
|
||||||
|
if not guids:
|
||||||
|
return {}
|
||||||
|
|
||||||
|
try:
|
||||||
|
from graph.auth import get_access_token
|
||||||
|
from graph.resolve import resolve_directory_object
|
||||||
|
|
||||||
|
token = await asyncio.to_thread(get_access_token)
|
||||||
|
cache: dict[str, dict] = {}
|
||||||
|
resolved = {}
|
||||||
|
for gid in guids:
|
||||||
|
result = await asyncio.to_thread(resolve_directory_object, gid, token, cache)
|
||||||
|
if result:
|
||||||
|
resolved[gid] = result["name"]
|
||||||
|
return resolved
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning("GUID resolution failed", error=str(exc))
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
async def _explain_event(event: dict, related: list[dict]) -> str:
|
async def _explain_event(event: dict, related: list[dict]) -> str:
|
||||||
if not LLM_API_KEY:
|
if not LLM_API_KEY:
|
||||||
raise RuntimeError("LLM_API_KEY not configured")
|
raise RuntimeError("LLM_API_KEY not configured")
|
||||||
|
|
||||||
|
# Resolve GUIDs to names before sending to LLM
|
||||||
|
resolved = await _resolve_guids_for_event(event)
|
||||||
|
|
||||||
event_text = json.dumps(event, indent=2, default=str)
|
event_text = json.dumps(event, indent=2, default=str)
|
||||||
|
resolution_text = ""
|
||||||
|
if resolved:
|
||||||
|
resolution_text = "\nResolved GUIDs:\n"
|
||||||
|
for gid, name in resolved.items():
|
||||||
|
resolution_text += f" {gid} → {name}\n"
|
||||||
|
|
||||||
related_text = ""
|
related_text = ""
|
||||||
if related:
|
if related:
|
||||||
@@ -492,7 +564,7 @@ async def _explain_event(event: dict, related: list[dict]) -> str:
|
|||||||
{"role": "system", "content": _EXPLAIN_SYSTEM_PROMPT},
|
{"role": "system", "content": _EXPLAIN_SYSTEM_PROMPT},
|
||||||
{
|
{
|
||||||
"role": "user",
|
"role": "user",
|
||||||
"content": f"Audit event:\n{event_text}{related_text}\n\nPlease explain this event.",
|
"content": f"Audit event:\n{event_text}{resolution_text}{related_text}\n\nPlease explain this event.",
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -525,6 +597,11 @@ async def explain_event(event_id: str, user: dict = Depends(require_auth)):
|
|||||||
if not event:
|
if not event:
|
||||||
raise HTTPException(status_code=404, detail="Event not found")
|
raise HTTPException(status_code=404, detail="Event not found")
|
||||||
|
|
||||||
|
if (
|
||||||
|
event.get("service") in PRIVACY_SERVICES or event.get("operation") in PRIVACY_SENSITIVE_OPERATIONS
|
||||||
|
) and not user_can_access_privacy_services(user):
|
||||||
|
raise HTTPException(status_code=403, detail="Access to this event is restricted")
|
||||||
|
|
||||||
event.pop("_id", None)
|
event.pop("_id", None)
|
||||||
|
|
||||||
# Fetch related events for context (same actor or target in last 24h)
|
# Fetch related events for context (same actor or target in last 24h)
|
||||||
@@ -615,6 +692,8 @@ async def ask_question(body: AskRequest, user: dict = Depends(require_auth)):
|
|||||||
# -----------------------------------------------------------------------
|
# -----------------------------------------------------------------------
|
||||||
# Build and run query
|
# Build and run query
|
||||||
# -----------------------------------------------------------------------
|
# -----------------------------------------------------------------------
|
||||||
|
privacy_excluded_services = [] if user_can_access_privacy_services(user) else list(PRIVACY_SERVICES)
|
||||||
|
privacy_excluded_ops = [] if user_can_access_privacy_services(user) else list(PRIVACY_SENSITIVE_OPERATIONS)
|
||||||
query = _build_event_query(
|
query = _build_event_query(
|
||||||
entity,
|
entity,
|
||||||
start,
|
start,
|
||||||
@@ -626,6 +705,13 @@ async def ask_question(body: AskRequest, user: dict = Depends(require_auth)):
|
|||||||
include_tags=body.include_tags,
|
include_tags=body.include_tags,
|
||||||
exclude_tags=body.exclude_tags,
|
exclude_tags=body.exclude_tags,
|
||||||
)
|
)
|
||||||
|
extra_filters = []
|
||||||
|
if privacy_excluded_services:
|
||||||
|
extra_filters.append({"service": {"$nin": privacy_excluded_services}})
|
||||||
|
if privacy_excluded_ops:
|
||||||
|
extra_filters.append({"operation": {"$nin": privacy_excluded_ops}})
|
||||||
|
if extra_filters:
|
||||||
|
query["$and"] = query.get("$and", []) + extra_filters
|
||||||
|
|
||||||
try:
|
try:
|
||||||
total = events_collection.count_documents(query)
|
total = events_collection.count_documents(query)
|
||||||
|
|||||||
@@ -3,8 +3,9 @@ import re
|
|||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
from audit_trail import log_action
|
from audit_trail import log_action
|
||||||
from auth import require_auth
|
from auth import require_auth, user_can_access_privacy_services
|
||||||
from bson import ObjectId
|
from bson import ObjectId
|
||||||
|
from config import PRIVACY_SENSITIVE_OPERATIONS, PRIVACY_SERVICES
|
||||||
from database import events_collection
|
from database import events_collection
|
||||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||||
from models.api import (
|
from models.api import (
|
||||||
@@ -44,6 +45,7 @@ def _build_query(
|
|||||||
cursor: str | None = None,
|
cursor: str | None = None,
|
||||||
include_tags: list[str] | None = None,
|
include_tags: list[str] | None = None,
|
||||||
exclude_tags: list[str] | None = None,
|
exclude_tags: list[str] | None = None,
|
||||||
|
exclude_operations: list[str] | None = None,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
filters = []
|
filters = []
|
||||||
|
|
||||||
@@ -51,6 +53,8 @@ def _build_query(
|
|||||||
filters.append({"service": service})
|
filters.append({"service": service})
|
||||||
if services:
|
if services:
|
||||||
filters.append({"service": {"$in": services}})
|
filters.append({"service": {"$in": services}})
|
||||||
|
if exclude_operations:
|
||||||
|
filters.append({"operation": {"$nin": exclude_operations}})
|
||||||
if actor:
|
if actor:
|
||||||
actor_safe = re.escape(actor)
|
actor_safe = re.escape(actor)
|
||||||
filters.append(
|
filters.append(
|
||||||
@@ -125,6 +129,8 @@ def list_events(
|
|||||||
exclude_tags: list[str] | None = Query(default=None),
|
exclude_tags: list[str] | None = Query(default=None),
|
||||||
user: dict = Depends(require_auth),
|
user: dict = Depends(require_auth),
|
||||||
):
|
):
|
||||||
|
privacy_excluded_services = [] if user_can_access_privacy_services(user) else list(PRIVACY_SERVICES)
|
||||||
|
privacy_excluded_ops = [] if user_can_access_privacy_services(user) else list(PRIVACY_SENSITIVE_OPERATIONS)
|
||||||
query = _build_query(
|
query = _build_query(
|
||||||
service=service,
|
service=service,
|
||||||
services=services,
|
services=services,
|
||||||
@@ -137,7 +143,13 @@ def list_events(
|
|||||||
cursor=cursor,
|
cursor=cursor,
|
||||||
include_tags=include_tags,
|
include_tags=include_tags,
|
||||||
exclude_tags=exclude_tags,
|
exclude_tags=exclude_tags,
|
||||||
|
exclude_operations=privacy_excluded_ops,
|
||||||
)
|
)
|
||||||
|
if privacy_excluded_services:
|
||||||
|
query = query if query else {}
|
||||||
|
if "$and" not in query:
|
||||||
|
query = {"$and": [query]} if query else {"$and": []}
|
||||||
|
query["$and"].append({"service": {"$nin": privacy_excluded_services}})
|
||||||
|
|
||||||
safe_page_size = max(1, min(page_size, 500))
|
safe_page_size = max(1, min(page_size, 500))
|
||||||
|
|
||||||
@@ -202,6 +214,8 @@ def bulk_tags(
|
|||||||
exclude_tags: list[str] | None = Query(default=None),
|
exclude_tags: list[str] | None = Query(default=None),
|
||||||
user: dict = Depends(require_auth),
|
user: dict = Depends(require_auth),
|
||||||
):
|
):
|
||||||
|
privacy_excluded_services = [] if user_can_access_privacy_services(user) else list(PRIVACY_SERVICES)
|
||||||
|
privacy_excluded_ops = [] if user_can_access_privacy_services(user) else list(PRIVACY_SENSITIVE_OPERATIONS)
|
||||||
query = _build_query(
|
query = _build_query(
|
||||||
service=service,
|
service=service,
|
||||||
services=services,
|
services=services,
|
||||||
@@ -213,7 +227,13 @@ def bulk_tags(
|
|||||||
search=search,
|
search=search,
|
||||||
include_tags=include_tags,
|
include_tags=include_tags,
|
||||||
exclude_tags=exclude_tags,
|
exclude_tags=exclude_tags,
|
||||||
|
exclude_operations=privacy_excluded_ops,
|
||||||
)
|
)
|
||||||
|
if privacy_excluded_services:
|
||||||
|
query = query if query else {}
|
||||||
|
if "$and" not in query:
|
||||||
|
query = {"$and": [query]} if query else {"$and": []}
|
||||||
|
query["$and"].append({"service": {"$nin": privacy_excluded_services}})
|
||||||
tags = [t.strip() for t in body.tags if t.strip()]
|
tags = [t.strip() for t in body.tags if t.strip()]
|
||||||
if not tags:
|
if not tags:
|
||||||
raise HTTPException(status_code=400, detail="No tags provided")
|
raise HTTPException(status_code=400, detail="No tags provided")
|
||||||
@@ -235,7 +255,10 @@ def bulk_tags(
|
|||||||
|
|
||||||
|
|
||||||
@router.get("/filter-options", response_model=FilterOptionsResponse)
|
@router.get("/filter-options", response_model=FilterOptionsResponse)
|
||||||
def filter_options(limit: int = Query(default=200, ge=1, le=1000)):
|
def filter_options(
|
||||||
|
limit: int = Query(default=200, ge=1, le=1000),
|
||||||
|
user: dict = Depends(require_auth),
|
||||||
|
):
|
||||||
safe_limit = max(1, min(limit, 1000))
|
safe_limit = max(1, min(limit, 1000))
|
||||||
try:
|
try:
|
||||||
services = sorted(events_collection.distinct("service"))[:safe_limit]
|
services = sorted(events_collection.distinct("service"))[:safe_limit]
|
||||||
@@ -247,6 +270,10 @@ def filter_options(limit: int = Query(default=200, ge=1, le=1000)):
|
|||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
raise HTTPException(status_code=500, detail=f"Failed to load filter options: {exc}") from exc
|
raise HTTPException(status_code=500, detail=f"Failed to load filter options: {exc}") from exc
|
||||||
|
|
||||||
|
if not user_can_access_privacy_services(user):
|
||||||
|
services = [s for s in services if s not in PRIVACY_SERVICES]
|
||||||
|
operations = [o for o in operations if o not in PRIVACY_SENSITIVE_OPERATIONS]
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"services": services,
|
"services": services,
|
||||||
"operations": operations,
|
"operations": operations,
|
||||||
|
|||||||
60
backend/routes/saved_searches.py
Normal file
60
backend/routes/saved_searches.py
Normal file
@@ -0,0 +1,60 @@
|
|||||||
|
"""CRUD for saved filter searches (bookmarks)."""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
import structlog
|
||||||
|
from auth import require_auth
|
||||||
|
from database import saved_searches_collection
|
||||||
|
from fastapi import APIRouter, Depends, HTTPException
|
||||||
|
|
||||||
|
router = APIRouter(dependencies=[Depends(require_auth)])
|
||||||
|
logger = structlog.get_logger("aoc.saved_searches")
|
||||||
|
|
||||||
|
|
||||||
|
def _user_sub(user: dict) -> str:
|
||||||
|
return user.get("sub", "anonymous")
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/saved-searches")
|
||||||
|
async def list_saved_searches(user: dict = Depends(require_auth)):
|
||||||
|
"""Return saved searches for the current user."""
|
||||||
|
sub = _user_sub(user)
|
||||||
|
cursor = saved_searches_collection.find({"created_by": sub}).sort("created_at", -1)
|
||||||
|
items = []
|
||||||
|
for doc in cursor:
|
||||||
|
doc["id"] = doc.pop("_id")
|
||||||
|
items.append(doc)
|
||||||
|
return items
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/saved-searches")
|
||||||
|
async def create_saved_search(body: dict, user: dict = Depends(require_auth)):
|
||||||
|
"""Save the current filter set."""
|
||||||
|
name = (body.get("name") or "").strip()
|
||||||
|
if not name:
|
||||||
|
raise HTTPException(status_code=400, detail="Name is required")
|
||||||
|
|
||||||
|
filters = body.get("filters") or {}
|
||||||
|
doc = {
|
||||||
|
"_id": str(uuid.uuid4()),
|
||||||
|
"name": name,
|
||||||
|
"filters": filters,
|
||||||
|
"created_at": datetime.now(UTC).isoformat().replace("+00:00", "Z"),
|
||||||
|
"created_by": _user_sub(user),
|
||||||
|
}
|
||||||
|
saved_searches_collection.insert_one(doc)
|
||||||
|
logger.info("Saved search created", name=name, user=doc["created_by"])
|
||||||
|
doc["id"] = doc.pop("_id")
|
||||||
|
return doc
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/saved-searches/{search_id}")
|
||||||
|
async def delete_saved_search(search_id: str, user: dict = Depends(require_auth)):
|
||||||
|
"""Delete a saved search (only if owned by current user)."""
|
||||||
|
sub = _user_sub(user)
|
||||||
|
result = saved_searches_collection.delete_one({"_id": search_id, "created_by": sub})
|
||||||
|
if result.deleted_count == 0:
|
||||||
|
raise HTTPException(status_code=404, detail="Saved search not found")
|
||||||
|
logger.info("Saved search deleted", search_id=search_id, user=sub)
|
||||||
|
return {"status": "deleted"}
|
||||||
@@ -22,15 +22,23 @@ def mock_watermarks_collection():
|
|||||||
@pytest.fixture(scope="function")
|
@pytest.fixture(scope="function")
|
||||||
def client(mock_events_collection, mock_watermarks_collection, monkeypatch):
|
def client(mock_events_collection, mock_watermarks_collection, monkeypatch):
|
||||||
monkeypatch.setattr("database.events_collection", mock_events_collection)
|
monkeypatch.setattr("database.events_collection", mock_events_collection)
|
||||||
|
monkeypatch.setattr("database.saved_searches_collection", mock_events_collection)
|
||||||
monkeypatch.setattr("routes.fetch.events_collection", mock_events_collection)
|
monkeypatch.setattr("routes.fetch.events_collection", mock_events_collection)
|
||||||
monkeypatch.setattr("routes.events.events_collection", mock_events_collection)
|
monkeypatch.setattr("routes.events.events_collection", mock_events_collection)
|
||||||
monkeypatch.setattr("routes.ask.events_collection", mock_events_collection)
|
monkeypatch.setattr("routes.ask.events_collection", mock_events_collection)
|
||||||
|
monkeypatch.setattr("routes.saved_searches.saved_searches_collection", mock_events_collection)
|
||||||
monkeypatch.setattr("watermark.watermarks_collection", mock_watermarks_collection)
|
monkeypatch.setattr("watermark.watermarks_collection", mock_watermarks_collection)
|
||||||
monkeypatch.setattr("routes.health.watermarks_collection", mock_watermarks_collection)
|
monkeypatch.setattr("routes.health.watermarks_collection", mock_watermarks_collection)
|
||||||
monkeypatch.setattr("routes.fetch.get_watermark", lambda source: None)
|
monkeypatch.setattr("routes.fetch.get_watermark", lambda source: None)
|
||||||
monkeypatch.setattr("routes.fetch.set_watermark", lambda source, ts: None)
|
monkeypatch.setattr("routes.fetch.set_watermark", lambda source, ts: None)
|
||||||
monkeypatch.setattr("auth.AUTH_ENABLED", False)
|
monkeypatch.setattr("auth.AUTH_ENABLED", False)
|
||||||
monkeypatch.setattr("routes.mcp.AUTH_ENABLED", False)
|
monkeypatch.setattr("routes.mcp.AUTH_ENABLED", False)
|
||||||
|
monkeypatch.setattr("config.PRIVACY_SERVICES", set())
|
||||||
|
monkeypatch.setattr("config.PRIVACY_SENSITIVE_OPERATIONS", set())
|
||||||
|
monkeypatch.setattr("routes.events.PRIVACY_SERVICES", set())
|
||||||
|
monkeypatch.setattr("routes.events.PRIVACY_SENSITIVE_OPERATIONS", set())
|
||||||
|
monkeypatch.setattr("routes.ask.PRIVACY_SERVICES", set())
|
||||||
|
monkeypatch.setattr("routes.ask.PRIVACY_SENSITIVE_OPERATIONS", set())
|
||||||
monkeypatch.setattr("database.db.command", lambda cmd: {"ok": 1} if cmd == "ping" else {})
|
monkeypatch.setattr("database.db.command", lambda cmd: {"ok": 1} if cmd == "ping" else {})
|
||||||
|
|
||||||
# Mock audit trail and rules collections so tests don't wait on real MongoDB
|
# Mock audit trail and rules collections so tests don't wait on real MongoDB
|
||||||
|
|||||||
@@ -107,6 +107,146 @@ def test_explain_event_with_llm_mock(client, mock_events_collection, monkeypatch
|
|||||||
assert data["llm_used"] is True
|
assert data["llm_used"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_saved_searches_crud(client, monkeypatch):
|
||||||
|
monkeypatch.setattr("auth.AUTH_ENABLED", False)
|
||||||
|
|
||||||
|
# Create
|
||||||
|
response = client.post(
|
||||||
|
"/api/saved-searches", json={"name": "Test search", "filters": {"actor": "alice", "result": "success"}}
|
||||||
|
)
|
||||||
|
assert response.status_code == 200
|
||||||
|
created = response.json()
|
||||||
|
assert created["name"] == "Test search"
|
||||||
|
assert created["filters"]["actor"] == "alice"
|
||||||
|
search_id = created["id"]
|
||||||
|
|
||||||
|
# List
|
||||||
|
response2 = client.get("/api/saved-searches")
|
||||||
|
assert response2.status_code == 200
|
||||||
|
items = response2.json()
|
||||||
|
assert len(items) == 1
|
||||||
|
assert items[0]["name"] == "Test search"
|
||||||
|
|
||||||
|
# Delete
|
||||||
|
response3 = client.delete(f"/api/saved-searches/{search_id}")
|
||||||
|
assert response3.status_code == 200
|
||||||
|
|
||||||
|
# List empty
|
||||||
|
response4 = client.get("/api/saved-searches")
|
||||||
|
assert response4.status_code == 200
|
||||||
|
assert len(response4.json()) == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_saved_searches_delete_not_found(client, monkeypatch):
|
||||||
|
monkeypatch.setattr("auth.AUTH_ENABLED", False)
|
||||||
|
response = client.delete("/api/saved-searches/nonexistent")
|
||||||
|
assert response.status_code == 404
|
||||||
|
|
||||||
|
|
||||||
|
def test_saved_searches_create_validation(client, monkeypatch):
|
||||||
|
monkeypatch.setattr("auth.AUTH_ENABLED", False)
|
||||||
|
response = client.post("/api/saved-searches", json={"name": " ", "filters": {}})
|
||||||
|
assert response.status_code == 400
|
||||||
|
|
||||||
|
|
||||||
|
def test_privacy_filtering_events_by_operation(client, mock_events_collection, monkeypatch):
|
||||||
|
monkeypatch.setattr("config.PRIVACY_SENSITIVE_OPERATIONS", {"MailItemsAccessed", "Send"})
|
||||||
|
monkeypatch.setattr("routes.events.PRIVACY_SENSITIVE_OPERATIONS", {"MailItemsAccessed", "Send"})
|
||||||
|
monkeypatch.setattr("auth.PRIVACY_SERVICE_ROLES", {"SecurityAdmin"})
|
||||||
|
monkeypatch.setattr("auth.user_can_access_privacy_services", lambda claims: False)
|
||||||
|
monkeypatch.setattr("routes.events.user_can_access_privacy_services", lambda claims: False)
|
||||||
|
|
||||||
|
mock_events_collection.insert_one(
|
||||||
|
{
|
||||||
|
"id": "evt-safe",
|
||||||
|
"timestamp": datetime.now(UTC).isoformat(),
|
||||||
|
"service": "Exchange",
|
||||||
|
"operation": "Add-MailboxPermission",
|
||||||
|
"result": "success",
|
||||||
|
"actor_display": "Alice",
|
||||||
|
"raw_text": "",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
mock_events_collection.insert_one(
|
||||||
|
{
|
||||||
|
"id": "evt-priv",
|
||||||
|
"timestamp": datetime.now(UTC).isoformat(),
|
||||||
|
"service": "Exchange",
|
||||||
|
"operation": "Send",
|
||||||
|
"result": "success",
|
||||||
|
"actor_display": "Bob",
|
||||||
|
"raw_text": "",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
response = client.get("/api/events")
|
||||||
|
assert response.status_code == 200
|
||||||
|
data = response.json()
|
||||||
|
ids = [e["id"] for e in data["items"]]
|
||||||
|
assert "evt-safe" in ids
|
||||||
|
assert "evt-priv" not in ids
|
||||||
|
|
||||||
|
|
||||||
|
def test_privacy_filter_options_shows_service_hides_ops(client, mock_events_collection, monkeypatch):
|
||||||
|
monkeypatch.setattr("config.PRIVACY_SENSITIVE_OPERATIONS", {"MailItemsAccessed"})
|
||||||
|
monkeypatch.setattr("routes.events.PRIVACY_SENSITIVE_OPERATIONS", {"MailItemsAccessed"})
|
||||||
|
monkeypatch.setattr("auth.PRIVACY_SERVICE_ROLES", {"SecurityAdmin"})
|
||||||
|
monkeypatch.setattr("auth.user_can_access_privacy_services", lambda claims: False)
|
||||||
|
monkeypatch.setattr("routes.events.user_can_access_privacy_services", lambda claims: False)
|
||||||
|
|
||||||
|
mock_events_collection.insert_one(
|
||||||
|
{
|
||||||
|
"id": "evt-1",
|
||||||
|
"timestamp": datetime.now(UTC).isoformat(),
|
||||||
|
"service": "Exchange",
|
||||||
|
"operation": "MailItemsAccessed",
|
||||||
|
"result": "success",
|
||||||
|
"actor_display": "Alice",
|
||||||
|
"raw_text": "",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
mock_events_collection.insert_one(
|
||||||
|
{
|
||||||
|
"id": "evt-2",
|
||||||
|
"timestamp": datetime.now(UTC).isoformat(),
|
||||||
|
"service": "Exchange",
|
||||||
|
"operation": "Add-MailboxPermission",
|
||||||
|
"result": "success",
|
||||||
|
"actor_display": "Bob",
|
||||||
|
"raw_text": "",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
response = client.get("/api/filter-options")
|
||||||
|
assert response.status_code == 200
|
||||||
|
data = response.json()
|
||||||
|
assert "Exchange" in data["services"]
|
||||||
|
assert "MailItemsAccessed" not in data["operations"]
|
||||||
|
assert "Add-MailboxPermission" in data["operations"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_privacy_explain_forbidden_by_operation(client, mock_events_collection, monkeypatch):
|
||||||
|
monkeypatch.setattr("config.PRIVACY_SENSITIVE_OPERATIONS", {"Send"})
|
||||||
|
monkeypatch.setattr("routes.ask.PRIVACY_SENSITIVE_OPERATIONS", {"Send"})
|
||||||
|
monkeypatch.setattr("auth.PRIVACY_SERVICE_ROLES", {"SecurityAdmin"})
|
||||||
|
monkeypatch.setattr("auth.user_can_access_privacy_services", lambda claims: False)
|
||||||
|
monkeypatch.setattr("routes.ask.user_can_access_privacy_services", lambda claims: False)
|
||||||
|
|
||||||
|
mock_events_collection.insert_one(
|
||||||
|
{
|
||||||
|
"id": "evt-send",
|
||||||
|
"timestamp": datetime.now(UTC).isoformat(),
|
||||||
|
"service": "Exchange",
|
||||||
|
"operation": "Send",
|
||||||
|
"result": "success",
|
||||||
|
"actor_display": "Bob",
|
||||||
|
"raw_text": "",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
response = client.post("/api/events/evt-send/explain")
|
||||||
|
assert response.status_code == 403
|
||||||
|
|
||||||
|
|
||||||
def test_health(client):
|
def test_health(client):
|
||||||
response = client.get("/health")
|
response = client.get("/health")
|
||||||
assert response.status_code == 200
|
assert response.status_code == 200
|
||||||
|
|||||||
Reference in New Issue
Block a user