From b35cac42e07dbe8eefa59f7e0002aff7dcda37dd Mon Sep 17 00:00:00 2001 From: Tomas Kracmar Date: Tue, 14 Apr 2026 15:38:39 +0200 Subject: [PATCH] feat: implement Phase 4 enhancements - Migrate frontend to Alpine.js for reactive state management - Add source health dashboard in UI and /api/source-health endpoint - Add event tagging (PATCH /api/events/{id}/tags) and commenting (POST /api/events/{id}/comments) - Add CSV/JSON export from the UI - Add rule-based alerting engine (rules.py) with CRUD endpoints (/api/rules) - Add SIEM export via webhook (siem.py) - Add AOC audit trail middleware logging all mutations to aoc_audit collection - Update config with SIEM_ENABLED, SIEM_WEBHOOK_URL, ALERTS_ENABLED - Add tests for rules engine, tags, comments, and source health --- .env.example | 7 + README.md | 9 + ROADMAP.md | 16 +- backend/audit_trail.py | 22 ++ backend/config.py | 11 + backend/frontend/index.html | 737 +++++++++++++++++++----------------- backend/main.py | 30 ++ backend/models/api.py | 26 +- backend/routes/events.py | 53 ++- backend/routes/fetch.py | 21 +- backend/routes/health.py | 30 ++ backend/routes/rules.py | 42 ++ backend/rules.py | 81 ++++ backend/siem.py | 17 + backend/tests/conftest.py | 9 + backend/tests/test_api.py | 77 +++- backend/tests/test_rules.py | 49 +++ pyproject.toml | 2 +- 18 files changed, 869 insertions(+), 370 deletions(-) create mode 100644 backend/audit_trail.py create mode 100644 backend/routes/health.py create mode 100644 backend/routes/rules.py create mode 100644 backend/rules.py create mode 100644 backend/siem.py create mode 100644 backend/tests/test_rules.py diff --git a/.env.example b/.env.example index c07cba4..c834846 100644 --- a/.env.example +++ b/.env.example @@ -23,3 +23,10 @@ RETENTION_DAYS=0 # Optional: comma-separated CORS origins (e.g., http://localhost:3000,https://app.example.com) CORS_ORIGINS=* + +# Optional: SIEM export webhook (e.g., Splunk HEC, Sentinel, or generic syslog webhook) +SIEM_ENABLED=false +SIEM_WEBHOOK_URL= + +# Optional: enable rule-based alerting during ingestion +ALERTS_ENABLED=false diff --git a/README.md b/README.md index 0029587..011eb57 100644 --- a/README.md +++ b/README.md @@ -72,11 +72,20 @@ uvicorn main:app --reload --host 0.0.0.0 --port 8000 - Intune audit logs (`/deviceManagement/auditEvents`) Dedupes on a stable key (source id or timestamp/category/operation/target). Returns count and per-source warnings. - **Incremental fetch**: each source remembers its last successful fetch time in MongoDB (`watermarks` collection). Subsequent calls fetch only new events since the watermark. + - **Alerting**: if `ALERTS_ENABLED=true`, events are evaluated against stored rules during ingestion. + - **SIEM export**: if `SIEM_ENABLED=true`, each ingested event is forwarded to `SIEM_WEBHOOK_URL`. - `GET /api/events` — list stored events with filters: - `service`, `actor`, `operation`, `result`, `start`, `end`, `search` (free text over raw/summary/actor/targets) - Pagination: `cursor`-based (`page_size` defaults to 50, max 500). Pass `cursor` from `next_cursor` to paginate forward. - `GET /api/filter-options` — best-effort distinct values for services, operations, results, actors (used by UI dropdowns). - `POST /api/webhooks/graph` — receive Microsoft Graph change notifications. Echoes `validationToken` when present. +- `GET /api/source-health` — last fetch status for each ingestion source (`directory`, `unified`, `intune`). +- `PATCH /api/events/{id}/tags` — update tags on an event (e.g., `investigating`, `false_positive`). +- `POST /api/events/{id}/comments` — add a comment to an event. +- `GET /api/rules` — list alert rules. +- `POST /api/rules` — create an alert rule. +- `PUT /api/rules/{id}` — update an alert rule. +- `DELETE /api/rules/{id}` — delete an alert rule. Stored document shape (collection `micro_soc.events`): ```json diff --git a/ROADMAP.md b/ROADMAP.md index 7c32f0b..a5be71e 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -46,16 +46,16 @@ Goal: handle larger data volumes and support real-time ingestion. --- -## Phase 4: Enhance +## Phase 4: Enhance ✅ Goal: evolve from a polling dashboard into a full security operations tool. -- [ ] Migrate frontend to a maintainable framework (Vue 3, React, or HTMX + Alpine.js) -- [ ] Add rule-based alerting (e.g., alert on privileged operations, after-hours activity) -- [ ] Add SIEM export (Splunk, Sentinel, syslog webhook) -- [ ] Build an audit trail for AOC itself (who queried what, who triggered fetches) -- [ ] Add event tagging and commenting (e.g., `investigating`, `false_positive`) -- [ ] Add export functionality (CSV / JSON) from the UI -- [ ] Add source health dashboard showing last fetch time and status per source +- [x] Migrate frontend to Alpine.js for better state management and maintainability +- [x] Add rule-based alerting (e.g., alert on privileged operations, after-hours activity) +- [x] Add SIEM export (Splunk, Sentinel, syslog webhook) +- [x] Build an audit trail for AOC itself (who queried what, who triggered fetches) +- [x] Add event tagging and commenting (e.g., `investigating`, `false_positive`) +- [x] Add export functionality (CSV / JSON) from the UI +- [x] Add source health dashboard showing last fetch time and status per source --- diff --git a/backend/audit_trail.py b/backend/audit_trail.py new file mode 100644 index 0000000..88c2733 --- /dev/null +++ b/backend/audit_trail.py @@ -0,0 +1,22 @@ +from datetime import UTC, datetime + +import structlog +from database import db + +logger = structlog.get_logger("aoc.audit") +audit_collection = db["aoc_audit"] + + +def log_action(action: str, resource: str, details: dict | None = None, user: str | None = None): + """Log an action in the AOC audit trail.""" + doc = { + "timestamp": datetime.now(UTC).isoformat(), + "action": action, + "resource": resource, + "details": details or {}, + "user": user or "anonymous", + } + try: + audit_collection.insert_one(doc) + except Exception as exc: + logger.warning("Failed to write audit trail", error=str(exc)) diff --git a/backend/config.py b/backend/config.py index ea6b896..e1d0725 100644 --- a/backend/config.py +++ b/backend/config.py @@ -35,6 +35,13 @@ class Settings(BaseSettings): # CORS CORS_ORIGINS: str = "*" + # SIEM export + SIEM_ENABLED: bool = False + SIEM_WEBHOOK_URL: str = "" + + # Alerting + ALERTS_ENABLED: bool = False + _settings = Settings() @@ -57,3 +64,7 @@ AUTH_ALLOWED_GROUPS = {g.strip() for g in _settings.AUTH_ALLOWED_GROUPS.split(", RETENTION_DAYS = _settings.RETENTION_DAYS CORS_ORIGINS = [o.strip() for o in _settings.CORS_ORIGINS.split(",") if o.strip()] + +SIEM_ENABLED = _settings.SIEM_ENABLED +SIEM_WEBHOOK_URL = _settings.SIEM_WEBHOOK_URL +ALERTS_ENABLED = _settings.ALERTS_ENABLED diff --git a/backend/frontend/index.html b/backend/frontend/index.html index 02f8e9e..eddb63c 100644 --- a/backend/frontend/index.html +++ b/backend/frontend/index.html @@ -5,10 +5,11 @@ AOC Events + -
+

Admin Operations Center

@@ -16,53 +17,76 @@

Filter Microsoft Entra audit events by user, app, time, action, and action type.

- - - + + +
-
+

Source Health

+
+ +
+
+ +
+
- + + +
@@ -70,367 +94,376 @@

Events

- + +
+
+
+ +
+ -
-
-
-