cqrenet/aoc
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 29 Wiki Activity

feat: implement Phase 3 scaling
Some checks failed
CI / lint-and-test (push) Has been cancelled
Details

Browse Source 
- Replace skip-based pagination with cursor-based pagination (timestamp|_id cursors)
- Add Prometheus /metrics endpoint with request latency, fetch volume, and error counters
- Implement incremental fetch watermarking per source (watermarks collection in MongoDB)
- Add Graph change notification webhook endpoint (/api/webhooks/graph)
- Add correlation ID middleware for distributed tracing (x-request-id header)
- Update frontend to use cursor-based pagination with Prev/Next navigation
- Update tests for cursor pagination, metrics, webhooks, and watermark mocking
This commit is contained in:
Tomas Kracmar
2026-04-14 14:58:50 +02:00
parent 9271b4e461
commit b0198012eb
17 changed files with 402 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

229
backend/frontend/index.html
View File

@@ -101,14 +101,15 @@
const modalBody = document.getElementById('modalBody');
const closeModal = document.getElementById('closeModal');
let currentEvents = [];
let currentPage = 1;
let totalItems = 0;
let pageSize = 50;
let authConfig = null;
let msalInstance = null;
let account = null;
let accessToken = null;
let authScopes = [];
let cursorStack = [];
let nextCursor = null;
let currentCursor = null;
let authConfig = null;
let msalInstance = null;
let account = null;
let accessToken = null;
let authScopes = [];
const lists = {
actor: document.getElementById('actorOptions'),
service: document.getElementById('serviceOptions'),
@@ -122,9 +123,10 @@ let authScopes = [];
return isNaN(date.getTime()) ? '' : date.toISOString();
};
async function loadEvents() {
const params = new URLSearchParams();
const data = new FormData(form);
async function loadEvents(cursor) {
currentCursor = cursor || null;
const params = new URLSearchParams();
const data = new FormData(form);
['actor', 'service', 'operation', 'result', 'search'].forEach((key) => {
const val = data.get(key)?.trim();
if (val) params.append(key, val);
@@ -141,16 +143,18 @@ async function loadEvents() {
} else {
params.append('page_size', pageSize);
}
params.append('page', currentPage);
if (cursor) {
params.append('cursor', cursor);
}
status.textContent = 'Loading events…';
eventsContainer.innerHTML = '';
count.textContent = '';
status.textContent = 'Loading events…';
eventsContainer.innerHTML = '';
count.textContent = '';
if (authConfig?.auth_enabled && !accessToken) {
status.textContent = 'Please sign in to load events.';
return;
}
if (authConfig?.auth_enabled && !accessToken) {
status.textContent = 'Please sign in to load events.';
return;
}
try {
const res = await fetch(`/api/events?${params.toString()}`, { headers: { Accept: 'application/json', ...authHeader() } });
@@ -160,11 +164,10 @@ async function loadEvents() {
}
const body = await res.json();
const events = body.items || [];
totalItems = body.total || events.length;
pageSize = body.page_size || pageSize;
currentPage = body.page || currentPage;
nextCursor = body.next_cursor || null;
currentEvents = events;
renderEvents(events);
renderEvents(events, body.total);
renderPagination();
status.textContent = events.length ? '' : 'No events found for these filters.';
} catch (err) {
@@ -172,14 +175,14 @@ async function loadEvents() {
}
}
async function fetchLogs() {
status.textContent = 'Fetching latest audit logs…';
if (authConfig?.auth_enabled && !accessToken) {
status.textContent = 'Please sign in first.';
return;
}
try {
const res = await fetch('/api/fetch-audit-logs', { headers: authHeader() });
async function fetchLogs() {
status.textContent = 'Fetching latest audit logs…';
if (authConfig?.auth_enabled && !accessToken) {
status.textContent = 'Please sign in first.';
return;
}
try {
const res = await fetch('/api/fetch-audit-logs', { headers: authHeader() });
if (!res.ok) {
const msg = await res.text();
throw new Error(`Fetch failed: ${res.status} ${msg}`);
@@ -187,6 +190,7 @@ async function fetchLogs() {
const body = await res.json();
const errs = Array.isArray(body.errors) && body.errors.length ? `Warnings: ${body.errors.join(' | ')}` : '';
status.textContent = `Fetched and stored ${body.stored_events || 0} events.${errs ? ' ' + errs : ''} Refreshing list…`;
resetPagination();
await loadEvents();
} catch (err) {
status.textContent = err.message || 'Failed to fetch audit logs.';
@@ -212,8 +216,9 @@ async function fetchLogs() {
}
}
function renderEvents(events) {
count.textContent = totalItems ? `${totalItems} event${totalItems === 1 ? '' : 's'}` : '';
function renderEvents(events, total) {
const totalText = total >= 0 ? `${total} event${total === 1 ? '' : 's'}` : '';
count.textContent = totalText;
eventsContainer.innerHTML = events
.map((e, idx) => {
const actor =
@@ -272,16 +277,34 @@ async function fetchLogs() {
function renderPagination() {
const pagination = document.getElementById('pagination');
if (!pagination) return;
const totalPages = Math.max(1, Math.ceil((totalItems || 0) / (pageSize || 1)));
const hasPrev = cursorStack.length > 0;
const hasNext = !!nextCursor;
const currentPageNum = cursorStack.length + 1;
pagination.innerHTML = `
<button type="button" id="prevPage" ${currentPage <= 1 ? 'disabled' : ''}>Prev</button>
<span>Page ${currentPage} / ${totalPages}</span>
<button type="button" id="nextPage" ${currentPage >= totalPages ? 'disabled' : ''}>Next</button>
<button type="button" id="prevPage" ${hasPrev ? '' : 'disabled'}>Prev</button>
<span>Page ${currentPageNum}</span>
<button type="button" id="nextPage" ${hasNext ? '' : 'disabled'}>Next</button>
`;
const prev = document.getElementById('prevPage');
const next = document.getElementById('nextPage');
if (prev) prev.addEventListener('click', () => { if (currentPage > 1) { currentPage -= 1; loadEvents(); } });
if (next) next.addEventListener('click', () => { if (currentPage < totalPages) { currentPage += 1; loadEvents(); } });
if (prev) prev.addEventListener('click', () => {
if (cursorStack.length) {
const prevCursor = cursorStack.pop();
loadEvents(prevCursor);
}
});
if (next) next.addEventListener('click', () => {
if (nextCursor) {
cursorStack.push(currentCursor);
loadEvents(nextCursor);
}
});
}
function resetPagination() {
cursorStack = [];
nextCursor = null;
currentCursor = null;
}
function authHeader() {
@@ -290,11 +313,11 @@ async function fetchLogs() {
const pickToken = (res) => (res ? (res.accessToken || res.idToken || null) : null);
async function initAuth() {
try {
const res = await fetch('/api/config/auth');
authConfig = await res.json();
} catch {
async function initAuth() {
try {
const res = await fetch('/api/config/auth');
authConfig = await res.json();
} catch {
authConfig = { auth_enabled: false };
}
@@ -316,78 +339,76 @@ async function initAuth() {
['openid', 'profile', 'email', ...baseScope.split(/[ ,]+/).filter(Boolean)]
)
);
const authority = `https://login.microsoftonline.com/${tenantId}`;
const redirectUri = window.location.origin;
const authority = `https://login.microsoftonline.com/${tenantId}`;
const redirectUri = window.location.origin;
msalInstance = new msal.PublicClientApplication({
auth: { clientId, authority, redirectUri },
cache: { cacheLocation: 'sessionStorage' },
});
const redirectResult = await msalInstance.handleRedirectPromise().catch(() => null);
if (redirectResult) {
account = redirectResult.account;
msalInstance.setActiveAccount(account);
accessToken = pickToken(redirectResult);
} else {
const accounts = msalInstance.getAllAccounts();
if (accounts.length) {
account = accounts[0];
msalInstance.setActiveAccount(account);
accessToken = await acquireToken(authScopes);
const redirectResult = await msalInstance.handleRedirectPromise().catch(() => null);
if (redirectResult) {
account = redirectResult.account;
msalInstance.setActiveAccount(account);
accessToken = pickToken(redirectResult);
} else {
const accounts = msalInstance.getAllAccounts();
if (accounts.length) {
account = accounts[0];
msalInstance.setActiveAccount(account);
accessToken = await acquireToken(authScopes);
}
}
updateAuthButtons();
if (accessToken) {
await loadFilterOptions();
await loadEvents();
}
}
}
updateAuthButtons();
if (accessToken) {
await loadFilterOptions();
await loadEvents();
}
}
async function acquireToken(scopes) {
if (!msalInstance || !account) return null;
const request = { scopes: scopes && scopes.length ? scopes : ['openid', 'profile', 'email'], account };
try {
const res = await msalInstance.acquireTokenSilent(request);
return pickToken(res);
} catch {
const res = await msalInstance.acquireTokenPopup(request);
async function acquireToken(scopes) {
if (!msalInstance || !account) return null;
const request = { scopes: scopes && scopes.length ? scopes : ['openid', 'profile', 'email'], account };
try {
const res = await msalInstance.acquireTokenSilent(request);
return pickToken(res);
} catch {
const res = await msalInstance.acquireTokenPopup(request);
return pickToken(res);
}
}
function updateAuthButtons() {
const loggedIn = !!account;
if (authConfig?.auth_enabled) {
authBtn.textContent = loggedIn ? 'Logout' : 'Login';
}
if (loggedIn) {
// Refresh token silently on page load if needed.
acquireToken(authScopes).then((t) => { if (t) accessToken = t; }).catch(() => {});
status.textContent = '';
} else if (authConfig?.auth_enabled) {
status.textContent = 'Please log in to view events.';
}
}
authBtn.addEventListener('click', async () => {
if (!authConfig?.auth_enabled || !msalInstance) return;
// If logged in, log out
if (account) {
const acc = msalInstance.getActiveAccount();
accessToken = null;
account = null;
updateAuthButtons();
if (acc) {
await msalInstance.logoutPopup({ account: acc });
function updateAuthButtons() {
const loggedIn = !!account;
if (authConfig?.auth_enabled) {
authBtn.textContent = loggedIn ? 'Logout' : 'Login';
}
if (loggedIn) {
acquireToken(authScopes).then((t) => { if (t) accessToken = t; }).catch(() => {});
status.textContent = '';
} else if (authConfig?.auth_enabled) {
status.textContent = 'Please log in to view events.';
}
}
return;
}
const scopes = authScopes && authScopes.length ? authScopes : ['openid', 'profile', 'email'];
status.textContent = 'Redirecting to sign in...';
msalInstance.loginRedirect({ scopes });
});
authBtn.addEventListener('click', async () => {
if (!authConfig?.auth_enabled || !msalInstance) return;
if (account) {
const acc = msalInstance.getActiveAccount();
accessToken = null;
account = null;
updateAuthButtons();
if (acc) {
await msalInstance.logoutPopup({ account: acc });
}
return;
}
const scopes = authScopes && authScopes.length ? authScopes : ['openid', 'profile', 'email'];
status.textContent = 'Redirecting to sign in...';
msalInstance.loginRedirect({ scopes });
});
closeModal.addEventListener('click', () => modal.classList.add('hidden'));
modal.addEventListener('click', (e) => {
@@ -396,16 +417,16 @@ authBtn.addEventListener('click', async () => {
form.addEventListener('submit', (e) => {
e.preventDefault();
currentPage = 1;
resetPagination();
loadEvents();
});
fetchBtn.addEventListener('click', () => fetchLogs());
refreshBtn.addEventListener('click', () => loadEvents());
refreshBtn.addEventListener('click', () => loadEvents(currentCursor));
clearBtn.addEventListener('click', () => {
form.reset();
currentPage = 1;
resetPagination();
loadEvents();
});