Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

fix(2.4): scope the idle sweep to a tenant, and give it a caller

Beenu Aroraff9aca82a43a
diff --git a/services/fusion/app/services/sessions.py b/services/fusion/app/services/sessions.py
index dcba68caf..7b6559967 100644
--- a/services/fusion/app/services/sessions.py
+++ b/services/fusion/app/services/sessions.py
@@ -165,11 +165,32 @@ ON CONFLICT (id) DO UPDATE SET
#: Close derived sessions that have gone quiet. Vendor sessions are left
#: alone: the vendor says when those end, and guessing on their behalf would
#: report a session as over while it is still being used.
+#:
+#: Scoped to one tenant per call rather than sweeping the table. The first
+#: version had no tenant predicate and relied on the RLS policy's
+#: `current_tenant_id() IS NULL` arm, which a worker satisfies by setting
+#: nothing -- so one statement reached every tenant's rows and
+#: `check_tenant_query_predicates.py` failed the build for it. That gate is
+#: right: RLS is defence in depth here, not the control, and a maintenance
+#: sweep that can touch every tenant at once is the shape this repository
+#: has been bitten by before. It also bounds the work, so one noisy tenant
+#: cannot stall the sweep for the rest.
_CLOSE_IDLE_SQL = """
UPDATE sessions
SET ended_at = last_seen_at,
ended_reason = 'idle',
updated_at = now()
+ WHERE tenant_id = $1
+ AND ended_at IS NULL
+ AND key_kind = 'derived'
+ AND last_seen_at < now() - make_interval(secs => $2)
+"""
+
+#: The tenants with at least one open derived session. Drives the sweep so
+#: it visits only tenants that can have work, rather than every tenant row.
+_IDLE_TENANTS_SQL = """
+SELECT DISTINCT tenant_id
+ FROM sessions
WHERE ended_at IS NULL
AND key_kind = 'derived'
AND last_seen_at < now() - make_interval(secs => $1)
@@ -289,15 +310,34 @@ class SessionStore:
parent_id,
)
- async def close_idle(self, idle_seconds: int = DERIVED_IDLE_SECONDS) -> int:
- """Close derived sessions that have gone quiet. Returns rows closed."""
+ async def close_idle(
+ self,
+ idle_seconds: int = DERIVED_IDLE_SECONDS,
+ *,
+ tenant_id: str | None = None,
+ ) -> int:
+ """Close derived sessions that have gone quiet. Returns rows closed.
+
+ With ``tenant_id`` the sweep touches that tenant only. Without it the
+ tenants that *have* idle sessions are read first and swept one at a
+ time, so the UPDATE always carries a tenant predicate and one
+ tenant's backlog cannot stall the others.
+ """
try:
async with self._pool.acquire() as conn:
- result = await conn.execute(_CLOSE_IDLE_SQL, float(idle_seconds))
+ if tenant_id is not None:
+ result = await conn.execute(_CLOSE_IDLE_SQL, tenant_id, float(idle_seconds))
+ return _rows_affected(result)
+
+ rows = await conn.fetch(_IDLE_TENANTS_SQL, float(idle_seconds))
+ closed = 0
+ for row in rows:
+ result = await conn.execute(_CLOSE_IDLE_SQL, row["tenant_id"], float(idle_seconds))
+ closed += _rows_affected(result)
+ return closed
except Exception as exc: # noqa: BLE001 — same reasoning as record_event
logger.error("fusion.session.close_idle_failed", extra={"error": str(exc)}, exc_info=True)
return 0
- return _rows_affected(result)
def _session_block(payload: dict[str, Any]) -> dict[str, Any] | None:
diff --git a/services/fusion/app/workers/consumer.py b/services/fusion/app/workers/consumer.py
index 9c6662af7..f14ffeb32 100644
--- a/services/fusion/app/workers/consumer.py
+++ b/services/fusion/app/workers/consumer.py
@@ -7,6 +7,7 @@ results back to Kafka, and persists non-duplicate alerts to Postgres.
import asyncio
import contextlib
import json
+import time
from typing import Any
import structlog
@@ -22,7 +23,7 @@ from app.services.event_schema import validate_event
from app.services.fusion_engine import FusionEngine
from app.services.lake_writer import LakeWriter
from app.services.promoter import promote_normalized_event
-from app.services.sessions import SessionRef, SessionStore
+from app.services.sessions import DERIVED_IDLE_SECONDS, SessionRef, SessionStore
from app.services.tenant_overlay import OverlayCache
from app.services.ueba_signal import UebaSignalCache
from app.services.windowed_detection import WindowedDetectionEngine
@@ -50,6 +51,12 @@ _METRICS = {
}
+#: How often the worker sweeps for derived sessions that have gone quiet.
+#: A quarter of the idle window, so a session is closed within roughly
+#: DERIVED_IDLE_SECONDS of its last event rather than up to twice that.
+SESSION_SWEEP_SECONDS = DERIVED_IDLE_SECONDS / 4
+
+
class FusionWorker:
"""Kafka consumer/producer pair that drives the fusion pipeline."""
@@ -135,11 +142,19 @@ class FusionWorker:
await self._producer.start()
if self._sink is not None:
await self._sink.start()
- if self._lake is not None:
- await self._lake.start()
+ if self._lake is not None or self._sessions is not None:
+ if self._lake is not None:
+ await self._lake.start()
# Low-traffic safety net: a single buffered event would otherwise
# wait for the NEXT message before flush_if_stale runs. A background
# ticker guarantees the lake batch flushes by age even when idle.
+ #
+ # The same ticker closes idle derived sessions. A derived session
+ # ends when the activity stops, so nothing on the event path can
+ # ever close one -- the closing condition is the *absence* of an
+ # event. Without a ticker every derived session would stay open
+ # forever and "what else did this sitting do" would accumulate
+ # months of unrelated activity under one id.
self._flush_task = asyncio.create_task(self._periodic_flush())
self._running = True
logger.info(
@@ -154,11 +169,26 @@ class FusionWorker:
async def _periodic_flush(self) -> None:
interval = max(0.5, settings.lake_batch_max_age_seconds)
+ # Elapsed wall-clock rather than a tick count, because the two
+ # cadences are set independently: a tick count derived from the
+ # lake's flush interval silently changes the sweep period whenever
+ # someone tunes the lake, which is a coupling nobody would expect
+ # from either setting's name.
+ last_sweep = time.monotonic()
try:
while self._running:
await asyncio.sleep(interval)
if self._lake is not None:
await self._lake.flush_if_stale()
+ if self._sessions is None:
+ continue
+ now = time.monotonic()
+ if now - last_sweep < SESSION_SWEEP_SECONDS:
+ continue
+ last_sweep = now
+ closed = await self._sessions.close_idle()
+ if closed:
+ logger.info("Closed idle derived sessions", closed=closed)
except asyncio.CancelledError: # pragma: no cover — shutdown path
pass
diff --git a/services/fusion/tests/test_sessions.py b/services/fusion/tests/test_sessions.py
index 896c8d913..eaab8b9d9 100644
--- a/services/fusion/tests/test_sessions.py
+++ b/services/fusion/tests/test_sessions.py
@@ -9,6 +9,8 @@ arguments have to satisfy is asserted against the migration itself.
from __future__ import annotations
+import asyncio
+import contextlib
import json
import re
import uuid
@@ -18,6 +20,7 @@ from pathlib import Path
import pytest
from app.services.sessions import (
+ DERIVED_IDLE_SECONDS,
KEY_KINDS,
SessionStore,
session_id,
@@ -29,9 +32,12 @@ TENANT = "11111111-1111-1111-1111-111111111111"
class RecordingConn:
- def __init__(self, fail: Exception | None = None) -> None:
+ def __init__(self, fail: Exception | None = None, tenants: list[str] | None = None) -> None:
self.calls: list[tuple[str, tuple]] = []
self._fail = fail
+ # What `SELECT DISTINCT tenant_id` returns. asyncpg hands back
+ # mapping-like Records, so a dict is the honest stand-in.
+ self._tenants = tenants if tenants is not None else []
async def execute(self, sql: str, *args):
if self._fail is not None:
@@ -39,6 +45,15 @@ class RecordingConn:
self.calls.append((sql, args))
return "INSERT 0 1"
+ async def fetch(self, sql: str, *args):
+ if self._fail is not None:
+ raise self._fail
+ self.calls.append((sql, args))
+ return [{"tenant_id": t} for t in self._tenants]
+
+ def all_sql_containing(self, needle: str) -> list[tuple[str, tuple]]:
+ return [(sql, args) for sql, args in self.calls if needle in sql]
+
def sql_containing(self, needle: str) -> tuple[str, tuple]:
for sql, args in self.calls:
if needle in sql:
@@ -191,16 +206,16 @@ async def test_a_call_that_issued_its_own_credential_is_not_its_own_parent():
async def test_the_id_is_deterministic_across_stores_and_replays():
"""Three stores have to agree on one id without coordinating, and a Kafka
replay must upsert the same row rather than minting a second session."""
- first = session_id(TENANT, "okta", "102bFs0Aq6VQ1qjqVqk")
- second = session_id(TENANT, "okta", "102bFs0Aq6VQ1qjqVqk")
+ first = session_id(TENANT, "okta", "00okta0session0EXAMPLE0001")
+ second = session_id(TENANT, "okta", "00okta0session0EXAMPLE0001")
assert first == second
assert isinstance(first, uuid.UUID)
# Namespaced by tenant and by source: two tenants may legitimately observe
# the same vendor identifier, and an Okta session id and a GitHub token
# hash share no namespace.
- assert session_id("22222222-2222-2222-2222-222222222222", "okta", "102bFs0Aq6VQ1qjqVqk") != first
- assert session_id(TENANT, "github", "102bFs0Aq6VQ1qjqVqk") != first
+ assert session_id("22222222-2222-2222-2222-222222222222", "okta", "00okta0session0EXAMPLE0001") != first
+ assert session_id(TENANT, "github", "00okta0session0EXAMPLE0001") != first
@pytest.mark.asyncio
@@ -220,11 +235,45 @@ async def test_only_derived_sessions_are_closed_on_an_idle_gap():
conn = RecordingConn()
store = SessionStore(RecordingPool(conn))
- await store.close_idle(900)
+ await store.close_idle(900, tenant_id=TENANT)
sql, args = conn.sql_containing("UPDATE sessions")
assert "key_kind = 'derived'" in sql
- assert args[0] == 900.0
+ assert args == (TENANT, 900.0)
+
+
+async def test_the_idle_sweep_never_issues_an_update_without_a_tenant():
+ """The first version of this swept the whole table and leaned on the RLS
+ policy's `current_tenant_id() IS NULL` arm, which a worker satisfies by
+ setting nothing -- so one statement reached every tenant's rows and
+ `check_tenant_query_predicates.py` failed the build for it.
+
+ RLS is defence in depth here, not the control. Every UPDATE the sweep
+ issues names exactly one tenant."""
+ other = "33333333-3333-3333-3333-333333333333"
+ conn = RecordingConn(tenants=[TENANT, other])
+ store = SessionStore(RecordingPool(conn))
+
+ closed = await store.close_idle(900)
+
+ updates = conn.all_sql_containing("UPDATE sessions")
+ assert len(updates) == 2
+ assert {args[0] for _, args in updates} == {TENANT, other}
+ for sql, _ in updates:
+ assert "tenant_id = $1" in sql
+ assert closed == 2
+
+
+async def test_a_sweep_with_no_idle_tenants_issues_no_update_at_all():
+ """The control. A quiet deployment must not write, or the test above
+ would pass on a sweep that updates unconditionally."""
+ conn = RecordingConn(tenants=[])
+ store = SessionStore(RecordingPool(conn))
+
+ assert await store.close_idle(900) == 0
+ assert conn.all_sql_containing("UPDATE sessions") == []
def test_the_closed_set_matches_the_migrations_check_constraint():
@@ -262,3 +311,66 @@ def test_event_time_handles_every_shape_the_spine_delivers(raw, expect_year):
payload = {"time": raw} if raw is not None else {}
assert _event_time(payload).year == expect_year
+
+
+async def test_the_idle_sweep_runs_from_the_worker_rather_than_nowhere():
+ """`close_idle` had no production caller at all.
+
+ A derived session ends when the activity stops, so nothing on the event
+ path can ever close one -- the closing condition is the *absence* of an
+ event. Without a ticker every derived session stays open forever and
+ "what else did this sitting do" accumulates months of unrelated activity
+ under one id, which is worse than having no session grouping at all.
+
+ Drives the worker's real ticker with the real interval arithmetic rather
+ than asserting the call exists in the source.
+ """
+ from app.workers import consumer as consumer_mod
+
+ swept: list[int] = []
+
+ class SweepOnlyStore:
+ async def close_idle(self, *_a, **_kw) -> int:
+ swept.append(1)
+ return 0
+
+ worker = consumer_mod.FusionWorker.__new__(consumer_mod.FusionWorker)
+ worker._running = True
+ worker._lake = None
+ worker._sessions = SweepOnlyStore()
+
+ # The real period is a quarter of the idle window (~7 minutes). Shortened
+ # here so the test exercises the real loop and its real clock arithmetic
+ # rather than a reimplementation of it.
+ monkeypatch = pytest.MonkeyPatch()
+ monkeypatch.setattr(consumer_mod, "SESSION_SWEEP_SECONDS", 0.0)
+ monkeypatch.setattr(consumer_mod.settings, "lake_batch_max_age_seconds", 0.01)
+ try:
+ task = asyncio.create_task(worker._periodic_flush())
+ for _ in range(200):
+ await asyncio.sleep(0.01)
+ if swept:
+ break
+ worker._running = False
+ task.cancel()
+ with contextlib.suppress(asyncio.CancelledError):
+ await task
+ finally:
+ monkeypatch.undo()
+
+ assert swept, "the worker ticker never called close_idle"
+
+
+def test_the_sweep_period_is_shorter_than_the_window_it_sweeps_for():
+ """Two independently-written numbers describing one budget.
+
+ If the sweep ran less often than the idle window, a session would stay
+ open for up to the sum of the two rather than the window -- the shape
+ this repository has already been bitten by, where a caller's deadline
+ was a sixth of the retry budget it wrapped and the shorter number won
+ silently. Asserted as a relation, not as either value.
+ """
+ from app.workers.consumer import SESSION_SWEEP_SECONDS
+
+ assert 0 < SESSION_SWEEP_SECONDS <= DERIVED_IDLE_SECONDS / 2
diff --git a/services/ingest/internal/activity/session_test.go b/services/ingest/internal/activity/session_test.go
index 19a0655b5..ef87cece2 100644
--- a/services/ingest/internal/activity/session_test.go
+++ b/services/ingest/internal/activity/session_test.go
@@ -39,9 +39,9 @@ func TestSessionKeyComesFromTheVendorsOwnField(t *testing.T) {
connector: "okta",
vendor: map[string]any{
"actor": map[string]any{"alternateId": "[email protected]", "type": "User"},
- "authenticationContext": map[string]any{"externalSessionId": "102bFs0Aq6VQ1qjqVqk"},
+ "authenticationContext": map[string]any{"externalSessionId": "00okta0session0EXAMPLE0001"},
},
- wantKey: "102bFs0Aq6VQ1qjqVqk",
+ wantKey: "00okta0session0EXAMPLE0001",
wantKind: SessionVendorSession,
wantSource: "authenticationContext.externalSessionId",
},
@@ -61,9 +61,9 @@ func TestSessionKeyComesFromTheVendorsOwnField(t *testing.T) {
connector: "azure_entra",
vendor: map[string]any{
"userPrincipalName": "[email protected]",
- "correlationId": "9f8e7d6c-5b4a-3210-fedc-ba9876543210",
+ "correlationId": "00000000-1111-2222-3333-444444444444",
},
- wantKey: "9f8e7d6c-5b4a-3210-fedc-ba9876543210",
+ wantKey: "00000000-1111-2222-3333-444444444444",
wantKind: SessionVendorFlow,
wantSource: "correlationId",
},
@@ -374,7 +374,7 @@ func TestEverySessionKindIsInTheClosedSet(t *testing.T) {
// the vendor's own console. A hash here would be a value that exists nowhere
// else and cannot be looked up anywhere.
func TestTheKeyIsTheVendorsOwnValue(t *testing.T) {
- const raw = "102bFs0Aq6VQ1qjqVqk"
+ const raw = "00okta0session0EXAMPLE0001"
got := Project("okta", canonical("okta", map[string]any{
"actor": map[string]any{"alternateId": "[email protected]"},
"authenticationContext": map[string]any{"externalSessionId": raw},