Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
fix(2.4): scope the idle sweep to a tenant, and give it a caller
Beenu Aroraff9aca82a43a
diff --git a/services/fusion/app/services/sessions.py b/services/fusion/app/services/sessions.pyindex dcba68caf..7b6559967 100644--- a/services/fusion/app/services/sessions.py+++ b/services/fusion/app/services/sessions.py@@ -165,11 +165,32 @@ ON CONFLICT (id) DO UPDATE SET#: Close derived sessions that have gone quiet. Vendor sessions are left#: alone: the vendor says when those end, and guessing on their behalf would#: report a session as over while it is still being used.+#:+#: Scoped to one tenant per call rather than sweeping the table. The first+#: version had no tenant predicate and relied on the RLS policy's+#: `current_tenant_id() IS NULL` arm, which a worker satisfies by setting+#: nothing -- so one statement reached every tenant's rows and+#: `check_tenant_query_predicates.py` failed the build for it. That gate is+#: right: RLS is defence in depth here, not the control, and a maintenance+#: sweep that can touch every tenant at once is the shape this repository+#: has been bitten by before. It also bounds the work, so one noisy tenant+#: cannot stall the sweep for the rest._CLOSE_IDLE_SQL = """UPDATE sessionsSET ended_at = last_seen_at,ended_reason = 'idle',updated_at = now()+ WHERE tenant_id = $1+ AND ended_at IS NULL+ AND key_kind = 'derived'+ AND last_seen_at < now() - make_interval(secs => $2)+"""++#: The tenants with at least one open derived session. Drives the sweep so+#: it visits only tenants that can have work, rather than every tenant row.+_IDLE_TENANTS_SQL = """+SELECT DISTINCT tenant_id+ FROM sessionsWHERE ended_at IS NULLAND key_kind = 'derived'AND last_seen_at < now() - make_interval(secs => $1)@@ -289,15 +310,34 @@ class SessionStore:parent_id,)- async def close_idle(self, idle_seconds: int = DERIVED_IDLE_SECONDS) -> int:- """Close derived sessions that have gone quiet. Returns rows closed."""+ async def close_idle(+ self,+ idle_seconds: int = DERIVED_IDLE_SECONDS,+ *,+ tenant_id: str | None = None,+ ) -> int:+ """Close derived sessions that have gone quiet. Returns rows closed.++ With ``tenant_id`` the sweep touches that tenant only. Without it the+ tenants that *have* idle sessions are read first and swept one at a+ time, so the UPDATE always carries a tenant predicate and one+ tenant's backlog cannot stall the others.+ """try:async with self._pool.acquire() as conn:- result = await conn.execute(_CLOSE_IDLE_SQL, float(idle_seconds))+ if tenant_id is not None:+ result = await conn.execute(_CLOSE_IDLE_SQL, tenant_id, float(idle_seconds))+ return _rows_affected(result)++ rows = await conn.fetch(_IDLE_TENANTS_SQL, float(idle_seconds))+ closed = 0+ for row in rows:+ result = await conn.execute(_CLOSE_IDLE_SQL, row["tenant_id"], float(idle_seconds))+ closed += _rows_affected(result)+ return closedexcept Exception as exc: # noqa: BLE001 — same reasoning as record_eventlogger.error("fusion.session.close_idle_failed", extra={"error": str(exc)}, exc_info=True)return 0- return _rows_affected(result)def _session_block(payload: dict[str, Any]) -> dict[str, Any] | None:diff --git a/services/fusion/app/workers/consumer.py b/services/fusion/app/workers/consumer.pyindex 9c6662af7..f14ffeb32 100644--- a/services/fusion/app/workers/consumer.py+++ b/services/fusion/app/workers/consumer.py@@ -7,6 +7,7 @@ results back to Kafka, and persists non-duplicate alerts to Postgres.import asyncioimport contextlibimport json+import timefrom typing import Anyimport structlog@@ -22,7 +23,7 @@ from app.services.event_schema import validate_eventfrom app.services.fusion_engine import FusionEnginefrom app.services.lake_writer import LakeWriterfrom app.services.promoter import promote_normalized_event-from app.services.sessions import SessionRef, SessionStore+from app.services.sessions import DERIVED_IDLE_SECONDS, SessionRef, SessionStorefrom app.services.tenant_overlay import OverlayCachefrom app.services.ueba_signal import UebaSignalCachefrom app.services.windowed_detection import WindowedDetectionEngine@@ -50,6 +51,12 @@ _METRICS = {}+#: How often the worker sweeps for derived sessions that have gone quiet.+#: A quarter of the idle window, so a session is closed within roughly+#: DERIVED_IDLE_SECONDS of its last event rather than up to twice that.+SESSION_SWEEP_SECONDS = DERIVED_IDLE_SECONDS / 4++class FusionWorker:"""Kafka consumer/producer pair that drives the fusion pipeline."""@@ -135,11 +142,19 @@ class FusionWorker:await self._producer.start()if self._sink is not None:await self._sink.start()- if self._lake is not None:- await self._lake.start()+ if self._lake is not None or self._sessions is not None:+ if self._lake is not None:+ await self._lake.start()# Low-traffic safety net: a single buffered event would otherwise# wait for the NEXT message before flush_if_stale runs. A background# ticker guarantees the lake batch flushes by age even when idle.+ #+ # The same ticker closes idle derived sessions. A derived session+ # ends when the activity stops, so nothing on the event path can+ # ever close one -- the closing condition is the *absence* of an+ # event. Without a ticker every derived session would stay open+ # forever and "what else did this sitting do" would accumulate+ # months of unrelated activity under one id.self._flush_task = asyncio.create_task(self._periodic_flush())self._running = Truelogger.info(@@ -154,11 +169,26 @@ class FusionWorker:async def _periodic_flush(self) -> None:interval = max(0.5, settings.lake_batch_max_age_seconds)+ # Elapsed wall-clock rather than a tick count, because the two+ # cadences are set independently: a tick count derived from the+ # lake's flush interval silently changes the sweep period whenever+ # someone tunes the lake, which is a coupling nobody would expect+ # from either setting's name.+ last_sweep = time.monotonic()try:while self._running:await asyncio.sleep(interval)if self._lake is not None:await self._lake.flush_if_stale()+ if self._sessions is None:+ continue+ now = time.monotonic()+ if now - last_sweep < SESSION_SWEEP_SECONDS:+ continue+ last_sweep = now+ closed = await self._sessions.close_idle()+ if closed:+ logger.info("Closed idle derived sessions", closed=closed)except asyncio.CancelledError: # pragma: no cover — shutdown pathpassdiff --git a/services/fusion/tests/test_sessions.py b/services/fusion/tests/test_sessions.pyindex 896c8d913..eaab8b9d9 100644--- a/services/fusion/tests/test_sessions.py+++ b/services/fusion/tests/test_sessions.py@@ -9,6 +9,8 @@ arguments have to satisfy is asserted against the migration itself.from __future__ import annotations+import asyncio+import contextlibimport jsonimport reimport uuid@@ -18,6 +20,7 @@ from pathlib import Pathimport pytestfrom app.services.sessions import (+ DERIVED_IDLE_SECONDS,KEY_KINDS,SessionStore,session_id,@@ -29,9 +32,12 @@ TENANT = "11111111-1111-1111-1111-111111111111"class RecordingConn:- def __init__(self, fail: Exception | None = None) -> None:+ def __init__(self, fail: Exception | None = None, tenants: list[str] | None = None) -> None:self.calls: list[tuple[str, tuple]] = []self._fail = fail+ # What `SELECT DISTINCT tenant_id` returns. asyncpg hands back+ # mapping-like Records, so a dict is the honest stand-in.+ self._tenants = tenants if tenants is not None else []async def execute(self, sql: str, *args):if self._fail is not None:@@ -39,6 +45,15 @@ class RecordingConn:self.calls.append((sql, args))return "INSERT 0 1"+ async def fetch(self, sql: str, *args):+ if self._fail is not None:+ raise self._fail+ self.calls.append((sql, args))+ return [{"tenant_id": t} for t in self._tenants]++ def all_sql_containing(self, needle: str) -> list[tuple[str, tuple]]:+ return [(sql, args) for sql, args in self.calls if needle in sql]+def sql_containing(self, needle: str) -> tuple[str, tuple]:for sql, args in self.calls:if needle in sql:@@ -191,16 +206,16 @@ async def test_a_call_that_issued_its_own_credential_is_not_its_own_parent():async def test_the_id_is_deterministic_across_stores_and_replays():"""Three stores have to agree on one id without coordinating, and a Kafkareplay must upsert the same row rather than minting a second session."""- first = session_id(TENANT, "okta", "102bFs0Aq6VQ1qjqVqk")- second = session_id(TENANT, "okta", "102bFs0Aq6VQ1qjqVqk")+ first = session_id(TENANT, "okta", "00okta0session0EXAMPLE0001")+ second = session_id(TENANT, "okta", "00okta0session0EXAMPLE0001")assert first == secondassert isinstance(first, uuid.UUID)# Namespaced by tenant and by source: two tenants may legitimately observe# the same vendor identifier, and an Okta session id and a GitHub token# hash share no namespace.- assert session_id("22222222-2222-2222-2222-222222222222", "okta", "102bFs0Aq6VQ1qjqVqk") != first- assert session_id(TENANT, "github", "102bFs0Aq6VQ1qjqVqk") != first+ assert session_id("22222222-2222-2222-2222-222222222222", "okta", "00okta0session0EXAMPLE0001") != first+ assert session_id(TENANT, "github", "00okta0session0EXAMPLE0001") != first@pytest.mark.asyncio@@ -220,11 +235,45 @@ async def test_only_derived_sessions_are_closed_on_an_idle_gap():conn = RecordingConn()store = SessionStore(RecordingPool(conn))- await store.close_idle(900)+ await store.close_idle(900, tenant_id=TENANT)sql, args = conn.sql_containing("UPDATE sessions")assert "key_kind = 'derived'" in sql- assert args[0] == 900.0+ assert args == (TENANT, 900.0)+++async def test_the_idle_sweep_never_issues_an_update_without_a_tenant():+ """The first version of this swept the whole table and leaned on the RLS+ policy's `current_tenant_id() IS NULL` arm, which a worker satisfies by+ setting nothing -- so one statement reached every tenant's rows and+ `check_tenant_query_predicates.py` failed the build for it.++ RLS is defence in depth here, not the control. Every UPDATE the sweep+ issues names exactly one tenant."""+ other = "33333333-3333-3333-3333-333333333333"+ conn = RecordingConn(tenants=[TENANT, other])+ store = SessionStore(RecordingPool(conn))++ closed = await store.close_idle(900)++ updates = conn.all_sql_containing("UPDATE sessions")+ assert len(updates) == 2+ assert {args[0] for _, args in updates} == {TENANT, other}+ for sql, _ in updates:+ assert "tenant_id = $1" in sql+ assert closed == 2+++async def test_a_sweep_with_no_idle_tenants_issues_no_update_at_all():+ """The control. A quiet deployment must not write, or the test above+ would pass on a sweep that updates unconditionally."""+ conn = RecordingConn(tenants=[])+ store = SessionStore(RecordingPool(conn))++ assert await store.close_idle(900) == 0+ assert conn.all_sql_containing("UPDATE sessions") == []def test_the_closed_set_matches_the_migrations_check_constraint():@@ -262,3 +311,66 @@ def test_event_time_handles_every_shape_the_spine_delivers(raw, expect_year):payload = {"time": raw} if raw is not None else {}assert _event_time(payload).year == expect_year+++async def test_the_idle_sweep_runs_from_the_worker_rather_than_nowhere():+ """`close_idle` had no production caller at all.++ A derived session ends when the activity stops, so nothing on the event+ path can ever close one -- the closing condition is the *absence* of an+ event. Without a ticker every derived session stays open forever and+ "what else did this sitting do" accumulates months of unrelated activity+ under one id, which is worse than having no session grouping at all.++ Drives the worker's real ticker with the real interval arithmetic rather+ than asserting the call exists in the source.+ """+ from app.workers import consumer as consumer_mod++ swept: list[int] = []++ class SweepOnlyStore:+ async def close_idle(self, *_a, **_kw) -> int:+ swept.append(1)+ return 0++ worker = consumer_mod.FusionWorker.__new__(consumer_mod.FusionWorker)+ worker._running = True+ worker._lake = None+ worker._sessions = SweepOnlyStore()++ # The real period is a quarter of the idle window (~7 minutes). Shortened+ # here so the test exercises the real loop and its real clock arithmetic+ # rather than a reimplementation of it.+ monkeypatch = pytest.MonkeyPatch()+ monkeypatch.setattr(consumer_mod, "SESSION_SWEEP_SECONDS", 0.0)+ monkeypatch.setattr(consumer_mod.settings, "lake_batch_max_age_seconds", 0.01)+ try:+ task = asyncio.create_task(worker._periodic_flush())+ for _ in range(200):+ await asyncio.sleep(0.01)+ if swept:+ break+ worker._running = False+ task.cancel()+ with contextlib.suppress(asyncio.CancelledError):+ await task+ finally:+ monkeypatch.undo()++ assert swept, "the worker ticker never called close_idle"+++def test_the_sweep_period_is_shorter_than_the_window_it_sweeps_for():+ """Two independently-written numbers describing one budget.++ If the sweep ran less often than the idle window, a session would stay+ open for up to the sum of the two rather than the window -- the shape+ this repository has already been bitten by, where a caller's deadline+ was a sixth of the retry budget it wrapped and the shorter number won+ silently. Asserted as a relation, not as either value.+ """+ from app.workers.consumer import SESSION_SWEEP_SECONDS++ assert 0 < SESSION_SWEEP_SECONDS <= DERIVED_IDLE_SECONDS / 2diff --git a/services/ingest/internal/activity/session_test.go b/services/ingest/internal/activity/session_test.goindex 19a0655b5..ef87cece2 100644--- a/services/ingest/internal/activity/session_test.go+++ b/services/ingest/internal/activity/session_test.go@@ -39,9 +39,9 @@ func TestSessionKeyComesFromTheVendorsOwnField(t *testing.T) {connector: "okta",vendor: map[string]any{"actor": map[string]any{"alternateId": "[email protected]", "type": "User"},- "authenticationContext": map[string]any{"externalSessionId": "102bFs0Aq6VQ1qjqVqk"},+ "authenticationContext": map[string]any{"externalSessionId": "00okta0session0EXAMPLE0001"},},- wantKey: "102bFs0Aq6VQ1qjqVqk",+ wantKey: "00okta0session0EXAMPLE0001",wantKind: SessionVendorSession,wantSource: "authenticationContext.externalSessionId",},@@ -61,9 +61,9 @@ func TestSessionKeyComesFromTheVendorsOwnField(t *testing.T) {connector: "azure_entra",vendor: map[string]any{"userPrincipalName": "[email protected]",- "correlationId": "9f8e7d6c-5b4a-3210-fedc-ba9876543210",+ "correlationId": "00000000-1111-2222-3333-444444444444",},- wantKey: "9f8e7d6c-5b4a-3210-fedc-ba9876543210",+ wantKey: "00000000-1111-2222-3333-444444444444",wantKind: SessionVendorFlow,wantSource: "correlationId",},@@ -374,7 +374,7 @@ func TestEverySessionKindIsInTheClosedSet(t *testing.T) {// the vendor's own console. A hash here would be a value that exists nowhere// else and cannot be looked up anywhere.func TestTheKeyIsTheVendorsOwnValue(t *testing.T) {- const raw = "102bFs0Aq6VQ1qjqVqk"+ const raw = "00okta0session0EXAMPLE0001"got := Project("okta", canonical("okta", map[string]any{"actor": map[string]any{"alternateId": "[email protected]"},"authenticationContext": map[string]any{"externalSessionId": raw},