Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
fix(ingest,fusion): stop dropping description, techniques and entities (#1286)
Beenu Aroraf347ab1087a0
diff --git a/services/api/app/api/v1/endpoints/metrics.py b/services/api/app/api/v1/endpoints/metrics.pyindex 67dda690c..b85a80b8d 100644--- a/services/api/app/api/v1/endpoints/metrics.py+++ b/services/api/app/api/v1/endpoints/metrics.py@@ -1036,6 +1036,14 @@ async def _funnel_window(db, tenant_id, start, end, *, mitre_total: int) -> dict# (i.e. multiple raw events fused into a single alert). Single-event# alerts are excluded so the ratio stays meaningful when one detection# rule fires repeatedly.+ #+ # The predicate was always right and was unsatisfiable until issue #1244:+ # the only pipeline writer appended one id *before* the insert, and a+ # deduplicated event's insert is filtered out by the sink's+ # `WHERE NOT EXISTS`, so no row could ever reach two. The tile read 0 by+ # construction on a pipeline that was deduplicating the whole time.+ # `AlertSink._record_deduplicated_event` is what feeds this now; a zero+ # here means no event deduplicated in the window, not that nobody looked.correlation_instances = (await db.scalar(select(func.count()).where(diff --git a/services/fusion/app/services/alert_sink.py b/services/fusion/app/services/alert_sink.pyindex 810c602e6..0425ef0d0 100644--- a/services/fusion/app/services/alert_sink.py+++ b/services/fusion/app/services/alert_sink.py@@ -72,7 +72,8 @@ INSERT INTO alerts (dedup_hash, confidence, confidence_label, confidence_rationale,narrative, anomaly_score, event_time,connector_id, connector_type, source_event_ids, ocsf_class_uid,- rule_id, rule_name, external_id+ rule_id, rule_name, external_id,+ affected_hosts, affected_users, affected_ips)SELECT$1, $2, $3, $4, $5, 'new',@@ -80,13 +81,51 @@ SELECT$11::text, $12, $13, $14::jsonb,$15, $16, COALESCE($17, NOW()),$18, $19, $20::jsonb, $21,- $22, $23, $24+ $22, $23, $24,+ $25::jsonb, $26::jsonb, $27::jsonbWHERE NOT EXISTS (SELECT 1 FROM alerts WHERE tenant_id = $2 AND dedup_hash = $11::text)ON CONFLICT (id) DO NOTHINGRETURNING id"""+# The three entity columns are appended rather than inserted beside+# `entities`, where they belong semantically, so that every existing+# positional index into `_insert_args` keeps its meaning. Renumbering 24+# placeholders to make a list read nicely is how a confidence score ends up+# in a narrative column.++#: Record a raw event that deduplicated onto an alert that already exists.+#:+#: Issue #1244. `source_event_ids` is "the raw events that produced this+#: alert" and the funnel's Correlation Instances tile counts the alerts where+#: it holds two or more. The only writer appended exactly one id *before* the+#: insert, and a deduplicated event's insert is filtered out by the+#: `WHERE NOT EXISTS` above — so the second, third and five-hundredth event to+#: fold onto an alert left no trace, the predicate could never be satisfied,+#: and the tile read zero on a pipeline that was deduplicating the whole time.+#:+#: One id per statement, with the `@>` guard making a replay a no-op rather+#: than a second entry — a consumer-group rebalance re-delivers the same event+#: and the provenance list must not grow on it. `$4` caps the array because a+#: hot dedup key would otherwise grow one row without bound; past the cap the+#: alert still exists and still deduplicates, only the list stops growing.+#: jsonb containment accepts a scalar against an array element, so+#: ``'["evt-1"]'::jsonb @> '"evt-1"'::jsonb`` is the membership test.+_DEDUP_SOURCE_EVENT_SQL = """+UPDATE alerts+ SET source_event_ids = source_event_ids || to_jsonb($3::text)+ WHERE tenant_id = $1+ AND dedup_hash = $2::text+ AND NOT (source_event_ids @> to_jsonb($3::text))+ AND jsonb_array_length(source_event_ids) < $4+"""++#: Ceiling on `source_event_ids` per alert. 200 ids is ~8 KB of JSONB, and the+#: question the column answers ("did more than one event produce this?") is+#: settled by the second entry.+_MAX_SOURCE_EVENT_IDS = 200+# The reconciliation row the two-way SIEM loop reads. Written here rather than# from the API because fusion is the only place that holds the alert id, the# connector instance and the vendor finding id at the same moment — asking the@@ -170,6 +209,29 @@ def _entities(fused: FusedAlert) -> list[dict[str, str]]:return out+def _affected(fused: FusedAlert) -> tuple[list[str], list[str], list[str]]:+ """The denormalized `affected_hosts` / `affected_users` / `affected_ips`.++ Issue #1244. These three columns have existed since migration 042, the+ ORM maps them, `GET /alerts/{id}` serialises them, and `feedback.py`,+ `rule_tuning.py`, `business_context.py`, `cases.py` and the GraphQL+ resolver all read them — but the production INSERT never named them, so+ the only rows that ever had a value were the ones `seed_demo.py` wrote.+ Every reader saw `[]` on a real alert and a populated list on a seeded+ one.++ Derived from the same fields `_entities` and `_iocs` already use rather+ than from a second source, so the row cannot disagree with itself.+ `entities` stays exactly as it was: it is the shape the Investigation+ Rail reads and these columns are the shape the filters read.+ """+ alert = fused.alert+ hosts = [alert.hostname] if alert.hostname else []+ users = [alert.username] if alert.username else []+ ips = [ip for ip in (alert.src_ip, alert.dst_ip) if ip]+ return hosts, users, ips++def _insert_args(fused: FusedAlert) -> tuple[Any, ...]:"""The positional arguments `_INSERT_SQL` takes, in order.@@ -179,6 +241,7 @@ def _insert_args(fused: FusedAlert) -> tuple[Any, ...]:narrative without anything failing."""alert = fused.alert+ hosts, users, ips = _affected(fused)return (fused.id,alert.tenant_id,@@ -204,6 +267,9 @@ def _insert_args(fused: FusedAlert) -> tuple[Any, ...]:alert.rule_id,alert.rule_name,alert.external_id,+ json.dumps(hosts),+ json.dumps(users),+ json.dumps(ips),)@@ -245,7 +311,19 @@ class AlertSink:DB is reachable."""canonical_id = str(fused.id)+ alert = fused.alertif fused.fusion_decision == FusionDecision.DUPLICATE:+ # Fusion has already decided: there is no row to write, and a+ # database outage cannot change that verdict. The folded-away+ # event is still part of the surviving alert's provenance, so+ # record it when there is a pool — best-effort, never a reconnect+ # attempt, and never a different outcome.+ if self._pool is not None:+ try:+ async with self._pool.acquire() as conn:+ await self._record_deduplicated_event(conn, alert)+ except Exception as exc: # noqa: BLE001 — provenance is not worth the verdict+ logger.warning("alert_sink.dedup_provenance_failed", error=str(exc))return PersistResult(PersistOutcome.DUPLICATE, canonical_id)pool = await self._ensure_pool()@@ -256,13 +334,13 @@ class AlertSink:return PersistResult(PersistOutcome.UNAVAILABLE, canonical_id)self._connect_failed_logged = False- alert = fused.alerttry:async with pool.acquire() as conn:row = await conn.fetchrow(_INSERT_SQL, *_insert_args(fused))- if row is None:- logger.debug("alert_sink.dedup_skip", fingerprint=alert.fingerprint())- return PersistResult(PersistOutcome.DUPLICATE, canonical_id)+ if row is None:+ logger.debug("alert_sink.dedup_skip", fingerprint=alert.fingerprint())+ await self._record_deduplicated_event(conn, alert)+ return PersistResult(PersistOutcome.DUPLICATE, canonical_id)await self._link_source_finding(pool, alert, alert_id=row["id"])return PersistResult(PersistOutcome.INSERTED, str(row["id"]))except asyncpg.ForeignKeyViolationError:@@ -319,6 +397,7 @@ class AlertSink:for fused in fused_batch:canonical_id = str(fused.id)if fused.fusion_decision == FusionDecision.DUPLICATE:+ await self._record_deduplicated_event(conn, fused.alert)results.append(PersistResult(PersistOutcome.DUPLICATE, canonical_id))continuetry:@@ -333,6 +412,7 @@ class AlertSink:continueif row is None:+ await self._record_deduplicated_event(conn, fused.alert)results.append(PersistResult(PersistOutcome.DUPLICATE, canonical_id))continueresults.append(PersistResult(PersistOutcome.INSERTED, str(row["id"])))@@ -346,6 +426,30 @@ class AlertSink:return results+ async def _record_deduplicated_event(self, conn: Any, alert: Any) -> None:+ """Attach a folded-away event's id to the alert it deduplicated onto.++ Best-effort, for the same reason `_link_source_finding` is: the+ provenance entry is worth one statement, never the verdict the caller+ is about to return. A failure here must not turn a correct DUPLICATE+ into a FAILED.+ """+ ids = [i for i in (alert.source_event_ids or []) if i]+ if not ids:+ return+ fingerprint = alert.fingerprint()+ try:+ for source_event_id in ids:+ await conn.execute(+ _DEDUP_SOURCE_EVENT_SQL,+ alert.tenant_id,+ fingerprint,+ str(source_event_id),+ _MAX_SOURCE_EVENT_IDS,+ )+ except Exception as exc: # noqa: BLE001 — provenance is not worth the alert+ logger.warning("alert_sink.dedup_provenance_failed", error=str(exc))+async def _link_source_finding(self, pool: asyncpg.Pool, alert: Any, *, alert_id: Any) -> None:"""Record which vendor finding produced this alert, when there is one.diff --git a/services/fusion/app/services/detection_engine.py b/services/fusion/app/services/detection_engine.pyindex b5d77c4fb..1815c7cea 100644--- a/services/fusion/app/services/detection_engine.py+++ b/services/fusion/app/services/detection_engine.py@@ -48,6 +48,7 @@ from app.models.alert import AlertSeverity, RawAlertfrom app.services.derived_fields import enrich, requested_derived_fieldsfrom app.services.detection_matcher import matchesfrom app.services.provenance import extract_provenance+from app.services.raw_data import decode_raw_datalogger = structlog.get_logger()@@ -157,15 +158,9 @@ class DetectionEngine:@staticmethoddef _decode_raw_data(ocsf: dict[str, Any]) -> dict[str, Any]:"""The connector's normalized dict, or the OCSF top level as fallback."""- raw = ocsf.get("raw_data")- if isinstance(raw, str) and raw.strip():- try:- parsed = json.loads(raw)- if isinstance(parsed, dict):- return parsed- except (ValueError, TypeError):- # Malformed raw_data JSON — fall through to the OCSF top level below.- pass+ decoded = decode_raw_data(ocsf)+ if decoded is not None:+ return decoded# Fall back to the OCSF top level (some connectors emit flat OCSF).return ocsf if isinstance(ocsf, dict) else {}diff --git a/services/fusion/app/services/promoter.py b/services/fusion/app/services/promoter.pyindex 0701d8637..fffa1b883 100644--- a/services/fusion/app/services/promoter.py+++ b/services/fusion/app/services/promoter.py@@ -46,6 +46,7 @@ import structlogfrom app.models.alert import AlertSeverity, RawAlertfrom app.services.provenance import extract_provenance, product_label+from app.services.raw_data import decode_raw_datalogger = structlog.get_logger()@@ -141,11 +142,17 @@ def _description(ocsf: dict[str, Any]) -> str:The raw payload is not lost: it stays on the alert's `raw_event`, which iswhat the investigation surfaces read.++ Issue #1244: that vendor-description branch used to be guarded by+ `isinstance(raw, dict)` while ingest writes `raw_data` as the JSON string+ `json.Marshal(raw.Payload)` produces, so on the production path it was+ unreachable and every supplied description was discarded. `decode_raw_data`+ is the decoder the detection engine already used for the same value."""- raw = ocsf.get("raw_data")- if isinstance(raw, dict):+ payload = decode_raw_data(ocsf)+ if payload is not None:for key in ("description", "message", "summary", "detail", "reason"):- value = raw.get(key)+ value = payload.get(key)if isinstance(value, str) and value.strip():return value.strip()[:2000]finding_desc = _get_nested(ocsf, "finding", "desc")@@ -369,11 +376,22 @@ def promote_normalized_event(message: dict[str, Any]) -> RawAlert | None:tactics, techniques = _mitre(ocsf)connector_id, connector_type, class_uid = extract_provenance(message, ocsf)+ title = _title(ocsf)+ description = _description(ocsf)+ # The last resort in `_description` is `ocsf["message"]`, which is the+ # first key `_title` reads, so a vendor that supplied only a title used to+ # get that one sentence rendered twice — once as the alert heading and+ # again as its body. Nothing is lost by leaving the slot empty: the+ # console falls back to the raw event, and an empty description is the+ # honest statement that the vendor wrote no prose.+ if description == title:+ description = ""+return RawAlert(tenant_id=tenant_id,source=_source(ocsf),- title=_title(ocsf),- description=_description(ocsf),+ title=title,+ description=description,severity=severity,src_ip=_get_nested(ocsf, "src_endpoint", "ip"),dst_ip=_get_nested(ocsf, "dst_endpoint", "ip"),diff --git a/services/fusion/app/services/raw_data.py b/services/fusion/app/services/raw_data.pynew file mode 100644index 000000000..a5d5f7c20--- /dev/null+++ b/services/fusion/app/services/raw_data.py@@ -0,0 +1,43 @@+"""One decoder for the connector payload ``services/ingest`` attaches.++``raw_data`` is a JSON **string**, not a dict:+``normalizer.go`` runs ``json.Marshal(raw.Payload)`` and stores the result as+``ocsf["raw_data"]``. Two fusion modules need the payload back — the detection+engine, to build the field namespace a rule matches against, and the promoter,+to find the vendor's own description — and before issue #1244 only the+detection engine knew the shape. The promoter guarded its branch with+``isinstance(raw, dict)``, which no ingest event satisfies, so the vendor's+description was dropped on every event that carried one.++A dict is accepted as well as a string. Some producers (the API's direct+``/alerts/submit`` path, fixtures written before ingest serialised) hand the+payload through unserialised, and refusing it would trade one dead branch for+another.+"""++from __future__ import annotations++import json+from typing import Any+++def decode_raw_data(ocsf: dict[str, Any]) -> dict[str, Any] | None:+ """The connector's own payload, or ``None`` when there isn't one.++ ``None`` rather than ``{}`` so a caller can distinguish "no payload" from+ "an empty payload" and apply its own fallback — the detection engine falls+ back to the OCSF top level, the promoter does not.+ """+ raw = ocsf.get("raw_data")+ if isinstance(raw, dict):+ return raw+ if isinstance(raw, str) and raw.strip():+ try:+ parsed = json.loads(raw)+ except (ValueError, TypeError):+ # Malformed JSON in raw_data. The caller decides what to do with+ # nothing; one bad payload must not raise out of the hot path.+ return None+ if isinstance(parsed, dict):+ return parsed+ return Nonediff --git a/services/fusion/app/workers/consumer.py b/services/fusion/app/workers/consumer.pyindex 2d6133b29..07f60a422 100644--- a/services/fusion/app/workers/consumer.py+++ b/services/fusion/app/workers/consumer.py@@ -304,6 +304,17 @@ class FusionWorker:# Phase A2 — run the executable detection corpus against the live# event. Each firing rule becomes a RawAlert routed through fusion,# so telemetry that isn't a vendor-asserted finding still alerts.+ #+ # One event firing N rules produces N alert rows, deliberately+ # (issue #1244 reported one Sysmon event becoming five or six).+ # Each row names the rule that fired, carries that rule's severity+ # and category, and is dispositioned on its own: collapsing them+ # would mean picking one rule's verdict to stand for the others,+ # and an analyst who closes "suspicious PowerShell" has not+ # thereby closed "LSASS access". `Correlator.correlate` already+ # groups them into one incident on+ # {tenant}:{entity}:{tactic} for the console, which is where the+ # N-into-1 view belongs.if self._detector is not None:for hit in await self._detector.evaluate_async(payload, overlay, self._first_seen):det_alert = self._detector.build_alert(payload, hit)diff --git a/services/fusion/tests/test_alert_sink_batch.py b/services/fusion/tests/test_alert_sink_batch.pyindex 521337ec3..d68d10939 100644--- a/services/fusion/tests/test_alert_sink_batch.py+++ b/services/fusion/tests/test_alert_sink_batch.py@@ -11,6 +11,8 @@ is pinned here is the contract rather than the speed.from __future__ import annotations+import re+import pytestfrom app.services.alert_sink import PersistOutcome@@ -56,9 +58,18 @@ class TestTheArgumentTupleHasOneDefinition:into a narrative without anything failing."""from app.services.alert_sink import _INSERT_SQL- placeholders = {int(tok[1:]) for tok in _INSERT_SQL.split() if tok.startswith("$") and tok[1:].isdigit()}+ # Matched with a regex rather than by splitting on whitespace: almost+ # every placeholder in this statement carries a cast or a trailing+ # comma (`$9::jsonb,`), so the whitespace form only ever saw the two+ # bare ones and the width assertion below was nearly vacuous.+ placeholders = {int(n) for n in re.findall(r"\$(\d+)", _INSERT_SQL)}assert placeholders, "no positional placeholders found; this test is checking nothing"assert max(placeholders) >= 20, "the insert shrank unexpectedly — re-check _insert_args"+ # No gaps: asyncpg binds positionally, so a skipped number means every+ # argument after it lands in the wrong column.+ assert placeholders == set(range(1, max(placeholders) + 1)), (+ f"placeholder numbering has holes: {sorted(set(range(1, max(placeholders) + 1)) - placeholders)}"+ )@pytest.mark.asynciodiff --git a/services/fusion/tests/test_alert_sink_issue_1244.py b/services/fusion/tests/test_alert_sink_issue_1244.pynew file mode 100644index 000000000..46dc12276--- /dev/null+++ b/services/fusion/tests/test_alert_sink_issue_1244.py@@ -0,0 +1,293 @@+"""The alert row carries the entities fusion computed, and records dedup.++Issue #1244, two independent gaps in what ``AlertSink`` writes.++**affected_hosts / affected_users / affected_ips were never written.** The+columns have existed since migration 042, the ORM maps them, the alerts+endpoint serialises them and five readers filter on them — and the only+writer in the tree was ``seed_demo.py``, so every one of those readers saw+``[]`` on a real alert and a populated list on a seeded one. Fusion already+derives the same host, user and IPs for ``entities`` and ``iocs``; it just+did not name the columns in its INSERT.++**A deduplicated event left no trace.** ``source_event_ids`` is the column+the funnel's "Correlation Instances" tile counts (``jsonb_array_length >= 2``+in ``services/api/app/api/v1/endpoints/metrics.py``), and the only writer+appends one id before the insert. When a second event dedups onto an alert the+insert is filtered by ``WHERE NOT EXISTS`` and the second event's id is+dropped on the floor, so the predicate could never be satisfied and the tile+read 0 by construction — "522 events formed 0 correlation instances" on a+pipeline that was deduplicating the whole time.+"""++from __future__ import annotations++import json+import re+import uuid+from contextlib import asynccontextmanager++import pytest+from app.models.alert import AlertSeverity, FusedAlert, FusionDecision, RawAlert+from app.services.alert_sink import AlertSink, PersistOutcome++TENANT = uuid.UUID("00000000-0000-0000-0000-000000000001")+CONNECTOR = uuid.UUID("00000000-0000-0000-0000-0000000000c1")+++def _fused(+ decision: FusionDecision = FusionDecision.NEW_ALERT,+ *,+ source_event_ids: list[str] | None = None,+) -> FusedAlert:+ raw = RawAlert(+ tenant_id=TENANT,+ source="CrowdStrike Falcon",+ title="Credential dumping via LSASS access",+ severity=AlertSeverity.CRITICAL,+ src_ip="10.20.30.40",+ dst_ip="10.20.30.99",+ hostname="WIN-DC01",+ username="svc-backup",+ mitre_techniques=["T1003"],+ connector_id=CONNECTOR,+ connector_type="crowdstrike_falcon",+ source_event_ids=["evt-1"] if source_event_ids is None else list(source_event_ids),+ ocsf_class_uid=2004,+ )+ raw.id = raw.deterministic_id()+ return FusedAlert(id=raw.id, tenant_id=TENANT, fusion_decision=decision, alert=raw)+++class _StubConn:+ """Records every statement, and answers fetchrow with a scripted row."""++ def __init__(self, row):+ self._row = row+ self.calls: list[tuple] = []++ async def fetchrow(self, sql, *args):+ self.calls.append((sql, args))+ return self._row++ async def execute(self, sql, *args):+ self.calls.append((sql, args))+ return "UPDATE 1"+++class _StubPool:+ def __init__(self, row):+ self.conn = _StubConn(row)++ @asynccontextmanager+ async def _acquire(self):+ yield self.conn++ def acquire(self):+ return self._acquire()+++def _insert_statement(pool: _StubPool) -> tuple[str, tuple]:+ for sql, args in pool.conn.calls:+ if "INSERT INTO alerts" in sql:+ return sql, args+ raise AssertionError(f"no INSERT INTO alerts issued; calls={[c[0][:40] for c in pool.conn.calls]}")+++def _split_top_level(expr_list: str) -> list[str]:+ """Split a SELECT list on commas that are not inside parentheses.++ `COALESCE($17, NOW())` is one expression, not two.+ """+ out: list[str] = []+ depth = 0+ current = ""+ for char in expr_list:+ if char == "(":+ depth += 1+ elif char == ")":+ depth -= 1+ if char == "," and depth == 0:+ out.append(current.strip())+ current = ""+ continue+ current += char+ if current.strip():+ out.append(current.strip())+ return out+++def _column_value(sql: str, args: tuple, column: str):+ """Read one column's bound value by name, via the placeholder it selects.++ By name rather than by position: a positional assertion silently follows+ the wrong value the moment a column is inserted ahead of it, which is the+ drift `_insert_args` exists to prevent. Column index and argument index+ are *not* the same — `status` is selected as the literal `'new'` and+ consumes no placeholder — so the column list is zipped against the SELECT+ list and the `$N` is read off the matching expression.+ """+ head, _, tail = sql.partition("INSERT INTO alerts (")+ assert head is not None and tail, "not an alerts INSERT"+ columns = [c.strip() for c in tail.split(")", 1)[0].split(",")]+ select_list = tail.split("SELECT", 1)[1].split("WHERE NOT EXISTS", 1)[0]+ exprs = _split_top_level(select_list)+ assert len(columns) == len(exprs), f"{len(columns)} columns against {len(exprs)} select expressions"+ assert column in columns, f"{column} is not in the INSERT column list: {columns}"++ expr = exprs[columns.index(column)]+ match = re.match(r"\$(\d+)", expr)+ assert match, f"{column} is selected as {expr!r}, not a bound parameter"+ return args[int(match.group(1)) - 1]+++# ── affected_* are written from the entities fusion already computes ──────+++async def test_affected_hosts_users_and_ips_are_persisted() -> None:+ fused = _fused()+ pool = _StubPool(row={"id": fused.id})+ sink = AlertSink("postgresql://x")+ sink._pool = pool++ result = await sink.persist(fused)+ assert result.outcome is PersistOutcome.INSERTED++ sql, args = _insert_statement(pool)+ assert json.loads(_column_value(sql, args, "affected_hosts")) == ["WIN-DC01"]+ assert json.loads(_column_value(sql, args, "affected_users")) == ["svc-backup"]+ assert json.loads(_column_value(sql, args, "affected_ips")) == ["10.20.30.40", "10.20.30.99"]+++async def test_absent_entities_persist_as_empty_lists_not_nulls() -> None:+ """The columns are ``JSONB NOT NULL DEFAULT '[]'`` — a NULL would fail."""+ bare = RawAlert(tenant_id=TENANT, source="ingest", title="No entities anywhere")+ bare.id = bare.deterministic_id()+ fused = FusedAlert(id=bare.id, tenant_id=TENANT, fusion_decision=FusionDecision.NEW_ALERT, alert=bare)+ pool = _StubPool(row={"id": fused.id})+ sink = AlertSink("postgresql://x")+ sink._pool = pool++ await sink.persist(fused)++ sql, args = _insert_statement(pool)+ for column in ("affected_hosts", "affected_users", "affected_ips"):+ assert json.loads(_column_value(sql, args, column)) == [], column+++async def test_the_batch_path_writes_the_same_columns() -> None:+ """`persist_many` shares `_insert_args`; prove it rather than assume it."""+ fused = _fused()+ pool = _StubPool(row={"id": fused.id})+ sink = AlertSink("postgresql://x")+ sink._pool = pool++ await sink.persist_many([fused])++ sql, args = _insert_statement(pool)+ assert json.loads(_column_value(sql, args, "affected_hosts")) == ["WIN-DC01"]+++# ── a deduplicated event is recorded on the row it deduped onto ───────────+++async def test_a_dedup_hit_records_the_second_events_id() -> None:+ """The defect behind "522 events formed 0 correlation instances"."""+ pool = _StubPool(row=None) # WHERE NOT EXISTS filtered the insert out+ sink = AlertSink("postgresql://x")+ sink._pool = pool+ fused = _fused(source_event_ids=["evt-2"])++ result = await sink.persist(fused)+ assert result.outcome is PersistOutcome.DUPLICATE++ updates = [(sql, args) for sql, args in pool.conn.calls if sql.strip().upper().startswith("UPDATE ALERTS")]+ assert updates, (+ "a deduplicated event left no trace on the alert it deduped onto, so "+ "source_event_ids can never reach length 2 and the Correlation "+ "Instances tile is a structural zero"+ )+ sql, args = updates[0]+ assert "source_event_ids" in sql+ assert TENANT in args+ assert fused.alert.fingerprint() in args+ assert "evt-2" in json.dumps(args, default=str)+++async def test_a_fusion_decided_duplicate_is_also_recorded() -> None:+ """Fusion's own DUPLICATE verdict returns before the insert is attempted."""+ pool = _StubPool(row=None)+ sink = AlertSink("postgresql://x")+ sink._pool = pool++ result = await sink.persist(_fused(FusionDecision.DUPLICATE, source_event_ids=["evt-3"]))+ assert result.outcome is PersistOutcome.DUPLICATE++ updates = [(sql, args) for sql, args in pool.conn.calls if sql.strip().upper().startswith("UPDATE ALERTS")]+ assert updates, "fusion's in-memory dedup verdict also loses the source event id"+ assert "evt-3" in json.dumps(updates[0][1], default=str)+++async def test_an_insert_that_succeeded_issues_no_dedup_update() -> None:+ """The extra round trip is paid only when there is something to record."""+ fused = _fused()+ pool = _StubPool(row={"id": fused.id})+ sink = AlertSink("postgresql://x")+ sink._pool = pool++ await sink.persist(fused)++ updates = [sql for sql, _ in pool.conn.calls if sql.strip().upper().startswith("UPDATE ALERTS")]+ assert updates == []+++async def test_an_event_with_no_source_id_issues_no_dedup_update() -> None:+ pool = _StubPool(row=None)+ sink = AlertSink("postgresql://x")+ sink._pool = pool++ await sink.persist(_fused(source_event_ids=[]))++ updates = [sql for sql, _ in pool.conn.calls if sql.strip().upper().startswith("UPDATE ALERTS")]+ assert updates == []+++async def test_a_duplicate_verdict_survives_an_unreachable_database() -> None:+ """Recording the provenance must not make DUPLICATE depend on the DB.++ Fusion decided; an outage changes only whether we can write the id down.+ The early return has to stay ahead of `_ensure_pool`, or a DUPLICATE+ starts reporting UNAVAILABLE and the consumer retries a decision that+ was already made.+ """+ sink = AlertSink("postgresql://nope:[email protected]:1/none")+ sink._pool = None++ result = await sink.persist(_fused(FusionDecision.DUPLICATE))+ assert result.outcome is PersistOutcome.DUPLICATE+++async def test_a_failing_dedup_update_does_not_change_the_outcome() -> None:+ """Recording provenance is best-effort; it must never cost the verdict."""++ class _BoomConn(_StubConn):+ async def execute(self, sql, *args):+ self.calls.append((sql, args))+ raise RuntimeError("connection reset")++ pool = _StubPool(row=None)+ pool.conn = _BoomConn(row=None)+ sink = AlertSink("postgresql://x")+ sink._pool = pool++ result = await sink.persist(_fused(source_event_ids=["evt-4"]))+ assert result.outcome is PersistOutcome.DUPLICATEdiff --git a/services/fusion/tests/test_promoter_raw_data_string.py b/services/fusion/tests/test_promoter_raw_data_string.pynew file mode 100644index 000000000..d552d52c2--- /dev/null+++ b/services/fusion/tests/test_promoter_raw_data_string.py@@ -0,0 +1,113 @@+"""The promoter reads ``raw_data`` in the shape ingest actually writes it.++Issue #1244. ``services/ingest`` serialises the connector payload before it+attaches it::++ if rawBytes, err := json.Marshal(raw.Payload); err == nil {+ ocsf["raw_data"] = string(rawBytes)+ }++so ``raw_data`` on every ingest-normalized event is a JSON **string**. The+promoter's description resolver guarded its only vendor-description branch+with ``isinstance(raw, dict)``, which that value can never satisfy, so the+branch was unreachable on the production path and a supplied description was+dropped. Execution fell through to ``ocsf["message"]`` — the same key+``_title`` reads first — so title and description came back identical.++The existing coverage in ``test_promoter_alert_text.py`` passes ``raw_data``+as a dict, which is the shape no ingest event has: the double was more+capable than the real thing, so the tests passed while the path was dead.+Every case here uses the serialised string.+"""++from __future__ import annotations++import json+import uuid++from app.services.promoter import _description, promote_normalized_event++_TENANT = str(uuid.uuid4())+++def _ingest_event(payload: dict) -> dict:+ """An OCSF event shaped the way ``services/ingest`` emits one."""+ return {+ "class_uid": 2001,+ "class_name": "Security Finding",+ "severity_id": 4,+ "message": payload.get("title", ""),+ "metadata": {"product": {"vendor_name": "CrowdStrike", "name": "Falcon"}},+ "raw_data": json.dumps(payload),+ "tenant_uid": _TENANT,+ }+++def test_a_serialized_raw_data_still_yields_the_vendors_description() -> None:+ """The defect: this returned the title, because the dict branch was dead."""+ ocsf = _ingest_event(+ {+ "title": "Credential dumping detected",+ "description": "lsass.exe memory read by rundll32.exe (comsvcs.dll MiniDump)",+ "host": "WIN-FIN-01",+ }+ )+ assert _description(ocsf) == "lsass.exe memory read by rundll32.exe (comsvcs.dll MiniDump)"+++def test_description_and_title_are_not_the_same_string() -> None:+ """The symptom an analyst sees: two panes showing one sentence."""+ alert = promote_normalized_event(+ {+ "tenant_id": _TENANT,+ "ocsf_event": _ingest_event(+ {+ "title": "Credential dumping detected",+ "description": "lsass.exe memory read by rundll32.exe",+ }+ ),+ }+ )+ assert alert is not None+ assert alert.title == "Credential dumping detected"+ assert alert.description == "lsass.exe memory read by rundll32.exe"+++def test_a_title_is_not_repeated_as_the_description() -> None:+ """With no description supplied, the slot is empty rather than a copy.++ Restating the title adds nothing an analyst can act on and it reads as if+ the vendor wrote two sentences. The console renders the raw event either+ way.+ """+ alert = promote_normalized_event(+ {+ "tenant_id": _TENANT,+ "ocsf_event": _ingest_event({"title": "Suspicious PowerShell"}),+ }+ )+ assert alert is not None+ assert alert.title == "Suspicious PowerShell"+ assert alert.description is None or alert.description == ""+++def test_the_other_human_keys_are_reached_through_the_string_too() -> None:+ for key in ("message", "summary", "detail", "reason"):+ ocsf = _ingest_event({"title": "Alert", key: f"human sentence via {key}"})+ assert _description(ocsf) == f"human sentence via {key}", key+++def test_a_malformed_raw_data_string_does_not_raise() -> None:+ """A truncated payload degrades to the existing fallbacks, never an error."""+ ocsf = _ingest_event({"title": "Alert"})+ ocsf["raw_data"] = '{"description": "truncated'+ assert _description(ocsf) == "Alert"+ assert promote_normalized_event({"tenant_id": _TENANT, "ocsf_event": ocsf}) is not None+++def test_a_serialized_payload_is_never_the_description() -> None:+ """The property that matters, now stated against the real shape."""+ ocsf = _ingest_event({"command_line": "whoami", "host": "WIN-01", "pid": 4242})+ result = _description(ocsf)+ assert not result.startswith("{"), f"description is a serialized payload: {result[:60]}"+ assert "4242" not in resultdiff --git a/services/ingest/internal/normalizer/issue_1244_test.go b/services/ingest/internal/normalizer/issue_1244_test.gonew file mode 100644index 000000000..c58585ae8--- /dev/null+++ b/services/ingest/internal/normalizer/issue_1244_test.go@@ -0,0 +1,199 @@+package normalizer++import (+ "testing"++ "github.com/beenuar/aisoc/services/ingest/internal/attck"+)++// Issue #1244, reported against `POST /v1/ingest/batch` with+// `connector_type: crowdstrike` and generic field names.+//+// Two distinct losses on the ingest side:+//+// - a top-level `description` reached no OCSF slot the fusion promoter+// reads, so the alert's description resolved to its title;+// - a vendor's `techniques: ["T1003.001"]` reached nothing either, because+// `extractTechniqueIDs` did not list the plural bare key.++func normalizeOrFail(t *testing.T, n *Normalizer, raw *RawEvent) map[string]interface{} {+ t.Helper()+ ev, err := n.Normalize(raw)+ if err != nil {+ t.Fatalf("Normalize returned error: %v", err)+ }+ return ev.OcsfEvent+}++func pushedEvent(payload map[string]interface{}) *RawEvent {+ return &RawEvent{+ ConnectorID: "conn-1",+ ConnectorType: "crowdstrike",+ TenantID: "11111111-1111-1111-1111-111111111111",+ ReceivedAt: "2026-08-02T00:00:00Z",+ Payload: payload,+ }+}++func findingField(t *testing.T, ocsf map[string]interface{}, key string) interface{} {+ t.Helper()+ finding, ok := ocsf["finding"].(map[string]interface{})+ if !ok {+ return nil+ }+ return finding[key]+}++// ── A supplied description must reach a slot the promoter reads ───────────++func TestSuppliedDescriptionReachesFindingDesc(t *testing.T) {+ ocsf := normalizeOrFail(t, newLenientNormalizer(), pushedEvent(map[string]interface{}{+ "title": "Credential dumping detected",+ "description": "lsass.exe memory read by rundll32.exe",+ "severity": "high",+ "hostname": "WIN-FIN-01",+ }))++ if got := findingField(t, ocsf, "desc"); got != "lsass.exe memory read by rundll32.exe" {+ t.Errorf("finding.desc = %v, want the supplied description", got)+ }+ // The title still owns `message`; the two must not collapse onto one slot.+ if got := ocsf["message"]; got != "Credential dumping detected" {+ t.Errorf("message = %v, want the supplied title", got)+ }+}++// A profile that names its own source for finding.desc keeps it: the alias+// only fills a destination the field map left empty. `email_inbox` resolves+// finding.desc from `body`, so a payload carrying both must not lose it.+func TestAProfilesOwnDescriptionFieldStillWins(t *testing.T) {+ raw := pushedEvent(map[string]interface{}{+ "subject": "Invoice overdue",+ "body": "the profile's own field",+ "description": "the generic key",+ "severity": "medium",+ })+ raw.ConnectorType = "email_inbox"++ ocsf := normalizeOrFail(t, newLenientNormalizer(), raw)+ if got := findingField(t, ocsf, "desc"); got != "the profile's own field" {+ t.Errorf("finding.desc = %v, want the profile's own mapping to win over the alias", got)+ }+}++// Security Hub mapped `Description` onto `raw_data`, which Normalize+// overwrites unconditionally a few lines later with the serialized payload —+// so the vendor's description was mapped and then thrown away.+func TestSecurityHubDescriptionSurvivesTheRawDataOverwrite(t *testing.T) {+ raw := pushedEvent(map[string]interface{}{+ "Title": "S3 bucket is publicly readable",+ "Description": "The bucket policy grants s3:GetObject to Principal *",+ "Severity": map[string]interface{}{"Label": "HIGH"},+ })+ raw.ConnectorType = "aws_security_hub"++ ocsf := normalizeOrFail(t, newLenientNormalizer(), raw)++ if got := findingField(t, ocsf, "desc"); got != "The bucket policy grants s3:GetObject to Principal *" {+ t.Errorf("finding.desc = %v, want the Security Hub Description", got)+ }+ // raw_data is the serialized payload and nothing else.+ rawData, ok := ocsf["raw_data"].(string)+ if !ok {+ t.Fatalf("raw_data = %T, want the serialized payload string", ocsf["raw_data"])+ }+ if rawData == "The bucket policy grants s3:GetObject to Principal *" {+ t.Error("raw_data still holds the Description — the field map and the serializer are fighting over one slot")+ }+}++// ── A bare `techniques` key must yield techniques ─────────────────────────++func TestBareTechniquesKeyIsExtracted(t *testing.T) {+ for _, key := range []string{"techniques", "technique"} {+ t.Run(key, func(t *testing.T) {+ got := extractTechniqueIDs(map[string]interface{}{+ key: []interface{}{"T1003.001"},+ })+ if len(got) != 1 || got[0] != "T1003.001" {+ t.Errorf("extractTechniqueIDs(%q) = %v, want [T1003.001]", key, got)+ }+ })+ }+}++func TestTechniquesAcceptsAScalarAsWellAsAList(t *testing.T) {+ got := extractTechniqueIDs(map[string]interface{}{"techniques": "T1059.001"})+ if len(got) != 1 || got[0] != "T1059.001" {+ t.Errorf("extractTechniqueIDs = %v, want [T1059.001]", got)+ }+}++// A well-formed technique id must survive to `mitre_attck` even when the+// ATT&CK corpus is unavailable. The corpus is fetched from raw.githubusercontent.com+// at boot, so on an air-gapped or egress-filtered deployment `attck.Loaded()`+// is false forever — and gating the whole block on it meant no event anywhere+// ever carried a technique, no matter what the vendor sent.+func TestAWellFormedTechniqueIsCarriedWithoutTheCorpus(t *testing.T) {+ if attck.Loaded() {+ t.Skip("ATT&CK corpus is loaded in this process; this test covers the unloaded deployment")+ }++ ocsf := normalizeOrFail(t, newLenientNormalizer(), pushedEvent(map[string]interface{}{+ "title": "Credential dumping detected",+ "severity": "high",+ "techniques": []interface{}{"T1003.001"},+ }))++ block, ok := ocsf["mitre_attck"].([]map[string]interface{})+ if !ok || len(block) != 1 {+ t.Fatalf("mitre_attck = %#v, want one unverified entry", ocsf["mitre_attck"])+ }+ if got := block[0]["technique_id"]; got != "T1003.001" {+ t.Errorf("technique_id = %v, want T1003.001", got)+ }+ // The reader has to be able to tell an id we looked up from one we only+ // passed through, or an unverified id reads as corpus-confirmed.+ if verified, ok := block[0]["verified"].(bool); !ok || verified {+ t.Errorf("verified = %v, want false on an entry the corpus never confirmed", block[0]["verified"])+ }+}++// Garbage must not be carried: the pass-through is for well-formed ids only.+//+// `TA0006` is the one that matters here. It is a *tactic* id, and the old+// shape check (leading `T`, 5–7 characters before the dot) accepted it. That+// was invisible while every id had to survive `attck.Lookup`, which has no+// entry for a tactic — but an unverified pass-through would have written a+// tactic into `mitre_techniques`, so the shape check has to be exact for the+// carry to be safe.+func TestAMalformedTechniqueIsNotCarried(t *testing.T) {+ for _, bad := range []string{"not-a-technique", "T12", "TA0006", "T10591", "T1059.1", ""} {+ if got := extractTechniqueIDs(map[string]interface{}{"techniques": bad}); len(got) != 0 {+ t.Errorf("extractTechniqueIDs(%q) = %v, want none", bad, got)+ }+ }+}++// Stated against the shape check itself, because the table above goes through+// a key the extractor did not read before this change and so would have+// passed vacuously.+func TestNormalizeTechniqueIDRejectsATacticID(t *testing.T) {+ if got := normalizeTechniqueID("TA0006"); got != "" {+ t.Errorf("normalizeTechniqueID(\"TA0006\") = %q, want \"\" — TA0006 is a tactic, not a technique", got)+ }+ if got := normalizeTechniqueID("T10591"); got != "" {+ t.Errorf("normalizeTechniqueID(\"T10591\") = %q, want \"\"", got)+ }+ if got := normalizeTechniqueID("T1059.001"); got != "T1059.001" {+ t.Errorf("normalizeTechniqueID(\"T1059.001\") = %q, want it accepted unchanged", got)+ }+}++func TestWellFormedTechniqueShapesAreAccepted(t *testing.T) {+ for _, good := range []string{"T1059", "T1059.001", "t1003.001"} {+ if got := extractTechniqueIDs(map[string]interface{}{"techniques": good}); len(got) != 1 {+ t.Errorf("extractTechniqueIDs(%q) = %v, want one id", good, got)+ }+ }+}diff --git a/services/ingest/internal/normalizer/normalizer.go b/services/ingest/internal/normalizer/normalizer.goindex f23b57d7f..003202ef5 100644--- a/services/ingest/internal/normalizer/normalizer.go+++ b/services/ingest/internal/normalizer/normalizer.go@@ -5,6 +5,7 @@ import ("context""encoding/json""fmt"+ "regexp""strings""time"@@ -210,9 +211,15 @@ var connectorProfiles = map[string]connectorProfile{classUID: 2001,className: "Security Finding",fieldMap: map[string]string{- "UpdatedAt": "time",- "Title": "message",- "Description": "raw_data",+ "UpdatedAt": "time",+ "Title": "message",+ // Not `raw_data`: Normalize overwrites that slot unconditionally+ // with the serialized payload a few dozen lines after the field+ // map runs, so the Description was mapped and then discarded+ // every time (issue #1244). `finding.desc` is where every other+ // profile puts a vendor description and is what the fusion+ // promoter reads.+ "Description": "finding.desc","Severity.Label": "severity",},severityMap: map[string]int{@@ -536,6 +543,15 @@ var _canonicalAliases = []struct {// to the vendor profiles without giving them a second mechanism.{"message", []string{"title"}},{"finding.uid", []string{"external_id"}},+ // Issue #1244. Nothing mapped a top-level `description` onto any OCSF+ // slot outside the handful of profiles that name it, so a pushed event+ // carrying both a title and a description arrived with only the title —+ // and the fusion promoter, finding no description anywhere, fell back to+ // `message`, which is the title. The alert then showed one sentence+ // twice. `finding.desc` is the destination the profiles that *do* map it+ // already use (aisoc_sample, ai_guardrail, email_inbox, cloudtrail), and+ // it is the first slot promoter._description reads after the payload.+ {"finding.desc", []string{"description"}},}// connectorTypeAliases maps a connector identifier the connectors service@@ -892,24 +908,40 @@ func (n *Normalizer) Normalize(raw *RawEvent) (*NormalizedEvent, error) {ocsf["raw_data"] = string(rawBytes)}- // ATT&CK technique enrichment- if attck.Loaded() {- if techIDs := extractTechniqueIDs(raw.Payload); len(techIDs) > 0 {- var enriched []map[string]interface{}- for _, tid := range techIDs {- if tech := attck.Lookup(tid); tech != nil {- enriched = append(enriched, map[string]interface{}{- "technique_id": tech.ID,- "technique_name": tech.Name,- "tactic_ids": tech.TacticIDs,- "tactic_names": tech.TacticNames,- "url": tech.URL,- })- }- }- if len(enriched) > 0 {- ocsf["mitre_attck"] = enriched+ // ATT&CK technique enrichment.+ //+ // The whole block used to sit behind `attck.Loaded()` and each id behind+ // `attck.Lookup(tid) != nil`, so a deployment that cannot reach+ // raw.githubusercontent.com — air-gapped, egress-filtered, or simply+ // offline at boot — published `mitre_techniques: []` on every event+ // forever, whatever the vendor sent (issue #1244).+ //+ // A well-formed id is now carried either way. What the corpus adds is the+ // technique name, its tactics and its URL; what the vendor asserted is+ // the id, and dropping the vendor's assertion because we could not look+ // it up loses information we were handed. `verified` says which is which+ // so a reader is never told an unconfirmed id was confirmed.+ if techIDs := extractTechniqueIDs(raw.Payload); len(techIDs) > 0 {+ var enriched []map[string]interface{}+ for _, tid := range techIDs {+ if tech := attck.Lookup(tid); tech != nil {+ enriched = append(enriched, map[string]interface{}{+ "technique_id": tech.ID,+ "technique_name": tech.Name,+ "tactic_ids": tech.TacticIDs,+ "tactic_names": tech.TacticNames,+ "url": tech.URL,+ "verified": true,+ })+ continue}+ enriched = append(enriched, map[string]interface{}{+ "technique_id": tid,+ "verified": false,+ })+ }+ if len(enriched) > 0 {+ ocsf["mitre_attck"] = enriched}}@@ -959,9 +991,17 @@ func extractTechniqueIDs(payload map[string]interface{}) []string {seen := map[string]struct{}{}var results []string+ // `techniques` and `technique` are the bare forms a pushed payload uses+ // (issue #1244: a CrowdStrike-shaped event sending+ // `techniques: ["T1003.001"]` produced `mitre_techniques: []`). The list+ // was all prefixed or suffixed spellings and had no entry for the plain+ // noun, singular or plural.+ //+ // `tactic_id` is kept for the payload that files a technique under it,+ // but a genuine tactic id no longer survives normalizeTechniqueID.candidateKeys := []string{"technique_id", "mitre_technique", "attck_technique", "tactic_id",- "mitre_techniques", "attack_technique",+ "mitre_techniques", "attack_technique", "techniques", "technique",}for _, key := range candidateKeys {val, ok := payload[key]@@ -992,15 +1032,22 @@ func extractTechniqueIDs(payload map[string]interface{}) []string {return results}+// techniqueIDRe is the exact ATT&CK technique shape: T followed by four+// digits, optionally a three-digit sub-technique.+//+// The check used to be a length range ("leading T, 5–7 characters before the+// dot"), which also accepted `TA0006` — a *tactic* id — and `T10591`. That+// was harmless only because every id then had to survive attck.Lookup, which+// has no entry for either, so the loose ones were filtered downstream. Since+// issue #1244 a well-formed id is carried even when the corpus cannot confirm+// it, so the shape check is now the only filter and has to be exact.+var techniqueIDRe = regexp.MustCompile(`^T\d{4}(?:\.\d{3})?$`)+// normalizeTechniqueID extracts a clean ATT&CK technique ID from a string.func normalizeTechniqueID(s string) string {s = strings.TrimSpace(strings.ToUpper(s))- // Accept T1234 or T1234.001- if len(s) >= 5 && s[0] == 'T' {- parts := strings.SplitN(s, ".", 2)- if len(parts[0]) >= 5 && len(parts[0]) <= 7 {- return s- }+ if techniqueIDRe.MatchString(s) {+ return s}return ""}