Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

fix(ingest,fusion): stop dropping description, techniques and entities (#1286)

Beenu Aroraf347ab1087a0
diff --git a/services/api/app/api/v1/endpoints/metrics.py b/services/api/app/api/v1/endpoints/metrics.py
index 67dda690c..b85a80b8d 100644
--- a/services/api/app/api/v1/endpoints/metrics.py
+++ b/services/api/app/api/v1/endpoints/metrics.py
@@ -1036,6 +1036,14 @@ async def _funnel_window(db, tenant_id, start, end, *, mitre_total: int) -> dict
# (i.e. multiple raw events fused into a single alert). Single-event
# alerts are excluded so the ratio stays meaningful when one detection
# rule fires repeatedly.
+ #
+ # The predicate was always right and was unsatisfiable until issue #1244:
+ # the only pipeline writer appended one id *before* the insert, and a
+ # deduplicated event's insert is filtered out by the sink's
+ # `WHERE NOT EXISTS`, so no row could ever reach two. The tile read 0 by
+ # construction on a pipeline that was deduplicating the whole time.
+ # `AlertSink._record_deduplicated_event` is what feeds this now; a zero
+ # here means no event deduplicated in the window, not that nobody looked.
correlation_instances = (
await db.scalar(
select(func.count()).where(
diff --git a/services/fusion/app/services/alert_sink.py b/services/fusion/app/services/alert_sink.py
index 810c602e6..0425ef0d0 100644
--- a/services/fusion/app/services/alert_sink.py
+++ b/services/fusion/app/services/alert_sink.py
@@ -72,7 +72,8 @@ INSERT INTO alerts (
dedup_hash, confidence, confidence_label, confidence_rationale,
narrative, anomaly_score, event_time,
connector_id, connector_type, source_event_ids, ocsf_class_uid,
- rule_id, rule_name, external_id
+ rule_id, rule_name, external_id,
+ affected_hosts, affected_users, affected_ips
)
SELECT
$1, $2, $3, $4, $5, 'new',
@@ -80,13 +81,51 @@ SELECT
$11::text, $12, $13, $14::jsonb,
$15, $16, COALESCE($17, NOW()),
$18, $19, $20::jsonb, $21,
- $22, $23, $24
+ $22, $23, $24,
+ $25::jsonb, $26::jsonb, $27::jsonb
WHERE NOT EXISTS (
SELECT 1 FROM alerts WHERE tenant_id = $2 AND dedup_hash = $11::text
)
ON CONFLICT (id) DO NOTHING
RETURNING id
"""
+# The three entity columns are appended rather than inserted beside
+# `entities`, where they belong semantically, so that every existing
+# positional index into `_insert_args` keeps its meaning. Renumbering 24
+# placeholders to make a list read nicely is how a confidence score ends up
+# in a narrative column.
+
+#: Record a raw event that deduplicated onto an alert that already exists.
+#:
+#: Issue #1244. `source_event_ids` is "the raw events that produced this
+#: alert" and the funnel's Correlation Instances tile counts the alerts where
+#: it holds two or more. The only writer appended exactly one id *before* the
+#: insert, and a deduplicated event's insert is filtered out by the
+#: `WHERE NOT EXISTS` above — so the second, third and five-hundredth event to
+#: fold onto an alert left no trace, the predicate could never be satisfied,
+#: and the tile read zero on a pipeline that was deduplicating the whole time.
+#:
+#: One id per statement, with the `@>` guard making a replay a no-op rather
+#: than a second entry — a consumer-group rebalance re-delivers the same event
+#: and the provenance list must not grow on it. `$4` caps the array because a
+#: hot dedup key would otherwise grow one row without bound; past the cap the
+#: alert still exists and still deduplicates, only the list stops growing.
+#: jsonb containment accepts a scalar against an array element, so
+#: ``'["evt-1"]'::jsonb @> '"evt-1"'::jsonb`` is the membership test.
+_DEDUP_SOURCE_EVENT_SQL = """
+UPDATE alerts
+ SET source_event_ids = source_event_ids || to_jsonb($3::text)
+ WHERE tenant_id = $1
+ AND dedup_hash = $2::text
+ AND NOT (source_event_ids @> to_jsonb($3::text))
+ AND jsonb_array_length(source_event_ids) < $4
+"""
+
+#: Ceiling on `source_event_ids` per alert. 200 ids is ~8 KB of JSONB, and the
+#: question the column answers ("did more than one event produce this?") is
+#: settled by the second entry.
+_MAX_SOURCE_EVENT_IDS = 200
+
# The reconciliation row the two-way SIEM loop reads. Written here rather than
# from the API because fusion is the only place that holds the alert id, the
# connector instance and the vendor finding id at the same moment — asking the
@@ -170,6 +209,29 @@ def _entities(fused: FusedAlert) -> list[dict[str, str]]:
return out
+def _affected(fused: FusedAlert) -> tuple[list[str], list[str], list[str]]:
+ """The denormalized `affected_hosts` / `affected_users` / `affected_ips`.
+
+ Issue #1244. These three columns have existed since migration 042, the
+ ORM maps them, `GET /alerts/{id}` serialises them, and `feedback.py`,
+ `rule_tuning.py`, `business_context.py`, `cases.py` and the GraphQL
+ resolver all read them — but the production INSERT never named them, so
+ the only rows that ever had a value were the ones `seed_demo.py` wrote.
+ Every reader saw `[]` on a real alert and a populated list on a seeded
+ one.
+
+ Derived from the same fields `_entities` and `_iocs` already use rather
+ than from a second source, so the row cannot disagree with itself.
+ `entities` stays exactly as it was: it is the shape the Investigation
+ Rail reads and these columns are the shape the filters read.
+ """
+ alert = fused.alert
+ hosts = [alert.hostname] if alert.hostname else []
+ users = [alert.username] if alert.username else []
+ ips = [ip for ip in (alert.src_ip, alert.dst_ip) if ip]
+ return hosts, users, ips
+
+
def _insert_args(fused: FusedAlert) -> tuple[Any, ...]:
"""The positional arguments `_INSERT_SQL` takes, in order.
@@ -179,6 +241,7 @@ def _insert_args(fused: FusedAlert) -> tuple[Any, ...]:
narrative without anything failing.
"""
alert = fused.alert
+ hosts, users, ips = _affected(fused)
return (
fused.id,
alert.tenant_id,
@@ -204,6 +267,9 @@ def _insert_args(fused: FusedAlert) -> tuple[Any, ...]:
alert.rule_id,
alert.rule_name,
alert.external_id,
+ json.dumps(hosts),
+ json.dumps(users),
+ json.dumps(ips),
)
@@ -245,7 +311,19 @@ class AlertSink:
DB is reachable.
"""
canonical_id = str(fused.id)
+ alert = fused.alert
if fused.fusion_decision == FusionDecision.DUPLICATE:
+ # Fusion has already decided: there is no row to write, and a
+ # database outage cannot change that verdict. The folded-away
+ # event is still part of the surviving alert's provenance, so
+ # record it when there is a pool — best-effort, never a reconnect
+ # attempt, and never a different outcome.
+ if self._pool is not None:
+ try:
+ async with self._pool.acquire() as conn:
+ await self._record_deduplicated_event(conn, alert)
+ except Exception as exc: # noqa: BLE001 — provenance is not worth the verdict
+ logger.warning("alert_sink.dedup_provenance_failed", error=str(exc))
return PersistResult(PersistOutcome.DUPLICATE, canonical_id)
pool = await self._ensure_pool()
@@ -256,13 +334,13 @@ class AlertSink:
return PersistResult(PersistOutcome.UNAVAILABLE, canonical_id)
self._connect_failed_logged = False
- alert = fused.alert
try:
async with pool.acquire() as conn:
row = await conn.fetchrow(_INSERT_SQL, *_insert_args(fused))
- if row is None:
- logger.debug("alert_sink.dedup_skip", fingerprint=alert.fingerprint())
- return PersistResult(PersistOutcome.DUPLICATE, canonical_id)
+ if row is None:
+ logger.debug("alert_sink.dedup_skip", fingerprint=alert.fingerprint())
+ await self._record_deduplicated_event(conn, alert)
+ return PersistResult(PersistOutcome.DUPLICATE, canonical_id)
await self._link_source_finding(pool, alert, alert_id=row["id"])
return PersistResult(PersistOutcome.INSERTED, str(row["id"]))
except asyncpg.ForeignKeyViolationError:
@@ -319,6 +397,7 @@ class AlertSink:
for fused in fused_batch:
canonical_id = str(fused.id)
if fused.fusion_decision == FusionDecision.DUPLICATE:
+ await self._record_deduplicated_event(conn, fused.alert)
results.append(PersistResult(PersistOutcome.DUPLICATE, canonical_id))
continue
try:
@@ -333,6 +412,7 @@ class AlertSink:
continue
if row is None:
+ await self._record_deduplicated_event(conn, fused.alert)
results.append(PersistResult(PersistOutcome.DUPLICATE, canonical_id))
continue
results.append(PersistResult(PersistOutcome.INSERTED, str(row["id"])))
@@ -346,6 +426,30 @@ class AlertSink:
return results
+ async def _record_deduplicated_event(self, conn: Any, alert: Any) -> None:
+ """Attach a folded-away event's id to the alert it deduplicated onto.
+
+ Best-effort, for the same reason `_link_source_finding` is: the
+ provenance entry is worth one statement, never the verdict the caller
+ is about to return. A failure here must not turn a correct DUPLICATE
+ into a FAILED.
+ """
+ ids = [i for i in (alert.source_event_ids or []) if i]
+ if not ids:
+ return
+ fingerprint = alert.fingerprint()
+ try:
+ for source_event_id in ids:
+ await conn.execute(
+ _DEDUP_SOURCE_EVENT_SQL,
+ alert.tenant_id,
+ fingerprint,
+ str(source_event_id),
+ _MAX_SOURCE_EVENT_IDS,
+ )
+ except Exception as exc: # noqa: BLE001 — provenance is not worth the alert
+ logger.warning("alert_sink.dedup_provenance_failed", error=str(exc))
+
async def _link_source_finding(self, pool: asyncpg.Pool, alert: Any, *, alert_id: Any) -> None:
"""Record which vendor finding produced this alert, when there is one.
diff --git a/services/fusion/app/services/detection_engine.py b/services/fusion/app/services/detection_engine.py
index b5d77c4fb..1815c7cea 100644
--- a/services/fusion/app/services/detection_engine.py
+++ b/services/fusion/app/services/detection_engine.py
@@ -48,6 +48,7 @@ from app.models.alert import AlertSeverity, RawAlert
from app.services.derived_fields import enrich, requested_derived_fields
from app.services.detection_matcher import matches
from app.services.provenance import extract_provenance
+from app.services.raw_data import decode_raw_data
logger = structlog.get_logger()
@@ -157,15 +158,9 @@ class DetectionEngine:
@staticmethod
def _decode_raw_data(ocsf: dict[str, Any]) -> dict[str, Any]:
"""The connector's normalized dict, or the OCSF top level as fallback."""
- raw = ocsf.get("raw_data")
- if isinstance(raw, str) and raw.strip():
- try:
- parsed = json.loads(raw)
- if isinstance(parsed, dict):
- return parsed
- except (ValueError, TypeError):
- # Malformed raw_data JSON — fall through to the OCSF top level below.
- pass
+ decoded = decode_raw_data(ocsf)
+ if decoded is not None:
+ return decoded
# Fall back to the OCSF top level (some connectors emit flat OCSF).
return ocsf if isinstance(ocsf, dict) else {}
diff --git a/services/fusion/app/services/promoter.py b/services/fusion/app/services/promoter.py
index 0701d8637..fffa1b883 100644
--- a/services/fusion/app/services/promoter.py
+++ b/services/fusion/app/services/promoter.py
@@ -46,6 +46,7 @@ import structlog
from app.models.alert import AlertSeverity, RawAlert
from app.services.provenance import extract_provenance, product_label
+from app.services.raw_data import decode_raw_data
logger = structlog.get_logger()
@@ -141,11 +142,17 @@ def _description(ocsf: dict[str, Any]) -> str:
The raw payload is not lost: it stays on the alert's `raw_event`, which is
what the investigation surfaces read.
+
+ Issue #1244: that vendor-description branch used to be guarded by
+ `isinstance(raw, dict)` while ingest writes `raw_data` as the JSON string
+ `json.Marshal(raw.Payload)` produces, so on the production path it was
+ unreachable and every supplied description was discarded. `decode_raw_data`
+ is the decoder the detection engine already used for the same value.
"""
- raw = ocsf.get("raw_data")
- if isinstance(raw, dict):
+ payload = decode_raw_data(ocsf)
+ if payload is not None:
for key in ("description", "message", "summary", "detail", "reason"):
- value = raw.get(key)
+ value = payload.get(key)
if isinstance(value, str) and value.strip():
return value.strip()[:2000]
finding_desc = _get_nested(ocsf, "finding", "desc")
@@ -369,11 +376,22 @@ def promote_normalized_event(message: dict[str, Any]) -> RawAlert | None:
tactics, techniques = _mitre(ocsf)
connector_id, connector_type, class_uid = extract_provenance(message, ocsf)
+ title = _title(ocsf)
+ description = _description(ocsf)
+ # The last resort in `_description` is `ocsf["message"]`, which is the
+ # first key `_title` reads, so a vendor that supplied only a title used to
+ # get that one sentence rendered twice — once as the alert heading and
+ # again as its body. Nothing is lost by leaving the slot empty: the
+ # console falls back to the raw event, and an empty description is the
+ # honest statement that the vendor wrote no prose.
+ if description == title:
+ description = ""
+
return RawAlert(
tenant_id=tenant_id,
source=_source(ocsf),
- title=_title(ocsf),
- description=_description(ocsf),
+ title=title,
+ description=description,
severity=severity,
src_ip=_get_nested(ocsf, "src_endpoint", "ip"),
dst_ip=_get_nested(ocsf, "dst_endpoint", "ip"),
diff --git a/services/fusion/app/services/raw_data.py b/services/fusion/app/services/raw_data.py
new file mode 100644
index 000000000..a5d5f7c20
--- /dev/null
+++ b/services/fusion/app/services/raw_data.py
@@ -0,0 +1,43 @@
+"""One decoder for the connector payload ``services/ingest`` attaches.
+
+``raw_data`` is a JSON **string**, not a dict:
+``normalizer.go`` runs ``json.Marshal(raw.Payload)`` and stores the result as
+``ocsf["raw_data"]``. Two fusion modules need the payload back — the detection
+engine, to build the field namespace a rule matches against, and the promoter,
+to find the vendor's own description — and before issue #1244 only the
+detection engine knew the shape. The promoter guarded its branch with
+``isinstance(raw, dict)``, which no ingest event satisfies, so the vendor's
+description was dropped on every event that carried one.
+
+A dict is accepted as well as a string. Some producers (the API's direct
+``/alerts/submit`` path, fixtures written before ingest serialised) hand the
+payload through unserialised, and refusing it would trade one dead branch for
+another.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+
+def decode_raw_data(ocsf: dict[str, Any]) -> dict[str, Any] | None:
+ """The connector's own payload, or ``None`` when there isn't one.
+
+ ``None`` rather than ``{}`` so a caller can distinguish "no payload" from
+ "an empty payload" and apply its own fallback — the detection engine falls
+ back to the OCSF top level, the promoter does not.
+ """
+ raw = ocsf.get("raw_data")
+ if isinstance(raw, dict):
+ return raw
+ if isinstance(raw, str) and raw.strip():
+ try:
+ parsed = json.loads(raw)
+ except (ValueError, TypeError):
+ # Malformed JSON in raw_data. The caller decides what to do with
+ # nothing; one bad payload must not raise out of the hot path.
+ return None
+ if isinstance(parsed, dict):
+ return parsed
+ return None
diff --git a/services/fusion/app/workers/consumer.py b/services/fusion/app/workers/consumer.py
index 2d6133b29..07f60a422 100644
--- a/services/fusion/app/workers/consumer.py
+++ b/services/fusion/app/workers/consumer.py
@@ -304,6 +304,17 @@ class FusionWorker:
# Phase A2 — run the executable detection corpus against the live
# event. Each firing rule becomes a RawAlert routed through fusion,
# so telemetry that isn't a vendor-asserted finding still alerts.
+ #
+ # One event firing N rules produces N alert rows, deliberately
+ # (issue #1244 reported one Sysmon event becoming five or six).
+ # Each row names the rule that fired, carries that rule's severity
+ # and category, and is dispositioned on its own: collapsing them
+ # would mean picking one rule's verdict to stand for the others,
+ # and an analyst who closes "suspicious PowerShell" has not
+ # thereby closed "LSASS access". `Correlator.correlate` already
+ # groups them into one incident on
+ # {tenant}:{entity}:{tactic} for the console, which is where the
+ # N-into-1 view belongs.
if self._detector is not None:
for hit in await self._detector.evaluate_async(payload, overlay, self._first_seen):
det_alert = self._detector.build_alert(payload, hit)
diff --git a/services/fusion/tests/test_alert_sink_batch.py b/services/fusion/tests/test_alert_sink_batch.py
index 521337ec3..d68d10939 100644
--- a/services/fusion/tests/test_alert_sink_batch.py
+++ b/services/fusion/tests/test_alert_sink_batch.py
@@ -11,6 +11,8 @@ is pinned here is the contract rather than the speed.
from __future__ import annotations
+import re
+
import pytest
from app.services.alert_sink import PersistOutcome
@@ -56,9 +58,18 @@ class TestTheArgumentTupleHasOneDefinition:
into a narrative without anything failing."""
from app.services.alert_sink import _INSERT_SQL
- placeholders = {int(tok[1:]) for tok in _INSERT_SQL.split() if tok.startswith("$") and tok[1:].isdigit()}
+ # Matched with a regex rather than by splitting on whitespace: almost
+ # every placeholder in this statement carries a cast or a trailing
+ # comma (`$9::jsonb,`), so the whitespace form only ever saw the two
+ # bare ones and the width assertion below was nearly vacuous.
+ placeholders = {int(n) for n in re.findall(r"\$(\d+)", _INSERT_SQL)}
assert placeholders, "no positional placeholders found; this test is checking nothing"
assert max(placeholders) >= 20, "the insert shrank unexpectedly — re-check _insert_args"
+ # No gaps: asyncpg binds positionally, so a skipped number means every
+ # argument after it lands in the wrong column.
+ assert placeholders == set(range(1, max(placeholders) + 1)), (
+ f"placeholder numbering has holes: {sorted(set(range(1, max(placeholders) + 1)) - placeholders)}"
+ )
@pytest.mark.asyncio
diff --git a/services/fusion/tests/test_alert_sink_issue_1244.py b/services/fusion/tests/test_alert_sink_issue_1244.py
new file mode 100644
index 000000000..46dc12276
--- /dev/null
+++ b/services/fusion/tests/test_alert_sink_issue_1244.py
@@ -0,0 +1,293 @@
+"""The alert row carries the entities fusion computed, and records dedup.
+
+Issue #1244, two independent gaps in what ``AlertSink`` writes.
+
+**affected_hosts / affected_users / affected_ips were never written.** The
+columns have existed since migration 042, the ORM maps them, the alerts
+endpoint serialises them and five readers filter on them — and the only
+writer in the tree was ``seed_demo.py``, so every one of those readers saw
+``[]`` on a real alert and a populated list on a seeded one. Fusion already
+derives the same host, user and IPs for ``entities`` and ``iocs``; it just
+did not name the columns in its INSERT.
+
+**A deduplicated event left no trace.** ``source_event_ids`` is the column
+the funnel's "Correlation Instances" tile counts (``jsonb_array_length >= 2``
+in ``services/api/app/api/v1/endpoints/metrics.py``), and the only writer
+appends one id before the insert. When a second event dedups onto an alert the
+insert is filtered by ``WHERE NOT EXISTS`` and the second event's id is
+dropped on the floor, so the predicate could never be satisfied and the tile
+read 0 by construction — "522 events formed 0 correlation instances" on a
+pipeline that was deduplicating the whole time.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import uuid
+from contextlib import asynccontextmanager
+
+import pytest
+from app.models.alert import AlertSeverity, FusedAlert, FusionDecision, RawAlert
+from app.services.alert_sink import AlertSink, PersistOutcome
+
+TENANT = uuid.UUID("00000000-0000-0000-0000-000000000001")
+CONNECTOR = uuid.UUID("00000000-0000-0000-0000-0000000000c1")
+
+
+def _fused(
+ decision: FusionDecision = FusionDecision.NEW_ALERT,
+ *,
+ source_event_ids: list[str] | None = None,
+) -> FusedAlert:
+ raw = RawAlert(
+ tenant_id=TENANT,
+ source="CrowdStrike Falcon",
+ title="Credential dumping via LSASS access",
+ severity=AlertSeverity.CRITICAL,
+ src_ip="10.20.30.40",
+ dst_ip="10.20.30.99",
+ hostname="WIN-DC01",
+ username="svc-backup",
+ mitre_techniques=["T1003"],
+ connector_id=CONNECTOR,
+ connector_type="crowdstrike_falcon",
+ source_event_ids=["evt-1"] if source_event_ids is None else list(source_event_ids),
+ ocsf_class_uid=2004,
+ )
+ raw.id = raw.deterministic_id()
+ return FusedAlert(id=raw.id, tenant_id=TENANT, fusion_decision=decision, alert=raw)
+
+
+class _StubConn:
+ """Records every statement, and answers fetchrow with a scripted row."""
+
+ def __init__(self, row):
+ self._row = row
+ self.calls: list[tuple] = []
+
+ async def fetchrow(self, sql, *args):
+ self.calls.append((sql, args))
+ return self._row
+
+ async def execute(self, sql, *args):
+ self.calls.append((sql, args))
+ return "UPDATE 1"
+
+
+class _StubPool:
+ def __init__(self, row):
+ self.conn = _StubConn(row)
+
+ @asynccontextmanager
+ async def _acquire(self):
+ yield self.conn
+
+ def acquire(self):
+ return self._acquire()
+
+
+def _insert_statement(pool: _StubPool) -> tuple[str, tuple]:
+ for sql, args in pool.conn.calls:
+ if "INSERT INTO alerts" in sql:
+ return sql, args
+ raise AssertionError(f"no INSERT INTO alerts issued; calls={[c[0][:40] for c in pool.conn.calls]}")
+
+
+def _split_top_level(expr_list: str) -> list[str]:
+ """Split a SELECT list on commas that are not inside parentheses.
+
+ `COALESCE($17, NOW())` is one expression, not two.
+ """
+ out: list[str] = []
+ depth = 0
+ current = ""
+ for char in expr_list:
+ if char == "(":
+ depth += 1
+ elif char == ")":
+ depth -= 1
+ if char == "," and depth == 0:
+ out.append(current.strip())
+ current = ""
+ continue
+ current += char
+ if current.strip():
+ out.append(current.strip())
+ return out
+
+
+def _column_value(sql: str, args: tuple, column: str):
+ """Read one column's bound value by name, via the placeholder it selects.
+
+ By name rather than by position: a positional assertion silently follows
+ the wrong value the moment a column is inserted ahead of it, which is the
+ drift `_insert_args` exists to prevent. Column index and argument index
+ are *not* the same — `status` is selected as the literal `'new'` and
+ consumes no placeholder — so the column list is zipped against the SELECT
+ list and the `$N` is read off the matching expression.
+ """
+ head, _, tail = sql.partition("INSERT INTO alerts (")
+ assert head is not None and tail, "not an alerts INSERT"
+ columns = [c.strip() for c in tail.split(")", 1)[0].split(",")]
+ select_list = tail.split("SELECT", 1)[1].split("WHERE NOT EXISTS", 1)[0]
+ exprs = _split_top_level(select_list)
+ assert len(columns) == len(exprs), f"{len(columns)} columns against {len(exprs)} select expressions"
+ assert column in columns, f"{column} is not in the INSERT column list: {columns}"
+
+ expr = exprs[columns.index(column)]
+ match = re.match(r"\$(\d+)", expr)
+ assert match, f"{column} is selected as {expr!r}, not a bound parameter"
+ return args[int(match.group(1)) - 1]
+
+
+# ── affected_* are written from the entities fusion already computes ──────
+
+
+async def test_affected_hosts_users_and_ips_are_persisted() -> None:
+ fused = _fused()
+ pool = _StubPool(row={"id": fused.id})
+ sink = AlertSink("postgresql://x")
+ sink._pool = pool
+
+ result = await sink.persist(fused)
+ assert result.outcome is PersistOutcome.INSERTED
+
+ sql, args = _insert_statement(pool)
+ assert json.loads(_column_value(sql, args, "affected_hosts")) == ["WIN-DC01"]
+ assert json.loads(_column_value(sql, args, "affected_users")) == ["svc-backup"]
+ assert json.loads(_column_value(sql, args, "affected_ips")) == ["10.20.30.40", "10.20.30.99"]
+
+
+async def test_absent_entities_persist_as_empty_lists_not_nulls() -> None:
+ """The columns are ``JSONB NOT NULL DEFAULT '[]'`` — a NULL would fail."""
+ bare = RawAlert(tenant_id=TENANT, source="ingest", title="No entities anywhere")
+ bare.id = bare.deterministic_id()
+ fused = FusedAlert(id=bare.id, tenant_id=TENANT, fusion_decision=FusionDecision.NEW_ALERT, alert=bare)
+ pool = _StubPool(row={"id": fused.id})
+ sink = AlertSink("postgresql://x")
+ sink._pool = pool
+
+ await sink.persist(fused)
+
+ sql, args = _insert_statement(pool)
+ for column in ("affected_hosts", "affected_users", "affected_ips"):
+ assert json.loads(_column_value(sql, args, column)) == [], column
+
+
+async def test_the_batch_path_writes_the_same_columns() -> None:
+ """`persist_many` shares `_insert_args`; prove it rather than assume it."""
+ fused = _fused()
+ pool = _StubPool(row={"id": fused.id})
+ sink = AlertSink("postgresql://x")
+ sink._pool = pool
+
+ await sink.persist_many([fused])
+
+ sql, args = _insert_statement(pool)
+ assert json.loads(_column_value(sql, args, "affected_hosts")) == ["WIN-DC01"]
+
+
+# ── a deduplicated event is recorded on the row it deduped onto ───────────
+
+
+async def test_a_dedup_hit_records_the_second_events_id() -> None:
+ """The defect behind "522 events formed 0 correlation instances"."""
+ pool = _StubPool(row=None) # WHERE NOT EXISTS filtered the insert out
+ sink = AlertSink("postgresql://x")
+ sink._pool = pool
+ fused = _fused(source_event_ids=["evt-2"])
+
+ result = await sink.persist(fused)
+ assert result.outcome is PersistOutcome.DUPLICATE
+
+ updates = [(sql, args) for sql, args in pool.conn.calls if sql.strip().upper().startswith("UPDATE ALERTS")]
+ assert updates, (
+ "a deduplicated event left no trace on the alert it deduped onto, so "
+ "source_event_ids can never reach length 2 and the Correlation "
+ "Instances tile is a structural zero"
+ )
+ sql, args = updates[0]
+ assert "source_event_ids" in sql
+ assert TENANT in args
+ assert fused.alert.fingerprint() in args
+ assert "evt-2" in json.dumps(args, default=str)
+
+
+async def test_a_fusion_decided_duplicate_is_also_recorded() -> None:
+ """Fusion's own DUPLICATE verdict returns before the insert is attempted."""
+ pool = _StubPool(row=None)
+ sink = AlertSink("postgresql://x")
+ sink._pool = pool
+
+ result = await sink.persist(_fused(FusionDecision.DUPLICATE, source_event_ids=["evt-3"]))
+ assert result.outcome is PersistOutcome.DUPLICATE
+
+ updates = [(sql, args) for sql, args in pool.conn.calls if sql.strip().upper().startswith("UPDATE ALERTS")]
+ assert updates, "fusion's in-memory dedup verdict also loses the source event id"
+ assert "evt-3" in json.dumps(updates[0][1], default=str)
+
+
+async def test_an_insert_that_succeeded_issues_no_dedup_update() -> None:
+ """The extra round trip is paid only when there is something to record."""
+ fused = _fused()
+ pool = _StubPool(row={"id": fused.id})
+ sink = AlertSink("postgresql://x")
+ sink._pool = pool
+
+ await sink.persist(fused)
+
+ updates = [sql for sql, _ in pool.conn.calls if sql.strip().upper().startswith("UPDATE ALERTS")]
+ assert updates == []
+
+
+async def test_an_event_with_no_source_id_issues_no_dedup_update() -> None:
+ pool = _StubPool(row=None)
+ sink = AlertSink("postgresql://x")
+ sink._pool = pool
+
+ await sink.persist(_fused(source_event_ids=[]))
+
+ updates = [sql for sql, _ in pool.conn.calls if sql.strip().upper().startswith("UPDATE ALERTS")]
+ assert updates == []
+
+
+async def test_a_duplicate_verdict_survives_an_unreachable_database() -> None:
+ """Recording the provenance must not make DUPLICATE depend on the DB.
+
+ Fusion decided; an outage changes only whether we can write the id down.
+ The early return has to stay ahead of `_ensure_pool`, or a DUPLICATE
+ starts reporting UNAVAILABLE and the consumer retries a decision that
+ was already made.
+ """
+ sink = AlertSink("postgresql://nope:[email protected]:1/none")
+ sink._pool = None
+
+ result = await sink.persist(_fused(FusionDecision.DUPLICATE))
+ assert result.outcome is PersistOutcome.DUPLICATE
+
+
+async def test_a_failing_dedup_update_does_not_change_the_outcome() -> None:
+ """Recording provenance is best-effort; it must never cost the verdict."""
+
+ class _BoomConn(_StubConn):
+ async def execute(self, sql, *args):
+ self.calls.append((sql, args))
+ raise RuntimeError("connection reset")
+
+ pool = _StubPool(row=None)
+ pool.conn = _BoomConn(row=None)
+ sink = AlertSink("postgresql://x")
+ sink._pool = pool
+
+ result = await sink.persist(_fused(source_event_ids=["evt-4"]))
+ assert result.outcome is PersistOutcome.DUPLICATE
diff --git a/services/fusion/tests/test_promoter_raw_data_string.py b/services/fusion/tests/test_promoter_raw_data_string.py
new file mode 100644
index 000000000..d552d52c2
--- /dev/null
+++ b/services/fusion/tests/test_promoter_raw_data_string.py
@@ -0,0 +1,113 @@
+"""The promoter reads ``raw_data`` in the shape ingest actually writes it.
+
+Issue #1244. ``services/ingest`` serialises the connector payload before it
+attaches it::
+
+ if rawBytes, err := json.Marshal(raw.Payload); err == nil {
+ ocsf["raw_data"] = string(rawBytes)
+ }
+
+so ``raw_data`` on every ingest-normalized event is a JSON **string**. The
+promoter's description resolver guarded its only vendor-description branch
+with ``isinstance(raw, dict)``, which that value can never satisfy, so the
+branch was unreachable on the production path and a supplied description was
+dropped. Execution fell through to ``ocsf["message"]`` — the same key
+``_title`` reads first — so title and description came back identical.
+
+The existing coverage in ``test_promoter_alert_text.py`` passes ``raw_data``
+as a dict, which is the shape no ingest event has: the double was more
+capable than the real thing, so the tests passed while the path was dead.
+Every case here uses the serialised string.
+"""
+
+from __future__ import annotations
+
+import json
+import uuid
+
+from app.services.promoter import _description, promote_normalized_event
+
+_TENANT = str(uuid.uuid4())
+
+
+def _ingest_event(payload: dict) -> dict:
+ """An OCSF event shaped the way ``services/ingest`` emits one."""
+ return {
+ "class_uid": 2001,
+ "class_name": "Security Finding",
+ "severity_id": 4,
+ "message": payload.get("title", ""),
+ "metadata": {"product": {"vendor_name": "CrowdStrike", "name": "Falcon"}},
+ "raw_data": json.dumps(payload),
+ "tenant_uid": _TENANT,
+ }
+
+
+def test_a_serialized_raw_data_still_yields_the_vendors_description() -> None:
+ """The defect: this returned the title, because the dict branch was dead."""
+ ocsf = _ingest_event(
+ {
+ "title": "Credential dumping detected",
+ "description": "lsass.exe memory read by rundll32.exe (comsvcs.dll MiniDump)",
+ "host": "WIN-FIN-01",
+ }
+ )
+ assert _description(ocsf) == "lsass.exe memory read by rundll32.exe (comsvcs.dll MiniDump)"
+
+
+def test_description_and_title_are_not_the_same_string() -> None:
+ """The symptom an analyst sees: two panes showing one sentence."""
+ alert = promote_normalized_event(
+ {
+ "tenant_id": _TENANT,
+ "ocsf_event": _ingest_event(
+ {
+ "title": "Credential dumping detected",
+ "description": "lsass.exe memory read by rundll32.exe",
+ }
+ ),
+ }
+ )
+ assert alert is not None
+ assert alert.title == "Credential dumping detected"
+ assert alert.description == "lsass.exe memory read by rundll32.exe"
+
+
+def test_a_title_is_not_repeated_as_the_description() -> None:
+ """With no description supplied, the slot is empty rather than a copy.
+
+ Restating the title adds nothing an analyst can act on and it reads as if
+ the vendor wrote two sentences. The console renders the raw event either
+ way.
+ """
+ alert = promote_normalized_event(
+ {
+ "tenant_id": _TENANT,
+ "ocsf_event": _ingest_event({"title": "Suspicious PowerShell"}),
+ }
+ )
+ assert alert is not None
+ assert alert.title == "Suspicious PowerShell"
+ assert alert.description is None or alert.description == ""
+
+
+def test_the_other_human_keys_are_reached_through_the_string_too() -> None:
+ for key in ("message", "summary", "detail", "reason"):
+ ocsf = _ingest_event({"title": "Alert", key: f"human sentence via {key}"})
+ assert _description(ocsf) == f"human sentence via {key}", key
+
+
+def test_a_malformed_raw_data_string_does_not_raise() -> None:
+ """A truncated payload degrades to the existing fallbacks, never an error."""
+ ocsf = _ingest_event({"title": "Alert"})
+ ocsf["raw_data"] = '{"description": "truncated'
+ assert _description(ocsf) == "Alert"
+ assert promote_normalized_event({"tenant_id": _TENANT, "ocsf_event": ocsf}) is not None
+
+
+def test_a_serialized_payload_is_never_the_description() -> None:
+ """The property that matters, now stated against the real shape."""
+ ocsf = _ingest_event({"command_line": "whoami", "host": "WIN-01", "pid": 4242})
+ result = _description(ocsf)
+ assert not result.startswith("{"), f"description is a serialized payload: {result[:60]}"
+ assert "4242" not in result
diff --git a/services/ingest/internal/normalizer/issue_1244_test.go b/services/ingest/internal/normalizer/issue_1244_test.go
new file mode 100644
index 000000000..c58585ae8
--- /dev/null
+++ b/services/ingest/internal/normalizer/issue_1244_test.go
@@ -0,0 +1,199 @@
+package normalizer
+
+import (
+ "testing"
+
+ "github.com/beenuar/aisoc/services/ingest/internal/attck"
+)
+
+// Issue #1244, reported against `POST /v1/ingest/batch` with
+// `connector_type: crowdstrike` and generic field names.
+//
+// Two distinct losses on the ingest side:
+//
+// - a top-level `description` reached no OCSF slot the fusion promoter
+// reads, so the alert's description resolved to its title;
+// - a vendor's `techniques: ["T1003.001"]` reached nothing either, because
+// `extractTechniqueIDs` did not list the plural bare key.
+
+func normalizeOrFail(t *testing.T, n *Normalizer, raw *RawEvent) map[string]interface{} {
+ t.Helper()
+ ev, err := n.Normalize(raw)
+ if err != nil {
+ t.Fatalf("Normalize returned error: %v", err)
+ }
+ return ev.OcsfEvent
+}
+
+func pushedEvent(payload map[string]interface{}) *RawEvent {
+ return &RawEvent{
+ ConnectorID: "conn-1",
+ ConnectorType: "crowdstrike",
+ TenantID: "11111111-1111-1111-1111-111111111111",
+ ReceivedAt: "2026-08-02T00:00:00Z",
+ Payload: payload,
+ }
+}
+
+func findingField(t *testing.T, ocsf map[string]interface{}, key string) interface{} {
+ t.Helper()
+ finding, ok := ocsf["finding"].(map[string]interface{})
+ if !ok {
+ return nil
+ }
+ return finding[key]
+}
+
+// ── A supplied description must reach a slot the promoter reads ───────────
+
+func TestSuppliedDescriptionReachesFindingDesc(t *testing.T) {
+ ocsf := normalizeOrFail(t, newLenientNormalizer(), pushedEvent(map[string]interface{}{
+ "title": "Credential dumping detected",
+ "description": "lsass.exe memory read by rundll32.exe",
+ "severity": "high",
+ "hostname": "WIN-FIN-01",
+ }))
+
+ if got := findingField(t, ocsf, "desc"); got != "lsass.exe memory read by rundll32.exe" {
+ t.Errorf("finding.desc = %v, want the supplied description", got)
+ }
+ // The title still owns `message`; the two must not collapse onto one slot.
+ if got := ocsf["message"]; got != "Credential dumping detected" {
+ t.Errorf("message = %v, want the supplied title", got)
+ }
+}
+
+// A profile that names its own source for finding.desc keeps it: the alias
+// only fills a destination the field map left empty. `email_inbox` resolves
+// finding.desc from `body`, so a payload carrying both must not lose it.
+func TestAProfilesOwnDescriptionFieldStillWins(t *testing.T) {
+ raw := pushedEvent(map[string]interface{}{
+ "subject": "Invoice overdue",
+ "body": "the profile's own field",
+ "description": "the generic key",
+ "severity": "medium",
+ })
+ raw.ConnectorType = "email_inbox"
+
+ ocsf := normalizeOrFail(t, newLenientNormalizer(), raw)
+ if got := findingField(t, ocsf, "desc"); got != "the profile's own field" {
+ t.Errorf("finding.desc = %v, want the profile's own mapping to win over the alias", got)
+ }
+}
+
+// Security Hub mapped `Description` onto `raw_data`, which Normalize
+// overwrites unconditionally a few lines later with the serialized payload —
+// so the vendor's description was mapped and then thrown away.
+func TestSecurityHubDescriptionSurvivesTheRawDataOverwrite(t *testing.T) {
+ raw := pushedEvent(map[string]interface{}{
+ "Title": "S3 bucket is publicly readable",
+ "Description": "The bucket policy grants s3:GetObject to Principal *",
+ "Severity": map[string]interface{}{"Label": "HIGH"},
+ })
+ raw.ConnectorType = "aws_security_hub"
+
+ ocsf := normalizeOrFail(t, newLenientNormalizer(), raw)
+
+ if got := findingField(t, ocsf, "desc"); got != "The bucket policy grants s3:GetObject to Principal *" {
+ t.Errorf("finding.desc = %v, want the Security Hub Description", got)
+ }
+ // raw_data is the serialized payload and nothing else.
+ rawData, ok := ocsf["raw_data"].(string)
+ if !ok {
+ t.Fatalf("raw_data = %T, want the serialized payload string", ocsf["raw_data"])
+ }
+ if rawData == "The bucket policy grants s3:GetObject to Principal *" {
+ t.Error("raw_data still holds the Description — the field map and the serializer are fighting over one slot")
+ }
+}
+
+// ── A bare `techniques` key must yield techniques ─────────────────────────
+
+func TestBareTechniquesKeyIsExtracted(t *testing.T) {
+ for _, key := range []string{"techniques", "technique"} {
+ t.Run(key, func(t *testing.T) {
+ got := extractTechniqueIDs(map[string]interface{}{
+ key: []interface{}{"T1003.001"},
+ })
+ if len(got) != 1 || got[0] != "T1003.001" {
+ t.Errorf("extractTechniqueIDs(%q) = %v, want [T1003.001]", key, got)
+ }
+ })
+ }
+}
+
+func TestTechniquesAcceptsAScalarAsWellAsAList(t *testing.T) {
+ got := extractTechniqueIDs(map[string]interface{}{"techniques": "T1059.001"})
+ if len(got) != 1 || got[0] != "T1059.001" {
+ t.Errorf("extractTechniqueIDs = %v, want [T1059.001]", got)
+ }
+}
+
+// A well-formed technique id must survive to `mitre_attck` even when the
+// ATT&CK corpus is unavailable. The corpus is fetched from raw.githubusercontent.com
+// at boot, so on an air-gapped or egress-filtered deployment `attck.Loaded()`
+// is false forever — and gating the whole block on it meant no event anywhere
+// ever carried a technique, no matter what the vendor sent.
+func TestAWellFormedTechniqueIsCarriedWithoutTheCorpus(t *testing.T) {
+ if attck.Loaded() {
+ t.Skip("ATT&CK corpus is loaded in this process; this test covers the unloaded deployment")
+ }
+
+ ocsf := normalizeOrFail(t, newLenientNormalizer(), pushedEvent(map[string]interface{}{
+ "title": "Credential dumping detected",
+ "severity": "high",
+ "techniques": []interface{}{"T1003.001"},
+ }))
+
+ block, ok := ocsf["mitre_attck"].([]map[string]interface{})
+ if !ok || len(block) != 1 {
+ t.Fatalf("mitre_attck = %#v, want one unverified entry", ocsf["mitre_attck"])
+ }
+ if got := block[0]["technique_id"]; got != "T1003.001" {
+ t.Errorf("technique_id = %v, want T1003.001", got)
+ }
+ // The reader has to be able to tell an id we looked up from one we only
+ // passed through, or an unverified id reads as corpus-confirmed.
+ if verified, ok := block[0]["verified"].(bool); !ok || verified {
+ t.Errorf("verified = %v, want false on an entry the corpus never confirmed", block[0]["verified"])
+ }
+}
+
+// Garbage must not be carried: the pass-through is for well-formed ids only.
+//
+// `TA0006` is the one that matters here. It is a *tactic* id, and the old
+// shape check (leading `T`, 5–7 characters before the dot) accepted it. That
+// was invisible while every id had to survive `attck.Lookup`, which has no
+// entry for a tactic — but an unverified pass-through would have written a
+// tactic into `mitre_techniques`, so the shape check has to be exact for the
+// carry to be safe.
+func TestAMalformedTechniqueIsNotCarried(t *testing.T) {
+ for _, bad := range []string{"not-a-technique", "T12", "TA0006", "T10591", "T1059.1", ""} {
+ if got := extractTechniqueIDs(map[string]interface{}{"techniques": bad}); len(got) != 0 {
+ t.Errorf("extractTechniqueIDs(%q) = %v, want none", bad, got)
+ }
+ }
+}
+
+// Stated against the shape check itself, because the table above goes through
+// a key the extractor did not read before this change and so would have
+// passed vacuously.
+func TestNormalizeTechniqueIDRejectsATacticID(t *testing.T) {
+ if got := normalizeTechniqueID("TA0006"); got != "" {
+ t.Errorf("normalizeTechniqueID(\"TA0006\") = %q, want \"\" — TA0006 is a tactic, not a technique", got)
+ }
+ if got := normalizeTechniqueID("T10591"); got != "" {
+ t.Errorf("normalizeTechniqueID(\"T10591\") = %q, want \"\"", got)
+ }
+ if got := normalizeTechniqueID("T1059.001"); got != "T1059.001" {
+ t.Errorf("normalizeTechniqueID(\"T1059.001\") = %q, want it accepted unchanged", got)
+ }
+}
+
+func TestWellFormedTechniqueShapesAreAccepted(t *testing.T) {
+ for _, good := range []string{"T1059", "T1059.001", "t1003.001"} {
+ if got := extractTechniqueIDs(map[string]interface{}{"techniques": good}); len(got) != 1 {
+ t.Errorf("extractTechniqueIDs(%q) = %v, want one id", good, got)
+ }
+ }
+}
diff --git a/services/ingest/internal/normalizer/normalizer.go b/services/ingest/internal/normalizer/normalizer.go
index f23b57d7f..003202ef5 100644
--- a/services/ingest/internal/normalizer/normalizer.go
+++ b/services/ingest/internal/normalizer/normalizer.go
@@ -5,6 +5,7 @@ import (
"context"
"encoding/json"
"fmt"
+ "regexp"
"strings"
"time"
@@ -210,9 +211,15 @@ var connectorProfiles = map[string]connectorProfile{
classUID: 2001,
className: "Security Finding",
fieldMap: map[string]string{
- "UpdatedAt": "time",
- "Title": "message",
- "Description": "raw_data",
+ "UpdatedAt": "time",
+ "Title": "message",
+ // Not `raw_data`: Normalize overwrites that slot unconditionally
+ // with the serialized payload a few dozen lines after the field
+ // map runs, so the Description was mapped and then discarded
+ // every time (issue #1244). `finding.desc` is where every other
+ // profile puts a vendor description and is what the fusion
+ // promoter reads.
+ "Description": "finding.desc",
"Severity.Label": "severity",
},
severityMap: map[string]int{
@@ -536,6 +543,15 @@ var _canonicalAliases = []struct {
// to the vendor profiles without giving them a second mechanism.
{"message", []string{"title"}},
{"finding.uid", []string{"external_id"}},
+ // Issue #1244. Nothing mapped a top-level `description` onto any OCSF
+ // slot outside the handful of profiles that name it, so a pushed event
+ // carrying both a title and a description arrived with only the title —
+ // and the fusion promoter, finding no description anywhere, fell back to
+ // `message`, which is the title. The alert then showed one sentence
+ // twice. `finding.desc` is the destination the profiles that *do* map it
+ // already use (aisoc_sample, ai_guardrail, email_inbox, cloudtrail), and
+ // it is the first slot promoter._description reads after the payload.
+ {"finding.desc", []string{"description"}},
}
// connectorTypeAliases maps a connector identifier the connectors service
@@ -892,24 +908,40 @@ func (n *Normalizer) Normalize(raw *RawEvent) (*NormalizedEvent, error) {
ocsf["raw_data"] = string(rawBytes)
}
- // ATT&CK technique enrichment
- if attck.Loaded() {
- if techIDs := extractTechniqueIDs(raw.Payload); len(techIDs) > 0 {
- var enriched []map[string]interface{}
- for _, tid := range techIDs {
- if tech := attck.Lookup(tid); tech != nil {
- enriched = append(enriched, map[string]interface{}{
- "technique_id": tech.ID,
- "technique_name": tech.Name,
- "tactic_ids": tech.TacticIDs,
- "tactic_names": tech.TacticNames,
- "url": tech.URL,
- })
- }
- }
- if len(enriched) > 0 {
- ocsf["mitre_attck"] = enriched
+ // ATT&CK technique enrichment.
+ //
+ // The whole block used to sit behind `attck.Loaded()` and each id behind
+ // `attck.Lookup(tid) != nil`, so a deployment that cannot reach
+ // raw.githubusercontent.com — air-gapped, egress-filtered, or simply
+ // offline at boot — published `mitre_techniques: []` on every event
+ // forever, whatever the vendor sent (issue #1244).
+ //
+ // A well-formed id is now carried either way. What the corpus adds is the
+ // technique name, its tactics and its URL; what the vendor asserted is
+ // the id, and dropping the vendor's assertion because we could not look
+ // it up loses information we were handed. `verified` says which is which
+ // so a reader is never told an unconfirmed id was confirmed.
+ if techIDs := extractTechniqueIDs(raw.Payload); len(techIDs) > 0 {
+ var enriched []map[string]interface{}
+ for _, tid := range techIDs {
+ if tech := attck.Lookup(tid); tech != nil {
+ enriched = append(enriched, map[string]interface{}{
+ "technique_id": tech.ID,
+ "technique_name": tech.Name,
+ "tactic_ids": tech.TacticIDs,
+ "tactic_names": tech.TacticNames,
+ "url": tech.URL,
+ "verified": true,
+ })
+ continue
}
+ enriched = append(enriched, map[string]interface{}{
+ "technique_id": tid,
+ "verified": false,
+ })
+ }
+ if len(enriched) > 0 {
+ ocsf["mitre_attck"] = enriched
}
}
@@ -959,9 +991,17 @@ func extractTechniqueIDs(payload map[string]interface{}) []string {
seen := map[string]struct{}{}
var results []string
+ // `techniques` and `technique` are the bare forms a pushed payload uses
+ // (issue #1244: a CrowdStrike-shaped event sending
+ // `techniques: ["T1003.001"]` produced `mitre_techniques: []`). The list
+ // was all prefixed or suffixed spellings and had no entry for the plain
+ // noun, singular or plural.
+ //
+ // `tactic_id` is kept for the payload that files a technique under it,
+ // but a genuine tactic id no longer survives normalizeTechniqueID.
candidateKeys := []string{
"technique_id", "mitre_technique", "attck_technique", "tactic_id",
- "mitre_techniques", "attack_technique",
+ "mitre_techniques", "attack_technique", "techniques", "technique",
}
for _, key := range candidateKeys {
val, ok := payload[key]
@@ -992,15 +1032,22 @@ func extractTechniqueIDs(payload map[string]interface{}) []string {
return results
}
+// techniqueIDRe is the exact ATT&CK technique shape: T followed by four
+// digits, optionally a three-digit sub-technique.
+//
+// The check used to be a length range ("leading T, 5–7 characters before the
+// dot"), which also accepted `TA0006` — a *tactic* id — and `T10591`. That
+// was harmless only because every id then had to survive attck.Lookup, which
+// has no entry for either, so the loose ones were filtered downstream. Since
+// issue #1244 a well-formed id is carried even when the corpus cannot confirm
+// it, so the shape check is now the only filter and has to be exact.
+var techniqueIDRe = regexp.MustCompile(`^T\d{4}(?:\.\d{3})?$`)
+
// normalizeTechniqueID extracts a clean ATT&CK technique ID from a string.
func normalizeTechniqueID(s string) string {
s = strings.TrimSpace(strings.ToUpper(s))
- // Accept T1234 or T1234.001
- if len(s) >= 5 && s[0] == 'T' {
- parts := strings.SplitN(s, ".", 2)
- if len(parts[0]) >= 5 && len(parts[0]) <= 7 {
- return s
- }
+ if techniqueIDRe.MatchString(s) {
+ return s
}
return ""
}