Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
feat(api): let analysts hunt AiSOC's own event lake (#668)
Beenu Aroraa4bd8443d7d7
diff --git a/docs/openapi.yaml b/docs/openapi.yamlindex 760a9d7e2..5d57ffde4 100644--- a/docs/openapi.yaml+++ b/docs/openapi.yaml@@ -21599,6 +21599,19 @@ components:- type: string- type: 'null'title: Execution Error+ executed_against:+ anyOf:+ - type: string+ enum:+ - elasticsearch+ - lake+ - type: 'null'+ title: Executed Against+ unsupported_filters:+ items:+ type: string+ type: array+ title: Unsupported Filterstype: objectrequired:- request_id@@ -24113,6 +24126,24 @@ components:last_run_at:type: stringtitle: Last Run At+ executed:+ type: boolean+ title: Executed+ default: false+ result:+ anyOf:+ - $ref: '#/components/schemas/SavedHuntRunResult'+ - type: 'null'+ execution_error:+ anyOf:+ - type: string+ - type: 'null'+ title: Execution Error+ unsupported_filters:+ items:+ type: string+ type: array+ title: Unsupported Filterstype: objectrequired:- id@@ -24124,15 +24155,22 @@ components:description: 'Synchronous run result returned by ``POST /saved-hunts/{id}/run``.- The endpoint *does not* execute the underlying ES|QL — that path is the+ This endpoint used to re-translate the question, stamp ``last_run_at`` and++ return — executing nothing. The UI then displayed "last run just now" for++ a hunt that had queried no data at all, which is indistinguishable from a++ hunt that ran and found nothing.- job of :mod:`app.api.v1.endpoints.nl_query` (``/nl-query/execute``) and- requires a configured Elasticsearch URL. The ``/run`` endpoint here only+ It now runs the compiled query against AiSOC''s own event lake, which is- re-translates and stamps ``last_run_at`` so the UI can show "last run+ where every connector''s events are archived. ``executed`` says plainly- just now" and so the scheduler treats a manual run as resetting cadence.'+ whether rows were fetched, so a lake that is unreachable or unconfigured++ reports that instead of looking like a clean run with zero hits.'RunSummary:properties:id:@@ -24720,6 +24758,31 @@ components:- updated_attitle: SavedHuntModeldescription: Wire format for a saved hunt row.+ SavedHuntRunResult:+ properties:+ columns:+ items:+ type: string+ type: array+ title: Columns+ rows:+ items:+ items: {}+ type: array+ type: array+ title: Rows+ total_rows:+ type: integer+ title: Total Rows+ default: 0+ took_ms:+ anyOf:+ - type: integer+ - type: 'null'+ title: Took Ms+ type: object+ title: SavedHuntRunResult+ description: Rows the hunt actually returned.SavedViewModel:properties:id:diff --git a/services/api/app/api/v1/endpoints/nl_query.py b/services/api/app/api/v1/endpoints/nl_query.pyindex 2e0fe1402..8a101c8da 100644--- a/services/api/app/api/v1/endpoints/nl_query.py+++ b/services/api/app/api/v1/endpoints/nl_query.py@@ -26,21 +26,28 @@ import sysimport uuidfrom datetime import UTC, datetimefrom pathlib import Path-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKING, Any, Literalimport httpx+import structlogfrom fastapi import APIRouter, statusfrom pydantic import BaseModel, Fieldfrom app.api.v1.deps import AuthUserfrom app.core.airgap import AirgapViolation, enforce_airgap_for_urlfrom app.core.config import settings+from app.db.clickhouse import (+ LakeQueryError,+ LakeQueryTimeoutError,+ execute_lake_query,+)from app.services.esql_runner import (ESQLExecutionError,ESQLNotConfigured,resolve_es_credentials,run_esql_query,)+from app.services.lake_hunt import HuntCompileError, compile_huntif TYPE_CHECKING:# Static-only re-export so type checkers can see the dataclass fields and@@ -146,6 +153,8 @@ if not TYPE_CHECKING:enhance_with_llm = _nl_query.enhance_with_llmdeterministic_translate = _nl_query.translate+logger = structlog.get_logger(__name__)+router = APIRouter(prefix="/nl-query", tags=["nl_query"])@@ -208,6 +217,13 @@ class QueryResult(BaseModel):class NLQueryExecuteResponse(NLQueryTranslateResponse):result: QueryResult | None = Noneexecution_error: str | None = None+ #: Which backend produced `result`. An analyst must be able to tell+ #: "nothing matched in your SIEM" from "nothing matched in AiSOC's lake".+ executed_against: Literal["elasticsearch", "lake"] | None = None+ #: Translator fields the lake does not store, when executing against the+ #: lake. A hunt that quietly drops half its filters returns more rows than+ #: was asked for, which reads as "nothing was filtered out".+ unsupported_filters: list[str] = Field(default_factory=list)# ────────────────────────────────────────────────────────────────────────────@@ -293,6 +309,55 @@ async def _execute_esql(esql: str, es_url: str, es_api_key: str, max_rows: int)# ────────────────────────────────────────────────────────────────────────────+async def _execute_against_lake(+ base: NLQueryExecuteResponse,+ translated: TranslatedQuery,+ body: NLQueryExecuteRequest,+ user: AuthUser,+) -> NLQueryExecuteResponse:+ """Run the hunt against AiSOC's own event lake.++ The translator's structured IR compiles straight to parameterised+ ClickHouse SQL, so no part of the analyst's question reaches the SQL+ string. The tenant predicate is generated as part of the WHERE clause+ rather than rewritten in afterwards.+ """+ base.executed_against = "lake"+ try:+ compiled = compile_hunt(+ translated.intents,+ tenant_id=str(user.tenant_id),+ hours=body.time_range_hours,+ limit=body.max_rows,+ )+ except HuntCompileError as exc:+ base.execution_error = f"Could not compile this question into a lake query: {exc}"+ return base++ base.unsupported_filters = compiled.unsupported_fields++ try:+ result = await execute_lake_query(compiled.sql, params=compiled.params)+ except LakeQueryTimeoutError as exc:+ base.execution_error = f"Lake query timed out: {exc}"+ return base+ except LakeQueryError as exc:+ base.execution_error = f"Lake query failed: {exc}"+ return base+ except Exception as exc: # noqa: BLE001 — surface, never fabricate rows+ logger.warning("nl_query.lake_execution_failed", error=str(exc))+ base.execution_error = f"Lake query failed: {exc}. Configure ES_URL to hunt an external SIEM instead."+ return base++ base.result = QueryResult(+ columns=list(result.columns),+ rows=[list(row) for row in result.rows],+ total_rows=result.row_count,+ took_ms=result.elapsed_ms,+ )+ return base++@router.post("/translate",response_model=NLQueryTranslateResponse,@@ -347,9 +412,13 @@ async def execute_query(try:es_url, es_api_key = resolve_es_credentials()except ESQLNotConfigured:- base.execution_error = "ES_URL or ES_API_KEY not configured. Set them in environment variables."- return base+ # No external SIEM configured. Every connector's events are archived+ # to AiSOC's own ClickHouse lake, so hunt that instead of refusing.+ # This endpoint used to return "ES_URL not configured" and stop, which+ # left a tenant unable to query any of the data they had ingested.+ return await _execute_against_lake(base, translated, body, user)+ base.executed_against = "elasticsearch"try:base.result = await _execute_esql(translated.esql,diff --git a/services/api/app/api/v1/endpoints/saved_hunts.py b/services/api/app/api/v1/endpoints/saved_hunts.pyindex 40110a39b..857915e36 100644--- a/services/api/app/api/v1/endpoints/saved_hunts.py+++ b/services/api/app/api/v1/endpoints/saved_hunts.py@@ -58,8 +58,10 @@ from app.api.v1.deps import AuthUserfrom app.api.v1.endpoints.nl_query import ( # noqa: E402deterministic_translate,)+from app.db.clickhouse import execute_lake_queryfrom app.db.rls import TenantDBSessionfrom app.models.saved_hunt import SavedHunt+from app.services.lake_hunt import HuntCompileError, compile_huntlogger = structlog.get_logger()@@ -192,14 +194,27 @@ class CreateSavedHuntRequest(BaseModel):schedule: str | None = Field(default=None, max_length=_MAX_SCHEDULE_LEN)+class SavedHuntRunResult(BaseModel):+ """Rows the hunt actually returned."""++ columns: list[str] = Field(default_factory=list)+ rows: list[list[Any]] = Field(default_factory=list)+ total_rows: int = 0+ took_ms: int | None = None++class RunSavedHuntResponse(BaseModel):"""Synchronous run result returned by ``POST /saved-hunts/{id}/run``.- The endpoint *does not* execute the underlying ES|QL — that path is the- job of :mod:`app.api.v1.endpoints.nl_query` (``/nl-query/execute``) and- requires a configured Elasticsearch URL. The ``/run`` endpoint here only- re-translates and stamps ``last_run_at`` so the UI can show "last run- just now" and so the scheduler treats a manual run as resetting cadence.+ This endpoint used to re-translate the question, stamp ``last_run_at`` and+ return — executing nothing. The UI then displayed "last run just now" for+ a hunt that had queried no data at all, which is indistinguishable from a+ hunt that ran and found nothing.++ It now runs the compiled query against AiSOC's own event lake, which is+ where every connector's events are archived. ``executed`` says plainly+ whether rows were fetched, so a lake that is unreachable or unconfigured+ reports that instead of looking like a clean run with zero hits."""id: str@@ -207,6 +222,14 @@ class RunSavedHuntResponse(BaseModel):nl_query: strtranslated_query: TranslatedQueryEnvelopelast_run_at: str+ #: True only when a query really ran against the lake.+ executed: bool = False+ result: SavedHuntRunResult | None = None+ #: Why no rows were fetched, when that is the case.+ execution_error: str | None = None+ #: Translator fields the lake does not store. A hunt that silently drops+ #: filters returns more rows than the analyst asked for.+ unsupported_filters: list[str] = Field(default_factory=list)# ---------------------------------------------------------------------------@@ -235,6 +258,43 @@ def _translate(nl_query: str) -> TranslatedQueryEnvelope:return TranslatedQueryEnvelope.from_translator(tq)+async def _execute_on_lake(+ nl_query: str,+ tenant_id: str,+) -> tuple[bool, SavedHuntRunResult | None, str | None, list[str]]:+ """Run the hunt against the tenant's slice of the event lake.++ Returns ``(executed, result, error, unsupported_fields)``. Never raises:+ a hunt run must not 500 because the lake is down, but it also must not+ report a successful run that did not happen.+ """+ try:+ compiled = compile_hunt(deterministic_translate(nl_query).intents, tenant_id=tenant_id)+ except HuntCompileError as exc:+ return False, None, f"Could not compile this hunt into a lake query: {exc}", []+ except Exception as exc: # noqa: BLE001 — report, never fabricate a run+ logger.warning("saved_hunt.compile_failed", error=str(exc))+ return False, None, f"Could not compile this hunt into a lake query: {exc}", []++ try:+ lake_result = await execute_lake_query(compiled.sql, params=compiled.params)+ except Exception as exc: # noqa: BLE001 — surface the real reason+ logger.warning("saved_hunt.lake_execution_failed", error=str(exc))+ return False, None, f"Lake query failed: {exc}", compiled.unsupported_fields++ return (+ True,+ SavedHuntRunResult(+ columns=list(lake_result.columns),+ rows=[list(r) for r in lake_result.rows],+ total_rows=lake_result.row_count,+ took_ms=lake_result.elapsed_ms,+ ),+ None,+ compiled.unsupported_fields,+ )++async def _load_owned_hunt(db: Any, # noqa: ANN401hunt_id: uuid.UUID,@@ -407,6 +467,7 @@ async def run_saved_hunt(translated = _translate(row.nl_query)now = datetime.now(UTC)+ executed, result, execution_error, unsupported = await _execute_on_lake(row.nl_query, str(user.tenant_id))# Persist the refreshed translation alongside the run timestamp so the# next list call surfaces the latest envelope. The schedule worker uses@@ -430,4 +491,8 @@ async def run_saved_hunt(nl_query=row.nl_query,translated_query=translated,last_run_at=now.isoformat(),+ executed=executed,+ result=result,+ execution_error=execution_error,+ unsupported_filters=unsupported,)diff --git a/services/api/app/db/clickhouse.py b/services/api/app/db/clickhouse.pyindex 95a2a028a..f3a944c05 100644--- a/services/api/app/db/clickhouse.py+++ b/services/api/app/db/clickhouse.py@@ -227,6 +227,7 @@ async def execute_lake_query(*,timeout_seconds: float | None = None,extra_settings: dict[str, Any] | None = None,+ params: dict[str, Any] | None = None,) -> LakeQueryResult:"""Run a rewritten lake SQL statement and return columns + rows.@@ -261,6 +262,13 @@ async def execute_lake_query(:data:`DEFAULT_QUERY_SETTINGS`. Used by the lake endpoints tolayer in per-tenant overrides (e.g. tighter row caps forunprivileged roles).+ params:+ Values for ``%(name)s`` placeholders, bound by the driver. Callers+ that build SQL from untrusted input — an analyst's hunt question, for+ instance — bind their values here so no user text is ever+ concatenated into the statement. This includes the tenant predicate:+ binding it as a parameter is stronger than injecting a literal and+ hoping the rewrite survived.Returns-------@@ -301,6 +309,7 @@ async def execute_lake_query(# which we need so the API can echo a stable column ordering.return client.execute(sql,+ params or {},with_column_types=True,settings=merged_settings,)diff --git a/services/api/app/services/lake_hunt.py b/services/api/app/services/lake_hunt.pynew file mode 100644index 000000000..98f15e110--- /dev/null+++ b/services/api/app/services/lake_hunt.py@@ -0,0 +1,290 @@+"""Compile a translated hunt into SQL against AiSOC's own event lake.++Every connector's events are archived to ClickHouse `aisoc.raw_events`, and+until now nothing could hunt them. `/nl-query/execute` only ever executed+against Elasticsearch, so a tenant whose data lives in the platform's own lake+got "ES_URL or ES_API_KEY not configured" and had no way to query anything they+had ingested. `POST /saved-hunts/{id}/run` was worse: it re-translated the+question, stamped `last_run_at`, and returned — so the UI displayed "last run+just now" having queried nothing at all.++This module closes that by compiling the translator's structured IR+(`QueryIntents`) into ClickHouse SQL. Deterministic, no LLM: the IR already+carries typed filters with a closed operator set, group-bys, aggregations, a+sort and a limit.++Two properties this must not get wrong:++**No user text ever reaches the SQL string.** Field names resolve through a+fixed alias map to real column names, operators come from a closed set, and+every value is bound as a query parameter. A question is untrusted input, and+the lake holds every tenant's events.++**Tenant scoping is structural, not rewritten in afterwards.** The tenant+predicate is part of the generated WHERE clause and its parameter is bound by+this module, rather than relying on `rewrite_for_tenant` — which exists to+constrain untrusted *operator* SQL, and which cannot help if a caller forgets+to use it.+"""++from __future__ import annotations++import json+from dataclasses import dataclass+from typing import Any++import structlog++logger = structlog.get_logger(__name__)++LAKE_TABLE = "aisoc.raw_events"++#: ECS-ish field names the translator emits, mapped to lake columns.+#:+#: The translator speaks ECS because it also targets Elasticsearch. Anything+#: absent from this map is a field the lake genuinely does not store, and is+#: reported to the caller rather than silently dropped — a hunt that quietly+#: ignores half its filters returns more rows than the analyst asked for, which+#: reads as "no results were filtered out" when the truth is "your filter was+#: discarded".+_FIELD_TO_COLUMN: dict[str, str] = {+ "@timestamp": "event_time",+ "event.created": "event_time",+ "user.name": "user_name",+ "source.ip": "source_ip",+ "destination.ip": "dest_ip",+ "source.port": "src_port",+ "destination.port": "dst_port",+ "host.name": "src_hostname",+ "destination.domain": "dst_hostname",+ "process.name": "process_name",+ "file.path": "file_path",+ "file.hash.sha256": "hash_sha256",+ "event.severity": "severity_id",+ "event.category": "connector_type",+ "event.provider": "connector_type",+ "network.protocol": "protocol",+ "rule.name": "connector_type",+ "threat.technique.id": "mitre_techniques",+ "process.command_line": "raw_payload",+ "url.full": "raw_payload",+ "user_agent.original": "raw_payload",+}++#: Columns holding an Array(String); membership tests differ from scalars.+_ARRAY_COLUMNS = frozenset({"mitre_techniques", "mitre_tactics", "iocs"})++#: Operators the IR can emit, mapped to ClickHouse syntax. A closed set: an+#: operator outside it is rejected rather than passed through.+_SCALAR_OPS = {+ "==": "=",+ "!=": "!=",+ ">": ">",+ "<": "<",+ ">=": ">=",+ "<=": "<=",+}++MAX_LIMIT = 1000+++@dataclass(frozen=True)+class CompiledHunt:+ """A parameterised lake query plus an honest account of what was dropped."""++ sql: str+ params: dict[str, Any]+ #: Translator fields with no lake column. Surfaced to the caller so a+ #: partial hunt is never presented as a complete one.+ unsupported_fields: list[str]++ @property+ def is_partial(self) -> bool:+ return bool(self.unsupported_fields)+++class HuntCompileError(ValueError):+ """The IR could not be compiled into a safe lake query."""+++def _column_for(field: str) -> str | None:+ if field in _FIELD_TO_COLUMN:+ return _FIELD_TO_COLUMN[field]+ # A bare column name is accepted only if it really is a lake column.+ if field in set(_FIELD_TO_COLUMN.values()) | _ARRAY_COLUMNS:+ return field+ return None+++def _predicate(column: str, op: str, value: str, key: str) -> tuple[str, Any]:+ """Build one parameterised predicate. Never interpolates the value."""+ placeholder = f"%({key})s"++ if column in _ARRAY_COLUMNS:+ if op in {"==", "IN"}:+ values = _as_list(value)+ return f"hasAny({column}, %({key})s)", values+ raise HuntCompileError(f"operator {op!r} is not supported on array column {column!r}")++ if op == "IN":+ return f"{column} IN %({key})s", _as_list(value)+ if op == "LIKE":+ # The translator emits LIKE for "contains" phrasing. Bind the whole+ # pattern, wildcards included, so the value stays data.+ pattern = value if "%" in value else f"%{value}%"+ return f"{column} ILIKE %({key})s", pattern+ if op in _SCALAR_OPS:+ return f"{column} {_SCALAR_OPS[op]} {placeholder}", _coerce(column, value)++ raise HuntCompileError(f"unsupported operator {op!r}")+++def _as_list(value: str) -> list[str]:+ """The IR JSON-encodes IN lists so its dedup set stays hashable."""+ try:+ parsed = json.loads(value)+ except (TypeError, ValueError):+ return [value]+ return [str(v) for v in parsed] if isinstance(parsed, list) else [str(parsed)]+++def _coerce(column: str, value: str) -> Any:+ """Numeric columns need numbers, not strings, or ClickHouse rejects them."""+ if column in {"severity_id", "src_port", "dst_port"}:+ try:+ return int(value)+ except (TypeError, ValueError) as exc:+ raise HuntCompileError(f"{column} needs a number, got {value!r}") from exc+ return value+++def compile_hunt(+ intents: Any,+ *,+ tenant_id: str,+ hours: int = 24,+ limit: int | None = None,+) -> CompiledHunt:+ """Compile translator intents into a tenant-scoped lake query.++ `intents` is a `QueryIntents` (or anything with the same attributes), so+ this works against both the API's vendored translator and the agents' copy+ without importing either.+ """+ params: dict[str, Any] = {"tenant_id": tenant_id, "hours": max(1, int(hours))}+ where = [+ # Structural, not rewritten in afterwards.+ "tenant_id = %(tenant_id)s",+ "event_time >= now() - INTERVAL %(hours)s HOUR",+ ]+ unsupported: list[str] = []++ for index, (field, op, value) in enumerate(getattr(intents, "filters", []) or []):+ column = _column_for(str(field))+ if column is None:+ unsupported.append(str(field))+ continue+ key = f"f{index}"+ clause, bound = _predicate(column, str(op), str(value), key)+ where.append(clause)+ params[key] = bound++ group_by: list[str] = []+ for field in getattr(intents, "group_by", []) or []:+ column = _column_for(str(field))+ if column is None:+ unsupported.append(str(field))+ continue+ group_by.append(column)++ select = _select_clause(intents, group_by, unsupported)+ sql = f"SELECT {', '.join(select)} FROM {LAKE_TABLE} WHERE {' AND '.join(where)}"++ if group_by:+ sql += f" GROUP BY {', '.join(group_by)}"++ sql += f" ORDER BY {_order_clause(intents, group_by)}"++ effective_limit = min(int(limit or getattr(intents, "limit", 500) or 500), MAX_LIMIT)+ sql += f" LIMIT {max(1, effective_limit)}"++ return CompiledHunt(sql=sql, params=params, unsupported_fields=sorted(set(unsupported)))+++def _select_clause(intents: Any, group_by: list[str], unsupported: list[str]) -> list[str]:+ """Projection: aggregations when asked for, otherwise the event columns."""+ aggregations = getattr(intents, "aggregations", []) or []+ distinct = getattr(intents, "distinct", None)++ if distinct:+ column = _column_for(str(distinct))+ if column is not None:+ return [f"DISTINCT {column}"]+ unsupported.append(str(distinct))++ if aggregations:+ select = list(group_by)+ for function, arg, alias in aggregations:+ select.append(_aggregation(str(function), arg, str(alias), unsupported))+ return select++ if group_by:+ return [*group_by, "count() AS event_count"]++ return [+ "event_time",+ "severity",+ "connector_type",+ "src_hostname",+ "user_name",+ "process_name",+ "toString(source_ip) AS source_ip",+ "toString(dest_ip) AS dest_ip",+ "mitre_techniques",+ ]+++def _aggregation(function: str, arg: Any, alias: str, unsupported: list[str]) -> str:+ """Render one aggregation from the closed set the translator can emit."""+ name = function.lower()+ safe_alias = "".join(c for c in alias if c.isalnum() or c == "_") or "value"++ if name in {"count", "count_distinct"} and not arg:+ return f"count() AS {safe_alias}"++ column = _column_for(str(arg)) if arg else None+ if column is None:+ if arg:+ unsupported.append(str(arg))+ return f"count() AS {safe_alias}"++ renderers = {+ "count": f"count({column})",+ "count_distinct": f"uniqExact({column})",+ "sum": f"sum({column})",+ "avg": f"avg({column})",+ "min": f"min({column})",+ "max": f"max({column})",+ }+ return f"{renderers.get(name, f'count({column})')} AS {safe_alias}"+++def _order_clause(intents: Any, group_by: list[str]) -> str:+ """Always returns a clause that is valid for the query being built.++ The default must respect `group_by`: ordering an aggregate query by+ `event_time` is a ClickHouse error, because the column is not in the+ grouping set. Same for a sort field that does not map to a lake column —+ falling back to a non-grouped column there would turn an unmappable sort+ into a failed query rather than a sensible default.+ """+ default = "count() DESC" if group_by else "event_time DESC"+ sort_by = getattr(intents, "sort_by", None)+ if not sort_by:+ return default++ field, direction = sort_by+ column = _column_for(str(field))+ if column is None or (group_by and column not in group_by):+ return default+ return f"{column} {'DESC' if str(direction).lower().startswith('desc') else 'ASC'}"diff --git a/services/api/tests/test_lake_hunt.py b/services/api/tests/test_lake_hunt.pynew file mode 100644index 000000000..8b8ceacda--- /dev/null+++ b/services/api/tests/test_lake_hunt.py@@ -0,0 +1,264 @@+"""Hunting AiSOC's own event lake.++Every connector's events are archived to ClickHouse `aisoc.raw_events`, and+nothing could query them. `/nl-query/execute` only ever executed against+Elasticsearch, so a tenant whose data lives in the platform's own lake got+"ES_URL or ES_API_KEY not configured" and could not query anything they had+ingested. `POST /saved-hunts/{id}/run` was worse: it re-translated the+question, stamped `last_run_at` and returned, so the console showed "last run+just now" for a hunt that had queried nothing — indistinguishable from a hunt+that ran and found no matches.++The compiler turns the translator's structured IR into ClickHouse SQL. The two+properties it must not get wrong are that no analyst text ever reaches the SQL+string, and that tenant scoping is generated rather than rewritten in+afterwards.+"""++from __future__ import annotations++import json+from dataclasses import dataclass, field++import pytest+from app.services.lake_hunt import (+ LAKE_TABLE,+ MAX_LIMIT,+ HuntCompileError,+ compile_hunt,+)++TENANT = "aaaaaaaa-0000-0000-0000-000000000001"+++@dataclass+class _Intents:+ """Stands in for the translator's QueryIntents."""++ filters: list[tuple[str, str, str]] = field(default_factory=list)+ group_by: list[str] = field(default_factory=list)+ aggregations: list[tuple[str, str | None, str]] = field(default_factory=list)+ sort_by: tuple[str, str] | None = None+ limit: int = 500+ distinct: str | None = None+ time_field: str = "@timestamp"+++# ── tenant scoping ────────────────────────────────────────────────────────+++def test_the_tenant_predicate_is_generated_and_bound():+ """Not rewritten in afterwards, and bound rather than interpolated.++ `rewrite_for_tenant` exists to constrain untrusted operator SQL and cannot+ help a caller who forgets to use it, so scoping is structural here.+ """+ compiled = compile_hunt(_Intents(), tenant_id=TENANT)+ assert "tenant_id = %(tenant_id)s" in compiled.sql+ assert compiled.params["tenant_id"] == TENANT+ assert TENANT not in compiled.sql+++def test_the_query_targets_the_lake_table():+ assert LAKE_TABLE in compile_hunt(_Intents(), tenant_id=TENANT).sql+++def test_a_time_window_is_always_applied():+ """An unbounded scan over the lake is a denial of service on itself."""+ compiled = compile_hunt(_Intents(), tenant_id=TENANT, hours=6)+ assert "event_time >= now() - INTERVAL %(hours)s HOUR" in compiled.sql+ assert compiled.params["hours"] == 6+++# ── no user text in the SQL ───────────────────────────────────────────────+++def test_filter_values_are_bound_never_interpolated():+ compiled = compile_hunt(+ _Intents(filters=[("user.name", "==", "alice")]),+ tenant_id=TENANT,+ )+ assert "alice" not in compiled.sql+ assert "alice" in compiled.params.values()+++def test_a_sql_injection_attempt_in_the_question_stays_data():+ """The question is untrusted input and the lake holds every tenant's events."""+ payload = "alice'; DROP TABLE aisoc.raw_events; --"+ compiled = compile_hunt(+ _Intents(filters=[("user.name", "==", payload)]),+ tenant_id=TENANT,+ )+ assert "DROP TABLE" not in compiled.sql+ assert payload in compiled.params.values()+++def test_an_unknown_field_cannot_become_a_column_name():+ """Otherwise a crafted field name is an injection point in the projection."""+ compiled = compile_hunt(+ _Intents(filters=[("evil) OR 1=1 --", "==", "x")]),+ tenant_id=TENANT,+ )+ assert "OR 1=1" not in compiled.sql+ assert "evil) OR 1=1 --" in compiled.unsupported_fields+++def test_an_unknown_operator_is_rejected():+ with pytest.raises(HuntCompileError):+ compile_hunt(_Intents(filters=[("user.name", "REGEXP", ".*")]), tenant_id=TENANT)+++def test_an_aggregation_alias_cannot_inject():+ compiled = compile_hunt(+ _Intents(aggregations=[("count", None, "x AS y, (SELECT 1)")]),+ tenant_id=TENANT,+ )+ assert "SELECT 1" not in compiled.sql+++# ── honesty about what was dropped ────────────────────────────────────────+++def test_fields_the_lake_does_not_store_are_reported():+ """A hunt that silently drops filters returns more rows than was asked for.++ Which reads to an analyst as "nothing was filtered out" when the truth is+ "your filter was discarded".+ """+ compiled = compile_hunt(+ _Intents(filters=[("source.geo.country_iso_code", "==", "IR")]),+ tenant_id=TENANT,+ )+ assert compiled.unsupported_fields == ["source.geo.country_iso_code"]+ assert compiled.is_partial+++def test_a_fully_supported_hunt_is_not_marked_partial():+ compiled = compile_hunt(+ _Intents(filters=[("host.name", "==", "WIN-DC01")]),+ tenant_id=TENANT,+ )+ assert compiled.unsupported_fields == []+ assert not compiled.is_partial+++# ── operators ─────────────────────────────────────────────────────────────+++ ("op", "expected"),+ [("==", "="), ("!=", "!="), (">", ">"), ("<", "<"), (">=", ">="), ("<=", "<=")],+)+def test_scalar_operators_render(op: str, expected: str):+ compiled = compile_hunt(+ _Intents(filters=[("destination.port", op, "445")]),+ tenant_id=TENANT,+ )+ assert f"dst_port {expected} %(f0)s" in compiled.sql+ assert compiled.params["f0"] == 445+++def test_numeric_columns_are_coerced_to_numbers():+ """ClickHouse rejects a string against a UInt column."""+ compiled = compile_hunt(+ _Intents(filters=[("event.severity", ">=", "4")]),+ tenant_id=TENANT,+ )+ assert compiled.params["f0"] == 4+++def test_a_non_numeric_value_on_a_numeric_column_is_refused():+ with pytest.raises(HuntCompileError):+ compile_hunt(_Intents(filters=[("destination.port", "==", "http")]), tenant_id=TENANT)+++def test_in_lists_are_decoded_from_the_ir_encoding():+ """The IR JSON-encodes IN lists so its dedup set stays hashable."""+ compiled = compile_hunt(+ _Intents(filters=[("user.name", "IN", json.dumps(["alice", "bob"]))]),+ tenant_id=TENANT,+ )+ assert "user_name IN %(f0)s" in compiled.sql+ assert compiled.params["f0"] == ["alice", "bob"]+++def test_like_becomes_a_bound_pattern():+ compiled = compile_hunt(+ _Intents(filters=[("process.name", "LIKE", "powershell")]),+ tenant_id=TENANT,+ )+ assert "ILIKE %(f0)s" in compiled.sql+ assert compiled.params["f0"] == "%powershell%"+++def test_array_columns_use_membership_not_equality():+ """`mitre_techniques` is an Array(String); `=` would never match."""+ compiled = compile_hunt(+ _Intents(filters=[("threat.technique.id", "==", json.dumps(["T1059"]))]),+ tenant_id=TENANT,+ )+ assert "hasAny(mitre_techniques, %(f0)s)" in compiled.sql+++def test_a_range_operator_on_an_array_column_is_refused():+ with pytest.raises(HuntCompileError):+ compile_hunt(+ _Intents(filters=[("threat.technique.id", ">", "T1059")]),+ tenant_id=TENANT,+ )+++# ── shape of the query ────────────────────────────────────────────────────+++def test_group_by_projects_the_grouping_and_a_count():+ compiled = compile_hunt(_Intents(group_by=["source.ip"]), tenant_id=TENANT)+ assert "GROUP BY source_ip" in compiled.sql+ assert "count() AS event_count" in compiled.sql+++def test_an_aggregate_query_is_not_ordered_by_an_ungrouped_column():+ """ClickHouse errors on ORDER BY event_time when it is not in GROUP BY.++ The unmappable-sort fallback used to emit exactly that, turning a+ reasonable question into a failed query.+ """+ compiled = compile_hunt(+ _Intents(group_by=["source.ip"], sort_by=("nonexistent.field", "desc")),+ tenant_id=TENANT,+ )+ assert "ORDER BY count() DESC" in compiled.sql+ assert "ORDER BY event_time" not in compiled.sql+++def test_a_flat_query_orders_by_time_descending():+ assert "ORDER BY event_time DESC" in compile_hunt(_Intents(), tenant_id=TENANT).sql+++def test_distinct_projects_a_single_column():+ compiled = compile_hunt(_Intents(distinct="user.name"), tenant_id=TENANT)+ assert "DISTINCT user_name" in compiled.sql+++def test_the_limit_is_capped():+ """An analyst asking for a million rows must not get a million rows."""+ compiled = compile_hunt(_Intents(limit=10_000_000), tenant_id=TENANT)+ assert f"LIMIT {MAX_LIMIT}" in compiled.sql+++def test_an_explicit_limit_is_honoured():+ assert "LIMIT 25" in compile_hunt(_Intents(limit=25), tenant_id=TENANT).sql+++# ── against the real translator ───────────────────────────────────────────+++def test_real_translator_output_compiles():+ """End to end from an analyst's sentence to bound SQL."""+ translator = pytest.importorskip("app._vendor.nl_query.translator")+ translated = translator.translate("processes named powershell.exe on host WIN-DC01")+ compiled = compile_hunt(translated.intents, tenant_id=TENANT)+ assert LAKE_TABLE in compiled.sql+ assert compiled.params["tenant_id"] == TENANT+ # Entity values arrived as parameters, not as SQL text.+ assert "win-dc01" in [str(v).lower() for v in compiled.params.values()]