Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

feat(api): let analysts hunt AiSOC's own event lake (#668)

Beenu Aroraa4bd8443d7d7
diff --git a/docs/openapi.yaml b/docs/openapi.yaml
index 760a9d7e2..5d57ffde4 100644
--- a/docs/openapi.yaml
+++ b/docs/openapi.yaml
@@ -21599,6 +21599,19 @@ components:
- type: string
- type: 'null'
title: Execution Error
+ executed_against:
+ anyOf:
+ - type: string
+ enum:
+ - elasticsearch
+ - lake
+ - type: 'null'
+ title: Executed Against
+ unsupported_filters:
+ items:
+ type: string
+ type: array
+ title: Unsupported Filters
type: object
required:
- request_id
@@ -24113,6 +24126,24 @@ components:
last_run_at:
type: string
title: Last Run At
+ executed:
+ type: boolean
+ title: Executed
+ default: false
+ result:
+ anyOf:
+ - $ref: '#/components/schemas/SavedHuntRunResult'
+ - type: 'null'
+ execution_error:
+ anyOf:
+ - type: string
+ - type: 'null'
+ title: Execution Error
+ unsupported_filters:
+ items:
+ type: string
+ type: array
+ title: Unsupported Filters
type: object
required:
- id
@@ -24124,15 +24155,22 @@ components:
description: 'Synchronous run result returned by ``POST /saved-hunts/{id}/run``.
- The endpoint *does not* execute the underlying ES|QL — that path is the
+ This endpoint used to re-translate the question, stamp ``last_run_at`` and
+
+ return — executing nothing. The UI then displayed "last run just now" for
+
+ a hunt that had queried no data at all, which is indistinguishable from a
+
+ hunt that ran and found nothing.
- job of :mod:`app.api.v1.endpoints.nl_query` (``/nl-query/execute``) and
- requires a configured Elasticsearch URL. The ``/run`` endpoint here only
+ It now runs the compiled query against AiSOC''s own event lake, which is
- re-translates and stamps ``last_run_at`` so the UI can show "last run
+ where every connector''s events are archived. ``executed`` says plainly
- just now" and so the scheduler treats a manual run as resetting cadence.'
+ whether rows were fetched, so a lake that is unreachable or unconfigured
+
+ reports that instead of looking like a clean run with zero hits.'
RunSummary:
properties:
id:
@@ -24720,6 +24758,31 @@ components:
- updated_at
title: SavedHuntModel
description: Wire format for a saved hunt row.
+ SavedHuntRunResult:
+ properties:
+ columns:
+ items:
+ type: string
+ type: array
+ title: Columns
+ rows:
+ items:
+ items: {}
+ type: array
+ type: array
+ title: Rows
+ total_rows:
+ type: integer
+ title: Total Rows
+ default: 0
+ took_ms:
+ anyOf:
+ - type: integer
+ - type: 'null'
+ title: Took Ms
+ type: object
+ title: SavedHuntRunResult
+ description: Rows the hunt actually returned.
SavedViewModel:
properties:
id:
diff --git a/services/api/app/api/v1/endpoints/nl_query.py b/services/api/app/api/v1/endpoints/nl_query.py
index 2e0fe1402..8a101c8da 100644
--- a/services/api/app/api/v1/endpoints/nl_query.py
+++ b/services/api/app/api/v1/endpoints/nl_query.py
@@ -26,21 +26,28 @@ import sys
import uuid
from datetime import UTC, datetime
from pathlib import Path
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Literal
import httpx
+import structlog
from fastapi import APIRouter, status
from pydantic import BaseModel, Field
from app.api.v1.deps import AuthUser
from app.core.airgap import AirgapViolation, enforce_airgap_for_url
from app.core.config import settings
+from app.db.clickhouse import (
+ LakeQueryError,
+ LakeQueryTimeoutError,
+ execute_lake_query,
+)
from app.services.esql_runner import (
ESQLExecutionError,
ESQLNotConfigured,
resolve_es_credentials,
run_esql_query,
)
+from app.services.lake_hunt import HuntCompileError, compile_hunt
if TYPE_CHECKING:
# Static-only re-export so type checkers can see the dataclass fields and
@@ -146,6 +153,8 @@ if not TYPE_CHECKING:
enhance_with_llm = _nl_query.enhance_with_llm
deterministic_translate = _nl_query.translate
+logger = structlog.get_logger(__name__)
+
router = APIRouter(prefix="/nl-query", tags=["nl_query"])
@@ -208,6 +217,13 @@ class QueryResult(BaseModel):
class NLQueryExecuteResponse(NLQueryTranslateResponse):
result: QueryResult | None = None
execution_error: str | None = None
+ #: Which backend produced `result`. An analyst must be able to tell
+ #: "nothing matched in your SIEM" from "nothing matched in AiSOC's lake".
+ executed_against: Literal["elasticsearch", "lake"] | None = None
+ #: Translator fields the lake does not store, when executing against the
+ #: lake. A hunt that quietly drops half its filters returns more rows than
+ #: was asked for, which reads as "nothing was filtered out".
+ unsupported_filters: list[str] = Field(default_factory=list)
# ────────────────────────────────────────────────────────────────────────────
@@ -293,6 +309,55 @@ async def _execute_esql(esql: str, es_url: str, es_api_key: str, max_rows: int)
# ────────────────────────────────────────────────────────────────────────────
+async def _execute_against_lake(
+ base: NLQueryExecuteResponse,
+ translated: TranslatedQuery,
+ body: NLQueryExecuteRequest,
+ user: AuthUser,
+) -> NLQueryExecuteResponse:
+ """Run the hunt against AiSOC's own event lake.
+
+ The translator's structured IR compiles straight to parameterised
+ ClickHouse SQL, so no part of the analyst's question reaches the SQL
+ string. The tenant predicate is generated as part of the WHERE clause
+ rather than rewritten in afterwards.
+ """
+ base.executed_against = "lake"
+ try:
+ compiled = compile_hunt(
+ translated.intents,
+ tenant_id=str(user.tenant_id),
+ hours=body.time_range_hours,
+ limit=body.max_rows,
+ )
+ except HuntCompileError as exc:
+ base.execution_error = f"Could not compile this question into a lake query: {exc}"
+ return base
+
+ base.unsupported_filters = compiled.unsupported_fields
+
+ try:
+ result = await execute_lake_query(compiled.sql, params=compiled.params)
+ except LakeQueryTimeoutError as exc:
+ base.execution_error = f"Lake query timed out: {exc}"
+ return base
+ except LakeQueryError as exc:
+ base.execution_error = f"Lake query failed: {exc}"
+ return base
+ except Exception as exc: # noqa: BLE001 — surface, never fabricate rows
+ logger.warning("nl_query.lake_execution_failed", error=str(exc))
+ base.execution_error = f"Lake query failed: {exc}. Configure ES_URL to hunt an external SIEM instead."
+ return base
+
+ base.result = QueryResult(
+ columns=list(result.columns),
+ rows=[list(row) for row in result.rows],
+ total_rows=result.row_count,
+ took_ms=result.elapsed_ms,
+ )
+ return base
+
+
@router.post(
"/translate",
response_model=NLQueryTranslateResponse,
@@ -347,9 +412,13 @@ async def execute_query(
try:
es_url, es_api_key = resolve_es_credentials()
except ESQLNotConfigured:
- base.execution_error = "ES_URL or ES_API_KEY not configured. Set them in environment variables."
- return base
+ # No external SIEM configured. Every connector's events are archived
+ # to AiSOC's own ClickHouse lake, so hunt that instead of refusing.
+ # This endpoint used to return "ES_URL not configured" and stop, which
+ # left a tenant unable to query any of the data they had ingested.
+ return await _execute_against_lake(base, translated, body, user)
+ base.executed_against = "elasticsearch"
try:
base.result = await _execute_esql(
translated.esql,
diff --git a/services/api/app/api/v1/endpoints/saved_hunts.py b/services/api/app/api/v1/endpoints/saved_hunts.py
index 40110a39b..857915e36 100644
--- a/services/api/app/api/v1/endpoints/saved_hunts.py
+++ b/services/api/app/api/v1/endpoints/saved_hunts.py
@@ -58,8 +58,10 @@ from app.api.v1.deps import AuthUser
from app.api.v1.endpoints.nl_query import ( # noqa: E402
deterministic_translate,
)
+from app.db.clickhouse import execute_lake_query
from app.db.rls import TenantDBSession
from app.models.saved_hunt import SavedHunt
+from app.services.lake_hunt import HuntCompileError, compile_hunt
logger = structlog.get_logger()
@@ -192,14 +194,27 @@ class CreateSavedHuntRequest(BaseModel):
schedule: str | None = Field(default=None, max_length=_MAX_SCHEDULE_LEN)
+class SavedHuntRunResult(BaseModel):
+ """Rows the hunt actually returned."""
+
+ columns: list[str] = Field(default_factory=list)
+ rows: list[list[Any]] = Field(default_factory=list)
+ total_rows: int = 0
+ took_ms: int | None = None
+
+
class RunSavedHuntResponse(BaseModel):
"""Synchronous run result returned by ``POST /saved-hunts/{id}/run``.
- The endpoint *does not* execute the underlying ES|QL — that path is the
- job of :mod:`app.api.v1.endpoints.nl_query` (``/nl-query/execute``) and
- requires a configured Elasticsearch URL. The ``/run`` endpoint here only
- re-translates and stamps ``last_run_at`` so the UI can show "last run
- just now" and so the scheduler treats a manual run as resetting cadence.
+ This endpoint used to re-translate the question, stamp ``last_run_at`` and
+ return — executing nothing. The UI then displayed "last run just now" for
+ a hunt that had queried no data at all, which is indistinguishable from a
+ hunt that ran and found nothing.
+
+ It now runs the compiled query against AiSOC's own event lake, which is
+ where every connector's events are archived. ``executed`` says plainly
+ whether rows were fetched, so a lake that is unreachable or unconfigured
+ reports that instead of looking like a clean run with zero hits.
"""
id: str
@@ -207,6 +222,14 @@ class RunSavedHuntResponse(BaseModel):
nl_query: str
translated_query: TranslatedQueryEnvelope
last_run_at: str
+ #: True only when a query really ran against the lake.
+ executed: bool = False
+ result: SavedHuntRunResult | None = None
+ #: Why no rows were fetched, when that is the case.
+ execution_error: str | None = None
+ #: Translator fields the lake does not store. A hunt that silently drops
+ #: filters returns more rows than the analyst asked for.
+ unsupported_filters: list[str] = Field(default_factory=list)
# ---------------------------------------------------------------------------
@@ -235,6 +258,43 @@ def _translate(nl_query: str) -> TranslatedQueryEnvelope:
return TranslatedQueryEnvelope.from_translator(tq)
+async def _execute_on_lake(
+ nl_query: str,
+ tenant_id: str,
+) -> tuple[bool, SavedHuntRunResult | None, str | None, list[str]]:
+ """Run the hunt against the tenant's slice of the event lake.
+
+ Returns ``(executed, result, error, unsupported_fields)``. Never raises:
+ a hunt run must not 500 because the lake is down, but it also must not
+ report a successful run that did not happen.
+ """
+ try:
+ compiled = compile_hunt(deterministic_translate(nl_query).intents, tenant_id=tenant_id)
+ except HuntCompileError as exc:
+ return False, None, f"Could not compile this hunt into a lake query: {exc}", []
+ except Exception as exc: # noqa: BLE001 — report, never fabricate a run
+ logger.warning("saved_hunt.compile_failed", error=str(exc))
+ return False, None, f"Could not compile this hunt into a lake query: {exc}", []
+
+ try:
+ lake_result = await execute_lake_query(compiled.sql, params=compiled.params)
+ except Exception as exc: # noqa: BLE001 — surface the real reason
+ logger.warning("saved_hunt.lake_execution_failed", error=str(exc))
+ return False, None, f"Lake query failed: {exc}", compiled.unsupported_fields
+
+ return (
+ True,
+ SavedHuntRunResult(
+ columns=list(lake_result.columns),
+ rows=[list(r) for r in lake_result.rows],
+ total_rows=lake_result.row_count,
+ took_ms=lake_result.elapsed_ms,
+ ),
+ None,
+ compiled.unsupported_fields,
+ )
+
+
async def _load_owned_hunt(
db: Any, # noqa: ANN401
hunt_id: uuid.UUID,
@@ -407,6 +467,7 @@ async def run_saved_hunt(
translated = _translate(row.nl_query)
now = datetime.now(UTC)
+ executed, result, execution_error, unsupported = await _execute_on_lake(row.nl_query, str(user.tenant_id))
# Persist the refreshed translation alongside the run timestamp so the
# next list call surfaces the latest envelope. The schedule worker uses
@@ -430,4 +491,8 @@ async def run_saved_hunt(
nl_query=row.nl_query,
translated_query=translated,
last_run_at=now.isoformat(),
+ executed=executed,
+ result=result,
+ execution_error=execution_error,
+ unsupported_filters=unsupported,
)
diff --git a/services/api/app/db/clickhouse.py b/services/api/app/db/clickhouse.py
index 95a2a028a..f3a944c05 100644
--- a/services/api/app/db/clickhouse.py
+++ b/services/api/app/db/clickhouse.py
@@ -227,6 +227,7 @@ async def execute_lake_query(
*,
timeout_seconds: float | None = None,
extra_settings: dict[str, Any] | None = None,
+ params: dict[str, Any] | None = None,
) -> LakeQueryResult:
"""Run a rewritten lake SQL statement and return columns + rows.
@@ -261,6 +262,13 @@ async def execute_lake_query(
:data:`DEFAULT_QUERY_SETTINGS`. Used by the lake endpoints to
layer in per-tenant overrides (e.g. tighter row caps for
unprivileged roles).
+ params:
+ Values for ``%(name)s`` placeholders, bound by the driver. Callers
+ that build SQL from untrusted input — an analyst's hunt question, for
+ instance — bind their values here so no user text is ever
+ concatenated into the statement. This includes the tenant predicate:
+ binding it as a parameter is stronger than injecting a literal and
+ hoping the rewrite survived.
Returns
-------
@@ -301,6 +309,7 @@ async def execute_lake_query(
# which we need so the API can echo a stable column ordering.
return client.execute(
sql,
+ params or {},
with_column_types=True,
settings=merged_settings,
)
diff --git a/services/api/app/services/lake_hunt.py b/services/api/app/services/lake_hunt.py
new file mode 100644
index 000000000..98f15e110
--- /dev/null
+++ b/services/api/app/services/lake_hunt.py
@@ -0,0 +1,290 @@
+"""Compile a translated hunt into SQL against AiSOC's own event lake.
+
+Every connector's events are archived to ClickHouse `aisoc.raw_events`, and
+until now nothing could hunt them. `/nl-query/execute` only ever executed
+against Elasticsearch, so a tenant whose data lives in the platform's own lake
+got "ES_URL or ES_API_KEY not configured" and had no way to query anything they
+had ingested. `POST /saved-hunts/{id}/run` was worse: it re-translated the
+question, stamped `last_run_at`, and returned — so the UI displayed "last run
+just now" having queried nothing at all.
+
+This module closes that by compiling the translator's structured IR
+(`QueryIntents`) into ClickHouse SQL. Deterministic, no LLM: the IR already
+carries typed filters with a closed operator set, group-bys, aggregations, a
+sort and a limit.
+
+Two properties this must not get wrong:
+
+**No user text ever reaches the SQL string.** Field names resolve through a
+fixed alias map to real column names, operators come from a closed set, and
+every value is bound as a query parameter. A question is untrusted input, and
+the lake holds every tenant's events.
+
+**Tenant scoping is structural, not rewritten in afterwards.** The tenant
+predicate is part of the generated WHERE clause and its parameter is bound by
+this module, rather than relying on `rewrite_for_tenant` — which exists to
+constrain untrusted *operator* SQL, and which cannot help if a caller forgets
+to use it.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from typing import Any
+
+import structlog
+
+logger = structlog.get_logger(__name__)
+
+LAKE_TABLE = "aisoc.raw_events"
+
+#: ECS-ish field names the translator emits, mapped to lake columns.
+#:
+#: The translator speaks ECS because it also targets Elasticsearch. Anything
+#: absent from this map is a field the lake genuinely does not store, and is
+#: reported to the caller rather than silently dropped — a hunt that quietly
+#: ignores half its filters returns more rows than the analyst asked for, which
+#: reads as "no results were filtered out" when the truth is "your filter was
+#: discarded".
+_FIELD_TO_COLUMN: dict[str, str] = {
+ "@timestamp": "event_time",
+ "event.created": "event_time",
+ "user.name": "user_name",
+ "source.ip": "source_ip",
+ "destination.ip": "dest_ip",
+ "source.port": "src_port",
+ "destination.port": "dst_port",
+ "host.name": "src_hostname",
+ "destination.domain": "dst_hostname",
+ "process.name": "process_name",
+ "file.path": "file_path",
+ "file.hash.sha256": "hash_sha256",
+ "event.severity": "severity_id",
+ "event.category": "connector_type",
+ "event.provider": "connector_type",
+ "network.protocol": "protocol",
+ "rule.name": "connector_type",
+ "threat.technique.id": "mitre_techniques",
+ "process.command_line": "raw_payload",
+ "url.full": "raw_payload",
+ "user_agent.original": "raw_payload",
+}
+
+#: Columns holding an Array(String); membership tests differ from scalars.
+_ARRAY_COLUMNS = frozenset({"mitre_techniques", "mitre_tactics", "iocs"})
+
+#: Operators the IR can emit, mapped to ClickHouse syntax. A closed set: an
+#: operator outside it is rejected rather than passed through.
+_SCALAR_OPS = {
+ "==": "=",
+ "!=": "!=",
+ ">": ">",
+ "<": "<",
+ ">=": ">=",
+ "<=": "<=",
+}
+
+MAX_LIMIT = 1000
+
+
+@dataclass(frozen=True)
+class CompiledHunt:
+ """A parameterised lake query plus an honest account of what was dropped."""
+
+ sql: str
+ params: dict[str, Any]
+ #: Translator fields with no lake column. Surfaced to the caller so a
+ #: partial hunt is never presented as a complete one.
+ unsupported_fields: list[str]
+
+ @property
+ def is_partial(self) -> bool:
+ return bool(self.unsupported_fields)
+
+
+class HuntCompileError(ValueError):
+ """The IR could not be compiled into a safe lake query."""
+
+
+def _column_for(field: str) -> str | None:
+ if field in _FIELD_TO_COLUMN:
+ return _FIELD_TO_COLUMN[field]
+ # A bare column name is accepted only if it really is a lake column.
+ if field in set(_FIELD_TO_COLUMN.values()) | _ARRAY_COLUMNS:
+ return field
+ return None
+
+
+def _predicate(column: str, op: str, value: str, key: str) -> tuple[str, Any]:
+ """Build one parameterised predicate. Never interpolates the value."""
+ placeholder = f"%({key})s"
+
+ if column in _ARRAY_COLUMNS:
+ if op in {"==", "IN"}:
+ values = _as_list(value)
+ return f"hasAny({column}, %({key})s)", values
+ raise HuntCompileError(f"operator {op!r} is not supported on array column {column!r}")
+
+ if op == "IN":
+ return f"{column} IN %({key})s", _as_list(value)
+ if op == "LIKE":
+ # The translator emits LIKE for "contains" phrasing. Bind the whole
+ # pattern, wildcards included, so the value stays data.
+ pattern = value if "%" in value else f"%{value}%"
+ return f"{column} ILIKE %({key})s", pattern
+ if op in _SCALAR_OPS:
+ return f"{column} {_SCALAR_OPS[op]} {placeholder}", _coerce(column, value)
+
+ raise HuntCompileError(f"unsupported operator {op!r}")
+
+
+def _as_list(value: str) -> list[str]:
+ """The IR JSON-encodes IN lists so its dedup set stays hashable."""
+ try:
+ parsed = json.loads(value)
+ except (TypeError, ValueError):
+ return [value]
+ return [str(v) for v in parsed] if isinstance(parsed, list) else [str(parsed)]
+
+
+def _coerce(column: str, value: str) -> Any:
+ """Numeric columns need numbers, not strings, or ClickHouse rejects them."""
+ if column in {"severity_id", "src_port", "dst_port"}:
+ try:
+ return int(value)
+ except (TypeError, ValueError) as exc:
+ raise HuntCompileError(f"{column} needs a number, got {value!r}") from exc
+ return value
+
+
+def compile_hunt(
+ intents: Any,
+ *,
+ tenant_id: str,
+ hours: int = 24,
+ limit: int | None = None,
+) -> CompiledHunt:
+ """Compile translator intents into a tenant-scoped lake query.
+
+ `intents` is a `QueryIntents` (or anything with the same attributes), so
+ this works against both the API's vendored translator and the agents' copy
+ without importing either.
+ """
+ params: dict[str, Any] = {"tenant_id": tenant_id, "hours": max(1, int(hours))}
+ where = [
+ # Structural, not rewritten in afterwards.
+ "tenant_id = %(tenant_id)s",
+ "event_time >= now() - INTERVAL %(hours)s HOUR",
+ ]
+ unsupported: list[str] = []
+
+ for index, (field, op, value) in enumerate(getattr(intents, "filters", []) or []):
+ column = _column_for(str(field))
+ if column is None:
+ unsupported.append(str(field))
+ continue
+ key = f"f{index}"
+ clause, bound = _predicate(column, str(op), str(value), key)
+ where.append(clause)
+ params[key] = bound
+
+ group_by: list[str] = []
+ for field in getattr(intents, "group_by", []) or []:
+ column = _column_for(str(field))
+ if column is None:
+ unsupported.append(str(field))
+ continue
+ group_by.append(column)
+
+ select = _select_clause(intents, group_by, unsupported)
+ sql = f"SELECT {', '.join(select)} FROM {LAKE_TABLE} WHERE {' AND '.join(where)}"
+
+ if group_by:
+ sql += f" GROUP BY {', '.join(group_by)}"
+
+ sql += f" ORDER BY {_order_clause(intents, group_by)}"
+
+ effective_limit = min(int(limit or getattr(intents, "limit", 500) or 500), MAX_LIMIT)
+ sql += f" LIMIT {max(1, effective_limit)}"
+
+ return CompiledHunt(sql=sql, params=params, unsupported_fields=sorted(set(unsupported)))
+
+
+def _select_clause(intents: Any, group_by: list[str], unsupported: list[str]) -> list[str]:
+ """Projection: aggregations when asked for, otherwise the event columns."""
+ aggregations = getattr(intents, "aggregations", []) or []
+ distinct = getattr(intents, "distinct", None)
+
+ if distinct:
+ column = _column_for(str(distinct))
+ if column is not None:
+ return [f"DISTINCT {column}"]
+ unsupported.append(str(distinct))
+
+ if aggregations:
+ select = list(group_by)
+ for function, arg, alias in aggregations:
+ select.append(_aggregation(str(function), arg, str(alias), unsupported))
+ return select
+
+ if group_by:
+ return [*group_by, "count() AS event_count"]
+
+ return [
+ "event_time",
+ "severity",
+ "connector_type",
+ "src_hostname",
+ "user_name",
+ "process_name",
+ "toString(source_ip) AS source_ip",
+ "toString(dest_ip) AS dest_ip",
+ "mitre_techniques",
+ ]
+
+
+def _aggregation(function: str, arg: Any, alias: str, unsupported: list[str]) -> str:
+ """Render one aggregation from the closed set the translator can emit."""
+ name = function.lower()
+ safe_alias = "".join(c for c in alias if c.isalnum() or c == "_") or "value"
+
+ if name in {"count", "count_distinct"} and not arg:
+ return f"count() AS {safe_alias}"
+
+ column = _column_for(str(arg)) if arg else None
+ if column is None:
+ if arg:
+ unsupported.append(str(arg))
+ return f"count() AS {safe_alias}"
+
+ renderers = {
+ "count": f"count({column})",
+ "count_distinct": f"uniqExact({column})",
+ "sum": f"sum({column})",
+ "avg": f"avg({column})",
+ "min": f"min({column})",
+ "max": f"max({column})",
+ }
+ return f"{renderers.get(name, f'count({column})')} AS {safe_alias}"
+
+
+def _order_clause(intents: Any, group_by: list[str]) -> str:
+ """Always returns a clause that is valid for the query being built.
+
+ The default must respect `group_by`: ordering an aggregate query by
+ `event_time` is a ClickHouse error, because the column is not in the
+ grouping set. Same for a sort field that does not map to a lake column —
+ falling back to a non-grouped column there would turn an unmappable sort
+ into a failed query rather than a sensible default.
+ """
+ default = "count() DESC" if group_by else "event_time DESC"
+ sort_by = getattr(intents, "sort_by", None)
+ if not sort_by:
+ return default
+
+ field, direction = sort_by
+ column = _column_for(str(field))
+ if column is None or (group_by and column not in group_by):
+ return default
+ return f"{column} {'DESC' if str(direction).lower().startswith('desc') else 'ASC'}"
diff --git a/services/api/tests/test_lake_hunt.py b/services/api/tests/test_lake_hunt.py
new file mode 100644
index 000000000..8b8ceacda
--- /dev/null
+++ b/services/api/tests/test_lake_hunt.py
@@ -0,0 +1,264 @@
+"""Hunting AiSOC's own event lake.
+
+Every connector's events are archived to ClickHouse `aisoc.raw_events`, and
+nothing could query them. `/nl-query/execute` only ever executed against
+Elasticsearch, so a tenant whose data lives in the platform's own lake got
+"ES_URL or ES_API_KEY not configured" and could not query anything they had
+ingested. `POST /saved-hunts/{id}/run` was worse: it re-translated the
+question, stamped `last_run_at` and returned, so the console showed "last run
+just now" for a hunt that had queried nothing — indistinguishable from a hunt
+that ran and found no matches.
+
+The compiler turns the translator's structured IR into ClickHouse SQL. The two
+properties it must not get wrong are that no analyst text ever reaches the SQL
+string, and that tenant scoping is generated rather than rewritten in
+afterwards.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+
+import pytest
+from app.services.lake_hunt import (
+ LAKE_TABLE,
+ MAX_LIMIT,
+ HuntCompileError,
+ compile_hunt,
+)
+
+TENANT = "aaaaaaaa-0000-0000-0000-000000000001"
+
+
+@dataclass
+class _Intents:
+ """Stands in for the translator's QueryIntents."""
+
+ filters: list[tuple[str, str, str]] = field(default_factory=list)
+ group_by: list[str] = field(default_factory=list)
+ aggregations: list[tuple[str, str | None, str]] = field(default_factory=list)
+ sort_by: tuple[str, str] | None = None
+ limit: int = 500
+ distinct: str | None = None
+ time_field: str = "@timestamp"
+
+
+# ── tenant scoping ────────────────────────────────────────────────────────
+
+
+def test_the_tenant_predicate_is_generated_and_bound():
+ """Not rewritten in afterwards, and bound rather than interpolated.
+
+ `rewrite_for_tenant` exists to constrain untrusted operator SQL and cannot
+ help a caller who forgets to use it, so scoping is structural here.
+ """
+ compiled = compile_hunt(_Intents(), tenant_id=TENANT)
+ assert "tenant_id = %(tenant_id)s" in compiled.sql
+ assert compiled.params["tenant_id"] == TENANT
+ assert TENANT not in compiled.sql
+
+
+def test_the_query_targets_the_lake_table():
+ assert LAKE_TABLE in compile_hunt(_Intents(), tenant_id=TENANT).sql
+
+
+def test_a_time_window_is_always_applied():
+ """An unbounded scan over the lake is a denial of service on itself."""
+ compiled = compile_hunt(_Intents(), tenant_id=TENANT, hours=6)
+ assert "event_time >= now() - INTERVAL %(hours)s HOUR" in compiled.sql
+ assert compiled.params["hours"] == 6
+
+
+# ── no user text in the SQL ───────────────────────────────────────────────
+
+
+def test_filter_values_are_bound_never_interpolated():
+ compiled = compile_hunt(
+ _Intents(filters=[("user.name", "==", "alice")]),
+ tenant_id=TENANT,
+ )
+ assert "alice" not in compiled.sql
+ assert "alice" in compiled.params.values()
+
+
+def test_a_sql_injection_attempt_in_the_question_stays_data():
+ """The question is untrusted input and the lake holds every tenant's events."""
+ payload = "alice'; DROP TABLE aisoc.raw_events; --"
+ compiled = compile_hunt(
+ _Intents(filters=[("user.name", "==", payload)]),
+ tenant_id=TENANT,
+ )
+ assert "DROP TABLE" not in compiled.sql
+ assert payload in compiled.params.values()
+
+
+def test_an_unknown_field_cannot_become_a_column_name():
+ """Otherwise a crafted field name is an injection point in the projection."""
+ compiled = compile_hunt(
+ _Intents(filters=[("evil) OR 1=1 --", "==", "x")]),
+ tenant_id=TENANT,
+ )
+ assert "OR 1=1" not in compiled.sql
+ assert "evil) OR 1=1 --" in compiled.unsupported_fields
+
+
+def test_an_unknown_operator_is_rejected():
+ with pytest.raises(HuntCompileError):
+ compile_hunt(_Intents(filters=[("user.name", "REGEXP", ".*")]), tenant_id=TENANT)
+
+
+def test_an_aggregation_alias_cannot_inject():
+ compiled = compile_hunt(
+ _Intents(aggregations=[("count", None, "x AS y, (SELECT 1)")]),
+ tenant_id=TENANT,
+ )
+ assert "SELECT 1" not in compiled.sql
+
+
+# ── honesty about what was dropped ────────────────────────────────────────
+
+
+def test_fields_the_lake_does_not_store_are_reported():
+ """A hunt that silently drops filters returns more rows than was asked for.
+
+ Which reads to an analyst as "nothing was filtered out" when the truth is
+ "your filter was discarded".
+ """
+ compiled = compile_hunt(
+ _Intents(filters=[("source.geo.country_iso_code", "==", "IR")]),
+ tenant_id=TENANT,
+ )
+ assert compiled.unsupported_fields == ["source.geo.country_iso_code"]
+ assert compiled.is_partial
+
+
+def test_a_fully_supported_hunt_is_not_marked_partial():
+ compiled = compile_hunt(
+ _Intents(filters=[("host.name", "==", "WIN-DC01")]),
+ tenant_id=TENANT,
+ )
+ assert compiled.unsupported_fields == []
+ assert not compiled.is_partial
+
+
+# ── operators ─────────────────────────────────────────────────────────────
+
+
+ ("op", "expected"),
+ [("==", "="), ("!=", "!="), (">", ">"), ("<", "<"), (">=", ">="), ("<=", "<=")],
+)
+def test_scalar_operators_render(op: str, expected: str):
+ compiled = compile_hunt(
+ _Intents(filters=[("destination.port", op, "445")]),
+ tenant_id=TENANT,
+ )
+ assert f"dst_port {expected} %(f0)s" in compiled.sql
+ assert compiled.params["f0"] == 445
+
+
+def test_numeric_columns_are_coerced_to_numbers():
+ """ClickHouse rejects a string against a UInt column."""
+ compiled = compile_hunt(
+ _Intents(filters=[("event.severity", ">=", "4")]),
+ tenant_id=TENANT,
+ )
+ assert compiled.params["f0"] == 4
+
+
+def test_a_non_numeric_value_on_a_numeric_column_is_refused():
+ with pytest.raises(HuntCompileError):
+ compile_hunt(_Intents(filters=[("destination.port", "==", "http")]), tenant_id=TENANT)
+
+
+def test_in_lists_are_decoded_from_the_ir_encoding():
+ """The IR JSON-encodes IN lists so its dedup set stays hashable."""
+ compiled = compile_hunt(
+ _Intents(filters=[("user.name", "IN", json.dumps(["alice", "bob"]))]),
+ tenant_id=TENANT,
+ )
+ assert "user_name IN %(f0)s" in compiled.sql
+ assert compiled.params["f0"] == ["alice", "bob"]
+
+
+def test_like_becomes_a_bound_pattern():
+ compiled = compile_hunt(
+ _Intents(filters=[("process.name", "LIKE", "powershell")]),
+ tenant_id=TENANT,
+ )
+ assert "ILIKE %(f0)s" in compiled.sql
+ assert compiled.params["f0"] == "%powershell%"
+
+
+def test_array_columns_use_membership_not_equality():
+ """`mitre_techniques` is an Array(String); `=` would never match."""
+ compiled = compile_hunt(
+ _Intents(filters=[("threat.technique.id", "==", json.dumps(["T1059"]))]),
+ tenant_id=TENANT,
+ )
+ assert "hasAny(mitre_techniques, %(f0)s)" in compiled.sql
+
+
+def test_a_range_operator_on_an_array_column_is_refused():
+ with pytest.raises(HuntCompileError):
+ compile_hunt(
+ _Intents(filters=[("threat.technique.id", ">", "T1059")]),
+ tenant_id=TENANT,
+ )
+
+
+# ── shape of the query ────────────────────────────────────────────────────
+
+
+def test_group_by_projects_the_grouping_and_a_count():
+ compiled = compile_hunt(_Intents(group_by=["source.ip"]), tenant_id=TENANT)
+ assert "GROUP BY source_ip" in compiled.sql
+ assert "count() AS event_count" in compiled.sql
+
+
+def test_an_aggregate_query_is_not_ordered_by_an_ungrouped_column():
+ """ClickHouse errors on ORDER BY event_time when it is not in GROUP BY.
+
+ The unmappable-sort fallback used to emit exactly that, turning a
+ reasonable question into a failed query.
+ """
+ compiled = compile_hunt(
+ _Intents(group_by=["source.ip"], sort_by=("nonexistent.field", "desc")),
+ tenant_id=TENANT,
+ )
+ assert "ORDER BY count() DESC" in compiled.sql
+ assert "ORDER BY event_time" not in compiled.sql
+
+
+def test_a_flat_query_orders_by_time_descending():
+ assert "ORDER BY event_time DESC" in compile_hunt(_Intents(), tenant_id=TENANT).sql
+
+
+def test_distinct_projects_a_single_column():
+ compiled = compile_hunt(_Intents(distinct="user.name"), tenant_id=TENANT)
+ assert "DISTINCT user_name" in compiled.sql
+
+
+def test_the_limit_is_capped():
+ """An analyst asking for a million rows must not get a million rows."""
+ compiled = compile_hunt(_Intents(limit=10_000_000), tenant_id=TENANT)
+ assert f"LIMIT {MAX_LIMIT}" in compiled.sql
+
+
+def test_an_explicit_limit_is_honoured():
+ assert "LIMIT 25" in compile_hunt(_Intents(limit=25), tenant_id=TENANT).sql
+
+
+# ── against the real translator ───────────────────────────────────────────
+
+
+def test_real_translator_output_compiles():
+ """End to end from an analyst's sentence to bound SQL."""
+ translator = pytest.importorskip("app._vendor.nl_query.translator")
+ translated = translator.translate("processes named powershell.exe on host WIN-DC01")
+ compiled = compile_hunt(translated.intents, tenant_id=TENANT)
+ assert LAKE_TABLE in compiled.sql
+ assert compiled.params["tenant_id"] == TENANT
+ # Entity values arrived as parameters, not as SQL text.
+ assert "win-dc01" in [str(v).lower() for v in compiled.params.values()]