Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

test(4.3): 38 cases, a deterministic claim proof, and the test it replaced

Beenu Aroraae0d8a1e2514
diff --git a/.github/workflows/lake-query-jobs-live.yml b/.github/workflows/lake-query-jobs-live.yml
new file mode 100644
index 000000000..d0efbdde8
--- /dev/null
+++ b/.github/workflows/lake-query-jobs-live.yml
@@ -0,0 +1,157 @@
+# Cold-tier lake query jobs, against real Postgres (depth plan 4.3).
+#
+# The properties here are about concurrency and state, and a mocked
+# session answers whatever the test asked it to. Two API replicas must not
+# both start one cold scan; a terminal state must never be left; a job id
+# must be tenant-scoped.
+#
+# The claim test is deliberately *deterministic* rather than a race. An
+# `asyncio.gather` of five concurrent claims passes under a
+# SELECT-then-UPDATE implementation as well — measured, by subverting the
+# claim and watching that version stay green — so the two transactions are
+# interleaved explicitly and the first holds its lock while the second
+# reaches the same row.
+#
+# No `paths:` filter: a cross-tenant read guard must grade every pull
+# request.
+name: Lake query jobs (live)
+
+on:
+ push:
+ branches: [main]
+ pull_request:
+ workflow_dispatch:
+
+concurrency:
+ group: lake-query-jobs-live-${{ github.ref }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.sha }}
+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+permissions:
+ contents: read
+
+jobs:
+ lake-query-jobs-live:
+ name: One replica claims a cold scan, and a finished job stays finished
+ runs-on: ubuntu-latest
+ timeout-minutes: 20
+
+ services:
+ postgres:
+ image: public.ecr.aws/docker/library/postgres:16-alpine
+ env:
+ POSTGRES_USER: aisoc
+ POSTGRES_PASSWORD: aisoc_ci_only
+ POSTGRES_DB: aisoc
+ ports: ["5432:5432"]
+ options: >-
+ --health-cmd "pg_isready -U aisoc"
+ --health-interval 5s
+ --health-timeout 5s
+ --health-retries 20
+
+ env:
+ LAKE_JOB_DSN: postgresql+asyncpg://aisoc:aisoc_ci_only@localhost:5432/aisoc
+ PGPASSWORD: aisoc_ci_only
+ ENVIRONMENT: test
+ SECRET_KEY: ci-lake-job-signing-secret-at-least-32-chars
+ OTEL_SDK_DISABLED: "true"
+
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+
+ - uses: actions/setup-python@e9d6f990972a57673cdb72ec29e19d42ba28880f # v6
+ with:
+ python-version: "3.11"
+
+ - name: Install api from its lockfile
+ working-directory: services/api
+ run: |
+ set -euo pipefail
+ python -m pip install --quiet --upgrade pip "poetry==2.4.1"
+ poetry config virtualenvs.create false
+ poetry install --no-root --no-interaction
+ python -m pip install --quiet pytest pytest-asyncio
+
+ - name: Apply every migration, in order
+ run: |
+ set -euo pipefail
+ for f in services/api/migrations/*.sql; do
+ psql -h localhost -U aisoc -d aisoc -v ON_ERROR_STOP=1 -q -f "$f"
+ done
+
+ # progress_ratio must be nullable. NOT NULL would force a zero where
+ # the honest answer is unknown, and a progress bar that renders a
+ # forced zero as 0% is as wrong as one that renders it as 100%.
+ - name: progress_ratio is nullable
+ run: |
+ set -euo pipefail
+ nullable=$(psql -h localhost -U aisoc -d aisoc -tAc \
+ "SELECT is_nullable FROM information_schema.columns
+ WHERE table_name = 'lake_query_jobs' AND column_name = 'progress_ratio'")
+ if [ "$nullable" != "YES" ]; then
+ echo "FAIL: progress_ratio is '$nullable'; unknown progress has nowhere to go but a zero."
+ exit 1
+ fi
+ echo "OK: progress_ratio is nullable."
+
+ - name: The claim, the terminal states and the tenant predicate
+ working-directory: services/api
+ env:
+ ISOLATION_LAKE_JOB_DSN: ${{ env.LAKE_JOB_DSN }}
+ DATABASE_URL: ${{ env.LAKE_JOB_DSN }}
+ run: python -m pytest ../../tests/isolation/test_lake_query_jobs_live.py -v --tb=short
+
+ - name: The suite actually ran
+ working-directory: services/api
+ env:
+ ISOLATION_LAKE_JOB_DSN: ${{ env.LAKE_JOB_DSN }}
+ DATABASE_URL: ${{ env.LAKE_JOB_DSN }}
+ run: |
+ set -euo pipefail
+ python -m pytest ../../tests/isolation/test_lake_query_jobs_live.py -q 2>&1 | tee /tmp/lakejob-run.txt
+ if grep -qE '[0-9]+ skipped' /tmp/lakejob-run.txt; then
+ echo "FAIL: a case skipped in the job whose whole purpose is to run it live."
+ exit 1
+ fi
+ if ! grep -qE '[0-9]+ passed' /tmp/lakejob-run.txt; then
+ echo "FAIL: no case reported as passed."
+ exit 1
+ fi
+
+ # The negative control.
+ #
+ # Remove the `status = 'queued'` predicate from the claim. Postgres
+ # then has nothing to re-evaluate after the row lock is released, so
+ # both interleaved transactions claim the same job and two replicas
+ # would both run the scan. The suite must go red.
+ - name: The gate fails when the claim stops re-checking the status
+ working-directory: services/api
+ env:
+ ISOLATION_LAKE_JOB_DSN: ${{ env.LAKE_JOB_DSN }}
+ DATABASE_URL: ${{ env.LAKE_JOB_DSN }}
+ run: |
+ set -euo pipefail
+ cp app/services/lake_query_runner.py /tmp/runner.bak
+ python - <<'PY'
+ import pathlib, sys
+ p = pathlib.Path("app/services/lake_query_runner.py")
+ s = p.read_text()
+ needle = ' .where(LakeQueryJob.id == job_id, LakeQueryJob.status == "queued")\n'
+ if needle not in s:
+ print("negative control could not find the claim predicate; the claim has been "
+ "reshaped and this control no longer proves anything")
+ sys.exit(1)
+ p.write_text(s.replace(needle, " .where(LakeQueryJob.id == job_id)\n", 1))
+ PY
+ set +e
+ python -m pytest \
+ "../../tests/isolation/test_lake_query_jobs_live.py::TestTheClaimIsAtomic::test_two_interleaved_transactions_produce_one_winner" \
+ -q >/dev/null 2>&1
+ rc=$?
+ set -e
+ cp /tmp/runner.bak app/services/lake_query_runner.py
+ if [ "$rc" -eq 0 ]; then
+ echo "FAIL: both transactions claimed the job and the suite stayed green."
+ exit 1
+ fi
+ echo "OK: without the predicate both transactions claim the job and the suite goes red, as it must."
diff --git a/scripts/check_retention_window.py b/scripts/check_retention_window.py
index f9e4b50c5..d5df41268 100644
--- a/scripts/check_retention_window.py
+++ b/scripts/check_retention_window.py
@@ -23,6 +23,18 @@ system. The purge worker must read holds and must skip what they cover.
purge and a purge that silently did nothing look the same in a log. The run
summary has to carry the count it withheld.
+**A cold read with no cost and no progress.** The plan's third bullet. A
+query over the cold volume either returned inside the published
+30-second synchronous bound or raised a bare timeout, which tells an
+analyst nothing about whether a longer deadline would have helped or how
+much data was involved — and the ``cost`` ``/lake/sql`` did report was
+the rate limiter's token price, a number about how often you may ask.
+The estimate must come from ClickHouse's own planner and parts, the
+money figure must be absent rather than invented when no retrieval price
+is configured, the job must be claimed with a predicate that two
+replicas cannot both satisfy, and a progress ratio with no denominator
+must be null rather than zero.
+
These are read out of the source rather than exercised, because the lake
TTL lives in DDL and the worker needs Postgres and ClickHouse. The
behavioural half is `services/api/tests/test_retention_legal_hold.py`,
@@ -56,6 +68,10 @@ TIERING_SQL = Path("services/api/clickhouse/tiering/002_tiering.sql")
RETENTION_PY = Path("services/api/app/services/retention.py")
WORKER_PY = Path("services/api/app/workers/retention_purge.py")
LAKE_MIGRATIONS = Path("services/api/app/db/lake_migrations.py")
+COLD_QUERY_PY = Path("services/api/app/services/lake_cold_query.py")
+JOB_RUNNER_PY = Path("services/api/app/services/lake_query_runner.py")
+LAKE_ENDPOINT = Path("services/api/app/api/v1/endpoints/lake.py")
+JOB_MIGRATION = Path("services/api/migrations/111_lake_query_jobs.sql")
#: The floor the plan names. A deployment that cannot keep a year and a bit
#: cannot answer "show me this account's activity over the last audit
@@ -236,6 +252,122 @@ def evaluate(root: Path) -> list[str]:
f"that silently did nothing look identical in a log, and only one of them is working."
)
+ failures.extend(_evaluate_cold_query(root))
+ return failures
+
+
+def _evaluate_cold_query(root: Path) -> list[str]:
+ """The plan's third bullet: cost and progress instead of a silent timeout."""
+ failures: list[str] = []
+
+ cold_src = _read(root, COLD_QUERY_PY)
+ if not cold_src:
+ failures.append(
+ f"{COLD_QUERY_PY} is missing, so a query over the cold volume still either returns inside the "
+ f"synchronous bound or raises a bare timeout with no indication of how much data was involved."
+ )
+ return failures
+ cold = _strip_py_comments(cold_src)
+
+ # The estimate has to be the server's, not ours.
+ if "EXPLAIN ESTIMATE" not in cold:
+ failures.append(
+ f"{COLD_QUERY_PY} does not ask ClickHouse for an estimate. A row count this code derived itself is a "
+ f"guess presented as a measurement."
+ )
+ if "system.parts" not in cold:
+ failures.append(
+ f"{COLD_QUERY_PY} does not read system.parts, so nothing says which of the scan is on the cold volume "
+ f"- which is the half that costs."
+ )
+ if "system.processes" not in cold:
+ failures.append(
+ f"{COLD_QUERY_PY} does not read system.processes, so a running scan has no progress to report and the "
+ f"only honest alternative is deriving one from elapsed time, which over a cold read is proportional "
+ f"to nothing."
+ )
+
+ # No invented money. This is the rule the repository cares about most.
+ if "RETRIEVAL_PRICE_ENV" not in cold:
+ failures.append(
+ f"{COLD_QUERY_PY} declares no configurable retrieval price, so any money figure it reports is either "
+ f"invented or taken from the storage rate card, which prices residency rather than reads."
+ )
+ if "storage-cost-model.json" not in cold_src:
+ failures.append(
+ f"{COLD_QUERY_PY} does not say why docs/decisions/storage-cost-model.json is the wrong source for a "
+ f"read price. The next person to want a dollar figure will reach for it."
+ )
+
+ # Progress that admits it does not know.
+ if not re.search(r"total_rows_approx\s*<=\s*0", cold):
+ failures.append(
+ f"{COLD_QUERY_PY} does not guard the progress denominator. ClickHouse publishes total_rows_approx "
+ f"best-effort, so a ratio computed without the guard renders 100% whenever the server has not "
+ f"published a total."
+ )
+
+ runner_src = _read(root, JOB_RUNNER_PY)
+ if not runner_src:
+ failures.append(f"{JOB_RUNNER_PY} is missing, so there is no job to carry a progress state.")
+ else:
+ runner = _strip_py_comments(runner_src)
+ if 'LakeQueryJob.status == "queued"' not in runner:
+ failures.append(
+ f"{JOB_RUNNER_PY} claims a job without re-checking its status in the same statement. Two API "
+ f"replicas would then both start one cold scan, which is two reads of the cold volume for one "
+ f"analyst's question."
+ )
+ # Matched as the *pair* rather than as the status string alone.
+ # list_stuck_jobs legitimately selects on status == "running", so
+ # a rule looking for the bare string passes a runner whose writes
+ # have lost their guard entirely - measured, by removing it and
+ # watching the first version of this rule stay green.
+ if 'LakeQueryJob.id == job_id, LakeQueryJob.status == "running"' not in runner:
+ failures.append(
+ f"{JOB_RUNNER_PY} writes to a job without checking it is still running, so a scan that completes "
+ f"after an operator cancelled it reports success, and a late progress read resurrects a finished "
+ f"job."
+ )
+
+ endpoint = _strip_py_comments(_read(root, LAKE_ENDPOINT))
+ if not endpoint:
+ failures.append(f"{LAKE_ENDPOINT} is missing; there is no surface for any of this.")
+ else:
+ # The *branch*, not the field name. LakeEstimateResponse
+ # declares a touches_cold field, so a rule looking for the bare
+ # name passes an endpoint that reports the flag and ignores it -
+ # which is the regression that matters, because the response
+ # still looks right.
+ if not re.search(r"estimate\.touches_cold", endpoint):
+ failures.append(
+ f"{LAKE_ENDPOINT} never branches on the estimate, so a cold query still goes down the "
+ f"synchronous path and still times out silently - while the estimate field in the response "
+ f"makes it look handled."
+ )
+ if '"/query-jobs"' not in endpoint:
+ failures.append(f"{LAKE_ENDPOINT} exposes no job surface, so a long scan has nowhere to report progress to.")
+ if "/estimate" not in endpoint:
+ failures.append(
+ f"{LAKE_ENDPOINT} offers no way to ask what a query would read without running it, so the only "
+ f"way to find out the cost is to pay it."
+ )
+
+ if not (root / JOB_MIGRATION).is_file():
+ failures.append(
+ f"{JOB_MIGRATION} is missing, so lake_query_jobs exists in the ORM and in no database - and an "
+ f"in-process registry loses every job on a restart and 404s half of them on a second replica."
+ )
+ else:
+ migration = _read(root, JOB_MIGRATION)
+ if "ENABLE ROW LEVEL SECURITY" not in migration:
+ failures.append(f"{JOB_MIGRATION} creates lake_query_jobs with no row-level security.")
+ if "progress_ratio DOUBLE PRECISION" not in migration and "progress_ratio DOUBLE PRECISION" not in migration:
+ failures.append(
+ f"{JOB_MIGRATION} does not declare progress_ratio as a nullable double. NOT NULL would force a "
+ f"zero where the honest answer is unknown."
+ )
+
return failures
@@ -256,12 +388,30 @@ def _self_test() -> int:
("a purge that never reads the holds in force", "NO_HOLD_READ"),
("a purge that reads holds and never compares a row against them", "NO_DECISION"),
("a purge that cannot report what a hold withheld", "SILENT_HOLD"),
+ ("a cold-query estimator that is absent entirely", "NO_COLD_ESTIMATOR"),
+ ("an estimate this code derives instead of asking the planner", "OWN_ESTIMATE"),
+ ("a scan with no warm/cold split, so nothing says what costs", "NO_PARTS_READ"),
+ ("a progress ratio with no denominator guard", "UNGUARDED_RATIO"),
+ ("a job claim two replicas can both win", "RACY_CLAIM"),
+ ("a terminal state a late finish can overwrite", "OVERWRITABLE_FINISH"),
+ ("a cold query still routed down the synchronous path", "NO_ROUTING"),
+ ("jobs that exist in the ORM and in no database", "NO_JOB_TABLE"),
)
for label, kind in cases:
tmp = Path(tempfile.mkdtemp(prefix="aisoc-retention-selftest-"))
try:
- for rel in (INIT_SQL, TIERING_SQL, RETENTION_PY, WORKER_PY, LAKE_MIGRATIONS):
+ for rel in (
+ INIT_SQL,
+ TIERING_SQL,
+ RETENTION_PY,
+ WORKER_PY,
+ LAKE_MIGRATIONS,
+ COLD_QUERY_PY,
+ JOB_RUNNER_PY,
+ LAKE_ENDPOINT,
+ JOB_MIGRATION,
+ ):
src = REPO_ROOT / rel
if src.is_file():
(tmp / rel).parent.mkdir(parents=True, exist_ok=True)
@@ -303,6 +453,46 @@ def _self_test() -> int:
worker.write_text(worker.read_text(encoding="utf-8").replace("may_purge(", "_unused_decision("), encoding="utf-8")
elif kind == "SILENT_HOLD" and worker.is_file():
worker.write_text(re.sub(r"held", "skipped", worker.read_text(encoding="utf-8")), encoding="utf-8")
+ elif kind == "NO_COLD_ESTIMATOR":
+ (tmp / COLD_QUERY_PY).unlink(missing_ok=True)
+ elif kind == "OWN_ESTIMATE":
+ path = tmp / COLD_QUERY_PY
+ path.write_text(
+ path.read_text(encoding="utf-8").replace("EXPLAIN ESTIMATE", "SELECT count() FROM"),
+ encoding="utf-8",
+ )
+ elif kind == "NO_PARTS_READ":
+ path = tmp / COLD_QUERY_PY
+ path.write_text(path.read_text(encoding="utf-8").replace("system.parts", "system.tables"), encoding="utf-8")
+ elif kind == "UNGUARDED_RATIO":
+ path = tmp / COLD_QUERY_PY
+ path.write_text(
+ path.read_text(encoding="utf-8").replace("if self.total_rows_approx <= 0:", "if False:"),
+ encoding="utf-8",
+ )
+ elif kind == "RACY_CLAIM":
+ path = tmp / JOB_RUNNER_PY
+ path.write_text(
+ path.read_text(encoding="utf-8").replace(
+ 'LakeQueryJob.id == job_id, LakeQueryJob.status == "queued"',
+ "LakeQueryJob.id == job_id",
+ ),
+ encoding="utf-8",
+ )
+ elif kind == "OVERWRITABLE_FINISH":
+ path = tmp / JOB_RUNNER_PY
+ path.write_text(
+ path.read_text(encoding="utf-8").replace(
+ 'LakeQueryJob.id == job_id, LakeQueryJob.status == "running"',
+ "LakeQueryJob.id == job_id",
+ ),
+ encoding="utf-8",
+ )
+ elif kind == "NO_ROUTING":
+ path = tmp / LAKE_ENDPOINT
+ path.write_text(path.read_text(encoding="utf-8").replace("estimate.touches_cold", "False"), encoding="utf-8")
+ elif kind == "NO_JOB_TABLE":
+ (tmp / JOB_MIGRATION).unlink(missing_ok=True)
results.append((f"catches {label}", bool(evaluate(tmp))))
finally:
diff --git a/services/api/app/services/lake_query_runner.py b/services/api/app/services/lake_query_runner.py
index f917686e8..7fb878ec3 100644
--- a/services/api/app/services/lake_query_runner.py
+++ b/services/api/app/services/lake_query_runner.py
@@ -107,20 +107,34 @@ async def read_progress(query_id: str) -> QueryProgress | None:
)
-async def claim_job(session: AsyncSession, job_id: uuid.UUID) -> bool:
- """Move one job from ``queued`` to ``running``, atomically.
-
- The ``status == 'queued'`` predicate is the whole mechanism: a second
- replica running the same statement updates zero rows and learns that
- it lost. A SELECT-then-UPDATE would have both replicas read `queued`
- and both start the scan.
+def claim_statement(job_id: uuid.UUID):
+ """The one statement that claims a job.
+
+ Factored out so a test can run *this* statement on two interleaved
+ connections rather than a hand-written copy of it. A concurrency
+ proof against a re-typed statement proves something about the test.
+
+ The ``status == 'queued'`` predicate is the whole mechanism. Postgres
+ takes a row lock on the UPDATE, so a second connection running the
+ same statement blocks until the first commits and then re-evaluates
+ the predicate against the committed row — finding `running`, matching
+ nothing, and learning that it lost. A SELECT followed by an
+ unconditional UPDATE has no such re-evaluation: the second
+ connection's SELECT cannot see the first's uncommitted change, reads
+ `queued`, and both replicas start the scan.
"""
- result = await session.execute(
+ now = datetime.now(UTC)
+ return (
update(LakeQueryJob)
.where(LakeQueryJob.id == job_id, LakeQueryJob.status == "queued")
- .values(status="running", started_at=datetime.now(UTC), updated_at=datetime.now(UTC))
+ .values(status="running", started_at=now, updated_at=now)
.returning(LakeQueryJob.id)
)
+
+
+async def claim_job(session: AsyncSession, job_id: uuid.UUID) -> bool:
+ """Move one job from ``queued`` to ``running``, atomically."""
+ result = await session.execute(claim_statement(job_id))
claimed = result.scalar_one_or_none() is not None
await session.commit()
return claimed
diff --git a/services/api/tests/test_lake_cold_query.py b/services/api/tests/test_lake_cold_query.py
new file mode 100644
index 000000000..afa2155f3
--- /dev/null
+++ b/services/api/tests/test_lake_cold_query.py
@@ -0,0 +1,216 @@
+"""The cold-query estimate and its honesty rules (depth 4.3).
+
+Pure-function tests over ``app.services.lake_cold_query``. The parsing,
+the warm/cold split and the "no money figure without a price" rule are
+arithmetic and policy, and both are testable without a ClickHouse.
+
+The job lifecycle — where the claim must be atomic and a terminal state
+must never be left — is covered against real Postgres in
+``tests/isolation/test_lake_query_jobs_live.py``, because a mocked
+session answers whatever the test asked it to and would hide exactly
+that.
+"""
+
+from __future__ import annotations
+
+import pytest
+from app.services.lake_cold_query import (
+ COLD_VOLUME_NAMES,
+ RETRIEVAL_PRICE_ENV,
+ ColdEstimateUnavailable,
+ QueryProgress,
+ ScanEstimate,
+ cold_disk_names_from_policy,
+ parse_explain_estimate,
+ price_cold_bytes,
+ split_bytes_by_volume,
+ validate_query_id,
+)
+
+_GB = 1024**3
+
+
+# ---------------------------------------------------------------------------
+# EXPLAIN ESTIMATE
+# ---------------------------------------------------------------------------
+
+
+def test_the_planner_output_is_summed_by_column_name():
+ columns = ["database", "table", "parts", "rows", "marks"]
+ rows = [["aisoc", "raw_events", 12, 4_000_000, 500]]
+ total_rows, parts, tables = parse_explain_estimate(columns, rows)
+ assert (total_rows, parts, tables) == (4_000_000, 12, ["raw_events"])
+
+
+def test_a_reordered_column_set_is_read_by_name_not_position():
+ # parts and rows are both plausible integers, so a silent reorder
+ # would swap them and nobody would notice until an estimate said a
+ # 4-million-row scan touched 12 rows.
+ columns = ["database", "table", "rows", "parts", "marks"]
+ rows = [["aisoc", "raw_events", 4_000_000, 12, 500]]
+ total_rows, parts, _ = parse_explain_estimate(columns, rows)
+ assert total_rows == 4_000_000
+ assert parts == 12
+
+
+def test_several_tables_are_summed_and_listed_once():
+ columns = ["database", "table", "parts", "rows", "marks"]
+ rows = [
+ ["aisoc", "raw_events", 4, 100, 1],
+ ["aisoc", "raw_events", 2, 50, 1],
+ ["aisoc", "alerts_archive", 1, 25, 1],
+ ]
+ total_rows, parts, tables = parse_explain_estimate(columns, rows)
+ assert (total_rows, parts) == (175, 7)
+ assert tables == ["raw_events", "alerts_archive"]
+
+
+def test_an_unreadable_column_set_raises_rather_than_returning_zero():
+ # Zero and "could not estimate" are opposite answers, and returning
+ # the first for the second waves a 400-day cold scan through as free.
+ with pytest.raises(ColdEstimateUnavailable):
+ parse_explain_estimate(["something", "unexpected"], [["a", "b"]])
+
+
+def test_no_rows_is_an_empty_estimate_not_a_failure():
+ assert parse_explain_estimate(["database", "table", "parts", "rows", "marks"], []) == (0, 0, [])
+
+
+# ---------------------------------------------------------------------------
+# Where the data lives
+# ---------------------------------------------------------------------------
+
+
+def test_cold_disks_are_read_from_the_servers_storage_policy():
+ # The volume is named in a mounted XML file an operator owns, so the
+ # disk names cannot be hardcoded here.
+ disks = cold_disk_names_from_policy([["hot", ["nvme0"]], ["cold", ["s3_disk", "s3_backup"]]])
+ assert disks == frozenset({"s3_disk", "s3_backup"})
+
+
+def test_a_deployment_with_no_cold_volume_has_no_cold_disks():
+ assert cold_disk_names_from_policy([["hot", ["nvme0"]]]) == frozenset()
+
+
+def test_bytes_are_split_by_the_disk_the_part_is_on():
+ cold, warm = split_bytes_by_volume(
+ [["nvme0", 1_000], ["s3_disk", 9_000]],
+ cold_disk_names=frozenset({"s3_disk"}),
+ )
+ assert (cold, warm) == (9_000, 1_000)
+
+
+def test_an_unrecognised_disk_counts_as_warm():
+ # Deliberate direction. Over-reporting cold would route ordinary
+ # queries into the job path and make the feature feel broken;
+ # under-reporting sends a cold query down the synchronous path where
+ # it fails the way it always did — no worse than before, and visible.
+ cold, warm = split_bytes_by_volume([["some_new_disk", 5_000]], cold_disk_names=frozenset({"s3_disk"}))
+ assert (cold, warm) == (0, 5_000)
+
+
+def test_the_fallback_volume_name_matches_the_tiering_migration():
+ # services/api/clickhouse/tiering/002_tiering.sql moves to VOLUME 'cold'.
+ assert "cold" in COLD_VOLUME_NAMES
+
+
+# ---------------------------------------------------------------------------
+# Money. The rule this repository cares about most.
+# ---------------------------------------------------------------------------
+
+
+def test_no_configured_price_means_no_figure_and_a_reason(monkeypatch):
+ monkeypatch.delenv(RETRIEVAL_PRICE_ENV, raising=False)
+ usd, basis = price_cold_bytes(10 * _GB)
+ assert usd is None, "a figure was produced with no price configured"
+ assert "storage rate card" in basis
+ assert RETRIEVAL_PRICE_ENV in basis
+
+
+def test_a_configured_price_produces_a_figure_derived_from_it(monkeypatch):
+ monkeypatch.setenv(RETRIEVAL_PRICE_ENV, "0.01")
+ usd, basis = price_cold_bytes(10 * _GB)
+ assert usd == pytest.approx(0.10)
+ assert "0.01" in basis
+
+
+def test_a_nonsense_price_is_refused_rather_than_coerced(monkeypatch):
+ for bad in ("free", "-1"):
+ monkeypatch.setenv(RETRIEVAL_PRICE_ENV, bad)
+ usd, _ = price_cold_bytes(10 * _GB)
+ assert usd is None, f"{bad!r} produced a figure"
+
+
+def test_zero_cold_bytes_with_a_price_is_zero_not_none(monkeypatch):
+ # The distinction the whole design rests on: None means "we do not
+ # know what this costs", 0.0 means "this costs nothing".
+ monkeypatch.setenv(RETRIEVAL_PRICE_ENV, "0.01")
+ usd, _ = price_cold_bytes(0)
+ assert usd == 0.0
+
+
+# ---------------------------------------------------------------------------
+# The estimate object
+# ---------------------------------------------------------------------------
+
+
+def _estimate(**overrides) -> ScanEstimate:
+ base = {
+ "estimated_rows": 1_000,
+ "estimated_parts": 3,
+ "estimated_bytes": 2_000,
+ "cold_bytes": 0,
+ "warm_bytes": 2_000,
+ "tables": ["raw_events"],
+ }
+ base.update(overrides)
+ return ScanEstimate(**base)
+
+
+def test_touches_cold_is_what_routes_a_query_to_a_job():
+ assert _estimate(cold_bytes=0).touches_cold is False
+ assert _estimate(cold_bytes=1).touches_cold is True
+
+
+def test_the_serialised_estimate_carries_the_routing_field():
+ payload = _estimate(cold_bytes=5).as_dict()
+ assert payload["touches_cold"] is True
+ assert payload["cold_bytes"] == 5
+ assert "estimated_usd" in payload
+
+
+# ---------------------------------------------------------------------------
+# Progress
+# ---------------------------------------------------------------------------
+
+
+def test_progress_with_no_denominator_is_unknown_rather_than_complete():
+ # ClickHouse publishes total_rows_approx best-effort. A bar that
+ # invents 100% because the denominator was zero is worse than one
+ # that says it does not know.
+ assert QueryProgress(rows_read=500, bytes_read=1, total_rows_approx=0, elapsed_seconds=1.0).ratio is None
+
+
+def test_progress_is_clamped_when_the_estimate_is_overtaken():
+ # total_rows_approx moves while a query runs and can end up below
+ # rows_read, which would otherwise render as 140%.
+ progress = QueryProgress(rows_read=1_400, bytes_read=1, total_rows_approx=1_000, elapsed_seconds=1.0)
+ assert progress.ratio == 1.0
+
+
+def test_progress_serialises_the_unknown_ratio_as_null():
+ payload = QueryProgress(rows_read=1, bytes_read=1, total_rows_approx=0, elapsed_seconds=0.5).as_dict()
+ assert payload["ratio"] is None
+
+
+ "bad",
+ ["", "a b", "a;DROP", "x" * 65, "../etc", "id'quote"],
+)
+def test_a_query_id_outside_the_safe_alphabet_is_refused(bad):
+ with pytest.raises(ValueError):
+ validate_query_id(bad)
+
+
+def test_a_derived_query_id_is_accepted():
+ assert validate_query_id("aisoc-lake-11111111-1111-1111-1111-111111111111")
diff --git a/tests/isolation/test_lake_query_jobs_live.py b/tests/isolation/test_lake_query_jobs_live.py
new file mode 100644
index 000000000..d2d172a9f
--- /dev/null
+++ b/tests/isolation/test_lake_query_jobs_live.py
@@ -0,0 +1,321 @@
+"""Cold-tier lake query jobs against real Postgres (depth 4.3).
+
+The three properties worth proving here are all about concurrency and
+state, and a mocked session answers whatever the test asked it to:
+
+* **The claim is atomic.** Two API replicas must not both start the same
+ cold scan. The claim is one conditional UPDATE, and the only way to
+ show that works is to run two of them against one row.
+* **A terminal state is never left.** A late progress write or a late
+ finish must not resurrect a cancelled job, or rewrite a failure as a
+ success.
+* **A job id is tenant-scoped.** It is a handle a console holds, so the
+ predicate is in the query and row-level security is defence in depth.
+
+Skips when ``ISOLATION_LAKE_JOB_DSN`` is unset; the workflow that owns
+this suite sets it unconditionally, because a skip in a job whose whole
+purpose is to run live is a pass for the wrong reason.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import uuid
+from datetime import UTC, datetime, timedelta
+
+import pytest
+import pytest_asyncio
+
+pytestmark = [
+ pytest.mark.asyncio(loop_scope="module"),
+ pytest.mark.skipif(
+ not os.environ.get("ISOLATION_LAKE_JOB_DSN", "").strip(),
+ reason="ISOLATION_LAKE_JOB_DSN is not set; this suite needs a live Postgres",
+ ),
+]
+
+
+def _dsn() -> str:
+ value = os.environ.get("ISOLATION_LAKE_JOB_DSN", "").strip()
+ if not value:
+ pytest.skip("ISOLATION_LAKE_JOB_DSN is not set")
+ if value.startswith("postgresql://"):
+ return value.replace("postgresql://", "postgresql+asyncpg://", 1)
+ return value
+
+
+@pytest_asyncio.fixture(scope="module", loop_scope="module")
+async def engine():
+ from sqlalchemy.ext.asyncio import create_async_engine
+
+ eng = create_async_engine(_dsn())
+ try:
+ yield eng
+ finally:
+ await eng.dispose()
+
+
+@pytest_asyncio.fixture(scope="module", loop_scope="module")
+async def session_factory(engine):
+ from sqlalchemy.ext.asyncio import async_sessionmaker
+
+ return async_sessionmaker(engine, expire_on_commit=False)
+
+
+@pytest_asyncio.fixture(scope="module", loop_scope="module")
+async def tenants(engine):
+ from sqlalchemy import text
+
+ a, b = uuid.uuid4(), uuid.uuid4()
+ async with engine.begin() as conn:
+ for tid, slug in ((a, "lakejob-a"), (b, "lakejob-b")):
+ await conn.execute(
+ text("INSERT INTO tenants (id, name, slug) VALUES (:id, :n, :s)"),
+ {"id": str(tid), "n": slug, "s": f"{slug}-{uuid.uuid4().hex[:8]}"},
+ )
+ yield a, b
+ async with engine.begin() as conn:
+ for tid in (a, b):
+ await conn.execute(text("DELETE FROM tenants WHERE id = :id"), {"id": str(tid)})
+
+
+async def _make_job(session_factory, tenant_id: uuid.UUID, **overrides):
+ from app.models.lake_query_job import LakeQueryJob
+ from app.services.lake_query_runner import new_query_id
+
+ job_id = uuid.uuid4()
+ values = {
+ "id": job_id,
+ "tenant_id": tenant_id,
+ "submitted_by": uuid.uuid4(),
+ "submitted_email": "[email protected]",
+ "rewritten_sql": "SELECT 1",
+ "row_cap": 100,
+ "referenced_tables": ["raw_events"],
+ "query_id": new_query_id(job_id),
+ "estimate": {"cold_bytes": 1234, "estimated_usd": None},
+ }
+ values.update(overrides)
+ async with session_factory() as session:
+ job = LakeQueryJob(**values)
+ session.add(job)
+ await session.commit()
+ return job_id
+
+
+class TestTheClaimIsAtomic:
+ async def test_two_interleaved_transactions_produce_one_winner(self, engine, session_factory, tenants):
+ """Two API replicas must not both start one cold scan.
+
+ Deterministic rather than a race. The first attempt runs the
+ claim and **holds its transaction open**; the second then runs
+ the same statement and blocks on the row lock; the first commits;
+ the second unblocks and re-evaluates ``status = 'queued'``
+ against the committed row.
+
+ The interleaving is forced rather than hoped for because a plain
+ ``asyncio.gather`` of five claims passes under a
+ SELECT-then-UPDATE implementation as well — measured, by
+ subverting the claim and watching that version stay green. A
+ concurrency test that cannot fail on the broken implementation
+ is not a concurrency test.
+
+ It runs ``claim_statement`` itself, not a re-typed copy: a proof
+ against a hand-written statement proves something about the
+ test.
+ """
+ from app.services.lake_query_runner import claim_statement
+
+ tenant_a, _ = tenants
+ job_id = await _make_job(session_factory, tenant_a)
+
+ first_claimed: list[bool] = []
+ second_claimed: list[bool] = []
+ first_has_run = asyncio.Event()
+
+ async def first() -> None:
+ async with engine.connect() as conn:
+ await conn.begin()
+ result = await conn.execute(claim_statement(job_id))
+ first_claimed.append(result.scalar_one_or_none() is not None)
+ first_has_run.set()
+ # Hold the lock long enough for the second attempt to
+ # reach the same row and block on it.
+ await asyncio.sleep(0.5)
+ await conn.commit()
+
+ async def second() -> None:
+ await first_has_run.wait()
+ await asyncio.sleep(0.1)
+ async with engine.connect() as conn:
+ await conn.begin()
+ result = await conn.execute(claim_statement(job_id))
+ second_claimed.append(result.scalar_one_or_none() is not None)
+ await conn.commit()
+
+ await asyncio.gather(first(), second())
+
+ assert first_claimed == [True], "the first attempt did not claim the job"
+ assert second_claimed == [False], "both transactions claimed the same job, so two replicas would both run the scan"
+
+ async def test_a_job_that_is_not_queued_cannot_be_claimed(self, session_factory, tenants):
+ from app.services.lake_query_runner import claim_job
+
+ tenant_a, _ = tenants
+ job_id = await _make_job(session_factory, tenant_a, status="succeeded")
+ async with session_factory() as session:
+ assert await claim_job(session, job_id) is False
+
+ async def test_claiming_records_when_it_started(self, session_factory, tenants):
+ from app.models.lake_query_job import LakeQueryJob
+ from app.services.lake_query_runner import claim_job
+
+ tenant_a, _ = tenants
+ job_id = await _make_job(session_factory, tenant_a)
+ async with session_factory() as session:
+ assert await claim_job(session, job_id) is True
+ async with session_factory() as session:
+ job = await session.get(LakeQueryJob, job_id)
+ assert job.status == "running"
+ assert job.started_at is not None
+
+
+class TestATerminalStateIsNeverLeft:
+ async def test_a_cancelled_job_is_not_overwritten_by_a_late_finish(self, session_factory, tenants):
+ """The predicate on ``_finish`` is the whole mechanism: a scan
+ that completes after an operator cancelled it must not report
+ success."""
+ from app.models.lake_query_job import LakeQueryJob
+ from app.services.lake_query_runner import _finish, claim_job
+
+ tenant_a, _ = tenants
+ job_id = await _make_job(session_factory, tenant_a)
+ async with session_factory() as session:
+ await claim_job(session, job_id)
+ async with session_factory() as session:
+ job = await session.get(LakeQueryJob, job_id)
+ job.status = "cancelled"
+ await session.commit()
+
+ await _finish(session_factory, job_id, status="succeeded", row_count=9)
+
+ async with session_factory() as session:
+ job = await session.get(LakeQueryJob, job_id)
+ assert job.status == "cancelled"
+ assert job.row_count is None
+
+ async def test_cancelling_an_ended_job_is_refused(self, session_factory, tenants):
+ from app.models.lake_query_job import LakeQueryJob
+ from app.services.lake_query_runner import cancel_job
+
+ tenant_a, _ = tenants
+ job_id = await _make_job(session_factory, tenant_a, status="succeeded")
+ async with session_factory() as session:
+ job = await session.get(LakeQueryJob, job_id)
+ assert await cancel_job(session, job) is False
+
+ async def test_progress_is_not_written_onto_a_finished_job(self, session_factory, tenants):
+ from app.models.lake_query_job import LakeQueryJob
+ from app.services.lake_cold_query import QueryProgress
+ from app.services.lake_query_runner import _record_progress
+
+ tenant_a, _ = tenants
+ job_id = await _make_job(session_factory, tenant_a, status="succeeded")
+ await _record_progress(
+ session_factory,
+ job_id,
+ QueryProgress(rows_read=42, bytes_read=1, total_rows_approx=100, elapsed_seconds=1.0),
+ )
+ async with session_factory() as session:
+ job = await session.get(LakeQueryJob, job_id)
+ assert job.rows_read == 0, "a late progress read resurrected a finished job"
+
+
+class TestProgressIsHonest:
+ async def test_an_unknown_denominator_stores_null_not_zero(self, session_factory, tenants):
+ from app.models.lake_query_job import LakeQueryJob
+ from app.services.lake_cold_query import QueryProgress
+ from app.services.lake_query_runner import _record_progress, claim_job
+
+ tenant_a, _ = tenants
+ job_id = await _make_job(session_factory, tenant_a)
+ async with session_factory() as session:
+ await claim_job(session, job_id)
+
+ await _record_progress(
+ session_factory,
+ job_id,
+ QueryProgress(rows_read=500, bytes_read=10, total_rows_approx=0, elapsed_seconds=2.0),
+ )
+ async with session_factory() as session:
+ job = await session.get(LakeQueryJob, job_id)
+ assert job.rows_read == 500
+ assert job.progress_ratio is None, "unknown progress was stored as a number"
+
+ async def test_a_known_denominator_stores_a_ratio(self, session_factory, tenants):
+ from app.models.lake_query_job import LakeQueryJob
+ from app.services.lake_cold_query import QueryProgress
+ from app.services.lake_query_runner import _record_progress, claim_job
+
+ tenant_a, _ = tenants
+ job_id = await _make_job(session_factory, tenant_a)
+ async with session_factory() as session:
+ await claim_job(session, job_id)
+ await _record_progress(
+ session_factory,
+ job_id,
+ QueryProgress(rows_read=250, bytes_read=10, total_rows_approx=1000, elapsed_seconds=2.0),
+ )
+ async with session_factory() as session:
+ job = await session.get(LakeQueryJob, job_id)
+ assert job.progress_ratio == pytest.approx(0.25)
+
+
+class TestTenantScoping:
+ async def test_another_tenants_job_is_not_found(self, session_factory, tenants):
+ from app.api.v1.endpoints.lake import _load_job
+ from fastapi import HTTPException
+
+ tenant_a, tenant_b = tenants
+ job_id = await _make_job(session_factory, tenant_b)
+ async with session_factory() as session:
+ with pytest.raises(HTTPException) as exc:
+ await _load_job(session, tenant_a, job_id)
+ assert exc.value.status_code == 404
+
+ async def test_the_callers_own_job_is_found(self, session_factory, tenants):
+ from app.api.v1.endpoints.lake import _load_job
+
+ tenant_a, _ = tenants
+ job_id = await _make_job(session_factory, tenant_a)
+ async with session_factory() as session:
+ job = await _load_job(session, tenant_a, job_id)
+ assert job.id == job_id
+
+
+class TestStuckJobsAreVisible:
+ async def test_a_job_whose_runner_died_is_reported(self, session_factory, tenants):
+ """A job stuck at ``running`` with no task behind it reads to an
+ analyst as a query that is still working."""
+ from app.services.lake_query_runner import list_stuck_jobs
+
+ tenant_a, _ = tenants
+ job_id = await _make_job(
+ session_factory,
+ tenant_a,
+ status="running",
+ started_at=datetime.now(UTC) - timedelta(hours=2),
+ )
+ async with session_factory() as session:
+ stuck = await list_stuck_jobs(session, older_than_seconds=900)
+ assert job_id in {job.id for job in stuck}
+
+ async def test_a_job_that_started_recently_is_not_reported(self, session_factory, tenants):
+ from app.services.lake_query_runner import list_stuck_jobs
+
+ tenant_a, _ = tenants
+ job_id = await _make_job(session_factory, tenant_a, status="running", started_at=datetime.now(UTC))
+ async with session_factory() as session:
+ stuck = await list_stuck_jobs(session, older_than_seconds=900)
+ assert job_id not in {job.id for job in stuck}