Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
test(4.3): 38 cases, a deterministic claim proof, and the test it replaced
Beenu Aroraae0d8a1e2514
diff --git a/.github/workflows/lake-query-jobs-live.yml b/.github/workflows/lake-query-jobs-live.ymlnew file mode 100644index 000000000..d0efbdde8--- /dev/null+++ b/.github/workflows/lake-query-jobs-live.yml@@ -0,0 +1,157 @@+# Cold-tier lake query jobs, against real Postgres (depth plan 4.3).+#+# The properties here are about concurrency and state, and a mocked+# session answers whatever the test asked it to. Two API replicas must not+# both start one cold scan; a terminal state must never be left; a job id+# must be tenant-scoped.+#+# The claim test is deliberately *deterministic* rather than a race. An+# `asyncio.gather` of five concurrent claims passes under a+# SELECT-then-UPDATE implementation as well — measured, by subverting the+# claim and watching that version stay green — so the two transactions are+# interleaved explicitly and the first holds its lock while the second+# reaches the same row.+#+# No `paths:` filter: a cross-tenant read guard must grade every pull+# request.+name: Lake query jobs (live)++on:+ push:+ branches: [main]+ pull_request:+ workflow_dispatch:++concurrency:+ group: lake-query-jobs-live-${{ github.ref }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.sha }}+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}++permissions:+ contents: read++jobs:+ lake-query-jobs-live:+ name: One replica claims a cold scan, and a finished job stays finished+ runs-on: ubuntu-latest+ timeout-minutes: 20++ services:+ postgres:+ image: public.ecr.aws/docker/library/postgres:16-alpine+ env:+ POSTGRES_USER: aisoc+ POSTGRES_PASSWORD: aisoc_ci_only+ POSTGRES_DB: aisoc+ ports: ["5432:5432"]+ options: >-+ --health-cmd "pg_isready -U aisoc"+ --health-interval 5s+ --health-timeout 5s+ --health-retries 20++ env:+ LAKE_JOB_DSN: postgresql+asyncpg://aisoc:aisoc_ci_only@localhost:5432/aisoc+ PGPASSWORD: aisoc_ci_only+ ENVIRONMENT: test+ SECRET_KEY: ci-lake-job-signing-secret-at-least-32-chars+ OTEL_SDK_DISABLED: "true"++ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7++ - uses: actions/setup-python@e9d6f990972a57673cdb72ec29e19d42ba28880f # v6+ with:+ python-version: "3.11"++ - name: Install api from its lockfile+ working-directory: services/api+ run: |+ set -euo pipefail+ python -m pip install --quiet --upgrade pip "poetry==2.4.1"+ poetry config virtualenvs.create false+ poetry install --no-root --no-interaction+ python -m pip install --quiet pytest pytest-asyncio++ - name: Apply every migration, in order+ run: |+ set -euo pipefail+ for f in services/api/migrations/*.sql; do+ psql -h localhost -U aisoc -d aisoc -v ON_ERROR_STOP=1 -q -f "$f"+ done++ # progress_ratio must be nullable. NOT NULL would force a zero where+ # the honest answer is unknown, and a progress bar that renders a+ # forced zero as 0% is as wrong as one that renders it as 100%.+ - name: progress_ratio is nullable+ run: |+ set -euo pipefail+ nullable=$(psql -h localhost -U aisoc -d aisoc -tAc \+ "SELECT is_nullable FROM information_schema.columns+ WHERE table_name = 'lake_query_jobs' AND column_name = 'progress_ratio'")+ if [ "$nullable" != "YES" ]; then+ echo "FAIL: progress_ratio is '$nullable'; unknown progress has nowhere to go but a zero."+ exit 1+ fi+ echo "OK: progress_ratio is nullable."++ - name: The claim, the terminal states and the tenant predicate+ working-directory: services/api+ env:+ ISOLATION_LAKE_JOB_DSN: ${{ env.LAKE_JOB_DSN }}+ DATABASE_URL: ${{ env.LAKE_JOB_DSN }}+ run: python -m pytest ../../tests/isolation/test_lake_query_jobs_live.py -v --tb=short++ - name: The suite actually ran+ working-directory: services/api+ env:+ ISOLATION_LAKE_JOB_DSN: ${{ env.LAKE_JOB_DSN }}+ DATABASE_URL: ${{ env.LAKE_JOB_DSN }}+ run: |+ set -euo pipefail+ python -m pytest ../../tests/isolation/test_lake_query_jobs_live.py -q 2>&1 | tee /tmp/lakejob-run.txt+ if grep -qE '[0-9]+ skipped' /tmp/lakejob-run.txt; then+ echo "FAIL: a case skipped in the job whose whole purpose is to run it live."+ exit 1+ fi+ if ! grep -qE '[0-9]+ passed' /tmp/lakejob-run.txt; then+ echo "FAIL: no case reported as passed."+ exit 1+ fi++ # The negative control.+ #+ # Remove the `status = 'queued'` predicate from the claim. Postgres+ # then has nothing to re-evaluate after the row lock is released, so+ # both interleaved transactions claim the same job and two replicas+ # would both run the scan. The suite must go red.+ - name: The gate fails when the claim stops re-checking the status+ working-directory: services/api+ env:+ ISOLATION_LAKE_JOB_DSN: ${{ env.LAKE_JOB_DSN }}+ DATABASE_URL: ${{ env.LAKE_JOB_DSN }}+ run: |+ set -euo pipefail+ cp app/services/lake_query_runner.py /tmp/runner.bak+ python - <<'PY'+ import pathlib, sys+ p = pathlib.Path("app/services/lake_query_runner.py")+ s = p.read_text()+ needle = ' .where(LakeQueryJob.id == job_id, LakeQueryJob.status == "queued")\n'+ if needle not in s:+ print("negative control could not find the claim predicate; the claim has been "+ "reshaped and this control no longer proves anything")+ sys.exit(1)+ p.write_text(s.replace(needle, " .where(LakeQueryJob.id == job_id)\n", 1))+ PY+ set +e+ python -m pytest \+ "../../tests/isolation/test_lake_query_jobs_live.py::TestTheClaimIsAtomic::test_two_interleaved_transactions_produce_one_winner" \+ -q >/dev/null 2>&1+ rc=$?+ set -e+ cp /tmp/runner.bak app/services/lake_query_runner.py+ if [ "$rc" -eq 0 ]; then+ echo "FAIL: both transactions claimed the job and the suite stayed green."+ exit 1+ fi+ echo "OK: without the predicate both transactions claim the job and the suite goes red, as it must."diff --git a/scripts/check_retention_window.py b/scripts/check_retention_window.pyindex f9e4b50c5..d5df41268 100644--- a/scripts/check_retention_window.py+++ b/scripts/check_retention_window.py@@ -23,6 +23,18 @@ system. The purge worker must read holds and must skip what they cover.purge and a purge that silently did nothing look the same in a log. The runsummary has to carry the count it withheld.+**A cold read with no cost and no progress.** The plan's third bullet. A+query over the cold volume either returned inside the published+30-second synchronous bound or raised a bare timeout, which tells an+analyst nothing about whether a longer deadline would have helped or how+much data was involved — and the ``cost`` ``/lake/sql`` did report was+the rate limiter's token price, a number about how often you may ask.+The estimate must come from ClickHouse's own planner and parts, the+money figure must be absent rather than invented when no retrieval price+is configured, the job must be claimed with a predicate that two+replicas cannot both satisfy, and a progress ratio with no denominator+must be null rather than zero.+These are read out of the source rather than exercised, because the lakeTTL lives in DDL and the worker needs Postgres and ClickHouse. Thebehavioural half is `services/api/tests/test_retention_legal_hold.py`,@@ -56,6 +68,10 @@ TIERING_SQL = Path("services/api/clickhouse/tiering/002_tiering.sql")RETENTION_PY = Path("services/api/app/services/retention.py")WORKER_PY = Path("services/api/app/workers/retention_purge.py")LAKE_MIGRATIONS = Path("services/api/app/db/lake_migrations.py")+COLD_QUERY_PY = Path("services/api/app/services/lake_cold_query.py")+JOB_RUNNER_PY = Path("services/api/app/services/lake_query_runner.py")+LAKE_ENDPOINT = Path("services/api/app/api/v1/endpoints/lake.py")+JOB_MIGRATION = Path("services/api/migrations/111_lake_query_jobs.sql")#: The floor the plan names. A deployment that cannot keep a year and a bit#: cannot answer "show me this account's activity over the last audit@@ -236,6 +252,122 @@ def evaluate(root: Path) -> list[str]:f"that silently did nothing look identical in a log, and only one of them is working.")+ failures.extend(_evaluate_cold_query(root))+ return failures+++def _evaluate_cold_query(root: Path) -> list[str]:+ """The plan's third bullet: cost and progress instead of a silent timeout."""+ failures: list[str] = []++ cold_src = _read(root, COLD_QUERY_PY)+ if not cold_src:+ failures.append(+ f"{COLD_QUERY_PY} is missing, so a query over the cold volume still either returns inside the "+ f"synchronous bound or raises a bare timeout with no indication of how much data was involved."+ )+ return failures+ cold = _strip_py_comments(cold_src)++ # The estimate has to be the server's, not ours.+ if "EXPLAIN ESTIMATE" not in cold:+ failures.append(+ f"{COLD_QUERY_PY} does not ask ClickHouse for an estimate. A row count this code derived itself is a "+ f"guess presented as a measurement."+ )+ if "system.parts" not in cold:+ failures.append(+ f"{COLD_QUERY_PY} does not read system.parts, so nothing says which of the scan is on the cold volume "+ f"- which is the half that costs."+ )+ if "system.processes" not in cold:+ failures.append(+ f"{COLD_QUERY_PY} does not read system.processes, so a running scan has no progress to report and the "+ f"only honest alternative is deriving one from elapsed time, which over a cold read is proportional "+ f"to nothing."+ )++ # No invented money. This is the rule the repository cares about most.+ if "RETRIEVAL_PRICE_ENV" not in cold:+ failures.append(+ f"{COLD_QUERY_PY} declares no configurable retrieval price, so any money figure it reports is either "+ f"invented or taken from the storage rate card, which prices residency rather than reads."+ )+ if "storage-cost-model.json" not in cold_src:+ failures.append(+ f"{COLD_QUERY_PY} does not say why docs/decisions/storage-cost-model.json is the wrong source for a "+ f"read price. The next person to want a dollar figure will reach for it."+ )++ # Progress that admits it does not know.+ if not re.search(r"total_rows_approx\s*<=\s*0", cold):+ failures.append(+ f"{COLD_QUERY_PY} does not guard the progress denominator. ClickHouse publishes total_rows_approx "+ f"best-effort, so a ratio computed without the guard renders 100% whenever the server has not "+ f"published a total."+ )++ runner_src = _read(root, JOB_RUNNER_PY)+ if not runner_src:+ failures.append(f"{JOB_RUNNER_PY} is missing, so there is no job to carry a progress state.")+ else:+ runner = _strip_py_comments(runner_src)+ if 'LakeQueryJob.status == "queued"' not in runner:+ failures.append(+ f"{JOB_RUNNER_PY} claims a job without re-checking its status in the same statement. Two API "+ f"replicas would then both start one cold scan, which is two reads of the cold volume for one "+ f"analyst's question."+ )+ # Matched as the *pair* rather than as the status string alone.+ # list_stuck_jobs legitimately selects on status == "running", so+ # a rule looking for the bare string passes a runner whose writes+ # have lost their guard entirely - measured, by removing it and+ # watching the first version of this rule stay green.+ if 'LakeQueryJob.id == job_id, LakeQueryJob.status == "running"' not in runner:+ failures.append(+ f"{JOB_RUNNER_PY} writes to a job without checking it is still running, so a scan that completes "+ f"after an operator cancelled it reports success, and a late progress read resurrects a finished "+ f"job."+ )++ endpoint = _strip_py_comments(_read(root, LAKE_ENDPOINT))+ if not endpoint:+ failures.append(f"{LAKE_ENDPOINT} is missing; there is no surface for any of this.")+ else:+ # The *branch*, not the field name. LakeEstimateResponse+ # declares a touches_cold field, so a rule looking for the bare+ # name passes an endpoint that reports the flag and ignores it -+ # which is the regression that matters, because the response+ # still looks right.+ if not re.search(r"estimate\.touches_cold", endpoint):+ failures.append(+ f"{LAKE_ENDPOINT} never branches on the estimate, so a cold query still goes down the "+ f"synchronous path and still times out silently - while the estimate field in the response "+ f"makes it look handled."+ )+ if '"/query-jobs"' not in endpoint:+ failures.append(f"{LAKE_ENDPOINT} exposes no job surface, so a long scan has nowhere to report progress to.")+ if "/estimate" not in endpoint:+ failures.append(+ f"{LAKE_ENDPOINT} offers no way to ask what a query would read without running it, so the only "+ f"way to find out the cost is to pay it."+ )++ if not (root / JOB_MIGRATION).is_file():+ failures.append(+ f"{JOB_MIGRATION} is missing, so lake_query_jobs exists in the ORM and in no database - and an "+ f"in-process registry loses every job on a restart and 404s half of them on a second replica."+ )+ else:+ migration = _read(root, JOB_MIGRATION)+ if "ENABLE ROW LEVEL SECURITY" not in migration:+ failures.append(f"{JOB_MIGRATION} creates lake_query_jobs with no row-level security.")+ if "progress_ratio DOUBLE PRECISION" not in migration and "progress_ratio DOUBLE PRECISION" not in migration:+ failures.append(+ f"{JOB_MIGRATION} does not declare progress_ratio as a nullable double. NOT NULL would force a "+ f"zero where the honest answer is unknown."+ )+return failures@@ -256,12 +388,30 @@ def _self_test() -> int:("a purge that never reads the holds in force", "NO_HOLD_READ"),("a purge that reads holds and never compares a row against them", "NO_DECISION"),("a purge that cannot report what a hold withheld", "SILENT_HOLD"),+ ("a cold-query estimator that is absent entirely", "NO_COLD_ESTIMATOR"),+ ("an estimate this code derives instead of asking the planner", "OWN_ESTIMATE"),+ ("a scan with no warm/cold split, so nothing says what costs", "NO_PARTS_READ"),+ ("a progress ratio with no denominator guard", "UNGUARDED_RATIO"),+ ("a job claim two replicas can both win", "RACY_CLAIM"),+ ("a terminal state a late finish can overwrite", "OVERWRITABLE_FINISH"),+ ("a cold query still routed down the synchronous path", "NO_ROUTING"),+ ("jobs that exist in the ORM and in no database", "NO_JOB_TABLE"),)for label, kind in cases:tmp = Path(tempfile.mkdtemp(prefix="aisoc-retention-selftest-"))try:- for rel in (INIT_SQL, TIERING_SQL, RETENTION_PY, WORKER_PY, LAKE_MIGRATIONS):+ for rel in (+ INIT_SQL,+ TIERING_SQL,+ RETENTION_PY,+ WORKER_PY,+ LAKE_MIGRATIONS,+ COLD_QUERY_PY,+ JOB_RUNNER_PY,+ LAKE_ENDPOINT,+ JOB_MIGRATION,+ ):src = REPO_ROOT / relif src.is_file():(tmp / rel).parent.mkdir(parents=True, exist_ok=True)@@ -303,6 +453,46 @@ def _self_test() -> int:worker.write_text(worker.read_text(encoding="utf-8").replace("may_purge(", "_unused_decision("), encoding="utf-8")elif kind == "SILENT_HOLD" and worker.is_file():worker.write_text(re.sub(r"held", "skipped", worker.read_text(encoding="utf-8")), encoding="utf-8")+ elif kind == "NO_COLD_ESTIMATOR":+ (tmp / COLD_QUERY_PY).unlink(missing_ok=True)+ elif kind == "OWN_ESTIMATE":+ path = tmp / COLD_QUERY_PY+ path.write_text(+ path.read_text(encoding="utf-8").replace("EXPLAIN ESTIMATE", "SELECT count() FROM"),+ encoding="utf-8",+ )+ elif kind == "NO_PARTS_READ":+ path = tmp / COLD_QUERY_PY+ path.write_text(path.read_text(encoding="utf-8").replace("system.parts", "system.tables"), encoding="utf-8")+ elif kind == "UNGUARDED_RATIO":+ path = tmp / COLD_QUERY_PY+ path.write_text(+ path.read_text(encoding="utf-8").replace("if self.total_rows_approx <= 0:", "if False:"),+ encoding="utf-8",+ )+ elif kind == "RACY_CLAIM":+ path = tmp / JOB_RUNNER_PY+ path.write_text(+ path.read_text(encoding="utf-8").replace(+ 'LakeQueryJob.id == job_id, LakeQueryJob.status == "queued"',+ "LakeQueryJob.id == job_id",+ ),+ encoding="utf-8",+ )+ elif kind == "OVERWRITABLE_FINISH":+ path = tmp / JOB_RUNNER_PY+ path.write_text(+ path.read_text(encoding="utf-8").replace(+ 'LakeQueryJob.id == job_id, LakeQueryJob.status == "running"',+ "LakeQueryJob.id == job_id",+ ),+ encoding="utf-8",+ )+ elif kind == "NO_ROUTING":+ path = tmp / LAKE_ENDPOINT+ path.write_text(path.read_text(encoding="utf-8").replace("estimate.touches_cold", "False"), encoding="utf-8")+ elif kind == "NO_JOB_TABLE":+ (tmp / JOB_MIGRATION).unlink(missing_ok=True)results.append((f"catches {label}", bool(evaluate(tmp))))finally:diff --git a/services/api/app/services/lake_query_runner.py b/services/api/app/services/lake_query_runner.pyindex f917686e8..7fb878ec3 100644--- a/services/api/app/services/lake_query_runner.py+++ b/services/api/app/services/lake_query_runner.py@@ -107,20 +107,34 @@ async def read_progress(query_id: str) -> QueryProgress | None:)-async def claim_job(session: AsyncSession, job_id: uuid.UUID) -> bool:- """Move one job from ``queued`` to ``running``, atomically.-- The ``status == 'queued'`` predicate is the whole mechanism: a second- replica running the same statement updates zero rows and learns that- it lost. A SELECT-then-UPDATE would have both replicas read `queued`- and both start the scan.+def claim_statement(job_id: uuid.UUID):+ """The one statement that claims a job.++ Factored out so a test can run *this* statement on two interleaved+ connections rather than a hand-written copy of it. A concurrency+ proof against a re-typed statement proves something about the test.++ The ``status == 'queued'`` predicate is the whole mechanism. Postgres+ takes a row lock on the UPDATE, so a second connection running the+ same statement blocks until the first commits and then re-evaluates+ the predicate against the committed row — finding `running`, matching+ nothing, and learning that it lost. A SELECT followed by an+ unconditional UPDATE has no such re-evaluation: the second+ connection's SELECT cannot see the first's uncommitted change, reads+ `queued`, and both replicas start the scan."""- result = await session.execute(+ now = datetime.now(UTC)+ return (update(LakeQueryJob).where(LakeQueryJob.id == job_id, LakeQueryJob.status == "queued")- .values(status="running", started_at=datetime.now(UTC), updated_at=datetime.now(UTC))+ .values(status="running", started_at=now, updated_at=now).returning(LakeQueryJob.id))+++async def claim_job(session: AsyncSession, job_id: uuid.UUID) -> bool:+ """Move one job from ``queued`` to ``running``, atomically."""+ result = await session.execute(claim_statement(job_id))claimed = result.scalar_one_or_none() is not Noneawait session.commit()return claimeddiff --git a/services/api/tests/test_lake_cold_query.py b/services/api/tests/test_lake_cold_query.pynew file mode 100644index 000000000..afa2155f3--- /dev/null+++ b/services/api/tests/test_lake_cold_query.py@@ -0,0 +1,216 @@+"""The cold-query estimate and its honesty rules (depth 4.3).++Pure-function tests over ``app.services.lake_cold_query``. The parsing,+the warm/cold split and the "no money figure without a price" rule are+arithmetic and policy, and both are testable without a ClickHouse.++The job lifecycle — where the claim must be atomic and a terminal state+must never be left — is covered against real Postgres in+``tests/isolation/test_lake_query_jobs_live.py``, because a mocked+session answers whatever the test asked it to and would hide exactly+that.+"""++from __future__ import annotations++import pytest+from app.services.lake_cold_query import (+ COLD_VOLUME_NAMES,+ RETRIEVAL_PRICE_ENV,+ ColdEstimateUnavailable,+ QueryProgress,+ ScanEstimate,+ cold_disk_names_from_policy,+ parse_explain_estimate,+ price_cold_bytes,+ split_bytes_by_volume,+ validate_query_id,+)++_GB = 1024**3+++# ---------------------------------------------------------------------------+# EXPLAIN ESTIMATE+# ---------------------------------------------------------------------------+++def test_the_planner_output_is_summed_by_column_name():+ columns = ["database", "table", "parts", "rows", "marks"]+ rows = [["aisoc", "raw_events", 12, 4_000_000, 500]]+ total_rows, parts, tables = parse_explain_estimate(columns, rows)+ assert (total_rows, parts, tables) == (4_000_000, 12, ["raw_events"])+++def test_a_reordered_column_set_is_read_by_name_not_position():+ # parts and rows are both plausible integers, so a silent reorder+ # would swap them and nobody would notice until an estimate said a+ # 4-million-row scan touched 12 rows.+ columns = ["database", "table", "rows", "parts", "marks"]+ rows = [["aisoc", "raw_events", 4_000_000, 12, 500]]+ total_rows, parts, _ = parse_explain_estimate(columns, rows)+ assert total_rows == 4_000_000+ assert parts == 12+++def test_several_tables_are_summed_and_listed_once():+ columns = ["database", "table", "parts", "rows", "marks"]+ rows = [+ ["aisoc", "raw_events", 4, 100, 1],+ ["aisoc", "raw_events", 2, 50, 1],+ ["aisoc", "alerts_archive", 1, 25, 1],+ ]+ total_rows, parts, tables = parse_explain_estimate(columns, rows)+ assert (total_rows, parts) == (175, 7)+ assert tables == ["raw_events", "alerts_archive"]+++def test_an_unreadable_column_set_raises_rather_than_returning_zero():+ # Zero and "could not estimate" are opposite answers, and returning+ # the first for the second waves a 400-day cold scan through as free.+ with pytest.raises(ColdEstimateUnavailable):+ parse_explain_estimate(["something", "unexpected"], [["a", "b"]])+++def test_no_rows_is_an_empty_estimate_not_a_failure():+ assert parse_explain_estimate(["database", "table", "parts", "rows", "marks"], []) == (0, 0, [])+++# ---------------------------------------------------------------------------+# Where the data lives+# ---------------------------------------------------------------------------+++def test_cold_disks_are_read_from_the_servers_storage_policy():+ # The volume is named in a mounted XML file an operator owns, so the+ # disk names cannot be hardcoded here.+ disks = cold_disk_names_from_policy([["hot", ["nvme0"]], ["cold", ["s3_disk", "s3_backup"]]])+ assert disks == frozenset({"s3_disk", "s3_backup"})+++def test_a_deployment_with_no_cold_volume_has_no_cold_disks():+ assert cold_disk_names_from_policy([["hot", ["nvme0"]]]) == frozenset()+++def test_bytes_are_split_by_the_disk_the_part_is_on():+ cold, warm = split_bytes_by_volume(+ [["nvme0", 1_000], ["s3_disk", 9_000]],+ cold_disk_names=frozenset({"s3_disk"}),+ )+ assert (cold, warm) == (9_000, 1_000)+++def test_an_unrecognised_disk_counts_as_warm():+ # Deliberate direction. Over-reporting cold would route ordinary+ # queries into the job path and make the feature feel broken;+ # under-reporting sends a cold query down the synchronous path where+ # it fails the way it always did — no worse than before, and visible.+ cold, warm = split_bytes_by_volume([["some_new_disk", 5_000]], cold_disk_names=frozenset({"s3_disk"}))+ assert (cold, warm) == (0, 5_000)+++def test_the_fallback_volume_name_matches_the_tiering_migration():+ # services/api/clickhouse/tiering/002_tiering.sql moves to VOLUME 'cold'.+ assert "cold" in COLD_VOLUME_NAMES+++# ---------------------------------------------------------------------------+# Money. The rule this repository cares about most.+# ---------------------------------------------------------------------------+++def test_no_configured_price_means_no_figure_and_a_reason(monkeypatch):+ monkeypatch.delenv(RETRIEVAL_PRICE_ENV, raising=False)+ usd, basis = price_cold_bytes(10 * _GB)+ assert usd is None, "a figure was produced with no price configured"+ assert "storage rate card" in basis+ assert RETRIEVAL_PRICE_ENV in basis+++def test_a_configured_price_produces_a_figure_derived_from_it(monkeypatch):+ monkeypatch.setenv(RETRIEVAL_PRICE_ENV, "0.01")+ usd, basis = price_cold_bytes(10 * _GB)+ assert usd == pytest.approx(0.10)+ assert "0.01" in basis+++def test_a_nonsense_price_is_refused_rather_than_coerced(monkeypatch):+ for bad in ("free", "-1"):+ monkeypatch.setenv(RETRIEVAL_PRICE_ENV, bad)+ usd, _ = price_cold_bytes(10 * _GB)+ assert usd is None, f"{bad!r} produced a figure"+++def test_zero_cold_bytes_with_a_price_is_zero_not_none(monkeypatch):+ # The distinction the whole design rests on: None means "we do not+ # know what this costs", 0.0 means "this costs nothing".+ monkeypatch.setenv(RETRIEVAL_PRICE_ENV, "0.01")+ usd, _ = price_cold_bytes(0)+ assert usd == 0.0+++# ---------------------------------------------------------------------------+# The estimate object+# ---------------------------------------------------------------------------+++def _estimate(**overrides) -> ScanEstimate:+ base = {+ "estimated_rows": 1_000,+ "estimated_parts": 3,+ "estimated_bytes": 2_000,+ "cold_bytes": 0,+ "warm_bytes": 2_000,+ "tables": ["raw_events"],+ }+ base.update(overrides)+ return ScanEstimate(**base)+++def test_touches_cold_is_what_routes_a_query_to_a_job():+ assert _estimate(cold_bytes=0).touches_cold is False+ assert _estimate(cold_bytes=1).touches_cold is True+++def test_the_serialised_estimate_carries_the_routing_field():+ payload = _estimate(cold_bytes=5).as_dict()+ assert payload["touches_cold"] is True+ assert payload["cold_bytes"] == 5+ assert "estimated_usd" in payload+++# ---------------------------------------------------------------------------+# Progress+# ---------------------------------------------------------------------------+++def test_progress_with_no_denominator_is_unknown_rather_than_complete():+ # ClickHouse publishes total_rows_approx best-effort. A bar that+ # invents 100% because the denominator was zero is worse than one+ # that says it does not know.+ assert QueryProgress(rows_read=500, bytes_read=1, total_rows_approx=0, elapsed_seconds=1.0).ratio is None+++def test_progress_is_clamped_when_the_estimate_is_overtaken():+ # total_rows_approx moves while a query runs and can end up below+ # rows_read, which would otherwise render as 140%.+ progress = QueryProgress(rows_read=1_400, bytes_read=1, total_rows_approx=1_000, elapsed_seconds=1.0)+ assert progress.ratio == 1.0+++def test_progress_serialises_the_unknown_ratio_as_null():+ payload = QueryProgress(rows_read=1, bytes_read=1, total_rows_approx=0, elapsed_seconds=0.5).as_dict()+ assert payload["ratio"] is None+++ "bad",+ ["", "a b", "a;DROP", "x" * 65, "../etc", "id'quote"],+)+def test_a_query_id_outside_the_safe_alphabet_is_refused(bad):+ with pytest.raises(ValueError):+ validate_query_id(bad)+++def test_a_derived_query_id_is_accepted():+ assert validate_query_id("aisoc-lake-11111111-1111-1111-1111-111111111111")diff --git a/tests/isolation/test_lake_query_jobs_live.py b/tests/isolation/test_lake_query_jobs_live.pynew file mode 100644index 000000000..d2d172a9f--- /dev/null+++ b/tests/isolation/test_lake_query_jobs_live.py@@ -0,0 +1,321 @@+"""Cold-tier lake query jobs against real Postgres (depth 4.3).++The three properties worth proving here are all about concurrency and+state, and a mocked session answers whatever the test asked it to:++* **The claim is atomic.** Two API replicas must not both start the same+ cold scan. The claim is one conditional UPDATE, and the only way to+ show that works is to run two of them against one row.+* **A terminal state is never left.** A late progress write or a late+ finish must not resurrect a cancelled job, or rewrite a failure as a+ success.+* **A job id is tenant-scoped.** It is a handle a console holds, so the+ predicate is in the query and row-level security is defence in depth.++Skips when ``ISOLATION_LAKE_JOB_DSN`` is unset; the workflow that owns+this suite sets it unconditionally, because a skip in a job whose whole+purpose is to run live is a pass for the wrong reason.+"""++from __future__ import annotations++import asyncio+import os+import uuid+from datetime import UTC, datetime, timedelta++import pytest+import pytest_asyncio++pytestmark = [+ pytest.mark.asyncio(loop_scope="module"),+ pytest.mark.skipif(+ not os.environ.get("ISOLATION_LAKE_JOB_DSN", "").strip(),+ reason="ISOLATION_LAKE_JOB_DSN is not set; this suite needs a live Postgres",+ ),+]+++def _dsn() -> str:+ value = os.environ.get("ISOLATION_LAKE_JOB_DSN", "").strip()+ if not value:+ pytest.skip("ISOLATION_LAKE_JOB_DSN is not set")+ if value.startswith("postgresql://"):+ return value.replace("postgresql://", "postgresql+asyncpg://", 1)+ return value+++@pytest_asyncio.fixture(scope="module", loop_scope="module")+async def engine():+ from sqlalchemy.ext.asyncio import create_async_engine++ eng = create_async_engine(_dsn())+ try:+ yield eng+ finally:+ await eng.dispose()+++@pytest_asyncio.fixture(scope="module", loop_scope="module")+async def session_factory(engine):+ from sqlalchemy.ext.asyncio import async_sessionmaker++ return async_sessionmaker(engine, expire_on_commit=False)+++@pytest_asyncio.fixture(scope="module", loop_scope="module")+async def tenants(engine):+ from sqlalchemy import text++ a, b = uuid.uuid4(), uuid.uuid4()+ async with engine.begin() as conn:+ for tid, slug in ((a, "lakejob-a"), (b, "lakejob-b")):+ await conn.execute(+ text("INSERT INTO tenants (id, name, slug) VALUES (:id, :n, :s)"),+ {"id": str(tid), "n": slug, "s": f"{slug}-{uuid.uuid4().hex[:8]}"},+ )+ yield a, b+ async with engine.begin() as conn:+ for tid in (a, b):+ await conn.execute(text("DELETE FROM tenants WHERE id = :id"), {"id": str(tid)})+++async def _make_job(session_factory, tenant_id: uuid.UUID, **overrides):+ from app.models.lake_query_job import LakeQueryJob+ from app.services.lake_query_runner import new_query_id++ job_id = uuid.uuid4()+ values = {+ "id": job_id,+ "tenant_id": tenant_id,+ "submitted_by": uuid.uuid4(),+ "submitted_email": "[email protected]",+ "rewritten_sql": "SELECT 1",+ "row_cap": 100,+ "referenced_tables": ["raw_events"],+ "query_id": new_query_id(job_id),+ "estimate": {"cold_bytes": 1234, "estimated_usd": None},+ }+ values.update(overrides)+ async with session_factory() as session:+ job = LakeQueryJob(**values)+ session.add(job)+ await session.commit()+ return job_id+++class TestTheClaimIsAtomic:+ async def test_two_interleaved_transactions_produce_one_winner(self, engine, session_factory, tenants):+ """Two API replicas must not both start one cold scan.++ Deterministic rather than a race. The first attempt runs the+ claim and **holds its transaction open**; the second then runs+ the same statement and blocks on the row lock; the first commits;+ the second unblocks and re-evaluates ``status = 'queued'``+ against the committed row.++ The interleaving is forced rather than hoped for because a plain+ ``asyncio.gather`` of five claims passes under a+ SELECT-then-UPDATE implementation as well — measured, by+ subverting the claim and watching that version stay green. A+ concurrency test that cannot fail on the broken implementation+ is not a concurrency test.++ It runs ``claim_statement`` itself, not a re-typed copy: a proof+ against a hand-written statement proves something about the+ test.+ """+ from app.services.lake_query_runner import claim_statement++ tenant_a, _ = tenants+ job_id = await _make_job(session_factory, tenant_a)++ first_claimed: list[bool] = []+ second_claimed: list[bool] = []+ first_has_run = asyncio.Event()++ async def first() -> None:+ async with engine.connect() as conn:+ await conn.begin()+ result = await conn.execute(claim_statement(job_id))+ first_claimed.append(result.scalar_one_or_none() is not None)+ first_has_run.set()+ # Hold the lock long enough for the second attempt to+ # reach the same row and block on it.+ await asyncio.sleep(0.5)+ await conn.commit()++ async def second() -> None:+ await first_has_run.wait()+ await asyncio.sleep(0.1)+ async with engine.connect() as conn:+ await conn.begin()+ result = await conn.execute(claim_statement(job_id))+ second_claimed.append(result.scalar_one_or_none() is not None)+ await conn.commit()++ await asyncio.gather(first(), second())++ assert first_claimed == [True], "the first attempt did not claim the job"+ assert second_claimed == [False], "both transactions claimed the same job, so two replicas would both run the scan"++ async def test_a_job_that_is_not_queued_cannot_be_claimed(self, session_factory, tenants):+ from app.services.lake_query_runner import claim_job++ tenant_a, _ = tenants+ job_id = await _make_job(session_factory, tenant_a, status="succeeded")+ async with session_factory() as session:+ assert await claim_job(session, job_id) is False++ async def test_claiming_records_when_it_started(self, session_factory, tenants):+ from app.models.lake_query_job import LakeQueryJob+ from app.services.lake_query_runner import claim_job++ tenant_a, _ = tenants+ job_id = await _make_job(session_factory, tenant_a)+ async with session_factory() as session:+ assert await claim_job(session, job_id) is True+ async with session_factory() as session:+ job = await session.get(LakeQueryJob, job_id)+ assert job.status == "running"+ assert job.started_at is not None+++class TestATerminalStateIsNeverLeft:+ async def test_a_cancelled_job_is_not_overwritten_by_a_late_finish(self, session_factory, tenants):+ """The predicate on ``_finish`` is the whole mechanism: a scan+ that completes after an operator cancelled it must not report+ success."""+ from app.models.lake_query_job import LakeQueryJob+ from app.services.lake_query_runner import _finish, claim_job++ tenant_a, _ = tenants+ job_id = await _make_job(session_factory, tenant_a)+ async with session_factory() as session:+ await claim_job(session, job_id)+ async with session_factory() as session:+ job = await session.get(LakeQueryJob, job_id)+ job.status = "cancelled"+ await session.commit()++ await _finish(session_factory, job_id, status="succeeded", row_count=9)++ async with session_factory() as session:+ job = await session.get(LakeQueryJob, job_id)+ assert job.status == "cancelled"+ assert job.row_count is None++ async def test_cancelling_an_ended_job_is_refused(self, session_factory, tenants):+ from app.models.lake_query_job import LakeQueryJob+ from app.services.lake_query_runner import cancel_job++ tenant_a, _ = tenants+ job_id = await _make_job(session_factory, tenant_a, status="succeeded")+ async with session_factory() as session:+ job = await session.get(LakeQueryJob, job_id)+ assert await cancel_job(session, job) is False++ async def test_progress_is_not_written_onto_a_finished_job(self, session_factory, tenants):+ from app.models.lake_query_job import LakeQueryJob+ from app.services.lake_cold_query import QueryProgress+ from app.services.lake_query_runner import _record_progress++ tenant_a, _ = tenants+ job_id = await _make_job(session_factory, tenant_a, status="succeeded")+ await _record_progress(+ session_factory,+ job_id,+ QueryProgress(rows_read=42, bytes_read=1, total_rows_approx=100, elapsed_seconds=1.0),+ )+ async with session_factory() as session:+ job = await session.get(LakeQueryJob, job_id)+ assert job.rows_read == 0, "a late progress read resurrected a finished job"+++class TestProgressIsHonest:+ async def test_an_unknown_denominator_stores_null_not_zero(self, session_factory, tenants):+ from app.models.lake_query_job import LakeQueryJob+ from app.services.lake_cold_query import QueryProgress+ from app.services.lake_query_runner import _record_progress, claim_job++ tenant_a, _ = tenants+ job_id = await _make_job(session_factory, tenant_a)+ async with session_factory() as session:+ await claim_job(session, job_id)++ await _record_progress(+ session_factory,+ job_id,+ QueryProgress(rows_read=500, bytes_read=10, total_rows_approx=0, elapsed_seconds=2.0),+ )+ async with session_factory() as session:+ job = await session.get(LakeQueryJob, job_id)+ assert job.rows_read == 500+ assert job.progress_ratio is None, "unknown progress was stored as a number"++ async def test_a_known_denominator_stores_a_ratio(self, session_factory, tenants):+ from app.models.lake_query_job import LakeQueryJob+ from app.services.lake_cold_query import QueryProgress+ from app.services.lake_query_runner import _record_progress, claim_job++ tenant_a, _ = tenants+ job_id = await _make_job(session_factory, tenant_a)+ async with session_factory() as session:+ await claim_job(session, job_id)+ await _record_progress(+ session_factory,+ job_id,+ QueryProgress(rows_read=250, bytes_read=10, total_rows_approx=1000, elapsed_seconds=2.0),+ )+ async with session_factory() as session:+ job = await session.get(LakeQueryJob, job_id)+ assert job.progress_ratio == pytest.approx(0.25)+++class TestTenantScoping:+ async def test_another_tenants_job_is_not_found(self, session_factory, tenants):+ from app.api.v1.endpoints.lake import _load_job+ from fastapi import HTTPException++ tenant_a, tenant_b = tenants+ job_id = await _make_job(session_factory, tenant_b)+ async with session_factory() as session:+ with pytest.raises(HTTPException) as exc:+ await _load_job(session, tenant_a, job_id)+ assert exc.value.status_code == 404++ async def test_the_callers_own_job_is_found(self, session_factory, tenants):+ from app.api.v1.endpoints.lake import _load_job++ tenant_a, _ = tenants+ job_id = await _make_job(session_factory, tenant_a)+ async with session_factory() as session:+ job = await _load_job(session, tenant_a, job_id)+ assert job.id == job_id+++class TestStuckJobsAreVisible:+ async def test_a_job_whose_runner_died_is_reported(self, session_factory, tenants):+ """A job stuck at ``running`` with no task behind it reads to an+ analyst as a query that is still working."""+ from app.services.lake_query_runner import list_stuck_jobs++ tenant_a, _ = tenants+ job_id = await _make_job(+ session_factory,+ tenant_a,+ status="running",+ started_at=datetime.now(UTC) - timedelta(hours=2),+ )+ async with session_factory() as session:+ stuck = await list_stuck_jobs(session, older_than_seconds=900)+ assert job_id in {job.id for job in stuck}++ async def test_a_job_that_started_recently_is_not_reported(self, session_factory, tenants):+ from app.services.lake_query_runner import list_stuck_jobs++ tenant_a, _ = tenants+ job_id = await _make_job(session_factory, tenant_a, status="running", started_at=datetime.now(UTC))+ async with session_factory() as session:+ stuck = await list_stuck_jobs(session, older_than_seconds=900)+ assert job_id not in {job.id for job in stuck}