Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

feat(gates): hold raw SQL to the same column contract the ORM already has (#970)

Beenu Aroraff4129fd1fd8
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c563620fb..fb226b5b1 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -763,6 +763,26 @@ jobs:
python3 scripts/check_orm_migration_parity.py --self-test
python3 scripts/check_orm_migration_parity.py --check
+ # The same rule for the raw `text()` SQL the gate above cannot see —
+ # 107 statements naming 44 tables, 35 of which have no ORM model at
+ # all, so raw SQL is the only thing that writes them.
+ # `detection_rule_proposals` had no `source` column and three inserts
+ # named it, so the auto-tuner, the scheduled-hunt bridge and the
+ # false-positive loop could not write a proposal on any deployment.
+ # Nothing caught it: no job runs `services/api/tests/` against a real
+ # database, and the one test covering the insert asserted only that the
+ # table name appeared somewhere in the SQL string — which the wrong
+ # name `aisoc_detection_rule_proposals` also satisfies.
+ #
+ # The self-test runs first, and deliberately: it injects a wrong table,
+ # a wrong column and a runtime-assembled statement into scratch trees,
+ # so a parser that has quietly stopped matching cannot report the
+ # repository clean.
+ - name: Verify raw SQL names only tables and columns migrations create
+ run: |
+ python3 scripts/check_raw_sql_columns.py --self-test
+ python3 scripts/check_raw_sql_columns.py
+
# The four alembic chains have to be invoked by something. They were
# not: every container command was a plain `uvicorn`, so the
# documented quickstart booted two services with no tables and no RLS
diff --git a/scripts/check_orm_migration_parity.py b/scripts/check_orm_migration_parity.py
index c15e77f00..4ca7eea7e 100644
--- a/scripts/check_orm_migration_parity.py
+++ b/scripts/check_orm_migration_parity.py
@@ -137,6 +137,21 @@ def _string(node: ast.AST) -> str | None:
return node.value if isinstance(node, ast.Constant) and isinstance(node.value, str) else None
+def _rel(path: pathlib.Path) -> str:
+ """``path`` relative to the repository, or as given when it is outside it.
+
+ The parsers below are reusable — ``check_raw_sql_columns`` reads the same
+ migrations through them — and a reusable parser has to survive being
+ pointed at a scratch tree. ``relative_to`` raises there, which would turn
+ "this gate was handed a temporary directory" into a traceback rather than
+ a scan. Inside the repository the output is unchanged.
+ """
+ try:
+ return str(path.relative_to(REPO_ROOT))
+ except ValueError:
+ return str(path)
+
+
#: SQL type name → family. Two spellings of the same family are not drift
#: (``Text`` against ``String``, ``BigInteger`` against ``Integer``); two
#: families are, and one such mismatch is why this mapping exists.
@@ -289,7 +304,7 @@ def model_tables(paths: list[pathlib.Path]) -> dict[str, dict]:
if tablename and columns:
tables[tablename] = {
"columns": columns,
- "source": str(path.relative_to(REPO_ROOT)),
+ "source": _rel(path),
"model": node.name,
}
return tables
@@ -302,17 +317,43 @@ def model_tables(paths: list[pathlib.Path]) -> dict[str, dict]:
#: ``CREATE TABLE [IF NOT EXISTS] name ( ... )`` — the body is scanned for
#: leading identifiers, one per column definition.
_CREATE_TABLE = re.compile(r"CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?[\"']?(\w+)[\"']?\s*\((.*?)\);", re.I | re.S)
-_ADD_COLUMN = re.compile(
- r"ALTER\s+TABLE\s+(?:ONLY\s+)?[\"']?(\w+)[\"']?\s+ADD\s+COLUMN\s+"
- r"(?:IF\s+NOT\s+EXISTS\s+)?[\"']?(\w+)[\"']?\s*([A-Za-z]*)\s*(?:\((\d+)\))?",
- re.I,
-)
-_DROP_COLUMN = re.compile(r"ALTER\s+TABLE\s+(?:ONLY\s+)?[\"']?(\w+)[\"']?\s+DROP\s+COLUMN\s+(?:IF\s+EXISTS\s+)?[\"']?(\w+)[\"']?", re.I)
-_ALTER_TYPE = re.compile(
- r"ALTER\s+TABLE\s+(?:ONLY\s+)?[\"']?(\w+)[\"']?\s+ALTER\s+COLUMN\s+[\"']?(\w+)[\"']?\s+"
- r"(?:SET\s+DATA\s+)?TYPE\s+([A-Za-z]+)\s*(?:\((\d+)\))?",
- re.I,
-)
+#: ``ALTER TABLE <name> <clauses>;`` — the target, then everything up to the
+#: statement terminator, because one ``ALTER TABLE`` may carry several
+#: comma-separated clauses.
+#:
+#: This was four regexes, each anchored on its own ``ALTER TABLE`` prefix,
+#: which between them read only the *first* clause of any statement.
+#: ``040_critical_severity_alert_confidence.sql`` adds three columns to
+#: ``alerts`` in one statement and this parser credited the table with one;
+#: ``063_cost_provenance.sql`` adds five to ``aisoc_run_costs`` and it credited
+#: one. Anchoring on the statement and then reading its clause list is the only
+#: shape that does not depend on how the author chose to punctuate.
+_ALTER_TABLE = re.compile(r"ALTER\s+TABLE\s+(?:IF\s+EXISTS\s+)?(?:ONLY\s+)?[\"']?(\w+)[\"']?\s+(.*?)(?:;|\Z)", re.I | re.S)
+_CLAUSE_ADD = re.compile(r"\bADD\s+COLUMN\s+(?:IF\s+NOT\s+EXISTS\s+)?[\"']?(\w+)[\"']?\s*([A-Za-z]*)\s*(?:\((\d+)\))?", re.I)
+_CLAUSE_DROP = re.compile(r"\bDROP\s+COLUMN\s+(?:IF\s+EXISTS\s+)?[\"']?(\w+)[\"']?", re.I)
+_CLAUSE_RENAME = re.compile(r"\bRENAME\s+COLUMN\s+[\"']?(\w+)[\"']?\s+TO\s+[\"']?(\w+)[\"']?", re.I)
+_CLAUSE_TYPE = re.compile(r"\bALTER\s+COLUMN\s+[\"']?(\w+)[\"']?\s+(?:SET\s+DATA\s+)?TYPE\s+([A-Za-z]+)\s*(?:\((\d+)\))?", re.I)
+
+_SQL_COMMENT = re.compile(r"--[^\n]*|/\*.*?\*/", re.S)
+
+
+def strip_sql_comments(sql: str) -> str:
+ """``sql`` with ``--`` and ``/* */`` comments removed, line structure kept.
+
+ Any parser that splits a clause list on commas has to do this first, and
+ both readers of SQL in this tree learned it the hard way.
+ ``_split_columns`` reads the first token of each comma-separated part as a
+ column name, so a ``--`` comment line sits inside the part that *follows*
+ it and hides the column behind it: ``detection_rule_proposals`` documents
+ four of its columns that way and this parser credited the table with none
+ of them. ``check_raw_sql_columns`` hit the same thing from the statement
+ side, where a comment inside a ``SET`` clause made three perfectly
+ ordinary upserts read as assembled at runtime.
+
+ Newlines survive so a comment cannot glue two statements together.
+ """
+ return _SQL_COMMENT.sub("", sql)
+
#: Words that begin a table-level constraint rather than a column.
_CONSTRAINT_WORDS = frozenset({"primary", "unique", "foreign", "constraint", "check", "exclude", "like"})
@@ -366,19 +407,40 @@ def _record(
store.setdefault(table, {})[column] = (family, length)
-def _scan_sql(text: str, store: dict[str, dict[str, tuple[str | None, int | None]]], dropped: set[tuple[str, str]]) -> None:
+def _scan_sql(raw: str, store: dict[str, dict[str, tuple[str | None, int | None]]], dropped: set[tuple[str, str]]) -> None:
+ text = strip_sql_comments(raw)
for table, body in _CREATE_TABLE.findall(text):
for column, family, length in _split_columns(body):
_record(store, table, column, family, length)
- for table, column, type_name, length in _ADD_COLUMN.findall(text):
- _record(store, table, column, _family(type_name), int(length) if length else None)
- for table, column, type_name, length in _ALTER_TYPE.findall(text):
- # Only widens what is already known; an ALTER TYPE on a column this
- # scan never saw created is still a column it never saw created.
- if column in store.get(table, {}):
+ # Each ALTER TABLE in document order, then its clauses, so a statement
+ # that adds a column and a later one that renames it compose the way the
+ # database would apply them.
+ for table, clauses in _ALTER_TABLE.findall(text):
+ for column, type_name, length in _CLAUSE_ADD.findall(clauses):
_record(store, table, column, _family(type_name), int(length) if length else None)
- for table, column in _DROP_COLUMN.findall(text):
- dropped.add((table, column))
+ for column, type_name, length in _CLAUSE_TYPE.findall(clauses):
+ # Only widens what is already known; an ALTER TYPE on a column
+ # this scan never saw created is still a column it never saw
+ # created.
+ if column in store.get(table, {}):
+ _record(store, table, column, _family(type_name), int(length) if length else None)
+ # A renamed column is the same column under a new name, and reading
+ # only the create statement leaves the parser holding the old one.
+ # Migration 025 renames four columns on `connectors` — `config` to
+ # `connector_config` among them — and every reader in the tree names
+ # the new spelling, so a parser that never applies the rename reports
+ # four columns that do not exist and misses four that do.
+ for before, after in _CLAUSE_RENAME.findall(clauses):
+ columns = store.get(table)
+ if columns is None:
+ continue
+ # The rename is itself proof the column is there, so a column
+ # this parser never saw created still lands under its new name
+ # rather than being silently dropped.
+ columns[after] = columns.pop(before, (None, None))
+ dropped.discard((table, before))
+ for column in _CLAUSE_DROP.findall(clauses):
+ dropped.add((table, column))
def _scan_alembic_module(tree: ast.AST, store: dict[str, dict[str, tuple[str | None, int | None]]], dropped: set[tuple[str, str]]) -> None:
@@ -461,12 +523,12 @@ def migration_columns(service: pathlib.Path) -> tuple[dict[str, dict[str, tuple[
tree = _upgrade_only(path)
if tree is not None:
_scan_alembic_module(tree, store, dropped)
- read.append(str(path.relative_to(REPO_ROOT)))
+ read.append(_rel(path))
for name in SQL_MIGRATION_DIRS:
for path in sorted((service / name).glob("*.sql")):
_scan_sql(path.read_text(encoding="utf-8", errors="replace"), store, dropped)
- read.append(str(path.relative_to(REPO_ROOT)))
+ read.append(_rel(path))
return store, dropped, read
@@ -508,7 +570,7 @@ def creates_tables_from_metadata(paths: list[pathlib.Path]) -> str | None:
continue
for node in ast.walk(tree):
if isinstance(node, ast.Attribute) and node.attr == "create_all":
- return str(path.relative_to(REPO_ROOT))
+ return _rel(path)
return None
diff --git a/scripts/check_raw_sql_columns.py b/scripts/check_raw_sql_columns.py
new file mode 100644
index 000000000..fc56c7bf9
--- /dev/null
+++ b/scripts/check_raw_sql_columns.py
@@ -0,0 +1,772 @@
+#!/usr/bin/env python3
+"""Every table and column a raw SQL statement names must be one a migration creates.
+
+Why this exists
+---------------
+
+``services/api/app/api/v1/endpoints/rule_tuning.py`` inserts an auto-tuner
+proposal with a raw ``sqlalchemy.text()`` statement naming fourteen columns of
+``detection_rule_proposals``. One of them, ``source``, is created by no
+migration in this repository — not in ``010_detection_as_code.sql`` where the
+table is defined, not in ``039_detection_proposal_github_pr.sql`` where it is
+altered, not anywhere. The statement raises ``UndefinedColumnError`` on the
+first row it tries to write, so the human-approval-gated tuning loop cannot
+produce a single proposal on any deployment. This gate found it.
+
+Nothing about that needed a database. The statement and the migrations are
+both in the tree and the disagreement is structural, which is the same
+observation ``check_orm_migration_parity.py`` was written on: UEBA declared
+``EntityBaseline.peer_group_id``, migration ``0001`` never created it, and
+UEBA could not write a baseline or an anomaly from the day the column was
+added. That gate reads ORM-declared models. A raw ``text()`` string is
+invisible to it, so the identical failure class had no gate at all over 107
+raw statements naming 44 tables — and 35 of those 44 have no ORM model in the
+tree at all, so raw SQL is the only thing that writes them and nothing was
+checking it.
+
+Two things hid it for as long as they did.
+
+*The claim that a real database already exercised it.* The claim-to-gate
+matrix recorded the analyst-disposition tuning loop as ``PARTIAL`` with the
+gap "end-to-end proposal insert exercised in api integration, not a unit DB",
+which reads as thin coverage of something that does run. No job runs
+``services/api/tests/`` against a real database. ``Python — Tests`` in
+``ci.yml`` declares no ``postgres`` service and sets
+``DATABASE_URL=postgresql+asyncpg://x:x@localhost/x``, a deliberate
+non-database. ``integration.yml`` and ``upgrade-test.yml`` are the only
+workflows with a live Postgres, and the single API test either of them runs is
+``services/api/tests/test_mssp_portfolio_isolation.py``. The insert had never
+touched a database anywhere.
+
+*A test that greps for the table name.* ``test_wave1_loop_edges.py`` asserted
+``"detection_rule_proposals" in str(db.execute.await_args.args[0])``. Being a
+substring test it also passes on ``aisoc_detection_rule_proposals``, which
+contains the right name inside the wrong one — and that exact confusion is
+recorded history here: three insert sites and two tests had codified the
+``aisoc_``-prefixed spelling of this very table. A mocked session cannot fail
+on a column, so the assertion had nothing else to check.
+
+What it checks, in both directions
+----------------------------------
+
+``unknown-table``
+ ``INSERT INTO t`` or ``UPDATE t SET`` naming a table no migration creates.
+ This is the ``aisoc_detection_rule_proposals`` class exactly, and unlike
+ the substring assertion it cannot be satisfied by a name that merely
+ contains the right one.
+
+``unknown-column``
+ A column in an ``INSERT`` column list, or on the left of a ``SET`` in an
+ ``UPDATE`` or an ``ON CONFLICT … DO UPDATE``, that no migration creates on
+ that table. ``detection_rule_proposals.source`` is this.
+
+``unparsed-statement``
+ The other direction, and the one that decides whether this gate is worth
+ anything: a statement whose table or column list is assembled at runtime
+ contributes zero comparisons, and zero findings over zero comparisons
+ prints the same word as a clean result. Rather than skipping those
+ quietly, each is a finding until it is recorded in ``DYNAMIC_SQL`` below
+ with a reason. The list is checked in both directions, so an entry nothing
+ needs any more fails the build instead of sitting there looking like
+ coverage.
+
+``--credits`` exists for the same reason: it enumerates what each verdict was
+based on, so a parser that has silently stopped matching can be caught by
+reading rather than by trusting the exit code.
+
+What it reads
+-------------
+
+Statements: Python ``ast`` over ``services/*/app/**/*.py``, every string
+constant and f-string in the module rather than only the arguments of
+``text()`` — the SQL in ``services/fusion/app/services/alert_sink.py`` and
+``services/agents/app/hunt/store.py`` is a module-level constant that
+something else wraps later, and a gate keyed on the call site would not see
+either. An f-string renders with ``{}`` where its interpolations were, which
+is how a runtime-assembled table or column list becomes visible as one.
+
+Schema: the migration parsers in ``check_orm_migration_parity.py``, imported
+rather than reimplemented. Two parsers of the same DDL drift the first time
+either learns something the other has not, and this repository has the scars:
+its graph-schema check declared OK while one side had 17 labels and the other
+28. Reusing them also fixed two gaps that only a column-level reader would
+notice — a ``--`` comment line swallowed the column definition after it, and
+``ALTER TABLE … RENAME COLUMN`` was never applied, so four ``connectors``
+columns were known to the parser under names nothing in the tree uses.
+
+Scope: every service with an ``app/`` directory, which on today's tree is 893
+modules yielding 107 statements — ``api`` 75, ``agents`` 20, ``fusion`` 5,
+``actions`` 3, ``slack-bot`` 2, ``osquery-tls`` 1, ``ueba`` 1. Small enough to
+triage one by one, so nothing is narrowed to keep the number down.
+
+The schema is the union of every migration source in the repository — 85 SQL
+files under ``services/api/migrations`` plus the four alembic chains, 131
+tables and 1,582 columns — because it is one Postgres. ``services/agents`` and
+``services/fusion`` own no migrations and write to tables ``services/api``
+creates, so a per-service comparison would have reported every one of their
+25 statements as unsourced.
+
+Usage
+-----
+
+::
+
+ python3 scripts/check_raw_sql_columns.py # gate
+ python3 scripts/check_raw_sql_columns.py --list # every statement read
+ python3 scripts/check_raw_sql_columns.py --credits # what it credited
+ python3 scripts/check_raw_sql_columns.py --json
+ python3 scripts/check_raw_sql_columns.py --self-test
+
+Exit codes: 0 clean, 1 findings, 2 the scan itself could not run.
+"""
+
+from __future__ import annotations
+
+import argparse
+import ast
+import contextlib
+import io
+import json
+import pathlib
+import re
+import sys
+import tempfile
+
+# `scripts/` is on sys.path when this file is run as a program, but not when a
+# test loads it by path with importlib. Both siblings sit beside it either way.
+sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
+
+# `--self-test` is answered in `main()` rather than through
+# `self_test_if_requested`, because that shortcut is for gates with nothing
+# bespoke to add and this one has five injected-drift cases to run alongside
+# the shared empty-tree refusal. Installing it here would exit before any of
+# them, leaving a self-test that proves only half of what it prints.
+from check_orm_migration_parity import migration_columns, strip_sql_comments
+from gate_toolkit import repo_root, self_test_main
+
+REPO_ROOT = repo_root()
+SERVICES_DIR = REPO_ROOT / "services"
+
+#: Directories under ``services/<svc>/app/`` that hold no production SQL.
+#: Tests are excluded because a fixture may name a table on purpose to assert
+#: what happens when it is wrong.
+_SKIP_PARTS = frozenset({"tests", "test", "_vendor", "node_modules", "__pycache__"})
+
+#: What an f-string interpolation renders as. Anything carrying this was
+#: assembled at runtime and cannot be compared against a migration.
+DYNAMIC = "{}"
+
+
+# ---------------------------------------------------------------------------
+# Recorded exceptions
+# ---------------------------------------------------------------------------
+#
+# Shrink-only, and verified in both directions: an entry that stops being
+# needed fails the build rather than sitting here.
+#
+# Two lists rather than one, because they excuse genuinely different things
+# and collapsing them would let either reason cover the other. A table no
+# migration owns is a fact about the table, so it is keyed on the table and
+# forgives the column check with it. A statement assembled at runtime is a
+# fact about that one statement, so it is keyed on (file, table as written) —
+# and it only ever excuses a statement this parser could not read, so an
+# ordinary INSERT in an allow-listed file is still held to the contract.
+
+#: Tables that exist but whose creation no migration under ``services/`` owns.
+UNMIGRATED_TABLES: dict[str, str] = {
+ "aisoc.raw_events": (
+ "ClickHouse, not Postgres. The event lake's table is created by "
+ "services/fusion/app/services/lake_writer.py with CREATE DATABASE / CREATE TABLE IF NOT EXISTS on "
+ "startup — the ECS ClickHouse task does not run docker-entrypoint-initdb.d — so no migration "
+ "declares it and no migration should."
+ ),
+ "aisoc_schema_migrations": (
+ "The Postgres migration runner's own ledger, created by CREATE_MIGRATIONS_TABLE in "
+ "services/api/app/scripts/run_migrations.py before any migration executes. A migration chain "
+ "cannot be asked to declare the table that records which migrations ran — the same reason "
+ "alembic's version table is in no revision."
+ ),
+}
+
+#: Statements whose table or column list is assembled at runtime, and so
+#: cannot be compared against anything, with the reason.
+DYNAMIC_SQL: dict[tuple[str, str], str] = {
+ (
+ "services/api/app/db/lake_migrations.py",
+ DYNAMIC,
+ ): (
+ "The ClickHouse lake's own migration ledger: the table is `MIGRATION_TABLE` "
+ "('aisoc._migrations'), interpolated, and this module creates it. Same reason as "
+ "`aisoc_schema_migrations` above, one store over."
+ ),
+ (
+ "services/api/app/api/v1/endpoints/cases.py",
+ "aisoc_cases",
+ ): (
+ "A PATCH handler: the SET clause is `', '.join(sets)` over whichever fields the request body "
+ "supplied, so there is no column list in the source to compare. The columns it can append are "
+ "literals in this function, which is the read a human has to do."
+ ),
+ (
+ "services/api/app/api/v1/endpoints/cases.py",
+ "aisoc_case_tasks",
+ ): "A PATCH handler on case tasks, same shape as the one above.",
+ (
+ "services/api/app/api/v1/endpoints/hunts.py",
+ "aisoc_hunts",
+ ): "A PATCH handler on saved hunts, same shape as the two above.",
+ (
+ "services/api/app/scripts/seed_demo.py",
+ DYNAMIC,
+ ): (
+ "The demo re-anchor sweep walks `_REANCHOR_TABLES` and shifts every timestamp column it names, so "
+ "both the table and the assignment list are interpolated per iteration. Demo-mode seeding, not a "
+ "production write path."
+ ),
+ (
+ "services/osquery-tls/app/db/env.py",
+ DYNAMIC,
+ ): ("Alembic's own bookkeeping: the table is `VERSION_TABLE`, interpolated, and alembic creates it rather than a revision."),
+}
+
+
+# ---------------------------------------------------------------------------
+# Schema side
+# ---------------------------------------------------------------------------
+
+
+def migration_schema(services_dir: pathlib.Path) -> tuple[dict[str, set[str]], dict[str, str], int]:
+ """``({table: columns}, {table: owning service}, migration files read)``.
+
+ The union across every service that ships migrations, because the services
+ share one Postgres: ``services/agents`` writes ``investigation_runs`` and
+ ``services/fusion`` writes ``alerts``, both created under
+ ``services/api/migrations``.
+ """
+ schema: dict[str, set[str]] = {}
+ owner: dict[str, str] = {}
+ files = 0
+ for service in sorted(p for p in services_dir.iterdir() if p.is_dir()):
+ created, dropped, read = migration_columns(service)
+ if not read:
+ continue
+ files += len(read)
+ for table, columns in created.items():
+ live = {c for c in columns if (table, c) not in dropped}
+ if not live:
+ continue
+ schema.setdefault(table, set()).update(live)
+ owner.setdefault(table, service.name)
+ return schema, owner, files
+
+
+# ---------------------------------------------------------------------------
+# Statement side
+# ---------------------------------------------------------------------------
+
+_INSERT = re.compile(r"\bINSERT\s+(?:OR\s+\w+\s+)?INTO\s+([^\s(;]+)", re.I)
+_UPDATE = re.compile(r"\bUPDATE\s+(?:ONLY\s+)?([^\s(;]+)\s+SET\b", re.I)
+_DO_UPDATE = re.compile(r"\bDO\s+UPDATE\s+SET\b", re.I)
+
+#: Where a ``SET`` clause ends. ``FROM`` is included because Postgres allows
+#: ``UPDATE t SET … FROM other``, and reading past it would collect the joined
+#: table's columns as if they were assignments.
+_SET_END = re.compile(r"\b(WHERE|RETURNING|FROM)\b", re.I)
+
+_IDENTIFIER = re.compile(r"^[A-Za-z_]\w*$")
+
+
+def _render(node: ast.AST) -> str | None:
+ """A string literal's text, with ``{}`` where an f-string interpolated.
+
+ Rendering rather than skipping is what makes a runtime-assembled table
+ name or column list *visible*. ``f"INSERT INTO {VERSION_TABLE} …"`` comes
+ back as ``INSERT INTO {} …``, which this gate reports as unparsed instead
+ of failing to notice.
+ """
+ if isinstance(node, ast.Constant):
+ text = node.value if isinstance(node.value, str) else None
+ elif isinstance(node, ast.JoinedStr):
+ parts: list[str] = []
+ for value in node.values:
+ if isinstance(value, ast.Constant) and isinstance(value.value, str):
+ parts.append(value.value)
+ else:
+ parts.append(DYNAMIC)
+ text = "".join(parts)
+ else:
+ return None
+ # Comments are stripped through the same helper the migration side uses.
+ # Three ordinary upserts here read as assembled at runtime purely because
+ # a `--` note sat inside their SET clause and the comma-split collected it
+ # as if it were an assignment.
+ return strip_sql_comments(text) if text else text
+
+
+def _balanced(text: str, start: int) -> tuple[str, int] | None:
+ """The contents of the parenthesised group beginning at or after ``start``.
+
+ Returns ``None`` when the next non-space character is not ``(``, which is
+ how ``INSERT INTO t VALUES (…)`` — no column list — is told apart from
+ ``INSERT INTO t (a, b) VALUES (…)``.
+ """
+ index = start
+ while index < len(text) and text[index].isspace():
+ index += 1
+ if index >= len(text) or text[index] != "(":
+ return None
+ depth = 0
+ for position in range(index, len(text)):
+ if text[position] == "(":
+ depth += 1
+ elif text[position] == ")":
+ depth -= 1
+ if depth == 0:
+ return text[index + 1 : position], position + 1
+ return None
+
+
+def _split_top_level(body: str) -> list[str]:
+ """``body`` split on commas outside any parentheses."""
+ depth = 0
+ current: list[str] = []
+ parts: list[str] = []
+ for char in body:
+ if char == "(":
+ depth += 1
+ elif char == ")":
+ depth -= 1
+ if char == "," and depth == 0:
+ parts.append("".join(current))
+ current = []
+ continue
+ current.append(char)
+ parts.append("".join(current))
+ return parts
+
+
+def _column_names(parts: list[str]) -> tuple[list[str], bool]:
+ """``(identifiers, every part was a plain identifier)``.
+
+ The flag is the honest half. A part this cannot read — an expression, a
+ cast, an interpolation — means the list is not fully accounted for, and
+ comparing the identifiers it *did* read while staying silent about the
+ rest would credit the statement with more scrutiny than it received.
+ """
+ names: list[str] = []
+ complete = True
+ for part in parts:
+ token = part.strip().strip('"').strip()
+ if _IDENTIFIER.match(token):
+ names.append(token)
+ elif token:
+ complete = False
+ return names, complete
+
+
+def _set_columns(text: str, start: int) -> tuple[list[str], bool]:
+ """Assigned column names in the ``SET`` clause beginning at ``start``."""
+ clause = text[start:]
+ if end := _SET_END.search(clause):
+ clause = clause[: end.start()]
+ names: list[str] = []
+ complete = True
+ for part in _split_top_level(clause):
+ left = part.split("=", 1)[0].strip().strip('"').strip()
+ if not left:
+ continue
+ # `t.col = …` is still an assignment to `col`.
+ if "." in left:
+ left = left.rsplit(".", 1)[-1].strip('"')
+ if _IDENTIFIER.match(left):
+ names.append(left)
+ else:
+ complete = False
+ return names, complete
+
+
+def statements_in(source: str, relpath: str) -> list[dict]:
+ """Every raw ``INSERT`` / ``UPDATE`` in one module's string literals."""
+ try:
+ tree = ast.parse(source)
+ except SyntaxError:
+ return []
+
+ found: list[dict] = []
+ for node in ast.walk(tree):
+ if not isinstance(node, ast.Constant | ast.JoinedStr):
+ continue
+ text = _render(node)
+ if not text or not ("INSERT" in text.upper() or "UPDATE" in text.upper()):
+ continue
+ line = getattr(node, "lineno", 0)
+
+ inserts: list[tuple[int, dict]] = []
+ for match in _INSERT.finditer(text):
+ group = _balanced(text, match.end())
+ columns, complete = _column_names(_split_top_level(group[0])) if group else ([], True)
+ statement = {
+ "kind": "INSERT",
+ "file": relpath,
+ "line": line,
+ "table": match.group(1).strip('"'),
+ "columns": columns,
+ # A statement with no column list names no column to check,
+ # which is not the same as one whose list could not be read.
+ "columns_declared": group is not None,
+ "columns_complete": complete,
+ }
+ inserts.append((match.start(), statement))
+ found.append(statement)
+
+ for match in _UPDATE.finditer(text):
+ columns, complete = _set_columns(text, match.end())
+ found.append(
+ {
+ "kind": "UPDATE",
+ "file": relpath,
+ "line": line,
+ "table": match.group(1).strip('"'),
+ "columns": columns,
+ "columns_declared": True,
+ "columns_complete": complete,
+ }
+ )
+
+ # `ON CONFLICT … DO UPDATE SET` assigns to the table of the INSERT it
+ # belongs to, so it is the same column contract under another spelling
+ # — and upserts are how most of this repository writes.
+ for match in _DO_UPDATE.finditer(text):
+ preceding = [statement for position, statement in inserts if position < match.start()]
+ if not preceding:
+ continue
+ columns, complete = _set_columns(text, match.end())
+ found.append(
+ {
+ "kind": "UPSERT",
+ "file": relpath,
+ "line": line,
+ "table": preceding[-1]["table"],
+ "columns": columns,
+ "columns_declared": True,
+ "columns_complete": complete,
+ }
+ )
+
+ return found
+
+
+def service_statements(services_dir: pathlib.Path, root: pathlib.Path) -> tuple[list[dict], int]:
+ """Statements across ``services/*/app/**/*.py``, and the file count read."""
+ found: list[dict] = []
+ read = 0
+ for service in sorted(p for p in services_dir.iterdir() if p.is_dir()):
+ app = service / "app"
+ if not app.is_dir():
+ continue
+ for path in sorted(app.rglob("*.py")):
+ if _SKIP_PARTS & set(path.relative_to(service).parts):
+ continue
+ read += 1
+ try:
+ relpath = str(path.relative_to(root))
+ except ValueError:
+ relpath = str(path)
+ found.extend(statements_in(path.read_text(encoding="utf-8", errors="replace"), relpath))
+ return found, read
+
+
+# ---------------------------------------------------------------------------
+# Comparison
+# ---------------------------------------------------------------------------
+
+
+def scan(root: pathlib.Path | None = None) -> dict:
+ root = root or REPO_ROOT
+ services_dir = root / "services"
+ if not services_dir.is_dir():
+ return {"error": f"no services directory at {services_dir}"}
+
+ schema, owner, migration_files = migration_schema(services_dir)
+ statements, files_read = service_statements(services_dir, root)
+
+ findings: list[dict] = []
+ credits: list[dict] = []
+ foreign_used: set[str] = set()
+ dynamic_used: set[tuple[str, str]] = set()
+
+ for statement in statements:
+ table = statement["table"]
+ key = (statement["file"], table)
+
+ if table in UNMIGRATED_TABLES:
+ foreign_used.add(table)
+ credits.append({**statement, "verdict": "unmigrated-table", "detail": UNMIGRATED_TABLES[table]})
+ continue
+
+ # Only a statement this parser could not read may be excused as
+ # dynamic. A file holding one assembled statement does not get its
+ # ordinary ones forgiven along with it.
+ if DYNAMIC in table or not statement["columns_complete"]:
+ if key in DYNAMIC_SQL:
+ dynamic_used.add(key)
+ credits.append({**statement, "verdict": "runtime-assembled", "detail": DYNAMIC_SQL[key]})
+ continue
+ findings.append(
+ {
+ **statement,
+ "kind_of_finding": "unparsed-statement",
+ "column": "*",
+ "detail": (
+ f"{statement['kind']} into {table!r} is assembled at runtime, so no table or column "
+ "here was compared against a migration. Record it in DYNAMIC_SQL with the reason."
+ ),
+ }
+ )
+ continue
+
+ if table not in schema:
+ findings.append(
+ {
+ **statement,
+ "kind_of_finding": "unknown-table",
+ "column": "*",
+ "detail": f"no migration creates a table named {table!r}. Every execution of this statement fails.",
+ }
+ )
+ continue
+
+ unknown = [column for column in statement["columns"] if column not in schema[table]]
+ for column in unknown:
+ findings.append(
+ {
+ **statement,
+ "kind_of_finding": "unknown-column",
+ "column": column,
+ "detail": (
+ f"{statement['kind']} names {table}.{column}; no migration creates it on that table "
+ f"(migrations for {table} come from services/{owner[table]})."
+ ),
+ }
+ )
+ credits.append(
+ {
+ **statement,
+ "verdict": "compared",
+ "detail": f"{len(statement['columns'])} column(s) against services/{owner[table]} migrations",
+ }
+ )
+
+ stale = [
+ f"UNMIGRATED_TABLES[{name!r}] — no raw statement names that table any more"
+ for name in UNMIGRATED_TABLES
+ if name not in foreign_used
+ ]
+ stale += [
+ f"DYNAMIC_SQL[{key!r}] — no statement there is assembled at runtime any more, so it is checkable"
+ for key in DYNAMIC_SQL
+ if key not in dynamic_used
+ ]
+
+ return {
+ "root": str(root),
+ "files_read": files_read,
+ "migration_files": migration_files,
+ "schema_tables": len(schema),
+ "schema_columns": sum(len(c) for c in schema.values()),
+ "statements": statements,
+ "credits": credits,
+ "compared": sum(1 for c in credits if c["verdict"] == "compared"),
+ "recorded": sum(1 for c in credits if c["verdict"] != "compared"),
+ "tables_touched": sorted({s["table"] for s in statements if DYNAMIC not in s["table"]}),
+ "findings": findings,
+ "stale": stale,
+ }
+
+
+# ---------------------------------------------------------------------------
+# Self-test
+# ---------------------------------------------------------------------------
+
+_GOOD_MIGRATION = """
+CREATE TABLE IF NOT EXISTS widgets (
+ id UUID PRIMARY KEY,
+ -- a comment line, which used to swallow the column behind it
+ tenant_id UUID NOT NULL,
+ name VARCHAR(200) NOT NULL
+);
+"""
+
+_GOOD_STATEMENT = """
+from sqlalchemy import text
+
+SQL = text("INSERT INTO widgets (id, tenant_id, name) VALUES (:id, :tid, :name)")
+"""
+
+
+def _scratch(root: pathlib.Path, statement: str, migration: str = _GOOD_MIGRATION) -> dict:
+ service = root / "services" / "widgetry"
+ (service / "app" / "api").mkdir(parents=True)
+ (service / "app" / "api" / "widgets.py").write_text(statement)
+ (service / "migrations").mkdir(parents=True)
+ (service / "migrations" / "001_init.sql").write_text(migration)
+ return scan(root)
+
+
+def _detects_injected_drift() -> list[tuple[str, bool]]:
+ """Prove the gate still catches each kind, on a tree built for the purpose.
+
+ Run before the gate renders any verdict about the real tree, which is the
+ point of a self-test: a gate whose parser has quietly stopped matching
+ anything reports a clean repository in exactly the same words as a clean
+ one.
+ """
+
+ def kinds(statement: str) -> set[str]:
+ with tempfile.TemporaryDirectory() as raw:
+ return {f["kind_of_finding"] for f in _scratch(pathlib.Path(raw), statement)["findings"]}
+
+ def counted(statement: str) -> int:
+ with tempfile.TemporaryDirectory() as raw:
+ return _scratch(pathlib.Path(raw), statement)["compared"]
+
+ # The defect this gate exists for, in the spelling it actually had: the
+ # wrong name contains the right one, so the substring assertion in
+ # `test_wave1_loop_edges.py` passed on it.
+ wrong_table = _GOOD_STATEMENT.replace("INSERT INTO widgets", "INSERT INTO aisoc_widgets")
+ wrong_column = _GOOD_STATEMENT.replace("id, tenant_id, name", "id, tenant_id, source")
+ dynamic = 'SQL = f"INSERT INTO {TABLE} (id) VALUES (:id)"\n'
+
+ return [
+ ("detects a table no migration creates, on a name containing a real one", kinds(wrong_table) == {"unknown-table"}),
+ ("detects a column no migration creates on that table", kinds(wrong_column) == {"unknown-column"}),
+ ("passes a statement whose table and every column exist", kinds(_GOOD_STATEMENT) == set()),
+ ("credits the passing statement rather than skipping it", counted(_GOOD_STATEMENT) == 1),
+ ("refuses a statement assembled at runtime rather than skipping it", kinds(dynamic) == {"unparsed-statement"}),
+ ]
+
+
+def _refuses_a_tree_with_no_statements() -> list[tuple[str, bool]]:
+ """And a services tree that parses to no SQL at all.
+
+ Separate from the above because it is the failure mode of this gate rather
+ than of the tree: reading no statement has to be a refusal, or a parser
+ that matches nothing reports every service clean. The scratch tree
+ ``gate_toolkit`` builds has no ``services/`` directory at all; this is the
+ other shape, where the directory is there and the corpus is gone.
+ """
+ with tempfile.TemporaryDirectory() as raw:
+ root = pathlib.Path(raw)
+ service = root / "services" / "widgetry"
+ (service / "app").mkdir(parents=True)
+ (service / "app" / "nothing.py").write_text("VALUE = 1\n")
+ (service / "migrations").mkdir(parents=True)
+ (service / "migrations" / "001_init.sql").write_text(_GOOD_MIGRATION)
+ # The probe runs the real `main`, so its refusal is the real refusal
+ # rather than a re-implementation of one — captured only so the
+ # self-test's own output stays readable.
+ sink = io.StringIO()
+ with contextlib.redirect_stdout(sink), contextlib.redirect_stderr(sink):
+ refused = main(["--repo-root", str(root)]) != 0
+
+ return [("refuses a services tree it parsed no SQL out of", refused)]
+
+
+# ---------------------------------------------------------------------------
+# Entry point
+# ---------------------------------------------------------------------------
+
+
+def main(argv: list[str] | None = None) -> int:
+ parser = argparse.ArgumentParser(description="Raw SQL / migration table and column parity")
+ parser.add_argument("--list", action="store_true", help="print every statement read")
+ parser.add_argument("--credits", action="store_true", help="print what each verdict was based on")
+ parser.add_argument("--json", action="store_true", help="machine-readable output")
+ parser.add_argument("--repo-root", help="the tree to inspect, instead of this checkout")
+ parser.add_argument("--self-test", action="store_true", help="prove this gate still detects drift")
+ args = parser.parse_args(argv)
+
+ if args.self_test:
+ return self_test_main(
+ pathlib.Path(__file__).name,
+ [],
+ extra=[*_detects_injected_drift(), *_refuses_a_tree_with_no_statements()],
+ )
+
+ result = scan(pathlib.Path(args.repo_root).resolve() if args.repo_root else None)
+ if "error" in result:
+ print(f"raw-sql-columns: {result['error']}", file=sys.stderr)
+ return 2
+
+ if args.json:
+ print(json.dumps(result, indent=2, sort_keys=True))
+
+ # Naming the root and the corpus size is not decoration. A gate that
+ # printed "OK" without them could be reporting on a different checkout,
+ # or on none: `found nothing` and `scanned nothing` print the same word.
+ if not args.json:
+ print(f"raw-sql-columns: root {result['root']}")
+ print(
+ f"read {result['files_read']} module(s) under services/*/app, "
+ f"{result['migration_files']} migration file(s) declaring "
+ f"{result['schema_tables']} table(s) / {result['schema_columns']} column(s)"
+ )
+ print(
+ f"parsed {len(result['statements'])} raw statement(s) against "
+ f"{len(result['tables_touched'])} table(s); {result['compared']} compared, "
+ f"{result['recorded']} recorded as unmigrated or runtime-assembled"
+ )
+
+ if args.list:
+ print("\nEvery statement read:")
+ for statement in result["statements"]:
+ columns = ", ".join(statement["columns"]) or "(no column list)"
+ print(f" {statement['file']}:{statement['line']} {statement['kind']} {statement['table']}")
+ print(f" {columns}")
+
+ if args.credits:
+ print("\nWhat each verdict was based on:")
+ for credit in result["credits"]:
+ print(f" [{credit['verdict']}] {credit['file']}:{credit['line']} {credit['kind']} {credit['table']}")
+ print(f" {credit['detail']}")
+
+ if not result["statements"] or not result["compared"]:
+ # The corpus refusal. A tree this parses no SQL out of produces no
+ # findings, and that is not a clean repository — it is a scan that
+ # read nothing, which must never print as a pass. It also catches the
+ # extraction above going wrong: a renamed helper, a changed string
+ # shape, a glob that stopped matching. None of them removes
+ # `services/`.
+ print(
+ f"\nraw-sql-columns: compared 0 raw statements under {result['root']}/services. Zero statements "
+ f"compared is not zero drift; check the root above is the tree you meant.",
+ file=sys.stderr,
+ )
+ return 1
+
+ if result["findings"]:
+ print(f"\nFAIL: {len(result['findings'])} raw SQL statement(s) naming something no migration creates:", file=sys.stderr)
+ for finding in result["findings"]:
+ print(
+ f" [{finding['kind_of_finding']}] {finding['file']}:{finding['line']} "
+ f"{finding['table']}.{finding['column']}\n {finding['detail']}",
+ file=sys.stderr,
+ )
+ print(
+ "\nFix: add the table or column in a new migration (the direction that breaks at runtime), "
+ "correct the statement, or — only for SQL genuinely assembled at runtime — record it in "
+ "DYNAMIC_SQL in this file with the reason.",
+ file=sys.stderr,
+ )
+
+ if result["stale"]:
+ print("\nFAIL: recorded exceptions that nothing needs any more:", file=sys.stderr)
+ for entry in result["stale"]:
+ print(f" {entry}", file=sys.stderr)
+ print("\nFix: delete them. This list only shrinks.", file=sys.stderr)
+
+ return 1 if (result["findings"] or result["stale"]) else 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/tests/test_check_raw_sql_columns.py b/scripts/tests/test_check_raw_sql_columns.py
new file mode 100644
index 000000000..92df2144a
--- /dev/null
+++ b/scripts/tests/test_check_raw_sql_columns.py
@@ -0,0 +1,193 @@
+"""The raw-SQL column gate detects what it claims, and only that.
+
+Proven against the two shapes that reached `main` — a table named wrongly with
+a name that *contains* the right one, and a column no migration creates — and
+against the shapes that would make the gate a nuisance if it fired on them: a
+correct statement, and SQL the source assembles at runtime and records why.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import subprocess
+import sys
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+GATE = REPO_ROOT / "scripts" / "check_raw_sql_columns.py"
+
+MIGRATION = """
+CREATE TABLE IF NOT EXISTS widgets (
+ id UUID PRIMARY KEY,
+ -- a comment line, which the shared parser used to read as a column name
+ tenant_id UUID NOT NULL,
+ name VARCHAR(200) NOT NULL
+);
+
+ALTER TABLE widgets
+ ADD COLUMN IF NOT EXISTS label TEXT,
+ ADD COLUMN IF NOT EXISTS retired BOOLEAN NOT NULL DEFAULT FALSE;
+"""
+
+
+def _load():
+ spec = importlib.util.spec_from_file_location("check_raw_sql_columns", GATE)
+ assert spec is not None and spec.loader is not None
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[spec.name] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+def _tree(tmp_path: Path, statement: str, migration: str = MIGRATION) -> Path:
+ """A one-service tree: a module holding `statement`, and `migration`."""
+ root = tmp_path / "tree"
+ service = root / "services" / "widgetry"
+ (service / "app" / "api").mkdir(parents=True)
+ (service / "app" / "api" / "widgets.py").write_text(statement, encoding="utf-8")
+ (service / "migrations").mkdir(parents=True)
+ (service / "migrations" / "001_init.sql").write_text(migration, encoding="utf-8")
+ return root
+
+
+def _kinds(result: dict) -> list[str]:
+ return [f["kind_of_finding"] for f in result["findings"]]
+
+
+def test_an_insert_naming_a_table_no_migration_creates_fails(tmp_path) -> None:
+ """The defect, in the spelling it actually had.
+
+ `aisoc_detection_rule_proposals` contains `detection_rule_proposals`, so
+ the substring assertion this gate was written beside passed on it.
+ """
+ module = _load()
+ statement = 'SQL = "INSERT INTO aisoc_widgets (id, tenant_id, name) VALUES (:id, :tid, :name)"\n'
+ result = module.scan(_tree(tmp_path, statement))
+ assert _kinds(result) == ["unknown-table"]
+ assert result["findings"][0]["table"] == "aisoc_widgets"
+ assert result["compared"] == 0
+
+
+def test_an_insert_naming_a_column_no_migration_creates_fails(tmp_path) -> None:
+ """`detection_rule_proposals.source`, reduced to its essentials."""
+ module = _load()
+ statement = 'SQL = "INSERT INTO widgets (id, tenant_id, source) VALUES (:id, :tid, :src)"\n'
+ result = module.scan(_tree(tmp_path, statement))
+ assert _kinds(result) == ["unknown-column"]
+ assert result["findings"][0]["column"] == "source"
+ # The statement is still credited: the table resolved and the other two
+ # columns were compared, which is what `--credits` has to be able to show.
+ assert result["compared"] == 1
+
+
+def test_a_correct_insert_passes(tmp_path) -> None:
+ module = _load()
+ statement = 'SQL = "INSERT INTO widgets (id, tenant_id, name) VALUES (:id, :tid, :name)"\n'
+ result = module.scan(_tree(tmp_path, statement))
+ assert result["findings"] == []
+ assert result["compared"] == 1
+
+
+def test_a_column_added_by_a_multi_clause_alter_table_is_credited(tmp_path) -> None:
+ """The parser gap that would have made this gate unusable.
+
+ `ALTER TABLE t ADD COLUMN a, ADD COLUMN b` is one statement with two
+ clauses. Reading only the first credited `alerts` with one of three new
+ columns and `aisoc_run_costs` with one of five, so eleven perfectly
+ ordinary writes read as naming columns that do not exist.
+ """
+ module = _load()
+ statement = 'SQL = "INSERT INTO widgets (id, label, retired) VALUES (:id, :l, :r)"\n'
+ result = module.scan(_tree(tmp_path, statement))
+ assert result["findings"] == []
+
+
+def test_an_update_naming_a_column_no_migration_creates_fails(tmp_path) -> None:
+ module = _load()
+ statement = 'SQL = "UPDATE widgets SET label = :l, ghost = :g WHERE id = :id"\n'
+ result = module.scan(_tree(tmp_path, statement))
+ assert _kinds(result) == ["unknown-column"]
+ assert result["findings"][0]["column"] == "ghost"
+
+
+def test_a_sql_comment_inside_a_set_clause_does_not_read_as_dynamic(tmp_path) -> None:
+ """Three real upserts were reported as runtime-assembled for this reason."""
+ module = _load()
+ statement = 'SQL = """UPDATE widgets\n SET label = :l,\n -- a note, with a comma in it\n name = :n\n WHERE id = :id"""\n'
+ result = module.scan(_tree(tmp_path, statement))
+ assert result["findings"] == []
+ assert result["compared"] == 1
+
+
+def test_a_runtime_assembled_statement_is_a_finding_until_it_is_recorded(tmp_path) -> None:
+ module = _load()
+ statement = 'SQL = f"UPDATE widgets SET {assignments} WHERE id = :id"\n'
+ result = module.scan(_tree(tmp_path, statement))
+ assert _kinds(result) == ["unparsed-statement"]
+
+
+def test_the_allow_list_suppresses_a_named_dynamic_statement(tmp_path) -> None:
+ module = _load()
+ statement = 'SQL = f"UPDATE widgets SET {assignments} WHERE id = :id"\n'
+ module.DYNAMIC_SQL = {("services/widgetry/app/api/widgets.py", "widgets"): "assembled from the request body"}
+ result = module.scan(_tree(tmp_path, statement))
+ assert result["findings"] == []
+ assert result["recorded"] == 1
+
+
+def test_the_allow_list_does_not_forgive_a_readable_statement_beside_a_dynamic_one(tmp_path) -> None:
+ """An entry excuses the statement it names, not the file it lives in.
+
+ Keyed on (file, table), a blanket exemption would have covered every
+ other write in the same module against the same table — which is how an
+ allow-list stops being a record and becomes a hole.
+ """
+ module = _load()
+ statement = 'ONE = f"UPDATE widgets SET {assignments} WHERE id = :id"\nTWO = "INSERT INTO widgets (id, ghost) VALUES (:id, :g)"\n'
+ module.DYNAMIC_SQL = {("services/widgetry/app/api/widgets.py", "widgets"): "assembled from the request body"}
+ result = module.scan(_tree(tmp_path, statement))
+ assert _kinds(result) == ["unknown-column"]
+ assert result["findings"][0]["column"] == "ghost"
+
+
+def test_an_allow_list_entry_nothing_needs_is_itself_a_finding(tmp_path) -> None:
+ """The list only shrinks, so it is verified in both directions."""
+ module = _load()
+ statement = 'SQL = "INSERT INTO widgets (id, tenant_id, name) VALUES (:id, :tid, :name)"\n'
+ module.DYNAMIC_SQL = {("services/widgetry/app/api/nowhere.py", "widgets"): "no longer true"}
+ module.UNMIGRATED_TABLES = {"sprockets": "another store, once"}
+ result = module.scan(_tree(tmp_path, statement))
+ assert result["findings"] == []
+ assert len(result["stale"]) == 2
+
+
+def test_a_tree_the_gate_parses_no_sql_out_of_is_refused(tmp_path) -> None:
+ """A gate that reads nothing finds nothing, so `main()` has to refuse it.
+
+ The shape that actually happens: `services/` is there, the modules are
+ there, and the extraction has stopped matching. Nothing removes
+ `services/`.
+ """
+ module = _load()
+ root = _tree(tmp_path, "VALUE = 1\n")
+ assert module.scan(root)["statements"] == []
+ assert module.main(["--repo-root", str(root)]) != 0
+
+
+def test_a_missing_services_directory_is_refused_rather_than_called_clean(tmp_path) -> None:
+ module = _load()
+ root = tmp_path / "bare"
+ root.mkdir()
+ assert "error" in module.scan(root)
+ assert module.main(["--repo-root", str(root)]) == 2
+
+
+def test_the_gate_answers_the_self_test_flag() -> None:
+ result = subprocess.run([sys.executable, str(GATE), "--self-test"], capture_output=True, text=True)
+ assert result.returncode == 0, result.stdout + result.stderr
+
+
+def test_the_live_repository_is_clean() -> None:
+ """The regression this gate exists for, asserted against the real tree."""
+ result = subprocess.run([sys.executable, str(GATE)], capture_output=True, text=True)
+ assert result.returncode == 0, result.stdout + result.stderr
diff --git a/services/api/migrations/073_detection_proposal_source.sql b/services/api/migrations/073_detection_proposal_source.sql
new file mode 100644
index 000000000..ac0d9b317
--- /dev/null
+++ b/services/api/migrations/073_detection_proposal_source.sql
@@ -0,0 +1,29 @@
+-- AiSOC: the column three proposal writers already name
+-- Author: Beenu - [email protected]
+--
+-- `detection_rule_proposals` has never had a `source` column, and three raw
+-- INSERT statements name it:
+--
+-- * app/api/v1/endpoints/rule_tuning.py writes 'auto-tuner'
+-- * app/workers/hunt_scheduler.py writes 'hunt-finding'
+-- * app/api/v1/endpoints/detection_loop.py writes 'detection-loop'
+--
+-- Each raises UndefinedColumnError on the first row it tries to write, so
+-- none of the three automated authoring paths can produce a proposal on any
+-- deployment. Nothing caught it: no CI job runs `services/api/tests/`
+-- against a real database, and the one test covering the insert asserted
+-- only that the table name appeared somewhere in the SQL string.
+--
+-- `scripts/check_raw_sql_columns.py` found it, and now holds raw SQL to the
+-- same table-and-column contract `check_orm_migration_parity.py` already
+-- holds the ORM models to.
+--
+-- Nullable with no default: rows written before this migration have no
+-- provenance to claim, and NULL is the honest value for "not recorded" —
+-- distinct from a proposal an analyst authored by hand.
+
+ALTER TABLE detection_rule_proposals
+ ADD COLUMN IF NOT EXISTS source TEXT;
+
+COMMENT ON COLUMN detection_rule_proposals.source IS
+ 'Which automated path authored this proposal: auto-tuner (analyst disposition history), hunt-finding (a scheduled hunt that hit), detection-loop (a false-positive fix). NULL for a hand-authored proposal or any row predating this column.';
diff --git a/services/api/tests/test_wave1_loop_edges.py b/services/api/tests/test_wave1_loop_edges.py
index 61d078738..d0133c69d 100644
--- a/services/api/tests/test_wave1_loop_edges.py
+++ b/services/api/tests/test_wave1_loop_edges.py
@@ -8,6 +8,7 @@ they run fast and deterministically.
from __future__ import annotations
+import re
from types import SimpleNamespace
from unittest.mock import AsyncMock
from uuid import uuid4
@@ -38,7 +39,14 @@ async def test_hunt_hits_open_a_detection_proposal(monkeypatch):
await hunt_scheduler._propose_detection_from_hunt(db, _hunt(), 5)
assert db.execute.await_count == 1
sql = str(db.execute.await_args.args[0])
- assert "detection_rule_proposals" in sql
+ # The parsed table name, not a substring of the SQL. `"detection_rule_
+ # proposals" in sql` also passes on `aisoc_detection_rule_proposals`,
+ # which contains the right name inside the wrong one — and that exact
+ # confusion is recorded history here: three insert sites and two tests had
+ # codified the `aisoc_`-prefixed spelling of this very table.
+ inserted = re.search(r"INSERT\s+INTO\s+(\w+)", sql)
+ assert inserted is not None, sql
+ assert inserted.group(1) == "detection_rule_proposals"
@pytest.mark.asyncio