Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
feat(gates): hold raw SQL to the same column contract the ORM already has (#970)
Beenu Aroraff4129fd1fd8
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex c563620fb..fb226b5b1 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -763,6 +763,26 @@ jobs:python3 scripts/check_orm_migration_parity.py --self-testpython3 scripts/check_orm_migration_parity.py --check+ # The same rule for the raw `text()` SQL the gate above cannot see —+ # 107 statements naming 44 tables, 35 of which have no ORM model at+ # all, so raw SQL is the only thing that writes them.+ # `detection_rule_proposals` had no `source` column and three inserts+ # named it, so the auto-tuner, the scheduled-hunt bridge and the+ # false-positive loop could not write a proposal on any deployment.+ # Nothing caught it: no job runs `services/api/tests/` against a real+ # database, and the one test covering the insert asserted only that the+ # table name appeared somewhere in the SQL string — which the wrong+ # name `aisoc_detection_rule_proposals` also satisfies.+ #+ # The self-test runs first, and deliberately: it injects a wrong table,+ # a wrong column and a runtime-assembled statement into scratch trees,+ # so a parser that has quietly stopped matching cannot report the+ # repository clean.+ - name: Verify raw SQL names only tables and columns migrations create+ run: |+ python3 scripts/check_raw_sql_columns.py --self-test+ python3 scripts/check_raw_sql_columns.py+# The four alembic chains have to be invoked by something. They were# not: every container command was a plain `uvicorn`, so the# documented quickstart booted two services with no tables and no RLSdiff --git a/scripts/check_orm_migration_parity.py b/scripts/check_orm_migration_parity.pyindex c15e77f00..4ca7eea7e 100644--- a/scripts/check_orm_migration_parity.py+++ b/scripts/check_orm_migration_parity.py@@ -137,6 +137,21 @@ def _string(node: ast.AST) -> str | None:return node.value if isinstance(node, ast.Constant) and isinstance(node.value, str) else None+def _rel(path: pathlib.Path) -> str:+ """``path`` relative to the repository, or as given when it is outside it.++ The parsers below are reusable — ``check_raw_sql_columns`` reads the same+ migrations through them — and a reusable parser has to survive being+ pointed at a scratch tree. ``relative_to`` raises there, which would turn+ "this gate was handed a temporary directory" into a traceback rather than+ a scan. Inside the repository the output is unchanged.+ """+ try:+ return str(path.relative_to(REPO_ROOT))+ except ValueError:+ return str(path)++#: SQL type name → family. Two spellings of the same family are not drift#: (``Text`` against ``String``, ``BigInteger`` against ``Integer``); two#: families are, and one such mismatch is why this mapping exists.@@ -289,7 +304,7 @@ def model_tables(paths: list[pathlib.Path]) -> dict[str, dict]:if tablename and columns:tables[tablename] = {"columns": columns,- "source": str(path.relative_to(REPO_ROOT)),+ "source": _rel(path),"model": node.name,}return tables@@ -302,17 +317,43 @@ def model_tables(paths: list[pathlib.Path]) -> dict[str, dict]:#: ``CREATE TABLE [IF NOT EXISTS] name ( ... )`` — the body is scanned for#: leading identifiers, one per column definition._CREATE_TABLE = re.compile(r"CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?[\"']?(\w+)[\"']?\s*\((.*?)\);", re.I | re.S)-_ADD_COLUMN = re.compile(- r"ALTER\s+TABLE\s+(?:ONLY\s+)?[\"']?(\w+)[\"']?\s+ADD\s+COLUMN\s+"- r"(?:IF\s+NOT\s+EXISTS\s+)?[\"']?(\w+)[\"']?\s*([A-Za-z]*)\s*(?:\((\d+)\))?",- re.I,-)-_DROP_COLUMN = re.compile(r"ALTER\s+TABLE\s+(?:ONLY\s+)?[\"']?(\w+)[\"']?\s+DROP\s+COLUMN\s+(?:IF\s+EXISTS\s+)?[\"']?(\w+)[\"']?", re.I)-_ALTER_TYPE = re.compile(- r"ALTER\s+TABLE\s+(?:ONLY\s+)?[\"']?(\w+)[\"']?\s+ALTER\s+COLUMN\s+[\"']?(\w+)[\"']?\s+"- r"(?:SET\s+DATA\s+)?TYPE\s+([A-Za-z]+)\s*(?:\((\d+)\))?",- re.I,-)+#: ``ALTER TABLE <name> <clauses>;`` — the target, then everything up to the+#: statement terminator, because one ``ALTER TABLE`` may carry several+#: comma-separated clauses.+#:+#: This was four regexes, each anchored on its own ``ALTER TABLE`` prefix,+#: which between them read only the *first* clause of any statement.+#: ``040_critical_severity_alert_confidence.sql`` adds three columns to+#: ``alerts`` in one statement and this parser credited the table with one;+#: ``063_cost_provenance.sql`` adds five to ``aisoc_run_costs`` and it credited+#: one. Anchoring on the statement and then reading its clause list is the only+#: shape that does not depend on how the author chose to punctuate.+_ALTER_TABLE = re.compile(r"ALTER\s+TABLE\s+(?:IF\s+EXISTS\s+)?(?:ONLY\s+)?[\"']?(\w+)[\"']?\s+(.*?)(?:;|\Z)", re.I | re.S)+_CLAUSE_ADD = re.compile(r"\bADD\s+COLUMN\s+(?:IF\s+NOT\s+EXISTS\s+)?[\"']?(\w+)[\"']?\s*([A-Za-z]*)\s*(?:\((\d+)\))?", re.I)+_CLAUSE_DROP = re.compile(r"\bDROP\s+COLUMN\s+(?:IF\s+EXISTS\s+)?[\"']?(\w+)[\"']?", re.I)+_CLAUSE_RENAME = re.compile(r"\bRENAME\s+COLUMN\s+[\"']?(\w+)[\"']?\s+TO\s+[\"']?(\w+)[\"']?", re.I)+_CLAUSE_TYPE = re.compile(r"\bALTER\s+COLUMN\s+[\"']?(\w+)[\"']?\s+(?:SET\s+DATA\s+)?TYPE\s+([A-Za-z]+)\s*(?:\((\d+)\))?", re.I)++_SQL_COMMENT = re.compile(r"--[^\n]*|/\*.*?\*/", re.S)+++def strip_sql_comments(sql: str) -> str:+ """``sql`` with ``--`` and ``/* */`` comments removed, line structure kept.++ Any parser that splits a clause list on commas has to do this first, and+ both readers of SQL in this tree learned it the hard way.+ ``_split_columns`` reads the first token of each comma-separated part as a+ column name, so a ``--`` comment line sits inside the part that *follows*+ it and hides the column behind it: ``detection_rule_proposals`` documents+ four of its columns that way and this parser credited the table with none+ of them. ``check_raw_sql_columns`` hit the same thing from the statement+ side, where a comment inside a ``SET`` clause made three perfectly+ ordinary upserts read as assembled at runtime.++ Newlines survive so a comment cannot glue two statements together.+ """+ return _SQL_COMMENT.sub("", sql)+#: Words that begin a table-level constraint rather than a column._CONSTRAINT_WORDS = frozenset({"primary", "unique", "foreign", "constraint", "check", "exclude", "like"})@@ -366,19 +407,40 @@ def _record(store.setdefault(table, {})[column] = (family, length)-def _scan_sql(text: str, store: dict[str, dict[str, tuple[str | None, int | None]]], dropped: set[tuple[str, str]]) -> None:+def _scan_sql(raw: str, store: dict[str, dict[str, tuple[str | None, int | None]]], dropped: set[tuple[str, str]]) -> None:+ text = strip_sql_comments(raw)for table, body in _CREATE_TABLE.findall(text):for column, family, length in _split_columns(body):_record(store, table, column, family, length)- for table, column, type_name, length in _ADD_COLUMN.findall(text):- _record(store, table, column, _family(type_name), int(length) if length else None)- for table, column, type_name, length in _ALTER_TYPE.findall(text):- # Only widens what is already known; an ALTER TYPE on a column this- # scan never saw created is still a column it never saw created.- if column in store.get(table, {}):+ # Each ALTER TABLE in document order, then its clauses, so a statement+ # that adds a column and a later one that renames it compose the way the+ # database would apply them.+ for table, clauses in _ALTER_TABLE.findall(text):+ for column, type_name, length in _CLAUSE_ADD.findall(clauses):_record(store, table, column, _family(type_name), int(length) if length else None)- for table, column in _DROP_COLUMN.findall(text):- dropped.add((table, column))+ for column, type_name, length in _CLAUSE_TYPE.findall(clauses):+ # Only widens what is already known; an ALTER TYPE on a column+ # this scan never saw created is still a column it never saw+ # created.+ if column in store.get(table, {}):+ _record(store, table, column, _family(type_name), int(length) if length else None)+ # A renamed column is the same column under a new name, and reading+ # only the create statement leaves the parser holding the old one.+ # Migration 025 renames four columns on `connectors` — `config` to+ # `connector_config` among them — and every reader in the tree names+ # the new spelling, so a parser that never applies the rename reports+ # four columns that do not exist and misses four that do.+ for before, after in _CLAUSE_RENAME.findall(clauses):+ columns = store.get(table)+ if columns is None:+ continue+ # The rename is itself proof the column is there, so a column+ # this parser never saw created still lands under its new name+ # rather than being silently dropped.+ columns[after] = columns.pop(before, (None, None))+ dropped.discard((table, before))+ for column in _CLAUSE_DROP.findall(clauses):+ dropped.add((table, column))def _scan_alembic_module(tree: ast.AST, store: dict[str, dict[str, tuple[str | None, int | None]]], dropped: set[tuple[str, str]]) -> None:@@ -461,12 +523,12 @@ def migration_columns(service: pathlib.Path) -> tuple[dict[str, dict[str, tuple[tree = _upgrade_only(path)if tree is not None:_scan_alembic_module(tree, store, dropped)- read.append(str(path.relative_to(REPO_ROOT)))+ read.append(_rel(path))for name in SQL_MIGRATION_DIRS:for path in sorted((service / name).glob("*.sql")):_scan_sql(path.read_text(encoding="utf-8", errors="replace"), store, dropped)- read.append(str(path.relative_to(REPO_ROOT)))+ read.append(_rel(path))return store, dropped, read@@ -508,7 +570,7 @@ def creates_tables_from_metadata(paths: list[pathlib.Path]) -> str | None:continuefor node in ast.walk(tree):if isinstance(node, ast.Attribute) and node.attr == "create_all":- return str(path.relative_to(REPO_ROOT))+ return _rel(path)return Nonediff --git a/scripts/check_raw_sql_columns.py b/scripts/check_raw_sql_columns.pynew file mode 100644index 000000000..fc56c7bf9--- /dev/null+++ b/scripts/check_raw_sql_columns.py@@ -0,0 +1,772 @@+#!/usr/bin/env python3+"""Every table and column a raw SQL statement names must be one a migration creates.++Why this exists+---------------++``services/api/app/api/v1/endpoints/rule_tuning.py`` inserts an auto-tuner+proposal with a raw ``sqlalchemy.text()`` statement naming fourteen columns of+``detection_rule_proposals``. One of them, ``source``, is created by no+migration in this repository — not in ``010_detection_as_code.sql`` where the+table is defined, not in ``039_detection_proposal_github_pr.sql`` where it is+altered, not anywhere. The statement raises ``UndefinedColumnError`` on the+first row it tries to write, so the human-approval-gated tuning loop cannot+produce a single proposal on any deployment. This gate found it.++Nothing about that needed a database. The statement and the migrations are+both in the tree and the disagreement is structural, which is the same+observation ``check_orm_migration_parity.py`` was written on: UEBA declared+``EntityBaseline.peer_group_id``, migration ``0001`` never created it, and+UEBA could not write a baseline or an anomaly from the day the column was+added. That gate reads ORM-declared models. A raw ``text()`` string is+invisible to it, so the identical failure class had no gate at all over 107+raw statements naming 44 tables — and 35 of those 44 have no ORM model in the+tree at all, so raw SQL is the only thing that writes them and nothing was+checking it.++Two things hid it for as long as they did.++*The claim that a real database already exercised it.* The claim-to-gate+matrix recorded the analyst-disposition tuning loop as ``PARTIAL`` with the+gap "end-to-end proposal insert exercised in api integration, not a unit DB",+which reads as thin coverage of something that does run. No job runs+``services/api/tests/`` against a real database. ``Python — Tests`` in+``ci.yml`` declares no ``postgres`` service and sets+``DATABASE_URL=postgresql+asyncpg://x:x@localhost/x``, a deliberate+non-database. ``integration.yml`` and ``upgrade-test.yml`` are the only+workflows with a live Postgres, and the single API test either of them runs is+``services/api/tests/test_mssp_portfolio_isolation.py``. The insert had never+touched a database anywhere.++*A test that greps for the table name.* ``test_wave1_loop_edges.py`` asserted+``"detection_rule_proposals" in str(db.execute.await_args.args[0])``. Being a+substring test it also passes on ``aisoc_detection_rule_proposals``, which+contains the right name inside the wrong one — and that exact confusion is+recorded history here: three insert sites and two tests had codified the+``aisoc_``-prefixed spelling of this very table. A mocked session cannot fail+on a column, so the assertion had nothing else to check.++What it checks, in both directions+----------------------------------++``unknown-table``+ ``INSERT INTO t`` or ``UPDATE t SET`` naming a table no migration creates.+ This is the ``aisoc_detection_rule_proposals`` class exactly, and unlike+ the substring assertion it cannot be satisfied by a name that merely+ contains the right one.++``unknown-column``+ A column in an ``INSERT`` column list, or on the left of a ``SET`` in an+ ``UPDATE`` or an ``ON CONFLICT … DO UPDATE``, that no migration creates on+ that table. ``detection_rule_proposals.source`` is this.++``unparsed-statement``+ The other direction, and the one that decides whether this gate is worth+ anything: a statement whose table or column list is assembled at runtime+ contributes zero comparisons, and zero findings over zero comparisons+ prints the same word as a clean result. Rather than skipping those+ quietly, each is a finding until it is recorded in ``DYNAMIC_SQL`` below+ with a reason. The list is checked in both directions, so an entry nothing+ needs any more fails the build instead of sitting there looking like+ coverage.++``--credits`` exists for the same reason: it enumerates what each verdict was+based on, so a parser that has silently stopped matching can be caught by+reading rather than by trusting the exit code.++What it reads+-------------++Statements: Python ``ast`` over ``services/*/app/**/*.py``, every string+constant and f-string in the module rather than only the arguments of+``text()`` — the SQL in ``services/fusion/app/services/alert_sink.py`` and+``services/agents/app/hunt/store.py`` is a module-level constant that+something else wraps later, and a gate keyed on the call site would not see+either. An f-string renders with ``{}`` where its interpolations were, which+is how a runtime-assembled table or column list becomes visible as one.++Schema: the migration parsers in ``check_orm_migration_parity.py``, imported+rather than reimplemented. Two parsers of the same DDL drift the first time+either learns something the other has not, and this repository has the scars:+its graph-schema check declared OK while one side had 17 labels and the other+28. Reusing them also fixed two gaps that only a column-level reader would+notice — a ``--`` comment line swallowed the column definition after it, and+``ALTER TABLE … RENAME COLUMN`` was never applied, so four ``connectors``+columns were known to the parser under names nothing in the tree uses.++Scope: every service with an ``app/`` directory, which on today's tree is 893+modules yielding 107 statements — ``api`` 75, ``agents`` 20, ``fusion`` 5,+``actions`` 3, ``slack-bot`` 2, ``osquery-tls`` 1, ``ueba`` 1. Small enough to+triage one by one, so nothing is narrowed to keep the number down.++The schema is the union of every migration source in the repository — 85 SQL+files under ``services/api/migrations`` plus the four alembic chains, 131+tables and 1,582 columns — because it is one Postgres. ``services/agents`` and+``services/fusion`` own no migrations and write to tables ``services/api``+creates, so a per-service comparison would have reported every one of their+25 statements as unsourced.++Usage+-----++::++ python3 scripts/check_raw_sql_columns.py # gate+ python3 scripts/check_raw_sql_columns.py --list # every statement read+ python3 scripts/check_raw_sql_columns.py --credits # what it credited+ python3 scripts/check_raw_sql_columns.py --json+ python3 scripts/check_raw_sql_columns.py --self-test++Exit codes: 0 clean, 1 findings, 2 the scan itself could not run.+"""++from __future__ import annotations++import argparse+import ast+import contextlib+import io+import json+import pathlib+import re+import sys+import tempfile++# `scripts/` is on sys.path when this file is run as a program, but not when a+# test loads it by path with importlib. Both siblings sit beside it either way.+sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))++# `--self-test` is answered in `main()` rather than through+# `self_test_if_requested`, because that shortcut is for gates with nothing+# bespoke to add and this one has five injected-drift cases to run alongside+# the shared empty-tree refusal. Installing it here would exit before any of+# them, leaving a self-test that proves only half of what it prints.+from check_orm_migration_parity import migration_columns, strip_sql_comments+from gate_toolkit import repo_root, self_test_main++REPO_ROOT = repo_root()+SERVICES_DIR = REPO_ROOT / "services"++#: Directories under ``services/<svc>/app/`` that hold no production SQL.+#: Tests are excluded because a fixture may name a table on purpose to assert+#: what happens when it is wrong.+_SKIP_PARTS = frozenset({"tests", "test", "_vendor", "node_modules", "__pycache__"})++#: What an f-string interpolation renders as. Anything carrying this was+#: assembled at runtime and cannot be compared against a migration.+DYNAMIC = "{}"+++# ---------------------------------------------------------------------------+# Recorded exceptions+# ---------------------------------------------------------------------------+#+# Shrink-only, and verified in both directions: an entry that stops being+# needed fails the build rather than sitting here.+#+# Two lists rather than one, because they excuse genuinely different things+# and collapsing them would let either reason cover the other. A table no+# migration owns is a fact about the table, so it is keyed on the table and+# forgives the column check with it. A statement assembled at runtime is a+# fact about that one statement, so it is keyed on (file, table as written) —+# and it only ever excuses a statement this parser could not read, so an+# ordinary INSERT in an allow-listed file is still held to the contract.++#: Tables that exist but whose creation no migration under ``services/`` owns.+UNMIGRATED_TABLES: dict[str, str] = {+ "aisoc.raw_events": (+ "ClickHouse, not Postgres. The event lake's table is created by "+ "services/fusion/app/services/lake_writer.py with CREATE DATABASE / CREATE TABLE IF NOT EXISTS on "+ "startup — the ECS ClickHouse task does not run docker-entrypoint-initdb.d — so no migration "+ "declares it and no migration should."+ ),+ "aisoc_schema_migrations": (+ "The Postgres migration runner's own ledger, created by CREATE_MIGRATIONS_TABLE in "+ "services/api/app/scripts/run_migrations.py before any migration executes. A migration chain "+ "cannot be asked to declare the table that records which migrations ran — the same reason "+ "alembic's version table is in no revision."+ ),+}++#: Statements whose table or column list is assembled at runtime, and so+#: cannot be compared against anything, with the reason.+DYNAMIC_SQL: dict[tuple[str, str], str] = {+ (+ "services/api/app/db/lake_migrations.py",+ DYNAMIC,+ ): (+ "The ClickHouse lake's own migration ledger: the table is `MIGRATION_TABLE` "+ "('aisoc._migrations'), interpolated, and this module creates it. Same reason as "+ "`aisoc_schema_migrations` above, one store over."+ ),+ (+ "services/api/app/api/v1/endpoints/cases.py",+ "aisoc_cases",+ ): (+ "A PATCH handler: the SET clause is `', '.join(sets)` over whichever fields the request body "+ "supplied, so there is no column list in the source to compare. The columns it can append are "+ "literals in this function, which is the read a human has to do."+ ),+ (+ "services/api/app/api/v1/endpoints/cases.py",+ "aisoc_case_tasks",+ ): "A PATCH handler on case tasks, same shape as the one above.",+ (+ "services/api/app/api/v1/endpoints/hunts.py",+ "aisoc_hunts",+ ): "A PATCH handler on saved hunts, same shape as the two above.",+ (+ "services/api/app/scripts/seed_demo.py",+ DYNAMIC,+ ): (+ "The demo re-anchor sweep walks `_REANCHOR_TABLES` and shifts every timestamp column it names, so "+ "both the table and the assignment list are interpolated per iteration. Demo-mode seeding, not a "+ "production write path."+ ),+ (+ "services/osquery-tls/app/db/env.py",+ DYNAMIC,+ ): ("Alembic's own bookkeeping: the table is `VERSION_TABLE`, interpolated, and alembic creates it rather than a revision."),+}+++# ---------------------------------------------------------------------------+# Schema side+# ---------------------------------------------------------------------------+++def migration_schema(services_dir: pathlib.Path) -> tuple[dict[str, set[str]], dict[str, str], int]:+ """``({table: columns}, {table: owning service}, migration files read)``.++ The union across every service that ships migrations, because the services+ share one Postgres: ``services/agents`` writes ``investigation_runs`` and+ ``services/fusion`` writes ``alerts``, both created under+ ``services/api/migrations``.+ """+ schema: dict[str, set[str]] = {}+ owner: dict[str, str] = {}+ files = 0+ for service in sorted(p for p in services_dir.iterdir() if p.is_dir()):+ created, dropped, read = migration_columns(service)+ if not read:+ continue+ files += len(read)+ for table, columns in created.items():+ live = {c for c in columns if (table, c) not in dropped}+ if not live:+ continue+ schema.setdefault(table, set()).update(live)+ owner.setdefault(table, service.name)+ return schema, owner, files+++# ---------------------------------------------------------------------------+# Statement side+# ---------------------------------------------------------------------------++_INSERT = re.compile(r"\bINSERT\s+(?:OR\s+\w+\s+)?INTO\s+([^\s(;]+)", re.I)+_UPDATE = re.compile(r"\bUPDATE\s+(?:ONLY\s+)?([^\s(;]+)\s+SET\b", re.I)+_DO_UPDATE = re.compile(r"\bDO\s+UPDATE\s+SET\b", re.I)++#: Where a ``SET`` clause ends. ``FROM`` is included because Postgres allows+#: ``UPDATE t SET … FROM other``, and reading past it would collect the joined+#: table's columns as if they were assignments.+_SET_END = re.compile(r"\b(WHERE|RETURNING|FROM)\b", re.I)++_IDENTIFIER = re.compile(r"^[A-Za-z_]\w*$")+++def _render(node: ast.AST) -> str | None:+ """A string literal's text, with ``{}`` where an f-string interpolated.++ Rendering rather than skipping is what makes a runtime-assembled table+ name or column list *visible*. ``f"INSERT INTO {VERSION_TABLE} …"`` comes+ back as ``INSERT INTO {} …``, which this gate reports as unparsed instead+ of failing to notice.+ """+ if isinstance(node, ast.Constant):+ text = node.value if isinstance(node.value, str) else None+ elif isinstance(node, ast.JoinedStr):+ parts: list[str] = []+ for value in node.values:+ if isinstance(value, ast.Constant) and isinstance(value.value, str):+ parts.append(value.value)+ else:+ parts.append(DYNAMIC)+ text = "".join(parts)+ else:+ return None+ # Comments are stripped through the same helper the migration side uses.+ # Three ordinary upserts here read as assembled at runtime purely because+ # a `--` note sat inside their SET clause and the comma-split collected it+ # as if it were an assignment.+ return strip_sql_comments(text) if text else text+++def _balanced(text: str, start: int) -> tuple[str, int] | None:+ """The contents of the parenthesised group beginning at or after ``start``.++ Returns ``None`` when the next non-space character is not ``(``, which is+ how ``INSERT INTO t VALUES (…)`` — no column list — is told apart from+ ``INSERT INTO t (a, b) VALUES (…)``.+ """+ index = start+ while index < len(text) and text[index].isspace():+ index += 1+ if index >= len(text) or text[index] != "(":+ return None+ depth = 0+ for position in range(index, len(text)):+ if text[position] == "(":+ depth += 1+ elif text[position] == ")":+ depth -= 1+ if depth == 0:+ return text[index + 1 : position], position + 1+ return None+++def _split_top_level(body: str) -> list[str]:+ """``body`` split on commas outside any parentheses."""+ depth = 0+ current: list[str] = []+ parts: list[str] = []+ for char in body:+ if char == "(":+ depth += 1+ elif char == ")":+ depth -= 1+ if char == "," and depth == 0:+ parts.append("".join(current))+ current = []+ continue+ current.append(char)+ parts.append("".join(current))+ return parts+++def _column_names(parts: list[str]) -> tuple[list[str], bool]:+ """``(identifiers, every part was a plain identifier)``.++ The flag is the honest half. A part this cannot read — an expression, a+ cast, an interpolation — means the list is not fully accounted for, and+ comparing the identifiers it *did* read while staying silent about the+ rest would credit the statement with more scrutiny than it received.+ """+ names: list[str] = []+ complete = True+ for part in parts:+ token = part.strip().strip('"').strip()+ if _IDENTIFIER.match(token):+ names.append(token)+ elif token:+ complete = False+ return names, complete+++def _set_columns(text: str, start: int) -> tuple[list[str], bool]:+ """Assigned column names in the ``SET`` clause beginning at ``start``."""+ clause = text[start:]+ if end := _SET_END.search(clause):+ clause = clause[: end.start()]+ names: list[str] = []+ complete = True+ for part in _split_top_level(clause):+ left = part.split("=", 1)[0].strip().strip('"').strip()+ if not left:+ continue+ # `t.col = …` is still an assignment to `col`.+ if "." in left:+ left = left.rsplit(".", 1)[-1].strip('"')+ if _IDENTIFIER.match(left):+ names.append(left)+ else:+ complete = False+ return names, complete+++def statements_in(source: str, relpath: str) -> list[dict]:+ """Every raw ``INSERT`` / ``UPDATE`` in one module's string literals."""+ try:+ tree = ast.parse(source)+ except SyntaxError:+ return []++ found: list[dict] = []+ for node in ast.walk(tree):+ if not isinstance(node, ast.Constant | ast.JoinedStr):+ continue+ text = _render(node)+ if not text or not ("INSERT" in text.upper() or "UPDATE" in text.upper()):+ continue+ line = getattr(node, "lineno", 0)++ inserts: list[tuple[int, dict]] = []+ for match in _INSERT.finditer(text):+ group = _balanced(text, match.end())+ columns, complete = _column_names(_split_top_level(group[0])) if group else ([], True)+ statement = {+ "kind": "INSERT",+ "file": relpath,+ "line": line,+ "table": match.group(1).strip('"'),+ "columns": columns,+ # A statement with no column list names no column to check,+ # which is not the same as one whose list could not be read.+ "columns_declared": group is not None,+ "columns_complete": complete,+ }+ inserts.append((match.start(), statement))+ found.append(statement)++ for match in _UPDATE.finditer(text):+ columns, complete = _set_columns(text, match.end())+ found.append(+ {+ "kind": "UPDATE",+ "file": relpath,+ "line": line,+ "table": match.group(1).strip('"'),+ "columns": columns,+ "columns_declared": True,+ "columns_complete": complete,+ }+ )++ # `ON CONFLICT … DO UPDATE SET` assigns to the table of the INSERT it+ # belongs to, so it is the same column contract under another spelling+ # — and upserts are how most of this repository writes.+ for match in _DO_UPDATE.finditer(text):+ preceding = [statement for position, statement in inserts if position < match.start()]+ if not preceding:+ continue+ columns, complete = _set_columns(text, match.end())+ found.append(+ {+ "kind": "UPSERT",+ "file": relpath,+ "line": line,+ "table": preceding[-1]["table"],+ "columns": columns,+ "columns_declared": True,+ "columns_complete": complete,+ }+ )++ return found+++def service_statements(services_dir: pathlib.Path, root: pathlib.Path) -> tuple[list[dict], int]:+ """Statements across ``services/*/app/**/*.py``, and the file count read."""+ found: list[dict] = []+ read = 0+ for service in sorted(p for p in services_dir.iterdir() if p.is_dir()):+ app = service / "app"+ if not app.is_dir():+ continue+ for path in sorted(app.rglob("*.py")):+ if _SKIP_PARTS & set(path.relative_to(service).parts):+ continue+ read += 1+ try:+ relpath = str(path.relative_to(root))+ except ValueError:+ relpath = str(path)+ found.extend(statements_in(path.read_text(encoding="utf-8", errors="replace"), relpath))+ return found, read+++# ---------------------------------------------------------------------------+# Comparison+# ---------------------------------------------------------------------------+++def scan(root: pathlib.Path | None = None) -> dict:+ root = root or REPO_ROOT+ services_dir = root / "services"+ if not services_dir.is_dir():+ return {"error": f"no services directory at {services_dir}"}++ schema, owner, migration_files = migration_schema(services_dir)+ statements, files_read = service_statements(services_dir, root)++ findings: list[dict] = []+ credits: list[dict] = []+ foreign_used: set[str] = set()+ dynamic_used: set[tuple[str, str]] = set()++ for statement in statements:+ table = statement["table"]+ key = (statement["file"], table)++ if table in UNMIGRATED_TABLES:+ foreign_used.add(table)+ credits.append({**statement, "verdict": "unmigrated-table", "detail": UNMIGRATED_TABLES[table]})+ continue++ # Only a statement this parser could not read may be excused as+ # dynamic. A file holding one assembled statement does not get its+ # ordinary ones forgiven along with it.+ if DYNAMIC in table or not statement["columns_complete"]:+ if key in DYNAMIC_SQL:+ dynamic_used.add(key)+ credits.append({**statement, "verdict": "runtime-assembled", "detail": DYNAMIC_SQL[key]})+ continue+ findings.append(+ {+ **statement,+ "kind_of_finding": "unparsed-statement",+ "column": "*",+ "detail": (+ f"{statement['kind']} into {table!r} is assembled at runtime, so no table or column "+ "here was compared against a migration. Record it in DYNAMIC_SQL with the reason."+ ),+ }+ )+ continue++ if table not in schema:+ findings.append(+ {+ **statement,+ "kind_of_finding": "unknown-table",+ "column": "*",+ "detail": f"no migration creates a table named {table!r}. Every execution of this statement fails.",+ }+ )+ continue++ unknown = [column for column in statement["columns"] if column not in schema[table]]+ for column in unknown:+ findings.append(+ {+ **statement,+ "kind_of_finding": "unknown-column",+ "column": column,+ "detail": (+ f"{statement['kind']} names {table}.{column}; no migration creates it on that table "+ f"(migrations for {table} come from services/{owner[table]})."+ ),+ }+ )+ credits.append(+ {+ **statement,+ "verdict": "compared",+ "detail": f"{len(statement['columns'])} column(s) against services/{owner[table]} migrations",+ }+ )++ stale = [+ f"UNMIGRATED_TABLES[{name!r}] — no raw statement names that table any more"+ for name in UNMIGRATED_TABLES+ if name not in foreign_used+ ]+ stale += [+ f"DYNAMIC_SQL[{key!r}] — no statement there is assembled at runtime any more, so it is checkable"+ for key in DYNAMIC_SQL+ if key not in dynamic_used+ ]++ return {+ "root": str(root),+ "files_read": files_read,+ "migration_files": migration_files,+ "schema_tables": len(schema),+ "schema_columns": sum(len(c) for c in schema.values()),+ "statements": statements,+ "credits": credits,+ "compared": sum(1 for c in credits if c["verdict"] == "compared"),+ "recorded": sum(1 for c in credits if c["verdict"] != "compared"),+ "tables_touched": sorted({s["table"] for s in statements if DYNAMIC not in s["table"]}),+ "findings": findings,+ "stale": stale,+ }+++# ---------------------------------------------------------------------------+# Self-test+# ---------------------------------------------------------------------------++_GOOD_MIGRATION = """+CREATE TABLE IF NOT EXISTS widgets (+ id UUID PRIMARY KEY,+ -- a comment line, which used to swallow the column behind it+ tenant_id UUID NOT NULL,+ name VARCHAR(200) NOT NULL+);+"""++_GOOD_STATEMENT = """+from sqlalchemy import text++SQL = text("INSERT INTO widgets (id, tenant_id, name) VALUES (:id, :tid, :name)")+"""+++def _scratch(root: pathlib.Path, statement: str, migration: str = _GOOD_MIGRATION) -> dict:+ service = root / "services" / "widgetry"+ (service / "app" / "api").mkdir(parents=True)+ (service / "app" / "api" / "widgets.py").write_text(statement)+ (service / "migrations").mkdir(parents=True)+ (service / "migrations" / "001_init.sql").write_text(migration)+ return scan(root)+++def _detects_injected_drift() -> list[tuple[str, bool]]:+ """Prove the gate still catches each kind, on a tree built for the purpose.++ Run before the gate renders any verdict about the real tree, which is the+ point of a self-test: a gate whose parser has quietly stopped matching+ anything reports a clean repository in exactly the same words as a clean+ one.+ """++ def kinds(statement: str) -> set[str]:+ with tempfile.TemporaryDirectory() as raw:+ return {f["kind_of_finding"] for f in _scratch(pathlib.Path(raw), statement)["findings"]}++ def counted(statement: str) -> int:+ with tempfile.TemporaryDirectory() as raw:+ return _scratch(pathlib.Path(raw), statement)["compared"]++ # The defect this gate exists for, in the spelling it actually had: the+ # wrong name contains the right one, so the substring assertion in+ # `test_wave1_loop_edges.py` passed on it.+ wrong_table = _GOOD_STATEMENT.replace("INSERT INTO widgets", "INSERT INTO aisoc_widgets")+ wrong_column = _GOOD_STATEMENT.replace("id, tenant_id, name", "id, tenant_id, source")+ dynamic = 'SQL = f"INSERT INTO {TABLE} (id) VALUES (:id)"\n'++ return [+ ("detects a table no migration creates, on a name containing a real one", kinds(wrong_table) == {"unknown-table"}),+ ("detects a column no migration creates on that table", kinds(wrong_column) == {"unknown-column"}),+ ("passes a statement whose table and every column exist", kinds(_GOOD_STATEMENT) == set()),+ ("credits the passing statement rather than skipping it", counted(_GOOD_STATEMENT) == 1),+ ("refuses a statement assembled at runtime rather than skipping it", kinds(dynamic) == {"unparsed-statement"}),+ ]+++def _refuses_a_tree_with_no_statements() -> list[tuple[str, bool]]:+ """And a services tree that parses to no SQL at all.++ Separate from the above because it is the failure mode of this gate rather+ than of the tree: reading no statement has to be a refusal, or a parser+ that matches nothing reports every service clean. The scratch tree+ ``gate_toolkit`` builds has no ``services/`` directory at all; this is the+ other shape, where the directory is there and the corpus is gone.+ """+ with tempfile.TemporaryDirectory() as raw:+ root = pathlib.Path(raw)+ service = root / "services" / "widgetry"+ (service / "app").mkdir(parents=True)+ (service / "app" / "nothing.py").write_text("VALUE = 1\n")+ (service / "migrations").mkdir(parents=True)+ (service / "migrations" / "001_init.sql").write_text(_GOOD_MIGRATION)+ # The probe runs the real `main`, so its refusal is the real refusal+ # rather than a re-implementation of one — captured only so the+ # self-test's own output stays readable.+ sink = io.StringIO()+ with contextlib.redirect_stdout(sink), contextlib.redirect_stderr(sink):+ refused = main(["--repo-root", str(root)]) != 0++ return [("refuses a services tree it parsed no SQL out of", refused)]+++# ---------------------------------------------------------------------------+# Entry point+# ---------------------------------------------------------------------------+++def main(argv: list[str] | None = None) -> int:+ parser = argparse.ArgumentParser(description="Raw SQL / migration table and column parity")+ parser.add_argument("--list", action="store_true", help="print every statement read")+ parser.add_argument("--credits", action="store_true", help="print what each verdict was based on")+ parser.add_argument("--json", action="store_true", help="machine-readable output")+ parser.add_argument("--repo-root", help="the tree to inspect, instead of this checkout")+ parser.add_argument("--self-test", action="store_true", help="prove this gate still detects drift")+ args = parser.parse_args(argv)++ if args.self_test:+ return self_test_main(+ pathlib.Path(__file__).name,+ [],+ extra=[*_detects_injected_drift(), *_refuses_a_tree_with_no_statements()],+ )++ result = scan(pathlib.Path(args.repo_root).resolve() if args.repo_root else None)+ if "error" in result:+ print(f"raw-sql-columns: {result['error']}", file=sys.stderr)+ return 2++ if args.json:+ print(json.dumps(result, indent=2, sort_keys=True))++ # Naming the root and the corpus size is not decoration. A gate that+ # printed "OK" without them could be reporting on a different checkout,+ # or on none: `found nothing` and `scanned nothing` print the same word.+ if not args.json:+ print(f"raw-sql-columns: root {result['root']}")+ print(+ f"read {result['files_read']} module(s) under services/*/app, "+ f"{result['migration_files']} migration file(s) declaring "+ f"{result['schema_tables']} table(s) / {result['schema_columns']} column(s)"+ )+ print(+ f"parsed {len(result['statements'])} raw statement(s) against "+ f"{len(result['tables_touched'])} table(s); {result['compared']} compared, "+ f"{result['recorded']} recorded as unmigrated or runtime-assembled"+ )++ if args.list:+ print("\nEvery statement read:")+ for statement in result["statements"]:+ columns = ", ".join(statement["columns"]) or "(no column list)"+ print(f" {statement['file']}:{statement['line']} {statement['kind']} {statement['table']}")+ print(f" {columns}")++ if args.credits:+ print("\nWhat each verdict was based on:")+ for credit in result["credits"]:+ print(f" [{credit['verdict']}] {credit['file']}:{credit['line']} {credit['kind']} {credit['table']}")+ print(f" {credit['detail']}")++ if not result["statements"] or not result["compared"]:+ # The corpus refusal. A tree this parses no SQL out of produces no+ # findings, and that is not a clean repository — it is a scan that+ # read nothing, which must never print as a pass. It also catches the+ # extraction above going wrong: a renamed helper, a changed string+ # shape, a glob that stopped matching. None of them removes+ # `services/`.+ print(+ f"\nraw-sql-columns: compared 0 raw statements under {result['root']}/services. Zero statements "+ f"compared is not zero drift; check the root above is the tree you meant.",+ file=sys.stderr,+ )+ return 1++ if result["findings"]:+ print(f"\nFAIL: {len(result['findings'])} raw SQL statement(s) naming something no migration creates:", file=sys.stderr)+ for finding in result["findings"]:+ print(+ f" [{finding['kind_of_finding']}] {finding['file']}:{finding['line']} "+ f"{finding['table']}.{finding['column']}\n {finding['detail']}",+ file=sys.stderr,+ )+ print(+ "\nFix: add the table or column in a new migration (the direction that breaks at runtime), "+ "correct the statement, or — only for SQL genuinely assembled at runtime — record it in "+ "DYNAMIC_SQL in this file with the reason.",+ file=sys.stderr,+ )++ if result["stale"]:+ print("\nFAIL: recorded exceptions that nothing needs any more:", file=sys.stderr)+ for entry in result["stale"]:+ print(f" {entry}", file=sys.stderr)+ print("\nFix: delete them. This list only shrinks.", file=sys.stderr)++ return 1 if (result["findings"] or result["stale"]) else 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/scripts/tests/test_check_raw_sql_columns.py b/scripts/tests/test_check_raw_sql_columns.pynew file mode 100644index 000000000..92df2144a--- /dev/null+++ b/scripts/tests/test_check_raw_sql_columns.py@@ -0,0 +1,193 @@+"""The raw-SQL column gate detects what it claims, and only that.++Proven against the two shapes that reached `main` — a table named wrongly with+a name that *contains* the right one, and a column no migration creates — and+against the shapes that would make the gate a nuisance if it fired on them: a+correct statement, and SQL the source assembles at runtime and records why.+"""++from __future__ import annotations++import importlib.util+import subprocess+import sys+from pathlib import Path++REPO_ROOT = Path(__file__).resolve().parents[2]+GATE = REPO_ROOT / "scripts" / "check_raw_sql_columns.py"++MIGRATION = """+CREATE TABLE IF NOT EXISTS widgets (+ id UUID PRIMARY KEY,+ -- a comment line, which the shared parser used to read as a column name+ tenant_id UUID NOT NULL,+ name VARCHAR(200) NOT NULL+);++ALTER TABLE widgets+ ADD COLUMN IF NOT EXISTS label TEXT,+ ADD COLUMN IF NOT EXISTS retired BOOLEAN NOT NULL DEFAULT FALSE;+"""+++def _load():+ spec = importlib.util.spec_from_file_location("check_raw_sql_columns", GATE)+ assert spec is not None and spec.loader is not None+ module = importlib.util.module_from_spec(spec)+ sys.modules[spec.name] = module+ spec.loader.exec_module(module)+ return module+++def _tree(tmp_path: Path, statement: str, migration: str = MIGRATION) -> Path:+ """A one-service tree: a module holding `statement`, and `migration`."""+ root = tmp_path / "tree"+ service = root / "services" / "widgetry"+ (service / "app" / "api").mkdir(parents=True)+ (service / "app" / "api" / "widgets.py").write_text(statement, encoding="utf-8")+ (service / "migrations").mkdir(parents=True)+ (service / "migrations" / "001_init.sql").write_text(migration, encoding="utf-8")+ return root+++def _kinds(result: dict) -> list[str]:+ return [f["kind_of_finding"] for f in result["findings"]]+++def test_an_insert_naming_a_table_no_migration_creates_fails(tmp_path) -> None:+ """The defect, in the spelling it actually had.++ `aisoc_detection_rule_proposals` contains `detection_rule_proposals`, so+ the substring assertion this gate was written beside passed on it.+ """+ module = _load()+ statement = 'SQL = "INSERT INTO aisoc_widgets (id, tenant_id, name) VALUES (:id, :tid, :name)"\n'+ result = module.scan(_tree(tmp_path, statement))+ assert _kinds(result) == ["unknown-table"]+ assert result["findings"][0]["table"] == "aisoc_widgets"+ assert result["compared"] == 0+++def test_an_insert_naming_a_column_no_migration_creates_fails(tmp_path) -> None:+ """`detection_rule_proposals.source`, reduced to its essentials."""+ module = _load()+ statement = 'SQL = "INSERT INTO widgets (id, tenant_id, source) VALUES (:id, :tid, :src)"\n'+ result = module.scan(_tree(tmp_path, statement))+ assert _kinds(result) == ["unknown-column"]+ assert result["findings"][0]["column"] == "source"+ # The statement is still credited: the table resolved and the other two+ # columns were compared, which is what `--credits` has to be able to show.+ assert result["compared"] == 1+++def test_a_correct_insert_passes(tmp_path) -> None:+ module = _load()+ statement = 'SQL = "INSERT INTO widgets (id, tenant_id, name) VALUES (:id, :tid, :name)"\n'+ result = module.scan(_tree(tmp_path, statement))+ assert result["findings"] == []+ assert result["compared"] == 1+++def test_a_column_added_by_a_multi_clause_alter_table_is_credited(tmp_path) -> None:+ """The parser gap that would have made this gate unusable.++ `ALTER TABLE t ADD COLUMN a, ADD COLUMN b` is one statement with two+ clauses. Reading only the first credited `alerts` with one of three new+ columns and `aisoc_run_costs` with one of five, so eleven perfectly+ ordinary writes read as naming columns that do not exist.+ """+ module = _load()+ statement = 'SQL = "INSERT INTO widgets (id, label, retired) VALUES (:id, :l, :r)"\n'+ result = module.scan(_tree(tmp_path, statement))+ assert result["findings"] == []+++def test_an_update_naming_a_column_no_migration_creates_fails(tmp_path) -> None:+ module = _load()+ statement = 'SQL = "UPDATE widgets SET label = :l, ghost = :g WHERE id = :id"\n'+ result = module.scan(_tree(tmp_path, statement))+ assert _kinds(result) == ["unknown-column"]+ assert result["findings"][0]["column"] == "ghost"+++def test_a_sql_comment_inside_a_set_clause_does_not_read_as_dynamic(tmp_path) -> None:+ """Three real upserts were reported as runtime-assembled for this reason."""+ module = _load()+ statement = 'SQL = """UPDATE widgets\n SET label = :l,\n -- a note, with a comma in it\n name = :n\n WHERE id = :id"""\n'+ result = module.scan(_tree(tmp_path, statement))+ assert result["findings"] == []+ assert result["compared"] == 1+++def test_a_runtime_assembled_statement_is_a_finding_until_it_is_recorded(tmp_path) -> None:+ module = _load()+ statement = 'SQL = f"UPDATE widgets SET {assignments} WHERE id = :id"\n'+ result = module.scan(_tree(tmp_path, statement))+ assert _kinds(result) == ["unparsed-statement"]+++def test_the_allow_list_suppresses_a_named_dynamic_statement(tmp_path) -> None:+ module = _load()+ statement = 'SQL = f"UPDATE widgets SET {assignments} WHERE id = :id"\n'+ module.DYNAMIC_SQL = {("services/widgetry/app/api/widgets.py", "widgets"): "assembled from the request body"}+ result = module.scan(_tree(tmp_path, statement))+ assert result["findings"] == []+ assert result["recorded"] == 1+++def test_the_allow_list_does_not_forgive_a_readable_statement_beside_a_dynamic_one(tmp_path) -> None:+ """An entry excuses the statement it names, not the file it lives in.++ Keyed on (file, table), a blanket exemption would have covered every+ other write in the same module against the same table — which is how an+ allow-list stops being a record and becomes a hole.+ """+ module = _load()+ statement = 'ONE = f"UPDATE widgets SET {assignments} WHERE id = :id"\nTWO = "INSERT INTO widgets (id, ghost) VALUES (:id, :g)"\n'+ module.DYNAMIC_SQL = {("services/widgetry/app/api/widgets.py", "widgets"): "assembled from the request body"}+ result = module.scan(_tree(tmp_path, statement))+ assert _kinds(result) == ["unknown-column"]+ assert result["findings"][0]["column"] == "ghost"+++def test_an_allow_list_entry_nothing_needs_is_itself_a_finding(tmp_path) -> None:+ """The list only shrinks, so it is verified in both directions."""+ module = _load()+ statement = 'SQL = "INSERT INTO widgets (id, tenant_id, name) VALUES (:id, :tid, :name)"\n'+ module.DYNAMIC_SQL = {("services/widgetry/app/api/nowhere.py", "widgets"): "no longer true"}+ module.UNMIGRATED_TABLES = {"sprockets": "another store, once"}+ result = module.scan(_tree(tmp_path, statement))+ assert result["findings"] == []+ assert len(result["stale"]) == 2+++def test_a_tree_the_gate_parses_no_sql_out_of_is_refused(tmp_path) -> None:+ """A gate that reads nothing finds nothing, so `main()` has to refuse it.++ The shape that actually happens: `services/` is there, the modules are+ there, and the extraction has stopped matching. Nothing removes+ `services/`.+ """+ module = _load()+ root = _tree(tmp_path, "VALUE = 1\n")+ assert module.scan(root)["statements"] == []+ assert module.main(["--repo-root", str(root)]) != 0+++def test_a_missing_services_directory_is_refused_rather_than_called_clean(tmp_path) -> None:+ module = _load()+ root = tmp_path / "bare"+ root.mkdir()+ assert "error" in module.scan(root)+ assert module.main(["--repo-root", str(root)]) == 2+++def test_the_gate_answers_the_self_test_flag() -> None:+ result = subprocess.run([sys.executable, str(GATE), "--self-test"], capture_output=True, text=True)+ assert result.returncode == 0, result.stdout + result.stderr+++def test_the_live_repository_is_clean() -> None:+ """The regression this gate exists for, asserted against the real tree."""+ result = subprocess.run([sys.executable, str(GATE)], capture_output=True, text=True)+ assert result.returncode == 0, result.stdout + result.stderrdiff --git a/services/api/migrations/073_detection_proposal_source.sql b/services/api/migrations/073_detection_proposal_source.sqlnew file mode 100644index 000000000..ac0d9b317--- /dev/null+++ b/services/api/migrations/073_detection_proposal_source.sql@@ -0,0 +1,29 @@+-- AiSOC: the column three proposal writers already name+-- Author: Beenu - [email protected]+--+-- `detection_rule_proposals` has never had a `source` column, and three raw+-- INSERT statements name it:+--+-- * app/api/v1/endpoints/rule_tuning.py writes 'auto-tuner'+-- * app/workers/hunt_scheduler.py writes 'hunt-finding'+-- * app/api/v1/endpoints/detection_loop.py writes 'detection-loop'+--+-- Each raises UndefinedColumnError on the first row it tries to write, so+-- none of the three automated authoring paths can produce a proposal on any+-- deployment. Nothing caught it: no CI job runs `services/api/tests/`+-- against a real database, and the one test covering the insert asserted+-- only that the table name appeared somewhere in the SQL string.+--+-- `scripts/check_raw_sql_columns.py` found it, and now holds raw SQL to the+-- same table-and-column contract `check_orm_migration_parity.py` already+-- holds the ORM models to.+--+-- Nullable with no default: rows written before this migration have no+-- provenance to claim, and NULL is the honest value for "not recorded" —+-- distinct from a proposal an analyst authored by hand.++ALTER TABLE detection_rule_proposals+ ADD COLUMN IF NOT EXISTS source TEXT;++COMMENT ON COLUMN detection_rule_proposals.source IS+ 'Which automated path authored this proposal: auto-tuner (analyst disposition history), hunt-finding (a scheduled hunt that hit), detection-loop (a false-positive fix). NULL for a hand-authored proposal or any row predating this column.';diff --git a/services/api/tests/test_wave1_loop_edges.py b/services/api/tests/test_wave1_loop_edges.pyindex 61d078738..d0133c69d 100644--- a/services/api/tests/test_wave1_loop_edges.py+++ b/services/api/tests/test_wave1_loop_edges.py@@ -8,6 +8,7 @@ they run fast and deterministically.from __future__ import annotations+import refrom types import SimpleNamespacefrom unittest.mock import AsyncMockfrom uuid import uuid4@@ -38,7 +39,14 @@ async def test_hunt_hits_open_a_detection_proposal(monkeypatch):await hunt_scheduler._propose_detection_from_hunt(db, _hunt(), 5)assert db.execute.await_count == 1sql = str(db.execute.await_args.args[0])- assert "detection_rule_proposals" in sql+ # The parsed table name, not a substring of the SQL. `"detection_rule_+ # proposals" in sql` also passes on `aisoc_detection_rule_proposals`,+ # which contains the right name inside the wrong one — and that exact+ # confusion is recorded history here: three insert sites and two tests had+ # codified the `aisoc_`-prefixed spelling of this very table.+ inserted = re.search(r"INSERT\s+INTO\s+(\w+)", sql)+ assert inserted is not None, sql+ assert inserted.group(1) == "detection_rule_proposals"@pytest.mark.asyncio