Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

fix(agents): stop warning that cost provenance is missing when it is present (#967)

Beenu Arorafb05a380d59b
diff --git a/services/agents/app/core/cost_telemetry.py b/services/agents/app/core/cost_telemetry.py
index 0e1395cbd..832a5fcb7 100644
--- a/services/agents/app/core/cost_telemetry.py
+++ b/services/agents/app/core/cost_telemetry.py
@@ -66,7 +66,7 @@ from typing import Any
import structlog
from app.core.gateway_cost import GatewayCost, extract_gateway_cost, extract_resolved_model
-from app.core.schema_bootstrap import ensure_table
+from app.core.schema_bootstrap import ensure_columns, ensure_table
logger = structlog.get_logger()
@@ -283,6 +283,17 @@ CREATE INDEX IF NOT EXISTS aisoc_run_costs_tenant_run
ON aisoc_run_costs (tenant_id, run_id);
"""
+#: The columns ``_RUN_COSTS_PROVENANCE_DDL`` adds, listed so the writer can
+#: ask whether they are there before asking for the privilege to add them.
+_RUN_COSTS_PROVENANCE_COLUMNS = (
+ "measured_cost_usd",
+ "measured_call_count",
+ "estimated_cost_usd",
+ "estimated_call_count",
+ "unpriced_call_count",
+ "resolved_model",
+)
+
#: Cost provenance, added by ``services/api/migrations/063_cost_provenance.sql``.
#: Applied here too because this writer has to run against a database whose
#: migrations it does not own, and a write that silently drops the provenance
@@ -322,14 +333,15 @@ async def _get_pool() -> Any | None:
if not await ensure_table(conn, "aisoc_run_costs", _RUN_COSTS_DDL):
await pool.close()
return None
- try:
- await conn.execute(_RUN_COSTS_PROVENANCE_DDL)
- except Exception as exc: # noqa: BLE001
- # The runtime role may hold DML only, in which case migration
- # 055 is the one that adds these. Loud enough to diagnose a
- # dashboard stuck on "not measured", quiet enough not to fail
- # the run: the token counts still land.
- logger.warning("cost_telemetry.provenance_columns_unavailable", error=str(exc))
+ # Probe before altering, for the same reason as the table above:
+ # ownership is checked before `IF NOT EXISTS`, so this raised
+ # `must be owner of table aisoc_run_costs` on every run against a
+ # database where all six columns were already present and being
+ # written. `ensure_columns` reports only when they are genuinely
+ # absent, and never fails the run either way — the token counts
+ # still land, and a dashboard stuck on "not measured" is
+ # diagnosable from the message it now prints.
+ await ensure_columns(conn, "aisoc_run_costs", _RUN_COSTS_PROVENANCE_COLUMNS, _RUN_COSTS_PROVENANCE_DDL)
_POOL = pool
return _POOL
except Exception as exc:
diff --git a/services/agents/app/core/schema_bootstrap.py b/services/agents/app/core/schema_bootstrap.py
index a58915d5a..b894f804d 100644
--- a/services/agents/app/core/schema_bootstrap.py
+++ b/services/agents/app/core/schema_bootstrap.py
@@ -66,3 +66,54 @@ async def ensure_table(conn: Any, table: str, ddl: str) -> bool:
)
return False
return True
+
+
+async def ensure_columns(conn: Any, table: str, columns: tuple[str, ...], ddl: str) -> bool:
+ """The same rule for a column set that ``ensure_table`` applies to a table.
+
+ ``ALTER TABLE ... ADD COLUMN IF NOT EXISTS`` has the identical trap, for
+ the identical reason: Postgres checks ownership *before* the existence
+ test, so the statement raises ``must be owner of table <t>`` on a table
+ whose columns are all already present. Measured against a live CORE stack,
+ where the cost writer logged
+
+ cost_telemetry.provenance_columns_unavailable
+ error='must be owner of table aisoc_run_costs'
+
+ on **every** triage run while all six columns existed and were being
+ written — a warning pointing an operator at a migration that had already
+ run, attached to a subsystem that was working.
+
+ Returns ``True`` when the columns are usable afterwards, so a caller can
+ tell "present" from "genuinely missing and I cannot add them".
+ """
+ try:
+ present = await conn.fetch(
+ "SELECT column_name FROM information_schema.columns WHERE table_schema = 'public' AND table_name = $1",
+ table,
+ )
+ except Exception as exc: # noqa: BLE001 — a dead connection is the caller's problem
+ logger.error("schema_bootstrap.column_probe_failed", table=table, error=str(exc))
+ return False
+
+ have = {row["column_name"] for row in present}
+ missing = [column for column in columns if column not in have]
+ if not missing:
+ return True
+
+ try:
+ await conn.execute(ddl)
+ except Exception as exc: # noqa: BLE001 — reported, not swallowed
+ logger.error(
+ "schema_bootstrap.add_columns_failed",
+ table=table,
+ missing=missing,
+ error=str(exc),
+ hint=(
+ f"{table} is missing {missing} and this role cannot add them. The runtime role holds "
+ "DML only by design; apply the API migration chain as the owner role "
+ "(DATABASE_MIGRATION_URL) instead of relying on this bootstrap."
+ ),
+ )
+ return False
+ return True
diff --git a/services/agents/tests/test_schema_bootstrap_columns.py b/services/agents/tests/test_schema_bootstrap_columns.py
new file mode 100644
index 000000000..5358bf030
--- /dev/null
+++ b/services/agents/tests/test_schema_bootstrap_columns.py
@@ -0,0 +1,119 @@
+"""`ensure_columns` must not ask for a privilege it does not need.
+
+The cost writer applied its provenance columns with
+
+ ALTER TABLE aisoc_run_costs ADD COLUMN IF NOT EXISTS ...
+
+and caught the failure into a warning. `IF NOT EXISTS` reads as "harmless
+when they are already there", and it is not: Postgres checks ownership
+*before* the existence test, so on a database where all six columns exist the
+statement still raises ``must be owner of table aisoc_run_costs`` for the
+DML-only runtime role the services connect as.
+
+Measured against a live CORE stack, that fired on **every** triage run:
+
+ cost_telemetry.provenance_columns_unavailable
+ error='must be owner of table aisoc_run_costs'
+
+while the six columns were present and the rows carrying them were being
+written. A warning that names a working subsystem, and points the operator at
+a migration that had already run, costs more than it is worth — it is the
+"diagnostic that sends you to debug something that is not broken" shape this
+repository keeps finding.
+
+`ensure_table` already had the answer in its own module docstring: look
+before leaping. These tests pin that `ensure_columns` does the same, in both
+directions — silent when the columns are there, loud when they genuinely are
+not.
+"""
+
+from __future__ import annotations
+
+import pytest
+from app.core.schema_bootstrap import ensure_columns
+
+COLUMNS = ("measured_cost_usd", "resolved_model")
+DDL = "ALTER TABLE aisoc_run_costs ADD COLUMN IF NOT EXISTS measured_cost_usd DOUBLE PRECISION;"
+
+
+class FakeConn:
+ """Postgres as the DML-only role sees it: `execute` may be refused."""
+
+ def __init__(self, present: tuple[str, ...], *, execute_error: Exception | None = None) -> None:
+ self._present = present
+ self._execute_error = execute_error
+ self.executed: list[str] = []
+ self.probes = 0
+
+ async def fetch(self, _sql: str, _table: str) -> list[dict[str, str]]:
+ self.probes += 1
+ return [{"column_name": name} for name in self._present]
+
+ async def execute(self, sql: str) -> None:
+ self.executed.append(sql)
+ if self._execute_error is not None:
+ raise self._execute_error
+
+
+async def test_present_columns_are_not_altered_at_all() -> None:
+ """The case that was firing a warning on every run."""
+ conn = FakeConn(present=COLUMNS + ("run_id", "tenant_id"))
+
+ assert await ensure_columns(conn, "aisoc_run_costs", COLUMNS, DDL) is True
+ assert conn.executed == [], "asked for ALTER privilege it did not need"
+ assert conn.probes == 1
+
+
+async def test_a_role_without_ownership_stays_quiet_when_the_columns_exist() -> None:
+ """The regression, stated as the deployment that produced it.
+
+ An `execute` that would raise `must be owner` must never be reached, so
+ configuring one and finding nothing logged is the assertion.
+ """
+ conn = FakeConn(
+ present=COLUMNS,
+ execute_error=RuntimeError("must be owner of table aisoc_run_costs"),
+ )
+
+ assert await ensure_columns(conn, "aisoc_run_costs", COLUMNS, DDL) is True
+ assert conn.executed == []
+
+
+async def test_genuinely_missing_columns_are_still_added() -> None:
+ conn = FakeConn(present=("run_id",))
+
+ assert await ensure_columns(conn, "aisoc_run_costs", COLUMNS, DDL) is True
+ assert conn.executed == [DDL]
+
+
+async def test_one_missing_column_is_enough_to_apply_the_ddl() -> None:
+ """A partially-migrated table must converge, not be read as done."""
+ conn = FakeConn(present=("measured_cost_usd",))
+
+ assert await ensure_columns(conn, "aisoc_run_costs", COLUMNS, DDL) is True
+ assert conn.executed == [DDL]
+
+
+async def test_a_refusal_on_genuinely_missing_columns_is_reported_and_not_raised() -> None:
+ """Loud, but never fatal: the token counts still land without them."""
+ conn = FakeConn(
+ present=("run_id",),
+ execute_error=RuntimeError("must be owner of table aisoc_run_costs"),
+ )
+
+ assert await ensure_columns(conn, "aisoc_run_costs", COLUMNS, DDL) is False
+ assert conn.executed == [DDL]
+
+
+async def test_a_failed_probe_does_not_take_the_agent_down() -> None:
+ class DeadConn(FakeConn):
+ async def fetch(self, _sql: str, _table: str): # type: ignore[override]
+ raise RuntimeError("connection is closed")
+
+ assert await ensure_columns(DeadConn(present=()), "aisoc_run_costs", COLUMNS, DDL) is False