Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

fix(api): SSO activation no longer locks out the admin; funnel lake column (#1569)

Beenu Arora013ecd389541
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 74f39e4a3..feece4a0c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,32 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
+### Fixed
+
+- **Activating SSO OIDC no longer locks the administrator out** (reported in
+ discussion #1568). `resolve_permissions` uses the static role map only while
+ a tenant holds zero `user_roles` grants; the first SSO sign-in provisions a
+ user and grants it a role, which flips the tenant to database-backed RBAC,
+ and from that instant the bootstrap admin — created by `bootstrap_admin.py`
+ *after* migration 092's membership backfill, so the one account with
+ `users.role = 'admin'` and no grant — resolved to **zero** permissions and
+ every route answered 403 (users, investigations, the alert queue alike).
+ `bootstrap_admin` now grants the admin its `admin` catalog role on every
+ startup, in an isolated session so it can never disturb admin creation, and
+ as a no-op when the catalog is unseeded. Because it runs on every startup it
+ also heals a deployment already locked out, on the next restart. The
+ resolver's deprovisioning semantics are unchanged and still separately
+ pinned; a live-Postgres test reproduces the lockout and proves the fix.
+- **The alert-funnel's "events of interest" count queried a column the lake
+ does not have.** `_events_of_interest` filtered `aisoc.raw_events` on
+ `ingested_at`, which raised ClickHouse `Code: 47, Missing columns:
+ 'ingested_at'` on every call (surfaced alongside #1568), so the funnel
+ silently served its Postgres fallback on every deployment with a reachable
+ lake. It now filters on `ingest_time` — when the event landed, the column
+ `usage_metering` already counts arrivals by — with a test that checks the
+ query's filter columns against the committed ClickHouse DDL so any
+ non-existent column is caught, not only this one.
+
## [20.3.0] - 2026-10-30
### Added
diff --git a/services/api/app/api/v1/endpoints/metrics.py b/services/api/app/api/v1/endpoints/metrics.py
index 89de1a1be..cd8c8811e 100644
--- a/services/api/app/api/v1/endpoints/metrics.py
+++ b/services/api/app/api/v1/endpoints/metrics.py
@@ -899,6 +899,32 @@ def _pct_delta(current: float, previous: float) -> float | None:
return round(((current - previous) / previous) * 100.0, 2)
+def _events_of_interest_sql(start, end) -> str:
+ """The lake count query for events that arrived in ``[start, end)``.
+
+ Filters on ``ingest_time`` — when the event *landed in the lake*
+ (``DEFAULT now64()`` in ``clickhouse/001_init.sql``), which is the
+ question an intake funnel asks: of the events that arrived in this
+ window, how many advanced. ``event_time`` is when the event *occurred*,
+ which late or backfilled data pushes outside the window, so it is the
+ wrong clock for an arrival funnel — and it is the column
+ ``usage_metering`` already counts arrivals by.
+
+ There is no ``ingested_at`` column: naming it made this query fail with
+ ClickHouse ``Code: 47, Missing columns: 'ingested_at'`` on every call, so
+ the funnel silently served the Postgres fallback on every deployment with
+ a reachable lake (reported in discussion #1568). A pure builder so the
+ column can be checked against the schema in a test without a live
+ ClickHouse.
+ """
+ return (
+ "SELECT count() AS cnt "
+ "FROM aisoc.raw_events "
+ f"WHERE ingest_time >= toDateTime('{start.strftime('%Y-%m-%d %H:%M:%S')}') "
+ f"AND ingest_time < toDateTime('{end.strftime('%Y-%m-%d %H:%M:%S')}')"
+ )
+
+
async def _events_of_interest(db, tenant_id, start, end) -> int:
"""Count tenant raw events that landed in the analytical lake.
@@ -925,12 +951,7 @@ async def _events_of_interest(db, tenant_id, start, end) -> int:
except Exception: # pragma: no cover — defensive
return await _events_of_interest_from_alerts(db, tenant_id, start, end)
- sql = (
- "SELECT count() AS cnt "
- "FROM aisoc.raw_events "
- f"WHERE ingested_at >= toDateTime('{start.strftime('%Y-%m-%d %H:%M:%S')}') "
- f"AND ingested_at < toDateTime('{end.strftime('%Y-%m-%d %H:%M:%S')}')"
- )
+ sql = _events_of_interest_sql(start, end)
try:
rewrite = rewrite_for_tenant(sql, tenant_id, row_cap=1)
diff --git a/services/api/app/scripts/bootstrap_admin.py b/services/api/app/scripts/bootstrap_admin.py
index 9f7289c9c..9ac217c22 100644
--- a/services/api/app/scripts/bootstrap_admin.py
+++ b/services/api/app/scripts/bootstrap_admin.py
@@ -40,6 +40,7 @@ from __future__ import annotations
import argparse
import asyncio
+import logging
import os
import secrets
import string
@@ -50,11 +51,14 @@ from email_validator import EmailNotValidError, validate_email
from sqlalchemy import func, select
from app.api.v1.dev_auth import DEMO_TENANT_ID
+from app.core.rbac_catalog import sync_user_catalog_role
from app.core.security import get_password_hash
from app.db.database import AsyncSessionLocal
from app.models.tenant import Tenant, User
from app.services.audit import emit_audit
+logger = logging.getLogger(__name__)
+
# Matches the tenant seeded by migration 001. Reused so a bootstrap on an
# already-migrated database adopts that tenant rather than creating a second
# one that none of the seeded rows belong to.
@@ -278,6 +282,55 @@ async def bootstrap(
async with AsyncSessionLocal() as session:
tenant = await _ensure_tenant(session)
+ async def _grant_admin_catalog_role(user_id) -> None:
+ """Give the break-glass admin its catalog ``user_roles`` grant.
+
+ The admin is created by this script, which runs *after* migration
+ 092's membership backfill, so it is the one account in a seeded
+ tenant with ``users.role = 'admin'`` and no ``user_roles`` row.
+ That is harmless while the tenant holds zero grants —
+ ``resolve_permissions`` uses the static map on a bootstrap tenant —
+ but the first SSO sign-in (or any first grant) flips the tenant to
+ database-backed RBAC, and from that moment an admin with no grant
+ resolves to *zero* permissions and every route answers 403. That is
+ exactly what activating SSO OIDC did to the administrator who set it
+ up (discussion #1568): the SSO user's grant switched the tenant, and
+ the admin — ungranted — was locked out of users, investigations and
+ the alert queue alike.
+
+ Granting the admin here closes it at the source, and because
+ bootstrap runs on every startup it also *heals* a deployment already
+ in that state on the next restart. A no-op when the catalog is not
+ seeded (``sync_user_catalog_role`` logs and returns), which is
+ correct: that tenant stays on the static-map path and the admin
+ needs no row. RLS does not block the write — bootstrap sets no
+ tenant context, and the ``user_roles`` policy admits a NULL context,
+ the same escape under which this script already writes ``users``.
+
+ Best-effort by design: creating the break-glass admin is the
+ critical path and must not be held hostage to a grant write. If the
+ catalog tables are absent entirely (an unmigrated database, or the
+ SQLite schema the unit tests build) the write raises, and the right
+ answer is the same as an unseeded catalog — leave the admin on the
+ static-map path, exactly where it was before this fix, so there is
+ no regression.
+
+ Runs in its *own* session: the admin row is already committed by the
+ time this is called, and a grant failure (a missing catalog, a
+ connection fault) must not be able to corrupt or roll back the
+ caller's transaction. Isolating it means the worst case is a logged
+ skip, never a half-written admin.
+ """
+ try:
+ async with AsyncSessionLocal() as grant_session:
+ await sync_user_catalog_role(grant_session, tenant_id=tenant.id, user_id=user_id, role_name="admin")
+ await grant_session.commit()
+ except Exception as exc: # noqa: BLE001 — hardening must not break admin creation
+ logger.warning(
+ "bootstrap_admin.catalog_grant_skipped error=%s",
+ str(exc).replace("\r", "").replace("\n", " ")[:200],
+ )
+
# Case-insensitive: Postgres `UNIQUE` on `email` is case-sensitive, so
# `Admin@…` and `admin@…` are two rows, and a second bootstrap would
# hand out a password for an account the operator does not realise is a
@@ -287,6 +340,10 @@ async def bootstrap(
if user is not None:
if not reset_password:
+ # Still ensure the grant: this is the path a restart takes
+ # against an already-provisioned admin, which is how a
+ # deployment locked out by an earlier SSO activation heals.
+ await _grant_admin_catalog_role(user.id)
return email, False
previous_role = user.role
previously_active = user.is_active
@@ -311,6 +368,7 @@ async def bootstrap(
},
)
await session.commit()
+ await _grant_admin_catalog_role(user.id)
return email, True
# An administrator under a different address still counts — re-running
@@ -318,6 +376,7 @@ async def bootstrap(
result = await session.execute(select(User).where(User.role == "admin", User.is_active.is_(True)).limit(1))
existing_admin = result.scalar_one_or_none()
if existing_admin is not None and not reset_password:
+ await _grant_admin_catalog_role(existing_admin.id)
return existing_admin.email, False
administrator = User(
@@ -349,6 +408,7 @@ async def bootstrap(
},
)
await session.commit()
+ await _grant_admin_catalog_role(administrator.id)
return email, True
diff --git a/services/api/tests/test_bootstrap_admin_keeps_access_after_sso.py b/services/api/tests/test_bootstrap_admin_keeps_access_after_sso.py
new file mode 100644
index 000000000..4c992139e
--- /dev/null
+++ b/services/api/tests/test_bootstrap_admin_keeps_access_after_sso.py
@@ -0,0 +1,152 @@
+"""Activating SSO must not lock the administrator who activated it out.
+
+Discussion #1568: an operator enabled SSO OIDC and the admin account
+immediately lost access — 403 on `/api/v1/alerts/queue`, on
+`/api/v1/admin/users`, on investigations. Nothing about the admin changed; a
+*different* user signing in did.
+
+The mechanism, proven by `test_bootstrap_fallback_survives_a_seeded_role.py`
+(whose last case asserts, correctly, that an ungranted user in a configured
+tenant resolves to nothing): `resolve_permissions` falls back to the static
+`ROLE_PERMISSIONS` map only while a tenant holds *zero* `user_roles` grants.
+The bootstrap admin is created by `bootstrap_admin.py`, which runs after
+migration 092's membership backfill, so it alone carries `users.role = admin`
+with no `user_roles` row. That is harmless — until the first SSO sign-in
+provisions a user and grants it a role. That first grant flips the tenant to
+database-backed RBAC, and from that instant the ungranted admin resolves to
+the empty set and every authorized route answers 403.
+
+The fix does not touch the resolver (its deprovisioning semantics are
+correct and separately pinned). It ensures the break-glass admin *has* its
+catalog grant, so it is never the ungranted user. This test reproduces the
+lockout, applies the fix exactly as `bootstrap_admin` now does, and asserts
+access is restored — with the pre-fix lockout standing as the negative
+control.
+
+Live Postgres, same rationale as the sibling file: a fake session cannot tell
+"counts grants" from "counts roles", and the migration chain is where the
+catalog lives. Skips when no database answers, but cannot skip where
+`integration.yml` requires it.
+"""
+
+from __future__ import annotations
+
+import os
+import uuid
+
+import pytest
+import pytest_asyncio
+from app.core.permission_cache import grants, reset_for_tests, resolve_permissions
+from app.core.rbac_catalog import seed_tenant_catalog, sync_user_catalog_role
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
+
+DSN = os.environ.get("DATABASE_URL", "")
+REQUIRED = os.environ.get("MSSP_ISOLATION_REQUIRED", "").strip() not in ("", "0", "false")
+
+TENANT = uuid.UUID("55081568-0000-0000-0000-00000000000b")
+ADMIN = uuid.UUID("55081568-0000-0000-0000-0000000000a1")
+SSO_USER = uuid.UUID("55081568-0000-0000-0000-0000000000a2")
+
+
+@pytest_asyncio.fixture
+async def db():
+ if "postgres" not in DSN and not REQUIRED:
+ pytest.skip("needs a live Postgres with the migration chain applied (integration.yml)")
+ engine = create_async_engine(DSN)
+ try:
+ async with engine.connect() as probe:
+ await probe.execute(text("SELECT 1"))
+ except Exception as exc:
+ await engine.dispose()
+ if REQUIRED:
+ pytest.fail(
+ "MSSP_ISOLATION_REQUIRED is set but no database answered at DATABASE_URL — "
+ f"the SSO-lockout proof did not run: {type(exc).__name__}: {exc}"
+ )
+ pytest.skip(f"no database at DATABASE_URL ({type(exc).__name__}) — runs in integration.yml")
+
+ factory = async_sessionmaker(engine, expire_on_commit=False)
+ async with factory() as session:
+ await _seed(session)
+ try:
+ yield session
+ finally:
+ await _teardown(session)
+ await engine.dispose()
+
+
+async def _seed(session) -> None:
+ """A tenant whose catalog is seeded, an admin created *after* that seed
+ (so it has no grant), and an SSO user who does hold one.
+
+ `seed_tenant_catalog` runs before any user exists, so its membership
+ backfill grants nobody — which is the production shape: the admin is
+ created later by `bootstrap_admin`, and the SSO user later still.
+ """
+ await session.rollback()
+ await session.execute(
+ text("INSERT INTO tenants (id, name, slug) VALUES (CAST(:t AS uuid), 'sso-lockout', :s) ON CONFLICT DO NOTHING"),
+ {"t": str(TENANT), "s": f"sso-lockout-{TENANT.hex[:8]}"},
+ )
+ await seed_tenant_catalog(session, TENANT)
+ await session.commit()
+
+ for uid, email, role in ((ADMIN, "[email protected]", "admin"), (SSO_USER, "[email protected]", "viewer")):
+ await session.execute(
+ text(
+ "INSERT INTO users (id, tenant_id, email, username, hashed_password, role, is_active) "
+ "VALUES (CAST(:i AS uuid), CAST(:t AS uuid), :e, :e, 'x', :r, true) ON CONFLICT DO NOTHING"
+ ),
+ {"i": str(uid), "t": str(TENANT), "e": email, "r": role},
+ )
+ await session.commit()
+ reset_for_tests()
+
+
+async def _teardown(session) -> None:
+ await session.rollback()
+ await session.execute(
+ text("DELETE FROM user_roles WHERE user_id IN (CAST(:a AS uuid), CAST(:u AS uuid))"),
+ {"a": str(ADMIN), "u": str(SSO_USER)},
+ )
+ await session.execute(text("DELETE FROM users WHERE tenant_id = CAST(:t AS uuid)"), {"t": str(TENANT)})
+ await session.commit()
+ reset_for_tests()
+
+
+class TestSsoActivationDoesNotLockOutTheAdmin:
+ async def test_the_first_sso_grant_locks_out_an_ungranted_admin(self, db) -> None:
+ """The reported 403, as an assertion — the negative control for the fix.
+
+ The admin holds no grant. The moment the SSO user is granted a role,
+ the tenant is database-backed and the admin resolves to nothing.
+ """
+ await sync_user_catalog_role(db, tenant_id=TENANT, user_id=SSO_USER, role_name="viewer")
+ await db.commit()
+ reset_for_tests()
+
+ resolved = await resolve_permissions(db, tenant_id=TENANT, user_id=ADMIN, static_role="admin")
+ assert resolved == frozenset(), (
+ "expected the pre-fix lockout: an ungranted admin should resolve to nothing once the tenant is DB-backed"
+ )
+
+ async def test_granting_the_admin_its_catalog_role_restores_access(self, db) -> None:
+ """The fix: `bootstrap_admin` grants the admin its catalog role, so it
+ is never the ungranted user when the SSO grant flips the tenant."""
+ await sync_user_catalog_role(db, tenant_id=TENANT, user_id=SSO_USER, role_name="viewer")
+ await db.commit()
+ reset_for_tests()
+ # Precondition: locked out, exactly as the control above.
+ assert await resolve_permissions(db, tenant_id=TENANT, user_id=ADMIN, static_role="admin") == frozenset()
+
+ # This is what bootstrap_admin now does on every startup.
+ await sync_user_catalog_role(db, tenant_id=TENANT, user_id=ADMIN, role_name="admin")
+ await db.commit()
+ reset_for_tests()
+
+ resolved = await resolve_permissions(db, tenant_id=TENANT, user_id=ADMIN, static_role="admin")
+ assert grants(resolved, "users:read"), "the admin cannot see the user list after the fix"
+ assert grants(resolved, "alerts:read"), "the admin cannot read the alert queue after the fix"
+ assert grants(resolved, "anything:at:all"), "the admin wildcard did not resolve"
diff --git a/services/api/tests/test_events_of_interest_lake_column.py b/services/api/tests/test_events_of_interest_lake_column.py
new file mode 100644
index 000000000..69c371183
--- /dev/null
+++ b/services/api/tests/test_events_of_interest_lake_column.py
@@ -0,0 +1,75 @@
+"""The funnel's lake query must filter on a column the lake actually has.
+
+Discussion #1568 attached, underneath an SSO lockout, a second symptom:
+
+ [aisoc-api] events_of_interest ClickHouse fallback: clickhouse query
+ failed: Code: 47. DB::Exception: Missing columns: 'ingested_at'
+
+`_events_of_interest` filtered `aisoc.raw_events` on `ingested_at`, a column
+that does not exist — the lake has `event_time` (when the event occurred) and
+`ingest_time` (when it landed). So the query raised `Code: 47` on every call
+and the funnel silently served its Postgres fallback on every deployment with
+a reachable lake. The primary source was dead and nothing said so.
+
+This test derives the real column set from the committed ClickHouse DDL and
+asserts every column the funnel query filters on is a member. It fails on the
+pre-fix tree (`ingested_at` is not in the DDL) and would catch any future
+query that invents a column, not only this one.
+"""
+
+from __future__ import annotations
+
+import re
+from datetime import UTC, datetime
+from pathlib import Path
+
+from app.api.v1.endpoints.metrics import _events_of_interest_sql
+
+_DDL = Path(__file__).resolve().parents[1] / "clickhouse" / "001_init.sql"
+
+
+def _raw_events_columns() -> set[str]:
+ """Column names declared for `aisoc.raw_events` in the canonical DDL."""
+ sql = _DDL.read_text(encoding="utf-8")
+ body = re.search(
+ r"CREATE TABLE IF NOT EXISTS aisoc\.raw_events\s*\((.*?)\)\s*ENGINE",
+ sql,
+ re.DOTALL | re.IGNORECASE,
+ )
+ assert body, "could not locate the aisoc.raw_events CREATE TABLE in 001_init.sql"
+ columns: set[str] = set()
+ for line in body.group(1).splitlines():
+ token = line.strip()
+ if not token or token.startswith("--"):
+ continue
+ name = token.split()[0].strip(",")
+ if re.fullmatch(r"[a-z_][a-z0-9_]*", name):
+ columns.add(name)
+ return columns
+
+
+def _filtered_columns(sql: str) -> set[str]:
+ """Columns the query compares against in its WHERE clause."""
+ return set(re.findall(r"(?:WHERE|AND)\s+([a-z_][a-z0-9_]*)\s*(?:>=|<=|<|>|=)", sql))
+
+
+def test_the_funnel_query_filters_on_a_real_lake_column() -> None:
+ start = datetime(2026, 10, 30, 0, 0, tzinfo=UTC)
+ end = datetime(2026, 10, 30, 1, 0, tzinfo=UTC)
+ sql = _events_of_interest_sql(start, end)
+
+ columns = _raw_events_columns()
+ # Guard the guard: if the DDL parse ever returns nothing, the membership
+ # assertion below would pass vacuously.
+ assert {"event_time", "ingest_time"} <= columns, columns
+
+ filtered = _filtered_columns(sql)
+ assert filtered, f"no filter column parsed from: {sql}"
+ missing = filtered - columns
+ assert not missing, f"funnel query filters on columns absent from aisoc.raw_events: {missing}"
+
+
+def test_the_dead_column_name_is_gone() -> None:
+ sql = _events_of_interest_sql(datetime(2026, 10, 30, tzinfo=UTC), datetime(2026, 10, 31, tzinfo=UTC))
+ assert "ingested_at" not in sql
+ assert "ingest_time" in sql