Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix: correct stale control-api migration references and guard them (#689)
ALFREDO J LOPEZ H (alfredolopez80.eth)ea2375678eb8
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex b88ea1a7d..9837b354e 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.584",+ "version": "0.1.586","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.584",+ "version": "0.1.586","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 7904ea41c..ddcea757c 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.584",+ "version": "0.1.586","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/db.ts b/control-api/src/db.tsindex 04cc4f61f..e710d5d98 100644--- a/control-api/src/db.ts+++ b/control-api/src/db.ts@@ -168,7 +168,8 @@ export const corePool = poolasync function applyBaselineSchema(db: DbClient): Promise<void> {// Baseline includes additive Phase 0 workflow-trigger tables for fresh- // clusters. Existing clusters receive the same tables through migration 0016;+ // clusters. Existing clusters receive the same tables through migration+ // 0016_workflow_trigger_shared_foundation;// that migration also remains responsible for backfill and preflight checks.await db.query(`CREATE EXTENSION IF NOT EXISTS "pgcrypto";@@ -1320,7 +1321,7 @@ async function alignWorkflowRunsAuditRecipeIndex(db: DbClient): Promise<void> {* Drop the `trigger_grants_audit.operator_user_id_fkey` foreign key on* clusters that bootstrapped before commit 68c81bea.*- * Context: the baseline migration (0001) originally declared this FK as+ * Context: the baseline migration (0001_control_api_baseline) originally declared this FK as* `REFERENCES users(id)`. Commit 68c81bea edited the baseline body in place* to add an `ALTER TABLE … DROP CONSTRAINT IF EXISTS …` — but baseline was* already recorded in `schema_migrations` on every long-lived cluster, so@@ -4410,8 +4411,9 @@ export const CONTROL_API_MIGRATIONS: DbMigration[] = [DROP TRIGGER IF EXISTS infrastructure_cost_daily_components_no_truncate ON infrastructure_cost_daily_components;CREATE TRIGGER infrastructure_cost_daily_components_no_truncate BEFORE TRUNCATE ON infrastructure_cost_daily_components FOR EACH STATEMENT EXECUTE FUNCTION governed_trace_reject_truncate();- -- Migration 0055 replaces this bootstrap function with the final- -- owner-bound retention implementation and dedicated runtime roles.+ -- Migration 0062_governed_trace_runtime_roles replaces this bootstrap+ -- function with the final owner-bound retention implementation and+ -- dedicated runtime roles.CREATE OR REPLACE FUNCTION governed_trace_prune_expired_events(requested_family TEXT,batch_limit INTEGER DEFAULT 1000@@ -6141,8 +6143,9 @@ async function consolidateWorkflowAllowedUsersToTriggers(db: DbClient): Promise<const roleRow = usersRoleColumn.rows[0] as { exists: boolean } | undefinedif (roleRow?.exists) {// Re-seed the sentinel binding (admins → mcp-host/standalone) into the- // canonical table. Migration 0006 seeded it into the legacy table; this is- // idempotent against admins that joined since 0006 ran.+ // canonical table. Migration 0006_seed_sentinel_allowlist_for_admins seeded+ // it into the legacy table; this is idempotent against admins that joined+ // since that migration ran.await db.query(`INSERT INTO user_workflow_triggers (user_id, recipe_namespace, recipe_name)SELECT id, 'mcp-host', 'standalone' FROM users WHERE role = 'admin'diff --git a/control-api/src/routes/admin/codexSubscription.ts b/control-api/src/routes/admin/codexSubscription.tsindex 15575ea12..cf16c186d 100644--- a/control-api/src/routes/admin/codexSubscription.ts+++ b/control-api/src/routes/admin/codexSubscription.ts@@ -349,7 +349,8 @@ export function createAdminCodexSubscriptionRouter(}/**- * Migration 0113 archives superseded tombstones as `<key>~revoked~<id>`.+ * Migration 0113_grok_subscription_terminal_connection_key archives+ * superseded tombstones as `<key>~revoked~<id>`.* Those keys sit outside the key grammar, so no keyed route can address* them; listing them would only surface unactionable duplicates. The key's* own terminal tombstone (status `revoked`, `revokedAt` set) stays listed.diff --git a/control-api/src/services/grokSubscriptionConnection.ts b/control-api/src/services/grokSubscriptionConnection.tsindex 727868ec2..74531956a 100644--- a/control-api/src/services/grokSubscriptionConnection.ts+++ b/control-api/src/services/grokSubscriptionConnection.ts@@ -56,7 +56,8 @@ export class GrokSubscriptionFingerprintConflictError extends Error {/*** A write lost the connection_key uniqueness race: another writer created the* key first, or the key is a revoked tombstone (revocation is terminal per key- * since migration 0113). Callers re-read with+ * since migration 0113_grok_subscription_terminal_connection_key). Callers+ * re-read with* `getGrokSubscriptionConnectionIncludingRevoked` to tell the two apart.*/export class GrokSubscriptionConnectionKeyConflictError extends Error {diff --git a/control-api/src/services/hostHeartbeatService.ts b/control-api/src/services/hostHeartbeatService.tsindex 1b18bfe65..eccd4ca2a 100644--- a/control-api/src/services/hostHeartbeatService.ts+++ b/control-api/src/services/hostHeartbeatService.ts@@ -1,7 +1,7 @@import { pool } from '../db.js'/**- * Stateless heartbeat persistence (host_heartbeats, migration 0051).+ * Stateless heartbeat persistence (host_heartbeats, migration 0051_host_heartbeats).** mcp-host pods POST D8 activity snapshots to control-api's /mcp-host facade* (`POST /api/v1/mcp-host/hosts/heartbeat`, identity bound to the runtimediff --git a/control-api/src/services/memberRegistrationCredentialsSchema.ts b/control-api/src/services/memberRegistrationCredentialsSchema.tsindex be99ed5c7..8d20b6663 100644--- a/control-api/src/services/memberRegistrationCredentialsSchema.ts+++ b/control-api/src/services/memberRegistrationCredentialsSchema.ts@@ -1,6 +1,6 @@import type { DbClient } from '../db.js'-// ─── Hosted member-registration credentials — schema (migration 0068) ──+// ─── Hosted member-registration credentials — schema (migration 0068_member_registration_credentials) ──// One row per (bound_domain, mint). The hub binds each credential to exactly one// destination hostname; hosted mode mints one per configured UI host (spec §8.2).// `secret_encrypted` is AES-256-GCM via src/oauth/encryption.ts — never plaintext.diff --git a/control-api/src/services/pluginWorkloadSdkDb.ts b/control-api/src/services/pluginWorkloadSdkDb.tsindex de4d91ea8..8141f35f7 100644--- a/control-api/src/services/pluginWorkloadSdkDb.ts+++ b/control-api/src/services/pluginWorkloadSdkDb.ts@@ -352,7 +352,8 @@ export interface PluginWorkloadSdkInvocationRecord {}// Schema lives in pluginWorkloadSdkSchema.ts (type-only db dependency) so-// db.ts can register migration 0027 without a runtime import cycle.+// db.ts can register migration 0034_plugin_workload_sdk without a runtime+// import cycle.export { applyPluginWorkloadSdkSchema } from './pluginWorkloadSdkSchema.js'// ─── Hashing helpers ─────────────────────────────────────────────────────diff --git a/control-api/src/services/pluginWorkloadSdkSchema.ts b/control-api/src/services/pluginWorkloadSdkSchema.tsindex ef62f61b7..865381a94 100644--- a/control-api/src/services/pluginWorkloadSdkSchema.ts+++ b/control-api/src/services/pluginWorkloadSdkSchema.ts@@ -13,7 +13,7 @@ export async function applyPluginWorkloadSdkSchema(db: DbClient): Promise<void>capability_family TEXT NOT NULL CHECK (capability_family IN ('promptBridge','clientNotifications')),-- Explicit provider bound to a promptBridge grant (R1: credentials resolve-- per-provider, not per-model). NULLABLE: rows written before migration- -- 0050 carry NULL and remain legacy/unreviewed until an operator+ -- 0055_plugin_workload_sdk_grant_provider carry NULL and remain legacy/unreviewed until an operator-- explicitly resaves an ordered target policy. clientNotifications-- grants leave it NULL.provider TEXT,@@ -358,7 +358,8 @@ export async function addPluginWorkloadSdkAttemptLedgerColumns(db: DbClient): Pr}/**- * Migration 0086 is intentionally a no-op for policy authority. A complete+ * Migration 0086_plugin_workload_sdk_legacy_policy_repair is intentionally a+ * no-op for policy authority. A complete* JSON shape is not provenance of an operator decision, so a legacy row must* never be reactivated automatically. The forward provenance migration below* fences rows on clusters that already ran the original unsafe implementation.diff --git a/control-api/test/db.gfsPermissionStoreMigration.test.ts b/control-api/test/db.gfsPermissionStoreMigration.test.tsindex d264de0cb..38de5eded 100644--- a/control-api/test/db.gfsPermissionStoreMigration.test.ts+++ b/control-api/test/db.gfsPermissionStoreMigration.test.ts@@ -131,7 +131,7 @@ describe('db migration 0048_gfs_permission_store', () => {expect(invalidation).toContain('gfs_notify_perm_invalidate')})- it('records the 0043 version exactly once', async () => {+ it('records the 0048_gfs_permission_store version exactly once', async () => {const { initDb } = await import('../src/db.js')await initDb()const recordCalls = clientQuery.mock.calls.filter(diff --git a/control-api/test/gfsBlobMigration.realPostgres.integration.test.ts b/control-api/test/gfsBlobMigration.realPostgres.integration.test.tsindex 20eea0d10..c04b50425 100644--- a/control-api/test/gfsBlobMigration.realPostgres.integration.test.ts+++ b/control-api/test/gfsBlobMigration.realPostgres.integration.test.ts@@ -46,7 +46,7 @@ describeRealPostgres('GFS immutable blob migration on real PostgreSQL', () => {await adminPool.end()})- it('applies 0068 exactly once with named constraints, indexes, and writer ACL', async () => {+ it('applies 0071_gfs_immutable_blob_generations exactly once with named constraints, indexes, and writer ACL', async () => {const versions = await pool.query<{ count: string }>(`SELECT count(*)::text AS count FROM schema_migrationsWHERE version = '0071_gfs_immutable_blob_generations'`diff --git a/control-api/test/gfsReadiness.realPostgres.integration.test.ts b/control-api/test/gfsReadiness.realPostgres.integration.test.tsindex 881c9544d..c91d4fe91 100644--- a/control-api/test/gfsReadiness.realPostgres.integration.test.ts+++ b/control-api/test/gfsReadiness.realPostgres.integration.test.ts@@ -88,7 +88,9 @@ describeRealPostgres('GFS Phase 0 real PostgreSQL readiness', () => {await client.query('BEGIN')await client.query('DROP TABLE gfs_blob_manifests')await client.query('SET LOCAL ROLE gfs_controller')- await expect(probeAs(client)()).rejects.toThrow(/gfs_blob_manifests|0068/i)+ await expect(probeAs(client)()).rejects.toThrow(+ /gfs_blob_manifests|0071_gfs_immutable_blob_generations/i+ )} finally {await client.query('ROLLBACK').catch(() => undefined)client.release()@@ -152,7 +154,7 @@ describeRealPostgres('GFS Phase 0 real PostgreSQL readiness', () => {await client.query(revokeSql)await client.query('SET LOCAL ROLE gfs_controller')await expect(probeAs(client)()).rejects.toThrow(- /coherence check failed|permission denied|0068|0048/i+ /coherence check failed|permission denied|0071_gfs_immutable_blob_generations|0048_gfs_permission_store/i)} finally {await client.query('ROLLBACK').catch(() => undefined)diff --git a/control-api/test/migrationReferences.test.ts b/control-api/test/migrationReferences.test.tsnew file mode 100644index 000000000..e4291f851--- /dev/null+++ b/control-api/test/migrationReferences.test.ts@@ -0,0 +1,166 @@+import { describe, expect, it } from 'vitest'+import { existsSync, readFileSync, readdirSync } from 'node:fs'+import { dirname, extname, join, relative } from 'node:path'+import { fileURLToPath } from 'node:url'+import { CONTROL_API_MIGRATIONS } from '../src/db.js'++// Operator-facing errors, comments and runbooks cite control-api migrations.+// Migrations have been renumbered before (5e6c990f8), which silently turned+// bare numbers like "migration 0068" into references to unrelated schema+// changes. In every scanned root, these checks pin each version-shaped token+// to a registered version and forbid citing a migration by bare number.+//+// Not covered: whether a registered version is the right one for the text+// around it (citing 0071_… where 0072_… belongs passes), and bare numbers not+// introduced by the word "migration(s)" (for example "the 0114 trigger").++const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), '../..')+const REGISTERED = new Set(CONTROL_API_MIGRATIONS.map(migration => migration.version))+const SCANNED_EXTENSIONS = new Set(['.ts', '.tsx', '.mjs', '.js', '.sh', '.md', '.yaml', '.yml'])+const SKIPPED_DIRECTORIES = new Set(['node_modules', 'dist', 'build', 'coverage', '.git'])+// control-api/test is not scanned: it holds seed fixtures and legacy-name+// lists that are version-shaped on purpose.+const SCANNED_ROOTS = ['gfs-controller/src', 'control-api/src', 'deploy', 'docs', 'scripts']++// Registered versions are four digits ("0071") or hex-style ("00a4").+const NUMBER = String.raw`(?:\d{4}|00[a-f]\d)`+const VERSION_TOKEN = new RegExp(String.raw`(?<![0-9A-Za-z_])${NUMBER}_[a-z][a-z0-9_]*`, 'g')+// Superseded names that the runner still accepts as already applied. Only+// tokens inside a legacyVersions array are exempt; the same name anywhere else+// is a stale reference.+const LEGACY_VERSIONS_ARRAY = /legacyVersions:\s*\[[^\]]*\]/g+// "migration(s)" followed by a list of versions, across line breaks and+// separators such as ":", "`", "(", ",", "/", "–", "and", "or".+const MIGRATION_LIST = new RegExp(+ String.raw`\bmigrations?\b((?:[\s:\`#(),/\-–]+|\b(?:and|or)\b|${NUMBER}(?:_[a-z][a-z0-9_]*)?(?![0-9A-Za-z_]))+)`,+ 'gi'+)+const BARE_NUMBER = new RegExp(String.raw`(?<![0-9A-Za-z_])${NUMBER}(?![0-9A-Za-z_])`, 'g')++interface MigrationReference {+ location: string+ token: string+}++function listSourceFiles(directory: string): string[] {+ return readdirSync(directory, { withFileTypes: true }).flatMap(entry => {+ const path = join(directory, entry.name)+ if (entry.isDirectory()) {+ return SKIPPED_DIRECTORIES.has(entry.name) ? [] : listSourceFiles(path)+ }+ return SCANNED_EXTENSIONS.has(extname(entry.name)) ? [path] : []+ })+}++function lineOf(text: string, index: number): number {+ return text.slice(0, index).split('\n').length+}++function legacySpans(text: string): Array<[number, number]> {+ return [...text.matchAll(LEGACY_VERSIONS_ARRAY)].map(match => [+ match.index,+ match.index + match[0].length,+ ])+}++function versionTokens(text: string): Array<{ index: number; token: string }> {+ const spans = legacySpans(text)+ return [...text.matchAll(VERSION_TOKEN)]+ .filter(match => !spans.some(([start, end]) => match.index >= start && match.index < end))+ .map(match => ({ index: match.index, token: match[0] }))+}++function bareNumbers(text: string): Array<{ index: number; token: string }> {+ return [...text.matchAll(MIGRATION_LIST)].flatMap(list => {+ const listStart = list.index + list[0].length - list[1].length+ return [...list[1].matchAll(BARE_NUMBER)].map(bare => ({+ index: listStart + bare.index,+ token: bare[0],+ }))+ })+}++function scan(+ root: string,+ find: (text: string) => Array<{ index: number; token: string }>+): { files: number; references: MigrationReference[] } {+ const directory = join(REPO_ROOT, root)+ if (!existsSync(directory)) {+ throw new Error(`scanned root ${root} does not exist; update SCANNED_ROOTS`)+ }+ const files = listSourceFiles(directory)+ const references = files.flatMap(file => {+ const text = readFileSync(file, 'utf8')+ return find(text).map(({ index, token }) => ({+ location: `${relative(REPO_ROOT, file)}:${lineOf(text, index)}`,+ token,+ }))+ })+ return { files: files.length, references }+}++function describeReferences(references: MigrationReference[]): string[] {+ return references.map(reference => `${reference.location} ${reference.token}`)+}++describe('control-api migration references', () => {+ it('detects bare numbers in every citation shape the guard relies on', () => {+ expect(+ [+ 'control-api migration 0068 not applied',+ 'migration: 0068',+ 'migration `0068`',+ 'migration\n 0068',+ 'migrations 0095_first, 0096 and 00a4',+ 'migrations 0095_first/0096',+ 'migrations 0109–0114',+ 'Migration 0055 replaces',+ ].map(sample => bareNumbers(sample).map(bare => bare.token))+ ).toEqual([+ ['0068'],+ ['0068'],+ ['0068'],+ ['0068'],+ ['0096', '00a4'],+ ['0096'],+ ['0109', '0114'],+ ['0055'],+ ])+ expect(+ bareNumbers('migration 0071_gfs_immutable_blob_generations not applied, or 0074_x')+ ).toEqual([])+ })++ it('exempts only tokens inside a legacyVersions array', () => {+ const text = [+ "legacyVersions: [\n '0055_governed_trace_runtime_roles',\n],",+ '// replaced by migration 0055_governed_trace_runtime_roles',+ ].join('\n')+ expect(versionTokens(text).map(found => found.token)).toEqual([+ '0055_governed_trace_runtime_roles',+ ])+ expect(versionTokens(text)[0].index).toBeGreaterThan(text.indexOf('],'))+ })++ it.each(SCANNED_ROOTS)('every version token in %s is a registered migration', root => {+ const { files, references } = scan(root, versionTokens)++ // Liveness witness: each root cites real versions today, so an empty+ // result means the scan stopped reading files, not that they are clean.+ expect(files).toBeGreaterThan(0)+ expect(references.length).toBeGreaterThan(0)+ expect(+ describeReferences(references.filter(reference => !REGISTERED.has(reference.token)))+ ).toEqual([])+ })++ it.each(SCANNED_ROOTS)('%s never cites a control-api migration by bare number', root => {+ const { references } = scan(root, bareNumbers)+ const { references: named } = scan(root, versionTokens)++ // Liveness witness: each root cites named migrations, so the files this+ // check reads are the ones that carry migration citations.+ expect(named.length).toBeGreaterThan(0)+ expect(describeReferences(references)).toEqual([])+ })+})diff --git a/deploy/scripts/provision-control-api-runtime-roles.sh b/deploy/scripts/provision-control-api-runtime-roles.shindex 1c4c16ffc..0fbf6f926 100755--- a/deploy/scripts/provision-control-api-runtime-roles.sh+++ b/deploy/scripts/provision-control-api-runtime-roles.sh@@ -1,7 +1,9 @@#!/usr/bin/env bash# Reconcile least-privilege Postgres logins used by control-api tracing paths.#-# Migration 0053 owns role creation and grants. This script owns only the+# Control-api migrations own role creation and grants: the roles are created by+# migration 0062_governed_trace_runtime_roles, and later migrations adjust+# their grants. This script owns only the# runtime credential lifecycle: it preserves an existing valid credential or# generates one on first provisioning, applies it to the fixed database role,# and patches the matching Kubernetes Secret. No credential is accepted fromdiff --git a/deploy/scripts/run-control-api-db-migration.sh b/deploy/scripts/run-control-api-db-migration.shindex 642b1e78f..2929e85d6 100755--- a/deploy/scripts/run-control-api-db-migration.sh+++ b/deploy/scripts/run-control-api-db-migration.sh@@ -351,7 +351,7 @@ build_job_manifest() {},"spec" => {# Retry a transient migration abort instead of failing the whole deploy.- # Migration 0090 takes an EXCLUSIVE lock on agent_run_events with a bounded+ # Migration 0090_plugin_workload_sdk_runtime_contract_reconciliation takes an EXCLUSIVE lock on agent_run_events with a bounded# lock_timeout; if a concurrent writer (e.g. the trace-retention prune# DELETE) holds its lock past that timeout the Job exits 1. The migration# body is idempotent, so re-running is safe; restartPolicy Never makes eachdiff --git a/docs/architecture/codex-subscription-transport-contract.md b/docs/architecture/codex-subscription-transport-contract.mdindex c22645472..b3044f226 100644--- a/docs/architecture/codex-subscription-transport-contract.md+++ b/docs/architecture/codex-subscription-transport-contract.md@@ -165,7 +165,7 @@ behavior changes:with `state_cancelled`.- When two first grants race on the same key, the losing active-key insertmaps to a stale-revision conflict instead of a 500.-- **Attempt integrity (migration 0114).** A constraint trigger requires a+- **Attempt integrity (migration 0114_llm_provider_attempts_connection_integrity).** A constraint trigger requires anon-null `llm_provider_attempts.connection_id` of a Codex attempt to exist in`codex_subscription_connections`. This replaces the FK that 0112 dropped.Historic rows are not revalidated, and the migration is forward-only.diff --git a/docs/architecture/grok-subscription-transport-contract.md b/docs/architecture/grok-subscription-transport-contract.mdindex c512e963d..56879cacb 100644--- a/docs/architecture/grok-subscription-transport-contract.md+++ b/docs/architecture/grok-subscription-transport-contract.md@@ -77,7 +77,7 @@ an error.## Connection lifecycle-- **Grok connection keys are terminal after revoke (migration 0113).**+- **Grok connection keys are terminal after revoke (migration 0113_grok_subscription_terminal_connection_key).**- A revoked key cannot be reconnected. Create a new connection key instead.- A device flow that started before the revoke cannot revive the key: itfails with `revoked grant cannot be reused`.@@ -87,7 +87,7 @@ an error.`connection_key`.- Archived `~revoked~` rows fall outside the key grammar. Admin list and GEThide them. The key's own terminal tombstone stays listed as `revoked`.-- **Attempt integrity (migration 0114).** A constraint trigger on+- **Attempt integrity (migration 0114_llm_provider_attempts_connection_integrity).** A constraint trigger on`llm_provider_attempts` INSERT, and on UPDATE of `provider` or`connection_id`, requires a non-null `connection_id` to exist in theprovider's own connection table:@@ -245,7 +245,8 @@ Grok annotations. A new control-api also republishes on boot.(`GROK_LLM_PROXY_CONTROL_API_TOKEN`) with`deploy/scripts/apply-inter-service-tokens.sh`. Both must be in place beforethe merge to `dev` triggers the automatic deploy.-2. **control-api** with migrations 0109–0114. Wait until every control-api pod+2. **control-api** with migrations 0109_grok_subscription_connections through+ 0114_llm_provider_attempts_connection_integrity. Wait until every control-api podruns the new image. An old control-api rejects the whole workflow-controltoken issue when the request includes the unknown `llm:grok:execute` scope(`invalid_workflow_control_scopes`). HCC or WRC deployed first could@@ -276,7 +277,8 @@ Grok annotations. A new control-api also republishes on boot.shape. In the UI, re-pick the connection.- A save that omits the annotations keeps the stored disagreeing pair, soit does not repair the grant.-4. Migrations 0113 (terminal Grok keys) and 0114 (attempt connection+4. Migrations 0113_grok_subscription_terminal_connection_key (terminal Grok keys)+ and 0114_llm_provider_attempts_connection_integrity (attempt connectionintegrity) are forward-only. An older control-api still runs against them:- The 0114 trigger only rejects connection ids that do not exist.- Under 0113, a revoked Grok key can no longer be reconnected by the olddiff --git a/docs/deploy/gfs-permission-store.md b/docs/deploy/gfs-permission-store.mdindex 8706cba33..983288719 100644--- a/docs/deploy/gfs-permission-store.md+++ b/docs/deploy/gfs-permission-store.md@@ -166,8 +166,9 @@ store. The store check has two layers (`gfs-controller/src/authz/storeProbe.ts`)the rotation and the kubelet probe cadence keeps one alive forever. Onlysuccess is cached; while failing, every probe retries.-Net effect: a stale DSN, a rotated role password, or missing migration-0048-grants flips the pod NotReady within roughly a minute, visibly in+Net effect: a stale DSN, a rotated role password, or missing role grants+(migration 0048_gfs_permission_store for the writer,+0072_gfs_reader_database_role for the reader) flips the pod NotReady within roughly a minute, visibly in`kubectl get pods -n gfs`, instead of serving chronic 503s to users.`/readyz` reasons you may see:@@ -176,7 +177,7 @@ grants flips the pod NotReady within roughly a minute, visibly in| ------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- || `storage volume not mounted …` | Drive PVC missing/unmounted (not a credential issue) || `permission store unreachable: password authentication failed …` | DSN/password drift — re-run provisioning |-| `permission store unreachable: … coherence check failed … gfs_resources … 0048` | Role exists but the SELECT grants/migration are missing — run control-api migrations, then provisioning |+| `permission store unreachable: … coherence check failed … gfs_resources … 0048_gfs_permission_store` (writer) or `… 0072_gfs_reader_database_role` (reader) | Role exists but the SELECT grants/migration are missing — run control-api migrations, then provisioning || `permission store unreachable: … coherence check failed … gfs_audit …` | Audit INSERT grant missing — every request would 503 (audit-write failures propagate); run migrations + provisioning || `permission store unreachable: … timed out after Nms` | Black-hole partition (unreachable, not refusing) — the probe hard-bounds connect/query (default 5s) instead of dangling || `permission store unreachable: connect ECONNREFUSED …` | PostgreSQL down/unreachable |@@ -222,7 +223,8 @@ CONTEXT=<kube-context> deploy/scripts/provision-gfs-db.sh rotate-readerMINIKUBE_PROFILE=<profile> make minikube-verify-gfs```-The script verifies the role exists (writer migration 0048; reader migration 0071),+The script verifies the role exists (writer migration 0048_gfs_permission_store; reader migration+0072_gfs_reader_database_role),rotates the password, patches the Secret, stamps the rotation annotation, androlls and waits only for the selected gfsc Deployment.diff --git a/gfs-controller/src/authz/storeProbe.test.ts b/gfs-controller/src/authz/storeProbe.test.tsindex 9f464f0e0..473444929 100644--- a/gfs-controller/src/authz/storeProbe.test.ts+++ b/gfs-controller/src/authz/storeProbe.test.ts@@ -176,33 +176,58 @@ describe("createPermissionStoreProbe", () => {await expect(probe()).rejects.toThrow(/password authentication failed/);});- it("rejects with a migration hint when the role lacks SELECT on gfs_resources", async () => {- const { factory } = fakeClientFactory({ canRead: false });- const probe = createPermissionStoreProbe({- pool: fakePool(),- connectionString: DSN,- intervalMs: INTERVAL,- storageRole: "reader",- clientFactory: factory,- now: () => 0,- });- await expect(probe()).rejects.toThrow(/gfs_resources.*0048/s);- });+ // Each role's base grants come from a different migration, so the hint must+ // name the one that actually granted the missing privilege to that role.+ it.each([+ ["writer", "0048_gfs_permission_store"],+ ["reader", "0072_gfs_reader_database_role"],+ ] as const)(+ "rejects a %s lacking SELECT on gfs_resources with its own grant migration hint",+ async (storageRole, grantMigration) => {+ const { factory } = fakeClientFactory({ canRead: false });+ const probe = createPermissionStoreProbe({+ pool: fakePool(),+ connectionString: DSN,+ intervalMs: INTERVAL,+ storageRole,+ clientFactory: factory,+ now: () => 0,+ });+ await expect(probe()).rejects.toThrow(+ new RegExp(+ `lacks SELECT on gfs_resources \\(control-api migration ${grantMigration} not applied, ` ++ "or 0074_gfs_runtime_role_exact_contract grants drifted\\?\\)"+ )+ );+ }+ );- it("rejects when the role lacks INSERT on gfs_audit (chronic per-request 503 mode)", async () => {- const { factory } = fakeClientFactory({ canAudit: false });- const probe = createPermissionStoreProbe({- pool: fakePool(),- connectionString: DSN,- intervalMs: INTERVAL,- storageRole: "reader",- clientFactory: factory,- now: () => 0,- });- await expect(probe()).rejects.toThrow(/gfs_audit.*INSERT|INSERT on gfs_audit/s);- });+ it.each([+ ["writer", "0048_gfs_permission_store"],+ ["reader", "0072_gfs_reader_database_role"],+ ] as const)(+ "rejects a %s lacking INSERT on gfs_audit (chronic per-request 503 mode) with its own grant migration hint",+ async (storageRole, grantMigration) => {+ const { factory } = fakeClientFactory({ canAudit: false });+ const probe = createPermissionStoreProbe({+ pool: fakePool(),+ connectionString: DSN,+ intervalMs: INTERVAL,+ storageRole,+ clientFactory: factory,+ now: () => 0,+ });+ await expect(probe()).rejects.toThrow(+ new RegExp(+ `lacks INSERT on gfs_audit \\(audit-write failures would 503 every request; ` ++ `control-api migration ${grantMigration} not applied, ` ++ "or 0074_gfs_runtime_role_exact_contract grants drifted\\?\\)"+ )+ );+ }+ );- it("rejects a pre-0068 reader schema before serving requests", async () => {+ it("rejects a pre-0071_gfs_immutable_blob_generations reader schema before serving requests", async () => {const { factory } = fakeClientFactory({ immutableSchemaReady: false });const probe = createPermissionStoreProbe({pool: fakePool(),@@ -212,9 +237,30 @@ describe("createPermissionStoreProbe", () => {clientFactory: factory,now: () => 0,});- await expect(probe()).rejects.toThrow(/migration 0068/);+ await expect(probe()).rejects.toThrow(+ /migration 0071_gfs_immutable_blob_generations not applied, or 0074_gfs_runtime_role_exact_contract grants drifted\?/+ );});+ it.each([+ ["schema", { auditDecisionEvidenceSchemaReady: false }],+ ["constraint set", { auditDecisionEvidenceConstraintsReady: false }],+ ] satisfies Array<[string, FakeClientBehavior]>)(+ "rejects readiness before 0073_gfs_audit_decision_evidence when the audit decision-evidence %s is missing",+ async (_part, behavior) => {+ const { factory } = fakeClientFactory(behavior);+ const probe = createPermissionStoreProbe({+ pool: fakePool(),+ connectionString: DSN,+ intervalMs: INTERVAL,+ storageRole: "reader",+ clientFactory: factory,+ now: () => 0,+ });+ await expect(probe()).rejects.toThrow(/migration 0073_gfs_audit_decision_evidence /);+ }+ );+it("keeps reader readiness independent of the writer-only manifest schema", async () => {const probeClient = fakeClientFactory({});const probe = createPermissionStoreProbe({@@ -233,7 +279,7 @@ describe("createPermissionStoreProbe", () => {expect(probeClient.lastSql()).toContain("gfs_audit_record_type_fields_valid");});- it("rejects readiness when migration 0096 default or backfill is incomplete", async () => {+ it("rejects readiness when migration 0096_control_admin_session_version_default default or backfill is incomplete", async () => {const { factory } = fakeClientFactory({ authorityAdminSessionEpochReady: false });const probe = createPermissionStoreProbe({pool: fakePool(),@@ -243,7 +289,7 @@ describe("createPermissionStoreProbe", () => {clientFactory: factory,now: () => 0,});- await expect(probe()).rejects.toThrow(/0095\/0096/);+ await expect(probe()).rejects.toThrow(/migrations 0095_gfs_lifecycle_authority_projection\/0096_control_admin_session_version_default /);});it.each([@@ -265,7 +311,7 @@ describe("createPermissionStoreProbe", () => {clientFactory: factory,now: () => 0,});- await expect(probe()).rejects.toThrow(/reader has forbidden GFS mutation privileges.*0069/s);+ await expect(probe()).rejects.toThrow(/reader has forbidden GFS mutation privileges.*migration 0074_gfs_runtime_role_exact_contract /s);});@@ -286,7 +332,7 @@ describe("createPermissionStoreProbe", () => {});it.each(["actor_on_behalf_of", "desktop_user_id", "authority_source"] as const)(- "rejects readiness before 0092 when the required %s audit column is absent or drifted",+ "rejects readiness before 0092_gfs_audit_actor_correlation when the required %s audit column is absent or drifted",async (_column) => {const { factory } = fakeClientFactory({ auditActorCorrelationColumnsReady: false });const probe = createPermissionStoreProbe({@@ -297,7 +343,7 @@ describe("createPermissionStoreProbe", () => {clientFactory: factory,now: () => 0,});- await expect(probe()).rejects.toThrow(/migration 0092/);+ await expect(probe()).rejects.toThrow(/migration 0092_gfs_audit_actor_correlation /);});@@ -309,7 +355,7 @@ describe("createPermissionStoreProbe", () => {["admin privileges", { authorityAdminPrivilegesReady: false }],["link privileges", { authorityLinksPrivilegesReady: false }],] satisfies Array<[string, FakeClientBehavior]>)(- "rejects readiness when the 0095 authority projection is missing (%s)",+ "rejects readiness when the 0095_gfs_lifecycle_authority_projection authority projection is missing (%s)",async (_projection, behavior) => {const { factory } = fakeClientFactory(behavior);const probe = createPermissionStoreProbe({@@ -320,12 +366,14 @@ describe("createPermissionStoreProbe", () => {clientFactory: factory,now: () => 0,});- await expect(probe()).rejects.toThrow(/migration 0095/);+ await expect(probe()).rejects.toThrow(+ /migrations 0095_gfs_lifecycle_authority_projection\/0096_control_admin_session_version_default /+ );});it.each(["reader", "writer"] as const)(- "rejects a %s when migration 0092 audit actor-correlation constraints are missing or unvalidated",+ "rejects a %s when migration 0092_gfs_audit_actor_correlation audit actor-correlation constraints are missing or unvalidated",async (storageRole) => {const { factory } = fakeClientFactory({ auditActorCorrelationConstraintReady: false });const probe = createPermissionStoreProbe({@@ -336,7 +384,7 @@ describe("createPermissionStoreProbe", () => {clientFactory: factory,now: () => 0,});- await expect(probe()).rejects.toThrow(/migration 0092/);+ await expect(probe()).rejects.toThrow(/migration 0092_gfs_audit_actor_correlation /);});@@ -358,7 +406,9 @@ describe("createPermissionStoreProbe", () => {clientFactory: factory,now: () => 0,});- await expect(probe()).rejects.toThrow(/writer lacks.*0068/s);+ await expect(probe()).rejects.toThrow(+ /writer lacks.*migration 0071_gfs_immutable_blob_generations not applied, or 0074_gfs_runtime_role_exact_contract grants drifted\?/s+ );});diff --git a/gfs-controller/src/authz/storeProbe.ts b/gfs-controller/src/authz/storeProbe.tsindex d003229e5..3517b16f0 100644--- a/gfs-controller/src/authz/storeProbe.ts+++ b/gfs-controller/src/authz/storeProbe.ts@@ -15,7 +15,10 @@ import { Client } from "pg";* 2. Amortized fresh-connection probe (at most once per `intervalMs`) — opens* a brand-new client (NEVER the pool) so a rotated credential fails to* authenticate, and verifies real coherence with `has_table_privilege`, so- * "password correct but migration 0048 grants missing" also fails loud.+ * "password correct but role grants missing" also fails loud. The writer's+ * grants come from migration 0048_gfs_permission_store, the reader's from+ * 0072_gfs_reader_database_role, and 0074_gfs_runtime_role_exact_contract+ * rebuilds both envelopes.** FAIL CLOSED: only SUCCESS is cached. While failing, every readiness call* retries the fresh connection (~1 new connection per probe period during an@@ -191,6 +194,13 @@ const WRITER_COHERENCE_SQL =export function createPermissionStoreProbe(opts: StoreProbeOptions): () => Promise<void> {const now = opts.now ?? Date.now;const connectTimeoutMs = opts.connectTimeoutMs ?? 5_000;+ // Each role gets its base grants from its own migration; 0074 later revokes+ // every table and column privilege and rebuilds both envelopes exactly.+ const roleGrantsMigration =+ opts.storageRole === "writer" ? "0048_gfs_permission_store" : "0072_gfs_reader_database_role";+ const roleGrantsHint =+ `control-api migration ${roleGrantsMigration} not applied, ` ++ "or 0074_gfs_runtime_role_exact_contract grants drifted?";// @types/pg resolves Client.connect() to Promise<Client>; the probe contract// wants Promise<void> — awaited and discarded (same adaptation as the LISTEN// client in index.ts). connectionTimeoutMillis is belt-and-suspenders under@@ -270,13 +280,13 @@ export function createPermissionStoreProbe(opts: StoreProbeOptions): () => Promiif (row?.can_read !== true) {throw new Error("permission store coherence check failed: role lacks SELECT on gfs_resources " +- "(control-api migration 0048 not applied?)"+ `(${roleGrantsHint})`);}if (row?.can_audit !== true) {throw new Error("permission store coherence check failed: role lacks INSERT on gfs_audit " +- "(audit-write failures would 503 every request; control-api migration 0048 not applied?)"+ `(audit-write failures would 503 every request; ${roleGrantsHint})`);}if (@@ -286,7 +296,8 @@ export function createPermissionStoreProbe(opts: StoreProbeOptions): () => Promi) {throw new Error("permission store coherence check failed: immutable GFS blob schema/reader privileges " +- "are missing (control-api migration 0068 not applied?)"+ "are missing (control-api migration 0071_gfs_immutable_blob_generations not applied, " ++ "or 0074_gfs_runtime_role_exact_contract grants drifted?)");}if (@@ -295,7 +306,7 @@ export function createPermissionStoreProbe(opts: StoreProbeOptions): () => Promi) {throw new Error("permission store coherence check failed: GFS audit decision-evidence schema is missing " +- "(control-api migration 0073 not applied?)"+ "(control-api migration 0073_gfs_audit_decision_evidence not applied?)");}if (@@ -304,7 +315,7 @@ export function createPermissionStoreProbe(opts: StoreProbeOptions): () => Promi) {throw new Error("permission store coherence check failed: GFS audit actor-correlation schema is missing " +- "(control-api migration 0092 not applied?)"+ "(control-api migration 0092_gfs_audit_actor_correlation not applied?)");}if (@@ -318,7 +329,7 @@ export function createPermissionStoreProbe(opts: StoreProbeOptions): () => Promi) {throw new Error("permission store coherence check failed: GFS lifecycle authority projection is missing " +- "(control-api migration 0095/0096 not applied?)"+ "(control-api migrations 0095_gfs_lifecycle_authority_projection/0096_control_admin_session_version_default not applied?)");}if (@@ -332,7 +343,7 @@ export function createPermissionStoreProbe(opts: StoreProbeOptions): () => Promi) {throw new Error("permission store coherence check failed: reader has forbidden GFS mutation privileges " +- "(control-api migration 0069 grants drifted?)"+ "(control-api migration 0074_gfs_runtime_role_exact_contract grants drifted?)");}if (@@ -347,7 +358,8 @@ export function createPermissionStoreProbe(opts: StoreProbeOptions): () => Promi) {throw new Error("permission store coherence check failed: writer lacks GFS manifest/resource mutation " +- "privileges (control-api migration 0068 not applied?)"+ "privileges (control-api migration 0071_gfs_immutable_blob_generations not applied, " ++ "or 0074_gfs_runtime_role_exact_contract grants drifted?)");}lastFreshOkAt = now(); // cache SUCCESS only — failures always retrydiff --git a/gfs-controller/src/index.ts b/gfs-controller/src/index.tsindex 1a4cf9f38..b8e475006 100644--- a/gfs-controller/src/index.ts+++ b/gfs-controller/src/index.ts@@ -138,7 +138,9 @@ async function main(): Promise<void> {// detect a rotated password — its idle clients authenticated before the// rotation and the readiness cadence keeps one alive forever. The probe// keeps the fast pool ping AND dials a brand-new client (amortized) so a- // stale DSN or missing migration-0048 grants flips the pod NotReady.+ // stale DSN or missing role grants (migration 0048_gfs_permission_store for+ // the writer, 0072_gfs_reader_database_role for the reader) flips the pod+ // NotReady.pingPermissionStore: createPermissionStoreProbe({pool,connectionString: config.pgConnectionString,diff --git a/scripts/e2e/e2e-figure-d-multibot.sh b/scripts/e2e/e2e-figure-d-multibot.shindex fe6d9db6c..947de205b 100755--- a/scripts/e2e/e2e-figure-d-multibot.sh+++ b/scripts/e2e/e2e-figure-d-multibot.sh@@ -6,7 +6,7 @@# Companion gate for PR1 (.ralph/plans/figure-d-multibot-pr1.md). Proves, on a# live cluster, what unit tests cannot:#-# B4 (always) — migration 0039 applied: communication_channel_ref column+# B4 (always) — migration 0039_wama_communication_channel_ref applied: communication_channel_ref column# + idx_wama_channel_ref index + status CHECK admits# skipped_no_bot + ONLY telegram NULL-ref rows force-disabled# (Slack / other models untouched).@@ -73,8 +73,8 @@ if command -v git >/dev/null 2>&1; thenlog "Evidence: HEAD=$(git rev-parse --short HEAD 2>/dev/null || echo '?') context=$KCTX"fi-# ─── B4: migration 0039 applied ────────────────────────────────────────-log "B4 — migration 0039 + schema"+# ─── B4: migration 0039_wama_communication_channel_ref applied ────────────────────────────────────────+log "B4 — migration 0039_wama_communication_channel_ref + schema"ver="$(psql_q "SELECT version FROM schema_migrations WHERE version = '0039_wama_communication_channel_ref';")"[ "$ver" = "0039_wama_communication_channel_ref" ] \diff --git a/scripts/minikube/full-setup.sh b/scripts/minikube/full-setup.shindex 97860ebdb..514df9120 100755--- a/scripts/minikube/full-setup.sh+++ b/scripts/minikube/full-setup.sh@@ -2181,7 +2181,7 @@ done# ----------------------------------------------------------------------# gfs serving provisioning (runs once pods are up, so control-api has applied-# migration 0048 that creates the gfs_controller role). gfsc fails closed+# migration 0048_gfs_permission_store that creates the gfs_controller role). gfsc fails closed# without BOTH: (a) its JWT public key in gfs-config, and (b) the gfs_controller# DB LOGIN + DSN in gfs-controller-db. The inline JWT sync (6g) only covers# mcp-host-config, so these gfs-specific steps are explicit here. Idempotent.diff --git a/scripts/minikube/pre-gate-sync.sh b/scripts/minikube/pre-gate-sync.shindex 40908fac9..15e860867 100755--- a/scripts/minikube/pre-gate-sync.sh+++ b/scripts/minikube/pre-gate-sync.sh@@ -772,7 +772,7 @@ if [[ "${cluster_changed}" == "true" ]]; thenrestore_control_api# The base manifest no longer declares connection-string (provisioning-owned# key), so deploy-all cannot clobber it. Provisioning must still run AFTER- # control-api migrations (0048 creates the least-privilege gfs_controller+ # control-api migrations (0048_gfs_permission_store creates the least-privilege gfs_controller# role) for fresh profiles and to converge any stale credential before the# rest of the gate observes service readiness.provision_gfs_serving