Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix: correct stale control-api migration references and guard them (#689)

ALFREDO J LOPEZ H (alfredolopez80.eth)ea2375678eb8
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index b88ea1a7d..9837b354e 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.584",
+ "version": "0.1.586",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.584",
+ "version": "0.1.586",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 7904ea41c..ddcea757c 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.584",
+ "version": "0.1.586",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/db.ts b/control-api/src/db.ts
index 04cc4f61f..e710d5d98 100644
--- a/control-api/src/db.ts
+++ b/control-api/src/db.ts
@@ -168,7 +168,8 @@ export const corePool = pool
async function applyBaselineSchema(db: DbClient): Promise<void> {
// Baseline includes additive Phase 0 workflow-trigger tables for fresh
- // clusters. Existing clusters receive the same tables through migration 0016;
+ // clusters. Existing clusters receive the same tables through migration
+ // 0016_workflow_trigger_shared_foundation;
// that migration also remains responsible for backfill and preflight checks.
await db.query(`
CREATE EXTENSION IF NOT EXISTS "pgcrypto";
@@ -1320,7 +1321,7 @@ async function alignWorkflowRunsAuditRecipeIndex(db: DbClient): Promise<void> {
* Drop the `trigger_grants_audit.operator_user_id_fkey` foreign key on
* clusters that bootstrapped before commit 68c81bea.
*
- * Context: the baseline migration (0001) originally declared this FK as
+ * Context: the baseline migration (0001_control_api_baseline) originally declared this FK as
* `REFERENCES users(id)`. Commit 68c81bea edited the baseline body in place
* to add an `ALTER TABLE … DROP CONSTRAINT IF EXISTS …` — but baseline was
* already recorded in `schema_migrations` on every long-lived cluster, so
@@ -4410,8 +4411,9 @@ export const CONTROL_API_MIGRATIONS: DbMigration[] = [
DROP TRIGGER IF EXISTS infrastructure_cost_daily_components_no_truncate ON infrastructure_cost_daily_components;
CREATE TRIGGER infrastructure_cost_daily_components_no_truncate BEFORE TRUNCATE ON infrastructure_cost_daily_components FOR EACH STATEMENT EXECUTE FUNCTION governed_trace_reject_truncate();
- -- Migration 0055 replaces this bootstrap function with the final
- -- owner-bound retention implementation and dedicated runtime roles.
+ -- Migration 0062_governed_trace_runtime_roles replaces this bootstrap
+ -- function with the final owner-bound retention implementation and
+ -- dedicated runtime roles.
CREATE OR REPLACE FUNCTION governed_trace_prune_expired_events(
requested_family TEXT,
batch_limit INTEGER DEFAULT 1000
@@ -6141,8 +6143,9 @@ async function consolidateWorkflowAllowedUsersToTriggers(db: DbClient): Promise<
const roleRow = usersRoleColumn.rows[0] as { exists: boolean } | undefined
if (roleRow?.exists) {
// Re-seed the sentinel binding (admins → mcp-host/standalone) into the
- // canonical table. Migration 0006 seeded it into the legacy table; this is
- // idempotent against admins that joined since 0006 ran.
+ // canonical table. Migration 0006_seed_sentinel_allowlist_for_admins seeded
+ // it into the legacy table; this is idempotent against admins that joined
+ // since that migration ran.
await db.query(`
INSERT INTO user_workflow_triggers (user_id, recipe_namespace, recipe_name)
SELECT id, 'mcp-host', 'standalone' FROM users WHERE role = 'admin'
diff --git a/control-api/src/routes/admin/codexSubscription.ts b/control-api/src/routes/admin/codexSubscription.ts
index 15575ea12..cf16c186d 100644
--- a/control-api/src/routes/admin/codexSubscription.ts
+++ b/control-api/src/routes/admin/codexSubscription.ts
@@ -349,7 +349,8 @@ export function createAdminCodexSubscriptionRouter(
}
/**
- * Migration 0113 archives superseded tombstones as `<key>~revoked~<id>`.
+ * Migration 0113_grok_subscription_terminal_connection_key archives
+ * superseded tombstones as `<key>~revoked~<id>`.
* Those keys sit outside the key grammar, so no keyed route can address
* them; listing them would only surface unactionable duplicates. The key's
* own terminal tombstone (status `revoked`, `revokedAt` set) stays listed.
diff --git a/control-api/src/services/grokSubscriptionConnection.ts b/control-api/src/services/grokSubscriptionConnection.ts
index 727868ec2..74531956a 100644
--- a/control-api/src/services/grokSubscriptionConnection.ts
+++ b/control-api/src/services/grokSubscriptionConnection.ts
@@ -56,7 +56,8 @@ export class GrokSubscriptionFingerprintConflictError extends Error {
/**
* A write lost the connection_key uniqueness race: another writer created the
* key first, or the key is a revoked tombstone (revocation is terminal per key
- * since migration 0113). Callers re-read with
+ * since migration 0113_grok_subscription_terminal_connection_key). Callers
+ * re-read with
* `getGrokSubscriptionConnectionIncludingRevoked` to tell the two apart.
*/
export class GrokSubscriptionConnectionKeyConflictError extends Error {
diff --git a/control-api/src/services/hostHeartbeatService.ts b/control-api/src/services/hostHeartbeatService.ts
index 1b18bfe65..eccd4ca2a 100644
--- a/control-api/src/services/hostHeartbeatService.ts
+++ b/control-api/src/services/hostHeartbeatService.ts
@@ -1,7 +1,7 @@
import { pool } from '../db.js'
/**
- * Stateless heartbeat persistence (host_heartbeats, migration 0051).
+ * Stateless heartbeat persistence (host_heartbeats, migration 0051_host_heartbeats).
*
* mcp-host pods POST D8 activity snapshots to control-api's /mcp-host facade
* (`POST /api/v1/mcp-host/hosts/heartbeat`, identity bound to the runtime
diff --git a/control-api/src/services/memberRegistrationCredentialsSchema.ts b/control-api/src/services/memberRegistrationCredentialsSchema.ts
index be99ed5c7..8d20b6663 100644
--- a/control-api/src/services/memberRegistrationCredentialsSchema.ts
+++ b/control-api/src/services/memberRegistrationCredentialsSchema.ts
@@ -1,6 +1,6 @@
import type { DbClient } from '../db.js'
-// ─── Hosted member-registration credentials — schema (migration 0068) ──
+// ─── Hosted member-registration credentials — schema (migration 0068_member_registration_credentials) ──
// One row per (bound_domain, mint). The hub binds each credential to exactly one
// destination hostname; hosted mode mints one per configured UI host (spec §8.2).
// `secret_encrypted` is AES-256-GCM via src/oauth/encryption.ts — never plaintext.
diff --git a/control-api/src/services/pluginWorkloadSdkDb.ts b/control-api/src/services/pluginWorkloadSdkDb.ts
index de4d91ea8..8141f35f7 100644
--- a/control-api/src/services/pluginWorkloadSdkDb.ts
+++ b/control-api/src/services/pluginWorkloadSdkDb.ts
@@ -352,7 +352,8 @@ export interface PluginWorkloadSdkInvocationRecord {
}
// Schema lives in pluginWorkloadSdkSchema.ts (type-only db dependency) so
-// db.ts can register migration 0027 without a runtime import cycle.
+// db.ts can register migration 0034_plugin_workload_sdk without a runtime
+// import cycle.
export { applyPluginWorkloadSdkSchema } from './pluginWorkloadSdkSchema.js'
// ─── Hashing helpers ─────────────────────────────────────────────────────
diff --git a/control-api/src/services/pluginWorkloadSdkSchema.ts b/control-api/src/services/pluginWorkloadSdkSchema.ts
index ef62f61b7..865381a94 100644
--- a/control-api/src/services/pluginWorkloadSdkSchema.ts
+++ b/control-api/src/services/pluginWorkloadSdkSchema.ts
@@ -13,7 +13,7 @@ export async function applyPluginWorkloadSdkSchema(db: DbClient): Promise<void>
capability_family TEXT NOT NULL CHECK (capability_family IN ('promptBridge','clientNotifications')),
-- Explicit provider bound to a promptBridge grant (R1: credentials resolve
-- per-provider, not per-model). NULLABLE: rows written before migration
- -- 0050 carry NULL and remain legacy/unreviewed until an operator
+ -- 0055_plugin_workload_sdk_grant_provider carry NULL and remain legacy/unreviewed until an operator
-- explicitly resaves an ordered target policy. clientNotifications
-- grants leave it NULL.
provider TEXT,
@@ -358,7 +358,8 @@ export async function addPluginWorkloadSdkAttemptLedgerColumns(db: DbClient): Pr
}
/**
- * Migration 0086 is intentionally a no-op for policy authority. A complete
+ * Migration 0086_plugin_workload_sdk_legacy_policy_repair is intentionally a
+ * no-op for policy authority. A complete
* JSON shape is not provenance of an operator decision, so a legacy row must
* never be reactivated automatically. The forward provenance migration below
* fences rows on clusters that already ran the original unsafe implementation.
diff --git a/control-api/test/db.gfsPermissionStoreMigration.test.ts b/control-api/test/db.gfsPermissionStoreMigration.test.ts
index d264de0cb..38de5eded 100644
--- a/control-api/test/db.gfsPermissionStoreMigration.test.ts
+++ b/control-api/test/db.gfsPermissionStoreMigration.test.ts
@@ -131,7 +131,7 @@ describe('db migration 0048_gfs_permission_store', () => {
expect(invalidation).toContain('gfs_notify_perm_invalidate')
})
- it('records the 0043 version exactly once', async () => {
+ it('records the 0048_gfs_permission_store version exactly once', async () => {
const { initDb } = await import('../src/db.js')
await initDb()
const recordCalls = clientQuery.mock.calls.filter(
diff --git a/control-api/test/gfsBlobMigration.realPostgres.integration.test.ts b/control-api/test/gfsBlobMigration.realPostgres.integration.test.ts
index 20eea0d10..c04b50425 100644
--- a/control-api/test/gfsBlobMigration.realPostgres.integration.test.ts
+++ b/control-api/test/gfsBlobMigration.realPostgres.integration.test.ts
@@ -46,7 +46,7 @@ describeRealPostgres('GFS immutable blob migration on real PostgreSQL', () => {
await adminPool.end()
})
- it('applies 0068 exactly once with named constraints, indexes, and writer ACL', async () => {
+ it('applies 0071_gfs_immutable_blob_generations exactly once with named constraints, indexes, and writer ACL', async () => {
const versions = await pool.query<{ count: string }>(
`SELECT count(*)::text AS count FROM schema_migrations
WHERE version = '0071_gfs_immutable_blob_generations'`
diff --git a/control-api/test/gfsReadiness.realPostgres.integration.test.ts b/control-api/test/gfsReadiness.realPostgres.integration.test.ts
index 881c9544d..c91d4fe91 100644
--- a/control-api/test/gfsReadiness.realPostgres.integration.test.ts
+++ b/control-api/test/gfsReadiness.realPostgres.integration.test.ts
@@ -88,7 +88,9 @@ describeRealPostgres('GFS Phase 0 real PostgreSQL readiness', () => {
await client.query('BEGIN')
await client.query('DROP TABLE gfs_blob_manifests')
await client.query('SET LOCAL ROLE gfs_controller')
- await expect(probeAs(client)()).rejects.toThrow(/gfs_blob_manifests|0068/i)
+ await expect(probeAs(client)()).rejects.toThrow(
+ /gfs_blob_manifests|0071_gfs_immutable_blob_generations/i
+ )
} finally {
await client.query('ROLLBACK').catch(() => undefined)
client.release()
@@ -152,7 +154,7 @@ describeRealPostgres('GFS Phase 0 real PostgreSQL readiness', () => {
await client.query(revokeSql)
await client.query('SET LOCAL ROLE gfs_controller')
await expect(probeAs(client)()).rejects.toThrow(
- /coherence check failed|permission denied|0068|0048/i
+ /coherence check failed|permission denied|0071_gfs_immutable_blob_generations|0048_gfs_permission_store/i
)
} finally {
await client.query('ROLLBACK').catch(() => undefined)
diff --git a/control-api/test/migrationReferences.test.ts b/control-api/test/migrationReferences.test.ts
new file mode 100644
index 000000000..e4291f851
--- /dev/null
+++ b/control-api/test/migrationReferences.test.ts
@@ -0,0 +1,166 @@
+import { describe, expect, it } from 'vitest'
+import { existsSync, readFileSync, readdirSync } from 'node:fs'
+import { dirname, extname, join, relative } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { CONTROL_API_MIGRATIONS } from '../src/db.js'
+
+// Operator-facing errors, comments and runbooks cite control-api migrations.
+// Migrations have been renumbered before (5e6c990f8), which silently turned
+// bare numbers like "migration 0068" into references to unrelated schema
+// changes. In every scanned root, these checks pin each version-shaped token
+// to a registered version and forbid citing a migration by bare number.
+//
+// Not covered: whether a registered version is the right one for the text
+// around it (citing 0071_… where 0072_… belongs passes), and bare numbers not
+// introduced by the word "migration(s)" (for example "the 0114 trigger").
+
+const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), '../..')
+const REGISTERED = new Set(CONTROL_API_MIGRATIONS.map(migration => migration.version))
+const SCANNED_EXTENSIONS = new Set(['.ts', '.tsx', '.mjs', '.js', '.sh', '.md', '.yaml', '.yml'])
+const SKIPPED_DIRECTORIES = new Set(['node_modules', 'dist', 'build', 'coverage', '.git'])
+// control-api/test is not scanned: it holds seed fixtures and legacy-name
+// lists that are version-shaped on purpose.
+const SCANNED_ROOTS = ['gfs-controller/src', 'control-api/src', 'deploy', 'docs', 'scripts']
+
+// Registered versions are four digits ("0071") or hex-style ("00a4").
+const NUMBER = String.raw`(?:\d{4}|00[a-f]\d)`
+const VERSION_TOKEN = new RegExp(String.raw`(?<![0-9A-Za-z_])${NUMBER}_[a-z][a-z0-9_]*`, 'g')
+// Superseded names that the runner still accepts as already applied. Only
+// tokens inside a legacyVersions array are exempt; the same name anywhere else
+// is a stale reference.
+const LEGACY_VERSIONS_ARRAY = /legacyVersions:\s*\[[^\]]*\]/g
+// "migration(s)" followed by a list of versions, across line breaks and
+// separators such as ":", "`", "(", ",", "/", "–", "and", "or".
+const MIGRATION_LIST = new RegExp(
+ String.raw`\bmigrations?\b((?:[\s:\`#(),/\-–]+|\b(?:and|or)\b|${NUMBER}(?:_[a-z][a-z0-9_]*)?(?![0-9A-Za-z_]))+)`,
+ 'gi'
+)
+const BARE_NUMBER = new RegExp(String.raw`(?<![0-9A-Za-z_])${NUMBER}(?![0-9A-Za-z_])`, 'g')
+
+interface MigrationReference {
+ location: string
+ token: string
+}
+
+function listSourceFiles(directory: string): string[] {
+ return readdirSync(directory, { withFileTypes: true }).flatMap(entry => {
+ const path = join(directory, entry.name)
+ if (entry.isDirectory()) {
+ return SKIPPED_DIRECTORIES.has(entry.name) ? [] : listSourceFiles(path)
+ }
+ return SCANNED_EXTENSIONS.has(extname(entry.name)) ? [path] : []
+ })
+}
+
+function lineOf(text: string, index: number): number {
+ return text.slice(0, index).split('\n').length
+}
+
+function legacySpans(text: string): Array<[number, number]> {
+ return [...text.matchAll(LEGACY_VERSIONS_ARRAY)].map(match => [
+ match.index,
+ match.index + match[0].length,
+ ])
+}
+
+function versionTokens(text: string): Array<{ index: number; token: string }> {
+ const spans = legacySpans(text)
+ return [...text.matchAll(VERSION_TOKEN)]
+ .filter(match => !spans.some(([start, end]) => match.index >= start && match.index < end))
+ .map(match => ({ index: match.index, token: match[0] }))
+}
+
+function bareNumbers(text: string): Array<{ index: number; token: string }> {
+ return [...text.matchAll(MIGRATION_LIST)].flatMap(list => {
+ const listStart = list.index + list[0].length - list[1].length
+ return [...list[1].matchAll(BARE_NUMBER)].map(bare => ({
+ index: listStart + bare.index,
+ token: bare[0],
+ }))
+ })
+}
+
+function scan(
+ root: string,
+ find: (text: string) => Array<{ index: number; token: string }>
+): { files: number; references: MigrationReference[] } {
+ const directory = join(REPO_ROOT, root)
+ if (!existsSync(directory)) {
+ throw new Error(`scanned root ${root} does not exist; update SCANNED_ROOTS`)
+ }
+ const files = listSourceFiles(directory)
+ const references = files.flatMap(file => {
+ const text = readFileSync(file, 'utf8')
+ return find(text).map(({ index, token }) => ({
+ location: `${relative(REPO_ROOT, file)}:${lineOf(text, index)}`,
+ token,
+ }))
+ })
+ return { files: files.length, references }
+}
+
+function describeReferences(references: MigrationReference[]): string[] {
+ return references.map(reference => `${reference.location} ${reference.token}`)
+}
+
+describe('control-api migration references', () => {
+ it('detects bare numbers in every citation shape the guard relies on', () => {
+ expect(
+ [
+ 'control-api migration 0068 not applied',
+ 'migration: 0068',
+ 'migration `0068`',
+ 'migration\n 0068',
+ 'migrations 0095_first, 0096 and 00a4',
+ 'migrations 0095_first/0096',
+ 'migrations 0109–0114',
+ 'Migration 0055 replaces',
+ ].map(sample => bareNumbers(sample).map(bare => bare.token))
+ ).toEqual([
+ ['0068'],
+ ['0068'],
+ ['0068'],
+ ['0068'],
+ ['0096', '00a4'],
+ ['0096'],
+ ['0109', '0114'],
+ ['0055'],
+ ])
+ expect(
+ bareNumbers('migration 0071_gfs_immutable_blob_generations not applied, or 0074_x')
+ ).toEqual([])
+ })
+
+ it('exempts only tokens inside a legacyVersions array', () => {
+ const text = [
+ "legacyVersions: [\n '0055_governed_trace_runtime_roles',\n],",
+ '// replaced by migration 0055_governed_trace_runtime_roles',
+ ].join('\n')
+ expect(versionTokens(text).map(found => found.token)).toEqual([
+ '0055_governed_trace_runtime_roles',
+ ])
+ expect(versionTokens(text)[0].index).toBeGreaterThan(text.indexOf('],'))
+ })
+
+ it.each(SCANNED_ROOTS)('every version token in %s is a registered migration', root => {
+ const { files, references } = scan(root, versionTokens)
+
+ // Liveness witness: each root cites real versions today, so an empty
+ // result means the scan stopped reading files, not that they are clean.
+ expect(files).toBeGreaterThan(0)
+ expect(references.length).toBeGreaterThan(0)
+ expect(
+ describeReferences(references.filter(reference => !REGISTERED.has(reference.token)))
+ ).toEqual([])
+ })
+
+ it.each(SCANNED_ROOTS)('%s never cites a control-api migration by bare number', root => {
+ const { references } = scan(root, bareNumbers)
+ const { references: named } = scan(root, versionTokens)
+
+ // Liveness witness: each root cites named migrations, so the files this
+ // check reads are the ones that carry migration citations.
+ expect(named.length).toBeGreaterThan(0)
+ expect(describeReferences(references)).toEqual([])
+ })
+})
diff --git a/deploy/scripts/provision-control-api-runtime-roles.sh b/deploy/scripts/provision-control-api-runtime-roles.sh
index 1c4c16ffc..0fbf6f926 100755
--- a/deploy/scripts/provision-control-api-runtime-roles.sh
+++ b/deploy/scripts/provision-control-api-runtime-roles.sh
@@ -1,7 +1,9 @@
#!/usr/bin/env bash
# Reconcile least-privilege Postgres logins used by control-api tracing paths.
#
-# Migration 0053 owns role creation and grants. This script owns only the
+# Control-api migrations own role creation and grants: the roles are created by
+# migration 0062_governed_trace_runtime_roles, and later migrations adjust
+# their grants. This script owns only the
# runtime credential lifecycle: it preserves an existing valid credential or
# generates one on first provisioning, applies it to the fixed database role,
# and patches the matching Kubernetes Secret. No credential is accepted from
diff --git a/deploy/scripts/run-control-api-db-migration.sh b/deploy/scripts/run-control-api-db-migration.sh
index 642b1e78f..2929e85d6 100755
--- a/deploy/scripts/run-control-api-db-migration.sh
+++ b/deploy/scripts/run-control-api-db-migration.sh
@@ -351,7 +351,7 @@ build_job_manifest() {
},
"spec" => {
# Retry a transient migration abort instead of failing the whole deploy.
- # Migration 0090 takes an EXCLUSIVE lock on agent_run_events with a bounded
+ # Migration 0090_plugin_workload_sdk_runtime_contract_reconciliation takes an EXCLUSIVE lock on agent_run_events with a bounded
# lock_timeout; if a concurrent writer (e.g. the trace-retention prune
# DELETE) holds its lock past that timeout the Job exits 1. The migration
# body is idempotent, so re-running is safe; restartPolicy Never makes each
diff --git a/docs/architecture/codex-subscription-transport-contract.md b/docs/architecture/codex-subscription-transport-contract.md
index c22645472..b3044f226 100644
--- a/docs/architecture/codex-subscription-transport-contract.md
+++ b/docs/architecture/codex-subscription-transport-contract.md
@@ -165,7 +165,7 @@ behavior changes:
with `state_cancelled`.
- When two first grants race on the same key, the losing active-key insert
maps to a stale-revision conflict instead of a 500.
-- **Attempt integrity (migration 0114).** A constraint trigger requires a
+- **Attempt integrity (migration 0114_llm_provider_attempts_connection_integrity).** A constraint trigger requires a
non-null `llm_provider_attempts.connection_id` of a Codex attempt to exist in
`codex_subscription_connections`. This replaces the FK that 0112 dropped.
Historic rows are not revalidated, and the migration is forward-only.
diff --git a/docs/architecture/grok-subscription-transport-contract.md b/docs/architecture/grok-subscription-transport-contract.md
index c512e963d..56879cacb 100644
--- a/docs/architecture/grok-subscription-transport-contract.md
+++ b/docs/architecture/grok-subscription-transport-contract.md
@@ -77,7 +77,7 @@ an error.
## Connection lifecycle
-- **Grok connection keys are terminal after revoke (migration 0113).**
+- **Grok connection keys are terminal after revoke (migration 0113_grok_subscription_terminal_connection_key).**
- A revoked key cannot be reconnected. Create a new connection key instead.
- A device flow that started before the revoke cannot revive the key: it
fails with `revoked grant cannot be reused`.
@@ -87,7 +87,7 @@ an error.
`connection_key`.
- Archived `~revoked~` rows fall outside the key grammar. Admin list and GET
hide them. The key's own terminal tombstone stays listed as `revoked`.
-- **Attempt integrity (migration 0114).** A constraint trigger on
+- **Attempt integrity (migration 0114_llm_provider_attempts_connection_integrity).** A constraint trigger on
`llm_provider_attempts` INSERT, and on UPDATE of `provider` or
`connection_id`, requires a non-null `connection_id` to exist in the
provider's own connection table:
@@ -245,7 +245,8 @@ Grok annotations. A new control-api also republishes on boot.
(`GROK_LLM_PROXY_CONTROL_API_TOKEN`) with
`deploy/scripts/apply-inter-service-tokens.sh`. Both must be in place before
the merge to `dev` triggers the automatic deploy.
-2. **control-api** with migrations 0109–0114. Wait until every control-api pod
+2. **control-api** with migrations 0109_grok_subscription_connections through
+ 0114_llm_provider_attempts_connection_integrity. Wait until every control-api pod
runs the new image. An old control-api rejects the whole workflow-control
token issue when the request includes the unknown `llm:grok:execute` scope
(`invalid_workflow_control_scopes`). HCC or WRC deployed first could
@@ -276,7 +277,8 @@ Grok annotations. A new control-api also republishes on boot.
shape. In the UI, re-pick the connection.
- A save that omits the annotations keeps the stored disagreeing pair, so
it does not repair the grant.
-4. Migrations 0113 (terminal Grok keys) and 0114 (attempt connection
+4. Migrations 0113_grok_subscription_terminal_connection_key (terminal Grok keys)
+ and 0114_llm_provider_attempts_connection_integrity (attempt connection
integrity) are forward-only. An older control-api still runs against them:
- The 0114 trigger only rejects connection ids that do not exist.
- Under 0113, a revoked Grok key can no longer be reconnected by the old
diff --git a/docs/deploy/gfs-permission-store.md b/docs/deploy/gfs-permission-store.md
index 8706cba33..983288719 100644
--- a/docs/deploy/gfs-permission-store.md
+++ b/docs/deploy/gfs-permission-store.md
@@ -166,8 +166,9 @@ store. The store check has two layers (`gfs-controller/src/authz/storeProbe.ts`)
the rotation and the kubelet probe cadence keeps one alive forever. Only
success is cached; while failing, every probe retries.
-Net effect: a stale DSN, a rotated role password, or missing migration-0048
-grants flips the pod NotReady within roughly a minute, visibly in
+Net effect: a stale DSN, a rotated role password, or missing role grants
+(migration 0048_gfs_permission_store for the writer,
+0072_gfs_reader_database_role for the reader) flips the pod NotReady within roughly a minute, visibly in
`kubectl get pods -n gfs`, instead of serving chronic 503s to users.
`/readyz` reasons you may see:
@@ -176,7 +177,7 @@ grants flips the pod NotReady within roughly a minute, visibly in
| ------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- |
| `storage volume not mounted …` | Drive PVC missing/unmounted (not a credential issue) |
| `permission store unreachable: password authentication failed …` | DSN/password drift — re-run provisioning |
-| `permission store unreachable: … coherence check failed … gfs_resources … 0048` | Role exists but the SELECT grants/migration are missing — run control-api migrations, then provisioning |
+| `permission store unreachable: … coherence check failed … gfs_resources … 0048_gfs_permission_store` (writer) or `… 0072_gfs_reader_database_role` (reader) | Role exists but the SELECT grants/migration are missing — run control-api migrations, then provisioning |
| `permission store unreachable: … coherence check failed … gfs_audit …` | Audit INSERT grant missing — every request would 503 (audit-write failures propagate); run migrations + provisioning |
| `permission store unreachable: … timed out after Nms` | Black-hole partition (unreachable, not refusing) — the probe hard-bounds connect/query (default 5s) instead of dangling |
| `permission store unreachable: connect ECONNREFUSED …` | PostgreSQL down/unreachable |
@@ -222,7 +223,8 @@ CONTEXT=<kube-context> deploy/scripts/provision-gfs-db.sh rotate-reader
MINIKUBE_PROFILE=<profile> make minikube-verify-gfs
```
-The script verifies the role exists (writer migration 0048; reader migration 0071),
+The script verifies the role exists (writer migration 0048_gfs_permission_store; reader migration
+0072_gfs_reader_database_role),
rotates the password, patches the Secret, stamps the rotation annotation, and
rolls and waits only for the selected gfsc Deployment.
diff --git a/gfs-controller/src/authz/storeProbe.test.ts b/gfs-controller/src/authz/storeProbe.test.ts
index 9f464f0e0..473444929 100644
--- a/gfs-controller/src/authz/storeProbe.test.ts
+++ b/gfs-controller/src/authz/storeProbe.test.ts
@@ -176,33 +176,58 @@ describe("createPermissionStoreProbe", () => {
await expect(probe()).rejects.toThrow(/password authentication failed/);
});
- it("rejects with a migration hint when the role lacks SELECT on gfs_resources", async () => {
- const { factory } = fakeClientFactory({ canRead: false });
- const probe = createPermissionStoreProbe({
- pool: fakePool(),
- connectionString: DSN,
- intervalMs: INTERVAL,
- storageRole: "reader",
- clientFactory: factory,
- now: () => 0,
- });
- await expect(probe()).rejects.toThrow(/gfs_resources.*0048/s);
- });
+ // Each role's base grants come from a different migration, so the hint must
+ // name the one that actually granted the missing privilege to that role.
+ it.each([
+ ["writer", "0048_gfs_permission_store"],
+ ["reader", "0072_gfs_reader_database_role"],
+ ] as const)(
+ "rejects a %s lacking SELECT on gfs_resources with its own grant migration hint",
+ async (storageRole, grantMigration) => {
+ const { factory } = fakeClientFactory({ canRead: false });
+ const probe = createPermissionStoreProbe({
+ pool: fakePool(),
+ connectionString: DSN,
+ intervalMs: INTERVAL,
+ storageRole,
+ clientFactory: factory,
+ now: () => 0,
+ });
+ await expect(probe()).rejects.toThrow(
+ new RegExp(
+ `lacks SELECT on gfs_resources \\(control-api migration ${grantMigration} not applied, ` +
+ "or 0074_gfs_runtime_role_exact_contract grants drifted\\?\\)"
+ )
+ );
+ }
+ );
- it("rejects when the role lacks INSERT on gfs_audit (chronic per-request 503 mode)", async () => {
- const { factory } = fakeClientFactory({ canAudit: false });
- const probe = createPermissionStoreProbe({
- pool: fakePool(),
- connectionString: DSN,
- intervalMs: INTERVAL,
- storageRole: "reader",
- clientFactory: factory,
- now: () => 0,
- });
- await expect(probe()).rejects.toThrow(/gfs_audit.*INSERT|INSERT on gfs_audit/s);
- });
+ it.each([
+ ["writer", "0048_gfs_permission_store"],
+ ["reader", "0072_gfs_reader_database_role"],
+ ] as const)(
+ "rejects a %s lacking INSERT on gfs_audit (chronic per-request 503 mode) with its own grant migration hint",
+ async (storageRole, grantMigration) => {
+ const { factory } = fakeClientFactory({ canAudit: false });
+ const probe = createPermissionStoreProbe({
+ pool: fakePool(),
+ connectionString: DSN,
+ intervalMs: INTERVAL,
+ storageRole,
+ clientFactory: factory,
+ now: () => 0,
+ });
+ await expect(probe()).rejects.toThrow(
+ new RegExp(
+ `lacks INSERT on gfs_audit \\(audit-write failures would 503 every request; ` +
+ `control-api migration ${grantMigration} not applied, ` +
+ "or 0074_gfs_runtime_role_exact_contract grants drifted\\?\\)"
+ )
+ );
+ }
+ );
- it("rejects a pre-0068 reader schema before serving requests", async () => {
+ it("rejects a pre-0071_gfs_immutable_blob_generations reader schema before serving requests", async () => {
const { factory } = fakeClientFactory({ immutableSchemaReady: false });
const probe = createPermissionStoreProbe({
pool: fakePool(),
@@ -212,9 +237,30 @@ describe("createPermissionStoreProbe", () => {
clientFactory: factory,
now: () => 0,
});
- await expect(probe()).rejects.toThrow(/migration 0068/);
+ await expect(probe()).rejects.toThrow(
+ /migration 0071_gfs_immutable_blob_generations not applied, or 0074_gfs_runtime_role_exact_contract grants drifted\?/
+ );
});
+ it.each([
+ ["schema", { auditDecisionEvidenceSchemaReady: false }],
+ ["constraint set", { auditDecisionEvidenceConstraintsReady: false }],
+ ] satisfies Array<[string, FakeClientBehavior]>)(
+ "rejects readiness before 0073_gfs_audit_decision_evidence when the audit decision-evidence %s is missing",
+ async (_part, behavior) => {
+ const { factory } = fakeClientFactory(behavior);
+ const probe = createPermissionStoreProbe({
+ pool: fakePool(),
+ connectionString: DSN,
+ intervalMs: INTERVAL,
+ storageRole: "reader",
+ clientFactory: factory,
+ now: () => 0,
+ });
+ await expect(probe()).rejects.toThrow(/migration 0073_gfs_audit_decision_evidence /);
+ }
+ );
+
it("keeps reader readiness independent of the writer-only manifest schema", async () => {
const probeClient = fakeClientFactory({});
const probe = createPermissionStoreProbe({
@@ -233,7 +279,7 @@ describe("createPermissionStoreProbe", () => {
expect(probeClient.lastSql()).toContain("gfs_audit_record_type_fields_valid");
});
- it("rejects readiness when migration 0096 default or backfill is incomplete", async () => {
+ it("rejects readiness when migration 0096_control_admin_session_version_default default or backfill is incomplete", async () => {
const { factory } = fakeClientFactory({ authorityAdminSessionEpochReady: false });
const probe = createPermissionStoreProbe({
pool: fakePool(),
@@ -243,7 +289,7 @@ describe("createPermissionStoreProbe", () => {
clientFactory: factory,
now: () => 0,
});
- await expect(probe()).rejects.toThrow(/0095\/0096/);
+ await expect(probe()).rejects.toThrow(/migrations 0095_gfs_lifecycle_authority_projection\/0096_control_admin_session_version_default /);
});
it.each([
@@ -265,7 +311,7 @@ describe("createPermissionStoreProbe", () => {
clientFactory: factory,
now: () => 0,
});
- await expect(probe()).rejects.toThrow(/reader has forbidden GFS mutation privileges.*0069/s);
+ await expect(probe()).rejects.toThrow(/reader has forbidden GFS mutation privileges.*migration 0074_gfs_runtime_role_exact_contract /s);
}
);
@@ -286,7 +332,7 @@ describe("createPermissionStoreProbe", () => {
});
it.each(["actor_on_behalf_of", "desktop_user_id", "authority_source"] as const)(
- "rejects readiness before 0092 when the required %s audit column is absent or drifted",
+ "rejects readiness before 0092_gfs_audit_actor_correlation when the required %s audit column is absent or drifted",
async (_column) => {
const { factory } = fakeClientFactory({ auditActorCorrelationColumnsReady: false });
const probe = createPermissionStoreProbe({
@@ -297,7 +343,7 @@ describe("createPermissionStoreProbe", () => {
clientFactory: factory,
now: () => 0,
});
- await expect(probe()).rejects.toThrow(/migration 0092/);
+ await expect(probe()).rejects.toThrow(/migration 0092_gfs_audit_actor_correlation /);
}
);
@@ -309,7 +355,7 @@ describe("createPermissionStoreProbe", () => {
["admin privileges", { authorityAdminPrivilegesReady: false }],
["link privileges", { authorityLinksPrivilegesReady: false }],
] satisfies Array<[string, FakeClientBehavior]>)(
- "rejects readiness when the 0095 authority projection is missing (%s)",
+ "rejects readiness when the 0095_gfs_lifecycle_authority_projection authority projection is missing (%s)",
async (_projection, behavior) => {
const { factory } = fakeClientFactory(behavior);
const probe = createPermissionStoreProbe({
@@ -320,12 +366,14 @@ describe("createPermissionStoreProbe", () => {
clientFactory: factory,
now: () => 0,
});
- await expect(probe()).rejects.toThrow(/migration 0095/);
+ await expect(probe()).rejects.toThrow(
+ /migrations 0095_gfs_lifecycle_authority_projection\/0096_control_admin_session_version_default /
+ );
}
);
it.each(["reader", "writer"] as const)(
- "rejects a %s when migration 0092 audit actor-correlation constraints are missing or unvalidated",
+ "rejects a %s when migration 0092_gfs_audit_actor_correlation audit actor-correlation constraints are missing or unvalidated",
async (storageRole) => {
const { factory } = fakeClientFactory({ auditActorCorrelationConstraintReady: false });
const probe = createPermissionStoreProbe({
@@ -336,7 +384,7 @@ describe("createPermissionStoreProbe", () => {
clientFactory: factory,
now: () => 0,
});
- await expect(probe()).rejects.toThrow(/migration 0092/);
+ await expect(probe()).rejects.toThrow(/migration 0092_gfs_audit_actor_correlation /);
}
);
@@ -358,7 +406,9 @@ describe("createPermissionStoreProbe", () => {
clientFactory: factory,
now: () => 0,
});
- await expect(probe()).rejects.toThrow(/writer lacks.*0068/s);
+ await expect(probe()).rejects.toThrow(
+ /writer lacks.*migration 0071_gfs_immutable_blob_generations not applied, or 0074_gfs_runtime_role_exact_contract grants drifted\?/s
+ );
}
);
diff --git a/gfs-controller/src/authz/storeProbe.ts b/gfs-controller/src/authz/storeProbe.ts
index d003229e5..3517b16f0 100644
--- a/gfs-controller/src/authz/storeProbe.ts
+++ b/gfs-controller/src/authz/storeProbe.ts
@@ -15,7 +15,10 @@ import { Client } from "pg";
* 2. Amortized fresh-connection probe (at most once per `intervalMs`) — opens
* a brand-new client (NEVER the pool) so a rotated credential fails to
* authenticate, and verifies real coherence with `has_table_privilege`, so
- * "password correct but migration 0048 grants missing" also fails loud.
+ * "password correct but role grants missing" also fails loud. The writer's
+ * grants come from migration 0048_gfs_permission_store, the reader's from
+ * 0072_gfs_reader_database_role, and 0074_gfs_runtime_role_exact_contract
+ * rebuilds both envelopes.
*
* FAIL CLOSED: only SUCCESS is cached. While failing, every readiness call
* retries the fresh connection (~1 new connection per probe period during an
@@ -191,6 +194,13 @@ const WRITER_COHERENCE_SQL =
export function createPermissionStoreProbe(opts: StoreProbeOptions): () => Promise<void> {
const now = opts.now ?? Date.now;
const connectTimeoutMs = opts.connectTimeoutMs ?? 5_000;
+ // Each role gets its base grants from its own migration; 0074 later revokes
+ // every table and column privilege and rebuilds both envelopes exactly.
+ const roleGrantsMigration =
+ opts.storageRole === "writer" ? "0048_gfs_permission_store" : "0072_gfs_reader_database_role";
+ const roleGrantsHint =
+ `control-api migration ${roleGrantsMigration} not applied, ` +
+ "or 0074_gfs_runtime_role_exact_contract grants drifted?";
// @types/pg resolves Client.connect() to Promise<Client>; the probe contract
// wants Promise<void> — awaited and discarded (same adaptation as the LISTEN
// client in index.ts). connectionTimeoutMillis is belt-and-suspenders under
@@ -270,13 +280,13 @@ export function createPermissionStoreProbe(opts: StoreProbeOptions): () => Promi
if (row?.can_read !== true) {
throw new Error(
"permission store coherence check failed: role lacks SELECT on gfs_resources " +
- "(control-api migration 0048 not applied?)"
+ `(${roleGrantsHint})`
);
}
if (row?.can_audit !== true) {
throw new Error(
"permission store coherence check failed: role lacks INSERT on gfs_audit " +
- "(audit-write failures would 503 every request; control-api migration 0048 not applied?)"
+ `(audit-write failures would 503 every request; ${roleGrantsHint})`
);
}
if (
@@ -286,7 +296,8 @@ export function createPermissionStoreProbe(opts: StoreProbeOptions): () => Promi
) {
throw new Error(
"permission store coherence check failed: immutable GFS blob schema/reader privileges " +
- "are missing (control-api migration 0068 not applied?)"
+ "are missing (control-api migration 0071_gfs_immutable_blob_generations not applied, " +
+ "or 0074_gfs_runtime_role_exact_contract grants drifted?)"
);
}
if (
@@ -295,7 +306,7 @@ export function createPermissionStoreProbe(opts: StoreProbeOptions): () => Promi
) {
throw new Error(
"permission store coherence check failed: GFS audit decision-evidence schema is missing " +
- "(control-api migration 0073 not applied?)"
+ "(control-api migration 0073_gfs_audit_decision_evidence not applied?)"
);
}
if (
@@ -304,7 +315,7 @@ export function createPermissionStoreProbe(opts: StoreProbeOptions): () => Promi
) {
throw new Error(
"permission store coherence check failed: GFS audit actor-correlation schema is missing " +
- "(control-api migration 0092 not applied?)"
+ "(control-api migration 0092_gfs_audit_actor_correlation not applied?)"
);
}
if (
@@ -318,7 +329,7 @@ export function createPermissionStoreProbe(opts: StoreProbeOptions): () => Promi
) {
throw new Error(
"permission store coherence check failed: GFS lifecycle authority projection is missing " +
- "(control-api migration 0095/0096 not applied?)"
+ "(control-api migrations 0095_gfs_lifecycle_authority_projection/0096_control_admin_session_version_default not applied?)"
);
}
if (
@@ -332,7 +343,7 @@ export function createPermissionStoreProbe(opts: StoreProbeOptions): () => Promi
) {
throw new Error(
"permission store coherence check failed: reader has forbidden GFS mutation privileges " +
- "(control-api migration 0069 grants drifted?)"
+ "(control-api migration 0074_gfs_runtime_role_exact_contract grants drifted?)"
);
}
if (
@@ -347,7 +358,8 @@ export function createPermissionStoreProbe(opts: StoreProbeOptions): () => Promi
) {
throw new Error(
"permission store coherence check failed: writer lacks GFS manifest/resource mutation " +
- "privileges (control-api migration 0068 not applied?)"
+ "privileges (control-api migration 0071_gfs_immutable_blob_generations not applied, " +
+ "or 0074_gfs_runtime_role_exact_contract grants drifted?)"
);
}
lastFreshOkAt = now(); // cache SUCCESS only — failures always retry
diff --git a/gfs-controller/src/index.ts b/gfs-controller/src/index.ts
index 1a4cf9f38..b8e475006 100644
--- a/gfs-controller/src/index.ts
+++ b/gfs-controller/src/index.ts
@@ -138,7 +138,9 @@ async function main(): Promise<void> {
// detect a rotated password — its idle clients authenticated before the
// rotation and the readiness cadence keeps one alive forever. The probe
// keeps the fast pool ping AND dials a brand-new client (amortized) so a
- // stale DSN or missing migration-0048 grants flips the pod NotReady.
+ // stale DSN or missing role grants (migration 0048_gfs_permission_store for
+ // the writer, 0072_gfs_reader_database_role for the reader) flips the pod
+ // NotReady.
pingPermissionStore: createPermissionStoreProbe({
pool,
connectionString: config.pgConnectionString,
diff --git a/scripts/e2e/e2e-figure-d-multibot.sh b/scripts/e2e/e2e-figure-d-multibot.sh
index fe6d9db6c..947de205b 100755
--- a/scripts/e2e/e2e-figure-d-multibot.sh
+++ b/scripts/e2e/e2e-figure-d-multibot.sh
@@ -6,7 +6,7 @@
# Companion gate for PR1 (.ralph/plans/figure-d-multibot-pr1.md). Proves, on a
# live cluster, what unit tests cannot:
#
-# B4 (always) — migration 0039 applied: communication_channel_ref column
+# B4 (always) — migration 0039_wama_communication_channel_ref applied: communication_channel_ref column
# + idx_wama_channel_ref index + status CHECK admits
# skipped_no_bot + ONLY telegram NULL-ref rows force-disabled
# (Slack / other models untouched).
@@ -73,8 +73,8 @@ if command -v git >/dev/null 2>&1; then
log "Evidence: HEAD=$(git rev-parse --short HEAD 2>/dev/null || echo '?') context=$KCTX"
fi
-# ─── B4: migration 0039 applied ────────────────────────────────────────
-log "B4 — migration 0039 + schema"
+# ─── B4: migration 0039_wama_communication_channel_ref applied ────────────────────────────────────────
+log "B4 — migration 0039_wama_communication_channel_ref + schema"
ver="$(psql_q "SELECT version FROM schema_migrations WHERE version = '0039_wama_communication_channel_ref';")"
[ "$ver" = "0039_wama_communication_channel_ref" ] \
diff --git a/scripts/minikube/full-setup.sh b/scripts/minikube/full-setup.sh
index 97860ebdb..514df9120 100755
--- a/scripts/minikube/full-setup.sh
+++ b/scripts/minikube/full-setup.sh
@@ -2181,7 +2181,7 @@ done
# ----------------------------------------------------------------------
# gfs serving provisioning (runs once pods are up, so control-api has applied
-# migration 0048 that creates the gfs_controller role). gfsc fails closed
+# migration 0048_gfs_permission_store that creates the gfs_controller role). gfsc fails closed
# without BOTH: (a) its JWT public key in gfs-config, and (b) the gfs_controller
# DB LOGIN + DSN in gfs-controller-db. The inline JWT sync (6g) only covers
# mcp-host-config, so these gfs-specific steps are explicit here. Idempotent.
diff --git a/scripts/minikube/pre-gate-sync.sh b/scripts/minikube/pre-gate-sync.sh
index 40908fac9..15e860867 100755
--- a/scripts/minikube/pre-gate-sync.sh
+++ b/scripts/minikube/pre-gate-sync.sh
@@ -772,7 +772,7 @@ if [[ "${cluster_changed}" == "true" ]]; then
restore_control_api
# The base manifest no longer declares connection-string (provisioning-owned
# key), so deploy-all cannot clobber it. Provisioning must still run AFTER
- # control-api migrations (0048 creates the least-privilege gfs_controller
+ # control-api migrations (0048_gfs_permission_store creates the least-privilege gfs_controller
# role) for fresh profiles and to converge any stale credential before the
# rest of the gate observes service readiness.
provision_gfs_serving