Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): make Grok grant revocation terminal and catalog sync atomic

Jose Gabriel Ramirez Encinas32fd94041ac3
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 730825a43..86e3b66ff 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.573",
+ "version": "0.1.574",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.573",
+ "version": "0.1.574",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 861b15295..ee5e96292 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.573",
+ "version": "0.1.574",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/db.ts b/control-api/src/db.ts
index f9e391fd9..3fdfaf294 100644
--- a/control-api/src/db.ts
+++ b/control-api/src/db.ts
@@ -23,6 +23,7 @@ import {
applyGrokCatalogModelsSchema,
applyGrokSubscriptionConnectionSchema,
applyGrokSubscriptionOAuthStateSchema,
+ applyGrokSubscriptionTerminalConnectionKeySchema,
applyLlmProviderAttemptsGrokBrokerSchema,
} from './services/grokSubscriptionSchema.js'
import {
@@ -6036,6 +6037,10 @@ export const CONTROL_API_MIGRATIONS: DbMigration[] = [
version: '0112_llm_provider_attempts_grok_broker',
apply: applyLlmProviderAttemptsGrokBrokerSchema,
},
+ {
+ version: '0113_grok_subscription_terminal_connection_key',
+ apply: applyGrokSubscriptionTerminalConnectionKeySchema,
+ },
]
async function consolidateWorkflowAllowedUsersToTriggers(db: DbClient): Promise<void> {
diff --git a/control-api/src/services/grokProviderAttemptRedemption.ts b/control-api/src/services/grokProviderAttemptRedemption.ts
index 4fc6d90d0..7bc27b11e 100644
--- a/control-api/src/services/grokProviderAttemptRedemption.ts
+++ b/control-api/src/services/grokProviderAttemptRedemption.ts
@@ -148,9 +148,17 @@ export async function redeemGrokProviderAttempt(
}
if (deps.ensureFreshAccessToken) {
+ const assigned = await deps.getConnectionById(deps.db, claims.connectionId)
+ if (!assigned) {
+ // The attempt's connection row is gone; never hand an empty key to the
+ // refresh path (it would surface as an unmapped 500).
+ throw new GrokProviderAttemptRedeemError(
+ 'connection_unavailable',
+ 'Grok subscription connection is not usable'
+ )
+ }
try {
- const assigned = await deps.getConnectionById(deps.db, claims.connectionId)
- await deps.ensureFreshAccessToken(assigned?.connectionKey)
+ await deps.ensureFreshAccessToken(assigned.connectionKey)
} catch (err) {
if (err instanceof GrokSubscriptionOAuthError) {
throw new GrokProviderAttemptRedeemError(
diff --git a/control-api/src/services/grokSubscriptionCatalog.ts b/control-api/src/services/grokSubscriptionCatalog.ts
index a90847313..4d4adb23d 100644
--- a/control-api/src/services/grokSubscriptionCatalog.ts
+++ b/control-api/src/services/grokSubscriptionCatalog.ts
@@ -1,7 +1,7 @@
import jwt from 'jsonwebtoken'
import { CATALOG_ORIGIN } from '@clerum/grok-provider-attempt-contract'
import { config } from '../config.js'
-import type { DbClient } from '../db.js'
+import { type DbClient, withTransaction } from '../db.js'
import { rootLogger } from '../observability/logger.js'
import {
type GrokSubscriptionCatalogStatus,
@@ -20,6 +20,16 @@ export const GROK_CATALOG_ORIGIN = CATALOG_ORIGIN
export type GrokCatalogOutcome = 'ready' | 'auth-rejected' | 'unavailable'
+/**
+ * Runs `work` inside one database transaction and commits only when it
+ * resolves. Production uses the Control API pool (`withTransaction` in db.ts),
+ * which is the same pool behind the admin routes' DbClient; tests inject a
+ * runner bound to their own pool.
+ */
+export type GrokTransactionRunner = <T>(work: (tx: DbClient) => Promise<T>) => Promise<T>
+
+export const defaultGrokTransactionRunner: GrokTransactionRunner = work => withTransaction(work)
+
export type GrokDiscoveredModel = {
model: string
displayName?: string
@@ -276,11 +286,21 @@ export async function rebuildLiveGrokUnionAllowlist(db: DbClient): Promise<void>
)
}
+/**
+ * Reconcile one connection's catalog. The provider call runs outside any
+ * transaction; the fenced readiness/revision write, the model row
+ * insert/refresh/stale and the live union rebuild commit together, so a
+ * failure in any phase leaves readiness, revision and rows untouched. The
+ * fenced UPDATE row-locks the connection first, so concurrent syncs on one
+ * revision serialize and exactly one commits an outcome. Callers publish the
+ * runtime allowlist only after this resolves (i.e. after commit).
+ */
export async function syncGrokSubscriptionCatalog(
db: DbClient,
transport: GrokCatalogTransport,
accessToken: string,
- expected: { credentialRevision?: number; catalogRevision?: number; connectionKey: string }
+ expected: { credentialRevision?: number; catalogRevision?: number; connectionKey: string },
+ options: { withTransaction?: GrokTransactionRunner } = {}
): Promise<{
outcome: GrokCatalogOutcome
connection: GrokSubscriptionSafeConnection | null
@@ -295,24 +315,27 @@ export async function syncGrokSubscriptionCatalog(
const expectedCredentialRevision = expected.credentialRevision ?? connection.credentialRevision
const expectedCatalogRevision = expected.catalogRevision ?? connection.catalogRevision
const result = await transport.listModels({ accessToken })
- const existing = await loadGrokRows(db, connection.id)
- const plan = planGrokCatalogReconcile(existing, result)
- const recorded = await recordGrokCatalogOutcome(db, {
- catalogStatus: plan.catalogStatus as GrokSubscriptionCatalogStatus,
- connectionStatus: plan.connectionStatus,
- expectedCredentialRevision,
- expectedCatalogRevision,
- connectionKey: expected.connectionKey,
+ const outcomePlan = planGrokCatalogReconcile([], result)
+ const runInTransaction = options.withTransaction ?? defaultGrokTransactionRunner
+ const committed = await runInTransaction(async tx => {
+ const recorded = await recordGrokCatalogOutcome(tx, {
+ catalogStatus: outcomePlan.catalogStatus as GrokSubscriptionCatalogStatus,
+ connectionStatus: outcomePlan.connectionStatus,
+ expectedCredentialRevision,
+ expectedCatalogRevision,
+ connectionKey: expected.connectionKey,
+ })
+ if (!recorded || !outcomePlan.mutateRows) {
+ return { recorded, added: 0, refreshed: 0, staled: 0 }
+ }
+ const plan = planGrokCatalogReconcile(await loadGrokRows(tx, connection.id), result)
+ const added = await insertDiscovered(tx, connection.id, plan.inserts)
+ const refreshed = await refreshDiscovered(tx, connection.id, plan.refresh)
+ const staled = await staleMissing(tx, connection.id, plan.stale)
+ await rebuildLiveGrokUnionAllowlist(tx)
+ return { recorded, added, refreshed, staled }
})
- let added = 0
- let refreshed = 0
- let staled = 0
- if (recorded && plan.mutateRows) {
- added = await insertDiscovered(db, connection.id, plan.inserts)
- refreshed = await refreshDiscovered(db, connection.id, plan.refresh)
- staled = await staleMissing(db, connection.id, plan.stale)
- await rebuildLiveGrokUnionAllowlist(db)
- }
+ const { recorded, added, refreshed, staled } = committed
if (!recorded) {
log.warn(
{ event: 'grok_catalog_stale_writer' },
@@ -322,7 +345,7 @@ export async function syncGrokSubscriptionCatalog(
log.info(
{
event: 'grok_catalog_reconciled',
- outcome: plan.catalogStatus,
+ outcome: outcomePlan.catalogStatus,
added,
refreshed,
staled,
@@ -331,7 +354,7 @@ export async function syncGrokSubscriptionCatalog(
)
}
return {
- outcome: plan.catalogStatus,
+ outcome: outcomePlan.catalogStatus,
connection: recorded,
added,
refreshed,
diff --git a/control-api/src/services/grokSubscriptionConnection.ts b/control-api/src/services/grokSubscriptionConnection.ts
index d84a2d4b6..727868ec2 100644
--- a/control-api/src/services/grokSubscriptionConnection.ts
+++ b/control-api/src/services/grokSubscriptionConnection.ts
@@ -53,6 +53,27 @@ export class GrokSubscriptionFingerprintConflictError extends Error {
}
}
+/**
+ * A write lost the connection_key uniqueness race: another writer created the
+ * key first, or the key is a revoked tombstone (revocation is terminal per key
+ * since migration 0113). Callers re-read with
+ * `getGrokSubscriptionConnectionIncludingRevoked` to tell the two apart.
+ */
+export class GrokSubscriptionConnectionKeyConflictError extends Error {
+ readonly code = 'connection_key_conflict'
+
+ constructor() {
+ super('Grok subscription connection key is already taken')
+ this.name = 'GrokSubscriptionConnectionKeyConflictError'
+ }
+}
+
+const CONNECTION_KEY_UNIQUE_CONSTRAINTS: ReadonlySet<string> = new Set([
+ 'grok_subscription_connections_key_unique',
+ // Pre-0113 live-only index name.
+ 'grok_subscription_connections_active_key',
+])
+
export class GrokSubscriptionStaleRevisionError extends Error {
readonly code = 'grok_subscription_stale_revision'
@@ -168,6 +189,28 @@ export async function getSafeGrokSubscriptionConnection(
return row ? toSafeConnection(row) : null
}
+/**
+ * Lifecycle lookup that also returns revoked tombstones. Use it wherever a
+ * decision must respect revocation (OAuth start/completion); the live-only
+ * getter above reports a revoked key as simply absent.
+ */
+export async function getGrokSubscriptionConnectionIncludingRevoked(
+ db: DbClient,
+ connectionKey: string
+): Promise<GrokSubscriptionSafeConnection | null> {
+ const key = assertGrokConnectionKey(connectionKey)
+ const result = await db.query(
+ `SELECT ${SAFE_CONNECTION_COLUMNS}
+ FROM grok_subscription_connections
+ WHERE connection_key = $1
+ ORDER BY revoked_at DESC NULLS FIRST, created_at DESC
+ LIMIT 1`,
+ [key]
+ )
+ const row = result.rows[0] as SafeConnectionRow | undefined
+ return row ? toSafeConnection(row) : null
+}
+
export async function getSafeGrokSubscriptionConnectionById(
db: DbClient,
connectionId: string
@@ -267,7 +310,7 @@ export async function insertInitialGrokSubscriptionConnection(
)
return toSafeConnection(result.rows[0] as SafeConnectionRow)
} catch (err) {
- throw remapFingerprintConflict(err)
+ throw remapUniqueConflict(err)
}
}
@@ -314,7 +357,7 @@ export async function rotateGrokSubscriptionCredentials(
if (!row) throw new GrokSubscriptionStaleRevisionError()
return toSafeConnection(row)
} catch (err) {
- throw remapFingerprintConflict(err)
+ throw remapUniqueConflict(err)
}
}
@@ -378,37 +421,57 @@ export async function markGrokRefreshSubjectMismatch(
return row ? toSafeConnection(row) : null
}
+/**
+ * Revoke the live grant for a key and invalidate every pending OAuth device
+ * state for that key in ONE statement (data-modifying CTEs share a snapshot and
+ * commit atomically), so no earlier authorization work can complete against a
+ * revoked key. Pending states are cancelled even when the key has no live row.
+ * The terminal tombstone plus the full connection_key unique index (0113) is
+ * the backstop for a completion that consumed its state before the revoke.
+ */
export async function revokeGrokSubscriptionConnection(
db: DbClient,
connectionKey: string
): Promise<GrokSubscriptionSafeConnection | null> {
const key = assertGrokConnectionKey(connectionKey)
const result = await db.query(
- `UPDATE grok_subscription_connections
- SET status = 'revoked',
- refresh_token_encrypted = NULL,
- access_token_encrypted = NULL,
- access_token_expires_at = NULL,
- catalog_status = 'never_synced',
- credential_revision = credential_revision + 1,
- refresh_lock_token = NULL,
- refresh_lock_expires_at = NULL,
- revoked_at = now(),
- updated_at = now()
- WHERE connection_key = $1
- AND revoked_at IS NULL
- RETURNING ${SAFE_CONNECTION_COLUMNS}`,
+ `WITH cancelled_states AS (
+ UPDATE grok_subscription_oauth_states
+ SET status = 'cancelled',
+ cancelled_at = now()
+ WHERE connection_key = $1
+ AND status = 'pending'
+ RETURNING 1
+ ),
+ revoked AS (
+ UPDATE grok_subscription_connections
+ SET status = 'revoked',
+ refresh_token_encrypted = NULL,
+ access_token_encrypted = NULL,
+ access_token_expires_at = NULL,
+ catalog_status = 'never_synced',
+ credential_revision = credential_revision + 1,
+ refresh_lock_token = NULL,
+ refresh_lock_expires_at = NULL,
+ revoked_at = now(),
+ updated_at = now()
+ WHERE connection_key = $1
+ AND revoked_at IS NULL
+ RETURNING ${SAFE_CONNECTION_COLUMNS}
+ ),
+ disabled_models AS (
+ UPDATE grok_catalog_models
+ SET enabled = false,
+ stale = true
+ WHERE connection_id IN (SELECT id FROM revoked)
+ RETURNING 1
+ )
+ SELECT ${SAFE_CONNECTION_COLUMNS}
+ FROM revoked`,
[key]
)
const row = result.rows[0] as SafeConnectionRow | undefined
if (!row) return getSafeGrokSubscriptionConnection(db, key)
- await db.query(
- `UPDATE grok_catalog_models
- SET enabled = false,
- stale = true
- WHERE connection_id = $1`,
- [row.id]
- )
return toSafeConnection(row)
}
@@ -599,15 +662,17 @@ export async function updateGrokSubscriptionConnectionMetadata(
return row ? toSafeConnection(row) : null
}
-function remapFingerprintConflict(err: unknown): never | Error {
+function remapUniqueConflict(err: unknown): unknown {
const conflict = err as { code?: string; constraint?: string } | null
- if (
- conflict?.code === '23505' &&
- conflict.constraint === 'grok_subscription_connections_active_fingerprint'
- ) {
- return new GrokSubscriptionFingerprintConflictError()
+ if (conflict?.code === '23505') {
+ if (conflict.constraint === 'grok_subscription_connections_active_fingerprint') {
+ return new GrokSubscriptionFingerprintConflictError()
+ }
+ if (conflict.constraint && CONNECTION_KEY_UNIQUE_CONSTRAINTS.has(conflict.constraint)) {
+ return new GrokSubscriptionConnectionKeyConflictError()
+ }
}
- throw err
+ return err
}
function toSafeConnection(row: SafeConnectionRow): GrokSubscriptionSafeConnection {
diff --git a/control-api/src/services/grokSubscriptionOAuth.ts b/control-api/src/services/grokSubscriptionOAuth.ts
index 5a670a82a..c15cf0a0f 100644
--- a/control-api/src/services/grokSubscriptionOAuth.ts
+++ b/control-api/src/services/grokSubscriptionOAuth.ts
@@ -4,15 +4,19 @@ import { rootLogger } from '../observability/logger.js'
import {
type GrokCatalogOutcome,
type GrokCatalogTransport,
+ type GrokTransactionRunner,
+ defaultGrokTransactionRunner,
rebuildLiveGrokUnionAllowlist,
syncGrokSubscriptionCatalog,
} from './grokSubscriptionCatalog.js'
import {
+ GrokSubscriptionConnectionKeyConflictError,
GrokSubscriptionFingerprintConflictError,
type GrokSubscriptionSafeConnection,
GrokSubscriptionStaleRevisionError,
acquireGrokSubscriptionRefreshLock,
assertGrokConnectionKey,
+ getGrokSubscriptionConnectionIncludingRevoked,
getSafeGrokSubscriptionConnection,
insertInitialGrokSubscriptionConnection,
loadGrokSubscriptionSecrets,
@@ -122,6 +126,8 @@ export type GrokOAuthDeps = {
clientId: string
enabled: boolean
connectionKey: string
+ /** Transaction runner bound to `db`'s pool; defaults to the Control API pool. */
+ withTransaction?: GrokTransactionRunner
}
function connectionKeyOf(deps: GrokOAuthDeps): string {
@@ -150,6 +156,24 @@ function requireEnabled(deps: GrokOAuthDeps): void {
}
}
+function revokedKeyError(): GrokSubscriptionOAuthError {
+ return new GrokSubscriptionOAuthError(
+ 'not_connected',
+ 'revoked grant cannot be reused; create a new connection'
+ )
+}
+
+function isRevoked(connection: GrokSubscriptionSafeConnection | null | undefined): boolean {
+ return Boolean(connection?.revokedAt || connection?.status === 'revoked')
+}
+
+/** Revocation is terminal per key: no OAuth work may start or finish on a tombstone. */
+async function assertKeyNotRevoked(deps: GrokOAuthDeps, key: string): Promise<void> {
+ if (isRevoked(await getGrokSubscriptionConnectionIncludingRevoked(deps.db, key))) {
+ throw revokedKeyError()
+ }
+}
+
export type GrokDeviceStartResult = {
userCode: string
verificationUri: string
@@ -191,6 +215,7 @@ export async function startGrokDeviceConnect(
): Promise<GrokDeviceStartResult> {
requireEnabled(deps)
const connectionKey = connectionKeyOf(deps)
+ await assertKeyNotRevoked(deps, connectionKey)
const started = await postForm(deps, GROK_OAUTH_DEVICE_URL, {
client_id: deps.clientId,
scope: GROK_OAUTH_SCOPES.join(' '),
@@ -235,6 +260,7 @@ export async function pollGrokDevice(
state: string
): Promise<GrokDevicePollResult> {
requireEnabled(deps)
+ await assertKeyNotRevoked(deps, connectionKeyOf(deps))
const pending = await peekPendingGrokSubscriptionOAuthState(deps.db, deps.encryptionKey, state)
if (!pending) {
throw new GrokSubscriptionOAuthError('state_replayed', 'device state is not pending')
@@ -314,8 +340,12 @@ export async function revokeGrokSubscription(
requireEnabled(deps)
const key = connectionKeyOf(deps)
const secrets = await loadGrokSubscriptionSecrets(deps.db, deps.encryptionKey, key)
- const local = await revokeGrokSubscriptionConnection(deps.db, key)
- await rebuildLiveGrokUnionAllowlist(deps.db)
+ const runInTransaction = deps.withTransaction ?? defaultGrokTransactionRunner
+ const local = await runInTransaction(async tx => {
+ const revoked = await revokeGrokSubscriptionConnection(tx, key)
+ await rebuildLiveGrokUnionAllowlist(tx)
+ return revoked
+ })
if (secrets?.refreshToken) {
try {
await postForm(deps, GROK_OAUTH_REVOKE_URL, {
@@ -392,9 +422,13 @@ export async function runGrokCatalogSync(
if (!secrets?.accessToken) {
return { ok: false, catalogStatus: 'never_synced', reason: 'no_grant' }
}
- const synced = await syncGrokSubscriptionCatalog(deps.db, transport, secrets.accessToken, {
- connectionKey: key,
- })
+ const synced = await syncGrokSubscriptionCatalog(
+ deps.db,
+ transport,
+ secrets.accessToken,
+ { connectionKey: key },
+ { withTransaction: deps.withTransaction }
+ )
if (!synced.connection) {
return { ok: false, catalogStatus: 'never_synced', reason: 'stale_revision' }
}
@@ -487,13 +521,9 @@ async function persistGrantedTokens(
connectionKey: string
): Promise<GrokSubscriptionSafeConnection> {
const key = assertGrokConnectionKey(connectionKey)
- const existing = await getSafeGrokSubscriptionConnection(deps.db, key)
- if (existing?.revokedAt || existing?.status === 'revoked') {
- throw new GrokSubscriptionOAuthError(
- 'not_connected',
- 'revoked grant cannot be reused; create a new connection'
- )
- }
+ // Tombstone-aware: a key revoked after this flow started must not be revived.
+ const existing = await getGrokSubscriptionConnectionIncludingRevoked(deps.db, key)
+ if (isRevoked(existing)) throw revokedKeyError()
if (
existing?.accountFingerprint &&
existing.accountFingerprint !== parsed.accountFingerprint &&
@@ -512,14 +542,27 @@ async function persistGrantedTokens(
status: 'connected' as const,
}
if (!existing) {
- const created = await insertInitialGrokSubscriptionConnection(
- deps.db,
- deps.encryptionKey,
- write,
- key
- )
- log.info({ event: 'grok_oauth_persisted', connectionKey: key }, 'Grok grant persisted')
- return created
+ try {
+ const created = await insertInitialGrokSubscriptionConnection(
+ deps.db,
+ deps.encryptionKey,
+ write,
+ key
+ )
+ log.info({ event: 'grok_oauth_persisted', connectionKey: key }, 'Grok grant persisted')
+ return created
+ } catch (err) {
+ if (err instanceof GrokSubscriptionConnectionKeyConflictError) {
+ throw await lostKeyRaceError(deps, key, 'a concurrent grant created this connection')
+ }
+ if (err instanceof GrokSubscriptionFingerprintConflictError) {
+ throw new GrokSubscriptionOAuthError(
+ 'fingerprint_in_use',
+ 'a live Grok subscription already uses this account'
+ )
+ }
+ throw err
+ }
}
try {
const rotated = await rotateGrokSubscriptionCredentials(
@@ -533,7 +576,7 @@ async function persistGrantedTokens(
return rotated
} catch (err) {
if (err instanceof GrokSubscriptionStaleRevisionError) {
- throw new GrokSubscriptionOAuthError('stale_revision', 'connection was replaced concurrently')
+ throw await lostKeyRaceError(deps, key, 'connection was replaced concurrently')
}
if (err instanceof GrokSubscriptionFingerprintConflictError) {
throw new GrokSubscriptionOAuthError(
@@ -545,6 +588,21 @@ async function persistGrantedTokens(
}
}
+/**
+ * A completion lost a write race on its key. If the winner was a revoke, the
+ * key is a terminal tombstone (not_connected); otherwise another writer holds
+ * the live row (stale_revision, 409).
+ */
+async function lostKeyRaceError(
+ deps: GrokOAuthDeps,
+ key: string,
+ message: string
+): Promise<GrokSubscriptionOAuthError> {
+ const latest = await getGrokSubscriptionConnectionIncludingRevoked(deps.db, key)
+ if (isRevoked(latest)) return revokedKeyError()
+ return new GrokSubscriptionOAuthError('stale_revision', message)
+}
+
type GrokTokenPair = {
accessToken: string
refreshToken: string
diff --git a/control-api/src/services/grokSubscriptionSchema.ts b/control-api/src/services/grokSubscriptionSchema.ts
index 8188790e5..3c968480e 100644
--- a/control-api/src/services/grokSubscriptionSchema.ts
+++ b/control-api/src/services/grokSubscriptionSchema.ts
@@ -183,3 +183,66 @@ export async function applyLlmProviderAttemptsGrokBrokerSchema(db: DbClient): Pr
END $$;
`)
}
+
+/**
+ * 0113 — revocation is terminal per Grok connection key.
+ *
+ * 0109 only enforced key uniqueness among live rows (`revoked_at IS NULL`), so
+ * a revoked key could be revived by an older pending device flow (or a raw
+ * insert) and existing key-based Host/recipe assignments would silently become
+ * usable again. From 0113 a connection_key names exactly one row for its whole
+ * lifetime; reconnecting after revoke requires a fresh key (the UI generates
+ * `grok-<hex>` keys).
+ *
+ * Additive and idempotent, safe to run against DBs where 0109–0112 already ran:
+ * 1. Archive superseded tombstones: when a key already has several rows (a
+ * live row plus tombstones, or several tombstones), every row except the
+ * live one — or, when no live row exists, the newest tombstone — gets its
+ * key rewritten to `<key>~revoked~<id>`. `~` is outside the key grammar, so
+ * archived keys can never collide with or be addressed as a real key.
+ * Rows are never deleted: catalog models and provider attempts keep
+ * pointing at their connection id.
+ * 2. Cancel pending OAuth device states whose key is now a tombstone.
+ * 3. Replace the live-only key index with a full unique index. The
+ * fingerprint index stays live-only (an account may reconnect under a new
+ * key).
+ * Pods still running pre-0113 code keep working: a revive attempt now fails
+ * closed with 23505 instead of creating a second row.
+ */
+export async function applyGrokSubscriptionTerminalConnectionKeySchema(
+ db: DbClient
+): Promise<void> {
+ await db.query(`
+ WITH ranked AS (
+ SELECT id,
+ row_number() OVER (
+ PARTITION BY connection_key
+ ORDER BY revoked_at DESC NULLS FIRST, created_at DESC, id DESC
+ ) AS key_rank
+ FROM grok_subscription_connections
+ )
+ UPDATE grok_subscription_connections c
+ SET connection_key = c.connection_key || '~revoked~' || c.id::text,
+ updated_at = now()
+ FROM ranked r
+ WHERE r.id = c.id
+ AND r.key_rank > 1
+ AND c.revoked_at IS NOT NULL;
+
+ UPDATE grok_subscription_oauth_states s
+ SET status = 'cancelled',
+ cancelled_at = now()
+ WHERE s.status = 'pending'
+ AND EXISTS (
+ SELECT 1
+ FROM grok_subscription_connections c
+ WHERE c.connection_key = s.connection_key
+ AND c.revoked_at IS NOT NULL
+ );
+
+ CREATE UNIQUE INDEX IF NOT EXISTS grok_subscription_connections_key_unique
+ ON grok_subscription_connections (connection_key);
+
+ DROP INDEX IF EXISTS grok_subscription_connections_active_key;
+ `)
+}
diff --git a/control-api/test/db.grokSubscriptionMigration.test.ts b/control-api/test/db.grokSubscriptionMigration.test.ts
index 34b06a153..f7946a964 100644
--- a/control-api/test/db.grokSubscriptionMigration.test.ts
+++ b/control-api/test/db.grokSubscriptionMigration.test.ts
@@ -66,4 +66,19 @@ describe('0109-0112 Grok subscription persistence', () => {
expect(profiles.get('grok_subscription_oauth_states')).toBe('upsert')
expect(profiles.get('grok_catalog_models')).toBe('upsert')
})
+
+ it('registers 0113 terminal Grok connection keys after 0112 as an additive migration', () => {
+ const v0112 = dbSource.indexOf("version: '0112_llm_provider_attempts_grok_broker'")
+ const v0113 = dbSource.indexOf("version: '0113_grok_subscription_terminal_connection_key'")
+ expect(v0113).toBeGreaterThan(v0112)
+ expect(dbSource).toContain('apply: applyGrokSubscriptionTerminalConnectionKeySchema')
+ const body = schemaSource.slice(
+ schemaSource.indexOf('export async function applyGrokSubscriptionTerminalConnectionKeySchema')
+ )
+ expect(body).toContain(
+ 'CREATE UNIQUE INDEX IF NOT EXISTS grok_subscription_connections_key_unique'
+ )
+ expect(body).toMatch(/ON grok_subscription_connections \(connection_key\);/)
+ expect(body).toContain("status = 'cancelled'")
+ })
})
diff --git a/control-api/test/db.oauthGrantsOwnerGeneralization.test.ts b/control-api/test/db.oauthGrantsOwnerGeneralization.test.ts
index f475ccab2..e7b14131d 100644
--- a/control-api/test/db.oauthGrantsOwnerGeneralization.test.ts
+++ b/control-api/test/db.oauthGrantsOwnerGeneralization.test.ts
@@ -21,7 +21,7 @@ describe('0101 oauth_grants owner generalization migration', () => {
const { CONTROL_API_MIGRATIONS } = await import('../src/db.js')
const versions = CONTROL_API_MIGRATIONS.map(m => m.version)
expect(versions).toContain('0106_oauth_grants_owner_generalization')
- expect(versions.at(-1)).toBe('0112_llm_provider_attempts_grok_broker')
+ expect(versions.at(-1)).toBe('0113_grok_subscription_terminal_connection_key')
expect(versions.indexOf('0100_seed_minimax_allowed_model')).toBeLessThan(
versions.indexOf('0106_oauth_grants_owner_generalization')
)
diff --git a/control-api/test/services.grokProviderAttemptRedemption.test.ts b/control-api/test/services.grokProviderAttemptRedemption.test.ts
index 334939068..db09d2732 100644
--- a/control-api/test/services.grokProviderAttemptRedemption.test.ts
+++ b/control-api/test/services.grokProviderAttemptRedemption.test.ts
@@ -119,4 +119,43 @@ describe('redeemGrokProviderAttempt', () => {
)
).rejects.toMatchObject({ code: 'no_grant' })
})
+
+ it('maps a missing connection row to connection_unavailable instead of an unmapped error', async () => {
+ vi.spyOn(store, 'peekLlmProviderAttemptTicket').mockResolvedValue({
+ jti: CLAIMS.jti,
+ providerAttemptId: CLAIMS.providerAttemptId,
+ status: 'issued',
+ expiresAt: new Date(Date.now() + 30_000),
+ receiptHash: null,
+ })
+ vi.spyOn(store, 'loadLlmProviderAttempt').mockResolvedValue({
+ id: CLAIMS.providerAttemptId,
+ provider: 'grok-subscription',
+ connectionId: CONNECTION_ID,
+ status: 'authorized',
+ } as never)
+ const ensureFresh = vi.fn(async (connectionKey?: string) => {
+ connection.assertGrokConnectionKey(connectionKey ?? '')
+ })
+ const withTransaction = vi.fn()
+ await expect(
+ redeemGrokProviderAttempt(
+ { executionTicket: 'ticket', requestHash: CLAIMS.requestHash },
+ {
+ enabled: true,
+ db: { query: vi.fn() },
+ withTransaction: withTransaction as never,
+ loadSecrets: vi.fn(),
+ getConnectionById: vi.fn().mockResolvedValue(null),
+ encryptionKey: Buffer.alloc(32),
+ ensureFreshAccessToken: ensureFresh,
+ }
+ )
+ ).rejects.toMatchObject({
+ name: 'GrokProviderAttemptRedeemError',
+ code: 'connection_unavailable',
+ })
+ expect(ensureFresh).not.toHaveBeenCalled()
+ expect(withTransaction).not.toHaveBeenCalled()
+ })
})
diff --git a/control-api/test/services.grokSubscriptionCatalog.test.ts b/control-api/test/services.grokSubscriptionCatalog.test.ts
index 8819f6997..b250717c9 100644
--- a/control-api/test/services.grokSubscriptionCatalog.test.ts
+++ b/control-api/test/services.grokSubscriptionCatalog.test.ts
@@ -1,9 +1,10 @@
-import { describe, expect, it } from 'vitest'
+import { describe, expect, it, vi } from 'vitest'
import { CATALOG_ORIGIN } from '@clerum/grok-provider-attempt-contract'
import {
GROK_CATALOG_ORIGIN,
pickGrokGrantModel,
planGrokCatalogReconcile,
+ syncGrokSubscriptionCatalog,
} from '../src/services/grokSubscriptionCatalog.js'
import {
isGrokUnassignedConnectionKey,
@@ -32,4 +33,100 @@ describe('grok subscription catalog', () => {
expect(isGrokUnassignedConnectionKey(readHostGrokConnectionRef(''))).toBe(true)
expect(isGrokUnassignedConnectionKey(readHostGrokConnectionRef('team-grok'))).toBe(false)
})
+
+ describe('syncGrokSubscriptionCatalog atomicity', () => {
+ const CONNECTION_ROW = {
+ id: '11111111-1111-4111-8111-111111111111',
+ connection_key: 'team-grok',
+ display_name: 'team-grok',
+ default_model: null,
+ created_by: null,
+ status: 'connected',
+ credential_revision: 2,
+ catalog_revision: 0,
+ account_fingerprint: 'fp',
+ catalog_status: 'never_synced',
+ catalog_synced_at: null,
+ last_refresh_at: null,
+ last_auth_at: null,
+ refresh_lock_token: null,
+ refresh_lock_expires_at: null,
+ revoked_at: null,
+ created_at: new Date(),
+ updated_at: new Date(),
+ }
+
+ function harness(opts: { failOn?: RegExp } = {}) {
+ let inTransaction = false
+ const writesOutside: string[] = []
+ const txLog: string[] = []
+ const outside = {
+ query: vi.fn(async (sql: string) => {
+ if (/^\s*(INSERT|UPDATE)/i.test(sql)) writesOutside.push(sql)
+ if (sql.includes('FROM grok_subscription_connections')) {
+ return { rows: [CONNECTION_ROW], rowCount: 1 }
+ }
+ return { rows: [], rowCount: 0 }
+ }),
+ }
+ const tx = {
+ query: vi.fn(async (sql: string) => {
+ if (!inTransaction) writesOutside.push(sql)
+ txLog.push(sql)
+ if (opts.failOn?.test(sql)) throw new Error('injected failure')
+ if (sql.includes('UPDATE grok_subscription_connections')) {
+ return { rows: [{ ...CONNECTION_ROW, catalog_status: 'ready' }], rowCount: 1 }
+ }
+ if (sql.includes('INSERT INTO grok_catalog_models')) return { rows: [], rowCount: 1 }
+ return { rows: [], rowCount: 0 }
+ }),
+ }
+ const withTransaction = vi.fn(async <T>(work: (db: typeof tx) => Promise<T>) => {
+ inTransaction = true
+ try {
+ return await work(tx)
+ } finally {
+ inTransaction = false
+ }
+ })
+ return { outside, tx, txLog, writesOutside, withTransaction }
+ }
+
+ it('records readiness, reconciles rows and rebuilds the union inside one transaction', async () => {
+ const h = harness()
+ const listModels = vi.fn(async () => {
+ expect(h.withTransaction).not.toHaveBeenCalled()
+ return { outcome: 'ready' as const, models: [{ model: 'grok-4.6' }] }
+ })
+ const synced = await syncGrokSubscriptionCatalog(
+ h.outside,
+ { listModels },
+ 'access-token',
+ { connectionKey: 'team-grok' },
+ { withTransaction: h.withTransaction as never }
+ )
+ expect(synced.outcome).toBe('ready')
+ expect(synced.added).toBe(1)
+ expect(h.withTransaction).toHaveBeenCalledTimes(1)
+ expect(h.writesOutside).toEqual([])
+ expect(h.txLog.some(sql => sql.includes('UPDATE grok_subscription_connections'))).toBe(true)
+ expect(h.txLog.some(sql => sql.includes('INSERT INTO grok_catalog_models'))).toBe(true)
+ expect(h.txLog.some(sql => sql.includes('INSERT INTO llm_allowed_models'))).toBe(true)
+ })
+
+ it('propagates a mid-reconcile failure out of the transaction instead of reporting ready', async () => {
+ const h = harness({ failOn: /INSERT INTO grok_catalog_models/ })
+ await expect(
+ syncGrokSubscriptionCatalog(
+ h.outside,
+ { listModels: async () => ({ outcome: 'ready', models: [{ model: 'grok-4.6' }] }) },
+ 'access-token',
+ { connectionKey: 'team-grok' },
+ { withTransaction: h.withTransaction as never }
+ )
+ ).rejects.toThrow('injected failure')
+ expect(h.writesOutside).toEqual([])
+ expect(h.txLog.some(sql => sql.includes('INSERT INTO llm_allowed_models'))).toBe(false)
+ })
+ })
})
diff --git a/control-api/test/services.grokSubscriptionConnection.realPostgres.integration.test.ts b/control-api/test/services.grokSubscriptionConnection.realPostgres.integration.test.ts
index aae1778c9..89e71dc77 100644
--- a/control-api/test/services.grokSubscriptionConnection.realPostgres.integration.test.ts
+++ b/control-api/test/services.grokSubscriptionConnection.realPostgres.integration.test.ts
@@ -1,14 +1,34 @@
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { randomBytes } from 'node:crypto'
import { Pool } from 'pg'
-import { initDb } from '../src/db.js'
+import { type DbClient, initDb } from '../src/db.js'
import { deriveOAuthEncryptionKey } from '../src/oauth/encryption.js'
import {
+ type GrokCatalogTransport,
+ type GrokTransactionRunner,
+ syncGrokSubscriptionCatalog,
+} from '../src/services/grokSubscriptionCatalog.js'
+import {
+ GrokSubscriptionConnectionKeyConflictError,
acquireGrokSubscriptionRefreshLock,
+ createNamedGrokSubscriptionConnection,
+ getSafeGrokSubscriptionConnection,
insertInitialGrokSubscriptionConnection,
releaseGrokSubscriptionRefreshLock,
} from '../src/services/grokSubscriptionConnection.js'
+import {
+ GROK_OAUTH_DEVICE_URL,
+ GROK_OAUTH_REVOKE_URL,
+ GROK_OAUTH_TOKEN_URL,
+ type GrokOAuthDeps,
+ pollGrokDevice,
+ revokeGrokSubscription,
+ startGrokDeviceConnect,
+} from '../src/services/grokSubscriptionOAuth.js'
+import { insertGrokSubscriptionOAuthState } from '../src/services/grokSubscriptionOAuthState.js'
+import { applyGrokSubscriptionTerminalConnectionKeySchema } from '../src/services/grokSubscriptionSchema.js'
import { insertLlmProviderAttempt } from '../src/services/llmProviderAttemptStore.js'
+import './realPostgres.requirement.ts'
const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
const describeRealPostgres = adminUrl ? describe : describe.skip
@@ -24,6 +44,48 @@ function quoteIdent(value: string): string {
return `"${value.replace(/"/g, '""')}"`
}
+function idTokenFor(subject: string): string {
+ return `hdr.${Buffer.from(JSON.stringify({ sub: subject })).toString('base64url')}.sig`
+}
+
+function jsonResponse(status: number, body: Record<string, unknown>): Response {
+ return { ok: status >= 200 && status < 300, status, json: async () => body } as Response
+}
+
+/**
+ * Simulated xAI device-authorization provider. Every device code completes
+ * immediately with a subject derived from the code, so parallel flows carry
+ * distinct account fingerprints.
+ */
+function simulatedGrokProvider(): typeof fetch {
+ let issued = 0
+ return (async (url: string | URL, init?: RequestInit) => {
+ const target = String(url)
+ const form = new URLSearchParams(String(init?.body ?? ''))
+ if (target === GROK_OAUTH_DEVICE_URL) {
+ issued += 1
+ return jsonResponse(200, {
+ device_code: `device-code-${issued}-${randomBytes(4).toString('hex')}`,
+ user_code: 'ABCD-EFGH',
+ verification_uri: 'https://auth.x.ai/activate',
+ expires_in: 900,
+ interval: 5,
+ })
+ }
+ if (target === GROK_OAUTH_TOKEN_URL) {
+ const deviceCode = form.get('device_code') ?? 'refresh'
+ return jsonResponse(200, {
+ access_token: `access-${deviceCode}`,
+ refresh_token: `refresh-${deviceCode}`,
+ expires_in: 3600,
+ id_token: idTokenFor(`subject-${deviceCode}`),
+ })
+ }
+ if (target === GROK_OAUTH_REVOKE_URL) return jsonResponse(200, {})
+ return jsonResponse(404, {})
+ }) as typeof fetch
+}
+
describeRealPostgres('Grok subscription connection on real PostgreSQL', () => {
const database = `grok_subscription_${randomBytes(6).toString('hex')}`
const connectionString = databaseUrl(
@@ -33,6 +95,51 @@ describeRealPostgres('Grok subscription connection on real PostgreSQL', () => {
let adminPool: Pool
let pool: Pool
+ const poolTransaction: GrokTransactionRunner = async work => {
+ const client = await pool.connect()
+ try {
+ await client.query('BEGIN')
+ const result = await work(client)
+ await client.query('COMMIT')
+ return result
+ } catch (err) {
+ await client.query('ROLLBACK').catch(() => undefined)
+ throw err
+ } finally {
+ client.release()
+ }
+ }
+
+ function oauthDeps(connectionKey: string, db: DbClient = pool): GrokOAuthDeps {
+ return {
+ db,
+ encryptionKey: KEY,
+ fetchFn: simulatedGrokProvider(),
+ clientId: 'b1a00492-073a-47ea-816f-4c329264a828',
+ enabled: true,
+ connectionKey,
+ withTransaction: poolTransaction,
+ }
+ }
+
+ async function rowsForKey(connectionKey: string) {
+ const result = await pool.query<{ id: string; status: string; revoked_at: Date | null }>(
+ `SELECT id, status, revoked_at
+ FROM grok_subscription_connections
+ WHERE connection_key = $1`,
+ [connectionKey]
+ )
+ return result.rows
+ }
+
+ async function stateStatus(state: string): Promise<string | undefined> {
+ const result = await pool.query<{ status: string }>(
+ `SELECT status FROM grok_subscription_oauth_states WHERE state = $1`,
+ [state]
+ )
+ return result.rows[0]?.status
+ }
+
beforeAll(async () => {
adminPool = new Pool({ connectionString: adminUrl })
await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)
@@ -148,4 +255,293 @@ describeRealPostgres('Grok subscription connection on real PostgreSQL', () => {
expect(row.provider).toBe('grok-subscription')
expect(row.connectionId).toBe(created.id)
})
+
+ it('revoke cancels pending device states and a revoked key never gets a new live row', async () => {
+ const key = 'team-grok-revoke'
+ await insertInitialGrokSubscriptionConnection(
+ pool,
+ KEY,
+ { refreshToken: 'refresh-revoke', accountFingerprint: 'fp-revoke' },
+ key
+ )
+ const deps = oauthDeps(key)
+ const started = await startGrokDeviceConnect(deps, 'reconnect')
+ expect(await stateStatus(started.state)).toBe('pending')
+
+ await revokeGrokSubscription(deps)
+ expect(await stateStatus(started.state)).toBe('cancelled')
+
+ await expect(pollGrokDevice(deps, started.state)).rejects.toMatchObject({
+ code: 'not_connected',
+ })
+ await expect(startGrokDeviceConnect(deps, 'connect')).rejects.toMatchObject({
+ code: 'not_connected',
+ })
+ const rows = await rowsForKey(key)
+ expect(rows).toHaveLength(1)
+ expect(rows[0]).toMatchObject({ status: 'revoked' })
+ expect(rows[0]?.revoked_at).toBeInstanceOf(Date)
+
+ // DB backstop: even a writer that skips the service checks cannot revive the key.
+ await expect(
+ insertInitialGrokSubscriptionConnection(
+ pool,
+ KEY,
+ { refreshToken: 'refresh-revive', accountFingerprint: 'fp-revive' },
+ key
+ )
+ ).rejects.toBeInstanceOf(GrokSubscriptionConnectionKeyConflictError)
+ await expect(
+ createNamedGrokSubscriptionConnection(pool, { connectionKey: key, displayName: 'revive' })
+ ).rejects.toMatchObject({ code: '23505' })
+ expect(await rowsForKey(key)).toHaveLength(1)
+ })
+
+ it('revoke of a key that never connected still invalidates its pending device state', async () => {
+ const key = 'team-grok-never'
+ const deps = oauthDeps(key)
+ const started = await startGrokDeviceConnect(deps, 'connect')
+ await revokeGrokSubscription(deps)
+ expect(await stateStatus(started.state)).toBe('cancelled')
+ await expect(pollGrokDevice(deps, started.state)).rejects.toMatchObject({
+ code: 'state_replayed',
+ })
+ expect(await rowsForKey(key)).toHaveLength(0)
+ })
+
+ it('parallel first grants for one key: one connects and the loser gets stale_revision', async () => {
+ const key = 'team-grok-race'
+ const held: Array<() => void> = []
+ // Hold each first-grant INSERT until both completions reached it, so both
+ // observed "no row" before either wrote (the M13 race).
+ const gatedDb: DbClient = {
+ query: async (text, values) => {
+ if (text.includes('INSERT INTO grok_subscription_connections')) {
+ await new Promise<void>(resolve => {
+ held.push(resolve)
+ if (held.length === 2) for (const release of held) release()
+ })
+ }
+ return pool.query(text, values)
+ },
+ }
+ const first = await startGrokDeviceConnect(oauthDeps(key), 'connect')
+ const second = await startGrokDeviceConnect(oauthDeps(key), 'connect')
+ const results = await Promise.allSettled([
+ pollGrokDevice(oauthDeps(key, gatedDb), first.state),
+ pollGrokDevice(oauthDeps(key, gatedDb), second.state),
+ ])
+ const fulfilled = results.filter(result => result.status === 'fulfilled')
+ const rejected = results.filter(
+ (result): result is PromiseRejectedResult => result.status === 'rejected'
+ )
+ expect(fulfilled).toHaveLength(1)
+ expect(rejected).toHaveLength(1)
+ expect(rejected[0]?.reason).toMatchObject({ code: 'stale_revision' })
+ const rows = await rowsForKey(key)
+ expect(rows).toHaveLength(1)
+ expect(rows[0]).toMatchObject({ status: 'connected', revoked_at: null })
+ })
+
+ it('concurrent raw inserts for one key yield exactly one row and one key conflict', async () => {
+ const key = 'team-grok-insert-race'
+ const results = await Promise.allSettled([
+ insertInitialGrokSubscriptionConnection(
+ pool,
+ KEY,
+ { refreshToken: 'refresh-a', accountFingerprint: 'fp-insert-a' },
+ key
+ ),
+ insertInitialGrokSubscriptionConnection(
+ pool,
+ KEY,
+ { refreshToken: 'refresh-b', accountFingerprint: 'fp-insert-b' },
+ key
+ ),
+ ])
+ expect(results.filter(result => result.status === 'fulfilled')).toHaveLength(1)
+ const rejected = results.find(
+ (result): result is PromiseRejectedResult => result.status === 'rejected'
+ )
+ expect(rejected?.reason).toBeInstanceOf(GrokSubscriptionConnectionKeyConflictError)
+ expect(await rowsForKey(key)).toHaveLength(1)
+ })
+
+ it('a catalog failure mid-sync leaves readiness, revision and model rows unchanged', async () => {
+ const key = 'team-grok-catalog'
+ const created = await insertInitialGrokSubscriptionConnection(
+ pool,
+ KEY,
+ { refreshToken: 'refresh-catalog', accessToken: 'access', accountFingerprint: 'fp-catalog' },
+ key
+ )
+ const failing: GrokCatalogTransport = {
+ listModels: async () => ({
+ outcome: 'ready',
+ // INTEGER overflow fails the second model INSERT after readiness was recorded.
+ models: [
+ { model: 'grok-catalog-ok' },
+ { model: 'grok-catalog-bad', contextWindowTokens: 3_000_000_000 },
+ ],
+ }),
+ }
+ await expect(
+ syncGrokSubscriptionCatalog(
+ pool,
+ failing,
+ 'access',
+ { connectionKey: key },
+ { withTransaction: poolTransaction }
+ )
+ ).rejects.toThrow()
+ const after = await getSafeGrokSubscriptionConnection(pool, key)
+ expect(after).toMatchObject({
+ catalogStatus: 'never_synced',
+ catalogRevision: created.catalogRevision,
+ catalogSyncedAt: null,
+ })
+ const models = await pool.query(
+ `SELECT model FROM grok_catalog_models WHERE connection_id = $1`,
+ [created.id]
+ )
+ expect(models.rows).toEqual([])
+ const union = await pool.query(
+ `SELECT model FROM llm_allowed_models WHERE provider = 'grok-subscription' AND model = $1`,
+ ['grok-catalog-ok']
+ )
+ expect(union.rows).toEqual([])
+
+ const healthy = await syncGrokSubscriptionCatalog(
+ pool,
+ { listModels: async () => ({ outcome: 'ready', models: [{ model: 'grok-catalog-ok' }] }) },
+ 'access',
+ { connectionKey: key },
+ { withTransaction: poolTransaction }
+ )
+ expect(healthy.connection).toMatchObject({
+ catalogStatus: 'ready',
+ catalogRevision: created.catalogRevision + 1,
+ })
+ expect(healthy.added).toBe(1)
+ })
+
+ it('concurrent catalog syncs on one revision commit exactly one outcome', async () => {
+ const key = 'team-grok-catalog-race'
+ const created = await insertInitialGrokSubscriptionConnection(
+ pool,
+ KEY,
+ { refreshToken: 'refresh-cr', accessToken: 'access', accountFingerprint: 'fp-catalog-race' },
+ key
+ )
+ const waiting: Array<() => void> = []
+ const transport: GrokCatalogTransport = {
+ listModels: async () => {
+ await new Promise<void>(resolve => {
+ waiting.push(resolve)
+ if (waiting.length === 2) for (const release of waiting) release()
+ })
+ return { outcome: 'ready', models: [{ model: 'grok-catalog-race' }] }
+ },
+ }
+ const results = await Promise.all([
+ syncGrokSubscriptionCatalog(
+ pool,
+ transport,
+ 'access',
+ { connectionKey: key },
+ {
+ withTransaction: poolTransaction,
+ }
+ ),
+ syncGrokSubscriptionCatalog(
+ pool,
+ transport,
+ 'access',
+ { connectionKey: key },
+ {
+ withTransaction: poolTransaction,
+ }
+ ),
+ ])
+ expect(results.filter(result => result.connection !== null)).toHaveLength(1)
+ const after = await getSafeGrokSubscriptionConnection(pool, key)
+ expect(after?.catalogRevision).toBe(created.catalogRevision + 1)
+ })
+
+ it('0113 archives superseded tombstones, cancels their pending states and is idempotent', async () => {
+ await pool.query(`DROP INDEX IF EXISTS grok_subscription_connections_key_unique`)
+ const insertRow = async (
+ key: string,
+ revokedAgoMinutes: number | null,
+ createdAgoMinutes: number
+ ) => {
+ const result = await pool.query<{ id: string }>(
+ `INSERT INTO grok_subscription_connections
+ (connection_key, status, revoked_at, created_at)
+ VALUES ($1, $2, $3, now() - ($4 * interval '1 minute'))
+ RETURNING id`,
+ [
+ key,
+ revokedAgoMinutes === null ? 'disconnected' : 'revoked',
+ revokedAgoMinutes === null ? null : new Date(Date.now() - revokedAgoMinutes * 60_000),
+ createdAgoMinutes,
+ ]
+ )
+ return result.rows[0]!.id
+ }
+ const staleTombstone = await insertRow('team-grok-dup', 50, 60)
+ const live = await insertRow('team-grok-dup', null, 10)
+ const olderTombstone = await insertRow('team-grok-tomb', 40, 60)
+ const newestTombstone = await insertRow('team-grok-tomb', 5, 30)
+ const pendingFor = async (key: string) =>
+ (
+ await insertGrokSubscriptionOAuthState(pool, KEY, {
+ state: `state-${randomBytes(6).toString('hex')}`,
+ intent: 'connect',
+ deviceCode: 'device-secret',
+ expiresAt: new Date(Date.now() + 600_000),
+ connectionKey: key,
+ })
+ ).state
+ const liveState = await pendingFor('team-grok-dup')
+ const tombState = await pendingFor('team-grok-tomb')
+
+ await applyGrokSubscriptionTerminalConnectionKeySchema(pool)
+ await applyGrokSubscriptionTerminalConnectionKeySchema(pool)
+
+ expect((await rowsForKey('team-grok-dup')).map(row => row.id)).toEqual([live])
+ expect((await rowsForKey('team-grok-tomb')).map(row => row.id)).toEqual([newestTombstone])
+ const archived = await pool.query<{
+ id: string
+ connection_key: string
+ revoked_at: Date | null
+ }>(
+ `SELECT id, connection_key, revoked_at
+ FROM grok_subscription_connections
+ WHERE id = ANY($1::uuid[])
+ ORDER BY id`,
+ [[staleTombstone, olderTombstone]]
+ )
+ expect(archived.rows).toHaveLength(2)
+ for (const row of archived.rows) {
+ expect(row.revoked_at).toBeInstanceOf(Date)
+ expect(row.connection_key).toContain(`~revoked~${row.id}`)
+ }
+ expect(await stateStatus(liveState)).toBe('pending')
+ expect(await stateStatus(tombState)).toBe('cancelled')
+ const index = await pool.query(
+ `SELECT indexdef FROM pg_indexes WHERE indexname = 'grok_subscription_connections_key_unique'`
+ )
+ expect(String((index.rows[0] as { indexdef?: string } | undefined)?.indexdef)).toMatch(
+ /UNIQUE INDEX .* \(connection_key\)$/
+ )
+ await expect(
+ insertInitialGrokSubscriptionConnection(
+ pool,
+ KEY,
+ { refreshToken: 'refresh-tomb', accountFingerprint: 'fp-tomb' },
+ 'team-grok-tomb'
+ )
+ ).rejects.toBeInstanceOf(GrokSubscriptionConnectionKeyConflictError)
+ })
})
diff --git a/control-api/test/services.grokSubscriptionConnection.test.ts b/control-api/test/services.grokSubscriptionConnection.test.ts
index 27dcaea89..d1e34b518 100644
--- a/control-api/test/services.grokSubscriptionConnection.test.ts
+++ b/control-api/test/services.grokSubscriptionConnection.test.ts
@@ -1,15 +1,18 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { deriveOAuthEncryptionKey, encryptOAuthSecret } from '../src/oauth/encryption.js'
import {
+ GrokSubscriptionConnectionKeyConflictError,
GrokSubscriptionFingerprintConflictError,
GrokSubscriptionInvalidConnectionKeyError,
GrokSubscriptionStaleRevisionError,
assertGrokConnectionKey,
generateGrokConnectionKey,
+ getGrokSubscriptionConnectionIncludingRevoked,
getSafeGrokSubscriptionConnection,
insertInitialGrokSubscriptionConnection,
persistGrokRefreshCiphertextFirst,
readHostGrokConnectionRef,
+ revokeGrokSubscriptionConnection,
} from '../src/services/grokSubscriptionConnection.js'
const KEY = deriveOAuthEncryptionKey('ab'.repeat(32))
@@ -154,6 +157,46 @@ describe('grok subscription connection repository', () => {
).rejects.toBeInstanceOf(GrokSubscriptionFingerprintConflictError)
})
+ it.each(['grok_subscription_connections_active_key', 'grok_subscription_connections_key_unique'])(
+ 'maps a %s unique violation to a connection key conflict',
+ async constraint => {
+ query.mockRejectedValueOnce(
+ Object.assign(new Error('duplicate'), { code: '23505', constraint })
+ )
+ await expect(
+ insertInitialGrokSubscriptionConnection(
+ { query },
+ KEY,
+ { refreshToken: 'plain-refresh', accountFingerprint: 'fp_race' },
+ 'team-grok'
+ )
+ ).rejects.toBeInstanceOf(GrokSubscriptionConnectionKeyConflictError)
+ }
+ )
+
+ it('reads revoked tombstones through the lifecycle lookup', async () => {
+ query.mockResolvedValueOnce({ rows: [], rowCount: 0 })
+ await expect(
+ getGrokSubscriptionConnectionIncludingRevoked({ query }, 'team-grok')
+ ).resolves.toBe(null)
+ const sql = String(query.mock.calls[0]?.[0])
+ expect(sql).toContain('connection_key = $1')
+ expect(sql).not.toContain('revoked_at IS NULL')
+ })
+
+ it('revokes the grant and cancels pending OAuth states for the key atomically', async () => {
+ query.mockResolvedValueOnce({ rows: [], rowCount: 0 })
+ query.mockResolvedValueOnce({ rows: [], rowCount: 0 })
+ await revokeGrokSubscriptionConnection({ query }, 'team-grok')
+ const sql = String(query.mock.calls[0]?.[0])
+ expect(sql).toContain('UPDATE grok_subscription_oauth_states')
+ expect(sql).toContain("status = 'cancelled'")
+ expect(sql).toContain("status = 'pending'")
+ expect(sql).toContain('UPDATE grok_subscription_connections')
+ expect(sql).toContain('UPDATE grok_catalog_models')
+ expect(query.mock.calls[0]?.[1]).toEqual(['team-grok'])
+ })
+
it('fences persist-first refresh on lock token, revision, and live lease', async () => {
query.mockResolvedValueOnce({ rows: [], rowCount: 0 })
await expect(
diff --git a/control-api/test/services.grokSubscriptionOAuth.test.ts b/control-api/test/services.grokSubscriptionOAuth.test.ts
index dd58af4cc..62da05865 100644
--- a/control-api/test/services.grokSubscriptionOAuth.test.ts
+++ b/control-api/test/services.grokSubscriptionOAuth.test.ts
@@ -1,7 +1,11 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { createHash } from 'node:crypto'
import { deriveOAuthEncryptionKey } from '../src/oauth/encryption.js'
-import { GrokSubscriptionStaleRevisionError } from '../src/services/grokSubscriptionConnection.js'
+import { rebuildLiveGrokUnionAllowlist } from '../src/services/grokSubscriptionCatalog.js'
+import {
+ GrokSubscriptionConnectionKeyConflictError,
+ GrokSubscriptionStaleRevisionError,
+} from '../src/services/grokSubscriptionConnection.js'
const repos = vi.hoisted(() => ({
insertState: vi.fn(),
@@ -10,6 +14,7 @@ const repos = vi.hoisted(() => ({
expireState: vi.fn(),
cancelState: vi.fn(),
getSafe: vi.fn(),
+ getAny: vi.fn(),
insertInitial: vi.fn(),
rotate: vi.fn(),
acquireLock: vi.fn(),
@@ -38,6 +43,7 @@ vi.mock('../src/services/grokSubscriptionConnection.js', async () => {
return {
...actual,
getSafeGrokSubscriptionConnection: repos.getSafe,
+ getGrokSubscriptionConnectionIncludingRevoked: repos.getAny,
insertInitialGrokSubscriptionConnection: repos.insertInitial,
rotateGrokSubscriptionCredentials: repos.rotate,
acquireGrokSubscriptionRefreshLock: repos.acquireLock,
@@ -67,6 +73,7 @@ const {
ensureFreshGrokAccessToken,
pollGrokDevice,
refreshGrokSubscriptionConnection,
+ revokeGrokSubscription,
startGrokDeviceConnect,
} = await import('../src/services/grokSubscriptionOAuth.js')
@@ -567,3 +574,160 @@ describe('grok subscription OAuth device broker', () => {
expect(repos.releaseLock).toHaveBeenCalled()
})
})
+
+function pendingState(state: string, status: 'pending' | 'consumed' = 'pending') {
+ return {
+ safe: {
+ state,
+ flow: 'device',
+ intent: 'connect',
+ status,
+ connectionKey: CONNECTION_KEY,
+ expiresAt: new Date(Date.now() + 60_000),
+ consumedAt: status === 'consumed' ? new Date() : null,
+ cancelledAt: null,
+ createdAt: new Date(),
+ },
+ deviceCode: 'device-secret',
+ }
+}
+
+function tokenFetch(subject = 'acct_raw_123') {
+ return vi.fn().mockResolvedValue({
+ ok: true,
+ status: 200,
+ json: async () => ({
+ access_token: 'access-secret',
+ refresh_token: 'refresh-secret',
+ expires_in: 60,
+ id_token: idTokenFor(subject),
+ }),
+ })
+}
+
+const REVOKED_ROW = {
+ id: '11111111-1111-4111-8111-111111111111',
+ connectionKey: CONNECTION_KEY,
+ status: 'revoked',
+ credentialRevision: 2,
+ accountFingerprint: null,
+ revokedAt: new Date(),
+}
+
+const LIVE_ROW = {
+ id: '11111111-1111-4111-8111-111111111111',
+ connectionKey: CONNECTION_KEY,
+ status: 'connected',
+ credentialRevision: 1,
+ accountFingerprint: fingerprint('acct_raw_123'),
+ revokedAt: null,
+}
+
+describe('grok subscription grant lifecycle (revocation is terminal per key)', () => {
+ beforeEach(() => {
+ for (const fn of Object.values(repos)) fn.mockReset()
+ vi.mocked(rebuildLiveGrokUnionAllowlist).mockReset()
+ })
+
+ it('refuses to start device authorization for a revoked key', async () => {
+ repos.getAny.mockResolvedValue(REVOKED_ROW)
+ const fetchFn = vi.fn()
+ await expect(startGrokDeviceConnect(deps(fetchFn), 'connect')).rejects.toMatchObject({
+ code: 'not_connected',
+ })
+ expect(fetchFn).not.toHaveBeenCalled()
+ expect(repos.insertState).not.toHaveBeenCalled()
+ })
+
+ it('refuses to poll a device flow for a revoked key before consuming state or calling upstream', async () => {
+ repos.getAny.mockResolvedValue(REVOKED_ROW)
+ repos.peekState.mockResolvedValue(pendingState('dev-revoked'))
+ const fetchFn = tokenFetch()
+ await expect(pollGrokDevice(deps(fetchFn), 'dev-revoked')).rejects.toMatchObject({
+ code: 'not_connected',
+ })
+ expect(fetchFn).not.toHaveBeenCalled()
+ expect(repos.consumeState).not.toHaveBeenCalled()
+ expect(repos.insertInitial).not.toHaveBeenCalled()
+ })
+
+ it('does not recreate a live row when the key is revoked after the state was consumed', async () => {
+ repos.getAny.mockResolvedValueOnce(null).mockResolvedValue(REVOKED_ROW)
+ repos.peekState.mockResolvedValue(pendingState('dev-late'))
+ repos.consumeState.mockResolvedValue(pendingState('dev-late', 'consumed'))
+ await expect(pollGrokDevice(deps(tokenFetch()), 'dev-late')).rejects.toMatchObject({
+ code: 'not_connected',
+ })
+ expect(repos.insertInitial).not.toHaveBeenCalled()
+ expect(repos.rotate).not.toHaveBeenCalled()
+ })
+
+ it('maps a first-grant key race against a live row to stale_revision', async () => {
+ repos.getAny.mockResolvedValueOnce(null).mockResolvedValueOnce(null).mockResolvedValue(LIVE_ROW)
+ repos.peekState.mockResolvedValue(pendingState('dev-race'))
+ repos.consumeState.mockResolvedValue(pendingState('dev-race', 'consumed'))
+ repos.insertInitial.mockRejectedValue(new GrokSubscriptionConnectionKeyConflictError())
+ await expect(pollGrokDevice(deps(tokenFetch()), 'dev-race')).rejects.toMatchObject({
+ code: 'stale_revision',
+ })
+ })
+
+ it('maps a first-grant key conflict against a revoked tombstone to not_connected', async () => {
+ repos.getAny
+ .mockResolvedValueOnce(null)
+ .mockResolvedValueOnce(null)
+ .mockResolvedValue(REVOKED_ROW)
+ repos.peekState.mockResolvedValue(pendingState('dev-tomb'))
+ repos.consumeState.mockResolvedValue(pendingState('dev-tomb', 'consumed'))
+ repos.insertInitial.mockRejectedValue(new GrokSubscriptionConnectionKeyConflictError())
+ await expect(pollGrokDevice(deps(tokenFetch()), 'dev-tomb')).rejects.toMatchObject({
+ code: 'not_connected',
+ })
+ })
+
+ it('maps a rotate that lost to a concurrent revoke to not_connected', async () => {
+ repos.getAny
+ .mockResolvedValueOnce(LIVE_ROW)
+ .mockResolvedValueOnce(LIVE_ROW)
+ .mockResolvedValue(REVOKED_ROW)
+ repos.peekState.mockResolvedValue(pendingState('dev-rotate'))
+ repos.consumeState.mockResolvedValue(pendingState('dev-rotate', 'consumed'))
+ repos.rotate.mockRejectedValue(new GrokSubscriptionStaleRevisionError())
+ await expect(pollGrokDevice(deps(tokenFetch()), 'dev-rotate')).rejects.toMatchObject({
+ code: 'not_connected',
+ })
+ })
+
+ it('revokes the grant and rebuilds the union inside one transaction, upstream revoke after commit', async () => {
+ const events: string[] = []
+ repos.loadSecrets.mockResolvedValue({
+ refreshToken: 'refresh-secret',
+ accessToken: null,
+ accessTokenExpiresAt: null,
+ credentialRevision: 1,
+ })
+ repos.revokeConnection.mockImplementation(async (db: unknown) => {
+ events.push(db === 'tx' ? 'revoke:tx' : 'revoke:outside')
+ return REVOKED_ROW
+ })
+ vi.mocked(rebuildLiveGrokUnionAllowlist).mockImplementation(async (db: unknown) => {
+ events.push(db === 'tx' ? 'rebuild:tx' : 'rebuild:outside')
+ })
+ const fetchFn = vi.fn().mockImplementation(async () => {
+ events.push('upstream')
+ return { ok: true, status: 200, json: async () => ({}) }
+ })
+ const withTransaction = vi.fn(async (work: (tx: never) => Promise<unknown>) => {
+ events.push('begin')
+ const result = await work('tx' as never)
+ events.push('commit')
+ return result
+ })
+ const revoked = await revokeGrokSubscription({
+ ...deps(fetchFn),
+ withTransaction: withTransaction as never,
+ })
+ expect(revoked).toMatchObject({ status: 'revoked' })
+ expect(events).toEqual(['begin', 'revoke:tx', 'rebuild:tx', 'commit', 'upstream'])
+ })
+})