Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(control-api): answer 503 instead of 500 when external session validation cannot reach Postgres
Alfredo Lopezda09507af009
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 8fa990243..e21fcf1e4 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.609",+ "version": "0.1.610","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.609",+ "version": "0.1.610","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 6b17fc92c..7e01ada66 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.609",+ "version": "0.1.610","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/middleware/externalSessionAuth.ts b/control-api/src/middleware/externalSessionAuth.tsindex 024f5a5cb..313ca7b47 100644--- a/control-api/src/middleware/externalSessionAuth.ts+++ b/control-api/src/middleware/externalSessionAuth.ts@@ -1,8 +1,12 @@import { NextFunction, Request, Response } from 'express'import { pool } from '../db.js'+import { rootLogger } from '../observability/logger.js'import { AuthClaims, TeamRole } from '../profileTypes.js'import { verifyExternalSessionToken } from '../utils/auth/externalSessionAuthToken.js'+/** Retry-After sent when the user row cannot be read to validate a session. */+export const SESSION_BACKEND_RETRY_AFTER_SECONDS = 2+export type ExternalAuthedRequest = Request & {externalAuth?: AuthClaims}@@ -66,7 +70,29 @@ async function requireValidExternalSessionTokenAsync(return}- if (!(await isCurrentExternalSession(claims))) {+ let current: boolean+ try {+ current = await isCurrentExternalSession(claims)+ } catch (error) {+ // The users-table lookup failed (pool acquire timeout, connection or+ // statement error): the session could not be judged, which is neither+ // a denial (401) nor a defect in this request (500).+ rootLogger.warn(+ {+ event: 'external_session_backend_unavailable',+ err: error instanceof Error ? error.message : String(error),+ },+ 'external session validation could not reach PostgreSQL'+ )+ res.setHeader('Retry-After', String(SESSION_BACKEND_RETRY_AFTER_SECONDS))+ res.setHeader('Cache-Control', 'no-store')+ res.status(503).json({+ error: 'session_backend_unavailable',+ retryAfterSeconds: SESSION_BACKEND_RETRY_AFTER_SECONDS,+ })+ return+ }+ if (!current) {res.status(401).json({ error: 'Unauthorized' })return}diff --git a/control-api/test/middleware.externalSessionAuth.lifecycle.test.ts b/control-api/test/middleware.externalSessionAuth.lifecycle.test.tsindex 9627e1329..a4a211145 100644--- a/control-api/test/middleware.externalSessionAuth.lifecycle.test.ts+++ b/control-api/test/middleware.externalSessionAuth.lifecycle.test.ts@@ -5,10 +5,14 @@ import request from 'supertest'const poolQuery = vi.hoisted(() => vi.fn())const verifyToken = vi.hoisted(() => vi.fn())+const loggerWarn = vi.hoisted(() => vi.fn())vi.mock('../src/db.js', () => ({pool: { query: (...args: unknown[]) => poolQuery(...args) },}))+vi.mock('../src/observability/logger.js', () => ({+ rootLogger: { warn: (...args: unknown[]) => loggerWarn(...args) },+}))vi.mock('../src/utils/auth/externalSessionAuthToken.js', () => ({verifyExternalSessionToken: (...args: unknown[]) => verifyToken(...args),}))@@ -34,13 +38,19 @@ function app() {},(_req, res) => res.status(200).json({ ok: true }))+ server.use(+ (_err: unknown, _req: express.Request, res: express.Response, _next: express.NextFunction) => {+ res.status(500).json({ error: 'internal' })+ }+ )return server}describe('external session lifecycle gate', () => {beforeEach(() => {- poolQuery.mockClear()- verifyToken.mockClear()+ poolQuery.mockReset()+ verifyToken.mockReset()+ loggerWarn.mockReset()poolQuery.mockResolvedValue({rows: [{ lifecycle_state: 'active', lifecycle_version: '4' }],rowCount: 1,@@ -74,4 +84,34 @@ describe('external session lifecycle gate', () => {expect(response.status).toBe(401)expect(poolQuery).not.toHaveBeenCalled()})++ it('answers 503 with Retry-After when the lifecycle row cannot be read', async () => {+ verifyToken.mockReturnValueOnce(claims)+ poolQuery.mockRejectedValueOnce(new Error('timeout exceeded when trying to connect'))+ const response = await request(app()).get('/protected').set('x-user-session-token', 'session')+ expect(response.status).toBe(503)+ expect(response.body).toEqual({ error: 'session_backend_unavailable', retryAfterSeconds: 2 })+ expect(response.headers['retry-after']).toBe('2')+ expect(response.headers['cache-control']).toBe('no-store')+ // Witness: the lookup was attempted exactly once, for the token's user.+ expect(poolQuery).toHaveBeenCalledTimes(1)+ expect(poolQuery.mock.calls[0]?.[1]).toEqual([claims.userId])+ expect(loggerWarn).toHaveBeenCalledTimes(1)+ expect(loggerWarn.mock.calls[0]?.[0]).toEqual({+ event: 'external_session_backend_unavailable',+ err: 'timeout exceeded when trying to connect',+ })+ })++ it('keeps 401 for an invalid token while the lifecycle backend is failing', async () => {+ verifyToken.mockReturnValueOnce(null)+ poolQuery.mockRejectedValue(new Error('timeout exceeded when trying to connect'))+ const response = await request(app()).get('/protected').set('x-user-session-token', 'forged')+ expect(response.status).toBe(401)+ expect(response.body).toEqual({ error: 'Unauthorized' })+ // Witness: the token was verified; the rejection came before any lookup.+ expect(verifyToken).toHaveBeenCalledWith('forged')+ expect(poolQuery).not.toHaveBeenCalled()+ expect(loggerWarn).not.toHaveBeenCalled()+ })})diff --git a/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts b/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.tsindex f501ae2ca..8c5bf1b53 100644--- a/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts+++ b/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts@@ -11,7 +11,10 @@ import request from 'supertest'// CONTROL_API_REAL_PG_ADMIN_URL.vi.mock('../src/utils/auth/externalSessionAuthToken.js', () => ({+ // A token that is not a JSON identity fails verification, as a bad+ // signature does in production.verifyExternalSessionToken: (token: string) => {+ if (!token.startsWith('{')) return nullconst parsed = JSON.parse(token) as { userId: string }return {userId: parsed.userId,@@ -32,6 +35,12 @@ const LIMITER_ACQUIRE_TIMEOUT_MS = 5_000// the budget covers that walk so the regression fails on the status assertion,// not on the test timeout.const T3_TIMEOUT_MS = (4 + 2) * LIMITER_ACQUIRE_TIMEOUT_MS+const CORE_POOL_MAX = 10+const CORE_ACQUIRE_TIMEOUT_MS = 2_000+// T5: serial same-row upserts per pool. 10 ms per upsert is the regression+// line; the plan's reversal rule (report and stop) sits at 2.5 ms with sync off.+const T5_UPSERTS = 200+const T5_MAX_TOTAL_MS = T5_UPSERTS * 10function databaseUrl(baseUrl: string, database: string): string {const url = new URL(baseUrl)@@ -294,4 +303,94 @@ describeRealPostgres('external GFS rate limiter backend (real PostgreSQL)', () =expect(keys.length).toBeGreaterThan(0)expect(keys.every(key => key.startsWith('gfs-ext:'))).toBe(true)}, 30_000)++ it('T4: with every core connection held, session validation answers 503 while an invalid token still answers 401', async () => {+ type Options = { options: { max: number; connectionTimeoutMillis: number } }+ expect((corePool as unknown as Options).options).toMatchObject({+ max: CORE_POOL_MAX,+ connectionTimeoutMillis: CORE_ACQUIRE_TIMEOUT_MS,+ })+ const app = mod.createApp(new mod.MockGateway())+ const userId = await seedUser()+ const internalToken = mod.config.internalServiceTokens['external-rest-api']+ if (!internalToken) throw new Error('config has no external-rest-api internal service token')+ const external = (req: request.Test, sessionToken: string) =>+ req+ .set('Authorization', `Bearer ${internalToken}`)+ .set('x-service-token', 'external-rest-api')+ .set('x-user-session-token', sessionToken)+ .set('x-forwarded-for', '203.0.113.7')+ const affordancesPath = () =>+ `/api/v1/external/gfs/resources/${randomUUID()}/affordances?drive=main`++ const held: PoolClient[] = []+ let unavailable: request.Response+ let invalid: request.Response+ let elapsedMs: number+ try {+ for (let index = 0; index < CORE_POOL_MAX; index += 1) {+ held.push(await corePool.connect())+ }+ // Witness: the core pool is exhausted, not merely busy.+ expect(corePool.totalCount).toBe(CORE_POOL_MAX)+ expect(corePool.idleCount).toBe(0)++ const startedAt = Date.now()+ unavailable = await external(request(app).get(affordancesPath()), JSON.stringify({ userId }))+ elapsedMs = Date.now() - startedAt+ invalid = await external(request(app).get(affordancesPath()), 'not-a-session-token')+ } finally {+ for (const client of held) client.release()+ }++ expect(unavailable.status).toBe(503)+ expect(unavailable.body).toEqual({+ error: 'session_backend_unavailable',+ retryAfterSeconds: 2,+ })+ expect(unavailable.headers['retry-after']).toBe('2')+ expect(unavailable.headers['cache-control']).toBe('no-store')+ // Witness: the 503 came from waiting out the core pool's acquire timeout.+ expect(elapsedMs).toBeGreaterThanOrEqual(CORE_ACQUIRE_TIMEOUT_MS - 250)++ expect(invalid.status).toBe(401)+ expect(invalid.body).toEqual({ error: 'Unauthorized' })++ // Liveness witness: the same valid request, once the pool is free, passes+ // session validation and reaches the handler.+ const allowed = await external(request(app).get(affordancesPath()), JSON.stringify({ userId }))+ expect(allowed.status).toBe(200)+ }, 30_000)++ it('T5 (informational): same-row limiter upserts with synchronous_commit off vs on', async () => {+ const { checkAndIncrementWithQuery } = await import('../src/services/rateLimiterService.js')+ const measure = async (pool: Pool, label: string): Promise<number> => {+ const key = `t5:${label}:${randomUUID()}`+ const nowMs = Date.now()+ const startedAt = process.hrtime.bigint()+ let last = 0+ for (let index = 0; index < T5_UPSERTS; index += 1) {+ const result = await checkAndIncrementWithQuery(+ (text, values) => pool.query(text, values),+ key,+ T5_UPSERTS,+ nowMs+ )+ expect(result.backendAvailable).toBe(true)+ last = result.count+ }+ const totalMs = Number(process.hrtime.bigint() - startedAt) / 1e6+ // Witness: every upsert landed on the same row.+ expect(last).toBe(T5_UPSERTS)+ return totalMs+ }++ const offMs = await measure(limiterPool, 'sync-off')+ const onMs = await measure(corePool, 'sync-on')+ console.info(+ `[T5] ${T5_UPSERTS} serial same-row upserts: synchronous_commit=off ` ++ `${(offMs / T5_UPSERTS).toFixed(3)} ms/upsert, on ${(onMs / T5_UPSERTS).toFixed(3)} ms/upsert`+ )+ expect(offMs).toBeLessThan(T5_MAX_TOTAL_MS)+ }, 30_000)})diff --git a/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts b/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.tsindex 8aa618cd7..8010d06ca 100644--- a/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts+++ b/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts@@ -24,9 +24,9 @@ import request from 'supertest'// resulting bounds are asserted before anything else.//// Route-level corollary: with the core pool held, an external GFS request-// returns 500, because session auth queries the users table first-// (externalSessionAuth.ts) and fails closed before the limiter runs. The only-// mock is the session-token verifier (identity). Skipped without+// returns 503 session_backend_unavailable, because session auth queries the+// users table first (externalSessionAuth.ts) and fails closed before the+// limiter runs. The only mock is the session-token verifier (identity). Skipped without// CONTROL_API_REAL_PG_ADMIN_URL.const mockVerifyExternalSessionToken = vi.fn()@@ -244,7 +244,7 @@ describeRealPostgres('rate limiter under pool saturation', () => {}}, 20_000)- it('answers 500 at the route while the core pool is held, because session auth fails first', async () => {+ it('answers 503 at the route while the core pool is held, because session auth fails first', async () => {const internalToken = mod.config.internalServiceTokens['external-rest-api']if (!internalToken) throw new Error('config has no external-rest-api internal service token')mockVerifyExternalSessionToken.mockReset()@@ -266,7 +266,7 @@ describeRealPostgres('rate limiter under pool saturation', () => {// Liveness witness with the pool free: the request passes internal auth,// reaches the session verifier, and is denied by the users-table lookup- // (no such user), so the 500 below comes from that same lookup.+ // (no such user), so the 503 below comes from that same lookup.const free = await send()expect(free.status).toBe(401)expect(mockVerifyExternalSessionToken).toHaveBeenCalledTimes(1)@@ -274,7 +274,12 @@ describeRealPostgres('rate limiter under pool saturation', () => {const held = await holdEveryClient(corePool, PRODUCTION_POOL_MAX)try {const saturated = await send()- expect(saturated.status).toBe(500)+ expect(saturated.status).toBe(503)+ expect(saturated.body).toEqual({+ error: 'session_backend_unavailable',+ retryAfterSeconds: 2,+ })+ expect(saturated.headers['retry-after']).toBe('2')expect(mockVerifyExternalSessionToken).toHaveBeenCalledTimes(2)} finally {for (const client of held) client.release()