Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): answer 503 instead of 500 when external session validation cannot reach Postgres

Alfredo Lopezda09507af009
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 8fa990243..e21fcf1e4 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.609",
+ "version": "0.1.610",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.609",
+ "version": "0.1.610",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 6b17fc92c..7e01ada66 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.609",
+ "version": "0.1.610",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/middleware/externalSessionAuth.ts b/control-api/src/middleware/externalSessionAuth.ts
index 024f5a5cb..313ca7b47 100644
--- a/control-api/src/middleware/externalSessionAuth.ts
+++ b/control-api/src/middleware/externalSessionAuth.ts
@@ -1,8 +1,12 @@
import { NextFunction, Request, Response } from 'express'
import { pool } from '../db.js'
+import { rootLogger } from '../observability/logger.js'
import { AuthClaims, TeamRole } from '../profileTypes.js'
import { verifyExternalSessionToken } from '../utils/auth/externalSessionAuthToken.js'
+/** Retry-After sent when the user row cannot be read to validate a session. */
+export const SESSION_BACKEND_RETRY_AFTER_SECONDS = 2
+
export type ExternalAuthedRequest = Request & {
externalAuth?: AuthClaims
}
@@ -66,7 +70,29 @@ async function requireValidExternalSessionTokenAsync(
return
}
- if (!(await isCurrentExternalSession(claims))) {
+ let current: boolean
+ try {
+ current = await isCurrentExternalSession(claims)
+ } catch (error) {
+ // The users-table lookup failed (pool acquire timeout, connection or
+ // statement error): the session could not be judged, which is neither
+ // a denial (401) nor a defect in this request (500).
+ rootLogger.warn(
+ {
+ event: 'external_session_backend_unavailable',
+ err: error instanceof Error ? error.message : String(error),
+ },
+ 'external session validation could not reach PostgreSQL'
+ )
+ res.setHeader('Retry-After', String(SESSION_BACKEND_RETRY_AFTER_SECONDS))
+ res.setHeader('Cache-Control', 'no-store')
+ res.status(503).json({
+ error: 'session_backend_unavailable',
+ retryAfterSeconds: SESSION_BACKEND_RETRY_AFTER_SECONDS,
+ })
+ return
+ }
+ if (!current) {
res.status(401).json({ error: 'Unauthorized' })
return
}
diff --git a/control-api/test/middleware.externalSessionAuth.lifecycle.test.ts b/control-api/test/middleware.externalSessionAuth.lifecycle.test.ts
index 9627e1329..a4a211145 100644
--- a/control-api/test/middleware.externalSessionAuth.lifecycle.test.ts
+++ b/control-api/test/middleware.externalSessionAuth.lifecycle.test.ts
@@ -5,10 +5,14 @@ import request from 'supertest'
const poolQuery = vi.hoisted(() => vi.fn())
const verifyToken = vi.hoisted(() => vi.fn())
+const loggerWarn = vi.hoisted(() => vi.fn())
vi.mock('../src/db.js', () => ({
pool: { query: (...args: unknown[]) => poolQuery(...args) },
}))
+vi.mock('../src/observability/logger.js', () => ({
+ rootLogger: { warn: (...args: unknown[]) => loggerWarn(...args) },
+}))
vi.mock('../src/utils/auth/externalSessionAuthToken.js', () => ({
verifyExternalSessionToken: (...args: unknown[]) => verifyToken(...args),
}))
@@ -34,13 +38,19 @@ function app() {
},
(_req, res) => res.status(200).json({ ok: true })
)
+ server.use(
+ (_err: unknown, _req: express.Request, res: express.Response, _next: express.NextFunction) => {
+ res.status(500).json({ error: 'internal' })
+ }
+ )
return server
}
describe('external session lifecycle gate', () => {
beforeEach(() => {
- poolQuery.mockClear()
- verifyToken.mockClear()
+ poolQuery.mockReset()
+ verifyToken.mockReset()
+ loggerWarn.mockReset()
poolQuery.mockResolvedValue({
rows: [{ lifecycle_state: 'active', lifecycle_version: '4' }],
rowCount: 1,
@@ -74,4 +84,34 @@ describe('external session lifecycle gate', () => {
expect(response.status).toBe(401)
expect(poolQuery).not.toHaveBeenCalled()
})
+
+ it('answers 503 with Retry-After when the lifecycle row cannot be read', async () => {
+ verifyToken.mockReturnValueOnce(claims)
+ poolQuery.mockRejectedValueOnce(new Error('timeout exceeded when trying to connect'))
+ const response = await request(app()).get('/protected').set('x-user-session-token', 'session')
+ expect(response.status).toBe(503)
+ expect(response.body).toEqual({ error: 'session_backend_unavailable', retryAfterSeconds: 2 })
+ expect(response.headers['retry-after']).toBe('2')
+ expect(response.headers['cache-control']).toBe('no-store')
+ // Witness: the lookup was attempted exactly once, for the token's user.
+ expect(poolQuery).toHaveBeenCalledTimes(1)
+ expect(poolQuery.mock.calls[0]?.[1]).toEqual([claims.userId])
+ expect(loggerWarn).toHaveBeenCalledTimes(1)
+ expect(loggerWarn.mock.calls[0]?.[0]).toEqual({
+ event: 'external_session_backend_unavailable',
+ err: 'timeout exceeded when trying to connect',
+ })
+ })
+
+ it('keeps 401 for an invalid token while the lifecycle backend is failing', async () => {
+ verifyToken.mockReturnValueOnce(null)
+ poolQuery.mockRejectedValue(new Error('timeout exceeded when trying to connect'))
+ const response = await request(app()).get('/protected').set('x-user-session-token', 'forged')
+ expect(response.status).toBe(401)
+ expect(response.body).toEqual({ error: 'Unauthorized' })
+ // Witness: the token was verified; the rejection came before any lookup.
+ expect(verifyToken).toHaveBeenCalledWith('forged')
+ expect(poolQuery).not.toHaveBeenCalled()
+ expect(loggerWarn).not.toHaveBeenCalled()
+ })
})
diff --git a/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts b/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts
index f501ae2ca..8c5bf1b53 100644
--- a/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts
+++ b/control-api/test/routes.externalGfsRateLimitFailClosed.realPostgres.integration.test.ts
@@ -11,7 +11,10 @@ import request from 'supertest'
// CONTROL_API_REAL_PG_ADMIN_URL.
vi.mock('../src/utils/auth/externalSessionAuthToken.js', () => ({
+ // A token that is not a JSON identity fails verification, as a bad
+ // signature does in production.
verifyExternalSessionToken: (token: string) => {
+ if (!token.startsWith('{')) return null
const parsed = JSON.parse(token) as { userId: string }
return {
userId: parsed.userId,
@@ -32,6 +35,12 @@ const LIMITER_ACQUIRE_TIMEOUT_MS = 5_000
// the budget covers that walk so the regression fails on the status assertion,
// not on the test timeout.
const T3_TIMEOUT_MS = (4 + 2) * LIMITER_ACQUIRE_TIMEOUT_MS
+const CORE_POOL_MAX = 10
+const CORE_ACQUIRE_TIMEOUT_MS = 2_000
+// T5: serial same-row upserts per pool. 10 ms per upsert is the regression
+// line; the plan's reversal rule (report and stop) sits at 2.5 ms with sync off.
+const T5_UPSERTS = 200
+const T5_MAX_TOTAL_MS = T5_UPSERTS * 10
function databaseUrl(baseUrl: string, database: string): string {
const url = new URL(baseUrl)
@@ -294,4 +303,94 @@ describeRealPostgres('external GFS rate limiter backend (real PostgreSQL)', () =
expect(keys.length).toBeGreaterThan(0)
expect(keys.every(key => key.startsWith('gfs-ext:'))).toBe(true)
}, 30_000)
+
+ it('T4: with every core connection held, session validation answers 503 while an invalid token still answers 401', async () => {
+ type Options = { options: { max: number; connectionTimeoutMillis: number } }
+ expect((corePool as unknown as Options).options).toMatchObject({
+ max: CORE_POOL_MAX,
+ connectionTimeoutMillis: CORE_ACQUIRE_TIMEOUT_MS,
+ })
+ const app = mod.createApp(new mod.MockGateway())
+ const userId = await seedUser()
+ const internalToken = mod.config.internalServiceTokens['external-rest-api']
+ if (!internalToken) throw new Error('config has no external-rest-api internal service token')
+ const external = (req: request.Test, sessionToken: string) =>
+ req
+ .set('Authorization', `Bearer ${internalToken}`)
+ .set('x-service-token', 'external-rest-api')
+ .set('x-user-session-token', sessionToken)
+ .set('x-forwarded-for', '203.0.113.7')
+ const affordancesPath = () =>
+ `/api/v1/external/gfs/resources/${randomUUID()}/affordances?drive=main`
+
+ const held: PoolClient[] = []
+ let unavailable: request.Response
+ let invalid: request.Response
+ let elapsedMs: number
+ try {
+ for (let index = 0; index < CORE_POOL_MAX; index += 1) {
+ held.push(await corePool.connect())
+ }
+ // Witness: the core pool is exhausted, not merely busy.
+ expect(corePool.totalCount).toBe(CORE_POOL_MAX)
+ expect(corePool.idleCount).toBe(0)
+
+ const startedAt = Date.now()
+ unavailable = await external(request(app).get(affordancesPath()), JSON.stringify({ userId }))
+ elapsedMs = Date.now() - startedAt
+ invalid = await external(request(app).get(affordancesPath()), 'not-a-session-token')
+ } finally {
+ for (const client of held) client.release()
+ }
+
+ expect(unavailable.status).toBe(503)
+ expect(unavailable.body).toEqual({
+ error: 'session_backend_unavailable',
+ retryAfterSeconds: 2,
+ })
+ expect(unavailable.headers['retry-after']).toBe('2')
+ expect(unavailable.headers['cache-control']).toBe('no-store')
+ // Witness: the 503 came from waiting out the core pool's acquire timeout.
+ expect(elapsedMs).toBeGreaterThanOrEqual(CORE_ACQUIRE_TIMEOUT_MS - 250)
+
+ expect(invalid.status).toBe(401)
+ expect(invalid.body).toEqual({ error: 'Unauthorized' })
+
+ // Liveness witness: the same valid request, once the pool is free, passes
+ // session validation and reaches the handler.
+ const allowed = await external(request(app).get(affordancesPath()), JSON.stringify({ userId }))
+ expect(allowed.status).toBe(200)
+ }, 30_000)
+
+ it('T5 (informational): same-row limiter upserts with synchronous_commit off vs on', async () => {
+ const { checkAndIncrementWithQuery } = await import('../src/services/rateLimiterService.js')
+ const measure = async (pool: Pool, label: string): Promise<number> => {
+ const key = `t5:${label}:${randomUUID()}`
+ const nowMs = Date.now()
+ const startedAt = process.hrtime.bigint()
+ let last = 0
+ for (let index = 0; index < T5_UPSERTS; index += 1) {
+ const result = await checkAndIncrementWithQuery(
+ (text, values) => pool.query(text, values),
+ key,
+ T5_UPSERTS,
+ nowMs
+ )
+ expect(result.backendAvailable).toBe(true)
+ last = result.count
+ }
+ const totalMs = Number(process.hrtime.bigint() - startedAt) / 1e6
+ // Witness: every upsert landed on the same row.
+ expect(last).toBe(T5_UPSERTS)
+ return totalMs
+ }
+
+ const offMs = await measure(limiterPool, 'sync-off')
+ const onMs = await measure(corePool, 'sync-on')
+ console.info(
+ `[T5] ${T5_UPSERTS} serial same-row upserts: synchronous_commit=off ` +
+ `${(offMs / T5_UPSERTS).toFixed(3)} ms/upsert, on ${(onMs / T5_UPSERTS).toFixed(3)} ms/upsert`
+ )
+ expect(offMs).toBeLessThan(T5_MAX_TOTAL_MS)
+ }, 30_000)
})
diff --git a/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts b/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts
index 8aa618cd7..8010d06ca 100644
--- a/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts
+++ b/control-api/test/services.rateLimiterPoolSaturation.realPostgres.integration.test.ts
@@ -24,9 +24,9 @@ import request from 'supertest'
// resulting bounds are asserted before anything else.
//
// Route-level corollary: with the core pool held, an external GFS request
-// returns 500, because session auth queries the users table first
-// (externalSessionAuth.ts) and fails closed before the limiter runs. The only
-// mock is the session-token verifier (identity). Skipped without
+// returns 503 session_backend_unavailable, because session auth queries the
+// users table first (externalSessionAuth.ts) and fails closed before the
+// limiter runs. The only mock is the session-token verifier (identity). Skipped without
// CONTROL_API_REAL_PG_ADMIN_URL.
const mockVerifyExternalSessionToken = vi.fn()
@@ -244,7 +244,7 @@ describeRealPostgres('rate limiter under pool saturation', () => {
}
}, 20_000)
- it('answers 500 at the route while the core pool is held, because session auth fails first', async () => {
+ it('answers 503 at the route while the core pool is held, because session auth fails first', async () => {
const internalToken = mod.config.internalServiceTokens['external-rest-api']
if (!internalToken) throw new Error('config has no external-rest-api internal service token')
mockVerifyExternalSessionToken.mockReset()
@@ -266,7 +266,7 @@ describeRealPostgres('rate limiter under pool saturation', () => {
// Liveness witness with the pool free: the request passes internal auth,
// reaches the session verifier, and is denied by the users-table lookup
- // (no such user), so the 500 below comes from that same lookup.
+ // (no such user), so the 503 below comes from that same lookup.
const free = await send()
expect(free.status).toBe(401)
expect(mockVerifyExternalSessionToken).toHaveBeenCalledTimes(1)
@@ -274,7 +274,12 @@ describeRealPostgres('rate limiter under pool saturation', () => {
const held = await holdEveryClient(corePool, PRODUCTION_POOL_MAX)
try {
const saturated = await send()
- expect(saturated.status).toBe(500)
+ expect(saturated.status).toBe(503)
+ expect(saturated.body).toEqual({
+ error: 'session_backend_unavailable',
+ retryAfterSeconds: 2,
+ })
+ expect(saturated.headers['retry-after']).toBe('2')
expect(mockVerifyExternalSessionToken).toHaveBeenCalledTimes(2)
} finally {
for (const client of held) client.release()