Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): answer 503 only for connection-class session-auth errors

Alfredo Lopezbb4676989505
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 32b011270..dc8f73ce9 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.614",
+ "version": "0.1.615",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.614",
+ "version": "0.1.615",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index d212d7c77..e3dc19c99 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.614",
+ "version": "0.1.615",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/middleware/externalSessionAuth.ts b/control-api/src/middleware/externalSessionAuth.ts
index 313ca7b47..3cf4a859f 100644
--- a/control-api/src/middleware/externalSessionAuth.ts
+++ b/control-api/src/middleware/externalSessionAuth.ts
@@ -1,4 +1,5 @@
import { NextFunction, Request, Response } from 'express'
+import { DatabaseError } from 'pg'
import { pool } from '../db.js'
import { rootLogger } from '../observability/logger.js'
import { AuthClaims, TeamRole } from '../profileTypes.js'
@@ -7,6 +8,22 @@ import { verifyExternalSessionToken } from '../utils/auth/externalSessionAuthTok
/** Retry-After sent when the user row cannot be read to validate a session. */
export const SESSION_BACKEND_RETRY_AFTER_SECONDS = 2
+// SQLSTATE classes that mean the server could not run the query: 08 connection
+// exception, 53 insufficient resources, 57 operator intervention (includes
+// 57014 statement timeout), 58 system error.
+const BACKEND_UNAVAILABLE_SQLSTATE_CLASSES = new Set(['08', '53', '57', '58'])
+
+/**
+ * True when the lookup failed because PostgreSQL could not be reached or could
+ * not run it. Pool acquire timeouts and socket failures are plain Errors with
+ * no SQLSTATE. A DatabaseError in any other class (22P02, 42P01, ...) is a
+ * defect in the query or the schema, which a 503 would hide.
+ */
+function isBackendUnavailableError(error: unknown): boolean {
+ if (!(error instanceof DatabaseError)) return true
+ return BACKEND_UNAVAILABLE_SQLSTATE_CLASSES.has(String(error.code).slice(0, 2))
+}
+
export type ExternalAuthedRequest = Request & {
externalAuth?: AuthClaims
}
@@ -74,9 +91,11 @@ async function requireValidExternalSessionTokenAsync(
try {
current = await isCurrentExternalSession(claims)
} catch (error) {
- // The users-table lookup failed (pool acquire timeout, connection or
- // statement error): the session could not be judged, which is neither
- // a denial (401) nor a defect in this request (500).
+ // A query or schema defect goes to the error handler (500).
+ if (!isBackendUnavailableError(error)) throw error
+ // The users-table lookup could not run (pool acquire timeout, connection
+ // or statement timeout): the session could not be judged, which is
+ // neither a denial (401) nor a defect in this request (500).
rootLogger.warn(
{
event: 'external_session_backend_unavailable',
diff --git a/control-api/test/middleware.externalSessionAuth.lifecycle.test.ts b/control-api/test/middleware.externalSessionAuth.lifecycle.test.ts
index a4a211145..0138cbcb9 100644
--- a/control-api/test/middleware.externalSessionAuth.lifecycle.test.ts
+++ b/control-api/test/middleware.externalSessionAuth.lifecycle.test.ts
@@ -1,6 +1,7 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import express from 'express'
import { rateLimit } from 'express-rate-limit'
+import { DatabaseError } from 'pg'
import request from 'supertest'
const poolQuery = vi.hoisted(() => vi.fn())
@@ -26,6 +27,13 @@ const claims = {
exp: Math.floor(Date.now() / 1000) + 3600,
}
+/** A server-side error as pg raises it: a DatabaseError carrying a SQLSTATE. */
+function databaseError(sqlstate: string): DatabaseError {
+ const error = new DatabaseError(`server error ${sqlstate}`, 0, 'error')
+ error.code = sqlstate
+ return error
+}
+
function app() {
const server = express()
server.get(
@@ -103,6 +111,44 @@ describe('external session lifecycle gate', () => {
})
})
+ it.each([
+ ['a pool acquire timeout', new Error('timeout exceeded when trying to connect')],
+ ['a dropped connection', new Error('Connection terminated unexpectedly')],
+ [
+ 'a refused socket',
+ Object.assign(new Error('connect ECONNREFUSED 127.0.0.1:5432'), { code: 'ECONNREFUSED' }),
+ ],
+ // Five characters, like a SQLSTATE, but a socket error: no SQLSTATE class.
+ ['a broken pipe', Object.assign(new Error('write EPIPE'), { code: 'EPIPE' })],
+ ['SQLSTATE 08006 connection_failure', databaseError('08006')],
+ ['SQLSTATE 53300 too_many_connections', databaseError('53300')],
+ ['SQLSTATE 57P01 admin_shutdown', databaseError('57P01')],
+ ['SQLSTATE 57014 query_canceled (statement timeout)', databaseError('57014')],
+ ['SQLSTATE 58000 system_error', databaseError('58000')],
+ ])('answers 503 for %s', async (_label, error) => {
+ verifyToken.mockReturnValueOnce(claims)
+ poolQuery.mockRejectedValueOnce(error)
+ const response = await request(app()).get('/protected').set('x-user-session-token', 'session')
+ expect(response.status).toBe(503)
+ expect(response.body).toEqual({ error: 'session_backend_unavailable', retryAfterSeconds: 2 })
+ expect(poolQuery).toHaveBeenCalledTimes(1)
+ })
+
+ it.each([
+ ['SQLSTATE 22P02 invalid_text_representation', databaseError('22P02')],
+ ['SQLSTATE 42P01 undefined_table', databaseError('42P01')],
+ ['SQLSTATE 42703 undefined_column', databaseError('42703')],
+ ])('passes %s to the error handler as a 500, not a 503', async (_label, error) => {
+ verifyToken.mockReturnValueOnce(claims)
+ poolQuery.mockRejectedValueOnce(error)
+ const response = await request(app()).get('/protected').set('x-user-session-token', 'session')
+ expect(response.status).toBe(500)
+ expect(response.headers['retry-after']).toBeUndefined()
+ // Witness: the lookup ran and failed; the defect was not relabelled.
+ expect(poolQuery).toHaveBeenCalledTimes(1)
+ expect(loggerWarn).not.toHaveBeenCalled()
+ })
+
it('keeps 401 for an invalid token while the lifecycle backend is failing', async () => {
verifyToken.mockReturnValueOnce(null)
poolQuery.mockRejectedValue(new Error('timeout exceeded when trying to connect'))