Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): name each external GFS edge backstop and report its denials

Alfredo Lopezef2f63578589
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index f64de5231..716800e60 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.600",
+ "version": "0.1.601",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.600",
+ "version": "0.1.601",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 20033d94b..a4c6f4544 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.600",
+ "version": "0.1.601",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/routes/external/gfs.ts b/control-api/src/routes/external/gfs.ts
index 9e6f33cab..d0825b19e 100644
--- a/control-api/src/routes/external/gfs.ts
+++ b/control-api/src/routes/external/gfs.ts
@@ -30,6 +30,7 @@ import {
externalGfsPreResolutionRateLimit,
externalGfsResolvedOperationRateLimit,
externalGfsSourceIp,
+ reportEdgeBackstopDenial,
} from '../../middleware/externalGfsRateLimit.js'
import {
type ExternalAuthedRequest,
@@ -106,6 +107,18 @@ function attachExternalGfsRequestId(
next()
}
+/** Name of each in-process edge backstop, reported as the `guard` log field. */
+type ExternalGfsEdgeGuard =
+ | 'ingress'
+ | 'token'
+ | 'resource'
+ | 'proxy-read'
+ | 'resource-mutation'
+ | 'grants-read'
+ | 'grants-mutation'
+ | 'shares-read'
+ | 'shares-mutation'
+
function externalGfsRateKey(value: string): string {
return createHash('sha256').update(value).digest('hex')
}
@@ -327,23 +340,50 @@ export function createExternalGfsRouter(): Router {
// distributed buckets retain the narrower security budgets. Edge rejects
// still expose a bounded Retry-After body/header so Desktop can recover
// without treating the backstop as an opaque 500.
- const edgeRateLimitHandler = (_req: Request, res: ExpressResponse): void => {
- const raw = res.getHeader('Retry-After')
- const retryAfterSeconds =
- typeof raw === 'number' && Number.isSafeInteger(raw) && raw > 0
- ? raw
- : typeof raw === 'string' && /^\d+$/.test(raw)
- ? Math.max(1, Number(raw))
- : 60
- res.status(429).json({ error: 'Too Many Requests', retryAfterSeconds })
- }
+ //
+ // The express-rate-limit MemoryStore window starts at the first hit for a
+ // key, while the Postgres limiters in externalGfsRateLimit.ts use windows
+ // aligned to floor(now / 60_000). A burst that straddles a minute boundary
+ // can stay under the limit in each clock window and still exceed this one,
+ // so a backstop can be the first, and sometimes the only, denier for such a
+ // burst. That is why its denials report here rather than via reportDecision.
+ const edgeRateLimitHandler =
+ (guard: ExternalGfsEdgeGuard, keyOf: (req: Request) => string) =>
+ (req: Request, res: ExpressResponse): void => {
+ const raw = res.getHeader('Retry-After')
+ const retryAfterSeconds =
+ typeof raw === 'number' && Number.isSafeInteger(raw) && raw > 0
+ ? raw
+ : typeof raw === 'string' && /^\d+$/.test(raw)
+ ? Math.max(1, Number(raw))
+ : 60
+ reportEdgeBackstopDenial({
+ req,
+ guard,
+ key: keyOf(req),
+ retryAfterSeconds,
+ // Only the ingress and token backstops are mounted before
+ // attachExternalGfsAuthority; the other seven run after it.
+ authorityResolutionAvoided: guard === 'ingress' || guard === 'token',
+ })
+ res.status(429).json({ error: 'Too Many Requests', retryAfterSeconds })
+ }
+ // One key-generator instance per guard, shared by keyGenerator and the
+ // handler, so the logged key digest is the key the store actually counted.
+ const resourceRateKey = externalGfsActorRateKey('resource')
+ const proxyReadRateKey = externalGfsActorRateKey('proxy-read')
+ const resourceMutationRateKey = externalGfsActorRateKey('resource-mutation')
+ const grantsReadRateKey = externalGfsActorRateKey('grants-read')
+ const grantsMutationRateKey = externalGfsActorRateKey('grants-mutation')
+ const sharesReadRateKey = externalGfsActorRateKey('shares-read')
+ const sharesMutationRateKey = externalGfsActorRateKey('shares-mutation')
const externalGfsIngressRateLimit = rateLimit({
windowMs: 60_000,
limit: config.externalGfsIngressRlPerMin,
standardHeaders: 'draft-7',
legacyHeaders: false,
keyGenerator: externalGfsIngressRateKey,
- handler: edgeRateLimitHandler,
+ handler: edgeRateLimitHandler('ingress', externalGfsIngressRateKey),
})
const externalGfsTokenRouteRateLimit = rateLimit({
windowMs: 60_000,
@@ -351,63 +391,63 @@ export function createExternalGfsRouter(): Router {
standardHeaders: 'draft-7',
legacyHeaders: false,
keyGenerator: externalGfsTokenRateKey,
- handler: edgeRateLimitHandler,
+ handler: edgeRateLimitHandler('token', externalGfsTokenRateKey),
})
const externalGfsResourceRouteRateLimit = rateLimit({
windowMs: 60_000,
limit: config.externalGfsReadRlPerMin,
standardHeaders: 'draft-7',
legacyHeaders: false,
- keyGenerator: externalGfsActorRateKey('resource'),
- handler: edgeRateLimitHandler,
+ keyGenerator: resourceRateKey,
+ handler: edgeRateLimitHandler('resource', resourceRateKey),
})
const externalGfsProxyReadRouteRateLimit = rateLimit({
windowMs: 60_000,
limit: config.externalGfsReadRlPerMin,
standardHeaders: 'draft-7',
legacyHeaders: false,
- keyGenerator: externalGfsActorRateKey('proxy-read'),
- handler: edgeRateLimitHandler,
+ keyGenerator: proxyReadRateKey,
+ handler: edgeRateLimitHandler('proxy-read', proxyReadRateKey),
})
const externalGfsMutationRouteRateLimit = rateLimit({
windowMs: 60_000,
limit: config.externalGfsOperationRlPerMin,
standardHeaders: 'draft-7',
legacyHeaders: false,
- keyGenerator: externalGfsActorRateKey('resource-mutation'),
- handler: edgeRateLimitHandler,
+ keyGenerator: resourceMutationRateKey,
+ handler: edgeRateLimitHandler('resource-mutation', resourceMutationRateKey),
})
const externalGfsGrantsReadRouteRateLimit = rateLimit({
windowMs: 60_000,
limit: config.externalGfsReadRlPerMin,
standardHeaders: 'draft-7',
legacyHeaders: false,
- keyGenerator: externalGfsActorRateKey('grants-read'),
- handler: edgeRateLimitHandler,
+ keyGenerator: grantsReadRateKey,
+ handler: edgeRateLimitHandler('grants-read', grantsReadRateKey),
})
const externalGfsGrantsMutationRouteRateLimit = rateLimit({
windowMs: 60_000,
limit: config.externalGfsOperationRlPerMin,
standardHeaders: 'draft-7',
legacyHeaders: false,
- keyGenerator: externalGfsActorRateKey('grants-mutation'),
- handler: edgeRateLimitHandler,
+ keyGenerator: grantsMutationRateKey,
+ handler: edgeRateLimitHandler('grants-mutation', grantsMutationRateKey),
})
const externalGfsSharesReadRouteRateLimit = rateLimit({
windowMs: 60_000,
limit: config.externalGfsReadRlPerMin,
standardHeaders: 'draft-7',
legacyHeaders: false,
- keyGenerator: externalGfsActorRateKey('shares-read'),
- handler: edgeRateLimitHandler,
+ keyGenerator: sharesReadRateKey,
+ handler: edgeRateLimitHandler('shares-read', sharesReadRateKey),
})
const externalGfsSharesMutationRouteRateLimit = rateLimit({
windowMs: 60_000,
limit: config.externalGfsOperationRlPerMin,
standardHeaders: 'draft-7',
legacyHeaders: false,
- keyGenerator: externalGfsActorRateKey('shares-mutation'),
- handler: edgeRateLimitHandler,
+ keyGenerator: sharesMutationRateKey,
+ handler: edgeRateLimitHandler('shares-mutation', sharesMutationRateKey),
})
// The source-IP guard intentionally precedes authentication and authority
diff --git a/control-api/test/routes.externalGfs.test.ts b/control-api/test/routes.externalGfs.test.ts
index 5dabe1865..99b579951 100644
--- a/control-api/test/routes.externalGfs.test.ts
+++ b/control-api/test/routes.externalGfs.test.ts
@@ -1,9 +1,15 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import express from 'express'
+import { ipKeyGenerator } from 'express-rate-limit'
+import { createHash } from 'node:crypto'
import request from 'supertest'
// The MOCKED config object (defined below); mutating gfscProxyTimeoutMs drives a
// real deadline abort in the read-proxy timeout test.
import { config } from '../src/config.js'
+// Real (unmocked) modules: the route imports these same instances, so a spy or
+// a counter read here observes exactly what the edge backstops emit.
+import { rootLogger } from '../src/observability/logger.js'
+import { externalGfsRateLimitRequestsTotal } from '../src/observability/metrics.js'
/**
* Unit tests for the end-user gfs surface on the /external (Session-JWT) plane.
@@ -274,6 +280,28 @@ function dbReturning(
})
}
+const sha256 = (value: string): string => createHash('sha256').update(value).digest('hex')
+
+/**
+ * Sum of the edge-backstop denial counter, narrowed to `labels` when given.
+ * The registry is process-wide, so callers compare a before/after delta.
+ */
+async function edgeBackstopDenials(
+ labels: Partial<Record<'operation_class' | 'route' | 'authority_resolution_avoided', string>> = {}
+): Promise<number> {
+ const { values } = await externalGfsRateLimitRequestsTotal.get()
+ return values
+ .filter(
+ sample =>
+ sample.labels.phase === 'edge-backstop' &&
+ sample.labels.outcome === 'denied' &&
+ (Object.keys(labels) as Array<keyof typeof labels>).every(
+ name => sample.labels[name] === labels[name]
+ )
+ )
+ .reduce((sum, sample) => sum + sample.value, 0)
+}
+
describe('POST /external/gfs/token (user mint — existing signer, sub=users.id)', () => {
it('allows a mixed GFS journey beyond the former 30/min ingress cap', async () => {
auth()
@@ -386,6 +414,197 @@ describe('POST /external/gfs/token (user mint — existing signer, sub=users.id)
}
})
+ // The three edge-backstop cases below mock every Postgres bucket as allowing
+ // (dbReturning → count 1), so each 429 can only come from an express backstop.
+ it('reports a resource edge-backstop denial on the rate metric and log', async () => {
+ const previousReadLimit = config.externalGfsReadRlPerMin
+ config.externalGfsReadRlPerMin = 2
+ const warn = vi.spyOn(rootLogger, 'warn').mockImplementation(() => {})
+ try {
+ auth()
+ dbReturning([])
+ const app = await buildApp()
+ const labels = {
+ operation_class: 'resource',
+ route: '/external/gfs/resources',
+ authority_resolution_avoided: 'false',
+ }
+ const allBefore = await edgeBackstopDenials()
+ const labelledBefore = await edgeBackstopDenials(labels)
+
+ for (let attempt = 0; attempt < 2; attempt += 1) {
+ await request(app)
+ .get('/external/gfs/resources')
+ .set('x-user-session-token', 'read-session')
+ .expect(200)
+ }
+ // Liveness witness for the silence below: the guard ran and allowed twice.
+ expect(warn).not.toHaveBeenCalled()
+ expect(await edgeBackstopDenials()).toBe(allBefore)
+
+ const exhausted = await request(app)
+ .get('/external/gfs/resources')
+ .set('x-user-session-token', 'read-session')
+
+ expect(exhausted.status).toBe(429)
+ expect(exhausted.body).toEqual({
+ error: 'Too Many Requests',
+ retryAfterSeconds: expect.any(Number),
+ })
+ const retryAfterSeconds = exhausted.body.retryAfterSeconds as number
+ expect(retryAfterSeconds).toBeGreaterThanOrEqual(1)
+ expect(warn).toHaveBeenCalledTimes(1)
+ expect(warn.mock.calls[0]).toEqual([
+ {
+ event: 'external_gfs_rate_limit',
+ phase: 'edge-backstop',
+ guard: 'resource',
+ operationClass: 'resource',
+ route: '/external/gfs/resources',
+ hashedKey: sha256(`external-gfs:resource:actor:${sha256(`user-session:${U1}`)}`),
+ retryAfterSeconds,
+ outcome: 'denied',
+ authorityResolutionAvoided: false,
+ },
+ 'external GFS edge backstop denied',
+ ])
+ expect(JSON.stringify(warn.mock.calls)).not.toContain(U1)
+ expect((await edgeBackstopDenials(labels)) - labelledBefore).toBe(1)
+ expect((await edgeBackstopDenials()) - allBefore).toBe(1)
+ } finally {
+ warn.mockRestore()
+ config.externalGfsReadRlPerMin = previousReadLimit
+ }
+ })
+
+ it('reports an ingress edge-backstop denial on a path outside the operation matrix', async () => {
+ const previousIngressLimit = config.externalGfsIngressRlPerMin
+ config.externalGfsIngressRlPerMin = 1
+ const warn = vi.spyOn(rootLogger, 'warn').mockImplementation(() => {})
+ const sourceIp = '203.0.113.9'
+ try {
+ auth()
+ const app = await buildApp()
+ const labels = {
+ operation_class: 'unclassified',
+ route: 'unclassified',
+ authority_resolution_avoided: 'true',
+ }
+ const allBefore = await edgeBackstopDenials()
+ const labelledBefore = await edgeBackstopDenials(labels)
+
+ await request(app)
+ .get('/external/gfs/not-classified')
+ .set('x-user-session-token', 'sess')
+ .set('x-forwarded-for', sourceIp)
+ .expect(404)
+ // Liveness witness for the silence below: the guard ran and allowed once.
+ expect(warn).not.toHaveBeenCalled()
+ expect(await edgeBackstopDenials()).toBe(allBefore)
+
+ const exhausted = await request(app)
+ .get('/external/gfs/not-classified')
+ .set('x-user-session-token', 'sess')
+ .set('x-forwarded-for', sourceIp)
+
+ expect(exhausted.status).toBe(429)
+ const retryAfterSeconds = exhausted.body.retryAfterSeconds as number
+ expect(retryAfterSeconds).toBeGreaterThanOrEqual(1)
+ // The ingress guard runs before authentication, so no session was checked.
+ expect(mockVerifyExternalSessionToken).toHaveBeenCalledTimes(1)
+ expect(warn).toHaveBeenCalledTimes(1)
+ expect(warn.mock.calls[0]).toEqual([
+ {
+ event: 'external_gfs_rate_limit',
+ phase: 'edge-backstop',
+ guard: 'ingress',
+ operationClass: 'unclassified',
+ route: 'unclassified',
+ hashedKey: sha256(`external-gfs:ingress:${sha256(ipKeyGenerator(sourceIp))}`),
+ retryAfterSeconds,
+ outcome: 'denied',
+ authorityResolutionAvoided: true,
+ },
+ 'external GFS edge backstop denied',
+ ])
+ expect(JSON.stringify(warn.mock.calls)).not.toContain(sourceIp)
+ expect((await edgeBackstopDenials(labels)) - labelledBefore).toBe(1)
+ expect((await edgeBackstopDenials()) - allBefore).toBe(1)
+ } finally {
+ warn.mockRestore()
+ config.externalGfsIngressRlPerMin = previousIngressLimit
+ }
+ })
+
+ it('reports a grants-mutation edge-backstop denial on the rate metric and log', async () => {
+ const previousOperationLimit = config.externalGfsOperationRlPerMin
+ config.externalGfsOperationRlPerMin = 2
+ const warn = vi.spyOn(rootLogger, 'warn').mockImplementation(() => {})
+ try {
+ auth()
+ // The caller holds manage_acl + read on R, so each delegation succeeds.
+ dbReturning([
+ {
+ subject_type: 'user',
+ subject_id: U1,
+ resource_id: R,
+ permissions: ['manage_acl', 'read'],
+ inherit: false,
+ },
+ ])
+ const app = await buildApp()
+ const grant = { resourceId: R, subject: { type: 'user', id: U2 }, permissions: ['read'] }
+ const labels = {
+ operation_class: 'grants-mutation',
+ route: '/external/gfs/grants',
+ authority_resolution_avoided: 'false',
+ }
+ const allBefore = await edgeBackstopDenials()
+ const labelledBefore = await edgeBackstopDenials(labels)
+
+ for (let attempt = 0; attempt < 2; attempt += 1) {
+ await request(app)
+ .put('/external/gfs/grants')
+ .set('x-user-session-token', 'sess')
+ .send(grant)
+ .expect(200)
+ }
+ // Liveness witness for the silence below: the guard ran and allowed twice.
+ expect(warn).not.toHaveBeenCalled()
+ expect(await edgeBackstopDenials()).toBe(allBefore)
+
+ const exhausted = await request(app)
+ .put('/external/gfs/grants')
+ .set('x-user-session-token', 'sess')
+ .send(grant)
+
+ expect(exhausted.status).toBe(429)
+ const retryAfterSeconds = exhausted.body.retryAfterSeconds as number
+ expect(retryAfterSeconds).toBeGreaterThanOrEqual(1)
+ expect(warn).toHaveBeenCalledTimes(1)
+ expect(warn.mock.calls[0]).toEqual([
+ {
+ event: 'external_gfs_rate_limit',
+ phase: 'edge-backstop',
+ guard: 'grants-mutation',
+ operationClass: 'grants-mutation',
+ route: '/external/gfs/grants',
+ hashedKey: sha256(`external-gfs:grants-mutation:actor:${sha256(`user-session:${U1}`)}`),
+ retryAfterSeconds,
+ outcome: 'denied',
+ authorityResolutionAvoided: false,
+ },
+ 'external GFS edge backstop denied',
+ ])
+ expect(JSON.stringify(warn.mock.calls)).not.toContain(U1)
+ expect((await edgeBackstopDenials(labels)) - labelledBefore).toBe(1)
+ expect((await edgeBackstopDenials()) - allBefore).toBe(1)
+ } finally {
+ warn.mockRestore()
+ config.externalGfsOperationRlPerMin = previousOperationLimit
+ }
+ })
+
it('mints a gfs token for the user with the EXISTING signGfsToken (sub=users.id)', async () => {
auth()
const app = await buildApp()