Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(control-api): name each external GFS edge backstop and report its denials
Alfredo Lopezef2f63578589
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex f64de5231..716800e60 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.600",+ "version": "0.1.601","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.600",+ "version": "0.1.601","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 20033d94b..a4c6f4544 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.600",+ "version": "0.1.601","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/routes/external/gfs.ts b/control-api/src/routes/external/gfs.tsindex 9e6f33cab..d0825b19e 100644--- a/control-api/src/routes/external/gfs.ts+++ b/control-api/src/routes/external/gfs.ts@@ -30,6 +30,7 @@ import {externalGfsPreResolutionRateLimit,externalGfsResolvedOperationRateLimit,externalGfsSourceIp,+ reportEdgeBackstopDenial,} from '../../middleware/externalGfsRateLimit.js'import {type ExternalAuthedRequest,@@ -106,6 +107,18 @@ function attachExternalGfsRequestId(next()}+/** Name of each in-process edge backstop, reported as the `guard` log field. */+type ExternalGfsEdgeGuard =+ | 'ingress'+ | 'token'+ | 'resource'+ | 'proxy-read'+ | 'resource-mutation'+ | 'grants-read'+ | 'grants-mutation'+ | 'shares-read'+ | 'shares-mutation'+function externalGfsRateKey(value: string): string {return createHash('sha256').update(value).digest('hex')}@@ -327,23 +340,50 @@ export function createExternalGfsRouter(): Router {// distributed buckets retain the narrower security budgets. Edge rejects// still expose a bounded Retry-After body/header so Desktop can recover// without treating the backstop as an opaque 500.- const edgeRateLimitHandler = (_req: Request, res: ExpressResponse): void => {- const raw = res.getHeader('Retry-After')- const retryAfterSeconds =- typeof raw === 'number' && Number.isSafeInteger(raw) && raw > 0- ? raw- : typeof raw === 'string' && /^\d+$/.test(raw)- ? Math.max(1, Number(raw))- : 60- res.status(429).json({ error: 'Too Many Requests', retryAfterSeconds })- }+ //+ // The express-rate-limit MemoryStore window starts at the first hit for a+ // key, while the Postgres limiters in externalGfsRateLimit.ts use windows+ // aligned to floor(now / 60_000). A burst that straddles a minute boundary+ // can stay under the limit in each clock window and still exceed this one,+ // so a backstop can be the first, and sometimes the only, denier for such a+ // burst. That is why its denials report here rather than via reportDecision.+ const edgeRateLimitHandler =+ (guard: ExternalGfsEdgeGuard, keyOf: (req: Request) => string) =>+ (req: Request, res: ExpressResponse): void => {+ const raw = res.getHeader('Retry-After')+ const retryAfterSeconds =+ typeof raw === 'number' && Number.isSafeInteger(raw) && raw > 0+ ? raw+ : typeof raw === 'string' && /^\d+$/.test(raw)+ ? Math.max(1, Number(raw))+ : 60+ reportEdgeBackstopDenial({+ req,+ guard,+ key: keyOf(req),+ retryAfterSeconds,+ // Only the ingress and token backstops are mounted before+ // attachExternalGfsAuthority; the other seven run after it.+ authorityResolutionAvoided: guard === 'ingress' || guard === 'token',+ })+ res.status(429).json({ error: 'Too Many Requests', retryAfterSeconds })+ }+ // One key-generator instance per guard, shared by keyGenerator and the+ // handler, so the logged key digest is the key the store actually counted.+ const resourceRateKey = externalGfsActorRateKey('resource')+ const proxyReadRateKey = externalGfsActorRateKey('proxy-read')+ const resourceMutationRateKey = externalGfsActorRateKey('resource-mutation')+ const grantsReadRateKey = externalGfsActorRateKey('grants-read')+ const grantsMutationRateKey = externalGfsActorRateKey('grants-mutation')+ const sharesReadRateKey = externalGfsActorRateKey('shares-read')+ const sharesMutationRateKey = externalGfsActorRateKey('shares-mutation')const externalGfsIngressRateLimit = rateLimit({windowMs: 60_000,limit: config.externalGfsIngressRlPerMin,standardHeaders: 'draft-7',legacyHeaders: false,keyGenerator: externalGfsIngressRateKey,- handler: edgeRateLimitHandler,+ handler: edgeRateLimitHandler('ingress', externalGfsIngressRateKey),})const externalGfsTokenRouteRateLimit = rateLimit({windowMs: 60_000,@@ -351,63 +391,63 @@ export function createExternalGfsRouter(): Router {standardHeaders: 'draft-7',legacyHeaders: false,keyGenerator: externalGfsTokenRateKey,- handler: edgeRateLimitHandler,+ handler: edgeRateLimitHandler('token', externalGfsTokenRateKey),})const externalGfsResourceRouteRateLimit = rateLimit({windowMs: 60_000,limit: config.externalGfsReadRlPerMin,standardHeaders: 'draft-7',legacyHeaders: false,- keyGenerator: externalGfsActorRateKey('resource'),- handler: edgeRateLimitHandler,+ keyGenerator: resourceRateKey,+ handler: edgeRateLimitHandler('resource', resourceRateKey),})const externalGfsProxyReadRouteRateLimit = rateLimit({windowMs: 60_000,limit: config.externalGfsReadRlPerMin,standardHeaders: 'draft-7',legacyHeaders: false,- keyGenerator: externalGfsActorRateKey('proxy-read'),- handler: edgeRateLimitHandler,+ keyGenerator: proxyReadRateKey,+ handler: edgeRateLimitHandler('proxy-read', proxyReadRateKey),})const externalGfsMutationRouteRateLimit = rateLimit({windowMs: 60_000,limit: config.externalGfsOperationRlPerMin,standardHeaders: 'draft-7',legacyHeaders: false,- keyGenerator: externalGfsActorRateKey('resource-mutation'),- handler: edgeRateLimitHandler,+ keyGenerator: resourceMutationRateKey,+ handler: edgeRateLimitHandler('resource-mutation', resourceMutationRateKey),})const externalGfsGrantsReadRouteRateLimit = rateLimit({windowMs: 60_000,limit: config.externalGfsReadRlPerMin,standardHeaders: 'draft-7',legacyHeaders: false,- keyGenerator: externalGfsActorRateKey('grants-read'),- handler: edgeRateLimitHandler,+ keyGenerator: grantsReadRateKey,+ handler: edgeRateLimitHandler('grants-read', grantsReadRateKey),})const externalGfsGrantsMutationRouteRateLimit = rateLimit({windowMs: 60_000,limit: config.externalGfsOperationRlPerMin,standardHeaders: 'draft-7',legacyHeaders: false,- keyGenerator: externalGfsActorRateKey('grants-mutation'),- handler: edgeRateLimitHandler,+ keyGenerator: grantsMutationRateKey,+ handler: edgeRateLimitHandler('grants-mutation', grantsMutationRateKey),})const externalGfsSharesReadRouteRateLimit = rateLimit({windowMs: 60_000,limit: config.externalGfsReadRlPerMin,standardHeaders: 'draft-7',legacyHeaders: false,- keyGenerator: externalGfsActorRateKey('shares-read'),- handler: edgeRateLimitHandler,+ keyGenerator: sharesReadRateKey,+ handler: edgeRateLimitHandler('shares-read', sharesReadRateKey),})const externalGfsSharesMutationRouteRateLimit = rateLimit({windowMs: 60_000,limit: config.externalGfsOperationRlPerMin,standardHeaders: 'draft-7',legacyHeaders: false,- keyGenerator: externalGfsActorRateKey('shares-mutation'),- handler: edgeRateLimitHandler,+ keyGenerator: sharesMutationRateKey,+ handler: edgeRateLimitHandler('shares-mutation', sharesMutationRateKey),})// The source-IP guard intentionally precedes authentication and authoritydiff --git a/control-api/test/routes.externalGfs.test.ts b/control-api/test/routes.externalGfs.test.tsindex 5dabe1865..99b579951 100644--- a/control-api/test/routes.externalGfs.test.ts+++ b/control-api/test/routes.externalGfs.test.ts@@ -1,9 +1,15 @@import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'import express from 'express'+import { ipKeyGenerator } from 'express-rate-limit'+import { createHash } from 'node:crypto'import request from 'supertest'// The MOCKED config object (defined below); mutating gfscProxyTimeoutMs drives a// real deadline abort in the read-proxy timeout test.import { config } from '../src/config.js'+// Real (unmocked) modules: the route imports these same instances, so a spy or+// a counter read here observes exactly what the edge backstops emit.+import { rootLogger } from '../src/observability/logger.js'+import { externalGfsRateLimitRequestsTotal } from '../src/observability/metrics.js'/*** Unit tests for the end-user gfs surface on the /external (Session-JWT) plane.@@ -274,6 +280,28 @@ function dbReturning(})}+const sha256 = (value: string): string => createHash('sha256').update(value).digest('hex')++/**+ * Sum of the edge-backstop denial counter, narrowed to `labels` when given.+ * The registry is process-wide, so callers compare a before/after delta.+ */+async function edgeBackstopDenials(+ labels: Partial<Record<'operation_class' | 'route' | 'authority_resolution_avoided', string>> = {}+): Promise<number> {+ const { values } = await externalGfsRateLimitRequestsTotal.get()+ return values+ .filter(+ sample =>+ sample.labels.phase === 'edge-backstop' &&+ sample.labels.outcome === 'denied' &&+ (Object.keys(labels) as Array<keyof typeof labels>).every(+ name => sample.labels[name] === labels[name]+ )+ )+ .reduce((sum, sample) => sum + sample.value, 0)+}+describe('POST /external/gfs/token (user mint — existing signer, sub=users.id)', () => {it('allows a mixed GFS journey beyond the former 30/min ingress cap', async () => {auth()@@ -386,6 +414,197 @@ describe('POST /external/gfs/token (user mint — existing signer, sub=users.id)}})+ // The three edge-backstop cases below mock every Postgres bucket as allowing+ // (dbReturning → count 1), so each 429 can only come from an express backstop.+ it('reports a resource edge-backstop denial on the rate metric and log', async () => {+ const previousReadLimit = config.externalGfsReadRlPerMin+ config.externalGfsReadRlPerMin = 2+ const warn = vi.spyOn(rootLogger, 'warn').mockImplementation(() => {})+ try {+ auth()+ dbReturning([])+ const app = await buildApp()+ const labels = {+ operation_class: 'resource',+ route: '/external/gfs/resources',+ authority_resolution_avoided: 'false',+ }+ const allBefore = await edgeBackstopDenials()+ const labelledBefore = await edgeBackstopDenials(labels)++ for (let attempt = 0; attempt < 2; attempt += 1) {+ await request(app)+ .get('/external/gfs/resources')+ .set('x-user-session-token', 'read-session')+ .expect(200)+ }+ // Liveness witness for the silence below: the guard ran and allowed twice.+ expect(warn).not.toHaveBeenCalled()+ expect(await edgeBackstopDenials()).toBe(allBefore)++ const exhausted = await request(app)+ .get('/external/gfs/resources')+ .set('x-user-session-token', 'read-session')++ expect(exhausted.status).toBe(429)+ expect(exhausted.body).toEqual({+ error: 'Too Many Requests',+ retryAfterSeconds: expect.any(Number),+ })+ const retryAfterSeconds = exhausted.body.retryAfterSeconds as number+ expect(retryAfterSeconds).toBeGreaterThanOrEqual(1)+ expect(warn).toHaveBeenCalledTimes(1)+ expect(warn.mock.calls[0]).toEqual([+ {+ event: 'external_gfs_rate_limit',+ phase: 'edge-backstop',+ guard: 'resource',+ operationClass: 'resource',+ route: '/external/gfs/resources',+ hashedKey: sha256(`external-gfs:resource:actor:${sha256(`user-session:${U1}`)}`),+ retryAfterSeconds,+ outcome: 'denied',+ authorityResolutionAvoided: false,+ },+ 'external GFS edge backstop denied',+ ])+ expect(JSON.stringify(warn.mock.calls)).not.toContain(U1)+ expect((await edgeBackstopDenials(labels)) - labelledBefore).toBe(1)+ expect((await edgeBackstopDenials()) - allBefore).toBe(1)+ } finally {+ warn.mockRestore()+ config.externalGfsReadRlPerMin = previousReadLimit+ }+ })++ it('reports an ingress edge-backstop denial on a path outside the operation matrix', async () => {+ const previousIngressLimit = config.externalGfsIngressRlPerMin+ config.externalGfsIngressRlPerMin = 1+ const warn = vi.spyOn(rootLogger, 'warn').mockImplementation(() => {})+ const sourceIp = '203.0.113.9'+ try {+ auth()+ const app = await buildApp()+ const labels = {+ operation_class: 'unclassified',+ route: 'unclassified',+ authority_resolution_avoided: 'true',+ }+ const allBefore = await edgeBackstopDenials()+ const labelledBefore = await edgeBackstopDenials(labels)++ await request(app)+ .get('/external/gfs/not-classified')+ .set('x-user-session-token', 'sess')+ .set('x-forwarded-for', sourceIp)+ .expect(404)+ // Liveness witness for the silence below: the guard ran and allowed once.+ expect(warn).not.toHaveBeenCalled()+ expect(await edgeBackstopDenials()).toBe(allBefore)++ const exhausted = await request(app)+ .get('/external/gfs/not-classified')+ .set('x-user-session-token', 'sess')+ .set('x-forwarded-for', sourceIp)++ expect(exhausted.status).toBe(429)+ const retryAfterSeconds = exhausted.body.retryAfterSeconds as number+ expect(retryAfterSeconds).toBeGreaterThanOrEqual(1)+ // The ingress guard runs before authentication, so no session was checked.+ expect(mockVerifyExternalSessionToken).toHaveBeenCalledTimes(1)+ expect(warn).toHaveBeenCalledTimes(1)+ expect(warn.mock.calls[0]).toEqual([+ {+ event: 'external_gfs_rate_limit',+ phase: 'edge-backstop',+ guard: 'ingress',+ operationClass: 'unclassified',+ route: 'unclassified',+ hashedKey: sha256(`external-gfs:ingress:${sha256(ipKeyGenerator(sourceIp))}`),+ retryAfterSeconds,+ outcome: 'denied',+ authorityResolutionAvoided: true,+ },+ 'external GFS edge backstop denied',+ ])+ expect(JSON.stringify(warn.mock.calls)).not.toContain(sourceIp)+ expect((await edgeBackstopDenials(labels)) - labelledBefore).toBe(1)+ expect((await edgeBackstopDenials()) - allBefore).toBe(1)+ } finally {+ warn.mockRestore()+ config.externalGfsIngressRlPerMin = previousIngressLimit+ }+ })++ it('reports a grants-mutation edge-backstop denial on the rate metric and log', async () => {+ const previousOperationLimit = config.externalGfsOperationRlPerMin+ config.externalGfsOperationRlPerMin = 2+ const warn = vi.spyOn(rootLogger, 'warn').mockImplementation(() => {})+ try {+ auth()+ // The caller holds manage_acl + read on R, so each delegation succeeds.+ dbReturning([+ {+ subject_type: 'user',+ subject_id: U1,+ resource_id: R,+ permissions: ['manage_acl', 'read'],+ inherit: false,+ },+ ])+ const app = await buildApp()+ const grant = { resourceId: R, subject: { type: 'user', id: U2 }, permissions: ['read'] }+ const labels = {+ operation_class: 'grants-mutation',+ route: '/external/gfs/grants',+ authority_resolution_avoided: 'false',+ }+ const allBefore = await edgeBackstopDenials()+ const labelledBefore = await edgeBackstopDenials(labels)++ for (let attempt = 0; attempt < 2; attempt += 1) {+ await request(app)+ .put('/external/gfs/grants')+ .set('x-user-session-token', 'sess')+ .send(grant)+ .expect(200)+ }+ // Liveness witness for the silence below: the guard ran and allowed twice.+ expect(warn).not.toHaveBeenCalled()+ expect(await edgeBackstopDenials()).toBe(allBefore)++ const exhausted = await request(app)+ .put('/external/gfs/grants')+ .set('x-user-session-token', 'sess')+ .send(grant)++ expect(exhausted.status).toBe(429)+ const retryAfterSeconds = exhausted.body.retryAfterSeconds as number+ expect(retryAfterSeconds).toBeGreaterThanOrEqual(1)+ expect(warn).toHaveBeenCalledTimes(1)+ expect(warn.mock.calls[0]).toEqual([+ {+ event: 'external_gfs_rate_limit',+ phase: 'edge-backstop',+ guard: 'grants-mutation',+ operationClass: 'grants-mutation',+ route: '/external/gfs/grants',+ hashedKey: sha256(`external-gfs:grants-mutation:actor:${sha256(`user-session:${U1}`)}`),+ retryAfterSeconds,+ outcome: 'denied',+ authorityResolutionAvoided: false,+ },+ 'external GFS edge backstop denied',+ ])+ expect(JSON.stringify(warn.mock.calls)).not.toContain(U1)+ expect((await edgeBackstopDenials(labels)) - labelledBefore).toBe(1)+ expect((await edgeBackstopDenials()) - allBefore).toBe(1)+ } finally {+ warn.mockRestore()+ config.externalGfsOperationRlPerMin = previousOperationLimit+ }+ })+it('mints a gfs token for the user with the EXISTING signGfsToken (sub=users.id)', async () => {auth()const app = await buildApp()