Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(control-api): join backstop and Postgres denials on one key and bound the backstop log
Alfredo Lopez323636418666
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 211f524b5..aa041be5d 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.604",+ "version": "0.1.605","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.604",+ "version": "0.1.605","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 64d00e152..adeb0d846 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.604",+ "version": "0.1.605","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/config.ts b/control-api/src/config.tsindex dba2b2fba..962126f29 100644--- a/control-api/src/config.ts+++ b/control-api/src/config.ts@@ -1204,6 +1204,11 @@ export const config: Config = {* - The per-actor read budget must not exceed the per-IP all-class bucket in* front of it; otherwise that bucket caps it under a different key and the* configured number is never reached.+ *+ * The per-IP check cannot fire from the environment today: the read ceiling+ * (960) is below the compiled per-IP budget (1200). It guards a change to+ * either compiled value, and test/config.externalGfsBudgets.test.ts pins the+ * relation.*/export function assertExternalGfsBudgetInvariants(budgets: {readPerMin: numberdiff --git a/control-api/src/middleware/externalGfsRateLimit.ts b/control-api/src/middleware/externalGfsRateLimit.tsindex 42e0c458a..0ea758bb1 100644--- a/control-api/src/middleware/externalGfsRateLimit.ts+++ b/control-api/src/middleware/externalGfsRateLimit.ts@@ -145,6 +145,22 @@ function digest(value: string): string {return createHash('sha256').update(value).digest('hex')}+/**+ * The two bucket keys that the express backstops in routes/external/gfs.ts+ * also use as their store keys. One key per actor on both limiters gives a+ * Postgres denial and a backstop denial the same hashedKey in the logs.+ */+export function externalGfsTokenUserBucketKey(desktopUserId: string): string {+ return `gfs-ext:pre:token:user:${desktopUserId}`+}++export function externalGfsResolvedActorBucketKey(+ operationClass: Exclude<ExternalGfsOperationClass, 'token'>,+ authority: Pick<ExternalGfsAuthority, 'kind' | 'tokenSubject'>+): string {+ return `gfs-ext:resolved:${operationClass}:actor:${authority.kind}:${authority.tokenSubject}`+}+function authenticatedSessionDigest(req: Request): string | null {const token = req.header('x-user-session-token')?.trim()return token ? digest(token) : null@@ -255,6 +271,10 @@ function reportDecision(input: {* `guard` is a log field only. The counter's label set is fixed in metrics.ts* and prom-client throws on an undeclared label, which here would turn a 429* into a 500.+ *+ * The counter records every denial. The warn line is written only for the+ * first denial of a key in its window, so a client retrying in a loop costs+ * one log line per minute instead of one per request.*/export function reportEdgeBackstopDenial(input: {req: Request@@ -262,6 +282,7 @@ export function reportEdgeBackstopDenial(input: {key: stringretryAfterSeconds: numberauthorityResolutionAvoided: boolean+ firstDenialInWindow: boolean}): void {// Route-scoped backstops only run on classified routes. The ingress// backstop runs on every /external/gfs path before the route table, so a@@ -277,6 +298,7 @@ export function reportEdgeBackstopDenial(input: {phase: 'edge-backstop',authority_resolution_avoided: input.authorityResolutionAvoided ? 'true' : 'false',})+ if (!input.firstDenialInWindow) returnrootLogger.warn({event: 'external_gfs_rate_limit',@@ -356,7 +378,7 @@ export function externalGfsPreResolutionRateLimit(operation.operationClass === 'token'? [{- key: `gfs-ext:pre:token:user:${desktopUserId}`,+ key: externalGfsTokenUserBucketKey(desktopUserId),maxPerMinute: config.externalGfsTokenUserRlPerMin,},{@@ -405,7 +427,7 @@ export function externalGfsResolvedOperationRateLimit(return}const bucket: Bucket = {- key: `gfs-ext:resolved:${operation.operationClass}:actor:${authority.kind}:${authority.tokenSubject}`,+ key: externalGfsResolvedActorBucketKey(operation.operationClass, authority),maxPerMinute: operationLimit(operation.operationClass),}if (diff --git a/control-api/src/routes/external/gfs.ts b/control-api/src/routes/external/gfs.tsindex d0825b19e..71169bca4 100644--- a/control-api/src/routes/external/gfs.ts+++ b/control-api/src/routes/external/gfs.ts@@ -1,5 +1,5 @@import { type Response as ExpressResponse, type NextFunction, type Request, Router } from 'express'-import { ipKeyGenerator, rateLimit } from 'express-rate-limit'+import { type RateLimitInfo, ipKeyGenerator, rateLimit } from 'express-rate-limit'import { createHash, randomUUID } from 'node:crypto'import {GFS_DELETE_SCOPE,@@ -27,9 +27,12 @@ import {} from '../../gfs/tree.js'import { asyncHandler } from '../../http/asyncHandler.js'import {+ type ExternalGfsOperationClass,externalGfsPreResolutionRateLimit,+ externalGfsResolvedActorBucketKey,externalGfsResolvedOperationRateLimit,externalGfsSourceIp,+ externalGfsTokenUserBucketKey,reportEdgeBackstopDenial,} from '../../middleware/externalGfsRateLimit.js'import {@@ -129,24 +132,33 @@ function externalGfsRateKey(value: string): string {* the public quota headers. These guards are deliberately direct* express-rate-limit middleware so CodeQL can prove every external GFS route* is bounded before auth, authority resolution, or route-specific work.+ *+ * The token and actor guards count under the same key as their Postgres+ * bucket, so both limiters log one hashedKey for one actor. The ingress guard+ * has no Postgres twin: it masks IPv6 to a /56 with ipKeyGenerator, while the+ * Postgres per-IP buckets key on the full address.*/function externalGfsIngressRateKey(req: import('express').Request): string {return `external-gfs:ingress:${externalGfsRateKey(ipKeyGenerator(externalGfsSourceIp(req)))}`}+// Mounted after requireValidExternalSessionToken and the pre-resolution gate,+// which answers 401 without a user; a missing user here is a routing defect.function externalGfsTokenRateKey(req: import('express').Request): string {- const userId = (req as ExternalGfsRequest).externalAuth?.userId ?? '__missing_user__'- return `external-gfs:token:user:${externalGfsRateKey(userId)}`+ const userId = (req as ExternalGfsRequest).externalAuth?.userId+ if (!userId) throw new Error('external GFS token backstop ran without an authenticated user')+ return externalGfsTokenUserBucketKey(userId)}-function externalGfsActorRateKey(operationClass: string) {+// Mounted after attachExternalGfsAuthority and the resolved-operation gate,+// which answers 401 without an authority; a missing one is a routing defect.+function externalGfsActorRateKey(operationClass: Exclude<ExternalGfsOperationClass, 'token'>) {return (req: import('express').Request): string => {- const externalReq = req as ExternalGfsRequest- const authority = externalReq.gfsAuthority- const actor = authority- ? `${authority.kind}:${authority.tokenSubject}`- : `session:${externalReq.externalAuth?.userId ?? '__missing_user__'}`- return `external-gfs:${operationClass}:actor:${externalGfsRateKey(actor)}`+ const authority = (req as ExternalGfsRequest).gfsAuthority+ if (!authority) {+ throw new Error(`external GFS ${operationClass} backstop ran without a resolved authority`)+ }+ return externalGfsResolvedActorBucketKey(operationClass, authority)}}@@ -347,29 +359,33 @@ export function createExternalGfsRouter(): Router {// can stay under the limit in each clock window and still exceed this one,// so a backstop can be the first, and sometimes the only, denier for such a// burst. That is why its denials report here rather than via reportDecision.+ //+ // express-rate-limit sets req.rateLimit before it calls the handler, with+ // the key it counted and the window's reset time. `used` includes denied+ // requests, so `used === limit + 1` is the first denial in the window.const edgeRateLimitHandler =- (guard: ExternalGfsEdgeGuard, keyOf: (req: Request) => string) =>+ (guard: ExternalGfsEdgeGuard) =>(req: Request, res: ExpressResponse): void => {- const raw = res.getHeader('Retry-After')- const retryAfterSeconds =- typeof raw === 'number' && Number.isSafeInteger(raw) && raw > 0- ? raw- : typeof raw === 'string' && /^\d+$/.test(raw)- ? Math.max(1, Number(raw))- : 60+ const info = (req as Request & { rateLimit?: RateLimitInfo }).rateLimit+ if (info === undefined || info.resetTime === undefined) {+ throw new Error(`external GFS ${guard} backstop denied without rate limit state`)+ }+ const retryAfterSeconds = Math.max(+ 1,+ Math.ceil((info.resetTime.getTime() - Date.now()) / 1000)+ )reportEdgeBackstopDenial({req,guard,- key: keyOf(req),+ key: info.key,retryAfterSeconds,// Only the ingress and token backstops are mounted before// attachExternalGfsAuthority; the other seven run after it.authorityResolutionAvoided: guard === 'ingress' || guard === 'token',+ firstDenialInWindow: info.used === info.limit + 1,})res.status(429).json({ error: 'Too Many Requests', retryAfterSeconds })}- // One key-generator instance per guard, shared by keyGenerator and the- // handler, so the logged key digest is the key the store actually counted.const resourceRateKey = externalGfsActorRateKey('resource')const proxyReadRateKey = externalGfsActorRateKey('proxy-read')const resourceMutationRateKey = externalGfsActorRateKey('resource-mutation')@@ -383,7 +399,7 @@ export function createExternalGfsRouter(): Router {standardHeaders: 'draft-7',legacyHeaders: false,keyGenerator: externalGfsIngressRateKey,- handler: edgeRateLimitHandler('ingress', externalGfsIngressRateKey),+ handler: edgeRateLimitHandler('ingress'),})const externalGfsTokenRouteRateLimit = rateLimit({windowMs: 60_000,@@ -391,7 +407,7 @@ export function createExternalGfsRouter(): Router {standardHeaders: 'draft-7',legacyHeaders: false,keyGenerator: externalGfsTokenRateKey,- handler: edgeRateLimitHandler('token', externalGfsTokenRateKey),+ handler: edgeRateLimitHandler('token'),})const externalGfsResourceRouteRateLimit = rateLimit({windowMs: 60_000,@@ -399,7 +415,7 @@ export function createExternalGfsRouter(): Router {standardHeaders: 'draft-7',legacyHeaders: false,keyGenerator: resourceRateKey,- handler: edgeRateLimitHandler('resource', resourceRateKey),+ handler: edgeRateLimitHandler('resource'),})const externalGfsProxyReadRouteRateLimit = rateLimit({windowMs: 60_000,@@ -407,7 +423,7 @@ export function createExternalGfsRouter(): Router {standardHeaders: 'draft-7',legacyHeaders: false,keyGenerator: proxyReadRateKey,- handler: edgeRateLimitHandler('proxy-read', proxyReadRateKey),+ handler: edgeRateLimitHandler('proxy-read'),})const externalGfsMutationRouteRateLimit = rateLimit({windowMs: 60_000,@@ -415,7 +431,7 @@ export function createExternalGfsRouter(): Router {standardHeaders: 'draft-7',legacyHeaders: false,keyGenerator: resourceMutationRateKey,- handler: edgeRateLimitHandler('resource-mutation', resourceMutationRateKey),+ handler: edgeRateLimitHandler('resource-mutation'),})const externalGfsGrantsReadRouteRateLimit = rateLimit({windowMs: 60_000,@@ -423,7 +439,7 @@ export function createExternalGfsRouter(): Router {standardHeaders: 'draft-7',legacyHeaders: false,keyGenerator: grantsReadRateKey,- handler: edgeRateLimitHandler('grants-read', grantsReadRateKey),+ handler: edgeRateLimitHandler('grants-read'),})const externalGfsGrantsMutationRouteRateLimit = rateLimit({windowMs: 60_000,@@ -431,7 +447,7 @@ export function createExternalGfsRouter(): Router {standardHeaders: 'draft-7',legacyHeaders: false,keyGenerator: grantsMutationRateKey,- handler: edgeRateLimitHandler('grants-mutation', grantsMutationRateKey),+ handler: edgeRateLimitHandler('grants-mutation'),})const externalGfsSharesReadRouteRateLimit = rateLimit({windowMs: 60_000,@@ -439,7 +455,7 @@ export function createExternalGfsRouter(): Router {standardHeaders: 'draft-7',legacyHeaders: false,keyGenerator: sharesReadRateKey,- handler: edgeRateLimitHandler('shares-read', sharesReadRateKey),+ handler: edgeRateLimitHandler('shares-read'),})const externalGfsSharesMutationRouteRateLimit = rateLimit({windowMs: 60_000,@@ -447,7 +463,7 @@ export function createExternalGfsRouter(): Router {standardHeaders: 'draft-7',legacyHeaders: false,keyGenerator: sharesMutationRateKey,- handler: edgeRateLimitHandler('shares-mutation', sharesMutationRateKey),+ handler: edgeRateLimitHandler('shares-mutation'),})// The source-IP guard intentionally precedes authentication and authoritydiff --git a/control-api/test/config.externalGfsBudgets.test.ts b/control-api/test/config.externalGfsBudgets.test.tsindex d3d68ca4d..2ed8f1dd9 100644--- a/control-api/test/config.externalGfsBudgets.test.ts+++ b/control-api/test/config.externalGfsBudgets.test.ts@@ -125,6 +125,24 @@ describe('external GFS actor budgets', () => {)})+ it('L9: the largest read budget the environment can set stays within the per-IP bucket', async () => {+ // The per-IP half of assertExternalGfsBudgetInvariants cannot fire from the+ // environment today: the read ceiling is below the compiled per-IP budget.+ // This pins that relation, so lowering the per-IP budget or raising the+ // read ceiling has to be a decision, not a boot failure found in a deploy.+ const { config } = await loadConfigModuleWith({+ CONTROL_API_EXTERNAL_GFS_READ_RL_PER_MIN: '960',+ })+ expect(config.externalGfsReadRlPerMin).toBe(960)+ expect(config.externalGfsReadRlPerMin).toBeLessThanOrEqual(config.externalGfsIpRlPerMin)+ // 960 is the ceiling: one more is refused by the parser, before the invariant.+ await expect(+ loadConfigModuleWith({ CONTROL_API_EXTERNAL_GFS_READ_RL_PER_MIN: '961' })+ ).rejects.toThrow(+ 'CONTROL_API_EXTERNAL_GFS_READ_RL_PER_MIN must be an integer between 1 and 960'+ )+ })+it('refuses a read budget above the per-IP budget and accepts it at the boundary', async () => {const { assertExternalGfsBudgetInvariants } = await loadConfigModuleWith({})expect(() =>diff --git a/control-api/test/middleware.externalGfsRateLimit.test.ts b/control-api/test/middleware.externalGfsRateLimit.test.tsindex c6d8a81a8..eb89e9a50 100644--- a/control-api/test/middleware.externalGfsRateLimit.test.ts+++ b/control-api/test/middleware.externalGfsRateLimit.test.ts@@ -464,7 +464,7 @@ describe('external GFS edge backstop denial report', () => {})it('reports a route backstop denial with the fixed label set and a hashed key', () => {- const rawKey = `external-gfs:resource:actor:${DESKTOP_USER_ID}`+ const rawKey = `gfs-ext:resolved:resource:actor:user-session:${DESKTOP_USER_ID}`reportEdgeBackstopDenial({req: backstopRequest('GET', '/resources'),@@ -472,6 +472,7 @@ describe('external GFS edge backstop denial report', () => {key: rawKey,retryAfterSeconds: 7,authorityResolutionAvoided: false,+ firstDenialInWindow: true,})// Literal comparison, not objectContaining: an extra label must fail here,@@ -516,6 +517,7 @@ describe('external GFS edge backstop denial report', () => {key: 'external-gfs:ingress:203.0.113.9',retryAfterSeconds: 3,authorityResolutionAvoided: true,+ firstDenialInWindow: true,})expect(metrics.externalGfsRateLimitRequestsTotal.inc.mock.calls).toEqual([@@ -538,4 +540,19 @@ describe('external GFS edge backstop denial report', () => {})expect(JSON.stringify(logger.rootLogger.warn.mock.calls)).not.toContain('203.0.113.9')})++ it('counts a repeated denial in the same window without logging it again', () => {+ reportEdgeBackstopDenial({+ req: backstopRequest('GET', '/resources'),+ guard: 'resource',+ key: `gfs-ext:resolved:resource:actor:user-session:${DESKTOP_USER_ID}`,+ retryAfterSeconds: 7,+ authorityResolutionAvoided: false,+ firstDenialInWindow: false,+ })++ // Liveness witness for the missing log line: the denial was counted.+ expect(metrics.externalGfsRateLimitRequestsTotal.inc).toHaveBeenCalledTimes(1)+ expect(logger.rootLogger.warn).not.toHaveBeenCalled()+ })})diff --git a/control-api/test/routes.externalGfs.test.ts b/control-api/test/routes.externalGfs.test.tsindex 99b579951..d7828df48 100644--- a/control-api/test/routes.externalGfs.test.ts+++ b/control-api/test/routes.externalGfs.test.ts@@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'import express from 'express'import { ipKeyGenerator } from 'express-rate-limit'import { createHash } from 'node:crypto'+import { readFileSync } from 'node:fs'import request from 'supertest'// The MOCKED config object (defined below); mutating gfscProxyTimeoutMs drives a// real deadline abort in the read-proxy timeout test.@@ -239,6 +240,8 @@ function dbReturning(callerAgentNames?: string[]grantListRows?: Record<string, unknown>[]rateLimitCount?: number+ /** Per-bucket count, keyed by the bucket key; wins over rateLimitCount. */+ rateLimitCountFor?: (bucketKey: string) => number} = {}) {// The external-plane eligibility guard resolves the caller's own agents via@@ -252,7 +255,10 @@ function dbReturning(return { rows: callerAgentNames.map(name => ({ agent_name: name })) }}if (text.includes('rate_limit_buckets')) {- return { rows: [{ count: opts.rateLimitCount ?? 1 }] }+ const count = opts.rateLimitCountFor+ ? opts.rateLimitCountFor(String(values?.[0]))+ : (opts.rateLimitCount ?? 1)+ return { rows: [{ count }] }}if (text.includes('ORDER BY created_at ASC, id ASC')) {return { rows: opts.grantListRows ?? [] }@@ -461,7 +467,7 @@ describe('POST /external/gfs/token (user mint — existing signer, sub=users.id)guard: 'resource',operationClass: 'resource',route: '/external/gfs/resources',- hashedKey: sha256(`external-gfs:resource:actor:${sha256(`user-session:${U1}`)}`),+ hashedKey: sha256(`gfs-ext:resolved:resource:actor:user-session:${U1}`),retryAfterSeconds,outcome: 'denied',authorityResolutionAvoided: false,@@ -589,7 +595,7 @@ describe('POST /external/gfs/token (user mint — existing signer, sub=users.id)guard: 'grants-mutation',operationClass: 'grants-mutation',route: '/external/gfs/grants',- hashedKey: sha256(`external-gfs:grants-mutation:actor:${sha256(`user-session:${U1}`)}`),+ hashedKey: sha256(`gfs-ext:resolved:grants-mutation:actor:user-session:${U1}`),retryAfterSeconds,outcome: 'denied',authorityResolutionAvoided: false,@@ -605,6 +611,295 @@ describe('POST /external/gfs/token (user mint — existing signer, sub=users.id)}})+ it('L5: a Postgres denial and a backstop denial for one actor log the same hashedKey', async () => {+ const previousReadLimit = config.externalGfsReadRlPerMin+ config.externalGfsReadRlPerMin = 2+ const warn = vi.spyOn(rootLogger, 'warn').mockImplementation(() => {})+ try {+ auth()+ let denyResolvedBucket = true+ dbReturning([], {+ rateLimitCountFor: key =>+ denyResolvedBucket && key.startsWith('gfs-ext:resolved:') ? 3 : 1,+ })+ const app = await buildApp()+ const read = () =>+ request(app).get('/external/gfs/resources').set('x-user-session-token', 'read-session')++ // The resolved Postgres bucket runs before the route backstop, so this+ // denial leaves the backstop's counter untouched.+ expect((await read()).status).toBe(429)+ denyResolvedBucket = false+ await read().expect(200)+ await read().expect(200)+ expect((await read()).status).toBe(429)++ const fields = warn.mock.calls.map(+ ([logged]) => logged as { phase: string; hashedKey: string }+ )+ const postgres = fields.filter(logged => logged.phase === 'resolved-operation')+ const backstop = fields.filter(logged => logged.phase === 'edge-backstop')+ expect(postgres).toHaveLength(1)+ expect(backstop).toHaveLength(1)+ expect(backstop[0].hashedKey).toBe(postgres[0].hashedKey)+ expect(postgres[0].hashedKey).toBe(+ sha256(`gfs-ext:resolved:resource:actor:user-session:${U1}`)+ )+ } finally {+ warn.mockRestore()+ config.externalGfsReadRlPerMin = previousReadLimit+ }+ })++ it('L10: logs the first backstop denial per key and window, and counts every denial', async () => {+ const previousReadLimit = config.externalGfsReadRlPerMin+ config.externalGfsReadRlPerMin = 2+ const warn = vi.spyOn(rootLogger, 'warn').mockImplementation(() => {})+ try {+ auth()+ dbReturning([])+ const app = await buildApp()+ const labels = {+ operation_class: 'resource',+ route: '/external/gfs/resources',+ authority_resolution_avoided: 'false',+ }+ const before = await edgeBackstopDenials(labels)+ const read = () =>+ request(app).get('/external/gfs/resources').set('x-user-session-token', 'read-session')++ await read().expect(200)+ await read().expect(200)+ const statuses: number[] = []+ for (let attempt = 0; attempt < 50; attempt += 1) statuses.push((await read()).status)++ expect(statuses).toEqual(Array.from({ length: 50 }, () => 429))+ expect((await edgeBackstopDenials(labels)) - before).toBe(50)+ // Liveness witness: the one line that is logged is the backstop's.+ expect(warn).toHaveBeenCalledTimes(1)+ expect(warn.mock.calls[0]).toEqual([+ expect.objectContaining({ phase: 'edge-backstop', guard: 'resource' }),+ 'external GFS edge backstop denied',+ ])+ } finally {+ warn.mockRestore()+ config.externalGfsReadRlPerMin = previousReadLimit+ }+ })++ it('L22: reports the exact seconds left in the backstop window', async () => {+ const previousReadLimit = config.externalGfsReadRlPerMin+ config.externalGfsReadRlPerMin = 1+ const warn = vi.spyOn(rootLogger, 'warn').mockImplementation(() => {})+ const windowOpenedAt = Date.now()+ vi.useFakeTimers({ toFake: ['Date'], now: windowOpenedAt })+ try {+ auth()+ dbReturning([])+ const app = await buildApp()+ const read = () =>+ request(app).get('/external/gfs/resources').set('x-user-session-token', 'read-session')++ await read().expect(200)+ // 42.6 s remain in the 60 s window, so the client is told 43.+ vi.setSystemTime(windowOpenedAt + 17_400)+ const exhausted = await read()++ expect(exhausted.status).toBe(429)+ expect(exhausted.body).toEqual({ error: 'Too Many Requests', retryAfterSeconds: 43 })+ expect(warn).toHaveBeenCalledTimes(1)+ expect(warn.mock.calls[0]?.[0]).toEqual(expect.objectContaining({ retryAfterSeconds: 43 }))+ } finally {+ vi.useRealTimers()+ warn.mockRestore()+ config.externalGfsReadRlPerMin = previousReadLimit+ }+ })++ type BudgetKey =+ | 'externalGfsIngressRlPerMin'+ | 'externalGfsTokenUserRlPerMin'+ | 'externalGfsReadRlPerMin'+ | 'externalGfsOperationRlPerMin'+ const SOURCE_IP = '203.0.113.9'+ const actorKey = (operationClass: string) =>+ sha256(`gfs-ext:resolved:${operationClass}:actor:user-session:${U1}`)+ const grantBody = { resourceId: R, subject: { type: 'user', id: U2 }, permissions: ['read'] }+ const shareBody = { resourceId: R, permissions: ['read'] }+ const edgeGuards: Array<{+ guard: string+ budget: BudgetKey+ method: 'get' | 'post' | 'put' | 'patch'+ path: string+ body?: Record<string, unknown>+ operationClass: string+ route: string+ authorityResolutionAvoided: boolean+ hashedKey: string+ }> = [+ {+ guard: 'ingress',+ budget: 'externalGfsIngressRlPerMin',+ method: 'get',+ path: '/external/gfs/not-classified',+ operationClass: 'unclassified',+ route: 'unclassified',+ authorityResolutionAvoided: true,+ hashedKey: sha256(`external-gfs:ingress:${sha256(ipKeyGenerator(SOURCE_IP))}`),+ },+ {+ guard: 'token',+ budget: 'externalGfsTokenUserRlPerMin',+ method: 'post',+ path: '/external/gfs/token',+ body: {},+ operationClass: 'token',+ route: '/external/gfs/token',+ authorityResolutionAvoided: true,+ hashedKey: sha256(`gfs-ext:pre:token:user:${U1}`),+ },+ {+ guard: 'resource',+ budget: 'externalGfsReadRlPerMin',+ method: 'get',+ path: '/external/gfs/resources',+ operationClass: 'resource',+ route: '/external/gfs/resources',+ authorityResolutionAvoided: false,+ hashedKey: actorKey('resource'),+ },+ {+ guard: 'proxy-read',+ budget: 'externalGfsReadRlPerMin',+ method: 'get',+ path: `/external/gfs/proxy/${R}`,+ operationClass: 'proxy-read',+ route: '/external/gfs/proxy/:rid',+ authorityResolutionAvoided: false,+ hashedKey: actorKey('proxy-read'),+ },+ {+ guard: 'resource-mutation',+ budget: 'externalGfsOperationRlPerMin',+ method: 'patch',+ path: `/external/gfs/resources/${R}`,+ body: { name: 'rename' },+ operationClass: 'resource-mutation',+ route: '/external/gfs/resources/:id',+ authorityResolutionAvoided: false,+ hashedKey: actorKey('resource-mutation'),+ },+ {+ guard: 'grants-read',+ budget: 'externalGfsReadRlPerMin',+ method: 'get',+ path: `/external/gfs/grants?resourceId=${R}`,+ operationClass: 'grants-read',+ route: '/external/gfs/grants',+ authorityResolutionAvoided: false,+ hashedKey: actorKey('grants-read'),+ },+ {+ guard: 'grants-mutation',+ budget: 'externalGfsOperationRlPerMin',+ method: 'put',+ path: '/external/gfs/grants',+ body: grantBody,+ operationClass: 'grants-mutation',+ route: '/external/gfs/grants',+ authorityResolutionAvoided: false,+ hashedKey: actorKey('grants-mutation'),+ },+ {+ guard: 'shares-read',+ budget: 'externalGfsReadRlPerMin',+ method: 'get',+ path: `/external/gfs/shares?resourceId=${R}`,+ operationClass: 'shares-read',+ route: '/external/gfs/shares',+ authorityResolutionAvoided: false,+ hashedKey: actorKey('shares-read'),+ },+ {+ guard: 'shares-mutation',+ budget: 'externalGfsOperationRlPerMin',+ method: 'post',+ path: '/external/gfs/shares',+ body: shareBody,+ operationClass: 'shares-mutation',+ route: '/external/gfs/shares',+ authorityResolutionAvoided: false,+ hashedKey: actorKey('shares-mutation'),+ },+ ]++ it('L11: the table covers every edge guard the router mounts', async () => {+ const source = readFileSync(new URL('../src/routes/external/gfs.ts', import.meta.url), 'utf-8')+ const mounted = [...source.matchAll(/edgeRateLimitHandler\('([a-z-]+)'/g)].map(m => m[1])+ expect(mounted).toHaveLength(9)+ expect([...mounted].sort()).toEqual(edgeGuards.map(row => row.guard).sort())+ })++ it.each(edgeGuards)(+ 'L11: the $guard backstop denies at its limit and reports once on the metric and log',+ async row => {+ const previousLimit = config[row.budget]+ config[row.budget] = 1+ const warn = vi.spyOn(rootLogger, 'warn').mockImplementation(() => {})+ // A handler that reaches gfsc gets an inert answer; the status of the+ // allowed request is irrelevant here, only that it is not a 429.+ vi.stubGlobal(+ 'fetch',+ vi.fn(async () => new Response('{}', { status: 200 }))+ )+ try {+ auth()+ dbReturning([])+ const app = await buildApp()+ const labels = {+ operation_class: row.operationClass,+ route: row.route,+ authority_resolution_avoided: String(row.authorityResolutionAvoided),+ }+ const before = await edgeBackstopDenials(labels)+ const send = () => {+ const pending = request(app)+ [row.method](row.path)+ .set('x-user-session-token', 'sess')+ .set('x-forwarded-for', SOURCE_IP)+ return row.body === undefined ? pending : pending.send(row.body)+ }++ const allowed = await send()+ expect(allowed.status).not.toBe(429)+ // Liveness witness for the silence: the guard admitted one request.+ expect(await edgeBackstopDenials(labels)).toBe(before)++ const denied = await send()+ expect(denied.status).toBe(429)+ expect((await edgeBackstopDenials(labels)) - before).toBe(1)+ expect(warn).toHaveBeenCalledTimes(1)+ expect(warn.mock.calls[0]).toEqual([+ {+ event: 'external_gfs_rate_limit',+ phase: 'edge-backstop',+ guard: row.guard,+ operationClass: row.operationClass,+ route: row.route,+ hashedKey: row.hashedKey,+ retryAfterSeconds: denied.body.retryAfterSeconds,+ outcome: 'denied',+ authorityResolutionAvoided: row.authorityResolutionAvoided,+ },+ 'external GFS edge backstop denied',+ ])+ } finally {+ warn.mockRestore()+ config[row.budget] = previousLimit+ }+ }+ )+it('mints a gfs token for the user with the EXISTING signGfsToken (sub=users.id)', async () => {auth()const app = await buildApp()diff --git a/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts b/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.tsindex 7a33930c4..785d1fce0 100644--- a/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts+++ b/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts@@ -539,6 +539,13 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postg'resolved-operation': 0,'edge-backstop': 0,})- for (const run of [a, b, c]) expect(run.logged).toEqual(run.metricDelta)+ for (const run of [a, c]) expect(run.logged).toEqual(run.metricDelta)+ // The backstop logs once per key per window and counts every denial: B's+ // 8 denials share one actor key and one backstop window, so one line.+ expect(b.logged).toEqual({+ 'pre-resolution': 0,+ 'resolved-operation': 0,+ 'edge-backstop': 1,+ })}, 120_000)})