Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): join backstop and Postgres denials on one key and bound the backstop log

Alfredo Lopez323636418666
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 211f524b5..aa041be5d 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.604",
+ "version": "0.1.605",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.604",
+ "version": "0.1.605",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 64d00e152..adeb0d846 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.604",
+ "version": "0.1.605",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/config.ts b/control-api/src/config.ts
index dba2b2fba..962126f29 100644
--- a/control-api/src/config.ts
+++ b/control-api/src/config.ts
@@ -1204,6 +1204,11 @@ export const config: Config = {
* - The per-actor read budget must not exceed the per-IP all-class bucket in
* front of it; otherwise that bucket caps it under a different key and the
* configured number is never reached.
+ *
+ * The per-IP check cannot fire from the environment today: the read ceiling
+ * (960) is below the compiled per-IP budget (1200). It guards a change to
+ * either compiled value, and test/config.externalGfsBudgets.test.ts pins the
+ * relation.
*/
export function assertExternalGfsBudgetInvariants(budgets: {
readPerMin: number
diff --git a/control-api/src/middleware/externalGfsRateLimit.ts b/control-api/src/middleware/externalGfsRateLimit.ts
index 42e0c458a..0ea758bb1 100644
--- a/control-api/src/middleware/externalGfsRateLimit.ts
+++ b/control-api/src/middleware/externalGfsRateLimit.ts
@@ -145,6 +145,22 @@ function digest(value: string): string {
return createHash('sha256').update(value).digest('hex')
}
+/**
+ * The two bucket keys that the express backstops in routes/external/gfs.ts
+ * also use as their store keys. One key per actor on both limiters gives a
+ * Postgres denial and a backstop denial the same hashedKey in the logs.
+ */
+export function externalGfsTokenUserBucketKey(desktopUserId: string): string {
+ return `gfs-ext:pre:token:user:${desktopUserId}`
+}
+
+export function externalGfsResolvedActorBucketKey(
+ operationClass: Exclude<ExternalGfsOperationClass, 'token'>,
+ authority: Pick<ExternalGfsAuthority, 'kind' | 'tokenSubject'>
+): string {
+ return `gfs-ext:resolved:${operationClass}:actor:${authority.kind}:${authority.tokenSubject}`
+}
+
function authenticatedSessionDigest(req: Request): string | null {
const token = req.header('x-user-session-token')?.trim()
return token ? digest(token) : null
@@ -255,6 +271,10 @@ function reportDecision(input: {
* `guard` is a log field only. The counter's label set is fixed in metrics.ts
* and prom-client throws on an undeclared label, which here would turn a 429
* into a 500.
+ *
+ * The counter records every denial. The warn line is written only for the
+ * first denial of a key in its window, so a client retrying in a loop costs
+ * one log line per minute instead of one per request.
*/
export function reportEdgeBackstopDenial(input: {
req: Request
@@ -262,6 +282,7 @@ export function reportEdgeBackstopDenial(input: {
key: string
retryAfterSeconds: number
authorityResolutionAvoided: boolean
+ firstDenialInWindow: boolean
}): void {
// Route-scoped backstops only run on classified routes. The ingress
// backstop runs on every /external/gfs path before the route table, so a
@@ -277,6 +298,7 @@ export function reportEdgeBackstopDenial(input: {
phase: 'edge-backstop',
authority_resolution_avoided: input.authorityResolutionAvoided ? 'true' : 'false',
})
+ if (!input.firstDenialInWindow) return
rootLogger.warn(
{
event: 'external_gfs_rate_limit',
@@ -356,7 +378,7 @@ export function externalGfsPreResolutionRateLimit(
operation.operationClass === 'token'
? [
{
- key: `gfs-ext:pre:token:user:${desktopUserId}`,
+ key: externalGfsTokenUserBucketKey(desktopUserId),
maxPerMinute: config.externalGfsTokenUserRlPerMin,
},
{
@@ -405,7 +427,7 @@ export function externalGfsResolvedOperationRateLimit(
return
}
const bucket: Bucket = {
- key: `gfs-ext:resolved:${operation.operationClass}:actor:${authority.kind}:${authority.tokenSubject}`,
+ key: externalGfsResolvedActorBucketKey(operation.operationClass, authority),
maxPerMinute: operationLimit(operation.operationClass),
}
if (
diff --git a/control-api/src/routes/external/gfs.ts b/control-api/src/routes/external/gfs.ts
index d0825b19e..71169bca4 100644
--- a/control-api/src/routes/external/gfs.ts
+++ b/control-api/src/routes/external/gfs.ts
@@ -1,5 +1,5 @@
import { type Response as ExpressResponse, type NextFunction, type Request, Router } from 'express'
-import { ipKeyGenerator, rateLimit } from 'express-rate-limit'
+import { type RateLimitInfo, ipKeyGenerator, rateLimit } from 'express-rate-limit'
import { createHash, randomUUID } from 'node:crypto'
import {
GFS_DELETE_SCOPE,
@@ -27,9 +27,12 @@ import {
} from '../../gfs/tree.js'
import { asyncHandler } from '../../http/asyncHandler.js'
import {
+ type ExternalGfsOperationClass,
externalGfsPreResolutionRateLimit,
+ externalGfsResolvedActorBucketKey,
externalGfsResolvedOperationRateLimit,
externalGfsSourceIp,
+ externalGfsTokenUserBucketKey,
reportEdgeBackstopDenial,
} from '../../middleware/externalGfsRateLimit.js'
import {
@@ -129,24 +132,33 @@ function externalGfsRateKey(value: string): string {
* the public quota headers. These guards are deliberately direct
* express-rate-limit middleware so CodeQL can prove every external GFS route
* is bounded before auth, authority resolution, or route-specific work.
+ *
+ * The token and actor guards count under the same key as their Postgres
+ * bucket, so both limiters log one hashedKey for one actor. The ingress guard
+ * has no Postgres twin: it masks IPv6 to a /56 with ipKeyGenerator, while the
+ * Postgres per-IP buckets key on the full address.
*/
function externalGfsIngressRateKey(req: import('express').Request): string {
return `external-gfs:ingress:${externalGfsRateKey(ipKeyGenerator(externalGfsSourceIp(req)))}`
}
+// Mounted after requireValidExternalSessionToken and the pre-resolution gate,
+// which answers 401 without a user; a missing user here is a routing defect.
function externalGfsTokenRateKey(req: import('express').Request): string {
- const userId = (req as ExternalGfsRequest).externalAuth?.userId ?? '__missing_user__'
- return `external-gfs:token:user:${externalGfsRateKey(userId)}`
+ const userId = (req as ExternalGfsRequest).externalAuth?.userId
+ if (!userId) throw new Error('external GFS token backstop ran without an authenticated user')
+ return externalGfsTokenUserBucketKey(userId)
}
-function externalGfsActorRateKey(operationClass: string) {
+// Mounted after attachExternalGfsAuthority and the resolved-operation gate,
+// which answers 401 without an authority; a missing one is a routing defect.
+function externalGfsActorRateKey(operationClass: Exclude<ExternalGfsOperationClass, 'token'>) {
return (req: import('express').Request): string => {
- const externalReq = req as ExternalGfsRequest
- const authority = externalReq.gfsAuthority
- const actor = authority
- ? `${authority.kind}:${authority.tokenSubject}`
- : `session:${externalReq.externalAuth?.userId ?? '__missing_user__'}`
- return `external-gfs:${operationClass}:actor:${externalGfsRateKey(actor)}`
+ const authority = (req as ExternalGfsRequest).gfsAuthority
+ if (!authority) {
+ throw new Error(`external GFS ${operationClass} backstop ran without a resolved authority`)
+ }
+ return externalGfsResolvedActorBucketKey(operationClass, authority)
}
}
@@ -347,29 +359,33 @@ export function createExternalGfsRouter(): Router {
// can stay under the limit in each clock window and still exceed this one,
// so a backstop can be the first, and sometimes the only, denier for such a
// burst. That is why its denials report here rather than via reportDecision.
+ //
+ // express-rate-limit sets req.rateLimit before it calls the handler, with
+ // the key it counted and the window's reset time. `used` includes denied
+ // requests, so `used === limit + 1` is the first denial in the window.
const edgeRateLimitHandler =
- (guard: ExternalGfsEdgeGuard, keyOf: (req: Request) => string) =>
+ (guard: ExternalGfsEdgeGuard) =>
(req: Request, res: ExpressResponse): void => {
- const raw = res.getHeader('Retry-After')
- const retryAfterSeconds =
- typeof raw === 'number' && Number.isSafeInteger(raw) && raw > 0
- ? raw
- : typeof raw === 'string' && /^\d+$/.test(raw)
- ? Math.max(1, Number(raw))
- : 60
+ const info = (req as Request & { rateLimit?: RateLimitInfo }).rateLimit
+ if (info === undefined || info.resetTime === undefined) {
+ throw new Error(`external GFS ${guard} backstop denied without rate limit state`)
+ }
+ const retryAfterSeconds = Math.max(
+ 1,
+ Math.ceil((info.resetTime.getTime() - Date.now()) / 1000)
+ )
reportEdgeBackstopDenial({
req,
guard,
- key: keyOf(req),
+ key: info.key,
retryAfterSeconds,
// Only the ingress and token backstops are mounted before
// attachExternalGfsAuthority; the other seven run after it.
authorityResolutionAvoided: guard === 'ingress' || guard === 'token',
+ firstDenialInWindow: info.used === info.limit + 1,
})
res.status(429).json({ error: 'Too Many Requests', retryAfterSeconds })
}
- // One key-generator instance per guard, shared by keyGenerator and the
- // handler, so the logged key digest is the key the store actually counted.
const resourceRateKey = externalGfsActorRateKey('resource')
const proxyReadRateKey = externalGfsActorRateKey('proxy-read')
const resourceMutationRateKey = externalGfsActorRateKey('resource-mutation')
@@ -383,7 +399,7 @@ export function createExternalGfsRouter(): Router {
standardHeaders: 'draft-7',
legacyHeaders: false,
keyGenerator: externalGfsIngressRateKey,
- handler: edgeRateLimitHandler('ingress', externalGfsIngressRateKey),
+ handler: edgeRateLimitHandler('ingress'),
})
const externalGfsTokenRouteRateLimit = rateLimit({
windowMs: 60_000,
@@ -391,7 +407,7 @@ export function createExternalGfsRouter(): Router {
standardHeaders: 'draft-7',
legacyHeaders: false,
keyGenerator: externalGfsTokenRateKey,
- handler: edgeRateLimitHandler('token', externalGfsTokenRateKey),
+ handler: edgeRateLimitHandler('token'),
})
const externalGfsResourceRouteRateLimit = rateLimit({
windowMs: 60_000,
@@ -399,7 +415,7 @@ export function createExternalGfsRouter(): Router {
standardHeaders: 'draft-7',
legacyHeaders: false,
keyGenerator: resourceRateKey,
- handler: edgeRateLimitHandler('resource', resourceRateKey),
+ handler: edgeRateLimitHandler('resource'),
})
const externalGfsProxyReadRouteRateLimit = rateLimit({
windowMs: 60_000,
@@ -407,7 +423,7 @@ export function createExternalGfsRouter(): Router {
standardHeaders: 'draft-7',
legacyHeaders: false,
keyGenerator: proxyReadRateKey,
- handler: edgeRateLimitHandler('proxy-read', proxyReadRateKey),
+ handler: edgeRateLimitHandler('proxy-read'),
})
const externalGfsMutationRouteRateLimit = rateLimit({
windowMs: 60_000,
@@ -415,7 +431,7 @@ export function createExternalGfsRouter(): Router {
standardHeaders: 'draft-7',
legacyHeaders: false,
keyGenerator: resourceMutationRateKey,
- handler: edgeRateLimitHandler('resource-mutation', resourceMutationRateKey),
+ handler: edgeRateLimitHandler('resource-mutation'),
})
const externalGfsGrantsReadRouteRateLimit = rateLimit({
windowMs: 60_000,
@@ -423,7 +439,7 @@ export function createExternalGfsRouter(): Router {
standardHeaders: 'draft-7',
legacyHeaders: false,
keyGenerator: grantsReadRateKey,
- handler: edgeRateLimitHandler('grants-read', grantsReadRateKey),
+ handler: edgeRateLimitHandler('grants-read'),
})
const externalGfsGrantsMutationRouteRateLimit = rateLimit({
windowMs: 60_000,
@@ -431,7 +447,7 @@ export function createExternalGfsRouter(): Router {
standardHeaders: 'draft-7',
legacyHeaders: false,
keyGenerator: grantsMutationRateKey,
- handler: edgeRateLimitHandler('grants-mutation', grantsMutationRateKey),
+ handler: edgeRateLimitHandler('grants-mutation'),
})
const externalGfsSharesReadRouteRateLimit = rateLimit({
windowMs: 60_000,
@@ -439,7 +455,7 @@ export function createExternalGfsRouter(): Router {
standardHeaders: 'draft-7',
legacyHeaders: false,
keyGenerator: sharesReadRateKey,
- handler: edgeRateLimitHandler('shares-read', sharesReadRateKey),
+ handler: edgeRateLimitHandler('shares-read'),
})
const externalGfsSharesMutationRouteRateLimit = rateLimit({
windowMs: 60_000,
@@ -447,7 +463,7 @@ export function createExternalGfsRouter(): Router {
standardHeaders: 'draft-7',
legacyHeaders: false,
keyGenerator: sharesMutationRateKey,
- handler: edgeRateLimitHandler('shares-mutation', sharesMutationRateKey),
+ handler: edgeRateLimitHandler('shares-mutation'),
})
// The source-IP guard intentionally precedes authentication and authority
diff --git a/control-api/test/config.externalGfsBudgets.test.ts b/control-api/test/config.externalGfsBudgets.test.ts
index d3d68ca4d..2ed8f1dd9 100644
--- a/control-api/test/config.externalGfsBudgets.test.ts
+++ b/control-api/test/config.externalGfsBudgets.test.ts
@@ -125,6 +125,24 @@ describe('external GFS actor budgets', () => {
)
})
+ it('L9: the largest read budget the environment can set stays within the per-IP bucket', async () => {
+ // The per-IP half of assertExternalGfsBudgetInvariants cannot fire from the
+ // environment today: the read ceiling is below the compiled per-IP budget.
+ // This pins that relation, so lowering the per-IP budget or raising the
+ // read ceiling has to be a decision, not a boot failure found in a deploy.
+ const { config } = await loadConfigModuleWith({
+ CONTROL_API_EXTERNAL_GFS_READ_RL_PER_MIN: '960',
+ })
+ expect(config.externalGfsReadRlPerMin).toBe(960)
+ expect(config.externalGfsReadRlPerMin).toBeLessThanOrEqual(config.externalGfsIpRlPerMin)
+ // 960 is the ceiling: one more is refused by the parser, before the invariant.
+ await expect(
+ loadConfigModuleWith({ CONTROL_API_EXTERNAL_GFS_READ_RL_PER_MIN: '961' })
+ ).rejects.toThrow(
+ 'CONTROL_API_EXTERNAL_GFS_READ_RL_PER_MIN must be an integer between 1 and 960'
+ )
+ })
+
it('refuses a read budget above the per-IP budget and accepts it at the boundary', async () => {
const { assertExternalGfsBudgetInvariants } = await loadConfigModuleWith({})
expect(() =>
diff --git a/control-api/test/middleware.externalGfsRateLimit.test.ts b/control-api/test/middleware.externalGfsRateLimit.test.ts
index c6d8a81a8..eb89e9a50 100644
--- a/control-api/test/middleware.externalGfsRateLimit.test.ts
+++ b/control-api/test/middleware.externalGfsRateLimit.test.ts
@@ -464,7 +464,7 @@ describe('external GFS edge backstop denial report', () => {
})
it('reports a route backstop denial with the fixed label set and a hashed key', () => {
- const rawKey = `external-gfs:resource:actor:${DESKTOP_USER_ID}`
+ const rawKey = `gfs-ext:resolved:resource:actor:user-session:${DESKTOP_USER_ID}`
reportEdgeBackstopDenial({
req: backstopRequest('GET', '/resources'),
@@ -472,6 +472,7 @@ describe('external GFS edge backstop denial report', () => {
key: rawKey,
retryAfterSeconds: 7,
authorityResolutionAvoided: false,
+ firstDenialInWindow: true,
})
// Literal comparison, not objectContaining: an extra label must fail here,
@@ -516,6 +517,7 @@ describe('external GFS edge backstop denial report', () => {
key: 'external-gfs:ingress:203.0.113.9',
retryAfterSeconds: 3,
authorityResolutionAvoided: true,
+ firstDenialInWindow: true,
})
expect(metrics.externalGfsRateLimitRequestsTotal.inc.mock.calls).toEqual([
@@ -538,4 +540,19 @@ describe('external GFS edge backstop denial report', () => {
})
expect(JSON.stringify(logger.rootLogger.warn.mock.calls)).not.toContain('203.0.113.9')
})
+
+ it('counts a repeated denial in the same window without logging it again', () => {
+ reportEdgeBackstopDenial({
+ req: backstopRequest('GET', '/resources'),
+ guard: 'resource',
+ key: `gfs-ext:resolved:resource:actor:user-session:${DESKTOP_USER_ID}`,
+ retryAfterSeconds: 7,
+ authorityResolutionAvoided: false,
+ firstDenialInWindow: false,
+ })
+
+ // Liveness witness for the missing log line: the denial was counted.
+ expect(metrics.externalGfsRateLimitRequestsTotal.inc).toHaveBeenCalledTimes(1)
+ expect(logger.rootLogger.warn).not.toHaveBeenCalled()
+ })
})
diff --git a/control-api/test/routes.externalGfs.test.ts b/control-api/test/routes.externalGfs.test.ts
index 99b579951..d7828df48 100644
--- a/control-api/test/routes.externalGfs.test.ts
+++ b/control-api/test/routes.externalGfs.test.ts
@@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import express from 'express'
import { ipKeyGenerator } from 'express-rate-limit'
import { createHash } from 'node:crypto'
+import { readFileSync } from 'node:fs'
import request from 'supertest'
// The MOCKED config object (defined below); mutating gfscProxyTimeoutMs drives a
// real deadline abort in the read-proxy timeout test.
@@ -239,6 +240,8 @@ function dbReturning(
callerAgentNames?: string[]
grantListRows?: Record<string, unknown>[]
rateLimitCount?: number
+ /** Per-bucket count, keyed by the bucket key; wins over rateLimitCount. */
+ rateLimitCountFor?: (bucketKey: string) => number
} = {}
) {
// The external-plane eligibility guard resolves the caller's own agents via
@@ -252,7 +255,10 @@ function dbReturning(
return { rows: callerAgentNames.map(name => ({ agent_name: name })) }
}
if (text.includes('rate_limit_buckets')) {
- return { rows: [{ count: opts.rateLimitCount ?? 1 }] }
+ const count = opts.rateLimitCountFor
+ ? opts.rateLimitCountFor(String(values?.[0]))
+ : (opts.rateLimitCount ?? 1)
+ return { rows: [{ count }] }
}
if (text.includes('ORDER BY created_at ASC, id ASC')) {
return { rows: opts.grantListRows ?? [] }
@@ -461,7 +467,7 @@ describe('POST /external/gfs/token (user mint — existing signer, sub=users.id)
guard: 'resource',
operationClass: 'resource',
route: '/external/gfs/resources',
- hashedKey: sha256(`external-gfs:resource:actor:${sha256(`user-session:${U1}`)}`),
+ hashedKey: sha256(`gfs-ext:resolved:resource:actor:user-session:${U1}`),
retryAfterSeconds,
outcome: 'denied',
authorityResolutionAvoided: false,
@@ -589,7 +595,7 @@ describe('POST /external/gfs/token (user mint — existing signer, sub=users.id)
guard: 'grants-mutation',
operationClass: 'grants-mutation',
route: '/external/gfs/grants',
- hashedKey: sha256(`external-gfs:grants-mutation:actor:${sha256(`user-session:${U1}`)}`),
+ hashedKey: sha256(`gfs-ext:resolved:grants-mutation:actor:user-session:${U1}`),
retryAfterSeconds,
outcome: 'denied',
authorityResolutionAvoided: false,
@@ -605,6 +611,295 @@ describe('POST /external/gfs/token (user mint — existing signer, sub=users.id)
}
})
+ it('L5: a Postgres denial and a backstop denial for one actor log the same hashedKey', async () => {
+ const previousReadLimit = config.externalGfsReadRlPerMin
+ config.externalGfsReadRlPerMin = 2
+ const warn = vi.spyOn(rootLogger, 'warn').mockImplementation(() => {})
+ try {
+ auth()
+ let denyResolvedBucket = true
+ dbReturning([], {
+ rateLimitCountFor: key =>
+ denyResolvedBucket && key.startsWith('gfs-ext:resolved:') ? 3 : 1,
+ })
+ const app = await buildApp()
+ const read = () =>
+ request(app).get('/external/gfs/resources').set('x-user-session-token', 'read-session')
+
+ // The resolved Postgres bucket runs before the route backstop, so this
+ // denial leaves the backstop's counter untouched.
+ expect((await read()).status).toBe(429)
+ denyResolvedBucket = false
+ await read().expect(200)
+ await read().expect(200)
+ expect((await read()).status).toBe(429)
+
+ const fields = warn.mock.calls.map(
+ ([logged]) => logged as { phase: string; hashedKey: string }
+ )
+ const postgres = fields.filter(logged => logged.phase === 'resolved-operation')
+ const backstop = fields.filter(logged => logged.phase === 'edge-backstop')
+ expect(postgres).toHaveLength(1)
+ expect(backstop).toHaveLength(1)
+ expect(backstop[0].hashedKey).toBe(postgres[0].hashedKey)
+ expect(postgres[0].hashedKey).toBe(
+ sha256(`gfs-ext:resolved:resource:actor:user-session:${U1}`)
+ )
+ } finally {
+ warn.mockRestore()
+ config.externalGfsReadRlPerMin = previousReadLimit
+ }
+ })
+
+ it('L10: logs the first backstop denial per key and window, and counts every denial', async () => {
+ const previousReadLimit = config.externalGfsReadRlPerMin
+ config.externalGfsReadRlPerMin = 2
+ const warn = vi.spyOn(rootLogger, 'warn').mockImplementation(() => {})
+ try {
+ auth()
+ dbReturning([])
+ const app = await buildApp()
+ const labels = {
+ operation_class: 'resource',
+ route: '/external/gfs/resources',
+ authority_resolution_avoided: 'false',
+ }
+ const before = await edgeBackstopDenials(labels)
+ const read = () =>
+ request(app).get('/external/gfs/resources').set('x-user-session-token', 'read-session')
+
+ await read().expect(200)
+ await read().expect(200)
+ const statuses: number[] = []
+ for (let attempt = 0; attempt < 50; attempt += 1) statuses.push((await read()).status)
+
+ expect(statuses).toEqual(Array.from({ length: 50 }, () => 429))
+ expect((await edgeBackstopDenials(labels)) - before).toBe(50)
+ // Liveness witness: the one line that is logged is the backstop's.
+ expect(warn).toHaveBeenCalledTimes(1)
+ expect(warn.mock.calls[0]).toEqual([
+ expect.objectContaining({ phase: 'edge-backstop', guard: 'resource' }),
+ 'external GFS edge backstop denied',
+ ])
+ } finally {
+ warn.mockRestore()
+ config.externalGfsReadRlPerMin = previousReadLimit
+ }
+ })
+
+ it('L22: reports the exact seconds left in the backstop window', async () => {
+ const previousReadLimit = config.externalGfsReadRlPerMin
+ config.externalGfsReadRlPerMin = 1
+ const warn = vi.spyOn(rootLogger, 'warn').mockImplementation(() => {})
+ const windowOpenedAt = Date.now()
+ vi.useFakeTimers({ toFake: ['Date'], now: windowOpenedAt })
+ try {
+ auth()
+ dbReturning([])
+ const app = await buildApp()
+ const read = () =>
+ request(app).get('/external/gfs/resources').set('x-user-session-token', 'read-session')
+
+ await read().expect(200)
+ // 42.6 s remain in the 60 s window, so the client is told 43.
+ vi.setSystemTime(windowOpenedAt + 17_400)
+ const exhausted = await read()
+
+ expect(exhausted.status).toBe(429)
+ expect(exhausted.body).toEqual({ error: 'Too Many Requests', retryAfterSeconds: 43 })
+ expect(warn).toHaveBeenCalledTimes(1)
+ expect(warn.mock.calls[0]?.[0]).toEqual(expect.objectContaining({ retryAfterSeconds: 43 }))
+ } finally {
+ vi.useRealTimers()
+ warn.mockRestore()
+ config.externalGfsReadRlPerMin = previousReadLimit
+ }
+ })
+
+ type BudgetKey =
+ | 'externalGfsIngressRlPerMin'
+ | 'externalGfsTokenUserRlPerMin'
+ | 'externalGfsReadRlPerMin'
+ | 'externalGfsOperationRlPerMin'
+ const SOURCE_IP = '203.0.113.9'
+ const actorKey = (operationClass: string) =>
+ sha256(`gfs-ext:resolved:${operationClass}:actor:user-session:${U1}`)
+ const grantBody = { resourceId: R, subject: { type: 'user', id: U2 }, permissions: ['read'] }
+ const shareBody = { resourceId: R, permissions: ['read'] }
+ const edgeGuards: Array<{
+ guard: string
+ budget: BudgetKey
+ method: 'get' | 'post' | 'put' | 'patch'
+ path: string
+ body?: Record<string, unknown>
+ operationClass: string
+ route: string
+ authorityResolutionAvoided: boolean
+ hashedKey: string
+ }> = [
+ {
+ guard: 'ingress',
+ budget: 'externalGfsIngressRlPerMin',
+ method: 'get',
+ path: '/external/gfs/not-classified',
+ operationClass: 'unclassified',
+ route: 'unclassified',
+ authorityResolutionAvoided: true,
+ hashedKey: sha256(`external-gfs:ingress:${sha256(ipKeyGenerator(SOURCE_IP))}`),
+ },
+ {
+ guard: 'token',
+ budget: 'externalGfsTokenUserRlPerMin',
+ method: 'post',
+ path: '/external/gfs/token',
+ body: {},
+ operationClass: 'token',
+ route: '/external/gfs/token',
+ authorityResolutionAvoided: true,
+ hashedKey: sha256(`gfs-ext:pre:token:user:${U1}`),
+ },
+ {
+ guard: 'resource',
+ budget: 'externalGfsReadRlPerMin',
+ method: 'get',
+ path: '/external/gfs/resources',
+ operationClass: 'resource',
+ route: '/external/gfs/resources',
+ authorityResolutionAvoided: false,
+ hashedKey: actorKey('resource'),
+ },
+ {
+ guard: 'proxy-read',
+ budget: 'externalGfsReadRlPerMin',
+ method: 'get',
+ path: `/external/gfs/proxy/${R}`,
+ operationClass: 'proxy-read',
+ route: '/external/gfs/proxy/:rid',
+ authorityResolutionAvoided: false,
+ hashedKey: actorKey('proxy-read'),
+ },
+ {
+ guard: 'resource-mutation',
+ budget: 'externalGfsOperationRlPerMin',
+ method: 'patch',
+ path: `/external/gfs/resources/${R}`,
+ body: { name: 'rename' },
+ operationClass: 'resource-mutation',
+ route: '/external/gfs/resources/:id',
+ authorityResolutionAvoided: false,
+ hashedKey: actorKey('resource-mutation'),
+ },
+ {
+ guard: 'grants-read',
+ budget: 'externalGfsReadRlPerMin',
+ method: 'get',
+ path: `/external/gfs/grants?resourceId=${R}`,
+ operationClass: 'grants-read',
+ route: '/external/gfs/grants',
+ authorityResolutionAvoided: false,
+ hashedKey: actorKey('grants-read'),
+ },
+ {
+ guard: 'grants-mutation',
+ budget: 'externalGfsOperationRlPerMin',
+ method: 'put',
+ path: '/external/gfs/grants',
+ body: grantBody,
+ operationClass: 'grants-mutation',
+ route: '/external/gfs/grants',
+ authorityResolutionAvoided: false,
+ hashedKey: actorKey('grants-mutation'),
+ },
+ {
+ guard: 'shares-read',
+ budget: 'externalGfsReadRlPerMin',
+ method: 'get',
+ path: `/external/gfs/shares?resourceId=${R}`,
+ operationClass: 'shares-read',
+ route: '/external/gfs/shares',
+ authorityResolutionAvoided: false,
+ hashedKey: actorKey('shares-read'),
+ },
+ {
+ guard: 'shares-mutation',
+ budget: 'externalGfsOperationRlPerMin',
+ method: 'post',
+ path: '/external/gfs/shares',
+ body: shareBody,
+ operationClass: 'shares-mutation',
+ route: '/external/gfs/shares',
+ authorityResolutionAvoided: false,
+ hashedKey: actorKey('shares-mutation'),
+ },
+ ]
+
+ it('L11: the table covers every edge guard the router mounts', async () => {
+ const source = readFileSync(new URL('../src/routes/external/gfs.ts', import.meta.url), 'utf-8')
+ const mounted = [...source.matchAll(/edgeRateLimitHandler\('([a-z-]+)'/g)].map(m => m[1])
+ expect(mounted).toHaveLength(9)
+ expect([...mounted].sort()).toEqual(edgeGuards.map(row => row.guard).sort())
+ })
+
+ it.each(edgeGuards)(
+ 'L11: the $guard backstop denies at its limit and reports once on the metric and log',
+ async row => {
+ const previousLimit = config[row.budget]
+ config[row.budget] = 1
+ const warn = vi.spyOn(rootLogger, 'warn').mockImplementation(() => {})
+ // A handler that reaches gfsc gets an inert answer; the status of the
+ // allowed request is irrelevant here, only that it is not a 429.
+ vi.stubGlobal(
+ 'fetch',
+ vi.fn(async () => new Response('{}', { status: 200 }))
+ )
+ try {
+ auth()
+ dbReturning([])
+ const app = await buildApp()
+ const labels = {
+ operation_class: row.operationClass,
+ route: row.route,
+ authority_resolution_avoided: String(row.authorityResolutionAvoided),
+ }
+ const before = await edgeBackstopDenials(labels)
+ const send = () => {
+ const pending = request(app)
+ [row.method](row.path)
+ .set('x-user-session-token', 'sess')
+ .set('x-forwarded-for', SOURCE_IP)
+ return row.body === undefined ? pending : pending.send(row.body)
+ }
+
+ const allowed = await send()
+ expect(allowed.status).not.toBe(429)
+ // Liveness witness for the silence: the guard admitted one request.
+ expect(await edgeBackstopDenials(labels)).toBe(before)
+
+ const denied = await send()
+ expect(denied.status).toBe(429)
+ expect((await edgeBackstopDenials(labels)) - before).toBe(1)
+ expect(warn).toHaveBeenCalledTimes(1)
+ expect(warn.mock.calls[0]).toEqual([
+ {
+ event: 'external_gfs_rate_limit',
+ phase: 'edge-backstop',
+ guard: row.guard,
+ operationClass: row.operationClass,
+ route: row.route,
+ hashedKey: row.hashedKey,
+ retryAfterSeconds: denied.body.retryAfterSeconds,
+ outcome: 'denied',
+ authorityResolutionAvoided: row.authorityResolutionAvoided,
+ },
+ 'external GFS edge backstop denied',
+ ])
+ } finally {
+ warn.mockRestore()
+ config[row.budget] = previousLimit
+ }
+ }
+ )
+
it('mints a gfs token for the user with the EXISTING signGfsToken (sub=users.id)', async () => {
auth()
const app = await buildApp()
diff --git a/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts b/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts
index 7a33930c4..785d1fce0 100644
--- a/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts
+++ b/control-api/test/routes.externalGfsRateLimitStress.realPostgres.integration.test.ts
@@ -539,6 +539,13 @@ describeRealPostgres('external GFS rate limits under concurrent load (real Postg
'resolved-operation': 0,
'edge-backstop': 0,
})
- for (const run of [a, b, c]) expect(run.logged).toEqual(run.metricDelta)
+ for (const run of [a, c]) expect(run.logged).toEqual(run.metricDelta)
+ // The backstop logs once per key per window and counts every denial: B's
+ // 8 denials share one actor key and one backstop window, so one line.
+ expect(b.logged).toEqual({
+ 'pre-resolution': 0,
+ 'resolved-operation': 0,
+ 'edge-backstop': 1,
+ })
}, 120_000)
})