Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(control-api): report edge-backstop denials on the external GFS rate metric and log
Alfredo Lopeza1032aa7a0c3
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex cc624b3db..f64de5231 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.599",+ "version": "0.1.600","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.599",+ "version": "0.1.600","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 944361935..20033d94b 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.599",+ "version": "0.1.600","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/middleware/externalGfsRateLimit.ts b/control-api/src/middleware/externalGfsRateLimit.tsindex 94c726acd..42e0c458a 100644--- a/control-api/src/middleware/externalGfsRateLimit.ts+++ b/control-api/src/middleware/externalGfsRateLimit.ts@@ -31,7 +31,7 @@ export type ExternalGfsOperation = {route: string}-type ExternalGfsRateLimitPhase = 'pre-resolution' | 'resolved-operation'+type ExternalGfsRateLimitPhase = 'pre-resolution' | 'resolved-operation' | 'edge-backstop'type Bucket = { key: string; maxPerMinute: number }type ExternalGfsAuthedRequest = Request & {@@ -247,6 +247,52 @@ function reportDecision(input: {}}+/**+ * Report a denial from one of the in-memory express backstops in+ * routes/external/gfs.ts with the event and counter that reportDecision uses+ * for the Postgres buckets, so both denial sources are visible in one place.+ *+ * `guard` is a log field only. The counter's label set is fixed in metrics.ts+ * and prom-client throws on an undeclared label, which here would turn a 429+ * into a 500.+ */+export function reportEdgeBackstopDenial(input: {+ req: Request+ guard: string+ key: string+ retryAfterSeconds: number+ authorityResolutionAvoided: boolean+}): void {+ // Route-scoped backstops only run on classified routes. The ingress+ // backstop runs on every /external/gfs path before the route table, so a+ // path outside the operation matrix has no class and is reported as such.+ const operation = externalGfsOperationFor(input.req)+ const operationClass = operation === null ? 'unclassified' : operation.operationClass+ const route = operation === null ? 'unclassified' : operation.route++ externalGfsRateLimitRequestsTotal.inc({+ operation_class: operationClass,+ route,+ outcome: 'denied',+ phase: 'edge-backstop',+ authority_resolution_avoided: input.authorityResolutionAvoided ? 'true' : 'false',+ })+ rootLogger.warn(+ {+ event: 'external_gfs_rate_limit',+ phase: 'edge-backstop',+ guard: input.guard,+ operationClass,+ route,+ hashedKey: digest(input.key),+ retryAfterSeconds: input.retryAfterSeconds,+ outcome: 'denied',+ authorityResolutionAvoided: input.authorityResolutionAvoided,+ },+ 'external GFS edge backstop denied'+ )+}+async function enforceBuckets(input: {req: Requestres: Responsediff --git a/control-api/test/middleware.externalGfsRateLimit.test.ts b/control-api/test/middleware.externalGfsRateLimit.test.tsindex 2efc419d7..c6d8a81a8 100644--- a/control-api/test/middleware.externalGfsRateLimit.test.ts+++ b/control-api/test/middleware.externalGfsRateLimit.test.ts@@ -1,5 +1,5 @@import { beforeEach, describe, expect, it, vi } from 'vitest'-import express from 'express'+import express, { type Request } from 'express'import { createHash } from 'node:crypto'import request from 'supertest'import {@@ -7,6 +7,7 @@ import {externalGfsPreResolutionRateLimit,externalGfsResolvedOperationRateLimit,externalGfsSourceIp,+ reportEdgeBackstopDenial,} from '../src/middleware/externalGfsRateLimit.js'const checkAndIncrement = vi.hoisted(() => vi.fn())@@ -444,3 +445,97 @@ describe('external GFS rate boundary', () => {expect(handler).not.toHaveBeenCalled()})})++describe('external GFS edge backstop denial report', () => {+ function backstopRequest(method: string, pathWithinGfs: string): Request {+ return {+ method,+ baseUrl: '/external/gfs',+ path: pathWithinGfs,+ originalUrl: `/external/gfs${pathWithinGfs}`,+ } as unknown as Request+ }++ beforeEach(() => {+ metrics.externalGfsRateLimitRequestsTotal.inc.mockReset()+ metrics.externalGfsRateLimitDurationSeconds.observe.mockReset()+ logger.rootLogger.debug.mockReset()+ logger.rootLogger.warn.mockReset()+ })++ it('reports a route backstop denial with the fixed label set and a hashed key', () => {+ const rawKey = `external-gfs:resource:actor:${DESKTOP_USER_ID}`++ reportEdgeBackstopDenial({+ req: backstopRequest('GET', '/resources'),+ guard: 'resource',+ key: rawKey,+ retryAfterSeconds: 7,+ authorityResolutionAvoided: false,+ })++ // Literal comparison, not objectContaining: an extra label must fail here,+ // because prom-client throws on it inside the 429 path.+ expect(metrics.externalGfsRateLimitRequestsTotal.inc.mock.calls).toEqual([+ [+ {+ operation_class: 'resource',+ route: '/external/gfs/resources',+ outcome: 'denied',+ phase: 'edge-backstop',+ authority_resolution_avoided: 'false',+ },+ ],+ ])+ expect(logger.rootLogger.warn.mock.calls).toEqual([+ [+ {+ event: 'external_gfs_rate_limit',+ phase: 'edge-backstop',+ guard: 'resource',+ operationClass: 'resource',+ route: '/external/gfs/resources',+ hashedKey: createHash('sha256').update(rawKey).digest('hex'),+ retryAfterSeconds: 7,+ outcome: 'denied',+ authorityResolutionAvoided: false,+ },+ 'external GFS edge backstop denied',+ ],+ ])+ const payload = logger.rootLogger.warn.mock.calls[0][0] as { hashedKey: string }+ expect(payload.hashedKey).toMatch(/^[0-9a-f]{64}$/)+ expect(JSON.stringify(logger.rootLogger.warn.mock.calls)).not.toContain(DESKTOP_USER_ID)+ expect(metrics.externalGfsRateLimitDurationSeconds.observe).not.toHaveBeenCalled()+ })++ it('reports an ingress denial on an unclassified path as unclassified', () => {+ reportEdgeBackstopDenial({+ req: backstopRequest('GET', '/not-an-external-gfs-route'),+ guard: 'ingress',+ key: 'external-gfs:ingress:203.0.113.9',+ retryAfterSeconds: 3,+ authorityResolutionAvoided: true,+ })++ expect(metrics.externalGfsRateLimitRequestsTotal.inc.mock.calls).toEqual([+ [+ {+ operation_class: 'unclassified',+ route: 'unclassified',+ outcome: 'denied',+ phase: 'edge-backstop',+ authority_resolution_avoided: 'true',+ },+ ],+ ])+ expect(logger.rootLogger.warn).toHaveBeenCalledTimes(1)+ expect(logger.rootLogger.warn.mock.calls[0][0]).toMatchObject({+ guard: 'ingress',+ operationClass: 'unclassified',+ route: 'unclassified',+ retryAfterSeconds: 3,+ })+ expect(JSON.stringify(logger.rootLogger.warn.mock.calls)).not.toContain('203.0.113.9')+ })+})