Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): report edge-backstop denials on the external GFS rate metric and log

Alfredo Lopeza1032aa7a0c3
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index cc624b3db..f64de5231 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.599",
+ "version": "0.1.600",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.599",
+ "version": "0.1.600",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 944361935..20033d94b 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.599",
+ "version": "0.1.600",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/middleware/externalGfsRateLimit.ts b/control-api/src/middleware/externalGfsRateLimit.ts
index 94c726acd..42e0c458a 100644
--- a/control-api/src/middleware/externalGfsRateLimit.ts
+++ b/control-api/src/middleware/externalGfsRateLimit.ts
@@ -31,7 +31,7 @@ export type ExternalGfsOperation = {
route: string
}
-type ExternalGfsRateLimitPhase = 'pre-resolution' | 'resolved-operation'
+type ExternalGfsRateLimitPhase = 'pre-resolution' | 'resolved-operation' | 'edge-backstop'
type Bucket = { key: string; maxPerMinute: number }
type ExternalGfsAuthedRequest = Request & {
@@ -247,6 +247,52 @@ function reportDecision(input: {
}
}
+/**
+ * Report a denial from one of the in-memory express backstops in
+ * routes/external/gfs.ts with the event and counter that reportDecision uses
+ * for the Postgres buckets, so both denial sources are visible in one place.
+ *
+ * `guard` is a log field only. The counter's label set is fixed in metrics.ts
+ * and prom-client throws on an undeclared label, which here would turn a 429
+ * into a 500.
+ */
+export function reportEdgeBackstopDenial(input: {
+ req: Request
+ guard: string
+ key: string
+ retryAfterSeconds: number
+ authorityResolutionAvoided: boolean
+}): void {
+ // Route-scoped backstops only run on classified routes. The ingress
+ // backstop runs on every /external/gfs path before the route table, so a
+ // path outside the operation matrix has no class and is reported as such.
+ const operation = externalGfsOperationFor(input.req)
+ const operationClass = operation === null ? 'unclassified' : operation.operationClass
+ const route = operation === null ? 'unclassified' : operation.route
+
+ externalGfsRateLimitRequestsTotal.inc({
+ operation_class: operationClass,
+ route,
+ outcome: 'denied',
+ phase: 'edge-backstop',
+ authority_resolution_avoided: input.authorityResolutionAvoided ? 'true' : 'false',
+ })
+ rootLogger.warn(
+ {
+ event: 'external_gfs_rate_limit',
+ phase: 'edge-backstop',
+ guard: input.guard,
+ operationClass,
+ route,
+ hashedKey: digest(input.key),
+ retryAfterSeconds: input.retryAfterSeconds,
+ outcome: 'denied',
+ authorityResolutionAvoided: input.authorityResolutionAvoided,
+ },
+ 'external GFS edge backstop denied'
+ )
+}
+
async function enforceBuckets(input: {
req: Request
res: Response
diff --git a/control-api/test/middleware.externalGfsRateLimit.test.ts b/control-api/test/middleware.externalGfsRateLimit.test.ts
index 2efc419d7..c6d8a81a8 100644
--- a/control-api/test/middleware.externalGfsRateLimit.test.ts
+++ b/control-api/test/middleware.externalGfsRateLimit.test.ts
@@ -1,5 +1,5 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
-import express from 'express'
+import express, { type Request } from 'express'
import { createHash } from 'node:crypto'
import request from 'supertest'
import {
@@ -7,6 +7,7 @@ import {
externalGfsPreResolutionRateLimit,
externalGfsResolvedOperationRateLimit,
externalGfsSourceIp,
+ reportEdgeBackstopDenial,
} from '../src/middleware/externalGfsRateLimit.js'
const checkAndIncrement = vi.hoisted(() => vi.fn())
@@ -444,3 +445,97 @@ describe('external GFS rate boundary', () => {
expect(handler).not.toHaveBeenCalled()
})
})
+
+describe('external GFS edge backstop denial report', () => {
+ function backstopRequest(method: string, pathWithinGfs: string): Request {
+ return {
+ method,
+ baseUrl: '/external/gfs',
+ path: pathWithinGfs,
+ originalUrl: `/external/gfs${pathWithinGfs}`,
+ } as unknown as Request
+ }
+
+ beforeEach(() => {
+ metrics.externalGfsRateLimitRequestsTotal.inc.mockReset()
+ metrics.externalGfsRateLimitDurationSeconds.observe.mockReset()
+ logger.rootLogger.debug.mockReset()
+ logger.rootLogger.warn.mockReset()
+ })
+
+ it('reports a route backstop denial with the fixed label set and a hashed key', () => {
+ const rawKey = `external-gfs:resource:actor:${DESKTOP_USER_ID}`
+
+ reportEdgeBackstopDenial({
+ req: backstopRequest('GET', '/resources'),
+ guard: 'resource',
+ key: rawKey,
+ retryAfterSeconds: 7,
+ authorityResolutionAvoided: false,
+ })
+
+ // Literal comparison, not objectContaining: an extra label must fail here,
+ // because prom-client throws on it inside the 429 path.
+ expect(metrics.externalGfsRateLimitRequestsTotal.inc.mock.calls).toEqual([
+ [
+ {
+ operation_class: 'resource',
+ route: '/external/gfs/resources',
+ outcome: 'denied',
+ phase: 'edge-backstop',
+ authority_resolution_avoided: 'false',
+ },
+ ],
+ ])
+ expect(logger.rootLogger.warn.mock.calls).toEqual([
+ [
+ {
+ event: 'external_gfs_rate_limit',
+ phase: 'edge-backstop',
+ guard: 'resource',
+ operationClass: 'resource',
+ route: '/external/gfs/resources',
+ hashedKey: createHash('sha256').update(rawKey).digest('hex'),
+ retryAfterSeconds: 7,
+ outcome: 'denied',
+ authorityResolutionAvoided: false,
+ },
+ 'external GFS edge backstop denied',
+ ],
+ ])
+ const payload = logger.rootLogger.warn.mock.calls[0][0] as { hashedKey: string }
+ expect(payload.hashedKey).toMatch(/^[0-9a-f]{64}$/)
+ expect(JSON.stringify(logger.rootLogger.warn.mock.calls)).not.toContain(DESKTOP_USER_ID)
+ expect(metrics.externalGfsRateLimitDurationSeconds.observe).not.toHaveBeenCalled()
+ })
+
+ it('reports an ingress denial on an unclassified path as unclassified', () => {
+ reportEdgeBackstopDenial({
+ req: backstopRequest('GET', '/not-an-external-gfs-route'),
+ guard: 'ingress',
+ key: 'external-gfs:ingress:203.0.113.9',
+ retryAfterSeconds: 3,
+ authorityResolutionAvoided: true,
+ })
+
+ expect(metrics.externalGfsRateLimitRequestsTotal.inc.mock.calls).toEqual([
+ [
+ {
+ operation_class: 'unclassified',
+ route: 'unclassified',
+ outcome: 'denied',
+ phase: 'edge-backstop',
+ authority_resolution_avoided: 'true',
+ },
+ ],
+ ])
+ expect(logger.rootLogger.warn).toHaveBeenCalledTimes(1)
+ expect(logger.rootLogger.warn.mock.calls[0][0]).toMatchObject({
+ guard: 'ingress',
+ operationClass: 'unclassified',
+ route: 'unclassified',
+ retryAfterSeconds: 3,
+ })
+ expect(JSON.stringify(logger.rootLogger.warn.mock.calls)).not.toContain('203.0.113.9')
+ })
+})