Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(gfs-controller): single-line denial log and shared agent budget wiring
Alfredo Lopez6c5300b0e9bc
diff --git a/gfs-controller/src/api/serve.agentRateLimit.test.ts b/gfs-controller/src/api/serve.agentRateLimit.test.tsindex 015c331ca..9897e690d 100644--- a/gfs-controller/src/api/serve.agentRateLimit.test.ts+++ b/gfs-controller/src/api/serve.agentRateLimit.test.ts@@ -5,7 +5,7 @@ import type { GfsVerifiedClaims } from "../auth/verify";import type { AuthzContext } from "../authz/permissionClient";import { loadConfig, type GfsConfig } from "../config";import { GfsMetrics } from "../metrics";-import { RateLimiter } from "../quota/rateLimit";+import { RateLimiter, buildAgentRateLimits } from "../quota/rateLimit";import { GfsServer } from "../server";import type { GfsResource } from "./read";import { GfsServingHandler, type ServingDeps } from "./serve";@@ -118,6 +118,14 @@ interface Plane {}function plane(limits: { reads: number; writes: number }): Plane {+ return planeWith((now) => ({+ reads: new RateLimiter({ limit: limits.reads, windowMs: WINDOW_MS, now }),+ writes: new RateLimiter({ limit: limits.writes, windowMs: WINDOW_MS, now }),+ }));+}++/** A plane whose limiters come from `build`, on the plane's injected clock. */+function planeWith(build: (now: () => number) => ServingDeps["rateLimit"]): Plane {const clock = { t: 1_000_000 };const store = { allow: true };const authorizeCalls: string[] = [];@@ -138,10 +146,7 @@ function plane(limits: { reads: number; writes: number }): Plane {},} as unknown as ServingDeps["writeService"],metrics,- rateLimit: {- reads: new RateLimiter({ limit: limits.reads, windowMs: WINDOW_MS, now: () => clock.t }),- writes: new RateLimiter({ limit: limits.writes, windowMs: WINDOW_MS, now: () => clock.t }),- },+ rateLimit: build(() => clock.t),};return { deps, handler: new GfsServingHandler(deps), clock, metrics, authorizeCalls, store };}@@ -181,10 +186,10 @@ const statuses = (rs: FakeRes[]): Record<number, number> =>rs.reduce<Record<number, number>>((acc, r) => ({ ...acc, [r.statusCode]: (acc[r.statusCode] ?? 0) + 1 }), {});let warn: ReturnType<typeof vi.spyOn>;-const denialLogs = (): Array<{ kind: string }> =>+const denialLogs = (): string[] =>warn.mock.calls- .map((c: unknown[]) => c[0] as { event?: string; kind: string })- .filter((e: { event?: string }) => e?.event === "gfs_rate_limit_denied");+ .map((c: unknown[]) => c[0])+ .filter((line: unknown): line is string => typeof line === "string" && line.startsWith("[gfsc] rate_limit_denied "));beforeEach(() => {warn = vi.spyOn(console, "warn").mockImplementation(() => undefined);@@ -218,7 +223,9 @@ describe("agent rate limit under concurrent load", () => {}expect(p.authorizeCalls).toHaveLength(AGENTS.length * R);expect(p.metrics.snapshot().rateLimitDenied).toEqual({ read: AGENTS.length * (10 - R), write: 0 });- expect(denialLogs()).toHaveLength(AGENTS.length * (10 - R));+ const logs = denialLogs();+ expect(logs).toHaveLength(AGENTS.length * (10 - R));+ expect(logs.every((line) => /^\[gfsc\] rate_limit_denied kind=read subject=[0-9a-f]{64}$/.test(line))).toBe(true);});it("G2: after the read budget is spent, writes spend their own budget; a following read is still denied", async () => {@@ -300,7 +307,10 @@ describe("agent rate limit through a real GfsServer", () => {const PER_CLIENT = R + 5;const AGENT = "host:1st:mcp-host/agent-wire";- /** The operator's value travels env → loadConfig → limiter, as in index.ts. */+ /**+ * The operator's value travels env → loadConfig → buildAgentRateLimits, the+ * same helper index.ts calls, so a wiring mistake there fails G5.+ */function operatorConfig(): GfsConfig {vi.stubEnv("GFS_STORAGE_ROLE", "reader");vi.stubEnv("GFS_DEV_MODE", "true");@@ -373,7 +383,7 @@ describe("agent rate limit through a real GfsServer", () => {it("G5: two keep-alive clients of one agent share one budget across both sockets", async () => {const config = operatorConfig();expect(config.port).toBe(0);- const p = plane({ reads: config.agentReadRlPerMinPerReplica, writes: config.agentWriteRlPerMinPerReplica });+ const p = planeWith((now) => buildAgentRateLimits(config, now));server = new GfsServer(config,{ isStorageMounted: async () => true, pingPermissionStore: async () => undefined },diff --git a/gfs-controller/src/api/serve.rateLimit.test.ts b/gfs-controller/src/api/serve.rateLimit.test.tsindex 3ffc94608..041f55293 100644--- a/gfs-controller/src/api/serve.rateLimit.test.ts+++ b/gfs-controller/src/api/serve.rateLimit.test.ts@@ -1,5 +1,7 @@import { afterEach, describe, expect, it, vi } from "vitest";+import { createHash } from "node:crypto";import type { IncomingMessage, ServerResponse } from "node:http";+import { format } from "node:util";import { Readable, Writable } from "node:stream";import type { GfsVerifiedClaims } from "../auth/verify";import type { AuthzContext } from "../authz/permissionClient";@@ -11,8 +13,9 @@ import { GfsServingHandler, type ServingDeps } from "./serve";/*** The per-subject agent rate limit (#699): every request from a host principal* spends the read or the write budget right after subject resolution and before- * authorization. User and linked-admin principals are metered by control-api- * and are never charged here.+ * authorization. User and linked-admin principals are never charged here:+ * control-api meters their Desktop requests, and the Control UI operator plane+ * (/api/v1/gfs/proxy) is not metered by either service yet (#762).*/class FakeRes extends Writable {@@ -249,16 +252,18 @@ describe("GfsServingHandler — per-subject agent rate limit", () => {expect(h.metrics.render()).toContain('gfs_rate_limit_denied_total{kind="write"} 0');});- it("case 8: logs a hashed subject and never the raw one", async () => {+ it("case 8: logs one line with a hashed subject and never the raw one", async () => {const h = harness({ writes: 1 });const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined);expect((await h.put(HOST_A)).statusCode).toBe(200);const denied = await h.put(HOST_A);expect(denied.statusCode).toBe(429);expect(warn).toHaveBeenCalledTimes(1);- const [payload] = warn.mock.calls[0] as [Record<string, unknown>];- expect(payload).toMatchObject({ event: "gfs_rate_limit_denied", kind: "write" });- expect(payload.hashedSubject).toMatch(/^[0-9a-f]{64}$/);+ const hashed = createHash("sha256").update(HOST_A).digest("hex");+ expect(warn.mock.calls[0]).toEqual([`[gfsc] rate_limit_denied kind=write subject=${hashed}`]);+ // console.warn renders its arguments with util.format; a log collector+ // splits records on newlines, so the rendered line must hold none.+ expect(format(...(warn.mock.calls[0] as unknown[]))).not.toContain("\n");expect(JSON.stringify(warn.mock.calls[0])).not.toContain(HOST_A);expect(JSON.stringify(warn.mock.calls[0])).not.toContain("agent-a");expect(denied.body).not.toContain(HOST_A);diff --git a/gfs-controller/src/api/serve.ts b/gfs-controller/src/api/serve.tsindex 92bcdd866..eaaf2871e 100644--- a/gfs-controller/src/api/serve.ts+++ b/gfs-controller/src/api/serve.ts@@ -71,8 +71,10 @@ export function sanitizeForLog(value: string): string {** Step 3 runs before authorization, so a flood is bounded by attempts, not by* successes, and a denied agent never reaches the permission store. User and- * linked-admin principals are not charged: they reach gfsc only through- * control-api, which meters every external GFS request before minting.+ * linked-admin principals are not charged here. Desktop requests reach gfsc+ * through control-api's /external/gfs routes, which meter every request before+ * minting. The Control UI operator plane (control-api /api/v1/gfs/proxy) is+ * metered by neither control-api nor gfsc today (#762).** Steps 4 AND 5 must BOTH allow — "neither alone authorizes" (§522). Authz runs* BEFORE any metadata is revealed, so an absent resource and an unauthorized one@@ -587,11 +589,11 @@ export class GfsServingHandler {} catch (err) {if (!(err instanceof RateLimitExceededError)) throw err;this.deps.metrics?.recordRateLimitDenial(kind);- console.warn({- event: "gfs_rate_limit_denied",- kind,- hashedSubject: sanitizeForLog(createHash("sha256").update(ctx.primarySubject).digest("hex")),- });+ // The hash is a correlation id, not anonymization: host subjects are+ // enumerable, so anyone holding the host list can reverse it. Hex output+ // needs no log sanitizing.+ const hashedSubject = createHash("sha256").update(ctx.primarySubject).digest("hex");+ console.warn(`[gfsc] rate_limit_denied kind=${kind} subject=${hashedSubject}`);throw new GfsError("rate_limited", "agent rate limit exceeded", undefined, {retryAfterSeconds: err.retryAfterSeconds,limit: write ? "agent_writes" : "agent_reads",diff --git a/gfs-controller/src/config.test.ts b/gfs-controller/src/config.test.tsindex c39b217fd..f4a3b4527 100644--- a/gfs-controller/src/config.test.ts+++ b/gfs-controller/src/config.test.ts@@ -292,8 +292,25 @@ describe('GFS_AGENT_*_RL_PER_MIN_PER_REPLICA', () => {for (const value of ['60001', '0', '-1', '1.5', 'many']) {vi.stubEnv(name, value)expect(() => loadConfig()).toThrow(- `[gfsc] ${name} must be an integer between 1 and 60000, got: ${value}`+ `[gfsc] ${name} must be an integer between 1 and 60000, got: ${JSON.stringify(value)}`)}})++ it.each([+ 'GFS_AGENT_READ_RL_PER_MIN_PER_REPLICA',+ 'GFS_AGENT_WRITE_RL_PER_MIN_PER_REPLICA',+ ])('L1: %s accepts only a canonical decimal, so a mistyped value fails startup', (name) => {+ for (const value of ['', '0x10', '1e3', ' 45 ', '45.0', '045', '+45']) {+ vi.stubEnv(name, value)+ expect(() => loadConfig(), `value ${JSON.stringify(value)}`).toThrow(+ `[gfsc] ${name} must be an integer between 1 and 60000, got: ${JSON.stringify(value)}`+ )+ }+ // Witness: the same variable still accepts a canonical value.+ vi.stubEnv(name, '45')+ expect(loadConfig()).toMatchObject({+ [name.includes('READ') ? 'agentReadRlPerMinPerReplica' : 'agentWriteRlPerMinPerReplica']: 45,+ })+ })})diff --git a/gfs-controller/src/config.ts b/gfs-controller/src/config.tsindex 85505b6c3..9712d54ff 100644--- a/gfs-controller/src/config.ts+++ b/gfs-controller/src/config.ts@@ -132,17 +132,22 @@ function positiveInteger(name: string, defaultValue: number, maximum: number): n}/**- * Copy admission limits intentionally have no compiled product ceiling. Only- * an absent variable selects the documented default; an explicitly empty or- * non-canonical positive decimal fails startup instead of falling back.+ * Canonical positive decimal, optionally bounded. Only an absent variable+ * selects the documented default; an explicitly empty or non-canonical value+ * (`0x10`, `1e3`, ` 45 `, `45.0`, `045`, `+45`) fails startup instead of+ * falling back. Copy admission limits pass no maximum: they intentionally have+ * no compiled product ceiling.*/-function syncCopyPositiveInteger(name: string, defaultValue: number): number {+function strictPositiveInteger(name: string, defaultValue: number, maximum?: number): number {const raw = process.env[name]if (raw === undefined) return defaultValue- if (!/^[1-9]\d*$/.test(raw)) {- throw new Error(`[gfsc] ${name} must be a positive safe integer, got: ${JSON.stringify(raw)}`)+ const value = /^[1-9]\d*$/.test(raw) ? Number(raw) : Number.NaN+ if (maximum !== undefined) {+ if (!Number.isSafeInteger(value) || value > maximum) {+ throw new Error(`[gfsc] ${name} must be an integer between 1 and ${maximum}, got: ${JSON.stringify(raw)}`)+ }+ return value}- const value = Number(raw)if (!Number.isSafeInteger(value)) {throw new Error(`[gfsc] ${name} must be a positive safe integer, got: ${JSON.stringify(raw)}`)}@@ -399,21 +404,21 @@ export function loadConfig(): GfsConfig {blobCleanupBatchSize: positiveInteger('GFS_BLOB_CLEANUP_BATCH_SIZE', 100, 10000),// The operator sets both explicitly (host-context-controller gfscEnv); the// defaults cover a gfsc image that rolls out before the operator's env.- agentReadRlPerMinPerReplica: positiveInteger(+ agentReadRlPerMinPerReplica: strictPositiveInteger('GFS_AGENT_READ_RL_PER_MIN_PER_REPLICA',AGENT_READ_RL_PER_MIN_DEFAULT,AGENT_RL_PER_MIN_MAX),- agentWriteRlPerMinPerReplica: positiveInteger(+ agentWriteRlPerMinPerReplica: strictPositiveInteger('GFS_AGENT_WRITE_RL_PER_MIN_PER_REPLICA',AGENT_WRITE_RL_PER_MIN_DEFAULT,AGENT_RL_PER_MIN_MAX),- syncCopyMaxObjects: syncCopyPositiveInteger('GFS_SYNC_COPY_MAX_OBJECTS', 1000),- syncCopyMaxBytes: syncCopyPositiveInteger('GFS_SYNC_COPY_MAX_BYTES', 1073741824),- syncCopyTimeoutMs: syncCopyPositiveInteger('GFS_SYNC_COPY_TIMEOUT_MS', 30000),- syncRenameMaxObjects: syncCopyPositiveInteger('GFS_SYNC_RENAME_MAX_OBJECTS', 1000),- syncRenameTimeoutMs: syncCopyPositiveInteger('GFS_SYNC_RENAME_TIMEOUT_MS', 30000),+ syncCopyMaxObjects: strictPositiveInteger('GFS_SYNC_COPY_MAX_OBJECTS', 1000),+ syncCopyMaxBytes: strictPositiveInteger('GFS_SYNC_COPY_MAX_BYTES', 1073741824),+ syncCopyTimeoutMs: strictPositiveInteger('GFS_SYNC_COPY_TIMEOUT_MS', 30000),+ syncRenameMaxObjects: strictPositiveInteger('GFS_SYNC_RENAME_MAX_OBJECTS', 1000),+ syncRenameTimeoutMs: strictPositiveInteger('GFS_SYNC_RENAME_TIMEOUT_MS', 30000),uploadV2: uploadConfig(),devMode,}diff --git a/gfs-controller/src/index.ts b/gfs-controller/src/index.tsindex 5b48ed395..a858ed8ba 100644--- a/gfs-controller/src/index.ts+++ b/gfs-controller/src/index.ts@@ -18,7 +18,7 @@ import { PgResourceStore } from "./db/resourceStore";import { GfsWriteService, PgTransactor } from "./db/writeStore";import { PgBlobStagingStore, reconcileExpiredBlobs } from "./db/blobStaging";import { GfsMetrics } from "./metrics";-import { RateLimiter } from "./quota/rateLimit";+import { buildAgentRateLimits } from "./quota/rateLimit";import { GfsServer, ReadinessDeps } from "./server";import { BlobStore } from "./storage/blobStore";import { GfsUploadSessionService, uploadCapabilities } from "./upload/uploadSession";@@ -156,10 +156,7 @@ async function main(): Promise<void> {// (fail-loud crash); dev mode may run probes-only with a loud warning.const metrics = new GfsMetrics();// Both roles enforce the agent budgets: a reader serves agent reads too.- const rateLimit = {- reads: new RateLimiter({ limit: config.agentReadRlPerMinPerReplica, windowMs: 60_000 }),- writes: new RateLimiter({ limit: config.agentWriteRlPerMinPerReplica, windowMs: 60_000 }),- };+ const rateLimit = buildAgentRateLimits(config);let serving: GfsServingHandler | undefined;let invalidation: { stop: () => Promise<void> } | undefined;let cleanupTimer: NodeJS.Timeout | undefined;diff --git a/gfs-controller/src/quota/quota.test.ts b/gfs-controller/src/quota/quota.test.tsindex e529a9f5e..cdef8847e 100644--- a/gfs-controller/src/quota/quota.test.ts+++ b/gfs-controller/src/quota/quota.test.ts@@ -1,7 +1,7 @@import { describe, expect, it } from "vitest";import { assertByteQuota, QuotaError } from "./bytes.js";import { assertObjectQuota } from "./objects.js";-import { RateLimiter } from "./rateLimit.js";+import { RateLimiter, RateLimitExceededError, buildAgentRateLimits } from "./rateLimit.js";/*** P4-S03 — byte + object quotas and per-subject rate limit. Exceeding any →@@ -63,4 +63,75 @@ describe("RateLimiter", () => {rl.check("a");expect(() => rl.check("b")).not.toThrow();});++ it("L2: never states a Retry-After longer than the window, even after the clock steps back", () => {+ let t = 1_000_000;+ const rl = new RateLimiter({ limit: 1, windowMs: 60_000, now: () => t });+ rl.check("s");+ t -= 300_000;+ let denial: unknown;+ try {+ rl.check("s");+ } catch (err) {+ denial = err;+ }+ expect(denial).toBeInstanceOf(RateLimitExceededError);+ expect((denial as RateLimitExceededError).retryAfterSeconds).toBe(60);+ });++ it("L13: denies and sweeps a window holding 130 000 hits without a RangeError", () => {+ let t = 1_000_000;+ const limit = 130_000;+ const rl = new RateLimiter({ limit, windowMs: 60_000, now: () => t });+ for (let i = 0; i < limit; i += 1) rl.check("s");+ let denial: unknown;+ try {+ rl.check("s");+ } catch (err) {+ denial = err;+ }+ expect(denial).toBeInstanceOf(RateLimitExceededError);+ expect((denial as RateLimitExceededError).retryAfterSeconds).toBe(60);+ t += 60_001;+ expect(() => rl.check("other")).not.toThrow();+ // Witness: the sweep ran and evicted the full subject.+ expect(rl.trackedSubjectCount).toBe(1);+ });++ it("L13: keeps each check constant-time on a full window at the configured maximum", () => {+ // 60 000/min is AGENT_RL_PER_MIN_MAX. One hit per millisecond for three+ // windows keeps the window full: every check from the second window on+ // evicts exactly one hit and records one. A check that rescans the window+ // costs 1.2e5 × 6e4 comparisons here, minutes rather than milliseconds, so+ // the bound separates the two by orders of magnitude on any machine.+ let t = 0;+ const limit = 60_000;+ const rl = new RateLimiter({ limit, windowMs: 60_000, now: () => t });+ const started = performance.now();+ for (t = 1; t <= 3 * limit; t += 1) rl.check("s");+ const elapsedMs = performance.now() - started;+ expect(elapsedMs).toBeLessThan(2_000);+ // Witness: the window is really full, so the loop ran at the limit.+ t -= 1;+ expect(() => rl.check("s")).toThrow(RateLimitExceededError);+ });+});++describe("buildAgentRateLimits", () => {+ it("builds the read and the write limiter from their own budgets on the given clock", () => {+ let t = 1_000_000;+ const limits = buildAgentRateLimits(+ { agentReadRlPerMinPerReplica: 5, agentWriteRlPerMinPerReplica: 2 },+ () => t+ );+ expect(limits.reads.limitPerWindow).toBe(5);+ expect(limits.writes.limitPerWindow).toBe(2);+ expect(limits.reads.windowLengthMs).toBe(60_000);+ expect(limits.writes.windowLengthMs).toBe(60_000);+ limits.writes.check("s");+ limits.writes.check("s");+ expect(() => limits.writes.check("s")).toThrow(RateLimitExceededError);+ t += 60_001;+ expect(() => limits.writes.check("s")).not.toThrow();+ });});diff --git a/gfs-controller/src/quota/rateLimit.ts b/gfs-controller/src/quota/rateLimit.tsindex be9ea6b58..639d2bdf3 100644--- a/gfs-controller/src/quota/rateLimit.ts+++ b/gfs-controller/src/quota/rateLimit.ts@@ -1,3 +1,4 @@+import type { GfsConfig } from "../config.js";import { QuotaError } from "./bytes.js";/**@@ -20,7 +21,7 @@ export class RateLimitExceededError extends QuotaError {* deterministic tests.*/export class RateLimiter {- private readonly hits = new Map<string, number[]>();+ private readonly hits = new Map<string, SubjectWindow>();private readonly limit: number;private readonly windowMs: number;private readonly now: () => number;@@ -52,22 +53,36 @@ export class RateLimiter {/*** Record a hit for `subject`; throw RateLimitExceededError if the window is* full. Retry-after is when the oldest retained hit leaves the window, the- * exact moment one slot frees.+ * exact moment one slot frees, clamped to [1, window] so a clock stepped+ * backwards cannot state a wait longer than the window itself.+ *+ * Hits are appended in time order, so the expired ones sit at the front:+ * advancing `head` past them makes each check O(1) amortized instead of a+ * rescan of the whole window. After a backward clock step the order breaks;+ * eviction from the front then keeps extra hits, which only denies sooner.*/check(subject: string): void {const now = this.now();const cutoff = now - this.windowMs;this.evictExpired(now, cutoff);- const recent = (this.hits.get(subject) ?? []).filter((t) => t > cutoff);- if (recent.length >= this.limit) {- this.hits.set(subject, recent);- const oldest = Math.min(...recent);+ const window = this.hits.get(subject) ?? { times: [], head: 0 };+ while (window.head < window.times.length && window.times[window.head] <= cutoff) {+ window.head += 1;+ }+ if (window.head > 0 && window.head * 2 >= window.times.length) {+ window.times = window.times.slice(window.head);+ window.head = 0;+ }+ if (window.times.length - window.head >= this.limit) {+ this.hits.set(subject, window);+ const oldest = window.times[window.head];+ const untilFreeSeconds = Math.ceil((oldest + this.windowMs - now) / 1000);throw new RateLimitExceededError(- Math.max(1, Math.ceil((oldest + this.windowMs - now) / 1000)),+ Math.min(Math.ceil(this.windowMs / 1000), Math.max(1, untilFreeSeconds)),);}- recent.push(now);- this.hits.set(subject, recent);+ window.times.push(now);+ this.hits.set(subject, window);}/**@@ -77,9 +92,30 @@ export class RateLimiter {*/private evictExpired(now: number, cutoff: number): void {if (now - this.lastSweep < this.windowMs) return;- for (const [subject, times] of this.hits) {- if (Math.max(...times) <= cutoff) this.hits.delete(subject);+ for (const [subject, window] of this.hits) {+ if (window.times[window.times.length - 1] <= cutoff) this.hits.delete(subject);}this.lastSweep = now;}}++/** One subject's hits in arrival order; entries before `head` have expired. */+interface SubjectWindow {+ times: number[];+ head: number;+}++/**+ * The agent read and write limiters, built from their per-replica budgets with+ * a one-minute window. The single place that maps each budget to its limiter,+ * shared by the process entry point and the tests that pin the wiring.+ */+export function buildAgentRateLimits(+ config: Pick<GfsConfig, "agentReadRlPerMinPerReplica" | "agentWriteRlPerMinPerReplica">,+ now?: () => number,+): { reads: RateLimiter; writes: RateLimiter } {+ return {+ reads: new RateLimiter({ limit: config.agentReadRlPerMinPerReplica, windowMs: 60_000, now }),+ writes: new RateLimiter({ limit: config.agentWriteRlPerMinPerReplica, windowMs: 60_000, now }),+ };+}