Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(gfs-controller): throttle the agent rate-limit denial log line
Alfredo Lopez50694221c296
diff --git a/gfs-controller/src/api/serve.agentRateLimit.test.ts b/gfs-controller/src/api/serve.agentRateLimit.test.tsindex 9897e690d..5612bd9a2 100644--- a/gfs-controller/src/api/serve.agentRateLimit.test.ts+++ b/gfs-controller/src/api/serve.agentRateLimit.test.ts@@ -147,6 +147,7 @@ function planeWith(build: (now: () => number) => ServingDeps["rateLimit"]): Plan} as unknown as ServingDeps["writeService"],metrics,rateLimit: build(() => clock.t),+ now: () => clock.t,};return { deps, handler: new GfsServingHandler(deps), clock, metrics, authorizeCalls, store };}@@ -223,9 +224,12 @@ describe("agent rate limit under concurrent load", () => {}expect(p.authorizeCalls).toHaveLength(AGENTS.length * R);expect(p.metrics.snapshot().rateLimitDenied).toEqual({ read: AGENTS.length * (10 - R), write: 0 });+ // Every denial is counted above, but each agent writes one line per minute:+ // the first denial logs, the other 10 - R - 1 wait for the next line.const logs = denialLogs();- expect(logs).toHaveLength(AGENTS.length * (10 - R));- expect(logs.every((line) => /^\[gfsc\] rate_limit_denied kind=read subject=[0-9a-f]{64}$/.test(line))).toBe(true);+ expect(logs).toHaveLength(AGENTS.length);+ expect(logs.every((line) => /^\[gfsc\] rate_limit_denied kind=read subject=[0-9a-f]{64} suppressed=0$/.test(line))).toBe(true);+ expect(new Set(logs).size).toBe(AGENTS.length);});it("G2: after the read budget is spent, writes spend their own budget; a following read is still denied", async () => {diff --git a/gfs-controller/src/api/serve.copy.test.ts b/gfs-controller/src/api/serve.copy.test.tsindex 938a62eda..add5d429a 100644--- a/gfs-controller/src/api/serve.copy.test.ts+++ b/gfs-controller/src/api/serve.copy.test.ts@@ -55,6 +55,7 @@ function fixture(options: { child?: boolean; snapshotCount?: number; omitLastPar], liveChildren: [] };const batch: Array<Array<{ resourceId: string; op: string }>> = [];const audits: AuditEvent[] = [];+ const times: number[] = [];let loads = 0;const copy = vi.fn(async (input: { requestId: string }) => {if (options.copyError) throw options.copyError;@@ -87,13 +88,17 @@ function fixture(options: { child?: boolean; snapshotCount?: number; omitLastParaudit: { record: async event => { audits.push(event); } },maxObjects: 1000, maxBytes: 1024 * 1024, timeoutMs: 30_000,},- now: () => options.times?.shift() ?? 10,+ now: () => times.shift() ?? 10,rateLimit: {reads: new RateLimiter({ limit: 1_000_000, windowMs: 60_000 }),writes: new RateLimiter({ limit: 1_000_000, windowMs: 60_000 }),},};- return { deps, destination, handler: new GfsServingHandler(deps), batch, audits, copy, get loads() { return loads; } };+ const handler = new GfsServingHandler(deps);+ // The scripted times are for the request path. They are armed after the+ // handler is built, because its constructor reads the clock once.+ times.push(...(options.times ?? []));+ return { deps, destination, handler, batch, audits, copy, get loads() { return loads; } };}describe("POST /v1/copy", () => {diff --git a/gfs-controller/src/api/serve.rateLimit.test.ts b/gfs-controller/src/api/serve.rateLimit.test.tsindex 041f55293..c4910c0c1 100644--- a/gfs-controller/src/api/serve.rateLimit.test.ts+++ b/gfs-controller/src/api/serve.rateLimit.test.ts@@ -133,6 +133,7 @@ function harness(opts: { reads?: number; writes?: number; windowMs?: number; allwriteService,metrics,rateLimit,+ now: () => clock.t,};const handler = new GfsServingHandler(deps);@@ -260,7 +261,7 @@ describe("GfsServingHandler — per-subject agent rate limit", () => {expect(denied.statusCode).toBe(429);expect(warn).toHaveBeenCalledTimes(1);const hashed = createHash("sha256").update(HOST_A).digest("hex");- expect(warn.mock.calls[0]).toEqual([`[gfsc] rate_limit_denied kind=write subject=${hashed}`]);+ expect(warn.mock.calls[0]).toEqual([`[gfsc] rate_limit_denied kind=write subject=${hashed} suppressed=0`]);// console.warn renders its arguments with util.format; a log collector// splits records on newlines, so the rendered line must hold none.expect(format(...(warn.mock.calls[0] as unknown[]))).not.toContain("\n");@@ -309,4 +310,27 @@ describe("GfsServingHandler — per-subject agent rate limit", () => {expect((await h.get(HOST_A)).statusCode).toBe(200);expect(h.rateLimit.reads.trackedSubjectCount).toBe(1);});++ it("case 12: writes one denial line per subject per minute, counts every denial, and reports the suppressed lines", async () => {+ const h = harness({ reads: 1, windowMs: 60_000 });+ const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined);+ const hashed = createHash("sha256").update(HOST_A).digest("hex");+ expect((await h.get(HOST_A)).statusCode).toBe(200);++ const statuses: number[] = [];+ for (let i = 0; i < 50; i += 1) statuses.push((await h.get(HOST_A)).statusCode);+ // Witness: all 50 requests were denied, and the counter saw every one.+ expect(statuses).toEqual(Array.from({ length: 50 }, () => 429));+ expect(h.metrics.snapshot().rateLimitDenied).toEqual({ read: 50, write: 0 });+ expect(warn.mock.calls).toEqual([[`[gfsc] rate_limit_denied kind=read subject=${hashed} suppressed=0`]]);++ // One minute later the read budget is back: one request passes, the next is denied.+ h.clock.t += 60_000;+ expect((await h.get(HOST_A)).statusCode).toBe(200);+ expect((await h.get(HOST_A)).statusCode).toBe(429);+ expect(h.metrics.snapshot().rateLimitDenied).toEqual({ read: 51, write: 0 });+ expect(warn).toHaveBeenCalledTimes(2);+ expect(warn.mock.calls[1]).toEqual([`[gfsc] rate_limit_denied kind=read subject=${hashed} suppressed=49`]);+ expect(format(...(warn.mock.calls[1] as unknown[]))).not.toContain("\n");+ });});diff --git a/gfs-controller/src/api/serve.ts b/gfs-controller/src/api/serve.tsindex eaaf2871e..af4cce4f4 100644--- a/gfs-controller/src/api/serve.ts+++ b/gfs-controller/src/api/serve.ts@@ -8,6 +8,7 @@ import type { GfsPermission } from "../authz/resolve";import { GfsSubjectResolutionDeniedError } from "../authz/subjectResolver";import { checkTokenCeiling } from "../authz/tokenCeiling";import type { GfsWriteService } from "../db/writeStore";+import { LogThrottle } from "../logThrottle";import type { GfsMetrics } from "../metrics";import { type RateLimiter, RateLimitExceededError } from "../quota/rateLimit";import type {@@ -195,6 +196,9 @@ function isAgentSub(sub: string): boolean {return sub.startsWith("host:");}+/** An agent retrying a denied request writes at most one denial line per kind per minute. */+const RATE_LIMIT_DENIAL_LOG_INTERVAL_MS = 60_000;+const RESOURCE_RE = /^\/v1\/resources\/([^/]+)(?:\/(children|content))?$/;const RESOLVE_RE = /^\/v1\/resolve$/;const ACCESSIBLE_RE = /^\/v1\/accessible$/;@@ -251,7 +255,12 @@ function abortConnectionHeader(req: IncomingMessage): Record<string, string> | u}export class GfsServingHandler {- constructor(private readonly deps: ServingDeps) {}+ /** Keyed by kind and hashed subject; bounds the rate_limit_denied line volume. */+ private readonly denialLogThrottle: LogThrottle;++ constructor(private readonly deps: ServingDeps) {+ this.denialLogThrottle = new LogThrottle(RATE_LIMIT_DENIAL_LOG_INTERVAL_MS, () => this.clock());+ }private clock(): number {return (this.deps.now ?? Date.now)();@@ -578,7 +587,9 @@ export class GfsServingHandler {* Step 3: spend one unit of the agent's write or read budget. Only host* principals are charged; after authContext a `host:` subject has matched the* canonical host form. A denial is counted, logged with the subject hashed,- * and answered 429 with the limiter's own limit and retry-after.+ * and answered 429 with the limiter's own limit and retry-after. The counter+ * sees every denial; the log line is throttled to one per subject and kind+ * per minute and carries how many lines it suppressed since the previous one.*/private chargeAgentBudget(ctx: AuthzContext, write: boolean): void {if (!isAgentSub(ctx.primarySubject)) return;@@ -593,7 +604,10 @@ export class GfsServingHandler {// enumerable, so anyone holding the host list can reverse it. Hex output// needs no log sanitizing.const hashedSubject = createHash("sha256").update(ctx.primarySubject).digest("hex");- console.warn(`[gfsc] rate_limit_denied kind=${kind} subject=${hashedSubject}`);+ const suppressed = this.denialLogThrottle.admit(`${kind}:${hashedSubject}`);+ if (suppressed !== undefined) {+ console.warn(`[gfsc] rate_limit_denied kind=${kind} subject=${hashedSubject} suppressed=${suppressed}`);+ }throw new GfsError("rate_limited", "agent rate limit exceeded", undefined, {retryAfterSeconds: err.retryAfterSeconds,limit: write ? "agent_writes" : "agent_reads",diff --git a/gfs-controller/src/logThrottle.test.ts b/gfs-controller/src/logThrottle.test.tsnew file mode 100644index 000000000..9543b6c10--- /dev/null+++ b/gfs-controller/src/logThrottle.test.ts@@ -0,0 +1,58 @@+import { describe, expect, it } from "vitest";+import { LogThrottle } from "./logThrottle";++describe("LogThrottle", () => {+ it("admits the first event per key, suppresses the rest of the interval, and reports how many it dropped", () => {+ let t = 1_000;+ const throttle = new LogThrottle(60_000, () => t);++ expect(throttle.admit("a")).toBe(0);+ const inInterval = Array.from({ length: 49 }, () => throttle.admit("a"));+ // Witness: 49 events were offered and every one was suppressed.+ expect(inInterval).toHaveLength(49);+ expect(inInterval.every((r) => r === undefined)).toBe(true);++ t += 60_000;+ expect(throttle.admit("a")).toBe(49);+ expect(throttle.admit("a")).toBeUndefined();+ });++ it("throttles each key on its own", () => {+ const t = 1_000;+ const throttle = new LogThrottle(60_000, () => t);++ expect(throttle.admit("a")).toBe(0);+ expect(throttle.admit("b")).toBe(0);+ expect(throttle.admit("a")).toBeUndefined();+ expect(throttle.admit("b")).toBeUndefined();+ });++ it("keeps a pending suppressed count through a sweep until the second interval ends", () => {+ let t = 1_000;+ const throttle = new LogThrottle(60_000, () => t);+ throttle.admit("a");+ throttle.admit("a");+ throttle.admit("a");++ // Another key's event at 1.5 intervals runs the sweep first.+ t += 90_000;+ throttle.admit("b");+ expect(throttle.admit("a")).toBe(2);+ });++ it("sweeps keys idle for two intervals, so the map is bounded by the keys active recently", () => {+ let t = 1_000;+ const throttle = new LogThrottle(60_000, () => t);+ for (let i = 0; i < 1_000; i += 1) throttle.admit(`k${i}`);+ // Witness: every key is tracked before the sweep.+ expect(throttle.trackedKeyCount).toBe(1_000);++ t += 120_000;+ throttle.admit("live");+ expect(throttle.trackedKeyCount).toBe(1);+ });++ it("rejects a non-positive interval", () => {+ expect(() => new LogThrottle(0)).toThrow(/intervalMs must be > 0/);+ });+});diff --git a/gfs-controller/src/logThrottle.ts b/gfs-controller/src/logThrottle.tsnew file mode 100644index 000000000..e282705e5--- /dev/null+++ b/gfs-controller/src/logThrottle.ts@@ -0,0 +1,56 @@+/**+ * Bounds how many lines one event writes: at most one per key per interval.+ * A suppressed event is only counted, and the next admitted line for that key+ * carries the count, so no occurrence disappears from the log without a trace.+ * Callers keep their metrics unthrottled; this only sheds log volume, such as+ * one warn line per request from a client retrying a rate-limit denial.+ *+ * Keys idle for two intervals are dropped by a sweep that runs at most once per+ * interval, so the map is bounded by the keys active in the last two intervals.+ * A count still pending when its key is swept is lost from the log, not from+ * the metrics.+ *+ * Same semantics as control-api's `src/observability/logThrottle.ts`.+ */+export class LogThrottle {+ private readonly entries = new Map<string, { emittedAt: number; suppressed: number }>();+ private lastSweep: number;++ constructor(+ private readonly intervalMs: number,+ private readonly now: () => number = () => Date.now()+ ) {+ if (!(intervalMs > 0)) throw new Error("LogThrottle: intervalMs must be > 0");+ this.lastSweep = now();+ }++ /** Number of keys currently held. Read by tests to observe the sweep. */+ get trackedKeyCount(): number {+ return this.entries.size;+ }++ /**+ * Record one event for `key`. Returns the number of events suppressed since+ * the key's previous line when a line is due now, or `undefined` when this+ * event is suppressed.+ */+ admit(key: string): number | undefined {+ const now = this.now();+ this.sweep(now);+ const entry = this.entries.get(key);+ if (entry && now - entry.emittedAt < this.intervalMs) {+ entry.suppressed += 1;+ return undefined;+ }+ this.entries.set(key, { emittedAt: now, suppressed: 0 });+ return entry?.suppressed ?? 0;+ }++ private sweep(now: number): void {+ if (now - this.lastSweep < this.intervalMs) return;+ for (const [key, entry] of this.entries) {+ if (now - entry.emittedAt >= 2 * this.intervalMs) this.entries.delete(key);+ }+ this.lastSweep = now;+ }+}