Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(gfs-controller): throttle the agent rate-limit denial log line

Alfredo Lopez50694221c296
diff --git a/gfs-controller/src/api/serve.agentRateLimit.test.ts b/gfs-controller/src/api/serve.agentRateLimit.test.ts
index 9897e690d..5612bd9a2 100644
--- a/gfs-controller/src/api/serve.agentRateLimit.test.ts
+++ b/gfs-controller/src/api/serve.agentRateLimit.test.ts
@@ -147,6 +147,7 @@ function planeWith(build: (now: () => number) => ServingDeps["rateLimit"]): Plan
} as unknown as ServingDeps["writeService"],
metrics,
rateLimit: build(() => clock.t),
+ now: () => clock.t,
};
return { deps, handler: new GfsServingHandler(deps), clock, metrics, authorizeCalls, store };
}
@@ -223,9 +224,12 @@ describe("agent rate limit under concurrent load", () => {
}
expect(p.authorizeCalls).toHaveLength(AGENTS.length * R);
expect(p.metrics.snapshot().rateLimitDenied).toEqual({ read: AGENTS.length * (10 - R), write: 0 });
+ // Every denial is counted above, but each agent writes one line per minute:
+ // the first denial logs, the other 10 - R - 1 wait for the next line.
const logs = denialLogs();
- expect(logs).toHaveLength(AGENTS.length * (10 - R));
- expect(logs.every((line) => /^\[gfsc\] rate_limit_denied kind=read subject=[0-9a-f]{64}$/.test(line))).toBe(true);
+ expect(logs).toHaveLength(AGENTS.length);
+ expect(logs.every((line) => /^\[gfsc\] rate_limit_denied kind=read subject=[0-9a-f]{64} suppressed=0$/.test(line))).toBe(true);
+ expect(new Set(logs).size).toBe(AGENTS.length);
});
it("G2: after the read budget is spent, writes spend their own budget; a following read is still denied", async () => {
diff --git a/gfs-controller/src/api/serve.copy.test.ts b/gfs-controller/src/api/serve.copy.test.ts
index 938a62eda..add5d429a 100644
--- a/gfs-controller/src/api/serve.copy.test.ts
+++ b/gfs-controller/src/api/serve.copy.test.ts
@@ -55,6 +55,7 @@ function fixture(options: { child?: boolean; snapshotCount?: number; omitLastPar
], liveChildren: [] };
const batch: Array<Array<{ resourceId: string; op: string }>> = [];
const audits: AuditEvent[] = [];
+ const times: number[] = [];
let loads = 0;
const copy = vi.fn(async (input: { requestId: string }) => {
if (options.copyError) throw options.copyError;
@@ -87,13 +88,17 @@ function fixture(options: { child?: boolean; snapshotCount?: number; omitLastPar
audit: { record: async event => { audits.push(event); } },
maxObjects: 1000, maxBytes: 1024 * 1024, timeoutMs: 30_000,
},
- now: () => options.times?.shift() ?? 10,
+ now: () => times.shift() ?? 10,
rateLimit: {
reads: new RateLimiter({ limit: 1_000_000, windowMs: 60_000 }),
writes: new RateLimiter({ limit: 1_000_000, windowMs: 60_000 }),
},
};
- return { deps, destination, handler: new GfsServingHandler(deps), batch, audits, copy, get loads() { return loads; } };
+ const handler = new GfsServingHandler(deps);
+ // The scripted times are for the request path. They are armed after the
+ // handler is built, because its constructor reads the clock once.
+ times.push(...(options.times ?? []));
+ return { deps, destination, handler, batch, audits, copy, get loads() { return loads; } };
}
describe("POST /v1/copy", () => {
diff --git a/gfs-controller/src/api/serve.rateLimit.test.ts b/gfs-controller/src/api/serve.rateLimit.test.ts
index 041f55293..c4910c0c1 100644
--- a/gfs-controller/src/api/serve.rateLimit.test.ts
+++ b/gfs-controller/src/api/serve.rateLimit.test.ts
@@ -133,6 +133,7 @@ function harness(opts: { reads?: number; writes?: number; windowMs?: number; all
writeService,
metrics,
rateLimit,
+ now: () => clock.t,
};
const handler = new GfsServingHandler(deps);
@@ -260,7 +261,7 @@ describe("GfsServingHandler — per-subject agent rate limit", () => {
expect(denied.statusCode).toBe(429);
expect(warn).toHaveBeenCalledTimes(1);
const hashed = createHash("sha256").update(HOST_A).digest("hex");
- expect(warn.mock.calls[0]).toEqual([`[gfsc] rate_limit_denied kind=write subject=${hashed}`]);
+ expect(warn.mock.calls[0]).toEqual([`[gfsc] rate_limit_denied kind=write subject=${hashed} suppressed=0`]);
// console.warn renders its arguments with util.format; a log collector
// splits records on newlines, so the rendered line must hold none.
expect(format(...(warn.mock.calls[0] as unknown[]))).not.toContain("\n");
@@ -309,4 +310,27 @@ describe("GfsServingHandler — per-subject agent rate limit", () => {
expect((await h.get(HOST_A)).statusCode).toBe(200);
expect(h.rateLimit.reads.trackedSubjectCount).toBe(1);
});
+
+ it("case 12: writes one denial line per subject per minute, counts every denial, and reports the suppressed lines", async () => {
+ const h = harness({ reads: 1, windowMs: 60_000 });
+ const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined);
+ const hashed = createHash("sha256").update(HOST_A).digest("hex");
+ expect((await h.get(HOST_A)).statusCode).toBe(200);
+
+ const statuses: number[] = [];
+ for (let i = 0; i < 50; i += 1) statuses.push((await h.get(HOST_A)).statusCode);
+ // Witness: all 50 requests were denied, and the counter saw every one.
+ expect(statuses).toEqual(Array.from({ length: 50 }, () => 429));
+ expect(h.metrics.snapshot().rateLimitDenied).toEqual({ read: 50, write: 0 });
+ expect(warn.mock.calls).toEqual([[`[gfsc] rate_limit_denied kind=read subject=${hashed} suppressed=0`]]);
+
+ // One minute later the read budget is back: one request passes, the next is denied.
+ h.clock.t += 60_000;
+ expect((await h.get(HOST_A)).statusCode).toBe(200);
+ expect((await h.get(HOST_A)).statusCode).toBe(429);
+ expect(h.metrics.snapshot().rateLimitDenied).toEqual({ read: 51, write: 0 });
+ expect(warn).toHaveBeenCalledTimes(2);
+ expect(warn.mock.calls[1]).toEqual([`[gfsc] rate_limit_denied kind=read subject=${hashed} suppressed=49`]);
+ expect(format(...(warn.mock.calls[1] as unknown[]))).not.toContain("\n");
+ });
});
diff --git a/gfs-controller/src/api/serve.ts b/gfs-controller/src/api/serve.ts
index eaaf2871e..af4cce4f4 100644
--- a/gfs-controller/src/api/serve.ts
+++ b/gfs-controller/src/api/serve.ts
@@ -8,6 +8,7 @@ import type { GfsPermission } from "../authz/resolve";
import { GfsSubjectResolutionDeniedError } from "../authz/subjectResolver";
import { checkTokenCeiling } from "../authz/tokenCeiling";
import type { GfsWriteService } from "../db/writeStore";
+import { LogThrottle } from "../logThrottle";
import type { GfsMetrics } from "../metrics";
import { type RateLimiter, RateLimitExceededError } from "../quota/rateLimit";
import type {
@@ -195,6 +196,9 @@ function isAgentSub(sub: string): boolean {
return sub.startsWith("host:");
}
+/** An agent retrying a denied request writes at most one denial line per kind per minute. */
+const RATE_LIMIT_DENIAL_LOG_INTERVAL_MS = 60_000;
+
const RESOURCE_RE = /^\/v1\/resources\/([^/]+)(?:\/(children|content))?$/;
const RESOLVE_RE = /^\/v1\/resolve$/;
const ACCESSIBLE_RE = /^\/v1\/accessible$/;
@@ -251,7 +255,12 @@ function abortConnectionHeader(req: IncomingMessage): Record<string, string> | u
}
export class GfsServingHandler {
- constructor(private readonly deps: ServingDeps) {}
+ /** Keyed by kind and hashed subject; bounds the rate_limit_denied line volume. */
+ private readonly denialLogThrottle: LogThrottle;
+
+ constructor(private readonly deps: ServingDeps) {
+ this.denialLogThrottle = new LogThrottle(RATE_LIMIT_DENIAL_LOG_INTERVAL_MS, () => this.clock());
+ }
private clock(): number {
return (this.deps.now ?? Date.now)();
@@ -578,7 +587,9 @@ export class GfsServingHandler {
* Step 3: spend one unit of the agent's write or read budget. Only host
* principals are charged; after authContext a `host:` subject has matched the
* canonical host form. A denial is counted, logged with the subject hashed,
- * and answered 429 with the limiter's own limit and retry-after.
+ * and answered 429 with the limiter's own limit and retry-after. The counter
+ * sees every denial; the log line is throttled to one per subject and kind
+ * per minute and carries how many lines it suppressed since the previous one.
*/
private chargeAgentBudget(ctx: AuthzContext, write: boolean): void {
if (!isAgentSub(ctx.primarySubject)) return;
@@ -593,7 +604,10 @@ export class GfsServingHandler {
// enumerable, so anyone holding the host list can reverse it. Hex output
// needs no log sanitizing.
const hashedSubject = createHash("sha256").update(ctx.primarySubject).digest("hex");
- console.warn(`[gfsc] rate_limit_denied kind=${kind} subject=${hashedSubject}`);
+ const suppressed = this.denialLogThrottle.admit(`${kind}:${hashedSubject}`);
+ if (suppressed !== undefined) {
+ console.warn(`[gfsc] rate_limit_denied kind=${kind} subject=${hashedSubject} suppressed=${suppressed}`);
+ }
throw new GfsError("rate_limited", "agent rate limit exceeded", undefined, {
retryAfterSeconds: err.retryAfterSeconds,
limit: write ? "agent_writes" : "agent_reads",
diff --git a/gfs-controller/src/logThrottle.test.ts b/gfs-controller/src/logThrottle.test.ts
new file mode 100644
index 000000000..9543b6c10
--- /dev/null
+++ b/gfs-controller/src/logThrottle.test.ts
@@ -0,0 +1,58 @@
+import { describe, expect, it } from "vitest";
+import { LogThrottle } from "./logThrottle";
+
+describe("LogThrottle", () => {
+ it("admits the first event per key, suppresses the rest of the interval, and reports how many it dropped", () => {
+ let t = 1_000;
+ const throttle = new LogThrottle(60_000, () => t);
+
+ expect(throttle.admit("a")).toBe(0);
+ const inInterval = Array.from({ length: 49 }, () => throttle.admit("a"));
+ // Witness: 49 events were offered and every one was suppressed.
+ expect(inInterval).toHaveLength(49);
+ expect(inInterval.every((r) => r === undefined)).toBe(true);
+
+ t += 60_000;
+ expect(throttle.admit("a")).toBe(49);
+ expect(throttle.admit("a")).toBeUndefined();
+ });
+
+ it("throttles each key on its own", () => {
+ const t = 1_000;
+ const throttle = new LogThrottle(60_000, () => t);
+
+ expect(throttle.admit("a")).toBe(0);
+ expect(throttle.admit("b")).toBe(0);
+ expect(throttle.admit("a")).toBeUndefined();
+ expect(throttle.admit("b")).toBeUndefined();
+ });
+
+ it("keeps a pending suppressed count through a sweep until the second interval ends", () => {
+ let t = 1_000;
+ const throttle = new LogThrottle(60_000, () => t);
+ throttle.admit("a");
+ throttle.admit("a");
+ throttle.admit("a");
+
+ // Another key's event at 1.5 intervals runs the sweep first.
+ t += 90_000;
+ throttle.admit("b");
+ expect(throttle.admit("a")).toBe(2);
+ });
+
+ it("sweeps keys idle for two intervals, so the map is bounded by the keys active recently", () => {
+ let t = 1_000;
+ const throttle = new LogThrottle(60_000, () => t);
+ for (let i = 0; i < 1_000; i += 1) throttle.admit(`k${i}`);
+ // Witness: every key is tracked before the sweep.
+ expect(throttle.trackedKeyCount).toBe(1_000);
+
+ t += 120_000;
+ throttle.admit("live");
+ expect(throttle.trackedKeyCount).toBe(1);
+ });
+
+ it("rejects a non-positive interval", () => {
+ expect(() => new LogThrottle(0)).toThrow(/intervalMs must be > 0/);
+ });
+});
diff --git a/gfs-controller/src/logThrottle.ts b/gfs-controller/src/logThrottle.ts
new file mode 100644
index 000000000..e282705e5
--- /dev/null
+++ b/gfs-controller/src/logThrottle.ts
@@ -0,0 +1,56 @@
+/**
+ * Bounds how many lines one event writes: at most one per key per interval.
+ * A suppressed event is only counted, and the next admitted line for that key
+ * carries the count, so no occurrence disappears from the log without a trace.
+ * Callers keep their metrics unthrottled; this only sheds log volume, such as
+ * one warn line per request from a client retrying a rate-limit denial.
+ *
+ * Keys idle for two intervals are dropped by a sweep that runs at most once per
+ * interval, so the map is bounded by the keys active in the last two intervals.
+ * A count still pending when its key is swept is lost from the log, not from
+ * the metrics.
+ *
+ * Same semantics as control-api's `src/observability/logThrottle.ts`.
+ */
+export class LogThrottle {
+ private readonly entries = new Map<string, { emittedAt: number; suppressed: number }>();
+ private lastSweep: number;
+
+ constructor(
+ private readonly intervalMs: number,
+ private readonly now: () => number = () => Date.now()
+ ) {
+ if (!(intervalMs > 0)) throw new Error("LogThrottle: intervalMs must be > 0");
+ this.lastSweep = now();
+ }
+
+ /** Number of keys currently held. Read by tests to observe the sweep. */
+ get trackedKeyCount(): number {
+ return this.entries.size;
+ }
+
+ /**
+ * Record one event for `key`. Returns the number of events suppressed since
+ * the key's previous line when a line is due now, or `undefined` when this
+ * event is suppressed.
+ */
+ admit(key: string): number | undefined {
+ const now = this.now();
+ this.sweep(now);
+ const entry = this.entries.get(key);
+ if (entry && now - entry.emittedAt < this.intervalMs) {
+ entry.suppressed += 1;
+ return undefined;
+ }
+ this.entries.set(key, { emittedAt: now, suppressed: 0 });
+ return entry?.suppressed ?? 0;
+ }
+
+ private sweep(now: number): void {
+ if (now - this.lastSweep < this.intervalMs) return;
+ for (const [key, entry] of this.entries) {
+ if (now - entry.emittedAt >= 2 * this.intervalMs) this.entries.delete(key);
+ }
+ this.lastSweep = now;
+ }
+}