Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(gfs-controller): enforce the per-subject agent rate limit on reads and writes

Alfredo Lopezc6a32e6bac5f
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.yml
index 8b5a6bb6f..5a509c804 100644
--- a/.github/workflows/ci-public.yml
+++ b/.github/workflows/ci-public.yml
@@ -126,9 +126,29 @@ jobs:
working-directory: desktop-app
run: npx playwright install --with-deps chromium
+ # PINNED_SUITES lists test files that must appear in the run's output.
+ # A rename or move that drops one would otherwise still pass `npm test`.
- name: Test
working-directory: ${{ matrix.service }}
- run: npm test
+ env:
+ PINNED_SUITES: ${{ matrix.service == 'gfs-controller' && 'serve.agentRateLimit.test.ts' || '' }}
+ run: |
+ set -euo pipefail
+ if [ -z "$PINNED_SUITES" ]; then
+ npm test
+ exit 0
+ fi
+ log_file="$(mktemp)"
+ normalized_log="$(mktemp)"
+ trap 'rm -f "$log_file" "$normalized_log"' EXIT
+ npm test 2>&1 | tee "$log_file"
+ sed $'s/\033\\[[0-9;]*m//g' "$log_file" > "$normalized_log"
+ for suite in $PINNED_SUITES; do
+ grep -Fq "$suite" "$normalized_log" || {
+ echo "${{ matrix.service }} test run omitted pinned suite ${suite}" >&2
+ exit 1
+ }
+ done
# These node:test suites live outside service packages and resolve the MCP
# SDK from mcp-host's installed dependencies.
diff --git a/gfs-controller/src/api/errors.ts b/gfs-controller/src/api/errors.ts
index 3ba87c480..0a2e2fd39 100644
--- a/gfs-controller/src/api/errors.ts
+++ b/gfs-controller/src/api/errors.ts
@@ -17,6 +17,7 @@ export type GfsErrorCode =
| "precondition_failed"
| "payload_too_large"
| "quota_exceeded"
+ | "rate_limited"
| "conflict"
| "idempotency_conflict"
| "upload_incomplete"
@@ -43,6 +44,7 @@ export const HTTP_STATUS: Record<GfsErrorCode, number> = {
precondition_failed: 412, // spec-pinned
payload_too_large: 413, // spec-pinned
quota_exceeded: 429,
+ rate_limited: 429,
conflict: 409,
idempotency_conflict: 409,
upload_incomplete: 409,
diff --git a/gfs-controller/src/api/serve.agentRateLimit.test.ts b/gfs-controller/src/api/serve.agentRateLimit.test.ts
new file mode 100644
index 000000000..015c331ca
--- /dev/null
+++ b/gfs-controller/src/api/serve.agentRateLimit.test.ts
@@ -0,0 +1,415 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import http, { type IncomingMessage, type ServerResponse } from "node:http";
+import { Readable, Writable } from "node:stream";
+import type { GfsVerifiedClaims } from "../auth/verify";
+import type { AuthzContext } from "../authz/permissionClient";
+import { loadConfig, type GfsConfig } from "../config";
+import { GfsMetrics } from "../metrics";
+import { RateLimiter } from "../quota/rateLimit";
+import { GfsServer } from "../server";
+import type { GfsResource } from "./read";
+import { GfsServingHandler, type ServingDeps } from "./serve";
+
+/**
+ * Stress suite for the per-subject agent rate limit (#699). Concurrent bursts
+ * from many host principals, with user and linked-admin traffic interleaved,
+ * against real `RateLimiter` instances on an injected clock. Every count is
+ * exact: the limiter is in-process and synchronous, so concurrency cannot make
+ * the outcome approximate.
+ */
+
+class FakeRes extends Writable {
+ statusCode = 0;
+ headers: Record<string, string> = {};
+ headersSent = false;
+ private chunks: Buffer[] = [];
+ writeHead(status: number, headers?: Record<string, string>): this {
+ this.statusCode = status;
+ this.headersSent = true;
+ if (headers) Object.assign(this.headers, headers);
+ return this;
+ }
+ setHeader(name: string, value: string): this {
+ this.headers[name] = value;
+ return this;
+ }
+ _write(chunk: Buffer | string, _enc: BufferEncoding, cb: (e?: Error | null) => void): void {
+ this.chunks.push(Buffer.from(chunk));
+ cb();
+ }
+ get json(): { ok: boolean; error?: { code: string; limit?: string } } {
+ return JSON.parse(Buffer.concat(this.chunks).toString("utf8"));
+ }
+}
+
+const WINDOW_MS = 60_000;
+const RID = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+const REQUEST_ID = "44444444-4444-4444-8444-444444444444";
+const ADMIN_ID = "22222222-2222-4222-8222-222222222222";
+const DESKTOP_USER_ID = "33333333-3333-4333-8333-333333333333";
+const LINKED_ADMIN = "linked-admin";
+
+const DIR: GfsResource = {
+ resourceId: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
+ drive: "main",
+ parentResourceId: "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
+ name: "docs",
+ kind: "directory",
+ pathCache: "/docs",
+ version: 2,
+ bytes: 0,
+ blobKey: null,
+ contentSha256: null,
+ deletedAt: null,
+ updatedAt: "2026-01-01T00:00:00.000Z",
+};
+
+const hosts = (n: number): string[] =>
+ Array.from({ length: n }, (_, i) => `host:1st:mcp-host/agent-${String(i).padStart(2, "0")}`);
+const users = (n: number): string[] =>
+ Array.from({ length: n }, (_, i) => `11111111-1111-4111-8111-${String(i).padStart(12, "0")}`);
+
+function claimsFor(token: string): GfsVerifiedClaims {
+ const base = {
+ drive: "main",
+ scopes: ["gfs.read", "gfs.write"] as GfsVerifiedClaims["scopes"],
+ pathBindings: [],
+ iat: 0,
+ exp: 0,
+ };
+ if (token === LINKED_ADMIN) {
+ return {
+ ...base,
+ sub: ADMIN_ID,
+ principalType: "control-admin",
+ brokeredAuthority: { desktopUserId: DESKTOP_USER_ID, controlAdminId: ADMIN_ID, authoritySource: "linked-admin" },
+ };
+ }
+ return { ...base, sub: token };
+}
+
+function contextFor(claims: Pick<GfsVerifiedClaims, "sub" | "brokeredAuthority">): AuthzContext {
+ if (claims.brokeredAuthority) {
+ return {
+ drive: "main",
+ subjects: ["operator:"],
+ isOperator: true,
+ primarySubject: ADMIN_ID,
+ effectiveControlAdminId: ADMIN_ID,
+ desktopUserId: DESKTOP_USER_ID,
+ authoritySource: "linked-admin",
+ };
+ }
+ if (claims.sub.startsWith("host:")) {
+ return { drive: "main", subjects: [claims.sub], isOperator: false, primarySubject: claims.sub };
+ }
+ return { drive: "main", subjects: [`user:${claims.sub}`], isOperator: false, primarySubject: claims.sub };
+}
+
+interface Plane {
+ deps: ServingDeps;
+ handler: GfsServingHandler;
+ clock: { t: number };
+ metrics: GfsMetrics;
+ /** Permission-store decisions, by the subject that asked. */
+ authorizeCalls: string[];
+ /** Flip to make the permission store deny every later request. */
+ store: { allow: boolean };
+}
+
+function plane(limits: { reads: number; writes: number }): Plane {
+ const clock = { t: 1_000_000 };
+ const store = { allow: true };
+ const authorizeCalls: string[] = [];
+ const metrics = new GfsMetrics();
+ const deps: ServingDeps = {
+ verifyToken: (token) => claimsFor(token),
+ resolveContext: async (claims) => contextFor(claims),
+ authorize: async (ctx) => {
+ authorizeCalls.push(ctx.primarySubject);
+ return { allowed: store.allow };
+ },
+ audit: { record: async () => undefined },
+ store: { getResource: async () => DIR, listChildren: async () => [] },
+ blobs: { read: async () => Readable.from([]) },
+ writeService: {
+ async replace() {
+ return { ...DIR, kind: "file" as const, version: 3 };
+ },
+ } as unknown as ServingDeps["writeService"],
+ metrics,
+ rateLimit: {
+ reads: new RateLimiter({ limit: limits.reads, windowMs: WINDOW_MS, now: () => clock.t }),
+ writes: new RateLimiter({ limit: limits.writes, windowMs: WINDOW_MS, now: () => clock.t }),
+ },
+ };
+ return { deps, handler: new GfsServingHandler(deps), clock, metrics, authorizeCalls, store };
+}
+
+async function get(p: Plane, token: string): Promise<FakeRes> {
+ const res = new FakeRes();
+ const req = { url: `/v1/resources/${RID}/children`, method: "GET", headers: { authorization: `Bearer ${token}` } };
+ await p.handler.tryHandle(req as unknown as IncomingMessage, res as unknown as ServerResponse);
+ return res;
+}
+
+async function put(p: Plane, token: string): Promise<FakeRes> {
+ const res = new FakeRes();
+ const req = Readable.from([Buffer.from(JSON.stringify({ content: "hi", ifMatch: 2 }))]) as unknown as IncomingMessage;
+ req.url = `/v1/resources/${RID}/content`;
+ req.method = "PUT";
+ req.headers = { authorization: `Bearer ${token}`, "x-request-id": REQUEST_ID };
+ await p.handler.tryHandle(req, res as unknown as ServerResponse);
+ return res;
+}
+
+/** Fire `perSubject` concurrent requests for every token in one Promise.all. */
+async function burst(
+ p: Plane,
+ tokens: string[],
+ perSubject: number,
+ send: (p: Plane, token: string) => Promise<FakeRes>
+): Promise<Map<string, FakeRes[]>> {
+ const flat = tokens.flatMap((token) => Array.from({ length: perSubject }, () => token));
+ const results = await Promise.all(flat.map((token) => send(p, token)));
+ const byToken = new Map<string, FakeRes[]>(tokens.map((t) => [t, []]));
+ flat.forEach((token, i) => byToken.get(token)!.push(results[i]));
+ return byToken;
+}
+
+const statuses = (rs: FakeRes[]): Record<number, number> =>
+ rs.reduce<Record<number, number>>((acc, r) => ({ ...acc, [r.statusCode]: (acc[r.statusCode] ?? 0) + 1 }), {});
+
+let warn: ReturnType<typeof vi.spyOn>;
+const denialLogs = (): Array<{ kind: string }> =>
+ warn.mock.calls
+ .map((c: unknown[]) => c[0] as { event?: string; kind: string })
+ .filter((e: { event?: string }) => e?.event === "gfs_rate_limit_denied");
+
+beforeEach(() => {
+ warn = vi.spyOn(console, "warn").mockImplementation(() => undefined);
+});
+afterEach(() => {
+ vi.restoreAllMocks();
+});
+
+describe("agent rate limit under concurrent load", () => {
+ const R = 5;
+ const W = 2;
+ const AGENTS = hosts(20);
+
+ it("G1+G6: 20 agents x 10 concurrent reads each get exactly R through, and only those reach the store", async () => {
+ const p = plane({ reads: R, writes: W });
+
+ const byAgent = await burst(p, AGENTS, 10, get);
+
+ for (const agent of AGENTS) {
+ const rs = byAgent.get(agent)!;
+ expect(statuses(rs)).toEqual({ 200: R, 429: 10 - R });
+ for (const denied of rs.filter((r) => r.statusCode === 429)) {
+ expect(denied.json.error?.code).toBe("rate_limited");
+ expect(denied.json.error?.limit).toBe("agent_reads");
+ expect(denied.headers["Retry-After"]).toBe("60");
+ expect(denied.headers["X-RateLimit-Limit"]).toBe(String(R));
+ expect(denied.headers["X-GFS-RateLimit-Scope"]).toBe("agent_reads");
+ }
+ // G6: the permission store answered exactly the allowed requests.
+ expect(p.authorizeCalls.filter((s) => s === agent)).toHaveLength(R);
+ }
+ expect(p.authorizeCalls).toHaveLength(AGENTS.length * R);
+ expect(p.metrics.snapshot().rateLimitDenied).toEqual({ read: AGENTS.length * (10 - R), write: 0 });
+ expect(denialLogs()).toHaveLength(AGENTS.length * (10 - R));
+ });
+
+ it("G2: after the read budget is spent, writes spend their own budget; a following read is still denied", async () => {
+ const p = plane({ reads: R, writes: W });
+ await burst(p, AGENTS, 10, get);
+ const authorizedAfterReads = p.authorizeCalls.length;
+ p.store.allow = false;
+
+ const byAgent = await burst(p, AGENTS, 5, put);
+
+ for (const agent of AGENTS) {
+ const rs = byAgent.get(agent)!;
+ expect(statuses(rs)).toEqual({ 403: W, 429: 5 - W });
+ for (const denied of rs.filter((r) => r.statusCode === 429)) {
+ expect(denied.json.error?.limit).toBe("agent_writes");
+ expect(denied.headers["X-RateLimit-Limit"]).toBe(String(W));
+ }
+ }
+ // Only the W writes per agent that passed the limiter asked the store.
+ expect(p.authorizeCalls.length - authorizedAfterReads).toBe(AGENTS.length * W);
+
+ const after = await burst(p, AGENTS, 1, get);
+ for (const agent of AGENTS) {
+ const [r] = after.get(agent)!;
+ expect(r.statusCode).toBe(429);
+ expect(r.json.error?.limit).toBe("agent_reads");
+ }
+ expect(p.metrics.snapshot().rateLimitDenied).toEqual({
+ read: AGENTS.length * (10 - R) + AGENTS.length,
+ write: AGENTS.length * (5 - W),
+ });
+ });
+
+ it("G3: user and linked-admin reads interleaved with the agent burst are never denied by gfsc", async () => {
+ const p = plane({ reads: R, writes: W });
+ const people = [...users(5), LINKED_ADMIN];
+ const perPerson = 3 * R;
+
+ const [agentResults, peopleResults] = await Promise.all([
+ burst(p, AGENTS, 10, get),
+ burst(p, people, perPerson, get),
+ ]);
+
+ for (const person of people) {
+ expect(statuses(peopleResults.get(person)!)).toEqual({ 200: perPerson });
+ }
+ // Liveness witness: the limiter was active in the same burst.
+ for (const agent of AGENTS) {
+ expect(statuses(agentResults.get(agent)!)).toEqual({ 200: R, 429: 10 - R });
+ }
+ expect(p.authorizeCalls.filter((s) => !s.startsWith("host:"))).toHaveLength(people.length * perPerson);
+ expect(p.deps.rateLimit.reads.trackedSubjectCount).toBe(AGENTS.length);
+ });
+
+ it("G4: the window slides; a full window restores the budget and half a window frees only the older half", async () => {
+ const p = plane({ reads: 6, writes: W });
+ const [agent] = AGENTS;
+ const t0 = p.clock.t;
+
+ expect(statuses((await burst(p, [agent], 3, get)).get(agent)!)).toEqual({ 200: 3 });
+ p.clock.t = t0 + WINDOW_MS / 2;
+ expect(statuses((await burst(p, [agent], 3, get)).get(agent)!)).toEqual({ 200: 3 });
+ const over = await get(p, agent);
+ expect(over.statusCode).toBe(429);
+ expect(over.headers["Retry-After"]).toBe(String(WINDOW_MS / 2 / 1000));
+
+ // The first three hits leave the window; the three from t0 + W/2 remain.
+ p.clock.t = t0 + WINDOW_MS + 1;
+ expect(statuses((await burst(p, [agent], 6, get)).get(agent)!)).toEqual({ 200: 3, 429: 3 });
+
+ // A full window after the last allowed hit, the whole budget is back.
+ p.clock.t = t0 + 2 * WINDOW_MS + 2;
+ expect(statuses((await burst(p, [agent], 8, get)).get(agent)!)).toEqual({ 200: 6, 429: 2 });
+ });
+});
+
+describe("agent rate limit through a real GfsServer", () => {
+ const R = 5;
+ const PER_CLIENT = R + 5;
+ const AGENT = "host:1st:mcp-host/agent-wire";
+
+ /** The operator's value travels env → loadConfig → limiter, as in index.ts. */
+ function operatorConfig(): GfsConfig {
+ vi.stubEnv("GFS_STORAGE_ROLE", "reader");
+ vi.stubEnv("GFS_DEV_MODE", "true");
+ vi.stubEnv("GFS_PORT", "0");
+ vi.stubEnv("GFS_AGENT_READ_RL_PER_MIN_PER_REPLICA", String(R));
+ vi.stubEnv("GFS_AGENT_WRITE_RL_PER_MIN_PER_REPLICA", "2");
+ return loadConfig();
+ }
+
+ let server: GfsServer | null = null;
+ const agents: http.Agent[] = [];
+ afterEach(async () => {
+ vi.unstubAllEnvs();
+ for (const a of agents.splice(0)) a.destroy();
+ if (server) await server.stop();
+ server = null;
+ });
+
+ interface WireResponse {
+ status: number;
+ retryAfter: string | undefined;
+ limit: string | undefined;
+ localPort: number;
+ }
+
+ function wireGet(port: number, agent: http.Agent): Promise<WireResponse> {
+ return new Promise((resolve, reject) => {
+ const req = http.get(
+ {
+ host: "127.0.0.1",
+ port,
+ path: `/v1/resources/${RID}/children`,
+ agent,
+ headers: { authorization: `Bearer ${AGENT}` },
+ },
+ (res) => {
+ // Read the socket before "end": keep-alive detaches it on release.
+ const localPort = res.socket.localPort;
+ if (localPort === undefined) {
+ reject(new Error("response socket has no local port"));
+ return;
+ }
+ res.resume();
+ res.on("end", () =>
+ resolve({
+ status: res.statusCode ?? 0,
+ retryAfter: res.headers["retry-after"] as string | undefined,
+ limit: res.headers["x-ratelimit-limit"] as string | undefined,
+ localPort,
+ })
+ );
+ }
+ );
+ req.on("error", reject);
+ });
+ }
+
+ function metricsText(port: number): Promise<string> {
+ return new Promise((resolve, reject) => {
+ http
+ .get({ host: "127.0.0.1", port, path: "/metrics" }, (res) => {
+ let text = "";
+ res.on("data", (c) => (text += c));
+ res.on("end", () => resolve(text));
+ })
+ .on("error", reject);
+ });
+ }
+
+ it("G5: two keep-alive clients of one agent share one budget across both sockets", async () => {
+ const config = operatorConfig();
+ expect(config.port).toBe(0);
+ const p = plane({ reads: config.agentReadRlPerMinPerReplica, writes: config.agentWriteRlPerMinPerReplica });
+ server = new GfsServer(
+ config,
+ { isStorageMounted: async () => true, pingPermissionStore: async () => undefined },
+ p.handler,
+ p.metrics
+ );
+ const port = await server.start();
+
+ const clients = [0, 1].map(() => {
+ const a = new http.Agent({ keepAlive: true, maxSockets: 1 });
+ agents.push(a);
+ return a;
+ });
+ const sequential = async (a: http.Agent): Promise<WireResponse[]> => {
+ const out: WireResponse[] = [];
+ for (let i = 0; i < PER_CLIENT; i++) out.push(await wireGet(port, a));
+ return out;
+ };
+ const [first, second] = await Promise.all(clients.map(sequential));
+ const all = [...first, ...second];
+
+ // Each client reused one socket, and the two sockets are distinct.
+ const firstPorts = new Set(first.map((r) => r.localPort));
+ const secondPorts = new Set(second.map((r) => r.localPort));
+ expect(firstPorts.size).toBe(1);
+ expect(secondPorts.size).toBe(1);
+ expect([...firstPorts][0]).not.toBe([...secondPorts][0]);
+
+ expect(all.filter((r) => r.status === 200)).toHaveLength(R);
+ const denied = all.filter((r) => r.status === 429);
+ expect(denied).toHaveLength(2 * PER_CLIENT - R);
+ for (const r of denied) {
+ expect(r.retryAfter).toBe("60");
+ expect(r.limit).toBe(String(R));
+ }
+ expect(p.authorizeCalls).toHaveLength(R);
+ expect(await metricsText(port)).toContain(`gfs_rate_limit_denied_total{kind="read"} ${2 * PER_CLIENT - R}`);
+ });
+});
diff --git a/gfs-controller/src/api/serve.copy.test.ts b/gfs-controller/src/api/serve.copy.test.ts
index 71cc173f2..938a62eda 100644
--- a/gfs-controller/src/api/serve.copy.test.ts
+++ b/gfs-controller/src/api/serve.copy.test.ts
@@ -5,6 +5,7 @@ import { describe, expect, it, vi } from "vitest";
import type { GfsVerifiedClaims } from "../auth/verify";
import type { AuditEvent } from "../authz/audit";
import type { AuthzContext } from "../authz/permissionClient";
+import { RateLimiter } from "../quota/rateLimit";
import type { CopyDestinationSnapshot, CopySnapshotNode } from "./copy";
import { GfsServingHandler, type ServingDeps } from "./serve";
@@ -87,6 +88,10 @@ function fixture(options: { child?: boolean; snapshotCount?: number; omitLastPar
maxObjects: 1000, maxBytes: 1024 * 1024, timeoutMs: 30_000,
},
now: () => options.times?.shift() ?? 10,
+ rateLimit: {
+ reads: new RateLimiter({ limit: 1_000_000, windowMs: 60_000 }),
+ writes: new RateLimiter({ limit: 1_000_000, windowMs: 60_000 }),
+ },
};
return { deps, destination, handler: new GfsServingHandler(deps), batch, audits, copy, get loads() { return loads; } };
}
diff --git a/gfs-controller/src/api/serve.rateLimit.test.ts b/gfs-controller/src/api/serve.rateLimit.test.ts
new file mode 100644
index 000000000..3ffc94608
--- /dev/null
+++ b/gfs-controller/src/api/serve.rateLimit.test.ts
@@ -0,0 +1,307 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import type { IncomingMessage, ServerResponse } from "node:http";
+import { Readable, Writable } from "node:stream";
+import type { GfsVerifiedClaims } from "../auth/verify";
+import type { AuthzContext } from "../authz/permissionClient";
+import { GfsMetrics } from "../metrics";
+import { RateLimiter } from "../quota/rateLimit";
+import type { GfsResource } from "./read";
+import { GfsServingHandler, type ServingDeps } from "./serve";
+
+/**
+ * The per-subject agent rate limit (#699): every request from a host principal
+ * spends the read or the write budget right after subject resolution and before
+ * authorization. User and linked-admin principals are metered by control-api
+ * and are never charged here.
+ */
+
+class FakeRes extends Writable {
+ statusCode = 0;
+ headers: Record<string, string> = {};
+ headersSent = false;
+ private chunks: Buffer[] = [];
+ writeHead(status: number, headers?: Record<string, string>): this {
+ this.statusCode = status;
+ this.headersSent = true;
+ if (headers) Object.assign(this.headers, headers);
+ return this;
+ }
+ setHeader(name: string, value: string): this {
+ this.headers[name] = value;
+ return this;
+ }
+ _write(chunk: Buffer | string, _enc: BufferEncoding, cb: (e?: Error | null) => void): void {
+ this.chunks.push(Buffer.from(chunk));
+ cb();
+ }
+ get body(): string {
+ return Buffer.concat(this.chunks).toString("utf8");
+ }
+ get json(): { ok: boolean; error?: { code: string; message: string; limit?: string } } {
+ return JSON.parse(this.body);
+ }
+}
+
+const HOST_A = "host:1st:mcp-host/agent-a";
+const HOST_B = "host:1st:mcp-host/agent-b";
+const USER_ID = "11111111-1111-4111-8111-111111111111";
+const ADMIN_ID = "22222222-2222-4222-8222-222222222222";
+const DESKTOP_USER_ID = "33333333-3333-4333-8333-333333333333";
+const RID = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+const REQUEST_ID = "44444444-4444-4444-8444-444444444444";
+
+const DIR: GfsResource = {
+ resourceId: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
+ drive: "main",
+ parentResourceId: "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
+ name: "docs",
+ kind: "directory",
+ pathCache: "/docs",
+ version: 2,
+ bytes: 0,
+ blobKey: null,
+ contentSha256: null,
+ deletedAt: null,
+ updatedAt: "2026-01-01T00:00:00.000Z",
+};
+
+function claimsFor(token: string): GfsVerifiedClaims {
+ const base = { drive: "main", scopes: ["gfs.read", "gfs.write"] as GfsVerifiedClaims["scopes"], pathBindings: [], iat: 0, exp: 0 };
+ if (token === "linked-admin") {
+ return {
+ ...base,
+ sub: ADMIN_ID,
+ principalType: "control-admin",
+ brokeredAuthority: { desktopUserId: DESKTOP_USER_ID, controlAdminId: ADMIN_ID, authoritySource: "linked-admin" },
+ };
+ }
+ return { ...base, sub: token };
+}
+
+function contextFor(claims: Pick<GfsVerifiedClaims, "sub" | "brokeredAuthority">): AuthzContext {
+ if (claims.brokeredAuthority) {
+ return {
+ drive: "main",
+ subjects: ["operator:"],
+ isOperator: true,
+ primarySubject: ADMIN_ID,
+ effectiveControlAdminId: ADMIN_ID,
+ desktopUserId: DESKTOP_USER_ID,
+ authoritySource: "linked-admin",
+ };
+ }
+ if (claims.sub.startsWith("host:")) {
+ return { drive: "main", subjects: [claims.sub], isOperator: false, primarySubject: claims.sub };
+ }
+ return { drive: "main", subjects: [`user:${claims.sub}`], isOperator: false, primarySubject: claims.sub };
+}
+
+function harness(opts: { reads?: number; writes?: number; windowMs?: number; allowed?: boolean } = {}) {
+ const clock = { t: 1_000_000 };
+ const windowMs = opts.windowMs ?? 60_000;
+ const rateLimit = {
+ reads: new RateLimiter({ limit: opts.reads ?? 100, windowMs, now: () => clock.t }),
+ writes: new RateLimiter({ limit: opts.writes ?? 100, windowMs, now: () => clock.t }),
+ };
+ const replaces: string[] = [];
+ const listChildrenBySubject: string[] = [];
+ const metrics = new GfsMetrics();
+ let current = "";
+ const listChildren = vi.fn(async () => {
+ listChildrenBySubject.push(current);
+ return [];
+ });
+ const writeService = {
+ async replace(input: { mutation: { subject: string } }) {
+ replaces.push(input.mutation.subject);
+ return { ...DIR, kind: "file" as const, version: 3 };
+ },
+ } as unknown as ServingDeps["writeService"];
+ const deps: ServingDeps = {
+ verifyToken: (token) => {
+ current = token;
+ return claimsFor(token);
+ },
+ resolveContext: async (claims) => contextFor(claims),
+ authorize: async () => ({ allowed: opts.allowed !== false }),
+ audit: { record: async () => undefined },
+ store: { getResource: async () => DIR, listChildren },
+ blobs: { read: async () => Readable.from([]) },
+ writeService,
+ metrics,
+ rateLimit,
+ };
+ const handler = new GfsServingHandler(deps);
+
+ async function get(token: string): Promise<FakeRes> {
+ const res = new FakeRes();
+ const request = { url: `/v1/resources/${RID}/children`, method: "GET", headers: { authorization: `Bearer ${token}` } };
+ await handler.tryHandle(request as unknown as IncomingMessage, res as unknown as ServerResponse);
+ return res;
+ }
+
+ async function put(token: string): Promise<FakeRes> {
+ const res = new FakeRes();
+ const stream = Readable.from([Buffer.from(JSON.stringify({ content: "hi", ifMatch: 2 }))]) as unknown as IncomingMessage;
+ stream.url = `/v1/resources/${RID}/content`;
+ stream.method = "PUT";
+ stream.headers = { authorization: `Bearer ${token}`, "x-request-id": REQUEST_ID };
+ await handler.tryHandle(stream, res as unknown as ServerResponse);
+ return res;
+ }
+
+ return { clock, rateLimit, replaces, listChildren, listChildrenBySubject, metrics, get, put };
+}
+
+afterEach(() => {
+ vi.restoreAllMocks();
+});
+
+describe("GfsServingHandler — per-subject agent rate limit", () => {
+ it("case 1: caps an agent's writes and answers 429 with the retry headers", async () => {
+ const h = harness({ writes: 2 });
+ vi.spyOn(console, "warn").mockImplementation(() => undefined);
+ expect((await h.put(HOST_A)).statusCode).toBe(200);
+ expect((await h.put(HOST_A)).statusCode).toBe(200);
+ const denied = await h.put(HOST_A);
+ expect(denied.statusCode).toBe(429);
+ expect(denied.json.error?.code).toBe("rate_limited");
+ expect(denied.json.error?.limit).toBe("agent_writes");
+ expect(denied.headers["Retry-After"]).toBe("60");
+ expect(denied.headers["X-RateLimit-Limit"]).toBe("2");
+ expect(denied.headers["X-GFS-RateLimit-Scope"]).toBe("agent_writes");
+ // The denied attempt never reached the executor.
+ expect(h.replaces).toHaveLength(2);
+ });
+
+ it("case 2: caps an agent's reads", async () => {
+ const h = harness({ reads: 2 });
+ vi.spyOn(console, "warn").mockImplementation(() => undefined);
+ expect((await h.get(HOST_A)).statusCode).toBe(200);
+ expect((await h.get(HOST_A)).statusCode).toBe(200);
+ const denied = await h.get(HOST_A);
+ expect(denied.statusCode).toBe(429);
+ expect(denied.json.error?.code).toBe("rate_limited");
+ expect(denied.headers["X-GFS-RateLimit-Scope"]).toBe("agent_reads");
+ expect(denied.headers["X-RateLimit-Limit"]).toBe("2");
+ expect(h.listChildren).toHaveBeenCalledTimes(2);
+ });
+
+ it("case 3: keeps separate read and write budgets, and the read budget stays finite", async () => {
+ const h = harness({ reads: 2, writes: 1 });
+ vi.spyOn(console, "warn").mockImplementation(() => undefined);
+ const readsCheck = vi.spyOn(h.rateLimit.reads, "check");
+ expect((await h.get(HOST_A)).statusCode).toBe(200);
+ expect((await h.put(HOST_A)).statusCode).toBe(200);
+ expect((await h.put(HOST_A)).statusCode).toBe(429);
+ // Exhausting the writes did not drain the reads...
+ expect((await h.get(HOST_A)).statusCode).toBe(200);
+ // ...and the reads are still capped.
+ const third = await h.get(HOST_A);
+ expect(third.statusCode).toBe(429);
+ expect(third.headers["X-GFS-RateLimit-Scope"]).toBe("agent_reads");
+ expect(readsCheck).toHaveBeenCalledTimes(3);
+ });
+
+ it("case 4: charges every attempt, including one that authorization denies", async () => {
+ const h = harness({ writes: 1, allowed: false });
+ vi.spyOn(console, "warn").mockImplementation(() => undefined);
+ const first = await h.put(HOST_A);
+ expect(first.statusCode).toBe(403);
+ expect(first.json.error?.code).toBe("forbidden");
+ const second = await h.put(HOST_A);
+ expect(second.statusCode).toBe(429);
+ expect(second.json.error?.code).toBe("rate_limited");
+ expect(h.replaces).toHaveLength(0);
+ });
+
+ it("case 5: meters each agent subject independently", async () => {
+ const h = harness({ reads: 1 });
+ vi.spyOn(console, "warn").mockImplementation(() => undefined);
+ expect((await h.get(HOST_A)).statusCode).toBe(200);
+ expect((await h.get(HOST_A)).statusCode).toBe(429);
+ expect((await h.get(HOST_A)).statusCode).toBe(429);
+ expect((await h.get(HOST_B)).statusCode).toBe(200);
+ expect(h.listChildrenBySubject).toEqual([HOST_A, HOST_B]);
+ });
+
+ it("case 6: restores the budget once the window has elapsed", async () => {
+ const h = harness({ reads: 1, windowMs: 60_000 });
+ vi.spyOn(console, "warn").mockImplementation(() => undefined);
+ expect((await h.get(HOST_A)).statusCode).toBe(200);
+ expect((await h.get(HOST_A)).statusCode).toBe(429);
+ expect(h.listChildren).toHaveBeenCalledTimes(1);
+ h.clock.t += 60_001;
+ expect((await h.get(HOST_A)).statusCode).toBe(200);
+ expect(h.listChildren).toHaveBeenCalledTimes(2);
+ });
+
+ it("case 7: reports one denial as exactly one warning and one counter increment", async () => {
+ const h = harness({ reads: 1 });
+ const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined);
+ expect((await h.get(HOST_A)).statusCode).toBe(200);
+ expect(warn).toHaveBeenCalledTimes(0);
+ expect(h.metrics.snapshot().rateLimitDenied).toEqual({ read: 0, write: 0 });
+ expect((await h.get(HOST_A)).statusCode).toBe(429);
+ expect(warn).toHaveBeenCalledTimes(1);
+ expect(h.metrics.snapshot().rateLimitDenied).toEqual({ read: 1, write: 0 });
+ expect(h.metrics.render()).toContain('gfs_rate_limit_denied_total{kind="read"} 1');
+ expect(h.metrics.render()).toContain('gfs_rate_limit_denied_total{kind="write"} 0');
+ });
+
+ it("case 8: logs a hashed subject and never the raw one", async () => {
+ const h = harness({ writes: 1 });
+ const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined);
+ expect((await h.put(HOST_A)).statusCode).toBe(200);
+ const denied = await h.put(HOST_A);
+ expect(denied.statusCode).toBe(429);
+ expect(warn).toHaveBeenCalledTimes(1);
+ const [payload] = warn.mock.calls[0] as [Record<string, unknown>];
+ expect(payload).toMatchObject({ event: "gfs_rate_limit_denied", kind: "write" });
+ expect(payload.hashedSubject).toMatch(/^[0-9a-f]{64}$/);
+ expect(JSON.stringify(warn.mock.calls[0])).not.toContain(HOST_A);
+ expect(JSON.stringify(warn.mock.calls[0])).not.toContain("agent-a");
+ expect(denied.body).not.toContain(HOST_A);
+ expect(denied.body).not.toContain("agent-a");
+ });
+
+ it("case 9: never charges user or linked-admin principals", async () => {
+ const h = harness({ reads: 2 });
+ vi.spyOn(console, "warn").mockImplementation(() => undefined);
+ const readsCheck = vi.spyOn(h.rateLimit.reads, "check");
+ for (let i = 0; i < 5; i += 1) {
+ expect((await h.get(USER_ID)).statusCode).toBe(200);
+ expect((await h.get("linked-admin")).statusCode).toBe(200);
+ }
+ expect(readsCheck).toHaveBeenCalledTimes(0);
+ expect(h.listChildren).toHaveBeenCalledTimes(10);
+ // The limiter is wired and live: a host subject is capped in the same test.
+ expect((await h.get(HOST_A)).statusCode).toBe(200);
+ expect((await h.get(HOST_A)).statusCode).toBe(200);
+ expect((await h.get(HOST_A)).statusCode).toBe(429);
+ expect(readsCheck).toHaveBeenCalledTimes(3);
+ });
+
+ it("case 10: reports Retry-After from the oldest hit still in the window", async () => {
+ const h = harness({ reads: 2, windowMs: 60_000 });
+ vi.spyOn(console, "warn").mockImplementation(() => undefined);
+ expect((await h.get(HOST_A)).statusCode).toBe(200);
+ h.clock.t += 20_000;
+ expect((await h.get(HOST_A)).statusCode).toBe(200);
+ h.clock.t += 10_000;
+ const denied = await h.get(HOST_A);
+ expect(denied.statusCode).toBe(429);
+ expect(denied.headers["Retry-After"]).toBe("30");
+ });
+
+ it("case 11: evicts subjects whose window has emptied", async () => {
+ const h = harness({ reads: 5, windowMs: 60_000 });
+ for (let i = 0; i < 50; i += 1) {
+ expect((await h.get(`host:1st:mcp-host/agent-${i}`)).statusCode).toBe(200);
+ }
+ expect(h.rateLimit.reads.trackedSubjectCount).toBe(50);
+ h.clock.t += 60_001;
+ expect((await h.get(HOST_A)).statusCode).toBe(200);
+ expect(h.rateLimit.reads.trackedSubjectCount).toBe(1);
+ });
+});
diff --git a/gfs-controller/src/api/serve.rename.test.ts b/gfs-controller/src/api/serve.rename.test.ts
index c5ccbd849..b87469c2d 100644
--- a/gfs-controller/src/api/serve.rename.test.ts
+++ b/gfs-controller/src/api/serve.rename.test.ts
@@ -4,6 +4,7 @@ import { Readable } from "node:stream";
import { describe, expect, it, vi } from "vitest";
import type { GfsVerifiedClaims } from "../auth/verify";
import type { AuthzContext } from "../authz/permissionClient";
+import { RateLimiter } from "../quota/rateLimit";
import { GfsServingHandler, type ServingDeps } from "./serve";
const RID = "00000000-0000-4000-8000-000000000011";
@@ -47,6 +48,10 @@ function fixture(options: { allowed?: boolean } = {}) {
writeService: { rename } as unknown as ServingDeps["writeService"],
audit: { record: async () => undefined },
rename: { maxObjects: 1000, timeoutMs: 30_000 },
+ rateLimit: {
+ reads: new RateLimiter({ limit: 1_000_000, windowMs: 60_000 }),
+ writes: new RateLimiter({ limit: 1_000_000, windowMs: 60_000 }),
+ },
};
return { deps, handler: new GfsServingHandler(deps), rename, operations };
}
diff --git a/gfs-controller/src/api/serve.test.ts b/gfs-controller/src/api/serve.test.ts
index 05a86cadc..9ba2f93c8 100644
--- a/gfs-controller/src/api/serve.test.ts
+++ b/gfs-controller/src/api/serve.test.ts
@@ -5,6 +5,7 @@ import { GfsAuthError, type GfsScope, type GfsVerifiedClaims } from "../auth/ver
import type { AuthzContext } from "../authz/permissionClient";
import type { GfsPermission } from "../authz/resolve";
import { GfsSubjectResolutionDeniedError } from "../authz/subjectResolver";
+import { RateLimiter } from "../quota/rateLimit";
import { GfsError } from "./errors";
import { GfsResource } from "./read";
import { GfsServingHandler, ServingDeps } from "./serve";
@@ -122,10 +123,19 @@ function deps(over: Partial<ServingDeps> = {}): ServingDeps {
listChildren: async () => [FILE],
},
blobs: { read: async () => Readable.from(Buffer.from("hello world")) },
+ rateLimit: unboundedRateLimit(),
...over,
};
}
+/** Agent budgets no test in this file reaches; serve.rateLimit.test.ts covers enforcement. */
+function unboundedRateLimit(): ServingDeps["rateLimit"] {
+ return {
+ reads: new RateLimiter({ limit: 1_000_000, windowMs: 60_000 }),
+ writes: new RateLimiter({ limit: 1_000_000, windowMs: 60_000 }),
+ };
+}
+
/** Run the handler against a FakeRes (cast to the ServerResponse the API wants). */
function run(d: ServingDeps, request: IncomingMessage, res: FakeRes): Promise<boolean> {
return new GfsServingHandler(d).tryHandle(request, res as unknown as ServerResponse);
diff --git a/gfs-controller/src/api/serve.ts b/gfs-controller/src/api/serve.ts
index cf66b13c0..92bcdd866 100644
--- a/gfs-controller/src/api/serve.ts
+++ b/gfs-controller/src/api/serve.ts
@@ -1,4 +1,4 @@
-import { randomUUID } from "node:crypto";
+import { createHash, randomUUID } from "node:crypto";
import type { IncomingMessage, ServerResponse } from "node:http";
import type { GfsVerifiedClaims } from "../auth/verify";
import type { AccessibleResourcePage } from "../authz/accessibleStore";
@@ -9,6 +9,7 @@ import { GfsSubjectResolutionDeniedError } from "../authz/subjectResolver";
import { checkTokenCeiling } from "../authz/tokenCeiling";
import type { GfsWriteService } from "../db/writeStore";
import type { GfsMetrics } from "../metrics";
+import { type RateLimiter, RateLimitExceededError } from "../quota/rateLimit";
import type {
GfsUploadPartGeometry,
GfsUploadSessionService,
@@ -60,13 +61,20 @@ export function sanitizeForLog(value: string): string {
*
* 1. Bearer token → verifyToken (sig/aud/exp, fail-closed → 401)
* 2. principal → subjects → resolveContext (DB; store error → 503)
- * 3. token CEILING → checkTokenCeiling (scopes + pathBindings; the
+ * 3. agent rate limit → rateLimit.reads / rateLimit.writes, charged for
+ * host principals only, per attempt (→ 429)
+ * 4. token CEILING → checkTokenCeiling (scopes + pathBindings; the
* upper bound the token itself carries → 403)
- * 4. permission STORE → PermissionClient.authorize (source of truth,
+ * 5. permission STORE → PermissionClient.authorize (source of truth,
* deny-by-default, audited, fail-closed → 503)
- * 5. read executor → stat / list / download (existence + bytes)
+ * 6. executor → stat / list / download / mutation
*
- * Steps 3 AND 4 must BOTH allow — "neither alone authorizes" (§522). Authz runs
+ * Step 3 runs before authorization, so a flood is bounded by attempts, not by
+ * successes, and a denied agent never reaches the permission store. User and
+ * linked-admin principals are not charged: they reach gfsc only through
+ * control-api, which meters every external GFS request before minting.
+ *
+ * Steps 4 AND 5 must BOTH allow — "neither alone authorizes" (§522). Authz runs
* BEFORE any metadata is revealed, so an absent resource and an unauthorized one
* are indistinguishable (both 403) — no existence leak. The `drive` is taken
* from the token claim, never from the query, so a token can only ever reach the
@@ -104,6 +112,8 @@ export interface ServingDeps {
/** Admission limits for the synchronous rename mutation; PATCH is not served without them. */
rename?: { maxObjects: number; timeoutMs: number };
metrics?: GfsMetrics;
+ /** Per-subject agent budgets (step 3 of the chain). Required: no caller can opt out. */
+ rateLimit: { reads: RateLimiter; writes: RateLimiter };
/** Injectable clock for deterministic latency tests; defaults to Date.now. */
now?: () => number;
/** Capability response is injected so PR1 can remain disabled and testable. */
@@ -314,6 +324,10 @@ export class GfsServingHandler {
);
const requestId = needsMutationRequestId ? requireRequestId(req) : undefined;
const ctx = await this.authContext(claims, req, requestId);
+ this.chargeAgentBudget(
+ ctx,
+ Boolean(resourceWrite || copyWrite || renameWrite || (uploadRoute && isWrite))
+ );
copyAbort?.signal.throwIfAborted();
if (capabilitiesMatch) {
@@ -558,6 +572,34 @@ export class GfsServingHandler {
}
}
+ /**
+ * Step 3: spend one unit of the agent's write or read budget. Only host
+ * principals are charged; after authContext a `host:` subject has matched the
+ * canonical host form. A denial is counted, logged with the subject hashed,
+ * and answered 429 with the limiter's own limit and retry-after.
+ */
+ private chargeAgentBudget(ctx: AuthzContext, write: boolean): void {
+ if (!isAgentSub(ctx.primarySubject)) return;
+ const kind = write ? "write" : "read";
+ const limiter = write ? this.deps.rateLimit.writes : this.deps.rateLimit.reads;
+ try {
+ limiter.check(ctx.primarySubject);
+ } catch (err) {
+ if (!(err instanceof RateLimitExceededError)) throw err;
+ this.deps.metrics?.recordRateLimitDenial(kind);
+ console.warn({
+ event: "gfs_rate_limit_denied",
+ kind,
+ hashedSubject: sanitizeForLog(createHash("sha256").update(ctx.primarySubject).digest("hex")),
+ });
+ throw new GfsError("rate_limited", "agent rate limit exceeded", undefined, {
+ retryAfterSeconds: err.retryAfterSeconds,
+ limit: write ? "agent_writes" : "agent_reads",
+ rateLimitLimit: limiter.limitPerWindow,
+ });
+ }
+ }
+
/** Resolve the principal to its subject set; a store failure is fail-closed 503. */
private async authContext(
claims: GfsVerifiedClaims,
diff --git a/gfs-controller/src/config.test.ts b/gfs-controller/src/config.test.ts
index c56690702..c39b217fd 100644
--- a/gfs-controller/src/config.test.ts
+++ b/gfs-controller/src/config.test.ts
@@ -258,3 +258,42 @@ describe('GFS_UPLOAD_V2 strict disabled contract', () => {
expect(loadConfig().uploadV2.stalePartLeaseMs).toBe(600001)
})
})
+
+describe('GFS_AGENT_*_RL_PER_MIN_PER_REPLICA', () => {
+ beforeEach(() => {
+ vi.stubEnv('GFS_STORAGE_ROLE', 'reader')
+ vi.stubEnv('GFS_DEV_MODE', 'true')
+ })
+ afterEach(() => vi.unstubAllEnvs())
+
+ it('uses the stated per-replica defaults only when the variables are absent', () => {
+ vi.stubEnv('GFS_AGENT_READ_RL_PER_MIN_PER_REPLICA', undefined)
+ vi.stubEnv('GFS_AGENT_WRITE_RL_PER_MIN_PER_REPLICA', undefined)
+ expect(loadConfig()).toMatchObject({
+ agentReadRlPerMinPerReplica: 300,
+ agentWriteRlPerMinPerReplica: 120,
+ })
+ })
+
+ it.each([
+ ['GFS_AGENT_READ_RL_PER_MIN_PER_REPLICA', 'agentReadRlPerMinPerReplica'],
+ ['GFS_AGENT_WRITE_RL_PER_MIN_PER_REPLICA', 'agentWriteRlPerMinPerReplica'],
+ ] as const)('%s accepts the operator value and the maximum', (name, key) => {
+ vi.stubEnv(name, '45')
+ expect(loadConfig()[key]).toBe(45)
+ vi.stubEnv(name, '60000')
+ expect(loadConfig()[key]).toBe(60000)
+ })
+
+ it.each([
+ 'GFS_AGENT_READ_RL_PER_MIN_PER_REPLICA',
+ 'GFS_AGENT_WRITE_RL_PER_MIN_PER_REPLICA',
+ ])('%s rejects values above the maximum and non-positive or non-integer values', (name) => {
+ for (const value of ['60001', '0', '-1', '1.5', 'many']) {
+ vi.stubEnv(name, value)
+ expect(() => loadConfig()).toThrow(
+ `[gfsc] ${name} must be an integer between 1 and 60000, got: ${value}`
+ )
+ }
+ })
+})
diff --git a/gfs-controller/src/config.ts b/gfs-controller/src/config.ts
index 668a06cae..85505b6c3 100644
--- a/gfs-controller/src/config.ts
+++ b/gfs-controller/src/config.ts
@@ -81,6 +81,10 @@ export interface GfsConfig {
blobCleanupSafetyWindowMs: number
blobCleanupIntervalMs: number
blobCleanupBatchSize: number
+ /** Per-subject agent (host principal) read budget per minute, per replica. */
+ agentReadRlPerMinPerReplica: number
+ /** Per-subject agent (host principal) write budget per minute, per replica. */
+ agentWriteRlPerMinPerReplica: number
/** Maximum source objects admitted by one synchronous Copy request. */
syncCopyMaxObjects: number
/** Maximum observed source bytes admitted by one synchronous Copy request. */
@@ -112,6 +116,11 @@ function tuningMs(name: string, defaultMs: number): number {
return value
}
+export const AGENT_READ_RL_PER_MIN_DEFAULT = 300
+export const AGENT_WRITE_RL_PER_MIN_DEFAULT = 120
+/** 1000 requests per second per replica; a larger value is a typo, not a budget. */
+export const AGENT_RL_PER_MIN_MAX = 60000
+
function positiveInteger(name: string, defaultValue: number, maximum: number): number {
const raw = process.env[name]
if (raw === undefined || raw === '') return defaultValue
@@ -388,6 +397,18 @@ export function loadConfig(): GfsConfig {
2147483647
),
blobCleanupBatchSize: positiveInteger('GFS_BLOB_CLEANUP_BATCH_SIZE', 100, 10000),
+ // The operator sets both explicitly (host-context-controller gfscEnv); the
+ // defaults cover a gfsc image that rolls out before the operator's env.
+ agentReadRlPerMinPerReplica: positiveInteger(
+ 'GFS_AGENT_READ_RL_PER_MIN_PER_REPLICA',
+ AGENT_READ_RL_PER_MIN_DEFAULT,
+ AGENT_RL_PER_MIN_MAX
+ ),
+ agentWriteRlPerMinPerReplica: positiveInteger(
+ 'GFS_AGENT_WRITE_RL_PER_MIN_PER_REPLICA',
+ AGENT_WRITE_RL_PER_MIN_DEFAULT,
+ AGENT_RL_PER_MIN_MAX
+ ),
syncCopyMaxObjects: syncCopyPositiveInteger('GFS_SYNC_COPY_MAX_OBJECTS', 1000),
syncCopyMaxBytes: syncCopyPositiveInteger('GFS_SYNC_COPY_MAX_BYTES', 1073741824),
syncCopyTimeoutMs: syncCopyPositiveInteger('GFS_SYNC_COPY_TIMEOUT_MS', 30000),
diff --git a/gfs-controller/src/index.ts b/gfs-controller/src/index.ts
index b8e475006..5b48ed395 100644
--- a/gfs-controller/src/index.ts
+++ b/gfs-controller/src/index.ts
@@ -18,6 +18,7 @@ import { PgResourceStore } from "./db/resourceStore";
import { GfsWriteService, PgTransactor } from "./db/writeStore";
import { PgBlobStagingStore, reconcileExpiredBlobs } from "./db/blobStaging";
import { GfsMetrics } from "./metrics";
+import { RateLimiter } from "./quota/rateLimit";
import { GfsServer, ReadinessDeps } from "./server";
import { BlobStore } from "./storage/blobStore";
import { GfsUploadSessionService, uploadCapabilities } from "./upload/uploadSession";
@@ -154,6 +155,11 @@ async function main(): Promise<void> {
// serve authorized reads without one. In production an absent key is fatal
// (fail-loud crash); dev mode may run probes-only with a loud warning.
const metrics = new GfsMetrics();
+ // Both roles enforce the agent budgets: a reader serves agent reads too.
+ const rateLimit = {
+ reads: new RateLimiter({ limit: config.agentReadRlPerMinPerReplica, windowMs: 60_000 }),
+ writes: new RateLimiter({ limit: config.agentWriteRlPerMinPerReplica, windowMs: 60_000 }),
+ };
let serving: GfsServingHandler | undefined;
let invalidation: { stop: () => Promise<void> } | undefined;
let cleanupTimer: NodeJS.Timeout | undefined;
@@ -224,6 +230,7 @@ async function main(): Promise<void> {
},
} : {}),
metrics,
+ rateLimit,
// Upload mutation and capability advertisement are writer-only. Reader
// replicas deliberately do not expose a v2 capability even if an
// operator accidentally carries the same tuning env into both pods.
diff --git a/gfs-controller/src/metrics.ts b/gfs-controller/src/metrics.ts
index d48daa859..f23b9f934 100644
--- a/gfs-controller/src/metrics.ts
+++ b/gfs-controller/src/metrics.ts
@@ -16,6 +16,7 @@ export interface SliSnapshot {
orphanCandidates: number
orphanBytes: number
blobCleanupFailures: number
+ rateLimitDenied: { read: number; write: number }
}
function percentile(values: number[], p: number): number {
@@ -37,6 +38,7 @@ export class GfsMetrics {
private orphanCandidates = 0
private orphanBytes = 0
private blobCleanupFailures = 0
+ private readonly rateLimitDenied = { read: 0, write: 0 }
/** Cap retained latency samples so a long-running writer cannot grow unbounded. */
private static readonly MAX_LAT_SAMPLES = 10_000
@@ -73,6 +75,10 @@ export class GfsMetrics {
recordBlobCleanupFailure(): void {
this.blobCleanupFailures += 1
}
+ /** One agent request denied by the per-subject read or write budget. */
+ recordRateLimitDenial(kind: 'read' | 'write'): void {
+ this.rateLimitDenied[kind] += 1
+ }
snapshot(): SliSnapshot {
const totalCache = this.cacheHits + this.cacheMisses
@@ -87,6 +93,7 @@ export class GfsMetrics {
orphanCandidates: this.orphanCandidates,
orphanBytes: this.orphanBytes,
blobCleanupFailures: this.blobCleanupFailures,
+ rateLimitDenied: { ...this.rateLimitDenied },
}
}
@@ -105,6 +112,8 @@ export class GfsMetrics {
`gfs_blob_orphan_candidates ${s.orphanCandidates}`,
`gfs_blob_orphan_bytes ${s.orphanBytes}`,
`gfs_blob_cleanup_failures_total ${s.blobCleanupFailures}`,
+ `gfs_rate_limit_denied_total{kind="read"} ${s.rateLimitDenied.read}`,
+ `gfs_rate_limit_denied_total{kind="write"} ${s.rateLimitDenied.write}`,
]
return lines.join('\n') + '\n'
}
diff --git a/gfs-controller/src/quota/rateLimit.ts b/gfs-controller/src/quota/rateLimit.ts
index bc9ff9685..be9ea6b58 100644
--- a/gfs-controller/src/quota/rateLimit.ts
+++ b/gfs-controller/src/quota/rateLimit.ts
@@ -1,16 +1,30 @@
import { QuotaError } from "./bytes.js";
+/**
+ * A `rate_limited` denial that carries when the caller may retry. The message
+ * never names the subject: the limiter is keyed by a host identity and the
+ * denial can reach the caller.
+ */
+export class RateLimitExceededError extends QuotaError {
+ constructor(readonly retryAfterSeconds: number) {
+ super("rate_limited", "rate limit exceeded");
+ this.name = "RateLimitExceededError";
+ }
+}
+
/**
* Per-subject sliding-window rate limiter (spec §gfs-controller Quotas; plan
- * P4-S03). Caps mint + write ops per subject so a compromised agent cannot flood
- * gfsc; exceeding → rate_limited. In-memory per reader; the durable counter is a
- * follow-up. Clock is injectable for deterministic tests.
+ * P4-S03). Caps an agent's reads or writes so a compromised agent cannot flood
+ * gfsc; exceeding → rate_limited. In-memory per process, so a budget is per
+ * replica; the durable counter is a follow-up. Clock is injectable for
+ * deterministic tests.
*/
export class RateLimiter {
private readonly hits = new Map<string, number[]>();
private readonly limit: number;
private readonly windowMs: number;
private readonly now: () => number;
+ private lastSweep: number;
constructor(opts: { limit: number; windowMs: number; now?: () => number }) {
if (!(opts.limit > 0) || !(opts.windowMs > 0)) {
@@ -19,17 +33,53 @@ export class RateLimiter {
this.limit = opts.limit;
this.windowMs = opts.windowMs;
this.now = opts.now ?? Date.now;
+ this.lastSweep = this.now();
+ }
+
+ get limitPerWindow(): number {
+ return this.limit;
}
- /** Record a hit for `subject`; throw rate_limited if over the limit in the window. */
+ get windowLengthMs(): number {
+ return this.windowMs;
+ }
+
+ /** Number of subjects currently holding a window. Read by tests to observe eviction. */
+ get trackedSubjectCount(): number {
+ return this.hits.size;
+ }
+
+ /**
+ * Record a hit for `subject`; throw RateLimitExceededError if the window is
+ * full. Retry-after is when the oldest retained hit leaves the window, the
+ * exact moment one slot frees.
+ */
check(subject: string): void {
const now = this.now();
const cutoff = now - this.windowMs;
+ this.evictExpired(now, cutoff);
const recent = (this.hits.get(subject) ?? []).filter((t) => t > cutoff);
if (recent.length >= this.limit) {
- throw new QuotaError("rate_limited", `rate limit exceeded for ${subject}`);
+ this.hits.set(subject, recent);
+ const oldest = Math.min(...recent);
+ throw new RateLimitExceededError(
+ Math.max(1, Math.ceil((oldest + this.windowMs - now) / 1000)),
+ );
}
recent.push(now);
this.hits.set(subject, recent);
}
+
+ /**
+ * Delete every subject whose newest hit has left the window. Runs at most once
+ * per window length, inside check(), so the map is bounded by the subjects
+ * active in the last two windows without a timer in the data path.
+ */
+ private evictExpired(now: number, cutoff: number): void {
+ if (now - this.lastSweep < this.windowMs) return;
+ for (const [subject, times] of this.hits) {
+ if (Math.max(...times) <= cutoff) this.hits.delete(subject);
+ }
+ this.lastSweep = now;
+ }
}
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index eba90ec51..70da79a11 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.348",
+ "version": "0.1.349",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.348",
+ "version": "0.1.349",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index ba549ad76..a82cc17fe 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.348",
+ "version": "0.1.349",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/gfsConfig.test.ts b/host-context-controller/src/gfsConfig.test.ts
index f0640671d..a2e73e5e5 100644
--- a/host-context-controller/src/gfsConfig.test.ts
+++ b/host-context-controller/src/gfsConfig.test.ts
@@ -16,6 +16,36 @@ afterEach(() => {
}
delete process.env.CONTEXT_MAPPER_GFSC_IMAGE_PULL_POLICY
delete process.env.CONTEXT_MAPPER_NODELOCAL_DNS_CIDR
+ delete process.env.CONTEXT_MAPPER_GFSC_AGENT_READ_RL_PER_MIN_PER_REPLICA
+ delete process.env.CONTEXT_MAPPER_GFSC_AGENT_WRITE_RL_PER_MIN_PER_REPLICA
+})
+
+describe('gfsDefaultFactoryConfig agent rate-limit budgets', () => {
+ it('sets the stated per-replica defaults when the variables are absent', () => {
+ const c = gfsDefaultFactoryConfig()
+ expect(c.agentReadRlPerMinPerReplica).toBe(300)
+ expect(c.agentWriteRlPerMinPerReplica).toBe(120)
+ })
+
+ it('takes explicit operator values', () => {
+ process.env.CONTEXT_MAPPER_GFSC_AGENT_READ_RL_PER_MIN_PER_REPLICA = '600'
+ process.env.CONTEXT_MAPPER_GFSC_AGENT_WRITE_RL_PER_MIN_PER_REPLICA = '60000'
+ const c = gfsDefaultFactoryConfig()
+ expect(c.agentReadRlPerMinPerReplica).toBe(600)
+ expect(c.agentWriteRlPerMinPerReplica).toBe(60000)
+ })
+
+ it.each([
+ 'CONTEXT_MAPPER_GFSC_AGENT_READ_RL_PER_MIN_PER_REPLICA',
+ 'CONTEXT_MAPPER_GFSC_AGENT_WRITE_RL_PER_MIN_PER_REPLICA',
+ ])('%s fails the operator at startup on a value gfsc would reject', key => {
+ for (const value of ['60001', '0', '-5', '2.5', 'lots']) {
+ process.env[key] = value
+ expect(() => gfsDefaultFactoryConfig()).toThrow(
+ `[gfs] ${key} must be an integer between 1 and 60000, got ${JSON.stringify(value)}`
+ )
+ }
+ })
})
describe('gfsDefaultFactoryConfig', () => {
diff --git a/host-context-controller/src/gfsConfig.ts b/host-context-controller/src/gfsConfig.ts
index f2e751cb8..a6adc60e7 100644
--- a/host-context-controller/src/gfsConfig.ts
+++ b/host-context-controller/src/gfsConfig.ts
@@ -1,4 +1,11 @@
-import { DEFAULT_GFSC_PORT, DEFAULT_INIT_IMAGE, GfsFactoryConfig } from './k8s/gfsFactory'
+import {
+ DEFAULT_GFSC_AGENT_READ_RL_PER_MIN_PER_REPLICA,
+ DEFAULT_GFSC_AGENT_WRITE_RL_PER_MIN_PER_REPLICA,
+ DEFAULT_GFSC_PORT,
+ DEFAULT_INIT_IMAGE,
+ GfsFactoryConfig,
+ MAX_GFSC_AGENT_RL_PER_MIN_PER_REPLICA,
+} from './k8s/gfsFactory'
import { parseNodeLocalDnsCidr } from './k8sApiCidrs'
/**
@@ -24,6 +31,26 @@ function envInt(key: string, fallback: number): number {
return parsed
}
+/**
+ * A gfsc agent budget. Validated here with gfsc's own bounds so a bad value fails
+ * the operator at startup instead of crash-looping every gfsc pod it reaches.
+ */
+function envAgentBudget(key: string, fallback: number): number {
+ const value = process.env[key]
+ if (value === undefined || value === '') return fallback
+ const parsed = Number(value)
+ if (
+ !Number.isSafeInteger(parsed) ||
+ parsed <= 0 ||
+ parsed > MAX_GFSC_AGENT_RL_PER_MIN_PER_REPLICA
+ ) {
+ throw new Error(
+ `[gfs] ${key} must be an integer between 1 and ${MAX_GFSC_AGENT_RL_PER_MIN_PER_REPLICA}, got ${JSON.stringify(value)}`
+ )
+ }
+ return parsed
+}
+
function envPullPolicy(key: string): 'Always' | 'IfNotPresent' | 'Never' {
const value = process.env[key]
if (value === undefined || value === '') return 'IfNotPresent'
@@ -64,6 +91,14 @@ export function gfsDefaultFactoryConfig(): GfsFactoryConfig {
readerPgSecretKey: env('CONTEXT_MAPPER_GFS_READER_PG_SECRET_KEY', 'connection-string'),
driveName: env('CONTEXT_MAPPER_GFS_DRIVE_NAME', 'main'),
tokenAudience: env('CONTEXT_MAPPER_GFS_TOKEN_AUDIENCE', 'gfs-controller'),
+ agentReadRlPerMinPerReplica: envAgentBudget(
+ 'CONTEXT_MAPPER_GFSC_AGENT_READ_RL_PER_MIN_PER_REPLICA',
+ DEFAULT_GFSC_AGENT_READ_RL_PER_MIN_PER_REPLICA
+ ),
+ agentWriteRlPerMinPerReplica: envAgentBudget(
+ 'CONTEXT_MAPPER_GFSC_AGENT_WRITE_RL_PER_MIN_PER_REPLICA',
+ DEFAULT_GFSC_AGENT_WRITE_RL_PER_MIN_PER_REPLICA
+ ),
syncCopyMaxObjects: process.env.GFS_SYNC_COPY_MAX_OBJECTS,
syncCopyMaxBytes: process.env.GFS_SYNC_COPY_MAX_BYTES,
syncCopyTimeoutMs: process.env.GFS_SYNC_COPY_TIMEOUT_MS,
diff --git a/host-context-controller/src/gfsReconciler.test.ts b/host-context-controller/src/gfsReconciler.test.ts
index 77519b85b..bc7ab97ff 100644
--- a/host-context-controller/src/gfsReconciler.test.ts
+++ b/host-context-controller/src/gfsReconciler.test.ts
@@ -28,6 +28,8 @@ const config: GfsFactoryConfig = {
readerPgSecretKey: 'connection-string',
driveName: 'main',
tokenAudience: 'gfs-controller',
+ agentReadRlPerMinPerReplica: 300,
+ agentWriteRlPerMinPerReplica: 120,
}
const gfs: GlobalFileSystemCRD = {
diff --git a/host-context-controller/src/k8s/gfsFactory.test.ts b/host-context-controller/src/k8s/gfsFactory.test.ts
index f4376239e..e8dd71082 100644
--- a/host-context-controller/src/k8s/gfsFactory.test.ts
+++ b/host-context-controller/src/k8s/gfsFactory.test.ts
@@ -39,6 +39,8 @@ const config: GfsFactoryConfig = {
readerPgSecretKey: 'connection-string',
driveName: 'main',
tokenAudience: 'gfs-controller',
+ agentReadRlPerMinPerReplica: 300,
+ agentWriteRlPerMinPerReplica: 120,
}
function gfs(spec: GlobalFileSystemCRD['spec'] = {}): GlobalFileSystemCRD {
@@ -221,6 +223,30 @@ describe('gfsFactory writer Deployment', () => {
}
})
+ it('sets both agent rate-limit budgets from the operator config on every role', () => {
+ const configured = {
+ ...config,
+ agentReadRlPerMinPerReplica: 45,
+ agentWriteRlPerMinPerReplica: 7,
+ }
+ const roles = ['writer', 'reader'] as const
+ const seen: string[] = []
+ for (const role of roles) {
+ const env =
+ buildDeployment(gfs(), configured, role).spec?.template.spec?.containers[0].env ?? []
+ const byName = (name: string) => env.filter(item => item.name === name)
+ expect(byName('GFS_STORAGE_ROLE')[0]?.value).toBe(role)
+ expect(byName('GFS_AGENT_READ_RL_PER_MIN_PER_REPLICA')).toEqual([
+ { name: 'GFS_AGENT_READ_RL_PER_MIN_PER_REPLICA', value: '45' },
+ ])
+ expect(byName('GFS_AGENT_WRITE_RL_PER_MIN_PER_REPLICA')).toEqual([
+ { name: 'GFS_AGENT_WRITE_RL_PER_MIN_PER_REPLICA', value: '7' },
+ ])
+ seen.push(role)
+ }
+ expect(seen).toEqual(['writer', 'reader'])
+ })
+
it('runs non-root, drops all caps, read-only rootfs (reconciler owns securityContext)', () => {
const pod = dep.spec?.template.spec
expect(pod?.securityContext?.runAsNonRoot).toBe(true)
diff --git a/host-context-controller/src/k8s/gfsFactory.ts b/host-context-controller/src/k8s/gfsFactory.ts
index be06825cf..1076a3d3c 100644
--- a/host-context-controller/src/k8s/gfsFactory.ts
+++ b/host-context-controller/src/k8s/gfsFactory.ts
@@ -39,6 +39,15 @@ export const DEFAULT_GFS_ACCESS_MODES: ReadonlyArray<string> = ['ReadWriteOnce']
export const DEFAULT_GFS_STORAGE_CLASS = 'standard-rwo'
export const DEFAULT_INIT_IMAGE = 'busybox:1.36'
export const DEFAULT_GFSC_PORT = 8087
+/**
+ * Per-subject agent budgets gfsc enforces, per replica (its limiter is in-memory).
+ * The writer is one replica, so 120 writes/min is the fleet bound; reads land on
+ * any of the pods behind the read Service, so the fleet read bound is 300 x pods.
+ * gfsc rejects values above 60000 at startup.
+ */
+export const DEFAULT_GFSC_AGENT_READ_RL_PER_MIN_PER_REPLICA = 300
+export const DEFAULT_GFSC_AGENT_WRITE_RL_PER_MIN_PER_REPLICA = 120
+export const MAX_GFSC_AGENT_RL_PER_MIN_PER_REPLICA = 60000
export const DEFAULT_READER_REPLICAS = 2
export const DEFAULT_RUN_AS_UID = 1000
export const DEFAULT_RUN_AS_GID = 1000
@@ -91,6 +100,10 @@ export interface GfsFactoryConfig {
driveName: string
/** Expected audience on inbound gfs access tokens. */
tokenAudience: string
+ /** Agent (host principal) read budget per minute per gfsc replica; set on every pod. */
+ agentReadRlPerMinPerReplica: number
+ /** Agent (host principal) write budget per minute per gfsc replica; set on every pod. */
+ agentWriteRlPerMinPerReplica: number
/** Optional synchronous copy limits passed through verbatim to gfsc. */
syncCopyMaxObjects?: string
syncCopyMaxBytes?: string
@@ -316,6 +329,15 @@ function gfscEnv(config: GfsFactoryConfig, role: GfscRole): k8s.V1EnvVar[] {
{ name: 'GFS_STORAGE_ROLE', value: role },
{ name: 'GFS_DRIVE_NAME', value: config.driveName },
{ name: 'GFS_TOKEN_AUDIENCE', value: config.tokenAudience },
+ // Both roles serve agent traffic, so both carry both budgets.
+ {
+ name: 'GFS_AGENT_READ_RL_PER_MIN_PER_REPLICA',
+ value: String(config.agentReadRlPerMinPerReplica),
+ },
+ {
+ name: 'GFS_AGENT_WRITE_RL_PER_MIN_PER_REPLICA',
+ value: String(config.agentWriteRlPerMinPerReplica),
+ },
...(role === 'writer' && config.uploadV2Enabled !== undefined
? [{ name: 'GFS_UPLOAD_V2_ENABLED', value: config.uploadV2Enabled }]
: []),
diff --git a/host-context-controller/src/k8s/gfsK8sApi.readFirst.test.ts b/host-context-controller/src/k8s/gfsK8sApi.readFirst.test.ts
index 29689de64..0a011ea48 100644
--- a/host-context-controller/src/k8s/gfsK8sApi.readFirst.test.ts
+++ b/host-context-controller/src/k8s/gfsK8sApi.readFirst.test.ts
@@ -374,6 +374,8 @@ const readerProducerConfig: GfsFactoryConfig = {
readerPgSecretKey: 'connection-string',
driveName: 'main',
tokenAudience: 'gfs-controller',
+ agentReadRlPerMinPerReplica: 300,
+ agentWriteRlPerMinPerReplica: 120,
}
function producedReader(): k8s.V1Deployment {