Skip to content

The open-source app everyone uses to manage agents at work

feat(evals): route Runner sessions through a controller gateway (#17440)

Dottad1fdfced9d28
diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md
index b7f8efad0c..a7114ed7c3 100644
--- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md
+++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md
@@ -104,6 +104,61 @@ Claude Managed uses its Agent version, while AgentCore uses its qualification
revision. Transport API or beta versions remain separate protocol metadata and
must not replace that runtime identity in a report.
+## Controller-assigned gateway route
+
+The Runner eval CLI accepts `--gateway-config /absolute/path/gateway.json` for
+native Codex and ACPX Claude. This is an explicit route for a controller
+that owns a local budget proxy. Normal invocations keep the direct path.
+ACPX Codex is rejected before service creation because its runtime prohibits
+preconfigured homes to protect Runner state.
+The descriptor must bind the exact model and attempt ID from `request.json`:
+
+```json
+{
+ "schema": "paperclip.runner-eval.gateway/v1",
+ "provider": "openai",
+ "model": "gpt-5.4-mini",
+ "baseUrl": "http://127.0.0.1:4567",
+ "campaignId": "campaign-id",
+ "executionId": "cell-id",
+ "attemptId": "attempt-id",
+ "upstreamRoute": "direct"
+}
+```
+
+Use `anthropic` for ACPX Claude. `upstreamRoute` is the controller's declaration:
+`direct` or `braintrust`. It is not an independent verification of the upstream.
+The base URL must be the proxy's explicit IPv4 loopback address and port, with
+no path, query, fragment, or embedded credential. The controller supplies its
+32-byte hexadecimal token through `PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN`.
+Do not put provider or Braintrust credentials in this child environment or file.
+
+```sh
+node /path/to/runner/dist/cli/eval-session.js \
+ --request /path/to/request.json --output /path/to/artifact.json \
+ --gateway-config /path/to/gateway.json
+```
+
+The CLI validates the descriptor before creating a provider service. It creates
+an owned temporary home, pins Codex's Responses endpoint or Claude's Messages
+endpoint and exact model, and passes only the assigned proxy credential.
+It does not copy ambient provider keys, subscription logins, proxy settings,
+startup hooks, or user homes into this route. Unsupported harnesses and
+mismatched descriptor identities fail before provider creation. The temporary
+home is removed on success and failure.
+
+Success and failure artifacts retain `requestRouting`: controller gateway mode,
+provider, declared upstream route, campaign/execution/attempt IDs, and the
+SHA-256 of the descriptor bytes. The token and endpoint are excluded from this
+metadata. Existing artifact usage and provider outcomes remain authoritative
+for their respective fields; the route declaration is not a billing receipt.
+The controller must retain its request ledger and link these identities to it.
+
+This entry point does not itself allocate a campaign budget, run the proxy, or
+change the hosted matrix. The controller must enforce request admission,
+deadlines, and limits before forwarding traffic. A local fake-API probe only
+proves routing; it does not qualify live provider or Braintrust behavior.
+
## Hosted campaign
Use the `Runner Direct Live Protocol Evals` workflow. Dispatch the workflow
diff --git a/packages/paperclip-runner/src/cli/eval-gateway-runtime.test.ts b/packages/paperclip-runner/src/cli/eval-gateway-runtime.test.ts
new file mode 100644
index 0000000000..cd1cb1e03e
--- /dev/null
+++ b/packages/paperclip-runner/src/cli/eval-gateway-runtime.test.ts
@@ -0,0 +1,156 @@
+import { createHash } from "node:crypto";
+import { existsSync } from "node:fs";
+import { chmod, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, expect, it, vi } from "vitest";
+import { createSanitizedCodexEnvironment } from "../drivers/codex/app-server-transport.js";
+import { createSanitizedAcpxSpawnInput } from "../drivers/acpx/environment.js";
+import { createCapabilityFixtureState } from "../mock-core/capability-control-plane-types.js";
+import type { CapabilityLiveSessionSnapshot } from "../live/live-session.js";
+import { parseEvalSessionRequest } from "./eval-session-contract.js";
+import { prepareEvalGatewayRuntime } from "./eval-gateway-runtime.js";
+import { runEvalSessionCli } from "./eval-session.js";
+
+const roots: string[] = [];
+afterEach(async () => {
+ vi.unstubAllEnvs();
+ for (const root of roots.splice(0)) {
+ for (const entry of await readdir(root).catch(() => [])) {
+ if (entry.startsWith(".paperclip-eval-runtime-context-")) await chmod(join(root, entry, "instructions"), 0o700).catch(() => undefined);
+ }
+ await rm(root, { recursive: true, force: true });
+ }
+});
+const token = "a".repeat(64);
+async function fixture(provider = "codex", acpxAgent?: string) {
+ const root = await mkdtemp(join(tmpdir(), "runner-gateway-test-")); roots.push(root);
+ const model = acpxAgent === "claude" ? "claude-sonnet-5" : "gpt-5.6-sol";
+ const binary = join(root, "runnerd");
+ await writeFile(binary, "fake runnerd");
+ const request = parseEvalSessionRequest({
+ schema: "paperclip-runner/eval-session-request/v1", attemptId: "attempt-1", prompt: "Read the governed task.",
+ provider, ...(acpxAgent ? { acpxAgent } : {}), model,
+ runnerd: { path: binary, sha256: createHash("sha256").update("fake runnerd").digest("hex") },
+ limits: { turnTimeoutMs: 120000, maxAgentTurns: 1, maxEstimatedCostNanodollars: 100000000 },
+ session: { workingDirectory: root },
+ });
+ const descriptor = { schema: "paperclip.runner-eval.gateway/v1", provider: acpxAgent === "claude" ? "anthropic" : "openai",
+ model, baseUrl: "http://127.0.0.1:4567", campaignId: "campaign-1", executionId: "cell-1", attemptId: "attempt-1", upstreamRoute: "braintrust" };
+ const descriptorPath = join(root, "gateway.json");
+ await writeFile(descriptorPath, JSON.stringify(descriptor));
+ return { root, request, descriptor, descriptorPath };
+}
+
+it.each([["codex", undefined], ["acpx", "claude"]])("routes %s/%s with only its proxy credential and an owned home", async (provider, agent) => {
+ const f = await fixture(provider, agent);
+ const gateway = await prepareEvalGatewayRuntime(f.descriptorPath, f.request, {
+ PATH: process.env.PATH, HOME: "/ambient/home", CODEX_HOME: "/ambient/codex", CLAUDE_CONFIG_DIR: "/ambient/claude",
+ PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: token, OPENAI_API_KEY: "ambient-openai", CODEX_API_KEY: "ambient-codex",
+ ANTHROPIC_API_KEY: "ambient-anthropic", ANTHROPIC_AUTH_TOKEN: "ambient-oauth", CLAUDE_CODE_OAUTH_TOKEN: "ambient-claude",
+ BRAINTRUST_API_KEY: "ambient-braintrust", AWS_SECRET_ACCESS_KEY: "ambient-aws", GH_TOKEN: "ambient-github",
+ HTTPS_PROXY: "https://ambient-proxy", NODE_OPTIONS: "--require ambient-hook", PAPERCLIP_CONFIGURED_ENVIRONMENT_KEYS: '["GH_TOKEN"]',
+ });
+ const env = gateway.transportOptions.environment;
+ try {
+ expect(gateway.transportOptions.homeMode).toBe("preconfigured");
+ expect(JSON.stringify(env)).not.toContain("ambient");
+ expect(env.OPENAI_API_KEY).toBeUndefined();
+ expect(env.CODEX_API_KEY).toBeUndefined();
+ expect(env.BRAINTRUST_API_KEY).toBeUndefined();
+ expect(env.PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN).toBeUndefined();
+ const child = provider === "codex" ? createSanitizedCodexEnvironment(env) : createSanitizedAcpxSpawnInput(env, agent as "codex" | "claude").env;
+ expect(child.HOME).toBe(env.HOME);
+ if (agent === "claude") {
+ expect(child.ANTHROPIC_API_KEY).toBe(token);
+ expect(child.ANTHROPIC_BASE_URL).toBe(f.descriptor.baseUrl);
+ expect(child.CLAUDE_CODE_SUBAGENT_MODEL).toBe(f.request.model);
+ expect(JSON.parse(await readFile(join(env.CLAUDE_CONFIG_DIR!, "settings.json"), "utf8"))).toEqual({ model: f.request.model, availableModels: [f.request.model] });
+ } else {
+ expect(child.PAPERCLIP_AI_PROVIDER_KEY).toBe(token);
+ const config = await readFile(join(env.CODEX_HOME!, "config.toml"), "utf8");
+ expect(config).toContain('base_url = "http://127.0.0.1:4567/v1"');
+ expect(config).toContain('env_key = "PAPERCLIP_AI_PROVIDER_KEY"');
+ expect(config).toContain("requires_openai_auth = false");
+ expect(config).toContain("shell_snapshot = false");
+ expect(config).not.toContain(token);
+ expect((await stat(join(env.CODEX_HOME!, "config.toml"))).mode & 0o777).toBe(0o600);
+ }
+ expect(JSON.stringify(gateway.evidence)).not.toContain(token);
+ expect(gateway.evidence).toMatchObject({ mode: "controller_gateway", attemptId: f.request.attemptId, declaredUpstreamRoute: "braintrust" });
+ } finally { await gateway.close(); }
+ expect(existsSync(env.HOME!)).toBe(false);
+});
+
+it.each([
+ { baseUrl: "https://api.openai.com" }, { baseUrl: "http://localhost:4567" },
+ { baseUrl: "http://127.0.0.1:4567/v1" }, { baseUrl: "http://127.0.0.1:99999" },
+ { baseUrl: "http://127.0.0.1:4567?token=secret" }, { attemptId: "another-attempt" },
+ { model: "another-model" }, { campaignId: "bad identity" }, { unexpectedSecret: "must-not-print" },
+])("rejects invalid descriptor binding %j", async (change) => {
+ const f = await fixture();
+ await writeFile(f.descriptorPath, JSON.stringify({ ...f.descriptor, ...change }));
+ await expect(prepareEvalGatewayRuntime(f.descriptorPath, f.request, { PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: token }))
+ .rejects.toThrow("Invalid controller gateway descriptor");
+});
+
+it("rejects missing tokens and unsupported provider routes before runtime creation", async () => {
+ const f = await fixture();
+ await expect(prepareEvalGatewayRuntime(f.descriptorPath, f.request, {})).rejects.toThrow("gateway token");
+ await expect(prepareEvalGatewayRuntime(f.descriptorPath, { ...f.request, provider: "opencode" }, { PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: token })).rejects.toThrow("does not support");
+ await writeFile(f.descriptorPath, JSON.stringify({ ...f.descriptor, provider: "anthropic" }));
+ await expect(prepareEvalGatewayRuntime(f.descriptorPath, f.request, { PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: token })).rejects.toThrow("does not support");
+});
+
+it.each([false, true])("the real CLI retains route identity and removes the owned home after failure=%s", async (fail) => {
+ const f = await fixture();
+ vi.stubEnv("PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN", token);
+ vi.stubEnv("OPENAI_API_KEY", "upstream-must-not-reach-provider");
+ const requestPath = join(f.root, "request.json"), outputPath = join(f.root, "artifact.json");
+ await writeFile(requestPath, JSON.stringify(f.request));
+ const state = JSON.stringify(createCapabilityFixtureState({}));
+ const snapshot = {
+ sessionId: "session-1", revision: 1, providerThreadId: "provider-1", providerSessionId: "provider-1",
+ providerModel: { id: f.request.model, provider: "openai" }, status: "idle", activeTurnId: null,
+ createdAt: "2026-10-24T00:00:00Z", updatedAt: "2026-10-24T00:00:01Z",
+ authority: { companyId: "company-1", actorId: "actor-1", taskId: "task-1", runId: "run-1", sessionId: "session-1", scenarioId: "gateway-test" },
+ config: { provider: "codex", driver: "codex_app_server", seedState: state, workingDirectory: f.root, scenario: { id: "gateway-test" }, capabilities: [], explicitClaims: [], turnTimeoutMs: 1000 },
+ mockState: state, process: null, networkEvidence: { realPaperclipRequests: 0, childPaperclipEnvironmentKeys: [] },
+ transcript: [], evidence: [], authorizationRecords: [], attempts: [], terminalTurns: [{ turnId: "turn-1", status: "completed" }],
+ usageLedger: [{ receiptId: "receipt-1", turnId: "turn-1", providerCalls: 1, providerRequests: 1, inputTokens: 100, outputTokens: 10, cachedInputTokens: 0, reasoningTokens: 0, costNanodollars: 0 }], usageUnavailable: [],
+ } as unknown as CapabilityLiveSessionSnapshot;
+ let home: string | undefined;
+ const factory = vi.fn((_binary, options) => {
+ expect(options.homeMode).toBe("preconfigured");
+ expect(options.environment.OPENAI_API_KEY).toBeUndefined();
+ expect(options.environment.PAPERCLIP_AI_PROVIDER_KEY).toBe(token);
+ home = options.environment.HOME;
+ expect(existsSync(home!)).toBe(true);
+ return { create: async () => ({
+ sendMessage: async () => { if (fail) throw new Error("simulated provider failure"); return { turnId: "turn-1", status: "completed", snapshot }; },
+ completeAttempt: async () => undefined, shutdown: async () => { snapshot.status = "closed"; }, snapshot: () => snapshot,
+ }) } as never;
+ });
+ const result = await runEvalSessionCli(["--request", requestPath, "--output", outputPath, "--gateway-config", f.descriptorPath], { serviceFactory: factory });
+ expect(result).toBe(fail ? 2 : 0);
+ expect(factory).toHaveBeenCalledTimes(1);
+ expect(existsSync(home!)).toBe(false);
+ const raw = await readFile(outputPath, "utf8"), artifact = JSON.parse(raw);
+ expect(artifact.requestRouting).toMatchObject({ mode: "controller_gateway", provider: "openai", campaignId: "campaign-1", executionId: "cell-1", attemptId: "attempt-1" });
+ expect(raw).not.toContain(token);
+ expect(raw).not.toContain("upstream-must-not-reach-provider");
+ if (fail) expect(artifact.infrastructureError).toBe("simulated provider failure");
+});
+
+
+it.each(["codex", undefined])("rejects ACPX Codex (%s) before creating any provider service", async (agent) => {
+ const f = await fixture("acpx", agent);
+ vi.stubEnv("PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN", token);
+ const requestPath = join(f.root, "request.json"), outputPath = join(f.root, "artifact.json");
+ await writeFile(requestPath, JSON.stringify(f.request));
+ const factory = vi.fn(() => { throw new Error("must not create a provider service"); });
+ await expect(runEvalSessionCli(["--request", requestPath, "--output", outputPath, "--gateway-config", f.descriptorPath], { serviceFactory: factory }))
+ .rejects.toThrow("Controller gateway does not support this eval provider");
+ expect(factory).not.toHaveBeenCalled();
+ expect(existsSync(outputPath)).toBe(false);
+});
diff --git a/packages/paperclip-runner/src/cli/eval-gateway-runtime.ts b/packages/paperclip-runner/src/cli/eval-gateway-runtime.ts
new file mode 100644
index 0000000000..0d32670c3a
--- /dev/null
+++ b/packages/paperclip-runner/src/cli/eval-gateway-runtime.ts
@@ -0,0 +1,104 @@
+import { createHash } from "node:crypto";
+import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import type { EvalSessionRequest } from "./eval-session-contract.js";
+
+export interface EvalGatewayDescriptor {
+ schema: "paperclip.runner-eval.gateway/v1";
+ provider: "openai" | "anthropic";
+ model: string;
+ baseUrl: string;
+ campaignId: string;
+ executionId: string;
+ attemptId: string;
+ upstreamRoute: "direct" | "braintrust";
+}
+
+const TOKEN_ENV = "PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN";
+const safeId = (value: unknown) => typeof value === "string" && /^[A-Za-z0-9][A-Za-z0-9_.:/-]{0,511}$/.test(value);
+
+export async function prepareEvalGatewayRuntime(
+ descriptorPath: string,
+ request: EvalSessionRequest,
+ source: NodeJS.ProcessEnv = process.env,
+) {
+ const raw = await readFile(descriptorPath, "utf8");
+ let descriptor: EvalGatewayDescriptor;
+ try {
+ if (Buffer.byteLength(raw) > 8192) throw new Error();
+ descriptor = JSON.parse(raw) as EvalGatewayDescriptor;
+ const fields = ["schema", "provider", "model", "baseUrl", "campaignId", "executionId", "attemptId", "upstreamRoute"];
+ if (!descriptor || typeof descriptor !== "object" || Array.isArray(descriptor)
+ || Object.keys(descriptor).length !== fields.length || Object.keys(descriptor).some(key => !fields.includes(key))
+ || descriptor.schema !== "paperclip.runner-eval.gateway/v1"
+ || !["openai", "anthropic"].includes(descriptor.provider)
+ || !["direct", "braintrust"].includes(descriptor.upstreamRoute)
+ || ![descriptor.campaignId, descriptor.executionId, descriptor.attemptId].every(safeId)
+ || descriptor.attemptId !== request.attemptId || descriptor.model !== request.model
+ || typeof descriptor.baseUrl !== "string" || !/^http:\/\/127\.0\.0\.1:[1-9][0-9]{0,4}$/.test(descriptor.baseUrl)) throw new Error();
+ const url = new URL(descriptor.baseUrl);
+ if (!url.port || Number(url.port) > 65535) throw new Error();
+ } catch {
+ throw new Error("Invalid controller gateway descriptor");
+ }
+ const provider = request.provider ?? "codex";
+ const codex = provider === "codex";
+ const claude = provider === "acpx" && request.acpxAgent === "claude";
+ if (!codex && !claude || descriptor.provider !== (codex ? "openai" : "anthropic")) {
+ throw new Error("Controller gateway does not support this eval provider");
+ }
+ const token = source[TOKEN_ENV];
+ if (typeof token !== "string" || !/^[a-f0-9]{64}$/.test(token)) throw new Error("Missing or invalid controller gateway token");
+ const root = await mkdtemp(join(tmpdir(), "paperclip-runner-eval-gateway-"));
+ try {
+ const home = join(root, "home"), codexHome = join(root, "codex"), claudeHome = join(root, "claude");
+ const environment: NodeJS.ProcessEnv = Object.fromEntries(
+ ["PATH", "LANG", "LANGUAGE", "LC_ALL", "TZ", "SystemRoot", "WINDIR", "PATHEXT"]
+ .flatMap(key => source[key] === undefined ? [] : [[key, source[key]!]]),
+ );
+ Object.assign(environment, {
+ HOME: home, CODEX_HOME: codexHome, CLAUDE_CONFIG_DIR: claudeHome,
+ XDG_CONFIG_HOME: join(root, "config"), XDG_DATA_HOME: join(root, "data"), XDG_CACHE_HOME: join(root, "cache"),
+ TMPDIR: join(root, "tmp"), TMP: join(root, "tmp"), TEMP: join(root, "tmp"),
+ PAPERCLIP_RUNNER_HOME_MODE: "preconfigured",
+ });
+ for (const directory of [home, codexHome, claudeHome, environment.XDG_CONFIG_HOME!, environment.XDG_DATA_HOME!, environment.XDG_CACHE_HOME!, environment.TMPDIR!]) {
+ await mkdir(directory, { mode: 0o700 });
+ }
+ if (codex) {
+ environment.PAPERCLIP_AI_PROVIDER_KEY = token;
+ await writeFile(join(codexHome, "config.toml"), [
+ 'model_provider = "paperclip_eval"',
+ `model = ${JSON.stringify(request.model)}`,
+ '[model_providers.paperclip_eval]',
+ 'name = "Paperclip eval controller"',
+ `base_url = ${JSON.stringify(descriptor.baseUrl + "/v1")}`,
+ 'wire_api = "responses"', 'requires_openai_auth = false', 'env_key = "PAPERCLIP_AI_PROVIDER_KEY"',
+ '[features]', 'shell_snapshot = false', '',
+ ].join("\n"), { mode: 0o600, flag: "wx" });
+ } else {
+ Object.assign(environment, {
+ ANTHROPIC_API_KEY: token, ANTHROPIC_BASE_URL: descriptor.baseUrl,
+ CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1", ACPX_CLAUDE_INCLUDE_USER_SETTINGS: "1",
+ ANTHROPIC_MODEL: request.model, ANTHROPIC_DEFAULT_OPUS_MODEL: request.model,
+ ANTHROPIC_DEFAULT_SONNET_MODEL: request.model, ANTHROPIC_DEFAULT_HAIKU_MODEL: request.model,
+ CLAUDE_CODE_SUBAGENT_MODEL: request.model,
+ });
+ await writeFile(join(claudeHome, "settings.json"), JSON.stringify({ model: request.model, availableModels: [request.model] }) + "\n", { mode: 0o600, flag: "wx" });
+ }
+ return {
+ transportOptions: { homeMode: "preconfigured" as const, environment },
+ evidence: {
+ mode: "controller_gateway" as const, provider: descriptor.provider,
+ declaredUpstreamRoute: descriptor.upstreamRoute,
+ campaignId: descriptor.campaignId, executionId: descriptor.executionId, attemptId: descriptor.attemptId,
+ descriptorSha256: createHash("sha256").update(raw).digest("hex"),
+ },
+ close: () => rm(root, { recursive: true, force: true }),
+ };
+ } catch (error) {
+ await rm(root, { recursive: true, force: true });
+ throw error;
+ }
+}
diff --git a/packages/paperclip-runner/src/cli/eval-session.ts b/packages/paperclip-runner/src/cli/eval-session.ts
index 788dc61602..c42a22ec93 100644
--- a/packages/paperclip-runner/src/cli/eval-session.ts
+++ b/packages/paperclip-runner/src/cli/eval-session.ts
@@ -23,6 +23,7 @@ import {
CapabilityLiveSessionService,
type CapabilityLiveSession,
type CapabilityLiveSessionSnapshot,
+ type CapabilityLiveSessionServiceOptions,
type CapabilityLiveTurnResult,
type CreateCapabilityLiveSessionInput,
} from "../live/live-session.js";
@@ -35,6 +36,7 @@ import {
type EvalSessionUsage,
} from "./eval-session-contract.js";
import { evalProviderTransportOptions } from "./eval-provider-runtime.js";
+import { prepareEvalGatewayRuntime } from "./eval-gateway-runtime.js";
import { NativeSessionCloseUnrecoverableError } from "../contracts/native-session-backend.js";
interface EvalSessionCliOptions {
@@ -42,6 +44,7 @@ interface EvalSessionCliOptions {
outputPath: string;
candidateProfile?: EvalCandidateProfile;
expectedAcpxProfile?: Record<string, unknown>;
+ gatewayConfigPath?: string;
}
function argument(args: string[], name: string): string {
@@ -52,7 +55,7 @@ function argument(args: string[], name: string): string {
}
export function parseEvalSessionCliArgs(args: string[]): EvalSessionCliOptions {
- const allowed = new Set(["--request", "--output", "--candidate-profile", "--expected-acpx-profile"]);
+ const allowed = new Set(["--request", "--output", "--candidate-profile", "--expected-acpx-profile", "--gateway-config"]);
const seen = new Set<string>();
for (let index = 0; index < args.length; index += 2) {
if (!allowed.has(args[index] ?? "")) {
@@ -89,6 +92,7 @@ export function parseEvalSessionCliArgs(args: string[]): EvalSessionCliOptions {
return {
...(candidateProfile === undefined ? {} : { candidateProfile }),
...(expectedAcpxProfile === undefined ? {} : { expectedAcpxProfile }),
+ ...(seen.has("--gateway-config") ? { gatewayConfigPath: argument(args, "--gateway-config") } : {}),
requestPath: argument(args, "--request"),
outputPath: argument(args, "--output"),
};
@@ -362,6 +366,7 @@ export async function runEvalSessionCli(
options: {
serviceFactory?: (
runnerBinary: string,
+ transportOptions: NonNullable<CapabilityLiveSessionServiceOptions["transportOptions"]>,
) => CapabilityLiveSessionService;
} = {},
): Promise<number> {
@@ -398,32 +403,31 @@ export async function runEvalSessionCli(
const requestedDriver = request.driver ??
expectedEvalSessionDriver(requestedProvider);
const requestedProviderVersion = evalSessionProviderVersion(request);
- const runtimeContext = await prepareEvalRuntimeContext(
- resolve(request.session.workingDirectory ?? process.cwd()),
- );
- const service = options.serviceFactory?.(runnerdPath) ??
- new CapabilityLiveSessionService({
- transportOptions: {
- ...evalProviderTransportOptions(requestedProvider, request.limits.turnTimeoutMs, request.session.workingDirectory),
- runnerBinary: runnerdPath,
- ...(cli.candidateProfile === undefined ? {} : { acpxCandidateProfile: cli.candidateProfile }),
- runtimeContext,
- baseInstructions: evalRuntimeSystemInstructions(runtimeContext),
- // The transport performs the provider-specific allowlisting. Supplying
- // the source environment here is still required: without it the
- // isolated Codex home has no credential source and runnerd receives no
- // executable PATH from the Evalbook CLI process.
- environment: process.env,
- onDiagnostic: (message) => {
- process.stderr.write(`[eval-session runnerd] ${message}\n`);
- },
- },
- });
+ const gateway = cli.gatewayConfigPath
+ ? await prepareEvalGatewayRuntime(cli.gatewayConfigPath, request)
+ : undefined;
let session: CapabilityLiveSession | null = null;
let turn: CapabilityLiveTurnResult | null = null;
let snapshot: CapabilityLiveSessionSnapshot | null = null;
try {
+ const runtimeContext = await prepareEvalRuntimeContext(
+ resolve(request.session.workingDirectory ?? process.cwd()),
+ );
+ const transportOptions: NonNullable<CapabilityLiveSessionServiceOptions["transportOptions"]> = {
+ ...evalProviderTransportOptions(requestedProvider, request.limits.turnTimeoutMs, request.session.workingDirectory),
+ runnerBinary: runnerdPath,
+ ...(cli.candidateProfile === undefined ? {} : { acpxCandidateProfile: cli.candidateProfile }),
+ runtimeContext,
+ baseInstructions: evalRuntimeSystemInstructions(runtimeContext),
+ environment: process.env,
+ ...gateway?.transportOptions,
+ onDiagnostic: (message) => {
+ process.stderr.write(`[eval-session runnerd] ${message}\n`);
+ },
+ };
+ const service = options.serviceFactory?.(runnerdPath, transportOptions)
+ ?? new CapabilityLiveSessionService({ transportOptions });
// PR3 expands this same service input with the two qualified remote
// profiles. The cast keeps this isolated PR typecheckable before PR3 lands;
// the runtime fields and their fail-closed validation are already present.
@@ -470,6 +474,7 @@ export async function runEvalSessionCli(
build: PAPERCLIP_RUNNER_BUILD_METADATA,
runnerd: { path: "[withheld]", sha256: `sha256:${actualDigest}` },
requestedModel: request.model,
+ ...(gateway ? { requestRouting: gateway.evidence } : {}),
...(cli.candidateProfile === undefined ? {} : { diagnosticCandidateProfile: cli.candidateProfile }),
provider: requestedProvider,
driver: requestedDriver,
@@ -549,6 +554,7 @@ export async function runEvalSessionCli(
build: PAPERCLIP_RUNNER_BUILD_METADATA,
runnerd: { path: "[withheld]", sha256: `sha256:${actualDigest}` },
requestedModel: request.model,
+ ...(gateway ? { requestRouting: gateway.evidence } : {}),
...(cli.candidateProfile === undefined ? {} : { diagnosticCandidateProfile: cli.candidateProfile }),
provider: requestedProvider,
driver: requestedDriver,
@@ -575,6 +581,8 @@ export async function runEvalSessionCli(
},
}, null, 2)}\n`);
return 2;
+ } finally {
+ await gateway?.close();
}
}