The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
feat(evals): route Runner sessions through a controller gateway (#17440)
Dottad1fdfced9d28
diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.mdindex b7f8efad0c..a7114ed7c3 100644--- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md+++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md@@ -104,6 +104,61 @@ Claude Managed uses its Agent version, while AgentCore uses its qualificationrevision. Transport API or beta versions remain separate protocol metadata andmust not replace that runtime identity in a report.+## Controller-assigned gateway route++The Runner eval CLI accepts `--gateway-config /absolute/path/gateway.json` for+native Codex and ACPX Claude. This is an explicit route for a controller+that owns a local budget proxy. Normal invocations keep the direct path.+ACPX Codex is rejected before service creation because its runtime prohibits+preconfigured homes to protect Runner state.+The descriptor must bind the exact model and attempt ID from `request.json`:++```json+{+ "schema": "paperclip.runner-eval.gateway/v1",+ "provider": "openai",+ "model": "gpt-5.4-mini",+ "baseUrl": "http://127.0.0.1:4567",+ "campaignId": "campaign-id",+ "executionId": "cell-id",+ "attemptId": "attempt-id",+ "upstreamRoute": "direct"+}+```++Use `anthropic` for ACPX Claude. `upstreamRoute` is the controller's declaration:+`direct` or `braintrust`. It is not an independent verification of the upstream.+The base URL must be the proxy's explicit IPv4 loopback address and port, with+no path, query, fragment, or embedded credential. The controller supplies its+32-byte hexadecimal token through `PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN`.+Do not put provider or Braintrust credentials in this child environment or file.++```sh+node /path/to/runner/dist/cli/eval-session.js \+ --request /path/to/request.json --output /path/to/artifact.json \+ --gateway-config /path/to/gateway.json+```++The CLI validates the descriptor before creating a provider service. It creates+an owned temporary home, pins Codex's Responses endpoint or Claude's Messages+endpoint and exact model, and passes only the assigned proxy credential.+It does not copy ambient provider keys, subscription logins, proxy settings,+startup hooks, or user homes into this route. Unsupported harnesses and+mismatched descriptor identities fail before provider creation. The temporary+home is removed on success and failure.++Success and failure artifacts retain `requestRouting`: controller gateway mode,+provider, declared upstream route, campaign/execution/attempt IDs, and the+SHA-256 of the descriptor bytes. The token and endpoint are excluded from this+metadata. Existing artifact usage and provider outcomes remain authoritative+for their respective fields; the route declaration is not a billing receipt.+The controller must retain its request ledger and link these identities to it.++This entry point does not itself allocate a campaign budget, run the proxy, or+change the hosted matrix. The controller must enforce request admission,+deadlines, and limits before forwarding traffic. A local fake-API probe only+proves routing; it does not qualify live provider or Braintrust behavior.+## Hosted campaignUse the `Runner Direct Live Protocol Evals` workflow. Dispatch the workflowdiff --git a/packages/paperclip-runner/src/cli/eval-gateway-runtime.test.ts b/packages/paperclip-runner/src/cli/eval-gateway-runtime.test.tsnew file mode 100644index 0000000000..cd1cb1e03e--- /dev/null+++ b/packages/paperclip-runner/src/cli/eval-gateway-runtime.test.ts@@ -0,0 +1,156 @@+import { createHash } from "node:crypto";+import { existsSync } from "node:fs";+import { chmod, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { afterEach, expect, it, vi } from "vitest";+import { createSanitizedCodexEnvironment } from "../drivers/codex/app-server-transport.js";+import { createSanitizedAcpxSpawnInput } from "../drivers/acpx/environment.js";+import { createCapabilityFixtureState } from "../mock-core/capability-control-plane-types.js";+import type { CapabilityLiveSessionSnapshot } from "../live/live-session.js";+import { parseEvalSessionRequest } from "./eval-session-contract.js";+import { prepareEvalGatewayRuntime } from "./eval-gateway-runtime.js";+import { runEvalSessionCli } from "./eval-session.js";++const roots: string[] = [];+afterEach(async () => {+ vi.unstubAllEnvs();+ for (const root of roots.splice(0)) {+ for (const entry of await readdir(root).catch(() => [])) {+ if (entry.startsWith(".paperclip-eval-runtime-context-")) await chmod(join(root, entry, "instructions"), 0o700).catch(() => undefined);+ }+ await rm(root, { recursive: true, force: true });+ }+});+const token = "a".repeat(64);+async function fixture(provider = "codex", acpxAgent?: string) {+ const root = await mkdtemp(join(tmpdir(), "runner-gateway-test-")); roots.push(root);+ const model = acpxAgent === "claude" ? "claude-sonnet-5" : "gpt-5.6-sol";+ const binary = join(root, "runnerd");+ await writeFile(binary, "fake runnerd");+ const request = parseEvalSessionRequest({+ schema: "paperclip-runner/eval-session-request/v1", attemptId: "attempt-1", prompt: "Read the governed task.",+ provider, ...(acpxAgent ? { acpxAgent } : {}), model,+ runnerd: { path: binary, sha256: createHash("sha256").update("fake runnerd").digest("hex") },+ limits: { turnTimeoutMs: 120000, maxAgentTurns: 1, maxEstimatedCostNanodollars: 100000000 },+ session: { workingDirectory: root },+ });+ const descriptor = { schema: "paperclip.runner-eval.gateway/v1", provider: acpxAgent === "claude" ? "anthropic" : "openai",+ model, baseUrl: "http://127.0.0.1:4567", campaignId: "campaign-1", executionId: "cell-1", attemptId: "attempt-1", upstreamRoute: "braintrust" };+ const descriptorPath = join(root, "gateway.json");+ await writeFile(descriptorPath, JSON.stringify(descriptor));+ return { root, request, descriptor, descriptorPath };+}++it.each([["codex", undefined], ["acpx", "claude"]])("routes %s/%s with only its proxy credential and an owned home", async (provider, agent) => {+ const f = await fixture(provider, agent);+ const gateway = await prepareEvalGatewayRuntime(f.descriptorPath, f.request, {+ PATH: process.env.PATH, HOME: "/ambient/home", CODEX_HOME: "/ambient/codex", CLAUDE_CONFIG_DIR: "/ambient/claude",+ PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: token, OPENAI_API_KEY: "ambient-openai", CODEX_API_KEY: "ambient-codex",+ ANTHROPIC_API_KEY: "ambient-anthropic", ANTHROPIC_AUTH_TOKEN: "ambient-oauth", CLAUDE_CODE_OAUTH_TOKEN: "ambient-claude",+ BRAINTRUST_API_KEY: "ambient-braintrust", AWS_SECRET_ACCESS_KEY: "ambient-aws", GH_TOKEN: "ambient-github",+ HTTPS_PROXY: "https://ambient-proxy", NODE_OPTIONS: "--require ambient-hook", PAPERCLIP_CONFIGURED_ENVIRONMENT_KEYS: '["GH_TOKEN"]',+ });+ const env = gateway.transportOptions.environment;+ try {+ expect(gateway.transportOptions.homeMode).toBe("preconfigured");+ expect(JSON.stringify(env)).not.toContain("ambient");+ expect(env.OPENAI_API_KEY).toBeUndefined();+ expect(env.CODEX_API_KEY).toBeUndefined();+ expect(env.BRAINTRUST_API_KEY).toBeUndefined();+ expect(env.PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN).toBeUndefined();+ const child = provider === "codex" ? createSanitizedCodexEnvironment(env) : createSanitizedAcpxSpawnInput(env, agent as "codex" | "claude").env;+ expect(child.HOME).toBe(env.HOME);+ if (agent === "claude") {+ expect(child.ANTHROPIC_API_KEY).toBe(token);+ expect(child.ANTHROPIC_BASE_URL).toBe(f.descriptor.baseUrl);+ expect(child.CLAUDE_CODE_SUBAGENT_MODEL).toBe(f.request.model);+ expect(JSON.parse(await readFile(join(env.CLAUDE_CONFIG_DIR!, "settings.json"), "utf8"))).toEqual({ model: f.request.model, availableModels: [f.request.model] });+ } else {+ expect(child.PAPERCLIP_AI_PROVIDER_KEY).toBe(token);+ const config = await readFile(join(env.CODEX_HOME!, "config.toml"), "utf8");+ expect(config).toContain('base_url = "http://127.0.0.1:4567/v1"');+ expect(config).toContain('env_key = "PAPERCLIP_AI_PROVIDER_KEY"');+ expect(config).toContain("requires_openai_auth = false");+ expect(config).toContain("shell_snapshot = false");+ expect(config).not.toContain(token);+ expect((await stat(join(env.CODEX_HOME!, "config.toml"))).mode & 0o777).toBe(0o600);+ }+ expect(JSON.stringify(gateway.evidence)).not.toContain(token);+ expect(gateway.evidence).toMatchObject({ mode: "controller_gateway", attemptId: f.request.attemptId, declaredUpstreamRoute: "braintrust" });+ } finally { await gateway.close(); }+ expect(existsSync(env.HOME!)).toBe(false);+});++it.each([+ { baseUrl: "https://api.openai.com" }, { baseUrl: "http://localhost:4567" },+ { baseUrl: "http://127.0.0.1:4567/v1" }, { baseUrl: "http://127.0.0.1:99999" },+ { baseUrl: "http://127.0.0.1:4567?token=secret" }, { attemptId: "another-attempt" },+ { model: "another-model" }, { campaignId: "bad identity" }, { unexpectedSecret: "must-not-print" },+])("rejects invalid descriptor binding %j", async (change) => {+ const f = await fixture();+ await writeFile(f.descriptorPath, JSON.stringify({ ...f.descriptor, ...change }));+ await expect(prepareEvalGatewayRuntime(f.descriptorPath, f.request, { PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: token }))+ .rejects.toThrow("Invalid controller gateway descriptor");+});++it("rejects missing tokens and unsupported provider routes before runtime creation", async () => {+ const f = await fixture();+ await expect(prepareEvalGatewayRuntime(f.descriptorPath, f.request, {})).rejects.toThrow("gateway token");+ await expect(prepareEvalGatewayRuntime(f.descriptorPath, { ...f.request, provider: "opencode" }, { PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: token })).rejects.toThrow("does not support");+ await writeFile(f.descriptorPath, JSON.stringify({ ...f.descriptor, provider: "anthropic" }));+ await expect(prepareEvalGatewayRuntime(f.descriptorPath, f.request, { PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: token })).rejects.toThrow("does not support");+});++it.each([false, true])("the real CLI retains route identity and removes the owned home after failure=%s", async (fail) => {+ const f = await fixture();+ vi.stubEnv("PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN", token);+ vi.stubEnv("OPENAI_API_KEY", "upstream-must-not-reach-provider");+ const requestPath = join(f.root, "request.json"), outputPath = join(f.root, "artifact.json");+ await writeFile(requestPath, JSON.stringify(f.request));+ const state = JSON.stringify(createCapabilityFixtureState({}));+ const snapshot = {+ sessionId: "session-1", revision: 1, providerThreadId: "provider-1", providerSessionId: "provider-1",+ providerModel: { id: f.request.model, provider: "openai" }, status: "idle", activeTurnId: null,+ createdAt: "2026-10-24T00:00:00Z", updatedAt: "2026-10-24T00:00:01Z",+ authority: { companyId: "company-1", actorId: "actor-1", taskId: "task-1", runId: "run-1", sessionId: "session-1", scenarioId: "gateway-test" },+ config: { provider: "codex", driver: "codex_app_server", seedState: state, workingDirectory: f.root, scenario: { id: "gateway-test" }, capabilities: [], explicitClaims: [], turnTimeoutMs: 1000 },+ mockState: state, process: null, networkEvidence: { realPaperclipRequests: 0, childPaperclipEnvironmentKeys: [] },+ transcript: [], evidence: [], authorizationRecords: [], attempts: [], terminalTurns: [{ turnId: "turn-1", status: "completed" }],+ usageLedger: [{ receiptId: "receipt-1", turnId: "turn-1", providerCalls: 1, providerRequests: 1, inputTokens: 100, outputTokens: 10, cachedInputTokens: 0, reasoningTokens: 0, costNanodollars: 0 }], usageUnavailable: [],+ } as unknown as CapabilityLiveSessionSnapshot;+ let home: string | undefined;+ const factory = vi.fn((_binary, options) => {+ expect(options.homeMode).toBe("preconfigured");+ expect(options.environment.OPENAI_API_KEY).toBeUndefined();+ expect(options.environment.PAPERCLIP_AI_PROVIDER_KEY).toBe(token);+ home = options.environment.HOME;+ expect(existsSync(home!)).toBe(true);+ return { create: async () => ({+ sendMessage: async () => { if (fail) throw new Error("simulated provider failure"); return { turnId: "turn-1", status: "completed", snapshot }; },+ completeAttempt: async () => undefined, shutdown: async () => { snapshot.status = "closed"; }, snapshot: () => snapshot,+ }) } as never;+ });+ const result = await runEvalSessionCli(["--request", requestPath, "--output", outputPath, "--gateway-config", f.descriptorPath], { serviceFactory: factory });+ expect(result).toBe(fail ? 2 : 0);+ expect(factory).toHaveBeenCalledTimes(1);+ expect(existsSync(home!)).toBe(false);+ const raw = await readFile(outputPath, "utf8"), artifact = JSON.parse(raw);+ expect(artifact.requestRouting).toMatchObject({ mode: "controller_gateway", provider: "openai", campaignId: "campaign-1", executionId: "cell-1", attemptId: "attempt-1" });+ expect(raw).not.toContain(token);+ expect(raw).not.toContain("upstream-must-not-reach-provider");+ if (fail) expect(artifact.infrastructureError).toBe("simulated provider failure");+});+++it.each(["codex", undefined])("rejects ACPX Codex (%s) before creating any provider service", async (agent) => {+ const f = await fixture("acpx", agent);+ vi.stubEnv("PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN", token);+ const requestPath = join(f.root, "request.json"), outputPath = join(f.root, "artifact.json");+ await writeFile(requestPath, JSON.stringify(f.request));+ const factory = vi.fn(() => { throw new Error("must not create a provider service"); });+ await expect(runEvalSessionCli(["--request", requestPath, "--output", outputPath, "--gateway-config", f.descriptorPath], { serviceFactory: factory }))+ .rejects.toThrow("Controller gateway does not support this eval provider");+ expect(factory).not.toHaveBeenCalled();+ expect(existsSync(outputPath)).toBe(false);+});diff --git a/packages/paperclip-runner/src/cli/eval-gateway-runtime.ts b/packages/paperclip-runner/src/cli/eval-gateway-runtime.tsnew file mode 100644index 0000000000..0d32670c3a--- /dev/null+++ b/packages/paperclip-runner/src/cli/eval-gateway-runtime.ts@@ -0,0 +1,104 @@+import { createHash } from "node:crypto";+import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import type { EvalSessionRequest } from "./eval-session-contract.js";++export interface EvalGatewayDescriptor {+ schema: "paperclip.runner-eval.gateway/v1";+ provider: "openai" | "anthropic";+ model: string;+ baseUrl: string;+ campaignId: string;+ executionId: string;+ attemptId: string;+ upstreamRoute: "direct" | "braintrust";+}++const TOKEN_ENV = "PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN";+const safeId = (value: unknown) => typeof value === "string" && /^[A-Za-z0-9][A-Za-z0-9_.:/-]{0,511}$/.test(value);++export async function prepareEvalGatewayRuntime(+ descriptorPath: string,+ request: EvalSessionRequest,+ source: NodeJS.ProcessEnv = process.env,+) {+ const raw = await readFile(descriptorPath, "utf8");+ let descriptor: EvalGatewayDescriptor;+ try {+ if (Buffer.byteLength(raw) > 8192) throw new Error();+ descriptor = JSON.parse(raw) as EvalGatewayDescriptor;+ const fields = ["schema", "provider", "model", "baseUrl", "campaignId", "executionId", "attemptId", "upstreamRoute"];+ if (!descriptor || typeof descriptor !== "object" || Array.isArray(descriptor)+ || Object.keys(descriptor).length !== fields.length || Object.keys(descriptor).some(key => !fields.includes(key))+ || descriptor.schema !== "paperclip.runner-eval.gateway/v1"+ || !["openai", "anthropic"].includes(descriptor.provider)+ || !["direct", "braintrust"].includes(descriptor.upstreamRoute)+ || ![descriptor.campaignId, descriptor.executionId, descriptor.attemptId].every(safeId)+ || descriptor.attemptId !== request.attemptId || descriptor.model !== request.model+ || typeof descriptor.baseUrl !== "string" || !/^http:\/\/127\.0\.0\.1:[1-9][0-9]{0,4}$/.test(descriptor.baseUrl)) throw new Error();+ const url = new URL(descriptor.baseUrl);+ if (!url.port || Number(url.port) > 65535) throw new Error();+ } catch {+ throw new Error("Invalid controller gateway descriptor");+ }+ const provider = request.provider ?? "codex";+ const codex = provider === "codex";+ const claude = provider === "acpx" && request.acpxAgent === "claude";+ if (!codex && !claude || descriptor.provider !== (codex ? "openai" : "anthropic")) {+ throw new Error("Controller gateway does not support this eval provider");+ }+ const token = source[TOKEN_ENV];+ if (typeof token !== "string" || !/^[a-f0-9]{64}$/.test(token)) throw new Error("Missing or invalid controller gateway token");+ const root = await mkdtemp(join(tmpdir(), "paperclip-runner-eval-gateway-"));+ try {+ const home = join(root, "home"), codexHome = join(root, "codex"), claudeHome = join(root, "claude");+ const environment: NodeJS.ProcessEnv = Object.fromEntries(+ ["PATH", "LANG", "LANGUAGE", "LC_ALL", "TZ", "SystemRoot", "WINDIR", "PATHEXT"]+ .flatMap(key => source[key] === undefined ? [] : [[key, source[key]!]]),+ );+ Object.assign(environment, {+ HOME: home, CODEX_HOME: codexHome, CLAUDE_CONFIG_DIR: claudeHome,+ XDG_CONFIG_HOME: join(root, "config"), XDG_DATA_HOME: join(root, "data"), XDG_CACHE_HOME: join(root, "cache"),+ TMPDIR: join(root, "tmp"), TMP: join(root, "tmp"), TEMP: join(root, "tmp"),+ PAPERCLIP_RUNNER_HOME_MODE: "preconfigured",+ });+ for (const directory of [home, codexHome, claudeHome, environment.XDG_CONFIG_HOME!, environment.XDG_DATA_HOME!, environment.XDG_CACHE_HOME!, environment.TMPDIR!]) {+ await mkdir(directory, { mode: 0o700 });+ }+ if (codex) {+ environment.PAPERCLIP_AI_PROVIDER_KEY = token;+ await writeFile(join(codexHome, "config.toml"), [+ 'model_provider = "paperclip_eval"',+ `model = ${JSON.stringify(request.model)}`,+ '[model_providers.paperclip_eval]',+ 'name = "Paperclip eval controller"',+ `base_url = ${JSON.stringify(descriptor.baseUrl + "/v1")}`,+ 'wire_api = "responses"', 'requires_openai_auth = false', 'env_key = "PAPERCLIP_AI_PROVIDER_KEY"',+ '[features]', 'shell_snapshot = false', '',+ ].join("\n"), { mode: 0o600, flag: "wx" });+ } else {+ Object.assign(environment, {+ ANTHROPIC_API_KEY: token, ANTHROPIC_BASE_URL: descriptor.baseUrl,+ CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1", ACPX_CLAUDE_INCLUDE_USER_SETTINGS: "1",+ ANTHROPIC_MODEL: request.model, ANTHROPIC_DEFAULT_OPUS_MODEL: request.model,+ ANTHROPIC_DEFAULT_SONNET_MODEL: request.model, ANTHROPIC_DEFAULT_HAIKU_MODEL: request.model,+ CLAUDE_CODE_SUBAGENT_MODEL: request.model,+ });+ await writeFile(join(claudeHome, "settings.json"), JSON.stringify({ model: request.model, availableModels: [request.model] }) + "\n", { mode: 0o600, flag: "wx" });+ }+ return {+ transportOptions: { homeMode: "preconfigured" as const, environment },+ evidence: {+ mode: "controller_gateway" as const, provider: descriptor.provider,+ declaredUpstreamRoute: descriptor.upstreamRoute,+ campaignId: descriptor.campaignId, executionId: descriptor.executionId, attemptId: descriptor.attemptId,+ descriptorSha256: createHash("sha256").update(raw).digest("hex"),+ },+ close: () => rm(root, { recursive: true, force: true }),+ };+ } catch (error) {+ await rm(root, { recursive: true, force: true });+ throw error;+ }+}diff --git a/packages/paperclip-runner/src/cli/eval-session.ts b/packages/paperclip-runner/src/cli/eval-session.tsindex 788dc61602..c42a22ec93 100644--- a/packages/paperclip-runner/src/cli/eval-session.ts+++ b/packages/paperclip-runner/src/cli/eval-session.ts@@ -23,6 +23,7 @@ import {CapabilityLiveSessionService,type CapabilityLiveSession,type CapabilityLiveSessionSnapshot,+ type CapabilityLiveSessionServiceOptions,type CapabilityLiveTurnResult,type CreateCapabilityLiveSessionInput,} from "../live/live-session.js";@@ -35,6 +36,7 @@ import {type EvalSessionUsage,} from "./eval-session-contract.js";import { evalProviderTransportOptions } from "./eval-provider-runtime.js";+import { prepareEvalGatewayRuntime } from "./eval-gateway-runtime.js";import { NativeSessionCloseUnrecoverableError } from "../contracts/native-session-backend.js";interface EvalSessionCliOptions {@@ -42,6 +44,7 @@ interface EvalSessionCliOptions {outputPath: string;candidateProfile?: EvalCandidateProfile;expectedAcpxProfile?: Record<string, unknown>;+ gatewayConfigPath?: string;}function argument(args: string[], name: string): string {@@ -52,7 +55,7 @@ function argument(args: string[], name: string): string {}export function parseEvalSessionCliArgs(args: string[]): EvalSessionCliOptions {- const allowed = new Set(["--request", "--output", "--candidate-profile", "--expected-acpx-profile"]);+ const allowed = new Set(["--request", "--output", "--candidate-profile", "--expected-acpx-profile", "--gateway-config"]);const seen = new Set<string>();for (let index = 0; index < args.length; index += 2) {if (!allowed.has(args[index] ?? "")) {@@ -89,6 +92,7 @@ export function parseEvalSessionCliArgs(args: string[]): EvalSessionCliOptions {return {...(candidateProfile === undefined ? {} : { candidateProfile }),...(expectedAcpxProfile === undefined ? {} : { expectedAcpxProfile }),+ ...(seen.has("--gateway-config") ? { gatewayConfigPath: argument(args, "--gateway-config") } : {}),requestPath: argument(args, "--request"),outputPath: argument(args, "--output"),};@@ -362,6 +366,7 @@ export async function runEvalSessionCli(options: {serviceFactory?: (runnerBinary: string,+ transportOptions: NonNullable<CapabilityLiveSessionServiceOptions["transportOptions"]>,) => CapabilityLiveSessionService;} = {},): Promise<number> {@@ -398,32 +403,31 @@ export async function runEvalSessionCli(const requestedDriver = request.driver ??expectedEvalSessionDriver(requestedProvider);const requestedProviderVersion = evalSessionProviderVersion(request);- const runtimeContext = await prepareEvalRuntimeContext(- resolve(request.session.workingDirectory ?? process.cwd()),- );- const service = options.serviceFactory?.(runnerdPath) ??- new CapabilityLiveSessionService({- transportOptions: {- ...evalProviderTransportOptions(requestedProvider, request.limits.turnTimeoutMs, request.session.workingDirectory),- runnerBinary: runnerdPath,- ...(cli.candidateProfile === undefined ? {} : { acpxCandidateProfile: cli.candidateProfile }),- runtimeContext,- baseInstructions: evalRuntimeSystemInstructions(runtimeContext),- // The transport performs the provider-specific allowlisting. Supplying- // the source environment here is still required: without it the- // isolated Codex home has no credential source and runnerd receives no- // executable PATH from the Evalbook CLI process.- environment: process.env,- onDiagnostic: (message) => {- process.stderr.write(`[eval-session runnerd] ${message}\n`);- },- },- });+ const gateway = cli.gatewayConfigPath+ ? await prepareEvalGatewayRuntime(cli.gatewayConfigPath, request)+ : undefined;let session: CapabilityLiveSession | null = null;let turn: CapabilityLiveTurnResult | null = null;let snapshot: CapabilityLiveSessionSnapshot | null = null;try {+ const runtimeContext = await prepareEvalRuntimeContext(+ resolve(request.session.workingDirectory ?? process.cwd()),+ );+ const transportOptions: NonNullable<CapabilityLiveSessionServiceOptions["transportOptions"]> = {+ ...evalProviderTransportOptions(requestedProvider, request.limits.turnTimeoutMs, request.session.workingDirectory),+ runnerBinary: runnerdPath,+ ...(cli.candidateProfile === undefined ? {} : { acpxCandidateProfile: cli.candidateProfile }),+ runtimeContext,+ baseInstructions: evalRuntimeSystemInstructions(runtimeContext),+ environment: process.env,+ ...gateway?.transportOptions,+ onDiagnostic: (message) => {+ process.stderr.write(`[eval-session runnerd] ${message}\n`);+ },+ };+ const service = options.serviceFactory?.(runnerdPath, transportOptions)+ ?? new CapabilityLiveSessionService({ transportOptions });// PR3 expands this same service input with the two qualified remote// profiles. The cast keeps this isolated PR typecheckable before PR3 lands;// the runtime fields and their fail-closed validation are already present.@@ -470,6 +474,7 @@ export async function runEvalSessionCli(build: PAPERCLIP_RUNNER_BUILD_METADATA,runnerd: { path: "[withheld]", sha256: `sha256:${actualDigest}` },requestedModel: request.model,+ ...(gateway ? { requestRouting: gateway.evidence } : {}),...(cli.candidateProfile === undefined ? {} : { diagnosticCandidateProfile: cli.candidateProfile }),provider: requestedProvider,driver: requestedDriver,@@ -549,6 +554,7 @@ export async function runEvalSessionCli(build: PAPERCLIP_RUNNER_BUILD_METADATA,runnerd: { path: "[withheld]", sha256: `sha256:${actualDigest}` },requestedModel: request.model,+ ...(gateway ? { requestRouting: gateway.evidence } : {}),...(cli.candidateProfile === undefined ? {} : { diagnosticCandidateProfile: cli.candidateProfile }),provider: requestedProvider,driver: requestedDriver,@@ -575,6 +581,8 @@ export async function runEvalSessionCli(},}, null, 2)}\n`);return 2;+ } finally {+ await gateway?.close();}}