The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix(evals): correlate canceled gateway requests with their ledger (#17354)
Dottae48f1867b3b4
diff --git a/tests/runner-e2e/OPERATIONS.md b/tests/runner-e2e/OPERATIONS.mdindex a51bd02276..6f4a3a643e 100644--- a/tests/runner-e2e/OPERATIONS.md+++ b/tests/runner-e2e/OPERATIONS.md@@ -156,7 +156,17 @@ keys. The server environment strips the gateway token and upstream credentials.Every upstream attempt reserves its maximum estimated charge in an fsyncedjournal before dispatch. Requests, retries, and cancelled calls retain distinct-IDs. Unknown usage keeps the reservation held. One shared writer serializes+IDs. Braintrust gateway requests propagate the reserved request ID, campaign,+execution, attempt and phase as allowlisted `paperclip_eval_*` metadata in the+SDK-compatible parent context. The exact context is retained in the private+receipt before dispatch, so a request canceled before response headers can+still be looked up by `metadata.paperclip_eval_request_id` in project logs.+This context contains no provider credential, prompt or response, and does not+create an extra model call or a synthetic parent span. Direct provider routes+receive no Braintrust context. Older project-only receipts cannot acquire this+identity retroactively; time or session matches remain candidate evidence.+A matching logged span does not itself prove complete usage or final billing.+Unknown usage keeps the reservation held. One shared writer serializesadmission across cells, so concurrent requests cannot each spend the same remainingallowance. Campaign deadline, request count, and in-flight count are also enforced.Private request receipts retain transport IDs, response IDs, usage and cache TTLs,diff --git a/tests/runner-e2e/braintrust-gateway-context.ts b/tests/runner-e2e/braintrust-gateway-context.tsnew file mode 100644index 0000000000..8364f371a2--- /dev/null+++ b/tests/runner-e2e/braintrust-gateway-context.ts@@ -0,0 +1,17 @@+import { validEvalLedgerId, type EvalChargeIdentity } from "./cost-ledger.js";++export function braintrustGatewayRequestParent(projectId: string, requestId: string, campaignId: string, identity: EvalChargeIdentity): string {+ if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(projectId)+ || ![requestId, campaignId, identity.executionId, identity.attemptId].every(validEvalLedgerId)+ || !["setup", "agent", "judge", "feedback_review", "tool", "infrastructure"].includes(identity.phase)) {+ throw new Error("Invalid eval gateway logging identity");+ }+ return Buffer.concat([Buffer.from([4, 2, 0]), Buffer.from(JSON.stringify({+ object_id: projectId,+ propagated_event: { metadata: {+ paperclip_eval_request_id: requestId, paperclip_eval_campaign_id: campaignId,+ paperclip_eval_execution_id: identity.executionId, paperclip_eval_attempt_id: identity.attemptId,+ paperclip_eval_phase: identity.phase,+ } },+ }))]).toString("base64");+}diff --git a/tests/runner-e2e/eval-gateway.test.ts b/tests/runner-e2e/eval-gateway.test.tsindex 4fa02239ca..ba5266e72f 100644--- a/tests/runner-e2e/eval-gateway.test.ts+++ b/tests/runner-e2e/eval-gateway.test.ts@@ -31,12 +31,54 @@ it("fsyncs reservation before dispatch, pins gateway routing, and replaces reserconst result = await forwardEvalRequest({ ...f, transport });expect(reservedBeforeDispatch).toBe(true);expect(transport.mock.calls[0]![0]).toBe("https://gateway.braintrust.dev/responses");- expect(captured).toMatchObject({ redirect: "error", headers: { authorization: `Bearer ${route.apiKey}`, "x-bt-use-cache": "never", "cache-control": "no-cache, no-store", "x-bt-project-id": route.projectId, "x-bt-endpoint-name": route.endpointName, "x-bt-parent": `project_id:${route.projectId}` } });+ expect(captured).toMatchObject({ redirect: "error", headers: { authorization: `Bearer ${route.apiKey}`, "x-bt-use-cache": "never", "cache-control": "no-cache, no-store", "x-bt-project-id": route.projectId, "x-bt-endpoint-name": route.endpointName, "x-bt-parent": result.braintrustParent } });expect(JSON.parse(captured!.body as string)).toMatchObject({ max_output_tokens: 100, service_tier: "default" });expect(result).toMatchObject({ outcome: "succeeded", costUsd: "0.000068000", final: true, usageComplete: true, gatewaySpanId: "gateway-span" });expect(f.ledger.snapshot()).toMatchObject({ knownUsd: "0.000068000", heldUsd: "0.000000000", inflight: 0 });expect(await readFile(f.file, "utf8")).not.toContain(route.apiKey);});+it.each(["openai", "anthropic"] as const)("correlates %s gateway logs before response headers even when the client cancels", async provider => {+ const f = await fixture();+ const selectedRoute: EvalGatewayRoute = provider === "openai" ? route : { ...route, provider, endpointName: "ANTHROPIC_API_KEY",+ price: { ...route.price, provider, cacheWrite5mUsdPerMillion: "2", cacheWrite1hUsdPerMillion: "4" } };+ const parents: string[] = [];+ const initialReceipts: EvalGatewayReceipt[] = [];+ for (let attempt = 0; attempt < 2; attempt++) {+ const controller = new AbortController();+ const receipt = await forwardEvalRequest({ ...f, route: selectedRoute, signal: controller.signal,+ transport: async (_url, init) => {+ parents.push(new Headers(init?.headers).get("x-bt-parent")!);+ initialReceipts.push(structuredClone(f.receipts.at(-1)!));+ controller.abort();+ throw new DOMException("client disconnected", "AbortError");+ },+ });+ expect(receipt).toMatchObject({ outcome: "cancelled", errorCode: "request_cancelled", costUsd: null, final: false });+ expect(receipt.gatewaySpanId).toBeUndefined();+ }+ expect(parents[0]).not.toBe(parents[1]);+ for (const [index, encoded] of parents.entries()) {+ const bytes = Buffer.from(encoded, "base64");+ expect([...bytes.subarray(0, 3)]).toEqual([4, 2, 0]);+ expect(JSON.parse(bytes.subarray(3).toString())).toEqual({ object_id: route.projectId, propagated_event: { metadata: {+ paperclip_eval_request_id: initialReceipts[index]!.requestId, paperclip_eval_campaign_id: "campaign",+ paperclip_eval_execution_id: identity.executionId, paperclip_eval_attempt_id: identity.attemptId, paperclip_eval_phase: "agent",+ } } });+ expect(initialReceipts[index]).toMatchObject({ braintrustParent: encoded, final: false, costUsd: null });+ }+ expect(f.ledger.snapshot()).toMatchObject({ inflight: 0, unknownRequests: 2, knownUsd: "0.000000000" });+ expect(f.ledger.snapshot().heldUsd).not.toBe("0.000000000");+ expect(JSON.stringify(initialReceipts)).not.toContain(route.apiKey);+});+it("keeps Braintrust request metadata out of direct provider requests", async () => {+ const f = await fixture();+ let headers: Headers | undefined;+ const receipt = await forwardEvalRequest({ ...f, route: { ...route, mode: "direct" }, transport: async (_url, init) => {+ headers = new Headers(init?.headers); return response();+ } });+ expect(headers!.has("x-bt-parent")).toBe(false);+ expect(receipt).not.toHaveProperty("braintrustParent");+});it("stops at the campaign cap before any upstream request", async () => {const f = await fixture("0.002"), transport = vi.fn<typeof fetch>();expect(evalRequestReservation(route)).toBe("0.003000000");diff --git a/tests/runner-e2e/eval-gateway.ts b/tests/runner-e2e/eval-gateway.tsindex 7252890866..26af76ff10 100644--- a/tests/runner-e2e/eval-gateway.ts+++ b/tests/runner-e2e/eval-gateway.ts@@ -1,6 +1,7 @@import { randomBytes, randomUUID, timingSafeEqual } from "node:crypto";import { createServer, type IncomingMessage, type ServerResponse } from "node:http";import { once } from "node:events";+import { braintrustGatewayRequestParent } from "./braintrust-gateway-context.js";import { usdToUnits } from "../../packages/shared/src/money.js";import { estimateProviderCost, ProviderUsageAccumulator, type EvalModelPrice, type ProviderUsageReceipt } from "./provider-usage.js";import { EvalAdmissionError, validEvalLedgerId, type EvalChargeIdentity, type openEvalCostLedger } from "./cost-ledger.js";@@ -19,7 +20,7 @@ export interface EvalGatewayRoute {}export interface EvalGatewayReceipt {requestId: string; status?: number; route: "direct" | "braintrust";- gatewayRequestId?: string; gatewaySpanId?: string; providerRequestId?: string;+ gatewayRequestId?: string; gatewaySpanId?: string; providerRequestId?: string; braintrustParent?: string;endpoint?: string; cache?: string; failoverFrom?: string; failoverTo?: string; errorOrigin?: string;outcome: "succeeded" | "failed" | "cancelled"; errorCode?: string;usageComplete: boolean; final: boolean; costUsd: string | null;@@ -111,7 +112,8 @@ export async function forwardEvalRequest(input: {if (input.signal?.aborted) throw new EvalRequestError("request_cancelled");if (Date.now() >= input.deadlineAt) throw new EvalAdmissionError("request_deadline_reached");const requestId = randomUUID();- const receipt: EvalGatewayReceipt = { requestId, route: route.mode, outcome: "failed", usageComplete: false, final: false, costUsd: null };+ const receipt: EvalGatewayReceipt = { requestId, route: route.mode, outcome: "failed", usageComplete: false, final: false, costUsd: null,+ ...(route.mode === "braintrust" ? { braintrustParent: braintrustGatewayRequestParent(route.projectId!, requestId, ledger.snapshot().campaignId, input.identity) } : {}) };const usage = new ProviderUsageAccumulator(route.provider, body);await ledger.reserve(requestId, input.identity, reservation);const abort = new AbortController();@@ -122,7 +124,7 @@ export async function forwardEvalRequest(input: {const headers: Record<string, string> = { "content-type": "application/json", "accept-encoding": "identity" };if (route.mode === "braintrust") Object.assign(headers, {authorization: `Bearer ${route.apiKey}`, "x-bt-project-id": route.projectId!, "x-bt-endpoint-name": route.endpointName!,- "x-bt-parent": `project_id:${route.projectId}`, "x-bt-use-cache": "never", "cache-control": "no-cache, no-store",+ "x-bt-parent": receipt.braintrustParent!, "x-bt-use-cache": "never", "cache-control": "no-cache, no-store",});else if (route.provider === "openai") headers.authorization = `Bearer ${route.apiKey}`;else headers["x-api-key"] = route.apiKey;