Skip to content

The open-source app everyone uses to manage agents at work

perf(evals): tighten measured campaign resource ceilings (#17381)

Dotta52a66af29ddc
diff --git a/tests/runner-e2e/OPERATIONS.md b/tests/runner-e2e/OPERATIONS.md
index 52650357b0..1dc7b6c325 100644
--- a/tests/runner-e2e/OPERATIONS.md
+++ b/tests/runner-e2e/OPERATIONS.md
@@ -364,8 +364,10 @@ rejects ambiguous singular fields and routing values.
Each relay allows at most 2,048 admitted exports, 8 MiB per request before and
after decompression, 64 MiB of forwarded payloads, and four simultaneous exports.
-It also limits each attempt to 16,384 records and 256 MiB of expanded source
-JSON. This measurement includes the full decoded source plus each record in
+The default allowance limits each attempt to 4,096 records and 16 MiB of
+expanded source JSON. Historical receipts with up to 16,384 records and 256 MiB
+remain readable; controller callers may still supply explicit limits up to
+those maxima. This measurement includes the full decoded source plus each record in
its own envelope with the shared resource and scope fields repeated. It counts
span names, IDs, times, events, links and log bodies, not only routing attributes.
Protobuf fields use the pinned OTLP v1.9 schema; unknown fields fail admission
@@ -408,7 +410,7 @@ or an invoice. Vendor charges remain unreconciled. The current two-attempt daily
maximum is 132.125 MiB and the four-attempt weekly maximum is 260.25 MiB,
including finalization and the campaign's experiment payload allowance. These
bounds do not include other projects, standalone imports or retained storage.
-The corresponding expanded-source ceilings are 514 MiB daily and 1,028 MiB
+The corresponding default expanded-source ceilings are 34 MiB daily and 68 MiB
weekly before the separate 4-MiB experiment allowance. These source measurements
support a priced forecast; they are not Braintrust's processed-data meter or a
vendor-enforced dollar limit. Gateway auto-logging is a separate input to that
@@ -463,7 +465,12 @@ the existing AWS/GitHub runner choice.
Daily selects native Claude's marker and feedback-persistence cases with a shared
$20 allowance. Weekly adds native Codex's two corresponding cases with a
-shared $100 allowance. Model requests and declared operating overhead share
+shared $100 allowance. Daily campaigns admit at most 30 model requests; weekly
+campaigns admit at most 60. The October 23 qualification campaigns used 10 model
+requests each. The largest observed native attempt exported 6,531,801 expanded
+source bytes and 2,455 records. The defaults retain headroom for those measured
+cases; exhausting a limit remains visible and never silently drops required
+evidence. Model requests and declared operating overhead share
this limit. Each campaign runs sequentially in one job with one
journal, one attempt per execution, zero automatic retries, a 30-minute request
deadline and a finite request count. Codex's worst-case request reservation exceeds
diff --git a/tests/runner-e2e/eval-otlp-relay.test.ts b/tests/runner-e2e/eval-otlp-relay.test.ts
index d241934242..7b220dc0f4 100644
--- a/tests/runner-e2e/eval-otlp-relay.test.ts
+++ b/tests/runner-e2e/eval-otlp-relay.test.ts
@@ -239,7 +239,7 @@ it.each(["json", "protobuf"])("meters repeated resource/scope data and full reco
it("accepts thousands of small native records while bounding their total count", async () => {
const upstream = vi.fn<typeof fetch>(async () => Response.json({}));
- const f = await fixture({ transport: upstream, limits: { ...EVAL_OTLP_LIMITS, maxRecords: 3000 } });
+ const f = await fixture({ transport: upstream });
const body = JSON.stringify({ resource_logs: [{ scope_logs: [{ log_records: Array.from({ length: 2533 }, () => ({ body: { stringValue: "event" } })) }] }] });
expect((await f.send(body, {}, "/v1/logs")).status).toBe(200);
expect((await f.send(body, {}, "/v1/logs")).status).toBe(429);
@@ -247,6 +247,23 @@ it("accepts thousands of small native records while bounding their total count",
expect((await f.records()).find(row => row.type === "admitted").recordCount).toBe(2533);
});
+it("enforces the smaller default expanded-source allowance before export", async () => {
+ const upstream = vi.fn<typeof fetch>(async () => Response.json({}));
+ const f = await fixture({ transport: upstream });
+ const body = JSON.stringify({ resourceLogs: [{ scopeLogs: [{ logRecords: [
+ { body: { stringValue: "x".repeat(3 * 1024 * 1024) } },
+ ] }] }] });
+ expect((await f.send(body, {}, "/v1/logs")).status).toBe(200);
+ expect((await f.send(body, {}, "/v1/logs")).status).toBe(200);
+ expect((await f.send(body, {}, "/v1/logs")).status).toBe(429);
+ expect(upstream).toHaveBeenCalledTimes(2);
+ await f.relay.close();
+ const rows = await f.records();
+ expect(rows[0].limits.maxExpandedBytes).toBe(16 * 1024 * 1024);
+ expect(rows.at(-1).admittedExpandedBytes).toBeGreaterThan(12 * 1024 * 1024);
+ expect(rows.filter(row => row.type === "quota_exhausted")).toHaveLength(1);
+});
+
it("rejects unknown protobuf fields instead of excluding forwarded data from sizing", async () => {
const upstream = vi.fn<typeof fetch>(async () => Response.json({}));
const f = await fixture({ transport: upstream });
diff --git a/tests/runner-e2e/eval-otlp-relay.ts b/tests/runner-e2e/eval-otlp-relay.ts
index 36e61d9dde..d34efd814e 100644
--- a/tests/runner-e2e/eval-otlp-relay.ts
+++ b/tests/runner-e2e/eval-otlp-relay.ts
@@ -9,12 +9,14 @@ import { evalOtlpRejected, evalOtlpResponse, isEvalProjectId, validateEvalOtlpPa
export const EVAL_OTLP_LIMITS = { maxRequests: 2048, maxRequestBytes: 8 * 1024 * 1024,
maxTotalBytes: 64 * 1024 * 1024, maxConcurrent: 4,
maxExpandedBytes: 256 * 1024 * 1024, maxRecords: 16_384 } as const;
+export const EVAL_OTLP_DEFAULT_LIMITS = { ...EVAL_OTLP_LIMITS,
+ maxExpandedBytes: 16 * 1024 * 1024, maxRecords: 4096 } as const;
export async function startEvalOtlpRelay(input: {
projectId: string; apiKey: string; campaignId: string; attemptId: string; receiptPath: string; deadlineAt: number;
limits?: { [K in keyof typeof EVAL_OTLP_LIMITS]: number }; transport?: typeof fetch;
}) {
- const limits = input.limits ?? EVAL_OTLP_LIMITS;
+ const limits = input.limits ?? EVAL_OTLP_DEFAULT_LIMITS;
if (!isEvalProjectId(input.projectId) || !input.apiKey.trim() || /[\r\n]/.test(input.apiKey)
|| ![input.campaignId, input.attemptId].every(value => /^[A-Za-z0-9_.:-]{1,512}$/.test(value))
|| !path.isAbsolute(input.receiptPath) || !Number.isFinite(input.deadlineAt)
diff --git a/tests/runner-e2e/ingestion-usage.test.ts b/tests/runner-e2e/ingestion-usage.test.ts
index fdb327f454..38e921b79f 100644
--- a/tests/runner-e2e/ingestion-usage.test.ts
+++ b/tests/runner-e2e/ingestion-usage.test.ts
@@ -80,6 +80,16 @@ it("binds the expected receipt set to admitted attempts and retains unstarted se
await expect(collectScheduledIngestion({ ...f.input, campaignId: "foreign" })).rejects.toThrow("campaign mismatch");
});
+it("keeps historical native receipts with the larger policy limits readable", async () => {
+ const f = await fixture(), file = f.files[0]!;
+ const rows = (await readFile(file, "utf8")).trim().split("\n").map(line => JSON.parse(line));
+ rows[0].limits.maxExpandedBytes = 256 * 1024 * 1024;
+ rows[0].limits.maxRecords = 16_384;
+ await writeFile(file, rows.map(row => JSON.stringify(row)).join("\n") + "\n");
+ expect(await collectScheduledIngestion(f.input)).toMatchObject({ status: "complete",
+ totalAdmittedPayloadBytes: f.expectedBytes, billedCostUsd: null });
+});
+
it("keeps version 1 transport receipts readable without inventing expanded data measurements", async () => {
const f = await fixture(), file = f.files[0]!;
const rows = (await readFile(file, "utf8")).trim().split("\n").map(line => JSON.parse(line));
diff --git a/tests/runner-e2e/scheduled-operations.test.ts b/tests/runner-e2e/scheduled-operations.test.ts
index a9856edd9c..5fc31097df 100644
--- a/tests/runner-e2e/scheduled-operations.test.ts
+++ b/tests/runner-e2e/scheduled-operations.test.ts
@@ -19,7 +19,7 @@ it.each(["daily", "weekly"] as const)("runs the exact %s selection through one s
const manifest = scheduledManifest({ ...input, cadence });
const selected = selectRunnerExecutions(parseRunnerSelectors(manifest.executionIds.flatMap(id => ["--id", id])), runnerMatrix);
expect(selected.map(row => row.id).sort()).toEqual([...scheduledSelection(cadence)].sort());
- expect(manifest.limits).toMatchObject({ maxUsd: cadence === "daily" ? "20" : "100", maxConcurrent: 1, maxAttemptsPerExecution: 1 });
+ expect(manifest.limits).toMatchObject({ maxUsd: cadence === "daily" ? "20" : "100", maxRequests: cadence === "daily" ? 30 : 60, maxConcurrent: 1, maxAttemptsPerExecution: 1 });
expect(Date.parse(manifest.limits.deadlineAt) - input.now).toBe(30 * 60_000);
expect(scheduledMatrix(cadence).include).toHaveLength(1);
expect(manifest.executionIds).toHaveLength(cadence === "daily" ? 2 : 4);
diff --git a/tests/runner-e2e/scheduled-operations.ts b/tests/runner-e2e/scheduled-operations.ts
index f92c8ff135..c2a5e80cfc 100644
--- a/tests/runner-e2e/scheduled-operations.ts
+++ b/tests/runner-e2e/scheduled-operations.ts
@@ -64,7 +64,7 @@ export function scheduledManifest(input: {
const { executionIds, executions, models } = validateScheduledBudget(input);
const manifest: EvalOperationsManifest = {
version: 1, limits: { campaignId: input.campaignId, maxUsd: budget(input.cadence), overhead: input.overhead,
- deadlineAt: new Date(input.now + 30 * 60_000).toISOString(), maxRequests: input.cadence === "daily" ? 60 : 120,
+ deadlineAt: new Date(input.now + 30 * 60_000).toISOString(), maxRequests: input.cadence === "daily" ? 30 : 60,
maxConcurrent: 1, maxAttemptsPerExecution: 1 },
ledgerPath: input.ledgerPath, executionIds, maxAttemptsPerExecution: 1, route: "braintrust", projectId: input.projectId,
models,