Skip to content

The open-source app everyone uses to manage agents at work

fix(evals): select operating allowance by cadence (#17326)

Dottaed3f067dbd71
diff --git a/tests/runner-e2e/OPERATIONS.md b/tests/runner-e2e/OPERATIONS.md
index 31b36ed856..5ec1bc4890 100644
--- a/tests/runner-e2e/OPERATIONS.md
+++ b/tests/runner-e2e/OPERATIONS.md
@@ -375,8 +375,12 @@ maximum allowances, not forecasts of actual spending.
This is an admission limit with retained reservations, not a reconciled invoice
total or a provider-enforced billing limit.
-Before enabling recurrence, set `RUNNER_E2E_OPERATING_OVERHEAD_JSON` to a reviewed
-allowance with `version: 1`, an evidence `reference`, the actual `runner`
+Before enabling recurrence, set `RUNNER_E2E_OPERATING_OVERHEAD_JSON` to an object
+with `daily` and `weekly` reviewed allowances. The catalog selects the allowance
+for the actual trigger, then passes only that validated allowance to preparation.
+A missing selected cadence or unknown outer field fails admission. A single
+allowance remains supported when the same bounds fit both schedules. Each
+allowance has `version: 1`, an evidence `reference`, the actual `runner`
(`github-hosted` for bounded schedules), and `verifiedAt`/`expiresAt` timestamps no more
than 31 days apart. Its `components` array must contain exactly `ci_compute`,
`artifact_storage`, `observability`, and `shared_subscription`. Each component
diff --git a/tests/runner-e2e/operating-overhead.ts b/tests/runner-e2e/operating-overhead.ts
index 08396df0ef..4f53cd7517 100644
--- a/tests/runner-e2e/operating-overhead.ts
+++ b/tests/runner-e2e/operating-overhead.ts
@@ -40,10 +40,16 @@ export function validateOperatingOverhead(value: EvalOperatingOverhead, totalUsd
return reserved;
}
-export function scheduledOperatingOverhead(raw: string | undefined, runner: string | undefined, totalUsd: string, now: number): EvalOperatingOverhead {
+export function scheduledOperatingOverhead(raw: string | undefined, runner: string | undefined, totalUsd: string, now: number, cadence?: "daily" | "weekly"): EvalOperatingOverhead {
if (!raw || raw.length > 16_384) throw new Error("Missing reviewed operating overhead allowance");
- let value: EvalOperatingOverhead;
- try { value = JSON.parse(raw); } catch { throw new Error("Invalid operating overhead JSON"); }
+ let parsed: unknown;
+ try { parsed = JSON.parse(raw); } catch { throw new Error("Invalid operating overhead JSON"); }
+ if (parsed && typeof parsed === "object" && !Array.isArray(parsed) && !("version" in parsed)) {
+ object(parsed, ["daily", "weekly"]);
+ if (!cadence) throw new Error("Missing operating overhead cadence");
+ parsed = (parsed as Record<string, unknown>)[cadence];
+ }
+ const value = parsed as EvalOperatingOverhead;
validateOperatingOverhead(value, totalUsd);
if (runner !== "ubuntu-latest" || value.runner !== "github-hosted") throw new Error("Bounded campaigns require the standard GitHub-hosted runner allowance");
if (!Number.isFinite(now) || now < Date.parse(value.verifiedAt) || now + 4 * 3_600_000 >= Date.parse(value.expiresAt)) {
diff --git a/tests/runner-e2e/scheduled-operations.test.ts b/tests/runner-e2e/scheduled-operations.test.ts
index 72ebe6d4d1..a9856edd9c 100644
--- a/tests/runner-e2e/scheduled-operations.test.ts
+++ b/tests/runner-e2e/scheduled-operations.test.ts
@@ -75,3 +75,26 @@ it("rejects an overhead allowance that cannot leave room for a full selected req
expect(() => validateScheduledBudget({ ...input, overhead: expensive })).toThrow("insufficient allowance");
expect(validateScheduledBudget({ ...input, overhead })).toMatchObject({ executionIds: scheduledSelection("daily") });
});
+
+
+it.each(["daily", "weekly"] as const)("prepares %s with its own reviewed operating allowance", async cadence => {
+ const directory = await mkdtemp(path.join(tmpdir(), "eval-cadence-overhead-"));
+ const allowance = (value: string): EvalOperatingOverhead => ({ ...overhead, reference: `fixture-${value}`,
+ components: overhead.components.map(component => ({ ...component,
+ maxUsd: component.component === "observability" ? value : component.component === "artifact_storage" ? "1" : "0" })) });
+ const allowances = { daily: allowance("10"), weekly: allowance("20") };
+ const env = { EVAL_EVENT: "workflow_dispatch", EVAL_CADENCE: cadence, EVAL_OPERATIONS_ENABLED: "true",
+ GITHUB_RUN_ID: "123", GITHUB_RUN_ATTEMPT: "1", GITHUB_WORKSPACE: directory,
+ EVAL_OPERATING_OVERHEAD_JSON: JSON.stringify(allowances), EVAL_TEST_RUNNER: "ubuntu-latest",
+ PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: input.projectId };
+ try {
+ await expect(prepareScheduledCampaign({ ...env,
+ EVAL_OPERATING_OVERHEAD_JSON: JSON.stringify({ ...allowances, [cadence]: undefined }) }, input.now)).rejects.toThrow();
+ await expect(prepareScheduledCampaign({ ...env,
+ EVAL_OPERATING_OVERHEAD_JSON: JSON.stringify({ ...allowances, typo: allowances.daily }) }, input.now)).rejects.toThrow();
+ const prepared = await prepareScheduledCampaign(env, input.now);
+ expect(prepared.manifest.limits.overhead).toEqual(allowances[cadence]);
+ expect(prepared.manifest.limits.maxUsd).toBe(cadence === "daily" ? "20" : "100");
+ expect(prepared.manifest.executionIds).toEqual(scheduledSelection(cadence));
+ } finally { await rm(directory, { recursive: true, force: true }); }
+});
diff --git a/tests/runner-e2e/scheduled-operations.ts b/tests/runner-e2e/scheduled-operations.ts
index 84bb950e08..f92c8ff135 100644
--- a/tests/runner-e2e/scheduled-operations.ts
+++ b/tests/runner-e2e/scheduled-operations.ts
@@ -87,7 +87,7 @@ export async function prepareScheduledCampaign(env: NodeJS.ProcessEnv, now = Dat
const directory = path.join(env.GITHUB_WORKSPACE, "tests/runner-e2e/results/scheduled");
const manifest = scheduledManifest({ cadence, campaignId, ledgerPath: path.join(directory, "ledger.jsonl"),
projectId: env.PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID ?? "", now, runAttempt: env.GITHUB_RUN_ATTEMPT ?? "",
- overhead: scheduledOperatingOverhead(env.EVAL_OPERATING_OVERHEAD_JSON, env.EVAL_TEST_RUNNER, budget(cadence), now) });
+ overhead: scheduledOperatingOverhead(env.EVAL_OPERATING_OVERHEAD_JSON, env.EVAL_TEST_RUNNER, budget(cadence), now, cadence) });
await mkdir(directory, { recursive: true, mode: 0o700 });
const manifestPath = path.join(directory, "manifest.json");
await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, { mode: 0o600, flag: "wx" });
@@ -109,7 +109,7 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href)
const cadence = scheduledCadence(process.env.EVAL_EVENT ?? "", process.env.EVAL_SCHEDULE ?? "", process.env.EVAL_CADENCE);
if (!cadence || process.env.EVAL_OPERATIONS_ENABLED !== "true") throw new Error("Bounded eval operations are not enabled");
const now = Date.now();
- const overhead = scheduledOperatingOverhead(process.env.EVAL_OPERATING_OVERHEAD_JSON, process.env.EVAL_TEST_RUNNER, budget(cadence), now);
+ const overhead = scheduledOperatingOverhead(process.env.EVAL_OPERATING_OVERHEAD_JSON, process.env.EVAL_TEST_RUNNER, budget(cadence), now, cadence);
validateScheduledBudget({ cadence, overhead, now });
console.log(JSON.stringify({ ...scheduledMatrix(cadence), overhead }));
} else if (command === "prepare") {