The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix(evals): select operating allowance by cadence (#17326)
Dottaed3f067dbd71
diff --git a/tests/runner-e2e/OPERATIONS.md b/tests/runner-e2e/OPERATIONS.mdindex 31b36ed856..5ec1bc4890 100644--- a/tests/runner-e2e/OPERATIONS.md+++ b/tests/runner-e2e/OPERATIONS.md@@ -375,8 +375,12 @@ maximum allowances, not forecasts of actual spending.This is an admission limit with retained reservations, not a reconciled invoicetotal or a provider-enforced billing limit.-Before enabling recurrence, set `RUNNER_E2E_OPERATING_OVERHEAD_JSON` to a reviewed-allowance with `version: 1`, an evidence `reference`, the actual `runner`+Before enabling recurrence, set `RUNNER_E2E_OPERATING_OVERHEAD_JSON` to an object+with `daily` and `weekly` reviewed allowances. The catalog selects the allowance+for the actual trigger, then passes only that validated allowance to preparation.+A missing selected cadence or unknown outer field fails admission. A single+allowance remains supported when the same bounds fit both schedules. Each+allowance has `version: 1`, an evidence `reference`, the actual `runner`(`github-hosted` for bounded schedules), and `verifiedAt`/`expiresAt` timestamps no morethan 31 days apart. Its `components` array must contain exactly `ci_compute`,`artifact_storage`, `observability`, and `shared_subscription`. Each componentdiff --git a/tests/runner-e2e/operating-overhead.ts b/tests/runner-e2e/operating-overhead.tsindex 08396df0ef..4f53cd7517 100644--- a/tests/runner-e2e/operating-overhead.ts+++ b/tests/runner-e2e/operating-overhead.ts@@ -40,10 +40,16 @@ export function validateOperatingOverhead(value: EvalOperatingOverhead, totalUsdreturn reserved;}-export function scheduledOperatingOverhead(raw: string | undefined, runner: string | undefined, totalUsd: string, now: number): EvalOperatingOverhead {+export function scheduledOperatingOverhead(raw: string | undefined, runner: string | undefined, totalUsd: string, now: number, cadence?: "daily" | "weekly"): EvalOperatingOverhead {if (!raw || raw.length > 16_384) throw new Error("Missing reviewed operating overhead allowance");- let value: EvalOperatingOverhead;- try { value = JSON.parse(raw); } catch { throw new Error("Invalid operating overhead JSON"); }+ let parsed: unknown;+ try { parsed = JSON.parse(raw); } catch { throw new Error("Invalid operating overhead JSON"); }+ if (parsed && typeof parsed === "object" && !Array.isArray(parsed) && !("version" in parsed)) {+ object(parsed, ["daily", "weekly"]);+ if (!cadence) throw new Error("Missing operating overhead cadence");+ parsed = (parsed as Record<string, unknown>)[cadence];+ }+ const value = parsed as EvalOperatingOverhead;validateOperatingOverhead(value, totalUsd);if (runner !== "ubuntu-latest" || value.runner !== "github-hosted") throw new Error("Bounded campaigns require the standard GitHub-hosted runner allowance");if (!Number.isFinite(now) || now < Date.parse(value.verifiedAt) || now + 4 * 3_600_000 >= Date.parse(value.expiresAt)) {diff --git a/tests/runner-e2e/scheduled-operations.test.ts b/tests/runner-e2e/scheduled-operations.test.tsindex 72ebe6d4d1..a9856edd9c 100644--- a/tests/runner-e2e/scheduled-operations.test.ts+++ b/tests/runner-e2e/scheduled-operations.test.ts@@ -75,3 +75,26 @@ it("rejects an overhead allowance that cannot leave room for a full selected reqexpect(() => validateScheduledBudget({ ...input, overhead: expensive })).toThrow("insufficient allowance");expect(validateScheduledBudget({ ...input, overhead })).toMatchObject({ executionIds: scheduledSelection("daily") });});+++it.each(["daily", "weekly"] as const)("prepares %s with its own reviewed operating allowance", async cadence => {+ const directory = await mkdtemp(path.join(tmpdir(), "eval-cadence-overhead-"));+ const allowance = (value: string): EvalOperatingOverhead => ({ ...overhead, reference: `fixture-${value}`,+ components: overhead.components.map(component => ({ ...component,+ maxUsd: component.component === "observability" ? value : component.component === "artifact_storage" ? "1" : "0" })) });+ const allowances = { daily: allowance("10"), weekly: allowance("20") };+ const env = { EVAL_EVENT: "workflow_dispatch", EVAL_CADENCE: cadence, EVAL_OPERATIONS_ENABLED: "true",+ GITHUB_RUN_ID: "123", GITHUB_RUN_ATTEMPT: "1", GITHUB_WORKSPACE: directory,+ EVAL_OPERATING_OVERHEAD_JSON: JSON.stringify(allowances), EVAL_TEST_RUNNER: "ubuntu-latest",+ PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: input.projectId };+ try {+ await expect(prepareScheduledCampaign({ ...env,+ EVAL_OPERATING_OVERHEAD_JSON: JSON.stringify({ ...allowances, [cadence]: undefined }) }, input.now)).rejects.toThrow();+ await expect(prepareScheduledCampaign({ ...env,+ EVAL_OPERATING_OVERHEAD_JSON: JSON.stringify({ ...allowances, typo: allowances.daily }) }, input.now)).rejects.toThrow();+ const prepared = await prepareScheduledCampaign(env, input.now);+ expect(prepared.manifest.limits.overhead).toEqual(allowances[cadence]);+ expect(prepared.manifest.limits.maxUsd).toBe(cadence === "daily" ? "20" : "100");+ expect(prepared.manifest.executionIds).toEqual(scheduledSelection(cadence));+ } finally { await rm(directory, { recursive: true, force: true }); }+});diff --git a/tests/runner-e2e/scheduled-operations.ts b/tests/runner-e2e/scheduled-operations.tsindex 84bb950e08..f92c8ff135 100644--- a/tests/runner-e2e/scheduled-operations.ts+++ b/tests/runner-e2e/scheduled-operations.ts@@ -87,7 +87,7 @@ export async function prepareScheduledCampaign(env: NodeJS.ProcessEnv, now = Datconst directory = path.join(env.GITHUB_WORKSPACE, "tests/runner-e2e/results/scheduled");const manifest = scheduledManifest({ cadence, campaignId, ledgerPath: path.join(directory, "ledger.jsonl"),projectId: env.PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID ?? "", now, runAttempt: env.GITHUB_RUN_ATTEMPT ?? "",- overhead: scheduledOperatingOverhead(env.EVAL_OPERATING_OVERHEAD_JSON, env.EVAL_TEST_RUNNER, budget(cadence), now) });+ overhead: scheduledOperatingOverhead(env.EVAL_OPERATING_OVERHEAD_JSON, env.EVAL_TEST_RUNNER, budget(cadence), now, cadence) });await mkdir(directory, { recursive: true, mode: 0o700 });const manifestPath = path.join(directory, "manifest.json");await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, { mode: 0o600, flag: "wx" });@@ -109,7 +109,7 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href)const cadence = scheduledCadence(process.env.EVAL_EVENT ?? "", process.env.EVAL_SCHEDULE ?? "", process.env.EVAL_CADENCE);if (!cadence || process.env.EVAL_OPERATIONS_ENABLED !== "true") throw new Error("Bounded eval operations are not enabled");const now = Date.now();- const overhead = scheduledOperatingOverhead(process.env.EVAL_OPERATING_OVERHEAD_JSON, process.env.EVAL_TEST_RUNNER, budget(cadence), now);+ const overhead = scheduledOperatingOverhead(process.env.EVAL_OPERATING_OVERHEAD_JSON, process.env.EVAL_TEST_RUNNER, budget(cadence), now, cadence);validateScheduledBudget({ cadence, overhead, now });console.log(JSON.stringify({ ...scheduledMatrix(cadence), overhead }));} else if (command === "prepare") {