The Open Framework for Building Agents
test(e2e): make Opus prompt-cache validation optional (#3279)
Chad Hietaladf62d7361afd
diff --git a/.github/scripts/discover-e2e-fixtures.mjs b/.github/scripts/discover-e2e-fixtures.mjsindex 5cf4540a3..b74f4a4ba 100644--- a/.github/scripts/discover-e2e-fixtures.mjs+++ b/.github/scripts/discover-e2e-fixtures.mjs@@ -5,12 +5,14 @@// A fixture qualifies when it has an `evals/` directory under one of the// fixture roots. Emits one GitHub Actions output per matrix://-// model_matrix `{ name, dir, model_name, model_id }` entries for-// the model suite (e2e-local). Fixtures marked-// `"e2e": { "modelMatrix": "full" }` in package.json-// run on every registry model; all other fixtures run-// once on the default (first) model. A fixture may add-// narrowly scoped model legs through `additionalModels`.+// model_matrix `{ name, dir, model_name, model_id, optional }`+// entries for the model suite (e2e-local). Fixtures+// marked `"e2e": { "modelMatrix": "full" }` in+// package.json run on every registry model; all other+// fixtures run once on the default (first) model. A+// fixture may add narrowly scoped model legs through+// `additionalModels` and make selected legs non-blocking+// through `optionalModels`.// world_matrix_<world> `{ name, dir[, world_package] }` entries for that// world's suite workflow, which runs every fixture// once with mock models (EVE_E2E_MODEL=mock).@@ -32,6 +34,7 @@ for (const root of roots) {let modelMatrix = "default";let additionalModels = [];+ let optionalModels = [];const packageJsonPath = join(dir, "package.json");if (existsSync(packageJsonPath)) {const pkg = JSON.parse(readFileSync(packageJsonPath, "utf8"));@@ -42,12 +45,16 @@ for (const root of roots) {["id"],{ allowEmpty: true },);+ optionalModels = validateNames(+ pkg.e2e?.optionalModels ?? [],+ `${packageJsonPath}: e2e.optionalModels`,+ );}if (modelMatrix !== "default" && modelMatrix !== "full") {throw new Error(`${packageJsonPath}: e2e.modelMatrix must be "default" or "full".`);}- fixtures.push({ name: entry, dir, modelMatrix, additionalModels });+ fixtures.push({ name: entry, dir, modelMatrix, additionalModels, optionalModels });}}@@ -56,16 +63,28 @@ if (fixtures.length === 0) {process.exit(1);}-const modelMatrix = fixtures.flatMap(({ name, dir, modelMatrix, additionalModels }) =>- uniqueModels([- ...(modelMatrix === "full" ? models : models.slice(0, 1)),- ...additionalModels,- ]).map((model) => ({- name,- dir,- model_name: model.name,- model_id: model.id,- })),+const modelMatrix = fixtures.flatMap(+ ({ name, dir, modelMatrix, additionalModels, optionalModels }) => {+ const fixtureModels = uniqueModels([+ ...(modelMatrix === "full" ? models : models.slice(0, 1)),+ ...additionalModels,+ ]);+ const selectedNames = new Set(fixtureModels.map((model) => model.name));+ for (const optionalModel of optionalModels) {+ if (!selectedNames.has(optionalModel)) {+ throw new Error(+ `${dir}/package.json: optional model "${optionalModel}" is not selected by this fixture.`,+ );+ }+ }+ return fixtureModels.map((model) => ({+ name,+ dir,+ model_name: model.name,+ model_id: model.id,+ optional: optionalModels.includes(model.name),+ }));+ },);const outputs = [`model_matrix=${JSON.stringify(modelMatrix)}`];@@ -110,6 +129,23 @@ function validateNamedEntries(entries, key, requiredFields, options = {}) {return entries;}+function validateNames(entries, key) {+ if (!Array.isArray(entries)) {+ throw new Error(`${key} must be an array.`);+ }+ const names = new Set();+ for (const entry of entries) {+ if (typeof entry !== "string" || !/^[a-z0-9-]+$/.test(entry)) {+ throw new Error(`${key} entries must be lowercase alphanumerics and dashes.`);+ }+ if (names.has(entry)) {+ throw new Error(`${key} contains duplicate model "${entry}".`);+ }+ names.add(entry);+ }+ return entries;+}+function uniqueModels(entries) {const ids = new Set();const names = new Set();diff --git a/.github/workflows/e2e-local.yml b/.github/workflows/e2e-local.ymlindex 4dee19dcc..f14a21d7a 100644--- a/.github/workflows/e2e-local.yml+++ b/.github/workflows/e2e-local.yml@@ -66,6 +66,7 @@ jobs:name: e2e-local (${{ matrix.name }}, ${{ matrix.model_name }})runs-on: ubuntu-latestneeds: [changes, discover-fixtures]+ continue-on-error: ${{ matrix.optional }}# No job-level `if`: the stable aggregate check below requires a successful# matrix result. Guarding each step lets an irrelevant diff satisfy it in# seconds without running setup or evals.diff --git a/e2e/README.md b/e2e/README.mdindex 81be80e88..c2843dd42 100644--- a/e2e/README.md+++ b/e2e/README.md@@ -173,6 +173,8 @@ matrices from the registry:on fixtures with `"e2e": { "modelMatrix": "full" }` in package.json.A fixture can add a narrowly scoped leg with `e2e.additionalModels` entriesshaped as `{ "name": "short-check-name", "id": "provider/model" }`.+ `e2e.optionalModels` can name selected model legs that should still run and+ report failures without blocking the aggregate check.- `world_matrix_<world>` — one leg per fixture for that world's suiteworkflow. A registered world's `package` reaches the job as`EVE_E2E_WORKFLOW_WORLD` (worlds without one, like `vercel`, use thediff --git a/e2e/fixtures/agent-prompt-cache/package.json b/e2e/fixtures/agent-prompt-cache/package.jsonindex f51440c8e..0f3ff7111 100644--- a/e2e/fixtures/agent-prompt-cache/package.json+++ b/e2e/fixtures/agent-prompt-cache/package.json@@ -21,6 +21,9 @@"typescript": "catalog:"},"e2e": {- "modelMatrix": "default"+ "modelMatrix": "full",+ "optionalModels": [+ "anthropic-opus"+ ]}}