Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

test(e2e): make Opus prompt-cache validation optional (#3279)

Chad Hietaladf62d7361afd
diff --git a/.github/scripts/discover-e2e-fixtures.mjs b/.github/scripts/discover-e2e-fixtures.mjs
index 5cf4540a3..b74f4a4ba 100644
--- a/.github/scripts/discover-e2e-fixtures.mjs
+++ b/.github/scripts/discover-e2e-fixtures.mjs
@@ -5,12 +5,14 @@
// A fixture qualifies when it has an `evals/` directory under one of the
// fixture roots. Emits one GitHub Actions output per matrix:
//
-// model_matrix `{ name, dir, model_name, model_id }` entries for
-// the model suite (e2e-local). Fixtures marked
-// `"e2e": { "modelMatrix": "full" }` in package.json
-// run on every registry model; all other fixtures run
-// once on the default (first) model. A fixture may add
-// narrowly scoped model legs through `additionalModels`.
+// model_matrix `{ name, dir, model_name, model_id, optional }`
+// entries for the model suite (e2e-local). Fixtures
+// marked `"e2e": { "modelMatrix": "full" }` in
+// package.json run on every registry model; all other
+// fixtures run once on the default (first) model. A
+// fixture may add narrowly scoped model legs through
+// `additionalModels` and make selected legs non-blocking
+// through `optionalModels`.
// world_matrix_<world> `{ name, dir[, world_package] }` entries for that
// world's suite workflow, which runs every fixture
// once with mock models (EVE_E2E_MODEL=mock).
@@ -32,6 +34,7 @@ for (const root of roots) {
let modelMatrix = "default";
let additionalModels = [];
+ let optionalModels = [];
const packageJsonPath = join(dir, "package.json");
if (existsSync(packageJsonPath)) {
const pkg = JSON.parse(readFileSync(packageJsonPath, "utf8"));
@@ -42,12 +45,16 @@ for (const root of roots) {
["id"],
{ allowEmpty: true },
);
+ optionalModels = validateNames(
+ pkg.e2e?.optionalModels ?? [],
+ `${packageJsonPath}: e2e.optionalModels`,
+ );
}
if (modelMatrix !== "default" && modelMatrix !== "full") {
throw new Error(`${packageJsonPath}: e2e.modelMatrix must be "default" or "full".`);
}
- fixtures.push({ name: entry, dir, modelMatrix, additionalModels });
+ fixtures.push({ name: entry, dir, modelMatrix, additionalModels, optionalModels });
}
}
@@ -56,16 +63,28 @@ if (fixtures.length === 0) {
process.exit(1);
}
-const modelMatrix = fixtures.flatMap(({ name, dir, modelMatrix, additionalModels }) =>
- uniqueModels([
- ...(modelMatrix === "full" ? models : models.slice(0, 1)),
- ...additionalModels,
- ]).map((model) => ({
- name,
- dir,
- model_name: model.name,
- model_id: model.id,
- })),
+const modelMatrix = fixtures.flatMap(
+ ({ name, dir, modelMatrix, additionalModels, optionalModels }) => {
+ const fixtureModels = uniqueModels([
+ ...(modelMatrix === "full" ? models : models.slice(0, 1)),
+ ...additionalModels,
+ ]);
+ const selectedNames = new Set(fixtureModels.map((model) => model.name));
+ for (const optionalModel of optionalModels) {
+ if (!selectedNames.has(optionalModel)) {
+ throw new Error(
+ `${dir}/package.json: optional model "${optionalModel}" is not selected by this fixture.`,
+ );
+ }
+ }
+ return fixtureModels.map((model) => ({
+ name,
+ dir,
+ model_name: model.name,
+ model_id: model.id,
+ optional: optionalModels.includes(model.name),
+ }));
+ },
);
const outputs = [`model_matrix=${JSON.stringify(modelMatrix)}`];
@@ -110,6 +129,23 @@ function validateNamedEntries(entries, key, requiredFields, options = {}) {
return entries;
}
+function validateNames(entries, key) {
+ if (!Array.isArray(entries)) {
+ throw new Error(`${key} must be an array.`);
+ }
+ const names = new Set();
+ for (const entry of entries) {
+ if (typeof entry !== "string" || !/^[a-z0-9-]+$/.test(entry)) {
+ throw new Error(`${key} entries must be lowercase alphanumerics and dashes.`);
+ }
+ if (names.has(entry)) {
+ throw new Error(`${key} contains duplicate model "${entry}".`);
+ }
+ names.add(entry);
+ }
+ return entries;
+}
+
function uniqueModels(entries) {
const ids = new Set();
const names = new Set();
diff --git a/.github/workflows/e2e-local.yml b/.github/workflows/e2e-local.yml
index 4dee19dcc..f14a21d7a 100644
--- a/.github/workflows/e2e-local.yml
+++ b/.github/workflows/e2e-local.yml
@@ -66,6 +66,7 @@ jobs:
name: e2e-local (${{ matrix.name }}, ${{ matrix.model_name }})
runs-on: ubuntu-latest
needs: [changes, discover-fixtures]
+ continue-on-error: ${{ matrix.optional }}
# No job-level `if`: the stable aggregate check below requires a successful
# matrix result. Guarding each step lets an irrelevant diff satisfy it in
# seconds without running setup or evals.
diff --git a/e2e/README.md b/e2e/README.md
index 81be80e88..c2843dd42 100644
--- a/e2e/README.md
+++ b/e2e/README.md
@@ -173,6 +173,8 @@ matrices from the registry:
on fixtures with `"e2e": { "modelMatrix": "full" }` in package.json.
A fixture can add a narrowly scoped leg with `e2e.additionalModels` entries
shaped as `{ "name": "short-check-name", "id": "provider/model" }`.
+ `e2e.optionalModels` can name selected model legs that should still run and
+ report failures without blocking the aggregate check.
- `world_matrix_<world>` — one leg per fixture for that world's suite
workflow. A registered world's `package` reaches the job as
`EVE_E2E_WORKFLOW_WORLD` (worlds without one, like `vercel`, use the
diff --git a/e2e/fixtures/agent-prompt-cache/package.json b/e2e/fixtures/agent-prompt-cache/package.json
index f51440c8e..0f3ff7111 100644
--- a/e2e/fixtures/agent-prompt-cache/package.json
+++ b/e2e/fixtures/agent-prompt-cache/package.json
@@ -21,6 +21,9 @@
"typescript": "catalog:"
},
"e2e": {
- "modelMatrix": "default"
+ "modelMatrix": "full",
+ "optionalModels": [
+ "anthropic-opus"
+ ]
}
}