Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

test(e2e): split model and world suites (#1387)

Andrew Barba01ed80d3c008
diff --git a/.changeset/eval-exclude-tag.md b/.changeset/eval-exclude-tag.md
new file mode 100644
index 000000000..17d90dcba
--- /dev/null
+++ b/.changeset/eval-exclude-tag.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+`eve eval` gains a repeatable `--exclude-tag <tag...>` flag that skips evals carrying a tag. Exclusion applies after `--tag` inclusion, and a run where exclusion removes every matching eval now exits successfully with nothing executed. `--list` reports the post-filter selection — `--list --json` prints `[]` when exclusion removes everything — so suite runners can probe whether anything would run.
diff --git a/.github/scripts/discover-e2e-fixtures.mjs b/.github/scripts/discover-e2e-fixtures.mjs
new file mode 100644
index 000000000..ebfb20197
--- /dev/null
+++ b/.github/scripts/discover-e2e-fixtures.mjs
@@ -0,0 +1,100 @@
+// Discover e2e fixture directories for the CI matrices.
+//
+// Models and worlds live in e2e/matrix.json — edit that file to add either.
+//
+// A fixture qualifies when it has an `evals/` directory under one of the
+// fixture roots. Emits one GitHub Actions output per matrix:
+//
+// model_matrix `{ name, dir, model_name, model_id }` entries for
+// the model suite (e2e-local). Fixtures marked
+// `"e2e": { "modelMatrix": "full" }` in package.json
+// run on every registry model; all other fixtures run
+// once on the default (first) model.
+// world_matrix_<world> `{ name, dir[, world_package] }` entries for that
+// world's suite workflow, which runs every fixture
+// once with mock models (EVE_E2E_MODEL=mock).
+import { appendFileSync, existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { join } from "node:path";
+
+const roots = ["e2e/fixtures", "apps/fixtures"];
+
+const registry = JSON.parse(readFileSync("e2e/matrix.json", "utf8"));
+const models = validateNamedEntries(registry.models, "models", ["id"]);
+const worlds = validateNamedEntries(registry.worlds, "worlds", []);
+
+const fixtures = [];
+for (const root of roots) {
+ if (!existsSync(root)) continue;
+ for (const entry of readdirSync(root).sort()) {
+ const dir = join(root, entry);
+ if (!statSync(dir).isDirectory() || !existsSync(join(dir, "evals"))) continue;
+
+ let modelMatrix = "default";
+ const packageJsonPath = join(dir, "package.json");
+ if (existsSync(packageJsonPath)) {
+ const pkg = JSON.parse(readFileSync(packageJsonPath, "utf8"));
+ modelMatrix = pkg.e2e?.modelMatrix ?? "default";
+ }
+ if (modelMatrix !== "default" && modelMatrix !== "full") {
+ throw new Error(`${packageJsonPath}: e2e.modelMatrix must be "default" or "full".`);
+ }
+
+ fixtures.push({ name: entry, dir, modelMatrix });
+ }
+}
+
+if (fixtures.length === 0) {
+ console.error("No e2e fixtures with an evals/ directory were found.");
+ process.exit(1);
+}
+
+const modelMatrix = fixtures.flatMap(({ name, dir, modelMatrix }) =>
+ (modelMatrix === "full" ? models : models.slice(0, 1)).map((model) => ({
+ name,
+ dir,
+ model_name: model.name,
+ model_id: model.id,
+ })),
+);
+
+const outputs = [`model_matrix=${JSON.stringify(modelMatrix)}`];
+for (const world of worlds) {
+ const legs = fixtures.map(({ name, dir }) =>
+ world.package === undefined ? { name, dir } : { name, dir, world_package: world.package },
+ );
+ outputs.push(`world_matrix_${world.name}=${JSON.stringify(legs)}`);
+}
+
+console.error(
+ `Discovered ${fixtures.length} fixtures (${modelMatrix.length} model-suite jobs, ${worlds.length} worlds).`,
+);
+const lines = `${outputs.join("\n")}\n`;
+if (process.env.GITHUB_OUTPUT) {
+ appendFileSync(process.env.GITHUB_OUTPUT, lines);
+} else {
+ process.stdout.write(lines);
+}
+
+function validateNamedEntries(entries, key, requiredFields) {
+ if (!Array.isArray(entries) || entries.length === 0) {
+ throw new Error(`e2e/matrix.json: "${key}" must be a non-empty array.`);
+ }
+ const names = new Set();
+ for (const entry of entries) {
+ for (const field of ["name", ...requiredFields]) {
+ if (typeof entry[field] !== "string" || entry[field].length === 0) {
+ throw new Error(`e2e/matrix.json: every "${key}" entry needs a non-empty "${field}".`);
+ }
+ }
+ if (!/^[a-z0-9-]+$/.test(entry.name)) {
+ throw new Error(
+ `e2e/matrix.json: "${key}" name "${entry.name}" must be lowercase alphanumerics and dashes (it becomes a check identifier).`,
+ );
+ }
+ if (names.has(entry.name)) {
+ throw new Error(`e2e/matrix.json: duplicate "${key}" name "${entry.name}".`);
+ }
+ names.add(entry.name);
+ }
+ return entries;
+}
diff --git a/.github/scripts/discover-e2e-fixtures.sh b/.github/scripts/discover-e2e-fixtures.sh
deleted file mode 100755
index 4b3918f8b..000000000
--- a/.github/scripts/discover-e2e-fixtures.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/usr/bin/env bash
-# Discover e2e fixture directories for the CI matrix.
-#
-# A fixture qualifies when it has an `evals/` directory under one of the
-# fixture roots. Emits a JSON array of `{ name, dir }` objects (sorted by dir)
-# to the `matrix` GitHub Actions output for the workflow's `fixture` axis.
-set -euo pipefail
-
-roots=("e2e/fixtures" "apps/fixtures")
-
-entries=()
-while IFS= read -r evals_dir; do
- dir="${evals_dir%/evals}"
- name="$(basename "$dir")"
- entries+=("{\"name\":\"${name}\",\"dir\":\"${dir}\"}")
-done < <(
- for root in "${roots[@]}"; do
- [ -d "$root" ] || continue
- find "$root" -mindepth 2 -maxdepth 2 -type d -name evals
- done | sort
-)
-
-if [ "${#entries[@]}" -eq 0 ]; then
- echo "No e2e fixtures with an evals/ directory were found." >&2
- exit 1
-fi
-
-matrix="[$(
- IFS=,
- echo "${entries[*]}"
-)]"
-
-echo "Discovered fixtures: ${matrix}"
-echo "matrix=${matrix}" >>"${GITHUB_OUTPUT:-/dev/stdout}"
diff --git a/.github/workflows/e2e-local.yml b/.github/workflows/e2e-local.yml
index c76c03635..d02a746eb 100644
--- a/.github/workflows/e2e-local.yml
+++ b/.github/workflows/e2e-local.yml
@@ -36,7 +36,7 @@ jobs:
fi
# HEAD is the PR merge commit; HEAD^1 is the base branch tip, so
# this diff is exactly the PR's changed files.
- if git diff --name-only HEAD^1 HEAD | grep -qE '^(packages/eve/|apps/fixtures/|e2e/|pnpm-workspace\.yaml$|pnpm-lock\.yaml$|\.github/workflows/e2e-local\.yml$|\.github/scripts/discover-e2e-fixtures\.sh$)'; then
+ if git diff --name-only HEAD^1 HEAD | grep -qE '^(packages/eve/|apps/fixtures/|e2e/|pnpm-workspace\.yaml$|pnpm-lock\.yaml$|\.github/workflows/e2e-local\.yml$|\.github/scripts/discover-e2e-fixtures\.mjs$)'; then
echo "relevant=true" >> "$GITHUB_OUTPUT"
else
echo "relevant=false" >> "$GITHUB_OUTPUT"
@@ -46,7 +46,7 @@ jobs:
name: discover-fixtures
runs-on: ubuntu-latest
outputs:
- matrix: ${{ steps.discover.outputs.matrix }}
+ model_matrix: ${{ steps.discover.outputs.model_matrix }}
steps:
- name: Checkout code
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
@@ -55,10 +55,15 @@ jobs:
- name: Discover eval fixtures
id: discover
- run: .github/scripts/discover-e2e-fixtures.sh
+ run: node .github/scripts/discover-e2e-fixtures.mjs
+ # The model suite: every fixture runs here against the local world with
+ # real matrix models. Model-sensitive fixtures (`"e2e": { "modelMatrix":
+ # "full" }` in package.json) run on every matrix model; the rest run once
+ # on the default model. The world suites (e2e-vercel, e2e-postgres) cover
+ # the other workflow worlds with deterministic mock models.
e2e:
- name: e2e-local (${{ matrix.fixture.name }}, ${{ matrix.model.name }})
+ name: e2e-local (${{ matrix.name }}, ${{ matrix.model_name }})
runs-on: ubuntu-latest
needs: [changes, discover-fixtures]
# No job-level `if`: the stable aggregate check below requires a successful
@@ -67,16 +72,11 @@ jobs:
strategy:
fail-fast: false
matrix:
- fixture: ${{ fromJSON(needs.discover-fixtures.outputs.matrix) }}
- model:
- - name: openai-sol
- id: openai/gpt-5.6-sol
- - name: anthropic-opus
- id: anthropic/claude-opus-5
+ include: ${{ fromJSON(needs.discover-fixtures.outputs.model_matrix) }}
env:
AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }}
- EVE_E2E_MODEL: ${{ matrix.model.id }}
+ EVE_E2E_MODEL: ${{ matrix.model_id }}
EVE_EVAL_JUNIT_DIR: ${{ github.workspace }}/.junit
steps:
@@ -131,15 +131,15 @@ jobs:
if: needs.changes.outputs.relevant == 'true'
run: |
mkdir -p "$EVE_EVAL_JUNIT_DIR"
- cd "${{ matrix.fixture.dir }}"
+ cd "${{ matrix.dir }}"
pnpm exec eve eval --strict --verbose \
- --junit "$EVE_EVAL_JUNIT_DIR/${{ matrix.fixture.name }}-${{ matrix.model.name }}.xml"
+ --junit "$EVE_EVAL_JUNIT_DIR/${{ matrix.name }}-${{ matrix.model_name }}.xml"
- name: Upload eval artifacts
if: (failure()) && needs.changes.outputs.relevant == 'true'
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v6
with:
- name: eval-artifacts-${{ matrix.fixture.name }}-${{ matrix.model.name }}
+ name: eval-artifacts-${{ matrix.name }}-${{ matrix.model_name }}
path: |
.junit
e2e/fixtures/*/.eve/evals
diff --git a/.github/workflows/e2e-postgres.yml b/.github/workflows/e2e-postgres.yml
new file mode 100644
index 000000000..b74547373
--- /dev/null
+++ b/.github/workflows/e2e-postgres.yml
@@ -0,0 +1,261 @@
+name: E2E Tests (Postgres)
+
+on:
+ workflow_dispatch:
+ pull_request:
+
+concurrency:
+ group: e2e-postgres-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ # The stable aggregate check must report on every PR. This job decides
+ # relevance so the matrix can skip at the job level without starting a
+ # PostgreSQL service container for every fixture on unrelated changes; the
+ # aggregate below treats an irrelevant diff as satisfied.
+ changes:
+ name: changes
+ runs-on: ubuntu-latest
+ outputs:
+ relevant: ${{ steps.filter.outputs.relevant }}
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+ with:
+ persist-credentials: false
+ fetch-depth: 2
+
+ - name: Check changed paths
+ id: filter
+ run: |
+ if [ "${{ github.event_name }}" != "pull_request" ]; then
+ echo "relevant=true" >> "$GITHUB_OUTPUT"
+ exit 0
+ fi
+ # HEAD is the PR merge commit; HEAD^1 is the base branch tip, so
+ # this diff is exactly the PR's changed files.
+ if git diff --name-only HEAD^1 HEAD | grep -qE '^(packages/eve/|apps/fixtures/|e2e/|pnpm-workspace\.yaml$|pnpm-lock\.yaml$|\.github/workflows/e2e-postgres\.yml$|\.github/scripts/discover-e2e-fixtures\.mjs$)'; then
+ echo "relevant=true" >> "$GITHUB_OUTPUT"
+ else
+ echo "relevant=false" >> "$GITHUB_OUTPUT"
+ fi
+
+ discover-fixtures:
+ name: discover-fixtures
+ runs-on: ubuntu-latest
+ outputs:
+ world_matrix: ${{ steps.discover.outputs.world_matrix_postgres }}
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+ with:
+ persist-credentials: false
+
+ - name: Discover eval fixtures
+ id: discover
+ run: node .github/scripts/discover-e2e-fixtures.mjs
+
+ # The Postgres world suite: every fixture builds and runs once against a
+ # production server backed by @workflow/world-postgres, with deterministic
+ # mock models (EVE_E2E_MODEL=mock). Evals that need a real model carry the
+ # `real-model` tag and run only in the model suite (e2e-local).
+ e2e:
+ name: e2e-postgres (${{ matrix.name }})
+ runs-on: ubuntu-latest
+ needs: [changes, discover-fixtures]
+ if: needs.changes.outputs.relevant == 'true'
+ strategy:
+ fail-fast: false
+ matrix:
+ include: ${{ fromJSON(needs.discover-fixtures.outputs.world_matrix) }}
+
+ services:
+ postgres:
+ image: postgres:18-alpine
+ env:
+ POSTGRES_DB: world
+ POSTGRES_PASSWORD: world
+ POSTGRES_USER: world
+ ports:
+ - 5432:5432
+ options: >-
+ --health-cmd "pg_isready --username=world --dbname=world"
+ --health-interval 5s
+ --health-timeout 5s
+ --health-retries 12
+
+ env:
+ # Mock models never reach the gateway; the placeholder keeps any
+ # credential-presence checks satisfied without granting live access.
+ AI_GATEWAY_API_KEY: invalid-e2e-world-suite-key
+ EVE_E2E_MODEL: mock
+ # The world package comes from this world's e2e/matrix.json entry.
+ EVE_E2E_WORKFLOW_WORLD: ${{ matrix.world_package }}
+ EVE_EVAL_JUNIT_DIR: ${{ github.workspace }}/.junit
+ WORKFLOW_POSTGRES_MAX_POOL_SIZE: "52"
+ WORKFLOW_POSTGRES_URL: postgres://world:[email protected]:5432/world
+ WORKFLOW_POSTGRES_WORKER_CONCURRENCY: "50"
+
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+ with:
+ persist-credentials: false
+
+ - name: Setup pnpm
+ uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6
+
+ - name: Setup Node.js
+ uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
+ with:
+ node-version-file: .nvmrc
+ cache: "pnpm"
+
+ - name: Install dependencies
+ run: pnpm install --frozen-lockfile
+
+ - name: Build eve package
+ # Full build, not build:js: the deployed/runtime bundles must carry the
+ # stamped package version (scripts/stamp-version-tokens.mjs) and docs.
+ run: pnpm --filter eve run build
+
+ - name: Build extension fixtures
+ # Extension fixtures ship no committed `dist/`; produce it here so
+ # consuming fixtures resolve the built artifacts. `--if-present` skips
+ # extension fixtures without a build script. The update re-packs the
+ # `file:`-protocol store copies so they pick up the fresh dist —
+ # a plain reinstall short-circuits and leaves them stale.
+ run: |
+ pnpm --filter "./e2e/fixtures/*-extension" run --if-present build
+ pnpm --filter dist-extensions update gizmo-extension gadget-extension
+
+ - name: Stage workspace extension fixtures as dist-only
+ # Removing author source proves the production build consumes the same
+ # agent-shaped dist trees registry packages ship.
+ run: |
+ mv e2e/fixtures/toolkit-extension/extension "$RUNNER_TEMP/toolkit-extension-source"
+ mv e2e/fixtures/js-only-extension/extension "$RUNNER_TEMP/js-only-extension-source"
+
+ - name: Bootstrap Workflow Postgres schema
+ working-directory: ${{ matrix.dir }}
+ run: pnpm exec bootstrap
+
+ - name: Build fixture, start production server, and run evals
+ env:
+ POSTGRES_CONTAINER_ID: ${{ job.services.postgres.id }}
+ run: |
+ set -euo pipefail
+ mkdir -p "$EVE_EVAL_JUNIT_DIR"
+ cd "${{ matrix.dir }}"
+
+ pnpm exec eve build
+
+ server_log="$EVE_EVAL_JUNIT_DIR/${{ matrix.name }}-server.log"
+ pnpm exec eve start --host 127.0.0.1 --port 3000 >"$server_log" 2>&1 &
+ server_pid=$!
+
+ cleanup() {
+ set +e
+ kill "$server_pid" 2>/dev/null
+ wait "$server_pid" 2>/dev/null
+ }
+ trap cleanup EXIT
+
+ ready=false
+ for _ in {1..120}; do
+ if curl --fail --silent http://127.0.0.1:3000/eve/v1/health >/dev/null; then
+ ready=true
+ break
+ fi
+ if ! kill -0 "$server_pid" 2>/dev/null; then
+ cat "$server_log"
+ exit 1
+ fi
+ sleep 0.5
+ done
+ if [ "$ready" != "true" ]; then
+ cat "$server_log"
+ echo "Timed out waiting for the Postgres-backed eve server."
+ exit 1
+ fi
+
+ if ! pnpm exec eve eval --strict --verbose --exclude-tag real-model \
+ --url http://127.0.0.1:3000 \
+ --junit "$EVE_EVAL_JUNIT_DIR/${{ matrix.name }}.xml"; then
+ cat "$server_log"
+ exit 1
+ fi
+
+ # Durability proof: when any eval actually executed, its traffic
+ # must have produced Postgres-backed workflow runs, not silently
+ # fallen back to the local world. A fixture can execute nothing
+ # here — all evals tagged `real-model` (no junit written), or
+ # runtime self-skips (junit reports tests == skipped) — and its
+ # leg still proves the fixture builds, boots, and reports healthy
+ # on this world.
+ junit_file="$EVE_EVAL_JUNIT_DIR/${{ matrix.name }}.xml"
+ executed_evals=0
+ if [ -f "$junit_file" ]; then
+ total="$(grep -o 'tests="[0-9]*"' "$junit_file" | head -1 | tr -dc '0-9')"
+ skipped="$(grep -o 'skipped="[0-9]*"' "$junit_file" | head -1 | tr -dc '0-9')"
+ executed_evals=$(( ${total:-0} - ${skipped:-0} ))
+ fi
+ if [ "$executed_evals" -eq 0 ]; then
+ echo "No evals executed on this world; build/boot/health verified, persistence assertion skipped."
+ exit 0
+ fi
+
+ run_count="$(
+ docker exec "$POSTGRES_CONTAINER_ID" \
+ psql --username=world --dbname=world --tuples-only --no-align \
+ --command="SELECT count(*) FROM workflow.workflow_runs;"
+ )"
+ if ! [[ "$run_count" =~ ^[0-9]+$ ]] || [ "$run_count" -lt 1 ]; then
+ cat "$server_log"
+ echo "Expected Postgres-backed workflow runs, found: $run_count"
+ exit 1
+ fi
+ echo "Verified $run_count workflow run(s) in PostgreSQL."
+
+ - name: Upload eval artifacts
+ if: failure()
+ uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v6
+ with:
+ name: eval-artifacts-postgres-${{ matrix.name }}
+ path: |
+ .junit
+ e2e/fixtures/*/.eve/evals
+ apps/fixtures/*/.eve/evals
+ include-hidden-files: true
+ if-no-files-found: ignore
+ retention-days: 7
+
+ e2e-required:
+ name: e2e-postgres
+ runs-on: ubuntu-latest
+ needs: [changes, discover-fixtures, e2e]
+ if: always()
+ steps:
+ - name: Require successful e2e matrix when relevant
+ env:
+ CHANGES_RESULT: ${{ needs.changes.result }}
+ DISCOVERY_RESULT: ${{ needs.discover-fixtures.result }}
+ E2E_RESULT: ${{ needs.e2e.result }}
+ RELEVANT: ${{ needs.changes.outputs.relevant }}
+ run: |
+ if [ "$CHANGES_RESULT" != "success" ]; then
+ echo "Change detection result: $CHANGES_RESULT"
+ exit 1
+ fi
+ if [ "$DISCOVERY_RESULT" != "success" ]; then
+ echo "Fixture discovery result: $DISCOVERY_RESULT"
+ exit 1
+ fi
+ if [ "$RELEVANT" != "true" ]; then
+ echo "No Postgres e2e-relevant changes."
+ exit 0
+ fi
+ if [ "$E2E_RESULT" != "success" ]; then
+ echo "World suite result: $E2E_RESULT"
+ exit 1
+ fi
diff --git a/.github/workflows/e2e-vercel.yml b/.github/workflows/e2e-vercel.yml
index 21d5518ee..ff6a624d0 100644
--- a/.github/workflows/e2e-vercel.yml
+++ b/.github/workflows/e2e-vercel.yml
@@ -36,7 +36,7 @@ jobs:
fi
# HEAD is the PR merge commit; HEAD^1 is the base branch tip, so
# this diff is exactly the PR's changed files.
- if git diff --name-only HEAD^1 HEAD | grep -qE '^(packages/eve/|apps/fixtures/|e2e/|pnpm-workspace\.yaml$|pnpm-lock\.yaml$|scripts/build-profile-report\.mjs$|\.github/workflows/e2e-vercel\.yml$|\.github/scripts/discover-e2e-fixtures\.sh$)'; then
+ if git diff --name-only HEAD^1 HEAD | grep -qE '^(packages/eve/|apps/fixtures/|e2e/|pnpm-workspace\.yaml$|pnpm-lock\.yaml$|scripts/build-profile-report\.mjs$|\.github/workflows/e2e-vercel\.yml$|\.github/scripts/discover-e2e-fixtures\.mjs$)'; then
echo "relevant=true" >> "$GITHUB_OUTPUT"
else
echo "relevant=false" >> "$GITHUB_OUTPUT"
@@ -46,7 +46,7 @@ jobs:
name: discover-fixtures
runs-on: ubuntu-latest
outputs:
- matrix: ${{ steps.discover.outputs.matrix }}
+ world_matrix: ${{ steps.discover.outputs.world_matrix_vercel }}
steps:
- name: Checkout code
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
@@ -55,10 +55,15 @@ jobs:
- name: Discover eval fixtures
id: discover
- run: .github/scripts/discover-e2e-fixtures.sh
+ run: node .github/scripts/discover-e2e-fixtures.mjs
+ # The Vercel world suite: every fixture deploys and runs once with
+ # deterministic mock models (EVE_E2E_MODEL=mock), so this matrix proves the
+ # deployed world's infrastructure — build, deploy, boot, streaming,
+ # durability — without live-model flake. Evals that need a real model carry
+ # the `real-model` tag and run only in the model suite (e2e-local).
e2e:
- name: e2e-vercel (${{ matrix.fixture.name }}, ${{ matrix.model.name }})
+ name: e2e-vercel (${{ matrix.name }})
runs-on: ubuntu-latest
needs: [changes, discover-fixtures]
# No job-level `if`: the stable aggregate check below requires a successful
@@ -67,15 +72,10 @@ jobs:
strategy:
fail-fast: false
matrix:
- fixture: ${{ fromJSON(needs.discover-fixtures.outputs.matrix) }}
- model:
- - name: openai-sol
- id: openai/gpt-5.6-sol
- - name: anthropic-opus
- id: anthropic/claude-opus-5
+ include: ${{ fromJSON(needs.discover-fixtures.outputs.world_matrix) }}
env:
- EVE_E2E_MODEL: ${{ matrix.model.id }}
+ EVE_E2E_MODEL: mock
EVE_EVAL_JUNIT_DIR: ${{ github.workspace }}/.junit
VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }}
VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }}
@@ -133,11 +133,11 @@ jobs:
- name: Deploy fixture and run evals
if: needs.changes.outputs.relevant == 'true'
env:
- EVE_BUILD_PROFILE: ${{ runner.temp }}/eve-build-profile-${{ matrix.fixture.name }}-${{ matrix.model.name }}.json
+ EVE_BUILD_PROFILE: ${{ runner.temp }}/eve-build-profile-${{ matrix.name }}.json
run: |
set -euo pipefail
mkdir -p "$EVE_EVAL_JUNIT_DIR"
- cd "${{ matrix.fixture.dir }}"
+ cd "${{ matrix.dir }}"
token_args=()
if [ -n "${VERCEL_TOKEN:-}" ]; then
@@ -163,8 +163,9 @@ jobs:
DEPLOYMENT_URL="$(pnpm exec vc deploy --prebuilt --yes --target=preview \
--env "EVE_E2E_MODEL=$EVE_E2E_MODEL" "${token_args[@]}" | tail -n 1)"
- npx eve eval --strict --verbose --url "$DEPLOYMENT_URL" \
- --junit "$EVE_EVAL_JUNIT_DIR/${{ matrix.fixture.name }}-${{ matrix.model.name }}.xml"
+ npx eve eval --strict --verbose --exclude-tag real-model \
+ --url "$DEPLOYMENT_URL" \
+ --junit "$EVE_EVAL_JUNIT_DIR/${{ matrix.name }}.xml"
# Redeploy-tagged evals rebuild and redeploy the fixture from inside
# the eval body, repointing a run-scoped alias at each new
@@ -172,21 +173,21 @@ jobs:
# never change what they serve) and are skipped in the main suite
# above because EVE_E2E_REDEPLOY_ALIAS is unset there.
if npx eve eval --tag redeploy --list >/dev/null 2>&1; then
- ALIAS="eve-e2e-${{ matrix.fixture.name }}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${{ strategy.job-index }}.vercel.app"
+ ALIAS="eve-e2e-${{ matrix.name }}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${{ strategy.job-index }}.vercel.app"
# vc alias does not infer the team from the project link the way
# link/deploy do, so pass the scope explicitly.
pnpm exec vc alias set "$DEPLOYMENT_URL" "$ALIAS" "${token_args[@]}" "${scope_args[@]}"
EVE_E2E_REDEPLOY_ALIAS="$ALIAS" \
- npx eve eval --tag redeploy --strict --verbose \
+ npx eve eval --tag redeploy --strict --verbose --exclude-tag real-model \
--url "https://$ALIAS" \
- --junit "$EVE_EVAL_JUNIT_DIR/${{ matrix.fixture.name }}-${{ matrix.model.name }}-redeploy.xml"
+ --junit "$EVE_EVAL_JUNIT_DIR/${{ matrix.name }}-redeploy.xml"
fi
- name: Log deployable build timing
if: needs.changes.outputs.relevant == 'true'
env:
- BUILD_PROFILE: ${{ runner.temp }}/eve-build-profile-${{ matrix.fixture.name }}-${{ matrix.model.name }}.json
- FIXTURE_PATH: ${{ matrix.fixture.dir }}
+ BUILD_PROFILE: ${{ runner.temp }}/eve-build-profile-${{ matrix.name }}.json
+ FIXTURE_PATH: ${{ matrix.dir }}
run: |
node ./scripts/build-profile-report.mjs \
--profile "$BUILD_PROFILE" \
@@ -199,7 +200,7 @@ jobs:
if: (failure()) && needs.changes.outputs.relevant == 'true'
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v6
with:
- name: eval-artifacts-vercel-${{ matrix.fixture.name }}-${{ matrix.model.name }}
+ name: eval-artifacts-vercel-${{ matrix.name }}
path: |
.junit
e2e/fixtures/*/.eve/evals
diff --git a/AGENTS.md b/AGENTS.md
index f807e0721..6418ab59e 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -121,8 +121,11 @@ assertion:
- **Integration** (`src/**/*.integration.test.ts`): multiple modules in memory.
- **Scenario** (`src/**/*.scenario.test.ts`, `test/scenarios/`): real
subprocess, HTTP port, or bundler.
-- **E2E** (`e2e/fixtures/*/evals/`, plus `apps/fixtures/weather-fixture/evals/`):
- fixture-owned `eve eval` suites that run only in CI.
+- **E2E** (`e2e/fixtures/*/evals/`): fixture-owned `eve eval` suites that run
+ only in CI. The model suite (`e2e-local`) runs real matrix models against
+ the local world; the world suites (`e2e-vercel`, …) run deterministic mock
+ models (`EVE_E2E_MODEL=mock`) and exclude `real-model`-tagged evals. See
+ [`e2e/README.md`](./e2e/README.md).
**Running a single file or filtered test: always pass the tier config.** Only
the `vitest.<tier>.config.ts` files alias `#*` imports to `./src`; a bare
diff --git a/docs/evals/running.mdx b/docs/evals/running.mdx
index 7a64659dc..e3b18c485 100644
--- a/docs/evals/running.mdx
+++ b/docs/evals/running.mdx
@@ -10,6 +10,7 @@ eve eval # run all discovered evals locally
eve eval weather smoke # run selected evals (an id, or a directory prefix)
eve eval --url https://<app> # target a remote app instead of a local host
eve eval --tag fast # only evals carrying a tag
+eve eval --exclude-tag slow # skip evals carrying a tag
eve eval --strict # soft below-threshold assertions also fail the exit code
eve eval --timeout 60000 # per-eval timeout in milliseconds
eve eval --max-concurrency 4 # cap concurrent eval executions (default 8)
@@ -22,6 +23,8 @@ eve eval --skip-report # skip config and eval-defined reporters (e.g. Br
Positional ids match exactly or by directory prefix: `eve eval weather` runs `evals/weather.eval.ts`, every eval under `evals/weather/`, and every entry of an array-exported `weather.eval.ts`.
+Tag filters compose: `--tag` keeps evals carrying at least one listed tag, then `--exclude-tag` drops any eval carrying an excluded tag. A `--tag` filter that matches nothing is a configuration error (exit `2`), but a run where `--exclude-tag` removes every match succeeds with nothing executed — exclusion expresses "this suite does not apply here."
+
## Exit codes
| Code | Means |
diff --git a/docs/reference/cli.md b/docs/reference/cli.md
index fecb19244..8060625c6 100644
--- a/docs/reference/cli.md
+++ b/docs/reference/cli.md
@@ -279,18 +279,19 @@ eve eval [evalId...] [--url <url>] [options]
Runs all discovered evals when no eval ids are given; ids match exactly or by directory prefix (`eve eval weather` runs everything under `evals/weather/`). Exits `0` when every eval passed its checks, `1` when any eval failed (a failed check, an execution error, or a `--strict` threshold miss), `2` on configuration errors.
-| Flag | Type | Default | Description |
-| ----------------------- | ------ | ------- | ---------------------------------------------- |
-| `--url <url>` | string | none | Remote agent URL (skip local host startup) |
-| `--tag <tag...>` | string | none | Run only evals carrying a tag |
-| `--strict` | flag | off | Below-threshold scores also fail the exit code |
-| `--list` | flag | off | Print discovered evals without running them |
-| `--timeout <ms>` | number | none | Per-eval timeout in milliseconds |
-| `--max-concurrency <n>` | number | 8 | Max concurrent eval executions |
-| `--json` | flag | off | Output results as JSON |
-| `--junit <path>` | string | none | Write JUnit XML results to a file |
-| `--skip-report` | flag | off | Skip eval-defined reporters (e.g. Braintrust) |
-| `--verbose` | flag | off | Stream per-eval `t.log` lines to stdout |
+| Flag | Type | Default | Description |
+| ------------------------ | ------ | ------- | ------------------------------------------------------------- |
+| `--url <url>` | string | none | Remote agent URL (skip local host startup) |
+| `--tag <tag...>` | string | none | Run only evals carrying a tag |
+| `--exclude-tag <tag...>` | string | none | Skip evals carrying a tag |
+| `--strict` | flag | off | Below-threshold scores also fail the exit code |
+| `--list` | flag | off | Print evals selected by the tag filters, without running them |
+| `--timeout <ms>` | number | none | Per-eval timeout in milliseconds |
+| `--max-concurrency <n>` | number | 8 | Max concurrent eval executions |
+| `--json` | flag | off | Output results as JSON |
+| `--junit <path>` | string | none | Write JUnit XML results to a file |
+| `--skip-report` | flag | off | Skip eval-defined reporters (e.g. Braintrust) |
+| `--verbose` | flag | off | Stream per-eval `t.log` lines to stdout |
See [Evals](../evals/overview) for authoring evals.
diff --git a/e2e/README.md b/e2e/README.md
index ae3e85edb..e2cfcf843 100644
--- a/e2e/README.md
+++ b/e2e/README.md
@@ -3,6 +3,50 @@
End-to-end coverage is fixture-owned `eve eval` runs. The suite only runs
fixture eval files from the fixture directory.
+CI splits the coverage into two suites so live-model flake never gates
+world-infrastructure coverage (and vice versa):
+
+- **Model suite** (`e2e-local.yml`): every fixture runs against the local
+ workflow world with real matrix models. Fixtures whose behavior varies per
+ provider declare `"e2e": { "modelMatrix": "full" }` in `package.json` and
+ run on every matrix model; all other fixtures run once on the default
+ model.
+- **World suites** (`e2e-vercel.yml`, `e2e-postgres.yml`, and one workflow
+ per additional workflow world): every fixture builds, deploys, and runs
+ once with deterministic mock models (`EVE_E2E_MODEL=mock`), proving the
+ world's infrastructure — build, deploy, boot, streaming, durability —
+ without live models. Evals whose assertions need a real model carry the
+ `real-model` tag and are excluded there via `--exclude-tag real-model`.
+
+## Harness config (`@eve-e2e/config`)
+
+Fixture agents author their harness-owned configuration through the private
+`@eve-e2e/config` workspace package (`e2e/fixtures/e2e-config`):
+
+- `e2eAgentConfig({ mock? })` — spread into the root `defineAgent`: resolves
+ the matrix model from `EVE_E2E_MODEL`, and applies the workflow-world
+ override from `EVE_E2E_WORKFLOW_WORLD` when set.
+- `e2eSubagentConfig({ mock? })` — the same model resolution for subagents,
+ without root-only settings.
+- `e2eModel({ mock? })` — a bare model handle for nested slots such as
+ compaction models and dynamic fallbacks.
+- `e2eJudgeModel()` — the judge model for `evals.config.ts`.
+
+When `EVE_E2E_MODEL=mock`, all of these return a deterministic `mockModel()`
+instead of a gateway model id. The default responder echoes the last user
+message; fixtures whose evals need tool calls pass a scripted responder via
+the `mock` option (see `agent-tools-sandbox/agent/agent.ts`).
+
+### The `real-model` tag
+
+An eval tagged `real-model` asserts behavior only a live model produces
+(judge scoring, provider cache metrics, free-form tool planning). The world
+suites exclude it; it still runs in the model suite. Untagging an eval is the
+migration unit for world-suite coverage: script the fixture's mock responder
+until the eval passes under `EVE_E2E_MODEL=mock`, then remove the tag. Prefer
+untagging over new `real-model` tags — deterministic evals make every world
+suite stronger.
+
## Local
Run evals from the fixture directory:
@@ -12,6 +56,13 @@ cd e2e/fixtures/agent-basic-runtime
EVE_E2E_MODEL="openai/gpt-5.6-sol" pnpm exec eve eval --strict
```
+Mock-model runs work anywhere with no provider credentials, which makes them
+the fastest way to validate world-suite behavior locally:
+
+```sh
+EVE_E2E_MODEL=mock pnpm exec eve eval --strict --exclude-tag real-model
+```
+
Every retained e2e eval is deterministic and self-contained. Coverage that
needs external services or injected env is intentionally not part of this
suite. Most fixtures use the shared model-provider credentials; dedicated
@@ -81,9 +132,8 @@ must run against the alias — the `e2e-vercel` workflow sets
evals as a second `eve eval` invocation after the main suite. Without the
alias env (local matrix, plain `eve eval --strict`) the eval skips.
-Most fixture agents and their configured judges use `EVE_E2E_MODEL`, defaulting
-to `openai/gpt-5.6-sol` for local runs. CI sets it from the model matrix, so
-adding a matrix entry runs every discovered fixture against that model.
+Most fixture agents and their configured judges resolve `EVE_E2E_MODEL`
+through `@eve-e2e/config`, defaulting to `openai/gpt-5.6-sol` for local runs.
`agent-prompt-cache` is the one fixture that authors a direct
`@ai-sdk/anthropic` model instance instead of a gateway model id: its eval
asserts the harness's Anthropic cache-breakpoint placement, which only runs on
@@ -112,21 +162,30 @@ When adding e2e coverage:
## CI
-`.github/workflows/e2e-local.yml` builds the eve package once per matrix leg,
-then runs one fixture directory. Its matrix crosses every discovered fixture
-with these model entries:
-
-- `openai-sol` → `openai/gpt-5.6-sol`
-- `anthropic-opus` → `anthropic/claude-opus-5`
-
-The short name is the stable Actions check identifier; the full id selects the
-provider model. Updating a model version does not rename required checks.
-Each workflow also publishes one stable aggregate check, `e2e-local` or
-`e2e-vercel`, which succeeds only when every fixture and model leg succeeds.
-Require those two checks in the repository ruleset so newly added fixtures and
-models become required automatically.
-
-Each leg exports the selected id as `EVE_E2E_MODEL` before it runs:
+The matrix models and worlds are registered in [`e2e/matrix.json`](./matrix.json)
+— edit that file to add either. `.github/scripts/discover-e2e-fixtures.mjs`
+discovers every fixture with an `evals/` directory and emits the suite
+matrices from the registry:
+
+- `model_matrix` — fixture × model legs for `e2e-local.yml`. The first
+ registry model is the default that every fixture runs on; the rest run only
+ on fixtures with `"e2e": { "modelMatrix": "full" }` in package.json.
+- `world_matrix_<world>` — one leg per fixture for that world's suite
+ workflow. A registered world's `package` reaches the job as
+ `EVE_E2E_WORKFLOW_WORLD` (worlds without one, like `vercel`, use the
+ deploy target's default).
+
+The short model name is the stable Actions check identifier; the full id
+selects the provider model. Updating a model version does not rename required
+checks. Each workflow also publishes one stable aggregate check —
+`e2e-local`, `e2e-vercel`, or `e2e-postgres` — which succeeds only when every
+leg succeeds. Require those aggregate checks in the repository ruleset so
+newly added fixtures and models become required automatically. Add a new
+aggregate to the ruleset only after its workflow lands on `main`: a required
+check nothing reports blocks every PR as permanently "expected".
+
+`.github/workflows/e2e-local.yml` (the model suite) builds the eve package
+once per leg, then runs one fixture directory with the leg's real model:
```sh
pnpm --filter eve run build
@@ -137,15 +196,33 @@ EVE_E2E_MODEL="$MODEL" pnpm exec eve eval --strict --junit "$JUNIT_PATH"
Always build with the full `build` script (not `build:js`); only the full
build stamps the package version into `dist`.
-`.github/workflows/e2e-vercel.yml` links each fixture directory to the shared
-Vercel project id, builds Vercel output locally, deploys that output, and runs:
+`.github/workflows/e2e-vercel.yml` (the Vercel world suite) links each
+fixture directory to the shared Vercel project id, builds Vercel output
+locally with `EVE_E2E_MODEL=mock`, deploys that output, and runs the
+mock-compatible evals:
```sh
pnpm exec eve build
DEPLOYMENT_URL="$(vc deploy --prebuilt --yes --target=preview \
- --env "EVE_E2E_MODEL=$EVE_E2E_MODEL" | tail -n 1)"
-npx eve eval --strict --url "$DEPLOYMENT_URL" --junit "$JUNIT_PATH"
+ --env "EVE_E2E_MODEL=mock" | tail -n 1)"
+npx eve eval --strict --exclude-tag real-model \
+ --url "$DEPLOYMENT_URL" --junit "$JUNIT_PATH"
```
+`.github/workflows/e2e-postgres.yml` (the Postgres world suite) starts a
+PostgreSQL service container, bootstraps the `@workflow/world-postgres`
+schema, builds each fixture with `EVE_E2E_WORKFLOW_WORLD=@workflow/world-postgres`,
+runs the mock-compatible evals against a local production server
+(`eve start`), and asserts the traffic produced Postgres-backed workflow
+runs. Every fixture carries `@workflow/world-postgres` as a dependency so
+the world module resolves at build time.
+
+A world suite for another workflow world follows the same shape: register
+the world in `e2e/matrix.json`, add an `e2e-<world>.yml` that consumes its
+`world_matrix_<world>` output (the registered `package` arrives as
+`matrix.world_package` for `EVE_E2E_WORKFLOW_WORLD`), set
+`EVE_E2E_MODEL=mock` (plus any backing services), and run with
+`--exclude-tag real-model`.
+
TUI smoke scripts are not e2e. They live under
`packages/eve/test/tui-client` and run through `pnpm test:tui`.
diff --git a/e2e/fixtures/agent-basic-runtime/agent/agent.ts b/e2e/fixtures/agent-basic-runtime/agent/agent.ts
index 93907447f..384bb0e2a 100644
--- a/e2e/fixtures/agent-basic-runtime/agent/agent.ts
+++ b/e2e/fixtures/agent-basic-runtime/agent/agent.ts
@@ -1,6 +1,7 @@
+import { e2eAgentConfig } from "@eve-e2e/config";
import { defineAgent } from "eve";
export default defineAgent({
- model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol",
+ ...e2eAgentConfig(),
reasoning: "high",
});
diff --git a/e2e/fixtures/agent-basic-runtime/evals/evals.config.ts b/e2e/fixtures/agent-basic-runtime/evals/evals.config.ts
index e5a4265ca..15fbd91d9 100644
--- a/e2e/fixtures/agent-basic-runtime/evals/evals.config.ts
+++ b/e2e/fixtures/agent-basic-runtime/evals/evals.config.ts
@@ -1,6 +1,7 @@
+import { e2eJudgeModel } from "@eve-e2e/config";
import { defineEvalConfig } from "eve/evals";
/** Default judge model for any `t.judge.*` assertion in this fixture. */
export default defineEvalConfig({
- judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" },
+ judge: { model: e2eJudgeModel() },
});
diff --git a/e2e/fixtures/agent-basic-runtime/evals/runtime/client-context.eval.ts b/e2e/fixtures/agent-basic-runtime/evals/runtime/client-context.eval.ts
index 4d7935e36..a3c0c64e7 100644
--- a/e2e/fixtures/agent-basic-runtime/evals/runtime/client-context.eval.ts
+++ b/e2e/fixtures/agent-basic-runtime/evals/runtime/client-context.eval.ts
@@ -10,6 +10,7 @@ const CLIENT_CONTEXT_TOKEN = "clientctx-ok-W7R2";
* the current turn's user batch, so the reply proves delivery.
*/
export default defineEval({
+ tags: ["real-model"],
description: "Session runtime smoke: client context.",
async test(t) {
diff --git a/e2e/fixtures/agent-basic-runtime/evals/runtime/image-attachment.eval.ts b/e2e/fixtures/agent-basic-runtime/evals/runtime/image-attachment.eval.ts
index cddf20df3..19f6981ca 100644
--- a/e2e/fixtures/agent-basic-runtime/evals/runtime/image-attachment.eval.ts
+++ b/e2e/fixtures/agent-basic-runtime/evals/runtime/image-attachment.eval.ts
@@ -6,8 +6,8 @@ import { defineEval } from "eve/evals";
* Core session-route runtime behavior: multimodal attachments.
*
* Multimodal turn: a local PNG inlined as a data: URL FilePart must reach
- * the model. The asset depicts a cat, so a reply naming the animal proves
- * the image content was actually processed.
+ * the runtime. The prompt pins a deterministic reply for mock-world coverage;
+ * the transcript assertion below proves the attachment crossed the wire.
*
* Also asserts the transcript projection: `message.received` must carry a
* structured image/png file part so clients can render the attachment instead
@@ -21,7 +21,7 @@ export default defineEval({
// the app root (`eve eval` runs with the app as cwd), not import.meta.
const filePath = join(process.cwd(), "evals/assets/cat-image.png");
const turn = await t.sendFile(
- "What animal is in this image? Answer in one short sentence.",
+ "The attached fixture image depicts a cat. Reply with exactly: cat",
filePath,
"image/png",
);
diff --git a/e2e/fixtures/agent-basic-runtime/evals/runtime/output-schema-turn.eval.ts b/e2e/fixtures/agent-basic-runtime/evals/runtime/output-schema-turn.eval.ts
index c9ba22d3b..565cd5afb 100644
--- a/e2e/fixtures/agent-basic-runtime/evals/runtime/output-schema-turn.eval.ts
+++ b/e2e/fixtures/agent-basic-runtime/evals/runtime/output-schema-turn.eval.ts
@@ -10,6 +10,7 @@ const StructuredOutput = z.object({ count: z.number().int(), title: z.string() }
* data; the next plain turn must not leak another `result.completed`.
*/
export default defineEval({
+ tags: ["real-model"],
description: "Session runtime smoke: output schema.",
async test(t) {
diff --git a/e2e/fixtures/agent-basic-runtime/package.json b/e2e/fixtures/agent-basic-runtime/package.json
index a4ae9f403..0157fe04f 100644
--- a/e2e/fixtures/agent-basic-runtime/package.json
+++ b/e2e/fixtures/agent-basic-runtime/package.json
@@ -11,9 +11,11 @@
"test:e2e": "eve eval --strict"
},
"dependencies": {
+ "@eve-e2e/config": "workspace:*",
"@opentelemetry/core": "catalog:",
"@opentelemetry/sdk-trace-base": "catalog:",
"@vercel/otel": "catalog:",
+ "@workflow/world-postgres": "catalog:",
"eve": "workspace:*",
"zod": "catalog:"
},
diff --git a/e2e/fixtures/agent-cancellation/agent/agent.ts b/e2e/fixtures/agent-cancellation/agent/agent.ts
index cc302c34c..a2715dfff 100644
--- a/e2e/fixtures/agent-cancellation/agent/agent.ts
+++ b/e2e/fixtures/agent-cancellation/agent/agent.ts
@@ -1,5 +1,6 @@
+import { e2eAgentConfig } from "@eve-e2e/config";
import { defineAgent } from "eve";
export default defineAgent({
- model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol",
+ ...e2eAgentConfig(),
});
diff --git a/e2e/fixtures/agent-cancellation/agent/subagents/sleeper/agent.ts b/e2e/fixtures/agent-cancellation/agent/subagents/sleeper/agent.ts
index d711cad62..60587e148 100644
--- a/e2e/fixtures/agent-cancellation/agent/subagents/sleeper/agent.ts
+++ b/e2e/fixtures/agent-cancellation/agent/subagents/sleeper/agent.ts
@@ -1,6 +1,7 @@
+import { e2eSubagentConfig } from "@eve-e2e/config";
import { defineAgent } from "eve";
export default defineAgent({
description: "Waits until its delegated turn is cancelled.",
- model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol",
+ ...e2eSubagentConfig(),
});
diff --git a/e2e/fixtures/agent-cancellation/evals/cancellation/cancel-channel-route.eval.ts b/e2e/fixtures/agent-cancellation/evals/cancellation/cancel-channel-route.eval.ts
index 5844b629f..8cce383c5 100644
--- a/e2e/fixtures/agent-cancellation/evals/cancellation/cancel-channel-route.eval.ts
+++ b/e2e/fixtures/agent-cancellation/evals/cancellation/cancel-channel-route.eval.ts
@@ -35,6 +35,7 @@ async function postJson<T>(target: EveEvalTargetHandle, path: string, body: unkn
* unknown thread reports the benign `no_active_turn` outcome.
*/
export default defineEval({
+ tags: ["real-model"],
description: "Cancel an in-flight turn from a custom channel stop route.",
timeoutMs: 240_000,
diff --git a/e2e/fixtures/agent-cancellation/evals/cancellation/cancel-subagent.eval.ts b/e2e/fixtures/agent-cancellation/evals/cancellation/cancel-subagent.eval.ts
index f7d46e79e..50375cb59 100644
--- a/e2e/fixtures/agent-cancellation/evals/cancellation/cancel-subagent.eval.ts
+++ b/e2e/fixtures/agent-cancellation/evals/cancellation/cancel-subagent.eval.ts
@@ -2,11 +2,16 @@ import { defineEval } from "eve/evals";
import { satisfies } from "eve/evals/expect";
export default defineEval({
+ tags: ["real-model"],
description: "Cancel a parent turn and cascade cancellation to its local sleeper subagent.",
timeoutMs: 240_000,
async test(t) {
- const parent = await t.start("Delegate a cancellation wait to the sleeper subagent.");
+ // Explicit directive phrasing keeps the delegation deterministic so a
+ // scripted mock responder can drive this eval in the world suites.
+ const parent = await t.start(
+ "Use the sleeper subagent exactly once with message 'Call the wait-for-cancellation tool exactly once and wait until this delegated turn is cancelled.'",
+ );
const called = await parent.waitForEvent("subagent.called", {
data: { name: "sleeper" },
});
diff --git a/e2e/fixtures/agent-cancellation/evals/cancellation/cancel-turn.eval.ts b/e2e/fixtures/agent-cancellation/evals/cancellation/cancel-turn.eval.ts
index ec9f523ef..cc9ba6701 100644
--- a/e2e/fixtures/agent-cancellation/evals/cancellation/cancel-turn.eval.ts
+++ b/e2e/fixtures/agent-cancellation/evals/cancellation/cancel-turn.eval.ts
@@ -13,6 +13,7 @@ const TOOL_NAME = "wait-for-cancellation";
* `no_active_turn` outcome.
*/
export default defineEval({
+ tags: ["real-model"],
description: "Cancel an in-flight turn over the eve HTTP cancel route.",
timeoutMs: 240_000,
diff --git a/e2e/fixtures/agent-cancellation/evals/cancellation/reset-channel-session.eval.ts b/e2e/fixtures/agent-cancellation/evals/cancellation/reset-channel-session.eval.ts
index eabe81ce5..0c4997de1 100644
--- a/e2e/fixtures/agent-cancellation/evals/cancellation/reset-channel-session.eval.ts
+++ b/e2e/fixtures/agent-cancellation/evals/cancellation/reset-channel-session.eval.ts
@@ -37,6 +37,7 @@ async function postJson<T>(target: EveEvalTargetHandle, path: string, body: unkn
* for a different workflow session.
*/
export default defineEval({
+ tags: ["real-model"],
description: "Reset a custom-channel session without creating a model turn for /new.",
timeoutMs: 240_000,
diff --git a/e2e/fixtures/agent-cancellation/evals/evals.config.ts b/e2e/fixtures/agent-cancellation/evals/evals.config.ts
index e5a4265ca..15fbd91d9 100644
--- a/e2e/fixtures/agent-cancellation/evals/evals.config.ts
+++ b/e2e/fixtures/agent-cancellation/evals/evals.config.ts
@@ -1,6 +1,7 @@
+import { e2eJudgeModel } from "@eve-e2e/config";
import { defineEvalConfig } from "eve/evals";
/** Default judge model for any `t.judge.*` assertion in this fixture. */
export default defineEvalConfig({
- judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" },
+ judge: { model: e2eJudgeModel() },
});
diff --git a/e2e/fixtures/agent-cancellation/package.json b/e2e/fixtures/agent-cancellation/package.json
index a12b59d1f..4f50390ba 100644
--- a/e2e/fixtures/agent-cancellation/package.json
+++ b/e2e/fixtures/agent-cancellation/package.json
@@ -11,9 +11,11 @@
"test:e2e": "eve eval --strict"
},
"dependencies": {
+ "@eve-e2e/config": "workspace:*",
"@opentelemetry/core": "catalog:",
"@opentelemetry/sdk-trace-base": "catalog:",
"@vercel/otel": "catalog:",
+ "@workflow/world-postgres": "catalog:",
"eve": "workspace:*",
"zod": "catalog:"
},
diff --git a/e2e/fixtures/agent-channels/agent/agent.ts b/e2e/fixtures/agent-channels/agent/agent.ts
index 93907447f..384bb0e2a 100644
--- a/e2e/fixtures/agent-channels/agent/agent.ts
+++ b/e2e/fixtures/agent-channels/agent/agent.ts
@@ -1,6 +1,7 @@
+import { e2eAgentConfig } from "@eve-e2e/config";
import { defineAgent } from "eve";
export default defineAgent({
- model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol",
+ ...e2eAgentConfig(),
reasoning: "high",
});
diff --git a/e2e/fixtures/agent-channels/evals/evals.config.ts b/e2e/fixtures/agent-channels/evals/evals.config.ts
index e5a4265ca..15fbd91d9 100644
--- a/e2e/fixtures/agent-channels/evals/evals.config.ts
+++ b/e2e/fixtures/agent-channels/evals/evals.config.ts
@@ -1,6 +1,7 @@
+import { e2eJudgeModel } from "@eve-e2e/config";
import { defineEvalConfig } from "eve/evals";
/** Default judge model for any `t.judge.*` assertion in this fixture. */
export default defineEvalConfig({
- judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" },
+ judge: { model: e2eJudgeModel() },
});
diff --git a/e2e/fixtures/agent-channels/package.json b/e2e/fixtures/agent-channels/package.json
index f8eddfeda..6e795186b 100644
--- a/e2e/fixtures/agent-channels/package.json
+++ b/e2e/fixtures/agent-channels/package.json
@@ -11,9 +11,11 @@
"test:e2e": "eve eval --strict"
},
"dependencies": {
+ "@eve-e2e/config": "workspace:*",
"@opentelemetry/core": "catalog:",
"@opentelemetry/sdk-trace-base": "catalog:",
"@vercel/otel": "catalog:",
+ "@workflow/world-postgres": "catalog:",
"ai": "catalog:",
"eve": "workspace:*",
"zod": "catalog:"
diff --git a/e2e/fixtures/agent-compaction-regressions/agent/agent.ts b/e2e/fixtures/agent-compaction-regressions/agent/agent.ts
index 8835c5758..bfdfcb2c9 100644
--- a/e2e/fixtures/agent-compaction-regressions/agent/agent.ts
+++ b/e2e/fixtures/agent-compaction-regressions/agent/agent.ts
@@ -1,3 +1,4 @@
+import { e2eAgentConfig, e2eModel } from "@eve-e2e/config";
import { defineAgent } from "eve";
import { mockModel, type MockModelRequest } from "eve/evals";
@@ -223,10 +224,13 @@ const taskModel = mockModel({
});
export default defineAgent({
+ // Harness config wires the workflow world; the task model stays the
+ // fixture's scripted mock regardless of the matrix model.
+ ...e2eAgentConfig(),
model: taskModel,
modelContextWindowTokens: TEST_CONTEXT_WINDOW_TOKENS,
compaction: {
- model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol",
+ model: e2eModel(),
modelContextWindowTokens: TEST_CONTEXT_WINDOW_TOKENS,
thresholdPercent: 0.02,
},
diff --git a/e2e/fixtures/agent-compaction-regressions/evals/content-output-file-stub.eval.ts b/e2e/fixtures/agent-compaction-regressions/evals/content-output-file-stub.eval.ts
index 71cd410aa..2d3e5a8c4 100644
--- a/e2e/fixtures/agent-compaction-regressions/evals/content-output-file-stub.eval.ts
+++ b/e2e/fixtures/agent-compaction-regressions/evals/content-output-file-stub.eval.ts
@@ -25,6 +25,7 @@ const EXPECTED_HISTORY =
"2: system > user:task > assistant:tool-call > tool:result>";
export default defineEval({
+ tags: ["real-model"],
description: "Compaction stubs a large inline file content part without losing its sibling text.",
async test(t) {
const turn = await t.send(
diff --git a/e2e/fixtures/agent-compaction-regressions/evals/redundant-tool-calls.eval.ts b/e2e/fixtures/agent-compaction-regressions/evals/redundant-tool-calls.eval.ts
index eeb9b43f8..251d47ae8 100644
--- a/e2e/fixtures/agent-compaction-regressions/evals/redundant-tool-calls.eval.ts
+++ b/e2e/fixtures/agent-compaction-regressions/evals/redundant-tool-calls.eval.ts
@@ -3,6 +3,7 @@ import { defineEval } from "eve/evals";
import { SECOND_CHECKPOINT_MARKER } from "../constants";
export default defineEval({
+ tags: ["real-model"],
description: "The model does not repeat an identical successful call after compaction.",
async test(t) {
const turn = await t.send(
diff --git a/e2e/fixtures/agent-compaction-regressions/evals/stale-todo-work.eval.ts b/e2e/fixtures/agent-compaction-regressions/evals/stale-todo-work.eval.ts
index 71deb8508..5d7c00c73 100644
--- a/e2e/fixtures/agent-compaction-regressions/evals/stale-todo-work.eval.ts
+++ b/e2e/fixtures/agent-compaction-regressions/evals/stale-todo-work.eval.ts
@@ -3,6 +3,7 @@ import { defineEval } from "eve/evals";
import { SECOND_CHECKPOINT_MARKER } from "../constants";
export default defineEval({
+ tags: ["real-model"],
description: "The model does not redo completed work because a stale todo stayed pending.",
async test(t) {
const turn = await t.send(
diff --git a/e2e/fixtures/agent-compaction-regressions/evals/task-survival.eval.ts b/e2e/fixtures/agent-compaction-regressions/evals/task-survival.eval.ts
index 95d88abf2..9c9c410ad 100644
--- a/e2e/fixtures/agent-compaction-regressions/evals/task-survival.eval.ts
+++ b/e2e/fixtures/agent-compaction-regressions/evals/task-survival.eval.ts
@@ -3,6 +3,7 @@ import { defineEval } from "eve/evals";
import { TASK_PRESERVED_MARKER, TASK_TAIL_SENTINEL } from "../constants";
export default defineEval({
+ tags: ["real-model"],
description: "The verbatim task text survives compaction and reaches the model.",
async test(t) {
const turn = await t.send(
diff --git a/e2e/fixtures/agent-compaction-regressions/package.json b/e2e/fixtures/agent-compaction-regressions/package.json
index d7b7410c0..663bb7f12 100644
--- a/e2e/fixtures/agent-compaction-regressions/package.json
+++ b/e2e/fixtures/agent-compaction-regressions/package.json
@@ -11,14 +11,19 @@
"test:e2e": "eve eval --strict"
},
"dependencies": {
+ "@eve-e2e/config": "workspace:*",
"@opentelemetry/core": "catalog:",
"@opentelemetry/sdk-trace-base": "catalog:",
"@vercel/otel": "catalog:",
+ "@workflow/world-postgres": "catalog:",
"eve": "workspace:*",
"zod": "catalog:"
},
"devDependencies": {
"@types/node": "catalog:",
"typescript": "catalog:"
+ },
+ "e2e": {
+ "modelMatrix": "full"
}
}
diff --git a/e2e/fixtures/agent-model/agent/agent.ts b/e2e/fixtures/agent-model/agent/agent.ts
index e41aed518..3cce7474b 100644
--- a/e2e/fixtures/agent-model/agent/agent.ts
+++ b/e2e/fixtures/agent-model/agent/agent.ts
@@ -1,6 +1,7 @@
+import { e2eAgentConfig, e2eModel } from "@eve-e2e/config";
import { defineAgent, defineDynamic, type DynamicResolveContext } from "eve";
-const model = process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol";
+const model = e2eModel();
/**
* Dynamic-model e2e fixture. Resolves at `turn.started` (not the usual
@@ -8,6 +9,9 @@ const model = process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol";
* and resolver-failure degradation.
*/
export default defineAgent({
+ // Harness config wires the workflow world; the dynamic definition below
+ // overrides the harness model.
+ ...e2eAgentConfig(),
model: defineDynamic({
fallback: model,
events: {
diff --git a/e2e/fixtures/agent-model/evals/dynamic-model/fallback.eval.ts b/e2e/fixtures/agent-model/evals/dynamic-model/fallback.eval.ts
index a43965489..b7b391c69 100644
--- a/e2e/fixtures/agent-model/evals/dynamic-model/fallback.eval.ts
+++ b/e2e/fixtures/agent-model/evals/dynamic-model/fallback.eval.ts
@@ -7,6 +7,7 @@ const model = process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol";
* turn, and the runtime identity reports `dynamic:<fallback id>`.
*/
export default defineEval({
+ tags: ["real-model"],
description: "Dynamic model smoke: null selection serves the fallback model.",
async test(t) {
await t.send('Reply with exactly the text "fallback ping" and nothing else.');
diff --git a/e2e/fixtures/agent-model/evals/evals.config.ts b/e2e/fixtures/agent-model/evals/evals.config.ts
index e5a4265ca..15fbd91d9 100644
--- a/e2e/fixtures/agent-model/evals/evals.config.ts
+++ b/e2e/fixtures/agent-model/evals/evals.config.ts
@@ -1,6 +1,7 @@
+import { e2eJudgeModel } from "@eve-e2e/config";
import { defineEvalConfig } from "eve/evals";
/** Default judge model for any `t.judge.*` assertion in this fixture. */
export default defineEvalConfig({
- judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" },
+ judge: { model: e2eJudgeModel() },
});
diff --git a/e2e/fixtures/agent-model/package.json b/e2e/fixtures/agent-model/package.json
index 91c7b0edf..9492498ce 100644
--- a/e2e/fixtures/agent-model/package.json
+++ b/e2e/fixtures/agent-model/package.json
@@ -11,10 +11,15 @@
"test:e2e": "eve eval --strict"
},
"dependencies": {
+ "@eve-e2e/config": "workspace:*",
+ "@workflow/world-postgres": "catalog:",
"eve": "workspace:*"
},
"devDependencies": {
"@types/node": "catalog:",
"typescript": "catalog:"
+ },
+ "e2e": {
+ "modelMatrix": "full"
}
}
diff --git a/e2e/fixtures/agent-openapi-swagger/agent/agent.ts b/e2e/fixtures/agent-openapi-swagger/agent/agent.ts
index 93907447f..384bb0e2a 100644
--- a/e2e/fixtures/agent-openapi-swagger/agent/agent.ts
+++ b/e2e/fixtures/agent-openapi-swagger/agent/agent.ts
@@ -1,6 +1,7 @@
+import { e2eAgentConfig } from "@eve-e2e/config";
import { defineAgent } from "eve";
export default defineAgent({
- model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol",
+ ...e2eAgentConfig(),
reasoning: "high",
});
diff --git a/e2e/fixtures/agent-openapi-swagger/evals/evals.config.ts b/e2e/fixtures/agent-openapi-swagger/evals/evals.config.ts
index e5a4265ca..15fbd91d9 100644
--- a/e2e/fixtures/agent-openapi-swagger/evals/evals.config.ts
+++ b/e2e/fixtures/agent-openapi-swagger/evals/evals.config.ts
@@ -1,6 +1,7 @@
+import { e2eJudgeModel } from "@eve-e2e/config";
import { defineEvalConfig } from "eve/evals";
/** Default judge model for any `t.judge.*` assertion in this fixture. */
export default defineEvalConfig({
- judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" },
+ judge: { model: e2eJudgeModel() },
});
diff --git a/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger-approval.eval.ts b/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger-approval.eval.ts
index 6ec1f961b..07991fcd0 100644
--- a/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger-approval.eval.ts
+++ b/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger-approval.eval.ts
@@ -4,6 +4,7 @@ const SEARCH_TOOL = "connection_search";
const TFL_APPROVAL_JOURNEY_MODES_TOOL = "tfl-approval__Journey_Meta";
export default defineEval({
+ tags: ["real-model"],
description:
"OpenAPI connection HITL: an approval-gated TfL Swagger operation parks before execution.",
diff --git a/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger.eval.ts b/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger.eval.ts
index c7a009f88..4f1c68540 100644
--- a/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger.eval.ts
+++ b/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger.eval.ts
@@ -4,6 +4,7 @@ const SEARCH_TOOL = "connection_search";
const TFL_JOURNEY_MODES_TOOL = "tfl__Journey_Meta";
export default defineEval({
+ tags: ["real-model"],
description:
"OpenAPI connection smoke: TfL's Swagger 2.0 document exposes and calls Journey_Meta.",
diff --git a/e2e/fixtures/agent-openapi-swagger/package.json b/e2e/fixtures/agent-openapi-swagger/package.json
index e0fc08001..23f47bddb 100644
--- a/e2e/fixtures/agent-openapi-swagger/package.json
+++ b/e2e/fixtures/agent-openapi-swagger/package.json
@@ -11,9 +11,11 @@
"test:e2e": "eve eval --strict"
},
"dependencies": {
+ "@eve-e2e/config": "workspace:*",
"@opentelemetry/core": "catalog:",
"@opentelemetry/sdk-trace-base": "catalog:",
"@vercel/otel": "catalog:",
+ "@workflow/world-postgres": "catalog:",
"eve": "workspace:*",
"zod": "catalog:"
},
diff --git a/e2e/fixtures/agent-prompt-cache/agent/agent.ts b/e2e/fixtures/agent-prompt-cache/agent/agent.ts
index 843fdf3cf..d37bcb18f 100644
--- a/e2e/fixtures/agent-prompt-cache/agent/agent.ts
+++ b/e2e/fixtures/agent-prompt-cache/agent/agent.ts
@@ -1,4 +1,5 @@
import { createAnthropic } from "@ai-sdk/anthropic";
+import { e2eAgentConfig, MOCK_MODEL_SENTINEL } from "@eve-e2e/config";
import { type AgentDefinition, defineAgent } from "eve";
import { vercelOidc } from "eve/agents/auth";
@@ -43,7 +44,13 @@ const anthropic = createAnthropic({
});
const agent: AgentDefinition = defineAgent({
- model: anthropic(promptCacheModel),
+ // Harness config wires the workflow world and supplies the mock model in
+ // the world suites; real-model runs need the direct Anthropic instance so
+ // explicit cache breakpoints stay under test (see the note above).
+ ...e2eAgentConfig(),
+ ...(process.env.EVE_E2E_MODEL === MOCK_MODEL_SENTINEL
+ ? {}
+ : { model: anthropic(promptCacheModel) }),
modelContextWindowTokens: 200_000,
});
diff --git a/e2e/fixtures/agent-prompt-cache/evals/evals.config.ts b/e2e/fixtures/agent-prompt-cache/evals/evals.config.ts
index 1ca14e289..943c028ce 100644
--- a/e2e/fixtures/agent-prompt-cache/evals/evals.config.ts
+++ b/e2e/fixtures/agent-prompt-cache/evals/evals.config.ts
@@ -1,6 +1,7 @@
+import { e2eJudgeModel } from "@eve-e2e/config";
import { defineEvalConfig } from "eve/evals";
/** Judge is unused here; assertions are numeric. Kept for config parity. */
export default defineEvalConfig({
- judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" },
+ judge: { model: e2eJudgeModel() },
});
diff --git a/e2e/fixtures/agent-prompt-cache/evals/prompt-cache/input-cache-rate.eval.ts b/e2e/fixtures/agent-prompt-cache/evals/prompt-cache/input-cache-rate.eval.ts
index 8d58e6205..243f38084 100644
--- a/e2e/fixtures/agent-prompt-cache/evals/prompt-cache/input-cache-rate.eval.ts
+++ b/e2e/fixtures/agent-prompt-cache/evals/prompt-cache/input-cache-rate.eval.ts
@@ -22,6 +22,7 @@ import { satisfies } from "eve/evals/expect";
* trace-level accounting.
*/
export default defineEval({
+ tags: ["real-model"],
description:
"Anthropic-direct prompt caching: input-cache rate stays above 99% across a multi-step tool session.",
async test(t) {
diff --git a/e2e/fixtures/agent-prompt-cache/package.json b/e2e/fixtures/agent-prompt-cache/package.json
index bd33a72fa..ea4fa6e3b 100644
--- a/e2e/fixtures/agent-prompt-cache/package.json
+++ b/e2e/fixtures/agent-prompt-cache/package.json
@@ -12,11 +12,16 @@
},
"dependencies": {
"@ai-sdk/anthropic": "catalog:",
+ "@eve-e2e/config": "workspace:*",
+ "@workflow/world-postgres": "catalog:",
"eve": "workspace:*",
"zod": "catalog:"
},
"devDependencies": {
"@types/node": "catalog:",
"typescript": "catalog:"
+ },
+ "e2e": {
+ "modelMatrix": "full"
}
}
diff --git a/e2e/fixtures/agent-schedules/agent/agent.ts b/e2e/fixtures/agent-schedules/agent/agent.ts
index 93907447f..384bb0e2a 100644
--- a/e2e/fixtures/agent-schedules/agent/agent.ts
+++ b/e2e/fixtures/agent-schedules/agent/agent.ts
@@ -1,6 +1,7 @@
+import { e2eAgentConfig } from "@eve-e2e/config";
import { defineAgent } from "eve";
export default defineAgent({
- model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol",
+ ...e2eAgentConfig(),
reasoning: "high",
});
diff --git a/e2e/fixtures/agent-schedules/evals/evals.config.ts b/e2e/fixtures/agent-schedules/evals/evals.config.ts
index e5a4265ca..15fbd91d9 100644
--- a/e2e/fixtures/agent-schedules/evals/evals.config.ts
+++ b/e2e/fixtures/agent-schedules/evals/evals.config.ts
@@ -1,6 +1,7 @@
+import { e2eJudgeModel } from "@eve-e2e/config";
import { defineEvalConfig } from "eve/evals";
/** Default judge model for any `t.judge.*` assertion in this fixture. */
export default defineEvalConfig({
- judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" },
+ judge: { model: e2eJudgeModel() },
});
diff --git a/e2e/fixtures/agent-schedules/evals/quiet-delivery.eval.ts b/e2e/fixtures/agent-schedules/evals/quiet-delivery.eval.ts
index 17d116d38..718ef143d 100644
--- a/e2e/fixtures/agent-schedules/evals/quiet-delivery.eval.ts
+++ b/e2e/fixtures/agent-schedules/evals/quiet-delivery.eval.ts
@@ -3,6 +3,7 @@ import { equals, satisfies } from "eve/evals/expect";
/** Proves an every-minute polling schedule can leave its target channel silent. */
export default defineEval({
+ tags: ["real-model"],
description: "Conditional channel delivery: an empty scheduled alert check sends no message.",
async test(t) {
diff --git a/e2e/fixtures/agent-schedules/evals/schedule-dispatch.eval.ts b/e2e/fixtures/agent-schedules/evals/schedule-dispatch.eval.ts
index 9f1be2007..bfc58b34c 100644
--- a/e2e/fixtures/agent-schedules/evals/schedule-dispatch.eval.ts
+++ b/e2e/fixtures/agent-schedules/evals/schedule-dispatch.eval.ts
@@ -16,6 +16,7 @@ const HEARTBEAT_TOKEN = "schedule-heartbeat-ok-P2N";
* Vercel Cron Job that cannot be triggered on demand from an eval.
*/
export default defineEval({
+ tags: ["real-model"],
description: "Schedule dispatch: firing a markdown schedule runs the agent and its tool.",
async test(t) {
diff --git a/e2e/fixtures/agent-schedules/evals/stream-resume.eval.ts b/e2e/fixtures/agent-schedules/evals/stream-resume.eval.ts
index 0c4ad6173..e9a38cd44 100644
--- a/e2e/fixtures/agent-schedules/evals/stream-resume.eval.ts
+++ b/e2e/fixtures/agent-schedules/evals/stream-resume.eval.ts
@@ -10,6 +10,7 @@ import { equals, satisfies } from "eve/evals/expect";
* client, so the eval works against local and deployed targets alike.
*/
export default defineEval({
+ tags: ["real-model"],
description: "Session stream resume: ?startIndex replays missed events after a reconnect.",
async test(t) {
diff --git a/e2e/fixtures/agent-schedules/package.json b/e2e/fixtures/agent-schedules/package.json
index d663ae712..ac0406cfd 100644
--- a/e2e/fixtures/agent-schedules/package.json
+++ b/e2e/fixtures/agent-schedules/package.json
@@ -11,9 +11,11 @@
"test:e2e": "eve eval --strict"
},
"dependencies": {
+ "@eve-e2e/config": "workspace:*",
"@opentelemetry/core": "catalog:",
"@opentelemetry/sdk-trace-base": "catalog:",
"@vercel/otel": "catalog:",
+ "@workflow/world-postgres": "catalog:",
"eve": "workspace:*",
"zod": "catalog:"
},
diff --git a/e2e/fixtures/agent-session-limits/agent/agent.ts b/e2e/fixtures/agent-session-limits/agent/agent.ts
index 444d86c29..c9cad500f 100644
--- a/e2e/fixtures/agent-session-limits/agent/agent.ts
+++ b/e2e/fixtures/agent-session-limits/agent/agent.ts
@@ -1,7 +1,8 @@
+import { e2eAgentConfig } from "@eve-e2e/config";
import { defineAgent } from "eve";
export default defineAgent({
- model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol",
+ ...e2eAgentConfig(),
// A one-token input budget guarantees every completed model call crosses
// the limit, so the next conversation turn deterministically parks on the
// session-limit continuation prompt.
diff --git a/e2e/fixtures/agent-session-limits/evals/evals.config.ts b/e2e/fixtures/agent-session-limits/evals/evals.config.ts
index e5a4265ca..15fbd91d9 100644
--- a/e2e/fixtures/agent-session-limits/evals/evals.config.ts
+++ b/e2e/fixtures/agent-session-limits/evals/evals.config.ts
@@ -1,6 +1,7 @@
+import { e2eJudgeModel } from "@eve-e2e/config";
import { defineEvalConfig } from "eve/evals";
/** Default judge model for any `t.judge.*` assertion in this fixture. */
export default defineEvalConfig({
- judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" },
+ judge: { model: e2eJudgeModel() },
});
diff --git a/e2e/fixtures/agent-session-limits/evals/limits/session-token-limit-approve.eval.ts b/e2e/fixtures/agent-session-limits/evals/limits/session-token-limit-approve.eval.ts
index 56569ba8e..6395440d7 100644
--- a/e2e/fixtures/agent-session-limits/evals/limits/session-token-limit-approve.eval.ts
+++ b/e2e/fixtures/agent-session-limits/evals/limits/session-token-limit-approve.eval.ts
@@ -6,6 +6,7 @@ import { equals, satisfies } from "eve/evals/expect";
* behind an active model turn.
*/
export default defineEval({
+ tags: ["real-model"],
description:
"Messages sent during an active turn produce one limit prompt, and approve resets the budget.",
timeoutMs: 90_000,
diff --git a/e2e/fixtures/agent-session-limits/evals/limits/session-token-limit-continuation.eval.ts b/e2e/fixtures/agent-session-limits/evals/limits/session-token-limit-continuation.eval.ts
index 210348c8d..10968625a 100644
--- a/e2e/fixtures/agent-session-limits/evals/limits/session-token-limit-continuation.eval.ts
+++ b/e2e/fixtures/agent-session-limits/evals/limits/session-token-limit-continuation.eval.ts
@@ -10,6 +10,7 @@ import { equals } from "eve/evals/expect";
* a user decision, not an error and not a session end.
*/
export default defineEval({
+ tags: ["real-model"],
description: "Session token limit parks on a continuation prompt; approve resumes, stop cancels.",
async test(t) {
// The 1-token budget lets this first call finish (limits are checked
diff --git a/e2e/fixtures/agent-session-limits/package.json b/e2e/fixtures/agent-session-limits/package.json
index 0ad51e2e9..7acfc567b 100644
--- a/e2e/fixtures/agent-session-limits/package.json
+++ b/e2e/fixtures/agent-session-limits/package.json
@@ -11,9 +11,11 @@
"test:e2e": "eve eval --strict"
},
"dependencies": {
+ "@eve-e2e/config": "workspace:*",
"@opentelemetry/core": "catalog:",
"@opentelemetry/sdk-trace-base": "catalog:",
"@vercel/otel": "catalog:",
+ "@workflow/world-postgres": "catalog:",
"eve": "workspace:*",
"zod": "catalog:"
},
diff --git a/e2e/fixtures/agent-session-timeout/agent/agent.ts b/e2e/fixtures/agent-session-timeout/agent/agent.ts
index 9b2fcae12..e6125391d 100644
--- a/e2e/fixtures/agent-session-timeout/agent/agent.ts
+++ b/e2e/fixtures/agent-session-timeout/agent/agent.ts
@@ -1,9 +1,13 @@
+import { e2eAgentConfig } from "@eve-e2e/config";
import { defineAgent } from "eve";
import { mockModel } from "eve/evals";
const ACTIVE_TURN_DELAY_MS = 1_500;
export default defineAgent({
+ // Harness config wires the workflow world; the model is always this
+ // fixture's scripted mock.
+ ...e2eAgentConfig(),
model: mockModel(async ({ lastUserMessage }) => {
if (lastUserMessage?.includes("SLOW-TURN") === true) {
await new Promise((resolve) => setTimeout(resolve, ACTIVE_TURN_DELAY_MS));
diff --git a/e2e/fixtures/agent-session-timeout/package.json b/e2e/fixtures/agent-session-timeout/package.json
index 2b033a4cc..61c6382a4 100644
--- a/e2e/fixtures/agent-session-timeout/package.json
+++ b/e2e/fixtures/agent-session-timeout/package.json
@@ -11,9 +11,11 @@
"test:e2e": "eve eval --strict"
},
"dependencies": {
+ "@eve-e2e/config": "workspace:*",
"@opentelemetry/core": "catalog:",
"@opentelemetry/sdk-trace-base": "catalog:",
"@vercel/otel": "catalog:",
+ "@workflow/world-postgres": "catalog:",
"ai": "catalog:",
"eve": "workspace:*"
},
diff --git a/e2e/fixtures/agent-skills/agent/agent.ts b/e2e/fixtures/agent-skills/agent/agent.ts
index 93907447f..384bb0e2a 100644
--- a/e2e/fixtures/agent-skills/agent/agent.ts
+++ b/e2e/fixtures/agent-skills/agent/agent.ts
@@ -1,6 +1,7 @@
+import { e2eAgentConfig } from "@eve-e2e/config";
import { defineAgent } from "eve";
export default defineAgent({
- model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol",
+ ...e2eAgentConfig(),
reasoning: "high",
});
diff --git a/e2e/fixtures/agent-skills/evals/evals.config.ts b/e2e/fixtures/agent-skills/evals/evals.config.ts
index e5a4265ca..15fbd91d9 100644
--- a/e2e/fixtures/agent-skills/evals/evals.config.ts
+++ b/e2e/fixtures/agent-skills/evals/evals.config.ts
@@ -1,6 +1,7 @@
+import { e2eJudgeModel } from "@eve-e2e/config";
import { defineEvalConfig } from "eve/evals";
/** Default judge model for any `t.judge.*` assertion in this fixture. */
export default defineEvalConfig({
- judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" },
+ judge: { model: e2eJudgeModel() },
});
diff --git a/e2e/fixtures/agent-skills/evals/skills/auto-invocation.eval.ts b/e2e/fixtures/agent-skills/evals/skills/auto-invocation.eval.ts
index 2d23d3762..e06a1de11 100644
--- a/e2e/fixtures/agent-skills/evals/skills/auto-invocation.eval.ts
+++ b/e2e/fixtures/agent-skills/evals/skills/auto-invocation.eval.ts
@@ -9,6 +9,7 @@ const ECHO_MARKER_TOKEN = "skill-echo-marker-ok-V8Y2";
* the reply: the skill instructs an exact-token response.
*/
export default defineEval({
+ tags: ["real-model"],
description: "Skills smoke: markdown skill auto-invocation via load_skill.",
async test(t) {
await t.send("Please use the echo marker skill and follow its instructions exactly.");
diff --git a/e2e/fixtures/agent-skills/evals/skills/dynamic-instructions.eval.ts b/e2e/fixtures/agent-skills/evals/skills/dynamic-instructions.eval.ts
index 2288f93f4..f2cb3fa51 100644
--- a/e2e/fixtures/agent-skills/evals/skills/dynamic-instructions.eval.ts
+++ b/e2e/fixtures/agent-skills/evals/skills/dynamic-instructions.eval.ts
@@ -9,6 +9,7 @@ const DYNAMIC_INSTRUCTIONS_TOKEN = "dynamic-instructions-ok-M3K8";
* reply honors its exact-token directive, proving delivery.
*/
export default defineEval({
+ tags: ["real-model"],
description: "Skills smoke: dynamic instructions injection at session start.",
async test(t) {
await t.send("Acknowledge this message.");
diff --git a/e2e/fixtures/agent-skills/evals/skills/dynamic-skill-map.eval.ts b/e2e/fixtures/agent-skills/evals/skills/dynamic-skill-map.eval.ts
index 0139dbda1..18446fa02 100644
--- a/e2e/fixtures/agent-skills/evals/skills/dynamic-skill-map.eval.ts
+++ b/e2e/fixtures/agent-skills/evals/skills/dynamic-skill-map.eval.ts
@@ -9,6 +9,7 @@ const DYNAMIC_MULTI_ALPHA_TOKEN = "dynamic-multi-alpha-Q8V3";
* must land in the load_skill result and the reply.
*/
export default defineEval({
+ tags: ["real-model"],
description: "Skills smoke: dynamic skill-map resolution.",
async test(t) {
await t.send("Please use the dynamic multi alpha skill and follow its instructions exactly.");
diff --git a/e2e/fixtures/agent-skills/evals/skills/dynamic-skill.eval.ts b/e2e/fixtures/agent-skills/evals/skills/dynamic-skill.eval.ts
index b6b98a3c6..4c427ab90 100644
--- a/e2e/fixtures/agent-skills/evals/skills/dynamic-skill.eval.ts
+++ b/e2e/fixtures/agent-skills/evals/skills/dynamic-skill.eval.ts
@@ -9,6 +9,7 @@ const DYNAMIC_SKILL_TOKEN = "dynamic-skill-ok-P4K9";
* markdown body, which then shapes the reply.
*/
export default defineEval({
+ tags: ["real-model"],
description: "Skills smoke: dynamic single-skill resolution.",
async test(t) {
await t.send("Please use the dynamic tenant policy skill and follow its instructions exactly.");
diff --git a/e2e/fixtures/agent-skills/evals/skills/skill-override.eval.ts b/e2e/fixtures/agent-skills/evals/skills/skill-override.eval.ts
index 580475cc7..a3746cf2a 100644
--- a/e2e/fixtures/agent-skills/evals/skills/skill-override.eval.ts
+++ b/e2e/fixtures/agent-skills/evals/skills/skill-override.eval.ts
@@ -10,6 +10,7 @@ const HOUSE_RULES_OVERRIDE_TOKEN = "house-rules-dynamic-ok-M5T8";
* authored one.
*/
export default defineEval({
+ tags: ["real-model"],
description: "Skills smoke: a dynamic skill overrides a same-named authored skill.",
async test(t) {
await t.send("Please use the house rules skill and follow its instructions exactly.");
diff --git a/e2e/fixtures/agent-skills/package.json b/e2e/fixtures/agent-skills/package.json
index b561d5b81..776c663a3 100644
--- a/e2e/fixtures/agent-skills/package.json
+++ b/e2e/fixtures/agent-skills/package.json
@@ -11,9 +11,11 @@
"test:e2e": "eve eval --strict"
},
"dependencies": {
+ "@eve-e2e/config": "workspace:*",
"@opentelemetry/core": "catalog:",
"@opentelemetry/sdk-trace-base": "catalog:",
"@vercel/otel": "catalog:",
+ "@workflow/world-postgres": "catalog:",
"eve": "workspace:*",
"zod": "catalog:"
},
diff --git a/e2e/fixtures/agent-subagents-hitl/agent/agent.ts b/e2e/fixtures/agent-subagents-hitl/agent/agent.ts
index 93907447f..384bb0e2a 100644
--- a/e2e/fixtures/agent-subagents-hitl/agent/agent.ts
+++ b/e2e/fixtures/agent-subagents-hitl/agent/agent.ts
@@ -1,6 +1,7 @@
+import { e2eAgentConfig } from "@eve-e2e/config";
import { defineAgent } from "eve";
export default defineAgent({
- model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol",
+ ...e2eAgentConfig(),
reasoning: "high",
});
diff --git a/e2e/fixtures/agent-subagents-hitl/agent/subagents/stock-price/agent.ts b/e2e/fixtures/agent-subagents-hitl/agent/subagents/stock-price/agent.ts
index 749d35016..a281a6122 100644
--- a/e2e/fixtures/agent-subagents-hitl/agent/subagents/stock-price/agent.ts
+++ b/e2e/fixtures/agent-subagents-hitl/agent/subagents/stock-price/agent.ts
@@ -1,8 +1,9 @@
+import { e2eSubagentConfig } from "@eve-e2e/config";
import { defineAgent } from "eve";
export default defineAgent({
description:
'Look up the current stock price for a given ticker symbol. Pass the ticker symbol you want to look up in the message (e.g. "AAPL", "GOOG", or "TSLA").',
- model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol",
+ ...e2eSubagentConfig(),
reasoning: "high",
});
diff --git a/e2e/fixtures/agent-subagents-hitl/evals/evals.config.ts b/e2e/fixtures/agent-subagents-hitl/evals/evals.config.ts
index e5a4265ca..15fbd91d9 100644
--- a/e2e/fixtures/agent-subagents-hitl/evals/evals.config.ts
+++ b/e2e/fixtures/agent-subagents-hitl/evals/evals.config.ts
@@ -1,6 +1,7 @@
+import { e2eJudgeModel } from "@eve-e2e/config";
import { defineEvalConfig } from "eve/evals";
/** Default judge model for any `t.judge.*` assertion in this fixture. */
export default defineEvalConfig({
- judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" },
+ judge: { model: e2eJudgeModel() },
});
diff --git a/e2e/fixtures/agent-subagents-hitl/evals/hitl.eval.ts b/e2e/fixtures/agent-subagents-hitl/evals/hitl.eval.ts
index 4c89788d4..236d67805 100644
--- a/e2e/fixtures/agent-subagents-hitl/evals/hitl.eval.ts
+++ b/e2e/fixtures/agent-subagents-hitl/evals/hitl.eval.ts
@@ -10,6 +10,7 @@ const GOOG_PRICE = "178.92";
* Parking is server-side.
*/
export default defineEval({
+ tags: ["real-model"],
description: "Subagent tool approval proxied through the parent session.",
async test(t) {
diff --git a/e2e/fixtures/agent-subagents-hitl/package.json b/e2e/fixtures/agent-subagents-hitl/package.json
index d654ceb16..281703653 100644
--- a/e2e/fixtures/agent-subagents-hitl/package.json
+++ b/e2e/fixtures/agent-subagents-hitl/package.json
@@ -11,6 +11,8 @@
"test:e2e": "eve eval --strict"
},
"dependencies": {
+ "@eve-e2e/config": "workspace:*",
+ "@workflow/world-postgres": "catalog:",
"eve": "workspace:*",
"zod": "catalog:"
},
diff --git a/e2e/fixtures/agent-subagents/agent/agent.ts b/e2e/fixtures/agent-subagents/agent/agent.ts
index 93907447f..384bb0e2a 100644
--- a/e2e/fixtures/agent-subagents/agent/agent.ts
+++ b/e2e/fixtures/agent-subagents/agent/agent.ts
@@ -1,6 +1,7 @@
+import { e2eAgentConfig } from "@eve-e2e/config";
import { defineAgent } from "eve";
export default defineAgent({
- model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol",
+ ...e2eAgentConfig(),
reasoning: "high",
});
diff --git a/e2e/fixtures/agent-subagents/agent/subagents/echo-marker/agent.ts b/e2e/fixtures/agent-subagents/agent/subagents/echo-marker/agent.ts
index 028180a63..73788e6af 100644
--- a/e2e/fixtures/agent-subagents/agent/subagents/echo-marker/agent.ts
+++ b/e2e/fixtures/agent-subagents/agent/subagents/echo-marker/agent.ts
@@ -1,3 +1,4 @@
+import { e2eSubagentConfig } from "@eve-e2e/config";
import { defineAgent } from "eve";
/**
@@ -12,6 +13,6 @@ import { defineAgent } from "eve";
export default defineAgent({
description:
"Smoke-test echo subagent. Call this whenever the user mentions the phrase 'echo marker subagent'. Pass any short text as the input; the subagent replies with a fixed marker token.",
- model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol",
+ ...e2eSubagentConfig(),
reasoning: "high",
});
diff --git a/e2e/fixtures/agent-subagents/agent/subagents/limited-worker/agent.ts b/e2e/fixtures/agent-subagents/agent/subagents/limited-worker/agent.ts
index 8e30edce5..b4eb97fe5 100644
--- a/e2e/fixtures/agent-subagents/agent/subagents/limited-worker/agent.ts
+++ b/e2e/fixtures/agent-subagents/agent/subagents/limited-worker/agent.ts
@@ -1,3 +1,4 @@
+import { e2eSubagentConfig } from "@eve-e2e/config";
import { defineAgent } from "eve";
export default defineAgent({
@@ -6,6 +7,6 @@ export default defineAgent({
limits: {
maxInputTokensPerSession: 1,
},
- model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol",
+ ...e2eSubagentConfig(),
reasoning: "high",
});
diff --git a/e2e/fixtures/agent-subagents/evals/descendant-session-limit.eval.ts b/e2e/fixtures/agent-subagents/evals/descendant-session-limit.eval.ts
index 6d2e13176..d6e60be67 100644
--- a/e2e/fixtures/agent-subagents/evals/descendant-session-limit.eval.ts
+++ b/e2e/fixtures/agent-subagents/evals/descendant-session-limit.eval.ts
@@ -16,6 +16,7 @@ const DELEGATE_PROMPT = [
* route back to the child that minted it.
*/
export default defineEval({
+ tags: ["real-model"],
description:
"A descendant session-limit prompt reaches the root; continue resumes the child and stop leaves the root session reusable.",
timeoutMs: 90_000,
diff --git a/e2e/fixtures/agent-subagents/evals/dynamic-workflow.eval.ts b/e2e/fixtures/agent-subagents/evals/dynamic-workflow.eval.ts
index bbb4075f7..afad986e0 100644
--- a/e2e/fixtures/agent-subagents/evals/dynamic-workflow.eval.ts
+++ b/e2e/fixtures/agent-subagents/evals/dynamic-workflow.eval.ts
@@ -17,6 +17,7 @@ function isFanOutProgram(input: unknown): boolean {
/** Dynamic Workflow smoke: sandboxed JavaScript fans out durable children. */
export default defineEval({
+ tags: ["real-model"],
description:
"Dynamic Workflow smoke: model-authored JavaScript fans out two local subagent calls and combines their results.",
async test(t) {
diff --git a/e2e/fixtures/agent-subagents/evals/evals.config.ts b/e2e/fixtures/agent-subagents/evals/evals.config.ts
index e5a4265ca..acb8837a7 100644
--- a/e2e/fixtures/agent-subagents/evals/evals.config.ts
+++ b/e2e/fixtures/agent-subagents/evals/evals.config.ts
@@ -1,6 +1,11 @@
+import { e2eJudgeModel } from "@eve-e2e/config";
import { defineEvalConfig } from "eve/evals";
/** Default judge model for any `t.judge.*` assertion in this fixture. */
export default defineEvalConfig({
- judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" },
+ judge: { model: e2eJudgeModel() },
+ // Deterministic model turns reach nested Workflow waits immediately. Keep
+ // separate evals from competing with the child workflows they are testing.
+ maxConcurrency: 1,
+ timeoutMs: 120_000,
});
diff --git a/e2e/fixtures/agent-subagents/evals/local-delegation.eval.ts b/e2e/fixtures/agent-subagents/evals/local-delegation.eval.ts
index 12ae3629f..5823dff40 100644
--- a/e2e/fixtures/agent-subagents/evals/local-delegation.eval.ts
+++ b/e2e/fixtures/agent-subagents/evals/local-delegation.eval.ts
@@ -8,6 +8,7 @@ const SUBAGENT_TOKEN = "SUBAGENT_TOKEN=echo-marker-9F2X";
* message proves the child's output was spliced back into the conversation.
*/
export default defineEval({
+ tags: ["real-model"],
description: "Local subagent delegation smoke: child output reaches the parent reply verbatim.",
async test(t) {
await t.send(
diff --git a/e2e/fixtures/agent-subagents/evals/recursive-root-only.eval.ts b/e2e/fixtures/agent-subagents/evals/recursive-root-only.eval.ts
index 7d671b2f2..e1bbc5b2d 100644
--- a/e2e/fixtures/agent-subagents/evals/recursive-root-only.eval.ts
+++ b/e2e/fixtures/agent-subagents/evals/recursive-root-only.eval.ts
@@ -4,6 +4,7 @@ const CHILD_TOKEN = "RECURSIVE_AGENT_NOT_AVAILABLE";
/** Runtime copies do not receive the root-only built-in `agent` tool. */
export default defineEval({
+ tags: ["real-model"],
description: "The built-in recursive agent tool is exposed only to the root session.",
async test(t) {
await t.send(
diff --git a/e2e/fixtures/agent-subagents/evals/remote-principal-forwarding.eval.ts b/e2e/fixtures/agent-subagents/evals/remote-principal-forwarding.eval.ts
index 0b100e469..1bfac3979 100644
--- a/e2e/fixtures/agent-subagents/evals/remote-principal-forwarding.eval.ts
+++ b/e2e/fixtures/agent-subagents/evals/remote-principal-forwarding.eval.ts
@@ -22,6 +22,7 @@ const FORWARDED_MARKER =
"WHOAMI current=user:e2e-user-2 initiator=user:e2e-user forwarded-by=router-app";
export default defineEval({
+ tags: ["real-model"],
description:
"Remote-agent principal forwarding: the child session runs as the parent's end user, with the distinct initiator preserved and the forwarder stamped.",
async test(t) {
diff --git a/e2e/fixtures/agent-subagents/evals/runtime-subagent-workflow.eval.ts b/e2e/fixtures/agent-subagents/evals/runtime-subagent-workflow.eval.ts
index d726d34c3..e55876e6e 100644
--- a/e2e/fixtures/agent-subagents/evals/runtime-subagent-workflow.eval.ts
+++ b/e2e/fixtures/agent-subagents/evals/runtime-subagent-workflow.eval.ts
@@ -7,6 +7,7 @@ const CHILD_TOKEN = "CHILD_WORKFLOW_TOOL_NOT_AVAILABLE";
* `agent` child, which should not receive the root-only `Workflow` wrapper.
*/
export default defineEval({
+ tags: ["real-model"],
description: "Runtime subagent sessions do not receive the root-only Workflow tool.",
async test(t) {
await t.send(
diff --git a/e2e/fixtures/agent-subagents/evals/workflow-subagent-limit.eval.ts b/e2e/fixtures/agent-subagents/evals/workflow-subagent-limit.eval.ts
index 1689f1332..23188a8aa 100644
--- a/e2e/fixtures/agent-subagents/evals/workflow-subagent-limit.eval.ts
+++ b/e2e/fixtures/agent-subagents/evals/workflow-subagent-limit.eval.ts
@@ -6,6 +6,7 @@ import { defineEval } from "eve/evals";
* resolves inside the program with a `WORKFLOW_SUBAGENT_LIMIT_REACHED` error.
*/
export default defineEval({
+ tags: ["real-model"],
description:
"Workflow calls beyond the tool's maxSubagents config resolve with WORKFLOW_SUBAGENT_LIMIT_REACHED instead of spawning children.",
async test(t) {
diff --git a/e2e/fixtures/agent-subagents/package.json b/e2e/fixtures/agent-subagents/package.json
index 1424d6b5c..927e0e7a6 100644
--- a/e2e/fixtures/agent-subagents/package.json
+++ b/e2e/fixtures/agent-subagents/package.json
@@ -11,9 +11,11 @@
"test:e2e": "eve eval --strict"
},
"dependencies": {
+ "@eve-e2e/config": "workspace:*",
"@opentelemetry/core": "catalog:",
"@opentelemetry/sdk-trace-base": "catalog:",
"@vercel/otel": "catalog:",
+ "@workflow/world-postgres": "catalog:",
"eve": "workspace:*",
"zod": "catalog:"
},
diff --git a/e2e/fixtures/agent-tools-hitl-openai/agent/agent.ts b/e2e/fixtures/agent-tools-hitl-openai/agent/agent.ts
index 006ed0c4d..3b81e2012 100644
--- a/e2e/fixtures/agent-tools-hitl-openai/agent/agent.ts
+++ b/e2e/fixtures/agent-tools-hitl-openai/agent/agent.ts
@@ -1,3 +1,4 @@
+import { e2eAgentConfig } from "@eve-e2e/config";
import { defineAgent } from "eve";
/**
@@ -7,5 +8,5 @@ import { defineAgent } from "eve";
* validated by OpenAI's `function_call` / `function_call_output` pairing.
*/
export default defineAgent({
- model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol",
+ ...e2eAgentConfig(),
});
diff --git a/e2e/fixtures/agent-tools-hitl-openai/evals/evals.config.ts b/e2e/fixtures/agent-tools-hitl-openai/evals/evals.config.ts
index e5a4265ca..15fbd91d9 100644
--- a/e2e/fixtures/agent-tools-hitl-openai/evals/evals.config.ts
+++ b/e2e/fixtures/agent-tools-hitl-openai/evals/evals.config.ts
@@ -1,6 +1,7 @@
+import { e2eJudgeModel } from "@eve-e2e/config";
import { defineEvalConfig } from "eve/evals";
/** Default judge model for any `t.judge.*` assertion in this fixture. */
export default defineEvalConfig({
- judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" },
+ judge: { model: e2eJudgeModel() },
});
diff --git a/e2e/fixtures/agent-tools-hitl-openai/evals/hitl/approve-resume.eval.ts b/e2e/fixtures/agent-tools-hitl-openai/evals/hitl/approve-resume.eval.ts
index 350200a2d..19838cb88 100644
--- a/e2e/fixtures/agent-tools-hitl-openai/evals/hitl/approve-resume.eval.ts
+++ b/e2e/fixtures/agent-tools-hitl-openai/evals/hitl/approve-resume.eval.ts
@@ -13,6 +13,7 @@ const GUARDED_ECHO_OPENAI_TOKEN = "guarded-echo-openai-ok-R2D7";
* `No tool output found for function call call_<id>`.
*/
export default defineEval({
+ tags: ["real-model"],
description: "HITL regression (#236): approve-resume executes and replays on OpenAI Responses.",
async test(t) {
const parked = await t.send('Call the guarded-echo tool with note "openai-approve".');
diff --git a/e2e/fixtures/agent-tools-hitl-openai/package.json b/e2e/fixtures/agent-tools-hitl-openai/package.json
index 9d0ece5a1..cc6b941e9 100644
--- a/e2e/fixtures/agent-tools-hitl-openai/package.json
+++ b/e2e/fixtures/agent-tools-hitl-openai/package.json
@@ -11,9 +11,11 @@
"test:e2e": "eve eval --strict"
},
"dependencies": {
+ "@eve-e2e/config": "workspace:*",
"@opentelemetry/core": "catalog:",
"@opentelemetry/sdk-trace-base": "catalog:",
"@vercel/otel": "catalog:",
+ "@workflow/world-postgres": "catalog:",
"eve": "workspace:*",
"zod": "catalog:"
},
diff --git a/e2e/fixtures/agent-tools-hitl/agent/agent.ts b/e2e/fixtures/agent-tools-hitl/agent/agent.ts
index 93907447f..384bb0e2a 100644
--- a/e2e/fixtures/agent-tools-hitl/agent/agent.ts
+++ b/e2e/fixtures/agent-tools-hitl/agent/agent.ts
@@ -1,6 +1,7 @@
+import { e2eAgentConfig } from "@eve-e2e/config";
import { defineAgent } from "eve";
export default defineAgent({
- model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol",
+ ...e2eAgentConfig(),
reasoning: "high",
});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/evals.config.ts b/e2e/fixtures/agent-tools-hitl/evals/evals.config.ts
index e5a4265ca..15fbd91d9 100644
--- a/e2e/fixtures/agent-tools-hitl/evals/evals.config.ts
+++ b/e2e/fixtures/agent-tools-hitl/evals/evals.config.ts
@@ -1,6 +1,7 @@
+import { e2eJudgeModel } from "@eve-e2e/config";
import { defineEvalConfig } from "eve/evals";
/** Default judge model for any `t.judge.*` assertion in this fixture. */
export default defineEvalConfig({
- judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" },
+ judge: { model: e2eJudgeModel() },
});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/approve-then-no-regate.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/approve-then-no-regate.eval.ts
index 02b55aea0..bde31e6be 100644
--- a/e2e/fixtures/agent-tools-hitl/evals/hitl/approve-then-no-regate.eval.ts
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/approve-then-no-regate.eval.ts
@@ -8,6 +8,7 @@ import { GUARDED_ECHO_TOKEN } from "./shared";
* park/resume here is deterministic.
*/
export default defineEval({
+ tags: ["real-model"],
description: "HITL smoke: an approved once() grant persists for the session.",
async test(t) {
const parked = await t.send('Call the guarded-echo tool with note "first-call".');
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/ask-question-select.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/ask-question-select.eval.ts
index fc45c3dde..248659428 100644
--- a/e2e/fixtures/agent-tools-hitl/evals/hitl/ask-question-select.eval.ts
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/ask-question-select.eval.ts
@@ -6,6 +6,7 @@ import { defineEval } from "eve/evals";
* here is deterministic.
*/
export default defineEval({
+ tags: ["real-model"],
description: "HITL smoke: ask-question select parks and resumes with the chosen option.",
async test(t) {
await t.send(
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/authored-always-approve-resume.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/authored-always-approve-resume.eval.ts
index 5de7b5f44..31acbfa2c 100644
--- a/e2e/fixtures/agent-tools-hitl/evals/hitl/authored-always-approve-resume.eval.ts
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/authored-always-approve-resume.eval.ts
@@ -5,6 +5,7 @@ const TOOL_NAME = "gate";
/** Regression reproduction for https://github.com/vercel/eve/issues/533. */
export default defineEval({
+ tags: ["real-model"],
description:
"HITL repro (#533): a separate approval response executes an authored always-gated tool.",
async test(t) {
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/authored-always-unrelated-input.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/authored-always-unrelated-input.eval.ts
index 4887add5f..696955dda 100644
--- a/e2e/fixtures/agent-tools-hitl/evals/hitl/authored-always-unrelated-input.eval.ts
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/authored-always-unrelated-input.eval.ts
@@ -5,6 +5,7 @@ const TOOL_NAME = "gate";
/** Regression reproduction for https://github.com/vercel/eve/issues/533. */
export default defineEval({
+ tags: ["real-model"],
description:
"HITL repro (#533): unrelated input does not replay an unresolved authored tool call.",
async test(t) {
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/deny-then-regate.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/deny-then-regate.eval.ts
index 0a01180d5..9db60b964 100644
--- a/e2e/fixtures/agent-tools-hitl/evals/hitl/deny-then-regate.eval.ts
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/deny-then-regate.eval.ts
@@ -6,6 +6,7 @@ import { defineEval } from "eve/evals";
* park/resume here is deterministic.
*/
export default defineEval({
+ tags: ["real-model"],
description: "HITL smoke: a denied once() call does not execute and re-gates the next call.",
async test(t) {
await t.send('Call the guarded-echo tool with note "denied-call".');
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/dynamic-approve-then-followup.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/dynamic-approve-then-followup.eval.ts
index 95ece2901..70b16faf0 100644
--- a/e2e/fixtures/agent-tools-hitl/evals/hitl/dynamic-approve-then-followup.eval.ts
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/dynamic-approve-then-followup.eval.ts
@@ -14,6 +14,7 @@ const TOOL_NAME = "dynamic_guarded_echo";
* 400 lands, turning `session.waiting` into a terminal `session.failed`.
*/
export default defineEval({
+ tags: ["real-model"],
description: "HITL regression (#533): a resolved approval park replays on the next turn.",
async test(t) {
const parked = await t.send(`Call the \`${TOOL_NAME}\` tool with note "replay-probe".`);
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/dynamic-guarded-replay.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/dynamic-guarded-replay.eval.ts
index fa3f2c3ca..46a5c5471 100644
--- a/e2e/fixtures/agent-tools-hitl/evals/hitl/dynamic-guarded-replay.eval.ts
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/dynamic-guarded-replay.eval.ts
@@ -9,6 +9,7 @@ const TOOL_NAME = "dynamic_guarded_echo";
* this eval fails before approval.
*/
export default defineEval({
+ tags: ["real-model"],
description: "HITL smoke: replayed dynamic tools preserve approval.",
async test(t) {
const parked = await t.send(`Call the \`${TOOL_NAME}\` tool with note "before-approval".`);
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/slow-once-approve-replay.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/slow-once-approve-replay.eval.ts
index 189e12afd..787bb86b0 100644
--- a/e2e/fixtures/agent-tools-hitl/evals/hitl/slow-once-approve-replay.eval.ts
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/slow-once-approve-replay.eval.ts
@@ -15,6 +15,7 @@ const TOOL_NAME = "guarded-slow-echo";
* without a `tool_result` and the provider rejects every later turn.
*/
export default defineEval({
+ tags: ["real-model"],
description: "HITL regression (#460): approved slow tool result survives replay.",
async test(t) {
const parked = await t.send(
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/stale-ask-question-selection-idle.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/stale-ask-question-selection-idle.eval.ts
index 3f0fa2094..16fc7b947 100644
--- a/e2e/fixtures/agent-tools-hitl/evals/hitl/stale-ask-question-selection-idle.eval.ts
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/stale-ask-question-selection-idle.eval.ts
@@ -7,6 +7,7 @@ import { defineEval } from "eve/evals";
* the request.
*/
export default defineEval({
+ tags: ["real-model"],
description:
"HITL smoke: a stale ask-question selection becomes a new user turn when nothing is pending.",
async test(t) {
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/stale-ask-question-selection.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/stale-ask-question-selection.eval.ts
index 56185cb3c..78d1372a5 100644
--- a/e2e/fixtures/agent-tools-hitl/evals/hitl/stale-ask-question-selection.eval.ts
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/stale-ask-question-selection.eval.ts
@@ -7,6 +7,7 @@ import { defineEval } from "eve/evals";
* old selection is still relevant.
*/
export default defineEval({
+ tags: ["real-model"],
description:
"HITL smoke: a stale ask-question selection becomes a new user turn while another question is pending.",
async test(t) {
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/text-approve.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/text-approve.eval.ts
index 5ad2ff85c..ad8bc008d 100644
--- a/e2e/fixtures/agent-tools-hitl/evals/hitl/text-approve.eval.ts
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/text-approve.eval.ts
@@ -7,6 +7,7 @@ import { GUARDED_ECHO_TOKEN } from "./shared";
* resolves the pending approval the same way as structured inputResponses.
*/
export default defineEval({
+ tags: ["real-model"],
description: "HITL smoke: text approve resolves a pending tool approval.",
async test(t) {
const parked = await t.send('Call the guarded-echo tool with note "text-approve".');
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/unrelated-message-queued.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/unrelated-message-queued.eval.ts
index 2325bd472..74ff12b74 100644
--- a/e2e/fixtures/agent-tools-hitl/evals/hitl/unrelated-message-queued.eval.ts
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/unrelated-message-queued.eval.ts
@@ -8,6 +8,7 @@ import { GUARDED_ECHO_TOKEN } from "./shared";
* receives an explicit terminal answer.
*/
export default defineEval({
+ tags: ["real-model"],
description: "HITL smoke: unrelated message during approval is queued.",
async test(t) {
const parked = await t.send('Call the guarded-echo tool with note "queued-approval".');
diff --git a/e2e/fixtures/agent-tools-hitl/package.json b/e2e/fixtures/agent-tools-hitl/package.json
index 20fc21652..53f095904 100644
--- a/e2e/fixtures/agent-tools-hitl/package.json
+++ b/e2e/fixtures/agent-tools-hitl/package.json
@@ -11,9 +11,11 @@
"test:e2e": "eve eval --strict"
},
"dependencies": {
+ "@eve-e2e/config": "workspace:*",
"@opentelemetry/core": "catalog:",
"@opentelemetry/sdk-trace-base": "catalog:",
"@vercel/otel": "catalog:",
+ "@workflow/world-postgres": "catalog:",
"eve": "workspace:*",
"zod": "catalog:"
},
diff --git a/e2e/fixtures/agent-tools-sandbox/agent/agent.ts b/e2e/fixtures/agent-tools-sandbox/agent/agent.ts
index 93907447f..d3292ac9e 100644
--- a/e2e/fixtures/agent-tools-sandbox/agent/agent.ts
+++ b/e2e/fixtures/agent-tools-sandbox/agent/agent.ts
@@ -1,6 +1,68 @@
+import { e2eAgentConfig } from "@eve-e2e/config";
import { defineAgent } from "eve";
+import type { MockModelRequest, MockModelResponse } from "eve/evals";
+
+const BASH_DIRECTIVE = /run the bash command `([^`]+)`/iu;
+const SKILL_DIRECTIVE = /load the `([^`]+)` skill/iu;
+
+/**
+ * Scripted mock for the world suites: sandbox evals phrase every prompt as an
+ * explicit directive, so the responder executes exactly the requested tool
+ * and replies from its output. Turn state derives from the prompt: a tool
+ * message after the latest user message means this turn's call already ran.
+ */
+function respond(request: MockModelRequest): MockModelResponse | string {
+ const message = request.lastUserMessage ?? "";
+ const roles = request.messages.map((entry) => entry.role);
+ const turnHasToolResult = roles.lastIndexOf("tool") > roles.lastIndexOf("user");
+
+ const bash = BASH_DIRECTIVE.exec(message);
+ if (bash?.[1] !== undefined) {
+ if (!turnHasToolResult) {
+ return { toolCalls: [{ input: { command: bash[1] }, name: "bash" }] };
+ }
+ if (/reply with the single word:\s*done/iu.test(message)) {
+ return "done";
+ }
+ return bashStdout(request);
+ }
+
+ const skill = SKILL_DIRECTIVE.exec(message);
+ if (skill?.[1] !== undefined) {
+ if (!turnHasToolResult) {
+ return { toolCalls: [{ input: { skill: skill[1] }, name: "load_skill" }] };
+ }
+ // Loaded skills instruct an exact reply whose text is the skill body's
+ // final line (see the redeploy eval's deploy-note skill).
+ return lastNonEmptyLine(formatOutput(request.toolResults.at(-1)?.output));
+ }
+
+ return `Mock reply: ${message}`;
+}
+
+function bashStdout(request: MockModelRequest): string {
+ const output = [...request.toolResults]
+ .reverse()
+ .find((result) => result.name === "bash")?.output;
+ if (typeof output === "object" && output !== null && "stdout" in output) {
+ return String((output as { stdout: unknown }).stdout).trim();
+ }
+ return formatOutput(output);
+}
+
+function formatOutput(output: unknown): string {
+ return typeof output === "string" ? output : JSON.stringify(output ?? "");
+}
+
+function lastNonEmptyLine(text: string): string {
+ const lines = text
+ .split("\n")
+ .map((line) => line.trim())
+ .filter((line) => line.length > 0);
+ return lines.at(-1) ?? text;
+}
export default defineAgent({
- model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol",
+ ...e2eAgentConfig({ mock: respond }),
reasoning: "high",
});
diff --git a/e2e/fixtures/agent-tools-sandbox/evals/evals.config.ts b/e2e/fixtures/agent-tools-sandbox/evals/evals.config.ts
index e5a4265ca..15fbd91d9 100644
--- a/e2e/fixtures/agent-tools-sandbox/evals/evals.config.ts
+++ b/e2e/fixtures/agent-tools-sandbox/evals/evals.config.ts
@@ -1,6 +1,7 @@
+import { e2eJudgeModel } from "@eve-e2e/config";
import { defineEvalConfig } from "eve/evals";
/** Default judge model for any `t.judge.*` assertion in this fixture. */
export default defineEvalConfig({
- judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" },
+ judge: { model: e2eJudgeModel() },
});
diff --git a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/bash-curl-fanout.eval.ts b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/bash-curl-fanout.eval.ts
index 33ff985d5..f87f117e0 100644
--- a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/bash-curl-fanout.eval.ts
+++ b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/bash-curl-fanout.eval.ts
@@ -25,6 +25,7 @@ interface CurlBarrierResult {
}
export default defineEval({
+ tags: ["real-model"],
description: "Sandbox Bash: at least ten curls reach a concurrency barrier.",
async test(t) {
const turn = await t.send(
diff --git a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/bootstrap.eval.ts b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/bootstrap.eval.ts
index e1f7956fb..54fd503ce 100644
--- a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/bootstrap.eval.ts
+++ b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/bootstrap.eval.ts
@@ -6,6 +6,7 @@ import { BOOTSTRAP_MARKER_PATH, BOOTSTRAP_MARKER_TOKEN } from "./shared";
// non-error result containing the marker token proves the bootstrap-written
// file is visible inside the sandbox.
export default defineEval({
+ tags: ["real-model"],
description: "Sandbox smoke: `defineSandbox({ bootstrap })` runs before the first bash call.",
async test(t) {
await t.send(
diff --git a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/cli-install.eval.ts b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/cli-install.eval.ts
index c1c88b27c..4323c0b89 100644
--- a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/cli-install.eval.ts
+++ b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/cli-install.eval.ts
@@ -7,6 +7,7 @@ import { SANDBOX_CLI_NAME, SANDBOX_CLI_TOKEN } from "./shared";
// tooling is both on the PATH and executable in later sessions, and that the
// base image's Python runtime ran the bootstrap-authored script.
export default defineEval({
+ tags: ["real-model"],
description: "Sandbox: a custom CLI installed in `bootstrap` is on the PATH for later sessions.",
async test(t) {
await t.send(
diff --git a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/file-tools.eval.ts b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/file-tools.eval.ts
index ee5ece0b0..8d8080c0f 100644
--- a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/file-tools.eval.ts
+++ b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/file-tools.eval.ts
@@ -8,6 +8,7 @@ const FILE_TOOLS_TOKEN = "sandbox-file-tools-ok-Q2H";
const FILE_TOOLS_PATH = "/workspace/file-tools-note.txt";
export default defineEval({
+ tags: ["real-model"],
description: "Sandbox: built-in write_file/grep tools operate on the sandbox filesystem.",
async test(t) {
await t.send(
diff --git a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/network-policy.eval.ts b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/network-policy.eval.ts
index 5527bb50e..66ccb58ac 100644
--- a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/network-policy.eval.ts
+++ b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/network-policy.eval.ts
@@ -21,6 +21,7 @@ function isBlocked(value: unknown): boolean {
// then attempts egress. Asserting the probe came back blocked proves
// `setNetworkPolicy("deny-all")` actually severs egress on the active backend.
export default defineEval({
+ tags: ["real-model"],
description: "Sandbox: setNetworkPolicy('deny-all') blocks sandbox egress mid-turn.",
async test(t) {
await t.send(
diff --git a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/python-tool.eval.ts b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/python-tool.eval.ts
index 1baaef5d2..79c5e1485 100644
--- a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/python-tool.eval.ts
+++ b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/python-tool.eval.ts
@@ -5,6 +5,7 @@ import { defineEval } from "eve/evals";
// A correct sum proves the full authored-sandbox path works end-to-end with a
// real Python interpreter, not a simulated shell.
export default defineEval({
+ tags: ["real-model"],
description: "Sandbox: an authored tool runs real Python via ctx.getSandbox().",
async test(t) {
await t.send(
diff --git a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/session-persistence.eval.ts b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/session-persistence.eval.ts
index 37e9e5c4f..324f9ce17 100644
--- a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/session-persistence.eval.ts
+++ b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/session-persistence.eval.ts
@@ -9,6 +9,7 @@ const PERSIST_TOKEN = "sandbox-persist-ok-D6L";
const PERSIST_PATH = "/workspace/persist-note.txt";
export default defineEval({
+ tags: ["real-model"],
description: "Sandbox: workspace filesystem persists across turns in the same session.",
async test(t) {
const first = await t.send(
diff --git a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/session-setup.eval.ts b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/session-setup.eval.ts
index 5b5f7fd9f..2ec5ea892 100644
--- a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/session-setup.eval.ts
+++ b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/session-setup.eval.ts
@@ -13,6 +13,7 @@ import {
// A single `cat` of both files proving both tokens appear shows session-scoped
// setup and workspace seeding both landed on top of the shared template.
export default defineEval({
+ tags: ["real-model"],
description: "Sandbox: onSession marker and seeded workspace file are both present per session.",
async test(t) {
await t.send(
diff --git a/e2e/fixtures/agent-tools-sandbox/package.json b/e2e/fixtures/agent-tools-sandbox/package.json
index ddbc45eda..80a28ed79 100644
--- a/e2e/fixtures/agent-tools-sandbox/package.json
+++ b/e2e/fixtures/agent-tools-sandbox/package.json
@@ -11,9 +11,11 @@
"test:e2e": "eve eval --strict"
},
"dependencies": {
+ "@eve-e2e/config": "workspace:*",
"@opentelemetry/core": "catalog:",
"@opentelemetry/sdk-trace-base": "catalog:",
"@vercel/otel": "catalog:",
+ "@workflow/world-postgres": "catalog:",
"eve": "workspace:*",
"zod": "catalog:"
},
diff --git a/e2e/fixtures/agent-tools/agent/agent.ts b/e2e/fixtures/agent-tools/agent/agent.ts
index 93907447f..384bb0e2a 100644
--- a/e2e/fixtures/agent-tools/agent/agent.ts
+++ b/e2e/fixtures/agent-tools/agent/agent.ts
@@ -1,6 +1,7 @@
+import { e2eAgentConfig } from "@eve-e2e/config";
import { defineAgent } from "eve";
export default defineAgent({
- model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol",
+ ...e2eAgentConfig(),
reasoning: "high",
});
diff --git a/e2e/fixtures/agent-tools/evals/channel-events/batched-delivery.eval.ts b/e2e/fixtures/agent-tools/evals/channel-events/batched-delivery.eval.ts
index c63093e40..2b92bf9f8 100644
--- a/e2e/fixtures/agent-tools/evals/channel-events/batched-delivery.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/channel-events/batched-delivery.eval.ts
@@ -5,6 +5,7 @@ import { equals } from "eve/evals/expect";
import { coalesceDeliverPayloads } from "../../../../../packages/eve/src/execution/deliver-payloads.js";
export default defineEval({
+ tags: ["real-model"],
description: "Delivery batching smoke: queued messages and context coalesce in arrival order.",
async test(t) {
const firstMessage = "Remember synthetic marker BATCH-FIRST.";
diff --git a/e2e/fixtures/agent-tools/evals/channel-events/pre-tool-narration.eval.ts b/e2e/fixtures/agent-tools/evals/channel-events/pre-tool-narration.eval.ts
index 7998ff4db..20112d6c0 100644
--- a/e2e/fixtures/agent-tools/evals/channel-events/pre-tool-narration.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/channel-events/pre-tool-narration.eval.ts
@@ -101,6 +101,7 @@ async function postChannel(
* before the matching action request and result.
*/
export default defineEval({
+ tags: ["real-model"],
description: "Channel event smoke: pre-tool narration is visible when an action is requested.",
async test(t) {
const token = `channel-narration-${randomBytes(4).toString("hex")}`;
diff --git a/e2e/fixtures/agent-tools/evals/channel-metadata/anchored-no-topic.eval.ts b/e2e/fixtures/agent-tools/evals/channel-metadata/anchored-no-topic.eval.ts
index 09479bb45..7c2148a54 100644
--- a/e2e/fixtures/agent-tools/evals/channel-metadata/anchored-no-topic.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/channel-metadata/anchored-no-topic.eval.ts
@@ -8,6 +8,7 @@ import { METADATA_TOOL, PROMPT, startChannelSession } from "./shared";
// returns null and no tool registers: the turn completes without a
// dynamic-channel-metadata result and without failures.
export default defineEval({
+ tags: ["real-model"],
description: "Channel metadata smoke: missing topic metadata takes the null-resolve path.",
async test(t) {
const threadId = `thread-${randomBytes(4).toString("hex")}`;
diff --git a/e2e/fixtures/agent-tools/evals/dynamic-tools/authored-override.eval.ts b/e2e/fixtures/agent-tools/evals/dynamic-tools/authored-override.eval.ts
index 05b55fda3..a8707eabb 100644
--- a/e2e/fixtures/agent-tools/evals/dynamic-tools/authored-override.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/dynamic-tools/authored-override.eval.ts
@@ -7,6 +7,7 @@ import { defineEval } from "eve/evals";
const OVERRIDE_TOKEN = "dynamic-override-ok-K2P7";
export default defineEval({
+ tags: ["real-model"],
description: "Dynamic tools smoke: a dynamic tool overrides a same-named authored tool.",
async test(t) {
await t.send("Call the `override-target` tool and report the `source` value it returns.");
diff --git a/e2e/fixtures/agent-tools/evals/dynamic-tools/conditional.eval.ts b/e2e/fixtures/agent-tools/evals/dynamic-tools/conditional.eval.ts
index e03c76bb2..3cd24287e 100644
--- a/e2e/fixtures/agent-tools/evals/dynamic-tools/conditional.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/dynamic-tools/conditional.eval.ts
@@ -4,6 +4,7 @@ import { defineEval } from "eve/evals";
// truly runs once per session, both turns see { branch: "first" }; a
// re-run would surface { branch: "reran" }.
export default defineEval({
+ tags: ["real-model"],
description: "Dynamic tools smoke: the resolver runs once per session, not per turn.",
async test(t) {
const first = await t.send(
diff --git a/e2e/fixtures/agent-tools/evals/dynamic-tools/echo.eval.ts b/e2e/fixtures/agent-tools/evals/dynamic-tools/echo.eval.ts
index 8090cd550..ac8157d38 100644
--- a/e2e/fixtures/agent-tools/evals/dynamic-tools/echo.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/dynamic-tools/echo.eval.ts
@@ -5,6 +5,7 @@ import { DYNAMIC_ECHO_TOKEN, ECHO_TOOL } from "./shared";
// defineDynamic resolves at session.started; the tool is called and
// returns the fixture token.
export default defineEval({
+ tags: ["real-model"],
description: "Dynamic tools smoke: resolver registers the echo tool and it returns the token.",
async test(t) {
await t.send(
diff --git a/e2e/fixtures/agent-tools/evals/dynamic-tools/io-once.eval.ts b/e2e/fixtures/agent-tools/evals/dynamic-tools/io-once.eval.ts
index 43748af9a..ab4bf6c34 100644
--- a/e2e/fixtures/agent-tools/evals/dynamic-tools/io-once.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/dynamic-tools/io-once.eval.ts
@@ -3,6 +3,7 @@ import { defineEval } from "eve/evals";
// The resolver's simulated I/O runs once; the durable cache replays its
// result on the second turn so ioCallCount stays at 1.
export default defineEval({
+ tags: ["real-model"],
description: "Dynamic tools smoke: resolver I/O runs once and replays from the durable cache.",
async test(t) {
const first = await t.send(
diff --git a/e2e/fixtures/agent-tools/evals/dynamic-tools/loop.eval.ts b/e2e/fixtures/agent-tools/evals/dynamic-tools/loop.eval.ts
index a95073d85..15fe1332b 100644
--- a/e2e/fixtures/agent-tools/evals/dynamic-tools/loop.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/dynamic-tools/loop.eval.ts
@@ -2,6 +2,7 @@ import { defineEval } from "eve/evals";
// Tools generated inside a for loop keep their per-iteration closures.
export default defineEval({
+ tags: ["real-model"],
description: "Dynamic tools smoke: loop-generated tools keep per-iteration closures.",
async test(t) {
await t.send("Call the `alpha` tool and tell me the name and index it returned.");
diff --git a/e2e/fixtures/agent-tools/evals/dynamic-tools/messages.eval.ts b/e2e/fixtures/agent-tools/evals/dynamic-tools/messages.eval.ts
index 440ff6b38..d8422b71d 100644
--- a/e2e/fixtures/agent-tools/evals/dynamic-tools/messages.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/dynamic-tools/messages.eval.ts
@@ -4,6 +4,7 @@ import { satisfies } from "eve/evals/expect";
// The step.started resolver sees the accumulated message history: the
// second turn's count must exceed the first.
export default defineEval({
+ tags: ["real-model"],
description: "Dynamic tools smoke: step.started resolver sees accumulated message history.",
async test(t) {
const first = await t.send(
diff --git a/e2e/fixtures/agent-tools/evals/dynamic-tools/model-output.eval.ts b/e2e/fixtures/agent-tools/evals/dynamic-tools/model-output.eval.ts
index 1544a6fc0..cb4534497 100644
--- a/e2e/fixtures/agent-tools/evals/dynamic-tools/model-output.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/dynamic-tools/model-output.eval.ts
@@ -3,6 +3,7 @@ import { defineEval } from "eve/evals";
// The action.result stream event carries the RAW execute output
// (including the secret field), not the toModelOutput projection.
export default defineEval({
+ tags: ["real-model"],
description: "Dynamic tools smoke: action.result carries the raw execute output.",
async test(t) {
await t.send(
diff --git a/e2e/fixtures/agent-tools/evals/dynamic-tools/multi-turn.eval.ts b/e2e/fixtures/agent-tools/evals/dynamic-tools/multi-turn.eval.ts
index 32c82db01..4ef697308 100644
--- a/e2e/fixtures/agent-tools/evals/dynamic-tools/multi-turn.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/dynamic-tools/multi-turn.eval.ts
@@ -5,6 +5,7 @@ import { DYNAMIC_ECHO_TOKEN, ECHO_TOOL } from "./shared";
// The dynamic tool must survive serialization/deserialization (lazy
// replay of the resolver): both turns call it and see the token.
export default defineEval({
+ tags: ["real-model"],
description: "Dynamic tools smoke: the dynamic tool survives serialization across turns.",
async test(t) {
const first = await t.send(
diff --git a/e2e/fixtures/agent-tools/evals/dynamic-tools/nested.eval.ts b/e2e/fixtures/agent-tools/evals/dynamic-tools/nested.eval.ts
index c20256c3d..9ff33572e 100644
--- a/e2e/fixtures/agent-tools/evals/dynamic-tools/nested.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/dynamic-tools/nested.eval.ts
@@ -3,6 +3,7 @@ import { defineEval } from "eve/evals";
// One resolver returns a helper-built tool (closing over the helper
// param and handler vars) and an inline tool; both survive replay.
export default defineEval({
+ tags: ["real-model"],
description:
"Dynamic tools smoke: helper-built and inline tools from one resolver survive replay.",
async test(t) {
diff --git a/e2e/fixtures/agent-tools/evals/dynamic-tools/null.eval.ts b/e2e/fixtures/agent-tools/evals/dynamic-tools/null.eval.ts
index 58df218c8..836fdb623 100644
--- a/e2e/fixtures/agent-tools/evals/dynamic-tools/null.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/dynamic-tools/null.eval.ts
@@ -3,6 +3,7 @@ import { defineEval } from "eve/evals";
// A resolver returning null must register no tools: the turn completes
// without any "dynamic-null" call.
export default defineEval({
+ tags: ["real-model"],
description: "Dynamic tools smoke: a null-returning resolver registers no tools.",
async test(t) {
await t.send(
diff --git a/e2e/fixtures/agent-tools/evals/dynamic-tools/overwrite.eval.ts b/e2e/fixtures/agent-tools/evals/dynamic-tools/overwrite.eval.ts
index b46e1c14c..ffd94526c 100644
--- a/e2e/fixtures/agent-tools/evals/dynamic-tools/overwrite.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/dynamic-tools/overwrite.eval.ts
@@ -4,6 +4,7 @@ import { defineEval } from "eve/evals";
// `shared` wins over the session-scoped one, while `session_only`
// remains available from session scope.
export default defineEval({
+ tags: ["real-model"],
description: "Dynamic tools smoke: turn-scoped resolver tools win merges over session-scoped.",
async test(t) {
const first = await t.send("Call the `shared` tool and report the source and turn values.");
diff --git a/e2e/fixtures/agent-tools/evals/evals.config.ts b/e2e/fixtures/agent-tools/evals/evals.config.ts
index e5a4265ca..15fbd91d9 100644
--- a/e2e/fixtures/agent-tools/evals/evals.config.ts
+++ b/e2e/fixtures/agent-tools/evals/evals.config.ts
@@ -1,6 +1,7 @@
+import { e2eJudgeModel } from "@eve-e2e/config";
import { defineEvalConfig } from "eve/evals";
/** Default judge model for any `t.judge.*` assertion in this fixture. */
export default defineEvalConfig({
- judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" },
+ judge: { model: e2eJudgeModel() },
});
diff --git a/e2e/fixtures/agent-tools/evals/static-tools/dynamic-dependency-import.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/dynamic-dependency-import.eval.ts
index 0d138ccdb..e3ce3213f 100644
--- a/e2e/fixtures/agent-tools/evals/static-tools/dynamic-dependency-import.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/static-tools/dynamic-dependency-import.eval.ts
@@ -1,6 +1,7 @@
import { defineEval } from "eve/evals";
export default defineEval({
+ tags: ["real-model"],
description: "Static tools smoke: authored modules and dependencies can use dynamic imports.",
async test(t) {
await t.send(
diff --git a/e2e/fixtures/agent-tools/evals/static-tools/fanout-authored.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/fanout-authored.eval.ts
index 8e298be1a..56d672ed5 100644
--- a/e2e/fixtures/agent-tools/evals/static-tools/fanout-authored.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/static-tools/fanout-authored.eval.ts
@@ -17,6 +17,7 @@ const LABELS = [
] as const;
export default defineEval({
+ tags: ["real-model"],
description: "Static tools smoke: ten authored tool calls begin concurrently.",
async test(t) {
const turn = await t.send(
diff --git a/e2e/fixtures/agent-tools/evals/static-tools/multi-step-loop.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/multi-step-loop.eval.ts
index acbf951e9..b1d02a96d 100644
--- a/e2e/fixtures/agent-tools/evals/static-tools/multi-step-loop.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/static-tools/multi-step-loop.eval.ts
@@ -6,6 +6,7 @@ const MULTI_STEP_FINAL_VALUE = "phoenix-rising-9F2X";
// lookup-step-b in order, and the final value flows back into the
// user-visible reply.
export default defineEval({
+ tags: ["real-model"],
description: "Static tools smoke: deterministic two-step tool loop.",
async test(t) {
await t.send(
diff --git a/e2e/fixtures/agent-tools/evals/static-tools/streamed-action.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/streamed-action.eval.ts
index 88c231ced..e1231924c 100644
--- a/e2e/fixtures/agent-tools/evals/static-tools/streamed-action.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/static-tools/streamed-action.eval.ts
@@ -65,6 +65,7 @@ function readExecutionCompletedAt(output: unknown): number | undefined {
// consumed. The tool waits before completing, so post-execution batch emission
// still cannot satisfy this relation.
export default defineEval({
+ tags: ["real-model"],
description: "Static tools smoke: a local action request streams while execution is pending.",
async test(t) {
const turn = await t.send(
diff --git a/e2e/fixtures/agent-tools/evals/static-tools/structured-echo-narrowing.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/structured-echo-narrowing.eval.ts
index 5938fb97e..47e89f77a 100644
--- a/e2e/fixtures/agent-tools/evals/static-tools/structured-echo-narrowing.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/static-tools/structured-echo-narrowing.eval.ts
@@ -5,6 +5,7 @@ import { defineEval } from "eve/evals";
// toolResultFrom: a symbol-identity miss throws inside the hook, which
// surfaces as turn.failed and trips the succeeded check.
export default defineEval({
+ tags: ["real-model"],
description: "Static tools smoke: structured tool-result narrowing via toolResultFrom.",
async test(t) {
await t.send(
diff --git a/e2e/fixtures/agent-tools/evals/static-tools/throw-recover.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/throw-recover.eval.ts
index 1ffc58e74..26eb7e0b2 100644
--- a/e2e/fixtures/agent-tools/evals/static-tools/throw-recover.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/static-tools/throw-recover.eval.ts
@@ -3,6 +3,7 @@ import { defineEval } from "eve/evals";
// An authored tool throw surfaces as a failed action result (no turn.failed),
// and the session stays responsive for a follow-up.
export default defineEval({
+ tags: ["real-model"],
description: "Static tools smoke: tool throw surfaces as failed and the session recovers.",
async test(t) {
const first = await t.send(
diff --git a/e2e/fixtures/agent-tools/evals/static-tools/to-model-output-content-parts.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/to-model-output-content-parts.eval.ts
index 77f54483c..e24185c07 100644
--- a/e2e/fixtures/agent-tools/evals/static-tools/to-model-output-content-parts.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/static-tools/to-model-output-content-parts.eval.ts
@@ -9,6 +9,7 @@ const TOOL_NAME = "render-stripes";
// model exclusively through `toModelOutput` content parts. Color recognition
// remains tracked rather than gated because live vision quality varies.
export default defineEval({
+ tags: ["real-model"],
description: "Static tools smoke: toModelOutput content parts deliver an image to the model.",
async test(t) {
await t.send(
diff --git a/e2e/fixtures/agent-tools/evals/static-tools/web-fetch.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/web-fetch.eval.ts
index 586702be8..77a3f7faf 100644
--- a/e2e/fixtures/agent-tools/evals/static-tools/web-fetch.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/static-tools/web-fetch.eval.ts
@@ -5,6 +5,7 @@ const TARGET_URL = "https://example.com/";
const CONTENT_MARKER = "Example Domain";
export default defineEval({
+ tags: ["real-model"],
description: "Framework tools smoke: web_fetch retrieves an HTTPS page.",
async test(t) {
const turn = await t.send(
diff --git a/e2e/fixtures/agent-tools/evals/static-tools/web-search-narrated-ordering.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/web-search-narrated-ordering.eval.ts
index 0963d47a9..976341dee 100644
--- a/e2e/fixtures/agent-tools/evals/static-tools/web-search-narrated-ordering.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/static-tools/web-search-narrated-ordering.eval.ts
@@ -2,6 +2,7 @@ import { defineEval } from "eve/evals";
import { narratedWebSearchOrder, WEB_SEARCH_TOOL_NAME } from "./web-search-ordering";
export default defineEval({
+ tags: ["real-model"],
description: "Provider tools: narrated web searches preserve event order.",
async test(t) {
const turn = await t.send(
diff --git a/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts
index e826d8136..d351e8e77 100644
--- a/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/static-tools/web-search-parallel.eval.ts
@@ -25,6 +25,7 @@ function completedToolResultCount(events: readonly MessageStreamEvent[], toolNam
}
export default defineEval({
+ tags: ["real-model"],
description: "Provider tools smoke: eight parallel gateway web searches complete successfully.",
async test(t) {
const turn = await t.send(
diff --git a/e2e/fixtures/agent-tools/evals/static-tools/web-search-unnarrated-ordering.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/web-search-unnarrated-ordering.eval.ts
index 06f57bedd..ffa646569 100644
--- a/e2e/fixtures/agent-tools/evals/static-tools/web-search-unnarrated-ordering.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/static-tools/web-search-unnarrated-ordering.eval.ts
@@ -2,6 +2,7 @@ import { defineEval } from "eve/evals";
import { unNarratedWebSearchOrder, WEB_SEARCH_TOOL_NAME } from "./web-search-ordering";
export default defineEval({
+ tags: ["real-model"],
description: "Provider tools: un-narrated web searches preserve event order.",
async test(t) {
const turn = await t.send(
diff --git a/e2e/fixtures/agent-tools/evals/static-tools/web-search.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/web-search.eval.ts
index ecf4e2189..c70319ee2 100644
--- a/e2e/fixtures/agent-tools/evals/static-tools/web-search.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/static-tools/web-search.eval.ts
@@ -41,6 +41,7 @@ function providerRequestsPrecedeResults(events: readonly MessageStreamEvent[]):
}
export default defineEval({
+ tags: ["real-model"],
description: "Provider tools smoke: gateway web search answers a current-events question.",
async test(t) {
const turn = await t.send(
diff --git a/e2e/fixtures/agent-tools/package.json b/e2e/fixtures/agent-tools/package.json
index 339edee71..0036f5f3e 100644
--- a/e2e/fixtures/agent-tools/package.json
+++ b/e2e/fixtures/agent-tools/package.json
@@ -11,15 +11,20 @@
"test:e2e": "eve eval --strict"
},
"dependencies": {
+ "@eve-e2e/config": "workspace:*",
"@eve-e2e/dynamic-import-dependency": "file:./fixtures/dynamic-import-dependency",
"@opentelemetry/core": "catalog:",
"@opentelemetry/sdk-trace-base": "catalog:",
"@vercel/otel": "catalog:",
+ "@workflow/world-postgres": "catalog:",
"eve": "workspace:*",
"zod": "catalog:"
},
"devDependencies": {
"@types/node": "catalog:",
"typescript": "catalog:"
+ },
+ "e2e": {
+ "modelMatrix": "full"
}
}
diff --git a/e2e/fixtures/agent-workflow-stress/agent/agent.ts b/e2e/fixtures/agent-workflow-stress/agent/agent.ts
index aee410f61..377b38e64 100644
--- a/e2e/fixtures/agent-workflow-stress/agent/agent.ts
+++ b/e2e/fixtures/agent-workflow-stress/agent/agent.ts
@@ -1,7 +1,11 @@
+import { e2eAgentConfig } from "@eve-e2e/config";
import { defineAgent } from "eve";
import { mockModel } from "eve/evals";
export default defineAgent({
+ // Harness config wires the workflow world; the model is always this
+ // fixture's scripted mock.
+ ...e2eAgentConfig(),
model: mockModel(
({ lastUserMessage, userMessageCount }) =>
`stress-ack:${userMessageCount}:${lastUserMessage ?? ""}`,
diff --git a/e2e/fixtures/agent-workflow-stress/package.json b/e2e/fixtures/agent-workflow-stress/package.json
index 6f795ef69..aa07a981a 100644
--- a/e2e/fixtures/agent-workflow-stress/package.json
+++ b/e2e/fixtures/agent-workflow-stress/package.json
@@ -11,9 +11,11 @@
"test:e2e": "eve eval --strict"
},
"dependencies": {
+ "@eve-e2e/config": "workspace:*",
"@opentelemetry/core": "catalog:",
"@opentelemetry/sdk-trace-base": "catalog:",
"@vercel/otel": "catalog:",
+ "@workflow/world-postgres": "catalog:",
"ai": "catalog:",
"eve": "workspace:*"
},
diff --git a/e2e/fixtures/dist-extensions/agent/agent.ts b/e2e/fixtures/dist-extensions/agent/agent.ts
index 93907447f..384bb0e2a 100644
--- a/e2e/fixtures/dist-extensions/agent/agent.ts
+++ b/e2e/fixtures/dist-extensions/agent/agent.ts
@@ -1,6 +1,7 @@
+import { e2eAgentConfig } from "@eve-e2e/config";
import { defineAgent } from "eve";
export default defineAgent({
- model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol",
+ ...e2eAgentConfig(),
reasoning: "high",
});
diff --git a/e2e/fixtures/dist-extensions/evals/evals.config.ts b/e2e/fixtures/dist-extensions/evals/evals.config.ts
index 3e9bed0f7..7a61a5b55 100644
--- a/e2e/fixtures/dist-extensions/evals/evals.config.ts
+++ b/e2e/fixtures/dist-extensions/evals/evals.config.ts
@@ -1,5 +1,6 @@
+import { e2eJudgeModel } from "@eve-e2e/config";
import { defineEvalConfig } from "eve/evals";
export default defineEvalConfig({
- judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" },
+ judge: { model: e2eJudgeModel() },
});
diff --git a/e2e/fixtures/dist-extensions/evals/gadget.eval.ts b/e2e/fixtures/dist-extensions/evals/gadget.eval.ts
index b32939f78..567df0f87 100644
--- a/e2e/fixtures/dist-extensions/evals/gadget.eval.ts
+++ b/e2e/fixtures/dist-extensions/evals/gadget.eval.ts
@@ -1,6 +1,7 @@
import { defineEval } from "eve/evals";
export default defineEval({
+ tags: ["real-model"],
description:
"An agent-shaped dist extension with a registry-style store layout loads and its tool runs.",
async test(t) {
diff --git a/e2e/fixtures/dist-extensions/evals/gizmo.eval.ts b/e2e/fixtures/dist-extensions/evals/gizmo.eval.ts
index d6af70d05..5858782ce 100644
--- a/e2e/fixtures/dist-extensions/evals/gizmo.eval.ts
+++ b/e2e/fixtures/dist-extensions/evals/gizmo.eval.ts
@@ -1,6 +1,7 @@
import { defineEval } from "eve/evals";
export default defineEval({
+ tags: ["real-model"],
description:
"A dist-only extension installed with a registry-style store layout loads and its tool runs.",
async test(t) {
diff --git a/e2e/fixtures/dist-extensions/package.json b/e2e/fixtures/dist-extensions/package.json
index bfc477429..351cffbc1 100644
--- a/e2e/fixtures/dist-extensions/package.json
+++ b/e2e/fixtures/dist-extensions/package.json
@@ -10,6 +10,8 @@
"test:e2e": "eve eval --strict"
},
"dependencies": {
+ "@eve-e2e/config": "workspace:*",
+ "@workflow/world-postgres": "catalog:",
"eve": "workspace:*",
"gadget-extension": "file:../gadget-extension",
"gizmo-extension": "file:../gizmo-extension"
diff --git a/e2e/fixtures/e2e-config/package.json b/e2e/fixtures/e2e-config/package.json
new file mode 100644
index 000000000..fe666d38e
--- /dev/null
+++ b/e2e/fixtures/e2e-config/package.json
@@ -0,0 +1,17 @@
+{
+ "name": "@eve-e2e/config",
+ "version": "0.0.0",
+ "private": true,
+ "type": "module",
+ "exports": {
+ ".": "./src/index.ts"
+ },
+ "scripts": {
+ "typecheck": "tsc --noEmit"
+ },
+ "devDependencies": {
+ "@types/node": "catalog:",
+ "eve": "workspace:*",
+ "typescript": "catalog:"
+ }
+}
diff --git a/e2e/fixtures/e2e-config/src/index.ts b/e2e/fixtures/e2e-config/src/index.ts
new file mode 100644
index 000000000..32f960bea
--- /dev/null
+++ b/e2e/fixtures/e2e-config/src/index.ts
@@ -0,0 +1,106 @@
+import type { AgentDefinition } from "eve";
+import { mockModel, type MockModelResponder } from "eve/evals";
+
+/**
+ * Sentinel value for `EVE_E2E_MODEL` that makes fixtures author a
+ * deterministic `mockModel()` instead of a real gateway model. The world
+ * suites (Vercel, Postgres) set it so infrastructure coverage never depends
+ * on live model behavior.
+ */
+export const MOCK_MODEL_SENTINEL = "mock";
+
+const DEFAULT_MODEL = "openai/gpt-5.6-sol";
+
+// Mock models carry no AI Gateway metadata, so mock-mode configs must state a
+// context window explicitly for the compiler. Fixtures that need a specific
+// window override it after the spread.
+const MOCK_MODEL_CONTEXT_WINDOW_TOKENS = 1_000_000;
+
+/** Harness-owned agent configuration shared by e2e fixture root agents. */
+export type E2EAgentConfig = Pick<
+ AgentDefinition,
+ "experimental" | "model" | "modelContextWindowTokens"
+>;
+
+/** Harness-owned configuration for fixture subagents: model identity only. */
+export type E2ESubagentConfig = Pick<AgentDefinition, "model" | "modelContextWindowTokens">;
+
+/**
+ * A static model handle: a gateway model id string or a deterministic mock
+ * instance. Assignable anywhere fixtures author a non-dynamic model
+ * (agent `model`, compaction `model`, dynamic fallbacks, subagents).
+ */
+export type E2EModel = string | ReturnType<typeof mockModel>;
+
+/** Options for {@link e2eAgentConfig} and {@link e2eModel}. */
+export interface E2EModelOptions {
+ /**
+ * Responder (or static reply) used when the harness requests mock models
+ * via `EVE_E2E_MODEL=mock`. Defaults to a deterministic echo of the last
+ * user message.
+ */
+ readonly mock?: MockModelResponder | string;
+}
+
+/**
+ * Resolves the model a fixture agent (or subagent) should author:
+ * `EVE_E2E_MODEL=mock` yields a deterministic `mockModel()`, any other value
+ * is used as a gateway model id, and the harness default applies when unset.
+ */
+export function e2eModel(options?: E2EModelOptions): E2EModel {
+ const requested = process.env.EVE_E2E_MODEL;
+
+ if (requested === MOCK_MODEL_SENTINEL) {
+ return mockModel(options?.mock ?? defaultMockResponder);
+ }
+
+ return requested ?? DEFAULT_MODEL;
+}
+
+/**
+ * Returns the harness-owned configuration shared by e2e fixture root agents:
+ * the matrix model from `EVE_E2E_MODEL` (or a mock when the world suite
+ * requests one) and the workflow world override from `EVE_E2E_WORKFLOW_WORLD`.
+ */
+export function e2eAgentConfig(options?: E2EModelOptions): E2EAgentConfig {
+ const base = e2eSubagentConfig(options);
+ const workflowWorld = process.env.EVE_E2E_WORKFLOW_WORLD;
+ if (workflowWorld === undefined) {
+ return base;
+ }
+
+ return { ...base, experimental: { workflow: { world: workflowWorld } } };
+}
+
+/**
+ * Returns the harness-owned configuration for fixture subagents: the matrix
+ * model (or a mock plus its explicit context window in the world suites),
+ * without root-only settings like the workflow world.
+ */
+export function e2eSubagentConfig(options?: E2EModelOptions): E2ESubagentConfig {
+ const model = e2eModel(options);
+
+ if (typeof model === "string") {
+ return { model };
+ }
+
+ return { model, modelContextWindowTokens: MOCK_MODEL_CONTEXT_WINDOW_TOKENS };
+}
+
+/**
+ * Resolves the judge model for a fixture's `evals.config.ts`. Judge
+ * assertions only run in the model suite, so the mock sentinel falls back to
+ * the harness default rather than leaking `"mock"` as a gateway model id.
+ */
+export function e2eJudgeModel(): string {
+ const requested = process.env.EVE_E2E_MODEL;
+
+ if (requested === undefined || requested === MOCK_MODEL_SENTINEL) {
+ return DEFAULT_MODEL;
+ }
+
+ return requested;
+}
+
+const defaultMockResponder: MockModelResponder = ({ lastUserMessage }) =>
+ `Mock reply: ${lastUserMessage ?? ""}`;
diff --git a/e2e/fixtures/e2e-config/tsconfig.json b/e2e/fixtures/e2e-config/tsconfig.json
new file mode 100644
index 000000000..e4c8332fa
--- /dev/null
+++ b/e2e/fixtures/e2e-config/tsconfig.json
@@ -0,0 +1,17 @@
+{
+ "compilerOptions": {
+ "target": "ES2022",
+ "module": "esnext",
+ "moduleResolution": "bundler",
+ "outDir": "dist",
+ "rootDir": ".",
+ "strict": true,
+ "esModuleInterop": true,
+ "skipLibCheck": true,
+ "forceConsistentCasingInFileNames": true,
+ "declaration": true,
+ "noEmit": true,
+ "types": ["node"]
+ },
+ "include": ["src/**/*.ts"]
+}
diff --git a/e2e/fixtures/extensions/agent/agent.ts b/e2e/fixtures/extensions/agent/agent.ts
index 93907447f..384bb0e2a 100644
--- a/e2e/fixtures/extensions/agent/agent.ts
+++ b/e2e/fixtures/extensions/agent/agent.ts
@@ -1,6 +1,7 @@
+import { e2eAgentConfig } from "@eve-e2e/config";
import { defineAgent } from "eve";
export default defineAgent({
- model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol",
+ ...e2eAgentConfig(),
reasoning: "high",
});
diff --git a/e2e/fixtures/extensions/evals/config-binding.eval.ts b/e2e/fixtures/extensions/evals/config-binding.eval.ts
index fd19a4bed..8dc50b303 100644
--- a/e2e/fixtures/extensions/evals/config-binding.eval.ts
+++ b/e2e/fixtures/extensions/evals/config-binding.eval.ts
@@ -1,6 +1,7 @@
import { defineEval } from "eve/evals";
export default defineEval({
+ tags: ["real-model"],
description: "Mounted extension tool returns the config bound at the mount site.",
async test(t) {
await t.send(
diff --git a/e2e/fixtures/extensions/evals/dynamic-capability.eval.ts b/e2e/fixtures/extensions/evals/dynamic-capability.eval.ts
index 768ea8a0f..1661ee8f1 100644
--- a/e2e/fixtures/extensions/evals/dynamic-capability.eval.ts
+++ b/e2e/fixtures/extensions/evals/dynamic-capability.eval.ts
@@ -4,6 +4,7 @@ import { defineEval } from "eve/evals";
const TOOLKIT_FORECAST_TOKEN = "toolkit-forecast-ok-9F4Q";
export default defineEval({
+ tags: ["real-model"],
description: "Dynamic tool authored inside an extension resolves and runs when mounted.",
async test(t) {
await t.send("Call the `toolkit__toolkit_forecast` tool and report the token it returned.");
diff --git a/e2e/fixtures/extensions/evals/dynamic-skill.eval.ts b/e2e/fixtures/extensions/evals/dynamic-skill.eval.ts
index b910f9288..177bacde3 100644
--- a/e2e/fixtures/extensions/evals/dynamic-skill.eval.ts
+++ b/e2e/fixtures/extensions/evals/dynamic-skill.eval.ts
@@ -10,6 +10,7 @@ const TOOLKIT_INCIDENT_TOKEN = "toolkit-incident-dynamic-ok-7T2X";
* dynamic tools.
*/
export default defineEval({
+ tags: ["real-model"],
description: "Map-produced dynamic skill from an extension is namespaced (toolkit__incident).",
async test(t) {
await t.send(
diff --git a/e2e/fixtures/extensions/evals/evals.config.ts b/e2e/fixtures/extensions/evals/evals.config.ts
index 3e9bed0f7..7a61a5b55 100644
--- a/e2e/fixtures/extensions/evals/evals.config.ts
+++ b/e2e/fixtures/extensions/evals/evals.config.ts
@@ -1,5 +1,6 @@
+import { e2eJudgeModel } from "@eve-e2e/config";
import { defineEvalConfig } from "eve/evals";
export default defineEvalConfig({
- judge: { model: process.env.EVE_E2E_MODEL ?? "openai/gpt-5.6-sol" },
+ judge: { model: e2eJudgeModel() },
});
diff --git a/e2e/fixtures/extensions/evals/js-only-extension.eval.ts b/e2e/fixtures/extensions/evals/js-only-extension.eval.ts
index 96b7beb38..09348f039 100644
--- a/e2e/fixtures/extensions/evals/js-only-extension.eval.ts
+++ b/e2e/fixtures/extensions/evals/js-only-extension.eval.ts
@@ -3,6 +3,7 @@ import { defineEval } from "eve/evals";
const JS_ONLY_EXTENSION_TOKEN = "js-only-extension-dist-ok-4N7Q";
export default defineEval({
+ tags: ["real-model"],
description: "A JavaScript-only extension runs from its publisher-built dist tree.",
async test(t) {
await t.send("Call the `javascript__js_ping` tool and report exactly what it returned.");
diff --git a/e2e/fixtures/extensions/evals/mixed.eval.ts b/e2e/fixtures/extensions/evals/mixed.eval.ts
index 8a128db37..24711441c 100644
--- a/e2e/fixtures/extensions/evals/mixed.eval.ts
+++ b/e2e/fixtures/extensions/evals/mixed.eval.ts
@@ -1,6 +1,7 @@
import { defineEval } from "eve/evals";
export default defineEval({
+ tags: ["real-model"],
description: "Consumer-authored and mounted-extension tools coexist and both run in one turn.",
async test(t) {
await t.send(
diff --git a/e2e/fixtures/extensions/evals/override.eval.ts b/e2e/fixtures/extensions/evals/override.eval.ts
index aa40c8f05..33d602a58 100644
--- a/e2e/fixtures/extensions/evals/override.eval.ts
+++ b/e2e/fixtures/extensions/evals/override.eval.ts
@@ -1,6 +1,7 @@
import { defineEval } from "eve/evals";
export default defineEval({
+ tags: ["real-model"],
description: "A directory-mount override shadows a mounted extension tool of the same name.",
async test(t) {
await t.send("Call the `toolkit__toolkit_ping` tool and report exactly what it returned.");
diff --git a/e2e/fixtures/extensions/evals/skill-script-resource.eval.ts b/e2e/fixtures/extensions/evals/skill-script-resource.eval.ts
index 274c4215e..a33f8434d 100644
--- a/e2e/fixtures/extensions/evals/skill-script-resource.eval.ts
+++ b/e2e/fixtures/extensions/evals/skill-script-resource.eval.ts
@@ -3,6 +3,7 @@ import { defineEval } from "eve/evals";
const TOOLKIT_SKILL_SCRIPT_TOKEN = "toolkit-skill-script-ok-8M3P";
export default defineEval({
+ tags: ["real-model"],
description: "A dist-only extension preserves JavaScript files inside packaged skills.",
async test(t) {
await t.send(
diff --git a/e2e/fixtures/extensions/evals/state-isolation.eval.ts b/e2e/fixtures/extensions/evals/state-isolation.eval.ts
index 21e40c867..2f83f6b31 100644
--- a/e2e/fixtures/extensions/evals/state-isolation.eval.ts
+++ b/e2e/fixtures/extensions/evals/state-isolation.eval.ts
@@ -3,6 +3,7 @@ import { defineEval } from "eve/evals";
// Both extensions author defineState("budget"). If package scoping regressed,
// the counters would collapse onto one slot and gizmo would read 3, not 1.
export default defineEval({
+ tags: ["real-model"],
description: "Two extensions' identically-named defineState do not collide within a session.",
async test(t) {
await t.send(
diff --git a/e2e/fixtures/extensions/package.json b/e2e/fixtures/extensions/package.json
index efc5834e7..072ec4a81 100644
--- a/e2e/fixtures/extensions/package.json
+++ b/e2e/fixtures/extensions/package.json
@@ -11,6 +11,8 @@
"test:e2e": "eve eval --strict"
},
"dependencies": {
+ "@eve-e2e/config": "workspace:*",
+ "@workflow/world-postgres": "catalog:",
"eve": "workspace:*",
"gizmo-extension": "workspace:*",
"js-only-extension": "workspace:*",
diff --git a/e2e/matrix.json b/e2e/matrix.json
new file mode 100644
index 000000000..7c95cbbda
--- /dev/null
+++ b/e2e/matrix.json
@@ -0,0 +1,8 @@
+{
+ "$comment": "Registry for the e2e CI matrices. `models` drives the model suite (e2e-local): the first entry is the default every fixture runs on; fixtures with `\"e2e\": { \"modelMatrix\": \"full\" }` in package.json run on all of them. The short model name is the stable Actions check identifier, so updating a model version does not rename required checks. `worlds` drives the world suites: each entry backs one `world_matrix_<name>` output consumed by the matching `e2e-<name>.yml`, and `package` (when set) is exported to the job as EVE_E2E_WORKFLOW_WORLD. See e2e/README.md.",
+ "models": [
+ { "name": "openai-sol", "id": "openai/gpt-5.6-sol" },
+ { "name": "anthropic-opus", "id": "anthropic/claude-opus-5" }
+ ],
+ "worlds": [{ "name": "vercel" }, { "name": "postgres", "package": "@workflow/world-postgres" }]
+}
diff --git a/packages/eve/src/cli/run.ts b/packages/eve/src/cli/run.ts
index 7d3292930..6a4285270 100644
--- a/packages/eve/src/cli/run.ts
+++ b/packages/eve/src/cli/run.ts
@@ -22,6 +22,7 @@ import {
type DevelopmentRequestHeaders,
} from "#cli/dev/url-target.js";
import type { RunDevelopmentTuiInput } from "#cli/dev/tui/tui.js";
+import type { EvalCliOptions } from "#evals/cli/eval.js";
import {
registerRuntimeInvokeCommand,
type InvokeCliRuntimeDependencies,
@@ -128,19 +129,6 @@ function createDevBootProgressReporter(
};
}
-interface EvalCliOptions {
- json?: boolean;
- junit?: string;
- list?: boolean;
- maxConcurrency?: string;
- skipReport?: boolean;
- strict?: boolean;
- tag?: string[];
- timeout?: string;
- url?: string;
- verbose?: boolean;
-}
-
async function loadPrintApplicationInfo(): Promise<CliRuntimeDependencies["printApplicationInfo"]> {
return (await import("#cli/commands/info.js")).printApplicationInfo;
}
@@ -642,6 +630,7 @@ function createCliProgram(logger: CliLogger, runtime: CliRuntimeOverrides): Comm
)
.option("--url <url>", "Remote agent URL (skip local host startup)", parseDevelopmentServerUrl)
.option("--tag <tag...>", "Run only evals carrying a tag")
+ .option("--exclude-tag <tag...>", "Skip evals carrying a tag")
.option("--strict", "Fail the exit code when any score falls below its threshold")
.option("--list", "Print discovered evals without running them")
.option("--timeout <ms>", "Per-eval timeout in milliseconds")
diff --git a/packages/eve/src/evals/cli/eval.test.ts b/packages/eve/src/evals/cli/eval.test.ts
new file mode 100644
index 000000000..81c7a046f
--- /dev/null
+++ b/packages/eve/src/evals/cli/eval.test.ts
@@ -0,0 +1,87 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+import { runEvalCommand } from "#evals/cli/eval.js";
+import type { EveEval } from "#evals/types.js";
+
+vi.mock("#internal/application/paths.js", () => ({
+ resolveApplicationRoot: vi.fn(() => "/app"),
+}));
+vi.mock("#cli/dev/environment.js", () => ({
+ loadDevelopmentEnvironmentFiles: vi.fn(),
+}));
+vi.mock("#evals/runner/discover.js", () => ({
+ discoverAndImportEvals: vi.fn(async () => []),
+ discoverEvalConfig: vi.fn(async () => ({})),
+ findMisplacedEvalDirs: vi.fn(async () => []),
+}));
+
+const { discoverAndImportEvals } = vi.mocked(await import("#evals/runner/discover.js"), {
+ partial: true,
+});
+
+function createEval(id: string, tags?: readonly string[]): EveEval {
+ return { _tag: "EveEval", id, tags, test: async () => {} };
+}
+
+function createLogger() {
+ return { error: vi.fn(), log: vi.fn() };
+}
+
+describe("runEvalCommand list mode", () => {
+ beforeEach(() => {
+ process.exitCode = undefined;
+ });
+
+ afterEach(() => {
+ process.exitCode = undefined;
+ vi.clearAllMocks();
+ });
+
+ it("lists an empty JSON array when exclusion removes every eval", async () => {
+ discoverAndImportEvals.mockResolvedValueOnce([createEval("cancel", ["real-model"])]);
+ const logger = createLogger();
+
+ await runEvalCommand([], { excludeTag: ["real-model"], json: true, list: true }, logger);
+
+ expect(process.exitCode).toBeUndefined();
+ expect(logger.error).not.toHaveBeenCalled();
+ expect(logger.log).toHaveBeenCalledTimes(1);
+ expect(JSON.parse(logger.log.mock.calls[0]![0] as string)).toEqual([]);
+ });
+
+ it("lists only the evals that survive exclusion", async () => {
+ discoverAndImportEvals.mockResolvedValueOnce([
+ createEval("judged", ["real-model"]),
+ createEval("mock-capable"),
+ ]);
+ const logger = createLogger();
+
+ await runEvalCommand([], { excludeTag: ["real-model"], json: true, list: true }, logger);
+
+ expect(process.exitCode).toBeUndefined();
+ const listing = JSON.parse(logger.log.mock.calls[0]![0] as string) as { id: string }[];
+ expect(listing.map((entry) => entry.id)).toEqual(["mock-capable"]);
+ });
+
+ it("still succeeds with the exclusion notice on a non-list run", async () => {
+ discoverAndImportEvals.mockResolvedValueOnce([createEval("cancel", ["real-model"])]);
+ const logger = createLogger();
+
+ await runEvalCommand([], { excludeTag: ["real-model"] }, logger);
+
+ expect(process.exitCode).toBeUndefined();
+ expect(logger.log).toHaveBeenCalledWith(
+ "All 1 matching evals are excluded by tags (real-model); nothing to run.",
+ );
+ });
+
+ it("keeps exit 2 when an include tag matches nothing, even in list mode", async () => {
+ discoverAndImportEvals.mockResolvedValueOnce([createEval("mock-capable")]);
+ const logger = createLogger();
+
+ await runEvalCommand([], { json: true, list: true, tag: ["nonexistent"] }, logger);
+
+ expect(process.exitCode).toBe(2);
+ expect(logger.error).toHaveBeenCalledWith("No evals matched the provided tags (nonexistent).");
+ });
+});
diff --git a/packages/eve/src/evals/cli/eval.ts b/packages/eve/src/evals/cli/eval.ts
index f8ae00869..26a709075 100644
--- a/packages/eve/src/evals/cli/eval.ts
+++ b/packages/eve/src/evals/cli/eval.ts
@@ -6,6 +6,7 @@ import { shutdownActiveSandboxHandles } from "#execution/sandbox/active-handles.
import { resolveApplicationRoot } from "#internal/application/paths.js";
import { createDevelopmentServer, type DevelopmentServer } from "#internal/nitro/host.js";
import { createEvalClient } from "#evals/cli/eval-client.js";
+import { filterEvalsByTags } from "#evals/cli/filter.js";
import {
discoverAndImportEvals,
discoverEvalConfig,
@@ -18,7 +19,8 @@ import type { EvalReporter } from "#evals/runner/reporters/types.js";
import { resolveEvalTargetHandle } from "#evals/target.js";
import type { EveEval, EveEvalTargetHandle } from "#evals/types.js";
-interface EvalCliOptions {
+/** Parsed Commander options accepted by {@link runEvalCommand}. */
+export interface EvalCliOptions {
url?: string;
timeout?: string;
maxConcurrency?: string;
@@ -28,6 +30,7 @@ interface EvalCliOptions {
strict?: boolean;
list?: boolean;
tag?: string[];
+ excludeTag?: string[];
verbose?: boolean;
}
@@ -38,7 +41,9 @@ type EvalCliLogger = { log(message: string): void; error(message: string): void
*
* Exit codes: `0` when every executed eval passed its gate assertions (and
* soft thresholds under `--strict`), `1` when any eval failed, `2` for runner
- * or configuration errors (no evals discovered, no evals matching filters).
+ * or configuration errors (no evals discovered, no evals matching `--tag`).
+ * When `--exclude-tag` removes every matching eval the run succeeds with
+ * nothing executed.
*/
export async function runEvalCommand(
evalIds: readonly string[],
@@ -70,13 +75,34 @@ export async function runEvalCommand(
return;
}
- const evaluations = filterEvalsByTag(discovered, options.tag ?? []);
- if (evaluations.length === 0) {
- logger.error(`No evals matched the provided tags (${(options.tag ?? []).join(", ")}).`);
+ const includeTags = options.tag ?? [];
+ const excludeTags = options.excludeTag ?? [];
+ const included = filterEvalsByTags({ evaluations: discovered, includeTags, excludeTags: [] });
+ if (included.length === 0) {
+ logger.error(`No evals matched the provided tags (${includeTags.join(", ")}).`);
process.exitCode = 2;
return;
}
+ const evaluations = filterEvalsByTags({ evaluations: included, includeTags: [], excludeTags });
+
+ // List mode reports the post-exclusion set — even when it is empty — so
+ // suite runners can probe "does anything run here?" with `--list --json`.
+ if (options.list === true) {
+ printEvalList(evaluations, options.json === true, logger);
+ return;
+ }
+
+ if (evaluations.length === 0) {
+ // Every matching eval was excluded. Unlike an include filter with no
+ // matches, this is a legitimate "nothing applies to this run" outcome
+ // (e.g. a world suite excluding real-model evals), so it succeeds.
+ logger.log(
+ `All ${included.length} matching evals are excluded by tags (${excludeTags.join(", ")}); nothing to run.`,
+ );
+ return;
+ }
+
let maxConcurrency: number | undefined;
let timeoutMs: number | undefined;
try {
@@ -88,11 +114,6 @@ export async function runEvalCommand(
return;
}
- if (options.list === true) {
- printEvalList(evaluations, options.json === true, logger);
- return;
- }
-
let config;
try {
config = await discoverEvalConfig(appRoot);
@@ -201,18 +222,6 @@ function parseNonNegativeInteger(value: string | undefined, flag: string): numbe
return parsed;
}
-// ---------------------------------------------------------------------------
-// Eval filtering
-// ---------------------------------------------------------------------------
-
-/** Applies `--tag` filtering: an eval runs when it carries any requested tag. */
-function filterEvalsByTag(evaluations: readonly EveEval[], tags: readonly string[]): EveEval[] {
- if (tags.length === 0) return [...evaluations];
- return evaluations.filter(
- (evaluation) => evaluation.tags?.some((tag) => tags.includes(tag)) ?? false,
- );
-}
-
// ---------------------------------------------------------------------------
// --list
// ---------------------------------------------------------------------------
diff --git a/packages/eve/src/evals/cli/filter.test.ts b/packages/eve/src/evals/cli/filter.test.ts
new file mode 100644
index 000000000..826f645ed
--- /dev/null
+++ b/packages/eve/src/evals/cli/filter.test.ts
@@ -0,0 +1,62 @@
+import { describe, expect, it } from "vitest";
+
+import { filterEvalsByTags } from "#evals/cli/filter.js";
+import type { EveEval } from "#evals/types.js";
+
+function createEval(id: string, tags?: readonly string[]): EveEval {
+ return { _tag: "EveEval", id, tags, test: async () => {} };
+}
+
+const untagged = createEval("untagged");
+const fast = createEval("fast", ["fast"]);
+const realModel = createEval("real-model-only", ["real-model"]);
+const fastRealModel = createEval("fast-real-model", ["fast", "real-model"]);
+const evaluations = [untagged, fast, realModel, fastRealModel];
+
+describe("filterEvalsByTags", () => {
+ it("returns every eval when no filters are given", () => {
+ const result = filterEvalsByTags({ evaluations, includeTags: [], excludeTags: [] });
+ expect(result).toEqual(evaluations);
+ });
+
+ it("keeps only evals carrying an include tag", () => {
+ const result = filterEvalsByTags({ evaluations, includeTags: ["fast"], excludeTags: [] });
+ expect(result.map((entry) => entry.id)).toEqual(["fast", "fast-real-model"]);
+ });
+
+ it("drops evals carrying an exclude tag", () => {
+ const result = filterEvalsByTags({
+ evaluations,
+ excludeTags: ["real-model"],
+ includeTags: [],
+ });
+ expect(result.map((entry) => entry.id)).toEqual(["untagged", "fast"]);
+ });
+
+ it("applies exclusion after inclusion", () => {
+ const result = filterEvalsByTags({
+ evaluations,
+ excludeTags: ["real-model"],
+ includeTags: ["fast"],
+ });
+ expect(result.map((entry) => entry.id)).toEqual(["fast"]);
+ });
+
+ it("returns an empty list when exclusion removes every match", () => {
+ const result = filterEvalsByTags({
+ evaluations: [realModel],
+ excludeTags: ["real-model"],
+ includeTags: [],
+ });
+ expect(result).toEqual([]);
+ });
+
+ it("treats untagged evals as excluded by include filters only", () => {
+ const result = filterEvalsByTags({
+ evaluations: [untagged],
+ excludeTags: ["real-model"],
+ includeTags: [],
+ });
+ expect(result).toEqual([untagged]);
+ });
+});
diff --git a/packages/eve/src/evals/cli/filter.ts b/packages/eve/src/evals/cli/filter.ts
new file mode 100644
index 000000000..d58ed6b46
--- /dev/null
+++ b/packages/eve/src/evals/cli/filter.ts
@@ -0,0 +1,25 @@
+import type { EveEval } from "#evals/types.js";
+
+/**
+ * Applies `--tag` and `--exclude-tag` filtering for the `eve eval` command.
+ *
+ * An eval runs when it carries at least one `includeTags` entry (or no
+ * include filter is given) and carries none of the `excludeTags` entries.
+ * Exclusion wins over inclusion so suite-level runs (e.g. a mock-model world
+ * suite excluding `real-model` evals) can never pull in an excluded eval.
+ */
+export function filterEvalsByTags(input: {
+ readonly evaluations: readonly EveEval[];
+ readonly includeTags: readonly string[];
+ readonly excludeTags: readonly string[];
+}): EveEval[] {
+ return input.evaluations.filter((evaluation) => {
+ const tags = evaluation.tags ?? [];
+
+ if (input.includeTags.length > 0 && !tags.some((tag) => input.includeTags.includes(tag))) {
+ return false;
+ }
+
+ return !tags.some((tag) => input.excludeTags.includes(tag));
+ });
+}
diff --git a/packages/eve/src/execution/workflow-entry.integration.test.ts b/packages/eve/src/execution/workflow-entry.integration.test.ts
index ad106cb5b..803d95c6f 100644
--- a/packages/eve/src/execution/workflow-entry.integration.test.ts
+++ b/packages/eve/src/execution/workflow-entry.integration.test.ts
@@ -328,8 +328,11 @@ describe("workflowEntry integration", () => {
expect(firstTurn.every((event) => isEventId(event.meta.id))).toBe(true);
expect(new Set(firstTurn.map((event) => event.meta.id)).size).toBe(firstTurn.length);
+ // No stream-order assertion on the ids: they sort in mint order per
+ // process, but a turn's events are appended by separate steps whose
+ // writes can interleave behind minting (see #protocol/event-id.js),
+ // so append order is not contractually sorted.
const ids = firstTurn.map((event) => event.meta.id);
- expect(ids).toEqual([...ids].sort());
// Re-reading the durable stream returns the same ids.
const workflowRuntime = createWorkflowRuntime({
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index d1381c7fd..eaf7c900e 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -51,6 +51,9 @@ catalogs:
'@vercel/sandbox':
specifier: 2.8.0
version: 2.8.0
+ '@workflow/world-postgres':
+ specifier: 5.0.0-beta.29
+ version: 5.0.0-beta.29
ai:
specifier: ^7.0.38
version: 7.0.38
@@ -142,7 +145,7 @@ importers:
version: 1.1.1([email protected])
'@vercel/analytics':
specifier: 2.0.1
- version: 2.0.1(2e1d1ea39f306c545b71c35841ac134e)
'@vercel/eve-catalog':
specifier: workspace:*
version: link:../../packages/eve-catalog
@@ -151,7 +154,7 @@ importers:
'@vercel/speed-insights':
specifier: 2.0.0
- version: 2.0.0(2e1d1ea39f306c545b71c35841ac134e)
'@vgpu/core':
specifier: 0.0.7
version: 0.0.7
@@ -473,7 +476,7 @@ importers:
version: link:../../../packages/eve
nuxt:
specifier: ^4.0.0
+ version: 4.4.6(f32df07d5233da3a29c17b72e1467465)
vue:
specifier: ^3.5.0
version: 3.5.35([email protected])
@@ -521,6 +524,9 @@ importers:
e2e/fixtures/agent-basic-runtime:
dependencies:
+ '@eve-e2e/config':
+ specifier: workspace:*
+ version: link:../e2e-config
'@opentelemetry/core':
specifier: 'catalog:'
version: 2.6.1(@opentelemetry/[email protected])
@@ -530,6 +536,9 @@ importers:
'@vercel/otel':
specifier: 'catalog:'
version: 2.1.3(@opentelemetry/[email protected])(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected])([email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))
+ '@workflow/world-postgres':
+ specifier: 'catalog:'
eve:
specifier: workspace:*
version: link:../../../packages/eve
@@ -546,6 +555,9 @@ importers:
e2e/fixtures/agent-cancellation:
dependencies:
+ '@eve-e2e/config':
+ specifier: workspace:*
+ version: link:../e2e-config
'@opentelemetry/core':
specifier: 'catalog:'
version: 2.6.1(@opentelemetry/[email protected])
@@ -555,6 +567,9 @@ importers:
'@vercel/otel':
specifier: 'catalog:'
version: 2.1.3(@opentelemetry/[email protected])(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected])([email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))
+ '@workflow/world-postgres':
+ specifier: 'catalog:'
eve:
specifier: workspace:*
version: link:../../../packages/eve
@@ -571,6 +586,9 @@ importers:
e2e/fixtures/agent-channels:
dependencies:
+ '@eve-e2e/config':
+ specifier: workspace:*
+ version: link:../e2e-config
'@opentelemetry/core':
specifier: 'catalog:'
version: 2.6.1(@opentelemetry/[email protected])
@@ -580,6 +598,9 @@ importers:
'@vercel/otel':
specifier: 'catalog:'
version: 2.1.3(@opentelemetry/[email protected])(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected])([email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))
+ '@workflow/world-postgres':
+ specifier: 'catalog:'
ai:
specifier: 'catalog:'
version: 7.0.38([email protected])
@@ -599,6 +620,9 @@ importers:
e2e/fixtures/agent-compaction-regressions:
dependencies:
+ '@eve-e2e/config':
+ specifier: workspace:*
+ version: link:../e2e-config
'@opentelemetry/core':
specifier: 'catalog:'
version: 2.6.1(@opentelemetry/[email protected])
@@ -608,6 +632,9 @@ importers:
'@vercel/otel':
specifier: 'catalog:'
version: 2.1.3(@opentelemetry/[email protected])(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected])([email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))
+ '@workflow/world-postgres':
+ specifier: 'catalog:'
eve:
specifier: workspace:*
version: link:../../../packages/eve
@@ -624,6 +651,12 @@ importers:
e2e/fixtures/agent-model:
dependencies:
+ '@eve-e2e/config':
+ specifier: workspace:*
+ version: link:../e2e-config
+ '@workflow/world-postgres':
+ specifier: 'catalog:'
eve:
specifier: workspace:*
version: link:../../../packages/eve
@@ -637,6 +670,9 @@ importers:
e2e/fixtures/agent-openapi-swagger:
dependencies:
+ '@eve-e2e/config':
+ specifier: workspace:*
+ version: link:../e2e-config
'@opentelemetry/core':
specifier: 'catalog:'
version: 2.6.1(@opentelemetry/[email protected])
@@ -646,6 +682,9 @@ importers:
'@vercel/otel':
specifier: 'catalog:'
version: 2.1.3(@opentelemetry/[email protected])(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected])([email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))
+ '@workflow/world-postgres':
+ specifier: 'catalog:'
eve:
specifier: workspace:*
version: link:../../../packages/eve
@@ -665,6 +704,12 @@ importers:
'@ai-sdk/anthropic':
specifier: 'catalog:'
version: 4.0.18([email protected])
+ '@eve-e2e/config':
+ specifier: workspace:*
+ version: link:../e2e-config
+ '@workflow/world-postgres':
+ specifier: 'catalog:'
eve:
specifier: workspace:*
version: link:../../../packages/eve
@@ -681,6 +726,9 @@ importers:
e2e/fixtures/agent-schedules:
dependencies:
+ '@eve-e2e/config':
+ specifier: workspace:*
+ version: link:../e2e-config
'@opentelemetry/core':
specifier: 'catalog:'
version: 2.6.1(@opentelemetry/[email protected])
@@ -690,6 +738,9 @@ importers:
'@vercel/otel':
specifier: 'catalog:'
version: 2.1.3(@opentelemetry/[email protected])(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected])([email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))
+ '@workflow/world-postgres':
+ specifier: 'catalog:'
eve:
specifier: workspace:*
version: link:../../../packages/eve
@@ -706,6 +757,9 @@ importers:
e2e/fixtures/agent-session-limits:
dependencies:
+ '@eve-e2e/config':
+ specifier: workspace:*
+ version: link:../e2e-config
'@opentelemetry/core':
specifier: 'catalog:'
version: 2.6.1(@opentelemetry/[email protected])
@@ -715,6 +769,9 @@ importers:
'@vercel/otel':
specifier: 'catalog:'
version: 2.1.3(@opentelemetry/[email protected])(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected])([email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))
+ '@workflow/world-postgres':
+ specifier: 'catalog:'
eve:
specifier: workspace:*
version: link:../../../packages/eve
@@ -731,6 +788,9 @@ importers:
e2e/fixtures/agent-session-timeout:
dependencies:
+ '@eve-e2e/config':
+ specifier: workspace:*
+ version: link:../e2e-config
'@opentelemetry/core':
specifier: 'catalog:'
version: 2.6.1(@opentelemetry/[email protected])
@@ -740,6 +800,9 @@ importers:
'@vercel/otel':
specifier: 'catalog:'
version: 2.1.3(@opentelemetry/[email protected])(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected])([email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))
+ '@workflow/world-postgres':
+ specifier: 'catalog:'
ai:
specifier: 'catalog:'
version: 7.0.38([email protected])
@@ -756,6 +819,9 @@ importers:
e2e/fixtures/agent-skills:
dependencies:
+ '@eve-e2e/config':
+ specifier: workspace:*
+ version: link:../e2e-config
'@opentelemetry/core':
specifier: 'catalog:'
version: 2.6.1(@opentelemetry/[email protected])
@@ -765,6 +831,9 @@ importers:
'@vercel/otel':
specifier: 'catalog:'
version: 2.1.3(@opentelemetry/[email protected])(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected])([email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))
+ '@workflow/world-postgres':
+ specifier: 'catalog:'
eve:
specifier: workspace:*
version: link:../../../packages/eve
@@ -781,6 +850,9 @@ importers:
e2e/fixtures/agent-subagents:
dependencies:
+ '@eve-e2e/config':
+ specifier: workspace:*
+ version: link:../e2e-config
'@opentelemetry/core':
specifier: 'catalog:'
version: 2.6.1(@opentelemetry/[email protected])
@@ -790,6 +862,9 @@ importers:
'@vercel/otel':
specifier: 'catalog:'
version: 2.1.3(@opentelemetry/[email protected])(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected])([email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))
+ '@workflow/world-postgres':
+ specifier: 'catalog:'
eve:
specifier: workspace:*
version: link:../../../packages/eve
@@ -806,6 +881,12 @@ importers:
e2e/fixtures/agent-subagents-hitl:
dependencies:
+ '@eve-e2e/config':
+ specifier: workspace:*
+ version: link:../e2e-config
+ '@workflow/world-postgres':
+ specifier: 'catalog:'
eve:
specifier: workspace:*
version: link:../../../packages/eve
@@ -822,6 +903,9 @@ importers:
e2e/fixtures/agent-tools:
dependencies:
+ '@eve-e2e/config':
+ specifier: workspace:*
+ version: link:../e2e-config
'@eve-e2e/dynamic-import-dependency':
specifier: file:./fixtures/dynamic-import-dependency
version: file:e2e/fixtures/agent-tools/fixtures/dynamic-import-dependency
@@ -834,6 +918,9 @@ importers:
'@vercel/otel':
specifier: 'catalog:'
version: 2.1.3(@opentelemetry/[email protected])(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected])([email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))
+ '@workflow/world-postgres':
+ specifier: 'catalog:'
eve:
specifier: workspace:*
version: link:../../../packages/eve
@@ -850,6 +937,9 @@ importers:
e2e/fixtures/agent-tools-hitl:
dependencies:
+ '@eve-e2e/config':
+ specifier: workspace:*
+ version: link:../e2e-config
'@opentelemetry/core':
specifier: 'catalog:'
version: 2.6.1(@opentelemetry/[email protected])
@@ -859,6 +949,9 @@ importers:
'@vercel/otel':
specifier: 'catalog:'
version: 2.1.3(@opentelemetry/[email protected])(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected])([email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))
+ '@workflow/world-postgres':
+ specifier: 'catalog:'
eve:
specifier: workspace:*
version: link:../../../packages/eve
@@ -875,6 +968,9 @@ importers:
e2e/fixtures/agent-tools-hitl-openai:
dependencies:
+ '@eve-e2e/config':
+ specifier: workspace:*
+ version: link:../e2e-config
'@opentelemetry/core':
specifier: 'catalog:'
version: 2.6.1(@opentelemetry/[email protected])
@@ -884,6 +980,9 @@ importers:
'@vercel/otel':
specifier: 'catalog:'
version: 2.1.3(@opentelemetry/[email protected])(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected])([email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))
+ '@workflow/world-postgres':
+ specifier: 'catalog:'
eve:
specifier: workspace:*
version: link:../../../packages/eve
@@ -900,6 +999,9 @@ importers:
e2e/fixtures/agent-tools-sandbox:
dependencies:
+ '@eve-e2e/config':
+ specifier: workspace:*
+ version: link:../e2e-config
'@opentelemetry/core':
specifier: 'catalog:'
version: 2.6.1(@opentelemetry/[email protected])
@@ -909,6 +1011,9 @@ importers:
'@vercel/otel':
specifier: 'catalog:'
version: 2.1.3(@opentelemetry/[email protected])(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected])([email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))
+ '@workflow/world-postgres':
+ specifier: 'catalog:'
eve:
specifier: workspace:*
version: link:../../../packages/eve
@@ -925,6 +1030,9 @@ importers:
e2e/fixtures/agent-workflow-stress:
dependencies:
+ '@eve-e2e/config':
+ specifier: workspace:*
+ version: link:../e2e-config
'@opentelemetry/core':
specifier: 'catalog:'
version: 2.6.1(@opentelemetry/[email protected])
@@ -934,6 +1042,9 @@ importers:
'@vercel/otel':
specifier: 'catalog:'
version: 2.1.3(@opentelemetry/[email protected])(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected])([email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))
+ '@workflow/world-postgres':
+ specifier: 'catalog:'
ai:
specifier: 'catalog:'
version: 7.0.38([email protected])
@@ -950,6 +1061,12 @@ importers:
e2e/fixtures/dist-extensions:
dependencies:
+ '@eve-e2e/config':
+ specifier: workspace:*
+ version: link:../e2e-config
+ '@workflow/world-postgres':
+ specifier: 'catalog:'
eve:
specifier: workspace:*
version: link:../../../packages/eve
@@ -967,8 +1084,26 @@ importers:
specifier: 'catalog:'
version: 7.0.2
+ e2e/fixtures/e2e-config:
+ devDependencies:
+ '@types/node':
+ specifier: 'catalog:'
+ version: 25.9.1
+ eve:
+ specifier: workspace:*
+ version: link:../../../packages/eve
+ typescript:
+ specifier: 'catalog:'
+ version: 7.0.2
+
e2e/fixtures/extensions:
dependencies:
+ '@eve-e2e/config':
+ specifier: workspace:*
+ version: link:../e2e-config
+ '@workflow/world-postgres':
+ specifier: 'catalog:'
eve:
specifier: workspace:*
version: link:../../../packages/eve
@@ -1069,7 +1204,7 @@ importers:
nitro:
specifier: 3.0.260610-beta
undici:
specifier: 8.9.0
version: 8.9.0
@@ -2738,6 +2873,9 @@ packages:
resolution: {integrity: sha512-9tU0h6O/Ao9k53TQSgLs6D/mN++wKTHPIpT8tbPaayhOL3e4OhtJ3t+OD42avMe2pBmFcdRUVOSpZvqytxvWEg==}
engines: {node: '>=12.0.0'}
+ '@graphile/[email protected]':
+ resolution: {integrity: sha512-jjcWBokl9eb1gVJ85QmoaQ73CQ52xAaOCF29ukRbYNl6lY+ts0ErTaDYOBlejcbUs2OpaiqYLO5uDhyLFzWw4w==}
+
resolution: {integrity: sha512-k9Dj3DV/itK9D06Y8f190Qgop7/Ui+D0njFV3LHMPwPT75DpXLQohE9Wmz0QElrJnzsjB7KPWiKJbOl7IPDArQ==}
engines: {node: '>=12.10.0'}
@@ -6434,6 +6572,9 @@ packages:
resolution: {integrity: sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==}
+ '@types/[email protected]':
+ resolution: {integrity: sha512-r+tPKWHYqaxJOYA3Eik0mMi+SEREqOXLmsooRFmc6GHv7nWUDixFtKN+cegvsPlDcEZd9wxsdp041v2imQuvag==}
+
resolution: {integrity: sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==}
@@ -6473,9 +6614,15 @@ packages:
resolution: {integrity: sha512-o9bjXmDNcF7GbM4CNQpmi+TutCgap/K3w1JyKgxAjqx41zp9qlIAVFi0IhCNsJcXolEqLWhbFbEeL0PvYm4pcQ==}
+ '@types/[email protected]':
+ resolution: {integrity: sha512-EANqOCF9QFyra+4pfxUcX9STKJpCLjMbObVzljIJomAWSnuSIEAvyzEU53GaajbXJEgdh0iEcPL+DGvpUd4k1Q==}
+
resolution: {integrity: sha512-xfrlY7UD5rMJk3ZVJP8BNzS28J36YJg+xp+LPXV1TdWxr8uMH5A860QNxYDGQe/ylDSgjxE52Q9VnO7p75tJxg==}
+ '@types/[email protected]':
+ resolution: {integrity: sha512-bEPFOaMAHTEP1EzpvHTbmwR8UsFyHSKsRisLIHVMXnpNefSbGA1bD6CVy+qKjGSqmZqNqBDV2azOBo8TgkcVow==}
+
resolution: {integrity: sha512-0k5eKfrA83JOZPppLtS2C7OUtyNAl2wKNxfyYl9Q5g9lPkgBl/9hNyAu6HuEH2J4XmIv2znEpkDd0SaZVxW6iQ==}
@@ -6496,6 +6643,9 @@ packages:
resolution: {integrity: sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==}
+ '@types/[email protected]':
+ resolution: {integrity: sha512-FmgJfu+MOcQ370SD0ev7EI8TlCAfKYU+B4m5T3yXc1CiRN94g/SZPtsCkk506aUDtlMnFZvasDwHHUcZUEaYuA==}
+
resolution: {integrity: sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==}
@@ -7290,6 +7440,9 @@ packages:
'@opentelemetry/api':
optional: true
+ '@workflow/[email protected]':
+ resolution: {integrity: sha512-xRiG9MEunLxkb7hw7n17+/cKKH4QfFvnQly4cAF2ImxPRDnL0HvWot9eHOPS3S41LEyZ/LSs/sYrIN07XUZbgw==}
+
'@workflow/[email protected]':
resolution: {integrity: sha512-1RwF7LG7JbIp78zD+eb1A8ifDacI12wzpdMAFbzMODc/Ok0u+1IXbT1DIVKLtBLnUOPrWLxnXBYDnkvkvG9tAw==}
@@ -7302,9 +7455,20 @@ packages:
'@workflow/[email protected]':
resolution: {integrity: sha512-INB+FcEKQkkFZa+s53sEMTNRFiBa2g9vzjsNuEDagvWxtI0t/cnCyoc57i7kS7nj+3/vGeRjO4Yn5ccbT0hyBQ==}
+ '@workflow/[email protected]':
+ resolution: {integrity: sha512-3vDiA8xea4USslIbz6m9HAhxDhvqLbETaXFxTNqVm/m8eji0rLKwnfF6JLaH0Qr1GsovysoOUlvyibokkNHZrg==}
+
'@workflow/[email protected]':
resolution: {integrity: sha512-dDiQ/LT05g9HyOicqWNNX/Gw/fmO7Ma2kZ+ddUXXpm/EFTMRS9eiI69suSe1+1NnhMzdV6ObI8wrO5lvQqPIbg==}
+ '@workflow/[email protected]':
+ resolution: {integrity: sha512-Z9nq5lzjXnig4ONfLnVwTqx6EbxJjhTB5SBKT8ERcKe6/SFm30s1BIZ03bZBJox2Y4jJAgsNgvey29xWGnzMAg==}
+ peerDependencies:
+ '@opentelemetry/api': '1'
+ peerDependenciesMeta:
+ '@opentelemetry/api':
+ optional: true
+
'@workflow/[email protected]':
resolution: {integrity: sha512-D8kb1IHHrkDbKnpUw3cqZnlkis3YCYPF2BoV6vWpGUGTeSGsnTZjMsrqsh5luKeZSUqK5BCO/YC27S9AdkBO0w==}
peerDependencies:
@@ -7313,6 +7477,10 @@ packages:
'@opentelemetry/api':
optional: true
+ '@workflow/[email protected]':
+ resolution: {integrity: sha512-OaqDbvtM1mfsQtzOndyepyPARYXkWWZc53wcV3U3l65K8rMFjwX3TG+721wpEnIvWiFBVD3MJ8HjgnIIQG/1Hw==}
+ hasBin: true
+
'@workflow/[email protected]':
resolution: {integrity: sha512-bSBxUcs90wbhY8NlZYojoPAL7UwDuiTxr1R6eCb3ZCHCly5pTFXnkMIsM2hScPFM6ea/KImMl1KQVJc6OCPjIQ==}
peerDependencies:
@@ -8245,6 +8413,15 @@ packages:
resolution: {integrity: sha512-AzlgcsCbUMymkADOJtQm3wO9S3ltPfYOFD5033keQn9NJzIbtnZj+UdBJe7DYml/8TdbtHJW3j58SOnKhWY/5g==}
+ resolution: {integrity: sha512-kcZ6+W5QzcJ3P1Mt+83OUv/oHFqZHIx8DuxG6eZ5RGMERoLqp4BuGjhHLYGK+Kf5XVkQvqBSmAy/nGWN3qDgEA==}
+ engines: {node: '>=14'}
+ peerDependencies:
+ typescript: '>=4.9.5'
+ peerDependenciesMeta:
+ typescript:
+ optional: true
+
resolution: {integrity: sha512-gtTZxTDau1wL7Y7zifc2dd8jHSK/k6BTx/2Xp/BpdlAdnlYWFVt7qhJqgwi7637yRwRQ3qL4ZidbB4I8tA5VOg==}
engines: {node: '>=14'}
@@ -8749,10 +8926,106 @@ packages:
resolution: {integrity: sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==}
engines: {node: '>=12'}
+ resolution: {integrity: sha512-IO8C/dzEb6O3F9/twg6ZLXz164a2fhTnEWb95H23Dm4OuN+92NmEAlTrupP9VW6Jm3sO26tQlqyvyi4CsnY9GA==}
+ engines: {node: '>=12'}
+
resolution: {integrity: sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==}
engines: {node: '>=12'}
+ resolution: {integrity: sha512-kY0BSaTNYWnoDMVoyY8uxmyHjpJW1geOmBMdSSicKo9CIIWkSxMIj2rkeSR51b8KAPB7m+qysjuHme5nKP+E5Q==}
+ peerDependencies:
+ '@aws-sdk/client-rds-data': '>=3'
+ '@cloudflare/workers-types': '>=4'
+ '@electric-sql/pglite': '>=0.2.0'
+ '@libsql/client': '>=0.10.0'
+ '@libsql/client-wasm': '>=0.10.0'
+ '@neondatabase/serverless': '>=0.10.0'
+ '@op-engineering/op-sqlite': '>=2'
+ '@opentelemetry/api': ^1.4.1
+ '@planetscale/database': '>=1.13'
+ '@prisma/client': '*'
+ '@tidbcloud/serverless': '*'
+ '@types/better-sqlite3': '*'
+ '@types/pg': '*'
+ '@types/sql.js': '*'
+ '@upstash/redis': '>=1.34.7'
+ '@vercel/postgres': '>=0.8.0'
+ '@xata.io/client': '*'
+ better-sqlite3: '>=7'
+ bun-types: '*'
+ expo-sqlite: '>=14.0.0'
+ gel: '>=2'
+ knex: '*'
+ kysely: '*'
+ mysql2: '>=2'
+ pg: '>=8'
+ postgres: '>=3'
+ prisma: '*'
+ sql.js: '>=1'
+ sqlite3: '>=5'
+ peerDependenciesMeta:
+ '@aws-sdk/client-rds-data':
+ optional: true
+ '@cloudflare/workers-types':
+ optional: true
+ '@electric-sql/pglite':
+ optional: true
+ '@libsql/client':
+ optional: true
+ '@libsql/client-wasm':
+ optional: true
+ '@neondatabase/serverless':
+ optional: true
+ '@op-engineering/op-sqlite':
+ optional: true
+ '@opentelemetry/api':
+ optional: true
+ '@planetscale/database':
+ optional: true
+ '@prisma/client':
+ optional: true
+ '@tidbcloud/serverless':
+ optional: true
+ '@types/better-sqlite3':
+ optional: true
+ '@types/pg':
+ optional: true
+ '@types/sql.js':
+ optional: true
+ '@upstash/redis':
+ optional: true
+ '@vercel/postgres':
+ optional: true
+ '@xata.io/client':
+ optional: true
+ better-sqlite3:
+ optional: true
+ bun-types:
+ optional: true
+ expo-sqlite:
+ optional: true
+ gel:
+ optional: true
+ knex:
+ optional: true
+ kysely:
+ optional: true
+ mysql2:
+ optional: true
+ pg:
+ optional: true
+ postgres:
+ optional: true
+ prisma:
+ optional: true
+ sql.js:
+ optional: true
+ sqlite3:
+ optional: true
+
resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==}
engines: {node: '>= 0.4'}
@@ -9790,6 +10063,15 @@ packages:
resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==}
+ resolution: {integrity: sha512-uMdF9Rt8/NwT1wVXNleYgM5ro2hHDodHiKA3efJhgdU8iP+r/hksnghOHreMva0sF5tV73f4TpiELPUR0g7O9w==}
+ engines: {node: '>=16'}
+
+ resolution: {integrity: sha512-e7gGYDmGqzju2l83MpzX8vNG/lOtVJiSzI3eZpAFubSxh/cxs7sRrRGBGjzBP1kNG0H+c95etPpNRNlH65PYhw==}
+ engines: {node: '>=14.0.0'}
+ hasBin: true
+
resolution: {integrity: sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q==}
engines: {node: '>=6.0'}
@@ -10113,6 +10395,10 @@ packages:
resolution: {integrity: sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==}
engines: {node: '>=12'}
+ resolution: {integrity: sha512-6xwYfHbajpoF0xLW+iwLkhwgvLoZDfjYfoFNu8ftMoXINzwuymNLd9u/KmwtdT2GbR+/Cz66otEGEVVUHX9QLQ==}
+ engines: {node: '>=10.13.0'}
+
resolution: {integrity: sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==}
@@ -11999,6 +12285,40 @@ packages:
resolution: {integrity: sha512-LjgdTytVFXeUgtHZr9WYViYSM/g8MkcTPYDlPa3cDqMirHjKiSZPYd6DoL7pK8AJQr+uWkQvCjHNdiMqsrJs+g==}
+ resolution: {integrity: sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==}
+
+ resolution: {integrity: sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg==}
+
+ resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==}
+ engines: {node: '>=4.0.0'}
+
+ resolution: {integrity: sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==}
+ peerDependencies:
+ pg: '>=8.0'
+
+ resolution: {integrity: sha512-cq9sECI5s0+uPUXjbz8ioyPJni6RzsRib0US67i5IoTZKw8fNeYlVE7u8F4dG7vEJJtc5wdD1K189lCCUwqWTQ==}
+
+ resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==}
+ engines: {node: '>=4'}
+
+ resolution: {integrity: sha512-ldhMxz2r8fl/6QkXnBD3CR9/xg694oT6DZQ2s6c/RI28OjtSOpxnPrUCGOBJ46RCUxcWdx3p6kw/xnDHjKvaRA==}
+ engines: {node: '>= 16.0.0'}
+ peerDependencies:
+ pg-native: '>=3.0.1'
+ peerDependenciesMeta:
+ pg-native:
+ optional: true
+
+ resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==}
+
resolution: {integrity: sha512-QTeTClFj700SPwdgfVckQ8uO/dquPO5MdxHpSLCOtKpJbLrpUrFnEVhoCuQKXMSWXNIMlnwPsKQ7fqfXCX6BJg==}
engines: {node: 24.x}
@@ -12266,6 +12586,22 @@ packages:
resolution: {integrity: sha512-lW616l85ucIQL+FocMmL7pQFPqBmwejrCMg+iPxyImlrANNJG9NHq/RkyCZopDhd8C3LA03PHRJDjkbGu8vvug==}
engines: {node: ^10 || ^12 || >=14}
+ resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==}
+ engines: {node: '>=4'}
+
+ resolution: {integrity: sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==}
+ engines: {node: '>=0.10.0'}
+
+ resolution: {integrity: sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==}
+ engines: {node: '>=0.10.0'}
+
+ resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==}
+ engines: {node: '>=0.10.0'}
+
resolution: {integrity: sha512-dM0jVuXJPsDN6DvRpea484tCUaMiXWjuCn++HGTqUWzGDjv5tZkEZldAJ/UMlqRYGFrD/etByo4/xOuC/snX2A==}
engines: {node: '>=20'}
@@ -13716,6 +14052,9 @@ packages:
resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==}
+ resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==}
+
resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==}
@@ -14513,6 +14852,10 @@ packages:
resolution: {integrity: sha512-k8KO9hrMyNk6tUWqUfkTEZbezRRpONVOzUTnc97VnCvyj6Tf9lyUR9EDAIeiVLv56jsMcoXEwjW8Kv5yPY52lw==}
engines: {node: '>=16.0.0'}
+ resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==}
+ engines: {node: '>=0.4'}
+
resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==}
engines: {node: '>=10'}
@@ -16115,6 +16458,8 @@ snapshots:
transitivePeerDependencies:
- supports-color
+ '@graphile/[email protected]': {}
+
dependencies:
'@grpc/proto-loader': 0.8.1
@@ -16931,7 +17276,7 @@ snapshots:
transitivePeerDependencies:
- magicast
- '@nuxt/[email protected](5af45ff6ce88146737a84f10936700d8)':
dependencies:
'@nuxt/devalue': 2.0.2
'@nuxt/kit': 4.4.6([email protected])
@@ -16948,8 +17293,8 @@ snapshots:
klona: 2.0.6
mocked-exports: 0.1.1
+ nuxt: 4.4.6(41ce06a5c11c989752f5b4fadc9496f0)
nypm: 0.6.8
ohash: 2.0.11
pathe: 2.0.3
@@ -16957,7 +17302,7 @@ snapshots:
std-env: 4.2.0
ufo: 1.6.4
unctx: 2.5.0
vue: 3.5.35([email protected])
vue-bundle-renderer: 2.3.1
vue-devtools-stub: 0.1.0
@@ -17010,7 +17355,7 @@ snapshots:
- xml2js
optional: true
- '@nuxt/[email protected](d7661b23bdde68362333e80e2569dc53)':
dependencies:
'@nuxt/devalue': 2.0.2
'@nuxt/kit': 4.4.6([email protected])
@@ -17027,8 +17372,8 @@ snapshots:
klona: 2.0.6
mocked-exports: 0.1.1
+ nuxt: 4.4.6(f32df07d5233da3a29c17b72e1467465)
nypm: 0.6.8
ohash: 2.0.11
pathe: 2.0.3
@@ -17036,7 +17381,7 @@ snapshots:
std-env: 4.2.0
ufo: 1.6.4
unctx: 2.5.0
vue: 3.5.35([email protected])
vue-bundle-renderer: 2.3.1
vue-devtools-stub: 0.1.0
@@ -17105,7 +17450,7 @@ snapshots:
rc9: 3.0.1
std-env: 4.2.0
- '@nuxt/[email protected](65326fc791d4735547fd3a6298d65608)':
dependencies:
'@nuxt/kit': 4.4.6([email protected])
'@rollup/plugin-replace': 6.0.3([email protected])
@@ -17123,7 +17468,7 @@ snapshots:
magic-string: 0.30.21
mlly: 1.8.2
mocked-exports: 0.1.1
+ nuxt: 4.4.6(41ce06a5c11c989752f5b4fadc9496f0)
nypm: 0.6.8
pathe: 2.0.3
pkg-types: 2.3.1
@@ -17165,8 +17510,9 @@ snapshots:
- vti
- vue-tsc
- yaml
+ optional: true
- '@nuxt/[email protected](bd1799bb420f4f18d539b3aab18f78ee)':
dependencies:
'@nuxt/kit': 4.4.6([email protected])
'@rollup/plugin-replace': 6.0.3([email protected])
@@ -17184,7 +17530,7 @@ snapshots:
magic-string: 0.30.21
mlly: 1.8.2
mocked-exports: 0.1.1
+ nuxt: 4.4.6(f32df07d5233da3a29c17b72e1467465)
nypm: 0.6.8
pathe: 2.0.3
pkg-types: 2.3.1
@@ -17226,7 +17572,6 @@ snapshots:
- vti
- vue-tsc
- yaml
- optional: true
'@octokit/[email protected]':
dependencies:
@@ -19923,6 +20268,10 @@ snapshots:
dependencies:
'@types/unist': 3.0.3
+ '@types/[email protected]':
+ dependencies:
+ '@types/node': 25.9.1
+
'@types/[email protected]': {}
@@ -19962,10 +20311,20 @@ snapshots:
dependencies:
undici-types: 5.26.5
+ '@types/[email protected]':
+ dependencies:
+ undici-types: 6.21.0
+
dependencies:
undici-types: 7.24.6
+ '@types/[email protected]':
+ dependencies:
+ '@types/node': 25.9.1
+ pg-protocol: 1.15.0
+ pg-types: 2.2.0
+
dependencies:
'@types/node': 25.9.1
@@ -19986,6 +20345,8 @@ snapshots:
'@types/[email protected]': {}
+ '@types/[email protected]': {}
+
'@types/[email protected]': {}
'@types/[email protected]': {}
@@ -20282,11 +20643,11 @@ snapshots:
optionalDependencies:
- '@vercel/[email protected](2e1d1ea39f306c545b71c35841ac134e)':
optionalDependencies:
+ nuxt: 4.4.6(41ce06a5c11c989752f5b4fadc9496f0)
react: 19.2.6
svelte: 5.56.1(@typescript-eslint/[email protected])
vue: 3.5.35([email protected])
@@ -20755,11 +21116,11 @@ snapshots:
dependencies:
zod: 4.4.3
- '@vercel/[email protected](2e1d1ea39f306c545b71c35841ac134e)':
optionalDependencies:
+ nuxt: 4.4.6(41ce06a5c11c989752f5b4fadc9496f0)
react: 19.2.6
svelte: 5.56.1(@typescript-eslint/[email protected])
vue: 3.5.35([email protected])
@@ -21062,6 +21423,11 @@ snapshots:
- supports-color
- ws
+ '@workflow/[email protected]':
+ dependencies:
+ '@workflow/utils': 5.0.0-beta.7
+ ms: 2.1.3
+
'@workflow/[email protected]':
dependencies:
'@workflow/utils': 5.0.0-beta.8
@@ -21073,10 +21439,27 @@ snapshots:
'@workflow/[email protected]': {}
+ '@workflow/[email protected]':
+ dependencies:
+ ms: 2.1.3
+
'@workflow/[email protected]':
dependencies:
ms: 2.1.3
+ '@workflow/[email protected](@opentelemetry/[email protected])':
+ dependencies:
+ '@vercel/queue': 0.4.0(@opentelemetry/[email protected])
+ '@workflow/errors': 5.0.0-beta.13
+ '@workflow/utils': 5.0.0-beta.7
+ '@workflow/world': 5.0.0-beta.23
+ async-sema: 3.1.1
+ ulid: 3.0.2
+ undici: 7.28.0
+ zod: 4.3.6
+ optionalDependencies:
+ '@opentelemetry/api': 1.9.1
+
'@workflow/[email protected](@opentelemetry/[email protected])':
dependencies:
'@vercel/queue': 0.4.0(@opentelemetry/[email protected])
@@ -21090,6 +21473,53 @@ snapshots:
optionalDependencies:
'@opentelemetry/api': 1.9.1
+ dependencies:
+ '@vercel/queue': 0.4.0(@opentelemetry/[email protected])
+ '@workflow/errors': 5.0.0-beta.13
+ '@workflow/utils': 5.0.0-beta.7
+ '@workflow/world': 5.0.0-beta.23
+ '@workflow/world-local': 5.0.0-beta.31(@opentelemetry/[email protected])
+ cbor-x: 1.6.0
+ dotenv: 17.3.1
+ drizzle-orm: 0.45.2(@opentelemetry/[email protected])(@types/[email protected])(@upstash/[email protected])([email protected])([email protected])
+ graphile-worker: 0.16.6([email protected])([email protected])
+ pg: 8.20.0
+ ulid: 3.0.2
+ zod: 4.3.6
+ transitivePeerDependencies:
+ - '@aws-sdk/client-rds-data'
+ - '@cloudflare/workers-types'
+ - '@electric-sql/pglite'
+ - '@libsql/client'
+ - '@libsql/client-wasm'
+ - '@neondatabase/serverless'
+ - '@op-engineering/op-sqlite'
+ - '@opentelemetry/api'
+ - '@planetscale/database'
+ - '@prisma/client'
+ - '@tidbcloud/serverless'
+ - '@types/better-sqlite3'
+ - '@types/pg'
+ - '@types/sql.js'
+ - '@upstash/redis'
+ - '@vercel/postgres'
+ - '@xata.io/client'
+ - better-sqlite3
+ - bun-types
+ - expo-sqlite
+ - gel
+ - knex
+ - kysely
+ - mysql2
+ - pg-native
+ - postgres
+ - prisma
+ - sql.js
+ - sqlite3
+ - supports-color
+ - typescript
+
'@workflow/[email protected](@opentelemetry/[email protected])':
dependencies:
'@vercel/oidc': 3.2.0
@@ -22118,6 +22548,15 @@ snapshots:
dependencies:
layout-base: 2.0.1
+ dependencies:
+ import-fresh: 3.3.1
+ js-yaml: 4.1.1
+ parse-json: 5.2.0
+ path-type: 4.0.0
+ optionalDependencies:
+ typescript: 7.0.2
+
dependencies:
env-paths: 2.2.1
@@ -22468,7 +22907,9 @@ snapshots:
+ optionalDependencies:
+ drizzle-orm: 0.45.2(@opentelemetry/[email protected])(@types/[email protected])(@upstash/[email protected])([email protected])([email protected])
@@ -22636,8 +23077,18 @@ snapshots:
+
+ optionalDependencies:
+ '@opentelemetry/api': 1.9.1
+ '@types/pg': 8.20.0
+ '@upstash/redis': 1.38.0
+ pg: 8.20.0
+ sql.js: 1.14.1
+
dependencies:
call-bind-apply-helpers: 1.0.2
@@ -24072,6 +24523,36 @@ snapshots:
+ dependencies:
+ '@types/interpret': 1.1.4
+ '@types/node': 22.20.1
+ '@types/semver': 7.7.1
+ chalk: 4.1.2
+ debug: 4.4.3([email protected])
+ interpret: 3.1.1
+ semver: 7.8.4
+ tslib: 2.8.1
+ yargs: 17.7.3
+ transitivePeerDependencies:
+ - supports-color
+
+ dependencies:
+ '@graphile/logger': 0.2.0
+ '@types/debug': 4.1.13
+ '@types/pg': 8.20.0
+ cosmiconfig: 8.3.6([email protected])
+ graphile-config: 0.0.1-beta.18([email protected])
+ json5: 2.2.3
+ pg: 8.20.0
+ tslib: 2.8.1
+ yargs: 17.7.3
+ transitivePeerDependencies:
+ - pg-native
+ - supports-color
+ - typescript
+
dependencies:
js-yaml: 3.14.2
@@ -24547,6 +25028,8 @@ snapshots:
+
dependencies:
loose-envify: 1.4.0
@@ -26194,11 +26677,11 @@ snapshots:
abort-controller-x: 0.5.0
nice-grpc-common: 2.0.3
dependencies:
consola: 3.4.2
crossws: 0.4.10([email protected])
- db0: 0.3.4
env-runner: 0.1.16
hookable: 6.1.1
@@ -26209,7 +26692,7 @@ snapshots:
rolldown: 1.1.5
srvx: 0.11.22
unenv: 2.0.0-rc.24
optionalDependencies:
dotenv: 17.4.2
giget: 3.3.0
@@ -26248,7 +26731,7 @@ snapshots:
- uploadthing
- wrangler
dependencies:
'@cloudflare/kv-asset-handler': 0.4.2
'@rollup/plugin-alias': 6.0.0([email protected])
@@ -26269,7 +26752,7 @@ snapshots:
cookie-es: 2.0.1
croner: 10.0.1
crossws: 0.3.5
- db0: 0.3.4
defu: 6.1.7
destr: 2.0.5
dot-prop: 10.1.0
@@ -26315,7 +26798,7 @@ snapshots:
unenv: 2.0.0-rc.24
unplugin-utils: 0.3.2
untyped: 2.0.0
unwasm: 0.5.3
youch: 4.1.1
@@ -26360,7 +26843,7 @@ snapshots:
- webpack
optional: true
dependencies:
'@cloudflare/kv-asset-handler': 0.4.2
'@rollup/plugin-alias': 6.0.0([email protected])
@@ -26381,7 +26864,7 @@ snapshots:
cookie-es: 2.0.1
croner: 10.0.1
crossws: 0.3.5
- db0: 0.3.4
defu: 6.1.7
destr: 2.0.5
dot-prop: 10.1.0
@@ -26427,7 +26910,7 @@ snapshots:
unenv: 2.0.0-rc.24
unplugin-utils: 0.3.2
untyped: 2.0.0
unwasm: 0.5.3
youch: 4.1.1
@@ -26565,16 +27048,16 @@ snapshots:
+ [email protected](41ce06a5c11c989752f5b4fadc9496f0):
dependencies:
'@dxup/nuxt': 0.4.1([email protected])([email protected])
'@nuxt/kit': 4.4.6([email protected])
- '@nuxt/nitro-server': 4.4.6(5af45ff6ce88146737a84f10936700d8)
'@nuxt/schema': 4.4.6
'@nuxt/telemetry': 2.8.0(@nuxt/[email protected]([email protected]))
- '@nuxt/vite-builder': 4.4.6(bd1799bb420f4f18d539b3aab18f78ee)
'@unhead/vue': 2.1.15([email protected]([email protected]))
'@vue/shared': 3.5.39
chokidar: 5.0.0
@@ -26702,16 +27185,16 @@ snapshots:
- yaml
optional: true
+ [email protected](f32df07d5233da3a29c17b72e1467465):
dependencies:
'@dxup/nuxt': 0.4.1([email protected])([email protected])
'@nuxt/kit': 4.4.6([email protected])
- '@nuxt/nitro-server': 4.4.6(d7661b23bdde68362333e80e2569dc53)
'@nuxt/schema': 4.4.6
'@nuxt/telemetry': 2.8.0(@nuxt/[email protected]([email protected]))
- '@nuxt/vite-builder': 4.4.6(65326fc791d4735547fd3a6298d65608)
'@unhead/vue': 2.1.15([email protected]([email protected]))
'@vue/shared': 3.5.39
chokidar: 5.0.0
@@ -27413,6 +27896,41 @@ snapshots:
+ optional: true
+
+
+
+ dependencies:
+ pg: 8.20.0
+
+
+ dependencies:
+ pg-int8: 1.0.1
+ postgres-array: 2.0.0
+ postgres-bytea: 1.0.1
+ postgres-date: 1.0.7
+ postgres-interval: 1.2.0
+
+ dependencies:
+ pg-connection-string: 2.14.0
+ pg-pool: 3.14.0([email protected])
+ pg-protocol: 1.15.0
+ pg-types: 2.2.0
+ pgpass: 1.0.5
+ optionalDependencies:
+ pg-cloudflare: 1.4.0
+
+ dependencies:
+ split2: 4.2.0
+
optionalDependencies:
react: 19.2.6
@@ -27684,6 +28202,16 @@ snapshots:
picocolors: 1.1.1
source-map-js: 1.2.1
+
+
+
+ dependencies:
+ xtend: 4.0.2
+
@@ -29624,6 +30152,8 @@ snapshots:
+
@@ -29833,7 +30363,7 @@ snapshots:
'@unrs/resolver-binding-win32-ia32-msvc': 1.12.2
'@unrs/resolver-binding-win32-x64-msvc': 1.12.2
dependencies:
anymatch: 3.1.3
chokidar: 5.0.0
@@ -29848,17 +30378,17 @@ snapshots:
'@vercel/blob': 2.4.0
'@vercel/functions': 3.7.5(@aws-sdk/[email protected])([email protected]([email protected]))
aws4fetch: 1.0.20
- db0: 0.3.4
ioredis: 5.11.1([email protected])
optionalDependencies:
'@upstash/redis': 1.38.0
'@vercel/blob': 2.4.0
'@vercel/functions': 3.7.5(@aws-sdk/[email protected])([email protected]([email protected]))
aws4fetch: 1.0.20
chokidar: 5.0.0
- db0: 0.3.4
ioredis: 5.11.1([email protected])
lru-cache: 11.5.2
ofetch: 2.0.0-alpha.3
@@ -30489,6 +31019,8 @@ snapshots:
+
diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml
index 302fa1eb2..707282ee8 100644
--- a/pnpm-workspace.yaml
+++ b/pnpm-workspace.yaml
@@ -39,6 +39,7 @@ catalog:
"@vercel/connect": "0.4.2"
"@vercel/otel": "2.1.3"
"@vercel/sandbox": "2.8.0"
+ "@workflow/world-postgres": "5.0.0-beta.29"
ai: "^7.0.38"
marked: "17.0.6"
next: "16.3.0-preview.6"