Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

fix(agents): eval tests no longer fail at random when the model catalog is slow (#16302)

Hazem Adel333ff69a8cac
diff --git a/package.json b/package.json
index 9e4a98e60c1..f4afb701024 100644
--- a/package.json
+++ b/package.json
@@ -36,7 +36,7 @@
"test-unit": "turbo run test $(npx tsx tools/scripts/test-filters.ts) && turbo run test-unit --filter=api",
"test-api": "turbo run check-migrations test-ce test-ee test-cloud --filter=api --concurrency=1",
"agent-evals": "set -a && . ./.env.dev && set +a && tsx --tsconfig packages/server/worker/test/lib/agent-eval/cli/tsconfig.json packages/server/worker/test/lib/agent-eval/cli/index.ts",
- "agent-evals:ci": "if [ -f ./.env.dev ]; then set -a && . ./.env.dev && set +a; fi && vitest run --dir packages/server/worker/test/lib/agent-eval",
+ "agent-evals:ci": "if [ -f ./.env.dev ]; then set -a && . ./.env.dev && set +a; fi && vitest run --config packages/server/worker/vitest.config.ts packages/server/worker/test/lib/agent-eval",
"agent-evals:live": "set -a && . ./.env.dev && set +a && tsx --tsconfig packages/server/worker/test/lib/agent-eval/cli/tsconfig.json packages/server/worker/test/lib/agent-eval/live/run.ts",
"cli": "npx ts-node -r tsconfig-paths/register --project packages/cli/tsconfig.json packages/cli/src/index.ts",
"create-piece": "npx ts-node -r tsconfig-paths/register --project packages/cli/tsconfig.json packages/cli/src/index.ts pieces create",
diff --git a/packages/server/worker/vitest.config.ts b/packages/server/worker/vitest.config.ts
index 78868510231..540c173fb19 100644
--- a/packages/server/worker/vitest.config.ts
+++ b/packages/server/worker/vitest.config.ts
@@ -8,6 +8,7 @@ export default defineConfig({
pool: 'forks',
include: [path.resolve(__dirname, 'test/**/*.test.ts')],
exclude: [path.resolve(__dirname, 'test/e2e/**')],
+ setupFiles: [path.resolve(__dirname, 'vitest.setup.ts')],
},
resolve: {
alias: {
diff --git a/packages/server/worker/vitest.setup.ts b/packages/server/worker/vitest.setup.ts
new file mode 100644
index 00000000000..6cc46d87c5f
--- /dev/null
+++ b/packages/server/worker/vitest.setup.ts
@@ -0,0 +1 @@
+process.env.AP_MODEL_CATALOG_URL = 'http://model-catalog.invalid/ai/model-catalog.json'