Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): deflake workflow and fixture tests (#3165)

Chad Hietalad630b218af24
diff --git a/.changeset/quiet-workflow-reads.md b/.changeset/quiet-workflow-reads.md
new file mode 100644
index 000000000..b6a6497d8
--- /dev/null
+++ b/.changeset/quiet-workflow-reads.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Retry transient Windows file-sharing errors when reading local workflow state while still surfacing persistent access errors and malformed state. Preserve pending tasks that fit the compaction budget and keep explicitly configured `mockModel()` responders during step-scoped dynamic model selection when authored-model mocking is enabled.
diff --git a/e2e/fixtures/agent-channels/agent/channels/cross-version.ts b/e2e/fixtures/agent-channels/agent/channels/cross-version.ts
index 7b32c2b7e..58f71799b 100644
--- a/e2e/fixtures/agent-channels/agent/channels/cross-version.ts
+++ b/e2e/fixtures/agent-channels/agent/channels/cross-version.ts
@@ -1,4 +1,7 @@
import { defineChannel, POST, type Session } from "eve/channels";
+import evePackage from "eve/package.json" with { type: "json" };
+
+import { checkDeploymentRevision } from "../lib/deployment-revision";
const smokeTestAuth = {
attributes: { source: "smoke-test" },
@@ -29,6 +32,8 @@ type CrossVersionRouteContext = {
export default defineChannel({
routes: [
POST("/cross-version-webhook", async (req, ctx) => {
+ const pendingDeployment = checkDeploymentRevision(req);
+ if (pendingDeployment !== undefined) return pendingDeployment;
const body = (await req.json().catch(() => ({}))) as {
message?: string;
sessionRef?: string;
@@ -53,7 +58,7 @@ export default defineChannel({
throw new Error("Expected the route context to expose from() or send().");
}
- return Response.json({ ok: true, sessionId: session.id });
+ return Response.json({ ok: true, sessionId: session.id, eveVersion: evePackage.version });
}),
],
});
diff --git a/e2e/fixtures/agent-channels/agent/lib/deployment-revision.ts b/e2e/fixtures/agent-channels/agent/lib/deployment-revision.ts
new file mode 100644
index 000000000..723ba6ba3
--- /dev/null
+++ b/e2e/fixtures/agent-channels/agent/lib/deployment-revision.ts
@@ -0,0 +1,17 @@
+export const DEPLOYMENT_REVISION_HEADER = "x-eve-e2e-deployment-revision";
+export const DEPLOYMENT_PENDING_CODE = "e2e_deployment_pending";
+
+export function checkDeploymentRevision(
+ request: Request,
+ revision = process.env.EVE_E2E_DEPLOYMENT_REVISION,
+): Response | undefined {
+ const expected = request.headers.get(DEPLOYMENT_REVISION_HEADER);
+ if (expected === null || expected === revision) return undefined;
+ return Response.json(
+ {
+ code: DEPLOYMENT_PENDING_CODE,
+ error: "The requested deployment is not serving this request yet.",
+ },
+ { status: 409 },
+ );
+}
diff --git a/e2e/fixtures/agent-channels/evals/custom-channels/cross-version-session-inbox.eval.ts b/e2e/fixtures/agent-channels/evals/custom-channels/cross-version-session-inbox.eval.ts
index 1a4bddd89..58de02246 100644
--- a/e2e/fixtures/agent-channels/evals/custom-channels/cross-version-session-inbox.eval.ts
+++ b/e2e/fixtures/agent-channels/evals/custom-channels/cross-version-session-inbox.eval.ts
@@ -7,7 +7,10 @@ import { defineEval } from "eve/evals";
import type { EveEvalContext } from "eve/evals";
import { satisfies } from "eve/evals/expect";
-import { postChannel } from "./shared";
+import {
+ DEPLOYMENT_PENDING_CODE,
+ DEPLOYMENT_REVISION_HEADER,
+} from "../../agent/lib/deployment-revision";
const ALIAS_ENV = "EVE_E2E_REDEPLOY_ALIAS";
const OLD_EVE_VERSION = "0.30.8";
@@ -26,7 +29,7 @@ const TOOL_NAME = "wait-for-cancellation";
const execFileAsync = promisify(execFile);
const EXEC_OPTIONS = { maxBuffer: 64 * 1024 * 1024 } as const;
-type MessageResponse = { ok: boolean; sessionId?: string };
+type MessageResponse = { ok: boolean; sessionId?: string; eveVersion: string };
/**
* Proves the complete mixed-version codec boundary through the real Workflow
@@ -53,6 +56,8 @@ export default defineEval({
}
const originalInstructions = await readFile(INSTRUCTIONS_PATH, "utf8");
+ const oldRevision = crypto.randomUUID();
+ const currentRevision = crypto.randomUUID();
const links = await Promise.all([
snapshotLink(FIXTURE_EVE_LINK),
snapshotLink(CONFIG_EVE_LINK),
@@ -74,18 +79,21 @@ export default defineEval({
INSTRUCTIONS_PATH,
`${originalInstructions}\nDeployment marker: ${OLD_DEPLOYMENT_MARKER}.\n`,
);
- await deployToAlias(t, alias, "[email protected]");
+ await deployToAlias(t, alias, "[email protected]", oldRevision, oldEvePackage);
await waitForAliasToServe(t, OLD_DEPLOYMENT_MARKER);
const sessionRef = crypto.randomUUID();
- const started = await postChannel<MessageResponse>(t.target, "/cross-version-webhook", {
+ const started = await postToDeployment(t, oldRevision, {
message: ACTIVE_TURN_MESSAGE,
sessionRef,
});
await t.require(
started,
satisfies(
- (value: MessageResponse) => value.ok === true && typeof value.sessionId === "string",
+ (value: MessageResponse) =>
+ value.ok === true &&
+ typeof value.sessionId === "string" &&
+ value.eveVersion === OLD_EVE_VERSION,
"[email protected] starts the durable session",
),
);
@@ -104,10 +112,10 @@ export default defineEval({
INSTRUCTIONS_PATH,
`${originalInstructions}\nDeployment marker: ${CURRENT_DEPLOYMENT_MARKER}.\n`,
);
- await deployToAlias(t, alias, `eve@${currentEveVersion}`);
+ await deployToAlias(t, alias, `eve@${currentEveVersion}`, currentRevision, currentEvePackage);
await waitForAliasToServe(t, CURRENT_DEPLOYMENT_MARKER);
- const replacement = await postChannel<MessageResponse>(t.target, "/cross-version-webhook", {
+ const replacement = await postToDeployment(t, currentRevision, {
message: `Reply with exactly ${FOLLOW_UP_TOKEN}.`,
sessionRef,
turnPolicy: "queue",
@@ -115,7 +123,10 @@ export default defineEval({
await t.require(
replacement,
satisfies(
- (value: MessageResponse) => value.ok === true && value.sessionId === sessionId,
+ (value: MessageResponse) =>
+ value.ok === true &&
+ value.sessionId === sessionId &&
+ value.eveVersion === currentEveVersion,
"the current producer targets the existing [email protected] session",
),
);
@@ -188,8 +199,15 @@ async function readPackageVersion(packagePath: string): Promise<string> {
}
/** Builds the fixture and repoints the run-scoped alias at the fresh deployment. */
-async function deployToAlias(t: EveEvalContext, alias: string, phase: string): Promise<void> {
- await execFileAsync("pnpm", ["exec", "eve", "build"], {
+async function deployToAlias(
+ t: EveEvalContext,
+ alias: string,
+ phase: string,
+ revision: string,
+ evePackagePath: string,
+): Promise<void> {
+ // pnpm's dependency synchronization can repair the intentionally replaced links.
+ await execFileAsync(process.execPath, [resolve(evePackagePath, "bin", "eve.js"), "build"], {
...EXEC_OPTIONS,
env: {
...process.env,
@@ -204,11 +222,22 @@ async function deployToAlias(t: EveEvalContext, alias: string, phase: string): P
process.env.EVE_E2E_MODEL === undefined
? []
: ["--env", `EVE_E2E_MODEL=${process.env.EVE_E2E_MODEL}`];
+ const revisionArgs = ["--env", `EVE_E2E_DEPLOYMENT_REVISION=${revision}`];
const scopeArgs =
process.env.VERCEL_ORG_ID === undefined ? [] : ["--scope", process.env.VERCEL_ORG_ID];
const deploy = await execFileAsync(
"pnpm",
- ["exec", "vc", "deploy", "--prebuilt", "--yes", "--target=preview", ...modelArgs, ...tokenArgs],
+ [
+ "exec",
+ "vc",
+ "deploy",
+ "--prebuilt",
+ "--yes",
+ "--target=preview",
+ ...modelArgs,
+ ...revisionArgs,
+ ...tokenArgs,
+ ],
EXEC_OPTIONS,
);
const deploymentUrl = deploy.stdout.trim().split("\n").at(-1)?.trim();
@@ -224,6 +253,35 @@ async function deployToAlias(t: EveEvalContext, alias: string, phase: string): P
);
}
+async function postToDeployment(
+ t: EveEvalContext,
+ revision: string,
+ body: unknown,
+): Promise<MessageResponse> {
+ const deadline = Date.now() + 120_000;
+ while (Date.now() < deadline) {
+ const response = await t.target.fetch("/cross-version-webhook", {
+ method: "POST",
+ headers: {
+ "content-type": "application/json",
+ [DEPLOYMENT_REVISION_HEADER]: revision,
+ },
+ body: JSON.stringify(body),
+ signal: t.signal,
+ });
+ const result = (await response.json()) as MessageResponse & { code?: string };
+ if (response.status === 409 && result.code === DEPLOYMENT_PENDING_CODE) {
+ await t.sleep(1_000);
+ continue;
+ }
+ if (!response.ok) {
+ throw new Error(`Cross-version message failed (${response.status}).`);
+ }
+ return result;
+ }
+ throw new Error("Timed out waiting for the cross-version message's deployment.");
+}
+
/** Waits until the alias exposes the marker from the expected deployment. */
async function waitForAliasToServe(t: EveEvalContext, marker: string): Promise<void> {
const deadline = Date.now() + 120_000;
diff --git a/e2e/fixtures/agent-channels/package.json b/e2e/fixtures/agent-channels/package.json
index 827cec9e2..ba341b10d 100644
--- a/e2e/fixtures/agent-channels/package.json
+++ b/e2e/fixtures/agent-channels/package.json
@@ -8,7 +8,8 @@
"dev": "eve dev",
"start": "eve start",
"typecheck": "eve build && tsc",
- "test:e2e": "eve eval --strict"
+ "test:e2e": "eve eval --strict",
+ "test:unit": "node --test test/*.test.mjs"
},
"dependencies": {
"@eve-e2e/config": "workspace:*",
diff --git a/e2e/fixtures/agent-channels/test/deployment-revision.test.mjs b/e2e/fixtures/agent-channels/test/deployment-revision.test.mjs
new file mode 100644
index 000000000..6b350e79f
--- /dev/null
+++ b/e2e/fixtures/agent-channels/test/deployment-revision.test.mjs
@@ -0,0 +1,35 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import {
+ checkDeploymentRevision,
+ DEPLOYMENT_PENDING_CODE,
+ DEPLOYMENT_REVISION_HEADER,
+} from "../agent/lib/deployment-revision.ts";
+
+function pinnedRequest() {
+ return new Request("https://fixture.example/cross-version-webhook", {
+ headers: { [DEPLOYMENT_REVISION_HEADER]: "requested-revision" },
+ });
+}
+
+test("rejects a stale deployment before dispatch", async () => {
+ const response = checkDeploymentRevision(pinnedRequest(), "previous-revision");
+
+ assert.equal(response.status, 409);
+ assert.equal((await response.json()).code, DEPLOYMENT_PENDING_CODE);
+});
+
+test("allows a request on its pinned deployment", () => {
+ assert.equal(checkDeploymentRevision(pinnedRequest(), "requested-revision"), undefined);
+});
+
+test("preserves requests without a deployment pin", () => {
+ assert.equal(
+ checkDeploymentRevision(
+ new Request("https://fixture.example/cross-version-webhook"),
+ "previous-revision",
+ ),
+ undefined,
+ );
+});
diff --git a/e2e/fixtures/agent-compaction-regressions/agent/agent.ts b/e2e/fixtures/agent-compaction-regressions/agent/agent.ts
index 1b56229ea..b281013da 100644
--- a/e2e/fixtures/agent-compaction-regressions/agent/agent.ts
+++ b/e2e/fixtures/agent-compaction-regressions/agent/agent.ts
@@ -15,6 +15,9 @@ import {
} from "../constants";
const TEST_CONTEXT_WINDOW_TOKENS = 32_000;
+// Scripted usage supplies pressure without starving the real summarizer's transcript.
+const TEST_COMPACTION_THRESHOLD_TOKENS = 2_048;
+const COMPACTION_PRESSURE_USAGE = { inputTokens: 4_096 };
const MAX_TOOL_CALLS = 10;
type RegressionCase =
@@ -170,12 +173,10 @@ const taskModel = mockModel({
// These are fixture markers, not compaction protocol fields. `marker` records the
// regression work tool; `SECOND_CHECKPOINT_MARKER` records the test-only tool
// whose output makes the harness cross the compaction threshold a second time.
- // Completion evidence is detected in any assistant message: compaction may
- // leave it as a summarization checkpoint or as an eviction trail line, and
- // the model must not repeat work in either case. User messages are
- // excluded because the eval instructions themselves quote the markers.
- if (assistantEvidenceContains(request.messages, marker)) {
- if (assistantEvidenceContains(request.messages, SECOND_CHECKPOINT_MARKER)) {
+ // Kept tool results, checkpoints, and eviction trails all prove completed work.
+ // User messages are excluded because the eval instructions quote the markers.
+ if (completionEvidenceContains(request.messages, marker)) {
+ if (completionEvidenceContains(request.messages, SECOND_CHECKPOINT_MARKER)) {
return `Done: ${marker}; ${SECOND_CHECKPOINT_MARKER}`;
}
@@ -186,6 +187,7 @@ const taskModel = mockModel({
checkpointAdvanceCallCounts.set(regressionCase, advanceCalls + 1);
return {
+ usage: COMPACTION_PRESSURE_USAGE,
toolCalls: [
{
id: `advance-checkpoint-${advanceCalls + 1}`,
@@ -206,6 +208,7 @@ const taskModel = mockModel({
return regressionCase === "redundant-tool-calls"
? {
+ usage: COMPACTION_PRESSURE_USAGE,
toolCalls: [
{
id: `inspect-repository-${attempt}`,
@@ -215,6 +218,7 @@ const taskModel = mockModel({
],
}
: {
+ usage: COMPACTION_PRESSURE_USAGE,
toolCalls: [
{
id: `perform-source-analysis-${attempt}`,
@@ -235,7 +239,7 @@ export default defineAgent({
compaction: {
model: e2eModel(),
modelContextWindowTokens: TEST_CONTEXT_WINDOW_TOKENS,
- thresholdPercent: 0.02,
+ thresholdPercent: TEST_COMPACTION_THRESHOLD_TOKENS / TEST_CONTEXT_WINDOW_TOKENS,
},
limits: {
maxInputTokensPerSession: 100_000,
@@ -267,9 +271,12 @@ function completionMarker(
: "SOURCE_ANALYSIS_COMPLETE";
}
-function assistantEvidenceContains(
+function completionEvidenceContains(
messages: MockModelRequest["messages"],
marker: string,
): boolean {
- return messages.some((message) => message.role === "assistant" && message.text.includes(marker));
+ return messages.some(
+ (message) =>
+ (message.role === "assistant" || message.role === "tool") && message.text.includes(marker),
+ );
}
diff --git a/e2e/fixtures/agent-compaction-regressions/evals/redundant-tool-calls.eval.ts b/e2e/fixtures/agent-compaction-regressions/evals/redundant-tool-calls.eval.ts
index 5f55a12e7..eb3c9c8a8 100644
--- a/e2e/fixtures/agent-compaction-regressions/evals/redundant-tool-calls.eval.ts
+++ b/e2e/fixtures/agent-compaction-regressions/evals/redundant-tool-calls.eval.ts
@@ -9,8 +9,10 @@ export default defineEval({
const turn = await t.send(
[
"[case: redundant-tool-calls]",
- "Call inspect-repository exactly once with scope repository.",
- "After it succeeds, report REPOSITORY_INSPECTION_COMPLETE and call no more tools.",
+ "Alice is reviewing a small catalog application and preparing a handoff for Bob.",
+ "Use inspect-repository once with scope repository to obtain her review notes.",
+ "Once the review is complete, include REPOSITORY_INSPECTION_COMPLETE in the handoff.",
+ "The completed inspection does not need to be repeated.",
].join("\n"),
);
diff --git a/e2e/fixtures/agent-compaction-regressions/evals/stale-todo-work.eval.ts b/e2e/fixtures/agent-compaction-regressions/evals/stale-todo-work.eval.ts
index 1b33e5e6f..5d02e09e4 100644
--- a/e2e/fixtures/agent-compaction-regressions/evals/stale-todo-work.eval.ts
+++ b/e2e/fixtures/agent-compaction-regressions/evals/stale-todo-work.eval.ts
@@ -9,9 +9,11 @@ export default defineEval({
const turn = await t.send(
[
"[case: stale-todo-work]",
- "Call perform-source-analysis exactly once with approach initial.",
- "The tool deliberately leaves its completed work in a pending todo.",
- "After it succeeds, report SOURCE_ANALYSIS_COMPLETE and call no more tools.",
+ "Alice and Bob are reviewing a small catalog application.",
+ "Use perform-source-analysis once with approach initial to prepare Alice's findings.",
+ "Bob updates their shared checklist separately, so it may still say pending after the analysis is complete.",
+ "Use the completed analysis for the handoff and include SOURCE_ANALYSIS_COMPLETE.",
+ "The pending checklist entry does not require repeating the completed analysis.",
].join("\n"),
);
diff --git a/e2e/fixtures/agent-subagents-hitl/agent/agent.ts b/e2e/fixtures/agent-subagents-hitl/agent/agent.ts
index 086c9b28e..637717abb 100644
--- a/e2e/fixtures/agent-subagents-hitl/agent/agent.ts
+++ b/e2e/fixtures/agent-subagents-hitl/agent/agent.ts
@@ -1,6 +1,6 @@
import { e2eAgentConfig } from "@eve-e2e/config";
import { defineAgent } from "eve";
-import type { MockModelRequest, MockModelResponse } from "eve/evals";
+import { mockModel, type MockModelRequest, type MockModelResponse } from "eve/evals";
const COLLISION_MARKER = "MIXED-PARK-COMPLETE-7K2M";
const STOCK_PRICE = "178.92";
@@ -58,6 +58,8 @@ function respond(request: MockModelRequest): MockModelResponse | string {
}
export default defineAgent({
- ...e2eAgentConfig({ mock: respond }),
- reasoning: "high",
+ ...e2eAgentConfig(),
+ // Parking coverage requires both actions in one step; children still use the matrix model.
+ model: mockModel(respond),
+ modelContextWindowTokens: 1_000_000,
});
diff --git a/e2e/fixtures/agent-subagents/agent/agent.ts b/e2e/fixtures/agent-subagents/agent/agent.ts
index 62583a4ca..5cabe7625 100644
--- a/e2e/fixtures/agent-subagents/agent/agent.ts
+++ b/e2e/fixtures/agent-subagents/agent/agent.ts
@@ -2,7 +2,7 @@ import { e2eAgentConfig } from "@eve-e2e/config";
import { defineAgent, defineDynamic } from "eve";
import { mockModel } from "eve/evals";
-import { WORKSPACE_LOOKUP_MESSAGE } from "../constants";
+import { WORKSPACE_FORWARDING_MARKER, WORKSPACE_LOOKUP_MESSAGE } from "../constants";
if (process.env.EVE_E2E_MODEL === "mock") {
process.env.EVE_MOCK_AUTHORED_MODELS = "1";
@@ -10,6 +10,7 @@ if (process.env.EVE_E2E_MODEL === "mock") {
const base = e2eAgentConfig();
const { model, modelContextWindowTokens, ...agentConfig } = base;
+const defaultModel = typeof model === "string" ? model : `${model.provider}/${model.modelId}`;
const workspaceReader = mockModel({
modelId: "principal-forwarding-workspace-reader",
respond: ({ messages }) => {
@@ -20,24 +21,70 @@ const workspaceReader = mockModel({
return { toolCalls: [{ name: "read-workspace-label", input: {} }] };
},
});
+const workspaceDispatcher = mockModel({
+ modelId: "principal-forwarding-workspace-dispatcher",
+ respond(request) {
+ let requestIndex = -1;
+ for (const [index, message] of request.messages.entries()) {
+ if (message.role === "user" && message.text.includes(WORKSPACE_FORWARDING_MARKER)) {
+ requestIndex = index;
+ }
+ }
+ if (
+ requestIndex < 0 ||
+ request.messages.slice(requestIndex + 1).some((message) => message.role === "tool")
+ ) {
+ return "The workspace lookup was submitted.";
+ }
+ const previous = [...request.toolResults]
+ .reverse()
+ .find((result) => result.name === "remote-loopback")?.output;
+ const agentId =
+ previous !== null &&
+ typeof previous === "object" &&
+ "agentId" in previous &&
+ typeof previous.agentId === "string"
+ ? previous.agentId
+ : undefined;
+ const requestCount = request.messages.filter(
+ (message) => message.role === "user" && message.text.includes(WORKSPACE_FORWARDING_MARKER),
+ ).length;
+ if (requestCount > 1 && agentId === undefined) {
+ throw new Error("Workspace continuation has no existing remote agent receipt.");
+ }
+ return {
+ toolCalls: [
+ {
+ id: `workspace-lookup-${requestCount}`,
+ name: "remote-loopback",
+ input: { agentId, message: WORKSPACE_LOOKUP_MESSAGE },
+ },
+ ],
+ };
+ },
+});
export default defineAgent({
...agentConfig,
model: defineDynamic({
events: {
"step.started": (_event, ctx) => {
- // Authorization must reach the real tool even if a provider declines the lookup.
- const isWorkspaceReader = ctx.messages.some((message) => {
- if (message.role !== "user") return false;
- const text =
+ const messages = ctx.messages.flatMap((message) => {
+ if (message.role !== "user") return [];
+ return [
typeof message.content === "string"
? message.content
- : message.content.map((part) => (part.type === "text" ? part.text : "")).join("");
- return text === WORKSPACE_LOOKUP_MESSAGE;
+ : message.content.map((part) => (part.type === "text" ? part.text : "")).join(""),
+ ];
});
- return isWorkspaceReader
- ? { model: workspaceReader, modelContextWindowTokens: 1_000_000 }
- : { model, modelContextWindowTokens };
+ // Both models must reach the real authorization boundary, including denied lookups.
+ if (messages.includes(WORKSPACE_LOOKUP_MESSAGE)) {
+ return { model: workspaceReader, modelContextWindowTokens: 1_000_000 };
+ }
+ if (messages.some((message) => message.includes(WORKSPACE_FORWARDING_MARKER))) {
+ return { model: workspaceDispatcher, modelContextWindowTokens: 1_000_000 };
+ }
+ return { model: defaultModel, modelContextWindowTokens };
},
},
}),
diff --git a/e2e/fixtures/agent-subagents/constants.ts b/e2e/fixtures/agent-subagents/constants.ts
index b7004745b..29b7cbf0a 100644
--- a/e2e/fixtures/agent-subagents/constants.ts
+++ b/e2e/fixtures/agent-subagents/constants.ts
@@ -3,3 +3,4 @@ export const WORKSPACE_LOOKUP_MESSAGE = [
"Make a fresh lookup: earlier answers in this session may belong to a different caller.",
"Report the returned name, or explain if access is denied.",
].join(" ");
+export const WORKSPACE_FORWARDING_MARKER = "Shared workspace membership check.";
diff --git a/e2e/fixtures/agent-subagents/evals/agent-messaging-remote.eval.ts b/e2e/fixtures/agent-subagents/evals/agent-messaging-remote.eval.ts
index 0d336b5f1..3d8fd3d57 100644
--- a/e2e/fixtures/agent-subagents/evals/agent-messaging-remote.eval.ts
+++ b/e2e/fixtures/agent-subagents/evals/agent-messaging-remote.eval.ts
@@ -1,6 +1,6 @@
import { defineEval } from "eve/evals";
-const MEMORABLE_FACT = "The tide station passphrase is HARBOR-LUMEN-4482.";
+const MEMORABLE_FACT = "The tide station passphrase is HARBOR-LUMEN-4482";
/**
* Cross-turn continuation of a remote child over a real HTTP hop: turn one
diff --git a/e2e/fixtures/agent-subagents/evals/dynamic-workflow.eval.ts b/e2e/fixtures/agent-subagents/evals/dynamic-workflow.eval.ts
index afad986e0..0321c386f 100644
--- a/e2e/fixtures/agent-subagents/evals/dynamic-workflow.eval.ts
+++ b/e2e/fixtures/agent-subagents/evals/dynamic-workflow.eval.ts
@@ -27,15 +27,25 @@ export default defineEval({
t.succeeded();
t.calledTool("Workflow", { input: isFanOutProgram, count: 1 });
- // Parallel fan-out: both echo-marker subagents are called before either
- // completes. subagent.completed can be emitted more than once if the
- // resolution step re-runs before commit (benign duplicate — client state is
- // built from callId-deduped action.result), so tolerate >=2 here; the exact
- // distinct count is pinned by calledSubagent below.
- turn.eventOrder([
- { type: "subagent.called", data: { name: "echo-marker" }, count: 2 },
- { type: "subagent.completed", data: { subagentName: "echo-marker" }, count: (n) => n >= 2 },
- ]);
+ // Workflow delivery can replay either event; count logical calls, not deliveries.
+ turn.eventsSatisfy("both distinct children start before either completes", (events) => {
+ const called = new Map<string, number>();
+ const completed = new Map<string, number>();
+ for (const [index, event] of events.entries()) {
+ if (event.type === "subagent.called" && event.data.name === "echo-marker") {
+ if (!called.has(event.data.callId)) called.set(event.data.callId, index);
+ }
+ if (event.type === "subagent.completed" && event.data.subagentName === "echo-marker") {
+ if (!completed.has(event.data.callId)) completed.set(event.data.callId, index);
+ }
+ }
+ return (
+ called.size === 2 &&
+ completed.size === 2 &&
+ [...completed.keys()].every((callId) => called.has(callId)) &&
+ Math.max(...called.values()) < Math.min(...completed.values())
+ );
+ });
t.calledSubagent("echo-marker", {
output: /SUBAGENT_TOKEN=echo-marker-9F2X/,
count: 2,
diff --git a/e2e/fixtures/agent-subagents/evals/remote-principal-forwarding.eval.ts b/e2e/fixtures/agent-subagents/evals/remote-principal-forwarding.eval.ts
index 660652d9a..f73798cc2 100644
--- a/e2e/fixtures/agent-subagents/evals/remote-principal-forwarding.eval.ts
+++ b/e2e/fixtures/agent-subagents/evals/remote-principal-forwarding.eval.ts
@@ -7,17 +7,19 @@ import {
} from "eve/evals";
import { satisfies } from "eve/evals/expect";
-import { WORKSPACE_LOOKUP_MESSAGE } from "../constants";
+import { WORKSPACE_FORWARDING_MARKER, WORKSPACE_LOOKUP_MESSAGE } from "../constants";
const ALICE_WORKSPACE_LABEL = "Maple Studio";
const BOB_WORKSPACE_LABEL = "Cedar Workshop";
const BOB_AUTHORIZATION = "Bearer e2e-workspace-label-bob";
const OBSERVER_AUTHORIZATION = "Bearer e2e-workspace-label-observer";
const CREATE_CHILD_MESSAGE = [
+ WORKSPACE_FORWARDING_MARKER,
"Use remote-loopback with this message:",
JSON.stringify(WORKSPACE_LOOKUP_MESSAGE),
].join(" ");
const CONTINUE_CHILD_MESSAGE = [
+ WORKSPACE_FORWARDING_MARKER,
"A different user is making this request now.",
"Continue that same remote-loopback agent using its agentId with this message:",
JSON.stringify(WORKSPACE_LOOKUP_MESSAGE),
@@ -30,7 +32,7 @@ const CLARIFICATION = [
/** Three users resume one remote child; each tool call resolves only its current caller's workspace membership. */
export default defineEval({
- tags: ["principal-forwarding", "real-model"],
+ tags: ["principal-forwarding"],
description:
"A persistent remote child switches between two workspace memberships and denies a third caller with none.",
async test(t) {
diff --git a/e2e/fixtures/agent-workflow-tools/evals/mixed-agent-resolution.eval.ts b/e2e/fixtures/agent-workflow-tools/evals/mixed-agent-resolution.eval.ts
index 91907428c..8d0ca3cb0 100644
--- a/e2e/fixtures/agent-workflow-tools/evals/mixed-agent-resolution.eval.ts
+++ b/e2e/fixtures/agent-workflow-tools/evals/mixed-agent-resolution.eval.ts
@@ -1,4 +1,4 @@
-import { defineEval } from "eve/evals";
+import { defineEval, type EveEvalSession, type EveEvalTurn } from "eve/evals";
export default defineEval({
description:
@@ -16,9 +16,21 @@ export default defineEval({
const sessionId = initial.sessionId;
if (sessionId === undefined) throw new Error("Mixed workflow turn has no session id.");
- const completed = await t.target
- .watchTurn(sessionId, { startIndex: requireStreamIndex(t) })
- .result();
+ let session: Pick<EveEvalSession, "state"> = t;
+ let completed: EveEvalTurn | undefined;
+ for (let attempt = 0; attempt < 5; attempt += 1) {
+ const live = t.target.watchTurn(sessionId, { startIndex: requireStreamIndex(session) });
+ const turn = await live.result();
+ turn.expectOk();
+ if (turn.message?.includes("WORKFLOW-REPORT-ACK")) {
+ completed = turn;
+ break;
+ }
+ session = live.session;
+ }
+ if (completed === undefined) {
+ throw new Error("Background workflow result did not reach the parent after five turns.");
+ }
completed.expectOk();
completed.messageIncludes("WORKFLOW-REPORT-ACK");
completed.event("message.received", { count: 1 });
diff --git a/packages/eve/scripts/vendor-compiled/@workflow/world-local.mjs b/packages/eve/scripts/vendor-compiled/@workflow/world-local.mjs
index 6f4ffb4fd..6d8936b75 100644
--- a/packages/eve/scripts/vendor-compiled/@workflow/world-local.mjs
+++ b/packages/eve/scripts/vendor-compiled/@workflow/world-local.mjs
@@ -60,6 +60,7 @@ export default {
packageName: "@workflow/world-local",
compiledPath: "@workflow/world-local",
chunkGroup: "workflow",
+ fingerprintFiles: ["dist/fs.js"],
declaration: await loadDeclaration("workflow-world-local.d.ts"),
plugins: [workflowWorldLocalVersionPlugin],
};
diff --git a/packages/eve/src/cli/dev/tui/prompt-command-handler.test.ts b/packages/eve/src/cli/dev/tui/prompt-command-handler.test.ts
index 13517bb75..e3ba04c8e 100644
--- a/packages/eve/src/cli/dev/tui/prompt-command-handler.test.ts
+++ b/packages/eve/src/cli/dev/tui/prompt-command-handler.test.ts
@@ -6,6 +6,19 @@ import type { RemoteAuthCompletion, RemoteConnectionController } from "./remote-
import type { AgentTUIRenderer, PromptCommandHandlerContext } from "./runner.js";
import type { SetupFlowRenderer } from "./setup-flow.js";
+vi.mock("#setup/flows/model.js", () => ({
+ modelChangeRefusalForUneditableModel: vi.fn(),
+}));
+vi.mock("#compiler/model-catalog.js", () => ({
+ createCompiledRuntimeModelCatalogLoader: vi.fn(),
+}));
+vi.mock("#discover/discover-agent.js", () => ({
+ discoverAgent: vi.fn(),
+}));
+vi.mock("#source-change/static-source-change.js", () => ({
+ createStaticSourceChange: vi.fn(),
+}));
+
const APP_ROOT = "/tmp/weather-agent";
const LOCAL_TARGET = {
kind: "local",
diff --git a/packages/eve/src/cli/telemetry/preference.test.ts b/packages/eve/src/cli/telemetry/preference.test.ts
index b47bb8eeb..25f248466 100644
--- a/packages/eve/src/cli/telemetry/preference.test.ts
+++ b/packages/eve/src/cli/telemetry/preference.test.ts
@@ -19,7 +19,10 @@ import {
setEveTelemetryEnabled,
} from "#cli/telemetry/preference.js";
+const originalPlatform = process.platform;
+
afterEach(() => {
+ Object.defineProperty(process, "platform", { value: originalPlatform });
vi.unstubAllEnvs();
vi.clearAllMocks();
});
@@ -65,6 +68,7 @@ describe("eve telemetry preference", () => {
});
it("ignores a relative XDG config home", async () => {
+ Object.defineProperty(process, "platform", { value: "linux" });
vi.stubEnv("XDG_CONFIG_HOME", "relative-config");
vi.mocked(readFile).mockRejectedValue(new Error("missing"));
@@ -74,6 +78,9 @@ describe("eve telemetry preference", () => {
});
it("merges telemetry changes into the eve config atomically", async () => {
+ Object.defineProperty(process, "platform", { value: "linux" });
+ const directory = join("/eve-config", "eve");
+ const temporaryPrefix = `${join(directory, "config.json")}.`;
vi.stubEnv("XDG_CONFIG_HOME", "/eve-config");
vi.mocked(readFile)
.mockResolvedValueOnce('{"other":"preserved","telemetry":{"future":"preserved"}}')
@@ -84,37 +91,57 @@ describe("eve telemetry preference", () => {
await setEveTelemetryEnabled(false);
await markEveTelemetryNotified();
- expect(mkdir).toHaveBeenCalledWith("/eve-config/eve", { recursive: true });
+ expect(mkdir).toHaveBeenCalledWith(directory, { recursive: true });
expect(writeFile).toHaveBeenNthCalledWith(
1,
- expect.stringMatching(/^\/eve-config\/eve\/config\.json\./),
+ expect.stringContaining(temporaryPrefix),
expect.stringContaining('"other": "preserved"'),
{ mode: 0o600 },
);
expect(writeFile).toHaveBeenNthCalledWith(
2,
- expect.stringMatching(/^\/eve-config\/eve\/config\.json\./),
+ expect.stringContaining(temporaryPrefix),
expect.stringContaining('"future": "preserved"'),
{ mode: 0o600 },
);
expect(writeFile).toHaveBeenNthCalledWith(
2,
- expect.stringMatching(/^\/eve-config\/eve\/config\.json\./),
+ expect.stringContaining(temporaryPrefix),
expect.stringContaining('"enabled": false'),
{ mode: 0o600 },
);
expect(writeFile).toHaveBeenNthCalledWith(
2,
- expect.stringMatching(/^\/eve-config\/eve\/config\.json\./),
+ expect.stringContaining(temporaryPrefix),
expect.stringContaining('"noticeVersion": 1'),
{ mode: 0o600 },
);
expect(writeFile).toHaveBeenNthCalledWith(
2,
- expect.stringMatching(/^\/eve-config\/eve\/config\.json\./),
+ expect.stringContaining(temporaryPrefix),
expect.stringContaining('"notifiedAt"'),
{ mode: 0o600 },
);
expect(rename).toHaveBeenCalledTimes(2);
+ expect(rename).toHaveBeenCalledWith(
+ expect.stringContaining(temporaryPrefix),
+ join(directory, "config.json"),
+ );
+ });
+
+ it.each([
+ ["darwin", join(homedir(), "Library", "Preferences", "eve")],
+ ["win32", join(homedir(), "AppData", "Roaming", "eve")],
+ ["linux", join(homedir(), ".config", "eve")],
+ ])("uses the native config directory on %s", async (platform, directory) => {
+ Object.defineProperty(process, "platform", { value: platform });
+ vi.stubEnv("XDG_CONFIG_HOME", "");
+ vi.stubEnv("APPDATA", "");
+ vi.mocked(readFile).mockRejectedValue(new Error("missing"));
+
+ await setEveTelemetryEnabled(false);
+
+ expect(mkdir).toHaveBeenCalledWith(directory, { recursive: true });
+ expect(rename).toHaveBeenCalledWith(expect.any(String), join(directory, "config.json"));
});
});
diff --git a/packages/eve/src/context/dynamic-model-lifecycle.test.ts b/packages/eve/src/context/dynamic-model-lifecycle.test.ts
index 95dd1060f..3e908f02f 100644
--- a/packages/eve/src/context/dynamic-model-lifecycle.test.ts
+++ b/packages/eve/src/context/dynamic-model-lifecycle.test.ts
@@ -1,4 +1,4 @@
-import type { LanguageModel } from "ai";
+import { generateText, type LanguageModel } from "ai";
import { afterEach, describe, expect, it, vi } from "vitest";
import { ROOT_COMPILED_AGENT_NODE_ID } from "#compiler/manifest.js";
@@ -14,6 +14,7 @@ import {
TurnDynamicModelReferenceKey,
} from "#context/keys.js";
import { defineDynamic } from "#dynamic/definition.js";
+import { mockModel } from "#evals/mock-model.js";
import {
createSessionStartedEvent,
createStepStartedEvent,
@@ -198,6 +199,58 @@ describe("dynamic model lifecycle", () => {
});
});
+ it.each([
+ { nodeEnv: "test", mockAuthored: "" },
+ { nodeEnv: "production", mockAuthored: "1" },
+ ])(
+ "keeps explicit mock responders in $nodeEnv with override $mockAuthored",
+ async ({ nodeEnv, mockAuthored }) => {
+ vi.stubEnv("NODE_ENV", nodeEnv);
+ vi.stubEnv("EVE_MOCK_AUTHORED_MODELS", mockAuthored);
+ const ctx = new ContextContainer();
+ const stepModel = mockModel({
+ modelId: "scripted-dispatcher",
+ provider: "custom-fixture",
+ respond: "Use the explicitly authored response.",
+ });
+ const moduleMap = createModuleMap({
+ default: {
+ model: defineDynamic({
+ events: {
+ "step.started": () => ({
+ model: stepModel,
+ modelContextWindowTokens: 64_000,
+ }),
+ },
+ }),
+ },
+ });
+
+ await dispatchDynamicModelEvent({
+ ctx,
+ dynamicModel: DYNAMIC_MODEL_SOURCE,
+ event: createStepStartedEvent({
+ modelId: "unresolved",
+ sequence: 0,
+ stepIndex: 0,
+ turnId: "turn_0",
+ }),
+ messages: [],
+ scope: { moduleMap, nodeId: undefined },
+ });
+
+ const selected = getActiveDynamicModelSelection(ctx);
+ expect(selected?.model).toBe(stepModel);
+ expect(selected?.reference.id).toBe("custom-fixture/scripted-dispatcher");
+ if (selected?.model === undefined) throw new Error("Expected the explicit mock model.");
+ const response = await generateText({
+ model: selected.model,
+ prompt: "Use the configured fixture response.",
+ });
+ expect(response.text).toBe("Use the explicitly authored response.");
+ },
+ );
+
it("rejects live provider instances at durable scopes", async () => {
const ctx = new ContextContainer();
const liveModel = createLanguageModel("openai.responses", "gpt-live");
diff --git a/packages/eve/src/context/dynamic-model-lifecycle.ts b/packages/eve/src/context/dynamic-model-lifecycle.ts
index 813b9d268..67585cd4d 100644
--- a/packages/eve/src/context/dynamic-model-lifecycle.ts
+++ b/packages/eve/src/context/dynamic-model-lifecycle.ts
@@ -3,6 +3,7 @@ import type { ModelMessage } from "ai";
import { buildResolveContext } from "#context/dynamic-resolve-context.js";
import type { AlsContext } from "#context/container.js";
import type { ContextKey } from "#context/key.js";
+import { isMockModel } from "#internal/mock-model-identity.js";
import {
LiveStepDynamicModelSelectionKey,
SessionDynamicModelReferenceKey,
@@ -134,9 +135,12 @@ function setSelectionForEvent(
selection: ResolvedRuntimeModelSelection | null,
): void {
if (eventType === "step.started") {
- // In mock mode drop the live instance so the mock adapter keeps precedence.
+ // Replace real providers in mock mode, but keep explicitly scripted responders.
const stored =
- selection !== null && selection.model !== undefined && shouldMockAuthoredRuntimeModels()
+ selection !== null &&
+ selection.model !== undefined &&
+ !isMockModel(selection.model) &&
+ shouldMockAuthoredRuntimeModels()
? { reference: selection.reference }
: selection;
ctx.setVirtualContext(LiveStepDynamicModelSelectionKey, stored);
diff --git a/packages/eve/src/evals/mock-model.ts b/packages/eve/src/evals/mock-model.ts
index 738a83550..3bf0cea81 100644
--- a/packages/eve/src/evals/mock-model.ts
+++ b/packages/eve/src/evals/mock-model.ts
@@ -1,6 +1,7 @@
import type { LanguageModel } from "ai";
import { MockLanguageModelV3 } from "ai/test";
+import { markMockModel } from "#internal/mock-model-identity.js";
import { AGENTS_SNIPPET_LABEL } from "#subagents/handles/prompt.js";
import { isPendingApprovalsSnippet } from "#harness/hitl/approval-prompt.js";
@@ -125,16 +126,18 @@ export function mockModel(
const respond = normalizeResponder(options.respond);
const modelId = options.modelId ?? DEFAULT_MODEL_ID;
- return new MockLanguageModelV3({
- modelId,
- provider: options.provider ?? DEFAULT_PROVIDER,
- doGenerate: async (callOptions) =>
- createGenerateResult(await respond(createRequest(callOptions)), callOptions, modelId),
- doStream: async (callOptions) =>
- createStreamResult(
+ return markMockModel(
+ new MockLanguageModelV3({
+ modelId,
+ provider: options.provider ?? DEFAULT_PROVIDER,
+ doGenerate: async (callOptions) =>
createGenerateResult(await respond(createRequest(callOptions)), callOptions, modelId),
- ),
- });
+ doStream: async (callOptions) =>
+ createStreamResult(
+ createGenerateResult(await respond(createRequest(callOptions)), callOptions, modelId),
+ ),
+ }),
+ );
}
function normalizeOptions(input: MockModelOptions | MockModelResponder | string): MockModelOptions {
diff --git a/packages/eve/src/execution/parked-delivery-wait.test.ts b/packages/eve/src/execution/parked-delivery-wait.test.ts
index 943a3ff1a..1c208f790 100644
--- a/packages/eve/src/execution/parked-delivery-wait.test.ts
+++ b/packages/eve/src/execution/parked-delivery-wait.test.ts
@@ -20,6 +20,11 @@ vi.mock("./cancel-indexed-session-tasks-step.js", () => ({
import { cancelAllIndexedSessionTasksStep } from "#execution/cancel-indexed-session-tasks-step.js";
+beforeEach(() => {
+ vi.mocked(routeDeliverToChildren).mockReset();
+ vi.mocked(cancelAllIndexedSessionTasksStep).mockReset();
+});
+
interface ScriptedRead {
readonly result: IteratorResult<SessionInboxPayload>;
readonly source: SessionInboxSource;
@@ -112,10 +117,6 @@ function waitInput(inbox: SessionCommandInbox): Parameters<typeof nextTurnDelive
}
describe("nextTurnDelivery", () => {
- afterEach(() => {
- vi.mocked(routeDeliverToChildren).mockReset();
- });
-
it("surfaces an authorization callback as its own instruction", async () => {
const inbox = createMockInbox([authorizationRead()]);
@@ -234,7 +235,6 @@ describe("nextTurnDelivery", () => {
});
describe("nextTurnDelivery routing", () => {
- beforeEach(() => vi.clearAllMocks());
it("keeps waiting instead of starting a parent turn for a fully routed task response", async () => {
const sessionState = {
continuationToken: "token",
diff --git a/packages/eve/src/execution/tools/workflow/workflow-tool-run.integration.test.ts b/packages/eve/src/execution/tools/workflow/workflow-tool-run.integration.test.ts
index abb28e5d3..833ae455e 100644
--- a/packages/eve/src/execution/tools/workflow/workflow-tool-run.integration.test.ts
+++ b/packages/eve/src/execution/tools/workflow/workflow-tool-run.integration.test.ts
@@ -688,12 +688,19 @@ describe("workflow tools", () => {
const asked = await stream.nextTurn();
expect(filterEventsByType(asked, "input.requested")).toHaveLength(1);
- const resumed = await stream.nextTurn();
- const outputs = filterEventsByType(resumed, "action.result").map((event) =>
- JSON.stringify(event.data.result.output),
- );
+ const outputs: string[] = [];
+ // A replayed parked boundary may arrive before the deadline result.
+ for (let attempt = 0; attempt < 5 && outputs.length === 0; attempt += 1) {
+ const resumed = await stream.nextTurn();
+ expect(filterEventsByType(resumed, "turn.failed")).toHaveLength(0);
+ expect(filterEventsByType(resumed, "session.failed")).toHaveLength(0);
+ outputs.push(
+ ...filterEventsByType(resumed, "action.result").map((event) =>
+ JSON.stringify(event.data.result.output),
+ ),
+ );
+ }
expect(outputs.some((output) => output.includes('"decided":"timed out"'))).toBe(true);
- expect(filterEventsByType(resumed, "turn.failed")).toHaveLength(0);
} finally {
stream.dispose();
await run.cancel();
diff --git a/packages/eve/src/execution/workflow-steps.test.ts b/packages/eve/src/execution/workflow-steps.test.ts
index 40cf56b3b..03980b8e3 100644
--- a/packages/eve/src/execution/workflow-steps.test.ts
+++ b/packages/eve/src/execution/workflow-steps.test.ts
@@ -1,5 +1,5 @@
import type { ModelMessage } from "ai";
-import { afterEach, describe, expect, it, vi } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { ChannelAdapter, ChannelAdapterContext } from "#channel/adapter.js";
import type {
DeliverPayload,
@@ -228,7 +228,16 @@ function createSerializedContext(
): Record<string, unknown> {
const ctx = new ContextContainer();
ctx.set(AuthKey, null);
- ctx.set(BundleKey, {
+ ctx.set(BundleKey, createStubBundle());
+ ctx.set(ChannelKey, threadContextAdapter);
+ ctx.set(ContinuationTokenKey, "http:thread-context");
+ ctx.set(ModeKey, mode);
+ ctx.set(SessionIdKey, "session-1");
+ return serializeContext(ctx);
+}
+
+function createStubBundle(): Awaited<ReturnType<typeof getCompiledRuntimeAgentBundle>> {
+ return {
adapterRegistry: {
adaptersByKind: new Map([[threadContextAdapter.kind, threadContextAdapter]]),
},
@@ -245,14 +254,13 @@ function createSerializedContext(
subagentRegistry: {},
toolRegistry: {},
turnAgent: TestTurnAgent,
- } as never);
- ctx.set(ChannelKey, threadContextAdapter);
- ctx.set(ContinuationTokenKey, "http:thread-context");
- ctx.set(ModeKey, mode);
- ctx.set(SessionIdKey, "session-1");
- return serializeContext(ctx);
+ } as never;
}
+beforeEach(() => {
+ vi.mocked(getCompiledRuntimeAgentBundle).mockReset().mockResolvedValue(createStubBundle());
+});
+
afterEach(() => {
getRunMock.mockReset();
resumeHookMock.mockReset();
diff --git a/packages/eve/src/harness/compaction.test.ts b/packages/eve/src/harness/compaction.test.ts
index 2939ef8a5..51775e82f 100644
--- a/packages/eve/src/harness/compaction.test.ts
+++ b/packages/eve/src/harness/compaction.test.ts
@@ -765,6 +765,38 @@ describe("compactMessages: summarization fallback", () => {
expect(result.slice(2, 4)).toEqual([recentCall, recentResult]);
});
+ it("replays a folded-away task when recent tool results survive", async () => {
+ const task = user("Keep the original request and its final task sentinel.");
+ const [call, resultMsg] = toolExchange({ callId: "call-1", payloadChars: 100 });
+ const { result, summarizer } = await compact(
+ [user("old notes ".repeat(4_000)), task, call, resultMsg],
+ { recentWindowSize: 2, threshold: 2_048 },
+ );
+
+ expect(summarizer).toHaveBeenCalled();
+ expect(result).toContainEqual(call);
+ expect(result).toContainEqual(resultMsg);
+ expect(result.at(-1)).toEqual(task);
+ });
+
+ it("makes room for the original task before keeping a large tool-result tail", async () => {
+ const task = user("Keep these requirements. ".repeat(25));
+ const [call, resultMsg] = toolExchange({ callId: "call-1", payloadChars: 600 });
+ const summary = "s".repeat(2_800);
+ const head = checkpointHead(summary);
+ const threshold = Math.floor(
+ (estimateTokens([...head, task]) + estimateTokens([...head, call, resultMsg, task])) / 2,
+ );
+ const { result } = await compact([user("old notes ".repeat(4_000)), task, call, resultMsg], {
+ recentWindowSize: 2,
+ summary,
+ threshold,
+ });
+
+ expect(result.at(-1)).toEqual(task);
+ expect(estimateTokens(result)).toBeLessThanOrEqual(threshold);
+ });
+
it("strips tool activity from the tail when verbatim does not fit but text does", async () => {
const oldProse = user("investigation notes ".repeat(2_000));
const [recentCall, recentResult] = toolExchange({
diff --git a/packages/eve/src/harness/compaction.ts b/packages/eve/src/harness/compaction.ts
index fe83ebf06..4e1323b41 100644
--- a/packages/eve/src/harness/compaction.ts
+++ b/packages/eve/src/harness/compaction.ts
@@ -141,6 +141,7 @@ function toolResultCapHeuristic(input: CompactionHeuristicInput): CompactionHeur
const capped = withResumptionGuard(
[...checkpointHead, ...capToolResults(input.older), ...input.recent],
input.conversation,
+ input.config.threshold,
);
// Evaluate on the same ruler shouldCompact uses (envelope included):
@@ -260,7 +261,11 @@ export async function compactMessages(
// Prefer keeping the recent tail verbatim — surviving tool results are the
// model's evidence that work already ran. Degrade to text-only, then to a
// smaller window, only under threshold pressure.
- const verbatim = withResumptionGuard([...summaryHead, ...recent], conversation);
+ const verbatim = withResumptionGuard(
+ [...summaryHead, ...recent],
+ conversation,
+ config.threshold,
+ );
if (evaluateThreshold(verbatim, config, "estimate").type === "within-limit") {
return verbatim;
}
@@ -268,6 +273,7 @@ export async function compactMessages(
const stripped = withResumptionGuard(
[...summaryHead, ...keepNonToolResultMessages(recent)],
conversation,
+ config.threshold,
);
if (evaluateThreshold(stripped, config, "estimate").type === "within-limit" || keep === 0) {
return stripped;
@@ -338,17 +344,28 @@ function capToolResults(messages: readonly ModelMessage[]): ModelMessage[] {
function withResumptionGuard(
messages: ModelMessage[],
conversation: readonly ModelMessage[],
+ threshold: number,
): ModelMessage[] {
const lastRole = messages.at(-1)?.role;
- if (lastRole !== undefined && lastRole !== "assistant") {
- return messages;
- }
-
const replay = findLastRealUserMessage(conversation);
const alreadyKept =
replay !== undefined &&
messages.some((message) => message.role === "user" && message.content === replay.content);
+ if (lastRole !== undefined && lastRole !== "assistant") {
+ // A retained tool tail must not displace a task that can fit in the budget.
+ // Including it here lets tail selection make room before accepting a candidate.
+ if (
+ lastRole === "tool" &&
+ replay !== undefined &&
+ !alreadyKept &&
+ estimateTokens([replay]) <= threshold
+ ) {
+ return [...messages, replay];
+ }
+ return messages;
+ }
+
return [
...messages,
replay !== undefined && !alreadyKept
diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.ts
index b0a168380..7e55a83bf 100644
--- a/packages/eve/src/harness/tool-loop.test.ts
+++ b/packages/eve/src/harness/tool-loop.test.ts
@@ -240,9 +240,11 @@ vi.mock("./compaction.js", () => ({
afterEach(() => {
vi.clearAllMocks();
+ vi.mocked(shouldCompact).mockReset().mockReturnValue(false);
+ vi.mocked(compactMessages).mockReset();
vi.unstubAllEnvs();
declareTelemetry(undefined);
- mockGetRegisteredTelemetryIntegrations.mockReturnValue([]);
+ mockGetRegisteredTelemetryIntegrations.mockReset().mockReturnValue([]);
});
function createTestSession(overrides?: Partial<HarnessSession>): HarnessSession {
diff --git a/packages/eve/src/internal/mock-model-identity.ts b/packages/eve/src/internal/mock-model-identity.ts
new file mode 100644
index 000000000..789e7208a
--- /dev/null
+++ b/packages/eve/src/internal/mock-model-identity.ts
@@ -0,0 +1,11 @@
+// Authored bundles and the host may load separate copies of this module.
+const MOCK_MODEL = Symbol.for("eve:mock-model");
+
+export function markMockModel<T extends object>(model: T): T {
+ Object.defineProperty(model, MOCK_MODEL, { value: true });
+ return model;
+}
+
+export function isMockModel(model: unknown): boolean {
+ return typeof model === "object" && model !== null && Reflect.get(model, MOCK_MODEL) === true;
+}
diff --git a/packages/eve/src/internal/nitro/host/create-application-nitro.scenario.test.ts b/packages/eve/src/internal/nitro/host/create-application-nitro.scenario.test.ts
index c9b8e77cc..78f3bb884 100644
--- a/packages/eve/src/internal/nitro/host/create-application-nitro.scenario.test.ts
+++ b/packages/eve/src/internal/nitro/host/create-application-nitro.scenario.test.ts
@@ -1,4 +1,4 @@
-import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { mkdir, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
@@ -29,12 +29,14 @@ import {
EVE_WORKFLOW_FLOW_ROUTE_PATH,
} from "#internal/nitro/host/vercel-build-output-config.js";
import { applyWorkflowTransform } from "#internal/workflow-bundle/workflow-builders.js";
+import { useTemporaryDirectories } from "#internal/testing/use-temporary-app-roots.js";
import { defineChannel, WS } from "#public/definitions/channel.js";
const configureDevelopmentNitroRoutes = vi.fn(async () => undefined);
const configureProductionNitroRoutes = vi.fn(async () => undefined);
const createNitroMock = vi.fn();
const registerScheduleTaskHandlers = vi.fn();
+const createTemporaryDirectory = useTemporaryDirectories();
vi.mock("nitro/builder", () => ({
createNitro: createNitroMock,
@@ -198,6 +200,7 @@ describe("application Nitro creation", () => {
beforeEach(() => {
vi.resetModules();
vi.clearAllMocks();
+ createNitroMock.mockReset();
});
afterEach(() => {
@@ -582,6 +585,16 @@ describe("application Nitro creation", () => {
const { createProductionApplicationNitro } =
await import("#internal/nitro/host/create-application-nitro.js");
const preparedHost = await createPreparedHost();
+ const sourceRoot = await realpath(await createTemporaryDirectory("eve-extension-trace-deps-"));
+ for (const name of ["zod", "sharp"]) {
+ const packageRoot = join(sourceRoot, "node_modules", name);
+ await mkdir(join(packageRoot, "lib"), { recursive: true });
+ await writeFile(
+ join(packageRoot, "package.json"),
+ JSON.stringify({ name, version: "1.0.0", main: "./lib/index.js" }),
+ );
+ await writeFile(join(packageRoot, "lib", "index.js"), "module.exports = {};\n");
+ }
preparedHost.compileResult.manifest.extensionMounts = [
{
externalDependencies: ["zod", "sharp"],
@@ -590,7 +603,7 @@ describe("application Nitro creation", () => {
namespace: "layout",
packageName: "layout-extension",
packageNamespace: "layout-extension",
- sourceRoot: process.cwd(),
+ sourceRoot,
},
];
preparedHost.compileResult.manifest.config = {
@@ -611,8 +624,8 @@ describe("application Nitro creation", () => {
id: "zod",
});
expect(createNitroMock.mock.calls[0]?.[0].traceOpts.nft.paths).toEqual({
- zod: expect.stringMatching(/zod[/\\].*index\.(?:c?js|mjs)$/),
- sharp: expect.stringMatching(/sharp[/\\](?:lib[/\\]index\.js|dist[/\\]index\.cjs)$/),
+ zod: join(sourceRoot, "node_modules", "zod", "lib", "index.js"),
+ sharp: join(sourceRoot, "node_modules", "sharp", "lib", "index.js"),
});
});
diff --git a/packages/eve/src/internal/nitro/host/start-development-server.test.ts b/packages/eve/src/internal/nitro/host/start-development-server.test.ts
index e6a4cdefe..1a2dc1e12 100644
--- a/packages/eve/src/internal/nitro/host/start-development-server.test.ts
+++ b/packages/eve/src/internal/nitro/host/start-development-server.test.ts
@@ -207,14 +207,14 @@ vi.mock("#execution/sandbox/development-prewarm.js", () => ({
startDevelopmentSandboxPrewarmInBackground: mocks.startDevelopmentSandboxPrewarmInBackground,
}));
-vi.mock("#execution/sandbox/bindings/local.js", async (importOriginal) => {
- const actual = await importOriginal<typeof import("#execution/sandbox/bindings/local.js")>();
+vi.mock("#execution/sandbox/bindings/local.js", () => ({
+ pruneLocalSandboxTemplatesInBackground: mocks.pruneLocalSandboxTemplatesInBackground,
+ stopDevelopmentSandboxResources: mocks.stopDevelopmentSandboxResources,
+}));
- return {
- ...actual,
- pruneLocalSandboxTemplatesInBackground: mocks.pruneLocalSandboxTemplatesInBackground,
- stopDevelopmentSandboxResources: mocks.stopDevelopmentSandboxResources,
- };
+beforeEach(() => {
+ mocks.fsControl.stateReadError = undefined;
+ mocks.fsControl.stateWriteError = undefined;
});
const developmentServerStatePath = join("/tmp/eve-test", ".eve", "dev-server-state.v1.json");
@@ -348,8 +348,6 @@ describe("createDevelopmentServer", () => {
beforeEach(() => {
vi.clearAllMocks();
mocks.fetch.mockImplementation(async () => Response.json({ revision: "test" }));
- mocks.fsControl.stateReadError = undefined;
- mocks.fsControl.stateWriteError = undefined;
mocks.authoredSourceWatcher.close.mockResolvedValue(undefined);
mocks.authoredSourceWatcher.flush.mockResolvedValue(undefined);
mocks.authoredSourceWatcher.rebuild.mockResolvedValue(undefined);
diff --git a/packages/eve/src/internal/testing/events.test.ts b/packages/eve/src/internal/testing/events.test.ts
index 1d9fa06e5..fdc877a08 100644
--- a/packages/eve/src/internal/testing/events.test.ts
+++ b/packages/eve/src/internal/testing/events.test.ts
@@ -4,6 +4,7 @@ import {
containsEventSequence,
captureTurnEvents,
filterEventsByType,
+ stampTestEvents,
type WorkflowRunHandle,
} from "#internal/testing/events.js";
@@ -83,6 +84,33 @@ describe("filterEventsByType", () => {
});
describe("captureTurnEvents", () => {
+ it("reads every buffered turn before requesting another chunk", async () => {
+ const turns = createTurns();
+ const stream = captureTurnEvents(createStaticRun(encodeEvents(turns.flat())));
+ try {
+ for (const turn of turns) {
+ await expect(stream.nextTurn()).resolves.toEqual(turn);
+ }
+ } finally {
+ stream.dispose();
+ }
+ });
+
+ it("preserves turns and UTF-8 at every two-chunk split", async () => {
+ const turns = createTurns();
+ const bytes = encodeEvents(turns.flat());
+ for (let split = 1; split < bytes.length; split += 1) {
+ const stream = captureTurnEvents(createStaticRun(bytes.slice(0, split), bytes.slice(split)));
+ try {
+ for (const turn of turns) {
+ await expect(stream.nextTurn(), `chunk split ${split}`).resolves.toEqual(turn);
+ }
+ } finally {
+ stream.dispose();
+ }
+ }
+ });
+
it("throws when nextTurn is called after dispose", async () => {
const encoder = new TextEncoder();
const run = createStaticRun(
@@ -101,10 +129,35 @@ describe("captureTurnEvents", () => {
});
});
-function createStaticRun(bytes: Uint8Array): WorkflowRunHandle {
+function createTurns() {
+ return ["caf\u00e9", "done"].map((message, index) =>
+ stampTestEvents([
+ {
+ type: "message.completed",
+ data: {
+ finishReason: "stop",
+ message,
+ sequence: index,
+ stepIndex: 0,
+ turnId: `turn_${index}`,
+ },
+ },
+ {
+ type: "session.waiting",
+ data: { continuationToken: "session-1", wait: "next-user-message" },
+ },
+ ]),
+ );
+}
+
+function encodeEvents(events: readonly unknown[]): Uint8Array {
+ return new TextEncoder().encode(events.map((event) => JSON.stringify(event) + "\n").join(""));
+}
+
+function createStaticRun(...chunks: Uint8Array[]): WorkflowRunHandle {
const readable = new ReadableStream<Uint8Array>({
start(controller) {
- controller.enqueue(bytes);
+ for (const chunk of chunks) controller.enqueue(chunk);
controller.close();
},
});
diff --git a/packages/eve/src/internal/testing/events.ts b/packages/eve/src/internal/testing/events.ts
index 387ea9c8b..387bcaaf3 100644
--- a/packages/eve/src/internal/testing/events.ts
+++ b/packages/eve/src/internal/testing/events.ts
@@ -137,14 +137,6 @@ async function readUntilBoundary(
const events: MessageStreamEvent[] = [];
while (true) {
- const { done, value } = await reader.read();
-
- if (done) {
- throw new Error("Workflow stream closed before reaching a turn boundary.");
- }
-
- state.buffer += decoder.decode(value);
-
for (
let newlineIndex = state.buffer.indexOf("\n");
newlineIndex !== -1;
@@ -164,6 +156,12 @@ async function readUntilBoundary(
return events;
}
}
+
+ const { done, value } = await reader.read();
+ if (done) {
+ throw new Error("Workflow stream closed before reaching a turn boundary.");
+ }
+ state.buffer += decoder.decode(value, { stream: true });
}
}
diff --git a/packages/eve/src/internal/testing/mocks/mock-skill.integration.test.ts b/packages/eve/src/internal/testing/mocks/mock-skill.integration.test.ts
index de7de5284..ffd3c4740 100644
--- a/packages/eve/src/internal/testing/mocks/mock-skill.integration.test.ts
+++ b/packages/eve/src/internal/testing/mocks/mock-skill.integration.test.ts
@@ -5,9 +5,9 @@ import { describe, expect, it } from "vitest";
import { mockSkill } from "#internal/testing/mocks/mock-skill.js";
describe("mockSkill", () => {
- let materializedRootPath: string | undefined;
-
- it("materializes skill package files", async () => {
+ it("materializes skill package files and cleans them after the test", async ({
+ onTestFinished,
+ }) => {
const skill = await mockSkill({
description: "Weather guidance.",
name: "weather",
@@ -23,7 +23,9 @@ describe("mockSkill", () => {
throw new Error("Expected mock skill source to be a skill package.");
}
- materializedRootPath = source.rootPath;
+ onTestFinished(async () => {
+ await expect(access(source.rootPath)).rejects.toMatchObject({ code: "ENOENT" });
+ });
const referencesPath = source.referencesPath;
expect(referencesPath).toBeDefined();
@@ -36,13 +38,14 @@ describe("mockSkill", () => {
await expect(access(referencesPath)).resolves.toBeUndefined();
});
- it("cleans materialized tmpdirs after each test", async () => {
- expect(materializedRootPath).toBeDefined();
-
- if (materializedRootPath === undefined) {
- throw new Error("Expected previous test to create a mock skill root.");
+ it("allows explicit cleanup to run more than once", async () => {
+ const skill = await mockSkill({ name: "weather", description: "Weather guidance." });
+ if (skill.source.sourceKind !== "skill-package") {
+ throw new Error("Expected mock skill source to be a skill package.");
}
-
- await expect(access(materializedRootPath)).rejects.toThrow();
+ await expect(access(skill.source.rootPath)).resolves.toBeUndefined();
+ await skill.cleanup();
+ await skill.cleanup();
+ await expect(access(skill.source.rootPath)).rejects.toMatchObject({ code: "ENOENT" });
});
});
diff --git a/packages/eve/src/internal/testing/scenario-app.ts b/packages/eve/src/internal/testing/scenario-app.ts
index af79d1271..a4e55cf59 100644
--- a/packages/eve/src/internal/testing/scenario-app.ts
+++ b/packages/eve/src/internal/testing/scenario-app.ts
@@ -284,7 +284,8 @@ async function installScenarioDependencies(input: {
"--no-audit",
"--no-fund",
"--ignore-scripts",
- "--prefer-offline",
+ // The workspace's installed AI SDK version may be newer than cached registry metadata.
+ "--prefer-online",
]);
return;
}
diff --git a/packages/eve/src/internal/testing/scenario-apps/next-eve-proxy.ts b/packages/eve/src/internal/testing/scenario-apps/next-eve-proxy.ts
index 233ca843f..5a7fd7008 100644
--- a/packages/eve/src/internal/testing/scenario-apps/next-eve-proxy.ts
+++ b/packages/eve/src/internal/testing/scenario-apps/next-eve-proxy.ts
@@ -3,6 +3,7 @@ import { createRequire } from "node:module";
import type { ScenarioAppDescriptor } from "#internal/testing/scenario-app.js";
const require = createRequire(import.meta.url);
+const nextRequire = createRequire(require.resolve("next/package.json"));
export interface NextEveProxyDescriptorOptions {
readonly installDependencies?: boolean;
@@ -58,7 +59,7 @@ export function proxy() {
}
function resolveInstalledPackageVersion(packageName: string): string {
- const manifest: unknown = require(`${packageName}/package.json`);
+ const manifest: unknown = nextRequire(`${packageName}/package.json`);
if (
typeof manifest !== "object" ||
diff --git a/packages/eve/src/internal/workflow/local-world-file-reads.test.ts b/packages/eve/src/internal/workflow/local-world-file-reads.test.ts
new file mode 100644
index 000000000..f62f0c6f8
--- /dev/null
+++ b/packages/eve/src/internal/workflow/local-world-file-reads.test.ts
@@ -0,0 +1,85 @@
+import { promises as fs } from "node:fs";
+import { createRequire } from "node:module";
+import { pathToFileURL } from "node:url";
+
+import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
+
+interface FileReader {
+ readJSON(path: string, decoder: { parse(value: unknown): unknown }): Promise<unknown>;
+}
+
+const readFile = vi.fn<() => Promise<string>>();
+const decoder = { parse: (value: unknown) => value };
+let reader: FileReader;
+
+beforeAll(async () => {
+ const require = createRequire(import.meta.url);
+ const url = new URL("./fs.js", pathToFileURL(require.resolve("@workflow/world-local")));
+ const platform = Object.getOwnPropertyDescriptor(process, "platform")!;
+ // Exercise the installed patch's Windows path on every CI operating system.
+ Object.defineProperty(process, "platform", { ...platform, value: "win32" });
+ try {
+ reader = await import(/* @vite-ignore */ url.href);
+ } finally {
+ Object.defineProperty(process, "platform", platform);
+ }
+});
+
+beforeEach(() => {
+ vi.useFakeTimers();
+ readFile.mockReset().mockResolvedValue('{"ready":true}');
+ vi.spyOn(fs, "readFile").mockImplementation(readFile);
+});
+
+afterEach(() => {
+ vi.restoreAllMocks();
+ vi.useRealTimers();
+});
+
+async function read() {
+ const settled = Promise.allSettled([reader.readJSON("hook.json", decoder)]);
+ await vi.runAllTimersAsync();
+ return (await settled)[0];
+}
+
+describe("local workflow file reads", () => {
+ it.each(["EPERM", "EACCES", "EBUSY"])("recovers from transient %s reads", async (code) => {
+ const error = Object.assign(new Error("temporarily busy"), { code });
+ readFile.mockRejectedValueOnce(error).mockRejectedValueOnce(error);
+
+ expect(await read()).toEqual({ status: "fulfilled", value: { ready: true } });
+ expect(readFile).toHaveBeenCalledTimes(3);
+ expect(readFile).toHaveBeenLastCalledWith("hook.json", "utf-8");
+ });
+
+ it("still fails when a sharing violation persists", async () => {
+ const error = Object.assign(new Error("busy"), { code: "EPERM" });
+ readFile.mockRejectedValue(error);
+
+ expect(await read()).toEqual({ status: "rejected", reason: error });
+ expect(readFile).toHaveBeenCalledTimes(6);
+ });
+
+ it("returns null when a concurrently deleted file disappears", async () => {
+ readFile.mockRejectedValueOnce(Object.assign(new Error("busy"), { code: "EPERM" }));
+ readFile.mockRejectedValueOnce(Object.assign(new Error("gone"), { code: "ENOENT" }));
+
+ expect(await read()).toEqual({ status: "fulfilled", value: null });
+ expect(readFile).toHaveBeenCalledTimes(2);
+ });
+
+ it("does not retry permanent read failures", async () => {
+ const error = Object.assign(new Error("I/O failure"), { code: "EIO" });
+ readFile.mockRejectedValue(error);
+
+ expect(await read()).toEqual({ status: "rejected", reason: error });
+ expect(readFile).toHaveBeenCalledTimes(1);
+ });
+
+ it("does not retry malformed JSON", async () => {
+ readFile.mockResolvedValue("{");
+
+ expect(await read()).toEqual({ status: "rejected", reason: expect.any(SyntaxError) });
+ expect(readFile).toHaveBeenCalledTimes(1);
+ });
+});
diff --git a/packages/eve/src/react/use-eve-agent.test.ts b/packages/eve/src/react/use-eve-agent.test.ts
index f2c31a640..f37d67425 100644
--- a/packages/eve/src/react/use-eve-agent.test.ts
+++ b/packages/eve/src/react/use-eve-agent.test.ts
@@ -1,5 +1,5 @@
import { createElement } from "react";
-import { act, create } from "react-test-renderer";
+import { act, create as createRenderer } from "react-test-renderer";
import { afterEach, describe, expect, it, vi } from "vitest";
import { useEveAgent, type UseEveAgentHelpers } from "#react/use-eve-agent.js";
@@ -131,8 +131,22 @@ function completedTurnData(input: {
};
}
-afterEach(() => {
- vi.restoreAllMocks();
+const renderers: ReturnType<typeof createRenderer>[] = [];
+
+function create(...args: Parameters<typeof createRenderer>) {
+ const renderer = createRenderer(...args);
+ renderers.push(renderer);
+ return renderer;
+}
+
+afterEach(async () => {
+ try {
+ await act(async () => {
+ for (const renderer of renderers.splice(0)) renderer.unmount();
+ });
+ } finally {
+ vi.restoreAllMocks();
+ }
});
describe("useEveAgent", () => {
@@ -705,6 +719,8 @@ describe("useEveAgent", () => {
await act(async () => {
create(createElement(TestComponent));
+ });
+ await act(async () => {
await vi.waitFor(() => expect(helpers?.status).toBe("ready"));
});
diff --git a/packages/eve/src/self-modification/package-consumption.scenario.test.ts b/packages/eve/src/self-modification/package-consumption.scenario.test.ts
index 38f87606e..6da4200c7 100644
--- a/packages/eve/src/self-modification/package-consumption.scenario.test.ts
+++ b/packages/eve/src/self-modification/package-consumption.scenario.test.ts
@@ -141,13 +141,7 @@ describe("packed package consumption", () => {
await run(
"pnpm",
- [
- "install",
- "--ignore-scripts",
- "--no-frozen-lockfile",
- "--prefer-offline",
- "--config.minimum-release-age=0",
- ],
+ ["install", "--ignore-scripts", "--no-frozen-lockfile", "--prefer-offline"],
appRoot,
);
await access(join(appRoot, "node_modules/eve/dist/src/self-modification/agent.js"));
diff --git a/packages/eve/test/scenarios/eve-init.scenario.test.ts b/packages/eve/test/scenarios/eve-init.scenario.test.ts
index 458fb7ad4..3ecc91406 100644
--- a/packages/eve/test/scenarios/eve-init.scenario.test.ts
+++ b/packages/eve/test/scenarios/eve-init.scenario.test.ts
@@ -10,15 +10,12 @@ import { CODING_AGENT_ENV_MARKERS } from "../../src/cli/agent-detection.js";
import { loadYaml } from "../../src/evals/loaders/yaml.js";
import { DEFAULT_AGENT_MODEL_ID } from "../../src/shared/default-agent-model.js";
import { pathExists } from "../../src/setup/path-exists.js";
+import { ensureScenarioEveTarballPath } from "../../src/internal/testing/scenario-app.js";
import { useTemporaryDirectories } from "../../src/internal/testing/use-temporary-app-roots.js";
const EVE_BIN_PATH = fileURLToPath(new URL("../../bin/eve.js", import.meta.url));
const runFile = promisify(execFile);
const RELEASE_AGE_MINUTES = "2880";
-// Changesets opens Version Packages PRs on `changeset-release/<base>`. Those
-// PRs bump package.json before npm has that version, so a real registry
-// install of the scaffolded eve range cannot succeed yet.
-const isChangesetReleasePr = process.env.GITHUB_HEAD_REF?.startsWith("changeset-release/") === true;
const createScratchDirectory = useTemporaryDirectories();
@@ -167,43 +164,66 @@ async function createFakeNpmEnvironment(scratch: string): Promise<{
}
describe("eve init smoke", () => {
- it.skipIf(isChangesetReleasePr)(
- "resolves a standalone pnpm scaffold under the release-age policy",
- async () => {
- const scratch = await createScratchDirectory("eve-init-release-age-");
- const workspacePolicy = (await loadYaml(
- fileURLToPath(new URL("../../../../pnpm-workspace.yaml", import.meta.url)),
- )) as { minimumReleaseAgeExclude: string[] };
- const env = {
- ...withoutCodingAgentMarkers(process.env),
- // The agent path skips the interactive dev handoff, which cannot run
- // against the real pnpm install this scenario performs.
- AI_AGENT: "claude",
- CI: "true",
- PNPM_CONFIG_MINIMUM_RELEASE_AGE: RELEASE_AGE_MINUTES,
- // Use the repository's reviewed dependency exceptions for the current
- // release without adding bypasses to scaffolded projects.
- PNPM_CONFIG_MINIMUM_RELEASE_AGE_EXCLUDE: JSON.stringify([
- "eve",
- ...workspacePolicy.minimumReleaseAgeExclude,
- ]),
- };
-
- const result = await runEveBin(scratch, ["init", "policy-agent"], env);
-
- expect(result.exitCode, result.stderr).toBe(0);
- const projectDir = join(scratch, "policy-agent");
- await expect(
- readFile(join(projectDir, "pnpm-workspace.yaml"), "utf8"),
- ).resolves.not.toContain("minimumReleaseAgeExclude:");
- await expect(
- runFile("pnpm", ["add", "--ignore-scripts", "--lockfile-only", "[email protected]"], {
- cwd: projectDir,
- env,
- }),
- ).resolves.toMatchObject({ stderr: expect.any(String) });
- },
- );
+ it("resolves a standalone pnpm scaffold under the release-age policy", async () => {
+ const scratch = await createScratchDirectory("eve-init-release-age-");
+ const eveTarball = `file:${await ensureScenarioEveTarballPath()}`;
+ const pnpmfile = join(scratch, "scaffold-pnpmfile.cjs");
+ // The package under test need not have reached npm; registry dependencies still use the policy.
+ await writeFile(
+ pnpmfile,
+ [
+ "module.exports = { hooks: { updateConfig(config) {",
+ ` return { ...config, overrides: { ...config.overrides, eve: ${JSON.stringify(eveTarball)} } };`,
+ "} } };\n",
+ ].join("\n"),
+ );
+ const workspacePolicy = (await loadYaml(
+ fileURLToPath(new URL("../../../../pnpm-workspace.yaml", import.meta.url)),
+ )) as { minimumReleaseAgeExclude: string[] };
+ const env = {
+ ...withoutCodingAgentMarkers(process.env),
+ // The agent path skips the interactive dev handoff, which cannot run
+ // against the real pnpm install this scenario performs.
+ AI_AGENT: "claude",
+ CI: "true",
+ PNPM_CONFIG_GLOBAL_PNPMFILE: pnpmfile,
+ PNPM_CONFIG_MINIMUM_RELEASE_AGE: RELEASE_AGE_MINUTES,
+ // Use the repository's reviewed dependency exceptions for the current
+ // release without adding bypasses to scaffolded projects.
+ PNPM_CONFIG_MINIMUM_RELEASE_AGE_EXCLUDE: JSON.stringify(
+ workspacePolicy.minimumReleaseAgeExclude,
+ ),
+ };
+
+ const result = await runEveBin(scratch, ["init", "policy-agent"], env);
+
+ expect(result.exitCode, result.stderr).toBe(0);
+ const projectDir = join(scratch, "policy-agent");
+ await expect(readFile(join(projectDir, "pnpm-workspace.yaml"), "utf8")).resolves.not.toContain(
+ "minimumReleaseAgeExclude:",
+ );
+ await expect(readFile(join(projectDir, "pnpm-workspace.yaml"), "utf8")).resolves.toContain(
+ "minimumReleaseAgeStrict: true",
+ );
+
+ // Exercise publication lag even when the checkout's version is already available on npm.
+ const manifestPath = join(projectDir, "package.json");
+ const manifest = JSON.parse(await readFile(manifestPath, "utf8")) as {
+ dependencies: Record<string, string>;
+ };
+ manifest.dependencies.eve = "0.0.0-eve-init-unpublished";
+ await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`);
+ await expect(
+ runFile("pnpm", ["add", "--ignore-scripts", "--lockfile-only", "[email protected]"], {
+ cwd: projectDir,
+ env,
+ }),
+ ).resolves.toMatchObject({ stderr: expect.any(String) });
+ const lockfile = (await loadYaml(join(projectDir, "pnpm-lock.yaml"))) as {
+ overrides?: Record<string, string>;
+ };
+ expect(lockfile.overrides?.eve).toBe(eveTarball);
+ });
it("creates the base template with the default model and no Vercel state", async () => {
const scratch = await createScratchDirectory("eve-init-");
diff --git a/patches/@[email protected] b/patches/@[email protected]
new file mode 100644
index 000000000..846f6cf7e
--- /dev/null
+++ b/patches/@[email protected]
@@ -0,0 +1,13 @@
+diff --git a/dist/fs.js b/dist/fs.js
+index f5b40f81f2c8581d333c653a26fb64b4730e959c..93cd6d177a22426afdad49947140f07e73ab283b 100644
+--- a/dist/fs.js
++++ b/dist/fs.js
+@@ -360,7 +360,7 @@ export async function write(filePath, data, opts) {
+ }
+ export async function readJSON(filePath, decoder) {
+ try {
+- const content = await fs.readFile(filePath, 'utf-8');
++ const content = await withWindowsRetry(() => fs.readFile(filePath, 'utf-8'));
+ return decoder.parse(JSON.parse(content, jsonReviver));
+ }
+ catch (error) {
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index c7bf8ac21..25611eacf 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -94,6 +94,9 @@ catalogs:
specifier: 4.5.4
version: 4.5.4
+patchedDependencies:
+ '@workflow/[email protected]': 593f7971cdce38c1a7e69dd1a613ba7eaec158484c8bf2519ec2d20e297deba8
+
importers:
.:
@@ -1432,7 +1435,7 @@ importers:
version: 5.0.0-beta.33
'@workflow/world-local':
specifier: 5.0.0-beta.42
- version: 5.0.0-beta.42(@opentelemetry/[email protected])
+ version: 5.0.0-beta.42(patch_hash=593f7971cdce38c1a7e69dd1a613ba7eaec158484c8bf2519ec2d20e297deba8)(@opentelemetry/[email protected])
'@workflow/world-vercel':
specifier: 5.0.0-beta.44
version: 5.0.0-beta.44(@aws-sdk/[email protected])(@opentelemetry/[email protected])([email protected])
@@ -23441,7 +23444,7 @@ snapshots:
'@workflow/serde': 5.0.0-beta.2
'@workflow/utils': 5.0.0-beta.10
'@workflow/world': 5.0.0-beta.33
- '@workflow/world-local': 5.0.0-beta.42(@opentelemetry/[email protected])
+ '@workflow/world-local': 5.0.0-beta.42(patch_hash=593f7971cdce38c1a7e69dd1a613ba7eaec158484c8bf2519ec2d20e297deba8)(@opentelemetry/[email protected])
'@workflow/world-vercel': 5.0.0-beta.44(@aws-sdk/[email protected])(@opentelemetry/[email protected])([email protected])
debug: 4.4.3([email protected])
devalue: 5.9.0
@@ -23479,7 +23482,7 @@ snapshots:
dependencies:
ms: 2.1.3
- '@workflow/[email protected](@opentelemetry/[email protected])':
+ '@workflow/[email protected](patch_hash=593f7971cdce38c1a7e69dd1a613ba7eaec158484c8bf2519ec2d20e297deba8)(@opentelemetry/[email protected])':
dependencies:
'@vercel/queue': 0.5.0(@opentelemetry/[email protected])
'@workflow/errors': 5.0.0-beta.20
@@ -23499,7 +23502,7 @@ snapshots:
'@workflow/errors': 5.0.0-beta.20
'@workflow/utils': 5.0.0-beta.10
'@workflow/world': 5.0.0-beta.33
- '@workflow/world-local': 5.0.0-beta.42(@opentelemetry/[email protected])
+ '@workflow/world-local': 5.0.0-beta.42(patch_hash=593f7971cdce38c1a7e69dd1a613ba7eaec158484c8bf2519ec2d20e297deba8)(@opentelemetry/[email protected])
cbor-x: 1.6.0
dotenv: 17.3.1
drizzle-orm: 0.45.2(@opentelemetry/[email protected])(@types/[email protected])(@upstash/[email protected])([email protected])([email protected])
diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml
index 8cd951376..b493e3fa2 100644
--- a/pnpm-workspace.yaml
+++ b/pnpm-workspace.yaml
@@ -73,3 +73,6 @@ minimumReleaseAgeExclude:
- run
- workflow
+
+patchedDependencies:
+ "@workflow/[email protected]": patches/@[email protected]