Skip to content

The open-source app everyone uses to manage agents at work

fix(evals): capture complete durable run event streams (#13977)

Dottaf56aee54236f
diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md
index ffe53d181b..db28e68025 100644
--- a/tests/runner-e2e/README.md
+++ b/tests/runner-e2e/README.md
@@ -381,6 +381,12 @@ published with permanent campaign dashboards; fixture authors must therefore
keep credentials and other private data out of every captured UI state. SVG is
active content and is rejected from the packaged evidence entirely.
+The standard task and chat evidence collectors read durable run events through the paginated
+public API, including completion events beyond the first 1,000 rows. It rejects
+missing, repeated, or out-of-order sequence numbers and fails capture after
+100 full pages instead of grading a truncated stream. Original incomplete
+captures remain failed evidence; qualifying a fix requires a new live attempt.
+
Every completed local campaign also writes
`tests/runner-e2e/results/<campaign>/dashboard.html`. The self-contained page
shows the complete profile/environment grid with screenshot thumbnails.
diff --git a/tests/runner-e2e/chat-flow.test.ts b/tests/runner-e2e/chat-flow.test.ts
index c0a1a174fa..f69ff74cd3 100644
--- a/tests/runner-e2e/chat-flow.test.ts
+++ b/tests/runner-e2e/chat-flow.test.ts
@@ -224,21 +224,34 @@ describe("chat acceptance contracts", () => {
);
});
it("retains reset events without requesting a provider log, and does not hide missing real logs", async () => {
- const get = vi.fn().mockResolvedValue([{ type: "session_reset" }]);
+ const get = vi.fn().mockResolvedValue([{ seq: 1, type: "session_reset" }]);
const reset = { ...run, resultJson: { conversationReset: true } };
await expect(collectChatRunEvidence({ get }, reset)).resolves.toEqual({
runId: run.id,
log: null,
- events: [{ type: "session_reset" }],
+ events: [{ seq: 1, type: "session_reset" }],
});
expect(get.mock.calls).toEqual([
- [`/api/heartbeat-runs/${run.id}/events?limit=1000`],
+ [`/api/heartbeat-runs/${run.id}/events?afterSeq=0&limit=1000`],
]);
get.mockRejectedValue(new Error("Run log not found"));
await expect(collectChatRunEvidence({ get }, run)).rejects.toThrow(
"Run log not found",
);
});
+ it("retains the tail of a long chat run for invariant checks", async () => {
+ const firstPage = Array.from({ length: 1000 }, (_, i) => ({ seq: i + 1, eventType: "item.delta" }));
+ const tail = { seq: 1001, eventType: "run.terminal" };
+ const get = vi.fn().mockResolvedValueOnce({ content: "fixture log" })
+ .mockResolvedValueOnce(firstPage).mockResolvedValueOnce([tail]);
+ const evidence = await collectChatRunEvidence({ get }, run);
+ expect(evidence.events).toEqual([...firstPage, tail]);
+ expect(get.mock.calls).toEqual([
+ [`/api/heartbeat-runs/${run.id}/log?limitBytes=1048576`],
+ [`/api/heartbeat-runs/${run.id}/events?afterSeq=0&limit=1000`],
+ [`/api/heartbeat-runs/${run.id}/events?afterSeq=1000&limit=1000`],
+ ]);
+ });
it("retains events for an unstarted dependency-blocked wake without asking for a nonexistent log", async () => {
const get = vi.fn().mockResolvedValue([]);
const suppressed = { ...run, status: "cancelled", errorCode: "issue_dependencies_blocked", startedAt: null };
diff --git a/tests/runner-e2e/chat-flow.ts b/tests/runner-e2e/chat-flow.ts
index db36a37124..a77712a674 100644
--- a/tests/runner-e2e/chat-flow.ts
+++ b/tests/runner-e2e/chat-flow.ts
@@ -7,6 +7,7 @@ import type {
import type { LiveFixtureValues } from "./live-fixtures.js";
import type { MatrixExecution } from "./types.js";
import { isBlockedUnstartedWake } from "./non-execution-wake.js";
+import { collectRunEvents } from "./run-observations.js";
import { chatMarker } from "./chat-cases.js";
import { assertChatRememberedAfterRestart, assertChatStartupStopped, isChatStopReady, runChatHardeningFlow } from "./chat-hardening.js";
import { enableChatThroughSettings, runChatInterruption, runChatSettingsLifecycle } from "./chat-stories.js";
@@ -239,7 +240,9 @@ export async function collectChatRunEvidence(
log: isResetRun(run) || isBlockedUnstartedWake({ ...run })
? null
: await api.get(`/api/heartbeat-runs/${run.id}/log?limitBytes=1048576`),
- events: await api.get(`/api/heartbeat-runs/${run.id}/events?limit=1000`),
+ events: await collectRunEvents((afterSeq, limit) =>
+ api.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=${limit}`),
+ ),
};
}
diff --git a/tests/runner-e2e/everyday.test.ts b/tests/runner-e2e/everyday.test.ts
index 5016adf268..c1322eb794 100644
--- a/tests/runner-e2e/everyday.test.ts
+++ b/tests/runner-e2e/everyday.test.ts
@@ -212,7 +212,7 @@ describe("manual everyday workflow catalog", () => {
const selected = selectRunnerExecutions(
parseRunnerSelectors(["--suite", "everyday-workflows"]),
);
- expect(selected).toHaveLength(38);
+ expect(selected).toHaveLength(47);
expect(selected.every((e) => e.profile.generation === "native")).toBe(true);
expect(
selectRunnerExecutions(
diff --git a/tests/runner-e2e/run-observations.ts b/tests/runner-e2e/run-observations.ts
index e0f18d49ce..1e5b1949e2 100644
--- a/tests/runner-e2e/run-observations.ts
+++ b/tests/runner-e2e/run-observations.ts
@@ -1,3 +1,28 @@
+/** Read the complete durable stream; never grade a silently truncated page. */
+export async function collectRunEvents<T extends { seq?: number }>(
+ loadPage: (afterSeq: number, limit: number) => Promise<unknown>,
+): Promise<T[]> {
+ const pageSize = 1000;
+ const events: T[] = [];
+ let afterSeq = 0;
+ for (let pageNumber = 0; pageNumber < 100; pageNumber += 1) {
+ const page = await loadPage(afterSeq, pageSize);
+ if (!Array.isArray(page) || page.length > pageSize) {
+ throw new Error("Run event evidence returned an invalid page");
+ }
+ for (const event of page) {
+ const seq = event?.seq;
+ if (!Number.isSafeInteger(seq) || seq <= afterSeq) {
+ throw new Error("Run event evidence has a missing or non-increasing sequence");
+ }
+ afterSeq = seq;
+ events.push(event as T);
+ }
+ if (page.length < pageSize) return events;
+ }
+ throw new Error("Run event evidence exceeded 100 pages; refusing incomplete evidence");
+}
+
export interface ObservableRunState {
status?: string | null;
errorCode?: string | null;
diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts
index 9649bf1da5..5e8392c529 100644
--- a/tests/runner-e2e/runner.spec.ts
+++ b/tests/runner-e2e/runner.spec.ts
@@ -21,6 +21,7 @@ import { setupLiveFixtures, type LiveFixtureValues } from "./live-fixtures.js";
import { evaluateMatcher, persistedFinalRunMessage, type MatcherResult } from "./matchers.js";
import {
acceptedPlanSessionResetFailures,
+ collectRunEvents,
hasTerminalMalformedPlanConfirmation,
isControlPlaneGovernedResponseWait,
isNonExecutingReviewFenceRun,
@@ -696,8 +697,8 @@ for (const execution of executions) {
),
),
events: await capture(() =>
- api.get<RunEventRecord[]>(
- `/api/heartbeat-runs/${candidate.id}/events?limit=1000`,
+ collectRunEvents<RunEventRecord>((afterSeq, limit) =>
+ api.get(`/api/heartbeat-runs/${candidate.id}/events?afterSeq=${afterSeq}&limit=${limit}`),
),
),
})),
@@ -1694,8 +1695,8 @@ for (const execution of executions) {
try {
return {
runId: candidate.id,
- events: await api.get<RunEventRecord[]>(
- `/api/heartbeat-runs/${candidate.id}/events?limit=1000`,
+ events: await collectRunEvents<RunEventRecord>((afterSeq, limit) =>
+ api.get(`/api/heartbeat-runs/${candidate.id}/events?afterSeq=${afterSeq}&limit=${limit}`),
),
error: null,
};
diff --git a/tests/runner-e2e/support.test.ts b/tests/runner-e2e/support.test.ts
index e7c4cb95ff..9c71367fa0 100644
--- a/tests/runner-e2e/support.test.ts
+++ b/tests/runner-e2e/support.test.ts
@@ -38,6 +38,7 @@ import {
} from "./ports.js";
import {
acceptedPlanSessionResetFailures,
+ collectRunEvents,
hasTerminalMalformedPlanConfirmation,
isControlPlaneGovernedResponseWait,
isNonExecutingReviewFenceRun,
@@ -48,6 +49,47 @@ import {
import { runnerE2EWebServerCommand } from "./web-server-command.js";
const cleanupDirectories: string[] = [];
+
+describe("complete run event evidence", () => {
+ const page = Array.from({ length: 1000 }, (_, i) => ({ seq: i + 1, eventType: "item.delta" }));
+ it("reads completion events beyond the first 1000 rows", async () => {
+ const terminal = ["run.result.proposed", "run.result.accepted", "run.terminal"]
+ .map((eventType, i) => ({ seq: 1001 + i, eventType }));
+ const load = vi.fn().mockResolvedValueOnce(page).mockResolvedValueOnce(terminal);
+ const events = await collectRunEvents(load);
+ expect(events).toEqual([...page, ...terminal]);
+ expect(load.mock.calls).toEqual([[0, 1000], [1000, 1000]]);
+ });
+ it("checks for another page even at an exact page boundary", async () => {
+ const load = vi.fn().mockResolvedValueOnce(page).mockResolvedValueOnce([]);
+ expect(await collectRunEvents(load)).toEqual(page);
+ expect(load).toHaveBeenCalledTimes(2);
+ });
+ it.each([
+ null, {}, [{ eventType: "run.terminal" }], [{ seq: 0 }], [{ seq: -1 }],
+ [{ seq: 1.5 }], [{ seq: "1" }], [{ seq: NaN }], [{ seq: Infinity }],
+ [{ seq: 2 }, { seq: 1 }], [{ seq: 1 }, { seq: 1 }], [...page, { seq: 1001 }],
+ ])("rejects malformed evidence page %#", async (malformed) => {
+ await expect(collectRunEvents(async () => malformed)).rejects.toThrow("Run event evidence");
+ });
+ it("rejects a repeated cursor instead of accepting duplicate events", async () => {
+ const load = vi.fn().mockResolvedValue(page);
+ await expect(collectRunEvents(load)).rejects.toThrow("non-increasing sequence");
+ expect(load).toHaveBeenCalledTimes(2);
+ });
+ it("propagates a missing later page without returning partial evidence", async () => {
+ const load = vi.fn().mockResolvedValueOnce(page).mockRejectedValueOnce(new Error("Unavailable"));
+ await expect(collectRunEvents(load)).rejects.toThrow("Unavailable");
+ });
+ it("fails closed when a stream never ends within the bounded capture", async () => {
+ const load = vi.fn(async (afterSeq: number) => page.map((event) => ({ ...event, seq: event.seq + afterSeq })));
+ await expect(collectRunEvents(load)).rejects.toThrow("refusing incomplete evidence");
+ expect(load).toHaveBeenCalledTimes(100);
+ });
+});
+
+
+
afterEach(async () => {
vi.unstubAllEnvs();
vi.unstubAllGlobals();