The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix(evals): capture complete durable run event streams (#13977)
Dottaf56aee54236f
diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.mdindex ffe53d181b..db28e68025 100644--- a/tests/runner-e2e/README.md+++ b/tests/runner-e2e/README.md@@ -381,6 +381,12 @@ published with permanent campaign dashboards; fixture authors must thereforekeep credentials and other private data out of every captured UI state. SVG isactive content and is rejected from the packaged evidence entirely.+The standard task and chat evidence collectors read durable run events through the paginated+public API, including completion events beyond the first 1,000 rows. It rejects+missing, repeated, or out-of-order sequence numbers and fails capture after+100 full pages instead of grading a truncated stream. Original incomplete+captures remain failed evidence; qualifying a fix requires a new live attempt.+Every completed local campaign also writes`tests/runner-e2e/results/<campaign>/dashboard.html`. The self-contained pageshows the complete profile/environment grid with screenshot thumbnails.diff --git a/tests/runner-e2e/chat-flow.test.ts b/tests/runner-e2e/chat-flow.test.tsindex c0a1a174fa..f69ff74cd3 100644--- a/tests/runner-e2e/chat-flow.test.ts+++ b/tests/runner-e2e/chat-flow.test.ts@@ -224,21 +224,34 @@ describe("chat acceptance contracts", () => {);});it("retains reset events without requesting a provider log, and does not hide missing real logs", async () => {- const get = vi.fn().mockResolvedValue([{ type: "session_reset" }]);+ const get = vi.fn().mockResolvedValue([{ seq: 1, type: "session_reset" }]);const reset = { ...run, resultJson: { conversationReset: true } };await expect(collectChatRunEvidence({ get }, reset)).resolves.toEqual({runId: run.id,log: null,- events: [{ type: "session_reset" }],+ events: [{ seq: 1, type: "session_reset" }],});expect(get.mock.calls).toEqual([- [`/api/heartbeat-runs/${run.id}/events?limit=1000`],+ [`/api/heartbeat-runs/${run.id}/events?afterSeq=0&limit=1000`],]);get.mockRejectedValue(new Error("Run log not found"));await expect(collectChatRunEvidence({ get }, run)).rejects.toThrow("Run log not found",);});+ it("retains the tail of a long chat run for invariant checks", async () => {+ const firstPage = Array.from({ length: 1000 }, (_, i) => ({ seq: i + 1, eventType: "item.delta" }));+ const tail = { seq: 1001, eventType: "run.terminal" };+ const get = vi.fn().mockResolvedValueOnce({ content: "fixture log" })+ .mockResolvedValueOnce(firstPage).mockResolvedValueOnce([tail]);+ const evidence = await collectChatRunEvidence({ get }, run);+ expect(evidence.events).toEqual([...firstPage, tail]);+ expect(get.mock.calls).toEqual([+ [`/api/heartbeat-runs/${run.id}/log?limitBytes=1048576`],+ [`/api/heartbeat-runs/${run.id}/events?afterSeq=0&limit=1000`],+ [`/api/heartbeat-runs/${run.id}/events?afterSeq=1000&limit=1000`],+ ]);+ });it("retains events for an unstarted dependency-blocked wake without asking for a nonexistent log", async () => {const get = vi.fn().mockResolvedValue([]);const suppressed = { ...run, status: "cancelled", errorCode: "issue_dependencies_blocked", startedAt: null };diff --git a/tests/runner-e2e/chat-flow.ts b/tests/runner-e2e/chat-flow.tsindex db36a37124..a77712a674 100644--- a/tests/runner-e2e/chat-flow.ts+++ b/tests/runner-e2e/chat-flow.ts@@ -7,6 +7,7 @@ import type {import type { LiveFixtureValues } from "./live-fixtures.js";import type { MatrixExecution } from "./types.js";import { isBlockedUnstartedWake } from "./non-execution-wake.js";+import { collectRunEvents } from "./run-observations.js";import { chatMarker } from "./chat-cases.js";import { assertChatRememberedAfterRestart, assertChatStartupStopped, isChatStopReady, runChatHardeningFlow } from "./chat-hardening.js";import { enableChatThroughSettings, runChatInterruption, runChatSettingsLifecycle } from "./chat-stories.js";@@ -239,7 +240,9 @@ export async function collectChatRunEvidence(log: isResetRun(run) || isBlockedUnstartedWake({ ...run })? null: await api.get(`/api/heartbeat-runs/${run.id}/log?limitBytes=1048576`),- events: await api.get(`/api/heartbeat-runs/${run.id}/events?limit=1000`),+ events: await collectRunEvents((afterSeq, limit) =>+ api.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=${limit}`),+ ),};}diff --git a/tests/runner-e2e/everyday.test.ts b/tests/runner-e2e/everyday.test.tsindex 5016adf268..c1322eb794 100644--- a/tests/runner-e2e/everyday.test.ts+++ b/tests/runner-e2e/everyday.test.ts@@ -212,7 +212,7 @@ describe("manual everyday workflow catalog", () => {const selected = selectRunnerExecutions(parseRunnerSelectors(["--suite", "everyday-workflows"]),);- expect(selected).toHaveLength(38);+ expect(selected).toHaveLength(47);expect(selected.every((e) => e.profile.generation === "native")).toBe(true);expect(selectRunnerExecutions(diff --git a/tests/runner-e2e/run-observations.ts b/tests/runner-e2e/run-observations.tsindex e0f18d49ce..1e5b1949e2 100644--- a/tests/runner-e2e/run-observations.ts+++ b/tests/runner-e2e/run-observations.ts@@ -1,3 +1,28 @@+/** Read the complete durable stream; never grade a silently truncated page. */+export async function collectRunEvents<T extends { seq?: number }>(+ loadPage: (afterSeq: number, limit: number) => Promise<unknown>,+): Promise<T[]> {+ const pageSize = 1000;+ const events: T[] = [];+ let afterSeq = 0;+ for (let pageNumber = 0; pageNumber < 100; pageNumber += 1) {+ const page = await loadPage(afterSeq, pageSize);+ if (!Array.isArray(page) || page.length > pageSize) {+ throw new Error("Run event evidence returned an invalid page");+ }+ for (const event of page) {+ const seq = event?.seq;+ if (!Number.isSafeInteger(seq) || seq <= afterSeq) {+ throw new Error("Run event evidence has a missing or non-increasing sequence");+ }+ afterSeq = seq;+ events.push(event as T);+ }+ if (page.length < pageSize) return events;+ }+ throw new Error("Run event evidence exceeded 100 pages; refusing incomplete evidence");+}+export interface ObservableRunState {status?: string | null;errorCode?: string | null;diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.tsindex 9649bf1da5..5e8392c529 100644--- a/tests/runner-e2e/runner.spec.ts+++ b/tests/runner-e2e/runner.spec.ts@@ -21,6 +21,7 @@ import { setupLiveFixtures, type LiveFixtureValues } from "./live-fixtures.js";import { evaluateMatcher, persistedFinalRunMessage, type MatcherResult } from "./matchers.js";import {acceptedPlanSessionResetFailures,+ collectRunEvents,hasTerminalMalformedPlanConfirmation,isControlPlaneGovernedResponseWait,isNonExecutingReviewFenceRun,@@ -696,8 +697,8 @@ for (const execution of executions) {),),events: await capture(() =>- api.get<RunEventRecord[]>(- `/api/heartbeat-runs/${candidate.id}/events?limit=1000`,+ collectRunEvents<RunEventRecord>((afterSeq, limit) =>+ api.get(`/api/heartbeat-runs/${candidate.id}/events?afterSeq=${afterSeq}&limit=${limit}`),),),})),@@ -1694,8 +1695,8 @@ for (const execution of executions) {try {return {runId: candidate.id,- events: await api.get<RunEventRecord[]>(- `/api/heartbeat-runs/${candidate.id}/events?limit=1000`,+ events: await collectRunEvents<RunEventRecord>((afterSeq, limit) =>+ api.get(`/api/heartbeat-runs/${candidate.id}/events?afterSeq=${afterSeq}&limit=${limit}`),),error: null,};diff --git a/tests/runner-e2e/support.test.ts b/tests/runner-e2e/support.test.tsindex e7c4cb95ff..9c71367fa0 100644--- a/tests/runner-e2e/support.test.ts+++ b/tests/runner-e2e/support.test.ts@@ -38,6 +38,7 @@ import {} from "./ports.js";import {acceptedPlanSessionResetFailures,+ collectRunEvents,hasTerminalMalformedPlanConfirmation,isControlPlaneGovernedResponseWait,isNonExecutingReviewFenceRun,@@ -48,6 +49,47 @@ import {import { runnerE2EWebServerCommand } from "./web-server-command.js";const cleanupDirectories: string[] = [];++describe("complete run event evidence", () => {+ const page = Array.from({ length: 1000 }, (_, i) => ({ seq: i + 1, eventType: "item.delta" }));+ it("reads completion events beyond the first 1000 rows", async () => {+ const terminal = ["run.result.proposed", "run.result.accepted", "run.terminal"]+ .map((eventType, i) => ({ seq: 1001 + i, eventType }));+ const load = vi.fn().mockResolvedValueOnce(page).mockResolvedValueOnce(terminal);+ const events = await collectRunEvents(load);+ expect(events).toEqual([...page, ...terminal]);+ expect(load.mock.calls).toEqual([[0, 1000], [1000, 1000]]);+ });+ it("checks for another page even at an exact page boundary", async () => {+ const load = vi.fn().mockResolvedValueOnce(page).mockResolvedValueOnce([]);+ expect(await collectRunEvents(load)).toEqual(page);+ expect(load).toHaveBeenCalledTimes(2);+ });+ it.each([+ null, {}, [{ eventType: "run.terminal" }], [{ seq: 0 }], [{ seq: -1 }],+ [{ seq: 1.5 }], [{ seq: "1" }], [{ seq: NaN }], [{ seq: Infinity }],+ [{ seq: 2 }, { seq: 1 }], [{ seq: 1 }, { seq: 1 }], [...page, { seq: 1001 }],+ ])("rejects malformed evidence page %#", async (malformed) => {+ await expect(collectRunEvents(async () => malformed)).rejects.toThrow("Run event evidence");+ });+ it("rejects a repeated cursor instead of accepting duplicate events", async () => {+ const load = vi.fn().mockResolvedValue(page);+ await expect(collectRunEvents(load)).rejects.toThrow("non-increasing sequence");+ expect(load).toHaveBeenCalledTimes(2);+ });+ it("propagates a missing later page without returning partial evidence", async () => {+ const load = vi.fn().mockResolvedValueOnce(page).mockRejectedValueOnce(new Error("Unavailable"));+ await expect(collectRunEvents(load)).rejects.toThrow("Unavailable");+ });+ it("fails closed when a stream never ends within the bounded capture", async () => {+ const load = vi.fn(async (afterSeq: number) => page.map((event) => ({ ...event, seq: event.seq + afterSeq })));+ await expect(collectRunEvents(load)).rejects.toThrow("refusing incomplete evidence");+ expect(load).toHaveBeenCalledTimes(100);+ });+});+++afterEach(async () => {vi.unstubAllEnvs();vi.unstubAllGlobals();