The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix: stop repeated notifications for finalized run failures (#13739)
Dottafd071748eec9
diff --git a/DESIGN.md b/DESIGN.mdindex 16f428bffa..d80f6a7075 100644--- a/DESIGN.md+++ b/DESIGN.md@@ -48,7 +48,10 @@ alignment in every step and conditional state, not just the first screen.Do not show a toast for task or run state already visible on the current screen.This includes descendant runs represented by the open subtree. Show local actionresults in place; keep failures actionable inline. Notifications for other work-remain useful. Expected cancellation is neutral gray, not an error. The composer's Stop action stops the current response and leaves the composer available for a new message. Pause work is a separate explicit task or subtree action. A paused task replaces the composer with an amber takeover. It says “Task is+remain useful. Repeated delivery of the same run outcome must refresh cached state+without repeating its toast, including after reconnecting. A terminal outcome+delivered more than five minutes after the run finished is historical and should+refresh state silently. Expected cancellation is neutral gray, not an error. The composer's Stop action stops the current response and leaves the composer available for a new message. Pause work is a separate explicit task or subtree action. A paused task replaces the composer with an amber takeover. It says “Task ispaused.” and “Resume this task to send a message.” with a “Resume task” action.Subtrees use “Subtree is paused.” and “Resume subtree.” The takeover cannot bedismissed, retains drafts, and hides message inputs until the pause is released.diff --git a/server/src/services/native-runtime/native-run-finalizer-telemetry.test.ts b/server/src/services/native-runtime/native-run-finalizer-telemetry.test.tsindex 4eb79127f5..aeaa9f0f35 100644--- a/server/src/services/native-runtime/native-run-finalizer-telemetry.test.ts+++ b/server/src/services/native-runtime/native-run-finalizer-telemetry.test.ts@@ -48,6 +48,7 @@ function captureRunFailureCallsFrom(fromIndex: number) {import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js";import { finalizeNativeRun, recordNativeFinalizationFailure } from "./native-run-finalizer.js";+import { deliverExecutionStatuses } from "../execution-status-delivery.js";import { commitNativeStatusDecision } from "./status-decision-committer.js";import { NATIVE_STATUS_ARBITER_POLICY_VERSION, type NativeStatusDecision } from "./status-arbiter.js";@@ -416,7 +417,7 @@ describeEmbeddedPostgres("native run finalizer / status decision committer — a},);- it("emits zero events when a reconciliation replay commits the same failed terminal result again", async () => {+ it.each(["in_progress", "cancelled"])("does not redeliver a committed failure during reconciliation of a %s task", async (issueStatus) => {const fixture = await seedNativeRun();await driveToCompleteResult(fixture, {...CONTROL_PLANE_CONFORMANCE_TERMINAL,@@ -450,24 +451,63 @@ describeEmbeddedPostgres("native run finalizer / status decision committer — arunStatus: "failed",});+ await db.update(issues).set({ status: issueStatus }).where(eq(issues.id, fixture.issueId));+ const publish = vi.fn();+ await deliverExecutionStatuses(db, { publish });+ expect(publish.mock.calls.filter(([event]) => event.payload.runId === fixture.runId)).toHaveLength(1);+ const [beforeReplay] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId));+ expect(beforeReplay.executionStatusDeliveryId).toBeNull();+ publish.mockClear();+const callsBefore = mockTelemetryClient.track.mock.calls.length;const captureCallsBeforeReplay = mockCaptureRunFailure.mock.calls.length;- // The coordinator is already "committed" with a "failed" terminal- // result, and the run row is already "failed". Like "succeeded",- // "failed" sits inside projectCommittedRun's WHERE clause, so its write- // matches the row. The write changes nothing (failed -> failed), so it- // must not emit a second event for the same committed result, and it- // must not report a second Sentry event either.- await finalizeNativeRun({- db,- runId: fixture.runId,- workspaceFinalizeStatus: "succeeded",- projectRunStatus: true,+ // Simulate several periodic sweeps, including overlapping reconciliation.+ for (let sweep = 0; sweep < 3; sweep += 1) {+ await Promise.all([0, 1].map(() => finalizeNativeRun({+ db,+ runId: fixture.runId,+ workspaceFinalizeStatus: "succeeded",+ projectRunStatus: true,+ })));+ await deliverExecutionStatuses(db, { publish });+ }+ expect(publish.mock.calls.filter(([event]) => event.payload.runId === fixture.runId)).toHaveLength(0);+ const [afterReplay] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId));+ expect(afterReplay).toMatchObject({+ executionStatusDeliveryId: null,+ updatedAt: beforeReplay.updatedAt,+ nativePhaseUpdatedAt: beforeReplay.nativePhaseUpdatedAt,+ finishedAt: beforeReplay.finishedAt,});expect(agentTaskRunCalls(callsBefore)).toHaveLength(0);expect(captureRunFailureCallsFrom(captureCallsBeforeReplay)).toHaveLength(0);});+ it("repairs an incomplete committed failure once and preserves its pending delivery on replay", async () => {+ const fixture = await seedNativeRun();+ await driveToCompleteResult(fixture, {+ ...CONTROL_PLANE_CONFORMANCE_TERMINAL,+ turnTerminalState: "failed",+ runTerminalState: "failed",+ });+ const finalize = () => finalizeNativeRun({ db, runId: fixture.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });+ await finalize();+ await db.update(heartbeatRuns).set({+ finishedAt: null,+ nativePhase: "arbitrating",+ executionStatusDeliveryId: null,+ }).where(eq(heartbeatRuns.id, fixture.runId));+ await finalize();+ const [repaired] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId));+ expect(repaired.status).toBe("failed");+ expect(repaired.nativePhase).toBe("committed");+ expect(repaired.finishedAt).toBeInstanceOf(Date);+ expect(repaired.executionStatusDeliveryId).toBeTypeOf("string");+ await finalize();+ const [replayed] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId));+ expect(replayed).toEqual(repaired);+ });+it("emits zero events when a retryable-failure write's conditional status spread is omitted", async () => {const fixture = await seedNativeRun();// recordNativeFinalizationFailure only needs the run and coordinator rowsdiff --git a/server/src/services/native-runtime/native-run-finalizer.ts b/server/src/services/native-runtime/native-run-finalizer.tsindex 48c8ff358a..8dfc177aef 100644--- a/server/src/services/native-runtime/native-run-finalizer.ts+++ b/server/src/services/native-runtime/native-run-finalizer.ts@@ -569,14 +569,15 @@ async function projectCommittedRun(input: {if (!["succeeded", "failed", "cancelled"].includes(String(terminalState))) {throw new Error("native_finalization_invalid");}+ const projectedStatus = projectNativeTerminalRunStatus(+ terminalState as "succeeded" | "failed" | "cancelled",+ );const now = new Date();const [updatedRun] = await input.db.update(heartbeatRuns).set({executionStatusDeliveryId: randomUUID(),- status: projectNativeTerminalRunStatus(- terminalState as "succeeded" | "failed" | "cancelled",- ),+ status: projectedStatus,finishedAt: input.run.finishedAt ?? now,nativePhase: "committed",nativePhaseUpdatedAt: now,@@ -619,17 +620,24 @@ async function projectCommittedRun(input: {),),),+ // Reconciliation revisits committed results periodically. Only repair+ // a changed projection; rewriting an unchanged failed run would mint a+ // fresh status delivery (and failure toast) on every sweep. Check the+ // current row so concurrent replays cannot both queue the same repair.+ or(+ sql`${heartbeatRuns.status} is distinct from ${projectedStatus}`,+ isNull(heartbeatRuns.finishedAt),+ sql`${heartbeatRuns.nativePhase} is distinct from 'committed'`,+ ...(terminalState === "succeeded"+ ? [isNotNull(heartbeatRuns.error), isNotNull(heartbeatRuns.errorCode)]+ : []),+ ),nativeRunnerOwnershipNotHeldCondition(),),).returning();- // The WHERE clause above allows "failed" as a source status, so a run that- // failed before its coordinator committed can still pick up the committed- // terminal state. A later reconciliation replay can enter this same path- // and match that clause again even when the committed state is still- // "failed" — the write changes nothing. Emit only when the status- // genuinely changed, so a reconciliation replay never emits twice for one- // committed terminal result.+ // Metadata repairs can preserve the terminal status. Only a genuine status+ // transition should emit another terminal event.if (updatedRun && updatedRun.status !== input.run.status) {await emitAgentTaskRun(input.db, updatedRun);void reportRunFailure(input.db, updatedRun);diff --git a/ui/src/context/LiveUpdatesProvider.hook.test.tsx b/ui/src/context/LiveUpdatesProvider.hook.test.tsxindex 41546fb571..8650bfc355 100644--- a/ui/src/context/LiveUpdatesProvider.hook.test.tsx+++ b/ui/src/context/LiveUpdatesProvider.hook.test.tsx@@ -170,17 +170,64 @@ describe("LiveUpdatesProvider socket run notification scope", () => {vi.unstubAllGlobals();});- async function receiveStatus(payload: Record<string, unknown>) {+ async function receiveStatus(payload: Record<string, unknown>, createdAt = "2026-09-09T18:00:00.000Z") {await reactAct(async () => {root!.render(<QueryClientProvider client={queryClient}><LiveUpdatesProvider><span>Visible task</span></LiveUpdatesProvider></QueryClientProvider>);});await vi.waitFor(() => expect(sockets).toHaveLength(1));expect(sockets[0]?.onmessage).toBeTypeOf("function");await reactAct(async () => sockets[0]!.onmessage!(new MessageEvent("message", {- data: JSON.stringify({ id: 1, companyId: "company-1", type: "heartbeat.run.status", createdAt: "2026-09-09T18:00:00.000Z", payload }),+ data: JSON.stringify({ id: Date.now(), companyId: "company-1", type: "heartbeat.run.status", createdAt, payload }),})));}+ it("announces a run outcome once across minute-apart redeliveries but still refreshes caches", async () => {+ const clock = vi.spyOn(Date, "now");+ const payload = { runId: "unrelated-run", agentId: "other-agent", status: "failed", finishedAt: "2026-09-09T18:00:00.000Z" };+ clock.mockReturnValue(Date.parse(payload.finishedAt));+ await receiveStatus({ ...payload, deliveryId: "first" });+ expect(pushToast).toHaveBeenCalledTimes(1);+ queryClient.setQueryData(queryKeys.runDetail(payload.runId), { id: payload.runId, status: "running" });+ for (let minute = 1; minute <= 3; minute += 1) {+ const now = Date.parse(payload.finishedAt) + minute * 60_000;+ clock.mockReturnValue(now);+ await receiveStatus({ ...payload, deliveryId: `retry-${minute}` }, new Date(now).toISOString());+ }+ expect(pushToast).toHaveBeenCalledTimes(1);+ expect(queryClient.getQueryData(queryKeys.runDetail(payload.runId))).toMatchObject({ status: "failed" });+ await receiveStatus({ ...payload, runId: "new-failure" });+ expect(pushToast).toHaveBeenCalledTimes(2);+ });++ it("refreshes an old terminal run without announcing its first delivery as a new failure", async () => {+ queryClient.setQueryData(queryKeys.runDetail("old-run"), { id: "old-run", status: "running" });+ await receiveStatus({+ runId: "old-run", agentId: "other-agent", status: "failed",+ finishedAt: "2026-09-07T18:00:00.000Z", deliveryId: "new-delivery-of-old-failure",+ });+ expect(pushToast).not.toHaveBeenCalled();+ expect(queryClient.getQueryData(queryKeys.runDetail("old-run"))).toMatchObject({ status: "failed" });+ });++ it("does not announce an outcome again after hiding and reopening the window", async () => {+ const payload = { runId: "unrelated-run", agentId: "other-agent", status: "failed" };+ await receiveStatus(payload);+ const visibility = vi.spyOn(document, "visibilityState", "get");+ await reactAct(async () => {+ visibility.mockReturnValue("hidden");+ document.dispatchEvent(new Event("visibilitychange"));+ });+ await reactAct(async () => {+ visibility.mockReturnValue("visible");+ document.dispatchEvent(new Event("visibilitychange"));+ });+ await vi.waitFor(() => expect(sockets).toHaveLength(2));+ await reactAct(async () => sockets[1]!.onmessage!(new MessageEvent("message", {+ data: JSON.stringify({ id: 2, companyId: "company-1", type: "heartbeat.run.status", createdAt: "2026-09-09T18:01:00.000Z", payload }),+ })));+ expect(pushToast).toHaveBeenCalledTimes(1);+ });+it("disconnects while hidden and reconciles active queries once on return", async () => {await receiveStatus({ runId: "child-run", agentId: "child-agent", status: "running" });const invalidate = vi.spyOn(queryClient, "invalidateQueries");diff --git a/ui/src/context/LiveUpdatesProvider.tsx b/ui/src/context/LiveUpdatesProvider.tsxindex 8ea8c42dd0..f86c71b47b 100644--- a/ui/src/context/LiveUpdatesProvider.tsx+++ b/ui/src/context/LiveUpdatesProvider.tsx@@ -54,6 +54,8 @@ import { tryCreateWebSocket } from "../lib/websocket";const TOAST_COOLDOWN_WINDOW_MS = 10_000;const TOAST_COOLDOWN_MAX = 3;const RECONNECT_SUPPRESS_MS = 2000;+const RUN_TOAST_MAX_AGE_MS = 5 * 60_000;+const MAX_OBSERVED_RUN_OUTCOMES = 2000;const DISCONNECTED_POLL_INTERVAL_MS = 15_000;const SOCKET_CONNECTING = 0;const SOCKET_OPEN = 1;@@ -1587,6 +1589,29 @@ function invalidateActivityQueries(interface ToastGate {cooldownHits: Map<string, number[]>;suppressUntil: number;+ observedRunOutcomes: Set<string>;+}++function observeRunOutcome(gate: ToastGate, event: LiveEvent): boolean {+ const payload = event.payload ?? {};+ const runId = readString(payload.runId);+ const status = readString(payload.status);+ if (!runId || !status || !TERMINAL_RUN_STATUSES.has(status)) return false;+ const key = `${event.companyId}:${runId}:${status}`;+ const observed = gate.observedRunOutcomes.has(key);+ // Refresh insertion order so repeated deliveries stay remembered even in a+ // busy company. Remember suppressed outcomes too (visible task/reconnect).+ gate.observedRunOutcomes.delete(key);+ gate.observedRunOutcomes.add(key);+ if (gate.observedRunOutcomes.size > MAX_OBSERVED_RUN_OUTCOMES) {+ gate.observedRunOutcomes.delete(gate.observedRunOutcomes.values().next().value!);+ }+ // Use the two server timestamps: delivery time is not failure time. This+ // also keeps historical failures quiet after a reload, without client clock+ // skew hiding fresh failures. Missing timestamps retain legacy behavior.+ const finishedAt = Date.parse(readString(payload.finishedAt) ?? "");+ const deliveredAt = Date.parse(event.createdAt);+ return observed || deliveredAt - finishedAt > RUN_TOAST_MAX_AGE_MS;}function shouldSuppressToast(gate: ToastGate, category: string): boolean {@@ -1636,7 +1661,8 @@ function handleLiveEvent(// Resolve membership before terminal lifecycle patches remove live-run rows.const suppressRunToast =event.type === "heartbeat.run.status" &&- shouldSuppressRunStatusToastForVisibleIssue(queryClient, pathname, payload);+ (observeRunOutcome(gate, event) ||+ shouldSuppressRunStatusToastForVisibleIssue(queryClient, pathname, payload));const liveStatusPatch = readRunLiveStatusPatchFromPayload(payload,event.createdAt,@@ -1837,6 +1863,7 @@ export function LiveUpdatesProvider({ children }: { children: ReactNode }) {const gateRef = useRef<ToastGate>({cooldownHits: new Map(),suppressUntil: 0,+ observedRunOutcomes: new Set(),});const pathnameRef = useRef(location.pathname);const { data: session, status: sessionStatus } = useQuery({