Skip to content

Automate browser based workflows with AI

Re-read a finalized run's timeline instead of latching a stale snapshot (SKY-15614) (#8446)

Aaron Perezdadc9f1c3038
diff --git a/skyvern-frontend/src/routes/workflows/hooks/useRunHealEpisodesQuery.test.tsx b/skyvern-frontend/src/routes/workflows/hooks/useRunHealEpisodesQuery.test.tsx
new file mode 100644
index 000000000..0a5e14aa0
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/hooks/useRunHealEpisodesQuery.test.tsx
@@ -0,0 +1,108 @@
+// @vitest-environment jsdom
+
+import type { ReactNode } from "react";
+import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
+import { renderHook, waitFor } from "@testing-library/react";
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+const { mockGet, mockGetClient, runState } = vi.hoisted(() => ({
+ mockGet: vi.fn(),
+ mockGetClient: vi.fn(),
+ runState: { status: "running" },
+}));
+
+vi.mock("@/api/AxiosClient", () => ({ getClient: mockGetClient }));
+vi.mock("@/hooks/useCredentialGetter", () => ({
+ useCredentialGetter: () => null,
+}));
+vi.mock("./useWorkflowRunWithWorkflowQuery", () => ({
+ useWorkflowRunWithWorkflowQuery: () => ({
+ data: {
+ status: runState.status,
+ workflow: { workflow_permanent_id: "wpid_1" },
+ },
+ dataUpdatedAt: 0,
+ }),
+}));
+
+import { useRunHealEpisodesQuery } from "./useRunHealEpisodesQuery";
+
+afterEach(() => {
+ vi.clearAllMocks();
+ runState.status = "running";
+});
+
+describe("useRunHealEpisodesQuery", () => {
+ // Episodes recorded after the reader unmounts are only picked up by the invalidation that
+ // follows a run-query poll, and a finalized run stops polling. Without the status in the key a
+ // reader that leaves mid-run and returns after the run finished is served the episodes as of
+ // the moment it left, for as long as that entry survives garbage collection.
+ it("re-reads episodes when a reader returns after the run finalized", async () => {
+ mockGet.mockResolvedValue({ data: { episodes: [] } });
+ mockGetClient.mockResolvedValue({ get: mockGet });
+ const queryClient = new QueryClient({
+ defaultOptions: { queries: { retry: false } },
+ });
+ function wrapper({ children }: { children: ReactNode }) {
+ return (
+ <QueryClientProvider client={queryClient}>
+ {children}
+ </QueryClientProvider>
+ );
+ }
+
+ const midRun = renderHook(
+ () => useRunHealEpisodesQuery({ workflowRunId: "wr_1" }),
+ { wrapper },
+ );
+ await waitFor(() => expect(midRun.result.current.isSuccess).toBe(true));
+ const midRunReads = mockGet.mock.calls.length;
+ midRun.unmount();
+
+ runState.status = "completed";
+ const afterRun = renderHook(
+ () => useRunHealEpisodesQuery({ workflowRunId: "wr_1" }),
+ { wrapper },
+ );
+
+ await waitFor(() => expect(afterRun.result.current.isSuccess).toBe(true));
+ expect(mockGet.mock.calls.length).toBeGreaterThan(midRunReads);
+ });
+
+ // The path the readers actually take when a run finishes under them: no remount, just the status
+ // swap onto an uncached key. RunHealChip and BlockHealPanel both render null on absent data, so
+ // dropping the previous episodes here blinks the chip and the panel out right at completion.
+ it("keeps episodes on screen while a mounted run finalizes", async () => {
+ const episodes = {
+ episodes: [{ workflow_run_block_id: "wrb_1" }],
+ summary: { blocks_with_heal_attempt: 1 },
+ };
+ mockGet.mockResolvedValue({ data: episodes });
+ mockGetClient.mockResolvedValue({ get: mockGet });
+ const queryClient = new QueryClient({
+ defaultOptions: { queries: { retry: false } },
+ });
+ function wrapper({ children }: { children: ReactNode }) {
+ return (
+ <QueryClientProvider client={queryClient}>
+ {children}
+ </QueryClientProvider>
+ );
+ }
+
+ const { result, rerender } = renderHook(
+ () => useRunHealEpisodesQuery({ workflowRunId: "wr_1" }),
+ { wrapper },
+ );
+ await waitFor(() => expect(result.current.isSuccess).toBe(true));
+ const readsWhileLive = mockGet.mock.calls.length;
+
+ runState.status = "completed";
+ rerender();
+
+ expect(result.current.data).toEqual(episodes);
+ await waitFor(() =>
+ expect(mockGet.mock.calls.length).toBeGreaterThan(readsWhileLive),
+ );
+ });
+});
diff --git a/skyvern-frontend/src/routes/workflows/hooks/useRunHealEpisodesQuery.ts b/skyvern-frontend/src/routes/workflows/hooks/useRunHealEpisodesQuery.ts
index c8a7cde33..bd8c538bf 100644
--- a/skyvern-frontend/src/routes/workflows/hooks/useRunHealEpisodesQuery.ts
+++ b/skyvern-frontend/src/routes/workflows/hooks/useRunHealEpisodesQuery.ts
@@ -2,7 +2,11 @@ import { useEffect, useRef } from "react";
import { getClient } from "@/api/AxiosClient";
import { useCredentialGetter } from "@/hooks/useCredentialGetter";
import { statusIsNotFinalized } from "@/routes/tasks/types";
-import { useQuery, useQueryClient } from "@tanstack/react-query";
+import {
+ keepPreviousData,
+ useQuery,
+ useQueryClient,
+} from "@tanstack/react-query";
import type { RunHealEpisodesResponse } from "../types/healTypes";
import { useWorkflowRunWithWorkflowQuery } from "./useWorkflowRunWithWorkflowQuery";
@@ -35,13 +39,19 @@ function useRunHealEpisodesQuery({
}, [dataUpdatedAt, workflowRunId, queryClient]);
return useQuery<RunHealEpisodesResponse>({
- queryKey: ["run-heal-episodes", workflowRunId],
+ // Same reason the run timeline carries its status: the invalidation above only fires while the
+ // run query is still polling, so a reader that unmounts mid-run and comes back after the run
+ // finished would otherwise be served the episodes as of the unmount.
+ queryKey: ["run-heal-episodes", workflowRunId, workflowRun?.status],
queryFn: async () => {
const client = await getClient(credentialGetter, "sans-api-v1");
return client
.get<RunHealEpisodesResponse>(`/runs/${workflowRunId}/heal_episodes`)
.then((response) => response.data);
},
+ // The status swap lands on an uncached key, and both readers render null on absent data — so
+ // without this the heal chip and the block heal panel blink out as the run finishes.
+ placeholderData: keepPreviousData,
refetchOnMount:
workflowRun && statusIsNotFinalized(workflowRun) ? "always" : false,
refetchOnWindowFocus:
diff --git a/skyvern-frontend/src/routes/workflows/hooks/useWorkflowRunTimelineQuery.test.tsx b/skyvern-frontend/src/routes/workflows/hooks/useWorkflowRunTimelineQuery.test.tsx
index 1870e78c0..6b43f61b5 100644
--- a/skyvern-frontend/src/routes/workflows/hooks/useWorkflowRunTimelineQuery.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/hooks/useWorkflowRunTimelineQuery.test.tsx
@@ -1,13 +1,18 @@
// @vitest-environment jsdom
import type { ReactNode } from "react";
-import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
+import {
+ focusManager,
+ QueryClient,
+ QueryClientProvider,
+} from "@tanstack/react-query";
import { renderHook, waitFor } from "@testing-library/react";
import { afterEach, describe, expect, it, vi } from "vitest";
-const { mockGet, mockGetClient } = vi.hoisted(() => ({
+const { mockGet, mockGetClient, runState } = vi.hoisted(() => ({
mockGet: vi.fn(),
mockGetClient: vi.fn(),
+ runState: { status: "completed" },
}));
vi.mock("@/api/AxiosClient", () => ({
@@ -25,14 +30,17 @@ vi.mock("./useGlobalWorkflowsQuery", () => ({
vi.mock("./useWorkflowRunWithWorkflowQuery", () => ({
useWorkflowRunWithWorkflowQuery: () => ({
data: {
- status: "completed",
+ status: runState.status,
workflow: { workflow_permanent_id: "wpid_1" },
},
dataUpdatedAt: 0,
}),
}));
-import { useWorkflowRunTimelineQuery } from "./useWorkflowRunTimelineQuery";
+import {
+ RUNNING_TIMELINE_REFETCH_INTERVAL_MS,
+ useWorkflowRunTimelineQuery,
+} from "./useWorkflowRunTimelineQuery";
function wrapper({ children }: { children: ReactNode }) {
const queryClient = new QueryClient({
@@ -45,6 +53,9 @@ function wrapper({ children }: { children: ReactNode }) {
afterEach(() => {
vi.clearAllMocks();
+ vi.useRealTimers();
+ focusManager.setFocused(undefined);
+ runState.status = "completed";
});
describe("useWorkflowRunTimelineQuery", () => {
@@ -79,4 +90,64 @@ describe("useWorkflowRunTimelineQuery", () => {
// Callers guard on `undefined`; a non-array must never reach them.
expect(result.current.data).toBeUndefined();
});
+
+ // The poll cannot be the thing that reads a run's last blocks: the run-status query can report
+ // the terminal state between two timeline ticks, which cancels the timer before it fires again.
+ // The reader stays mounted through that, so no mount or focus refetch covers it either, and the
+ // timeline keeps whatever partial snapshot it last saw — rendered as blocks that "did not
+ // execute", or as empty when the run had not written its first block yet.
+ it.each([
+ ["nothing yet", []],
+ ["a partial timeline", [{ type: "block" }]],
+ ])(
+ "re-reads when a mounted run finalizes having read %s",
+ async (_label, body) => {
+ runState.status = "running";
+ mockGet.mockResolvedValue({ data: body });
+ mockGetClient.mockResolvedValue({ get: mockGet });
+
+ const { result, rerender } = renderHook(
+ () => useWorkflowRunTimelineQuery(),
+ { wrapper },
+ );
+ await waitFor(() => expect(result.current.isSuccess).toBe(true));
+ const readsWhileLive = mockGet.mock.calls.length;
+
+ runState.status = "completed";
+ rerender();
+
+ // The swap lands on a key with nothing cached, so without keepPreviousData every reader
+ // gating on isLoading would flash a skeleton the moment a run completes.
+ expect(result.current.data).toEqual(body);
+ expect(result.current.isLoading).toBe(false);
+
+ await waitFor(() =>
+ expect(mockGet.mock.calls.length).toBeGreaterThan(readsWhileLive),
+ );
+ },
+ );
+
+ // Background polling exists so a run watched from another window still fills in its rows. Only a
+ // running run writes any, and a paused one waits on a human — polling through that would turn a
+ // tab left open on a paused run into hours of requests it never made before.
+ it.each([
+ ["polls through a running run", "running", true],
+ ["stays quiet through a paused run", "paused", false],
+ ])("a backgrounded tab %s", async (_label, status, expectedToPoll) => {
+ vi.useFakeTimers({ shouldAdvanceTime: true });
+ runState.status = status;
+ mockGet.mockResolvedValue({ data: [] });
+ mockGetClient.mockResolvedValue({ get: mockGet });
+
+ const { result } = renderHook(() => useWorkflowRunTimelineQuery(), {
+ wrapper,
+ });
+ await waitFor(() => expect(result.current.isSuccess).toBe(true));
+
+ focusManager.setFocused(false);
+ const readsWhileFocused = mockGet.mock.calls.length;
+ await vi.advanceTimersByTimeAsync(3 * RUNNING_TIMELINE_REFETCH_INTERVAL_MS);
+
+ expect(mockGet.mock.calls.length > readsWhileFocused).toBe(expectedToPoll);
+ });
});
diff --git a/skyvern-frontend/src/routes/workflows/hooks/useWorkflowRunTimelineQuery.ts b/skyvern-frontend/src/routes/workflows/hooks/useWorkflowRunTimelineQuery.ts
index 0efc28cdc..08dc40a3a 100644
--- a/skyvern-frontend/src/routes/workflows/hooks/useWorkflowRunTimelineQuery.ts
+++ b/skyvern-frontend/src/routes/workflows/hooks/useWorkflowRunTimelineQuery.ts
@@ -1,4 +1,5 @@
import { getClient } from "@/api/AxiosClient";
+import { Status } from "@/api/types";
import { useCredentialGetter } from "@/hooks/useCredentialGetter";
import { statusIsNotFinalized } from "@/routes/tasks/types";
import { keepPreviousData, useQuery } from "@tanstack/react-query";
@@ -26,10 +27,25 @@ function useWorkflowRunTimelineQuery(options?: { workflowRunId?: string }) {
const { data: workflowRun } = useWorkflowRunWithWorkflowQuery(options);
const workflow = workflowRun?.workflow;
const workflowPermanentId = workflow?.workflow_permanent_id;
+ const runIsLive = !!workflowRun && statusIsNotFinalized(workflowRun);
+ // Only a running run writes timeline rows. Created, queued and paused are live but idle, and a
+ // paused run waits on a human for as long as that takes.
+ const runIsWriting = workflowRun?.status === Status.Running;
return useQuery<Array<WorkflowRunTimelineItem>>({
+ // The run's status is part of what was read: a timeline fetched while the run was still
+ // writing is a different, incomplete answer than one fetched after it stopped. Keying on it
+ // makes reaching a terminal state re-read exactly once — the poll cannot be relied on for
+ // that, because the run-status query can report the terminal state between two ticks and
+ // cancel the timer before it fires again. keepPreviousData holds the last rows on screen
+ // across the swap.
queryKey: getOrgScopedQueryKey(
- ["workflowRunTimeline", workflowPermanentId, workflowRunId],
+ [
+ "workflowRunTimeline",
+ workflowPermanentId,
+ workflowRunId,
+ workflowRun?.status,
+ ],
activeOrgQueryKeyScope,
),
queryFn: async ({ signal }) => {
@@ -61,17 +77,17 @@ function useWorkflowRunTimelineQuery(options?: { workflowRunId?: string }) {
if (query.state.status === "error") {
return false;
}
- return workflowRun && statusIsNotFinalized(workflowRun)
- ? RUNNING_TIMELINE_REFETCH_INTERVAL_MS
- : false;
+ return runIsLive ? RUNNING_TIMELINE_REFETCH_INTERVAL_MS : false;
},
+ // The interval otherwise pauses while the window is unfocused, and a run watched from another
+ // window keeps writing blocks the whole time. Scoped to a running run so a backgrounded tab
+ // does not poll through an idle one — a paused run can sit there for hours.
+ refetchIntervalInBackground: runIsWriting,
placeholderData: keepPreviousData,
- refetchOnMount:
- workflowRun && statusIsNotFinalized(workflowRun) ? "always" : false,
- refetchOnWindowFocus:
- workflowRun && statusIsNotFinalized(workflowRun) ? "always" : false,
+ refetchOnMount: runIsLive ? "always" : false,
+ refetchOnWindowFocus: runIsLive ? "always" : false,
enabled: !!globalWorkflows && !!workflowPermanentId && !!workflowRunId,
});
}
-export { useWorkflowRunTimelineQuery };
+export { RUNNING_TIMELINE_REFETCH_INTERVAL_MS, useWorkflowRunTimelineQuery };