Skip to content

Automate browser based workflows with AI

fix(SKY-11560): mirror legacy run page in studio run-tab hero (per-action & per-block) (#6930)

Celal Zamanoğlua23465353a6b
diff --git a/skyvern-frontend/src/routes/workflows/studio/runProjections.ts b/skyvern-frontend/src/routes/workflows/studio/runProjections.ts
index e56fded13..db31cf984 100644
--- a/skyvern-frontend/src/routes/workflows/studio/runProjections.ts
+++ b/skyvern-frontend/src/routes/workflows/studio/runProjections.ts
@@ -77,7 +77,7 @@ export type FilmstripFrame = {
actionOrder: number | null;
};
-function actionLabel(action: ActionsApiResponse): string {
+export function actionLabel(action: ActionsApiResponse): string {
const candidate =
action.intention?.trim() ||
action.description?.trim() ||
diff --git a/skyvern-frontend/src/routes/workflows/studio/runview/HeroScreenshot.test.tsx b/skyvern-frontend/src/routes/workflows/studio/runview/HeroScreenshot.test.tsx
new file mode 100644
index 000000000..d27432b9e
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/studio/runview/HeroScreenshot.test.tsx
@@ -0,0 +1,84 @@
+// @vitest-environment jsdom
+
+import { cleanup, render, screen, waitFor } from "@testing-library/react";
+import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
+import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
+
+const { getSpy } = vi.hoisted(() => ({ getSpy: vi.fn() }));
+
+vi.mock("@/api/AxiosClient", () => ({
+ getClient: async () => ({
+ get: (url: string) => {
+ getSpy(url);
+ return Promise.resolve({ data: null });
+ },
+ }),
+}));
+
+vi.mock("@/hooks/useCredentialGetter", () => ({
+ useCredentialGetter: () => undefined,
+}));
+
+import { HeroScreenshot, type HeroSelection } from "./HeroScreenshot";
+
+function renderHero(selection: HeroSelection | null) {
+ const client = new QueryClient({
+ defaultOptions: { queries: { retry: false } },
+ });
+ return render(
+ <QueryClientProvider client={client}>
+ <HeroScreenshot selection={selection} running={false} />
+ </QueryClientProvider>,
+ );
+}
+
+beforeEach(() => getSpy.mockReset());
+afterEach(cleanup);
+
+describe("HeroScreenshot selection", () => {
+ test("fetches the action's own screenshot artifact when an artifactId is present", async () => {
+ renderHero({
+ kind: "action",
+ artifactId: "art_1",
+ stepId: "stp_1",
+ actionOrder: 0,
+ });
+ await waitFor(() => expect(getSpy).toHaveBeenCalledTimes(1));
+ expect(getSpy.mock.calls[0]![0]).toContain("/artifacts/art_1");
+ });
+
+ test("falls back to the step's action screenshots when the action has no artifactId", async () => {
+ renderHero({
+ kind: "action",
+ artifactId: null,
+ stepId: "stp_1",
+ actionOrder: 0,
+ });
+ await waitFor(() => expect(getSpy).toHaveBeenCalledTimes(1));
+ expect(getSpy.mock.calls[0]![0]).toContain("/step/stp_1/artifacts");
+ });
+
+ test("fetches block artifacts for a block selection", async () => {
+ renderHero({
+ kind: "block",
+ workflowRunBlockId: "wrb_1",
+ blockType: "task",
+ });
+ await waitFor(() => expect(getSpy).toHaveBeenCalledTimes(1));
+ expect(getSpy.mock.calls[0]![0]).toContain(
+ "/workflow_run_block/wrb_1/artifacts",
+ );
+ });
+
+ test("fetches thought artifacts for a thought selection", async () => {
+ renderHero({ kind: "thought", thoughtId: "tht_1" });
+ await waitFor(() => expect(getSpy).toHaveBeenCalledTimes(1));
+ expect(getSpy.mock.calls[0]![0]).toContain("/thought/tht_1/artifacts");
+ });
+
+ test("renders the empty state and fetches nothing without a selection", () => {
+ renderHero(null);
+ screen.getByText("No screenshot for this selection.");
+ expect(getSpy).not.toHaveBeenCalled();
+ });
+});
diff --git a/skyvern-frontend/src/routes/workflows/studio/runview/HeroScreenshot.tsx b/skyvern-frontend/src/routes/workflows/studio/runview/HeroScreenshot.tsx
index a9c435aab..311dfe241 100644
--- a/skyvern-frontend/src/routes/workflows/studio/runview/HeroScreenshot.tsx
+++ b/skyvern-frontend/src/routes/workflows/studio/runview/HeroScreenshot.tsx
@@ -3,7 +3,7 @@ import { ReloadIcon } from "@radix-ui/react-icons";
import { useQuery } from "@tanstack/react-query";
import { getClient } from "@/api/AxiosClient";
-import { ArtifactApiResponse } from "@/api/types";
+import { ArtifactApiResponse, ArtifactType } from "@/api/types";
import { useCredentialGetter } from "@/hooks/useCredentialGetter";
import { getImageURL } from "@/routes/tasks/detail/artifactUtils";
import { apiPathPrefix } from "@/util/env";
@@ -11,35 +11,92 @@ import { apiPathPrefix } from "@/util/env";
import { selectBlockScreenshot } from "../../workflowRun/blockScreenshot";
import { screenshotZoomClasses } from "./HeroScreenshot.utils";
+export type HeroSelection =
+ | {
+ kind: "action";
+ artifactId: string | null;
+ stepId: string | null;
+ actionOrder: number | null;
+ }
+ | {
+ kind: "block";
+ workflowRunBlockId: string;
+ blockType: string | null;
+ }
+ | {
+ kind: "thought";
+ thoughtId: string;
+ };
+
/**
- * The active block's screenshot, fit to the run-hero width and scrollable for long
- * captures. Prefers the full-page LLM screenshot (what the agent saw) over the
- * per-action screenshot via `selectBlockScreenshot`, matching the legacy run page.
+ * The selected element's screenshot, fit to the run-hero width and scrollable for
+ * long captures. Mirrors the legacy run page: an action shows its own post-action
+ * screenshot (by artifact id, falling back to the step's action screenshots), a
+ * block shows its representative screenshot via `selectBlockScreenshot`.
*/
export function HeroScreenshot({
- workflowRunBlockId,
- blockType,
+ selection,
running,
}: {
- workflowRunBlockId: string | null;
- blockType: string | null;
+ selection: HeroSelection | null;
running: boolean;
}) {
const credentialGetter = useCredentialGetter();
const [zoomed, setZoomed] = useState(false);
const containerRef = useRef<HTMLDivElement>(null);
- const { data: artifacts, isLoading } = useQuery<Array<ArtifactApiResponse>>({
- queryKey: ["workflowRunBlock", workflowRunBlockId, "artifacts"],
+ const action = selection?.kind === "action" ? selection : null;
+ const block = selection?.kind === "block" ? selection : null;
+ const thought = selection?.kind === "thought" ? selection : null;
+
+ const { data: artifactById, isLoading: loadingArtifact } =
+ useQuery<ArtifactApiResponse>({
+ queryKey: ["artifact", action?.artifactId],
+ queryFn: async () => {
+ const client = await getClient(credentialGetter, "sans-api-v1");
+ return client
+ .get(`/artifacts/${action!.artifactId}`)
+ .then((response) => response.data);
+ },
+ enabled: Boolean(action?.artifactId),
+ refetchOnWindowFocus: false,
+ staleTime: Infinity,
+ retry: 1,
+ });
+
+ // Fallback path only when the action carries no explicit screenshot id: the
+ // step's action screenshots, indexed by action order (newest-first), like legacy.
+ const useStepFallback = Boolean(action?.stepId) && !action?.artifactId;
+ const { data: stepArtifacts, isLoading: loadingStep } = useQuery<
+ Array<ArtifactApiResponse>
+ >({
+ queryKey: ["step", action?.stepId, "artifacts"],
+ queryFn: async () => {
+ const client = await getClient(credentialGetter);
+ return client
+ .get(`${apiPathPrefix}/step/${action!.stepId}/artifacts`)
+ .then((response) => response.data);
+ },
+ enabled: useStepFallback,
+ refetchInterval: running ? 5000 : false,
+ refetchOnWindowFocus: false,
+ staleTime: running ? 0 : Infinity,
+ retry: 1,
+ });
+
+ const { data: blockArtifacts, isLoading: loadingBlock } = useQuery<
+ Array<ArtifactApiResponse>
+ >({
+ queryKey: ["workflowRunBlock", block?.workflowRunBlockId, "artifacts"],
queryFn: async () => {
const client = await getClient(credentialGetter);
return client
.get(
- `${apiPathPrefix}/workflow_run_block/${workflowRunBlockId}/artifacts`,
+ `${apiPathPrefix}/workflow_run_block/${block!.workflowRunBlockId}/artifacts`,
)
.then((response) => response.data);
},
- enabled: Boolean(workflowRunBlockId),
+ enabled: Boolean(block?.workflowRunBlockId),
refetchInterval: running ? 5000 : false,
refetchOnWindowFocus: false,
// Artifacts are immutable once the run finishes; only keep polling while live.
@@ -47,8 +104,49 @@ export function HeroScreenshot({
retry: 1,
});
- const screenshot = selectBlockScreenshot(artifacts, blockType ?? undefined);
+ const { data: thoughtArtifacts, isLoading: loadingThought } = useQuery<
+ Array<ArtifactApiResponse>
+ >({
+ queryKey: ["observerThought", thought?.thoughtId, "artifacts"],
+ queryFn: async () => {
+ const client = await getClient(credentialGetter);
+ return client
+ .get(`${apiPathPrefix}/thought/${thought!.thoughtId}/artifacts`)
+ .then((response) => response.data);
+ },
+ enabled: Boolean(thought?.thoughtId),
+ refetchInterval: running ? 5000 : false,
+ refetchOnWindowFocus: false,
+ staleTime: running ? 0 : Infinity,
+ retry: 1,
+ });
+
+ let screenshot: ArtifactApiResponse | undefined;
+ if (action) {
+ const actionShots = stepArtifacts?.filter(
+ (artifact) => artifact.artifact_type === ArtifactType.ActionScreenshot,
+ );
+ const fromStep =
+ actionShots && action.actionOrder != null
+ ? actionShots[actionShots.length - action.actionOrder - 1]
+ : undefined;
+ screenshot = artifactById ?? fromStep;
+ } else if (block) {
+ screenshot = selectBlockScreenshot(
+ blockArtifacts,
+ block.blockType ?? undefined,
+ );
+ } else if (thought) {
+ const thoughtShots = thoughtArtifacts?.filter(
+ (artifact) => artifact.artifact_type === ArtifactType.LLMScreenshot,
+ );
+ // Thought LLM screenshots arrive newest-first; the last is the capture.
+ screenshot = thoughtShots?.[thoughtShots.length - 1];
+ }
+
const screenshotId = screenshot?.artifact_id ?? null;
+ const isLoading =
+ loadingArtifact || loadingStep || loadingBlock || loadingThought;
useEffect(() => {
setZoomed(false);
@@ -65,15 +163,15 @@ export function HeroScreenshot({
el.scrollLeft = zoomed ? (el.scrollWidth - el.clientWidth) / 2 : 0;
}, [zoomed, screenshotId]);
- if (!workflowRunBlockId) {
+ if (!selection) {
return (
<div className="absolute inset-0 grid place-items-center text-sm text-muted-foreground">
- No screenshot for this action.
+ No screenshot for this selection.
</div>
);
}
// Spinner only on the first load, not on poll refetches (which keep prior data).
- if (isLoading && !artifacts) {
+ if (isLoading && !screenshot) {
return (
<div className="absolute inset-0 flex items-center justify-center gap-2 text-sm text-muted-foreground">
<ReloadIcon className="h-5 w-5 animate-spin" /> Loading screenshot…
@@ -107,7 +205,7 @@ export function HeroScreenshot({
>
<img
src={getImageURL(screenshot)}
- alt="block screenshot"
+ alt="screenshot"
className={zoom.image}
/>
</div>
diff --git a/skyvern-frontend/src/routes/workflows/studio/runview/RunHero.test.tsx b/skyvern-frontend/src/routes/workflows/studio/runview/RunHero.test.tsx
index 1e18d2bcd..1daf6a841 100644
--- a/skyvern-frontend/src/routes/workflows/studio/runview/RunHero.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/studio/runview/RunHero.test.tsx
@@ -22,7 +22,8 @@ import { RunHero } from "./RunHero";
const baseProps = {
workflowRunId: "wr_1",
- shownFrame: null,
+ heroSelection: null,
+ heroLabel: "",
running: true,
provisioning: false,
isPaused: false,
diff --git a/skyvern-frontend/src/routes/workflows/studio/runview/RunHero.tsx b/skyvern-frontend/src/routes/workflows/studio/runview/RunHero.tsx
index e18e215fe..6581dba89 100644
--- a/skyvern-frontend/src/routes/workflows/studio/runview/RunHero.tsx
+++ b/skyvern-frontend/src/routes/workflows/studio/runview/RunHero.tsx
@@ -16,15 +16,15 @@ import { useRunViewStore } from "@/store/RunViewStore";
import { cn } from "@/util/utils";
import { WorkflowRunCode } from "../../workflowRun/WorkflowRunCode";
-import { FilmstripFrame } from "../runProjections";
import { useStudioShellContext } from "../StudioShellContext";
import { HeroRecording } from "./HeroRecording";
-import { HeroScreenshot } from "./HeroScreenshot";
+import { HeroScreenshot, type HeroSelection } from "./HeroScreenshot";
import { RunLiveStream } from "./RunLiveStream";
type RunHeroProps = {
workflowRunId: string;
- shownFrame: FilmstripFrame | null;
+ heroSelection: HeroSelection | null;
+ heroLabel: string;
running: boolean;
// A block run shows the shared debug-session stream (re-parented in by the
// shell), view-only, instead of mounting a separate run stream.
@@ -92,7 +92,8 @@ function ViewToggle({
export function RunHero({
workflowRunId,
- shownFrame,
+ heroSelection,
+ heroLabel,
running,
showDebugStream,
provisioning,
@@ -185,7 +186,7 @@ export function RunHero({
? "Inputs"
: center === "outputs"
? "Outputs"
- : (shownFrame?.label ?? "Screenshot");
+ : (heroLabel ?? "Screenshot");
const headerIcon =
center === "recording" ? (
@@ -355,23 +356,19 @@ export function RunHero({
recordingUrls={recordingUrls}
onPlay={onRecordingPlay}
/>
- ) : shownFrame ? (
- <HeroScreenshot
- workflowRunBlockId={shownFrame.blockId}
- blockType={shownFrame.blockType}
- running={running}
- />
+ ) : heroSelection ? (
+ <HeroScreenshot selection={heroSelection} running={running} />
) : (
<div className="absolute inset-0 grid place-items-center text-sm text-muted-foreground">
Waiting for the first action…
</div>
)}
- {center === "screenshot" && scrubbing && shownFrame ? (
+ {center === "screenshot" && scrubbing && heroSelection ? (
<div className="absolute left-3 top-3 flex max-w-[26rem] items-center gap-2 rounded-md bg-black/70 px-3 py-1.5 text-xs text-white backdrop-blur">
<CounterClockwiseClockIcon className="h-3.5 w-3.5 shrink-0" />
<span className="truncate">
- Inspecting step · <b>{shownFrame.label}</b>
+ Inspecting · <b>{heroLabel}</b>
</span>
{running || showDebugStream ? (
<button
diff --git a/skyvern-frontend/src/routes/workflows/studio/runview/RunView.tsx b/skyvern-frontend/src/routes/workflows/studio/runview/RunView.tsx
index f61d48ad8..a87295299 100644
--- a/skyvern-frontend/src/routes/workflows/studio/runview/RunView.tsx
+++ b/skyvern-frontend/src/routes/workflows/studio/runview/RunView.tsx
@@ -8,6 +8,11 @@ import { ApiWebhookActionsMenu } from "@/components/ApiWebhookActionsMenu";
import { WebhookReplayDialog } from "@/components/WebhookReplayDialog";
import { useApiCredential } from "@/hooks/useApiCredential";
import { statusIsFinalized } from "@/routes/tasks/types";
+import {
+ isAction,
+ isObserverThought,
+ isWorkflowRunBlock,
+} from "@/routes/workflows/types/workflowRunTypes";
import { useRunViewStore } from "@/store/RunViewStore";
import { useStudioShellStore } from "@/store/StudioShellStore";
import { type ApiCommandOptions } from "@/util/apiCommands";
@@ -22,14 +27,20 @@ import { WorkflowRunTimeline } from "../../workflowRun/WorkflowRunTimeline";
import { WorkflowRunVerificationCodeForm } from "../../workflowRun/WorkflowRunVerificationCodeForm";
import { pickDownloadedFileFilename } from "../../workflowRun/blockDownloadedFiles";
import { getRecordingUrls } from "../../workflowRun/recordingUrls";
-import { findActiveItem } from "../../workflowRun/workflowTimelineUtils";
+import {
+ findActiveItem,
+ findTimelineBlock,
+ resolveScreenshotBlockId,
+} from "../../workflowRun/workflowTimelineUtils";
import { getOrderedRunParameters } from "../../utils";
import {
+ actionLabel,
buildFilmstrip,
formatElapsed,
runOutcomeFromStatus,
} from "../runProjections";
import { RunHero } from "./RunHero";
+import { type HeroSelection } from "./HeroScreenshot";
import { buildRunFixMessage } from "./runFixMessage";
import { RunInputsSection, type RunInputMeta } from "./RunInputsSection";
import { RunOutputsSection, type RunOutputFile } from "./RunOutputsSection";
@@ -190,10 +201,6 @@ export function RunView({
);
const lastFrame = frames.length > 0 ? frames[frames.length - 1] : null;
- const shownFrame =
- (pinnedFrameId ? frames.find((f) => f.id === pinnedFrameId) : undefined) ??
- lastFrame ??
- null;
const finalized = workflowRun ? statusIsFinalized(workflowRun) : false;
const finallyBlockLabel =
@@ -206,6 +213,47 @@ export function RunView({
[timeline, selectedId, finalized, finallyBlockLabel],
);
+ // Mirror the legacy run page: an action shows its own screenshot, a block shows
+ // its representative screenshot (a loop/conditional container resolves to its leaf).
+ const heroSelection = useMemo<HeroSelection | null>(() => {
+ if (isAction(activeItem)) {
+ return {
+ kind: "action",
+ artifactId: activeItem.screenshot_artifact_id ?? null,
+ stepId: activeItem.step_id ?? null,
+ actionOrder: activeItem.action_order ?? null,
+ };
+ }
+ if (isWorkflowRunBlock(activeItem)) {
+ const screenshotBlockId = resolveScreenshotBlockId(
+ timeline ?? [],
+ activeItem,
+ activeIteration,
+ );
+ const blockType =
+ findTimelineBlock(timeline ?? [], screenshotBlockId)?.block_type ??
+ activeItem.block_type ??
+ null;
+ return {
+ kind: "block",
+ workflowRunBlockId: screenshotBlockId,
+ blockType,
+ };
+ }
+ if (isObserverThought(activeItem)) {
+ return { kind: "thought", thoughtId: activeItem.thought_id };
+ }
+ return null;
+ }, [activeItem, timeline, activeIteration]);
+
+ const heroLabel = isAction(activeItem)
+ ? actionLabel(activeItem)
+ : isWorkflowRunBlock(activeItem)
+ ? (activeItem.label ?? "Screenshot")
+ : isObserverThought(activeItem)
+ ? (activeItem.thought ?? "Thought")
+ : "Screenshot";
+
const extractedInformation = useMemo<Record<string, unknown> | null>(() => {
const outputs = workflowRun?.outputs;
return typeof outputs === "object" &&
@@ -282,7 +330,8 @@ export function RunView({
/>
<RunHero
workflowRunId={workflowRun.workflow_run_id}
- shownFrame={shownFrame}
+ heroSelection={heroSelection}
+ heroLabel={heroLabel}
running={running}
showDebugStream={showDebugStream}
provisioning={