Skip to content

Automate browser based workflows with AI

fix(SKY-11079): show code-block action screenshot on the run Overview (#6605)

Aaron Perezb6159bfbbc4c
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunBlockScreenshot.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunBlockScreenshot.tsx
index e1dde0931..d63f04e98 100644
--- a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunBlockScreenshot.tsx
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunBlockScreenshot.tsx
@@ -1,6 +1,6 @@
import { useEffect, useState } from "react";
import { getClient } from "@/api/AxiosClient";
-import { ArtifactApiResponse, ArtifactType, Status } from "@/api/types";
+import { ArtifactApiResponse, Status } from "@/api/types";
import { ZoomableImage } from "@/components/ZoomableImage";
import { useCredentialGetter } from "@/hooks/useCredentialGetter";
import { useQuery } from "@tanstack/react-query";
@@ -11,13 +11,19 @@ import {
import { statusIsNotFinalized } from "@/routes/tasks/types";
import { getImageURL } from "@/routes/tasks/detail/artifactUtils";
import { apiPathPrefix } from "@/util/env";
+import { isBlockScreenshot, selectBlockScreenshot } from "./blockScreenshot";
type Props = {
workflowRunBlockId: string;
+ blockType?: string;
runStatus?: Status;
};
-function WorkflowRunBlockScreenshot({ workflowRunBlockId, runStatus }: Props) {
+function WorkflowRunBlockScreenshot({
+ workflowRunBlockId,
+ blockType,
+ runStatus,
+}: Props) {
const credentialGetter = useCredentialGetter();
const [imageFailed, setImageFailed] = useState(false);
@@ -32,22 +38,18 @@ function WorkflowRunBlockScreenshot({ workflowRunBlockId, runStatus }: Props) {
.then((response) => response.data);
},
refetchInterval: (query) => {
- const data = query.state.data;
- const screenshot = data?.filter(
- (artifact) => artifact.artifact_type === ArtifactType.LLMScreenshot,
- )?.[0];
- if (!screenshot) {
- return 5000;
+ // While the run is active, keep polling so the latest screenshot shows (a code block's
+ // action screenshots land after its pre-execution LLM screenshot). Once finalized, stop:
+ // no further screenshots are captured, including for promptless code blocks that never
+ // produce one. Status unknown -> poll until any screenshot appears.
+ if (runStatus !== undefined) {
+ return statusIsNotFinalized({ status: runStatus }) ? 5000 : false;
}
- return false;
+ return query.state.data?.some(isBlockScreenshot) ? false : 5000;
},
});
- const llmScreenshots = artifacts?.filter(
- (artifact) => artifact.artifact_type === ArtifactType.LLMScreenshot,
- );
-
- const screenshot = llmScreenshots?.[0];
+ const screenshot = selectBlockScreenshot(artifacts, blockType);
useEffect(() => {
setImageFailed(false);
@@ -112,7 +114,7 @@ function WorkflowRunBlockScreenshot({ workflowRunBlockId, runStatus }: Props) {
<figure className="mx-auto flex max-w-full flex-col items-center gap-2 overflow-hidden rounded">
<ZoomableImage
src={getImageURL(screenshot)}
- alt="llm-screenshot"
+ alt="block screenshot"
onError={() => setImageFailed(true)}
/>
</figure>
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx
index 92fb84255..312fb6cbf 100644
--- a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx
@@ -18,6 +18,7 @@ import { WorkflowRunStream } from "./WorkflowRunStream";
import { useSearchParams } from "react-router-dom";
import {
findActiveItem,
+ findTimelineBlock,
parseActiveIterationParam,
resolveScreenshotBlockId,
} from "./workflowTimelineUtils";
@@ -155,16 +156,27 @@ function WorkflowRunOverview() {
stepId={selection.step_id ?? ""}
/>
)}
- {isWorkflowRunBlock(selection) && !isStreamActive && (
- <WorkflowRunBlockScreenshot
- workflowRunBlockId={resolveScreenshotBlockId(
+ {isWorkflowRunBlock(selection) &&
+ !isStreamActive &&
+ (() => {
+ // A container selection (loop/conditional) resolves to a descendant leaf, so read the
+ // leaf's type rather than the container's so a nested code block is treated as one.
+ const screenshotBlockId = resolveScreenshotBlockId(
workflowRunTimeline,
selection,
activeIteration,
- )}
- runStatus={workflowRun?.status}
- />
- )}
+ );
+ const screenshotBlockType =
+ findTimelineBlock(workflowRunTimeline, screenshotBlockId)
+ ?.block_type ?? selection.block_type;
+ return (
+ <WorkflowRunBlockScreenshot
+ workflowRunBlockId={screenshotBlockId}
+ blockType={screenshotBlockType}
+ runStatus={workflowRun?.status}
+ />
+ );
+ })()}
{isObserverThought(selection) && (
<ObserverThoughtScreenshot observerThoughtId={selection.thought_id} />
)}
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/blockScreenshot.test.ts b/skyvern-frontend/src/routes/workflows/workflowRun/blockScreenshot.test.ts
new file mode 100644
index 000000000..c59b4df09
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/blockScreenshot.test.ts
@@ -0,0 +1,81 @@
+import { describe, expect, test } from "vitest";
+import { ArtifactApiResponse, ArtifactType } from "@/api/types";
+import { WorkflowBlockTypes } from "@/routes/workflows/types/workflowTypes";
+import { isBlockScreenshot, selectBlockScreenshot } from "./blockScreenshot";
+
+function artifact(
+ artifactType: ArtifactType,
+ artifactId = "a_1",
+): ArtifactApiResponse {
+ return {
+ created_at: "2026-06-16T00:00:00Z",
+ modified_at: "2026-06-16T00:00:00Z",
+ artifact_id: artifactId,
+ task_id: "tsk_1",
+ step_id: "stp_1",
+ artifact_type: artifactType,
+ uri: "s3://bucket/key",
+ organization_id: "o_1",
+ };
+}
+
+describe("isBlockScreenshot", () => {
+ test("true for LLM and action screenshots", () => {
+ expect(isBlockScreenshot(artifact(ArtifactType.LLMScreenshot))).toBe(true);
+ expect(isBlockScreenshot(artifact(ArtifactType.ActionScreenshot))).toBe(
+ true,
+ );
+ });
+
+ test("false for non-screenshot artifacts", () => {
+ expect(isBlockScreenshot(artifact(ArtifactType.Recording))).toBe(false);
+ });
+});
+
+describe("selectBlockScreenshot", () => {
+ test("agent block prefers the LLM screenshot when present", () => {
+ const artifacts = [
+ artifact(ArtifactType.ActionScreenshot, "a_action"),
+ artifact(ArtifactType.LLMScreenshot, "a_llm"),
+ ];
+ expect(selectBlockScreenshot(artifacts)?.artifact_id).toBe("a_llm");
+ });
+
+ test("code block prefers the action screenshot over the pre-execution LLM screenshot", () => {
+ // execute_safe captures a block-start LLM screenshot before the code runs; the action
+ // screenshot is the block's real output and must win for code blocks.
+ const artifacts = [
+ artifact(ArtifactType.ActionScreenshot, "a_action"),
+ artifact(ArtifactType.LLMScreenshot, "a_llm_preexec"),
+ ];
+ expect(
+ selectBlockScreenshot(artifacts, WorkflowBlockTypes.Code)?.artifact_id,
+ ).toBe("a_action");
+ });
+
+ test("code block falls back to the LLM screenshot when no action screenshot exists", () => {
+ const artifacts = [artifact(ArtifactType.LLMScreenshot, "a_llm")];
+ expect(
+ selectBlockScreenshot(artifacts, WorkflowBlockTypes.Code)?.artifact_id,
+ ).toBe("a_llm");
+ });
+
+ test("falls back to the latest action screenshot when no LLM screenshot exists", () => {
+ // Artifacts arrive newest-first, so the first action screenshot is the latest.
+ const artifacts = [
+ artifact(ArtifactType.ActionScreenshot, "a_action_newest"),
+ artifact(ArtifactType.ActionScreenshot, "a_action_older"),
+ ];
+ expect(selectBlockScreenshot(artifacts)?.artifact_id).toBe(
+ "a_action_newest",
+ );
+ });
+
+ test("returns undefined when no screenshot is present", () => {
+ expect(
+ selectBlockScreenshot([artifact(ArtifactType.Recording)]),
+ ).toBeUndefined();
+ expect(selectBlockScreenshot([])).toBeUndefined();
+ expect(selectBlockScreenshot(undefined)).toBeUndefined();
+ });
+});
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/blockScreenshot.ts b/skyvern-frontend/src/routes/workflows/workflowRun/blockScreenshot.ts
new file mode 100644
index 000000000..314a0f529
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/blockScreenshot.ts
@@ -0,0 +1,30 @@
+import { ArtifactApiResponse, ArtifactType } from "@/api/types";
+import { WorkflowBlockTypes } from "@/routes/workflows/types/workflowTypes";
+
+// Agent blocks emit an LLM screenshot; code blocks emit per-action screenshots. Either one is a
+// valid block-level screenshot for the run Overview.
+function isBlockScreenshot(artifact: ArtifactApiResponse): boolean {
+ return (
+ artifact.artifact_type === ArtifactType.LLMScreenshot ||
+ artifact.artifact_type === ArtifactType.ActionScreenshot
+ );
+}
+
+// Artifacts arrive newest-first. An agent block is best represented by its LLM screenshot, but a
+// code block's only LLM screenshot is the pre-execution one `Block.execute_safe` takes before the
+// block runs; its real output is the per-action screenshot captured during execution, so prefer
+// that and fall back to the LLM screenshot only when no action screenshot exists.
+function selectBlockScreenshot(
+ artifacts: Array<ArtifactApiResponse> | undefined,
+ blockType?: string,
+): ArtifactApiResponse | undefined {
+ const findType = (type: ArtifactType) =>
+ artifacts?.find((artifact) => artifact.artifact_type === type);
+ const [primary, fallback] =
+ blockType === WorkflowBlockTypes.Code
+ ? [ArtifactType.ActionScreenshot, ArtifactType.LLMScreenshot]
+ : [ArtifactType.LLMScreenshot, ArtifactType.ActionScreenshot];
+ return findType(primary) ?? findType(fallback);
+}
+
+export { isBlockScreenshot, selectBlockScreenshot };
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/workflowTimelineUtils.test.ts b/skyvern-frontend/src/routes/workflows/workflowRun/workflowTimelineUtils.test.ts
index 242e36094..b60132ed5 100644
--- a/skyvern-frontend/src/routes/workflows/workflowRun/workflowTimelineUtils.test.ts
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/workflowTimelineUtils.test.ts
@@ -16,6 +16,7 @@ import {
findBlockSurroundingThought,
findLastExecutedBlock,
findRunningBlock,
+ findTimelineBlock,
flattenTimelineChronologically,
resolveScreenshotBlockId,
} from "./workflowTimelineUtils";
@@ -906,6 +907,28 @@ describe("classifyUnexecutedDefinedBlocks", () => {
});
});
+describe("findTimelineBlock", () => {
+ test("returns a nested leaf block by id so its real type is available", () => {
+ const codeLeaf = buildBlock({
+ workflow_run_block_id: "wrb_code_leaf",
+ block_type: "code",
+ });
+ const loop = buildBlock({
+ workflow_run_block_id: "wrb_loop",
+ block_type: "for_loop",
+ });
+ const timeline = [buildBlockItem(loop, [buildBlockItem(codeLeaf)])];
+ expect(findTimelineBlock(timeline, "wrb_code_leaf")?.block_type).toBe(
+ "code",
+ );
+ });
+
+ test("returns null for an unknown id", () => {
+ const leaf = buildBlock({ workflow_run_block_id: "wrb_leaf" });
+ expect(findTimelineBlock([buildBlockItem(leaf)], "wrb_missing")).toBeNull();
+ });
+});
+
describe("resolveScreenshotBlockId", () => {
test("returns the block's own id when it isn't a container", () => {
const leaf = buildBlock({
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/workflowTimelineUtils.ts b/skyvern-frontend/src/routes/workflows/workflowRun/workflowTimelineUtils.ts
index cbbe4b66d..64214e171 100644
--- a/skyvern-frontend/src/routes/workflows/workflowRun/workflowTimelineUtils.ts
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/workflowTimelineUtils.ts
@@ -430,6 +430,14 @@ function findTimelineBlockItem(
return null;
}
+function findTimelineBlock(
+ items: Array<WorkflowRunTimelineItem>,
+ blockId: string,
+): WorkflowRunBlock | null {
+ const item = findTimelineBlockItem(items, blockId);
+ return item && isBlockItem(item) ? item.block : null;
+}
+
function findFirstLeafBlockId(
items: Array<WorkflowRunTimelineItem>,
): string | null {
@@ -617,6 +625,7 @@ export {
findLastExecutedBlock,
findRunningBlock,
findThoughtsForBlock,
+ findTimelineBlock,
flattenTimelineChronologically,
parseActiveIterationParam,
resolveScreenshotBlockId,