Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-11079): show code-block action screenshot on the run Overview (#6605)
Aaron Perezb6159bfbbc4c
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunBlockScreenshot.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunBlockScreenshot.tsxindex e1dde0931..d63f04e98 100644--- a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunBlockScreenshot.tsx+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunBlockScreenshot.tsx@@ -1,6 +1,6 @@import { useEffect, useState } from "react";import { getClient } from "@/api/AxiosClient";-import { ArtifactApiResponse, ArtifactType, Status } from "@/api/types";+import { ArtifactApiResponse, Status } from "@/api/types";import { ZoomableImage } from "@/components/ZoomableImage";import { useCredentialGetter } from "@/hooks/useCredentialGetter";import { useQuery } from "@tanstack/react-query";@@ -11,13 +11,19 @@ import {import { statusIsNotFinalized } from "@/routes/tasks/types";import { getImageURL } from "@/routes/tasks/detail/artifactUtils";import { apiPathPrefix } from "@/util/env";+import { isBlockScreenshot, selectBlockScreenshot } from "./blockScreenshot";type Props = {workflowRunBlockId: string;+ blockType?: string;runStatus?: Status;};-function WorkflowRunBlockScreenshot({ workflowRunBlockId, runStatus }: Props) {+function WorkflowRunBlockScreenshot({+ workflowRunBlockId,+ blockType,+ runStatus,+}: Props) {const credentialGetter = useCredentialGetter();const [imageFailed, setImageFailed] = useState(false);@@ -32,22 +38,18 @@ function WorkflowRunBlockScreenshot({ workflowRunBlockId, runStatus }: Props) {.then((response) => response.data);},refetchInterval: (query) => {- const data = query.state.data;- const screenshot = data?.filter(- (artifact) => artifact.artifact_type === ArtifactType.LLMScreenshot,- )?.[0];- if (!screenshot) {- return 5000;+ // While the run is active, keep polling so the latest screenshot shows (a code block's+ // action screenshots land after its pre-execution LLM screenshot). Once finalized, stop:+ // no further screenshots are captured, including for promptless code blocks that never+ // produce one. Status unknown -> poll until any screenshot appears.+ if (runStatus !== undefined) {+ return statusIsNotFinalized({ status: runStatus }) ? 5000 : false;}- return false;+ return query.state.data?.some(isBlockScreenshot) ? false : 5000;},});- const llmScreenshots = artifacts?.filter(- (artifact) => artifact.artifact_type === ArtifactType.LLMScreenshot,- );-- const screenshot = llmScreenshots?.[0];+ const screenshot = selectBlockScreenshot(artifacts, blockType);useEffect(() => {setImageFailed(false);@@ -112,7 +114,7 @@ function WorkflowRunBlockScreenshot({ workflowRunBlockId, runStatus }: Props) {<figure className="mx-auto flex max-w-full flex-col items-center gap-2 overflow-hidden rounded"><ZoomableImagesrc={getImageURL(screenshot)}- alt="llm-screenshot"+ alt="block screenshot"onError={() => setImageFailed(true)}/></figure>diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsxindex 92fb84255..312fb6cbf 100644--- a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx@@ -18,6 +18,7 @@ import { WorkflowRunStream } from "./WorkflowRunStream";import { useSearchParams } from "react-router-dom";import {findActiveItem,+ findTimelineBlock,parseActiveIterationParam,resolveScreenshotBlockId,} from "./workflowTimelineUtils";@@ -155,16 +156,27 @@ function WorkflowRunOverview() {stepId={selection.step_id ?? ""}/>)}- {isWorkflowRunBlock(selection) && !isStreamActive && (- <WorkflowRunBlockScreenshot- workflowRunBlockId={resolveScreenshotBlockId(+ {isWorkflowRunBlock(selection) &&+ !isStreamActive &&+ (() => {+ // A container selection (loop/conditional) resolves to a descendant leaf, so read the+ // leaf's type rather than the container's so a nested code block is treated as one.+ const screenshotBlockId = resolveScreenshotBlockId(workflowRunTimeline,selection,activeIteration,- )}- runStatus={workflowRun?.status}- />- )}+ );+ const screenshotBlockType =+ findTimelineBlock(workflowRunTimeline, screenshotBlockId)+ ?.block_type ?? selection.block_type;+ return (+ <WorkflowRunBlockScreenshot+ workflowRunBlockId={screenshotBlockId}+ blockType={screenshotBlockType}+ runStatus={workflowRun?.status}+ />+ );+ })()}{isObserverThought(selection) && (<ObserverThoughtScreenshot observerThoughtId={selection.thought_id} />)}diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/blockScreenshot.test.ts b/skyvern-frontend/src/routes/workflows/workflowRun/blockScreenshot.test.tsnew file mode 100644index 000000000..c59b4df09--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/workflowRun/blockScreenshot.test.ts@@ -0,0 +1,81 @@+import { describe, expect, test } from "vitest";+import { ArtifactApiResponse, ArtifactType } from "@/api/types";+import { WorkflowBlockTypes } from "@/routes/workflows/types/workflowTypes";+import { isBlockScreenshot, selectBlockScreenshot } from "./blockScreenshot";++function artifact(+ artifactType: ArtifactType,+ artifactId = "a_1",+): ArtifactApiResponse {+ return {+ created_at: "2026-06-16T00:00:00Z",+ modified_at: "2026-06-16T00:00:00Z",+ artifact_id: artifactId,+ task_id: "tsk_1",+ step_id: "stp_1",+ artifact_type: artifactType,+ uri: "s3://bucket/key",+ organization_id: "o_1",+ };+}++describe("isBlockScreenshot", () => {+ test("true for LLM and action screenshots", () => {+ expect(isBlockScreenshot(artifact(ArtifactType.LLMScreenshot))).toBe(true);+ expect(isBlockScreenshot(artifact(ArtifactType.ActionScreenshot))).toBe(+ true,+ );+ });++ test("false for non-screenshot artifacts", () => {+ expect(isBlockScreenshot(artifact(ArtifactType.Recording))).toBe(false);+ });+});++describe("selectBlockScreenshot", () => {+ test("agent block prefers the LLM screenshot when present", () => {+ const artifacts = [+ artifact(ArtifactType.ActionScreenshot, "a_action"),+ artifact(ArtifactType.LLMScreenshot, "a_llm"),+ ];+ expect(selectBlockScreenshot(artifacts)?.artifact_id).toBe("a_llm");+ });++ test("code block prefers the action screenshot over the pre-execution LLM screenshot", () => {+ // execute_safe captures a block-start LLM screenshot before the code runs; the action+ // screenshot is the block's real output and must win for code blocks.+ const artifacts = [+ artifact(ArtifactType.ActionScreenshot, "a_action"),+ artifact(ArtifactType.LLMScreenshot, "a_llm_preexec"),+ ];+ expect(+ selectBlockScreenshot(artifacts, WorkflowBlockTypes.Code)?.artifact_id,+ ).toBe("a_action");+ });++ test("code block falls back to the LLM screenshot when no action screenshot exists", () => {+ const artifacts = [artifact(ArtifactType.LLMScreenshot, "a_llm")];+ expect(+ selectBlockScreenshot(artifacts, WorkflowBlockTypes.Code)?.artifact_id,+ ).toBe("a_llm");+ });++ test("falls back to the latest action screenshot when no LLM screenshot exists", () => {+ // Artifacts arrive newest-first, so the first action screenshot is the latest.+ const artifacts = [+ artifact(ArtifactType.ActionScreenshot, "a_action_newest"),+ artifact(ArtifactType.ActionScreenshot, "a_action_older"),+ ];+ expect(selectBlockScreenshot(artifacts)?.artifact_id).toBe(+ "a_action_newest",+ );+ });++ test("returns undefined when no screenshot is present", () => {+ expect(+ selectBlockScreenshot([artifact(ArtifactType.Recording)]),+ ).toBeUndefined();+ expect(selectBlockScreenshot([])).toBeUndefined();+ expect(selectBlockScreenshot(undefined)).toBeUndefined();+ });+});diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/blockScreenshot.ts b/skyvern-frontend/src/routes/workflows/workflowRun/blockScreenshot.tsnew file mode 100644index 000000000..314a0f529--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/workflowRun/blockScreenshot.ts@@ -0,0 +1,30 @@+import { ArtifactApiResponse, ArtifactType } from "@/api/types";+import { WorkflowBlockTypes } from "@/routes/workflows/types/workflowTypes";++// Agent blocks emit an LLM screenshot; code blocks emit per-action screenshots. Either one is a+// valid block-level screenshot for the run Overview.+function isBlockScreenshot(artifact: ArtifactApiResponse): boolean {+ return (+ artifact.artifact_type === ArtifactType.LLMScreenshot ||+ artifact.artifact_type === ArtifactType.ActionScreenshot+ );+}++// Artifacts arrive newest-first. An agent block is best represented by its LLM screenshot, but a+// code block's only LLM screenshot is the pre-execution one `Block.execute_safe` takes before the+// block runs; its real output is the per-action screenshot captured during execution, so prefer+// that and fall back to the LLM screenshot only when no action screenshot exists.+function selectBlockScreenshot(+ artifacts: Array<ArtifactApiResponse> | undefined,+ blockType?: string,+): ArtifactApiResponse | undefined {+ const findType = (type: ArtifactType) =>+ artifacts?.find((artifact) => artifact.artifact_type === type);+ const [primary, fallback] =+ blockType === WorkflowBlockTypes.Code+ ? [ArtifactType.ActionScreenshot, ArtifactType.LLMScreenshot]+ : [ArtifactType.LLMScreenshot, ArtifactType.ActionScreenshot];+ return findType(primary) ?? findType(fallback);+}++export { isBlockScreenshot, selectBlockScreenshot };diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/workflowTimelineUtils.test.ts b/skyvern-frontend/src/routes/workflows/workflowRun/workflowTimelineUtils.test.tsindex 242e36094..b60132ed5 100644--- a/skyvern-frontend/src/routes/workflows/workflowRun/workflowTimelineUtils.test.ts+++ b/skyvern-frontend/src/routes/workflows/workflowRun/workflowTimelineUtils.test.ts@@ -16,6 +16,7 @@ import {findBlockSurroundingThought,findLastExecutedBlock,findRunningBlock,+ findTimelineBlock,flattenTimelineChronologically,resolveScreenshotBlockId,} from "./workflowTimelineUtils";@@ -906,6 +907,28 @@ describe("classifyUnexecutedDefinedBlocks", () => {});});+describe("findTimelineBlock", () => {+ test("returns a nested leaf block by id so its real type is available", () => {+ const codeLeaf = buildBlock({+ workflow_run_block_id: "wrb_code_leaf",+ block_type: "code",+ });+ const loop = buildBlock({+ workflow_run_block_id: "wrb_loop",+ block_type: "for_loop",+ });+ const timeline = [buildBlockItem(loop, [buildBlockItem(codeLeaf)])];+ expect(findTimelineBlock(timeline, "wrb_code_leaf")?.block_type).toBe(+ "code",+ );+ });++ test("returns null for an unknown id", () => {+ const leaf = buildBlock({ workflow_run_block_id: "wrb_leaf" });+ expect(findTimelineBlock([buildBlockItem(leaf)], "wrb_missing")).toBeNull();+ });+});+describe("resolveScreenshotBlockId", () => {test("returns the block's own id when it isn't a container", () => {const leaf = buildBlock({diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/workflowTimelineUtils.ts b/skyvern-frontend/src/routes/workflows/workflowRun/workflowTimelineUtils.tsindex cbbe4b66d..64214e171 100644--- a/skyvern-frontend/src/routes/workflows/workflowRun/workflowTimelineUtils.ts+++ b/skyvern-frontend/src/routes/workflows/workflowRun/workflowTimelineUtils.ts@@ -430,6 +430,14 @@ function findTimelineBlockItem(return null;}+function findTimelineBlock(+ items: Array<WorkflowRunTimelineItem>,+ blockId: string,+): WorkflowRunBlock | null {+ const item = findTimelineBlockItem(items, blockId);+ return item && isBlockItem(item) ? item.block : null;+}+function findFirstLeafBlockId(items: Array<WorkflowRunTimelineItem>,): string | null {@@ -617,6 +625,7 @@ export {findLastExecutedBlock,findRunningBlock,findThoughtsForBlock,+ findTimelineBlock,flattenTimelineChronologically,parseActiveIterationParam,resolveScreenshotBlockId,