Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(SKY-11519): show full-page LLM screenshot in studio run hero (#6850)
Celal Zamanoğlu24318bf30279
diff --git a/skyvern-frontend/src/routes/workflows/studio/runProjections.test.ts b/skyvern-frontend/src/routes/workflows/studio/runProjections.test.tsindex 8de4eb1bf..f8986030f 100644--- a/skyvern-frontend/src/routes/workflows/studio/runProjections.test.ts+++ b/skyvern-frontend/src/routes/workflows/studio/runProjections.test.ts@@ -129,6 +129,7 @@ describe("buildFilmstrip", () => {expect(frames.map((f) => f.id)).toEqual(["oldest", "newest"]);expect(frames.map((f) => f.index)).toEqual([1, 2]);expect(frames.map((f) => f.isBlockStart)).toEqual([true, false]);+ expect(frames.map((f) => f.blockType)).toEqual(["task", "task"]);});test("marks block boundaries across a multi-block run", () => {diff --git a/skyvern-frontend/src/routes/workflows/studio/runProjections.ts b/skyvern-frontend/src/routes/workflows/studio/runProjections.tsindex dad6d27e6..5650a7fc4 100644--- a/skyvern-frontend/src/routes/workflows/studio/runProjections.ts+++ b/skyvern-frontend/src/routes/workflows/studio/runProjections.ts@@ -62,7 +62,7 @@ export type FilmstripFrame = {actionType: ActionType;label: string;status: Status;- screenshotArtifactId: string | null;+ blockType: string | null;stepId: string | null;actionOrder: number | null;};@@ -108,7 +108,7 @@ export function buildFilmstrip(actionType: action.action_type,label: actionLabel(action),status: action.status,- screenshotArtifactId: action.screenshot_artifact_id ?? null,+ blockType: block.block_type ?? null,stepId: action.step_id,actionOrder: action.action_order,});diff --git a/skyvern-frontend/src/routes/workflows/studio/runview/HeroScreenshot.tsx b/skyvern-frontend/src/routes/workflows/studio/runview/HeroScreenshot.tsxindex 3de1424a9..a9c435aab 100644--- a/skyvern-frontend/src/routes/workflows/studio/runview/HeroScreenshot.tsx+++ b/skyvern-frontend/src/routes/workflows/studio/runview/HeroScreenshot.tsx@@ -6,21 +6,53 @@ import { getClient } from "@/api/AxiosClient";import { ArtifactApiResponse } from "@/api/types";import { useCredentialGetter } from "@/hooks/useCredentialGetter";import { getImageURL } from "@/routes/tasks/detail/artifactUtils";+import { apiPathPrefix } from "@/util/env";+import { selectBlockScreenshot } from "../../workflowRun/blockScreenshot";import { screenshotZoomClasses } from "./HeroScreenshot.utils";/**- * The pinned/active action screenshot, fit to the run-hero width and scrollable- * for long captures. Shares the ["artifact", id] cache with the filmstrip- * thumbnails so each is fetched once.+ * The active block's screenshot, fit to the run-hero width and scrollable for long+ * captures. Prefers the full-page LLM screenshot (what the agent saw) over the+ * per-action screenshot via `selectBlockScreenshot`, matching the legacy run page.*/-export function HeroScreenshot({ artifactId }: { artifactId: string | null }) {+export function HeroScreenshot({+ workflowRunBlockId,+ blockType,+ running,+}: {+ workflowRunBlockId: string | null;+ blockType: string | null;+ running: boolean;+}) {const credentialGetter = useCredentialGetter();const [zoomed, setZoomed] = useState(false);const containerRef = useRef<HTMLDivElement>(null);++ const { data: artifacts, isLoading } = useQuery<Array<ArtifactApiResponse>>({+ queryKey: ["workflowRunBlock", workflowRunBlockId, "artifacts"],+ queryFn: async () => {+ const client = await getClient(credentialGetter);+ return client+ .get(+ `${apiPathPrefix}/workflow_run_block/${workflowRunBlockId}/artifacts`,+ )+ .then((response) => response.data);+ },+ enabled: Boolean(workflowRunBlockId),+ refetchInterval: running ? 5000 : false,+ refetchOnWindowFocus: false,+ // Artifacts are immutable once the run finishes; only keep polling while live.+ staleTime: running ? 0 : Infinity,+ retry: 1,+ });++ const screenshot = selectBlockScreenshot(artifacts, blockType ?? undefined);+ const screenshotId = screenshot?.artifact_id ?? null;+useEffect(() => {setZoomed(false);- }, [artifactId]);+ }, [screenshotId]);// Start each view at the top; when zoomed, center horizontally too (margin-auto// resolves to 0 once the image overflows, so the scroll would default to left).@@ -31,43 +63,31 @@ export function HeroScreenshot({ artifactId }: { artifactId: string | null }) {}el.scrollTop = 0;el.scrollLeft = zoomed ? (el.scrollWidth - el.clientWidth) / 2 : 0;- }, [zoomed, artifactId]);- const { data, isLoading } = useQuery<ArtifactApiResponse>({- queryKey: ["artifact", artifactId],- queryFn: async () => {- const client = await getClient(credentialGetter, "sans-api-v1");- return client- .get(`/artifacts/${artifactId}`)- .then((response) => response.data);- },- enabled: Boolean(artifactId),- refetchOnWindowFocus: false,- refetchOnMount: false,- staleTime: Infinity,- retry: 1,- });+ }, [zoomed, screenshotId]);- if (!artifactId) {+ if (!workflowRunBlockId) {return (<div className="absolute inset-0 grid place-items-center text-sm text-muted-foreground">No screenshot for this action.</div>);}- if (isLoading) {+ // Spinner only on the first load, not on poll refetches (which keep prior data).+ if (isLoading && !artifacts) {return (<div className="absolute inset-0 flex items-center justify-center gap-2 text-sm text-muted-foreground"><ReloadIcon className="h-5 w-5 animate-spin" /> Loading screenshot…</div>);}- if (!data || data.archived) {+ if (!screenshot || screenshot.archived) {return (<div className="absolute inset-0 grid place-items-center text-sm text-muted-foreground">Screenshot unavailable.</div>);}+const toggleZoom = () => setZoomed((z) => !z);const zoom = screenshotZoomClasses(zoomed);return (@@ -86,8 +106,8 @@ export function HeroScreenshot({ artifactId }: { artifactId: string | null }) {}}><img- src={getImageURL(data)}- alt="action screenshot"+ src={getImageURL(screenshot)}+ alt="block screenshot"className={zoom.image}/></div>diff --git a/skyvern-frontend/src/routes/workflows/studio/runview/RunHero.tsx b/skyvern-frontend/src/routes/workflows/studio/runview/RunHero.tsxindex 85f8848f0..264683e5e 100644--- a/skyvern-frontend/src/routes/workflows/studio/runview/RunHero.tsx+++ b/skyvern-frontend/src/routes/workflows/studio/runview/RunHero.tsx@@ -239,7 +239,11 @@ export function RunHero({onPlay={onRecordingPlay}/>) : shownFrame ? (- <HeroScreenshot artifactId={shownFrame.screenshotArtifactId} />+ <HeroScreenshot+ workflowRunBlockId={shownFrame.blockId}+ blockType={shownFrame.blockType}+ running={running}+ />) : (<div className="absolute inset-0 grid place-items-center text-sm text-muted-foreground">Waiting for the first action…