Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-11560): mirror legacy run page in studio run-tab hero (per-action & per-block) (#6930)
Celal Zamanoğlua23465353a6b
diff --git a/skyvern-frontend/src/routes/workflows/studio/runProjections.ts b/skyvern-frontend/src/routes/workflows/studio/runProjections.tsindex e56fded13..db31cf984 100644--- a/skyvern-frontend/src/routes/workflows/studio/runProjections.ts+++ b/skyvern-frontend/src/routes/workflows/studio/runProjections.ts@@ -77,7 +77,7 @@ export type FilmstripFrame = {actionOrder: number | null;};-function actionLabel(action: ActionsApiResponse): string {+export function actionLabel(action: ActionsApiResponse): string {const candidate =action.intention?.trim() ||action.description?.trim() ||diff --git a/skyvern-frontend/src/routes/workflows/studio/runview/HeroScreenshot.test.tsx b/skyvern-frontend/src/routes/workflows/studio/runview/HeroScreenshot.test.tsxnew file mode 100644index 000000000..d27432b9e--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/studio/runview/HeroScreenshot.test.tsx@@ -0,0 +1,84 @@+// @vitest-environment jsdom++import { cleanup, render, screen, waitFor } from "@testing-library/react";+import { QueryClient, QueryClientProvider } from "@tanstack/react-query";+import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";++const { getSpy } = vi.hoisted(() => ({ getSpy: vi.fn() }));++vi.mock("@/api/AxiosClient", () => ({+ getClient: async () => ({+ get: (url: string) => {+ getSpy(url);+ return Promise.resolve({ data: null });+ },+ }),+}));++vi.mock("@/hooks/useCredentialGetter", () => ({+ useCredentialGetter: () => undefined,+}));++import { HeroScreenshot, type HeroSelection } from "./HeroScreenshot";++function renderHero(selection: HeroSelection | null) {+ const client = new QueryClient({+ defaultOptions: { queries: { retry: false } },+ });+ return render(+ <QueryClientProvider client={client}>+ <HeroScreenshot selection={selection} running={false} />+ </QueryClientProvider>,+ );+}++beforeEach(() => getSpy.mockReset());+afterEach(cleanup);++describe("HeroScreenshot selection", () => {+ test("fetches the action's own screenshot artifact when an artifactId is present", async () => {+ renderHero({+ kind: "action",+ artifactId: "art_1",+ stepId: "stp_1",+ actionOrder: 0,+ });+ await waitFor(() => expect(getSpy).toHaveBeenCalledTimes(1));+ expect(getSpy.mock.calls[0]![0]).toContain("/artifacts/art_1");+ });++ test("falls back to the step's action screenshots when the action has no artifactId", async () => {+ renderHero({+ kind: "action",+ artifactId: null,+ stepId: "stp_1",+ actionOrder: 0,+ });+ await waitFor(() => expect(getSpy).toHaveBeenCalledTimes(1));+ expect(getSpy.mock.calls[0]![0]).toContain("/step/stp_1/artifacts");+ });++ test("fetches block artifacts for a block selection", async () => {+ renderHero({+ kind: "block",+ workflowRunBlockId: "wrb_1",+ blockType: "task",+ });+ await waitFor(() => expect(getSpy).toHaveBeenCalledTimes(1));+ expect(getSpy.mock.calls[0]![0]).toContain(+ "/workflow_run_block/wrb_1/artifacts",+ );+ });++ test("fetches thought artifacts for a thought selection", async () => {+ renderHero({ kind: "thought", thoughtId: "tht_1" });+ await waitFor(() => expect(getSpy).toHaveBeenCalledTimes(1));+ expect(getSpy.mock.calls[0]![0]).toContain("/thought/tht_1/artifacts");+ });++ test("renders the empty state and fetches nothing without a selection", () => {+ renderHero(null);+ screen.getByText("No screenshot for this selection.");+ expect(getSpy).not.toHaveBeenCalled();+ });+});diff --git a/skyvern-frontend/src/routes/workflows/studio/runview/HeroScreenshot.tsx b/skyvern-frontend/src/routes/workflows/studio/runview/HeroScreenshot.tsxindex a9c435aab..311dfe241 100644--- a/skyvern-frontend/src/routes/workflows/studio/runview/HeroScreenshot.tsx+++ b/skyvern-frontend/src/routes/workflows/studio/runview/HeroScreenshot.tsx@@ -3,7 +3,7 @@ import { ReloadIcon } from "@radix-ui/react-icons";import { useQuery } from "@tanstack/react-query";import { getClient } from "@/api/AxiosClient";-import { ArtifactApiResponse } from "@/api/types";+import { ArtifactApiResponse, ArtifactType } from "@/api/types";import { useCredentialGetter } from "@/hooks/useCredentialGetter";import { getImageURL } from "@/routes/tasks/detail/artifactUtils";import { apiPathPrefix } from "@/util/env";@@ -11,35 +11,92 @@ import { apiPathPrefix } from "@/util/env";import { selectBlockScreenshot } from "../../workflowRun/blockScreenshot";import { screenshotZoomClasses } from "./HeroScreenshot.utils";+export type HeroSelection =+ | {+ kind: "action";+ artifactId: string | null;+ stepId: string | null;+ actionOrder: number | null;+ }+ | {+ kind: "block";+ workflowRunBlockId: string;+ blockType: string | null;+ }+ | {+ kind: "thought";+ thoughtId: string;+ };+/**- * The active block's screenshot, fit to the run-hero width and scrollable for long- * captures. Prefers the full-page LLM screenshot (what the agent saw) over the- * per-action screenshot via `selectBlockScreenshot`, matching the legacy run page.+ * The selected element's screenshot, fit to the run-hero width and scrollable for+ * long captures. Mirrors the legacy run page: an action shows its own post-action+ * screenshot (by artifact id, falling back to the step's action screenshots), a+ * block shows its representative screenshot via `selectBlockScreenshot`.*/export function HeroScreenshot({- workflowRunBlockId,- blockType,+ selection,running,}: {- workflowRunBlockId: string | null;- blockType: string | null;+ selection: HeroSelection | null;running: boolean;}) {const credentialGetter = useCredentialGetter();const [zoomed, setZoomed] = useState(false);const containerRef = useRef<HTMLDivElement>(null);- const { data: artifacts, isLoading } = useQuery<Array<ArtifactApiResponse>>({- queryKey: ["workflowRunBlock", workflowRunBlockId, "artifacts"],+ const action = selection?.kind === "action" ? selection : null;+ const block = selection?.kind === "block" ? selection : null;+ const thought = selection?.kind === "thought" ? selection : null;++ const { data: artifactById, isLoading: loadingArtifact } =+ useQuery<ArtifactApiResponse>({+ queryKey: ["artifact", action?.artifactId],+ queryFn: async () => {+ const client = await getClient(credentialGetter, "sans-api-v1");+ return client+ .get(`/artifacts/${action!.artifactId}`)+ .then((response) => response.data);+ },+ enabled: Boolean(action?.artifactId),+ refetchOnWindowFocus: false,+ staleTime: Infinity,+ retry: 1,+ });++ // Fallback path only when the action carries no explicit screenshot id: the+ // step's action screenshots, indexed by action order (newest-first), like legacy.+ const useStepFallback = Boolean(action?.stepId) && !action?.artifactId;+ const { data: stepArtifacts, isLoading: loadingStep } = useQuery<+ Array<ArtifactApiResponse>+ >({+ queryKey: ["step", action?.stepId, "artifacts"],+ queryFn: async () => {+ const client = await getClient(credentialGetter);+ return client+ .get(`${apiPathPrefix}/step/${action!.stepId}/artifacts`)+ .then((response) => response.data);+ },+ enabled: useStepFallback,+ refetchInterval: running ? 5000 : false,+ refetchOnWindowFocus: false,+ staleTime: running ? 0 : Infinity,+ retry: 1,+ });++ const { data: blockArtifacts, isLoading: loadingBlock } = useQuery<+ Array<ArtifactApiResponse>+ >({+ queryKey: ["workflowRunBlock", block?.workflowRunBlockId, "artifacts"],queryFn: async () => {const client = await getClient(credentialGetter);return client.get(- `${apiPathPrefix}/workflow_run_block/${workflowRunBlockId}/artifacts`,+ `${apiPathPrefix}/workflow_run_block/${block!.workflowRunBlockId}/artifacts`,).then((response) => response.data);},- enabled: Boolean(workflowRunBlockId),+ enabled: Boolean(block?.workflowRunBlockId),refetchInterval: running ? 5000 : false,refetchOnWindowFocus: false,// Artifacts are immutable once the run finishes; only keep polling while live.@@ -47,8 +104,49 @@ export function HeroScreenshot({retry: 1,});- const screenshot = selectBlockScreenshot(artifacts, blockType ?? undefined);+ const { data: thoughtArtifacts, isLoading: loadingThought } = useQuery<+ Array<ArtifactApiResponse>+ >({+ queryKey: ["observerThought", thought?.thoughtId, "artifacts"],+ queryFn: async () => {+ const client = await getClient(credentialGetter);+ return client+ .get(`${apiPathPrefix}/thought/${thought!.thoughtId}/artifacts`)+ .then((response) => response.data);+ },+ enabled: Boolean(thought?.thoughtId),+ refetchInterval: running ? 5000 : false,+ refetchOnWindowFocus: false,+ staleTime: running ? 0 : Infinity,+ retry: 1,+ });++ let screenshot: ArtifactApiResponse | undefined;+ if (action) {+ const actionShots = stepArtifacts?.filter(+ (artifact) => artifact.artifact_type === ArtifactType.ActionScreenshot,+ );+ const fromStep =+ actionShots && action.actionOrder != null+ ? actionShots[actionShots.length - action.actionOrder - 1]+ : undefined;+ screenshot = artifactById ?? fromStep;+ } else if (block) {+ screenshot = selectBlockScreenshot(+ blockArtifacts,+ block.blockType ?? undefined,+ );+ } else if (thought) {+ const thoughtShots = thoughtArtifacts?.filter(+ (artifact) => artifact.artifact_type === ArtifactType.LLMScreenshot,+ );+ // Thought LLM screenshots arrive newest-first; the last is the capture.+ screenshot = thoughtShots?.[thoughtShots.length - 1];+ }+const screenshotId = screenshot?.artifact_id ?? null;+ const isLoading =+ loadingArtifact || loadingStep || loadingBlock || loadingThought;useEffect(() => {setZoomed(false);@@ -65,15 +163,15 @@ export function HeroScreenshot({el.scrollLeft = zoomed ? (el.scrollWidth - el.clientWidth) / 2 : 0;}, [zoomed, screenshotId]);- if (!workflowRunBlockId) {+ if (!selection) {return (<div className="absolute inset-0 grid place-items-center text-sm text-muted-foreground">- No screenshot for this action.+ No screenshot for this selection.</div>);}// Spinner only on the first load, not on poll refetches (which keep prior data).- if (isLoading && !artifacts) {+ if (isLoading && !screenshot) {return (<div className="absolute inset-0 flex items-center justify-center gap-2 text-sm text-muted-foreground"><ReloadIcon className="h-5 w-5 animate-spin" /> Loading screenshot…@@ -107,7 +205,7 @@ export function HeroScreenshot({><imgsrc={getImageURL(screenshot)}- alt="block screenshot"+ alt="screenshot"className={zoom.image}/></div>diff --git a/skyvern-frontend/src/routes/workflows/studio/runview/RunHero.test.tsx b/skyvern-frontend/src/routes/workflows/studio/runview/RunHero.test.tsxindex 1e18d2bcd..1daf6a841 100644--- a/skyvern-frontend/src/routes/workflows/studio/runview/RunHero.test.tsx+++ b/skyvern-frontend/src/routes/workflows/studio/runview/RunHero.test.tsx@@ -22,7 +22,8 @@ import { RunHero } from "./RunHero";const baseProps = {workflowRunId: "wr_1",- shownFrame: null,+ heroSelection: null,+ heroLabel: "",running: true,provisioning: false,isPaused: false,diff --git a/skyvern-frontend/src/routes/workflows/studio/runview/RunHero.tsx b/skyvern-frontend/src/routes/workflows/studio/runview/RunHero.tsxindex e18e215fe..6581dba89 100644--- a/skyvern-frontend/src/routes/workflows/studio/runview/RunHero.tsx+++ b/skyvern-frontend/src/routes/workflows/studio/runview/RunHero.tsx@@ -16,15 +16,15 @@ import { useRunViewStore } from "@/store/RunViewStore";import { cn } from "@/util/utils";import { WorkflowRunCode } from "../../workflowRun/WorkflowRunCode";-import { FilmstripFrame } from "../runProjections";import { useStudioShellContext } from "../StudioShellContext";import { HeroRecording } from "./HeroRecording";-import { HeroScreenshot } from "./HeroScreenshot";+import { HeroScreenshot, type HeroSelection } from "./HeroScreenshot";import { RunLiveStream } from "./RunLiveStream";type RunHeroProps = {workflowRunId: string;- shownFrame: FilmstripFrame | null;+ heroSelection: HeroSelection | null;+ heroLabel: string;running: boolean;// A block run shows the shared debug-session stream (re-parented in by the// shell), view-only, instead of mounting a separate run stream.@@ -92,7 +92,8 @@ function ViewToggle({export function RunHero({workflowRunId,- shownFrame,+ heroSelection,+ heroLabel,running,showDebugStream,provisioning,@@ -185,7 +186,7 @@ export function RunHero({? "Inputs": center === "outputs"? "Outputs"- : (shownFrame?.label ?? "Screenshot");+ : (heroLabel ?? "Screenshot");const headerIcon =center === "recording" ? (@@ -355,23 +356,19 @@ export function RunHero({recordingUrls={recordingUrls}onPlay={onRecordingPlay}/>- ) : shownFrame ? (- <HeroScreenshot- workflowRunBlockId={shownFrame.blockId}- blockType={shownFrame.blockType}- running={running}- />+ ) : heroSelection ? (+ <HeroScreenshot selection={heroSelection} running={running} />) : (<div className="absolute inset-0 grid place-items-center text-sm text-muted-foreground">Waiting for the first action…</div>)}- {center === "screenshot" && scrubbing && shownFrame ? (+ {center === "screenshot" && scrubbing && heroSelection ? (<div className="absolute left-3 top-3 flex max-w-[26rem] items-center gap-2 rounded-md bg-black/70 px-3 py-1.5 text-xs text-white backdrop-blur"><CounterClockwiseClockIcon className="h-3.5 w-3.5 shrink-0" /><span className="truncate">- Inspecting step · <b>{shownFrame.label}</b>+ Inspecting · <b>{heroLabel}</b></span>{running || showDebugStream ? (<buttondiff --git a/skyvern-frontend/src/routes/workflows/studio/runview/RunView.tsx b/skyvern-frontend/src/routes/workflows/studio/runview/RunView.tsxindex f61d48ad8..a87295299 100644--- a/skyvern-frontend/src/routes/workflows/studio/runview/RunView.tsx+++ b/skyvern-frontend/src/routes/workflows/studio/runview/RunView.tsx@@ -8,6 +8,11 @@ import { ApiWebhookActionsMenu } from "@/components/ApiWebhookActionsMenu";import { WebhookReplayDialog } from "@/components/WebhookReplayDialog";import { useApiCredential } from "@/hooks/useApiCredential";import { statusIsFinalized } from "@/routes/tasks/types";+import {+ isAction,+ isObserverThought,+ isWorkflowRunBlock,+} from "@/routes/workflows/types/workflowRunTypes";import { useRunViewStore } from "@/store/RunViewStore";import { useStudioShellStore } from "@/store/StudioShellStore";import { type ApiCommandOptions } from "@/util/apiCommands";@@ -22,14 +27,20 @@ import { WorkflowRunTimeline } from "../../workflowRun/WorkflowRunTimeline";import { WorkflowRunVerificationCodeForm } from "../../workflowRun/WorkflowRunVerificationCodeForm";import { pickDownloadedFileFilename } from "../../workflowRun/blockDownloadedFiles";import { getRecordingUrls } from "../../workflowRun/recordingUrls";-import { findActiveItem } from "../../workflowRun/workflowTimelineUtils";+import {+ findActiveItem,+ findTimelineBlock,+ resolveScreenshotBlockId,+} from "../../workflowRun/workflowTimelineUtils";import { getOrderedRunParameters } from "../../utils";import {+ actionLabel,buildFilmstrip,formatElapsed,runOutcomeFromStatus,} from "../runProjections";import { RunHero } from "./RunHero";+import { type HeroSelection } from "./HeroScreenshot";import { buildRunFixMessage } from "./runFixMessage";import { RunInputsSection, type RunInputMeta } from "./RunInputsSection";import { RunOutputsSection, type RunOutputFile } from "./RunOutputsSection";@@ -190,10 +201,6 @@ export function RunView({);const lastFrame = frames.length > 0 ? frames[frames.length - 1] : null;- const shownFrame =- (pinnedFrameId ? frames.find((f) => f.id === pinnedFrameId) : undefined) ??- lastFrame ??- null;const finalized = workflowRun ? statusIsFinalized(workflowRun) : false;const finallyBlockLabel =@@ -206,6 +213,47 @@ export function RunView({[timeline, selectedId, finalized, finallyBlockLabel],);+ // Mirror the legacy run page: an action shows its own screenshot, a block shows+ // its representative screenshot (a loop/conditional container resolves to its leaf).+ const heroSelection = useMemo<HeroSelection | null>(() => {+ if (isAction(activeItem)) {+ return {+ kind: "action",+ artifactId: activeItem.screenshot_artifact_id ?? null,+ stepId: activeItem.step_id ?? null,+ actionOrder: activeItem.action_order ?? null,+ };+ }+ if (isWorkflowRunBlock(activeItem)) {+ const screenshotBlockId = resolveScreenshotBlockId(+ timeline ?? [],+ activeItem,+ activeIteration,+ );+ const blockType =+ findTimelineBlock(timeline ?? [], screenshotBlockId)?.block_type ??+ activeItem.block_type ??+ null;+ return {+ kind: "block",+ workflowRunBlockId: screenshotBlockId,+ blockType,+ };+ }+ if (isObserverThought(activeItem)) {+ return { kind: "thought", thoughtId: activeItem.thought_id };+ }+ return null;+ }, [activeItem, timeline, activeIteration]);++ const heroLabel = isAction(activeItem)+ ? actionLabel(activeItem)+ : isWorkflowRunBlock(activeItem)+ ? (activeItem.label ?? "Screenshot")+ : isObserverThought(activeItem)+ ? (activeItem.thought ?? "Thought")+ : "Screenshot";+const extractedInformation = useMemo<Record<string, unknown> | null>(() => {const outputs = workflowRun?.outputs;return typeof outputs === "object" &&@@ -282,7 +330,8 @@ export function RunView({/><RunHeroworkflowRunId={workflowRun.workflow_run_id}- shownFrame={shownFrame}+ heroSelection={heroSelection}+ heroLabel={heroLabel}running={running}showDebugStream={showDebugStream}provisioning={