Skip to content

Automate browser based workflows with AI

add workflow run block screenshots (#1443)

Shuchang Zheng4e436395315c
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/ObserverThoughtScreenshot.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/ObserverThoughtScreenshot.tsx
index 705e73217..5daa23895 100644
--- a/skyvern-frontend/src/routes/workflows/workflowRun/ObserverThoughtScreenshot.tsx
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/ObserverThoughtScreenshot.tsx
@@ -61,7 +61,7 @@ function ObserverThoughtScreenshot({ observerThoughtId, taskStatus }: Props) {
if (!screenshot) {
return (
<div className="flex h-full items-center justify-center bg-slate-elevation1">
- No screenshot found for this action.
+ No screenshot found for this thought.
</div>
);
}
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunBlockScreenshot.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunBlockScreenshot.tsx
new file mode 100644
index 000000000..4ac6bb6b2
--- /dev/null
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunBlockScreenshot.tsx
@@ -0,0 +1,66 @@
+import { getClient } from "@/api/AxiosClient";
+import { ArtifactApiResponse, ArtifactType } from "@/api/types";
+import { ZoomableImage } from "@/components/ZoomableImage";
+import { useCredentialGetter } from "@/hooks/useCredentialGetter";
+import { useQuery } from "@tanstack/react-query";
+import { ReloadIcon } from "@radix-ui/react-icons";
+import { getImageURL } from "@/routes/tasks/detail/artifactUtils";
+
+type Props = {
+ workflowRunBlockId: string;
+};
+
+function WorkflowRunBlockScreenshot({ workflowRunBlockId }: Props) {
+ const credentialGetter = useCredentialGetter();
+
+ const { data: artifacts, isLoading } = useQuery<Array<ArtifactApiResponse>>({
+ queryKey: ["workflowRunBlock", workflowRunBlockId, "artifacts"],
+ queryFn: async () => {
+ const client = await getClient(credentialGetter);
+ return client
+ .get(`/workflow_run_block/${workflowRunBlockId}/artifacts`)
+ .then((response) => response.data);
+ },
+ refetchInterval: (query) => {
+ const data = query.state.data;
+ const screenshot = data?.filter(
+ (artifact) => artifact.artifact_type === ArtifactType.LLMScreenshot,
+ )?.[0];
+ if (!screenshot) {
+ return 5000;
+ }
+ return false;
+ },
+ });
+
+ const llmScreenshots = artifacts?.filter(
+ (artifact) => artifact.artifact_type === ArtifactType.LLMScreenshot,
+ );
+
+ const screenshot = llmScreenshots?.[0];
+
+ if (isLoading) {
+ return (
+ <div className="flex h-full items-center justify-center gap-2 bg-slate-elevation1">
+ <ReloadIcon className="h-6 w-6 animate-spin" />
+ <div>Loading screenshot...</div>
+ </div>
+ );
+ }
+
+ if (!screenshot) {
+ return (
+ <div className="flex h-full items-center justify-center bg-slate-elevation1">
+ No screenshot found for this workflow run block.
+ </div>
+ );
+ }
+
+ return (
+ <figure className="mx-auto flex max-w-full flex-col items-center gap-2 overflow-hidden rounded">
+ <ZoomableImage src={getImageURL(screenshot)} alt="llm-screenshot" />
+ </figure>
+ );
+}
+
+export { WorkflowRunBlockScreenshot };
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx
index bfa655b25..143c98382 100644
--- a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx
@@ -23,6 +23,7 @@ import { DotFilledIcon } from "@radix-ui/react-icons";
import { WorkflowRunTimelineItemInfoSection } from "./WorkflowRunTimelineItemInfoSection";
import { ObserverThoughtScreenshot } from "./ObserverThoughtScreenshot";
import { ScrollArea, ScrollAreaViewport } from "@/components/ui/scroll-area";
+import { WorkflowRunBlockScreenshot } from "./WorkflowRunBlockScreenshot";
export type ActionItem = {
block: WorkflowRunBlock;
@@ -103,9 +104,9 @@ function WorkflowRunOverview() {
/>
)}
{isWorkflowRunBlock(selection) && (
- <div className="flex h-full w-full items-center justify-center bg-slate-elevation1">
- No screenshot found for this block
- </div>
+ <WorkflowRunBlockScreenshot
+ workflowRunBlockId={selection.workflow_run_block_id}
+ />
)}
{isObserverThought(selection) && (
<ObserverThoughtScreenshot