Skip to content

Automate browser based workflows with AI

link actions to their screenshots - frontend (#4403)

Celal Zamanogluf5cb826a37f4
diff --git a/skyvern-frontend/src/api/types.ts b/skyvern-frontend/src/api/types.ts
index 2070f4cc1..adf561a7c 100644
--- a/skyvern-frontend/src/api/types.ts
+++ b/skyvern-frontend/src/api/types.ts
@@ -328,6 +328,7 @@ export type ActionApiResponse = {
option: Option | null;
file_url: string | null;
created_by: string | null;
+ screenshot_artifact_id?: string | null;
};
export type Action = {
@@ -339,6 +340,7 @@ export type Action = {
stepId: string;
index: number;
created_by: string | null;
+ screenshotArtifactId?: string | null;
};
export type EvalKind = "workflow" | "task";
@@ -464,6 +466,7 @@ export type ActionsApiResponse = {
response: string | null;
created_by: string | null;
text: string | null;
+ screenshot_artifact_id?: string | null;
};
export type TaskV2 = {
diff --git a/skyvern-frontend/src/routes/tasks/detail/ActionScreenshot.tsx b/skyvern-frontend/src/routes/tasks/detail/ActionScreenshot.tsx
index 78452d22c..079ed2c3f 100644
--- a/skyvern-frontend/src/routes/tasks/detail/ActionScreenshot.tsx
+++ b/skyvern-frontend/src/routes/tasks/detail/ActionScreenshot.tsx
@@ -9,18 +9,37 @@ import { statusIsNotFinalized } from "../types";
import { apiPathPrefix } from "@/util/env";
type Props = {
+ artifactId?: string;
stepId: string;
index: number;
taskStatus?: Status; // to give a hint that screenshot may not be available if task is not finalized
};
-function ActionScreenshot({ stepId, index, taskStatus }: Props) {
+function ActionScreenshot({ artifactId, stepId, index, taskStatus }: Props) {
const credentialGetter = useCredentialGetter();
+ const {
+ data: artifactById,
+ isLoading: isLoadingArtifactById,
+ isFetching: isFetchingArtifactById,
+ } = useQuery<ArtifactApiResponse>({
+ queryKey: ["artifact", artifactId],
+ queryFn: async () => {
+ const client = await getClient(credentialGetter);
+ return client
+ .get(`${apiPathPrefix}/artifacts/${artifactId}`)
+ .then((response) => response.data);
+ },
+ enabled: Boolean(artifactId),
+ refetchOnWindowFocus: false,
+ refetchOnMount: false,
+ retry: 1,
+ });
+
const {
data: artifacts,
- isLoading,
- isFetching,
+ isLoading: isLoadingStepArtifacts,
+ isFetching: isFetchingStepArtifacts,
} = useQuery<Array<ArtifactApiResponse>>({
queryKey: ["step", stepId, "artifacts"],
queryFn: async () => {
@@ -39,14 +58,19 @@ function ActionScreenshot({ stepId, index, taskStatus }: Props) {
}
return false;
},
+ enabled: !artifactId, // fallback path only when explicit artifact id is absent
});
const actionScreenshots = artifacts?.filter(
(artifact) => artifact.artifact_type === ArtifactType.ActionScreenshot,
);
- // action screenshots are reverse ordered w.r.t action order
- const screenshot = actionScreenshots?.[actionScreenshots.length - index - 1];
+ const screenshotFromStep =
+ actionScreenshots?.[actionScreenshots.length - index - 1];
+ const screenshot = artifactById ?? screenshotFromStep;
+
+ const isLoading = isLoadingArtifactById || isLoadingStepArtifacts;
+ const isFetching = isFetchingArtifactById || isFetchingStepArtifacts;
if (isLoading) {
return (
diff --git a/skyvern-frontend/src/routes/tasks/detail/TaskActions.tsx b/skyvern-frontend/src/routes/tasks/detail/TaskActions.tsx
index 669072945..329ea8076 100644
--- a/skyvern-frontend/src/routes/tasks/detail/TaskActions.tsx
+++ b/skyvern-frontend/src/routes/tasks/detail/TaskActions.tsx
@@ -237,6 +237,7 @@ function TaskActions() {
{activeSelection === "stream" ? getStream() : null}
{typeof activeSelection === "number" && activeAction ? (
<ActionScreenshot
+ artifactId={activeAction.screenshotArtifactId ?? undefined}
stepId={activeAction.stepId}
index={activeAction.index}
taskStatus={task?.status}
diff --git a/skyvern-frontend/src/routes/tasks/detail/hooks/useActions.ts b/skyvern-frontend/src/routes/tasks/detail/hooks/useActions.ts
index 3185a0c00..5e1ac7082 100644
--- a/skyvern-frontend/src/routes/tasks/detail/hooks/useActions.ts
+++ b/skyvern-frontend/src/routes/tasks/detail/hooks/useActions.ts
@@ -127,6 +127,7 @@ function useActions({ id }: Props): {
stepId: action.step_id ?? "",
index: index,
created_by: action.created_by,
+ screenshotArtifactId: action.screenshot_artifact_id ?? undefined,
};
});
diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx
index a9c3c9417..b3f479abf 100644
--- a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx
@@ -122,6 +122,7 @@ function WorkflowRunOverview() {
!showStreamingBrowser &&
isAction(selection) && (
<ActionScreenshot
+ artifactId={selection.screenshot_artifact_id ?? undefined}
index={selection.action_order ?? 0}
stepId={selection.step_id ?? ""}
/>