Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
link actions to their screenshots - frontend (#4403)
Celal Zamanogluf5cb826a37f4
diff --git a/skyvern-frontend/src/api/types.ts b/skyvern-frontend/src/api/types.tsindex 2070f4cc1..adf561a7c 100644--- a/skyvern-frontend/src/api/types.ts+++ b/skyvern-frontend/src/api/types.ts@@ -328,6 +328,7 @@ export type ActionApiResponse = {option: Option | null;file_url: string | null;created_by: string | null;+ screenshot_artifact_id?: string | null;};export type Action = {@@ -339,6 +340,7 @@ export type Action = {stepId: string;index: number;created_by: string | null;+ screenshotArtifactId?: string | null;};export type EvalKind = "workflow" | "task";@@ -464,6 +466,7 @@ export type ActionsApiResponse = {response: string | null;created_by: string | null;text: string | null;+ screenshot_artifact_id?: string | null;};export type TaskV2 = {diff --git a/skyvern-frontend/src/routes/tasks/detail/ActionScreenshot.tsx b/skyvern-frontend/src/routes/tasks/detail/ActionScreenshot.tsxindex 78452d22c..079ed2c3f 100644--- a/skyvern-frontend/src/routes/tasks/detail/ActionScreenshot.tsx+++ b/skyvern-frontend/src/routes/tasks/detail/ActionScreenshot.tsx@@ -9,18 +9,37 @@ import { statusIsNotFinalized } from "../types";import { apiPathPrefix } from "@/util/env";type Props = {+ artifactId?: string;stepId: string;index: number;taskStatus?: Status; // to give a hint that screenshot may not be available if task is not finalized};-function ActionScreenshot({ stepId, index, taskStatus }: Props) {+function ActionScreenshot({ artifactId, stepId, index, taskStatus }: Props) {const credentialGetter = useCredentialGetter();+ const {+ data: artifactById,+ isLoading: isLoadingArtifactById,+ isFetching: isFetchingArtifactById,+ } = useQuery<ArtifactApiResponse>({+ queryKey: ["artifact", artifactId],+ queryFn: async () => {+ const client = await getClient(credentialGetter);+ return client+ .get(`${apiPathPrefix}/artifacts/${artifactId}`)+ .then((response) => response.data);+ },+ enabled: Boolean(artifactId),+ refetchOnWindowFocus: false,+ refetchOnMount: false,+ retry: 1,+ });+const {data: artifacts,- isLoading,- isFetching,+ isLoading: isLoadingStepArtifacts,+ isFetching: isFetchingStepArtifacts,} = useQuery<Array<ArtifactApiResponse>>({queryKey: ["step", stepId, "artifacts"],queryFn: async () => {@@ -39,14 +58,19 @@ function ActionScreenshot({ stepId, index, taskStatus }: Props) {}return false;},+ enabled: !artifactId, // fallback path only when explicit artifact id is absent});const actionScreenshots = artifacts?.filter((artifact) => artifact.artifact_type === ArtifactType.ActionScreenshot,);- // action screenshots are reverse ordered w.r.t action order- const screenshot = actionScreenshots?.[actionScreenshots.length - index - 1];+ const screenshotFromStep =+ actionScreenshots?.[actionScreenshots.length - index - 1];+ const screenshot = artifactById ?? screenshotFromStep;++ const isLoading = isLoadingArtifactById || isLoadingStepArtifacts;+ const isFetching = isFetchingArtifactById || isFetchingStepArtifacts;if (isLoading) {return (diff --git a/skyvern-frontend/src/routes/tasks/detail/TaskActions.tsx b/skyvern-frontend/src/routes/tasks/detail/TaskActions.tsxindex 669072945..329ea8076 100644--- a/skyvern-frontend/src/routes/tasks/detail/TaskActions.tsx+++ b/skyvern-frontend/src/routes/tasks/detail/TaskActions.tsx@@ -237,6 +237,7 @@ function TaskActions() {{activeSelection === "stream" ? getStream() : null}{typeof activeSelection === "number" && activeAction ? (<ActionScreenshot+ artifactId={activeAction.screenshotArtifactId ?? undefined}stepId={activeAction.stepId}index={activeAction.index}taskStatus={task?.status}diff --git a/skyvern-frontend/src/routes/tasks/detail/hooks/useActions.ts b/skyvern-frontend/src/routes/tasks/detail/hooks/useActions.tsindex 3185a0c00..5e1ac7082 100644--- a/skyvern-frontend/src/routes/tasks/detail/hooks/useActions.ts+++ b/skyvern-frontend/src/routes/tasks/detail/hooks/useActions.ts@@ -127,6 +127,7 @@ function useActions({ id }: Props): {stepId: action.step_id ?? "",index: index,created_by: action.created_by,+ screenshotArtifactId: action.screenshot_artifact_id ?? undefined,};});diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsxindex a9c3c9417..b3f479abf 100644--- a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx@@ -122,6 +122,7 @@ function WorkflowRunOverview() {!showStreamingBrowser &&isAction(selection) && (<ActionScreenshot+ artifactId={selection.screenshot_artifact_id ?? undefined}index={selection.action_order ?? 0}stepId={selection.step_id ?? ""}/>