Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Use actions api for new tasks (#1027)
Shuchang Zhengd6fd4f8923bd
diff --git a/skyvern-frontend/src/api/types.ts b/skyvern-frontend/src/api/types.tsindex cfebe57a1..ea0134793 100644--- a/skyvern-frontend/src/api/types.ts+++ b/skyvern-frontend/src/api/types.ts@@ -276,3 +276,17 @@ export type TaskGenerationApiResponse = {navigation_payload: Record<string, unknown> | null;extracted_information_schema: Record<string, unknown> | null;};++export type ActionsApiResponse = {+ action_type: ActionType;+ status: Status;+ task_id: string | null;+ step_id: string | null;+ step_order: number | null;+ action_order: number | null;+ confidence_float: number | null;+ description: string | null;+ reasoning: string | null;+ intention: string | null;+ response: string | null;+};diff --git a/skyvern-frontend/src/routes/tasks/detail/TaskActions.tsx b/skyvern-frontend/src/routes/tasks/detail/TaskActions.tsxindex 4ebc07ceb..799979a94 100644--- a/skyvern-frontend/src/routes/tasks/detail/TaskActions.tsx+++ b/skyvern-frontend/src/routes/tasks/detail/TaskActions.tsx@@ -1,23 +1,18 @@-import { useEffect, useState } from "react";-import { useParams } from "react-router-dom";-import { ActionScreenshot } from "./ActionScreenshot";-import { ScrollableActionList } from "./ScrollableActionList";-import { keepPreviousData, useQuery } from "@tanstack/react-query";-import {- ActionApiResponse,- ActionTypes,- Status,- StepApiResponse,- TaskApiResponse,-} from "@/api/types";import { getClient } from "@/api/AxiosClient";-import { useCredentialGetter } from "@/hooks/useCredentialGetter";+import { Status, StepApiResponse, TaskApiResponse } from "@/api/types";import { Skeleton } from "@/components/ui/skeleton";import { toast } from "@/components/ui/use-toast";-import { envCredential } from "@/util/env";-import { statusIsNotFinalized, statusIsRunningOrQueued } from "../types";import { ZoomableImage } from "@/components/ZoomableImage";import { useCostCalculator } from "@/hooks/useCostCalculator";+import { useCredentialGetter } from "@/hooks/useCredentialGetter";+import { envCredential } from "@/util/env";+import { keepPreviousData, useQuery } from "@tanstack/react-query";+import { useEffect, useState } from "react";+import { useParams } from "react-router-dom";+import { statusIsNotFinalized, statusIsRunningOrQueued } from "../types";+import { ActionScreenshot } from "./ActionScreenshot";+import { useActions } from "./hooks/useActions";+import { ScrollableActionList } from "./ScrollableActionList";const formatter = Intl.NumberFormat("en-US", {style: "currency",@@ -34,18 +29,6 @@ let socket: WebSocket | null = null;const wssBaseUrl = import.meta.env.VITE_WSS_BASE_URL;-function getActionInput(action: ActionApiResponse) {- let input = "";- if (action.action_type === ActionTypes.InputText && action.text) {- input = action.text;- } else if (action.action_type === ActionTypes.Click) {- input = "Click";- } else if (action.action_type === ActionTypes.SelectOption && action.option) {- input = action.option.label;- }- return input;-}-function TaskActions() {const { taskId } = useParams();const credentialGetter = useCredentialGetter();@@ -165,31 +148,11 @@ function TaskActions() {placeholderData: keepPreviousData,});- const actions = steps- ?.map((step) => {- const actionsAndResults = step.output?.actions_and_results ?? [];-- const actions = actionsAndResults.map((actionAndResult, index) => {- const action = actionAndResult[0];- const actionResult = actionAndResult[1];- if (actionResult.length === 0) {- return null;- }- return {- reasoning: action.reasoning,- confidence: action.confidence_float,- input: getActionInput(action),- type: action.action_type,- success: actionResult?.[0]?.success ?? false,- stepId: step.step_id,- index,- };- });- return actions;- })- .flat();+ const { data: actions, isLoading: actionsIsLoading } = useActions({+ id: taskId,+ });- if (taskIsLoading || stepsIsLoading) {+ if (taskIsLoading || actionsIsLoading || stepsIsLoading) {return (<div className="flex gap-2"><div className="h-[40rem] w-3/4">diff --git a/skyvern-frontend/src/routes/tasks/detail/hooks/useActions.ts b/skyvern-frontend/src/routes/tasks/detail/hooks/useActions.tsnew file mode 100644index 000000000..4137a60d9--- /dev/null+++ b/skyvern-frontend/src/routes/tasks/detail/hooks/useActions.ts@@ -0,0 +1,133 @@+import { getClient } from "@/api/AxiosClient";+import {+ Action,+ ActionApiResponse,+ ActionsApiResponse,+ ActionTypes,+ Status,+ StepApiResponse,+ TaskApiResponse,+} from "@/api/types";+import { useCredentialGetter } from "@/hooks/useCredentialGetter";+import { keepPreviousData, useQuery } from "@tanstack/react-query";+import { statusIsNotFinalized } from "../../types";++function getActionInput(action: ActionApiResponse) {+ let input = "";+ if (action.action_type === ActionTypes.InputText && action.text) {+ input = action.text;+ } else if (action.action_type === ActionTypes.Click) {+ input = "Click";+ } else if (action.action_type === ActionTypes.SelectOption && action.option) {+ input = action.option.label;+ }+ return input;+}++type Props = {+ id?: string;+};++function isOld(task: TaskApiResponse) {+ return new Date(task.created_at) < new Date(2024, 9, 21);+}++function useActions({ id }: Props): {+ data: Array<Action | null>;+ isLoading: boolean;+} {+ const credentialGetter = useCredentialGetter();++ const { data: task, isLoading: taskIsLoading } = useQuery<TaskApiResponse>({+ queryKey: ["task", id],+ queryFn: async () => {+ const client = await getClient(credentialGetter);+ return client.get(`/tasks/${id}`).then((response) => response.data);+ },+ refetchInterval: (query) => {+ if (!query.state.data) {+ return false;+ }+ if (statusIsNotFinalized(query.state.data)) {+ return 5000;+ }+ return false;+ },+ placeholderData: keepPreviousData,+ });++ const taskIsNotFinalized = task && statusIsNotFinalized(task);++ const { data: taskActions, isLoading: taskActionsIsLoading } = useQuery<+ Array<ActionsApiResponse>+ >({+ queryKey: ["tasks", id, "actions"],+ queryFn: async () => {+ const client = await getClient(credentialGetter);+ return client+ .get(`/tasks/${id}/actions`)+ .then((response) => response.data);+ },+ refetchInterval: taskIsNotFinalized ? 5000 : false,+ placeholderData: keepPreviousData,+ enabled: Boolean(task && !isOld(task)),+ });++ const { data: steps, isLoading: stepsIsLoading } = useQuery<+ Array<StepApiResponse>+ >({+ queryKey: ["task", id, "steps"],+ queryFn: async () => {+ const client = await getClient(credentialGetter);+ return client.get(`/tasks/${id}/steps`).then((response) => response.data);+ },+ enabled: Boolean(task && isOld(task)),+ refetchOnWindowFocus: taskIsNotFinalized,+ refetchInterval: taskIsNotFinalized ? 5000 : false,+ placeholderData: keepPreviousData,+ });++ const actions =+ task && isOld(task)+ ? steps+ ?.map((step) => {+ const actionsAndResults = step.output?.actions_and_results ?? [];++ const actions = actionsAndResults.map((actionAndResult, index) => {+ const action = actionAndResult[0];+ const actionResult = actionAndResult[1];+ if (actionResult.length === 0) {+ return null;+ }+ return {+ reasoning: action.reasoning,+ confidence: action.confidence_float,+ input: getActionInput(action),+ type: action.action_type,+ success: actionResult?.[0]?.success ?? false,+ stepId: step.step_id,+ index,+ };+ });+ return actions;+ })+ .flat()+ : taskActions?.map((action) => {+ return {+ reasoning: action.reasoning ?? "",+ confidence: action.confidence_float ?? undefined,+ input: action.response ?? "",+ type: action.action_type,+ success: action.status === Status.Completed,+ stepId: action.step_id ?? "",+ index: action.action_order ?? 0,+ };+ });++ return {+ data: actions ?? [],+ isLoading: taskIsLoading || taskActionsIsLoading || stepsIsLoading,+ };+}++export { useActions };