Skip to content

Automate browser based workflows with AI

Use actions api for new tasks (#1027)

Shuchang Zhengd6fd4f8923bd
diff --git a/skyvern-frontend/src/api/types.ts b/skyvern-frontend/src/api/types.ts
index cfebe57a1..ea0134793 100644
--- a/skyvern-frontend/src/api/types.ts
+++ b/skyvern-frontend/src/api/types.ts
@@ -276,3 +276,17 @@ export type TaskGenerationApiResponse = {
navigation_payload: Record<string, unknown> | null;
extracted_information_schema: Record<string, unknown> | null;
};
+
+export type ActionsApiResponse = {
+ action_type: ActionType;
+ status: Status;
+ task_id: string | null;
+ step_id: string | null;
+ step_order: number | null;
+ action_order: number | null;
+ confidence_float: number | null;
+ description: string | null;
+ reasoning: string | null;
+ intention: string | null;
+ response: string | null;
+};
diff --git a/skyvern-frontend/src/routes/tasks/detail/TaskActions.tsx b/skyvern-frontend/src/routes/tasks/detail/TaskActions.tsx
index 4ebc07ceb..799979a94 100644
--- a/skyvern-frontend/src/routes/tasks/detail/TaskActions.tsx
+++ b/skyvern-frontend/src/routes/tasks/detail/TaskActions.tsx
@@ -1,23 +1,18 @@
-import { useEffect, useState } from "react";
-import { useParams } from "react-router-dom";
-import { ActionScreenshot } from "./ActionScreenshot";
-import { ScrollableActionList } from "./ScrollableActionList";
-import { keepPreviousData, useQuery } from "@tanstack/react-query";
-import {
- ActionApiResponse,
- ActionTypes,
- Status,
- StepApiResponse,
- TaskApiResponse,
-} from "@/api/types";
import { getClient } from "@/api/AxiosClient";
-import { useCredentialGetter } from "@/hooks/useCredentialGetter";
+import { Status, StepApiResponse, TaskApiResponse } from "@/api/types";
import { Skeleton } from "@/components/ui/skeleton";
import { toast } from "@/components/ui/use-toast";
-import { envCredential } from "@/util/env";
-import { statusIsNotFinalized, statusIsRunningOrQueued } from "../types";
import { ZoomableImage } from "@/components/ZoomableImage";
import { useCostCalculator } from "@/hooks/useCostCalculator";
+import { useCredentialGetter } from "@/hooks/useCredentialGetter";
+import { envCredential } from "@/util/env";
+import { keepPreviousData, useQuery } from "@tanstack/react-query";
+import { useEffect, useState } from "react";
+import { useParams } from "react-router-dom";
+import { statusIsNotFinalized, statusIsRunningOrQueued } from "../types";
+import { ActionScreenshot } from "./ActionScreenshot";
+import { useActions } from "./hooks/useActions";
+import { ScrollableActionList } from "./ScrollableActionList";
const formatter = Intl.NumberFormat("en-US", {
style: "currency",
@@ -34,18 +29,6 @@ let socket: WebSocket | null = null;
const wssBaseUrl = import.meta.env.VITE_WSS_BASE_URL;
-function getActionInput(action: ActionApiResponse) {
- let input = "";
- if (action.action_type === ActionTypes.InputText && action.text) {
- input = action.text;
- } else if (action.action_type === ActionTypes.Click) {
- input = "Click";
- } else if (action.action_type === ActionTypes.SelectOption && action.option) {
- input = action.option.label;
- }
- return input;
-}
-
function TaskActions() {
const { taskId } = useParams();
const credentialGetter = useCredentialGetter();
@@ -165,31 +148,11 @@ function TaskActions() {
placeholderData: keepPreviousData,
});
- const actions = steps
- ?.map((step) => {
- const actionsAndResults = step.output?.actions_and_results ?? [];
-
- const actions = actionsAndResults.map((actionAndResult, index) => {
- const action = actionAndResult[0];
- const actionResult = actionAndResult[1];
- if (actionResult.length === 0) {
- return null;
- }
- return {
- reasoning: action.reasoning,
- confidence: action.confidence_float,
- input: getActionInput(action),
- type: action.action_type,
- success: actionResult?.[0]?.success ?? false,
- stepId: step.step_id,
- index,
- };
- });
- return actions;
- })
- .flat();
+ const { data: actions, isLoading: actionsIsLoading } = useActions({
+ id: taskId,
+ });
- if (taskIsLoading || stepsIsLoading) {
+ if (taskIsLoading || actionsIsLoading || stepsIsLoading) {
return (
<div className="flex gap-2">
<div className="h-[40rem] w-3/4">
diff --git a/skyvern-frontend/src/routes/tasks/detail/hooks/useActions.ts b/skyvern-frontend/src/routes/tasks/detail/hooks/useActions.ts
new file mode 100644
index 000000000..4137a60d9
--- /dev/null
+++ b/skyvern-frontend/src/routes/tasks/detail/hooks/useActions.ts
@@ -0,0 +1,133 @@
+import { getClient } from "@/api/AxiosClient";
+import {
+ Action,
+ ActionApiResponse,
+ ActionsApiResponse,
+ ActionTypes,
+ Status,
+ StepApiResponse,
+ TaskApiResponse,
+} from "@/api/types";
+import { useCredentialGetter } from "@/hooks/useCredentialGetter";
+import { keepPreviousData, useQuery } from "@tanstack/react-query";
+import { statusIsNotFinalized } from "../../types";
+
+function getActionInput(action: ActionApiResponse) {
+ let input = "";
+ if (action.action_type === ActionTypes.InputText && action.text) {
+ input = action.text;
+ } else if (action.action_type === ActionTypes.Click) {
+ input = "Click";
+ } else if (action.action_type === ActionTypes.SelectOption && action.option) {
+ input = action.option.label;
+ }
+ return input;
+}
+
+type Props = {
+ id?: string;
+};
+
+function isOld(task: TaskApiResponse) {
+ return new Date(task.created_at) < new Date(2024, 9, 21);
+}
+
+function useActions({ id }: Props): {
+ data: Array<Action | null>;
+ isLoading: boolean;
+} {
+ const credentialGetter = useCredentialGetter();
+
+ const { data: task, isLoading: taskIsLoading } = useQuery<TaskApiResponse>({
+ queryKey: ["task", id],
+ queryFn: async () => {
+ const client = await getClient(credentialGetter);
+ return client.get(`/tasks/${id}`).then((response) => response.data);
+ },
+ refetchInterval: (query) => {
+ if (!query.state.data) {
+ return false;
+ }
+ if (statusIsNotFinalized(query.state.data)) {
+ return 5000;
+ }
+ return false;
+ },
+ placeholderData: keepPreviousData,
+ });
+
+ const taskIsNotFinalized = task && statusIsNotFinalized(task);
+
+ const { data: taskActions, isLoading: taskActionsIsLoading } = useQuery<
+ Array<ActionsApiResponse>
+ >({
+ queryKey: ["tasks", id, "actions"],
+ queryFn: async () => {
+ const client = await getClient(credentialGetter);
+ return client
+ .get(`/tasks/${id}/actions`)
+ .then((response) => response.data);
+ },
+ refetchInterval: taskIsNotFinalized ? 5000 : false,
+ placeholderData: keepPreviousData,
+ enabled: Boolean(task && !isOld(task)),
+ });
+
+ const { data: steps, isLoading: stepsIsLoading } = useQuery<
+ Array<StepApiResponse>
+ >({
+ queryKey: ["task", id, "steps"],
+ queryFn: async () => {
+ const client = await getClient(credentialGetter);
+ return client.get(`/tasks/${id}/steps`).then((response) => response.data);
+ },
+ enabled: Boolean(task && isOld(task)),
+ refetchOnWindowFocus: taskIsNotFinalized,
+ refetchInterval: taskIsNotFinalized ? 5000 : false,
+ placeholderData: keepPreviousData,
+ });
+
+ const actions =
+ task && isOld(task)
+ ? steps
+ ?.map((step) => {
+ const actionsAndResults = step.output?.actions_and_results ?? [];
+
+ const actions = actionsAndResults.map((actionAndResult, index) => {
+ const action = actionAndResult[0];
+ const actionResult = actionAndResult[1];
+ if (actionResult.length === 0) {
+ return null;
+ }
+ return {
+ reasoning: action.reasoning,
+ confidence: action.confidence_float,
+ input: getActionInput(action),
+ type: action.action_type,
+ success: actionResult?.[0]?.success ?? false,
+ stepId: step.step_id,
+ index,
+ };
+ });
+ return actions;
+ })
+ .flat()
+ : taskActions?.map((action) => {
+ return {
+ reasoning: action.reasoning ?? "",
+ confidence: action.confidence_float ?? undefined,
+ input: action.response ?? "",
+ type: action.action_type,
+ success: action.status === Status.Completed,
+ stepId: action.step_id ?? "",
+ index: action.action_order ?? 0,
+ };
+ });
+
+ return {
+ data: actions ?? [],
+ isLoading: taskIsLoading || taskActionsIsLoading || stepsIsLoading,
+ };
+}
+
+export { useActions };