Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
infer action type from instruction (#1231)
LawyZhengbb6d3e6a37fc
diff --git a/skyvern/forge/prompts/skyvern/infer-action-type.j2 b/skyvern/forge/prompts/skyvern/infer-action-type.j2new file mode 100644index 000000000..cc343ac1c--- /dev/null+++ b/skyvern/forge/prompts/skyvern/infer-action-type.j2@@ -0,0 +1,16 @@+You are a browser agent performing actions on the web. You are instructed to take a single action. Help to identify which action type should be taken according to the action instruction.++MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.++Reply in the following JSON format:+{+ "thought": str, // A string to describe how to infer the action type from the action instruction.+ "confidence_float": float, // The confidence of the action. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence+ "action_type": str, // It's a string enum: "CLICK", "INPUT_TEXT", "UPLOAD_FILE", "SELECT_OPTION". "CLICK" means user wants to click. "INPUT_TEXT" means user wants to input text. "UPLOAD_FILE" means user wants to upload a file. "SELECT_OPTION" means user wants to select an option.+ "error": str, // It's a string enum to describe error. Null if you can identify the action as one of the defined action type. Use "UNKNOWN_ACTION" if none of the defined action type matched. Use "MULTIPLE_ACTIONS" if the instruction includes multiple actions.+}++Action instruction+```+{{ navigation_goal }}+```\ No newline at end of filediff --git a/skyvern/forge/sdk/workflow/exceptions.py b/skyvern/forge/sdk/workflow/exceptions.pyindex 96e09813d..a49d6fd11 100644--- a/skyvern/forge/sdk/workflow/exceptions.py+++ b/skyvern/forge/sdk/workflow/exceptions.py@@ -107,3 +107,10 @@ class WorkflowParameterMissingRequiredValue(BaseWorkflowHTTPException):f"Missing required value for workflow parameter. Workflow parameter type: {workflow_parameter_type}. workflow_parameter_key: {workflow_parameter_key}. Required value: {required_value}",status_code=status.HTTP_400_BAD_REQUEST,)+++class FailedToParseActionInstruction(SkyvernException):+ def __init__(self, reason: str | None, error_type: str | None):+ super().__init__(+ f"Failed to parse the action instruction as '{reason}({error_type})'",+ )diff --git a/skyvern/forge/sdk/workflow/models/yaml.py b/skyvern/forge/sdk/workflow/models/yaml.pyindex 5dc4dcd0c..d22c93b27 100644--- a/skyvern/forge/sdk/workflow/models/yaml.py+++ b/skyvern/forge/sdk/workflow/models/yaml.py@@ -6,7 +6,6 @@ from pydantic import BaseModel, Fieldfrom skyvern.forge.sdk.schemas.tasks import ProxyLocationfrom skyvern.forge.sdk.workflow.models.block import BlockType, FileTypefrom skyvern.forge.sdk.workflow.models.parameter import ParameterType, WorkflowParameterType-from skyvern.webeye.actions.actions import ActionTypeclass ParameterYAML(BaseModel, abc.ABC):@@ -219,7 +218,6 @@ class ValidationBlockYAML(BlockYAML):class ActionBlockYAML(BlockYAML):- action_type: ActionTypeblock_type: Literal[BlockType.ACTION] = BlockType.ACTION # type: ignoreurl: str | None = Nonediff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex 51fa82e29..48506a255 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -14,6 +14,7 @@ from skyvern.exceptions import (WorkflowRunNotFound,)from skyvern.forge import app+from skyvern.forge.prompts import prompt_enginefrom skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.security import generate_skyvern_signature@@ -23,6 +24,7 @@ from skyvern.forge.sdk.models import Organization, Stepfrom skyvern.forge.sdk.schemas.tasks import ProxyLocation, Taskfrom skyvern.forge.sdk.workflow.exceptions import (ContextParameterSourceNotDefined,+ FailedToParseActionInstruction,InvalidWorkflowDefinition,WorkflowDefinitionHasDuplicateParameterKeys,WorkflowDefinitionHasReservedParameterKeys,@@ -1366,18 +1368,35 @@ class WorkflowService:if block_yaml.parameter_keyselse [])++ if not block_yaml.navigation_goal:+ raise Exception("empty action instruction")++ prompt = prompt_engine.load_prompt("infer-action-type", navigation_goal=block_yaml.navigation_goal)+ # TODO: no step here, so LLM call won't be saved as an artifact+ json_response = await app.LLM_API_HANDLER(prompt=prompt)+ if json_response.get("error"):+ raise FailedToParseActionInstruction(+ reason=json_response.get("thought"), error_type=json_response.get("error")+ )++ action_type: str = json_response.get("action_type") or ""+ action_type = action_type.lower()+prompt_template = ""- if block_yaml.action_type == ActionType.CLICK:+ if action_type == ActionType.CLICK:prompt_template = TaskPromptTemplate.SingleClickAction- elif block_yaml.action_type == ActionType.INPUT_TEXT:+ elif action_type == ActionType.INPUT_TEXT:prompt_template = TaskPromptTemplate.SingleInputAction- elif block_yaml.action_type == ActionType.UPLOAD_FILE:+ elif action_type == ActionType.UPLOAD_FILE:prompt_template = TaskPromptTemplate.SingleUploadAction- elif block_yaml.action_type == ActionType.SELECT_OPTION:+ elif action_type == ActionType.SELECT_OPTION:prompt_template = TaskPromptTemplate.SingleSelectActionif not prompt_template:- raise Exception("not supported action type for action block")+ raise Exception(+ f"Not supported action for action block. Currently we only support [click, input_text, upload_file, select_option], but got [{action_type}]"+ )return ActionBlock(prompt_template=prompt_template,