Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
add new workflow block (#1228)
LawyZheng3f209404f7b7
diff --git a/alembic/versions/2024_11_21_0708-2d79d5fc1baa_add_task_prompt_template_complete_.py b/alembic/versions/2024_11_21_0708-2d79d5fc1baa_add_task_prompt_template_complete_.pynew file mode 100644index 000000000..e6c58d052--- /dev/null+++ b/alembic/versions/2024_11_21_0708-2d79d5fc1baa_add_task_prompt_template_complete_.py@@ -0,0 +1,35 @@+"""Add task prompt template, complete criterion and complete criterion++Revision ID: 2d79d5fc1baa+Revises: 1909715536dc+Create Date: 2024-11-21 07:08:19.177274+00:00++"""++from typing import Sequence, Union++import sqlalchemy as sa++from alembic import op++# revision identifiers, used by Alembic.+revision: str = "2d79d5fc1baa"+down_revision: Union[str, None] = "1909715536dc"+branch_labels: Union[str, Sequence[str], None] = None+depends_on: Union[str, Sequence[str], None] = None+++def upgrade() -> None:+ # ### commands auto generated by Alembic - please adjust! ###+ op.add_column("tasks", sa.Column("prompt_template", sa.String(), nullable=True))+ op.add_column("tasks", sa.Column("complete_criterion", sa.String(), nullable=True))+ op.add_column("tasks", sa.Column("terminate_criterion", sa.String(), nullable=True))+ # ### end Alembic commands ###+++def downgrade() -> None:+ # ### commands auto generated by Alembic - please adjust! ###+ op.drop_column("tasks", "terminate_criterion")+ op.drop_column("tasks", "complete_criterion")+ op.drop_column("tasks", "prompt_template")+ # ### end Alembic commands ###diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex c6dae5c9b..165e59074 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -39,11 +39,12 @@ from skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.security import generate_skyvern_signaturefrom skyvern.forge.sdk.core.validators import validate_url+from skyvern.forge.sdk.db.enums import TaskPromptTemplatefrom skyvern.forge.sdk.models import Organization, Step, StepStatusfrom skyvern.forge.sdk.schemas.tasks import Task, TaskRequest, TaskStatusfrom skyvern.forge.sdk.settings_manager import SettingsManagerfrom skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext-from skyvern.forge.sdk.workflow.models.block import TaskBlock+from skyvern.forge.sdk.workflow.models.block import ActionBlock, BaseTaskBlockfrom skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowRun, WorkflowRunStatusfrom skyvern.webeye.actions.actions import (Action,@@ -98,7 +99,7 @@ class ForgeAgent:async def create_task_and_step_from_block(self,- task_block: TaskBlock,+ task_block: BaseTaskBlock,workflow: Workflow,workflow_run: WorkflowRun,workflow_run_context: WorkflowRunContext,@@ -132,6 +133,9 @@ class ForgeAgent:task_url = validate_url(task_url)task = await app.DATABASE.create_task(url=task_url,+ prompt_template=task_block.prompt_template,+ complete_criterion=task_block.complete_criterion,+ terminate_criterion=task_block.terminate_criterion,title=task_block.title or task_block.label,webhook_callback_url=None,totp_verification_url=task_block.totp_verification_url,@@ -195,6 +199,8 @@ class ForgeAgent:totp_verification_url=totp_verification_url,totp_identifier=task_request.totp_identifier,navigation_goal=task_request.navigation_goal,+ complete_criterion=task_request.complete_criterion,+ terminate_criterion=task_request.terminate_criterion,data_extraction_goal=task_request.data_extraction_goal,navigation_payload=task_request.navigation_payload,organization_id=organization_id,@@ -222,7 +228,7 @@ class ForgeAgent:step: Step,api_key: str | None = None,close_browser_on_completion: bool = True,- task_block: TaskBlock | None = None,+ task_block: BaseTaskBlock | None = None,) -> Tuple[Step, DetailedAgentStepOutput | None, Step | None]:workflow_run: WorkflowRun | None = Noneif task.workflow_run_id:@@ -362,7 +368,13 @@ class ForgeAgent:is_task_completed,maybe_last_step,maybe_next_step,- ) = await self.handle_completed_step(organization, task, step, await browser_state.get_working_page())+ ) = await self.handle_completed_step(+ organization=organization,+ task=task,+ step=step,+ page=await browser_state.get_working_page(),+ task_block=task_block,+ )if is_task_completed is not None and maybe_last_step:last_step = maybe_last_stepawait self.clean_up_task(@@ -582,7 +594,7 @@ class ForgeAgent:step: Step,browser_state: BrowserState,organization: Organization | None = None,- task_block: TaskBlock | None = None,+ task_block: BaseTaskBlock | None = None,) -> tuple[Step, DetailedAgentStepOutput]:detailed_agent_step_output = DetailedAgentStepOutput(scraped_page=None,@@ -620,7 +632,7 @@ class ForgeAgent:actions: list[Action]using_cached_action_plan = False- if not task.navigation_goal:+ if not task.workflow_run_id and not task.navigation_goal:actions = [CompleteAction(reasoning="Task has no navigation goal.",@@ -870,7 +882,7 @@ class ForgeAgent:breaktask_completes_on_download = task_block and task_block.complete_on_download and task.workflow_run_id- if not has_decisive_action and not task_completes_on_download:+ if not has_decisive_action and not task_completes_on_download and not isinstance(task_block, ActionBlock):disable_user_goal_check = app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached("DISABLE_USER_GOAL_CHECK",task.task_id,@@ -1225,8 +1237,10 @@ class ForgeAgent:final_navigation_payload = self._build_navigation_payload(task, expire_verification_code=expire_verification_code)++ template = task.prompt_template if task.prompt_template else TaskPromptTemplate.ExtractActionreturn prompt_engine.load_prompt(- "extract-action",+ template=template,navigation_goal=navigation_goal,navigation_payload_str=json.dumps(final_navigation_payload),starting_url=starting_url,@@ -1237,6 +1251,8 @@ class ForgeAgent:error_code_mapping_str=(json.dumps(task.error_code_mapping) if task.error_code_mapping else None),utc_datetime=datetime.utcnow().strftime("%Y-%m-%d %H:%M"),verification_code_check=verification_code_check,+ complete_criterion=task.complete_criterion,+ terminate_criterion=task.terminate_criterion,)def _build_navigation_payload(@@ -1770,6 +1786,7 @@ class ForgeAgent:task: Task,step: Step,page: Page | None,+ task_block: BaseTaskBlock | None = None,) -> tuple[bool | None, Step | None, Step | None]:if step.is_goal_achieved():LOG.info(@@ -1810,6 +1827,24 @@ class ForgeAgent:or organization.max_steps_per_runor SettingsManager.get_settings().MAX_STEPS_PER_RUN)++ # HACK: action block only have one step to execute without complete action, so we consider the task is completed as long as the step is completed+ if isinstance(task_block, ActionBlock) and step.is_success():+ LOG.info(+ "Step completed for the action block, marking task as completed",+ task_id=task.task_id,+ step_id=step.step_id,+ step_order=step.order,+ step_retry=step.retry_index,+ output=step.output,+ )+ last_step = await self.update_step(step, is_last=True)+ await self.update_task(+ task,+ status=TaskStatus.completed,+ )+ return True, last_step, None+if step.order + 1 >= max_steps_per_run:LOG.info("Step completed but max steps reached, marking task as failed",diff --git a/skyvern/forge/prompts/skyvern/decisive-criterion-validate.j2 b/skyvern/forge/prompts/skyvern/decisive-criterion-validate.j2new file mode 100644index 000000000..68978ad0c--- /dev/null+++ b/skyvern/forge/prompts/skyvern/decisive-criterion-validate.j2@@ -0,0 +1,55 @@+Your are here to help the user determine if the current page has met the complete/terminte criterion. Use the criterions of complete/terminate, the content of the elements parsed from the page, the screenshots of the page, and user details to determine whether the criterions has been met.+++Reply in JSON format with the following keys:+{+ "page_info": str, // Think step by step. Describe all the useful information in the page related to the complete/terminate criterion.+ "thoughts": str, // Think step by step. What information makes you believe whether and which criterion has been met. Use information you see on the site to explain.+ "actions": array // You are supposed to give only one action("COMPLETE" or "TERMINATE") in the action list. Here's the format of the action:+ [{+ "reasoning": str, // The reasoning behind the action. This reasoning must be user information agnostic. Mention why you chose the action type, and why you chose the element id. Keep the reasoning short and to the point.+ "confidence_float": float, // The confidence of the action. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence+ "action_type": str, // It's a string enum: "COMPLETE", "TERMINATE". "COMPLETE" is used when the current page info has met the complete criterion. If there is no complete criterion, use "COMPLETE" as long as the page info hasn't met the terminate criterion. "TERMINATE" is used to terminate with a failure when the current page info has met the terminate criterion. It there is no terminate criterion, use "TERMINATE" as long as the page info hasn't met the complete criterion.+{% if error_code_mapping_str %}+ "errors": array // A list of errors. This is used to surface any errors that matches the current situation for COMPLETE and TERMINATE actions. For other actions or if no error description suits the current situation on the screenshots, return an empty list. You are allowed to return multiple errors if there are multiple errors on the page.+ [{+ "error_code": str, // The error code from the user's error code list+ "reasoning": str, // The reasoning behind the error. Be specific, referencing any user information and their fields in your reasoning. Keep the reasoning short and to the point.+ "confidence_float": float // The confidence of the error. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence+ }]+{% endif %}+ }]+}++HTML elements from `{{ current_url }}`:+```+{{ elements }}+```++The URL of the page you're on right now is `{{ current_url }}`.++{% if complete_criterion %}+Complete Criterion:+```+{{ complete_criterion }}+```{% endif %}+{% if terminate_criterion %}+Terminate Criterion:+```+{{ terminate_criterion }}+```{% endif %}+{% if error_code_mapping_str %}+Use the error codes and their descriptions to surface user-defined errors. Do not return any error that's not defined by the user. User defined errors:+```+{{ error_code_mapping_str }}+```{% endif %}++User details:+```+{{ navigation_payload_str }}+```++Current datetime in UTC, YYYY-MM-DD HH:MM format:+```+{{ utc_datetime }}+```diff --git a/skyvern/forge/prompts/skyvern/single-click-action.j2 b/skyvern/forge/prompts/skyvern/single-click-action.j2new file mode 100644index 000000000..6ab528740--- /dev/null+++ b/skyvern/forge/prompts/skyvern/single-click-action.j2@@ -0,0 +1,39 @@+Your are here to help the user to perform a CLICK action on the web page. Use the user instruction, the content of the elements parsed from the page, the screenshots of the page, and user details to determine which element to click.+Each actionable element is tagged with an ID. Only take the action on the elements provided in the HTML elements, do not image any new element.+MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.++Reply in JSON format with the following keys:+{+ "page_info": str, // Think step by step. Describe all the useful information in the page related to the user instruction.+ "thoughts": str, // Think step by step. What information makes you believe which element to click. Use information you see on the site to explain.+ "actions": array // You are supposed to give only one action("CLICK") in the action list. Here's the format of the action:+ [{+ "reasoning": str, // The reasoning behind the action. This reasoning must be user information agnostic. Mention why you chose the action type, and why you chose the element id. Keep the reasoning short and to the point.+ "confidence_float": float, // The confidence of the action. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence+ "action_type": str, // It's a string enum: "CLICK". "CLICK" is an element you'd like to click.+ "id": str, // The id of the element to take action on. The id has to be one from the elements list.+ "download": bool, // If true, the browser will trigger a download by clicking the element. If false, the browser will click the element without triggering a download.+ }]+}++The URL of the page you're on right now is `{{ current_url }}`.++HTML elements from `{{ current_url }}`:+```+{{ elements }}+```++User instruction:+```+{{ navigation_goal }}+```++User details:+```+{{ navigation_payload_str }}+```++Current datetime in UTC, YYYY-MM-DD HH:MM format:+```+{{ utc_datetime }}+```diff --git a/skyvern/forge/prompts/skyvern/single-input-action.j2 b/skyvern/forge/prompts/skyvern/single-input-action.j2new file mode 100644index 000000000..bb220b3a3--- /dev/null+++ b/skyvern/forge/prompts/skyvern/single-input-action.j2@@ -0,0 +1,41 @@+Your are here to help the user to perform an INPUT_TEXT action on the web page. Use the user instruction, the content of the elements parsed from the page, the screenshots of the page, and user details to determine which element to input.+Each actionable element is tagged with an ID. Only take the action on the elements provided in the HTML elements, do not image any new element.+MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.++Reply in JSON format with the following keys:+{+ "page_info": str, // Think step by step. Describe all the useful information in the page related to the user instruction.+ "thoughts": str, // Think step by step. What information makes you believe which element to input the value. Use information you see on the site to explain.+ "actions": array // You are supposed to give only one action("INPUT_TEXT") in the action list. Here's the format of the action:+ [{+ "reasoning": str, // The reasoning behind the action. This reasoning must be user information agnostic. Mention why you chose the action type, and why you chose the element id. Keep the reasoning short and to the point.+ "confidence_float": float, // The confidence of the action. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence+ "action_type": str, // It's a string enum: "INPUT_TEXT". "INPUT_TEXT" is an element you'd like to input text into.+ "id": str, // The id of the element to take action on. The id has to be one from the elements list.+ "text": str, // The text to input.+ }]{% if verification_code_check %}+ "verification_code_reasoning": str, // Let's think step by step. Describe what you see and think if a verification code is needed for login or any verification step. Explain why you believe a verification code is needed or not. Has the code been sent and is code available somewhere (email, phone or 2FA device)?+ "need_verification_code": bool, // Whether a verification code is needed to proceed. True only if the code is available to user. If the code is not sent, return false {% endif %}+}++The URL of the page you're on right now is `{{ current_url }}`.++HTML elements from `{{ current_url }}`:+```+{{ elements }}+```++User instruction:+```+{{ navigation_goal }}+```++User details:+```+{{ navigation_payload_str }}+```++Current datetime in UTC, YYYY-MM-DD HH:MM format:+```+{{ utc_datetime }}+```diff --git a/skyvern/forge/prompts/skyvern/single-select-action.j2 b/skyvern/forge/prompts/skyvern/single-select-action.j2new file mode 100644index 000000000..d99ae548f--- /dev/null+++ b/skyvern/forge/prompts/skyvern/single-select-action.j2@@ -0,0 +1,43 @@+Your are here to help the user to perform an SELECT_OPTION action on the web page. Use the user instruction, the content of the elements parsed from the page, the screenshots of the page, and user details to determine which element to select.+Each actionable element is tagged with an ID. Only take the action on the elements provided in the HTML elements, do not image any new element.+MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.++Reply in JSON format with the following keys:+{+ "page_info": str, // Think step by step. Describe all the useful information in the page related to the user instruction.+ "thoughts": str, // Think step by step. What information makes you believe which element to select. Use information you see on the site to explain.+ "actions": array // You are supposed to give only one action("SELECT_OPTION") in the action list. Here's the format of the action:+ [{+ "reasoning": str, // The reasoning behind the action. This reasoning must be user information agnostic. Mention why you chose the action type, and why you chose the element id. Keep the reasoning short and to the point.+ "confidence_float": float, // The confidence of the action. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence+ "action_type": str, // It's a string enum: "SELECT_OPTION". "SELECT_OPTION" is an element you'd like to select an option from.+ "id": str, // The id of the element to take action on. The id has to be one from the elements list.+ "option": { // The option to select.+ "label": str, // the label of the option if any. MAKE SURE YOU USE THIS LABEL TO SELECT THE OPTION. DO NOT PUT ANYTHING OTHER THAN A VALID OPTION LABEL HERE+ "index": int, // the index corresponding to the option index under the select element.+ "value": str // the value of the option. MAKE SURE YOU USE THIS VALUE TO SELECT THE OPTION. DO NOT PUT ANYTHING OTHER THAN A VALID OPTION VALUE HERE+ },+ }]+}++The URL of the page you're on right now is `{{ current_url }}`.++HTML elements from `{{ current_url }}`:+```+{{ elements }}+```++User instruction:+```+{{ navigation_goal }}+```++User details:+```+{{ navigation_payload_str }}+```++Current datetime in UTC, YYYY-MM-DD HH:MM format:+```+{{ utc_datetime }}+```diff --git a/skyvern/forge/prompts/skyvern/single-upload-action.j2 b/skyvern/forge/prompts/skyvern/single-upload-action.j2new file mode 100644index 000000000..df2ef32b6--- /dev/null+++ b/skyvern/forge/prompts/skyvern/single-upload-action.j2@@ -0,0 +1,39 @@+Your are here to help the user to perform an UPLOAD_FILE action on the web page. Use the user instruction, the content of the elements parsed from the page, the screenshots of the page, and user details to determine which element to upload.+Each actionable element is tagged with an ID. Only take the action on the elements provided in the HTML elements, do not image any new element.+MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.++Reply in JSON format with the following keys:+{+ "page_info": str, // Think step by step. Describe all the useful information in the page related to the user instruction.+ "thoughts": str, // Think step by step. What information makes you believe which element to upload the file. Use information you see on the site to explain.+ "actions": array // You are supposed to give only one action("UPLOAD_FILE") in the action list. Here's the format of the action:+ [{+ "reasoning": str, // The reasoning behind the action. This reasoning must be user information agnostic. Mention why you chose the action type, and why you chose the element id. Keep the reasoning short and to the point.+ "confidence_float": float, // The confidence of the action. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence+ "action_type": str, // It's a string enum: "UPLOAD_FILE". "UPLOAD_FILE" is an element you'd like to upload a file into.+ "id": str, // The id of the element to take action on. The id has to be one from the elements list.+ "file_url": str, // The url of the file to upload.+ }]+}++The URL of the page you're on right now is `{{ current_url }}`.++HTML elements from `{{ current_url }}`:+```+{{ elements }}+```++User instruction:+```+{{ navigation_goal }}+```++User details:+```+{{ navigation_payload_str }}+```++Current datetime in UTC, YYYY-MM-DD HH:MM format:+```+{{ utc_datetime }}+```diff --git a/skyvern/forge/sdk/db/client.py b/skyvern/forge/sdk/db/client.pyindex a946a92dc..31dcc9579 100644--- a/skyvern/forge/sdk/db/client.py+++ b/skyvern/forge/sdk/db/client.py@@ -10,7 +10,7 @@ from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_enginefrom skyvern.config import settingsfrom skyvern.exceptions import WorkflowParameterNotFoundfrom skyvern.forge.sdk.artifact.models import Artifact, ArtifactType-from skyvern.forge.sdk.db.enums import OrganizationAuthTokenType+from skyvern.forge.sdk.db.enums import OrganizationAuthTokenType, TaskPromptTemplatefrom skyvern.forge.sdk.db.exceptions import NotFoundErrorfrom skyvern.forge.sdk.db.models import (ActionModel,@@ -97,6 +97,8 @@ class AgentDB:self,url: str,title: str | None,+ complete_criterion: str | None,+ terminate_criterion: str | None,navigation_goal: str | None,data_extraction_goal: str | None,navigation_payload: dict[str, Any] | list | str | None,@@ -111,17 +113,21 @@ class AgentDB:retry: int | None = None,max_steps_per_run: int | None = None,error_code_mapping: dict[str, str] | None = None,+ prompt_template: str = TaskPromptTemplate.ExtractAction,) -> Task:try:async with self.Session() as session:new_task = TaskModel(status="created",+ prompt_template=prompt_template,url=url,title=title,webhook_callback_url=webhook_callback_url,totp_verification_url=totp_verification_url,totp_identifier=totp_identifier,navigation_goal=navigation_goal,+ complete_criterion=complete_criterion,+ terminate_criterion=terminate_criterion,data_extraction_goal=data_extraction_goal,navigation_payload=navigation_payload,organization_id=organization_id,diff --git a/skyvern/forge/sdk/db/enums.py b/skyvern/forge/sdk/db/enums.pyindex d42eb6175..c521012d9 100644--- a/skyvern/forge/sdk/db/enums.py+++ b/skyvern/forge/sdk/db/enums.py@@ -3,3 +3,19 @@ from enum import StrEnumclass OrganizationAuthTokenType(StrEnum):api = "api"+++class TaskPromptTemplate(StrEnum):+ ExtractAction = "extract-action"+ DecisiveCriterionValidate = "decisive-criterion-validate"+ SingleClickAction = "single-click-action"+ SingleInputAction = "single-input-action"+ SingleUploadAction = "single-upload-action"+ SingleSelectAction = "single-select-action"+++class ActionType(StrEnum):+ Click = "CLICK"+ InputText = "INPUT_TEXT"+ UploadFile = "UPLOAD_FILE"+ SelectOption = "SELECT_OPTION"diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.pyindex 42baeda3e..6dbb6c42b 100644--- a/skyvern/forge/sdk/db/models.py+++ b/skyvern/forge/sdk/db/models.py@@ -17,7 +17,7 @@ from sqlalchemy import (from sqlalchemy.ext.asyncio import AsyncAttrsfrom sqlalchemy.orm import DeclarativeBase-from skyvern.forge.sdk.db.enums import OrganizationAuthTokenType+from skyvern.forge.sdk.db.enums import OrganizationAuthTokenType, TaskPromptTemplatefrom skyvern.forge.sdk.db.id import (generate_action_id,generate_artifact_id,@@ -54,9 +54,12 @@ class TaskModel(Base):totp_verification_url = Column(String)totp_identifier = Column(String)title = Column(String)+ prompt_template = Column(String, default=TaskPromptTemplate.ExtractAction)url = Column(String)navigation_goal = Column(String)data_extraction_goal = Column(String)+ complete_criterion = Column(String)+ terminate_criterion = Column(String)navigation_payload = Column(JSON)extracted_information = Column(JSON)failure_reason = Column(String)diff --git a/skyvern/forge/sdk/db/utils.py b/skyvern/forge/sdk/db/utils.pyindex a17e08fcc..d8c7a1afc 100644--- a/skyvern/forge/sdk/db/utils.py+++ b/skyvern/forge/sdk/db/utils.py@@ -60,8 +60,11 @@ def convert_to_task(task_obj: TaskModel, debug_enabled: bool = False) -> Task:status=TaskStatus(task_obj.status),created_at=task_obj.created_at,modified_at=task_obj.modified_at,+ prompt_template=task_obj.prompt_template,title=task_obj.title,url=task_obj.url,+ complete_criterion=task_obj.complete_criterion,+ terminate_criterion=task_obj.terminate_criterion,webhook_callback_url=task_obj.webhook_callback_url,totp_verification_url=task_obj.totp_verification_url,totp_identifier=task_obj.totp_identifier,diff --git a/skyvern/forge/sdk/models.py b/skyvern/forge/sdk/models.pyindex 5c28886a0..e40b0fcc6 100644--- a/skyvern/forge/sdk/models.py+++ b/skyvern/forge/sdk/models.py@@ -104,6 +104,14 @@ class Step(BaseModel):return False+ def is_success(self) -> bool:+ if self.status != StepStatus.completed:+ return False+ # TODO (kerem): Remove this check once we have backfilled all the steps+ if self.output is None or self.output.actions_and_results is None:+ return False+ return True+def is_terminated(self) -> bool:if self.status != StepStatus.completed:return Falsediff --git a/skyvern/forge/sdk/schemas/tasks.py b/skyvern/forge/sdk/schemas/tasks.pyindex 7b449fa56..a38aaed49 100644--- a/skyvern/forge/sdk/schemas/tasks.py+++ b/skyvern/forge/sdk/schemas/tasks.py@@ -8,6 +8,7 @@ from pydantic import BaseModel, Field, HttpUrl, field_validatorfrom skyvern.exceptions import BlockedHost, InvalidTaskStatusTransition, TaskAlreadyCanceledfrom skyvern.forge.sdk.core.validators import is_blocked_host+from skyvern.forge.sdk.db.enums import TaskPromptTemplateclass ProxyLocation(StrEnum):@@ -77,6 +78,19 @@ class TaskBase(BaseModel):default=None,description="The requested schema of the extracted information.",)+ complete_criterion: str | None = Field(+ default=None, description="Criterion to complete", examples=["Complete if 'hello world' shows up on the page"]+ )+ terminate_criterion: str | None = Field(+ default=None,+ description="Criterion to terminate",+ examples=["Terminate if 'existing account' shows up on the page"],+ )+ prompt_template: str | None = Field(+ default=TaskPromptTemplate.ExtractAction,+ description="The prompt template used for task",+ examples=[TaskPromptTemplate.ExtractAction, TaskPromptTemplate.DecisiveCriterionValidate],+ )class TaskRequest(TaskBase):diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex 63e026dcb..ed5269ed2 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -40,6 +40,7 @@ from skyvern.forge.sdk.api.files import (get_path_for_workflow_download_directory,)from skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory+from skyvern.forge.sdk.db.enums import TaskPromptTemplatefrom skyvern.forge.sdk.schemas.tasks import Task, TaskOutput, TaskStatusfrom skyvern.forge.sdk.settings_manager import SettingsManagerfrom skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext@@ -69,6 +70,8 @@ class BlockType(StrEnum):UPLOAD_TO_S3 = "upload_to_s3"SEND_EMAIL = "send_email"FILE_URL_PARSER = "file_url_parser"+ VALIDATION = "validation"+ ACTION = "action"class BlockStatus(StrEnum):@@ -174,11 +177,12 @@ class Block(BaseModel, abc.ABC):pass-class TaskBlock(Block):- block_type: Literal[BlockType.TASK] = BlockType.TASK-+class BaseTaskBlock(Block):+ prompt_template: str = TaskPromptTemplate.ExtractActionurl: str | None = Nonetitle: str = ""+ complete_criterion: str | None = None+ terminate_criterion: str | None = Nonenavigation_goal: str | None = Nonedata_extraction_goal: str | None = Nonedata_schema: dict[str, Any] | list | None = None@@ -464,6 +468,10 @@ class TaskBlock(Block):)+class TaskBlock(BaseTaskBlock):+ block_type: Literal[BlockType.TASK] = BlockType.TASK++class ForLoopBlock(Block):block_type: Literal[BlockType.FOR_LOOP] = BlockType.FOR_LOOP@@ -1264,6 +1272,36 @@ class FileParserBlock(Block):)+class ValidationBlock(BaseTaskBlock):+ block_type: Literal[BlockType.VALIDATION] = BlockType.VALIDATION++ def get_all_parameters(+ self,+ workflow_run_id: str,+ ) -> list[PARAMETER_TYPE]:+ return self.parameters++ async def execute(self, workflow_run_id: str, **kwargs: dict) -> BlockResult:+ task_order, _ = await self.get_task_order(workflow_run_id, 0)+ is_first_task = task_order == 0+ if is_first_task:+ return self.build_block_result(+ success=False,+ failure_reason="Validation block should not be the first block",+ output_parameter_value=None,+ status=BlockStatus.terminated,+ )++ return await super().execute(workflow_run_id=workflow_run_id, kwargs=kwargs)+++class ActionBlock(BaseTaskBlock):+ block_type: Literal[BlockType.ACTION] = BlockType.ACTION++ async def execute(self, workflow_run_id: str, **kwargs: dict) -> BlockResult:+ return await super().execute(workflow_run_id=workflow_run_id, kwargs=kwargs)++BlockSubclasses = Union[ForLoopBlock,TaskBlock,@@ -1273,5 +1311,7 @@ BlockSubclasses = Union[UploadToS3Block,SendEmailBlock,FileParserBlock,+ ValidationBlock,+ ActionBlock,]BlockTypeVar = Annotated[BlockSubclasses, Field(discriminator="block_type")]diff --git a/skyvern/forge/sdk/workflow/models/yaml.py b/skyvern/forge/sdk/workflow/models/yaml.pyindex fbe431b44..8baea2b32 100644--- a/skyvern/forge/sdk/workflow/models/yaml.py+++ b/skyvern/forge/sdk/workflow/models/yaml.py@@ -3,6 +3,7 @@ from typing import Annotated, Any, Literalfrom pydantic import BaseModel, Field+from skyvern.forge.sdk.db.enums import ActionTypefrom skyvern.forge.sdk.schemas.tasks import ProxyLocationfrom skyvern.forge.sdk.workflow.models.block import BlockType, FileTypefrom skyvern.forge.sdk.workflow.models.parameter import ParameterType, WorkflowParameterType@@ -208,6 +209,32 @@ class FileParserBlockYAML(BlockYAML):file_type: FileType+class ValidationBlockYAML(BlockYAML):+ block_type: Literal[BlockType.VALIDATION] = BlockType.VALIDATION # type: ignore++ complete_criterion: str | None = None+ terminate_criterion: str | None = None+ error_code_mapping: dict[str, str] | None = None+ parameter_keys: list[str] | None = None+++class ActionBlockYAML(BlockYAML):+ action_type: ActionType+ block_type: Literal[BlockType.ACTION] = BlockType.ACTION # type: ignore++ url: str | None = None+ title: str = ""+ navigation_goal: str | None = None+ error_code_mapping: dict[str, str] | None = None+ max_retries: int = 0+ parameter_keys: list[str] | None = None+ complete_on_download: bool = False+ download_suffix: str | None = None+ totp_verification_url: str | None = None+ totp_identifier: str | None = None+ cache_actions: bool = False++PARAMETER_YAML_SUBCLASSES = (AWSSecretParameterYAML| BitwardenLoginCredentialParameterYAML@@ -228,6 +255,8 @@ BLOCK_YAML_SUBCLASSES = (| UploadToS3BlockYAML| SendEmailBlockYAML| FileParserBlockYAML+ | ValidationBlockYAML+ | ActionBlockYAML)BLOCK_YAML_TYPES = Annotated[BLOCK_YAML_SUBCLASSES, Field(discriminator="block_type")]diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex 9d52fc554..02bf57f3f 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -18,6 +18,7 @@ from skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.security import generate_skyvern_signaturefrom skyvern.forge.sdk.core.skyvern_context import SkyvernContext+from skyvern.forge.sdk.db.enums import ActionType, TaskPromptTemplatefrom skyvern.forge.sdk.models import Organization, Stepfrom skyvern.forge.sdk.schemas.tasks import ProxyLocation, Taskfrom skyvern.forge.sdk.workflow.exceptions import (@@ -28,6 +29,7 @@ from skyvern.forge.sdk.workflow.exceptions import (WorkflowParameterMissingRequiredValue,)from skyvern.forge.sdk.workflow.models.block import (+ ActionBlock,BlockStatus,BlockType,BlockTypeVar,@@ -39,6 +41,7 @@ from skyvern.forge.sdk.workflow.models.block import (TaskBlock,TextPromptBlock,UploadToS3Block,+ ValidationBlock,)from skyvern.forge.sdk.workflow.models.parameter import (PARAMETER_TYPE,@@ -1333,4 +1336,65 @@ class WorkflowService:file_type=block_yaml.file_type,continue_on_failure=block_yaml.continue_on_failure,)+ elif block_yaml.block_type == BlockType.VALIDATION:+ validation_block_parameters = (+ [parameters[parameter_key] for parameter_key in block_yaml.parameter_keys]+ if block_yaml.parameter_keys+ else []+ )++ if not block_yaml.complete_criterion and not block_yaml.terminate_criterion:+ raise Exception("Both complete criterion and terminate criterion are empty")++ return ValidationBlock(+ label=block_yaml.label,+ prompt_template=TaskPromptTemplate.DecisiveCriterionValidate,+ parameters=validation_block_parameters,+ output_parameter=output_parameter,+ complete_criterion=block_yaml.complete_criterion,+ terminate_criterion=block_yaml.terminate_criterion,+ error_code_mapping=block_yaml.error_code_mapping,+ continue_on_failure=block_yaml.continue_on_failure,+ # only need one step for validation block+ max_steps_per_run=1,+ )++ elif block_yaml.block_type == BlockType.ACTION:+ action_block_parameters = (+ [parameters[parameter_key] for parameter_key in block_yaml.parameter_keys]+ if block_yaml.parameter_keys+ else []+ )+ prompt_template = ""+ if block_yaml.action_type == ActionType.Click:+ prompt_template = TaskPromptTemplate.SingleClickAction+ elif block_yaml.action_type == ActionType.InputText:+ prompt_template = TaskPromptTemplate.SingleInputAction+ elif block_yaml.action_type == ActionType.UploadFile:+ prompt_template = TaskPromptTemplate.SingleUploadAction+ elif block_yaml.action_type == ActionType.SelectOption:+ prompt_template = TaskPromptTemplate.SingleSelectAction++ if not prompt_template:+ raise Exception("not supported action type for action block")++ return ActionBlock(+ prompt_template=prompt_template,+ label=block_yaml.label,+ url=block_yaml.url,+ title=block_yaml.title,+ parameters=action_block_parameters,+ output_parameter=output_parameter,+ navigation_goal=block_yaml.navigation_goal,+ error_code_mapping=block_yaml.error_code_mapping,+ max_retries=block_yaml.max_retries,+ complete_on_download=block_yaml.complete_on_download,+ download_suffix=block_yaml.download_suffix,+ continue_on_failure=block_yaml.continue_on_failure,+ totp_verification_url=block_yaml.totp_verification_url,+ totp_identifier=block_yaml.totp_identifier,+ cache_actions=block_yaml.cache_actions,+ max_steps_per_run=1,+ )+raise ValueError(f"Invalid block type {block_yaml.block_type}")