Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
add endpoint/logic for creating a taskv2-workflow from a prompt (#3352)
Jonathan Dobson8b163ba37158
diff --git a/skyvern/forge/prompts/skyvern/conversational_ui_goal.j2 b/skyvern/forge/prompts/skyvern/conversational_ui_goal.j2new file mode 100644index 000000000..cf92fe648--- /dev/null+++ b/skyvern/forge/prompts/skyvern/conversational_ui_goal.j2@@ -0,0 +1,14 @@+Given the user goal, come up with a title for it, and a block label.++MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.++Reply in JSON format with the following keys:+{+ "title": str, // A descriptive and informative title for the goal. Use no more than 5 words+ "block_label": str, // A label for the block. Use 1 word only. Based off of the "title".+}++User goal:+```+{{ user_goal }}+```diff --git a/skyvern/forge/sdk/routes/agent_protocol.py b/skyvern/forge/sdk/routes/agent_protocol.pyindex 8e297dee0..d0b8dc875 100644--- a/skyvern/forge/sdk/routes/agent_protocol.py+++ b/skyvern/forge/sdk/routes/agent_protocol.py@@ -525,6 +525,55 @@ async def create_workflow(raise FailedToCreateWorkflow(str(e))+@base_router.post(+ "/workflows/create-from-prompt",+ include_in_schema=False,+)+async def create_workflow_from_prompt(+ data: TaskV2Request,+ organization: Organization = Depends(org_auth_service.get_current_org),+ x_max_iterations_override: Annotated[int | str | None, Header()] = None,+ x_max_steps_override: Annotated[int | str | None, Header()] = None,+) -> dict[str, Any]:+ if x_max_iterations_override or x_max_steps_override:+ LOG.info(+ "Overriding max steps for workflow-from-prompt",+ max_iterations_override=x_max_iterations_override,+ max_steps_override=x_max_steps_override,+ )+ await PermissionCheckerFactory.get_instance().check(organization, browser_session_id=data.browser_session_id)++ if isinstance(x_max_iterations_override, str):+ try:+ x_max_iterations_override = int(x_max_iterations_override)+ except ValueError:+ x_max_iterations_override = None++ if isinstance(x_max_steps_override, str):+ try:+ x_max_steps_override = int(x_max_steps_override)+ except ValueError:+ x_max_steps_override = None+ try:+ workflow = await app.WORKFLOW_SERVICE.create_workflow_from_prompt(+ organization=organization,+ user_prompt=data.user_prompt,+ totp_identifier=data.totp_identifier,+ totp_verification_url=data.totp_verification_url,+ webhook_callback_url=data.webhook_callback_url,+ proxy_location=data.proxy_location,+ max_screenshot_scrolling_times=data.max_screenshot_scrolls,+ extra_http_headers=data.extra_http_headers,+ max_iterations=x_max_iterations_override,+ max_steps=x_max_steps_override,+ )+ except Exception as e:+ LOG.error("Failed to create workflow from prompt", exc_info=True, organization_id=organization.organization_id)+ raise FailedToCreateWorkflow(str(e))++ return workflow.model_dump(by_alias=True)++@legacy_base_router.put("/workflows/{workflow_id}",openapi_extra={diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex 6aba935e2..a554b3eea 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -1,6 +1,7 @@import asyncioimport base64import json+import uuidfrom datetime import UTC, datetimefrom typing import Any@@ -25,6 +26,7 @@ from skyvern.exceptions import (WorkflowRunNotFound,)from skyvern.forge import app+from skyvern.forge.prompts import prompt_enginefrom skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.security import generate_skyvern_webhook_headers@@ -110,6 +112,9 @@ from skyvern.webeye.browser_factory import BrowserStateLOG = structlog.get_logger()+DEFAULT_FIRST_BLOCK_LABEL = "block_1"+DEFAULT_WORKFLOW_TITLE = "New Workflow"+class WorkflowService:@staticmethod@@ -669,6 +674,62 @@ class WorkflowService:ai_fallback=False if ai_fallback is None else ai_fallback,)+ async def create_workflow_from_prompt(+ self,+ organization: Organization,+ user_prompt: str,+ totp_identifier: str | None = None,+ totp_verification_url: str | None = None,+ webhook_callback_url: str | None = None,+ proxy_location: ProxyLocation | None = None,+ max_screenshot_scrolling_times: int | None = None,+ extra_http_headers: dict[str, str] | None = None,+ max_iterations: int | None = None,+ max_steps: int | None = None,+ ) -> Workflow:+ metadata_prompt = prompt_engine.load_prompt(+ "conversational_ui_goal",+ user_goal=user_prompt,+ )++ metadata_response = await app.LLM_API_HANDLER(+ prompt=metadata_prompt,+ prompt_name="conversational_ui_goal",+ )++ block_label: str = metadata_response.get("block_label", DEFAULT_FIRST_BLOCK_LABEL)+ title: str = metadata_response.get("title", DEFAULT_WORKFLOW_TITLE)++ task_v2_block = TaskV2Block(+ prompt=user_prompt,+ totp_identifier=totp_identifier,+ totp_verification_url=totp_verification_url,+ label=block_label,+ max_iterations=max_iterations or settings.MAX_ITERATIONS_PER_TASK_V2,+ max_steps=max_steps or settings.MAX_STEPS_PER_TASK_V2,+ output_parameter=OutputParameter(+ output_parameter_id=str(uuid.uuid4()),+ key=f"{block_label}_output",+ workflow_id="",+ created_at=datetime.now(UTC),+ modified_at=datetime.now(UTC),+ ),+ )++ new_workflow = await self.create_workflow(+ title=title,+ workflow_definition=WorkflowDefinition(parameters=[], blocks=[task_v2_block]),+ organization_id=organization.organization_id,+ proxy_location=proxy_location,+ webhook_callback_url=webhook_callback_url,+ totp_verification_url=totp_verification_url,+ totp_identifier=totp_identifier,+ max_screenshot_scrolling_times=max_screenshot_scrolling_times,+ extra_http_headers=extra_http_headers,+ )++ return new_workflow+async def get_workflow(self, workflow_id: str, organization_id: str | None = None) -> Workflow:workflow = await app.DATABASE.get_workflow(workflow_id=workflow_id, organization_id=organization_id)if not workflow: