Skip to content

Automate browser based workflows with AI

add endpoint/logic for creating a taskv2-workflow from a prompt (#3352)

Jonathan Dobson8b163ba37158
diff --git a/skyvern/forge/prompts/skyvern/conversational_ui_goal.j2 b/skyvern/forge/prompts/skyvern/conversational_ui_goal.j2
new file mode 100644
index 000000000..cf92fe648
--- /dev/null
+++ b/skyvern/forge/prompts/skyvern/conversational_ui_goal.j2
@@ -0,0 +1,14 @@
+Given the user goal, come up with a title for it, and a block label.
+
+MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.
+
+Reply in JSON format with the following keys:
+{
+ "title": str, // A descriptive and informative title for the goal. Use no more than 5 words
+ "block_label": str, // A label for the block. Use 1 word only. Based off of the "title".
+}
+
+User goal:
+```
+{{ user_goal }}
+```
diff --git a/skyvern/forge/sdk/routes/agent_protocol.py b/skyvern/forge/sdk/routes/agent_protocol.py
index 8e297dee0..d0b8dc875 100644
--- a/skyvern/forge/sdk/routes/agent_protocol.py
+++ b/skyvern/forge/sdk/routes/agent_protocol.py
@@ -525,6 +525,55 @@ async def create_workflow(
raise FailedToCreateWorkflow(str(e))
+@base_router.post(
+ "/workflows/create-from-prompt",
+ include_in_schema=False,
+)
+async def create_workflow_from_prompt(
+ data: TaskV2Request,
+ organization: Organization = Depends(org_auth_service.get_current_org),
+ x_max_iterations_override: Annotated[int | str | None, Header()] = None,
+ x_max_steps_override: Annotated[int | str | None, Header()] = None,
+) -> dict[str, Any]:
+ if x_max_iterations_override or x_max_steps_override:
+ LOG.info(
+ "Overriding max steps for workflow-from-prompt",
+ max_iterations_override=x_max_iterations_override,
+ max_steps_override=x_max_steps_override,
+ )
+ await PermissionCheckerFactory.get_instance().check(organization, browser_session_id=data.browser_session_id)
+
+ if isinstance(x_max_iterations_override, str):
+ try:
+ x_max_iterations_override = int(x_max_iterations_override)
+ except ValueError:
+ x_max_iterations_override = None
+
+ if isinstance(x_max_steps_override, str):
+ try:
+ x_max_steps_override = int(x_max_steps_override)
+ except ValueError:
+ x_max_steps_override = None
+ try:
+ workflow = await app.WORKFLOW_SERVICE.create_workflow_from_prompt(
+ organization=organization,
+ user_prompt=data.user_prompt,
+ totp_identifier=data.totp_identifier,
+ totp_verification_url=data.totp_verification_url,
+ webhook_callback_url=data.webhook_callback_url,
+ proxy_location=data.proxy_location,
+ max_screenshot_scrolling_times=data.max_screenshot_scrolls,
+ extra_http_headers=data.extra_http_headers,
+ max_iterations=x_max_iterations_override,
+ max_steps=x_max_steps_override,
+ )
+ except Exception as e:
+ LOG.error("Failed to create workflow from prompt", exc_info=True, organization_id=organization.organization_id)
+ raise FailedToCreateWorkflow(str(e))
+
+ return workflow.model_dump(by_alias=True)
+
+
@legacy_base_router.put(
"/workflows/{workflow_id}",
openapi_extra={
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 6aba935e2..a554b3eea 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -1,6 +1,7 @@
import asyncio
import base64
import json
+import uuid
from datetime import UTC, datetime
from typing import Any
@@ -25,6 +26,7 @@ from skyvern.exceptions import (
WorkflowRunNotFound,
)
from skyvern.forge import app
+from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.security import generate_skyvern_webhook_headers
@@ -110,6 +112,9 @@ from skyvern.webeye.browser_factory import BrowserState
LOG = structlog.get_logger()
+DEFAULT_FIRST_BLOCK_LABEL = "block_1"
+DEFAULT_WORKFLOW_TITLE = "New Workflow"
+
class WorkflowService:
@staticmethod
@@ -669,6 +674,62 @@ class WorkflowService:
ai_fallback=False if ai_fallback is None else ai_fallback,
)
+ async def create_workflow_from_prompt(
+ self,
+ organization: Organization,
+ user_prompt: str,
+ totp_identifier: str | None = None,
+ totp_verification_url: str | None = None,
+ webhook_callback_url: str | None = None,
+ proxy_location: ProxyLocation | None = None,
+ max_screenshot_scrolling_times: int | None = None,
+ extra_http_headers: dict[str, str] | None = None,
+ max_iterations: int | None = None,
+ max_steps: int | None = None,
+ ) -> Workflow:
+ metadata_prompt = prompt_engine.load_prompt(
+ "conversational_ui_goal",
+ user_goal=user_prompt,
+ )
+
+ metadata_response = await app.LLM_API_HANDLER(
+ prompt=metadata_prompt,
+ prompt_name="conversational_ui_goal",
+ )
+
+ block_label: str = metadata_response.get("block_label", DEFAULT_FIRST_BLOCK_LABEL)
+ title: str = metadata_response.get("title", DEFAULT_WORKFLOW_TITLE)
+
+ task_v2_block = TaskV2Block(
+ prompt=user_prompt,
+ totp_identifier=totp_identifier,
+ totp_verification_url=totp_verification_url,
+ label=block_label,
+ max_iterations=max_iterations or settings.MAX_ITERATIONS_PER_TASK_V2,
+ max_steps=max_steps or settings.MAX_STEPS_PER_TASK_V2,
+ output_parameter=OutputParameter(
+ output_parameter_id=str(uuid.uuid4()),
+ key=f"{block_label}_output",
+ workflow_id="",
+ created_at=datetime.now(UTC),
+ modified_at=datetime.now(UTC),
+ ),
+ )
+
+ new_workflow = await self.create_workflow(
+ title=title,
+ workflow_definition=WorkflowDefinition(parameters=[], blocks=[task_v2_block]),
+ organization_id=organization.organization_id,
+ proxy_location=proxy_location,
+ webhook_callback_url=webhook_callback_url,
+ totp_verification_url=totp_verification_url,
+ totp_identifier=totp_identifier,
+ max_screenshot_scrolling_times=max_screenshot_scrolling_times,
+ extra_http_headers=extra_http_headers,
+ )
+
+ return new_workflow
+
async def get_workflow(self, workflow_id: str, organization_id: str | None = None) -> Workflow:
workflow = await app.DATABASE.get_workflow(workflow_id=workflow_id, organization_id=organization_id)
if not workflow: