Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Workflow Copilot: backend side of the first version (#4401)
Stanislav Novosade3dd75d7c163
diff --git a/alembic/versions/2026_01_06_2148-db8667f8ce63_workflow_copilot_chat.py b/alembic/versions/2026_01_06_2148-db8667f8ce63_workflow_copilot_chat.pynew file mode 100644index 000000000..55b342e6f--- /dev/null+++ b/alembic/versions/2026_01_06_2148-db8667f8ce63_workflow_copilot_chat.py@@ -0,0 +1,68 @@+"""workflow copilot chat++Revision ID: db8667f8ce63+Revises: e393f33ec711+Create Date: 2026-01-06 21:48:28.396490+00:00++"""++from typing import Sequence, Union++import sqlalchemy as sa++from alembic import op++# revision identifiers, used by Alembic.+revision: str = "db8667f8ce63"+down_revision: Union[str, None] = "e393f33ec711"+branch_labels: Union[str, Sequence[str], None] = None+depends_on: Union[str, Sequence[str], None] = None+++def upgrade() -> None:+ # ### commands auto generated by Alembic - please adjust! ###+ op.create_table(+ "workflow_copilot_chat_messages",+ sa.Column("workflow_copilot_chat_message_id", sa.String(), nullable=False),+ sa.Column("workflow_copilot_chat_id", sa.String(), nullable=False),+ sa.Column("organization_id", sa.String(), nullable=False),+ sa.Column("sender", sa.String(), nullable=False),+ sa.Column("content", sa.UnicodeText(), nullable=False),+ sa.Column("global_llm_context", sa.UnicodeText(), nullable=True),+ sa.Column("created_at", sa.DateTime(), nullable=False),+ sa.Column("modified_at", sa.DateTime(), nullable=False),+ sa.PrimaryKeyConstraint("workflow_copilot_chat_message_id"),+ )+ op.create_index(+ op.f("ix_workflow_copilot_chat_messages_workflow_copilot_chat_id"),+ "workflow_copilot_chat_messages",+ ["workflow_copilot_chat_id"],+ unique=False,+ )+ op.create_table(+ "workflow_copilot_chats",+ sa.Column("workflow_copilot_chat_id", sa.String(), nullable=False),+ sa.Column("organization_id", sa.String(), nullable=False),+ sa.Column("workflow_permanent_id", sa.String(), nullable=False),+ sa.Column("created_at", sa.DateTime(), nullable=False),+ sa.Column("modified_at", sa.DateTime(), nullable=False),+ sa.PrimaryKeyConstraint("workflow_copilot_chat_id"),+ )+ op.create_index(+ op.f("ix_workflow_copilot_chats_workflow_permanent_id"),+ "workflow_copilot_chats",+ ["workflow_permanent_id"],+ unique=False,+ )+ # ### end Alembic commands ###+++def downgrade() -> None:+ # ### commands auto generated by Alembic - please adjust! ###+ op.drop_index(op.f("ix_workflow_copilot_chats_workflow_permanent_id"), table_name="workflow_copilot_chats")+ op.drop_table("workflow_copilot_chats")+ op.drop_index(+ op.f("ix_workflow_copilot_chat_messages_workflow_copilot_chat_id"), table_name="workflow_copilot_chat_messages"+ )+ op.drop_table("workflow_copilot_chat_messages")+ # ### end Alembic commands ###diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot.j2new file mode 100644index 000000000..dbae58fef--- /dev/null+++ b/skyvern/forge/prompts/skyvern/workflow-copilot.j2@@ -0,0 +1,183 @@+You are an expert Skyvern Workflow assistant helping users build and modify browser automation workflows.++Your role is to understand the user's intent and help them construct or modify workflow YAML definitions that will automate browser-based tasks.++WORKFLOW KNOWLEDGE BASE:++{{ workflow_knowledge_base }}++TASK:++The user is working on building or updating a Skyvern workflow. They may want to:+- Create a new workflow from scratch+- Add new blocks to an existing workflow+- Modify existing blocks+- Fix errors or improve the workflow+- Get clarification on how to structure their workflow++Your job is to help them achieve their goal by either:+1. Providing a complete replacement workflow YAML+2. Providing a new block to add to their workflow+3. Asking clarifying questions if you need more information++CURRENT WORKFLOW YAML:++{% if workflow_yaml %}+The user's current workflow definition is:++```yaml+{{ workflow_yaml }}+```+{% else %}+The user is starting with an empty workflow.+{% endif %}++PREVIOUS CONTEXT:++{% if chat_history %}+Recent conversation history:+{{ chat_history }}+{% endif %}++{% if global_llm_context %}+Overall goal (long-term memory):+{{ global_llm_context }}+{% endif %}++{% if not chat_history and not global_llm_context %}+No previous context available.+{% endif %}++DEBUGGER RUN INFORMATION:++{% if debug_run_info %}+The user has run the workflow in the debugger. Here's the most recent block execution information:++{{ debug_run_info }}++Use this information to help diagnose issues, suggest fixes, or explain what might be going wrong.+If there's a failure, analyze the failure reason and visible elements to provide specific guidance.++{% else %}+No debugger run information available. The workflow hasn't been run yet, or no run data is accessible.+{% endif %}++USER MESSAGE:++The user says:++```+{{ user_message }}+```++INSTRUCTIONS:++Analyze the user's request and the current workflow YAML.++IMPORTANT RULES:+* Always generate valid YAML that conforms to the Skyvern workflow schema+* Preserve existing blocks unless the user explicitly asks to modify or remove them+* Use appropriate block types based on the user's intent:+ - Use "task_v2" blocks for complex, multi-step workflows (may be slightly slower)+ - Use "task" blocks for combined navigation and extraction (faster, but less flexible)+ - Use "goto_url" blocks for pure navigation without data extraction+ - Use "extraction" blocks for data extraction from the current page+ - Use "login" blocks for authentication flows+* Include all required fields for each block type (label, next_block_label, block_type, etc.)+* Use descriptive, unique labels for blocks (snake_case format)+* Reference parameters using Jinja2 syntax: {% raw %}{{ parameters.param_key }}{% endraw %}+* If the user's request is unclear or ambiguous, ask for clarification+* Generate complete, runnable workflows - don't use placeholders or TODOs+* KEEP IT MINIMAL (BUT NOT LESS) - CRITICAL:+ - Generate ONLY the blocks that the user explicitly requests+ - DO NOT add extra verification, validation, or confirmation blocks unless specifically asked+ - Keep in mind that users often want partial workflows that they will extend later+* BE PROACTIVE - DON'T ASK REDUNDANT QUESTIONS:+ - If you asked for information and received it, PROCEED immediately - don't ask for confirmation+ - Only ask follow-up questions if there's genuine ambiguity or multiple valid approaches+* PARAMETER CONSISTENCY - CRITICAL:+ - For ANY parameter referenced anywhere in the workflow (parameter_keys OR Jinja2 like {% raw %}{{ param_foo }}){% endraw %},+ you MUST add a matching parameter definition under the top-level workflow parameters.+ - This includes derived values you introduce (e.g., last_week_date); if you cannot infer a default_value,+ ask the user and do not invent placeholders.+* CREDENTIAL HANDLING - CRITICAL:+ - If the user's request requires authentication (login to private accounts, authenticated actions, etc.)+ but NO credential ID is provided, you MUST ask for credentials:+ - Use the ASK_QUESTION response type+ - Ask the user to provide their credential ID that's securely stored in Skyvern+ - ALWAYS include an example format in your question: "(e.g., cred_123)"+ - ALWAYS emphasize: "Please provide the credential ID securely stored in Skyvern, not raw login/password"+ - ALWAYS include "DO NOT PROVIDE RAW LOGIN/PASSWORD" all in uppercase+ - DO NOT generate a workflow without credentials when authentication is required++ Example question format:+ "Please provide your credential ID (e.g., cred_123) that's securely stored in Skyvern for authentication. DO NOT PROVIDE RAW LOGIN/PASSWORD."++ - When the user provides a credential ID (e.g., "cred_123"), you MUST:+ - Create a credential parameter with a descriptive key (e.g., "login_credentials")+ - Put the credential ID in the default_value field, NOT in the key field+ - Reference the descriptive key in parameter_keys, NOT the credential ID++ - If login fails with TOTP/2FA errors (including rate limiting), ask the user for a TOTP-enabled credential (e.g., cred_123)++Note that user usually works with UI and not aware of YAML, avoid mentioning YAML in user-facing response.++RESPONSE FORMAT:++You must respond with a JSON object (and nothing else) containing ONE of these actions.++**COMMON FIELDS (ALL RESPONSES MUST INCLUDE):**++Every response must include these fields:+- "type": The action type (REPLACE_WORKFLOW, REPLY, or ASK_QUESTION)+- "user_response": A short, user-facing message to show in chat+- "global_llm_context": Long-term memory for overall workflow goal++**CONTEXT FIELDS - CRITICAL:**+**global_llm_context (Long-term memory):**+ - For the overall user goal and workflow direction+ - Persist throughout the entire workflow building session+ - Update ONLY when the goal is clarified or modified+ - Include: user's main objective, workflow type being built, key decisions made+ - Example: "Building workflow to track GitHub PRs from last week. Using minimal approach (login only, no verification). Will use cred_123 for authentication."+ - Keep this stable - don't change unless the user's goal actually changes+ - Set on first user request, update only when goal changes or is clarified.++**ACTION TYPES:**++**Option 1: Replace the entire workflow**+Use this when the user wants to create a new workflow, make major structural changes, or when the current workflow needs significant modifications.++{+ "type": "REPLACE_WORKFLOW",+ "user_response": "A short response to show the user in chat",+ "workflow_yaml": "The complete new workflow YAML definition as a string",+ "global_llm_context": "Long-term goal/objective"+}++**Option 2: Reply without updating workflow**+Use this when answering questions, providing explanations, debugging help, or any response that doesn't require modifying the workflow YAML.++{+ "type": "REPLY",+ "user_response": "A short response to show the user in chat",+ "global_llm_context": "Long-term goal/objective"+}++**Option 3: Ask for clarification**+Use this when the user's request is ambiguous, missing critical information, or could be interpreted multiple ways.++{+ "type": "ASK_QUESTION",+ "user_response": "A short response to show the user in chat",+ "question": "A clear, specific question to ask the user",+ "global_llm_context": "User's overall goal (preserve from previous)"+}++MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes.+All YAML content must be properly escaped as JSON strings.++Current datetime, ISO format:+```+{{ current_datetime }}+```diff --git a/skyvern/forge/prompts/skyvern/workflow_knowledge_base.txt b/skyvern/forge/prompts/skyvern/workflow_knowledge_base.txtnew file mode 100644index 000000000..b6a79e7e1--- /dev/null+++ b/skyvern/forge/prompts/skyvern/workflow_knowledge_base.txt@@ -0,0 +1,625 @@+SKYVERN WORKFLOW YAML KNOWLEDGE BASE++This document provides comprehensive information about Skyvern Workflow YAML structure and blocks. Use this to understand how to construct, modify, and validate workflow definitions.++** WORKFLOW STRUCTURE OVERVIEW **++A Skyvern workflow is defined in YAML format with the following top-level structure+for a workflow definition (embedded under workflow_definition in full specs):++title: "<workflow title>"+description: "<optional description>"+workflow_definition:+ version: 2 # IMPORTANT: Always use version 2+ parameters: []+ blocks: []+webhook_callback_url: "<optional_https_url>" # Optional: Webhook URL to receive workflow run updates++Key Concepts:+- Workflows consist of sequential or conditional blocks that represent specific tasks+- Each block has a unique label for identification and navigation+- Blocks can reference workflow parameters using Jinja2 templating+- Block execution is defined by next_block_label on every non-terminal block++** WORKFLOW PARAMETERS **++Parameters provide input values and credentials to workflows. They are defined in the "parameters" list.++Common Parameter Types:++* WORKFLOW PARAMETERS (user inputs)+parameter_type: workflow+key: <unique_key>+workflow_parameter_type: <string|integer|float|boolean|json|file_url|credential_id>+description: <optional description>+default_value: <optional default>++Example:+parameters:+ - parameter_type: workflow+ key: search_query+ workflow_parameter_type: string+ description: "Search term to use"+ default_value: "example"++* OUTPUT PARAMETERS (block outputs)+parameter_type: output+key: <unique_key>+description: <optional description>++* CREDENTIAL PARAMETERS+parameter_type: workflow+workflow_parameter_type: credential_id+key: <unique_key>+default_value: <credential_id>++Using Parameters in Blocks:+- Reference using Jinja2: {{ param_key }}+- List parameter_keys in blocks that use them+- Parameters are resolved before block execution. ALL PARAMETER KEYS REFERENCED IN BLOCKS MUST FIRST BE DEFINED IN THE WORKFLOW PARAMETERS LIST++Example:+workflow_definition:+ version: 2+ parameters:+ - key: topics_count+ description: null+ parameter_type: workflow+ workflow_parameter_type: integer+ default_value: "3"+ blocks:+ - label: block_1+ block_type: task_v2+ prompt: Give me top {{topics_count}} news items+ url: https://news.ycombinator.com/+ next_block_label: null++** COMMON BLOCK FIELDS **++All blocks inherit these base fields:++block_type: <type> # Required: Defines the block type+label: <unique_label> # Required: Unique identifier for this block+next_block_label: <label|null> # Required: Label of next block; use null only for terminal blocks+continue_on_failure: false # Optional: Continue workflow if block fails+next_loop_on_failure: false # Optional: Continue to next loop iteration on failure (for loop blocks only)+model: {} # Optional: Override model settings for this block++Important Rules:+- Labels must be unique within a workflow+- Labels cannot be empty or contain only whitespace+- next_block_label is required for all non-terminal blocks+- Use next_block_label for explicit flow control+- Set next_block_label to null to mark the end of a flow+- continue_on_failure allows graceful error handling++** TASK BLOCK (task) **++Purpose: Navigate to a URL, perform actions based on natural language goals, and optionally extract data.++Structure:+block_type: task+label: <unique_label>+url: <starting_url> # Optional: URL to navigate to; omit to continue on current page+title: str # Required: The title of the block+navigation_goal: <action_description> # Optional: What actions to perform+data_extraction_goal: <extraction_description> # Optional: What data to extract+data_schema: <json_schema> # Optional: Schema for extracted data+error_code_mapping: {} # Optional: Map errors to custom codes+max_retries: 0 # Optional: Number of retry attempts+max_steps_per_run: null # Optional: Limit steps per execution+parameter_keys: [] # Optional: Parameters used in this block+complete_on_download: false # Optional: Complete when file downloads+download_suffix: null # Optional: Downloaded file name+totp_verification_url: null # Optional: TOTP verification URL+disable_cache: false # Optional: Disable caching+complete_criterion: null # Optional: Condition to mark complete+terminate_criterion: null # Optional: Condition to terminate+complete_verification: true # Optional: Verify completion+include_action_history_in_verification: false # Optional: Include history in verification++Use Cases:+- Fill out forms on websites+- Navigate complex multi-step processes+- Extract structured data from pages+- Combine navigation and extraction in one step++Example:+blocks:+ - block_type: task+ label: search_and_extract+ next_block_label: null+ url: "https://example.com/search"+ navigation_goal: "Search for {{ query }} and click the first result"+ data_extraction_goal: "Extract the product name, price, and availability"+ data_schema:+ type: object+ properties:+ name: {type: string}+ price: {type: number}+ available: {type: boolean}+ parameter_keys:+ - query+ max_retries: 2++** URL BLOCK (goto_url) **++Purpose: Navigate directly to a URL without any additional instructions.++Structure:+block_type: goto_url+label: <unique_label>+url: <target_url> # Required: URL to navigate to+error_code_mapping: {} # Optional: Custom error codes+max_retries: 0 # Optional: Retry attempts+parameter_keys: [] # Optional: Parameters used++Use Cases:+- Jump to a known page before a task block+- Reset the browser state to a specific URL+- Split URL navigation from subsequent actions++Example:+blocks:+ - block_type: goto_url+ label: open_cart+ next_block_label: null+ url: "https://example.com/cart"++** ACTION BLOCK (action) **++Purpose: Perform a single focused action on the current page without data extraction.++Structure:+block_type: action+label: <unique_label>+navigation_goal: <action_description> # Required: Single action to perform+url: <starting_url> # Optional: URL to start from+error_code_mapping: {} # Optional: Custom error codes+max_retries: 0 # Optional: Retry attempts+parameter_keys: [] # Optional: Parameters used+complete_on_download: false # Optional: Complete on download+download_suffix: null # Optional: Download file name+totp_verification_url: null # Optional: TOTP verification URL+totp_identifier: null # Optional: TOTP identifier+disable_cache: false # Optional: Disable cache++Use Cases:+- Click a specific button or link+- Fill a single field or selection+- Trigger a download with one action++Example:+blocks:+ - block_type: action+ label: accept_terms+ next_block_label: null+ url: "https://example.com/checkout"+ navigation_goal: "Check the terms checkbox"+ max_retries: 1++** TASK V2 BLOCK (task_v2) **++Purpose: Task block that can handle complex, multi-step workflows using a single natural language prompt. Can handle more complex scenarios than task blocks but may be slightly slower.++Structure:+block_type: task_v2+label: <unique_label>+prompt: <natural_language_instruction> # Required: What to do+url: <starting_url> # Optional: URL to navigate to; omit to continue on current page+disable_cache: false # Optional: Disable caching++Use Cases:+- Complex, multi-step workflows that can be described in natural language+- Scenarios requiring multiple actions and decision-making+- General-purpose automation with flexible requirements+- When you need to handle more complex scenarios and are okay with potentially slower execution++Differences from Task Block:+- Uses single "prompt" field instead of separate "navigation_goal" and "data_extraction_goal"+- Can handle more complex scenarios and longer sequences of actions+- May be slightly slower than task blocks+- No data_schema (extraction format described in prompt)+- More flexible configuration++Example:+blocks:+ - block_type: task_v2+ label: simple_booking+ next_block_label: null+ url: "https://booking.example.com"+ prompt: "Book a flight from {{ origin }} to {{ destination }} on {{ date }}. Return the booking confirmation number."+ max_iterations: 10++** LOGIN BLOCK (login) **++Purpose: Handle authentication flows including username/password and TOTP/2FA.++Structure:+block_type: login+label: <unique_label>+url: <login_page_url> # Optional: Login page URL+title: str # Required: The title of the block+navigation_goal: null # Optional: Additional navigation after login+error_code_mapping: {} # Optional: Custom error codes+max_retries: 0 # Optional: Retry attempts+max_steps_per_run: null # Optional: Step limit+parameter_keys: [] # Required: Should include credential parameters+complete_criterion: null # Optional: Completion condition+terminate_criterion: null # Optional: Termination condition+complete_verification: true # Optional: Verify successful login++Use Cases:+- Login to websites with username/password+- Handle 2FA/TOTP authentication+- Manage credential-protected workflows+- Session initialization++Important Notes:+- Credentials should be stored as parameters (credential, bitwarden_login_credential, etc.)+- TOTP is automatically handled if the credential parameter has TOTP configured++Example:+parameters:+ - parameter_type: workflow+ workflow_parameter_type: credential_id+ key: my_credentials+ default_value: "cred_uuid_here"+blocks:+ - block_type: login+ label: login_to_portal+ next_block_label: null+ url: "https://portal.example.com/login"+ parameter_keys:+ - my_credentials # This must match a 'key' from the parameters list above+ complete_criterion: "Current URL is 'https://portal.example.com/dashboard'"+ max_retries: 2++** EXTRACTION BLOCK (extraction) **++Purpose: Extract structured data from the current page without navigation.++Structure:+block_type: extraction+label: <unique_label>+title: str # Required: The title of the block+data_extraction_goal: <what_to_extract> # Required: Description of data to extract+data_schema: <json_schema> # Optional: Structure of extracted data+url: <page_url> # Optional: URL to navigate to first+max_retries: 0 # Optional: Retry attempts+max_steps_per_run: null # Optional: Step limit+parameter_keys: [] # Optional: Parameters used+disable_cache: false # Optional: Disable cache+Use Cases:+- Extract structured data after other blocks+- Parse tables, lists, or forms+- Collect multiple data points from a page+- Data mining from web pages++Data Schema Formats:+* JSON Schema object:+ data_schema:+ type: object+ properties:+ field1: {type: string}+ field2: {type: number}++* JSON Schema array:+ data_schema:+ type: array+ items:+ type: object+ properties:+ name: {type: string}++* String format (for simple extractions):+ data_schema: "csv_string"++Example:+blocks:+ - block_type: extraction+ label: extract_product_list+ next_block_label: null+ data_extraction_goal: "Extract all products with their names, prices, and stock status"+ data_schema:+ type: array+ items:+ type: object+ properties:+ product_name: {type: string}+ price: {type: number}+ in_stock: {type: boolean}+ rating: {type: number}+ max_retries: 1++** PARAMETER TEMPLATING **++All string fields in blocks support Jinja2 templating to reference parameters.++Syntax (preferred):+{{ param_key }}++Examples:++* In URL:+url: "https://example.com/search?q={{ search_term }}"++* In goals:+navigation_goal: "Search for {{ product_name }} and filter by {{ category }}"++* In data extraction:+data_extraction_goal: "Extract {{ field_name }} from the results"++* Complex expressions:+navigation_goal: "Enter {{ first_name }} {{ last_name }} in the name field"++* In schemas (as descriptions):+data_schema:+ type: object+ properties:+ query_result:+ type: string+ description: "Result for query: {{ query }}"++** ERROR HANDLING AND RETRIES **++Error Code Mapping:+Map internal errors to custom error codes for easier handling:++error_code_mapping:+ "ElementNotFound": "ELEMENT_MISSING"+ "TimeoutError": "PAGE_TIMEOUT"+ "NavigationFailed": "NAV_ERROR"++Retry Configuration:+max_retries: 3 # Block will retry up to 3 times on failure++Conditional Continuation:+continue_on_failure: true # Workflow continues even if block fails++Loop Continuation:+next_loop_on_failure: true # Skip to next iteration in loops++Completion Criteria:+complete_criterion: "URL contains '/success'" # Condition for success+terminate_criterion: "Element with text 'Error' exists" # Condition to stop++** WORKFLOW EXECUTION FLOW **++Sequential Execution:+blocks:+ - block_type: goto_url+ label: step1+ next_block_label: step2+ url: "https://example.com/start"+ - block_type: extraction+ label: step2+ next_block_label: step3+ - block_type: task+ label: step3+ next_block_label: null+# Executes: step1 → step2 → step3++Explicit Flow Control (Skip blocks):+blocks:+ - block_type: goto_url+ label: login+ next_block_label: extract_data+ url: "https://app.example.com/login"+ - block_type: task+ label: handle_error+ next_block_label: null+ - block_type: extraction+ label: extract_data+ next_block_label: null+# Executes: login → extract_data (skips handle_error)++Error Recovery Flow:+blocks:+ - block_type: task+ label: primary_task+ next_block_label: verify_result+ continue_on_failure: true+ - block_type: validation+ label: verify_result+ next_block_label: null++** BEST PRACTICES **++* Naming Conventions:+ - Use descriptive labels: "login_to_portal" not "step1"+ - Use snake_case for labels and parameter keys+ - Make labels unique and meaningful++* Goal Writing:+ - Be specific: "Click the blue 'Submit' button" vs "Submit the form"+ - Include context: "After clicking Search, wait for results to load"+ - Natural language: Write as you would instruct a human++* Parameter Usage:+ - Always list parameter_keys when using parameters in a block+ - Validate parameter types match usage+ - Provide default values for optional parameters++* Error Handling:+ - Set appropriate max_retries for flaky operations+ - Use complete_criterion for validation+ - Map errors to meaningful codes for debugging++* Data Extraction:+ - Always provide data_schema for structured extraction+ - Use specific extraction goals+ - Handle arrays vs objects appropriately++* Performance:+ - Use disable_cache: true for dynamic content+ - Set max_steps_per_run to prevent infinite loops+ - Combine navigation and extraction in task blocks when possible++* Security:+ - Never hardcode credentials in workflows+ - Use credential parameters for sensitive data+ - Use AWS secrets or vault integrations++** COMMON PATTERNS **++Pattern 1: Login → Navigate → Extract+parameters:+ - parameter_type: workflow+ workflow_parameter_type: credential_id+ key: my_credentials+ default_value: "uuid"+ - parameter_type: output+ key: extracted_data+blocks:+ - block_type: login+ label: authenticate+ next_block_label: go_to_reports+ url: "https://app.example.com/login"+ parameter_keys: [my_credentials]+ - block_type: task+ label: go_to_reports+ next_block_label: get_report_data+ navigation_goal: "Navigate to Reports section"+ - block_type: extraction+ label: get_report_data+ next_block_label: null+ data_extraction_goal: "Extract all report entries"+ data_schema:+ type: array+ items: {type: object}++Pattern 2: Search with Dynamic Input+parameters:+ - parameter_type: workflow+ key: search_query+ workflow_parameter_type: string+blocks:+ - block_type: task_v2+ label: search_and_extract+ next_block_label: null+ url: "https://example.com"+ prompt: "Search for '{{ search_query }}' and extract the first 10 results with titles and URLs"++Pattern 3: Multi-Step Form Filling+blocks:+ - block_type: goto_url+ label: open_form+ next_block_label: fill_personal_info+ url: "https://forms.example.com/application"+ - block_type: task+ label: fill_personal_info+ next_block_label: fill_address+ navigation_goal: "Fill in name as {{ name }}, email as {{ email }}"+ parameter_keys: [name, email]+ - block_type: task+ label: fill_address+ next_block_label: submit+ navigation_goal: "Fill in address fields and click Continue"+ parameter_keys: [address, city, zip]+ - block_type: task+ label: submit+ next_block_label: null+ navigation_goal: "Review information and click Submit"++Pattern 4: Conditional Extraction+blocks:+ - block_type: task+ label: search_product+ next_block_label: check_availability+ navigation_goal: "Search for {{ product }}"+ - block_type: extraction+ label: check_availability+ next_block_label: add_to_cart+ data_extraction_goal: "Check if product is in stock"+ data_schema:+ type: object+ properties:+ in_stock: {type: boolean}+ - block_type: task+ label: add_to_cart+ next_block_label: null+ navigation_goal: "If product is in stock, add to cart"++** VALIDATION RULES **++Workflow-Level:+- All block labels must be unique+- Parameters referenced in blocks must be defined+- next_block_label must point to existing block labels or be null+- The last block in execution flow should have next_block_label: null++Block-Level:+- label is required and cannot be empty+- block_type must be a valid type+- For task blocks: either navigation_goal or data_extraction_goal should be present+- For extraction blocks: data_extraction_goal is required+- For action blocks: navigation_goal is required+- For login blocks: parameter_keys should include credentials++Parameter-Level:+- key must be unique across parameters+- key cannot contain whitespace+- parameter_type must be valid+- Referenced keys (like source_parameter_key) must exist++** COMPLETE WORKFLOW EXAMPLE **++title: E-commerce Product Search and Purchase+description: Search for a product, extract details, and add to cart+workflow_definition:+ version: 2+ parameters:+ - parameter_type: workflow+ key: product_name+ workflow_parameter_type: string+ description: "Product to search for"+ - parameter_type: workflow+ key: max_price+ workflow_parameter_type: float+ description: "Maximum price willing to pay"+ - parameter_type: workflow+ workflow_parameter_type: credential_id+ key: account_creds+ default_value: "cred_12345"+ - parameter_type: output+ key: product_details+ description: "Extracted product information"+ blocks:+ - block_type: login+ label: login_to_store+ next_block_label: search_and_filter+ url: "https://shop.example.com/login"+ parameter_keys:+ - account_creds+ complete_criterion: "URL contains '/dashboard'"++ - block_type: task+ label: search_and_filter+ next_block_label: get_product_info+ url: "https://shop.example.com/search"+ navigation_goal: "Search for {{ product_name }} and filter results by price under ${{ max_price }}"+ parameter_keys:+ - product_name+ - max_price+ max_retries: 2++ - block_type: extraction+ label: get_product_info+ next_block_label: add_to_cart+ data_extraction_goal: "Extract product name, price, rating, and availability"+ data_schema:+ type: object+ properties:+ name: {type: string}+ price: {type: number}+ rating: {type: number}+ available: {type: boolean}++ - block_type: task_v2+ label: add_to_cart+ next_block_label: null+ prompt: "Click on the first available product and add it to cart"+ max_iterations: 5++END OF KNOWLEDGE BASEdiff --git a/skyvern/forge/sdk/db/agent_db.py b/skyvern/forge/sdk/db/agent_db.pyindex a1660d4fb..033eef095 100644--- a/skyvern/forge/sdk/db/agent_db.py+++ b/skyvern/forge/sdk/db/agent_db.py@@ -47,6 +47,8 @@ from skyvern.forge.sdk.db.models import (TaskV2Model,ThoughtModel,TOTPCodeModel,+ WorkflowCopilotChatMessageModel,+ WorkflowCopilotChatModel,WorkflowModel,WorkflowParameterModel,WorkflowRunBlockModel,@@ -72,6 +74,7 @@ from skyvern.forge.sdk.db.utils import (convert_to_task,convert_to_task_v2,convert_to_workflow,+ convert_to_workflow_copilot_chat_message,convert_to_workflow_parameter,convert_to_workflow_run,convert_to_workflow_run_block,@@ -100,6 +103,11 @@ from skyvern.forge.sdk.schemas.task_generations import TaskGenerationfrom skyvern.forge.sdk.schemas.task_v2 import TaskV2, TaskV2Status, Thought, ThoughtTypefrom skyvern.forge.sdk.schemas.tasks import OrderBy, SortDirection, Task, TaskStatusfrom skyvern.forge.sdk.schemas.totp_codes import OTPType, TOTPCode+from skyvern.forge.sdk.schemas.workflow_copilot import (+ WorkflowCopilotChat,+ WorkflowCopilotChatMessage,+ WorkflowCopilotChatSender,+)from skyvern.forge.sdk.schemas.workflow_runs import WorkflowRunBlockfrom skyvern.forge.sdk.workflow.models.parameter import (AWSSecretParameter,@@ -3640,6 +3648,91 @@ class AgentDB(BaseAlchemyDB):await session.refresh(new_ai_suggestion)return AISuggestion.model_validate(new_ai_suggestion)+ async def create_workflow_copilot_chat(+ self,+ organization_id: str,+ workflow_permanent_id: str,+ ) -> WorkflowCopilotChat:+ async with self.Session() as session:+ new_chat = WorkflowCopilotChatModel(+ organization_id=organization_id,+ workflow_permanent_id=workflow_permanent_id,+ )+ session.add(new_chat)+ await session.commit()+ await session.refresh(new_chat)+ return WorkflowCopilotChat.model_validate(new_chat)++ async def create_workflow_copilot_chat_message(+ self,+ organization_id: str,+ workflow_copilot_chat_id: str,+ sender: WorkflowCopilotChatSender,+ content: str,+ global_llm_context: str | None = None,+ ) -> WorkflowCopilotChatMessage:+ async with self.Session() as session:+ new_message = WorkflowCopilotChatMessageModel(+ workflow_copilot_chat_id=workflow_copilot_chat_id,+ organization_id=organization_id,+ sender=sender,+ content=content,+ global_llm_context=global_llm_context,+ )+ session.add(new_message)+ await session.commit()+ await session.refresh(new_message)+ return convert_to_workflow_copilot_chat_message(new_message, self.debug_enabled)++ async def get_workflow_copilot_chat_messages(+ self,+ workflow_copilot_chat_id: str,+ ) -> list[WorkflowCopilotChatMessage]:+ async with self.Session() as session:+ query = (+ select(WorkflowCopilotChatMessageModel)+ .filter(WorkflowCopilotChatMessageModel.workflow_copilot_chat_id == workflow_copilot_chat_id)+ .order_by(WorkflowCopilotChatMessageModel.workflow_copilot_chat_message_id.asc())+ )+ messages = (await session.scalars(query)).all()+ return [convert_to_workflow_copilot_chat_message(message, self.debug_enabled) for message in messages]++ async def get_workflow_copilot_chat_by_id(+ self,+ organization_id: str,+ workflow_copilot_chat_id: str,+ ) -> WorkflowCopilotChat | None:+ async with self.Session() as session:+ query = (+ select(WorkflowCopilotChatModel)+ .filter(WorkflowCopilotChatModel.organization_id == organization_id)+ .filter(WorkflowCopilotChatModel.workflow_copilot_chat_id == workflow_copilot_chat_id)+ .order_by(WorkflowCopilotChatModel.created_at.desc())+ .limit(1)+ )+ chat = (await session.scalars(query)).first()+ if not chat:+ return None+ return WorkflowCopilotChat.model_validate(chat)++ async def get_latest_workflow_copilot_chat(+ self,+ organization_id: str,+ workflow_permanent_id: str,+ ) -> WorkflowCopilotChat | None:+ async with self.Session() as session:+ query = (+ select(WorkflowCopilotChatModel)+ .filter(WorkflowCopilotChatModel.organization_id == organization_id)+ .filter(WorkflowCopilotChatModel.workflow_permanent_id == workflow_permanent_id)+ .order_by(WorkflowCopilotChatModel.created_at.desc())+ .limit(1)+ )+ chat = (await session.scalars(query)).first()+ if not chat:+ return None+ return WorkflowCopilotChat.model_validate(chat)+async def get_task_generation_by_prompt_hash(self,user_prompt_hash: str,diff --git a/skyvern/forge/sdk/db/id.py b/skyvern/forge/sdk/db/id.pyindex 98c8f9179..ad0582c4f 100644--- a/skyvern/forge/sdk/db/id.py+++ b/skyvern/forge/sdk/db/id.py@@ -69,6 +69,8 @@ WORKFLOW_RUN_PREFIX = "wr"WORKFLOW_SCRIPT_PREFIX = "ws"WORKFLOW_TEMPLATE_PREFIX = "wt"ORGANIZATION_BILLING_PREFIX = "ob"+WORKFLOW_COPILOT_CHAT_PREFIX = "wcc"+WORKFLOW_COPILOT_CHAT_MESSAGE_PREFIX = "wccm"def generate_workflow_id() -> str:@@ -266,6 +268,16 @@ def generate_billing_id() -> str:return f"{ORGANIZATION_BILLING_PREFIX}_{int_id}"+def generate_workflow_copilot_chat_id() -> str:+ int_id = generate_id()+ return f"{WORKFLOW_COPILOT_CHAT_PREFIX}_{int_id}"+++def generate_workflow_copilot_chat_message_id() -> str:+ int_id = generate_id()+ return f"{WORKFLOW_COPILOT_CHAT_MESSAGE_PREFIX}_{int_id}"++############# Helper functions below ##############def generate_id() -> int:"""diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.pyindex e89f46d66..1a23f0ad0 100644--- a/skyvern/forge/sdk/db/models.py+++ b/skyvern/forge/sdk/db/models.py@@ -51,6 +51,8 @@ from skyvern.forge.sdk.db.id import (generate_task_v2_id,generate_thought_id,generate_totp_code_id,+ generate_workflow_copilot_chat_id,+ generate_workflow_copilot_chat_message_id,generate_workflow_id,generate_workflow_parameter_id,generate_workflow_permanent_id,@@ -1081,3 +1083,40 @@ class ScriptBlockModel(Base):created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False)modified_at = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow, nullable=False)deleted_at = Column(DateTime, nullable=True)+++class WorkflowCopilotChatModel(Base):+ __tablename__ = "workflow_copilot_chats"++ workflow_copilot_chat_id = Column(String, primary_key=True, default=generate_workflow_copilot_chat_id)+ organization_id = Column(String, nullable=False)+ workflow_permanent_id = Column(String, nullable=False, index=True)++ created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False)+ modified_at = Column(+ DateTime,+ default=datetime.datetime.utcnow,+ onupdate=datetime.datetime.utcnow,+ nullable=False,+ )+++class WorkflowCopilotChatMessageModel(Base):+ __tablename__ = "workflow_copilot_chat_messages"++ workflow_copilot_chat_message_id = Column(+ String, primary_key=True, default=generate_workflow_copilot_chat_message_id+ )+ workflow_copilot_chat_id = Column(String, nullable=False, index=True)+ organization_id = Column(String, nullable=False)+ sender = Column(String, nullable=False)+ content = Column(UnicodeText, nullable=False)+ global_llm_context = Column(UnicodeText, nullable=True)++ created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False)+ modified_at = Column(+ DateTime,+ default=datetime.datetime.utcnow,+ onupdate=datetime.datetime.utcnow,+ nullable=False,+ )diff --git a/skyvern/forge/sdk/db/utils.py b/skyvern/forge/sdk/db/utils.pyindex 7e4811ee6..45c2a8922 100644--- a/skyvern/forge/sdk/db/utils.py+++ b/skyvern/forge/sdk/db/utils.py@@ -21,6 +21,7 @@ from skyvern.forge.sdk.db.models import (StepModel,TaskModel,TaskV2Model,+ WorkflowCopilotChatMessageModel,WorkflowModel,WorkflowParameterModel,WorkflowRunBlockModel,@@ -39,6 +40,7 @@ from skyvern.forge.sdk.schemas.organizations import ()from skyvern.forge.sdk.schemas.task_v2 import TaskV2from skyvern.forge.sdk.schemas.tasks import Task, TaskStatus+from skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotChatMessage as WorkflowCopilotChatMessageSchemafrom skyvern.forge.sdk.schemas.workflow_runs import WorkflowRunBlockfrom skyvern.forge.sdk.workflow.models.parameter import (AWSSecretParameter,@@ -217,6 +219,17 @@ def convert_to_task_v2(task_v2_model: TaskV2Model, debug_enabled: bool = False)return TaskV2.model_validate(task_v2_data)+def convert_to_workflow_copilot_chat_message(+ message_model: WorkflowCopilotChatMessageModel, debug_enabled: bool = False+) -> WorkflowCopilotChatMessageSchema:+ if debug_enabled:+ LOG.debug(+ "Converting WorkflowCopilotChatMessage to WorkflowCopilotChatMessageSchema",+ workflow_copilot_chat_message_id=message_model.workflow_copilot_chat_message_id,+ )+ return WorkflowCopilotChatMessageSchema.model_validate(message_model)++def convert_to_step(step_model: StepModel, debug_enabled: bool = False) -> Step:if debug_enabled:LOG.debug("Converting StepModel to Step", step_id=step_model.step_id)diff --git a/skyvern/forge/sdk/routes/__init__.py b/skyvern/forge/sdk/routes/__init__.pyindex 0ae1ab5dd..a24204fca 100644--- a/skyvern/forge/sdk/routes/__init__.py+++ b/skyvern/forge/sdk/routes/__init__.py@@ -9,6 +9,7 @@ from skyvern.forge.sdk.routes import run_blocks # noqa: F401from skyvern.forge.sdk.routes import scripts # noqa: F401from skyvern.forge.sdk.routes import sdk # noqa: F401from skyvern.forge.sdk.routes import webhooks # noqa: F401+from skyvern.forge.sdk.routes import workflow_copilot # noqa: F401from skyvern.forge.sdk.routes.streaming import messages # noqa: F401from skyvern.forge.sdk.routes.streaming import screenshot # noqa: F401from skyvern.forge.sdk.routes.streaming import vnc # noqa: F401diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.pynew file mode 100644index 000000000..61b47a225--- /dev/null+++ b/skyvern/forge/sdk/routes/workflow_copilot.py@@ -0,0 +1,348 @@+from dataclasses import dataclass+from datetime import datetime, timezone+from pathlib import Path+from typing import Any++import structlog+import yaml+from fastapi import Depends, HTTPException, status++from skyvern.forge import app+from skyvern.forge.prompts import prompt_engine+from skyvern.forge.sdk.api.llm.exceptions import LLMProviderError+from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType+from skyvern.forge.sdk.routes.routers import base_router+from skyvern.forge.sdk.routes.run_blocks import DEFAULT_LOGIN_PROMPT+from skyvern.forge.sdk.schemas.organizations import Organization+from skyvern.forge.sdk.schemas.workflow_copilot import (+ WorkflowCopilotChatHistoryMessage,+ WorkflowCopilotChatHistoryResponse,+ WorkflowCopilotChatMessage,+ WorkflowCopilotChatRequest,+ WorkflowCopilotChatResponse,+ WorkflowCopilotChatSender,+)+from skyvern.forge.sdk.services import org_auth_service+from skyvern.schemas.workflows import LoginBlockYAML, WorkflowCreateYAMLRequest++WORKFLOW_KNOWLEDGE_BASE_PATH = Path("skyvern/forge/prompts/skyvern/workflow_knowledge_base.txt")+CHAT_HISTORY_CONTEXT_MESSAGES = 10++LOG = structlog.get_logger()+++@dataclass(frozen=True)+class RunInfo:+ block_label: str | None+ block_type: str+ block_status: str | None+ failure_reason: str | None+ html: str | None+++async def _get_debug_artifact(organization_id: str, workflow_run_id: str) -> Artifact | None:+ artifacts = await app.DATABASE.get_artifacts_for_run(+ run_id=workflow_run_id, organization_id=organization_id, artifact_types=[ArtifactType.VISIBLE_ELEMENTS_TREE]+ )+ return artifacts[0] if isinstance(artifacts, list) else None+++async def _get_debug_run_info(organization_id: str, workflow_run_id: str | None) -> RunInfo | None:+ if not workflow_run_id:+ return None++ blocks = await app.DATABASE.get_workflow_run_blocks(+ workflow_run_id=workflow_run_id, organization_id=organization_id+ )+ if not blocks:+ return None++ block = blocks[0]++ artifact = await _get_debug_artifact(organization_id, workflow_run_id)+ if artifact:+ artifact_bytes = await app.ARTIFACT_MANAGER.retrieve_artifact(artifact)+ html = artifact_bytes.decode("utf-8") if artifact_bytes else None+ else:+ html = None++ return RunInfo(+ block_label=block.label,+ block_type=block.block_type.name,+ block_status=block.status,+ failure_reason=block.failure_reason,+ html=html,+ )+++async def copilot_call_llm(+ organization_id: str,+ chat_request: WorkflowCopilotChatRequest,+ chat_history: list[WorkflowCopilotChatHistoryMessage],+ global_llm_context: str | None,+ debug_run_info_text: str,+) -> tuple[str, str | None, str | None]:+ current_datetime = datetime.now(timezone.utc).isoformat()++ chat_history_text = ""+ if chat_history:+ history_lines = [f"{msg.sender}: {msg.content}" for msg in chat_history]+ chat_history_text = "\n".join(history_lines)++ workflow_knowledge_base = WORKFLOW_KNOWLEDGE_BASE_PATH.read_text(encoding="utf-8")++ llm_prompt = prompt_engine.load_prompt(+ template="workflow-copilot",+ workflow_knowledge_base=workflow_knowledge_base,+ workflow_yaml=chat_request.workflow_yaml or "",+ user_message=chat_request.message,+ chat_history=chat_history_text,+ global_llm_context=global_llm_context or "",+ current_datetime=current_datetime,+ debug_run_info=debug_run_info_text,+ )++ LOG.info(+ "Calling LLM for workflow copilot",+ prompt_length=len(llm_prompt),+ )+ llm_response = await app.LLM_API_HANDLER(+ prompt=llm_prompt,+ prompt_name="workflow-copilot",+ organization_id=organization_id,+ )++ if isinstance(llm_response, dict) and "output" in llm_response:+ action_data = llm_response["output"]+ else:+ action_data = llm_response++ if not isinstance(action_data, dict):+ LOG.error(+ "LLM response is not valid JSON",+ organization_id=organization_id,+ response_type=type(action_data).__name__,+ )+ raise HTTPException(+ status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,+ detail="Invalid response from LLM",+ )++ action_type = action_data.get("type")+ user_response_value = action_data.get("user_response")+ if user_response_value is None:+ user_response = "I received your request but I'm not sure how to help. Could you rephrase?"+ else:+ user_response = str(user_response_value)+ LOG.info(+ "LLM response received",+ organization_id=organization_id,+ action_type=action_type,+ )++ global_llm_context = action_data.get("global_llm_context")+ if global_llm_context is not None:+ global_llm_context = str(global_llm_context)++ if action_type == "REPLACE_WORKFLOW":+ workflow_yaml = await _process_workflow_yaml(action_data)+ return user_response, workflow_yaml, global_llm_context+ elif action_type == "REPLY":+ return user_response, None, global_llm_context+ elif action_type == "ASK_QUESTION":+ return user_response, None, global_llm_context+ else:+ LOG.error(+ "Unknown action type from LLM",+ organization_id=organization_id,+ action_type=action_type,+ )+ return "I received your request but I'm not sure how to help. Could you rephrase?", None, None+++async def _process_workflow_yaml(action_data: dict[str, Any]) -> None | str:+ workflow_yaml = action_data.get("workflow_yaml", "")++ try:+ parsed_yaml = yaml.safe_load(workflow_yaml)+ except yaml.YAMLError as e:+ LOG.error(+ "Invalid YAML from LLM",+ error=str(e),+ yaml=f"\n{str(e)}\n{workflow_yaml}",+ )+ raise HTTPException(+ status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,+ detail=f"LLM generated invalid YAML: {str(e)}",+ )++ try:+ # Fixing trivial common LLM mistakes+ workflow_definition = parsed_yaml.get("workflow_definition", None)+ if workflow_definition:+ blocks = workflow_definition.get("blocks", [])+ for block in blocks:+ block["title"] = block.get("title", "")++ workflow = WorkflowCreateYAMLRequest.model_validate(parsed_yaml)++ # Post-processing+ for block in workflow.workflow_definition.blocks:+ if isinstance(block, LoginBlockYAML) and not block.navigation_goal:+ block.navigation_goal = DEFAULT_LOGIN_PROMPT++ workflow_yaml = yaml.safe_dump(workflow.model_dump(mode="json"), sort_keys=False)+ except Exception as e:+ LOG.error(+ "YAML from LLM does not conform to Skyvern workflow schema",+ error=str(e),+ yaml=f"\n{str(e)}\n{workflow_yaml}",+ )+ raise HTTPException(+ status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,+ detail=f"LLM generated YAML that doesn't match workflow schema: {str(e)}",+ )+ return workflow_yaml+++@base_router.post("/workflow/copilot/chat-post", include_in_schema=False)+async def workflow_copilot_chat_post(+ chat_request: WorkflowCopilotChatRequest,+ organization: Organization = Depends(org_auth_service.get_current_org),+) -> WorkflowCopilotChatResponse:+ LOG.info(+ "Workflow copilot chat request",+ workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id,+ workflow_run_id=chat_request.workflow_run_id,+ message=chat_request.message,+ workflow_yaml_length=len(chat_request.workflow_yaml),+ organization_id=organization.organization_id,+ )++ request_started_at = datetime.now(timezone.utc)++ if chat_request.workflow_copilot_chat_id:+ chat = await app.DATABASE.get_workflow_copilot_chat_by_id(+ organization_id=organization.organization_id,+ workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id,+ )+ if not chat:+ raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Chat not found")+ if chat_request.workflow_permanent_id != chat.workflow_permanent_id:+ raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Wrong workflow permanent ID")+ else:+ chat = await app.DATABASE.create_workflow_copilot_chat(+ organization_id=organization.organization_id,+ workflow_permanent_id=chat_request.workflow_permanent_id,+ )++ chat_messages = await app.DATABASE.get_workflow_copilot_chat_messages(+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,+ )+ global_llm_context = None+ for message in reversed(chat_messages):+ if message.global_llm_context is not None:+ global_llm_context = message.global_llm_context+ break++ debug_run_info = await _get_debug_run_info(organization.organization_id, chat_request.workflow_run_id)++ # Format debug run info for prompt+ debug_run_info_text = ""+ if debug_run_info:+ debug_run_info_text = f"Block Label: {debug_run_info.block_label}"+ debug_run_info_text += f" Block Type: {debug_run_info.block_type}"+ debug_run_info_text += f" Status: {debug_run_info.block_status}"+ if debug_run_info.failure_reason:+ debug_run_info_text += f"\nFailure Reason: {debug_run_info.failure_reason}"+ if debug_run_info.html:+ debug_run_info_text += f"\n\nVisible Elements Tree (HTML):\n{debug_run_info.html}"++ await app.DATABASE.create_workflow_copilot_chat_message(+ organization_id=chat.organization_id,+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,+ sender=WorkflowCopilotChatSender.USER,+ content=chat_request.message,+ )++ try:+ user_response, updated_workflow_yaml, updated_global_llm_context = await copilot_call_llm(+ organization.organization_id,+ chat_request,+ convert_to_history_messages(chat_messages[-CHAT_HISTORY_CONTEXT_MESSAGES:]),+ global_llm_context,+ debug_run_info_text,+ )+ except HTTPException:+ raise+ except LLMProviderError as e:+ LOG.error(+ "LLM provider error",+ organization_id=organization.organization_id,+ error=str(e),+ exc_info=True,+ )+ raise HTTPException(+ status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,+ detail="Failed to process your request. Please try again.",+ )+ except Exception as e:+ LOG.error(+ "Unexpected error in workflow copilot",+ organization_id=organization.organization_id,+ error=str(e),+ exc_info=True,+ )+ raise HTTPException(+ status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,+ detail=f"An error occurred: {str(e)}",+ )++ assistant_message = await app.DATABASE.create_workflow_copilot_chat_message(+ organization_id=chat.organization_id,+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,+ sender=WorkflowCopilotChatSender.AI,+ content=user_response,+ global_llm_context=updated_global_llm_context,+ )++ return WorkflowCopilotChatResponse(+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,+ message=user_response,+ updated_workflow_yaml=updated_workflow_yaml,+ request_time=request_started_at,+ response_time=assistant_message.created_at,+ )+++@base_router.get("/workflow/copilot/chat-history", include_in_schema=False)+async def workflow_copilot_chat_history(+ workflow_permanent_id: str,+ organization: Organization = Depends(org_auth_service.get_current_org),+) -> WorkflowCopilotChatHistoryResponse:+ latest_chat = await app.DATABASE.get_latest_workflow_copilot_chat(+ organization_id=organization.organization_id,+ workflow_permanent_id=workflow_permanent_id,+ )+ if not latest_chat:+ return WorkflowCopilotChatHistoryResponse(workflow_copilot_chat_id=None, chat_history=[])+ chat_messages = await app.DATABASE.get_workflow_copilot_chat_messages(+ workflow_copilot_chat_id=latest_chat.workflow_copilot_chat_id,+ )+ return WorkflowCopilotChatHistoryResponse(+ workflow_copilot_chat_id=latest_chat.workflow_copilot_chat_id,+ chat_history=convert_to_history_messages(chat_messages),+ )+++def convert_to_history_messages(+ messages: list[WorkflowCopilotChatMessage],+) -> list[WorkflowCopilotChatHistoryMessage]:+ return [+ WorkflowCopilotChatHistoryMessage(+ sender=message.sender,+ content=message.content,+ created_at=message.created_at,+ )+ for message in messages+ ]diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.pynew file mode 100644index 000000000..29de46189--- /dev/null+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py@@ -0,0 +1,58 @@+from datetime import datetime+from enum import StrEnum++from pydantic import BaseModel, ConfigDict, Field+++class WorkflowCopilotChat(BaseModel):+ model_config = ConfigDict(from_attributes=True)++ workflow_copilot_chat_id: str = Field(..., description="ID for the workflow copilot chat")+ organization_id: str = Field(..., description="Organization ID for the chat")+ workflow_permanent_id: str = Field(..., description="Workflow permanent ID for the chat")+ created_at: datetime = Field(..., description="When the chat was created")+ modified_at: datetime = Field(..., description="When the chat was last modified")+++class WorkflowCopilotChatSender(StrEnum):+ USER = "user"+ AI = "ai"+++class WorkflowCopilotChatMessage(BaseModel):+ model_config = ConfigDict(from_attributes=True)++ workflow_copilot_chat_message_id: str = Field(..., description="ID for the workflow copilot chat message")+ workflow_copilot_chat_id: str = Field(..., description="ID of the parent workflow copilot chat")+ sender: WorkflowCopilotChatSender = Field(..., description="Message sender")+ content: str = Field(..., description="Message content")+ global_llm_context: str | None = Field(None, description="Optional global LLM context for the message")+ created_at: datetime = Field(..., description="When the message was created")+ modified_at: datetime = Field(..., description="When the message was last modified")+++class WorkflowCopilotChatRequest(BaseModel):+ workflow_permanent_id: str = Field(..., description="Workflow permanent ID for the chat")+ workflow_copilot_chat_id: str | None = Field(None, description="The chat ID to send the message to")+ workflow_run_id: str | None = Field(None, description="The workflow run ID to use for the context")+ message: str = Field(..., description="The message that user sends")+ workflow_yaml: str = Field(..., description="Current workflow YAML including unsaved changes")+++class WorkflowCopilotChatResponse(BaseModel):+ workflow_copilot_chat_id: str = Field(..., description="The chat ID")+ message: str = Field(..., description="The message sent to the user")+ updated_workflow_yaml: str | None = Field(None, description="The updated workflow yaml")+ request_time: datetime = Field(..., description="When the request was received")+ response_time: datetime = Field(..., description="When the assistant message was created")+++class WorkflowCopilotChatHistoryMessage(BaseModel):+ sender: WorkflowCopilotChatSender = Field(..., description="Message sender")+ content: str = Field(..., description="Message content")+ created_at: datetime = Field(..., description="When the message was created")+++class WorkflowCopilotChatHistoryResponse(BaseModel):+ workflow_copilot_chat_id: str | None = Field(None, description="Latest chat ID for the workflow")+ chat_history: list[WorkflowCopilotChatHistoryMessage] = Field(default_factory=list, description="Chat messages")