Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-9352): copilot asks for sample value before testing parameterized workflows (#5733)
Andrew Neilson89400b0dad78
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2index 79001b58b..10b4da02d 100644--- a/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2@@ -149,6 +149,14 @@ PASSING RUNTIME VALUES WHEN TESTING:* For credential-shaped values (password, secret, token, api_key, credential, totp/otp/one_time_code, private_key, auth), follow the CREDENTIAL HANDLING refusal rule above instead of routing them through `parameters`.* Creating new credentials from inline values is not available to you. If the user wants a reusable credential, tell them to create one in the UI.+PARAMETERIZED REQUESTS WITHOUT A SAMPLE VALUE:+* Defer to CREDENTIAL HANDLING first. If the request contains a raw credential, follow the credential-safety refusal flow above. If the request requires authentication and you have no `credential_id`, run `list_credentials` (with paging) before asking about a sample value. Only when no credential issue applies does this section take over.+* If the user's request signals a parameterized workflow ("for any X we input/provide", "for each Y", "for every Z", "for a list of Ns") but does NOT include a concrete value to test the workflow against, you MUST:+ 1. Decide which runtime parameter(s) the workflow needs and pick descriptive keys.+ 2. Respond with `ASK_QUESTION` on this turn — do NOT call any block-running, navigation, or page-exploration tool first. Name the parameter(s) you propose and ask for one sample value (per parameter) you can use to verify the workflow end-to-end. Example: "I'll build a parameterized workflow that takes `applicant_name`. What sample person should I use to test it?"+ 3. Do NOT loop exploring the page or testing with invented placeholders to discover what the parameter values should be — without a real value the form returns no results, every test reads as suspicious-success or null-data, and the budget burns without producing a committed workflow.+ 4. Once the user provides the sample, build the parameterized workflow, test it with the sample passed via `parameters`, and finalize.+WORKFLOW SUBMISSION RULE:Whenever you build or modify a workflow, you MUST call the `update_workflow` tool with the complete workflow YAML. Do NOT embed workflow YAML in your final JSON response. If the tool returns a validation error, fix the YAML and call `update_workflow` again. Only move to your final response after the tool succeeds.diff --git a/skyvern/forge/prompts/skyvern/workflow_knowledge_base.txt b/skyvern/forge/prompts/skyvern/workflow_knowledge_base.txtindex 81a44a45e..94dabe054 100644--- a/skyvern/forge/prompts/skyvern/workflow_knowledge_base.txt+++ b/skyvern/forge/prompts/skyvern/workflow_knowledge_base.txt@@ -7,7 +7,7 @@ This document provides comprehensive information about Skyvern Workflow YAML strA Skyvern workflow is defined in YAML format with the following top-level structurefor a workflow definition (embedded under workflow_definition in full specs):-title: "<workflow title>"+title: "<workflow title>" # Requireddescription: "<optional description>"workflow_definition:version: 2 # IMPORTANT: Always use version 2diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.pyindex c389dcf9d..40461bb48 100644--- a/skyvern/forge/sdk/routes/workflow_copilot.py+++ b/skyvern/forge/sdk/routes/workflow_copilot.py@@ -889,6 +889,8 @@ def _normalize_copilot_yaml(workflow_yaml: str) -> WorkflowCreateYAMLRequest:# Fixing trivial common LLM mistakes; non-dict YAML falls through to model_validate.if isinstance(parsed_yaml, dict):+ # title is schema-required; coerce rather than force a self-healing round-trip.+ parsed_yaml.setdefault("title", "")workflow_definition = parsed_yaml.get("workflow_definition", None)if workflow_definition:blocks = workflow_definition.get("blocks", []) or []diff --git a/tests/unit/test_workflow_copilot_prompt_injection.py b/tests/unit/test_workflow_copilot_prompt_injection.pyindex 09410718b..f9a75524c 100644--- a/tests/unit/test_workflow_copilot_prompt_injection.py+++ b/tests/unit/test_workflow_copilot_prompt_injection.py@@ -258,6 +258,24 @@ class TestAgentTemplateSecurity:assert "DEBUGGER RUN INFORMATION:" not in rendered+class TestAgentTemplateParameterizedRequestsRule:+ def test_agent_template_has_parameterized_requests_section(self) -> None:+ rendered = prompt_engine.load_prompt(+ "workflow-copilot-agent",+ **_AGENT_TEMPLATE_DEFAULTS,+ )+ assert "PARAMETERIZED REQUESTS WITHOUT A SAMPLE VALUE:" in rendered++ def test_agent_template_keeps_sample_anchor_word(self) -> None:+ # Eval cases with `clarification_must_mention=["sample"]` silently break+ # if this word is dropped from the prompt; assertion surfaces it locally.+ rendered = prompt_engine.load_prompt(+ "workflow-copilot-agent",+ **_AGENT_TEMPLATE_DEFAULTS,+ )+ assert "sample" in rendered.lower()++class TestBuildSystemPromptSecurityRules:"""Verify _build_system_prompt passes security_rules through to the rendered prompt."""diff --git a/tests/unit/test_workflow_copilot_route_helpers.py b/tests/unit/test_workflow_copilot_route_helpers.pyindex f9674108d..fa160e6da 100644--- a/tests/unit/test_workflow_copilot_route_helpers.py+++ b/tests/unit/test_workflow_copilot_route_helpers.py@@ -12,11 +12,11 @@ from __future__ import annotationsfrom unittest.mock import MagicMock+from skyvern.forge.sdk.routes.workflow_copilot import _normalize_copilot_yaml, _should_restore_persisted_workflow+class TestShouldRestorePersistedWorkflow:def test_restores_for_non_auto_accept_and_persisted_workflow(self) -> None:- from skyvern.forge.sdk.routes.workflow_copilot import _should_restore_persisted_workflow-agent_result = MagicMock()agent_result.workflow_was_persisted = True@@ -24,8 +24,6 @@ class TestShouldRestorePersistedWorkflow:assert _should_restore_persisted_workflow(None, agent_result) is Truedef test_does_not_restore_for_auto_accept_or_unpersisted_result(self) -> None:- from skyvern.forge.sdk.routes.workflow_copilot import _should_restore_persisted_workflow-persisted = MagicMock()persisted.workflow_was_persisted = Truenot_persisted = MagicMock()@@ -34,3 +32,15 @@ class TestShouldRestorePersistedWorkflow:assert _should_restore_persisted_workflow(True, persisted) is Falseassert _should_restore_persisted_workflow(False, not_persisted) is Falseassert _should_restore_persisted_workflow(False, None) is False+++class TestNormalizeCopilotYamlTitleCoercion:+ def test_missing_top_level_title_is_coerced_to_empty(self) -> None:+ yaml_str = "workflow_definition:\n blocks: []\n parameters: []\n"+ request = _normalize_copilot_yaml(yaml_str)+ assert request.title == ""++ def test_explicit_top_level_title_is_preserved(self) -> None:+ yaml_str = "title: My Workflow\nworkflow_definition:\n blocks: []\n parameters: []\n"+ request = _normalize_copilot_yaml(yaml_str)+ assert request.title == "My Workflow"