Skip to content

Automate browser based workflows with AI

fix(SKY-9352): copilot asks for sample value before testing parameterized workflows (#5733)

Andrew Neilson89400b0dad78
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2
index 79001b58b..10b4da02d 100644
--- a/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2
+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2
@@ -149,6 +149,14 @@ PASSING RUNTIME VALUES WHEN TESTING:
* For credential-shaped values (password, secret, token, api_key, credential, totp/otp/one_time_code, private_key, auth), follow the CREDENTIAL HANDLING refusal rule above instead of routing them through `parameters`.
* Creating new credentials from inline values is not available to you. If the user wants a reusable credential, tell them to create one in the UI.
+PARAMETERIZED REQUESTS WITHOUT A SAMPLE VALUE:
+* Defer to CREDENTIAL HANDLING first. If the request contains a raw credential, follow the credential-safety refusal flow above. If the request requires authentication and you have no `credential_id`, run `list_credentials` (with paging) before asking about a sample value. Only when no credential issue applies does this section take over.
+* If the user's request signals a parameterized workflow ("for any X we input/provide", "for each Y", "for every Z", "for a list of Ns") but does NOT include a concrete value to test the workflow against, you MUST:
+ 1. Decide which runtime parameter(s) the workflow needs and pick descriptive keys.
+ 2. Respond with `ASK_QUESTION` on this turn — do NOT call any block-running, navigation, or page-exploration tool first. Name the parameter(s) you propose and ask for one sample value (per parameter) you can use to verify the workflow end-to-end. Example: "I'll build a parameterized workflow that takes `applicant_name`. What sample person should I use to test it?"
+ 3. Do NOT loop exploring the page or testing with invented placeholders to discover what the parameter values should be — without a real value the form returns no results, every test reads as suspicious-success or null-data, and the budget burns without producing a committed workflow.
+ 4. Once the user provides the sample, build the parameterized workflow, test it with the sample passed via `parameters`, and finalize.
+
WORKFLOW SUBMISSION RULE:
Whenever you build or modify a workflow, you MUST call the `update_workflow` tool with the complete workflow YAML. Do NOT embed workflow YAML in your final JSON response. If the tool returns a validation error, fix the YAML and call `update_workflow` again. Only move to your final response after the tool succeeds.
diff --git a/skyvern/forge/prompts/skyvern/workflow_knowledge_base.txt b/skyvern/forge/prompts/skyvern/workflow_knowledge_base.txt
index 81a44a45e..94dabe054 100644
--- a/skyvern/forge/prompts/skyvern/workflow_knowledge_base.txt
+++ b/skyvern/forge/prompts/skyvern/workflow_knowledge_base.txt
@@ -7,7 +7,7 @@ This document provides comprehensive information about Skyvern Workflow YAML str
A Skyvern workflow is defined in YAML format with the following top-level structure
for a workflow definition (embedded under workflow_definition in full specs):
-title: "<workflow title>"
+title: "<workflow title>" # Required
description: "<optional description>"
workflow_definition:
version: 2 # IMPORTANT: Always use version 2
diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py
index c389dcf9d..40461bb48 100644
--- a/skyvern/forge/sdk/routes/workflow_copilot.py
+++ b/skyvern/forge/sdk/routes/workflow_copilot.py
@@ -889,6 +889,8 @@ def _normalize_copilot_yaml(workflow_yaml: str) -> WorkflowCreateYAMLRequest:
# Fixing trivial common LLM mistakes; non-dict YAML falls through to model_validate.
if isinstance(parsed_yaml, dict):
+ # title is schema-required; coerce rather than force a self-healing round-trip.
+ parsed_yaml.setdefault("title", "")
workflow_definition = parsed_yaml.get("workflow_definition", None)
if workflow_definition:
blocks = workflow_definition.get("blocks", []) or []
diff --git a/tests/unit/test_workflow_copilot_prompt_injection.py b/tests/unit/test_workflow_copilot_prompt_injection.py
index 09410718b..f9a75524c 100644
--- a/tests/unit/test_workflow_copilot_prompt_injection.py
+++ b/tests/unit/test_workflow_copilot_prompt_injection.py
@@ -258,6 +258,24 @@ class TestAgentTemplateSecurity:
assert "DEBUGGER RUN INFORMATION:" not in rendered
+class TestAgentTemplateParameterizedRequestsRule:
+ def test_agent_template_has_parameterized_requests_section(self) -> None:
+ rendered = prompt_engine.load_prompt(
+ "workflow-copilot-agent",
+ **_AGENT_TEMPLATE_DEFAULTS,
+ )
+ assert "PARAMETERIZED REQUESTS WITHOUT A SAMPLE VALUE:" in rendered
+
+ def test_agent_template_keeps_sample_anchor_word(self) -> None:
+ # Eval cases with `clarification_must_mention=["sample"]` silently break
+ # if this word is dropped from the prompt; assertion surfaces it locally.
+ rendered = prompt_engine.load_prompt(
+ "workflow-copilot-agent",
+ **_AGENT_TEMPLATE_DEFAULTS,
+ )
+ assert "sample" in rendered.lower()
+
+
class TestBuildSystemPromptSecurityRules:
"""Verify _build_system_prompt passes security_rules through to the rendered prompt."""
diff --git a/tests/unit/test_workflow_copilot_route_helpers.py b/tests/unit/test_workflow_copilot_route_helpers.py
index f9674108d..fa160e6da 100644
--- a/tests/unit/test_workflow_copilot_route_helpers.py
+++ b/tests/unit/test_workflow_copilot_route_helpers.py
@@ -12,11 +12,11 @@ from __future__ import annotations
from unittest.mock import MagicMock
+from skyvern.forge.sdk.routes.workflow_copilot import _normalize_copilot_yaml, _should_restore_persisted_workflow
+
class TestShouldRestorePersistedWorkflow:
def test_restores_for_non_auto_accept_and_persisted_workflow(self) -> None:
- from skyvern.forge.sdk.routes.workflow_copilot import _should_restore_persisted_workflow
-
agent_result = MagicMock()
agent_result.workflow_was_persisted = True
@@ -24,8 +24,6 @@ class TestShouldRestorePersistedWorkflow:
assert _should_restore_persisted_workflow(None, agent_result) is True
def test_does_not_restore_for_auto_accept_or_unpersisted_result(self) -> None:
- from skyvern.forge.sdk.routes.workflow_copilot import _should_restore_persisted_workflow
-
persisted = MagicMock()
persisted.workflow_was_persisted = True
not_persisted = MagicMock()
@@ -34,3 +32,15 @@ class TestShouldRestorePersistedWorkflow:
assert _should_restore_persisted_workflow(True, persisted) is False
assert _should_restore_persisted_workflow(False, not_persisted) is False
assert _should_restore_persisted_workflow(False, None) is False
+
+
+class TestNormalizeCopilotYamlTitleCoercion:
+ def test_missing_top_level_title_is_coerced_to_empty(self) -> None:
+ yaml_str = "workflow_definition:\n blocks: []\n parameters: []\n"
+ request = _normalize_copilot_yaml(yaml_str)
+ assert request.title == ""
+
+ def test_explicit_top_level_title_is_preserved(self) -> None:
+ yaml_str = "title: My Workflow\nworkflow_definition:\n blocks: []\n parameters: []\n"
+ request = _normalize_copilot_yaml(yaml_str)
+ assert request.title == "My Workflow"