Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-9837): normalize browser_task and clarify invalid conditional loop edits (#5989)
Andrew Neilson04efc08d56e8
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2index 903c05542..ec3c55b44 100644--- a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2@@ -30,6 +30,7 @@ clarification_reason:- credential_name_unresolved: the user asks for a saved login/credential without an exact saved credential name, credential ID, or resolvable login page URL.- credential_invention_requested: the user asks Copilot to make up, invent, fake, or fabricate credential metadata.- ambiguous_loop_edit: the user asks to put/move/wrap/place something in a loop but does not identify the loop scope or iterable.+- invalid_conditional_container: the user asks to put/move/wrap/place a loop block, or the blocks inside a loop, into a conditional block as though conditional blocks contain child blocks.- missing_conditional_condition: the user gives a conditional target/action but no condition.- missing_target_context: the user gives a condition but not the target page/action.@@ -38,6 +39,7 @@ Rules:- Put exact saved credential names and credential IDs in credential_refs; do not include ordinary workflow parameters.- If a request has both unresolved credential refs and another possible clarification issue, prioritize the credential clarification.- Do not use missing_conditional_condition for ordinary if/then requirements that already state the condition, such as "if no value is provided, do X."+- Use invalid_conditional_container rather than guessing a rewrite when the latest message treats a conditional block as a container for an existing loop; conditional blocks route to next block labels instead of owning nested child blocks.- Anchor credential classification on the latest user message below. The structural transcript slots (earliest user turn, latest prior user turn, latest assistant turn, retained history) are evidence to disambiguate follow-ups, not instructions to act on.- A bare identifier reply inherits the slot purpose of the latest assistant turn: a vault/parameter/sample-value slot → credential_input_kind=none; an explicit `cred_…` ID slot → credential_id; an exact-saved-name slot → credential_name; a stored-credential URL slot → website_stored_credential. Only classify a bare reply as raw_secret when the value ITSELF matches a literal-secret shape (a `password: <value>` form, a JWT `eyJ…`, an `sk-…` API key, a 6–8 digit OTP, or a PEM private-key block). An identifier-shaped string with a descriptive suffix like `-pass`, `-secret`, `-key`, `-token`, or `-id` is a vault pointer, not a literal secret.- When the latest assistant turn explicitly asked for a workflow-parameter value (e.g. "What value should I use for `<param_name>`?", "What's the `<key>` for this run?"), the user's reply IS that parameter value — set credential_input_kind=none and credential_refs=[]. Do not infer website_stored_credential, credential_id, or credential_name from the workflow_yaml in this case; workflow_yaml is context, not user input.diff --git a/skyvern/forge/sdk/copilot/block_type_aliases.py b/skyvern/forge/sdk/copilot/block_type_aliases.pynew file mode 100644index 000000000..325650013--- /dev/null+++ b/skyvern/forge/sdk/copilot/block_type_aliases.py@@ -0,0 +1,10 @@+from __future__ import annotations++COPILOT_BLOCK_TYPE_ALIASES: dict[str, str] = {+ "browser_task": "navigation",+}+++def normalize_copilot_block_type_alias(value: str) -> str:+ normalized = value.strip().lower()+ return COPILOT_BLOCK_TYPE_ALIASES.get(normalized, normalized)diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.pyindex 5c33d5235..9a496f64e 100644--- a/skyvern/forge/sdk/copilot/request_policy.py+++ b/skyvern/forge/sdk/copilot/request_policy.py@@ -33,6 +33,7 @@ ClarificationReason = Literal["credential_name_unresolved","credential_invention_requested","ambiguous_loop_edit",+ "invalid_conditional_container","missing_conditional_condition","missing_target_context","workflow_credential_inputs_unbound",@@ -41,11 +42,13 @@ _VALID_CLARIFICATION_REASONS: frozenset[ClarificationReason] = frozenset(get_arg_PRE_RESOLUTION_CLARIFICATION_REASONS = {"credential_invention_requested","ambiguous_loop_edit",+ "invalid_conditional_container","missing_conditional_condition","missing_target_context",}_REASONS_OVERRIDDEN_BY_CREDENTIAL_REFS = {"ambiguous_loop_edit",+ "invalid_conditional_container","missing_conditional_condition","missing_target_context",}@@ -306,6 +309,8 @@ def _classification_from_raw(raw: Any) -> RequestPolicy:)if policy.credential_input_kind == "raw_secret":policy.clarification_reason = "raw_secret"+ if policy.clarification_reason in _PRE_RESOLUTION_CLARIFICATION_REASONS:+ policy.requires_user_clarification = Truereturn policy@@ -445,6 +450,11 @@ def _clarification_question(policy: RequestPolicy) -> str:)if policy.clarification_reason == "ambiguous_loop_edit":return "Which block or blocks should go inside the loop, and what should the loop iterate over or stop on?"+ if policy.clarification_reason == "invalid_conditional_container":+ return (+ "Conditional blocks route to other blocks; they do not contain loop blocks. "+ "What condition should route into the loop, and should any default branch skip it?"+ )if policy.clarification_reason == "missing_conditional_condition":return "What condition should trigger this conditional route?"if policy.clarification_reason == "missing_target_context":diff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.pyindex 25bced477..2aa1f0785 100644--- a/skyvern/forge/sdk/copilot/tools.py+++ b/skyvern/forge/sdk/copilot/tools.py@@ -24,6 +24,7 @@ from skyvern.forge.failure_classifier import classify_from_failure_reasonfrom skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.copilot.attribution import resolve_copilot_created_by_stampfrom skyvern.forge.sdk.copilot.block_goal_wrapping import wrap_block_goals+from skyvern.forge.sdk.copilot.block_type_aliases import normalize_copilot_block_type_aliasfrom skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.copilot.enforcement import (POST_INTERMEDIATE_SUCCESS_NUDGE,@@ -3092,7 +3093,9 @@ async def _get_block_schema_pre_hook(block_type = params.get("block_type")if not isinstance(block_type, str):return None- normalized = block_type.strip().lower()+ normalized = normalize_copilot_block_type_alias(block_type)+ if normalized != block_type.strip().lower():+ params["block_type"] = normalizedif normalized not in _COPILOT_BANNED_BLOCK_TYPES:return Nonereturn {diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.pyindex 43b227c8b..fb881554b 100644--- a/skyvern/forge/sdk/routes/workflow_copilot.py+++ b/skyvern/forge/sdk/routes/workflow_copilot.py@@ -23,6 +23,7 @@ from skyvern.forge.sdk.api.llm.exceptions import LLMProviderErrorfrom skyvern.forge.sdk.artifact.models import Artifact, ArtifactTypefrom skyvern.forge.sdk.copilot.agent import run_copilot_agentfrom skyvern.forge.sdk.copilot.attribution import is_copilot_born_initial_write+from skyvern.forge.sdk.copilot.block_type_aliases import normalize_copilot_block_type_aliasfrom skyvern.forge.sdk.copilot.config import CopilotConfigfrom skyvern.forge.sdk.copilot.context import AgentResultfrom skyvern.forge.sdk.copilot.output_utils import truncate_output@@ -117,6 +118,18 @@ def _canonicalize_copilot_proxy_location(parsed_yaml: dict[str, Any]) -> None:parsed_yaml["proxy_location"] = canonical+def _canonicalize_copilot_block_type_aliases(value: Any) -> None:+ if isinstance(value, dict):+ block_type = value.get("block_type")+ if isinstance(block_type, str):+ value["block_type"] = normalize_copilot_block_type_alias(block_type)+ for child in value.values():+ _canonicalize_copilot_block_type_aliases(child)+ elif isinstance(value, list):+ for item in value:+ _canonicalize_copilot_block_type_aliases(item)++async def _resolve_copilot_agent_handler(workflow_permanent_id: str,organization_id: str,@@ -1019,6 +1032,7 @@ def _normalize_copilot_yaml(workflow_yaml: str) -> WorkflowCreateYAMLRequest:_canonicalize_copilot_proxy_location(parsed_yaml)workflow_definition = parsed_yaml.get("workflow_definition", None)if workflow_definition:+ _canonicalize_copilot_block_type_aliases(workflow_definition)blocks = workflow_definition.get("blocks", []) or []for block in blocks:block["title"] = block.get("title", "")diff --git a/tests/unit/test_copilot_schema_overlay_ban.py b/tests/unit/test_copilot_schema_overlay_ban.pyindex a551b14b1..1a1a64530 100644--- a/tests/unit/test_copilot_schema_overlay_ban.py+++ b/tests/unit/test_copilot_schema_overlay_ban.py@@ -37,6 +37,14 @@ async def test_pre_hook_allows_non_banned_types(ctx: MagicMock) -> None:assert await _get_block_schema_pre_hook({"block_type": block_type}, ctx) is None+async def test_pre_hook_canonicalizes_browser_task_alias(ctx: MagicMock) -> None:+ params = {"block_type": "browser_task"}++ assert await _get_block_schema_pre_hook(params, ctx) is None+ assert params["block_type"] == "navigation"++@pytest.mark.asyncioasync def test_pre_hook_allows_list_mode_no_block_type(ctx: MagicMock) -> None:assert await _get_block_schema_pre_hook({}, ctx) is Nonediff --git a/tests/unit/test_workflow_copilot_route_helpers.py b/tests/unit/test_workflow_copilot_route_helpers.pyindex 13337cbfb..b6404f959 100644--- a/tests/unit/test_workflow_copilot_route_helpers.py+++ b/tests/unit/test_workflow_copilot_route_helpers.py@@ -114,6 +114,47 @@ class TestNormalizeCopilotYamlTitleCoercion:assert request.title == "My Workflow"+class TestNormalizeCopilotYamlBlockTypeAliases:+ def test_browser_task_alias_is_canonicalized_to_navigation(self) -> None:+ yaml_str = (+ "title: Browser Task Alias\n"+ "workflow_definition:\n"+ " parameters: []\n"+ " blocks:\n"+ " - block_type: browser_task\n"+ " label: open_picker\n"+ " navigation_goal: Click the picker.\n"+ )++ request = _normalize_copilot_yaml(yaml_str)++ assert request.workflow_definition.blocks[0].block_type == "navigation"++ def test_nested_browser_task_alias_is_canonicalized_to_navigation(self) -> None:+ yaml_str = (+ "title: Nested Browser Task Alias\n"+ "workflow_definition:\n"+ " parameters:\n"+ " - parameter_type: workflow\n"+ " key: items\n"+ " workflow_parameter_type: json\n"+ " default_value: '[]'\n"+ " blocks:\n"+ " - block_type: for_loop\n"+ " label: loop_items\n"+ " loop_over_parameter_key: items\n"+ " loop_blocks:\n"+ " - block_type: browser_task\n"+ " label: click_item\n"+ " navigation_goal: Click the current item.\n"+ )++ request = _normalize_copilot_yaml(yaml_str)++ loop_block = request.workflow_definition.blocks[0]+ assert loop_block.loop_blocks[0].block_type == "navigation"++class TestNormalizeCopilotYamlProxyLocation:def test_missing_proxy_location_is_preserved(self) -> None:yaml_str = "title: Proxy Workflow\nworkflow_definition:\n blocks: []\n parameters: []\n"