Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-10317): classify blank-workflow browser-task copilot turns as build (#6186)
Andrew Neilson2fb4ef8a5154
diff --git a/skyvern/forge/sdk/copilot/turn_intent.py b/skyvern/forge/sdk/copilot/turn_intent.pyindex 8bb6daacd..71af1fef2 100644--- a/skyvern/forge/sdk/copilot/turn_intent.py+++ b/skyvern/forge/sdk/copilot/turn_intent.py@@ -171,6 +171,10 @@ _NEW_BROWSER_TASK_TERMS = ("go to", "navigate to", "open", "visit", "search for"_EDIT_TERMS = ("edit", "update", "change", "modify", "replace", "fix")_DIAGNOSE_TERMS = ("debug", "diagnose", "failed", "failure", "error", "result")_RUN_CONTEXT_REQUEST_TERMS = ("get_run_results", "workflow_run_id", "run result", "run results")+# Subset of _DIAGNOSE_TERMS that name unambiguous diagnose intent. "result" is in+# _DIAGNOSE_TERMS but absent here: a browser task on a no-blocks workflow that+# mentions "results" is a build request, not a run-result inspection.+_CLEAR_DIAGNOSE_TERMS = ("debug", "diagnose", "failed", "failure", "error")_DOCS_TERMS = ("explain","how do",@@ -225,17 +229,32 @@ def _is_explicit_run_context_request(user_message: str) -> bool:return _contains_any(user_message, _RUN_CONTEXT_REQUEST_TERMS)-def _workflow_block_labels(workflow_yaml: str | None) -> set[str]:+def _workflow_definition_dict(workflow_yaml: str | None) -> dict | None:if not workflow_yaml:- return set()+ return Nonetry:parsed = safe_load_no_dates(workflow_yaml)except yaml.YAMLError:- return set()+ return Noneif not isinstance(parsed, dict):- return set()+ return Noneworkflow_definition = parsed.get("workflow_definition")- if not isinstance(workflow_definition, dict):+ return workflow_definition if isinstance(workflow_definition, dict) else None+++def _workflow_has_blocks(workflow_yaml: str | None) -> bool:+ workflow_definition = _workflow_definition_dict(workflow_yaml)+ if workflow_definition is None:+ return False+ blocks = workflow_definition.get("blocks")+ if not isinstance(blocks, list):+ return False+ return any(isinstance(block, dict) for block in blocks)+++def _workflow_block_labels(workflow_yaml: str | None) -> set[str]:+ workflow_definition = _workflow_definition_dict(workflow_yaml)+ if workflow_definition is None:return set()blocks = workflow_definition.get("blocks")if not isinstance(blocks, list):@@ -251,16 +270,8 @@ def _workflow_block_labels(workflow_yaml: str | None) -> set[str]:def _workflow_parameter_keys(workflow_yaml: str | None) -> set[str]:- if not workflow_yaml:- return set()- try:- parsed = safe_load_no_dates(workflow_yaml)- except yaml.YAMLError:- return set()- if not isinstance(parsed, dict):- return set()- workflow_definition = parsed.get("workflow_definition")- if not isinstance(workflow_definition, dict):+ workflow_definition = _workflow_definition_dict(workflow_yaml)+ if workflow_definition is None:return set()parameters = workflow_definition.get("parameters")if not isinstance(parameters, list):@@ -309,10 +320,27 @@ def _unresolved_explicit_block_refs(user_message: str, workflow_yaml: str | Nonedef _mode_from_keywords(- user_message: str, *, has_workflow: bool+ user_message: str,+ *,+ has_workflow: bool,+ has_workflow_blocks: bool,+ has_prior_run_signal: bool,) -> tuple[TurnIntentMode, TurnIntentExpectedOutput] | None:if has_workflow and _contains_any(user_message, _EDIT_TERMS):return TurnIntentMode.EDIT, TurnIntentExpectedOutput.WORKFLOW_UPDATE+ # A browser-task verb on a saved workflow with no buildable blocks is a new+ # build request even when the message mentions "results"; suppress only when+ # the user names diagnosis or docs intent, or a prior run is attached. The+ # no-yaml case still falls through to the trailing browser-task fallback.+ if (+ has_workflow+ and not has_workflow_blocks+ and not has_prior_run_signal+ and _contains_any(user_message, _NEW_BROWSER_TASK_TERMS)+ and not _contains_any(user_message, _CLEAR_DIAGNOSE_TERMS)+ and not _contains_any(user_message, _DOCS_TERMS)+ ):+ return TurnIntentMode.BUILD, TurnIntentExpectedOutput.WORKFLOW_DRAFTif _contains_any(user_message, _DIAGNOSE_TERMS):return TurnIntentMode.DIAGNOSE, TurnIntentExpectedOutput.RUN_RESULTif _contains_any(user_message, _DOCS_TERMS):@@ -447,7 +475,11 @@ def _user_signals_non_progress(user_message: str, chat_history: list[WorkflowCopdef _carryover_mode_from_prior_turn(- chat_history: list[WorkflowCopilotChatHistoryMessage], *, has_workflow: bool+ chat_history: list[WorkflowCopilotChatHistoryMessage],+ *,+ has_workflow: bool,+ has_workflow_blocks: bool,+ has_prior_run_signal: bool,) -> tuple[TurnIntentMode, TurnIntentExpectedOutput] | None:"""Mode of the most recent prior user turn that keyword-classifies. A bareconfirmation carries no keywords of its own, so it inherits the turn it confirms."""@@ -459,7 +491,12 @@ def _carryover_mode_from_prior_turn(# back to the substantive request, not to the nearest confirmation.if not content or _is_bare_affirmative(content):continue- if carried := _mode_from_keywords(content, has_workflow=has_workflow):+ if carried := _mode_from_keywords(+ content,+ has_workflow=has_workflow,+ has_workflow_blocks=has_workflow_blocks,+ has_prior_run_signal=has_prior_run_signal,+ ):return carriedreturn None@@ -477,6 +514,7 @@ def build_turn_intent(browser_session_id: str | None = None,) -> TurnIntent:has_workflow = bool((workflow_yaml or "").strip())+ has_workflow_blocks = _workflow_has_blocks(workflow_yaml)has_prior_context = bool((global_llm_context or "").strip())target_entities: dict[str, list[str]] = {}required_context: list[RequiredContextKey] = []@@ -542,12 +580,22 @@ def build_turn_intent(expected_output = TurnIntentExpectedOutput.WORKFLOW_DRAFTconfidence = 0.6reason_codes.append(TurnIntentReasonCode.TESTING_INTENT_SKIP_TEST)- elif keyword_mode := _mode_from_keywords(user_message, has_workflow=has_workflow):+ elif keyword_mode := _mode_from_keywords(+ user_message,+ has_workflow=has_workflow,+ has_workflow_blocks=has_workflow_blocks,+ has_prior_run_signal=has_prior_run_signal,+ ):mode, expected_output = keyword_modeconfidence = 0.35reason_codes.append(TurnIntentReasonCode.KEYWORD_HEURISTIC)elif _is_bare_affirmative(user_message) and (- carried_mode := _carryover_mode_from_prior_turn(chat_history, has_workflow=has_workflow)+ carried_mode := _carryover_mode_from_prior_turn(+ chat_history,+ has_workflow=has_workflow,+ has_workflow_blocks=has_workflow_blocks,+ has_prior_run_signal=has_prior_run_signal,+ )):mode, expected_output = carried_modeconfidence = 0.3diff --git a/tests/unit/test_copilot_turn_intent.py b/tests/unit/test_copilot_turn_intent.pyindex 20d711f33..4951ec4ad 100644--- a/tests/unit/test_copilot_turn_intent.py+++ b/tests/unit/test_copilot_turn_intent.py@@ -203,6 +203,109 @@ def test_build_turn_intent_marks_blank_workflow_browser_task_as_build() -> None:assert intent.authority.may_run_blocks is True+_BLANK_SAVED_WORKFLOW_YAML = "title: Saved blank workflow\nworkflow_definition:\n blocks: []\n parameters: []\n"+++def test_build_turn_intent_marks_blank_saved_workflow_browser_task_with_results_word_as_build() -> None:+ intent = build_turn_intent(+ user_message=(+ "Go to https://example.test/registry, search for ABC-1234, expand the entry, "+ "and report the credential type and expiration date from the results."+ ),+ workflow_yaml=_BLANK_SAVED_WORKFLOW_YAML,+ chat_history=[],+ global_llm_context="",+ request_policy=RequestPolicy(),+ )++ assert intent.mode == TurnIntentMode.BUILD+ assert intent.authority.may_update_workflow is True+ assert intent.authority.may_run_blocks is True+ assert RequiredContextKey.LATEST_RUN_RESULT not in intent.required_context+++def test_build_turn_intent_marks_create_workflow_with_browser_task_and_results_as_build() -> None:+ intent = build_turn_intent(+ user_message=("Create a workflow that opens https://example.test and reports the results."),+ workflow_yaml=_BLANK_SAVED_WORKFLOW_YAML,+ chat_history=[],+ global_llm_context="",+ request_policy=RequestPolicy(),+ )++ assert intent.mode == TurnIntentMode.BUILD+ assert intent.authority.may_update_workflow is True+ assert intent.authority.may_run_blocks is True+ assert RequiredContextKey.LATEST_RUN_RESULT not in intent.required_context+++def test_build_turn_intent_keeps_clear_diagnose_on_blank_saved_workflow_with_browser_task_verb() -> None:+ intent = build_turn_intent(+ user_message="Open the failed page and tell me what went wrong.",+ workflow_yaml=_BLANK_SAVED_WORKFLOW_YAML,+ chat_history=[],+ global_llm_context="",+ request_policy=RequestPolicy(),+ )++ assert intent.mode == TurnIntentMode.DIAGNOSE+++def test_build_turn_intent_keeps_run_result_diagnose_with_build_keyword_on_existing_workflow() -> None:+ existing_workflow_yaml = (+ "title: Existing\nworkflow_definition:\n blocks:\n - block_type: navigation\n label: nav_to_site\n"+ )+ intent = build_turn_intent(+ user_message="Generate a report from the last result.",+ workflow_yaml=existing_workflow_yaml,+ chat_history=[],+ global_llm_context="",+ request_policy=RequestPolicy(),+ )++ assert intent.mode == TurnIntentMode.DIAGNOSE+++def test_build_turn_intent_keeps_run_result_diagnose_when_no_browser_task_verb() -> None:+ intent = build_turn_intent(+ user_message="What is the result of the last run?",+ workflow_yaml=_BLANK_SAVED_WORKFLOW_YAML,+ chat_history=[],+ global_llm_context="",+ request_policy=RequestPolicy(),+ )++ assert intent.mode == TurnIntentMode.DIAGNOSE+++def test_build_turn_intent_keeps_diagnose_when_run_id_attached_on_blank_saved_workflow() -> None:+ intent = build_turn_intent(+ user_message="Open the result.",+ workflow_yaml=_BLANK_SAVED_WORKFLOW_YAML,+ chat_history=[],+ global_llm_context="",+ request_policy=RequestPolicy(),+ workflow_run_id="wr_abc123",+ )++ assert intent.mode == TurnIntentMode.DIAGNOSE+ assert RequiredContextKey.LATEST_RUN_RESULT in intent.required_context+++def test_build_turn_intent_routes_docs_question_with_browser_task_verb_on_blank_saved_workflow() -> None:+ intent = build_turn_intent(+ user_message="How do I navigate to my workflow?",+ workflow_yaml=_BLANK_SAVED_WORKFLOW_YAML,+ chat_history=[],+ global_llm_context="",+ request_policy=RequestPolicy(),+ )++ assert intent.mode == TurnIntentMode.DOCS_ANSWER+ assert intent.authority.may_update_workflow is False+ assert intent.authority.may_run_blocks is False++def test_build_turn_intent_marks_run_context_for_diagnose() -> None:intent = build_turn_intent(user_message="Diagnose the failed run",