Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Fix first compose coverage guard (#SKY-11287) (#6776)
Andrew Neilsonbc4056fa22af
diff --git a/skyvern/forge/sdk/copilot/tools/__init__.py b/skyvern/forge/sdk/copilot/tools/__init__.pyindex 8da1f05bc..2b144f183 100644--- a/skyvern/forge/sdk/copilot/tools/__init__.py+++ b/skyvern/forge/sdk/copilot/tools/__init__.py@@ -244,7 +244,6 @@ from .workflow_update import CodeArtifactMetadata as CodeArtifactMetadatafrom .workflow_update import _code_artifact_metadata_as_tool_argument as _code_artifact_metadata_as_tool_argumentfrom .workflow_update import _code_block_safety_errors as _code_block_safety_errorsfrom .workflow_update import _normalize_code_artifact_metadata as _normalize_code_artifact_metadata-from .workflow_update import _pre_run_workflow_coverage_error as _pre_run_workflow_coverage_errorfrom .workflow_update import _record_workflow_proxy_location_span as _record_workflow_proxy_location_spanfrom .workflow_update import _record_workflow_update_result as _record_workflow_update_resultfrom .workflow_update import _update_workflow as _update_workflow@@ -668,34 +667,6 @@ async def update_and_run_blocks_tool(sanitized = sanitize_tool_result_for_llm("update_workflow", update_result)return json.dumps(sanitized)- coverage_error = _pre_run_workflow_coverage_error(copilot_ctx)- if coverage_error:- user_facing_summary = (- "Workflow draft saved; I still need to add the remaining requested actions before testing it."- )- result = {- "ok": False,- "error": coverage_error,- "data": {- "block_count": copilot_ctx.last_update_block_count,- "workflow_updated": True,- "workflow_run_skipped": True,- "control_signal": {- "kind": "intermediate_success",- "user_facing_summary": user_facing_summary,- },- "user_facing_summary": user_facing_summary,- },- }- record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, result)- _record_diagnosis_repair_contract(- copilot_ctx,- source_tool="update_and_run_blocks",- result=result,- workflow_updated=True,- )- return json.dumps(result)-if skip_run_after_update:skip_message = "Skipped test run: required credentials are not configured."skip_result = {diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.pyindex 9c7bcff57..3cea0daef 100644--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py@@ -54,9 +54,6 @@ from skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.copilot.enforcement import (MAX_CODE_AUTHORING_GUARDRAIL_REJECTS,MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS,- POST_INTERMEDIATE_SUCCESS_NUDGE,- _completion_contract_unknown_due_to_policy_fallback,- _goal_likely_needs_more_blocks,code_authoring_churn_stop_signal,credential_priority_authoring_churn_stop_signal,)@@ -93,7 +90,6 @@ from ._shared import (_enum_or_string_name,_proxy_location_trace_value,_raw_yaml_proxy_location,- _workflow_definition_as_dict,)from .banned_blocks import (_banned_block_reject_message,@@ -3552,47 +3548,3 @@ def _record_workflow_update_result(clear_failed_step_tracker_for_tools_in_ctx(copilot_ctx, BLOCK_RUNNING_TOOLS)_invalidate_verified_state_on_edit(copilot_ctx, prior_definition, getattr(wf, "workflow_definition", None))---def _last_update_is_single_goto_bootstrap(copilot_ctx: CopilotContext) -> bool:- last_workflow = copilot_ctx.last_workflow- definition = _workflow_definition_as_dict(last_workflow.workflow_definition if last_workflow is not None else None)- blocks = definition.get("blocks")- if not isinstance(blocks, list) or len(blocks) != 1:- return False- block = blocks[0]- if not isinstance(block, dict):- return False- return str(block.get("block_type") or "").strip().lower() == "goto_url"---def _pre_run_workflow_coverage_error(copilot_ctx: Any) -> str | None:- block_count = getattr(copilot_ctx, "last_update_block_count", None)- if not isinstance(block_count, int):- return None- if block_count == 1 and _last_update_is_single_goto_bootstrap(copilot_ctx):- return None-- user_message = getattr(copilot_ctx, "user_message", "")- request_policy = getattr(copilot_ctx, "request_policy", None)- completion_contract = getattr(request_policy, "completion_contract", None)- if isinstance(completion_contract, str):- completion_contract = completion_contract.strip() or None- else:- completion_contract = None-- if _completion_contract_unknown_due_to_policy_fallback(copilot_ctx):- return None-- if not _goal_likely_needs_more_blocks(user_message, block_count, completion_contract):- return None-- nudge_count = getattr(copilot_ctx, "coverage_nudge_count", 0)- if nudge_count >= 1:- return None- copilot_ctx.coverage_nudge_count = nudge_count + 1- return (- f"{POST_INTERMEDIATE_SUCCESS_NUDGE} The workflow was saved with {block_count} block"- f"{'' if block_count == 1 else 's'}, but it has not been run because the request-policy "- "completion contract still leaves distinct requested actions uncovered."- )diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex 466406f9f..d329fb231 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -140,72 +140,6 @@ class TestFailedTestResponseNormalization:assert "test failed" in rewritten.lower()assert "Call log:" not in rewritten- def test_pre_run_coverage_guard_uses_completion_contract(self) -> None:- from skyvern.forge.sdk.copilot.tools import _pre_run_workflow_coverage_error-- ctx = _ctx(- user_message="Go to https://the-internet.herokuapp.com/download and then download the first file.",- request_policy=SimpleNamespace(completion_contract="complete when the download starts"),- last_update_block_count=1,- coverage_nudge_count=0,- )-- error = _pre_run_workflow_coverage_error(ctx)-- assert error is not None- assert "has not been run" in error- assert ctx.coverage_nudge_count == 1-- def test_pre_run_coverage_guard_skips_unknown_completion_contract(self) -> None:- from skyvern.forge.sdk.copilot.tools import _pre_run_workflow_coverage_error-- ctx = _ctx(- user_message="Go to https://example.com/contact. Fill out the contact form and submit it.",- request_policy=SimpleNamespace(completion_contract_status="unknown"),- last_update_block_count=1,- coverage_nudge_count=0,- )-- assert _pre_run_workflow_coverage_error(ctx) is None- assert ctx.coverage_nudge_count == 0-- def test_pre_run_coverage_guard_allows_single_final_action_contract(self) -> None:- from skyvern.forge.sdk.copilot.tools import _pre_run_workflow_coverage_error-- ctx = _ctx(- user_message="Click Delete. Your goal is complete when the Delete button disappears.",- request_policy=SimpleNamespace(completion_contract="complete when the Delete button disappears"),- last_update_block_count=1,- coverage_nudge_count=0,- )-- assert _pre_run_workflow_coverage_error(ctx) is None- assert ctx.coverage_nudge_count == 0-- def test_pre_run_coverage_guard_allows_single_goto_bootstrap(self) -> None:- from skyvern.forge.sdk.copilot.tools import _pre_run_workflow_coverage_error-- ctx = _ctx(- user_message="Go to example.com, fill the search form, and extract the results.",- request_policy=SimpleNamespace(completion_contract="complete when result rows are extracted"),- last_update_block_count=1,- last_workflow=SimpleNamespace(- workflow_definition={- "blocks": [- {- "label": "open_search_page",- "block_type": "goto_url",- "url": "https://example.com/search",- }- ]- }- ),- coverage_nudge_count=0,- )-- assert _pre_run_workflow_coverage_error(ctx) is None- assert ctx.coverage_nudge_count == 0-def test_failed_run_does_not_clear_last_workflow_state(self) -> None:from skyvern.forge.sdk.copilot.tools import _record_run_blocks_result@@ -1531,7 +1465,7 @@ workflow_definition:label: submitnavigation_goal: Submit the contact form."""- captured: dict[str, str] = {}+ captured: dict[str, str | bool] = {}async def fake_update_workflow(payload, ctx, allow_missing_credentials=False):captured["workflow_yaml"] = payload["workflow_yaml"]@@ -1545,6 +1479,7 @@ workflow_definition:return {"ok": True, "_workflow": workflow, "data": {"block_count": 1}}async def fake_run_blocks(params, ctx, **kwargs):+ captured["run_called"] = Truereturn {"ok": True,"data": {@@ -1559,15 +1494,15 @@ workflow_definition:monkeypatch.setattr(tools_module, "_tool_loop_error", lambda *args, **kwargs: None)monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", AsyncMock(return_value=None))monkeypatch.setattr(tools_module, "_update_workflow", fake_update_workflow)- monkeypatch.setattr(tools_module, "_pre_run_workflow_coverage_error", lambda *args: None)monkeypatch.setattr(tools_module, "_plan_frontier", lambda *args: (["submit"], {}, "submit"))monkeypatch.setattr(tools_module, "_run_blocks_and_collect_debug", fake_run_blocks)monkeypatch.setattr(tools_module, "_record_diagnosis_repair_contract", lambda *args, **kwargs: None)monkeypatch.setattr(tools_module, "enqueue_screenshot_from_result", lambda *args, **kwargs: None)ctx = _ctx(- user_message="Submit a contact form.",+ user_message="Go to https://the-internet.herokuapp.com/download and then download the first file.",block_goal_main_goal="Submit a contact form.",+ request_policy=RequestPolicy(completion_contract="complete when the download starts"),)result = await tools_module.update_and_run_blocks_tool.on_invoke_tool(SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"),@@ -1576,6 +1511,8 @@ workflow_definition:assert json.loads(result)["ok"] is Trueassert captured["workflow_yaml"] == clean_yaml+ assert captured["run_called"] is True+ assert ctx.coverage_nudge_count == 0assert "Achieve the following mini goal" not in captured["workflow_yaml"]@@ -1844,15 +1781,10 @@ workflow_definition:assert agent_result.workflow_yaml is Nonedef test_inline_replace_with_invalid_yaml_keeps_prior_pass(self, monkeypatch) -> None:- # _process_workflow_yaml raising on a malformed REPLACE must leave- # ctx untouched — no spurious last_test_ok reset, no workflow swap —- # so a prior tested workflow remains available.- import yaml as yaml_mod-tested_wf = SimpleNamespace(name="tested")def boom(**kwargs):- raise yaml_mod.YAMLError("mangled yaml")+ raise yaml.YAMLError("mangled yaml")monkeypatch.setattr("skyvern.forge.sdk.copilot.tools._process_workflow_yaml", boom)@@ -4619,7 +4551,6 @@ class TestRunBlocksCredentialApproval:monkeypatch.setattr(tools_module, "_tool_loop_error", lambda *args, **kwargs: None)monkeypatch.setattr(tools_module, "_update_and_run_blocks_composition_evidence_precheck", lambda *args: None)monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", AsyncMock(return_value=None))- monkeypatch.setattr(tools_module, "_pre_run_workflow_coverage_error", lambda *args: None)monkeypatch.setattr(tools_module, "_plan_frontier", lambda *args: (["login"], {}, "login"))monkeypatch.setattr(tools_module, "_frontier_run_size_error", lambda *args: None)monkeypatch.setattr(tools_module, "_verify_and_record_run_blocks_result", AsyncMock(return_value=None))diff --git a/tests/unit/test_copilot_output_policy.py b/tests/unit/test_copilot_output_policy.pyindex 23261fafb..7feb737a8 100644--- a/tests/unit/test_copilot_output_policy.py+++ b/tests/unit/test_copilot_output_policy.py@@ -1837,7 +1837,6 @@ workflow_definition:monkeypatch.setattr(tools_module, "_tool_loop_error", lambda *args, **kwargs: None)monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", fake_prior_definition)monkeypatch.setattr(tools_module, "_update_workflow", fake_update_workflow)- monkeypatch.setattr(tools_module, "_pre_run_workflow_coverage_error", lambda *args: None)monkeypatch.setattr(tools_module, "_plan_frontier", lambda *args: (["open_results"], {}, "open_results"))monkeypatch.setattr(tools_module, "_frontier_run_size_error", lambda *args: None)monkeypatch.setattr(tools_module, "_run_blocks_and_collect_debug", fake_run_blocks)