Skip to content

Automate browser based workflows with AI

Fix first compose coverage guard (#SKY-11287) (#6776)

Andrew Neilsonbc4056fa22af
diff --git a/skyvern/forge/sdk/copilot/tools/__init__.py b/skyvern/forge/sdk/copilot/tools/__init__.py
index 8da1f05bc..2b144f183 100644
--- a/skyvern/forge/sdk/copilot/tools/__init__.py
+++ b/skyvern/forge/sdk/copilot/tools/__init__.py
@@ -244,7 +244,6 @@ from .workflow_update import CodeArtifactMetadata as CodeArtifactMetadata
from .workflow_update import _code_artifact_metadata_as_tool_argument as _code_artifact_metadata_as_tool_argument
from .workflow_update import _code_block_safety_errors as _code_block_safety_errors
from .workflow_update import _normalize_code_artifact_metadata as _normalize_code_artifact_metadata
-from .workflow_update import _pre_run_workflow_coverage_error as _pre_run_workflow_coverage_error
from .workflow_update import _record_workflow_proxy_location_span as _record_workflow_proxy_location_span
from .workflow_update import _record_workflow_update_result as _record_workflow_update_result
from .workflow_update import _update_workflow as _update_workflow
@@ -668,34 +667,6 @@ async def update_and_run_blocks_tool(
sanitized = sanitize_tool_result_for_llm("update_workflow", update_result)
return json.dumps(sanitized)
- coverage_error = _pre_run_workflow_coverage_error(copilot_ctx)
- if coverage_error:
- user_facing_summary = (
- "Workflow draft saved; I still need to add the remaining requested actions before testing it."
- )
- result = {
- "ok": False,
- "error": coverage_error,
- "data": {
- "block_count": copilot_ctx.last_update_block_count,
- "workflow_updated": True,
- "workflow_run_skipped": True,
- "control_signal": {
- "kind": "intermediate_success",
- "user_facing_summary": user_facing_summary,
- },
- "user_facing_summary": user_facing_summary,
- },
- }
- record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, result)
- _record_diagnosis_repair_contract(
- copilot_ctx,
- source_tool="update_and_run_blocks",
- result=result,
- workflow_updated=True,
- )
- return json.dumps(result)
-
if skip_run_after_update:
skip_message = "Skipped test run: required credentials are not configured."
skip_result = {
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index 9c7bcff57..3cea0daef 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -54,9 +54,6 @@ from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.enforcement import (
MAX_CODE_AUTHORING_GUARDRAIL_REJECTS,
MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS,
- POST_INTERMEDIATE_SUCCESS_NUDGE,
- _completion_contract_unknown_due_to_policy_fallback,
- _goal_likely_needs_more_blocks,
code_authoring_churn_stop_signal,
credential_priority_authoring_churn_stop_signal,
)
@@ -93,7 +90,6 @@ from ._shared import (
_enum_or_string_name,
_proxy_location_trace_value,
_raw_yaml_proxy_location,
- _workflow_definition_as_dict,
)
from .banned_blocks import (
_banned_block_reject_message,
@@ -3552,47 +3548,3 @@ def _record_workflow_update_result(
clear_failed_step_tracker_for_tools_in_ctx(copilot_ctx, BLOCK_RUNNING_TOOLS)
_invalidate_verified_state_on_edit(copilot_ctx, prior_definition, getattr(wf, "workflow_definition", None))
-
-
-def _last_update_is_single_goto_bootstrap(copilot_ctx: CopilotContext) -> bool:
- last_workflow = copilot_ctx.last_workflow
- definition = _workflow_definition_as_dict(last_workflow.workflow_definition if last_workflow is not None else None)
- blocks = definition.get("blocks")
- if not isinstance(blocks, list) or len(blocks) != 1:
- return False
- block = blocks[0]
- if not isinstance(block, dict):
- return False
- return str(block.get("block_type") or "").strip().lower() == "goto_url"
-
-
-def _pre_run_workflow_coverage_error(copilot_ctx: Any) -> str | None:
- block_count = getattr(copilot_ctx, "last_update_block_count", None)
- if not isinstance(block_count, int):
- return None
- if block_count == 1 and _last_update_is_single_goto_bootstrap(copilot_ctx):
- return None
-
- user_message = getattr(copilot_ctx, "user_message", "")
- request_policy = getattr(copilot_ctx, "request_policy", None)
- completion_contract = getattr(request_policy, "completion_contract", None)
- if isinstance(completion_contract, str):
- completion_contract = completion_contract.strip() or None
- else:
- completion_contract = None
-
- if _completion_contract_unknown_due_to_policy_fallback(copilot_ctx):
- return None
-
- if not _goal_likely_needs_more_blocks(user_message, block_count, completion_contract):
- return None
-
- nudge_count = getattr(copilot_ctx, "coverage_nudge_count", 0)
- if nudge_count >= 1:
- return None
- copilot_ctx.coverage_nudge_count = nudge_count + 1
- return (
- f"{POST_INTERMEDIATE_SUCCESS_NUDGE} The workflow was saved with {block_count} block"
- f"{'' if block_count == 1 else 's'}, but it has not been run because the request-policy "
- "completion contract still leaves distinct requested actions uncovered."
- )
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 466406f9f..d329fb231 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -140,72 +140,6 @@ class TestFailedTestResponseNormalization:
assert "test failed" in rewritten.lower()
assert "Call log:" not in rewritten
- def test_pre_run_coverage_guard_uses_completion_contract(self) -> None:
- from skyvern.forge.sdk.copilot.tools import _pre_run_workflow_coverage_error
-
- ctx = _ctx(
- user_message="Go to https://the-internet.herokuapp.com/download and then download the first file.",
- request_policy=SimpleNamespace(completion_contract="complete when the download starts"),
- last_update_block_count=1,
- coverage_nudge_count=0,
- )
-
- error = _pre_run_workflow_coverage_error(ctx)
-
- assert error is not None
- assert "has not been run" in error
- assert ctx.coverage_nudge_count == 1
-
- def test_pre_run_coverage_guard_skips_unknown_completion_contract(self) -> None:
- from skyvern.forge.sdk.copilot.tools import _pre_run_workflow_coverage_error
-
- ctx = _ctx(
- user_message="Go to https://example.com/contact. Fill out the contact form and submit it.",
- request_policy=SimpleNamespace(completion_contract_status="unknown"),
- last_update_block_count=1,
- coverage_nudge_count=0,
- )
-
- assert _pre_run_workflow_coverage_error(ctx) is None
- assert ctx.coverage_nudge_count == 0
-
- def test_pre_run_coverage_guard_allows_single_final_action_contract(self) -> None:
- from skyvern.forge.sdk.copilot.tools import _pre_run_workflow_coverage_error
-
- ctx = _ctx(
- user_message="Click Delete. Your goal is complete when the Delete button disappears.",
- request_policy=SimpleNamespace(completion_contract="complete when the Delete button disappears"),
- last_update_block_count=1,
- coverage_nudge_count=0,
- )
-
- assert _pre_run_workflow_coverage_error(ctx) is None
- assert ctx.coverage_nudge_count == 0
-
- def test_pre_run_coverage_guard_allows_single_goto_bootstrap(self) -> None:
- from skyvern.forge.sdk.copilot.tools import _pre_run_workflow_coverage_error
-
- ctx = _ctx(
- user_message="Go to example.com, fill the search form, and extract the results.",
- request_policy=SimpleNamespace(completion_contract="complete when result rows are extracted"),
- last_update_block_count=1,
- last_workflow=SimpleNamespace(
- workflow_definition={
- "blocks": [
- {
- "label": "open_search_page",
- "block_type": "goto_url",
- "url": "https://example.com/search",
- }
- ]
- }
- ),
- coverage_nudge_count=0,
- )
-
- assert _pre_run_workflow_coverage_error(ctx) is None
- assert ctx.coverage_nudge_count == 0
-
def test_failed_run_does_not_clear_last_workflow_state(self) -> None:
from skyvern.forge.sdk.copilot.tools import _record_run_blocks_result
@@ -1531,7 +1465,7 @@ workflow_definition:
label: submit
navigation_goal: Submit the contact form.
"""
- captured: dict[str, str] = {}
+ captured: dict[str, str | bool] = {}
async def fake_update_workflow(payload, ctx, allow_missing_credentials=False):
captured["workflow_yaml"] = payload["workflow_yaml"]
@@ -1545,6 +1479,7 @@ workflow_definition:
return {"ok": True, "_workflow": workflow, "data": {"block_count": 1}}
async def fake_run_blocks(params, ctx, **kwargs):
+ captured["run_called"] = True
return {
"ok": True,
"data": {
@@ -1559,15 +1494,15 @@ workflow_definition:
monkeypatch.setattr(tools_module, "_tool_loop_error", lambda *args, **kwargs: None)
monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", AsyncMock(return_value=None))
monkeypatch.setattr(tools_module, "_update_workflow", fake_update_workflow)
- monkeypatch.setattr(tools_module, "_pre_run_workflow_coverage_error", lambda *args: None)
monkeypatch.setattr(tools_module, "_plan_frontier", lambda *args: (["submit"], {}, "submit"))
monkeypatch.setattr(tools_module, "_run_blocks_and_collect_debug", fake_run_blocks)
monkeypatch.setattr(tools_module, "_record_diagnosis_repair_contract", lambda *args, **kwargs: None)
monkeypatch.setattr(tools_module, "enqueue_screenshot_from_result", lambda *args, **kwargs: None)
ctx = _ctx(
- user_message="Submit a contact form.",
+ user_message="Go to https://the-internet.herokuapp.com/download and then download the first file.",
block_goal_main_goal="Submit a contact form.",
+ request_policy=RequestPolicy(completion_contract="complete when the download starts"),
)
result = await tools_module.update_and_run_blocks_tool.on_invoke_tool(
SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"),
@@ -1576,6 +1511,8 @@ workflow_definition:
assert json.loads(result)["ok"] is True
assert captured["workflow_yaml"] == clean_yaml
+ assert captured["run_called"] is True
+ assert ctx.coverage_nudge_count == 0
assert "Achieve the following mini goal" not in captured["workflow_yaml"]
@@ -1844,15 +1781,10 @@ workflow_definition:
assert agent_result.workflow_yaml is None
def test_inline_replace_with_invalid_yaml_keeps_prior_pass(self, monkeypatch) -> None:
- # _process_workflow_yaml raising on a malformed REPLACE must leave
- # ctx untouched — no spurious last_test_ok reset, no workflow swap —
- # so a prior tested workflow remains available.
- import yaml as yaml_mod
-
tested_wf = SimpleNamespace(name="tested")
def boom(**kwargs):
- raise yaml_mod.YAMLError("mangled yaml")
+ raise yaml.YAMLError("mangled yaml")
monkeypatch.setattr("skyvern.forge.sdk.copilot.tools._process_workflow_yaml", boom)
@@ -4619,7 +4551,6 @@ class TestRunBlocksCredentialApproval:
monkeypatch.setattr(tools_module, "_tool_loop_error", lambda *args, **kwargs: None)
monkeypatch.setattr(tools_module, "_update_and_run_blocks_composition_evidence_precheck", lambda *args: None)
monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", AsyncMock(return_value=None))
- monkeypatch.setattr(tools_module, "_pre_run_workflow_coverage_error", lambda *args: None)
monkeypatch.setattr(tools_module, "_plan_frontier", lambda *args: (["login"], {}, "login"))
monkeypatch.setattr(tools_module, "_frontier_run_size_error", lambda *args: None)
monkeypatch.setattr(tools_module, "_verify_and_record_run_blocks_result", AsyncMock(return_value=None))
diff --git a/tests/unit/test_copilot_output_policy.py b/tests/unit/test_copilot_output_policy.py
index 23261fafb..7feb737a8 100644
--- a/tests/unit/test_copilot_output_policy.py
+++ b/tests/unit/test_copilot_output_policy.py
@@ -1837,7 +1837,6 @@ workflow_definition:
monkeypatch.setattr(tools_module, "_tool_loop_error", lambda *args, **kwargs: None)
monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", fake_prior_definition)
monkeypatch.setattr(tools_module, "_update_workflow", fake_update_workflow)
- monkeypatch.setattr(tools_module, "_pre_run_workflow_coverage_error", lambda *args: None)
monkeypatch.setattr(tools_module, "_plan_frontier", lambda *args: (["open_results"], {}, "open_results"))
monkeypatch.setattr(tools_module, "_frontier_run_size_error", lambda *args: None)
monkeypatch.setattr(tools_module, "_run_blocks_and_collect_debug", fake_run_blocks)