Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-10985: reconcile WIP-rescue terminal frame on coverage-complete adjudication failure (#6671)
Andrew Neilson7b64c080f8ea
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 0ffcd664a..43283a441 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -771,6 +771,11 @@ _FAILURE_FOLLOW_UP = {}+def _join_capped_labels(labels: list[str], cap: int = 6) -> str:+ shown = ", ".join(labels[:cap])+ return f"{shown}, ..." if len(labels) > cap else shown++def _partial_verification_response(ctx: CopilotContext) -> str | None:evidence = ctx.workflow_verification_evidenceif not evidence.has_evidence():@@ -778,16 +783,34 @@ def _partial_verification_response(ctx: CopilotContext) -> str | None:if evidence.full_workflow_verified:return None+ coverage_complete = (+ bool(evidence.block_verified) and not evidence.unverified_block_labels and not evidence.per_tool_budget_on_block+ )+ if coverage_complete:+ count = len(evidence.block_verified)+ block_word = "block" if count == 1 else "blocks"+ labels = _join_capped_labels(evidence.block_verified)+ failure_reason = (evidence.failure_reason or "").strip()+ if failure_reason:+ return (+ f"I saved a draft workflow and ran all {count} {block_word} ({labels}), but the run did not "+ f"confirm the workflow end-to-end: {failure_reason}. Keep the draft to iterate on, or discard."+ )+ return (+ f"I saved a draft workflow and ran all {count} {block_word} ({labels}), but I couldn't confirm the "+ "workflow end-to-end this turn. Keep the draft to iterate on, or discard."+ )+detail_parts: list[str] = []if evidence.block_verified:- detail_parts.append("verified block(s): " + ", ".join(evidence.block_verified[:6]))+ detail_parts.append("verified block(s): " + _join_capped_labels(evidence.block_verified))if evidence.live_page_state_verified:page = evidence.page_title or evidence.current_url or "the current browser page"detail_parts.append(f"verified current browser state: {page}")if evidence.per_tool_budget_on_block:- detail_parts.append("per-tool budget hit on: " + ", ".join(evidence.per_tool_budget_on_block[:6]))+ detail_parts.append("per-tool budget hit on: " + _join_capped_labels(evidence.per_tool_budget_on_block))if evidence.unverified_block_labels:- detail_parts.append("unverified block(s): " + ", ".join(evidence.unverified_block_labels[:6]))+ detail_parts.append("unverified block(s): " + _join_capped_labels(evidence.unverified_block_labels))details = " ".join(detail_parts)if details:diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex b5dd9d589..5afe35381 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -452,6 +452,194 @@ workflow_definition:assert "verified block(s): extract_results" in rewrittenassert "per-tool budget hit on: search_registry" in rewritten+ def test_coverage_complete_adjudication_failure_drops_chain_coverage_frame(self) -> None:+ from skyvern.forge.sdk.copilot.agent import _rewrite_failed_test_response++ failure_reason = (+ "The run completed but did not demonstrate the goal outcome(s): "+ "workflow title is 'Example Heading Extractor v2'. Add an end-state confirmation."+ )+ ctx = _ctx(+ last_workflow=object(),+ last_workflow_yaml="title: drafted",+ last_test_ok=True,+ last_full_workflow_test_ok=False,+ workflow_verification_evidence=WorkflowVerificationEvidence(+ block_verified=["extract_main_heading", "open_example_page"],+ unverified_block_labels=[],+ per_tool_budget_on_block=[],+ failure_reason=failure_reason,+ ),+ )++ rewritten = _rewrite_failed_test_response("I verified the workflow end-to-end.", ctx)++ assert "verified part of it" not in rewritten+ assert "verified block(s)" not in rewritten+ assert "extract_main_heading" in rewritten+ assert "open_example_page" in rewritten+ assert failure_reason in rewritten++ def test_coverage_complete_appends_ellipsis_when_blocks_exceed_cap(self) -> None:+ from skyvern.forge.sdk.copilot.agent import _rewrite_failed_test_response++ blocks = [f"block_{i}" for i in range(8)]+ ctx = _ctx(+ last_workflow=object(),+ last_workflow_yaml="title: drafted",+ last_test_ok=True,+ last_full_workflow_test_ok=False,+ workflow_verification_evidence=WorkflowVerificationEvidence(+ block_verified=blocks,+ unverified_block_labels=[],+ per_tool_budget_on_block=[],+ failure_reason="The run completed but did not demonstrate the goal outcome(s): title check.",+ ),+ )++ rewritten = _rewrite_failed_test_response("done", ctx)++ assert "ran all 8 blocks" in rewritten+ assert "block_0, block_1, block_2, block_3, block_4, block_5, ..." in rewritten+ assert "block_6" not in rewritten++ def test_partial_frame_appends_ellipsis_when_unverified_blocks_exceed_cap(self) -> None:+ from skyvern.forge.sdk.copilot.agent import _rewrite_failed_test_response++ ctx = _ctx(+ last_workflow=object(),+ last_workflow_yaml="title: drafted",+ last_test_ok=True,+ last_full_workflow_test_ok=False,+ workflow_verification_evidence=WorkflowVerificationEvidence(+ block_verified=["open_example_page"],+ unverified_block_labels=[f"block_{i}" for i in range(7)],+ per_tool_budget_on_block=[],+ ),+ )++ rewritten = _rewrite_failed_test_response("done", ctx)++ assert "verified part of it" in rewritten+ assert "unverified block(s): block_0, block_1, block_2, block_3, block_4, block_5, ..." in rewritten++ def test_genuinely_partial_via_unverified_blocks_keeps_existing_frame(self) -> None:+ from skyvern.forge.sdk.copilot.agent import _rewrite_failed_test_response++ ctx = _ctx(+ last_workflow=object(),+ last_workflow_yaml="title: drafted",+ last_test_ok=True,+ last_full_workflow_test_ok=False,+ workflow_verification_evidence=WorkflowVerificationEvidence(+ block_verified=["open_example_page"],+ unverified_block_labels=["extract_main_heading"],+ per_tool_budget_on_block=[],+ ),+ )++ rewritten = _rewrite_failed_test_response("done", ctx)++ assert "verified part of it" in rewritten+ assert "unverified block(s): extract_main_heading" in rewritten++ def test_genuinely_partial_via_per_tool_budget_keeps_existing_frame(self) -> None:+ from skyvern.forge.sdk.copilot.agent import _rewrite_failed_test_response++ ctx = _ctx(+ last_workflow=object(),+ last_workflow_yaml="title: drafted",+ last_test_ok=True,+ last_full_workflow_test_ok=False,+ workflow_verification_evidence=WorkflowVerificationEvidence(+ block_verified=["open_example_page"],+ unverified_block_labels=[],+ per_tool_budget_on_block=["search_block"],+ ),+ )++ rewritten = _rewrite_failed_test_response("done", ctx)++ assert "verified part of it" in rewritten+ assert "per-tool budget hit on: search_block" in rewritten++ def test_coverage_complete_without_failure_reason_uses_no_details_fallback(self) -> None:+ from skyvern.forge.sdk.copilot.agent import _rewrite_failed_test_response++ ctx = _ctx(+ last_workflow=object(),+ last_workflow_yaml="title: drafted",+ last_test_ok=True,+ last_full_workflow_test_ok=False,+ workflow_verification_evidence=WorkflowVerificationEvidence(+ block_verified=["open_example_page"],+ unverified_block_labels=[],+ per_tool_budget_on_block=[],+ failure_reason=None,+ ),+ )++ rewritten = _rewrite_failed_test_response("done", ctx)++ assert "verified part of it" not in rewritten+ assert "verified block(s): open_example_page" not in rewritten+ assert "keep the draft" in rewritten.lower()++ def test_empty_detail_evidence_keeps_bare_fallback(self) -> None:+ from skyvern.forge.sdk.copilot.agent import _rewrite_failed_test_response++ ctx = _ctx(+ last_workflow=object(),+ last_workflow_yaml="title: drafted",+ last_test_ok=True,+ last_full_workflow_test_ok=False,+ workflow_verification_evidence=WorkflowVerificationEvidence(+ test_attempted_but_incomplete=True,+ ),+ )++ rewritten = _rewrite_failed_test_response("done", ctx)++ assert "full workflow chain has not been verified end-to-end" in rewritten+ assert "verified part of it" not in rewritten+ assert "verified block(s)" not in rewritten++ def test_no_frame_asserts_end_to_end_verified_while_unverified(self) -> None:+ from skyvern.forge.sdk.copilot.agent import _rewrite_failed_test_response++ # Frame A renders execution vocabulary only; the word "verified" must never appear.+ ctx_a = _ctx(+ last_workflow=object(),+ last_workflow_yaml="title: drafted",+ last_test_ok=True,+ last_full_workflow_test_ok=False,+ workflow_verification_evidence=WorkflowVerificationEvidence(+ block_verified=["extract_main_heading", "open_example_page"],+ unverified_block_labels=[],+ per_tool_budget_on_block=[],+ failure_reason="The run completed but did not demonstrate the goal outcome(s): title check.",+ ),+ )+ reply_a = _rewrite_failed_test_response("done", ctx_a)+ assert "ran all 2 blocks" in reply_a+ assert "verified" not in reply_a.lower()++ # Frame B keeps the partial frame; its only end-to-end claim is the truthful negation.+ ctx_b = _ctx(+ last_workflow=object(),+ last_workflow_yaml="title: drafted",+ last_test_ok=True,+ last_full_workflow_test_ok=False,+ workflow_verification_evidence=WorkflowVerificationEvidence(+ block_verified=["open_example_page"],+ unverified_block_labels=["extract_main_heading"],+ per_tool_budget_on_block=[],+ ),+ )+ reply_b = _rewrite_failed_test_response("done", ctx_b)+ assert "verified part of it" in reply_b+ assert "has not been verified end-to-end" in reply_b+def test_runtime_verification_evidence_prompt_surfaces_state_for_agent(self) -> None:ctx = _ctx(workflow_verification_evidence=WorkflowVerificationEvidence(