Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-11638): persist Copilot review proof outputs (#7000)
Andrew Neilsone164e2616424
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.pyindex f8a90bf92..863fd93ff 100644--- a/skyvern/forge/sdk/copilot/completion_verification.py+++ b/skyvern/forge/sdk/copilot/completion_verification.py@@ -1389,9 +1389,13 @@ def _review_page_with_final_controls_visible(record: dict[str, Any]) -> bool:if not _has_review_page_evidence(record):return Falsefinal_controls = record.get("final_controls_visible")+ expected_controls = {"submit request", "submit", "back"}+ if not isinstance(final_controls, list):+ final_controls = record.get("submit_controls_visible")+ expected_controls = {"submit request", "submit"}if isinstance(final_controls, list):visible = {_normalize_present_value(str(value)) for value in final_controls if isinstance(value, str)}- if visible & {"submit request", "submit", "back"}:+ if visible & expected_controls:return Truereturn any("final" in _key_word_tokens(key)diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.pyindex 90c4322aa..5edc29650 100644--- a/skyvern/forge/sdk/copilot/tools/run_execution.py+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py@@ -927,7 +927,11 @@ def _workflow_output_parameter_indexes(if workflow is None:return {}, {}workflow_definition = getattr(workflow, "workflow_definition", None)- blocks = getattr(workflow_definition, "blocks", None)+ blocks = (+ workflow_definition.get("blocks")+ if isinstance(workflow_definition, Mapping)+ else getattr(workflow_definition, "blocks", None)+ )if not isinstance(blocks, list):return {}, {}diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.pyindex 8cee28950..43191ad93 100644--- a/tests/unit/test_copilot_completion_verification.py+++ b/tests/unit/test_copilot_completion_verification.py@@ -1872,6 +1872,70 @@ def test_fallback_floor_accepts_live_validation_review_output_parameter_shape()]+def test_fallback_floor_accepts_live_validation_review_submit_controls_shape_without_url() -> None:+ payload = _live_validation_review_output_payload(final_submit_controls_present=None)+ payload["submit_controls_visible"] = ["Submit Request", "Back"]+ snapshot = RunEvidenceSnapshot(block_outputs={"validate_business_start_service_review_output": payload})++ verdicts = grade_fallback_floor_reached_end_state_criteria(build_classifier_fallback_floor([]), snapshot)++ assert verdicts == [+ CriterionVerdict(+ criterion_id="__copilot_fallback_floor__run",+ state="satisfied",+ reason_code="evidence_confirms",+ evidence_ref="block_outputs:validate_business_start_service_review_output",+ )+ ]+++def test_fallback_floor_rejects_live_validation_review_submit_controls_after_final_click() -> None:+ payload = _live_validation_review_output_payload(+ final_submit_controls_present=None,+ submit_finalize_control_clicked=True,+ )+ payload["submit_controls_visible"] = ["Submit Request", "Back"]+ snapshot = RunEvidenceSnapshot(block_outputs={"validate_business_start_service_review_output": payload})++ assert grade_fallback_floor_reached_end_state_criteria(build_classifier_fallback_floor([]), snapshot) == []+++def test_fallback_floor_rejects_live_validation_review_back_only_submit_controls() -> None:+ payload = _live_validation_review_output_payload(final_submit_controls_present=None)+ payload["submit_controls_visible"] = ["Back"]+ snapshot = RunEvidenceSnapshot(block_outputs={"validate_business_start_service_review_output": payload})++ assert grade_fallback_floor_reached_end_state_criteria(build_classifier_fallback_floor([]), snapshot) == []+++def test_fallback_floor_rejects_observed_end_state_url_without_review_contract() -> None:+ snapshot = RunEvidenceSnapshot(+ current_url="http://localhost:8900/utility_services/peach_electric/",+ page_title="Peach Electric - Start Service",+ )++ assert grade_fallback_floor_reached_end_state_criteria(build_classifier_fallback_floor([]), snapshot) == []+++def test_fallback_floor_rejects_page_title_without_observed_end_state_url() -> None:+ snapshot = RunEvidenceSnapshot(page_title="Start Service - Review")++ assert grade_fallback_floor_reached_end_state_criteria(build_classifier_fallback_floor([]), snapshot) == []+++ "snapshot",+ [+ RunEvidenceSnapshot(current_url="https://example.test/review", failed_block_labels=["submit_request"]),+ RunEvidenceSnapshot(current_url="https://example.test/review", failure_classes=["ActionFailed"]),+ ],+)+def test_fallback_floor_rejects_observed_end_state_url_with_typed_failure_evidence(+ snapshot: RunEvidenceSnapshot,+) -> None:+ assert grade_fallback_floor_reached_end_state_criteria(build_classifier_fallback_floor([]), snapshot) == []++def test_fallback_floor_rejects_generated_validation_review_after_final_click() -> None:snapshot = RunEvidenceSnapshot(block_outputs={@@ -7183,13 +7247,10 @@ async def test_maybe_run_completion_verification_terminal_goal_uses_workflow_run@pytest.mark.asyncio-async def test_maybe_run_completion_verification_fallback_floor_rejects_judged_page_end_state_evidence(+async def test_maybe_run_completion_verification_fallback_floor_rejects_clean_url_without_review_contract(monkeypatch: pytest.MonkeyPatch,) -> None:- captured: dict[str, str] = {}-- async def handler(*, prompt: str, prompt_name: str) -> dict:- captured["prompt"] = prompt+ async def handler(**_: object) -> dict:return {"verdicts": [{@@ -7213,8 +7274,8 @@ async def test_maybe_run_completion_verification_fallback_floor_rejects_judged_p"workflow_run_id": "wr_review","overall_status": "completed","executed_block_labels": [],- "current_url": "https://example.test/review",- "page_title": "Start Service - Review",+ "current_url": "http://localhost:8900/utility_services/peach_electric/",+ "page_title": "Peach Electric - Start Service","blocks": [],},}@@ -7226,8 +7287,7 @@ async def test_maybe_run_completion_verification_fallback_floor_rejects_judged_passert verification.is_fully_satisfied() is Falseassert verification.verdicts[0].state == "unsatisfied"assert verification.verdicts[0].reason_code == "no_evidence"- assert "observed_end_state_url: https://example.test/review" in captured["prompt"]- assert "observed_end_state_page_title: Start Service - Review" in captured["prompt"]+ assert verification.verdicts[0].evidence_ref is None@pytest.mark.asyncio@@ -7263,7 +7323,7 @@ async def test_maybe_run_completion_verification_fallback_floor_without_evidence@pytest.mark.asyncio-async def test_maybe_run_completion_verification_fallback_floor_rejects_judged_login_page_output(+async def test_maybe_run_completion_verification_fallback_floor_rejects_typed_failure_evidence(monkeypatch: pytest.MonkeyPatch,) -> None:async def handler(**_: object) -> object:@@ -7289,7 +7349,7 @@ async def test_maybe_run_completion_verification_fallback_floor_rejects_judged_l"ok": True,"data": {"workflow_run_id": "wr_login_only",- "overall_status": "completed",+ "overall_status": "failed","executed_block_labels": ["open_demo_utility_login"],"current_url": "http://localhost:8900/utility_services/demo_utility/","page_title": "Login",@@ -7297,7 +7357,8 @@ async def test_maybe_run_completion_verification_fallback_floor_rejects_judged_l{"label": "open_demo_utility_login","block_type": "CODE",- "status": "completed",+ "status": "failed",+ "failure_reason": "Page.evaluate: SyntaxError: Unexpected token ')'","extracted_data": {"login_page_reached": True,"evidence_text": "Sample Utility Portal Log in to your account User ID Password",diff --git a/tests/unit/test_copilot_run_output_content_gate.py b/tests/unit/test_copilot_run_output_content_gate.pyindex 57a0630d3..466de5f23 100644--- a/tests/unit/test_copilot_run_output_content_gate.py+++ b/tests/unit/test_copilot_run_output_content_gate.py@@ -12,10 +12,14 @@ from typing import Anyimport pytestfrom skyvern.forge.sdk.copilot.agent import _verified_workflow_or_none-from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict+from skyvern.forge.sdk.copilot.completion_verification import (+ CompletionVerificationResult,+ CriterionVerdict,+ grade_fallback_floor_reached_end_state_criteria,+)from skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.copilot.enforcement import verified_goal_satisfied_context-from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy+from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy, build_classifier_fallback_floorfrom skyvern.forge.sdk.copilot.terminal_predicates import outcome_fully_verifiedfrom skyvern.forge.sdk.copilot.tools import (_analyze_run_blocks,@@ -24,6 +28,7 @@ from skyvern.forge.sdk.copilot.tools import (_is_unfinished_run_verification_candidate,_record_run_blocks_result,_run_blocks_structured_blocker_message,+ run_execution,)from skyvern.forge.sdk.copilot.tools._shared import _registered_output_parameter_payloadsfrom skyvern.forge.sdk.copilot.tools.blockers import _code_output_has_goal_content@@ -93,6 +98,27 @@ def _structured_record_payload(**overrides: Any) -> dict[str, Any]:return payload+def _validation_review_payload(**overrides: Any) -> dict[str, Any]:+ payload = {+ "validation_only": True,+ "submit_or_finalize_clicked": False,+ "pre_submit_review_reached": True,+ "final_controls_visible": ["Submit Request", "Back"],+ "review_values": {+ "Service Address": "1234 Sample Utility Way, Testville, CA 94016",+ "Requested Start Date": "2026-06-22",+ },+ "evidence_text": (+ "Start Service - Review\n"+ "Service Address\n1234 Sample Utility Way, Testville, CA 94016\n"+ "Requested Start Date\n2026-06-22\n"+ "Submit Request\nBack"+ ),+ }+ payload.update(overrides)+ return payload++def _ctx(blocks: list[dict[str, Any]] | None = None) -> CopilotContext:ctx = CopilotContext(organization_id="o",@@ -702,8 +728,6 @@ def test_structured_record_top_level_output_record_is_meaningful() -> None:async def test_registered_output_adapter_fetches_db_values_and_synthesizes_block(monkeypatch: pytest.MonkeyPatch,) -> None:- from skyvern.forge.sdk.copilot.tools import run_execution-async def fake_get_workflow_run_output_parameters(*, workflow_run_id: str) -> list[SimpleNamespace]:assert workflow_run_id == "wr_test"return [@@ -759,6 +783,60 @@ async def test_registered_output_adapter_fetches_db_values_and_synthesizes_blockassert data["blocks"][0]["extracted_data"]["extract_record_status_details_output"]["record_number"] == "1234567890"+async def test_registered_validation_review_output_from_dict_workflow_merges_into_block_and_verifies(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ review_payload = _validation_review_payload()++ async def fake_get_workflow_run_output_parameters(*, workflow_run_id: str) -> list[SimpleNamespace]:+ assert workflow_run_id == "wr_test"+ return [+ SimpleNamespace(+ workflow_run_id="wr_test",+ output_parameter_id="op_review",+ value=review_payload,+ )+ ]++ monkeypatch.setattr(+ run_execution.app.DATABASE,+ "workflow_runs",+ SimpleNamespace(get_workflow_run_output_parameters=fake_get_workflow_run_output_parameters),+ )+ workflow = SimpleNamespace(+ workflow_definition={+ "blocks": [+ {+ "label": "validate_business_start_service_review",+ "block_type": "CODE",+ "output_parameter": {+ "output_parameter_id": "op_review",+ "key": "validate_business_start_service_review_output",+ },+ }+ ]+ }+ )+ data: dict[str, Any] = {"workflow_run_id": "wr_test", "blocks": []}++ by_label = await _attach_registered_output_parameter_values(+ workflow_run_id="wr_test",+ workflow=workflow, # type: ignore[arg-type]+ data=data,+ )+ ctx = _ctx([{"label": "validate_business_start_service_review"}])+ snapshot = _build_run_evidence_snapshot(ctx, {"ok": True, "data": data})+ verdicts = grade_fallback_floor_reached_end_state_criteria(build_classifier_fallback_floor([]), snapshot)++ assert by_label == {+ "validate_business_start_service_review": {"validate_business_start_service_review_output": review_payload}+ }+ assert data["registered_output_parameter_values"][0]["block_label"] == "validate_business_start_service_review"+ assert data["blocks"][0]["extracted_data"]["validate_business_start_service_review_output"] == review_payload+ assert verdicts[0].evidence_ref == "block_outputs:validate_business_start_service_review"++def test_satisfied_completion_prevents_empty_output_suspicious_success() -> None:result = _empty_extraction_run_result()ctx = _ctx(result["data"]["blocks"])