Skip to content

Automate browser based workflows with AI

fix(SKY-11638): persist Copilot review proof outputs (#7000)

Andrew Neilsone164e2616424
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.py
index f8a90bf92..863fd93ff 100644
--- a/skyvern/forge/sdk/copilot/completion_verification.py
+++ b/skyvern/forge/sdk/copilot/completion_verification.py
@@ -1389,9 +1389,13 @@ def _review_page_with_final_controls_visible(record: dict[str, Any]) -> bool:
if not _has_review_page_evidence(record):
return False
final_controls = record.get("final_controls_visible")
+ expected_controls = {"submit request", "submit", "back"}
+ if not isinstance(final_controls, list):
+ final_controls = record.get("submit_controls_visible")
+ expected_controls = {"submit request", "submit"}
if isinstance(final_controls, list):
visible = {_normalize_present_value(str(value)) for value in final_controls if isinstance(value, str)}
- if visible & {"submit request", "submit", "back"}:
+ if visible & expected_controls:
return True
return any(
"final" in _key_word_tokens(key)
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 90c4322aa..5edc29650 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -927,7 +927,11 @@ def _workflow_output_parameter_indexes(
if workflow is None:
return {}, {}
workflow_definition = getattr(workflow, "workflow_definition", None)
- blocks = getattr(workflow_definition, "blocks", None)
+ blocks = (
+ workflow_definition.get("blocks")
+ if isinstance(workflow_definition, Mapping)
+ else getattr(workflow_definition, "blocks", None)
+ )
if not isinstance(blocks, list):
return {}, {}
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index 8cee28950..43191ad93 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -1872,6 +1872,70 @@ def test_fallback_floor_accepts_live_validation_review_output_parameter_shape()
]
+def test_fallback_floor_accepts_live_validation_review_submit_controls_shape_without_url() -> None:
+ payload = _live_validation_review_output_payload(final_submit_controls_present=None)
+ payload["submit_controls_visible"] = ["Submit Request", "Back"]
+ snapshot = RunEvidenceSnapshot(block_outputs={"validate_business_start_service_review_output": payload})
+
+ verdicts = grade_fallback_floor_reached_end_state_criteria(build_classifier_fallback_floor([]), snapshot)
+
+ assert verdicts == [
+ CriterionVerdict(
+ criterion_id="__copilot_fallback_floor__run",
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_ref="block_outputs:validate_business_start_service_review_output",
+ )
+ ]
+
+
+def test_fallback_floor_rejects_live_validation_review_submit_controls_after_final_click() -> None:
+ payload = _live_validation_review_output_payload(
+ final_submit_controls_present=None,
+ submit_finalize_control_clicked=True,
+ )
+ payload["submit_controls_visible"] = ["Submit Request", "Back"]
+ snapshot = RunEvidenceSnapshot(block_outputs={"validate_business_start_service_review_output": payload})
+
+ assert grade_fallback_floor_reached_end_state_criteria(build_classifier_fallback_floor([]), snapshot) == []
+
+
+def test_fallback_floor_rejects_live_validation_review_back_only_submit_controls() -> None:
+ payload = _live_validation_review_output_payload(final_submit_controls_present=None)
+ payload["submit_controls_visible"] = ["Back"]
+ snapshot = RunEvidenceSnapshot(block_outputs={"validate_business_start_service_review_output": payload})
+
+ assert grade_fallback_floor_reached_end_state_criteria(build_classifier_fallback_floor([]), snapshot) == []
+
+
+def test_fallback_floor_rejects_observed_end_state_url_without_review_contract() -> None:
+ snapshot = RunEvidenceSnapshot(
+ current_url="http://localhost:8900/utility_services/peach_electric/",
+ page_title="Peach Electric - Start Service",
+ )
+
+ assert grade_fallback_floor_reached_end_state_criteria(build_classifier_fallback_floor([]), snapshot) == []
+
+
+def test_fallback_floor_rejects_page_title_without_observed_end_state_url() -> None:
+ snapshot = RunEvidenceSnapshot(page_title="Start Service - Review")
+
+ assert grade_fallback_floor_reached_end_state_criteria(build_classifier_fallback_floor([]), snapshot) == []
+
+
+ "snapshot",
+ [
+ RunEvidenceSnapshot(current_url="https://example.test/review", failed_block_labels=["submit_request"]),
+ RunEvidenceSnapshot(current_url="https://example.test/review", failure_classes=["ActionFailed"]),
+ ],
+)
+def test_fallback_floor_rejects_observed_end_state_url_with_typed_failure_evidence(
+ snapshot: RunEvidenceSnapshot,
+) -> None:
+ assert grade_fallback_floor_reached_end_state_criteria(build_classifier_fallback_floor([]), snapshot) == []
+
+
def test_fallback_floor_rejects_generated_validation_review_after_final_click() -> None:
snapshot = RunEvidenceSnapshot(
block_outputs={
@@ -7183,13 +7247,10 @@ async def test_maybe_run_completion_verification_terminal_goal_uses_workflow_run
@pytest.mark.asyncio
-async def test_maybe_run_completion_verification_fallback_floor_rejects_judged_page_end_state_evidence(
+async def test_maybe_run_completion_verification_fallback_floor_rejects_clean_url_without_review_contract(
monkeypatch: pytest.MonkeyPatch,
) -> None:
- captured: dict[str, str] = {}
-
- async def handler(*, prompt: str, prompt_name: str) -> dict:
- captured["prompt"] = prompt
+ async def handler(**_: object) -> dict:
return {
"verdicts": [
{
@@ -7213,8 +7274,8 @@ async def test_maybe_run_completion_verification_fallback_floor_rejects_judged_p
"workflow_run_id": "wr_review",
"overall_status": "completed",
"executed_block_labels": [],
- "current_url": "https://example.test/review",
- "page_title": "Start Service - Review",
+ "current_url": "http://localhost:8900/utility_services/peach_electric/",
+ "page_title": "Peach Electric - Start Service",
"blocks": [],
},
}
@@ -7226,8 +7287,7 @@ async def test_maybe_run_completion_verification_fallback_floor_rejects_judged_p
assert verification.is_fully_satisfied() is False
assert verification.verdicts[0].state == "unsatisfied"
assert verification.verdicts[0].reason_code == "no_evidence"
- assert "observed_end_state_url: https://example.test/review" in captured["prompt"]
- assert "observed_end_state_page_title: Start Service - Review" in captured["prompt"]
+ assert verification.verdicts[0].evidence_ref is None
@pytest.mark.asyncio
@@ -7263,7 +7323,7 @@ async def test_maybe_run_completion_verification_fallback_floor_without_evidence
@pytest.mark.asyncio
-async def test_maybe_run_completion_verification_fallback_floor_rejects_judged_login_page_output(
+async def test_maybe_run_completion_verification_fallback_floor_rejects_typed_failure_evidence(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def handler(**_: object) -> object:
@@ -7289,7 +7349,7 @@ async def test_maybe_run_completion_verification_fallback_floor_rejects_judged_l
"ok": True,
"data": {
"workflow_run_id": "wr_login_only",
- "overall_status": "completed",
+ "overall_status": "failed",
"executed_block_labels": ["open_demo_utility_login"],
"current_url": "http://localhost:8900/utility_services/demo_utility/",
"page_title": "Login",
@@ -7297,7 +7357,8 @@ async def test_maybe_run_completion_verification_fallback_floor_rejects_judged_l
{
"label": "open_demo_utility_login",
"block_type": "CODE",
- "status": "completed",
+ "status": "failed",
+ "failure_reason": "Page.evaluate: SyntaxError: Unexpected token ')'",
"extracted_data": {
"login_page_reached": True,
"evidence_text": "Sample Utility Portal Log in to your account User ID Password",
diff --git a/tests/unit/test_copilot_run_output_content_gate.py b/tests/unit/test_copilot_run_output_content_gate.py
index 57a0630d3..466de5f23 100644
--- a/tests/unit/test_copilot_run_output_content_gate.py
+++ b/tests/unit/test_copilot_run_output_content_gate.py
@@ -12,10 +12,14 @@ from typing import Any
import pytest
from skyvern.forge.sdk.copilot.agent import _verified_workflow_or_none
-from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
+from skyvern.forge.sdk.copilot.completion_verification import (
+ CompletionVerificationResult,
+ CriterionVerdict,
+ grade_fallback_floor_reached_end_state_criteria,
+)
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.enforcement import verified_goal_satisfied_context
-from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy
+from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy, build_classifier_fallback_floor
from skyvern.forge.sdk.copilot.terminal_predicates import outcome_fully_verified
from skyvern.forge.sdk.copilot.tools import (
_analyze_run_blocks,
@@ -24,6 +28,7 @@ from skyvern.forge.sdk.copilot.tools import (
_is_unfinished_run_verification_candidate,
_record_run_blocks_result,
_run_blocks_structured_blocker_message,
+ run_execution,
)
from skyvern.forge.sdk.copilot.tools._shared import _registered_output_parameter_payloads
from skyvern.forge.sdk.copilot.tools.blockers import _code_output_has_goal_content
@@ -93,6 +98,27 @@ def _structured_record_payload(**overrides: Any) -> dict[str, Any]:
return payload
+def _validation_review_payload(**overrides: Any) -> dict[str, Any]:
+ payload = {
+ "validation_only": True,
+ "submit_or_finalize_clicked": False,
+ "pre_submit_review_reached": True,
+ "final_controls_visible": ["Submit Request", "Back"],
+ "review_values": {
+ "Service Address": "1234 Sample Utility Way, Testville, CA 94016",
+ "Requested Start Date": "2026-06-22",
+ },
+ "evidence_text": (
+ "Start Service - Review\n"
+ "Service Address\n1234 Sample Utility Way, Testville, CA 94016\n"
+ "Requested Start Date\n2026-06-22\n"
+ "Submit Request\nBack"
+ ),
+ }
+ payload.update(overrides)
+ return payload
+
+
def _ctx(blocks: list[dict[str, Any]] | None = None) -> CopilotContext:
ctx = CopilotContext(
organization_id="o",
@@ -702,8 +728,6 @@ def test_structured_record_top_level_output_record_is_meaningful() -> None:
async def test_registered_output_adapter_fetches_db_values_and_synthesizes_block(
monkeypatch: pytest.MonkeyPatch,
) -> None:
- from skyvern.forge.sdk.copilot.tools import run_execution
-
async def fake_get_workflow_run_output_parameters(*, workflow_run_id: str) -> list[SimpleNamespace]:
assert workflow_run_id == "wr_test"
return [
@@ -759,6 +783,60 @@ async def test_registered_output_adapter_fetches_db_values_and_synthesizes_block
assert data["blocks"][0]["extracted_data"]["extract_record_status_details_output"]["record_number"] == "1234567890"
+async def test_registered_validation_review_output_from_dict_workflow_merges_into_block_and_verifies(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ review_payload = _validation_review_payload()
+
+ async def fake_get_workflow_run_output_parameters(*, workflow_run_id: str) -> list[SimpleNamespace]:
+ assert workflow_run_id == "wr_test"
+ return [
+ SimpleNamespace(
+ workflow_run_id="wr_test",
+ output_parameter_id="op_review",
+ value=review_payload,
+ )
+ ]
+
+ monkeypatch.setattr(
+ run_execution.app.DATABASE,
+ "workflow_runs",
+ SimpleNamespace(get_workflow_run_output_parameters=fake_get_workflow_run_output_parameters),
+ )
+ workflow = SimpleNamespace(
+ workflow_definition={
+ "blocks": [
+ {
+ "label": "validate_business_start_service_review",
+ "block_type": "CODE",
+ "output_parameter": {
+ "output_parameter_id": "op_review",
+ "key": "validate_business_start_service_review_output",
+ },
+ }
+ ]
+ }
+ )
+ data: dict[str, Any] = {"workflow_run_id": "wr_test", "blocks": []}
+
+ by_label = await _attach_registered_output_parameter_values(
+ workflow_run_id="wr_test",
+ workflow=workflow, # type: ignore[arg-type]
+ data=data,
+ )
+ ctx = _ctx([{"label": "validate_business_start_service_review"}])
+ snapshot = _build_run_evidence_snapshot(ctx, {"ok": True, "data": data})
+ verdicts = grade_fallback_floor_reached_end_state_criteria(build_classifier_fallback_floor([]), snapshot)
+
+ assert by_label == {
+ "validate_business_start_service_review": {"validate_business_start_service_review_output": review_payload}
+ }
+ assert data["registered_output_parameter_values"][0]["block_label"] == "validate_business_start_service_review"
+ assert data["blocks"][0]["extracted_data"]["validate_business_start_service_review_output"] == review_payload
+ assert verdicts[0].evidence_ref == "block_outputs:validate_business_start_service_review"
+
+
def test_satisfied_completion_prevents_empty_output_suspicious_success() -> None:
result = _empty_extraction_run_result()
ctx = _ctx(result["data"]["blocks"])