Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Stop recording completed Copilot runs with registered outputs as failures (SKY-12765) (#7613)
Andrew Neilson14974a4baa08
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.pyindex de2852bd7..10350ae6d 100644--- a/skyvern/forge/sdk/copilot/build_test_outcome.py+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py@@ -54,6 +54,7 @@ BuildTestOutcomeReasonCode = Literal["terminal_challenge_blocker","blocker_reported","failed_run",+ "run_completed_unevaluated","suspicious_success","missing_structural_evidence","unchanged_after_recorded_outcome",@@ -1139,7 +1140,11 @@ def recorded_outcome_from_run_blocks_result(verdict: BuildTestOutcomeVerdict = "repairable_failure" if bool(result.get("ok")) is False else "progress_observed"if not structural_identity and not page_refs and not output_refs:verdict = "not_authoritative"- reason_code = "runtime_block_failure" if failed_block is not None or not bool(result.get("ok")) else "failed_run"+ reason_code = (+ "runtime_block_failure"+ if failed_block is not None or not bool(result.get("ok"))+ else "run_completed_unevaluated"+ )has_runtime_failure_evidence = bool(failure_categories or failure_type or runtime_failure_identity or failed_block)if (verdict == "repairable_failure"@@ -1555,6 +1560,8 @@ def _run_outcome_reason_code(recorded_run_outcome: RecordedRunOutcome) -> BuildTreturn reason_codeif recorded_run_outcome.verdict == "demonstrated":return "verified_success"+ if recorded_run_outcome.verdict == "not_evaluated":+ return "run_completed_unevaluated"return "failed_run"diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.pyindex 34574ea6d..45917b73a 100644--- a/skyvern/forge/sdk/copilot/enforcement.py+++ b/skyvern/forge/sdk/copilot/enforcement.py@@ -905,12 +905,13 @@ def _completion_verification_only_structural_abstentions(ctx: CopilotContext) ->def verified_goal_satisfied_context(ctx: CopilotContext) -> bool:- if outcome_fully_verified(ctx):- return True- # The judge verdict is authoritative: once it has evaluated, an unconfirmed- # criterion means the outcome is unmet regardless of run status. The block-count- # heuristic (which counts method verbs in the request) governs only when there- # is no evaluated verdict.+ return outcome_fully_verified(ctx)+++def built_complete_without_evaluated_outcome(ctx: CopilotContext) -> bool:+ """A run that looks built and repair-inert but carries no evaluated verdict.+ It ends the turn like ``built_unverified_repair_inert_context`` does, but must+ never authorize a verified-satisfaction claim."""if _outcome_criteria_evaluated(ctx):return Falseif not (@@ -952,6 +953,7 @@ def gate_decision_trace_fields(ctx: CopilotContext) -> dict[str, bool]:return {"gate_satisfied": verified_goal_satisfied_context(ctx),"gate_built_unverified_repair_inert": built_unverified_repair_inert_context(ctx),+ "gate_built_complete_without_evaluated_outcome": built_complete_without_evaluated_outcome(ctx),"gate_claim_authorized": verified_goal_claim_authorized(ctx),"gate_last_test_ok": ctx.last_test_ok is True,"gate_last_full_workflow_test_ok": ctx.last_full_workflow_test_ok is True,@@ -1546,7 +1548,7 @@ def _check_enforcement(if verified_goal_satisfied_context(ctx):raise CopilotGoalSatisfied()- if built_unverified_repair_inert_context(ctx):+ if built_unverified_repair_inert_context(ctx) or built_complete_without_evaluated_outcome(ctx):raise CopilotBuiltUnverified()if _needs_repair_ceiling_halt(ctx):diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.pyindex 6bc9c81c1..027edf7f3 100644--- a/skyvern/forge/sdk/copilot/tools/completion.py+++ b/skyvern/forge/sdk/copilot/tools/completion.py@@ -254,9 +254,9 @@ def _completion_verification_criteria(copilot_ctx: Any) -> list[CompletionCriterformed_criteria = policy.graded_completion_criteria() if policy is not None else []criteria = gradeable_completion_criteria(formed_criteria)authored_output_criteria = _authored_output_contract_criteria(copilot_ctx)- if authored_output_criteria and (- not formed_criteria or all(is_fallback_floor_criterion(c) for c in formed_criteria)- ):+ # With nothing gradeable left (no criteria formed, or every formed criterion degraded+ # out), the authored output contract is the only remaining evidence surface.+ if authored_output_criteria and (not criteria or all(is_fallback_floor_criterion(c) for c in formed_criteria)):return authored_output_criteriaif _accepted_staged_output_contract_missing(copilot_ctx) and (not formed_criteria or all(is_fallback_floor_criterion(c) for c in formed_criteria)diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex 6d649895f..2a70dc841 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -42,7 +42,11 @@ from skyvern.forge.sdk.copilot.completion_criteria_store import (plan_persistence,reconcile_completion_criteria,)-from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict+from skyvern.forge.sdk.copilot.completion_verification import (+ CompletionVerificationResult,+ CriterionVerdict,+ gradeable_completion_criteria,+)from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfigfrom skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContextfrom skyvern.forge.sdk.copilot.diagnosis_repair_contract import (@@ -66,6 +70,7 @@ from skyvern.forge.sdk.copilot.enforcement import (verified_goal_satisfied_context,)from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE+from skyvern.forge.sdk.copilot.hooks import CopilotRunHooksfrom skyvern.forge.sdk.copilot.output_policy import (ACTUATION_OBLIGATION_BROWSER_ACTION_KEY,ACTUATION_OBLIGATION_STEER_REASON_CODE,@@ -1181,6 +1186,11 @@ class TestVerifiedGoalSatisfiedStop:last_full_workflow_test_ok=True,latest_diagnosis_repair_contract=_verified_goal_contract(),)+ ctx.completion_verification_result = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["c0"],+ verdicts=[CriterionVerdict(criterion_id="c0", state="satisfied", reason_code="evidence_confirms")],+ )hook = CopilotRunHooks(ctx)result = json.dumps({@@ -1214,9 +1224,60 @@ class TestVerifiedGoalSatisfiedStop:last_full_workflow_test_ok=True,latest_diagnosis_repair_contract=_verified_goal_contract(),)+ ctx.completion_verification_result = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["c0"],+ verdicts=[CriterionVerdict(criterion_id="c0", state="satisfied", reason_code="evidence_confirms")],+ )assert verified_goal_satisfied_context(ctx)+ @pytest.mark.asyncio+ async def test_block_run_hook_does_not_claim_goal_satisfied_without_evaluated_outcome(self) -> None:+ ctx = _ctx(+ last_test_ok=True,+ last_full_workflow_test_ok=True,+ latest_diagnosis_repair_contract=_verified_goal_contract(),+ )+ hook = CopilotRunHooks(ctx)+ result = json.dumps(+ {+ "ok": True,+ "data": {+ "workflow_run_id": "wr_1",+ "blocks": [{"label": "search", "output": {"status": "found"}}],+ },+ }+ )++ assert ctx.completion_verification_result is None+ assert verified_goal_satisfied_context(ctx) is False++ await hook.on_tool_end(+ context=MagicMock(),+ agent=MagicMock(),+ tool=SimpleNamespace(name="update_and_run_blocks"),+ result=result,+ )++ assert ctx.goal_satisfied_tool_name is None++ def test_turn_telemetry_distinguishes_unevaluated_gate_from_repair_inert(self) -> None:+ from skyvern.forge.sdk.copilot.enforcement import gate_decision_trace_fields++ ctx = _ctx(+ last_test_ok=True,+ last_full_workflow_test_ok=True,+ latest_diagnosis_repair_contract=_verified_goal_contract(),+ )++ assert ctx.completion_verification_result is None+ fields = gate_decision_trace_fields(ctx)++ assert fields["gate_built_complete_without_evaluated_outcome"] is True+ assert fields["gate_built_unverified_repair_inert"] is False+ assert fields["gate_satisfied"] is False+@pytest.mark.asyncioasync def test_wrapped_exception_fallback_reaches_goal_satisfied_after_verified_consume(self) -> None:ctx = _ctx(@@ -6869,6 +6930,74 @@ class TestDeclaredEqualsGradedCompletionCriteria:assert not any(verdict.satisfied for verdict in verification.verdicts)assert all(verdict.reason_code != "evidence_confirms" for verdict in verification.verdicts)+ @pytest.mark.asyncio+ async def test_ungradeable_formed_criteria_fall_back_to_authored_output_contract(self) -> None:+ label = "collect_top_entry"+ ctx = SimpleNamespace(+ request_policy=RequestPolicy(+ completion_criteria=[+ CompletionCriterion(+ id="c0",+ outcome="the top listed entry is returned",+ mint_degrade="undecidable_judgment",+ )+ ],+ classifier_status="success",+ ),+ code_artifact_metadata={+ label: {"claimed_outcomes": [{"goal_value_paths": ["output.top_entry"]}]},+ },+ workflow_yaml=(+ "title: Utility path\n"+ "workflow_definition:\n"+ " blocks:\n"+ " - block_type: code\n"+ f" label: {label}\n"+ " code: |\n"+ " return {}\n"+ ),+ last_workflow_yaml=None,+ completion_verification_result=None,+ copilot_total_timeout_exceeded=False,+ reached_download_target=None,+ workflow_verification_evidence=SimpleNamespace(block_verified=[]),+ verified_prefix_labels=[],+ verified_block_outputs={},+ post_run_page_observation_after_failed_test=False,+ composition_page_evidence=None,+ completion_criteria_turn_state=None,+ )++ assert ctx.request_policy.graded_completion_criteria() != []+ assert gradeable_completion_criteria(ctx.request_policy.graded_completion_criteria()) == []++ criteria = _completion_verification_criteria(ctx)+ assert [criterion.output_path for criterion in criteria] == ["output.top_entry"]++ verification = await _maybe_run_completion_verification(+ ctx,+ {+ "ok": True,+ "data": {+ "workflow_run_id": "wr_completed",+ "overall_status": "completed",+ "blocks": [+ {+ "label": label,+ "status": "completed",+ "extracted_data": {"output": {"top_entry": "First listed entry"}},+ }+ ],+ "executed_block_labels": [label],+ },+ },+ 0,+ )++ assert verification is not None+ assert verification.status == "evaluated"+ assert [verdict.output_path for verdict in verification.verdicts] == ["output.top_entry"]+def test_fallback_floor_uses_repair_context_output_contract_paths_when_metadata_missing(self) -> None:ctx = SimpleNamespace(request_policy=RequestPolicy(diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.pyindex c2ab22481..ed9562c05 100644--- a/tests/unit/test_copilot_build_test_outcome.py+++ b/tests/unit/test_copilot_build_test_outcome.py@@ -845,10 +845,91 @@ def test_not_evaluated_recorded_outcome_is_not_authoritative_repair_failure() ->assert outcome is not Noneassert outcome.verdict == "not_authoritative"- assert outcome.reason_code == "failed_run"+ assert outcome.reason_code == "run_completed_unevaluated"assert outcome.is_authoritative is False+def test_completed_run_with_registered_outputs_is_not_classified_as_failed_run() -> None:+ result = {+ "ok": True,+ "data": {+ "workflow_run_id": "wr_completed",+ "overall_status": "completed",+ "blocks": [{"label": "collect_top_entry", "status": "completed"}],+ "registered_output_parameter_values": [+ {"output_parameter_id": "op_1", "value": {"output": {"top_entry": "First listed entry"}}}+ ],+ },+ }++ outcome = recorded_outcome_from_run_blocks_result(+ result,+ recorded_run_outcome=RecordedRunOutcome(verdict="not_evaluated", workflow_run_id="wr_completed"),+ )++ assert outcome is not None+ assert outcome.reason_code != "failed_run"+ assert outcome.reason_code == "run_completed_unevaluated"+++def test_failed_run_classification_preserved_for_not_ok_run() -> None:+ result = {+ "ok": False,+ "data": {+ "workflow_run_id": "wr_not_ok",+ "overall_status": "failed",+ "failure_type": "block_failure",+ "blocks": [{"label": "collect_top_entry", "status": "completed"}],+ },+ }++ outcome = recorded_outcome_from_run_blocks_result(result)++ assert outcome is not None+ assert outcome.reason_code == "runtime_block_failure"+ assert outcome.verdict == "repairable_failure"+++def test_failed_run_classification_preserved_for_failed_block() -> None:+ result = {+ "ok": True,+ "data": {+ "workflow_run_id": "wr_failed_block",+ "overall_status": "completed",+ "failure_type": "block_failure",+ "blocks": [{"label": "collect_top_entry", "status": "failed", "failure_reason": "selector not found"}],+ },+ }++ outcome = recorded_outcome_from_run_blocks_result(result)++ assert outcome is not None+ assert outcome.reason_code == "runtime_block_failure"+++def test_judge_evaluated_non_satisfaction_keeps_failure_classification() -> None:+ result = {+ "ok": True,+ "data": {+ "workflow_run_id": "wr_judged",+ "overall_status": "completed",+ "blocks": [{"label": "collect_top_entry", "status": "completed"}],+ },+ }++ outcome = recorded_outcome_from_run_blocks_result(+ result,+ recorded_run_outcome=RecordedRunOutcome(+ verdict="not_demonstrated",+ reason_code="outcome_not_demonstrated",+ workflow_run_id="wr_judged",+ ),+ )++ assert outcome is not None+ assert outcome.reason_code == "outcome_not_demonstrated"++def test_outcome_not_demonstrated_does_not_mark_presence_only_abstention_as_missing_output() -> None:result = {"ok": True,diff --git a/tests/unit/test_copilot_completion_criteria_lifecycle.py b/tests/unit/test_copilot_completion_criteria_lifecycle.pyindex c8923e296..0e407f333 100644--- a/tests/unit/test_copilot_completion_criteria_lifecycle.py+++ b/tests/unit/test_copilot_completion_criteria_lifecycle.py@@ -50,6 +50,9 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (VerificationResult,)from skyvern.forge.sdk.copilot.enforcement import (+ CopilotBuiltUnverified,+ _check_enforcement,+ built_complete_without_evaluated_outcome,built_unverified_repair_inert_context,verified_goal_claim_authorized,verified_goal_satisfied_context,@@ -1426,10 +1429,14 @@ def _no_repair_unverified_contract() -> DiagnosisRepairContract:def test_claim_closure_turn_still_ends_but_claim_downgrades() -> None:ctx = _legacy_verified_ctx()- # Turn completion is unchanged: the legacy conjunction still satisfies the gate.- assert verified_goal_satisfied_context(ctx) is True- # The claim tier is closed: no adjudicated evidence, no tested-success claim.++ assert ctx.completion_verification_result is None+ assert verified_goal_satisfied_context(ctx) is Falseassert verified_goal_claim_authorized(ctx) is False+ assert built_complete_without_evaluated_outcome(ctx) is True++ with pytest.raises(CopilotBuiltUnverified):+ _check_enforcement(ctx)def test_structural_abstention_no_repair_terminalizes_without_authorizing_success_claim() -> None: