Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-11558: Preserve committed Copilot run outcomes (#6917)
Andrew Neilson3e75fccb1e33
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 355d954bf..be437fdfa 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -1177,6 +1177,8 @@ def _shape_ask_question_response(user_response: str, ctx: CopilotContext) -> strdef _completion_contract_not_violated(ctx: CopilotContext) -> bool:if artifact_health_blocked(ctx):return False+ if outcome_fully_verified(ctx):+ return Trueresult = ctx.completion_verification_resultif result is None:return Truediff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.pyindex 8569879f9..abbc25a46 100644--- a/skyvern/forge/sdk/copilot/completion_verification.py+++ b/skyvern/forge/sdk/copilot/completion_verification.py@@ -104,6 +104,9 @@ class CompletionVerificationResult:if self.status != "evaluated" or not self.criterion_ids:return Falseverdict_by_id = {verdict.criterion_id: verdict for verdict in self.verdicts}+ has_observed_reach_state = any(+ _is_satisfied_observed_end_state_verdict(verdict) for verdict in verdict_by_id.values()+ )satisfied_run_plane_count = 0for criterion_id in self.criterion_ids:verdict = verdict_by_id.get(criterion_id)@@ -126,6 +129,8 @@ class CompletionVerificationResult:continueif verdict is not None and _is_structural_requested_output_abstention(verdict):continue+ if has_observed_reach_state and verdict is not None and _is_reperception_contradiction(verdict):+ continuereturn Falsereturn satisfied_run_plane_count > 0@@ -1835,6 +1840,24 @@ def _is_meaningful_contingent_antecedent_value(value: Any) -> bool:_DEFINITION_REASON_PREFIX = "definition_"+_OBSERVED_END_STATE_EVIDENCE_REF = "observed_end_state_url"+_REPERCEPTION_CONTRADICTION_EVIDENCE_REFS = frozenset({"scout_synthesized_browser_steps_output"})+++def _is_satisfied_observed_end_state_verdict(verdict: CriterionVerdict) -> bool:+ return (+ verdict.state == "satisfied"+ and verdict.reason_code == "evidence_confirms"+ and verdict.evidence_ref == _OBSERVED_END_STATE_EVIDENCE_REF+ )+++def _is_reperception_contradiction(verdict: CriterionVerdict) -> bool:+ return (+ verdict.state == "unsatisfied"+ and verdict.reason_code == "evidence_contradicts"+ and verdict.evidence_ref in _REPERCEPTION_CONTRADICTION_EVIDENCE_REFS+ )def _is_definition_plane_abstention(verdict: CriterionVerdict) -> bool:diff --git a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.pyindex e6f91b220..588f64164 100644--- a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py+++ b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py@@ -191,6 +191,7 @@ def build_diagnosis_repair_contract(if next_action == RepairNextAction.REPAIR and not target_blocks and repair_context is not None:target_blocks = [repair_context.block_label]user_goal_satisfied, completion_contract_satisfied = _verification_satisfaction(+ ctx,run_ok,suspicious,run_status,@@ -612,6 +613,7 @@ def _last_test_anti_bot_is_terminal(ctx: CopilotContext, data: dict[str, Any]) -def _verification_satisfaction(+ ctx: CopilotContext,run_ok: bool,suspicious: bool,run_status: str | None,@@ -627,6 +629,8 @@ def _verification_satisfaction(return fully_satisfied, fully_satisfiedif failure_type == DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT:return False, False+ if outcome_fully_verified(ctx):+ return True, Trueif (failure_type == DiagnosisFailureType.NO_FAILUREand completion_verification is not Nonediff --git a/skyvern/forge/sdk/copilot/terminal_predicates.py b/skyvern/forge/sdk/copilot/terminal_predicates.pyindex 91f379dd5..efd895391 100644--- a/skyvern/forge/sdk/copilot/terminal_predicates.py+++ b/skyvern/forge/sdk/copilot/terminal_predicates.py@@ -23,6 +23,17 @@ def outcome_criteria_evaluated(ctx: CopilotContext) -> bool:return result is not None and result.status == "evaluated"+def _has_same_run_committed_demonstrated_outcome(ctx: CopilotContext) -> bool:+ outcome = getattr(ctx, "last_run_outcome", None)+ run_id = getattr(ctx, "last_run_blocks_workflow_run_id", None)+ return (+ getattr(outcome, "verdict", None) == "demonstrated"+ and isinstance(run_id, str)+ and bool(run_id)+ and getattr(outcome, "workflow_run_id", None) == run_id+ )++def outcome_fully_verified(ctx: CopilotContext) -> bool:"""Whether the judge confirmed every outcome criterion from this run's evidence.@@ -31,6 +42,8 @@ def outcome_fully_verified(ctx: CopilotContext) -> bool:"""if artifact_health_blocked(ctx):return False+ if _has_same_run_committed_demonstrated_outcome(ctx):+ return Trueif not outcome_criteria_evaluated(ctx):return Falseresult = ctx.completion_verification_resultdiff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.pyindex 9a5de96d7..f6adaef54 100644--- a/skyvern/forge/sdk/copilot/tools/completion.py+++ b/skyvern/forge/sdk/copilot/tools/completion.py@@ -45,6 +45,7 @@ from skyvern.forge.sdk.copilot.reached_download_target import (derive_from_block_outputs,)from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion+from skyvern.forge.sdk.copilot.terminal_predicates import outcome_fully_verifiedfrom skyvern.forge.sdk.copilot.tracing_setup import copilot_spanfrom ._shared import (@@ -1234,6 +1235,8 @@ def _tool_visible_result_after_completion_verification(result: dict[str, Any],completion_verification: CompletionVerificationResult | None,) -> dict[str, Any]:+ if outcome_fully_verified(copilot_ctx):+ return resultoutcome_unverified_reason = _outcome_unverified_reason(copilot_ctx, completion_verification)if outcome_unverified_reason is None:return resultdiff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.pyindex d9e85a2db..5d8b758aa 100644--- a/skyvern/forge/sdk/copilot/tools/run_execution.py+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py@@ -165,6 +165,14 @@ from .scouting import _mark_page_inspected, _mark_post_run_page_observedLOG = structlog.get_logger()++class _RunIdUnset:+ pass+++# Distinguishes omitted run id (read ctx) from explicit None (no current run).+_RUN_ID_UNSET = _RunIdUnset()+_ACTIVE_RUN_TERMINAL_MONITOR_INITIAL_DELAY_SECONDS = 30.0_ACTIVE_RUN_TERMINAL_MONITOR_INTERVAL_SECONDS = 30.0_ACTIVE_RUN_TERMINAL_MONITOR_MAX_SAMPLES = 8@@ -2035,9 +2043,13 @@ def _record_run_blocks_result(# Prefer the final run-result extracted_data for terminal replies;# it is the same persisted run evidence completion verification saw.copilot_ctx.verified_terminal_block_outputs = terminal_outputs+ prior_committed_outcome = _same_run_committed_demonstrated_outcome(+ copilot_ctx, run_id if isinstance(run_id, str) else None+ )copilot_ctx.completion_verification_result = completion_verification- record_completion_verification(copilot_ctx, completion_verification)- _record_adjudication_on_turn_state(copilot_ctx, completion_verification)+ if prior_committed_outcome is None or _verification_fully_satisfied(completion_verification):+ record_completion_verification(copilot_ctx, completion_verification)+ _record_adjudication_on_turn_state(copilot_ctx, completion_verification)if completion_verification is not None and completion_verification.status == "evaluated":_emit_completion_verification_trace(copilot_ctx, completion_verification)copilot_ctx.last_run_blocks_workflow_run_id = run_id if isinstance(run_id, str) else None@@ -2060,8 +2072,9 @@ def _record_run_blocks_result(copilot_ctx.last_outcome_gate_reason = _outcome_unverified_reason(copilot_ctx, completion_verification)copilot_ctx.last_outcome_gate_workflow_run_id = copilot_ctx.last_run_blocks_workflow_run_idcopilot_ctx.last_test_suspicious_success = False- copilot_ctx.last_run_outcome = None- copilot_ctx.last_run_outcome_block_labels = []+ if prior_committed_outcome is None:+ copilot_ctx.last_run_outcome = None+ copilot_ctx.last_run_outcome_block_labels = []copilot_ctx.suspicious_success_nudge_count = 0copilot_ctx.last_test_anti_bot = Noneprior_budget_flag = copilot_ctx.last_failure_category_top == PER_TOOL_BUDGET_FAILURE_CATEGORY@@ -2244,6 +2257,19 @@ def _record_run_blocks_result(display_reason=run_outcome_display_reason(copilot_ctx.last_test_failure_reason),),)+ if prior_committed_outcome is not None and artifact_reason is None:+ # artifact_reason is current-run health; prior_committed_outcome already passed prior ctx artifact-health.+ copilot_ctx.last_full_workflow_test_ok = True+ copilot_ctx.last_unverified_block_labels = []+ copilot_ctx.last_good_workflow = copilot_ctx.last_workflow+ copilot_ctx.last_good_workflow_yaml = copilot_ctx.last_workflow_yaml+ copilot_ctx.last_test_suspicious_success = False+ copilot_ctx.last_test_failure_reason = None+ copilot_ctx.last_outcome_gate_reason = None+ copilot_ctx.last_outcome_gate_workflow_run_id = None+ update_repeated_failure_state(copilot_ctx, result)+ _update_verification_evidence_from_run_result(copilot_ctx, result)+ return _stash_recorded_run_outcome(copilot_ctx, prior_committed_outcome)unverified = _unverified_current_workflow_labels(copilot_ctx)copilot_ctx.last_unverified_block_labels = unverifiedoutcome_unverified_reason = _outcome_unverified_reason(copilot_ctx, completion_verification)@@ -2330,14 +2356,45 @@ def _record_run_blocks_result(def _stash_recorded_run_outcome(copilot_ctx: Any, outcome: RecordedRunOutcome) -> RecordedRunOutcome:if outcome.workflow_run_id is None:outcome = replace(outcome, workflow_run_id=getattr(copilot_ctx, "last_run_blocks_workflow_run_id", None))+ committed = _same_run_committed_demonstrated_outcome(copilot_ctx, outcome.workflow_run_id)+ if committed is not None and outcome.reason_code == "outcome_not_demonstrated":+ return committedcopilot_ctx.last_run_outcome = outcomecopilot_ctx.last_run_outcome_block_labels = list(getattr(copilot_ctx, "last_run_blocks_block_labels", []) or [])return outcome+def _verification_fully_satisfied(completion_verification: CompletionVerificationResult | None) -> bool:+ return completion_verification is not None and completion_verification.is_fully_satisfied()+++def _same_run_committed_demonstrated_outcome(+ copilot_ctx: Any, workflow_run_id: str | None | _RunIdUnset = _RUN_ID_UNSET+) -> RecordedRunOutcome | None:+ artifact_reason = getattr(copilot_ctx, "last_artifact_health_blocker_reason", None)+ if isinstance(artifact_reason, str) and artifact_reason.strip():+ return None+ run_id = (+ getattr(copilot_ctx, "last_run_blocks_workflow_run_id", None)+ if workflow_run_id is _RUN_ID_UNSET+ else workflow_run_id+ )+ if not isinstance(run_id, str) or not run_id:+ return None+ outcome = getattr(copilot_ctx, "last_run_outcome", None)+ if not isinstance(outcome, RecordedRunOutcome):+ return None+ if outcome.verdict != "demonstrated" or outcome.workflow_run_id != run_id:+ return None+ return outcome++def _adjudicated_run_outcome(copilot_ctx: Any, completion_verification: CompletionVerificationResult | None) -> RecordedRunOutcome:+ committed = _same_run_committed_demonstrated_outcome(copilot_ctx)+ if committed is not None:+ return committedif completion_verification is not None and completion_verification.status == "evaluated":if not completion_verification.is_fully_satisfied():return RecordedRunOutcome(diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.pyindex 748448323..e3ff52981 100644--- a/tests/unit/test_copilot_completion_verification.py+++ b/tests/unit/test_copilot_completion_verification.py@@ -66,6 +66,7 @@ from skyvern.forge.sdk.copilot.request_policy import (_parse_completion_criteria,build_classifier_fallback_floor,)+from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcomefrom skyvern.forge.sdk.copilot.tools import (ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,_active_run_terminal_evidence_needs_visual_fallback,@@ -356,6 +357,63 @@ def test_is_fully_satisfied_requires_every_criterion() -> None:assert _evaluated(("c0", True), ("c1", False)).is_fully_satisfied() is False+def test_observed_end_state_satisfaction_takes_precedence_over_reperception_contradiction() -> None:+ result = _mixed(+ CriterionVerdict(+ criterion_id="c_reach",+ state="satisfied",+ reason_code="evidence_confirms",+ evidence_ref="observed_end_state_url",+ ),+ CriterionVerdict(+ criterion_id="c_reperception",+ state="unsatisfied",+ reason_code="evidence_contradicts",+ evidence_ref="scout_synthesized_browser_steps_output",+ ),+ )++ assert result.is_fully_satisfied() is True+++def test_reperception_contradiction_without_observed_end_state_satisfaction_still_blocks() -> None:+ result = _mixed(+ CriterionVerdict(+ criterion_id="c_output",+ state="satisfied",+ reason_code="evidence_confirms",+ evidence_ref="block_outputs:submit_request",+ ),+ CriterionVerdict(+ criterion_id="c_reperception",+ state="unsatisfied",+ reason_code="evidence_contradicts",+ evidence_ref="scout_synthesized_browser_steps_output",+ ),+ )++ assert result.is_fully_satisfied() is False+++def test_observed_end_state_satisfaction_does_not_override_requested_output_contradiction() -> None:+ result = _mixed(+ CriterionVerdict(+ criterion_id="c_reach",+ state="satisfied",+ reason_code="evidence_confirms",+ evidence_ref="observed_end_state_url",+ ),+ CriterionVerdict(+ criterion_id="c_requested_output",+ state="unsatisfied",+ reason_code="evidence_contradicts",+ evidence_ref="block_outputs:submit_request.confirmation_number",+ ),+ )++ assert result.is_fully_satisfied() is False++def _mixed(*verdicts: CriterionVerdict) -> CompletionVerificationResult:return CompletionVerificationResult(status="evaluated", criterion_ids=[v.criterion_id for v in verdicts], verdicts=list(verdicts)@@ -2011,20 +2069,44 @@ def test_parse_caps_count() -> None:assert len(_parse_completion_criteria(raw)) == 8+def _verification_satisfaction_ctx(+ completion_verification: CompletionVerificationResult | None = None,+) -> SimpleNamespace:+ return SimpleNamespace(+ completion_verification_result=completion_verification,+ last_artifact_health_blocker_reason=None,+ last_run_blocks_workflow_run_id=None,+ last_run_outcome=None,+ )++def test_verification_satisfaction_no_cvr_uses_prior_proxy() -> None:- assert _verification_satisfaction(True, False, "completed", None) == (True, True)- assert _verification_satisfaction(True, True, "completed", None) == (False, False)- assert _verification_satisfaction(False, False, None, None) == (None, None)+ ctx = _verification_satisfaction_ctx()+ assert _verification_satisfaction(ctx, True, False, "completed", None) == (True, True)+ assert _verification_satisfaction(ctx, True, True, "completed", None) == (False, False)+ assert _verification_satisfaction(ctx, False, False, None, None) == (None, None)def test_verification_satisfaction_evaluated_drives_contract_signal() -> None:- assert _verification_satisfaction(True, False, "completed", _evaluated(("c0", True))) == (True, True)- _, contract = _verification_satisfaction(True, False, "completed", _evaluated(("c0", False)))+ satisfied = _evaluated(("c0", True))+ unsatisfied = _evaluated(("c0", False))+ assert _verification_satisfaction(+ _verification_satisfaction_ctx(satisfied), True, False, "completed", satisfied+ ) == (+ True,+ True,+ )+ _, contract = _verification_satisfaction(+ _verification_satisfaction_ctx(unsatisfied), True, False, "completed", unsatisfied+ )assert contract is Falsedef test_verification_satisfaction_unavailable_fails_closed() -> None:- _, contract = _verification_satisfaction(True, False, "completed", CompletionVerificationResult("unavailable"))+ unavailable = CompletionVerificationResult("unavailable")+ _, contract = _verification_satisfaction(+ _verification_satisfaction_ctx(unavailable), True, False, "completed", unavailable+ )assert contract is False@@ -2458,6 +2540,29 @@ def test_tool_visible_result_keeps_mid_build_run_visible_success() -> None:assert visible["ok"] is True+def test_tool_visible_result_keeps_committed_same_run_success_after_later_contradiction() -> None:+ ctx = _ctx_with_blocks("extraction")+ result = _clean_success_result()++ _record_run_blocks_result(ctx, result, completion_verification=_evaluated(("c0", True)))+ visible = _tool_visible_result_after_completion_verification(ctx, result, _contradicted("c0"))++ assert visible["ok"] is True+ assert visible is result+ assert "failure_categories" not in visible["data"]+ assert "completion_verification" not in visible["data"]+++def test_tool_visible_result_downgrades_first_pass_contradiction_without_committed_outcome() -> None:+ ctx = _ctx_with_blocks("extraction")+ result = _clean_success_result()++ visible = _tool_visible_result_after_completion_verification(ctx, result, _contradicted("c0"))++ assert visible["ok"] is False+ assert visible["data"]["failure_categories"][0]["category"] == "OUTCOME_UNVERIFIED"++def test_record_run_blocks_keeps_building_on_mid_build_no_evidence() -> None:ctx = _ctx_with_blocks("goto_url", "navigation")_record_run_blocks_result(ctx, _clean_success_result(), completion_verification=_evaluated(("c0", False)))@@ -2494,6 +2599,138 @@ def test_record_run_blocks_demonstrated_when_lone_definition_abstention_with_conassert verified_goal_satisfied_context(ctx) is True+def test_committed_same_run_outcome_survives_later_contradictory_overwrite() -> None:+ ctx = _ctx_with_blocks("extraction")+ verification = _mixed(+ CriterionVerdict(criterion_id="c0", state="satisfied", reason_code="evidence_confirms"),+ CriterionVerdict(criterion_id="c1", state="unknown", reason_code="definition_parameters_absent"),+ )++ recorded = _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=verification)++ assert recorded == RecordedRunOutcome(verdict="demonstrated", workflow_run_id="wr_x")+ assert outcome_fully_verified(ctx) is True++ ctx.completion_verification_result = _mixed(+ CriterionVerdict(criterion_id="c0", state="unsatisfied", reason_code="evidence_contradicts"),+ CriterionVerdict(criterion_id="c1", state="unknown", reason_code="definition_parameters_absent"),+ )++ assert outcome_fully_verified(ctx) is True+ assert verified_goal_satisfied_context(ctx) is True+++def test_first_pass_contradiction_without_committed_run_outcome_still_fails() -> None:+ ctx = _ctx_with_blocks("extraction")++ recorded = _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=_contradicted("c0"))++ assert recorded is not None+ assert recorded.verdict == "not_demonstrated"+ assert outcome_fully_verified(ctx) is False+ assert verified_goal_satisfied_context(ctx) is False+++def test_same_run_contradiction_after_committed_outcome_does_not_churn() -> None:+ ctx = _ctx_with_blocks("extraction")++ _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=_evaluated(("c0", True)))+ recorded = _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=_contradicted("c0"))++ assert recorded == RecordedRunOutcome(verdict="demonstrated", workflow_run_id="wr_x")+ assert ctx.last_test_suspicious_success is False+ assert ctx.last_test_failure_reason is None+ assert ctx.last_full_workflow_test_ok is True+ assert outcome_fully_verified(ctx) is True+++def test_missing_run_id_does_not_preserve_committed_same_run_outcome() -> None:+ ctx = _ctx_with_blocks("extraction")+ malformed_result = _clean_success_result()+ malformed_result["data"].pop("workflow_run_id")++ _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=_evaluated(("c0", True)))+ recorded = _record_run_blocks_result(ctx, malformed_result, completion_verification=_contradicted("c0"))++ assert recorded is not None+ assert recorded.verdict == "not_demonstrated"+ assert ctx.last_run_outcome == recorded+ assert outcome_fully_verified(ctx) is False+++def test_committed_same_run_outcome_surfaces_verified_workflow_after_later_contradiction() -> None:+ ctx = _ctx_with_blocks("extraction")+ ctx.last_workflow_yaml = "workflow: {}"++ _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=_evaluated(("c0", True)))+ ctx.completion_verification_result = _contradicted("c0")++ assert _completion_contract_not_violated(ctx) is True+ assert _verified_workflow_or_none(ctx) == (ctx.last_workflow, "workflow: {}")+++def test_first_pass_contradiction_does_not_surface_verified_workflow() -> None:+ ctx = _ctx_with_blocks("extraction")+ ctx.last_workflow_yaml = "workflow: {}"++ _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=_contradicted("c0"))++ assert _completion_contract_not_violated(ctx) is False+ assert _verified_workflow_or_none(ctx) == (None, None)+++def test_different_run_id_committed_outcome_does_not_surface_verified_workflow() -> None:+ ctx = _ctx_with_blocks("extraction")+ ctx.last_workflow_yaml = "workflow: {}"+ ctx.last_test_ok = True+ ctx.last_full_workflow_test_ok = True+ ctx.last_run_blocks_workflow_run_id = "wr_new"+ ctx.last_run_outcome = RecordedRunOutcome(verdict="demonstrated", workflow_run_id="wr_prior")+ ctx.completion_verification_result = _contradicted("c0")++ assert outcome_fully_verified(ctx) is False+ assert _completion_contract_not_violated(ctx) is False+ assert _verified_workflow_or_none(ctx) == (None, None)+++def test_missing_current_run_id_does_not_surface_verified_workflow_from_prior_outcome() -> None:+ ctx = _ctx_with_blocks("extraction")+ ctx.last_workflow_yaml = "workflow: {}"+ ctx.last_test_ok = True+ ctx.last_full_workflow_test_ok = True+ ctx.last_run_blocks_workflow_run_id = None+ ctx.last_run_outcome = RecordedRunOutcome(verdict="demonstrated", workflow_run_id="wr_prior")+ ctx.completion_verification_result = _contradicted("c0")++ assert outcome_fully_verified(ctx) is False+ assert _completion_contract_not_violated(ctx) is False+ assert _verified_workflow_or_none(ctx) == (None, None)+++def test_artifact_health_blocks_verified_workflow_surfacing_with_committed_outcome() -> None:+ ctx = _ctx_with_blocks("extraction")+ ctx.last_workflow_yaml = "workflow: {}"++ _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=_evaluated(("c0", True)))+ ctx.last_artifact_health_blocker_reason = "Code block failed with SyntaxError."++ assert outcome_fully_verified(ctx) is False+ assert _completion_contract_not_violated(ctx) is False+ assert _verified_workflow_or_none(ctx) == (None, None)+++def test_unavailable_verification_without_committed_outcome_fails_closed_for_surfacing() -> None:+ ctx = _ctx_with_blocks("extraction")+ ctx.last_workflow_yaml = "workflow: {}"+ ctx.last_test_ok = True+ ctx.last_full_workflow_test_ok = True+ ctx.completion_verification_result = CompletionVerificationResult(status="unavailable")++ assert outcome_fully_verified(ctx) is False+ assert _completion_contract_not_violated(ctx) is False+ assert _verified_workflow_or_none(ctx) == (None, None)++def _goto_only_result() -> dict:return {"ok": True,@@ -3166,6 +3403,17 @@ def test_artifact_health_blocks_fully_satisfied_failed_run() -> None:assert _verified_workflow_or_none(ctx) == (None, None)+def test_artifact_health_blocks_committed_same_run_outcome() -> None:+ ctx = _ctx_with_blocks("extraction")+ verification = _evaluated(("c0", True))++ _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=verification)+ ctx.last_artifact_health_blocker_reason = "Code block failed with SyntaxError."++ assert outcome_fully_verified(ctx) is False+ assert verified_goal_satisfied_context(ctx) is False++@pytest.mark.asyncioasync def test_maybe_run_completion_verification_runs_on_canceled_run(monkeypatch: pytest.MonkeyPatch) -> None:async def handler(**_: object) -> dict:diff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.pyindex 61f57065b..60dc3d9f7 100644--- a/tests/unit/test_copilot_diagnosis_repair_contract.py+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py@@ -18,9 +18,11 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (RepairNextAction,build_diagnosis_repair_contract,)+from skyvern.forge.sdk.copilot.enforcement import latest_diagnosis_contract_satisfies_goalfrom skyvern.forge.sdk.copilot.run_outcome import (TERMINAL_CHALLENGE_BLOCKER_REASON_CODE,TERMINAL_CHALLENGE_RUN_OUTCOME_REASON_CODE,+ RecordedRunOutcome,)from skyvern.forge.sdk.copilot.runtime_authoring_repair import (finalize_runtime_authoring_repair_context_from_page_observation,@@ -57,6 +59,14 @@ def _satisfied_completion_verification() -> CompletionVerificationResult:)+def _contradictory_completion_verification() -> CompletionVerificationResult:+ return CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["c0"],+ verdicts=[CriterionVerdict(criterion_id="c0", state="unsatisfied", reason_code="evidence_contradicts")],+ )++def _clean_completed_result() -> dict[str, object]:return {"ok": True,@@ -969,6 +979,46 @@ def test_clean_run_with_satisfied_completion_verification_has_no_repair_or_blockassert contract.verification_result.remaining_blocker is None+def test_committed_same_run_outcome_satisfies_diagnosis_after_later_contradiction() -> None:+ ctx = _ctx()+ ctx.last_run_blocks_workflow_run_id = "wr_clean"+ ctx.last_run_outcome = RecordedRunOutcome(verdict="demonstrated", workflow_run_id="wr_clean")+ ctx.completion_verification_result = _contradictory_completion_verification()++ contract = build_diagnosis_repair_contract(+ source_tool="update_and_run_blocks",+ result=_clean_completed_result(),+ ctx=ctx,+ workflow_updated=True,+ )+ ctx.latest_diagnosis_repair_contract = contract++ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.NO_FAILURE+ assert contract.repair_decision.next_action == RepairNextAction.NO_CHANGE+ assert contract.verification_result.user_goal_satisfied is True+ assert contract.verification_result.completion_contract_satisfied is True+ assert latest_diagnosis_contract_satisfies_goal(ctx) is True+++def test_first_pass_contradiction_does_not_satisfy_latest_diagnosis_contract() -> None:+ ctx = _ctx()+ ctx.completion_verification_result = _contradictory_completion_verification()++ contract = build_diagnosis_repair_contract(+ source_tool="update_and_run_blocks",+ result=_clean_completed_result(),+ ctx=ctx,+ workflow_updated=True,+ )+ ctx.latest_diagnosis_repair_contract = contract++ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.SUSPICIOUS_SUCCESS+ assert contract.repair_decision.next_action == RepairNextAction.REPAIR+ assert contract.verification_result.user_goal_satisfied is False+ assert contract.verification_result.completion_contract_satisfied is False+ assert latest_diagnosis_contract_satisfies_goal(ctx) is False++def test_no_change_contracts_do_not_carry_remaining_blocker() -> None:satisfied_ctx = _ctx()satisfied_ctx.completion_verification_result = _satisfied_completion_verification()diff --git a/tests/unit/test_copilot_run_outcome_event.py b/tests/unit/test_copilot_run_outcome_event.pyindex c5fbc90bf..207a8d362 100644--- a/tests/unit/test_copilot_run_outcome_event.py+++ b/tests/unit/test_copilot_run_outcome_event.py@@ -18,11 +18,14 @@ from skyvern.forge.sdk.copilot import tools as copilot_toolsfrom skyvern.forge.sdk.copilot.agent import _build_narrative_payloadfrom skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdictfrom skyvern.forge.sdk.copilot.context import CopilotContext+from skyvern.forge.sdk.copilot.enforcement import outcome_fully_verifiedfrom skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicyfrom skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome, run_outcome_display_reasonfrom skyvern.forge.sdk.copilot.tools import run_executionfrom skyvern.forge.sdk.copilot.tools.run_execution import (+ _adjudicated_run_outcome,_record_run_blocks_result,+ _stash_recorded_run_outcome,_verify_and_record_run_blocks_result,)from skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotRunOutcomeUpdate@@ -136,6 +139,48 @@ def _evaluated(satisfied: bool) -> CompletionVerificationResult:return CompletionVerificationResult(status="evaluated", criterion_ids=["c0"], verdicts=[verdict])+def _mixed_observed_reach_state_with_reperception_contradiction() -> CompletionVerificationResult:+ return CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["c_reach", "c_reperception"],+ verdicts=[+ CriterionVerdict(+ criterion_id="c_reach",+ state="satisfied",+ reason_code="evidence_confirms",+ evidence_ref="observed_end_state_url",+ ),+ CriterionVerdict(+ criterion_id="c_reperception",+ state="unsatisfied",+ reason_code="evidence_contradicts",+ evidence_ref="scout_synthesized_browser_steps_output",+ ),+ ],+ )+++def _mixed_observed_reach_state_with_requested_output_contradiction() -> CompletionVerificationResult:+ return CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["c_reach", "c_requested_output"],+ verdicts=[+ CriterionVerdict(+ criterion_id="c_reach",+ state="satisfied",+ reason_code="evidence_confirms",+ evidence_ref="observed_end_state_url",+ ),+ CriterionVerdict(+ criterion_id="c_requested_output",+ state="unsatisfied",+ reason_code="evidence_contradicts",+ evidence_ref="block_outputs:search_registry_person.confirmation_number",+ ),+ ],+ )++def _run_outcome_frames(stream: _FakeStream) -> list[WorkflowCopilotRunOutcomeUpdate]:return [frame for frame in stream.sent if isinstance(frame, WorkflowCopilotRunOutcomeUpdate)]@@ -406,6 +451,94 @@ def test_recorded_run_outcome_carries_producing_workflow_run_id() -> None:assert ctx.last_run_outcome.workflow_run_id == "wr_test"+def test_recorded_run_outcome_commits_same_adjudication_reach_state_over_reperception_contradiction() -> None:+ ctx = _ctx([_code_block("search_registry_person", {"records": []})])++ outcome = _record_run_blocks_result(+ ctx,+ _run_result([_code_block("search_registry_person", {"records": []})]),+ completion_verification=_mixed_observed_reach_state_with_reperception_contradiction(),+ )++ assert outcome == RecordedRunOutcome(verdict="demonstrated", workflow_run_id="wr_test")+ assert ctx.last_run_outcome == outcome+ assert ctx.last_full_workflow_test_ok is True+ assert ctx.last_test_suspicious_success is False+ assert ctx.last_test_failure_reason is None+ assert outcome_fully_verified(ctx) is True+++def test_recorded_run_outcome_does_not_commit_requested_output_contradiction() -> None:+ ctx = _ctx([_code_block("search_registry_person", {"records": []})])++ outcome = _record_run_blocks_result(+ ctx,+ _run_result([_code_block("search_registry_person", {"records": []})]),+ completion_verification=_mixed_observed_reach_state_with_requested_output_contradiction(),+ )++ assert outcome is not None+ assert outcome.verdict == "not_demonstrated"+ assert ctx.last_run_outcome == outcome+ assert outcome_fully_verified(ctx) is False+++def test_adjudication_keeps_committed_same_run_demonstrated_outcome() -> None:+ ctx = _ctx([_code_block("search_registry_person", {"records": []})])+ ctx.last_run_blocks_workflow_run_id = "wr_test"+ committed = _stash_recorded_run_outcome(ctx, RecordedRunOutcome(verdict="demonstrated"))++ assert committed == RecordedRunOutcome(verdict="demonstrated", workflow_run_id="wr_test")++ adjudicated = _adjudicated_run_outcome(ctx, _evaluated(satisfied=False))+ stashed = _stash_recorded_run_outcome(+ ctx,+ RecordedRunOutcome(+ verdict="not_demonstrated",+ reason_code="outcome_not_demonstrated",+ workflow_run_id="wr_test",+ ),+ )++ assert adjudicated == committed+ assert stashed == committed+ assert ctx.last_run_outcome == committed+++def test_adjudication_does_not_keep_committed_outcome_for_new_run() -> None:+ ctx = _ctx([_code_block("search_registry_person", {"records": []})])+ ctx.last_run_outcome = RecordedRunOutcome(verdict="demonstrated", workflow_run_id="wr_prior")+ ctx.last_run_blocks_workflow_run_id = "wr_test"++ adjudicated = _adjudicated_run_outcome(ctx, _evaluated(satisfied=False))++ assert adjudicated.verdict == "not_demonstrated"+++async def test_missing_run_id_does_not_emit_committed_demonstrated_outcome(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ result = _clean_run_result()+ result["data"].pop("workflow_run_id")+ ctx = _ctx(result["data"]["blocks"])+ ctx.last_run_outcome = RecordedRunOutcome(verdict="demonstrated", workflow_run_id="wr_test")+ ctx.last_run_blocks_workflow_run_id = "wr_test"++ async def _stub_verification(*args: Any, **kwargs: Any) -> CompletionVerificationResult:+ return _evaluated(satisfied=False)++ monkeypatch.setattr(run_execution, "_maybe_run_completion_verification", _stub_verification)++ await _verify_and_record_run_blocks_result(ctx, result, time.monotonic())++ frames = _run_outcome_frames(ctx.stream) # type: ignore[arg-type]+ assert [frame.verdict for frame in frames] == ["evaluating", "not_demonstrated"]+ assert frames[-1].workflow_run_id != "wr_test"+ assert ctx.last_run_outcome is not None+ assert ctx.last_run_outcome.verdict == "not_demonstrated"++def test_both_consumers_route_through_single_producer() -> None:source = inspect.getsource(copilot_tools)assert source.count("await _verify_and_record_run_blocks_result(") == 2