Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-11642 repair read-only extraction verification (#6996)
Andrew Neilsonfb6ff274ae9f
diff --git a/skyvern/forge/sdk/copilot/completion_criteria_store.py b/skyvern/forge/sdk/copilot/completion_criteria_store.pyindex aabae500d..f56430597 100644--- a/skyvern/forge/sdk/copilot/completion_criteria_store.py+++ b/skyvern/forge/sdk/copilot/completion_criteria_store.py@@ -116,8 +116,9 @@ class PersistencePlan:def criteria_to_json(criteria: tuple[CompletionCriterion, ...] | list[CompletionCriterion]) -> list[dict[str, Any]]:- return [- {+ items: list[dict[str, Any]] = []+ for criterion in criteria:+ item = {"id": criterion.id,"outcome": criterion.outcome,"contingent_on": criterion.contingent_on,@@ -134,8 +135,10 @@ def criteria_to_json(criteria: tuple[CompletionCriterion, ...] | list[Completion"classification_output_key": criterion.classification_output_key,"expected_classification": criterion.expected_classification,}- for criterion in criteria- ]+ if criterion.requested_output_corroborator:+ item["requested_output_corroborator"] = True+ items.append(item)+ return itemsdef criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:@@ -195,6 +198,7 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:terminal_action_family=cast(TerminalActionFamily | None, terminal_action_family),classification_output_key=classification_output_key,expected_classification=expected_classification,+ requested_output_corroborator=bool(item.get("requested_output_corroborator")),))return tuple(criteria)diff --git a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.pyindex add505051..f0fa05a7f 100644--- a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py+++ b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py@@ -16,6 +16,7 @@ from skyvern.forge.sdk.copilot.composition_evidence import interactive_challengefrom skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContextfrom skyvern.forge.sdk.copilot.failure_tracking import (ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,+ ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE,RepairRootCauseIdentity,compute_repair_root_cause_signature,)@@ -511,6 +512,8 @@ def _failure_type(ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY in categoriesor data.get("active_run_terminal_evidence_detected") is True):+ if outcome_verified and _active_run_terminal_evidence_reason_code(data):+ return DiagnosisFailureType.NO_FAILUREreturn DiagnosisFailureType.ACTIVE_RUN_TERMINAL_EVIDENCEif (category_set & _PRE_RUN_CREDENTIAL_FAILURE_CATEGORIES@@ -538,6 +541,10 @@ def _failure_type(return DiagnosisFailureType.FAILED_RUN if result.get("ok") is False else DiagnosisFailureType.UNKNOWN+def _active_run_terminal_evidence_reason_code(data: dict[str, Any]) -> bool:+ return _safe_str(data.get("active_run_terminal_evidence_reason_code")) == ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE++def _next_action(failure_type: DiagnosisFailureType,ctx: CopilotContext,diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.pyindex 840502411..d324cf77c 100644--- a/skyvern/forge/sdk/copilot/request_policy.py+++ b/skyvern/forge/sdk/copilot/request_policy.py@@ -256,6 +256,7 @@ class CompletionCriterion:terminal_action_family: TerminalActionFamily | None = Noneclassification_output_key: str | None = Noneexpected_classification: ClassificationTarget | None = None+ requested_output_corroborator: bool = False@dataclass@@ -1102,6 +1103,8 @@ def _generic_completion_criterion(criterion: CompletionCriterion) -> bool:def _criterion_drop_priority(criterion: CompletionCriterion, requested_output_paths: set[str]) -> int:if criterion.output_path in requested_output_paths:return 0+ if criterion.requested_output_corroborator:+ return 0if criterion.level == "definition":return 0if is_fallback_floor_criterion(criterion):@@ -1155,6 +1158,110 @@ def _preserve_after_requested_output_canonicalization(return True+def _non_requested_output_run_corroborator(criterion: CompletionCriterion) -> bool:+ return (+ criterion.level == "run"+ and criterion.kind == "outcome"+ and not criterion.method_mandated+ and criterion.output_path is None+ )+++def _validation_classification_output_path(output_path: str | None) -> bool:+ return (+ output_path in _VALIDATION_CLASSIFICATION_BOOLEAN_OUTPUT_TARGETS+ or output_path in _VALIDATION_CLASSIFICATION_LABEL_OUTPUT_TARGETS+ )+++def completion_criterion_requires_active_run_terminal_monitor(criterion: CompletionCriterion) -> bool:+ if criterion.requested_output_corroborator:+ return False+ if (+ criterion.output_path+ and criterion.level != "definition"+ and not criterion.method_mandated+ and criterion.kind != "validation_classification"+ ):+ return False+ return True+++def _source_requested_output_corroborator(+ criteria: list[CompletionCriterion],+ requested_fields: list[str],+ requested_output_paths: set[str],+) -> CompletionCriterion | None:+ for criterion in criteria:+ if criterion.level != "run" or criterion.kind != "outcome" or criterion.method_mandated:+ continue+ if _validation_classification_output_path(criterion.output_path):+ continue+ if criterion.output_path in requested_output_paths or _criterion_text_covers_any_requested_output(+ criterion, requested_fields+ ):+ return replace(+ criterion,+ output_path=None,+ expected_output_value=None,+ expected_output_shape=None,+ requested_output_corroborator=True,+ )+ for criterion in criteria:+ if is_fallback_floor_base_criterion(criterion):+ return replace(criterion, requested_output_corroborator=True)+ return None+++def _requested_output_corroborator_id(source_id: str, used_ids: set[str]) -> str:+ base_id = f"{source_id}__requested_output_corroborator"+ if base_id not in used_ids:+ return base_id+ suffix = 2+ while f"{base_id}_{suffix}" in used_ids:+ suffix += 1+ return f"{base_id}_{suffix}"+++def _apply_classifier_typed_requested_output_corroborators(policy: RequestPolicy) -> None:+ if any(_non_requested_output_run_corroborator(criterion) for criterion in policy.completion_criteria):+ return+ source_criteria = [+ criterion+ for criterion in policy.completion_criteria+ if criterion.level == "run"+ and criterion.kind == "outcome"+ and not criterion.method_mandated+ and criterion.output_path is not None+ and not _validation_classification_output_path(criterion.output_path)+ and not criterion.id.startswith(_REQUESTED_OUTPUT_CRITERION_ID_PREFIX)+ ]+ if not source_criteria:+ return++ used_ids = {criterion.id for criterion in policy.completion_criteria}+ corroborators: list[CompletionCriterion] = []+ for criterion in source_criteria:+ corroborator_id = _requested_output_corroborator_id(criterion.id, used_ids)+ used_ids.add(corroborator_id)+ corroborators.append(+ replace(+ criterion,+ id=corroborator_id,+ output_path=None,+ expected_output_value=None,+ expected_output_shape=None,+ requested_output_corroborator=True,+ )+ )++ requested_output_paths = {criterion.output_path for criterion in source_criteria if criterion.output_path}+ policy.completion_criteria = _cap_completion_criteria(+ policy.completion_criteria + corroborators,+ requested_output_paths,+ )++def _apply_requested_output_completion_criteria(policy: RequestPolicy, user_message: str, aliases: dict[str, str] | None = None) -> None:@@ -1237,8 +1344,17 @@ def _apply_requested_output_completion_criteria()for _field_name, output_path, field_label in requested_specs]+ criteria = preserved_criteria + canonical_requested_criteria+ if not any(_non_requested_output_run_corroborator(criterion) for criterion in criteria):+ corroborator = _source_requested_output_corroborator(+ policy.completion_criteria,+ requested_fields,+ requested_output_paths,+ )+ if corroborator is not None:+ criteria = preserved_criteria + [corroborator] + canonical_requested_criteriapolicy.completion_criteria = _cap_completion_criteria(- preserved_criteria + canonical_requested_criteria,+ criteria,requested_output_paths,)@@ -1320,6 +1436,8 @@ def _render_active_criteria_for_prompt(criteria: list[CompletionCriterion] | Nonitem["classification_output_key"] = criterion.classification_output_keyif criterion.expected_classification is not None:item["expected_classification"] = criterion.expected_classification+ if criterion.requested_output_corroborator:+ item["requested_output_corroborator"] = Truerendered.append(item)return json.dumps(rendered)@@ -1395,6 +1513,7 @@ def _classifier_fallback_policy(completion_contract_status="present" if fallback_criteria else "unknown",)_apply_requested_output_completion_criteria(policy, user_message, requested_output_path_aliases)+ _apply_classifier_typed_requested_output_corroborators(policy)if policy.graded_completion_criteria():policy.completion_contract_status = "present"return policy@@ -1637,6 +1756,7 @@ async def _classify_request(policy.raw_secret_evidence = None_apply_requested_output_completion_criteria(policy, user_message, requested_output_path_aliases)_apply_validation_classification_completion_criteria(policy)+ _apply_classifier_typed_requested_output_corroborators(policy)policy.completion_contract_status = ("present" if policy.completion_contract or policy.graded_completion_criteria() else "absent")diff --git a/skyvern/forge/sdk/copilot/tools/composition_capture.py b/skyvern/forge/sdk/copilot/tools/composition_capture.pyindex f6dd3533b..a60db52d0 100644--- a/skyvern/forge/sdk/copilot/tools/composition_capture.py+++ b/skyvern/forge/sdk/copilot/tools/composition_capture.py@@ -25,9 +25,13 @@ from skyvern.forge.sdk.copilot.composition_evidence import (parse_composition_html,)from skyvern.forge.sdk.copilot.context import CopilotContext-from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY+from skyvern.forge.sdk.copilot.failure_tracking import (+ ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,+ ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE,+)from skyvern.forge.sdk.copilot.llm_config import resolve_fast_copilot_handlerfrom skyvern.forge.sdk.copilot.loop_detection import record_tool_step_result_for_ctx+from skyvern.forge.sdk.copilot.request_policy import completion_criterion_requires_active_run_terminal_monitorfrom skyvern.forge.sdk.copilot.runtime_authoring_repair import (finalize_runtime_authoring_repair_context_from_page_observation,)@@ -79,7 +83,10 @@ async def _active_run_terminal_monitor_enabled(copilot_ctx: Any) -> bool:return Falseif not getattr(copilot_ctx, "discovery_mcp_server", None):return False- if not _completion_verification_criteria(copilot_ctx):+ criteria = _completion_verification_criteria(copilot_ctx)+ if not criteria:+ return False+ if not any(completion_criterion_requires_active_run_terminal_monitor(criterion) for criterion in criteria):return Falsereturn await _completion_verification_handler(copilot_ctx) is not None@@ -194,6 +201,7 @@ def _active_run_terminal_evidence_result("current_url": observed_url,"page_title": observed_title,"active_run_terminal_evidence_detected": True,+ "active_run_terminal_evidence_reason_code": ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE,"active_run_terminal_evidence_sample_index": getattr(sample, "sample_index", None),"full_workflow_verified": False,"current_page_evidence": getattr(sample, "page_evidence", None),diff --git a/skyvern/forge/sdk/copilot/turn_halt.py b/skyvern/forge/sdk/copilot/turn_halt.pyindex 0b0fd621b..459daf1f0 100644--- a/skyvern/forge/sdk/copilot/turn_halt.py+++ b/skyvern/forge/sdk/copilot/turn_halt.py@@ -90,6 +90,8 @@ _INVOLUNTARY_BLOCKER_REASON_CODES = (| _SCHEMA_INCOMPATIBILITY_REASON_CODES| frozenset({REPAIR_CEILING_REASON_CODE}))+_VERIFIED_SUPPRESSIBLE_ACTIVE_TERMINAL_REASON_CODES = frozenset({ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE})+_VERIFIED_SUPPRESSIBLE_ACTIVE_TERMINAL_SOURCES = frozenset({"run_execution"})@dataclass(frozen=True)@@ -183,9 +185,21 @@ def raise_if_turn_halt(ctx: Any, *, verified: bool = False) -> None:halt = getattr(ctx, "turn_halt", None)if not isinstance(halt, TurnHalt):return- if verified and halt.kind in _INVOLUNTARY_TURN_HALT_KINDS:+ suppressible_reason_codes = _INVOLUNTARY_BLOCKER_REASON_CODES+ if (+ halt.kind == TurnHaltKind.ACTIVE_TERMINAL_CHALLENGE+ and halt.blocker_signal is not None+ and halt.blocker_signal.internal_reason_code in _VERIFIED_SUPPRESSIBLE_ACTIVE_TERMINAL_REASON_CODES+ and halt.extra.get("source") in _VERIFIED_SUPPRESSIBLE_ACTIVE_TERMINAL_SOURCES+ ):+ suppressible_reason_codes = (+ _INVOLUNTARY_BLOCKER_REASON_CODES | _VERIFIED_SUPPRESSIBLE_ACTIVE_TERMINAL_REASON_CODES+ )+ elif halt.kind not in _INVOLUNTARY_TURN_HALT_KINDS:+ suppressible_reason_codes = frozenset()+ if verified and suppressible_reason_codes:ctx.turn_halt = None- clear_tool_blocker_signals_for_reason_codes(ctx, _INVOLUNTARY_BLOCKER_REASON_CODES)+ clear_tool_blocker_signals_for_reason_codes(ctx, suppressible_reason_codes)LOG.info("copilot turn halt suppressed by verified outcome",**turn_halt_to_trace_data(halt),diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex d3c68b898..859e14734 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -55,6 +55,7 @@ from skyvern.forge.sdk.copilot.enforcement import (verified_goal_claim_authorized,verified_goal_satisfied_context,)+from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODEfrom skyvern.forge.sdk.copilot.recoverable_failure import build_recoverable_failurefrom skyvern.forge.sdk.copilot.request_policy import (_MAX_COMPLETION_CRITERIA,@@ -73,6 +74,7 @@ from skyvern.forge.sdk.copilot.run_outcome import TERMINAL_CHALLENGE_BLOCKER_REAfrom skyvern.forge.sdk.copilot.tools.completion import _completion_verification_criteriafrom skyvern.forge.sdk.copilot.turn_context import TranscriptContext, TurnContextOmission, TurnContextPacketfrom skyvern.forge.sdk.copilot.turn_halt import (+ CopilotTurnHalt,TurnHalt,TurnHaltKind,raise_if_turn_halt,@@ -1057,6 +1059,48 @@ class TestVerifiedGoalSatisfiedStop:assert result.user_response == _VERIFIED_WORKFLOW_SUCCESS_REPLYassert result.proposal_disposition != "no_proposal"+ def test_verified_turn_halt_suppresses_active_run_terminal_evidence_only(self) -> None:+ ctx = _ctx()+ signal = CopilotToolBlockerSignal(+ blocker_kind="tool_error",+ agent_steering_text="active run reached requested state",+ user_facing_reason="active run reached requested state",+ recovery_hint="report_blocker_to_user",+ internal_reason_code=ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE,+ blocked_tool="update_and_run_blocks",+ )+ ctx.blocker_signal = signal+ ctx.latest_tool_blocker_signal = signal+ ctx.tool_blocker_signals.append(signal)+ ctx.turn_halt = TurnHalt(+ kind=TurnHaltKind.ACTIVE_TERMINAL_CHALLENGE,+ blocker_signal=signal,+ extra={"source": "run_execution"},+ )++ raise_if_turn_halt(ctx, verified=True)++ assert not isinstance(ctx.turn_halt, TurnHalt)+ assert ctx.blocker_signal is None+ assert ctx.latest_tool_blocker_signal is None+ assert ctx.tool_blocker_signals == []++ def test_verified_turn_halt_keeps_terminal_challenge_blocker(self) -> None:+ ctx = _ctx()+ signal = CopilotToolBlockerSignal(+ blocker_kind="tool_error",+ agent_steering_text="stop on terminal challenge",+ user_facing_reason="The site requires human verification.",+ recovery_hint="stop",+ internal_reason_code=TERMINAL_CHALLENGE_BLOCKER_REASON_CODE,+ blocked_tool="update_and_run_blocks",+ )+ ctx.blocker_signal = signal+ ctx.turn_halt = TurnHalt(kind=TurnHaltKind.ACTIVE_TERMINAL_CHALLENGE, blocker_signal=signal)++ with pytest.raises(CopilotTurnHalt):+ raise_if_turn_halt(ctx, verified=True)+@pytest.mark.asyncioasync def test_wrapped_exception_resolver_renders_success_over_involuntary_halt(self) -> None:ctx = _ctx(@@ -1304,6 +1348,35 @@ class TestVerifiedGoalSatisfiedStop:assert result.narrative_payload is not Noneassert result.narrative_payload["verifiedSuccess"] is False+ def test_corroborated_structural_abstention_avoids_built_unverified_terminal(self) -> None:+ ctx = _ctx(+ last_test_ok=True,+ last_full_workflow_test_ok=True,+ latest_diagnosis_repair_contract=_unverified_no_repair_contract(),+ completion_verification_result=CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["c0", "c0__requested_output_corroborator"],+ verdicts=[+ CriterionVerdict(+ criterion_id="c0",+ state="unsatisfied",+ reason_code="structurally_abstained",+ evidence_ref="block_outputs:extract_first_three_quotes.quotes",+ output_path="output.quotes",+ grounding_mode="missing",+ ),+ CriterionVerdict(+ criterion_id="c0__requested_output_corroborator",+ state="satisfied",+ reason_code="evidence_confirms",+ ),+ ],+ ),+ )++ assert verified_goal_satisfied_context(ctx) is True+ assert built_unverified_repair_inert_context(ctx) is False+@pytest.mark.asyncioasync def test_goal_satisfied_exit_result_carries_outcome_adjudication(self) -> None:from skyvern.forge.sdk.copilot.agent import _build_goal_satisfied_exit_resultdiff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.pyindex 63d791acf..8cee28950 100644--- a/tests/unit/test_copilot_completion_verification.py+++ b/tests/unit/test_copilot_completion_verification.py@@ -4,6 +4,7 @@ import asyncioimport textwrapimport timefrom collections.abc import Awaitable, Callable+from dataclasses import replacefrom types import SimpleNamespacefrom typing import Any@@ -46,6 +47,7 @@ from skyvern.forge.sdk.copilot.completion_verification import ()from skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.copilot.diagnosis_repair_contract import (+ DiagnosisFailureType,DiagnosisInput,DiagnosisRepairContract,DiagnosisResult,@@ -53,12 +55,14 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (RepairNextAction,VerificationResult,_verification_satisfaction,+ build_diagnosis_repair_contract,)from skyvern.forge.sdk.copilot.enforcement import (built_unverified_repair_inert_context,outcome_fully_verified,verified_goal_satisfied_context,)+from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODEfrom skyvern.forge.sdk.copilot.hooks import _tool_completion_satisfies_turnfrom skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTargetfrom skyvern.forge.sdk.copilot.request_policy import (@@ -92,6 +96,7 @@ from skyvern.forge.sdk.copilot.tools.completion import (_artifact_health_blocker_from_result,_reconcile_download_completion_criterion,)+from skyvern.forge.sdk.copilot.tools.composition_capture import _active_run_terminal_monitor_enableddef _criterion(@@ -109,6 +114,7 @@ def _criterion(expected_output_shape: str | None = None,classification_output_key: str | None = None,expected_classification: str | bool | None = None,+ requested_output_corroborator: bool = False,) -> CompletionCriterion:return CompletionCriterion(id=cid,@@ -124,6 +130,7 @@ def _criterion(expected_output_shape=expected_output_shape, # type: ignore[arg-type]classification_output_key=classification_output_key,expected_classification=expected_classification,+ requested_output_corroborator=requested_output_corroborator,)@@ -482,6 +489,30 @@ def test_structural_requested_output_abstention_without_typed_corroboration_getsassert result.is_fully_satisfied() is False+def test_structural_requested_output_abstention_with_classifier_corroborator_satisfies() -> None:+ result = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["c0", "c0__requested_output_corroborator"],+ verdicts=[+ CriterionVerdict(+ criterion_id="c0",+ state="unsatisfied",+ reason_code="structurally_abstained",+ evidence_ref="block_outputs:extract_first_three_quotes.quotes",+ output_path="output.quotes",+ grounding_mode="missing",+ ),+ CriterionVerdict(+ criterion_id="c0__requested_output_corroborator",+ state="satisfied",+ reason_code="evidence_confirms",+ ),+ ],+ )++ assert result.is_fully_satisfied() is True++def test_requested_output_no_evidence_with_observed_end_state_still_blocks() -> None:result = _mixed(CriterionVerdict(@@ -2230,6 +2261,169 @@ def test_visual_prompt_requests_outcome_relevant_page_state() -> None:assert "human-verification" in prompt+async def test_active_run_terminal_monitor_skips_requested_output_only_criteria(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def handler(**_: object) -> dict:+ return {}++ async def fake_completion_verification_handler(_ctx: object) -> object:+ return handler++ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.tools.composition_capture._completion_verification_handler",+ fake_completion_verification_handler,+ )+ ctx = _run_ctx()+ ctx.browser_session_id = "bs_1"+ ctx.discovery_mcp_server = object()+ ctx.request_policy = RequestPolicy(+ completion_criteria=[_criterion("c0", "return record id", output_path="output.record_id")]+ )++ assert await _active_run_terminal_monitor_enabled(ctx) is False+++async def test_active_run_terminal_monitor_keeps_mixed_requested_output_armed(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def handler(**_: object) -> dict:+ return {}++ async def fake_completion_verification_handler(_ctx: object) -> object:+ return handler++ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.tools.composition_capture._completion_verification_handler",+ fake_completion_verification_handler,+ )+ ctx = _run_ctx()+ ctx.browser_session_id = "bs_1"+ ctx.discovery_mcp_server = object()+ ctx.request_policy = RequestPolicy(+ completion_criteria=[+ _criterion("c0", "return record id", output_path="output.record_id"),+ _criterion("c1", "cart page is visible"),+ ]+ )++ assert await _active_run_terminal_monitor_enabled(ctx) is True+++async def test_active_run_terminal_monitor_skips_requested_output_corroborator(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def handler(**_: object) -> dict:+ return {}++ async def fake_completion_verification_handler(_ctx: object) -> object:+ return handler++ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.tools.composition_capture._completion_verification_handler",+ fake_completion_verification_handler,+ )+ ctx = _run_ctx()+ ctx.browser_session_id = "bs_1"+ ctx.discovery_mcp_server = object()+ ctx.request_policy = RequestPolicy(+ completion_criteria=[+ _criterion("c_requested", "return quotes", output_path="output.quotes"),+ _criterion(+ "c_quotes",+ "The run extracts the first 3 quotes and authors into the returned quotes JSON.",+ requested_output_corroborator=True,+ ),+ ]+ )++ assert await _active_run_terminal_monitor_enabled(ctx) is False+++async def test_active_run_terminal_monitor_keeps_unmarked_fallback_floor_armed(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def handler(**_: object) -> dict:+ return {}++ async def fake_completion_verification_handler(_ctx: object) -> object:+ return handler++ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.tools.composition_capture._completion_verification_handler",+ fake_completion_verification_handler,+ )+ ctx = _run_ctx()+ ctx.browser_session_id = "bs_1"+ ctx.discovery_mcp_server = object()+ ctx.request_policy = RequestPolicy(completion_criteria=build_classifier_fallback_floor([]))++ assert await _active_run_terminal_monitor_enabled(ctx) is True+++async def test_active_run_terminal_monitor_skips_marked_fallback_floor_corroborator(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def handler(**_: object) -> dict:+ return {}++ async def fake_completion_verification_handler(_ctx: object) -> object:+ return handler++ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.tools.composition_capture._completion_verification_handler",+ fake_completion_verification_handler,+ )+ ctx = _run_ctx()+ ctx.browser_session_id = "bs_1"+ ctx.discovery_mcp_server = object()+ floor = replace(build_classifier_fallback_floor([])[0], requested_output_corroborator=True)+ ctx.request_policy = RequestPolicy(+ completion_criteria=[+ _criterion("c_requested", "return record id", output_path="output.record_id"),+ floor,+ ]+ )++ assert await _active_run_terminal_monitor_enabled(ctx) is False+++async def test_active_run_terminal_monitor_keeps_terminal_action_criteria_armed(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def handler(**_: object) -> dict:+ return {}++ async def fake_completion_verification_handler(_ctx: object) -> object:+ return handler++ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.tools.composition_capture._completion_verification_handler",+ fake_completion_verification_handler,+ )+ ctx = _run_ctx()+ ctx.browser_session_id = "bs_1"+ ctx.discovery_mcp_server = object()+ ctx.request_policy = RequestPolicy(+ completion_criteria=[+ _criterion(+ "c0",+ "service request is submitted",+ kind="terminal_action",+ terminal_action_family="request",+ )+ ]+ )++ assert await _active_run_terminal_monitor_enabled(ctx) is True++def test_summarize_unsatisfied_lists_unmet_outcomes() -> None:criteria = [_criterion("c0", "item in cart"), _criterion("c1", "added exactly once")]result = _evaluated(("c0", True), ("c1", False))@@ -2818,6 +3012,29 @@ def test_record_run_blocks_keeps_clean_structural_abstention_as_built_unverifiedassert outcome.is_authoritative is False+def test_record_run_blocks_verifies_structural_requested_output_with_run_corroborator() -> None:+ ctx = _ctx_with_blocks("extraction")+ verification = _mixed(+ CriterionVerdict(criterion_id="c_quotes", state="satisfied", reason_code="evidence_confirms"),+ CriterionVerdict(+ criterion_id="c_requested_output",+ state="unsatisfied",+ reason_code="structurally_abstained",+ evidence_ref="block_outputs:extract_first_three_quotes.quotes",+ output_path="output.quotes",+ grounding_mode="missing",+ ),+ )++ recorded = _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=verification)++ assert recorded is not None+ assert recorded.verdict == "demonstrated"+ assert verified_goal_satisfied_context(ctx) is True+ assert built_unverified_repair_inert_context(ctx) is False+ assert outcome_fully_verified(ctx) is True++def test_committed_same_run_outcome_survives_later_contradictory_overwrite() -> None:ctx = _ctx_with_blocks("extraction")verification = _mixed(@@ -3801,10 +4018,87 @@ def test_active_run_terminal_evidence_result_shape_is_not_final_success() -> Nonassert result["ok"] is Falseassert result["data"]["active_run_terminal_evidence_detected"] is True+ assert result["data"]["active_run_terminal_evidence_reason_code"] == ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODEassert result["data"]["full_workflow_verified"] is Falseassert result["data"]["failure_categories"][0]["category"] == ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY+def test_active_run_terminal_evidence_contract_noops_when_outcome_fully_verified() -> None:+ sample = SimpleNamespace(+ current_url="https://example.com/cart",+ page_title="Cart",+ sample_index=2,+ completion_verification=_evaluated(("c0", True)),+ page_evidence={+ "current_url": "https://example.com/cart",+ "page_title": "Cart",+ "visible_text_excerpt": "Cart TESTBRAND PART-001-TEST quantity 1",+ },+ )+ result = _active_run_terminal_evidence_result(+ workflow_run_id="wr_active",+ run_status="canceled",+ sample=sample,+ requested_block_labels=["search_and_add"],+ executed_block_labels=["search_and_add"],+ )+ ctx = _run_ctx()+ ctx.completion_verification_result = _evaluated(("c0", True))++ contract = build_diagnosis_repair_contract(source_tool="update_and_run_blocks", result=result, ctx=ctx)++ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.NO_FAILURE+ assert contract.repair_decision.next_action == RepairNextAction.NO_CHANGE+ assert contract.verification_result.user_goal_satisfied is True+ assert contract.verification_result.completion_contract_satisfied is True+++def test_active_run_terminal_evidence_contract_requires_reason_code_for_verified_noop() -> None:+ sample = SimpleNamespace(+ current_url="https://example.com/cart",+ page_title="Cart",+ sample_index=2,+ completion_verification=_evaluated(("c0", True)),+ page_evidence={"current_url": "https://example.com/cart"},+ )+ result = _active_run_terminal_evidence_result(+ workflow_run_id="wr_active",+ run_status="canceled",+ sample=sample,+ requested_block_labels=["search_and_add"],+ executed_block_labels=["search_and_add"],+ )+ result["data"].pop("active_run_terminal_evidence_reason_code")+ ctx = _run_ctx()+ ctx.completion_verification_result = _evaluated(("c0", True))++ contract = build_diagnosis_repair_contract(source_tool="update_and_run_blocks", result=result, ctx=ctx)++ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.ACTIVE_RUN_TERMINAL_EVIDENCE+ assert contract.repair_decision.next_action == RepairNextAction.STOP+++def test_terminal_challenge_contract_still_stops_when_outcome_fully_verified() -> None:+ ctx = _run_ctx()+ ctx.completion_verification_result = _evaluated(("c0", True))+ result = {+ "ok": False,+ "error": "blocked by human verification",+ "data": {+ "workflow_run_id": "wr_blocked",+ "overall_status": "failed",+ "failure_categories": [{"category": "ANTI_BOT_DETECTION", "confidence_float": 1.0}],+ },+ }++ contract = build_diagnosis_repair_contract(source_tool="update_and_run_blocks", result=result, ctx=ctx)++ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.TERMINAL_CHALLENGE_BLOCKER+ assert contract.repair_decision.next_action == RepairNextAction.STOP+ assert contract.verification_result.user_goal_satisfied is False+ assert contract.verification_result.completion_contract_satisfied is False++def test_record_active_run_terminal_evidence_keeps_workflow_unverified() -> None:sample = SimpleNamespace(current_url="https://example.com/cart",diff --git a/tests/unit/test_copilot_request_policy_completion_criteria.py b/tests/unit/test_copilot_request_policy_completion_criteria.pyindex d8548e427..c7f43bb8d 100644--- a/tests/unit/test_copilot_request_policy_completion_criteria.py+++ b/tests/unit/test_copilot_request_policy_completion_criteria.py@@ -21,6 +21,8 @@ from skyvern.forge.sdk.copilot.request_policy import (_classify_request,_parse_completion_criteria,_render_active_criteria_for_prompt,+ build_classifier_fallback_floor,+ is_fallback_floor_base_criterion,request_policy_has_present_completion_contract,)@@ -109,6 +111,17 @@ def _requested_output_subset(policy: RequestPolicy, requested_output_paths: set[]+def _run_outcome_corroborators(policy: RequestPolicy) -> list[CompletionCriterion]:+ return [+ criterion+ for criterion in policy.completion_criteria+ if criterion.level == "run"+ and criterion.kind == "outcome"+ and not criterion.method_mandated+ and criterion.output_path is None+ ]++def test_present_completion_contract_helper_accepts_present_status() -> None:policy = RequestPolicy(completion_contract_status="present")@@ -351,9 +364,11 @@ async def test_resale_docs_requested_output_uses_explicit_document_name_and_bloc],)- assert [criterion.output_path for criterion in policy.completion_criteria] == ["output.document_name"]- assert policy.completion_criteria[0].expected_output_value is None- assert policy.completion_criteria[0].expected_output_shape is None+ requested_outputs = [criterion for criterion in policy.completion_criteria if criterion.output_path is not None]+ assert [criterion.output_path for criterion in requested_outputs] == ["output.document_name"]+ assert requested_outputs[0].expected_output_value is None+ assert requested_outputs[0].expected_output_shape is None+ assert _run_outcome_corroborators(policy)[0].requested_output_corroborator is True@pytest.mark.asyncio@@ -367,6 +382,162 @@ async def test_named_customer_prose_does_not_become_requested_output_field_name(assert _criteria_for_path(policy, "output.status")+async def test_requested_output_canonicalization_preserves_output_corroborator() -> None:+ policy = await _policy_for_message(+ "Extract the first 3 quotes and authors, then return JSON with quotes.",+ [+ {+ "id": "c_quotes",+ "outcome": "The run extracts the first 3 quotes and authors into the returned quotes JSON.",+ "output_path": "output.quotes",+ }+ ],+ )++ requested = _requested_output_subset(policy, {"output.quotes"})+ corroborators = _run_outcome_corroborators(policy)++ assert len(requested) == 1+ assert requested[0].id == "__copilot_requested_output__output_quotes"+ assert len(corroborators) == 1+ assert corroborators[0].requested_output_corroborator is True+ assert corroborators[0].outcome == "The run extracts the first 3 quotes and authors into the returned quotes JSON."+++async def test_classifier_typed_requested_output_adds_distinct_corroborator() -> None:+ policy = await _policy_for_message(+ "Run the read-only extraction.",+ [+ {+ "outcome": "The returned JSON contains the first three public quotes and authors.",+ "output_path": "output.quotes",+ }+ ],+ )++ requested = _requested_output_subset(policy, {"output.quotes"})+ corroborators = _run_outcome_corroborators(policy)++ assert len(requested) == 1+ assert requested[0].id == "c0"+ assert requested[0].output_path == "output.quotes"+ assert requested[0].requested_output_corroborator is False+ assert len(corroborators) == 1+ assert corroborators[0].id != requested[0].id+ assert corroborators[0].id == "c0__requested_output_corroborator"+ assert corroborators[0].output_path is None+ assert corroborators[0].expected_output_value is None+ assert corroborators[0].expected_output_shape is None+ assert corroborators[0].requested_output_corroborator is True+ assert corroborators[0].outcome == requested[0].outcome+++async def test_plain_requested_output_text_does_not_fabricate_corroborator() -> None:+ policy = await _policy_for_message("Return a final record with record id.", [])++ requested = _requested_output_subset(policy, {"output.record_id"})++ assert len(requested) == 1+ assert requested[0].id == "__copilot_requested_output__output_record_id"+ assert _run_outcome_corroborators(policy) == []+++def test_requested_output_canonicalization_preserves_fallback_floor_base_when_it_is_source() -> None:+ policy = RequestPolicy(completion_criteria=build_classifier_fallback_floor([]))++ _apply_requested_output_completion_criteria(policy, "Return a final record with record id.")++ assert len(_requested_output_subset(policy, {"output.record_id"})) == 1+ fallback = [+ criterion for criterion in _run_outcome_corroborators(policy) if is_fallback_floor_base_criterion(criterion)+ ]+ assert len(fallback) == 1+ assert fallback[0].requested_output_corroborator is True+++def test_requested_output_canonicalization_does_not_promote_method_mandated_only_corroborator() -> None:+ policy = RequestPolicy(+ completion_criteria=[+ CompletionCriterion(+ id="method_floor",+ outcome="The workflow runs to its intended end state with the expected output.",+ method_mandated=True,+ )+ ]+ )++ _apply_requested_output_completion_criteria(policy, "Return a final record with record id.")++ assert len(_requested_output_subset(policy, {"output.record_id"})) == 1+ assert _run_outcome_corroborators(policy) == []+++def test_requested_output_corroborator_respects_completion_criteria_cap() -> None:+ policy = RequestPolicy(+ completion_criteria=[+ CompletionCriterion(id=f"specific_{index}", outcome=f"Specific retained outcome {index}.")+ for index in range(7)+ ]+ + [+ CompletionCriterion(+ id="quotes",+ outcome="The run extracts the requested record id into the returned JSON.",+ output_path="output.record_id",+ )+ ]+ )++ _apply_requested_output_completion_criteria(policy, "Return a final record with record id and status.")++ assert len(policy.completion_criteria) == 8+ assert {+ criterion.output_path for criterion in _requested_output_subset(policy, {"output.record_id", "output.status"})+ } == {+ "output.record_id",+ "output.status",+ }+++def test_requested_output_corroborator_survives_when_requested_outputs_exceed_cap() -> None:+ policy = RequestPolicy(+ completion_criteria=[+ CompletionCriterion(+ id="name_source",+ outcome="The run extracts the requested name into the returned JSON.",+ output_path="output.name",+ )+ ]+ )++ _apply_requested_output_completion_criteria(+ policy,+ "Return a final record with name, record id, status, phone, email, license, taxonomy, specialty, and date.",+ )++ requested_output_paths = {+ "output.name",+ "output.record_id",+ "output.status",+ "output.phone",+ "output.email",+ "output.license",+ "output.taxonomy",+ "output.specialty",+ "output.date",+ }+ corroborators = _run_outcome_corroborators(policy)++ assert {+ criterion.output_path for criterion in _requested_output_subset(policy, requested_output_paths)+ } == requested_output_paths+ assert len(corroborators) == 1+ assert corroborators[0].id == "name_source"+ assert corroborators[0].requested_output_corroborator is True++@pytest.mark.asyncioasync def test_requested_output_criteria_do_not_infer_shapes_from_generated_provider_field_names() -> None:policy = await _policy_for_message(