Skip to content

Automate browser based workflows with AI

SKY-11642 repair read-only extraction verification (#6996)

Andrew Neilsonfb6ff274ae9f
diff --git a/skyvern/forge/sdk/copilot/completion_criteria_store.py b/skyvern/forge/sdk/copilot/completion_criteria_store.py
index aabae500d..f56430597 100644
--- a/skyvern/forge/sdk/copilot/completion_criteria_store.py
+++ b/skyvern/forge/sdk/copilot/completion_criteria_store.py
@@ -116,8 +116,9 @@ class PersistencePlan:
def criteria_to_json(criteria: tuple[CompletionCriterion, ...] | list[CompletionCriterion]) -> list[dict[str, Any]]:
- return [
- {
+ items: list[dict[str, Any]] = []
+ for criterion in criteria:
+ item = {
"id": criterion.id,
"outcome": criterion.outcome,
"contingent_on": criterion.contingent_on,
@@ -134,8 +135,10 @@ def criteria_to_json(criteria: tuple[CompletionCriterion, ...] | list[Completion
"classification_output_key": criterion.classification_output_key,
"expected_classification": criterion.expected_classification,
}
- for criterion in criteria
- ]
+ if criterion.requested_output_corroborator:
+ item["requested_output_corroborator"] = True
+ items.append(item)
+ return items
def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:
@@ -195,6 +198,7 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:
terminal_action_family=cast(TerminalActionFamily | None, terminal_action_family),
classification_output_key=classification_output_key,
expected_classification=expected_classification,
+ requested_output_corroborator=bool(item.get("requested_output_corroborator")),
)
)
return tuple(criteria)
diff --git a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
index add505051..f0fa05a7f 100644
--- a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
+++ b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
@@ -16,6 +16,7 @@ from skyvern.forge.sdk.copilot.composition_evidence import interactive_challenge
from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext
from skyvern.forge.sdk.copilot.failure_tracking import (
ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,
+ ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE,
RepairRootCauseIdentity,
compute_repair_root_cause_signature,
)
@@ -511,6 +512,8 @@ def _failure_type(
ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY in categories
or data.get("active_run_terminal_evidence_detected") is True
):
+ if outcome_verified and _active_run_terminal_evidence_reason_code(data):
+ return DiagnosisFailureType.NO_FAILURE
return DiagnosisFailureType.ACTIVE_RUN_TERMINAL_EVIDENCE
if (
category_set & _PRE_RUN_CREDENTIAL_FAILURE_CATEGORIES
@@ -538,6 +541,10 @@ def _failure_type(
return DiagnosisFailureType.FAILED_RUN if result.get("ok") is False else DiagnosisFailureType.UNKNOWN
+def _active_run_terminal_evidence_reason_code(data: dict[str, Any]) -> bool:
+ return _safe_str(data.get("active_run_terminal_evidence_reason_code")) == ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE
+
+
def _next_action(
failure_type: DiagnosisFailureType,
ctx: CopilotContext,
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index 840502411..d324cf77c 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -256,6 +256,7 @@ class CompletionCriterion:
terminal_action_family: TerminalActionFamily | None = None
classification_output_key: str | None = None
expected_classification: ClassificationTarget | None = None
+ requested_output_corroborator: bool = False
@dataclass
@@ -1102,6 +1103,8 @@ def _generic_completion_criterion(criterion: CompletionCriterion) -> bool:
def _criterion_drop_priority(criterion: CompletionCriterion, requested_output_paths: set[str]) -> int:
if criterion.output_path in requested_output_paths:
return 0
+ if criterion.requested_output_corroborator:
+ return 0
if criterion.level == "definition":
return 0
if is_fallback_floor_criterion(criterion):
@@ -1155,6 +1158,110 @@ def _preserve_after_requested_output_canonicalization(
return True
+def _non_requested_output_run_corroborator(criterion: CompletionCriterion) -> bool:
+ return (
+ criterion.level == "run"
+ and criterion.kind == "outcome"
+ and not criterion.method_mandated
+ and criterion.output_path is None
+ )
+
+
+def _validation_classification_output_path(output_path: str | None) -> bool:
+ return (
+ output_path in _VALIDATION_CLASSIFICATION_BOOLEAN_OUTPUT_TARGETS
+ or output_path in _VALIDATION_CLASSIFICATION_LABEL_OUTPUT_TARGETS
+ )
+
+
+def completion_criterion_requires_active_run_terminal_monitor(criterion: CompletionCriterion) -> bool:
+ if criterion.requested_output_corroborator:
+ return False
+ if (
+ criterion.output_path
+ and criterion.level != "definition"
+ and not criterion.method_mandated
+ and criterion.kind != "validation_classification"
+ ):
+ return False
+ return True
+
+
+def _source_requested_output_corroborator(
+ criteria: list[CompletionCriterion],
+ requested_fields: list[str],
+ requested_output_paths: set[str],
+) -> CompletionCriterion | None:
+ for criterion in criteria:
+ if criterion.level != "run" or criterion.kind != "outcome" or criterion.method_mandated:
+ continue
+ if _validation_classification_output_path(criterion.output_path):
+ continue
+ if criterion.output_path in requested_output_paths or _criterion_text_covers_any_requested_output(
+ criterion, requested_fields
+ ):
+ return replace(
+ criterion,
+ output_path=None,
+ expected_output_value=None,
+ expected_output_shape=None,
+ requested_output_corroborator=True,
+ )
+ for criterion in criteria:
+ if is_fallback_floor_base_criterion(criterion):
+ return replace(criterion, requested_output_corroborator=True)
+ return None
+
+
+def _requested_output_corroborator_id(source_id: str, used_ids: set[str]) -> str:
+ base_id = f"{source_id}__requested_output_corroborator"
+ if base_id not in used_ids:
+ return base_id
+ suffix = 2
+ while f"{base_id}_{suffix}" in used_ids:
+ suffix += 1
+ return f"{base_id}_{suffix}"
+
+
+def _apply_classifier_typed_requested_output_corroborators(policy: RequestPolicy) -> None:
+ if any(_non_requested_output_run_corroborator(criterion) for criterion in policy.completion_criteria):
+ return
+ source_criteria = [
+ criterion
+ for criterion in policy.completion_criteria
+ if criterion.level == "run"
+ and criterion.kind == "outcome"
+ and not criterion.method_mandated
+ and criterion.output_path is not None
+ and not _validation_classification_output_path(criterion.output_path)
+ and not criterion.id.startswith(_REQUESTED_OUTPUT_CRITERION_ID_PREFIX)
+ ]
+ if not source_criteria:
+ return
+
+ used_ids = {criterion.id for criterion in policy.completion_criteria}
+ corroborators: list[CompletionCriterion] = []
+ for criterion in source_criteria:
+ corroborator_id = _requested_output_corroborator_id(criterion.id, used_ids)
+ used_ids.add(corroborator_id)
+ corroborators.append(
+ replace(
+ criterion,
+ id=corroborator_id,
+ output_path=None,
+ expected_output_value=None,
+ expected_output_shape=None,
+ requested_output_corroborator=True,
+ )
+ )
+
+ requested_output_paths = {criterion.output_path for criterion in source_criteria if criterion.output_path}
+ policy.completion_criteria = _cap_completion_criteria(
+ policy.completion_criteria + corroborators,
+ requested_output_paths,
+ )
+
+
def _apply_requested_output_completion_criteria(
policy: RequestPolicy, user_message: str, aliases: dict[str, str] | None = None
) -> None:
@@ -1237,8 +1344,17 @@ def _apply_requested_output_completion_criteria(
)
for _field_name, output_path, field_label in requested_specs
]
+ criteria = preserved_criteria + canonical_requested_criteria
+ if not any(_non_requested_output_run_corroborator(criterion) for criterion in criteria):
+ corroborator = _source_requested_output_corroborator(
+ policy.completion_criteria,
+ requested_fields,
+ requested_output_paths,
+ )
+ if corroborator is not None:
+ criteria = preserved_criteria + [corroborator] + canonical_requested_criteria
policy.completion_criteria = _cap_completion_criteria(
- preserved_criteria + canonical_requested_criteria,
+ criteria,
requested_output_paths,
)
@@ -1320,6 +1436,8 @@ def _render_active_criteria_for_prompt(criteria: list[CompletionCriterion] | Non
item["classification_output_key"] = criterion.classification_output_key
if criterion.expected_classification is not None:
item["expected_classification"] = criterion.expected_classification
+ if criterion.requested_output_corroborator:
+ item["requested_output_corroborator"] = True
rendered.append(item)
return json.dumps(rendered)
@@ -1395,6 +1513,7 @@ def _classifier_fallback_policy(
completion_contract_status="present" if fallback_criteria else "unknown",
)
_apply_requested_output_completion_criteria(policy, user_message, requested_output_path_aliases)
+ _apply_classifier_typed_requested_output_corroborators(policy)
if policy.graded_completion_criteria():
policy.completion_contract_status = "present"
return policy
@@ -1637,6 +1756,7 @@ async def _classify_request(
policy.raw_secret_evidence = None
_apply_requested_output_completion_criteria(policy, user_message, requested_output_path_aliases)
_apply_validation_classification_completion_criteria(policy)
+ _apply_classifier_typed_requested_output_corroborators(policy)
policy.completion_contract_status = (
"present" if policy.completion_contract or policy.graded_completion_criteria() else "absent"
)
diff --git a/skyvern/forge/sdk/copilot/tools/composition_capture.py b/skyvern/forge/sdk/copilot/tools/composition_capture.py
index f6dd3533b..a60db52d0 100644
--- a/skyvern/forge/sdk/copilot/tools/composition_capture.py
+++ b/skyvern/forge/sdk/copilot/tools/composition_capture.py
@@ -25,9 +25,13 @@ from skyvern.forge.sdk.copilot.composition_evidence import (
parse_composition_html,
)
from skyvern.forge.sdk.copilot.context import CopilotContext
-from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY
+from skyvern.forge.sdk.copilot.failure_tracking import (
+ ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,
+ ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE,
+)
from skyvern.forge.sdk.copilot.llm_config import resolve_fast_copilot_handler
from skyvern.forge.sdk.copilot.loop_detection import record_tool_step_result_for_ctx
+from skyvern.forge.sdk.copilot.request_policy import completion_criterion_requires_active_run_terminal_monitor
from skyvern.forge.sdk.copilot.runtime_authoring_repair import (
finalize_runtime_authoring_repair_context_from_page_observation,
)
@@ -79,7 +83,10 @@ async def _active_run_terminal_monitor_enabled(copilot_ctx: Any) -> bool:
return False
if not getattr(copilot_ctx, "discovery_mcp_server", None):
return False
- if not _completion_verification_criteria(copilot_ctx):
+ criteria = _completion_verification_criteria(copilot_ctx)
+ if not criteria:
+ return False
+ if not any(completion_criterion_requires_active_run_terminal_monitor(criterion) for criterion in criteria):
return False
return await _completion_verification_handler(copilot_ctx) is not None
@@ -194,6 +201,7 @@ def _active_run_terminal_evidence_result(
"current_url": observed_url,
"page_title": observed_title,
"active_run_terminal_evidence_detected": True,
+ "active_run_terminal_evidence_reason_code": ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE,
"active_run_terminal_evidence_sample_index": getattr(sample, "sample_index", None),
"full_workflow_verified": False,
"current_page_evidence": getattr(sample, "page_evidence", None),
diff --git a/skyvern/forge/sdk/copilot/turn_halt.py b/skyvern/forge/sdk/copilot/turn_halt.py
index 0b0fd621b..459daf1f0 100644
--- a/skyvern/forge/sdk/copilot/turn_halt.py
+++ b/skyvern/forge/sdk/copilot/turn_halt.py
@@ -90,6 +90,8 @@ _INVOLUNTARY_BLOCKER_REASON_CODES = (
| _SCHEMA_INCOMPATIBILITY_REASON_CODES
| frozenset({REPAIR_CEILING_REASON_CODE})
)
+_VERIFIED_SUPPRESSIBLE_ACTIVE_TERMINAL_REASON_CODES = frozenset({ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE})
+_VERIFIED_SUPPRESSIBLE_ACTIVE_TERMINAL_SOURCES = frozenset({"run_execution"})
@dataclass(frozen=True)
@@ -183,9 +185,21 @@ def raise_if_turn_halt(ctx: Any, *, verified: bool = False) -> None:
halt = getattr(ctx, "turn_halt", None)
if not isinstance(halt, TurnHalt):
return
- if verified and halt.kind in _INVOLUNTARY_TURN_HALT_KINDS:
+ suppressible_reason_codes = _INVOLUNTARY_BLOCKER_REASON_CODES
+ if (
+ halt.kind == TurnHaltKind.ACTIVE_TERMINAL_CHALLENGE
+ and halt.blocker_signal is not None
+ and halt.blocker_signal.internal_reason_code in _VERIFIED_SUPPRESSIBLE_ACTIVE_TERMINAL_REASON_CODES
+ and halt.extra.get("source") in _VERIFIED_SUPPRESSIBLE_ACTIVE_TERMINAL_SOURCES
+ ):
+ suppressible_reason_codes = (
+ _INVOLUNTARY_BLOCKER_REASON_CODES | _VERIFIED_SUPPRESSIBLE_ACTIVE_TERMINAL_REASON_CODES
+ )
+ elif halt.kind not in _INVOLUNTARY_TURN_HALT_KINDS:
+ suppressible_reason_codes = frozenset()
+ if verified and suppressible_reason_codes:
ctx.turn_halt = None
- clear_tool_blocker_signals_for_reason_codes(ctx, _INVOLUNTARY_BLOCKER_REASON_CODES)
+ clear_tool_blocker_signals_for_reason_codes(ctx, suppressible_reason_codes)
LOG.info(
"copilot turn halt suppressed by verified outcome",
**turn_halt_to_trace_data(halt),
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index d3c68b898..859e14734 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -55,6 +55,7 @@ from skyvern.forge.sdk.copilot.enforcement import (
verified_goal_claim_authorized,
verified_goal_satisfied_context,
)
+from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE
from skyvern.forge.sdk.copilot.recoverable_failure import build_recoverable_failure
from skyvern.forge.sdk.copilot.request_policy import (
_MAX_COMPLETION_CRITERIA,
@@ -73,6 +74,7 @@ from skyvern.forge.sdk.copilot.run_outcome import TERMINAL_CHALLENGE_BLOCKER_REA
from skyvern.forge.sdk.copilot.tools.completion import _completion_verification_criteria
from skyvern.forge.sdk.copilot.turn_context import TranscriptContext, TurnContextOmission, TurnContextPacket
from skyvern.forge.sdk.copilot.turn_halt import (
+ CopilotTurnHalt,
TurnHalt,
TurnHaltKind,
raise_if_turn_halt,
@@ -1057,6 +1059,48 @@ class TestVerifiedGoalSatisfiedStop:
assert result.user_response == _VERIFIED_WORKFLOW_SUCCESS_REPLY
assert result.proposal_disposition != "no_proposal"
+ def test_verified_turn_halt_suppresses_active_run_terminal_evidence_only(self) -> None:
+ ctx = _ctx()
+ signal = CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text="active run reached requested state",
+ user_facing_reason="active run reached requested state",
+ recovery_hint="report_blocker_to_user",
+ internal_reason_code=ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE,
+ blocked_tool="update_and_run_blocks",
+ )
+ ctx.blocker_signal = signal
+ ctx.latest_tool_blocker_signal = signal
+ ctx.tool_blocker_signals.append(signal)
+ ctx.turn_halt = TurnHalt(
+ kind=TurnHaltKind.ACTIVE_TERMINAL_CHALLENGE,
+ blocker_signal=signal,
+ extra={"source": "run_execution"},
+ )
+
+ raise_if_turn_halt(ctx, verified=True)
+
+ assert not isinstance(ctx.turn_halt, TurnHalt)
+ assert ctx.blocker_signal is None
+ assert ctx.latest_tool_blocker_signal is None
+ assert ctx.tool_blocker_signals == []
+
+ def test_verified_turn_halt_keeps_terminal_challenge_blocker(self) -> None:
+ ctx = _ctx()
+ signal = CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text="stop on terminal challenge",
+ user_facing_reason="The site requires human verification.",
+ recovery_hint="stop",
+ internal_reason_code=TERMINAL_CHALLENGE_BLOCKER_REASON_CODE,
+ blocked_tool="update_and_run_blocks",
+ )
+ ctx.blocker_signal = signal
+ ctx.turn_halt = TurnHalt(kind=TurnHaltKind.ACTIVE_TERMINAL_CHALLENGE, blocker_signal=signal)
+
+ with pytest.raises(CopilotTurnHalt):
+ raise_if_turn_halt(ctx, verified=True)
+
@pytest.mark.asyncio
async def test_wrapped_exception_resolver_renders_success_over_involuntary_halt(self) -> None:
ctx = _ctx(
@@ -1304,6 +1348,35 @@ class TestVerifiedGoalSatisfiedStop:
assert result.narrative_payload is not None
assert result.narrative_payload["verifiedSuccess"] is False
+ def test_corroborated_structural_abstention_avoids_built_unverified_terminal(self) -> None:
+ ctx = _ctx(
+ last_test_ok=True,
+ last_full_workflow_test_ok=True,
+ latest_diagnosis_repair_contract=_unverified_no_repair_contract(),
+ completion_verification_result=CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c0", "c0__requested_output_corroborator"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="c0",
+ state="unsatisfied",
+ reason_code="structurally_abstained",
+ evidence_ref="block_outputs:extract_first_three_quotes.quotes",
+ output_path="output.quotes",
+ grounding_mode="missing",
+ ),
+ CriterionVerdict(
+ criterion_id="c0__requested_output_corroborator",
+ state="satisfied",
+ reason_code="evidence_confirms",
+ ),
+ ],
+ ),
+ )
+
+ assert verified_goal_satisfied_context(ctx) is True
+ assert built_unverified_repair_inert_context(ctx) is False
+
@pytest.mark.asyncio
async def test_goal_satisfied_exit_result_carries_outcome_adjudication(self) -> None:
from skyvern.forge.sdk.copilot.agent import _build_goal_satisfied_exit_result
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index 63d791acf..8cee28950 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -4,6 +4,7 @@ import asyncio
import textwrap
import time
from collections.abc import Awaitable, Callable
+from dataclasses import replace
from types import SimpleNamespace
from typing import Any
@@ -46,6 +47,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (
)
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
+ DiagnosisFailureType,
DiagnosisInput,
DiagnosisRepairContract,
DiagnosisResult,
@@ -53,12 +55,14 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
RepairNextAction,
VerificationResult,
_verification_satisfaction,
+ build_diagnosis_repair_contract,
)
from skyvern.forge.sdk.copilot.enforcement import (
built_unverified_repair_inert_context,
outcome_fully_verified,
verified_goal_satisfied_context,
)
+from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE
from skyvern.forge.sdk.copilot.hooks import _tool_completion_satisfies_turn
from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTarget
from skyvern.forge.sdk.copilot.request_policy import (
@@ -92,6 +96,7 @@ from skyvern.forge.sdk.copilot.tools.completion import (
_artifact_health_blocker_from_result,
_reconcile_download_completion_criterion,
)
+from skyvern.forge.sdk.copilot.tools.composition_capture import _active_run_terminal_monitor_enabled
def _criterion(
@@ -109,6 +114,7 @@ def _criterion(
expected_output_shape: str | None = None,
classification_output_key: str | None = None,
expected_classification: str | bool | None = None,
+ requested_output_corroborator: bool = False,
) -> CompletionCriterion:
return CompletionCriterion(
id=cid,
@@ -124,6 +130,7 @@ def _criterion(
expected_output_shape=expected_output_shape, # type: ignore[arg-type]
classification_output_key=classification_output_key,
expected_classification=expected_classification,
+ requested_output_corroborator=requested_output_corroborator,
)
@@ -482,6 +489,30 @@ def test_structural_requested_output_abstention_without_typed_corroboration_gets
assert result.is_fully_satisfied() is False
+def test_structural_requested_output_abstention_with_classifier_corroborator_satisfies() -> None:
+ result = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c0", "c0__requested_output_corroborator"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="c0",
+ state="unsatisfied",
+ reason_code="structurally_abstained",
+ evidence_ref="block_outputs:extract_first_three_quotes.quotes",
+ output_path="output.quotes",
+ grounding_mode="missing",
+ ),
+ CriterionVerdict(
+ criterion_id="c0__requested_output_corroborator",
+ state="satisfied",
+ reason_code="evidence_confirms",
+ ),
+ ],
+ )
+
+ assert result.is_fully_satisfied() is True
+
+
def test_requested_output_no_evidence_with_observed_end_state_still_blocks() -> None:
result = _mixed(
CriterionVerdict(
@@ -2230,6 +2261,169 @@ def test_visual_prompt_requests_outcome_relevant_page_state() -> None:
assert "human-verification" in prompt
+async def test_active_run_terminal_monitor_skips_requested_output_only_criteria(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def handler(**_: object) -> dict:
+ return {}
+
+ async def fake_completion_verification_handler(_ctx: object) -> object:
+ return handler
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.tools.composition_capture._completion_verification_handler",
+ fake_completion_verification_handler,
+ )
+ ctx = _run_ctx()
+ ctx.browser_session_id = "bs_1"
+ ctx.discovery_mcp_server = object()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[_criterion("c0", "return record id", output_path="output.record_id")]
+ )
+
+ assert await _active_run_terminal_monitor_enabled(ctx) is False
+
+
+async def test_active_run_terminal_monitor_keeps_mixed_requested_output_armed(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def handler(**_: object) -> dict:
+ return {}
+
+ async def fake_completion_verification_handler(_ctx: object) -> object:
+ return handler
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.tools.composition_capture._completion_verification_handler",
+ fake_completion_verification_handler,
+ )
+ ctx = _run_ctx()
+ ctx.browser_session_id = "bs_1"
+ ctx.discovery_mcp_server = object()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ _criterion("c0", "return record id", output_path="output.record_id"),
+ _criterion("c1", "cart page is visible"),
+ ]
+ )
+
+ assert await _active_run_terminal_monitor_enabled(ctx) is True
+
+
+async def test_active_run_terminal_monitor_skips_requested_output_corroborator(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def handler(**_: object) -> dict:
+ return {}
+
+ async def fake_completion_verification_handler(_ctx: object) -> object:
+ return handler
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.tools.composition_capture._completion_verification_handler",
+ fake_completion_verification_handler,
+ )
+ ctx = _run_ctx()
+ ctx.browser_session_id = "bs_1"
+ ctx.discovery_mcp_server = object()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ _criterion("c_requested", "return quotes", output_path="output.quotes"),
+ _criterion(
+ "c_quotes",
+ "The run extracts the first 3 quotes and authors into the returned quotes JSON.",
+ requested_output_corroborator=True,
+ ),
+ ]
+ )
+
+ assert await _active_run_terminal_monitor_enabled(ctx) is False
+
+
+async def test_active_run_terminal_monitor_keeps_unmarked_fallback_floor_armed(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def handler(**_: object) -> dict:
+ return {}
+
+ async def fake_completion_verification_handler(_ctx: object) -> object:
+ return handler
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.tools.composition_capture._completion_verification_handler",
+ fake_completion_verification_handler,
+ )
+ ctx = _run_ctx()
+ ctx.browser_session_id = "bs_1"
+ ctx.discovery_mcp_server = object()
+ ctx.request_policy = RequestPolicy(completion_criteria=build_classifier_fallback_floor([]))
+
+ assert await _active_run_terminal_monitor_enabled(ctx) is True
+
+
+async def test_active_run_terminal_monitor_skips_marked_fallback_floor_corroborator(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def handler(**_: object) -> dict:
+ return {}
+
+ async def fake_completion_verification_handler(_ctx: object) -> object:
+ return handler
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.tools.composition_capture._completion_verification_handler",
+ fake_completion_verification_handler,
+ )
+ ctx = _run_ctx()
+ ctx.browser_session_id = "bs_1"
+ ctx.discovery_mcp_server = object()
+ floor = replace(build_classifier_fallback_floor([])[0], requested_output_corroborator=True)
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ _criterion("c_requested", "return record id", output_path="output.record_id"),
+ floor,
+ ]
+ )
+
+ assert await _active_run_terminal_monitor_enabled(ctx) is False
+
+
+async def test_active_run_terminal_monitor_keeps_terminal_action_criteria_armed(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def handler(**_: object) -> dict:
+ return {}
+
+ async def fake_completion_verification_handler(_ctx: object) -> object:
+ return handler
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.tools.composition_capture._completion_verification_handler",
+ fake_completion_verification_handler,
+ )
+ ctx = _run_ctx()
+ ctx.browser_session_id = "bs_1"
+ ctx.discovery_mcp_server = object()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ _criterion(
+ "c0",
+ "service request is submitted",
+ kind="terminal_action",
+ terminal_action_family="request",
+ )
+ ]
+ )
+
+ assert await _active_run_terminal_monitor_enabled(ctx) is True
+
+
def test_summarize_unsatisfied_lists_unmet_outcomes() -> None:
criteria = [_criterion("c0", "item in cart"), _criterion("c1", "added exactly once")]
result = _evaluated(("c0", True), ("c1", False))
@@ -2818,6 +3012,29 @@ def test_record_run_blocks_keeps_clean_structural_abstention_as_built_unverified
assert outcome.is_authoritative is False
+def test_record_run_blocks_verifies_structural_requested_output_with_run_corroborator() -> None:
+ ctx = _ctx_with_blocks("extraction")
+ verification = _mixed(
+ CriterionVerdict(criterion_id="c_quotes", state="satisfied", reason_code="evidence_confirms"),
+ CriterionVerdict(
+ criterion_id="c_requested_output",
+ state="unsatisfied",
+ reason_code="structurally_abstained",
+ evidence_ref="block_outputs:extract_first_three_quotes.quotes",
+ output_path="output.quotes",
+ grounding_mode="missing",
+ ),
+ )
+
+ recorded = _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=verification)
+
+ assert recorded is not None
+ assert recorded.verdict == "demonstrated"
+ assert verified_goal_satisfied_context(ctx) is True
+ assert built_unverified_repair_inert_context(ctx) is False
+ assert outcome_fully_verified(ctx) is True
+
+
def test_committed_same_run_outcome_survives_later_contradictory_overwrite() -> None:
ctx = _ctx_with_blocks("extraction")
verification = _mixed(
@@ -3801,10 +4018,87 @@ def test_active_run_terminal_evidence_result_shape_is_not_final_success() -> Non
assert result["ok"] is False
assert result["data"]["active_run_terminal_evidence_detected"] is True
+ assert result["data"]["active_run_terminal_evidence_reason_code"] == ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE
assert result["data"]["full_workflow_verified"] is False
assert result["data"]["failure_categories"][0]["category"] == ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY
+def test_active_run_terminal_evidence_contract_noops_when_outcome_fully_verified() -> None:
+ sample = SimpleNamespace(
+ current_url="https://example.com/cart",
+ page_title="Cart",
+ sample_index=2,
+ completion_verification=_evaluated(("c0", True)),
+ page_evidence={
+ "current_url": "https://example.com/cart",
+ "page_title": "Cart",
+ "visible_text_excerpt": "Cart TESTBRAND PART-001-TEST quantity 1",
+ },
+ )
+ result = _active_run_terminal_evidence_result(
+ workflow_run_id="wr_active",
+ run_status="canceled",
+ sample=sample,
+ requested_block_labels=["search_and_add"],
+ executed_block_labels=["search_and_add"],
+ )
+ ctx = _run_ctx()
+ ctx.completion_verification_result = _evaluated(("c0", True))
+
+ contract = build_diagnosis_repair_contract(source_tool="update_and_run_blocks", result=result, ctx=ctx)
+
+ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.NO_FAILURE
+ assert contract.repair_decision.next_action == RepairNextAction.NO_CHANGE
+ assert contract.verification_result.user_goal_satisfied is True
+ assert contract.verification_result.completion_contract_satisfied is True
+
+
+def test_active_run_terminal_evidence_contract_requires_reason_code_for_verified_noop() -> None:
+ sample = SimpleNamespace(
+ current_url="https://example.com/cart",
+ page_title="Cart",
+ sample_index=2,
+ completion_verification=_evaluated(("c0", True)),
+ page_evidence={"current_url": "https://example.com/cart"},
+ )
+ result = _active_run_terminal_evidence_result(
+ workflow_run_id="wr_active",
+ run_status="canceled",
+ sample=sample,
+ requested_block_labels=["search_and_add"],
+ executed_block_labels=["search_and_add"],
+ )
+ result["data"].pop("active_run_terminal_evidence_reason_code")
+ ctx = _run_ctx()
+ ctx.completion_verification_result = _evaluated(("c0", True))
+
+ contract = build_diagnosis_repair_contract(source_tool="update_and_run_blocks", result=result, ctx=ctx)
+
+ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.ACTIVE_RUN_TERMINAL_EVIDENCE
+ assert contract.repair_decision.next_action == RepairNextAction.STOP
+
+
+def test_terminal_challenge_contract_still_stops_when_outcome_fully_verified() -> None:
+ ctx = _run_ctx()
+ ctx.completion_verification_result = _evaluated(("c0", True))
+ result = {
+ "ok": False,
+ "error": "blocked by human verification",
+ "data": {
+ "workflow_run_id": "wr_blocked",
+ "overall_status": "failed",
+ "failure_categories": [{"category": "ANTI_BOT_DETECTION", "confidence_float": 1.0}],
+ },
+ }
+
+ contract = build_diagnosis_repair_contract(source_tool="update_and_run_blocks", result=result, ctx=ctx)
+
+ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.TERMINAL_CHALLENGE_BLOCKER
+ assert contract.repair_decision.next_action == RepairNextAction.STOP
+ assert contract.verification_result.user_goal_satisfied is False
+ assert contract.verification_result.completion_contract_satisfied is False
+
+
def test_record_active_run_terminal_evidence_keeps_workflow_unverified() -> None:
sample = SimpleNamespace(
current_url="https://example.com/cart",
diff --git a/tests/unit/test_copilot_request_policy_completion_criteria.py b/tests/unit/test_copilot_request_policy_completion_criteria.py
index d8548e427..c7f43bb8d 100644
--- a/tests/unit/test_copilot_request_policy_completion_criteria.py
+++ b/tests/unit/test_copilot_request_policy_completion_criteria.py
@@ -21,6 +21,8 @@ from skyvern.forge.sdk.copilot.request_policy import (
_classify_request,
_parse_completion_criteria,
_render_active_criteria_for_prompt,
+ build_classifier_fallback_floor,
+ is_fallback_floor_base_criterion,
request_policy_has_present_completion_contract,
)
@@ -109,6 +111,17 @@ def _requested_output_subset(policy: RequestPolicy, requested_output_paths: set[
]
+def _run_outcome_corroborators(policy: RequestPolicy) -> list[CompletionCriterion]:
+ return [
+ criterion
+ for criterion in policy.completion_criteria
+ if criterion.level == "run"
+ and criterion.kind == "outcome"
+ and not criterion.method_mandated
+ and criterion.output_path is None
+ ]
+
+
def test_present_completion_contract_helper_accepts_present_status() -> None:
policy = RequestPolicy(completion_contract_status="present")
@@ -351,9 +364,11 @@ async def test_resale_docs_requested_output_uses_explicit_document_name_and_bloc
],
)
- assert [criterion.output_path for criterion in policy.completion_criteria] == ["output.document_name"]
- assert policy.completion_criteria[0].expected_output_value is None
- assert policy.completion_criteria[0].expected_output_shape is None
+ requested_outputs = [criterion for criterion in policy.completion_criteria if criterion.output_path is not None]
+ assert [criterion.output_path for criterion in requested_outputs] == ["output.document_name"]
+ assert requested_outputs[0].expected_output_value is None
+ assert requested_outputs[0].expected_output_shape is None
+ assert _run_outcome_corroborators(policy)[0].requested_output_corroborator is True
@pytest.mark.asyncio
@@ -367,6 +382,162 @@ async def test_named_customer_prose_does_not_become_requested_output_field_name(
assert _criteria_for_path(policy, "output.status")
+async def test_requested_output_canonicalization_preserves_output_corroborator() -> None:
+ policy = await _policy_for_message(
+ "Extract the first 3 quotes and authors, then return JSON with quotes.",
+ [
+ {
+ "id": "c_quotes",
+ "outcome": "The run extracts the first 3 quotes and authors into the returned quotes JSON.",
+ "output_path": "output.quotes",
+ }
+ ],
+ )
+
+ requested = _requested_output_subset(policy, {"output.quotes"})
+ corroborators = _run_outcome_corroborators(policy)
+
+ assert len(requested) == 1
+ assert requested[0].id == "__copilot_requested_output__output_quotes"
+ assert len(corroborators) == 1
+ assert corroborators[0].requested_output_corroborator is True
+ assert corroborators[0].outcome == "The run extracts the first 3 quotes and authors into the returned quotes JSON."
+
+
+async def test_classifier_typed_requested_output_adds_distinct_corroborator() -> None:
+ policy = await _policy_for_message(
+ "Run the read-only extraction.",
+ [
+ {
+ "outcome": "The returned JSON contains the first three public quotes and authors.",
+ "output_path": "output.quotes",
+ }
+ ],
+ )
+
+ requested = _requested_output_subset(policy, {"output.quotes"})
+ corroborators = _run_outcome_corroborators(policy)
+
+ assert len(requested) == 1
+ assert requested[0].id == "c0"
+ assert requested[0].output_path == "output.quotes"
+ assert requested[0].requested_output_corroborator is False
+ assert len(corroborators) == 1
+ assert corroborators[0].id != requested[0].id
+ assert corroborators[0].id == "c0__requested_output_corroborator"
+ assert corroborators[0].output_path is None
+ assert corroborators[0].expected_output_value is None
+ assert corroborators[0].expected_output_shape is None
+ assert corroborators[0].requested_output_corroborator is True
+ assert corroborators[0].outcome == requested[0].outcome
+
+
+async def test_plain_requested_output_text_does_not_fabricate_corroborator() -> None:
+ policy = await _policy_for_message("Return a final record with record id.", [])
+
+ requested = _requested_output_subset(policy, {"output.record_id"})
+
+ assert len(requested) == 1
+ assert requested[0].id == "__copilot_requested_output__output_record_id"
+ assert _run_outcome_corroborators(policy) == []
+
+
+def test_requested_output_canonicalization_preserves_fallback_floor_base_when_it_is_source() -> None:
+ policy = RequestPolicy(completion_criteria=build_classifier_fallback_floor([]))
+
+ _apply_requested_output_completion_criteria(policy, "Return a final record with record id.")
+
+ assert len(_requested_output_subset(policy, {"output.record_id"})) == 1
+ fallback = [
+ criterion for criterion in _run_outcome_corroborators(policy) if is_fallback_floor_base_criterion(criterion)
+ ]
+ assert len(fallback) == 1
+ assert fallback[0].requested_output_corroborator is True
+
+
+def test_requested_output_canonicalization_does_not_promote_method_mandated_only_corroborator() -> None:
+ policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="method_floor",
+ outcome="The workflow runs to its intended end state with the expected output.",
+ method_mandated=True,
+ )
+ ]
+ )
+
+ _apply_requested_output_completion_criteria(policy, "Return a final record with record id.")
+
+ assert len(_requested_output_subset(policy, {"output.record_id"})) == 1
+ assert _run_outcome_corroborators(policy) == []
+
+
+def test_requested_output_corroborator_respects_completion_criteria_cap() -> None:
+ policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(id=f"specific_{index}", outcome=f"Specific retained outcome {index}.")
+ for index in range(7)
+ ]
+ + [
+ CompletionCriterion(
+ id="quotes",
+ outcome="The run extracts the requested record id into the returned JSON.",
+ output_path="output.record_id",
+ )
+ ]
+ )
+
+ _apply_requested_output_completion_criteria(policy, "Return a final record with record id and status.")
+
+ assert len(policy.completion_criteria) == 8
+ assert {
+ criterion.output_path for criterion in _requested_output_subset(policy, {"output.record_id", "output.status"})
+ } == {
+ "output.record_id",
+ "output.status",
+ }
+
+
+def test_requested_output_corroborator_survives_when_requested_outputs_exceed_cap() -> None:
+ policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="name_source",
+ outcome="The run extracts the requested name into the returned JSON.",
+ output_path="output.name",
+ )
+ ]
+ )
+
+ _apply_requested_output_completion_criteria(
+ policy,
+ "Return a final record with name, record id, status, phone, email, license, taxonomy, specialty, and date.",
+ )
+
+ requested_output_paths = {
+ "output.name",
+ "output.record_id",
+ "output.status",
+ "output.phone",
+ "output.email",
+ "output.license",
+ "output.taxonomy",
+ "output.specialty",
+ "output.date",
+ }
+ corroborators = _run_outcome_corroborators(policy)
+
+ assert {
+ criterion.output_path for criterion in _requested_output_subset(policy, requested_output_paths)
+ } == requested_output_paths
+ assert len(corroborators) == 1
+ assert corroborators[0].id == "name_source"
+ assert corroborators[0].requested_output_corroborator is True
+
+
@pytest.mark.asyncio
async def test_requested_output_criteria_do_not_infer_shapes_from_generated_provider_field_names() -> None:
policy = await _policy_for_message(