Skip to content

Automate browser based workflows with AI

SKY-11558: Preserve committed Copilot run outcomes (#6917)

Andrew Neilson3e75fccb1e33
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 355d954bf..be437fdfa 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -1177,6 +1177,8 @@ def _shape_ask_question_response(user_response: str, ctx: CopilotContext) -> str
def _completion_contract_not_violated(ctx: CopilotContext) -> bool:
if artifact_health_blocked(ctx):
return False
+ if outcome_fully_verified(ctx):
+ return True
result = ctx.completion_verification_result
if result is None:
return True
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.py
index 8569879f9..abbc25a46 100644
--- a/skyvern/forge/sdk/copilot/completion_verification.py
+++ b/skyvern/forge/sdk/copilot/completion_verification.py
@@ -104,6 +104,9 @@ class CompletionVerificationResult:
if self.status != "evaluated" or not self.criterion_ids:
return False
verdict_by_id = {verdict.criterion_id: verdict for verdict in self.verdicts}
+ has_observed_reach_state = any(
+ _is_satisfied_observed_end_state_verdict(verdict) for verdict in verdict_by_id.values()
+ )
satisfied_run_plane_count = 0
for criterion_id in self.criterion_ids:
verdict = verdict_by_id.get(criterion_id)
@@ -126,6 +129,8 @@ class CompletionVerificationResult:
continue
if verdict is not None and _is_structural_requested_output_abstention(verdict):
continue
+ if has_observed_reach_state and verdict is not None and _is_reperception_contradiction(verdict):
+ continue
return False
return satisfied_run_plane_count > 0
@@ -1835,6 +1840,24 @@ def _is_meaningful_contingent_antecedent_value(value: Any) -> bool:
_DEFINITION_REASON_PREFIX = "definition_"
+_OBSERVED_END_STATE_EVIDENCE_REF = "observed_end_state_url"
+_REPERCEPTION_CONTRADICTION_EVIDENCE_REFS = frozenset({"scout_synthesized_browser_steps_output"})
+
+
+def _is_satisfied_observed_end_state_verdict(verdict: CriterionVerdict) -> bool:
+ return (
+ verdict.state == "satisfied"
+ and verdict.reason_code == "evidence_confirms"
+ and verdict.evidence_ref == _OBSERVED_END_STATE_EVIDENCE_REF
+ )
+
+
+def _is_reperception_contradiction(verdict: CriterionVerdict) -> bool:
+ return (
+ verdict.state == "unsatisfied"
+ and verdict.reason_code == "evidence_contradicts"
+ and verdict.evidence_ref in _REPERCEPTION_CONTRADICTION_EVIDENCE_REFS
+ )
def _is_definition_plane_abstention(verdict: CriterionVerdict) -> bool:
diff --git a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
index e6f91b220..588f64164 100644
--- a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
+++ b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
@@ -191,6 +191,7 @@ def build_diagnosis_repair_contract(
if next_action == RepairNextAction.REPAIR and not target_blocks and repair_context is not None:
target_blocks = [repair_context.block_label]
user_goal_satisfied, completion_contract_satisfied = _verification_satisfaction(
+ ctx,
run_ok,
suspicious,
run_status,
@@ -612,6 +613,7 @@ def _last_test_anti_bot_is_terminal(ctx: CopilotContext, data: dict[str, Any]) -
def _verification_satisfaction(
+ ctx: CopilotContext,
run_ok: bool,
suspicious: bool,
run_status: str | None,
@@ -627,6 +629,8 @@ def _verification_satisfaction(
return fully_satisfied, fully_satisfied
if failure_type == DiagnosisFailureType.MISSING_CREDENTIAL_OR_INIT:
return False, False
+ if outcome_fully_verified(ctx):
+ return True, True
if (
failure_type == DiagnosisFailureType.NO_FAILURE
and completion_verification is not None
diff --git a/skyvern/forge/sdk/copilot/terminal_predicates.py b/skyvern/forge/sdk/copilot/terminal_predicates.py
index 91f379dd5..efd895391 100644
--- a/skyvern/forge/sdk/copilot/terminal_predicates.py
+++ b/skyvern/forge/sdk/copilot/terminal_predicates.py
@@ -23,6 +23,17 @@ def outcome_criteria_evaluated(ctx: CopilotContext) -> bool:
return result is not None and result.status == "evaluated"
+def _has_same_run_committed_demonstrated_outcome(ctx: CopilotContext) -> bool:
+ outcome = getattr(ctx, "last_run_outcome", None)
+ run_id = getattr(ctx, "last_run_blocks_workflow_run_id", None)
+ return (
+ getattr(outcome, "verdict", None) == "demonstrated"
+ and isinstance(run_id, str)
+ and bool(run_id)
+ and getattr(outcome, "workflow_run_id", None) == run_id
+ )
+
+
def outcome_fully_verified(ctx: CopilotContext) -> bool:
"""Whether the judge confirmed every outcome criterion from this run's evidence.
@@ -31,6 +42,8 @@ def outcome_fully_verified(ctx: CopilotContext) -> bool:
"""
if artifact_health_blocked(ctx):
return False
+ if _has_same_run_committed_demonstrated_outcome(ctx):
+ return True
if not outcome_criteria_evaluated(ctx):
return False
result = ctx.completion_verification_result
diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.py
index 9a5de96d7..f6adaef54 100644
--- a/skyvern/forge/sdk/copilot/tools/completion.py
+++ b/skyvern/forge/sdk/copilot/tools/completion.py
@@ -45,6 +45,7 @@ from skyvern.forge.sdk.copilot.reached_download_target import (
derive_from_block_outputs,
)
from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion
+from skyvern.forge.sdk.copilot.terminal_predicates import outcome_fully_verified
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
from ._shared import (
@@ -1234,6 +1235,8 @@ def _tool_visible_result_after_completion_verification(
result: dict[str, Any],
completion_verification: CompletionVerificationResult | None,
) -> dict[str, Any]:
+ if outcome_fully_verified(copilot_ctx):
+ return result
outcome_unverified_reason = _outcome_unverified_reason(copilot_ctx, completion_verification)
if outcome_unverified_reason is None:
return result
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index d9e85a2db..5d8b758aa 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -165,6 +165,14 @@ from .scouting import _mark_page_inspected, _mark_post_run_page_observed
LOG = structlog.get_logger()
+
+class _RunIdUnset:
+ pass
+
+
+# Distinguishes omitted run id (read ctx) from explicit None (no current run).
+_RUN_ID_UNSET = _RunIdUnset()
+
_ACTIVE_RUN_TERMINAL_MONITOR_INITIAL_DELAY_SECONDS = 30.0
_ACTIVE_RUN_TERMINAL_MONITOR_INTERVAL_SECONDS = 30.0
_ACTIVE_RUN_TERMINAL_MONITOR_MAX_SAMPLES = 8
@@ -2035,9 +2043,13 @@ def _record_run_blocks_result(
# Prefer the final run-result extracted_data for terminal replies;
# it is the same persisted run evidence completion verification saw.
copilot_ctx.verified_terminal_block_outputs = terminal_outputs
+ prior_committed_outcome = _same_run_committed_demonstrated_outcome(
+ copilot_ctx, run_id if isinstance(run_id, str) else None
+ )
copilot_ctx.completion_verification_result = completion_verification
- record_completion_verification(copilot_ctx, completion_verification)
- _record_adjudication_on_turn_state(copilot_ctx, completion_verification)
+ if prior_committed_outcome is None or _verification_fully_satisfied(completion_verification):
+ record_completion_verification(copilot_ctx, completion_verification)
+ _record_adjudication_on_turn_state(copilot_ctx, completion_verification)
if completion_verification is not None and completion_verification.status == "evaluated":
_emit_completion_verification_trace(copilot_ctx, completion_verification)
copilot_ctx.last_run_blocks_workflow_run_id = run_id if isinstance(run_id, str) else None
@@ -2060,8 +2072,9 @@ def _record_run_blocks_result(
copilot_ctx.last_outcome_gate_reason = _outcome_unverified_reason(copilot_ctx, completion_verification)
copilot_ctx.last_outcome_gate_workflow_run_id = copilot_ctx.last_run_blocks_workflow_run_id
copilot_ctx.last_test_suspicious_success = False
- copilot_ctx.last_run_outcome = None
- copilot_ctx.last_run_outcome_block_labels = []
+ if prior_committed_outcome is None:
+ copilot_ctx.last_run_outcome = None
+ copilot_ctx.last_run_outcome_block_labels = []
copilot_ctx.suspicious_success_nudge_count = 0
copilot_ctx.last_test_anti_bot = None
prior_budget_flag = copilot_ctx.last_failure_category_top == PER_TOOL_BUDGET_FAILURE_CATEGORY
@@ -2244,6 +2257,19 @@ def _record_run_blocks_result(
display_reason=run_outcome_display_reason(copilot_ctx.last_test_failure_reason),
),
)
+ if prior_committed_outcome is not None and artifact_reason is None:
+ # artifact_reason is current-run health; prior_committed_outcome already passed prior ctx artifact-health.
+ copilot_ctx.last_full_workflow_test_ok = True
+ copilot_ctx.last_unverified_block_labels = []
+ copilot_ctx.last_good_workflow = copilot_ctx.last_workflow
+ copilot_ctx.last_good_workflow_yaml = copilot_ctx.last_workflow_yaml
+ copilot_ctx.last_test_suspicious_success = False
+ copilot_ctx.last_test_failure_reason = None
+ copilot_ctx.last_outcome_gate_reason = None
+ copilot_ctx.last_outcome_gate_workflow_run_id = None
+ update_repeated_failure_state(copilot_ctx, result)
+ _update_verification_evidence_from_run_result(copilot_ctx, result)
+ return _stash_recorded_run_outcome(copilot_ctx, prior_committed_outcome)
unverified = _unverified_current_workflow_labels(copilot_ctx)
copilot_ctx.last_unverified_block_labels = unverified
outcome_unverified_reason = _outcome_unverified_reason(copilot_ctx, completion_verification)
@@ -2330,14 +2356,45 @@ def _record_run_blocks_result(
def _stash_recorded_run_outcome(copilot_ctx: Any, outcome: RecordedRunOutcome) -> RecordedRunOutcome:
if outcome.workflow_run_id is None:
outcome = replace(outcome, workflow_run_id=getattr(copilot_ctx, "last_run_blocks_workflow_run_id", None))
+ committed = _same_run_committed_demonstrated_outcome(copilot_ctx, outcome.workflow_run_id)
+ if committed is not None and outcome.reason_code == "outcome_not_demonstrated":
+ return committed
copilot_ctx.last_run_outcome = outcome
copilot_ctx.last_run_outcome_block_labels = list(getattr(copilot_ctx, "last_run_blocks_block_labels", []) or [])
return outcome
+def _verification_fully_satisfied(completion_verification: CompletionVerificationResult | None) -> bool:
+ return completion_verification is not None and completion_verification.is_fully_satisfied()
+
+
+def _same_run_committed_demonstrated_outcome(
+ copilot_ctx: Any, workflow_run_id: str | None | _RunIdUnset = _RUN_ID_UNSET
+) -> RecordedRunOutcome | None:
+ artifact_reason = getattr(copilot_ctx, "last_artifact_health_blocker_reason", None)
+ if isinstance(artifact_reason, str) and artifact_reason.strip():
+ return None
+ run_id = (
+ getattr(copilot_ctx, "last_run_blocks_workflow_run_id", None)
+ if workflow_run_id is _RUN_ID_UNSET
+ else workflow_run_id
+ )
+ if not isinstance(run_id, str) or not run_id:
+ return None
+ outcome = getattr(copilot_ctx, "last_run_outcome", None)
+ if not isinstance(outcome, RecordedRunOutcome):
+ return None
+ if outcome.verdict != "demonstrated" or outcome.workflow_run_id != run_id:
+ return None
+ return outcome
+
+
def _adjudicated_run_outcome(
copilot_ctx: Any, completion_verification: CompletionVerificationResult | None
) -> RecordedRunOutcome:
+ committed = _same_run_committed_demonstrated_outcome(copilot_ctx)
+ if committed is not None:
+ return committed
if completion_verification is not None and completion_verification.status == "evaluated":
if not completion_verification.is_fully_satisfied():
return RecordedRunOutcome(
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index 748448323..e3ff52981 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -66,6 +66,7 @@ from skyvern.forge.sdk.copilot.request_policy import (
_parse_completion_criteria,
build_classifier_fallback_floor,
)
+from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
from skyvern.forge.sdk.copilot.tools import (
ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,
_active_run_terminal_evidence_needs_visual_fallback,
@@ -356,6 +357,63 @@ def test_is_fully_satisfied_requires_every_criterion() -> None:
assert _evaluated(("c0", True), ("c1", False)).is_fully_satisfied() is False
+def test_observed_end_state_satisfaction_takes_precedence_over_reperception_contradiction() -> None:
+ result = _mixed(
+ CriterionVerdict(
+ criterion_id="c_reach",
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_ref="observed_end_state_url",
+ ),
+ CriterionVerdict(
+ criterion_id="c_reperception",
+ state="unsatisfied",
+ reason_code="evidence_contradicts",
+ evidence_ref="scout_synthesized_browser_steps_output",
+ ),
+ )
+
+ assert result.is_fully_satisfied() is True
+
+
+def test_reperception_contradiction_without_observed_end_state_satisfaction_still_blocks() -> None:
+ result = _mixed(
+ CriterionVerdict(
+ criterion_id="c_output",
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_ref="block_outputs:submit_request",
+ ),
+ CriterionVerdict(
+ criterion_id="c_reperception",
+ state="unsatisfied",
+ reason_code="evidence_contradicts",
+ evidence_ref="scout_synthesized_browser_steps_output",
+ ),
+ )
+
+ assert result.is_fully_satisfied() is False
+
+
+def test_observed_end_state_satisfaction_does_not_override_requested_output_contradiction() -> None:
+ result = _mixed(
+ CriterionVerdict(
+ criterion_id="c_reach",
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_ref="observed_end_state_url",
+ ),
+ CriterionVerdict(
+ criterion_id="c_requested_output",
+ state="unsatisfied",
+ reason_code="evidence_contradicts",
+ evidence_ref="block_outputs:submit_request.confirmation_number",
+ ),
+ )
+
+ assert result.is_fully_satisfied() is False
+
+
def _mixed(*verdicts: CriterionVerdict) -> CompletionVerificationResult:
return CompletionVerificationResult(
status="evaluated", criterion_ids=[v.criterion_id for v in verdicts], verdicts=list(verdicts)
@@ -2011,20 +2069,44 @@ def test_parse_caps_count() -> None:
assert len(_parse_completion_criteria(raw)) == 8
+def _verification_satisfaction_ctx(
+ completion_verification: CompletionVerificationResult | None = None,
+) -> SimpleNamespace:
+ return SimpleNamespace(
+ completion_verification_result=completion_verification,
+ last_artifact_health_blocker_reason=None,
+ last_run_blocks_workflow_run_id=None,
+ last_run_outcome=None,
+ )
+
+
def test_verification_satisfaction_no_cvr_uses_prior_proxy() -> None:
- assert _verification_satisfaction(True, False, "completed", None) == (True, True)
- assert _verification_satisfaction(True, True, "completed", None) == (False, False)
- assert _verification_satisfaction(False, False, None, None) == (None, None)
+ ctx = _verification_satisfaction_ctx()
+ assert _verification_satisfaction(ctx, True, False, "completed", None) == (True, True)
+ assert _verification_satisfaction(ctx, True, True, "completed", None) == (False, False)
+ assert _verification_satisfaction(ctx, False, False, None, None) == (None, None)
def test_verification_satisfaction_evaluated_drives_contract_signal() -> None:
- assert _verification_satisfaction(True, False, "completed", _evaluated(("c0", True))) == (True, True)
- _, contract = _verification_satisfaction(True, False, "completed", _evaluated(("c0", False)))
+ satisfied = _evaluated(("c0", True))
+ unsatisfied = _evaluated(("c0", False))
+ assert _verification_satisfaction(
+ _verification_satisfaction_ctx(satisfied), True, False, "completed", satisfied
+ ) == (
+ True,
+ True,
+ )
+ _, contract = _verification_satisfaction(
+ _verification_satisfaction_ctx(unsatisfied), True, False, "completed", unsatisfied
+ )
assert contract is False
def test_verification_satisfaction_unavailable_fails_closed() -> None:
- _, contract = _verification_satisfaction(True, False, "completed", CompletionVerificationResult("unavailable"))
+ unavailable = CompletionVerificationResult("unavailable")
+ _, contract = _verification_satisfaction(
+ _verification_satisfaction_ctx(unavailable), True, False, "completed", unavailable
+ )
assert contract is False
@@ -2458,6 +2540,29 @@ def test_tool_visible_result_keeps_mid_build_run_visible_success() -> None:
assert visible["ok"] is True
+def test_tool_visible_result_keeps_committed_same_run_success_after_later_contradiction() -> None:
+ ctx = _ctx_with_blocks("extraction")
+ result = _clean_success_result()
+
+ _record_run_blocks_result(ctx, result, completion_verification=_evaluated(("c0", True)))
+ visible = _tool_visible_result_after_completion_verification(ctx, result, _contradicted("c0"))
+
+ assert visible["ok"] is True
+ assert visible is result
+ assert "failure_categories" not in visible["data"]
+ assert "completion_verification" not in visible["data"]
+
+
+def test_tool_visible_result_downgrades_first_pass_contradiction_without_committed_outcome() -> None:
+ ctx = _ctx_with_blocks("extraction")
+ result = _clean_success_result()
+
+ visible = _tool_visible_result_after_completion_verification(ctx, result, _contradicted("c0"))
+
+ assert visible["ok"] is False
+ assert visible["data"]["failure_categories"][0]["category"] == "OUTCOME_UNVERIFIED"
+
+
def test_record_run_blocks_keeps_building_on_mid_build_no_evidence() -> None:
ctx = _ctx_with_blocks("goto_url", "navigation")
_record_run_blocks_result(ctx, _clean_success_result(), completion_verification=_evaluated(("c0", False)))
@@ -2494,6 +2599,138 @@ def test_record_run_blocks_demonstrated_when_lone_definition_abstention_with_con
assert verified_goal_satisfied_context(ctx) is True
+def test_committed_same_run_outcome_survives_later_contradictory_overwrite() -> None:
+ ctx = _ctx_with_blocks("extraction")
+ verification = _mixed(
+ CriterionVerdict(criterion_id="c0", state="satisfied", reason_code="evidence_confirms"),
+ CriterionVerdict(criterion_id="c1", state="unknown", reason_code="definition_parameters_absent"),
+ )
+
+ recorded = _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=verification)
+
+ assert recorded == RecordedRunOutcome(verdict="demonstrated", workflow_run_id="wr_x")
+ assert outcome_fully_verified(ctx) is True
+
+ ctx.completion_verification_result = _mixed(
+ CriterionVerdict(criterion_id="c0", state="unsatisfied", reason_code="evidence_contradicts"),
+ CriterionVerdict(criterion_id="c1", state="unknown", reason_code="definition_parameters_absent"),
+ )
+
+ assert outcome_fully_verified(ctx) is True
+ assert verified_goal_satisfied_context(ctx) is True
+
+
+def test_first_pass_contradiction_without_committed_run_outcome_still_fails() -> None:
+ ctx = _ctx_with_blocks("extraction")
+
+ recorded = _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=_contradicted("c0"))
+
+ assert recorded is not None
+ assert recorded.verdict == "not_demonstrated"
+ assert outcome_fully_verified(ctx) is False
+ assert verified_goal_satisfied_context(ctx) is False
+
+
+def test_same_run_contradiction_after_committed_outcome_does_not_churn() -> None:
+ ctx = _ctx_with_blocks("extraction")
+
+ _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=_evaluated(("c0", True)))
+ recorded = _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=_contradicted("c0"))
+
+ assert recorded == RecordedRunOutcome(verdict="demonstrated", workflow_run_id="wr_x")
+ assert ctx.last_test_suspicious_success is False
+ assert ctx.last_test_failure_reason is None
+ assert ctx.last_full_workflow_test_ok is True
+ assert outcome_fully_verified(ctx) is True
+
+
+def test_missing_run_id_does_not_preserve_committed_same_run_outcome() -> None:
+ ctx = _ctx_with_blocks("extraction")
+ malformed_result = _clean_success_result()
+ malformed_result["data"].pop("workflow_run_id")
+
+ _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=_evaluated(("c0", True)))
+ recorded = _record_run_blocks_result(ctx, malformed_result, completion_verification=_contradicted("c0"))
+
+ assert recorded is not None
+ assert recorded.verdict == "not_demonstrated"
+ assert ctx.last_run_outcome == recorded
+ assert outcome_fully_verified(ctx) is False
+
+
+def test_committed_same_run_outcome_surfaces_verified_workflow_after_later_contradiction() -> None:
+ ctx = _ctx_with_blocks("extraction")
+ ctx.last_workflow_yaml = "workflow: {}"
+
+ _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=_evaluated(("c0", True)))
+ ctx.completion_verification_result = _contradicted("c0")
+
+ assert _completion_contract_not_violated(ctx) is True
+ assert _verified_workflow_or_none(ctx) == (ctx.last_workflow, "workflow: {}")
+
+
+def test_first_pass_contradiction_does_not_surface_verified_workflow() -> None:
+ ctx = _ctx_with_blocks("extraction")
+ ctx.last_workflow_yaml = "workflow: {}"
+
+ _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=_contradicted("c0"))
+
+ assert _completion_contract_not_violated(ctx) is False
+ assert _verified_workflow_or_none(ctx) == (None, None)
+
+
+def test_different_run_id_committed_outcome_does_not_surface_verified_workflow() -> None:
+ ctx = _ctx_with_blocks("extraction")
+ ctx.last_workflow_yaml = "workflow: {}"
+ ctx.last_test_ok = True
+ ctx.last_full_workflow_test_ok = True
+ ctx.last_run_blocks_workflow_run_id = "wr_new"
+ ctx.last_run_outcome = RecordedRunOutcome(verdict="demonstrated", workflow_run_id="wr_prior")
+ ctx.completion_verification_result = _contradicted("c0")
+
+ assert outcome_fully_verified(ctx) is False
+ assert _completion_contract_not_violated(ctx) is False
+ assert _verified_workflow_or_none(ctx) == (None, None)
+
+
+def test_missing_current_run_id_does_not_surface_verified_workflow_from_prior_outcome() -> None:
+ ctx = _ctx_with_blocks("extraction")
+ ctx.last_workflow_yaml = "workflow: {}"
+ ctx.last_test_ok = True
+ ctx.last_full_workflow_test_ok = True
+ ctx.last_run_blocks_workflow_run_id = None
+ ctx.last_run_outcome = RecordedRunOutcome(verdict="demonstrated", workflow_run_id="wr_prior")
+ ctx.completion_verification_result = _contradicted("c0")
+
+ assert outcome_fully_verified(ctx) is False
+ assert _completion_contract_not_violated(ctx) is False
+ assert _verified_workflow_or_none(ctx) == (None, None)
+
+
+def test_artifact_health_blocks_verified_workflow_surfacing_with_committed_outcome() -> None:
+ ctx = _ctx_with_blocks("extraction")
+ ctx.last_workflow_yaml = "workflow: {}"
+
+ _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=_evaluated(("c0", True)))
+ ctx.last_artifact_health_blocker_reason = "Code block failed with SyntaxError."
+
+ assert outcome_fully_verified(ctx) is False
+ assert _completion_contract_not_violated(ctx) is False
+ assert _verified_workflow_or_none(ctx) == (None, None)
+
+
+def test_unavailable_verification_without_committed_outcome_fails_closed_for_surfacing() -> None:
+ ctx = _ctx_with_blocks("extraction")
+ ctx.last_workflow_yaml = "workflow: {}"
+ ctx.last_test_ok = True
+ ctx.last_full_workflow_test_ok = True
+ ctx.completion_verification_result = CompletionVerificationResult(status="unavailable")
+
+ assert outcome_fully_verified(ctx) is False
+ assert _completion_contract_not_violated(ctx) is False
+ assert _verified_workflow_or_none(ctx) == (None, None)
+
+
def _goto_only_result() -> dict:
return {
"ok": True,
@@ -3166,6 +3403,17 @@ def test_artifact_health_blocks_fully_satisfied_failed_run() -> None:
assert _verified_workflow_or_none(ctx) == (None, None)
+def test_artifact_health_blocks_committed_same_run_outcome() -> None:
+ ctx = _ctx_with_blocks("extraction")
+ verification = _evaluated(("c0", True))
+
+ _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=verification)
+ ctx.last_artifact_health_blocker_reason = "Code block failed with SyntaxError."
+
+ assert outcome_fully_verified(ctx) is False
+ assert verified_goal_satisfied_context(ctx) is False
+
+
@pytest.mark.asyncio
async def test_maybe_run_completion_verification_runs_on_canceled_run(monkeypatch: pytest.MonkeyPatch) -> None:
async def handler(**_: object) -> dict:
diff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.py
index 61f57065b..60dc3d9f7 100644
--- a/tests/unit/test_copilot_diagnosis_repair_contract.py
+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py
@@ -18,9 +18,11 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
RepairNextAction,
build_diagnosis_repair_contract,
)
+from skyvern.forge.sdk.copilot.enforcement import latest_diagnosis_contract_satisfies_goal
from skyvern.forge.sdk.copilot.run_outcome import (
TERMINAL_CHALLENGE_BLOCKER_REASON_CODE,
TERMINAL_CHALLENGE_RUN_OUTCOME_REASON_CODE,
+ RecordedRunOutcome,
)
from skyvern.forge.sdk.copilot.runtime_authoring_repair import (
finalize_runtime_authoring_repair_context_from_page_observation,
@@ -57,6 +59,14 @@ def _satisfied_completion_verification() -> CompletionVerificationResult:
)
+def _contradictory_completion_verification() -> CompletionVerificationResult:
+ return CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c0"],
+ verdicts=[CriterionVerdict(criterion_id="c0", state="unsatisfied", reason_code="evidence_contradicts")],
+ )
+
+
def _clean_completed_result() -> dict[str, object]:
return {
"ok": True,
@@ -969,6 +979,46 @@ def test_clean_run_with_satisfied_completion_verification_has_no_repair_or_block
assert contract.verification_result.remaining_blocker is None
+def test_committed_same_run_outcome_satisfies_diagnosis_after_later_contradiction() -> None:
+ ctx = _ctx()
+ ctx.last_run_blocks_workflow_run_id = "wr_clean"
+ ctx.last_run_outcome = RecordedRunOutcome(verdict="demonstrated", workflow_run_id="wr_clean")
+ ctx.completion_verification_result = _contradictory_completion_verification()
+
+ contract = build_diagnosis_repair_contract(
+ source_tool="update_and_run_blocks",
+ result=_clean_completed_result(),
+ ctx=ctx,
+ workflow_updated=True,
+ )
+ ctx.latest_diagnosis_repair_contract = contract
+
+ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.NO_FAILURE
+ assert contract.repair_decision.next_action == RepairNextAction.NO_CHANGE
+ assert contract.verification_result.user_goal_satisfied is True
+ assert contract.verification_result.completion_contract_satisfied is True
+ assert latest_diagnosis_contract_satisfies_goal(ctx) is True
+
+
+def test_first_pass_contradiction_does_not_satisfy_latest_diagnosis_contract() -> None:
+ ctx = _ctx()
+ ctx.completion_verification_result = _contradictory_completion_verification()
+
+ contract = build_diagnosis_repair_contract(
+ source_tool="update_and_run_blocks",
+ result=_clean_completed_result(),
+ ctx=ctx,
+ workflow_updated=True,
+ )
+ ctx.latest_diagnosis_repair_contract = contract
+
+ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.SUSPICIOUS_SUCCESS
+ assert contract.repair_decision.next_action == RepairNextAction.REPAIR
+ assert contract.verification_result.user_goal_satisfied is False
+ assert contract.verification_result.completion_contract_satisfied is False
+ assert latest_diagnosis_contract_satisfies_goal(ctx) is False
+
+
def test_no_change_contracts_do_not_carry_remaining_blocker() -> None:
satisfied_ctx = _ctx()
satisfied_ctx.completion_verification_result = _satisfied_completion_verification()
diff --git a/tests/unit/test_copilot_run_outcome_event.py b/tests/unit/test_copilot_run_outcome_event.py
index c5fbc90bf..207a8d362 100644
--- a/tests/unit/test_copilot_run_outcome_event.py
+++ b/tests/unit/test_copilot_run_outcome_event.py
@@ -18,11 +18,14 @@ from skyvern.forge.sdk.copilot import tools as copilot_tools
from skyvern.forge.sdk.copilot.agent import _build_narrative_payload
from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.enforcement import outcome_fully_verified
from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome, run_outcome_display_reason
from skyvern.forge.sdk.copilot.tools import run_execution
from skyvern.forge.sdk.copilot.tools.run_execution import (
+ _adjudicated_run_outcome,
_record_run_blocks_result,
+ _stash_recorded_run_outcome,
_verify_and_record_run_blocks_result,
)
from skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotRunOutcomeUpdate
@@ -136,6 +139,48 @@ def _evaluated(satisfied: bool) -> CompletionVerificationResult:
return CompletionVerificationResult(status="evaluated", criterion_ids=["c0"], verdicts=[verdict])
+def _mixed_observed_reach_state_with_reperception_contradiction() -> CompletionVerificationResult:
+ return CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c_reach", "c_reperception"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="c_reach",
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_ref="observed_end_state_url",
+ ),
+ CriterionVerdict(
+ criterion_id="c_reperception",
+ state="unsatisfied",
+ reason_code="evidence_contradicts",
+ evidence_ref="scout_synthesized_browser_steps_output",
+ ),
+ ],
+ )
+
+
+def _mixed_observed_reach_state_with_requested_output_contradiction() -> CompletionVerificationResult:
+ return CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c_reach", "c_requested_output"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="c_reach",
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_ref="observed_end_state_url",
+ ),
+ CriterionVerdict(
+ criterion_id="c_requested_output",
+ state="unsatisfied",
+ reason_code="evidence_contradicts",
+ evidence_ref="block_outputs:search_registry_person.confirmation_number",
+ ),
+ ],
+ )
+
+
def _run_outcome_frames(stream: _FakeStream) -> list[WorkflowCopilotRunOutcomeUpdate]:
return [frame for frame in stream.sent if isinstance(frame, WorkflowCopilotRunOutcomeUpdate)]
@@ -406,6 +451,94 @@ def test_recorded_run_outcome_carries_producing_workflow_run_id() -> None:
assert ctx.last_run_outcome.workflow_run_id == "wr_test"
+def test_recorded_run_outcome_commits_same_adjudication_reach_state_over_reperception_contradiction() -> None:
+ ctx = _ctx([_code_block("search_registry_person", {"records": []})])
+
+ outcome = _record_run_blocks_result(
+ ctx,
+ _run_result([_code_block("search_registry_person", {"records": []})]),
+ completion_verification=_mixed_observed_reach_state_with_reperception_contradiction(),
+ )
+
+ assert outcome == RecordedRunOutcome(verdict="demonstrated", workflow_run_id="wr_test")
+ assert ctx.last_run_outcome == outcome
+ assert ctx.last_full_workflow_test_ok is True
+ assert ctx.last_test_suspicious_success is False
+ assert ctx.last_test_failure_reason is None
+ assert outcome_fully_verified(ctx) is True
+
+
+def test_recorded_run_outcome_does_not_commit_requested_output_contradiction() -> None:
+ ctx = _ctx([_code_block("search_registry_person", {"records": []})])
+
+ outcome = _record_run_blocks_result(
+ ctx,
+ _run_result([_code_block("search_registry_person", {"records": []})]),
+ completion_verification=_mixed_observed_reach_state_with_requested_output_contradiction(),
+ )
+
+ assert outcome is not None
+ assert outcome.verdict == "not_demonstrated"
+ assert ctx.last_run_outcome == outcome
+ assert outcome_fully_verified(ctx) is False
+
+
+def test_adjudication_keeps_committed_same_run_demonstrated_outcome() -> None:
+ ctx = _ctx([_code_block("search_registry_person", {"records": []})])
+ ctx.last_run_blocks_workflow_run_id = "wr_test"
+ committed = _stash_recorded_run_outcome(ctx, RecordedRunOutcome(verdict="demonstrated"))
+
+ assert committed == RecordedRunOutcome(verdict="demonstrated", workflow_run_id="wr_test")
+
+ adjudicated = _adjudicated_run_outcome(ctx, _evaluated(satisfied=False))
+ stashed = _stash_recorded_run_outcome(
+ ctx,
+ RecordedRunOutcome(
+ verdict="not_demonstrated",
+ reason_code="outcome_not_demonstrated",
+ workflow_run_id="wr_test",
+ ),
+ )
+
+ assert adjudicated == committed
+ assert stashed == committed
+ assert ctx.last_run_outcome == committed
+
+
+def test_adjudication_does_not_keep_committed_outcome_for_new_run() -> None:
+ ctx = _ctx([_code_block("search_registry_person", {"records": []})])
+ ctx.last_run_outcome = RecordedRunOutcome(verdict="demonstrated", workflow_run_id="wr_prior")
+ ctx.last_run_blocks_workflow_run_id = "wr_test"
+
+ adjudicated = _adjudicated_run_outcome(ctx, _evaluated(satisfied=False))
+
+ assert adjudicated.verdict == "not_demonstrated"
+
+
+async def test_missing_run_id_does_not_emit_committed_demonstrated_outcome(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ result = _clean_run_result()
+ result["data"].pop("workflow_run_id")
+ ctx = _ctx(result["data"]["blocks"])
+ ctx.last_run_outcome = RecordedRunOutcome(verdict="demonstrated", workflow_run_id="wr_test")
+ ctx.last_run_blocks_workflow_run_id = "wr_test"
+
+ async def _stub_verification(*args: Any, **kwargs: Any) -> CompletionVerificationResult:
+ return _evaluated(satisfied=False)
+
+ monkeypatch.setattr(run_execution, "_maybe_run_completion_verification", _stub_verification)
+
+ await _verify_and_record_run_blocks_result(ctx, result, time.monotonic())
+
+ frames = _run_outcome_frames(ctx.stream) # type: ignore[arg-type]
+ assert [frame.verdict for frame in frames] == ["evaluating", "not_demonstrated"]
+ assert frames[-1].workflow_run_id != "wr_test"
+ assert ctx.last_run_outcome is not None
+ assert ctx.last_run_outcome.verdict == "not_demonstrated"
+
+
def test_both_consumers_route_through_single_producer() -> None:
source = inspect.getsource(copilot_tools)
assert source.count("await _verify_and_record_run_blocks_result(") == 2