Skip to content

Automate browser based workflows with AI

Stop recording completed Copilot runs with registered outputs as failures (SKY-12765) (#7613)

Andrew Neilson14974a4baa08
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.py
index de2852bd7..10350ae6d 100644
--- a/skyvern/forge/sdk/copilot/build_test_outcome.py
+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py
@@ -54,6 +54,7 @@ BuildTestOutcomeReasonCode = Literal[
"terminal_challenge_blocker",
"blocker_reported",
"failed_run",
+ "run_completed_unevaluated",
"suspicious_success",
"missing_structural_evidence",
"unchanged_after_recorded_outcome",
@@ -1139,7 +1140,11 @@ def recorded_outcome_from_run_blocks_result(
verdict: BuildTestOutcomeVerdict = "repairable_failure" if bool(result.get("ok")) is False else "progress_observed"
if not structural_identity and not page_refs and not output_refs:
verdict = "not_authoritative"
- reason_code = "runtime_block_failure" if failed_block is not None or not bool(result.get("ok")) else "failed_run"
+ reason_code = (
+ "runtime_block_failure"
+ if failed_block is not None or not bool(result.get("ok"))
+ else "run_completed_unevaluated"
+ )
has_runtime_failure_evidence = bool(failure_categories or failure_type or runtime_failure_identity or failed_block)
if (
verdict == "repairable_failure"
@@ -1555,6 +1560,8 @@ def _run_outcome_reason_code(recorded_run_outcome: RecordedRunOutcome) -> BuildT
return reason_code
if recorded_run_outcome.verdict == "demonstrated":
return "verified_success"
+ if recorded_run_outcome.verdict == "not_evaluated":
+ return "run_completed_unevaluated"
return "failed_run"
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index 34574ea6d..45917b73a 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -905,12 +905,13 @@ def _completion_verification_only_structural_abstentions(ctx: CopilotContext) ->
def verified_goal_satisfied_context(ctx: CopilotContext) -> bool:
- if outcome_fully_verified(ctx):
- return True
- # The judge verdict is authoritative: once it has evaluated, an unconfirmed
- # criterion means the outcome is unmet regardless of run status. The block-count
- # heuristic (which counts method verbs in the request) governs only when there
- # is no evaluated verdict.
+ return outcome_fully_verified(ctx)
+
+
+def built_complete_without_evaluated_outcome(ctx: CopilotContext) -> bool:
+ """A run that looks built and repair-inert but carries no evaluated verdict.
+ It ends the turn like ``built_unverified_repair_inert_context`` does, but must
+ never authorize a verified-satisfaction claim."""
if _outcome_criteria_evaluated(ctx):
return False
if not (
@@ -952,6 +953,7 @@ def gate_decision_trace_fields(ctx: CopilotContext) -> dict[str, bool]:
return {
"gate_satisfied": verified_goal_satisfied_context(ctx),
"gate_built_unverified_repair_inert": built_unverified_repair_inert_context(ctx),
+ "gate_built_complete_without_evaluated_outcome": built_complete_without_evaluated_outcome(ctx),
"gate_claim_authorized": verified_goal_claim_authorized(ctx),
"gate_last_test_ok": ctx.last_test_ok is True,
"gate_last_full_workflow_test_ok": ctx.last_full_workflow_test_ok is True,
@@ -1546,7 +1548,7 @@ def _check_enforcement(
if verified_goal_satisfied_context(ctx):
raise CopilotGoalSatisfied()
- if built_unverified_repair_inert_context(ctx):
+ if built_unverified_repair_inert_context(ctx) or built_complete_without_evaluated_outcome(ctx):
raise CopilotBuiltUnverified()
if _needs_repair_ceiling_halt(ctx):
diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.py
index 6bc9c81c1..027edf7f3 100644
--- a/skyvern/forge/sdk/copilot/tools/completion.py
+++ b/skyvern/forge/sdk/copilot/tools/completion.py
@@ -254,9 +254,9 @@ def _completion_verification_criteria(copilot_ctx: Any) -> list[CompletionCriter
formed_criteria = policy.graded_completion_criteria() if policy is not None else []
criteria = gradeable_completion_criteria(formed_criteria)
authored_output_criteria = _authored_output_contract_criteria(copilot_ctx)
- if authored_output_criteria and (
- not formed_criteria or all(is_fallback_floor_criterion(c) for c in formed_criteria)
- ):
+ # With nothing gradeable left (no criteria formed, or every formed criterion degraded
+ # out), the authored output contract is the only remaining evidence surface.
+ if authored_output_criteria and (not criteria or all(is_fallback_floor_criterion(c) for c in formed_criteria)):
return authored_output_criteria
if _accepted_staged_output_contract_missing(copilot_ctx) and (
not formed_criteria or all(is_fallback_floor_criterion(c) for c in formed_criteria)
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 6d649895f..2a70dc841 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -42,7 +42,11 @@ from skyvern.forge.sdk.copilot.completion_criteria_store import (
plan_persistence,
reconcile_completion_criteria,
)
-from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
+from skyvern.forge.sdk.copilot.completion_verification import (
+ CompletionVerificationResult,
+ CriterionVerdict,
+ gradeable_completion_criteria,
+)
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfig
from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext
from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
@@ -66,6 +70,7 @@ from skyvern.forge.sdk.copilot.enforcement import (
verified_goal_satisfied_context,
)
from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE
+from skyvern.forge.sdk.copilot.hooks import CopilotRunHooks
from skyvern.forge.sdk.copilot.output_policy import (
ACTUATION_OBLIGATION_BROWSER_ACTION_KEY,
ACTUATION_OBLIGATION_STEER_REASON_CODE,
@@ -1181,6 +1186,11 @@ class TestVerifiedGoalSatisfiedStop:
last_full_workflow_test_ok=True,
latest_diagnosis_repair_contract=_verified_goal_contract(),
)
+ ctx.completion_verification_result = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c0"],
+ verdicts=[CriterionVerdict(criterion_id="c0", state="satisfied", reason_code="evidence_confirms")],
+ )
hook = CopilotRunHooks(ctx)
result = json.dumps(
{
@@ -1214,9 +1224,60 @@ class TestVerifiedGoalSatisfiedStop:
last_full_workflow_test_ok=True,
latest_diagnosis_repair_contract=_verified_goal_contract(),
)
+ ctx.completion_verification_result = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c0"],
+ verdicts=[CriterionVerdict(criterion_id="c0", state="satisfied", reason_code="evidence_confirms")],
+ )
assert verified_goal_satisfied_context(ctx)
+ @pytest.mark.asyncio
+ async def test_block_run_hook_does_not_claim_goal_satisfied_without_evaluated_outcome(self) -> None:
+ ctx = _ctx(
+ last_test_ok=True,
+ last_full_workflow_test_ok=True,
+ latest_diagnosis_repair_contract=_verified_goal_contract(),
+ )
+ hook = CopilotRunHooks(ctx)
+ result = json.dumps(
+ {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_1",
+ "blocks": [{"label": "search", "output": {"status": "found"}}],
+ },
+ }
+ )
+
+ assert ctx.completion_verification_result is None
+ assert verified_goal_satisfied_context(ctx) is False
+
+ await hook.on_tool_end(
+ context=MagicMock(),
+ agent=MagicMock(),
+ tool=SimpleNamespace(name="update_and_run_blocks"),
+ result=result,
+ )
+
+ assert ctx.goal_satisfied_tool_name is None
+
+ def test_turn_telemetry_distinguishes_unevaluated_gate_from_repair_inert(self) -> None:
+ from skyvern.forge.sdk.copilot.enforcement import gate_decision_trace_fields
+
+ ctx = _ctx(
+ last_test_ok=True,
+ last_full_workflow_test_ok=True,
+ latest_diagnosis_repair_contract=_verified_goal_contract(),
+ )
+
+ assert ctx.completion_verification_result is None
+ fields = gate_decision_trace_fields(ctx)
+
+ assert fields["gate_built_complete_without_evaluated_outcome"] is True
+ assert fields["gate_built_unverified_repair_inert"] is False
+ assert fields["gate_satisfied"] is False
+
@pytest.mark.asyncio
async def test_wrapped_exception_fallback_reaches_goal_satisfied_after_verified_consume(self) -> None:
ctx = _ctx(
@@ -6869,6 +6930,74 @@ class TestDeclaredEqualsGradedCompletionCriteria:
assert not any(verdict.satisfied for verdict in verification.verdicts)
assert all(verdict.reason_code != "evidence_confirms" for verdict in verification.verdicts)
+ @pytest.mark.asyncio
+ async def test_ungradeable_formed_criteria_fall_back_to_authored_output_contract(self) -> None:
+ label = "collect_top_entry"
+ ctx = SimpleNamespace(
+ request_policy=RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="c0",
+ outcome="the top listed entry is returned",
+ mint_degrade="undecidable_judgment",
+ )
+ ],
+ classifier_status="success",
+ ),
+ code_artifact_metadata={
+ label: {"claimed_outcomes": [{"goal_value_paths": ["output.top_entry"]}]},
+ },
+ workflow_yaml=(
+ "title: Utility path\n"
+ "workflow_definition:\n"
+ " blocks:\n"
+ " - block_type: code\n"
+ f" label: {label}\n"
+ " code: |\n"
+ " return {}\n"
+ ),
+ last_workflow_yaml=None,
+ completion_verification_result=None,
+ copilot_total_timeout_exceeded=False,
+ reached_download_target=None,
+ workflow_verification_evidence=SimpleNamespace(block_verified=[]),
+ verified_prefix_labels=[],
+ verified_block_outputs={},
+ post_run_page_observation_after_failed_test=False,
+ composition_page_evidence=None,
+ completion_criteria_turn_state=None,
+ )
+
+ assert ctx.request_policy.graded_completion_criteria() != []
+ assert gradeable_completion_criteria(ctx.request_policy.graded_completion_criteria()) == []
+
+ criteria = _completion_verification_criteria(ctx)
+ assert [criterion.output_path for criterion in criteria] == ["output.top_entry"]
+
+ verification = await _maybe_run_completion_verification(
+ ctx,
+ {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_completed",
+ "overall_status": "completed",
+ "blocks": [
+ {
+ "label": label,
+ "status": "completed",
+ "extracted_data": {"output": {"top_entry": "First listed entry"}},
+ }
+ ],
+ "executed_block_labels": [label],
+ },
+ },
+ 0,
+ )
+
+ assert verification is not None
+ assert verification.status == "evaluated"
+ assert [verdict.output_path for verdict in verification.verdicts] == ["output.top_entry"]
+
def test_fallback_floor_uses_repair_context_output_contract_paths_when_metadata_missing(self) -> None:
ctx = SimpleNamespace(
request_policy=RequestPolicy(
diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.py
index c2ab22481..ed9562c05 100644
--- a/tests/unit/test_copilot_build_test_outcome.py
+++ b/tests/unit/test_copilot_build_test_outcome.py
@@ -845,10 +845,91 @@ def test_not_evaluated_recorded_outcome_is_not_authoritative_repair_failure() ->
assert outcome is not None
assert outcome.verdict == "not_authoritative"
- assert outcome.reason_code == "failed_run"
+ assert outcome.reason_code == "run_completed_unevaluated"
assert outcome.is_authoritative is False
+def test_completed_run_with_registered_outputs_is_not_classified_as_failed_run() -> None:
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_completed",
+ "overall_status": "completed",
+ "blocks": [{"label": "collect_top_entry", "status": "completed"}],
+ "registered_output_parameter_values": [
+ {"output_parameter_id": "op_1", "value": {"output": {"top_entry": "First listed entry"}}}
+ ],
+ },
+ }
+
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ recorded_run_outcome=RecordedRunOutcome(verdict="not_evaluated", workflow_run_id="wr_completed"),
+ )
+
+ assert outcome is not None
+ assert outcome.reason_code != "failed_run"
+ assert outcome.reason_code == "run_completed_unevaluated"
+
+
+def test_failed_run_classification_preserved_for_not_ok_run() -> None:
+ result = {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_not_ok",
+ "overall_status": "failed",
+ "failure_type": "block_failure",
+ "blocks": [{"label": "collect_top_entry", "status": "completed"}],
+ },
+ }
+
+ outcome = recorded_outcome_from_run_blocks_result(result)
+
+ assert outcome is not None
+ assert outcome.reason_code == "runtime_block_failure"
+ assert outcome.verdict == "repairable_failure"
+
+
+def test_failed_run_classification_preserved_for_failed_block() -> None:
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_failed_block",
+ "overall_status": "completed",
+ "failure_type": "block_failure",
+ "blocks": [{"label": "collect_top_entry", "status": "failed", "failure_reason": "selector not found"}],
+ },
+ }
+
+ outcome = recorded_outcome_from_run_blocks_result(result)
+
+ assert outcome is not None
+ assert outcome.reason_code == "runtime_block_failure"
+
+
+def test_judge_evaluated_non_satisfaction_keeps_failure_classification() -> None:
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_judged",
+ "overall_status": "completed",
+ "blocks": [{"label": "collect_top_entry", "status": "completed"}],
+ },
+ }
+
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ recorded_run_outcome=RecordedRunOutcome(
+ verdict="not_demonstrated",
+ reason_code="outcome_not_demonstrated",
+ workflow_run_id="wr_judged",
+ ),
+ )
+
+ assert outcome is not None
+ assert outcome.reason_code == "outcome_not_demonstrated"
+
+
def test_outcome_not_demonstrated_does_not_mark_presence_only_abstention_as_missing_output() -> None:
result = {
"ok": True,
diff --git a/tests/unit/test_copilot_completion_criteria_lifecycle.py b/tests/unit/test_copilot_completion_criteria_lifecycle.py
index c8923e296..0e407f333 100644
--- a/tests/unit/test_copilot_completion_criteria_lifecycle.py
+++ b/tests/unit/test_copilot_completion_criteria_lifecycle.py
@@ -50,6 +50,9 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
VerificationResult,
)
from skyvern.forge.sdk.copilot.enforcement import (
+ CopilotBuiltUnverified,
+ _check_enforcement,
+ built_complete_without_evaluated_outcome,
built_unverified_repair_inert_context,
verified_goal_claim_authorized,
verified_goal_satisfied_context,
@@ -1426,10 +1429,14 @@ def _no_repair_unverified_contract() -> DiagnosisRepairContract:
def test_claim_closure_turn_still_ends_but_claim_downgrades() -> None:
ctx = _legacy_verified_ctx()
- # Turn completion is unchanged: the legacy conjunction still satisfies the gate.
- assert verified_goal_satisfied_context(ctx) is True
- # The claim tier is closed: no adjudicated evidence, no tested-success claim.
+
+ assert ctx.completion_verification_result is None
+ assert verified_goal_satisfied_context(ctx) is False
assert verified_goal_claim_authorized(ctx) is False
+ assert built_complete_without_evaluated_outcome(ctx) is True
+
+ with pytest.raises(CopilotBuiltUnverified):
+ _check_enforcement(ctx)
def test_structural_abstention_no_repair_terminalizes_without_authorizing_success_claim() -> None: