Skip to content

Automate browser based workflows with AI

fix(SKY-11302): fail-safe completion verdict on zero gradeable run-plane criteria (#6852)

Andrew Neilsona8783cfbdbe9
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.py
index 6ff5f1109..e056e1478 100644
--- a/skyvern/forge/sdk/copilot/completion_verification.py
+++ b/skyvern/forge/sdk/copilot/completion_verification.py
@@ -75,23 +75,28 @@ class CompletionVerificationResult:
status: VerificationStatus
criterion_ids: list[str] = field(default_factory=list)
verdicts: list[CriterionVerdict] = field(default_factory=list)
+ no_gradeable_run_plane: bool = False
def is_fully_satisfied(self) -> bool:
if self.status != "evaluated" or not self.criterion_ids:
return False
verdict_by_id = {verdict.criterion_id: verdict for verdict in self.verdicts}
- satisfied_count = 0
+ satisfied_run_plane_count = 0
for criterion_id in self.criterion_ids:
verdict = verdict_by_id.get(criterion_id)
if verdict is not None and verdict.satisfied:
- satisfied_count += 1
+ # A definition-plane satisfied verdict proves the workflow is configurable,
+ # never that a run reached the outcome, so it cannot authorize verified success
+ # on its own — only a satisfied run-plane verdict can.
+ if not verdict.reason_code.startswith(_DEFINITION_REASON_PREFIX):
+ satisfied_run_plane_count += 1
continue
# A definition-plane ``unknown`` is a YAML-grader abstention, not a refutation,
# so it must not veto a run whose observable outcome evidence is fully confirmed.
if verdict is not None and _is_definition_plane_abstention(verdict):
continue
return False
- return satisfied_count > 0
+ return satisfied_run_plane_count > 0
def to_trace_data(self) -> dict[str, Any]:
unmet = [verdict for verdict in self.verdicts if not verdict.satisfied]
@@ -107,6 +112,7 @@ class CompletionVerificationResult:
"unsatisfied_count": sum(1 for verdict in self.verdicts if verdict.state == "unsatisfied"),
"unknown_count": sum(1 for verdict in self.verdicts if verdict.state == "unknown"),
"fully_satisfied": self.is_fully_satisfied(),
+ "no_gradeable_run_plane": self.no_gradeable_run_plane,
"reason_codes": [verdict.reason_code for verdict in self.verdicts],
"unmet_criterion_ids": [verdict.criterion_id for verdict in unmet],
"missing_evidence": missing_evidence,
@@ -1155,7 +1161,12 @@ def combine_verification_results(
verdict_by_id.get(cid, CriterionVerdict(criterion_id=cid, state="unknown", reason_code="unknown"))
for cid in criterion_ids
]
- return CompletionVerificationResult(status="evaluated", criterion_ids=list(criterion_ids), verdicts=verdicts)
+ return CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=list(criterion_ids),
+ verdicts=verdicts,
+ no_gradeable_run_plane=run_result is None,
+ )
async def evaluate_completion_criteria(
diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.py
index 6903d4878..43e126192 100644
--- a/skyvern/forge/sdk/copilot/tools/completion.py
+++ b/skyvern/forge/sdk/copilot/tools/completion.py
@@ -62,6 +62,23 @@ def _split_criteria_by_plane(criteria: list[Any]) -> tuple[list[CompletionCriter
return run_criteria, definition_criteria
+def _classifier_status(copilot_ctx: Any) -> str:
+ policy = getattr(copilot_ctx, "request_policy", None)
+ return policy.classifier_status if policy is not None else "not_run"
+
+
+def _no_gradeable_run_plane_result(criterion_ids: list[str]) -> CompletionVerificationResult:
+ return CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=list(criterion_ids),
+ verdicts=[
+ CriterionVerdict(criterion_id=criterion_id, state="unknown", reason_code="unknown")
+ for criterion_id in criterion_ids
+ ],
+ no_gradeable_run_plane=True,
+ )
+
+
def _definition_plane_workflow_yaml(copilot_ctx: Any) -> str | None:
last_yaml = getattr(copilot_ctx, "last_workflow_yaml", None)
if isinstance(last_yaml, str) and last_yaml.strip():
@@ -134,10 +151,17 @@ async def _maybe_run_completion_verification_from_page_observation(
if getattr(copilot_ctx, "post_run_page_observation_after_failed_test", False) is not True:
return None
criteria = _completion_verification_criteria(copilot_ctx)
- if not criteria:
- return None
run_criteria, definition_criteria = _split_criteria_by_plane(criteria)
criterion_ids = [criterion.id for criterion in criteria]
+ if _classifier_status(copilot_ctx) == "fallback" and not run_criteria:
+ verification = _no_gradeable_run_plane_result(criterion_ids)
+ copilot_ctx.completion_verification_result = verification
+ record_completion_verification(copilot_ctx, verification)
+ _record_adjudication_on_turn_state(copilot_ctx, verification)
+ _emit_completion_verification_trace(copilot_ctx, verification)
+ return verification
+ if not criteria:
+ return None
definition_verdicts = (
grade_definition_criteria(definition_criteria, _definition_plane_workflow_yaml(copilot_ctx))
if definition_criteria
@@ -538,16 +562,18 @@ async def _maybe_run_completion_verification(
) -> CompletionVerificationResult | None:
if getattr(copilot_ctx, "copilot_total_timeout_exceeded", False):
return None
- criteria = _completion_verification_criteria(copilot_ctx)
- if not criteria:
- return None
if not (
_is_outcome_evidence_candidate(copilot_ctx, result)
or _is_unfinished_run_verification_candidate(copilot_ctx, result)
):
return None
+ criteria = _completion_verification_criteria(copilot_ctx)
run_criteria, definition_criteria = _split_criteria_by_plane(criteria)
criterion_ids = [criterion.id for criterion in criteria]
+ if _classifier_status(copilot_ctx) == "fallback" and not run_criteria:
+ return _no_gradeable_run_plane_result(criterion_ids)
+ if not criteria:
+ return None
definition_verdicts = (
grade_definition_criteria(definition_criteria, _definition_plane_workflow_yaml(copilot_ctx))
if definition_criteria
@@ -635,6 +661,11 @@ def _outcome_unverified_reason(
if completion_verification.status == "evaluated":
if completion_verification.is_fully_satisfied():
return None
+ if completion_verification.no_gradeable_run_plane:
+ return (
+ "The run completed but the goal outcome could not be independently verified "
+ "(no run-gradeable outcome in this contract). Review the draft before using it."
+ )
policy = getattr(copilot_ctx, "request_policy", None)
criteria: list[CompletionCriterion] = list(policy.completion_criteria) if policy is not None else []
known_good = _known_good_revision_hint(copilot_ctx)
diff --git a/tests/unit/test_copilot_completion_zero_run_plane.py b/tests/unit/test_copilot_completion_zero_run_plane.py
new file mode 100644
index 000000000..a604eaa11
--- /dev/null
+++ b/tests/unit/test_copilot_completion_zero_run_plane.py
@@ -0,0 +1,368 @@
+from __future__ import annotations
+
+import time
+from types import SimpleNamespace
+from typing import Any
+
+import pytest
+
+from skyvern.forge.sdk.copilot.agent import _should_apply_code_only_success_without_review
+from skyvern.forge.sdk.copilot.completion_criteria_store import (
+ CompletionCriteriaTurnState,
+ note_adjudication_on_turn_state,
+)
+from skyvern.forge.sdk.copilot.completion_verification import (
+ CompletionVerificationResult,
+ CriterionVerdict,
+ combine_verification_results,
+)
+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
+from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy
+from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
+from skyvern.forge.sdk.copilot.tools import _record_run_blocks_result
+from skyvern.forge.sdk.copilot.tools.completion import (
+ _completion_evidence_payload,
+ _maybe_run_completion_verification,
+ _maybe_run_completion_verification_from_page_observation,
+ _outcome_unverified_reason,
+)
+
+_NO_GRADEABLE_PROSE = "could not be independently verified"
+_ADD_OR_FIX_PROSE = "Add or fix the block"
+
+
+def _criterion(cid: str, outcome: str, *, level: str = "run", method_mandated: bool = False) -> CompletionCriterion:
+ return CompletionCriterion(id=cid, outcome=outcome, level=level, method_mandated=method_mandated) # type: ignore[arg-type]
+
+
+def _ctx() -> CopilotContext:
+ return CopilotContext(
+ organization_id="o",
+ workflow_id="w",
+ workflow_permanent_id="wp",
+ workflow_yaml="",
+ browser_session_id=None,
+ stream=SimpleNamespace(), # type: ignore[arg-type]
+ user_message="look up the order and report the order number",
+ )
+
+
+def _real_output_result() -> dict[str, Any]:
+ return {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_real",
+ "overall_status": "completed",
+ "executed_block_labels": ["lookup"],
+ "current_url": "https://example.com/orders/A1B2C3",
+ "page_title": "Order Confirmation",
+ "blocks": [
+ {
+ "label": "lookup",
+ "block_type": "EXTRACTION",
+ "status": "completed",
+ "extracted_data": {"extracted_information": {"order_number": "A1B2C3"}},
+ }
+ ],
+ },
+ }
+
+
+def _no_handler(_ctx: object) -> object:
+ raise AssertionError("the zero-run-plane gate must return before the completion judge handler is resolved")
+
+
+def _handler_lookup(handler: object) -> object:
+ async def _lookup(_ctx: object) -> object:
+ return handler
+
+ return _lookup
+
+
+async def test_empty_fallback_gate_fires_above_criteria_guard(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.tools.completion._completion_verification_handler", _no_handler)
+ ctx = _ctx()
+ ctx.request_policy = RequestPolicy(completion_criteria=[], classifier_status="fallback")
+
+ verification = await _maybe_run_completion_verification(ctx, _real_output_result(), time.monotonic())
+
+ assert verification is not None
+ assert verification.status == "evaluated"
+ assert verification.no_gradeable_run_plane is True
+ assert verification.is_fully_satisfied() is False
+ assert verification.criterion_ids == []
+
+
+async def test_all_method_mandated_floor_fallback_gate_fires(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.tools.completion._completion_verification_handler", _no_handler)
+ ctx = _ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[_criterion("floor", "navigated to the directory", method_mandated=True)],
+ classifier_status="fallback",
+ )
+
+ verification = await _maybe_run_completion_verification(ctx, _real_output_result(), time.monotonic())
+
+ assert verification is not None
+ assert verification.no_gradeable_run_plane is True
+ assert verification.is_fully_satisfied() is False
+ assert verification.criterion_ids == []
+
+
+async def test_fallback_with_real_run_plane_criterion_does_not_fire(monkeypatch: pytest.MonkeyPatch) -> None:
+ async def handler(**_: object) -> dict:
+ return {"verdicts": [{"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"}]}
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.tools.completion._completion_verification_handler",
+ _handler_lookup(handler),
+ )
+ ctx = _ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[_criterion("c0", "the order number is reported")],
+ classifier_status="fallback",
+ )
+
+ verification = await _maybe_run_completion_verification(ctx, _real_output_result(), time.monotonic())
+
+ assert verification is not None
+ assert verification.no_gradeable_run_plane is False
+ assert verification.is_fully_satisfied() is True
+
+
+async def test_registered_download_in_block_output_reaches_judge_but_does_not_satisfy(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ captured: dict[str, Any] = {}
+
+ async def handler(*, prompt: str, prompt_name: str) -> dict:
+ captured["prompt"] = prompt
+ return {"verdicts": [{"criterion_id": "c0", "satisfied": False, "reason_code": "no_evidence"}]}
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.tools.completion._completion_verification_handler",
+ _handler_lookup(handler),
+ )
+ ctx = _ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[_criterion("c0", "the requested file is downloaded")],
+ classifier_status="success",
+ )
+ ctx.last_workflow = SimpleNamespace(
+ workflow_definition=SimpleNamespace(blocks=[SimpleNamespace(block_type="extraction", label="lookup")])
+ )
+ block_extracted_data = {
+ "extracted_information": {"summary": "file saved"},
+ "downloaded_files": ["receipt.pdf"],
+ }
+ output: dict[str, Any] = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_download",
+ "overall_status": "completed",
+ "executed_block_labels": ["lookup"],
+ "blocks": [
+ {
+ "label": "lookup",
+ "block_type": "EXTRACTION",
+ "status": "completed",
+ "extracted_data": block_extracted_data,
+ }
+ ],
+ },
+ }
+
+ verification = await _maybe_run_completion_verification(ctx, output, time.monotonic())
+
+ assert _completion_evidence_payload(block_extracted_data)["download_registered"] is True
+ assert captured.get("prompt") is not None
+ assert "download_registered" in captured["prompt"]
+ assert verification is not None
+ assert verification.status == "evaluated"
+ assert verification.is_fully_satisfied() is False
+ c0_verdict = next(verdict for verdict in verification.verdicts if verdict.criterion_id == "c0")
+ assert c0_verdict.reason_code == "no_evidence"
+
+
+async def test_non_fallback_param_only_uses_kept_floor_not_gate(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.tools.completion._completion_verification_handler", _no_handler)
+ ctx = _ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[_criterion("c0", "inputs are reusable", level="definition")],
+ classifier_status="success",
+ )
+
+ verification = await _maybe_run_completion_verification(ctx, _real_output_result(), time.monotonic())
+
+ assert verification is not None
+ assert verification.status == "evaluated"
+ assert verification.no_gradeable_run_plane is True
+ assert verification.is_fully_satisfied() is False
+ reason = _outcome_unverified_reason(ctx, verification)
+ assert reason is not None
+ assert _NO_GRADEABLE_PROSE in reason
+ assert _ADD_OR_FIX_PROSE not in reason
+
+
+async def test_chat_only_no_run_fallback_does_not_fire(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.tools.completion._completion_verification_handler", _no_handler)
+ ctx = _ctx()
+ ctx.request_policy = RequestPolicy(completion_criteria=[], classifier_status="fallback")
+ no_run_result: dict[str, Any] = {"ok": False, "data": {}}
+
+ verification = await _maybe_run_completion_verification(ctx, no_run_result, time.monotonic())
+
+ assert verification is None
+
+
+async def test_observation_seam_fires_and_persists_marker(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.tools.completion._completion_verification_handler", _no_handler)
+ ctx = _ctx()
+ ctx.request_policy = RequestPolicy(completion_criteria=[], classifier_status="fallback")
+ ctx.post_run_page_observation_after_failed_test = True
+
+ verification = await _maybe_run_completion_verification_from_page_observation(
+ ctx, url="https://example.com/orders/A1B2C3", title="Order Confirmation"
+ )
+
+ assert verification is not None
+ assert verification.no_gradeable_run_plane is True
+ assert ctx.completion_verification_result is verification
+
+
+async def test_observation_seam_non_candidate_does_not_fire() -> None:
+ ctx = _ctx()
+ ctx.request_policy = RequestPolicy(completion_criteria=[], classifier_status="fallback")
+ ctx.post_run_page_observation_after_failed_test = False
+
+ verification = await _maybe_run_completion_verification_from_page_observation(ctx, url="https://example.com/orders")
+
+ assert verification is None
+
+
+def test_empty_fallback_marker_records_built_unverified_not_no_meaningful_output() -> None:
+ ctx = _ctx()
+ ctx.request_policy = RequestPolicy(completion_criteria=[], classifier_status="fallback")
+ ctx.last_workflow = SimpleNamespace(
+ workflow_definition=SimpleNamespace(blocks=[SimpleNamespace(block_type="extraction", label="lookup")])
+ )
+ ctx.verified_prefix_labels = ["lookup"]
+ marker = CompletionVerificationResult(
+ status="evaluated", criterion_ids=[], verdicts=[], no_gradeable_run_plane=True
+ )
+
+ recorded = _record_run_blocks_result(ctx, _real_output_result(), completion_verification=marker)
+
+ assert isinstance(recorded, RecordedRunOutcome)
+ assert recorded.verdict == "not_demonstrated"
+ assert recorded.reason_code != "no_meaningful_output"
+ assert ctx.last_test_suspicious_success is False
+ assert ctx.last_full_workflow_test_ok is False
+ assert _NO_GRADEABLE_PROSE in (recorded.display_reason or "")
+
+
+def test_marker_does_not_leak_into_verdict_consumers() -> None:
+ empty = combine_verification_results([], None, [])
+ marked = CompletionVerificationResult(
+ status="evaluated", criterion_ids=[], verdicts=[], no_gradeable_run_plane=True
+ )
+ assert marked.verdict_state_counts() == empty.verdict_state_counts()
+ trace = marked.to_trace_data()
+ assert trace["no_gradeable_run_plane"] is True
+ assert trace["unmet_criterion_ids"] == []
+ assert trace["missing_evidence"] == []
+ assert trace["satisfied_count"] == 0
+ assert trace["fully_satisfied"] is False
+
+
+def test_kept_floor_rejects_definition_only_and_accepts_run_plane() -> None:
+ definition_only = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c0"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="c0",
+ state="satisfied",
+ reason_code="definition_parameters_referenced",
+ evidence_ref="workflow_yaml:first_name",
+ )
+ ],
+ )
+ assert all(v.satisfied for v in definition_only.verdicts)
+ assert definition_only.is_fully_satisfied() is False
+
+ run_satisfied = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c0"],
+ verdicts=[CriterionVerdict(criterion_id="c0", state="satisfied", reason_code="evidence_confirms")],
+ )
+ assert run_satisfied.is_fully_satisfied() is True
+
+
+def test_unsatisfied_definition_verdict_keeps_actionable_prose() -> None:
+ unsatisfied = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c0"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="c0",
+ state="unsatisfied",
+ reason_code="definition_parameters_unreferenced",
+ missing_evidence="workflow does not reference the parameter",
+ )
+ ],
+ )
+ ctx = _ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[_criterion("c0", "inputs are reusable", level="definition")]
+ )
+
+ reason = _outcome_unverified_reason(ctx, unsatisfied)
+
+ assert reason is not None
+ assert _ADD_OR_FIX_PROSE in reason
+ assert _NO_GRADEABLE_PROSE not in reason
+
+
+def test_rollback_anchor_not_recorded_for_definition_only_contract() -> None:
+ turn_state = CompletionCriteriaTurnState()
+ verification = combine_verification_results(
+ ["c0"],
+ None,
+ [
+ CriterionVerdict(
+ criterion_id="c0",
+ state="satisfied",
+ reason_code="definition_parameters_referenced",
+ evidence_ref="workflow_yaml:first_name",
+ )
+ ],
+ )
+
+ note_adjudication_on_turn_state(turn_state, verification, fully_satisfied_workflow_yaml="x: y")
+
+ assert turn_state.fully_satisfied_workflow_yaml is None
+
+
+def test_gated_state_does_not_auto_apply_code_only_success() -> None:
+ ctx = _ctx()
+ ctx.request_policy = RequestPolicy(completion_criteria=[], classifier_status="fallback")
+ ctx.completion_verification_result = CompletionVerificationResult(
+ status="evaluated", criterion_ids=[], verdicts=[], no_gradeable_run_plane=True
+ )
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ctx.has_staged_proposal = True
+ ctx.staged_workflow = SimpleNamespace() # type: ignore[assignment]
+ ctx.last_test_ok = True
+ ctx.last_full_workflow_test_ok = False
+
+ assert _should_apply_code_only_success_without_review(ctx, "auto_applicable") is False