Skip to content

Automate browser based workflows with AI

SKY-11611 credit corroborated requested-output abstentions (#6957)

Andrew Neilson5fafb3bbcc0d
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.py
index abbc25a46..de5fb9e67 100644
--- a/skyvern/forge/sdk/copilot/completion_verification.py
+++ b/skyvern/forge/sdk/copilot/completion_verification.py
@@ -45,6 +45,7 @@ _CONTINGENT_ABSTENTION_REASON_CODES = frozenset(
{"unknown", "no_evidence", "evidence_contradicts", "missing_exact_field", "unproducible"}
)
REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID = "__copilot_registered_download__downloaded_files_non_empty"
+TERMINAL_RECORD_CORROBORATION_CRITERION_ID = "__copilot_terminal_record_corroboration__"
_REGISTERED_DOWNLOAD_COUNT_KEYS = (
"downloaded_file_count",
"downloaded_file_url_count",
@@ -80,7 +81,7 @@ class CriterionVerdict:
evidence_ref: str | None = None
missing_evidence: str | None = None
output_path: str | None = None
- grounding_mode: Literal["exact_value", "shape", "missing"] | None = None
+ grounding_mode: Literal["exact_value", "shape", "missing", "terminal_record"] | None = None
expected_output_shape: str | None = None
has_exact_value: bool = False
@@ -107,6 +108,10 @@ class CompletionVerificationResult:
has_observed_reach_state = any(
_is_satisfied_observed_end_state_verdict(verdict) for verdict in verdict_by_id.values()
)
+ has_requested_output_abstention_corroboration = any(
+ _is_satisfied_observed_end_state_verdict(verdict) or _is_satisfied_terminal_record_verdict(verdict)
+ for verdict in verdict_by_id.values()
+ )
satisfied_run_plane_count = 0
for criterion_id in self.criterion_ids:
verdict = verdict_by_id.get(criterion_id)
@@ -127,6 +132,11 @@ class CompletionVerificationResult:
self.structural_unfired_criterion_ids,
):
continue
+ if verdict is not None and _is_corroborated_structural_requested_output_abstention(
+ verdict, has_requested_output_abstention_corroboration
+ ):
+ satisfied_run_plane_count += 1
+ continue
if verdict is not None and _is_structural_requested_output_abstention(verdict):
continue
if has_observed_reach_state and verdict is not None and _is_reperception_contradiction(verdict):
@@ -1181,6 +1191,7 @@ def grade_terminal_goal_record_criteria(
state="satisfied",
reason_code="evidence_confirms",
evidence_ref=f"block_outputs:{label}",
+ grounding_mode="terminal_record",
)
)
if verdicts:
@@ -1188,6 +1199,24 @@ def grade_terminal_goal_record_criteria(
return []
+def grade_terminal_goal_record_corroboration(snapshot: RunEvidenceSnapshot) -> list[CriterionVerdict]:
+ for label, payload in snapshot.block_outputs.items():
+ record = _structured_record_payload(payload)
+ if record is None:
+ continue
+ if any(_terminal_goal_record_confirmed(record, family) for family in _TERMINAL_RECORD_FAMILIES):
+ return [
+ CriterionVerdict(
+ criterion_id=TERMINAL_RECORD_CORROBORATION_CRITERION_ID,
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_ref=f"block_outputs:{label}",
+ grounding_mode="terminal_record",
+ )
+ ]
+ return []
+
+
def grade_fallback_floor_reached_end_state_criteria(
criteria: list[CompletionCriterion], snapshot: RunEvidenceSnapshot
) -> list[CriterionVerdict]:
@@ -1852,6 +1881,14 @@ def _is_satisfied_observed_end_state_verdict(verdict: CriterionVerdict) -> bool:
)
+def _is_satisfied_terminal_record_verdict(verdict: CriterionVerdict) -> bool:
+ return (
+ verdict.state == "satisfied"
+ and verdict.reason_code == "evidence_confirms"
+ and verdict.grounding_mode == "terminal_record"
+ )
+
+
def _is_reperception_contradiction(verdict: CriterionVerdict) -> bool:
return (
verdict.state == "unsatisfied"
@@ -1868,6 +1905,16 @@ def _is_structural_requested_output_abstention(verdict: CriterionVerdict) -> boo
return verdict.state == "unsatisfied" and verdict.reason_code == _STRUCTURAL_ABSTENTION_REASON_CODE
+def _is_corroborated_structural_requested_output_abstention(verdict: CriterionVerdict, has_corroboration: bool) -> bool:
+ return (
+ has_corroboration
+ and _is_structural_requested_output_abstention(verdict)
+ and bool(verdict.evidence_ref)
+ and bool(verdict.output_path)
+ and not verdict.has_exact_value
+ )
+
+
def _is_contingent_abstention(
verdict: CriterionVerdict,
contingent_criterion_ids: Iterable[str],
@@ -1934,6 +1981,12 @@ def combine_verification_results(
verdict_by_id.get(cid, CriterionVerdict(criterion_id=cid, state="unknown", reason_code="unknown"))
for cid in criterion_ids
]
+ if run_result is not None:
+ verdicts.extend(
+ verdict
+ for verdict in run_result.verdicts
+ if verdict.criterion_id not in criterion_ids and _is_satisfied_terminal_record_verdict(verdict)
+ )
return CompletionVerificationResult(
status="evaluated",
criterion_ids=list(criterion_ids),
diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.py
index f6adaef54..20c18923c 100644
--- a/skyvern/forge/sdk/copilot/tools/completion.py
+++ b/skyvern/forge/sdk/copilot/tools/completion.py
@@ -23,6 +23,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (
grade_record_semantic_consistency,
grade_registered_download_criteria,
grade_structured_record_criteria,
+ grade_terminal_goal_record_corroboration,
grade_terminal_goal_record_criteria,
grade_validation_classification_criteria,
is_fallback_floor_base_criterion,
@@ -762,6 +763,15 @@ def _merge_run_verdicts(
verdicts=[
verdict_by_id.get(criterion_id, CriterionVerdict(criterion_id, "unknown", "unknown"))
for criterion_id in criterion_ids
+ ]
+ + [
+ verdict
+ for verdicts in verdict_groups
+ for verdict in verdicts
+ if verdict.criterion_id not in criterion_ids
+ and verdict.state == "satisfied"
+ and verdict.reason_code == "evidence_confirms"
+ and verdict.grounding_mode == "terminal_record"
],
contingent_criterion_ids=contingent_criterion_ids or default_contingent_ids,
contingent_on_by_criterion_id=contingent_on_by_criterion_id or default_contingent_on_by_id,
@@ -776,14 +786,17 @@ def _merge_run_verdicts_if_requested_output_exists(
run_criteria: list[CompletionCriterion],
requested_output_verdicts: list[CriterionVerdict],
*verdict_groups: list[CriterionVerdict],
+ snapshot: RunEvidenceSnapshot | None = None,
structural_unfired_criterion_ids: list[str] | None = None,
) -> CompletionVerificationResult | None:
if not requested_output_verdicts:
return None
+ extra_verdicts = [grade_terminal_goal_record_corroboration(snapshot)] if snapshot is not None else []
return _merge_run_verdicts(
run_criteria,
requested_output_verdicts,
*verdict_groups,
+ *extra_verdicts,
structural_unfired_criterion_ids=structural_unfired_criterion_ids,
)
@@ -1004,6 +1017,7 @@ async def _maybe_run_completion_verification(
run_result = _merge_run_verdicts(
run_criteria,
requested_output_verdicts,
+ grade_terminal_goal_record_corroboration(snapshot),
structural_unfired_criterion_ids=run_structural_unfired_ids,
)
else:
@@ -1023,6 +1037,7 @@ async def _maybe_run_completion_verification(
requested_output_result = _merge_run_verdicts_if_requested_output_exists(
run_criteria,
requested_output_verdicts,
+ snapshot=snapshot,
structural_unfired_criterion_ids=run_structural_unfired_ids,
)
if requested_output_result is not None:
@@ -1052,6 +1067,7 @@ async def _maybe_run_completion_verification(
requested_output_result = _merge_run_verdicts_if_requested_output_exists(
run_criteria,
requested_output_verdicts,
+ snapshot=snapshot,
structural_unfired_criterion_ids=run_structural_unfired_ids,
)
if requested_output_result is not None:
@@ -1078,6 +1094,7 @@ async def _maybe_run_completion_verification(
_merge_run_verdicts_if_requested_output_exists(
run_criteria,
requested_output_verdicts,
+ snapshot=snapshot,
structural_unfired_criterion_ids=run_structural_unfired_ids,
)
or judged_result
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index e3ff52981..9281b2614 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -414,6 +414,189 @@ def test_observed_end_state_satisfaction_does_not_override_requested_output_cont
assert result.is_fully_satisfied() is False
+def test_observed_end_state_corroborates_structural_requested_output_abstention() -> None:
+ result = _mixed(
+ CriterionVerdict(
+ criterion_id="c_reach",
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_ref="observed_end_state_url",
+ ),
+ CriterionVerdict(
+ criterion_id="c_requested_output",
+ state="unsatisfied",
+ reason_code="structurally_abstained",
+ evidence_ref="block_outputs:extract_profile.customer_name",
+ output_path="output.customer_name",
+ grounding_mode="shape",
+ ),
+ )
+
+ assert result.is_fully_satisfied() is True
+
+
+def test_structural_requested_output_abstention_without_typed_corroboration_does_not_veto() -> None:
+ result = _mixed(
+ CriterionVerdict(
+ criterion_id="c_output",
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_ref="block_outputs:extract_profile",
+ ),
+ CriterionVerdict(
+ criterion_id="c_requested_output",
+ state="unsatisfied",
+ reason_code="structurally_abstained",
+ evidence_ref="block_outputs:extract_profile.customer_name",
+ output_path="output.customer_name",
+ grounding_mode="shape",
+ ),
+ )
+
+ assert result.is_fully_satisfied() is True
+
+
+def test_structural_requested_output_abstention_without_typed_corroboration_gets_no_floor_credit() -> None:
+ result = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c_requested_output"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="c_output",
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_ref="block_outputs:extract_profile",
+ ),
+ CriterionVerdict(
+ criterion_id="c_requested_output",
+ state="unsatisfied",
+ reason_code="structurally_abstained",
+ evidence_ref="block_outputs:extract_profile.customer_name",
+ output_path="output.customer_name",
+ grounding_mode="shape",
+ ),
+ ],
+ )
+
+ assert result.is_fully_satisfied() is False
+
+
+def test_requested_output_no_evidence_with_observed_end_state_still_blocks() -> None:
+ result = _mixed(
+ CriterionVerdict(
+ criterion_id="c_reach",
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_ref="observed_end_state_url",
+ ),
+ CriterionVerdict(
+ criterion_id="c_requested_output",
+ state="unsatisfied",
+ reason_code="no_evidence",
+ evidence_ref="block_outputs:extract_profile.customer_name",
+ output_path="output.customer_name",
+ grounding_mode="missing",
+ ),
+ )
+
+ assert result.is_fully_satisfied() is False
+
+
+def test_terminal_record_corroborates_structural_requested_output_abstention() -> None:
+ result = _mixed(
+ CriterionVerdict(
+ criterion_id="c_submit",
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_ref="block_outputs:submit_request",
+ grounding_mode="terminal_record",
+ ),
+ CriterionVerdict(
+ criterion_id="c_requested_output",
+ state="unsatisfied",
+ reason_code="structurally_abstained",
+ evidence_ref="block_outputs:submit_request.confirmation_number",
+ output_path="output.confirmation_number",
+ grounding_mode="shape",
+ ),
+ )
+
+ assert result.is_fully_satisfied() is True
+
+
+def test_terminal_record_corroboration_gives_structural_requested_output_abstention_floor_credit() -> None:
+ result = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c_requested_output"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="c_submit",
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_ref="block_outputs:submit_request",
+ grounding_mode="terminal_record",
+ ),
+ CriterionVerdict(
+ criterion_id="c_requested_output",
+ state="unsatisfied",
+ reason_code="structurally_abstained",
+ evidence_ref="block_outputs:submit_request.confirmation_number",
+ output_path="output.confirmation_number",
+ grounding_mode="shape",
+ ),
+ ],
+ )
+
+ assert result.is_fully_satisfied() is True
+
+
+def test_unmarked_block_output_does_not_corroborate_structural_requested_output_abstention() -> None:
+ result = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c_requested_output"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="c_submit",
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_ref="block_outputs:submit_request",
+ ),
+ CriterionVerdict(
+ criterion_id="c_requested_output",
+ state="unsatisfied",
+ reason_code="structurally_abstained",
+ evidence_ref="block_outputs:submit_request.confirmation_number",
+ output_path="output.confirmation_number",
+ grounding_mode="shape",
+ ),
+ ],
+ )
+
+ assert result.is_fully_satisfied() is False
+
+
+def test_requested_output_wrong_exact_value_with_observed_end_state_still_blocks() -> None:
+ result = _mixed(
+ CriterionVerdict(
+ criterion_id="c_reach",
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_ref="observed_end_state_url",
+ ),
+ CriterionVerdict(
+ criterion_id="c_requested_output",
+ state="unsatisfied",
+ reason_code="evidence_contradicts",
+ evidence_ref="block_outputs:submit_request.confirmation_number",
+ output_path="output.confirmation_number",
+ grounding_mode="exact_value",
+ has_exact_value=True,
+ ),
+ )
+
+ assert result.is_fully_satisfied() is False
+
+
def _mixed(*verdicts: CriterionVerdict) -> CompletionVerificationResult:
return CompletionVerificationResult(
status="evaluated", criterion_ids=[v.criterion_id for v in verdicts], verdicts=list(verdicts)
@@ -1011,6 +1194,7 @@ def test_terminal_goal_record_satisfies_flat_submit_payload() -> None:
state="satisfied",
reason_code="evidence_confirms",
evidence_ref="block_outputs:submit_water_request",
+ grounding_mode="terminal_record",
)
]
@@ -5669,6 +5853,92 @@ async def test_all_abstained_requested_outputs_do_not_satisfy_completion(
assert verification.verdicts[0].reason_code == "structurally_abstained"
+async def test_requested_output_only_terminal_record_corroboration_satisfies_completion(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def fail_handler(**_: object) -> object:
+ raise AssertionError("requested-output only terminal record corroboration should bypass the judge")
+
+ _patch_completion_handler(monkeypatch, fail_handler)
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "extract_profile")
+ ctx.code_artifact_metadata = _metadata_for_requested_paths(
+ "confirmation_number",
+ "account_number",
+ "selected_start_date",
+ )
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="c_confirmation",
+ outcome="The returned record includes confirmation number.",
+ output_path="output.confirmation_number",
+ ),
+ CompletionCriterion(
+ id="c_account",
+ outcome="The returned record includes account number.",
+ output_path="output.account_number",
+ ),
+ CompletionCriterion(
+ id="c_start_date",
+ outcome="The returned record includes selected start date.",
+ output_path="output.selected_start_date",
+ ),
+ ]
+ )
+
+ verification = await _maybe_run_completion_verification(
+ ctx,
+ _requested_output_result(_terminal_goal_payload()),
+ time.monotonic(),
+ )
+
+ assert verification is not None
+ assert verification.is_fully_satisfied() is True
+ verdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}
+ assert {verdicts[criterion_id].reason_code for criterion_id in ("c_confirmation", "c_account", "c_start_date")} == {
+ "structurally_abstained"
+ }
+ terminal_record_verdicts = [
+ verdict for verdict in verification.verdicts if verdict.grounding_mode == "terminal_record"
+ ]
+ assert len(terminal_record_verdicts) == 1
+ assert terminal_record_verdicts[0].evidence_ref == "block_outputs:extract_profile"
+
+
+async def test_requested_output_only_generic_output_has_no_terminal_record_corroboration(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def fail_handler(**_: object) -> object:
+ raise AssertionError("requested-output only generic output should bypass the judge")
+
+ _patch_completion_handler(monkeypatch, fail_handler)
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "extract_profile")
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("customer_name")
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="c_customer_name",
+ outcome="The returned record includes customer name.",
+ output_path="output.customer_name",
+ )
+ ]
+ )
+
+ verification = await _maybe_run_completion_verification(
+ ctx,
+ _requested_output_result({"customer_name": "Sample Customer"}),
+ time.monotonic(),
+ )
+
+ assert verification is not None
+ assert verification.is_fully_satisfied() is False
+ assert all(verdict.grounding_mode != "terminal_record" for verdict in verification.verdicts)
+
+
@pytest.mark.asyncio
async def test_requested_output_bypasses_judge_satisfaction_without_exact_field(
monkeypatch: pytest.MonkeyPatch,