Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-11611 credit corroborated requested-output abstentions (#6957)
Andrew Neilson5fafb3bbcc0d
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.pyindex abbc25a46..de5fb9e67 100644--- a/skyvern/forge/sdk/copilot/completion_verification.py+++ b/skyvern/forge/sdk/copilot/completion_verification.py@@ -45,6 +45,7 @@ _CONTINGENT_ABSTENTION_REASON_CODES = frozenset({"unknown", "no_evidence", "evidence_contradicts", "missing_exact_field", "unproducible"})REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID = "__copilot_registered_download__downloaded_files_non_empty"+TERMINAL_RECORD_CORROBORATION_CRITERION_ID = "__copilot_terminal_record_corroboration__"_REGISTERED_DOWNLOAD_COUNT_KEYS = ("downloaded_file_count","downloaded_file_url_count",@@ -80,7 +81,7 @@ class CriterionVerdict:evidence_ref: str | None = Nonemissing_evidence: str | None = Noneoutput_path: str | None = None- grounding_mode: Literal["exact_value", "shape", "missing"] | None = None+ grounding_mode: Literal["exact_value", "shape", "missing", "terminal_record"] | None = Noneexpected_output_shape: str | None = Nonehas_exact_value: bool = False@@ -107,6 +108,10 @@ class CompletionVerificationResult:has_observed_reach_state = any(_is_satisfied_observed_end_state_verdict(verdict) for verdict in verdict_by_id.values())+ has_requested_output_abstention_corroboration = any(+ _is_satisfied_observed_end_state_verdict(verdict) or _is_satisfied_terminal_record_verdict(verdict)+ for verdict in verdict_by_id.values()+ )satisfied_run_plane_count = 0for criterion_id in self.criterion_ids:verdict = verdict_by_id.get(criterion_id)@@ -127,6 +132,11 @@ class CompletionVerificationResult:self.structural_unfired_criterion_ids,):continue+ if verdict is not None and _is_corroborated_structural_requested_output_abstention(+ verdict, has_requested_output_abstention_corroboration+ ):+ satisfied_run_plane_count += 1+ continueif verdict is not None and _is_structural_requested_output_abstention(verdict):continueif has_observed_reach_state and verdict is not None and _is_reperception_contradiction(verdict):@@ -1181,6 +1191,7 @@ def grade_terminal_goal_record_criteria(state="satisfied",reason_code="evidence_confirms",evidence_ref=f"block_outputs:{label}",+ grounding_mode="terminal_record",))if verdicts:@@ -1188,6 +1199,24 @@ def grade_terminal_goal_record_criteria(return []+def grade_terminal_goal_record_corroboration(snapshot: RunEvidenceSnapshot) -> list[CriterionVerdict]:+ for label, payload in snapshot.block_outputs.items():+ record = _structured_record_payload(payload)+ if record is None:+ continue+ if any(_terminal_goal_record_confirmed(record, family) for family in _TERMINAL_RECORD_FAMILIES):+ return [+ CriterionVerdict(+ criterion_id=TERMINAL_RECORD_CORROBORATION_CRITERION_ID,+ state="satisfied",+ reason_code="evidence_confirms",+ evidence_ref=f"block_outputs:{label}",+ grounding_mode="terminal_record",+ )+ ]+ return []++def grade_fallback_floor_reached_end_state_criteria(criteria: list[CompletionCriterion], snapshot: RunEvidenceSnapshot) -> list[CriterionVerdict]:@@ -1852,6 +1881,14 @@ def _is_satisfied_observed_end_state_verdict(verdict: CriterionVerdict) -> bool:)+def _is_satisfied_terminal_record_verdict(verdict: CriterionVerdict) -> bool:+ return (+ verdict.state == "satisfied"+ and verdict.reason_code == "evidence_confirms"+ and verdict.grounding_mode == "terminal_record"+ )++def _is_reperception_contradiction(verdict: CriterionVerdict) -> bool:return (verdict.state == "unsatisfied"@@ -1868,6 +1905,16 @@ def _is_structural_requested_output_abstention(verdict: CriterionVerdict) -> booreturn verdict.state == "unsatisfied" and verdict.reason_code == _STRUCTURAL_ABSTENTION_REASON_CODE+def _is_corroborated_structural_requested_output_abstention(verdict: CriterionVerdict, has_corroboration: bool) -> bool:+ return (+ has_corroboration+ and _is_structural_requested_output_abstention(verdict)+ and bool(verdict.evidence_ref)+ and bool(verdict.output_path)+ and not verdict.has_exact_value+ )++def _is_contingent_abstention(verdict: CriterionVerdict,contingent_criterion_ids: Iterable[str],@@ -1934,6 +1981,12 @@ def combine_verification_results(verdict_by_id.get(cid, CriterionVerdict(criterion_id=cid, state="unknown", reason_code="unknown"))for cid in criterion_ids]+ if run_result is not None:+ verdicts.extend(+ verdict+ for verdict in run_result.verdicts+ if verdict.criterion_id not in criterion_ids and _is_satisfied_terminal_record_verdict(verdict)+ )return CompletionVerificationResult(status="evaluated",criterion_ids=list(criterion_ids),diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.pyindex f6adaef54..20c18923c 100644--- a/skyvern/forge/sdk/copilot/tools/completion.py+++ b/skyvern/forge/sdk/copilot/tools/completion.py@@ -23,6 +23,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (grade_record_semantic_consistency,grade_registered_download_criteria,grade_structured_record_criteria,+ grade_terminal_goal_record_corroboration,grade_terminal_goal_record_criteria,grade_validation_classification_criteria,is_fallback_floor_base_criterion,@@ -762,6 +763,15 @@ def _merge_run_verdicts(verdicts=[verdict_by_id.get(criterion_id, CriterionVerdict(criterion_id, "unknown", "unknown"))for criterion_id in criterion_ids+ ]+ + [+ verdict+ for verdicts in verdict_groups+ for verdict in verdicts+ if verdict.criterion_id not in criterion_ids+ and verdict.state == "satisfied"+ and verdict.reason_code == "evidence_confirms"+ and verdict.grounding_mode == "terminal_record"],contingent_criterion_ids=contingent_criterion_ids or default_contingent_ids,contingent_on_by_criterion_id=contingent_on_by_criterion_id or default_contingent_on_by_id,@@ -776,14 +786,17 @@ def _merge_run_verdicts_if_requested_output_exists(run_criteria: list[CompletionCriterion],requested_output_verdicts: list[CriterionVerdict],*verdict_groups: list[CriterionVerdict],+ snapshot: RunEvidenceSnapshot | None = None,structural_unfired_criterion_ids: list[str] | None = None,) -> CompletionVerificationResult | None:if not requested_output_verdicts:return None+ extra_verdicts = [grade_terminal_goal_record_corroboration(snapshot)] if snapshot is not None else []return _merge_run_verdicts(run_criteria,requested_output_verdicts,*verdict_groups,+ *extra_verdicts,structural_unfired_criterion_ids=structural_unfired_criterion_ids,)@@ -1004,6 +1017,7 @@ async def _maybe_run_completion_verification(run_result = _merge_run_verdicts(run_criteria,requested_output_verdicts,+ grade_terminal_goal_record_corroboration(snapshot),structural_unfired_criterion_ids=run_structural_unfired_ids,)else:@@ -1023,6 +1037,7 @@ async def _maybe_run_completion_verification(requested_output_result = _merge_run_verdicts_if_requested_output_exists(run_criteria,requested_output_verdicts,+ snapshot=snapshot,structural_unfired_criterion_ids=run_structural_unfired_ids,)if requested_output_result is not None:@@ -1052,6 +1067,7 @@ async def _maybe_run_completion_verification(requested_output_result = _merge_run_verdicts_if_requested_output_exists(run_criteria,requested_output_verdicts,+ snapshot=snapshot,structural_unfired_criterion_ids=run_structural_unfired_ids,)if requested_output_result is not None:@@ -1078,6 +1094,7 @@ async def _maybe_run_completion_verification(_merge_run_verdicts_if_requested_output_exists(run_criteria,requested_output_verdicts,+ snapshot=snapshot,structural_unfired_criterion_ids=run_structural_unfired_ids,)or judged_resultdiff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.pyindex e3ff52981..9281b2614 100644--- a/tests/unit/test_copilot_completion_verification.py+++ b/tests/unit/test_copilot_completion_verification.py@@ -414,6 +414,189 @@ def test_observed_end_state_satisfaction_does_not_override_requested_output_contassert result.is_fully_satisfied() is False+def test_observed_end_state_corroborates_structural_requested_output_abstention() -> None:+ result = _mixed(+ CriterionVerdict(+ criterion_id="c_reach",+ state="satisfied",+ reason_code="evidence_confirms",+ evidence_ref="observed_end_state_url",+ ),+ CriterionVerdict(+ criterion_id="c_requested_output",+ state="unsatisfied",+ reason_code="structurally_abstained",+ evidence_ref="block_outputs:extract_profile.customer_name",+ output_path="output.customer_name",+ grounding_mode="shape",+ ),+ )++ assert result.is_fully_satisfied() is True+++def test_structural_requested_output_abstention_without_typed_corroboration_does_not_veto() -> None:+ result = _mixed(+ CriterionVerdict(+ criterion_id="c_output",+ state="satisfied",+ reason_code="evidence_confirms",+ evidence_ref="block_outputs:extract_profile",+ ),+ CriterionVerdict(+ criterion_id="c_requested_output",+ state="unsatisfied",+ reason_code="structurally_abstained",+ evidence_ref="block_outputs:extract_profile.customer_name",+ output_path="output.customer_name",+ grounding_mode="shape",+ ),+ )++ assert result.is_fully_satisfied() is True+++def test_structural_requested_output_abstention_without_typed_corroboration_gets_no_floor_credit() -> None:+ result = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["c_requested_output"],+ verdicts=[+ CriterionVerdict(+ criterion_id="c_output",+ state="satisfied",+ reason_code="evidence_confirms",+ evidence_ref="block_outputs:extract_profile",+ ),+ CriterionVerdict(+ criterion_id="c_requested_output",+ state="unsatisfied",+ reason_code="structurally_abstained",+ evidence_ref="block_outputs:extract_profile.customer_name",+ output_path="output.customer_name",+ grounding_mode="shape",+ ),+ ],+ )++ assert result.is_fully_satisfied() is False+++def test_requested_output_no_evidence_with_observed_end_state_still_blocks() -> None:+ result = _mixed(+ CriterionVerdict(+ criterion_id="c_reach",+ state="satisfied",+ reason_code="evidence_confirms",+ evidence_ref="observed_end_state_url",+ ),+ CriterionVerdict(+ criterion_id="c_requested_output",+ state="unsatisfied",+ reason_code="no_evidence",+ evidence_ref="block_outputs:extract_profile.customer_name",+ output_path="output.customer_name",+ grounding_mode="missing",+ ),+ )++ assert result.is_fully_satisfied() is False+++def test_terminal_record_corroborates_structural_requested_output_abstention() -> None:+ result = _mixed(+ CriterionVerdict(+ criterion_id="c_submit",+ state="satisfied",+ reason_code="evidence_confirms",+ evidence_ref="block_outputs:submit_request",+ grounding_mode="terminal_record",+ ),+ CriterionVerdict(+ criterion_id="c_requested_output",+ state="unsatisfied",+ reason_code="structurally_abstained",+ evidence_ref="block_outputs:submit_request.confirmation_number",+ output_path="output.confirmation_number",+ grounding_mode="shape",+ ),+ )++ assert result.is_fully_satisfied() is True+++def test_terminal_record_corroboration_gives_structural_requested_output_abstention_floor_credit() -> None:+ result = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["c_requested_output"],+ verdicts=[+ CriterionVerdict(+ criterion_id="c_submit",+ state="satisfied",+ reason_code="evidence_confirms",+ evidence_ref="block_outputs:submit_request",+ grounding_mode="terminal_record",+ ),+ CriterionVerdict(+ criterion_id="c_requested_output",+ state="unsatisfied",+ reason_code="structurally_abstained",+ evidence_ref="block_outputs:submit_request.confirmation_number",+ output_path="output.confirmation_number",+ grounding_mode="shape",+ ),+ ],+ )++ assert result.is_fully_satisfied() is True+++def test_unmarked_block_output_does_not_corroborate_structural_requested_output_abstention() -> None:+ result = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["c_requested_output"],+ verdicts=[+ CriterionVerdict(+ criterion_id="c_submit",+ state="satisfied",+ reason_code="evidence_confirms",+ evidence_ref="block_outputs:submit_request",+ ),+ CriterionVerdict(+ criterion_id="c_requested_output",+ state="unsatisfied",+ reason_code="structurally_abstained",+ evidence_ref="block_outputs:submit_request.confirmation_number",+ output_path="output.confirmation_number",+ grounding_mode="shape",+ ),+ ],+ )++ assert result.is_fully_satisfied() is False+++def test_requested_output_wrong_exact_value_with_observed_end_state_still_blocks() -> None:+ result = _mixed(+ CriterionVerdict(+ criterion_id="c_reach",+ state="satisfied",+ reason_code="evidence_confirms",+ evidence_ref="observed_end_state_url",+ ),+ CriterionVerdict(+ criterion_id="c_requested_output",+ state="unsatisfied",+ reason_code="evidence_contradicts",+ evidence_ref="block_outputs:submit_request.confirmation_number",+ output_path="output.confirmation_number",+ grounding_mode="exact_value",+ has_exact_value=True,+ ),+ )++ assert result.is_fully_satisfied() is False++def _mixed(*verdicts: CriterionVerdict) -> CompletionVerificationResult:return CompletionVerificationResult(status="evaluated", criterion_ids=[v.criterion_id for v in verdicts], verdicts=list(verdicts)@@ -1011,6 +1194,7 @@ def test_terminal_goal_record_satisfies_flat_submit_payload() -> None:state="satisfied",reason_code="evidence_confirms",evidence_ref="block_outputs:submit_water_request",+ grounding_mode="terminal_record",)]@@ -5669,6 +5853,92 @@ async def test_all_abstained_requested_outputs_do_not_satisfy_completion(assert verification.verdicts[0].reason_code == "structurally_abstained"+async def test_requested_output_only_terminal_record_corroboration_satisfies_completion(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def fail_handler(**_: object) -> object:+ raise AssertionError("requested-output only terminal record corroboration should bypass the judge")++ _patch_completion_handler(monkeypatch, fail_handler)+ ctx = _run_ctx()+ _set_workflow_labels(ctx, "extract_profile")+ ctx.code_artifact_metadata = _metadata_for_requested_paths(+ "confirmation_number",+ "account_number",+ "selected_start_date",+ )+ ctx.request_policy = RequestPolicy(+ completion_criteria=[+ CompletionCriterion(+ id="c_confirmation",+ outcome="The returned record includes confirmation number.",+ output_path="output.confirmation_number",+ ),+ CompletionCriterion(+ id="c_account",+ outcome="The returned record includes account number.",+ output_path="output.account_number",+ ),+ CompletionCriterion(+ id="c_start_date",+ outcome="The returned record includes selected start date.",+ output_path="output.selected_start_date",+ ),+ ]+ )++ verification = await _maybe_run_completion_verification(+ ctx,+ _requested_output_result(_terminal_goal_payload()),+ time.monotonic(),+ )++ assert verification is not None+ assert verification.is_fully_satisfied() is True+ verdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}+ assert {verdicts[criterion_id].reason_code for criterion_id in ("c_confirmation", "c_account", "c_start_date")} == {+ "structurally_abstained"+ }+ terminal_record_verdicts = [+ verdict for verdict in verification.verdicts if verdict.grounding_mode == "terminal_record"+ ]+ assert len(terminal_record_verdicts) == 1+ assert terminal_record_verdicts[0].evidence_ref == "block_outputs:extract_profile"+++async def test_requested_output_only_generic_output_has_no_terminal_record_corroboration(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def fail_handler(**_: object) -> object:+ raise AssertionError("requested-output only generic output should bypass the judge")++ _patch_completion_handler(monkeypatch, fail_handler)+ ctx = _run_ctx()+ _set_workflow_labels(ctx, "extract_profile")+ ctx.code_artifact_metadata = _metadata_for_requested_paths("customer_name")+ ctx.request_policy = RequestPolicy(+ completion_criteria=[+ CompletionCriterion(+ id="c_customer_name",+ outcome="The returned record includes customer name.",+ output_path="output.customer_name",+ )+ ]+ )++ verification = await _maybe_run_completion_verification(+ ctx,+ _requested_output_result({"customer_name": "Sample Customer"}),+ time.monotonic(),+ )++ assert verification is not None+ assert verification.is_fully_satisfied() is False+ assert all(verdict.grounding_mode != "terminal_record" for verdict in verification.verdicts)++@pytest.mark.asyncioasync def test_requested_output_bypasses_judge_satisfaction_without_exact_field(monkeypatch: pytest.MonkeyPatch,