Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-11495): grade fallback floor end states (#6856)
Andrew Neilson3fee904db76f
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.pyindex e056e1478..99b13c222 100644--- a/skyvern/forge/sdk/copilot/completion_verification.py+++ b/skyvern/forge/sdk/copilot/completion_verification.py@@ -23,7 +23,11 @@ import yamlfrom skyvern.config import settingsfrom skyvern.forge.prompts import prompt_enginefrom skyvern.forge.sdk.copilot.output_utils import parse_final_response-from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, redact_raw_secrets_for_prompt+from skyvern.forge.sdk.copilot.request_policy import (+ CompletionCriterion,+ is_fallback_floor_base_criterion,+ redact_raw_secrets_for_prompt,+)from skyvern.utils.strings import escape_code_fencesLOG = structlog.get_logger()@@ -743,6 +747,36 @@ def grade_terminal_goal_record_criteria(return []+def grade_fallback_floor_reached_end_state_criteria(+ criteria: list[CompletionCriterion], snapshot: RunEvidenceSnapshot+) -> list[CriterionVerdict]:+ eligible_criteria = [criterion for criterion in criteria if is_fallback_floor_base_criterion(criterion)]+ if not eligible_criteria:+ return []+ evidence_ref = _fallback_floor_reached_end_state_evidence_ref(snapshot)+ if evidence_ref is None:+ return []+ return [+ CriterionVerdict(+ criterion_id=criterion.id,+ state="satisfied",+ reason_code="evidence_confirms",+ evidence_ref=evidence_ref,+ )+ for criterion in eligible_criteria+ ]+++def _fallback_floor_reached_end_state_evidence_ref(snapshot: RunEvidenceSnapshot) -> str | None:+ for label, payload in snapshot.block_outputs.items():+ record = _structured_record_payload(payload)+ if record is None:+ continue+ if any(_terminal_goal_record_confirmed(record, family) for family in _TERMINAL_RECORD_FAMILY_ARTIFACTS):+ return f"block_outputs:{label}"+ return None++def _terminal_goal_criterion_family(outcome: str) -> str | None:if _extract_present_value_literals(outcome):return Nonediff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.pyindex 74e98df13..681560276 100644--- a/skyvern/forge/sdk/copilot/request_policy.py+++ b/skyvern/forge/sdk/copilot/request_policy.py@@ -916,13 +916,17 @@ def is_fallback_floor_criterion(criterion: CompletionCriterion) -> bool:return criterion.id.startswith(FALLBACK_FLOOR_CRITERION_ID_PREFIX)+def is_fallback_floor_base_criterion(criterion: CompletionCriterion) -> bool:+ return criterion.id == _FALLBACK_FLOOR_BASE_ID++def build_classifier_fallback_floor(ids: list[str]) -> list[CompletionCriterion]:floor = [CompletionCriterion(id=_FALLBACK_FLOOR_BASE_ID,outcome=_FALLBACK_FLOOR_BASE_OUTCOME,implicit=True,- method_mandated=True,+ method_mandated=False,level="run",)]diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.pyindex 181849668..9d2ac2eba 100644--- a/skyvern/forge/sdk/copilot/tools/completion.py+++ b/skyvern/forge/sdk/copilot/tools/completion.py@@ -17,6 +17,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (combine_verification_results,evaluate_completion_criteria,grade_definition_criteria,+ grade_fallback_floor_reached_end_state_criteria,grade_present_value_criteria,grade_record_semantic_consistency,grade_structured_record_criteria,@@ -508,6 +509,12 @@ def _apply_present_value_upgrades({verdict.criterion_id: verdict for verdict in grade_structured_record_criteria(run_criteria, snapshot)})if include_terminal_goal_records:+ upgrades.update(+ {+ verdict.criterion_id: verdict+ for verdict in grade_fallback_floor_reached_end_state_criteria(run_criteria, snapshot)+ }+ )upgrades.update({verdict.criterion_id: verdict for verdict in grade_terminal_goal_record_criteria(run_criteria, snapshot)})@@ -596,6 +603,8 @@ def _deterministic_run_verification_result(deterministic_by_id[verdict.criterion_id] = verdictfor verdict in grade_structured_record_criteria(run_criteria, snapshot):deterministic_by_id[verdict.criterion_id] = verdict+ for verdict in grade_fallback_floor_reached_end_state_criteria(run_criteria, snapshot):+ deterministic_by_id[verdict.criterion_id] = verdictfor verdict in grade_terminal_goal_record_criteria(run_criteria, snapshot):deterministic_by_id[verdict.criterion_id] = verdictfor verdict in semantic_verdicts:diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex 96f783a9e..e9e0728f1 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -2965,7 +2965,7 @@ class TestRequestPolicyCredentialResolution:assert policy.classifier_status == "fallback"assert policy.classifier_failure_kind == "timeout"assert policy.classifier_retry_count == 0- assert policy.completion_contract_status == "unknown"+ assert policy.completion_contract_status == "present"@pytest.mark.asyncioasync def test_request_policy_classifier_slow_but_within_budget_returns_real_policy(self, monkeypatch) -> None:@@ -3020,7 +3020,7 @@ class TestRequestPolicyCredentialResolution:assert policy.classifier_status == "fallback"assert policy.classifier_failure_kind == "provider_error"assert policy.classifier_retry_count == 0- assert policy.completion_contract_status == "unknown"+ assert policy.completion_contract_status == "present"@pytest.mark.asyncioasync def test_request_policy_classifier_retries_transient_error_then_succeeds(self) -> None:@@ -3078,7 +3078,7 @@ class TestRequestPolicyCredentialResolution:assert policy.classifier_status == "fallback"assert policy.classifier_failure_kind == "provider_error"assert policy.classifier_retry_count == 1- assert policy.completion_contract_status == "unknown"+ assert policy.completion_contract_status == "present"@pytest.mark.asyncioasync def test_request_policy_classifier_non_retriable_error_does_not_retry(self) -> None:@@ -3101,7 +3101,7 @@ class TestRequestPolicyCredentialResolution:assert policy.classifier_status == "fallback"assert policy.classifier_failure_kind == "provider_error"assert policy.classifier_retry_count == 0- assert policy.completion_contract_status == "unknown"+ assert policy.completion_contract_status == "present"@pytest.mark.asyncioasync def test_request_policy_classifier_transient_error_exhausting_budget_labels_transient(@@ -3135,7 +3135,7 @@ class TestRequestPolicyCredentialResolution:assert policy.classifier_status == "fallback"assert policy.classifier_failure_kind == "transient_error"assert policy.classifier_retry_count == 1- assert policy.completion_contract_status == "unknown"+ assert policy.completion_contract_status == "present"@pytest.mark.asyncioasync def test_missing_user_supplied_credential_ids_ask_for_clarification(self, monkeypatch) -> None:@@ -3623,7 +3623,7 @@ workflow_definition:assert policy.classifier_status == "fallback"assert policy.classifier_failure_kind == "provider_error"- assert policy.completion_contract_status == "unknown"+ assert policy.completion_contract_status == "present"assert policy.credential_input_kind == "credential_name"assert policy.credential_refs == ["mock-portal-login"]assert policy.resolved_credentials == [credential]@@ -3650,7 +3650,7 @@ workflow_definition:)assert policy.classifier_status == "fallback"- assert policy.completion_contract_status == "unknown"+ assert policy.completion_contract_status == "present"assert policy.credential_input_kind == "none"assert policy.credential_refs == []credentials.get_credentials.assert_not_called()@@ -5562,7 +5562,7 @@ class TestStructuralInfeasibilityQuestion:class TestClassifierFallbackCompletionFloor:@pytest.mark.asyncio- async def test_timeout_fallback_emits_method_mandated_run_floor(self, monkeypatch) -> None:+ async def test_timeout_fallback_emits_gradeable_run_floor(self, monkeypatch) -> None:monkeypatch.setattr(settings, "COPILOT_REQUEST_POLICY_CLASSIFIER_TIMEOUT_SECONDS", 0.05)async def handler(*, prompt: str, prompt_name: str) -> dict[str, object]:@@ -5582,22 +5582,22 @@ class TestClassifierFallbackCompletionFloor:assert policy.completion_criteriaassert all(c.level == "run" for c in policy.completion_criteria)assert all(c.implicit for c in policy.completion_criteria)- assert all(c.method_mandated for c in policy.completion_criteria)+ assert policy.completion_criteria[0].method_mandated is Falseassert all(is_fallback_floor_criterion(c) for c in policy.completion_criteria)assert policy.completion_contract is None- assert policy.completion_contract_status == "unknown"+ assert policy.completion_contract_status == "present"trace = policy.to_trace_data()- assert trace["completion_criteria_count"] == 0- assert trace["completion_criteria_method_mandated_count"] == len(policy.completion_criteria)+ assert trace["completion_criteria_count"] == 1+ assert trace["completion_criteria_method_mandated_count"] == len(policy.completion_criteria) - 1assert trace["has_completion_contract"] is False- def test_floor_is_excluded_from_judge_satisfaction_set(self) -> None:+ def test_floor_base_reaches_judge_satisfaction_set(self) -> None:policy = _classifier_fallback_policy([], raw_secret_present=False, failure_kind="timeout")ctx = SimpleNamespace(request_policy=policy)assert policy.completion_criteria- assert _completion_verification_criteria(ctx) == []+ assert _completion_verification_criteria(ctx) == [policy.completion_criteria[0]]def test_credential_aware_floor_adds_one_run_plane_criterion(self) -> None:base = build_classifier_fallback_floor([])@@ -5605,7 +5605,9 @@ class TestClassifierFallbackCompletionFloor:assert len(credentialed) == len(base) + 1assert all(c.level == "run" for c in credentialed)- assert all(c.implicit and c.method_mandated for c in credentialed)+ assert all(c.implicit for c in credentialed)+ assert credentialed[0].method_mandated is False+ assert credentialed[1].method_mandated is Trueassert len({c.id for c in credentialed}) == len(credentialed)def test_floor_respects_max_criteria_cap(self) -> None:diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.pyindex 9e89ee05b..3bc8a3c05 100644--- a/tests/unit/test_copilot_completion_verification.py+++ b/tests/unit/test_copilot_completion_verification.py@@ -46,7 +46,12 @@ from skyvern.forge.sdk.copilot.enforcement import (verified_goal_satisfied_context,)from skyvern.forge.sdk.copilot.hooks import _tool_completion_satisfies_turn-from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy, _parse_completion_criteria+from skyvern.forge.sdk.copilot.request_policy import (+ CompletionCriterion,+ RequestPolicy,+ _parse_completion_criteria,+ build_classifier_fallback_floor,+)from skyvern.forge.sdk.copilot.tools import (ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,_active_run_terminal_evidence_needs_visual_fallback,@@ -2857,6 +2862,108 @@ async def test_maybe_run_completion_verification_terminal_goal_without_boolean_bassert verification.verdicts[0].evidence_ref == "block_outputs:submit_water_request"+async def test_maybe_run_completion_verification_fallback_floor_uses_recorded_terminal_goal(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def fail_handler(**_: object) -> object:+ raise AssertionError("recorded fallback-floor outcome must not call the judge")++ _patch_completion_handler(monkeypatch, fail_handler)+ ctx = _run_ctx()+ _set_workflow_labels(ctx, "submit_water_request")+ ctx.request_policy = RequestPolicy(+ completion_criteria=build_classifier_fallback_floor([]),+ classifier_status="fallback",+ )++ verification = await _maybe_run_completion_verification(ctx, _terminal_goal_output_result(), time.monotonic())++ assert verification is not None+ assert verification.no_gradeable_run_plane is False+ assert verification.is_fully_satisfied() is True+ assert verification.to_trace_data()["criterion_count"] == 1+ assert verification.verdicts[0].evidence_ref == "block_outputs:submit_water_request"+++async def test_maybe_run_completion_verification_fallback_floor_uses_page_end_state_evidence(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ captured: dict[str, str] = {}++ async def handler(*, prompt: str, prompt_name: str) -> dict:+ captured["prompt"] = prompt+ return {+ "verdicts": [+ {+ "criterion_id": "__copilot_fallback_floor__run",+ "satisfied": True,+ "reason_code": "evidence_confirms",+ "evidence_ref": "observed_end_state_url",+ }+ ]+ }++ _patch_completion_handler(monkeypatch, handler)+ ctx = _run_ctx()+ ctx.request_policy = RequestPolicy(+ completion_criteria=build_classifier_fallback_floor([]),+ classifier_status="fallback",+ )+ result = {+ "ok": True,+ "data": {+ "workflow_run_id": "wr_review",+ "overall_status": "completed",+ "executed_block_labels": [],+ "current_url": "https://example.test/review",+ "page_title": "Start Service - Review",+ "blocks": [],+ },+ }++ verification = await _maybe_run_completion_verification(ctx, result, time.monotonic())++ assert verification is not None+ assert verification.no_gradeable_run_plane is False+ assert verification.is_fully_satisfied() is True+ assert "observed_end_state_url: https://example.test/review" in captured["prompt"]+ assert "observed_end_state_page_title: Start Service - Review" in captured["prompt"]+++async def test_maybe_run_completion_verification_fallback_floor_without_evidence_fails_closed(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def fail_handler(**_: object) -> object:+ raise AssertionError("true no-evidence fallback floor must not call the judge")++ _patch_completion_handler(monkeypatch, fail_handler)+ ctx = _run_ctx()+ ctx.request_policy = RequestPolicy(+ completion_criteria=build_classifier_fallback_floor([]),+ classifier_status="fallback",+ )+ result = {+ "ok": True,+ "data": {+ "workflow_run_id": "wr_empty",+ "overall_status": "completed",+ "executed_block_labels": [],+ "blocks": [],+ },+ }++ verification = await _maybe_run_completion_verification(ctx, result, time.monotonic())++ assert verification is not None+ assert verification.no_gradeable_run_plane is False+ assert verification.is_fully_satisfied() is False+ assert verification.verdicts[0].state == "unsatisfied"+ assert verification.verdicts[0].reason_code == "no_evidence"++@pytest.mark.asyncioasync def test_maybe_run_completion_verification_mixed_terminal_goal_upgrades_judge_unknown(monkeypatch: pytest.MonkeyPatch,diff --git a/tests/unit/test_copilot_request_policy_prompt.py b/tests/unit/test_copilot_request_policy_prompt.pyindex a3e12764e..45d97de88 100644--- a/tests/unit/test_copilot_request_policy_prompt.py+++ b/tests/unit/test_copilot_request_policy_prompt.py@@ -397,10 +397,10 @@ class TestClassifierFallbackCompletionCriteria:"present",{"output.user_name", "output.id", "output.address", "output.status"},),- ("Open https://example.com and click the pricing link.", "unknown", set()),+ ("Open https://example.com and click the pricing link.", "present", set()),# Substring matches ("read" in "already", "name" in "filename", "id" in "decided")# must not satisfy the whole-word gate.- ("I already decided the filename and reviewed the status of locations.", "unknown", set()),+ ("I already decided the filename and reviewed the status of locations.", "present", set()),# Generic structural group term ("entries") satisfies the group gate.("Return a record with the entity name, identifier 1234567890, status, and the grouped entries.",