Skip to content

Automate browser based workflows with AI

fix(SKY-11495): grade fallback floor end states (#6856)

Andrew Neilson3fee904db76f
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.py
index e056e1478..99b13c222 100644
--- a/skyvern/forge/sdk/copilot/completion_verification.py
+++ b/skyvern/forge/sdk/copilot/completion_verification.py
@@ -23,7 +23,11 @@ import yaml
from skyvern.config import settings
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.copilot.output_utils import parse_final_response
-from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, redact_raw_secrets_for_prompt
+from skyvern.forge.sdk.copilot.request_policy import (
+ CompletionCriterion,
+ is_fallback_floor_base_criterion,
+ redact_raw_secrets_for_prompt,
+)
from skyvern.utils.strings import escape_code_fences
LOG = structlog.get_logger()
@@ -743,6 +747,36 @@ def grade_terminal_goal_record_criteria(
return []
+def grade_fallback_floor_reached_end_state_criteria(
+ criteria: list[CompletionCriterion], snapshot: RunEvidenceSnapshot
+) -> list[CriterionVerdict]:
+ eligible_criteria = [criterion for criterion in criteria if is_fallback_floor_base_criterion(criterion)]
+ if not eligible_criteria:
+ return []
+ evidence_ref = _fallback_floor_reached_end_state_evidence_ref(snapshot)
+ if evidence_ref is None:
+ return []
+ return [
+ CriterionVerdict(
+ criterion_id=criterion.id,
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_ref=evidence_ref,
+ )
+ for criterion in eligible_criteria
+ ]
+
+
+def _fallback_floor_reached_end_state_evidence_ref(snapshot: RunEvidenceSnapshot) -> str | None:
+ for label, payload in snapshot.block_outputs.items():
+ record = _structured_record_payload(payload)
+ if record is None:
+ continue
+ if any(_terminal_goal_record_confirmed(record, family) for family in _TERMINAL_RECORD_FAMILY_ARTIFACTS):
+ return f"block_outputs:{label}"
+ return None
+
+
def _terminal_goal_criterion_family(outcome: str) -> str | None:
if _extract_present_value_literals(outcome):
return None
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index 74e98df13..681560276 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -916,13 +916,17 @@ def is_fallback_floor_criterion(criterion: CompletionCriterion) -> bool:
return criterion.id.startswith(FALLBACK_FLOOR_CRITERION_ID_PREFIX)
+def is_fallback_floor_base_criterion(criterion: CompletionCriterion) -> bool:
+ return criterion.id == _FALLBACK_FLOOR_BASE_ID
+
+
def build_classifier_fallback_floor(ids: list[str]) -> list[CompletionCriterion]:
floor = [
CompletionCriterion(
id=_FALLBACK_FLOOR_BASE_ID,
outcome=_FALLBACK_FLOOR_BASE_OUTCOME,
implicit=True,
- method_mandated=True,
+ method_mandated=False,
level="run",
)
]
diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.py
index 181849668..9d2ac2eba 100644
--- a/skyvern/forge/sdk/copilot/tools/completion.py
+++ b/skyvern/forge/sdk/copilot/tools/completion.py
@@ -17,6 +17,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (
combine_verification_results,
evaluate_completion_criteria,
grade_definition_criteria,
+ grade_fallback_floor_reached_end_state_criteria,
grade_present_value_criteria,
grade_record_semantic_consistency,
grade_structured_record_criteria,
@@ -508,6 +509,12 @@ def _apply_present_value_upgrades(
{verdict.criterion_id: verdict for verdict in grade_structured_record_criteria(run_criteria, snapshot)}
)
if include_terminal_goal_records:
+ upgrades.update(
+ {
+ verdict.criterion_id: verdict
+ for verdict in grade_fallback_floor_reached_end_state_criteria(run_criteria, snapshot)
+ }
+ )
upgrades.update(
{verdict.criterion_id: verdict for verdict in grade_terminal_goal_record_criteria(run_criteria, snapshot)}
)
@@ -596,6 +603,8 @@ def _deterministic_run_verification_result(
deterministic_by_id[verdict.criterion_id] = verdict
for verdict in grade_structured_record_criteria(run_criteria, snapshot):
deterministic_by_id[verdict.criterion_id] = verdict
+ for verdict in grade_fallback_floor_reached_end_state_criteria(run_criteria, snapshot):
+ deterministic_by_id[verdict.criterion_id] = verdict
for verdict in grade_terminal_goal_record_criteria(run_criteria, snapshot):
deterministic_by_id[verdict.criterion_id] = verdict
for verdict in semantic_verdicts:
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 96f783a9e..e9e0728f1 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -2965,7 +2965,7 @@ class TestRequestPolicyCredentialResolution:
assert policy.classifier_status == "fallback"
assert policy.classifier_failure_kind == "timeout"
assert policy.classifier_retry_count == 0
- assert policy.completion_contract_status == "unknown"
+ assert policy.completion_contract_status == "present"
@pytest.mark.asyncio
async def test_request_policy_classifier_slow_but_within_budget_returns_real_policy(self, monkeypatch) -> None:
@@ -3020,7 +3020,7 @@ class TestRequestPolicyCredentialResolution:
assert policy.classifier_status == "fallback"
assert policy.classifier_failure_kind == "provider_error"
assert policy.classifier_retry_count == 0
- assert policy.completion_contract_status == "unknown"
+ assert policy.completion_contract_status == "present"
@pytest.mark.asyncio
async def test_request_policy_classifier_retries_transient_error_then_succeeds(self) -> None:
@@ -3078,7 +3078,7 @@ class TestRequestPolicyCredentialResolution:
assert policy.classifier_status == "fallback"
assert policy.classifier_failure_kind == "provider_error"
assert policy.classifier_retry_count == 1
- assert policy.completion_contract_status == "unknown"
+ assert policy.completion_contract_status == "present"
@pytest.mark.asyncio
async def test_request_policy_classifier_non_retriable_error_does_not_retry(self) -> None:
@@ -3101,7 +3101,7 @@ class TestRequestPolicyCredentialResolution:
assert policy.classifier_status == "fallback"
assert policy.classifier_failure_kind == "provider_error"
assert policy.classifier_retry_count == 0
- assert policy.completion_contract_status == "unknown"
+ assert policy.completion_contract_status == "present"
@pytest.mark.asyncio
async def test_request_policy_classifier_transient_error_exhausting_budget_labels_transient(
@@ -3135,7 +3135,7 @@ class TestRequestPolicyCredentialResolution:
assert policy.classifier_status == "fallback"
assert policy.classifier_failure_kind == "transient_error"
assert policy.classifier_retry_count == 1
- assert policy.completion_contract_status == "unknown"
+ assert policy.completion_contract_status == "present"
@pytest.mark.asyncio
async def test_missing_user_supplied_credential_ids_ask_for_clarification(self, monkeypatch) -> None:
@@ -3623,7 +3623,7 @@ workflow_definition:
assert policy.classifier_status == "fallback"
assert policy.classifier_failure_kind == "provider_error"
- assert policy.completion_contract_status == "unknown"
+ assert policy.completion_contract_status == "present"
assert policy.credential_input_kind == "credential_name"
assert policy.credential_refs == ["mock-portal-login"]
assert policy.resolved_credentials == [credential]
@@ -3650,7 +3650,7 @@ workflow_definition:
)
assert policy.classifier_status == "fallback"
- assert policy.completion_contract_status == "unknown"
+ assert policy.completion_contract_status == "present"
assert policy.credential_input_kind == "none"
assert policy.credential_refs == []
credentials.get_credentials.assert_not_called()
@@ -5562,7 +5562,7 @@ class TestStructuralInfeasibilityQuestion:
class TestClassifierFallbackCompletionFloor:
@pytest.mark.asyncio
- async def test_timeout_fallback_emits_method_mandated_run_floor(self, monkeypatch) -> None:
+ async def test_timeout_fallback_emits_gradeable_run_floor(self, monkeypatch) -> None:
monkeypatch.setattr(settings, "COPILOT_REQUEST_POLICY_CLASSIFIER_TIMEOUT_SECONDS", 0.05)
async def handler(*, prompt: str, prompt_name: str) -> dict[str, object]:
@@ -5582,22 +5582,22 @@ class TestClassifierFallbackCompletionFloor:
assert policy.completion_criteria
assert all(c.level == "run" for c in policy.completion_criteria)
assert all(c.implicit for c in policy.completion_criteria)
- assert all(c.method_mandated for c in policy.completion_criteria)
+ assert policy.completion_criteria[0].method_mandated is False
assert all(is_fallback_floor_criterion(c) for c in policy.completion_criteria)
assert policy.completion_contract is None
- assert policy.completion_contract_status == "unknown"
+ assert policy.completion_contract_status == "present"
trace = policy.to_trace_data()
- assert trace["completion_criteria_count"] == 0
- assert trace["completion_criteria_method_mandated_count"] == len(policy.completion_criteria)
+ assert trace["completion_criteria_count"] == 1
+ assert trace["completion_criteria_method_mandated_count"] == len(policy.completion_criteria) - 1
assert trace["has_completion_contract"] is False
- def test_floor_is_excluded_from_judge_satisfaction_set(self) -> None:
+ def test_floor_base_reaches_judge_satisfaction_set(self) -> None:
policy = _classifier_fallback_policy([], raw_secret_present=False, failure_kind="timeout")
ctx = SimpleNamespace(request_policy=policy)
assert policy.completion_criteria
- assert _completion_verification_criteria(ctx) == []
+ assert _completion_verification_criteria(ctx) == [policy.completion_criteria[0]]
def test_credential_aware_floor_adds_one_run_plane_criterion(self) -> None:
base = build_classifier_fallback_floor([])
@@ -5605,7 +5605,9 @@ class TestClassifierFallbackCompletionFloor:
assert len(credentialed) == len(base) + 1
assert all(c.level == "run" for c in credentialed)
- assert all(c.implicit and c.method_mandated for c in credentialed)
+ assert all(c.implicit for c in credentialed)
+ assert credentialed[0].method_mandated is False
+ assert credentialed[1].method_mandated is True
assert len({c.id for c in credentialed}) == len(credentialed)
def test_floor_respects_max_criteria_cap(self) -> None:
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index 9e89ee05b..3bc8a3c05 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -46,7 +46,12 @@ from skyvern.forge.sdk.copilot.enforcement import (
verified_goal_satisfied_context,
)
from skyvern.forge.sdk.copilot.hooks import _tool_completion_satisfies_turn
-from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy, _parse_completion_criteria
+from skyvern.forge.sdk.copilot.request_policy import (
+ CompletionCriterion,
+ RequestPolicy,
+ _parse_completion_criteria,
+ build_classifier_fallback_floor,
+)
from skyvern.forge.sdk.copilot.tools import (
ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,
_active_run_terminal_evidence_needs_visual_fallback,
@@ -2857,6 +2862,108 @@ async def test_maybe_run_completion_verification_terminal_goal_without_boolean_b
assert verification.verdicts[0].evidence_ref == "block_outputs:submit_water_request"
+async def test_maybe_run_completion_verification_fallback_floor_uses_recorded_terminal_goal(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def fail_handler(**_: object) -> object:
+ raise AssertionError("recorded fallback-floor outcome must not call the judge")
+
+ _patch_completion_handler(monkeypatch, fail_handler)
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "submit_water_request")
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=build_classifier_fallback_floor([]),
+ classifier_status="fallback",
+ )
+
+ verification = await _maybe_run_completion_verification(ctx, _terminal_goal_output_result(), time.monotonic())
+
+ assert verification is not None
+ assert verification.no_gradeable_run_plane is False
+ assert verification.is_fully_satisfied() is True
+ assert verification.to_trace_data()["criterion_count"] == 1
+ assert verification.verdicts[0].evidence_ref == "block_outputs:submit_water_request"
+
+
+async def test_maybe_run_completion_verification_fallback_floor_uses_page_end_state_evidence(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ captured: dict[str, str] = {}
+
+ async def handler(*, prompt: str, prompt_name: str) -> dict:
+ captured["prompt"] = prompt
+ return {
+ "verdicts": [
+ {
+ "criterion_id": "__copilot_fallback_floor__run",
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "observed_end_state_url",
+ }
+ ]
+ }
+
+ _patch_completion_handler(monkeypatch, handler)
+ ctx = _run_ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=build_classifier_fallback_floor([]),
+ classifier_status="fallback",
+ )
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_review",
+ "overall_status": "completed",
+ "executed_block_labels": [],
+ "current_url": "https://example.test/review",
+ "page_title": "Start Service - Review",
+ "blocks": [],
+ },
+ }
+
+ verification = await _maybe_run_completion_verification(ctx, result, time.monotonic())
+
+ assert verification is not None
+ assert verification.no_gradeable_run_plane is False
+ assert verification.is_fully_satisfied() is True
+ assert "observed_end_state_url: https://example.test/review" in captured["prompt"]
+ assert "observed_end_state_page_title: Start Service - Review" in captured["prompt"]
+
+
+async def test_maybe_run_completion_verification_fallback_floor_without_evidence_fails_closed(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def fail_handler(**_: object) -> object:
+ raise AssertionError("true no-evidence fallback floor must not call the judge")
+
+ _patch_completion_handler(monkeypatch, fail_handler)
+ ctx = _run_ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=build_classifier_fallback_floor([]),
+ classifier_status="fallback",
+ )
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_empty",
+ "overall_status": "completed",
+ "executed_block_labels": [],
+ "blocks": [],
+ },
+ }
+
+ verification = await _maybe_run_completion_verification(ctx, result, time.monotonic())
+
+ assert verification is not None
+ assert verification.no_gradeable_run_plane is False
+ assert verification.is_fully_satisfied() is False
+ assert verification.verdicts[0].state == "unsatisfied"
+ assert verification.verdicts[0].reason_code == "no_evidence"
+
+
@pytest.mark.asyncio
async def test_maybe_run_completion_verification_mixed_terminal_goal_upgrades_judge_unknown(
monkeypatch: pytest.MonkeyPatch,
diff --git a/tests/unit/test_copilot_request_policy_prompt.py b/tests/unit/test_copilot_request_policy_prompt.py
index a3e12764e..45d97de88 100644
--- a/tests/unit/test_copilot_request_policy_prompt.py
+++ b/tests/unit/test_copilot_request_policy_prompt.py
@@ -397,10 +397,10 @@ class TestClassifierFallbackCompletionCriteria:
"present",
{"output.user_name", "output.id", "output.address", "output.status"},
),
- ("Open https://example.com and click the pricing link.", "unknown", set()),
+ ("Open https://example.com and click the pricing link.", "present", set()),
# Substring matches ("read" in "already", "name" in "filename", "id" in "decided")
# must not satisfy the whole-word gate.
- ("I already decided the filename and reviewed the status of locations.", "unknown", set()),
+ ("I already decided the filename and reviewed the status of locations.", "present", set()),
# Generic structural group term ("entries") satisfies the group gate.
(
"Return a record with the entity name, identifier 1234567890, status, and the grouped entries.",