Skip to content

Automate browser based workflows with AI

fix(SKY-11301): completion-criteria floor on classifier fallback (#6759)

Andrew Neilson6ab1fddda715
diff --git a/skyvern/forge/sdk/copilot/completion_criteria_store.py b/skyvern/forge/sdk/copilot/completion_criteria_store.py
index 08f507f4a..d7f479fbd 100644
--- a/skyvern/forge/sdk/copilot/completion_criteria_store.py
+++ b/skyvern/forge/sdk/copilot/completion_criteria_store.py
@@ -17,6 +17,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (
)
from skyvern.forge.sdk.copilot.request_policy import (
CompletionCriterion,
+ is_fallback_floor_criterion,
normalized_criterion_outcome_key,
)
@@ -159,6 +160,8 @@ def reconcile_completion_criteria(
"""
stored = snapshot.active if snapshot is not None else None
next_epoch = snapshot.next_epoch if snapshot is not None else 1
+ if fresh and all(is_fallback_floor_criterion(criterion) for criterion in fresh):
+ fresh = []
if stored is None:
if not fresh:
return ReconcileDecision(action="none", reason="no_criteria", epoch=0, criteria=())
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index 67ee13660..aa1236e9a 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -145,6 +145,12 @@ _CREDENTIAL_CODE_MARKERS = ("saved credential", "login_credentials", ".otp()", "
_MAX_COMPLETION_CRITERIA = 8
_COMPLETION_CRITERION_OUTCOME_MAX_CHARS = 200
+FALLBACK_FLOOR_CRITERION_ID_PREFIX = "__copilot_fallback_floor__"
+_FALLBACK_FLOOR_BASE_ID = f"{FALLBACK_FLOOR_CRITERION_ID_PREFIX}run"
+_FALLBACK_FLOOR_CREDENTIAL_ID = f"{FALLBACK_FLOOR_CRITERION_ID_PREFIX}credential"
+_FALLBACK_FLOOR_BASE_OUTCOME = "The workflow runs to its intended end state with the expected output."
+_FALLBACK_FLOOR_CREDENTIAL_OUTCOME = "The credentialed step authenticates and reaches the post-login state."
+
CriterionLevel = Literal["definition", "run"]
_CRITERION_LEVELS: frozenset[str] = frozenset({"definition", "run"})
@@ -561,6 +567,33 @@ def _render_active_criteria_for_prompt(criteria: list[CompletionCriterion] | Non
)
+def is_fallback_floor_criterion(criterion: CompletionCriterion) -> bool:
+ return criterion.id.startswith(FALLBACK_FLOOR_CRITERION_ID_PREFIX)
+
+
+def build_classifier_fallback_floor(ids: list[str]) -> list[CompletionCriterion]:
+ floor = [
+ CompletionCriterion(
+ id=_FALLBACK_FLOOR_BASE_ID,
+ outcome=_FALLBACK_FLOOR_BASE_OUTCOME,
+ implicit=True,
+ method_mandated=True,
+ level="run",
+ )
+ ]
+ if ids:
+ floor.append(
+ CompletionCriterion(
+ id=_FALLBACK_FLOOR_CREDENTIAL_ID,
+ outcome=_FALLBACK_FLOOR_CREDENTIAL_OUTCOME,
+ implicit=True,
+ method_mandated=True,
+ level="run",
+ )
+ )
+ return floor[:_MAX_COMPLETION_CRITERIA]
+
+
def _classifier_fallback_policy(
ids: list[str],
*,
@@ -593,7 +626,7 @@ def _classifier_fallback_policy(
return RequestPolicy(
credential_input_kind="credential_id" if ids else "none",
credential_refs=ids,
- completion_criteria=fallback_criteria,
+ completion_criteria=fallback_criteria or build_classifier_fallback_floor(ids),
classifier_status="fallback",
classifier_failure_kind=failure_kind,
classifier_retry_count=retry_count,
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 20b35faff..466406f9f 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -11,6 +11,7 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
import yaml
+from skyvern.config import settings
from skyvern.forge.sdk.api.llm.exceptions import LLMProviderError
from skyvern.forge.sdk.copilot import agent as agent_module
from skyvern.forge.sdk.copilot.agent import (
@@ -20,6 +21,13 @@ from skyvern.forge.sdk.copilot.agent import (
)
from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
from skyvern.forge.sdk.copilot.build_phase import BuildPhase
+from skyvern.forge.sdk.copilot.completion_criteria_store import (
+ StoredCriteriaSet,
+ StoredCriteriaSnapshot,
+ build_turn_state,
+ plan_persistence,
+ reconcile_completion_criteria,
+)
from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfig
from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
@@ -39,15 +47,20 @@ from skyvern.forge.sdk.copilot.enforcement import (
)
from skyvern.forge.sdk.copilot.recoverable_failure import build_recoverable_failure
from skyvern.forge.sdk.copilot.request_policy import (
+ _MAX_COMPLETION_CRITERIA,
_REDACTED_REFUSED_SECRET_TURN,
TRANSCRIPT_ANCHOR_CHAR_CAP,
CompletionCriterion,
RequestPolicy,
+ _classifier_fallback_policy,
_classify_request,
+ build_classifier_fallback_floor,
build_transcript_context,
+ is_fallback_floor_criterion,
redact_raw_secrets_for_prompt,
)
from skyvern.forge.sdk.copilot.run_outcome import TERMINAL_CHALLENGE_BLOCKER_REASON_CODE, RecordedRunOutcome
+from skyvern.forge.sdk.copilot.tools.completion import _completion_verification_criteria
from skyvern.forge.sdk.copilot.turn_context import TranscriptContext, TurnContextOmission, TurnContextPacket
from skyvern.forge.sdk.copilot.turn_halt import (
TurnHalt,
@@ -5404,3 +5417,117 @@ class TestStructuralInfeasibilityQuestion:
def test_returns_none_for_non_turn_intent(self) -> None:
assert agent_module._structural_infeasibility_question(None) is None
+
+
+class TestClassifierFallbackCompletionFloor:
+ @pytest.mark.asyncio
+ async def test_timeout_fallback_emits_method_mandated_run_floor(self, monkeypatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_REQUEST_POLICY_CLASSIFIER_TIMEOUT_SECONDS", 0.05)
+
+ async def handler(*, prompt: str, prompt_name: str) -> dict[str, object]:
+ await asyncio.sleep(0.5)
+ return {"credential_input_kind": "none", "completion_contract": None}
+
+ policy = await _classify_request(
+ user_message="Build a workflow for https://example.com.",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ handler=handler,
+ )
+
+ assert policy.classifier_status == "fallback"
+ assert policy.classifier_failure_kind == "timeout"
+ assert policy.completion_criteria
+ assert all(c.level == "run" for c in policy.completion_criteria)
+ assert all(c.implicit for c in policy.completion_criteria)
+ assert all(c.method_mandated for c in policy.completion_criteria)
+ assert all(is_fallback_floor_criterion(c) for c in policy.completion_criteria)
+ assert policy.completion_contract is None
+ assert policy.completion_contract_status == "unknown"
+
+ trace = policy.to_trace_data()
+ assert trace["completion_criteria_count"] > 0
+ assert trace["has_completion_contract"] is False
+
+ def test_floor_is_excluded_from_judge_satisfaction_set(self) -> None:
+ policy = _classifier_fallback_policy([], raw_secret_present=False, failure_kind="timeout")
+ ctx = SimpleNamespace(request_policy=policy)
+
+ assert policy.completion_criteria
+ assert _completion_verification_criteria(ctx) == []
+
+ def test_credential_aware_floor_adds_one_run_plane_criterion(self) -> None:
+ base = build_classifier_fallback_floor([])
+ credentialed = build_classifier_fallback_floor(["cred_1"])
+
+ assert len(credentialed) == len(base) + 1
+ assert all(c.level == "run" for c in credentialed)
+ assert all(c.implicit and c.method_mandated for c in credentialed)
+ assert len({c.id for c in credentialed}) == len(credentialed)
+
+ def test_floor_respects_max_criteria_cap(self) -> None:
+ floor = build_classifier_fallback_floor([f"cred_{i}" for i in range(50)])
+
+ assert len(floor) <= _MAX_COMPLETION_CRITERIA
+
+ def test_raw_secret_fallback_still_blocks_with_empty_criteria(self) -> None:
+ policy = _classifier_fallback_policy(["cred_1"], raw_secret_present=True, failure_kind="timeout")
+
+ assert policy.raw_secret_handling == "block"
+ assert policy.completion_criteria == []
+ assert policy.completion_contract_status == "unknown"
+
+ def test_reconcile_all_floor_fresh_with_no_stored_set_does_not_create(self) -> None:
+ snapshot = StoredCriteriaSnapshot(active=None, next_epoch=1)
+ floor = build_classifier_fallback_floor(["cred_1"])
+
+ decision = reconcile_completion_criteria(snapshot, list(floor), actionable=True)
+
+ assert decision.action == "none"
+ assert decision.reason == "no_criteria"
+ assert decision.criteria == ()
+
+ turn_state = build_turn_state(snapshot, decision)
+ plan = plan_persistence(turn_state)
+
+ assert plan is None or plan.create_epoch is None
+
+ def test_reconcile_all_floor_fresh_keeps_stored_real_epoch(self) -> None:
+ stored = StoredCriteriaSet(
+ set_id="wccs_real",
+ goal_epoch=3,
+ criteria=(CompletionCriterion(id="c0", outcome="The confirmation number is captured"),),
+ )
+ snapshot = StoredCriteriaSnapshot(active=stored, next_epoch=4)
+ floor = build_classifier_fallback_floor(["cred_1"])
+
+ decision = reconcile_completion_criteria(snapshot, list(floor), actionable=True)
+
+ assert decision.action == "adopt_stored"
+ assert decision.reason == "empty_fresh"
+ assert decision.superseded_set_id is None
+ assert decision.criteria == stored.criteria
+
+ def test_reconcile_real_all_implicit_set_still_supersedes(self) -> None:
+ stored = StoredCriteriaSet(
+ set_id="wccs_old",
+ goal_epoch=1,
+ criteria=(CompletionCriterion(id="c0", outcome="The old goal is reached", implicit=True),),
+ )
+ snapshot = StoredCriteriaSnapshot(active=stored, next_epoch=2)
+ real_fresh = [CompletionCriterion(id="c0", outcome="A different goal is reached", implicit=True)]
+
+ decision = reconcile_completion_criteria(snapshot, real_fresh, actionable=True)
+
+ assert decision.action == "create"
+ assert decision.reason == "not_subset"
+ assert decision.superseded_set_id == "wccs_old"
+
+ def test_successful_classifier_turn_keeps_full_criteria_without_sentinel(self) -> None:
+ criteria = [
+ CompletionCriterion(id="c0", outcome="The account page is visible"),
+ CompletionCriterion(id="c1", outcome="The confirmation number is captured"),
+ ]
+
+ assert not any(is_fallback_floor_criterion(c) for c in criteria)
diff --git a/tests/unit/test_copilot_request_policy_prompt.py b/tests/unit/test_copilot_request_policy_prompt.py
index 4a59df3ab..87b8c82e3 100644
--- a/tests/unit/test_copilot_request_policy_prompt.py
+++ b/tests/unit/test_copilot_request_policy_prompt.py
@@ -16,6 +16,7 @@ from skyvern.forge.sdk.copilot.request_policy import (
_credential_ids,
_raw_secret_detected,
contains_email_password_pair,
+ is_fallback_floor_criterion,
redact_raw_secrets_for_prompt,
)
from skyvern.forge.sdk.schemas.workflow_copilot import (
@@ -413,9 +414,11 @@ class TestClassifierFallbackCompletionCriteria:
assert policy.classifier_status == "fallback"
assert policy.completion_contract_status == ("present" if expect_criteria else "unknown")
if not expect_criteria:
- assert policy.completion_criteria == []
+ assert policy.completion_criteria
+ assert all(is_fallback_floor_criterion(criterion) for criterion in policy.completion_criteria)
return
+ assert not any(is_fallback_floor_criterion(criterion) for criterion in policy.completion_criteria)
assert [criterion.id for criterion in policy.completion_criteria] == [
"fallback_record_identity",
"fallback_record_identifier",