Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-11301): completion-criteria floor on classifier fallback (#6759)
Andrew Neilson6ab1fddda715
diff --git a/skyvern/forge/sdk/copilot/completion_criteria_store.py b/skyvern/forge/sdk/copilot/completion_criteria_store.pyindex 08f507f4a..d7f479fbd 100644--- a/skyvern/forge/sdk/copilot/completion_criteria_store.py+++ b/skyvern/forge/sdk/copilot/completion_criteria_store.py@@ -17,6 +17,7 @@ from skyvern.forge.sdk.copilot.completion_verification import ()from skyvern.forge.sdk.copilot.request_policy import (CompletionCriterion,+ is_fallback_floor_criterion,normalized_criterion_outcome_key,)@@ -159,6 +160,8 @@ def reconcile_completion_criteria("""stored = snapshot.active if snapshot is not None else Nonenext_epoch = snapshot.next_epoch if snapshot is not None else 1+ if fresh and all(is_fallback_floor_criterion(criterion) for criterion in fresh):+ fresh = []if stored is None:if not fresh:return ReconcileDecision(action="none", reason="no_criteria", epoch=0, criteria=())diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.pyindex 67ee13660..aa1236e9a 100644--- a/skyvern/forge/sdk/copilot/request_policy.py+++ b/skyvern/forge/sdk/copilot/request_policy.py@@ -145,6 +145,12 @@ _CREDENTIAL_CODE_MARKERS = ("saved credential", "login_credentials", ".otp()", "_MAX_COMPLETION_CRITERIA = 8_COMPLETION_CRITERION_OUTCOME_MAX_CHARS = 200+FALLBACK_FLOOR_CRITERION_ID_PREFIX = "__copilot_fallback_floor__"+_FALLBACK_FLOOR_BASE_ID = f"{FALLBACK_FLOOR_CRITERION_ID_PREFIX}run"+_FALLBACK_FLOOR_CREDENTIAL_ID = f"{FALLBACK_FLOOR_CRITERION_ID_PREFIX}credential"+_FALLBACK_FLOOR_BASE_OUTCOME = "The workflow runs to its intended end state with the expected output."+_FALLBACK_FLOOR_CREDENTIAL_OUTCOME = "The credentialed step authenticates and reaches the post-login state."+CriterionLevel = Literal["definition", "run"]_CRITERION_LEVELS: frozenset[str] = frozenset({"definition", "run"})@@ -561,6 +567,33 @@ def _render_active_criteria_for_prompt(criteria: list[CompletionCriterion] | Non)+def is_fallback_floor_criterion(criterion: CompletionCriterion) -> bool:+ return criterion.id.startswith(FALLBACK_FLOOR_CRITERION_ID_PREFIX)+++def build_classifier_fallback_floor(ids: list[str]) -> list[CompletionCriterion]:+ floor = [+ CompletionCriterion(+ id=_FALLBACK_FLOOR_BASE_ID,+ outcome=_FALLBACK_FLOOR_BASE_OUTCOME,+ implicit=True,+ method_mandated=True,+ level="run",+ )+ ]+ if ids:+ floor.append(+ CompletionCriterion(+ id=_FALLBACK_FLOOR_CREDENTIAL_ID,+ outcome=_FALLBACK_FLOOR_CREDENTIAL_OUTCOME,+ implicit=True,+ method_mandated=True,+ level="run",+ )+ )+ return floor[:_MAX_COMPLETION_CRITERIA]++def _classifier_fallback_policy(ids: list[str],*,@@ -593,7 +626,7 @@ def _classifier_fallback_policy(return RequestPolicy(credential_input_kind="credential_id" if ids else "none",credential_refs=ids,- completion_criteria=fallback_criteria,+ completion_criteria=fallback_criteria or build_classifier_fallback_floor(ids),classifier_status="fallback",classifier_failure_kind=failure_kind,classifier_retry_count=retry_count,diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex 20b35faff..466406f9f 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -11,6 +11,7 @@ from unittest.mock import AsyncMock, MagicMockimport pytestimport yaml+from skyvern.config import settingsfrom skyvern.forge.sdk.api.llm.exceptions import LLMProviderErrorfrom skyvern.forge.sdk.copilot import agent as agent_modulefrom skyvern.forge.sdk.copilot.agent import (@@ -20,6 +21,13 @@ from skyvern.forge.sdk.copilot.agent import ()from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignalfrom skyvern.forge.sdk.copilot.build_phase import BuildPhase+from skyvern.forge.sdk.copilot.completion_criteria_store import (+ StoredCriteriaSet,+ StoredCriteriaSnapshot,+ build_turn_state,+ plan_persistence,+ reconcile_completion_criteria,+)from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdictfrom skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfigfrom skyvern.forge.sdk.copilot.diagnosis_repair_contract import (@@ -39,15 +47,20 @@ from skyvern.forge.sdk.copilot.enforcement import ()from skyvern.forge.sdk.copilot.recoverable_failure import build_recoverable_failurefrom skyvern.forge.sdk.copilot.request_policy import (+ _MAX_COMPLETION_CRITERIA,_REDACTED_REFUSED_SECRET_TURN,TRANSCRIPT_ANCHOR_CHAR_CAP,CompletionCriterion,RequestPolicy,+ _classifier_fallback_policy,_classify_request,+ build_classifier_fallback_floor,build_transcript_context,+ is_fallback_floor_criterion,redact_raw_secrets_for_prompt,)from skyvern.forge.sdk.copilot.run_outcome import TERMINAL_CHALLENGE_BLOCKER_REASON_CODE, RecordedRunOutcome+from skyvern.forge.sdk.copilot.tools.completion import _completion_verification_criteriafrom skyvern.forge.sdk.copilot.turn_context import TranscriptContext, TurnContextOmission, TurnContextPacketfrom skyvern.forge.sdk.copilot.turn_halt import (TurnHalt,@@ -5404,3 +5417,117 @@ class TestStructuralInfeasibilityQuestion:def test_returns_none_for_non_turn_intent(self) -> None:assert agent_module._structural_infeasibility_question(None) is None+++class TestClassifierFallbackCompletionFloor:+ @pytest.mark.asyncio+ async def test_timeout_fallback_emits_method_mandated_run_floor(self, monkeypatch) -> None:+ monkeypatch.setattr(settings, "COPILOT_REQUEST_POLICY_CLASSIFIER_TIMEOUT_SECONDS", 0.05)++ async def handler(*, prompt: str, prompt_name: str) -> dict[str, object]:+ await asyncio.sleep(0.5)+ return {"credential_input_kind": "none", "completion_contract": None}++ policy = await _classify_request(+ user_message="Build a workflow for https://example.com.",+ workflow_yaml="",+ chat_history=[],+ global_llm_context="",+ handler=handler,+ )++ assert policy.classifier_status == "fallback"+ assert policy.classifier_failure_kind == "timeout"+ assert policy.completion_criteria+ assert all(c.level == "run" for c in policy.completion_criteria)+ assert all(c.implicit for c in policy.completion_criteria)+ assert all(c.method_mandated for c in policy.completion_criteria)+ assert all(is_fallback_floor_criterion(c) for c in policy.completion_criteria)+ assert policy.completion_contract is None+ assert policy.completion_contract_status == "unknown"++ trace = policy.to_trace_data()+ assert trace["completion_criteria_count"] > 0+ assert trace["has_completion_contract"] is False++ def test_floor_is_excluded_from_judge_satisfaction_set(self) -> None:+ policy = _classifier_fallback_policy([], raw_secret_present=False, failure_kind="timeout")+ ctx = SimpleNamespace(request_policy=policy)++ assert policy.completion_criteria+ assert _completion_verification_criteria(ctx) == []++ def test_credential_aware_floor_adds_one_run_plane_criterion(self) -> None:+ base = build_classifier_fallback_floor([])+ credentialed = build_classifier_fallback_floor(["cred_1"])++ assert len(credentialed) == len(base) + 1+ assert all(c.level == "run" for c in credentialed)+ assert all(c.implicit and c.method_mandated for c in credentialed)+ assert len({c.id for c in credentialed}) == len(credentialed)++ def test_floor_respects_max_criteria_cap(self) -> None:+ floor = build_classifier_fallback_floor([f"cred_{i}" for i in range(50)])++ assert len(floor) <= _MAX_COMPLETION_CRITERIA++ def test_raw_secret_fallback_still_blocks_with_empty_criteria(self) -> None:+ policy = _classifier_fallback_policy(["cred_1"], raw_secret_present=True, failure_kind="timeout")++ assert policy.raw_secret_handling == "block"+ assert policy.completion_criteria == []+ assert policy.completion_contract_status == "unknown"++ def test_reconcile_all_floor_fresh_with_no_stored_set_does_not_create(self) -> None:+ snapshot = StoredCriteriaSnapshot(active=None, next_epoch=1)+ floor = build_classifier_fallback_floor(["cred_1"])++ decision = reconcile_completion_criteria(snapshot, list(floor), actionable=True)++ assert decision.action == "none"+ assert decision.reason == "no_criteria"+ assert decision.criteria == ()++ turn_state = build_turn_state(snapshot, decision)+ plan = plan_persistence(turn_state)++ assert plan is None or plan.create_epoch is None++ def test_reconcile_all_floor_fresh_keeps_stored_real_epoch(self) -> None:+ stored = StoredCriteriaSet(+ set_id="wccs_real",+ goal_epoch=3,+ criteria=(CompletionCriterion(id="c0", outcome="The confirmation number is captured"),),+ )+ snapshot = StoredCriteriaSnapshot(active=stored, next_epoch=4)+ floor = build_classifier_fallback_floor(["cred_1"])++ decision = reconcile_completion_criteria(snapshot, list(floor), actionable=True)++ assert decision.action == "adopt_stored"+ assert decision.reason == "empty_fresh"+ assert decision.superseded_set_id is None+ assert decision.criteria == stored.criteria++ def test_reconcile_real_all_implicit_set_still_supersedes(self) -> None:+ stored = StoredCriteriaSet(+ set_id="wccs_old",+ goal_epoch=1,+ criteria=(CompletionCriterion(id="c0", outcome="The old goal is reached", implicit=True),),+ )+ snapshot = StoredCriteriaSnapshot(active=stored, next_epoch=2)+ real_fresh = [CompletionCriterion(id="c0", outcome="A different goal is reached", implicit=True)]++ decision = reconcile_completion_criteria(snapshot, real_fresh, actionable=True)++ assert decision.action == "create"+ assert decision.reason == "not_subset"+ assert decision.superseded_set_id == "wccs_old"++ def test_successful_classifier_turn_keeps_full_criteria_without_sentinel(self) -> None:+ criteria = [+ CompletionCriterion(id="c0", outcome="The account page is visible"),+ CompletionCriterion(id="c1", outcome="The confirmation number is captured"),+ ]++ assert not any(is_fallback_floor_criterion(c) for c in criteria)diff --git a/tests/unit/test_copilot_request_policy_prompt.py b/tests/unit/test_copilot_request_policy_prompt.pyindex 4a59df3ab..87b8c82e3 100644--- a/tests/unit/test_copilot_request_policy_prompt.py+++ b/tests/unit/test_copilot_request_policy_prompt.py@@ -16,6 +16,7 @@ from skyvern.forge.sdk.copilot.request_policy import (_credential_ids,_raw_secret_detected,contains_email_password_pair,+ is_fallback_floor_criterion,redact_raw_secrets_for_prompt,)from skyvern.forge.sdk.schemas.workflow_copilot import (@@ -413,9 +414,11 @@ class TestClassifierFallbackCompletionCriteria:assert policy.classifier_status == "fallback"assert policy.completion_contract_status == ("present" if expect_criteria else "unknown")if not expect_criteria:- assert policy.completion_criteria == []+ assert policy.completion_criteria+ assert all(is_fallback_floor_criterion(criterion) for criterion in policy.completion_criteria)return+ assert not any(is_fallback_floor_criterion(criterion) for criterion in policy.completion_criteria)assert [criterion.id for criterion in policy.completion_criteria] == ["fallback_record_identity","fallback_record_identifier",