Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-11363: make declared completion-criteria count match the graded set (#6794)
Andrew Neilson64ac33305cd5
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.pyindex aa1236e9a..9b1703475 100644--- a/skyvern/forge/sdk/copilot/request_policy.py+++ b/skyvern/forge/sdk/copilot/request_policy.py@@ -197,6 +197,9 @@ class RequestPolicy:classifier_retry_count: int = 0completion_contract_status: str = "absent"+ def graded_completion_criteria(self) -> list[CompletionCriterion]:+ return [criterion for criterion in self.completion_criteria if not criterion.method_mandated]+def to_trace_data(self) -> dict[str, Any]:return {"testing_intent": self.testing_intent,@@ -207,7 +210,7 @@ class RequestPolicy:"allow_missing_credentials_in_draft": self.allow_missing_credentials_in_draft,"resolved_credential_count": len(self.resolved_credentials),"has_completion_contract": bool(self.completion_contract),- "completion_criteria_count": len(self.completion_criteria),+ "completion_criteria_count": len(self.graded_completion_criteria()),"completion_criteria_implicit_count": sum(1 for criterion in self.completion_criteria if criterion.implicit),diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.pyindex 3e29b7d84..0d15dece4 100644--- a/skyvern/forge/sdk/copilot/tools/completion.py+++ b/skyvern/forge/sdk/copilot/tools/completion.py@@ -48,11 +48,11 @@ from .blockers import (LOG = structlog.get_logger()-def _completion_verification_criteria(copilot_ctx: Any) -> list[Any]:+def _completion_verification_criteria(copilot_ctx: Any) -> list[CompletionCriterion]:policy = getattr(copilot_ctx, "request_policy", None)# A method-mandated criterion asserts HOW the goal was reached; the outcome# judge sees only end-state evidence.- return [c for c in (policy.completion_criteria if policy is not None else []) if not c.method_mandated]+ return policy.graded_completion_criteria() if policy is not None else []def _split_criteria_by_plane(criteria: list[Any]) -> tuple[list[CompletionCriterion], list[CompletionCriterion]]:diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex 3d536c35e..4f6b5401f 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -5519,7 +5519,8 @@ class TestClassifierFallbackCompletionFloor:assert policy.completion_contract_status == "unknown"trace = policy.to_trace_data()- assert trace["completion_criteria_count"] > 0+ assert trace["completion_criteria_count"] == 0+ assert trace["completion_criteria_method_mandated_count"] == len(policy.completion_criteria)assert trace["has_completion_contract"] is Falsedef test_floor_is_excluded_from_judge_satisfaction_set(self) -> None:@@ -5603,3 +5604,32 @@ class TestClassifierFallbackCompletionFloor:]assert not any(is_fallback_floor_criterion(c) for c in criteria)+++class TestDeclaredEqualsGradedCompletionCriteria:+ @staticmethod+ def _policy(total: int, method_mandated: int) -> RequestPolicy:+ criteria = [+ CompletionCriterion(id=f"c{i}", outcome=f"outcome {i}", method_mandated=i < method_mandated)+ for i in range(total)+ ]+ return RequestPolicy(completion_criteria=criteria)++ @pytest.mark.parametrize("total, method_mandated", [(6, 2), (8, 1), (5, 0)])+ def test_declared_count_equals_graded_set_across_shapes(self, total: int, method_mandated: int) -> None:+ policy = self._policy(total, method_mandated)+ ctx = SimpleNamespace(request_policy=policy)+ graded = total - method_mandated++ declared = policy.to_trace_data()["completion_criteria_count"]+ assert declared == len(policy.graded_completion_criteria())+ assert declared == len(_completion_verification_criteria(ctx))+ assert declared == graded+ assert policy.to_trace_data()["completion_criteria_method_mandated_count"] == method_mandated++ def test_graded_set_excludes_only_method_mandated_criteria(self) -> None:+ policy = self._policy(6, 2)++ graded = policy.graded_completion_criteria()+ assert all(not criterion.method_mandated for criterion in graded)+ assert {criterion.id for criterion in graded} == {"c2", "c3", "c4", "c5"}diff --git a/tests/unit/test_copilot_outcome_verification_trace.py b/tests/unit/test_copilot_outcome_verification_trace.pyindex 3d7a61935..13b977ae4 100644--- a/tests/unit/test_copilot_outcome_verification_trace.py+++ b/tests/unit/test_copilot_outcome_verification_trace.py@@ -102,7 +102,7 @@ def test_outcome_verification_turn_fields_composes_all_sources() -> None:assert fields["gate_satisfied"] is Trueassert fields["completion_verification_status"] == "evaluated"assert fields["verification_evidence_full_workflow_verified"] is True- assert fields["request_policy_completion_criteria_count"] == 2+ assert fields["request_policy_completion_criteria_count"] == 1assert fields["request_policy_completion_criteria_implicit_count"] == 1assert fields["request_policy_completion_criteria_method_mandated_count"] == 1