Skip to content

Automate browser based workflows with AI

SKY-11363: make declared completion-criteria count match the graded set (#6794)

Andrew Neilson64ac33305cd5
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index aa1236e9a..9b1703475 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -197,6 +197,9 @@ class RequestPolicy:
classifier_retry_count: int = 0
completion_contract_status: str = "absent"
+ def graded_completion_criteria(self) -> list[CompletionCriterion]:
+ return [criterion for criterion in self.completion_criteria if not criterion.method_mandated]
+
def to_trace_data(self) -> dict[str, Any]:
return {
"testing_intent": self.testing_intent,
@@ -207,7 +210,7 @@ class RequestPolicy:
"allow_missing_credentials_in_draft": self.allow_missing_credentials_in_draft,
"resolved_credential_count": len(self.resolved_credentials),
"has_completion_contract": bool(self.completion_contract),
- "completion_criteria_count": len(self.completion_criteria),
+ "completion_criteria_count": len(self.graded_completion_criteria()),
"completion_criteria_implicit_count": sum(
1 for criterion in self.completion_criteria if criterion.implicit
),
diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.py
index 3e29b7d84..0d15dece4 100644
--- a/skyvern/forge/sdk/copilot/tools/completion.py
+++ b/skyvern/forge/sdk/copilot/tools/completion.py
@@ -48,11 +48,11 @@ from .blockers import (
LOG = structlog.get_logger()
-def _completion_verification_criteria(copilot_ctx: Any) -> list[Any]:
+def _completion_verification_criteria(copilot_ctx: Any) -> list[CompletionCriterion]:
policy = getattr(copilot_ctx, "request_policy", None)
# A method-mandated criterion asserts HOW the goal was reached; the outcome
# judge sees only end-state evidence.
- return [c for c in (policy.completion_criteria if policy is not None else []) if not c.method_mandated]
+ return policy.graded_completion_criteria() if policy is not None else []
def _split_criteria_by_plane(criteria: list[Any]) -> tuple[list[CompletionCriterion], list[CompletionCriterion]]:
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 3d536c35e..4f6b5401f 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -5519,7 +5519,8 @@ class TestClassifierFallbackCompletionFloor:
assert policy.completion_contract_status == "unknown"
trace = policy.to_trace_data()
- assert trace["completion_criteria_count"] > 0
+ assert trace["completion_criteria_count"] == 0
+ assert trace["completion_criteria_method_mandated_count"] == len(policy.completion_criteria)
assert trace["has_completion_contract"] is False
def test_floor_is_excluded_from_judge_satisfaction_set(self) -> None:
@@ -5603,3 +5604,32 @@ class TestClassifierFallbackCompletionFloor:
]
assert not any(is_fallback_floor_criterion(c) for c in criteria)
+
+
+class TestDeclaredEqualsGradedCompletionCriteria:
+ @staticmethod
+ def _policy(total: int, method_mandated: int) -> RequestPolicy:
+ criteria = [
+ CompletionCriterion(id=f"c{i}", outcome=f"outcome {i}", method_mandated=i < method_mandated)
+ for i in range(total)
+ ]
+ return RequestPolicy(completion_criteria=criteria)
+
+ @pytest.mark.parametrize("total, method_mandated", [(6, 2), (8, 1), (5, 0)])
+ def test_declared_count_equals_graded_set_across_shapes(self, total: int, method_mandated: int) -> None:
+ policy = self._policy(total, method_mandated)
+ ctx = SimpleNamespace(request_policy=policy)
+ graded = total - method_mandated
+
+ declared = policy.to_trace_data()["completion_criteria_count"]
+ assert declared == len(policy.graded_completion_criteria())
+ assert declared == len(_completion_verification_criteria(ctx))
+ assert declared == graded
+ assert policy.to_trace_data()["completion_criteria_method_mandated_count"] == method_mandated
+
+ def test_graded_set_excludes_only_method_mandated_criteria(self) -> None:
+ policy = self._policy(6, 2)
+
+ graded = policy.graded_completion_criteria()
+ assert all(not criterion.method_mandated for criterion in graded)
+ assert {criterion.id for criterion in graded} == {"c2", "c3", "c4", "c5"}
diff --git a/tests/unit/test_copilot_outcome_verification_trace.py b/tests/unit/test_copilot_outcome_verification_trace.py
index 3d7a61935..13b977ae4 100644
--- a/tests/unit/test_copilot_outcome_verification_trace.py
+++ b/tests/unit/test_copilot_outcome_verification_trace.py
@@ -102,7 +102,7 @@ def test_outcome_verification_turn_fields_composes_all_sources() -> None:
assert fields["gate_satisfied"] is True
assert fields["completion_verification_status"] == "evaluated"
assert fields["verification_evidence_full_workflow_verified"] is True
- assert fields["request_policy_completion_criteria_count"] == 2
+ assert fields["request_policy_completion_criteria_count"] == 1
assert fields["request_policy_completion_criteria_implicit_count"] == 1
assert fields["request_policy_completion_criteria_method_mandated_count"] == 1