Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(tier3): handle negative controls in insights
rookepooleb5b479eb63c7
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex cc2d4dd3..ceb96acc 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -90,6 +90,10 @@ All notable changes to SkillEvaluator are documented in this file.### Fixed+- Tier 3 LLM insights now receive explicit labels and bounded expected-behavior+ context for `expected_skill: null` negative controls, and the judge is+ instructed not to flag unrelated successes without invocation or+ failed-routing evidence.- Quality scoring now uses boundary-aware and context-aware matching for XML tags,reserved names, MCP guidance, README references, time references, exclusivitylanguage, instruction action verbs, and nested Markdown links, avoidingdiff --git a/src/skillevaluator/evaluation/insights_judge.py b/src/skillevaluator/evaluation/insights_judge.pyindex 9d167040..63278d39 100644--- a/src/skillevaluator/evaluation/insights_judge.py+++ b/src/skillevaluator/evaluation/insights_judge.py@@ -65,6 +65,20 @@ The runner already provides three deterministic baseline observations(\"best performing agent\", \"weakest dimension\", and an optional\"coverage expansion\" suggestion). DO NOT repeat those exact observations.+NEGATIVE-CONTROL INTERPRETATION:+- A dataset case whose expected_skill is null is an intentional negative+ control: the agent should complete the unrelated task without activating+ the evaluated skill.+- A successful negative-control case is correct routing behavior. Do not infer+ unintended skill application, scope misalignment, or overfitting merely+ because the agent answered the unrelated task correctly.+- Report unintended skill application only when explicit execution evidence+ shows that the evaluated skill was read or invoked, or routing evidence for+ that case failed. A passing routing result is evidence against that warning;+ when invocation evidence is absent or ambiguous, do not issue the warning.+- Do not recommend removing an unrelated negative-control case merely because+ it is outside the skill's scope; that is what the case is designed to test.+Your job is to produce ADDITIONAL, contextual insights:CONCLUSIONS (up to {max_conclusions}): observations the reviewer should know@@ -172,13 +186,34 @@ def _summarize_dataset(dataset: list[dict]) -> list[dict]:"""Trim dataset cases to the question + ground truth (plus id)."""summary: list[dict] = []for case in (dataset or [])[:5]:+ if "expected_skill" not in case:+ case_type = "unlabeled"+ skill_expected_to_activate = None+ elif case.get("expected_skill") is None:+ case_type = "negative_control"+ skill_expected_to_activate = False+ else:+ case_type = "skill_activation"+ skill_expected_to_activate = True++ expected_behavior = case.get("expected_behavior")+ if isinstance(expected_behavior, list):+ expected_behavior = [str(item)[:300] for item in expected_behavior[:5]]+ elif isinstance(expected_behavior, str):+ expected_behavior = expected_behavior[:300]+ else:+ expected_behavior = None+summary.append({"id": case.get("id"),+ "case_type": case_type,+ "skill_expected_to_activate": skill_expected_to_activate,"question": (case.get("question") or "")[:300],"ground_truth": (case.get("ground_truth") or "")[:300],"expected_skill": case.get("expected_skill"),"expected_script": case.get("expected_script"),+ "expected_behavior": expected_behavior,})return summarydiff --git a/tests/evaluation/test_judges.py b/tests/evaluation/test_judges.pyindex a2a80e80..f033a713 100644--- a/tests/evaluation/test_judges.py+++ b/tests/evaluation/test_judges.py@@ -10,10 +10,13 @@ network access.from __future__ import annotations+import json+import pytestfrom skillevaluator.constants import DIMENSION_MAPPINGfrom skillevaluator.evaluation import (+ InsightsJudge,build_insights,compute_dimensions,compute_dimensions_deterministic,@@ -71,3 +74,82 @@ class TestInsightsJudgeFallback:canonical = {"dimensions": compute_dimensions_deterministic(evaluators)}out = build_insights(canonical, deterministic, use_llm=False)assert out == {"conclusions": [], "recommendations": []}+++class TestInsightsJudgeNegativeControls:+ def test_prompt_marks_expected_skill_null_as_negative_control(self) -> None:+ prompt = InsightsJudge().create_user_prompt(+ canonical={+ "dataset": [+ {+ "id": "negative-001",+ "question": "Convert 42 degrees Fahrenheit to Celsius.",+ "ground_truth": "5.6",+ "expected_skill": None,+ "expected_behavior": [+ "The agent did not read or invoke the evaluated skill",+ ],+ }+ ]+ },+ deterministic={},+ )+ context = json.loads(prompt.split("\n\n", 1)[1])++ case = context["dataset"][0]+ assert case["case_type"] == "negative_control"+ assert case["skill_expected_to_activate"] is False+ assert case["expected_skill"] is None+ assert case["expected_behavior"] == [+ "The agent did not read or invoke the evaluated skill",+ ]++ @pytest.mark.parametrize(+ ("case", "expected_type", "expected_activation"),+ [+ ({"id": "positive-001", "expected_skill": "calculator"}, "skill_activation", True),+ ({"id": "unlabeled-001"}, "unlabeled", None),+ ],+ )+ def test_prompt_preserves_positive_and_unlabeled_case_semantics(+ self,+ case: dict,+ expected_type: str,+ expected_activation: bool | None,+ ) -> None:+ prompt = InsightsJudge().create_user_prompt(+ canonical={"dataset": [case]},+ deterministic={},+ )+ context = json.loads(prompt.split("\n\n", 1)[1])++ assert context["dataset"][0]["case_type"] == expected_type+ assert context["dataset"][0]["skill_expected_to_activate"] is expected_activation++ def test_prompt_bounds_negative_control_assertions(self) -> None:+ prompt = InsightsJudge().create_user_prompt(+ canonical={+ "dataset": [+ {+ "id": "negative-001",+ "expected_skill": None,+ "expected_behavior": ["x" * 500] * 10,+ }+ ]+ },+ deterministic={},+ )+ context = json.loads(prompt.split("\n\n", 1)[1])++ assertions = context["dataset"][0]["expected_behavior"]+ assert len(assertions) == 5+ assert all(len(assertion) == 300 for assertion in assertions)++ def test_system_prompt_requires_execution_evidence_for_negative_control_warning(self) -> None:+ prompt = InsightsJudge().get_system_prompt()++ assert "expected_skill is null" in prompt+ assert "successful negative-control" in prompt+ assert "explicit execution evidence" in prompt+ assert "invocation evidence is absent or ambiguous" in prompt+ assert "Do not recommend removing an unrelated negative-control case" in prompt