Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
Fix unchecked behavior-step qualification reporting (#504)
Narendran Raghavan82a2ead2cdfc
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 8c207c41..9bac6f5c 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -35,6 +35,9 @@ All notable changes to SkillEvaluator are documented in this file.### Fixed+- Grader qualification reports null behavior-step results with a matching+ `not_checked` expectation as insufficient evidence instead of errors. Raw+ scores and unjudgeable rates are unchanged.- A failed Tier 3 execution (for example a backend launch failure) now marks thereport payload, its summary and the affected agents as failed with theirerrors, instead of only the top-level `result.json`. A report re-rendered bydiff --git a/src/skillevaluator/tier3/grader_validation.py b/src/skillevaluator/tier3/grader_validation.pyindex 091bf477..16da6774 100644--- a/src/skillevaluator/tier3/grader_validation.py+++ b/src/skillevaluator/tier3/grader_validation.py@@ -307,7 +307,8 @@ def observations(case, reward, thresholds):"score": score,})- steps = reward.get("details", {}).get(metric, {}).get("results", [])+ detail = reward.get("details", {}).get(metric, {})+ steps = detail.get("results", [])for index, passed in enumerate(expected):verdict = observedstep_score = None@@ -315,6 +316,21 @@ def observations(case, reward, thresholds):step = next((s for s in steps if s.get("step") == index + 1), {})step_score = step.get("passed")verdict = ("pass" if step_score else "fail") if type(step_score) is bool else "error"+ expectations = detail.get("expectations", [])+ if (+ "passed" in step+ and step_score is None+ and type(step.get("step")) is int+ and isinstance(expectations, list)+ and any(+ isinstance(item, dict)+ and type(item.get("step")) is int+ and item["step"] == index + 1+ and item.get("status") == "not_checked"+ for item in expectations+ )+ ):+ verdict = "insufficient_evidence"result.append({"metric": "behavior_steps",diff --git a/tests/tier3/test_grader_validation.py b/tests/tier3/test_grader_validation.pyindex 03bf6fe8..fb3541c1 100644--- a/tests/tier3/test_grader_validation.py+++ b/tests/tier3/test_grader_validation.py@@ -216,6 +216,133 @@ def test_behavior_pool():assert all(r["observed"] == "error" for r in gv.observations(case, reward, gv.DEFAULT_THRESHOLDS)[1:])+def unchecked_behavior():+ case = {"id": "baseline", "expected": {"overall": "fail", "metrics": {"behavior_check": [False, True]}}}+ reward = {+ "overall": 0.4,+ "behavior_check": 0.5,+ "details": {+ "behavior_check": {+ "score": 0.5,+ "reason": "The numeric answer was returned, but instruction reading cannot be verified.",+ "results": [+ {"step": 1, "passed": None, "reason": "Tool execution details are not provided."},+ {"step": 2, "passed": True, "reason": "The final response was only the number 14."},+ ],+ "expectations": [+ {+ "step": 1,+ "requirement": "Read calculator instructions before running calc.py.",+ "status": "not_checked",+ "reason": "Tool execution details are not provided.",+ },+ {+ "step": 2,+ "requirement": "Return only the numeric answer.",+ "status": "met",+ "reason": "The final response was only the number 14.",+ },+ ],+ }+ },+ }+ return case, reward++[email protected]("reverse_expectations", [False, True])+def test_unchecked_behavior_step_projects_insufficient_evidence(unchecked_behavior, reverse_expectations):+ case, reward = unchecked_behavior+ if reverse_expectations:+ reward["details"]["behavior_check"]["expectations"].reverse()+ original = copy.deepcopy(reward)+ rows = gv.observations(case, reward, gv.DEFAULT_THRESHOLDS)+ assert [(r["metric"], r["expected"], r["observed"], r["score"]) for r in rows] == [+ ("overall", "fail", "fail", 0.4),+ ("behavior_check", "fail", "fail", 0.5),+ ("behavior_steps", "fail", "insufficient_evidence", None),+ ("behavior_steps", "pass", "pass", 1),+ ]+ assert reward == original+ summary = gv.summarize([r for r in rows if r["metric"] == "behavior_steps"])+ assert summary["errors"] == 0+ assert summary["insufficient_evidence"] == 1+ assert summary["confusion_matrix"]["fail"]["insufficient_evidence"] == 1+ assert summary["unjudgeable_rate"]["numerator"] == 1+ assert summary["unjudgeable_rate"]["denominator"] == 2+ without_status = copy.deepcopy(reward)+ without_status["details"]["behavior_check"].pop("expectations")+ error_summary = gv.summarize(+ [r for r in gv.observations(case, without_status, gv.DEFAULT_THRESHOLDS) if r["metric"] == "behavior_steps"]+ )+ assert error_summary["errors"] == 1+ assert error_summary["insufficient_evidence"] == 0+ assert summary["unjudgeable_rate"] == error_summary["unjudgeable_rate"]+++ "expectations",+ [+ None,+ {},+ [],+ [None],+ [{"status": "not_checked"}],+ [{"step": 2, "status": "not_checked"}],+ [{"step": "1", "status": "not_checked"}],+ [{"step": True, "status": "not_checked"}],+ [{"step": 1.0, "status": "not_checked"}],+ [{"step": 1}],+ *[{"step": 1, "status": status} for status in (None, "met", "unmet", "error", "unknown", [], {})],+ ],+)+def test_null_behavior_step_without_valid_matching_status_is_error(unchecked_behavior, expectations):+ case, reward = unchecked_behavior+ reward["details"]["behavior_check"]["expectations"] = expectations+ steps = [r for r in gv.observations(case, reward, gv.DEFAULT_THRESHOLDS) if r["metric"] == "behavior_steps"]+ assert [(r["observed"], r["score"]) for r in steps] == [("error", None), ("pass", 1)]+++ "result",+ [+ {},+ {"step": 1},+ {"step": 2, "passed": None},+ {"step": "1", "passed": None},+ {"step": True, "passed": None},+ {"step": 1.0, "passed": None},+ *[{"step": 1, "passed": passed} for passed in ("true", 1, 0, {}, [])],+ ],+)+def test_malformed_behavior_result_cannot_abstain(unchecked_behavior, result):+ case, reward = unchecked_behavior+ reward["details"]["behavior_check"]["results"][0] = result+ first = gv.observations(case, reward, gv.DEFAULT_THRESHOLDS)[2]+ assert (first["observed"], first["score"]) == ("error", None)++[email protected]("passed", [False, True])+def test_boolean_behavior_result_keeps_verdict_with_not_checked_status(unchecked_behavior, passed):+ case, reward = unchecked_behavior+ reward["details"]["behavior_check"]["results"][0]["passed"] = passed+ first = gv.observations(case, reward, gv.DEFAULT_THRESHOLDS)[2]+ assert (first["observed"], first["score"]) == ("pass" if passed else "fail", int(passed))++[email protected]("failure", ["status", "process", "score"])+def test_aggregate_behavior_error_overrides_not_checked_step(unchecked_behavior, failure):+ case, reward = unchecked_behavior+ if failure == "status":+ reward["details"]["behavior_check"]["status"] = "error"+ elif failure == "process":+ reward["_process_error"] = True+ else:+ reward.pop("behavior_check")+ rows = gv.observations(case, reward, gv.DEFAULT_THRESHOLDS)+ assert [r["observed"] for r in rows[1:]] == ["error", "error", "error"]++def test_behavior_expected_class_uses_suite_threshold():case = {"id": "mixed", "expected": {"overall": "fail", "metrics": {"behavior_check": [True, False]}}}reward = {