Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(doctor): label scanners that SkillEvaluator provisions itself (#540)
Christopher Kevinda42ece43fe2
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 58bd6731..7997326e 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -233,6 +233,11 @@ All notable changes to SkillEvaluator are documented in this file.### Fixed+- `health-check --tier 1` (and `doctor --tier 1`) labels a scanner that+ SkillEvaluator provisions itself: `(uv tool run)` for SkillSpector and+ `(managed)` for the downloaded Gitleaks binary. Before, the Gitleaks row+ ended in `(from None)` and the SkillSpector row named+ `SKILLEVALUATOR_SKILLSPECTOR_PATH` although no override was set.- The comments-only consumer template now grants its required read and PR-commentpermissions only to the comment job, with no workflow-wide token permissions.- The behavior evidence gate no longer keeps a score for an expectation thatdiff --git a/src/skillevaluator/doctor.py b/src/skillevaluator/doctor.pyindex 7be5aeed..49f56f37 100644--- a/src/skillevaluator/doctor.py+++ b/src/skillevaluator/doctor.py@@ -73,8 +73,11 @@ def _scanner_row(tool: Any, timeout: float) -> Row:if not tool.is_available:return (label, "fail", f"not installed. {tool.get_install_hint()}")location = tool.path- if tool.resolution_source != "automatic":+ if tool.resolution_source.startswith("override:"):location += f" (from {tool.override_env})"+ elif tool.resolution_source != "automatic":+ # Managed provisioning: "uv tool run" for SkillSpector, "managed" for Gitleaks.+ location += f" ({tool.resolution_source})"version_args = ["version"] if tool.command == "gitleaks" else ["--version"]result = tool.run(version_args, timeout=timeout, log_command=False, **_probe_run_options(tool.command))if result.error_message:diff --git a/tests/test_doctor.py b/tests/test_doctor.pyindex 9271e55c..4c185ad1 100644--- a/tests/test_doctor.py+++ b/tests/test_doctor.py@@ -133,6 +133,32 @@ def test_tier1_reports_an_explicit_executable_override(scanners, monkeypatch, tmassert rows["Bandit"] == ("pass", f"1.9.4 at {executable.resolve()} (from SKILLEVALUATOR_BANDIT_PATH)")+def test_tier1_reports_managed_provisioning_without_naming_an_override(scanners, monkeypatch, tmp_path) -> None:+ """No scanner is installed, so SkillSpector runs through uv and Gitleaks is the managed binary."""+ monkeypatch.setenv("SKILLEVALUATOR_RUNTIME_DIR", str(tmp_path))+ monkeypatch.setattr(+ tool_runner, "resolve_tool_path", lambda command, **_kwargs: "/env/bin/uv" if command == "uv" else None+ )+ monkeypatch.setattr(tool_runner, "gitleaks_platform", lambda: ("linux", "x64"))+ skillspector = ExternalTool(+ "skillspector", "skillspector", override_env="SKILLEVALUATOR_SKILLSPECTOR_PATH", managed=True+ )+ gitleaks = ExternalTool("Gitleaks", "gitleaks", managed=True)+ # The wheel and the binary are already in place, so the probes download nothing.+ monkeypatch.setattr(skillspector, "_ensure", None)+ monkeypatch.setattr(gitleaks, "_ensure", None)+ monkeypatch.setattr(Tools, "skillspector", skillspector)+ monkeypatch.setattr(Tools, "gitleaks", gitleaks)+ scanners.output["uv"] = "SkillSpector v2.13.4\n"++ rows = _rows(tier="1", checks="security,code")++ assert rows["SkillSpector"] == ("pass", "2.13.4 at /env/bin/uv (uv tool run)")+ assert rows["Gitleaks"] == ("pass", f"8.21.2 at {gitleaks.path} (managed)")+ assert scanners.calls[0][0][:3] == ["/env/bin/uv", "tool", "run"]+ assert scanners.calls[0][0][-2:] == ["skillspector", "--version"]++@pytest.mark.parametrize("failure", ["missing", "malformed override", "timeout", "exit status"])def test_tier1_scanner_failure_does_not_hide_other_scanners(scanners, monkeypatch, failure) -> None:if failure == "missing":