Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(doctor): label scanners that SkillEvaluator provisions itself (#540)

Christopher Kevinda42ece43fe2
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 58bd6731..7997326e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -233,6 +233,11 @@ All notable changes to SkillEvaluator are documented in this file.
### Fixed
+- `health-check --tier 1` (and `doctor --tier 1`) labels a scanner that
+ SkillEvaluator provisions itself: `(uv tool run)` for SkillSpector and
+ `(managed)` for the downloaded Gitleaks binary. Before, the Gitleaks row
+ ended in `(from None)` and the SkillSpector row named
+ `SKILLEVALUATOR_SKILLSPECTOR_PATH` although no override was set.
- The comments-only consumer template now grants its required read and PR-comment
permissions only to the comment job, with no workflow-wide token permissions.
- The behavior evidence gate no longer keeps a score for an expectation that
diff --git a/src/skillevaluator/doctor.py b/src/skillevaluator/doctor.py
index 7be5aeed..49f56f37 100644
--- a/src/skillevaluator/doctor.py
+++ b/src/skillevaluator/doctor.py
@@ -73,8 +73,11 @@ def _scanner_row(tool: Any, timeout: float) -> Row:
if not tool.is_available:
return (label, "fail", f"not installed. {tool.get_install_hint()}")
location = tool.path
- if tool.resolution_source != "automatic":
+ if tool.resolution_source.startswith("override:"):
location += f" (from {tool.override_env})"
+ elif tool.resolution_source != "automatic":
+ # Managed provisioning: "uv tool run" for SkillSpector, "managed" for Gitleaks.
+ location += f" ({tool.resolution_source})"
version_args = ["version"] if tool.command == "gitleaks" else ["--version"]
result = tool.run(version_args, timeout=timeout, log_command=False, **_probe_run_options(tool.command))
if result.error_message:
diff --git a/tests/test_doctor.py b/tests/test_doctor.py
index 9271e55c..4c185ad1 100644
--- a/tests/test_doctor.py
+++ b/tests/test_doctor.py
@@ -133,6 +133,32 @@ def test_tier1_reports_an_explicit_executable_override(scanners, monkeypatch, tm
assert rows["Bandit"] == ("pass", f"1.9.4 at {executable.resolve()} (from SKILLEVALUATOR_BANDIT_PATH)")
+def test_tier1_reports_managed_provisioning_without_naming_an_override(scanners, monkeypatch, tmp_path) -> None:
+ """No scanner is installed, so SkillSpector runs through uv and Gitleaks is the managed binary."""
+ monkeypatch.setenv("SKILLEVALUATOR_RUNTIME_DIR", str(tmp_path))
+ monkeypatch.setattr(
+ tool_runner, "resolve_tool_path", lambda command, **_kwargs: "/env/bin/uv" if command == "uv" else None
+ )
+ monkeypatch.setattr(tool_runner, "gitleaks_platform", lambda: ("linux", "x64"))
+ skillspector = ExternalTool(
+ "skillspector", "skillspector", override_env="SKILLEVALUATOR_SKILLSPECTOR_PATH", managed=True
+ )
+ gitleaks = ExternalTool("Gitleaks", "gitleaks", managed=True)
+ # The wheel and the binary are already in place, so the probes download nothing.
+ monkeypatch.setattr(skillspector, "_ensure", None)
+ monkeypatch.setattr(gitleaks, "_ensure", None)
+ monkeypatch.setattr(Tools, "skillspector", skillspector)
+ monkeypatch.setattr(Tools, "gitleaks", gitleaks)
+ scanners.output["uv"] = "SkillSpector v2.13.4\n"
+
+ rows = _rows(tier="1", checks="security,code")
+
+ assert rows["SkillSpector"] == ("pass", "2.13.4 at /env/bin/uv (uv tool run)")
+ assert rows["Gitleaks"] == ("pass", f"8.21.2 at {gitleaks.path} (managed)")
+ assert scanners.calls[0][0][:3] == ["/env/bin/uv", "tool", "run"]
+ assert scanners.calls[0][0][-2:] == ["skillspector", "--version"]
+
+
@pytest.mark.parametrize("failure", ["missing", "malformed override", "timeout", "exit status"])
def test_tier1_scanner_failure_does_not_hide_other_scanners(scanners, monkeypatch, failure) -> None:
if failure == "missing":