Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(doctor): suggest scanner installs that work on the host platform (#796)
sm18lr883ecd69bd7fb1
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 48382297..bf760594 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -22,6 +22,10 @@ All notable changes to SkillEvaluator are documented in this file.### Fixed+- Scanner install hints in `health-check` and `doctor` now match the host:+ Windows suggests `winget install Gitleaks.Gitleaks` and `uv tool install semgrep`+ instead of `brew`, macOS keeps `brew`, and Linux suggests `go install` or a+ release binary.- Malformed optional trajectory `final_metrics` leave condition usage unavailableand no longer abort fresh or reused Tier 3 result collection.- Allow GitHub all-tier CI to opt out of deployment records while retainingdiff --git a/src/skillevaluator/utils/tool_runner.py b/src/skillevaluator/utils/tool_runner.pyindex 71cc1752..c0064f45 100644--- a/src/skillevaluator/utils/tool_runner.py+++ b/src/skillevaluator/utils/tool_runner.py@@ -325,14 +325,24 @@ class ExternalTool:"Bundled with the security extra — reinstall with: "'uv tool install "skillevaluator[all] @ git+https://github.com/NVIDIA/SkillEvaluator.git"')+ gitleaks_go = "go install github.com/zricethezav/gitleaks/v8@latest"+ if sys.platform == "win32":+ gitleaks_hint = f"Install with: winget install Gitleaks.Gitleaks or {gitleaks_go}"+ semgrep_hint = "Optional; install separately with: uv tool install semgrep"+ elif sys.platform == "darwin":+ gitleaks_hint = f"Install with: brew install gitleaks or {gitleaks_go}"+ semgrep_hint = "Optional; install separately with: brew install semgrep or uv tool install semgrep"+ else:+ gitleaks_hint = (+ f"Install with: {gitleaks_go}, or a release binary from https://github.com/gitleaks/gitleaks/releases"+ )+ semgrep_hint = "Optional; install separately with: uv tool install semgrep"hints = {- "gitleaks": (- "Install with: brew install gitleaks (macOS) or go install github.com/gitleaks/gitleaks/v8@latest"- ),+ "gitleaks": gitleaks_hint,"pip-audit": security_extra_hint,"safety": "Optional secondary scanner; install separately with: uv tool install safety","bandit": bundled_scanner_hint,- "semgrep": "Optional; install separately with: brew install semgrep (macOS) or uv tool install semgrep",+ "semgrep": semgrep_hint,"skillspector": f"Install with: uv tool install {SKILLSPECTOR_WHEEL_URL}","skillevaluator": ('Install with: uv tool install "skillevaluator[all] @ git+https://github.com/NVIDIA/SkillEvaluator.git"'diff --git a/tests/utils/test_tool_runner.py b/tests/utils/test_tool_runner.pyindex 0f5e2ce7..4dce3db2 100644--- a/tests/utils/test_tool_runner.py+++ b/tests/utils/test_tool_runner.py@@ -792,3 +792,25 @@ def test_version_parses_tool_output(tmp_path, monkeypatch):tool.chmod(0o755)monkeypatch.setenv("PATH", str(tmp_path))assert ExternalTool("Bandit", "bandit").version == "1.9.4"+++ ("platform", "gitleaks", "semgrep"),+ [+ ("win32", "winget install Gitleaks.Gitleaks", "uv tool install semgrep"),+ ("darwin", "brew install gitleaks", "brew install semgrep"),+ ("linux", "go install github.com/zricethezav/gitleaks/v8@latest", "uv tool install semgrep"),+ ],+)+def test_scanner_install_hints_match_the_host_platform(monkeypatch, platform, gitleaks, semgrep):+ gitleaks_tool = ExternalTool("Gitleaks", "gitleaks")+ semgrep_tool = ExternalTool("Semgrep", "semgrep")+ monkeypatch.setattr(tool_runner.sys, "platform", platform)++ gitleaks_hint = gitleaks_tool.get_install_hint()+ semgrep_hint = semgrep_tool.get_install_hint()++ assert gitleaks in gitleaks_hint+ assert semgrep in semgrep_hint+ if platform != "darwin":+ assert "brew" not in gitleaks_hint + semgrep_hint