Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
feat(cli): write catalog-summary.json after catalog validate (#121)
Mohammed Imran Khan2af26f52eed0
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 0fc1c927..d0c1dfed 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -6,6 +6,9 @@ All notable changes to SkillEvaluator are documented in this file.### Added+- Catalog validation now writes `catalog-summary.json` at the reports root with+ per-skill pass/fail status, optional severity rollups from child JSON reports,+ and paths to per-skill report directories.- `SKILL_EVAL_MODEL_CATALOG_ALLOW_HTTP_HOSTS` names hosts whose model catalog maybe read over plain HTTP. Catalog reads still require HTTPS for every othernon-loopback host. Entries match one whole host as written, with no namediff --git a/src/skillevaluator/cli.py b/src/skillevaluator/cli.pyindex 8aa423ab..04660976 100644--- a/src/skillevaluator/cli.py+++ b/src/skillevaluator/cli.py@@ -6,8 +6,11 @@from __future__ import annotationsimport copy+import jsonimport loggingimport math+from contextvars import ContextVar+from datetime import UTC, datetimefrom pathlib import Pathimport click@@ -325,6 +328,29 @@ def _report_options(func):)(func)+_validate_json_report_var: ContextVar[str | None] = ContextVar(+ "_validate_json_report_var",+ default=None,+)+++def _effective_report_formats(report_formats: tuple[str, ...], *, quiet: bool) -> tuple[str, ...]:+ """Resolve the report formats ``validate`` will actually emit."""+ if quiet and not _report_formats_explicit():+ return tuple(dict.fromkeys([fmt for fmt in report_formats if fmt != "cli"] + ["html", "json"]))+ return report_formats+++def _record_validate_json_report(report_name: str | None) -> None:+ _validate_json_report_var.set(report_name)+++def _consume_validate_json_report() -> str | None:+ report_name = _validate_json_report_var.get()+ _validate_json_report_var.set(None)+ return report_name++def _report_formats_explicit() -> bool:"""True when the user passed ``-r``/``--report`` on the command line.@@ -692,6 +718,88 @@ def _print_catalog_summary(total: int, failures: list[tuple[str, str]], reports_console_.print(Text.assemble((" reports ", MUTED), (f"{reports_root}/<skill>/", MUTED)))+CATALOG_SUMMARY_FILENAME = "catalog-summary.json"+++def _catalog_skill_entry(+ skill_name: str,+ skill_report_dir: Path,+ *,+ passed: bool,+ reason: str,+ json_report_name: str | None = None,+) -> dict[str, object]:+ entry: dict[str, object] = {+ "name": skill_name,+ "passed": passed,+ "report_dir": skill_name,+ }+ if not passed:+ entry["reason"] = reason++ if json_report_name:+ json_report = skill_report_dir / json_report_name+ else:+ return entry++ if not json_report.is_file():+ return entry++ entry["json_report"] = json_report.name+ try:+ payload = json.loads(json_report.read_text(encoding="utf-8"))+ except (json.JSONDecodeError, OSError):+ return entry+ if not isinstance(payload, dict):+ return entry++ for key in ("overall_passed", "overall_status", "incomplete_scans", "severity_counts"):+ if key in payload:+ entry[key] = payload[key]+ return entry+++def _write_catalog_summary(output_dir: Path, skills: list[dict[str, object]]) -> Path:+ """Write a machine-readable fleet rollup for catalog validation."""+ from skillevaluator.reporting.base import _write_report_atomically++ total = len(skills)+ passed = sum(1 for skill in skills if skill.get("passed"))+ failed = total - passed+ severity_totals = {+ "critical": 0,+ "high": 0,+ "medium": 0,+ "low": 0,+ }+ for skill in skills:+ counts = skill.get("severity_counts")+ if not isinstance(counts, dict):+ continue+ for key in severity_totals:+ value = counts.get(key)+ if isinstance(value, int):+ severity_totals[key] += value++ summary: dict[str, object] = {+ "total": total,+ "passed": passed,+ "failed": failed,+ "overall_passed": failed == 0,+ "reports_root": output_dir.name,+ "summary_path": CATALOG_SUMMARY_FILENAME,+ "severity_totals": severity_totals,+ "skills": skills,+ "generated_at": datetime.now(tz=UTC).isoformat(),+ }+ output_path = output_dir / CATALOG_SUMMARY_FILENAME+ _write_report_atomically(+ output_path,+ json.dumps(summary, indent=2, default=str, allow_nan=False).encode("utf-8"),+ )+ return output_path++def _validate_catalog(ctx: click.Context,*,@@ -712,13 +820,15 @@ def _validate_catalog(skill_dirs = sorted(marker.parent for marker in resolved_target.glob("*/SKILL.md"))failures: list[tuple[str, str]] = []+ skill_reports: dict[str, str | None] = {}for index, skill_dir in enumerate(skill_dirs, start=1):_print_catalog_divider(index, len(skill_dirs), skill_dir.name)+ skill_output = output_dir / skill_dir.nameoverrides = {**ctx.params,"target_path": skill_dir,"content_type": "skill",- "output_dir": output_dir / skill_dir.name,+ "output_dir": skill_output,}try:ctx.invoke(validate, **overrides)@@ -726,6 +836,19 @@ def _validate_catalog(failures.append((skill_dir.name, str(getattr(exc, "message", exc))))except Exception as exc: # unexpected: keep the catalog running, report it on the scoreboardfailures.append((skill_dir.name, f"unexpected error: {exc}"))+ skill_reports[skill_dir.name] = _consume_validate_json_report()+ failure_map = dict(failures)+ skill_entries = [+ _catalog_skill_entry(+ skill_dir.name,+ output_dir / skill_dir.name,+ passed=skill_dir.name not in failure_map,+ reason=failure_map.get(skill_dir.name, ""),+ json_report_name=skill_reports.get(skill_dir.name),+ )+ for skill_dir in skill_dirs+ ]+ _write_catalog_summary(output_dir, skill_entries)_print_catalog_summary(len(skill_dirs), failures, output_dir)if failures:raise click.ClickException(@@ -1165,6 +1288,7 @@ def validate(validated against its public contract. Quality/lint/version checks areskill-only and skipped for plugins."""+ _record_validate_json_report(None)if dedup:_reject_linked_tier2_root(target_path)target_path = target_path.resolve()@@ -1452,10 +1576,7 @@ def validate(# the summary; the files carry the findings) and points at them from the# footer. An EXPLICIT -r is a contract and is honored exactly — including# "cli", which renders the full Rich report below the pipeline view.- if quiet and not _report_formats_explicit():- effective_formats = tuple(dict.fromkeys([f for f in report_formats if f != "cli"] + ["html", "json"]))- else:- effective_formats = report_formats+ effective_formats = _effective_report_formats(report_formats, quiet=quiet)report_basename_value = make_timestamped_basename(f"{REPORT_PREFIX}-output")emit_reports(results,@@ -1469,6 +1590,9 @@ def validate(sarif_scan_root=target_path,sarif_repository_root=sarif_repository_root,)+ _record_validate_json_report(+ f"{report_basename_value}.json" if "json" in effective_formats else None+ )# BENCHMARK.md is generated compulsorily for skills (matches SkillEvaluator), even on# failure, so the publication card always reflects the latest evaluation --diff --git a/tests/test_commands.py b/tests/test_commands.pyindex a91abe09..56e812b7 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -3,13 +3,17 @@from __future__ import annotations+import contextvarsimport jsonimport shutil+import threading+from collections.abc import Callablefrom pathlib import Pathimport pytestfrom click.testing import CliRunner+from skillevaluator import cli as cli_modulefrom skillevaluator.cli import clifrom skillevaluator.tier3.commands import parse_agent_model_overrides, parse_agents@@ -349,6 +353,258 @@ def test_validate_catalog_runs_each_skill_as_separate_job() -> None:assert result.exit_code == 0, result.outputassert any(Path("out/simple").glob("*.html"))assert any(Path("out/simple2").glob("*.html"))+ summary_path = Path("out/catalog-summary.json")+ assert summary_path.is_file()+ summary = json.loads(summary_path.read_text(encoding="utf-8"))+ assert summary["total"] == 2+ assert summary["passed"] == 2+ assert summary["overall_passed"] is True+ assert len(summary["skills"]) == 2+++def test_catalog_skill_entry_skips_stale_json_without_report_name(tmp_path: Path) -> None:+ skill_dir = tmp_path / "simple"+ skill_dir.mkdir()+ stale = skill_dir / "skillevaluator-output-19990101T000000.json"+ stale.write_text(+ json.dumps({"overall_passed": True, "severity_counts": {"high": 7}}),+ encoding="utf-8",+ )+ entry = cli_module._catalog_skill_entry(+ "simple",+ skill_dir,+ passed=False,+ reason="validation failed",+ json_report_name=None,+ )+ assert "overall_passed" not in entry+ assert "severity_counts" not in entry+++def test_validate_records_json_report_name_for_catalog_binding(monkeypatch) -> None:+ from skillevaluator import cli as cli_module++ monkeypatch.setattr(cli_module, "run_validation", lambda *_args, **_kwargs: [])+ monkeypatch.setattr(cli_module, "emit_reports", lambda *_args, **_kwargs: True)++ runner = CliRunner()+ with runner.isolated_filesystem():+ result = runner.invoke(+ cli,+ ["validate", str(FIXTURE), "--no-llm", "--no-tier2", "--checks", "schema", "-o", "out"],+ )++ assert result.exit_code == 0, result.output+ assert cli_module._consume_validate_json_report() is not None+++def _handoff_overlap_results(+ *,+ record_alpha: str,+ record_beta: str,+ run_in_context: bool,+) -> dict[str, str | None]:+ """Record two handoffs, then consume both after a barrier (overlap window)."""+ alpha_ctx = contextvars.Context()+ beta_ctx = contextvars.Context()+ barrier = threading.Barrier(2)+ results: dict[str, str | None] = {}+ thread_errors: list[BaseException] = []++ def alpha_lane() -> None:+ cli_module._record_validate_json_report(record_alpha)+ barrier.wait()+ results["alpha"] = cli_module._consume_validate_json_report()++ def beta_lane() -> None:+ cli_module._record_validate_json_report(record_beta)+ barrier.wait()+ results["beta"] = cli_module._consume_validate_json_report()++ def run_lane(ctx: contextvars.Context, lane: Callable[[], None]) -> None:+ try:+ if run_in_context:+ ctx.run(lane)+ else:+ lane()+ except BaseException as exc:+ thread_errors.append(exc)++ threads = (+ threading.Thread(target=run_lane, args=(alpha_ctx, alpha_lane)),+ threading.Thread(target=run_lane, args=(beta_ctx, beta_lane)),+ )+ for thread in threads:+ thread.start()+ for thread in threads:+ thread.join()+ if thread_errors:+ raise thread_errors[0]+ return results+++def test_validate_json_report_handoff_isolated_between_invocations() -> None:+ """Distinct Context instances keep each basename when record/consume overlap."""+ results = _handoff_overlap_results(+ record_alpha="skillevaluator-output-alpha.json",+ record_beta="skillevaluator-output-beta.json",+ run_in_context=True,+ )+ assert results["alpha"] == "skillevaluator-output-alpha.json"+ assert results["beta"] == "skillevaluator-output-beta.json"+++def test_validate_json_report_handoff_module_global_leaks_under_overlap(+ monkeypatch,+) -> None:+ """A single module-global slot returns the last writer after overlapping records."""+ slot: str | None = None++ def _record(report_name: str | None) -> None:+ nonlocal slot+ slot = report_name++ def _consume() -> str | None:+ nonlocal slot+ report_name = slot+ slot = None+ return report_name++ monkeypatch.setattr(cli_module, "_record_validate_json_report", _record)+ monkeypatch.setattr(cli_module, "_consume_validate_json_report", _consume)++ results = _handoff_overlap_results(+ record_alpha="skillevaluator-output-alpha.json",+ record_beta="skillevaluator-output-beta.json",+ run_in_context=False,+ )+ assert results != {+ "alpha": "skillevaluator-output-alpha.json",+ "beta": "skillevaluator-output-beta.json",+ }+++def test_catalog_summary_binds_exact_json_report_not_sarif_sidecar(monkeypatch) -> None:+ from skillevaluator.models.result import ValidationResult++ def _emit(results, *, report_formats, output_dir, basename, **_kwargs):+ output_dir.mkdir(parents=True, exist_ok=True)+ standard = output_dir / f"{basename}.json"+ sarif = output_dir / f"{basename}.sarif.json"+ standard.write_text(+ json.dumps({"overall_status": "pass", "severity_counts": {"high": 2}}),+ encoding="utf-8",+ )+ sarif.write_text("{}", encoding="utf-8")+ return all(result.passed for result in results)++ monkeypatch.setattr(cli_module, "run_validation", lambda *_args, **_kwargs: [ValidationResult(validator_name="Schema")])+ monkeypatch.setattr(cli_module, "emit_reports", _emit)++ runner = CliRunner()+ with runner.isolated_filesystem():+ catalog = Path("catalog")+ shutil.copytree(FIXTURE, catalog / "simple")+ result = runner.invoke(+ cli,+ [+ "validate",+ str(catalog.resolve()),+ "--no-llm",+ "--no-dedup",+ "--checks",+ "schema",+ "-r",+ "json",+ "-r",+ "sarif",+ "-o",+ "out",+ ],+ )++ assert result.exit_code == 0, result.output+ summary = json.loads(Path("out/catalog-summary.json").read_text(encoding="utf-8"))+ skill = summary["skills"][0]+ assert skill["json_report"].endswith(".json")+ assert not skill["json_report"].endswith(".sarif.json")+ assert skill["severity_counts"] == {"high": 2}+++def test_catalog_summary_same_second_rerun_overwrites_json(monkeypatch) -> None:+ from skillevaluator.models.result import Finding, Severity, ValidationResult++ failing = ValidationResult(validator_name="Schema")+ failing.add_structured_finding(+ Finding(+ category="SCHEMA",+ severity=Severity.HIGH,+ check_name="author_missing",+ message="missing author",+ file_path="SKILL.md",+ ),+ is_error=True,+ )+ outcomes = [[ValidationResult(validator_name="Schema")], [failing]]++ def _run_validation(_target: Path, **_kwargs):+ return outcomes.pop(0)++ def _emit(results, *, report_formats, output_dir, basename, **_kwargs):+ output_dir.mkdir(parents=True, exist_ok=True)+ report = output_dir / f"{basename}.json"+ report.write_text(+ json.dumps(+ {+ "overall_passed": all(result.passed for result in results),+ "overall_status": "pass" if all(result.passed for result in results) else "fail",+ "severity_counts": {"high": 0 if all(result.passed for result in results) else 3},+ }+ ),+ encoding="utf-8",+ )+ return all(result.passed for result in results)++ monkeypatch.setattr(cli_module, "run_validation", _run_validation)+ monkeypatch.setattr(cli_module, "emit_reports", _emit)+ monkeypatch.setattr(+ "skillevaluator.utils.helpers.make_timestamped_basename",+ lambda _prefix: "skillevaluator-output-20260908T000000",+ )++ runner = CliRunner()+ with runner.isolated_filesystem():+ catalog = Path("catalog")+ shutil.copytree(FIXTURE, catalog / "simple")+ first = runner.invoke(+ cli,+ ["validate", str(catalog.resolve()), "--no-llm", "--no-dedup", "--checks", "schema", "-r", "json", "-o", "out"],+ )+ second = runner.invoke(+ cli,+ ["validate", str(catalog.resolve()), "--no-llm", "--no-dedup", "--checks", "schema", "-r", "json", "-o", "out"],+ )++ assert first.exit_code == 0, first.output+ assert second.exit_code != 0, second.output+ summary = json.loads(Path("out/catalog-summary.json").read_text(encoding="utf-8"))+ skill = summary["skills"][0]+ assert skill["json_report"] == "skillevaluator-output-20260908T000000.json"+ assert skill["overall_status"] == "fail"+ assert skill["severity_counts"] == {"high": 3}+++def test_write_catalog_summary_rejects_symlinked_parent(tmp_path: Path) -> None:+ external = tmp_path / "external"+ external.mkdir()+ reports = tmp_path / "reports"+ reports.symlink_to(external, target_is_directory=True)+ output_dir = reports / "new" / "nested"++ with pytest.raises(ValueError, match=r"symlink|reparse"):+ cli_module._write_catalog_summary(output_dir, [])++ assert not (external / "new").exists()def test_validate_catalog_rejects_one_previous_version_for_every_skill() -> None:@@ -749,11 +1005,15 @@ def test_validate_catalog_survives_failing_skills(monkeypatch) -> None:cli, ["validate", str(catalog.resolve()), "--no-llm", "--no-dedup", "--checks", "schema", "-o", "out"])- out = _plain_text(result.output)- assert "skill 1/2" in out and "skill 2/2" in out- assert "Catalog Result" in out- assert "0/2 skills passed" in out- assert result.exit_code != 0+ out = _plain_text(result.output)+ assert "skill 1/2" in out and "skill 2/2" in out+ assert "Catalog Result" in out+ assert "0/2 skills passed" in out+ assert result.exit_code != 0+ summary = json.loads(Path("out/catalog-summary.json").read_text(encoding="utf-8"))+ assert summary["failed"] == 2+ assert summary["overall_passed"] is False+ assert all(not skill["passed"] for skill in summary["skills"])def test_render_evaluation_result_invokes_findings_report(monkeypatch) -> None:@@ -893,10 +1153,13 @@ def test_validate_catalog_reports_unexpected_errors(monkeypatch) -> None:cli, ["validate", str(catalog.resolve()), "--no-llm", "--no-dedup", "--checks", "schema", "-o", "out"])- out = _plain_text(result.output)- assert "Catalog Result" in out- assert "unexpected error: validator exploded" in out- assert result.exit_code != 0+ out = _plain_text(result.output)+ assert "Catalog Result" in out+ assert "unexpected error: validator exploded" in out+ assert result.exit_code != 0+ summary = json.loads(Path("out/catalog-summary.json").read_text(encoding="utf-8"))+ assert summary["failed"] == 2+ assert summary["skills"][0]["reason"] == "unexpected error: validator exploded"def test_summarize_tier2_empty_results_name_a_reason() -> None: