Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(reporting): preserve local and CI skill identity (#489)
Narendran Raghavan5aacb5e1aca0
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 3cf33d99..c1ff3d40 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -122,6 +122,9 @@ All notable changes to SkillEvaluator are documented in this file.`gating.tiers` entry adds `validator_policies`, which says for each checkwhether it gates the exit code; the tier's `blocking` is `true` when any oneof them does. The keys are additive, and existing keys keep their shape.+ Local skill aliases such as `.` retain the folder name in `skills`, and+ native Tier 3 CI reports retain their single skill identity. Published CI+ JSON omits absolute host `target_path` values.- Import production trace exports into redacted review candidates with`tier3 import-traces`, then promote only explicitly human-reviewed expectationsdiff --git a/ci/run_tiers.py b/ci/run_tiers.pyindex cfb32d42..b45c3ba5 100644--- a/ci/run_tiers.py+++ b/ci/run_tiers.py@@ -276,7 +276,7 @@ def native_tier3(skill: Path, output: Path, env: dict) -> int:result.metadata["evaluated_source"] = options.evaluated_sourceif service.failure_reason(engine):result.mark_scan_incomplete("Tier 3 execution")- document = JSONReporter(expected_skill_name=skill.name).render_all([result])+ document = JSONReporter(expected_skill_name=skill.name, explicit_skill_name=skill.name).render_all([result])(output / "skillevaluator-tier3.json").write_text(document, encoding="utf-8")return int(resolve_run_status([result]).headline.value != "passed")diff --git a/ci/secret_redaction.py b/ci/secret_redaction.pyindex d78cea02..3b345762 100644--- a/ci/secret_redaction.py+++ b/ci/secret_redaction.py@@ -10,7 +10,7 @@ import jsonimport osimport refrom collections.abc import Iterable-from pathlib import Path+from pathlib import Path, PurePosixPath, PureWindowsPathfrom skillevaluator.reporting import HTMLReporter, MarkdownReporter, SARIFReporterfrom skillevaluator.reporting.finding_summary import refresh_finding_identities@@ -163,7 +163,13 @@ def sanitized_outputs(raw: Path, tokens: set[str], *, target_path: str | None =document = redactor.object(json.loads(text))if isinstance(document, dict):if target_path is not None and isinstance(document.get("target_path"), str):- document["target_path"] = target_path+ document["target_path"] = redactor.text(target_path)+ target = document.get("target_path")+ if isinstance(target, str) and (+ PurePosixPath(target).is_absolute() or PureWindowsPath(target).is_absolute()+ ):+ # Published CI evidence must not disclose runner paths.+ document["target_path"] = None# IDs hashed from the unredacted text could confirm a guessed# value. Refresh even without collected tokens: the redactor# also masks database userinfo that no token names.diff --git a/src/skillevaluator/reporting/json_reporter.py b/src/skillevaluator/reporting/json_reporter.pyindex bfd0adbc..081c9be6 100644--- a/src/skillevaluator/reporting/json_reporter.py+++ b/src/skillevaluator/reporting/json_reporter.py@@ -16,6 +16,7 @@ from __future__ import annotationsimport jsonfrom datetime import UTC, datetime+from pathlib import Pathfrom typing import TYPE_CHECKING, Anyfrom skillevaluator import __version__@@ -153,8 +154,15 @@ class JSONReporter(ReporterBase):# The explicit name keeps a single-skill run to one ``skills`` entry# when no finding path or quality score names the skill, and the# target path is the same fallback the HTML report uses.+ identity_target = self.target_path+ if identity_target and "://" not in identity_target:+ local_target = Path(identity_target)+ if (local_target / "SKILL.md").exists() or (local_target / "skill.md").exists():+ # Preserve the requested target in JSON while resolving aliases+ # such as "." for the shared single-skill identity fallback.+ identity_target = str(local_target.resolve())html_reporter = HTMLReporter(- target_path=self.target_path,+ target_path=identity_target,explicit_skill_name=self.explicit_skill_name,expected_skill_name=self.expected_skill_name,)diff --git a/tests/ci/test_run_tiers.py b/tests/ci/test_run_tiers.pyindex ba504554..03730f05 100644--- a/tests/ci/test_run_tiers.py+++ b/tests/ci/test_run_tiers.py@@ -150,6 +150,7 @@ def test_native_tier3_engine_failure_still_emits_incomplete_evidence(consumer, t== 0)report = json.loads((tmp_path / "skillevaluator-tier3.json").read_text())+ assert [entry["name"] for entry in report["skills"]] == ["alpha"]assert report["results"][0]["status"] == "incomplete"assert report["gating"]["tiers"]["3"]["blocking"] is Falseassert report["results"][0]["gating"] == {"tier": 3, "blocking": False}diff --git a/tests/ci/test_secret_redaction.py b/tests/ci/test_secret_redaction.pyindex 7cbedb81..2820a71e 100644--- a/tests/ci/test_secret_redaction.py+++ b/tests/ci/test_secret_redaction.py@@ -868,3 +868,60 @@ def test_invisible_character_in_credential_never_reaches_reports_or_pages(tmp_paif path.is_file():_assert_no_fragment(path.read_bytes(), value)assert (site / "runs/123/1/skill-001/skillevaluator-tier1.html").is_file()+++ ("target", "path_override", "published"),+ [+ ("/home/runner/work/private/skills/demo", None, None),+ (r"C:\runner\work\private\skills\demo", None, None),+ ("C:/runner/work/private/skills/demo", None, None),+ (r"\\server\private\skills\demo", None, None),+ ("skills/demo", None, "skills/demo"),+ (".", None, "."),+ ("https://github.com/example/skills/tree/main/demo", None, "https://github.com/example/skills/tree/main/demo"),+ (None, None, None),+ ("/home/runner/work/private/skills/demo", "skills/demo", "skills/demo"),+ ("skills/demo", "skills/public-demo", "skills/public-demo"),+ ("/home/runner/work/private/skills/demo", "/private/skills/demo", None),+ ],+)+def test_published_json_omits_absolute_host_target_without_changing_evidence(+ tmp_path, target, path_override, published+):+ result = ValidationResult(validator_name="Schema", validator_description="Check skill manifest")+ result.add_success("manifest_exists", "Found SKILL.md")+ result.metadata["gating"] = {"tier": 1, "blocking": True}+ raw_document = json.loads(+ JSONReporter(include_timestamp=False, target_path=target, explicit_skill_name="demo").render_all([result])+ )+ raw = tmp_path / "raw"+ raw.mkdir()+ (raw / "skillevaluator-tier1.json").write_text(json.dumps(raw_document), encoding="utf-8")++ outputs = runner.sanitized_outputs(raw, set(), target_path=path_override)+ document = json.loads(outputs["skillevaluator-tier1.json"])++ assert raw_document["target_path"] == target+ assert document["target_path"] == published+ for field in ("skills", "gating", "version", "results"):+ assert document[field] == raw_document[field]+++def test_published_json_redacts_known_credentials_in_repository_relative_target_override(tmp_path):+ token = "example-secret-0123456789"+ result = ValidationResult(validator_name="Schema", validator_description="Check skill manifest")+ result.add_success("manifest_exists", "Found SKILL.md")+ raw_document = JSONReporter(target_path="/runner/private/skills/demo", explicit_skill_name="demo").render_all(+ [result]+ )+ raw = tmp_path / "raw"+ raw.mkdir()+ (raw / "skillevaluator-tier1.json").write_text(raw_document, encoding="utf-8")++ outputs = runner.sanitized_outputs(raw, {token}, target_path=f"skills/{token}/demo")+ document = json.loads(outputs["skillevaluator-tier1.json"])++ assert document["target_path"] == "skills/[REDACTED]/demo"+ assert token.encode() not in outputs["skillevaluator-tier1.json"]+ assert (raw / "skillevaluator-tier1.json").read_text(encoding="utf-8") == raw_documentdiff --git a/tests/reporting/test_reporters.py b/tests/reporting/test_reporters.pyindex db042b3f..fb5d440d 100644--- a/tests/reporting/test_reporters.py+++ b/tests/reporting/test_reporters.py@@ -545,6 +545,7 @@ class TestJSONReporter:("working_directory", "target"),[("simple", "."),+ ("simple", "./"),("simple/nested", ".."),(".", "simple"),(".", "./simple"),