Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(reporting): preserve local and CI skill identity (#489)

Narendran Raghavan5aacb5e1aca0
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3cf33d99..c1ff3d40 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -122,6 +122,9 @@ All notable changes to SkillEvaluator are documented in this file.
`gating.tiers` entry adds `validator_policies`, which says for each check
whether it gates the exit code; the tier's `blocking` is `true` when any one
of them does. The keys are additive, and existing keys keep their shape.
+ Local skill aliases such as `.` retain the folder name in `skills`, and
+ native Tier 3 CI reports retain their single skill identity. Published CI
+ JSON omits absolute host `target_path` values.
- Import production trace exports into redacted review candidates with
`tier3 import-traces`, then promote only explicitly human-reviewed expectations
diff --git a/ci/run_tiers.py b/ci/run_tiers.py
index cfb32d42..b45c3ba5 100644
--- a/ci/run_tiers.py
+++ b/ci/run_tiers.py
@@ -276,7 +276,7 @@ def native_tier3(skill: Path, output: Path, env: dict) -> int:
result.metadata["evaluated_source"] = options.evaluated_source
if service.failure_reason(engine):
result.mark_scan_incomplete("Tier 3 execution")
- document = JSONReporter(expected_skill_name=skill.name).render_all([result])
+ document = JSONReporter(expected_skill_name=skill.name, explicit_skill_name=skill.name).render_all([result])
(output / "skillevaluator-tier3.json").write_text(document, encoding="utf-8")
return int(resolve_run_status([result]).headline.value != "passed")
diff --git a/ci/secret_redaction.py b/ci/secret_redaction.py
index d78cea02..3b345762 100644
--- a/ci/secret_redaction.py
+++ b/ci/secret_redaction.py
@@ -10,7 +10,7 @@ import json
import os
import re
from collections.abc import Iterable
-from pathlib import Path
+from pathlib import Path, PurePosixPath, PureWindowsPath
from skillevaluator.reporting import HTMLReporter, MarkdownReporter, SARIFReporter
from skillevaluator.reporting.finding_summary import refresh_finding_identities
@@ -163,7 +163,13 @@ def sanitized_outputs(raw: Path, tokens: set[str], *, target_path: str | None =
document = redactor.object(json.loads(text))
if isinstance(document, dict):
if target_path is not None and isinstance(document.get("target_path"), str):
- document["target_path"] = target_path
+ document["target_path"] = redactor.text(target_path)
+ target = document.get("target_path")
+ if isinstance(target, str) and (
+ PurePosixPath(target).is_absolute() or PureWindowsPath(target).is_absolute()
+ ):
+ # Published CI evidence must not disclose runner paths.
+ document["target_path"] = None
# IDs hashed from the unredacted text could confirm a guessed
# value. Refresh even without collected tokens: the redactor
# also masks database userinfo that no token names.
diff --git a/src/skillevaluator/reporting/json_reporter.py b/src/skillevaluator/reporting/json_reporter.py
index bfd0adbc..081c9be6 100644
--- a/src/skillevaluator/reporting/json_reporter.py
+++ b/src/skillevaluator/reporting/json_reporter.py
@@ -16,6 +16,7 @@ from __future__ import annotations
import json
from datetime import UTC, datetime
+from pathlib import Path
from typing import TYPE_CHECKING, Any
from skillevaluator import __version__
@@ -153,8 +154,15 @@ class JSONReporter(ReporterBase):
# The explicit name keeps a single-skill run to one ``skills`` entry
# when no finding path or quality score names the skill, and the
# target path is the same fallback the HTML report uses.
+ identity_target = self.target_path
+ if identity_target and "://" not in identity_target:
+ local_target = Path(identity_target)
+ if (local_target / "SKILL.md").exists() or (local_target / "skill.md").exists():
+ # Preserve the requested target in JSON while resolving aliases
+ # such as "." for the shared single-skill identity fallback.
+ identity_target = str(local_target.resolve())
html_reporter = HTMLReporter(
- target_path=self.target_path,
+ target_path=identity_target,
explicit_skill_name=self.explicit_skill_name,
expected_skill_name=self.expected_skill_name,
)
diff --git a/tests/ci/test_run_tiers.py b/tests/ci/test_run_tiers.py
index ba504554..03730f05 100644
--- a/tests/ci/test_run_tiers.py
+++ b/tests/ci/test_run_tiers.py
@@ -150,6 +150,7 @@ def test_native_tier3_engine_failure_still_emits_incomplete_evidence(consumer, t
== 0
)
report = json.loads((tmp_path / "skillevaluator-tier3.json").read_text())
+ assert [entry["name"] for entry in report["skills"]] == ["alpha"]
assert report["results"][0]["status"] == "incomplete"
assert report["gating"]["tiers"]["3"]["blocking"] is False
assert report["results"][0]["gating"] == {"tier": 3, "blocking": False}
diff --git a/tests/ci/test_secret_redaction.py b/tests/ci/test_secret_redaction.py
index 7cbedb81..2820a71e 100644
--- a/tests/ci/test_secret_redaction.py
+++ b/tests/ci/test_secret_redaction.py
@@ -868,3 +868,60 @@ def test_invisible_character_in_credential_never_reaches_reports_or_pages(tmp_pa
if path.is_file():
_assert_no_fragment(path.read_bytes(), value)
assert (site / "runs/123/1/skill-001/skillevaluator-tier1.html").is_file()
+
+
+ ("target", "path_override", "published"),
+ [
+ ("/home/runner/work/private/skills/demo", None, None),
+ (r"C:\runner\work\private\skills\demo", None, None),
+ ("C:/runner/work/private/skills/demo", None, None),
+ (r"\\server\private\skills\demo", None, None),
+ ("skills/demo", None, "skills/demo"),
+ (".", None, "."),
+ ("https://github.com/example/skills/tree/main/demo", None, "https://github.com/example/skills/tree/main/demo"),
+ (None, None, None),
+ ("/home/runner/work/private/skills/demo", "skills/demo", "skills/demo"),
+ ("skills/demo", "skills/public-demo", "skills/public-demo"),
+ ("/home/runner/work/private/skills/demo", "/private/skills/demo", None),
+ ],
+)
+def test_published_json_omits_absolute_host_target_without_changing_evidence(
+ tmp_path, target, path_override, published
+):
+ result = ValidationResult(validator_name="Schema", validator_description="Check skill manifest")
+ result.add_success("manifest_exists", "Found SKILL.md")
+ result.metadata["gating"] = {"tier": 1, "blocking": True}
+ raw_document = json.loads(
+ JSONReporter(include_timestamp=False, target_path=target, explicit_skill_name="demo").render_all([result])
+ )
+ raw = tmp_path / "raw"
+ raw.mkdir()
+ (raw / "skillevaluator-tier1.json").write_text(json.dumps(raw_document), encoding="utf-8")
+
+ outputs = runner.sanitized_outputs(raw, set(), target_path=path_override)
+ document = json.loads(outputs["skillevaluator-tier1.json"])
+
+ assert raw_document["target_path"] == target
+ assert document["target_path"] == published
+ for field in ("skills", "gating", "version", "results"):
+ assert document[field] == raw_document[field]
+
+
+def test_published_json_redacts_known_credentials_in_repository_relative_target_override(tmp_path):
+ token = "example-secret-0123456789"
+ result = ValidationResult(validator_name="Schema", validator_description="Check skill manifest")
+ result.add_success("manifest_exists", "Found SKILL.md")
+ raw_document = JSONReporter(target_path="/runner/private/skills/demo", explicit_skill_name="demo").render_all(
+ [result]
+ )
+ raw = tmp_path / "raw"
+ raw.mkdir()
+ (raw / "skillevaluator-tier1.json").write_text(raw_document, encoding="utf-8")
+
+ outputs = runner.sanitized_outputs(raw, {token}, target_path=f"skills/{token}/demo")
+ document = json.loads(outputs["skillevaluator-tier1.json"])
+
+ assert document["target_path"] == "skills/[REDACTED]/demo"
+ assert token.encode() not in outputs["skillevaluator-tier1.json"]
+ assert (raw / "skillevaluator-tier1.json").read_text(encoding="utf-8") == raw_document
diff --git a/tests/reporting/test_reporters.py b/tests/reporting/test_reporters.py
index db042b3f..fb5d440d 100644
--- a/tests/reporting/test_reporters.py
+++ b/tests/reporting/test_reporters.py
@@ -545,6 +545,7 @@ class TestJSONReporter:
("working_directory", "target"),
[
("simple", "."),
+ ("simple", "./"),
("simple/nested", ".."),
(".", "simple"),
(".", "./simple"),