Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(reporting): name the skill when the target is the current directory (#480)

Christopher Kevineab725cb0b74
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b1bab71a..058f8048 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -643,6 +643,14 @@ All notable changes to SkillEvaluator are documented in this file.
instead of `Evaluator version: not recorded (legacy or non-live result)`; a
version recorded by a Tier 3 run still wins.
+- `skillevaluator tier1 .` and `tier2 .`, run from inside the skill folder, name
+ the skill in the JSON `skills` and `contributors` lists, as `tier1 <folder>`
+ does. The report took the name from the path as typed, which is empty for
+ `.`, so a clean run with a narrowed check list, such as `--checks pii`,
+ listed no skill or one skill per check ID. The JSON that CI retains records
+ the skill's path relative to the repository root as `target_path`, no longer
+ the absolute path of the runner's workspace.
+
- Tier 3 improvement recommendations respect each case's expected skill routing:
passing negative controls no longer inherit activation-loss advice from positive
cases, while failed negatives retain task, environment, and run-error diagnostics.
@@ -1078,7 +1086,6 @@ All notable changes to SkillEvaluator are documented in this file.
with a `destructive_command` finding, already gets FAIL from that check (see
Changed).
-
## 0.8.0 - 2026-11-05
### Added
diff --git a/ci/run_tier1.py b/ci/run_tier1.py
index 10bf833a..9fff145c 100644
--- a/ci/run_tier1.py
+++ b/ci/run_tier1.py
@@ -470,7 +470,7 @@ def evaluate_skill(
process = execute(
argv, cwd=workspace, env=env, stdout=stdout, stderr=stderr, check=False, timeout=timeout
)
- safe = sanitized_outputs(raw, tokens)
+ safe = sanitized_outputs(raw, tokens, target_path=target.relative_to(workspace).as_posix())
for name, content in safe.items():
(output / name).write_bytes(content)
fields["exit_code"] = process.returncode
diff --git a/ci/run_tiers.py b/ci/run_tiers.py
index 4e6f6dc6..b99b543e 100644
--- a/ci/run_tiers.py
+++ b/ci/run_tiers.py
@@ -317,7 +317,9 @@ def _combined(statuses: list[str]) -> str:
)
-def _retain(raw: Path, destination: Path, tier: int, tokens: set[str], skill: Path, code: int) -> dict:
+def _retain(
+ raw: Path, destination: Path, tier: int, tokens: set[str], skill: Path, code: int, target_path: str | None = None
+) -> dict:
from skillevaluator.reporting.retained import retained_results
try:
@@ -330,6 +332,10 @@ def _retain(raw: Path, destination: Path, tier: int, tokens: set[str], skill: Pa
if len(documents) != 1:
raise ConfigurationError("The evaluator did not produce one canonical JSON report.")
report = Redactor(tokens).object(read_json(documents[0]))
+ if target_path is not None and isinstance(report.get("target_path"), str):
+ # The evaluator records the target as the CI passed it, an absolute path
+ # in the runner's workspace; the retained report keeps the repository path.
+ report["target_path"] = target_path
results = retained_results(report, allowed_tiers={tier})
if tier == 2:
# Standalone Tier 2 gates its CLI exit; CI keeps the same evidence and
@@ -562,7 +568,7 @@ def evaluate(tier: int, workspace: Path, reports: Path, env: dict[str, str]) ->
)
if code == 124:
skill_row["error"] = "Evaluation exceeded the configured time budget."
- retained = _retain(raw, leaf, tier, tokens, skill, code)
+ retained = _retain(raw, leaf, tier, tokens, skill, code, skill_row["path"])
complete = retained.pop("complete")
skill_row.update(retained)
if tier == 3 and dataset_preflight(skill) != digests:
diff --git a/ci/secret_redaction.py b/ci/secret_redaction.py
index d5e0be39..d78cea02 100644
--- a/ci/secret_redaction.py
+++ b/ci/secret_redaction.py
@@ -105,8 +105,13 @@ def collect_sensitive_bytes(contents: Iterable[bytes]) -> set[str]:
return tokens
-def sanitized_outputs(raw: Path, tokens: set[str]) -> dict[str, bytes]:
- """Read and sanitize bounded outputs before writing any shared artifact."""
+def sanitized_outputs(raw: Path, tokens: set[str], *, target_path: str | None = None) -> dict[str, bytes]:
+ """Read and sanitize bounded outputs before writing any shared artifact.
+
+ The evaluator records the target as the CI passed it, an absolute path in
+ the runner's workspace. When *target_path* is given, the JSON keeps that
+ repository-relative path instead.
+ """
allowed = (
"skillevaluator-tier1.json",
"skillevaluator-tier1.html",
@@ -157,6 +162,8 @@ def sanitized_outputs(raw: Path, tokens: set[str]) -> dict[str, bytes]:
if name.endswith(".json"):
document = redactor.object(json.loads(text))
if isinstance(document, dict):
+ if target_path is not None and isinstance(document.get("target_path"), str):
+ document["target_path"] = target_path
# IDs hashed from the unredacted text could confirm a guessed
# value. Refresh even without collected tokens: the redactor
# also masks database userinfo that no token names.
diff --git a/docs/ci-integration.mdx b/docs/ci-integration.mdx
index c4d1a68d..3c69f57f 100644
--- a/docs/ci-integration.mdx
+++ b/docs/ci-integration.mdx
@@ -243,7 +243,7 @@ When the exit code is not enough — dashboards, custom thresholds, badge genera
| Field | Type | Contents |
| --- | --- | --- |
| `version` | string | SkillEvaluator version that wrote the report |
-| `target_path` | string or null | The validated target. `validate` records its absolute path, or its browsable URL when it is in a Git checkout with an `origin` remote. `tier1` and `tier2` record the path as given on the command line. The single-check commands such as `security-scan` give `null` |
+| `target_path` | string or null | The validated target. `validate` records its absolute path, or its browsable URL when it is in a Git checkout with an `origin` remote. `tier1` and `tier2` record the path as given on the command line; in a report that CI retains, the path is the skill's path relative to the repository root, never the runner's absolute path. The single-check commands such as `security-scan` give `null` |
| `overall_passed` | boolean | `true` when every check passed |
| `overall_status` | string | `passed`, `failed`, or `incomplete` (a required scanner produced no evidence) |
| `publication_status` | string | `pass`, `neutral`, `fail`, or `incomplete`; use this field for publication eligibility |
diff --git a/src/skillevaluator/reporting/html.py b/src/skillevaluator/reporting/html.py
index 49da6560..7e719e4a 100644
--- a/src/skillevaluator/reporting/html.py
+++ b/src/skillevaluator/reporting/html.py
@@ -2043,7 +2043,9 @@ class HTMLReporter(ReporterBase):
if self.target_path:
tp = Path(self.target_path)
if (tp / "SKILL.md").exists() or (tp / "skill.md").exists():
- return tp.name
+ # ``.`` and ``..`` (``tier1 .`` from inside the skill folder) have
+ # no folder name of their own; the folder they open does.
+ return (tp.resolve().name if tp.name in {"", ".", ".."} else tp.name) or None
return None
diff --git a/tests/ci/test_run_tier1.py b/tests/ci/test_run_tier1.py
index 37326cf4..22666673 100644
--- a/tests/ci/test_run_tier1.py
+++ b/tests/ci/test_run_tier1.py
@@ -539,6 +539,29 @@ def test_successful_rollup_preserves_all_counts(tmp_path):
assert summary["skills"][0]["path"] == "skills/a"
[email protected]("recorded", [True, False])
+def test_retained_report_records_the_repository_relative_target_not_the_runner_path(tmp_path, recorded):
+ workspace = tmp_path / "workspace"
+ skill(workspace, "skills/a")
+ reports = tmp_path / "reports"
+
+ def invoke(argv, **_kwargs):
+ # The evaluator records the target as the CI passed it: an absolute
+ # path in the runner's workspace. An older evaluator image records none.
+ report = {**payload(), "target_path": argv[argv.index("tier1") + 1]}
+ if not recorded:
+ report.pop("target_path")
+ output = Path(argv[argv.index("-o") + 1])
+ (output / "skillevaluator-tier1.json").write_text(json.dumps(report), encoding="utf-8")
+ return CompletedProcess(argv, 0)
+
+ assert runner.run(workspace, reports, {"SKILL_PATHS": "skills"}, execute=invoke) == 0
+
+ retained = json.loads((reports / "skill-001" / "skillevaluator-tier1.json").read_text())
+ assert retained.get("target_path") == ("skills/a" if recorded else None)
+ assert str(workspace) not in json.dumps(retained)
+
+
def test_invalid_json_is_reported_and_later_skills_still_run(tmp_path):
workspace = tmp_path / "workspace"
skill(workspace, "skills/a")
diff --git a/tests/ci/test_run_tiers.py b/tests/ci/test_run_tiers.py
index 031bd402..d61523cb 100644
--- a/tests/ci/test_run_tiers.py
+++ b/tests/ci/test_run_tiers.py
@@ -542,6 +542,33 @@ def test_skill_directories_with_url_delimiters_still_aggregate(tmp_path, monkeyp
assert sorted(s["path"] for s in tier2["skills"]) == sorted(f"skills/{name}" for name in names)
+def test_retained_reports_record_the_repository_relative_target_not_the_runner_path(consumer, tmp_path, monkeypatch):
+ from skillevaluator.models import ValidationResult
+ from skillevaluator.reporting.json_reporter import JSONReporter
+
+ def execute(args, **_kwargs):
+ if "catalog" in args:
+ return 0
+ result = ValidationResult(
+ validator_name="Context Deduplication", metadata={"gating": {"tier": 2, "blocking": False}}
+ )
+ # The evaluator records the target as the CI passed it, an absolute runner path.
+ document = JSONReporter(target_path=args[args.index("tier2") + 1]).render_all([result])
+ (Path(args[args.index("-o") + 1]) / "skillevaluator-tier2.json").write_text(document)
+ return 0
+
+ monkeypatch.setattr(run_tiers, "run_command", execute)
+ reports = tmp_path / "reports"
+ env = env_for(consumer, NVIDIA_API_KEY="test-provider-value-1234")
+ assert run_tiers.evaluate(2, consumer, reports, env) == 0
+
+ targets = {
+ child.name: json.loads((child / "skillevaluator-tier2.json").read_text())["target_path"]
+ for child in sorted((reports / "tier2").glob("skill-*"))
+ }
+ assert targets == {"skill-001": "skills/alpha", "skill-002": "skills/beta"}
+
+
def write_attempt(staging, tier, attempt, env, status):
"""Lay out one downloaded tier artifact as actions/download-artifact does for a pattern."""
artifact = staging / f"skill-evaluator-tier{tier}-attempt-{attempt}"
@@ -736,8 +763,9 @@ def test_retained_tier2_reports_apply_ci_advisory_policy(consumer, tmp_path, fai
env = env_for(consumer)
source = {"repository": "example/consumer", "commit": env["REPORT_COMMIT"]}
result.metadata["evaluated_source"] = source
- document = JSONReporter().render_all([result])
+ document = JSONReporter(target_path=str(skill)).render_all([result])
original = json.loads(document)
+ assert original["target_path"] == str(skill)
assert original["overall_passed"] is False
assert original["results"][0]["gating"] == {"tier": 2, "blocking": True}
assert original["gating"]["tiers"]["2"]["validator_policies"] == [
@@ -749,7 +777,7 @@ def test_retained_tier2_reports_apply_ci_advisory_policy(consumer, tmp_path, fai
reports = tmp_path / "reports"
destination = reports / "tier2/skill-001"
- outcome = run_tiers._retain(raw, destination, 2, set(), skill, 1)
+ outcome = run_tiers._retain(raw, destination, 2, set(), skill, 1, "skills/alpha")
assert outcome == {"status": status, "metrics": {"findings": int(failed)}, "complete": not incomplete}
retained = json.loads((destination / "skillevaluator-tier2.json").read_text())
@@ -763,7 +791,9 @@ def test_retained_tier2_reports_apply_ci_advisory_policy(consumer, tmp_path, fai
assert retained["results"][0]["status"] == status
assert retained["results"][0]["passed"] is False
assert retained["evaluated_source"] == source
- for key in set(original) - {"gating", "overall_passed", "overall_status", "results"}:
+ # The evaluator saw the runner's absolute path; the retained report keeps the repository path.
+ assert retained["target_path"] == "skills/alpha"
+ for key in set(original) - {"gating", "overall_passed", "overall_status", "results", "target_path"}:
assert retained[key] == original[key], key
for key in set(original["results"][0]) - {"gating"}:
assert retained["results"][0][key] == original["results"][0][key], key
diff --git a/tests/cli/test_tier_workflows.py b/tests/cli/test_tier_workflows.py
index 27517e9b..38c883cb 100644
--- a/tests/cli/test_tier_workflows.py
+++ b/tests/cli/test_tier_workflows.py
@@ -6,6 +6,7 @@
from __future__ import annotations
import json
+from pathlib import Path
import pytest
from click.testing import CliRunner
@@ -244,11 +245,16 @@ def test_tier2_without_catalog_does_not_manufacture_a_similarity_pass(skill, tmp
("tier", "validator"),
[("tier1", "Schema Validation"), ("tier2", "Context Deduplication")],
)
+ ("directory", "target"),
+ [("parent", "sample"), ("parent", "sample/"), ("skill", ".")],
+)
def test_direct_tier_json_names_the_clean_skill_and_records_the_target_as_given(
- tier, validator, skill, tmp_path, monkeypatch
+ tier, validator, directory, target, skill, tmp_path, monkeypatch
):
"""A clean result names no file and has no quality score, so the JSON
- names the skill from the target path, which records the path as typed."""
+ names the skill from the target path, which records the path as typed.
+ ``.`` has no folder name of its own, so the folder it opens names the skill."""
from skillevaluator.tier1 import commands as tier1_commands
from skillevaluator.tier2 import commands as tier2_commands
@@ -260,15 +266,15 @@ def test_direct_tier_json_names_the_clean_skill_and_records_the_target_as_given(
monkeypatch.setattr(tier2_commands, "run_context_optimization_check", lambda *_a, **_kw: _result(validator))
monkeypatch.setenv("SKILL_EVAL_LLM_PROVIDER", "nv_build")
monkeypatch.setenv("NVIDIA_API_KEY", "test-credential")
- monkeypatch.chdir(skill.parent)
+ monkeypatch.chdir({"parent": skill.parent, "skill": skill}[directory])
- result = CliRunner().invoke(cli, [tier, skill.name, *options, "-r", "json", "-o", str(tmp_path / "reports")])
+ result = CliRunner().invoke(cli, [tier, target, *options, "-r", "json", "-o", str(tmp_path / "reports")])
assert result.exit_code == 0, result.output
payload = _report(tmp_path / "reports")
assert payload["skills"] == [{"name": "sample", "passed": True, "issue_count": 0}]
assert [item["name"] for contributor in payload["contributors"] for item in contributor["items"]] == ["sample"]
- assert payload["target_path"] == "sample"
+ assert payload["target_path"] == str(Path(target))
number = tier.removeprefix("tier")
assert payload["gating"]["tiers"] == {
number: {
diff --git a/tests/reporting/test_reporters.py b/tests/reporting/test_reporters.py
index 8f874cbf..db042b3f 100644
--- a/tests/reporting/test_reporters.py
+++ b/tests/reporting/test_reporters.py
@@ -541,6 +541,46 @@ class TestJSONReporter:
assert [entry["name"] for entry in data["skills"]] == ["simple"]
assert data["target_path"] == str(skill)
+ @pytest.mark.parametrize(
+ ("working_directory", "target"),
+ [
+ ("simple", "."),
+ ("simple/nested", ".."),
+ (".", "simple"),
+ (".", "./simple"),
+ (".", "simple/"),
+ ("simple/nested", "../../simple"),
+ ],
+ )
+ def test_render_all_names_the_skill_for_every_spelling_of_a_local_target(
+ self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, working_directory: str, target: str
+ ) -> None:
+ """``.`` and ``..`` have no folder name of their own; the folder they open does."""
+ skill = tmp_path / "simple"
+ (skill / "nested").mkdir(parents=True)
+ (skill / "SKILL.md").write_text("---\nname: simple\n---\n", encoding="utf-8")
+ monkeypatch.chdir(tmp_path / working_directory)
+
+ data = json.loads(
+ JSONReporter(include_timestamp=False, target_path=target).render_all([self._clean_schema_result()])
+ )
+
+ assert data["skills"] == [{"name": "simple", "passed": True, "issue_count": 0}]
+ assert data["target_path"] == target
+
+ def test_render_all_does_not_name_a_current_directory_that_holds_no_skill(
+ self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ folder = tmp_path / "plain"
+ folder.mkdir()
+ monkeypatch.chdir(folder)
+
+ data = json.loads(
+ JSONReporter(include_timestamp=False, target_path=".").render_all([self._clean_schema_result()])
+ )
+
+ assert "plain" not in [entry["name"] for entry in data["skills"]]
+
def test_render_all_records_the_evaluator_version_and_target(self, success_result: ValidationResult) -> None:
from skillevaluator import __version__