Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(grader): canonicalize verifier paths before provenance checks (#467)

Narendran Raghavan03c27bf15d1e
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a2a491bc..0c074d43 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -386,6 +386,10 @@ All notable changes to SkillEvaluator are documented in this file.
- Read JSON, JSONL and YAML evaluation datasets as UTF-8 so Unicode prompts and
answers load correctly on hosts with a non-UTF-8 default text encoding.
+- Grader qualification resolves verifier source paths before provenance hashing,
+ so source-directory aliases (including macOS `/tmp`) retain the same relative
+ file identities and hashes while files outside the source root are rejected.
+
- Estimate Tier 1 instruction tokens from the parsed Markdown body, so `---`
inside YAML values or comments does not count frontmatter as instructions.
Body separators without valid frontmatter no longer produce a spurious estimate.
diff --git a/src/skillevaluator/tier3/grader_validation.py b/src/skillevaluator/tier3/grader_validation.py
index 740e22fb..d031817e 100644
--- a/src/skillevaluator/tier3/grader_validation.py
+++ b/src/skillevaluator/tier3/grader_validation.py
@@ -469,7 +469,8 @@ def qualify(root, suite, cases, output, repeats, concurrency, timeout, split, pr
source_root = Path(__file__).resolve().parents[2]
verifier_file_sha256 = {}
verifier_digest = hashlib.sha256()
- for path in sorted(_VERIFIER_FILES, key=lambda item: item.relative_to(source_root).as_posix()):
+ verifier_paths = (path.resolve() for path in _VERIFIER_FILES)
+ for path in sorted(verifier_paths, key=lambda item: item.relative_to(source_root).as_posix()):
name = path.relative_to(source_root).as_posix()
content = path.read_bytes()
verifier_file_sha256[name] = hashlib.sha256(content).hexdigest()
diff --git a/tests/tier3/test_grader_validation.py b/tests/tier3/test_grader_validation.py
index cf870b9d..6f75201a 100644
--- a/tests/tier3/test_grader_validation.py
+++ b/tests/tier3/test_grader_validation.py
@@ -4,6 +4,7 @@
from __future__ import annotations
import copy
+import hashlib
import json
import re
import threading
@@ -445,6 +446,91 @@ def test_provenance_records_resolved_judge_controls_and_verifier_files(tmp_path)
assert len(provenance["verifier_sha256"]) == 64
[email protected]("package_root", ["src", "site-packages"])
[email protected]("use_alias", [False, True], ids=["canonical", "symlink-alias"])
+def test_verifier_provenance_preserves_relative_hashes_through_source_alias(
+ monkeypatch, tmp_path, package_root, use_alias
+):
+ from skillevaluator.tier3.harbor import adapter
+
+ source = tmp_path / package_root
+ module = source / "skillevaluator/tier3/grader_validation.py"
+ module.parent.mkdir(parents=True)
+ module.touch()
+ contents = {
+ "skillevaluator/tier3/eval_core/checks.py": b"checks\n",
+ "skillevaluator/tier3/harbor/templates/eval.py": b"verifier\n",
+ }
+ for name, content in contents.items():
+ path = source / name
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_bytes(content)
+ imported_root = source
+ if use_alias:
+ imported_root = tmp_path / "source-alias"
+ imported_root.symlink_to(source, target_is_directory=True)
+ monkeypatch.setattr(gv, "__file__", str(imported_root / module.relative_to(source)))
+ monkeypatch.setattr(adapter, "_VERIFIER_FILES", [imported_root / name for name in reversed(contents)])
+ output = tmp_path / "qualification"
+ output.mkdir()
+ report = gv.qualify(
+ gv.REFERENCE_SUITE,
+ gv.load_suite(gv.REFERENCE_SUITE),
+ [],
+ output,
+ 1,
+ 1,
+ 1,
+ "assessment",
+ SimpleNamespace(provider="openai", model="test-model"),
+ {},
+ {},
+ )
+ expected_digest = hashlib.sha256()
+ for name, content in contents.items():
+ expected_digest.update(name.encode() + b"\0" + len(content).to_bytes(8, "big") + content)
+ assert report["provenance"]["verifier_file_sha256"] == {
+ name: hashlib.sha256(content).hexdigest() for name, content in contents.items()
+ }
+ assert report["provenance"]["verifier_sha256"] == expected_digest.hexdigest()
+ assert json.loads((output / "grader-validation.json").read_text())["provenance"] == report["provenance"]
+
+
[email protected]("symlink_escape", [False, True], ids=["outside-file", "symlink-escape"])
+def test_verifier_provenance_rejects_files_outside_source_root(monkeypatch, tmp_path, symlink_escape):
+ from skillevaluator.tier3.harbor import adapter
+
+ source = tmp_path / "site-packages"
+ module = source / "skillevaluator/tier3/grader_validation.py"
+ module.parent.mkdir(parents=True)
+ module.touch()
+ outside = tmp_path / "outside.py"
+ outside.write_text("outside source boundary\n")
+ verifier = outside
+ if symlink_escape:
+ verifier = module.parent / "escaped.py"
+ verifier.symlink_to(outside)
+ monkeypatch.setattr(gv, "__file__", str(module))
+ monkeypatch.setattr(adapter, "_VERIFIER_FILES", [verifier])
+ output = tmp_path / "qualification"
+ output.mkdir()
+ with pytest.raises(ValueError, match="not in the subpath"):
+ gv.qualify(
+ gv.REFERENCE_SUITE,
+ gv.load_suite(gv.REFERENCE_SUITE),
+ [],
+ output,
+ 1,
+ 1,
+ 1,
+ "assessment",
+ SimpleNamespace(provider="openai", model="test-model"),
+ {},
+ {},
+ )
+ assert not (output / "grader-validation.json").exists()
+
+
def test_split_separation_and_gates(judge_server, tmp_path):
result = CliRunner().invoke(
cli,