Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(grader): canonicalize verifier paths before provenance checks (#467)
Narendran Raghavan03c27bf15d1e
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex a2a491bc..0c074d43 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -386,6 +386,10 @@ All notable changes to SkillEvaluator are documented in this file.- Read JSON, JSONL and YAML evaluation datasets as UTF-8 so Unicode prompts andanswers load correctly on hosts with a non-UTF-8 default text encoding.+- Grader qualification resolves verifier source paths before provenance hashing,+ so source-directory aliases (including macOS `/tmp`) retain the same relative+ file identities and hashes while files outside the source root are rejected.+- Estimate Tier 1 instruction tokens from the parsed Markdown body, so `---`inside YAML values or comments does not count frontmatter as instructions.Body separators without valid frontmatter no longer produce a spurious estimate.diff --git a/src/skillevaluator/tier3/grader_validation.py b/src/skillevaluator/tier3/grader_validation.pyindex 740e22fb..d031817e 100644--- a/src/skillevaluator/tier3/grader_validation.py+++ b/src/skillevaluator/tier3/grader_validation.py@@ -469,7 +469,8 @@ def qualify(root, suite, cases, output, repeats, concurrency, timeout, split, prsource_root = Path(__file__).resolve().parents[2]verifier_file_sha256 = {}verifier_digest = hashlib.sha256()- for path in sorted(_VERIFIER_FILES, key=lambda item: item.relative_to(source_root).as_posix()):+ verifier_paths = (path.resolve() for path in _VERIFIER_FILES)+ for path in sorted(verifier_paths, key=lambda item: item.relative_to(source_root).as_posix()):name = path.relative_to(source_root).as_posix()content = path.read_bytes()verifier_file_sha256[name] = hashlib.sha256(content).hexdigest()diff --git a/tests/tier3/test_grader_validation.py b/tests/tier3/test_grader_validation.pyindex cf870b9d..6f75201a 100644--- a/tests/tier3/test_grader_validation.py+++ b/tests/tier3/test_grader_validation.py@@ -4,6 +4,7 @@from __future__ import annotationsimport copy+import hashlibimport jsonimport reimport threading@@ -445,6 +446,91 @@ def test_provenance_records_resolved_judge_controls_and_verifier_files(tmp_path)assert len(provenance["verifier_sha256"]) == 64[email protected]("package_root", ["src", "site-packages"])[email protected]("use_alias", [False, True], ids=["canonical", "symlink-alias"])+def test_verifier_provenance_preserves_relative_hashes_through_source_alias(+ monkeypatch, tmp_path, package_root, use_alias+):+ from skillevaluator.tier3.harbor import adapter++ source = tmp_path / package_root+ module = source / "skillevaluator/tier3/grader_validation.py"+ module.parent.mkdir(parents=True)+ module.touch()+ contents = {+ "skillevaluator/tier3/eval_core/checks.py": b"checks\n",+ "skillevaluator/tier3/harbor/templates/eval.py": b"verifier\n",+ }+ for name, content in contents.items():+ path = source / name+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_bytes(content)+ imported_root = source+ if use_alias:+ imported_root = tmp_path / "source-alias"+ imported_root.symlink_to(source, target_is_directory=True)+ monkeypatch.setattr(gv, "__file__", str(imported_root / module.relative_to(source)))+ monkeypatch.setattr(adapter, "_VERIFIER_FILES", [imported_root / name for name in reversed(contents)])+ output = tmp_path / "qualification"+ output.mkdir()+ report = gv.qualify(+ gv.REFERENCE_SUITE,+ gv.load_suite(gv.REFERENCE_SUITE),+ [],+ output,+ 1,+ 1,+ 1,+ "assessment",+ SimpleNamespace(provider="openai", model="test-model"),+ {},+ {},+ )+ expected_digest = hashlib.sha256()+ for name, content in contents.items():+ expected_digest.update(name.encode() + b"\0" + len(content).to_bytes(8, "big") + content)+ assert report["provenance"]["verifier_file_sha256"] == {+ name: hashlib.sha256(content).hexdigest() for name, content in contents.items()+ }+ assert report["provenance"]["verifier_sha256"] == expected_digest.hexdigest()+ assert json.loads((output / "grader-validation.json").read_text())["provenance"] == report["provenance"]++[email protected]("symlink_escape", [False, True], ids=["outside-file", "symlink-escape"])+def test_verifier_provenance_rejects_files_outside_source_root(monkeypatch, tmp_path, symlink_escape):+ from skillevaluator.tier3.harbor import adapter++ source = tmp_path / "site-packages"+ module = source / "skillevaluator/tier3/grader_validation.py"+ module.parent.mkdir(parents=True)+ module.touch()+ outside = tmp_path / "outside.py"+ outside.write_text("outside source boundary\n")+ verifier = outside+ if symlink_escape:+ verifier = module.parent / "escaped.py"+ verifier.symlink_to(outside)+ monkeypatch.setattr(gv, "__file__", str(module))+ monkeypatch.setattr(adapter, "_VERIFIER_FILES", [verifier])+ output = tmp_path / "qualification"+ output.mkdir()+ with pytest.raises(ValueError, match="not in the subpath"):+ gv.qualify(+ gv.REFERENCE_SUITE,+ gv.load_suite(gv.REFERENCE_SUITE),+ [],+ output,+ 1,+ 1,+ 1,+ "assessment",+ SimpleNamespace(provider="openai", model="test-model"),+ {},+ {},+ )+ assert not (output / "grader-validation.json").exists()++def test_split_separation_and_gates(judge_server, tmp_path):result = CliRunner().invoke(cli,