Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
ci: trust base branch for path classification
Christopher Kevin324f9c958476
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 738c0795..940d10fe 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -30,7 +30,16 @@ jobs:env:BASE_SHA: ${{ github.event.pull_request.base.sha }}HEAD_SHA: ${{ github.event.pull_request.head.sha }}- run: python3 scripts/classify_ci_changes.py --base "$BASE_SHA" --head "$HEAD_SHA"+ shell: bash+ run: |+ set -euo pipefail+ classifier="$RUNNER_TEMP/classify_ci_changes.py"+ if ! git show "$BASE_SHA:scripts/classify_ci_changes.py" > "$classifier"; then+ echo "::notice::Trusted classifier is not present on the base branch; running full CI."+ echo "docs_only=false" >> "$GITHUB_OUTPUT"+ exit 0+ fi+ python3 "$classifier" --base "$BASE_SHA" --head "$HEAD_SHA"test-python-312:name: Tests (Python 3.12)diff --git a/.github/workflows/security.yml b/.github/workflows/security.ymlindex 2e66b74a..6bf9e4f8 100644--- a/.github/workflows/security.yml+++ b/.github/workflows/security.yml@@ -35,7 +35,16 @@ jobs:env:BASE_SHA: ${{ github.event.pull_request.base.sha }}HEAD_SHA: ${{ github.event.pull_request.head.sha }}- run: python3 scripts/classify_ci_changes.py --base "$BASE_SHA" --head "$HEAD_SHA"+ shell: bash+ run: |+ set -euo pipefail+ classifier="$RUNNER_TEMP/classify_ci_changes.py"+ if ! git show "$BASE_SHA:scripts/classify_ci_changes.py" > "$classifier"; then+ echo "::notice::Trusted classifier is not present on the base branch; running full CI."+ echo "docs_only=false" >> "$GITHUB_OUTPUT"+ exit 0+ fi+ python3 "$classifier" --base "$BASE_SHA" --head "$HEAD_SHA"gitleaks:name: Gitleaksdiff --git a/CHANGELOG.md b/CHANGELOG.mdindex 8aefe394..c60346ac 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -46,6 +46,8 @@ All notable changes to SkillEvaluator are documented in this file.and non-docs changes retain the complete Linux, macOS, Windows, packaging,and security matrix. Superseded pull-request CI and security runs arecancelled so they do not consume runners after a newer commit is pushed.+ Path classification executes from the pull request base revision so a+ change cannot weaken its own CI routing.- Simplified the repository README into a concise documentation landing page,retained a compact keyless `validate` quickstart, LLM-provider setup, and aone-command `validate --full` path through all three tiers, broadened thediff --git a/scripts/classify_ci_changes.py b/scripts/classify_ci_changes.pyindex 5141da8f..aadb1f4b 100644--- a/scripts/classify_ci_changes.py+++ b/scripts/classify_ci_changes.py@@ -79,7 +79,7 @@ def changed_paths(repo: Path, base: str, head: str) -> list[bytes]:"--name-status","-z","--find-renames",- "--find-copies",+ "--find-copies-harder",f"{base}...{head}","--",],diff --git a/tests/test_ci_change_classifier.py b/tests/test_ci_change_classifier.pyindex 1106df10..fa2ee383 100644--- a/tests/test_ci_change_classifier.py+++ b/tests/test_ci_change_classifier.py@@ -8,7 +8,7 @@ import sysfrom pathlib import Pathimport pytest-from scripts.classify_ci_changes import is_docs_only, main, parse_name_status_z+from scripts.classify_ci_changes import changed_paths, is_docs_only, main, parse_name_status_zdef _git(repo: Path, *args: str) -> str:@@ -33,8 +33,8 @@ def git_repo(tmp_path: Path) -> tuple[Path, str]:_git(tmp_path, "config", "user.email", "[email protected]")_git(tmp_path, "config", "user.name", "CI Test")(tmp_path / "docs").mkdir()- (tmp_path / "docs" / "index.mdx").write_text("initial\n", encoding="utf-8")- (tmp_path / "README.md").write_text("initial\n", encoding="utf-8")+ (tmp_path / "docs" / "index.mdx").write_text("docs source\n", encoding="utf-8")+ (tmp_path / "README.md").write_text("root source\n", encoding="utf-8")return tmp_path, _commit(tmp_path, "initial")@@ -199,6 +199,20 @@ def test_main_checks_the_source_of_a_rename_into_docs(assert (tmp_path / "github-output").read_text(encoding="utf-8") == "docs_only=false\n"+def test_changed_paths_detects_an_unmodified_copy_source_outside_docs(+ git_repo: tuple[Path, str],+) -> None:+ repo, base = git_repo+ (repo / "docs" / "copied-readme.mdx").write_bytes((repo / "README.md").read_bytes())+ head = _commit(repo, "copy root file into docs")++ paths = changed_paths(repo, base, head)++ assert b"README.md" in paths+ assert b"docs/copied-readme.mdx" in paths+ assert is_docs_only(paths) is False++def test_main_uses_the_merge_base_when_the_base_branch_advances(git_repo: tuple[Path, str],tmp_path: Path,diff --git a/tests/test_ci_workflows.py b/tests/test_ci_workflows.pyindex cf24613d..e92a708d 100644--- a/tests/test_ci_workflows.py+++ b/tests/test_ci_workflows.py@@ -75,8 +75,11 @@ def test_ci_classifier_is_pull_request_only_and_exports_docs_only() -> None:assert classifier["steps"][0]["with"]["fetch-depth"] == "0"assert classifier["steps"][0]["with"]["persist-credentials"] == "false"assert classifier["steps"][1]["id"] == "changes"- assert "scripts/classify_ci_changes.py" in classifier["steps"][1]["run"]- assert classifier["steps"][1]["run"].startswith("python3 ")+ classifier_run = classifier["steps"][1]["run"]+ assert 'git show "$BASE_SHA:scripts/classify_ci_changes.py"' in classifier_run+ assert 'python3 "$classifier"' in classifier_run+ assert 'echo "docs_only=false" >> "$GITHUB_OUTPUT"' in classifier_run+ assert "python3 scripts/classify_ci_changes.py" not in classifier_runassert classifier["steps"][1]["env"] == {"BASE_SHA": "${{ github.event.pull_request.base.sha }}","HEAD_SHA": "${{ github.event.pull_request.head.sha }}",@@ -151,7 +154,11 @@ def test_security_keeps_gitleaks_always_on_and_skips_only_nonessential_jobs() ->assert jobs["classify-changes"]["if"] == "${{ github.event_name == 'pull_request' }}"assert jobs["classify-changes"]["outputs"]["docs_only"] == "${{ steps.changes.outputs.docs_only }}"assert jobs["classify-changes"]["steps"][0]["with"]["persist-credentials"] == "false"- assert jobs["classify-changes"]["steps"][1]["run"].startswith("python3 ")+ classifier_run = jobs["classify-changes"]["steps"][1]["run"]+ assert 'git show "$BASE_SHA:scripts/classify_ci_changes.py"' in classifier_run+ assert 'python3 "$classifier"' in classifier_run+ assert 'echo "docs_only=false" >> "$GITHUB_OUTPUT"' in classifier_run+ assert "python3 scripts/classify_ci_changes.py" not in classifier_rundependency_if = " ".join(jobs["dependency-review"]["if"].split())codeql_if = " ".join(jobs["codeql"]["if"].split())@@ -175,6 +182,16 @@ def test_dco_stays_unconditional_and_has_no_path_filter() -> None:assert "needs" not in dco["jobs"]["dco"]+def test_non_pr_workflow_triggers_are_preserved() -> None:+ ci = _load("ci.yml")+ security = _load("security.yml")++ assert ci["on"]["push"] == {"branches": ["main"]}+ assert security["on"]["push"] == {"branches": ["main"]}+ assert security["on"]["schedule"] == [{"cron": "23 7 * * 1"}]+ assert "workflow_dispatch" in security["on"]++def test_changed_workflows_pin_every_action_to_a_commit() -> None:for workflow_name in ("ci.yml", "security.yml"):for uses in _all_uses(_load(workflow_name)):