Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
merge(main): refresh benchmark publication rollout
Narendran Raghavane5056a795cc8
diff --cc .github/workflows/ci.ymlindex a1bbc251,85123d9e..80c2f8aa--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@@ -9,33 -9,68 +9,71 @@@ onpermissions:contents: read+ concurrency:+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}+ cancel-in-progress: true+jobs:- test:- name: Tests (Python ${{ matrix.python-version }})- strategy:- fail-fast: false- matrix:- python-version: ["3.12", "3.13"]+ classify-changes:+ name: Classify changes+ if: ${{ github.event_name == 'pull_request' }}runs-on: ubuntu-latest+ outputs:+ docs_only: ${{ steps.changes.outputs.docs_only }}steps:- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0- - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0with:- python-version: ${{ matrix.python-version }}- - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0+ fetch-depth: 0+ persist-credentials: false+ - name: Classify pull-request paths+ id: changes+ env:+ BASE_SHA: ${{ github.event.pull_request.base.sha }}+ HEAD_SHA: ${{ github.event.pull_request.head.sha }}+ shell: bash+ run: |+ set -euo pipefail+ classifier="$RUNNER_TEMP/classify_ci_changes.py"+ if ! git show "$BASE_SHA:scripts/classify_ci_changes.py" > "$classifier"; then+ echo "::notice::Trusted classifier is not present on the base branch; running full CI."+ echo "docs_only=false" >> "$GITHUB_OUTPUT"+ exit 0+ fi+ python3 "$classifier" --base "$BASE_SHA" --head "$HEAD_SHA"++ test-python-312:+ name: Tests (Python 3.12)+ needs: classify-changes+ if: ${{ !cancelled() }}+ runs-on: ubuntu-latest+ steps:+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0+ with:+ persist-credentials: false+ - name: Set up Python+ if: ${{ needs.classify-changes.outputs.docs_only != 'true' }}+ uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0+ with:+ python-version: "3.12"+ - name: Set up uv+ if: ${{ needs.classify-changes.outputs.docs_only != 'true' }}+ uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0with:version: "0.11.27"- - run: uv sync --all-extras --locked --python "${{ matrix.python-version }}"+ - name: Install dependencies+ if: ${{ needs.classify-changes.outputs.docs_only != 'true' }}+ run: uv sync --all-extras --locked --python 3.12- name: Scan OSS source boundary- if: matrix.python-version == '3.12'+ if: ${{ needs.classify-changes.outputs.docs_only != 'true' }}run: python scripts/check_oss_boundary.py --root . --allowlist config/oss_boundary_allowlist.json+ - name: Check public benchmark publication cards- if: matrix.python-version == '3.12'++ if: ${{ needs.classify-changes.outputs.docs_only != 'true' }}+ run: python scripts/ci/check_public_benchmarks.py --require-files tests/golden- - if: matrix.python-version == '3.12'+ - name: Lint+ if: ${{ needs.classify-changes.outputs.docs_only != 'true' }}run: uv run ruff check .- name: Run tests with coverage- if: matrix.python-version == '3.12'+ if: ${{ needs.classify-changes.outputs.docs_only != 'true' }}run: >-uv run pytest -q--cov=skillevaluatordiff --cc src/skillevaluator/tier3/harbor/runner.pyindex eb0ead0f,2c08f3d7..5e8eaedf--- a/src/skillevaluator/tier3/harbor/runner.py+++ b/src/skillevaluator/tier3/harbor/runner.py@@@ -25,8 -26,8 +26,9 @@@ from pathlib import Patfrom queue import Empty, SimpleQueuefrom types import MappingProxyTypefrom typing import Any+ from uuid import uuid4+from skillevaluator import __version__from skillevaluator.evaluation.tier3_report import render_agent_eval_html_reportfrom skillevaluator.provider_config import ProviderConfig, ProviderConfigurationError, resolve_llm_providerfrom skillevaluator.tier3.evals_config import EvalsConfigError, load_evals_config@@@ -48,12 -57,15 +58,19 @@@ from skillevaluator.tier3.harbor.progresafe_progress_reporter,secret_values_from_environment,)+from skillevaluator.tier3.harbor.report_data import (+ build_dataset_snapshot,+ load_staged_harbor_dataset,+)from skillevaluator.tier3.harbor.secure_copy import copytree_securefrom skillevaluator.tier3.harbor.secure_docker_environment import SECURE_DOCKER_ENV_IMPORT_PATH+ from skillevaluator.tier3.output_provenance import (+ mark_generated_output_root,+ remove_generated_output_root_if_owned,+ remove_output_reservation_if_identity_matches,+ write_output_file_atomically,+ )+ from skillevaluator.tier3.results_location import publish_latest_resultsfrom skillevaluator.tier3_environments import DEFAULT_ENV_MODE, ENV_MODE_LOCAL, HARBOR_ENV_MODESlogger = logging.getLogger(__name__)