Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

merge(main): refresh benchmark publication rollout

Narendran Raghavane5056a795cc8
diff --cc .github/workflows/ci.yml
index a1bbc251,85123d9e..80c2f8aa
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@@ -9,33 -9,68 +9,71 @@@ on
permissions:
contents: read
+ concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
+ cancel-in-progress: true
+
jobs:
- test:
- name: Tests (Python ${{ matrix.python-version }})
- strategy:
- fail-fast: false
- matrix:
- python-version: ["3.12", "3.13"]
+ classify-changes:
+ name: Classify changes
+ if: ${{ github.event_name == 'pull_request' }}
runs-on: ubuntu-latest
+ outputs:
+ docs_only: ${{ steps.changes.outputs.docs_only }}
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
- python-version: ${{ matrix.python-version }}
- - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0
+ fetch-depth: 0
+ persist-credentials: false
+ - name: Classify pull-request paths
+ id: changes
+ env:
+ BASE_SHA: ${{ github.event.pull_request.base.sha }}
+ HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+ shell: bash
+ run: |
+ set -euo pipefail
+ classifier="$RUNNER_TEMP/classify_ci_changes.py"
+ if ! git show "$BASE_SHA:scripts/classify_ci_changes.py" > "$classifier"; then
+ echo "::notice::Trusted classifier is not present on the base branch; running full CI."
+ echo "docs_only=false" >> "$GITHUB_OUTPUT"
+ exit 0
+ fi
+ python3 "$classifier" --base "$BASE_SHA" --head "$HEAD_SHA"
+
+ test-python-312:
+ name: Tests (Python 3.12)
+ needs: classify-changes
+ if: ${{ !cancelled() }}
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+ with:
+ persist-credentials: false
+ - name: Set up Python
+ if: ${{ needs.classify-changes.outputs.docs_only != 'true' }}
+ uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+ with:
+ python-version: "3.12"
+ - name: Set up uv
+ if: ${{ needs.classify-changes.outputs.docs_only != 'true' }}
+ uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0
with:
version: "0.11.27"
- - run: uv sync --all-extras --locked --python "${{ matrix.python-version }}"
+ - name: Install dependencies
+ if: ${{ needs.classify-changes.outputs.docs_only != 'true' }}
+ run: uv sync --all-extras --locked --python 3.12
- name: Scan OSS source boundary
- if: matrix.python-version == '3.12'
+ if: ${{ needs.classify-changes.outputs.docs_only != 'true' }}
run: python scripts/check_oss_boundary.py --root . --allowlist config/oss_boundary_allowlist.json
+ - name: Check public benchmark publication cards
- if: matrix.python-version == '3.12'
++ if: ${{ needs.classify-changes.outputs.docs_only != 'true' }}
+ run: python scripts/ci/check_public_benchmarks.py --require-files tests/golden
- - if: matrix.python-version == '3.12'
+ - name: Lint
+ if: ${{ needs.classify-changes.outputs.docs_only != 'true' }}
run: uv run ruff check .
- name: Run tests with coverage
- if: matrix.python-version == '3.12'
+ if: ${{ needs.classify-changes.outputs.docs_only != 'true' }}
run: >-
uv run pytest -q
--cov=skillevaluator
diff --cc src/skillevaluator/tier3/harbor/runner.py
index eb0ead0f,2c08f3d7..5e8eaedf
--- a/src/skillevaluator/tier3/harbor/runner.py
+++ b/src/skillevaluator/tier3/harbor/runner.py
@@@ -25,8 -26,8 +26,9 @@@ from pathlib import Pat
from queue import Empty, SimpleQueue
from types import MappingProxyType
from typing import Any
+ from uuid import uuid4
+from skillevaluator import __version__
from skillevaluator.evaluation.tier3_report import render_agent_eval_html_report
from skillevaluator.provider_config import ProviderConfig, ProviderConfigurationError, resolve_llm_provider
from skillevaluator.tier3.evals_config import EvalsConfigError, load_evals_config
@@@ -48,12 -57,15 +58,19 @@@ from skillevaluator.tier3.harbor.progre
safe_progress_reporter,
secret_values_from_environment,
)
+from skillevaluator.tier3.harbor.report_data import (
+ build_dataset_snapshot,
+ load_staged_harbor_dataset,
+)
from skillevaluator.tier3.harbor.secure_copy import copytree_secure
from skillevaluator.tier3.harbor.secure_docker_environment import SECURE_DOCKER_ENV_IMPORT_PATH
+ from skillevaluator.tier3.output_provenance import (
+ mark_generated_output_root,
+ remove_generated_output_root_if_owned,
+ remove_output_reservation_if_identity_matches,
+ write_output_file_atomically,
+ )
+ from skillevaluator.tier3.results_location import publish_latest_results
from skillevaluator.tier3_environments import DEFAULT_ENV_MODE, ENV_MODE_LOCAL, HARBOR_ENV_MODES
logger = logging.getLogger(__name__)