Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

6.1 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Contracts for the reusable GitLab job and consumer overrides."""

from pathlib import Path

import yaml

ROOT = Path(__file__).parents[2]


def test_user_can_override_included_global_defaults():
    config = yaml.safe_load((ROOT / "ci/gitlab/skill-evaluation.yml").read_text())
    job = config["skill-evaluation"]
    for name in ("SKILL_EVAL_PATHS", "SKILL_EVAL_MIN_SCORE", "SKILL_EVAL_SCAN_MODE"):
        assert name in config["variables"]
        assert name not in job.get("variables", {})
    assert config["variables"]["SKILL_EVAL_MIN_SCORE"] == "70"
    assert "LLM_ENABLED" not in config["variables"]


def _run_template(tmp_path, extra_env, *, scanner_exit=0, existing_report=False):
    import os
    import subprocess

    config = yaml.safe_load((ROOT / "ci/gitlab/skill-evaluation.yml").read_text())
    script = config["skill-evaluation"]["script"][0]
    workspace = tmp_path / "workspace"
    workspace.mkdir()
    if existing_report:
        (workspace / "skill-evaluator-report").mkdir()
    bin_dir = tmp_path / "bin"
    bin_dir.mkdir()
    stub = bin_dir / "python"
    stub.write_text(
        "#!/bin/sh\n"
        'while [ "$#" -gt 0 ]; do\n'
        '  if [ "$1" = --reports ]; then shift; report_dir=$1; fi\n'
        "  shift\n"
        "done\n"
        'printf \'%s\\n%s\\n%s\\n%s\\n\' "$SCAN_MODE" "${DIFF_BASE_SHA:-}" "${DIFF_HEAD_SHA:-}" "$MIN_SCORE" > "$report_dir/evidence.txt"\n'
        'printf \'%s\' "${REPORT_RUN_URL:-}" > "$report_dir/run_url.txt"\n'
        'printf \'%s\' "$report_dir" > "$report_dir/temp_location.txt"\n'
        f"exit {scanner_exit}\n"
    )
    stub.chmod(0o755)
    temporary = tmp_path / "temporary reports"
    temporary.mkdir()
    # Start from a clean environment so a suite run inside GitLab CI cannot leak its own job variables.
    outer = {key: value for key, value in os.environ.items() if not key.startswith(("CI_", "GITLAB_", "SKILL_EVAL_"))}
    env = {
        **outer,
        "PATH": str(bin_dir) + os.pathsep + os.environ["PATH"],
        "CI_PROJECT_DIR": str(workspace),
        "CI_PIPELINE_SOURCE": "push",
        "CI_COMMIT_SHA": "b" * 40,
        "SKILL_EVAL_IMAGE": "registry.example/test@sha256:" + "a" * 64,
        "SKILL_EVAL_PATHS": "skills",
        "SKILL_EVAL_MIN_SCORE": "80",
        "SKILL_EVAL_SCAN_MODE": "auto",
        "TMPDIR": str(temporary),
        **extra_env,
    }
    result = subprocess.run(["bash", "-c", script], env=env, capture_output=True, text=True, check=False)
    report = workspace / "skill-evaluator-report/evidence.txt"
    return result, report.read_text().splitlines() if report.exists() else None


def test_gitlab_job_runs_all_skills_on_push_and_keeps_output(tmp_path):
    result, evidence = _run_template(tmp_path, {})
    assert result.returncode == 0
    assert evidence == ["all", "", "", "80"]


def test_gitlab_job_uses_and_cleans_configured_temporary_storage(tmp_path):
    temporary = tmp_path / "custom temporary storage"
    temporary.mkdir()
    result, evidence = _run_template(tmp_path, {"TMPDIR": str(temporary)})
    assert result.returncode == 0, result.stderr
    assert evidence == ["all", "", "", "80"]
    location = Path((tmp_path / "workspace/skill-evaluator-report/temp_location.txt").read_text())
    assert location.parent == temporary
    assert not location.exists()


def test_gitlab_baseline_opt_in_rejects_noncanonical_boolean_and_inherited_policy(tmp_path):
    result, evidence = _run_template(
        tmp_path,
        {
            "SKILL_EVAL_SECURITY_BASELINES": "TRUE",
            "POLICY_SOURCE_SHA": "a" * 40,
        },
    )
    assert result.returncode != 0
    assert evidence is None


def test_gitlab_job_uses_mr_diff_and_retains_failed_results(tmp_path):
    result, evidence = _run_template(
        tmp_path,
        {"CI_PIPELINE_SOURCE": "merge_request_event", "CI_MERGE_REQUEST_DIFF_BASE_SHA": "a" * 40},
        scanner_exit=1,
    )
    assert result.returncode == 1
    assert evidence == ["changed", "a" * 40, "b" * 40, "80"]


def test_gitlab_job_rejects_merged_results_for_changed_selection(tmp_path):
    result, evidence = _run_template(
        tmp_path,
        {"CI_PIPELINE_SOURCE": "merge_request_event", "CI_MERGE_REQUEST_EVENT_TYPE": "merged_result"},
    )
    assert result.returncode != 0
    assert evidence is None
    assert "detached merge-request" in result.stderr


def test_gitlab_job_all_mode_supports_merged_result_pipelines(tmp_path):
    result, evidence = _run_template(
        tmp_path,
        {
            "CI_PIPELINE_SOURCE": "merge_request_event",
            "CI_MERGE_REQUEST_EVENT_TYPE": "merged_result",
            "SKILL_EVAL_SCAN_MODE": "all",
        },
    )
    assert result.returncode == 0
    assert evidence == ["all", "", "", "80"]


def test_gitlab_job_does_not_overwrite_existing_report_directory(tmp_path):
    result, evidence = _run_template(tmp_path, {}, existing_report=True)
    assert result.returncode != 0
    assert evidence is None
    assert "already exists" in result.stderr


def test_gitlab_job_passes_only_https_job_links(tmp_path):
    for job_url, expected in (
        ("http://gitlab.example.com/g/p/-/jobs/1", ""),
        ("https://gitlab.example.com/g/p/-/jobs/1", "https://gitlab.example.com/g/p/-/jobs/1"),
    ):
        case = tmp_path / ("https" if expected else "http")
        case.mkdir()
        result, evidence = _run_template(case, {"CI_JOB_URL": job_url})
        assert result.returncode == 0
        assert evidence == ["all", "", "", "80"]
        assert (case / "workspace/skill-evaluator-report/run_url.txt").read_text() == expected


def test_gitlab_template_ignores_the_outer_ci_environment(tmp_path, monkeypatch):
    monkeypatch.setenv("CI_MERGE_REQUEST_EVENT_TYPE", "merged_result")
    monkeypatch.setenv("SKILL_EVAL_SCAN_MODE", "all")
    result, evidence = _run_template(
        tmp_path,
        {"CI_PIPELINE_SOURCE": "merge_request_event", "CI_MERGE_REQUEST_DIFF_BASE_SHA": "a" * 40},
    )
    assert result.returncode == 0
    assert evidence == ["changed", "a" * 40, "b" * 40, "80"]