Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-License-Identifier: Apache-2.0
"""Validate the security boundaries of the opt-in three-tier pipeline."""
import json
from pathlib import Path
import yaml
ROOT = Path(__file__).parents[2]
def load(path):
return list(yaml.load_all((ROOT / path).read_text(), Loader=yaml.BaseLoader))[-1]
def test_candidate_onboarding_uses_the_released_tier1_image():
candidate = json.loads((ROOT / "src/skillevaluator/ci-release-all-tiers.json").read_text())
released = json.loads((ROOT / "src/skillevaluator/ci-release.json").read_text())
# Released schema capabilities, such as multi-valued metadata.domain,
# must also work for new all-tier consumers of the same package.
assert candidate["image"] == released["image"]
def test_required_tier1_reuses_hardened_workflow_and_parallel_paid_tiers():
workflow = load(".github/workflows/evaluate-all-tiers.yml")
inputs = workflow["on"]["workflow_call"]["inputs"]
assert inputs["env-mode"]["default"] == "docker"
assert inputs["tier3-required"]["default"] == "false"
assert workflow["jobs"]["tier1"]["uses"] == "./.github/workflows/evaluate-skills.yml"
paid = workflow["jobs"]["paid-tiers"]
assert paid["strategy"]["fail-fast"] == "false"
assert paid["strategy"]["matrix"]["tier"] == ["2", "3"]
# Existing consumers retain deployment tracking and custom protection apps.
assert inputs["environment-deployment"]["type"] == "boolean"
assert inputs["environment-deployment"]["default"] == "true"
for name in ("paid-tiers", "tier1-llm"):
assert workflow["jobs"][name]["environment"] == {
"name": "${{ inputs.environment }}",
"deployment": "${{ inputs.environment-deployment }}",
}
assert "eligibility" in paid["needs"]
assert "needs.eligibility.outputs.allowed" in paid["if"]
checkouts = [s for s in paid["steps"] if s.get("uses", "").startswith("actions/checkout@")]
assert {s["with"]["path"] for s in checkouts} == {"candidate", "evaluator"}
tools = next(s for s in checkouts if s["with"]["path"] == "evaluator")
assert tools["with"]["ref"] == "${{ inputs.tools-ref }}"
assert tools["with"]["persist-credentials"] == "false"
assert all("secrets." not in str(s) for s in workflow["jobs"]["eligibility"]["steps"])
def test_aggregate_retains_every_tier_after_failure():
workflow = load(".github/workflows/evaluate-all-tiers.yml")
aggregate = workflow["jobs"]["aggregate"]
assert aggregate["if"] == "${{ always() }}"
# Eligibility tells the aggregate when the paid tiers were skipped for no skill change.
assert set(aggregate["needs"]) == {"eligibility", "tier1", "tier1-llm", "paid-tiers"}
downloads = [s for s in aggregate["steps"] if s.get("uses", "").startswith("actions/download-artifact@")]
assert [s["with"]["pattern"] for s in downloads] == ["skill-evaluator-tier[123]-attempt-*"]
assert downloads[0]["with"]["path"] == "${{ runner.temp }}/skill-artifacts"
upload = next(s for s in aggregate["steps"] if s.get("uses", "").startswith("actions/upload-artifact@"))
assert upload["if"] == "${{ always() }}"
assert upload["with"]["retention-days"] == "14"
assert upload["with"]["name"] == "skill-evaluator-results-attempt-${{ github.run_attempt }}"
def test_aggregate_reuses_tier_evidence_from_jobs_not_rerun():
import fnmatch
workflow = load(".github/workflows/evaluate-all-tiers.yml")
steps = workflow["jobs"]["aggregate"]["steps"]
download = next(s for s in steps if s.get("uses", "").startswith("actions/download-artifact@"))
# "Re-run failed jobs" leaves passing tier jobs' artifacts under their own earlier attempt.
assert "github.run_attempt" not in str(download["with"])
assert "name" not in download["with"]
assert download["with"].get("merge-multiple", "false") == "false"
pattern = download["with"]["pattern"]
tier1 = load(".github/workflows/evaluate-skills.yml")["jobs"]["tier1"]["steps"]
produced = [
# The default/fork path retains canonical static evidence. Protected
# enrichment emits the same canonical name from its separate job.
s["with"]["name"].replace("${{ inputs.artifact-name }}", "skill-evaluator-tier1")
for s in tier1
if s.get("uses", "").startswith("actions/upload-artifact@")
]
produced += [
s["with"]["name"].replace("${{ matrix.tier }}", tier)
for s in workflow["jobs"]["paid-tiers"]["steps"]
if s.get("uses", "").startswith("actions/upload-artifact@")
for tier in workflow["jobs"]["paid-tiers"]["strategy"]["matrix"]["tier"]
]
produced += [s["with"]["name"] for s in steps if s.get("uses", "").startswith("actions/upload-artifact@")]
for attempt in ("1", "2", "12"):
names = [name.replace("${{ github.run_attempt }}", attempt) for name in produced]
matched = sorted(name for name in names if fnmatch.fnmatchcase(name, pattern))
assert matched == [f"skill-evaluator-tier{tier}-attempt-{attempt}" for tier in (1, 2, 3)]
run = [s.get("run", "") for s in steps]
select = next(i for i, script in enumerate(run) if "run_tiers.py select-attempts" in script)
assert (
steps.index(download) < select < next(i for i, script in enumerate(run) if "run_tiers.py aggregate" in script)
)
assert '--artifacts "$RUNNER_TEMP/skill-artifacts" --reports "$RUNNER_TEMP/skill-reports"' in run[select]
assert "continue-on-error" not in steps[select]
def test_paid_tier_evaluation_stops_before_the_job_timeout():
import subprocess
paid = load(".github/workflows/evaluate-all-tiers.yml")["jobs"]["paid-tiers"]
assert paid["timeout-minutes"] == "${{ inputs.timeout-minutes }}"
assert paid["env"]["SKILL_EVAL_TIMEOUT_MINUTES"] == "${{ inputs.timeout-minutes }}"
step = next(s for s in paid["steps"] if s.get("name") == "Evaluate selected tier and retain native evidence")
setup = step["run"].split("python -I", 1)[0]
for minutes, expected in ((300, 280), (60, 45), (10, 8), (1, 1)):
result = subprocess.run(
["bash", "-eo", "pipefail", "-c", setup + 'printf %s "$SKILL_EVAL_TIMEOUT_SECONDS"'],
env={"PATH": "/usr/bin:/bin", "SKILL_EVAL_TIMEOUT_MINUTES": str(minutes)},
capture_output=True,
text=True,
check=True,
)
assert int(result.stdout) == expected * 60
def test_paid_tiers_leave_the_embedding_default_to_the_runner():
paid = load(".github/workflows/evaluate-all-tiers.yml")["jobs"]["paid-tiers"]
# An Anthropic chat provider defaults to OpenAI embeddings inside run_tiers.
assert paid["env"]["SKILL_EVAL_EMBEDDING_PROVIDER"] == "${{ inputs.embedding-provider }}"
def test_feedback_checks_out_only_pinned_tools_and_uses_scoped_token():
workflow = load(".github/workflows/feedback-all-tiers.yml")
job = workflow["jobs"]["feedback"]
assert job["permissions"]["pull-requests"] == "write"
checkouts = [s for s in job["steps"] if s.get("uses", "").startswith("actions/checkout@")]
assert len(checkouts) == 1
assert checkouts[0]["with"]["ref"] == "${{ inputs.tools-ref }}"
assert "pull_request.head" not in str(checkouts)
assert any("ci/feedback.py" in s.get("run", "") for s in job["steps"])
def test_gitlab_feedback_token_is_confined_to_trusted_default_branch_job():
evaluation = load("ci/gitlab/all-tiers.yml")
assert "SKILLEVALUATOR_GITLAB_TOKEN" not in str(evaluation)
paid = evaluation["skill-evaluation-native"]
assert paid["parallel"]["matrix"] == [{"SKILL_EVAL_TIER": ["3"]}]
assert evaluation["skill-evaluation-native-tier2"]["parallel"]["matrix"] == [{"SKILL_EVAL_TIER": ["2"]}]
assert paid["environment"]["name"] == "$SKILL_EVAL_ENVIRONMENT"
assert evaluation["skill-evaluation-aggregate"]["when"] == "always"
feedback = load("ci/gitlab/feedback.yml")["skill-evaluation-feedback"]
assert all("CI_DEFAULT_BRANCH" in r["if"] and "SKILL_EVAL_FEEDBACK" in r["if"] for r in feedback["rules"])
assert "SKILLEVALUATOR_GITLAB_TOKEN" in str(feedback)
def test_gitlab_tier1_runs_only_digest_pinned_legacy_image():
job = load("ci/gitlab/all-tiers.yml")["skill-evaluation"]
assert job["image"]["name"] == "$SKILL_EVAL_IMAGE"
assert job["image"]["entrypoint"] == [""]
assert "/opt/skillevaluator/ci/run_tier1.py" in str(job["script"])
assert "python -m pip install" not in str(job["script"])
def test_both_paid_tiers_are_advisory_even_for_legacy_required_callers():
workflow = load(".github/workflows/evaluate-all-tiers.yml")
legacy = workflow["on"]["workflow_call"]["inputs"]["tier3-required"]
assert legacy["type"] == "boolean"
assert legacy["default"] == "false"
assert "deprecated" in legacy["description"].lower()
for name in ("paid-tiers", "aggregate"):
job = workflow["jobs"][name]
assert job["env"]["SKILL_EVAL_TIER3_REQUIRED"] == "false"
assert "inputs.tier3-required" not in str(job)
assert workflow["jobs"]["paid-tiers"]["continue-on-error"] == "true"
def test_new_candidate_callers_do_not_expose_a_paid_tier_requirement():
github = load("ci/templates/all-tiers/skill-checks-all-tiers.yml")
assert "tier3-required" not in github["jobs"]["skills"]["with"]
for path in ("ci/templates/all-tiers/gitlab-all-tiers.yml", "ci/gitlab/all-tiers.yml"):
assert "SKILL_EVAL_TIER3_REQUIRED" not in load(path)["variables"]
def test_trusted_aggregate_receipt_is_captured_before_untrusted_artifact_read():
job = load(".github/workflows/evaluate-all-tiers.yml")["jobs"]["aggregate"]
assert job["permissions"]["id-token"] == "write"
first = job["steps"][0]
assert first["id"] == "provenance"
assert "ACTIONS_ID_TOKEN_REQUEST_URL" in first["run"]
assert "skill-evaluator-results-provenance-attempt-" in str(job["steps"])
def test_setup_configuration_changes_force_a_real_tier1_scan():
jobs = load(".github/workflows/evaluate-all-tiers.yml")["jobs"]
assert jobs["tier1"]["needs"] == "eligibility"
assert "needs.eligibility.outputs.scan-mode" in jobs["tier1"]["with"]["scan-mode"]
gate = str(jobs["eligibility"]["steps"])
assert "/files?per_page=100&page=" in gate
assert ".github/" in gate and ".gitlab-ci.yml" in gate
gitlab = str(load("ci/gitlab/all-tiers.yml")["skill-evaluation"]["script"])
assert "from select_changed_skills import _git" in gitlab and 'print("all" if setup else "changed")' in gitlab
def test_gitlab_native_reports_are_outside_candidate_until_evaluation_completes():
script = str(load("ci/gitlab/all-tiers.yml")["skill-evaluation-native"]["script"])
assert "mktemp -d /tmp/skillevaluator-reports." in script
assert '--reports "$reports_dir"' in script
assert 'cp -R "$reports_dir/."' in script
def test_github_head_is_rechecked_after_environment_approval_before_provider_secrets():
steps = load(".github/workflows/evaluate-all-tiers.yml")["jobs"]["paid-tiers"]["steps"]
check = next(s for s in steps if s.get("name") == "Recheck current head after environment approval")
assert "secrets." not in str(check)
assert "current.get('head', {}).get('sha')" in check["run"]
assert "pulls/" in check["run"]
first_secret = next(i for i, s in enumerate(steps) if "secrets." in str(s))
assert steps.index(check) < first_secret
def test_provider_steps_cannot_run_when_post_approval_head_check_failed():
steps = load(".github/workflows/evaluate-all-tiers.yml")["jobs"]["paid-tiers"]["steps"]
for step in steps:
if "secrets." in str(step):
assert "steps.current-head.outputs.allowed == 'true'" in step["if"]
def test_candidate_templates_do_not_change_legacy_bundle_generation():
candidates = ROOT / "ci/templates/all-tiers"
assert (candidates / "skill-checks-all-tiers.yml").is_file()
assert not list((ROOT / "ci/templates").glob("*all-tiers.yml"))
def test_gitlab_privileged_bootstrap_uses_fixed_include_inputs():
source = (ROOT / "ci/gitlab/feedback.yml").read_text()
header = next(yaml.load_all(source, Loader=yaml.BaseLoader))
assert set(header["spec"]["inputs"]) == {"tools-ref", "tools-repository", "api-url", "project-id"}
job = load("ci/gitlab/feedback.yml")["skill-evaluation-feedback"]
script = str(job["script"])
assert "$SKILL_EVAL_TOOLS_REF" not in script
assert "$SKILL_EVAL_TOOLS_REPOSITORY" not in script
assert "$CI_API_V4_URL" not in script
assert "$[[ inputs.tools-ref ]]" in script
assert job["id_tokens"]["SKILL_EVAL_GITLAB_ID_TOKEN"]["aud"] == "urn:skillevaluator:feedback:v1"
def test_gitlab_feedback_request_uses_job_token_api_without_candidate_checkout():
job = load("ci/gitlab/all-tiers.yml")["skill-evaluation-request-feedback"]
assert "trigger" not in job
assert "environment" not in job
assert job["variables"] == {"GIT_STRATEGY": "none"}
assert job["extends"] == ".skill-evaluation-tools"
assert job["needs"] == [{"job": "skill-evaluation-aggregate", "artifacts": "false"}]
assert job["when"] == "always"
assert job["allow_failure"] == "true"
script = "\n".join(job["script"])
assert 'python -I "$SKILL_EVAL_TOOLS_DIR/ci/request_gitlab_feedback.py"' in script
assert "uv" not in script and "bootstrap" not in script
def test_gitlab_bridge_rules_do_not_disable_their_own_feedback_flag():
bridge = load("ci/gitlab/all-tiers.yml")["skill-evaluation-request-feedback"]
assert all("SKILL_EVAL_FEEDBACK" not in rule["if"] for rule in bridge["rules"])
def test_gitlab_feedback_bootstrap_isolates_process_configuration():
script = "\n".join(load("ci/gitlab/feedback.yml")["skill-evaluation-feedback"]["script"])
assert "export PATH=/usr/local/bin:/usr/bin:/bin" in script
assert "GIT_CONFIG_COUNT" in script and "GIT_EXEC_PATH" in script
assert "python -I -m pip --isolated install" in script
assert "env -i" in script
def test_gitlab_aggregate_enforces_required_gates_for_all_matrix_outcomes():
jobs = load("ci/gitlab/all-tiers.yml")
assert jobs["skill-evaluation-native"]["allow_failure"] == "true"
assert jobs["skill-evaluation-aggregate"].get("allow_failure", "false") == "false"
assert jobs["skill-evaluation-aggregate"]["when"] == "always"
def test_gitlab_tier2_has_no_docker_service_or_tls_settings():
jobs = load("ci/gitlab/all-tiers.yml")
tier2 = jobs["skill-evaluation-native-tier2"]
assert tier2["extends"] == "skill-evaluation-native"
assert tier2["services"] == []
assert tier2["variables"] == {
"DOCKER_HOST": "",
"DOCKER_TLS_CERTDIR": "",
"DOCKER_TLS_VERIFY": "",
"DOCKER_CERT_PATH": "",
}
def test_gitlab_native_caller_overrides_and_advisory_policy_apply_to_both_tiers():
from skillevaluator._ci_onboarding_all_tiers import _gitlab_inherited
jobs = load("ci/gitlab/all-tiers.yml")
jobs["skill-evaluation-native"].update({"timeout": "45 minutes", "tags": ["native-linux"], "services": []})
for name in ("skill-evaluation-native", "skill-evaluation-native-tier2"):
job = jobs[name]
assert _gitlab_inherited(jobs, job, "timeout") == "45 minutes"
assert _gitlab_inherited(jobs, job, "tags") == ["native-linux"]
assert _gitlab_inherited(jobs, job, "services") == []
assert _gitlab_inherited(jobs, job, "allow_failure") == "true"
assert _gitlab_inherited(jobs, job, "environment") == {"name": "$SKILL_EVAL_ENVIRONMENT", "action": "prepare"}
assert _gitlab_inherited(jobs, job, "rules")[0] == {"if": '$SKILL_EVAL_FEEDBACK == "true"', "when": "never"}
def test_gitlab_aggregate_downloads_evidence_from_both_advisory_tiers():
needs = load("ci/gitlab/all-tiers.yml")["skill-evaluation-aggregate"]["needs"]
assert needs == [
{"job": "skill-evaluation", "artifacts": "true", "optional": "true"},
{"job": "skill-evaluation-native", "artifacts": "true", "optional": "true"},
{"job": "skill-evaluation-native-tier2", "artifacts": "true", "optional": "true"},
{"job": "skill-evaluation-tier1-llm", "artifacts": "true", "optional": "true"},
]
def test_trusted_native_workflow_entrypoints_and_pip_ignore_consumer_import_paths():
for file in (
".github/workflows/evaluate-all-tiers.yml",
".github/workflows/feedback-all-tiers.yml",
"ci/gitlab/all-tiers.yml",
):
document = load(file)
jobs = document.get("jobs", document)
for job in jobs.values():
if not isinstance(job, dict):
continue
scripts = job.get("script", []) + [step["run"] for step in job.get("steps", []) if "run" in step]
for script in scripts:
for line in script.splitlines():
if "python " in line and "/opt/skillevaluator/ci/run_tier1.py" not in line:
assert "python -I " in line, (file, line)
if "-m pip" in line:
assert "-m pip --isolated install" in line, (file, line)
def test_candidate_caller_propagates_reusable_read_permissions():
candidate = load("ci/templates/all-tiers/skill-checks-all-tiers.yml")
assert candidate["permissions"]["actions"] == "read"
assert candidate["permissions"]["pull-requests"] == "read"
def test_all_tier_runtimes_sync_the_reviewed_lock_with_pinned_uv():
evaluation = load(".github/workflows/evaluate-all-tiers.yml")
paid_install = next(
s for s in evaluation["jobs"]["paid-tiers"]["steps"] if s.get("name", "").startswith("Install only")
)["run"]
assert "uv==0.13.8" in paid_install
assert "sync --locked --no-dev" in paid_install
assert '--extra "tier${TIER}"' in paid_install
assert 'evaluator/.venv/bin" >> "$GITHUB_PATH"' in paid_install
aggregate = "\n".join(s["run"] for s in evaluation["jobs"]["aggregate"]["steps"] if "run" in s)
assert "sync --locked --no-dev" in aggregate
assert "--default-index https://pypi.org/simple" in aggregate
feedback = "\n".join(
s["run"] for s in load(".github/workflows/feedback-all-tiers.yml")["jobs"]["feedback"]["steps"] if "run" in s
)
assert "--extra publisher" in feedback
for name in ("skill-evaluation-native", "skill-evaluation-aggregate"):
script = "\n".join(load("ci/gitlab/all-tiers.yml")[name]["script"])
assert "uv==0.13.8" in script
assert "sync --locked --no-dev" in script
assert '--project "$SKILL_EVAL_TOOLS_DIR"' in script
assert 'export PATH="$SKILL_EVAL_TOOLS_DIR/.venv/bin:$PATH"' in script
privileged = "\n".join(load("ci/gitlab/feedback.yml")["skill-evaluation-feedback"]["script"])
assert "env -i" in privileged and "--no-config" in privileged
assert "sync --locked --no-dev --extra publisher" in privileged
assert '"$tools_dir/.venv/bin/python" -I - ' in privileged
def test_github_native_and_aggregate_record_the_exact_tools_revision():
jobs = load(".github/workflows/evaluate-all-tiers.yml")["jobs"]
for name in ("paid-tiers", "aggregate"):
assert jobs[name]["env"]["SKILLEVALUATOR_TOOLS_REF"] == "${{ inputs.tools-ref }}"
def test_feedback_mutation_jobs_are_serialized_per_pr_source_without_dropping_pending_runs():
import re
github = load(".github/workflows/feedback-all-tiers.yml")["jobs"]["feedback"]
concurrency = github["concurrency"]
# The default `queue: single` cancels an already pending run whenever another joins the group.
assert concurrency["cancel-in-progress"] == "false"
assert concurrency["queue"] == "max"
assert set(concurrency) == {"group", "cancel-in-progress", "queue"}
assert "github.event.workflow_run.head_repository.full_name == github.repository" in github["if"]
def rendered(repository, head_repository, head_branch, run_id, head_sha):
# Unknown expressions raise KeyError; `workflow_run.pull_requests` is empty for fork PRs.
event = {
"github.repository": repository,
"github.event.workflow_run.head_repository.full_name": head_repository,
"github.event.workflow_run.head_branch": head_branch,
"github.event.workflow_run.id": run_id,
"github.event.workflow_run.head_sha": head_sha,
}
return re.sub(r"\$\{\{ ([^}]+) \}\}", lambda match: event[match.group(1)], concurrency["group"])
first = rendered("acme/skills", "acme/skills", "feature-a", "101", "a" * 40)
# Later producer runs for the same PR source serialize behind the comment update in progress.
assert rendered("acme/skills", "acme/skills", "feature-a", "102", "b" * 40) == first
# Other PRs, repositories and head repositories never compete for one pending slot.
assert rendered("acme/skills", "acme/skills", "feature-b", "103", "c" * 40) != first
assert rendered("acme/other", "acme/other", "feature-a", "104", "a" * 40) != first
assert rendered("acme/skills", "fork/skills", "feature-a", "105", "a" * 40) != first
assert load("ci/gitlab/feedback.yml")["skill-evaluation-feedback"]["resource_group"] == "skillevaluator-feedback"
def test_gitlab_dind_bootstrap_installs_client_compose_and_buildx_without_a_second_daemon():
script = "\n".join(load("ci/gitlab/all-tiers.yml")["skill-evaluation-native"]["script"])
assert "apt-get install -y --no-install-recommends docker-cli docker-compose docker-buildx" in script
assert "--no-install-recommends docker.io" not in script
assert "docker compose version" in script
assert "docker buildx version" in script
assert script.index("bootstrap_tiers.py") < script.index('run_tiers.py" evaluate')
def test_gitlab_shared_native_timeout_is_a_literal_duration():
config = load("ci/gitlab/all-tiers.yml")
assert config["skill-evaluation-native"]["timeout"] == "5 hours"
assert "SKILL_EVAL_JOB_TIMEOUT" not in config["variables"]
candidate = load("ci/templates/all-tiers/gitlab-all-tiers.yml")
assert candidate["skill-evaluation-native"]["timeout"] == "5 hours"
def test_gitlab_setup_selector_uses_pinned_safe_git_reader():
script = "\n".join(load("ci/gitlab/all-tiers.yml")["skill-evaluation"]["script"])
assert 'ci_changed="$(git diff' not in script
assert 'sys.path.insert(0, "/opt/skillevaluator/ci")' in script
assert "from select_changed_skills import _git" in script
assert "--name-only" in script and '"-z"' in script and "--no-renames" in script
assert 're.fullmatch(r"[0-9a-f]{40}", value)' in script
def test_gitlab_read_token_is_removed_before_provider_bootstrap_and_feedback_is_advisory():
config = load("ci/gitlab/all-tiers.yml")
script = "\n".join(config["skill-evaluation-native"]["script"])
assert (
script.index("eligible_gitlab.py")
< script.index("unset SKILLEVALUATOR_GITLAB_READ_TOKEN")
< script.index("-m pip")
)
assert config["skill-evaluation-request-feedback"]["allow_failure"] == "true"
def _gitlab_minutes(duration):
import re
match = re.fullmatch(r"(\d+) (minutes|hours)", duration)
assert match, duration
return int(match[1]) * (60 if match[2] == "hours" else 1)
def test_gitlab_evaluator_budget_ends_before_the_native_job_timeout():
for path in ("ci/gitlab/all-tiers.yml", "ci/templates/all-tiers/gitlab-all-tiers.yml"):
config = load(path)
minutes = _gitlab_minutes(config["skill-evaluation-native"]["timeout"])
assert "$" not in config["skill-evaluation-native"]["timeout"]
reserve = min(20, minutes // 4)
assert config["variables"]["SKILL_EVAL_TIMEOUT_SECONDS"] == str((minutes - reserve) * 60) == "16800", path
def test_gitlab_dind_daemon_requires_tls_client_certificates():
job = load("ci/gitlab/all-tiers.yml")["skill-evaluation-native"]
assert job["services"] == [{"name": "docker:28.5.1-dind", "alias": "docker"}]
variables = job["variables"]
assert variables["DOCKER_HOST"] == "tcp://docker:2376"
assert variables["DOCKER_TLS_CERTDIR"] == "/certs"
assert variables["DOCKER_TLS_VERIFY"] == "1"
assert variables["DOCKER_CERT_PATH"] == "/certs/client"
assert "2375" not in str(job)
docs = " ".join((ROOT / "ci/templates/all-tiers/README.md").read_text().split())
assert 'volumes = ["/certs/client"]' in docs
def test_gitlab_native_jobs_use_environment_without_recording_deployments():
job = load("ci/gitlab/all-tiers.yml")["skill-evaluation-native"]
assert job["environment"] == {"name": "$SKILL_EVAL_ENVIRONMENT", "action": "prepare"}
docs = " ".join((ROOT / "ci/templates/all-tiers/README.md").read_text().split())
assert "`environment: action: prepare`" in docs and "Prevent outdated deployment jobs" in docs
def test_gitlab_feedback_job_uses_its_protected_environment_without_recording_deployments():
# Scoped publisher token and deploy access still apply; no deployment means
# "Prevent outdated deployment jobs" cannot drop an older MR's feedback.
job = load("ci/gitlab/feedback.yml")["skill-evaluation-feedback"]
assert job["environment"] == {"name": "skillevaluator-feedback", "action": "prepare"}
# The publisher authenticates its job, pipeline and branch, never a deployment record.
assert "/deployments" not in (ROOT / "ci/feedback.py").read_text()
docs = " ".join((ROOT / "ci/templates/all-tiers/README.md").read_text().split())
assert "The native jobs and the trusted feedback job use `environment: action: prepare`" in docs
# Some GitLab versions turn prepare jobs on an environment with approval
# rules into manual jobs, so deploy access is the only safe restriction.
assert "Deployment approval rules apply only to jobs that create deployments" not in docs
assert "Do not add deployment approval rules to these environments" in docs
assert "turn them into manual jobs" in docs and "`prevent_blocking_non_deployment_jobs`" in docs
assert "Restrict who can run them with the protected environments' deploy access" in docs
onboarding = " ".join((ROOT / "ci/templates/all-tiers/README.md").read_text().split())
assert "so deployment approvals do not gate them" not in onboarding
assert "Do not add deployment approval rules to their environments" in onboarding
assert "turn them into manual jobs" in onboarding
def test_gitlab_feedback_pipeline_guard_is_offered_and_documented():
guard = {"if": '$SKILL_EVAL_FEEDBACK == "true"', "when": "never"}
hidden = load("ci/gitlab/all-tiers.yml")[".skill-evaluation-not-feedback"]
assert hidden["rules"][0] == guard
assert hidden["rules"][1] == {"when": "on_success"}
for path in ("ci/templates/all-tiers/README.md", "ci/templates/all-tiers/gitlab-all-tiers.yml"):
text = (ROOT / path).read_text()
assert "- if: '$SKILL_EVAL_FEEDBACK == \"true\"'" in text, path
docs = " ".join((ROOT / "ci/templates/all-tiers/README.md").read_text().split())
assert "extends: .skill-evaluation-not-feedback" in docs
assert "no comment is posted or updated" in docs
assert "Merge these guards to the default branch before the setup MR" in docs
# Included jobs use stage test, so a merged root's stages must keep it.
assert "If the root defines `stages`, keep `test` in the list" in docs
assert "neither `rules` nor `only`/`except`" in docs
# Guards merge before the setup MR, while only all-tiers.yml defines the
# hidden job, so they use rules; a job without rules needs a fallback rule.
assert "gets the guard followed by `- when: on_success`" in docs
assert "- when: on_success" in (ROOT / "ci/templates/all-tiers/README.md").read_text()
assert "Write these guards as rules" in docs
assert "until the setup MR merges, `extends: .skill-evaluation-not-feedback` fails validation" in docs
assert ".skill-evaluation-not-feedback" not in (ROOT / "ci/gitlab/skill-evaluation.yml").read_text()
template = " ".join((ROOT / "ci/templates/all-tiers/gitlab-all-tiers.yml").read_text().split())
assert "If the root defines stages, keep test" in template
assert "gets the guard followed by `- when: on_success`" in template
assert "Write the guards as rules" in template
assert "added after this file is on the default branch" in template
# The upgrade path must order the guards before the setup MR as well.
onboarding = " ".join((ROOT / "ci/templates/all-tiers/README.md").read_text().split())
guards = onboarding.index("merge those guards to the default branch")
assert guards < onboarding.index("Then, in your setup MR")
assert "keep `test` in any `stages` list" in onboarding
def test_pre_merge_paid_tier_advice_does_not_route_through_ci_demo():
# `ci demo` needs the scan caller on the default branch (the onboarding tests
# cover the rejection), so the docs must not offer it for an unmerged setup.
docs = " ".join((ROOT / "ci/templates/all-tiers/README.md").read_text().split())
assert "(`ci demo` opens one)" not in docs
assert "`ci demo` cannot exercise an unmerged setup" in docs
assert "rejects a setup that has not merged" in docs
assert "Use `ci demo` after the setup merges." in docs
advanced = " ".join((ROOT / "docs/advanced-ci.mdx").read_text().split())
assert "`skillevaluator ci demo --tiers 1,2,3` works only after the setup has merged." in advanced
def _heredoc(script, marker):
return script.split(f"<<'{marker}'\n", 1)[1].split(f"\n{marker}\n", 1)[0] + "\n"
def _setup_literals(source):
"""Return the prefixes and exact paths that an inline setup predicate checks."""
import ast
prefixes, files = set(), set()
for node in ast.walk(ast.parse(source)):
if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and node.func.attr == "startswith":
prefixes.update(ast.literal_eval(node.args[0]))
elif isinstance(node, ast.Compare) and isinstance(node.left, ast.Name) and node.left.id == "path":
assert all(isinstance(op, ast.Eq) for op in node.ops)
files.update(ast.literal_eval(value) for value in node.comparators)
text = {value.decode() if isinstance(value, bytes) else value for value in prefixes}
return text, {value.decode() if isinstance(value, bytes) else value for value in files}
def test_tier1_setup_predicates_equal_the_shared_native_predicate():
from ci.select_changed_skills import SETUP_FILES, SETUP_PREFIXES
jobs = load(".github/workflows/evaluate-all-tiers.yml")["jobs"]
github = _heredoc(jobs["eligibility"]["steps"][0]["run"], "PY")
gitlab = _heredoc("\n".join(load("ci/gitlab/all-tiers.yml")["skill-evaluation"]["script"]), "PYTHON")
for source in (github, gitlab):
assert _setup_literals(source) == (set(SETUP_PREFIXES), set(SETUP_FILES))
# GitHub also widens when the PR file listing is too long to certify; the
# paid tiers take that decision instead of recomputing a different one.
paid = jobs["paid-tiers"]
assert paid["env"]["SKILL_EVAL_SCAN_MODE"] == jobs["tier1"]["with"]["scan-mode"]
assert "needs.eligibility.outputs.scan-mode" in paid["env"]["SKILL_EVAL_SCAN_MODE"]
def test_gitlab_tier1_and_native_tiers_select_the_same_skills(tmp_path):
import os
import subprocess
import sys
from ci import run_tiers
heredoc = _heredoc("\n".join(load("ci/gitlab/all-tiers.yml")["skill-evaluation"]["script"]), "PYTHON")
heredoc = heredoc.replace('"/opt/skillevaluator/ci"', repr(str(ROOT / "ci")))
cases = {
"skill only": ([("skills/alpha/SKILL.md", "# alpha v2\n")], []),
"consumer ci script": ([("ci/build.sh", "echo build\n"), ("skills/alpha/SKILL.md", "# alpha v3\n")], []),
"workflow": ([(".github/workflows/skills.yml", "name: skills\n")], []),
"root pipeline": ([(".gitlab-ci.yml", "include: []\n")], []),
"renamed feedback include": ([], [(".gitlab/skill-feedback.yml", "docs/skill-feedback.yml")]),
"ignored setup gitlink": ([("skills/alpha/SKILL.md", "# alpha with setup gitlink\n")], []),
"visible setup gitlink": ([("skills/alpha/SKILL.md", "# alpha with setup gitlink\n")], []),
}
for name, (writes, renames) in cases.items():
repo = tmp_path / name.replace(" ", "-")
repo.mkdir()
git = ["git", "-C", str(repo), "-c", "user.name=CI", "-c", "[email protected]"]
subprocess.run([*git, "init", "-q", "-b", "main"], check=True)
for skill in ("alpha", "beta"):
(repo / "skills" / skill).mkdir(parents=True)
(repo / "skills" / skill / "SKILL.md").write_text(f"# {skill}\n")
(repo / ".gitlab").mkdir()
(repo / ".gitlab/skill-feedback.yml").write_text("include: []\n" * 20)
gitlink = name in {"ignored setup gitlink", "visible setup gitlink"}
if gitlink:
ignore = "all" if name == "ignored setup gitlink" else "none"
(repo / ".gitmodules").write_text(
f'[submodule "actions"]\n path = .github/actions\n url = ./actions\n ignore = {ignore}\n'
)
subprocess.run([*git, "add", "-A"], check=True)
if gitlink:
subprocess.run([*git, "update-index", "--add", "--cacheinfo", f"160000,{'a' * 40},.github/actions"], check=True)
subprocess.run([*git, "commit", "-qm", "base"], check=True)
base = subprocess.check_output([*git, "rev-parse", "HEAD"], text=True).strip()
for relative, text in writes:
(repo / relative).parent.mkdir(parents=True, exist_ok=True)
(repo / relative).write_text(text)
for source, target in renames:
(repo / target).parent.mkdir(parents=True, exist_ok=True)
subprocess.run([*git, "mv", source, target], check=True)
subprocess.run([*git, "add", "-A"], check=True)
if gitlink:
subprocess.run([*git, "update-index", "--add", "--cacheinfo", f"160000,{'b' * 40},.github/actions"], check=True)
subprocess.run([*git, "commit", "-qm", "head"], check=True)
head = subprocess.check_output([*git, "rev-parse", "HEAD"], text=True).strip()
tier1 = subprocess.run(
[sys.executable, "-I", "-"],
input=heredoc,
env={"PATH": os.environ["PATH"], "DIFF_BASE_SHA": base, "DIFF_HEAD_SHA": head, "CI_PROJECT_DIR": str(repo)},
capture_output=True,
text=True,
check=True,
).stdout.strip()
selected, selection = run_tiers.select_skills(
repo, {"SKILL_EVAL_PATHS": "skills", "DIFF_BASE_SHA": base, "REPORT_COMMIT": head}
)
native = "all" if selection.get("reason") == "ci_configuration_changed" else "changed"
assert tier1 == native, name
assert (native == "all") == (name not in {"skill only", "consumer ci script"}), name
if native == "changed":
assert [p.name for p in selected] == ["alpha"], name
else:
assert [p.name for p in selected] == ["alpha", "beta"], name