Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# ruff: noqa: INP001, T201 -- standalone CI entrypoint with escaped diagnostic output.
"""Run complete Tier 1 checks on configured repository skills and retain evidence.
Raw scanner output belongs in artifacts. Structured findings are escaped before
being shown in logs or GitHub annotations. SCAN_WORKERS (default 4) bounds how
many skills are scanned at once; rows, logs and annotations keep skill order.
"""
from __future__ import annotations
import argparse
import contextlib
import functools
import json
import os
import queue
import re
import signal
import stat
import subprocess
import tempfile
import threading
import time
from pathlib import Path
from skillevaluator.reporting.html import _validated_ci_context
from skillevaluator.utils.secret_redaction import Redactor
try:
from security_baselines import BaselineConfigurationError, load_security_baselines
except ModuleNotFoundError:
from ci.security_baselines import BaselineConfigurationError, load_security_baselines
try:
from secret_redaction import collect_sensitive_values, sanitized_outputs
except ModuleNotFoundError:
from ci.secret_redaction import collect_sensitive_values, sanitized_outputs
try:
from native_reports import render_native_reports
except ModuleNotFoundError:
from ci.native_reports import render_native_reports
try:
from publish_reports import PublicationError, validate_report
except ModuleNotFoundError:
from ci.publish_reports import PublicationError, validate_report
try:
from report_views import finding_repository_path, render_reports
except ModuleNotFoundError:
from ci.report_views import finding_repository_path, render_reports
try:
from select_changed_skills import (
SelectionError,
has_control_character,
select_changed_skills,
skill_root,
skill_roots,
)
except ModuleNotFoundError:
from ci.select_changed_skills import (
SelectionError,
has_control_character,
select_changed_skills,
skill_root,
skill_roots,
)
MAX_SKILLS = 100
MAX_REPORT_BYTES = 32 * 1024 * 1024
SCAN_TIMEOUT_SECONDS = 3000
# An interrupted run asks running scans to stop, then kills them after this.
STOP_GRACE_SECONDS = 5
# How often a running scan checks whether the run is ending.
POLL_SECONDS = 0.1
DEFAULT_SCAN_WORKERS = 4
MAX_SCAN_WORKERS = 16
PROVIDER_KEYS = {"openai": "OPENAI_API_KEY", "anthropic": "ANTHROPIC_API_KEY", "nv_build": "NVIDIA_API_KEY"}
PROCESS_ENV = {
"PATH",
"LANG",
"LC_ALL",
"LC_CTYPE",
"TMPDIR",
"SSL_CERT_FILE",
"SSL_CERT_DIR",
"REQUESTS_CA_BUNDLE",
"CURL_CA_BUNDLE",
"HTTP_PROXY",
"HTTPS_PROXY",
"NO_PROXY",
"http_proxy",
"https_proxy",
"no_proxy",
}
class ConfigurationError(ValueError):
"""An invalid request, without echoing untrusted paths or credentials."""
def discover_skills(workspace: Path, roots: str) -> list[Path]:
"""Discover a bounded unique set from SKILL_PATHS; linked scan content is rejected."""
workspace = workspace.resolve(strict=True)
return skill_trees(workspace, skill_roots(workspace, roots, ConfigurationError))
def skill_trees(workspace: Path, roots: list[str]) -> list[Path]:
"""Walk validated relative roots and return the skills found under them."""
discovered: set[Path] = set()
for relative in roots:
root = workspace / skill_root(workspace, relative, ConfigurationError)
if not root.is_dir() or not root.resolve(strict=True).is_relative_to(workspace):
raise ConfigurationError("A configured skill root is missing or outside the workspace.")
def inaccessible(_error):
raise ConfigurationError("A skill directory could not be read.")
for directory, directories, files in os.walk(root, followlinks=False, onerror=inaccessible):
parent = Path(directory)
inside_skill = any(parent.is_relative_to(skill) for skill in discovered) or "SKILL.md" in files
for name in [*directories, *files]:
child = parent / name
if has_control_character(name):
raise ConfigurationError("Skill paths must not contain control characters.")
if child.is_symlink() and (inside_skill or name == "SKILL.md"):
raise ConfigurationError("Symlinks are not supported in skill roots or scan content.")
if inside_skill and not child.is_dir() and not child.is_file():
raise ConfigurationError("Only regular files and directories are supported in skill trees.")
if "SKILL.md" in files:
discovered.add(parent)
if len(discovered) > MAX_SKILLS:
raise ConfigurationError("The CI integration supports at most 100 skills per run.")
if not discovered:
raise ConfigurationError("No SKILL.md files were found under the configured skill roots.")
return sorted(discovered, key=lambda item: item.relative_to(workspace).as_posix())
def child_environment(environ: dict[str, str]) -> tuple[dict[str, str], bool]:
"""Limit provider selection to the explicit CI inputs, including static runs."""
enabled = environ.get("LLM_ENABLED", "false").strip().lower()
if enabled not in {"true", "false"}:
raise ConfigurationError("LLM_ENABLED must be true or false.")
env = {key: value for key, value in environ.items() if key in PROCESS_ENV}
env.setdefault("PATH", os.defpath)
env["HOME"] = "/tmp/evaluator-home"
env["PYTHONDONTWRITEBYTECODE"] = "1"
env["PYTHON_DOTENV_DISABLED"] = "1"
env["SEMGREP_SEND_METRICS"] = "off"
env["SEMGREP_ENABLE_VERSION_CHECK"] = "0"
# The complete CI runtime requires Semgrep even when local installs make it optional.
env["SKILLEVALUATOR_REQUIRE_SEMGREP"] = "1"
env["NO_COLOR"] = "1"
Path(env["HOME"]).mkdir(parents=True, exist_ok=True)
if enabled == "true":
provider = environ.get("LLM_PROVIDER", "").strip()
if provider not in PROVIDER_KEYS:
raise ConfigurationError("LLM_PROVIDER must be openai, anthropic, or nv_build when LLM is enabled.")
key = environ.get("LLM_API_KEY", "").strip()
if not key:
raise ConfigurationError("LLM_API_KEY is required when LLM_ENABLED is true.")
env["SKILL_EVAL_LLM_PROVIDER"] = provider
env[PROVIDER_KEYS[provider]] = key
model = environ.get("LLM_MODEL", "").strip()
if model:
if any(ord(character) < 32 for character in model):
raise ConfigurationError("LLM_MODEL must not contain control characters.")
env["SKILL_EVAL_LLM_MODEL"] = model
return env, enabled == "true"
def scan_workers(environ: dict[str, str]) -> int:
"""Read the bound on concurrent skill scans; 1 scans one skill at a time."""
value = environ.get("SCAN_WORKERS", "").strip() or str(DEFAULT_SCAN_WORKERS)
if not re.fullmatch(r"[0-9]{1,2}", value) or not 1 <= int(value) <= MAX_SCAN_WORKERS:
raise ConfigurationError(f"SCAN_WORKERS must be a whole number from 1 to {MAX_SCAN_WORKERS}.")
return int(value)
def worker_environment(env: dict[str, str], worker: int) -> dict[str, str]:
"""Give each scan worker its own private home and temporary directory.
The paths are fixed, so outside Docker another local user could create
them first in a shared /tmp. An existing directory is used only if this
user owns it and no one else has access to it.
"""
isolated = dict(env)
isolated["HOME"] = str(Path(env["HOME"]) / f"worker-{worker}")
isolated["TMPDIR"] = str(Path(env.get("TMPDIR") or tempfile.gettempdir()) / f"skillevaluator-worker-{worker}")
for name in ("HOME", "TMPDIR"):
directory = Path(isolated[name])
directory.mkdir(mode=0o700, parents=True, exist_ok=True)
info = directory.lstat()
private = os.name != "posix" or (info.st_uid == os.getuid() and not info.st_mode & 0o077)
if not stat.S_ISDIR(info.st_mode) or not private:
raise ConfigurationError("Scan worker directories must be private directories owned by this user.")
return isolated
def _write_json(path: Path, payload: dict) -> None:
temporary = path.with_suffix(path.suffix + ".tmp")
with temporary.open("x", encoding="utf-8") as handle:
json.dump(payload, handle, indent=2, ensure_ascii=True, allow_nan=False)
handle.write("\n")
temporary.replace(path)
def _load_report(path: Path) -> dict:
if path.is_symlink() or not path.is_file() or path.stat().st_size > MAX_REPORT_BYTES:
raise ValueError("The evaluator did not produce a readable JSON report.")
data = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(data, dict):
raise ValueError("The evaluator JSON report is malformed.")
if type(data.get("overall_passed")) is not bool:
raise ValueError("The evaluator JSON report lacks a boolean verdict.")
if data.get("overall_status") not in {"passed", "failed", "incomplete"}:
raise ValueError("The evaluator JSON report lacks a recognized status.")
scans = data.get("incomplete_scans")
if not isinstance(scans, list) or any(not isinstance(scan, str) for scan in scans):
raise ValueError("The evaluator JSON report lacks scanner completeness evidence.")
results = data.get("results")
if not isinstance(results, list) or not results or any(not isinstance(result, dict) for result in results):
raise ValueError("The evaluator JSON report lacks validator results.")
for name in ("total_validators", "total_errors", "total_warnings"):
if type(data.get(name)) is not int or data[name] < 0:
raise ValueError("The evaluator JSON report has invalid result counts.")
if data["total_validators"] != len(results):
raise ValueError("The evaluator JSON report has inconsistent validator counts.")
counts = data.get("severity_counts")
if not isinstance(counts, dict) or any(
type(counts.get(name)) is not int or counts[name] < 0 for name in ("critical", "high", "medium", "low")
):
raise ValueError("The evaluator JSON report has invalid severity counts.")
from skillevaluator.reporting.retained import retained_results
# Validation only: the results are discarded, so skip copying their metadata.
retained_results(data, copy_metadata=False)
return data
def _summary(skills: list[dict], error: str | None = None) -> dict:
passed = sum(item["passed"] for item in skills)
incomplete = sum(item["overall_status"] in {"incomplete", "error"} for item in skills)
payload = {
"total": len(skills),
"passed": passed,
"failed": len(skills) - passed,
"incomplete": incomplete,
"overall_passed": bool(skills) and passed == len(skills) and error is None,
"overall_status": "error"
if error
else "failed"
if any(item["overall_status"] == "failed" for item in skills)
else "incomplete"
if incomplete
else "passed"
if passed == len(skills) and skills
else "failed",
"total_validators": sum(item.get("total_validators", 0) for item in skills),
"total_errors": sum(item.get("total_errors", 0) for item in skills),
"total_warnings": sum(item.get("total_warnings", 0) for item in skills),
"severity_counts": {
name: sum(item.get("severity_counts", {}).get(name, 0) for item in skills)
for name in ("critical", "high", "medium", "low")
},
"skills": skills,
}
if error:
payload["error"] = error
return payload
def _write_summary(reports: Path, summary: dict) -> None:
_write_json(reports / "summary.json", summary)
render_native_reports(reports, summary, summary.get("execution", {}))
def annotation_text(value: str, *, property_value: bool = False) -> str:
"""Escape GitHub command syntax; finding text must never create commands."""
escaped = value.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A")
return escaped.replace(":", "%3A").replace(",", "%2C") if property_value else escaped
def log_result(
row: dict,
data: dict | None,
*,
annotations: bool,
budget: dict[str, int],
enforcement_mode: str = "strict",
) -> None:
"""Explain verdicts promptly without replaying arbitrary subprocess logs."""
print(f"Skill result: {json.dumps(row['path'])} — {row['overall_status'].upper()}", flush=True)
if not data:
print(f" Reason: {json.dumps(row.get('error', 'No complete report was produced.'))}", flush=True)
return
for quality in data.get("quality_summary", []):
print(
f" Quality: {json.dumps(quality.get('overall_score', 'unknown'))}/100, grade {json.dumps(quality.get('grade'))}",
flush=True,
)
shown = 0
for result in data.get("results", []):
print(f" Check: {json.dumps(result.get('validator'))} — {json.dumps(result.get('status'))}", flush=True)
if result.get("incomplete_scans"):
print(f" Incomplete scanners: {json.dumps(result['incomplete_scans'])}", flush=True)
for finding in result.get("findings", []):
severity = str(finding.get("severity", "")).lower()
if severity not in {"critical", "high"} or shown >= 20:
continue
shown += 1
message = str(finding.get("message", "Finding details unavailable."))[:4000]
suggestion = str(finding.get("suggestion") or "Consult the full report.")[:2000]
location = str(finding.get("file_path") or row["path"])
print(f" Failure: {json.dumps(message)}", flush=True)
print(f" Location: {json.dumps(location)}", flush=True)
print(f" Suggested fix: {json.dumps(suggestion)}", flush=True)
if annotations and result.get("passed") is False and budget["errors"] < 10:
budget["errors"] += 1
annotation_level = "warning" if enforcement_mode == "advisory" else "error"
properties = "title=" + annotation_text(str(result.get("validator", "Tier 1")), property_value=True)
relative = finding_repository_path(row["path"], finding.get("file_path"))
if relative:
properties += ",file=" + annotation_text(relative, property_value=True)
line = finding.get("line_number")
if type(line) is int and line > 0:
properties += f",line={line}"
print(
f"::{annotation_level} {properties}::{annotation_text(message + ' Suggested fix: ' + suggestion)}",
flush=True,
)
if result.get("status") in {"incomplete", "failed"}:
for message in result.get("legacy", {}).get("errors", [])[:3]:
print(f" Scanner error: {json.dumps(str(message)[:4000])}", flush=True)
def execution_context(environ: dict[str, str]) -> dict:
return {
"run_url": environ.get("REPORT_RUN_URL", ""),
"commit": environ.get("REPORT_COMMIT", ""),
"image": environ.get("REPORT_IMAGE", ""),
"report_url": environ.get("REPORT_URL", ""),
"llm_enabled": environ.get("LLM_ENABLED", "false").lower() == "true",
"workflow_sha": environ.get("CALLER_WORKFLOW_SHA", ""),
}
def _signal_scan(process: subprocess.Popen, *, kill: bool) -> None:
"""Signal an evaluator's process group, which also holds the scanner tools it started."""
if os.name != "posix":
if process.poll() is None:
if kill:
process.kill()
else:
process.terminate()
return
try:
os.killpg(process.pid, signal.SIGKILL if kill else signal.SIGTERM)
except ProcessLookupError:
pass
except PermissionError:
# macOS reports EPERM instead of ESRCH once the group leader has exited.
if process.poll() is None:
raise
def run_scan(
argv: list[str], *, abort: threading.Event, timeout: float, check: bool = False, **options
) -> subprocess.CompletedProcess:
"""Run one evaluator process like subprocess.run, and end it early when the run aborts.
Scan worker threads never see Ctrl-C, and Docker sends it only to this
runner (PID 1), so the runner ends running scans itself. The evaluator
leads its own process group, so a timeout or abort also ends its tools.
"""
process = subprocess.Popen(argv, start_new_session=os.name == "posix", **options)
deadline = time.monotonic() + timeout
grace_ends = None
try:
while True:
now = time.monotonic()
if now >= deadline:
raise subprocess.TimeoutExpired(argv, timeout)
if abort.is_set() and grace_ends is None:
grace_ends = now + STOP_GRACE_SECONDS
_signal_scan(process, kill=False)
if grace_ends is not None and now >= grace_ends:
break
try:
process.wait(timeout=min(POLL_SECONDS, deadline - now))
break
except subprocess.TimeoutExpired:
pass
finally:
if process.returncode is None or grace_ends is not None:
# Also end scanner tools that outlived an aborted evaluator.
_signal_scan(process, kill=True)
if process.returncode is None:
process.wait()
if check and process.returncode:
raise subprocess.CalledProcessError(process.returncode, argv)
return subprocess.CompletedProcess(argv, process.returncode)
def evaluate_skill(
target: Path,
output: Path,
*,
workspace: Path,
env: dict[str, str],
llm: bool,
min_score: str,
timeout: float,
execute,
tokens: set[str],
baseline: dict | None = None,
) -> tuple[dict, dict | None, bool]:
"""Run one evaluator process; return row fields, its report and whether time ran out.
Scan workers call this concurrently. Raw scanner output stays in a private
directory under the worker's TMPDIR; only redacted files reach the skill's
own report directory. An ``error`` field of None means the row has no error.
"""
fields: dict = {"error": "Evaluation interrupted before complete evidence was retained."}
data = None
try:
output.mkdir()
# Raw scanner files and logs live only in a private temporary
# directory, never in the upload directory. Docker mounts /audit-tmp
# as tmpfs, so cancellation cannot upload half-written secrets. The
# name carries the report directory so concurrent scans are traceable.
with (
tempfile.TemporaryDirectory(
prefix=f"skillevaluator-private-{output.name}-", dir=env.get("TMPDIR")
) as temporary,
tempfile.TemporaryDirectory(prefix="skillevaluator-policy-", dir=env.get("TMPDIR"))
if baseline is not None
else contextlib.nullcontext() as policy_directory,
):
raw = Path(temporary).resolve(strict=True)
if raw.is_relative_to(workspace) or raw.is_relative_to(output.parent):
raise ValueError("A private scanner directory must be outside the workspace and reports.")
argv = [
"skillevaluator",
"tier1",
str(target),
"--llm" if llm else "--no-llm",
"--profile",
"default",
"--min-score",
min_score,
# native_reports renders each skill's HTML from the retained JSON
# and replaces any CLI copy, so the CLI does not render it.
"-r",
"json,markdown,sarif",
"-o",
str(raw),
]
if baseline is not None:
policy_path = Path(policy_directory) / Path(baseline["file"]).name
policy_path.write_bytes(baseline["content"])
policy_path.chmod(0o400)
argv.extend(["--skillspector-baseline", str(policy_path)])
with (raw / "stdout.log").open("xb") as stdout, (raw / "stderr.log").open("xb") as stderr:
process = execute(
argv, cwd=workspace, env=env, stdout=stdout, stderr=stderr, check=False, timeout=timeout
)
safe = sanitized_outputs(raw, tokens, target_path=target.relative_to(workspace).as_posix())
for name, content in safe.items():
(output / name).write_bytes(content)
fields["exit_code"] = process.returncode
data = _load_report(output / "skillevaluator-tier1.json")
for key in (
"overall_status",
"incomplete_scans",
"total_validators",
"total_errors",
"total_warnings",
"severity_counts",
):
fields[key] = data[key]
fields["passed"] = (
process.returncode == 0
and data["overall_passed"] is True
and data["overall_status"] == "passed"
and not data["incomplete_scans"]
and all(
result.get("passed") is True
and result.get("status") not in {"incomplete", "failed"}
and not result.get("incomplete_scans")
for result in data["results"]
)
)
try:
# The publisher rejects the whole artifact for one contradictory
# report. Check it the same way here so only this skill is lost.
validate_report(data, fields)
consistent = True
except PublicationError:
consistent = False
if not consistent or (not fields["passed"] and fields["overall_status"] == "passed"):
fields["overall_status"] = "error"
fields["incomplete_scans"] = []
fields["passed"] = False
fields["error"] = "Process exit or validator results contradict the report verdict."
else:
fields["error"] = None
except subprocess.TimeoutExpired:
fields["error"] = "Scan time limit reached; this skill did not finish."
return fields, None, True
except (OSError, ValueError, UnicodeError, RecursionError):
fields["error"] = "Evaluator execution failed or its JSON report was missing, unreadable, or invalid."
return fields, data, False
def _set_enforcement(summary: dict, *, mode: str, downgraded: bool) -> None:
summary["enforcement"] = {"mode": mode, "downgraded": downgraded}
def run(
workspace: Path,
reports: Path,
environ: dict[str, str],
*,
execute=None,
clock=time.monotonic,
enforcement_mode: str = "strict",
) -> int:
"""Run selected skills; advisory mode only relaxes complete policy verdicts.
``execute`` is called like subprocess.run. The default, run_scan, also
ends running scans when the run stops early on an interrupt or error.
"""
if enforcement_mode not in {"strict", "advisory"}:
raise ConfigurationError("enforcement-mode must be strict or advisory.")
if reports.is_symlink():
raise ConfigurationError("The report directory must not be a symlink.")
reports.mkdir(parents=True, exist_ok=True)
if any(reports.iterdir()):
raise ConfigurationError("The report directory must be empty to avoid stale evidence.")
reports = reports.resolve(strict=True)
rows = []
context = execution_context(environ)
annotation_budget = {"errors": 0}
selection = None
tokens: set[str] = set()
try:
try:
_validated_ci_context(context)
if environ.get("REPORT_BASE_URL"):
_validated_ci_context({"report_url": environ["REPORT_BASE_URL"]})
except ValueError as exc:
# Invalid context must not crash rendering of the configuration error.
context = {"llm_enabled": context["llm_enabled"]}
raise ConfigurationError(
"Report links must be bounded HTTPS URLs without credentials or control characters."
) from exc
workspace = workspace.resolve(strict=True)
if reports.is_relative_to(workspace):
raise ConfigurationError("The report directory must be outside the read-only workspace.")
env, llm = child_environment(environ)
baselines, baseline_context, policy_tokens = load_security_baselines(workspace, environ)
tokens.update(policy_tokens)
if baseline_context is not None:
context["security_baselines"] = baseline_context
score = environ.get("MIN_SCORE", "70")
if not score.isascii() or not score.isdigit() or len(score) > 3 or int(score) > 100:
raise ConfigurationError("MIN_SCORE must be an integer between 0 and 100.")
min_score = str(int(score))
workers = scan_workers(environ)
mode = environ.get("SCAN_MODE", "all")
if mode not in {"all", "changed"}:
raise ConfigurationError("SCAN_MODE must be all or changed.")
roots = environ.get("SKILL_PATHS", "skills")
if mode == "changed":
selected, selection = select_changed_skills(
workspace, roots, environ.get("DIFF_BASE_SHA", ""), environ.get("DIFF_HEAD_SHA", "")
)
if selected:
# Validate scan trees without adding nested unchanged targets.
skill_trees(workspace, [path.relative_to(workspace).as_posix() for path in selected])
else:
selected = discover_skills(workspace, roots)
selection = {
"mode": "all",
"base": None,
"head": None,
"discovered": len(selected),
"selected": [path.relative_to(workspace).as_posix() for path in selected],
"deleted": [],
}
deadline = clock() + SCAN_TIMEOUT_SECONDS
try:
if llm:
provider_key = environ.get("LLM_API_KEY", "").strip()
if len(provider_key) < 4:
raise ValueError("The provider key is too short for safe redaction.")
tokens.add(provider_key)
# A credential found in one selected skill can also be echoed by
# another. Collect the union before writing or logging any paths.
for target in selected:
tokens.update(collect_sensitive_values(target))
if clock() >= deadline:
raise ValueError("Credential preflight exceeded the scan time limit.")
redactor = Redactor(tokens)
metadata = json.dumps(
{
"selection": selection,
"execution": context,
"baseline_files": [baseline["file"] for baseline in baselines.values()],
},
ensure_ascii=True,
)
if redactor.text(metadata) != metadata:
raise ValueError("A credential is present in report context.")
except (OSError, ValueError, UnicodeError, RecursionError) as exc:
# Neither a path nor a context field is safe to echo on failure.
selection = None
context = {"llm_enabled": context.get("llm_enabled", False)}
raise ConfigurationError(
"Credential preflight could not safely prepare the scan; no results were published."
) from exc
environments = [worker_environment(env, slot) for slot in range(1, min(workers, len(selected)) + 1)]
except (ConfigurationError, SelectionError, BaselineConfigurationError, OSError) as exc:
message = (
str(exc)
if isinstance(exc, (ConfigurationError, SelectionError, BaselineConfigurationError))
else "The workspace or runtime configuration could not be read."
)
summary = _summary([], error=message)
if isinstance(exc, SelectionError):
selection = exc.selection
if selection is not None:
summary["selection"] = selection
summary["execution"] = context
_set_enforcement(summary, mode=enforcement_mode, downgraded=False)
_write_summary(reports, summary)
print(f"SkillEvaluator configuration failed: {json.dumps(message)}", flush=True)
return 1
if selection["deleted"]:
print(f"Deleted skill directories skipped (removed completely): {json.dumps(selection['deleted'])}", flush=True)
if not selected:
summary = _summary([])
summary.update(overall_status="skipped", overall_passed=False, selection=selection, execution=context)
_set_enforcement(summary, mode=enforcement_mode, downgraded=False)
_write_summary(reports, summary)
print("NO SKILLS CHANGED: no skill was evaluated between the selected Git commits.", flush=True)
return 0
rows = [
{
"path": target.relative_to(workspace).as_posix(),
"report_dir": f"skill-{index:03d}",
"exit_code": None,
"overall_status": "error",
"incomplete_scans": [],
"passed": False,
"error": "Not evaluated: the scan has not reached this skill.",
}
for index, target in enumerate(selected, 1)
]
for row in rows:
if baseline := baselines.get(row["path"]):
row["security_baseline"] = {key: baseline[key] for key in ("file", "digest")}
def checkpoint() -> None:
summary = _summary(rows)
summary.update(selection=selection, execution=context)
_set_enforcement(summary, mode=enforcement_mode, downgraded=False)
_write_json(reports / "summary.json", summary)
render_reports(reports, summary, context)
checkpoint()
pending: queue.SimpleQueue[int] = queue.SimpleQueue()
for index in range(len(rows)):
pending.put(index)
events: queue.SimpleQueue[tuple[str, int, tuple | None]] = queue.SimpleQueue()
stop = threading.Event()
# Set when the run stops early on an interrupt or error: running scans end.
abort = threading.Event()
if execute is None:
execute = functools.partial(run_scan, abort=abort)
def ending() -> bool:
return stop.is_set() or abort.is_set()
def work(worker_env: dict[str, str]) -> None:
# Workers only run processes. The main thread owns rows, logs and
# report files, so nothing races and logs keep skill order.
try:
while not ending():
try:
index = pending.get_nowait()
except queue.Empty:
return
remaining = deadline - clock()
if remaining <= 0:
stop.set()
return
events.put(("started", index, None))
try:
outcome = evaluate_skill(
selected[index],
reports / rows[index]["report_dir"],
workspace=workspace,
env=worker_env,
llm=llm,
min_score=min_score,
timeout=remaining,
execute=execute,
tokens=tokens,
baseline=baselines.get(rows[index]["path"]),
)
except Exception:
outcome = ({"error": "Evaluator execution failed unexpectedly; no result is claimed."}, None, False)
if outcome[2]:
# The budget is spent: running scans end at the deadline.
stop.set()
saved = threading.Event()
events.put(("finished", index, (outcome, saved)))
# Start another skill only after this result is checkpointed.
# Once the run is ending no skill starts, and an interrupted
# run never checkpoints again, so stop waiting.
while not saved.wait(POLL_SECONDS):
if ending():
return
finally:
events.put(("exited", -1, None))
threads = [
threading.Thread(target=work, args=(worker_env,), name=f"skill-scan-{slot}")
for slot, worker_env in enumerate(environments, 1)
]
if len(threads) > 1:
print(f"SkillEvaluator: scanning {len(rows)} skills, up to {len(threads)} at a time.", flush=True)
started: set[int] = set()
announced: set[int] = set()
results: dict[int, dict | None] = {}
logged = 0
def flush_logs(*, final: bool = False) -> None:
# Print each skill's lines in skill order, whatever order scans end in,
# so logs and the annotation budget match a sequential run.
nonlocal logged
while logged < len(rows):
if logged in started:
if logged not in announced:
announced.add(logged)
print(
f"SkillEvaluator: evaluating skill {logged + 1} of {len(rows)}: {json.dumps(rows[logged]['path'])}",
flush=True,
)
if logged not in results:
return
# Drop each report once it is logged; only unlogged ones stay in memory.
log_result(
rows[logged],
results.pop(logged),
annotations=environ.get("GITHUB_ACTIONS") == "true",
budget=annotation_budget,
enforcement_mode=enforcement_mode,
)
elif not final:
return
logged += 1
running = 0
live: list[threading.Thread] = []
unsaved: list[threading.Event] = []
try:
for thread in threads:
thread.start()
live.append(thread)
running += 1
while running:
try:
batch = [events.get(timeout=POLL_SECONDS)]
except queue.Empty:
# Ctrl-C is acted on in this thread, but its signal handler can run on a
# scan thread, and that does not interrupt a wait without a time limit.
continue
while True:
try:
batch.append(events.get_nowait())
except queue.Empty:
break
for kind, index, payload in batch:
if kind == "exited":
running -= 1
elif kind == "started":
started.add(index)
rows[index]["error"] = "Evaluation interrupted before complete evidence was retained."
else:
(fields, data, _), saved = payload
for key, value in fields.items():
if key == "error" and value is None:
rows[index].pop("error", None)
else:
rows[index][key] = value
results[index] = data
unsaved.append(saved)
flush_logs()
if unsaved:
# One checkpoint covers every result in the batch.
checkpoint()
for saved in unsaved:
saved.set()
unsaved.clear()
except BaseException:
# Ctrl-C or an error such as a full report disk: end running scans
# now instead of waiting up to the whole scan budget for them.
abort.set()
raise
finally:
stop.set()
for saved in unsaved:
saved.set()
# Workers exit within the stop grace once the run ends. A repeated
# Ctrl-C must not leave them behind, so it is raised after they exit.
interrupt = None
for thread in live:
while thread.is_alive():
try:
thread.join()
except KeyboardInterrupt as exc:
abort.set()
interrupt = exc
if interrupt is not None:
raise interrupt
flush_logs(final=True)
if len(started) < len(rows):
print(
f"SkillEvaluator: scan time limit reached; {len(rows) - len(started)} skills were not started.",
flush=True,
)
summary = _summary(rows)
summary["selection"] = selection
summary["execution"] = context
downgraded = (
enforcement_mode == "advisory"
and summary["overall_status"] == "failed"
and all(
row["overall_status"] in {"passed", "failed"}
and not row["incomplete_scans"]
and row["exit_code"] in {0, 1}
and not row.get("error")
for row in rows
)
)
_set_enforcement(summary, mode=enforcement_mode, downgraded=downgraded)
_write_summary(reports, summary)
print(f"SkillEvaluator finished: {summary['passed']} of {summary['total']} skills passed.", flush=True)
# Reports have been validated and rendered. A failed headline can coexist
# with runtime errors or incomplete scanners, so inspect every skill too.
if downgraded:
print(
"::warning title=Tier 1 (advisory)::Tier 1 did not pass; enforcement-mode=advisory "
"keeps policy findings non-blocking. See the uploaded report.",
flush=True,
)
return 0
return 0 if summary["overall_passed"] else 1
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--workspace", type=Path, default=Path("/workspace"))
parser.add_argument("--reports", type=Path, default=Path("/reports"))
parser.add_argument("--enforcement-mode", choices=("strict", "advisory"), default="strict")
args = parser.parse_args()
try:
return run(args.workspace, args.reports, dict(os.environ), enforcement_mode=args.enforcement_mode)
except (ConfigurationError, OSError):
print("SkillEvaluator could not safely initialize the report directory.", flush=True)
return 1
if __name__ == "__main__":
raise SystemExit(main())