Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Test the CI adapter's boundaries; actual scanner coverage is tested by live CI."""
import contextlib
import errno
import gc
import importlib.util
import io
import json
import os
import re
import shutil
import signal
import stat
import subprocess
import sys
import threading
import time
import types
import weakref
import zipfile
from pathlib import Path
from subprocess import CompletedProcess
import pytest
from skillevaluator.models import Finding, ValidationResult
from skillevaluator.reporting.json_reporter import JSONReporter
MODULE_PATH = Path(__file__).parents[2] / "ci" / "run_tier1.py"
SPEC = importlib.util.spec_from_file_location("ci_run_tier1", MODULE_PATH)
runner = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(runner)
SELECTOR_SPEC = importlib.util.spec_from_file_location(
"ci_selection", MODULE_PATH.with_name("select_changed_skills.py")
)
selector = importlib.util.module_from_spec(SELECTOR_SPEC)
SELECTOR_SPEC.loader.exec_module(selector)
# Upper bound for cross-thread waits; generous for loaded CI runners.
WAIT = 60
def skill(workspace, path):
directory = workspace / path
directory.mkdir(parents=True)
(directory / "SKILL.md").write_text("---\nname: example\n---\n", encoding="utf-8")
return directory
def payload(passed=True, status="passed", incomplete=None, findings=1):
results = [
ValidationResult(
validator_name=f"Tier 1 fixture validator {index + 1}",
validator_description="Controlled CI process result fixture",
)
for index in range(12)
]
for result in results:
result.metadata["gating"] = {"tier": 1, "blocking": True}
for number in range(0 if passed else findings):
results[0].add_finding(
Finding(
category="SCHEMA",
severity="high",
check_name="fixture_failure",
message="Fixture validation failed" + (f" ({number + 1})" if number else ""),
file_path="SKILL.md",
)
)
if status == "incomplete":
results[0].metadata["incomplete_scans"] = incomplete or ["semgrep"]
if not passed and not findings:
results[0].add_error("Scanner execution did not complete")
return json.loads(JSONReporter(include_timestamp=False).render_all(results))
def report_dir(output: Path) -> str:
"""Name the skill report directory behind the private scanner directory the runner passes."""
match = re.fullmatch(r"skillevaluator-private-(skill-[0-9]{3})-.+", output.name)
assert match, output.name
return match.group(1)
def fake_cli(responses, calls):
# Responses follow skill order, not call order: skills may run in parallel.
def invoke(argv, **kwargs):
calls.append((argv, kwargs))
output = Path(argv[argv.index("-o") + 1])
exit_code, report = responses[int(report_dir(output).removeprefix("skill-")) - 1]
kwargs["stdout"].write(b"::error::untrusted scanner text\n")
if report is not None:
(output / "skillevaluator-tier1.json").write_text(json.dumps(report), encoding="utf-8")
return CompletedProcess(argv, exit_code)
return invoke
def test_selection_deduplicates_overlapping_roots(tmp_path):
first = skill(tmp_path, "skills/a")
second = skill(tmp_path, "skills/b")
assert runner.discover_skills(tmp_path, "skills\nskills/a\nskills") == [first, second]
@pytest.mark.parametrize("mode", ["strict", "advisory"])
@pytest.mark.parametrize(
"case",
["passed", "failed", "incomplete", "mixed", "mixed-error", "missing", "malformed", "crash", "config"],
)
def test_enforcement_only_downgrades_complete_policy_verdicts(tmp_path, capsys, mode, case):
workspace = tmp_path / "workspace"
skill(workspace, "skills/a")
responses = [(0, payload())]
if case in {"failed", "mixed", "mixed-error"}:
responses = [(1, payload(False, "failed"))]
if case in {"incomplete", "mixed"}:
incomplete = (1, payload(False, "incomplete", ["semgrep"], findings=0))
if case == "mixed":
skill(workspace, "skills/b")
responses.append(incomplete)
else:
responses = [incomplete]
if case == "missing":
responses = [(1, None)]
if case == "mixed-error":
skill(workspace, "skills/b")
responses.append((1, None))
if case == "malformed":
responses = [(1, {"overall_passed": False})]
if case == "crash":
responses = [(137, payload(False, "failed"))]
reports = tmp_path / "reports"
result = runner.run(
workspace,
reports,
{
"SKILL_PATHS": "missing" if case == "config" else "skills",
"LLM_ENABLED": "false",
"GITHUB_ACTIONS": "true",
},
execute=fake_cli(responses, []),
enforcement_mode=mode,
)
expected = 0 if case == "passed" or (mode == "advisory" and case == "failed") else 1
assert result == expected
summary = json.loads((reports / "summary.json").read_text())
assert summary["overall_passed"] is (case == "passed")
assert summary["enforcement"]["mode"] == mode
assert summary["enforcement"]["downgraded"] is (mode == "advisory" and case == "failed")
if case == "failed":
assert summary["overall_status"] == "failed"
assert (reports / "skill-001/skillevaluator-tier1.json").is_file()
assert (reports / "index.html").is_file()
output = capsys.readouterr().out
assert ("::warning title=Tier 1 (advisory)::" in output) is (mode == "advisory" and case == "failed")
@pytest.mark.parametrize("mode,marker", [("strict", "::error "), ("advisory", "::warning ")])
def test_annotation_level_follows_enforcement_mode(capsys, mode, marker):
row = {"path": "skills/a", "overall_status": "failed"}
data = {
"results": [
{
"validator": "Tier 1 fixture validator 1",
"status": "failed",
"passed": False,
"incomplete_scans": [],
"legacy": {"errors": []},
"findings": [
{
"severity": "HIGH",
"message": "Fixture validation failed",
"suggestion": "Fix the fixture",
"file_path": "SKILL.md",
"line_number": 1,
}
],
}
]
}
runner.log_result(row, data, annotations=True, budget={"errors": 0}, enforcement_mode=mode)
output = capsys.readouterr().out
assert marker in output
def test_advisory_render_failure_still_fails(tmp_path, monkeypatch):
workspace = tmp_path / "workspace"
skill(workspace, "skills/a")
def failed_render(*_args):
raise OSError("Report disk is full")
monkeypatch.setattr(runner, "_write_summary", failed_render)
with pytest.raises(OSError, match="disk is full"):
runner.run(
workspace,
tmp_path / "reports",
{"SKILL_PATHS": "skills", "LLM_ENABLED": "false"},
execute=fake_cli([(1, payload(False, "failed"))], []),
enforcement_mode="advisory",
)
@pytest.mark.parametrize("mode,expected", [("strict", 1), ("advisory", 0)])
def test_cli_enforcement_flag_preserves_failed_evidence(tmp_path, mode, expected):
workspace = tmp_path / "workspace"
skill(workspace, "skills/a")
binary = tmp_path / "skillevaluator"
binary.write_text(
f"#!{sys.executable}\nimport pathlib,sys\n"
"output=pathlib.Path(sys.argv[sys.argv.index('-o')+1])\n"
f"(output/'skillevaluator-tier1.json').write_text({json.dumps(payload(False, 'failed'))!r})\n"
"raise SystemExit(1)\n"
)
binary.chmod(0o755)
reports = tmp_path / "reports"
result = subprocess.run(
[
sys.executable,
str(MODULE_PATH),
"--workspace",
str(workspace),
"--reports",
str(reports),
"--enforcement-mode",
mode,
],
env={
**os.environ,
"PATH": str(tmp_path) + os.pathsep + os.environ["PATH"],
"SKILL_PATHS": "skills",
"LLM_ENABLED": "false",
},
capture_output=True,
text=True,
)
assert result.returncode == expected, result.stderr
assert json.loads((reports / "summary.json").read_text())["overall_status"] == "failed"
assert ("::warning title=Tier 1 (advisory)::" in result.stdout) is (mode == "advisory")
def test_contradictory_passing_report_with_incomplete_scanners_has_publishable_error_row(tmp_path):
import ci.publish_reports as publisher
workspace = tmp_path / "workspace"
skill(workspace, "skills/a")
report = payload()
report["incomplete_scans"] = ["semgrep"]
reports = tmp_path / "reports"
exit_code = runner.run(
workspace,
reports,
{"SKILL_PATHS": "skills", "LLM_ENABLED": "false"},
execute=fake_cli([(0, report)], []),
)
summary = json.loads((reports / "summary.json").read_text())
assert exit_code == 1
assert summary["skills"][0]["overall_status"] == "error"
publisher.validate_summary(summary)
artifact = io.BytesIO()
with zipfile.ZipFile(artifact, "w") as archive:
archive.write(reports / "summary.json", "summary.json")
archive.write(reports / "skill-001/skillevaluator-tier1.json", "skill-001/skillevaluator-tier1.json")
retained, children = publisher.read_artifact(artifact.getvalue())
assert retained["skills"][0]["overall_status"] == "error"
assert children == {}
@pytest.mark.parametrize(
"exit_code,status,changes",
[
(1, "failed", {"incomplete_scans": ["semgrep"]}),
(0, "failed", {}),
(1, "incomplete", {"incomplete_scans": []}),
(1, "incomplete", {"incomplete_scans": ["trivy"]}),
(0, "incomplete", {}),
(0, "passed", {"total_errors": 1}),
],
)
def test_any_contradictory_report_becomes_error_row_without_losing_other_skills(tmp_path, exit_code, status, changes):
import ci.publish_reports as publisher
workspace = tmp_path / "workspace"
skill(workspace, "skills/a")
skill(workspace, "skills/b")
report = {**payload(status == "passed", status), **changes}
reports = tmp_path / "reports"
runner.run(
workspace,
reports,
{"SKILL_PATHS": "skills", "LLM_ENABLED": "false"},
execute=fake_cli([(exit_code, report), (0, payload())], []),
)
summary = json.loads((reports / "summary.json").read_text())
assert [row["overall_status"] for row in summary["skills"]] == ["error", "passed"]
assert summary["skills"][0]["incomplete_scans"] == []
assert summary["skills"][0]["passed"] is False
artifact = io.BytesIO()
with zipfile.ZipFile(artifact, "w") as archive:
for path in sorted(reports.rglob("*.json")):
archive.write(path, path.relative_to(reports).as_posix())
retained, children = publisher.read_artifact(artifact.getvalue())
assert [row["overall_status"] for row in retained["skills"]] == ["error", "passed"]
assert list(children) == ["skill-002"]
assert "Report unavailable" in (reports / "skill-001/skillevaluator-tier1.html").read_text()
@pytest.mark.parametrize(
"root", ["/etc", "../skills", "skills/../other", "C:\\skills", "skills\\a", "sk\tills", "skills/m\x7fskill"]
)
def test_rejects_unsafe_roots(tmp_path, root):
skill(tmp_path, "sk\tills/a")
skill(tmp_path, "skills/m\x7fskill")
with pytest.raises(runner.ConfigurationError, match="without parent traversal"):
runner.discover_skills(tmp_path, root)
LONG_NAME = "x" * 170
@pytest.mark.parametrize(
"roots",
[
"skills/tab\tname",
"skills/escape\x1bname",
"\n".join(["skills"] * 101),
"\n".join([f"skills/{LONG_NAME}"] * 100),
],
)
def test_full_scan_and_changed_selection_share_skill_root_rules(tmp_path, roots):
for name in ("tab\tname", "escape\x1bname", LONG_NAME):
skill(tmp_path, f"skills/{name}")
with pytest.raises(runner.ConfigurationError):
runner.discover_skills(tmp_path, roots)
with pytest.raises(selector.SelectionError):
selector.skill_roots(tmp_path, roots)
def test_empty_selection_and_missing_root_fail(tmp_path):
for roots in ("", "skills"):
with pytest.raises(runner.ConfigurationError):
runner.discover_skills(tmp_path, roots)
(tmp_path / "skills").mkdir()
with pytest.raises(runner.ConfigurationError):
runner.discover_skills(tmp_path, "skills")
@pytest.mark.parametrize("link_kind", ["root", "ancestor", "file", "directory"])
def test_rejects_symlinks_anywhere_in_tree(tmp_path, link_kind):
target = skill(tmp_path, "skills/a")
if link_kind == "root":
(tmp_path / "alias").symlink_to(target, target_is_directory=True)
configured = "alias"
elif link_kind == "ancestor":
(tmp_path / "alias").symlink_to(tmp_path / "skills", target_is_directory=True)
configured = "alias/a"
else:
(target / "linked").symlink_to(target if link_kind == "directory" else target / "SKILL.md")
configured = "skills"
with pytest.raises(runner.ConfigurationError):
runner.discover_skills(tmp_path, configured)
def test_target_limit_is_global(tmp_path):
for index in range(101):
skill(tmp_path, f"skills/a{index:03d}")
with pytest.raises(runner.ConfigurationError):
runner.discover_skills(tmp_path, "skills")
def test_all_skills_run_after_failure_and_scanner_stdout_is_not_printed(tmp_path, capsys):
workspace = tmp_path / "workspace"
skill(workspace, "skills/a")
skill(workspace, "skills/b")
calls = []
reports = tmp_path / "reports"
exit_code = runner.run(
workspace,
reports,
{"SKILL_PATHS": "skills", "LLM_ENABLED": "false"},
execute=fake_cli([(1, payload(False, "failed")), (0, payload())], calls),
)
summary = json.loads((reports / "summary.json").read_text())
assert exit_code == 1
assert len(calls) == 2
assert (summary["total"], summary["passed"], summary["failed"]) == (2, 1, 1)
assert summary["total_validators"] == 24
assert "::error::" not in capsys.readouterr().out
assert "::error::" in (reports / "skill-001" / "stdout.log").read_text()
assert "--no-llm" in calls[0][0]
assert calls[0][1]["env"]["HOME"].startswith("/tmp/evaluator-home/worker-")
# The CLI writes no HTML; the native renderer supplies each skill page.
assert calls[0][0][calls[0][0].index("-r") + 1] == "json,markdown,sarif"
for child in ("skill-001", "skill-002"):
page = (reports / child / "skillevaluator-tier1.html").read_text()
assert "Content-Security-Policy" in page
def test_structured_failure_logs_and_annotations_are_escaped(capsys):
row = {"path": "skills/example", "overall_status": "failed"}
report = {
"quality_summary": [{"overall_score": 71.8, "grade": "C"}],
"results": [
{
"validator": "Schema,metadata",
"status": "failed",
"passed": False,
"findings": [
{
"severity": "high",
"message": "Bad name\n::notice::injected",
"suggestion": "Use kebab-case",
"file_path": "/workspace/skills/example/SKILL.md",
"line_number": None,
}
],
}
],
}
runner.log_result(row, report, annotations=True, budget={"errors": 0})
output = capsys.readouterr().out
assert "71.8/100" in output
assert "Suggested fix:" in output
assert "\n::notice::injected" not in output
commands = [line for line in output.splitlines() if line.startswith("::")]
assert len(commands) == 1
assert "title=Schema%2Cmetadata,file=skills/example/SKILL.md" in commands[0]
assert "Bad name%0A::notice::injected" in commands[0]
assert ",line=" not in commands[0]
def test_annotation_budget_does_not_hide_log_details(capsys):
row = {"path": "skills/example", "overall_status": "failed"}
report = {
"results": [
{
"validator": "Schema",
"status": "failed",
"passed": False,
"findings": [{"severity": "high", "message": "Bad metadata"}],
}
]
}
runner.log_result(row, report, annotations=True, budget={"errors": 10})
output = capsys.readouterr().out
assert "Bad metadata" in output
assert "::error" not in output
@pytest.mark.parametrize(
"exit_code,report",
[
(0, None),
(0, {}),
(1, payload()),
(0, payload(False, "failed")),
(0, payload(True, "incomplete", ["semgrep"])),
(0, {**payload(), "results": [{"passed": False, "status": "failed"}]}),
(0, {**payload(), "overall_passed": "true"}),
(0, {**payload(), "total_validators": 100}),
(0, {**payload(), "results": [{"passed": True, "status": "incomplete"}] * 12}),
],
)
def test_exit_or_report_disagreement_never_passes(tmp_path, exit_code, report):
workspace = tmp_path / "workspace"
skill(workspace, "skills/a")
reports = tmp_path / "reports"
result = runner.run(workspace, reports, {"SKILL_PATHS": "skills"}, execute=fake_cli([(exit_code, report)], []))
assert result == 1
assert json.loads((reports / "summary.json").read_text())["overall_passed"] is False
def test_llm_maps_only_explicit_provider_and_credentials(tmp_path):
workspace = tmp_path / "workspace"
skill(workspace, "skills/a")
calls = []
environment = {
"SKILL_PATHS": "skills",
"LLM_ENABLED": "true",
"LLM_PROVIDER": "nv_build",
"LLM_API_KEY": "test-key",
"LLM_MODEL": "test-model",
"MIN_SCORE": "85",
"OPENAI_API_KEY": "ambient-key",
"SKILLSPECTOR_PROVIDER": "untrusted-ambient-provider",
"SKILL_EVAL_LLM_BASE_URL": "https://unexpected.invalid",
}
assert runner.run(workspace, tmp_path / "reports", environment, execute=fake_cli([(0, payload())], calls)) == 0
argv, kwargs = calls[0]
assert "--llm" in argv
assert argv[argv.index("--min-score") + 1] == "85"
assert kwargs["env"]["NVIDIA_API_KEY"] == "test-key"
assert kwargs["env"]["SKILL_EVAL_LLM_PROVIDER"] == "nv_build"
assert kwargs["env"]["SKILL_EVAL_LLM_MODEL"] == "test-model"
assert "OPENAI_API_KEY" not in kwargs["env"]
assert "SKILLSPECTOR_PROVIDER" not in kwargs["env"]
assert "SKILL_EVAL_LLM_BASE_URL" not in kwargs["env"]
@pytest.mark.parametrize(
"extra",
[
{"LLM_ENABLED": "true"},
{"LLM_ENABLED": "true", "LLM_PROVIDER": "arbitrary", "LLM_API_KEY": "key"},
{"LLM_ENABLED": "yes"},
],
)
def test_invalid_llm_configuration_is_reported_without_running(tmp_path, extra):
workspace = tmp_path / "workspace"
skill(workspace, "skills/a")
reports = tmp_path / "reports"
calls = []
assert runner.run(workspace, reports, {"SKILL_PATHS": "skills", **extra}, execute=fake_cli([], calls)) == 1
assert not calls
assert json.loads((reports / "summary.json").read_text())["overall_status"] == "error"
def test_markdown_paths_cannot_inject_html_or_table_rows(tmp_path):
workspace = tmp_path / "workspace"
skill(workspace, "skills/<script>|[x]`danger`")
reports = tmp_path / "reports"
assert runner.run(workspace, reports, {"SKILL_PATHS": "skills"}, execute=fake_cli([(0, payload())], [])) == 0
markdown = (reports / "details.md").read_text()
assert "<script>" not in markdown
assert "<script>" in markdown
assert "\\|" in markdown
def test_successful_rollup_preserves_all_counts(tmp_path):
workspace = tmp_path / "workspace"
skill(workspace, "skills/a")
reports = tmp_path / "reports"
assert runner.run(workspace, reports, {"SKILL_PATHS": "skills"}, execute=fake_cli([(0, payload())], [])) == 0
summary = json.loads((reports / "summary.json").read_text())
assert summary["overall_passed"] is True
assert summary["overall_status"] == "passed"
assert summary["incomplete"] == 0
assert summary["skills"][0]["path"] == "skills/a"
@pytest.mark.parametrize("recorded", [True, False])
def test_retained_report_records_the_repository_relative_target_not_the_runner_path(tmp_path, recorded):
workspace = tmp_path / "workspace"
skill(workspace, "skills/a")
reports = tmp_path / "reports"
def invoke(argv, **_kwargs):
# The evaluator records the target as the CI passed it: an absolute
# path in the runner's workspace. An older evaluator image records none.
report = {**payload(), "target_path": argv[argv.index("tier1") + 1]}
if not recorded:
report.pop("target_path")
output = Path(argv[argv.index("-o") + 1])
(output / "skillevaluator-tier1.json").write_text(json.dumps(report), encoding="utf-8")
return CompletedProcess(argv, 0)
assert runner.run(workspace, reports, {"SKILL_PATHS": "skills"}, execute=invoke) == 0
retained = json.loads((reports / "skill-001" / "skillevaluator-tier1.json").read_text())
assert retained.get("target_path") == ("skills/a" if recorded else None)
assert str(workspace) not in json.dumps(retained)
def test_invalid_json_is_reported_and_later_skills_still_run(tmp_path):
workspace = tmp_path / "workspace"
skill(workspace, "skills/a")
skill(workspace, "skills/b")
reports = tmp_path / "reports"
calls = []
original = fake_cli([(0, payload()), (0, payload())], calls)
def invoke(argv, **kwargs):
completed = original(argv, **kwargs)
output = Path(argv[argv.index("-o") + 1])
if report_dir(output) == "skill-001":
(output / "skillevaluator-tier1.json").write_text("not json", encoding="utf-8")
return completed
assert runner.run(workspace, reports, {"SKILL_PATHS": "skills"}, execute=invoke) == 1
summary = json.loads((reports / "summary.json").read_text())
assert summary["incomplete"] == 1
assert summary["passed"] == 1
assert len(calls) == 2
def test_report_loading_validates_without_copying_metadata(tmp_path, monkeypatch):
from skillevaluator.reporting import retained
report = payload(passed=False)
report["results"][0]["findings"][0]["metadata"] = {"rule": {"nested": ["evidence"]}}
path = tmp_path / "skillevaluator-tier1.json"
path.write_text(json.dumps(report), encoding="utf-8")
def no_copies(value, memo=None):
raise AssertionError("validation copied report metadata")
monkeypatch.setattr(retained.copy, "deepcopy", no_copies)
assert runner._load_report(path) == report
report["results"][0]["passed"] = "false"
path.write_text(json.dumps(report), encoding="utf-8")
with pytest.raises(ValueError, match="passed must be boolean"):
runner._load_report(path)
def test_process_start_error_does_not_skip_later_skills(tmp_path):
workspace = tmp_path / "workspace"
skill(workspace, "skills/a")
skill(workspace, "skills/b")
reports = tmp_path / "reports"
calls = []
def invoke(argv, **kwargs):
calls.append(argv)
output = Path(argv[argv.index("-o") + 1])
if report_dir(output) == "skill-001":
raise OSError("untrusted failure text")
(output / "skillevaluator-tier1.json").write_text(json.dumps(payload()), encoding="utf-8")
return CompletedProcess(argv, 0)
assert runner.run(workspace, reports, {"SKILL_PATHS": "skills"}, execute=invoke) == 1
assert len(calls) == 2
summary = json.loads((reports / "summary.json").read_text())
assert [row["passed"] for row in summary["skills"]] == [False, True]
def test_static_run_does_not_receive_ambient_model_credentials():
env, enabled = runner.child_environment(
{
"LLM_ENABLED": "false",
"LLM_API_KEY": "secret",
"OPENAI_API_KEY": "ambient",
"SKILL_EVAL_LLM_PROVIDER": "openai",
"SKILLSPECTOR_PROVIDER": "anthropic",
}
)
assert enabled is False
assert not any("KEY" in name or "PROVIDER" in name for name in env)
assert env["PYTHON_DOTENV_DISABLED"] == "1"
assert env["SEMGREP_SEND_METRICS"] == "off"
assert env["SEMGREP_ENABLE_VERSION_CHECK"] == "0"
@pytest.mark.parametrize("llm", ["false", "true"])
@pytest.mark.parametrize("ambient_requirement", [None, "0", "1"])
def test_ci_requires_semgrep_even_when_ambient_configuration_disables_it(llm, ambient_requirement):
configuration = {"LLM_ENABLED": llm, "LLM_PROVIDER": "openai", "LLM_API_KEY": "test-key"}
if ambient_requirement is not None:
configuration["SKILLEVALUATOR_REQUIRE_SEMGREP"] = ambient_requirement
env, _ = runner.child_environment(configuration)
assert env["SKILLEVALUATOR_REQUIRE_SEMGREP"] == "1"
def test_incomplete_runtime_rollup_is_not_mislabeled_as_inconsistent_evidence(tmp_path):
workspace = tmp_path / "workspace"
skill(workspace, "skills/a")
reports = tmp_path / "reports"
assert (
runner.run(
workspace,
reports,
{"SKILL_PATHS": "skills"},
execute=fake_cli([(1, payload(False, "incomplete", ["semgrep"], findings=0))], []),
)
== 1
)
summary = json.loads((reports / "summary.json").read_text())
assert summary["skills"][0]["overall_status"] == "incomplete"
assert (summary["failed"], summary["incomplete"]) == (1, 1)
compact = (reports / "summary.md").read_text()
assert "**Tier 1 scan result: INCOMPLETE**" in compact
assert "1 skills · 0 passed · 0 failed · 1 incomplete" in compact
assert "Report evidence is missing or inconsistent" not in compact
def test_confirmed_failure_and_incomplete_scanner_survive_retained_publication(tmp_path):
import ci.publish_reports as publisher
from skillevaluator.reporting.retained import retained_results
workspace = tmp_path / "workspace"
skill(workspace, "skills/a")
report = payload(False, "incomplete", ["semgrep"])
assert report["overall_status"] == "failed"
assert report["results"][0]["status"] == "failed"
assert retained_results(report)[0].is_incomplete
reports = tmp_path / "reports"
assert runner.run(workspace, reports, {"SKILL_PATHS": "skills"}, execute=fake_cli([(1, report)], [])) == 1
summary = json.loads((reports / "summary.json").read_text())
row = summary["skills"][0]
assert row["overall_status"] == "failed"
assert row["incomplete_scans"] == ["semgrep"]
assert row.get("error") is None
assert (summary["failed"], summary["incomplete"]) == (1, 0)
publisher.validate_summary(summary)
artifact = io.BytesIO()
with zipfile.ZipFile(artifact, "w") as archive:
archive.write(reports / "summary.json", "summary.json")
archive.write(reports / "skill-001/skillevaluator-tier1.json", "skill-001/skillevaluator-tier1.json")
retained, children = publisher.read_artifact(artifact.getvalue())
assert retained["skills"][0]["overall_status"] == "failed"
assert children["skill-001"]["results"][0]["incomplete_scans"] == ["semgrep"]
def test_confirmed_skill_failure_precedes_another_skills_incomplete_scan(tmp_path):
import ci.publish_reports as publisher
workspace = tmp_path / "workspace"
skill(workspace, "skills/a")
skill(workspace, "skills/b")
reports = tmp_path / "reports"
responses = [(1, payload(False)), (1, payload(False, "incomplete", ["semgrep"], findings=0))]
assert runner.run(workspace, reports, {"SKILL_PATHS": "skills"}, execute=fake_cli(responses, [])) == 1
summary = json.loads((reports / "summary.json").read_text())
assert summary["overall_status"] == "failed"
assert [row["overall_status"] for row in summary["skills"]] == ["failed", "incomplete"]
assert (summary["failed"], summary["incomplete"]) == (2, 1)
publisher.validate_summary(summary)
def repository_git(workspace, *arguments):
return subprocess.check_output(
["git", "-C", str(workspace), "-c", "core.hooksPath=/dev/null", "-c", "commit.gpgSign=false", *arguments],
text=True,
stderr=subprocess.DEVNULL,
env={**os.environ, "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_NOSYSTEM": "1"},
).strip()
def repository_commit(workspace):
repository_git(workspace, "add", "-A")
repository_git(workspace, "commit", "--allow-empty", "-m", "fixture")
return repository_git(workspace, "rev-parse", "HEAD")
def selection_checkout(tmp_path):
workspace = tmp_path / "workspace"
workspace.mkdir()
repository_git(workspace, "init", "-q")
repository_git(workspace, "config", "user.name", "Runner Test")
repository_git(workspace, "config", "user.email", "[email protected]")
skill(workspace, "skills/a")
skill(workspace, "skills/b")
return workspace, repository_commit(workspace)
def baseline_checkout(tmp_path, manifest=None):
workspace, _ = selection_checkout(tmp_path)
policy = workspace / ".skillevaluator/skillspector-baselines.yaml"
policy.parent.mkdir()
policy.write_text(manifest or "version: 1\nbaselines:\n skills/a: policies/a.json\n")
baseline = workspace / "policies/a.json"
baseline.parent.mkdir()
baseline.write_text(json.dumps({"version": 2, "rules": [{"id": "PE5", "reason": "Reviewed exact policy"}]}))
return workspace, repository_commit(workspace), baseline.read_bytes()
def test_security_baselines_use_reviewed_mapping_and_bytes_not_pr_head(tmp_path):
workspace, approved, approved_bytes = baseline_checkout(tmp_path)
(workspace / ".skillevaluator/skillspector-baselines.yaml").write_text(
"version: 1\nbaselines:\n skills/b: policies/a.json\n"
)
(workspace / "policies/a.json").write_text('{"version":2,"rules":[]}')
head = repository_commit(workspace)
calls, observed = [], []
fixture = fake_cli([(0, payload()), (0, payload())], calls)
def execute(argv, **kwargs):
if "--skillspector-baseline" in argv:
path = Path(argv[argv.index("--skillspector-baseline") + 1])
observed.append((argv[2], path.read_bytes(), path.is_relative_to(workspace)))
return fixture(argv, **kwargs)
reports = tmp_path / "reports"
assert (
runner.run(
workspace,
reports,
{
"SKILL_PATHS": "skills",
"SECURITY_BASELINES": "true",
"POLICY_SOURCE_SHA": approved,
"REPORT_COMMIT": head,
},
execute=execute,
)
== 0
)
assert observed == [(str(workspace / "skills/a"), approved_bytes, False)]
summary = json.loads((reports / "summary.json").read_text())
evidence = summary["execution"]["security_baselines"]
assert evidence["source_commit"] == approved
assert evidence["manifest"] == ".skillevaluator/skillspector-baselines.yaml"
assert re.fullmatch(r"sha256:[0-9a-f]{64}", evidence["manifest_digest"])
assert summary["skills"][0]["security_baseline"]["file"] == "policies/a.json"
assert "security_baseline" not in summary["skills"][1]
def test_security_baselines_can_use_current_push_commit(tmp_path):
workspace, head, expected = baseline_checkout(tmp_path)
calls, observed = [], []
fixture = fake_cli([(0, payload()), (0, payload())], calls)
def execute(argv, **kwargs):
if "--skillspector-baseline" in argv:
observed.append(Path(argv[argv.index("--skillspector-baseline") + 1]).read_bytes())
return fixture(argv, **kwargs)
assert (
runner.run(
workspace,
tmp_path / "reports",
{
"SKILL_PATHS": "skills",
"SECURITY_BASELINES": "true",
"POLICY_SOURCE_SHA": head,
},
execute=execute,
)
== 0
)
assert observed == [expected]
def test_security_baseline_audit_field_does_not_collide_with_configured_secret_fragment(tmp_path):
workspace, approved, _ = baseline_checkout(tmp_path)
(workspace / "skills/a/example.txt").write_text('secret="configuredSuperSecret987654321"\n')
calls, reports = [], tmp_path / "reports"
assert (
runner.run(
workspace,
reports,
{"SKILL_PATHS": "skills/a", "SECURITY_BASELINES": "true", "POLICY_SOURCE_SHA": approved},
execute=fake_cli([(0, payload())], calls),
)
== 0
)
assert len(calls) == 1
summary = json.loads((reports / "summary.json").read_text())
assert summary["execution"]["security_baselines"]["mapped_skill_count"] == 1
assert "configured_skills" not in summary["execution"]["security_baselines"]
assert "withheld" in (reports / "skill-001/stdout.log").read_text()
def test_security_baselines_require_explicit_opt_in(tmp_path):
workspace, head, _ = baseline_checkout(tmp_path)
calls = []
assert (
runner.run(
workspace,
tmp_path / "reports",
{
"SKILL_PATHS": "skills",
"POLICY_SOURCE_SHA": head,
},
execute=fake_cli([(0, payload()), (0, payload())], calls),
)
== 0
)
assert all("--skillspector-baseline" not in argv for argv, _ in calls)
@pytest.mark.parametrize("enabled", ["TRUE", " true ", "False", "yes"])
def test_security_baselines_reject_noncanonical_boolean_input(tmp_path, enabled):
workspace, head, _ = baseline_checkout(tmp_path)
calls = []
assert (
runner.run(
workspace,
tmp_path / "reports",
{
"SKILL_PATHS": "skills",
"SECURITY_BASELINES": enabled,
"POLICY_SOURCE_SHA": head,
},
execute=fake_cli([(0, payload()), (0, payload())], calls),
)
== 1
)
assert not calls
@pytest.mark.parametrize("source", ["", "HEAD", "0" * 40])
def test_security_baselines_fail_closed_without_exact_available_policy_commit(tmp_path, source):
workspace, _, _ = baseline_checkout(tmp_path)
calls = []
assert (
runner.run(
workspace,
tmp_path / "reports",
{
"SKILL_PATHS": "skills",
"SECURITY_BASELINES": "true",
"POLICY_SOURCE_SHA": source,
},
execute=fake_cli([(0, payload()), (0, payload())], calls),
)
== 1
)
assert not calls
@pytest.mark.parametrize(
"manifest",
[
"version: 1\nbaselines:\n skills/a: policies/a.json\n skills/a: policies/b.json\n",
"version: 1\nbaselines:\n skills/a: ../a.json\n",
"version: 1\nbaselines:\n ../skills/a: policies/a.json\n",
"version: 2\nbaselines: {}\n",
"version: 1\nbaselines:\n skills/a: policies/missing.json\n",
"version: 1\nbaselines:\n skills/a: &approved policies/a.json\n skills/b: *approved\n",
'version: 1\nbaselines:\n "skills/\\uD800": policies/a.json\n',
'version: 1\nbaselines:\n skills/a: "policies/\\uD800.json"\n',
],
)
def test_security_baselines_reject_invalid_reviewed_manifest_before_scanning(tmp_path, manifest):
workspace, approved, _ = baseline_checkout(tmp_path, manifest)
calls = []
assert (
runner.run(
workspace,
tmp_path / "reports",
{
"SKILL_PATHS": "skills",
"SECURITY_BASELINES": "true",
"POLICY_SOURCE_SHA": approved,
},
execute=fake_cli([(0, payload()), (0, payload())], calls),
)
== 1
)
assert not calls
def test_security_baselines_reject_symlink_blob_from_approved_commit(tmp_path):
workspace, _, _ = baseline_checkout(tmp_path)
(workspace / "policies/a.json").unlink()
(workspace / "policies/a.json").symlink_to("../secret")
approved = repository_commit(workspace)
calls = []
assert (
runner.run(
workspace,
tmp_path / "reports",
{
"SKILL_PATHS": "skills",
"SECURITY_BASELINES": "true",
"POLICY_SOURCE_SHA": approved,
},
execute=fake_cli([(0, payload()), (0, payload())], calls),
)
== 1
)
assert not calls
@pytest.mark.parametrize("escaped", [False, True])
def test_security_baselines_redact_credentials_only_in_approved_policy_bytes(tmp_path, escaped):
workspace, _, _ = baseline_checkout(tmp_path)
token = "nvapi-" + "A1" * 25
approved_content = json.dumps({"version": 2, "scanner_version": token, "rules": []})
if escaped:
approved_content = approved_content.replace(token, "\\u006e" + token[1:])
(workspace / "policies/a.json").write_text(approved_content)
approved = repository_commit(workspace)
(workspace / "policies/a.json").write_text('{"version":2,"rules":[]}')
repository_commit(workspace)
report = payload(passed=False)
report["results"][0]["findings"][0]["message"] = "Baseline scanner version mismatch: " + token
reports = tmp_path / "reports"
assert (
runner.run(
workspace,
reports,
{
"SKILL_PATHS": "skills/a",
"SECURITY_BASELINES": "true",
"POLICY_SOURCE_SHA": approved,
},
execute=fake_cli([(1, report)], []),
)
== 1
)
assert all(token.encode() not in path.read_bytes() for path in reports.rglob("*") if path.is_file())
assert "withheld" in (reports / "skill-001/stdout.log").read_text()
@pytest.mark.parametrize("content", ['!!int ""', '!!timestamp ""', '!!bool ""'])
def test_security_baselines_reject_malformed_scalar_before_scanning(tmp_path, content):
workspace, _, _ = baseline_checkout(tmp_path)
(workspace / "policies/a.json").write_text(content)
approved = repository_commit(workspace)
calls, reports = [], tmp_path / "reports"
assert (
runner.run(
workspace,
reports,
{"SKILL_PATHS": "skills", "SECURITY_BASELINES": "true", "POLICY_SOURCE_SHA": approved},
execute=fake_cli([(0, payload()), (0, payload())], calls),
)
== 1
)
assert not calls
summary = json.loads((reports / "summary.json").read_text())
assert "Credential preflight" in summary["error"]
def test_changed_mode_runs_only_changed_skill_from_real_git_history(tmp_path):
workspace, base = selection_checkout(tmp_path)
(workspace / "skills/a/SKILL.md").write_text("# Changed skill\n")
head = repository_commit(workspace)
reports, calls = tmp_path / "reports", []
assert (
runner.run(
workspace,
reports,
{
"SKILL_PATHS": "skills\nskills/a",
"SCAN_MODE": "changed",
"DIFF_BASE_SHA": base,
"DIFF_HEAD_SHA": head,
},
execute=fake_cli([(0, payload())], calls),
)
== 0
)
summary = json.loads((reports / "summary.json").read_text())
assert len(calls) == 1 and str(workspace / "skills/a") in calls[0][0]
assert summary["selection"]["selected"] == ["skills/a"]
assert summary["selection"]["discovered"] == 2
def test_unchanged_skill_pr_is_successful_neutral_noop_with_reports(tmp_path, capsys):
workspace, base = selection_checkout(tmp_path)
(workspace / "README.md").write_text("Unrelated documentation\n")
head = repository_commit(workspace)
reports, calls = tmp_path / "reports", []
assert (
runner.run(
workspace,
reports,
{
"SKILL_PATHS": "skills",
"SCAN_MODE": "changed",
"DIFF_BASE_SHA": base,
"DIFF_HEAD_SHA": head,
},
execute=fake_cli([], calls),
)
== 0
)
summary = json.loads((reports / "summary.json").read_text())
assert not calls and summary["overall_status"] == "skipped" and summary["overall_passed"] is False
assert (summary["total"], summary["passed"], summary["failed"], summary["incomplete"]) == (0, 0, 0, 0)
assert summary["skills"] == [] and summary["selection"]["selected"] == []
assert "NO SKILLS CHANGED" in capsys.readouterr().out
assert "NO SKILLS CHANGED" in (reports / "summary.md").read_text()
assert "Tier 1: Not run" in (reports / "summary.md").read_text()
assert (reports / "index.html").exists() and (reports / "details.md").exists()
assert not list(reports.glob("skill-*"))
def test_completely_deleted_configured_skill_is_disclosed_not_evaluated(tmp_path):
workspace, base = selection_checkout(tmp_path)
shutil.rmtree(workspace / "skills/a")
head = repository_commit(workspace)
reports, calls = tmp_path / "reports", []
assert (
runner.run(
workspace,
reports,
{
"SKILL_PATHS": "skills/a",
"SCAN_MODE": "changed",
"DIFF_BASE_SHA": base,
"DIFF_HEAD_SHA": head,
},
execute=fake_cli([], calls),
)
== 0
)
summary = json.loads((reports / "summary.json").read_text())
assert not calls and summary["selection"]["deleted"] == ["skills/a"]
assert "skills/a" in (reports / "summary.md").read_text()
assert "Deleted skill directories" in (reports / "summary.md").read_text()
assert summary["overall_passed"] is False
def test_deleted_manifest_with_remaining_content_is_error_before_evaluation(tmp_path):
workspace, _ = selection_checkout(tmp_path)
(workspace / "skills/a/helper.py").write_text("print('fixture')\n")
base = repository_commit(workspace)
(workspace / "skills/a/SKILL.md").unlink()
head = repository_commit(workspace)
reports, calls = tmp_path / "reports", []
assert (
runner.run(
workspace,
reports,
{
"SKILL_PATHS": "skills",
"SCAN_MODE": "changed",
"DIFF_BASE_SHA": base,
"DIFF_HEAD_SHA": head,
},
execute=fake_cli([], calls),
)
== 1
)
summary = json.loads((reports / "summary.json").read_text())
assert not calls and summary["overall_status"] == "error"
assert "Deleted SKILL.md left tracked content" in summary["error"]
assert "NO SKILLS CHANGED" not in (reports / "summary.md").read_text()
@pytest.mark.parametrize(
"selection",
[
{"SCAN_MODE": "changed"},
{"SCAN_MODE": "changed", "DIFF_BASE_SHA": "0" * 40, "DIFF_HEAD_SHA": "0" * 40},
{"SCAN_MODE": "unknown"},
],
)
def test_invalid_selection_is_error_without_full_scan_fallback(tmp_path, selection):
workspace, _ = selection_checkout(tmp_path)
reports, calls = tmp_path / "reports", []
assert runner.run(workspace, reports, {"SKILL_PATHS": "skills", **selection}, execute=fake_cli([], calls)) == 1
summary = json.loads((reports / "summary.json").read_text())
assert not calls and summary["overall_status"] == "error" and summary["overall_passed"] is False
@pytest.mark.parametrize("variable", ["REPORT_URL", "REPORT_RUN_URL", "REPORT_BASE_URL"])
@pytest.mark.parametrize(
"url", ["http://example.com", "https://user:[email protected]", "https://example.com/\nextra"]
)
def test_invalid_report_urls_fail_before_scanning_and_keep_diagnostics(tmp_path, variable, url):
workspace = tmp_path / "workspace"
skill(workspace, "skills/a")
reports, calls = tmp_path / "reports", []
assert runner.run(workspace, reports, {variable: url}, execute=fake_cli([], calls)) == 1
assert not calls
assert "HTTPS" in json.loads((reports / "summary.json").read_text())["error"]
assert "HTTPS" in (reports / "summary.md").read_text()
assert (reports / "index.html").is_file()
assert "user:secret" not in (reports / "pr-comment.md").read_text()
def test_completed_results_survive_timeout_and_unstarted_skills_stay_incomplete(tmp_path):
# One worker pins the sequential order: each result is saved before the next skill starts.
workspace = tmp_path / "workspace"
for name in ("a", "b", "c"):
skill(workspace, f"skills/{name}")
reports, calls = tmp_path / "reports", []
first = fake_cli([(0, payload())], calls)
def execute(argv, **kwargs):
snapshot = json.loads((reports / "summary.json").read_text())
assert snapshot["total"] == 3 and snapshot["overall_passed"] is False
assert kwargs["timeout"] <= runner.SCAN_TIMEOUT_SECONDS
if not calls:
assert snapshot["incomplete"] == 3
return first(argv, **kwargs)
assert snapshot["passed"] == 1 and snapshot["incomplete"] == 2
assert "INCOMPLETE" in (reports / "summary.md").read_text()
raise subprocess.TimeoutExpired(argv, kwargs["timeout"])
assert runner.run(workspace, reports, {"SCAN_WORKERS": "1"}, execute=execute) == 1
summary = json.loads((reports / "summary.json").read_text())
assert summary["passed"] == 1 and summary["incomplete"] == 2
assert "time limit" in summary["skills"][1]["error"]
assert "Not evaluated" in summary["skills"][2]["error"]
assert (reports / "index.html").is_file()
def test_unrelated_root_symlink_does_not_break_full_scan(tmp_path):
target = skill(tmp_path, "skills/a")
(tmp_path / "skills/README.md").symlink_to(tmp_path / "other.md")
assert runner.discover_skills(tmp_path, "skills") == [target]
def test_control_character_skill_path_fails_before_scanning(tmp_path):
workspace = tmp_path / "workspace"
skill(workspace, "skills/a\tbad")
calls = []
assert runner.run(workspace, tmp_path / "reports", {}, execute=fake_cli([], calls)) == 1
assert not calls
@pytest.mark.parametrize(
"path,expected", [("scripts/check.py", "skills/example/scripts/check.py"), ("../secret", None)]
)
def test_relative_annotation_location_is_confined_to_skill(capsys, path, expected):
report = {
"results": [
{
"validator": "Security",
"passed": False,
"status": "failed",
"findings": [{"severity": "high", "message": "Unsafe operation", "file_path": path, "line_number": 7}],
}
]
}
runner.log_result(
{"path": "skills/example", "overall_status": "failed"}, report, annotations=True, budget={"errors": 0}
)
command = next(line for line in capsys.readouterr().out.splitlines() if line.startswith("::error"))
if expected:
assert f"file={expected},line=7" in command
else:
assert "file=" not in command and "line=" not in command
@pytest.mark.parametrize("score", ["0", "75", "100"])
@pytest.mark.parametrize("workers", ["1", "4"])
def test_quality_threshold_is_forwarded_to_evaluator(tmp_path, score, workers):
workspace = tmp_path / "workspace"
skill(workspace, "skills/a")
skill(workspace, "skills/b")
calls = []
assert (
runner.run(
workspace,
tmp_path / "reports",
{"SKILL_PATHS": "skills", "MIN_SCORE": score, "SCAN_WORKERS": workers},
execute=fake_cli([(0, payload()), (0, payload())], calls),
)
== 0
)
assert len(calls) == 2
for argv, _env in calls:
assert argv[argv.index("--min-score") + 1] == score
@pytest.mark.parametrize("score", ["-1", "101", "12.5", "abc", "\uff17\uff10", ""])
def test_invalid_quality_threshold_fails_without_running_scanner(tmp_path, score):
workspace = tmp_path / "workspace"
skill(workspace, "skills/a")
calls = []
assert (
runner.run(
workspace,
tmp_path / "reports",
{"SKILL_PATHS": "skills", "MIN_SCORE": score},
execute=fake_cli([(0, payload())], calls),
)
== 1
)
assert not calls
summary = json.loads((tmp_path / "reports/summary.json").read_text())
assert summary["overall_status"] == "error"
def parallel_cli(responses, calls, *, before=None):
"""Fake CLI for concurrent runs; ``before(name)`` runs inside the call."""
lock = threading.Lock()
def invoke(argv, **kwargs):
output = Path(argv[argv.index("-o") + 1])
name = report_dir(output)
with lock:
calls.append((name, dict(kwargs["env"])))
if before:
before(name)
exit_code, report = responses[name]
kwargs["stdout"].write(b"::error::untrusted scanner text\n")
(output / "skillevaluator-tier1.json").write_text(json.dumps(report), encoding="utf-8")
return CompletedProcess(argv, exit_code)
return invoke
def scan_outputs(reports):
names = ["summary.json", "summary.md", "details.md", "pr-comment.md", "index.html"]
names += sorted(path.relative_to(reports).as_posix() for path in reports.glob("skill-*/skillevaluator-tier1.html"))
return {name: (reports / name).read_text() for name in names}
def test_parallel_scan_matches_sequential_rows_logs_annotations_and_reports(tmp_path, capsys):
workspace = tmp_path / "workspace"
names = [f"skills/s{index}" for index in range(1, 7)]
for name in names:
skill(workspace, name)
failing = {"skill-002", "skill-003", "skill-005"}
# Three failing skills with four findings each exceed the 10-annotation budget.
responses = {
f"skill-{index:03d}": (1, payload(False, "failed", findings=4))
if f"skill-{index:03d}" in failing
else (0, payload())
for index in range(1, 7)
}
environment = {"SKILL_PATHS": "skills", "GITHUB_ACTIONS": "true"}
outputs, logs, finished = {}, {}, {}
for workers in ("1", "4"):
order = finished[workers] = []
done = {name: threading.Event() for name in responses}
def before(name, workers=workers, order=order, done=done):
if workers != "1" and name == "skill-001":
# Finish skill 1 last so completion order differs from skill order.
assert done["skill-002"].wait(WAIT)
order.append(name)
done[name].set()
reports = tmp_path / f"reports-{workers}"
result = runner.run(
workspace,
reports,
{**environment, "SCAN_WORKERS": workers},
execute=parallel_cli(responses, [], before=before),
)
assert result == 1
outputs[workers] = scan_outputs(reports)
logs[workers] = [line for line in capsys.readouterr().out.splitlines() if "at a time" not in line]
assert finished["1"] == sorted(responses)
assert finished["4"].index("skill-002") < finished["4"].index("skill-001")
assert logs["1"] == logs["4"]
assert outputs["1"] == outputs["4"]
annotations = [line for line in logs["4"] if line.startswith("::error")]
assert len(annotations) == 10
assert all("file=skills/s2/SKILL.md" in line or "file=skills/s3/SKILL.md" in line for line in annotations[:8])
headers = [line for line in logs["4"] if line.startswith("SkillEvaluator: evaluating skill")]
assert headers == [
f"SkillEvaluator: evaluating skill {index} of 6: {json.dumps(name)}" for index, name in enumerate(names, 1)
]
def test_default_scan_runs_four_skills_at_once(tmp_path, capsys):
workspace = tmp_path / "workspace"
for index in range(5):
skill(workspace, f"skills/s{index}")
responses = {f"skill-{index:03d}": (0, payload()) for index in range(1, 6)}
barrier = threading.Barrier(4, timeout=WAIT)
lock = threading.Lock()
state = {"active": 0, "peak": 0, "calls": 0}
def before(name):
with lock:
state["calls"] += 1
state["active"] += 1
state["peak"] = max(state["peak"], state["active"])
first_wave = state["calls"] <= 4
if first_wave:
barrier.wait()
with lock:
state["active"] -= 1
calls = []
assert runner.run(workspace, tmp_path / "reports", {}, execute=parallel_cli(responses, calls, before=before)) == 0
assert state["peak"] == 4 and len(calls) == 5
assert "SkillEvaluator: scanning 5 skills, up to 4 at a time." in capsys.readouterr().out
def test_one_worker_runs_skills_strictly_one_at_a_time(tmp_path):
workspace = tmp_path / "workspace"
for index in range(3):
skill(workspace, f"skills/s{index}")
reports = tmp_path / "reports"
responses = {f"skill-{index:03d}": (0, payload()) for index in range(1, 4)}
seen = []
def before(name):
# The previous skill's result is saved before this one starts.
seen.append(json.loads((reports / "summary.json").read_text())["passed"])
assert (
runner.run(workspace, reports, {"SCAN_WORKERS": "1"}, execute=parallel_cli(responses, [], before=before)) == 0
)
assert seen == [0, 1, 2]
def test_each_worker_gets_its_own_home_and_temporary_directory(tmp_path):
workspace = tmp_path / "workspace"
for index in range(4):
skill(workspace, f"skills/s{index}")
audit = tmp_path / "audit-tmp"
audit.mkdir()
responses = {f"skill-{index:03d}": (0, payload()) for index in range(1, 5)}
barrier = threading.Barrier(2, timeout=WAIT)
pairs = []
def before(name):
barrier.wait()
pairs.append(name)
calls = []
environment = {"SCAN_WORKERS": "2", "TMPDIR": str(audit)}
assert (
runner.run(workspace, tmp_path / "reports", environment, execute=parallel_cli(responses, calls, before=before))
== 0
)
assert len(calls) == 4
homes = {env["HOME"] for _, env in calls}
assert homes == {"/tmp/evaluator-home/worker-1", "/tmp/evaluator-home/worker-2"}
for _, env in calls:
worker = env["HOME"].rsplit("-", 1)[1]
assert env["TMPDIR"] == str(audit / f"skillevaluator-worker-{worker}")
assert Path(env["HOME"]).is_dir() and Path(env["TMPDIR"]).is_dir()
# Calls 1-2 and 3-4 ran together, each pair on two different workers.
assert calls[0][1]["HOME"] != calls[1][1]["HOME"] and calls[2][1]["HOME"] != calls[3][1]["HOME"]
def test_concurrent_timeout_keeps_finished_results_and_starts_no_more_skills(tmp_path, capsys):
workspace = tmp_path / "workspace"
for name in "abcde":
skill(workspace, f"skills/{name}")
reports = tmp_path / "reports"
now = [0.0]
second_started, first_timed_out = threading.Event(), threading.Event()
calls = []
def execute(argv, **kwargs):
output = Path(argv[argv.index("-o") + 1])
calls.append(report_dir(output))
if report_dir(output) == "skill-001":
assert second_started.wait(WAIT)
now[0] = runner.SCAN_TIMEOUT_SECONDS + 1
first_timed_out.set()
raise subprocess.TimeoutExpired(argv, kwargs["timeout"])
second_started.set()
# Skill 2 was already running when the budget ran out; its result counts.
assert first_timed_out.wait(WAIT)
(output / "skillevaluator-tier1.json").write_text(json.dumps(payload()), encoding="utf-8")
return CompletedProcess(argv, 0)
environment = {"SCAN_WORKERS": "2"}
assert runner.run(workspace, reports, environment, execute=execute, clock=lambda: now[0]) == 1
assert sorted(calls) == ["skill-001", "skill-002"]
summary = json.loads((reports / "summary.json").read_text())
assert (summary["passed"], summary["incomplete"]) == (1, 4)
rows = summary["skills"]
assert "time limit" in rows[0]["error"] and rows[1]["passed"] is True
assert all("Not evaluated" in row["error"] for row in rows[2:])
output = capsys.readouterr().out
assert 'evaluating skill 2 of 5: "skills/b"' in output and "evaluating skill 3" not in output
assert "Scan time limit reached; this skill did not finish." in output
assert "3 skills were not started" in output
def test_checkpoint_saves_a_later_skill_before_an_earlier_one_finishes(tmp_path):
workspace = tmp_path / "workspace"
skill(workspace, "skills/a")
skill(workspace, "skills/b")
reports = tmp_path / "reports"
responses = {"skill-001": (0, payload()), "skill-002": (0, payload())}
snapshots = []
def before(name):
if name != "skill-001":
return
deadline = time.monotonic() + WAIT
while time.monotonic() < deadline:
rows = json.loads((reports / "summary.json").read_text())["skills"]
if rows[1]["passed"]:
snapshots.append(rows)
return
time.sleep(0.01)
assert (
runner.run(workspace, reports, {"SCAN_WORKERS": "2"}, execute=parallel_cli(responses, [], before=before)) == 0
)
assert len(snapshots) == 1 and snapshots[0][0]["passed"] is False
@pytest.mark.parametrize("value", ["0", "17", "four", "-1", "2.5", "٣"])
def test_invalid_scan_worker_count_fails_before_scanning(tmp_path, value):
workspace = tmp_path / "workspace"
skill(workspace, "skills/a")
reports, calls = tmp_path / "reports", []
assert runner.run(workspace, reports, {"SCAN_WORKERS": value}, execute=fake_cli([], calls)) == 1
assert not calls
assert "SCAN_WORKERS" in json.loads((reports / "summary.json").read_text())["error"]
posix_only = pytest.mark.skipif(os.name != "posix", reason="uses a shell script as the evaluator")
def scan_shim(tmp_path, *, fast=(), ignore_term=False):
"""Put a fake ``skillevaluator`` on a PATH and return that PATH.
A scan records its PID and its background tool's PID in ``scan.pids``,
then runs for five minutes. Skills named in ``fast`` pass at once instead.
"""
tool = tmp_path / "bin"
tool.mkdir()
report = tmp_path / "passing-report.json"
report.write_text(json.dumps(payload()), encoding="utf-8")
cases = "".join(f'*/{name}) cp "{report}" "$out/skillevaluator-tier1.json"; exit 0 ;;\n' for name in fast)
script = tool / "skillevaluator"
script.write_text(
"#!/bin/sh\n"
+ ("trap '' TERM\n" if ignore_term else "")
+ "for out; do :; done\n"
+ f'case "$2" in\n{cases}*) ;;\nesac\n'
+ "sleep 300 &\n"
+ 'echo "$$ $!" > "$out/scan.tmp" && mv "$out/scan.tmp" "$out/scan.pids"\n'
+ "exec sleep 300\n",
encoding="utf-8",
)
script.chmod(0o755)
return f"{tool}{os.pathsep}/usr/bin{os.pathsep}/bin"
def scan_pids(output, deadline):
pids = output / "scan.pids"
while not pids.exists():
assert time.monotonic() < deadline, "the scan did not start"
time.sleep(0.02)
return [int(pid) for pid in pids.read_text().split()]
def private_scan_pids(temporary, name, deadline):
"""Read the PIDs of the scan for report directory ``name`` from its private directory under TMPDIR."""
while not (found := sorted(temporary.rglob(f"skillevaluator-private-{name}-*/scan.pids"))):
assert time.monotonic() < deadline, "the scan did not start"
time.sleep(0.02)
return scan_pids(found[0].parent, deadline)
def still_running(pids):
"""Wait up to WAIT for the processes to end; return those still running."""
deadline = time.monotonic() + WAIT
running = [pid for pid in pids if not ended(pid)]
while running and time.monotonic() < deadline:
time.sleep(0.05)
running = [pid for pid in running if not ended(pid)]
return running
def ended(pid):
try:
os.kill(pid, 0)
except ProcessLookupError:
return True
return False
def kill_quietly(pids):
# Only for processes not seen to end: a PID that ended may be reused.
for pid in pids:
with contextlib.suppress(OSError):
os.kill(pid, signal.SIGKILL)
@posix_only
def test_scan_timeout_also_ends_the_scanner_tools(tmp_path, monkeypatch):
path = scan_shim(tmp_path)
output = tmp_path / "out"
output.mkdir()
clock = types.SimpleNamespace(offset=0.0, read=threading.Event())
def monotonic():
now = time.monotonic() + clock.offset
clock.read.set()
return now
clock.monotonic = monotonic
monkeypatch.setattr(runner, "time", clock)
outcome = {}
def scan():
try:
runner.run_scan(
["skillevaluator", "tier1", "skills/a", "-o", str(output)],
abort=threading.Event(),
timeout=WAIT,
env={"PATH": path},
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
except subprocess.TimeoutExpired as exc:
outcome["error"] = exc
thread = threading.Thread(target=scan, daemon=True)
thread.start()
pids = survivors = scan_pids(output, time.monotonic() + WAIT)
try:
# The scan sets its time limit after it starts the evaluator, which can write its
# PIDs first. Move the clock only once the scan has read it, or the limit moves too.
clock.read.clear()
assert clock.read.wait(WAIT), "the scan did not read the clock"
clock.offset = WAIT
thread.join(WAIT)
assert not thread.is_alive() and "error" in outcome
survivors = still_running(pids)
assert not survivors
finally:
kill_quietly(survivors)
@posix_only
def test_aborted_scan_is_killed_after_the_grace_period(tmp_path, monkeypatch):
monkeypatch.setattr(runner, "STOP_GRACE_SECONDS", 0.5)
path = scan_shim(tmp_path, ignore_term=True)
output = tmp_path / "out"
output.mkdir()
abort, outcome = threading.Event(), {}
def scan():
outcome["process"] = runner.run_scan(
["skillevaluator", "tier1", "skills/a", "-o", str(output)],
abort=abort,
timeout=WAIT,
env={"PATH": path},
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
thread = threading.Thread(target=scan, daemon=True)
thread.start()
pids = survivors = scan_pids(output, time.monotonic() + WAIT)
try:
abort.set()
thread.join(WAIT)
assert not thread.is_alive()
# The scan and its tool ignore SIGTERM, so only SIGKILL ends them.
assert outcome["process"].returncode == -signal.SIGKILL
survivors = still_running(pids)
assert not survivors
finally:
kill_quietly(survivors)
@posix_only
def test_checkpoint_error_ends_running_scans_at_once(tmp_path, monkeypatch):
workspace = tmp_path / "workspace"
skill(workspace, "skills/fast")
skill(workspace, "skills/slow")
audit = tmp_path / "audit-tmp"
audit.mkdir()
reports = tmp_path / "reports"
environment = {"PATH": scan_shim(tmp_path, fast=["fast"]), "TMPDIR": str(audit)}
render, pids, outcome = runner.render_reports, [], {}
survivors = pids
def render_reports(*args):
if (reports / "skill-001" / "skillevaluator-tier1.json").exists():
# The slow scan is running when saving the fast result fails.
pids.extend(private_scan_pids(audit, "skill-002", time.monotonic() + WAIT))
raise OSError(errno.ENOSPC, "No space left on device")
render(*args)
monkeypatch.setattr(runner, "render_reports", render_reports)
def scan():
try:
runner.run(workspace, reports, environment)
except BaseException as exc:
outcome["error"] = exc
thread = threading.Thread(target=scan, daemon=True)
thread.start()
try:
thread.join(WAIT)
assert not thread.is_alive(), "the run waited for a running scan"
assert isinstance(outcome["error"], OSError)
survivors = still_running(pids)
assert pids and not survivors
finally:
kill_quietly(survivors)
def start_runner_with_endless_scans(tmp_path, *, ignore_term=False):
"""Start the runner alone in its session, as Docker runs it, and wait for two endless scans."""
workspace = tmp_path / "workspace"
for name in ("a", "b"):
skill(workspace, f"skills/{name}")
audit = tmp_path / "audit-tmp"
audit.mkdir()
reports = tmp_path / "reports"
environment = {"PATH": scan_shim(tmp_path, ignore_term=ignore_term), "TMPDIR": str(audit)}
environment.update(
(name, os.environ[name]) for name in ("PYTHONPATH", "PYTHONDONTWRITEBYTECODE") if name in os.environ
)
with (tmp_path / "runner.log").open("wb") as log:
process = subprocess.Popen(
[sys.executable, str(MODULE_PATH), "--workspace", str(workspace), "--reports", str(reports)],
env=environment,
stdout=log,
stderr=subprocess.STDOUT,
start_new_session=True,
)
pids = []
try:
deadline = time.monotonic() + WAIT
for name in ("skill-001", "skill-002"):
pids += private_scan_pids(audit, name, deadline)
except BaseException:
process.kill()
process.wait()
kill_quietly(pids)
raise
return process, pids
@posix_only
def test_ctrl_c_ends_running_scans_and_their_tools(tmp_path):
process, pids = start_runner_with_endless_scans(tmp_path)
survivors = pids
try:
# Docker sends Ctrl-C to the runner (PID 1) only, not to its scans.
os.kill(process.pid, signal.SIGINT)
assert process.wait(WAIT) == -signal.SIGINT
survivors = still_running(pids)
assert not survivors
finally:
if process.poll() is None:
process.kill()
process.wait()
kill_quietly(survivors)
@posix_only
def test_second_ctrl_c_does_not_leave_the_runner_waiting(tmp_path):
# The scans ignore SIGTERM, so the second Ctrl-C arrives while the
# runner is still waiting for them to end.
process, pids = start_runner_with_endless_scans(tmp_path, ignore_term=True)
survivors = pids
try:
os.kill(process.pid, signal.SIGINT)
time.sleep(1)
os.kill(process.pid, signal.SIGINT)
assert process.wait(WAIT) == -signal.SIGINT
survivors = still_running(pids)
assert not survivors
finally:
if process.poll() is None:
process.kill()
process.wait()
kill_quietly(survivors)
@pytest.mark.skipif(not hasattr(signal, "pthread_kill"), reason="sends a signal to one thread")
def test_ctrl_c_handled_on_a_scan_thread_still_ends_the_run(tmp_path, monkeypatch):
# Python acts on Ctrl-C in the main thread, but the system can run the signal handler on
# another one. Linux hands a waiting SIGINT to a thread that unblocks signals, as a scan
# thread does when it starts an evaluator. That does not wake a main thread that waits
# for scan events without a time limit, so the run would go on until a scan ended.
workspace = tmp_path / "workspace"
skill(workspace, "skills/a")
waits, aborted = threading.Semaphore(0), []
class Events(runner.queue.SimpleQueue):
def get(self, *args, **kwargs):
waits.release()
return super().get(*args, **kwargs)
def run_scan(argv, *, abort, **options):
# A busy main thread acts on any signal, and this scan's start event ends its first
# wait when that wait has no time limit. Signal once it has started a second wait.
for _ in range(2):
waits.acquire(timeout=WAIT)
signal.pthread_kill(threading.get_ident(), signal.SIGINT)
aborted.append(abort.wait(WAIT))
return CompletedProcess(argv, 1)
monkeypatch.setattr(runner, "queue", types.SimpleNamespace(SimpleQueue=Events, Empty=runner.queue.Empty))
monkeypatch.setattr(runner, "run_scan", run_scan)
with pytest.raises(KeyboardInterrupt):
runner.run(workspace, tmp_path / "reports", {})
assert aborted == [True]
def test_each_report_is_released_once_its_result_is_logged(tmp_path, monkeypatch):
# Reports can be 32 MiB each; 100 of them must not stay in memory.
workspace = tmp_path / "workspace"
for index in range(5):
skill(workspace, f"skills/s{index}")
responses = {f"skill-{index:03d}": (0, payload()) for index in range(1, 6)}
load, reports, alive = runner._load_report, [], []
class Report(dict):
pass
def tracked(path):
report = Report(load(path))
reports.append(weakref.ref(report))
return report
def before(name):
gc.collect()
alive.append(sum(report() is not None for report in reports))
monkeypatch.setattr(runner, "_load_report", tracked)
environment = {"SCAN_WORKERS": "1"}
assert (
runner.run(workspace, tmp_path / "reports", environment, execute=parallel_cli(responses, [], before=before))
== 0
)
# Only the latest result may still be referenced when the next scan starts.
assert alive == [0, 1, 1, 1, 1]
def worker_directory(env, name):
return Path(env[name]) / ("worker-1" if name == "HOME" else "skillevaluator-worker-1")
@pytest.mark.parametrize("name", ["HOME", "TMPDIR"])
def test_symlinked_worker_directory_is_rejected(tmp_path, name):
env = {"HOME": str(tmp_path / "home"), "TMPDIR": str(tmp_path / "tmp")}
directory = worker_directory(env, name)
directory.parent.mkdir()
(tmp_path / "elsewhere").mkdir(mode=0o700)
directory.symlink_to(tmp_path / "elsewhere", target_is_directory=True)
with pytest.raises(runner.ConfigurationError):
runner.worker_environment(env, 1)
@posix_only
@pytest.mark.parametrize("name", ["HOME", "TMPDIR"])
def test_existing_worker_directory_open_to_others_is_rejected(tmp_path, name):
env = {"HOME": str(tmp_path / "home"), "TMPDIR": str(tmp_path / "tmp")}
directory = worker_directory(env, name)
directory.mkdir(parents=True)
directory.chmod(0o777)
with pytest.raises(runner.ConfigurationError):
runner.worker_environment(env, 1)
assert stat.S_IMODE(directory.stat().st_mode) == 0o777
@posix_only
def test_worker_directory_owned_by_another_user_is_rejected(tmp_path, monkeypatch):
env = {"HOME": str(tmp_path / "home"), "TMPDIR": str(tmp_path / "tmp")}
uid = os.getuid()
monkeypatch.setattr(runner.os, "getuid", lambda: uid + 1)
with pytest.raises(runner.ConfigurationError):
runner.worker_environment(env, 1)
@posix_only
def test_existing_private_worker_directories_are_reused(tmp_path):
env = {"HOME": str(tmp_path / "home"), "TMPDIR": str(tmp_path / "tmp")}
for name in ("HOME", "TMPDIR"):
worker_directory(env, name).mkdir(mode=0o700, parents=True)
(worker_directory(env, name) / "kept").write_text("", encoding="utf-8")
isolated = runner.worker_environment(env, 1)
for name in ("HOME", "TMPDIR"):
assert isolated[name] == str(worker_directory(env, name))
assert (worker_directory(env, name) / "kept").is_file()
def test_symlinked_worker_directory_fails_before_scanning(tmp_path):
workspace = tmp_path / "workspace"
skill(workspace, "skills/a")
audit = tmp_path / "audit-tmp"
audit.mkdir()
(tmp_path / "elsewhere").mkdir(mode=0o700)
(audit / "skillevaluator-worker-1").symlink_to(tmp_path / "elsewhere", target_is_directory=True)
reports, calls = tmp_path / "reports", []
assert runner.run(workspace, reports, {"TMPDIR": str(audit)}, execute=fake_cli([], calls)) == 1
assert not calls
summary = json.loads((reports / "summary.json").read_text())
assert summary["overall_status"] == "error"
assert summary["error"] == "Scan worker directories must be private directories owned by this user."
@pytest.mark.parametrize("workers", ["1", "4"])
def test_unexpected_worker_error_is_a_row_error_and_later_skills_still_run(tmp_path, capsys, workers):
workspace = tmp_path / "workspace"
for name in ("a", "b", "c"):
skill(workspace, f"skills/{name}")
reports, calls = tmp_path / "reports", []
passing = parallel_cli({"skill-002": (0, payload()), "skill-003": (0, payload())}, calls)
def execute(argv, **kwargs):
if report_dir(Path(argv[argv.index("-o") + 1])) == "skill-001":
raise RuntimeError("untrusted worker text")
return passing(argv, **kwargs)
assert runner.run(workspace, reports, {"SCAN_WORKERS": workers}, execute=execute) == 1
assert sorted(name for name, _ in calls) == ["skill-002", "skill-003"]
rows = json.loads((reports / "summary.json").read_text())["skills"]
assert rows[0]["error"] == "Evaluator execution failed unexpectedly; no result is claimed."
assert [row["passed"] for row in rows] == [False, True, True]
captured = capsys.readouterr()
for name in ("a", "b", "c"):
assert f'Skill result: "skills/{name}"' in captured.out
assert "untrusted worker text" not in captured.out + captured.err
assert all("untrusted worker text" not in text for text in scan_outputs(reports).values())