Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(cli): preflight before paid autopilot dataset generation (#434)
Narendran Raghavand616c5efc017
diff --git a/src/skillevaluator/cli.py b/src/skillevaluator/cli.pyindex 62294095..8d54c47d 100644--- a/src/skillevaluator/cli.py+++ b/src/skillevaluator/cli.py@@ -78,6 +78,8 @@ from skillevaluator.validators.pii_waivers import ()if TYPE_CHECKING:+ from collections.abc import Callable+from skillevaluator.reporting.presentation import FinalRenderContext@@ -1064,7 +1066,14 @@ def _offer_local_runtime_install(raw_agents: str | None) -> None:console.print(f" [green]✓ {agent} installed[/green]")-def _ensure_autopilot_dataset(skill_path: Path, *, quiet: bool = False, progress: str = "auto") -> str | None:+def _ensure_autopilot_dataset(+ skill_path: Path,+ *,+ quiet: bool = False,+ progress: str = "auto",+ evaluation_params: dict[str, Any] | None = None,+ prepare_environment: Callable[[], None] | None = None,+) -> str | None:"""Ensure an evaluation source exists, generating one when missing.Mirrors the standalone ``evaluate --autopilot`` behavior: reuse an existing@@ -1099,6 +1108,22 @@ def _ensure_autopilot_dataset(skill_path: Path, *, quiet: bool = False, progressecho("Autopilot: no public provider key is configured; generating one deterministic case.")else:no_llm = False+ if evaluation_params is not None:+ from skillevaluator.tier3.workflow import (+ _preflight_environment,+ _preflight_options,+ _source_snapshot,+ )++ # Match the direct Tier 3 workflow before spending a generation+ # request. Keep keyless deterministic starters available, and+ # leave existing authored sources untouched above.+ params = {**evaluation_params, "skill_path": skill_path}+ with _source_snapshot(skill_path) as source_path:+ environment_kwargs = _preflight_options(source_path, params)+ if prepare_environment is not None:+ prepare_environment()+ _preflight_environment(params, environment_kwargs=environment_kwargs)provider_name, model_name = safe_llm_labels(provider.provider, provider.model)echo(f"Autopilot: generating one case with {provider_name} / {model_name}.")@@ -2339,14 +2364,16 @@ def validate(view.tier_start(0)view.tier_progress(0, [stage_hint_row("status", "preparing evaluation dataset…")])try:- autopilot_dataset_note = _ensure_autopilot_dataset(tier3_path, quiet=quiet)+ autopilot_dataset_note = _ensure_autopilot_dataset(+ tier3_path, quiet=quiet, evaluation_params=click.get_current_context().params+ )except (Exception, SystemExit) as exc:try:- error_detail = sanitize_tier3_console_text(str(exc), limit=72)+ error_detail = sanitize_tier3_console_text(str(exc), limit=1024)except Exception:error_detail = type(exc).__name__autopilot_error = sanitize_tier3_console_text(- f"autopilot dataset generation failed: {error_detail or type(exc).__name__}"+ f"autopilot dataset generation failed: {error_detail or type(exc).__name__}", limit=1200)if not quiet:click.echo(f"Warning: {autopilot_error}", err=True)@@ -2512,6 +2539,12 @@ def validate(repo_context_exclude_paths=tuple(repo_context_exclude_paths),)results.append(tier3_result)+ if autopilot_error:+ # Invalid/missing sources may be reported as a failed Tier 3 check,+ # rather than a skip. Preserve the preparation diagnostic in both+ # cases so users can recover from the reason generation stopped.+ tier3_result.metadata["autopilot_error"] = autopilot_error+ tier3_result.add_warning(autopilot_error)# The panel follows the formats this run emits: with cli the full report# lists the source errors under the Tier 3 finding, so the panel keeps# the count; it shortens its own list only when an HTML or JSON report@@ -3259,8 +3292,20 @@ def evaluate(evaluated_source_revision,evaluator_container_revision,)+ local_runtime_offered = False++ def prepare_local_runtime() -> None:+ nonlocal local_runtime_offered+ _offer_local_runtime_install(agents)+ local_runtime_offered = True+if autopilot:- _ensure_autopilot_dataset(skill_path, progress=progress)+ _ensure_autopilot_dataset(+ skill_path,+ progress=progress,+ evaluation_params=click.get_current_context().params,+ prepare_environment=prepare_local_runtime if env_mode == "local" else None,+ )parsed_environment_kwargs = _parse_environment_kwargs_cli(environment_kwarg, env_mode=env_mode)@@ -3296,7 +3341,8 @@ def evaluate()try:if env_mode == "local":- _offer_local_runtime_install(agents)+ if not local_runtime_offered:+ prepare_local_runtime()from rich.panel import Panelfrom rich.text import Textdiff --git a/src/skillevaluator/tier3/workflow.py b/src/skillevaluator/tier3/workflow.pyindex a667f212..aa2884d2 100644--- a/src/skillevaluator/tier3/workflow.py+++ b/src/skillevaluator/tier3/workflow.py@@ -149,6 +149,16 @@ def _preflight_options(skill_path: Path, params: dict[str, Any]) -> dict[str, Anresolved_environment_kwargs = _resolve_workflow_environment_kwargs(params)if harbor.get("stop_on_pass", False) and harbor.get("n_attempts", 1) == 1:raise ValueError("stop_on_pass requires n_attempts > 1")+ if harbor.get("pre_agent_setup") and not params.get("skip_baseline"):+ if params.get("skip_with_skill"):+ raise ValueError(+ "harbor.pre_agent_setup/setup_commands cannot run in a baseline-only run (--skip-with-skill); "+ "the baseline arm must stay isolated from skill-owned setup commands"+ )+ raise ValueError(+ "harbor.pre_agent_setup/setup_commands cannot run in a paired evaluation; "+ "use --skip-baseline for a with-skill-only run"+ )workspace = effective.get("skill_workspace", {})# Match the expert callback: CLI paths are relative to the caller, whilediff --git a/tests/test_evaluation_service.py b/tests/test_evaluation_service.pyindex f7959d23..17a4e61f 100644--- a/tests/test_evaluation_service.py+++ b/tests/test_evaluation_service.py@@ -296,6 +296,198 @@ def _write_one_case(skill: Path, marker: str = "generated") -> None:)[email protected]("command", ["validate", "evaluate", "tier3 evaluate"])+ ("flags", "diagnostic"),+ [+ (["--agents", "unknown-agent"], "Unknown agent"),+ (["--agent-model", "not-an-override"], "AGENT=MODEL"),+ (["--n-attempts", "0"], "n_attempts"),+ (["--stop-on-pass", "--n-attempts", "1"], "stop_on_pass"),+ (["--agent-model", "opencode=other-model"], "catalog model ID"),+ ],+)+def test_autopilot_rejects_invalid_run_before_paid_generation(monkeypatch, tmp_path, command, flags, diagnostic):+ from skillevaluator import cli as cli_module++ skill = _autopilot_skill(tmp_path)+ monkeypatch.setenv("SKILL_EVAL_LLM_PROVIDER", "nv_build")+ monkeypatch.setenv("NVIDIA_API_KEY", "test-key-never-sent")+ monkeypatch.setattr(cli_module, "_offer_scanner_install", lambda *_a, **_kw: None)+ monkeypatch.setattr(cli_module, "run_validation", lambda *_a, **_kw: [])+ generated = []++ def generate(_self, path, *, use_llm):+ generated.append(use_llm)+ _write_one_case(path)++ monkeypatch.setattr(EvaluationService, "create_autopilot_dataset", generate)+ monkeypatch.setattr(+ EvaluationService,+ "evaluate",+ lambda *_a, **_kw: {"execution_status": "succeeded", "execution_errors": []},+ )+ args = [*command.split(), str(skill), *flags]+ args += ["--no-tier2", "-o", str(tmp_path / "reports")] if command == "validate" else ["--autopilot"]++ result = CliRunner().invoke(cli, args)++ assert generated == [], result.output+ assert not (skill / "evals" / "evals.json").exists()+ if command == "validate":+ report = json.loads(next((tmp_path / "reports").glob("*.json")).read_text())+ assert diagnostic in json.dumps(report)+ assert result.exit_code == 0, result.output+ else:+ assert result.exit_code != 0+ assert diagnostic in result.output++[email protected]("command", ["validate", "evaluate", "tier3 evaluate"])+def test_autopilot_missing_docker_does_not_spend_generation_request(monkeypatch, tmp_path, command):+ from skillevaluator import cli as cli_module+ from skillevaluator.tier3.harbor import runner++ skill = _autopilot_skill(tmp_path)+ monkeypatch.setenv("SKILL_EVAL_LLM_PROVIDER", "nv_build")+ monkeypatch.setenv("NVIDIA_API_KEY", "test-key-never-sent")+ monkeypatch.setattr(cli_module, "_offer_scanner_install", lambda *_a, **_kw: None)+ monkeypatch.setattr(cli_module, "run_validation", lambda *_a, **_kw: [])+ monkeypatch.setattr(runner, "_harbor_bin", lambda: "/test/harbor")++ def missing_docker(*_a, **_kw):+ raise FileNotFoundError("docker")++ monkeypatch.setattr(runner.subprocess, "run", missing_docker)+ monkeypatch.setattr(+ EvaluationService, "create_autopilot_dataset", lambda *_a, **_kw: pytest.fail("must not spend generation")+ )+ args = [*command.split(), str(skill)]+ args += ["--no-tier2", "-o", str(tmp_path / "reports")] if command == "validate" else ["--autopilot"]++ result = CliRunner().invoke(cli, args)++ assert not (skill / "evals").exists()+ if command == "validate":+ report = json.loads(next((tmp_path / "reports").glob("*.json")).read_text())+ assert "Docker Compose v2 is required" in json.dumps(report)+ assert result.exit_code == 0, result.output+ else:+ assert result.exit_code != 0+ assert "Docker Compose v2 is required" in result.output++[email protected]("command", ["evaluate", "tier3 evaluate"])[email protected]("invalid_options", [False, True])+def test_local_autopilot_offers_missing_runtime_before_environment_check(+ monkeypatch, tmp_path, command, invalid_options+):+ from skillevaluator import cli as cli_module+ from skillevaluator.tier3.harbor import runner++ skill = _autopilot_skill(tmp_path)+ monkeypatch.setenv("SKILL_EVAL_LLM_PROVIDER", "nv_build")+ monkeypatch.setenv("NVIDIA_API_KEY", "test-key-never-sent")+ calls = []++ def prerequisites(**_kw):+ return [] if "offer-install" in calls else ["Local opencode CLI missing; install opencode"]++ def generate(_self, path, *, use_llm):+ assert use_llm is True+ calls.append("generate")+ _write_one_case(path)++ def evaluate(*_a, **_kw):+ calls.append("evaluate")+ return {"execution_status": "succeeded", "execution_errors": []}++ monkeypatch.setattr(runner, "_check_prerequisites", prerequisites)+ monkeypatch.setattr(cli_module, "_offer_local_runtime_install", lambda _agents: calls.append("offer-install"))+ monkeypatch.setattr(EvaluationService, "create_autopilot_dataset", generate)+ monkeypatch.setattr(EvaluationService, "evaluate", evaluate)++ args = [*command.split(), str(skill), "--autopilot", "--env-mode", "local"]+ if invalid_options:+ args.extend(["--n-attempts", "0"])+ result = CliRunner().invoke(cli, args)++ if invalid_options:+ assert result.exit_code != 0+ assert "n_attempts" in result.output+ assert calls == []+ else:+ assert result.exit_code == 0, result.output+ assert calls == ["offer-install", "generate", "evaluate"]++[email protected]("setup_field", ["pre_agent_setup", "setup_commands"])[email protected]("setup_value", ["echo setup", ["echo setup"]])+ ("command", "arm_flags", "compatible"),+ [+ ("validate", [], False),+ ("evaluate", [], False),+ ("tier3 evaluate", [], False),+ ("evaluate", ["--skip-with-skill"], False),+ ("tier3 evaluate", ["--skip-with-skill"], False),+ ("validate", ["--skip-baseline"], True),+ ("evaluate", ["--skip-baseline"], True),+ ("tier3 evaluate", ["--skip-baseline"], True),+ ],+)+def test_autopilot_validates_setup_commands_against_evaluation_arms(+ monkeypatch, tmp_path, command, arm_flags, compatible, setup_field, setup_value+):+ from skillevaluator import cli as cli_module+ from skillevaluator.tier3.harbor import runner++ skill = _autopilot_skill(tmp_path)+ (skill / "evals").mkdir()+ (skill / "evals" / "config.yml").write_text(json.dumps({"schema_version": 1, "harbor": {setup_field: setup_value}}))+ monkeypatch.setenv("SKILL_EVAL_LLM_PROVIDER", "nv_build")+ monkeypatch.setenv("NVIDIA_API_KEY", "test-key-never-sent")+ monkeypatch.setattr(cli_module, "_offer_scanner_install", lambda *_a, **_kw: None)+ monkeypatch.setattr(cli_module, "run_validation", lambda *_a, **_kw: [])+ monkeypatch.setattr(runner, "_check_prerequisites", lambda **_kw: [])+ generated = []++ def generate(_self, path, *, use_llm):+ generated.append(use_llm)+ (path / "evals" / "evals.json").write_text(+ json.dumps(+ {+ "skill_name": path.name,+ "evals": [{"id": "setup-001", "prompt": "Run the task.", "expected_output": "Done."}],+ }+ )+ )++ monkeypatch.setattr(EvaluationService, "create_autopilot_dataset", generate)+ monkeypatch.setattr(+ EvaluationService,+ "evaluate",+ lambda *_a, **_kw: {"execution_status": "succeeded", "execution_errors": []},+ )+ args = [*command.split(), str(skill), *arm_flags]+ args += ["--no-tier2", "-o", str(tmp_path / "reports")] if command == "validate" else ["--autopilot"]++ result = CliRunner().invoke(cli, args)++ assert generated == ([True] if compatible else []), result.output+ assert (skill / "evals" / "evals.json").exists() is compatible+ if compatible or command == "validate":+ assert result.exit_code == 0, result.output+ else:+ assert result.exit_code != 0+ if not compatible:+ diagnostic = "baseline-only run" if "--skip-with-skill" in arm_flags else "paired evaluation"+ output = result.output+ if command == "validate":+ output += next((tmp_path / "reports").glob("*.json")).read_text()+ assert diagnostic in output++def test_cli_autopilot_reuses_existing_dataset_unchanged_and_preserves_skip_baseline(monkeypatch: pytest.MonkeyPatch,tmp_path: Path,@@ -329,12 +521,14 @@ def test_cli_autopilot_uses_keyed_llm_one_case_generation_and_forwards_options(monkeypatch: pytest.MonkeyPatch,tmp_path: Path,) -> None:- from skillevaluator import provider_config+ from skillevaluator.tier3.harbor import runnerskill = _autopilot_skill(tmp_path)calls = []captured: dict[str, object] = {}- monkeypatch.setattr(provider_config, "resolve_llm_provider", lambda: SimpleNamespace(provider="openai", model="m"))+ monkeypatch.setenv("SKILL_EVAL_LLM_PROVIDER", "nv_build")+ monkeypatch.setenv("NVIDIA_API_KEY", "test-key-never-sent")+ monkeypatch.setattr(runner, "_check_prerequisites", lambda **_kw: [])def create(_self, _skill_path, *, use_llm):calls.append(use_llm)@@ -407,11 +601,13 @@ def test_cli_autopilot_warns_and_falls_back_when_llm_generation_errors(monkeypatch: pytest.MonkeyPatch,tmp_path: Path,) -> None:- from skillevaluator import provider_configfrom skillevaluator.inference.client import LLMClient+ from skillevaluator.tier3.harbor import runnerskill = _autopilot_skill(tmp_path)- monkeypatch.setattr(provider_config, "resolve_llm_provider", lambda: SimpleNamespace(provider="openai", model="m"))+ monkeypatch.setenv("SKILL_EVAL_LLM_PROVIDER", "nv_build")+ monkeypatch.setenv("NVIDIA_API_KEY", "test-key-never-sent")+ monkeypatch.setattr(runner, "_check_prerequisites", lambda **_kw: [])monkeypatch.setattr(LLMClient,"completions",