Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
{
"commands": {
"ci": {
"commands": {
"demo": {
"params": [
{
"choices": [
"auto",
"github",
"gitlab"
],
"default": "auto",
"name": "platform",
"opts": [
"--platform"
],
"param_type": "option"
},
{
"name": "tiers",
"opts": [
"--tiers"
],
"param_type": "option",
"type": "text"
},
{
"default": "<callable pathlib.Path.cwd>",
"name": "repo",
"opts": [
"--repo"
],
"param_type": "option",
"type": "directory"
},
{
"default": "False",
"is_flag": true,
"name": "no_input",
"opts": [
"--no-input"
],
"param_type": "option",
"type": "boolean"
}
]
},
"init": {
"params": [
{
"name": "tiers",
"opts": [
"--tiers"
],
"param_type": "option",
"type": "text"
},
{
"choices": [
"auto",
"github",
"gitlab"
],
"default": "auto",
"name": "platform",
"opts": [
"--platform"
],
"param_type": "option"
},
{
"choices": [
"docker",
"local"
],
"default": "docker",
"name": "env_mode",
"opts": [
"--env-mode"
],
"param_type": "option"
},
{
"choices": [
"openai",
"anthropic",
"nv_build",
"openai-compatible"
],
"default": "openai",
"name": "provider",
"opts": [
"--provider"
],
"param_type": "option"
},
{
"choices": [
"openai",
"nv_build",
"openai-compatible"
],
"name": "embedding_provider",
"opts": [
"--embedding-provider"
],
"param_type": "option"
},
{
"name": "model",
"opts": [
"--model"
],
"param_type": "option",
"type": "text"
},
{
"name": "embedding_model",
"opts": [
"--embedding-model"
],
"param_type": "option",
"type": "text"
},
{
"default": "False",
"is_flag": true,
"name": "tier1_llm",
"opts": [
"--tier1-llm"
],
"param_type": "option",
"type": "boolean"
},
{
"choices": [
"openai",
"anthropic",
"nv_build"
],
"name": "tier1_provider",
"opts": [
"--tier1-provider"
],
"param_type": "option"
},
{
"default": "",
"name": "tier1_model",
"opts": [
"--tier1-model"
],
"param_type": "option",
"type": "text"
},
{
"default": "",
"name": "base_url",
"opts": [
"--base-url"
],
"param_type": "option",
"type": "text"
},
{
"default": "",
"name": "embedding_base_url",
"opts": [
"--embedding-base-url"
],
"param_type": "option",
"type": "text"
},
{
"choices": [
"openai",
"anthropic",
"nv_build",
"openai-compatible"
],
"name": "tier3_provider",
"opts": [
"--tier3-provider"
],
"param_type": "option"
},
{
"default": "",
"name": "tier3_model",
"opts": [
"--tier3-model"
],
"param_type": "option",
"type": "text"
},
{
"default": "",
"name": "tier3_base_url",
"opts": [
"--tier3-base-url"
],
"param_type": "option",
"type": "text"
},
{
"multiple": true,
"name": "agent_model",
"opts": [
"--agent-model"
],
"param_type": "option",
"type": "text"
},
{
"default": "SKILL_EVAL_LLM_API_KEY",
"name": "gateway_secret",
"opts": [
"--gateway-secret"
],
"param_type": "option",
"type": "text"
},
{
"default": "False",
"is_flag": true,
"name": "benchmark_pr",
"opts": [
"--benchmark-pr"
],
"param_type": "option",
"type": "boolean"
},
{
"name": "agents",
"opts": [
"--agents"
],
"param_type": "option",
"type": "text"
},
{
"name": "runner",
"opts": [
"--runner"
],
"param_type": "option",
"type": "text"
},
{
"default": "False",
"is_flag": true,
"name": "tier3_required",
"opts": [
"--tier3-required"
],
"param_type": "option",
"type": "boolean"
},
{
"default": "1",
"name": "n_attempts",
"opts": [
"--n-attempts"
],
"param_type": "option",
"type": "integer range"
},
{
"default": "1",
"name": "n_concurrent",
"opts": [
"--n-concurrent"
],
"param_type": "option",
"type": "integer range"
},
{
"default": "300",
"name": "timeout_minutes",
"opts": [
"--timeout-minutes"
],
"param_type": "option",
"type": "integer range"
},
{
"default": "skillevaluator",
"name": "environment",
"opts": [
"--environment"
],
"param_type": "option",
"type": "text"
},
{
"default": "<callable pathlib.Path.cwd>",
"name": "repo",
"opts": [
"--repo"
],
"param_type": "option",
"type": "directory"
},
{
"name": "release_path",
"opts": [
"--release"
],
"param_type": "option",
"type": "file"
},
{
"multiple": true,
"name": "paths",
"opts": [
"--paths"
],
"param_type": "option",
"type": "text"
},
{
"name": "min_score",
"opts": [
"--min-score"
],
"param_type": "option",
"type": "integer range"
},
{
"choices": [
"auto",
"all"
],
"name": "scan_mode",
"opts": [
"--scan-mode"
],
"param_type": "option"
},
{
"name": "default_branch",
"opts": [
"--default-branch"
],
"param_type": "option",
"type": "text"
},
{
"is_flag": true,
"name": "full",
"opts": [
"--full"
],
"param_type": "option",
"secondary_opts": [
"--checks-only"
],
"type": "boolean"
},
{
"name": "custom_url",
"opts": [
"--report-url"
],
"param_type": "option",
"type": "text"
},
{
"choices": [
"preview",
"save",
"pr"
],
"name": "mode",
"opts": [
"--mode"
],
"param_type": "option"
},
{
"default": "False",
"is_flag": true,
"name": "no_input",
"opts": [
"--no-input"
],
"param_type": "option",
"type": "boolean"
},
{
"default": "False",
"is_flag": true,
"name": "show_files",
"opts": [
"--show-files"
],
"param_type": "option",
"type": "boolean"
},
{
"name": "output",
"opts": [
"--output"
],
"param_type": "option",
"type": "path"
}
]
},
"status": {
"params": [
{
"choices": [
"auto",
"github",
"gitlab"
],
"default": "auto",
"name": "platform",
"opts": [
"--platform"
],
"param_type": "option"
},
{
"name": "tiers",
"opts": [
"--tiers"
],
"param_type": "option",
"type": "text"
},
{
"default": "<callable pathlib.Path.cwd>",
"name": "repo",
"opts": [
"--repo"
],
"param_type": "option",
"type": "directory"
}
]
}
},
"params": []
},
"compare": {
"params": [
{
"name": "skill_path",
"opts": [
"skill_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"name": "results_dir",
"opts": [
"--results-dir"
],
"param_type": "option",
"type": "directory"
}
]
},
"context-optimization-check": {
"params": [
{
"name": "skill_path",
"opts": [
"skill_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"default": "0.8",
"name": "threshold",
"opts": [
"--threshold"
],
"param_type": "option",
"type": "float"
},
{
"name": "model",
"opts": [
"--model"
],
"param_type": "option",
"type": "text"
},
{
"name": "llm_model",
"opts": [
"--llm-model"
],
"param_type": "option",
"type": "text"
},
{
"choices": [
"cli",
"json",
"html",
"markdown",
"sarif",
"junit"
],
"default": "('cli',)",
"multiple": true,
"name": "report_formats",
"opts": [
"--report",
"-r"
],
"param_type": "option"
},
{
"default": "reports",
"name": "output_dir",
"opts": [
"--output-dir",
"-o"
],
"param_type": "option",
"type": "directory"
}
]
},
"create-eval-dataset": {
"params": [
{
"name": "skill_path",
"opts": [
"skill_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"default": "False",
"is_flag": true,
"name": "full",
"opts": [
"--full"
],
"param_type": "option",
"type": "boolean"
},
{
"default": "False",
"is_flag": true,
"name": "no_llm",
"opts": [
"--no-llm"
],
"param_type": "option",
"type": "boolean"
},
{
"default": "False",
"is_flag": true,
"name": "dry_run",
"opts": [
"--dry-run"
],
"param_type": "option",
"type": "boolean"
},
{
"default": "False",
"is_flag": true,
"name": "force",
"opts": [
"--force"
],
"param_type": "option",
"type": "boolean"
},
{
"name": "prompt",
"opts": [
"--prompt"
],
"param_type": "option",
"type": "path"
},
{
"choices": [
"auto",
"skill",
"intent",
"trace"
],
"default": "auto",
"name": "grounding",
"opts": [
"--grounding"
],
"param_type": "option"
},
{
"default": "False",
"is_flag": true,
"name": "refine",
"opts": [
"--refine"
],
"param_type": "option",
"type": "boolean"
},
{
"name": "from_results",
"opts": [
"--from-results"
],
"param_type": "option",
"type": "path"
},
{
"name": "results_dir",
"opts": [
"--results-dir"
],
"param_type": "option",
"type": "directory"
}
]
},
"dashboard": {
"params": [
{
"name": "paths",
"opts": [
"paths"
],
"param_type": "argument",
"type": "path"
},
{
"name": "port",
"opts": [
"--port"
],
"param_type": "option",
"type": "integer range"
},
{
"default": "False",
"is_flag": true,
"name": "no_browser",
"opts": [
"--no-browser"
],
"param_type": "option",
"type": "boolean"
}
]
},
"dedup-scan": {
"params": [
{
"name": "skill_path",
"opts": [
"skill_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"default": "0.8",
"name": "threshold",
"opts": [
"--threshold"
],
"param_type": "option",
"type": "float"
},
{
"name": "llm_model",
"opts": [
"--llm-model"
],
"param_type": "option",
"type": "text"
},
{
"name": "model",
"opts": [
"--model"
],
"param_type": "option",
"type": "text"
},
{
"choices": [
"cli",
"json",
"html",
"markdown",
"sarif",
"junit"
],
"default": "('cli',)",
"multiple": true,
"name": "report_formats",
"opts": [
"--report",
"-r"
],
"param_type": "option"
},
{
"default": "reports",
"name": "output_dir",
"opts": [
"--output-dir",
"-o"
],
"param_type": "option",
"type": "directory"
}
]
},
"doctor": {
"params": [
{
"choices": [
"1",
"2",
"3",
"all"
],
"default": "3",
"name": "tier",
"opts": [
"--tier"
],
"param_type": "option"
},
{
"name": "checks",
"opts": [
"--checks"
],
"param_type": "option",
"type": "text"
},
{
"default": "60.0",
"name": "timeout",
"opts": [
"--timeout"
],
"param_type": "option",
"type": "float range"
},
{
"name": "agents",
"opts": [
"--agents",
"-a"
],
"param_type": "option",
"type": "text"
},
{
"choices": [
"docker",
"kata",
"daytona",
"e2b",
"modal",
"runloop",
"langsmith",
"ec2",
"gke",
"ack",
"openshift",
"novita",
"apple-container",
"singularity",
"islo",
"tensorlake",
"cwsandbox",
"wandb",
"use-computer",
"blaxel",
"beam",
"skypilot",
"hyperbrowser",
"vercel",
"runta",
"mosaic",
"local"
],
"default": "docker",
"name": "env_mode",
"opts": [
"--env-mode"
],
"param_type": "option"
},
{
"multiple": true,
"name": "environment_kwarg",
"opts": [
"--ek",
"--environment-kwarg"
],
"param_type": "option",
"type": "text"
},
{
"multiple": true,
"name": "agent_model",
"opts": [
"--agent-model"
],
"param_type": "option",
"type": "text"
},
{
"choices": [
"harbor",
"nemo-gym"
],
"default": "harbor",
"name": "backend",
"opts": [
"--backend"
],
"param_type": "option"
},
{
"default": "False",
"is_flag": true,
"name": "verify_models",
"opts": [
"--verify-models"
],
"param_type": "option",
"type": "boolean"
}
]
},
"evaluate": {
"params": [
{
"name": "skill_path",
"opts": [
"skill_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"name": "agents",
"opts": [
"--agents",
"-a"
],
"param_type": "option",
"type": "text"
},
{
"choices": [
"docker",
"kata",
"daytona",
"e2b",
"modal",
"runloop",
"langsmith",
"ec2",
"gke",
"ack",
"openshift",
"novita",
"apple-container",
"singularity",
"islo",
"tensorlake",
"cwsandbox",
"wandb",
"use-computer",
"blaxel",
"beam",
"skypilot",
"hyperbrowser",
"vercel",
"runta",
"mosaic",
"local"
],
"default": "docker",
"name": "env_mode",
"opts": [
"--env-mode"
],
"param_type": "option"
},
{
"choices": [
"harbor",
"nemo-gym"
],
"default": "harbor",
"name": "backend",
"opts": [
"--backend"
],
"param_type": "option"
},
{
"multiple": true,
"name": "environment_kwarg",
"opts": [
"--ek",
"--environment-kwarg"
],
"param_type": "option",
"type": "text"
},
{
"default": "False",
"is_flag": true,
"name": "autopilot",
"opts": [
"--autopilot"
],
"param_type": "option",
"type": "boolean"
},
{
"default": "False",
"is_flag": true,
"name": "skip_baseline",
"opts": [
"--skip-baseline"
],
"param_type": "option",
"type": "boolean"
},
{
"name": "reuse_baseline",
"opts": [
"--reuse-baseline"
],
"param_type": "option",
"type": "text"
},
{
"name": "compare_to",
"opts": [
"--compare-to"
],
"param_type": "option",
"type": "text"
},
{
"default": "False",
"is_flag": true,
"name": "skip_with_skill",
"opts": [
"--skip-with-skill"
],
"param_type": "option",
"type": "boolean"
},
{
"name": "n_attempts",
"opts": [
"--n-attempts"
],
"param_type": "option",
"type": "integer"
},
{
"name": "max_agent_tokens",
"opts": [
"--max-agent-tokens"
],
"param_type": "option",
"type": "integer range"
},
{
"name": "max_judge_calls",
"opts": [
"--max-judge-calls"
],
"param_type": "option",
"type": "integer range"
},
{
"name": "max_attempts_total",
"opts": [
"--max-attempts-total"
],
"param_type": "option",
"type": "integer range"
},
{
"name": "max_cost_usd",
"opts": [
"--max-cost-usd"
],
"param_type": "option",
"type": "float range"
},
{
"name": "price_file",
"opts": [
"--price-file"
],
"param_type": "option",
"type": "file"
},
{
"name": "pass_threshold",
"opts": [
"--pass-threshold"
],
"param_type": "option",
"type": "float"
},
{
"is_flag": true,
"name": "stop_on_pass",
"opts": [
"--stop-on-pass"
],
"param_type": "option",
"secondary_opts": [
"--no-stop-on-pass"
],
"type": "boolean"
},
{
"name": "n_concurrent",
"opts": [
"--n-concurrent"
],
"param_type": "option",
"type": "integer"
},
{
"name": "max_agents",
"opts": [
"--max-agents"
],
"param_type": "option",
"type": "integer"
},
{
"name": "max_cases",
"opts": [
"--max-cases"
],
"param_type": "option",
"type": "integer range"
},
{
"name": "model",
"opts": [
"--model"
],
"param_type": "option",
"type": "text"
},
{
"multiple": true,
"name": "agent_model",
"opts": [
"--agent-model"
],
"param_type": "option",
"type": "text"
},
{
"choices": [
"preserve",
"rebase"
],
"name": "custom_dockerfile_mode",
"opts": [
"--custom-dockerfile-mode"
],
"param_type": "option"
},
{
"choices": [
"isolated",
"group"
],
"name": "skill_workspace_mode",
"opts": [
"--skill-workspace-mode"
],
"param_type": "option"
},
{
"multiple": true,
"name": "include_skills",
"opts": [
"--include-skills"
],
"param_type": "option",
"type": "path"
},
{
"default": "False",
"is_flag": true,
"name": "copy_repo",
"opts": [
"--copy-repo"
],
"param_type": "option",
"type": "boolean"
},
{
"choices": [
"default",
"default_plus_custom",
"custom_only"
],
"name": "grading_mode",
"opts": [
"--grading-mode"
],
"param_type": "option"
},
{
"name": "results_dir",
"opts": [
"--results-dir"
],
"param_type": "option",
"type": "directory"
},
{
"default": "False",
"is_flag": true,
"name": "harbor_keep_jobs",
"opts": [
"--harbor-keep-jobs"
],
"param_type": "option",
"type": "boolean"
},
{
"is_flag": true,
"name": "agent_runtime_preflight",
"opts": [
"--agent-runtime-preflight"
],
"param_type": "option",
"secondary_opts": [
"--no-agent-runtime-preflight"
],
"type": "boolean"
},
{
"is_flag": true,
"name": "tolerate_baseline_timeouts",
"opts": [
"--tolerate-baseline-timeouts"
],
"param_type": "option",
"secondary_opts": [
"--no-tolerate-baseline-timeouts"
],
"type": "boolean"
},
{
"name": "timeout_multiplier",
"opts": [
"--timeout-multiplier"
],
"param_type": "option",
"type": "float"
},
{
"name": "override_cpus",
"opts": [
"--override-cpus"
],
"param_type": "option",
"type": "integer"
},
{
"name": "override_memory_mb",
"opts": [
"--override-memory-mb"
],
"param_type": "option",
"type": "integer"
},
{
"name": "override_storage_mb",
"opts": [
"--override-storage-mb"
],
"param_type": "option",
"type": "integer"
},
{
"name": "evaluated_source_repository",
"opts": [
"--evaluated-source-repository"
],
"param_type": "option",
"type": "text"
},
{
"name": "evaluated_source_revision",
"opts": [
"--evaluated-source-revision"
],
"param_type": "option",
"type": "text"
},
{
"name": "evaluator_container_revision",
"opts": [
"--evaluator-container-revision"
],
"param_type": "option",
"type": "text"
},
{
"choices": [
"auto",
"rich",
"plain",
"off"
],
"default": "auto",
"name": "progress",
"opts": [
"--progress"
],
"param_type": "option"
},
{
"name": "progress_interval",
"opts": [
"--progress-interval"
],
"param_type": "option",
"type": "float"
},
{
"default": "False",
"is_flag": true,
"name": "verbose",
"opts": [
"--verbose"
],
"param_type": "option",
"type": "boolean"
}
]
},
"harbor-view": {
"params": [
{
"name": "jobs_dir",
"opts": [
"jobs_dir"
],
"param_type": "argument",
"required": true,
"type": "directory"
}
]
},
"harness-check": {
"params": [
{
"name": "target_path",
"opts": [
"target_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"default": "all",
"name": "harness",
"opts": [
"--harness"
],
"param_type": "option",
"type": "text"
},
{
"name": "harness_profiles",
"opts": [
"--harness-profiles"
],
"param_type": "option",
"type": "directory"
},
{
"default": "False",
"is_flag": true,
"name": "verbose",
"opts": [
"--verbose",
"-v"
],
"param_type": "option",
"type": "boolean"
},
{
"name": "json_path",
"opts": [
"--json"
],
"param_type": "option",
"type": "file"
},
{
"default": "0",
"name": "context_tokens",
"opts": [
"--context-tokens"
],
"param_type": "option",
"type": "integer range"
},
{
"default": "False",
"is_flag": true,
"name": "custom_provider",
"opts": [
"--custom-provider"
],
"param_type": "option",
"type": "boolean"
},
{
"default": "False",
"is_flag": true,
"name": "explicit_invocation",
"opts": [
"--explicit-invocation"
],
"param_type": "option",
"type": "boolean"
},
{
"default": "False",
"is_flag": true,
"name": "headless",
"opts": [
"--headless"
],
"param_type": "option",
"type": "boolean"
},
{
"multiple": true,
"name": "harness_versions",
"opts": [
"--harness-version"
],
"param_type": "option",
"type": "text"
}
]
},
"health-check": {
"params": [
{
"choices": [
"1",
"2",
"3",
"all"
],
"default": "3",
"name": "tier",
"opts": [
"--tier"
],
"param_type": "option"
},
{
"name": "checks",
"opts": [
"--checks"
],
"param_type": "option",
"type": "text"
},
{
"default": "60.0",
"name": "timeout",
"opts": [
"--timeout"
],
"param_type": "option",
"type": "float range"
},
{
"name": "agents",
"opts": [
"--agents",
"-a"
],
"param_type": "option",
"type": "text"
},
{
"choices": [
"docker",
"kata",
"daytona",
"e2b",
"modal",
"runloop",
"langsmith",
"ec2",
"gke",
"ack",
"openshift",
"novita",
"apple-container",
"singularity",
"islo",
"tensorlake",
"cwsandbox",
"wandb",
"use-computer",
"blaxel",
"beam",
"skypilot",
"hyperbrowser",
"vercel",
"runta",
"mosaic",
"local"
],
"default": "docker",
"name": "env_mode",
"opts": [
"--env-mode"
],
"param_type": "option"
},
{
"multiple": true,
"name": "environment_kwarg",
"opts": [
"--ek",
"--environment-kwarg"
],
"param_type": "option",
"type": "text"
}
]
},
"init-custom-grader": {
"params": [
{
"name": "skill_path",
"opts": [
"skill_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"choices": [
"default_plus_custom",
"custom_only"
],
"default": "default_plus_custom",
"name": "mode",
"opts": [
"--mode"
],
"param_type": "option"
},
{
"choices": [
"python",
"shell"
],
"default": "python",
"name": "language",
"opts": [
"--language"
],
"param_type": "option"
},
{
"default": "False",
"is_flag": true,
"name": "force",
"opts": [
"--force"
],
"param_type": "option",
"type": "boolean"
},
{
"default": "False",
"is_flag": true,
"name": "no_config",
"opts": [
"--no-config"
],
"param_type": "option",
"type": "boolean"
}
]
},
"init-harbor-task": {
"params": [
{
"name": "skill_path",
"opts": [
"skill_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"default": "False",
"is_flag": true,
"name": "force",
"opts": [
"--force"
],
"param_type": "option",
"type": "boolean"
},
{
"default": "case-001",
"name": "case_id",
"opts": [
"--case-id"
],
"param_type": "option",
"type": "text"
},
{
"choices": [
"default",
"default_plus_custom",
"custom_only"
],
"default": "custom_only",
"name": "mode",
"opts": [
"--mode"
],
"param_type": "option"
},
{
"choices": [
"python",
"shell"
],
"default": "python",
"name": "language",
"opts": [
"--language"
],
"param_type": "option"
},
{
"default": "False",
"is_flag": true,
"name": "with_config",
"opts": [
"--with-config"
],
"param_type": "option",
"type": "boolean"
}
]
},
"lint-scripts": {
"params": [
{
"name": "target_path",
"opts": [
"target_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"choices": [
"cli",
"json",
"html",
"markdown",
"sarif",
"junit"
],
"default": "('cli',)",
"multiple": true,
"name": "report_formats",
"opts": [
"--report",
"-r"
],
"param_type": "option"
},
{
"default": "reports",
"name": "output_dir",
"opts": [
"--output-dir",
"-o"
],
"param_type": "option",
"type": "directory"
}
]
},
"models": {
"params": [
{
"default": "10",
"name": "limit",
"opts": [
"--limit"
],
"param_type": "option",
"type": "integer range"
},
{
"default": "False",
"is_flag": true,
"name": "as_json",
"opts": [
"--json"
],
"param_type": "option",
"type": "boolean"
}
]
},
"pii-scan": {
"params": [
{
"name": "target_path",
"opts": [
"target_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"default": "False",
"is_flag": true,
"name": "llm_verify",
"opts": [
"--llm-verify"
],
"param_type": "option",
"type": "boolean"
},
{
"name": "pii_waivers",
"opts": [
"--pii-waivers"
],
"param_type": "option",
"type": "file"
},
{
"choices": [
"cli",
"json",
"html",
"markdown",
"sarif",
"junit"
],
"default": "('cli',)",
"multiple": true,
"name": "report_formats",
"opts": [
"--report",
"-r"
],
"param_type": "option"
},
{
"default": "reports",
"name": "output_dir",
"opts": [
"--output-dir",
"-o"
],
"param_type": "option",
"type": "directory"
}
]
},
"precommit-run": {
"params": [
{
"default": "70",
"name": "min_score",
"opts": [
"--min-score"
],
"param_type": "option",
"type": "integer"
},
{
"name": "quality_config",
"opts": [
"--quality-config"
],
"param_type": "option",
"type": "file"
},
{
"choices": [
"quality",
"schema",
"lint-scripts"
],
"name": "check",
"opts": [
"check"
],
"param_type": "argument",
"required": true
},
{
"name": "paths",
"opts": [
"paths"
],
"param_type": "argument",
"type": "text"
}
]
},
"quality-check": {
"params": [
{
"name": "target_path",
"opts": [
"target_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"default": "70",
"name": "min_score",
"opts": [
"--min-score"
],
"param_type": "option",
"type": "integer"
},
{
"name": "quality_config",
"opts": [
"--quality-config"
],
"param_type": "option",
"type": "file"
},
{
"choices": [
"cli",
"json",
"html",
"markdown",
"sarif",
"junit"
],
"default": "('cli',)",
"multiple": true,
"name": "report_formats",
"opts": [
"--report",
"-r"
],
"param_type": "option"
},
{
"default": "reports",
"name": "output_dir",
"opts": [
"--output-dir",
"-o"
],
"param_type": "option",
"type": "directory"
}
]
},
"rubric-eval": {
"params": [
{
"name": "target_path",
"opts": [
"target_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"default": "70",
"name": "min_score",
"opts": [
"--min-score"
],
"param_type": "option",
"type": "integer"
},
{
"choices": [
"cli",
"json",
"html",
"markdown",
"sarif",
"junit"
],
"default": "('cli',)",
"multiple": true,
"name": "report_formats",
"opts": [
"--report",
"-r"
],
"param_type": "option"
},
{
"default": "reports",
"name": "output_dir",
"opts": [
"--output-dir",
"-o"
],
"param_type": "option",
"type": "directory"
}
]
},
"security-scan": {
"params": [
{
"name": "target_path",
"opts": [
"target_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"default": "False",
"is_flag": true,
"name": "llm",
"opts": [
"--llm"
],
"param_type": "option",
"secondary_opts": [
"--no-llm"
],
"type": "boolean"
},
{
"default": "False",
"is_flag": true,
"name": "llm_verify",
"opts": [
"--llm-verify"
],
"param_type": "option",
"type": "boolean"
},
{
"name": "skillspector_baseline",
"opts": [
"--skillspector-baseline"
],
"param_type": "option",
"type": "file"
},
{
"choices": [
"cli",
"json",
"html",
"markdown",
"sarif",
"junit"
],
"default": "('cli',)",
"multiple": true,
"name": "report_formats",
"opts": [
"--report",
"-r"
],
"param_type": "option"
},
{
"default": "reports",
"name": "output_dir",
"opts": [
"--output-dir",
"-o"
],
"param_type": "option",
"type": "directory"
}
]
},
"similarity-check": {
"params": [
{
"name": "content_path",
"opts": [
"content_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"choices": [
"skill",
"rules",
"workflows",
"auto"
],
"default": "auto",
"name": "content_type",
"opts": [
"--type"
],
"param_type": "option"
},
{
"default": "0.75",
"name": "threshold",
"opts": [
"--threshold"
],
"param_type": "option",
"type": "float"
},
{
"default": "False",
"is_flag": true,
"name": "full_body",
"opts": [
"--full-body"
],
"param_type": "option",
"type": "boolean"
},
{
"name": "model",
"opts": [
"--model"
],
"param_type": "option",
"type": "text"
},
{
"default": "1024",
"name": "max_entries",
"opts": [
"--max-entries"
],
"param_type": "option",
"type": "integer range"
},
{
"default": "128000000",
"name": "max_scalar_comparisons",
"opts": [
"--max-scalar-comparisons"
],
"param_type": "option",
"type": "integer range"
},
{
"name": "catalog",
"opts": [
"--catalog"
],
"param_type": "option",
"type": "file"
},
{
"name": "save_catalog",
"opts": [
"--save-catalog"
],
"param_type": "option",
"type": "file"
},
{
"name": "cache",
"opts": [
"--cache"
],
"param_type": "option",
"type": "path"
},
{
"name": "save_cache",
"opts": [
"--save-cache"
],
"param_type": "option",
"type": "path"
},
{
"choices": [
"cli",
"json",
"html",
"markdown",
"sarif",
"junit"
],
"default": "('cli',)",
"multiple": true,
"name": "report_formats",
"opts": [
"--report",
"-r"
],
"param_type": "option"
},
{
"default": "reports",
"name": "output_dir",
"opts": [
"--output-dir",
"-o"
],
"param_type": "option",
"type": "directory"
}
]
},
"tier1": {
"commands": {
"lint-scripts": {
"params": [
{
"name": "target_path",
"opts": [
"target_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"choices": [
"cli",
"json",
"html",
"markdown",
"sarif",
"junit"
],
"default": "('cli',)",
"multiple": true,
"name": "report_formats",
"opts": [
"--report",
"-r"
],
"param_type": "option"
},
{
"default": "reports",
"name": "output_dir",
"opts": [
"--output-dir",
"-o"
],
"param_type": "option",
"type": "directory"
}
]
},
"pii-scan": {
"params": [
{
"name": "target_path",
"opts": [
"target_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"default": "False",
"is_flag": true,
"name": "llm_verify",
"opts": [
"--llm-verify"
],
"param_type": "option",
"type": "boolean"
},
{
"name": "pii_waivers",
"opts": [
"--pii-waivers"
],
"param_type": "option",
"type": "file"
},
{
"choices": [
"cli",
"json",
"html",
"markdown",
"sarif",
"junit"
],
"default": "('cli',)",
"multiple": true,
"name": "report_formats",
"opts": [
"--report",
"-r"
],
"param_type": "option"
},
{
"default": "reports",
"name": "output_dir",
"opts": [
"--output-dir",
"-o"
],
"param_type": "option",
"type": "directory"
}
]
},
"quality-check": {
"params": [
{
"name": "target_path",
"opts": [
"target_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"default": "70",
"name": "min_score",
"opts": [
"--min-score"
],
"param_type": "option",
"type": "integer"
},
{
"name": "quality_config",
"opts": [
"--quality-config"
],
"param_type": "option",
"type": "file"
},
{
"choices": [
"cli",
"json",
"html",
"markdown",
"sarif",
"junit"
],
"default": "('cli',)",
"multiple": true,
"name": "report_formats",
"opts": [
"--report",
"-r"
],
"param_type": "option"
},
{
"default": "reports",
"name": "output_dir",
"opts": [
"--output-dir",
"-o"
],
"param_type": "option",
"type": "directory"
}
]
},
"rubric-eval": {
"params": [
{
"name": "target_path",
"opts": [
"target_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"default": "70",
"name": "min_score",
"opts": [
"--min-score"
],
"param_type": "option",
"type": "integer"
},
{
"choices": [
"cli",
"json",
"html",
"markdown",
"sarif",
"junit"
],
"default": "('cli',)",
"multiple": true,
"name": "report_formats",
"opts": [
"--report",
"-r"
],
"param_type": "option"
},
{
"default": "reports",
"name": "output_dir",
"opts": [
"--output-dir",
"-o"
],
"param_type": "option",
"type": "directory"
}
]
},
"security-scan": {
"params": [
{
"name": "target_path",
"opts": [
"target_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"default": "False",
"is_flag": true,
"name": "llm",
"opts": [
"--llm"
],
"param_type": "option",
"secondary_opts": [
"--no-llm"
],
"type": "boolean"
},
{
"default": "False",
"is_flag": true,
"name": "llm_verify",
"opts": [
"--llm-verify"
],
"param_type": "option",
"type": "boolean"
},
{
"name": "skillspector_baseline",
"opts": [
"--skillspector-baseline"
],
"param_type": "option",
"type": "file"
},
{
"choices": [
"cli",
"json",
"html",
"markdown",
"sarif",
"junit"
],
"default": "('cli',)",
"multiple": true,
"name": "report_formats",
"opts": [
"--report",
"-r"
],
"param_type": "option"
},
{
"default": "reports",
"name": "output_dir",
"opts": [
"--output-dir",
"-o"
],
"param_type": "option",
"type": "directory"
}
]
},
"validate": {
"params": [
{
"name": "skill_path",
"opts": [
"skill_path"
],
"param_type": "argument",
"required": true,
"type": "directory"
},
{
"name": "checks",
"opts": [
"--checks"
],
"param_type": "option",
"type": "text"
},
{
"name": "skip",
"opts": [
"--skip"
],
"param_type": "option",
"type": "text"
},
{
"is_flag": true,
"name": "llm",
"opts": [
"--llm"
],
"param_type": "option",
"secondary_opts": [
"--no-llm"
],
"type": "boolean"
},
{
"default": "70",
"name": "min_score",
"opts": [
"--min-score"
],
"param_type": "option",
"type": "integer range"
},
{
"name": "quality_config",
"opts": [
"--quality-config"
],
"param_type": "option",
"type": "file"
},
{
"name": "pii_waivers",
"opts": [
"--pii-waivers"
],
"param_type": "option",
"type": "file"
},
{
"name": "profile",
"opts": [
"--profile"
],
"param_type": "option",
"type": "text"
},
{
"name": "policy_path",
"opts": [
"--policy"
],
"param_type": "option",
"type": "file"
},
{
"name": "previous_version",
"opts": [
"--previous-version"
],
"param_type": "option",
"type": "text"
},
{
"name": "skillspector_baseline",
"opts": [
"--skillspector-baseline"
],
"param_type": "option",
"type": "file"
},
{
"choices": [
"cli",
"json",
"html",
"markdown",
"sarif",
"junit"
],
"default": "('cli', 'json', 'html')",
"multiple": true,
"name": "report_formats",
"opts": [
"--report",
"-r"
],
"param_type": "option"
},
{
"default": "reports",
"name": "output_dir",
"opts": [
"--output-dir",
"-o"
],
"param_type": "option",
"type": "directory"
}
]
}
},
"params": [],
"workflow": {
"params": [
{
"name": "skill_path",
"opts": [
"skill_path"
],
"param_type": "argument",
"required": true,
"type": "directory"
},
{
"name": "checks",
"opts": [
"--checks"
],
"param_type": "option",
"type": "text"
},
{
"name": "skip",
"opts": [
"--skip"
],
"param_type": "option",
"type": "text"
},
{
"is_flag": true,
"name": "llm",
"opts": [
"--llm"
],
"param_type": "option",
"secondary_opts": [
"--no-llm"
],
"type": "boolean"
},
{
"default": "70",
"name": "min_score",
"opts": [
"--min-score"
],
"param_type": "option",
"type": "integer range"
},
{
"name": "quality_config",
"opts": [
"--quality-config"
],
"param_type": "option",
"type": "file"
},
{
"name": "pii_waivers",
"opts": [
"--pii-waivers"
],
"param_type": "option",
"type": "file"
},
{
"name": "profile",
"opts": [
"--profile"
],
"param_type": "option",
"type": "text"
},
{
"name": "policy_path",
"opts": [
"--policy"
],
"param_type": "option",
"type": "file"
},
{
"name": "previous_version",
"opts": [
"--previous-version"
],
"param_type": "option",
"type": "text"
},
{
"name": "skillspector_baseline",
"opts": [
"--skillspector-baseline"
],
"param_type": "option",
"type": "file"
},
{
"choices": [
"cli",
"json",
"html",
"markdown",
"sarif",
"junit"
],
"default": "('cli', 'json', 'html')",
"multiple": true,
"name": "report_formats",
"opts": [
"--report",
"-r"
],
"param_type": "option"
},
{
"default": "reports",
"name": "output_dir",
"opts": [
"--output-dir",
"-o"
],
"param_type": "option",
"type": "directory"
}
]
}
},
"tier2": {
"commands": {
"context-optimization-check": {
"params": [
{
"name": "skill_path",
"opts": [
"skill_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"default": "0.8",
"name": "threshold",
"opts": [
"--threshold"
],
"param_type": "option",
"type": "float"
},
{
"name": "model",
"opts": [
"--model"
],
"param_type": "option",
"type": "text"
},
{
"name": "llm_model",
"opts": [
"--llm-model"
],
"param_type": "option",
"type": "text"
},
{
"choices": [
"cli",
"json",
"html",
"markdown",
"sarif",
"junit"
],
"default": "('cli',)",
"multiple": true,
"name": "report_formats",
"opts": [
"--report",
"-r"
],
"param_type": "option"
},
{
"default": "reports",
"name": "output_dir",
"opts": [
"--output-dir",
"-o"
],
"param_type": "option",
"type": "directory"
}
]
},
"dedup-scan": {
"params": [
{
"name": "skill_path",
"opts": [
"skill_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"default": "0.8",
"name": "threshold",
"opts": [
"--threshold"
],
"param_type": "option",
"type": "float"
},
{
"name": "llm_model",
"opts": [
"--llm-model"
],
"param_type": "option",
"type": "text"
},
{
"name": "model",
"opts": [
"--model"
],
"param_type": "option",
"type": "text"
},
{
"choices": [
"cli",
"json",
"html",
"markdown",
"sarif",
"junit"
],
"default": "('cli',)",
"multiple": true,
"name": "report_formats",
"opts": [
"--report",
"-r"
],
"param_type": "option"
},
{
"default": "reports",
"name": "output_dir",
"opts": [
"--output-dir",
"-o"
],
"param_type": "option",
"type": "directory"
}
]
},
"similarity-check": {
"params": [
{
"name": "content_path",
"opts": [
"content_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"choices": [
"skill",
"rules",
"workflows",
"auto"
],
"default": "auto",
"name": "content_type",
"opts": [
"--type"
],
"param_type": "option"
},
{
"default": "0.75",
"name": "threshold",
"opts": [
"--threshold"
],
"param_type": "option",
"type": "float"
},
{
"default": "False",
"is_flag": true,
"name": "full_body",
"opts": [
"--full-body"
],
"param_type": "option",
"type": "boolean"
},
{
"name": "model",
"opts": [
"--model"
],
"param_type": "option",
"type": "text"
},
{
"default": "1024",
"name": "max_entries",
"opts": [
"--max-entries"
],
"param_type": "option",
"type": "integer range"
},
{
"default": "128000000",
"name": "max_scalar_comparisons",
"opts": [
"--max-scalar-comparisons"
],
"param_type": "option",
"type": "integer range"
},
{
"name": "catalog",
"opts": [
"--catalog"
],
"param_type": "option",
"type": "file"
},
{
"name": "save_catalog",
"opts": [
"--save-catalog"
],
"param_type": "option",
"type": "file"
},
{
"name": "cache",
"opts": [
"--cache"
],
"param_type": "option",
"type": "path"
},
{
"name": "save_cache",
"opts": [
"--save-cache"
],
"param_type": "option",
"type": "path"
},
{
"choices": [
"cli",
"json",
"html",
"markdown",
"sarif",
"junit"
],
"default": "('cli',)",
"multiple": true,
"name": "report_formats",
"opts": [
"--report",
"-r"
],
"param_type": "option"
},
{
"default": "reports",
"name": "output_dir",
"opts": [
"--output-dir",
"-o"
],
"param_type": "option",
"type": "directory"
}
]
}
},
"params": [],
"workflow": {
"params": [
{
"name": "skill_path",
"opts": [
"skill_path"
],
"param_type": "argument",
"required": true,
"type": "directory"
},
{
"name": "catalog",
"opts": [
"--catalog"
],
"param_type": "option",
"type": "file"
},
{
"default": "0.8",
"name": "threshold",
"opts": [
"--threshold"
],
"param_type": "option",
"type": "float"
},
{
"default": "0.75",
"name": "similarity_threshold",
"opts": [
"--similarity-threshold"
],
"param_type": "option",
"type": "float"
},
{
"default": "False",
"is_flag": true,
"name": "full_body",
"opts": [
"--full-body"
],
"param_type": "option",
"type": "boolean"
},
{
"choices": [
"cli",
"json",
"html",
"markdown",
"sarif",
"junit"
],
"default": "('cli', 'json', 'html')",
"multiple": true,
"name": "report_formats",
"opts": [
"--report",
"-r"
],
"param_type": "option"
},
{
"default": "reports",
"name": "output_dir",
"opts": [
"--output-dir",
"-o"
],
"param_type": "option",
"type": "directory"
}
]
}
},
"tier3": {
"commands": {
"accept-traces": {
"params": [
{
"name": "skill_path",
"opts": [
"skill_path"
],
"param_type": "argument",
"required": true,
"type": "directory"
},
{
"name": "review_dir",
"opts": [
"review_dir"
],
"param_type": "argument",
"required": true,
"type": "directory"
},
{
"default": "False",
"is_flag": true,
"name": "dry_run",
"opts": [
"--dry-run"
],
"param_type": "option",
"type": "boolean"
},
{
"default": "False",
"is_flag": true,
"name": "allow_native_harbor",
"opts": [
"--allow-native-harbor"
],
"param_type": "option",
"type": "boolean"
}
]
},
"compare": {
"params": [
{
"name": "skill_path",
"opts": [
"skill_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"name": "results_dir",
"opts": [
"--results-dir"
],
"param_type": "option",
"type": "directory"
}
]
},
"compare-versions": {
"params": [
{
"name": "skill_path",
"opts": [
"skill_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"name": "current",
"opts": [
"--current"
],
"param_type": "option",
"required": true,
"type": "text"
},
{
"name": "prior",
"opts": [
"--prior"
],
"param_type": "option",
"required": true,
"type": "text"
},
{
"name": "results_dir",
"opts": [
"--results-dir"
],
"param_type": "option",
"type": "directory"
},
{
"default": "False",
"is_flag": true,
"name": "as_json",
"opts": [
"--json"
],
"param_type": "option",
"type": "boolean"
}
]
},
"create-eval-dataset": {
"params": [
{
"name": "skill_path",
"opts": [
"skill_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"default": "False",
"is_flag": true,
"name": "full",
"opts": [
"--full"
],
"param_type": "option",
"type": "boolean"
},
{
"default": "False",
"is_flag": true,
"name": "no_llm",
"opts": [
"--no-llm"
],
"param_type": "option",
"type": "boolean"
},
{
"default": "False",
"is_flag": true,
"name": "dry_run",
"opts": [
"--dry-run"
],
"param_type": "option",
"type": "boolean"
},
{
"default": "False",
"is_flag": true,
"name": "force",
"opts": [
"--force"
],
"param_type": "option",
"type": "boolean"
},
{
"name": "prompt",
"opts": [
"--prompt"
],
"param_type": "option",
"type": "path"
},
{
"choices": [
"auto",
"skill",
"intent",
"trace"
],
"default": "auto",
"name": "grounding",
"opts": [
"--grounding"
],
"param_type": "option"
},
{
"default": "False",
"is_flag": true,
"name": "refine",
"opts": [
"--refine"
],
"param_type": "option",
"type": "boolean"
},
{
"name": "from_results",
"opts": [
"--from-results"
],
"param_type": "option",
"type": "path"
},
{
"name": "results_dir",
"opts": [
"--results-dir"
],
"param_type": "option",
"type": "directory"
}
]
},
"doctor": {
"params": [
{
"name": "agents",
"opts": [
"--agents",
"-a"
],
"param_type": "option",
"type": "text"
},
{
"choices": [
"docker",
"kata",
"daytona",
"e2b",
"modal",
"runloop",
"langsmith",
"ec2",
"gke",
"ack",
"openshift",
"novita",
"apple-container",
"singularity",
"islo",
"tensorlake",
"cwsandbox",
"wandb",
"use-computer",
"blaxel",
"beam",
"skypilot",
"hyperbrowser",
"vercel",
"runta",
"mosaic",
"local"
],
"default": "docker",
"name": "env_mode",
"opts": [
"--env-mode"
],
"param_type": "option"
},
{
"multiple": true,
"name": "environment_kwarg",
"opts": [
"--ek",
"--environment-kwarg"
],
"param_type": "option",
"type": "text"
},
{
"multiple": true,
"name": "agent_model",
"opts": [
"--agent-model"
],
"param_type": "option",
"type": "text"
},
{
"choices": [
"harbor",
"nemo-gym"
],
"default": "harbor",
"name": "backend",
"opts": [
"--backend"
],
"param_type": "option"
},
{
"default": "False",
"is_flag": true,
"name": "verify_models",
"opts": [
"--verify-models"
],
"param_type": "option",
"type": "boolean"
}
]
},
"evaluate": {
"params": [
{
"name": "skill_path",
"opts": [
"skill_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"name": "agents",
"opts": [
"--agents",
"-a"
],
"param_type": "option",
"type": "text"
},
{
"choices": [
"docker",
"kata",
"daytona",
"e2b",
"modal",
"runloop",
"langsmith",
"ec2",
"gke",
"ack",
"openshift",
"novita",
"apple-container",
"singularity",
"islo",
"tensorlake",
"cwsandbox",
"wandb",
"use-computer",
"blaxel",
"beam",
"skypilot",
"hyperbrowser",
"vercel",
"runta",
"mosaic",
"local"
],
"default": "docker",
"name": "env_mode",
"opts": [
"--env-mode"
],
"param_type": "option"
},
{
"choices": [
"harbor",
"nemo-gym"
],
"default": "harbor",
"name": "backend",
"opts": [
"--backend"
],
"param_type": "option"
},
{
"multiple": true,
"name": "environment_kwarg",
"opts": [
"--ek",
"--environment-kwarg"
],
"param_type": "option",
"type": "text"
},
{
"default": "False",
"is_flag": true,
"name": "autopilot",
"opts": [
"--autopilot"
],
"param_type": "option",
"type": "boolean"
},
{
"default": "False",
"is_flag": true,
"name": "skip_baseline",
"opts": [
"--skip-baseline"
],
"param_type": "option",
"type": "boolean"
},
{
"name": "reuse_baseline",
"opts": [
"--reuse-baseline"
],
"param_type": "option",
"type": "text"
},
{
"name": "compare_to",
"opts": [
"--compare-to"
],
"param_type": "option",
"type": "text"
},
{
"default": "False",
"is_flag": true,
"name": "skip_with_skill",
"opts": [
"--skip-with-skill"
],
"param_type": "option",
"type": "boolean"
},
{
"name": "n_attempts",
"opts": [
"--n-attempts"
],
"param_type": "option",
"type": "integer"
},
{
"name": "max_agent_tokens",
"opts": [
"--max-agent-tokens"
],
"param_type": "option",
"type": "integer range"
},
{
"name": "max_judge_calls",
"opts": [
"--max-judge-calls"
],
"param_type": "option",
"type": "integer range"
},
{
"name": "max_attempts_total",
"opts": [
"--max-attempts-total"
],
"param_type": "option",
"type": "integer range"
},
{
"name": "max_cost_usd",
"opts": [
"--max-cost-usd"
],
"param_type": "option",
"type": "float range"
},
{
"name": "price_file",
"opts": [
"--price-file"
],
"param_type": "option",
"type": "file"
},
{
"name": "pass_threshold",
"opts": [
"--pass-threshold"
],
"param_type": "option",
"type": "float"
},
{
"is_flag": true,
"name": "stop_on_pass",
"opts": [
"--stop-on-pass"
],
"param_type": "option",
"secondary_opts": [
"--no-stop-on-pass"
],
"type": "boolean"
},
{
"name": "n_concurrent",
"opts": [
"--n-concurrent"
],
"param_type": "option",
"type": "integer"
},
{
"name": "max_agents",
"opts": [
"--max-agents"
],
"param_type": "option",
"type": "integer"
},
{
"name": "max_cases",
"opts": [
"--max-cases"
],
"param_type": "option",
"type": "integer range"
},
{
"name": "model",
"opts": [
"--model"
],
"param_type": "option",
"type": "text"
},
{
"multiple": true,
"name": "agent_model",
"opts": [
"--agent-model"
],
"param_type": "option",
"type": "text"
},
{
"choices": [
"preserve",
"rebase"
],
"name": "custom_dockerfile_mode",
"opts": [
"--custom-dockerfile-mode"
],
"param_type": "option"
},
{
"choices": [
"isolated",
"group"
],
"name": "skill_workspace_mode",
"opts": [
"--skill-workspace-mode"
],
"param_type": "option"
},
{
"multiple": true,
"name": "include_skills",
"opts": [
"--include-skills"
],
"param_type": "option",
"type": "path"
},
{
"default": "False",
"is_flag": true,
"name": "copy_repo",
"opts": [
"--copy-repo"
],
"param_type": "option",
"type": "boolean"
},
{
"choices": [
"default",
"default_plus_custom",
"custom_only"
],
"name": "grading_mode",
"opts": [
"--grading-mode"
],
"param_type": "option"
},
{
"name": "results_dir",
"opts": [
"--results-dir"
],
"param_type": "option",
"type": "directory"
},
{
"default": "False",
"is_flag": true,
"name": "harbor_keep_jobs",
"opts": [
"--harbor-keep-jobs"
],
"param_type": "option",
"type": "boolean"
},
{
"is_flag": true,
"name": "agent_runtime_preflight",
"opts": [
"--agent-runtime-preflight"
],
"param_type": "option",
"secondary_opts": [
"--no-agent-runtime-preflight"
],
"type": "boolean"
},
{
"is_flag": true,
"name": "tolerate_baseline_timeouts",
"opts": [
"--tolerate-baseline-timeouts"
],
"param_type": "option",
"secondary_opts": [
"--no-tolerate-baseline-timeouts"
],
"type": "boolean"
},
{
"name": "timeout_multiplier",
"opts": [
"--timeout-multiplier"
],
"param_type": "option",
"type": "float"
},
{
"name": "override_cpus",
"opts": [
"--override-cpus"
],
"param_type": "option",
"type": "integer"
},
{
"name": "override_memory_mb",
"opts": [
"--override-memory-mb"
],
"param_type": "option",
"type": "integer"
},
{
"name": "override_storage_mb",
"opts": [
"--override-storage-mb"
],
"param_type": "option",
"type": "integer"
},
{
"name": "evaluated_source_repository",
"opts": [
"--evaluated-source-repository"
],
"param_type": "option",
"type": "text"
},
{
"name": "evaluated_source_revision",
"opts": [
"--evaluated-source-revision"
],
"param_type": "option",
"type": "text"
},
{
"name": "evaluator_container_revision",
"opts": [
"--evaluator-container-revision"
],
"param_type": "option",
"type": "text"
},
{
"choices": [
"auto",
"rich",
"plain",
"off"
],
"default": "auto",
"name": "progress",
"opts": [
"--progress"
],
"param_type": "option"
},
{
"name": "progress_interval",
"opts": [
"--progress-interval"
],
"param_type": "option",
"type": "float"
},
{
"default": "False",
"is_flag": true,
"name": "verbose",
"opts": [
"--verbose"
],
"param_type": "option",
"type": "boolean"
}
]
},
"feedback": {
"params": [
{
"name": "run_dir",
"opts": [
"run_dir"
],
"param_type": "argument",
"required": true,
"type": "directory"
},
{
"name": "skill_path",
"opts": [
"--skill"
],
"param_type": "option",
"type": "directory"
}
]
},
"harbor-view": {
"params": [
{
"name": "jobs_dir",
"opts": [
"jobs_dir"
],
"param_type": "argument",
"required": true,
"type": "directory"
}
]
},
"history": {
"params": [
{
"name": "skill_path",
"opts": [
"skill_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"name": "results_dir",
"opts": [
"--results-dir"
],
"param_type": "option",
"type": "directory"
},
{
"name": "agent",
"opts": [
"--agent"
],
"param_type": "option",
"type": "text"
},
{
"default": "False",
"is_flag": true,
"name": "as_json",
"opts": [
"--json"
],
"param_type": "option",
"type": "boolean"
},
{
"name": "output",
"opts": [
"--output"
],
"param_type": "option",
"type": "file"
}
]
},
"import-metrics": {
"params": [
{
"name": "source",
"opts": [
"source"
],
"param_type": "argument",
"required": true,
"type": "file"
},
{
"name": "output_dir",
"opts": [
"--output-dir",
"-o"
],
"param_type": "option",
"required": true,
"type": "directory"
}
]
},
"import-traces": {
"params": [
{
"name": "skill_path",
"opts": [
"skill_path"
],
"param_type": "argument",
"required": true,
"type": "directory"
},
{
"name": "export",
"opts": [
"export"
],
"param_type": "argument",
"required": true,
"type": "file"
},
{
"name": "output",
"opts": [
"--output",
"-o"
],
"param_type": "option",
"type": "directory"
},
{
"default": "False",
"is_flag": true,
"name": "force",
"opts": [
"--force"
],
"param_type": "option",
"type": "boolean"
},
{
"default": "False",
"is_flag": true,
"name": "dry_run",
"opts": [
"--dry-run"
],
"param_type": "option",
"type": "boolean"
}
]
},
"init-custom-grader": {
"params": [
{
"name": "skill_path",
"opts": [
"skill_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"choices": [
"default_plus_custom",
"custom_only"
],
"default": "default_plus_custom",
"name": "mode",
"opts": [
"--mode"
],
"param_type": "option"
},
{
"choices": [
"python",
"shell"
],
"default": "python",
"name": "language",
"opts": [
"--language"
],
"param_type": "option"
},
{
"default": "False",
"is_flag": true,
"name": "force",
"opts": [
"--force"
],
"param_type": "option",
"type": "boolean"
},
{
"default": "False",
"is_flag": true,
"name": "no_config",
"opts": [
"--no-config"
],
"param_type": "option",
"type": "boolean"
}
]
},
"init-harbor-task": {
"params": [
{
"name": "skill_path",
"opts": [
"skill_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"default": "False",
"is_flag": true,
"name": "force",
"opts": [
"--force"
],
"param_type": "option",
"type": "boolean"
},
{
"default": "case-001",
"name": "case_id",
"opts": [
"--case-id"
],
"param_type": "option",
"type": "text"
},
{
"choices": [
"default",
"default_plus_custom",
"custom_only"
],
"default": "custom_only",
"name": "mode",
"opts": [
"--mode"
],
"param_type": "option"
},
{
"choices": [
"python",
"shell"
],
"default": "python",
"name": "language",
"opts": [
"--language"
],
"param_type": "option"
},
{
"default": "False",
"is_flag": true,
"name": "with_config",
"opts": [
"--with-config"
],
"param_type": "option",
"type": "boolean"
}
]
},
"interact": {
"params": [
{
"name": "skills",
"opts": [
"skills"
],
"param_type": "argument",
"required": true,
"type": "directory"
},
{
"name": "eval_source",
"opts": [
"--eval-source"
],
"param_type": "option",
"required": true,
"type": "directory"
},
{
"name": "agents",
"opts": [
"--agents",
"-a"
],
"param_type": "option",
"type": "text"
},
{
"name": "model",
"opts": [
"--model"
],
"param_type": "option",
"type": "text"
},
{
"choices": [
"docker",
"kata",
"daytona",
"e2b",
"modal",
"runloop",
"langsmith",
"ec2",
"gke",
"ack",
"openshift",
"novita",
"apple-container",
"singularity",
"islo",
"tensorlake",
"cwsandbox",
"wandb",
"use-computer",
"blaxel",
"beam",
"skypilot",
"hyperbrowser",
"vercel",
"runta",
"mosaic",
"local"
],
"default": "docker",
"name": "env_mode",
"opts": [
"--env-mode"
],
"param_type": "option"
},
{
"multiple": true,
"name": "environment_kwarg",
"opts": [
"--ek",
"--environment-kwarg"
],
"param_type": "option",
"type": "text"
},
{
"default": "1",
"name": "n_attempts",
"opts": [
"--n-attempts"
],
"param_type": "option",
"type": "integer range"
},
{
"name": "n_concurrent",
"opts": [
"--n-concurrent"
],
"param_type": "option",
"type": "integer range"
},
{
"choices": [
"goal_accuracy",
"accuracy",
"overall"
],
"default": "goal_accuracy",
"name": "metric",
"opts": [
"--metric"
],
"param_type": "option"
},
{
"name": "results_dir",
"opts": [
"--results-dir"
],
"param_type": "option",
"type": "directory"
},
{
"default": "False",
"is_flag": true,
"name": "harbor_keep_jobs",
"opts": [
"--harbor-keep-jobs"
],
"param_type": "option",
"type": "boolean"
},
{
"default": "False",
"is_flag": true,
"name": "as_json",
"opts": [
"--json"
],
"param_type": "option",
"type": "boolean"
}
]
},
"migrate-dataset": {
"params": [
{
"name": "source",
"opts": [
"source"
],
"param_type": "argument",
"required": true,
"type": "file"
},
{
"name": "skill_name",
"opts": [
"--skill-name"
],
"param_type": "option",
"type": "text"
},
{
"name": "destination",
"opts": [
"--output"
],
"param_type": "option",
"type": "file"
}
]
},
"select-decoys": {
"params": [
{
"name": "skill_path",
"opts": [
"skill_path"
],
"param_type": "argument",
"required": true,
"type": "directory"
},
{
"name": "catalog_path",
"opts": [
"--catalog"
],
"param_type": "option",
"required": true,
"type": "file"
},
{
"name": "catalog_root",
"opts": [
"--catalog-root"
],
"param_type": "option",
"required": true,
"type": "directory"
},
{
"default": "4",
"name": "count",
"opts": [
"--count"
],
"param_type": "option",
"type": "integer range"
},
{
"multiple": true,
"name": "exclude",
"opts": [
"--exclude"
],
"param_type": "option",
"type": "text"
},
{
"name": "model",
"opts": [
"--model"
],
"param_type": "option",
"type": "text"
},
{
"name": "output_path",
"opts": [
"--output",
"-o"
],
"param_type": "option",
"required": true,
"type": "file"
}
]
},
"validate": {
"params": [
{
"name": "skill_path",
"opts": [
"skill_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"default": "False",
"is_flag": true,
"name": "as_json",
"opts": [
"--json"
],
"param_type": "option",
"type": "boolean"
},
{
"default": "False",
"is_flag": true,
"name": "strict",
"opts": [
"--strict"
],
"param_type": "option",
"type": "boolean"
},
{
"default": "False",
"is_flag": true,
"name": "harbor_contract",
"opts": [
"--harbor-contract"
],
"param_type": "option",
"type": "boolean"
}
]
},
"validate-grader": {
"params": [
{
"name": "suite_dir",
"opts": [
"suite_dir"
],
"param_type": "argument",
"type": "directory"
},
{
"default": "3",
"name": "repeats",
"opts": [
"--repeats"
],
"param_type": "option",
"type": "integer range"
},
{
"choices": [
"assessment",
"tuning",
"all"
],
"default": "assessment",
"name": "split",
"opts": [
"--split"
],
"param_type": "option"
},
{
"multiple": true,
"name": "case_ids",
"opts": [
"--case"
],
"param_type": "option",
"type": "text"
},
{
"default": "grader-validation-output",
"name": "output",
"opts": [
"--output",
"-o"
],
"param_type": "option",
"type": "directory"
},
{
"default": "False",
"is_flag": true,
"name": "as_json",
"opts": [
"--json"
],
"param_type": "option",
"type": "boolean"
},
{
"default": "False",
"is_flag": true,
"name": "check_suite",
"opts": [
"--check-suite"
],
"param_type": "option",
"type": "boolean"
},
{
"default": "1",
"name": "concurrency",
"opts": [
"--concurrency"
],
"param_type": "option",
"type": "integer range"
},
{
"default": "300.0",
"name": "timeout",
"opts": [
"--timeout"
],
"param_type": "option",
"type": "float range"
},
{
"name": "max_false_pass_rate",
"opts": [
"--max-false-pass-rate"
],
"param_type": "option",
"type": "float range"
},
{
"name": "max_false_fail_rate",
"opts": [
"--max-false-fail-rate"
],
"param_type": "option",
"type": "float range"
},
{
"name": "max_unjudgeable_rate",
"opts": [
"--max-unjudgeable-rate"
],
"param_type": "option",
"type": "float range"
},
{
"name": "qualification_policy",
"opts": [
"--qualification-policy"
],
"param_type": "option",
"type": "path"
}
]
},
"view": {
"params": [
{
"name": "skill_path",
"opts": [
"skill_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"name": "results_dir",
"opts": [
"--results-dir"
],
"param_type": "option",
"type": "directory"
}
]
}
},
"params": [],
"workflow": {
"params": [
{
"name": "skill_path",
"opts": [
"skill_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"name": "agents",
"opts": [
"--agents",
"-a"
],
"param_type": "option",
"type": "text"
},
{
"choices": [
"docker",
"kata",
"daytona",
"e2b",
"modal",
"runloop",
"langsmith",
"ec2",
"gke",
"ack",
"openshift",
"novita",
"apple-container",
"singularity",
"islo",
"tensorlake",
"cwsandbox",
"wandb",
"use-computer",
"blaxel",
"beam",
"skypilot",
"hyperbrowser",
"vercel",
"runta",
"mosaic",
"local"
],
"default": "docker",
"name": "env_mode",
"opts": [
"--env-mode"
],
"param_type": "option"
},
{
"choices": [
"harbor",
"nemo-gym"
],
"default": "harbor",
"name": "backend",
"opts": [
"--backend"
],
"param_type": "option"
},
{
"multiple": true,
"name": "environment_kwarg",
"opts": [
"--ek",
"--environment-kwarg"
],
"param_type": "option",
"type": "text"
},
{
"default": "False",
"is_flag": true,
"name": "autopilot",
"opts": [
"--autopilot"
],
"param_type": "option",
"type": "boolean"
},
{
"default": "False",
"is_flag": true,
"name": "skip_baseline",
"opts": [
"--skip-baseline"
],
"param_type": "option",
"type": "boolean"
},
{
"name": "reuse_baseline",
"opts": [
"--reuse-baseline"
],
"param_type": "option",
"type": "text"
},
{
"name": "compare_to",
"opts": [
"--compare-to"
],
"param_type": "option",
"type": "text"
},
{
"default": "False",
"is_flag": true,
"name": "skip_with_skill",
"opts": [
"--skip-with-skill"
],
"param_type": "option",
"type": "boolean"
},
{
"name": "n_attempts",
"opts": [
"--n-attempts"
],
"param_type": "option",
"type": "integer"
},
{
"name": "max_agent_tokens",
"opts": [
"--max-agent-tokens"
],
"param_type": "option",
"type": "integer range"
},
{
"name": "max_judge_calls",
"opts": [
"--max-judge-calls"
],
"param_type": "option",
"type": "integer range"
},
{
"name": "max_attempts_total",
"opts": [
"--max-attempts-total"
],
"param_type": "option",
"type": "integer range"
},
{
"name": "max_cost_usd",
"opts": [
"--max-cost-usd"
],
"param_type": "option",
"type": "float range"
},
{
"name": "price_file",
"opts": [
"--price-file"
],
"param_type": "option",
"type": "file"
},
{
"name": "pass_threshold",
"opts": [
"--pass-threshold"
],
"param_type": "option",
"type": "float"
},
{
"is_flag": true,
"name": "stop_on_pass",
"opts": [
"--stop-on-pass"
],
"param_type": "option",
"secondary_opts": [
"--no-stop-on-pass"
],
"type": "boolean"
},
{
"name": "n_concurrent",
"opts": [
"--n-concurrent"
],
"param_type": "option",
"type": "integer"
},
{
"name": "max_agents",
"opts": [
"--max-agents"
],
"param_type": "option",
"type": "integer"
},
{
"name": "max_cases",
"opts": [
"--max-cases"
],
"param_type": "option",
"type": "integer range"
},
{
"name": "model",
"opts": [
"--model"
],
"param_type": "option",
"type": "text"
},
{
"multiple": true,
"name": "agent_model",
"opts": [
"--agent-model"
],
"param_type": "option",
"type": "text"
},
{
"choices": [
"preserve",
"rebase"
],
"name": "custom_dockerfile_mode",
"opts": [
"--custom-dockerfile-mode"
],
"param_type": "option"
},
{
"choices": [
"isolated",
"group"
],
"name": "skill_workspace_mode",
"opts": [
"--skill-workspace-mode"
],
"param_type": "option"
},
{
"multiple": true,
"name": "include_skills",
"opts": [
"--include-skills"
],
"param_type": "option",
"type": "path"
},
{
"default": "False",
"is_flag": true,
"name": "copy_repo",
"opts": [
"--copy-repo"
],
"param_type": "option",
"type": "boolean"
},
{
"choices": [
"default",
"default_plus_custom",
"custom_only"
],
"name": "grading_mode",
"opts": [
"--grading-mode"
],
"param_type": "option"
},
{
"name": "results_dir",
"opts": [
"--results-dir"
],
"param_type": "option",
"type": "directory"
},
{
"default": "False",
"is_flag": true,
"name": "harbor_keep_jobs",
"opts": [
"--harbor-keep-jobs"
],
"param_type": "option",
"type": "boolean"
},
{
"is_flag": true,
"name": "agent_runtime_preflight",
"opts": [
"--agent-runtime-preflight"
],
"param_type": "option",
"secondary_opts": [
"--no-agent-runtime-preflight"
],
"type": "boolean"
},
{
"is_flag": true,
"name": "tolerate_baseline_timeouts",
"opts": [
"--tolerate-baseline-timeouts"
],
"param_type": "option",
"secondary_opts": [
"--no-tolerate-baseline-timeouts"
],
"type": "boolean"
},
{
"name": "timeout_multiplier",
"opts": [
"--timeout-multiplier"
],
"param_type": "option",
"type": "float"
},
{
"name": "override_cpus",
"opts": [
"--override-cpus"
],
"param_type": "option",
"type": "integer"
},
{
"name": "override_memory_mb",
"opts": [
"--override-memory-mb"
],
"param_type": "option",
"type": "integer"
},
{
"name": "override_storage_mb",
"opts": [
"--override-storage-mb"
],
"param_type": "option",
"type": "integer"
},
{
"name": "evaluated_source_repository",
"opts": [
"--evaluated-source-repository"
],
"param_type": "option",
"type": "text"
},
{
"name": "evaluated_source_revision",
"opts": [
"--evaluated-source-revision"
],
"param_type": "option",
"type": "text"
},
{
"name": "evaluator_container_revision",
"opts": [
"--evaluator-container-revision"
],
"param_type": "option",
"type": "text"
},
{
"choices": [
"auto",
"rich",
"plain",
"off"
],
"default": "auto",
"name": "progress",
"opts": [
"--progress"
],
"param_type": "option"
},
{
"name": "progress_interval",
"opts": [
"--progress-interval"
],
"param_type": "option",
"type": "float"
},
{
"default": "False",
"is_flag": true,
"name": "verbose",
"opts": [
"--verbose"
],
"param_type": "option",
"type": "boolean"
}
]
}
},
"validate": {
"params": [
{
"name": "target_path",
"opts": [
"target_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"choices": [
"skill",
"rules",
"workflows",
"plugin",
"auto"
],
"default": "auto",
"name": "content_type",
"opts": [
"--type"
],
"param_type": "option"
},
{
"name": "tiers",
"opts": [
"--tiers"
],
"param_type": "option",
"type": "text"
},
{
"default": "False",
"is_flag": true,
"name": "full",
"opts": [
"--full"
],
"param_type": "option",
"type": "boolean"
},
{
"default": "False",
"is_flag": true,
"name": "verbose",
"opts": [
"--verbose"
],
"param_type": "option",
"type": "boolean"
},
{
"name": "checks",
"opts": [
"--checks",
"--tier1-checks"
],
"param_type": "option",
"type": "text"
},
{
"name": "harness",
"opts": [
"--harness"
],
"param_type": "option",
"type": "text"
},
{
"name": "harness_profiles",
"opts": [
"--harness-profiles"
],
"param_type": "option",
"type": "directory"
},
{
"name": "skip",
"opts": [
"--skip"
],
"param_type": "option",
"type": "text"
},
{
"name": "previous_version",
"opts": [
"--previous-version"
],
"param_type": "option",
"type": "text"
},
{
"default": "False",
"is_flag": true,
"name": "fail_fast",
"opts": [
"--fail-fast"
],
"param_type": "option",
"type": "boolean"
},
{
"default": "False",
"is_flag": true,
"name": "continue_on_failure",
"opts": [
"--continue-on-failure",
"-c"
],
"param_type": "option",
"type": "boolean"
},
{
"default": "False",
"is_flag": true,
"name": "llm",
"opts": [
"--llm",
"--tier1-llm"
],
"param_type": "option",
"secondary_opts": [
"--no-llm",
"--no-tier1-llm"
],
"type": "boolean"
},
{
"default": "False",
"is_flag": true,
"name": "llm_verify",
"opts": [
"--llm-verify"
],
"param_type": "option",
"type": "boolean"
},
{
"name": "skillspector_baseline",
"opts": [
"--skillspector-baseline"
],
"param_type": "option",
"type": "file"
},
{
"default": "70",
"name": "min_score",
"opts": [
"--min-score"
],
"param_type": "option",
"type": "integer"
},
{
"name": "quality_config",
"opts": [
"--quality-config"
],
"param_type": "option",
"type": "file"
},
{
"name": "pii_waivers",
"opts": [
"--pii-waivers"
],
"param_type": "option",
"type": "file"
},
{
"name": "profile",
"opts": [
"--profile"
],
"param_type": "option",
"type": "text"
},
{
"default": "False",
"is_flag": true,
"name": "external",
"opts": [
"--external"
],
"param_type": "option",
"type": "boolean"
},
{
"name": "policy_path",
"opts": [
"--policy"
],
"param_type": "option",
"type": "file"
},
{
"default": "1",
"name": "workers",
"opts": [
"--workers"
],
"param_type": "option",
"type": "integer range"
},
{
"name": "catalog_root",
"opts": [
"--catalog-root"
],
"param_type": "option",
"type": "directory"
},
{
"default": "True",
"is_flag": true,
"name": "dedup",
"opts": [
"--dedup",
"--tier2"
],
"param_type": "option",
"secondary_opts": [
"--no-dedup",
"--no-tier2"
],
"type": "boolean"
},
{
"is_flag": true,
"name": "block_on_dedup",
"opts": [
"--block-on-dedup"
],
"param_type": "option",
"secondary_opts": [
"--no-block-on-dedup"
],
"type": "boolean"
},
{
"is_flag": true,
"name": "agent_eval",
"opts": [
"--agent-eval",
"--tier3"
],
"param_type": "option",
"secondary_opts": [
"--no-agent-eval",
"--no-tier3"
],
"type": "boolean"
},
{
"is_flag": true,
"name": "block_on_agent_eval",
"opts": [
"--block-on-agent-eval"
],
"param_type": "option",
"secondary_opts": [
"--no-block-on-agent-eval"
],
"type": "boolean"
},
{
"is_flag": true,
"name": "autopilot",
"opts": [
"--autopilot"
],
"param_type": "option",
"secondary_opts": [
"--no-autopilot"
],
"type": "boolean"
},
{
"name": "agents",
"opts": [
"--agents",
"-a"
],
"param_type": "option",
"type": "text"
},
{
"choices": [
"docker",
"kata",
"daytona",
"e2b",
"modal",
"runloop",
"langsmith",
"ec2",
"gke",
"ack",
"openshift",
"novita",
"apple-container",
"singularity",
"islo",
"tensorlake",
"cwsandbox",
"wandb",
"use-computer",
"blaxel",
"beam",
"skypilot",
"hyperbrowser",
"vercel",
"runta",
"mosaic",
"local"
],
"default": "docker",
"name": "env_mode",
"opts": [
"--env-mode"
],
"param_type": "option"
},
{
"multiple": true,
"name": "environment_kwarg",
"opts": [
"--ek",
"--environment-kwarg"
],
"param_type": "option",
"type": "text"
},
{
"default": "False",
"is_flag": true,
"name": "skip_baseline",
"opts": [
"--skip-baseline"
],
"param_type": "option",
"type": "boolean"
},
{
"name": "n_concurrent",
"opts": [
"--n-concurrent"
],
"param_type": "option",
"type": "integer"
},
{
"name": "max_agents",
"opts": [
"--max-agents"
],
"param_type": "option",
"type": "integer"
},
{
"name": "max_cases",
"opts": [
"--max-cases"
],
"param_type": "option",
"type": "integer range"
},
{
"name": "n_attempts",
"opts": [
"--n-attempts"
],
"param_type": "option",
"type": "integer"
},
{
"name": "max_agent_tokens",
"opts": [
"--max-agent-tokens"
],
"param_type": "option",
"type": "integer range"
},
{
"name": "max_judge_calls",
"opts": [
"--max-judge-calls"
],
"param_type": "option",
"type": "integer range"
},
{
"name": "max_attempts_total",
"opts": [
"--max-attempts-total"
],
"param_type": "option",
"type": "integer range"
},
{
"name": "max_cost_usd",
"opts": [
"--max-cost-usd"
],
"param_type": "option",
"type": "float range"
},
{
"name": "price_file",
"opts": [
"--price-file"
],
"param_type": "option",
"type": "file"
},
{
"name": "pass_threshold",
"opts": [
"--pass-threshold"
],
"param_type": "option",
"type": "float"
},
{
"is_flag": true,
"name": "stop_on_pass",
"opts": [
"--stop-on-pass"
],
"param_type": "option",
"secondary_opts": [
"--no-stop-on-pass"
],
"type": "boolean"
},
{
"name": "model",
"opts": [
"--model"
],
"param_type": "option",
"type": "text"
},
{
"multiple": true,
"name": "agent_model",
"opts": [
"--agent-model"
],
"param_type": "option",
"type": "text"
},
{
"choices": [
"default",
"default_plus_custom",
"custom_only"
],
"name": "grading_mode",
"opts": [
"--grading-mode"
],
"param_type": "option"
},
{
"name": "results_dir",
"opts": [
"--results-dir"
],
"param_type": "option",
"type": "directory"
},
{
"multiple": true,
"name": "include_skills",
"opts": [
"--include-skills"
],
"param_type": "option",
"type": "path"
},
{
"default": "False",
"is_flag": true,
"name": "copy_repo",
"opts": [
"--copy-repo"
],
"param_type": "option",
"type": "boolean"
},
{
"name": "timeout_multiplier",
"opts": [
"--timeout-multiplier"
],
"param_type": "option",
"type": "float"
},
{
"default": "False",
"is_flag": true,
"name": "harbor_keep_jobs",
"opts": [
"--harbor-keep-jobs"
],
"param_type": "option",
"type": "boolean"
},
{
"is_flag": true,
"name": "agent_runtime_preflight",
"opts": [
"--agent-runtime-preflight"
],
"param_type": "option",
"secondary_opts": [
"--no-agent-runtime-preflight"
],
"type": "boolean"
},
{
"is_flag": true,
"name": "tolerate_baseline_timeouts",
"opts": [
"--tolerate-baseline-timeouts"
],
"param_type": "option",
"secondary_opts": [
"--no-tolerate-baseline-timeouts"
],
"type": "boolean"
},
{
"name": "evaluated_source_repository",
"opts": [
"--evaluated-source-repository"
],
"param_type": "option",
"type": "text"
},
{
"name": "evaluated_source_revision",
"opts": [
"--evaluated-source-revision"
],
"param_type": "option",
"type": "text"
},
{
"name": "evaluator_container_revision",
"opts": [
"--evaluator-container-revision"
],
"param_type": "option",
"type": "text"
},
{
"choices": [
"cli",
"json",
"html",
"markdown",
"sarif",
"junit"
],
"default": "('cli',)",
"multiple": true,
"name": "report_formats",
"opts": [
"--report",
"-r"
],
"param_type": "option"
},
{
"default": "reports",
"name": "output_dir",
"opts": [
"--output-dir",
"-o"
],
"param_type": "option",
"type": "directory"
}
]
},
"view": {
"params": [
{
"name": "skill_path",
"opts": [
"skill_path"
],
"param_type": "argument",
"required": true,
"type": "path"
},
{
"name": "results_dir",
"opts": [
"--results-dir"
],
"param_type": "option",
"type": "directory"
}
]
}
},
"params": [
{
"default": "False",
"is_flag": true,
"name": "version",
"opts": [
"--version"
],
"param_type": "option",
"type": "boolean"
},
{
"default": "False",
"is_flag": true,
"name": "verbose",
"opts": [
"--verbose",
"-v"
],
"param_type": "option",
"type": "boolean"
}
]
}