Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Deterministic introspection of the Click command tree.
Used to freeze the CLI surface (commands, options, choices, defaults) as a
golden baseline so the Phase 1 rename and later refactors can prove they only
change *names* and not the command structure.
"""
from __future__ import annotations
from pathlib import Path
from typing import Any
import click
def _default_to_string(value: Any) -> str:
if not callable(value):
return str(value)
# pathlib moved this classmethod between private implementations in 3.13.
# Keep its public identity, without evaluating the current working directory.
if value == Path.cwd:
return "<callable pathlib.Path.cwd>"
module = getattr(value, "__module__", type(value).__module__)
name = getattr(value, "__qualname__", type(value).__qualname__)
return f"<callable {module}.{name}>"
def _param_to_dict(param: click.Parameter) -> dict[str, Any]:
data: dict[str, Any] = {
"name": param.name,
"param_type": param.param_type_name, # "option" or "argument"
}
opts = list(getattr(param, "opts", []) or [])
secondary = list(getattr(param, "secondary_opts", []) or [])
if opts:
data["opts"] = sorted(opts)
if secondary:
data["secondary_opts"] = sorted(secondary)
if getattr(param, "multiple", False):
data["multiple"] = True
if getattr(param, "is_flag", False):
data["is_flag"] = True
if getattr(param, "required", False):
data["required"] = True
default = getattr(param, "default", None)
if default is not None and default is not getattr(click.core, "UNSET", None):
data["default"] = _default_to_string(default)
param_type = getattr(param, "type", None)
if isinstance(param_type, click.Choice):
data["choices"] = list(param_type.choices)
elif param_type is not None:
data["type"] = param_type.name
return data
def _command_to_dict(command: click.Command) -> dict[str, Any]:
data: dict[str, Any] = {
"params": [_param_to_dict(p) for p in command.params],
}
if isinstance(command, click.Group):
data["commands"] = {name: _command_to_dict(command.commands[name]) for name in sorted(command.commands)}
workflow = getattr(command, "workflow", None)
if workflow is not None:
data["workflow"] = _command_to_dict(workflow)
return data
def build_cli_surface(group: click.Group) -> dict[str, Any]:
"""Return a JSON-serializable snapshot of the CLI command tree."""
return _command_to_dict(group)