Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Shared bounded permission rules for private MCP and hook policy analysis."""
from __future__ import annotations
import re
from collections.abc import Iterable, Iterator
from dataclasses import dataclass
from typing import Any, Literal
from skillevaluator.models.result import Severity
from skillevaluator.utils.structured_data import MAX_STRUCTURED_NODES
from skillevaluator.validators.url_policy import report_text
_DOC_KEYS = frozenset({"description", "title", "summary", "notes", "note", "comment", "comments", "help"})
_MAX_SCAN_NODES = MAX_STRUCTURED_NODES
@dataclass(frozen=True)
class OverrideIssue:
"""A dangerous flag/env/config override, independent of where it was declared."""
concept: Literal[
"permission_bypass_flag",
"permission_bypass_scan_truncated",
"permission_mode_flag",
"permission_allow_flag",
"env_code_injection",
"env_traffic_redirect",
"env_insecure_tls",
"env_inline_secret",
"auto_approve",
]
severity: Severity
message: str
suggestion: str
class _ConfigWalk:
"""Bounded, iterative walk yielding ``(json_path, node)`` for a config value.
``complete`` turns False when the walk stops at ``_MAX_SCAN_NODES`` with nodes
still unvisited. With ``skip_doc_keys``, prose under a documentation key
(``description``, ``notes``, ``help``, ...) is skipped; a table under one is
walked.
"""
def __init__(self, value: Any, *, skip_doc_keys: bool = True) -> None:
self.value = value
self.skip_doc_keys = skip_doc_keys
self.complete = True
def __iter__(self) -> Iterator[tuple[str, Any]]:
stack: list[tuple[str, Any]] = [("", self.value)]
seen = 0
while stack:
if seen >= _MAX_SCAN_NODES:
self.complete = False
return
path, node = stack.pop()
seen += 1
yield path, node
if isinstance(node, dict):
for key, child in node.items():
if self.skip_doc_keys and str(key).lower() in _DOC_KEYS and _is_prose(child):
continue
stack.append((f"{path}.{key}" if path else str(key), child))
elif isinstance(node, list):
for index, child in enumerate(node):
stack.append((f"{path}[{index}]", child))
def _is_prose(value: Any) -> bool:
"""Text under a documentation key: a string, or a list of strings.
A table or object under such a key is config (a server named ``notes`` or
``help``), so the walk still reads it.
"""
return isinstance(value, str) or (isinstance(value, list) and all(isinstance(item, str) for item in value))
def iter_config_strings(value: Any, *, skip_doc_keys: bool = True) -> Iterator[tuple[str, str]]:
"""Yield ``(json_path, string)`` for string leaves of a config value (bounded, iterative)."""
for path, node in _ConfigWalk(value, skip_doc_keys=skip_doc_keys):
if isinstance(node, str):
yield path, node
_bounded = report_text
_TOOL_RULE_RE = re.compile(r"\s*([A-Za-z_][\w-]*)\s*(?:\((.*)\))?\s*", re.DOTALL)
# Claude Code's interpreter prefixes, shared by its Bash and PowerShell dangerous-allow-rule checks.
_INTERPRETER_PREFIXES: tuple[str, ...] = (
"python",
"python3",
"python2",
"node",
"deno",
"tsx",
"ruby",
"perl",
"php",
"lua",
"npx",
"bunx",
"npm run",
"yarn run",
"pnpm run",
"bun run",
"bash",
"sh",
"ssh",
)
_DANGEROUS_BASH_PREFIXES: tuple[str, ...] = (
*_INTERPRETER_PREFIXES,
"zsh",
"fish",
"eval",
"exec",
"env",
"xargs",
"sudo",
)
# PowerShell launchers and code-running cmdlets (each also as ``<name>.exe``), as Claude Code lists them.
_DANGEROUS_POWERSHELL_PREFIXES: tuple[str, ...] = (
*_INTERPRETER_PREFIXES,
"pwsh",
"powershell",
"cmd",
"wsl",
"iex",
"invoke-expression",
"icm",
"invoke-command",
"start-process",
"saps",
"start",
"start-job",
"sajb",
"start-threadjob",
"invoke-wmimethod",
"iwmi",
"invoke-cimmethod",
"icim",
"wmic",
"register-objectevent",
"register-engineevent",
"register-wmievent",
"register-scheduledjob",
"new-pssession",
"nsn",
"enter-pssession",
"etsn",
"add-type",
"new-object",
)
_PYTHON_DOTTED_MODULE_RE = re.compile(r"-m\s+\w+\.[\w.]+(?:\s*:|\s+)")
def parse_tool_list(value: Any) -> list[str] | None:
"""Parse a ``tools`` / ``allowed-tools`` value; ``None`` when the field is absent.
Accepts a YAML list or a comma- or whitespace-separated string. Separators
inside parentheses (``Bash(git add *)``) do not split an entry. Every entry
is returned (the frontmatter is already byte-bounded), so risk checks see
all of them; privilege records store at most ``MAX_TOOL_ENTRIES``.
"""
if value is None:
return None
if isinstance(value, list):
return [text for text in (str(item).strip() for item in value) if text]
if not isinstance(value, str):
return [str(value)]
entries: list[str] = []
current: list[str] = []
depth = 0
for char in value:
if char == "(":
depth += 1
elif char == ")":
depth = max(0, depth - 1)
if depth == 0 and (char == "," or char.isspace()):
if current:
entries.append("".join(current).strip())
current = []
continue
current.append(char)
if current:
entries.append("".join(current).strip())
return [entry for entry in entries if entry]
def bash_rule_risk(entry: str) -> str | None:
"""Classify one shell-tool (``Bash`` or ``PowerShell``) rule the way Claude Code classifies its allow rules.
``"unrestricted"``: the tool with no content, empty content, or only ``*``
and whitespace (``Bash()``, ``Bash(*)``, ``PowerShell(**)``), plus the
legacy ``Bash(:*)`` / ``Bash(*:*)``. ``"interpreter"``: an interpreter or
wrapper prefix that runs any command (``Bash(python3:*)``, ``Bash(sh -c
*)``, ``Bash(env *)``, ``Bash(npm run:*)``; for PowerShell also its
launchers such as ``PowerShell(pwsh *)``, ``PowerShell(iex *)``, and
``PowerShell(cmd.exe *)``). ``None`` for a scoped rule or another tool.
"""
match = _TOOL_RULE_RE.fullmatch(entry)
tool = match.group(1).lower() if match is not None else None
if match is None or tool not in {"bash", "powershell"}:
return None
content = match.group(2)
if content is None or not content.strip(" \t\n\r*"):
return "unrestricted"
if "".join(content.split()) in {":*", "*:*"}:
return "unrestricted"
if tool == "powershell":
return "interpreter" if _dangerous_powershell_prefix(content) else None
return "interpreter" if _dangerous_bash_prefix(content) else None
def _dangerous_bash_prefix(content: str) -> bool:
"""Claude Code's ``dangerous_prefix`` test for Bash rule content."""
rule = content.strip().lower()
if rule == "*":
return True
for prefix in _DANGEROUS_BASH_PREFIXES:
if rule in {prefix, f"{prefix}:*", f"{prefix} *", f"{prefix}*"}:
return True
if rule.startswith(f"{prefix} ") and rule.endswith("*"):
rest = rule[len(prefix) + 1 :]
module_rule = prefix.startswith("python") and _PYTHON_DOTTED_MODULE_RE.fullmatch(rest[:-1])
if rest.startswith("-") and not module_rule:
return True
return False
def _dangerous_powershell_prefix(content: str) -> bool:
"""Claude Code's dangerous-prefix test for PowerShell rule content: each prefix also as ``<word>.exe``."""
rule = content.strip().lower()
if rule == "*":
return True
for prefix in _DANGEROUS_POWERSHELL_PREFIXES:
first, space, rest = prefix.partition(" ")
for name in (prefix, f"{first}.exe{space}{rest}"):
if rule in {name, f"{name}:*", f"{name}*", f"{name} *"}:
return True
if rule.startswith(f"{name} -") and rule.endswith("*"):
return True
return False
def is_unrestricted_bash(entry: str) -> bool:
"""A ``Bash`` or ``PowerShell`` grant that runs any command: unrestricted, or an interpreter or wrapper prefix."""
return bash_rule_risk(entry) is not None
def is_broad_allow_rule(rule: str) -> bool:
"""A permission ``allow`` rule that pre-approves any shell command, or every tool (``*``)."""
return is_unrestricted_bash(rule) or "".join(rule.split()) == "*"
_ALLOWED_TOOLS_VALUE = r"\"[^\"\n]{0,1024}\"|'[^'\n]{0,1024}'|[^\s;&|]{1,1024}"
_ALLOWED_TOOLS_VALUE_RE = re.compile(_ALLOWED_TOOLS_VALUE)
_ALLOWED_TOOLS_FLAG_RE = re.compile(
rf"(?<![\w-])--allowed-?tools(?:=(?P<value>{_ALLOWED_TOOLS_VALUE})"
rf"|\s+(?P<values>(?:{_ALLOWED_TOOLS_VALUE})(?:[ \t]+(?!-)(?:{_ALLOWED_TOOLS_VALUE}))*))",
re.IGNORECASE,
)
_ALLOWED_TOOLS_FLAGS = frozenset({"--allowedtools", "--allowed-tools"})
_LIST_ITEM_PATH_RE = re.compile(r"^(?P<parent>.*)\[(?P<index>\d+)\]$")
def allowed_tools_flag_issues(value: Any) -> list[OverrideIssue]:
"""``--allowedTools`` passed to an agent CLI with a grant that runs any command (or ``*``), HIGH each.
The launched agent then runs any shell command without a prompt, like an
unrestricted ``allowed-tools`` grant. Scoped grants (``Bash(git status *)``)
are not flagged. Read in any config string or argv list.
"""
hits: dict[tuple[str, str], None] = {}
items: dict[str, dict[int, str]] = {}
def _broad(values: Iterable[str]) -> str | None:
return next(
(rule for raw in values for rule in parse_tool_list(raw.strip("\"'")) or [] if is_broad_allow_rule(rule)),
None,
)
for path, text in iter_config_strings(value):
for match in _ALLOWED_TOOLS_FLAG_RE.finditer(text):
spaced = match.group("values")
rule = _broad([match.group("value")] if spaced is None else _ALLOWED_TOOLS_VALUE_RE.findall(spaced))
if rule is not None:
hits.setdefault((path, rule))
item = _LIST_ITEM_PATH_RE.match(path)
if item is not None:
items.setdefault(item.group("parent"), {})[int(item.group("index"))] = text
for parent, tokens in items.items():
for index, token in tokens.items():
if token.strip().lower() in _ALLOWED_TOOLS_FLAGS:
# Every item up to the next option is a value: ['--allowedTools', 'Read', 'Bash', '-p', 'x'].
end = index + 1
while end in tokens and not tokens[end].lstrip().startswith("-"):
end += 1
rule = _broad(tokens[position] for position in range(index + 1, end))
if rule is not None:
hits.setdefault((f"{parent}[{index}]", rule))
return [
OverrideIssue(
"permission_allow_flag",
Severity.HIGH,
f"agent-CLI flag '--allowedTools {_bounded(rule, 60)}'{f' in {path!r}' if path else ''} pre-approves "
"any shell command (or every tool) in the launched agent, so it runs without a prompt",
"Remove the flag, or scope it to exact commands such as Bash(git status *).",
)
for path, rule in hits
]