Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Bounded, client-compatible frontmatter parsing for private plugin analysis."""
from __future__ import annotations
import re
from dataclasses import dataclass
from typing import Any
from skillevaluator.utils.structured_data import StructuredDataError, StructuredDataLimitError, load_bounded_yaml
@dataclass(frozen=True)
class _Markdown:
name: str | None
description: str | None
body: str
frontmatter: dict[str, Any]
# Why frontmatter over a parser limit was not read; its fields are unknown, not absent.
frontmatter_error: str | None = None
_LENIENT_KEY_RE = re.compile(r"^([a-zA-Z_-]+):\s+(.+)$")
_LENIENT_SPECIAL_RE = re.compile(r"[{}\[\]*&#!|>%@`]|: ")
_LEADING_TABS_RE = re.compile(r"^\t+", re.MULTILINE)
# Claude Code's frontmatter split: the block ends at the first ``---`` after the opening line, even mid-line.
_FRONTMATTER_RE = re.compile(r"---\s*\n([\s\S]*?)---\s*\n?")
def _quote_lenient_values(raw: str) -> str:
"""Quote the plain frontmatter values strict YAML rejects, the way Claude Code retries a failed parse.
Claude Code 2.1.x parses frontmatter with a YAML parser and, when that
fails, quotes each top-level ``key: value`` whose value contains a YAML
indicator (``{}[]*&#!|>%@`` or a backtick) or ``": "`` (unless it is
already quoted or is a valid flow list), turns leading tabs into two spaces
each, and parses again. So
``description: Deploy: runs the script`` and ``tools: *`` load, and the
privileges and context cost must be read the same way.
"""
lines: list[str] = []
for line in raw.split("\n"):
match = _LENIENT_KEY_RE.match(line)
if match is not None:
key, value = match.groups()
if (value.startswith('"') and value.endswith('"')) or (value.startswith("'") and value.endswith("'")):
lines.append(line)
continue
if value.startswith("[") and value.endswith("]"):
try:
if isinstance(load_bounded_yaml(value), list):
lines.append(line)
continue
except StructuredDataError:
pass
if _LENIENT_SPECIAL_RE.search(value):
escaped = value.replace("\\", "\\\\").replace('"', '\\"')
lines.append(f'{key}: "{escaped}"')
continue
lines.append(line)
return _LEADING_TABS_RE.sub(lambda tabs: " " * len(tabs.group(0)), "\n".join(lines))
def _load_frontmatter(raw: str) -> Any:
"""Bounded YAML frontmatter, with Claude Code's lenient retry when strict YAML rejects it.
A key written twice keeps its last value, as in Claude Code.
"""
if not raw.strip():
return {}
try:
return load_bounded_yaml(raw, last_key_wins=True)
except StructuredDataLimitError:
raise
except (StructuredDataError, ValueError):
return load_bounded_yaml(_quote_lenient_values(raw), last_key_wins=True)
def parse_markdown(text: str) -> _Markdown:
"""Split optional ``---`` YAML frontmatter from a Markdown body (bounded YAML).
Frontmatter that strict YAML rejects is read again the way Claude Code
reads it (:func:`_quote_lenient_values`), so a value such as ``Deploy:
runs the script`` does not hide the fields after it. Frontmatter that
still fails to parse gives no fields, as in Claude Code. Frontmatter over a
parser limit (a 65,536-character value, 1,024 list or mapping items) gives
no fields and a ``frontmatter_error``: Claude Code still reads it, so its
grants are unknown and the caller must fail closed.
The block is split the way Claude Code splits it: after a leading BOM, an
opening ``---`` line, up to the first ``---`` that follows, even when it
begins a longer line such as ``----`` or ``--- # end``. CRLF line endings
in the block become LF, so the lenient retry still sees each value whole.
"""
match = _FRONTMATTER_RE.match(text.removeprefix("\ufeff"))
if match is None:
return _Markdown(None, None, text.strip(), {})
raw = match.group(1).replace("\r\n", "\n")
body = match.string[match.end() :].strip()
try:
data = _load_frontmatter(raw)
except StructuredDataLimitError as exc:
return _Markdown(None, None, body, {}, frontmatter_error=str(exc))
except (StructuredDataError, ValueError, RecursionError):
return _Markdown(None, None, body, {})
if not isinstance(data, dict):
return _Markdown(None, None, body, {})
name = data.get("name") if isinstance(data.get("name"), str) else None
description = data.get("description") if isinstance(data.get("description"), str) else None
return _Markdown(name, description, body, data)