Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
Score a missing command named after the skill as the agent's choice (#662)
Christopher Kevin6f0e2fca8304
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 6927cca2..f0dfe504 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -321,6 +321,10 @@ All notable changes to SkillEvaluator are documented in this file.- Budget-limited Tier 3 coverage checks retain admitted attempt ordinals, avoiding false missing/excess errors when a denied slot leaves a gap.- HTML reports retain budget context for fully budget-denied runs withoutrestoring untrusted score evidence.+- A trial where the agent runs the skill's own name as a shell command (for example+ `text-analyzer input.txt`, `command not found`) is now scored as the agent's tool+ choice instead of an undeclared environment dependency, unless the skill declares a+ tool by that name.- Fully budget-denied runs exit successfully and finish progress without anexecution error, while retaining skipped arms and an incomplete quality verdict.- `--backend nemo-gym` no longer rejects a malformed `evals/environment/mcp_servers.toml`diff --git a/src/skillevaluator/tier3/eval_core/checks.py b/src/skillevaluator/tier3/eval_core/checks.pyindex 95e778b9..788034a8 100644--- a/src/skillevaluator/tier3/eval_core/checks.py+++ b/src/skillevaluator/tier3/eval_core/checks.py@@ -6260,7 +6260,10 @@ def _classify_execution_failure(}declared_names = {normalized(str(name)) for name in dependencies}declared = normalized(missing) in declared_names or normalized(aliases.get(missing.lower(), "")) in declared_names- used = bool(re.search(r"(?<![\w-])" + re.escape(missing) + r"(?![\w-])", context.get("skill_text", "")))+ skill_name = normalized(str(context.get("skill_name") or ""))+ used = skill_name != normalized(missing) and bool(+ re.search(r"(?<![\w-])" + re.escape(missing) + r"(?![\w-])", context.get("skill_text", ""))+ )used = (used or missing in context.get("script_imports", []) or _command_executes_skill(command, expected_script, paths))diff --git a/src/skillevaluator/tier3/harbor/templates/eval.py b/src/skillevaluator/tier3/harbor/templates/eval.pyindex 4cf085de..82688d3f 100644--- a/src/skillevaluator/tier3/harbor/templates/eval.py+++ b/src/skillevaluator/tier3/harbor/templates/eval.py@@ -43,6 +43,7 @@ import timeimport unicodedataimport urllib.errorimport urllib.request+from collections.abc import Iteratorfrom contextvars import ContextVarfrom functools import partialfrom pathlib import Path@@ -4164,12 +4165,238 @@ def write_reward_outputs(result, overall):REWARD_TXT.write_text(str(sanitized_overall))+_YAML_FRONTMATTER = re.compile(r"\ufeff?---[ \t]*\r?\n(.*?)^---[ \t]*\r?$", re.MULTILINE | re.DOTALL)+_YAML_LINE_PREFIX = re.compile(r"[ \t]*(?:[-?:][ \t]+)*")+# An anchor (`&a`), a tag (`!`, `!!str`, `!local`, `!<verbatim>`) or both, in either order. They belong to+# the node that follows them, so a quote after them still opens a string.+_YAML_PROPERTY = r"(?:&[^\s,\[\]{}]+|!<[^>\s]*>|![^\s,\[\]{}]*)(?=[\s,\[\]{}]|\Z)"+_YAML_PROPERTIES = re.compile(rf"(?:{_YAML_PROPERTY}[ \t]*)+")+_YAML_KEY = re.compile(+ rf"""(?:{_YAML_PROPERTY}[ \t]+)*"""+ r"""(?:"((?:[^"\\\n]|\\.)*)"|'((?:[^'\n]|'')*)'|([^\s#'"\-?:\[\]{}|>&*!%@`,][^:\n]*?))[ \t]*:(?=[ \t\r\n]|\Z)"""+)+_YAML_QUOTED_BODY = {'"': re.compile(r"(?:[^\"\\]|\\.)*", re.DOTALL), "'": re.compile(r"(?:[^']|'')*")}+_YAML_BLOCK_SCALAR = re.compile(r"([|>])[+-]?\d?[+-]?[ \t]*(?:#[^\n]*)?(?:\r?\n|\Z)")+_YAML_FLOW_TOKEN = re.compile(+ r"""(?P<space>\s+)|(?P<comment>(?<!\S)\#[^\n]*)|(?P<open>[\[{])|(?P<close>[\]}])|(?P<comma>,)"""+ r"""|(?P<colon>:(?=[\s,\[\]{}]|\Z))|(?P<explicit>\?(?=[\s,\[\]{}]|\Z))"""+ rf"""|(?P<quoted>"(?:[^"\\]|\\.)*"|'(?:[^']|'')*')|(?P<prop>{_YAML_PROPERTY})|(?P<alias>\*[^\s,\[\]{{}}]*)""",+ re.DOTALL,+)+_YAML_FLOW_PLAIN = re.compile(r"(?:[^\s,\[\]{}:]|:(?![\s,\[\]{}]|\Z))+")+_YAML_ESCAPE = re.compile(r"\\(x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8}|.)", re.DOTALL)+_YAML_SIMPLE_ESCAPES = {"0": "\0", "n": "\n", "r": "\r", "t": "\t"}+++def _yaml_unescape(match: re.Match[str]) -> str:+ code = match.group(1)+ if len(code) > 1:+ try:+ return chr(int(code[1:], 16))+ except (ValueError, OverflowError):+ return match.group(0)+ return _YAML_SIMPLE_ESCAPES.get(code, code)+++def _yaml_line_end(text: str, position: int) -> int:+ end = text.find("\n", position)+ return len(text) if end < 0 else end+++def _yaml_unquote(raw: str) -> str:+ """Return the value of the single- or double-quoted scalar ``raw`` (quotes included)."""+ body = re.sub(r"[ \t]*\r?\n[ \t]*", " ", raw[1:-1])+ return _YAML_ESCAPE.sub(_yaml_unescape, body) if raw[0] == '"' else body.replace("''", "'")+++def _yaml_flow_tokens(text: str, start: int) -> Iterator[tuple[str, str, int, int]]:+ """Yield ``(kind, token, depth, end)`` for the flow collection that opens at ``text[start]``.++ A flow collection may continue over lines whose keys are not indented, so+ brackets are counted while skipping quoted strings and comments. A quote+ opens a string only where a node begins: after ``[``, ``{``, ``,``, a+ ``:`` or ``?`` indicator, or after the anchor and tag that belong to that+ node. Inside a plain scalar it is an ordinary character. ``depth`` is the+ nesting after the token.+ """+ depth, node_starts, closed, position = 0, True, False, start+ while position < len(text):+ if closed and text[position] == ":": # JSON-style `"key":value`+ node_starts, closed, position = True, False, position + 1+ yield "colon", ":", depth, position+ continue+ token = _YAML_FLOW_TOKEN.match(text, position)+ kind = token.lastgroup if token else None+ if kind in {"quoted", "prop", "alias", "explicit"} and not node_starts:+ kind = None # in the middle of a plain scalar these are ordinary characters+ if kind is None:+ plain = _YAML_FLOW_PLAIN.match(text, position)+ end = plain.end() if plain else position + 1+ node_starts, closed = False, False+ yield "plain", text[position:end], depth, end+ position = end+ continue+ position = token.end()+ if kind in {"space", "comment"}:+ continue+ closed = kind in {"close", "quoted"}+ if kind != "prop": # an anchor or tag leaves the node it belongs to still to begin+ node_starts = kind in {"open", "comma", "colon", "explicit"}+ depth += {"open": 1, "close": -1}.get(kind, 0)+ yield kind, token.group(), depth, position+ if depth == 0 and kind == "close":+ return+++def _yaml_flow_end(text: str, start: int) -> int:+ """Return the index just past the flow collection that opens at ``text[start]``."""+ for kind, _token, depth, end in _yaml_flow_tokens(text, start):+ if kind == "close" and depth == 0:+ return end+ return len(text)+++def _yaml_flow_mapping_name(text: str, start: int) -> str:+ """Return the string ``name`` entry of the flow mapping that opens at ``text[start]``."""+ name, key, value, in_value, nested = "", [], [], False, False+ for kind, token, depth, _end in _yaml_flow_tokens(text, start):+ if kind == "open" and depth == 2:+ nested = True+ elif depth >= 2 or (kind == "close" and depth == 1):+ continue # inside a nested collection, or its end+ elif kind in {"quoted", "plain"}:+ (value if in_value else key).append(_yaml_unquote(token) if kind == "quoted" else token)+ elif kind == "alias":+ nested = True # the anchored value is not resolved+ elif kind == "colon":+ in_value = True+ elif kind in {"comma", "close"}:+ if " ".join(key) == "name":+ name = "" if nested else " ".join(value)+ key, value, in_value, nested = [], [], False, False+ return name+++def _yaml_node(text: str, start: int, parent: int, *, lookahead: bool = False) -> tuple[str | None, int]:+ """Consume the node whose value begins at ``text[start]``.++ ``parent`` is the column of the key or ``-`` that owns the node; a plain or+ block scalar continues over the lines indented deeper than it. Returns the+ scalar value (``None`` for no value or a flow collection) and the index of+ the newline that ends the node. With ``lookahead``, an empty value takes a+ scalar written on the following lines.+ """+ while text[start : start + 1] in {" ", "\t"}:+ start += 1+ properties = _YAML_PROPERTIES.match(text, start)+ start = properties.end() if properties else start+ first = text[start : start + 1]+ if first == "*": # an alias is a whole node+ return None, _yaml_line_end(text, start)+ if first in {'"', "'"}:+ body = _YAML_QUOTED_BODY[first].match(text, start + 1)+ end = body.end()+ if text[end : end + 1] != first:+ return None, len(text) # unterminated: not valid YAML+ return _yaml_unquote(text[start : end + 1]), _yaml_line_end(text, end + 1)+ if first in {"[", "{"}:+ return None, _yaml_line_end(text, _yaml_flow_end(text, start))+ block = _YAML_BLOCK_SCALAR.match(text, start)+ if block:+ position, lines, last_end = block.end(), [], block.end() - 1+ while position < len(text):+ end = _yaml_line_end(text, position)+ line = text[position:end]+ if line.strip() and len(line) - len(line.lstrip(" \t")) <= parent:+ break # the block ends at the next line that is not indented under its key+ if line.strip():+ lines.append(line.strip())+ position, last_end = end + 1, end+ return (" " if block.group(1) == ">" else "\n").join(lines), min(last_end, len(text))+ line_end = _yaml_line_end(text, start)+ head = re.split(r"(?:^|[ \t])#", text[start:line_end], maxsplit=1)+ value, commented = head[0].strip(), len(head) == 2+ if not value:+ position = line_end + 1+ while lookahead and position < len(text): # no value here: it may be written on the next lines+ end = _yaml_line_end(text, position)+ line = text[position:end]+ if line.strip() and not line.lstrip().startswith("#"):+ indent = len(line) - len(line.lstrip(" \t"))+ if (+ indent > parent+ and not _YAML_KEY.match(line, indent)+ and not re.match(r"[-?](?:[ \t]|$)", line[indent:])+ ):+ return _yaml_node(text, position + indent, parent)+ break+ position = end + 1+ return None, line_end+ lines, end = [value], line_end+ while not commented: # a plain scalar continues over the lines indented under its key+ position = end + 1+ while position < len(text) and not text[position : _yaml_line_end(text, position)].strip():+ position = _yaml_line_end(text, position) + 1+ if position >= len(text):+ break+ next_end = _yaml_line_end(text, position)+ line = text[position:next_end]+ if len(line) - len(line.lstrip(" \t")) <= parent or line.lstrip().startswith("#"):+ break+ head = re.split(r"(?:^|[ \t])#", line, maxsplit=1)+ lines.append(head[0].strip())+ end, commented = next_end, len(head) == 2+ return " ".join(lines), end+++def _frontmatter_skill_name(content: str) -> str:+ """Read the top-level ``name`` of a SKILL.md the way ``yaml.safe_load`` does.++ The verifier image gets PyYAML only as a dependency of the RAGAS packages,+ so this does not import it. It reads just enough of the structure to tell+ a root key from text that merely sits on an unindented line: nested+ mappings and sequences, multi-line flow collections, quoted strings, and+ plain or block scalars, each with the anchor and tag that may precede it.+ A quote opens a string only where a node begins. The whole frontmatter may+ also be one flow mapping. Like ``yaml.safe_load``, a later ``name`` key+ wins. An alias (``name: *anchor``) is not resolved.+ """+ frontmatter = _YAML_FRONTMATTER.match(content)+ if not frontmatter:+ return ""+ text, name, position, first_node = frontmatter.group(1), "", 0, True+ while position < len(text):+ end = _yaml_line_end(text, position)+ line = text[position:end]+ if not line.strip() or line.lstrip().startswith("#"):+ position = end + 1+ continue+ prefix = _YAML_LINE_PREFIX.match(line).group()+ value_start = position + len(prefix)+ if first_node: # the whole frontmatter may be one flow mapping+ properties = _YAML_PROPERTIES.match(text, value_start)+ brace = properties.end() if properties else value_start+ if not prefix and text[brace : brace + 1] == "{":+ return _yaml_flow_mapping_name(text, brace)+ first_node = False+ key = _YAML_KEY.match(text, value_start)+ marker = max(prefix.rfind(indicator) for indicator in "-?:")+ parent = len(prefix) if key else (marker if marker >= 0 else max(len(prefix) - 1, 0))+ root_name = bool(key) and not prefix and (key.group(1) or key.group(2) or key.group(3)) == "name"+ value, end = _yaml_node(text, key.end() if key else value_start, parent, lookahead=root_name)+ if root_name:+ name = (value or "").strip()+ position = end + 1+ return name++def _dependency_context(skill: str, *, skills_root: Path = Path("/workspace/skills")) -> dict[str, Any]:"""Read bounded dependency evidence from the evaluator-staged skill."""context: dict[str, Any] = {"apt": [],"pip": [],"skill_text": "",+ "skill_name": "","skill_dirs": [],"skill_modules": set(),"script_imports": set(),@@ -4217,6 +4444,8 @@ def _dependency_context(skill: str, *, skills_root: Path = Path("/workspace/skil)else:context["skill_text"] += "\n" + content+ if relative.as_posix().lower() == "skill.md":+ context["skill_name"] = _frontmatter_skill_name(content) or context["skill_name"]if "scripts" in relative.parts and path.suffix == ".py":imports = re.finditer(r"(?m)^\s*(?:from\s+([\w.]+)\s+import\b|import\s+([\w.]+))", content)context["script_imports"].update((match.group(1) or match.group(2)).split(".")[0] for match in imports)diff --git a/tests/tier3/test_error_recovery.py b/tests/tier3/test_error_recovery.pyindex f5d1fbdb..89e83fd2 100644--- a/tests/tier3/test_error_recovery.py+++ b/tests/tier3/test_error_recovery.py@@ -10,6 +10,7 @@ command must NOT be scored as first-attempt clean.from __future__ import annotationsimport pytest+import yamlfrom skillevaluator.tier3.eval_core.atif_helpers import extract_tool_calls_as_dictsfrom skillevaluator.tier3.eval_core.checks import check_error_recovery, score_skill_execution@@ -190,6 +191,209 @@ def test_dependency_context_records_modules_provided_by_skill(tmp_path):assert "rasterio" in context["script_imports"]+def _skill_named_text_analyzer(tmp_path, apt: str | None = None, frontmatter: str | None = None) -> dict:+ skill = tmp_path / "text-analyzer"+ (skill / "scripts").mkdir(parents=True)+ if frontmatter is None:+ frontmatter = "name: text-analyzer\ndescription: Count words. Use when a request names text-analyzer.\n"+ (skill / "SKILL.md").write_bytes(+ f"---\n{frontmatter}---\nRun scripts/analyze_text.py; pipe the result to jq if needed.\n".encode()+ )+ if apt is not None:+ (skill / "scripts" / "apt-packages.txt").write_text(apt)+ context = eval_template._dependency_context("text-analyzer", skills_root=tmp_path)+ assert context["skill_name"] == "text-analyzer"+ return context+++def test_missing_command_named_after_the_skill_is_agent_tool_choice(checker, tmp_path):+ result = checker(+ [_call("text-analyzer input.txt", "bash: line 1: text-analyzer: command not found")],+ dependency_context=_skill_named_text_analyzer(tmp_path),+ )+ assert result["failures"][0]["fault"] == "agent"+ assert result["failures"][0]["fault_kind"] == "agent_tool_choice"+ assert result["environment_faults"] == 0+++# Every spelling below is the same skill name to yaml.safe_load, which the schema validator uses.+_DESCRIPTION = "description: Count words. Use when a request names text-analyzer.\n"+_QUOTED_DESCRIPTION = 'description: Counts "words" in a file.\n'+# A nested `name` must not be read as the skill name, whether it comes before or after the root `name`.+# Anchors, tags and aliases belong to the node after them, so a quote right after one still opens a string.+_NESTED_NAME_FRAGMENTS = {+ "anchored-quoted-flow-value": 'metadata: {k: &value "closing } inside",\nname: other\n}\n',+ "anchor-before-quoted-flow-value": 'meta: {k: &a "x}",\nname: other\n}\n',+ "tag-before-quoted-flow-value": 'meta: {k: !!str "x}",\nname: other\n}\n',+ "tag-then-anchor-before-quoted-value": 'meta: {k: !!str &a "x}",\nname: other\n}\n',+ "anchor-then-tag-before-quoted-value": 'meta: {k: &a !!str "x}",\nname: other\n}\n',+ "verbatim-tag-before-quoted-value": 'meta: {k: !<tag:yaml.org,2002:str> "x}",\nname: other\n}\n',+ "single-quoted-value-with-tag-and-anchor": "meta: {k: !!str &a 'x}',\nname: other\n}\n",+ "properties-before-quoted-sequence-items": 'tags: [&a "x]", !!str "y]",\nname: other]\n',+ "anchored-quoted-flow-key": 'meta: {&a "k}": v,\nname: other\n}\n',+ "tagged-quoted-flow-key": 'meta: {!!str "k}": v,\nname: other\n}\n',+ "explicit-anchored-quoted-flow-key": 'meta: {? &a "k}": v,\nname: other\n}\n',+ "explicit-quoted-key-in-flow-sequence": 'tags: [? "k]": v,\nname: other]\n',+ "anchor-before-flow-mapping-with-quoted-brace": 'meta: {k: &a {x: "}"},\nname: other\n}\n',+ "tag-before-flow-sequence-with-quoted-bracket": 'tags: [!!seq ["]"],\nname: other]\n',+ "properties-before-top-level-flow-mapping": 'meta: &m !!map {k: "}",\nname: other\n}\n',+ "anchored-empty-flow-value": "meta: {k: &x ,\nname: other\n}\n",+ "anchor-then-newline-then-quoted-flow-value": 'meta: {k: &a\n"x}",\nname: other\n}\n',+ "properties-before-block-literal": 'description: !!str |\n {[" name: nope\n',+ "properties-before-block-folded": "description: &d !!str >-\n text {\n",+ "properties-before-block-sequence-literal": "tags:\n- !!str |\n { unclosed\n- b\n",+ "anchored-multiline-double-quoted": 'description: &a "multi\nname: nope"\n',+ "tagged-multiline-single-quoted": "description: !!str 'multi\nname: nope'\n",+ "anchored-multiline-quoted-sequence-item": 'tags:\n- &a "x\nname: nope"\n',+ "anchored-key-with-flow-value": '&a "k": {x: 1,\nname: other}\n',+ "tagged-key-with-quoted-flow-value": "!!str k: {x: '}',\nname: other}\n",+ "explicit-key-with-flow-value": "? meta\n: {x: 1,\nname: other}\n",+}+_TEXT_ANALYZER_FRONTMATTERS = {+ "double-quoted-value": f'name: "text-analyzer"\n{_DESCRIPTION}',+ "single-quoted-value": f"name: 'text-analyzer'\n{_DESCRIPTION}",+ "double-quoted-key": f'"name": text-analyzer\n{_DESCRIPTION}',+ "single-quoted-key-and-value": f"'name': 'text-analyzer'\n{_DESCRIPTION}",+ "space-before-colon": f"name : text-analyzer\n{_DESCRIPTION}",+ "folded-block-scalar": f"name: >-\n text-analyzer\n{_DESCRIPTION}",+ "literal-block-scalar": f"name: |\n text-analyzer\n{_DESCRIPTION}",+ "value-on-next-line": f"name:\n text-analyzer\n{_DESCRIPTION}",+ "plain-trailing-comment": f"name: text-analyzer # shown in listings\n{_DESCRIPTION}",+ "quoted-trailing-comment": f'name: "text-analyzer" # shown in listings\n{_DESCRIPTION}',+ "name-after-block-description": "description: |\n Count words.\n name: not-this-one\nname: text-analyzer\n",+ "nested-name-is-ignored": f"metadata:\n name: not-this-one\nname: text-analyzer\n{_DESCRIPTION}",+ "name-inside-multiline-quoted-text": 'description: "Count words.\nname: not-this-one"\nname: text-analyzer\n',+ # Keys named `name` below the top level never replace the skill name, wherever they sit.+ "nested-block-mapping-after-name": f"name: text-analyzer\nmetadata:\n name: other\n{_DESCRIPTION}",+ "nested-block-mapping-last": f"{_DESCRIPTION}name: text-analyzer\nmetadata:\n version: 1\n name: other\n",+ "single-line-flow-mapping-after-name": f"name: text-analyzer\nmetadata: {{name: other}}\n{_DESCRIPTION}",+ "single-line-flow-mapping-before-name": f"metadata: {{name: other}}\nname: text-analyzer\n{_DESCRIPTION}",+ "multiline-flow-mapping-after-name": f"name: text-analyzer\n{_DESCRIPTION}metadata: {{\nname: other\n}}\n",+ "multiline-flow-mapping-before-name": f"metadata: {{\nname: other\n}}\nname: text-analyzer\n{_DESCRIPTION}",+ "multiline-flow-sequence-with-name-entry": "name: text-analyzer\ntags: [x,\nname: other\n]\n",+ "multiline-flow-mapping-in-block-mapping": "name: text-analyzer\nmeta:\n k: {x: 1,\nname: other}\n",+ "multiline-flow-mapping-in-block-sequence": "name: text-analyzer\ntags:\n- {x: 1,\nname: other}\n",+ "multiline-flow-mapping-on-next-line": "name: text-analyzer\nmeta:\n {x: 1,\nname: other}\n",+ "multiline-flow-mapping-with-anchor": "name: text-analyzer\nbase: &base {\nname: other\n}\n",+ "nested-flow-collections": "name: text-analyzer\nmeta: {a: [1, 2],\nname: other,\nb: {c: d}\n}\n",+ "flow-with-quoted-bracket-and-comment": 'name: text-analyzer\nmeta: {"}": 1, # }\nname: other\n}\n',+ "flow-with-apostrophe-in-plain-value": "name: text-analyzer\nmeta: {x: it's,\nname: other\n}\n",+ "multiline-quoted-in-block-sequence": 'name: text-analyzer\ntags:\n- "x\nname: other"\n',+ "block-sequence-of-mappings": "name: text-analyzer\nitems:\n- name: other\n- name: another\n",+ "name-after-multiline-flow-mapping": f"metadata: {{a: 1,\nb: 2}}\nname: text-analyzer\n{_DESCRIPTION}",+ # A quote starts a string only where a node starts, never inside a plain scalar.+ "inch-mark-in-flow-mapping": f'metadata: {{size: 5 "}}\nname: text-analyzer\n{_QUOTED_DESCRIPTION}',+ "adjacent-inch-mark-in-flow-mapping": f'metadata: {{size: 5"}}\nname: text-analyzer\n{_QUOTED_DESCRIPTION}',+ "quote-inside-flow-sequence-item": f'tags: [a "b]\nname: text-analyzer\n{_QUOTED_DESCRIPTION}',+ "apostrophe-in-flow-mapping-value": f"meta: {{k: it's}}\nname: text-analyzer\n{_QUOTED_DESCRIPTION}",+ "spaced-apostrophe-in-flow-mapping": "meta: {k: rock 'n roll}\nname: text-analyzer\nnote: 'quoted'\n",+ "inch-mark-in-multiline-flow-mapping": f'meta: {{size: 5 ",\nname: other\n}}\nname: text-analyzer\n{_QUOTED_DESCRIPTION}',+ "inch-mark-before-multiline-flow-close": f'meta: {{a: 1 "\n,\nname: other\n}}\nname: text-analyzer\n{_QUOTED_DESCRIPTION}',+ "quoted-string-hides-brace-in-flow-mapping": "meta: {k: \"closing } inside\",\nname: other,\nj: 'it''s ]'\n}\nname: text-analyzer\n",+ "quoted-strings-in-multiline-flow-sequence": "tags: [\"a, b\",\n'c' ,\nname: other]\nname: text-analyzer\n",+ "json-style-flow-mapping": 'meta: {"a":"b",\n"name":"other"\n}\nname: text-analyzer\n',+ "quoted-key-and-value-in-flow-mapping": "meta: {\"name\": \"other\",\n'k': 'v'}\nname: text-analyzer\n",+ "quote-in-block-plain-text": "description: 5\" pipe and 'it' fine\nname: text-analyzer\n",+ # Anchors, tags and aliases belong to the node that follows them, so a quote right after+ # them still opens a string and its brackets stay inside it.+ "anchored-name-key": "&a name: text-analyzer\n",+ "tagged-name-key": "!!str name: text-analyzer\n",+ "tagged-and-anchored-quoted-name-key": "!!str &a 'name': text-analyzer\n",+ "tagged-name-value": 'name: !!str "text-analyzer"\n',+ "anchored-name-value": "name: &n text-analyzer\n",+ "tagged-name-on-next-line": "name: !!str\n text-analyzer\n",+ "tag-and-anchor-quoted-name-value": "name: !!str &n 'text-analyzer'\n",+ "anchored-block-scalar-name": "name: &n >-\n text-analyzer\n",+ # Brackets and quotes inside plain text or block scalars do not open anything.+ "unbalanced-brackets-in-plain-text": "description: Edits {file or [path, it's # not a comment\nname: text-analyzer\n",+ "brackets-in-block-scalar": "description: |\n {[\"' # x\nname: text-analyzer\n",+ "plain-continuation-starting-with-bracket": 'description: Count words.\n [optional] "Use when" asked.\nname: text-analyzer\n',+}+_TEXT_ANALYZER_FRONTMATTERS |= {+ f"{case}-before-name": f"{fragment}name: text-analyzer\n" for case, fragment in _NESTED_NAME_FRAGMENTS.items()+} | {f"{case}-after-name": f"name: text-analyzer\n{fragment}" for case, fragment in _NESTED_NAME_FRAGMENTS.items()}+# The whole frontmatter can be one flow mapping.+_TEXT_ANALYZER_FRONTMATTERS |= {+ "root-flow-mapping": "{name: text-analyzer, description: Count words.}\n",+ "root-flow-mapping-quoted": '{"name": \'text-analyzer\', description: "x}"}\n',+ "root-flow-mapping-multiline": '{description: Count "words",\nname: text-analyzer,\nmetadata: {name: other}\n}\n',+ "root-flow-mapping-nested-name-last": "{name: text-analyzer,\nmetadata: {name: other},\nlist: [name]}\n",+ "root-flow-mapping-with-properties": "!!map &m {name: !!str &n 'text-analyzer', k: &a \"}\"}\n",+}+_TEXT_ANALYZER_FRONTMATTERS["alias-in-flow-mapping"] = (+ 'name: text-analyzer\nbase: &a "x}"\nmeta: {k: *a,\nname: other\n}\n'+)++[email protected]("frontmatter", _TEXT_ANALYZER_FRONTMATTERS.values(), ids=_TEXT_ANALYZER_FRONTMATTERS.keys())+def test_skill_name_is_read_from_any_yaml_spelling_of_the_name(checker, tmp_path, frontmatter):+ """A missing command named after the skill is the agent's choice however the manifest spells the name."""+ context = _skill_named_text_analyzer(tmp_path, frontmatter=frontmatter)+ result = checker(+ [_call("text-analyzer input.txt", "bash: line 1: text-analyzer: command not found")],+ dependency_context=context,+ )+ assert result["failures"][0]["fault_kind"] == "agent_tool_choice"+ assert result["environment_faults"] == 0+ other = checker([_call("jq . out.json", "bash: line 1: jq: command not found")], dependency_context=context)+ assert other["failures"][0]["fault_kind"] == "undeclared_dependency"++[email protected]("frontmatter", _TEXT_ANALYZER_FRONTMATTERS.values(), ids=_TEXT_ANALYZER_FRONTMATTERS.keys())+def test_skill_name_matches_yaml_safe_load(frontmatter):+ """The verifier's reader agrees with the parser the schema validator uses."""+ expected = yaml.safe_load(frontmatter)["name"].strip()+ assert expected == "text-analyzer"+ assert eval_template._frontmatter_skill_name(f"---\n{frontmatter}---\nBody\n") == expected+++def test_crlf_frontmatter_name_is_read(tmp_path):+ skill = tmp_path / "text-analyzer"+ skill.mkdir()+ (skill / "SKILL.md").write_bytes(+ b"---\r\nname: 'text-analyzer'\r\ndescription: Count words.\r\n---\r\nRun scripts/run.py\r\n"+ )+ context = eval_template._dependency_context("text-analyzer", skills_root=tmp_path)+ assert context["skill_name"] == "text-analyzer"+++def test_bom_prefixed_frontmatter_name_is_read(tmp_path):+ skill = tmp_path / "text-analyzer"+ skill.mkdir()+ (skill / "SKILL.md").write_bytes(b"\xef\xbb\xbf---\nname: text-analyzer\ndescription: Count words.\n---\nBody\n")+ assert eval_template._dependency_context("text-analyzer", skills_root=tmp_path)["skill_name"] == "text-analyzer"+++ "frontmatter",+ ["description: Count words.\n", "name:\ndescription: x\n", "name: ''\n"],+ ids=["missing", "empty-value", "empty-string"],+)+def test_skill_without_a_readable_name_records_no_name(tmp_path, frontmatter):+ skill = tmp_path / "demo"+ skill.mkdir()+ (skill / "SKILL.md").write_text(f"---\n{frontmatter}---\nBody\n")+ assert eval_template._dependency_context("demo", skills_root=tmp_path)["skill_name"] == ""+++def test_declared_tool_named_after_the_skill_stays_environment_fault(checker, tmp_path):+ result = checker(+ [_call("text-analyzer input.txt", "bash: line 1: text-analyzer: command not found")],+ dependency_context=_skill_named_text_analyzer(tmp_path, apt="text-analyzer\n"),+ )+ assert result["failures"][0]["fault_kind"] == "declared_dependency_unavailable"+ assert result["environment_faults"] == 1+++def test_other_command_mentioned_by_the_skill_stays_undeclared_dependency(checker, tmp_path):+ result = checker(+ [_call("jq . out.json", "bash: line 1: jq: command not found")],+ dependency_context=_skill_named_text_analyzer(tmp_path),+ )+ assert result["failures"][0]["fault_kind"] == "undeclared_dependency"+ assert result["environment_faults"] == 1++def test_environment_failure_reason_retains_agent_failure(checker):result = checker([