Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(tier3): exclude failed skill reads from execution credit (#201)

Narendran Raghavan2e152d9d476b
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3615279d..bf60e670 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -95,9 +95,46 @@ All notable changes to SkillEvaluator are documented in this file.
- Preserve explicit Claude and OpenCode read failure metadata in reconstructed
ATIF trajectories. Failed isolated SKILL.md reads and failed native Skill
- calls lose credit; compound shell failures remain ambiguous and retain
- prior credit in host and verifier grading. Unattributed errors do not cancel
- successful reads, and shell reads count dash-prefixed filenames after `--`.
+ calls lose credit; compound shell failures and shell reads whose path may
+ expand to several files (braces, globs, `$` or backticks) remain ambiguous
+ and retain prior credit in host and verifier grading. Unattributed errors do
+ not cancel successful reads, and shell reads count dash-prefixed filenames
+ after `--`.
+- Tier 1 pyproject dependency audits now inspect static target declarations
+ rather than the evaluator environment. Exact direct pins (including optional
+ groups) are queried without installation or build-script execution;
+ unresolved declarations and invalid scanner evidence are incomplete.
+- Tier 1 PII scanning now includes `.mdc` rules and workflows.
+- Tier 3 negative-case accuracy excludes inapplicable skill identification and
+ records its scoring policy. Reports separately expose measured token and
+ agent execution-time differences with complete matching-arm coverage, while
+ efficiency continues to measure routing and productive tool use.
+- Derive accuracy scores from complete criterion verdicts in both judge
+ runtimes (passing applicable criteria divided by applicable criteria), so an
+ inconsistent model-supplied total cannot distort Skill Lift. Correct the
+ Harbor prompt example to score five passing criteria as 1.0.
+- Include schema-valid ATIF message content parts in final answers, conversation
+ summaries, and judge evidence references in the host and Harbor verifier.
+ Image and audio parts use markers without exposing their source paths.
+- Preserve scored Harbor trials when agent answers mention authentication errors
+ and per-step token usage is present but aggregate ATIF token metrics are absent
+ or zero. Explicit runtime exceptions and error events still invalidate trials.
+- Safety-classifier refusals, content filters and output-token truncation are
+ detected instead of being read as empty or malformed output. `--llm-verify`
+ and rubric evaluation report the cause, and `--llm-verify` keeps the
+ complete verdicts of a truncated reply. Claude 5.5 or later requests to the
+ native Claude API use server-side fallback to Claude Opus 4.8. In the Harbor
+ verifier, Bedrock retries a filtered Claude 5.5 reply on Claude Opus 4.8, and
+ a refusal or empty judge reply moves on to the next
+ `LLM_JUDGE_FALLBACK_MODELS` entry.
+- The Harbor verifier's last-resort judge model follows the selected provider.
+ With no judge or chat model configured, it used to request an OpenAI model
+ from Anthropic, Bedrock, NVIDIA Build and gateways. The unused host-side
+ `llm_judge.DEFAULT_JUDGE_MODEL` constant is removed.
+- Fixed Docker Codex and Claude Code setup wasting its timeout installing
+ distribution Node/npm before Harbor's own installers. Native, gateway Codex,
+ and NVIDIA Build agents now provision Node/npm only on the installer branches
+ that need them, while preserving installed-agent and version-pin checks.
- Preserved multiline skill descriptions when saving and loading Tier 2
embedding catalogs. Descriptions can contain line breaks and tabs, while
catalog identifiers and other control characters remain strictly validated.
diff --git a/src/skillevaluator/tier3/eval_core/checks.py b/src/skillevaluator/tier3/eval_core/checks.py
index e12db644..aa21ec94 100644
--- a/src/skillevaluator/tier3/eval_core/checks.py
+++ b/src/skillevaluator/tier3/eval_core/checks.py
@@ -640,11 +640,18 @@ def _command_reads_skill_md_arg(command: list[str], cmd_idx: int, assignments: d
return False
+# Brace, glob, parameter and command expansion can turn one shell word into
+# several files. Quotes are gone after tokenizing, so any operand holding one of
+# these characters is treated as possibly naming more than one file.
+_SHELL_EXPANSION_CHARS = frozenset("{*?[$`")
+
+
def _failed_single_skill_read(call):
"""A shell failure only proves the skill read failed for an isolated read.
- Compound commands and multi-file reads may print SKILL.md successfully
- before an unrelated operation fails.
+ Compound commands, multi-file reads and operands that the shell may expand
+ into several files can print SKILL.md successfully before another file or
+ operation fails.
"""
if call.get("is_error") is not True:
return False
@@ -662,7 +669,7 @@ def _failed_single_skill_read(call):
options_ended = True
elif options_ended or arg == "-" or not arg.startswith("-"):
files.append(arg)
- return len(files) == 1 and _skill_md_arg(files[0], {})
+ return len(files) == 1 and not _SHELL_EXPANSION_CHARS.intersection(files[0]) and _skill_md_arg(files[0], {})
def _cmd_reads_skill_md(cmd) -> bool:
diff --git a/src/skillevaluator/tier3/harbor/templates/eval.py b/src/skillevaluator/tier3/harbor/templates/eval.py
index 3cb2c82f..c7538b7c 100644
--- a/src/skillevaluator/tier3/harbor/templates/eval.py
+++ b/src/skillevaluator/tier3/harbor/templates/eval.py
@@ -3268,11 +3268,18 @@ def _command_reads_skill_md_arg(command, cmd_idx, assignments):
return False
+# Brace, glob, parameter and command expansion can turn one shell word into
+# several files. Quotes are gone after tokenizing, so any operand holding one of
+# these characters is treated as possibly naming more than one file.
+_SHELL_EXPANSION_CHARS = frozenset("{*?[$`")
+
+
def _failed_single_skill_read(call):
"""A shell failure only proves the skill read failed for an isolated read.
- Compound commands and multi-file reads may print SKILL.md successfully
- before an unrelated operation fails.
+ Compound commands, multi-file reads and operands that the shell may expand
+ into several files can print SKILL.md successfully before another file or
+ operation fails.
"""
if call.get("is_error") is not True:
return False
@@ -3290,7 +3297,7 @@ def _failed_single_skill_read(call):
options_ended = True
elif options_ended or arg == "-" or not arg.startswith("-"):
files.append(arg)
- return len(files) == 1 and _skill_md_arg(files[0], {})
+ return len(files) == 1 and not _SHELL_EXPANSION_CHARS.intersection(files[0]) and _skill_md_arg(files[0], {})
def _cmd_reads_skill_md(cmd) -> bool:
diff --git a/tests/tier3/test_failed_skill_reads.py b/tests/tier3/test_failed_skill_reads.py
index df4fe143..e69ea0de 100644
--- a/tests/tier3/test_failed_skill_reads.py
+++ b/tests/tier3/test_failed_skill_reads.py
@@ -312,6 +312,67 @@ def test_failed_cat_with_dash_prefixed_second_file_keeps_read_credit(
assert scorer.check_routing(calls, "demo-skill")["score"] == 1.0
+def test_failed_brace_expanded_cat_keeps_read_credit(grader: tuple[ModuleType, ModuleType], tmp_path: Path) -> None:
+ extractor, scorer = grader
+ skill = tmp_path / "skills" / "demo-skill" / "SKILL.md"
+ skill.parent.mkdir(parents=True)
+ skill.write_text("---\nname: demo-skill\n---\n# Demo skill\n", encoding="utf-8")
+ command = f"cat {shlex.quote(str(tmp_path / 'skills'))}/{{demo-skill,missing}}/SKILL.md"
+ result = subprocess.run(["bash", "-c", command], cwd=tmp_path, capture_output=True, text=True, encoding="utf-8")
+ assert result.returncode == 1
+ assert result.stdout == skill.read_text(encoding="utf-8")
+ assert "missing/SKILL.md" in result.stderr
+ trajectory = _claude_tool_result(
+ name="Bash",
+ arguments={"command": command},
+ is_error=True,
+ content=result.stdout + result.stderr,
+ )
+ calls = extractor.extract_tool_calls_as_dicts(trajectory)
+
+ assert calls[0]["is_error"] is True
+ assert scorer.check_activation(calls, "demo-skill")["score"] == 0.75
+ assert scorer.check_workflow_order(calls, expected_skill="demo-skill")["score"] == 1.0
+ assert scorer.check_routing(calls, "demo-skill")["score"] == 1.0
+
+
+ "command",
+ (
+ "cat /skills/*/SKILL.md",
+ "cat /skills/[a-z]*/SKILL.md",
+ "cat /skills/demo-skill?/SKILL.md",
+ "cat $SKILLS_DIR/SKILL.md",
+ "cat `pwd`/SKILL.md",
+ ),
+)
+def test_failed_cat_with_expanding_operand_keeps_read_credit(grader, command: str) -> None:
+ _, scorer = grader
+ read = {
+ "action": "Bash",
+ "action_input": {"command": command},
+ "observation": "---\nname: demo-skill\n---\n# Demo skill\ncat: /skills/other/SKILL.md: Is a directory",
+ "is_error": True,
+ }
+ execute = {"action": "Bash", "action_input": {"command": "python scripts/run.py"}}
+ assert scorer.check_activation([read], "demo-skill")["score"] == 0.75
+ assert scorer.check_workflow_order([read, execute], expected_skill="demo-skill")["score"] == 1.0
+ assert scorer.check_routing([read], "demo-skill")["score"] == 1.0
+
+
+def test_failed_cat_of_home_relative_skill_loses_credit(grader) -> None:
+ # Tilde expansion always yields one path, so this is still an isolated read.
+ _, scorer = grader
+ read = {
+ "action": "Bash",
+ "action_input": {"command": "cat ~/.claude/skills/demo-skill/SKILL.md"},
+ "observation": "cat: /root/.claude/skills/demo-skill/SKILL.md: No such file or directory",
+ "is_error": True,
+ }
+ assert scorer.check_activation([read], "demo-skill")["score"] == 0.0
+ assert scorer.check_routing([read], "demo-skill")["score"] == 0.0
+
+
@pytest.mark.parametrize("options", ("--", "'--'", "-n", "-n --"))
@pytest.mark.parametrize("is_error", (False, True))
def test_single_cat_read_with_options(grader: tuple[ModuleType, ModuleType], options: str, is_error: bool) -> None: