Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

Note Claude Code skill fields whose value Claude Code ignores (#623)

Christopher Kevin2ff4fc3a7b8b
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f832de5d..547abf66 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -298,6 +298,10 @@ All notable changes to SkillEvaluator are documented in this file.
- Claude Code `harness` results count `when_to_use` toward the 1,536-character
listing cut (`CC-DESC-CUT`): Claude Code lists `<description> - <when_to_use>`
and cuts that whole text, so a long `when_to_use` loses its end.
+- Claude Code `harness` results note skill fields whose value Claude Code ignores
+ (`CC-FIELD-VALUE`): an `effort` other than low, medium, high, xhigh, max or a number,
+ a `shell` other than bash or powershell, `background` without `context: fork`, and
+ `maxTurns`. The skill still loads.
- Copilot CLI `harness` results flag an empty or whitespace-only `description`
(`CP-DESC-EMPTY`): Copilot lists the skill with a blank description, so the
model sees only its name.
diff --git a/src/skillevaluator/harness/overlay.py b/src/skillevaluator/harness/overlay.py
index 420da8b6..8c96bdfa 100644
--- a/src/skillevaluator/harness/overlay.py
+++ b/src/skillevaluator/harness/overlay.py
@@ -400,6 +400,30 @@ def _openclaw_gates(data: dict[str, Any]) -> str:
return "; ".join(gates)
+_CLAUDE_EFFORTS = ("low", "medium", "high", "xhigh", "max")
+_CLAUDE_SHELLS = ("bash", "powershell")
+
+
+def _claude_ignored_values(data: dict[str, Any]) -> list[str]:
+ """Claude Code skill fields whose value it ignores, each with what it accepts."""
+ ignored = []
+ effort = data.get("effort")
+ text = str(effort).strip().lower()
+ whole_number = isinstance(effort, int) and not isinstance(effort, bool)
+ whole_number = whole_number or (isinstance(effort, float) and effort.is_integer())
+ numeric = whole_number or text.isdigit()
+ if effort is not None and not numeric and text not in _CLAUDE_EFFORTS:
+ ignored.append(f"effort '{effort}' (use {', '.join(_CLAUDE_EFFORTS)} or a number)")
+ shell = data.get("shell")
+ if shell is not None and str(shell).strip().lower() not in ("", *_CLAUDE_SHELLS):
+ ignored.append(f"shell '{shell}' (use bash or powershell; Claude Code falls back to bash)")
+ if _is_true(data.get("background")) and data.get("context") != "fork":
+ ignored.append("background without context: fork")
+ if "maxTurns" in data:
+ ignored.append("maxTurns (an agent field, not a skill field)")
+ return ignored
+
+
def skill_facts(skill_path: Path, catalog: Catalog | None = None) -> dict[str, Any]:
"""Compute the facts profile rules refer to (see profiles README).
@@ -469,6 +493,7 @@ def skill_facts(skill_path: Path, catalog: Catalog | None = None) -> dict[str, A
"disable_model_invocation": _is_true(data.get("disable-model-invocation")),
"codex_implicit_disabled": _codex_implicit_disabled(skill_path),
"claude_only_fields": ", ".join(f for f in _CLAUDE_ONLY_FIELDS if f in data),
+ "claude_ignored_values": "; ".join(_claude_ignored_values(data)),
"custom_fields": ", ".join(map(str, custom)),
"custom_field_chars": sum(len(v) if isinstance(v, str) else len(yaml.safe_dump(v)) for v in custom.values()),
"platforms": ", ".join(map(str, platforms)) if isinstance(platforms, list) else str(platforms or ""),
diff --git a/src/skillevaluator/harness/profiles/claude-code.yaml b/src/skillevaluator/harness/profiles/claude-code.yaml
index 4bde682f..a42fdfbd 100644
--- a/src/skillevaluator/harness/profiles/claude-code.yaml
+++ b/src/skillevaluator/harness/profiles/claude-code.yaml
@@ -21,6 +21,7 @@ rules:
- {id: CC-CASING, when: "not manifest_name_exact", effect: degraded, message: "Non-canonical name (skill.md/Skill.md): loaded on case-insensitive filesystems (macOS default APFS, 3/3), ignored on case-sensitive Linux (3/3). Works or breaks depending on the OS.", fix: "Rename the file to SKILL.md (exact casing).", confidence: high}
- {id: CC-CODEX-POLICY, when: "codex_implicit_disabled and not disable_model_invocation", effect: degraded, message: "agents/openai.yaml implicit-invocation policy is Codex-only: here the model can still use this skill on its own.", fix: "Also set disable-model-invocation: true (Claude Code, Copilot and OpenClaw honour it).", confidence: high}
- {id: CC-FM-STRIPPED, when: "custom_field_chars > 200", effect: advisory, message: "Frontmatter is stripped when the skill is used, so the text in {custom_fields} never reaches the model.", fix: "Move instructions from frontmatter into the SKILL.md body.", confidence: medium}
+ - {id: CC-FIELD-VALUE, when: "claude_ignored_values != ''", effect: advisory, message: "Loaded, but Claude Code ignores {claude_ignored_values}.", fix: "Use a value Claude Code accepts, or remove the field.", confidence: high}
notes:
- "Follows symlinked skill folders and symlinked SKILL.md files, inside or outside the root."
- "Nested SKILL.md, CRLF, BOM, tabs, uppercase names: no effect observed."
diff --git a/tests/harness/test_overlay.py b/tests/harness/test_overlay.py
index ed59e94a..530cb910 100644
--- a/tests/harness/test_overlay.py
+++ b/tests/harness/test_overlay.py
@@ -840,6 +840,33 @@ def test_claude_allowed_tools_blocks_headless_load(tmp_path: Path):
assert "CC-ALLOWED-TOOLS-HEADLESS" in _rule_ids(skill, "claude-code", {"headless": True})
+def test_claude_ignored_field_values_are_noted(tmp_path: Path):
+ bad = _skill(tmp_path / "bad", extra="effort: extreme\nshell: zsh\nbackground: true\nmaxTurns: 5\n")
+ good = _skill(tmp_path / "good", extra="effort: XHigh\nshell: powershell\ncontext: fork\nbackground: true\n")
+ numeric = _skill(tmp_path / "numeric", extra="effort: 3\n")
+
+ finding = next(f for r in evaluate_harnesses(bad, "claude-code") for f in r.findings if f["id"] == "CC-FIELD-VALUE")
+ assert finding["effect"] == "advisory"
+ for part in ("effort 'extreme'", "shell 'zsh'", "background without context: fork", "maxTurns"):
+ assert part in finding["message"]
+ assert _status(bad)["claude-code"] == "loads"
+ assert "CC-FIELD-VALUE" not in _rule_ids(good, "claude-code")
+ assert "CC-FIELD-VALUE" not in _rule_ids(numeric, "claude-code")
+
+
+def test_claude_effort_accepts_whole_number_floats_only(tmp_path: Path):
+ whole = _skill(tmp_path / "whole", extra="effort: 3.0\n")
+ fraction = _skill(tmp_path / "fraction", extra="effort: 3.5\n")
+ flag = _skill(tmp_path / "flag", extra="effort: true\n")
+
+ assert "CC-FIELD-VALUE" not in _rule_ids(whole, "claude-code")
+ for skill, shown in ((fraction, "effort '3.5'"), (flag, "effort 'True'")):
+ finding = next(
+ f for r in evaluate_harnesses(skill, "claude-code") for f in r.findings if f["id"] == "CC-FIELD-VALUE"
+ )
+ assert shown in finding["message"]
+
+
def test_native_root_skips_root_rules(tmp_path):
skill = tmp_path / ".claude" / "skills" / "demo"
skill.mkdir(parents=True)