Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

feat(harness): flag Hermes platforms and OpenClaw requirements (#537)

Christopher Kevinb010cc0257d7
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4c8e04a9..97bdc433 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,8 @@ All notable changes to SkillEvaluator are documented in this file.
comment helpers, and needs no Pages configuration or history branch. Future
generated bundles include the template; archived bundles and the released
onboarding CLI retain their existing hosted publisher setup.
+- `harness` rules for harness gates: Hermes `platforms` and OpenClaw
+ `metadata.openclaw.requires`.
- `harness` rules for invocation fields: `disable-model-invocation`, Codex's
`agents/openai.yaml` policy, `allowed-tools`, Claude Code-only fields and Hermes
project trust. Tier 1 `schema` now reports fields outside the agentskills.io
diff --git a/docs/tier1-validation.mdx b/docs/tier1-validation.mdx
index 88c496fa..730cbebb 100644
--- a/docs/tier1-validation.mdx
+++ b/docs/tier1-validation.mdx
@@ -292,6 +292,8 @@ For `validate`, the default set is `schema`, `version`, `security`, `pii`, `lice
Placeholder rules: `$ARGUMENTS` and `${CLAUDE_SKILL_DIR}` are substituted only by Claude Code, and price-like `$N` text is replaced with arguments by Claude Code and OpenCode.
Invocation rules: `disable-model-invocation` (ignored by Codex, Hermes and OpenCode), Codex's `agents/openai.yaml` implicit-invocation policy (Codex only), `allowed-tools`, Claude Code-only fields, and Hermes project trust.
+
+ Gate rules: Hermes `platforms` and OpenClaw `metadata.openclaw.requires` hide the skill on hosts that don't match.
</Accordion>
<Accordion title="security — vulnerability scanning">
diff --git a/src/skillevaluator/harness/overlay.py b/src/skillevaluator/harness/overlay.py
index 871bf838..264d3cce 100644
--- a/src/skillevaluator/harness/overlay.py
+++ b/src/skillevaluator/harness/overlay.py
@@ -355,6 +355,34 @@ def _codex_implicit_disabled(skill_path: Path) -> bool:
return isinstance(policy, dict) and policy.get("allow_implicit_invocation") is False
+_HERMES_OS = {"linux": "linux", "macos": "darwin", "windows": "win32"}
+
+
+def _excluded_os(platforms: Any) -> str:
+ """Operating systems Hermes hides the skill on (it maps names, then prefix-matches sys.platform)."""
+ if not platforms:
+ return ""
+ wanted = [str(p).lower().strip() for p in (platforms if isinstance(platforms, list) else [platforms])]
+ mapped = [_HERMES_OS.get(p, p) for p in wanted]
+ return ", ".join(os for os, sys_name in _HERMES_OS.items() if not any(sys_name.startswith(m) for m in mapped))
+
+
+def _openclaw_gates(data: dict[str, Any]) -> str:
+ metadata = data.get("metadata")
+ claw = metadata.get("openclaw") if isinstance(metadata, dict) else None
+ if not isinstance(claw, dict):
+ return ""
+ requires = claw.get("requires") if isinstance(claw.get("requires"), dict) else {}
+ gates = [
+ f"{key} {', '.join(map(str, value)) if isinstance(value, list) else value}"
+ for key in ("bins", "anyBins", "env", "config")
+ if (value := requires.get(key))
+ ]
+ if claw.get("os"):
+ gates.append(f"os {', '.join(map(str, claw['os'])) if isinstance(claw['os'], list) else claw['os']}")
+ return "; ".join(gates)
+
+
def skill_facts(skill_path: Path, catalog: Catalog | None = None) -> dict[str, Any]:
"""Compute the facts profile rules refer to (see profiles README).
@@ -380,6 +408,7 @@ def skill_facts(skill_path: Path, catalog: Catalog | None = None) -> dict[str, A
match = FRONTMATTER_PATTERN.match(text)
body = match.group(2) if match else text
custom = {k: v for k, v in data.items() if str(k) not in _KNOWN_FIELDS}
+ platforms = data.get("platforms")
name = _skill_name(skill_path, data)
duplicates = catalog.duplicates(skill_path, name) if catalog else []
return {
@@ -418,6 +447,9 @@ def skill_facts(skill_path: Path, catalog: Catalog | None = None) -> dict[str, A
"claude_only_fields": ", ".join(f for f in _CLAUDE_ONLY_FIELDS if f in data),
"custom_fields": ", ".join(map(str, custom)),
"custom_field_chars": sum(len(v) if isinstance(v, str) else len(yaml.safe_dump(v)) for v in custom.values()),
+ "platforms": ", ".join(map(str, platforms)) if isinstance(platforms, list) else str(platforms or ""),
+ "platforms_excluded": _excluded_os(platforms),
+ "openclaw_gates": _openclaw_gates(data),
"skill_md_chars": len(text),
"skill_md_tokens": len(raw) // 4,
"description_over": max(0, (len(description) if isinstance(description, str) else 0) - 1024),
diff --git a/src/skillevaluator/harness/profiles/hermes.yaml b/src/skillevaluator/harness/profiles/hermes.yaml
index 4b8ac6d6..996feae2 100644
--- a/src/skillevaluator/harness/profiles/hermes.yaml
+++ b/src/skillevaluator/harness/profiles/hermes.yaml
@@ -19,6 +19,7 @@ rules:
- {id: HM-ALLOWED-TOOLS, when: "allowed_tools_shape != 'none'", effect: advisory, message: "allowed-tools is ignored by Hermes.", fix: "Don't rely on allowed-tools to restrict tools; use the harness's permission settings."}
- {id: HM-CASING, when: "not manifest_name_exact", effect: not_loaded, message: "Non-canonical name (skill.md/Skill.md): never loaded, on Linux or macOS (3/3 each).", fix: "Rename the file to SKILL.md (exact casing)."}
- {id: HM-DUP-NAME, when: "duplicate_name", effect: advisory, message: "Another skill is also named '{skill_name}' ({duplicate_paths}); Hermes uses one: project, then ~/.hermes/skills.", fix: "Give each skill a unique name."}
+ - {id: HM-PLATFORMS, when: "platforms_excluded != ''", effect: advisory, message: "Hidden from the skill list on {platforms_excluded} (platforms: {platforms}).", fix: "Add those systems to platforms, or remove platforms if the skill works everywhere."}
- {id: HM-RESOURCE-SYMLINK, when: "resource_symlinks > 0", effect: degraded, message: "Symlinked files ({resource_symlink_paths}) are left out of the skill's file list.", fix: "Copy those files into the skill instead of symlinking them."}
- {id: HM-LINKED-LAYOUT, when: "hermes_unlisted_files > 0", effect: advisory, message: "{hermes_unlisted_files} files outside references/, templates/, assets/ and scripts/ ({hermes_unlisted_dirs}) are left out of the skill's file list; the model finds them only if SKILL.md names their paths.", fix: "Move supporting files under references/, templates/, assets/ or scripts/."}
- {id: HM-VIEW-BUDGET, when: "hermes_view_chars > 100000", effect: degraded, outcome: instructions not delivered, message: "When the model loads the skill, skill_view returns ~{hermes_view_chars} chars, over the 100,000-char tool-result budget (lower on models under ~167K context): the model gets a 1,500-char preview, not the instructions.", fix: "Keep SKILL.md under ~95,000 characters; move detail into references/ files."}
diff --git a/src/skillevaluator/harness/profiles/openclaw.yaml b/src/skillevaluator/harness/profiles/openclaw.yaml
index dee064a3..f3cedc45 100644
--- a/src/skillevaluator/harness/profiles/openclaw.yaml
+++ b/src/skillevaluator/harness/profiles/openclaw.yaml
@@ -22,6 +22,7 @@ rules:
- {id: OW-CASING, when: "not manifest_name_exact", effect: degraded, message: "Non-canonical name (skill.md/Skill.md): loaded on case-insensitive filesystems (macOS default APFS, 3/3), ignored on case-sensitive Linux (3/3). Works or breaks depending on the OS.", fix: "Rename the file to SKILL.md (exact casing)."}
- {id: OW-PROMPT-CAP, when: "catalog_size > 150 or catalog_desc_chars > 18000", effect: advisory, outcome: listing shortened, message: "{catalog_size} skills / {catalog_desc_chars} description chars here: past 150 skills or 18,000 chars OpenClaw also leaves skills out of the list (121 of 225 listed in our test).", fix: "Shorten descriptions (trigger first) or install fewer skills in one folder."}
- {id: OW-DUP-NAME, when: "duplicate_name", effect: advisory, message: "Another skill is also named '{skill_name}' ({duplicate_paths}); OpenClaw uses one: workspace skills/, then .agents/skills, then user folders.", fix: "Give each skill a unique name."}
+ - {id: OW-REQUIRES, when: "openclaw_gates != ''", effect: advisory, message: "Left out of the skill list unless the host has: {openclaw_gates}.", fix: "Make sure target hosts provide these, or drop requirements the skill doesn't need."}
- {id: OW-REF-BUDGET, when: "files_over_64000 > 0", effect: advisory, outcome: needs paging, message: "{files_over_64000} files over 64,000 bytes ({files_over_64000_paths}): OpenClaw's read cap is 16,000-64,000 UTF-8 bytes by model context, so each takes several reads.", fix: "Split large reference files into pieces under 16,000 UTF-8 bytes."}
- {id: OW-CODEX-POLICY, when: "codex_implicit_disabled and not disable_model_invocation", effect: degraded, message: "agents/openai.yaml implicit-invocation policy is Codex-only: here the model can still use this skill on its own.", fix: "Also set disable-model-invocation: true (Claude Code, Copilot and OpenClaw honour it)."}
- {id: OW-ARGUMENTS, when: "uses_arguments_placeholder", effect: degraded, message: "$ARGUMENTS is not substituted: the model sees the literal text.", fix: "Describe the expected input in words instead of $ARGUMENTS."}
diff --git a/tests/harness/test_overlay.py b/tests/harness/test_overlay.py
index f2a1b24f..e8f433e1 100644
--- a/tests/harness/test_overlay.py
+++ b/tests/harness/test_overlay.py
@@ -526,6 +526,10 @@ def test_gating_and_claude_only_fields(tmp_path: Path):
)
rows = _rows(skill)
+ assert "HM-PLATFORMS" in _ids(rows, "hermes")
+ assert "Left out of the skill list unless the host has: bins pdftotext." in {
+ f["message"] for f in rows["openclaw"]["findings"]
+ }
assert "OC-CLAUDE-ONLY" in _ids(rows, "opencode")
assert not any(f["id"].endswith("CLAUDE-ONLY") for f in rows["claude-code"]["findings"])
@@ -564,6 +568,16 @@ def test_fix_reaches_findings_and_panels(tmp_path: Path):
assert "Description in the model's skill list" in out
+def test_hermes_platforms_listing_every_os_is_not_flagged(tmp_path: Path):
+ everywhere = _rows(_skill(tmp_path / "a", extra="platforms: [linux, macos, windows]\n"))
+ mac_only = _rows(_skill(tmp_path / "b", extra="platforms: macos\n"))
+
+ assert "HM-PLATFORMS" not in _ids(everywhere, "hermes")
+ assert "Hidden from the skill list on linux, windows (platforms: macos)." in {
+ f["message"] for f in mac_only["hermes"]["findings"]
+ }
+
+
def _sized(root: Path, fact: str, target: int) -> Path:
"""A skill whose ``fact`` (a body-length measure) is exactly ``target``."""
probe = _skill(root / "probe", body="a")