Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

feat(tier1): accept a list of values for metadata.domain (#279)

Narendran Raghavan9e34dd4cbf7d
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9137700d..77053ca1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -109,6 +109,17 @@ All notable changes to SkillEvaluator are documented in this file.
expected-case checks cover only the cases that ran, `run_config.json` records
the selection as `case_selection`, and a run that skipped cases is reported
as publication-incomplete in JSON, HTML, and BENCHMARK.md.
+- Tier 1 schema validation accepts a list for `metadata.domain` in `SKILL.md`
+ (for example `domain: [backend, ai-ml]`) instead of reporting a HIGH
+ `Input should be a valid string` finding that also stopped the remaining
+ schema checks for that skill. A single string is still accepted and is
+ treated as a one-entry list. The parsed value is now always a list: entries
+ are trimmed and deduplicated in order, and a missing, null, or blank value
+ becomes an empty list. Lists longer than 16 entries and non-string, blank, or
+ over-128-character entries are rejected. This adds a length limit that
+ `metadata.domain` did not have before: a single string longer than 128
+ characters, which used to pass, now fails Tier 1 with a HIGH
+ `Field 'metadata.domain'` finding.
- `doctor` and `health-check` take `--tier 1|2|3|all`, so Tier 1 and Tier 2
readiness can be checked before a run instead of surfacing mid-validation.
Tier 1 reports each scanner that the `--checks` selection runs: SkillSpector,
diff --git a/docs/tier1-validation.mdx b/docs/tier1-validation.mdx
index 1d53aa13..c9b54d81 100644
--- a/docs/tier1-validation.mdx
+++ b/docs/tier1-validation.mdx
@@ -163,6 +163,7 @@ For `validate`, the default set is `schema`, `version`, `security`, `pii`, `lice
- Recognized skill-root subdirectories are `agents/`, `references/`, `scripts/`, `assets/`, `evals/`, `tests/`, `tools/`, and `config/`; anything else gets a LOW advisory finding. Extend the set per repository with `SKILLEVALUATOR_SCHEMA_ALLOWED_DIRS` (comma-separated names are added to the defaults, never replace them).
- `SKILL.md` should stay at or below 500 lines; larger files receive an advisory finding.
- The default profile expects `metadata.author` in `Name <email@host>` form but does not require a particular email domain.
+ - `metadata.domain` accepts one category string or a list of them, such as `domain: [backend, ai-ml]`. Values are trimmed and deduplicated in order; a blank or null scalar means no domain. A list with more than 16 entries, or a non-string, blank, or over-128-character entry, is a HIGH `Field 'metadata.domain'` finding. The 128-character limit also applies to a single string.
**To fix common findings:** rename the directory to match `name`, remove forbidden fields, and move the skill under `skills/`.
</Accordion>
diff --git a/src/skillevaluator/models/skill.py b/src/skillevaluator/models/skill.py
index 9915317a..f2b95ed7 100644
--- a/src/skillevaluator/models/skill.py
+++ b/src/skillevaluator/models/skill.py
@@ -30,6 +30,10 @@ XML_TAG_RE = re.compile(r"</?[A-Za-z][A-Za-z0-9_-]*(?:\s[^<>]*)?/?>|</?[A-Za-z][
_SEMVER_COMPONENT = r"(?:0|[1-9][0-9]{0,8})"
SEMVER_RE = re.compile(rf"^{_SEMVER_COMPONENT}\.{_SEMVER_COMPONENT}\.{_SEMVER_COMPONENT}$", re.ASCII)
+#: Bounds for metadata.domain, which accepts a single string or a list of strings.
+DOMAIN_MAX_ITEMS = 16
+DOMAIN_MAX_LENGTH = 128
+
class SkillMetadata(BaseModel):
"""Metadata object for SKILL.md frontmatter.
@@ -49,7 +53,10 @@ class SkillMetadata(BaseModel):
tags: list[str] | None = Field(default=None, description="Categorization tags")
languages: list[str] | None = Field(default=None, description="Programming languages")
frameworks: list[str] | None = Field(default=None, description="Frameworks used")
- domain: str | None = Field(default=None, description="Domain category (e.g., backend, ai-ml)")
+ domain: list[str] = Field(
+ default_factory=list,
+ description="Domain categories (e.g., backend, ai-ml)",
+ )
@field_validator("tags", "languages", "frameworks", mode="before")
@classmethod
@@ -63,6 +70,39 @@ class SkillMetadata(BaseModel):
return [str(item) for item in v]
return v
+ @field_validator("domain", mode="before")
+ @classmethod
+ def normalize_domain(cls, v: Any) -> list[str]:
+ """Normalize a scalar domain to a list and validate list entries."""
+ if v is None:
+ return []
+ if isinstance(v, str):
+ if not v.strip():
+ # A blank scalar such as ``domain: ""`` means "no domain".
+ return []
+ values = [v]
+ else:
+ values = v
+ if not isinstance(values, list):
+ raise ValueError("domain must be a string or list of strings")
+ if len(values) > DOMAIN_MAX_ITEMS:
+ raise ValueError(f"domain must contain at most {DOMAIN_MAX_ITEMS} values")
+
+ normalized: list[str] = []
+ seen: set[str] = set()
+ for value in values:
+ if not isinstance(value, str):
+ raise ValueError("domain values must be strings")
+ value = value.strip()
+ if not value:
+ raise ValueError("domain values must not be blank")
+ if len(value) > DOMAIN_MAX_LENGTH:
+ raise ValueError(f"domain values must be at most {DOMAIN_MAX_LENGTH} characters")
+ if value not in seen:
+ normalized.append(value)
+ seen.add(value)
+ return normalized
+
@field_validator("author")
@classmethod
def validate_author_format(cls, v: str | None) -> str | None:
diff --git a/tests/validators/test_schema.py b/tests/validators/test_schema.py
index 0cf14883..e038988b 100644
--- a/tests/validators/test_schema.py
+++ b/tests/validators/test_schema.py
@@ -10,6 +10,7 @@ from pathlib import Path
import pytest
+from skillevaluator.models.skill import SkillMetadata
from skillevaluator.validators.schema import SchemaValidator
@@ -329,7 +330,9 @@ metadata:
- python
frameworks:
- pytest
- domain: testing
+ domain:
+ - testing
+ - quality
---
# Complete Metadata Skill
@@ -351,6 +354,82 @@ metadata:
result = validator.validate(skill_dir)
assert result.passed, f"Expected complete metadata skill to pass. Errors: {result.errors}"
+ assert result.metadata["frontmatter"].metadata.domain == ["testing", "quality"]
+
+ @pytest.mark.parametrize(
+ ("metadata", "expected"),
+ [
+ pytest.param({"domain": "backend"}, ["backend"], id="scalar"),
+ pytest.param({"domain": ["backend", "ai-ml"]}, ["backend", "ai-ml"], id="list"),
+ pytest.param(
+ {"domain": [" backend ", "ai-ml", "backend"]},
+ ["backend", "ai-ml"],
+ id="strip-and-stable-dedupe",
+ ),
+ pytest.param({}, [], id="missing"),
+ pytest.param({"domain": None}, [], id="null"),
+ pytest.param({"domain": []}, [], id="empty"),
+ pytest.param({"domain": ""}, [], id="blank-scalar"),
+ pytest.param({"domain": " "}, [], id="whitespace-scalar"),
+ ],
+ )
+ def test_metadata_domain_normalization(self, metadata: dict[str, object], expected: list[str]):
+ assert SkillMetadata(**metadata).domain == expected
+
+ @pytest.mark.parametrize(
+ ("domain", "message"),
+ [
+ pytest.param(["backend", 1], "domain values must be strings", id="non-string"),
+ pytest.param(["backend", " "], "domain values must not be blank", id="blank-list-entry"),
+ pytest.param(["backend"] * 17, "domain must contain at most 16 values", id="too-many-raw-items"),
+ pytest.param("x" * 129, "domain values must be at most 128 characters", id="too-long"),
+ pytest.param({"name": "backend"}, "domain must be a string or list of strings", id="mapping"),
+ ],
+ )
+ def test_metadata_domain_rejects_invalid_values(self, domain: object, message: str):
+ with pytest.raises(ValueError, match=message):
+ SkillMetadata(domain=domain)
+
+ @pytest.mark.parametrize(
+ ("domain_yaml", "message"),
+ [
+ pytest.param(
+ " domain:\n - backend\n - 42\n",
+ "domain values must be strings",
+ id="non-string-list-entry",
+ ),
+ pytest.param(
+ f" domain: {'x' * 129}\n",
+ "domain values must be at most 128 characters",
+ id="over-long-single-string",
+ ),
+ ],
+ )
+ def test_invalid_metadata_domain_reports_field_finding(self, tmp_path: Path, domain_yaml: str, message: str):
+ """An invalid domain, including an over-long single string, is a HIGH metadata.domain finding."""
+ from skillevaluator.models.result import Severity
+
+ skill_dir = tmp_path / "bad-domain"
+ skill_dir.mkdir()
+ (skill_dir / "SKILL.md").write_text(
+ "---\n"
+ "name: bad-domain\n"
+ "description: A skill with an invalid domain\n"
+ "metadata:\n"
+ " author: Test User <[email protected]>\n"
+ f"{domain_yaml}"
+ "---\n\n# Bad Domain\n",
+ encoding="utf-8",
+ )
+
+ result = SchemaValidator().validate(skill_dir)
+
+ assert not result.passed
+ findings = [f for f in result.findings if f.check_name == "frontmatter_field"]
+ assert len(findings) == 1
+ assert findings[0].severity == Severity.HIGH
+ assert findings[0].metadata["field"] == "metadata.domain"
+ assert message in findings[0].message
# =========================================================================
# agentskills.io spec + validate_skill_md.py CI checks