Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(tier2): name the skill whose frontmatter stops the CI comparison catalog (#549)

Narendran Raghavan48fe16a0d553
diff --git a/CHANGELOG.md b/CHANGELOG.md
index efce7c6e..9e59eaff 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -723,6 +723,26 @@ All notable changes to SkillEvaluator are documented in this file.
by default, instead of staying on its first frame. These follow the 0.9.0
Fixed entry that starts "The interactive Tier 3 progress view
(`--progress rich`, or `auto` in a terminal) now keeps one height".
+- All-tier CI now says when a skill's frontmatter stops the Tier 2 comparison
+ catalog, and names every such skill. When the configured skill roots hold
+ more than one skill, CI builds the catalog from all of them, including skills
+ that did not change. One `SKILL.md` among them whose frontmatter is not a
+ valid YAML mapping with a non-empty string `name` and `description` stops it
+ before any embedding request, and Tier 2 then checks no skill and is
+ incomplete. This includes a value that is not a string, such as
+ `name: 2048`, `name: yes`, or `description: [Draft]`, which Tier 2 commands
+ such as `similarity-check` converted to text before 0.9.0. With one skill
+ there is no catalog, and such frontmatter does not stop Tier 2. The Tier 2
+ reason pointed at provider and embedding configuration; it now says that a
+ skill's frontmatter is the cause, and the retained catalog log under
+ `tier2/catalog/diagnostics/` names every such skill by its path in the
+ repository. Quote such a value, as in `name: "2048"`. A `SKILL.md` that is
+ not valid UTF-8, such as one saved as UTF-16 or as Windows-1252 with an
+ accented letter, stops the catalog with the same reason and is named the same
+ way; save it as UTF-8. Frontmatter over a structure or length limit in any
+ skill still gets the reason that points at provider and embedding
+ configuration, and the log then names no skill, not even one whose
+ frontmatter is invalid.
## 0.9.0 - 2026-11-13
diff --git a/ci/run_tiers.py b/ci/run_tiers.py
index 267da880..e7797262 100644
--- a/ci/run_tiers.py
+++ b/ci/run_tiers.py
@@ -401,13 +401,24 @@ def _retain(
return {"status": status, "metrics": metrics, "complete": complete}
-def _build_catalog(workspace: Path, selected: list[Path], path: Path) -> None:
+# Catalog helper exit status when a discovered skill has no valid SKILL.md frontmatter.
+INVALID_CATALOG_SKILL = 3
+
+
+def _build_catalog(workspace: Path, selected: list[Path], path: Path) -> int:
from skillevaluator.embedding.client import EmbeddingClient
- from skillevaluator.embedding.registry import EmbeddingRegistry
+ from skillevaluator.embedding.registry import EmbeddingRegistry, InvalidExplicitSkillError
registry = EmbeddingRegistry(EmbeddingClient())
- registry.build_from_paths(workspace, selected)
+ try:
+ registry.build_from_paths(workspace, selected)
+ except InvalidExplicitSkillError as exc:
+ # Raised before any embedding request; it names only the skill's
+ # checkout-relative path, never provider output.
+ print(exc, file=sys.stderr)
+ return INVALID_CATALOG_SKILL
registry.save_catalog(path)
+ return 0
def evaluate(tier: int, workspace: Path, reports: Path, env: dict[str, str]) -> int:
@@ -488,11 +499,18 @@ def evaluate(tier: int, workspace: Path, reports: Path, env: dict[str, str]) ->
"--output",
str(catalog),
]
- if run_command(
+ catalog_status = run_command(
args, env=child, cwd=workspace, timeout=deadline - time.monotonic(), log=scratch / "catalog.log"
- ):
+ )
+ if catalog_status:
+ # The PR/MR comment shows about 110 characters of a tier reason; the
+ # cause and the pointer to the diagnostics must both fit.
row["reason"] = (
- "The complete comparison catalog could not be built; inspect provider and embedding configuration."
+ "A skill's invalid SKILL.md frontmatter stopped the comparison catalog; its diagnostics name "
+ "the skill."
+ if catalog_status == INVALID_CATALOG_SKILL
+ else "The complete comparison catalog could not be built; inspect provider and embedding "
+ "configuration."
)
diagnostics = scratch / "catalog-diagnostics"
diagnostics.mkdir()
@@ -810,8 +828,7 @@ def main(argv: list[str] | None = None) -> int:
return 0
if args.command == "catalog":
request = read_json(args.input)
- _build_catalog(Path(request["workspace"]), [Path(p) for p in request["skills"]], args.output)
- return 0
+ return _build_catalog(Path(request["workspace"]), [Path(p) for p in request["skills"]], args.output)
if args.command == "native-tier3":
config = read_json(args.config)
config["SKILL_EVAL_ENV_MODE"] = args.env_mode
diff --git a/docs/tier2-deduplication.mdx b/docs/tier2-deduplication.mdx
index 760ce098..1233a314 100644
--- a/docs/tier2-deduplication.mdx
+++ b/docs/tier2-deduplication.mdx
@@ -24,6 +24,23 @@ secrets. Tier 2 does not read `evals/`; committed native datasets are needed
only for [Tier 3](/skills/skillevaluator/tier3-live-evaluation). Missing credentials or a central
dataset mapping produce an actionable incomplete result before paid work.
+When the configured skill roots hold more than one skill, CI compares the
+selected skills with a catalog built from all of them, including skills that
+did not change. One `SKILL.md` among them whose frontmatter is not a valid YAML
+mapping with a non-empty string `name` and `description`, such as `name: 2048`,
+`name: yes`, or `description: [Draft]`, stops that catalog before any embedding
+request. Tier 2 then checks no skill and is incomplete. Its reason says that a
+skill's frontmatter is the cause, and the retained catalog diagnostics name
+every such skill by its path in the repository. Quote such a value, as in
+`name: "2048"`. A `SKILL.md` that is not valid UTF-8, such as one saved as
+UTF-16 or as Windows-1252 with an accented letter, stops the catalog with the
+same reason and is named the same way; save it as UTF-8.
+Frontmatter over a [structure or length limit](#defaults-and-safety-limits)
+also stops the catalog, but with the reason that points at provider and
+embedding configuration, and the diagnostics then name no skill, not even one
+whose frontmatter is invalid as described above. Run `similarity-check` on the
+skill root to find the manifest over the limit.
+
The aggregate artifact preserves native Tier 2 JSON and HTML, including
deduplication and catalog-similarity findings. Failed, missing or incomplete
Tier 2 results stay advisory and do not block a passing Tier 1 CI gate; they are
@@ -592,7 +609,7 @@ handling terms before sending confidential skill content to a hosted endpoint.
| Catalog path is missing | Returns an error and does not rebuild it silently |
| Candidate lacks a root `SKILL.md`, or its frontmatter is not a valid YAML mapping with a non-empty string name and description | Rejects the catalog query and says what the frontmatter needs |
| A manifest's frontmatter exceeds a structure limit, or its name or description is longer than 16,384 characters | Fails with an error that names the manifest, before any embedding request |
- | A manifest's name or description is not a string, such as `name: 2048`, or its frontmatter is not a mapping | Excludes the manifest with a coverage reason, as for invalid YAML, and reports the scan as incomplete |
+ | A manifest's name or description is not a string, such as `name: 2048`, or its frontmatter is not a mapping | Excludes the manifest with a coverage reason, as for invalid YAML, and reports the scan as incomplete. In [automatic CI](#tier-2-in-automatic-ci) with more than one skill, stops the comparison catalog and names the skill |
| A skill contains an exactly named, single-link regular `AGENTS.md` plus `CLAUDE.md -> AGENTS.md` | Validates and skips the unrelated compatibility alias while discovering `SKILL.md` |
| The collection contains a relative directory alias such as `.agents -> .claude` whose target is a regular directory in the same collection | Skips the alias without following it, so manifests in the target are discovered once |
| Candidate matches multiple catalog entries | Reports matches in descending similarity order |
diff --git a/src/skillevaluator/embedding/extractor.py b/src/skillevaluator/embedding/extractor.py
index 7971f0cc..6f64ae42 100644
--- a/src/skillevaluator/embedding/extractor.py
+++ b/src/skillevaluator/embedding/extractor.py
@@ -64,6 +64,10 @@ MAX_EXCLUSION_PATH_CHARS = 512
MAX_FRONTMATTER_FIELD_CHARS = 16_384
+class ManifestEncodingError(ValueError):
+ """A manifest's bytes are not valid UTF-8, so its frontmatter cannot be read."""
+
+
@dataclass
class ContentEntry:
"""Unified representation of a content item for embedding.
@@ -440,7 +444,7 @@ def _read_bounded_manifest(
try:
return raw_bytes.decode("utf-8")
except UnicodeDecodeError as exc:
- raise ValueError(f"Manifest is not valid UTF-8: {file_path.name}") from exc
+ raise ManifestEncodingError(f"Manifest is not valid UTF-8: {file_path.name}") from exc
def _is_discovery_manifest_name(name: str) -> bool:
diff --git a/src/skillevaluator/embedding/registry.py b/src/skillevaluator/embedding/registry.py
index 46fa01fd..1efeb389 100644
--- a/src/skillevaluator/embedding/registry.py
+++ b/src/skillevaluator/embedding/registry.py
@@ -49,6 +49,7 @@ from skillevaluator.embedding.extractor import (
MAX_MANIFEST_BYTES,
ContentEntry,
ExtractionResult,
+ ManifestEncodingError,
discover_and_extract_detailed,
extract_from_skill,
)
@@ -105,6 +106,16 @@ _CATALOG_ENTRY_FIELDS = frozenset(
)
+class InvalidExplicitSkillError(ValueError):
+ """Explicit skills have no root UTF-8 ``SKILL.md`` with valid name and description frontmatter.
+
+ :meth:`EmbeddingRegistry.build_from_paths` raises it before any embedding
+ request. The message names every such skill, in sorted order, only by its
+ path under the checkout root, so callers can show it without exposing host
+ directories.
+ """
+
+
@dataclass
class RegistryEntry:
"""A single item in the embedding index."""
@@ -255,6 +266,10 @@ class EmbeddingRegistry:
if not unique or len(unique) > self._max_entries:
raise ValueError("Explicit skill collection exceeds its supported entry count")
entries = []
+ # Every skill is checked, so one error names all skills whose frontmatter
+ # needs fixing, including a SKILL.md that is not UTF-8. Limit and link
+ # errors still stop at their own skill.
+ invalid = []
for path in unique:
try:
relative = path.absolute().relative_to(root)
@@ -267,10 +282,19 @@ class EmbeddingRegistry:
raise ValueError("Explicit skill paths must not contain links or parent traversal")
if not current.resolve(strict=True).is_relative_to(root):
raise ValueError("An explicit skill is outside the checkout root")
- entry = extract_from_skill(current)
+ try:
+ entry = extract_from_skill(current)
+ except ManifestEncodingError:
+ entry = None
if entry is None:
- raise ValueError("Each explicit path must contain a valid SKILL.md")
- entries.append(entry)
+ invalid.append(relative.as_posix())
+ else:
+ entries.append(entry)
+ if invalid:
+ raise InvalidExplicitSkillError(
+ "Each explicit skill requires a root SKILL.md, saved as UTF-8, whose frontmatter is a valid YAML "
+ f"mapping with a non-empty string name and description: {', '.join(invalid)}"
+ )
return self._build_entries(root, entries, 1, False)
def _build_entries(
diff --git a/tests/ci/test_run_tiers.py b/tests/ci/test_run_tiers.py
index 03730f05..da82cb54 100644
--- a/tests/ci/test_run_tiers.py
+++ b/tests/ci/test_run_tiers.py
@@ -991,3 +991,106 @@ def test_no_skill_change_never_excuses_missing_tier1_evidence(consumer, tmp_path
rows = {row["tier"]: row for row in json.loads((reports / "ci-results.json").read_text())["tiers"]}
assert rows[1]["status"] == "incomplete"
assert rows[2]["status"] == "skipped"
+
+
+def _evaluate_tier2_with_unselected_game(consumer, tmp_path, monkeypatch, frontmatter):
+ """Run Tier 2 with only alpha selected and an unselected ``skills/game`` carrying *frontmatter*."""
+ import contextlib
+
+ from skillevaluator.embedding.registry import EmbeddingRegistry
+
+ game = consumer / "skills" / "game"
+ game.mkdir()
+ (game / "SKILL.md").write_bytes(f"---\n{frontmatter}\n---\nInstructions.\n".encode())
+ git = ["git", "-C", str(consumer), "-c", "user.name=CI tests", "-c", "[email protected]"]
+ subprocess.run([*git, "add", "."], check=True)
+ subprocess.run([*git, "commit", "-m", "game"], check=True, capture_output=True)
+ # Only alpha changed; the catalog still covers every discovered skill.
+ monkeypatch.setattr(run_tiers, "select_skills", lambda workspace, _env: ([workspace / "skills/alpha"], {}))
+
+ def no_embedding(*_args, **_kwargs):
+ raise AssertionError("A catalog skill with this frontmatter must be rejected before any embedding request")
+
+ monkeypatch.setattr(EmbeddingRegistry, "_build_entries", no_embedding)
+ commands = []
+
+ def execute(args, *, log, **_kwargs):
+ # Run the catalog helper in this process, with its stderr in the command log.
+ commands.append(args[3])
+ with log.open("w", encoding="utf-8") as stream, contextlib.redirect_stderr(stream):
+ return run_tiers.main(args[3:])
+
+ monkeypatch.setattr(run_tiers, "run_command", execute)
+ reports = tmp_path / "reports"
+ env = env_for(consumer, NVIDIA_API_KEY="test-provider-value-1234")
+ assert run_tiers.evaluate(2, consumer, reports, env) == 0
+ assert commands == ["catalog"]
+ tier = json.loads((reports / "tier2/ci-tier.json").read_text(encoding="utf-8"))["tiers"][0]
+ assert tier["status"] == "incomplete"
+ log = (reports / "tier2/catalog/diagnostics/command.log").read_text(encoding="utf-8")
+ return reports, env, tier, log
+
+
[email protected]("value", ["name: 2048", "name: yes", "name: [Draft]"])
+def test_skill_without_valid_frontmatter_stops_the_catalog_with_a_frontmatter_reason(
+ consumer, tmp_path, monkeypatch, value
+):
+ from ci.comment_summary import render_comment_summary
+
+ reports, env, tier, log = _evaluate_tier2_with_unselected_game(
+ consumer, tmp_path, monkeypatch, f"{value}\ndescription: Play the game."
+ )
+ reason = "A skill's invalid SKILL.md frontmatter stopped the comparison catalog; its diagnostics name the skill."
+ assert tier["reason"] == reason
+ assert "non-empty string name and description: skills/game" in log
+ assert str(consumer) not in log
+ # The feedback comment cuts each cell to a fixed length; the whole reason,
+ # with its pointer to the catalog diagnostics, must survive that cut.
+ assert run_tiers.aggregate(reports, (2,), env) == 0
+ manifest = json.loads((reports / "ci-results.json").read_text(encoding="utf-8"))
+ body = render_comment_summary(manifest, "https://github.com/example/consumer/actions/runs/123/artifacts/9")
+ alpha = next(line for line in body.splitlines() if line.startswith("| skills/alpha |"))
+ assert alpha.split(" | ")[2] == f"INCOMPLETE · issues unknown · {reason}"
+
+
[email protected]("encoding", ["cp1252", "utf-16"])
+def test_skill_md_that_is_not_utf8_stops_the_catalog_with_a_frontmatter_reason(
+ consumer, tmp_path, monkeypatch, encoding
+):
+ # A SKILL.md saved as Windows-1252 or UTF-16 is named with the skills whose
+ # frontmatter is invalid, and the reason points at the frontmatter.
+ zeta = consumer / "skills" / "zeta"
+ zeta.mkdir()
+ (zeta / "SKILL.md").write_bytes("---\nname: zeta\ndescription: Café menu.\n---\nInstructions.\n".encode(encoding))
+ _reports, _env, tier, log = _evaluate_tier2_with_unselected_game(
+ consumer, tmp_path, monkeypatch, "name: 2048\ndescription: Play the game."
+ )
+ assert tier["reason"] == (
+ "A skill's invalid SKILL.md frontmatter stopped the comparison catalog; its diagnostics name the skill."
+ )
+ assert log.strip() == (
+ "Each explicit skill requires a root SKILL.md, saved as UTF-8, whose frontmatter is a valid YAML "
+ "mapping with a non-empty string name and description: skills/game, skills/zeta"
+ )
+
+
+ "frontmatter",
+ [
+ pytest.param("name: " + "x" * 16_385 + "\ndescription: Play the game.", id="name-over-length-limit"),
+ pytest.param(
+ "name: game\ndescription: Play the game.\nextra: " + "[" * 1_500 + "safe" + "]" * 1_500,
+ id="frontmatter-over-structure-limit",
+ ),
+ ],
+)
+def test_frontmatter_over_a_limit_stops_the_catalog_with_the_provider_reason(
+ consumer, tmp_path, monkeypatch, frontmatter
+):
+ # Only the frontmatter rejection gets its own exit status and reason. Any
+ # other catalog failure, including a limit error, keeps the generic ones.
+ _reports, _env, tier, log = _evaluate_tier2_with_unselected_game(consumer, tmp_path, monkeypatch, frontmatter)
+ assert tier["reason"] == (
+ "The complete comparison catalog could not be built; inspect provider and embedding configuration."
+ )
+ assert log.strip() == "CI execution could not complete safely."
diff --git a/tests/embedding/test_registry.py b/tests/embedding/test_registry.py
index ec200f95..2a67ddae 100644
--- a/tests/embedding/test_registry.py
+++ b/tests/embedding/test_registry.py
@@ -104,6 +104,151 @@ def test_build_explicit_paths_rejects_outside_root_before_embedding(tmp_path: Pa
client.embed.assert_not_called()
+ "frontmatter",
+ [
+ "name: 2048\ndescription: Play the game.",
+ "name: yes\ndescription: Play the game.",
+ "name: game\ndescription: [Draft]",
+ "name: game\ndescription: Use when: debugging logs",
+ "- name: game",
+ ],
+)
+def test_build_explicit_paths_names_a_skill_without_valid_frontmatter(
+ tmp_path: Path, write_skill, frontmatter: str
+) -> None:
+ alpha = write_skill(tmp_path / "skills", "alpha", description="First fixture")
+ game = tmp_path / "skills" / "game"
+ game.mkdir()
+ (game / "SKILL.md").write_bytes(f"---\n{frontmatter}\n---\n\n# Game\n".encode())
+ client = _make_mock_client([[1.0, 0.0], [0.0, 1.0]])
+
+ with pytest.raises(ValueError) as caught:
+ EmbeddingRegistry(client).build_from_paths(tmp_path, [alpha, game])
+
+ assert str(caught.value) == (
+ "Each explicit skill requires a root SKILL.md, saved as UTF-8, whose frontmatter is a valid YAML "
+ "mapping with a non-empty string name and description: skills/game"
+ )
+ assert isinstance(caught.value, registry_module.InvalidExplicitSkillError)
+ assert str(tmp_path) not in str(caught.value)
+ client.embed.assert_not_called()
+ client.embed_single.assert_not_called()
+ client.embed_chunked.assert_not_called()
+
+
+def test_build_explicit_paths_names_every_skill_without_valid_frontmatter(tmp_path: Path, write_skill) -> None:
+ # One error names every such skill, so CI needs one run to find them all.
+ alpha = write_skill(tmp_path / "skills", "alpha", description="First fixture")
+ skills = [alpha]
+ for name, frontmatter in (
+ ("zeta", "name: zeta\ndescription: [Draft]"),
+ ("game", "name: 2048\ndescription: Play the game."),
+ ("quiz", "name: yes\ndescription: Answer questions."),
+ ):
+ skill = tmp_path / "skills" / name
+ skill.mkdir()
+ (skill / "SKILL.md").write_bytes(f"---\n{frontmatter}\n---\n\n# {name}\n".encode())
+ skills.append(skill)
+ client = _make_mock_client([[1.0, 0.0], [0.0, 1.0]])
+
+ with pytest.raises(registry_module.InvalidExplicitSkillError) as caught:
+ EmbeddingRegistry(client).build_from_paths(tmp_path, skills)
+
+ assert str(caught.value) == (
+ "Each explicit skill requires a root SKILL.md, saved as UTF-8, whose frontmatter is a valid YAML "
+ "mapping with a non-empty string name and description: skills/game, skills/quiz, skills/zeta"
+ )
+ client.embed.assert_not_called()
+ client.embed_single.assert_not_called()
+ client.embed_chunked.assert_not_called()
+
+
[email protected]("encoding", ["cp1252", "utf-16", "utf-16-le"])
+def test_build_explicit_paths_names_a_skill_whose_manifest_is_not_utf8(
+ tmp_path: Path, write_skill, encoding: str
+) -> None:
+ # A SKILL.md saved in another encoding, for example by a Windows editor or a
+ # PowerShell redirect, is named with the skills whose frontmatter is invalid.
+ alpha = write_skill(tmp_path / "skills", "alpha", description="First fixture")
+ skills = [alpha]
+ for name, manifest in (
+ ("zeta", "---\nname: zeta\ndescription: Café menu.\n---\n\n# Zeta\n".encode(encoding)),
+ ("game", b"---\nname: 2048\ndescription: Play the game.\n---\n\n# Game\n"),
+ ("beta", "---\nname: beta\ndescription: Café menu.\n---\n\n# Beta\n".encode(encoding)),
+ ):
+ skill = tmp_path / "skills" / name
+ skill.mkdir()
+ (skill / "SKILL.md").write_bytes(manifest)
+ skills.append(skill)
+ client = _make_mock_client([[1.0, 0.0], [0.0, 1.0]])
+
+ with pytest.raises(registry_module.InvalidExplicitSkillError) as caught:
+ EmbeddingRegistry(client).build_from_paths(tmp_path, skills)
+
+ assert str(caught.value) == (
+ "Each explicit skill requires a root SKILL.md, saved as UTF-8, whose frontmatter is a valid YAML "
+ "mapping with a non-empty string name and description: skills/beta, skills/game, skills/zeta"
+ )
+ assert str(tmp_path) not in str(caught.value)
+ client.embed.assert_not_called()
+ client.embed_single.assert_not_called()
+ client.embed_chunked.assert_not_called()
+
+
+def test_manifest_that_is_not_utf8_keeps_its_error_outside_explicit_paths(tmp_path: Path, write_skill) -> None:
+ # Only build_from_paths names such a skill; a directory scan still stops
+ # with the extractor's error.
+ write_skill(tmp_path / "skills", "alpha", description="First fixture")
+ zeta = tmp_path / "skills" / "zeta"
+ zeta.mkdir()
+ (zeta / "SKILL.md").write_bytes(b"---\nname: zeta\ndescription: Caf\xe9 menu.\n---\n\n# Zeta\n")
+ client = _make_mock_client([[1.0, 0.0], [0.0, 1.0]])
+
+ with pytest.raises(ValueError) as caught:
+ EmbeddingRegistry(client).build_from_directory(tmp_path / "skills", "skill")
+
+ assert str(caught.value) == "Manifest is not valid UTF-8: SKILL.md"
+ assert not isinstance(caught.value, registry_module.InvalidExplicitSkillError)
+ client.embed.assert_not_called()
+
+
+ ("frontmatter", "message"),
+ [
+ pytest.param(
+ "name: " + "x" * 16_385 + "\ndescription: Play the game.",
+ "Manifest frontmatter name exceeds the 16384-character limit: game/SKILL.md",
+ id="name-over-length-limit",
+ ),
+ pytest.param(
+ "name: game\ndescription: Play the game.\nextra: " + "[" * 1_500 + "safe" + "]" * 1_500,
+ "Manifest structured-data complexity limit exceeded: game/SKILL.md",
+ id="frontmatter-over-structure-limit",
+ ),
+ ],
+)
+def test_build_explicit_paths_keeps_frontmatter_limit_errors_generic(
+ tmp_path: Path, write_skill, frontmatter: str, message: str
+) -> None:
+ # Callers treat InvalidExplicitSkillError as a frontmatter shape problem;
+ # a limit error stays a plain ValueError with the extractor's message.
+ alpha = write_skill(tmp_path / "skills", "alpha", description="First fixture")
+ game = tmp_path / "skills" / "game"
+ game.mkdir()
+ (game / "SKILL.md").write_bytes(f"---\n{frontmatter}\n---\n\n# Game\n".encode())
+ client = _make_mock_client([[1.0, 0.0], [0.0, 1.0]])
+
+ with pytest.raises(ValueError) as caught:
+ EmbeddingRegistry(client).build_from_paths(tmp_path, [alpha, game])
+
+ assert str(caught.value) == message
+ assert type(caught.value) is ValueError
+ client.embed.assert_not_called()
+ client.embed_single.assert_not_called()
+ client.embed_chunked.assert_not_called()
+
+
class TestBuildFromDirectory:
def test_empty_collection_debug_logs_use_label_without_host_path(self, tmp_path: Path, monkeypatch) -> None:
collection = tmp_path / "nested" / "external-skills"
diff --git a/tests/test_oss_packaging.py b/tests/test_oss_packaging.py
index f3bdac24..eb10bd72 100644
--- a/tests/test_oss_packaging.py
+++ b/tests/test_oss_packaging.py
@@ -815,6 +815,29 @@ def test_public_docs_show_tier_two_collection_and_catalog_workflows() -> None:
assert "sends each discovered `SKILL.md` in full" in public_docs
assert "Only candidate clusters found by the embedding stage are" in public_docs
assert "sent to the configured chat LLM for classification" in public_docs
+ # One unchanged skill with invalid frontmatter stops the CI comparison catalog.
+ ci_catalog = "`name: 2048`, `name: yes`, or `description: [Draft]`, stops that catalog"
+ assert ci_catalog in " ".join(guide.split())
+ # CI builds that catalog only from more than one skill; the release note says so too.
+ condition = "When the configured skill roots hold more than one skill, CI"
+ assert condition in " ".join(guide.split())
+ changelog = (REPO_ROOT / "CHANGELOG.md").read_text(encoding="utf-8")
+ lead = "- All-tier CI now says when a skill's frontmatter stops the Tier 2 comparison catalog"
+ entries = [" ".join(entry.split()) for entry in re.split(r"\n(?=- )|\n\s*\n", changelog)]
+ catalog_entries = [entry for entry in entries if entry.startswith(lead)]
+ assert len(catalog_entries) == 1
+ assert f"{condition} builds the catalog from all of them" in catalog_entries[0]
+ assert "With one skill there is no catalog, and such frontmatter does not stop Tier 2." in catalog_entries[0]
+ # The edge-case table states the same condition.
+ assert "In [automatic CI](#tier-2-in-automatic-ci) with more than one skill, stops the comparison catalog" in guide
+ # A limit error stops at its own skill, so CI then names no invalid skill.
+ unnamed = "no skill, not even one whose frontmatter is invalid"
+ assert f"the diagnostics then name {unnamed}" in " ".join(guide.split())
+ assert f"the log then names {unnamed}." in catalog_entries[0]
+ # A SKILL.md that is not UTF-8 is named with them, under the same reason.
+ encoding = "that is not valid UTF-8, such as one saved as UTF-16 or as Windows-1252 with an accented letter"
+ assert f"{encoding}, stops the catalog with the same reason and is named the same way" in " ".join(guide.split())
+ assert f"{encoding}, stops the catalog with the same reason and is named the same way" in catalog_entries[0]
assert "NVI" + "DIA" + "_INFERENCE_KEY" not in public_docs # oss-boundary-anchor: docs-retired-credential