Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(embedding): preserve multiline descriptions in catalogs

Rio Yu5298d4d861f0
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7b35b49e..7e669fdf 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,9 @@ All notable changes to SkillEvaluator are documented in this file.
### Fixed
+- Preserved multiline skill descriptions when saving and loading Tier 2
+ embedding catalogs. Descriptions can contain line breaks and tabs, while
+ catalog identifiers and other control characters remain strictly validated.
- Brought the published documentation in line with 0.5.0. It now covers how
standard grading builds judge evidence and rebuilds trajectories from
OpenCode and Codex logs, structured judge output and retries, the evidence
diff --git a/src/skillevaluator/embedding/registry.py b/src/skillevaluator/embedding/registry.py
index 59edf926..ad23e4e5 100644
--- a/src/skillevaluator/embedding/registry.py
+++ b/src/skillevaluator/embedding/registry.py
@@ -259,7 +259,9 @@ class EmbeddingRegistry:
batch = texts[start : start + batch_size]
vectors = [self._client.embed_chunked(batch[0])] if self._full_body else self._client.embed(batch)
if len(vectors) != len(batch):
- raise ValueError(f"Embedding provider returned {len(vectors)} vectors for {len(batch)} entries in batch")
+ raise ValueError(
+ f"Embedding provider returned {len(vectors)} vectors for {len(batch)} entries in batch"
+ )
for vector in vectors:
vector_dimension = _validate_vector(vector, vector_dimension)
validated_vectors.append(vector)
@@ -659,13 +661,16 @@ def _validate_catalog_identity(entry_id: object, path: object, content_type: obj
def _validate_text_fields(entry_id: str, name: object, description: object) -> None:
_validate_catalog_string(name, f"Catalog entry '{entry_id}' name")
- _validate_catalog_string(description, f"Catalog entry '{entry_id}' description")
+ _validate_catalog_string(description, f"Catalog entry '{entry_id}' description", allow_text_whitespace=True)
-def _validate_catalog_string(value: object, label: str) -> str:
+def _validate_catalog_string(value: object, label: str, *, allow_text_whitespace: bool = False) -> str:
if not isinstance(value, str) or not value or len(value) > MAX_CATALOG_TEXT_LENGTH:
raise ValueError(f"{label} must be a non-empty bounded string")
- if any(unicodedata.category(character) in {"Cc", "Cf", "Cs"} for character in value):
+ if any(
+ unicodedata.category(character) in {"Cc", "Cf", "Cs"} and not (allow_text_whitespace and character in "\t\n\r")
+ for character in value
+ ):
raise ValueError(f"{label} contains unsafe control or surrogate characters")
return value
diff --git a/tests/embedding/test_catalog.py b/tests/embedding/test_catalog.py
index 85c82a4f..a6ac06bf 100644
--- a/tests/embedding/test_catalog.py
+++ b/tests/embedding/test_catalog.py
@@ -72,6 +72,76 @@ def _write_catalog(path: Path, data: dict | None = None) -> Path:
class TestCatalogPersistence:
+ @pytest.mark.parametrize("full_body", [False, True])
+ @pytest.mark.parametrize(
+ ("style", "description"),
+ [
+ (">", "Summarize reports and extract findings.\n"),
+ ("|", "Summarize reports\nand extract findings.\n"),
+ (">-", "Summarize reports and extract findings."),
+ ],
+ )
+ def test_yaml_multiline_description_roundtrip(
+ self, tmp_path: Path, full_body: bool, style: str, description: str
+ ) -> None:
+ skill_dir = tmp_path / "skills" / "summarize-reports"
+ skill_dir.mkdir(parents=True)
+ manifest = (
+ f"---\nname: summarize-reports\ndescription: {style}\n"
+ " Summarize reports\n and extract findings.\nlicense: MIT\n---\n\n# Reports\n"
+ )
+ (skill_dir / "SKILL.md").write_text(manifest, encoding="utf-8", newline="")
+ client = _client([[1.0, 0.0]])
+ registry = EmbeddingRegistry(client, full_body=full_body)
+ assert registry.build_from_directory(tmp_path / "skills", "skill") == 1
+ original = registry._entries["skill:summarize-reports"]
+ assert original.description == description
+ if full_body:
+ client.embed_chunked.assert_called_once_with(manifest)
+ else:
+ client.embed.assert_called_once_with([f"summarize-reports: {description}"])
+
+ catalog_path = tmp_path / "catalog.json"
+ registry.save_catalog(catalog_path)
+ restored = EmbeddingRegistry(client, full_body=full_body)
+ restored.load_catalog(catalog_path)
+
+ assert restored._entries["skill:summarize-reports"] == original
+ matches = restored.query("Summarize reports", threshold=0.9)
+ assert len(matches) == 1
+ assert matches[0].entry_b == "summarize-reports"
+ assert matches[0].score == pytest.approx(1.0)
+
+ @pytest.mark.parametrize("whitespace", ["\n", "\r", "\t"])
+ def test_description_whitespace_survives_load_and_save(self, tmp_path: Path, whitespace: str) -> None:
+ data = _catalog_data()
+ description = f"First{whitespace}catalog skill"
+ data["entries"][0]["description"] = description
+ registry = EmbeddingRegistry(_client())
+ registry.load_catalog(_write_catalog(tmp_path / "input.json", data))
+ output = tmp_path / "output.json"
+ registry.save_catalog(output)
+ assert json.loads(output.read_text(encoding="utf-8"))["entries"][0]["description"] == description
+
+ @pytest.mark.parametrize(
+ ("field", "character"),
+ [("name", char) for char in "\n\r\t"] + [("description", char) for char in "\x00\x0b\x0c\x1b\x85\u202e\ud800"],
+ )
+ def test_catalog_still_rejects_unsafe_text_on_load_and_save(
+ self, tmp_path: Path, field: str, character: str
+ ) -> None:
+ data = _catalog_data()
+ data["entries"][0][field] = f"First{character}skill"
+ with pytest.raises(ValueError, match="unsafe control or surrogate"):
+ EmbeddingRegistry(_client()).load_catalog(_write_catalog(tmp_path / "unsafe.json", data))
+
+ registry = EmbeddingRegistry(_client())
+ registry.load_catalog(_write_catalog(tmp_path / "safe.json"))
+ setattr(registry._entries["skill:team-a"], field, data["entries"][0][field])
+ with pytest.raises(ValueError, match="unsafe control or surrogate"):
+ registry.save_catalog(tmp_path / "output.json")
+ assert not (tmp_path / "output.json").exists()
+
def test_save_catalog_is_versioned_relative_and_preserves_duplicate_names(self, tmp_path: Path) -> None:
for directory, description in (("team-a", "First"), ("team-b", "Second")):
skill_dir = tmp_path / "skills" / directory