Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(tier2): keep catalog formatting within the byte budget (#628)

Christopher Kevin5bc1fc861ee7
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3ccdd10f..3adb721b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -296,6 +296,9 @@ All notable changes to SkillEvaluator are documented in this file.
- The Tier 2 scale guide names the fixed 8 MiB selected-manifest input budget
and 1 MiB per-manifest limit, including in description-only embedding mode.
+- Saved Tier 2 catalogs use compact JSON so indentation no longer rejects dense
+ indexes that fit the unchanged 32 MiB file limit. Vector values and catalog
+ compatibility are unchanged.
- Arms fully denied by a run budget are reported as not run instead of failed
empty Harbor jobs. Budget-partial runs remain incomplete without an execution
error when all admitted attempts succeed, including stop-on-pass runs. Reused
diff --git a/docs/tier2-deduplication.mdx b/docs/tier2-deduplication.mdx
index 6bd7e664..22516f4f 100644
--- a/docs/tier2-deduplication.mdx
+++ b/docs/tier2-deduplication.mdx
@@ -435,7 +435,8 @@ existing set — save the collection once and query candidates against it.
```
`--save-catalog` builds a versioned JSON catalog from the supplied
- collection.
+ collection. Catalogs use compact JSON so formatting whitespace does not
+ consume the 32 MiB file budget; vector values and the schema are unchanged.
</Step>
<Step title="Screen a candidate">
```bash title="Compare one candidate against the catalog"
diff --git a/src/skillevaluator/embedding/registry.py b/src/skillevaluator/embedding/registry.py
index 1efeb389..30a35f3c 100644
--- a/src/skillevaluator/embedding/registry.py
+++ b/src/skillevaluator/embedding/registry.py
@@ -606,7 +606,7 @@ class EmbeddingRegistry:
"created_at": datetime.now(tz=UTC).isoformat(),
"entries": entries,
}
- serialized = json.dumps(data, indent=2, allow_nan=False) + "\n"
+ serialized = json.dumps(data, separators=(",", ":"), allow_nan=False) + "\n"
if len(serialized.encode("utf-8")) > MAX_CATALOG_BYTES:
raise ValueError(f"Catalog size limit exceeded ({MAX_CATALOG_BYTES} bytes)")
_write_catalog_atomically(catalog_path, serialized.encode("utf-8"))
diff --git a/tests/embedding/test_catalog.py b/tests/embedding/test_catalog.py
index 32a12298..68be70f1 100644
--- a/tests/embedding/test_catalog.py
+++ b/tests/embedding/test_catalog.py
@@ -74,6 +74,37 @@ def _write_catalog(path: Path, data: dict | None = None) -> Path:
class TestCatalogPersistence:
+ def test_dense_catalog_uses_compact_json_with_exact_byte_limit(self, tmp_path: Path, monkeypatch) -> None:
+ data = _catalog_data()
+ data["vector_dimension"] = 128
+ data["entries"][0]["embedding"] = [(index + 1) / 129 for index in range(128)]
+ registry = EmbeddingRegistry(_client())
+ registry.load_catalog(_write_catalog(tmp_path / "input.json", data))
+ clock = MagicMock(wraps=registry_module.datetime)
+ clock.now.return_value = registry_module.datetime.fromisoformat(data["created_at"])
+ monkeypatch.setattr(registry_module, "datetime", clock)
+ compact_bytes = len(json.dumps(data, separators=(",", ":")).encode("utf-8"))
+ indented_bytes = len(json.dumps(data, indent=2).encode("utf-8"))
+ monkeypatch.setattr(registry_module, "MAX_CATALOG_BYTES", (compact_bytes + indented_bytes) // 2)
+
+ output = tmp_path / "output.json"
+ registry.save_catalog(output)
+ encoded = output.read_bytes()
+ assert len(encoded) < indented_bytes
+ assert json.loads(encoded)["entries"] == data["entries"]
+
+ monkeypatch.setattr(registry_module, "MAX_CATALOG_BYTES", len(encoded))
+ restored = EmbeddingRegistry(_client())
+ restored.load_catalog(output)
+ assert restored._entries == registry._entries
+ registry.save_catalog(output)
+
+ previous = output.read_bytes()
+ monkeypatch.setattr(registry_module, "MAX_CATALOG_BYTES", len(previous) - 1)
+ with pytest.raises(ValueError, match="Catalog size limit exceeded"):
+ registry.save_catalog(output)
+ assert output.read_bytes() == previous
+
@pytest.mark.parametrize("full_body", [False, True])
@pytest.mark.parametrize(
("style", "description"),