Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(embedding): reject non-progressing chunk windows (#142)

Deepak Jain (DJ)cdb4dbbd8f0e
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0412e0a8..6be4ceed 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -27,6 +27,9 @@ All notable changes to SkillEvaluator are documented in this file.
- Fully covered documentation-only skills no longer fail security validation
solely because non-applicable SkillSpector analyzers report a partial status
([#137](https://github.com/NVIDIA/SkillEvaluator/issues/137)).
+- Embedding chunking now rejects zero-sized or non-progressing windows before
+ entering the splitter or contacting the embedding provider
+ ([#139](https://github.com/NVIDIA/SkillEvaluator/issues/139)).
- Scoped network exfiltration command flag patterns in security checks, enforcing command-position anchoring, quote-aware argument segmentation, explicit HTTP method flags, and case-sensitive `-F`/`-d`/`-T` flags to prevent false-positive flags on safe URLs, packages, or download scripts while reliably detecting quoted secrets and subshell wrappers.
- Malformed, non-UTF-8, or unreadable bundled and custom policy files now
produce path-specific CLI errors instead of leaking raw parser or I/O errors
diff --git a/src/skillevaluator/embedding/client.py b/src/skillevaluator/embedding/client.py
index c5d4805c..776b230c 100644
--- a/src/skillevaluator/embedding/client.py
+++ b/src/skillevaluator/embedding/client.py
@@ -250,6 +250,17 @@ def _split_into_chunks(
fixed-size overlapping windows.
3. Guarantee at least one chunk is returned.
"""
+ if type(chunk_size) is not int:
+ raise SimilarityConfigError("Embedding chunk size must be an integer.")
+ if type(overlap) is not int:
+ raise SimilarityConfigError("Embedding chunk overlap must be an integer.")
+ if chunk_size <= 0:
+ raise SimilarityConfigError("Embedding chunk size must be greater than zero.")
+ if overlap < 0:
+ raise SimilarityConfigError("Embedding chunk overlap must not be negative.")
+ if overlap >= chunk_size:
+ raise SimilarityConfigError("Embedding chunk overlap must be smaller than the chunk size.")
+
max_chars = chunk_size * _CHARS_PER_TOKEN
overlap_chars = overlap * _CHARS_PER_TOKEN
diff --git a/tests/embedding/test_client.py b/tests/embedding/test_client.py
index da892b33..f2fa49ca 100644
--- a/tests/embedding/test_client.py
+++ b/tests/embedding/test_client.py
@@ -210,6 +210,51 @@ class TestEmbedChunked:
assert result == pytest.approx([0.5, 0.5])
+ @pytest.mark.parametrize("overlap", [10, 11])
+ def test_embed_chunked_rejects_non_progressing_windows_before_provider_call(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ overlap: int,
+ ) -> None:
+ monkeypatch.setenv("NVIDIA_API_KEY", "test-key")
+ client = EmbeddingClient()
+
+ with (
+ patch.object(client, "embed") as mock_embed,
+ pytest.raises(SimilarityConfigError, match="overlap must be smaller"),
+ ):
+ client.embed_chunked("x" * 100, chunk_size=10, overlap=overlap)
+
+ mock_embed.assert_not_called()
+
+ @pytest.mark.parametrize(
+ ("chunk_size", "overlap"),
+ [
+ (True, 0),
+ (10.0, 0),
+ (10, True),
+ (10, 1.5),
+ (10, math.nan),
+ ],
+ ids=["bool-size", "float-size", "bool-overlap", "float-overlap", "nan-overlap"],
+ )
+ def test_embed_chunked_rejects_non_integer_windows_before_provider_call(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ chunk_size: object,
+ overlap: object,
+ ) -> None:
+ monkeypatch.setenv("NVIDIA_API_KEY", "test-key")
+ client = EmbeddingClient()
+
+ with (
+ patch.object(client, "embed") as mock_embed,
+ pytest.raises(SimilarityConfigError, match="must be an integer"),
+ ):
+ client.embed_chunked("x" * 100, chunk_size=chunk_size, overlap=overlap) # type: ignore[arg-type]
+
+ mock_embed.assert_not_called()
+
class TestSplitIntoChunks:
def test_short_text_single_chunk(self) -> None:
@@ -236,6 +281,19 @@ class TestSplitIntoChunks:
chunks = _split_into_chunks("", chunk_size=100, overlap=10)
assert chunks == [""]
+ @pytest.mark.parametrize(
+ ("chunk_size", "overlap", "message"),
+ [
+ (0, 0, "chunk size must be greater than zero"),
+ (10, -1, "chunk overlap must not be negative"),
+ (10, 10, "overlap must be smaller than the chunk size"),
+ (10, 11, "overlap must be smaller than the chunk size"),
+ ],
+ )
+ def test_rejects_invalid_chunk_windows(self, chunk_size: int, overlap: int, message: str) -> None:
+ with pytest.raises(SimilarityConfigError, match=message):
+ _split_into_chunks("x" * 100, chunk_size=chunk_size, overlap=overlap)
+
def test_heading_split_preserves_heading_with_content(self) -> None:
text = "# Title\nSome intro\n## Part 1\nDetails"
sections = _split_by_headings(text)