Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(embedding): reject non-progressing chunk windows (#142)
Deepak Jain (DJ)cdb4dbbd8f0e
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 0412e0a8..6be4ceed 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -27,6 +27,9 @@ All notable changes to SkillEvaluator are documented in this file.- Fully covered documentation-only skills no longer fail security validationsolely because non-applicable SkillSpector analyzers report a partial status([#137](https://github.com/NVIDIA/SkillEvaluator/issues/137)).+- Embedding chunking now rejects zero-sized or non-progressing windows before+ entering the splitter or contacting the embedding provider+ ([#139](https://github.com/NVIDIA/SkillEvaluator/issues/139)).- Scoped network exfiltration command flag patterns in security checks, enforcing command-position anchoring, quote-aware argument segmentation, explicit HTTP method flags, and case-sensitive `-F`/`-d`/`-T` flags to prevent false-positive flags on safe URLs, packages, or download scripts while reliably detecting quoted secrets and subshell wrappers.- Malformed, non-UTF-8, or unreadable bundled and custom policy files nowproduce path-specific CLI errors instead of leaking raw parser or I/O errorsdiff --git a/src/skillevaluator/embedding/client.py b/src/skillevaluator/embedding/client.pyindex c5d4805c..776b230c 100644--- a/src/skillevaluator/embedding/client.py+++ b/src/skillevaluator/embedding/client.py@@ -250,6 +250,17 @@ def _split_into_chunks(fixed-size overlapping windows.3. Guarantee at least one chunk is returned."""+ if type(chunk_size) is not int:+ raise SimilarityConfigError("Embedding chunk size must be an integer.")+ if type(overlap) is not int:+ raise SimilarityConfigError("Embedding chunk overlap must be an integer.")+ if chunk_size <= 0:+ raise SimilarityConfigError("Embedding chunk size must be greater than zero.")+ if overlap < 0:+ raise SimilarityConfigError("Embedding chunk overlap must not be negative.")+ if overlap >= chunk_size:+ raise SimilarityConfigError("Embedding chunk overlap must be smaller than the chunk size.")+max_chars = chunk_size * _CHARS_PER_TOKENoverlap_chars = overlap * _CHARS_PER_TOKENdiff --git a/tests/embedding/test_client.py b/tests/embedding/test_client.pyindex da892b33..f2fa49ca 100644--- a/tests/embedding/test_client.py+++ b/tests/embedding/test_client.py@@ -210,6 +210,51 @@ class TestEmbedChunked:assert result == pytest.approx([0.5, 0.5])+ @pytest.mark.parametrize("overlap", [10, 11])+ def test_embed_chunked_rejects_non_progressing_windows_before_provider_call(+ self,+ monkeypatch: pytest.MonkeyPatch,+ overlap: int,+ ) -> None:+ monkeypatch.setenv("NVIDIA_API_KEY", "test-key")+ client = EmbeddingClient()++ with (+ patch.object(client, "embed") as mock_embed,+ pytest.raises(SimilarityConfigError, match="overlap must be smaller"),+ ):+ client.embed_chunked("x" * 100, chunk_size=10, overlap=overlap)++ mock_embed.assert_not_called()++ @pytest.mark.parametrize(+ ("chunk_size", "overlap"),+ [+ (True, 0),+ (10.0, 0),+ (10, True),+ (10, 1.5),+ (10, math.nan),+ ],+ ids=["bool-size", "float-size", "bool-overlap", "float-overlap", "nan-overlap"],+ )+ def test_embed_chunked_rejects_non_integer_windows_before_provider_call(+ self,+ monkeypatch: pytest.MonkeyPatch,+ chunk_size: object,+ overlap: object,+ ) -> None:+ monkeypatch.setenv("NVIDIA_API_KEY", "test-key")+ client = EmbeddingClient()++ with (+ patch.object(client, "embed") as mock_embed,+ pytest.raises(SimilarityConfigError, match="must be an integer"),+ ):+ client.embed_chunked("x" * 100, chunk_size=chunk_size, overlap=overlap) # type: ignore[arg-type]++ mock_embed.assert_not_called()+class TestSplitIntoChunks:def test_short_text_single_chunk(self) -> None:@@ -236,6 +281,19 @@ class TestSplitIntoChunks:chunks = _split_into_chunks("", chunk_size=100, overlap=10)assert chunks == [""]+ @pytest.mark.parametrize(+ ("chunk_size", "overlap", "message"),+ [+ (0, 0, "chunk size must be greater than zero"),+ (10, -1, "chunk overlap must not be negative"),+ (10, 10, "overlap must be smaller than the chunk size"),+ (10, 11, "overlap must be smaller than the chunk size"),+ ],+ )+ def test_rejects_invalid_chunk_windows(self, chunk_size: int, overlap: int, message: str) -> None:+ with pytest.raises(SimilarityConfigError, match=message):+ _split_into_chunks("x" * 100, chunk_size=chunk_size, overlap=overlap)+def test_heading_split_preserves_heading_with_content(self) -> None:text = "# Title\nSome intro\n## Part 1\nDetails"sections = _split_by_headings(text)