Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(tier2): retain trailing shell comments in deduplication (#357)

YMuskrate76fcc98fad3
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1a957183..a32cf9c7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -145,6 +145,10 @@ All notable changes to SkillEvaluator are documented in this file.
### Fixed
+- Tier 2 includes qualifying shell comment blocks at end of file in context
+ deduplication. Previously those blocks disappeared unless a blank line or
+ code followed them, which could hide repeated guidance across shell files.
+
- All-tier CI retains Tier 2's native reports as advisory while preserving
failed and incomplete evaluation outcomes. GitHub `ci status` validates the
pinned evaluation and feedback configuration before reporting readiness,
diff --git a/src/skillevaluator/deduplication/utils/chunker.py b/src/skillevaluator/deduplication/utils/chunker.py
index cbdfe593..b2b64e3d 100644
--- a/src/skillevaluator/deduplication/utils/chunker.py
+++ b/src/skillevaluator/deduplication/utils/chunker.py
@@ -15,6 +15,7 @@ import ast
import logging
import re
from dataclasses import dataclass, field, replace
+from itertools import chain
from typing import TYPE_CHECKING
from markdown_it import MarkdownIt
@@ -310,7 +311,8 @@ def chunk_shell(
comment_start = None
comment_lines: list[str] = []
- for i, line in enumerate(lines):
+ # A final empty sentinel flushes a trailing comment block at end of file.
+ for i, line in enumerate(chain(lines, ("",))):
stripped = line.strip()
if stripped.startswith("#") and not stripped.startswith("#!"):
if comment_start is None:
diff --git a/tests/deduplication/intra_skill/test_intra_skill_validator.py b/tests/deduplication/intra_skill/test_intra_skill_validator.py
index d3169107..45416730 100644
--- a/tests/deduplication/intra_skill/test_intra_skill_validator.py
+++ b/tests/deduplication/intra_skill/test_intra_skill_validator.py
@@ -106,6 +106,34 @@ class TestEffectiveChunkLimit:
class TestIntraSkillValidatorValidate:
+ @patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.analyze_cluster")
+ @patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.LLMClient")
+ @patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.EmbeddingClient")
+ def test_duplicate_shell_comments_at_eof_are_compared(
+ self, mock_embed, _mock_llm, mock_analyze, tmp_path: Path
+ ) -> None:
+ comments = (
+ "# Document the required setup before invoking this helper script.\n"
+ "# Use the configured workspace for every generated output file.\n"
+ "# Keep the source input unchanged while the helper processes it.\n"
+ )
+ (tmp_path / "first.sh").write_text("#!/bin/sh\necho ready\n" + comments, encoding="utf-8")
+ (tmp_path / "second.sh").write_text("#!/bin/sh\necho ready\n" + comments, encoding="utf-8")
+ mock_embed.return_value.embed.side_effect = _unit_embeddings(2)
+ mock_analyze.return_value = LLMVerdict(
+ verdict="DUPLICATE", confidence=0.9, reasoning="Repeated instructions", suggestion="Consolidate comments"
+ )
+
+ result = IntraSkillValidator().validate(tmp_path)
+
+ mock_embed.return_value.embed.assert_called_once()
+ assert len(mock_embed.return_value.embed.call_args.args[0]) == 2
+ mock_analyze.assert_called_once()
+ assert result.passed is False
+ assert len(result.findings) == 1
+ assert "first.sh" in result.findings[0].message
+ assert "second.sh" in result.findings[0].message
+
def test_collection_log_uses_target_label_without_host_path(self, tmp_path: Path, caplog) -> None:
skill_dir = tmp_path / "nested" / "external-skill"
skill_dir.mkdir(parents=True)
diff --git a/tests/deduplication/utils/test_chunker.py b/tests/deduplication/utils/test_chunker.py
index 568ef64d..0706f156 100644
--- a/tests/deduplication/utils/test_chunker.py
+++ b/tests/deduplication/utils/test_chunker.py
@@ -5,6 +5,8 @@
from __future__ import annotations
+import pytest
+
from skillevaluator.deduplication.utils.chunker import (
ContentChunk,
chunk_file,
@@ -191,6 +193,38 @@ class TestChunkPython:
class TestChunkShell:
+ @pytest.mark.parametrize("prefix", ["", "#!/bin/sh\necho ready\n"])
+ @pytest.mark.parametrize("ending", ["", "\n", "\n\n"])
+ def test_comment_block_at_eof(self, prefix: str, ending: str) -> None:
+ comments = [
+ "Document the required setup before invoking this helper script.",
+ "Use the configured workspace for every generated output file.",
+ "Keep the source input unchanged while the helper processes it.",
+ ]
+ content = prefix + "\n".join(f"# {line}" for line in comments) + ending
+
+ chunks = chunk_shell(_cf(content, ".sh", "run.sh"), min_chars=80)
+
+ assert len(chunks) == 1
+ assert chunks[0].heading == "(comment)"
+ assert chunks[0].text == "\n".join(comments)
+ first_line = len(prefix.splitlines()) + 1
+ assert (chunks[0].start_line, chunks[0].end_line) == (first_line, first_line + 2)
+
+ @pytest.mark.parametrize(
+ "content",
+ [
+ "",
+ "#!/bin/sh\n",
+ "# Short\n# Two lines",
+ "# This first long comment line explains the helper setup in detail.\n"
+ "# This second long comment line explains the workspace output rules.",
+ "# A\n# B\n# C",
+ ],
+ )
+ def test_eof_comments_keep_existing_filters(self, content: str) -> None:
+ assert chunk_shell(_cf(content, ".sh", "run.sh"), min_chars=80) == []
+
def test_function_body(self) -> None:
body = "setup() {\n" + " echo step\n" * 15 + "}\n"
chunks = chunk_shell(_cf(body, ".sh", "run.sh"), min_chars=80)