Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(tier2): chunk shell scripts in one pass instead of once per function header (#423)

Narendran Raghavancfeb58366d6d
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1ee5b02b..ed2b29e0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -418,6 +418,15 @@ All notable changes to SkillEvaluator are documented in this file.
characters: a text that saves only a few characters stays inline when sharing
it would push another text past block 9 and cost more than it saves.
+- Tier 2 deduplication no longer rescans a shell script once per function
+ header. In 0.8.0 each `name() {` header without a closing brace walked the
+ rest of the script, and line numbers were counted from the start of the file
+ for every function, so chunking grew with the square of the script's size: a
+ 53 KiB script of such headers took about 20 seconds, and a 1 MiB one could
+ take hours. Braces and line breaks are now indexed in one pass. Python files
+ are also split into lines once instead of once per function or class. Chunk
+ text and line ranges are unchanged.
+
- Tier 3 **Measured Agent Resources** no longer count idle or user time as LLM
seconds when timing is inferred from step timestamps and an agent step with
tool calls is followed by a user or system step. The whole gap up to that
diff --git a/docs/tier2-deduplication.mdx b/docs/tier2-deduplication.mdx
index e8606879..9003d3d4 100644
--- a/docs/tier2-deduplication.mdx
+++ b/docs/tier2-deduplication.mdx
@@ -138,6 +138,16 @@ empty, or non-mapping frontmatter remains part of the analyzed text. Every
retained chunk keeps its source file and line range, so findings point you to
the exact overlapping sections.
+A shell function starts at a `name() {` or `function name() {` header at the
+start of a line and runs to the matching closing brace. The name is letters,
+digits, and underscores, and the `{` may sit on the next line. Indented
+headers, `function name {` without parentheses, and names with hyphens are not
+chunked as functions. Every `{` and `}` inside a function counts, including
+those in strings and comments. A header without a matching closing brace
+yields no function chunk. Braces and line numbers are indexed in one pass over
+the script, so a script with many functions or unclosed headers is not
+rescanned once per header.
+
Each embedding input is capped at 3,000 characters, including long table rows,
Python docstrings, and shell functions. Oversized Markdown sections are packed
paragraph by paragraph up to the cap, and paragraphs under 80 characters stay
diff --git a/src/skillevaluator/deduplication/utils/chunker.py b/src/skillevaluator/deduplication/utils/chunker.py
index b2b64e3d..a0423a18 100644
--- a/src/skillevaluator/deduplication/utils/chunker.py
+++ b/src/skillevaluator/deduplication/utils/chunker.py
@@ -12,6 +12,7 @@ Three strategies:
from __future__ import annotations
import ast
+import bisect
import logging
import re
from dataclasses import dataclass, field, replace
@@ -258,6 +259,7 @@ def chunk_python(
return []
chunks: list[ContentChunk] = []
+ source_lines = collected.content.splitlines()
module_doc = ast.get_docstring(tree)
if module_doc and len(module_doc) >= min_chars:
@@ -282,7 +284,7 @@ def chunk_python(
else:
heading = f"{name}()"
- sig_line = collected.content.splitlines()[node.lineno - 1]
+ sig_line = source_lines[node.lineno - 1]
text = f"{sig_line}\n{doc}" if doc else sig_line
if len(text) >= min_chars:
@@ -336,22 +338,25 @@ def chunk_shell(
comment_lines = []
full_text = collected.content
+ brace_stack: list[int] = []
+ brace_ends: dict[int, int] = {}
+ newline_positions: list[int] = []
+ for position, char in enumerate(full_text):
+ if char == "\n":
+ newline_positions.append(position)
+ elif char == "{":
+ brace_stack.append(position)
+ elif char == "}" and brace_stack:
+ brace_ends[brace_stack.pop()] = position + 1
+
for match in SHELL_FUNC_PATTERN.finditer(full_text):
func_name = match.group(1)
brace_start = match.end() - 1
- depth = 1
- pos = brace_start + 1
- while pos < len(full_text) and depth > 0:
- if full_text[pos] == "{":
- depth += 1
- elif full_text[pos] == "}":
- depth -= 1
- pos += 1
-
- if depth == 0:
+ pos = brace_ends.get(brace_start)
+ if pos is not None:
func_text = full_text[match.start() : pos]
- start_line = full_text[: match.start()].count("\n") + 1
- end_line = full_text[:pos].count("\n") + 1
+ start_line = bisect.bisect_left(newline_positions, match.start()) + 1
+ end_line = bisect.bisect_left(newline_positions, pos) + 1
if len(func_text) >= min_chars:
chunks.append(
ContentChunk(
diff --git a/tests/deduplication/utils/test_chunker.py b/tests/deduplication/utils/test_chunker.py
index 0706f156..4e196b34 100644
--- a/tests/deduplication/utils/test_chunker.py
+++ b/tests/deduplication/utils/test_chunker.py
@@ -37,6 +37,62 @@ def _cf(
)
+class _ReadCountingText(str):
+ """A str that adds up the characters read through some of its methods.
+
+ Indexing and slicing add the length of the result, iteration adds one per
+ character, ``find`` and ``index`` add the span searched, and ``count`` and
+ ``splitlines`` add the span scanned. Nothing else is counted:
+
+ - reads through ``re``, which reads the string's buffer directly;
+ - other ``str`` methods, such as ``split``, ``strip`` or ``startswith``;
+ - any read of a plain ``str`` made from it, for example with ``str()``,
+ ``replace()`` or ``"".join()``.
+
+ A quadratic rewrite that reads only in those ways passes the read-count
+ tests. They pin the old per-character and per-function walks, not every
+ possible quadratic scan.
+ """
+
+ __slots__ = ()
+ read = 0
+
+ def _span(self, start, end):
+ first, last, _ = slice(start, end).indices(len(self))
+ return first, last
+
+ def __getitem__(self, key):
+ result = super().__getitem__(key)
+ type(self).read += len(result)
+ return result
+
+ def __iter__(self):
+ for char in super().__iter__():
+ type(self).read += 1
+ yield char
+
+ def find(self, sub, start=None, end=None):
+ found = super().find(sub, start, end)
+ first, last = self._span(start, end)
+ type(self).read += max((found + len(sub) if found >= 0 else last) - first, 0)
+ return found
+
+ def index(self, sub, start=None, end=None):
+ found = self.find(sub, start, end)
+ if found < 0:
+ raise ValueError("substring not found")
+ return found
+
+ def count(self, sub, start=None, end=None):
+ first, last = self._span(start, end)
+ type(self).read += max(last - first, 0)
+ return super().count(sub, start, end)
+
+ def splitlines(self, keepends=False):
+ type(self).read += len(self)
+ return super().splitlines(keepends)
+
+
class TestContentChunk:
def test_char_count(self) -> None:
chunk = ContentChunk("f.md", "## H", 1, 3, "hello world", "markdown")
@@ -191,6 +247,29 @@ class TestChunkPython:
chunks = chunk_python(_cf(content, ".py", "mod.py"), min_chars=80)
assert chunks == []
+ def test_source_lines_are_split_once_for_many_functions(self) -> None:
+ class _CountingText(str):
+ __slots__ = ()
+ calls = 0
+
+ def splitlines(self, *args, **kwargs):
+ type(self).calls += 1
+ return super().splitlines(*args, **kwargs)
+
+ content = _CountingText(
+ "\n".join(
+ f'def function_{index}():\n """' + ("documented function " * 6) + '"""\n pass'
+ for index in range(100)
+ )
+ )
+ collected = _cf(content, ".py", "many.py")
+ _CountingText.calls = 0
+
+ chunks = chunk_python(collected, min_chars=80)
+
+ assert len(chunks) == 100
+ assert _CountingText.calls == 1
+
class TestChunkShell:
@pytest.mark.parametrize("prefix", ["", "#!/bin/sh\necho ready\n"])
@@ -232,6 +311,48 @@ class TestChunkShell:
assert chunks[0].heading == "setup()"
assert chunks[0].source_format == "shell"
+ def test_function_ranges_follow_matching_braces(self) -> None:
+ lines = [
+ "}",
+ "broken() {",
+ "outer() {",
+ " if true; then { echo nested; }; fi",
+ " echo ${HOME}",
+ "}",
+ "inner() { echo one; }",
+ ]
+ content = "\n".join(lines) + "\n"
+ chunks = chunk_shell(_cf(content, ".sh", "run.sh"), min_chars=1)
+
+ assert [(c.heading, c.start_line, c.end_line) for c in chunks] == [
+ ("outer()", 3, 6),
+ ("inner()", 7, 7),
+ ]
+ assert chunks[0].text == "\n".join(lines[2:6])
+
+ def test_many_closed_functions_get_line_ranges_without_rereading_the_script(self) -> None:
+ content = _ReadCountingText("".join(f"step_{index}() {{ :; }}\n" for index in range(1_000)))
+ collected = _cf(content, ".sh", "many.sh")
+ _ReadCountingText.read = 0
+
+ chunks = chunk_shell(collected, min_chars=1)
+
+ assert [(c.heading, c.start_line, c.end_line) for c in chunks] == [
+ (f"step_{index}()", index + 1, index + 1) for index in range(1_000)
+ ]
+ assert chunks[-1].text == "step_999() { :; }"
+ # One pass reads each character a few times. Rescanning or slicing the
+ # script once per function reads it hundreds of times.
+ assert _ReadCountingText.read <= len(content) * 10
+
+ def test_many_unclosed_function_headers_read_the_script_a_bounded_number_of_times(self) -> None:
+ content = _ReadCountingText("".join(f"function_{index}() {{\n" for index in range(1_000)))
+ collected = _cf(content, ".sh", "malformed.sh")
+ _ReadCountingText.read = 0
+
+ assert chunk_shell(collected, min_chars=1) == []
+ assert _ReadCountingText.read <= len(content) * 10
+
def test_comment_block_3_plus_lines(self) -> None:
content = "# Comment line one about setup\n# Comment line two about config\n# Comment line three about usage\n\necho hello\n"
chunks = chunk_shell(_cf(content, ".sh", "run.sh"), min_chars=80)
@@ -249,6 +370,23 @@ class TestChunkShell:
comment_chunks = [c for c in chunks if c.heading == "(comment)"]
assert len(comment_chunks) == 0
+ def test_many_unclosed_function_headers_have_linear_character_scanning(self) -> None:
+ class _CountingText(str):
+ __slots__ = ()
+ indexed = 0
+
+ def __getitem__(self, key):
+ if isinstance(key, int):
+ type(self).indexed += 1
+ return super().__getitem__(key)
+
+ content = _CountingText("\n".join(f"function_{index}() {{" for index in range(1_000)))
+ collected = _cf(content, ".sh", "malformed.sh")
+ _CountingText.indexed = 0
+
+ assert chunk_shell(collected, min_chars=1) == []
+ assert _CountingText.indexed <= len(content) * 3
+
class TestSplitIntoParagraphs:
def test_splits_on_blank_lines(self) -> None: