π OpenHands: AI-Driven Development
feat: implement semantic file chunking using tree-sitter AST parsing (#14699)
Yuvraj Singh Chauhanb897ce421df0
diff --git a/openhands/app_server/utils/chunk_localizer.py b/openhands/app_server/utils/chunk_localizer.pyindex 6e6a086ab8..18bb40feee 100644--- a/openhands/app_server/utils/chunk_localizer.py+++ b/openhands/app_server/utils/chunk_localizer.py@@ -6,6 +6,7 @@ for a given query (e.g. edit draft produced by the agent).from pydantic import BaseModelfrom rapidfuzz.distance import LCSseq+from tree_sitter import Node, Treefrom tree_sitter_language_pack import get_parserfrom openhands.app_server.utils.logger import openhands_logger as logger@@ -25,9 +26,9 @@ class Chunk(BaseModel):return ret-def _create_chunks_from_raw_string(content: str, size: int):+def _create_chunks_from_raw_string(content: str, size: int) -> list[Chunk]:lines = content.split('\n')- ret = []+ ret: list[Chunk] = []for i in range(0, len(lines), size):_cur_lines = lines[i : i + size]ret.append(@@ -39,12 +40,121 @@ def _create_chunks_from_raw_string(content: str, size: int):return ret+def _collect_semantic_split_lines(+ node: Node,+ max_chunk_lines: int,+) -> set[int]:+ """Walk the AST to find 0-indexed line numbers where splitting is appropriate.++ A "split-after" line is the last line of a top-level AST child: splitting+ after it preserves the semantic unit. For oversized children, we recurse into+ their sub-children to discover finer split points.++ Args:+ node: A tree-sitter Node whose children define semantic units.+ max_chunk_lines: Target maximum lines per chunk. Nodes spanning more+ than this are recursed into.++ Returns:+ Set of 0-indexed line numbers where splitting *after* that line+ is semantically appropriate.+ """+ split_after: set[int] = set()+ if not node.children:+ return split_after++ for child in node.children:+ child_end_line = child.end_point[0]+ child_span = child_end_line - child.start_point[0] + 1++ # Mark the end of every child as a valid split point.+ split_after.add(child_end_line)++ # Recurse into children that are themselves oversized.+ if child_span > max_chunk_lines and child.children:+ split_after.update(_collect_semantic_split_lines(child, max_chunk_lines))++ return split_after+++def _create_chunks_from_tree_sitter(+ tree: Tree,+ text: str,+ max_chunk_lines: int,+) -> list[Chunk]:+ """Create semantically-aware chunks from a tree-sitter parse tree.++ Args:+ tree: A tree_sitter.Tree returned by parser.parse().+ text: The original source text that was parsed.+ max_chunk_lines: Maximum number of lines per chunk.++ Returns:+ A list of Chunk objects covering the entire source text.+ """+ text_lines = text.split('\n')+ total_lines = len(text_lines)++ root = tree.root_node+ if not root.children:+ return _create_chunks_from_raw_string(text, max_chunk_lines)++ # Phase 1: discover where we prefer to split.+ split_after_set = _collect_semantic_split_lines(root, max_chunk_lines)++ # Phase 2: greedy line-based chunking with preferred split points.+ chunks: list[Chunk] = []+ chunk_start = 0 # 0-indexed, inclusive++ while chunk_start < total_lines:+ # The farthest line we could include without exceeding the budget.+ budget_end = min(chunk_start + max_chunk_lines - 1, total_lines - 1)++ if budget_end >= total_lines - 1:+ # Remaining lines fit in one chunk β emit and finish.+ chunk_text = '\n'.join(text_lines[chunk_start:total_lines])+ chunks.append(+ Chunk(+ text=chunk_text,+ line_range=(chunk_start + 1, total_lines),+ )+ )+ break++ # Search backward from budget_end for the latest semantic split point.+ best_split: int | None = None+ for candidate in range(budget_end, chunk_start - 1, -1):+ if candidate in split_after_set:+ best_split = candidate+ break++ if best_split is not None and best_split >= chunk_start:+ chunk_end = best_split+ else:+ # No semantic boundary found - raw split at budget limit.+ chunk_end = budget_end++ chunk_text = '\n'.join(text_lines[chunk_start : chunk_end + 1])+ chunks.append(+ Chunk(+ text=chunk_text,+ line_range=(chunk_start + 1, chunk_end + 1),+ )+ )+ chunk_start = chunk_end + 1++ return chunks++def create_chunks(text: str, size: int = 100, language: str | None = None) -> list[Chunk]:+ if size <= 0:+ raise ValueError(f'size must be a positive integer, got {size}')+try:parser = get_parser(language) if language is not None else None- except AttributeError:+ except (AttributeError, LookupError):logger.debug(f'Language {language} not supported. Falling back to raw string.')parser = None@@ -52,9 +162,9 @@ def create_chunks(# fallback to raw stringreturn _create_chunks_from_raw_string(text, size)- # TODO: implement tree-sitter chunking- # return _create_chunks_from_tree_sitter(parser.parse(bytes(text, 'utf-8')), max_chunk_lines=size)- raise NotImplementedError('Tree-sitter chunking not implemented yet.')+ return _create_chunks_from_tree_sitter(+ parser.parse(text.encode('utf-8')), text, max_chunk_lines=size+ )def normalized_lcs(chunk: str, query: str) -> float:@@ -95,7 +205,7 @@ def get_top_k_chunk_matches(]sorted_chunks = sorted(chunks_with_lcs,- key=lambda x: x.normalized_lcs, # type: ignore+ key=lambda x: x.normalized_lcs if x.normalized_lcs is not None else 0.0,reverse=True,)return sorted_chunks[:k]diff --git a/tests/unit/utils/test_chunk_localizer.py b/tests/unit/utils/test_chunk_localizer.pyindex 0bffe19054..df8e630a66 100644--- a/tests/unit/utils/test_chunk_localizer.py+++ b/tests/unit/utils/test_chunk_localizer.py@@ -2,12 +2,95 @@ import pytestfrom openhands.app_server.utils.chunk_localizer import (Chunk,+ _create_chunks_from_raw_string,create_chunks,get_top_k_chunk_matches,normalized_lcs,)+def assert_chunk_invariants(+ text: str,+ size: int,+ language: str | None = None,+ *,+ strict_max: bool = True,+) -> list[Chunk]:+ """Run ``create_chunks`` and assert all core invariants.++ Calls ``create_chunks(text, size, language)`` and checks:+ 1. At least one chunk is produced+ 2. First chunk starts at line 1+ 3. Each chunk has start <= end (no inverted ranges)+ 4. Each chunk is within bounds (end <= total_lines)+ 5. Text line count matches declared range width+ 6. Chunk.visualize() does not raise+ 7. chunks[i].end + 1 == chunks[i+1].start (contiguity)+ 8. Last chunk ends at EOF+ 9. '\\n'.join(c.text for c in chunks) == text (reconstruction)+ 10. (optional) every chunk has at most ``size`` lines++ Args:+ text: Source text to chunk.+ size: Maximum lines per chunk (passed to ``create_chunks``).+ language: Language hint passed to ``create_chunks``.+ strict_max: When True (default), also assert that no chunk+ exceeds ``size`` lines.++ Returns:+ The produced chunk list, so callers can make additional assertions.+ """+ chunks = create_chunks(text, size=size, language=language)+ N = len(text.split('\n'))++ assert len(chunks) >= 1, f'size={size}: expected at least one chunk, got 0'++ assert chunks[0].line_range[0] == 1, (+ f'size={size}: first chunk starts at {chunks[0].line_range[0]}, expected 1'+ )++ assert chunks[-1].line_range[1] == N, (+ f'size={size}: last chunk ends at {chunks[-1].line_range[1]}, expected {N}'+ )++ for i, c in enumerate(chunks):+ s, e = c.line_range+ assert s <= e, f'size={size}: chunk[{i}] has inverted range ({s}, {e})'+ assert 1 <= s <= N, (+ f'size={size}: chunk[{i}] start={s} is out of bounds [1, {N}]'+ )+ assert 1 <= e <= N, f'size={size}: chunk[{i}] end={e} is out of bounds [1, {N}]'+ declared = e - s + 1+ actual = len(c.text.split('\n'))+ assert actual == declared, (+ f'size={size}: chunk[{i}] range implies {declared} lines '+ f'but text has {actual}'+ )+ c.visualize()+ if strict_max:+ assert declared <= size, (+ f'size={size}: chunk[{i}] has {declared} lines, '+ f'exceeding limit: {c.line_range}'+ )++ for i in range(len(chunks) - 1):+ curr_end = chunks[i].line_range[1]+ next_start = chunks[i + 1].line_range[0]+ assert curr_end + 1 == next_start, (+ f'size={size}: gap/overlap between chunk[{i}] (ends={curr_end}) '+ f'and chunk[{i + 1}] (starts={next_start})'+ )++ reconstructed = '\n'.join(c.text for c in chunks)+ assert reconstructed == text, (+ f'size={size}: reconstruction failed.\n'+ f' expected : {repr(text[:120])}\n'+ f' got : {repr(reconstructed[:120])}'+ )++ return chunks++def test_chunk_creation():chunk = Chunk(text='test chunk', line_range=(1, 1))assert chunk.text == 'test chunk'@@ -20,6 +103,11 @@ def test_chunk_visualization(capsys):assert chunk.visualize() == '1|line1\n2|line2\n'+def test_chunk_visualization_with_special_characters():+ chunk = Chunk(text='line1\nline2\t\nline3\r', line_range=(1, 3))+ assert chunk.visualize() == '1|line1\n2|line2\t\n3|line3\r\n'++def test_create_chunks_raw_string():text = 'line1\nline2\nline3\nline4\nline5'chunks = create_chunks(text, size=2)@@ -32,26 +120,6 @@ def test_create_chunks_raw_string():assert chunks[2].line_range == (5, 5)-def test_normalized_lcs():- chunk = 'abcdef'- edit_draft = 'abcxyz'- assert normalized_lcs(chunk, edit_draft) == 0.5---def test_get_top_k_chunk_matches():- text = 'chunk1\nchunk2\nchunk3\nchunk4'- query = 'chunk2'- matches = get_top_k_chunk_matches(text, query, k=2, max_chunk_size=1)- assert len(matches) == 2- assert matches[0].text == 'chunk2'- assert matches[0].line_range == (2, 2)- assert matches[0].normalized_lcs == 1.0- assert matches[1].text == 'chunk1'- assert matches[1].line_range == (1, 1)- assert matches[1].normalized_lcs == 5 / 6- assert matches[0].normalized_lcs > matches[1].normalized_lcs--def test_create_chunks_with_empty_lines():text = 'line1\n\nline3\n\n\nline6'chunks = create_chunks(text, size=2)@@ -82,6 +150,20 @@ def test_create_chunks_with_last_chunk_smaller():assert chunks[1].line_range == (3, 3)[email protected]('chunk_size', [1, 2, 3, 4])+def test_create_chunks_different_sizes(chunk_size):+ text = 'line1\nline2\nline3\nline4'+ chunks = create_chunks(text, size=chunk_size)+ assert len(chunks) == (4 + chunk_size - 1) // chunk_size+ assert sum(len(chunk.text.split('\n')) for chunk in chunks) == 4+++def test_normalized_lcs():+ chunk = 'abcdef'+ edit_draft = 'abcxyz'+ assert normalized_lcs(chunk, edit_draft) == 0.5++def test_normalized_lcs_edge_cases():assert normalized_lcs('', '') == 0.0assert normalized_lcs('a', '') == 0.0@@ -89,6 +171,26 @@ def test_normalized_lcs_edge_cases():assert normalized_lcs('abcde', 'ace') == 0.6+def test_normalized_lcs_with_unicode():+ chunk = 'Hello, δΈη!'+ edit_draft = 'Hello, world!'+ assert 0 < normalized_lcs(chunk, edit_draft) < 1+++def test_get_top_k_chunk_matches():+ text = 'chunk1\nchunk2\nchunk3\nchunk4'+ query = 'chunk2'+ matches = get_top_k_chunk_matches(text, query, k=2, max_chunk_size=1)+ assert len(matches) == 2+ assert matches[0].text == 'chunk2'+ assert matches[0].line_range == (2, 2)+ assert matches[0].normalized_lcs == 1.0+ assert matches[1].text == 'chunk1'+ assert matches[1].line_range == (1, 1)+ assert matches[1].normalized_lcs == 5 / 6+ assert matches[0].normalized_lcs > matches[1].normalized_lcs++def test_get_top_k_chunk_matches_with_ties():text = 'chunk1\nchunk2\nchunk3\nchunk1'query = 'chunk'@@ -102,26 +204,7 @@ def test_get_top_k_chunk_matches_with_large_k():text = 'chunk1\nchunk2\nchunk3'query = 'chunk'matches = get_top_k_chunk_matches(text, query, k=10, max_chunk_size=1)- assert len(matches) == 3 # Should return all chunks even if k is larger--[email protected]('chunk_size', [1, 2, 3, 4])-def test_create_chunks_different_sizes(chunk_size):- text = 'line1\nline2\nline3\nline4'- chunks = create_chunks(text, size=chunk_size)- assert len(chunks) == (4 + chunk_size - 1) // chunk_size- assert sum(len(chunk.text.split('\n')) for chunk in chunks) == 4---def test_chunk_visualization_with_special_characters():- chunk = Chunk(text='line1\nline2\t\nline3\r', line_range=(1, 3))- assert chunk.visualize() == '1|line1\n2|line2\t\n3|line3\r\n'---def test_normalized_lcs_with_unicode():- chunk = 'Hello, δΈη!'- edit_draft = 'Hello, world!'- assert 0 < normalized_lcs(chunk, edit_draft) < 1+ assert len(matches) == 3def test_get_top_k_chunk_matches_with_overlapping_chunks():@@ -134,3 +217,489 @@ def test_get_top_k_chunk_matches_with_overlapping_chunks():assert matches[1].text == 'chunk3\nchunk4'assert matches[1].line_range == (3, 4)assert matches[0].normalized_lcs == matches[1].normalized_lcs+++def test_create_chunks_unsupported_language_fallback():+ """Unsupported language falls back to raw string chunking."""+ text = 'line1\nline2\nline3\nline4'+ chunks = create_chunks(text, size=2, language='brainfuck_not_real')+ assert len(chunks) == 2+ assert chunks[0].text == 'line1\nline2'+ assert chunks[0].line_range == (1, 2)+ assert chunks[1].text == 'line3\nline4'+ assert chunks[1].line_range == (3, 4)++[email protected]('size', [0, -1, -100])[email protected]('language', [None, 'python'])+def test_create_chunks_non_positive_size_raises(size, language):+ """A non-positive size must fail fast rather than loop forever.++ The tree-sitter path would otherwise spin indefinitely (the chunk cursor+ never advances when the budget is empty), so guard it explicitly.+ """+ with pytest.raises(ValueError):+ create_chunks('def foo():\n pass', size=size, language=language)+++def test_create_chunks_no_language_uses_raw():+ """When language=None the raw string chunker is used."""+ text = 'a\nb\nc\nd\ne'+ chunks = create_chunks(text, size=2, language=None)+ assert len(chunks) == 3+ assert chunks[0].line_range == (1, 2)+ assert chunks[1].line_range == (3, 4)+ assert chunks[2].line_range == (5, 5)+++def test_create_chunks_empty_file():+ chunks = create_chunks('', size=10, language='python')+ assert len(chunks) == 1+ assert chunks[0].text == ''+ assert chunks[0].line_range == (1, 1)+++def test_create_chunks_empty_file_raw():+ chunks = create_chunks('', size=10)+ assert len(chunks) == 1+ assert chunks[0].text == ''+ assert chunks[0].line_range == (1, 1)+++def test_create_chunks_tree_sitter_whitespace_only():+ text = '\n\n\n\n\n'+ assert_chunk_invariants(text, size=2, language='python')+++def test_create_chunks_tree_sitter_python_basic():+ text = """\n def foo():\n print("foo")\n def bar():\n print("bar")\n """+ chunks = create_chunks(text, size=3, language='python')+ assert len(chunks) == 2+ assert chunks[0].line_range == (1, 3)+ assert 'def foo():' in chunks[0].text+ assert chunks[1].line_range == (4, 6)+ assert 'def bar():' in chunks[1].text+++def test_create_chunks_tree_sitter_python_oversized():+ text = """\n class MyClass:\n def method1(self):\n a = 1\n b = 2\n\n def method2(self):\n c = 3\n d = 4\n """+ assert_chunk_invariants(text, size=4, language='python')+++def test_create_chunks_tree_sitter_prefix_respects_max_lines():+ """Lines before the first AST node must not push a chunk past max_chunk_lines."""+ text = (+ '# comment 1\n# comment 2\n# comment 3\n'+ '# comment 4\n# comment 5\ndef foo():\n pass'+ )+ assert_chunk_invariants(text, size=3, language='python')+++def test_create_chunks_tree_sitter_gap_respects_max_lines():+ """Blank lines between AST nodes must not push a chunk past max_chunk_lines."""+ lines = [+ 'def foo():',+ ' pass',+ '',+ '',+ '',+ '',+ '',+ 'def bar():',+ ' pass',+ ]+ text = '\n'.join(lines)+ assert_chunk_invariants(text, size=3, language='python')+++def test_create_chunks_tree_sitter_suffix_respects_max_lines():+ """Trailing lines after the last AST node must not push a chunk past max_chunk_lines."""+ text = 'def foo():\n pass\n\n\n\n\n'+ assert_chunk_invariants(text, size=3, language='python')+++def test_create_chunks_tree_sitter_single_line_functions():+ """Multiple single-line statements are grouped up to max_chunk_lines."""+ text = 'a = 1\nb = 2\nc = 3\nd = 4\ne = 5\nf = 6'+ assert_chunk_invariants(text, size=2, language='python')+++def test_create_chunks_tree_sitter_tuple_literal_size_5():+ text = 'DATA = tuple([\n' + '\n'.join(f' {i},' for i in range(12)) + '\n])'+ assert_chunk_invariants(text, size=5, language='python')+++def test_create_chunks_tree_sitter_tuple_literal_size_100():+ text = 'DATA = tuple([\n' + '\n'.join(f' {i},' for i in range(12)) + '\n])'+ chunks = assert_chunk_invariants(text, size=100, language='python')+ assert len(chunks) == 1+++def test_create_chunks_tree_sitter_large_tuple_literal():+ text = 'DATA = tuple([\n' + '\n'.join(f' {i},' for i in range(200)) + '\n])'+ assert_chunk_invariants(text, size=100, language='python')+++def test_create_chunks_tree_sitter_deeply_nested():+ """Deeply nested AST produces valid chunks at small size."""+ text = '\n'.join(+ [+ 'class Outer:',+ ' class Inner:',+ ' def method(self):',+ ' if True:',+ ' for i in range(10):',+ ' x = i',+ ' y = i + 1',+ ' z = i + 2',+ ]+ )+ assert_chunk_invariants(text, size=3, language='python')+++def test_create_chunks_tree_sitter_single_huge_function():+ """A function longer than max_chunk_lines is still chunked correctly."""+ body_lines = [f' x{i} = {i}' for i in range(20)]+ text = 'def big():\n' + '\n'.join(body_lines)+ assert_chunk_invariants(text, size=5, language='python')+++def test_create_chunks_tree_sitter_same_row_nested_nodes():+ """Nodes sharing the same start row must not produce inverted or duplicate ranges."""+ text = 'x = [1, 2, 3]'+ assert_chunk_invariants(text, size=1, language='python')+++def test_create_chunks_tree_sitter_multiline_dict():+ """Multi-line dict literal with a same-row opening brace."""+ text = 'config = {\n' + '\n'.join(f' "key{i}": {i},' for i in range(15)) + '\n}'+ assert_chunk_invariants(text, size=5, language='python')+++def test_create_chunks_tree_sitter_nested_function_calls():+ """Deeply nested function calls on a single line."""+ text = 'result = foo(bar(baz(qux(42))))'+ assert_chunk_invariants(text, size=1, language='python')+++def test_create_chunks_tree_sitter_mixed_constructs():+ """File mixing imports, decorators, functions, and classes."""+ text = '\n'.join(+ [+ 'import os',+ 'import sys',+ '',+ '',+ '@decorator',+ 'def helper():',+ ' return 42',+ '',+ '',+ 'class MyClass:',+ ' """A docstring."""',+ '',+ ' def method(self):',+ ' pass',+ '',+ ' @staticmethod',+ ' def static_method():',+ ' return 1',+ ]+ )+ assert_chunk_invariants(text, size=5, language='python')+++def test_create_chunks_tree_sitter_visualize_all_chunks():+ """Chunk.visualize() must not raise for any chunk produced by the tree-sitter path."""+ text = 'DATA = tuple([\n' + '\n'.join(f' {i},' for i in range(12)) + '\n])'+ for size in [1, 2, 3, 5, 10, 100]:+ chunks = create_chunks(text, size=size, language='python')+ for c in chunks:+ c.visualize()+++_CODE_SAMPLES: dict[str, str] = {+ 'tuple_literal': (+ 'DATA = tuple([\n' + '\n'.join(f' {i},' for i in range(12)) + '\n])'+ ),+ 'large_tuple': (+ 'DATA = tuple([\n' + '\n'.join(f' {i},' for i in range(200)) + '\n])'+ ),+ 'functions': '\n'.join(+ [+ 'import os',+ 'import sys',+ '',+ 'def foo():',+ ' return 1',+ '',+ 'def bar():',+ ' return 2',+ '',+ 'class Baz:',+ ' def method(self):',+ ' pass',+ ]+ ),+ 'deeply_nested': '\n'.join(+ [+ 'class Outer:',+ ' class Inner:',+ ' def method(self):',+ ' if True:',+ ' for i in range(10):',+ ' x = i',+ ' y = i + 1',+ ' z = i + 2',+ ]+ ),+ 'single_huge_function': (+ 'def big():\n' + '\n'.join(f' x{i} = {i}' for i in range(50))+ ),+ 'single_line': 'x = 42',+ 'empty': '',+ 'whitespace_only': '\n\n\n\n\n',+ 'assignments': 'a = 1\nb = 2\nc = 3\nd = 4\ne = 5\nf = 6',+ 'multiline_string': 'x = """\nline1\nline2\nline3\nline4\nline5\n"""',+ 'list_comprehension': (+ 'result = [\n' + '\n'.join(f' item_{i},' for i in range(20)) + '\n]'+ ),+}++[email protected]('size', [1, 2, 3, 5, 10, 100])[email protected]('sample_name', list(_CODE_SAMPLES.keys()))+def test_invariants_parametrized(sample_name: str, size: int):+ """Invariant sweep across multiple code samples and chunk sizes."""+ text = _CODE_SAMPLES[sample_name]+ assert_chunk_invariants(text, size=size, language='python')++[email protected]('size', [1, 2, 3, 5, 10])+def test_max_chunk_lines_enforced(size):+ """max_chunk_lines is strictly respected."""+ text = '\n'.join(+ [+ 'import os',+ 'import sys',+ '',+ 'def foo():',+ ' return 1',+ '',+ 'def bar():',+ ' return 2',+ '',+ 'class Baz:',+ ' def method(self):',+ ' pass',+ ]+ )+ assert_chunk_invariants(text, size=size, language='python')+++class TestChunkInvariantPython:+ """Invariant tests for the tree-sitter chunking path."""++ SIZES = [1, 2, 3, 5, 7, 10, 15, 50, 100]++ def _check_all_sizes(self, text: str) -> None:+ for size in self.SIZES:+ assert_chunk_invariants(text, size=size, language='python')++ def test_tuple_literal_all_sizes(self):+ """Multi-line tuple literal with nested same-row AST nodes."""+ text = 'DATA = tuple([\n' + '\n'.join(f' {i},' for i in range(12)) + '\n])'+ self._check_all_sizes(text)++ def test_flat_functions(self):+ """Multiple top-level functions."""+ text = '\n'.join(+ [+ 'def foo():',+ ' return 1',+ '',+ 'def bar():',+ ' return 2',+ '',+ 'def baz():',+ ' return 3',+ ]+ )+ self._check_all_sizes(text)++ def test_single_oversized_function(self):+ """Single function whose body exceeds max_chunk_lines."""+ body = '\n'.join(f' x_{i} = {i}' for i in range(30))+ text = f'def big_function():\n{body}\n return x_0'+ self._check_all_sizes(text)++ def test_class_with_many_methods(self):+ """Class whose total size exceeds max_chunk_lines."""+ methods = []+ for i in range(8):+ methods.append(f' def method_{i}(self):')+ methods.append(f' return {i}')+ methods.append('')+ text = 'class MyClass:\n' + '\n'.join(methods)+ self._check_all_sizes(text)++ def test_deeply_nested_class(self):+ """Nested class definition forcing multi-level descent."""+ text = (+ 'class Outer:\n'+ ' class Inner:\n'+ + '\n'.join(f' def m{i}(self): return {i}' for i in range(10))+ + '\n def outer_method(self):\n pass\n'+ )+ self._check_all_sizes(text)++ def test_leading_and_trailing_blank_lines(self):+ """Blank lines before the first and after the last AST node."""+ text = '\n\n\ndef foo():\n return 1\n\n\n'+ self._check_all_sizes(text)++ def test_same_row_siblings(self):+ """Multiple top-level statements, some spanning multiple lines."""+ text = (+ 'x = [1, 2, 3]\n'+ "y = {'a': 1, 'b': 2}\n"+ 'z = (i for i in range(100))\n'+ 'DATA = tuple([\n' + '\n'.join(f' {i},' for i in range(8)) + '\n])\n'+ 'W = list(range(20))\n'+ )+ self._check_all_sizes(text)++ def test_empty_file(self):+ """Empty input produces exactly one chunk."""+ chunks = create_chunks('', size=10, language='python')+ assert len(chunks) == 1+ assert chunks[0].text == ''++ def test_single_line(self):+ """Single-line file produces exactly one chunk regardless of size."""+ text = 'x = 1'+ self._check_all_sizes(text)++ def test_large_file_default_size(self):+ """Large list literal followed by multiple functions."""+ big_list = (+ 'QUERIES = [\n'+ + '\n'.join(f" 'query_{i}'," for i in range(120))+ + '\n]\n'+ )+ functions = '\n'.join(+ f'def handler_{i}(ctx):\n return QUERIES[{i}]\n' for i in range(10)+ )+ text = big_list + '\n' + functions+ assert_chunk_invariants(text, size=100, language='python')+ assert_chunk_invariants(text, size=50, language='python')+ assert_chunk_invariants(text, size=10, language='python')++ def test_fallback_for_unsupported_language(self):+ """Unsupported language falls back to raw-string chunking."""+ text = 'line one\nline two\nline three\nline four\nline five\n'+ assert_chunk_invariants(text, size=2, language='brainfuck')++ def test_fallback_for_none_language(self):+ """language=None uses raw-string chunking."""+ text = '\n'.join(f'line {i}' for i in range(20))+ assert_chunk_invariants(text, size=5, language=None)+++class TestSemanticBoundaries:+ """Assert that tree-sitter produces *different* boundaries than raw splitting.++ Each test constructs input where semantic chunking and raw line splitting+ produce different chunk boundaries, then asserts the semantic boundaries+ explicitly. If ``_create_chunks_from_tree_sitter`` were swapped for the+ raw splitter, these tests would fail.+ """++ def test_unequal_sibling_functions_split_on_function_boundary(self):+ """A short function followed by a longer one: chunk ends at the function+ boundary, not at the raw budget line.+ """+ text = '\n'.join(+ [+ 'def short():',+ ' return 1',+ '',+ 'def long_func():',+ ' a = 1',+ ' b = 2',+ ' return a + b',+ ]+ )+ chunks = create_chunks(text, size=5, language='python')++ # Semantic boundary: first chunk contains only short() (lines 1-2),+ # second chunk contains the blank line + long_func() (lines 3-7).+ boundaries = [c.line_range for c in chunks]+ assert boundaries == [(1, 2), (3, 7)], (+ f'Expected semantic split after short(), got {boundaries}'+ )++ # Verify this differs from raw splitting.+ raw_chunks = _create_chunks_from_raw_string(text, 5)+ raw_boundaries = [c.line_range for c in raw_chunks]+ assert raw_boundaries != boundaries, (+ 'Semantic and raw boundaries should differ for this input'+ )++ def test_oversized_class_splits_between_methods(self):+ """A class with three methods: chunks split between methods, not mid-method."""+ text = '\n'.join(+ [+ 'class MyClass:',+ ' def method_a(self):',+ ' return 1',+ '',+ ' def method_b(self):',+ ' return 2',+ '',+ ' def method_c(self):',+ ' return 3',+ ]+ )+ chunks = create_chunks(text, size=4, language='python')++ boundaries = [c.line_range for c in chunks]+ assert boundaries == [(1, 3), (4, 6), (7, 9)], (+ f'Expected splits between methods, got {boundaries}'+ )++ raw_chunks = _create_chunks_from_raw_string(text, 4)+ raw_boundaries = [c.line_range for c in raw_chunks]+ assert raw_boundaries != boundaries++ def test_semantic_split_keeps_decorator_with_function(self):+ """A decorated function stays in the same chunk as its decorator."""+ text = '\n'.join(+ [+ 'import os',+ 'import sys',+ '',+ '@my_decorator',+ 'def decorated():',+ ' return 42',+ '',+ ]+ )+ chunks = create_chunks(text, size=5, language='python')+ boundaries = [c.line_range for c in chunks]++ # The decorator + function must be in the same chunk.+ # Find the chunk containing '@my_decorator'.+ decorator_chunk = [c for c in chunks if '@my_decorator' in c.text]+ assert len(decorator_chunk) == 1+ assert 'def decorated():' in decorator_chunk[0].text, (+ 'Decorator and function definition should be in the same chunk'+ )+ assert 'return 42' in decorator_chunk[0].text, (+ 'Function body should be in the same chunk as its decorator'+ )++ # Verify this differs from raw.+ raw_chunks = _create_chunks_from_raw_string(text, 5)+ raw_boundaries = [c.line_range for c in raw_chunks]+ assert raw_boundaries != boundaries