Personal AI, On Personal Devices
Python
fix(memory): preserve short paragraphs during chunking (#885)
Elliot Sluskyca419968fbc4
diff --git a/src/openjarvis/cli/memory_cmd.py b/src/openjarvis/cli/memory_cmd.pyindex c24840dd..d66b1ae1 100644--- a/src/openjarvis/cli/memory_cmd.py+++ b/src/openjarvis/cli/memory_cmd.py@@ -51,13 +51,13 @@ def memory() -> None:@click.option("--chunk-size",default=512,- type=int,+ type=click.IntRange(min=1),help="Chunk size in tokens.",)@click.option("--chunk-overlap",default=64,- type=int,+ type=click.IntRange(min=0),help="Overlap between chunks in tokens.",)def index(diff --git a/src/openjarvis/tools/storage/chunking.py b/src/openjarvis/tools/storage/chunking.pyindex abdbdaab..b8ccd214 100644--- a/src/openjarvis/tools/storage/chunking.py+++ b/src/openjarvis/tools/storage/chunking.py@@ -19,6 +19,24 @@ class ChunkConfig:chunk_overlap: int = 64min_chunk_size: int = 50+ def __post_init__(self) -> None:+ self.validate()++ def validate(self) -> None:+ """Reject values that cannot produce a finite, well-formed chunk stream."""+ if (+ isinstance(self.chunk_size, bool)+ or not isinstance(self.chunk_size, int)+ or self.chunk_size < 1+ ):+ raise ValueError("chunk_size must be an integer >= 1")+ if (+ isinstance(self.chunk_overlap, bool)+ or not isinstance(self.chunk_overlap, int)+ or self.chunk_overlap < 0+ ):+ raise ValueError("chunk_overlap must be an integer >= 0")+@dataclass(slots=True)class Chunk:@@ -57,11 +75,15 @@ def chunk_text(-------List of :class:`Chunk` objects, in order."""+ cfg = config or ChunkConfig()+ # ``ChunkConfig`` is intentionally mutable for API compatibility. Recheck+ # here so a caller cannot construct a valid config, mutate it, and enter a+ # non-progress loop during ingestion.+ cfg.validate()+if not text or not text.strip():return []- cfg = config or ChunkConfig()-# Split into paragraphs (double newline)paragraphs = [p for p in text.split("\n\n") if p.strip()]@@ -69,91 +91,86 @@ def chunk_text(current_tokens: List[str] = []current_offset = 0chunk_start_offset = 0+ force_final_chunk = False++ def emit_current(consumed_offset: int) -> None:+ """Emit the buffer and retain a bounded overlap for the next chunk."""+ nonlocal current_tokens, chunk_start_offset, force_final_chunk++ chunks.append(+ Chunk(+ content=" ".join(current_tokens),+ source=source,+ offset=chunk_start_offset,+ index=len(chunks),+ )+ )++ # Even a malformed overlap >= chunk_size must leave room for forward+ # progress and may never make the next chunk exceed the hard bound.+ keep = min(cfg.chunk_overlap, max(0, len(current_tokens) - 1))+ current_tokens = current_tokens[-keep:] if keep else []+ chunk_start_offset = consumed_offset - keep+ force_final_chunk = Falsefor para in paragraphs:para_tokens = para.split()-- # If adding this paragraph would exceed chunk_size and we already- # have content, flush the current chunk first.- if current_tokens and len(current_tokens) + len(para_tokens) > cfg.chunk_size:- chunk_content = " ".join(current_tokens)- if _count_tokens(chunk_content) >= cfg.min_chunk_size:- chunks.append(- Chunk(- content=chunk_content,- source=source,- offset=chunk_start_offset,- index=len(chunks),- )- )-- # Keep the overlap tail for the next chunk- if cfg.chunk_overlap > 0 and len(current_tokens) > cfg.chunk_overlap:- overlap = current_tokens[-cfg.chunk_overlap :]- current_tokens = list(overlap)+ would_overflow = len(current_tokens) + len(para_tokens) > cfg.chunk_size++ # Preserve a paragraph boundary when the buffered paragraph is large+ # enough to stand alone. A sub-floor lead-in instead stays attached+ # to the following text; dropping it was the original #754 bug.+ preserve_sequence = force_final_chunk+ if current_tokens and would_overflow:+ if len(current_tokens) >= cfg.min_chunk_size:+ emit_current(current_offset)else:- current_tokens = []- chunk_start_offset = current_offset-- # If a single paragraph exceeds chunk_size, split it directly- if len(para_tokens) > cfg.chunk_size:- # Flush anything accumulated first- if current_tokens:- chunk_content = " ".join(current_tokens)- if _count_tokens(chunk_content) >= cfg.min_chunk_size:- chunks.append(- Chunk(- content=chunk_content,- source=source,- offset=chunk_start_offset,- index=len(chunks),- )- )- current_tokens = []-- # Split the oversized paragraph into fixed windows- idx = 0- while idx < len(para_tokens):- window = para_tokens[idx : idx + cfg.chunk_size]- chunk_content = " ".join(window)- if _count_tokens(chunk_content) >= cfg.min_chunk_size:- chunks.append(- Chunk(- content=chunk_content,- source=source,- offset=current_offset + idx,- index=len(chunks),- )- )- step = max(1, cfg.chunk_size - cfg.chunk_overlap)- idx += step-- current_offset += len(para_tokens)- chunk_start_offset = current_offset- continue-- current_tokens.extend(para_tokens)+ preserve_sequence = True++ para_index = 0+ while para_index < len(para_tokens):+ capacity = cfg.chunk_size - len(current_tokens)+ remaining = len(para_tokens) - para_index++ if remaining <= capacity:+ current_tokens.extend(para_tokens[para_index:])+ para_index = len(para_tokens)+ force_final_chunk = preserve_sequence+ break++ take = capacity+ remainder = remaining - take+ overlap = min(cfg.chunk_overlap, max(0, cfg.chunk_size - 1))++ # With no (or a small) configured overlap, a full window can+ # strand a sub-floor tail. Rebalance the two windows when both+ # can satisfy the minimum; otherwise the preservation flag below+ # keeps the unavoidable short tail rather than losing content.+ tail_size = overlap + remainder+ if preserve_sequence and tail_size < cfg.min_chunk_size:+ shift = cfg.min_chunk_size - tail_size+ if len(current_tokens) + take - shift >= cfg.min_chunk_size:+ take -= shift++ current_tokens.extend(para_tokens[para_index : para_index + take])+ para_index += take+ emit_current(current_offset + para_index)+ force_final_chunk = preserve_sequence+current_offset += len(para_tokens)- # Flush remaining tokens.- #- # ``min_chunk_size`` exists to discard tiny *trailing* fragments once a- # document has already produced at least one chunk. It must NOT silently- # drop an entire short document: indexing a folder of short notes would- # otherwise report success while storing nothing (#502 follow-up). So if no- # chunk has been emitted yet, keep the remaining content regardless of the- # floor.+ # The buffer contains new document content, even when it also retains an+ # overlap. The minimum guides boundary selection; it must not discard a+ # short final paragraph or the end of an oversized paragraph (#754).if current_tokens:- chunk_content = " ".join(current_tokens)- if not chunks or _count_tokens(chunk_content) >= cfg.min_chunk_size:- chunks.append(- Chunk(- content=chunk_content,- source=source,- offset=chunk_start_offset,- index=len(chunks),- )+ chunks.append(+ Chunk(+ content=" ".join(current_tokens),+ source=source,+ offset=chunk_start_offset,+ index=len(chunks),)+ )return chunksdiff --git a/src/openjarvis/tools/storage_tools.py b/src/openjarvis/tools/storage_tools.pyindex fd588fef..04cd5c3d 100644--- a/src/openjarvis/tools/storage_tools.py+++ b/src/openjarvis/tools/storage_tools.py@@ -243,10 +243,12 @@ class MemoryIndexTool(BaseTool):},"chunk_size": {"type": "integer",+ "minimum": 1,"description": "Chunk size in characters (default 512).",},"chunk_overlap": {"type": "integer",+ "minimum": 0,"description": "Overlap between chunks (default 64).",},},@@ -281,9 +283,11 @@ class MemoryIndexTool(BaseTool):from openjarvis.tools.storage.chunking import ChunkConfigfrom openjarvis.tools.storage.ingest import ingest_path+ chunk_size = params.get("chunk_size", 512)+ chunk_overlap = params.get("chunk_overlap", 64)chunk_cfg = ChunkConfig(- chunk_size=int(params.get("chunk_size", 512)),- chunk_overlap=int(params.get("chunk_overlap", 64)),+ chunk_size=chunk_size,+ chunk_overlap=chunk_overlap,)chunks = ingest_path(Path(path), config=chunk_cfg)stored = 0diff --git a/tests/cli/test_memory_cmd.py b/tests/cli/test_memory_cmd.pyindex 9419651e..50d80595 100644--- a/tests/cli/test_memory_cmd.py+++ b/tests/cli/test_memory_cmd.py@@ -76,6 +76,25 @@ def test_memory_index_nonexistent(tmp_path: Path):assert result.exit_code != 0+def test_memory_index_rejects_non_progress_chunk_values(tmp_path: Path):+ doc = tmp_path / "doc.txt"+ doc.write_text("a b", encoding="utf-8")++ zero_size = CliRunner().invoke(+ cli,+ ["memory", "index", str(doc), "--chunk-size", "0"],+ )+ negative_overlap = CliRunner().invoke(+ cli,+ ["memory", "index", str(doc), "--chunk-overlap", "-1"],+ )++ assert zero_size.exit_code != 0+ assert "0 is not in the range x>=1" in zero_size.output+ assert negative_overlap.exit_code != 0+ assert "-1 is not in the range x>=0" in negative_overlap.output++def test_memory_search_returns_results(tmp_path: Path, monkeypatch):"""Search returns results from pre-populated backend."""_register_sqlite()diff --git a/tests/memory/test_chunking.py b/tests/memory/test_chunking.pyindex 7f6b4741..4ebefeee 100644--- a/tests/memory/test_chunking.py+++ b/tests/memory/test_chunking.py@@ -2,6 +2,8 @@from __future__ import annotations+import pytest+from openjarvis.tools.storage.chunking import ChunkConfig, chunk_text@@ -13,6 +15,55 @@ def test_whitespace_only_returns_empty():assert chunk_text(" \n\n ") == []+ ("kwargs", "message"),+ [+ ({"chunk_size": 0}, "chunk_size must be an integer >= 1"),+ ({"chunk_size": -1}, "chunk_size must be an integer >= 1"),+ ({"chunk_size": True}, "chunk_size must be an integer >= 1"),+ ({"chunk_overlap": -1}, "chunk_overlap must be an integer >= 0"),+ ({"chunk_overlap": False}, "chunk_overlap must be an integer >= 0"),+ ],+)+def test_invalid_chunk_config_is_rejected(kwargs, message):+ with pytest.raises(ValueError, match=message):+ ChunkConfig(**kwargs)+++def test_mutated_invalid_config_is_rejected_before_chunking():+ cfg = ChunkConfig(chunk_size=2, chunk_overlap=0, min_chunk_size=1)+ cfg.chunk_size = 0++ with pytest.raises(ValueError, match="chunk_size"):+ chunk_text("a b", config=cfg)+++def test_valid_boundary_sweep_preserves_content_size_and_offsets():+ tokens = [f"t{i}" for i in range(23)]+ text = " ".join(tokens[:3]) + "\n\n" + " ".join(tokens[3:])++ for chunk_size in range(1, 9):+ for chunk_overlap in range(0, 11):+ chunks = chunk_text(+ text,+ config=ChunkConfig(+ chunk_size=chunk_size,+ chunk_overlap=chunk_overlap,+ min_chunk_size=1,+ ),+ )+ emitted = []+ for chunk in chunks:+ chunk_tokens = chunk.content.split()+ emitted.extend(chunk_tokens)+ assert 0 < len(chunk_tokens) <= chunk_size+ assert tokens[chunk.offset : chunk.offset + len(chunk_tokens)] == (+ chunk_tokens+ )+ assert set(tokens).issubset(emitted)+ assert [chunk.index for chunk in chunks] == list(range(len(chunks)))++def test_short_text_single_chunk():# Need >= 50 words (default min_chunk_size)words = [f"word{i}" for i in range(60)]@@ -71,7 +122,7 @@ def test_short_only_document_not_dropped():Regression for #502 follow-up: previously a folder of short notes indexedto ``chunks_indexed: 0`` (HTTP 200), silently storing nothing. ``min_chunk_size``- should only discard tiny *trailing fragments*, never an entire short doc.+ must not discard an entire short document."""cfg = ChunkConfig(chunk_size=100, chunk_overlap=0, min_chunk_size=50)# 30 words is below min_chunk_size=50, but it's the entire document.@@ -89,17 +140,102 @@ def test_short_real_world_note_not_dropped():assert "hello world" in chunks[0].content-def test_min_chunk_size_filters_tiny_trailing_fragment():- """A tiny fragment trailing a real chunk is still dropped by the floor."""+def test_min_chunk_size_preserves_short_final_paragraph():+ """A short final paragraph must be searchable after indexing (#754)."""cfg = ChunkConfig(chunk_size=50, chunk_overlap=0, min_chunk_size=10)# Two paragraphs: the first fills a real chunk, the second is a tiny tail.para1 = " ".join(f"a{i}" for i in range(50))para2 = " ".join(f"b{i}" for i in range(3)) # 3 words < min_chunk_size=10text = f"{para1}\n\n{para2}"chunks = chunk_text(text, config=cfg)- # The 3-word trailing fragment is discarded; only the real chunk remains.- assert len(chunks) == 1- assert "b0" not in chunks[0].content+ assert len(chunks) == 2+ assert chunks[1].content == para2+ assert chunks[1].offset == 50++[email protected]("paragraph_lengths", [(11,), (5, 2), (2, 9, 1), (5, 5, 1)])[email protected]("overlap", [0, 1, 4, 7])+def test_short_tails_preserve_complete_document_and_offsets(paragraph_lengths, overlap):+ """Every input token remains covered, including sub-floor final tails."""+ tokens = [f"word{i}" for i in range(sum(paragraph_lengths))]+ paragraphs = []+ start = 0+ for length in paragraph_lengths:+ paragraphs.append(" ".join(tokens[start : start + length]))+ start += length+ chunks = chunk_text(+ "\n\n".join(paragraphs),+ config=ChunkConfig(chunk_size=5, chunk_overlap=overlap, min_chunk_size=4),+ )+ covered = set()+ for index, chunk in enumerate(chunks):+ words = chunk.content.split()+ assert 0 < len(words) <= 5+ assert chunk.index == index+ assert words == tokens[chunk.offset : chunk.offset + len(words)]+ covered.update(range(chunk.offset, chunk.offset + len(words)))+ assert covered == set(range(len(tokens)))+++def test_short_lead_in_before_oversized_paragraph_is_not_dropped():+ """A sub-floor lead-in must be carried into an oversized window (#754)."""+ lead_in = "IMPORTANT my API key rotation policy is documented here"+ large_paragraph = " ".join(f"word{i}" for i in range(600))++ chunks = chunk_text(f"{lead_in}\n\n{large_paragraph}")++ assert any(lead_in in chunk.content for chunk in chunks)+ output_tokens = {token for chunk in chunks for token in chunk.content.split()}+ assert set(lead_in.split()).issubset(output_tokens)+ assert {f"word{i}" for i in range(600)}.issubset(output_tokens)+++def test_short_paragraph_before_full_normal_paragraph_is_not_dropped():+ """The ordinary paragraph-boundary flush also preserves sub-floor text."""+ cfg = ChunkConfig(chunk_size=10, chunk_overlap=0, min_chunk_size=5)+ short = "keep every word"+ full = " ".join(f"next{i}" for i in range(10))++ chunks = chunk_text(f"{short}\n\n{full}", config=cfg)++ assert any(short in chunk.content for chunk in chunks)+ assert set(short.split()) | set(full.split()) == {+ token for chunk in chunks for token in chunk.content.split()+ }+ assert all(len(chunk.content.split()) <= cfg.chunk_size for chunk in chunks)+++ ("chunk_size", "chunk_overlap", "min_chunk_size", "lead_size", "body_size"),+ [+ (10, 0, 5, 3, 10),+ (10, 2, 5, 3, 10),+ (8, 7, 6, 2, 8),+ (7, 0, 4, 3, 12),+ (16, 4, 8, 4, 33),+ ],+)+def test_preserved_short_lead_in_never_breaks_hard_chunk_bound(+ chunk_size, chunk_overlap, min_chunk_size, lead_size, body_size+):+ """Adversarial boundaries preserve every token in bounded chunks."""+ cfg = ChunkConfig(+ chunk_size=chunk_size,+ chunk_overlap=chunk_overlap,+ min_chunk_size=min_chunk_size,+ )+ lead = [f"lead{i}" for i in range(lead_size)]+ body = [f"body{i}" for i in range(body_size)]++ chunks = chunk_text(+ f"{' '.join(lead)}\n\n{' '.join(body)}",+ config=cfg,+ )++ output_tokens = {token for chunk in chunks for token in chunk.content.split()}+ assert set(lead + body).issubset(output_tokens)+ assert all(0 < len(chunk.content.split()) <= chunk_size for chunk in chunks)+ assert [chunk.index for chunk in chunks] == list(range(len(chunks)))def test_source_propagated():diff --git a/tests/tools/test_storage_tools.py b/tests/tools/test_storage_tools.pyindex 558ac143..2dfbe4a7 100644--- a/tests/tools/test_storage_tools.py+++ b/tests/tools/test_storage_tools.py@@ -161,6 +161,9 @@ class TestMemoryIndexTool:def test_spec(self):tool = MemoryIndexTool()assert tool.spec.name == "memory_index"+ properties = tool.spec.parameters["properties"]+ assert properties["chunk_size"]["minimum"] == 1+ assert properties["chunk_overlap"]["minimum"] == 0def test_index_no_backend(self):tool = MemoryIndexTool()@@ -196,6 +199,28 @@ class TestMemoryIndexTool:finally:os.unlink(path)+ @pytest.mark.parametrize(+ "params",+ [+ {"chunk_size": 0},+ {"chunk_size": -1},+ {"chunk_size": True},+ {"chunk_overlap": -1},+ {"chunk_overlap": False},+ ],+ )+ def test_index_rejects_invalid_chunk_config(self, backend, params):+ with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f:+ f.write("a b c")+ path = f.name+ try:+ result = MemoryIndexTool(backend).execute(path=path, **params)+ assert result.success is False+ assert "must be an integer" in result.content+ assert backend._data == {}+ finally:+ os.unlink(path)+class TestStorageToolsRegistration:def test_storage_tools_importable(self):