Skip to content

Personal AI, On Personal Devices

Python

fix(memory): preserve short paragraphs during chunking (#885)

Elliot Sluskyca419968fbc4
diff --git a/src/openjarvis/cli/memory_cmd.py b/src/openjarvis/cli/memory_cmd.py
index c24840dd..d66b1ae1 100644
--- a/src/openjarvis/cli/memory_cmd.py
+++ b/src/openjarvis/cli/memory_cmd.py
@@ -51,13 +51,13 @@ def memory() -> None:
@click.option(
"--chunk-size",
default=512,
- type=int,
+ type=click.IntRange(min=1),
help="Chunk size in tokens.",
)
@click.option(
"--chunk-overlap",
default=64,
- type=int,
+ type=click.IntRange(min=0),
help="Overlap between chunks in tokens.",
)
def index(
diff --git a/src/openjarvis/tools/storage/chunking.py b/src/openjarvis/tools/storage/chunking.py
index abdbdaab..b8ccd214 100644
--- a/src/openjarvis/tools/storage/chunking.py
+++ b/src/openjarvis/tools/storage/chunking.py
@@ -19,6 +19,24 @@ class ChunkConfig:
chunk_overlap: int = 64
min_chunk_size: int = 50
+ def __post_init__(self) -> None:
+ self.validate()
+
+ def validate(self) -> None:
+ """Reject values that cannot produce a finite, well-formed chunk stream."""
+ if (
+ isinstance(self.chunk_size, bool)
+ or not isinstance(self.chunk_size, int)
+ or self.chunk_size < 1
+ ):
+ raise ValueError("chunk_size must be an integer >= 1")
+ if (
+ isinstance(self.chunk_overlap, bool)
+ or not isinstance(self.chunk_overlap, int)
+ or self.chunk_overlap < 0
+ ):
+ raise ValueError("chunk_overlap must be an integer >= 0")
+
@dataclass(slots=True)
class Chunk:
@@ -57,11 +75,15 @@ def chunk_text(
-------
List of :class:`Chunk` objects, in order.
"""
+ cfg = config or ChunkConfig()
+ # ``ChunkConfig`` is intentionally mutable for API compatibility. Recheck
+ # here so a caller cannot construct a valid config, mutate it, and enter a
+ # non-progress loop during ingestion.
+ cfg.validate()
+
if not text or not text.strip():
return []
- cfg = config or ChunkConfig()
-
# Split into paragraphs (double newline)
paragraphs = [p for p in text.split("\n\n") if p.strip()]
@@ -69,91 +91,86 @@ def chunk_text(
current_tokens: List[str] = []
current_offset = 0
chunk_start_offset = 0
+ force_final_chunk = False
+
+ def emit_current(consumed_offset: int) -> None:
+ """Emit the buffer and retain a bounded overlap for the next chunk."""
+ nonlocal current_tokens, chunk_start_offset, force_final_chunk
+
+ chunks.append(
+ Chunk(
+ content=" ".join(current_tokens),
+ source=source,
+ offset=chunk_start_offset,
+ index=len(chunks),
+ )
+ )
+
+ # Even a malformed overlap >= chunk_size must leave room for forward
+ # progress and may never make the next chunk exceed the hard bound.
+ keep = min(cfg.chunk_overlap, max(0, len(current_tokens) - 1))
+ current_tokens = current_tokens[-keep:] if keep else []
+ chunk_start_offset = consumed_offset - keep
+ force_final_chunk = False
for para in paragraphs:
para_tokens = para.split()
-
- # If adding this paragraph would exceed chunk_size and we already
- # have content, flush the current chunk first.
- if current_tokens and len(current_tokens) + len(para_tokens) > cfg.chunk_size:
- chunk_content = " ".join(current_tokens)
- if _count_tokens(chunk_content) >= cfg.min_chunk_size:
- chunks.append(
- Chunk(
- content=chunk_content,
- source=source,
- offset=chunk_start_offset,
- index=len(chunks),
- )
- )
-
- # Keep the overlap tail for the next chunk
- if cfg.chunk_overlap > 0 and len(current_tokens) > cfg.chunk_overlap:
- overlap = current_tokens[-cfg.chunk_overlap :]
- current_tokens = list(overlap)
+ would_overflow = len(current_tokens) + len(para_tokens) > cfg.chunk_size
+
+ # Preserve a paragraph boundary when the buffered paragraph is large
+ # enough to stand alone. A sub-floor lead-in instead stays attached
+ # to the following text; dropping it was the original #754 bug.
+ preserve_sequence = force_final_chunk
+ if current_tokens and would_overflow:
+ if len(current_tokens) >= cfg.min_chunk_size:
+ emit_current(current_offset)
else:
- current_tokens = []
- chunk_start_offset = current_offset
-
- # If a single paragraph exceeds chunk_size, split it directly
- if len(para_tokens) > cfg.chunk_size:
- # Flush anything accumulated first
- if current_tokens:
- chunk_content = " ".join(current_tokens)
- if _count_tokens(chunk_content) >= cfg.min_chunk_size:
- chunks.append(
- Chunk(
- content=chunk_content,
- source=source,
- offset=chunk_start_offset,
- index=len(chunks),
- )
- )
- current_tokens = []
-
- # Split the oversized paragraph into fixed windows
- idx = 0
- while idx < len(para_tokens):
- window = para_tokens[idx : idx + cfg.chunk_size]
- chunk_content = " ".join(window)
- if _count_tokens(chunk_content) >= cfg.min_chunk_size:
- chunks.append(
- Chunk(
- content=chunk_content,
- source=source,
- offset=current_offset + idx,
- index=len(chunks),
- )
- )
- step = max(1, cfg.chunk_size - cfg.chunk_overlap)
- idx += step
-
- current_offset += len(para_tokens)
- chunk_start_offset = current_offset
- continue
-
- current_tokens.extend(para_tokens)
+ preserve_sequence = True
+
+ para_index = 0
+ while para_index < len(para_tokens):
+ capacity = cfg.chunk_size - len(current_tokens)
+ remaining = len(para_tokens) - para_index
+
+ if remaining <= capacity:
+ current_tokens.extend(para_tokens[para_index:])
+ para_index = len(para_tokens)
+ force_final_chunk = preserve_sequence
+ break
+
+ take = capacity
+ remainder = remaining - take
+ overlap = min(cfg.chunk_overlap, max(0, cfg.chunk_size - 1))
+
+ # With no (or a small) configured overlap, a full window can
+ # strand a sub-floor tail. Rebalance the two windows when both
+ # can satisfy the minimum; otherwise the preservation flag below
+ # keeps the unavoidable short tail rather than losing content.
+ tail_size = overlap + remainder
+ if preserve_sequence and tail_size < cfg.min_chunk_size:
+ shift = cfg.min_chunk_size - tail_size
+ if len(current_tokens) + take - shift >= cfg.min_chunk_size:
+ take -= shift
+
+ current_tokens.extend(para_tokens[para_index : para_index + take])
+ para_index += take
+ emit_current(current_offset + para_index)
+ force_final_chunk = preserve_sequence
+
current_offset += len(para_tokens)
- # Flush remaining tokens.
- #
- # ``min_chunk_size`` exists to discard tiny *trailing* fragments once a
- # document has already produced at least one chunk. It must NOT silently
- # drop an entire short document: indexing a folder of short notes would
- # otherwise report success while storing nothing (#502 follow-up). So if no
- # chunk has been emitted yet, keep the remaining content regardless of the
- # floor.
+ # The buffer contains new document content, even when it also retains an
+ # overlap. The minimum guides boundary selection; it must not discard a
+ # short final paragraph or the end of an oversized paragraph (#754).
if current_tokens:
- chunk_content = " ".join(current_tokens)
- if not chunks or _count_tokens(chunk_content) >= cfg.min_chunk_size:
- chunks.append(
- Chunk(
- content=chunk_content,
- source=source,
- offset=chunk_start_offset,
- index=len(chunks),
- )
+ chunks.append(
+ Chunk(
+ content=" ".join(current_tokens),
+ source=source,
+ offset=chunk_start_offset,
+ index=len(chunks),
)
+ )
return chunks
diff --git a/src/openjarvis/tools/storage_tools.py b/src/openjarvis/tools/storage_tools.py
index fd588fef..04cd5c3d 100644
--- a/src/openjarvis/tools/storage_tools.py
+++ b/src/openjarvis/tools/storage_tools.py
@@ -243,10 +243,12 @@ class MemoryIndexTool(BaseTool):
},
"chunk_size": {
"type": "integer",
+ "minimum": 1,
"description": "Chunk size in characters (default 512).",
},
"chunk_overlap": {
"type": "integer",
+ "minimum": 0,
"description": "Overlap between chunks (default 64).",
},
},
@@ -281,9 +283,11 @@ class MemoryIndexTool(BaseTool):
from openjarvis.tools.storage.chunking import ChunkConfig
from openjarvis.tools.storage.ingest import ingest_path
+ chunk_size = params.get("chunk_size", 512)
+ chunk_overlap = params.get("chunk_overlap", 64)
chunk_cfg = ChunkConfig(
- chunk_size=int(params.get("chunk_size", 512)),
- chunk_overlap=int(params.get("chunk_overlap", 64)),
+ chunk_size=chunk_size,
+ chunk_overlap=chunk_overlap,
)
chunks = ingest_path(Path(path), config=chunk_cfg)
stored = 0
diff --git a/tests/cli/test_memory_cmd.py b/tests/cli/test_memory_cmd.py
index 9419651e..50d80595 100644
--- a/tests/cli/test_memory_cmd.py
+++ b/tests/cli/test_memory_cmd.py
@@ -76,6 +76,25 @@ def test_memory_index_nonexistent(tmp_path: Path):
assert result.exit_code != 0
+def test_memory_index_rejects_non_progress_chunk_values(tmp_path: Path):
+ doc = tmp_path / "doc.txt"
+ doc.write_text("a b", encoding="utf-8")
+
+ zero_size = CliRunner().invoke(
+ cli,
+ ["memory", "index", str(doc), "--chunk-size", "0"],
+ )
+ negative_overlap = CliRunner().invoke(
+ cli,
+ ["memory", "index", str(doc), "--chunk-overlap", "-1"],
+ )
+
+ assert zero_size.exit_code != 0
+ assert "0 is not in the range x>=1" in zero_size.output
+ assert negative_overlap.exit_code != 0
+ assert "-1 is not in the range x>=0" in negative_overlap.output
+
+
def test_memory_search_returns_results(tmp_path: Path, monkeypatch):
"""Search returns results from pre-populated backend."""
_register_sqlite()
diff --git a/tests/memory/test_chunking.py b/tests/memory/test_chunking.py
index 7f6b4741..4ebefeee 100644
--- a/tests/memory/test_chunking.py
+++ b/tests/memory/test_chunking.py
@@ -2,6 +2,8 @@
from __future__ import annotations
+import pytest
+
from openjarvis.tools.storage.chunking import ChunkConfig, chunk_text
@@ -13,6 +15,55 @@ def test_whitespace_only_returns_empty():
assert chunk_text(" \n\n ") == []
+ ("kwargs", "message"),
+ [
+ ({"chunk_size": 0}, "chunk_size must be an integer >= 1"),
+ ({"chunk_size": -1}, "chunk_size must be an integer >= 1"),
+ ({"chunk_size": True}, "chunk_size must be an integer >= 1"),
+ ({"chunk_overlap": -1}, "chunk_overlap must be an integer >= 0"),
+ ({"chunk_overlap": False}, "chunk_overlap must be an integer >= 0"),
+ ],
+)
+def test_invalid_chunk_config_is_rejected(kwargs, message):
+ with pytest.raises(ValueError, match=message):
+ ChunkConfig(**kwargs)
+
+
+def test_mutated_invalid_config_is_rejected_before_chunking():
+ cfg = ChunkConfig(chunk_size=2, chunk_overlap=0, min_chunk_size=1)
+ cfg.chunk_size = 0
+
+ with pytest.raises(ValueError, match="chunk_size"):
+ chunk_text("a b", config=cfg)
+
+
+def test_valid_boundary_sweep_preserves_content_size_and_offsets():
+ tokens = [f"t{i}" for i in range(23)]
+ text = " ".join(tokens[:3]) + "\n\n" + " ".join(tokens[3:])
+
+ for chunk_size in range(1, 9):
+ for chunk_overlap in range(0, 11):
+ chunks = chunk_text(
+ text,
+ config=ChunkConfig(
+ chunk_size=chunk_size,
+ chunk_overlap=chunk_overlap,
+ min_chunk_size=1,
+ ),
+ )
+ emitted = []
+ for chunk in chunks:
+ chunk_tokens = chunk.content.split()
+ emitted.extend(chunk_tokens)
+ assert 0 < len(chunk_tokens) <= chunk_size
+ assert tokens[chunk.offset : chunk.offset + len(chunk_tokens)] == (
+ chunk_tokens
+ )
+ assert set(tokens).issubset(emitted)
+ assert [chunk.index for chunk in chunks] == list(range(len(chunks)))
+
+
def test_short_text_single_chunk():
# Need >= 50 words (default min_chunk_size)
words = [f"word{i}" for i in range(60)]
@@ -71,7 +122,7 @@ def test_short_only_document_not_dropped():
Regression for #502 follow-up: previously a folder of short notes indexed
to ``chunks_indexed: 0`` (HTTP 200), silently storing nothing. ``min_chunk_size``
- should only discard tiny *trailing fragments*, never an entire short doc.
+ must not discard an entire short document.
"""
cfg = ChunkConfig(chunk_size=100, chunk_overlap=0, min_chunk_size=50)
# 30 words is below min_chunk_size=50, but it's the entire document.
@@ -89,17 +140,102 @@ def test_short_real_world_note_not_dropped():
assert "hello world" in chunks[0].content
-def test_min_chunk_size_filters_tiny_trailing_fragment():
- """A tiny fragment trailing a real chunk is still dropped by the floor."""
+def test_min_chunk_size_preserves_short_final_paragraph():
+ """A short final paragraph must be searchable after indexing (#754)."""
cfg = ChunkConfig(chunk_size=50, chunk_overlap=0, min_chunk_size=10)
# Two paragraphs: the first fills a real chunk, the second is a tiny tail.
para1 = " ".join(f"a{i}" for i in range(50))
para2 = " ".join(f"b{i}" for i in range(3)) # 3 words < min_chunk_size=10
text = f"{para1}\n\n{para2}"
chunks = chunk_text(text, config=cfg)
- # The 3-word trailing fragment is discarded; only the real chunk remains.
- assert len(chunks) == 1
- assert "b0" not in chunks[0].content
+ assert len(chunks) == 2
+ assert chunks[1].content == para2
+ assert chunks[1].offset == 50
+
+
[email protected]("paragraph_lengths", [(11,), (5, 2), (2, 9, 1), (5, 5, 1)])
[email protected]("overlap", [0, 1, 4, 7])
+def test_short_tails_preserve_complete_document_and_offsets(paragraph_lengths, overlap):
+ """Every input token remains covered, including sub-floor final tails."""
+ tokens = [f"word{i}" for i in range(sum(paragraph_lengths))]
+ paragraphs = []
+ start = 0
+ for length in paragraph_lengths:
+ paragraphs.append(" ".join(tokens[start : start + length]))
+ start += length
+ chunks = chunk_text(
+ "\n\n".join(paragraphs),
+ config=ChunkConfig(chunk_size=5, chunk_overlap=overlap, min_chunk_size=4),
+ )
+ covered = set()
+ for index, chunk in enumerate(chunks):
+ words = chunk.content.split()
+ assert 0 < len(words) <= 5
+ assert chunk.index == index
+ assert words == tokens[chunk.offset : chunk.offset + len(words)]
+ covered.update(range(chunk.offset, chunk.offset + len(words)))
+ assert covered == set(range(len(tokens)))
+
+
+def test_short_lead_in_before_oversized_paragraph_is_not_dropped():
+ """A sub-floor lead-in must be carried into an oversized window (#754)."""
+ lead_in = "IMPORTANT my API key rotation policy is documented here"
+ large_paragraph = " ".join(f"word{i}" for i in range(600))
+
+ chunks = chunk_text(f"{lead_in}\n\n{large_paragraph}")
+
+ assert any(lead_in in chunk.content for chunk in chunks)
+ output_tokens = {token for chunk in chunks for token in chunk.content.split()}
+ assert set(lead_in.split()).issubset(output_tokens)
+ assert {f"word{i}" for i in range(600)}.issubset(output_tokens)
+
+
+def test_short_paragraph_before_full_normal_paragraph_is_not_dropped():
+ """The ordinary paragraph-boundary flush also preserves sub-floor text."""
+ cfg = ChunkConfig(chunk_size=10, chunk_overlap=0, min_chunk_size=5)
+ short = "keep every word"
+ full = " ".join(f"next{i}" for i in range(10))
+
+ chunks = chunk_text(f"{short}\n\n{full}", config=cfg)
+
+ assert any(short in chunk.content for chunk in chunks)
+ assert set(short.split()) | set(full.split()) == {
+ token for chunk in chunks for token in chunk.content.split()
+ }
+ assert all(len(chunk.content.split()) <= cfg.chunk_size for chunk in chunks)
+
+
+ ("chunk_size", "chunk_overlap", "min_chunk_size", "lead_size", "body_size"),
+ [
+ (10, 0, 5, 3, 10),
+ (10, 2, 5, 3, 10),
+ (8, 7, 6, 2, 8),
+ (7, 0, 4, 3, 12),
+ (16, 4, 8, 4, 33),
+ ],
+)
+def test_preserved_short_lead_in_never_breaks_hard_chunk_bound(
+ chunk_size, chunk_overlap, min_chunk_size, lead_size, body_size
+):
+ """Adversarial boundaries preserve every token in bounded chunks."""
+ cfg = ChunkConfig(
+ chunk_size=chunk_size,
+ chunk_overlap=chunk_overlap,
+ min_chunk_size=min_chunk_size,
+ )
+ lead = [f"lead{i}" for i in range(lead_size)]
+ body = [f"body{i}" for i in range(body_size)]
+
+ chunks = chunk_text(
+ f"{' '.join(lead)}\n\n{' '.join(body)}",
+ config=cfg,
+ )
+
+ output_tokens = {token for chunk in chunks for token in chunk.content.split()}
+ assert set(lead + body).issubset(output_tokens)
+ assert all(0 < len(chunk.content.split()) <= chunk_size for chunk in chunks)
+ assert [chunk.index for chunk in chunks] == list(range(len(chunks)))
def test_source_propagated():
diff --git a/tests/tools/test_storage_tools.py b/tests/tools/test_storage_tools.py
index 558ac143..2dfbe4a7 100644
--- a/tests/tools/test_storage_tools.py
+++ b/tests/tools/test_storage_tools.py
@@ -161,6 +161,9 @@ class TestMemoryIndexTool:
def test_spec(self):
tool = MemoryIndexTool()
assert tool.spec.name == "memory_index"
+ properties = tool.spec.parameters["properties"]
+ assert properties["chunk_size"]["minimum"] == 1
+ assert properties["chunk_overlap"]["minimum"] == 0
def test_index_no_backend(self):
tool = MemoryIndexTool()
@@ -196,6 +199,28 @@ class TestMemoryIndexTool:
finally:
os.unlink(path)
+ @pytest.mark.parametrize(
+ "params",
+ [
+ {"chunk_size": 0},
+ {"chunk_size": -1},
+ {"chunk_size": True},
+ {"chunk_overlap": -1},
+ {"chunk_overlap": False},
+ ],
+ )
+ def test_index_rejects_invalid_chunk_config(self, backend, params):
+ with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f:
+ f.write("a b c")
+ path = f.name
+ try:
+ result = MemoryIndexTool(backend).execute(path=path, **params)
+ assert result.success is False
+ assert "must be an integer" in result.content
+ assert backend._data == {}
+ finally:
+ os.unlink(path)
+
class TestStorageToolsRegistration:
def test_storage_tools_importable(self):