Skip to content

Personal AI, On Personal Devices

Python
9.0 KB
Raw Blame
"""Tests for the document chunking pipeline."""

from __future__ import annotations

import pytest

from openjarvis.tools.storage.chunking import ChunkConfig, chunk_text


def test_empty_string_returns_empty():
    assert chunk_text("") == []


def test_whitespace_only_returns_empty():
    assert chunk_text("   \n\n  ") == []


@pytest.mark.parametrize(
    ("kwargs", "message"),
    [
        ({"chunk_size": 0}, "chunk_size must be an integer >= 1"),
        ({"chunk_size": -1}, "chunk_size must be an integer >= 1"),
        ({"chunk_size": True}, "chunk_size must be an integer >= 1"),
        ({"chunk_overlap": -1}, "chunk_overlap must be an integer >= 0"),
        ({"chunk_overlap": False}, "chunk_overlap must be an integer >= 0"),
    ],
)
def test_invalid_chunk_config_is_rejected(kwargs, message):
    with pytest.raises(ValueError, match=message):
        ChunkConfig(**kwargs)


def test_mutated_invalid_config_is_rejected_before_chunking():
    cfg = ChunkConfig(chunk_size=2, chunk_overlap=0, min_chunk_size=1)
    cfg.chunk_size = 0

    with pytest.raises(ValueError, match="chunk_size"):
        chunk_text("a b", config=cfg)


def test_valid_boundary_sweep_preserves_content_size_and_offsets():
    tokens = [f"t{i}" for i in range(23)]
    text = " ".join(tokens[:3]) + "\n\n" + " ".join(tokens[3:])

    for chunk_size in range(1, 9):
        for chunk_overlap in range(0, 11):
            chunks = chunk_text(
                text,
                config=ChunkConfig(
                    chunk_size=chunk_size,
                    chunk_overlap=chunk_overlap,
                    min_chunk_size=1,
                ),
            )
            emitted = []
            for chunk in chunks:
                chunk_tokens = chunk.content.split()
                emitted.extend(chunk_tokens)
                assert 0 < len(chunk_tokens) <= chunk_size
                assert tokens[chunk.offset : chunk.offset + len(chunk_tokens)] == (
                    chunk_tokens
                )
            assert set(tokens).issubset(emitted)
            assert [chunk.index for chunk in chunks] == list(range(len(chunks)))


def test_short_text_single_chunk():
    # Need >= 50 words (default min_chunk_size)
    words = [f"word{i}" for i in range(60)]
    text = " ".join(words)
    chunks = chunk_text(text, source="test.txt")
    assert len(chunks) == 1
    assert chunks[0].source == "test.txt"
    assert chunks[0].index == 0
    assert "word0" in chunks[0].content


def test_long_text_multiple_chunks():
    # Build text that exceeds 512 tokens
    words = [f"word{i}" for i in range(600)]
    text = " ".join(words)
    chunks = chunk_text(text)
    assert len(chunks) >= 2


def test_chunk_overlap():
    cfg = ChunkConfig(chunk_size=100, chunk_overlap=20, min_chunk_size=5)
    words = [f"w{i}" for i in range(250)]
    text = " ".join(words)
    chunks = chunk_text(text, config=cfg)
    assert len(chunks) >= 2

    # The end of chunk 0 and start of chunk 1 should overlap
    first_tokens = chunks[0].content.split()
    second_tokens = chunks[1].content.split()
    tail = first_tokens[-20:]
    head = second_tokens[:20]
    assert tail == head


def test_paragraph_boundary_respected():
    cfg = ChunkConfig(chunk_size=20, chunk_overlap=0, min_chunk_size=3)
    para1 = " ".join(f"a{i}" for i in range(10))
    para2 = " ".join(f"b{i}" for i in range(10))
    text = f"{para1}\n\n{para2}"
    chunks = chunk_text(text, config=cfg)
    # Both paragraphs fit in one chunk (10 + 10 = 20 <= 20)
    assert len(chunks) == 1


def test_custom_config():
    cfg = ChunkConfig(chunk_size=50, chunk_overlap=10, min_chunk_size=5)
    words = [f"tok{i}" for i in range(200)]
    text = " ".join(words)
    chunks = chunk_text(text, config=cfg)
    # Should produce multiple chunks
    assert len(chunks) >= 3


def test_short_only_document_not_dropped():
    """A whole document below min_chunk_size must still produce a chunk.

    Regression for #502 follow-up: previously a folder of short notes indexed
    to ``chunks_indexed: 0`` (HTTP 200), silently storing nothing. ``min_chunk_size``
    must not discard an entire short document.
    """
    cfg = ChunkConfig(chunk_size=100, chunk_overlap=0, min_chunk_size=50)
    # 30 words is below min_chunk_size=50, but it's the entire document.
    words = [f"w{i}" for i in range(30)]
    text = " ".join(words)
    chunks = chunk_text(text, config=cfg)
    assert len(chunks) == 1
    assert chunks[0].content == text


def test_short_real_world_note_not_dropped():
    """The exact repro from the issue: a ~4-word note must not vanish."""
    chunks = chunk_text("hello world\nsome content\n", source="a.txt")
    assert len(chunks) == 1
    assert "hello world" in chunks[0].content


def test_min_chunk_size_preserves_short_final_paragraph():
    """A short final paragraph must be searchable after indexing (#754)."""
    cfg = ChunkConfig(chunk_size=50, chunk_overlap=0, min_chunk_size=10)
    # Two paragraphs: the first fills a real chunk, the second is a tiny tail.
    para1 = " ".join(f"a{i}" for i in range(50))
    para2 = " ".join(f"b{i}" for i in range(3))  # 3 words < min_chunk_size=10
    text = f"{para1}\n\n{para2}"
    chunks = chunk_text(text, config=cfg)
    assert len(chunks) == 2
    assert chunks[1].content == para2
    assert chunks[1].offset == 50


@pytest.mark.parametrize("paragraph_lengths", [(11,), (5, 2), (2, 9, 1), (5, 5, 1)])
@pytest.mark.parametrize("overlap", [0, 1, 4, 7])
def test_short_tails_preserve_complete_document_and_offsets(paragraph_lengths, overlap):
    """Every input token remains covered, including sub-floor final tails."""
    tokens = [f"word{i}" for i in range(sum(paragraph_lengths))]
    paragraphs = []
    start = 0
    for length in paragraph_lengths:
        paragraphs.append(" ".join(tokens[start : start + length]))
        start += length
    chunks = chunk_text(
        "\n\n".join(paragraphs),
        config=ChunkConfig(chunk_size=5, chunk_overlap=overlap, min_chunk_size=4),
    )
    covered = set()
    for index, chunk in enumerate(chunks):
        words = chunk.content.split()
        assert 0 < len(words) <= 5
        assert chunk.index == index
        assert words == tokens[chunk.offset : chunk.offset + len(words)]
        covered.update(range(chunk.offset, chunk.offset + len(words)))
    assert covered == set(range(len(tokens)))


def test_short_lead_in_before_oversized_paragraph_is_not_dropped():
    """A sub-floor lead-in must be carried into an oversized window (#754)."""
    lead_in = "IMPORTANT my API key rotation policy is documented here"
    large_paragraph = " ".join(f"word{i}" for i in range(600))

    chunks = chunk_text(f"{lead_in}\n\n{large_paragraph}")

    assert any(lead_in in chunk.content for chunk in chunks)
    output_tokens = {token for chunk in chunks for token in chunk.content.split()}
    assert set(lead_in.split()).issubset(output_tokens)
    assert {f"word{i}" for i in range(600)}.issubset(output_tokens)


def test_short_paragraph_before_full_normal_paragraph_is_not_dropped():
    """The ordinary paragraph-boundary flush also preserves sub-floor text."""
    cfg = ChunkConfig(chunk_size=10, chunk_overlap=0, min_chunk_size=5)
    short = "keep every word"
    full = " ".join(f"next{i}" for i in range(10))

    chunks = chunk_text(f"{short}\n\n{full}", config=cfg)

    assert any(short in chunk.content for chunk in chunks)
    assert set(short.split()) | set(full.split()) == {
        token for chunk in chunks for token in chunk.content.split()
    }
    assert all(len(chunk.content.split()) <= cfg.chunk_size for chunk in chunks)


@pytest.mark.parametrize(
    ("chunk_size", "chunk_overlap", "min_chunk_size", "lead_size", "body_size"),
    [
        (10, 0, 5, 3, 10),
        (10, 2, 5, 3, 10),
        (8, 7, 6, 2, 8),
        (7, 0, 4, 3, 12),
        (16, 4, 8, 4, 33),
    ],
)
def test_preserved_short_lead_in_never_breaks_hard_chunk_bound(
    chunk_size, chunk_overlap, min_chunk_size, lead_size, body_size
):
    """Adversarial boundaries preserve every token in bounded chunks."""
    cfg = ChunkConfig(
        chunk_size=chunk_size,
        chunk_overlap=chunk_overlap,
        min_chunk_size=min_chunk_size,
    )
    lead = [f"lead{i}" for i in range(lead_size)]
    body = [f"body{i}" for i in range(body_size)]

    chunks = chunk_text(
        f"{' '.join(lead)}\n\n{' '.join(body)}",
        config=cfg,
    )

    output_tokens = {token for chunk in chunks for token in chunk.content.split()}
    assert set(lead + body).issubset(output_tokens)
    assert all(0 < len(chunk.content.split()) <= chunk_size for chunk in chunks)
    assert [chunk.index for chunk in chunks] == list(range(len(chunks)))


def test_source_propagated():
    words = [f"word{i}" for i in range(60)]
    text = " ".join(words)
    chunks = chunk_text(text, source="myfile.md")
    assert len(chunks) == 1
    assert chunks[0].source == "myfile.md"


def test_chunk_index_sequential():
    cfg = ChunkConfig(chunk_size=50, chunk_overlap=0, min_chunk_size=5)
    words = [f"w{i}" for i in range(200)]
    text = " ".join(words)
    chunks = chunk_text(text, config=cfg)
    for i, chunk in enumerate(chunks):
        assert chunk.index == i