Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

3.8 KB
Raw Blame
"""Tests for the character-based chunker with boundary expansion."""

from __future__ import annotations

import pytest

from deeptutor.services.memory.consolidator.chunker import (
    ChunkSpan,
    chunk_with_boundary,
)


def test_empty_input_returns_no_chunks() -> None:
    assert (
        chunk_with_boundary(
            "", budget=5, overlap_ratio=0.1, min_chunk_chars=100, max_chunk_chars=1000
        )
        == []
    )
    assert (
        chunk_with_boundary(
            "    \n\n", budget=5, overlap_ratio=0.1, min_chunk_chars=100, max_chunk_chars=1000
        )
        == []
    )


def test_short_input_yields_one_chunk_covering_everything() -> None:
    chunks = chunk_with_boundary(
        "Hello world.", budget=5, overlap_ratio=0.1, min_chunk_chars=100, max_chunk_chars=1000
    )
    assert len(chunks) == 1
    assert chunks[0].start == 0
    assert chunks[0].end == len("Hello world.")


def test_paragraph_boundary_extends_right_edge() -> None:
    text = "A" * 100 + "\n\n" + "B" * 100 + "\n\n" + "C" * 100
    chunks = chunk_with_boundary(
        text,
        budget=2,
        overlap_ratio=0.0,
        min_chunk_chars=50,
        max_chunk_chars=400,
        boundary="paragraph",
    )
    # First chunk's end should land at a paragraph boundary (or end of text when budget=1).
    assert chunks[0].text.endswith("\n\n") or chunks[0].end == len(text)
    # Nothing is left out across the chunks (allowing for overlap).
    assert chunks[-1].end == len(text)


def test_sentence_boundary_used_when_configured() -> None:
    sentences = " ".join(f"Sentence number {i}." for i in range(40))
    chunks = chunk_with_boundary(
        sentences,
        budget=4,
        overlap_ratio=0.05,
        min_chunk_chars=80,
        max_chunk_chars=400,
        boundary="sentence",
    )
    # Each non-last chunk ends with terminal punctuation + space (the
    # sentence boundary regex matches `.` followed by space).
    for chunk in chunks[:-1]:
        assert chunk.text[-1].isspace() or chunk.text.rstrip()[-1] in ".!?"


def test_budget_caps_chunk_count() -> None:
    text = ("paragraph body. " * 20 + "\n\n") * 30  # ~ 10k chars
    chunks = chunk_with_boundary(
        text, budget=5, overlap_ratio=0.1, min_chunk_chars=200, max_chunk_chars=8000
    )
    assert len(chunks) <= 5


def test_overlap_visible_in_adjacent_chunks() -> None:
    text = ("paragraph body. " * 20 + "\n\n") * 10
    chunks = chunk_with_boundary(
        text, budget=3, overlap_ratio=0.2, min_chunk_chars=200, max_chunk_chars=2000
    )
    if len(chunks) >= 2:
        # The second chunk should start before the first ends.
        assert chunks[1].start < chunks[0].end


def test_min_chunk_chars_protects_against_tiny_chunks() -> None:
    text = "x" * 500
    chunks = chunk_with_boundary(
        text, budget=50, overlap_ratio=0.0, min_chunk_chars=400, max_chunk_chars=1000
    )
    # 500 chars / budget=50 would suggest 10-char chunks; min_chunk_chars
    # raises that to 400, so we get 1-2 chunks.
    assert len(chunks) <= 2


def test_max_chunk_chars_protects_against_huge_chunks() -> None:
    text = "x" * 100_000
    chunks = chunk_with_boundary(
        text, budget=1, overlap_ratio=0.0, min_chunk_chars=200, max_chunk_chars=5000
    )
    # 1 chunk is impossible if max=5000 and input=100k; multiple chunks expected.
    assert len(chunks) > 1
    assert all(c.end - c.start <= 5000 * 2 for c in chunks)  # boundary extension may push over


@pytest.mark.parametrize("budget", [1, 5, 20, 100])
def test_no_data_loss_chunks_cover_full_text(budget: int) -> None:
    text = "\n\n".join(f"para {i}: " + "x" * 50 for i in range(50))
    chunks = chunk_with_boundary(
        text, budget=budget, overlap_ratio=0.1, min_chunk_chars=100, max_chunk_chars=2000
    )
    assert chunks[0].start == 0
    assert chunks[-1].end == len(text)