Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
6.0 KB
Raw Blame
"""The ingest flow's Split Text component never returns a chunk over chunk_size.

Docling joins consecutive lines that have no blank line between them into one
Markdown paragraph, so a plain-text file with one sentence per line reaches the
splitter as a few huge newline-free paragraphs. Langflow's built-in Split Text
only cuts on its separator, and kept each such paragraph as a single chunk of
hundreds of thousands of characters.
"""

import hashlib
import importlib
import importlib.util
import json
import sys
from pathlib import Path
from types import ModuleType

import pytest
from langchain_core.documents import Document

BUNDLE = Path("custom_components/openrag")
SOURCE = BUNDLE / "split_text.py"
NODE_TYPE = "ext:openrag:OpenRAGSplitTextComponent@extra"
FLOWS_WITH_SPLITTER = ("ingestion_flow.json", "openrag_url_mcp.json")


class FakeData:
    def __init__(self, text: str = "", data: dict | None = None) -> None:
        self.text = text
        self.data = data or {}
        self.text_key = "text"

    def to_lc_document(self) -> Document:
        return Document(page_content=self.text, metadata=dict(self.data))


class FakeInput:
    def __init__(self, **kwargs: object) -> None:
        self.name = kwargs.get("name")


@pytest.fixture
def component_module(monkeypatch: pytest.MonkeyPatch):
    modules = {
        name: ModuleType(name)
        for name in (
            "lfx",
            "lfx.custom",
            "lfx.custom.custom_component",
            "lfx.custom.custom_component.component",
            "lfx.io",
            "lfx.schema",
            "lfx.schema.data",
            "lfx.schema.dataframe",
            "lfx.schema.message",
            "lfx.utils",
            "lfx.utils.util",
        )
    }
    vars(modules["lfx.custom.custom_component.component"])["Component"] = object
    for input_name in (
        "BoolInput",
        "DropdownInput",
        "HandleInput",
        "IntInput",
        "MessageTextInput",
        "Output",
    ):
        setattr(modules["lfx.io"], input_name, FakeInput)
    vars(modules["lfx.schema.data"])["Data"] = FakeData
    vars(modules["lfx.schema.dataframe"])["DataFrame"] = type("DataFrame", (), {})
    vars(modules["lfx.schema.message"])["Message"] = type("Message", (), {})
    vars(modules["lfx.utils.util"])["unescape_string"] = lambda s: s.replace("\\n", "\n")
    for name, module in modules.items():
        monkeypatch.setitem(sys.modules, name, module)

    module_name = "custom_components.openrag.split_text"
    sys.modules.pop(module_name, None)
    yield importlib.import_module(module_name)
    sys.modules.pop(module_name, None)


def _split(module, text: str, *, chunk_size=1000, chunk_overlap=200, separator="\\n"):
    component = module.OpenRAGSplitTextComponent()
    component.data_inputs = [FakeData(text, {"filename": "wiki.txt"})]
    component.chunk_size = chunk_size
    component.chunk_overlap = chunk_overlap
    component.separator = separator
    component.text_key = "text"
    component.keep_separator = "False"
    return component.split_text_base()


def _sentences(count: int) -> list[str]:
    return [f"東京は日本の首都であり、第{i}番目の文はここで終わる。" for i in range(count)]


def test_newline_free_paragraph_is_split_to_chunk_size(component_module):
    sentences = _sentences(5000)
    text = " ".join(sentences)
    assert "\n" not in text

    docs = _split(component_module, text)

    assert len(docs) > 1
    assert max(len(doc.page_content) for doc in docs) <= 1000
    joined = "\n".join(doc.page_content for doc in docs)
    # With Keep Separator off, a "。" that falls on a chunk boundary is dropped.
    assert all(sentence.rstrip("。") in joined for sentence in sentences)
    assert all(doc.metadata == {"filename": "wiki.txt"} for doc in docs)


def test_single_long_line_among_short_lines_is_split(component_module):
    long_line = "x" * 5000
    text = "\n".join(["short line one", long_line, "short line two"])

    docs = _split(component_module, text)

    assert max(len(doc.page_content) for doc in docs) <= 1000
    assert sum(doc.page_content.count("x") for doc in docs) >= 5000


def test_configured_separator_is_still_preferred(component_module):
    lines = [f"line {i:03d} " + "y" * 80 for i in range(40)]

    docs = _split(component_module, "\n".join(lines))

    for doc in docs:
        assert all(part in lines for part in doc.page_content.split("\n"))


def test_separators_try_the_configured_one_first_without_duplicates(component_module):
    separators = component_module.build_separators("\n")

    assert separators[0] == "\n"
    assert separators[-1] == ""
    assert len(separators) == len(set(separators))
    assert "。" in separators


def test_component_type_differs_from_langflow_builtin():
    """Restricted Langflow swaps drifted code of a known built-in type for its own copy."""
    source = SOURCE.read_text(encoding="utf-8")
    assert 'name = "OpenRAGSplitText"' in source
    assert 'display_name: str = "Split Text"' in source


@pytest.mark.parametrize("flow_name", FLOWS_WITH_SPLITTER)
def test_flows_embed_the_openrag_split_text_component(flow_name):
    source = SOURCE.read_text(encoding="utf-8")
    expected_hash = hashlib.sha256(source.encode("utf-8")).hexdigest()[:12]
    flow = json.loads((Path("flows") / flow_name).read_text(encoding="utf-8"))
    nodes = flow["data"]["nodes"]

    assert not [n for n in nodes if n["data"].get("type") == "SplitText"]
    splitters = [n for n in nodes if n["data"].get("type") == NODE_TYPE]
    assert len(splitters) == 1
    component = splitters[0]["data"]["node"]
    # flows_service and the ingest tweaks find this node by display name
    assert component["display_name"] == "Split Text"
    assert component["template"]["code"]["value"] == source
    assert component["metadata"]["code_hash"] == expected_hash

    node_id = splitters[0]["id"]
    for edge in flow["data"]["edges"]:
        if edge["source"] == node_id:
            assert edge["data"]["sourceHandle"]["dataType"] == NODE_TYPE
            assert "œdataTypeœ:œSplitTextœ" not in edge["sourceHandle"]