OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
import importlib
import json
import sys
from types import ModuleType, SimpleNamespace
import pytest
def _load_opensearch_module(monkeypatch: pytest.MonkeyPatch):
class Input:
def __init__(self, *args: object, **kwargs: object) -> None:
self.name = kwargs.get("name", "")
class VectorStore:
inputs: list[object] = []
class Data(dict):
def __init__(self, text=None, **kwargs):
super().__init__(text=text, **kwargs)
class Table(list):
pass
logger = SimpleNamespace(
info=lambda *args, **kwargs: None,
warning=lambda *args, **kwargs: None,
error=lambda *args, **kwargs: None,
debug=lambda *args, **kwargs: None,
)
modules = {
"lfx": ModuleType("lfx"),
"lfx.base": ModuleType("lfx.base"),
"lfx.base.vectorstores": ModuleType("lfx.base.vectorstores"),
"lfx.base.vectorstores.model": ModuleType("lfx.base.vectorstores.model"),
"lfx.base.vectorstores.vector_store_connection_decorator": ModuleType(
"lfx.base.vectorstores.vector_store_connection_decorator"
),
"lfx.io": ModuleType("lfx.io"),
"lfx.log": ModuleType("lfx.log"),
"lfx.schema": ModuleType("lfx.schema"),
"lfx.schema.data": ModuleType("lfx.schema.data"),
"lfx.schema.dataframe": ModuleType("lfx.schema.dataframe"),
}
vars(modules["lfx.base.vectorstores.model"])["LCVectorStoreComponent"] = VectorStore
vars(modules["lfx.base.vectorstores.model"])["check_cached_vector_store"] = lambda func: func
vars(modules["lfx.base.vectorstores.vector_store_connection_decorator"])[
"vector_store_connection"
] = lambda cls: cls
for name in (
"BoolInput",
"DropdownInput",
"HandleInput",
"IntInput",
"MultilineInput",
"Output",
"SecretStrInput",
"StrInput",
"TableInput",
):
setattr(modules["lfx.io"], name, Input)
vars(modules["lfx.log"])["logger"] = logger
vars(modules["lfx.schema.data"])["Data"] = Data
vars(modules["lfx.schema.dataframe"])["Table"] = Table
for name, module in modules.items():
monkeypatch.setitem(sys.modules, name, module)
sys.modules.pop("custom_components.openrag.opensearch_multimodal", None)
return importlib.import_module("custom_components.openrag.opensearch_multimodal")
class _Indices:
def __init__(self, fields):
self.fields = fields
def get_mapping(self, *, index):
return {index: {"mappings": {"properties": self.fields}}}
class _SearchClient:
def __init__(self, aggregation, fields):
self.aggregation = aggregation
self.indices = _Indices(fields)
self.query = None
def search(self, *, index, body, params):
if body.get("size") == 0:
aggregation_name = next(iter(body["aggs"]))
return {
"aggregations": {
aggregation_name: self.aggregation["aggregations"].get(
aggregation_name, {"buckets": []}
)
}
}
self.query = body
return {"hits": {"hits": []}}
def _component(module, client, embedding):
component = module.OpenSearchVectorStoreComponentMultimodalMultiEmbedding()
component.embedding = embedding
component.index_name = "documents"
component.ingest_data = None
component.filter_expression = ""
component.number_of_results = 10
component.num_candidates = 0
component.vector_field = "chunk_embedding"
component.build_client = lambda: client
component.log = lambda message: None
return component
def test_detection_separates_provider_qualified_and_legacy_spaces(
monkeypatch: pytest.MonkeyPatch,
) -> None:
module = _load_opensearch_module(monkeypatch)
result = {
"aggregations": {
"embedding_spaces": {
"buckets": [{"key": {"space_id": "azure:text-embedding-3-small"}, "doc_count": 4}]
},
"legacy_embedding_models": {
"buckets": [{"key": {"model": "text-embedding-3-small"}, "doc_count": 3}],
},
}
}
spaces = module.embedding_spaces_from_aggregation(result)
assert [(space.space_id, space.route_model, space.field_identity) for space in spaces] == [
(
"azure:text-embedding-3-small",
"space:azure:text-embedding-3-small",
"azure:text-embedding-3-small",
),
(
"legacy:text-embedding-3-small",
"legacy:text-embedding-3-small",
"text-embedding-3-small",
),
]
def test_detection_ignores_missing_bucket_keys(monkeypatch: pytest.MonkeyPatch) -> None:
module = _load_opensearch_module(monkeypatch)
result = {
"aggregations": {
"embedding_spaces": {"buckets": [{"key": {}}, {"key": None}]},
"legacy_embedding_models": {"buckets": [{"key": {}}, {"key": None}]},
}
}
assert module.embedding_spaces_from_aggregation(result) == []
def test_detection_aggregation_only_treats_documents_without_space_id_as_legacy(
monkeypatch: pytest.MonkeyPatch,
) -> None:
module = _load_opensearch_module(monkeypatch)
aggregation = module.build_embedding_space_aggregation(size=25)
assert aggregation["embedding_spaces"]["composite"] == {
"size": 25,
"sources": [{"space_id": {"terms": {"field": "embedding_space_id"}}}],
}
assert aggregation["legacy_embedding_models"]["composite"] == {
"size": 25,
"sources": [{"model": {"terms": {"field": "embedding_model"}}}],
}
def test_search_queries_every_resolvable_provider_space(monkeypatch: pytest.MonkeyPatch) -> None:
module = _load_opensearch_module(monkeypatch)
aggregation = {
"aggregations": {
"embedding_spaces": {
"buckets": [{"key": {"space_id": "azure:text-embedding-3-small"}, "doc_count": 2}]
},
"legacy_embedding_models": {
"buckets": [{"key": {"model": "text-embedding-3-small"}, "doc_count": 2}]
},
}
}
fields = {
"chunk_embedding_azure_text_embedding_3_small": {
"type": "knn_vector",
"dimension": 2,
},
"chunk_embedding_text_embedding_3_small": {
"type": "knn_vector",
"dimension": 2,
},
"filename": {"type": "keyword"},
}
client = _SearchClient(aggregation, fields)
routes = []
class Child:
def __init__(self, route):
self.model = route
def embed_query(self, query):
return [0.1, 0.2]
class Root(Child):
deployment = "selected-model"
def for_model(self, route):
routes.append(route)
return Child(route)
_component(module, client, Root("selected-model")).search("needle")
assert routes == [
"space:azure:text-embedding-3-small",
"legacy:text-embedding-3-small",
]
knn_queries = client.query["query"]["bool"]["should"][0]["dis_max"]["queries"]
assert {next(iter(query["knn"])) for query in knn_queries} == {
"chunk_embedding_azure_text_embedding_3_small",
"chunk_embedding_text_embedding_3_small",
}
def test_detection_paginates_all_embedding_spaces(monkeypatch: pytest.MonkeyPatch) -> None:
module = _load_opensearch_module(monkeypatch)
class PagingClient:
def __init__(self):
self.calls = []
def search(self, *, index, body, params):
self.calls.append(body)
exact = body["aggs"].get("embedding_spaces")
if exact and exact["composite"].get("after"):
return {
"aggregations": {
"embedding_spaces": {"buckets": [{"key": {"space_id": "watsonx:slate"}}]}
}
}
if exact:
return {
"aggregations": {
"embedding_spaces": {
"buckets": [{"key": {"space_id": "azure:embed-v1"}}],
"after_key": {"space_id": "azure:embed-v1"},
}
}
}
return {
"aggregations": {
"legacy_embedding_models": {"buckets": []},
}
}
client = PagingClient()
component = _component(module, client, None)
spaces = component._detect_available_models(client)
assert [space.space_id for space in spaces] == ["azure:embed-v1", "watsonx:slate"]
assert len(client.calls) == 3
assert "legacy_embedding_models" not in client.calls[1]["aggs"]
assert client.calls[2]["query"]["bool"]["must_not"] == [
{"exists": {"field": "embedding_space_id"}}
]
def test_search_falls_back_to_keywords_when_no_space_resolves(
monkeypatch: pytest.MonkeyPatch,
) -> None:
module = _load_opensearch_module(monkeypatch)
aggregation = {
"aggregations": {
"embedding_spaces": {"buckets": []},
"legacy_embedding_models": {
"buckets": [{"key": {"model": "unknown-model"}, "doc_count": 1}]
},
}
}
client = _SearchClient(aggregation, {"filename": {"type": "keyword"}})
class Root:
model = "selected-model"
deployment = "selected-model"
def for_model(self, route):
raise RuntimeError("unmapped legacy model")
_component(module, client, Root()).search("needle")
should = client.query["query"]["bool"]["should"]
assert len(should) == 1
assert "multi_match" in should[0]
assert client.query["query"]["bool"]["filter"] == []
def test_search_remains_keyword_capable_without_embedding_adapter(
monkeypatch: pytest.MonkeyPatch,
) -> None:
module = _load_opensearch_module(monkeypatch)
aggregation = {
"aggregations": {
"embedding_spaces": {
"buckets": [{"key": {"space_id": "azure:text-embedding-3-small"}, "doc_count": 1}]
},
"legacy_embedding_models": {"buckets": []},
}
}
client = _SearchClient(aggregation, {"filename": {"type": "keyword"}})
_component(module, client, None).search("needle")
assert client.query["query"]["bool"]["should"] == [
{
"multi_match": {
"query": "needle",
"fields": ["text^2", "filename^1.5"],
"type": "best_fields",
"fuzziness": "AUTO",
"boost": 1.0,
}
}
]
def test_direct_ingest_persists_provider_qualified_space(
monkeypatch: pytest.MonkeyPatch,
) -> None:
module = _load_opensearch_module(monkeypatch)
component = module.OpenSearchVectorStoreComponentMultimodalMultiEmbedding()
component._openrag_ingest_callback_config = lambda: None
component.log = lambda message: None
captured = []
monkeypatch.setattr(
module.helpers, "bulk", lambda client, requests, **kwargs: captured.extend(requests)
)
component._bulk_ingest_embeddings(
client=object(),
index_name="documents",
embeddings=[[0.1, 0.2]],
texts=["hello"],
vector_field="chunk_embedding_azure_text_embedding_3_small",
embedding_model="text-embedding-3-small",
embedding_provider="azure",
embedding_space_id="azure:text-embedding-3-small",
)
assert captured[0]["embedding_model"] == "text-embedding-3-small"
assert captured[0]["embedding_provider"] == "azure"
assert captured[0]["embedding_space_id"] == "azure:text-embedding-3-small"
def test_ingest_normalizes_kangxi_radicals_before_embedding_and_indexing(
monkeypatch: pytest.MonkeyPatch,
) -> None:
module = _load_opensearch_module(monkeypatch)
class _Doc:
text_key = "text"
default_value = ""
def model_dump_json(self):
return '{"text": "11 \\u2f49 27 \\u2f47 ( \\u2f4a )", "filename": "slip.pdf"}'
class _WatsonxEmbedding:
model = "ibm/granite-embedding-278m-multilingual"
def __init__(self):
self.embedded = []
def embed_documents(self, texts):
self.embedded.extend(texts)
return [[0.1, 0.2] for _ in texts]
embedding = _WatsonxEmbedding()
component = _component(module, client=object(), embedding=embedding)
component.embedding_model_name = ""
component.docs_metadata = []
component._prepare_ingest_data = lambda: [_Doc()]
component._openrag_ingest_callback_config = lambda: ("url", "token", "task")
indexed = []
def _bulk_ingest(**kwargs):
indexed.extend(kwargs["texts"])
return ["id"]
component._bulk_ingest_embeddings = _bulk_ingest
component._add_documents_to_vector_store(object())
assert embedding.embedded == ["11 月 27 日 ( 木 )"]
assert indexed == ["11 月 27 日 ( 木 )"]
_RANGE = {"range": {"indexed_time": {"gte": "2026-10-07T00:00:00Z", "lte": "2026-10-10T23:59:59Z"}}}
_PDF = {"term": {"mimetype": "application/pdf"}}
@pytest.mark.parametrize(
("filter_obj", "expected"),
[
({"filter": [_RANGE]}, [_RANGE]),
({"filter": [_PDF, _RANGE]}, [_PDF, _RANGE]),
({"filter": [{"range": {"indexed_time": {}}}]}, []),
({"filter": [{"range": {"indexed_time": {"gte": None}}}]}, []),
({"filter": [{"range": "bad"}]}, []),
(
{"filter": [{"range": {"a": {"gte": 1}, "b": {"gte": 2}}}]},
[],
),
],
)
def test_filter_parser_keeps_valid_range_clauses(
monkeypatch: pytest.MonkeyPatch, filter_obj, expected
) -> None:
module = _load_opensearch_module(monkeypatch)
component = module.OpenSearchVectorStoreComponentMultimodalMultiEmbedding()
assert component._coerce_filter_clauses(filter_obj) == expected
assert component._coerce_filter_clauses(json.dumps(filter_obj)) == expected
@pytest.mark.parametrize("clauses", [[_RANGE], [_PDF, _RANGE]], ids=["date-only", "pdf-plus-date"])
def test_search_applies_date_filter_outside_and_inside_each_knn(
monkeypatch: pytest.MonkeyPatch, clauses
) -> None:
module = _load_opensearch_module(monkeypatch)
aggregation = {
"aggregations": {
"embedding_spaces": {
"buckets": [{"key": {"space_id": "azure:text-embedding-3-small"}, "doc_count": 2}]
},
"legacy_embedding_models": {"buckets": []},
}
}
fields = {
"chunk_embedding_azure_text_embedding_3_small": {
"type": "knn_vector",
"dimension": 2,
"method": {"engine": "jvector", "name": "disk_ann"},
}
}
client = _SearchClient(aggregation, fields)
class Child:
def embed_query(self, query):
return [0.1, 0.2]
class Root(Child):
deployment = "selected-model"
def for_model(self, route):
return Child()
component = _component(module, client, Root())
component.filter_expression = json.dumps({"filter": clauses})
component.search("needle")
bool_query = client.query["query"]["bool"]
assert bool_query["filter"] == clauses
knn_queries = bool_query["should"][0]["dis_max"]["queries"]
assert knn_queries
for query in knn_queries:
(knn,) = query["knn"].values()
assert knn["filter"] == {"bool": {"filter": clauses}}
def test_search_without_filter_leaves_knn_unfiltered(monkeypatch: pytest.MonkeyPatch) -> None:
module = _load_opensearch_module(monkeypatch)
aggregation = {
"aggregations": {
"embedding_spaces": {
"buckets": [{"key": {"space_id": "azure:text-embedding-3-small"}, "doc_count": 2}]
},
"legacy_embedding_models": {"buckets": []},
}
}
fields = {
"chunk_embedding_azure_text_embedding_3_small": {"type": "knn_vector", "dimension": 2}
}
client = _SearchClient(aggregation, fields)
class Child:
def embed_query(self, query):
return [0.1, 0.2]
class Root(Child):
deployment = "selected-model"
def for_model(self, route):
return Child()
_component(module, client, Root()).search("needle")
for query in client.query["query"]["bool"]["should"][0]["dis_max"]["queries"]:
(knn,) = query["knn"].values()
assert "filter" not in knn
@pytest.mark.parametrize("use_callback", [True, False], ids=["callback", "direct_write"])
def test_partial_embed_failure_aborts_before_any_write(
monkeypatch: pytest.MonkeyPatch, use_callback: bool
) -> None:
"""A document with a missing chunk must never be indexed, in either mode.
Direct writes carry no ingest_run_id, so the backend cannot clean up
survivors afterwards; a partial document would stay searchable and be
mistaken for a duplicate. The component therefore fails before writing.
"""
module = _load_opensearch_module(monkeypatch)
class _Doc:
text_key = "text"
default_value = ""
def __init__(self, text):
self._text = text
def model_dump_json(self):
return json.dumps({"text": self._text, "document_id": "doc"})
class _Embedding:
model = "text-embedding-3-small"
def embed_documents(self, texts):
if texts == ["bad"]:
raise ValueError("embedding backend rejected the chunk")
return [[0.1, 0.2] for _ in texts]
class _Indices:
def exists(self, **kwargs):
raise AssertionError("index must not be touched")
create = put_mapping = get_mapping = exists
class _Client:
indices = _Indices()
def index(self, **kwargs):
raise AssertionError("direct write must not happen")
component = _component(module, client=_Client(), embedding=_Embedding())
component.embedding_model_name = ""
component.docs_metadata = []
component._prepare_ingest_data = lambda: [_Doc("ok-0"), _Doc("bad"), _Doc("ok-2")]
if use_callback:
component._openrag_ingest_callback_config = lambda: ("url", "token", "task")
else:
component._openrag_ingest_callback_config = lambda: None
writes: list[dict] = []
component._bulk_ingest_embeddings = lambda **kwargs: writes.append(kwargs)
with pytest.raises(RuntimeError, match=r"Embedding failed for 1 of 3 chunks"):
component._add_documents_to_vector_store(_Client())
assert writes == []
@pytest.mark.parametrize(
("engine", "expect_inner"),
[("jvector", True), ("lucene", True), ("faiss", True), ("nmslib", False), (None, False)],
)
def test_inner_knn_filter_only_for_engines_that_support_it(
monkeypatch: pytest.MonkeyPatch, engine, expect_inner
) -> None:
module = _load_opensearch_module(monkeypatch)
aggregation = {
"aggregations": {
"embedding_spaces": {
"buckets": [{"key": {"space_id": "azure:text-embedding-3-small"}, "doc_count": 2}]
},
"legacy_embedding_models": {"buckets": []},
}
}
field = {"type": "knn_vector", "dimension": 2}
if engine:
field["method"] = {"engine": engine, "name": "hnsw"}
client = _SearchClient(aggregation, {"chunk_embedding_azure_text_embedding_3_small": field})
class Child:
def embed_query(self, query):
return [0.1, 0.2]
class Root(Child):
deployment = "selected-model"
def for_model(self, route):
return Child()
component = _component(module, client, Root())
component.filter_expression = json.dumps({"filter": [_RANGE]})
component.search("needle")
bool_query = client.query["query"]["bool"]
assert bool_query["filter"] == [_RANGE] # the outer filter is always kept
(clause,) = bool_query["should"][0]["dis_max"]["queries"]
if expect_inner:
(knn,) = clause["knn"].values()
assert knn["filter"] == {"bool": {"filter": [_RANGE]}}
else:
# nmslib cannot pre-filter: score the filtered documents exactly instead.
assert "knn" not in clause
script_score = clause["script_score"]
assert _RANGE in script_score["query"]["bool"]["filter"]
assert script_score["script"]["lang"] == "knn"
assert script_score["script"]["params"]["field"].startswith("chunk_embedding_")
def test_unfiltered_nmslib_search_keeps_the_plain_knn_clause(
monkeypatch: pytest.MonkeyPatch,
) -> None:
module = _load_opensearch_module(monkeypatch)
aggregation = {
"aggregations": {
"embedding_spaces": {
"buckets": [{"key": {"space_id": "azure:text-embedding-3-small"}, "doc_count": 2}]
},
"legacy_embedding_models": {"buckets": []},
}
}
field = {"type": "knn_vector", "dimension": 2, "method": {"engine": "nmslib"}}
client = _SearchClient(aggregation, {"chunk_embedding_azure_text_embedding_3_small": field})
class Child:
def embed_query(self, query):
return [0.1, 0.2]
class Root(Child):
deployment = "selected-model"
def for_model(self, route):
return Child()
_component(module, client, Root()).search("needle")
(clause,) = client.query["query"]["bool"]["should"][0]["dis_max"]["queries"]
assert "knn" in clause
def test_unreadable_mapping_keeps_the_native_knn_clause(monkeypatch: pytest.MonkeyPatch) -> None:
module = _load_opensearch_module(monkeypatch)
aggregation = {
"aggregations": {
"embedding_spaces": {
"buckets": [{"key": {"space_id": "azure:text-embedding-3-small"}, "doc_count": 2}]
},
"legacy_embedding_models": {"buckets": []},
}
}
client = _SearchClient(aggregation, {})
client.indices = type(
"Indices",
(),
{
"get_mapping": staticmethod(
lambda index: (_ for _ in ()).throw(module.OpenSearchException("denied"))
)
},
)()
class Child:
def embed_query(self, query):
return [0.1, 0.2]
class Root(Child):
deployment = "selected-model"
def for_model(self, route):
return Child()
component = _component(module, client, Root())
component.filter_expression = json.dumps({"filter": [_RANGE]})
component.search("needle")
(clause,) = client.query["query"]["bool"]["should"][0]["dis_max"]["queries"]
assert "script_score" not in clause
(knn,) = clause["knn"].values()
assert knn["filter"] == {"bool": {"filter": [_RANGE]}}