Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
13.3 KB
Raw Blame
"""OpenAI-compatible Langflow components bind proxy globals at runtime."""

import hashlib
import importlib
import json
import sys
from pathlib import Path
from types import ModuleType

import pytest

# The Langflow bundle is the single home for these components: the former
# ``flows/components`` copies were removed once the bundle became canonical.
BUNDLE = Path("custom_components/openrag")
# Langflow 1.11 rewrites the bare class name to ``ext:elastic:...@official``
# via its extension migration table, so the OpenRAG copy must be referenced by
# its own canonical extension id everywhere (index key and flow node types).
OPENSEARCH_EXT_ID = "ext:openrag:OpenSearchVectorStoreComponentMultimodalMultiEmbedding@extra"
OPENSEARCH_MODULE = "_lfx_ext.extra.openrag.opensearch_multimodal"


def test_llm_component_binds_proxy_globals():
    code = (BUNDLE / "openai_compatible_llm.py").read_text(encoding="utf-8")
    assert 'OPENRAG_LLM_BASE_URL_VAR = "OPENRAG_LLM_BASE_URL"' in code
    assert 'OPENRAG_LLM_TOKEN_VAR = "OPENRAG_LLM_TOKEN"' in code
    assert 'SELECTED_LANGUAGE_MODEL_VAR = "SELECTED_LANGUAGE_MODEL"' in code
    assert 'name="api_key"' in code
    assert 'name="api_base"' in code
    assert "load_from_db=True" in code
    assert "ChatOpenAI" in code
    assert "class OpenAICompatibleLLMComponent" in code


def test_embedding_component_uses_the_same_proxy_globals():
    """Embeddings hit /v1/embeddings with the same hop token and /v1 base URL."""
    llm = (BUNDLE / "openai_compatible_llm.py").read_text(encoding="utf-8")
    embedding = (BUNDLE / "openai_compatible_embedding.py").read_text(encoding="utf-8")

    assert 'OPENRAG_LLM_BASE_URL_VAR = "OPENRAG_LLM_BASE_URL"' in embedding
    assert 'OPENRAG_LLM_TOKEN_VAR = "OPENRAG_LLM_TOKEN"' in embedding
    assert 'SELECTED_EMBEDDING_MODEL_VAR = "SELECTED_EMBEDDING_MODEL"' in embedding
    assert 'name="api_key"' in embedding
    assert 'name="api_base"' in embedding
    assert "load_from_db=True" in embedding
    assert "OpenAIEmbeddings" in embedding
    assert "check_embedding_ctx_length" in embedding
    assert "class OpenAICompatibleEmbeddingComponent" in embedding

    assert 'OPENRAG_LLM_BASE_URL_VAR = "OPENRAG_LLM_BASE_URL"' in llm
    assert 'OPENRAG_LLM_TOKEN_VAR = "OPENRAG_LLM_TOKEN"' in llm


def test_langflow_runtime_globals_cover_chat_and_embeddings():
    from api.settings.langflow_sync import (
        LANGFLOW_CREDENTIAL_GLOBAL_VARIABLES,
        LANGFLOW_GENERIC_GLOBAL_VARIABLES,
        LANGFLOW_RUNTIME_CREDENTIAL_PLACEHOLDERS,
    )

    assert "OPENRAG_LLM_TOKEN" in LANGFLOW_CREDENTIAL_GLOBAL_VARIABLES
    assert "OPENRAG_LLM_TOKEN" in LANGFLOW_RUNTIME_CREDENTIAL_PLACEHOLDERS
    assert {
        "OPENRAG_LLM_BASE_URL",
        "SELECTED_LANGUAGE_MODEL",
        "SELECTED_EMBEDDING_MODEL",
        "SELECTED_EMBEDDING_PROVIDER",
    } <= LANGFLOW_GENERIC_GLOBAL_VARIABLES

    source = Path("src/services/flows_service.py").read_text(encoding="utf-8")
    assert '"api_key": "OPENRAG_LLM_TOKEN"' in source
    assert '"openai_api_key": "OPENRAG_LLM_TOKEN"' in source
    assert '"api_base": "OPENRAG_LLM_BASE_URL"' in source
    assert '"openai_api_base": "OPENRAG_LLM_BASE_URL"' in source


def test_langflow_image_installs_the_openrag_component_bundle():
    """The Langflow image includes the OpenRAG component source bundle."""
    dockerfile = Path("Dockerfile.langflow").read_text(encoding="utf-8")
    assert "COPY custom_components/ /app/custom_components/" in dockerfile
    assert "ENV LANGFLOW_COMPONENTS_PATH=/app/custom_components" in dockerfile

    for name in (
        "openai_compatible_llm.py",
        "openai_compatible_embedding.py",
        "opensearch_multimodal.py",
    ):
        assert (BUNDLE / name).is_file(), name

    opensearch_source = (BUNDLE / "opensearch_multimodal.py").read_text(encoding="utf-8")
    assert "from .embedding_spaces import" not in opensearch_source
    assert not (BUNDLE / "embedding_spaces.py").exists()


def test_docker_compose_seeds_openrag_llm_token_placeholder():
    compose = Path("docker-compose.yml").read_text(encoding="utf-8")
    assert "OPENRAG_LLM_TOKEN=None" in compose
    assert (
        "OPENRAG_LLM_TOKEN"
        in compose.split("LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=", 1)[1].split("\n", 1)[0]
    )
    helm = Path("kubernetes/helm/openrag/templates/langflow/langflow-dotenv.yaml").read_text(
        encoding="utf-8"
    )
    assert 'OPENRAG_LLM_TOKEN="None"' in helm


def test_component_index_sha256_matches_langflow_integrity_check():
    """Langflow recomputes this digest and silently drops a mismatching index.

    It hashes ``orjson.dumps(index_without_sha, option=OPT_SORT_KEYS)``, which
    emits UTF-8 rather than the ASCII escapes ``json.dumps`` produces by
    default. A digest built the other way makes Langflow log
    "SHA256 mismatch" and fall back to a scan with no OpenRAG bundle, which
    blocks every flow that uses one of our components.
    """
    raw = json.loads(Path("flows/component_index.json").read_text(encoding="utf-8"))
    stored = raw.pop("sha256")
    payload = json.dumps(raw, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode(
        "utf-8"
    )
    assert stored == hashlib.sha256(payload).hexdigest()


def test_flows_reference_opensearch_by_canonical_extension_id():
    """A bare class-name reference gets hijacked by Langflow's migration table."""
    for path in sorted(Path("flows").glob("*.json")):
        if path.name == "component_index.json":
            continue
        flow = json.loads(path.read_text(encoding="utf-8"))
        for node in flow.get("data", {}).get("nodes", []):
            node_type = node.get("data", {}).get("type")
            if node_type and "OpenSearchVectorStoreComponentMultimodalMultiEmbedding" in node_type:
                assert node_type == OPENSEARCH_EXT_ID, path.name
                metadata = node["data"]["node"].get("metadata") or {}
                assert metadata.get("module") == OPENSEARCH_MODULE, path.name


def test_openrag_bundle_directory_is_lowercase_snake_case():
    """Langflow rejects inline bundle dirs that are not lowercase snake_case."""
    assert BUNDLE.is_dir()
    assert BUNDLE.name.islower()
    assert not Path("custom_components/OpenRAG").is_dir() or BUNDLE.samefile(
        Path("custom_components/OpenRAG")
    )


def test_embedding_component_pins_deployment_to_the_configured_model():
    """`OpenAIEmbeddings.deployment` must not keep LangChain's class default.

    That default is the literal "text-embedding-ada-002" for every instance,
    whatever model is configured. The OpenSearch component keys its embedding
    lookup on `deployment` as well as `model`, so the default registers a
    768-dim watsonx embedder under ada-002 and OpenSearch then rejects the
    search with "Query vector has invalid dimension: 768. Dimension should be:
    1536".
    """
    code = (BUNDLE / "openai_compatible_embedding.py").read_text(encoding="utf-8")
    assert '"deployment": self.model_name' in code


def test_embedding_component_can_resolve_immutable_retrieval_adapters():
    code = (BUNDLE / "openai_compatible_embedding.py").read_text(encoding="utf-8")

    assert "class OpenRAGEmbeddings(Embeddings):" in code
    assert "def for_model(self, model_name: str) -> Embeddings:" in code
    assert "return OpenRAGEmbeddings(" in code
    assert "dimensions=dimensions" in code


def test_embedding_adapter_preserves_dimensions_only_for_selected_provider_route(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Retrieval keeps a selected model's configured vector dimensions."""
    created: list[dict[str, object]] = []

    class FakeOpenAIEmbeddings:
        def __init__(self, **kwargs: object) -> None:
            created.append(kwargs)

    class FakeInput:
        def __init__(self, **kwargs: object) -> None:
            self.name = kwargs.get("name")

    modules: dict[str, ModuleType] = {
        "langchain_core": ModuleType("langchain_core"),
        "langchain_core.embeddings": ModuleType("langchain_core.embeddings"),
        "langchain_openai": ModuleType("langchain_openai"),
        "lfx": ModuleType("lfx"),
        "lfx.base": ModuleType("lfx.base"),
        "lfx.base.embeddings": ModuleType("lfx.base.embeddings"),
        "lfx.base.embeddings.model": ModuleType("lfx.base.embeddings.model"),
        "lfx.io": ModuleType("lfx.io"),
    }
    vars(modules["langchain_core.embeddings"])["Embeddings"] = object
    vars(modules["langchain_openai"])["OpenAIEmbeddings"] = FakeOpenAIEmbeddings
    vars(modules["lfx.base.embeddings.model"])["LCEmbeddingsModel"] = object
    for input_name in ("IntInput", "SecretStrInput", "StrInput"):
        setattr(modules["lfx.io"], input_name, FakeInput)
    for module_name, module in modules.items():
        monkeypatch.setitem(sys.modules, module_name, module)

    module_name = "custom_components.openrag.openai_compatible_embedding"
    sys.modules.pop(module_name, None)
    component = importlib.import_module(module_name)

    root = component.OpenRAGEmbeddings(
        model_name="text-embedding-3-small",
        api_key="token",
        api_base="http://openrag/v1",
        provider_name="openai",
        dimensions=512,
    )
    root.for_model("space:openai:text-embedding-3-small")
    root.for_model("legacy:text-embedding-3-small")
    root.for_model("space:azure:text-embedding-3-small")
    root.for_model("text-embedding-3-small")

    assert [kwargs.get("dimensions") for kwargs in created] == [512, 512, None, None, None]


def test_flows_embed_the_current_proxy_component_sources():
    """Flow JSONs carry their own copy of each component; keep them in step.

    A fix applied only to `custom_components/` never reaches a running flow,
    because Langflow executes the code embedded in the flow node.
    """
    for name, node_type in (
        ("openai_compatible_embedding.py", "OpenAICompatibleEmbedding"),
        ("openai_compatible_llm.py", "OpenAICompatibleLLM"),
    ):
        source = (BUNDLE / name).read_text(encoding="utf-8")
        expected_hash = hashlib.sha256(source.encode("utf-8")).hexdigest()[:12]

        for path in sorted(Path("flows").glob("*.json")):
            if path.name == "component_index.json":
                continue
            flow = json.loads(path.read_text(encoding="utf-8"))
            for node in flow.get("data", {}).get("nodes", []):
                if node.get("data", {}).get("type") != node_type:
                    continue
                component = node["data"]["node"]
                assert component["template"]["code"]["value"] == source, (
                    f"{path.name}: embedded {name} is out of sync with the bundle"
                )
                assert component.get("metadata", {}).get("code_hash") == expected_hash, (
                    f"{path.name}: {node_type} metadata.code_hash is stale"
                )


def test_llm_component_pins_chat_completions_api(monkeypatch: pytest.MonkeyPatch) -> None:
    """The gateway serves only /chat/completions, never /responses (#2534).

    Left to infer, ChatOpenAI routes some models (gpt-6* with tools) to the
    Responses API, whose client then rejects the component's default `seed`
    with "AsyncResponses.create() got an unexpected keyword argument 'seed'".
    """
    created: list[dict[str, object]] = []

    class FakeChatOpenAI:
        def __init__(self, **kwargs: object) -> None:
            created.append(kwargs)

    class FakeInput:
        def __init__(self, **kwargs: object) -> None:
            self.name = kwargs.get("name")

    modules: dict[str, ModuleType] = {
        "langchain_openai": ModuleType("langchain_openai"),
        "lfx": ModuleType("lfx"),
        "lfx.base": ModuleType("lfx.base"),
        "lfx.base.models": ModuleType("lfx.base.models"),
        "lfx.base.models.model": ModuleType("lfx.base.models.model"),
        "lfx.field_typing": ModuleType("lfx.field_typing"),
        "lfx.field_typing.constants": ModuleType("lfx.field_typing.constants"),
        "lfx.field_typing.range_spec": ModuleType("lfx.field_typing.range_spec"),
        "lfx.inputs": ModuleType("lfx.inputs"),
        "lfx.inputs.inputs": ModuleType("lfx.inputs.inputs"),
        "lfx.io": ModuleType("lfx.io"),
    }
    vars(modules["langchain_openai"])["ChatOpenAI"] = FakeChatOpenAI
    vars(modules["lfx.base.models.model"])["LCModelComponent"] = object
    vars(modules["lfx.field_typing.constants"])["LanguageModel"] = object
    vars(modules["lfx.field_typing.range_spec"])["RangeSpec"] = FakeInput
    for input_name in ("BoolInput", "StrInput"):
        setattr(modules["lfx.inputs.inputs"], input_name, FakeInput)
    for input_name in (
        "IntInput",
        "MessageInput",
        "MultilineInput",
        "SecretStrInput",
        "SliderInput",
    ):
        setattr(modules["lfx.io"], input_name, FakeInput)
    for module_name, module in modules.items():
        monkeypatch.setitem(sys.modules, module_name, module)

    module_name = "custom_components.openrag.openai_compatible_llm"
    sys.modules.pop(module_name, None)
    llm = importlib.import_module(module_name)

    component = llm.OpenAICompatibleLLMComponent()
    component.model_name = "gpt-6.1-sol"
    component.api_key = "token"
    component.api_base = "http://openrag/v1/"
    component.stream = True
    component.temperature = 0.1
    component.max_tokens = None
    component.seed = 1
    component.build_model()

    assert len(created) == 1
    assert created[0]["use_responses_api"] is False
    assert created[0]["seed"] == 1