Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

refactor(knn): consolidate knn_vector field mapping into shared helper

zznate96c16c19f78d
diff --git a/scripts/migrate_embedding_model_field.py b/scripts/migrate_embedding_model_field.py
index 254da6580..2b1bc7dd8 100644
--- a/scripts/migrate_embedding_model_field.py
+++ b/scripts/migrate_embedding_model_field.py
@@ -55,8 +55,6 @@ from opensearchpy._async.http_aiohttp import AIOHttpConnection
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src'))
from config.settings import (
- KNN_EF_CONSTRUCTION,
- KNN_M,
OPENSEARCH_HOST,
OPENSEARCH_PORT,
OPENSEARCH_USERNAME,
@@ -64,7 +62,7 @@ from config.settings import (
get_index_name,
)
from utils.logging_config import get_logger
-from utils.embedding_fields import get_embedding_field_name
+from utils.embedding_fields import build_knn_vector_field, get_embedding_field_name
logger = get_logger(__name__)
@@ -78,16 +76,7 @@ async def ensure_new_field_exists(
"""Ensure the new embedding field exists in the index."""
mapping = {
"properties": {
- field_name: {
- "type": "knn_vector",
- "dimension": dimensions,
- "method": {
- "name": "disk_ann",
- "engine": "jvector",
- "space_type": "l2",
- "parameters": {"ef_construction": KNN_EF_CONSTRUCTION, "m": KNN_M},
- },
- },
+ field_name: build_knn_vector_field(dimensions),
"embedding_model": {
"type": "keyword"
}
diff --git a/src/config/settings.py b/src/config/settings.py
index b992d2dbc..371a4e199 100644
--- a/src/config/settings.py
+++ b/src/config/settings.py
@@ -10,6 +10,7 @@ from opensearchpy import AsyncOpenSearch
from opensearchpy._async.http_aiohttp import AIOHttpConnection
from utils.container_utils import get_container_host
+from utils.embedding_fields import build_knn_vector_field
from utils.logging_config import get_logger
# Import configuration manager
from .config_manager import config_manager
@@ -160,16 +161,7 @@ INDEX_BODY = {
"text": {"type": "text"},
# Legacy field - kept for backward compatibility
# New documents will use chunk_embedding_{model_name} fields
- "chunk_embedding": {
- "type": "knn_vector",
- "dimension": VECTOR_DIM,
- "method": {
- "name": "disk_ann",
- "engine": "jvector",
- "space_type": "l2",
- "parameters": {"ef_construction": KNN_EF_CONSTRUCTION, "m": KNN_M},
- },
- },
+ "chunk_embedding": build_knn_vector_field(VECTOR_DIM),
# Track which embedding model was used for this chunk
"embedding_model": {"type": "keyword"},
"source_url": {"type": "keyword"},
diff --git a/src/utils/embedding_fields.py b/src/utils/embedding_fields.py
index 388228e95..e74fd5a8c 100644
--- a/src/utils/embedding_fields.py
+++ b/src/utils/embedding_fields.py
@@ -4,16 +4,52 @@ Utility functions for managing dynamic embedding field names in OpenSearch.
This module provides helpers for:
- Normalizing embedding model names to valid OpenSearch field names
- Generating dynamic field names based on embedding models
+- Building the shared knn_vector field mapping used across the codebase
- Ensuring embedding fields exist in the OpenSearch index
"""
-from typing import Dict, Any
+from typing import Any, Dict
from utils.logging_config import get_logger
logger = get_logger(__name__)
+def build_knn_vector_field(dimension: int) -> Dict[str, Any]:
+ """Build a knn_vector field mapping for OpenSearch using OpenRAG's JVector settings.
+
+ All knn_vector fields in the documents index share the same JVector/DiskANN
+ method configuration, differing only in their vector dimension. This helper
+ is the single source of truth for that configuration so tuning changes
+ apply uniformly wherever an embedding field is declared.
+
+ Args:
+ dimension: Vector dimension for the embedding model that will populate
+ the field. Must match the output dimension of the embedding model.
+
+ Returns:
+ A fresh dict suitable for use as a field entry in an OpenSearch
+ mapping's ``properties``. Each call returns a new dict, so callers
+ may safely extend it in place (for example, to add ``advanced.*``
+ parameters for a specific field) without affecting other call sites.
+ """
+ from config.settings import KNN_EF_CONSTRUCTION, KNN_M
+
+ return {
+ "type": "knn_vector",
+ "dimension": dimension,
+ "method": {
+ "name": "disk_ann",
+ "engine": "jvector",
+ "space_type": "l2",
+ "parameters": {
+ "ef_construction": KNN_EF_CONSTRUCTION,
+ "m": KNN_M,
+ },
+ },
+ }
+
+
def normalize_model_name(model_name: str) -> str:
"""
Convert an embedding model name to a valid OpenSearch field suffix.
@@ -86,7 +122,7 @@ async def ensure_embedding_field_exists(
Raises:
Exception: If unable to add the field mapping
"""
- from config.settings import KNN_EF_CONSTRUCTION, KNN_M, get_index_name
+ from config.settings import get_index_name
from utils.embeddings import get_embedding_dimensions
if index_name is None:
@@ -130,16 +166,7 @@ async def ensure_embedding_field_exists(
# Define the field mapping for both the vector field and the tracking field
mapping = {
"properties": {
- field_name: {
- "type": "knn_vector",
- "dimension": dimensions,
- "method": {
- "name": "disk_ann",
- "engine": "jvector",
- "space_type": "l2",
- "parameters": {"ef_construction": KNN_EF_CONSTRUCTION, "m": KNN_M},
- },
- },
+ field_name: build_knn_vector_field(dimensions),
# Also ensure the embedding_model tracking field exists as keyword
"embedding_model": {
"type": "keyword"
diff --git a/src/utils/embeddings.py b/src/utils/embeddings.py
index 96861d431..f79746545 100644
--- a/src/utils/embeddings.py
+++ b/src/utils/embeddings.py
@@ -1,6 +1,7 @@
import httpx
-from config.settings import KNN_EF_CONSTRUCTION, KNN_M, OPENAI_EMBEDDING_DIMENSIONS, VECTOR_DIM, WATSONX_EMBEDDING_DIMENSIONS
+from config.settings import OPENAI_EMBEDDING_DIMENSIONS, VECTOR_DIM, WATSONX_EMBEDDING_DIMENSIONS
from utils.container_utils import transform_localhost_url
+from utils.embedding_fields import build_knn_vector_field
from utils.logging_config import get_logger
@@ -169,16 +170,7 @@ async def create_dynamic_index_body(
"text": {"type": "text"},
# Legacy field - kept for backward compatibility
# New documents will use chunk_embedding_{model_name} fields
- "chunk_embedding": {
- "type": "knn_vector",
- "dimension": dimensions,
- "method": {
- "name": "disk_ann",
- "engine": "jvector",
- "space_type": "l2",
- "parameters": {"ef_construction": KNN_EF_CONSTRUCTION, "m": KNN_M},
- },
- },
+ "chunk_embedding": build_knn_vector_field(dimensions),
# Track which embedding model was used for this chunk
"embedding_model": {"type": "keyword"},
"embedding_dimensions": {"type": "integer"},
diff --git a/tests/unit/test_embedding_fields.py b/tests/unit/test_embedding_fields.py
new file mode 100644
index 000000000..79e6ded21
--- /dev/null
+++ b/tests/unit/test_embedding_fields.py
@@ -0,0 +1,104 @@
+"""
+Unit tests for the embedding field helpers.
+
+Focuses on ``build_knn_vector_field``, the single source of truth for
+OpenRAG's ``knn_vector`` field mapping. Callers across ``config.settings``,
+``utils.embeddings``, ``utils.embedding_fields``, and
+``scripts.migrate_embedding_model_field`` rely on it producing a consistent
+JVector/DiskANN method configuration with only the dimension varying per
+embedding model.
+"""
+from typing import Any, Dict
+
+import pytest
+
+from utils.embedding_fields import build_knn_vector_field
+
+
+class TestBuildKnnVectorFieldStructure:
+ """Shape of the returned mapping dict."""
+
+ def test_returns_dict(self) -> None:
+ result = build_knn_vector_field(1536)
+ assert isinstance(result, dict)
+
+ def test_top_level_keys(self) -> None:
+ result = build_knn_vector_field(1536)
+ assert set(result.keys()) == {"type", "dimension", "method"}
+
+ def test_type_is_knn_vector(self) -> None:
+ assert build_knn_vector_field(1536)["type"] == "knn_vector"
+
+ def test_method_name_is_disk_ann(self) -> None:
+ assert build_knn_vector_field(1536)["method"]["name"] == "disk_ann"
+
+ def test_method_engine_is_jvector(self) -> None:
+ assert build_knn_vector_field(1536)["method"]["engine"] == "jvector"
+
+ def test_method_space_type_is_l2(self) -> None:
+ assert build_knn_vector_field(1536)["method"]["space_type"] == "l2"
+
+ def test_method_parameters_keys(self) -> None:
+ params = build_knn_vector_field(1536)["method"]["parameters"]
+ assert set(params.keys()) == {"ef_construction", "m"}
+
+
+class TestBuildKnnVectorFieldDimensionPropagation:
+ """Dimension should be the only value that varies between calls."""
+
+ @pytest.mark.parametrize("dimension", [384, 768, 1024, 1536, 3072])
+ def test_dimension_propagates(self, dimension: int) -> None:
+ assert build_knn_vector_field(dimension)["dimension"] == dimension
+
+ def test_method_block_identical_across_dimensions(self) -> None:
+ small = build_knn_vector_field(384)
+ large = build_knn_vector_field(3072)
+ assert small["method"] == large["method"]
+
+
+class TestBuildKnnVectorFieldSettingsResolution:
+ """The helper must read KNN_M and KNN_EF_CONSTRUCTION from config.settings."""
+
+ def test_matches_current_settings(self) -> None:
+ from config import settings
+
+ params = build_knn_vector_field(1536)["method"]["parameters"]
+ assert params["m"] == settings.KNN_M
+ assert params["ef_construction"] == settings.KNN_EF_CONSTRUCTION
+
+ def test_picks_up_settings_override(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr("config.settings.KNN_M", 32)
+ monkeypatch.setattr("config.settings.KNN_EF_CONSTRUCTION", 200)
+
+ params = build_knn_vector_field(1536)["method"]["parameters"]
+ assert params["m"] == 32
+ assert params["ef_construction"] == 200
+
+
+class TestBuildKnnVectorFieldIsolation:
+ """Each call must return a fresh dict so callers can mutate safely."""
+
+ def test_returns_new_dict_each_call(self) -> None:
+ first = build_knn_vector_field(1536)
+ second = build_knn_vector_field(1536)
+ assert first is not second
+ assert first["method"] is not second["method"]
+ assert first["method"]["parameters"] is not second["method"]["parameters"]
+
+ def test_mutation_does_not_leak(self) -> None:
+ first = build_knn_vector_field(1536)
+ first["method"]["parameters"]["advanced.hierarchy_enabled"] = True
+
+ second = build_knn_vector_field(1536)
+ assert "advanced.hierarchy_enabled" not in second["method"]["parameters"]
+
+
+class TestBuildKnnVectorFieldCallSitesMatch:
+ """Ensure every caller produces the same shape the helper promises."""
+
+ def test_index_body_uses_helper_output(self) -> None:
+ from config.settings import INDEX_BODY, VECTOR_DIM
+
+ chunk_field: Dict[str, Any] = INDEX_BODY["mappings"]["properties"]["chunk_embedding"]
+ expected = build_knn_vector_field(VECTOR_DIM)
+ assert chunk_field == expected