OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
refactor(knn): consolidate knn_vector field mapping into shared helper
zznate96c16c19f78d
diff --git a/scripts/migrate_embedding_model_field.py b/scripts/migrate_embedding_model_field.pyindex 254da6580..2b1bc7dd8 100644--- a/scripts/migrate_embedding_model_field.py+++ b/scripts/migrate_embedding_model_field.py@@ -55,8 +55,6 @@ from opensearchpy._async.http_aiohttp import AIOHttpConnectionsys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src'))from config.settings import (- KNN_EF_CONSTRUCTION,- KNN_M,OPENSEARCH_HOST,OPENSEARCH_PORT,OPENSEARCH_USERNAME,@@ -64,7 +62,7 @@ from config.settings import (get_index_name,)from utils.logging_config import get_logger-from utils.embedding_fields import get_embedding_field_name+from utils.embedding_fields import build_knn_vector_field, get_embedding_field_namelogger = get_logger(__name__)@@ -78,16 +76,7 @@ async def ensure_new_field_exists("""Ensure the new embedding field exists in the index."""mapping = {"properties": {- field_name: {- "type": "knn_vector",- "dimension": dimensions,- "method": {- "name": "disk_ann",- "engine": "jvector",- "space_type": "l2",- "parameters": {"ef_construction": KNN_EF_CONSTRUCTION, "m": KNN_M},- },- },+ field_name: build_knn_vector_field(dimensions),"embedding_model": {"type": "keyword"}diff --git a/src/config/settings.py b/src/config/settings.pyindex b992d2dbc..371a4e199 100644--- a/src/config/settings.py+++ b/src/config/settings.py@@ -10,6 +10,7 @@ from opensearchpy import AsyncOpenSearchfrom opensearchpy._async.http_aiohttp import AIOHttpConnectionfrom utils.container_utils import get_container_host+from utils.embedding_fields import build_knn_vector_fieldfrom utils.logging_config import get_logger# Import configuration managerfrom .config_manager import config_manager@@ -160,16 +161,7 @@ INDEX_BODY = {"text": {"type": "text"},# Legacy field - kept for backward compatibility# New documents will use chunk_embedding_{model_name} fields- "chunk_embedding": {- "type": "knn_vector",- "dimension": VECTOR_DIM,- "method": {- "name": "disk_ann",- "engine": "jvector",- "space_type": "l2",- "parameters": {"ef_construction": KNN_EF_CONSTRUCTION, "m": KNN_M},- },- },+ "chunk_embedding": build_knn_vector_field(VECTOR_DIM),# Track which embedding model was used for this chunk"embedding_model": {"type": "keyword"},"source_url": {"type": "keyword"},diff --git a/src/utils/embedding_fields.py b/src/utils/embedding_fields.pyindex 388228e95..e74fd5a8c 100644--- a/src/utils/embedding_fields.py+++ b/src/utils/embedding_fields.py@@ -4,16 +4,52 @@ Utility functions for managing dynamic embedding field names in OpenSearch.This module provides helpers for:- Normalizing embedding model names to valid OpenSearch field names- Generating dynamic field names based on embedding models+- Building the shared knn_vector field mapping used across the codebase- Ensuring embedding fields exist in the OpenSearch index"""-from typing import Dict, Any+from typing import Any, Dictfrom utils.logging_config import get_loggerlogger = get_logger(__name__)+def build_knn_vector_field(dimension: int) -> Dict[str, Any]:+ """Build a knn_vector field mapping for OpenSearch using OpenRAG's JVector settings.++ All knn_vector fields in the documents index share the same JVector/DiskANN+ method configuration, differing only in their vector dimension. This helper+ is the single source of truth for that configuration so tuning changes+ apply uniformly wherever an embedding field is declared.++ Args:+ dimension: Vector dimension for the embedding model that will populate+ the field. Must match the output dimension of the embedding model.++ Returns:+ A fresh dict suitable for use as a field entry in an OpenSearch+ mapping's ``properties``. Each call returns a new dict, so callers+ may safely extend it in place (for example, to add ``advanced.*``+ parameters for a specific field) without affecting other call sites.+ """+ from config.settings import KNN_EF_CONSTRUCTION, KNN_M++ return {+ "type": "knn_vector",+ "dimension": dimension,+ "method": {+ "name": "disk_ann",+ "engine": "jvector",+ "space_type": "l2",+ "parameters": {+ "ef_construction": KNN_EF_CONSTRUCTION,+ "m": KNN_M,+ },+ },+ }++def normalize_model_name(model_name: str) -> str:"""Convert an embedding model name to a valid OpenSearch field suffix.@@ -86,7 +122,7 @@ async def ensure_embedding_field_exists(Raises:Exception: If unable to add the field mapping"""- from config.settings import KNN_EF_CONSTRUCTION, KNN_M, get_index_name+ from config.settings import get_index_namefrom utils.embeddings import get_embedding_dimensionsif index_name is None:@@ -130,16 +166,7 @@ async def ensure_embedding_field_exists(# Define the field mapping for both the vector field and the tracking fieldmapping = {"properties": {- field_name: {- "type": "knn_vector",- "dimension": dimensions,- "method": {- "name": "disk_ann",- "engine": "jvector",- "space_type": "l2",- "parameters": {"ef_construction": KNN_EF_CONSTRUCTION, "m": KNN_M},- },- },+ field_name: build_knn_vector_field(dimensions),# Also ensure the embedding_model tracking field exists as keyword"embedding_model": {"type": "keyword"diff --git a/src/utils/embeddings.py b/src/utils/embeddings.pyindex 96861d431..f79746545 100644--- a/src/utils/embeddings.py+++ b/src/utils/embeddings.py@@ -1,6 +1,7 @@import httpx-from config.settings import KNN_EF_CONSTRUCTION, KNN_M, OPENAI_EMBEDDING_DIMENSIONS, VECTOR_DIM, WATSONX_EMBEDDING_DIMENSIONS+from config.settings import OPENAI_EMBEDDING_DIMENSIONS, VECTOR_DIM, WATSONX_EMBEDDING_DIMENSIONSfrom utils.container_utils import transform_localhost_url+from utils.embedding_fields import build_knn_vector_fieldfrom utils.logging_config import get_logger@@ -169,16 +170,7 @@ async def create_dynamic_index_body("text": {"type": "text"},# Legacy field - kept for backward compatibility# New documents will use chunk_embedding_{model_name} fields- "chunk_embedding": {- "type": "knn_vector",- "dimension": dimensions,- "method": {- "name": "disk_ann",- "engine": "jvector",- "space_type": "l2",- "parameters": {"ef_construction": KNN_EF_CONSTRUCTION, "m": KNN_M},- },- },+ "chunk_embedding": build_knn_vector_field(dimensions),# Track which embedding model was used for this chunk"embedding_model": {"type": "keyword"},"embedding_dimensions": {"type": "integer"},diff --git a/tests/unit/test_embedding_fields.py b/tests/unit/test_embedding_fields.pynew file mode 100644index 000000000..79e6ded21--- /dev/null+++ b/tests/unit/test_embedding_fields.py@@ -0,0 +1,104 @@+"""+Unit tests for the embedding field helpers.++Focuses on ``build_knn_vector_field``, the single source of truth for+OpenRAG's ``knn_vector`` field mapping. Callers across ``config.settings``,+``utils.embeddings``, ``utils.embedding_fields``, and+``scripts.migrate_embedding_model_field`` rely on it producing a consistent+JVector/DiskANN method configuration with only the dimension varying per+embedding model.+"""+from typing import Any, Dict++import pytest++from utils.embedding_fields import build_knn_vector_field+++class TestBuildKnnVectorFieldStructure:+ """Shape of the returned mapping dict."""++ def test_returns_dict(self) -> None:+ result = build_knn_vector_field(1536)+ assert isinstance(result, dict)++ def test_top_level_keys(self) -> None:+ result = build_knn_vector_field(1536)+ assert set(result.keys()) == {"type", "dimension", "method"}++ def test_type_is_knn_vector(self) -> None:+ assert build_knn_vector_field(1536)["type"] == "knn_vector"++ def test_method_name_is_disk_ann(self) -> None:+ assert build_knn_vector_field(1536)["method"]["name"] == "disk_ann"++ def test_method_engine_is_jvector(self) -> None:+ assert build_knn_vector_field(1536)["method"]["engine"] == "jvector"++ def test_method_space_type_is_l2(self) -> None:+ assert build_knn_vector_field(1536)["method"]["space_type"] == "l2"++ def test_method_parameters_keys(self) -> None:+ params = build_knn_vector_field(1536)["method"]["parameters"]+ assert set(params.keys()) == {"ef_construction", "m"}+++class TestBuildKnnVectorFieldDimensionPropagation:+ """Dimension should be the only value that varies between calls."""++ @pytest.mark.parametrize("dimension", [384, 768, 1024, 1536, 3072])+ def test_dimension_propagates(self, dimension: int) -> None:+ assert build_knn_vector_field(dimension)["dimension"] == dimension++ def test_method_block_identical_across_dimensions(self) -> None:+ small = build_knn_vector_field(384)+ large = build_knn_vector_field(3072)+ assert small["method"] == large["method"]+++class TestBuildKnnVectorFieldSettingsResolution:+ """The helper must read KNN_M and KNN_EF_CONSTRUCTION from config.settings."""++ def test_matches_current_settings(self) -> None:+ from config import settings++ params = build_knn_vector_field(1536)["method"]["parameters"]+ assert params["m"] == settings.KNN_M+ assert params["ef_construction"] == settings.KNN_EF_CONSTRUCTION++ def test_picks_up_settings_override(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr("config.settings.KNN_M", 32)+ monkeypatch.setattr("config.settings.KNN_EF_CONSTRUCTION", 200)++ params = build_knn_vector_field(1536)["method"]["parameters"]+ assert params["m"] == 32+ assert params["ef_construction"] == 200+++class TestBuildKnnVectorFieldIsolation:+ """Each call must return a fresh dict so callers can mutate safely."""++ def test_returns_new_dict_each_call(self) -> None:+ first = build_knn_vector_field(1536)+ second = build_knn_vector_field(1536)+ assert first is not second+ assert first["method"] is not second["method"]+ assert first["method"]["parameters"] is not second["method"]["parameters"]++ def test_mutation_does_not_leak(self) -> None:+ first = build_knn_vector_field(1536)+ first["method"]["parameters"]["advanced.hierarchy_enabled"] = True++ second = build_knn_vector_field(1536)+ assert "advanced.hierarchy_enabled" not in second["method"]["parameters"]+++class TestBuildKnnVectorFieldCallSitesMatch:+ """Ensure every caller produces the same shape the helper promises."""++ def test_index_body_uses_helper_output(self) -> None:+ from config.settings import INDEX_BODY, VECTOR_DIM++ chunk_field: Dict[str, Any] = INDEX_BODY["mappings"]["properties"]["chunk_embedding"]+ expected = build_knn_vector_field(VECTOR_DIM)+ assert chunk_field == expected