Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat: split vectordb engine by cpu variant (#656)
zhoujiahui758ccd8986ab
diff --git a/.github/workflows/_build.yml b/.github/workflows/_build.ymlindex 28a708dd5..89a69ef49 100644--- a/.github/workflows/_build.yml+++ b/.github/workflows/_build.yml@@ -455,21 +455,22 @@ jobs:shell: bashrun: |python - <<'PY'+ import importlibimport importlib.util- import pathlib- import site-- candidates = []- for base in site.getsitepackages():- candidates.extend(pathlib.Path(base).glob("openviking/storage/vectordb/engine*.so"))-- if not candidates:- raise SystemExit("openviking storage engine extension was not installed")-- engine_path = candidates[0]- spec = importlib.util.spec_from_file_location("engine", engine_path)- module = importlib.util.module_from_spec(spec)- assert spec.loader is not None- spec.loader.exec_module(module)- print(f"Loaded native extension from {engine_path}")++ import openviking.storage.vectordb.engine as engine++ native_spec = importlib.util.find_spec("openviking.storage.vectordb.engine._native")+ if native_spec is None or native_spec.origin is None:+ raise SystemExit("openviking storage native backend extension was not installed")++ native_module = importlib.import_module("openviking.storage.vectordb.engine._native")+ if engine.ENGINE_VARIANT != "native":+ raise SystemExit(+ f"expected native engine variant on macOS arm64 wheel, got {engine.ENGINE_VARIANT}"+ )++ print(f"Loaded runtime engine variant {engine.ENGINE_VARIANT}")+ print(f"Loaded native extension from {native_spec.origin}")+ print(f"Imported backend module {native_module.__name__}")PYdiff --git a/build_support/__init__.py b/build_support/__init__.pynew file mode 100644index 000000000..b9c0eff11--- /dev/null+++ b/build_support/__init__.py@@ -0,0 +1 @@+"""Build helpers for OpenViking native artifacts."""diff --git a/build_support/x86_profiles.py b/build_support/x86_profiles.pynew file mode 100644index 000000000..cb0f1904a--- /dev/null+++ b/build_support/x86_profiles.py@@ -0,0 +1,64 @@+from __future__ import annotations++import os+from dataclasses import dataclass+from typing import Iterable++DEFAULT_X86_VARIANTS = ("sse3", "avx2", "avx512")+KNOWN_X86_VARIANTS = frozenset(DEFAULT_X86_VARIANTS)+X86_ARCHITECTURES = ("x86_64", "amd64", "x64", "i386", "i686")+++@dataclass(frozen=True)+class EngineBuildConfig:+ is_x86: bool+ primary_extension: str+ cmake_variants: tuple[str, ...]+++def _normalize_machine(machine: str | None) -> str:+ return (machine or "").strip().lower()+++def is_x86_machine(machine: str | None) -> bool:+ normalized = _normalize_machine(machine)+ return any(token in normalized for token in X86_ARCHITECTURES)+++def _normalize_x86_variants(raw_variants: Iterable[str]) -> tuple[str, ...]:+ requested = []+ for variant in raw_variants:+ normalized = variant.strip().lower()+ if not normalized or normalized not in KNOWN_X86_VARIANTS or normalized in requested:+ continue+ requested.append(normalized)++ if "sse3" not in requested:+ requested.insert(0, "sse3")++ return tuple(requested or DEFAULT_X86_VARIANTS)+++def get_requested_x86_build_variants(raw_value: str | None = None) -> tuple[str, ...]:+ if raw_value is None:+ raw_value = os.environ.get("OV_X86_BUILD_VARIANTS", "")++ if not raw_value.strip():+ return DEFAULT_X86_VARIANTS++ return _normalize_x86_variants(raw_value.replace(";", ",").split(","))+++def get_host_engine_build_config(machine: str | None) -> EngineBuildConfig:+ if is_x86_machine(machine):+ return EngineBuildConfig(+ is_x86=True,+ primary_extension="openviking.storage.vectordb.engine._x86_sse3",+ cmake_variants=get_requested_x86_build_variants(),+ )++ return EngineBuildConfig(+ is_x86=False,+ primary_extension="openviking.storage.vectordb.engine._native",+ cmake_variants=(),+ )diff --git a/openviking/storage/vectordb/engine/__init__.py b/openviking/storage/vectordb/engine/__init__.pynew file mode 100644index 000000000..b1075f35d--- /dev/null+++ b/openviking/storage/vectordb/engine/__init__.py@@ -0,0 +1,176 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: Apache-2.0+"""Stable runtime loader for vectordb native engine variants."""++from __future__ import annotations++import importlib+import importlib.util+import os+import platform+from types import ModuleType++_BACKEND_MODULES = {+ "x86_sse3": "_x86_sse3",+ "x86_avx2": "_x86_avx2",+ "x86_avx512": "_x86_avx512",+ "native": "_native",+}+_X86_DISPLAY_ORDER = ("x86_sse3", "x86_avx2", "x86_avx512")+_X86_PRIORITY = ("x86_avx512", "x86_avx2", "x86_sse3")+_REQUEST_ALIASES = {+ "sse3": "x86_sse3",+ "avx2": "x86_avx2",+ "avx512": "x86_avx512",+}+++def _is_x86_machine(machine: str | None = None) -> bool:+ normalized = (machine or platform.machine() or "").strip().lower()+ return any(token in normalized for token in ("x86_64", "amd64", "x64", "i386", "i686"))+++def _module_exists(module_name: str) -> bool:+ return importlib.util.find_spec(f".{module_name}", __name__) is not None+++def _available_variants(is_x86: bool) -> tuple[str, ...]:+ ordered = _X86_DISPLAY_ORDER if is_x86 else ("native",)+ return tuple(variant for variant in ordered if _module_exists(_BACKEND_MODULES[variant]))+++def _supported_x86_variants() -> set[str]:+ supported = {"x86_sse3"}+ if not _module_exists("_x86_caps"):+ return supported++ try:+ caps = importlib.import_module("._x86_caps", __name__)+ except ImportError:+ return supported++ reported = getattr(caps, "get_supported_variants", lambda: [])()+ for variant in reported:+ normalized = str(variant).strip().lower()+ if normalized in _BACKEND_MODULES:+ supported.add(normalized)+ return supported+++def _normalize_requested_variant(value: str | None) -> str:+ normalized = (value or "auto").strip().lower()+ return _REQUEST_ALIASES.get(normalized, normalized)+++def _validate_forced_variant(+ requested: str, *, is_x86: bool, available: tuple[str, ...], supported_x86: set[str]+) -> None:+ if is_x86 and requested == "native":+ raise ImportError("OV_ENGINE_VARIANT=native is only valid on non-x86 platforms")++ if not is_x86 and requested != "native":+ raise ImportError(+ f"OV_ENGINE_VARIANT={requested} is not valid on non-x86 platforms; use native"+ )++ if requested not in _BACKEND_MODULES:+ raise ImportError(f"Unknown OV_ENGINE_VARIANT={requested}")++ if requested not in available:+ raise ImportError(+ f"Requested engine variant {requested} is not packaged in this wheel. "+ f"Available variants: {', '.join(available) or 'none'}"+ )++ if is_x86 and requested not in supported_x86:+ raise ImportError(f"Requested engine variant {requested} is not supported by this CPU")+++def _select_variant() -> tuple[str | None, tuple[str, ...], str | None]:+ is_x86 = _is_x86_machine()+ available = _available_variants(is_x86)+ requested = _normalize_requested_variant(os.environ.get("OV_ENGINE_VARIANT"))++ if requested != "auto":+ supported_x86 = _supported_x86_variants() if is_x86 else set()+ _validate_forced_variant(+ requested, is_x86=is_x86, available=available, supported_x86=supported_x86+ )+ return requested, available, None++ if not is_x86:+ if "native" not in available:+ return None, available, "Native engine backend is missing from this wheel"+ return "native", available, None++ supported_x86 = _supported_x86_variants()+ for variant in _X86_PRIORITY:+ if variant in available and variant in supported_x86:+ return variant, available, None++ if "x86_sse3" in available:+ return "x86_sse3", available, None++ return None, available, "No compatible x86 engine backend was packaged in this wheel"+++def _load_backend(variant: str) -> ModuleType:+ return importlib.import_module(f".{_BACKEND_MODULES[variant]}", __name__)+++def _export_backend(module: ModuleType) -> tuple[str, ...]:+ names = getattr(module, "__all__", None)+ if names is None:+ names = tuple(name for name in dir(module) if not name.startswith("_"))++ for name in names:+ globals()[name] = getattr(module, name)++ return tuple(names)+++class _MissingBackendSymbol:+ def __init__(self, symbol_name: str, message: str):+ self._symbol_name = symbol_name+ self._message = message++ def __call__(self, *args, **kwargs):+ raise ImportError(f"{self._message}. Missing symbol: {self._symbol_name}")++ def __getattr__(self, name: str):+ return _MissingBackendSymbol(f"{self._symbol_name}.{name}", self._message)++ def __bool__(self) -> bool:+ return False++ def __repr__(self) -> str:+ return f"<missing vectordb engine symbol {self._symbol_name}>"+++_SELECTED_VARIANT, AVAILABLE_ENGINE_VARIANTS, _ENGINE_IMPORT_ERROR = _select_variant()+if _SELECTED_VARIANT is None:+ ENGINE_VARIANT = "unavailable"+ _BACKEND = None+ _EXPORTED_NAMES = ()+else:+ ENGINE_VARIANT = _SELECTED_VARIANT+ _BACKEND = _load_backend(ENGINE_VARIANT)+ _EXPORTED_NAMES = _export_backend(_BACKEND)+++def __getattr__(name: str):+ if _BACKEND is None and _ENGINE_IMPORT_ERROR is not None:+ return _MissingBackendSymbol(name, _ENGINE_IMPORT_ERROR)+ raise AttributeError(name)+++__all__ = tuple(+ sorted(+ set(_EXPORTED_NAMES).union(+ {+ "AVAILABLE_ENGINE_VARIANTS",+ "ENGINE_VARIANT",+ }+ )+ )+)diff --git a/openviking/storage/vectordb/store/bytes_row.py b/openviking/storage/vectordb/store/bytes_row.pyindex 5455621c1..8eaa2679d 100644--- a/openviking/storage/vectordb/store/bytes_row.py+++ b/openviking/storage/vectordb/store/bytes_row.py@@ -274,6 +274,9 @@ class _PyBytesRow:try:import openviking.storage.vectordb.engine as engine+ if getattr(engine, "ENGINE_VARIANT", "unavailable") == "unavailable":+ raise ImportError("vectordb engine backend is unavailable")+# Use C++ implementation if availableBytesRow = engine.BytesRowSchema = engine.Schemadiff --git a/pyproject.toml b/pyproject.tomlindex 63b19c283..c26aa7cfc 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -177,7 +177,9 @@ openviking = ["lib/libagfsbinding.dylib","lib/libagfsbinding.dll","bin/ov",- "bin/ov.exe"+ "bin/ov.exe",+ "storage/vectordb/engine/*.so",+ "storage/vectordb/engine/*.pyd",]vikingbot = ["**/*.mjs",diff --git a/setup.py b/setup.pyindex f91a3c42a..87cc38807 100644--- a/setup.py+++ b/setup.py@@ -1,5 +1,7 @@+import importlibimport jsonimport os+import platformimport shutilimport subprocessimport sys@@ -10,10 +12,19 @@ import pybind11from setuptools import Extension, setupfrom setuptools.command.build_ext import build_ext+SETUP_DIR = Path(__file__).resolve().parent+if str(SETUP_DIR) not in sys.path:+ sys.path.insert(0, str(SETUP_DIR))++get_host_engine_build_config = importlib.import_module(+ "build_support.x86_profiles"+).get_host_engine_build_config+CMAKE_PATH = shutil.which("cmake") or "cmake"C_COMPILER_PATH = shutil.which("gcc") or "gcc"CXX_COMPILER_PATH = shutil.which("g++") or "g++"ENGINE_SOURCE_DIR = "src/"+ENGINE_BUILD_CONFIG = get_host_engine_build_config(platform.machine())class OpenVikingBuildExt(build_ext):@@ -320,6 +331,9 @@ class OpenVikingBuildExt(build_ext):def build_extension(self, ext):"""Build a single Python native extension artifact using CMake."""+ if getattr(self, "_engine_extensions_built", False):+ return+ext_fullpath = Path(self.get_ext_fullpath(ext.name))ext_dir = ext_fullpath.parent.resolve()build_dir = Path(self.build_temp) / "cmake_build"@@ -330,19 +344,19 @@ class OpenVikingBuildExt(build_ext):lambda: self._build_extension_impl(ext_fullpath, ext_dir, build_dir),[(ext_fullpath, f"native extension '{ext.name}'")],)+ self._engine_extensions_built = Truedef _build_extension_impl(self, ext_fullpath, ext_dir, build_dir):"""Invoke CMake to build the Python native extension."""- py_output_name = ext_fullpath.stem- py_output_suffix = ext_fullpath.suffix+ py_ext_suffix = sysconfig.get_config_var("EXT_SUFFIX") or ext_fullpath.suffixcmake_args = [f"-S{Path(ENGINE_SOURCE_DIR).resolve()}",f"-B{build_dir}","-DCMAKE_BUILD_TYPE=Release",- f"-DPY_OUTPUT_DIR={ext_dir}",- f"-DPY_OUTPUT_NAME={py_output_name}",- f"-DPY_OUTPUT_SUFFIX={py_output_suffix}",+ f"-DOV_PY_OUTPUT_DIR={ext_dir}",+ f"-DOV_PY_EXT_SUFFIX={py_ext_suffix}",+ f"-DOV_X86_BUILD_VARIANTS={';'.join(ENGINE_BUILD_CONFIG.cmake_variants)}","-DCMAKE_VERBOSE_MAKEFILE=ON","-DCMAKE_INSTALL_RPATH=$ORIGIN",f"-DPython3_EXECUTABLE={sys.executable}",@@ -351,7 +365,6 @@ class OpenVikingBuildExt(build_ext):f"-Dpybind11_DIR={pybind11.get_cmake_dir()}",f"-DCMAKE_C_COMPILER={C_COMPILER_PATH}",f"-DCMAKE_CXX_COMPILER={CXX_COMPILER_PATH}",- f"-DOV_X86_SIMD_LEVEL={os.environ.get('OV_X86_SIMD_LEVEL', 'AVX2')}",]if sys.platform == "darwin":@@ -374,7 +387,7 @@ setup(# ],ext_modules=[Extension(- name="openviking.storage.vectordb.engine",+ name=ENGINE_BUILD_CONFIG.primary_extension,sources=[],)],@@ -390,6 +403,8 @@ setup("lib/libagfsbinding.dll","bin/ov","bin/ov.exe",+ "storage/vectordb/engine/*.so",+ "storage/vectordb/engine/*.pyd",],},include_package_data=True,diff --git a/src/CMakeLists.txt b/src/CMakeLists.txtindex d844427a1..33b0aa5a3 100644--- a/src/CMakeLists.txt+++ b/src/CMakeLists.txt@@ -3,73 +3,24 @@ cmake_minimum_required(VERSION 3.12)project(openviking_cpp)include(CheckCXXCompilerFlag)+include(CMakeParseArguments)-set(OV_X86_SIMD_LEVEL "AVX2" CACHE STRING "x86 SIMD level: SSE3|AVX2|AVX512|NATIVE")-set_property(CACHE OV_X86_SIMD_LEVEL PROPERTY STRINGS SSE3 AVX2 AVX512 NATIVE)+set(OV_X86_BUILD_VARIANTS "sse3;avx2;avx512" CACHE STRING "x86 engine variants to build")+set(OV_PY_OUTPUT_DIR "" CACHE PATH "Output directory for Python extension modules")+set(OV_PY_EXT_SUFFIX ".so" CACHE STRING "Python extension suffix, including ABI tag if needed")++if(NOT OV_PY_OUTPUT_DIR)+ set(OV_PY_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/python_engine")+endif()set(OV_PLATFORM_X86 OFF)if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64|AMD64|i[3-6]86")set(OV_PLATFORM_X86 ON)endif()-if(OV_PLATFORM_X86)- string(TOUPPER "${OV_X86_SIMD_LEVEL}" OV_X86_SIMD_LEVEL_UPPER)- set(OV_X86_COMPILE_FLAGS)-- if(OV_X86_SIMD_LEVEL_UPPER STREQUAL "NATIVE")- check_cxx_compiler_flag("-march=native" HAVE_MARCH_NATIVE)- if(HAVE_MARCH_NATIVE)- list(APPEND OV_X86_COMPILE_FLAGS -march=native)- else()- message(WARNING "-march=native is not supported by this compiler; falling back to AVX2")- set(OV_X86_SIMD_LEVEL_UPPER "AVX2")- endif()- endif()-- if(OV_X86_SIMD_LEVEL_UPPER STREQUAL "AVX512")- foreach(FLAG -mavx512f -mavx512bw -mavx512dq -mavx512vl)- string(REPLACE "-" "_" FLAG_VAR_SUFFIX "${FLAG}")- set(FLAG_VAR "HAVE_${FLAG_VAR_SUFFIX}")- check_cxx_compiler_flag("${FLAG}" ${FLAG_VAR})- if(NOT ${FLAG_VAR})- message(FATAL_ERROR "Compiler does not support ${FLAG}; cannot build with OV_X86_SIMD_LEVEL=AVX512")- endif()- list(APPEND OV_X86_COMPILE_FLAGS ${FLAG})- endforeach()- elseif(OV_X86_SIMD_LEVEL_UPPER STREQUAL "AVX2")- check_cxx_compiler_flag("-mavx2" HAVE_MAVX2)- if(HAVE_MAVX2)- list(APPEND OV_X86_COMPILE_FLAGS -mavx2)- add_compile_definitions(OV_DISABLE_AVX512=1)- foreach(FLAG -mno-avx512f -mno-avx512bw -mno-avx512dq -mno-avx512vl)- string(REPLACE "-" "_" FLAG_VAR_SUFFIX "${FLAG}")- set(FLAG_VAR "HAVE_${FLAG_VAR_SUFFIX}")- check_cxx_compiler_flag("${FLAG}" ${FLAG_VAR})- if(${FLAG_VAR})- list(APPEND OV_X86_COMPILE_FLAGS ${FLAG})- endif()- endforeach()- else()- message(WARNING "Compiler does not support -mavx2; falling back to SSE3")- set(OV_X86_SIMD_LEVEL_UPPER "SSE3")- endif()- endif()-- if(OV_X86_SIMD_LEVEL_UPPER STREQUAL "SSE3")- check_cxx_compiler_flag("-msse3" HAVE_SSE3)- if(HAVE_SSE3)- list(APPEND OV_X86_COMPILE_FLAGS -msse3)- add_compile_definitions(OV_DISABLE_AVX512=1)- else()- message(FATAL_ERROR "Compiler does not support -msse3 on x86 target")- endif()- endif()-- if(OV_X86_COMPILE_FLAGS)- add_compile_options(${OV_X86_COMPILE_FLAGS})- endif()-- message(STATUS "OpenViking x86 SIMD level: ${OV_X86_SIMD_LEVEL_UPPER}")+set(OV_PLATFORM_ARM OFF)+if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|ARM64|arm64")+ set(OV_PLATFORM_ARM ON)endif()set(CMAKE_CXX_STANDARD 17)@@ -80,7 +31,6 @@ if(APPLE)endif()set(THREADS_PREFER_PTHREAD_FLAG ON)-set(CMAKE_STRIP FALSE)add_compile_definitions(HAVE_CXX17_HAS_INCLUDE=1)@@ -89,15 +39,11 @@ if(WIN32)endif()set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error -Wno-deprecated-declarations -Wno-format -Wno-inconsistent-missing-override")-set(CMAKE_CXX_LINK_EXECUTABLE "${CMAKE_CXX_LINK_EXECUTABLE} -lpthread")-set(Python3_ARCH_INCLUDE_DIR "/usr/include/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/")find_package(Python3 COMPONENTS Interpreter Development REQUIRED)-# On Linux, pybind11 modules don't need to link against libpython-# This prevents issues with static libpython that wasn't built with -fPICif(UNIX AND NOT APPLE)set(Python3_LIBRARIES "")endif()@@ -105,7 +51,6 @@ endif()find_package(pybind11 REQUIRED)find_package(Threads REQUIRED)-# Force static libraries for dependenciesset(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE)set(SPDLOG_BUILD_SHARED OFF CACHE BOOL "" FORCE)set(LEVELDB_BUILD_TESTS OFF CACHE BOOL "" FORCE)@@ -115,17 +60,13 @@ set(SPDLOG_BUILD_TESTS OFF CACHE BOOL "" FORCE)set(SPDLOG_BUILD_EXAMPLE OFF CACHE BOOL "" FORCE)add_subdirectory(../third_party/leveldb-1.23 ${CMAKE_BINARY_DIR}/leveldb_build)-if(TARGET leveldb)- target_compile_options(leveldb PRIVATE -fPIC)+ target_compile_options(leveldb PRIVATE -fPIC)endif()add_subdirectory(../third_party/spdlog-1.14.1 ${CMAKE_BINARY_DIR}/spdlog_build)-# ARM platform detection and KRL integration-set(OV_PLATFORM_ARM OFF)-if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|ARM64|arm64")- set(OV_PLATFORM_ARM ON)+if(OV_PLATFORM_ARM)message(STATUS "Building for ARM platform with KRL support")add_subdirectory(../third_party/krl ${CMAKE_BINARY_DIR}/krl_build)endif()@@ -135,103 +76,243 @@ include_directories(../third_party/)include_directories(../third_party/leveldb-1.23/include/)include_directories(../third_party/spdlog-1.14.1/include/)-# Add KRL include directory for ARM platformif(OV_PLATFORM_ARM)include_directories(../third_party/krl/include/)endif()if(NOT DEFINED Python3_INCLUDE_DIRS)- set(Python3_INCLUDE_DIRS+ set(Python3_INCLUDE_DIRS${Python3_ARCH_INCLUDE_DIR}"/usr/include/../include/")endif()-file(GLOB_RECURSE ALL_SOURCES- "index/*.cpp"- "store/*.cpp"- "common/*.cpp"-)--add_library(- engine_impl- STATIC- ${ALL_SOURCES}+set(OV_COMMON_SOURCES+ common/log_utils.cpp+ store/bytes_row.cpp+ store/persist_store.cpp+ store/volatile_store.cpp)-target_link_libraries(engine_impl PRIVATE- Threads::Threads- leveldb+set(OV_INDEX_SOURCES+ index/detail/index_manager_impl.cpp+ index/detail/meta/scalar_index_meta.cpp+ index/detail/meta/vector_index_meta.cpp+ index/detail/scalar/bitmap_holder/bitmap.cpp+ index/detail/scalar/bitmap_holder/bitmap_field_group.cpp+ index/detail/scalar/bitmap_holder/dir_index.cpp+ index/detail/scalar/bitmap_holder/ranged_map.cpp+ index/detail/scalar/filter/filter_ops.cpp+ index/detail/scalar/filter/op_base.cpp+ index/detail/scalar/filter/sort_ops.cpp+ index/detail/scalar/scalar_index.cpp+ index/detail/vector/sparse_retrieval/sparse_datapoint.cpp+ index/detail/vector/sparse_retrieval/sparse_row_index.cpp+ index/index_engine.cpp)-# Link KRL library for ARM platform-if(OV_PLATFORM_ARM)- target_link_libraries(engine_impl PRIVATE krl)-endif()+add_library(engine_common STATIC ${OV_COMMON_SOURCES})+target_compile_options(engine_common PRIVATE -fPIC)+target_link_libraries(engine_common PUBLIC Threads::Threads leveldb)-if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")- if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "9.0")- target_link_libraries(engine_impl PRIVATE stdc++fs)+function(ov_link_filesystem_libs target_name)+ if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")+ if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "9.0")+ target_link_libraries(${target_name} PRIVATE stdc++fs)+ endif()+ elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")+ if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "10.0")+ target_link_libraries(${target_name} PRIVATE c++fs)+ endif()+ elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")+ if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "11.0")+ target_link_libraries(${target_name} PRIVATE c++fs)+ endif()endif()-elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")- if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "10.0")- target_link_libraries(engine_impl PRIVATE c++fs)+endfunction()++function(ov_get_x86_variant_flags variant out_flags out_defs out_supported)+ string(TOLOWER "${variant}" OV_VARIANT)+ set(OV_FLAGS)+ set(OV_DEFS)+ set(OV_SUPPORTED TRUE)++ if(OV_VARIANT STREQUAL "sse3")+ check_cxx_compiler_flag("-msse3" HAVE_OV_SSE3)+ if(HAVE_OV_SSE3)+ list(APPEND OV_FLAGS -msse3)+ list(APPEND OV_DEFS OV_DISABLE_AVX512=1)+ else()+ set(OV_SUPPORTED FALSE)+ endif()+ elseif(OV_VARIANT STREQUAL "avx2")+ check_cxx_compiler_flag("-mavx2" HAVE_OV_AVX2)+ if(HAVE_OV_AVX2)+ list(APPEND OV_FLAGS -mavx2)+ list(APPEND OV_DEFS OV_DISABLE_AVX512=1)+ foreach(FLAG -mno-avx512f -mno-avx512bw -mno-avx512dq -mno-avx512vl)+ string(REPLACE "-" "_" FLAG_VAR_SUFFIX "${FLAG}")+ set(FLAG_VAR "HAVE_${FLAG_VAR_SUFFIX}")+ check_cxx_compiler_flag("${FLAG}" ${FLAG_VAR})+ if(${FLAG_VAR})+ list(APPEND OV_FLAGS ${FLAG})+ endif()+ endforeach()+ else()+ set(OV_SUPPORTED FALSE)+ endif()+ elseif(OV_VARIANT STREQUAL "avx512")+ foreach(FLAG -mavx512f -mavx512bw -mavx512dq -mavx512vl)+ string(REPLACE "-" "_" FLAG_VAR_SUFFIX "${FLAG}")+ set(FLAG_VAR "HAVE_${FLAG_VAR_SUFFIX}")+ check_cxx_compiler_flag("${FLAG}" ${FLAG_VAR})+ if(NOT ${FLAG_VAR})+ set(OV_SUPPORTED FALSE)+ else()+ list(APPEND OV_FLAGS ${FLAG})+ endif()+ endforeach()+ else()+ set(OV_SUPPORTED FALSE)endif()-elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")- if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "11.0")- target_link_libraries(engine_impl PRIVATE c++fs)++ set(${out_flags} "${OV_FLAGS}" PARENT_SCOPE)+ set(${out_defs} "${OV_DEFS}" PARENT_SCOPE)+ set(${out_supported} ${OV_SUPPORTED} PARENT_SCOPE)+endfunction()++function(ov_add_python_backend backend_suffix module_name)+ set(oneValueArgs INDEX_LIBRARY)+ set(multiValueArgs COMPILE_OPTIONS COMPILE_DEFINITIONS)+ cmake_parse_arguments(OV_BACKEND "" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})++ set(MODULE_TARGET "engine_module_${backend_suffix}")+ pybind11_add_module(${MODULE_TARGET} MODULE pybind11_interface.cpp)++ target_include_directories(${MODULE_TARGET} PRIVATE ${Python3_INCLUDE_DIRS})+ target_compile_options(${MODULE_TARGET} PRIVATE -fPIC ${OV_BACKEND_COMPILE_OPTIONS})+ target_compile_definitions(+ ${MODULE_TARGET}+ PRIVATE+ OV_PY_MODULE_NAME=${module_name}+ ${OV_BACKEND_COMPILE_DEFINITIONS}+ )+ target_link_libraries(+ ${MODULE_TARGET}+ PRIVATE+ engine_common+ ${OV_BACKEND_INDEX_LIBRARY}+ Threads::Threads+ )+ ov_link_filesystem_libs(${MODULE_TARGET})++ if(MINGW)+ target_link_libraries(${MODULE_TARGET} PRIVATE+ -static-libgcc+ -static-libstdc+++ -Wl,-Bstatic+ -lstdc+++ -lpthread+ -Wl,-Bdynamic+ )endif()-endif()-target_compile_options(engine_impl PRIVATE -fPIC)+ set_target_properties(+ ${MODULE_TARGET}+ PROPERTIES+ LIBRARY_OUTPUT_DIRECTORY "${OV_PY_OUTPUT_DIR}"+ RUNTIME_OUTPUT_DIRECTORY "${OV_PY_OUTPUT_DIR}"+ OUTPUT_NAME "${module_name}"+ SUFFIX "${OV_PY_EXT_SUFFIX}"+ )+endfunction()-pybind11_add_module(- engine- MODULE- pybind11_interface.cpp-)+set(OV_ENGINE_IMPL_TARGET "")-if(NOT DEFINED PY_OUTPUT_NAME)- set(PY_OUTPUT_NAME "engine")-endif()+if(OV_PLATFORM_X86)+ set(OV_BUILT_X86_VARIANTS)-if(NOT DEFINED PY_OUTPUT_SUFFIX)- set(PY_OUTPUT_SUFFIX ".so")- if(WIN32)- set(PY_OUTPUT_SUFFIX ".pyd")- endif()-endif()+ foreach(OV_VARIANT IN LISTS OV_X86_BUILD_VARIANTS)+ ov_get_x86_variant_flags("${OV_VARIANT}" OV_VARIANT_FLAGS OV_VARIANT_DEFS OV_VARIANT_SUPPORTED)+ if(NOT OV_VARIANT_SUPPORTED)+ message(STATUS "Skipping unsupported x86 engine variant: ${OV_VARIANT}")+ continue()+ endif()-set_target_properties(- engine- PROPERTIES- LIBRARY_OUTPUT_DIRECTORY "${PY_OUTPUT_DIR}"- RUNTIME_OUTPUT_DIRECTORY "${PY_OUTPUT_DIR}"- SUFFIX "${PY_OUTPUT_SUFFIX}"- PYBIND11_MODULE_NAME "engine"- OUTPUT_NAME "${PY_OUTPUT_NAME}"-)+ set(INDEX_TARGET "engine_index_${OV_VARIANT}")+ add_library(${INDEX_TARGET} STATIC ${OV_INDEX_SOURCES})+ target_compile_options(${INDEX_TARGET} PRIVATE -fPIC ${OV_VARIANT_FLAGS})+ target_compile_definitions(${INDEX_TARGET} PRIVATE ${OV_VARIANT_DEFS})+ target_link_libraries(${INDEX_TARGET} PUBLIC Threads::Threads)++ ov_add_python_backend(+ "${OV_VARIANT}"+ "_x86_${OV_VARIANT}"+ INDEX_LIBRARY ${INDEX_TARGET}+ COMPILE_OPTIONS ${OV_VARIANT_FLAGS}+ COMPILE_DEFINITIONS ${OV_VARIANT_DEFS}+ )++ list(APPEND OV_BUILT_X86_VARIANTS "${OV_VARIANT}")+ endforeach()++ pybind11_add_module(engine_module_x86_caps MODULE cpu_feature_probe.cpp)+ target_include_directories(engine_module_x86_caps PRIVATE ${Python3_INCLUDE_DIRS})+ set_target_properties(+ engine_module_x86_caps+ PROPERTIES+ LIBRARY_OUTPUT_DIRECTORY "${OV_PY_OUTPUT_DIR}"+ RUNTIME_OUTPUT_DIRECTORY "${OV_PY_OUTPUT_DIR}"+ OUTPUT_NAME "_x86_caps"+ SUFFIX "${OV_PY_EXT_SUFFIX}"+ )-target_include_directories(engine PRIVATE- ${Python3_INCLUDE_DIRS}-)+ if(TARGET engine_index_sse3)+ add_library(engine_impl INTERFACE)+ target_link_libraries(engine_impl INTERFACE engine_common engine_index_sse3 Threads::Threads)+ if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")+ if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "9.0")+ target_link_libraries(engine_impl INTERFACE stdc++fs)+ endif()+ elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")+ if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "10.0")+ target_link_libraries(engine_impl INTERFACE c++fs)+ endif()+ elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")+ if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "11.0")+ target_link_libraries(engine_impl INTERFACE c++fs)+ endif()+ endif()+ set(OV_ENGINE_IMPL_TARGET "engine_impl")+ endif()-target_link_libraries(engine PRIVATE- engine_impl- Threads::Threads-)+ message(STATUS "OpenViking x86 engine variants: ${OV_BUILT_X86_VARIANTS}")+else()+ add_library(engine_index_native STATIC ${OV_INDEX_SOURCES})+ target_compile_options(engine_index_native PRIVATE -fPIC)+ target_link_libraries(engine_index_native PUBLIC Threads::Threads)+ if(OV_PLATFORM_ARM)+ target_link_libraries(engine_index_native PUBLIC krl)+ endif()-if(MINGW)- target_link_libraries(engine PRIVATE- -static-libgcc- -static-libstdc++- -Wl,-Bstatic- -lstdc++- -lpthread- -Wl,-Bdynamic- )-endif()+ ov_add_python_backend("native" "_native" INDEX_LIBRARY engine_index_native)++ add_library(engine_impl INTERFACE)+ target_link_libraries(engine_impl INTERFACE engine_common engine_index_native Threads::Threads)+ if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")+ if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "9.0")+ target_link_libraries(engine_impl INTERFACE stdc++fs)+ endif()+ elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")+ if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "10.0")+ target_link_libraries(engine_impl INTERFACE c++fs)+ endif()+ elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")+ if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "11.0")+ target_link_libraries(engine_impl INTERFACE c++fs)+ endif()+ endif()+ set(OV_ENGINE_IMPL_TARGET "engine_impl")-if(NOT DEFINED PY_OUTPUT_DIR)- set(PY_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/${PY_PACKAGE_NAME})+ message(STATUS "OpenViking native engine backend: _native")endif()diff --git a/src/cpu_feature_probe.cpp b/src/cpu_feature_probe.cppnew file mode 100644index 000000000..f1df44f1a--- /dev/null+++ b/src/cpu_feature_probe.cpp@@ -0,0 +1,118 @@+// Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+// SPDX-License-Identifier: Apache-2.0+#include <pybind11/pybind11.h>+#include <pybind11/stl.h>++#include <vector>++#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86)+#if defined(_MSC_VER)+#include <immintrin.h>+#include <intrin.h>+#else+#include <cpuid.h>+#include <immintrin.h>+#endif+#endif++namespace py = pybind11;++namespace {++struct CpuFeatures {+ bool sse3 = false;+ bool avx = false;+ bool avx2 = false;+ bool avx512f = false;+ bool avx512dq = false;+ bool avx512bw = false;+ bool avx512vl = false;+};++#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86)+void cpuid(int regs[4], int leaf, int subleaf) {+#if defined(_MSC_VER)+ __cpuidex(regs, leaf, subleaf);+#else+ __cpuid_count(leaf, subleaf, regs[0], regs[1], regs[2], regs[3]);+#endif+}++unsigned long long xgetbv(unsigned int index) {+#if defined(_MSC_VER)+ return _xgetbv(index);+#else+ unsigned int eax = 0;+ unsigned int edx = 0;+ __asm__ volatile(".byte 0x0f, 0x01, 0xd0"+ : "=a"(eax), "=d"(edx)+ : "c"(index));+ return (static_cast<unsigned long long>(edx) << 32) | eax;+#endif+}++CpuFeatures detect_cpu_features() {+ CpuFeatures features;+ int regs[4] = {0, 0, 0, 0};++ cpuid(regs, 1, 0);+ features.sse3 = (regs[2] & (1 << 0)) != 0;+ const bool osxsave = (regs[2] & (1 << 27)) != 0;+ const bool avx_hw = (regs[2] & (1 << 28)) != 0;++ if (!(osxsave && avx_hw)) {+ return features;+ }++ const auto xcr0 = xgetbv(0);+ const bool avx_os = (xcr0 & 0x6) == 0x6;+ if (!avx_os) {+ return features;+ }++ features.avx = true;++ cpuid(regs, 7, 0);+ features.avx2 = (regs[1] & (1 << 5)) != 0;+ features.avx512f = (regs[1] & (1 << 16)) != 0;+ features.avx512dq = (regs[1] & (1 << 17)) != 0;+ features.avx512bw = (regs[1] & (1 << 30)) != 0;+ features.avx512vl = (regs[1] & (1u << 31)) != 0;++ const bool avx512_os = (xcr0 & 0xe6) == 0xe6;+ if (!avx512_os) {+ features.avx512f = false;+ features.avx512dq = false;+ features.avx512bw = false;+ features.avx512vl = false;+ }++ return features;+}+#else+CpuFeatures detect_cpu_features() { return CpuFeatures{}; }+#endif++std::vector<std::string> get_supported_variants() {+ std::vector<std::string> variants;+ const auto features = detect_cpu_features();++ if (features.sse3) {+ variants.emplace_back("x86_sse3");+ }+ if (features.avx && features.avx2) {+ variants.emplace_back("x86_avx2");+ }+ if (features.avx && features.avx512f && features.avx512dq &&+ features.avx512bw && features.avx512vl) {+ variants.emplace_back("x86_avx512");+ }+ return variants;+}++} // namespace++PYBIND11_MODULE(_x86_caps, m) {+ m.def("get_supported_variants", &get_supported_variants,+ "Return CPU-supported x86 engine variants");+}diff --git a/src/pybind11_interface.cpp b/src/pybind11_interface.cppindex 55c55e2d5..6533cd2fd 100644--- a/src/pybind11_interface.cpp+++ b/src/pybind11_interface.cpp@@ -15,7 +15,13 @@namespace py = pybind11;namespace vdb = vectordb;-PYBIND11_MODULE(engine, m) {+#ifndef OV_PY_MODULE_NAME+#define OV_PY_MODULE_NAME engine+#endif++#define OV_EXPAND_MACRO(name) name++PYBIND11_MODULE(OV_EXPAND_MACRO(OV_PY_MODULE_NAME), m) {m.def("init_logging", &vdb::init_logging, "Initialize logging");py::enum_<vdb::FieldType>(m, "FieldType")diff --git a/tests/engine/CMakeLists.txt b/tests/engine/CMakeLists.txtindex 5eedb5338..4461cfca2 100644--- a/tests/engine/CMakeLists.txt+++ b/tests/engine/CMakeLists.txt@@ -25,6 +25,10 @@ add_executable(test_index_enginetest_index_engine.cpp)+if(NOT TARGET engine_impl)+ message(FATAL_ERROR "engine_impl compatibility target was not created")+endif()+target_link_libraries(test_index_engine PRIVATEengine_implThreads::Threadsdiff --git a/tests/misc/test_vectordb_engine_loader.py b/tests/misc/test_vectordb_engine_loader.pynew file mode 100644index 000000000..a3d9442be--- /dev/null+++ b/tests/misc/test_vectordb_engine_loader.py@@ -0,0 +1,120 @@+import importlib+import importlib.util+import platform+import sys+import types+from pathlib import Path++import pytest++REPO_ROOT = Path(__file__).resolve().parents[2]+ENGINE_INIT = REPO_ROOT / "openviking" / "storage" / "vectordb" / "engine" / "__init__.py"+++def _install_package_stubs(monkeypatch):+ packages = {+ "openviking": REPO_ROOT / "openviking",+ "openviking.storage": REPO_ROOT / "openviking" / "storage",+ "openviking.storage.vectordb": REPO_ROOT / "openviking" / "storage" / "vectordb",+ }+ for name, path in packages.items():+ module = types.ModuleType(name)+ module.__path__ = [str(path)] # type: ignore[attr-defined]+ monkeypatch.setitem(sys.modules, name, module)+++def _load_engine_module(+ monkeypatch, *, machine, available_backends, cpu_variants, env_variant=None+):+ _install_package_stubs(monkeypatch)++ monkeypatch.setattr(platform, "machine", lambda: machine)+ if env_variant is None:+ monkeypatch.delenv("OV_ENGINE_VARIANT", raising=False)+ else:+ monkeypatch.setenv("OV_ENGINE_VARIANT", env_variant)++ original_import_module = importlib.import_module+ original_find_spec = importlib.util.find_spec++ def fake_import_module(name, package=None):+ if package == "openviking.storage.vectordb.engine" and name == "._x86_caps":+ caps = types.SimpleNamespace(+ get_supported_variants=lambda: list(cpu_variants),+ )+ return caps++ if package == "openviking.storage.vectordb.engine" and name.startswith("._"):+ backend_name = name[2:].lstrip("_")+ if backend_name not in available_backends:+ raise ModuleNotFoundError(name)+ return types.SimpleNamespace(+ BACKEND_NAME=backend_name,+ IndexEngine=f"IndexEngine:{backend_name}",+ PersistStore=f"PersistStore:{backend_name}",+ VolatileStore=f"VolatileStore:{backend_name}",+ )++ return original_import_module(name, package)++ def fake_find_spec(name, package=None):+ fullname = importlib.util.resolve_name(name, package) if name.startswith(".") else name+ if fullname == "openviking.storage.vectordb.engine._x86_caps":+ return object()+ if fullname.startswith("openviking.storage.vectordb.engine."):+ backend_name = fullname.rsplit(".", 1)[-1].lstrip("_")+ if backend_name in available_backends:+ return object()+ return None+ return original_find_spec(name, package)++ monkeypatch.setattr(importlib, "import_module", fake_import_module)+ monkeypatch.setattr(importlib.util, "find_spec", fake_find_spec)++ spec = importlib.util.spec_from_file_location(+ "openviking.storage.vectordb.engine",+ ENGINE_INIT,+ submodule_search_locations=[str(ENGINE_INIT.parent)],+ )+ module = importlib.util.module_from_spec(spec)+ monkeypatch.setitem(sys.modules, "openviking.storage.vectordb.engine", module)+ assert spec.loader is not None+ spec.loader.exec_module(module)+ return module+++def test_engine_loader_auto_selects_best_supported_x86_backend(monkeypatch):+ module = _load_engine_module(+ monkeypatch,+ machine="x86_64",+ available_backends={"x86_sse3", "x86_avx2", "x86_avx512"},+ cpu_variants={"x86_sse3", "x86_avx2"},+ )++ assert module.ENGINE_VARIANT == "x86_avx2"+ assert module.IndexEngine == "IndexEngine:x86_avx2"+ assert module.AVAILABLE_ENGINE_VARIANTS == ("x86_sse3", "x86_avx2", "x86_avx512")+++def test_engine_loader_uses_native_backend_on_non_x86(monkeypatch):+ module = _load_engine_module(+ monkeypatch,+ machine="arm64",+ available_backends={"native"},+ cpu_variants=set(),+ )++ assert module.ENGINE_VARIANT == "native"+ assert module.PersistStore == "PersistStore:native"+ assert module.AVAILABLE_ENGINE_VARIANTS == ("native",)+++def test_engine_loader_rejects_forced_unsupported_variant(monkeypatch):+ with pytest.raises(ImportError, match="x86_avx512"):+ _load_engine_module(+ monkeypatch,+ machine="x86_64",+ available_backends={"x86_sse3", "x86_avx2"},+ cpu_variants={"x86_sse3", "x86_avx2"},+ env_variant="x86_avx512",+ )diff --git a/tests/misc/test_x86_profiles.py b/tests/misc/test_x86_profiles.pynew file mode 100644index 000000000..df49b1459--- /dev/null+++ b/tests/misc/test_x86_profiles.py@@ -0,0 +1,17 @@+from build_support.x86_profiles import get_host_engine_build_config+++def test_x86_host_uses_sse3_extension_baseline():+ config = get_host_engine_build_config("x86_64")++ assert config.primary_extension == "openviking.storage.vectordb.engine._x86_sse3"+ assert config.cmake_variants == ("sse3", "avx2", "avx512")+ assert config.is_x86 is True+++def test_non_x86_host_uses_native_extension_baseline():+ config = get_host_engine_build_config("aarch64")++ assert config.primary_extension == "openviking.storage.vectordb.engine._native"+ assert config.cmake_variants == ()+ assert config.is_x86 is False