Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
Fix Windows engine wheel runtime packaging (#993)
Jiahui Zhou84d2cf8900d8
diff --git a/.github/workflows/_build.yml b/.github/workflows/_build.ymlindex da029f210..b77e505a7 100644--- a/.github/workflows/_build.yml+++ b/.github/workflows/_build.yml@@ -429,6 +429,31 @@ jobs:- name: Build package (Wheel Only)run: uv build --wheel+ - name: Smoke test built wheel (Windows)+ if: runner.os == 'Windows'+ shell: bash+ run: |+ python -m pip install --upgrade pip+ python -m pip install --force-reinstall dist/*.whl++ cd "$RUNNER_TEMP"+ python - <<'PY'+ import importlib.util++ import openviking.storage.vectordb.engine as engine++ print(f"Loaded runtime engine variant {engine.ENGINE_VARIANT}")+ print(f"Available engine variants {engine.AVAILABLE_ENGINE_VARIANTS}")++ module_name = f"openviking.storage.vectordb.engine._{engine.ENGINE_VARIANT}"+ backend_spec = importlib.util.find_spec(module_name)+ if backend_spec is None or backend_spec.origin is None:+ raise SystemExit(f"backend module {module_name} was not installed")++ print(f"Imported backend module {module_name}")+ print(f"Backend module origin {backend_spec.origin}")+ PY+- name: Store the distribution packagesuses: actions/upload-artifact@v7with:diff --git a/openviking/storage/vectordb/engine/__init__.py b/openviking/storage/vectordb/engine/__init__.pyindex f87b45e82..843103e92 100644--- a/openviking/storage/vectordb/engine/__init__.py+++ b/openviking/storage/vectordb/engine/__init__.py@@ -27,6 +27,7 @@ _REQUEST_ALIASES = {"avx2": "x86_avx2","avx512": "x86_avx512",}+_WINDOWS_DLL_DIR_HANDLES = []def _is_x86_machine(machine: str | None = None) -> bool:@@ -143,6 +144,25 @@ def _load_backend(variant: str) -> ModuleType:return importlib.import_module(f".{module_name}", __name__)+def _register_windows_dll_dirs(module_path: Path) -> None:+ if sys.platform != "win32" or not hasattr(os, "add_dll_directory"):+ return++ package_root = module_path.parents[2]+ search_dirs = [+ module_path,+ package_root / "lib",+ package_root / "bin",+ ]+ seen = set()+ for search_dir in search_dirs:+ resolved = search_dir.resolve()+ if resolved in seen or not resolved.exists():+ continue+ seen.add(resolved)+ _WINDOWS_DLL_DIR_HANDLES.append(os.add_dll_directory(str(resolved)))++def _export_backend(module: ModuleType) -> tuple[str, ...]:if getattr(module, "_ENGINE_BACKEND_API", None) == "abi3-v1":exports = build_abi3_exports(module)@@ -185,6 +205,7 @@ if _SELECTED_VARIANT is None:_EXPORTED_NAMES = ()else:ENGINE_VARIANT = _SELECTED_VARIANT+ _register_windows_dll_dirs(Path(__file__).resolve().parent)_BACKEND = _load_backend(ENGINE_VARIANT)_EXPORTED_NAMES = _export_backend(_BACKEND)diff --git a/setup.py b/setup.pyindex 91be3535b..f7baaf65c 100644--- a/setup.py+++ b/setup.py@@ -34,6 +34,41 @@ ENGINE_SOURCE_DIR = "src/"ENGINE_BUILD_CONFIG = get_host_engine_build_config(platform.machine())+def _get_windows_python_sabi_library() -> Path:+ """Return the stable-ABI Python library path for Windows abi3 extensions."""+ candidate_roots = []+ for raw_root in (+ sys.base_prefix,+ sys.base_exec_prefix,+ sysconfig.get_config_var("installed_base"),+ sysconfig.get_config_var("base"),+ ):+ if not raw_root:+ continue+ candidate_root = Path(raw_root).resolve()+ if candidate_root not in candidate_roots:+ candidate_roots.append(candidate_root)++ candidate_paths = []+ for root in candidate_roots:+ candidate_paths.extend(+ [+ root / "libs" / "python3.lib",+ root / "python3.dll",+ ]+ )++ for candidate_path in candidate_paths:+ if candidate_path.exists():+ return candidate_path++ searched = ", ".join(str(path) for path in candidate_paths) or "<none>"+ raise RuntimeError(+ "Could not locate the Windows stable-ABI Python library for abi3 engine modules. "+ f"Searched: {searched}"+ )++class OpenVikingBuildExt(build_ext):"""Build OpenViking runtime artifacts and Python native extensions."""@@ -397,6 +432,8 @@ class OpenVikingBuildExt(build_ext):if target_arch:cmake_args.append(f"-DCMAKE_OSX_ARCHITECTURES={target_arch}")elif sys.platform == "win32":+ windows_python_sabi_library = _get_windows_python_sabi_library()+ cmake_args.append(f"-DOV_PYTHON_SABI_LIBRARY={windows_python_sabi_library}")cmake_args.extend(["-G", "MinGW Makefiles"])self.spawn([self.cmake_executable] + cmake_args)diff --git a/src/CMakeLists.txt b/src/CMakeLists.txtindex ed1663e75..1a79b8a43 100644--- a/src/CMakeLists.txt+++ b/src/CMakeLists.txt@@ -8,6 +8,7 @@ include(CMakeParseArguments)set(OV_X86_BUILD_VARIANTS "sse3;avx2;avx512" CACHE STRING "x86 engine variants to build")set(OV_PY_OUTPUT_DIR "" CACHE PATH "Output directory for Python extension modules")set(OV_PY_EXT_SUFFIX ".so" CACHE STRING "Python extension suffix, including ABI tag if needed")+set(OV_PYTHON_SABI_LIBRARY "" CACHE FILEPATH "Stable-ABI Python import library or DLL for Windows abi3 modules")if(NOT OV_PY_OUTPUT_DIR)set(OV_PY_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/python_engine")@@ -43,6 +44,9 @@ set(CMAKE_CXX_LINK_EXECUTABLE "${CMAKE_CXX_LINK_EXECUTABLE} -lpthread")set(Python3_ARCH_INCLUDE_DIR "/usr/include/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/")find_package(Python3 COMPONENTS Interpreter Development REQUIRED)+if(WIN32 AND CMAKE_VERSION VERSION_GREATER_EQUAL "3.26")+ find_package(Python3 COMPONENTS Development.SABIModule QUIET)+endif()if(UNIX AND NOT APPLE)set(Python3_LIBRARIES "")@@ -183,6 +187,25 @@ function(ov_get_x86_variant_flags variant out_flags out_defs out_supported)set(${out_supported} ${OV_SUPPORTED} PARENT_SCOPE)endfunction()+function(ov_link_python_abi target_name)+ if(WIN32)+ if(OV_PYTHON_SABI_LIBRARY)+ target_link_libraries(${target_name} PRIVATE "${OV_PYTHON_SABI_LIBRARY}")+ elseif(TARGET Python3::SABIModule)+ target_link_libraries(${target_name} PRIVATE Python3::SABIModule)+ else()+ message(+ FATAL_ERROR+ "Windows abi3 modules require OV_PYTHON_SABI_LIBRARY or Python3::SABIModule"+ )+ endif()+ endif()++ if(APPLE)+ target_link_options(${target_name} PRIVATE "-undefined" "dynamic_lookup")+ endif()+endfunction()+function(ov_add_python_backend backend_suffix module_name)set(oneValueArgs INDEX_LIBRARY)set(multiValueArgs COMPILE_OPTIONS COMPILE_DEFINITIONS)@@ -207,9 +230,7 @@ function(ov_add_python_backend backend_suffix module_name)${OV_BACKEND_INDEX_LIBRARY}Threads::Threads)- if(WIN32 AND TARGET Python3::Python)- target_link_libraries(${MODULE_TARGET} PRIVATE Python3::Python)- endif()+ ov_link_python_abi(${MODULE_TARGET})ov_link_filesystem_libs(${MODULE_TARGET})if(MINGW)@@ -232,10 +253,6 @@ function(ov_add_python_backend backend_suffix module_name)OUTPUT_NAME "${module_name}"SUFFIX "${OV_PY_EXT_SUFFIX}")-- if(APPLE)- target_link_options(${MODULE_TARGET} PRIVATE "-undefined" "dynamic_lookup")- endif()endfunction()set(OV_ENGINE_IMPL_TARGET "")@@ -270,9 +287,7 @@ if(OV_PLATFORM_X86)add_library(engine_module_x86_caps MODULE abi3_x86_caps.cpp)target_include_directories(engine_module_x86_caps PRIVATE ${Python3_INCLUDE_DIRS})target_compile_definitions(engine_module_x86_caps PRIVATE Py_LIMITED_API=0x030A0000)- if(WIN32 AND TARGET Python3::Python)- target_link_libraries(engine_module_x86_caps PRIVATE Python3::Python)- endif()+ ov_link_python_abi(engine_module_x86_caps)set_target_properties(engine_module_x86_capsPROPERTIES@@ -282,9 +297,6 @@ if(OV_PLATFORM_X86)OUTPUT_NAME "_x86_caps"SUFFIX "${OV_PY_EXT_SUFFIX}")- if(APPLE)- target_link_options(engine_module_x86_caps PRIVATE "-undefined" "dynamic_lookup")- endif()if(TARGET engine_index_sse3)add_library(engine_impl INTERFACE)diff --git a/tests/misc/test_abi3_packaging_config.py b/tests/misc/test_abi3_packaging_config.pyindex 2638c1c8b..3df323254 100644--- a/tests/misc/test_abi3_packaging_config.py+++ b/tests/misc/test_abi3_packaging_config.py@@ -13,10 +13,28 @@ def test_packaging_only_includes_abi3_engine_extensions():assert "storage/vectordb/engine/*.abi3.so" in setup_pyassert "storage/vectordb/engine/*.abi3.so" in pyproject+ assert "storage/vectordb/engine/*.dll" not in setup_py+ assert "storage/vectordb/engine/*.dll" not in pyprojectassert "storage/vectordb/engine/*.so" not in setup_pyassert "storage/vectordb/engine/*.so" not in pyproject+def test_windows_engine_loader_registers_dll_search_paths():+ engine_init = _read_text("openviking/storage/vectordb/engine/__init__.py")++ assert "add_dll_directory" in engine_init+ assert "module_path" in engine_init+ assert 'package_root / "lib"' in engine_init+ assert 'package_root / "bin"' in engine_init+++def test_setup_no_longer_bundles_mingw_runtime_dlls_for_engine():+ setup_py = _read_text("setup.py")++ assert "WINDOWS_ENGINE_RUNTIME_DLL_PATTERNS" not in setup_py+ assert "_stage_windows_engine_runtime_dlls" not in setup_py++def test_release_workflows_default_to_single_cp310_and_drop_pybind11():build_workflow = _read_text(".github/workflows/_build.yml")release_workflow = _read_text(".github/workflows/release.yml")@@ -46,6 +64,33 @@ def test_release_build_workflow_no_longer_defines_extra_wheel_verify_jobs():assert "verify-macos-14-wheel-on-macos-15:" not in build_workflow+def test_build_workflow_smoke_tests_windows_wheel_engine_import():+ build_workflow = _read_text(".github/workflows/_build.yml")++ assert "Smoke test built wheel (Windows)" in build_workflow+ assert "python -m pip install --force-reinstall dist/*.whl" in build_workflow+ assert 'cd "$RUNNER_TEMP"' in build_workflow+ assert "import openviking.storage.vectordb.engine as engine" in build_workflow+ assert "engine.ENGINE_VARIANT" in build_workflow+++def test_windows_abi3_backend_uses_stable_python_linkage():+ setup_py = _read_text("setup.py")+ src_cmake = _read_text("src/CMakeLists.txt")++ assert "OV_PYTHON_SABI_LIBRARY" in setup_py+ assert "python3.dll" in setup_py+ assert "OV_PYTHON_SABI_LIBRARY" in src_cmake+ assert "Python3::Python" not in src_cmake+++def test_build_workflow_no_longer_defines_windows_python312_verify_job():+ build_workflow = _read_text(".github/workflows/_build.yml")++ assert "verify-windows-abi3-wheel-on-python312:" not in build_workflow+ assert "Smoke test Windows abi3 wheel on Python 3.12" not in build_workflow++def test_abi3_backend_releases_gil_and_rejects_invalid_storage_op_type():backend_source = _read_text("src/abi3_engine_backend.cpp")