Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

Fix Windows engine wheel runtime packaging (#993)

Jiahui Zhou84d2cf8900d8
diff --git a/.github/workflows/_build.yml b/.github/workflows/_build.yml
index da029f210..b77e505a7 100644
--- a/.github/workflows/_build.yml
+++ b/.github/workflows/_build.yml
@@ -429,6 +429,31 @@ jobs:
- name: Build package (Wheel Only)
run: uv build --wheel
+ - name: Smoke test built wheel (Windows)
+ if: runner.os == 'Windows'
+ shell: bash
+ run: |
+ python -m pip install --upgrade pip
+ python -m pip install --force-reinstall dist/*.whl
+
+ cd "$RUNNER_TEMP"
+ python - <<'PY'
+ import importlib.util
+
+ import openviking.storage.vectordb.engine as engine
+
+ print(f"Loaded runtime engine variant {engine.ENGINE_VARIANT}")
+ print(f"Available engine variants {engine.AVAILABLE_ENGINE_VARIANTS}")
+
+ module_name = f"openviking.storage.vectordb.engine._{engine.ENGINE_VARIANT}"
+ backend_spec = importlib.util.find_spec(module_name)
+ if backend_spec is None or backend_spec.origin is None:
+ raise SystemExit(f"backend module {module_name} was not installed")
+
+ print(f"Imported backend module {module_name}")
+ print(f"Backend module origin {backend_spec.origin}")
+ PY
+
- name: Store the distribution packages
uses: actions/upload-artifact@v7
with:
diff --git a/openviking/storage/vectordb/engine/__init__.py b/openviking/storage/vectordb/engine/__init__.py
index f87b45e82..843103e92 100644
--- a/openviking/storage/vectordb/engine/__init__.py
+++ b/openviking/storage/vectordb/engine/__init__.py
@@ -27,6 +27,7 @@ _REQUEST_ALIASES = {
"avx2": "x86_avx2",
"avx512": "x86_avx512",
}
+_WINDOWS_DLL_DIR_HANDLES = []
def _is_x86_machine(machine: str | None = None) -> bool:
@@ -143,6 +144,25 @@ def _load_backend(variant: str) -> ModuleType:
return importlib.import_module(f".{module_name}", __name__)
+def _register_windows_dll_dirs(module_path: Path) -> None:
+ if sys.platform != "win32" or not hasattr(os, "add_dll_directory"):
+ return
+
+ package_root = module_path.parents[2]
+ search_dirs = [
+ module_path,
+ package_root / "lib",
+ package_root / "bin",
+ ]
+ seen = set()
+ for search_dir in search_dirs:
+ resolved = search_dir.resolve()
+ if resolved in seen or not resolved.exists():
+ continue
+ seen.add(resolved)
+ _WINDOWS_DLL_DIR_HANDLES.append(os.add_dll_directory(str(resolved)))
+
+
def _export_backend(module: ModuleType) -> tuple[str, ...]:
if getattr(module, "_ENGINE_BACKEND_API", None) == "abi3-v1":
exports = build_abi3_exports(module)
@@ -185,6 +205,7 @@ if _SELECTED_VARIANT is None:
_EXPORTED_NAMES = ()
else:
ENGINE_VARIANT = _SELECTED_VARIANT
+ _register_windows_dll_dirs(Path(__file__).resolve().parent)
_BACKEND = _load_backend(ENGINE_VARIANT)
_EXPORTED_NAMES = _export_backend(_BACKEND)
diff --git a/setup.py b/setup.py
index 91be3535b..f7baaf65c 100644
--- a/setup.py
+++ b/setup.py
@@ -34,6 +34,41 @@ ENGINE_SOURCE_DIR = "src/"
ENGINE_BUILD_CONFIG = get_host_engine_build_config(platform.machine())
+def _get_windows_python_sabi_library() -> Path:
+ """Return the stable-ABI Python library path for Windows abi3 extensions."""
+ candidate_roots = []
+ for raw_root in (
+ sys.base_prefix,
+ sys.base_exec_prefix,
+ sysconfig.get_config_var("installed_base"),
+ sysconfig.get_config_var("base"),
+ ):
+ if not raw_root:
+ continue
+ candidate_root = Path(raw_root).resolve()
+ if candidate_root not in candidate_roots:
+ candidate_roots.append(candidate_root)
+
+ candidate_paths = []
+ for root in candidate_roots:
+ candidate_paths.extend(
+ [
+ root / "libs" / "python3.lib",
+ root / "python3.dll",
+ ]
+ )
+
+ for candidate_path in candidate_paths:
+ if candidate_path.exists():
+ return candidate_path
+
+ searched = ", ".join(str(path) for path in candidate_paths) or "<none>"
+ raise RuntimeError(
+ "Could not locate the Windows stable-ABI Python library for abi3 engine modules. "
+ f"Searched: {searched}"
+ )
+
+
class OpenVikingBuildExt(build_ext):
"""Build OpenViking runtime artifacts and Python native extensions."""
@@ -397,6 +432,8 @@ class OpenVikingBuildExt(build_ext):
if target_arch:
cmake_args.append(f"-DCMAKE_OSX_ARCHITECTURES={target_arch}")
elif sys.platform == "win32":
+ windows_python_sabi_library = _get_windows_python_sabi_library()
+ cmake_args.append(f"-DOV_PYTHON_SABI_LIBRARY={windows_python_sabi_library}")
cmake_args.extend(["-G", "MinGW Makefiles"])
self.spawn([self.cmake_executable] + cmake_args)
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index ed1663e75..1a79b8a43 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -8,6 +8,7 @@ include(CMakeParseArguments)
set(OV_X86_BUILD_VARIANTS "sse3;avx2;avx512" CACHE STRING "x86 engine variants to build")
set(OV_PY_OUTPUT_DIR "" CACHE PATH "Output directory for Python extension modules")
set(OV_PY_EXT_SUFFIX ".so" CACHE STRING "Python extension suffix, including ABI tag if needed")
+set(OV_PYTHON_SABI_LIBRARY "" CACHE FILEPATH "Stable-ABI Python import library or DLL for Windows abi3 modules")
if(NOT OV_PY_OUTPUT_DIR)
set(OV_PY_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/python_engine")
@@ -43,6 +44,9 @@ set(CMAKE_CXX_LINK_EXECUTABLE "${CMAKE_CXX_LINK_EXECUTABLE} -lpthread")
set(Python3_ARCH_INCLUDE_DIR "/usr/include/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu/")
find_package(Python3 COMPONENTS Interpreter Development REQUIRED)
+if(WIN32 AND CMAKE_VERSION VERSION_GREATER_EQUAL "3.26")
+ find_package(Python3 COMPONENTS Development.SABIModule QUIET)
+endif()
if(UNIX AND NOT APPLE)
set(Python3_LIBRARIES "")
@@ -183,6 +187,25 @@ function(ov_get_x86_variant_flags variant out_flags out_defs out_supported)
set(${out_supported} ${OV_SUPPORTED} PARENT_SCOPE)
endfunction()
+function(ov_link_python_abi target_name)
+ if(WIN32)
+ if(OV_PYTHON_SABI_LIBRARY)
+ target_link_libraries(${target_name} PRIVATE "${OV_PYTHON_SABI_LIBRARY}")
+ elseif(TARGET Python3::SABIModule)
+ target_link_libraries(${target_name} PRIVATE Python3::SABIModule)
+ else()
+ message(
+ FATAL_ERROR
+ "Windows abi3 modules require OV_PYTHON_SABI_LIBRARY or Python3::SABIModule"
+ )
+ endif()
+ endif()
+
+ if(APPLE)
+ target_link_options(${target_name} PRIVATE "-undefined" "dynamic_lookup")
+ endif()
+endfunction()
+
function(ov_add_python_backend backend_suffix module_name)
set(oneValueArgs INDEX_LIBRARY)
set(multiValueArgs COMPILE_OPTIONS COMPILE_DEFINITIONS)
@@ -207,9 +230,7 @@ function(ov_add_python_backend backend_suffix module_name)
${OV_BACKEND_INDEX_LIBRARY}
Threads::Threads
)
- if(WIN32 AND TARGET Python3::Python)
- target_link_libraries(${MODULE_TARGET} PRIVATE Python3::Python)
- endif()
+ ov_link_python_abi(${MODULE_TARGET})
ov_link_filesystem_libs(${MODULE_TARGET})
if(MINGW)
@@ -232,10 +253,6 @@ function(ov_add_python_backend backend_suffix module_name)
OUTPUT_NAME "${module_name}"
SUFFIX "${OV_PY_EXT_SUFFIX}"
)
-
- if(APPLE)
- target_link_options(${MODULE_TARGET} PRIVATE "-undefined" "dynamic_lookup")
- endif()
endfunction()
set(OV_ENGINE_IMPL_TARGET "")
@@ -270,9 +287,7 @@ if(OV_PLATFORM_X86)
add_library(engine_module_x86_caps MODULE abi3_x86_caps.cpp)
target_include_directories(engine_module_x86_caps PRIVATE ${Python3_INCLUDE_DIRS})
target_compile_definitions(engine_module_x86_caps PRIVATE Py_LIMITED_API=0x030A0000)
- if(WIN32 AND TARGET Python3::Python)
- target_link_libraries(engine_module_x86_caps PRIVATE Python3::Python)
- endif()
+ ov_link_python_abi(engine_module_x86_caps)
set_target_properties(
engine_module_x86_caps
PROPERTIES
@@ -282,9 +297,6 @@ if(OV_PLATFORM_X86)
OUTPUT_NAME "_x86_caps"
SUFFIX "${OV_PY_EXT_SUFFIX}"
)
- if(APPLE)
- target_link_options(engine_module_x86_caps PRIVATE "-undefined" "dynamic_lookup")
- endif()
if(TARGET engine_index_sse3)
add_library(engine_impl INTERFACE)
diff --git a/tests/misc/test_abi3_packaging_config.py b/tests/misc/test_abi3_packaging_config.py
index 2638c1c8b..3df323254 100644
--- a/tests/misc/test_abi3_packaging_config.py
+++ b/tests/misc/test_abi3_packaging_config.py
@@ -13,10 +13,28 @@ def test_packaging_only_includes_abi3_engine_extensions():
assert "storage/vectordb/engine/*.abi3.so" in setup_py
assert "storage/vectordb/engine/*.abi3.so" in pyproject
+ assert "storage/vectordb/engine/*.dll" not in setup_py
+ assert "storage/vectordb/engine/*.dll" not in pyproject
assert "storage/vectordb/engine/*.so" not in setup_py
assert "storage/vectordb/engine/*.so" not in pyproject
+def test_windows_engine_loader_registers_dll_search_paths():
+ engine_init = _read_text("openviking/storage/vectordb/engine/__init__.py")
+
+ assert "add_dll_directory" in engine_init
+ assert "module_path" in engine_init
+ assert 'package_root / "lib"' in engine_init
+ assert 'package_root / "bin"' in engine_init
+
+
+def test_setup_no_longer_bundles_mingw_runtime_dlls_for_engine():
+ setup_py = _read_text("setup.py")
+
+ assert "WINDOWS_ENGINE_RUNTIME_DLL_PATTERNS" not in setup_py
+ assert "_stage_windows_engine_runtime_dlls" not in setup_py
+
+
def test_release_workflows_default_to_single_cp310_and_drop_pybind11():
build_workflow = _read_text(".github/workflows/_build.yml")
release_workflow = _read_text(".github/workflows/release.yml")
@@ -46,6 +64,33 @@ def test_release_build_workflow_no_longer_defines_extra_wheel_verify_jobs():
assert "verify-macos-14-wheel-on-macos-15:" not in build_workflow
+def test_build_workflow_smoke_tests_windows_wheel_engine_import():
+ build_workflow = _read_text(".github/workflows/_build.yml")
+
+ assert "Smoke test built wheel (Windows)" in build_workflow
+ assert "python -m pip install --force-reinstall dist/*.whl" in build_workflow
+ assert 'cd "$RUNNER_TEMP"' in build_workflow
+ assert "import openviking.storage.vectordb.engine as engine" in build_workflow
+ assert "engine.ENGINE_VARIANT" in build_workflow
+
+
+def test_windows_abi3_backend_uses_stable_python_linkage():
+ setup_py = _read_text("setup.py")
+ src_cmake = _read_text("src/CMakeLists.txt")
+
+ assert "OV_PYTHON_SABI_LIBRARY" in setup_py
+ assert "python3.dll" in setup_py
+ assert "OV_PYTHON_SABI_LIBRARY" in src_cmake
+ assert "Python3::Python" not in src_cmake
+
+
+def test_build_workflow_no_longer_defines_windows_python312_verify_job():
+ build_workflow = _read_text(".github/workflows/_build.yml")
+
+ assert "verify-windows-abi3-wheel-on-python312:" not in build_workflow
+ assert "Smoke test Windows abi3 wheel on Python 3.12" not in build_workflow
+
+
def test_abi3_backend_releases_gil_and_rejects_invalid_storage_op_type():
backend_source = _read_text("src/abi3_engine_backend.cpp")