Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
import importlib
import json
import os
import platform
import shutil
import subprocess
import sys
import sysconfig
from pathlib import Path
from setuptools import Distribution, Extension, setup
from setuptools.command.build_ext import build_ext
from setuptools.command.build_py import build_py
from setuptools.command.sdist import sdist
try:
from wheel.bdist_wheel import bdist_wheel
except ImportError: # pragma: no cover - local build_ext may not have wheel installed
bdist_wheel = None
SETUP_DIR = Path(__file__).resolve().parent
if str(SETUP_DIR) not in sys.path:
sys.path.insert(0, str(SETUP_DIR))
get_host_engine_build_config = importlib.import_module(
"scripts.build_support.x86_profiles"
).get_host_engine_build_config
resolve_openviking_version = importlib.import_module(
"scripts.build_support.versioning"
).resolve_openviking_version
pypi_long_description = importlib.import_module(
"scripts.build_support.readme"
).pypi_long_description
write_system_assets_manifest = importlib.import_module(
"scripts.build_support.system_assets_manifest"
).write_system_assets_manifest
CMAKE_PATH = shutil.which("cmake") or "cmake"
C_COMPILER_PATH = os.environ.get("CC") or shutil.which("gcc") or "gcc"
CXX_COMPILER_PATH = os.environ.get("CXX") or shutil.which("g++") or "g++"
ENGINE_SOURCE_DIR = "src/"
ENGINE_BUILD_CONFIG = get_host_engine_build_config(platform.machine())
SKIP_CPP_BUILD = os.environ.get("OV_SKIP_CPP_BUILD") == "1"
def _sanitize_native_build_env(env):
"""Keep Rust native builds from accidentally linking against Linuxbrew libs.
On older glibc systems, Homebrew-provided native libraries can require a newer
libc than the host linker/runtime supports. When pkg-config resolves xz/bzip2
from Linuxbrew, Cargo inherits those library search paths and link fails.
"""
sanitized_env = env.copy()
pkg_config = sanitized_env.get("PKG_CONFIG") or shutil.which("pkg-config")
if pkg_config and "linuxbrew" in os.path.realpath(pkg_config).lower():
system_pkg_config = "/usr/bin/pkg-config"
if Path(system_pkg_config).exists():
sanitized_env["PKG_CONFIG"] = system_pkg_config
for key in ("PKG_CONFIG_PATH", "LIBRARY_PATH", "LD_LIBRARY_PATH"):
value = sanitized_env.get(key)
if not value:
continue
kept_paths = [
path
for path in value.split(os.pathsep)
if path and "linuxbrew" not in os.path.realpath(path).lower()
]
if kept_paths:
sanitized_env[key] = os.pathsep.join(kept_paths)
else:
sanitized_env.pop(key, None)
return sanitized_env
def _get_windows_python_sabi_library() -> Path:
"""Return the stable-ABI Python library path for Windows abi3 extensions."""
candidate_roots = []
for raw_root in (
sys.base_prefix,
sys.base_exec_prefix,
sysconfig.get_config_var("installed_base"),
sysconfig.get_config_var("base"),
):
if not raw_root:
continue
candidate_root = Path(raw_root).resolve()
if candidate_root not in candidate_roots:
candidate_roots.append(candidate_root)
candidate_paths = []
for root in candidate_roots:
candidate_paths.extend(
[
root / "libs" / "python3.lib",
root / "python3.dll",
]
)
for candidate_path in candidate_paths:
if candidate_path.exists():
return candidate_path
searched = ", ".join(str(path) for path in candidate_paths) or "<none>"
raise RuntimeError(
"Could not locate the Windows stable-ABI Python library for abi3 engine modules. "
f"Searched: {searched}"
)
class OpenVikingBuildExt(build_ext):
"""Build OpenViking runtime artifacts and Python native extensions."""
def run(self):
self.build_ov_cli_artifact()
self.build_ragfs_python_artifact()
self.cmake_executable = CMAKE_PATH
if SKIP_CPP_BUILD:
print("[SKIP] C++ vector engine build disabled by OV_SKIP_CPP_BUILD=1")
else:
for ext in self.extensions:
self.build_extension(ext)
def _copy_artifact(self, src, dst):
"""Copy a build artifact into the package tree and preserve executability."""
print(f"Copying artifact from {src} to {dst}")
dst.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(str(src), str(dst))
if sys.platform != "win32":
os.chmod(str(dst), 0o755)
def _copy_artifacts_to_build_lib(self, target_binary=None, target_lib=None):
"""Copy built artifacts into build_lib so wheel packaging can include them."""
if self.build_lib:
build_pkg_dir = Path(self.build_lib) / "openviking"
if target_binary and target_binary.exists():
self._copy_artifact(target_binary, build_pkg_dir / "bin" / target_binary.name)
if target_lib and target_lib.exists():
self._copy_artifact(target_lib, build_pkg_dir / "lib" / target_lib.name)
def _require_artifact(self, artifact_path, artifact_name, stage_name):
"""Abort the build immediately when a required artifact is missing."""
if artifact_path.exists():
return
raise RuntimeError(
f"{stage_name} did not produce required {artifact_name} at {artifact_path}"
)
def _run_stage_with_artifact_checks(
self, stage_name, build_fn, required_artifacts, on_success=None
):
"""Run a build stage and always validate its required outputs on normal return."""
build_fn()
for artifact_path, artifact_name in required_artifacts:
self._require_artifact(artifact_path, artifact_name, stage_name)
if on_success:
on_success()
def _resolve_cargo_target_dir(self, cargo_project_dir, env):
"""Resolve the Cargo target directory for workspace and overridden builds."""
configured_target_dir = env.get("CARGO_TARGET_DIR")
if configured_target_dir:
return Path(configured_target_dir).resolve()
try:
result = subprocess.run(
["cargo", "metadata", "--format-version", "1", "--no-deps"],
cwd=str(cargo_project_dir),
env=env,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
metadata = json.loads(result.stdout.decode("utf-8"))
target_directory = metadata.get("target_directory")
if target_directory:
return Path(target_directory).resolve()
except Exception as exc:
print(f"[Warning] Failed to resolve Cargo target directory via metadata: {exc}")
return cargo_project_dir.parents[1] / "target"
def build_ov_cli_artifact(self):
"""Build or reuse the ov Rust CLI binary."""
if os.environ.get("OV_SKIP_OV_BUILD") == "1":
print("[SKIP] ov CLI build disabled by OV_SKIP_OV_BUILD=1")
return
binary_name = "ov.exe" if sys.platform == "win32" else "ov"
ov_cli_dir = Path("crates/ov_cli").resolve()
ov_target_binary = Path("openviking/bin").resolve() / binary_name
self._run_stage_with_artifact_checks(
"ov CLI build",
lambda: self._build_ov_cli_artifact_impl(ov_cli_dir, binary_name, ov_target_binary),
[(ov_target_binary, binary_name)],
on_success=lambda: self._copy_artifacts_to_build_lib(ov_target_binary, None),
)
def _build_ov_cli_artifact_impl(self, ov_cli_dir, binary_name, ov_target_binary):
"""Implement ov CLI building without final artifact checks."""
prebuilt_dir = os.environ.get("OV_PREBUILT_BIN_DIR")
if prebuilt_dir:
src_bin = Path(prebuilt_dir).resolve() / binary_name
if src_bin.exists():
self._copy_artifact(src_bin, ov_target_binary)
return
if ov_cli_dir.exists() and shutil.which("cargo"):
print("Building ov CLI from source...")
try:
env = _sanitize_native_build_env(os.environ.copy())
env["OPENVIKING_VERSION"] = resolve_openviking_version(
env=env, project_root=SETUP_DIR
)
build_args = ["cargo", "build", "--release"]
target = env.get("CARGO_BUILD_TARGET")
if target:
print(f"Cross-compiling with CARGO_BUILD_TARGET={target}")
build_args.extend(["--target", target])
result = subprocess.run(
build_args,
cwd=str(ov_cli_dir),
env=env,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
if result.stdout:
print(f"Build stdout: {result.stdout.decode('utf-8', errors='replace')}")
if result.stderr:
print(f"Build stderr: {result.stderr.decode('utf-8', errors='replace')}")
cargo_target_dir = self._resolve_cargo_target_dir(ov_cli_dir, env)
if target:
built_bin = cargo_target_dir / target / "release" / binary_name
else:
built_bin = cargo_target_dir / "release" / binary_name
self._require_artifact(built_bin, binary_name, "ov CLI build")
self._copy_artifact(built_bin, ov_target_binary)
print("[OK] ov CLI built successfully from source")
except Exception as exc:
error_msg = f"Failed to build ov CLI from source: {exc}"
if isinstance(exc, subprocess.CalledProcessError):
if exc.stdout:
error_msg += (
f"\nBuild stdout:\n{exc.stdout.decode('utf-8', errors='replace')}"
)
if exc.stderr:
error_msg += (
f"\nBuild stderr:\n{exc.stderr.decode('utf-8', errors='replace')}"
)
print(f"[Error] {error_msg}")
raise RuntimeError(error_msg)
else:
if ov_target_binary.exists():
print("[Info] ov CLI binary already exists locally. Skipping source build.")
elif not ov_cli_dir.exists():
print(f"[Warning] ov CLI source directory not found at {ov_cli_dir}")
else:
print("[Warning] Cargo not found. Cannot build ov CLI from source.")
def build_ragfs_python_artifact(self):
"""Build ragfs-python (Rust RAGFS binding) via maturin and copy the native
extension into ``openviking/lib/`` so it ships inside the openviking wheel.
"""
require_ragfs_artifact = self._should_require_ragfs_artifact()
ragfs_python_dir = Path("crates/ragfs-python").resolve()
ragfs_lib_dir = Path("openviking/lib").resolve()
if not ragfs_python_dir.exists():
message = "ragfs-python source directory not found."
if require_ragfs_artifact:
raise RuntimeError(message)
print(f"[Info] {message} Skipping.")
return
if os.environ.get("OV_SKIP_RAGFS_BUILD") == "1":
message = "Skipping ragfs-python build (OV_SKIP_RAGFS_BUILD=1)"
if require_ragfs_artifact:
raise RuntimeError(f"{message} is incompatible with required wheel artifacts.")
print(f"[OK] {message}")
return
if importlib.util.find_spec("maturin") is None:
message = (
"maturin not found. ragfs-python (Rust binding) will not be built.\n"
" Install maturin to enable: pip install maturin"
)
if require_ragfs_artifact:
raise RuntimeError(message)
print(f"[SKIP] {message}")
return
import tempfile
import zipfile
with tempfile.TemporaryDirectory() as tmpdir:
try:
print("Building ragfs-python (Rust RAGFS binding) via maturin...")
env = _sanitize_native_build_env(os.environ.copy())
build_args = [
sys.executable,
"-m",
"maturin",
"build",
"--release",
"--out",
tmpdir,
]
# Respect CARGO_BUILD_TARGET for cross-compilation
target = env.get("CARGO_BUILD_TARGET")
if target:
build_args.extend(["--target", target])
result = subprocess.run(
build_args,
cwd=str(ragfs_python_dir),
env=env,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
if result.stdout:
print(result.stdout.decode("utf-8", errors="replace"))
if result.stderr:
print(result.stderr.decode("utf-8", errors="replace"))
# Extract the stable-ABI native extension from the built wheel.
whl_files = list(Path(tmpdir).glob("ragfs_python-*.whl"))
if not whl_files:
message = "maturin produced no wheel for ragfs-python."
if require_ragfs_artifact:
raise RuntimeError(message)
print(f"[Warning] {message}")
return
ragfs_lib_dir.mkdir(parents=True, exist_ok=True)
for stale_artifact in ragfs_lib_dir.glob("ragfs_python*.so"):
stale_artifact.unlink()
for stale_artifact in ragfs_lib_dir.glob("ragfs_python*.pyd"):
stale_artifact.unlink()
for stale_artifact in ragfs_lib_dir.glob("ragfs_python*.dylib"):
stale_artifact.unlink()
extracted = False
with zipfile.ZipFile(str(whl_files[0])) as zf:
for name in zf.namelist():
basename = Path(name).name
is_ragfs_extension = basename == "ragfs_python.pyd" or (
basename.startswith("ragfs_python.abi3.")
and basename.endswith((".so", ".pyd"))
)
if is_ragfs_extension:
target_path = ragfs_lib_dir / basename
with zf.open(name) as src, open(target_path, "wb") as dst:
dst.write(src.read())
if sys.platform != "win32":
os.chmod(str(target_path), 0o755)
print(f"[OK] ragfs-python: extracted {basename} -> {target_path}")
extracted = True
break
if not extracted:
message = (
"Could not find ragfs_python stable-ABI native extension in built wheel."
)
if require_ragfs_artifact:
raise RuntimeError(message)
print(f"[Warning] {message}")
else:
self._copy_artifacts_to_build_lib(target_lib=target_path)
except Exception as exc:
error_detail = ""
if isinstance(exc, subprocess.CalledProcessError):
if exc.stdout:
error_detail += exc.stdout.decode("utf-8", errors="replace")
if exc.stderr:
error_detail += exc.stderr.decode("utf-8", errors="replace")
if require_ragfs_artifact:
error_message = f"Failed to build ragfs-python: {exc}"
if error_detail:
error_message += f"\n{error_detail}"
raise RuntimeError(error_message) from exc
print(f"[Warning] Failed to build ragfs-python: {exc}")
if error_detail:
print(error_detail)
def _should_require_ragfs_artifact(self) -> bool:
"""Fail wheel builds closed when ragfs-python cannot be bundled."""
required = os.environ.get("OV_REQUIRE_RAGFS_BUILD")
if required is not None:
return required == "1"
return "bdist_wheel" in sys.argv
def build_extension(self, ext):
"""Build a single Python native extension artifact using CMake."""
if getattr(self, "_engine_extensions_built", False):
return
ext_fullpath = Path(self.get_ext_fullpath(ext.name))
ext_dir = ext_fullpath.parent.resolve()
build_dir = Path(self.build_temp) / "cmake_build"
build_dir.mkdir(parents=True, exist_ok=True)
self._clean_stale_engine_artifacts(ext_dir)
self._run_stage_with_artifact_checks(
"CMake build",
lambda: self._build_extension_impl(ext_fullpath, ext_dir, build_dir),
[(ext_fullpath, f"native extension '{ext.name}'")],
)
self._engine_extensions_built = True
def _clean_stale_engine_artifacts(self, ext_dir: Path):
"""Remove stale non-abi3 engine binaries from wheel build output directories."""
source_engine_dir = (SETUP_DIR / "openviking" / "storage" / "vectordb" / "engine").resolve()
if ext_dir == source_engine_dir:
return
for pattern in ("*.so", "*.pyd"):
for artifact in ext_dir.glob(pattern):
artifact.unlink()
def _build_extension_impl(self, ext_fullpath, ext_dir, build_dir):
"""Invoke CMake to build the Python native extension."""
ext_basename = ext_fullpath.stem.split(".")[0]
built_filename = Path(self.get_ext_filename(self.extensions[0].name)).name
py_ext_suffix = built_filename.removeprefix(ext_basename)
if not py_ext_suffix:
py_ext_suffix = sysconfig.get_config_var("EXT_SUFFIX") or ext_fullpath.suffix
cmake_args = [
f"-S{Path(ENGINE_SOURCE_DIR).resolve()}",
f"-B{build_dir}",
"-DCMAKE_BUILD_TYPE=Release",
f"-DOV_PY_OUTPUT_DIR={ext_dir}",
f"-DOV_PY_EXT_SUFFIX={py_ext_suffix}",
f"-DOV_X86_BUILD_VARIANTS={';'.join(ENGINE_BUILD_CONFIG.cmake_variants)}",
"-DCMAKE_VERBOSE_MAKEFILE=ON",
"-DCMAKE_INSTALL_RPATH=$ORIGIN",
f"-DPython3_EXECUTABLE={sys.executable}",
f"-DPython3_INCLUDE_DIRS={sysconfig.get_path('include')}",
f"-DPython3_LIBRARIES={sysconfig.get_config_vars().get('LIBRARY')}",
f"-DCMAKE_C_COMPILER={C_COMPILER_PATH}",
f"-DCMAKE_CXX_COMPILER={CXX_COMPILER_PATH}",
]
if sys.platform == "darwin":
cmake_args.append("-DCMAKE_OSX_DEPLOYMENT_TARGET=10.15")
target_arch = os.environ.get("CMAKE_OSX_ARCHITECTURES")
if target_arch:
cmake_args.append(f"-DCMAKE_OSX_ARCHITECTURES={target_arch}")
elif sys.platform == "win32":
windows_python_sabi_library = _get_windows_python_sabi_library()
cmake_args.append(f"-DOV_PYTHON_SABI_LIBRARY={windows_python_sabi_library}")
cmake_args.extend(["-G", "MinGW Makefiles"])
self.spawn([self.cmake_executable] + cmake_args)
build_args = ["--build", str(build_dir), "--config", "Release", f"-j{os.cpu_count() or 4}"]
self.spawn([self.cmake_executable] + build_args)
def _build_web_studio():
"""Build the web-studio SPA and copy dist into the Python package tree.
Skipped when OV_SKIP_STUDIO_BUILD=1 or when the bundle already exists.
Falls back gracefully (warning, not error) when npm is unavailable.
"""
if os.environ.get("OV_SKIP_STUDIO_BUILD") == "1":
print(" [SKIP] web-studio build disabled by OV_SKIP_STUDIO_BUILD=1")
return
dest = SETUP_DIR / "openviking" / "web_studio" / "dist"
if (dest / "index.html").is_file():
print(" [OK] web-studio bundle already present")
return
source = SETUP_DIR / "web-studio"
if not (source / "package.json").is_file():
print(" [SKIP] web-studio source not found; /studio will be unavailable")
return
npm = shutil.which("npm")
if not npm:
print(" [SKIP] npm not found; install Node.js to enable /studio")
return
print("Building web-studio (Vite SPA)...")
try:
subprocess.check_call([npm, "ci"], cwd=str(source))
subprocess.check_call(
[npm, "run", "build", "--", "--base=/studio/"],
cwd=str(source),
)
except subprocess.CalledProcessError as exc:
print(f" [WARNING] web-studio npm build failed ({exc}); /studio will be unavailable")
return
built = source / "dist"
if not (built / "index.html").is_file():
print(" [WARNING] web-studio build produced no index.html; /studio will be unavailable")
return
dest.parent.mkdir(parents=True, exist_ok=True)
if dest.exists():
shutil.rmtree(dest)
shutil.copytree(built, dest)
print(f" [OK] web-studio bundle copied to {dest}")
class OpenVikingBuildPy(build_py):
def run(self):
_build_web_studio()
write_system_assets_manifest(SETUP_DIR / "openviking" / "system_assets")
super().run()
package_root = Path(self.build_lib) / "vikingbot"
for asset_name in ("workspace", "bridge"):
source = SETUP_DIR / "bot" / asset_name
if not source.is_dir():
raise FileNotFoundError(f"required VikingBot asset directory not found: {source}")
destination = package_root / asset_name
if destination.exists():
shutil.rmtree(destination)
shutil.copytree(
source,
destination,
ignore=shutil.ignore_patterns("node_modules", "dist", "__pycache__", "*.pyc"),
)
class OpenVikingSdist(sdist):
def make_release_tree(self, base_dir, files):
super().make_release_tree(base_dir, files)
write_system_assets_manifest(Path(base_dir) / "openviking" / "system_assets")
if bdist_wheel is not None:
class OpenVikingBdistWheel(bdist_wheel):
def finalize_options(self):
super().finalize_options()
self.py_limited_api = "cp310"
else:
OpenVikingBdistWheel = None
class OpenVikingDistribution(Distribution):
"""Keep wheels platform-specific because ragfs-python is a native artifact."""
def has_ext_modules(self):
return True
cmdclass = {
"build_ext": OpenVikingBuildExt,
"build_py": OpenVikingBuildPy,
"sdist": OpenVikingSdist,
}
if OpenVikingBdistWheel is not None:
cmdclass["bdist_wheel"] = OpenVikingBdistWheel
setup(
distclass=OpenVikingDistribution,
long_description=pypi_long_description(SETUP_DIR / "README.md"),
long_description_content_type="text/markdown",
ext_modules=(
[]
if SKIP_CPP_BUILD
else [
Extension(
name=ENGINE_BUILD_CONFIG.primary_extension,
sources=[],
py_limited_api=True,
)
]
),
cmdclass=cmdclass,
package_data={
"openviking_gateway": ["client-rules.json"],
"openviking.scheduling.expiry": ["lua/*.lua"],
"openviking": [
"lib/ragfs_python*.so",
"lib/ragfs_python*.pyd",
"bin/ov",
"bin/ov.exe",
"server/account_stores/sql/*.sql",
"server/account_stores/sql/changes/*.sql",
"service/sql/*.sql",
"service/sql/changes/*.sql",
"server/static/**/*",
"web_studio/dist/**/*",
"storage/vectordb/engine/*.abi3.so",
"storage/vectordb/engine/*.pyd",
],
"vikingbot": [
"workspace/**/*",
"bridge/**/*",
],
},
include_package_data=True,
)