Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

23.9 KB
Raw Blame
import importlib
import json
import os
import platform
import shutil
import subprocess
import sys
import sysconfig
from pathlib import Path

from setuptools import Distribution, Extension, setup
from setuptools.command.build_ext import build_ext
from setuptools.command.build_py import build_py
from setuptools.command.sdist import sdist

try:
    from wheel.bdist_wheel import bdist_wheel
except ImportError:  # pragma: no cover - local build_ext may not have wheel installed
    bdist_wheel = None

SETUP_DIR = Path(__file__).resolve().parent
if str(SETUP_DIR) not in sys.path:
    sys.path.insert(0, str(SETUP_DIR))

get_host_engine_build_config = importlib.import_module(
    "scripts.build_support.x86_profiles"
).get_host_engine_build_config
resolve_openviking_version = importlib.import_module(
    "scripts.build_support.versioning"
).resolve_openviking_version
pypi_long_description = importlib.import_module(
    "scripts.build_support.readme"
).pypi_long_description
write_system_assets_manifest = importlib.import_module(
    "scripts.build_support.system_assets_manifest"
).write_system_assets_manifest

CMAKE_PATH = shutil.which("cmake") or "cmake"
C_COMPILER_PATH = os.environ.get("CC") or shutil.which("gcc") or "gcc"
CXX_COMPILER_PATH = os.environ.get("CXX") or shutil.which("g++") or "g++"
ENGINE_SOURCE_DIR = "src/"
ENGINE_BUILD_CONFIG = get_host_engine_build_config(platform.machine())
SKIP_CPP_BUILD = os.environ.get("OV_SKIP_CPP_BUILD") == "1"


def _sanitize_native_build_env(env):
    """Keep Rust native builds from accidentally linking against Linuxbrew libs.

    On older glibc systems, Homebrew-provided native libraries can require a newer
    libc than the host linker/runtime supports. When pkg-config resolves xz/bzip2
    from Linuxbrew, Cargo inherits those library search paths and link fails.
    """

    sanitized_env = env.copy()

    pkg_config = sanitized_env.get("PKG_CONFIG") or shutil.which("pkg-config")
    if pkg_config and "linuxbrew" in os.path.realpath(pkg_config).lower():
        system_pkg_config = "/usr/bin/pkg-config"
        if Path(system_pkg_config).exists():
            sanitized_env["PKG_CONFIG"] = system_pkg_config

    for key in ("PKG_CONFIG_PATH", "LIBRARY_PATH", "LD_LIBRARY_PATH"):
        value = sanitized_env.get(key)
        if not value:
            continue
        kept_paths = [
            path
            for path in value.split(os.pathsep)
            if path and "linuxbrew" not in os.path.realpath(path).lower()
        ]
        if kept_paths:
            sanitized_env[key] = os.pathsep.join(kept_paths)
        else:
            sanitized_env.pop(key, None)

    return sanitized_env


def _get_windows_python_sabi_library() -> Path:
    """Return the stable-ABI Python library path for Windows abi3 extensions."""
    candidate_roots = []
    for raw_root in (
        sys.base_prefix,
        sys.base_exec_prefix,
        sysconfig.get_config_var("installed_base"),
        sysconfig.get_config_var("base"),
    ):
        if not raw_root:
            continue
        candidate_root = Path(raw_root).resolve()
        if candidate_root not in candidate_roots:
            candidate_roots.append(candidate_root)

    candidate_paths = []
    for root in candidate_roots:
        candidate_paths.extend(
            [
                root / "libs" / "python3.lib",
                root / "python3.dll",
            ]
        )

    for candidate_path in candidate_paths:
        if candidate_path.exists():
            return candidate_path

    searched = ", ".join(str(path) for path in candidate_paths) or "<none>"
    raise RuntimeError(
        "Could not locate the Windows stable-ABI Python library for abi3 engine modules. "
        f"Searched: {searched}"
    )


class OpenVikingBuildExt(build_ext):
    """Build OpenViking runtime artifacts and Python native extensions."""

    def run(self):
        self.build_ov_cli_artifact()
        self.build_ragfs_python_artifact()
        self.cmake_executable = CMAKE_PATH

        if SKIP_CPP_BUILD:
            print("[SKIP] C++ vector engine build disabled by OV_SKIP_CPP_BUILD=1")
        else:
            for ext in self.extensions:
                self.build_extension(ext)

    def _copy_artifact(self, src, dst):
        """Copy a build artifact into the package tree and preserve executability."""
        print(f"Copying artifact from {src} to {dst}")
        dst.parent.mkdir(parents=True, exist_ok=True)
        shutil.copy2(str(src), str(dst))
        if sys.platform != "win32":
            os.chmod(str(dst), 0o755)

    def _copy_artifacts_to_build_lib(self, target_binary=None, target_lib=None):
        """Copy built artifacts into build_lib so wheel packaging can include them."""
        if self.build_lib:
            build_pkg_dir = Path(self.build_lib) / "openviking"
            if target_binary and target_binary.exists():
                self._copy_artifact(target_binary, build_pkg_dir / "bin" / target_binary.name)
            if target_lib and target_lib.exists():
                self._copy_artifact(target_lib, build_pkg_dir / "lib" / target_lib.name)

    def _require_artifact(self, artifact_path, artifact_name, stage_name):
        """Abort the build immediately when a required artifact is missing."""
        if artifact_path.exists():
            return
        raise RuntimeError(
            f"{stage_name} did not produce required {artifact_name} at {artifact_path}"
        )

    def _run_stage_with_artifact_checks(
        self, stage_name, build_fn, required_artifacts, on_success=None
    ):
        """Run a build stage and always validate its required outputs on normal return."""
        build_fn()
        for artifact_path, artifact_name in required_artifacts:
            self._require_artifact(artifact_path, artifact_name, stage_name)
        if on_success:
            on_success()

    def _resolve_cargo_target_dir(self, cargo_project_dir, env):
        """Resolve the Cargo target directory for workspace and overridden builds."""
        configured_target_dir = env.get("CARGO_TARGET_DIR")
        if configured_target_dir:
            return Path(configured_target_dir).resolve()

        try:
            result = subprocess.run(
                ["cargo", "metadata", "--format-version", "1", "--no-deps"],
                cwd=str(cargo_project_dir),
                env=env,
                check=True,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
            )
            metadata = json.loads(result.stdout.decode("utf-8"))
            target_directory = metadata.get("target_directory")
            if target_directory:
                return Path(target_directory).resolve()
        except Exception as exc:
            print(f"[Warning] Failed to resolve Cargo target directory via metadata: {exc}")

        return cargo_project_dir.parents[1] / "target"

    def build_ov_cli_artifact(self):
        """Build or reuse the ov Rust CLI binary."""
        if os.environ.get("OV_SKIP_OV_BUILD") == "1":
            print("[SKIP] ov CLI build disabled by OV_SKIP_OV_BUILD=1")
            return

        binary_name = "ov.exe" if sys.platform == "win32" else "ov"
        ov_cli_dir = Path("crates/ov_cli").resolve()
        ov_target_binary = Path("openviking/bin").resolve() / binary_name

        self._run_stage_with_artifact_checks(
            "ov CLI build",
            lambda: self._build_ov_cli_artifact_impl(ov_cli_dir, binary_name, ov_target_binary),
            [(ov_target_binary, binary_name)],
            on_success=lambda: self._copy_artifacts_to_build_lib(ov_target_binary, None),
        )

    def _build_ov_cli_artifact_impl(self, ov_cli_dir, binary_name, ov_target_binary):
        """Implement ov CLI building without final artifact checks."""

        prebuilt_dir = os.environ.get("OV_PREBUILT_BIN_DIR")
        if prebuilt_dir:
            src_bin = Path(prebuilt_dir).resolve() / binary_name
            if src_bin.exists():
                self._copy_artifact(src_bin, ov_target_binary)
                return

        if ov_cli_dir.exists() and shutil.which("cargo"):
            print("Building ov CLI from source...")
            try:
                env = _sanitize_native_build_env(os.environ.copy())
                env["OPENVIKING_VERSION"] = resolve_openviking_version(
                    env=env, project_root=SETUP_DIR
                )
                build_args = ["cargo", "build", "--release"]
                target = env.get("CARGO_BUILD_TARGET")
                if target:
                    print(f"Cross-compiling with CARGO_BUILD_TARGET={target}")
                    build_args.extend(["--target", target])

                result = subprocess.run(
                    build_args,
                    cwd=str(ov_cli_dir),
                    env=env,
                    check=True,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE,
                )
                if result.stdout:
                    print(f"Build stdout: {result.stdout.decode('utf-8', errors='replace')}")
                if result.stderr:
                    print(f"Build stderr: {result.stderr.decode('utf-8', errors='replace')}")

                cargo_target_dir = self._resolve_cargo_target_dir(ov_cli_dir, env)
                if target:
                    built_bin = cargo_target_dir / target / "release" / binary_name
                else:
                    built_bin = cargo_target_dir / "release" / binary_name

                self._require_artifact(built_bin, binary_name, "ov CLI build")
                self._copy_artifact(built_bin, ov_target_binary)
                print("[OK] ov CLI built successfully from source")
            except Exception as exc:
                error_msg = f"Failed to build ov CLI from source: {exc}"
                if isinstance(exc, subprocess.CalledProcessError):
                    if exc.stdout:
                        error_msg += (
                            f"\nBuild stdout:\n{exc.stdout.decode('utf-8', errors='replace')}"
                        )
                    if exc.stderr:
                        error_msg += (
                            f"\nBuild stderr:\n{exc.stderr.decode('utf-8', errors='replace')}"
                        )
                print(f"[Error] {error_msg}")
                raise RuntimeError(error_msg)
        else:
            if ov_target_binary.exists():
                print("[Info] ov CLI binary already exists locally. Skipping source build.")
            elif not ov_cli_dir.exists():
                print(f"[Warning] ov CLI source directory not found at {ov_cli_dir}")
            else:
                print("[Warning] Cargo not found. Cannot build ov CLI from source.")

    def build_ragfs_python_artifact(self):
        """Build ragfs-python (Rust RAGFS binding) via maturin and copy the native
        extension into ``openviking/lib/`` so it ships inside the openviking wheel.
        """
        require_ragfs_artifact = self._should_require_ragfs_artifact()
        ragfs_python_dir = Path("crates/ragfs-python").resolve()
        ragfs_lib_dir = Path("openviking/lib").resolve()

        if not ragfs_python_dir.exists():
            message = "ragfs-python source directory not found."
            if require_ragfs_artifact:
                raise RuntimeError(message)
            print(f"[Info] {message} Skipping.")
            return

        if os.environ.get("OV_SKIP_RAGFS_BUILD") == "1":
            message = "Skipping ragfs-python build (OV_SKIP_RAGFS_BUILD=1)"
            if require_ragfs_artifact:
                raise RuntimeError(f"{message} is incompatible with required wheel artifacts.")
            print(f"[OK] {message}")
            return

        if importlib.util.find_spec("maturin") is None:
            message = (
                "maturin not found. ragfs-python (Rust binding) will not be built.\n"
                "       Install maturin to enable: pip install maturin"
            )
            if require_ragfs_artifact:
                raise RuntimeError(message)
            print(f"[SKIP] {message}")
            return

        import tempfile
        import zipfile

        with tempfile.TemporaryDirectory() as tmpdir:
            try:
                print("Building ragfs-python (Rust RAGFS binding) via maturin...")
                env = _sanitize_native_build_env(os.environ.copy())
                build_args = [
                    sys.executable,
                    "-m",
                    "maturin",
                    "build",
                    "--release",
                    "--out",
                    tmpdir,
                ]
                # Respect CARGO_BUILD_TARGET for cross-compilation
                target = env.get("CARGO_BUILD_TARGET")
                if target:
                    build_args.extend(["--target", target])

                result = subprocess.run(
                    build_args,
                    cwd=str(ragfs_python_dir),
                    env=env,
                    check=True,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE,
                )
                if result.stdout:
                    print(result.stdout.decode("utf-8", errors="replace"))
                if result.stderr:
                    print(result.stderr.decode("utf-8", errors="replace"))

                # Extract the stable-ABI native extension from the built wheel.
                whl_files = list(Path(tmpdir).glob("ragfs_python-*.whl"))
                if not whl_files:
                    message = "maturin produced no wheel for ragfs-python."
                    if require_ragfs_artifact:
                        raise RuntimeError(message)
                    print(f"[Warning] {message}")
                    return

                ragfs_lib_dir.mkdir(parents=True, exist_ok=True)
                for stale_artifact in ragfs_lib_dir.glob("ragfs_python*.so"):
                    stale_artifact.unlink()
                for stale_artifact in ragfs_lib_dir.glob("ragfs_python*.pyd"):
                    stale_artifact.unlink()
                for stale_artifact in ragfs_lib_dir.glob("ragfs_python*.dylib"):
                    stale_artifact.unlink()

                extracted = False
                with zipfile.ZipFile(str(whl_files[0])) as zf:
                    for name in zf.namelist():
                        basename = Path(name).name
                        is_ragfs_extension = basename == "ragfs_python.pyd" or (
                            basename.startswith("ragfs_python.abi3.")
                            and basename.endswith((".so", ".pyd"))
                        )
                        if is_ragfs_extension:
                            target_path = ragfs_lib_dir / basename
                            with zf.open(name) as src, open(target_path, "wb") as dst:
                                dst.write(src.read())
                            if sys.platform != "win32":
                                os.chmod(str(target_path), 0o755)
                            print(f"[OK] ragfs-python: extracted {basename} -> {target_path}")
                            extracted = True
                            break

                if not extracted:
                    message = (
                        "Could not find ragfs_python stable-ABI native extension in built wheel."
                    )
                    if require_ragfs_artifact:
                        raise RuntimeError(message)
                    print(f"[Warning] {message}")
                else:
                    self._copy_artifacts_to_build_lib(target_lib=target_path)

            except Exception as exc:
                error_detail = ""
                if isinstance(exc, subprocess.CalledProcessError):
                    if exc.stdout:
                        error_detail += exc.stdout.decode("utf-8", errors="replace")
                    if exc.stderr:
                        error_detail += exc.stderr.decode("utf-8", errors="replace")
                if require_ragfs_artifact:
                    error_message = f"Failed to build ragfs-python: {exc}"
                    if error_detail:
                        error_message += f"\n{error_detail}"
                    raise RuntimeError(error_message) from exc
                print(f"[Warning] Failed to build ragfs-python: {exc}")
                if error_detail:
                    print(error_detail)

    def _should_require_ragfs_artifact(self) -> bool:
        """Fail wheel builds closed when ragfs-python cannot be bundled."""
        required = os.environ.get("OV_REQUIRE_RAGFS_BUILD")
        if required is not None:
            return required == "1"
        return "bdist_wheel" in sys.argv

    def build_extension(self, ext):
        """Build a single Python native extension artifact using CMake."""
        if getattr(self, "_engine_extensions_built", False):
            return

        ext_fullpath = Path(self.get_ext_fullpath(ext.name))
        ext_dir = ext_fullpath.parent.resolve()
        build_dir = Path(self.build_temp) / "cmake_build"
        build_dir.mkdir(parents=True, exist_ok=True)
        self._clean_stale_engine_artifacts(ext_dir)

        self._run_stage_with_artifact_checks(
            "CMake build",
            lambda: self._build_extension_impl(ext_fullpath, ext_dir, build_dir),
            [(ext_fullpath, f"native extension '{ext.name}'")],
        )
        self._engine_extensions_built = True

    def _clean_stale_engine_artifacts(self, ext_dir: Path):
        """Remove stale non-abi3 engine binaries from wheel build output directories."""
        source_engine_dir = (SETUP_DIR / "openviking" / "storage" / "vectordb" / "engine").resolve()
        if ext_dir == source_engine_dir:
            return

        for pattern in ("*.so", "*.pyd"):
            for artifact in ext_dir.glob(pattern):
                artifact.unlink()

    def _build_extension_impl(self, ext_fullpath, ext_dir, build_dir):
        """Invoke CMake to build the Python native extension."""
        ext_basename = ext_fullpath.stem.split(".")[0]
        built_filename = Path(self.get_ext_filename(self.extensions[0].name)).name
        py_ext_suffix = built_filename.removeprefix(ext_basename)
        if not py_ext_suffix:
            py_ext_suffix = sysconfig.get_config_var("EXT_SUFFIX") or ext_fullpath.suffix

        cmake_args = [
            f"-S{Path(ENGINE_SOURCE_DIR).resolve()}",
            f"-B{build_dir}",
            "-DCMAKE_BUILD_TYPE=Release",
            f"-DOV_PY_OUTPUT_DIR={ext_dir}",
            f"-DOV_PY_EXT_SUFFIX={py_ext_suffix}",
            f"-DOV_X86_BUILD_VARIANTS={';'.join(ENGINE_BUILD_CONFIG.cmake_variants)}",
            "-DCMAKE_VERBOSE_MAKEFILE=ON",
            "-DCMAKE_INSTALL_RPATH=$ORIGIN",
            f"-DPython3_EXECUTABLE={sys.executable}",
            f"-DPython3_INCLUDE_DIRS={sysconfig.get_path('include')}",
            f"-DPython3_LIBRARIES={sysconfig.get_config_vars().get('LIBRARY')}",
            f"-DCMAKE_C_COMPILER={C_COMPILER_PATH}",
            f"-DCMAKE_CXX_COMPILER={CXX_COMPILER_PATH}",
        ]

        if sys.platform == "darwin":
            cmake_args.append("-DCMAKE_OSX_DEPLOYMENT_TARGET=10.15")
            target_arch = os.environ.get("CMAKE_OSX_ARCHITECTURES")
            if target_arch:
                cmake_args.append(f"-DCMAKE_OSX_ARCHITECTURES={target_arch}")
        elif sys.platform == "win32":
            windows_python_sabi_library = _get_windows_python_sabi_library()
            cmake_args.append(f"-DOV_PYTHON_SABI_LIBRARY={windows_python_sabi_library}")
            cmake_args.extend(["-G", "MinGW Makefiles"])

        self.spawn([self.cmake_executable] + cmake_args)

        build_args = ["--build", str(build_dir), "--config", "Release", f"-j{os.cpu_count() or 4}"]
        self.spawn([self.cmake_executable] + build_args)


def _build_web_studio():
    """Build the web-studio SPA and copy dist into the Python package tree.

    Skipped when OV_SKIP_STUDIO_BUILD=1 or when the bundle already exists.
    Falls back gracefully (warning, not error) when npm is unavailable.
    """
    if os.environ.get("OV_SKIP_STUDIO_BUILD") == "1":
        print("  [SKIP] web-studio build disabled by OV_SKIP_STUDIO_BUILD=1")
        return

    dest = SETUP_DIR / "openviking" / "web_studio" / "dist"
    if (dest / "index.html").is_file():
        print("  [OK] web-studio bundle already present")
        return

    source = SETUP_DIR / "web-studio"
    if not (source / "package.json").is_file():
        print("  [SKIP] web-studio source not found; /studio will be unavailable")
        return

    npm = shutil.which("npm")
    if not npm:
        print("  [SKIP] npm not found; install Node.js to enable /studio")
        return

    print("Building web-studio (Vite SPA)...")
    try:
        subprocess.check_call([npm, "ci"], cwd=str(source))
        subprocess.check_call(
            [npm, "run", "build", "--", "--base=/studio/"],
            cwd=str(source),
        )
    except subprocess.CalledProcessError as exc:
        print(f"  [WARNING] web-studio npm build failed ({exc}); /studio will be unavailable")
        return

    built = source / "dist"
    if not (built / "index.html").is_file():
        print("  [WARNING] web-studio build produced no index.html; /studio will be unavailable")
        return

    dest.parent.mkdir(parents=True, exist_ok=True)
    if dest.exists():
        shutil.rmtree(dest)
    shutil.copytree(built, dest)
    print(f"  [OK] web-studio bundle copied to {dest}")


class OpenVikingBuildPy(build_py):
    def run(self):
        _build_web_studio()
        write_system_assets_manifest(SETUP_DIR / "openviking" / "system_assets")
        super().run()
        package_root = Path(self.build_lib) / "vikingbot"
        for asset_name in ("workspace", "bridge"):
            source = SETUP_DIR / "bot" / asset_name
            if not source.is_dir():
                raise FileNotFoundError(f"required VikingBot asset directory not found: {source}")
            destination = package_root / asset_name
            if destination.exists():
                shutil.rmtree(destination)
            shutil.copytree(
                source,
                destination,
                ignore=shutil.ignore_patterns("node_modules", "dist", "__pycache__", "*.pyc"),
            )


class OpenVikingSdist(sdist):
    def make_release_tree(self, base_dir, files):
        super().make_release_tree(base_dir, files)
        write_system_assets_manifest(Path(base_dir) / "openviking" / "system_assets")


if bdist_wheel is not None:

    class OpenVikingBdistWheel(bdist_wheel):
        def finalize_options(self):
            super().finalize_options()
            self.py_limited_api = "cp310"
else:
    OpenVikingBdistWheel = None


class OpenVikingDistribution(Distribution):
    """Keep wheels platform-specific because ragfs-python is a native artifact."""

    def has_ext_modules(self):
        return True


cmdclass = {
    "build_ext": OpenVikingBuildExt,
    "build_py": OpenVikingBuildPy,
    "sdist": OpenVikingSdist,
}
if OpenVikingBdistWheel is not None:
    cmdclass["bdist_wheel"] = OpenVikingBdistWheel


setup(
    distclass=OpenVikingDistribution,
    long_description=pypi_long_description(SETUP_DIR / "README.md"),
    long_description_content_type="text/markdown",
    ext_modules=(
        []
        if SKIP_CPP_BUILD
        else [
            Extension(
                name=ENGINE_BUILD_CONFIG.primary_extension,
                sources=[],
                py_limited_api=True,
            )
        ]
    ),
    cmdclass=cmdclass,
    package_data={
        "openviking_gateway": ["client-rules.json"],
        "openviking.scheduling.expiry": ["lua/*.lua"],
        "openviking": [
            "lib/ragfs_python*.so",
            "lib/ragfs_python*.pyd",
            "bin/ov",
            "bin/ov.exe",
            "server/account_stores/sql/*.sql",
            "server/account_stores/sql/changes/*.sql",
            "service/sql/*.sql",
            "service/sql/changes/*.sql",
            "server/static/**/*",
            "web_studio/dist/**/*",
            "storage/vectordb/engine/*.abi3.so",
            "storage/vectordb/engine/*.pyd",
        ],
        "vikingbot": [
            "workspace/**/*",
            "bridge/**/*",
        ],
    },
    include_package_data=True,
)