Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

8.3 KB
Raw Blame
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""Local GGUF embedders powered by llama-cpp-python."""

from __future__ import annotations

import importlib
import os
from dataclasses import dataclass
from pathlib import Path
from threading import Lock
from typing import Any, Dict, List, Optional

import requests

from openviking.models.embedder.base import DenseEmbedderBase, EmbedResult
from openviking.storage.errors import EmbeddingConfigurationError
from openviking_cli.utils import get_logger

logger = get_logger(__name__)

DEFAULT_LOCAL_MODEL_CACHE_DIR = "~/.cache/openviking/models"
DEFAULT_LOCAL_DENSE_MODEL = "bge-small-zh-v1.5-f16"
DEFAULT_BGE_ZH_QUERY_INSTRUCTION = "为这个句子生成表示以用于检索相关文章:"


@dataclass(frozen=True)
class LocalModelSpec:
    model_name: str
    dimension: int
    filename: str
    download_url: str
    query_instruction: Optional[str] = None


LOCAL_DENSE_MODEL_SPECS: Dict[str, LocalModelSpec] = {
    DEFAULT_LOCAL_DENSE_MODEL: LocalModelSpec(
        model_name=DEFAULT_LOCAL_DENSE_MODEL,
        dimension=512,
        filename="bge-small-zh-v1.5-f16.gguf",
        download_url=(
            "https://huggingface.co/CompendiumLabs/bge-small-zh-v1.5-gguf/resolve/main/"
            "bge-small-zh-v1.5-f16.gguf?download=true"
        ),
        query_instruction=DEFAULT_BGE_ZH_QUERY_INSTRUCTION,
    )
}


def get_local_model_spec(model_name: str) -> LocalModelSpec:
    try:
        return LOCAL_DENSE_MODEL_SPECS[model_name]
    except KeyError as exc:
        raise ValueError(
            f"Unknown local embedding model '{model_name}'. "
            f"Supported models: {list(LOCAL_DENSE_MODEL_SPECS.keys())}"
        ) from exc


def get_local_model_default_dimension(model_name: str) -> int:
    return get_local_model_spec(model_name).dimension


def get_local_model_cache_path(model_name: str, cache_dir: Optional[str] = None) -> Path:
    spec = get_local_model_spec(model_name)
    cache_root = Path(cache_dir or DEFAULT_LOCAL_MODEL_CACHE_DIR).expanduser().resolve()
    return cache_root / spec.filename


def get_local_model_identity(model_name: str, model_path: Optional[str] = None) -> str:
    if model_path:
        resolved = Path(model_path).expanduser().resolve()
        return str(resolved)
    return get_local_model_spec(model_name).filename


class LocalDenseEmbedder(DenseEmbedderBase):
    """Dense embedder backed by a local GGUF model via llama-cpp-python."""

    def __init__(
        self,
        model_name: str = DEFAULT_LOCAL_DENSE_MODEL,
        model_path: Optional[str] = None,
        cache_dir: Optional[str] = None,
        dimension: Optional[int] = None,
        query_instruction: Optional[str] = None,
        config: Optional[Dict[str, Any]] = None,
    ):
        runtime_config = dict(config or {})
        runtime_config.setdefault("provider", "local")
        super().__init__(model_name, runtime_config)

        self.model_spec = get_local_model_spec(model_name)
        self.model_path = model_path
        self.cache_dir = cache_dir or DEFAULT_LOCAL_MODEL_CACHE_DIR
        self.query_instruction = (
            query_instruction
            if query_instruction is not None
            else self.model_spec.query_instruction
        )
        self._dimension = dimension or self.model_spec.dimension
        if self._dimension != self.model_spec.dimension:
            raise ValueError(
                f"Local model '{model_name}' has fixed dimension {self.model_spec.dimension}, "
                f"but got dimension={self._dimension}"
            )

        self._resolved_model_path = self._resolve_model_path()
        self._llama = self._load_model()
        self._llama_lock = Lock()

    def _import_llama(self):
        try:
            module = importlib.import_module("llama_cpp")
        except ImportError as exc:
            raise EmbeddingConfigurationError(
                "Local embedding is enabled but 'llama-cpp-python' is not installed. "
                'Install it with: pip install "openviking[local-embed]". '
                "If you prefer a remote provider, set embedding.dense.provider explicitly in ov.conf."
            ) from exc

        llama_cls = getattr(module, "Llama", None)
        if llama_cls is None:
            raise EmbeddingConfigurationError(
                "llama_cpp.Llama is unavailable in the installed llama-cpp-python package."
            )
        return llama_cls

    def _resolve_model_path(self) -> Path:
        if self.model_path:
            resolved = Path(self.model_path).expanduser().resolve()
            if not resolved.exists():
                raise EmbeddingConfigurationError(
                    f"Local embedding model file not found: {resolved}"
                )
            return resolved

        cache_root = Path(self.cache_dir).expanduser().resolve()
        cache_root.mkdir(parents=True, exist_ok=True)
        target = get_local_model_cache_path(self.model_name, self.cache_dir)
        if target.exists():
            return target

        self._download_model(self.model_spec.download_url, target)
        return target

    def _download_model(self, url: str, target: Path) -> None:
        logger.info("Downloading local embedding model %s to %s", self.model_name, target)
        tmp_target = target.with_suffix(target.suffix + ".part")
        try:
            with requests.get(url, stream=True, timeout=(10, 300)) as response:
                response.raise_for_status()
                with tmp_target.open("wb") as fh:
                    for chunk in response.iter_content(chunk_size=1024 * 1024):
                        if chunk:
                            fh.write(chunk)
            os.replace(tmp_target, target)
        except Exception as exc:
            tmp_target.unlink(missing_ok=True)
            raise EmbeddingConfigurationError(
                f"Failed to download local embedding model '{self.model_name}' from {url} "
                f"to {target}: {exc}"
            ) from exc

    def _load_model(self):
        llama_cls = self._import_llama()
        try:
            return llama_cls(
                model_path=str(self._resolved_model_path),
                embedding=True,
                verbose=False,
            )
        except Exception as exc:
            raise EmbeddingConfigurationError(
                f"Failed to load GGUF embedding model from {self._resolved_model_path}: {exc}"
            ) from exc

    def _format_text(self, text: str, *, is_query: bool) -> str:
        if is_query and self.query_instruction:
            return f"{self.query_instruction}{text}"
        return text

    @staticmethod
    def _extract_embedding(payload: Any) -> List[float]:
        if isinstance(payload, dict):
            data = payload.get("data")
            if isinstance(data, list) and data:
                item = data[0]
                if isinstance(item, dict) and "embedding" in item:
                    return list(item["embedding"])
            if "embedding" in payload:
                return list(payload["embedding"])
        raise RuntimeError("Unexpected llama-cpp-python embedding response format")

    def _embed_formatted_text(self, formatted: str) -> EmbedResult:
        # llama.cpp mutates the shared context, including during native calls that release the GIL.
        with self._llama_lock:
            payload = self._llama.create_embedding(formatted)
            return EmbedResult(dense_vector=self._extract_embedding(payload))

    def embed(self, text: str, is_query: bool = False) -> EmbedResult:
        formatted = self._format_text(text, is_query=is_query)

        try:
            result = self._run_once(
                lambda: self._embed_formatted_text(formatted),
                logger=logger,
                operation_name="local embedding",
            )
        except Exception as exc:
            raise RuntimeError(f"Local embedding failed: {exc}") from exc

        estimated_tokens = self._estimate_tokens(formatted)
        self.update_token_usage(
            model_name=self.model_name,
            provider="local",
            prompt_tokens=estimated_tokens,
            completion_tokens=0,
        )
        return result

    def get_dimension(self) -> int:
        return self._dimension

    def close(self):
        with self._llama_lock:
            close_fn = getattr(self._llama, "close", None)
            if callable(close_fn):
                close_fn()