Skip to content

Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.

24.7 KB
Raw Blame
"""Client Factory - Create OpenAI LLM and Embedder clients from config.

Provides three levels of API:
    - create_client(): Unified creation for both LLM and Embedder
    - create_llm() / create_embedder(): Separate creation for advanced use
    - get_client(): Read from config.toml (backward compatible)

String shorthand format: provider:model@url
    - "bailian"              → provider only, use preset defaults
    - "bailian:qwen-plus"    → provider + model, use preset URL
    - "vllm:Qwen3.5-9B@http://localhost:8000/v1" → full specification
"""

import logging
import os
from pathlib import Path
from typing import Any

from langchain_core.embeddings import Embeddings
from langchain_core.language_models.chat_models import BaseChatModel

logger = logging.getLogger(__name__)

DEFAULT_CONFIG_DIR = Path.home() / ".he"
DEFAULT_CONFIG_FILE = DEFAULT_CONFIG_DIR / "config.toml"

# Official OpenAI API base URL — only this endpoint accepts pre-tokenized input
OPENAI_API_URL = "https://api.openai.com/v1"

# Provider presets: base_url and default models for each provider
PROVIDER_PRESETS: dict[str, dict[str, str | None]] = {
    "openai": {
        "base_url": "https://api.openai.com/v1",
        "default_llm": "gpt-4o-mini",
        "default_embedder": "text-embedding-3-small",
    },
    "bailian": {
        "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
        "default_llm": "qwen3.6-plus",
        "default_embedder": "text-embedding-v4",
    },
    "vllm": {
        "base_url": None,
        "default_llm": None,
        "default_embedder": None,
    },
    # DeepSeek. OpenAI-compatible endpoint. Current models are deepseek-v4-flash
    # and deepseek-v4-pro; both default to "thinking" mode, which rejects the
    # tool_choice that method="function_calling" relies on, so create_llm()
    # disables thinking for this provider by default (override via extra_body).
    # DeepSeek has no embeddings API — pair it with an OpenAI-compatible embedder.
    "deepseek": {
        "base_url": "https://api.deepseek.com",
        "default_llm": "deepseek-v4-flash",
        "default_embedder": None,
    },
    # OrcaRouter. OpenAI-compatible model routing gateway exposing 150+ models
    # from OpenAI, Anthropic, Google, DeepSeek, Qwen, MiniMax, xAI and others
    # behind a single endpoint and API key. Namespaced model ids such as
    # "orcarouter/auto" route to a matching upstream.
    "orcarouter": {
        "base_url": "https://api.orcarouter.ai/v1",
        "default_llm": "orcarouter/auto",
        "default_embedder": "openai/text-embedding-3-small",
    },
    # Cheaper Inference. OpenAI-compatible LLM gateway with one key for models
    # from several labs, using bare model ids such as "gpt-5.4-mini". It has no
    # embeddings API — pair it with an OpenAI-compatible embedder.
    "cheaperinference": {
        "base_url": "https://api.cheaperinference.com/v1",
        "default_llm": "gpt-5.4-mini",
        "default_embedder": None,
    },
    "api-route": {
        "base_url": "https://global.api-route.com/v1",
        "default_llm": "gpt-6.1-sol",
        "default_embedder": None,
    },
    # Atlas Cloud. OpenAI-compatible aggregator serving DeepSeek, Z.ai, Moonshot,
    # Qwen, MiniMax, OpenAI, Anthropic and Google models under vendor-prefixed
    # ids such as "deepseek-ai/deepseek-v4-flash". Forced tool_choice (which
    # method="function_calling" relies on) works for these models without
    # disabling thinking, except glm-5.x, which answers 400. Atlas Cloud has no
    # embeddings API — pair it with an OpenAI-compatible embedder.
    "atlascloud": {
        "base_url": "https://api.atlascloud.ai/v1",
        "default_llm": "deepseek-ai/deepseek-v4-flash",
        "default_embedder": None,
    },
    # Runcrate. OpenAI-compatible API for open models under vendor-prefixed ids
    # such as "deepseek-ai/DeepSeek-V4.1-Flash". It also serves embeddings, so
    # one key covers both the LLM and the embedder ("BAAI/bge-m3", 1024 dims).
    "runcrate": {
        "base_url": "https://api.runcrate.ai/v1",
        "default_llm": "deepseek-ai/DeepSeek-V4.1-Flash",
        "default_embedder": "BAAI/bge-m3",
    },
    # Anthropic (Claude). Uses the native ChatAnthropic client, so base_url is
    # left empty (the SDK targets api.anthropic.com by default). Anthropic has
    # no embeddings API, hence default_embedder is None — pair it with an
    # OpenAI-compatible embedder.
    "anthropic": {
        "base_url": "",
        "default_llm": "claude-opus-4-8",
        "default_embedder": None,
    },
    "claude": {
        "base_url": "",
        "default_llm": "claude-opus-4-8",
        "default_embedder": None,
    },
    # Google Gemini. Uses the native ChatGoogleGenerativeAI client
    # (langchain-google-genai), so base_url is empty. Gemini Developer API
    # has no mature embedder path in this repo — pair with an OpenAI-compatible
    # embedder. Default model is the current documented stable Flash id
    # (Gemini API models page, 2026-09-04): gemini-3.8-flash.
    "google": {
        "base_url": "",
        "default_llm": "gemini-3.8-flash",
        "default_embedder": None,
    },
    "gemini": {
        "base_url": "",
        "default_llm": "gemini-3.8-flash",
        "default_embedder": None,
    },
}

# Providers handled by the native langchain-anthropic client rather than the
# OpenAI-compatible path.
ANTHROPIC_PROVIDERS = ("anthropic", "claude")

# Providers handled by the native langchain-google-genai client.
GOOGLE_PROVIDERS = ("google", "gemini")

# Native (non-OpenAI-compatible) LLM providers — do not fall back to
# OPENAI_API_KEY when their own env vars are unset.
_NATIVE_LLM_PROVIDERS = ANTHROPIC_PROVIDERS + GOOGLE_PROVIDERS

# Environment variables checked (in order) for each provider's API key.
PROVIDER_API_KEY_ENV: dict[str, tuple[str, ...]] = {
    "anthropic": ("ANTHROPIC_API_KEY", "CLAUDE_API_KEY"),
    "claude": ("ANTHROPIC_API_KEY", "CLAUDE_API_KEY"),
    "google": ("GOOGLE_API_KEY", "GEMINI_API_KEY"),
    "gemini": ("GOOGLE_API_KEY", "GEMINI_API_KEY"),
    "deepseek": ("DEEPSEEK_API_KEY",),
    "orcarouter": ("ORCAROUTER_API_KEY",),
    "cheaperinference": ("CHEAPER_INFERENCE_API_KEY",),
    "api-route": ("API_ROUTE_API_KEY",),
    "atlascloud": ("ATLASCLOUD_API_KEY",),
    "runcrate": ("RUNCRATE_API_KEY",),
}


def _env_api_key(provider: str) -> str:
    """Return the first non-empty API key from the provider's env vars.

    Falls back to OPENAI_API_KEY for OpenAI-compatible providers.
    """
    for var in PROVIDER_API_KEY_ENV.get(provider, ()):
        value = os.environ.get(var, "")
        if value:
            return value
    if provider not in _NATIVE_LLM_PROVIDERS:
        return os.environ.get("OPENAI_API_KEY", "")
    return ""


class CompatibleEmbeddings(Embeddings):
    """Embeddings for OpenAI-compatible providers that only accept string input.

    langchain_openai's OpenAIEmbeddings with tiktoken_enabled=True sends
    pre-tokenized integer lists to the API, which OpenAI supports but most
    OpenAI-compatible providers (Ollama, LiteLLM, etc.) do not. This class
    works around that by always sending strings, using tiktoken for chunking
    with a fallback encoding when the model name isn't tiktoken-compatible.
    """

    def __init__(
        self,
        model: str = "text-embedding-ada-002",
        api_key: str | None = None,
        base_url: str | None = None,
        max_batch_size: int = 10,
        chunk_size: int | None = None,
        max_retries: int = 2,
        **kwargs: Any,
    ):
        from openai import OpenAI

        self._client = OpenAI(
            api_key=api_key or os.environ.get("OPENAI_API_KEY", ""),
            base_url=base_url,
            max_retries=max_retries,
        )
        self._model = model

        # max_batch_size caps how many inputs are sent per embeddings request.
        # Many OpenAI-compatible providers reject large batches (e.g. Bailian /
        # DashScope caps at 10), so the default is intentionally conservative.
        # `chunk_size` is the legacy name for this knob and is kept as an alias.
        self._max_batch_size = chunk_size if chunk_size is not None else max_batch_size

        # Determine the tiktoken encoding to use for chunking
        import tiktoken

        try:
            self._encoding = tiktoken.encoding_for_model(model)
        except KeyError:
            # Model not recognized by tiktoken; use cl100k_base (used by
            # text-embedding-ada-002, text-embedding-3-small, etc.)
            logger.debug(
                "Model '%s' not recognized by tiktoken, using cl100k_base encoding",
                model,
            )
            self._encoding = tiktoken.get_encoding("cl100k_base")

        # Max tokens per request (8191 is the limit for most OpenAI embedders)
        self._max_tokens = kwargs.get("embedding_ctx_length", 8191)

    def _split_texts(self, texts: list[str]) -> list[tuple[str, int]]:
        """Split texts into chunks that fit within token limits.

        Returns list of (text_chunk, original_index) tuples.
        """
        chunks: list[tuple[str, int]] = []
        for i, text in enumerate(texts):
            tokens = self._encoding.encode(text)
            if len(tokens) <= self._max_tokens:
                chunks.append((text, i))
            else:
                # Split into chunks
                for j in range(0, len(tokens), self._max_tokens):
                    chunk_tokens = tokens[j : j + self._max_tokens]
                    chunk_text = self._encoding.decode(chunk_tokens)
                    chunks.append((chunk_text, i))
        return chunks

    def embed_documents(self, texts: list[str]) -> list[list[float]]:
        if not texts:
            return []

        chunks = self._split_texts(texts)
        if not chunks:
            return []

        # Group chunks into batches no larger than max_batch_size
        all_embeddings: list[list[float] | None] = [None] * len(texts)
        batch: list[tuple[str, int]] = []

        # Accumulate a running sum and count per text so multi-chunk texts get a
        # true mean. A pairwise (prev + curr) / 2 only yields the mean for two
        # chunks; for three or more it biases toward later chunks.
        sums: dict[int, list[float]] = {}
        counts: dict[int, int] = {}

        def _embed_batch(b: list[tuple[str, int]]) -> None:
            # Skip blank chunks: many OpenAI-compatible providers (e.g. Bailian /
            # DashScope) reject empty-string input with a 400. Blank texts are
            # backfilled with a zero vector after all batches complete.
            non_blank = [(text, idx) for text, idx in b if text.strip()]
            if not non_blank:
                return
            response = self._client.embeddings.create(
                input=[text for text, _ in non_blank],
                model=self._model,
            )
            for (text, orig_idx), emb_data in zip(
                non_blank, response.data, strict=False
            ):
                emb = emb_data.embedding
                if orig_idx not in sums:
                    sums[orig_idx] = list(emb)
                    counts[orig_idx] = 1
                else:
                    running = sums[orig_idx]
                    sums[orig_idx] = [a + b for a, b in zip(running, emb, strict=False)]
                    counts[orig_idx] += 1

        for chunk in chunks:
            batch.append(chunk)
            if len(batch) >= self._max_batch_size:
                _embed_batch(batch)
                batch = []

        if batch:
            _embed_batch(batch)

        # Divide each running sum by its chunk count to get the mean embedding.
        dim: int | None = None
        for orig_idx, running in sums.items():
            count = counts[orig_idx]
            mean = [v / count for v in running]
            all_embeddings[orig_idx] = mean
            dim = len(mean)

        # Backfill blank texts (which were skipped above) with a zero vector so
        # the output stays aligned with the input. We never send an empty string
        # to the API. If every input was blank we have no reference dimension, so
        # probe once with a non-empty placeholder to learn it.
        missing_indices = [i for i, e in enumerate(all_embeddings) if e is None]
        if missing_indices:
            if dim is None:
                # Non-whitespace placeholder: a single space can be stripped to
                # empty by strict providers and rejected like an empty string.
                response = self._client.embeddings.create(
                    input=".",
                    model=self._model,
                )
                dim = len(response.data[0].embedding)
            zero_vector = [0.0] * dim
            for i in missing_indices:
                all_embeddings[i] = list(zero_vector)

        return all_embeddings  # type: ignore[return-value]

    def embed_query(self, text: str) -> list[float]:
        return self.embed_documents([text])[0]


def _parse_client_spec(
    spec: str | dict,
    *,
    api_key: str = "",
    default_kind: str = "llm",
) -> dict[str, Any]:
    """Parse a client specification string or dict into a config dict.

    String formats:
        - "bailian"              → provider only, use preset defaults
        - "bailian:qwen-plus"    → provider + model, use preset URL
        - "vllm:Qwen3.5-9B@http://localhost:8000/v1" → full specification

    Args:
        spec: String shorthand or dict config.
        api_key: Fallback API key.
        default_kind: "llm" or "embedder", used for default model selection.

    Returns:
        Dict with keys: provider, model, base_url, api_key
    """
    if isinstance(spec, dict):
        return {
            "provider": spec.get("provider", ""),
            "model": spec.get("model", ""),
            "base_url": spec.get("base_url", ""),
            "api_key": spec.get("api_key") or api_key,
            **{
                k: v
                for k, v in spec.items()
                if k not in ("provider", "model", "base_url", "api_key")
            },
        }

    # Parse string shorthand: provider:model@url
    provider = ""
    model = ""
    base_url = ""

    parts = spec.split(":", 1)
    provider = parts[0].strip()

    if len(parts) > 1:
        rest = parts[1].strip()
        if "@" in rest:
            model, base_url = rest.split("@", 1)
            model = model.strip()
            base_url = base_url.strip()
        else:
            model = rest.strip()

    # Fill defaults from preset
    preset = PROVIDER_PRESETS.get(provider, {})
    if not base_url:
        preset_url = preset.get("base_url")
        if preset_url is not None:
            base_url = preset_url
    if not model:
        default_key = f"default_{default_kind}"
        model = preset.get(default_key) or ""

    return {
        "provider": provider,
        "model": model,
        "base_url": base_url,
        "api_key": api_key,
    }


def create_llm(
    spec: str | dict,
    *,
    api_key: str = "",
    **kwargs: Any,
) -> BaseChatModel:
    """Create an LLM client from a specification string or dict.

    Args:
        spec: String shorthand (e.g. "bailian:qwen-plus") or dict config.
        api_key: API key fallback.
        **kwargs: Extra args forwarded to ChatOpenAI (e.g. temperature).

    Returns:
        Configured ChatOpenAI instance.

    Examples:
        >>> llm = create_llm("bailian:qwen-plus", api_key="sk-xxx")
        >>> llm = create_llm("vllm:Qwen3.5-9B@localhost:8000/v1", api_key="dummy")
        >>> llm = create_llm({"provider": "bailian", "model": "qwen-plus", "temperature": 0.5})
    """
    config = _parse_client_spec(spec, api_key=api_key, default_kind="llm")
    config.update(kwargs)

    provider = config.get("provider", "")

    if provider in ANTHROPIC_PROVIDERS:
        try:
            from langchain_anthropic import ChatAnthropic
        except ImportError as e:
            raise ImportError(
                "The Anthropic provider requires 'langchain-anthropic'. "
                "Install it with: pip install 'hyperextract[anthropic]'"
            ) from e

        resolved_key = config["api_key"] or _env_api_key(provider)
        if not resolved_key:
            raise ValueError(
                "No Anthropic API key found. Set ANTHROPIC_API_KEY (or CLAUDE_API_KEY), "
                "or pass api_key=..."
            )

        return ChatAnthropic(
            model=config["model"],
            api_key=resolved_key,
            temperature=config.get("temperature", 0),
            # Anthropic requires max_tokens; default low (1024) truncates large
            # structured extractions, so use a more generous default.
            max_tokens=config.get("max_tokens", 4096),
        )

    if provider in GOOGLE_PROVIDERS:
        try:
            from langchain_google_genai import ChatGoogleGenerativeAI
        except ImportError as e:
            raise ImportError(
                "The Google / Gemini provider requires 'langchain-google-genai'. "
                "Install it with: pip install 'hyperextract[google]'"
            ) from e

        resolved_key = config["api_key"] or _env_api_key(provider)
        if not resolved_key:
            raise ValueError(
                "No Google API key found. Set GOOGLE_API_KEY (or GEMINI_API_KEY), "
                "or pass api_key=..."
            )

        return ChatGoogleGenerativeAI(
            model=config["model"],
            api_key=resolved_key,
            temperature=config.get("temperature", 0),
        )

    from langchain_openai import ChatOpenAI

    # DeepSeek V4 models default to "thinking" mode, which rejects the
    # tool_choice that method="function_calling" sends for structured output
    # (HTTP 400: "Thinking mode does not support this tool_choice"). Disable it
    # by default so Hyper-Extract's structured extraction works out of the box.
    # Users who want thinking can override: extra_body={"thinking": {"type": "enabled"}}.
    extra_body = dict(config.get("extra_body") or {})
    if provider == "deepseek" and "thinking" not in extra_body:
        extra_body["thinking"] = {"type": "disabled"}

    chat_kwargs: dict[str, Any] = {
        "model": config["model"],
        "api_key": config["api_key"] or _env_api_key(provider),
        "base_url": config.get("base_url") or None,
        "temperature": config.get("temperature", 0),
    }
    if extra_body:
        chat_kwargs["extra_body"] = extra_body
    return ChatOpenAI(**chat_kwargs)


def create_embedder(
    spec: str | dict,
    *,
    api_key: str = "",
    **kwargs: Any,
) -> Embeddings:
    """Create an Embedder client from a specification string or dict.

    Args:
        spec: String shorthand (e.g. "bailian:text-embedding-v4") or dict config.
        api_key: API key fallback.
        **kwargs: Extra args forwarded to the embedder class.

    Returns:
        Configured OpenAIEmbeddings or CompatibleEmbeddings instance.

    Examples:
        >>> emb = create_embedder("bailian", api_key="sk-xxx")
        >>> emb = create_embedder("vllm:bge-m3@localhost:8001/v1", api_key="dummy")
    """
    config = _parse_client_spec(spec, api_key=api_key, default_kind="embedder")

    provider = config.get("provider", "")
    if provider in ANTHROPIC_PROVIDERS:
        raise ValueError(
            "Anthropic does not provide an embeddings API. Configure a separate "
            "OpenAI-compatible embedder, e.g. "
            'create_client(llm="anthropic", embedder="openai:text-embedding-3-small", ...) '
            "or a local vLLM/bge-m3 endpoint."
        )
    if provider in GOOGLE_PROVIDERS:
        raise ValueError(
            "Google / Gemini has no mature embeddings path in this repo. Configure a "
            "separate OpenAI-compatible embedder, e.g. "
            'create_client(llm="google", embedder="openai:text-embedding-3-small", ...) '
            "or a local vLLM/bge-m3 endpoint."
        )

    base_url = config.get("base_url", "")
    uses_custom = bool(base_url and base_url.rstrip("/") != OPENAI_API_URL)

    resolved_key = config["api_key"] or _env_api_key(config.get("provider", ""))

    if uses_custom:
        return CompatibleEmbeddings(
            model=config["model"],
            api_key=resolved_key,
            base_url=base_url,
            **kwargs,
        )
    else:
        from langchain_openai import OpenAIEmbeddings

        return OpenAIEmbeddings(
            model=config["model"],
            api_key=resolved_key,
            **kwargs,
        )


def create_client(
    llm: str | dict | None = None,
    embedder: str | dict | None = None,
    *,
    provider: str = "",
    api_key: str = "",
    **kwargs: Any,
) -> tuple[BaseChatModel, Embeddings]:
    """Create both LLM and Embedder clients in one call.

    Supports three patterns:

    **Pattern A: Single provider string (cloud API, simplest)**
    >>> llm, emb = create_client("bailian", api_key="sk-xxx")
    # Uses preset defaults: qwen3.6-plus + text-embedding-v4

    **Pattern B: Separate specs (vLLM, two independent services)**
    >>> llm, emb = create_client(
    ...     llm="vllm:Qwen3.5-9B@http://localhost:8000/v1",
    ...     embedder="vllm:bge-m3@http://localhost:8001/v1",
    ...     api_key="dummy",
    ... )

    **Pattern C: Mixed deployment**
    >>> llm = create_client(
    ...     llm="bailian:qwen-plus",
    ...     embedder="vllm:bge-m3@localhost:8001/v1",
    ...     api_key="sk-xxx",
    ... )

    Args:
        llm: LLM spec string/dict, or None to auto-infer from provider.
        embedder: Embedder spec string/dict, or None to auto-infer from provider.
        provider: Fallback provider when llm/embedder are bare strings without provider prefix.
        api_key: Shared API key for both services.
        **kwargs: Extra args forwarded to ChatOpenAI (e.g. temperature).

    Returns:
        (llm_client, embedder_client) tuple.
    """
    # Pattern A: Single provider shorthand
    if llm is None and embedder is None:
        if not provider:
            raise ValueError(
                "Must provide llm=, embedder=, or provider= argument.\n"
                "Examples:\n"
                '  create_client("bailian", api_key="...")\n'
                '  create_client(llm="bailian:qwen-plus", embedder="bailian", api_key="...")\n'
                '  create_client(llm="vllm:Qwen3.5-9B@localhost:8000", '
                'embedder="vllm:bge-m3@localhost:8001", api_key="dummy")'
            )
        # Use provider preset defaults for both
        llm = provider
        embedder = provider

    # Parse llm config
    llm_config = _parse_client_spec(
        llm or provider, api_key=api_key, default_kind="llm"
    )
    llm_config.update(kwargs)

    # Parse embedder config
    # For cloud providers, embedder defaults to llm's provider
    embedder_spec = embedder or llm_config.get("provider", "")
    emb_config = _parse_client_spec(
        embedder_spec, api_key=api_key, default_kind="embedder"
    )

    # Build clients
    llm_client = create_llm(llm_config, api_key=api_key)
    embedder_client = create_embedder(emb_config, api_key=api_key)

    return llm_client, embedder_client


def get_client(
    config_path: str | Path | None = None,
) -> tuple[BaseChatModel, Embeddings]:
    """Get OpenAI LLM client and Embedder from config.

    Backward-compatible: reads ~/.he/config.toml.

    Args:
        config_path: Config file path, default ~/.he/config.toml

    Returns:
        (llm_client, embedder) tuple

    Examples:
        >>> from hyperextract.utils import get_client
        >>> llm, emb = get_client()
        >>> # Or with custom config path
        >>> llm, emb = get_client("/path/to/config.toml")
    """
    import os

    from hyperextract.cli.config import ConfigManager

    path = Path(config_path) if config_path else DEFAULT_CONFIG_FILE
    if not path.exists() and not os.getenv("OPENAI_API_KEY"):
        raise FileNotFoundError(
            f"No Hyper-Extract config found at {path}. Choose one:\n"
            "  1. he config init -p openai -k YOUR_KEY\n"
            "  2. Local models: he config llm -p vllm -u http://localhost:8000/v1 "
            "-k dummy -m Qwen/Qwen3.5-9B (plus he config embedder -p vllm "
            "-u http://localhost:8001/v1 -k dummy -m BAAI/bge-m3)\n"
            "  3. Pass clients explicitly: Template.create(..., "
            "llm_client=..., embedder=...)"
        )
    manager = ConfigManager(path)

    llm_config = manager.get_llm_config()
    emb_config = manager.get_embedder_config()

    # Build LLM client
    llm_client = create_llm(
        {
            "provider": llm_config.provider,
            "model": llm_config.model,
            "base_url": llm_config.base_url,
            "api_key": llm_config.api_key,
        }
    )

    # Build embedder client
    embedder_client = create_embedder(
        {
            "provider": emb_config.provider,
            "model": emb_config.model,
            "base_url": emb_config.base_url,
            "api_key": emb_config.api_key,
        }
    )

    return llm_client, embedder_client