LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Add a dedicated OpenAI-compatible LLM adapter (#1895)
jimmy055905773107
diff --git a/README.md b/README.mdindex a3ab108c6..9e282b3e1 100644--- a/README.md+++ b/README.md@@ -176,9 +176,10 @@ Also see [architecture](docs/ARCHITECTURE.md).| Provider | Status | Provider | Status ||----------|--------|----------|--------|| OpenAI | ✅ | Azure OpenAI | ✅ |-| Anthropic Claude | ✅ | Google Gemini | ✅ |-| AWS Bedrock | ✅ | Mistral AI | ✅ |-| Ollama (local) | ✅ | Anyscale | ✅ |+| OpenAI Compatible | ✅ | Anthropic Claude | ✅ |+| AWS Bedrock | ✅ | Google Gemini | ✅ |+| Ollama (local) | ✅ | Mistral AI | ✅ |+| Anyscale | ✅ | | |### Vector Databasesdiff --git a/frontend/public/icons/adapter-icons/OpenAICompatible.png b/frontend/public/icons/adapter-icons/OpenAICompatible.pngnew file mode 100644index 000000000..ec23189d9Binary files /dev/null and b/frontend/public/icons/adapter-icons/OpenAICompatible.png differdiff --git a/unstract/sdk1/src/unstract/sdk1/adapters/base1.py b/unstract/sdk1/src/unstract/sdk1/adapters/base1.pyindex 666aa50c7..7bc58296c 100644--- a/unstract/sdk1/src/unstract/sdk1/adapters/base1.py+++ b/unstract/sdk1/src/unstract/sdk1/adapters/base1.py@@ -346,6 +346,32 @@ class OpenAILLMParameters(BaseChatCompletionParameters):return f"openai/{model}"+class OpenAICompatibleLLMParameters(BaseChatCompletionParameters):+ """See https://docs.litellm.ai/docs/providers/openai_compatible/."""++ api_key: str | None = None+ api_base: str++ @staticmethod+ def validate(adapter_metadata: dict[str, "Any"]) -> dict[str, "Any"]:+ adapter_metadata["model"] = OpenAICompatibleLLMParameters.validate_model(+ adapter_metadata+ )+ api_key = adapter_metadata.get("api_key")+ if isinstance(api_key, str) and not api_key.strip():+ adapter_metadata["api_key"] = None+ return OpenAICompatibleLLMParameters(**adapter_metadata).model_dump()++ @staticmethod+ def validate_model(adapter_metadata: dict[str, "Any"]) -> str:+ model = str(adapter_metadata.get("model", "")).strip()+ if not model:+ raise ValueError("model is required for the OpenAI Compatible adapter.")+ if model.startswith("custom_openai/"):+ return model+ return f"custom_openai/{model}"++class AzureOpenAILLMParameters(BaseChatCompletionParameters):"""See https://docs.litellm.ai/docs/providers/azure/#completion---using-azure_ad_token-api_base-api_version."""diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/__init__.py b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/__init__.pyindex c23a33390..1da3590f5 100644--- a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/__init__.py+++ b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/__init__.py@@ -8,6 +8,7 @@ from unstract.sdk1.adapters.llm1.azure_openai import AzureOpenAILLMAdapterfrom unstract.sdk1.adapters.llm1.bedrock import AWSBedrockLLMAdapterfrom unstract.sdk1.adapters.llm1.ollama import OllamaLLMAdapterfrom unstract.sdk1.adapters.llm1.openai import OpenAILLMAdapter+from unstract.sdk1.adapters.llm1.openai_compatible import OpenAICompatibleLLMAdapterfrom unstract.sdk1.adapters.llm1.vertexai import VertexAILLMAdapteradapters: dict[str, dict[str, Any]] = {}@@ -22,5 +23,6 @@ __all__ = ["AzureOpenAILLMAdapter","OllamaLLMAdapter","OpenAILLMAdapter",+ "OpenAICompatibleLLMAdapter","VertexAILLMAdapter",]diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/openai_compatible.py b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/openai_compatible.pynew file mode 100644index 000000000..3cb3ceafc--- /dev/null+++ b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/openai_compatible.py@@ -0,0 +1,46 @@+from typing import Any++from unstract.sdk1.adapters.base1 import BaseAdapter, OpenAICompatibleLLMParameters+from unstract.sdk1.adapters.enums import AdapterTypes++DESCRIPTION = (+ "Adapter for servers that implement the OpenAI Chat Completions API "+ "(vLLM, LM Studio, self-hosted gateways, and third-party providers). "+ "Use OpenAI for the official OpenAI service."+)+++class OpenAICompatibleLLMAdapter(OpenAICompatibleLLMParameters, BaseAdapter):+ @staticmethod+ def get_id() -> str:+ return "openaicompatible|b6d10f33-2c41-49fc-a8c2-58d2b247fc09"++ @staticmethod+ def get_metadata() -> dict[str, Any]:+ return {+ "name": "OpenAI Compatible",+ "version": "1.0.0",+ "adapter": OpenAICompatibleLLMAdapter,+ "description": DESCRIPTION,+ "is_active": True,+ }++ @staticmethod+ def get_name() -> str:+ return "OpenAI Compatible"++ @staticmethod+ def get_description() -> str:+ return DESCRIPTION++ @staticmethod+ def get_provider() -> str:+ return "custom_openai"++ @staticmethod+ def get_icon() -> str:+ return "/icons/adapter-icons/OpenAICompatible.png"++ @staticmethod+ def get_adapter_type() -> AdapterTypes:+ return AdapterTypes.LLMdiff --git a/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/custom_openai.json b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/custom_openai.jsonnew file mode 100644index 000000000..8767ffdcf--- /dev/null+++ b/unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/custom_openai.json@@ -0,0 +1,60 @@+{+ "title": "OpenAI Compatible",+ "type": "object",+ "required": [+ "adapter_name",+ "api_base"+ ],+ "properties": {+ "adapter_name": {+ "type": "string",+ "title": "Name",+ "default": "",+ "description": "Provide a unique name for this adapter instance. Example: compatible-gateway-1"+ },+ "api_key": {+ "type": [+ "string",+ "null"+ ],+ "title": "API Key",+ "format": "password",+ "description": "API key for your OpenAI-compatible endpoint. Leave empty if the endpoint does not require one."+ },+ "model": {+ "type": "string",+ "title": "Model",+ "description": "The model name expected by your OpenAI-compatible endpoint. Examples: gateway-model, gpt-4o-mini, openai/gpt-4o"+ },+ "api_base": {+ "type": "string",+ "format": "url",+ "title": "API Base",+ "description": "Base URL for the OpenAI-compatible endpoint. Examples: https://gateway.example.com/v1, https://llm.example.net/openai/v1"+ },+ "max_tokens": {+ "type": "number",+ "minimum": 0,+ "multipleOf": 1,+ "title": "Maximum Output Tokens",+ "default": 4096,+ "description": "Maximum number of output tokens to limit LLM replies. Leave it empty to use the provider default."+ },+ "max_retries": {+ "type": "number",+ "minimum": 0,+ "multipleOf": 1,+ "title": "Max Retries",+ "default": 5,+ "description": "The maximum number of times to retry a request if it fails."+ },+ "timeout": {+ "type": "number",+ "minimum": 0,+ "multipleOf": 1,+ "title": "Timeout",+ "default": 900,+ "description": "Timeout in seconds."+ }+ }+}diff --git a/unstract/sdk1/tests/test_openai_compatible_adapter.py b/unstract/sdk1/tests/test_openai_compatible_adapter.pynew file mode 100644index 000000000..addbf080c--- /dev/null+++ b/unstract/sdk1/tests/test_openai_compatible_adapter.py@@ -0,0 +1,172 @@+import json+from functools import lru_cache+from importlib import import_module+from unittest.mock import MagicMock, patch++from unstract.sdk1.adapters.base1 import OpenAICompatibleLLMParameters+from unstract.sdk1.adapters.constants import Common+from unstract.sdk1.adapters.llm1 import adapters+from unstract.sdk1.adapters.llm1.openai_compatible import OpenAICompatibleLLMAdapter++OPENAI_COMPATIBLE_DESCRIPTION = (+ "Adapter for servers that implement the OpenAI Chat Completions API "+ "(vLLM, LM Studio, self-hosted gateways, and third-party providers). "+ "Use OpenAI for the official OpenAI service."+)+++@lru_cache(maxsize=1)+def _load_llm_module() -> object:+ import sys+ from types import ModuleType++ # Stub python-magic so importing LLM does not depend on libmagic+ # being available in the test environment. sys.modules entries set+ # here must persist (no patch.dict) so litellm and other lazy-loaded+ # modules stay resolvable across tests.+ sys.modules.setdefault("magic", ModuleType("magic"))+ return import_module("unstract.sdk1.llm")+++def _load_llm_class() -> type:+ return _load_llm_module().LLM+++def test_openai_compatible_adapter_is_registered() -> None:+ adapter_id = OpenAICompatibleLLMAdapter.get_id()++ assert adapter_id in adapters+ assert adapters[adapter_id][Common.MODULE] is OpenAICompatibleLLMAdapter+++def test_openai_compatible_validate_prefixes_model() -> None:+ validated = OpenAICompatibleLLMParameters.validate(+ {+ "api_base": "https://gateway.example.com/v1",+ "api_key": "test-key",+ "model": "gateway-model",+ }+ )++ assert validated["model"] == "custom_openai/gateway-model"+++def test_openai_compatible_validate_preserves_prefixed_model() -> None:+ validated = OpenAICompatibleLLMParameters.validate(+ {+ "api_base": "https://gateway.example.com/v1",+ "model": "custom_openai/openai/gpt-4o",+ }+ )++ assert validated["model"] == "custom_openai/openai/gpt-4o"+ assert validated["api_key"] is None+++def test_openai_compatible_validate_normalizes_blank_api_key_to_none() -> None:+ validated = OpenAICompatibleLLMParameters.validate(+ {+ "api_base": "https://gateway.example.com/v1",+ "api_key": " ",+ "model": "gateway-model",+ }+ )++ assert validated["api_key"] is None+++def test_openai_compatible_schema_is_loadable() -> None:+ schema = json.loads(OpenAICompatibleLLMAdapter.get_json_schema())++ assert schema["title"] == "OpenAI Compatible"+ assert schema["properties"]["api_key"]["type"] == ["string", "null"]+ assert "default" not in schema["properties"]["model"]+ assert "gateway-model" in schema["properties"]["model"]["description"]+ assert "ERNIE" not in schema["properties"]["model"]["description"]+ assert "qianfan" not in schema["properties"]["api_base"]["description"].lower()+ assert "default" not in schema["properties"]["api_base"]+++def test_openai_compatible_adapter_uses_distinct_description_and_icon() -> None:+ metadata = OpenAICompatibleLLMAdapter.get_metadata()++ assert OpenAICompatibleLLMAdapter.get_description() == OPENAI_COMPATIBLE_DESCRIPTION+ assert metadata["description"] == OPENAI_COMPATIBLE_DESCRIPTION+ assert OpenAICompatibleLLMAdapter.get_icon() == (+ "/icons/adapter-icons/OpenAICompatible.png"+ )+++def _build_llm_for_record_usage(llm_cls: type) -> object:+ llm = llm_cls.__new__(llm_cls)+ llm._platform_api_key = "platform-key"+ llm.platform_kwargs = {"run_id": "run-1"}+ llm._usage_kwargs = {}+ llm._pending_usage = []+ llm.adapter = MagicMock()+ llm.adapter.get_provider.return_value = "custom_openai"+ return llm+++def test_record_usage_uses_reported_prompt_tokens_without_estimating() -> None:+ llm_module = _load_llm_module()+ llm = _build_llm_for_record_usage(llm_module.LLM)++ with (+ patch.object(llm_module.litellm, "token_counter") as mock_token_counter,+ patch.object(llm_module.litellm, "cost_per_token", return_value=(0.0, 0.0)),+ ):+ llm._record_usage(+ model="custom_openai/gateway-model",+ messages=[{"role": "user", "content": "hello"}],+ usage={"prompt_tokens": 3, "completion_tokens": 4, "total_tokens": 7},+ llm_api="complete",+ )++ mock_token_counter.assert_not_called()+ assert len(llm._pending_usage) == 1+ assert llm._pending_usage[0]["prompt_tokens"] == 3+++def test_record_usage_tolerates_unmapped_models_without_prompt_tokens() -> None:+ llm_module = _load_llm_module()+ llm = _build_llm_for_record_usage(llm_module.LLM)++ with (+ patch.object(+ llm_module.litellm, "token_counter", side_effect=Exception("unmapped")+ ),+ patch.object(llm_module.litellm, "cost_per_token", return_value=(0.0, 0.0)),+ patch.object(llm_module.logger, "warning") as mock_warning,+ ):+ llm._record_usage(+ model="custom_openai/gateway-model",+ messages=[{"role": "user", "content": "hello"}],+ usage={"completion_tokens": 4, "total_tokens": 7},+ llm_api="complete",+ )++ assert len(llm._pending_usage) == 1+ assert llm._pending_usage[0]["prompt_tokens"] == 0+ assert "litellm.token_counter() fallback failed" in mock_warning.call_args.args[0]+++def test_record_usage_uses_estimated_prompt_tokens_when_usage_has_none() -> None:+ llm_module = _load_llm_module()+ llm = _build_llm_for_record_usage(llm_module.LLM)++ with (+ patch.object(+ llm_module.litellm, "token_counter", return_value=9+ ) as mock_token_counter,+ patch.object(llm_module.litellm, "cost_per_token", return_value=(0.0, 0.0)),+ ):+ llm._record_usage(+ model="custom_openai/gateway-model",+ messages=[{"role": "user", "content": "hello"}],+ usage={"completion_tokens": 4, "total_tokens": 13},+ llm_api="complete",+ )++ mock_token_counter.assert_called_once()+ assert llm._pending_usage[0]["prompt_tokens"] == 9