Skip to content

BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.

Add Anthropic Vertex client for Claude Haiku on GCP.

Christian557f5c39d140
diff --git a/.env.example b/.env.example
index a98fa2c..b1b6ee6 100644
--- a/.env.example
+++ b/.env.example
@@ -1,16 +1,17 @@
# LLM Implementation envs
# The following are just examples of the adapter implementation, you can have completely different envs
LLM_SMALL_PROVIDER="gcp_vertex"
-LLM_LARGE_PROVIDER="azure"
+LLM_LARGE_PROVIDER="gcp_vertex"
EMBEDDINGS_PROVIDER="azure"
AZURE_SMALL_LLM_MODEL="gpt-4o-mini"
AZURE_LARGE_LLM_MODEL="gpt-4o"
AZURE_LLM_API_VERSION="2024-12-01-preview"
AZURE_LLM_ENDPOINT="https://yourproject.openai.azure.com"
AZURE_LLM_SUBSCRIPTION_KEY="xyz"
-GCP_EXTRA_SMALL_LLM_MODEL="gemini-3-flash-preview"
-GCP_SMALL_LLM_MODEL="gemini-3-flash-preview"
-GCP_LARGE_LLM_MODEL="gemini-3-flash-preview"
+GCP_EXTRA_SMALL_LLM_MODEL="claude-haiku-5-5"
+GCP_SMALL_LLM_MODEL="claude-haiku-5-5"
+GCP_LARGE_LLM_MODEL="claude-haiku-5-5"
+GCP_VERTEX_REGION="global"
GCP_EMBEDDING_MODEL="text-embedding-005"
GCP_PROJECT_ID="your-project-id"
GCP_CREDENTIALS_PATH=".path-to-credentials-file.json"
@@ -27,8 +28,8 @@ OPENAI_SMALL_LLM_MODEL="gpt-4o-mini"
OPENAI_LARGE_LLM_MODEL="gpt-4o"
OPENAI_EMBEDDING_MODEL="text-embedding-3-large"
# DeepSeek chat (OpenAI-compatible). No embeddings — keep EMBEDDINGS_PROVIDER on azure/openai/gcp_vertex/etc.
-# LLM_SMALL_PROVIDER="deepseek"
-# LLM_LARGE_PROVIDER="deepseek"
+# LLM_SMALL_PROVIDER="gcp_vertex"
+# LLM_LARGE_PROVIDER="gcp_vertex"
DEEPSEEK_API_KEY="sk-..."
DEEPSEEK_SMALL_LLM_MODEL="deepseek-v4-flash"
DEEPSEEK_LARGE_LLM_MODEL="deepseek-v4-pro"
@@ -117,8 +118,8 @@ PUBLIC_DEMO_MAX_K=10
# GCP
# Used on the oss project for the small LLM but can be changed to any other model
-GCP_EXTRA_SMALL_LLM_MODEL="gemini-3-flash-preview"
-GCP_SMALL_LLM_MODEL="gemini-3-flash-preview"
+GCP_EXTRA_SMALL_LLM_MODEL="claude-haiku-5-5"
+GCP_SMALL_LLM_MODEL="claude-haiku-5-5"
GCP_PROJECT_ID="your-project-id"
GCP_CREDENTIALS_PATH=".path-to-credentials-file.json"
diff --git a/deploy/env.heavy.example b/deploy/env.heavy.example
index bb99c8f..94985d3 100644
--- a/deploy/env.heavy.example
+++ b/deploy/env.heavy.example
@@ -4,17 +4,18 @@
# LLM Implementation envs
LLM_SMALL_PROVIDER="gcp_vertex"
-LLM_LARGE_PROVIDER="azure"
+LLM_LARGE_PROVIDER="gcp_vertex"
EMBEDDINGS_PROVIDER="azure"
AZURE_SMALL_LLM_MODEL="gpt-4o-mini"
AZURE_LARGE_LLM_MODEL="gpt-4o"
AZURE_LLM_API_VERSION="2024-12-01-preview"
AZURE_LLM_ENDPOINT="https://yourproject.openai.azure.com"
AZURE_LLM_SUBSCRIPTION_KEY="xyz"
-GCP_EXTRA_SMALL_LLM_MODEL="gemini-3-flash-preview"
-GCP_SMALL_LLM_MODEL="gemini-3-flash-preview"
-GCP_LARGE_LLM_MODEL="gemini-3-flash-preview"
+GCP_EXTRA_SMALL_LLM_MODEL="claude-haiku-5-5"
+GCP_SMALL_LLM_MODEL="claude-haiku-5-5"
+GCP_LARGE_LLM_MODEL="claude-haiku-5-5"
GCP_EMBEDDING_MODEL="text-embedding-005"
+GCP_VERTEX_REGION="global"
GCP_PROJECT_ID="your-project-id"
GCP_CREDENTIALS_PATH=".path-to-credentials-file.json"
BEDROCK_REGION="us-east-1"
@@ -30,8 +31,8 @@ OPENAI_SMALL_LLM_MODEL="gpt-4o-mini"
OPENAI_LARGE_LLM_MODEL="gpt-4o"
OPENAI_EMBEDDING_MODEL="text-embedding-3-large"
# DeepSeek chat (OpenAI-compatible). No embeddings — keep EMBEDDINGS_PROVIDER on azure/openai/gcp_vertex/etc.
-# LLM_SMALL_PROVIDER="deepseek"
-# LLM_LARGE_PROVIDER="deepseek"
+# LLM_SMALL_PROVIDER="gcp_vertex"
+# LLM_LARGE_PROVIDER="gcp_vertex"
DEEPSEEK_API_KEY="sk-..."
DEEPSEEK_SMALL_LLM_MODEL="deepseek-v4-flash"
DEEPSEEK_LARGE_LLM_MODEL="deepseek-v4-pro"
@@ -116,8 +117,8 @@ PUBLIC_DEMO_MAX_K=10
# GCP
# Used on the oss project for the small LLM but can be changed to any other model
-GCP_EXTRA_SMALL_LLM_MODEL="gemini-3-flash-preview"
-GCP_SMALL_LLM_MODEL="gemini-3-flash-preview"
+GCP_EXTRA_SMALL_LLM_MODEL="claude-haiku-5-5"
+GCP_SMALL_LLM_MODEL="claude-haiku-5-5"
GCP_PROJECT_ID="your-project-id"
GCP_CREDENTIALS_PATH=".path-to-credentials-file.json"
diff --git a/deploy/env.light.example b/deploy/env.light.example
index f828647..02d50fc 100644
--- a/deploy/env.light.example
+++ b/deploy/env.light.example
@@ -1,20 +1,21 @@
# BrainAPI deploy profile: light
# Pair with deploy/docker-compose.light.yaml
-# Stack: Postgres/pgvector + NetworkX + Redis; LLMs: DeepSeek; embeddings: OpenAI
+# Stack: Postgres/pgvector + NetworkX + Redis; LLMs: Vertex Claude Haiku; embeddings: OpenAI
# LLM Implementation envs
-LLM_SMALL_PROVIDER="deepseek"
-LLM_LARGE_PROVIDER="deepseek"
+LLM_SMALL_PROVIDER="gcp_vertex"
+LLM_LARGE_PROVIDER="gcp_vertex"
EMBEDDINGS_PROVIDER="openai"
AZURE_SMALL_LLM_MODEL="gpt-4o-mini"
AZURE_LARGE_LLM_MODEL="gpt-4o"
AZURE_LLM_API_VERSION="2024-12-01-preview"
AZURE_LLM_ENDPOINT="https://yourproject.openai.azure.com"
AZURE_LLM_SUBSCRIPTION_KEY="xyz"
-GCP_EXTRA_SMALL_LLM_MODEL="gemini-3-flash-preview"
-GCP_SMALL_LLM_MODEL="gemini-3-flash-preview"
-GCP_LARGE_LLM_MODEL="gemini-3-flash-preview"
+GCP_EXTRA_SMALL_LLM_MODEL="claude-haiku-5-5"
+GCP_SMALL_LLM_MODEL="claude-haiku-5-5"
+GCP_LARGE_LLM_MODEL="claude-haiku-5-5"
GCP_EMBEDDING_MODEL="text-embedding-005"
+GCP_VERTEX_REGION="global"
GCP_PROJECT_ID="your-project-id"
GCP_CREDENTIALS_PATH=".path-to-credentials-file.json"
BEDROCK_REGION="us-east-1"
@@ -114,8 +115,8 @@ PUBLIC_DEMO_MAX_K=10
# GCP
# Used on the oss project for the small LLM but can be changed to any other model
-GCP_EXTRA_SMALL_LLM_MODEL="gemini-3-flash-preview"
-GCP_SMALL_LLM_MODEL="gemini-3-flash-preview"
+GCP_EXTRA_SMALL_LLM_MODEL="claude-haiku-5-5"
+GCP_SMALL_LLM_MODEL="claude-haiku-5-5"
GCP_PROJECT_ID="your-project-id"
GCP_CREDENTIALS_PATH=".path-to-credentials-file.json"
diff --git a/src/config.py b/src/config.py
index 00eca61..78f00cf 100644
--- a/src/config.py
+++ b/src/config.py
@@ -240,6 +240,9 @@ class GCPConfig:
logger.debug("GCP credentials_path: %s", credentials_path)
self.credentials_path = credentials_path
self.project_id = os.getenv("GCP_PROJECT_ID")
+ self.location = (
+ os.getenv("GCP_VERTEX_REGION") or os.getenv("GCP_LOCATION") or "global"
+ )
self.small_llm_model = os.getenv("GCP_SMALL_LLM_MODEL")
self.large_llm_model = os.getenv("GCP_LARGE_LLM_MODEL")
self.embedding_model = os.getenv("GCP_EMBEDDING_MODEL", "text-embedding-005")
@@ -510,6 +513,7 @@ _PROVIDERS = (
"azure",
"openai",
"anthropic",
+ "anthropic_vertex",
"deepseek",
"gcp_vertex",
"amazon_bedrock",
@@ -560,8 +564,11 @@ class Config:
use_azure_llm = self.llm_small_provider == "azure" or self.llm_large_provider == "azure"
use_azure_embeddings = self.embeddings_provider == "azure"
- use_gcp_llm = self.llm_small_provider == "gcp_vertex" or self.llm_large_provider == "gcp_vertex"
- use_gcp_large = self.llm_large_provider == "gcp_vertex"
+ use_gcp_llm = self.llm_small_provider in (
+ "gcp_vertex",
+ "anthropic_vertex",
+ ) or self.llm_large_provider in ("gcp_vertex", "anthropic_vertex")
+ use_gcp_large = self.llm_large_provider in ("gcp_vertex", "anthropic_vertex")
use_gcp_embeddings = self.embeddings_provider == "gcp_vertex"
use_bedrock_llm = (
self.llm_small_provider == "amazon_bedrock"
diff --git a/src/core/instances.py b/src/core/instances.py
index 5030a13..1d7f12c 100644
--- a/src/core/instances.py
+++ b/src/core/instances.py
@@ -6,6 +6,11 @@ from src.adapters.llm import LLMAdapter
from src.config import config
from src.lib.redis.client import _redis_client
+
+def _is_claude_model(model: str | None) -> bool:
+ return bool(model) and model.lower().startswith("claude")
+
+
def _build_small_llm(provider: str):
if provider == "ollama":
from src.lib.llm.client_ollama import _llm_small_client
@@ -23,11 +28,23 @@ def _build_small_llm(provider: str):
from src.lib.llm.client_anthropic import _llm_small_client_anthropic
return _llm_small_client_anthropic
+ if provider == "anthropic_vertex":
+ from src.lib.llm.client_anthropic_vertex import (
+ _llm_small_client_anthropic_vertex,
+ )
+
+ return _llm_small_client_anthropic_vertex
if provider == "deepseek":
from src.lib.llm.client_deepseek import _llm_small_client_deepseek
return _llm_small_client_deepseek
if provider == "gcp_vertex":
+ if config.gcp and _is_claude_model(config.gcp.small_llm_model):
+ from src.lib.llm.client_anthropic_vertex import (
+ _llm_small_client_anthropic_vertex,
+ )
+
+ return _llm_small_client_anthropic_vertex
from src.lib.llm.client_small import _llm_small_client
return _llm_small_client
@@ -55,11 +72,25 @@ def _build_large_llm(provider: str):
from src.lib.llm.client_anthropic import _llm_large_client_anthropic
return _llm_large_client_anthropic
+ if provider == "anthropic_vertex":
+ from src.lib.llm.client_anthropic_vertex import (
+ _llm_large_client_anthropic_vertex,
+ )
+
+ return _llm_large_client_anthropic_vertex
if provider == "deepseek":
from src.lib.llm.client_deepseek import _llm_large_client_deepseek
return _llm_large_client_deepseek
if provider == "gcp_vertex":
+ if config.gcp and _is_claude_model(
+ config.gcp.large_llm_model or config.gcp.small_llm_model
+ ):
+ from src.lib.llm.client_anthropic_vertex import (
+ _llm_large_client_anthropic_vertex,
+ )
+
+ return _llm_large_client_anthropic_vertex
from src.lib.llm.client_vertex import _llm_large_client_vertex
return _llm_large_client_vertex
diff --git a/src/lib/llm/client_anthropic_vertex.py b/src/lib/llm/client_anthropic_vertex.py
new file mode 100644
index 0000000..e6033cc
--- /dev/null
+++ b/src/lib/llm/client_anthropic_vertex.py
@@ -0,0 +1,101 @@
+import json
+import os
+import threading
+
+from anthropic import AnthropicVertex
+from langchain_google_vertexai.model_garden import ChatAnthropicVertex
+
+from src.adapters.interfaces.llm import LLM
+from src.config import config
+
+
+def _vertex_region() -> str:
+ return getattr(config.gcp, "location", None) or os.getenv(
+ "GCP_VERTEX_REGION", "global"
+ )
+
+
+class _AnthropicVertexLLMBase(LLM):
+ def __init__(self, model: str):
+ self.model = model
+ self._lock = threading.Lock()
+ self._pid = os.getpid()
+ self._client = None
+ self._langchain_model = None
+
+ def _check_fork(self):
+ if os.getpid() != self._pid:
+ self._client = None
+ self._langchain_model = None
+ self._pid = os.getpid()
+
+ def _prepare_credentials(self) -> None:
+ os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = config.gcp.credentials_path
+
+ @property
+ def client(self):
+ self._check_fork()
+ if self._client is None:
+ with self._lock:
+ if self._client is None:
+ self._prepare_credentials()
+ self._client = AnthropicVertex(
+ region=_vertex_region(),
+ project_id=config.gcp.project_id,
+ )
+ return self._client
+
+ @property
+ def langchain_model(self):
+ self._check_fork()
+ if self._langchain_model is None:
+ with self._lock:
+ if self._langchain_model is None:
+ self._prepare_credentials()
+ self._langchain_model = ChatAnthropicVertex(
+ model_name=self.model,
+ project=config.gcp.project_id,
+ location=_vertex_region(),
+ max_retries=5,
+ timeout=120,
+ )
+ return self._langchain_model
+
+ def generate_text(self, prompt: str, max_new_tokens: int = None) -> str:
+ response = self.client.messages.create(
+ model=self.model,
+ max_tokens=max_new_tokens or 1024,
+ messages=[{"role": "user", "content": prompt}],
+ )
+ parts = [
+ block.text for block in response.content if getattr(block, "text", None)
+ ]
+ return "".join(parts)
+
+ def generate_json(
+ self, prompt: str, max_new_tokens: int = None, max_retries: int = 3
+ ) -> dict:
+ response = None
+ while max_retries > 0 and response is None:
+ try:
+ raw = self.generate_text(prompt, max_new_tokens=max_new_tokens)
+ response = json.loads(raw.strip().strip("```json").strip("```"))
+ except Exception as e:
+ max_retries -= 1
+ if max_retries <= 0:
+ raise e
+ return response
+
+
+class AnthropicVertexLLMClientSmall(_AnthropicVertexLLMBase):
+ def __init__(self):
+ super().__init__(config.gcp.small_llm_model)
+
+
+class AnthropicVertexLLMClientLarge(_AnthropicVertexLLMBase):
+ def __init__(self):
+ super().__init__(config.gcp.large_llm_model or config.gcp.small_llm_model)
+
+
+_llm_small_client_anthropic_vertex = AnthropicVertexLLMClientSmall()
+_llm_large_client_anthropic_vertex = AnthropicVertexLLMClientLarge()
diff --git a/tui/src/constants.ts b/tui/src/constants.ts
index e3ebcb6..f2fdc0c 100644
--- a/tui/src/constants.ts
+++ b/tui/src/constants.ts
@@ -14,8 +14,8 @@ export const OLLAMA_DEFAULT_PORT = 11434;
export const OLLAMA_DEFAULT_SMALL_MODEL = "hf.co/unsloth/Qwen3-14B-GGUF:Q4_K_M";
export const OLLAMA_DEFAULT_LARGE_MODEL = "hf.co/unsloth/Qwen3-14B-GGUF:Q4_K_M";
-export const GCP_DEFAULT_SMALL_MODEL = "gemini-3-flash-preview";
-export const GCP_DEFAULT_LARGE_MODEL = "gemini-3-flash-preview";
+export const GCP_DEFAULT_SMALL_MODEL = "claude-haiku-5-5";
+export const GCP_DEFAULT_LARGE_MODEL = "claude-haiku-5-5";
export const GCP_DEFAULT_EMBEDDING_MODEL = "text-embedding-005";
export const AZURE_DEFAULT_LARGE_MODEL = "gpt-4o";