Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

chore: enable vertex experimental (#2053)

Andy Li77e7e9555f9b
diff --git a/letta/llm_api/google_ai_client.py b/letta/llm_api/google_ai_client.py
index c41ad0501..ad650c5f1 100644
--- a/letta/llm_api/google_ai_client.py
+++ b/letta/llm_api/google_ai_client.py
@@ -165,10 +165,12 @@ class GoogleAIClient(LLMClientBase):
# NOTE: this also involves stripping the inner monologue out of the function
if llm_config.put_inner_thoughts_in_kwargs:
- from letta.local_llm.constants import INNER_THOUGHTS_KWARG
+ from letta.local_llm.constants import INNER_THOUGHTS_KWARG_VERTEX
- assert INNER_THOUGHTS_KWARG in function_args, f"Couldn't find inner thoughts in function args:\n{function_call}"
- inner_thoughts = function_args.pop(INNER_THOUGHTS_KWARG)
+ assert (
+ INNER_THOUGHTS_KWARG_VERTEX in function_args
+ ), f"Couldn't find inner thoughts in function args:\n{function_call}"
+ inner_thoughts = function_args.pop(INNER_THOUGHTS_KWARG_VERTEX)
assert inner_thoughts is not None, f"Expected non-null inner thoughts function arg:\n{function_call}"
else:
inner_thoughts = None
@@ -380,13 +382,13 @@ class GoogleAIClient(LLMClientBase):
# Add inner thoughts
if llm_config.put_inner_thoughts_in_kwargs:
- from letta.local_llm.constants import INNER_THOUGHTS_KWARG, INNER_THOUGHTS_KWARG_DESCRIPTION
+ from letta.local_llm.constants import INNER_THOUGHTS_KWARG_DESCRIPTION, INNER_THOUGHTS_KWARG_VERTEX
- func["parameters"]["properties"][INNER_THOUGHTS_KWARG] = {
+ func["parameters"]["properties"][INNER_THOUGHTS_KWARG_VERTEX] = {
"type": "string",
"description": INNER_THOUGHTS_KWARG_DESCRIPTION,
}
- func["parameters"]["required"].append(INNER_THOUGHTS_KWARG)
+ func["parameters"]["required"].append(INNER_THOUGHTS_KWARG_VERTEX)
return [{"functionDeclarations": function_list}]
diff --git a/letta/llm_api/google_vertex_client.py b/letta/llm_api/google_vertex_client.py
index 9fd66558b..5afb7fb01 100644
--- a/letta/llm_api/google_vertex_client.py
+++ b/letta/llm_api/google_vertex_client.py
@@ -168,10 +168,12 @@ class GoogleVertexClient(GoogleAIClient):
# NOTE: this also involves stripping the inner monologue out of the function
if llm_config.put_inner_thoughts_in_kwargs:
- from letta.local_llm.constants import INNER_THOUGHTS_KWARG
+ from letta.local_llm.constants import INNER_THOUGHTS_KWARG_VERTEX
- assert INNER_THOUGHTS_KWARG in function_args, f"Couldn't find inner thoughts in function args:\n{function_call}"
- inner_thoughts = function_args.pop(INNER_THOUGHTS_KWARG)
+ assert (
+ INNER_THOUGHTS_KWARG_VERTEX in function_args
+ ), f"Couldn't find inner thoughts in function args:\n{function_call}"
+ inner_thoughts = function_args.pop(INNER_THOUGHTS_KWARG_VERTEX)
assert inner_thoughts is not None, f"Expected non-null inner thoughts function arg:\n{function_call}"
else:
inner_thoughts = None
diff --git a/letta/local_llm/constants.py b/letta/local_llm/constants.py
index f4c66a47c..53f4e717e 100644
--- a/letta/local_llm/constants.py
+++ b/letta/local_llm/constants.py
@@ -26,6 +26,7 @@ DEFAULT_WRAPPER = ChatMLInnerMonologueWrapper
DEFAULT_WRAPPER_NAME = "chatml"
INNER_THOUGHTS_KWARG = "inner_thoughts"
+INNER_THOUGHTS_KWARG_VERTEX = "thinking"
INNER_THOUGHTS_KWARG_DESCRIPTION = "Deep inner monologue private to you only."
INNER_THOUGHTS_KWARG_DESCRIPTION_GO_FIRST = f"Deep inner monologue private to you only. Think before you act, so always generate arg '{INNER_THOUGHTS_KWARG}' first before any other arg."
INNER_THOUGHTS_CLI_SYMBOL = "💭"
diff --git a/letta/schemas/message.py b/letta/schemas/message.py
index 7fbe1fd4d..182f608ea 100644
--- a/letta/schemas/message.py
+++ b/letta/schemas/message.py
@@ -16,7 +16,7 @@ from pydantic import BaseModel, Field, field_validator
from letta.constants import DEFAULT_MESSAGE_TOOL, DEFAULT_MESSAGE_TOOL_KWARG, TOOL_CALL_ID_MAX_LEN
from letta.helpers.datetime_helpers import get_utc_time, is_utc_datetime
from letta.helpers.json_helpers import json_dumps
-from letta.local_llm.constants import INNER_THOUGHTS_KWARG
+from letta.local_llm.constants import INNER_THOUGHTS_KWARG, INNER_THOUGHTS_KWARG_VERTEX
from letta.schemas.enums import MessageRole
from letta.schemas.letta_base import OrmMetadataBase
from letta.schemas.letta_message import (
@@ -914,9 +914,9 @@ class Message(BaseMessage):
function_args = {"args": function_args}
if put_inner_thoughts_in_kwargs and text_content is not None:
- assert "inner_thoughts" not in function_args, function_args
+ assert INNER_THOUGHTS_KWARG not in function_args, function_args
assert len(self.tool_calls) == 1
- function_args[INNER_THOUGHTS_KWARG] = text_content
+ function_args[INNER_THOUGHTS_KWARG_VERTEX] = text_content
parts.append(
{
diff --git a/letta/server/rest_api/routers/v1/agents.py b/letta/server/rest_api/routers/v1/agents.py
index 4fe1d155d..8d9fd7f65 100644
--- a/letta/server/rest_api/routers/v1/agents.py
+++ b/letta/server/rest_api/routers/v1/agents.py
@@ -631,12 +631,17 @@ async def send_message(
# TODO: This is redundant, remove soon
agent = server.agent_manager.get_agent_by_id(agent_id, actor)
- if (
+ if all(
+ (
+ settings.use_experimental,
+ not agent.enable_sleeptime,
+ not agent.multi_agent_group,
+ not agent.agent_type == AgentType.sleeptime_agent,
+ )
+ ) and (
+ # LLM Model Check: (1) Anthropic or (2) Google Vertex + Flag
agent.llm_config.model_endpoint_type == "anthropic"
- and not agent.enable_sleeptime
- and not agent.multi_agent_group
- and not agent.agent_type == AgentType.sleeptime_agent
- and settings.use_experimental
+ or (agent.llm_config.model_endpoint_type == "google_vertex" and settings.use_vertex_async_loop_experimental)
):
experimental_agent = LettaAgent(
agent_id=agent_id,