Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: Cache anthropic tokenizer (#3286)
Matthew Zhou0804774f6326
diff --git a/letta/services/context_window_calculator/context_window_calculator.py b/letta/services/context_window_calculator/context_window_calculator.pyindex 47a9aacd4..b3a890280 100644--- a/letta/services/context_window_calculator/context_window_calculator.py+++ b/letta/services/context_window_calculator/context_window_calculator.py@@ -4,6 +4,7 @@ from typing import Any, List, Optional, Tuplefrom openai.types.beta.function_tool import FunctionTool as OpenAIToolfrom letta.log import get_logger+from letta.schemas.agent import AgentStatefrom letta.schemas.enums import MessageRolefrom letta.schemas.letta_message_content import TextContentfrom letta.schemas.memory import ContextWindowOverview@@ -56,7 +57,7 @@ class ContextWindowCalculator:return None, 1async def calculate_context_window(- self, agent_state: Any, actor: PydanticUser, token_counter: TokenCounter, message_manager: Any, passage_manager: Any+ self, agent_state: AgentState, actor: PydanticUser, token_counter: TokenCounter, message_manager: Any, passage_manager: Any) -> ContextWindowOverview:"""Calculate context window information using the provided token counter"""diff --git a/letta/services/context_window_calculator/token_counter.py b/letta/services/context_window_calculator/token_counter.pyindex 3e1de4f7c..1ec4a3fc9 100644--- a/letta/services/context_window_calculator/token_counter.py+++ b/letta/services/context_window_calculator/token_counter.py@@ -1,6 +1,9 @@+import hashlib+import jsonfrom abc import ABC, abstractmethodfrom typing import Any, Dict, List+from letta.helpers.decorators import async_redis_cachefrom letta.llm_api.anthropic_client import AnthropicClientfrom letta.schemas.openai.chat_completion_request import Tool as OpenAIToolfrom letta.utils import count_tokens@@ -33,16 +36,31 @@ class AnthropicTokenCounter(TokenCounter):self.client = anthropic_clientself.model = model+ @async_redis_cache(+ key_func=lambda self, text: f"anthropic_text_tokens:{self.model}:{hashlib.sha256(text.encode()).hexdigest()[:16]}",+ prefix="token_counter",+ ttl_s=3600, # cache for 1 hour+ )async def count_text_tokens(self, text: str) -> int:if not text:return 0return await self.client.count_tokens(model=self.model, messages=[{"role": "user", "content": text}])+ @async_redis_cache(+ key_func=lambda self, messages: f"anthropic_message_tokens:{self.model}:{hashlib.sha256(json.dumps(messages, sort_keys=True).encode()).hexdigest()[:16]}",+ prefix="token_counter",+ ttl_s=3600, # cache for 1 hour+ )async def count_message_tokens(self, messages: List[Dict[str, Any]]) -> int:if not messages:return 0return await self.client.count_tokens(model=self.model, messages=messages)+ @async_redis_cache(+ key_func=lambda self, tools: f"anthropic_tool_tokens:{self.model}:{hashlib.sha256(json.dumps([t.model_dump() for t in tools], sort_keys=True).encode()).hexdigest()[:16]}",+ prefix="token_counter",+ ttl_s=3600, # cache for 1 hour+ )async def count_tool_tokens(self, tools: List[OpenAITool]) -> int:if not tools:return 0@@ -58,11 +76,21 @@ class TiktokenCounter(TokenCounter):def __init__(self, model: str):self.model = model+ @async_redis_cache(+ key_func=lambda self, text: f"tiktoken_text_tokens:{self.model}:{hashlib.sha256(text.encode()).hexdigest()[:16]}",+ prefix="token_counter",+ ttl_s=3600, # cache for 1 hour+ )async def count_text_tokens(self, text: str) -> int:if not text:return 0return count_tokens(text)+ @async_redis_cache(+ key_func=lambda self, messages: f"tiktoken_message_tokens:{self.model}:{hashlib.sha256(json.dumps(messages, sort_keys=True).encode()).hexdigest()[:16]}",+ prefix="token_counter",+ ttl_s=3600, # cache for 1 hour+ )async def count_message_tokens(self, messages: List[Dict[str, Any]]) -> int:if not messages:return 0@@ -70,6 +98,11 @@ class TiktokenCounter(TokenCounter):return num_tokens_from_messages(messages=messages, model=self.model)+ @async_redis_cache(+ key_func=lambda self, tools: f"tiktoken_tool_tokens:{self.model}:{hashlib.sha256(json.dumps([t.model_dump() for t in tools], sort_keys=True).encode()).hexdigest()[:16]}",+ prefix="token_counter",+ ttl_s=3600, # cache for 1 hour+ )async def count_tool_tokens(self, tools: List[OpenAITool]) -> int:if not tools:return 0