Agent Zero AI framework
rate limiter polishing
frdel27b2a691fdc0
diff --git a/.gitignore b/.gitignoreindex aad645a53..17d47e1c8 100644--- a/.gitignore+++ b/.gitignore@@ -39,6 +39,4 @@ instruments/**# Global rule to include .gitkeep files anywhere!**/.gitkeep-agent_history.gif--tests/**+agent_history.gif\ No newline at end of filediff --git a/models.py b/models.pyindex 5e2170457..f93cfbe46 100644--- a/models.py+++ b/models.py@@ -22,7 +22,6 @@ from python.helpers.dotenv import load_dotenvfrom python.helpers.providers import get_provider_configfrom python.helpers.rate_limiter import RateLimiterfrom python.helpers.tokens import approximate_tokens-import asynciofrom langchain_core.language_models.chat_models import SimpleChatModelfrom langchain_core.outputs.chat_generation import ChatGenerationChunk@@ -109,6 +108,28 @@ def get_rate_limiter(limiter.limits["output"] = output or 0return limiter+async def apply_rate_limiter(model_config: ModelConfig|None, input_text: str):+ if not model_config:+ return+ limiter = get_rate_limiter(+ model_config.provider,+ model_config.name,+ model_config.limit_requests,+ model_config.limit_input,+ model_config.limit_output,+ )+ limiter.add(input=approximate_tokens(input_text))+ limiter.add(requests=1)+ await limiter.wait()+ return limiter++def apply_rate_limiter_sync(model_config: ModelConfig|None, input_text: str):+ if not model_config:+ return+ import asyncio, nest_asyncio+ nest_asyncio.apply()+ return asyncio.run(apply_rate_limiter(model_config, input_text))+class LiteLLMChatWrapper(SimpleChatModel):model_name: str@@ -123,30 +144,13 @@ class LiteLLMChatWrapper(SimpleChatModel):def __init__(self, model: str, provider: str, model_config: Optional[ModelConfig] = None, **kwargs: Any):model_value = f"{provider}/{model}"super().__init__(model_name=model_value, provider=provider, kwargs=kwargs) # type: ignore- # Set rate limiting config as instance attribute after parent init- self.rate_limit_config = model_config+ # Set A0 model config as instance attribute after parent init+ self.a0_model_conf = model_config@propertydef _llm_type(self) -> str:return "litellm-chat"- async def _apply_rate_limiting(self, input_text: str, background: bool = False):- """Apply rate limiting if rate_limit_config is provided"""- if not self.rate_limit_config:- return None-- limiter = get_rate_limiter(- self.rate_limit_config.provider,- self.rate_limit_config.name,- self.rate_limit_config.limit_requests,- self.rate_limit_config.limit_input,- self.rate_limit_config.limit_output,- )- limiter.add(input=approximate_tokens(input_text))- limiter.add(requests=1)- await limiter.wait()- return limiter-def _convert_messages(self, messages: List[BaseMessage]) -> List[dict]:result = []# Map LangChain message types to LiteLLM roles@@ -207,18 +211,14 @@ class LiteLLMChatWrapper(SimpleChatModel):msgs = self._convert_messages(messages)# Apply rate limiting if configured- if self.rate_limit_config:- try:- # Convert messages to text for token counting- messages_text = str(msgs)- asyncio.run(self._apply_rate_limiting(messages_text))- except Exception as e:- # Don't let rate limiting break the model call- print(f"Rate limiting warning: {e}")+ apply_rate_limiter_sync(self.a0_model_conf, str(msgs))+ # Call the modelresp = completion(model=self.model_name, messages=msgs, stop=stop, **{**self.kwargs, **kwargs})++ # Parse outputparsed = _parse_chunk(resp)return parsed["response_delta"]@@ -234,14 +234,7 @@ class LiteLLMChatWrapper(SimpleChatModel):msgs = self._convert_messages(messages)# Apply rate limiting if configured- if self.rate_limit_config:- try:- # Convert messages to text for token counting- messages_text = str(msgs)- asyncio.run(self._apply_rate_limiting(messages_text))- except Exception as e:- # Don't let rate limiting break the model call- print(f"Rate limiting warning: {e}")+ apply_rate_limiter_sync(self.a0_model_conf, str(msgs))for chunk in completion(model=self.model_name,@@ -267,10 +260,8 @@ class LiteLLMChatWrapper(SimpleChatModel):msgs = self._convert_messages(messages)# Apply rate limiting if configured- if self.rate_limit_config:- # Convert messages to text for token counting- messages_text = str(msgs)- await self._apply_rate_limiting(messages_text)+ await apply_rate_limiter(self.a0_model_conf, str(msgs))+response = await acompletion(model=self.model_name,@@ -312,11 +303,7 @@ class LiteLLMChatWrapper(SimpleChatModel):msgs_conv = self._convert_messages(messages)# Apply rate limiting if configured- limiter = None- if self.rate_limit_config:- # Convert messages to text for token counting- messages_text = str(msgs_conv)- limiter = await self._apply_rate_limiting(messages_text)+ limiter = await apply_rate_limiter(self.a0_model_conf, str(msgs_conv))# call model_completion = await acompletion(@@ -402,54 +389,16 @@ class BrowserCompatibleChatWrapper(LiteLLMChatWrapper):class LiteLLMEmbeddingWrapper(Embeddings):model_name: strkwargs: dict = {}- rate_limit_config: Optional[ModelConfig] = None+ a0_model_conf: Optional[ModelConfig] = Nonedef __init__(self, model: str, provider: str, model_config: Optional[ModelConfig] = None, **kwargs: Any):self.model_name = f"{provider}/{model}" if provider != "openai" else modelself.kwargs = kwargs- self.rate_limit_config = model_config+ self.a0_model_conf = model_config- def _apply_rate_limiting_sync(self, input_text: str):- """Apply rate limiting synchronously if model_config is provided"""- if not self.rate_limit_config:- return None-- limiter = get_rate_limiter(- self.rate_limit_config.provider,- self.rate_limit_config.name,- self.rate_limit_config.limit_requests,- self.rate_limit_config.limit_input,- self.rate_limit_config.limit_output,- )- limiter.add(input=approximate_tokens(input_text))- limiter.add(requests=1)- # Note: Embeddings typically don't have streaming, so we do synchronous rate limiting- import asyncio- try:- asyncio.run(limiter.wait())- except RuntimeError:- # If we're already in an event loop, create a new thread- import threading- import concurrent.futures-- def wait_for_limiter():- new_loop = asyncio.new_event_loop()- asyncio.set_event_loop(new_loop)- try:- new_loop.run_until_complete(limiter.wait())- finally:- new_loop.close()-- with concurrent.futures.ThreadPoolExecutor() as executor:- executor.submit(wait_for_limiter).result()- return limiter-def embed_documents(self, texts: List[str]) -> List[List[float]]:# Apply rate limiting if configured- if self.rate_limit_config:- # Convert texts to combined string for token counting- texts_combined = " ".join(texts)- self._apply_rate_limiting_sync(texts_combined)+ apply_rate_limiter_sync(self.a0_model_conf, " ".join(texts))resp = embedding(model=self.model_name, input=texts, **self.kwargs)return [@@ -459,8 +408,7 @@ class LiteLLMEmbeddingWrapper(Embeddings):def embed_query(self, text: str) -> List[float]:# Apply rate limiting if configured- if self.rate_limit_config:- self._apply_rate_limiting_sync(text)+ apply_rate_limiter_sync(self.a0_model_conf, text)resp = embedding(model=self.model_name, input=[text], **self.kwargs)item = resp.data[0] # type: ignore@@ -480,57 +428,18 @@ class LocalSentenceTransformerWrapper(Embeddings):self.model = SentenceTransformer(model, **kwargs)self.model_name = model- self.rate_limit_config = model_config+ self.a0_model_conf = model_config- def _apply_rate_limiting_sync(self, input_text: str):- """Apply rate limiting synchronously if model_config is provided"""- if not self.rate_limit_config:- return None-- limiter = get_rate_limiter(- self.rate_limit_config.provider,- self.rate_limit_config.name,- self.rate_limit_config.limit_requests,- self.rate_limit_config.limit_input,- self.rate_limit_config.limit_output,- )- limiter.add(input=approximate_tokens(input_text))- limiter.add(requests=1)- # Note: Local models still respect rate limiting for consistency- import asyncio- try:- asyncio.run(limiter.wait())- except RuntimeError:- # If we're already in an event loop, create a new thread- import threading- import concurrent.futures-- def wait_for_limiter():- new_loop = asyncio.new_event_loop()- asyncio.set_event_loop(new_loop)- try:- new_loop.run_until_complete(limiter.wait())- finally:- new_loop.close()-- with concurrent.futures.ThreadPoolExecutor() as executor:- executor.submit(wait_for_limiter).result()- return limiter-def embed_documents(self, texts: List[str]) -> List[List[float]]:# Apply rate limiting if configured- if self.rate_limit_config:- # Convert texts to combined string for token counting- texts_combined = " ".join(texts)- self._apply_rate_limiting_sync(texts_combined)+ apply_rate_limiter_sync(self.a0_model_conf, " ".join(texts))embeddings = self.model.encode(texts, convert_to_tensor=False) # type: ignorereturn embeddings.tolist() if hasattr(embeddings, "tolist") else embeddings # type: ignoredef embed_query(self, text: str) -> List[float]:# Apply rate limiting if configured- if self.rate_limit_config:- self._apply_rate_limiting_sync(text)+ apply_rate_limiter_sync(self.a0_model_conf, text)embedding = self.model.encode([text], convert_to_tensor=False) # type: ignoreresult = (diff --git a/python/helpers/settings.py b/python/helpers/settings.pyindex 4ea9a23eb..9c842630d 100644--- a/python/helpers/settings.py+++ b/python/helpers/settings.py@@ -48,6 +48,9 @@ class Settings(TypedDict):browser_model_name: strbrowser_model_api_base: strbrowser_model_vision: bool+ browser_model_rl_requests: int+ browser_model_rl_input: int+ browser_model_rl_output: intbrowser_model_kwargs: dict[str, str]agent_profile: str@@ -447,6 +450,36 @@ def convert_out(settings: Settings) -> SettingsOutput:})+ browser_model_fields.append(+ {+ "id": "browser_model_rl_requests",+ "title": "Web Browser model rate limit requests",+ "description": "Rate limit requests for web browser model.",+ "type": "number",+ "value": settings["browser_model_rl_requests"],+ }+ )++ browser_model_fields.append(+ {+ "id": "browser_model_rl_input",+ "title": "Web Browser model rate limit input",+ "description": "Rate limit input for web browser model.",+ "type": "number",+ "value": settings["browser_model_rl_input"],+ }+ )++ browser_model_fields.append(+ {+ "id": "browser_model_rl_output",+ "title": "Web Browser model rate limit output",+ "description": "Rate limit output for web browser model.",+ "type": "number",+ "value": settings["browser_model_rl_output"],+ }+ )+browser_model_fields.append({"id": "browser_model_kwargs",@@ -1199,6 +1232,9 @@ def get_default_settings() -> Settings:browser_model_name="openai/gpt-4.1",browser_model_api_base="",browser_model_vision=True,+ browser_model_rl_requests=0,+ browser_model_rl_input=0,+ browser_model_rl_output=0,browser_model_kwargs={"temperature": "0"},memory_recall_enabled=True,memory_recall_interval=3,diff --git a/tests/rate_limiter_test.py b/tests/rate_limiter_test.pynew file mode 100644index 000000000..1902f4b0a--- /dev/null+++ b/tests/rate_limiter_test.py@@ -0,0 +1,32 @@++import sys+import os+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))+import models++provider = "openai"+name = "gpt-4.1-mini"++model = models.get_chat_model(+ provider=provider,+ name=name,+ model_config=models.ModelConfig(+ type=models.ModelType.CHAT,+ provider=provider,+ name=name,+ limit_requests = 5,+ limit_input = 15000,+ limit_output = 1000,+ )+ )++async def run():+ response, reasoning = await model.unified_call(+ user_message="Tell me a joke"+ )+ print("Response: ", response)+ print("Reasoning: ", reasoning)+++import asyncio+asyncio.run(run())\ No newline at end of file