Skip to content

Agent Zero AI framework

rate limiter polishing, file download polishing

frdel765470796304
diff --git a/agent.py b/agent.py
index 2a4ae73d7..3e2d30ed2 100644
--- a/agent.py
+++ b/agent.py
@@ -613,15 +613,6 @@ class Agent:
):
model = self.get_utility_model()
- # rate limiter
- limiter = await self.rate_limiter(
- self.config.utility_model, f"SYSTEM: {system}\nUSER: {message}", background
- )
-
- # add output tokens to rate limiter in tokens callback
- async def tokens_callback(delta: str, tokens: int):
- await self.handle_intervention()
- limiter.add(output=tokens)
# propagate stream to callback if set
async def stream_callback(chunk: str, total: str):
@@ -632,7 +623,7 @@ class Agent:
system_message=system,
user_message=message,
response_callback=stream_callback,
- tokens_callback=tokens_callback,
+ rate_limiter_callback=self.rate_limiter_callback if not background else None,
)
return response
@@ -642,63 +633,29 @@ class Agent:
messages: list[BaseMessage],
response_callback: Callable[[str, str], Awaitable[None]] | None = None,
reasoning_callback: Callable[[str, str], Awaitable[None]] | None = None,
+ background: bool = False,
):
response = ""
# model class
model = self.get_chat_model()
- # rate limiter
- limiter = await self.rate_limiter(
- self.config.chat_model, ChatPromptTemplate.from_messages(messages).format()
- )
-
- # add output tokens to rate limiter in tokens callback
- async def tokens_callback(delta: str, tokens: int):
- await self.handle_intervention()
- limiter.add(output=tokens)
-
# call model
response, reasoning = await model.unified_call(
messages=messages,
reasoning_callback=reasoning_callback,
response_callback=response_callback,
- tokens_callback=tokens_callback,
+ rate_limiter_callback=self.rate_limiter_callback if not background else None,
)
return response, reasoning
- async def rate_limiter(
- self, model_config: models.ModelConfig, input: str, background: bool = False
+ async def rate_limiter_callback(
+ self, message:str, key:str, total:int, limit:int
):
- # rate limiter log
- wait_log = None
-
- async def wait_callback(msg: str, key: str, total: int, limit: int):
- nonlocal wait_log
- if not wait_log:
- wait_log = self.context.log.log(
- type="util",
- update_progress="none",
- heading=msg,
- model=f"{model_config.provider}\\{model_config.name}",
- )
- wait_log.update(heading=msg, key=key, value=total, limit=limit)
- if not background:
- self.context.log.set_progress(msg, -1)
-
- # rate limiter
- limiter = models.get_rate_limiter(
- model_config.provider,
- model_config.name,
- model_config.limit_requests,
- model_config.limit_input,
- model_config.limit_output,
- )
- limiter.add(input=tokens.approximate_tokens(input))
- limiter.add(requests=1)
- await limiter.wait(callback=wait_callback)
- return limiter
+ # show the rate limit waiting in a progress bar, no need to spam the chat history
+ self.context.log.set_progress(message, True)
+ return False
async def handle_intervention(self, progress: str = ""):
while self.context.paused:
diff --git a/models.py b/models.py
index f93cfbe46..25e2e5b00 100644
--- a/models.py
+++ b/models.py
@@ -108,7 +108,7 @@ def get_rate_limiter(
limiter.limits["output"] = output or 0
return limiter
-async def apply_rate_limiter(model_config: ModelConfig|None, input_text: str):
+async def apply_rate_limiter(model_config: ModelConfig|None, input_text: str, rate_limiter_callback: Callable[[str, str, int, int], Awaitable[bool]] | None = None):
if not model_config:
return
limiter = get_rate_limiter(
@@ -120,15 +120,15 @@ async def apply_rate_limiter(model_config: ModelConfig|None, input_text: str):
)
limiter.add(input=approximate_tokens(input_text))
limiter.add(requests=1)
- await limiter.wait()
+ await limiter.wait(rate_limiter_callback)
return limiter
-def apply_rate_limiter_sync(model_config: ModelConfig|None, input_text: str):
+def apply_rate_limiter_sync(model_config: ModelConfig|None, input_text: str, rate_limiter_callback: Callable[[str, str, int, int], Awaitable[bool]] | None = None):
if not model_config:
return
import asyncio, nest_asyncio
nest_asyncio.apply()
- return asyncio.run(apply_rate_limiter(model_config, input_text))
+ return asyncio.run(apply_rate_limiter(model_config, input_text, rate_limiter_callback))
class LiteLLMChatWrapper(SimpleChatModel):
@@ -286,6 +286,7 @@ class LiteLLMChatWrapper(SimpleChatModel):
response_callback: Callable[[str, str], Awaitable[None]] | None = None,
reasoning_callback: Callable[[str, str], Awaitable[None]] | None = None,
tokens_callback: Callable[[str, int], Awaitable[None]] | None = None,
+ rate_limiter_callback: Callable[[str, str, int, int], Awaitable[bool]] | None = None,
**kwargs: Any,
) -> Tuple[str, str]:
@@ -303,7 +304,7 @@ class LiteLLMChatWrapper(SimpleChatModel):
msgs_conv = self._convert_messages(messages)
# Apply rate limiting if configured
- limiter = await apply_rate_limiter(self.a0_model_conf, str(msgs_conv))
+ limiter = await apply_rate_limiter(self.a0_model_conf, str(msgs_conv), rate_limiter_callback)
# call model
_completion = await acompletion(
diff --git a/python/api/get_work_dir_files.py b/python/api/get_work_dir_files.py
index ed2fddb5f..1783e3d8a 100644
--- a/python/api/get_work_dir_files.py
+++ b/python/api/get_work_dir_files.py
@@ -1,8 +1,6 @@
from python.helpers.api import ApiHandler, Request, Response
from python.helpers.file_browser import FileBrowser
from python.helpers import runtime
-import python.api.get_work_dir_files as get_work_dir_files_module
-
class GetWorkDirFiles(ApiHandler):
@@ -21,7 +19,7 @@ class GetWorkDirFiles(ApiHandler):
# browser = FileBrowser()
# result = browser.get_files(current_path)
- result = await runtime.call_development_function(get_work_dir_files_module.get_files, current_path)
+ result = await runtime.call_development_function(get_files, current_path)
return {"data": result}
diff --git a/python/helpers/document_query.py b/python/helpers/document_query.py
index df7fae415..d60a86574 100644
--- a/python/helpers/document_query.py
+++ b/python/helpers/document_query.py
@@ -140,13 +140,7 @@ class DocumentQueryStore:
PrintStyle.error(f"No chunks created for document: {document_uri}")
return False, []
- # Apply rate limiter
try:
- docs_text = "".join(chunk.page_content for chunk in docs)
- await self.agent.rate_limiter(
- model_config=self.agent.config.embeddings_model, input=docs_text
- )
-
# Initialize vector db if not already initialized
if not self.vector_db:
self.vector_db = self.init_vector_db()
diff --git a/python/helpers/memory.py b/python/helpers/memory.py
index 68ad9454f..fcbcf302f 100644
--- a/python/helpers/memory.py
+++ b/python/helpers/memory.py
@@ -300,11 +300,6 @@ class Memory:
):
comparator = Memory._get_comparator(filter) if filter else None
- # rate limiter
- await self.agent.rate_limiter(
- model_config=self.agent.config.embeddings_model, input=query
- )
-
return await self.db.asearch(
query,
search_type="similarity_score_threshold",
@@ -376,12 +371,6 @@ class Memory:
if not doc.metadata.get("area", ""):
doc.metadata["area"] = Memory.Area.MAIN.value
- # rate limiter
- docs_txt = "".join(self.format_docs_plain(docs))
- await self.agent.rate_limiter(
- model_config=self.agent.config.embeddings_model, input=docs_txt
- )
-
await self.db.aadd_documents(documents=docs, ids=ids)
self._save_db() # persist
return ids
diff --git a/python/helpers/rate_limiter.py b/python/helpers/rate_limiter.py
index 48eda8bd5..efb57d35f 100644
--- a/python/helpers/rate_limiter.py
+++ b/python/helpers/rate_limiter.py
@@ -32,7 +32,7 @@ class RateLimiter:
async def wait(
self,
- callback: Callable[[str, str, int, int], Awaitable[None]] | None = None,
+ callback: Callable[[str, str, int, int], Awaitable[bool]] | None = None,
):
while True:
await self.cleanup()
@@ -46,8 +46,9 @@ class RateLimiter:
if total > limit:
if callback:
msg = f"Rate limit exceeded for {key} ({total}/{limit}), waiting..."
- await callback(msg, key, total, limit)
- should_wait = True
+ should_wait = not await callback(msg, key, total, limit)
+ else:
+ should_wait = True
break
if not should_wait:
diff --git a/python/helpers/vector_db.py b/python/helpers/vector_db.py
index c37a742b0..20254685b 100644
--- a/python/helpers/vector_db.py
+++ b/python/helpers/vector_db.py
@@ -77,11 +77,6 @@ class VectorDB:
):
comparator = get_comparator(filter) if filter else None
- # rate limiter
- await self.agent.rate_limiter(
- model_config=self.agent.config.embeddings_model, input=query
- )
-
return await self.db.asearch(
query,
search_type="similarity_score_threshold",
@@ -109,12 +104,6 @@ class VectorDB:
for doc, id in zip(docs, ids):
doc.metadata["id"] = id # add ids to documents metadata
- # rate limiter
- docs_txt = "".join(format_docs_plain(docs))
- await self.agent.rate_limiter(
- model_config=self.agent.config.embeddings_model, input=docs_txt
- )
-
self.db.add_documents(documents=docs, ids=ids)
return ids