Agent Zero AI framework
rate limiter polishing, file download polishing
frdel765470796304
diff --git a/agent.py b/agent.pyindex 2a4ae73d7..3e2d30ed2 100644--- a/agent.py+++ b/agent.py@@ -613,15 +613,6 @@ class Agent:):model = self.get_utility_model()- # rate limiter- limiter = await self.rate_limiter(- self.config.utility_model, f"SYSTEM: {system}\nUSER: {message}", background- )-- # add output tokens to rate limiter in tokens callback- async def tokens_callback(delta: str, tokens: int):- await self.handle_intervention()- limiter.add(output=tokens)# propagate stream to callback if setasync def stream_callback(chunk: str, total: str):@@ -632,7 +623,7 @@ class Agent:system_message=system,user_message=message,response_callback=stream_callback,- tokens_callback=tokens_callback,+ rate_limiter_callback=self.rate_limiter_callback if not background else None,)return response@@ -642,63 +633,29 @@ class Agent:messages: list[BaseMessage],response_callback: Callable[[str, str], Awaitable[None]] | None = None,reasoning_callback: Callable[[str, str], Awaitable[None]] | None = None,+ background: bool = False,):response = ""# model classmodel = self.get_chat_model()- # rate limiter- limiter = await self.rate_limiter(- self.config.chat_model, ChatPromptTemplate.from_messages(messages).format()- )-- # add output tokens to rate limiter in tokens callback- async def tokens_callback(delta: str, tokens: int):- await self.handle_intervention()- limiter.add(output=tokens)-# call modelresponse, reasoning = await model.unified_call(messages=messages,reasoning_callback=reasoning_callback,response_callback=response_callback,- tokens_callback=tokens_callback,+ rate_limiter_callback=self.rate_limiter_callback if not background else None,)return response, reasoning- async def rate_limiter(- self, model_config: models.ModelConfig, input: str, background: bool = False+ async def rate_limiter_callback(+ self, message:str, key:str, total:int, limit:int):- # rate limiter log- wait_log = None-- async def wait_callback(msg: str, key: str, total: int, limit: int):- nonlocal wait_log- if not wait_log:- wait_log = self.context.log.log(- type="util",- update_progress="none",- heading=msg,- model=f"{model_config.provider}\\{model_config.name}",- )- wait_log.update(heading=msg, key=key, value=total, limit=limit)- if not background:- self.context.log.set_progress(msg, -1)-- # rate limiter- limiter = models.get_rate_limiter(- model_config.provider,- model_config.name,- model_config.limit_requests,- model_config.limit_input,- model_config.limit_output,- )- limiter.add(input=tokens.approximate_tokens(input))- limiter.add(requests=1)- await limiter.wait(callback=wait_callback)- return limiter+ # show the rate limit waiting in a progress bar, no need to spam the chat history+ self.context.log.set_progress(message, True)+ return Falseasync def handle_intervention(self, progress: str = ""):while self.context.paused:diff --git a/models.py b/models.pyindex f93cfbe46..25e2e5b00 100644--- a/models.py+++ b/models.py@@ -108,7 +108,7 @@ def get_rate_limiter(limiter.limits["output"] = output or 0return limiter-async def apply_rate_limiter(model_config: ModelConfig|None, input_text: str):+async def apply_rate_limiter(model_config: ModelConfig|None, input_text: str, rate_limiter_callback: Callable[[str, str, int, int], Awaitable[bool]] | None = None):if not model_config:returnlimiter = get_rate_limiter(@@ -120,15 +120,15 @@ async def apply_rate_limiter(model_config: ModelConfig|None, input_text: str):)limiter.add(input=approximate_tokens(input_text))limiter.add(requests=1)- await limiter.wait()+ await limiter.wait(rate_limiter_callback)return limiter-def apply_rate_limiter_sync(model_config: ModelConfig|None, input_text: str):+def apply_rate_limiter_sync(model_config: ModelConfig|None, input_text: str, rate_limiter_callback: Callable[[str, str, int, int], Awaitable[bool]] | None = None):if not model_config:returnimport asyncio, nest_asyncionest_asyncio.apply()- return asyncio.run(apply_rate_limiter(model_config, input_text))+ return asyncio.run(apply_rate_limiter(model_config, input_text, rate_limiter_callback))class LiteLLMChatWrapper(SimpleChatModel):@@ -286,6 +286,7 @@ class LiteLLMChatWrapper(SimpleChatModel):response_callback: Callable[[str, str], Awaitable[None]] | None = None,reasoning_callback: Callable[[str, str], Awaitable[None]] | None = None,tokens_callback: Callable[[str, int], Awaitable[None]] | None = None,+ rate_limiter_callback: Callable[[str, str, int, int], Awaitable[bool]] | None = None,**kwargs: Any,) -> Tuple[str, str]:@@ -303,7 +304,7 @@ class LiteLLMChatWrapper(SimpleChatModel):msgs_conv = self._convert_messages(messages)# Apply rate limiting if configured- limiter = await apply_rate_limiter(self.a0_model_conf, str(msgs_conv))+ limiter = await apply_rate_limiter(self.a0_model_conf, str(msgs_conv), rate_limiter_callback)# call model_completion = await acompletion(diff --git a/python/api/get_work_dir_files.py b/python/api/get_work_dir_files.pyindex ed2fddb5f..1783e3d8a 100644--- a/python/api/get_work_dir_files.py+++ b/python/api/get_work_dir_files.py@@ -1,8 +1,6 @@from python.helpers.api import ApiHandler, Request, Responsefrom python.helpers.file_browser import FileBrowserfrom python.helpers import runtime-import python.api.get_work_dir_files as get_work_dir_files_module-class GetWorkDirFiles(ApiHandler):@@ -21,7 +19,7 @@ class GetWorkDirFiles(ApiHandler):# browser = FileBrowser()# result = browser.get_files(current_path)- result = await runtime.call_development_function(get_work_dir_files_module.get_files, current_path)+ result = await runtime.call_development_function(get_files, current_path)return {"data": result}diff --git a/python/helpers/document_query.py b/python/helpers/document_query.pyindex df7fae415..d60a86574 100644--- a/python/helpers/document_query.py+++ b/python/helpers/document_query.py@@ -140,13 +140,7 @@ class DocumentQueryStore:PrintStyle.error(f"No chunks created for document: {document_uri}")return False, []- # Apply rate limitertry:- docs_text = "".join(chunk.page_content for chunk in docs)- await self.agent.rate_limiter(- model_config=self.agent.config.embeddings_model, input=docs_text- )-# Initialize vector db if not already initializedif not self.vector_db:self.vector_db = self.init_vector_db()diff --git a/python/helpers/memory.py b/python/helpers/memory.pyindex 68ad9454f..fcbcf302f 100644--- a/python/helpers/memory.py+++ b/python/helpers/memory.py@@ -300,11 +300,6 @@ class Memory:):comparator = Memory._get_comparator(filter) if filter else None- # rate limiter- await self.agent.rate_limiter(- model_config=self.agent.config.embeddings_model, input=query- )-return await self.db.asearch(query,search_type="similarity_score_threshold",@@ -376,12 +371,6 @@ class Memory:if not doc.metadata.get("area", ""):doc.metadata["area"] = Memory.Area.MAIN.value- # rate limiter- docs_txt = "".join(self.format_docs_plain(docs))- await self.agent.rate_limiter(- model_config=self.agent.config.embeddings_model, input=docs_txt- )-await self.db.aadd_documents(documents=docs, ids=ids)self._save_db() # persistreturn idsdiff --git a/python/helpers/rate_limiter.py b/python/helpers/rate_limiter.pyindex 48eda8bd5..efb57d35f 100644--- a/python/helpers/rate_limiter.py+++ b/python/helpers/rate_limiter.py@@ -32,7 +32,7 @@ class RateLimiter:async def wait(self,- callback: Callable[[str, str, int, int], Awaitable[None]] | None = None,+ callback: Callable[[str, str, int, int], Awaitable[bool]] | None = None,):while True:await self.cleanup()@@ -46,8 +46,9 @@ class RateLimiter:if total > limit:if callback:msg = f"Rate limit exceeded for {key} ({total}/{limit}), waiting..."- await callback(msg, key, total, limit)- should_wait = True+ should_wait = not await callback(msg, key, total, limit)+ else:+ should_wait = Truebreakif not should_wait:diff --git a/python/helpers/vector_db.py b/python/helpers/vector_db.pyindex c37a742b0..20254685b 100644--- a/python/helpers/vector_db.py+++ b/python/helpers/vector_db.py@@ -77,11 +77,6 @@ class VectorDB:):comparator = get_comparator(filter) if filter else None- # rate limiter- await self.agent.rate_limiter(- model_config=self.agent.config.embeddings_model, input=query- )-return await self.db.asearch(query,search_type="similarity_score_threshold",@@ -109,12 +104,6 @@ class VectorDB:for doc, id in zip(docs, ids):doc.metadata["id"] = id # add ids to documents metadata- # rate limiter- docs_txt = "".join(format_docs_plain(docs))- await self.agent.rate_limiter(- model_config=self.agent.config.embeddings_model, input=docs_txt- )-self.db.add_documents(documents=docs, ids=ids)return ids