Agent Zero AI framework
Optimize response stream update cadence
Alessandro75bbc1d71adf
diff --git a/agent.py b/agent.pyindex 8dde01f76..906ceb740 100644--- a/agent.py+++ b/agent.py@@ -1,4 +1,4 @@-import asyncio, json, random, re, string, threading+import asyncio, json, random, re, string, threading, timefrom collections import OrderedDictfrom dataclasses import dataclass, field@@ -42,6 +42,10 @@ from helpers.llm_result import (from helpers.litellm_transport import ResponsesTransportfrom helpers.responses_tools import build_responses_function_tools, original_tool_name+_RESPONSE_STREAM_UPDATE_CHARS = 128+_RESPONSE_STREAM_UPDATE_SECONDS = 0.05++class AgentContextType(Enum):USER = "user"TASK = "task"@@ -403,6 +407,9 @@ class Agent:self.loop_data.iteration += 1self.loop_data.params_temporary = {} # clear temporary paramslast_response_stream_full = ""+ last_response_stream_chars = 0+ last_response_stream_at = time.monotonic()+ response_stream_pending = False# call message_loop_start extensionsawait extension.call_extensions_async(@@ -440,7 +447,8 @@ class Agent:await self.handle_reasoning_stream(stream_data["full"])async def stream_callback(chunk: str, full: str):- nonlocal last_response_stream_full+ nonlocal last_response_stream_full, last_response_stream_chars+ nonlocal last_response_stream_at, response_stream_pendingawait self.handle_intervention()# output the agent response streamif chunk == full:@@ -455,6 +463,7 @@ class Agent:passelse:await self.handle_response_stream(full)+ response_stream_pending = Falsereturn full.strip()await extension.call_extensions_async(@@ -466,9 +475,19 @@ class Agent:# Stream masked chunk after extensions processed itif stream_data.get("chunk"):printer.stream(stream_data["chunk"])- # Use the potentially modified full text for downstream processing- await self.handle_response_stream(stream_data["full"])last_response_stream_full = stream_data["full"]+ response_stream_pending = True+ now = time.monotonic()+ if (+ len(full) - last_response_stream_chars+ >= _RESPONSE_STREAM_UPDATE_CHARS+ or now - last_response_stream_at+ >= _RESPONSE_STREAM_UPDATE_SECONDS+ ):+ await self.handle_response_stream(last_response_stream_full)+ last_response_stream_chars = len(full)+ last_response_stream_at = time.monotonic()+ response_stream_pending = False# call main LLMllm_result = await self.call_chat_model_turn(@@ -479,6 +498,9 @@ class Agent:agent_response = llm_result.responseawait self.handle_intervention(agent_response)+ if response_stream_pending:+ await self.handle_response_stream(last_response_stream_full)+# Notify extensions to finalize their stream filtersawait extension.call_extensions_async("reasoning_stream_end", self, loop_data=self.loop_datadiff --git a/tests/test_response_stream_cadence.py b/tests/test_response_stream_cadence.pynew file mode 100644index 000000000..ce9254a73--- /dev/null+++ b/tests/test_response_stream_cadence.py@@ -0,0 +1,109 @@+from types import SimpleNamespace++import pytest++import agent as agent_module+from agent import Agent+from helpers import extension+from helpers.llm_result import LLMResult+++async def _run_monologue(monkeypatch, chunks, clock, *, advance=0.0):+ agent = object.__new__(Agent)+ agent.context = SimpleNamespace(streaming_agent=None, task=None)+ agent.last_user_message = None+ events = []+ provider_callbacks = 0++ async def call_extensions(name, _agent=None, **kwargs):+ events.append(("extension", name))+ if name == "response_stream_chunk":+ stream_data = kwargs["stream_data"]+ stream_data["full"] = "masked:" + stream_data["full"]++ async def call_model(*, response_callback, **_kwargs):+ nonlocal provider_callbacks+ full = ""+ for chunk in chunks:+ provider_callbacks += 1+ clock.now += advance+ full += chunk+ stopped = await response_callback(chunk, full)+ if stopped:+ return LLMResult(response=stopped)+ return LLMResult(response=full)++ async def handle_response_stream(full):+ events.append(("handle", full))++ async def done(_result):+ return "done"++ async def no_intervention(*_args):+ return None++ async def no_prompt(**_kwargs):+ return []++ monkeypatch.setattr(extension, "call_extensions_async", call_extensions)+ monkeypatch.setattr(agent_module.time, "monotonic", lambda: clock.now)+ agent.prepare_prompt = no_prompt+ agent.handle_intervention = no_intervention+ agent.call_chat_model_turn = call_model+ agent.handle_response_stream = handle_response_stream+ agent.hist_add_ai_response = lambda *_args, **_kwargs: SimpleNamespace(id="")+ agent._remember_llm_result_state = lambda *_args: None+ agent.process_llm_result_tools = done++ result = await Agent.monologue.__wrapped__(agent)+ return result, events, provider_callbacks+++async def test_response_stream_coalesces_fast_fragments_and_flushes_final(monkeypatch):+ clock = SimpleNamespace(now=0.0)+ result, events, provider_callbacks = await _run_monologue(+ monkeypatch, list("x" * 130), clock+ )++ handled = [value for kind, value in events if kind == "handle"]+ chunk_hooks = [+ event+ for event in events+ if event == ("extension", "response_stream_chunk")+ ]++ assert result == "done"+ assert provider_callbacks == len(chunk_hooks) == 130+ assert handled == ["masked:" + "x" * size for size in (128, 130)]+ assert events.index(("handle", handled[-1])) < events.index(+ ("extension", "response_stream_end")+ )+ assert events.index(("extension", "response_stream_end")) < events.index(+ ("extension", "message_loop_result")+ )+++async def test_response_stream_time_bound_keeps_slow_fragments_live(monkeypatch):+ clock = SimpleNamespace(now=0.0)+ _, events, _ = await _run_monologue(+ monkeypatch, ["a" * 10] * 3, clock, advance=0.03+ )++ handled = [value for kind, value in events if kind == "handle"]+ assert handled == ["masked:" + "a" * size for size in (20, 30)]+++async def test_response_stream_still_stops_on_exact_canonical_root(monkeypatch):+ clock = SimpleNamespace(now=0.0)+ message = '{"tool_name":"response","tool_args":{"text":"ok"}}'+ result, events, provider_callbacks = await _run_monologue(+ monkeypatch, [message[:-1], message[-1], " unreachable"], clock+ )++ assert result == "done"+ assert provider_callbacks == 2+ assert [value for kind, value in events if kind == "handle"] == [message]+ assert events.count(("extension", "response_stream_chunk")) == 1