Skip to content

Agent Zero AI framework

Optimize response stream update cadence

Alessandro75bbc1d71adf
diff --git a/agent.py b/agent.py
index 8dde01f76..906ceb740 100644
--- a/agent.py
+++ b/agent.py
@@ -1,4 +1,4 @@
-import asyncio, json, random, re, string, threading
+import asyncio, json, random, re, string, threading, time
from collections import OrderedDict
from dataclasses import dataclass, field
@@ -42,6 +42,10 @@ from helpers.llm_result import (
from helpers.litellm_transport import ResponsesTransport
from helpers.responses_tools import build_responses_function_tools, original_tool_name
+_RESPONSE_STREAM_UPDATE_CHARS = 128
+_RESPONSE_STREAM_UPDATE_SECONDS = 0.05
+
+
class AgentContextType(Enum):
USER = "user"
TASK = "task"
@@ -403,6 +407,9 @@ class Agent:
self.loop_data.iteration += 1
self.loop_data.params_temporary = {} # clear temporary params
last_response_stream_full = ""
+ last_response_stream_chars = 0
+ last_response_stream_at = time.monotonic()
+ response_stream_pending = False
# call message_loop_start extensions
await extension.call_extensions_async(
@@ -440,7 +447,8 @@ class Agent:
await self.handle_reasoning_stream(stream_data["full"])
async def stream_callback(chunk: str, full: str):
- nonlocal last_response_stream_full
+ nonlocal last_response_stream_full, last_response_stream_chars
+ nonlocal last_response_stream_at, response_stream_pending
await self.handle_intervention()
# output the agent response stream
if chunk == full:
@@ -455,6 +463,7 @@ class Agent:
pass
else:
await self.handle_response_stream(full)
+ response_stream_pending = False
return full.strip()
await extension.call_extensions_async(
@@ -466,9 +475,19 @@ class Agent:
# Stream masked chunk after extensions processed it
if stream_data.get("chunk"):
printer.stream(stream_data["chunk"])
- # Use the potentially modified full text for downstream processing
- await self.handle_response_stream(stream_data["full"])
last_response_stream_full = stream_data["full"]
+ response_stream_pending = True
+ now = time.monotonic()
+ if (
+ len(full) - last_response_stream_chars
+ >= _RESPONSE_STREAM_UPDATE_CHARS
+ or now - last_response_stream_at
+ >= _RESPONSE_STREAM_UPDATE_SECONDS
+ ):
+ await self.handle_response_stream(last_response_stream_full)
+ last_response_stream_chars = len(full)
+ last_response_stream_at = time.monotonic()
+ response_stream_pending = False
# call main LLM
llm_result = await self.call_chat_model_turn(
@@ -479,6 +498,9 @@ class Agent:
agent_response = llm_result.response
await self.handle_intervention(agent_response)
+ if response_stream_pending:
+ await self.handle_response_stream(last_response_stream_full)
+
# Notify extensions to finalize their stream filters
await extension.call_extensions_async(
"reasoning_stream_end", self, loop_data=self.loop_data
diff --git a/tests/test_response_stream_cadence.py b/tests/test_response_stream_cadence.py
new file mode 100644
index 000000000..ce9254a73
--- /dev/null
+++ b/tests/test_response_stream_cadence.py
@@ -0,0 +1,109 @@
+from types import SimpleNamespace
+
+import pytest
+
+import agent as agent_module
+from agent import Agent
+from helpers import extension
+from helpers.llm_result import LLMResult
+
+
+async def _run_monologue(monkeypatch, chunks, clock, *, advance=0.0):
+ agent = object.__new__(Agent)
+ agent.context = SimpleNamespace(streaming_agent=None, task=None)
+ agent.last_user_message = None
+ events = []
+ provider_callbacks = 0
+
+ async def call_extensions(name, _agent=None, **kwargs):
+ events.append(("extension", name))
+ if name == "response_stream_chunk":
+ stream_data = kwargs["stream_data"]
+ stream_data["full"] = "masked:" + stream_data["full"]
+
+ async def call_model(*, response_callback, **_kwargs):
+ nonlocal provider_callbacks
+ full = ""
+ for chunk in chunks:
+ provider_callbacks += 1
+ clock.now += advance
+ full += chunk
+ stopped = await response_callback(chunk, full)
+ if stopped:
+ return LLMResult(response=stopped)
+ return LLMResult(response=full)
+
+ async def handle_response_stream(full):
+ events.append(("handle", full))
+
+ async def done(_result):
+ return "done"
+
+ async def no_intervention(*_args):
+ return None
+
+ async def no_prompt(**_kwargs):
+ return []
+
+ monkeypatch.setattr(extension, "call_extensions_async", call_extensions)
+ monkeypatch.setattr(agent_module.time, "monotonic", lambda: clock.now)
+ agent.prepare_prompt = no_prompt
+ agent.handle_intervention = no_intervention
+ agent.call_chat_model_turn = call_model
+ agent.handle_response_stream = handle_response_stream
+ agent.hist_add_ai_response = lambda *_args, **_kwargs: SimpleNamespace(id="")
+ agent._remember_llm_result_state = lambda *_args: None
+ agent.process_llm_result_tools = done
+
+ result = await Agent.monologue.__wrapped__(agent)
+ return result, events, provider_callbacks
+
+
+async def test_response_stream_coalesces_fast_fragments_and_flushes_final(monkeypatch):
+ clock = SimpleNamespace(now=0.0)
+ result, events, provider_callbacks = await _run_monologue(
+ monkeypatch, list("x" * 130), clock
+ )
+
+ handled = [value for kind, value in events if kind == "handle"]
+ chunk_hooks = [
+ event
+ for event in events
+ if event == ("extension", "response_stream_chunk")
+ ]
+
+ assert result == "done"
+ assert provider_callbacks == len(chunk_hooks) == 130
+ assert handled == ["masked:" + "x" * size for size in (128, 130)]
+ assert events.index(("handle", handled[-1])) < events.index(
+ ("extension", "response_stream_end")
+ )
+ assert events.index(("extension", "response_stream_end")) < events.index(
+ ("extension", "message_loop_result")
+ )
+
+
+async def test_response_stream_time_bound_keeps_slow_fragments_live(monkeypatch):
+ clock = SimpleNamespace(now=0.0)
+ _, events, _ = await _run_monologue(
+ monkeypatch, ["a" * 10] * 3, clock, advance=0.03
+ )
+
+ handled = [value for kind, value in events if kind == "handle"]
+ assert handled == ["masked:" + "a" * size for size in (20, 30)]
+
+
+async def test_response_stream_still_stops_on_exact_canonical_root(monkeypatch):
+ clock = SimpleNamespace(now=0.0)
+ message = '{"tool_name":"response","tool_args":{"text":"ok"}}'
+ result, events, provider_callbacks = await _run_monologue(
+ monkeypatch, [message[:-1], message[-1], " unreachable"], clock
+ )
+
+ assert result == "done"
+ assert provider_callbacks == 2
+ assert [value for kind, value in events if kind == "handle"] == [message]
+ assert events.count(("extension", "response_stream_chunk")) == 1