Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
feat(agent): add max_image_num to ContextConfig to limit images in context (#2362)
DavdGao3ff581989df7
diff --git a/src/agentscope/agent/_agent.py b/src/agentscope/agent/_agent.pyindex 962d42f03..b98333bb5 100644--- a/src/agentscope/agent/_agent.py+++ b/src/agentscope/agent/_agent.py@@ -426,6 +426,10 @@ class Agent:"""cfg: ContextConfig = context_config or self.context_config+ # Limit the number of images in the context first, so that the token+ # counting below reflects the images that actually remain+ await self._limit_context_images(cfg)+# Count the current tokenskwargs = await self._prepare_model_input()estimated_tokens = await self.model.count_tokens(**kwargs)@@ -665,6 +669,110 @@ class Agent:await apply_taskraise+ async def _limit_context_images(self, cfg: ContextConfig) -> None:+ """Limit the number of images in the context according to+ ``cfg.max_image_num``. The oldest images exceeding the limit are+ offloaded to the workspace (if an offloader is provided) and replaced+ by a hint recording the offloaded path; otherwise they are dropped and+ replaced by a hint without path information.++ Image data blocks at the top level of a non-user message are+ replaced by :class:`HintBlock`, while those in user messages or+ nested inside a :class:`ToolResultBlock` / :class:`HintBlock` are+ replaced by :class:`TextBlock` (as required by their type+ constraints).++ Args:+ cfg (`ContextConfig`):+ The context config that provides ``max_image_num``.+ """+ max_image_num = cfg.max_image_num++ def _is_image(block: Any) -> bool:+ """Check whether the given block is an image data block."""+ return isinstance(+ block,+ DataBlock,+ ) and block.source.media_type.startswith("image/")++ # Collect all the image data blocks in chronological order, recorded+ # as (container list, index, block, is_top_level, role)+ images: list[tuple[list, int, DataBlock, bool, str]] = []+ for msg in self.state.context:+ for i, block in enumerate(msg.content):+ if _is_image(block):+ images.append((msg.content, i, block, True, msg.role))+ elif isinstance(block, ToolResultBlock) and isinstance(+ block.output,+ list,+ ):+ for j, sub in enumerate(block.output):+ if _is_image(sub):+ images.append(+ (block.output, j, sub, False, msg.role),+ )+ elif isinstance(block, HintBlock) and isinstance(+ block.hint,+ list,+ ):+ for j, sub in enumerate(block.hint):+ if _is_image(sub):+ images.append(+ (block.hint, j, sub, False, msg.role),+ )++ n_exceed = len(images) - max_image_num+ if n_exceed <= 0:+ return++ logger.info(+ "[AGENT %s]: The number of images in context (%d) exceeds the "+ "limit (%d), removing the oldest %d image(s).",+ self.name,+ len(images),+ max_image_num,+ n_exceed,+ )++ # Offload the images first (which may involve I/O), and then apply+ # the replacements together to avoid partial modification on+ # interruption+ replacements: list[tuple[list, int, HintBlock | TextBlock]] = []+ for container, idx, block, is_top_level, role in images[:n_exceed]:+ url = ""+ if isinstance(block.source, URLSource):+ url = str(block.source.url)+ elif self.offloader is not None:+ saved = await self.offloader.offload_data_block(block)+ if isinstance(saved.source, URLSource):+ url = str(saved.source.url)++ name = f"named '{block.name}' " if block.name else ""+ if url:+ text = (+ f"<system-reminder>The image {name}is offloaded into "+ f"{url}, you can refer to it when needed."+ f"</system-reminder>"+ )+ else:+ text = (+ f"<system-reminder>The image {name}is removed to free "+ f"up context space.</system-reminder>"+ )++ new_block: HintBlock | TextBlock+ if is_top_level and role != "user":+ new_block = HintBlock(hint=text)+ else:+ # User messages only accept text/data blocks (see the+ # validator in Msg), and a hint block is equivalent to a user+ # text block for the model anyway.+ new_block = TextBlock(type="text", text=text)+ replacements.append((container, idx, new_block))++ for container, idx, new_block in replacements:+ container[idx] = new_block+# ======================================================================# Agent core methods, including _reply, _reasoning, _acting, etc.# ======================================================================diff --git a/src/agentscope/agent/_config.py b/src/agentscope/agent/_config.pyindex 75ce585da..519f28bed 100644--- a/src/agentscope/agent/_config.py+++ b/src/agentscope/agent/_config.py@@ -138,6 +138,21 @@ class ContextConfig(BaseModel):)"""The tool result limit to avoid tool result bursting."""+ max_image_num: int = Field(+ title="Max Image Number",+ default=5,+ ge=0,+ description=(+ "The maximum number of images kept in the context. The oldest "+ "images exceeding the limit will be removed."+ ),+ )+ """The maximum number of images kept in the context. When the number of+ images exceeds this limit, the oldest images will be offloaded to the+ workspace (if an offloader is provided) and replaced by a hint that+ records the offloaded path; otherwise they are dropped and replaced by a+ hint without path information."""+class InjectionConfig(BaseModel):"""The state injection related configuration in AgentScope."""diff --git a/tests/compress_context_test.py b/tests/compress_context_test.pyindex 945bc843c..314f0ba29 100644--- a/tests/compress_context_test.py+++ b/tests/compress_context_test.py@@ -1,6 +1,7 @@# -*- coding: utf-8 -*-"""A template test case."""# pylint: disable=protected-access+import hashlibimport jsonimport osimport tempfile@@ -22,8 +23,86 @@ from agentscope.message import (ToolResultState,HintBlock,Msg,+ DataBlock,+ Base64Source,+ URLSource,)from agentscope.tool import Toolkit+from agentscope.workspace import LocalWorkspace+++# A 1x1 transparent PNG image+_PNG_BASE64 = (+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB"+ "0C8AAAAASUVORK5CYII="+)+++def _image_block(name: str) -> DataBlock:+ """Create a base64 image data block."""+ return DataBlock(+ source=Base64Source(data=_PNG_BASE64, media_type="image/png"),+ name=name,+ )+++def _build_image_context() -> list[Msg]:+ """Build a context with 6 images located at the message top level (user+ and assistant), inside a tool result and inside a hint block, plus one+ audio block."""+ return [+ UserMsg(+ "User",+ [_image_block("img1"), TextBlock(type="text", text="hello")],+ id="1",+ ),+ AssistantMsg(+ "Friday",+ [+ ToolCallBlock(+ type="tool_call",+ id="call_1",+ name="view",+ input="{}",+ ),+ ToolResultBlock(+ type="tool_result",+ id="call_1",+ name="view",+ output=[+ TextBlock(type="text", text="the image:"),+ _image_block("img2"),+ ],+ state=ToolResultState.SUCCESS,+ ),+ HintBlock(+ hint=[+ TextBlock(type="text", text="a hint image:"),+ _image_block("img3"),+ ],+ ),+ _image_block("img4"),+ ],+ id="2",+ ),+ UserMsg(+ "User",+ [+ DataBlock(+ source=Base64Source(data="AAAA", media_type="audio/wav"),+ ),+ DataBlock(+ source=URLSource(+ url="https://example.com/img5.png",+ media_type="image/png",+ ),+ name="img5",+ ),+ _image_block("img6"),+ ],+ id="3",+ ),+ ]class RecordingStructuredMockModel(MockModel):@@ -1441,6 +1520,623 @@ class ContextCompressionTest(IsolatedAsyncioTestCase):),)+ async def test_max_image_num_without_offloader(self) -> None:+ """The oldest images exceeding the limit are dropped and replaced by+ hints without path information when no offloader is provided."""+ agent = Agent(+ name="Friday",+ system_prompt="You're a helpful assistant.",+ model=MockModel(context_size=100000),+ context_config=ContextConfig(max_image_num=2),+ state=AgentState(session_id="123", context=_build_image_context()),+ toolkit=Toolkit(),+ )++ # The token count is far below the trigger threshold, so only the+ # image limitation takes effect+ await agent.compress_context()++ expected = [+ {+ "name": "User",+ "content": [+ {+ "type": "text",+ "text": (+ "<system-reminder>The image named 'img1' is "+ "removed to free up context "+ "space.</system-reminder>"+ ),+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": None,+ },+ {+ "type": "text",+ "text": "hello",+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": None,+ },+ ],+ "role": "user",+ "id": "1",+ "metadata": {},+ "created_at": AnyString(),+ "usage": None,+ "finished_at": AnyString(),+ "finished_reason": None,+ "structured_output": None,+ "error": None,+ },+ {+ "name": "Friday",+ "content": [+ {+ "type": "tool_call",+ "id": "call_1",+ "name": "view",+ "input": "{}",+ "state": "pending",+ "suggested_rules": [],+ "created_at": AnyString(),+ "finished_at": None,+ },+ {+ "type": "tool_result",+ "id": "call_1",+ "name": "view",+ "output": [+ {+ "type": "text",+ "text": "the image:",+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": None,+ },+ {+ "type": "text",+ "text": (+ "<system-reminder>The image named 'img2' "+ "is removed to free up context "+ "space.</system-reminder>"+ ),+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": None,+ },+ ],+ "state": "success",+ "metadata": {},+ "created_at": AnyString(),+ "finished_at": None,+ },+ {+ "type": "hint",+ "hint": [+ {+ "type": "text",+ "text": "a hint image:",+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": None,+ },+ {+ "type": "text",+ "text": (+ "<system-reminder>The image named 'img3' "+ "is removed to free up context "+ "space.</system-reminder>"+ ),+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": None,+ },+ ],+ "id": AnyString(),+ "source": None,+ "created_at": AnyString(),+ "finished_at": AnyString(),+ },+ {+ "type": "hint",+ "hint": (+ "<system-reminder>The image named 'img4' is "+ "removed to free up context "+ "space.</system-reminder>"+ ),+ "id": AnyString(),+ "source": None,+ "created_at": AnyString(),+ "finished_at": AnyString(),+ },+ ],+ "role": "assistant",+ "id": "2",+ "metadata": {},+ "created_at": AnyString(),+ "usage": None,+ "finished_at": None,+ "finished_reason": None,+ "structured_output": None,+ "error": None,+ },+ {+ "name": "User",+ "content": [+ {+ "type": "data",+ "id": AnyString(),+ "source": {+ "type": "base64",+ "data": "AAAA",+ "media_type": "audio/wav",+ },+ "name": None,+ "created_at": AnyString(),+ "finished_at": None,+ },+ {+ "type": "data",+ "id": AnyString(),+ "source": {+ "type": "url",+ "url": "https://example.com/img5.png",+ "media_type": "image/png",+ },+ "name": "img5",+ "created_at": AnyString(),+ "finished_at": None,+ },+ {+ "type": "data",+ "id": AnyString(),+ "source": {+ "type": "base64",+ "data": _PNG_BASE64,+ "media_type": "image/png",+ },+ "name": "img6",+ "created_at": AnyString(),+ "finished_at": None,+ },+ ],+ "role": "user",+ "id": "3",+ "metadata": {},+ "created_at": AnyString(),+ "usage": None,+ "finished_at": AnyString(),+ "finished_reason": None,+ "structured_output": None,+ "error": None,+ },+ ]+ self.assertListEqual(+ [_.model_dump() for _ in agent.state.context],+ expected,+ )++ # Calling again should be a no-op+ await agent.compress_context()+ self.assertListEqual(+ [_.model_dump() for _ in agent.state.context],+ expected,+ )++ async def test_max_image_num_with_offloader(self) -> None:+ """The oldest images exceeding the limit are offloaded and replaced+ by hints recording the offloaded path when an offloader is+ provided; URL images keep their original URL."""+ with tempfile.TemporaryDirectory() as workdir:+ agent = Agent(+ name="Friday",+ system_prompt="You're a helpful assistant.",+ model=MockModel(context_size=100000),+ context_config=ContextConfig(max_image_num=1),+ state=AgentState(+ session_id="123",+ context=_build_image_context(),+ ),+ toolkit=Toolkit(),+ offloader=LocalWorkspace(workdir=workdir),+ )++ await agent.compress_context()++ # All the base64 images share the same content, thus the same+ # offloaded path+ rel = "data/" + hashlib.sha256(_PNG_BASE64.encode()).hexdigest()+ url = f"workspace:///{rel}.png"+ self.assertTrue(+ os.path.exists(os.path.join(workdir, rel + ".png")),+ )++ self.assertListEqual(+ [_.model_dump() for _ in agent.state.context],+ [+ {+ "name": "User",+ "content": [+ {+ "type": "text",+ "text": (+ "<system-reminder>The image named 'img1' "+ "is offloaded into "+ + url+ + ", you can refer to it when "+ "needed.</system-reminder>"+ ),+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": None,+ },+ {+ "type": "text",+ "text": "hello",+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": None,+ },+ ],+ "role": "user",+ "id": "1",+ "metadata": {},+ "created_at": AnyString(),+ "usage": None,+ "finished_at": AnyString(),+ "finished_reason": None,+ "structured_output": None,+ "error": None,+ },+ {+ "name": "Friday",+ "content": [+ {+ "type": "tool_call",+ "id": "call_1",+ "name": "view",+ "input": "{}",+ "state": "pending",+ "suggested_rules": [],+ "created_at": AnyString(),+ "finished_at": None,+ },+ {+ "type": "tool_result",+ "id": "call_1",+ "name": "view",+ "output": [+ {+ "type": "text",+ "text": "the image:",+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": None,+ },+ {+ "type": "text",+ "text": (+ "<system-reminder>The image named "+ "'img2' is offloaded into "+ + url+ + ", you can refer to it when "+ "needed.</system-reminder>"+ ),+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": None,+ },+ ],+ "state": "success",+ "metadata": {},+ "created_at": AnyString(),+ "finished_at": None,+ },+ {+ "type": "hint",+ "hint": [+ {+ "type": "text",+ "text": "a hint image:",+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": None,+ },+ {+ "type": "text",+ "text": (+ "<system-reminder>The image named "+ "'img3' is offloaded into "+ + url+ + ", you can refer to it when "+ "needed.</system-reminder>"+ ),+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": None,+ },+ ],+ "id": AnyString(),+ "source": None,+ "created_at": AnyString(),+ "finished_at": AnyString(),+ },+ {+ "type": "hint",+ "hint": (+ "<system-reminder>The image named 'img4' "+ "is offloaded into "+ + url+ + ", you can refer to it when "+ "needed.</system-reminder>"+ ),+ "id": AnyString(),+ "source": None,+ "created_at": AnyString(),+ "finished_at": AnyString(),+ },+ ],+ "role": "assistant",+ "id": "2",+ "metadata": {},+ "created_at": AnyString(),+ "usage": None,+ "finished_at": None,+ "finished_reason": None,+ "structured_output": None,+ "error": None,+ },+ {+ "name": "User",+ "content": [+ {+ "type": "data",+ "id": AnyString(),+ "source": {+ "type": "base64",+ "data": "AAAA",+ "media_type": "audio/wav",+ },+ "name": None,+ "created_at": AnyString(),+ "finished_at": None,+ },+ {+ "type": "text",+ "text": (+ "<system-reminder>The image named 'img5' "+ "is offloaded into "+ "https://example.com/img5.png, you can "+ "refer to it when needed."+ "</system-reminder>"+ ),+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": None,+ },+ {+ "type": "data",+ "id": AnyString(),+ "source": {+ "type": "base64",+ "data": _PNG_BASE64,+ "media_type": "image/png",+ },+ "name": "img6",+ "created_at": AnyString(),+ "finished_at": None,+ },+ ],+ "role": "user",+ "id": "3",+ "metadata": {},+ "created_at": AnyString(),+ "usage": None,+ "finished_at": AnyString(),+ "finished_reason": None,+ "structured_output": None,+ "error": None,+ },+ ],+ )++ async def test_max_image_num_default(self) -> None:+ """The default limit is 5, so only the oldest image is removed for a+ context with 6 images."""+ agent = Agent(+ name="Friday",+ system_prompt="You're a helpful assistant.",+ model=MockModel(context_size=100000),+ state=AgentState(session_id="123", context=_build_image_context()),+ toolkit=Toolkit(),+ )+ self.assertEqual(agent.context_config.max_image_num, 5)++ await agent.compress_context()++ self.assertListEqual(+ [_.model_dump() for _ in agent.state.context],+ [+ {+ "name": "User",+ "content": [+ {+ "type": "text",+ "text": (+ "<system-reminder>The image named 'img1' is "+ "removed to free up context "+ "space.</system-reminder>"+ ),+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": None,+ },+ {+ "type": "text",+ "text": "hello",+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": None,+ },+ ],+ "role": "user",+ "id": "1",+ "metadata": {},+ "created_at": AnyString(),+ "usage": None,+ "finished_at": AnyString(),+ "finished_reason": None,+ "structured_output": None,+ "error": None,+ },+ {+ "name": "Friday",+ "content": [+ {+ "type": "tool_call",+ "id": "call_1",+ "name": "view",+ "input": "{}",+ "state": "pending",+ "suggested_rules": [],+ "created_at": AnyString(),+ "finished_at": None,+ },+ {+ "type": "tool_result",+ "id": "call_1",+ "name": "view",+ "output": [+ {+ "type": "text",+ "text": "the image:",+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": None,+ },+ {+ "type": "data",+ "id": AnyString(),+ "source": {+ "type": "base64",+ "data": _PNG_BASE64,+ "media_type": "image/png",+ },+ "name": "img2",+ "created_at": AnyString(),+ "finished_at": None,+ },+ ],+ "state": "success",+ "metadata": {},+ "created_at": AnyString(),+ "finished_at": None,+ },+ {+ "type": "hint",+ "hint": [+ {+ "type": "text",+ "text": "a hint image:",+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": None,+ },+ {+ "type": "data",+ "id": AnyString(),+ "source": {+ "type": "base64",+ "data": _PNG_BASE64,+ "media_type": "image/png",+ },+ "name": "img3",+ "created_at": AnyString(),+ "finished_at": None,+ },+ ],+ "id": AnyString(),+ "source": None,+ "created_at": AnyString(),+ "finished_at": AnyString(),+ },+ {+ "type": "data",+ "id": AnyString(),+ "source": {+ "type": "base64",+ "data": _PNG_BASE64,+ "media_type": "image/png",+ },+ "name": "img4",+ "created_at": AnyString(),+ "finished_at": None,+ },+ ],+ "role": "assistant",+ "id": "2",+ "metadata": {},+ "created_at": AnyString(),+ "usage": None,+ "finished_at": None,+ "finished_reason": None,+ "structured_output": None,+ "error": None,+ },+ {+ "name": "User",+ "content": [+ {+ "type": "data",+ "id": AnyString(),+ "source": {+ "type": "base64",+ "data": "AAAA",+ "media_type": "audio/wav",+ },+ "name": None,+ "created_at": AnyString(),+ "finished_at": None,+ },+ {+ "type": "data",+ "id": AnyString(),+ "source": {+ "type": "url",+ "url": "https://example.com/img5.png",+ "media_type": "image/png",+ },+ "name": "img5",+ "created_at": AnyString(),+ "finished_at": None,+ },+ {+ "type": "data",+ "id": AnyString(),+ "source": {+ "type": "base64",+ "data": _PNG_BASE64,+ "media_type": "image/png",+ },+ "name": "img6",+ "created_at": AnyString(),+ "finished_at": None,+ },+ ],+ "role": "user",+ "id": "3",+ "metadata": {},+ "created_at": AnyString(),+ "usage": None,+ "finished_at": AnyString(),+ "finished_reason": None,+ "structured_output": None,+ "error": None,+ },+ ],+ )+async def test_summary_failure_truncates_context(self) -> None:"""Summary failures fall back to lossy context truncation."""agent, _ = _make_failing_compression_agent()