Skip to content

Build and run agents you can see, understand and trust.

feat(agent): add max_image_num to ContextConfig to limit images in context (#2362)

DavdGao3ff581989df7
diff --git a/src/agentscope/agent/_agent.py b/src/agentscope/agent/_agent.py
index 962d42f03..b98333bb5 100644
--- a/src/agentscope/agent/_agent.py
+++ b/src/agentscope/agent/_agent.py
@@ -426,6 +426,10 @@ class Agent:
"""
cfg: ContextConfig = context_config or self.context_config
+ # Limit the number of images in the context first, so that the token
+ # counting below reflects the images that actually remain
+ await self._limit_context_images(cfg)
+
# Count the current tokens
kwargs = await self._prepare_model_input()
estimated_tokens = await self.model.count_tokens(**kwargs)
@@ -665,6 +669,110 @@ class Agent:
await apply_task
raise
+ async def _limit_context_images(self, cfg: ContextConfig) -> None:
+ """Limit the number of images in the context according to
+ ``cfg.max_image_num``. The oldest images exceeding the limit are
+ offloaded to the workspace (if an offloader is provided) and replaced
+ by a hint recording the offloaded path; otherwise they are dropped and
+ replaced by a hint without path information.
+
+ Image data blocks at the top level of a non-user message are
+ replaced by :class:`HintBlock`, while those in user messages or
+ nested inside a :class:`ToolResultBlock` / :class:`HintBlock` are
+ replaced by :class:`TextBlock` (as required by their type
+ constraints).
+
+ Args:
+ cfg (`ContextConfig`):
+ The context config that provides ``max_image_num``.
+ """
+ max_image_num = cfg.max_image_num
+
+ def _is_image(block: Any) -> bool:
+ """Check whether the given block is an image data block."""
+ return isinstance(
+ block,
+ DataBlock,
+ ) and block.source.media_type.startswith("image/")
+
+ # Collect all the image data blocks in chronological order, recorded
+ # as (container list, index, block, is_top_level, role)
+ images: list[tuple[list, int, DataBlock, bool, str]] = []
+ for msg in self.state.context:
+ for i, block in enumerate(msg.content):
+ if _is_image(block):
+ images.append((msg.content, i, block, True, msg.role))
+ elif isinstance(block, ToolResultBlock) and isinstance(
+ block.output,
+ list,
+ ):
+ for j, sub in enumerate(block.output):
+ if _is_image(sub):
+ images.append(
+ (block.output, j, sub, False, msg.role),
+ )
+ elif isinstance(block, HintBlock) and isinstance(
+ block.hint,
+ list,
+ ):
+ for j, sub in enumerate(block.hint):
+ if _is_image(sub):
+ images.append(
+ (block.hint, j, sub, False, msg.role),
+ )
+
+ n_exceed = len(images) - max_image_num
+ if n_exceed <= 0:
+ return
+
+ logger.info(
+ "[AGENT %s]: The number of images in context (%d) exceeds the "
+ "limit (%d), removing the oldest %d image(s).",
+ self.name,
+ len(images),
+ max_image_num,
+ n_exceed,
+ )
+
+ # Offload the images first (which may involve I/O), and then apply
+ # the replacements together to avoid partial modification on
+ # interruption
+ replacements: list[tuple[list, int, HintBlock | TextBlock]] = []
+ for container, idx, block, is_top_level, role in images[:n_exceed]:
+ url = ""
+ if isinstance(block.source, URLSource):
+ url = str(block.source.url)
+ elif self.offloader is not None:
+ saved = await self.offloader.offload_data_block(block)
+ if isinstance(saved.source, URLSource):
+ url = str(saved.source.url)
+
+ name = f"named '{block.name}' " if block.name else ""
+ if url:
+ text = (
+ f"<system-reminder>The image {name}is offloaded into "
+ f"{url}, you can refer to it when needed."
+ f"</system-reminder>"
+ )
+ else:
+ text = (
+ f"<system-reminder>The image {name}is removed to free "
+ f"up context space.</system-reminder>"
+ )
+
+ new_block: HintBlock | TextBlock
+ if is_top_level and role != "user":
+ new_block = HintBlock(hint=text)
+ else:
+ # User messages only accept text/data blocks (see the
+ # validator in Msg), and a hint block is equivalent to a user
+ # text block for the model anyway.
+ new_block = TextBlock(type="text", text=text)
+ replacements.append((container, idx, new_block))
+
+ for container, idx, new_block in replacements:
+ container[idx] = new_block
+
# ======================================================================
# Agent core methods, including _reply, _reasoning, _acting, etc.
# ======================================================================
diff --git a/src/agentscope/agent/_config.py b/src/agentscope/agent/_config.py
index 75ce585da..519f28bed 100644
--- a/src/agentscope/agent/_config.py
+++ b/src/agentscope/agent/_config.py
@@ -138,6 +138,21 @@ class ContextConfig(BaseModel):
)
"""The tool result limit to avoid tool result bursting."""
+ max_image_num: int = Field(
+ title="Max Image Number",
+ default=5,
+ ge=0,
+ description=(
+ "The maximum number of images kept in the context. The oldest "
+ "images exceeding the limit will be removed."
+ ),
+ )
+ """The maximum number of images kept in the context. When the number of
+ images exceeds this limit, the oldest images will be offloaded to the
+ workspace (if an offloader is provided) and replaced by a hint that
+ records the offloaded path; otherwise they are dropped and replaced by a
+ hint without path information."""
+
class InjectionConfig(BaseModel):
"""The state injection related configuration in AgentScope."""
diff --git a/tests/compress_context_test.py b/tests/compress_context_test.py
index 945bc843c..314f0ba29 100644
--- a/tests/compress_context_test.py
+++ b/tests/compress_context_test.py
@@ -1,6 +1,7 @@
# -*- coding: utf-8 -*-
"""A template test case."""
# pylint: disable=protected-access
+import hashlib
import json
import os
import tempfile
@@ -22,8 +23,86 @@ from agentscope.message import (
ToolResultState,
HintBlock,
Msg,
+ DataBlock,
+ Base64Source,
+ URLSource,
)
from agentscope.tool import Toolkit
+from agentscope.workspace import LocalWorkspace
+
+
+# A 1x1 transparent PNG image
+_PNG_BASE64 = (
+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB"
+ "0C8AAAAASUVORK5CYII="
+)
+
+
+def _image_block(name: str) -> DataBlock:
+ """Create a base64 image data block."""
+ return DataBlock(
+ source=Base64Source(data=_PNG_BASE64, media_type="image/png"),
+ name=name,
+ )
+
+
+def _build_image_context() -> list[Msg]:
+ """Build a context with 6 images located at the message top level (user
+ and assistant), inside a tool result and inside a hint block, plus one
+ audio block."""
+ return [
+ UserMsg(
+ "User",
+ [_image_block("img1"), TextBlock(type="text", text="hello")],
+ id="1",
+ ),
+ AssistantMsg(
+ "Friday",
+ [
+ ToolCallBlock(
+ type="tool_call",
+ id="call_1",
+ name="view",
+ input="{}",
+ ),
+ ToolResultBlock(
+ type="tool_result",
+ id="call_1",
+ name="view",
+ output=[
+ TextBlock(type="text", text="the image:"),
+ _image_block("img2"),
+ ],
+ state=ToolResultState.SUCCESS,
+ ),
+ HintBlock(
+ hint=[
+ TextBlock(type="text", text="a hint image:"),
+ _image_block("img3"),
+ ],
+ ),
+ _image_block("img4"),
+ ],
+ id="2",
+ ),
+ UserMsg(
+ "User",
+ [
+ DataBlock(
+ source=Base64Source(data="AAAA", media_type="audio/wav"),
+ ),
+ DataBlock(
+ source=URLSource(
+ url="https://example.com/img5.png",
+ media_type="image/png",
+ ),
+ name="img5",
+ ),
+ _image_block("img6"),
+ ],
+ id="3",
+ ),
+ ]
class RecordingStructuredMockModel(MockModel):
@@ -1441,6 +1520,623 @@ class ContextCompressionTest(IsolatedAsyncioTestCase):
),
)
+ async def test_max_image_num_without_offloader(self) -> None:
+ """The oldest images exceeding the limit are dropped and replaced by
+ hints without path information when no offloader is provided."""
+ agent = Agent(
+ name="Friday",
+ system_prompt="You're a helpful assistant.",
+ model=MockModel(context_size=100000),
+ context_config=ContextConfig(max_image_num=2),
+ state=AgentState(session_id="123", context=_build_image_context()),
+ toolkit=Toolkit(),
+ )
+
+ # The token count is far below the trigger threshold, so only the
+ # image limitation takes effect
+ await agent.compress_context()
+
+ expected = [
+ {
+ "name": "User",
+ "content": [
+ {
+ "type": "text",
+ "text": (
+ "<system-reminder>The image named 'img1' is "
+ "removed to free up context "
+ "space.</system-reminder>"
+ ),
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ {
+ "type": "text",
+ "text": "hello",
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ ],
+ "role": "user",
+ "id": "1",
+ "metadata": {},
+ "created_at": AnyString(),
+ "usage": None,
+ "finished_at": AnyString(),
+ "finished_reason": None,
+ "structured_output": None,
+ "error": None,
+ },
+ {
+ "name": "Friday",
+ "content": [
+ {
+ "type": "tool_call",
+ "id": "call_1",
+ "name": "view",
+ "input": "{}",
+ "state": "pending",
+ "suggested_rules": [],
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ {
+ "type": "tool_result",
+ "id": "call_1",
+ "name": "view",
+ "output": [
+ {
+ "type": "text",
+ "text": "the image:",
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ {
+ "type": "text",
+ "text": (
+ "<system-reminder>The image named 'img2' "
+ "is removed to free up context "
+ "space.</system-reminder>"
+ ),
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ ],
+ "state": "success",
+ "metadata": {},
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ {
+ "type": "hint",
+ "hint": [
+ {
+ "type": "text",
+ "text": "a hint image:",
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ {
+ "type": "text",
+ "text": (
+ "<system-reminder>The image named 'img3' "
+ "is removed to free up context "
+ "space.</system-reminder>"
+ ),
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ ],
+ "id": AnyString(),
+ "source": None,
+ "created_at": AnyString(),
+ "finished_at": AnyString(),
+ },
+ {
+ "type": "hint",
+ "hint": (
+ "<system-reminder>The image named 'img4' is "
+ "removed to free up context "
+ "space.</system-reminder>"
+ ),
+ "id": AnyString(),
+ "source": None,
+ "created_at": AnyString(),
+ "finished_at": AnyString(),
+ },
+ ],
+ "role": "assistant",
+ "id": "2",
+ "metadata": {},
+ "created_at": AnyString(),
+ "usage": None,
+ "finished_at": None,
+ "finished_reason": None,
+ "structured_output": None,
+ "error": None,
+ },
+ {
+ "name": "User",
+ "content": [
+ {
+ "type": "data",
+ "id": AnyString(),
+ "source": {
+ "type": "base64",
+ "data": "AAAA",
+ "media_type": "audio/wav",
+ },
+ "name": None,
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ {
+ "type": "data",
+ "id": AnyString(),
+ "source": {
+ "type": "url",
+ "url": "https://example.com/img5.png",
+ "media_type": "image/png",
+ },
+ "name": "img5",
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ {
+ "type": "data",
+ "id": AnyString(),
+ "source": {
+ "type": "base64",
+ "data": _PNG_BASE64,
+ "media_type": "image/png",
+ },
+ "name": "img6",
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ ],
+ "role": "user",
+ "id": "3",
+ "metadata": {},
+ "created_at": AnyString(),
+ "usage": None,
+ "finished_at": AnyString(),
+ "finished_reason": None,
+ "structured_output": None,
+ "error": None,
+ },
+ ]
+ self.assertListEqual(
+ [_.model_dump() for _ in agent.state.context],
+ expected,
+ )
+
+ # Calling again should be a no-op
+ await agent.compress_context()
+ self.assertListEqual(
+ [_.model_dump() for _ in agent.state.context],
+ expected,
+ )
+
+ async def test_max_image_num_with_offloader(self) -> None:
+ """The oldest images exceeding the limit are offloaded and replaced
+ by hints recording the offloaded path when an offloader is
+ provided; URL images keep their original URL."""
+ with tempfile.TemporaryDirectory() as workdir:
+ agent = Agent(
+ name="Friday",
+ system_prompt="You're a helpful assistant.",
+ model=MockModel(context_size=100000),
+ context_config=ContextConfig(max_image_num=1),
+ state=AgentState(
+ session_id="123",
+ context=_build_image_context(),
+ ),
+ toolkit=Toolkit(),
+ offloader=LocalWorkspace(workdir=workdir),
+ )
+
+ await agent.compress_context()
+
+ # All the base64 images share the same content, thus the same
+ # offloaded path
+ rel = "data/" + hashlib.sha256(_PNG_BASE64.encode()).hexdigest()
+ url = f"workspace:///{rel}.png"
+ self.assertTrue(
+ os.path.exists(os.path.join(workdir, rel + ".png")),
+ )
+
+ self.assertListEqual(
+ [_.model_dump() for _ in agent.state.context],
+ [
+ {
+ "name": "User",
+ "content": [
+ {
+ "type": "text",
+ "text": (
+ "<system-reminder>The image named 'img1' "
+ "is offloaded into "
+ + url
+ + ", you can refer to it when "
+ "needed.</system-reminder>"
+ ),
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ {
+ "type": "text",
+ "text": "hello",
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ ],
+ "role": "user",
+ "id": "1",
+ "metadata": {},
+ "created_at": AnyString(),
+ "usage": None,
+ "finished_at": AnyString(),
+ "finished_reason": None,
+ "structured_output": None,
+ "error": None,
+ },
+ {
+ "name": "Friday",
+ "content": [
+ {
+ "type": "tool_call",
+ "id": "call_1",
+ "name": "view",
+ "input": "{}",
+ "state": "pending",
+ "suggested_rules": [],
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ {
+ "type": "tool_result",
+ "id": "call_1",
+ "name": "view",
+ "output": [
+ {
+ "type": "text",
+ "text": "the image:",
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ {
+ "type": "text",
+ "text": (
+ "<system-reminder>The image named "
+ "'img2' is offloaded into "
+ + url
+ + ", you can refer to it when "
+ "needed.</system-reminder>"
+ ),
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ ],
+ "state": "success",
+ "metadata": {},
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ {
+ "type": "hint",
+ "hint": [
+ {
+ "type": "text",
+ "text": "a hint image:",
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ {
+ "type": "text",
+ "text": (
+ "<system-reminder>The image named "
+ "'img3' is offloaded into "
+ + url
+ + ", you can refer to it when "
+ "needed.</system-reminder>"
+ ),
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ ],
+ "id": AnyString(),
+ "source": None,
+ "created_at": AnyString(),
+ "finished_at": AnyString(),
+ },
+ {
+ "type": "hint",
+ "hint": (
+ "<system-reminder>The image named 'img4' "
+ "is offloaded into "
+ + url
+ + ", you can refer to it when "
+ "needed.</system-reminder>"
+ ),
+ "id": AnyString(),
+ "source": None,
+ "created_at": AnyString(),
+ "finished_at": AnyString(),
+ },
+ ],
+ "role": "assistant",
+ "id": "2",
+ "metadata": {},
+ "created_at": AnyString(),
+ "usage": None,
+ "finished_at": None,
+ "finished_reason": None,
+ "structured_output": None,
+ "error": None,
+ },
+ {
+ "name": "User",
+ "content": [
+ {
+ "type": "data",
+ "id": AnyString(),
+ "source": {
+ "type": "base64",
+ "data": "AAAA",
+ "media_type": "audio/wav",
+ },
+ "name": None,
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ {
+ "type": "text",
+ "text": (
+ "<system-reminder>The image named 'img5' "
+ "is offloaded into "
+ "https://example.com/img5.png, you can "
+ "refer to it when needed."
+ "</system-reminder>"
+ ),
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ {
+ "type": "data",
+ "id": AnyString(),
+ "source": {
+ "type": "base64",
+ "data": _PNG_BASE64,
+ "media_type": "image/png",
+ },
+ "name": "img6",
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ ],
+ "role": "user",
+ "id": "3",
+ "metadata": {},
+ "created_at": AnyString(),
+ "usage": None,
+ "finished_at": AnyString(),
+ "finished_reason": None,
+ "structured_output": None,
+ "error": None,
+ },
+ ],
+ )
+
+ async def test_max_image_num_default(self) -> None:
+ """The default limit is 5, so only the oldest image is removed for a
+ context with 6 images."""
+ agent = Agent(
+ name="Friday",
+ system_prompt="You're a helpful assistant.",
+ model=MockModel(context_size=100000),
+ state=AgentState(session_id="123", context=_build_image_context()),
+ toolkit=Toolkit(),
+ )
+ self.assertEqual(agent.context_config.max_image_num, 5)
+
+ await agent.compress_context()
+
+ self.assertListEqual(
+ [_.model_dump() for _ in agent.state.context],
+ [
+ {
+ "name": "User",
+ "content": [
+ {
+ "type": "text",
+ "text": (
+ "<system-reminder>The image named 'img1' is "
+ "removed to free up context "
+ "space.</system-reminder>"
+ ),
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ {
+ "type": "text",
+ "text": "hello",
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ ],
+ "role": "user",
+ "id": "1",
+ "metadata": {},
+ "created_at": AnyString(),
+ "usage": None,
+ "finished_at": AnyString(),
+ "finished_reason": None,
+ "structured_output": None,
+ "error": None,
+ },
+ {
+ "name": "Friday",
+ "content": [
+ {
+ "type": "tool_call",
+ "id": "call_1",
+ "name": "view",
+ "input": "{}",
+ "state": "pending",
+ "suggested_rules": [],
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ {
+ "type": "tool_result",
+ "id": "call_1",
+ "name": "view",
+ "output": [
+ {
+ "type": "text",
+ "text": "the image:",
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ {
+ "type": "data",
+ "id": AnyString(),
+ "source": {
+ "type": "base64",
+ "data": _PNG_BASE64,
+ "media_type": "image/png",
+ },
+ "name": "img2",
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ ],
+ "state": "success",
+ "metadata": {},
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ {
+ "type": "hint",
+ "hint": [
+ {
+ "type": "text",
+ "text": "a hint image:",
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ {
+ "type": "data",
+ "id": AnyString(),
+ "source": {
+ "type": "base64",
+ "data": _PNG_BASE64,
+ "media_type": "image/png",
+ },
+ "name": "img3",
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ ],
+ "id": AnyString(),
+ "source": None,
+ "created_at": AnyString(),
+ "finished_at": AnyString(),
+ },
+ {
+ "type": "data",
+ "id": AnyString(),
+ "source": {
+ "type": "base64",
+ "data": _PNG_BASE64,
+ "media_type": "image/png",
+ },
+ "name": "img4",
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ ],
+ "role": "assistant",
+ "id": "2",
+ "metadata": {},
+ "created_at": AnyString(),
+ "usage": None,
+ "finished_at": None,
+ "finished_reason": None,
+ "structured_output": None,
+ "error": None,
+ },
+ {
+ "name": "User",
+ "content": [
+ {
+ "type": "data",
+ "id": AnyString(),
+ "source": {
+ "type": "base64",
+ "data": "AAAA",
+ "media_type": "audio/wav",
+ },
+ "name": None,
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ {
+ "type": "data",
+ "id": AnyString(),
+ "source": {
+ "type": "url",
+ "url": "https://example.com/img5.png",
+ "media_type": "image/png",
+ },
+ "name": "img5",
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ {
+ "type": "data",
+ "id": AnyString(),
+ "source": {
+ "type": "base64",
+ "data": _PNG_BASE64,
+ "media_type": "image/png",
+ },
+ "name": "img6",
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ ],
+ "role": "user",
+ "id": "3",
+ "metadata": {},
+ "created_at": AnyString(),
+ "usage": None,
+ "finished_at": AnyString(),
+ "finished_reason": None,
+ "structured_output": None,
+ "error": None,
+ },
+ ],
+ )
+
async def test_summary_failure_truncates_context(self) -> None:
"""Summary failures fall back to lossy context truncation."""
agent, _ = _make_failing_compression_agent()