Skip to content

Build and run agents you can see, understand and trust.

23.6 KB
Raw Blame
# -*- coding: utf-8 -*-
"""Unit tests for the runtime state injection of the agent, i.e. the
``Agent._inject_runtime_state`` method."""
# pylint: disable=too-many-public-methods
from datetime import datetime, tzinfo
from unittest.async_case import IsolatedAsyncioTestCase
from unittest.mock import AsyncMock, patch

from pydantic import ValidationError

from utils import AnyString, MockModel

from agentscope.agent import Agent, ContextConfig, InjectionConfig
from agentscope.message import (
    HintBlock,
    ToolCallBlock,
    ToolResultBlock,
    ToolResultState,
)
from agentscope.state import Task
from agentscope.tool import Toolkit


# The fixed source used by the agent to mark its own runtime-state injection.
INJECTION_SOURCE = '{"label": "System", "sublabel": "Runtime State"}'

# The frozen "now" used across the tests, so time-related assertions are
# deterministic.
FROZEN_NOW = datetime(2026, 7, 1, 12, 0, 0)


class _FrozenDatetime(datetime):
    """A ``datetime`` subclass whose ``now`` always returns a fixed instant,
    while keeping the other classmethods (``strptime``/``strftime``) intact."""

    @classmethod
    def now(  # type: ignore[override]
        cls,
        tz: tzinfo | None = None,
    ) -> datetime:
        """Return the frozen instant (optionally attached with ``tz``)."""
        if tz is not None:
            return FROZEN_NOW.replace(tzinfo=tz)
        return FROZEN_NOW


class AgentInjectionTest(IsolatedAsyncioTestCase):
    """Test cases for the runtime state injection."""

    async def asyncSetUp(self) -> None:
        """Create a fresh agent with a mock model for each test."""
        self.model = MockModel(context_size=1000)
        self.agent = Agent(
            name="Friday",
            system_prompt="You are a helpful assistant.",
            model=self.model,
            toolkit=Toolkit(),
            injection_config=InjectionConfig(),
        )
        self.agent.state.reply_id = "reply-1"
        self.agent.state.cur_iter = 0
        self._batch_index = 0

        # Freeze the wall-clock time for all the test cases
        patcher = patch("agentscope.agent._agent.datetime", _FrozenDatetime)
        patcher.start()
        self.addCleanup(patcher.stop)

    # ------------------------------------------------------------------ utils
    async def _run_injection(self) -> list:
        """Drive the async generator and collect the yielded events."""
        return [
            # pylint: disable=protected-access
            evt
            async for evt in self.agent._inject_runtime_state()
        ]

    def _add_injection(self, time_str: str, timezone: str = "UTC") -> None:
        """Append an existing runtime-state injection carrying ``time_str``,
        which is the wall-clock time of ``timezone``."""
        self.agent.state.append_context(
            self.agent.name,
            [
                HintBlock(
                    source=INJECTION_SOURCE,
                    hint=(
                        f"<current-time>{time_str}</current-time>\n"
                        f"<timezone>{timezone}</timezone>"
                    ),
                ),
            ],
        )

    def _add_tool_batch(
        self,
        *calls: tuple[str, str, ToolResultState],
    ) -> None:
        """Append one reasoning-acting iteration to the context, where each
        call is a ``(tool name, arguments, result state)`` triple."""
        self._batch_index += 1
        ids = [f"tc-{self._batch_index}-{_}" for _ in range(len(calls))]
        self.agent.state.append_context(
            self.agent.name,
            [
                ToolCallBlock(id=id_, name=name, input=arguments)
                for id_, (name, arguments, _) in zip(ids, calls)
            ]
            + [
                ToolResultBlock(id=id_, name=name, output="boom", state=state)
                for id_, (name, _, state) in zip(ids, calls)
            ],
        )

    @staticmethod
    def _expected_event(hint: str, reply_id: str = "reply-1") -> dict:
        """Build the expected ``HintBlockEvent`` dump for the given hint."""
        return {
            "id": AnyString(),
            "created_at": AnyString(),
            "metadata": {},
            "type": "HINT_BLOCK",
            "reply_id": reply_id,
            "block_id": AnyString(),
            "source": INJECTION_SOURCE,
            "hint": hint,
        }

    @staticmethod
    def _expected_hint_block(hint: str) -> dict:
        """Build the expected persisted ``HintBlock`` dump."""
        return {
            "type": "hint",
            "created_at": AnyString(),
            "finished_at": AnyString(),
            "hint": hint,
            "id": AnyString(),
            "source": INJECTION_SOURCE,
        }

    # ------------------------------------------------------------------ tests
    async def test_first_reply_triggers_time_injection(self) -> None:
        """The first reply (empty context) should trigger a time injection."""
        expected_hint = (
            "<system-reminder>Treat the following as the ground truth at this "
            "point of the conversation. Anything stated earlier is outdated, "
            "and a later reminder, if any, supersedes this one:\n"
            "<current-time>2026-07-01T12:00:00</current-time>\n"
            "<timezone>UTC</timezone>\n"
            "</system-reminder>"
        )
        events = await self._run_injection()

        self.assertEqual(
            [self._expected_event(expected_hint)],
            [evt.model_dump() for evt in events],
        )
        self.assertEqual(
            [self._expected_hint_block(expected_hint)],
            [_.model_dump() for _ in self.agent.state.context[-1].content],
        )

    async def test_long_interval_triggers_time_injection(self) -> None:
        """A stale last injection (long elapsed time) should re-inject, while a
        recent one should not."""
        expected_hint = (
            "<system-reminder>Treat the following as the ground truth at this "
            "point of the conversation. Anything stated earlier is outdated, "
            "and a later reminder, if any, supersedes this one:\n"
            "<current-time>2026-07-01T12:00:00</current-time>\n"
            "<timezone>UTC</timezone>\n"
            "</system-reminder>"
        )
        # Avoid the context-length branch, which only runs on the first iter.
        self.agent.state.cur_iter = 1

        # Case 1: last injection was 6 hours ago -> re-inject.
        self._add_injection("2026-07-01T06:00:00")
        events = await self._run_injection()
        self.assertEqual(
            [self._expected_event(expected_hint)],
            [evt.model_dump() for evt in events],
        )

        # Case 2: last injection was 10 minutes ago (< time_interval) -> skip.
        self.agent.state.context = []
        self._add_injection("2026-07-01T11:50:00")
        events = await self._run_injection()
        self.assertEqual([], events)

    async def test_injection_after_compression(self) -> None:
        """A recent injection should not re-inject, but once the context is
        compressed away, the next call should inject again."""
        expected_hint = (
            "<system-reminder>Treat the following as the ground truth at this "
            "point of the conversation. Anything stated earlier is outdated, "
            "and a later reminder, if any, supersedes this one:\n"
            "<current-time>2026-07-01T12:00:00</current-time>\n"
            "<timezone>UTC</timezone>\n"
            "</system-reminder>"
        )
        self.agent.state.cur_iter = 1

        # There is a recent injection before compression -> no new injection.
        self._add_injection("2026-07-01T12:00:00")
        events = await self._run_injection()
        self.assertEqual([], events)

        # Simulate a compression that drops the old context (and injection).
        self.agent.state.context = []
        self.agent.state.summary = "A summary of the previous work."
        events = await self._run_injection()
        self.assertEqual(
            [self._expected_event(expected_hint)],
            [evt.model_dump() for evt in events],
        )

    async def test_pending_task_triggers_injection(self) -> None:
        """Pending tasks without task-related tool calls in the context should
        trigger a tasks injection."""
        expected_hint = (
            "<system-reminder>Treat the following as the ground truth at this "
            "point of the conversation. Anything stated earlier is outdated, "
            "and a later reminder, if any, supersedes this one:\n"
            "<tasks>You have 0 in-progress tasks and 1 pending tasks. "
            "Use `TaskList` to view them if you don't know.</tasks>\n"
            "</system-reminder>"
        )
        self.agent.state.cur_iter = 1
        # A recent injection so the time branch is not triggered.
        self._add_injection("2026-07-01T12:00:00")
        self.agent.state.tasks_context.tasks = [
            Task(
                subject="Write the report",
                description="Draft the quarterly report.",
                metadata={},
                state="pending",
            ),
        ]
        events = await self._run_injection()

        self.assertEqual(
            [self._expected_event(expected_hint)],
            [evt.model_dump() for evt in events],
        )

        # The tasks reminder is already in the context -> no repeated injection
        events = await self._run_injection()
        self.assertEqual([], events)

    async def test_recorded_timezone_is_honored(self) -> None:
        """The recorded timezone should be used to restore the recorded time,
        so a changed ``timezone`` config doesn't distort the elapsed time."""
        self.agent.state.cur_iter = 1

        # The frozen now is 12:00 UTC, i.e. 20:00 in Shanghai. An injection
        # recorded 10 minutes ago in Shanghai -> within the interval, skip.
        self._add_injection("2026-07-01T19:50:00", timezone="Asia/Shanghai")
        events = await self._run_injection()
        self.assertEqual([], events)

        # The same wall-clock time read as UTC would be 7h50m in the future,
        # so a negative elapsed time must trigger an injection instead of
        # being silently swallowed.
        self.agent.state.context = []
        self._add_injection("2026-07-01T19:50:00")
        events = await self._run_injection()
        self.assertEqual(1, len(events))

    async def test_extra_fields_are_attached(self) -> None:
        """The extra fields should be attached to a triggered injection."""
        expected_hint = (
            "<system-reminder>Treat the following as the ground truth at this "
            "point of the conversation. Anything stated earlier is outdated, "
            "and a later reminder, if any, supersedes this one:\n"
            "<current-time>2026-07-01T12:00:00</current-time>\n"
            "<timezone>UTC</timezone>\n"
            "<workspace>/home/friday</workspace>\n"
            "</system-reminder>"
        )
        self.agent.injection_config = InjectionConfig(
            extra_fields={"workspace": "/home/friday"},
        )
        events = await self._run_injection()

        self.assertEqual(
            [self._expected_event(expected_hint)],
            [evt.model_dump() for evt in events],
        )

    async def test_extra_fields_do_not_trigger_injection(self) -> None:
        """The extra fields alone should not trigger an injection."""
        self.agent.injection_config = InjectionConfig(
            extra_fields={"workspace": "/home/friday"},
        )
        self.agent.state.cur_iter = 1
        # A recent injection so the time branch is not triggered.
        self._add_injection("2026-07-01T12:00:00")
        events = await self._run_injection()

        self.assertEqual([], events)

    async def test_disabled_injection(self) -> None:
        """Nothing should be injected when the injection is turned off."""
        self.agent.injection_config = InjectionConfig(
            inject_runtime_state=False,
        )
        events = await self._run_injection()

        self.assertEqual([], events)
        self.assertEqual([], self.agent.state.context)

    async def test_context_size_triggers_injection(self) -> None:
        """When the input tokens are close to the compression threshold, a
        context-length injection should be triggered."""
        expected_hint = (
            "<system-reminder>Treat the following as the ground truth at this "
            "point of the conversation. Anything stated earlier is outdated, "
            "and a later reminder, if any, supersedes this one:\n"
            "<context-length>Your current context contains 700 tokens. "
            "When reaching 800 tokens, your context will be compressed."
            "</context-length>\n"
            "</system-reminder>"
        )
        # First iteration is required for the context-length branch.
        self.agent.state.cur_iter = 0
        # A recent injection so the time branch is not triggered.
        self._add_injection("2026-07-01T12:00:00")
        # 700 > max(0, 0.8 - 0.2) * 1000 == 600 -> triggers the injection.
        self.model.count_tokens = AsyncMock(return_value=700)

        events = await self._run_injection()

        self.assertEqual(
            [self._expected_event(expected_hint)],
            [evt.model_dump() for evt in events],
        )

    async def test_context_size_is_independent_of_the_other_fields(
        self,
    ) -> None:
        """The context length should be reported even when the other
        dimensions are triggered in the same injection."""
        expected_hint = (
            "<system-reminder>Treat the following as the ground truth at this "
            "point of the conversation. Anything stated earlier is outdated, "
            "and a later reminder, if any, supersedes this one:\n"
            "<current-time>2026-07-01T12:00:00</current-time>\n"
            "<timezone>UTC</timezone>\n"
            "<context-length>Your current context contains 700 tokens. "
            "When reaching 800 tokens, your context will be compressed."
            "</context-length>\n"
            "</system-reminder>"
        )
        # The first reply, where the time injection is always triggered.
        self.agent.state.cur_iter = 0
        self.model.count_tokens = AsyncMock(return_value=700)

        events = await self._run_injection()

        self.assertEqual(
            [self._expected_event(expected_hint)],
            [evt.model_dump() for evt in events],
        )

    async def test_compression_tool_is_recommended_between_tasks(self) -> None:
        """A long context at a task boundary recommends the agent tool."""
        expected_hint = (
            "<system-reminder>Treat the following as the ground truth at this "
            "point of the conversation. Anything stated earlier is outdated, "
            "and a later reminder, if any, supersedes this one:\n"
            "<context-length>Your current context contains 700 tokens. "
            "When reaching 800 tokens, your context will be compressed. "
            "No task is in progress, so judge by yourself whether the "
            "context should be compressed now by calling `CompressContext`."
            "</context-length>\n"
            "</system-reminder>"
        )
        self.agent.context_config = ContextConfig(
            trigger_ratio=0.8,
            reserve_ratio=0.1,
            compression_tool_enabled=True,
        )
        self.agent.state.cur_iter = 0
        self._add_injection("2026-07-01T12:00:00")
        self.model.count_tokens = AsyncMock(return_value=700)

        events = await self._run_injection()

        self.assertEqual(
            [self._expected_event(expected_hint)],
            [evt.model_dump() for evt in events],
        )

    async def test_compression_tool_is_not_recommended_within_a_task(
        self,
    ) -> None:
        """An in-progress task isn't a boundary to compress at."""
        self.agent.context_config = ContextConfig(
            trigger_ratio=0.8,
            reserve_ratio=0.1,
            compression_tool_enabled=True,
        )
        self.agent.state.cur_iter = 0
        self._add_injection("2026-07-01T12:00:00")
        self.agent.state.tasks_context.tasks = [
            Task(
                subject="Write the report",
                description="Draft the quarterly report.",
                metadata={},
                state="in_progress",
            ),
        ]
        self.model.count_tokens = AsyncMock(return_value=700)

        events = await self._run_injection()

        self.assertNotIn("CompressContext", events[0].hint)
        self.assertIn(
            "<context-length>Your current context contains 700 tokens. "
            "When reaching 800 tokens, your context will be compressed."
            "</context-length>",
            events[0].hint,
        )

    async def test_disabled_compression_tool_keeps_awareness_boundary(
        self,
    ) -> None:
        """The regular context reminder retains its strict boundary."""
        self.agent.state.cur_iter = 0
        self._add_injection("2026-07-01T12:00:00")
        # Exactly max(0, 0.8 - 0.2) * 1000; the original behavior requires
        # token usage to be strictly greater than this boundary.
        self.model.count_tokens = AsyncMock(return_value=600)

        events = await self._run_injection()

        self.assertEqual([], events)

    async def test_template_without_placeholder_is_rejected(self) -> None:
        """A template that would silently drop the injected fields should be
        rejected at the config level."""
        with self.assertRaises(ValidationError):
            InjectionConfig(template="<system-reminder></system-reminder>")

    async def test_template_with_curly_braces_is_kept(self) -> None:
        """The curly braces other than the placeholder should survive."""
        self.agent.injection_config = InjectionConfig(
            template='{"reminder": "{runtime_state}"}',
        )
        events = await self._run_injection()

        self.assertEqual(
            [
                self._expected_event(
                    '{"reminder": "'
                    "<current-time>2026-07-01T12:00:00</current-time>\n"
                    "<timezone>UTC</timezone>"
                    '"}',
                ),
            ],
            [evt.model_dump() for evt in events],
        )

    async def test_repeated_tool_errors_trigger_injection(self) -> None:
        """The same tool call failing in consecutive iterations should trigger
        a tool-error injection."""
        expected_hint = (
            "<system-reminder>Treat the following as the ground truth at this "
            "point of the conversation. Anything stated earlier is outdated, "
            "and a later reminder, if any, supersedes this one:\n"
            "<tool-error>The last 3 calls to 'read' with the same arguments "
            "all failed. Stop retrying the same call as-is, check the error "
            "message and try a different approach.</tool-error>\n"
            "</system-reminder>"
        )
        self.agent.state.cur_iter = 1
        # A recent injection so the time branch is not triggered.
        self._add_injection("2026-07-01T12:00:00")

        # Two failures are still below the default threshold of three.
        self._add_tool_batch(
            ("read", '{"path": "a.py"}', ToolResultState.ERROR),
        )
        self._add_tool_batch(
            ("read", '{"path": "a.py"}', ToolResultState.ERROR),
        )
        self.assertEqual([], await self._run_injection())

        self._add_tool_batch(
            ("read", '{"path": "a.py"}', ToolResultState.ERROR),
        )
        self.assertEqual(
            [self._expected_event(expected_hint)],
            [evt.model_dump() for evt in await self._run_injection()],
        )

    async def test_arguments_are_normalized(self) -> None:
        """Semantically equal arguments should count as the same call, while
        different ones break the streak."""
        expected_hint = (
            "<system-reminder>Treat the following as the ground truth at this "
            "point of the conversation. Anything stated earlier is outdated, "
            "and a later reminder, if any, supersedes this one:\n"
            "<tool-error>The last 3 calls to 'read' with the same arguments "
            "all failed. Stop retrying the same call as-is, check the error "
            "message and try a different approach.</tool-error>\n"
            "</system-reminder>"
        )
        self.agent.state.cur_iter = 1
        self._add_injection("2026-07-01T12:00:00")

        # Different arguments, which the streak below shouldn't count in.
        self._add_tool_batch(
            ("read", '{"path": "other.py"}', ToolResultState.ERROR),
        )
        # The same arguments in a different key order and spacing.
        self._add_tool_batch(
            ("read", '{"a": 1, "b": 2}', ToolResultState.ERROR),
        )
        self._add_tool_batch(("read", '{"b":2,"a":1}', ToolResultState.ERROR))
        self.assertEqual([], await self._run_injection())

        self._add_tool_batch(
            ("read", '{"a":  1, "b": 2}', ToolResultState.ERROR),
        )
        self.assertEqual(
            [self._expected_event(expected_hint)],
            [evt.model_dump() for evt in await self._run_injection()],
        )

    async def test_fan_out_errors_do_not_trigger_injection(self) -> None:
        """Failing calls with different arguments are not retries, even when
        they repeat across iterations."""
        self.agent.state.cur_iter = 1
        self._add_injection("2026-07-01T12:00:00")

        for _ in range(3):
            self._add_tool_batch(
                ("read", '{"path": "a.py"}', ToolResultState.ERROR),
                ("read", '{"path": "b.py"}', ToolResultState.ERROR),
            )

        self.assertEqual([], await self._run_injection())

    async def test_interleaved_results_break_the_streak(self) -> None:
        """The streak is counted over the trailing tool results, so another
        tool succeeding in between breaks it."""
        self.agent.state.cur_iter = 1
        self._add_injection("2026-07-01T12:00:00")

        for _ in range(3):
            self._add_tool_batch(
                ("read", '{"path": "a.py"}', ToolResultState.ERROR),
                ("TaskUpdate", "{}", ToolResultState.SUCCESS),
            )

        self.assertEqual([], await self._run_injection())

    async def test_non_error_results_break_the_streak(self) -> None:
        """Only the ``ERROR`` state counts, so denied or successful results
        break the streak."""
        self.agent.state.cur_iter = 1
        self._add_injection("2026-07-01T12:00:00")

        self._add_tool_batch(("read", "{}", ToolResultState.ERROR))
        self._add_tool_batch(("read", "{}", ToolResultState.DENIED))
        self._add_tool_batch(("read", "{}", ToolResultState.ERROR))
        self._add_tool_batch(("read", "{}", ToolResultState.ERROR))

        self.assertEqual([], await self._run_injection())

    async def test_invalid_timezone_falls_back_to_utc(self) -> None:
        """An unresolvable timezone shouldn't break the reply loop."""
        expected_hint = (
            "<system-reminder>Treat the following as the ground truth at this "
            "point of the conversation. Anything stated earlier is outdated, "
            "and a later reminder, if any, supersedes this one:\n"
            "<current-time>2026-07-01T12:00:00</current-time>\n"
            "<timezone>Mars/Olympus_Mons</timezone>\n"
            "</system-reminder>"
        )
        self.agent.injection_config = InjectionConfig(
            timezone="Mars/Olympus_Mons",
        )
        events = await self._run_injection()

        self.assertEqual(
            [self._expected_event(expected_hint)],
            [evt.model_dump() for evt in events],
        )