Skip to content

Build and run agents you can see, understand and trust.

fix(model): disable thinking for structured output calls and add compression fallback (#2140)

qbc4a121d8749c4
diff --git a/src/agentscope/agent/_agent.py b/src/agentscope/agent/_agent.py
index fa3c7c09a..962d42f03 100644
--- a/src/agentscope/agent/_agent.py
+++ b/src/agentscope/agent/_agent.py
@@ -544,13 +544,14 @@ class Agent:
context_overflow = True
# Compress the messages
+ res = None
try:
res = await self.model.generate_structured_output(
messages=messages,
structured_model=cfg.summary_schema,
)
- except Exception as e:
+ except Exception as error:
if context_overflow:
logger.warning(
"Failed to compress context, which may be caused by "
@@ -583,15 +584,25 @@ class Agent:
):
break
- res = await self.model.generate_structured_output(
- messages=messages,
- structured_model=cfg.summary_schema,
- )
+ try:
+ res = await self.model.generate_structured_output(
+ messages=messages,
+ structured_model=cfg.summary_schema,
+ )
+ except Exception as retry_error:
+ error = retry_error
- else:
- raise e from None
+ if res is None:
+ logger.warning(
+ "[AGENT %s]: Summary generation failed: %s. "
+ "Falling back to context truncation.",
+ self.name,
+ error,
+ )
- if res.finished_reason == FinishedReason.INTERRUPTED:
+ if res is not None and (
+ res.finished_reason == FinishedReason.INTERRUPTED
+ ):
logger.warning(
"The context compression was interrupted and skipped. ",
)
@@ -600,22 +611,45 @@ class Agent:
# Update the summary
async def _apply_change() -> None:
"""Apply the context change with interruption protection."""
- new_summary = cfg.summary_template.format(**res.content)
+ if res is not None:
+ new_summary = cfg.summary_template.format(**res.content)
+ else:
+ # Keep the previous summary if the compression failed
+ new_summary = self.state.summary or (
+ "<system-info>Some earlier messages were truncated for "
+ "limited context.</system-info>"
+ )
+
+ # Offload the compressed context if offloader is provided
if self.offloader:
path = await self.offloader.offload_context(
self.state.session_id,
msgs=msgs_to_compress,
)
- new_summary += (
- f"\n<system-reminder>The compressed context is offloaded "
- f"to '{path}', you can refer to it when needed."
- f"</system-reminder>"
+ offload_reminder = (
+ f"<system-reminder>The compressed context"
+ f" is offloaded to '{path}', you can refer"
+ f" to it when needed.</system-reminder>"
)
+ # Avoid duplicating the reminder when the previous summary
+ # is reused after a failed compression
+ if isinstance(new_summary, list):
+ if not any(
+ isinstance(block, TextBlock)
+ and block.text == offload_reminder
+ for block in new_summary
+ ):
+ new_summary = [
+ *new_summary,
+ TextBlock(text=offload_reminder),
+ ]
+ elif offload_reminder not in new_summary:
+ new_summary += f"\n{offload_reminder}"
- # Protected from interruption
+ # Clear the read tool cache
await self._clear_unreserved_read_cache(msgs_to_reserve)
- # Update the context
+ # Update the context and summary
self.state.summary = new_summary
self.state.context = msgs_to_reserve
diff --git a/src/agentscope/exception/__init__.py b/src/agentscope/exception/__init__.py
index 83be2bf25..8313c1828 100644
--- a/src/agentscope/exception/__init__.py
+++ b/src/agentscope/exception/__init__.py
@@ -5,6 +5,7 @@ from ._base import (
AgentOrientedException,
DeveloperOrientedException,
)
+from ._model import StructuredOutputError
from ._tool import (
ToolInterruptedError,
ToolNotFoundError,
@@ -15,6 +16,7 @@ from ._tool import (
__all__ = [
"AgentOrientedException",
"DeveloperOrientedException",
+ "StructuredOutputError",
"ToolInterruptedError",
"ToolNotFoundError",
"ToolJSONDecodeError",
diff --git a/src/agentscope/exception/_model.py b/src/agentscope/exception/_model.py
new file mode 100644
index 000000000..4866c235a
--- /dev/null
+++ b/src/agentscope/exception/_model.py
@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+"""The model related exceptions."""
+from ._base import DeveloperOrientedException
+
+
+class StructuredOutputError(DeveloperOrientedException):
+ """Raised when the model fails to produce a valid structured output,
+ e.g. it does not call the structured-output tool, returns an empty
+ response, or the returned arguments fail JSON/schema validation."""
diff --git a/src/agentscope/model/_anthropic/_model.py b/src/agentscope/model/_anthropic/_model.py
index 19e1996fe..3334452ec 100644
--- a/src/agentscope/model/_anthropic/_model.py
+++ b/src/agentscope/model/_anthropic/_model.py
@@ -8,7 +8,7 @@ from typing import Literal, Any, AsyncGenerator, TYPE_CHECKING, List, Type
from pydantic import BaseModel, Field
from .._base import ChatModelBase, _TOOL_CHOICE_LITERAL_MODES
-from .._model_response import ChatResponse, StructuredResponse
+from .._model_response import ChatResponse
from .._model_usage import ChatUsage
from ..._utils._common import _generate_id
from ...credential import AnthropicCredential
@@ -166,6 +166,14 @@ class AnthropicChatModel(ChatModelBase):
anthropic.InternalServerError,
)
+ @classmethod
+ def _get_structured_output_fallback_exceptions(
+ cls,
+ ) -> tuple[Type[Exception], ...]:
+ import anthropic
+
+ return (anthropic.BadRequestError,)
+
def _thinking_mode(self) -> str | None:
"""Resolve the effective thinking mode.
@@ -585,55 +593,6 @@ class AnthropicChatModel(ChatModelBase):
}
return fmt_tools, type_mapping[mode]
- async def _call_api_with_structured_output(
- self,
- model_name: str,
- messages: list[Msg],
- structured_model: Type[BaseModel] | dict,
- tool_choice: ToolChoice | None = None,
- **kwargs: Any,
- ) -> StructuredResponse:
- """Anthropic-specific override for structured output.
-
- Anthropic's budget-based thinking mode only supports
- ``tool_choice={"type": "auto"}`` or ``{"type": "none"}``; any
- forcing form (``"any"`` or a specific tool) raises an API error.
- In that mode we default ``tool_choice`` to ``"auto"`` and rely on
- the base class's injected system-reminder prompt to guide the
- model. Adaptive thinking accepts forced tool use, so it falls
- through to the base implementation (force the structured-output
- tool) just like thinking being off entirely.
-
- See:
- https://platform.claude.com/docs/en/build-with-claude/extended-thinking#extended-thinking-with-tool-use
-
- Args:
- model_name (`str`):
- The model name to use for this call.
- messages (`list[Msg]`):
- The context for the LLM to generate the structured output.
- structured_model (`Type[BaseModel] | dict`):
- A Pydantic model class or a JSON schema dict describing the
- required output structure.
- tool_choice (`ToolChoice | None`, defaults to `None`):
- The tool_choice forwarded to ``_call_api``. When ``None``
- and thinking mode is enabled, it is downgraded to
- ``ToolChoice(mode="auto")``; otherwise the base default
- (force the structured-output tool) is used.
- **kwargs (`Any`):
- Additional keyword arguments forwarded to ``_call_api``.
-
- Returns:
- `StructuredResponse`:
- The structured response whose ``content`` is the validated
- output dict matching ``structured_model``.
- """
- if tool_choice is None and self._thinking_mode() == "enabled":
- tool_choice = ToolChoice(mode="auto")
- return await super()._call_api_with_structured_output(
- model_name=model_name,
- messages=messages,
- structured_model=structured_model,
- tool_choice=tool_choice,
- **kwargs,
- )
+ def _get_disable_thinking_kwargs(self) -> dict:
+ """Anthropic uses ``thinking.type=disabled`` as a top-level kwarg."""
+ return {"thinking": {"type": "disabled"}}
diff --git a/src/agentscope/model/_base.py b/src/agentscope/model/_base.py
index c031823bf..87092d6e3 100644
--- a/src/agentscope/model/_base.py
+++ b/src/agentscope/model/_base.py
@@ -9,7 +9,7 @@ from pathlib import Path
from typing import Type, Any, AsyncGenerator
import jsonschema
-from pydantic import BaseModel
+from pydantic import BaseModel, ValidationError as PydanticValidationError
from ._model_response import StructuredResponse, ChatResponse, FinishedReason
from ._model_card import ModelCard
@@ -17,6 +17,7 @@ from ._utils import _StreamAccumulator
from .._logging import logger
from .._utils._common import _json_loads_with_repair
from ..credential import CredentialBase
+from ..exception import StructuredOutputError, ToolJSONDecodeError
from ..message import (
Msg,
TextBlock,
@@ -107,6 +108,29 @@ class ChatModelBase:
"""
return ()
+ def _get_disable_thinking_kwargs(self) -> dict:
+ """Return kwargs that disable thinking for this provider.
+
+ Subclasses whose API supports a thinking/reasoning toggle
+ should override this to return the appropriate kwargs dict
+ (e.g. ``{"extra_body": {"enable_thinking": False}}``).
+
+ Used by the structured-output fallback mechanism.
+ """
+ return {}
+
+ @classmethod
+ def _get_structured_output_fallback_exceptions(
+ cls,
+ ) -> tuple[Type[Exception], ...]:
+ """Provider exceptions that indicate the request shape was rejected
+ (e.g. a forced ``tool_choice`` while thinking is enabled), so that
+ another structured-output strategy may succeed. Subclasses should
+ return the provider's bad-request exception types; the default is
+ empty, i.e. only local structured-output failures trigger a fallback.
+ """
+ return ()
+
@classmethod
def list_models(
cls,
@@ -436,10 +460,24 @@ class ChatModelBase:
structured_model: Type[BaseModel] | dict,
**kwargs: Any,
) -> StructuredResponse:
- """Generate required structured output by the given model.
+ """Generate structured output, trying fallback strategies in order.
+
+ Each strategy is retried on transient (retryable) errors. A
+ :class:`~agentscope.exception.StructuredOutputError` (the model did
+ not produce a valid structured output) or a provider error listed in
+ ``_get_structured_output_fallback_exceptions()`` (e.g. a provider
+ rejecting a forced ``tool_choice`` while thinking is enabled) moves
+ on to the next strategy; any other error is raised immediately. The
+ strategies are immutable and local to each call:
- Shares the same retry settings (``max_retries``, ``retry_delay``, and
- ``_get_retryable_exceptions()``) as the ``__call__`` method.
+ - ``forced``: current config + forced ``tool_choice``
+ - ``auto``: current config + ``auto`` ``tool_choice``
+ - ``no_think``: thinking disabled + forced ``tool_choice`` (skipped
+ when the provider exposes no thinking toggle)
+ - ``none``: current config + no ``tool_choice``
+
+ An explicit ``tool_choice`` in ``kwargs`` bypasses the strategy
+ ladder and is forwarded unchanged.
Args:
messages (`list[Msg]`):
@@ -451,49 +489,108 @@ class ChatModelBase:
`StructuredResponse`:
The structured response generated by the model.
"""
-
if len(messages) == 0:
raise ValueError(
"The input messages cannot be empty for the "
"`generate_structured_output` method.",
)
+ user_tool_choice = kwargs.pop("tool_choice", None)
+ if user_tool_choice is None:
+ forced_tc = ToolChoice(mode="generate_structured_output")
+ disable_kwargs = self._get_disable_thinking_kwargs()
+ # (name, extra `_call_api` kwargs, tool_choice), best first.
+ # The no-think strategy only applies when the provider can
+ # toggle it.
+ strategies = (
+ ("forced", {}, forced_tc),
+ ("auto", {}, ToolChoice(mode="auto")),
+ *(
+ (("no_think", disable_kwargs, forced_tc),)
+ if disable_kwargs
+ else ()
+ ),
+ ("none", {}, None),
+ )
+ else:
+ strategies = (("explicit", {}, user_tool_choice),)
+
retryable = tuple(self._get_retryable_exceptions())
+ fallback: tuple[Type[Exception], ...] = (
+ StructuredOutputError,
+ *self._get_structured_output_fallback_exceptions(),
+ )
+ first_error: Exception | None = None
last_error: Exception | None = None
- for attempt in range(self.max_retries + 1):
- try:
- return await self._call_api_with_structured_output(
- self.model,
- messages=messages,
- structured_model=structured_model,
- **kwargs,
- )
- except Exception as e:
- if not isinstance(e, retryable):
- raise
- last_error = e
- if attempt < self.max_retries:
- logger.warning(
- "Attempt %d failed for model %s: %s. "
- "Retrying in %.1fs...",
- attempt + 1,
+ for name, extra_kwargs, tool_choice in strategies:
+ # Overlay disable-thinking kwargs; a nested `extra_body` is
+ # merged rather than overwritten so caller keys survive.
+ merged = {**kwargs, **extra_kwargs}
+ for key, val in extra_kwargs.items():
+ if isinstance(val, dict) and isinstance(kwargs.get(key), dict):
+ merged[key] = {**kwargs[key], **val}
+
+ for attempt in range(self.max_retries + 1):
+ try:
+ result = await self._call_api_with_structured_output(
self.model,
- str(e),
- self.retry_delay,
+ messages=messages,
+ structured_model=structured_model,
+ tool_choice=tool_choice,
+ **merged,
)
- await asyncio.sleep(self.retry_delay)
- else:
- logger.warning(
- "All %d attempt(s) failed for model %s.",
- self.max_retries + 1,
+ if name not in ("forced", "explicit"):
+ logger.info(
+ "Structured output for %s: using fallback "
+ "strategy '%s'.",
+ self.model,
+ name,
+ )
+ return result
+ except Exception as e: # pylint: disable=broad-except
+ # CancelledError / KeyboardInterrupt derive from
+ # BaseException and are not caught here, so they
+ # propagate as expected.
+ if first_error is None:
+ first_error = e
+ last_error = e
+ if isinstance(e, retryable):
+ # Transient error: retry the same strategy. A
+ # different strategy hits the same endpoint, so
+ # switching would not help a rate limit or timeout.
+ if attempt < self.max_retries:
+ logger.warning(
+ "Structured output attempt %d for %s "
+ "failed: %s. Retrying in %.1fs...",
+ attempt + 1,
+ self.model,
+ e,
+ self.retry_delay,
+ )
+ await asyncio.sleep(self.retry_delay)
+ continue
+ raise # retries exhausted -> give up
+ if not isinstance(e, fallback):
+ raise
+ # Structured-output compatibility failure: try the next
+ # strategy.
+ logger.debug(
+ "Structured output strategy '%s' failed for %s: %s. "
+ "Trying next.",
+ name,
self.model,
+ e,
)
- if last_error is not None:
- raise last_error
- raise RuntimeError(
- f"Failed to generate structured output after "
- f"{self.max_retries + 1} retries.",
- )
+ break
+
+ if last_error is None:
+ raise RuntimeError(
+ f"No structured-output strategy is available for "
+ f"{self.model}.",
+ )
+ if first_error is not None and first_error is not last_error:
+ raise last_error from first_error
+ raise last_error
async def _call_api_with_structured_output(
self,
@@ -503,22 +600,15 @@ class ChatModelBase:
tool_choice: ToolChoice | None = None,
**kwargs: Any,
) -> StructuredResponse:
- """This function constructs a 'generate_structured_output' tool to
- help LLM generate structured output as a compromise for LLM APIs that
- don't support structured output.
-
- If your subclasses inherit from `ChatModelBase` and the underlying
- API supports structured output, you can override this method to
- provide a more accurate implementation.
-
- Note by default this method forces LLM to call the
- 'generate_structured_output' tool via tool_choice, and adds
- instructions into the input messages. Subclasses whose underlying
- API rejects forced tool_choice in certain modes (e.g. DashScope in
- thinking mode) can pass ``tool_choice=ToolChoice(mode="auto")`` and
- rely solely on the injected system-reminder prompt. LLM APIs that
- don't support "required" tool choice may still fail (e.g. generate
- text output and ignore the tool call, or fail in validation).
+ """Run a single structured-output attempt via a forced tool call.
+
+ Constructs a ``generate_structured_output`` tool, asks the LLM to
+ call it (via ``tool_choice``), and validates the returned
+ arguments against ``structured_model``. Strategy selection and the
+ retry loop live in ``generate_structured_output``; this method
+ performs exactly one attempt with the given ``tool_choice`` and
+ ``kwargs``, and uses ``tool_choice`` as-is (``None`` is forwarded
+ unchanged, i.e. no forcing).
Args:
model_name (`str`):
@@ -529,20 +619,16 @@ class ChatModelBase:
A Pydantic model class or a JSON schema dict describing the
required output structure.
tool_choice (`ToolChoice | None`, defaults to `None`):
- The tool_choice forwarded to ``_call_api``. When ``None``,
- defaults to forcing the ``generate_structured_output`` tool.
+ The tool_choice forwarded to ``_call_api``, used as-is.
**kwargs (`Any`):
Additional keyword arguments forwarded to ``_call_api``.
"""
-
+ func_name = "generate_structured_output"
if isinstance(structured_model, dict):
input_schema = structured_model
else:
input_schema = structured_model.model_json_schema()
- func_name = "generate_structured_output"
- if tool_choice is None:
- tool_choice = ToolChoice(mode=func_name)
instruction = (
"<system-reminder>Now you **MUST** call the tool named "
f"'{func_name}' to generate the structured output required "
@@ -604,38 +690,47 @@ class ChatModelBase:
completed_response = res
if completed_response is None or not completed_response.content:
- raise RuntimeError(
+ raise StructuredOutputError(
f"Failed to get the completed response from model "
f"{model_name}.",
)
structured_output: dict[str, Any] | None = None
- for _ in completed_response.content:
- if isinstance(_, ToolCallBlock) and _.name == func_name:
- structured_output = _json_loads_with_repair(
- _.input,
- input_schema,
- )
- break
+ try:
+ for _ in completed_response.content:
+ if isinstance(_, ToolCallBlock) and _.name == func_name:
+ structured_output = _json_loads_with_repair(
+ _.input,
+ input_schema,
+ )
+ break
- if structured_output is None:
- raise RuntimeError(
- "Failed to generate structured output for model.",
- )
+ if structured_output is None:
+ raise StructuredOutputError(
+ "Failed to generate structured output for model.",
+ )
- # Validate the output
- if isinstance(structured_model, dict):
- jsonschema.validate(structured_output, structured_model)
+ # Validate the output
+ if isinstance(structured_model, dict):
+ jsonschema.validate(structured_output, structured_model)
- elif issubclass(structured_model, BaseModel):
- structured_model.model_validate(structured_output)
+ elif issubclass(structured_model, BaseModel):
+ structured_model.model_validate(structured_output)
- else:
- raise ValueError(
- "The structured_model is expected to be a subclass of "
- "Pydantic.BaseModel or a dict, "
- f"but got {type(structured_model)}.",
- )
+ else:
+ raise ValueError(
+ "The structured_model is expected to be a subclass of "
+ "Pydantic.BaseModel or a dict, "
+ f"but got {type(structured_model)}.",
+ )
+ except (
+ ToolJSONDecodeError,
+ jsonschema.ValidationError,
+ PydanticValidationError,
+ ) as e:
+ raise StructuredOutputError(
+ f"Invalid structured output from model {model_name}: {e}",
+ ) from e
return StructuredResponse(
id=completed_response.id,
diff --git a/src/agentscope/model/_dashscope/_model.py b/src/agentscope/model/_dashscope/_model.py
index e6f48ad41..6d1e512e8 100644
--- a/src/agentscope/model/_dashscope/_model.py
+++ b/src/agentscope/model/_dashscope/_model.py
@@ -11,7 +11,7 @@ from pydantic import BaseModel, Field
from ..._utils._audio import _build_streaming_wav_header
from ..._utils._common import _generate_id
from .._base import ChatModelBase, _TOOL_CHOICE_LITERAL_MODES
-from .._model_response import ChatResponse, StructuredResponse
+from .._model_response import ChatResponse
from .._model_usage import ChatUsage
from ...credential import DashScopeCredential
from ...formatter import FormatterBase, DashScopeChatFormatter
@@ -181,6 +181,14 @@ class DashScopeChatModel(ChatModelBase):
openai.InternalServerError,
)
+ @classmethod
+ def _get_structured_output_fallback_exceptions(
+ cls,
+ ) -> tuple[Type[Exception], ...]:
+ import openai
+
+ return (openai.BadRequestError,)
+
async def _call_api(
self,
model_name: str,
@@ -244,17 +252,21 @@ class DashScopeChatModel(ChatModelBase):
if fmt_tool_choice is not None:
request_kwargs["tool_choice"] = fmt_tool_choice
- extra_body: dict[str, Any] = {}
+ eb = dict(request_kwargs.get("extra_body") or {})
+ request_kwargs["extra_body"] = eb
+
if self.parameters.thinking_enable is not None:
- extra_body["enable_thinking"] = self.parameters.thinking_enable
+ eb.setdefault(
+ "enable_thinking",
+ self.parameters.thinking_enable,
+ )
if self.parameters.thinking_budget is not None:
- extra_body["thinking_budget"] = self.parameters.thinking_budget
+ eb.setdefault(
+ "thinking_budget",
+ self.parameters.thinking_budget,
+ )
if self.parameters.top_k is not None:
- extra_body["top_k"] = self.parameters.top_k
-
- if extra_body:
- request_kwargs.setdefault("extra_body", {})
- request_kwargs["extra_body"].update(extra_body)
+ eb.setdefault("top_k", self.parameters.top_k)
if self.stream:
request_kwargs["stream_options"] = {"include_usage": True}
@@ -529,51 +541,6 @@ class DashScopeChatModel(ChatModelBase):
return fmt_tools, mode
- async def _call_api_with_structured_output(
- self,
- model_name: str,
- messages: list[Msg],
- structured_model: Type[BaseModel] | dict,
- tool_choice: ToolChoice | None = None,
- **kwargs: Any,
- ) -> StructuredResponse:
- """DashScope-specific override for structured output.
-
- DashScope rejects ``tool_choice="required"`` or an object-form
- ``tool_choice`` when thinking mode is enabled. In that case we
- default ``tool_choice`` to ``"auto"`` and rely on the base class's
- injected system-reminder prompt to guide the model. When thinking
- is disabled, this falls through to the base implementation.
-
- See: https://help.aliyun.com/en/model-studio/qwen-function-calling
-
- Args:
- model_name (`str`):
- The model name to use for this call.
- messages (`list[Msg]`):
- The context for the LLM to generate the structured output.
- structured_model (`Type[BaseModel] | dict`):
- A Pydantic model class or a JSON schema dict describing the
- required output structure.
- tool_choice (`ToolChoice | None`, defaults to `None`):
- The tool_choice forwarded to ``_call_api``. When ``None``
- and thinking mode is enabled, it is downgraded to
- ``ToolChoice(mode="auto")``; otherwise the base default
- (force the structured-output tool) is used.
- **kwargs (`Any`):
- Additional keyword arguments forwarded to ``_call_api``.
-
- Returns:
- `StructuredResponse`:
- The structured response whose ``content`` is the validated
- output dict matching ``structured_model``.
- """
- if tool_choice is None and self.parameters.thinking_enable:
- tool_choice = ToolChoice(mode="auto")
- return await super()._call_api_with_structured_output(
- model_name=model_name,
- messages=messages,
- structured_model=structured_model,
- tool_choice=tool_choice,
- **kwargs,
- )
+ def _get_disable_thinking_kwargs(self) -> dict:
+ """DashScope uses ``enable_thinking=False`` in extra_body."""
+ return {"extra_body": {"enable_thinking": False}}
diff --git a/src/agentscope/model/_deepseek/_model.py b/src/agentscope/model/_deepseek/_model.py
index d62a757fc..ff6c5f41f 100644
--- a/src/agentscope/model/_deepseek/_model.py
+++ b/src/agentscope/model/_deepseek/_model.py
@@ -7,7 +7,7 @@ from typing import Literal, Any, AsyncGenerator, TYPE_CHECKING, List, Type
from pydantic import BaseModel, Field
from .._base import ChatModelBase, _TOOL_CHOICE_LITERAL_MODES
-from .._model_response import ChatResponse, StructuredResponse
+from .._model_response import ChatResponse
from .._model_usage import ChatUsage
from ..._utils._common import _generate_id
from ...credential import DeepSeekCredential
@@ -146,6 +146,14 @@ class DeepSeekChatModel(ChatModelBase):
openai.InternalServerError,
)
+ @classmethod
+ def _get_structured_output_fallback_exceptions(
+ cls,
+ ) -> tuple[Type[Exception], ...]:
+ import openai
+
+ return (openai.BadRequestError,)
+
async def _call_api(
self,
model_name: str,
@@ -428,49 +436,8 @@ class DeepSeekChatModel(ChatModelBase):
return tools, mode
- async def _call_api_with_structured_output(
- self,
- model_name: str,
- messages: list[Msg],
- structured_model: Type[BaseModel] | dict,
- tool_choice: ToolChoice | None = None,
- **kwargs: Any,
- ) -> StructuredResponse:
- """DeepSeek-specific override for structured output.
-
- DeepSeek rejects ``tool_choice="required"`` or an object-form
- ``tool_choice`` when thinking mode is enabled. In that case we
- default ``tool_choice`` to ``"auto"`` and rely on the base class's
- injected system-reminder prompt to guide the model. When thinking
- is disabled, this falls through to the base implementation.
-
- Args:
- model_name (`str`):
- The model name to use for this call.
- messages (`list[Msg]`):
- The context for the LLM to generate the structured output.
- structured_model (`Type[BaseModel] | dict`):
- A Pydantic model class or a JSON schema dict describing the
- required output structure.
- tool_choice (`ToolChoice | None`, defaults to `None`):
- The tool_choice forwarded to ``_call_api``. When ``None``
- and thinking mode is enabled, it is downgraded to
- ``ToolChoice(mode="auto")``; otherwise the base default
- (force the structured-output tool) is used.
- **kwargs (`Any`):
- Additional keyword arguments forwarded to ``_call_api``.
-
- Returns:
- `StructuredResponse`:
- The structured response whose ``content`` is the validated
- output dict matching ``structured_model``.
- """
- if tool_choice is None and self.parameters.thinking_enable:
- tool_choice = ToolChoice(mode="auto")
- return await super()._call_api_with_structured_output(
- model_name=model_name,
- messages=messages,
- structured_model=structured_model,
- tool_choice=tool_choice,
- **kwargs,
- )
+ def _get_disable_thinking_kwargs(self) -> dict:
+ """DeepSeek uses ``thinking.type=disabled`` in extra_body."""
+ return {
+ "extra_body": {"thinking": {"type": "disabled"}},
+ }
diff --git a/src/agentscope/model/_moonshot/_model.py b/src/agentscope/model/_moonshot/_model.py
index ac1a12f77..eff22c8e3 100644
--- a/src/agentscope/model/_moonshot/_model.py
+++ b/src/agentscope/model/_moonshot/_model.py
@@ -7,7 +7,7 @@ from typing import Literal, Any, AsyncGenerator, TYPE_CHECKING, List, Type
from pydantic import BaseModel, Field
from .._base import ChatModelBase, _TOOL_CHOICE_LITERAL_MODES
-from .._model_response import ChatResponse, StructuredResponse
+from .._model_response import ChatResponse
from .._model_usage import ChatUsage
from ..._utils._common import _generate_id
from ...credential import MoonshotCredential
@@ -145,6 +145,14 @@ class MoonshotChatModel(ChatModelBase):
openai.InternalServerError,
)
+ @classmethod
+ def _get_structured_output_fallback_exceptions(
+ cls,
+ ) -> tuple[Type[Exception], ...]:
+ import openai
+
+ return (openai.BadRequestError,)
+
async def _call_api(
self,
model_name: str,
@@ -432,49 +440,8 @@ class MoonshotChatModel(ChatModelBase):
return tools, mode
- async def _call_api_with_structured_output(
- self,
- model_name: str,
- messages: list[Msg],
- structured_model: Type[BaseModel] | dict,
- tool_choice: ToolChoice | None = None,
- **kwargs: Any,
- ) -> StructuredResponse:
- """Moonshot-specific override for structured output.
-
- Moonshot rejects ``tool_choice="required"`` or an object-form
- ``tool_choice`` when thinking mode is enabled. In that case we
- default ``tool_choice`` to ``"auto"`` and rely on the base class's
- injected system-reminder prompt to guide the model. When thinking
- is disabled, this falls through to the base implementation.
-
- Args:
- model_name (`str`):
- The model name to use for this call.
- messages (`list[Msg]`):
- The context for the LLM to generate the structured output.
- structured_model (`Type[BaseModel] | dict`):
- A Pydantic model class or a JSON schema dict describing the
- required output structure.
- tool_choice (`ToolChoice | None`, defaults to `None`):
- The tool_choice forwarded to ``_call_api``. When ``None``
- and thinking mode is enabled, it is downgraded to
- ``ToolChoice(mode="auto")``; otherwise the base default
- (force the structured-output tool) is used.
- **kwargs (`Any`):
- Additional keyword arguments forwarded to ``_call_api``.
-
- Returns:
- `StructuredResponse`:
- The structured response whose ``content`` is the validated
- output dict matching ``structured_model``.
- """
- if tool_choice is None and self.parameters.thinking_enable:
- tool_choice = ToolChoice(mode="auto")
- return await super()._call_api_with_structured_output(
- model_name=model_name,
- messages=messages,
- structured_model=structured_model,
- tool_choice=tool_choice,
- **kwargs,
- )
+ def _get_disable_thinking_kwargs(self) -> dict:
+ """Moonshot uses ``thinking.type=disabled`` in extra_body."""
+ return {
+ "extra_body": {"thinking": {"type": "disabled"}},
+ }
diff --git a/src/agentscope/model/_openai_chat/_model.py b/src/agentscope/model/_openai_chat/_model.py
index 87369dcd9..57de22a23 100644
--- a/src/agentscope/model/_openai_chat/_model.py
+++ b/src/agentscope/model/_openai_chat/_model.py
@@ -1,6 +1,5 @@
# -*- coding: utf-8 -*-
"""The OpenAI Chat Completions model implementation."""
-import warnings
import base64
from collections import OrderedDict
from datetime import datetime
@@ -11,7 +10,7 @@ from pydantic import BaseModel, Field
from ..._utils._audio import _build_streaming_wav_header
from ..._utils._common import _generate_id, _flatten_json_schema
from .._base import ChatModelBase, _TOOL_CHOICE_LITERAL_MODES
-from .._model_response import ChatResponse, StructuredResponse
+from .._model_response import ChatResponse
from .._model_usage import ChatUsage
from ...credential import OpenAICredential
from ...formatter import FormatterBase, OpenAIChatFormatter
@@ -185,6 +184,14 @@ class OpenAIChatModel(ChatModelBase):
openai.InternalServerError,
)
+ @classmethod
+ def _get_structured_output_fallback_exceptions(
+ cls,
+ ) -> tuple[Type[Exception], ...]:
+ import openai
+
+ return (openai.BadRequestError,)
+
async def _call_api(
self,
model_name: str,
@@ -544,48 +551,6 @@ class OpenAIChatModel(ChatModelBase):
return ChatResponse(**resp_kwargs)
- async def _call_api_with_structured_output(
- self,
- model_name: str,
- messages: list[Msg],
- structured_model: Type[BaseModel] | dict,
- tool_choice: ToolChoice | None = None,
- **kwargs: Any,
- ) -> StructuredResponse:
- """OpenAI-compatible override for structured output.
-
- Some third-party providers that expose an OpenAI-compatible API (e.g.
- DeepSeek via DashScope) reject forced ``tool_choice`` when thinking
- mode is active. When such a ``BadRequestError`` is encountered, this
- method automatically retries with ``tool_choice="auto"``.
- """
- import openai
-
- try:
- return await super()._call_api_with_structured_output(
- model_name=model_name,
- messages=messages,
- structured_model=structured_model,
- tool_choice=tool_choice,
- **kwargs,
- )
- except openai.BadRequestError as e:
- if "tool_choice" not in str(e):
- raise
- # Thinking mode rejects forced tool_choice; fall back to auto
- warnings.warn(
- f"Forced tool_choice rejected by provider ({e}), "
- "retrying with tool_choice='auto'.",
- stacklevel=2,
- )
- return await super()._call_api_with_structured_output(
- model_name=model_name,
- messages=messages,
- structured_model=structured_model,
- tool_choice=ToolChoice(mode="auto"),
- **kwargs,
- )
-
def _format_tools(
self,
tools: list[dict] | None,
diff --git a/src/agentscope/model/_openai_response/_model.py b/src/agentscope/model/_openai_response/_model.py
index 1ce73ed9a..3f3ed3834 100644
--- a/src/agentscope/model/_openai_response/_model.py
+++ b/src/agentscope/model/_openai_response/_model.py
@@ -152,6 +152,14 @@ class OpenAIResponseModel(ChatModelBase):
openai.InternalServerError,
)
+ @classmethod
+ def _get_structured_output_fallback_exceptions(
+ cls,
+ ) -> tuple[Type[Exception], ...]:
+ import openai
+
+ return (openai.BadRequestError,)
+
async def _call_api(
self,
model_name: str,
diff --git a/tests/compress_context_test.py b/tests/compress_context_test.py
index 2279ec09d..945bc843c 100644
--- a/tests/compress_context_test.py
+++ b/tests/compress_context_test.py
@@ -81,6 +81,63 @@ class RecordingStructuredMockModel(MockModel):
)
+class FixedPathOffloader:
+ """Return one stable offload path for reminder tests."""
+
+ async def offload_context(
+ self,
+ session_id: str,
+ msgs: list[Msg],
+ ) -> str:
+ """Return the fixed path without persisting test messages."""
+ del session_id, msgs
+ return "sessions/123/context.jsonl"
+
+
+def _make_failing_compression_agent(
+ summary: str = "",
+ offloader: Any = None,
+) -> tuple[Agent, RecordingStructuredMockModel]:
+ """Build an agent whose summary generation fails once."""
+ model = RecordingStructuredMockModel(
+ context_size=100,
+ fail_structured_output_times=1,
+ )
+ agent = Agent(
+ name="Friday",
+ system_prompt="".join(["0" for _ in range(20 * 4)]),
+ model=model,
+ context_config=ContextConfig(
+ trigger_ratio=0.7,
+ reserve_ratio=0.4,
+ ),
+ state=AgentState(
+ session_id="123",
+ summary=summary,
+ context=[
+ UserMsg(
+ "User",
+ "".join(["1" for _ in range(30 * 4)]),
+ id="1",
+ ),
+ AssistantMsg(
+ "Friday",
+ "".join(["2" for _ in range(10 * 4)]),
+ id="2",
+ ),
+ UserMsg(
+ "User",
+ "".join(["3" for _ in range(10 * 4)]),
+ id="3",
+ ),
+ ],
+ ),
+ offloader=offloader,
+ toolkit=Toolkit(),
+ )
+ return agent, model
+
+
def _has_instruction_hint(
messages: list[Msg],
instructions: HintBlock,
@@ -1384,5 +1441,37 @@ class ContextCompressionTest(IsolatedAsyncioTestCase):
),
)
+ async def test_summary_failure_truncates_context(self) -> None:
+ """Summary failures fall back to lossy context truncation."""
+ agent, _ = _make_failing_compression_agent()
+ expected_state = agent.state.model_copy(deep=True)
+ expected_state.context = expected_state.context[1:]
+ expected_state.summary = (
+ "<system-info>Some earlier messages were truncated for limited "
+ "context.</system-info>"
+ )
+
+ await agent.compress_context()
+
+ self.assertEqual(agent.state, expected_state)
+
+ async def test_offload_reminder_is_not_duplicated(self) -> None:
+ """Repeated fallback preserves one reminder for a stable path."""
+ reminder = (
+ "<system-reminder>The compressed context is offloaded to "
+ "'sessions/123/context.jsonl', you can refer to it when "
+ "needed.</system-reminder>"
+ )
+ agent, _ = _make_failing_compression_agent(
+ summary=reminder,
+ offloader=FixedPathOffloader(),
+ )
+ expected_state = agent.state.model_copy(deep=True)
+ expected_state.context = []
+
+ await agent.compress_context()
+
+ self.assertEqual(agent.state, expected_state)
+
async def asyncTearDown(self) -> None:
"""The async teardown method."""
diff --git a/tests/model_base_test.py b/tests/model_base_test.py
index 81d4a9066..a79c3069d 100644
--- a/tests/model_base_test.py
+++ b/tests/model_base_test.py
@@ -3,6 +3,7 @@
retry / accumulation / interrupt wrapper around ``_call_api``."""
import asyncio
import base64
+from typing import Any
from unittest.async_case import IsolatedAsyncioTestCase
from utils import AnyString, MockModel
@@ -16,7 +17,67 @@ from agentscope.message import (
URLSource,
UserMsg,
)
-from agentscope.model import ChatResponse, ChatUsage, FinishedReason
+from agentscope.model import (
+ ChatResponse,
+ ChatUsage,
+ FinishedReason,
+ StructuredResponse,
+)
+from agentscope.exception import StructuredOutputError
+from agentscope.tool import ToolChoice
+
+
+class _BadRequestError(Exception):
+ """A provider "bad request" error used to exercise strategy fallback."""
+
+
+class StructuredOutputStrategyMockModel(MockModel):
+ """Record and control structured-output strategy attempts."""
+
+ def __init__(
+ self,
+ responses: list[StructuredResponse | Exception] | None = None,
+ reject_forced: bool = False,
+ ) -> None:
+ """Initialize the strategy mock model."""
+ super().__init__()
+ self.responses = list(responses or [])
+ self.reject_forced = reject_forced
+ self.structured_calls: list[tuple[str | None, dict[str, Any]]] = []
+
+ def _get_disable_thinking_kwargs(self) -> dict:
+ """Expose a provider-specific thinking toggle."""
+ return {"extra_body": {"enable_thinking": False}}
+
+ @classmethod
+ def _get_structured_output_fallback_exceptions(
+ cls,
+ ) -> tuple[type[Exception], ...]:
+ """Declare the provider error that permits a strategy fallback."""
+ return (_BadRequestError,)
+
+ async def _call_api_with_structured_output(
+ self,
+ model_name: str,
+ messages: list,
+ structured_model: Any,
+ tool_choice: ToolChoice | None = None,
+ **kwargs: Any,
+ ) -> StructuredResponse:
+ """Return or raise the configured result for one strategy."""
+ del model_name, messages, structured_model
+ mode = tool_choice.mode if tool_choice is not None else None
+ self.structured_calls.append((mode, kwargs))
+ await asyncio.sleep(0)
+
+ if self.reject_forced and mode == "generate_structured_output":
+ raise _BadRequestError("tool_choice is unsupported")
+ if self.responses:
+ response = self.responses.pop(0)
+ if isinstance(response, Exception):
+ raise response
+ return response
+ return StructuredResponse(content={})
def _dump(chat_response: ChatResponse) -> dict:
@@ -1068,3 +1129,158 @@ class ChatModelBaseCallTest(IsolatedAsyncioTestCase):
async def asyncTearDown(self) -> None:
"""The async teardown method."""
+
+
+class StructuredOutputStrategyTest(IsolatedAsyncioTestCase):
+ """Test structured-output fallback strategy behavior."""
+
+ async def asyncSetUp(self) -> None:
+ """Prepare a common structured-output request."""
+ self.messages = [UserMsg(name="user", content="hi")]
+ self.schema = {"type": "object"}
+
+ async def test_strategy_order(self) -> None:
+ """Preserve thinking before falling back to disabling it."""
+ model = StructuredOutputStrategyMockModel(
+ responses=[
+ _BadRequestError("tool_choice is unsupported"),
+ StructuredOutputError(
+ "Failed to generate structured output for model.",
+ ),
+ StructuredResponse(content={}),
+ ],
+ )
+
+ await model.generate_structured_output(self.messages, self.schema)
+
+ self.assertEqual(
+ model.structured_calls,
+ [
+ ("generate_structured_output", {}),
+ ("auto", {}),
+ (
+ "generate_structured_output",
+ {"extra_body": {"enable_thinking": False}},
+ ),
+ ],
+ )
+
+ async def test_success_does_not_permanently_promote_strategy(self) -> None:
+ """Every call starts from the strongest immutable strategy."""
+ model = StructuredOutputStrategyMockModel(
+ responses=[
+ _BadRequestError("tool_choice is unsupported"),
+ StructuredResponse(content={}),
+ StructuredResponse(content={}),
+ ],
+ )
+
+ await model.generate_structured_output(self.messages, self.schema)
+ await model.generate_structured_output(self.messages, self.schema)
+
+ self.assertEqual(
+ model.structured_calls,
+ [
+ ("generate_structured_output", {}),
+ ("auto", {}),
+ ("generate_structured_output", {}),
+ ],
+ )
+
+ async def test_explicit_tool_choice_bypasses_fallbacks(self) -> None:
+ """A caller-provided tool choice is forwarded exactly once."""
+ model = StructuredOutputStrategyMockModel(
+ responses=[StructuredResponse(content={})],
+ )
+
+ await model.generate_structured_output(
+ self.messages,
+ self.schema,
+ tool_choice=ToolChoice(mode="auto"),
+ )
+
+ self.assertEqual(model.structured_calls, [("auto", {})])
+
+ async def test_unrelated_error_fails_fast(self) -> None:
+ """Non-compatibility errors do not walk the strategy ladder."""
+ expected = PermissionError("invalid API key")
+ model = StructuredOutputStrategyMockModel(responses=[expected])
+
+ with self.assertRaises(PermissionError) as raised:
+ await model.generate_structured_output(
+ self.messages,
+ self.schema,
+ )
+
+ self.assertEqual(
+ (raised.exception, model.structured_calls),
+ (
+ expected,
+ [("generate_structured_output", {})],
+ ),
+ )
+
+ async def test_final_error_is_chained_from_first_error(self) -> None:
+ """The first provider failure remains visible as the root cause."""
+ first_error = _BadRequestError("tool_choice is unsupported")
+ final_error = StructuredOutputError(
+ "Failed to generate structured output for model.",
+ )
+ model = StructuredOutputStrategyMockModel(
+ responses=[
+ first_error,
+ StructuredOutputError(
+ "Failed to generate structured output for model.",
+ ),
+ StructuredOutputError(
+ "Failed to generate structured output for model.",
+ ),
+ final_error,
+ ],
+ )
+
+ with self.assertRaises(StructuredOutputError) as raised:
+ await model.generate_structured_output(
+ self.messages,
+ self.schema,
+ )
+
+ self.assertEqual(
+ (
+ raised.exception,
+ raised.exception.__cause__,
+ model.structured_calls,
+ ),
+ (
+ final_error,
+ first_error,
+ [
+ ("generate_structured_output", {}),
+ ("auto", {}),
+ (
+ "generate_structured_output",
+ {"extra_body": {"enable_thinking": False}},
+ ),
+ (None, {}),
+ ],
+ ),
+ )
+
+ async def test_concurrent_calls_do_not_share_strategy_state(self) -> None:
+ """Concurrent calls independently traverse immutable strategies."""
+ model = StructuredOutputStrategyMockModel(reject_forced=True)
+
+ await asyncio.gather(
+ model.generate_structured_output(self.messages, self.schema),
+ model.generate_structured_output(self.messages, self.schema),
+ )
+
+ self.assertEqual(
+ model.structured_calls,
+ [
+ ("generate_structured_output", {}),
+ ("generate_structured_output", {}),
+ ("auto", {}),
+ ("auto", {}),
+ ],
+ )
diff --git a/tests/model_dashscope_test.py b/tests/model_dashscope_test.py
index 03a38f0ac..13803b5e8 100644
--- a/tests/model_dashscope_test.py
+++ b/tests/model_dashscope_test.py
@@ -251,6 +251,34 @@ class TestDashScopeNonStream(IsolatedAsyncioTestCase):
),
)
+ async def test_extra_body_is_not_mutated(self) -> None:
+ """Model defaults do not mutate the caller-owned extra body."""
+ mock_create = AsyncMock(
+ return_value=_mock_completion(text="Hello!"),
+ )
+ self.mock_client.chat.completions.create = mock_create
+ extra_body = {"custom": "value"}
+
+ await self.model([], extra_body=extra_body)
+
+ self.assertEqual(
+ (extra_body, mock_create.await_args.kwargs),
+ (
+ {"custom": "value"},
+ {
+ "model": "qwen3-max",
+ "messages": [],
+ "stream": False,
+ "max_tokens": 1000,
+ "extra_body": {
+ "custom": "value",
+ "enable_thinking": True,
+ "thinking_budget": 100,
+ },
+ },
+ ),
+ )
+
# ---------------------------------------------------------------------------
# Streaming tests