Skip to content

Build and run agents you can see, understand and trust.

feat(classifier): add OpenAI Decisions API support (#3174)

YingchaoX98dff833628a
diff --git a/src/agentscope/classifier/__init__.py b/src/agentscope/classifier/__init__.py
index 2811148f5..968037a15 100644
--- a/src/agentscope/classifier/__init__.py
+++ b/src/agentscope/classifier/__init__.py
@@ -14,10 +14,12 @@ from ._response import (
ChoiceAnswer,
ClassifierAnswer,
ClassifierResponse,
+ RefusalAnswer,
ScoreAnswer,
)
from ._usage import ClassifierUsage
from ._jev import JevClassifierModel
+from ._openai import OpenAIClassifierModel
__all__ = [
"BinaryAnswer",
@@ -31,6 +33,8 @@ __all__ = [
"ClassifierResponse",
"ClassifierUsage",
"JevClassifierModel",
+ "OpenAIClassifierModel",
+ "RefusalAnswer",
"ScoreAnswer",
"ScoreQuestion",
]
diff --git a/src/agentscope/classifier/_openai/__init__.py b/src/agentscope/classifier/_openai/__init__.py
new file mode 100644
index 000000000..bf790dfce
--- /dev/null
+++ b/src/agentscope/classifier/_openai/__init__.py
@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+"""The OpenAI Decisions classifier model."""
+
+from ._model import OpenAIClassifierModel
+
+__all__ = ["OpenAIClassifierModel"]
diff --git a/src/agentscope/classifier/_openai/_model.py b/src/agentscope/classifier/_openai/_model.py
new file mode 100644
index 000000000..f5610c7c5
--- /dev/null
+++ b/src/agentscope/classifier/_openai/_model.py
@@ -0,0 +1,219 @@
+# -*- coding: utf-8 -*-
+"""The OpenAI Decisions API classifier implementation."""
+import json
+from time import perf_counter
+from typing import Any, Mapping
+
+import openai
+from pydantic import ConfigDict
+
+from .._base import ClassifierModelBase
+from .._question import (
+ BinaryQuestion,
+ ChoiceQuestion,
+ ClassifierQuestion,
+ ScoreQuestion,
+)
+from .._response import (
+ BinaryAnswer,
+ ChoiceAnswer,
+ ClassifierAnswer,
+ ClassifierResponse,
+ RefusalAnswer,
+ ScoreAnswer,
+)
+from .._usage import ClassifierUsage
+from ...credential import OpenAICredential
+from ...message import Base64Source, DataBlock, TextBlock
+
+
+def _to_sdk_question(name: str, question: ClassifierQuestion) -> dict:
+ """Translate a named classifier question into a Decisions question."""
+ instructions = question.instructions or name
+ match question:
+ case BinaryQuestion():
+ if question.criteria is not None:
+ for outcome, description in question.criteria.model_dump(
+ exclude_none=True,
+ ).items():
+ instructions += f"\n{outcome.capitalize()}: {description}"
+ return {
+ "type": "predicate",
+ "name": name,
+ "instructions": instructions,
+ }
+ case ChoiceQuestion():
+ return {
+ "type": "choice",
+ "name": name,
+ "instructions": instructions,
+ "choices": [
+ {"value": value, **({"description": desc} if desc else {})}
+ for value, desc in question.criteria.items()
+ ],
+ }
+ case ScoreQuestion():
+ return {
+ "type": "score",
+ "name": name,
+ "instructions": instructions,
+ "levels": [{"label": label} for label in question.criteria],
+ }
+ raise TypeError(f"Unsupported classifier question: {type(question)}.")
+
+
+def _from_sdk_answer(answer: Any) -> ClassifierAnswer:
+ """Translate a Decisions answer without changing its probabilities."""
+ match answer.type:
+ case "predicate":
+ return BinaryAnswer(probability=answer.probability)
+ case "choice":
+ return ChoiceAnswer(
+ choice=answer.choice,
+ confidence=answer.confidence,
+ probabilities={
+ p.value: p.probability for p in answer.probabilities
+ },
+ )
+ case "score":
+ return ScoreAnswer(
+ score=answer.score,
+ confidence=answer.confidence,
+ legend={p.value: p.label for p in answer.probabilities},
+ probabilities={
+ p.value: p.probability for p in answer.probabilities
+ },
+ )
+ case "refusal":
+ return RefusalAnswer()
+ raise ValueError(f"Unsupported OpenAI answer type: {answer.type!r}.")
+
+
+def _to_sdk_input(state: str | dict | list[TextBlock | DataBlock]) -> Any:
+ """Convert text, JSON, or ordered text/image blocks to Decisions input."""
+ if isinstance(state, str):
+ return state
+ if isinstance(state, dict):
+ return json.dumps(state, ensure_ascii=False)
+
+ content = []
+ for block in state:
+ if isinstance(block, TextBlock):
+ content.append({"type": "input_text", "text": block.text})
+ elif (
+ isinstance(block, DataBlock)
+ and isinstance(block.source, Base64Source)
+ and block.source.media_type.startswith("image/")
+ ):
+ content.append(
+ {
+ "type": "input_image",
+ "image_url": (
+ f"data:{block.source.media_type};base64,"
+ f"{block.source.data}"
+ ),
+ },
+ )
+ else:
+ raise ValueError(
+ "Decisions input supports TextBlock and image DataBlock "
+ "with Base64Source only.",
+ )
+ return [{"role": "user", "content": content}]
+
+
+class OpenAIClassifierModel(ClassifierModelBase):
+ """A classifier backed by the OpenAI Decisions API.
+
+ Requires ``openai>=3.26.0``. Binary questions map to predicates; choice
+ and score questions retain their alternatives and ordered levels.
+ When instructions are omitted, the question name is used instead.
+ Binary criteria are appended to the instructions as true/false outcomes.
+
+ Refusals are returned as :class:`RefusalAnswer` for the affected question,
+ alongside any other answers from the same request.
+ """
+
+ class Parameters(ClassifierModelBase.Parameters):
+ """Provider-specific classifier parameters."""
+
+ model_config = ConfigDict(extra="forbid")
+
+ def __init__(
+ self,
+ credential: OpenAICredential,
+ model: str = "gpt-6-luna",
+ parameters: "OpenAIClassifierModel.Parameters | None" = None,
+ timeout: float = 30.0,
+ max_retries: int = 2,
+ ) -> None:
+ """Initialize the classifier and its reusable OpenAI client.
+
+ Args:
+ credential (`OpenAICredential`):
+ API key, optional organization and custom endpoint.
+ model (`str`, defaults to ``"gpt-6-luna"``):
+ The model name accepted by the Decisions endpoint.
+ parameters (`OpenAIClassifierModel.Parameters | None`):
+ Provider-specific classifier parameters.
+ timeout (`float`, defaults to `30.0`):
+ Per-request timeout in seconds.
+ max_retries (`int`, defaults to `2`):
+ Maximum retries after the initial request, handled by the SDK.
+ """
+ super().__init__(credential, model, parameters)
+ self.client = openai.AsyncOpenAI(
+ api_key=credential.api_key.get_secret_value(),
+ organization=credential.organization,
+ base_url=credential.base_url,
+ timeout=timeout,
+ max_retries=max_retries,
+ )
+ if not hasattr(self.client, "decisions"):
+ raise ImportError(
+ "OpenAIClassifierModel requires openai>=3.26.0. "
+ "Upgrade with `pip install 'openai>=3.26.0'`.",
+ )
+
+ async def __call__(
+ self,
+ state: str | dict | list[TextBlock | DataBlock],
+ questions: Mapping[str, ClassifierQuestion],
+ **kwargs: Any,
+ ) -> ClassifierResponse:
+ """Evaluate questions about text, JSON, or inline images.
+
+ Args:
+ state (`str | dict | list[TextBlock | DataBlock]`):
+ Shared evidence. Dictionaries are serialized as JSON text.
+ Content lists preserve order and support text plus image
+ blocks with :class:`Base64Source`. URL sources and other
+ media types are not supported.
+ questions (`Mapping[str, ClassifierQuestion]`):
+ Typed questions keyed by their names.
+ **kwargs (`Any`):
+ Options forwarded to ``client.decisions.create``, such as
+ ``safety_identifier``, ``extra_headers``, and ``extra_body``.
+
+ Returns:
+ `ClassifierResponse`:
+ Answers and refusals keyed by question name, with usage.
+ """
+ start_time = perf_counter()
+ response = await self.client.decisions.create(
+ model=self.model,
+ input=_to_sdk_input(state),
+ questions=[
+ _to_sdk_question(name, q) for name, q in questions.items()
+ ],
+ **kwargs,
+ )
+ return ClassifierResponse(
+ model=response.model,
+ content={a.name: _from_sdk_answer(a) for a in response.answers},
+ usage=ClassifierUsage(
+ input_tokens=response.usage.input_tokens,
+ output_tokens=response.usage.output_tokens,
+ time=perf_counter() - start_time,
+ ),
+ )
diff --git a/src/agentscope/classifier/_response.py b/src/agentscope/classifier/_response.py
index 9043025eb..d3916e914 100644
--- a/src/agentscope/classifier/_response.py
+++ b/src/agentscope/classifier/_response.py
@@ -69,8 +69,20 @@ class ScoreAnswer(DictMixin):
"""The answer type discriminator."""
+@dataclass
+class RefusalAnswer(DictMixin):
+ """A refusal to answer one question, independent of other answers."""
+
+ type: Literal["refusal_answer"] = field(
+ default_factory=lambda: "refusal_answer",
+ )
+ """The answer type discriminator."""
+
+
#: A provider-independent classifier answer.
-ClassifierAnswer: TypeAlias = BinaryAnswer | ChoiceAnswer | ScoreAnswer
+ClassifierAnswer: TypeAlias = (
+ BinaryAnswer | ChoiceAnswer | ScoreAnswer | RefusalAnswer
+)
@dataclass
diff --git a/tests/classifier_openai_test.py b/tests/classifier_openai_test.py
new file mode 100644
index 000000000..1f8891bd8
--- /dev/null
+++ b/tests/classifier_openai_test.py
@@ -0,0 +1,285 @@
+# -*- coding: utf-8 -*-
+"""Tests for the OpenAI Decisions classifier adapter."""
+from dataclasses import asdict
+from types import SimpleNamespace
+from unittest import IsolatedAsyncioTestCase
+from unittest.mock import AsyncMock, MagicMock, patch
+
+from utils import AnyValue
+from agentscope import classifier
+from agentscope.classifier import (
+ BinaryCriteria,
+ BinaryQuestion,
+ ChoiceQuestion,
+ ScoreQuestion,
+)
+from agentscope.credential import OpenAICredential
+from agentscope.message import Base64Source, DataBlock, TextBlock, URLSource
+
+
+class OpenAIClassifierModelTest(IsolatedAsyncioTestCase):
+ """Exercise request and response translation without network calls."""
+
+ def setUp(self) -> None:
+ """Replace the OpenAI client with an asynchronous stub."""
+ client_patch = patch("openai.AsyncOpenAI")
+ self.client_cls = client_patch.start()
+ self.addCleanup(client_patch.stop)
+ self.client = self.client_cls.return_value
+ self.client.decisions.create = AsyncMock(
+ return_value=SimpleNamespace(
+ model="gpt-6-luna",
+ answers=[],
+ usage=SimpleNamespace(input_tokens=12, output_tokens=0),
+ ),
+ )
+ self.credential = OpenAICredential(api_key="secret")
+
+ async def test_call_translates_all_answers(self) -> None:
+ """Keep all answers, including a refusal, and provider usage."""
+ self.client.decisions.create.return_value.answers = [
+ SimpleNamespace(type="predicate", name="urgent", probability=0.8),
+ SimpleNamespace(
+ type="choice",
+ name="route",
+ choice="billing",
+ confidence=0.9,
+ probabilities=[
+ SimpleNamespace(value="billing", probability=0.9),
+ SimpleNamespace(value="support", probability=0.1),
+ ],
+ ),
+ SimpleNamespace(
+ type="score",
+ name="priority",
+ score=1.7,
+ confidence=0.85,
+ probabilities=[
+ SimpleNamespace(value=0, label="low", probability=0.05),
+ SimpleNamespace(value=1, label="medium", probability=0.2),
+ SimpleNamespace(value=2, label="high", probability=0.75),
+ ],
+ ),
+ SimpleNamespace(type="refusal", name="restricted"),
+ ]
+ model = classifier.OpenAIClassifierModel(
+ OpenAICredential(
+ api_key="secret",
+ organization="org-test",
+ base_url="https://openai.example/v1",
+ ),
+ timeout=10,
+ max_retries=4,
+ )
+ response = await model(
+ state="I was charged twice.",
+ questions={
+ "urgent": BinaryQuestion(
+ instructions="Is this urgent?",
+ criteria=BinaryCriteria(true="Urgent."),
+ ),
+ "route": ChoiceQuestion(
+ instructions="Select a route.",
+ criteria={"billing": None, "support": None},
+ ),
+ "priority": ScoreQuestion(
+ instructions="Rate priority.",
+ criteria=["low", "medium", "high"],
+ ),
+ "restricted": BinaryQuestion(),
+ },
+ extra_headers={"X-Test": "value"},
+ )
+ self.assertDictEqual(
+ self.client_cls.call_args.kwargs,
+ {
+ "api_key": "secret",
+ "organization": "org-test",
+ "base_url": "https://openai.example/v1",
+ "timeout": 10,
+ "max_retries": 4,
+ },
+ )
+ self.assertDictEqual(
+ self.client.decisions.create.call_args.kwargs,
+ {
+ "model": "gpt-6-luna",
+ "input": "I was charged twice.",
+ "questions": [
+ {
+ "type": "predicate",
+ "name": "urgent",
+ "instructions": "Is this urgent?\nTrue: Urgent.",
+ },
+ {
+ "type": "choice",
+ "name": "route",
+ "instructions": "Select a route.",
+ "choices": [
+ {"value": "billing"},
+ {"value": "support"},
+ ],
+ },
+ {
+ "type": "score",
+ "name": "priority",
+ "instructions": "Rate priority.",
+ "levels": [
+ {"label": "low"},
+ {"label": "medium"},
+ {"label": "high"},
+ ],
+ },
+ {
+ "type": "predicate",
+ "name": "restricted",
+ "instructions": "restricted",
+ },
+ ],
+ "extra_headers": {"X-Test": "value"},
+ },
+ )
+ self.assertDictEqual(
+ asdict(response),
+ {
+ "model": "gpt-6-luna",
+ "content": {
+ "urgent": {"type": "binary_answer", "probability": 0.8},
+ "route": {
+ "type": "choice_answer",
+ "choice": "billing",
+ "confidence": 0.9,
+ "probabilities": {"billing": 0.9, "support": 0.1},
+ },
+ "priority": {
+ "type": "score_answer",
+ "score": 1.7,
+ "confidence": 0.85,
+ "legend": {0: "low", 1: "medium", 2: "high"},
+ "probabilities": {0: 0.05, 1: 0.2, 2: 0.75},
+ },
+ "restricted": {"type": "refusal_answer"},
+ },
+ "usage": {
+ "time": AnyValue(),
+ "input_tokens": 12,
+ "output_tokens": 0,
+ "type": "classifier",
+ },
+ "id": AnyValue(),
+ "created_at": AnyValue(),
+ "type": "classifier_response",
+ "metadata": {},
+ },
+ )
+
+ async def test_input_conversion(self) -> None:
+ """Convert input and retain criteria when instructions are omitted."""
+ model = classifier.OpenAIClassifierModel(self.credential)
+ questions = {
+ "safe": BinaryQuestion(criteria=BinaryCriteria(false="Unsafe.")),
+ "route": ChoiceQuestion(criteria={"billing": "Payments."}),
+ "priority": ScoreQuestion(criteria=["low", "high"]),
+ }
+ for state, expected in [
+ ({"text": "你好"}, '{"text": "你好"}'),
+ (
+ [
+ TextBlock(text="Compare."),
+ DataBlock(
+ source=Base64Source(
+ data="YWJj",
+ media_type="image/png",
+ ),
+ ),
+ DataBlock(
+ source=Base64Source(
+ data="ZGVm",
+ media_type="image/jpeg",
+ ),
+ ),
+ ],
+ [
+ {
+ "role": "user",
+ "content": [
+ {"type": "input_text", "text": "Compare."},
+ {
+ "type": "input_image",
+ "image_url": "data:image/png;base64,YWJj",
+ },
+ {
+ "type": "input_image",
+ "image_url": "data:image/jpeg;base64,ZGVm",
+ },
+ ],
+ },
+ ],
+ ),
+ ]:
+ with self.subTest(state=state):
+ await model(state, questions)
+ self.assertDictEqual(
+ self.client.decisions.create.call_args.kwargs,
+ {
+ "model": "gpt-6-luna",
+ "input": expected,
+ "questions": [
+ {
+ "type": "predicate",
+ "name": "safe",
+ "instructions": "safe\nFalse: Unsafe.",
+ },
+ {
+ "type": "choice",
+ "name": "route",
+ "instructions": "route",
+ "choices": [
+ {
+ "value": "billing",
+ "description": "Payments.",
+ },
+ ],
+ },
+ {
+ "type": "score",
+ "name": "priority",
+ "instructions": "priority",
+ "levels": [
+ {"label": "low"},
+ {"label": "high"},
+ ],
+ },
+ ],
+ },
+ )
+
+ async def test_unsupported_media_is_rejected(self) -> None:
+ """Do not silently discard unsupported input or fetch URL sources."""
+ model = classifier.OpenAIClassifierModel(self.credential)
+ for source in [
+ Base64Source(data="YWJj", media_type="audio/wav"),
+ URLSource(url="https://example.com/a.png", media_type="image/png"),
+ ]:
+ with self.subTest(source=source):
+ with self.assertRaisesRegex(ValueError, "Base64Source"):
+ await model(
+ [DataBlock(source=source)],
+ {"safe": BinaryQuestion()},
+ )
+ self.client.decisions.create.assert_not_awaited()
+
+ async def test_provider_error_is_raised(self) -> None:
+ """Leave retry and error handling to the SDK."""
+ error = RuntimeError("failed")
+ self.client.decisions.create.side_effect = error
+ model = classifier.OpenAIClassifierModel(self.credential)
+ with self.assertRaises(RuntimeError) as caught:
+ await model("hello", {"safe": BinaryQuestion()})
+ self.assertIs(caught.exception, error)
+
+ async def test_old_sdk_has_clear_error(self) -> None:
+ """Fail with an upgrade instruction when Decisions is unavailable."""
+ self.client_cls.return_value = MagicMock(spec=[])
+ with self.assertRaisesRegex(ImportError, "openai>=3.26.0"):
+ classifier.OpenAIClassifierModel(self.credential)
diff --git a/tests/classifier_test.py b/tests/classifier_test.py
index 987f98f86..55f9dcb58 100644
--- a/tests/classifier_test.py
+++ b/tests/classifier_test.py
@@ -14,6 +14,7 @@ from agentscope.classifier import (
ClassifierModelBase,
ClassifierQuestion,
ClassifierResponse,
+ RefusalAnswer,
ScoreQuestion,
)
from agentscope.credential import CredentialBase
@@ -90,6 +91,13 @@ class ClassifierQuestionTest(TestCase):
class ClassifierModelBaseTest(IsolatedAsyncioTestCase):
"""Test the classifier model contract."""
+ async def test_refusal_answer(self) -> None:
+ """A refusal carries no fabricated score or explanation."""
+ self.assertDictEqual(
+ asdict(RefusalAnswer()),
+ {"type": "refusal_answer"},
+ )
+
async def test_call(self) -> None:
"""A call returns the typed answers keyed by question name."""
response = await _MockClassifier()(
diff --git a/tests/middleware_model_router_test.py b/tests/middleware_model_router_test.py
index 1a9366860..6e6bc8b0b 100644
--- a/tests/middleware_model_router_test.py
+++ b/tests/middleware_model_router_test.py
@@ -2,6 +2,7 @@
"""Tests for the model router middleware."""
from typing import Any, Mapping
from unittest import IsolatedAsyncioTestCase
+from unittest.mock import AsyncMock, patch
from pydantic import BaseModel
@@ -12,6 +13,7 @@ from agentscope.classifier import (
ClassifierModelBase,
ClassifierQuestion,
ClassifierResponse,
+ RefusalAnswer,
)
from agentscope.credential import CredentialBase
from agentscope.event import ModelCallStartEvent, ReplyStartEvent
@@ -341,6 +343,32 @@ class ModelRouterMiddlewareTest(IsolatedAsyncioTestCase):
{"ModelRouterMiddleware": {agent.state.reply_id: None}},
)
+ async def test_refusal_keeps_the_agents_model(self) -> None:
+ """A classifier refusal leaves the reply on the original model."""
+ middleware = ModelRouterMiddleware(
+ _MockClassifier([]),
+ self.candidates,
+ )
+ with patch.object(
+ _MockClassifier,
+ "__call__",
+ new=AsyncMock(
+ return_value=ClassifierResponse(
+ model="classifier",
+ content={"chat_model": RefusalAnswer()},
+ ),
+ ),
+ ):
+ agent, called = await self._reply(
+ middleware,
+ [TextBlock(text="Hi.")],
+ )
+ self.assertListEqual(called, ["primary"])
+ self.assertDictEqual(
+ agent.state.middle_context,
+ {"ModelRouterMiddleware": {agent.state.reply_id: None}},
+ )
+
def test_duplicate_candidates_are_rejected(self) -> None:
"""Candidate names must be unique."""
with self.assertRaisesRegex(ValueError, "Duplicate"):