Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
feat(classifier): add OpenAI Decisions API support (#3174)
YingchaoX98dff833628a
diff --git a/src/agentscope/classifier/__init__.py b/src/agentscope/classifier/__init__.pyindex 2811148f5..968037a15 100644--- a/src/agentscope/classifier/__init__.py+++ b/src/agentscope/classifier/__init__.py@@ -14,10 +14,12 @@ from ._response import (ChoiceAnswer,ClassifierAnswer,ClassifierResponse,+ RefusalAnswer,ScoreAnswer,)from ._usage import ClassifierUsagefrom ._jev import JevClassifierModel+from ._openai import OpenAIClassifierModel__all__ = ["BinaryAnswer",@@ -31,6 +33,8 @@ __all__ = ["ClassifierResponse","ClassifierUsage","JevClassifierModel",+ "OpenAIClassifierModel",+ "RefusalAnswer","ScoreAnswer","ScoreQuestion",]diff --git a/src/agentscope/classifier/_openai/__init__.py b/src/agentscope/classifier/_openai/__init__.pynew file mode 100644index 000000000..bf790dfce--- /dev/null+++ b/src/agentscope/classifier/_openai/__init__.py@@ -0,0 +1,6 @@+# -*- coding: utf-8 -*-+"""The OpenAI Decisions classifier model."""++from ._model import OpenAIClassifierModel++__all__ = ["OpenAIClassifierModel"]diff --git a/src/agentscope/classifier/_openai/_model.py b/src/agentscope/classifier/_openai/_model.pynew file mode 100644index 000000000..f5610c7c5--- /dev/null+++ b/src/agentscope/classifier/_openai/_model.py@@ -0,0 +1,219 @@+# -*- coding: utf-8 -*-+"""The OpenAI Decisions API classifier implementation."""+import json+from time import perf_counter+from typing import Any, Mapping++import openai+from pydantic import ConfigDict++from .._base import ClassifierModelBase+from .._question import (+ BinaryQuestion,+ ChoiceQuestion,+ ClassifierQuestion,+ ScoreQuestion,+)+from .._response import (+ BinaryAnswer,+ ChoiceAnswer,+ ClassifierAnswer,+ ClassifierResponse,+ RefusalAnswer,+ ScoreAnswer,+)+from .._usage import ClassifierUsage+from ...credential import OpenAICredential+from ...message import Base64Source, DataBlock, TextBlock+++def _to_sdk_question(name: str, question: ClassifierQuestion) -> dict:+ """Translate a named classifier question into a Decisions question."""+ instructions = question.instructions or name+ match question:+ case BinaryQuestion():+ if question.criteria is not None:+ for outcome, description in question.criteria.model_dump(+ exclude_none=True,+ ).items():+ instructions += f"\n{outcome.capitalize()}: {description}"+ return {+ "type": "predicate",+ "name": name,+ "instructions": instructions,+ }+ case ChoiceQuestion():+ return {+ "type": "choice",+ "name": name,+ "instructions": instructions,+ "choices": [+ {"value": value, **({"description": desc} if desc else {})}+ for value, desc in question.criteria.items()+ ],+ }+ case ScoreQuestion():+ return {+ "type": "score",+ "name": name,+ "instructions": instructions,+ "levels": [{"label": label} for label in question.criteria],+ }+ raise TypeError(f"Unsupported classifier question: {type(question)}.")+++def _from_sdk_answer(answer: Any) -> ClassifierAnswer:+ """Translate a Decisions answer without changing its probabilities."""+ match answer.type:+ case "predicate":+ return BinaryAnswer(probability=answer.probability)+ case "choice":+ return ChoiceAnswer(+ choice=answer.choice,+ confidence=answer.confidence,+ probabilities={+ p.value: p.probability for p in answer.probabilities+ },+ )+ case "score":+ return ScoreAnswer(+ score=answer.score,+ confidence=answer.confidence,+ legend={p.value: p.label for p in answer.probabilities},+ probabilities={+ p.value: p.probability for p in answer.probabilities+ },+ )+ case "refusal":+ return RefusalAnswer()+ raise ValueError(f"Unsupported OpenAI answer type: {answer.type!r}.")+++def _to_sdk_input(state: str | dict | list[TextBlock | DataBlock]) -> Any:+ """Convert text, JSON, or ordered text/image blocks to Decisions input."""+ if isinstance(state, str):+ return state+ if isinstance(state, dict):+ return json.dumps(state, ensure_ascii=False)++ content = []+ for block in state:+ if isinstance(block, TextBlock):+ content.append({"type": "input_text", "text": block.text})+ elif (+ isinstance(block, DataBlock)+ and isinstance(block.source, Base64Source)+ and block.source.media_type.startswith("image/")+ ):+ content.append(+ {+ "type": "input_image",+ "image_url": (+ f"data:{block.source.media_type};base64,"+ f"{block.source.data}"+ ),+ },+ )+ else:+ raise ValueError(+ "Decisions input supports TextBlock and image DataBlock "+ "with Base64Source only.",+ )+ return [{"role": "user", "content": content}]+++class OpenAIClassifierModel(ClassifierModelBase):+ """A classifier backed by the OpenAI Decisions API.++ Requires ``openai>=3.26.0``. Binary questions map to predicates; choice+ and score questions retain their alternatives and ordered levels.+ When instructions are omitted, the question name is used instead.+ Binary criteria are appended to the instructions as true/false outcomes.++ Refusals are returned as :class:`RefusalAnswer` for the affected question,+ alongside any other answers from the same request.+ """++ class Parameters(ClassifierModelBase.Parameters):+ """Provider-specific classifier parameters."""++ model_config = ConfigDict(extra="forbid")++ def __init__(+ self,+ credential: OpenAICredential,+ model: str = "gpt-6-luna",+ parameters: "OpenAIClassifierModel.Parameters | None" = None,+ timeout: float = 30.0,+ max_retries: int = 2,+ ) -> None:+ """Initialize the classifier and its reusable OpenAI client.++ Args:+ credential (`OpenAICredential`):+ API key, optional organization and custom endpoint.+ model (`str`, defaults to ``"gpt-6-luna"``):+ The model name accepted by the Decisions endpoint.+ parameters (`OpenAIClassifierModel.Parameters | None`):+ Provider-specific classifier parameters.+ timeout (`float`, defaults to `30.0`):+ Per-request timeout in seconds.+ max_retries (`int`, defaults to `2`):+ Maximum retries after the initial request, handled by the SDK.+ """+ super().__init__(credential, model, parameters)+ self.client = openai.AsyncOpenAI(+ api_key=credential.api_key.get_secret_value(),+ organization=credential.organization,+ base_url=credential.base_url,+ timeout=timeout,+ max_retries=max_retries,+ )+ if not hasattr(self.client, "decisions"):+ raise ImportError(+ "OpenAIClassifierModel requires openai>=3.26.0. "+ "Upgrade with `pip install 'openai>=3.26.0'`.",+ )++ async def __call__(+ self,+ state: str | dict | list[TextBlock | DataBlock],+ questions: Mapping[str, ClassifierQuestion],+ **kwargs: Any,+ ) -> ClassifierResponse:+ """Evaluate questions about text, JSON, or inline images.++ Args:+ state (`str | dict | list[TextBlock | DataBlock]`):+ Shared evidence. Dictionaries are serialized as JSON text.+ Content lists preserve order and support text plus image+ blocks with :class:`Base64Source`. URL sources and other+ media types are not supported.+ questions (`Mapping[str, ClassifierQuestion]`):+ Typed questions keyed by their names.+ **kwargs (`Any`):+ Options forwarded to ``client.decisions.create``, such as+ ``safety_identifier``, ``extra_headers``, and ``extra_body``.++ Returns:+ `ClassifierResponse`:+ Answers and refusals keyed by question name, with usage.+ """+ start_time = perf_counter()+ response = await self.client.decisions.create(+ model=self.model,+ input=_to_sdk_input(state),+ questions=[+ _to_sdk_question(name, q) for name, q in questions.items()+ ],+ **kwargs,+ )+ return ClassifierResponse(+ model=response.model,+ content={a.name: _from_sdk_answer(a) for a in response.answers},+ usage=ClassifierUsage(+ input_tokens=response.usage.input_tokens,+ output_tokens=response.usage.output_tokens,+ time=perf_counter() - start_time,+ ),+ )diff --git a/src/agentscope/classifier/_response.py b/src/agentscope/classifier/_response.pyindex 9043025eb..d3916e914 100644--- a/src/agentscope/classifier/_response.py+++ b/src/agentscope/classifier/_response.py@@ -69,8 +69,20 @@ class ScoreAnswer(DictMixin):"""The answer type discriminator."""+@dataclass+class RefusalAnswer(DictMixin):+ """A refusal to answer one question, independent of other answers."""++ type: Literal["refusal_answer"] = field(+ default_factory=lambda: "refusal_answer",+ )+ """The answer type discriminator."""++#: A provider-independent classifier answer.-ClassifierAnswer: TypeAlias = BinaryAnswer | ChoiceAnswer | ScoreAnswer+ClassifierAnswer: TypeAlias = (+ BinaryAnswer | ChoiceAnswer | ScoreAnswer | RefusalAnswer+)@dataclassdiff --git a/tests/classifier_openai_test.py b/tests/classifier_openai_test.pynew file mode 100644index 000000000..1f8891bd8--- /dev/null+++ b/tests/classifier_openai_test.py@@ -0,0 +1,285 @@+# -*- coding: utf-8 -*-+"""Tests for the OpenAI Decisions classifier adapter."""+from dataclasses import asdict+from types import SimpleNamespace+from unittest import IsolatedAsyncioTestCase+from unittest.mock import AsyncMock, MagicMock, patch++from utils import AnyValue+from agentscope import classifier+from agentscope.classifier import (+ BinaryCriteria,+ BinaryQuestion,+ ChoiceQuestion,+ ScoreQuestion,+)+from agentscope.credential import OpenAICredential+from agentscope.message import Base64Source, DataBlock, TextBlock, URLSource+++class OpenAIClassifierModelTest(IsolatedAsyncioTestCase):+ """Exercise request and response translation without network calls."""++ def setUp(self) -> None:+ """Replace the OpenAI client with an asynchronous stub."""+ client_patch = patch("openai.AsyncOpenAI")+ self.client_cls = client_patch.start()+ self.addCleanup(client_patch.stop)+ self.client = self.client_cls.return_value+ self.client.decisions.create = AsyncMock(+ return_value=SimpleNamespace(+ model="gpt-6-luna",+ answers=[],+ usage=SimpleNamespace(input_tokens=12, output_tokens=0),+ ),+ )+ self.credential = OpenAICredential(api_key="secret")++ async def test_call_translates_all_answers(self) -> None:+ """Keep all answers, including a refusal, and provider usage."""+ self.client.decisions.create.return_value.answers = [+ SimpleNamespace(type="predicate", name="urgent", probability=0.8),+ SimpleNamespace(+ type="choice",+ name="route",+ choice="billing",+ confidence=0.9,+ probabilities=[+ SimpleNamespace(value="billing", probability=0.9),+ SimpleNamespace(value="support", probability=0.1),+ ],+ ),+ SimpleNamespace(+ type="score",+ name="priority",+ score=1.7,+ confidence=0.85,+ probabilities=[+ SimpleNamespace(value=0, label="low", probability=0.05),+ SimpleNamespace(value=1, label="medium", probability=0.2),+ SimpleNamespace(value=2, label="high", probability=0.75),+ ],+ ),+ SimpleNamespace(type="refusal", name="restricted"),+ ]+ model = classifier.OpenAIClassifierModel(+ OpenAICredential(+ api_key="secret",+ organization="org-test",+ base_url="https://openai.example/v1",+ ),+ timeout=10,+ max_retries=4,+ )+ response = await model(+ state="I was charged twice.",+ questions={+ "urgent": BinaryQuestion(+ instructions="Is this urgent?",+ criteria=BinaryCriteria(true="Urgent."),+ ),+ "route": ChoiceQuestion(+ instructions="Select a route.",+ criteria={"billing": None, "support": None},+ ),+ "priority": ScoreQuestion(+ instructions="Rate priority.",+ criteria=["low", "medium", "high"],+ ),+ "restricted": BinaryQuestion(),+ },+ extra_headers={"X-Test": "value"},+ )+ self.assertDictEqual(+ self.client_cls.call_args.kwargs,+ {+ "api_key": "secret",+ "organization": "org-test",+ "base_url": "https://openai.example/v1",+ "timeout": 10,+ "max_retries": 4,+ },+ )+ self.assertDictEqual(+ self.client.decisions.create.call_args.kwargs,+ {+ "model": "gpt-6-luna",+ "input": "I was charged twice.",+ "questions": [+ {+ "type": "predicate",+ "name": "urgent",+ "instructions": "Is this urgent?\nTrue: Urgent.",+ },+ {+ "type": "choice",+ "name": "route",+ "instructions": "Select a route.",+ "choices": [+ {"value": "billing"},+ {"value": "support"},+ ],+ },+ {+ "type": "score",+ "name": "priority",+ "instructions": "Rate priority.",+ "levels": [+ {"label": "low"},+ {"label": "medium"},+ {"label": "high"},+ ],+ },+ {+ "type": "predicate",+ "name": "restricted",+ "instructions": "restricted",+ },+ ],+ "extra_headers": {"X-Test": "value"},+ },+ )+ self.assertDictEqual(+ asdict(response),+ {+ "model": "gpt-6-luna",+ "content": {+ "urgent": {"type": "binary_answer", "probability": 0.8},+ "route": {+ "type": "choice_answer",+ "choice": "billing",+ "confidence": 0.9,+ "probabilities": {"billing": 0.9, "support": 0.1},+ },+ "priority": {+ "type": "score_answer",+ "score": 1.7,+ "confidence": 0.85,+ "legend": {0: "low", 1: "medium", 2: "high"},+ "probabilities": {0: 0.05, 1: 0.2, 2: 0.75},+ },+ "restricted": {"type": "refusal_answer"},+ },+ "usage": {+ "time": AnyValue(),+ "input_tokens": 12,+ "output_tokens": 0,+ "type": "classifier",+ },+ "id": AnyValue(),+ "created_at": AnyValue(),+ "type": "classifier_response",+ "metadata": {},+ },+ )++ async def test_input_conversion(self) -> None:+ """Convert input and retain criteria when instructions are omitted."""+ model = classifier.OpenAIClassifierModel(self.credential)+ questions = {+ "safe": BinaryQuestion(criteria=BinaryCriteria(false="Unsafe.")),+ "route": ChoiceQuestion(criteria={"billing": "Payments."}),+ "priority": ScoreQuestion(criteria=["low", "high"]),+ }+ for state, expected in [+ ({"text": "你好"}, '{"text": "你好"}'),+ (+ [+ TextBlock(text="Compare."),+ DataBlock(+ source=Base64Source(+ data="YWJj",+ media_type="image/png",+ ),+ ),+ DataBlock(+ source=Base64Source(+ data="ZGVm",+ media_type="image/jpeg",+ ),+ ),+ ],+ [+ {+ "role": "user",+ "content": [+ {"type": "input_text", "text": "Compare."},+ {+ "type": "input_image",+ "image_url": "data:image/png;base64,YWJj",+ },+ {+ "type": "input_image",+ "image_url": "data:image/jpeg;base64,ZGVm",+ },+ ],+ },+ ],+ ),+ ]:+ with self.subTest(state=state):+ await model(state, questions)+ self.assertDictEqual(+ self.client.decisions.create.call_args.kwargs,+ {+ "model": "gpt-6-luna",+ "input": expected,+ "questions": [+ {+ "type": "predicate",+ "name": "safe",+ "instructions": "safe\nFalse: Unsafe.",+ },+ {+ "type": "choice",+ "name": "route",+ "instructions": "route",+ "choices": [+ {+ "value": "billing",+ "description": "Payments.",+ },+ ],+ },+ {+ "type": "score",+ "name": "priority",+ "instructions": "priority",+ "levels": [+ {"label": "low"},+ {"label": "high"},+ ],+ },+ ],+ },+ )++ async def test_unsupported_media_is_rejected(self) -> None:+ """Do not silently discard unsupported input or fetch URL sources."""+ model = classifier.OpenAIClassifierModel(self.credential)+ for source in [+ Base64Source(data="YWJj", media_type="audio/wav"),+ URLSource(url="https://example.com/a.png", media_type="image/png"),+ ]:+ with self.subTest(source=source):+ with self.assertRaisesRegex(ValueError, "Base64Source"):+ await model(+ [DataBlock(source=source)],+ {"safe": BinaryQuestion()},+ )+ self.client.decisions.create.assert_not_awaited()++ async def test_provider_error_is_raised(self) -> None:+ """Leave retry and error handling to the SDK."""+ error = RuntimeError("failed")+ self.client.decisions.create.side_effect = error+ model = classifier.OpenAIClassifierModel(self.credential)+ with self.assertRaises(RuntimeError) as caught:+ await model("hello", {"safe": BinaryQuestion()})+ self.assertIs(caught.exception, error)++ async def test_old_sdk_has_clear_error(self) -> None:+ """Fail with an upgrade instruction when Decisions is unavailable."""+ self.client_cls.return_value = MagicMock(spec=[])+ with self.assertRaisesRegex(ImportError, "openai>=3.26.0"):+ classifier.OpenAIClassifierModel(self.credential)diff --git a/tests/classifier_test.py b/tests/classifier_test.pyindex 987f98f86..55f9dcb58 100644--- a/tests/classifier_test.py+++ b/tests/classifier_test.py@@ -14,6 +14,7 @@ from agentscope.classifier import (ClassifierModelBase,ClassifierQuestion,ClassifierResponse,+ RefusalAnswer,ScoreQuestion,)from agentscope.credential import CredentialBase@@ -90,6 +91,13 @@ class ClassifierQuestionTest(TestCase):class ClassifierModelBaseTest(IsolatedAsyncioTestCase):"""Test the classifier model contract."""+ async def test_refusal_answer(self) -> None:+ """A refusal carries no fabricated score or explanation."""+ self.assertDictEqual(+ asdict(RefusalAnswer()),+ {"type": "refusal_answer"},+ )+async def test_call(self) -> None:"""A call returns the typed answers keyed by question name."""response = await _MockClassifier()(diff --git a/tests/middleware_model_router_test.py b/tests/middleware_model_router_test.pyindex 1a9366860..6e6bc8b0b 100644--- a/tests/middleware_model_router_test.py+++ b/tests/middleware_model_router_test.py@@ -2,6 +2,7 @@"""Tests for the model router middleware."""from typing import Any, Mappingfrom unittest import IsolatedAsyncioTestCase+from unittest.mock import AsyncMock, patchfrom pydantic import BaseModel@@ -12,6 +13,7 @@ from agentscope.classifier import (ClassifierModelBase,ClassifierQuestion,ClassifierResponse,+ RefusalAnswer,)from agentscope.credential import CredentialBasefrom agentscope.event import ModelCallStartEvent, ReplyStartEvent@@ -341,6 +343,32 @@ class ModelRouterMiddlewareTest(IsolatedAsyncioTestCase):{"ModelRouterMiddleware": {agent.state.reply_id: None}},)+ async def test_refusal_keeps_the_agents_model(self) -> None:+ """A classifier refusal leaves the reply on the original model."""+ middleware = ModelRouterMiddleware(+ _MockClassifier([]),+ self.candidates,+ )+ with patch.object(+ _MockClassifier,+ "__call__",+ new=AsyncMock(+ return_value=ClassifierResponse(+ model="classifier",+ content={"chat_model": RefusalAnswer()},+ ),+ ),+ ):+ agent, called = await self._reply(+ middleware,+ [TextBlock(text="Hi.")],+ )+ self.assertListEqual(called, ["primary"])+ self.assertDictEqual(+ agent.state.middle_context,+ {"ModelRouterMiddleware": {agent.state.reply_id: None}},+ )+def test_duplicate_candidates_are_rejected(self) -> None:"""Candidate names must be unique."""with self.assertRaisesRegex(ValueError, "Duplicate"):