Skip to content

Build and run agents you can see, understand and trust.

feat(asr): add transcription interface and OpenAI batch model

wangtao812eaecd7816
diff --git a/src/agentscope/asr/__init__.py b/src/agentscope/asr/__init__.py
new file mode 100644
index 000000000..5da4cdfd7
--- /dev/null
+++ b/src/agentscope/asr/__init__.py
@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+"""Automatic speech recognition models."""
+
+from ._asr_base import ASRModelBase
+from ._asr_model_card import ASRModelCard
+from ._asr_response import ASRResponse
+from ._openai import OpenAIASRModel
+
+__all__ = [
+ "ASRModelBase",
+ "ASRModelCard",
+ "ASRResponse",
+ "OpenAIASRModel",
+]
diff --git a/src/agentscope/asr/_asr_base.py b/src/agentscope/asr/_asr_base.py
new file mode 100644
index 000000000..e9cddd202
--- /dev/null
+++ b/src/agentscope/asr/_asr_base.py
@@ -0,0 +1,239 @@
+# -*- coding: utf-8 -*-
+"""Base interface for speech recognition models."""
+
+from abc import ABC, abstractmethod
+import inspect
+from pathlib import Path
+from typing import Any, TYPE_CHECKING
+
+from pydantic import BaseModel
+
+from .._logging import logger
+from ..credential import CredentialBase
+from ._asr_response import ASRResponse
+
+if TYPE_CHECKING:
+ from ._asr_model_card import ASRModelCard
+
+
+class ASRModelBase(ABC):
+ """Base class for batch and realtime speech recognition models.
+
+ Batch implementations transcribe complete audio by calling
+ :meth:`transcribe` directly. Realtime implementations use the asynchronous
+ context manager, call :meth:`push` for each audio chunk, and finally call
+ :meth:`transcribe` without audio to finalize the current transcription.
+
+ Every concrete implementation must override :meth:`transcribe`. Realtime
+ implementations must additionally override :meth:`connect`,
+ :meth:`close`, and :meth:`push`.
+ """
+
+ class Parameters(BaseModel):
+ """Base schema for provider-specific transcription parameters.
+
+ Concrete ASR implementations should override this class with the
+ parameters supported by their provider.
+ """
+
+ credential: CredentialBase
+ """The credential used to authenticate against the ASR provider."""
+
+ model: str
+ """The name of the ASR model."""
+
+ parameters: BaseModel
+ """The provider-specific ASR model parameters."""
+
+ realtime: bool = False
+ """Whether the model supports realtime streaming input."""
+
+ def __init__(
+ self,
+ credential: CredentialBase,
+ model: str,
+ parameters: BaseModel | None = None,
+ ) -> None:
+ """Initialize the ASR model.
+
+ Args:
+ credential (`CredentialBase`):
+ The credential used to authenticate with the provider.
+ model (`str`):
+ The provider model name.
+ parameters (`BaseModel | None`, defaults to `None`):
+ Provider-specific parameters. When omitted, an instance of
+ :attr:`Parameters` is created.
+ """
+ self.credential = credential
+ self.model = model
+ self.parameters = parameters or self.Parameters()
+
+ async def __aenter__(self) -> "ASRModelBase":
+ """Enter the model's asynchronous context.
+
+ Realtime models establish their provider connection by calling
+ :meth:`connect`. Entering a batch model is a no-op.
+
+ Returns:
+ `ASRModelBase`:
+ This model instance.
+ """
+ if self.realtime:
+ await self.connect()
+ return self
+
+ async def __aexit__(
+ self,
+ exc_type: Any,
+ exc_value: Any,
+ traceback: Any,
+ ) -> None:
+ """Exit the model's asynchronous context.
+
+ Realtime models close their provider connection by calling
+ :meth:`close`, including when the context exits with an exception.
+ Exiting a batch model is a no-op.
+
+ Args:
+ exc_type (`Any`):
+ The type of the exception that caused the context to exit, or
+ ``None`` for a normal exit.
+ exc_value (`Any`):
+ The exception instance, or ``None`` for a normal exit.
+ traceback (`Any`):
+ The exception traceback, or ``None`` for a normal exit.
+ """
+ if self.realtime:
+ await self.close()
+
+ async def connect(self) -> None:
+ """Establish the realtime provider connection.
+
+ Realtime implementations must override this method and prepare any
+ provider resources needed by :meth:`push` and :meth:`transcribe`. The
+ default is a no-op so batch implementations do not need lifecycle
+ hooks.
+ """
+ return
+
+ async def close(self) -> None:
+ """Close the realtime provider connection.
+
+ Realtime implementations must override this method and release any
+ resources created by :meth:`connect`. The default is a no-op so batch
+ implementations do not need lifecycle hooks.
+ """
+ return
+
+ async def push( # pylint: disable=unused-argument
+ self,
+ audio: bytes,
+ **kwargs: Any,
+ ) -> ASRResponse:
+ """Send an audio chunk and return currently available text.
+
+ Realtime implementations must override this method. It should send the
+ audio without waiting for the complete transcription and return any
+ text already available from the provider.
+
+ Args:
+ audio (`bytes`):
+ An incremental audio chunk in the format expected by the
+ provider.
+ **kwargs (`Any`):
+ Provider-specific options for this audio chunk.
+
+ Returns:
+ `ASRResponse`:
+ A non-final incremental response. Its content is ``None`` when
+ no new text is available yet.
+
+ Raises:
+ RuntimeError:
+ If this model does not support realtime audio input.
+ """
+ raise RuntimeError(
+ f"{type(self).__name__} does not support realtime audio input",
+ )
+
+ @classmethod
+ def list_models(
+ cls,
+ custom_yaml_dir: str | None = None,
+ ) -> list["ASRModelCard"]:
+ """List model cards associated with the concrete implementation.
+
+ Cards whose ``realtime`` value differs from the implementation are
+ excluded. Invalid cards are logged and skipped so they do not prevent
+ other cards from being discovered.
+
+ Args:
+ custom_yaml_dir (`str | None`, defaults to `None`):
+ Directory containing model-card YAML files. When omitted, the
+ ``_models`` directory beside the concrete implementation is
+ used.
+
+ Returns:
+ `list[ASRModelCard]`:
+ Valid model cards compatible with the implementation's
+ realtime mode.
+ """
+ from ._asr_model_card import ASRModelCard
+
+ yaml_dir = (
+ Path(custom_yaml_dir)
+ if custom_yaml_dir is not None
+ else Path(inspect.getfile(cls)).parent / "_models"
+ )
+ model_cards = []
+ for path in sorted(yaml_dir.glob("*.yaml")):
+ try:
+ card = ASRModelCard.from_yaml(
+ str(path),
+ cls.Parameters,
+ )
+ if card.realtime != cls.realtime:
+ continue
+ model_cards.append(card)
+ # Keep one invalid card from hiding other valid cards.
+ except Exception as error:
+ logger.warning(
+ "Failed to load ASR model card %s: %s",
+ path,
+ error,
+ )
+ return model_cards
+
+ @abstractmethod
+ async def transcribe(
+ self,
+ audio: bytes | None = None,
+ filename: str = "audio.wav",
+ **kwargs: Any,
+ ) -> ASRResponse:
+ """Transcribe complete audio or finalize realtime input.
+
+ Batch subclasses require complete ``audio``. Realtime subclasses use
+ ``audio=None`` to finalize previously pushed chunks and may accept a
+ final audio chunk before finalizing. The final response has
+ ``is_last=True``.
+
+ ``filename`` supplies the format extension required by batch
+ multipart APIs when ``audio`` comes from memory.
+
+ Args:
+ audio (`bytes | None`, defaults to `None`):
+ Complete in-memory audio for a batch model. For a realtime
+ model, ``None`` finalizes previously pushed audio; a byte
+ string may provide one last chunk before finalization.
+ filename (`str`, defaults to `"audio.wav"`):
+ Filename metadata used by multipart batch APIs to identify the
+ in-memory audio format.
+ **kwargs (`Any`):
+ Provider-specific transcription options.
+
+ Returns:
+ `ASRResponse`:
+ The completed transcription response with ``is_last=True``.
+ """
diff --git a/src/agentscope/asr/_asr_model_card.py b/src/agentscope/asr/_asr_model_card.py
new file mode 100644
index 000000000..8cb3f6adc
--- /dev/null
+++ b/src/agentscope/asr/_asr_model_card.py
@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+"""Model metadata for speech recognition implementations."""
+
+from typing import Literal, Self, Type
+
+from pydantic import BaseModel, Field
+import yaml
+
+
+class ASRModelCard(BaseModel):
+ """A model and its supported audio input types."""
+
+ type: Literal["asr_model"] = "asr_model"
+ """The model card type discriminator."""
+
+ name: str = Field(
+ description="The model identifier used in provider API calls.",
+ )
+ """The model identifier used in provider API calls."""
+
+ label: str = Field(description="The human-readable model label.")
+ """The human-readable model label."""
+
+ input_types: list[str] = Field(
+ default_factory=list,
+ description="The supported audio input media types.",
+ )
+ """The supported audio input media types."""
+
+ output_types: list[str] = Field(
+ default_factory=lambda: ["text/plain"],
+ description="The supported transcription output media types.",
+ )
+ """The supported transcription output media types."""
+
+ realtime: bool = Field(
+ default=False,
+ description="Whether the model supports realtime streaming input.",
+ )
+ """Whether the model supports realtime streaming input."""
+
+ parameter_schema: dict = Field(
+ description="The JSON schema for provider-specific parameters.",
+ )
+ """The JSON schema for provider-specific parameters."""
+
+ @classmethod
+ def from_yaml(
+ cls,
+ yaml_path: str,
+ parameter_class: Type[BaseModel],
+ ) -> Self:
+ """Load a card and the implementation's parameter schema."""
+ with open(yaml_path, "r", encoding="utf-8") as file:
+ config = yaml.safe_load(file)
+ return cls(
+ name=config["name"],
+ label=config["label"],
+ input_types=config.get("input_types", []),
+ output_types=config.get("output_types", ["text/plain"]),
+ realtime=config.get("realtime", False),
+ parameter_schema=parameter_class.model_json_schema(),
+ )
diff --git a/src/agentscope/asr/_asr_response.py b/src/agentscope/asr/_asr_response.py
new file mode 100644
index 000000000..701638976
--- /dev/null
+++ b/src/agentscope/asr/_asr_response.py
@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+"""Speech recognition response."""
+
+from dataclasses import dataclass, field
+from typing import Literal
+
+from .._utils._common import _generate_timestamp, _get_timestamp
+from .._utils._mixin import DictMixin
+from ..message import TextBlock
+from ..types import JSONSerializableObject
+
+
+@dataclass
+class ASRResponse(DictMixin):
+ """The text produced by an ASR model."""
+
+ content: TextBlock | None
+ """The transcribed text content, or ``None`` when no text is ready."""
+
+ id: str = field(default_factory=lambda: _get_timestamp(True))
+ """The unique identifier of the response."""
+
+ created_at: str = field(default_factory=_generate_timestamp)
+ """The timestamp when the response was created."""
+
+ type: Literal["asr"] = field(default_factory=lambda: "asr")
+ """The response type discriminator."""
+
+ metadata: dict[str, JSONSerializableObject] | None = field(
+ default_factory=lambda: None,
+ )
+ """Optional provider-specific response metadata."""
+
+ is_last: bool = field(default_factory=lambda: True)
+ """Whether this is the last response in the transcription stream."""
diff --git a/src/agentscope/asr/_openai/__init__.py b/src/agentscope/asr/_openai/__init__.py
new file mode 100644
index 000000000..08670effe
--- /dev/null
+++ b/src/agentscope/asr/_openai/__init__.py
@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+"""OpenAI speech recognition model."""
+
+from ._model import OpenAIASRModel
+
+__all__ = ["OpenAIASRModel"]
diff --git a/src/agentscope/asr/_openai/_model.py b/src/agentscope/asr/_openai/_model.py
new file mode 100644
index 000000000..2ccc960c9
--- /dev/null
+++ b/src/agentscope/asr/_openai/_model.py
@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+"""OpenAI Audio Transcriptions API implementation."""
+
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+from ...credential import OpenAICredential
+from ...message import TextBlock
+from .._asr_base import ASRModelBase
+from .._asr_response import ASRResponse
+
+
+class OpenAIASRModel(ASRModelBase):
+ """Batch speech recognition using OpenAI's Audio Transcriptions API."""
+
+ class Parameters(BaseModel):
+ """Options supported by OpenAI transcription models."""
+
+ language: str | None = Field(
+ default=None,
+ description="Optional ISO-639-1 language code for the audio.",
+ )
+ prompt: str | None = Field(
+ default=None,
+ description="Optional transcription guidance.",
+ )
+
+ type: Literal["openai_asr"] = "openai_asr"
+ """The model implementation type discriminator."""
+
+ realtime: bool = False
+ """OpenAI Audio Transcriptions accepts complete audio files."""
+
+ def __init__(
+ self,
+ credential: OpenAICredential,
+ model: str = "gpt-4o-mini-transcribe",
+ parameters: "OpenAIASRModel.Parameters | None" = None,
+ ) -> None:
+ super().__init__(credential, model, parameters)
+
+ import openai
+
+ self.client: openai.AsyncClient = openai.AsyncClient(
+ api_key=credential.api_key.get_secret_value(),
+ organization=credential.organization,
+ base_url=credential.base_url,
+ )
+
+ async def transcribe(
+ self,
+ audio: bytes | None = None,
+ filename: str = "audio.wav",
+ **kwargs: Any,
+ ) -> ASRResponse:
+ """Transcribe in-memory audio, using ``filename`` for its format."""
+ if not audio:
+ raise ValueError("audio must not be empty")
+ if not filename:
+ raise ValueError("filename must not be empty")
+
+ options = self.parameters.model_dump(exclude_none=True)
+ options.update(kwargs)
+ result = await self.client.audio.transcriptions.create(
+ file=(filename, audio),
+ model=self.model,
+ response_format="json",
+ **options,
+ )
+ return ASRResponse(content=TextBlock(text=result.text))
diff --git a/src/agentscope/asr/_openai/_models/gpt-4o-mini-transcribe.yaml b/src/agentscope/asr/_openai/_models/gpt-4o-mini-transcribe.yaml
new file mode 100644
index 000000000..d566f16a7
--- /dev/null
+++ b/src/agentscope/asr/_openai/_models/gpt-4o-mini-transcribe.yaml
@@ -0,0 +1,9 @@
+name: gpt-4o-mini-transcribe
+label: GPT-4o Mini Transcribe
+input_types:
+ - audio/flac
+ - audio/mpeg
+ - audio/mp4
+ - audio/ogg
+ - audio/wav
+ - audio/webm
diff --git a/src/agentscope/asr/_openai/_models/gpt-4o-transcribe.yaml b/src/agentscope/asr/_openai/_models/gpt-4o-transcribe.yaml
new file mode 100644
index 000000000..58493deeb
--- /dev/null
+++ b/src/agentscope/asr/_openai/_models/gpt-4o-transcribe.yaml
@@ -0,0 +1,9 @@
+name: gpt-4o-transcribe
+label: GPT-4o Transcribe
+input_types:
+ - audio/flac
+ - audio/mpeg
+ - audio/mp4
+ - audio/ogg
+ - audio/wav
+ - audio/webm
diff --git a/src/agentscope/credential/_base.py b/src/agentscope/credential/_base.py
index 262141706..7ec8f8e69 100644
--- a/src/agentscope/credential/_base.py
+++ b/src/agentscope/credential/_base.py
@@ -7,6 +7,7 @@ from pydantic import BaseModel, Field
from .._utils._common import _generate_id
if TYPE_CHECKING:
+ from ..asr import ASRModelBase, ASRModelCard
from ..embedding import EmbeddingModelBase
from ..model import ChatModelBase, ModelCard
from ..realtime import RealtimeModelBase, RealtimeModelCard
@@ -55,6 +56,19 @@ class CredentialBase(BaseModel):
"""
return []
+ @classmethod
+ def get_asr_model_classes(cls) -> list[Type["ASRModelBase"]]:
+ """Return supported speech recognition implementations."""
+ return []
+
+ @classmethod
+ def list_asr_models(cls) -> list["ASRModelCard"]:
+ """List candidate ASR models for this credential."""
+ cards: list["ASRModelCard"] = []
+ for asr_cls in cls.get_asr_model_classes():
+ cards.extend(asr_cls.list_models())
+ return cards
+
@classmethod
def list_tts_models(cls) -> list["TTSModelCard"]:
"""List the candidate TTS models available under this credential.
diff --git a/src/agentscope/credential/_openai.py b/src/agentscope/credential/_openai.py
index 8470df358..2f5cea3f9 100644
--- a/src/agentscope/credential/_openai.py
+++ b/src/agentscope/credential/_openai.py
@@ -7,6 +7,7 @@ from pydantic import ConfigDict, Field, SecretStr
from ._base import CredentialBase
if TYPE_CHECKING:
+ from ..asr import ASRModelBase
from ..embedding import EmbeddingModelBase
from ..model import ChatModelBase
from ..realtime import RealtimeModelBase
@@ -64,6 +65,13 @@ class OpenAICredential(CredentialBase):
return [OpenAITTSModel]
+ @classmethod
+ def get_asr_model_classes(cls) -> list[Type["ASRModelBase"]]:
+ """Return the OpenAI transcription implementation."""
+ from ..asr import OpenAIASRModel
+
+ return [OpenAIASRModel]
+
@classmethod
def get_realtime_model_classes(cls) -> list[Type["RealtimeModelBase"]]:
"""Return the OpenAI realtime model classes."""
diff --git a/tests/asr_openai_test.py b/tests/asr_openai_test.py
new file mode 100644
index 000000000..e56ef159e
--- /dev/null
+++ b/tests/asr_openai_test.py
@@ -0,0 +1,507 @@
+# -*- coding: utf-8 -*-
+"""Unit tests for batch and realtime ASR model behavior."""
+
+from typing import Any
+from unittest import IsolatedAsyncioTestCase
+from unittest.mock import AsyncMock, MagicMock, patch
+
+from agentscope.asr import ASRModelBase, ASRResponse, OpenAIASRModel
+from agentscope.credential import OpenAICredential
+from agentscope.message import TextBlock
+
+
+class _RealtimeASRModel(ASRModelBase):
+ """Minimal realtime implementation used to verify the base lifecycle."""
+
+ realtime: bool = True
+
+ def __init__(self, credential: OpenAICredential) -> None:
+ """Initialize the test model."""
+ super().__init__(credential=credential, model="realtime-test")
+ self.events: list[str] = []
+ self.push_options: list[dict[str, Any]] = []
+ self.response_index = 0
+
+ async def connect(self) -> None:
+ """Record connection establishment."""
+ self.events.append("connect")
+
+ async def close(self) -> None:
+ """Record connection closure."""
+ self.events.append("close")
+
+ async def push(
+ self,
+ audio: bytes,
+ **kwargs: Any,
+ ) -> ASRResponse:
+ """Send audio and return an available transcription event."""
+ text = audio.decode("utf-8")
+ self.events.append(f"push:{text}")
+ self.push_options.append(kwargs)
+ self.response_index += 1
+ content = (
+ TextBlock(
+ text=text,
+ id=f"text-id-{self.response_index}",
+ created_at=f"text-created-at-{self.response_index}",
+ )
+ if text
+ else None
+ )
+ return ASRResponse(
+ content=content,
+ id=f"response-id-{self.response_index}",
+ created_at=f"response-created-at-{self.response_index}",
+ is_last=False,
+ )
+
+ async def transcribe( # pylint: disable=unused-argument
+ self,
+ audio: bytes | None = None,
+ filename: str = "audio.wav",
+ **kwargs: Any,
+ ) -> ASRResponse:
+ """Finalize realtime input and return the last event."""
+ text = f"{audio.decode('utf-8')}." if audio is not None else "."
+ self.events.append(
+ f"transcribe:{audio.decode('utf-8')}"
+ if audio is not None
+ else "transcribe",
+ )
+ self.response_index += 1
+ return ASRResponse(
+ content=TextBlock(
+ text=text,
+ id=f"text-id-{self.response_index}",
+ created_at=f"text-created-at-{self.response_index}",
+ ),
+ id=f"response-id-{self.response_index}",
+ created_at=f"response-created-at-{self.response_index}",
+ )
+
+
+class _IncompleteASRModel(ASRModelBase):
+ """ASR implementation missing the required transcription method."""
+
+
+class TestASRModelBase(IsolatedAsyncioTestCase):
+ """Test the shared ASR model lifecycle."""
+
+ def test_requires_transcribe_implementation(self) -> None:
+ """Reject concrete models without a transcription implementation."""
+ model_class: Any = _IncompleteASRModel
+
+ with self.assertRaisesRegex(
+ TypeError,
+ "Can't instantiate abstract class _IncompleteASRModel",
+ ):
+ type.__call__(
+ model_class,
+ credential=OpenAICredential(api_key="test"),
+ model="incomplete-test",
+ )
+
+ async def test_realtime_lifecycle(self) -> None:
+ """Push incremental audio and finalize within one connection."""
+ model = _RealtimeASRModel(OpenAICredential(api_key="test"))
+
+ async with model:
+ responses = [
+ await model.push(b"hello ", sample_rate=16000),
+ await model.push(b"world", sample_rate=16000),
+ await model.transcribe(),
+ ]
+
+ self.assertEqual(
+ model.events,
+ [
+ "connect",
+ "push:hello ",
+ "push:world",
+ "transcribe",
+ "close",
+ ],
+ )
+ self.assertEqual(
+ model.push_options,
+ [
+ {"sample_rate": 16000},
+ {"sample_rate": 16000},
+ ],
+ )
+ self.assertEqual(
+ [dict(response) for response in responses],
+ [
+ {
+ "content": TextBlock(
+ text="hello ",
+ id="text-id-1",
+ created_at="text-created-at-1",
+ ),
+ "id": "response-id-1",
+ "created_at": "response-created-at-1",
+ "type": "asr",
+ "metadata": None,
+ "is_last": False,
+ },
+ {
+ "content": TextBlock(
+ text="world",
+ id="text-id-2",
+ created_at="text-created-at-2",
+ ),
+ "id": "response-id-2",
+ "created_at": "response-created-at-2",
+ "type": "asr",
+ "metadata": None,
+ "is_last": False,
+ },
+ {
+ "content": TextBlock(
+ text=".",
+ id="text-id-3",
+ created_at="text-created-at-3",
+ ),
+ "id": "response-id-3",
+ "created_at": "response-created-at-3",
+ "type": "asr",
+ "metadata": None,
+ "is_last": True,
+ },
+ ],
+ )
+
+ async def test_closes_realtime_model_on_error(self) -> None:
+ """Close the realtime connection when processing fails."""
+ model = _RealtimeASRModel(OpenAICredential(api_key="test"))
+
+ with self.assertRaisesRegex(RuntimeError, "processing stopped"):
+ async with model:
+ response = await model.push(b"hello")
+ raise RuntimeError("processing stopped")
+
+ self.assertEqual(
+ dict(response),
+ {
+ "content": TextBlock(
+ text="hello",
+ id="text-id-1",
+ created_at="text-created-at-1",
+ ),
+ "id": "response-id-1",
+ "created_at": "response-created-at-1",
+ "type": "asr",
+ "metadata": None,
+ "is_last": False,
+ },
+ )
+ self.assertEqual(
+ model.events,
+ [
+ "connect",
+ "push:hello",
+ "close",
+ ],
+ )
+
+ async def test_returns_empty_realtime_response(self) -> None:
+ """Represent a push with no available text using empty content."""
+ model = _RealtimeASRModel(OpenAICredential(api_key="test"))
+
+ async with model:
+ responses = [
+ await model.push(b""),
+ await model.push(b"hello"),
+ await model.transcribe(),
+ ]
+
+ self.assertEqual(
+ [dict(response) for response in responses],
+ [
+ {
+ "content": None,
+ "id": "response-id-1",
+ "created_at": "response-created-at-1",
+ "type": "asr",
+ "metadata": None,
+ "is_last": False,
+ },
+ {
+ "content": TextBlock(
+ text="hello",
+ id="text-id-2",
+ created_at="text-created-at-2",
+ ),
+ "id": "response-id-2",
+ "created_at": "response-created-at-2",
+ "type": "asr",
+ "metadata": None,
+ "is_last": False,
+ },
+ {
+ "content": TextBlock(
+ text=".",
+ id="text-id-3",
+ created_at="text-created-at-3",
+ ),
+ "id": "response-id-3",
+ "created_at": "response-created-at-3",
+ "type": "asr",
+ "metadata": None,
+ "is_last": True,
+ },
+ ],
+ )
+ self.assertEqual(
+ model.events,
+ [
+ "connect",
+ "push:",
+ "push:hello",
+ "transcribe",
+ "close",
+ ],
+ )
+
+ async def test_realtime_transcribe_accepts_final_audio(self) -> None:
+ """Send an optional final chunk while finalizing realtime input."""
+ model = _RealtimeASRModel(OpenAICredential(api_key="test"))
+
+ async with model:
+ response = await model.transcribe(b"last")
+
+ self.assertEqual(
+ dict(response),
+ {
+ "content": TextBlock(
+ text="last.",
+ id="text-id-1",
+ created_at="text-created-at-1",
+ ),
+ "id": "response-id-1",
+ "created_at": "response-created-at-1",
+ "type": "asr",
+ "metadata": None,
+ "is_last": True,
+ },
+ )
+ self.assertEqual(
+ model.events,
+ [
+ "connect",
+ "transcribe:last",
+ "close",
+ ],
+ )
+
+ async def test_batch_model_rejects_realtime_input(self) -> None:
+ """Reject realtime operations on a batch-only model."""
+ model = OpenAIASRModel(
+ credential=OpenAICredential(api_key="test"),
+ )
+
+ with self.assertRaisesRegex(
+ RuntimeError,
+ "OpenAIASRModel does not support realtime audio input",
+ ):
+ await model.push(b"audio")
+
+
+class TestOpenAIASRModel(IsolatedAsyncioTestCase):
+ """Test transcription without making network requests."""
+
+ def test_response_uses_timestamp_factory(self) -> None:
+ """Use the configurable timestamp factory for response creation."""
+ content = TextBlock(
+ text="Hello world",
+ id="text-id",
+ created_at="text-created-at",
+ )
+ with patch(
+ "agentscope._utils._common._timestamp_factory",
+ return_value="FROZEN-TS",
+ ):
+ response = ASRResponse(
+ content=content,
+ id="response-id",
+ metadata={"language": "en"},
+ is_last=False,
+ )
+
+ self.assertEqual(
+ dict(response),
+ {
+ "content": content,
+ "id": "response-id",
+ "created_at": "FROZEN-TS",
+ "type": "asr",
+ "metadata": {"language": "en"},
+ "is_last": False,
+ },
+ )
+ self.assertEqual(
+ (response.metadata, response.is_last),
+ ({"language": "en"}, False),
+ )
+
+ def make_model(
+ self,
+ parameters: OpenAIASRModel.Parameters | None = None,
+ ) -> OpenAIASRModel:
+ """Build a model whose transcription call is stubbed out."""
+ model = OpenAIASRModel(
+ credential=OpenAICredential(api_key="test"),
+ parameters=parameters,
+ )
+ model.client = MagicMock()
+ model.client.audio.transcriptions.create = AsyncMock(
+ return_value=MagicMock(text="Hello world"),
+ )
+ return model
+
+ async def test_transcribes_audio(self) -> None:
+ """Transcribe audio and send the default model and format."""
+ model = self.make_model()
+ with (
+ patch(
+ "agentscope._utils._common._id_factory",
+ return_value="text-id",
+ ),
+ patch(
+ "agentscope._utils._common._timestamp_factory",
+ return_value="FROZEN-TS",
+ ),
+ patch(
+ "agentscope.asr._asr_response._get_timestamp",
+ return_value="response-id",
+ ),
+ ):
+ response = await model.transcribe(
+ b"audio bytes",
+ "sample.mp3",
+ )
+
+ self.assertIsInstance(model, ASRModelBase)
+ self.assertIsInstance(response, ASRResponse)
+ self.assertEqual(
+ dict(response),
+ {
+ "content": TextBlock(
+ text="Hello world",
+ id="text-id",
+ created_at="FROZEN-TS",
+ ),
+ "id": "response-id",
+ "created_at": "FROZEN-TS",
+ "type": "asr",
+ "metadata": None,
+ "is_last": True,
+ },
+ )
+ model.client.audio.transcriptions.create.assert_awaited_once_with(
+ file=("sample.mp3", b"audio bytes"),
+ model="gpt-4o-mini-transcribe",
+ response_format="json",
+ )
+
+ async def test_forwards_parameters(self) -> None:
+ """Forward the optional language and prompt parameters."""
+ model = self.make_model(
+ OpenAIASRModel.Parameters(language="en", prompt="AgentScope"),
+ )
+ await model.transcribe(b"audio")
+
+ model.client.audio.transcriptions.create.assert_awaited_once_with(
+ file=("audio.wav", b"audio"),
+ model="gpt-4o-mini-transcribe",
+ response_format="json",
+ language="en",
+ prompt="AgentScope",
+ )
+
+ async def test_call_parameters_override_model_parameters(self) -> None:
+ """Prefer per-call options over configured model parameters."""
+ model = self.make_model(
+ OpenAIASRModel.Parameters(language="en", prompt="AgentScope"),
+ )
+ await model.transcribe(
+ b"audio",
+ language="fr",
+ prompt="Override",
+ )
+
+ model.client.audio.transcriptions.create.assert_awaited_once_with(
+ file=("audio.wav", b"audio"),
+ model="gpt-4o-mini-transcribe",
+ response_format="json",
+ language="fr",
+ prompt="Override",
+ )
+
+ async def test_rejects_empty_audio(self) -> None:
+ """Reject empty audio before reaching the API."""
+ model = self.make_model()
+ for audio in [None, b""]:
+ with (
+ self.subTest(audio=audio),
+ self.assertRaisesRegex(
+ ValueError,
+ "audio must not be empty",
+ ),
+ ):
+ await model.transcribe(audio)
+ model.client.audio.transcriptions.create.assert_not_awaited()
+
+ async def test_rejects_missing_filename(self) -> None:
+ """Reject a missing filename before reaching the API."""
+ model = self.make_model()
+ with self.assertRaisesRegex(ValueError, "filename must not be empty"):
+ await model.transcribe(b"audio", "")
+ model.client.audio.transcriptions.create.assert_not_awaited()
+
+ async def test_credential_discovers_models(self) -> None:
+ """Expose the ASR model class and its cards via the credential."""
+ self.assertEqual(
+ OpenAICredential.get_asr_model_classes(),
+ [OpenAIASRModel],
+ )
+ cards = OpenAICredential.list_asr_models()
+ parameter_schema = OpenAIASRModel.Parameters.model_json_schema()
+ self.assertEqual(
+ [card.model_dump(mode="json") for card in cards],
+ [
+ {
+ "type": "asr_model",
+ "name": "gpt-4o-mini-transcribe",
+ "label": "GPT-4o Mini Transcribe",
+ "input_types": [
+ "audio/flac",
+ "audio/mpeg",
+ "audio/mp4",
+ "audio/ogg",
+ "audio/wav",
+ "audio/webm",
+ ],
+ "output_types": ["text/plain"],
+ "realtime": False,
+ "parameter_schema": parameter_schema,
+ },
+ {
+ "type": "asr_model",
+ "name": "gpt-4o-transcribe",
+ "label": "GPT-4o Transcribe",
+ "input_types": [
+ "audio/flac",
+ "audio/mpeg",
+ "audio/mp4",
+ "audio/ogg",
+ "audio/wav",
+ "audio/webm",
+ ],
+ "output_types": ["text/plain"],
+ "realtime": False,
+ "parameter_schema": parameter_schema,
+ },
+ ],
+ )