Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix: load Agent text-to-speech voices from the dedicated endpoint (#42575)

Joelbc38ce59f74a
diff --git a/api/controllers/console/agent/app_helpers.py b/api/controllers/console/agent/app_helpers.py
index 0de65d5a89f..de401351980 100644
--- a/api/controllers/console/agent/app_helpers.py
+++ b/api/controllers/console/agent/app_helpers.py
@@ -19,3 +19,8 @@ def resolve_agent_runtime_app_model(*, session: Session, tenant_id: str, agent_i
"""
return AgentRosterService(session).get_agent_runtime_app_model(tenant_id=tenant_id, agent_id=str(agent_id))
+
+
+def resolve_existing_agent_runtime_app_model(*, session: Session, tenant_id: str, agent_id: UUID) -> App:
+ """Resolve persisted runtime state for read-only Agent endpoints."""
+ return AgentRosterService(session).get_existing_agent_runtime_app_model(tenant_id=tenant_id, agent_id=str(agent_id))
diff --git a/api/controllers/console/app/audio.py b/api/controllers/console/app/audio.py
index f6c8cc43ed6..14e9642408c 100644
--- a/api/controllers/console/app/audio.py
+++ b/api/controllers/console/app/audio.py
@@ -9,10 +9,13 @@ from werkzeug.datastructures import FileStorage
from werkzeug.exceptions import HTTPException, InternalServerError
import services
-from controllers.common.rbac import PlainApp, RBACCheck, enforce_rbac_checks
+from controllers.common.rbac import AgentId, PlainApp, RBACCheck, enforce_rbac_checks
from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models
from controllers.console import console_ns
-from controllers.console.agent.app_helpers import resolve_agent_runtime_app_model
+from controllers.console.agent.app_helpers import (
+ resolve_agent_runtime_app_model,
+ resolve_existing_agent_runtime_app_model,
+)
from controllers.console.app.error import (
AppUnavailableError,
AudioTooLargeError,
@@ -36,10 +39,12 @@ from controllers.console.wraps import (
with_current_tenant_id,
with_current_user,
)
+from core.base.tts.audio_mime import SUPPORTED_TTS_AUDIO_MIME_TYPES
from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
from extensions.ext_database import db
from fields.base import ResponseModel
from graphon.model_runtime.errors.invoke import InvokeError
+from libs.flask_restx_compat import BINARY_RESPONSE_MEDIA_TYPES_VENDOR_KEY
from libs.helper import dump_response
from libs.login import current_user, login_required
from models import Account, App, AppMode
@@ -262,6 +267,53 @@ class AgentChatMessageAudioApi(Resource):
)
+def _text_to_speech(*, payload: TextToSpeechPayload, app_model: App, session: Session, account_id: str | None):
+ try:
+ message_ref = None
+ if payload.message_id:
+ app_ref = AppRefService.create_app_ref(app_model)
+ message_ref = AppRefService.create_message_ref(
+ app_ref,
+ payload.message_id,
+ account_id=account_id,
+ )
+
+ return AudioService.transcript_tts(
+ app_model=app_model,
+ session=session,
+ text=payload.text,
+ voice=payload.voice,
+ message_ref=message_ref,
+ is_draft=True,
+ )
+ except services.errors.app_model_config.AppModelConfigBrokenError:
+ logger.exception("App model config broken.")
+ raise AppUnavailableError()
+ except NoAudioUploadedServiceError:
+ raise NoAudioUploadedError()
+ except AudioTooLargeServiceError as e:
+ raise AudioTooLargeError(str(e))
+ except UnsupportedAudioTypeServiceError:
+ raise UnsupportedAudioTypeError()
+ except ProviderNotSupportSpeechToTextServiceError:
+ raise ProviderNotSupportSpeechToTextError()
+ except ProviderTokenNotInitError as ex:
+ raise ProviderNotInitializeError(ex.description)
+ except QuotaExceededError:
+ raise ProviderQuotaExceededError()
+ except ModelCurrentlyNotSupportError:
+ raise ProviderModelCurrentlyNotSupportError()
+ except InvokeError as e:
+ raise CompletionRequestError(e.description)
+ except HTTPException:
+ raise
+ except ValueError:
+ raise
+ except Exception as e:
+ logger.exception("Failed to convert text to speech")
+ raise InternalServerError() from e
+
+
@console_ns.route("/apps/<uuid:app_id>/text-to-audio")
class ChatMessageTextApi(Resource):
@console_ns.doc("chat_message_text_to_speech")
@@ -277,49 +329,91 @@ class ChatMessageTextApi(Resource):
@get_app_model
@model_validate(TextToSpeechPayload)
def post(self, req_data: TextToSpeechPayload, app_model: App):
- try:
- message_ref = None
- if req_data.message_id:
- app_ref = AppRefService.create_app_ref(app_model)
- message_ref = AppRefService.create_message_ref(
- app_ref,
- req_data.message_id,
- account_id=current_user.id,
- )
-
- # response-contract:ignore
- return AudioService.transcript_tts(
- app_model=app_model,
- session=db.session(),
- text=req_data.text,
- voice=req_data.voice,
- message_ref=message_ref,
- is_draft=True,
- )
- except services.errors.app_model_config.AppModelConfigBrokenError:
- logger.exception("App model config broken.")
- raise AppUnavailableError()
- except NoAudioUploadedServiceError:
- raise NoAudioUploadedError()
- except AudioTooLargeServiceError as e:
- raise AudioTooLargeError(str(e))
- except UnsupportedAudioTypeServiceError:
- raise UnsupportedAudioTypeError()
- except ProviderNotSupportSpeechToTextServiceError:
- raise ProviderNotSupportSpeechToTextError()
- except ProviderTokenNotInitError as ex:
- raise ProviderNotInitializeError(ex.description)
- except QuotaExceededError:
- raise ProviderQuotaExceededError()
- except ModelCurrentlyNotSupportError:
- raise ProviderModelCurrentlyNotSupportError()
- except InvokeError as e:
- raise CompletionRequestError(e.description)
- except ValueError as e:
- raise e
- except Exception as e:
- logger.exception("Failed to handle post request to ChatMessageTextApi")
- raise InternalServerError()
+ # response-contract:ignore provider audio bytes
+ return _text_to_speech(
+ payload=req_data,
+ app_model=app_model,
+ session=db.session(),
+ account_id=current_user.id if req_data.message_id else None,
+ )
+
+
+@console_ns.route("/agent/<uuid:agent_id>/text-to-audio")
+class AgentChatMessageTextApi(Resource):
+ @console_ns.doc("agent_chat_message_text_to_speech")
+ @console_ns.doc(description="Preview an Agent TTS voice or read an Agent chat message")
+ @console_ns.doc(params={"agent_id": "Agent ID"})
+ @console_ns.doc(
+ produces=list(SUPPORTED_TTS_AUDIO_MIME_TYPES),
+ vendor={BINARY_RESPONSE_MEDIA_TYPES_VENDOR_KEY: list(SUPPORTED_TTS_AUDIO_MIME_TYPES)},
+ )
+ @console_ns.expect(console_ns.models[TextToSpeechPayload.__name__])
+ @console_ns.response(200, "Generated audio bytes in the provider audio format")
+ @console_ns.response(400, "Invalid text or voice")
+ @console_ns.response(403, "Insufficient permissions")
+ @console_ns.response(404, "Agent not found")
+ @setup_required
+ @login_required
+ @account_initialization_required
+ @edit_permission_required
+ @with_current_user
+ @with_current_tenant_id
+ @with_session(write=False)
+ @model_validate(TextToSpeechPayload)
+ def post(
+ self,
+ req_data: TextToSpeechPayload,
+ session: Session,
+ current_tenant_id: str,
+ current_user: Account,
+ agent_id: UUID,
+ ):
+ enforce_rbac_checks(
+ tenant_id=current_tenant_id,
+ account_id=current_user.id,
+ checks=[RBACCheck(RBACPermission.AGENT_TEST_AND_RUN, AgentId())],
+ path_args={"agent_id": str(agent_id)},
+ )
+ app_model = resolve_existing_agent_runtime_app_model(
+ session=session,
+ tenant_id=current_tenant_id,
+ agent_id=agent_id,
+ )
+ # response-contract:ignore provider audio bytes
+ return _text_to_speech(payload=req_data, app_model=app_model, session=session, account_id=current_user.id)
+
+
+def _get_text_to_speech_voices(*, tenant_id: str, language: str):
+ try:
+ response = AudioService.transcript_tts_voices(
+ tenant_id=tenant_id,
+ language=language,
+ )
+
+ return dump_response(TextToSpeechVoiceListResponse, response)
+ except services.errors.audio.ProviderNotSupportTextToSpeechLanageServiceError:
+ raise AppUnavailableError("Text to audio voices language parameter loss.")
+ except NoAudioUploadedServiceError:
+ raise NoAudioUploadedError()
+ except AudioTooLargeServiceError as e:
+ raise AudioTooLargeError(str(e))
+ except UnsupportedAudioTypeServiceError:
+ raise UnsupportedAudioTypeError()
+ except ProviderNotSupportSpeechToTextServiceError:
+ raise ProviderNotSupportSpeechToTextError()
+ except ProviderTokenNotInitError as ex:
+ raise ProviderNotInitializeError(ex.description)
+ except QuotaExceededError:
+ raise ProviderQuotaExceededError()
+ except ModelCurrentlyNotSupportError:
+ raise ProviderModelCurrentlyNotSupportError()
+ except InvokeError as e:
+ raise CompletionRequestError(e.description)
+ except ValueError as e:
+ raise e
+ except Exception as e:
+ logger.exception("Failed to handle get request to TextModesApi")
+ raise InternalServerError()
@console_ns.route("/apps/<uuid:app_id>/text-to-audio/voices")
@@ -340,33 +434,44 @@ class TextModesApi(Resource):
@get_app_model
@model_validate(TextToSpeechVoiceQuery)
def get(self, req_data: TextToSpeechVoiceQuery, app_model: App):
- try:
- response = AudioService.transcript_tts_voices(
- tenant_id=app_model.tenant_id,
- language=req_data.language,
- )
+ return _get_text_to_speech_voices(tenant_id=app_model.tenant_id, language=req_data.language)
- return dump_response(TextToSpeechVoiceListResponse, response)
- except services.errors.audio.ProviderNotSupportTextToSpeechLanageServiceError:
- raise AppUnavailableError("Text to audio voices language parameter loss.")
- except NoAudioUploadedServiceError:
- raise NoAudioUploadedError()
- except AudioTooLargeServiceError as e:
- raise AudioTooLargeError(str(e))
- except UnsupportedAudioTypeServiceError:
- raise UnsupportedAudioTypeError()
- except ProviderNotSupportSpeechToTextServiceError:
- raise ProviderNotSupportSpeechToTextError()
- except ProviderTokenNotInitError as ex:
- raise ProviderNotInitializeError(ex.description)
- except QuotaExceededError:
- raise ProviderQuotaExceededError()
- except ModelCurrentlyNotSupportError:
- raise ProviderModelCurrentlyNotSupportError()
- except InvokeError as e:
- raise CompletionRequestError(e.description)
- except ValueError as e:
- raise e
- except Exception as e:
- logger.exception("Failed to handle get request to TextModesApi")
- raise InternalServerError()
+
+@console_ns.route("/agent/<uuid:agent_id>/text-to-audio/voices")
+class AgentTextToSpeechVoicesApi(Resource):
+ @console_ns.doc("get_agent_text_to_speech_voices")
+ @console_ns.doc(description="Get available TTS voices for an Agent and language")
+ @console_ns.doc(params={"agent_id": "Agent ID", **query_params_from_model(TextToSpeechVoiceQuery)})
+ @console_ns.response(
+ 200,
+ "TTS voices retrieved successfully",
+ console_ns.models[TextToSpeechVoiceListResponse.__name__],
+ )
+ @console_ns.response(400, "Invalid language parameter")
+ @console_ns.response(404, "Agent not found")
+ @setup_required
+ @login_required
+ @account_initialization_required
+ @with_current_user
+ @with_current_tenant_id
+ @with_session(write=False)
+ def get(
+ self,
+ session: Session,
+ current_tenant_id: str,
+ current_user: Account,
+ agent_id: UUID,
+ ):
+ query = TextToSpeechVoiceQuery.model_validate(request.args.to_dict(flat=True))
+ enforce_rbac_checks(
+ tenant_id=current_tenant_id,
+ account_id=current_user.id,
+ checks=[RBACCheck(RBACPermission.AGENT_PREVIEW, AgentId())],
+ path_args={"agent_id": str(agent_id)},
+ )
+ app_model = resolve_existing_agent_runtime_app_model(
+ session=session,
+ tenant_id=current_tenant_id,
+ agent_id=agent_id,
+ )
+ return _get_text_to_speech_voices(tenant_id=app_model.tenant_id, language=query.language)
diff --git a/api/libs/flask_restx_compat.py b/api/libs/flask_restx_compat.py
index 13b5b38ab4f..8e5b166d6d7 100644
--- a/api/libs/flask_restx_compat.py
+++ b/api/libs/flask_restx_compat.py
@@ -355,11 +355,13 @@ def install_swagger_compatibility() -> None:
def responses_for_with_status_specific_media(self: Swagger, doc: dict[str, object], method: str):
responses = original_responses_for(self, doc, method)
blueprint = self.api.blueprint
- if blueprint is None or blueprint.name != "service_api":
- return responses
method_doc = doc.get(method)
if not isinstance(method_doc, dict):
return responses
+ vendor = method_doc.get("vendor", {})
+ is_binary_response = isinstance(vendor, dict) and BINARY_RESPONSE_MEDIA_TYPES_VENDOR_KEY in vendor
+ if blueprint is None or (blueprint.name != "service_api" and not is_binary_response):
+ return responses
for status, response in responses.items():
if not isinstance(response, dict) or str(status).startswith("2"):
diff --git a/api/openapi/markdown/console-openapi.md b/api/openapi/markdown/console-openapi.md
index 21f8ce97dce..d4ac9dc5661 100644
--- a/api/openapi/markdown/console-openapi.md
+++ b/api/openapi/markdown/console-openapi.md
@@ -1261,6 +1261,48 @@ Read a text/binary preview file in an Agent App conversation sandbox
| ---- | ----------- | ------ |
| 200 | Agent monitoring summary and chart data | **application/json**: [AgentStatisticSummaryEnvelopeResponse](#agentstatisticsummaryenveloperesponse)<br> |
+### [POST] /agent/{agent_id}/text-to-audio
+Preview an Agent TTS voice or read an Agent chat message
+
+#### Parameters
+
+| Name | Located in | Description | Required | Schema |
+| ---- | ---------- | ----------- | -------- | ------ |
+| agent_id | path | Agent ID | Yes | string (uuid) |
+
+#### Request Body
+
+| Required | Schema |
+| -------- | ------ |
+| Yes | **application/json**: [TextToSpeechPayload](#texttospeechpayload)<br> |
+
+#### Responses
+
+| Code | Description | Schema |
+| ---- | ----------- | ------ |
+| 200 | Generated audio bytes in the provider audio format | **audio/aac**: binary<br>**audio/flac**: binary<br>**audio/mp4**: binary<br>**audio/mpeg**: binary<br>**audio/ogg**: binary<br>**audio/wav**: binary<br>**audio/webm**: binary<br> |
+| 400 | Invalid text or voice | |
+| 403 | Insufficient permissions | |
+| 404 | Agent not found | |
+
+### [GET] /agent/{agent_id}/text-to-audio/voices
+Get available TTS voices for an Agent and language
+
+#### Parameters
+
+| Name | Located in | Description | Required | Schema |
+| ---- | ---------- | ----------- | -------- | ------ |
+| agent_id | path | Agent ID | Yes | string (uuid) |
+| language | query | Language code | Yes | string |
+
+#### Responses
+
+| Code | Description | Schema |
+| ---- | ----------- | ------ |
+| 200 | TTS voices retrieved successfully | **application/json**: [TextToSpeechVoiceListResponse](#texttospeechvoicelistresponse)<br> |
+| 400 | Invalid language parameter | |
+| 404 | Agent not found | |
+
### [GET] /agent/{agent_id}/versions
#### Parameters
@@ -2683,7 +2725,7 @@ Export application configuration as DSL
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 200 | App exported successfully | **application/json**: [AppExportResponse](#appexportresponse)<br>**application/zip**: binary<br> |
-| 403 | Insufficient permissions | **application/zip**: binary<br> |
+| 403 | Insufficient permissions | |
### [POST] /apps/{app_id}/feedbacks
Create or update message feedback (like/dislike)
diff --git a/api/services/agent/roster_service.py b/api/services/agent/roster_service.py
index 370e31c5323..6ad45814668 100644
--- a/api/services/agent/roster_service.py
+++ b/api/services/agent/roster_service.py
@@ -1003,7 +1003,7 @@ class AgentRosterService:
def _get_runtime_resolvable_agent(self, *, tenant_id: str, agent_id: str) -> Agent | None:
"""Load an Agent that is eligible to resolve to a runtime backing App.
- Shared by the runtime resolver and the read-only authorization resolver
+ Shared by the runtime resolvers and the read-only authorization resolver
so both agree on what counts as a resolvable Agent.
"""
@@ -1043,6 +1043,20 @@ class AgentRosterService:
return None
return agent.app_id
+ def get_existing_agent_runtime_app_model(self, *, tenant_id: str, agent_id: str) -> App:
+ """Resolve an existing runtime App without creating or committing state.
+
+ Workflow-only Agents must already have a hidden backing App. Their
+ parent workflow App must never stand in for a missing runtime App.
+ """
+ agent = self._get_runtime_resolvable_agent(tenant_id=tenant_id, agent_id=agent_id)
+ if agent is None:
+ raise AgentNotFoundError()
+ app_id = agent.backing_app_id if agent.scope == AgentScope.WORKFLOW_ONLY else self.runtime_backing_app_id(agent)
+ if not app_id:
+ raise AgentNotFoundError()
+ return self._get_runtime_app_model_by_id(tenant_id=tenant_id, app_id=app_id)
+
def get_agent_runtime_app_model(self, *, tenant_id: str, agent_id: str) -> App:
"""Resolve the App that backs an Agent runtime surface.
@@ -1065,11 +1079,14 @@ class AgentRosterService:
if should_commit_backing_app:
self._session.commit()
+ return self._get_runtime_app_model_by_id(tenant_id=tenant_id, app_id=backing_app_id)
+
+ def _get_runtime_app_model_by_id(self, *, tenant_id: str, app_id: str) -> App:
app = self._session.scalar(
select(App)
.where(
App.tenant_id == tenant_id,
- App.id == backing_app_id,
+ App.id == app_id,
App.mode == AppMode.AGENT,
App.status == AppStatus.NORMAL,
)
diff --git a/api/tests/unit_tests/controllers/console/app/test_agent_audio_readonly.py b/api/tests/unit_tests/controllers/console/app/test_agent_audio_readonly.py
new file mode 100644
index 00000000000..ac2d8c808fb
--- /dev/null
+++ b/api/tests/unit_tests/controllers/console/app/test_agent_audio_readonly.py
@@ -0,0 +1,209 @@
+from collections.abc import Callable, Iterator
+from unittest.mock import Mock
+from uuid import UUID
+
+import pytest
+from flask import Flask, Response
+from sqlalchemy import Engine, event
+from sqlalchemy.orm import Session
+from werkzeug.exceptions import InternalServerError
+
+from controllers.console import wraps
+from controllers.console.app import audio
+from controllers.console.app.error import (
+ CompletionRequestError,
+ ProviderModelCurrentlyNotSupportError,
+ ProviderNotInitializeError,
+ ProviderQuotaExceededError,
+)
+from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
+from graphon.model_runtime.errors.invoke import InvokeError
+from models.agent import Agent, AgentKind, AgentScope, AgentSource, AgentStatus
+from models.model import App, AppMode
+from services.agent.errors import AgentNotFoundError
+from services.audio_service import AudioService
+from tests.unit_tests.model_factories import make_account, make_app
+
+AGENT_ID = UUID("019ef3d2-b24c-7803-b428-18b5ee8fb853")
+
+
[email protected](autouse=True)
+def audio_identity(monkeypatch: pytest.MonkeyPatch, config_overrides: Callable[..., None]) -> None:
+ config_overrides(LOGIN_DISABLED=True, RBAC_ENABLED=True)
+ monkeypatch.setattr(wraps, "_is_setup_completed", lambda: True)
+ monkeypatch.setattr(wraps, "current_account_with_tenant", lambda: (make_account(), "tenant-1"))
+ monkeypatch.setattr(audio, "enforce_rbac_checks", lambda **_kwargs: None)
+
+
+def persistence_events(sqlite_engine: Engine) -> Iterator[list[str]]:
+ observed: list[str] = []
+
+ def record_statement(
+ _connection: object,
+ _cursor: object,
+ statement: str,
+ _parameters: object,
+ _context: object,
+ _executemany: bool,
+ ) -> None:
+ operation = statement.lstrip().split(maxsplit=1)[0].upper()
+ if operation in {"INSERT", "UPDATE", "DELETE"}:
+ observed.append(operation)
+
+ def record_commit(_session: Session) -> None:
+ observed.append("COMMIT")
+
+ event.listen(sqlite_engine, "before_cursor_execute", record_statement)
+ event.listen(Session, "before_commit", record_commit)
+ try:
+ yield observed
+ finally:
+ event.remove(sqlite_engine, "before_cursor_execute", record_statement)
+ event.remove(Session, "before_commit", record_commit)
+
+
[email protected](params=["voices", "preview"])
+def request_audio(
+ request: pytest.FixtureRequest, app: Flask, monkeypatch: pytest.MonkeyPatch
+) -> tuple[Callable[[], object], Mock]:
+ if request.param == "voices":
+ provider = Mock(return_value=[{"name": "Echo", "value": "echo"}])
+ monkeypatch.setattr(AudioService, "transcript_tts_voices", provider)
+
+ def invoke() -> object:
+ with app.test_request_context(f"/console/api/agent/{AGENT_ID}/text-to-audio/voices?language=en-US"):
+ return audio.AgentTextToSpeechVoicesApi().get(agent_id=AGENT_ID)
+
+ else:
+ provider = Mock(return_value=Response(b"ID3audio", content_type="audio/mpeg"))
+ monkeypatch.setattr(AudioService, "transcript_tts", provider)
+
+ def invoke() -> object:
+ with app.test_request_context(
+ f"/console/api/agent/{AGENT_ID}/text-to-audio",
+ method="POST",
+ json={"text": "Preview this voice", "voice": "echo"},
+ ):
+ return audio.AgentChatMessageTextApi().post(agent_id=AGENT_ID)
+
+ return invoke, provider
+
+
+def _agent(scope: AgentScope = AgentScope.WORKFLOW_ONLY) -> Agent:
+ workflow_only = scope == AgentScope.WORKFLOW_ONLY
+ return Agent(
+ id=str(AGENT_ID),
+ tenant_id="tenant-1",
+ name="Audio agent",
+ agent_kind=AgentKind.DIFY_AGENT,
+ scope=scope,
+ source=AgentSource.WORKFLOW if workflow_only else AgentSource.AGENT_APP,
+ status=AgentStatus.ACTIVE,
+ app_id="parent-app-1" if workflow_only else "runtime-app-1",
+ backing_app_id="runtime-app-1" if workflow_only else None,
+ workflow_id="workflow-1" if workflow_only else None,
+ workflow_node_id="node-1" if workflow_only else None,
+ created_by="account-1",
+ updated_by="account-1",
+ )
+
+
[email protected]("scope", [AgentScope.ROSTER, AgentScope.WORKFLOW_ONLY])
+def test_audio_uses_existing_runtime_state_without_writes(
+ sqlite_session: Session,
+ request_audio: tuple[Callable[[], object], Mock],
+ persistence_events: list[str],
+ scope: AgentScope,
+) -> None:
+ agent = _agent(scope)
+ runtime_app = make_app(app_id="runtime-app-1", mode=AppMode.AGENT)
+ sqlite_session.add_all([agent, runtime_app])
+ sqlite_session.commit()
+ persistence_events.clear()
+ invoke, provider = request_audio
+
+ response = invoke()
+
+ provider.assert_called_once()
+ assert response is not None
+ assert persistence_events == []
+
+
+def test_audio_does_not_materialize_a_missing_workflow_backing_app(
+ sqlite_session: Session,
+ request_audio: tuple[Callable[[], object], Mock],
+ persistence_events: list[str],
+) -> None:
+ agent = _agent()
+ agent.backing_app_id = None
+ parent_app = make_app(app_id="parent-app-1", mode=AppMode.WORKFLOW)
+ sqlite_session.add_all([agent, parent_app])
+ sqlite_session.commit()
+ persistence_events.clear()
+ invoke, provider = request_audio
+
+ with pytest.raises(AgentNotFoundError):
+ invoke()
+
+ provider.assert_not_called()
+ assert persistence_events == []
+ sqlite_session.refresh(agent)
+ assert agent.backing_app_id is None
+ assert sqlite_session.get(App, "runtime-app-1") is None
+
+
[email protected]("foreign_owner", ["agent", "app"])
+def test_audio_scopes_both_agent_and_runtime_app_to_the_current_tenant(
+ sqlite_session: Session,
+ request_audio: tuple[Callable[[], object], Mock],
+ persistence_events: list[str],
+ foreign_owner: str,
+) -> None:
+ agent = _agent()
+ runtime_app = make_app(app_id="runtime-app-1", mode=AppMode.AGENT)
+ if foreign_owner == "agent":
+ agent.tenant_id = "tenant-2"
+ else:
+ runtime_app.tenant_id = "tenant-2"
+ sqlite_session.add_all([agent, runtime_app])
+ sqlite_session.commit()
+ persistence_events.clear()
+ invoke, provider = request_audio
+
+ with pytest.raises(AgentNotFoundError):
+ invoke()
+
+ provider.assert_not_called()
+ assert persistence_events == []
+
+
+ ("provider_error", "expected_error"),
+ [
+ (ProviderTokenNotInitError("No TTS model configured"), ProviderNotInitializeError),
+ (QuotaExceededError(), ProviderQuotaExceededError),
+ (ModelCurrentlyNotSupportError(), ProviderModelCurrentlyNotSupportError),
+ (InvokeError("TTS provider unavailable"), CompletionRequestError),
+ (ValueError("Unknown voice"), ValueError),
+ (RuntimeError("Unexpected provider failure"), InternalServerError),
+ ],
+)
+def test_provider_failures_preserve_error_contract_and_readonly_state(
+ sqlite_session: Session,
+ request_audio: tuple[Callable[[], object], Mock],
+ persistence_events: list[str],
+ provider_error: Exception,
+ expected_error: type[Exception],
+) -> None:
+ sqlite_session.add_all([_agent(AgentScope.ROSTER), make_app(app_id="runtime-app-1", mode=AppMode.AGENT)])
+ sqlite_session.commit()
+ persistence_events.clear()
+ invoke, provider = request_audio
+ provider.side_effect = provider_error
+
+ with pytest.raises(expected_error):
+ invoke()
+
+ provider.assert_called_once()
+ assert persistence_events == []
diff --git a/api/tests/unit_tests/controllers/console/app/test_audio.py b/api/tests/unit_tests/controllers/console/app/test_audio.py
index 6e3efa6c985..8fa69b8fbcc 100644
--- a/api/tests/unit_tests/controllers/console/app/test_audio.py
+++ b/api/tests/unit_tests/controllers/console/app/test_audio.py
@@ -6,7 +6,7 @@ from unittest.mock import patch
from uuid import UUID
import pytest
-from flask import Flask
+from flask import Flask, Response
from sqlalchemy.orm import Session
from werkzeug.datastructures import FileStorage
from werkzeug.exceptions import Forbidden, InternalServerError
@@ -14,6 +14,8 @@ from werkzeug.exceptions import Forbidden, InternalServerError
from controllers.console.app import audio as audio_module
from controllers.console.app.audio import (
AgentChatMessageAudioApi,
+ AgentChatMessageTextApi,
+ AgentTextToSpeechVoicesApi,
ChatMessageAudioApi,
ChatMessageTextApi,
TextModesApi,
@@ -38,7 +40,7 @@ from models import Account, App, AppMode
from models.agent import AgentConfigDraftType
from models.agent_config_entities import AgentSoulConfig
from services.agent.composer_service import AgentComposerService
-from services.agent.errors import AgentVersionNotFoundError
+from services.agent.errors import AgentNotFoundError, AgentVersionNotFoundError
from services.app_ref_service import AppRef, MessageRef
from services.audio_service import AudioService
from services.errors.app_model_config import AppModelConfigBrokenError
@@ -522,3 +524,158 @@ def test_text_to_audio_voices_with_language_filter(app: Flask, monkeypatch: pyte
):
response = method(api, TextToSpeechVoiceQuery(language="en-US"), app_model=app_model)
assert isinstance(response, list)
+
+
+def test_agent_text_to_speech_voices_uses_backing_app_and_language(app: Flask, unbound_session: Session) -> None:
+ agent_id = UUID("019ef3d2-b24c-7803-b428-18b5ee8fb853")
+ app_model = _app(app_id="backing-app-1")
+ current_user = _account()
+ voices = [{"name": "Voice 1", "value": "voice-1"}]
+ api = AgentTextToSpeechVoicesApi()
+ with (
+ patch.object(audio_module, "resolve_existing_agent_runtime_app_model", return_value=app_model) as resolve_app,
+ patch.object(audio_module, "enforce_rbac_checks") as check_access,
+ patch.object(AudioService, "transcript_tts_voices", return_value=voices) as get_voices,
+ app.test_request_context(f"/console/api/agent/{agent_id}/text-to-audio/voices?language=en-US"),
+ ):
+ response = unwrap(api.get)(
+ api,
+ session=unbound_session,
+ current_tenant_id="tenant-1",
+ current_user=current_user,
+ agent_id=agent_id,
+ )
+
+ assert response == voices
+ resolve_app.assert_called_once_with(session=unbound_session, tenant_id="tenant-1", agent_id=agent_id)
+ check_access.assert_called_once()
+ rbac_call = check_access.call_args.kwargs
+ assert rbac_call["tenant_id"] == "tenant-1"
+ assert rbac_call["account_id"] == current_user.id
+ assert rbac_call["path_args"] == {"agent_id": str(agent_id)}
+ (rbac_check,) = rbac_call["checks"]
+ assert rbac_check.scene is audio_module.RBACPermission.AGENT_PREVIEW
+ assert isinstance(rbac_check.locator, audio_module.AgentId)
+ get_voices.assert_called_once_with(tenant_id="tenant-1", language="en-US")
+
+
+def test_agent_text_to_speech_voices_does_not_query_provider_without_permission(
+ app: Flask, unbound_session: Session
+) -> None:
+ agent_id = UUID("019ef3d2-b24c-7803-b428-18b5ee8fb853")
+ api = AgentTextToSpeechVoicesApi()
+ with (
+ patch.object(audio_module, "resolve_existing_agent_runtime_app_model") as resolve_app,
+ patch.object(audio_module, "enforce_rbac_checks", side_effect=Forbidden()),
+ patch.object(AudioService, "transcript_tts_voices") as get_voices,
+ app.test_request_context(f"/console/api/agent/{agent_id}/text-to-audio/voices?language=en-US"),
+ pytest.raises(Forbidden),
+ ):
+ unwrap(api.get)(
+ api,
+ session=unbound_session,
+ current_tenant_id="tenant-1",
+ current_user=_account(),
+ agent_id=agent_id,
+ )
+
+ get_voices.assert_not_called()
+ resolve_app.assert_not_called()
+
+
[email protected]("message_id", [None, "019ef3d2-b24c-7803-b428-18b5ee8fb854"])
+def test_agent_text_to_speech_returns_audio_with_scoped_message(
+ app: Flask, unbound_session: Session, message_id: str | None
+) -> None:
+ agent_id = UUID("019ef3d2-b24c-7803-b428-18b5ee8fb853")
+ app_model = _app(app_id="backing-app-1")
+ current_user = _account()
+ payload = TextToSpeechPayload(text="Preview this voice", voice="voice-1", message_id=message_id)
+ audio = Response(b"RIFF\x00\x00\x00\x00WAVE", content_type="audio/wav")
+ api = AgentChatMessageTextApi()
+ with (
+ patch.object(audio_module, "resolve_existing_agent_runtime_app_model", return_value=app_model) as resolve_app,
+ patch.object(audio_module, "enforce_rbac_checks") as check_access,
+ patch.object(AudioService, "transcript_tts", return_value=audio) as synthesize,
+ app.test_request_context(f"/console/api/agent/{agent_id}/text-to-audio", method="POST"),
+ ):
+ response = unwrap(api.post)(
+ api,
+ req_data=payload,
+ session=unbound_session,
+ current_tenant_id="tenant-1",
+ current_user=current_user,
+ agent_id=agent_id,
+ )
+
+ assert response.get_data() == audio.get_data()
+ assert response.content_type == "audio/wav"
+ resolve_app.assert_called_once_with(session=unbound_session, tenant_id="tenant-1", agent_id=agent_id)
+ check_access.assert_called_once()
+ rbac_call = check_access.call_args.kwargs
+ assert rbac_call["tenant_id"] == "tenant-1"
+ assert rbac_call["account_id"] == current_user.id
+ assert rbac_call["path_args"] == {"agent_id": str(agent_id)}
+ (rbac_check,) = rbac_call["checks"]
+ assert rbac_check.scene is audio_module.RBACPermission.AGENT_TEST_AND_RUN
+ assert isinstance(rbac_check.locator, audio_module.AgentId)
+ synthesize.assert_called_once_with(
+ app_model=app_model,
+ session=unbound_session,
+ text=payload.text,
+ voice=payload.voice,
+ message_ref=MessageRef(AppRef("tenant-1", "backing-app-1"), message_id, account_id=current_user.id)
+ if message_id
+ else None,
+ is_draft=True,
+ )
+
+
+def test_agent_text_to_speech_denies_access_before_resolving_app(app: Flask, unbound_session: Session) -> None:
+ agent_id = UUID("019ef3d2-b24c-7803-b428-18b5ee8fb853")
+ api = AgentChatMessageTextApi()
+ with (
+ patch.object(audio_module, "resolve_existing_agent_runtime_app_model") as resolve_app,
+ patch.object(audio_module, "enforce_rbac_checks", side_effect=Forbidden()),
+ patch.object(AudioService, "transcript_tts") as synthesize,
+ app.test_request_context(f"/console/api/agent/{agent_id}/text-to-audio", method="POST"),
+ pytest.raises(Forbidden),
+ ):
+ unwrap(api.post)(
+ api,
+ req_data=TextToSpeechPayload(text="Preview"),
+ session=unbound_session,
+ current_tenant_id="tenant-1",
+ current_user=_account(),
+ agent_id=agent_id,
+ )
+
+ resolve_app.assert_not_called()
+ synthesize.assert_not_called()
+
+
[email protected]("voices", [False, True])
+def test_agent_tts_preserves_missing_agent_error(app: Flask, unbound_session: Session, voices: bool) -> None:
+ agent_id = UUID("019ef3d2-b24c-7803-b428-18b5ee8fb853")
+ api = AgentTextToSpeechVoicesApi() if voices else AgentChatMessageTextApi()
+ handler = unwrap(api.get if isinstance(api, AgentTextToSpeechVoicesApi) else api.post)
+ args: dict[str, object] = {
+ "session": unbound_session,
+ "current_tenant_id": "tenant-1",
+ "current_user": _account(),
+ "agent_id": agent_id,
+ }
+ if not voices:
+ args["req_data"] = TextToSpeechPayload(text="Preview")
+ with (
+ patch.object(audio_module, "resolve_existing_agent_runtime_app_model", side_effect=AgentNotFoundError()),
+ patch.object(audio_module, "enforce_rbac_checks"),
+ patch.object(AudioService, "transcript_tts") as synthesize,
+ patch.object(AudioService, "transcript_tts_voices") as get_voices,
+ app.test_request_context(f"/console/api/agent/{agent_id}/text-to-audio?language=en-US"),
+ pytest.raises(AgentNotFoundError),
+ ):
+ handler(api, **args)
+
+ synthesize.assert_not_called()
+ get_voices.assert_not_called()
diff --git a/api/tests/unit_tests/controllers/test_swagger.py b/api/tests/unit_tests/controllers/test_swagger.py
index 958f86827b2..a24a90133c3 100644
--- a/api/tests/unit_tests/controllers/test_swagger.py
+++ b/api/tests/unit_tests/controllers/test_swagger.py
@@ -194,6 +194,32 @@ def test_openapi_json_endpoints_render():
assert app.config["RESTX_INCLUDE_ALL_MODELS"] is True
+def test_agent_tts_routes_document_voice_queries_and_binary_audio():
+ from controllers.console import bp as console_bp
+ from core.base.tts.audio_mime import SUPPORTED_TTS_AUDIO_MIME_TYPES
+
+ app = Flask(__name__)
+ app.config["TESTING"] = True
+ app.register_blueprint(console_bp)
+ payload = app.test_client().get("/console/api/openapi.json").get_json()
+
+ voices = payload["paths"]["/agent/{agent_id}/text-to-audio/voices"]["get"]
+ language = _parameters_by_name(voices)["language"]
+ assert language["in"] == "query"
+ assert language["required"] is True
+ assert "requestBody" not in voices
+
+ preview = payload["paths"]["/agent/{agent_id}/text-to-audio"]["post"]
+ required_fields = _json_body_schema(payload, preview)["required"]
+ assert isinstance(required_fields, list)
+ assert "text" in required_fields
+ assert _response_content_types(preview) == set(SUPPORTED_TTS_AUDIO_MIME_TYPES)
+ for media in preview["responses"]["200"]["content"].values():
+ assert media["schema"] == {"type": "string", "format": "binary"}
+ for status in ("400", "403", "404"):
+ assert _response_content_types(preview, status) == {"application/json"}
+
+
def test_service_document_file_routes_document_multipart_form_data():
from controllers.service_api import bp as service_api_bp
diff --git a/packages/contracts/generated/api/console/agent/orpc.gen.ts b/packages/contracts/generated/api/console/agent/orpc.gen.ts
index fe1094a123a..7c46bd356b5 100644
--- a/packages/contracts/generated/api/console/agent/orpc.gen.ts
+++ b/packages/contracts/generated/api/console/agent/orpc.gen.ts
@@ -85,6 +85,9 @@ import {
zGetAgentByAgentIdStatisticsSummaryPath,
zGetAgentByAgentIdStatisticsSummaryQuery,
zGetAgentByAgentIdStatisticsSummaryResponse,
+ zGetAgentByAgentIdTextToAudioVoicesPath,
+ zGetAgentByAgentIdTextToAudioVoicesQuery,
+ zGetAgentByAgentIdTextToAudioVoicesResponse,
zGetAgentByAgentIdVersionsByVersionIdPath,
zGetAgentByAgentIdVersionsByVersionIdResponse,
zGetAgentByAgentIdVersionsPath,
@@ -139,6 +142,9 @@ import {
zPostAgentByAgentIdSandboxFilesDownloadBody,
zPostAgentByAgentIdSandboxFilesDownloadPath,
zPostAgentByAgentIdSandboxFilesDownloadResponse,
+ zPostAgentByAgentIdTextToAudioBody,
+ zPostAgentByAgentIdTextToAudioPath,
+ zPostAgentByAgentIdTextToAudioResponse,
zPostAgentByAgentIdVersionsByVersionIdRestorePath,
zPostAgentByAgentIdVersionsByVersionIdRestoreResponse,
zPostAgentResponse,
@@ -1111,7 +1117,56 @@ export const statistics = {
summary,
}
+/**
+ * Get available TTS voices for an Agent and language
+ */
+export const get28 = oc
+ .route({
+ description: 'Get available TTS voices for an Agent and language',
+ inputStructure: 'detailed',
+ method: 'GET',
+ operationId: 'getAgentByAgentIdTextToAudioVoices',
+ path: '/agent/{agent_id}/text-to-audio/voices',
+ tags: ['console'],
+ })
+ .input(
+ z.object({
+ params: zGetAgentByAgentIdTextToAudioVoicesPath,
+ query: zGetAgentByAgentIdTextToAudioVoicesQuery,
+ }),
+ )
+ .output(zGetAgentByAgentIdTextToAudioVoicesResponse)
+
+export const voices = {
+ get: get28,
+}
+
+/**
+ * Preview an Agent TTS voice or read an Agent chat message
+ */
export const post17 = oc
+ .route({
+ description: 'Preview an Agent TTS voice or read an Agent chat message',
+ inputStructure: 'detailed',
+ method: 'POST',
+ operationId: 'postAgentByAgentIdTextToAudio',
+ path: '/agent/{agent_id}/text-to-audio',
+ tags: ['console'],
+ })
+ .input(
+ z.object({
+ body: zPostAgentByAgentIdTextToAudioBody,
+ params: zPostAgentByAgentIdTextToAudioPath,
+ }),
+ )
+ .output(zPostAgentByAgentIdTextToAudioResponse)
+
+export const textToAudio = {
+ post: post17,
+ voices,
+}
+
+export const post18 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@@ -1123,10 +1178,10 @@ export const post17 = oc
.output(zPostAgentByAgentIdVersionsByVersionIdRestoreResponse)
export const restore = {
- post: post17,
+ post: post18,
}
-export const get28 = oc
+export const get29 = oc
.route({
inputStructure: 'detailed',
method: 'GET',
@@ -1138,11 +1193,11 @@ export const get28 = oc
.output(zGetAgentByAgentIdVersionsByVersionIdResponse)
export const byVersionId = {
- get: get28,
+ get: get29,
restore,
}
-export const get29 = oc
+export const get30 = oc
.route({
inputStructure: 'detailed',
method: 'GET',
@@ -1154,7 +1209,7 @@ export const get29 = oc
.output(zGetAgentByAgentIdVersionsResponse)
export const versions = {
- get: get29,
+ get: get30,
byVersionId,
}
@@ -1170,7 +1225,7 @@ export const delete5 = oc
.input(z.object({ params: zDeleteAgentByAgentIdPath }))
.output(zDeleteAgentByAgentIdResponse)
-export const get30 = oc
+export const get31 = oc
.route({
inputStructure: 'detailed',
method: 'GET',
@@ -1194,7 +1249,7 @@ export const put3 = oc
export const byAgentId = {
delete: delete5,
- get: get30,
+ get: get31,
put: put3,
apiAccess,
apiEnable,
@@ -1216,10 +1271,11 @@ export const byAgentId = {
referencingWorkflows,
sandbox,
statistics,
+ textToAudio,
versions,
}
-export const get31 = oc
+export const get32 = oc
.route({
inputStructure: 'detailed',
method: 'GET',
@@ -1230,7 +1286,7 @@ export const get31 = oc
.input(z.object({ query: zGetAgentQuery.optional() }))
.output(zGetAgentResponse)
-export const post18 = oc
+export const post19 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@@ -1243,8 +1299,8 @@ export const post18 = oc
.output(zPostAgentResponse)
export const agent = {
- get: get31,
- post: post18,
+ get: get32,
+ post: post19,
inviteOptions,
byAgentId,
}
diff --git a/packages/contracts/generated/api/console/agent/types.gen.ts b/packages/contracts/generated/api/console/agent/types.gen.ts
index 83680198b6b..b3aa59a6e69 100644
--- a/packages/contracts/generated/api/console/agent/types.gen.ts
+++ b/packages/contracts/generated/api/console/agent/types.gen.ts
@@ -407,6 +407,15 @@ export type AgentStatisticSummaryEnvelopeResponse = {
summary: AgentStatisticSummaryResponse
}
+export type TextToSpeechPayload = {
+ message_id?: string | null
+ streaming?: boolean | null
+ text: string
+ voice?: string | null
+}
+
+export type TextToSpeechVoiceListResponse = Array<TextToSpeechVoiceResponse>
+
export type AgentConfigSnapshotListResponse = {
data: Array<AgentConfigSnapshotSummaryResponse>
}
@@ -958,6 +967,11 @@ export type AgentStatisticSummaryResponse = {
user_satisfaction_rate: number
}
+export type TextToSpeechVoiceResponse = {
+ name: string
+ value: string
+}
+
export type AgentConfigRevisionResponse = {
created_at?: number | null
created_by?: string | null
@@ -2902,6 +2916,51 @@ export type GetAgentByAgentIdStatisticsSummaryResponses = {
export type GetAgentByAgentIdStatisticsSummaryResponse =
GetAgentByAgentIdStatisticsSummaryResponses[keyof GetAgentByAgentIdStatisticsSummaryResponses]
+export type PostAgentByAgentIdTextToAudioData = {
+ body: TextToSpeechPayload
+ path: {
+ agent_id: string
+ }
+ query?: never
+ url: '/agent/{agent_id}/text-to-audio'
+}
+
+export type PostAgentByAgentIdTextToAudioErrors = {
+ 400: unknown
+ 403: unknown
+ 404: unknown
+}
+
+export type PostAgentByAgentIdTextToAudioResponses = {
+ 200: Blob | File
+}
+
+export type PostAgentByAgentIdTextToAudioResponse =
+ PostAgentByAgentIdTextToAudioResponses[keyof PostAgentByAgentIdTextToAudioResponses]
+
+export type GetAgentByAgentIdTextToAudioVoicesData = {
+ body?: never
+ path: {
+ agent_id: string
+ }
+ query: {
+ language: string
+ }
+ url: '/agent/{agent_id}/text-to-audio/voices'
+}
+
+export type GetAgentByAgentIdTextToAudioVoicesErrors = {
+ 400: unknown
+ 404: unknown
+}
+
+export type GetAgentByAgentIdTextToAudioVoicesResponses = {
+ 200: TextToSpeechVoiceListResponse
+}
+
+export type GetAgentByAgentIdTextToAudioVoicesResponse =
+ GetAgentByAgentIdTextToAudioVoicesResponses[keyof GetAgentByAgentIdTextToAudioVoicesResponses]
+
export type GetAgentByAgentIdVersionsData = {
body?: never
path: {
diff --git a/packages/contracts/generated/api/console/agent/zod.gen.ts b/packages/contracts/generated/api/console/agent/zod.gen.ts
index 87f89cb82ac..f0505e311f7 100644
--- a/packages/contracts/generated/api/console/agent/zod.gen.ts
+++ b/packages/contracts/generated/api/console/agent/zod.gen.ts
@@ -195,6 +195,16 @@ export const zSandboxReadResponse = z.object({
truncated: z.boolean(),
})
+/**
+ * TextToSpeechPayload
+ */
+export const zTextToSpeechPayload = z.object({
+ message_id: z.string().nullish(),
+ streaming: z.boolean().nullish(),
+ text: z.string(),
+ voice: z.string().nullish(),
+})
+
/**
* AgentConfigSnapshotRestoreResponse
*/
@@ -809,6 +819,21 @@ export const zAgentStatisticSummaryResponse = z.object({
user_satisfaction_rate: z.number(),
})
+/**
+ * TextToSpeechVoiceResponse
+ */
+export const zTextToSpeechVoiceResponse = z.object({
+ name: z.string(),
+ value: z.string(),
+})
+
+/**
+ * TextToSpeechVoiceListResponse
+ *
+ * Available voices
+ */
+export const zTextToSpeechVoiceListResponse = z.array(zTextToSpeechVoiceResponse)
+
/**
* ModelConfigPartial
*/
@@ -3265,6 +3290,32 @@ export const zGetAgentByAgentIdStatisticsSummaryQuery = z.object({
*/
export const zGetAgentByAgentIdStatisticsSummaryResponse = zAgentStatisticSummaryEnvelopeResponse
+export const zPostAgentByAgentIdTextToAudioBody = zTextToSpeechPayload
+
+export const zPostAgentByAgentIdTextToAudioPath = z.object({
+ agent_id: z.uuid(),
+})
+
+/**
+ * Generated audio bytes in the provider audio format
+ */
+export const zPostAgentByAgentIdTextToAudioResponse = z.custom<Blob | File>(
+ (value) => value instanceof Blob || value instanceof File,
+)
+
+export const zGetAgentByAgentIdTextToAudioVoicesPath = z.object({
+ agent_id: z.uuid(),
+})
+
+export const zGetAgentByAgentIdTextToAudioVoicesQuery = z.object({
+ language: z.string(),
+})
+
+/**
+ * TTS voices retrieved successfully
+ */
+export const zGetAgentByAgentIdTextToAudioVoicesResponse = zTextToSpeechVoiceListResponse
+
export const zGetAgentByAgentIdVersionsPath = z.object({
agent_id: z.uuid(),
})
diff --git a/web/app/components/base/audio-btn/__tests__/index.spec.tsx b/web/app/components/base/audio-btn/__tests__/index.spec.tsx
index 6cda0c33fbf..a1ef1e74dc8 100644
--- a/web/app/components/base/audio-btn/__tests__/index.spec.tsx
+++ b/web/app/components/base/audio-btn/__tests__/index.spec.tsx
@@ -70,7 +70,7 @@ describe('AudioBtn', () => {
})
})
- // URL path resolution for app/public audio endpoints.
+ // URL path resolution for app, agent, and public audio endpoints.
describe('URL routing', () => {
it('should call public text-to-audio endpoint when token exists', async () => {
mockUseParams({ token: 'public-token' })
@@ -109,6 +109,25 @@ describe('AudioBtn', () => {
expect(call![0]).toBe('/installed-apps/456/text-to-audio')
expect(call![1]).toBe(false)
})
+
+ it('should play the selected voice preview through the agent endpoint', async () => {
+ const user = userEvent.setup()
+ mockUseParams({ agentId: 'agent-123' })
+ mockUsePathname('/agents/agent-123/configure')
+
+ render(<AudioBtn isAudition value="This is a voice preview." voice="alloy" />)
+ await user.click(screen.getByRole('button', { name: 'play' }))
+
+ expect(mockGetAudioPlayer).toHaveBeenCalledWith(
+ '/agent/agent-123/text-to-audio',
+ false,
+ undefined,
+ 'This is a voice preview.',
+ 'alloy',
+ expect.any(Function),
+ )
+ expect(mockPlayAudio).toHaveBeenCalledTimes(1)
+ })
})
// User-visible playback state transitions.
@@ -184,7 +203,7 @@ describe('AudioBtn', () => {
expect(call![4]).toBe('en-US')
})
- it('should keep empty route when neither token nor appId is present', async () => {
+ it('should keep empty route when no token, appId, or agentId is present', async () => {
render(<AudioBtn />)
await userEvent.click(getButton())
diff --git a/web/app/components/base/audio-btn/index.tsx b/web/app/components/base/audio-btn/index.tsx
index d1288f4b84a..f660125a610 100644
--- a/web/app/components/base/audio-btn/index.tsx
+++ b/web/app/components/base/audio-btn/index.tsx
@@ -48,6 +48,8 @@ export function AudioBtn({ id, voice, value, className, isAudition }: AudioBtnPr
if (params.token) {
url = '/text-to-audio'
isPublic = true
+ } else if (params.agentId) {
+ url = `/agent/${params.agentId}/text-to-audio`
} else if (params.appId) {
if (isInstalledAppPath(pathname)) url = `/installed-apps/${params.appId}/text-to-audio`
else url = `/apps/${params.appId}/text-to-audio`
diff --git a/web/app/components/base/features/new-feature-panel/text-to-speech/__tests__/param-config-content.spec.tsx b/web/app/components/base/features/new-feature-panel/text-to-speech/__tests__/param-config-content.spec.tsx
index 36397b7c24b..76938c29f1a 100644
--- a/web/app/components/base/features/new-feature-panel/text-to-speech/__tests__/param-config-content.spec.tsx
+++ b/web/app/components/base/features/new-feature-panel/text-to-speech/__tests__/param-config-content.spec.tsx
@@ -3,6 +3,7 @@ import type { OnFeaturesChange } from '@/app/components/base/features/types'
import { skipToken } from '@tanstack/react-query'
import { render, screen } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
+import i18next from 'i18next'
import { TtsAutoPlay } from '@/types/app'
import { FeaturesProvider } from '../../../context'
import ParamConfigContent from '../param-config-content'
@@ -12,33 +13,48 @@ let mockLanguages = [
{ value: 'zh-Hans', name: '中文', example: '你好' },
]
-let mockPathname = '/app/test-app-id/configuration'
+let mockParams: { appId?: string; agentId?: string } = { appId: 'test-app-id' }
let mockVoiceItems: { value: string; name: string }[] | undefined = [
{ value: 'alloy', name: 'Alloy' },
{ value: 'echo', name: 'Echo' },
]
-const mockVoicesQuery = vi.fn(
- (_options: {
- enabled: boolean
- input: typeof skipToken | { params: { app_id: string }; query: { language: string } }
- }) => ({
- data: mockVoiceItems,
- }),
-)
+type VoicesQueryOptions = {
+ enabled?: boolean
+ input:
+ | typeof skipToken
+ | {
+ params: { app_id: string } | { agent_id: string }
+ query: { language: string }
+ }
+}
+
+const mockVoicesQuery = vi.fn((_options: VoicesQueryOptions) => ({
+ data: mockVoiceItems,
+}))
+
+const mockGetAudioPlayer = vi.fn(() => ({ playAudio: vi.fn(), pauseAudio: vi.fn() }))
+
+vi.mock('@/app/components/base/audio-btn/audio.player.manager', () => ({
+ AudioPlayerManager: {
+ getInstance: () => ({ getAudioPlayer: mockGetAudioPlayer }),
+ },
+}))
vi.mock('@tanstack/react-query', async (importOriginal) => ({
...(await importOriginal<typeof import('@tanstack/react-query')>()),
- useQuery: (options: {
- enabled: boolean
- input: typeof skipToken | { params: { app_id: string }; query: { language: string } }
- }) => mockVoicesQuery(options),
+ useQuery: (options: VoicesQueryOptions) => mockVoicesQuery(options),
}))
vi.mock('@/next/navigation', () => ({
- usePathname: () => mockPathname,
- useParams: () => ({}),
+ usePathname: () =>
+ mockParams.appId
+ ? `/app/${mockParams.appId}/configuration`
+ : mockParams.agentId
+ ? `/agents/${mockParams.agentId}/configure`
+ : '/configuration',
+ useParams: () => mockParams,
}))
vi.mock('@/i18n/language', () => ({
@@ -76,9 +92,13 @@ const getLanguageSelect = () =>
const getVoiceSelect = () => screen.getByRole('combobox', { name: /voice\.voiceSettings\.voice/ })
describe('ParamConfigContent', () => {
+ beforeAll(async () => {
+ await i18next.init({})
+ })
+
beforeEach(() => {
vi.clearAllMocks()
- mockPathname = '/app/test-app-id/configuration'
+ mockParams = { appId: 'test-app-id' }
mockLanguages = [
{ value: 'en-US', name: 'English', example: 'Hello world' },
{ value: 'zh-Hans', name: '中文', example: '你好' },
@@ -187,6 +207,80 @@ describe('ParamConfigContent', () => {
// User-triggered behavior and callbacks.
describe('User Interactions', () => {
+ it('should audition the displayed fallback voice on an agent page', async () => {
+ const user = userEvent.setup()
+ mockParams = { agentId: 'agent-1' }
+ renderWithProvider(
+ {},
+ { text2speech: { ...defaultFeatures.text2speech, enabled: true, voice: 'removed-voice' } },
+ )
+
+ expect(getVoiceSelect()).toHaveTextContent('Alloy')
+ await user.click(screen.getByRole('button', { name: /play/i }))
+
+ expect(mockGetAudioPlayer).toHaveBeenCalledWith(
+ '/agent/agent-1/text-to-audio',
+ false,
+ undefined,
+ 'Hello world',
+ 'alloy',
+ expect.any(Function),
+ )
+ })
+
+ it.each([
+ {
+ route: 'Chatflow',
+ params: { appId: 'test-app-id' },
+ requestParams: { app_id: 'test-app-id' },
+ operation: ['console', 'apps', 'byAppId', 'textToAudio', 'voices', 'get'],
+ },
+ {
+ route: 'Agent',
+ params: { agentId: 'test-agent-id' },
+ requestParams: { agent_id: 'test-agent-id' },
+ operation: ['console', 'agent', 'byAgentId', 'textToAudio', 'voices', 'get'],
+ },
+ ])(
+ 'should load and select voices using the $route route ID',
+ async ({ params, requestParams, operation }) => {
+ const user = userEvent.setup()
+ const onChange = vi.fn()
+ mockParams = params
+
+ renderWithProvider({ onChange })
+
+ expect(mockVoicesQuery).toHaveBeenCalledWith(
+ expect.objectContaining({
+ queryKey: expect.arrayContaining([operation]),
+ input: {
+ params: requestParams,
+ query: { language: 'en-US' },
+ },
+ }),
+ )
+
+ await user.click(getVoiceSelect())
+ await user.click(await screen.findByRole('option', { name: 'Echo' }))
+
+ expect(getVoiceSelect()).toHaveTextContent('Echo')
+ expect(onChange).toHaveBeenCalled()
+
+ await user.click(getLanguageSelect())
+ await user.click(await screen.findByRole('option', { name: /voice\.language\.zhHans/ }))
+
+ expect(mockVoicesQuery).toHaveBeenLastCalledWith(
+ expect.objectContaining({
+ queryKey: expect.arrayContaining([operation]),
+ input: {
+ params: requestParams,
+ query: { language: 'zh-Hans' },
+ },
+ }),
+ )
+ },
+ )
+
it('should call onClose when close button is clicked', async () => {
const onClose = vi.fn()
renderWithProvider({ onClose })
@@ -341,8 +435,8 @@ describe('ParamConfigContent', () => {
expect(getVoiceSelect()).toHaveAttribute('data-disabled')
})
- it('should disable the voices query when pathname has no app segment', () => {
- mockPathname = '/configuration'
+ it('should disable the voices query when neither an app nor Agent ID is available', () => {
+ mockParams = {}
renderWithProvider()
diff --git a/web/app/components/base/features/new-feature-panel/text-to-speech/param-config-content.tsx b/web/app/components/base/features/new-feature-panel/text-to-speech/param-config-content.tsx
index 77d6ca9658e..22c8389616c 100644
--- a/web/app/components/base/features/new-feature-panel/text-to-speech/param-config-content.tsx
+++ b/web/app/components/base/features/new-feature-panel/text-to-speech/param-config-content.tsx
@@ -21,7 +21,7 @@ import { AudioBtn } from '@/app/components/base/audio-btn'
import { useFeatures, useFeaturesStore } from '@/app/components/base/features/hooks'
import { Infotip } from '@/app/components/base/infotip'
import { languages } from '@/i18n/language'
-import { usePathname } from '@/next/navigation'
+import { useParams } from '@/next/navigation'
import { consoleQuery } from '@/service/console'
import { TtsAutoPlay } from '@/types/app'
@@ -36,9 +36,7 @@ type VoiceParamConfigProps = {
}
const VoiceParamConfig = ({ onClose, onChange }: VoiceParamConfigProps) => {
const { t } = useTranslation()
- const pathname = usePathname()
- const matched = /\/app\/([^/]+)/.exec(pathname)
- const appId = matched?.length && matched[1] ? matched[1] : ''
+ const params = useParams<{ appId?: string; agentId?: string }>()
const text2speech = useFeatures((state) => state.features.text2speech)
const featuresStore = useFeaturesStore()
const formatLanguageName = (item: SelectOption) => {
@@ -56,14 +54,21 @@ const VoiceParamConfig = ({ onClose, onChange }: VoiceParamConfigProps) => {
const language = languageItem?.value
const { data: voiceItems } = useQuery(
- consoleQuery.apps.byAppId.textToAudio.voices.get.queryOptions({
- input: appId
- ? {
- params: { app_id: appId },
+ params.agentId
+ ? consoleQuery.agent.byAgentId.textToAudio.voices.get.queryOptions({
+ input: {
+ params: { agent_id: params.agentId },
query: { language: language || 'en-US' },
- }
- : skipToken,
- }),
+ },
+ })
+ : consoleQuery.apps.byAppId.textToAudio.voices.get.queryOptions({
+ input: params.appId
+ ? {
+ params: { app_id: params.appId },
+ query: { language: language || 'en-US' },
+ }
+ : skipToken,
+ }),
)
let voiceItem = voiceItems?.find((item) => item.value === text2speech?.voice)
if (voiceItems && !voiceItem) voiceItem = voiceItems[0]
@@ -188,7 +193,7 @@ const VoiceParamConfig = ({ onClose, onChange }: VoiceParamConfigProps) => {
role="group"
aria-label={t(($) => $.play, { ns: 'appApi', defaultValue: 'Play' })}
>
- <AudioBtn value={languageItem?.example} isAudition voice={text2speech?.voice} />
+ <AudioBtn value={languageItem?.example} isAudition voice={voiceItem?.value} />
</div>
)}
</div>
diff --git a/web/service/__tests__/share-text-to-audio.spec.ts b/web/service/__tests__/share-text-to-audio.spec.ts
new file mode 100644
index 00000000000..d641bf4b30d
--- /dev/null
+++ b/web/service/__tests__/share-text-to-audio.spec.ts
@@ -0,0 +1,51 @@
+import { consoleClient } from '@/service/console'
+import { AppSourceType, textToAudioStream } from '../share'
+
+vi.mock('@/service/console', () => ({
+ consoleClient: {
+ agent: { byAgentId: { textToAudio: { post: vi.fn() } } },
+ },
+}))
+
+describe('Agent text-to-audio transport', () => {
+ beforeEach(() => {
+ vi.clearAllMocks()
+ })
+
+ it('adapts generated audio downloads to the shared player response contract', async () => {
+ const bytes = new Uint8Array([0x52, 0x49, 0x46, 0x46, 0x00, 0xff])
+ vi.mocked(consoleClient.agent.byAgentId.textToAudio.post).mockResolvedValue(
+ new Blob([bytes], { type: 'audio/wav' }),
+ )
+
+ const response = await textToAudioStream(
+ '/agent/agent-1/text-to-audio',
+ AppSourceType.installedApp,
+ {
+ text: 'Preview this voice',
+ voice: 'echo',
+ streaming: true,
+ },
+ )
+
+ expect(consoleClient.agent.byAgentId.textToAudio.post).toHaveBeenCalledWith({
+ params: { agent_id: 'agent-1' },
+ body: { text: 'Preview this voice', voice: 'echo', streaming: true },
+ })
+ expect(response).toBeInstanceOf(Response)
+ expect(response.headers.get('content-type')).toBe('audio/wav')
+ expect(new Uint8Array(await response.arrayBuffer())).toEqual(bytes)
+ })
+
+ it('propagates failed synthesis to the player', async () => {
+ const failure = new Error('No TTS model is configured')
+ vi.mocked(consoleClient.agent.byAgentId.textToAudio.post).mockRejectedValue(failure)
+
+ await expect(
+ textToAudioStream('/agent/agent-1/text-to-audio', AppSourceType.installedApp, {
+ text: 'Preview this voice',
+ streaming: true,
+ }),
+ ).rejects.toBe(failure)
+ })
+})
diff --git a/web/service/console/client.spec.ts b/web/service/console/client.spec.ts
index 59c6bd75114..e444b775258 100644
--- a/web/service/console/client.spec.ts
+++ b/web/service/console/client.spec.ts
@@ -258,6 +258,30 @@ describe('consoleQuery transport context', () => {
expect(new Uint8Array(await result.arrayBuffer())).toEqual(bytes)
})
+ it('preserves agent audition audio bytes and the provider content type', async () => {
+ const bytes = new Uint8Array([0x52, 0x49, 0x46, 0x46, 0x00, 0xff])
+ const request = vi
+ .fn()
+ .mockResolvedValue(new Response(bytes, { headers: { 'content-type': 'audio/wav' } }))
+ const consoleQuery = await loadConsoleQueryWithRequest(request)
+ const mutation = new MutationObserver(
+ new QueryClient(),
+ consoleQuery.agent.byAgentId.textToAudio.post.mutationOptions(),
+ )
+ const audio = await mutation.mutate({
+ params: { agent_id: 'agent-1' },
+ body: { text: 'Preview this voice', voice: 'echo' },
+ })
+
+ const outgoing = request.mock.calls[0]?.[2]?.request as Request
+ expect(outgoing.url).toContain('/agent/agent-1/text-to-audio')
+ expect(outgoing.method).toBe('POST')
+ expect(await outgoing.json()).toEqual({ text: 'Preview this voice', voice: 'echo' })
+ expect(audio).toBeInstanceOf(Blob)
+ expect(audio.type).toBe('audio/wav')
+ expect(new Uint8Array(await audio.arrayBuffer())).toEqual(bytes)
+ })
+
it('should forward silent context to the base request transport', async () => {
const request = vi.fn().mockResolvedValue(
new Response(JSON.stringify({}), {
diff --git a/web/service/share.ts b/web/service/share.ts
index e398cf4a9fc..246409f7d07 100644
--- a/web/service/share.ts
+++ b/web/service/share.ts
@@ -5,6 +5,7 @@ import type { ChatConfig } from '@/app/components/base/chat/types'
import type { AccessMode } from '@/models/access-control'
import type { AppConversationData, AppData, AppMeta, ConversationItem } from '@/models/share'
import { WEB_APP_SHARE_CODE_HEADER_NAME } from '@/config'
+import { consoleClient } from '@/service/console'
import {
del as consoleDel,
get as consoleGet,
@@ -408,8 +409,21 @@ export const textToAudioStream = (
message_id?: string
text?: string | null | undefined
},
-) => {
- return getAction('post', appSourceType)(url, { body }, { needAllResponseContent: true })
+): Promise<Response> => {
+ const agentId =
+ appSourceType !== AppSourceType.webApp && /^\/agent\/([^/]+)\/text-to-audio$/.exec(url)?.[1]
+ if (agentId) {
+ return (
+ consoleClient.agent.byAgentId.textToAudio
+ .post({
+ params: { agent_id: agentId },
+ body: { ...body, text: body.text ?? '' },
+ })
+ // The shared player consumes Response streams; generated binary calls return Blobs.
+ .then((audio) => new Response(audio))
+ )
+ }
+ return getAction('post', appSourceType)<Response>(url, { body }, { needAllResponseContent: true })
}
type AccessTokenResponse = { access_token: string }