Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

10.6 KB
Raw Blame
import logging

from flask import request
from flask_restx import Resource
from sqlalchemy.orm import Session
from werkzeug.exceptions import InternalServerError

import services
from controllers.common.controller_schemas import TextToAudioPayload
from controllers.common.fields import AudioBinaryResponse, AudioTranscriptResponse
from controllers.common.schema import register_response_schema_models, register_schema_model
from controllers.common.session import with_session
from controllers.console.wraps import model_validate
from controllers.service_api import service_api_ns
from controllers.service_api.app.error import (
    AppUnavailableError,
    AudioTooLargeError,
    CompletionRequestError,
    NoAudioUploadedError,
    ProviderModelCurrentlyNotSupportError,
    ProviderNotInitializeError,
    ProviderNotSupportSpeechToTextError,
    ProviderQuotaExceededError,
    SpeechToTextDisabledError,
    UnsupportedAudioTypeError,
)
from controllers.service_api.schema import binary_response, expect_with_user, multipart_file_params
from controllers.service_api.wraps import FetchUserArg, WhereisUserArg, validate_app_token
from core.base.tts.audio_mime import SUPPORTED_TTS_AUDIO_MIME_TYPES
from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
from graphon.model_runtime.errors.invoke import InvokeError
from libs.external_api import error_responses
from libs.helper import dump_response
from models.model import App, EndUser
from services.app_ref_service import AppRefService
from services.audio_service import AudioService
from services.errors.audio import (
    AudioTooLargeServiceError,
    NoAudioUploadedServiceError,
    ProviderNotSupportSpeechToTextServiceError,
    SpeechToTextDisabledServiceError,
    UnsupportedAudioTypeServiceError,
)

logger = logging.getLogger(__name__)

register_response_schema_models(service_api_ns, AudioBinaryResponse, AudioTranscriptResponse)


@service_api_ns.route("/audio-to-text")
class AudioApi(Resource):
    @error_responses(
        AppUnavailableError,
        AudioTooLargeError,
        CompletionRequestError,
        NoAudioUploadedError,
        ProviderModelCurrentlyNotSupportError,
        ProviderNotInitializeError,
        ProviderNotSupportSpeechToTextError,
        ProviderQuotaExceededError,
        SpeechToTextDisabledError,
        UnsupportedAudioTypeError,
    )
    @service_api_ns.doc(
        summary="Convert Audio to Text",
        description=(
            "Convert audio file to text. Supported MIME types: `audio/mp3`, `audio/mpga`, `audio/m4a`, "
            "`audio/x-m4a`, `audio/wav`, and `audio/amr`. File size limit is `30 MB`."
        ),
        tags=["TTS"],
        responses={
            200: "Successfully converted audio to text.",
            400: (
                "- `app_unavailable` : App unavailable or misconfigured.\n"
                "- `speech_to_text_disabled` : Speech-to-text is disabled for this app.\n"
                "- `provider_not_support_speech_to_text` : Model provider does not support speech-to-text.\n"
                "- `provider_not_initialize` : No valid model provider credentials found.\n"
                "- `provider_quota_exceeded` : Model provider quota exhausted.\n"
                "- `model_currently_not_support` : Current model does not support this operation.\n"
                "- `completion_request_error` : Speech recognition request failed."
            ),
            413: "`audio_too_large` : Audio file size exceeded the limit.",
            415: "`unsupported_audio_type` : Audio type is not allowed.",
            500: "`internal_server_error` : Internal server error.",
        },
    )
    @service_api_ns.doc("audio_to_text")
    @service_api_ns.doc(description="Convert audio to text using speech-to-text")
    @service_api_ns.doc(
        consumes=["multipart/form-data"],
        params=multipart_file_params(
            include_user=True,
            file_description=(
                "Audio file to transcribe. Supported MIME types: `audio/mp3`, `audio/mpga`, `audio/m4a`, "
                "`audio/x-m4a`, `audio/wav`, and `audio/amr`. File size limit is `30 MB`."
            ),
        ),
    )
    @service_api_ns.doc(
        responses={
            200: "Audio successfully transcribed",
            400: "Bad request - no audio or invalid audio",
            401: "Unauthorized - invalid API token",
            413: "Audio file too large",
            415: "Unsupported audio type",
            500: "Internal server error",
        }
    )
    @service_api_ns.response(
        200,
        "Audio successfully transcribed",
        service_api_ns.models[AudioTranscriptResponse.__name__],
    )
    @validate_app_token(fetch_user_arg=FetchUserArg(fetch_from=WhereisUserArg.FORM))
    @with_session(write=False)
    def post(self, session: Session, app_model: App, end_user: EndUser):
        """Convert audio to text using speech-to-text.

        Accepts an audio file upload and returns the transcribed text.
        """
        file = request.files.get("file")

        try:
            response = AudioService.transcript_asr(
                app_model=app_model,
                file=file,
                session=session,
                end_user=end_user.id,
            )

            return dump_response(AudioTranscriptResponse, response)
        except services.errors.app_model_config.AppModelConfigBrokenError as caught_error:
            logger.exception("App model config broken.")
            raise AppUnavailableError() from caught_error
        except NoAudioUploadedServiceError as caught_error:
            raise NoAudioUploadedError() from caught_error
        except AudioTooLargeServiceError as e:
            raise AudioTooLargeError(str(e)) from e
        except UnsupportedAudioTypeServiceError as caught_error:
            raise UnsupportedAudioTypeError() from caught_error
        except ProviderNotSupportSpeechToTextServiceError as caught_error:
            raise ProviderNotSupportSpeechToTextError() from caught_error
        except SpeechToTextDisabledServiceError as caught_error:
            raise SpeechToTextDisabledError() from caught_error
        except ProviderTokenNotInitError as ex:
            raise ProviderNotInitializeError(ex.description) from ex
        except QuotaExceededError as caught_error:
            raise ProviderQuotaExceededError() from caught_error
        except ModelCurrentlyNotSupportError as caught_error:
            raise ProviderModelCurrentlyNotSupportError() from caught_error
        except InvokeError as e:
            raise CompletionRequestError(e.description) from e
        except ValueError as e:
            raise e
        except Exception as e:
            logger.exception("internal server error.")
            raise InternalServerError() from e


register_schema_model(service_api_ns, TextToAudioPayload)


@service_api_ns.route("/text-to-audio")
class TextApi(Resource):
    @error_responses(
        AppUnavailableError,
        CompletionRequestError,
        ProviderModelCurrentlyNotSupportError,
        ProviderNotInitializeError,
        ProviderQuotaExceededError,
    )
    @service_api_ns.doc(
        summary="Convert Text to Audio",
        description="Convert text to speech.",
        tags=["TTS"],
        responses={
            200: (
                "Returns the generated audio. The `Content-Type` header reflects the provider audio container, "
                "verified from the response bytes when recognizable. The binary response can be AAC, FLAC, MP4, "
                "MP3, Ogg, WAV, or WebM."
            ),
            400: (
                "- `app_unavailable` : App unavailable or misconfigured.\n"
                "- `provider_not_initialize` : No valid model provider credentials found.\n"
                "- `provider_quota_exceeded` : Model provider quota exhausted.\n"
                "- `model_currently_not_support` : Current model does not support this operation.\n"
                "- `completion_request_error` : Text-to-speech request failed."
            ),
            500: "`internal_server_error` : Internal server error.",
        },
    )
    @expect_with_user(service_api_ns, TextToAudioPayload)
    @binary_response(service_api_ns, SUPPORTED_TTS_AUDIO_MIME_TYPES)
    @service_api_ns.doc("text_to_audio")
    @service_api_ns.doc(description="Convert text to audio using text-to-speech")
    @service_api_ns.doc(
        responses={
            200: "Text successfully converted to audio",
            400: "Bad request - invalid parameters",
            401: "Unauthorized - invalid API token",
            500: "Internal server error",
        }
    )
    # TTS returns provider audio bytes, so the success response is intentionally schema-less.
    @service_api_ns.response(200, "Text successfully converted to audio")
    @validate_app_token(fetch_user_arg=FetchUserArg(fetch_from=WhereisUserArg.JSON))
    @with_session(write=False)
    @model_validate(TextToAudioPayload)
    def post(self, payload: TextToAudioPayload, session: Session, app_model: App, end_user: EndUser):
        """Convert text to audio using text-to-speech.

        Converts the provided text to audio using the specified voice.
        """
        try:
            message_id = payload.message_id
            text = payload.text
            voice = payload.voice
            message_ref = None
            if message_id:
                app_ref = AppRefService.create_app_ref(app_model)
                message_ref = AppRefService.create_message_ref(
                    app_ref,
                    message_id,
                    end_user_id=end_user.id,
                )
            return AudioService.transcript_tts(
                app_model=app_model,
                session=session,
                text=text,
                voice=voice,
                end_user=end_user.external_user_id,
                message_ref=message_ref,
            )
        except services.errors.app_model_config.AppModelConfigBrokenError as caught_error:
            logger.exception("App model config broken.")
            raise AppUnavailableError() from caught_error
        except ProviderTokenNotInitError as ex:
            raise ProviderNotInitializeError(ex.description) from ex
        except QuotaExceededError as caught_error:
            raise ProviderQuotaExceededError() from caught_error
        except ModelCurrentlyNotSupportError as caught_error:
            raise ProviderModelCurrentlyNotSupportError() from caught_error
        except InvokeError as e:
            raise CompletionRequestError(e.description) from e
        except ValueError as e:
            raise e
        except Exception as e:
            logger.exception("internal server error.")
            raise InternalServerError() from e