Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: implement Tongyi-Qianwen TTS via DashScope OpenAI-compatible endpoint (#17770)

euvreb4c2431a8be1
diff --git a/conf/models/aliyun.json b/conf/models/aliyun.json
index affa529a13..e0cc8f4ef4 100644
--- a/conf/models/aliyun.json
+++ b/conf/models/aliyun.json
@@ -10,7 +10,8 @@
"chat": "compatible-mode/v1/chat/completions",
"embedding": "compatible-mode/v1/embeddings",
"rerank": "compatible-api/v1/reranks",
- "models": "compatible-mode/v1/models"
+ "models": "compatible-mode/v1/models",
+ "tts": "compatible-mode/v1/audio/speech"
},
"models": [
{
diff --git a/internal/entity/models/aliyun.go b/internal/entity/models/aliyun.go
index 5e50a1a640..59f30f7d8e 100644
--- a/internal/entity/models/aliyun.go
+++ b/internal/entity/models/aliyun.go
@@ -389,13 +389,104 @@ func (a *AliyunModel) TranscribeAudioWithSender(ctx context.Context, modelName *
return fmt.Errorf("%s, no such method", a.Name())
}
+// aliyunTTSDefaultVoice is used when the caller does not specify a voice;
+// DashScope's Qwen TTS models require one.
+const aliyunTTSDefaultVoice = "Cherry"
+
+// newAliyunTTSRequest builds the OpenAI-compatible audio/speech request
+// against DashScope's compatible mode.
+func (a *AliyunModel) newAliyunTTSRequest(ctx context.Context, modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*http.Request, error) {
+ if err := a.baseModel.APIConfigCheck(apiConfig); err != nil {
+ return nil, err
+ }
+ if modelName == nil || *modelName == "" {
+ return nil, fmt.Errorf("model name is required")
+ }
+ if audioContent == nil || *audioContent == "" {
+ return nil, fmt.Errorf("audio content is empty")
+ }
+ if strings.TrimSpace(a.baseModel.URLSuffix.TTS) == "" {
+ return nil, fmt.Errorf("aliyun TTS URL suffix is required")
+ }
+
+ resolvedBaseURL, err := a.baseModel.GetBaseURL(apiConfig)
+ if err != nil {
+ return nil, err
+ }
+ url := fmt.Sprintf("%s/%s", strings.TrimSuffix(resolvedBaseURL, "/"), strings.TrimPrefix(a.baseModel.URLSuffix.TTS, "/"))
+
+ reqBody := map[string]interface{}{
+ "model": *modelName,
+ "input": *audioContent,
+ "voice": aliyunTTSDefaultVoice,
+ }
+ if ttsConfig != nil {
+ for key, value := range ttsConfig.Params {
+ reqBody[key] = value
+ }
+ if ttsConfig.Format != "" {
+ reqBody["response_format"] = ttsConfig.Format
+ }
+ }
+
+ jsonData, err := json.Marshal(reqBody)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+ return req, nil
+}
+
// AudioSpeech convert text to audio
func (a *AliyunModel) AudioSpeech(ctx context.Context, modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, modelUsage *common.ModelUsage) (*TTSResponse, error) {
- return nil, fmt.Errorf("%s, no such method", a.Name())
+ ctx, cancel := context.WithTimeout(ctx, nonStreamCallTimeout)
+ defer cancel()
+
+ req, err := a.newAliyunTTSRequest(ctx, modelName, audioContent, apiConfig, ttsConfig)
+ if err != nil {
+ return nil, err
+ }
+
+ resp, err := a.baseModel.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response body: %w", err)
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("aliyun TTS API error: %s, body: %s", resp.Status, string(body))
+ }
+ if len(body) == 0 {
+ return nil, fmt.Errorf("aliyun TTS API returned empty audio")
+ }
+
+ return &TTSResponse{Audio: body}, nil
}
func (a *AliyunModel) AudioSpeechWithSender(ctx context.Context, modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, modelUsage *common.ModelUsage, sender func(*string, *string) error) error {
- return fmt.Errorf("%s, no such method", a.Name())
+ if sender == nil {
+ return fmt.Errorf("sender is required")
+ }
+
+ // DashScope's compatible-mode audio/speech endpoint returns the whole
+ // audio in one response; forward it as a single chunk.
+ resp, err := a.AudioSpeech(ctx, modelName, audioContent, apiConfig, ttsConfig, modelUsage)
+ if err != nil {
+ return err
+ }
+ chunk := string(resp.Audio)
+ return sender(&chunk, nil)
}
// OCRFile OCR file
diff --git a/internal/entity/models/aliyun_test.go b/internal/entity/models/aliyun_test.go
index e9ae59263e..4cd7074435 100644
--- a/internal/entity/models/aliyun_test.go
+++ b/internal/entity/models/aliyun_test.go
@@ -306,3 +306,178 @@ func TestAliyunChatStreamlyWithSenderRejectsStreamFalse(t *testing.T) {
t.Fatalf("error = %v, want stream validation error", err)
}
}
+
+func TestAliyunAudioSpeechSynthesizesViaCompatibleEndpoint(t *testing.T) {
+ withSSRFBypass(t)
+ requestBody := make(chan map[string]interface{}, 1)
+ requestPath := make(chan string, 1)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var body map[string]interface{}
+ if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+ requestPath <- r.URL.Path
+ requestBody <- body
+ w.Header().Set("Content-Type", "audio/mpeg")
+ _, _ = w.Write([]byte("fake-mp3-bytes"))
+ }))
+ defer server.Close()
+ ctx := t.Context()
+
+ model := NewAliyunModel(
+ map[string]string{"default": server.URL},
+ URLSuffix{TTS: "compatible-mode/v1/audio/speech"},
+ )
+ apiKey := "test-key"
+ modelName := "qwen-tts-flash"
+ text := "你好,世界"
+
+ response, err := model.AudioSpeech(
+ ctx,
+ &modelName,
+ &text,
+ &APIConfig{ApiKey: &apiKey},
+ &TTSConfig{Format: "mp3"},
+ nil,
+ )
+ if err != nil {
+ t.Fatalf("AudioSpeech: %v", err)
+ }
+ if string(response.Audio) != "fake-mp3-bytes" {
+ t.Errorf("audio = %q, want fake-mp3-bytes", string(response.Audio))
+ }
+
+ if got := <-requestPath; got != "/compatible-mode/v1/audio/speech" {
+ t.Errorf("request path = %q, want /compatible-mode/v1/audio/speech", got)
+ }
+ body := <-requestBody
+ if body["model"] != "qwen-tts-flash" {
+ t.Errorf("model = %v, want qwen-tts-flash", body["model"])
+ }
+ if body["input"] != "你好,世界" {
+ t.Errorf("input = %v, want 你好,世界", body["input"])
+ }
+ if body["voice"] != aliyunTTSDefaultVoice {
+ t.Errorf("voice = %v, want default %s", body["voice"], aliyunTTSDefaultVoice)
+ }
+ if body["response_format"] != "mp3" {
+ t.Errorf("response_format = %v, want mp3", body["response_format"])
+ }
+}
+
+func TestAliyunAudioSpeechHonorsExplicitVoice(t *testing.T) {
+ withSSRFBypass(t)
+ requestBody := make(chan map[string]interface{}, 1)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var body map[string]interface{}
+ if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+ requestBody <- body
+ _, _ = w.Write([]byte("audio"))
+ }))
+ defer server.Close()
+ ctx := t.Context()
+
+ model := NewAliyunModel(
+ map[string]string{"default": server.URL},
+ URLSuffix{TTS: "compatible-mode/v1/audio/speech"},
+ )
+ apiKey := "test-key"
+ modelName := "qwen-tts-flash"
+ text := "hello"
+
+ if _, err := model.AudioSpeech(
+ ctx,
+ &modelName,
+ &text,
+ &APIConfig{ApiKey: &apiKey},
+ &TTSConfig{Params: map[string]any{"voice": "Serena"}},
+ nil,
+ ); err != nil {
+ t.Fatalf("AudioSpeech: %v", err)
+ }
+ if got := (<-requestBody)["voice"]; got != "Serena" {
+ t.Errorf("voice = %v, want Serena", got)
+ }
+}
+
+func TestAliyunAudioSpeechSurfacesAPIError(t *testing.T) {
+ withSSRFBypass(t)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ http.Error(w, `{"error":{"message":"Invalid api-key"}}`, http.StatusUnauthorized)
+ }))
+ defer server.Close()
+ ctx := t.Context()
+
+ model := NewAliyunModel(
+ map[string]string{"default": server.URL},
+ URLSuffix{TTS: "compatible-mode/v1/audio/speech"},
+ )
+ apiKey := "bad-key"
+ modelName := "qwen-tts-flash"
+ text := "hello"
+
+ _, err := model.AudioSpeech(ctx, &modelName, &text, &APIConfig{ApiKey: &apiKey}, nil, nil)
+ if err == nil {
+ t.Fatal("error = nil, want API error")
+ }
+}
+
+func TestAliyunAudioSpeechRequiresTTSSuffix(t *testing.T) {
+ withSSRFBypass(t)
+ ctx := t.Context()
+ model := NewAliyunModel(
+ map[string]string{"default": "https://dashscope.example"},
+ URLSuffix{Chat: "compatible-mode/v1/chat/completions"},
+ )
+ apiKey := "test-key"
+ modelName := "qwen-tts-flash"
+ text := "hello"
+
+ _, err := model.AudioSpeech(ctx, &modelName, &text, &APIConfig{ApiKey: &apiKey}, nil, nil)
+ if err == nil || err.Error() != "aliyun TTS URL suffix is required" {
+ t.Fatalf("error = %v, want missing TTS suffix error", err)
+ }
+}
+
+func TestAliyunAudioSpeechWithSenderSendsSingleChunk(t *testing.T) {
+ withSSRFBypass(t)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _, _ = w.Write([]byte("whole-audio"))
+ }))
+ defer server.Close()
+ ctx := t.Context()
+
+ model := NewAliyunModel(
+ map[string]string{"default": server.URL},
+ URLSuffix{TTS: "compatible-mode/v1/audio/speech"},
+ )
+ apiKey := "test-key"
+ modelName := "qwen-tts-flash"
+ text := "hello"
+
+ var chunks []string
+ err := model.AudioSpeechWithSender(
+ ctx,
+ &modelName,
+ &text,
+ &APIConfig{ApiKey: &apiKey},
+ nil,
+ nil,
+ func(content, _ *string) error {
+ if content != nil {
+ chunks = append(chunks, *content)
+ }
+ return nil
+ },
+ )
+ if err != nil {
+ t.Fatalf("AudioSpeechWithSender: %v", err)
+ }
+ if len(chunks) != 1 || chunks[0] != "whole-audio" {
+ t.Fatalf("chunks = %v, want [whole-audio]", chunks)
+ }
+}