RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: implement Tongyi-Qianwen TTS via DashScope OpenAI-compatible endpoint (#17770)
euvreb4c2431a8be1
diff --git a/conf/models/aliyun.json b/conf/models/aliyun.jsonindex affa529a13..e0cc8f4ef4 100644--- a/conf/models/aliyun.json+++ b/conf/models/aliyun.json@@ -10,7 +10,8 @@"chat": "compatible-mode/v1/chat/completions","embedding": "compatible-mode/v1/embeddings","rerank": "compatible-api/v1/reranks",- "models": "compatible-mode/v1/models"+ "models": "compatible-mode/v1/models",+ "tts": "compatible-mode/v1/audio/speech"},"models": [{diff --git a/internal/entity/models/aliyun.go b/internal/entity/models/aliyun.goindex 5e50a1a640..59f30f7d8e 100644--- a/internal/entity/models/aliyun.go+++ b/internal/entity/models/aliyun.go@@ -389,13 +389,104 @@ func (a *AliyunModel) TranscribeAudioWithSender(ctx context.Context, modelName *return fmt.Errorf("%s, no such method", a.Name())}+// aliyunTTSDefaultVoice is used when the caller does not specify a voice;+// DashScope's Qwen TTS models require one.+const aliyunTTSDefaultVoice = "Cherry"++// newAliyunTTSRequest builds the OpenAI-compatible audio/speech request+// against DashScope's compatible mode.+func (a *AliyunModel) newAliyunTTSRequest(ctx context.Context, modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig) (*http.Request, error) {+ if err := a.baseModel.APIConfigCheck(apiConfig); err != nil {+ return nil, err+ }+ if modelName == nil || *modelName == "" {+ return nil, fmt.Errorf("model name is required")+ }+ if audioContent == nil || *audioContent == "" {+ return nil, fmt.Errorf("audio content is empty")+ }+ if strings.TrimSpace(a.baseModel.URLSuffix.TTS) == "" {+ return nil, fmt.Errorf("aliyun TTS URL suffix is required")+ }++ resolvedBaseURL, err := a.baseModel.GetBaseURL(apiConfig)+ if err != nil {+ return nil, err+ }+ url := fmt.Sprintf("%s/%s", strings.TrimSuffix(resolvedBaseURL, "/"), strings.TrimPrefix(a.baseModel.URLSuffix.TTS, "/"))++ reqBody := map[string]interface{}{+ "model": *modelName,+ "input": *audioContent,+ "voice": aliyunTTSDefaultVoice,+ }+ if ttsConfig != nil {+ for key, value := range ttsConfig.Params {+ reqBody[key] = value+ }+ if ttsConfig.Format != "" {+ reqBody["response_format"] = ttsConfig.Format+ }+ }++ jsonData, err := json.Marshal(reqBody)+ if err != nil {+ return nil, fmt.Errorf("failed to marshal request: %w", err)+ }++ req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData))+ if err != nil {+ return nil, fmt.Errorf("failed to create request: %w", err)+ }+ req.Header.Set("Content-Type", "application/json")+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))+ return req, nil+}+// AudioSpeech convert text to audiofunc (a *AliyunModel) AudioSpeech(ctx context.Context, modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, modelUsage *common.ModelUsage) (*TTSResponse, error) {- return nil, fmt.Errorf("%s, no such method", a.Name())+ ctx, cancel := context.WithTimeout(ctx, nonStreamCallTimeout)+ defer cancel()++ req, err := a.newAliyunTTSRequest(ctx, modelName, audioContent, apiConfig, ttsConfig)+ if err != nil {+ return nil, err+ }++ resp, err := a.baseModel.httpClient.Do(req)+ if err != nil {+ return nil, fmt.Errorf("failed to send request: %w", err)+ }+ defer resp.Body.Close()++ body, err := io.ReadAll(resp.Body)+ if err != nil {+ return nil, fmt.Errorf("failed to read response body: %w", err)+ }++ if resp.StatusCode != http.StatusOK {+ return nil, fmt.Errorf("aliyun TTS API error: %s, body: %s", resp.Status, string(body))+ }+ if len(body) == 0 {+ return nil, fmt.Errorf("aliyun TTS API returned empty audio")+ }++ return &TTSResponse{Audio: body}, nil}func (a *AliyunModel) AudioSpeechWithSender(ctx context.Context, modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, modelUsage *common.ModelUsage, sender func(*string, *string) error) error {- return fmt.Errorf("%s, no such method", a.Name())+ if sender == nil {+ return fmt.Errorf("sender is required")+ }++ // DashScope's compatible-mode audio/speech endpoint returns the whole+ // audio in one response; forward it as a single chunk.+ resp, err := a.AudioSpeech(ctx, modelName, audioContent, apiConfig, ttsConfig, modelUsage)+ if err != nil {+ return err+ }+ chunk := string(resp.Audio)+ return sender(&chunk, nil)}// OCRFile OCR filediff --git a/internal/entity/models/aliyun_test.go b/internal/entity/models/aliyun_test.goindex e9ae59263e..4cd7074435 100644--- a/internal/entity/models/aliyun_test.go+++ b/internal/entity/models/aliyun_test.go@@ -306,3 +306,178 @@ func TestAliyunChatStreamlyWithSenderRejectsStreamFalse(t *testing.T) {t.Fatalf("error = %v, want stream validation error", err)}}++func TestAliyunAudioSpeechSynthesizesViaCompatibleEndpoint(t *testing.T) {+ withSSRFBypass(t)+ requestBody := make(chan map[string]interface{}, 1)+ requestPath := make(chan string, 1)+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ var body map[string]interface{}+ if err := json.NewDecoder(r.Body).Decode(&body); err != nil {+ http.Error(w, err.Error(), http.StatusBadRequest)+ return+ }+ requestPath <- r.URL.Path+ requestBody <- body+ w.Header().Set("Content-Type", "audio/mpeg")+ _, _ = w.Write([]byte("fake-mp3-bytes"))+ }))+ defer server.Close()+ ctx := t.Context()++ model := NewAliyunModel(+ map[string]string{"default": server.URL},+ URLSuffix{TTS: "compatible-mode/v1/audio/speech"},+ )+ apiKey := "test-key"+ modelName := "qwen-tts-flash"+ text := "你好,世界"++ response, err := model.AudioSpeech(+ ctx,+ &modelName,+ &text,+ &APIConfig{ApiKey: &apiKey},+ &TTSConfig{Format: "mp3"},+ nil,+ )+ if err != nil {+ t.Fatalf("AudioSpeech: %v", err)+ }+ if string(response.Audio) != "fake-mp3-bytes" {+ t.Errorf("audio = %q, want fake-mp3-bytes", string(response.Audio))+ }++ if got := <-requestPath; got != "/compatible-mode/v1/audio/speech" {+ t.Errorf("request path = %q, want /compatible-mode/v1/audio/speech", got)+ }+ body := <-requestBody+ if body["model"] != "qwen-tts-flash" {+ t.Errorf("model = %v, want qwen-tts-flash", body["model"])+ }+ if body["input"] != "你好,世界" {+ t.Errorf("input = %v, want 你好,世界", body["input"])+ }+ if body["voice"] != aliyunTTSDefaultVoice {+ t.Errorf("voice = %v, want default %s", body["voice"], aliyunTTSDefaultVoice)+ }+ if body["response_format"] != "mp3" {+ t.Errorf("response_format = %v, want mp3", body["response_format"])+ }+}++func TestAliyunAudioSpeechHonorsExplicitVoice(t *testing.T) {+ withSSRFBypass(t)+ requestBody := make(chan map[string]interface{}, 1)+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ var body map[string]interface{}+ if err := json.NewDecoder(r.Body).Decode(&body); err != nil {+ http.Error(w, err.Error(), http.StatusBadRequest)+ return+ }+ requestBody <- body+ _, _ = w.Write([]byte("audio"))+ }))+ defer server.Close()+ ctx := t.Context()++ model := NewAliyunModel(+ map[string]string{"default": server.URL},+ URLSuffix{TTS: "compatible-mode/v1/audio/speech"},+ )+ apiKey := "test-key"+ modelName := "qwen-tts-flash"+ text := "hello"++ if _, err := model.AudioSpeech(+ ctx,+ &modelName,+ &text,+ &APIConfig{ApiKey: &apiKey},+ &TTSConfig{Params: map[string]any{"voice": "Serena"}},+ nil,+ ); err != nil {+ t.Fatalf("AudioSpeech: %v", err)+ }+ if got := (<-requestBody)["voice"]; got != "Serena" {+ t.Errorf("voice = %v, want Serena", got)+ }+}++func TestAliyunAudioSpeechSurfacesAPIError(t *testing.T) {+ withSSRFBypass(t)+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ http.Error(w, `{"error":{"message":"Invalid api-key"}}`, http.StatusUnauthorized)+ }))+ defer server.Close()+ ctx := t.Context()++ model := NewAliyunModel(+ map[string]string{"default": server.URL},+ URLSuffix{TTS: "compatible-mode/v1/audio/speech"},+ )+ apiKey := "bad-key"+ modelName := "qwen-tts-flash"+ text := "hello"++ _, err := model.AudioSpeech(ctx, &modelName, &text, &APIConfig{ApiKey: &apiKey}, nil, nil)+ if err == nil {+ t.Fatal("error = nil, want API error")+ }+}++func TestAliyunAudioSpeechRequiresTTSSuffix(t *testing.T) {+ withSSRFBypass(t)+ ctx := t.Context()+ model := NewAliyunModel(+ map[string]string{"default": "https://dashscope.example"},+ URLSuffix{Chat: "compatible-mode/v1/chat/completions"},+ )+ apiKey := "test-key"+ modelName := "qwen-tts-flash"+ text := "hello"++ _, err := model.AudioSpeech(ctx, &modelName, &text, &APIConfig{ApiKey: &apiKey}, nil, nil)+ if err == nil || err.Error() != "aliyun TTS URL suffix is required" {+ t.Fatalf("error = %v, want missing TTS suffix error", err)+ }+}++func TestAliyunAudioSpeechWithSenderSendsSingleChunk(t *testing.T) {+ withSSRFBypass(t)+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ _, _ = w.Write([]byte("whole-audio"))+ }))+ defer server.Close()+ ctx := t.Context()++ model := NewAliyunModel(+ map[string]string{"default": server.URL},+ URLSuffix{TTS: "compatible-mode/v1/audio/speech"},+ )+ apiKey := "test-key"+ modelName := "qwen-tts-flash"+ text := "hello"++ var chunks []string+ err := model.AudioSpeechWithSender(+ ctx,+ &modelName,+ &text,+ &APIConfig{ApiKey: &apiKey},+ nil,+ nil,+ func(content, _ *string) error {+ if content != nil {+ chunks = append(chunks, *content)+ }+ return nil+ },+ )+ if err != nil {+ t.Fatalf("AudioSpeechWithSender: %v", err)+ }+ if len(chunks) != 1 || chunks[0] != "whole-audio" {+ t.Fatalf("chunks = %v, want [whole-audio]", chunks)+ }+}