RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(tts): send required voice payload for SiliconFlow TTS and surface synthesis failures (#19812)
euvred1b2d80ad7d6
diff --git a/internal/entity/models/siliconflow.go b/internal/entity/models/siliconflow.goindex 96ff7e31f1..8d51a1b8c8 100644--- a/internal/entity/models/siliconflow.go+++ b/internal/entity/models/siliconflow.go@@ -626,6 +626,33 @@ func (s *SiliconflowModel) TranscribeAudioWithSender(ctx context.Context, modelNreturn fmt.Errorf("%s, no such method", s.Name())}+// siliconflowTTSRequestBody mirrors rag/llm/tts_model.py SILICONFLOWTTS._build_payload:+// /v1/audio/speech requires voice as "{model}:{voice_name}" and only synthesizes+// mp3 at 32000 or 44100 Hz, so the defaults pin the payload to a shape the+// provider accepts. ttsConfig.Params overrides any field; Format overrides+// response_format last.+func siliconflowTTSRequestBody(modelName, audioContent string, ttsConfig *TTSConfig, stream bool) map[string]interface{} {+ reqBody := map[string]interface{}{+ "model": modelName,+ "input": audioContent,+ "voice": fmt.Sprintf("%s:anna", modelName),+ "response_format": "mp3",+ "sample_rate": 32000,+ "speed": 1,+ "gain": 0,+ "stream": stream,+ }+ if ttsConfig != nil {+ for key, value := range ttsConfig.Params {+ reqBody[key] = value+ }+ if ttsConfig.Format != "" {+ reqBody["response_format"] = ttsConfig.Format+ }+ }+ return reqBody+}+// AudioSpeech convert text to audiofunc (s *SiliconflowModel) AudioSpeech(ctx context.Context, modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, modelUsage *common.ModelUsage) (*TTSResponse, error) {if err := s.baseModel.APIConfigCheck(apiConfig); err != nil {@@ -642,22 +669,7 @@ func (s *SiliconflowModel) AudioSpeech(ctx context.Context, modelName *string, a}url := fmt.Sprintf("%s/%s", resolvedBaseURL, s.baseModel.URLSuffix.TTS)- reqBody := map[string]interface{}{- "model": *modelName,- "input": *audioContent,- "stream": false,- }-- if ttsConfig != nil && ttsConfig.Params != nil {- for key, value := range ttsConfig.Params {- reqBody[key] = value- }- }- if ttsConfig != nil && ttsConfig.Format != "" {- reqBody["response_format"] = ttsConfig.Format- }-- jsonData, err := json.Marshal(reqBody)+ jsonData, err := json.Marshal(siliconflowTTSRequestBody(*modelName, *audioContent, ttsConfig, false))if err != nil {return nil, fmt.Errorf("failed to marshal request: %w", err)}@@ -685,7 +697,7 @@ func (s *SiliconflowModel) AudioSpeech(ctx context.Context, modelName *string, a}if resp.StatusCode != http.StatusOK {- return nil, fmt.Errorf("%s - %s", resp.Status, string(body))+ return nil, fmt.Errorf("SiliconFlow TTS API error: %s - %s", resp.Status, string(body))}return &TTSResponse{Audio: body}, nil@@ -706,22 +718,7 @@ func (s *SiliconflowModel) AudioSpeechWithSender(ctx context.Context, modelName}url := fmt.Sprintf("%s/%s", resolvedBaseURL, s.baseModel.URLSuffix.TTS)- reqBody := map[string]interface{}{- "model": *modelName,- "input": *audioContent,- "stream": true,- }-- if ttsConfig != nil && ttsConfig.Params != nil {- for key, value := range ttsConfig.Params {- reqBody[key] = value- }- }- if ttsConfig != nil && ttsConfig.Format != "" {- reqBody["response_format"] = ttsConfig.Format- }-- jsonData, err := json.Marshal(reqBody)+ jsonData, err := json.Marshal(siliconflowTTSRequestBody(*modelName, *audioContent, ttsConfig, true))if err != nil {return fmt.Errorf("failed to marshal request: %w", err)}diff --git a/internal/entity/models/siliconflow_test.go b/internal/entity/models/siliconflow_test.goindex f87cdb12f4..c723b52c61 100644--- a/internal/entity/models/siliconflow_test.go+++ b/internal/entity/models/siliconflow_test.go@@ -21,6 +21,7 @@ import ("io""net/http""net/http/httptest"+ "strings""testing")@@ -111,6 +112,147 @@ func TestSiliconflowChatWithMessagesExtractsResponseAndUsage(t *testing.T) {}}+func TestSiliconflowAudioSpeechSendsRequiredVoice(t *testing.T) {+ withSSRFBypass(t)+ var gotPayload map[string]any+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if err := json.NewDecoder(r.Body).Decode(&gotPayload); err != nil {+ t.Errorf("decode request: %v", err)+ return+ }+ voice, _ := gotPayload["voice"].(string)+ if voice != "FunAudioLLM/CosyVoice2-0.5B:anna" {+ w.WriteHeader(http.StatusBadRequest)+ _, _ = w.Write([]byte(`{"code":20052,"message":"Voice or reference audio should be set","data":null}`))+ return+ }+ w.Header().Set("Content-Type", "audio/mpeg")+ _, _ = w.Write([]byte("mp3-bytes"))+ }))+ defer server.Close()++ apiKey := "test-key"+ modelName := "FunAudioLLM/CosyVoice2-0.5B"+ text := "hello"+ resp, err := NewSiliconflowModel(+ map[string]string{"default": server.URL},+ URLSuffix{TTS: "audio/speech"},+ ).AudioSpeech(t.Context(), &modelName, &text, &APIConfig{ApiKey: &apiKey}, nil, nil)+ if err != nil {+ t.Fatalf("AudioSpeech: %v", err)+ }+ if string(resp.Audio) != "mp3-bytes" {+ t.Fatalf("Audio=%q, want mp3-bytes", resp.Audio)+ }+ want := map[string]any{+ "model": "FunAudioLLM/CosyVoice2-0.5B",+ "input": "hello",+ "voice": "FunAudioLLM/CosyVoice2-0.5B:anna",+ "response_format": "mp3",+ "sample_rate": float64(32000),+ "speed": float64(1),+ "gain": float64(0),+ "stream": false,+ }+ for key, value := range want {+ if gotPayload[key] != value {+ t.Errorf("payload[%q]=%#v, want %#v", key, gotPayload[key], value)+ }+ }+}++func TestSiliconflowAudioSpeechParamsOverrideVoice(t *testing.T) {+ withSSRFBypass(t)+ var gotPayload map[string]any+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ _ = json.NewDecoder(r.Body).Decode(&gotPayload)+ w.Header().Set("Content-Type", "audio/mpeg")+ _, _ = w.Write([]byte("mp3-bytes"))+ }))+ defer server.Close()++ apiKey := "test-key"+ modelName := "FunAudioLLM/CosyVoice2-0.5B"+ text := "hello"+ resp, err := NewSiliconflowModel(+ map[string]string{"default": server.URL},+ URLSuffix{TTS: "audio/speech"},+ ).AudioSpeech(t.Context(), &modelName, &text, &APIConfig{ApiKey: &apiKey},+ &TTSConfig{Params: map[string]interface{}{"voice": "FunAudioLLM/CosyVoice2-0.5B:benjamin"}}, nil)+ if err != nil {+ t.Fatalf("AudioSpeech: %v", err)+ }+ if string(resp.Audio) != "mp3-bytes" {+ t.Fatalf("Audio=%q, want mp3-bytes", resp.Audio)+ }+ if gotPayload["voice"] != "FunAudioLLM/CosyVoice2-0.5B:benjamin" {+ t.Fatalf("voice=%#v, want the Params override", gotPayload["voice"])+ }+}++func TestSiliconflowAudioSpeechErrorIncludesProviderAndBody(t *testing.T) {+ withSSRFBypass(t)+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ w.WriteHeader(http.StatusBadRequest)+ _, _ = w.Write([]byte(`{"code":20052,"message":"Voice or reference audio should be set","data":null}`))+ }))+ defer server.Close()++ apiKey := "test-key"+ modelName := "FunAudioLLM/CosyVoice2-0.5B"+ text := "hello"+ _, err := NewSiliconflowModel(+ map[string]string{"default": server.URL},+ URLSuffix{TTS: "audio/speech"},+ ).AudioSpeech(t.Context(), &modelName, &text, &APIConfig{ApiKey: &apiKey}, nil, nil)+ if err == nil {+ t.Fatal("expected error for provider rejection")+ }+ for _, want := range []string{"SiliconFlow TTS API error", "400 Bad Request", "Voice or reference audio should be set"} {+ if !strings.Contains(err.Error(), want) {+ t.Errorf("err = %v, want it to contain %q", err, want)+ }+ }+}++func TestSiliconflowAudioSpeechWithSenderSendsRequiredVoice(t *testing.T) {+ withSSRFBypass(t)+ var gotPayload map[string]any+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ _ = json.NewDecoder(r.Body).Decode(&gotPayload)+ w.Header().Set("Content-Type", "audio/mpeg")+ _, _ = w.Write([]byte("mp3-bytes"))+ }))+ defer server.Close()++ apiKey := "test-key"+ modelName := "FunAudioLLM/CosyVoice2-0.5B"+ text := "hello"+ var chunks []string+ err := NewSiliconflowModel(+ map[string]string{"default": server.URL},+ URLSuffix{TTS: "audio/speech"},+ ).AudioSpeechWithSender(t.Context(), &modelName, &text, &APIConfig{ApiKey: &apiKey}, nil, nil,+ func(content *string, _ *string) error {+ if content != nil {+ chunks = append(chunks, *content)+ }+ return nil+ })+ if err != nil {+ t.Fatalf("AudioSpeechWithSender: %v", err)+ }+ if gotPayload["voice"] != "FunAudioLLM/CosyVoice2-0.5B:anna" {+ t.Fatalf("voice=%#v, want model-qualified default voice", gotPayload["voice"])+ }+ if gotPayload["stream"] != true {+ t.Fatalf("stream=%#v, want true", gotPayload["stream"])+ }+ if len(chunks) == 0 {+ t.Fatal("expected streamed audio chunks")+ }+}+func TestSiliconflowChatStreamlyWithSenderCollectsUsage(t *testing.T) {withSSRFBypass(t)server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {diff --git a/internal/handler/chat_audio.go b/internal/handler/chat_audio.goindex 9b39d25e16..e08f3061e9 100644--- a/internal/handler/chat_audio.go+++ b/internal/handler/chat_audio.go@@ -102,6 +102,7 @@ func (h *ChatHandler) ChatAudioSpeech(c *gin.Context) {// a single WAV before being written.var wavFmt *wavFormatvar wavPCM []byte+ var firstSynthErr errorfor i, seg := range segments {seg = strings.TrimSpace(seg)if seg == "" {@@ -109,6 +110,9 @@ func (h *ChatHandler) ChatAudioSpeech(c *gin.Context) {}resp, err := driver.AudioSpeech(ctx, &modelName, &seg, apiConfig, &modelModule.TTSConfig{Format: "mp3"}, nil)if err != nil {+ if firstSynthErr == nil {+ firstSynthErr = err+ }common.Warn("chat TTS synthesis failed",zap.Int("segmentIndex", i),zap.Int("segmentLen", len(seg)),@@ -169,7 +173,13 @@ func (h *ChatHandler) ChatAudioSpeech(c *gin.Context) {c.Writer.Flush()}if !headerWritten {- common.ErrorWithCode(c, common.CodeServerError, "TTS synthesis produced no audio")+ if firstSynthErr != nil {+ common.ErrorWithCode(c, common.CodeServerError,+ fmt.Sprintf("TTS synthesis failed for model %s (%s)", modelName, driver.Name()))+ return+ }+ common.ErrorWithCode(c, common.CodeServerError,+ fmt.Sprintf("TTS synthesis produced no audio for model %s (%s)", modelName, driver.Name()))}}diff --git a/internal/handler/chat_audio_test.go b/internal/handler/chat_audio_test.gonew file mode 100644index 0000000000..c8efc8924a--- /dev/null+++ b/internal/handler/chat_audio_test.go@@ -0,0 +1,111 @@+package handler++import (+ "encoding/json"+ "fmt"+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"++ "github.com/glebarez/sqlite"+ "gorm.io/gorm"++ "ragflow/internal/common"+ "ragflow/internal/dao"+ "ragflow/internal/entity"+ "ragflow/internal/service"+ "ragflow/internal/utility"+)++// The chat TTS response may carry model/driver context but never the provider's+// raw error body; the full synthesis error stays in the server-side log.+func TestChatAudioSpeechFailureOmitsProviderError(t *testing.T) {+ providerBody := `{"code":20052,"message":"Voice or reference audio should be set","request_id":"secret-internal-diagnostic"}`+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ w.WriteHeader(http.StatusBadRequest)+ _, _ = w.Write([]byte(providerBody))+ }))+ defer server.Close()++ prevAllow := utility.AllowAnyHostForTest+ utility.AllowAnyHostForTest = true+ t.Cleanup(func() { utility.AllowAnyHostForTest = prevAllow })++ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{TranslateError: true})+ if err != nil {+ t.Fatalf("failed to open sqlite: %v", err)+ }+ if err = db.AutoMigrate(+ &entity.Tenant{},+ &entity.UserTenant{},+ &entity.TenantModel{},+ &entity.TenantModelProvider{},+ &entity.TenantModelInstance{},+ ); err != nil {+ t.Fatalf("failed to migrate test schema: %v", err)+ }+ origDB := dao.DB+ dao.DB = db+ t.Cleanup(func() { dao.DB = origDB })++ tenantName := "tts-test-tenant"+ ttsModelID := "tm-tts-1"+ if err = db.Create(&entity.Tenant{+ ID: "user-1",+ Name: &tenantName,+ TTSID: &ttsModelID,+ ParserIDs: "naive",+ }).Error; err != nil {+ t.Fatalf("failed to create tenant: %v", err)+ }+ if err = db.Create(&entity.TenantModel{+ ID: ttsModelID,+ ModelName: "FunAudioLLM/CosyVoice2-0.5B",+ ProviderID: "prov-1",+ InstanceID: "inst-1",+ ModelType: int(entity.ModelTypeTTS),+ Status: "active",+ Extra: "{}",+ }).Error; err != nil {+ t.Fatalf("failed to create tenant model: %v", err)+ }+ if err = db.Create(&entity.TenantModelProvider{+ ID: "prov-1",+ ProviderName: "SILICONFLOW",+ TenantID: "user-1",+ }).Error; err != nil {+ t.Fatalf("failed to create provider: %v", err)+ }+ if err = db.Create(&entity.TenantModelInstance{+ ID: "inst-1",+ InstanceName: "default",+ ProviderID: "prov-1",+ APIKey: "test-key",+ Status: "active",+ Extra: fmt.Sprintf(`{"base_url":%q}`, server.URL),+ }).Error; err != nil {+ t.Fatalf("failed to create instance: %v", err)+ }++ h := NewChatHandler(service.NewChatService(), service.NewUserService())+ h.SetMindMapDependencies(nil, nil, service.NewModelProviderService(), nil)++ c, w := setupGinContextWithUser(http.MethodPost, "/api/v1/chat/audio/speech", `{"text":"hello"}`)+ h.ChatAudioSpeech(c)++ body := w.Body.String()+ var resp map[string]any+ if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {+ t.Fatalf("failed to decode response: %v (body=%s)", err, body)+ }+ if resp["code"] != float64(common.CodeServerError) {+ t.Fatalf("code = %v, want %d (body=%s)", resp["code"], common.CodeServerError, body)+ }+ if !strings.Contains(body, "TTS synthesis failed for model FunAudioLLM/CosyVoice2-0.5B (SILICONFLOW)") {+ t.Fatalf("response missing model/driver context: %s", body)+ }+ if strings.Contains(body, "secret-internal-diagnostic") || strings.Contains(body, "SiliconFlow TTS API error") {+ t.Fatalf("response leaks provider error detail: %s", body)+ }+}