Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(tts): send required voice payload for SiliconFlow TTS and surface synthesis failures (#19812)

euvred1b2d80ad7d6
diff --git a/internal/entity/models/siliconflow.go b/internal/entity/models/siliconflow.go
index 96ff7e31f1..8d51a1b8c8 100644
--- a/internal/entity/models/siliconflow.go
+++ b/internal/entity/models/siliconflow.go
@@ -626,6 +626,33 @@ func (s *SiliconflowModel) TranscribeAudioWithSender(ctx context.Context, modelN
return fmt.Errorf("%s, no such method", s.Name())
}
+// siliconflowTTSRequestBody mirrors rag/llm/tts_model.py SILICONFLOWTTS._build_payload:
+// /v1/audio/speech requires voice as "{model}:{voice_name}" and only synthesizes
+// mp3 at 32000 or 44100 Hz, so the defaults pin the payload to a shape the
+// provider accepts. ttsConfig.Params overrides any field; Format overrides
+// response_format last.
+func siliconflowTTSRequestBody(modelName, audioContent string, ttsConfig *TTSConfig, stream bool) map[string]interface{} {
+ reqBody := map[string]interface{}{
+ "model": modelName,
+ "input": audioContent,
+ "voice": fmt.Sprintf("%s:anna", modelName),
+ "response_format": "mp3",
+ "sample_rate": 32000,
+ "speed": 1,
+ "gain": 0,
+ "stream": stream,
+ }
+ if ttsConfig != nil {
+ for key, value := range ttsConfig.Params {
+ reqBody[key] = value
+ }
+ if ttsConfig.Format != "" {
+ reqBody["response_format"] = ttsConfig.Format
+ }
+ }
+ return reqBody
+}
+
// AudioSpeech convert text to audio
func (s *SiliconflowModel) AudioSpeech(ctx context.Context, modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, modelUsage *common.ModelUsage) (*TTSResponse, error) {
if err := s.baseModel.APIConfigCheck(apiConfig); err != nil {
@@ -642,22 +669,7 @@ func (s *SiliconflowModel) AudioSpeech(ctx context.Context, modelName *string, a
}
url := fmt.Sprintf("%s/%s", resolvedBaseURL, s.baseModel.URLSuffix.TTS)
- reqBody := map[string]interface{}{
- "model": *modelName,
- "input": *audioContent,
- "stream": false,
- }
-
- if ttsConfig != nil && ttsConfig.Params != nil {
- for key, value := range ttsConfig.Params {
- reqBody[key] = value
- }
- }
- if ttsConfig != nil && ttsConfig.Format != "" {
- reqBody["response_format"] = ttsConfig.Format
- }
-
- jsonData, err := json.Marshal(reqBody)
+ jsonData, err := json.Marshal(siliconflowTTSRequestBody(*modelName, *audioContent, ttsConfig, false))
if err != nil {
return nil, fmt.Errorf("failed to marshal request: %w", err)
}
@@ -685,7 +697,7 @@ func (s *SiliconflowModel) AudioSpeech(ctx context.Context, modelName *string, a
}
if resp.StatusCode != http.StatusOK {
- return nil, fmt.Errorf("%s - %s", resp.Status, string(body))
+ return nil, fmt.Errorf("SiliconFlow TTS API error: %s - %s", resp.Status, string(body))
}
return &TTSResponse{Audio: body}, nil
@@ -706,22 +718,7 @@ func (s *SiliconflowModel) AudioSpeechWithSender(ctx context.Context, modelName
}
url := fmt.Sprintf("%s/%s", resolvedBaseURL, s.baseModel.URLSuffix.TTS)
- reqBody := map[string]interface{}{
- "model": *modelName,
- "input": *audioContent,
- "stream": true,
- }
-
- if ttsConfig != nil && ttsConfig.Params != nil {
- for key, value := range ttsConfig.Params {
- reqBody[key] = value
- }
- }
- if ttsConfig != nil && ttsConfig.Format != "" {
- reqBody["response_format"] = ttsConfig.Format
- }
-
- jsonData, err := json.Marshal(reqBody)
+ jsonData, err := json.Marshal(siliconflowTTSRequestBody(*modelName, *audioContent, ttsConfig, true))
if err != nil {
return fmt.Errorf("failed to marshal request: %w", err)
}
diff --git a/internal/entity/models/siliconflow_test.go b/internal/entity/models/siliconflow_test.go
index f87cdb12f4..c723b52c61 100644
--- a/internal/entity/models/siliconflow_test.go
+++ b/internal/entity/models/siliconflow_test.go
@@ -21,6 +21,7 @@ import (
"io"
"net/http"
"net/http/httptest"
+ "strings"
"testing"
)
@@ -111,6 +112,147 @@ func TestSiliconflowChatWithMessagesExtractsResponseAndUsage(t *testing.T) {
}
}
+func TestSiliconflowAudioSpeechSendsRequiredVoice(t *testing.T) {
+ withSSRFBypass(t)
+ var gotPayload map[string]any
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if err := json.NewDecoder(r.Body).Decode(&gotPayload); err != nil {
+ t.Errorf("decode request: %v", err)
+ return
+ }
+ voice, _ := gotPayload["voice"].(string)
+ if voice != "FunAudioLLM/CosyVoice2-0.5B:anna" {
+ w.WriteHeader(http.StatusBadRequest)
+ _, _ = w.Write([]byte(`{"code":20052,"message":"Voice or reference audio should be set","data":null}`))
+ return
+ }
+ w.Header().Set("Content-Type", "audio/mpeg")
+ _, _ = w.Write([]byte("mp3-bytes"))
+ }))
+ defer server.Close()
+
+ apiKey := "test-key"
+ modelName := "FunAudioLLM/CosyVoice2-0.5B"
+ text := "hello"
+ resp, err := NewSiliconflowModel(
+ map[string]string{"default": server.URL},
+ URLSuffix{TTS: "audio/speech"},
+ ).AudioSpeech(t.Context(), &modelName, &text, &APIConfig{ApiKey: &apiKey}, nil, nil)
+ if err != nil {
+ t.Fatalf("AudioSpeech: %v", err)
+ }
+ if string(resp.Audio) != "mp3-bytes" {
+ t.Fatalf("Audio=%q, want mp3-bytes", resp.Audio)
+ }
+ want := map[string]any{
+ "model": "FunAudioLLM/CosyVoice2-0.5B",
+ "input": "hello",
+ "voice": "FunAudioLLM/CosyVoice2-0.5B:anna",
+ "response_format": "mp3",
+ "sample_rate": float64(32000),
+ "speed": float64(1),
+ "gain": float64(0),
+ "stream": false,
+ }
+ for key, value := range want {
+ if gotPayload[key] != value {
+ t.Errorf("payload[%q]=%#v, want %#v", key, gotPayload[key], value)
+ }
+ }
+}
+
+func TestSiliconflowAudioSpeechParamsOverrideVoice(t *testing.T) {
+ withSSRFBypass(t)
+ var gotPayload map[string]any
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _ = json.NewDecoder(r.Body).Decode(&gotPayload)
+ w.Header().Set("Content-Type", "audio/mpeg")
+ _, _ = w.Write([]byte("mp3-bytes"))
+ }))
+ defer server.Close()
+
+ apiKey := "test-key"
+ modelName := "FunAudioLLM/CosyVoice2-0.5B"
+ text := "hello"
+ resp, err := NewSiliconflowModel(
+ map[string]string{"default": server.URL},
+ URLSuffix{TTS: "audio/speech"},
+ ).AudioSpeech(t.Context(), &modelName, &text, &APIConfig{ApiKey: &apiKey},
+ &TTSConfig{Params: map[string]interface{}{"voice": "FunAudioLLM/CosyVoice2-0.5B:benjamin"}}, nil)
+ if err != nil {
+ t.Fatalf("AudioSpeech: %v", err)
+ }
+ if string(resp.Audio) != "mp3-bytes" {
+ t.Fatalf("Audio=%q, want mp3-bytes", resp.Audio)
+ }
+ if gotPayload["voice"] != "FunAudioLLM/CosyVoice2-0.5B:benjamin" {
+ t.Fatalf("voice=%#v, want the Params override", gotPayload["voice"])
+ }
+}
+
+func TestSiliconflowAudioSpeechErrorIncludesProviderAndBody(t *testing.T) {
+ withSSRFBypass(t)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusBadRequest)
+ _, _ = w.Write([]byte(`{"code":20052,"message":"Voice or reference audio should be set","data":null}`))
+ }))
+ defer server.Close()
+
+ apiKey := "test-key"
+ modelName := "FunAudioLLM/CosyVoice2-0.5B"
+ text := "hello"
+ _, err := NewSiliconflowModel(
+ map[string]string{"default": server.URL},
+ URLSuffix{TTS: "audio/speech"},
+ ).AudioSpeech(t.Context(), &modelName, &text, &APIConfig{ApiKey: &apiKey}, nil, nil)
+ if err == nil {
+ t.Fatal("expected error for provider rejection")
+ }
+ for _, want := range []string{"SiliconFlow TTS API error", "400 Bad Request", "Voice or reference audio should be set"} {
+ if !strings.Contains(err.Error(), want) {
+ t.Errorf("err = %v, want it to contain %q", err, want)
+ }
+ }
+}
+
+func TestSiliconflowAudioSpeechWithSenderSendsRequiredVoice(t *testing.T) {
+ withSSRFBypass(t)
+ var gotPayload map[string]any
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _ = json.NewDecoder(r.Body).Decode(&gotPayload)
+ w.Header().Set("Content-Type", "audio/mpeg")
+ _, _ = w.Write([]byte("mp3-bytes"))
+ }))
+ defer server.Close()
+
+ apiKey := "test-key"
+ modelName := "FunAudioLLM/CosyVoice2-0.5B"
+ text := "hello"
+ var chunks []string
+ err := NewSiliconflowModel(
+ map[string]string{"default": server.URL},
+ URLSuffix{TTS: "audio/speech"},
+ ).AudioSpeechWithSender(t.Context(), &modelName, &text, &APIConfig{ApiKey: &apiKey}, nil, nil,
+ func(content *string, _ *string) error {
+ if content != nil {
+ chunks = append(chunks, *content)
+ }
+ return nil
+ })
+ if err != nil {
+ t.Fatalf("AudioSpeechWithSender: %v", err)
+ }
+ if gotPayload["voice"] != "FunAudioLLM/CosyVoice2-0.5B:anna" {
+ t.Fatalf("voice=%#v, want model-qualified default voice", gotPayload["voice"])
+ }
+ if gotPayload["stream"] != true {
+ t.Fatalf("stream=%#v, want true", gotPayload["stream"])
+ }
+ if len(chunks) == 0 {
+ t.Fatal("expected streamed audio chunks")
+ }
+}
+
func TestSiliconflowChatStreamlyWithSenderCollectsUsage(t *testing.T) {
withSSRFBypass(t)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
diff --git a/internal/handler/chat_audio.go b/internal/handler/chat_audio.go
index 9b39d25e16..e08f3061e9 100644
--- a/internal/handler/chat_audio.go
+++ b/internal/handler/chat_audio.go
@@ -102,6 +102,7 @@ func (h *ChatHandler) ChatAudioSpeech(c *gin.Context) {
// a single WAV before being written.
var wavFmt *wavFormat
var wavPCM []byte
+ var firstSynthErr error
for i, seg := range segments {
seg = strings.TrimSpace(seg)
if seg == "" {
@@ -109,6 +110,9 @@ func (h *ChatHandler) ChatAudioSpeech(c *gin.Context) {
}
resp, err := driver.AudioSpeech(ctx, &modelName, &seg, apiConfig, &modelModule.TTSConfig{Format: "mp3"}, nil)
if err != nil {
+ if firstSynthErr == nil {
+ firstSynthErr = err
+ }
common.Warn("chat TTS synthesis failed",
zap.Int("segmentIndex", i),
zap.Int("segmentLen", len(seg)),
@@ -169,7 +173,13 @@ func (h *ChatHandler) ChatAudioSpeech(c *gin.Context) {
c.Writer.Flush()
}
if !headerWritten {
- common.ErrorWithCode(c, common.CodeServerError, "TTS synthesis produced no audio")
+ if firstSynthErr != nil {
+ common.ErrorWithCode(c, common.CodeServerError,
+ fmt.Sprintf("TTS synthesis failed for model %s (%s)", modelName, driver.Name()))
+ return
+ }
+ common.ErrorWithCode(c, common.CodeServerError,
+ fmt.Sprintf("TTS synthesis produced no audio for model %s (%s)", modelName, driver.Name()))
}
}
diff --git a/internal/handler/chat_audio_test.go b/internal/handler/chat_audio_test.go
new file mode 100644
index 0000000000..c8efc8924a
--- /dev/null
+++ b/internal/handler/chat_audio_test.go
@@ -0,0 +1,111 @@
+package handler
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+
+ "github.com/glebarez/sqlite"
+ "gorm.io/gorm"
+
+ "ragflow/internal/common"
+ "ragflow/internal/dao"
+ "ragflow/internal/entity"
+ "ragflow/internal/service"
+ "ragflow/internal/utility"
+)
+
+// The chat TTS response may carry model/driver context but never the provider's
+// raw error body; the full synthesis error stays in the server-side log.
+func TestChatAudioSpeechFailureOmitsProviderError(t *testing.T) {
+ providerBody := `{"code":20052,"message":"Voice or reference audio should be set","request_id":"secret-internal-diagnostic"}`
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusBadRequest)
+ _, _ = w.Write([]byte(providerBody))
+ }))
+ defer server.Close()
+
+ prevAllow := utility.AllowAnyHostForTest
+ utility.AllowAnyHostForTest = true
+ t.Cleanup(func() { utility.AllowAnyHostForTest = prevAllow })
+
+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{TranslateError: true})
+ if err != nil {
+ t.Fatalf("failed to open sqlite: %v", err)
+ }
+ if err = db.AutoMigrate(
+ &entity.Tenant{},
+ &entity.UserTenant{},
+ &entity.TenantModel{},
+ &entity.TenantModelProvider{},
+ &entity.TenantModelInstance{},
+ ); err != nil {
+ t.Fatalf("failed to migrate test schema: %v", err)
+ }
+ origDB := dao.DB
+ dao.DB = db
+ t.Cleanup(func() { dao.DB = origDB })
+
+ tenantName := "tts-test-tenant"
+ ttsModelID := "tm-tts-1"
+ if err = db.Create(&entity.Tenant{
+ ID: "user-1",
+ Name: &tenantName,
+ TTSID: &ttsModelID,
+ ParserIDs: "naive",
+ }).Error; err != nil {
+ t.Fatalf("failed to create tenant: %v", err)
+ }
+ if err = db.Create(&entity.TenantModel{
+ ID: ttsModelID,
+ ModelName: "FunAudioLLM/CosyVoice2-0.5B",
+ ProviderID: "prov-1",
+ InstanceID: "inst-1",
+ ModelType: int(entity.ModelTypeTTS),
+ Status: "active",
+ Extra: "{}",
+ }).Error; err != nil {
+ t.Fatalf("failed to create tenant model: %v", err)
+ }
+ if err = db.Create(&entity.TenantModelProvider{
+ ID: "prov-1",
+ ProviderName: "SILICONFLOW",
+ TenantID: "user-1",
+ }).Error; err != nil {
+ t.Fatalf("failed to create provider: %v", err)
+ }
+ if err = db.Create(&entity.TenantModelInstance{
+ ID: "inst-1",
+ InstanceName: "default",
+ ProviderID: "prov-1",
+ APIKey: "test-key",
+ Status: "active",
+ Extra: fmt.Sprintf(`{"base_url":%q}`, server.URL),
+ }).Error; err != nil {
+ t.Fatalf("failed to create instance: %v", err)
+ }
+
+ h := NewChatHandler(service.NewChatService(), service.NewUserService())
+ h.SetMindMapDependencies(nil, nil, service.NewModelProviderService(), nil)
+
+ c, w := setupGinContextWithUser(http.MethodPost, "/api/v1/chat/audio/speech", `{"text":"hello"}`)
+ h.ChatAudioSpeech(c)
+
+ body := w.Body.String()
+ var resp map[string]any
+ if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+ t.Fatalf("failed to decode response: %v (body=%s)", err, body)
+ }
+ if resp["code"] != float64(common.CodeServerError) {
+ t.Fatalf("code = %v, want %d (body=%s)", resp["code"], common.CodeServerError, body)
+ }
+ if !strings.Contains(body, "TTS synthesis failed for model FunAudioLLM/CosyVoice2-0.5B (SILICONFLOW)") {
+ t.Fatalf("response missing model/driver context: %s", body)
+ }
+ if strings.Contains(body, "secret-internal-diagnostic") || strings.Contains(body, "SiliconFlow TTS API error") {
+ t.Fatalf("response leaks provider error detail: %s", body)
+ }
+}