Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(aliyun): transcribe audio via DashScope multimodal-generation (#20975)

britzee12f193ed6b
diff --git a/conf/models/aliyun.json b/conf/models/aliyun.json
index 180312fa7a..2a56b17d6a 100644
--- a/conf/models/aliyun.json
+++ b/conf/models/aliyun.json
@@ -11,7 +11,8 @@
"embedding": "compatible-mode/v1/embeddings",
"rerank": "compatible-api/v1/reranks",
"models": "compatible-mode/v1/models",
- "tts": "api/v1/services/aigc/multimodal-generation/generation"
+ "tts": "api/v1/services/aigc/multimodal-generation/generation",
+ "asr": "api/v1/services/aigc/multimodal-generation/generation"
},
"models": [
{
diff --git a/internal/entity/models/aliyun.go b/internal/entity/models/aliyun.go
index 120662b9d9..84b5b17aa1 100644
--- a/internal/entity/models/aliyun.go
+++ b/internal/entity/models/aliyun.go
@@ -19,12 +19,14 @@ package models
import (
"bytes"
"context"
+ "encoding/base64"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
+ "path/filepath"
"strings"
"sync"
@@ -552,13 +554,214 @@ func (a *AliyunModel) Rerank(ctx context.Context, modelName *string, request Rer
return rerankResponse, nil
}
-// TranscribeAudio transcribe audio
+// aliyunASRRequest is the DashScope multimodal-generation request for Qwen ASR
+// models. It mirrors dashscope.MultiModalConversation: the audio is a data URL
+// on the shared multimodal-generation endpoint.
+type aliyunASRRequest struct {
+ Model string `json:"model"`
+ Input aliyunASRInput `json:"input"`
+ Parameters *aliyunASRParameters `json:"parameters,omitempty"`
+}
+
+type aliyunASRInput struct {
+ Messages []aliyunASRMessage `json:"messages"`
+}
+
+type aliyunASRMessage struct {
+ Role string `json:"role"`
+ Content []aliyunASRContentPart `json:"content"`
+}
+
+type aliyunASRContentPart struct {
+ Audio string `json:"audio,omitempty"`
+ Text string `json:"text,omitempty"`
+}
+
+type aliyunASRParameters struct {
+ ResultFormat string `json:"result_format,omitempty"`
+ ASROptions map[string]interface{} `json:"asr_options,omitempty"`
+}
+
+// aliyunASRResponse is the non-streaming DashScope multimodal-generation
+// response. The transcription lives in
+// output.choices[0].message.content[*].text.
+type aliyunASRResponse struct {
+ Output struct {
+ Choices []struct {
+ FinishReason string `json:"finish_reason"`
+ Message struct {
+ Content []struct {
+ Text string `json:"text"`
+ } `json:"content"`
+ } `json:"message"`
+ } `json:"choices"`
+ } `json:"output"`
+ Code string `json:"code"`
+ Message string `json:"message"`
+ RequestID string `json:"request_id"`
+}
+
+// TranscribeAudio transcribes an audio file with a DashScope ASR model
+// (qwen-asr-flash / qwen-audio-asr). It calls the same synchronized
+// multimodal-generation endpoint used for TTS, passing the audio as a base64
+// data URL, matching the legacy dashscope.MultiModalConversation path.
func (a *AliyunModel) TranscribeAudio(ctx context.Context, modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, modelUsage *common.ModelUsage) (*ASRResponse, error) {
- return nil, fmt.Errorf("%s, no such method", a.Name())
+ if err := a.baseModel.APIConfigCheck(apiConfig); err != nil {
+ return nil, err
+ }
+ if modelName == nil || strings.TrimSpace(*modelName) == "" {
+ return nil, fmt.Errorf("model name is required")
+ }
+ if file == nil || strings.TrimSpace(*file) == "" {
+ return nil, fmt.Errorf("audio file is required")
+ }
+ if strings.TrimSpace(a.baseModel.URLSuffix.ASR) == "" {
+ return nil, fmt.Errorf("aliyun ASR URL suffix is required")
+ }
+
+ // codeql[go/path-injection] False positive: *file is the audio file path the caller passes in to upload. The user (or operator-supplied pipeline) explicitly chose this path, and the OS access check enforces permissions anyway.
+ audio, err := os.ReadFile(*file)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read audio file: %w", err)
+ }
+ if len(audio) == 0 {
+ return nil, fmt.Errorf("audio file is empty")
+ }
+
+ contextText, options := aliyunASROptions(asrConfig)
+ reqBody := aliyunASRRequest{
+ Model: strings.TrimSpace(*modelName),
+ Input: aliyunASRInput{Messages: []aliyunASRMessage{
+ {Role: "system", Content: []aliyunASRContentPart{{Text: contextText}}},
+ {Role: "user", Content: []aliyunASRContentPart{{Audio: aliyunAudioDataURL(*file, audio)}}},
+ }},
+ Parameters: &aliyunASRParameters{ResultFormat: "message", ASROptions: options},
+ }
+
+ jsonData, err := json.Marshal(reqBody)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal request: %w", err)
+ }
+
+ url, err := a.baseModel.EndpointURL(apiConfig, a.baseModel.URLSuffix.ASR)
+ if err != nil {
+ return nil, err
+ }
+
+ ctx, cancel := context.WithTimeout(ctx, longOpCallTimeout)
+ defer cancel()
+
+ req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create request: %w", err)
+ }
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))
+ req.Header.Set("X-DashScope-SSE", "disable")
+
+ resp, err := a.baseModel.httpClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send request: %w", err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ body, err := readModelErrorBody(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("API request failed with status %d; failed to read error response: %w", resp.StatusCode, err)
+ }
+ return nil, &APIStatusError{Status: resp.StatusCode, Body: string(body)}
+ }
+
+ body, err := readModelResponseBody(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response: %w", err)
+ }
+
+ var parsed aliyunASRResponse
+ if err := json.Unmarshal(body, &parsed); err != nil {
+ return nil, fmt.Errorf("failed to parse aliyun ASR response: %w, body: %s", err, string(body))
+ }
+ if parsed.Code != "" {
+ return nil, fmt.Errorf("aliyun ASR API error: %s: %s", parsed.Code, parsed.Message)
+ }
+ if len(parsed.Output.Choices) == 0 {
+ return nil, fmt.Errorf("aliyun ASR response has no choices, body: %s", string(body))
+ }
+
+ var text strings.Builder
+ for _, part := range parsed.Output.Choices[0].Message.Content {
+ text.WriteString(part.Text)
+ }
+ return &ASRResponse{Text: text.String()}, nil
}
func (a *AliyunModel) TranscribeAudioWithSender(ctx context.Context, modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, modelUsage *common.ModelUsage, sender func(*string, *string) error) error {
- return fmt.Errorf("%s, no such method", a.Name())
+ if sender == nil {
+ return fmt.Errorf("sender is required")
+ }
+ // The synchronized DashScope ASR endpoint has no incremental-result
+ // stream; forward the finished transcription as a single chunk.
+ resp, err := a.TranscribeAudio(ctx, modelName, file, apiConfig, asrConfig, modelUsage)
+ if err != nil {
+ return err
+ }
+ chunk := resp.Text
+ return sender(&chunk, nil)
+}
+
+// aliyunASROptions splits the caller's ASRConfig into the recognition context
+// (system message text) and the DashScope asr_options. The defaults match the
+// legacy dashscope.MultiModalConversation path; only documented options are
+// forwarded so a stray UI parameter cannot fail the request.
+func aliyunASROptions(asrConfig *ASRConfig) (string, map[string]interface{}) {
+ options := map[string]interface{}{"enable_lid": true, "enable_itn": false}
+ if asrConfig == nil || asrConfig.Params == nil {
+ return "", options
+ }
+ contextText, _ := asrConfig.Params["context"].(string)
+ if raw, ok := asrConfig.Params["asr_options"].(map[string]interface{}); ok {
+ for key, value := range raw {
+ options[key] = value
+ }
+ }
+ for _, key := range []string{"language", "enable_lid", "enable_itn"} {
+ if value, ok := asrConfig.Params[key]; ok {
+ options[key] = value
+ }
+ }
+ return contextText, options
+}
+
+// aliyunAudioDataURL encodes audio bytes as the DashScope data URL
+// "data:<mime>;base64,<data>" accepted by the ASR endpoint.
+func aliyunAudioDataURL(filename string, audio []byte) string {
+ return "data:" + aliyunAudioMIME(filename) + ";base64," + base64.StdEncoding.EncodeToString(audio)
+}
+
+// aliyunAudioMIME maps an audio file extension to the MIME type DashScope
+// expects; WAV is the default because it is the most common RAGFlow output.
+func aliyunAudioMIME(filename string) string {
+ switch strings.ToLower(filepath.Ext(filename)) {
+ case ".mp3":
+ return "audio/mpeg"
+ case ".m4a":
+ return "audio/mp4"
+ case ".aac":
+ return "audio/aac"
+ case ".flac":
+ return "audio/flac"
+ case ".ogg":
+ return "audio/ogg"
+ case ".opus":
+ return "audio/opus"
+ case ".amr":
+ return "audio/amr"
+ case ".wma":
+ return "audio/x-ms-wma"
+ default:
+ return "audio/wav"
+ }
}
// aliyunTTSDefaultVoice is used when the caller does not specify a voice;
diff --git a/internal/entity/models/aliyun_test.go b/internal/entity/models/aliyun_test.go
index 41b3e308d3..b22c1644c2 100644
--- a/internal/entity/models/aliyun_test.go
+++ b/internal/entity/models/aliyun_test.go
@@ -17,11 +17,14 @@
package models
import (
+ "encoding/base64"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
+ "os"
+ "path/filepath"
"strings"
"testing"
)
@@ -935,3 +938,274 @@ func TestAliyunEmbedNativeRejectsTooNegativeIndex(t *testing.T) {
t.Fatalf("err = %v, want out-of-range error", err)
}
}
+
+const aliyunASRTestSuffix = "api/v1/services/aigc/multimodal-generation/generation"
+
+// newAliyunASRTestServer stubs the DashScope multimodal-generation endpoint for
+// ASR: POST returns one message choice whose content carries the transcription.
+func newAliyunASRTestServer(t *testing.T, transcription string, requestBody chan<- map[string]interface{}, requestPath chan<- string) *httptest.Server {
+ t.Helper()
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodPost {
+ http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+ return
+ }
+ var body map[string]interface{}
+ if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+ if requestPath != nil {
+ requestPath <- r.URL.Path
+ }
+ if requestBody != nil {
+ requestBody <- body
+ }
+ w.Header().Set("Content-Type", "application/json")
+ if _, err := fmt.Fprintf(w, `{"output":{"choices":[{"finish_reason":"stop","message":{"role":"assistant","content":[{"text":%q}]}}]},"request_id":"req-1"}`, transcription); err != nil {
+ t.Errorf("failed to write ASR response: %v", err)
+ }
+ }))
+ t.Cleanup(server.Close)
+ return server
+}
+
+func writeAliyunASRTestFile(t *testing.T, name string, content []byte) string {
+ t.Helper()
+ path := filepath.Join(t.TempDir(), name)
+ if err := os.WriteFile(path, content, 0o600); err != nil {
+ t.Fatalf("write audio file: %v", err)
+ }
+ return path
+}
+
+func aliyunASRMessages(t *testing.T, body map[string]interface{}) []map[string]interface{} {
+ t.Helper()
+ input, ok := body["input"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("input = %T, want JSON object", body["input"])
+ }
+ raw, ok := input["messages"].([]interface{})
+ if !ok {
+ t.Fatalf("input.messages = %T, want JSON array", input["messages"])
+ }
+ messages := make([]map[string]interface{}, 0, len(raw))
+ for _, item := range raw {
+ message, ok := item.(map[string]interface{})
+ if !ok {
+ t.Fatalf("message = %T, want JSON object", item)
+ }
+ messages = append(messages, message)
+ }
+ return messages
+}
+
+func aliyunASRContentString(t *testing.T, message map[string]interface{}, key string) string {
+ t.Helper()
+ content, ok := message["content"].([]interface{})
+ if !ok || len(content) == 0 {
+ t.Fatalf("content = %v, want non-empty JSON array", message["content"])
+ }
+ part, ok := content[0].(map[string]interface{})
+ if !ok {
+ t.Fatalf("content[0] = %T, want JSON object", content[0])
+ }
+ value, _ := part[key].(string)
+ return value
+}
+
+func TestAliyunTranscribeAudioSendsDataURLAndParsesText(t *testing.T) {
+ withSSRFBypass(t)
+ requestBody := make(chan map[string]interface{}, 1)
+ requestPath := make(chan string, 1)
+ server := newAliyunASRTestServer(t, "hello world", requestBody, requestPath)
+ ctx := t.Context()
+
+ model := NewAliyunModel(
+ map[string]string{"default": server.URL},
+ URLSuffix{ASR: aliyunASRTestSuffix},
+ )
+ apiKey := "test-key"
+ modelName := "qwen-asr-flash"
+ audio := []byte("RIFF....WAVE")
+ file := writeAliyunASRTestFile(t, "clip.wav", audio)
+
+ response, err := model.TranscribeAudio(ctx, &modelName, &file, &APIConfig{ApiKey: &apiKey}, nil, nil)
+ if err != nil {
+ t.Fatalf("TranscribeAudio: %v", err)
+ }
+ if response.Text != "hello world" {
+ t.Errorf("text = %q, want %q", response.Text, "hello world")
+ }
+
+ if got := <-requestPath; got != "/"+aliyunASRTestSuffix {
+ t.Errorf("request path = %q, want /%s", got, aliyunASRTestSuffix)
+ }
+ body := <-requestBody
+ if body["model"] != "qwen-asr-flash" {
+ t.Errorf("model = %v, want qwen-asr-flash", body["model"])
+ }
+ messages := aliyunASRMessages(t, body)
+ if len(messages) != 2 {
+ t.Fatalf("messages = %d, want 2 (system + user)", len(messages))
+ }
+ if messages[0]["role"] != "system" {
+ t.Errorf("messages[0].role = %v, want system", messages[0]["role"])
+ }
+ if messages[1]["role"] != "user" {
+ t.Errorf("messages[1].role = %v, want user", messages[1]["role"])
+ }
+ wantURL := "data:audio/wav;base64," + base64.StdEncoding.EncodeToString(audio)
+ if got := aliyunASRContentString(t, messages[1], "audio"); got != wantURL {
+ t.Errorf("audio = %q, want %q", got, wantURL)
+ }
+
+ params, ok := body["parameters"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("parameters = %T, want JSON object", body["parameters"])
+ }
+ if params["result_format"] != "message" {
+ t.Errorf("result_format = %v, want message", params["result_format"])
+ }
+ options, ok := params["asr_options"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("asr_options = %T, want JSON object", params["asr_options"])
+ }
+ if options["enable_lid"] != true || options["enable_itn"] != false {
+ t.Errorf("asr_options = %v, want defaults enable_lid=true enable_itn=false", options)
+ }
+}
+
+func TestAliyunTranscribeAudioHonorsContextAndASROptions(t *testing.T) {
+ withSSRFBypass(t)
+ requestBody := make(chan map[string]interface{}, 1)
+ server := newAliyunASRTestServer(t, "ok", requestBody, nil)
+ ctx := t.Context()
+
+ model := NewAliyunModel(
+ map[string]string{"default": server.URL},
+ URLSuffix{ASR: aliyunASRTestSuffix},
+ )
+ apiKey := "test-key"
+ modelName := "qwen-asr-flash"
+ file := writeAliyunASRTestFile(t, "clip.mp3", []byte("ID3...."))
+
+ _, err := model.TranscribeAudio(ctx, &modelName, &file, &APIConfig{ApiKey: &apiKey}, &ASRConfig{Params: map[string]interface{}{
+ "context": "NAVI vs Faze",
+ "language": "en",
+ "enable_itn": true,
+ }}, nil)
+ if err != nil {
+ t.Fatalf("TranscribeAudio: %v", err)
+ }
+ body := <-requestBody
+ messages := aliyunASRMessages(t, body)
+ if got := aliyunASRContentString(t, messages[0], "text"); got != "NAVI vs Faze" {
+ t.Errorf("system text = %q, want NAVI vs Faze", got)
+ }
+ if got := aliyunASRContentString(t, messages[1], "audio"); !strings.HasPrefix(got, "data:audio/mpeg;base64,") {
+ t.Errorf("audio data URL = %q, want mp3 MIME prefix", got)
+ }
+ params, ok := body["parameters"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("parameters = %T, want JSON object", body["parameters"])
+ }
+ options, ok := params["asr_options"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("asr_options = %T, want JSON object", params["asr_options"])
+ }
+ if options["language"] != "en" {
+ t.Errorf("language = %v, want en", options["language"])
+ }
+ if options["enable_itn"] != true {
+ t.Errorf("enable_itn = %v, want true (explicit override)", options["enable_itn"])
+ }
+ if options["enable_lid"] != true {
+ t.Errorf("enable_lid = %v, want default true", options["enable_lid"])
+ }
+}
+
+func TestAliyunTranscribeAudioSurfacesAPIError(t *testing.T) {
+ withSSRFBypass(t)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(http.StatusUnauthorized)
+ _, _ = w.Write([]byte(`{"code":"InvalidApiKey","message":"Invalid API-key provided."}`))
+ }))
+ defer server.Close()
+
+ model := NewAliyunModel(
+ map[string]string{"default": server.URL},
+ URLSuffix{ASR: aliyunASRTestSuffix},
+ )
+ apiKey := "bad-key"
+ modelName := "qwen-asr-flash"
+ file := writeAliyunASRTestFile(t, "clip.wav", []byte("data"))
+
+ _, err := model.TranscribeAudio(t.Context(), &modelName, &file, &APIConfig{ApiKey: &apiKey}, nil, nil)
+ var statusErr *APIStatusError
+ if !errors.As(err, &statusErr) || statusErr.Status != http.StatusUnauthorized {
+ t.Fatalf("error = %v, want APIStatusError with status 401", err)
+ }
+ if !strings.Contains(statusErr.Body, "InvalidApiKey") {
+ t.Fatalf("error body = %q, want to include InvalidApiKey", statusErr.Body)
+ }
+}
+
+func TestAliyunTranscribeAudioRejectsEmptyFile(t *testing.T) {
+ withSSRFBypass(t)
+ model := NewAliyunModel(
+ map[string]string{"default": "https://dashscope.example"},
+ URLSuffix{ASR: aliyunASRTestSuffix},
+ )
+ apiKey := "test-key"
+ modelName := "qwen-asr-flash"
+ file := writeAliyunASRTestFile(t, "empty.wav", nil)
+
+ _, err := model.TranscribeAudio(t.Context(), &modelName, &file, &APIConfig{ApiKey: &apiKey}, nil, nil)
+ if err == nil || err.Error() != "audio file is empty" {
+ t.Fatalf("error = %v, want empty audio error", err)
+ }
+}
+
+func TestAliyunTranscribeAudioRequiresASRSuffix(t *testing.T) {
+ withSSRFBypass(t)
+ model := NewAliyunModel(
+ map[string]string{"default": "https://dashscope.example"},
+ URLSuffix{Chat: "compatible-mode/v1/chat/completions"},
+ )
+ apiKey := "test-key"
+ modelName := "qwen-asr-flash"
+ file := writeAliyunASRTestFile(t, "clip.wav", []byte("data"))
+
+ _, err := model.TranscribeAudio(t.Context(), &modelName, &file, &APIConfig{ApiKey: &apiKey}, nil, nil)
+ if err == nil || err.Error() != "aliyun ASR URL suffix is required" {
+ t.Fatalf("error = %v, want missing ASR suffix error", err)
+ }
+}
+
+func TestAliyunTranscribeAudioWithSenderSendsSingleChunk(t *testing.T) {
+ withSSRFBypass(t)
+ server := newAliyunASRTestServer(t, "transcribed", nil, nil)
+ model := NewAliyunModel(
+ map[string]string{"default": server.URL},
+ URLSuffix{ASR: aliyunASRTestSuffix},
+ )
+ apiKey := "test-key"
+ modelName := "qwen-asr-flash"
+ file := writeAliyunASRTestFile(t, "clip.wav", []byte("data"))
+
+ var chunks []string
+ err := model.TranscribeAudioWithSender(t.Context(), &modelName, &file, &APIConfig{ApiKey: &apiKey}, nil, nil, func(content, _ *string) error {
+ if content != nil {
+ chunks = append(chunks, *content)
+ }
+ return nil
+ })
+ if err != nil {
+ t.Fatalf("TranscribeAudioWithSender: %v", err)
+ }
+ if len(chunks) != 1 || chunks[0] != "transcribed" {
+ t.Fatalf("chunks = %v, want [transcribed]", chunks)
+ }
+}