RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(aliyun): transcribe audio via DashScope multimodal-generation (#20975)
britzee12f193ed6b
diff --git a/conf/models/aliyun.json b/conf/models/aliyun.jsonindex 180312fa7a..2a56b17d6a 100644--- a/conf/models/aliyun.json+++ b/conf/models/aliyun.json@@ -11,7 +11,8 @@"embedding": "compatible-mode/v1/embeddings","rerank": "compatible-api/v1/reranks","models": "compatible-mode/v1/models",- "tts": "api/v1/services/aigc/multimodal-generation/generation"+ "tts": "api/v1/services/aigc/multimodal-generation/generation",+ "asr": "api/v1/services/aigc/multimodal-generation/generation"},"models": [{diff --git a/internal/entity/models/aliyun.go b/internal/entity/models/aliyun.goindex 120662b9d9..84b5b17aa1 100644--- a/internal/entity/models/aliyun.go+++ b/internal/entity/models/aliyun.go@@ -19,12 +19,14 @@ package modelsimport ("bytes""context"+ "encoding/base64""encoding/json""fmt""io""net/http""net/url""os"+ "path/filepath""strings""sync"@@ -552,13 +554,214 @@ func (a *AliyunModel) Rerank(ctx context.Context, modelName *string, request Rerreturn rerankResponse, nil}-// TranscribeAudio transcribe audio+// aliyunASRRequest is the DashScope multimodal-generation request for Qwen ASR+// models. It mirrors dashscope.MultiModalConversation: the audio is a data URL+// on the shared multimodal-generation endpoint.+type aliyunASRRequest struct {+ Model string `json:"model"`+ Input aliyunASRInput `json:"input"`+ Parameters *aliyunASRParameters `json:"parameters,omitempty"`+}++type aliyunASRInput struct {+ Messages []aliyunASRMessage `json:"messages"`+}++type aliyunASRMessage struct {+ Role string `json:"role"`+ Content []aliyunASRContentPart `json:"content"`+}++type aliyunASRContentPart struct {+ Audio string `json:"audio,omitempty"`+ Text string `json:"text,omitempty"`+}++type aliyunASRParameters struct {+ ResultFormat string `json:"result_format,omitempty"`+ ASROptions map[string]interface{} `json:"asr_options,omitempty"`+}++// aliyunASRResponse is the non-streaming DashScope multimodal-generation+// response. The transcription lives in+// output.choices[0].message.content[*].text.+type aliyunASRResponse struct {+ Output struct {+ Choices []struct {+ FinishReason string `json:"finish_reason"`+ Message struct {+ Content []struct {+ Text string `json:"text"`+ } `json:"content"`+ } `json:"message"`+ } `json:"choices"`+ } `json:"output"`+ Code string `json:"code"`+ Message string `json:"message"`+ RequestID string `json:"request_id"`+}++// TranscribeAudio transcribes an audio file with a DashScope ASR model+// (qwen-asr-flash / qwen-audio-asr). It calls the same synchronized+// multimodal-generation endpoint used for TTS, passing the audio as a base64+// data URL, matching the legacy dashscope.MultiModalConversation path.func (a *AliyunModel) TranscribeAudio(ctx context.Context, modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, modelUsage *common.ModelUsage) (*ASRResponse, error) {- return nil, fmt.Errorf("%s, no such method", a.Name())+ if err := a.baseModel.APIConfigCheck(apiConfig); err != nil {+ return nil, err+ }+ if modelName == nil || strings.TrimSpace(*modelName) == "" {+ return nil, fmt.Errorf("model name is required")+ }+ if file == nil || strings.TrimSpace(*file) == "" {+ return nil, fmt.Errorf("audio file is required")+ }+ if strings.TrimSpace(a.baseModel.URLSuffix.ASR) == "" {+ return nil, fmt.Errorf("aliyun ASR URL suffix is required")+ }++ // codeql[go/path-injection] False positive: *file is the audio file path the caller passes in to upload. The user (or operator-supplied pipeline) explicitly chose this path, and the OS access check enforces permissions anyway.+ audio, err := os.ReadFile(*file)+ if err != nil {+ return nil, fmt.Errorf("failed to read audio file: %w", err)+ }+ if len(audio) == 0 {+ return nil, fmt.Errorf("audio file is empty")+ }++ contextText, options := aliyunASROptions(asrConfig)+ reqBody := aliyunASRRequest{+ Model: strings.TrimSpace(*modelName),+ Input: aliyunASRInput{Messages: []aliyunASRMessage{+ {Role: "system", Content: []aliyunASRContentPart{{Text: contextText}}},+ {Role: "user", Content: []aliyunASRContentPart{{Audio: aliyunAudioDataURL(*file, audio)}}},+ }},+ Parameters: &aliyunASRParameters{ResultFormat: "message", ASROptions: options},+ }++ jsonData, err := json.Marshal(reqBody)+ if err != nil {+ return nil, fmt.Errorf("failed to marshal request: %w", err)+ }++ url, err := a.baseModel.EndpointURL(apiConfig, a.baseModel.URLSuffix.ASR)+ if err != nil {+ return nil, err+ }++ ctx, cancel := context.WithTimeout(ctx, longOpCallTimeout)+ defer cancel()++ req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData))+ if err != nil {+ return nil, fmt.Errorf("failed to create request: %w", err)+ }+ req.Header.Set("Content-Type", "application/json")+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", *apiConfig.ApiKey))+ req.Header.Set("X-DashScope-SSE", "disable")++ resp, err := a.baseModel.httpClient.Do(req)+ if err != nil {+ return nil, fmt.Errorf("failed to send request: %w", err)+ }+ defer resp.Body.Close()++ if resp.StatusCode != http.StatusOK {+ body, err := readModelErrorBody(resp.Body)+ if err != nil {+ return nil, fmt.Errorf("API request failed with status %d; failed to read error response: %w", resp.StatusCode, err)+ }+ return nil, &APIStatusError{Status: resp.StatusCode, Body: string(body)}+ }++ body, err := readModelResponseBody(resp.Body)+ if err != nil {+ return nil, fmt.Errorf("failed to read response: %w", err)+ }++ var parsed aliyunASRResponse+ if err := json.Unmarshal(body, &parsed); err != nil {+ return nil, fmt.Errorf("failed to parse aliyun ASR response: %w, body: %s", err, string(body))+ }+ if parsed.Code != "" {+ return nil, fmt.Errorf("aliyun ASR API error: %s: %s", parsed.Code, parsed.Message)+ }+ if len(parsed.Output.Choices) == 0 {+ return nil, fmt.Errorf("aliyun ASR response has no choices, body: %s", string(body))+ }++ var text strings.Builder+ for _, part := range parsed.Output.Choices[0].Message.Content {+ text.WriteString(part.Text)+ }+ return &ASRResponse{Text: text.String()}, nil}func (a *AliyunModel) TranscribeAudioWithSender(ctx context.Context, modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, modelUsage *common.ModelUsage, sender func(*string, *string) error) error {- return fmt.Errorf("%s, no such method", a.Name())+ if sender == nil {+ return fmt.Errorf("sender is required")+ }+ // The synchronized DashScope ASR endpoint has no incremental-result+ // stream; forward the finished transcription as a single chunk.+ resp, err := a.TranscribeAudio(ctx, modelName, file, apiConfig, asrConfig, modelUsage)+ if err != nil {+ return err+ }+ chunk := resp.Text+ return sender(&chunk, nil)+}++// aliyunASROptions splits the caller's ASRConfig into the recognition context+// (system message text) and the DashScope asr_options. The defaults match the+// legacy dashscope.MultiModalConversation path; only documented options are+// forwarded so a stray UI parameter cannot fail the request.+func aliyunASROptions(asrConfig *ASRConfig) (string, map[string]interface{}) {+ options := map[string]interface{}{"enable_lid": true, "enable_itn": false}+ if asrConfig == nil || asrConfig.Params == nil {+ return "", options+ }+ contextText, _ := asrConfig.Params["context"].(string)+ if raw, ok := asrConfig.Params["asr_options"].(map[string]interface{}); ok {+ for key, value := range raw {+ options[key] = value+ }+ }+ for _, key := range []string{"language", "enable_lid", "enable_itn"} {+ if value, ok := asrConfig.Params[key]; ok {+ options[key] = value+ }+ }+ return contextText, options+}++// aliyunAudioDataURL encodes audio bytes as the DashScope data URL+// "data:<mime>;base64,<data>" accepted by the ASR endpoint.+func aliyunAudioDataURL(filename string, audio []byte) string {+ return "data:" + aliyunAudioMIME(filename) + ";base64," + base64.StdEncoding.EncodeToString(audio)+}++// aliyunAudioMIME maps an audio file extension to the MIME type DashScope+// expects; WAV is the default because it is the most common RAGFlow output.+func aliyunAudioMIME(filename string) string {+ switch strings.ToLower(filepath.Ext(filename)) {+ case ".mp3":+ return "audio/mpeg"+ case ".m4a":+ return "audio/mp4"+ case ".aac":+ return "audio/aac"+ case ".flac":+ return "audio/flac"+ case ".ogg":+ return "audio/ogg"+ case ".opus":+ return "audio/opus"+ case ".amr":+ return "audio/amr"+ case ".wma":+ return "audio/x-ms-wma"+ default:+ return "audio/wav"+ }}// aliyunTTSDefaultVoice is used when the caller does not specify a voice;diff --git a/internal/entity/models/aliyun_test.go b/internal/entity/models/aliyun_test.goindex 41b3e308d3..b22c1644c2 100644--- a/internal/entity/models/aliyun_test.go+++ b/internal/entity/models/aliyun_test.go@@ -17,11 +17,14 @@package modelsimport (+ "encoding/base64""encoding/json""errors""fmt""net/http""net/http/httptest"+ "os"+ "path/filepath""strings""testing")@@ -935,3 +938,274 @@ func TestAliyunEmbedNativeRejectsTooNegativeIndex(t *testing.T) {t.Fatalf("err = %v, want out-of-range error", err)}}++const aliyunASRTestSuffix = "api/v1/services/aigc/multimodal-generation/generation"++// newAliyunASRTestServer stubs the DashScope multimodal-generation endpoint for+// ASR: POST returns one message choice whose content carries the transcription.+func newAliyunASRTestServer(t *testing.T, transcription string, requestBody chan<- map[string]interface{}, requestPath chan<- string) *httptest.Server {+ t.Helper()+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.Method != http.MethodPost {+ http.Error(w, "method not allowed", http.StatusMethodNotAllowed)+ return+ }+ var body map[string]interface{}+ if err := json.NewDecoder(r.Body).Decode(&body); err != nil {+ http.Error(w, err.Error(), http.StatusBadRequest)+ return+ }+ if requestPath != nil {+ requestPath <- r.URL.Path+ }+ if requestBody != nil {+ requestBody <- body+ }+ w.Header().Set("Content-Type", "application/json")+ if _, err := fmt.Fprintf(w, `{"output":{"choices":[{"finish_reason":"stop","message":{"role":"assistant","content":[{"text":%q}]}}]},"request_id":"req-1"}`, transcription); err != nil {+ t.Errorf("failed to write ASR response: %v", err)+ }+ }))+ t.Cleanup(server.Close)+ return server+}++func writeAliyunASRTestFile(t *testing.T, name string, content []byte) string {+ t.Helper()+ path := filepath.Join(t.TempDir(), name)+ if err := os.WriteFile(path, content, 0o600); err != nil {+ t.Fatalf("write audio file: %v", err)+ }+ return path+}++func aliyunASRMessages(t *testing.T, body map[string]interface{}) []map[string]interface{} {+ t.Helper()+ input, ok := body["input"].(map[string]interface{})+ if !ok {+ t.Fatalf("input = %T, want JSON object", body["input"])+ }+ raw, ok := input["messages"].([]interface{})+ if !ok {+ t.Fatalf("input.messages = %T, want JSON array", input["messages"])+ }+ messages := make([]map[string]interface{}, 0, len(raw))+ for _, item := range raw {+ message, ok := item.(map[string]interface{})+ if !ok {+ t.Fatalf("message = %T, want JSON object", item)+ }+ messages = append(messages, message)+ }+ return messages+}++func aliyunASRContentString(t *testing.T, message map[string]interface{}, key string) string {+ t.Helper()+ content, ok := message["content"].([]interface{})+ if !ok || len(content) == 0 {+ t.Fatalf("content = %v, want non-empty JSON array", message["content"])+ }+ part, ok := content[0].(map[string]interface{})+ if !ok {+ t.Fatalf("content[0] = %T, want JSON object", content[0])+ }+ value, _ := part[key].(string)+ return value+}++func TestAliyunTranscribeAudioSendsDataURLAndParsesText(t *testing.T) {+ withSSRFBypass(t)+ requestBody := make(chan map[string]interface{}, 1)+ requestPath := make(chan string, 1)+ server := newAliyunASRTestServer(t, "hello world", requestBody, requestPath)+ ctx := t.Context()++ model := NewAliyunModel(+ map[string]string{"default": server.URL},+ URLSuffix{ASR: aliyunASRTestSuffix},+ )+ apiKey := "test-key"+ modelName := "qwen-asr-flash"+ audio := []byte("RIFF....WAVE")+ file := writeAliyunASRTestFile(t, "clip.wav", audio)++ response, err := model.TranscribeAudio(ctx, &modelName, &file, &APIConfig{ApiKey: &apiKey}, nil, nil)+ if err != nil {+ t.Fatalf("TranscribeAudio: %v", err)+ }+ if response.Text != "hello world" {+ t.Errorf("text = %q, want %q", response.Text, "hello world")+ }++ if got := <-requestPath; got != "/"+aliyunASRTestSuffix {+ t.Errorf("request path = %q, want /%s", got, aliyunASRTestSuffix)+ }+ body := <-requestBody+ if body["model"] != "qwen-asr-flash" {+ t.Errorf("model = %v, want qwen-asr-flash", body["model"])+ }+ messages := aliyunASRMessages(t, body)+ if len(messages) != 2 {+ t.Fatalf("messages = %d, want 2 (system + user)", len(messages))+ }+ if messages[0]["role"] != "system" {+ t.Errorf("messages[0].role = %v, want system", messages[0]["role"])+ }+ if messages[1]["role"] != "user" {+ t.Errorf("messages[1].role = %v, want user", messages[1]["role"])+ }+ wantURL := "data:audio/wav;base64," + base64.StdEncoding.EncodeToString(audio)+ if got := aliyunASRContentString(t, messages[1], "audio"); got != wantURL {+ t.Errorf("audio = %q, want %q", got, wantURL)+ }++ params, ok := body["parameters"].(map[string]interface{})+ if !ok {+ t.Fatalf("parameters = %T, want JSON object", body["parameters"])+ }+ if params["result_format"] != "message" {+ t.Errorf("result_format = %v, want message", params["result_format"])+ }+ options, ok := params["asr_options"].(map[string]interface{})+ if !ok {+ t.Fatalf("asr_options = %T, want JSON object", params["asr_options"])+ }+ if options["enable_lid"] != true || options["enable_itn"] != false {+ t.Errorf("asr_options = %v, want defaults enable_lid=true enable_itn=false", options)+ }+}++func TestAliyunTranscribeAudioHonorsContextAndASROptions(t *testing.T) {+ withSSRFBypass(t)+ requestBody := make(chan map[string]interface{}, 1)+ server := newAliyunASRTestServer(t, "ok", requestBody, nil)+ ctx := t.Context()++ model := NewAliyunModel(+ map[string]string{"default": server.URL},+ URLSuffix{ASR: aliyunASRTestSuffix},+ )+ apiKey := "test-key"+ modelName := "qwen-asr-flash"+ file := writeAliyunASRTestFile(t, "clip.mp3", []byte("ID3...."))++ _, err := model.TranscribeAudio(ctx, &modelName, &file, &APIConfig{ApiKey: &apiKey}, &ASRConfig{Params: map[string]interface{}{+ "context": "NAVI vs Faze",+ "language": "en",+ "enable_itn": true,+ }}, nil)+ if err != nil {+ t.Fatalf("TranscribeAudio: %v", err)+ }+ body := <-requestBody+ messages := aliyunASRMessages(t, body)+ if got := aliyunASRContentString(t, messages[0], "text"); got != "NAVI vs Faze" {+ t.Errorf("system text = %q, want NAVI vs Faze", got)+ }+ if got := aliyunASRContentString(t, messages[1], "audio"); !strings.HasPrefix(got, "data:audio/mpeg;base64,") {+ t.Errorf("audio data URL = %q, want mp3 MIME prefix", got)+ }+ params, ok := body["parameters"].(map[string]interface{})+ if !ok {+ t.Fatalf("parameters = %T, want JSON object", body["parameters"])+ }+ options, ok := params["asr_options"].(map[string]interface{})+ if !ok {+ t.Fatalf("asr_options = %T, want JSON object", params["asr_options"])+ }+ if options["language"] != "en" {+ t.Errorf("language = %v, want en", options["language"])+ }+ if options["enable_itn"] != true {+ t.Errorf("enable_itn = %v, want true (explicit override)", options["enable_itn"])+ }+ if options["enable_lid"] != true {+ t.Errorf("enable_lid = %v, want default true", options["enable_lid"])+ }+}++func TestAliyunTranscribeAudioSurfacesAPIError(t *testing.T) {+ withSSRFBypass(t)+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ w.WriteHeader(http.StatusUnauthorized)+ _, _ = w.Write([]byte(`{"code":"InvalidApiKey","message":"Invalid API-key provided."}`))+ }))+ defer server.Close()++ model := NewAliyunModel(+ map[string]string{"default": server.URL},+ URLSuffix{ASR: aliyunASRTestSuffix},+ )+ apiKey := "bad-key"+ modelName := "qwen-asr-flash"+ file := writeAliyunASRTestFile(t, "clip.wav", []byte("data"))++ _, err := model.TranscribeAudio(t.Context(), &modelName, &file, &APIConfig{ApiKey: &apiKey}, nil, nil)+ var statusErr *APIStatusError+ if !errors.As(err, &statusErr) || statusErr.Status != http.StatusUnauthorized {+ t.Fatalf("error = %v, want APIStatusError with status 401", err)+ }+ if !strings.Contains(statusErr.Body, "InvalidApiKey") {+ t.Fatalf("error body = %q, want to include InvalidApiKey", statusErr.Body)+ }+}++func TestAliyunTranscribeAudioRejectsEmptyFile(t *testing.T) {+ withSSRFBypass(t)+ model := NewAliyunModel(+ map[string]string{"default": "https://dashscope.example"},+ URLSuffix{ASR: aliyunASRTestSuffix},+ )+ apiKey := "test-key"+ modelName := "qwen-asr-flash"+ file := writeAliyunASRTestFile(t, "empty.wav", nil)++ _, err := model.TranscribeAudio(t.Context(), &modelName, &file, &APIConfig{ApiKey: &apiKey}, nil, nil)+ if err == nil || err.Error() != "audio file is empty" {+ t.Fatalf("error = %v, want empty audio error", err)+ }+}++func TestAliyunTranscribeAudioRequiresASRSuffix(t *testing.T) {+ withSSRFBypass(t)+ model := NewAliyunModel(+ map[string]string{"default": "https://dashscope.example"},+ URLSuffix{Chat: "compatible-mode/v1/chat/completions"},+ )+ apiKey := "test-key"+ modelName := "qwen-asr-flash"+ file := writeAliyunASRTestFile(t, "clip.wav", []byte("data"))++ _, err := model.TranscribeAudio(t.Context(), &modelName, &file, &APIConfig{ApiKey: &apiKey}, nil, nil)+ if err == nil || err.Error() != "aliyun ASR URL suffix is required" {+ t.Fatalf("error = %v, want missing ASR suffix error", err)+ }+}++func TestAliyunTranscribeAudioWithSenderSendsSingleChunk(t *testing.T) {+ withSSRFBypass(t)+ server := newAliyunASRTestServer(t, "transcribed", nil, nil)+ model := NewAliyunModel(+ map[string]string{"default": server.URL},+ URLSuffix{ASR: aliyunASRTestSuffix},+ )+ apiKey := "test-key"+ modelName := "qwen-asr-flash"+ file := writeAliyunASRTestFile(t, "clip.wav", []byte("data"))++ var chunks []string+ err := model.TranscribeAudioWithSender(t.Context(), &modelName, &file, &APIConfig{ApiKey: &apiKey}, nil, nil, func(content, _ *string) error {+ if content != nil {+ chunks = append(chunks, *content)+ }+ return nil+ })+ if err != nil {+ t.Fatalf("TranscribeAudioWithSender: %v", err)+ }+ if len(chunks) != 1 || chunks[0] != "transcribed" {+ t.Fatalf("chunks = %v, want [transcribed]", chunks)+ }+}