RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: handle Jina VLM streaming responses (#18855)
buua436559bc17432c7
diff --git a/internal/entity/models/jina.go b/internal/entity/models/jina.goindex 9558c969a0..4450111d90 100644--- a/internal/entity/models/jina.go+++ b/internal/entity/models/jina.go@@ -17,15 +17,20 @@package modelsimport (+ "bufio""bytes""context""encoding/json""fmt""io""net/http"+ "net/url""ragflow/internal/common""strings""time"+ "unicode/utf8"++ "go.uber.org/zap")type JinaModel struct {@@ -106,6 +111,7 @@ func (j *JinaModel) ChatWithMessages(ctx context.Context, modelName string, messreturn nil, err}baseURL = strings.TrimSuffix(baseURL, "/")+ messages = prepareJinaMessages(baseURL, messages)url := fmt.Sprintf("%s/%s", baseURL, j.baseModel.URLSuffix.Chat)reqBody := buildRequestBody(chatModelConfig, modelName, messages, false)@@ -125,6 +131,9 @@ func (j *JinaModel) ChatWithMessages(ctx context.Context, modelName string, mess}func (j *JinaModel) ChatStreamlyWithSender(ctx context.Context, modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig, modelUsage *common.ModelUsage, sender func(*string, *string) error) error {+ if sender == nil {+ return fmt.Errorf("sender is required")+ }if err := j.baseModel.APIConfigCheck(apiConfig); err != nil {return err}@@ -143,10 +152,10 @@ func (j *JinaModel) ChatStreamlyWithSender(ctx context.Context, modelName stringreturn err}baseURL = strings.TrimSuffix(baseURL, "/")+ messages = prepareJinaMessages(baseURL, messages)url := fmt.Sprintf("%s/%s", baseURL, j.baseModel.URLSuffix.Chat)reqBody := buildRequestBody(chatModelConfig, modelName, messages, true)- reqBody["stream_options"] = map[string]interface{}{"include_usage": true}if chatModelConfig != nil {chatModelConfig.ToolCallsResult = nil@@ -156,9 +165,267 @@ func (j *JinaModel) ChatStreamlyWithSender(ctx context.Context, modelName string}}- return j.baseModel.doStreamRequest(ctx, url, apiConfig, reqBody, streamCallTimeout, func(body io.ReadCloser) error {- return HandleStreamingResponse(body, modelUsage, chatModelConfig, OpenAIParserConfig, sender)+ err = j.baseModel.doStreamRequest(ctx, url, apiConfig, reqBody, streamCallTimeout, func(body io.ReadCloser) error {+ return handleJinaStreamingResponse(body, modelUsage, chatModelConfig, sender)})+ if err != nil {+ upstreamResponse := err.Error()+ errorChunk := "**ERROR**: " + upstreamResponse+ if sendErr := sender(&errorChunk, nil); sendErr != nil {+ return fmt.Errorf("forward Jina stream error: %w", sendErr)+ }+ endOfStream := "[DONE]"+ if sendErr := sender(&endOfStream, nil); sendErr != nil {+ return fmt.Errorf("finish Jina error stream: %w", sendErr)+ }+ return nil+ }+ return nil+}++func prepareJinaMessages(baseURL string, messages []Message) []Message {+ parsedURL, err := url.Parse(baseURL)+ if err != nil || !strings.EqualFold(parsedURL.Hostname(), "api.jina.ai") {+ return messages+ }+ return mergeLeadingJinaSystemMessages(messages)+}++func mergeLeadingJinaSystemMessages(messages []Message) []Message {+ if len(messages) == 0 {+ return nil+ }+ firstNonSystem := 0+ systemParts := make([]string, 0, 1)+ for firstNonSystem < len(messages) && strings.EqualFold(strings.TrimSpace(messages[firstNonSystem].Role), "system") {+ if content, ok := messages[firstNonSystem].Content.(string); ok && strings.TrimSpace(content) != "" {+ systemParts = append(systemParts, content)+ }+ firstNonSystem+++ }+ if firstNonSystem == 0 {+ return messages+ }++ out := append([]Message(nil), messages[firstNonSystem:]...)+ systemText := strings.Join(systemParts, "\n\n")+ if len(out) == 0 || !strings.EqualFold(strings.TrimSpace(out[0].Role), "user") {+ return append([]Message{{Role: "user", Content: systemText}}, out...)+ }+ out[0].Content = prependJinaSystemText(out[0].Content, systemText)+ return out+}++func prependJinaSystemText(content any, systemText string) any {+ if strings.TrimSpace(systemText) == "" {+ return content+ }+ switch value := content.(type) {+ case string:+ if strings.TrimSpace(value) == "" {+ return systemText+ }+ return systemText + "\n\n" + value+ case []interface{}:+ parts := make([]interface{}, 0, len(value)+1)+ parts = append(parts, map[string]interface{}{"type": "text", "text": systemText})+ return append(parts, value...)+ default:+ return systemText+ }+}++// handleJinaStreamingResponse adapts Jina-VLM's thinking stream to the+// internal content/reasoning split. Jina emits both kinds of text in+// delta.content and marks thinking chunks with delta.type="think"; the+// generic OpenAI handler treats all of them as answer content.+func handleJinaStreamingResponse(body io.Reader, modelUsage *common.ModelUsage, chatConfig *ChatConfig, sender func(*string, *string) error) error {+ if sender == nil {+ return fmt.Errorf("sender is required")+ }++ var streamUsage *TokenUsage+ accumulatedToolCalls := make(map[int]map[string]any)+ thinking := false+ thinkingClosed := false+ var pending strings.Builder+ streamModel := ""+ streamResponseID := ""++ emit := func(text string, reasoning bool) error {+ if text == "" {+ return nil+ }+ if reasoning {+ return sender(nil, &text)+ }+ return sender(&text, nil)+ }++ flushThinking := func(final bool) error {+ value := pending.String()+ if !final && len(value) > len("</think>")-1 {+ safeEnd := len(value) - (len("</think>") - 1)+ for safeEnd > 0 && !utf8.RuneStart(value[safeEnd]) {+ safeEnd--+ }+ safe := value[:safeEnd]+ pending.Reset()+ pending.WriteString(value[len(safe):])+ return emit(safe, true)+ }+ pending.Reset()+ return emit(value, true)+ }++ consumeContent := func(content string, typ string) error {+ if typ == "think" && !thinkingClosed {+ thinking = true+ }+ if !thinking {+ return emit(content, false)+ }++ pending.WriteString(content)+ for thinking {+ value := pending.String()+ idx := strings.Index(value, "</think>")+ if idx < 0 {+ return flushThinking(false)+ }+ if err := emit(value[:idx], true); err != nil {+ return err+ }+ pending.Reset()+ pending.WriteString(value[idx+len("</think>"):])+ thinking = false+ thinkingClosed = true+ }+ value := pending.String()+ pending.Reset()+ return emit(value, false)+ }++ _, eventCount, err := parseJinaStream(body, func(event map[string]any) error {+ if id, ok := event["id"].(string); ok && id != "" {+ streamResponseID = id+ }+ if u, ok := OpenAIParserConfig.StreamParser(event); ok {+ streamUsage = u+ }+ if m, ok := event["model"].(string); ok {+ streamModel = m+ }+ if apiErr, ok := event["error"]; ok && apiErr != nil {+ return fmt.Errorf("upstream stream error: %v", apiErr)+ }+ choices, ok := event["choices"].([]any)+ if !ok || len(choices) == 0 {+ return nil+ }+ choice, ok := choices[0].(map[string]any)+ if !ok {+ return nil+ }+ delta, ok := choice["delta"].(map[string]any)+ if !ok {+ return nil+ }+ accumulateToolCallDeltas(delta, accumulatedToolCalls)+ content, _ := delta["content"].(string)+ typ, _ := delta["type"].(string)+ return consumeContent(content, typ)+ })+ if err != nil {+ return fmt.Errorf("failed to scan Jina response body: %w", err)+ }+ if thinking {+ if err := flushThinking(true); err != nil {+ return err+ }+ }+ if eventCount == 0 {+ return fmt.Errorf("Jina stream contained no JSON events")+ }+ if chatConfig != nil {+ setSortedToolCallsResult(chatConfig, accumulatedToolCalls)+ }+ if streamUsage != nil {+ recordResponseUsage(modelUsage, streamResponseID, streamUsage, "chat")+ if chatConfig != nil {+ chatConfig.UsageResult = streamUsage+ common.Info("StreamUsage", zap.String("model", streamModel), zap.Int("prompt", streamUsage.PromptTokens), zap.Int("completion", streamUsage.CompletionTokens), zap.Int("total", streamUsage.TotalTokens))+ }+ }+ endOfStream := "[DONE]"+ return sender(&endOfStream, nil)+}++// parseJinaStream accepts both OpenAI-style SSE frames and Jina-VLM's+// newline-delimited JSON stream. Jina may terminate a successful stream by+// closing the connection instead of emitting a final [DONE] frame.+func parseJinaStream(body io.Reader, onEvent func(map[string]any) error) (done bool, eventCount int, err error) {+ scanner := bufio.NewScanner(body)+ scanner.Buffer(make([]byte, 64*1024), 1024*1024)+ var sseData []string+ flushSSE := func() error {+ if len(sseData) == 0 {+ return nil+ }+ line := strings.Join(sseData, "\n")+ sseData = nil+ if line == "[DONE]" {+ done = true+ return nil+ }+ var event map[string]any+ if err := json.Unmarshal([]byte(line), &event); err != nil {+ return fmt.Errorf("invalid Jina stream event: %w", err)+ }+ eventCount+++ return onEvent(event)+ }+ for scanner.Scan() {+ line := strings.TrimRight(scanner.Text(), "\r")+ if line == "" {+ if err := flushSSE(); err != nil {+ return false, eventCount, err+ }+ if done {+ return true, eventCount, nil+ }+ continue+ }+ if strings.HasPrefix(line, "data:") {+ sseData = append(sseData, strings.TrimSpace(line[len("data:"):]))+ continue+ } else if strings.Contains(line, ":") && !strings.HasPrefix(line, "{") && !strings.HasPrefix(line, "[") {+ // Ignore non-data SSE fields such as event:, id:, and retry:.+ continue+ }+ line = strings.TrimSpace(line)+ if strings.HasPrefix(line, "{") || strings.HasPrefix(line, "[") {+ var event map[string]any+ if err := json.Unmarshal([]byte(line), &event); err != nil {+ return false, eventCount, fmt.Errorf("invalid Jina stream event: %w", err)+ }+ eventCount+++ if err := onEvent(event); err != nil {+ return false, eventCount, err+ }+ continue+ }+ }+ if err := scanner.Err(); err != nil {+ return false, eventCount, err+ }+ if err := flushSSE(); err != nil {+ return false, eventCount, err+ }+ if done {+ return true, eventCount, nil+ }+ return false, eventCount, nil}func (j *JinaModel) Embed(ctx context.Context, modelName *string, request EmbedRequest, apiConfig *APIConfig, embeddingConfig *EmbeddingConfig, modelUsage *common.ModelUsage) ([]EmbeddingData, error) {diff --git a/internal/entity/models/jina_test.go b/internal/entity/models/jina_test.goindex 317b6b5dc2..0b15723099 100644--- a/internal/entity/models/jina_test.go+++ b/internal/entity/models/jina_test.go@@ -91,6 +91,187 @@ func TestJinaChatHappyPath(t *testing.T) {}}+func TestJinaChatStreamSeparatesThinkingContent(t *testing.T) {+ withSSRFBypass(t)+ srv := newJinaServer(t, "/chat/completions", func(t *testing.T, body map[string]interface{}, w http.ResponseWriter) {+ if body["stream"] != true {+ t.Errorf("stream=%v, want true", body["stream"])+ }+ if _, ok := body["stream_options"]; ok {+ t.Error("stream_options should not be sent to Jina")+ }+ _, _ = io.WriteString(w, "data: {\"model\":\"jina-vlm\",\"choices\":[{\"delta\":{\"content\":\"reasoning\",\"type\":\"think\"}}]}\n\n")+ _, _ = io.WriteString(w, "data: {\"choices\":[{\"delta\":{\"content\":\"</thi\"}}]}\n\n")+ _, _ = io.WriteString(w, "data: {\"choices\":[{\"delta\":{\"content\":\"nk>\",\"type\":\"think\"}}]}\n\n")+ _, _ = io.WriteString(w, "data: {\"choices\":[{\"delta\":{\"content\":\"answer\",\"type\":\"think\"}}]}\n\n")+ _, _ = io.WriteString(w, "data: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n")+ _, _ = io.WriteString(w, "data: [DONE]\n\n")+ })+ defer srv.Close()++ var content, reasoning strings.Builder+ apiKey := "test-key"+ err := newJinaForTest(srv.URL).ChatStreamlyWithSender(+ t.Context(), "jina-vlm", []Message{{Role: "user", Content: "ping"}},+ &APIConfig{ApiKey: &apiKey}, nil, nil,+ func(delta, reason *string) error {+ if delta != nil && *delta != "[DONE]" {+ content.WriteString(*delta)+ }+ if reason != nil {+ reasoning.WriteString(*reason)+ }+ return nil+ },+ )+ if err != nil {+ t.Fatalf("ChatStreamlyWithSender: %v", err)+ }+ if got := reasoning.String(); got != "reasoning" {+ t.Errorf("reasoning=%q, want %q", got, "reasoning")+ }+ if got := content.String(); got != "answer" {+ t.Errorf("content=%q, want %q", got, "answer")+ }+}++func TestJinaChatStreamAcceptsNDJSONAndCleanEOF(t *testing.T) {+ withSSRFBypass(t)+ srv := newJinaServer(t, "/chat/completions", func(t *testing.T, _ map[string]interface{}, w http.ResponseWriter) {+ _, _ = io.WriteString(w, "{\"model\":\"jina-vlm\",\"choices\":[{\"delta\":{\"content\":\"plan\",\"type\":\"think\"}}]}\n")+ _, _ = io.WriteString(w, "{\"model\":\"jina-vlm\",\"choices\":[{\"delta\":{\"content\":\"</think>\"}}]}\n")+ _, _ = io.WriteString(w, "{\"model\":\"jina-vlm\",\"choices\":[{\"delta\":{\"content\":\"hello\"}}]}\n")+ })+ defer srv.Close()++ var content, reasoning strings.Builder+ apiKey := "test-key"+ err := newJinaForTest(srv.URL).ChatStreamlyWithSender(+ t.Context(), "jina-vlm", []Message{{Role: "user", Content: "ping"}},+ &APIConfig{ApiKey: &apiKey}, nil, nil,+ func(delta, reason *string) error {+ if delta != nil && *delta != "[DONE]" {+ content.WriteString(*delta)+ }+ if reason != nil {+ reasoning.WriteString(*reason)+ }+ return nil+ },+ )+ if err != nil {+ t.Fatalf("ChatStreamlyWithSender: %v", err)+ }+ if got := reasoning.String(); got != "plan" {+ t.Errorf("reasoning=%q, want %q", got, "plan")+ }+ if got := content.String(); got != "hello" {+ t.Errorf("content=%q, want %q", got, "hello")+ }+}++func TestParseJinaStreamAcceptsMultilineSSEData(t *testing.T) {+ var events []map[string]any+ done, count, err := parseJinaStream(strings.NewReader("data: {\"choices\":\n"++ "data: [{\"delta\":{\"content\":\"hello\"}}]}\n\n"), func(event map[string]any) error {+ events = append(events, event)+ return nil+ })+ if err != nil {+ t.Fatalf("parseJinaStream: %v", err)+ }+ if done || count != 1 || len(events) != 1 {+ t.Fatalf("done=%v count=%d events=%d, want false/1/1", done, count, len(events))+ }+}++func TestJinaThinkingFlushPreservesUTF8(t *testing.T) {+ var reasoning strings.Builder+ err := handleJinaStreamingResponse(strings.NewReader("{"++ "\"choices\":[{\"delta\":{\"content\":\"思考过程很长\",\"type\":\"think\"}}]}\n"), nil, nil,+ func(_ *string, reason *string) error {+ if reason != nil {+ reasoning.WriteString(*reason)+ }+ return nil+ })+ if err != nil {+ t.Fatalf("handleJinaStreamingResponse: %v", err)+ }+ if !strings.Contains(reasoning.String(), "思考过程很长") {+ t.Errorf("reasoning=%q, want valid UTF-8 content", reasoning.String())+ }+}++func TestJinaChatStreamForwardsHTTPError(t *testing.T) {+ withSSRFBypass(t)+ srv := newJinaServer(t, "/chat/completions", func(t *testing.T, _ map[string]interface{}, w http.ResponseWriter) {+ w.WriteHeader(http.StatusBadRequest)+ _, _ = io.WriteString(w, `{"detail":{"message":"roles must alternate"}}`)+ })+ defer srv.Close()++ var chunks []string+ apiKey := "test-key"+ err := newJinaForTest(srv.URL).ChatStreamlyWithSender(+ t.Context(), "jina-vlm", []Message{{Role: "user", Content: "ping"}},+ &APIConfig{ApiKey: &apiKey}, nil, nil,+ func(content, _ *string) error {+ if content != nil {+ chunks = append(chunks, *content)+ }+ return nil+ },+ )+ if err != nil {+ t.Fatalf("ChatStreamlyWithSender: %v", err)+ }+ if len(chunks) != 2 {+ t.Fatalf("chunks=%q, want error and done chunks", chunks)+ }+ if !strings.Contains(chunks[0], `**ERROR**: API request failed with status 400`) || !strings.Contains(chunks[0], `roles must alternate`) {+ t.Errorf("error chunk=%q", chunks[0])+ }+ if chunks[1] != "[DONE]" {+ t.Errorf("terminal chunk=%q, want [DONE]", chunks[1])+ }+}++func TestPrepareJinaMessagesMergesSystemOnlyForAPIEndpoint(t *testing.T) {+ messages := []Message{+ {Role: "system", Content: "be helpful"},+ {Role: "user", Content: "hello"},+ }++ apiMessages := prepareJinaMessages("https://api.jina.ai/v1", messages)+ if len(apiMessages) != 1 || apiMessages[0].Role != "user" || apiMessages[0].Content != "be helpful\n\nhello" {+ t.Errorf("api.jina.ai messages=%#v", apiMessages)+ }+ deepSearchMessages := prepareJinaMessages("https://deepsearch.jina.ai/v1", messages)+ if len(deepSearchMessages) != 2 || deepSearchMessages[0].Role != "system" || deepSearchMessages[1].Role != "user" {+ t.Errorf("deepsearch.jina.ai messages=%#v", deepSearchMessages)+ }+}++func TestPrepareJinaMessagesPreservesMultimodalUserContent(t *testing.T) {+ image := map[string]interface{}{"type": "image_url", "image_url": map[string]interface{}{"url": "data:image/png;base64,x"}}+ messages := prepareJinaMessages("https://api.jina.ai/v1", []Message{+ {Role: "system", Content: "describe the image"},+ {Role: "user", Content: []interface{}{image}},+ })+ parts, ok := messages[0].Content.([]interface{})+ if !ok || len(parts) != 2 {+ t.Fatalf("content=%#v, want text and image parts", messages[0].Content)+ }+ textPart, ok := parts[0].(map[string]interface{})+ if !ok || textPart["type"] != "text" || textPart["text"] != "describe the image" {+ t.Errorf("text part=%#v", parts[0])+ }+ if parts[1].(map[string]interface{})["type"] != "image_url" {+ t.Errorf("image part=%#v", parts[1])+ }+}+func TestJinaChatPreservesReasoningContent(t *testing.T) {withSSRFBypass(t)srv := newJinaServer(t, "/chat/completions", func(t *testing.T, _ map[string]interface{}, w http.ResponseWriter) {