RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(ollama): preserve multimodal images (#17663)
Loi Nguyenf420e47f952d
diff --git a/internal/entity/models/ollama.go b/internal/entity/models/ollama.goindex e82fa29bef..c01584221b 100644--- a/internal/entity/models/ollama.go+++ b/internal/entity/models/ollama.go@@ -52,6 +52,92 @@ func (o *OllamaModel) Name() string {return "Ollama"}+func buildOllamaRequestBody(cfg *ChatConfig, modelName string, messages []Message, stream bool) map[string]any {+ reqBody := buildRequestBody(cfg, modelName, messages, stream)+ reqBody["messages"] = buildOllamaMessages(messages)+ return reqBody+}++func buildOllamaMessages(messages []Message) []map[string]any {+ apiMessages := buildChatMessages(messages)+ for i, message := range messages {+ content, images, ok := ollamaMultimodalContent(message.Content)+ if !ok {+ continue+ }+ apiMessages[i]["content"] = content+ if len(images) > 0 {+ apiMessages[i]["images"] = images+ }+ }+ return apiMessages+}++func ollamaMultimodalContent(content interface{}) (string, []string, bool) {+ var parts []interface{}+ switch value := content.(type) {+ case []interface{}:+ parts = value+ case []map[string]interface{}:+ parts = make([]interface{}, len(value))+ for i := range value {+ parts[i] = value[i]+ }+ default:+ return "", nil, false+ }++ var textParts []string+ var images []string+ for _, part := range parts {+ partMap, ok := part.(map[string]interface{})+ if !ok {+ if text, ok := part.(string); ok {+ textParts = append(textParts, text)+ }+ continue+ }++ partType, _ := partMap["type"].(string)+ switch partType {+ case "text", "input_text":+ if text, ok := partMap["text"].(string); ok {+ textParts = append(textParts, text)+ }+ case "image_url":+ if imageURL := ollamaImageURL(partMap["image_url"]); imageURL != "" {+ images = append(images, cleanOllamaImageData(imageURL))+ }+ }+ }++ return strings.Join(textParts, "\n"), images, true+}++func ollamaImageURL(value interface{}) string {+ switch image := value.(type) {+ case string:+ return image+ case map[string]interface{}:+ url, _ := image["url"].(string)+ return url+ case map[string]string:+ return image["url"]+ default:+ return ""+ }+}++func cleanOllamaImageData(image string) string {+ const base64Marker = ";base64,"+ if strings.HasPrefix(image, "data:") {+ if marker := strings.Index(image, base64Marker); marker >= 0 {+ return image[marker+len(base64Marker):]+ }+ }+ return image+}+func (o *OllamaModel) ChatWithMessages(ctx context.Context, modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig, modelUsage *common.ModelUsage) (*ChatResponse, error) {if len(messages) == 0 {return nil, fmt.Errorf("message is nil")@@ -70,7 +156,7 @@ func (o *OllamaModel) ChatWithMessages(ctx context.Context, modelName string, me}// Build request body- reqBody := buildRequestBody(chatModelConfig, modelName, messages, false)+ reqBody := buildOllamaRequestBody(chatModelConfig, modelName, messages, false)if chatModelConfig != nil {if chatModelConfig.Effort != nil && *chatModelConfig.Effort != "" {@@ -108,7 +194,7 @@ func (o *OllamaModel) ChatStreamlyWithSender(ctx context.Context, modelName stri}// Build request body with streaming enabled- reqBody := buildRequestBody(modelConfig, modelName, messages, true)+ reqBody := buildOllamaRequestBody(modelConfig, modelName, messages, true)if modelConfig.Effort != nil && *modelConfig.Effort != "" {if strings.HasPrefix(strings.ToLower(modelName), "gpt-oss") {diff --git a/internal/entity/models/ollama_test.go b/internal/entity/models/ollama_test.goindex 786ad13f40..e625ed763a 100644--- a/internal/entity/models/ollama_test.go+++ b/internal/entity/models/ollama_test.go@@ -17,9 +17,11 @@package modelsimport (+ "encoding/json""io""net/http""net/http/httptest"+ "ragflow/internal/common""testing")@@ -27,6 +29,113 @@ func newOllamaForListModelsTest(baseURL string) *OllamaModel {return NewOllamaModel(map[string]string{"default": baseURL}, URLSuffix{Models: "api/tags"})}+func newOllamaForChatTest(baseURL string) *OllamaModel {+ return NewOllamaModel(map[string]string{"default": baseURL}, URLSuffix{Chat: "api/chat", AsyncChat: "api/chat"})+}++func ollamaMultimodalTestMessages() []Message {+ return []Message{{Role: "user", Content: []interface{}{+ map[string]interface{}{"type": "text", "text": "describe"},+ map[string]interface{}{"type": "image_url", "image_url": map[string]interface{}{"url": "data:image/png;base64,aGVsbG8="}},+ map[string]interface{}{"type": "text", "text": "carefully"},+ map[string]interface{}{"type": "image_url", "image_url": map[string]interface{}{"url": "cmF3LWltYWdl"}},+ map[string]interface{}{"type": "image_url", "image_url": map[string]interface{}{"url": "https://example.com/cat.png"}},+ }}}+}++func assertOllamaMultimodalRequest(t *testing.T, body map[string]interface{}, stream bool) {+ t.Helper()+ if body["stream"] != stream {+ t.Errorf("stream=%v, want %v", body["stream"], stream)+ }+ messages, ok := body["messages"].([]interface{})+ if !ok || len(messages) != 1 {+ t.Errorf("messages=%v, want one message", body["messages"])+ return+ }+ message, ok := messages[0].(map[string]interface{})+ if !ok {+ t.Errorf("message=%v, want object", messages[0])+ return+ }+ if message["content"] != "describe\ncarefully" {+ t.Errorf("content=%v, want joined text parts", message["content"])+ }+ images, ok := message["images"].([]interface{})+ if !ok {+ t.Errorf("images=%v, want array", message["images"])+ return+ }+ want := []string{"aGVsbG8=", "cmF3LWltYWdl", "https://example.com/cat.png"}+ if len(images) != len(want) {+ t.Errorf("images=%v, want %v", images, want)+ return+ }+ for i, expected := range want {+ if images[i] != expected {+ t.Errorf("images[%d]=%v, want %q", i, images[i], expected)+ }+ }+}++func TestOllamaChatMapsMultimodalImages(t *testing.T) {+ withSSRFBypass(t)+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ var body map[string]interface{}+ if err := json.NewDecoder(r.Body).Decode(&body); err != nil {+ t.Errorf("decode request: %v", err)+ http.Error(w, "invalid request", http.StatusBadRequest)+ return+ }+ assertOllamaMultimodalRequest(t, body, false)+ _, _ = io.WriteString(w, `{"message":{"content":"ok","thinking":""}}`)+ }))+ defer srv.Close()++ response, err := newOllamaForChatTest(srv.URL).ChatWithMessages(+ t.Context(),+ "llava",+ ollamaMultimodalTestMessages(),+ &APIConfig{},+ &ChatConfig{},+ &common.ModelUsage{},+ )+ if err != nil {+ t.Fatalf("ChatWithMessages: %v", err)+ }+ if response.Answer == nil || *response.Answer != "ok" {+ t.Fatalf("answer=%v, want ok", response.Answer)+ }+}++func TestOllamaStreamingChatMapsMultimodalImages(t *testing.T) {+ withSSRFBypass(t)+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ var body map[string]interface{}+ if err := json.NewDecoder(r.Body).Decode(&body); err != nil {+ t.Errorf("decode request: %v", err)+ http.Error(w, "invalid request", http.StatusBadRequest)+ return+ }+ assertOllamaMultimodalRequest(t, body, true)+ _, _ = io.WriteString(w, "{\"message\":{\"content\":\"ok\"},\"done\":false}\n{\"done\":true}\n")+ }))+ defer srv.Close()++ err := newOllamaForChatTest(srv.URL).ChatStreamlyWithSender(+ t.Context(),+ "llava",+ ollamaMultimodalTestMessages(),+ &APIConfig{},+ &ChatConfig{},+ &common.ModelUsage{},+ func(*string, *string) error { return nil },+ )+ if err != nil {+ t.Fatalf("ChatStreamlyWithSender: %v", err)+ }+}+func TestOllamaListModels(t *testing.T) {withSSRFBypass(t)ctx := t.Context()