Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(ollama): preserve multimodal images (#17663)

Loi Nguyenf420e47f952d
diff --git a/internal/entity/models/ollama.go b/internal/entity/models/ollama.go
index e82fa29bef..c01584221b 100644
--- a/internal/entity/models/ollama.go
+++ b/internal/entity/models/ollama.go
@@ -52,6 +52,92 @@ func (o *OllamaModel) Name() string {
return "Ollama"
}
+func buildOllamaRequestBody(cfg *ChatConfig, modelName string, messages []Message, stream bool) map[string]any {
+ reqBody := buildRequestBody(cfg, modelName, messages, stream)
+ reqBody["messages"] = buildOllamaMessages(messages)
+ return reqBody
+}
+
+func buildOllamaMessages(messages []Message) []map[string]any {
+ apiMessages := buildChatMessages(messages)
+ for i, message := range messages {
+ content, images, ok := ollamaMultimodalContent(message.Content)
+ if !ok {
+ continue
+ }
+ apiMessages[i]["content"] = content
+ if len(images) > 0 {
+ apiMessages[i]["images"] = images
+ }
+ }
+ return apiMessages
+}
+
+func ollamaMultimodalContent(content interface{}) (string, []string, bool) {
+ var parts []interface{}
+ switch value := content.(type) {
+ case []interface{}:
+ parts = value
+ case []map[string]interface{}:
+ parts = make([]interface{}, len(value))
+ for i := range value {
+ parts[i] = value[i]
+ }
+ default:
+ return "", nil, false
+ }
+
+ var textParts []string
+ var images []string
+ for _, part := range parts {
+ partMap, ok := part.(map[string]interface{})
+ if !ok {
+ if text, ok := part.(string); ok {
+ textParts = append(textParts, text)
+ }
+ continue
+ }
+
+ partType, _ := partMap["type"].(string)
+ switch partType {
+ case "text", "input_text":
+ if text, ok := partMap["text"].(string); ok {
+ textParts = append(textParts, text)
+ }
+ case "image_url":
+ if imageURL := ollamaImageURL(partMap["image_url"]); imageURL != "" {
+ images = append(images, cleanOllamaImageData(imageURL))
+ }
+ }
+ }
+
+ return strings.Join(textParts, "\n"), images, true
+}
+
+func ollamaImageURL(value interface{}) string {
+ switch image := value.(type) {
+ case string:
+ return image
+ case map[string]interface{}:
+ url, _ := image["url"].(string)
+ return url
+ case map[string]string:
+ return image["url"]
+ default:
+ return ""
+ }
+}
+
+func cleanOllamaImageData(image string) string {
+ const base64Marker = ";base64,"
+ if strings.HasPrefix(image, "data:") {
+ if marker := strings.Index(image, base64Marker); marker >= 0 {
+ return image[marker+len(base64Marker):]
+ }
+ }
+ return image
+}
+
func (o *OllamaModel) ChatWithMessages(ctx context.Context, modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig, modelUsage *common.ModelUsage) (*ChatResponse, error) {
if len(messages) == 0 {
return nil, fmt.Errorf("message is nil")
@@ -70,7 +156,7 @@ func (o *OllamaModel) ChatWithMessages(ctx context.Context, modelName string, me
}
// Build request body
- reqBody := buildRequestBody(chatModelConfig, modelName, messages, false)
+ reqBody := buildOllamaRequestBody(chatModelConfig, modelName, messages, false)
if chatModelConfig != nil {
if chatModelConfig.Effort != nil && *chatModelConfig.Effort != "" {
@@ -108,7 +194,7 @@ func (o *OllamaModel) ChatStreamlyWithSender(ctx context.Context, modelName stri
}
// Build request body with streaming enabled
- reqBody := buildRequestBody(modelConfig, modelName, messages, true)
+ reqBody := buildOllamaRequestBody(modelConfig, modelName, messages, true)
if modelConfig.Effort != nil && *modelConfig.Effort != "" {
if strings.HasPrefix(strings.ToLower(modelName), "gpt-oss") {
diff --git a/internal/entity/models/ollama_test.go b/internal/entity/models/ollama_test.go
index 786ad13f40..e625ed763a 100644
--- a/internal/entity/models/ollama_test.go
+++ b/internal/entity/models/ollama_test.go
@@ -17,9 +17,11 @@
package models
import (
+ "encoding/json"
"io"
"net/http"
"net/http/httptest"
+ "ragflow/internal/common"
"testing"
)
@@ -27,6 +29,113 @@ func newOllamaForListModelsTest(baseURL string) *OllamaModel {
return NewOllamaModel(map[string]string{"default": baseURL}, URLSuffix{Models: "api/tags"})
}
+func newOllamaForChatTest(baseURL string) *OllamaModel {
+ return NewOllamaModel(map[string]string{"default": baseURL}, URLSuffix{Chat: "api/chat", AsyncChat: "api/chat"})
+}
+
+func ollamaMultimodalTestMessages() []Message {
+ return []Message{{Role: "user", Content: []interface{}{
+ map[string]interface{}{"type": "text", "text": "describe"},
+ map[string]interface{}{"type": "image_url", "image_url": map[string]interface{}{"url": "data:image/png;base64,aGVsbG8="}},
+ map[string]interface{}{"type": "text", "text": "carefully"},
+ map[string]interface{}{"type": "image_url", "image_url": map[string]interface{}{"url": "cmF3LWltYWdl"}},
+ map[string]interface{}{"type": "image_url", "image_url": map[string]interface{}{"url": "https://example.com/cat.png"}},
+ }}}
+}
+
+func assertOllamaMultimodalRequest(t *testing.T, body map[string]interface{}, stream bool) {
+ t.Helper()
+ if body["stream"] != stream {
+ t.Errorf("stream=%v, want %v", body["stream"], stream)
+ }
+ messages, ok := body["messages"].([]interface{})
+ if !ok || len(messages) != 1 {
+ t.Errorf("messages=%v, want one message", body["messages"])
+ return
+ }
+ message, ok := messages[0].(map[string]interface{})
+ if !ok {
+ t.Errorf("message=%v, want object", messages[0])
+ return
+ }
+ if message["content"] != "describe\ncarefully" {
+ t.Errorf("content=%v, want joined text parts", message["content"])
+ }
+ images, ok := message["images"].([]interface{})
+ if !ok {
+ t.Errorf("images=%v, want array", message["images"])
+ return
+ }
+ want := []string{"aGVsbG8=", "cmF3LWltYWdl", "https://example.com/cat.png"}
+ if len(images) != len(want) {
+ t.Errorf("images=%v, want %v", images, want)
+ return
+ }
+ for i, expected := range want {
+ if images[i] != expected {
+ t.Errorf("images[%d]=%v, want %q", i, images[i], expected)
+ }
+ }
+}
+
+func TestOllamaChatMapsMultimodalImages(t *testing.T) {
+ withSSRFBypass(t)
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var body map[string]interface{}
+ if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+ t.Errorf("decode request: %v", err)
+ http.Error(w, "invalid request", http.StatusBadRequest)
+ return
+ }
+ assertOllamaMultimodalRequest(t, body, false)
+ _, _ = io.WriteString(w, `{"message":{"content":"ok","thinking":""}}`)
+ }))
+ defer srv.Close()
+
+ response, err := newOllamaForChatTest(srv.URL).ChatWithMessages(
+ t.Context(),
+ "llava",
+ ollamaMultimodalTestMessages(),
+ &APIConfig{},
+ &ChatConfig{},
+ &common.ModelUsage{},
+ )
+ if err != nil {
+ t.Fatalf("ChatWithMessages: %v", err)
+ }
+ if response.Answer == nil || *response.Answer != "ok" {
+ t.Fatalf("answer=%v, want ok", response.Answer)
+ }
+}
+
+func TestOllamaStreamingChatMapsMultimodalImages(t *testing.T) {
+ withSSRFBypass(t)
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var body map[string]interface{}
+ if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+ t.Errorf("decode request: %v", err)
+ http.Error(w, "invalid request", http.StatusBadRequest)
+ return
+ }
+ assertOllamaMultimodalRequest(t, body, true)
+ _, _ = io.WriteString(w, "{\"message\":{\"content\":\"ok\"},\"done\":false}\n{\"done\":true}\n")
+ }))
+ defer srv.Close()
+
+ err := newOllamaForChatTest(srv.URL).ChatStreamlyWithSender(
+ t.Context(),
+ "llava",
+ ollamaMultimodalTestMessages(),
+ &APIConfig{},
+ &ChatConfig{},
+ &common.ModelUsage{},
+ func(*string, *string) error { return nil },
+ )
+ if err != nil {
+ t.Fatalf("ChatStreamlyWithSender: %v", err)
+ }
+}
+
func TestOllamaListModels(t *testing.T) {
withSSRFBypass(t)
ctx := t.Context()