Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
Support image input for OpenAI chat compatibility (#5208)
royjhane9f7f3602961
diff --git a/openai/openai.go b/openai/openai.goindex 1707da14b..b289d73e8 100644--- a/openai/openai.go+++ b/openai/openai.go@@ -3,11 +3,13 @@ package openaiimport ("bytes"+ "encoding/base64""encoding/json""fmt""io""math/rand""net/http"+ "strings""time""github.com/gin-gonic/gin"@@ -28,7 +30,7 @@ type ErrorResponse struct {type Message struct {Role string `json:"role"`- Content string `json:"content"`+ Content any `json:"content"`}type Choice struct {@@ -269,10 +271,66 @@ func toModel(r api.ShowResponse, m string) Model {}}-func fromChatRequest(r ChatCompletionRequest) api.ChatRequest {+func fromChatRequest(r ChatCompletionRequest) (*api.ChatRequest, error) {var messages []api.Messagefor _, msg := range r.Messages {- messages = append(messages, api.Message{Role: msg.Role, Content: msg.Content})+ switch content := msg.Content.(type) {+ case string:+ messages = append(messages, api.Message{Role: msg.Role, Content: content})+ case []any:+ message := api.Message{Role: msg.Role}+ for _, c := range content {+ data, ok := c.(map[string]any)+ if !ok {+ return nil, fmt.Errorf("invalid message format")+ }+ switch data["type"] {+ case "text":+ text, ok := data["text"].(string)+ if !ok {+ return nil, fmt.Errorf("invalid message format")+ }+ message.Content = text+ case "image_url":+ var url string+ if urlMap, ok := data["image_url"].(map[string]any); ok {+ if url, ok = urlMap["url"].(string); !ok {+ return nil, fmt.Errorf("invalid message format")+ }+ } else {+ if url, ok = data["image_url"].(string); !ok {+ return nil, fmt.Errorf("invalid message format")+ }+ }++ types := []string{"jpeg", "jpg", "png"}+ valid := false+ for _, t := range types {+ prefix := "data:image/" + t + ";base64,"+ if strings.HasPrefix(url, prefix) {+ url = strings.TrimPrefix(url, prefix)+ valid = true+ break+ }+ }++ if !valid {+ return nil, fmt.Errorf("invalid image input")+ }++ img, err := base64.StdEncoding.DecodeString(url)+ if err != nil {+ return nil, fmt.Errorf("invalid message format")+ }+ message.Images = append(message.Images, img)+ default:+ return nil, fmt.Errorf("invalid message format")+ }+ }+ messages = append(messages, message)+ default:+ return nil, fmt.Errorf("invalid message content type: %T", content)+ }}options := make(map[string]interface{})@@ -323,13 +381,13 @@ func fromChatRequest(r ChatCompletionRequest) api.ChatRequest {format = "json"}- return api.ChatRequest{+ return &api.ChatRequest{Model: r.Model,Messages: messages,Format: format,Options: options,Stream: &r.Stream,- }+ }, nil}func fromCompleteRequest(r CompletionRequest) (api.GenerateRequest, error) {@@ -656,7 +714,13 @@ func ChatMiddleware() gin.HandlerFunc {}var b bytes.Buffer- if err := json.NewEncoder(&b).Encode(fromChatRequest(req)); err != nil {++ chatReq, err := fromChatRequest(req)+ if err != nil {+ c.AbortWithStatusJSON(http.StatusBadRequest, NewError(http.StatusBadRequest, err.Error()))+ }++ if err := json.NewEncoder(&b).Encode(chatReq); err != nil {c.AbortWithStatusJSON(http.StatusInternalServerError, NewError(http.StatusInternalServerError, err.Error()))return}diff --git a/openai/openai_test.go b/openai/openai_test.goindex 5f1ae52e9..99f8baaf3 100644--- a/openai/openai_test.go+++ b/openai/openai_test.go@@ -2,6 +2,7 @@ package openaiimport ("bytes"+ "encoding/base64""encoding/json""io""net/http"@@ -15,6 +16,10 @@ import ("github.com/stretchr/testify/assert")+const prefix = `data:image/jpeg;base64,`+const image = `iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=`+const imageURL = prefix + image+func TestMiddlewareRequests(t *testing.T) {type testCase struct {Name string@@ -112,6 +117,50 @@ func TestMiddlewareRequests(t *testing.T) {}},},+ {+ Name: "chat handler with image content",+ Method: http.MethodPost,+ Path: "/api/chat",+ Handler: ChatMiddleware,+ Setup: func(t *testing.T, req *http.Request) {+ body := ChatCompletionRequest{+ Model: "test-model",+ Messages: []Message{+ {+ Role: "user", Content: []map[string]any{+ {"type": "text", "text": "Hello"},+ {"type": "image_url", "image_url": map[string]string{"url": imageURL}},+ },+ },+ },+ }++ bodyBytes, _ := json.Marshal(body)++ req.Body = io.NopCloser(bytes.NewReader(bodyBytes))+ req.Header.Set("Content-Type", "application/json")+ },+ Expected: func(t *testing.T, req *http.Request) {+ var chatReq api.ChatRequest+ if err := json.NewDecoder(req.Body).Decode(&chatReq); err != nil {+ t.Fatal(err)+ }++ if chatReq.Messages[0].Role != "user" {+ t.Fatalf("expected 'user', got %s", chatReq.Messages[0].Role)+ }++ if chatReq.Messages[0].Content != "Hello" {+ t.Fatalf("expected 'Hello', got %s", chatReq.Messages[0].Content)+ }++ img, _ := base64.StdEncoding.DecodeString(imageURL[len(prefix):])++ if !bytes.Equal(chatReq.Messages[0].Images[0], img) {+ t.Fatalf("expected image encoding, got %s", chatReq.Messages[0].Images[0])+ }+ },+ },}gin.SetMode(gin.TestMode)