Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

openai: keep image positions when merging tool result parts

Dong Chenbea35c898464
diff --git a/openai/openai.go b/openai/openai.go
index a3cc19a65..8a8550977 100644
--- a/openai/openai.go
+++ b/openai/openai.go
@@ -684,11 +684,12 @@ func FromChatRequest(r ChatCompletionRequest, thinking ...*model.Thinking) (*api
}
}
// a tool message is a single result, so keep its parts together and
- // preserve the tool call it answers
+ // preserve the tool call it answers. an [img] placeholder keeps each
+ // image at its position among the text parts
if strings.ToLower(msg.Role) == "tool" && (len(messages) > start || toolName != "" || msg.ToolCallID != "") {
tool := api.Message{Role: msg.Role, ToolName: toolName, ToolCallID: msg.ToolCallID}
for _, part := range messages[start:] {
- tool.Content += part.Content
+ tool.Content += part.Content + strings.Repeat("[img]", len(part.Images))
tool.Images = append(tool.Images, part.Images...)
}
messages = append(messages[:start], tool)
diff --git a/openai/openai_test.go b/openai/openai_test.go
index 6d72a9907..1b9ec1e7c 100644
--- a/openai/openai_test.go
+++ b/openai/openai_test.go
@@ -158,7 +158,9 @@ func TestFromChatRequest_WithImage(t *testing.T) {
}
func TestFromChatRequest_ToolMessageArrayContent(t *testing.T) {
+ const image2 = `iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==`
imgData, _ := base64.StdEncoding.DecodeString(image)
+ img2Data, _ := base64.StdEncoding.DecodeString(image2)
req := ChatCompletionRequest{
Model: "test-model",
@@ -179,12 +181,16 @@ func TestFromChatRequest_ToolMessageArrayContent(t *testing.T) {
Role: "tool",
ToolCallID: "call_1",
Content: []any{
- map[string]any{"type": "text", "text": "Captured "},
+ map[string]any{"type": "text", "text": "Before: "},
map[string]any{
"type": "image_url",
"image_url": map[string]any{"url": prefix + image},
},
- map[string]any{"type": "text", "text": "screen.png"},
+ map[string]any{"type": "text", "text": " After: "},
+ map[string]any{
+ "type": "image_url",
+ "image_url": map[string]any{"url": prefix + image2},
+ },
},
},
},
@@ -203,11 +209,12 @@ func TestFromChatRequest_ToolMessageArrayContent(t *testing.T) {
if tool.Role != "tool" || tool.ToolCallID != "call_1" || tool.ToolName != "screenshot" {
t.Errorf("expected tool message for call_1/screenshot, got role=%q id=%q name=%q", tool.Role, tool.ToolCallID, tool.ToolName)
}
- if tool.Content != "Captured screen.png" {
- t.Errorf("expected content 'Captured screen.png', got %q", tool.Content)
+ // each image keeps its position among the text parts
+ if tool.Content != "Before: [img] After: [img]" {
+ t.Errorf("expected content 'Before: [img] After: [img]', got %q", tool.Content)
}
- if len(tool.Images) != 1 || string(tool.Images[0]) != string(imgData) {
- t.Errorf("expected the tool image to be kept, got %d images", len(tool.Images))
+ if len(tool.Images) != 2 || string(tool.Images[0]) != string(imgData) || string(tool.Images[1]) != string(img2Data) {
+ t.Errorf("expected both tool images to be kept in order, got %d images", len(tool.Images))
}
}