Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

gateway: a prompt's inline image comes to the text view as a picture it can show

yetone069bf0583b66
diff --git a/internal/gateway/prompt.go b/internal/gateway/prompt.go
index c80fe0805..609115083 100644
--- a/internal/gateway/prompt.go
+++ b/internal/gateway/prompt.go
@@ -108,7 +108,8 @@ type promptBuilder struct {
// PromptPiece is a stretch of an item's text as the request sent it: a
// message, a tool's definition, a call and its input, what a tool
-// answered. An image or sealed reasoning is only named, with its size.
+// answered. Sealed reasoning is only named, with its size; an image is
+// its picture when the request carried it inline (Src).
type PromptPiece struct {
// Role: system, user, assistant, thinking, call, result, tool, image,
// sealed; "" for text that is the item itself (an instruction file)
@@ -116,7 +117,11 @@ type PromptPiece struct {
Name string `json:"name,omitempty"` // the tool called or defined
Text string `json:"text,omitempty"`
Size int `json:"size,omitempty"` // a sealed piece's bytes
- raw []byte // JSON (or a custom tool's text), Text once read out
+ // Src: an image's picture as a data: URL, when the request sent it
+ // inline as a PNG, JPEG, GIF or WebP; a linked image stays a name
+ Src string `json:"src,omitempty"`
+ raw []byte // JSON (or a custom tool's text), Text once read out
+ img []byte // an image's source as the request sent it, Src once read out
}
func txt(role, s string) PromptPiece { return PromptPiece{Role: role, Text: s} }
@@ -125,7 +130,9 @@ func rawPiece(role, name string, raw []byte) PromptPiece {
return PromptPiece{Role: role, Name: name, raw: raw}
}
-var imagePiece = PromptPiece{Role: "image"}
+// imgPiece is an image, its source (an Anthropic source, an image_url,
+// Gemini's inlineData) kept to be read out as its picture.
+func imgPiece(src []byte) PromptPiece { return PromptPiece{Role: "image", img: src} }
func sealedPiece(n int) PromptPiece { return PromptPiece{Role: "sealed", Size: n} }
@@ -714,7 +721,7 @@ func (b *promptBuilder) anthropicBlock(role string, x rawBlock) {
for _, c := range blocksOf(x.Content) {
switch c.Type {
case "image", "document":
- b.add(PartFiles, c.Type, "image", imageTokens, imagePiece)
+ b.add(PartFiles, c.Type, "image", imageTokens, imgPiece(c.Source))
default:
n += tokensOf(c.Text)
texts = append(texts, c.Text)
@@ -731,7 +738,7 @@ func (b *promptBuilder) anthropicBlock(role string, x rawBlock) {
b.add(PartFiles, "document", "image", rawTokens(x.Source), rawPiece("", "", x.Source))
return
}
- b.add(PartFiles, x.Type, "image", imageTokens, imagePiece)
+ b.add(PartFiles, x.Type, "image", imageTokens, imgPiece(x.Source))
default:
b.chat(tokensOf(x.Text), txt(role, x.Text))
}
@@ -853,7 +860,7 @@ func (b *promptBuilder) responsesItem(raw json.RawMessage) {
}
for _, p := range parts {
if p.image {
- b.add(PartFiles, "image", "image", imageTokens, imagePiece)
+ b.add(PartFiles, "image", "image", imageTokens, imgPiece(p.src))
continue
}
b.userText(p.text)
@@ -897,7 +904,7 @@ func (b *promptBuilder) responsesItem(raw json.RawMessage) {
var texts []string
for _, p := range responsesContent(it.Output) {
if p.image {
- b.add(PartFiles, "image", "image", imageTokens, imagePiece)
+ b.add(PartFiles, "image", "image", imageTokens, imgPiece(p.src))
continue
}
n += tokensOf(p.text)
@@ -921,6 +928,7 @@ func (b *promptBuilder) responsesItem(raw json.RawMessage) {
type rPart struct {
text string
image bool
+ src json.RawMessage // an image's image_url
}
func responsesContent(c json.RawMessage) []rPart {
@@ -929,13 +937,14 @@ func responsesContent(c json.RawMessage) []rPart {
return []rPart{{text: s}}
}
var ps []struct {
- Type string `json:"type"`
- Text string `json:"text"`
+ Type string `json:"type"`
+ Text string `json:"text"`
+ ImageURL json.RawMessage `json:"image_url"`
}
_ = decodeJSON(c, &ps)
out := make([]rPart, 0, len(ps))
for _, p := range ps {
- out = append(out, rPart{text: p.Text, image: p.Type == "input_image" || p.Type == "input_file" || p.Type == "image_url"})
+ out = append(out, rPart{text: p.Text, image: p.Type == "input_image" || p.Type == "input_file" || p.Type == "image_url", src: p.ImageURL})
}
return out
}
@@ -978,7 +987,7 @@ func (b *promptBuilder) chatCompletions(body []byte) bool {
}
for _, p := range parts {
if p.image {
- b.add(PartFiles, "image", "image", imageTokens, imagePiece)
+ b.add(PartFiles, "image", "image", imageTokens, imgPiece(p.src))
continue
}
b.userText(p.text)
@@ -988,7 +997,7 @@ func (b *promptBuilder) chatCompletions(body []byte) bool {
var texts []string
for _, p := range parts {
if p.image {
- b.add(PartFiles, "image", "image", imageTokens, imagePiece)
+ b.add(PartFiles, "image", "image", imageTokens, imgPiece(p.src))
continue
}
n += tokensOf(p.text)
@@ -1086,7 +1095,7 @@ func (b *promptBuilder) gemini(body []byte) bool {
}
b.result(id, rawTokens(p.FunctionResponse.Response), rawPiece("result", "", p.FunctionResponse.Response))
case len(p.InlineData) > 0 || len(p.FileData) > 0:
- b.add(PartFiles, "image", "image", imageTokens, imagePiece)
+ b.add(PartFiles, "image", "image", imageTokens, imgPiece(p.InlineData))
case c.Role == "model":
role := "assistant"
if p.Thought {
diff --git a/internal/gateway/prompt_text.go b/internal/gateway/prompt_text.go
index dd041b77f..08005829a 100644
--- a/internal/gateway/prompt_text.go
+++ b/internal/gateway/prompt_text.go
@@ -166,6 +166,10 @@ func boolInt(b bool) int {
// read is the piece with its JSON laid out to be read: indented, or a
// string's text when the JSON is a string.
func (pc PromptPiece) read() PromptPiece {
+ if pc.img != nil {
+ pc.Src = promptImageSrc(pc.img)
+ pc.img = nil
+ }
if pc.raw == nil {
return pc
}
@@ -192,6 +196,58 @@ func (pc PromptPiece) read() PromptPiece {
return pc
}
+// promptImageSrc is an image's source as a data: URL the GUI can draw: an
+// Anthropic source ({type: base64, media_type, data}), an image_url (a
+// string, or {url}) or Gemini's inlineData ({mimeType, data}). "" for an
+// image that is a link, a file id, or anything but base64 PNG, JPEG, GIF
+// or WebP: the GUI never fetches a picture from elsewhere, and an SVG
+// could carry script.
+func promptImageSrc(raw []byte) string {
+ var s string
+ if json.Unmarshal(raw, &s) == nil {
+ return dataImage(s)
+ }
+ var src struct {
+ URL string `json:"url"`
+ MediaType string `json:"media_type"`
+ MimeType string `json:"mimeType"`
+ Data string `json:"data"`
+ }
+ if json.Unmarshal(raw, &src) != nil {
+ return ""
+ }
+ if src.URL != "" {
+ return dataImage(src.URL)
+ }
+ mt := src.MediaType
+ if mt == "" {
+ mt = src.MimeType
+ }
+ return dataImage("data:" + mt + ";base64," + src.Data)
+}
+
+// dataImage is u when it is a base64 data: URL of a picture the GUI
+// draws, else "".
+func dataImage(u string) string {
+ head, data, ok := strings.Cut(u, ",")
+ if !ok || data == "" {
+ return ""
+ }
+ switch strings.ToLower(head) {
+ case "data:image/png;base64", "data:image/jpeg;base64", "data:image/jpg;base64",
+ "data:image/gif;base64", "data:image/webp;base64":
+ default:
+ return ""
+ }
+ for i := 0; i < len(data); i++ {
+ c := data[i]
+ if !('A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' || c == '+' || c == '/' || c == '=') {
+ return ""
+ }
+ }
+ return u
+}
+
// promptPieces reads a request's prompt keeping each item's text.
func promptPieces(from provider.Protocol, body []byte) (b *promptBuilder, ok bool) {
defer func() {
diff --git a/internal/gateway/prompt_text_test.go b/internal/gateway/prompt_text_test.go
index 5cadebfbe..43fcb373b 100644
--- a/internal/gateway/prompt_text_test.go
+++ b/internal/gateway/prompt_text_test.go
@@ -160,3 +160,63 @@ func TestPromptTextChatPieces(t *testing.T) {
t.Fatalf("sizes differ")
}
}
+
+// An image the request sent inline is shown as its picture in the text of
+// the context window's Files (the owner: "这里要是能够显示图片就更牛逼了"),
+// in each protocol's shape, a tool's result's image too. A linked image,
+// an SVG or a source that isn't base64 stays a name: the GUI draws only
+// what the request itself carried.
+func TestPromptTextImagesArePictures(t *testing.T) {
+ const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
+ const url = "data:image/png;base64," + png
+ cases := []struct {
+ from provider.Protocol
+ body string
+ want []string
+ }{
+ {provider.Anthropic, `{"messages":[{"role":"user","content":[
+ {"type":"text","text":"look"},
+ {"type":"image","source":{"type":"base64","media_type":"image/png","data":"` + png + `"}},
+ {"type":"image","source":{"type":"url","url":"https://example.com/a.png"}},
+ {"type":"image","source":{"type":"base64","media_type":"image/svg+xml","data":"PHN2Zz4="}}]},
+ {"role":"assistant","content":[{"type":"tool_use","id":"c1","name":"Shot","input":{}}]},
+ {"role":"user","content":[{"type":"tool_result","tool_use_id":"c1","content":[
+ {"type":"image","source":{"type":"base64","media_type":"image/jpeg","data":"` + png + `"}}]}]}]}`,
+ []string{url, "", "", "data:image/jpeg;base64," + png}},
+ {provider.Responses, `{"input":[{"role":"user","content":[
+ {"type":"input_text","text":"look"},
+ {"type":"input_image","image_url":"` + url + `"},
+ {"type":"input_image","image_url":"https://example.com/a.png"}]}]}`,
+ []string{url, ""}},
+ {provider.Chat, `{"messages":[{"role":"user","content":[
+ {"type":"text","text":"look"},
+ {"type":"image_url","image_url":{"url":"` + url + `"}},
+ {"type":"image_url","image_url":{"url":"data:image/png;base64,<script>"}}]}]}`,
+ []string{url, ""}},
+ {provider.Gemini, `{"contents":[{"role":"user","parts":[
+ {"text":"look"},
+ {"inlineData":{"mimeType":"image/webp","data":"` + png + `"}},
+ {"fileData":{"mimeType":"image/png","fileUri":"https://example.com/a.png"}}]}]}`,
+ []string{"data:image/webp;base64," + png, ""}},
+ }
+ for _, c := range cases {
+ b, ok := promptPieces(c.from, []byte(c.body))
+ if !ok {
+ t.Fatalf("%s: not read", c.from)
+ }
+ var got []string
+ for key, pcs := range b.pieces {
+ if !strings.HasPrefix(key, PartFiles+"\x00") {
+ continue
+ }
+ for _, pc := range pcs {
+ if pc = pc.read(); pc.Role == "image" {
+ got = append(got, pc.Src)
+ }
+ }
+ }
+ if strings.Join(got, " | ") != strings.Join(c.want, " | ") {
+ t.Fatalf("%s: srcs %q, want %q", c.from, got, c.want)
+ }
+ }
+}