Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
gateway: a prompt's inline image comes to the text view as a picture it can show
yetone069bf0583b66
diff --git a/internal/gateway/prompt.go b/internal/gateway/prompt.goindex c80fe0805..609115083 100644--- a/internal/gateway/prompt.go+++ b/internal/gateway/prompt.go@@ -108,7 +108,8 @@ type promptBuilder struct {// PromptPiece is a stretch of an item's text as the request sent it: a// message, a tool's definition, a call and its input, what a tool-// answered. An image or sealed reasoning is only named, with its size.+// answered. Sealed reasoning is only named, with its size; an image is+// its picture when the request carried it inline (Src).type PromptPiece struct {// Role: system, user, assistant, thinking, call, result, tool, image,// sealed; "" for text that is the item itself (an instruction file)@@ -116,7 +117,11 @@ type PromptPiece struct {Name string `json:"name,omitempty"` // the tool called or definedText string `json:"text,omitempty"`Size int `json:"size,omitempty"` // a sealed piece's bytes- raw []byte // JSON (or a custom tool's text), Text once read out+ // Src: an image's picture as a data: URL, when the request sent it+ // inline as a PNG, JPEG, GIF or WebP; a linked image stays a name+ Src string `json:"src,omitempty"`+ raw []byte // JSON (or a custom tool's text), Text once read out+ img []byte // an image's source as the request sent it, Src once read out}func txt(role, s string) PromptPiece { return PromptPiece{Role: role, Text: s} }@@ -125,7 +130,9 @@ func rawPiece(role, name string, raw []byte) PromptPiece {return PromptPiece{Role: role, Name: name, raw: raw}}-var imagePiece = PromptPiece{Role: "image"}+// imgPiece is an image, its source (an Anthropic source, an image_url,+// Gemini's inlineData) kept to be read out as its picture.+func imgPiece(src []byte) PromptPiece { return PromptPiece{Role: "image", img: src} }func sealedPiece(n int) PromptPiece { return PromptPiece{Role: "sealed", Size: n} }@@ -714,7 +721,7 @@ func (b *promptBuilder) anthropicBlock(role string, x rawBlock) {for _, c := range blocksOf(x.Content) {switch c.Type {case "image", "document":- b.add(PartFiles, c.Type, "image", imageTokens, imagePiece)+ b.add(PartFiles, c.Type, "image", imageTokens, imgPiece(c.Source))default:n += tokensOf(c.Text)texts = append(texts, c.Text)@@ -731,7 +738,7 @@ func (b *promptBuilder) anthropicBlock(role string, x rawBlock) {b.add(PartFiles, "document", "image", rawTokens(x.Source), rawPiece("", "", x.Source))return}- b.add(PartFiles, x.Type, "image", imageTokens, imagePiece)+ b.add(PartFiles, x.Type, "image", imageTokens, imgPiece(x.Source))default:b.chat(tokensOf(x.Text), txt(role, x.Text))}@@ -853,7 +860,7 @@ func (b *promptBuilder) responsesItem(raw json.RawMessage) {}for _, p := range parts {if p.image {- b.add(PartFiles, "image", "image", imageTokens, imagePiece)+ b.add(PartFiles, "image", "image", imageTokens, imgPiece(p.src))continue}b.userText(p.text)@@ -897,7 +904,7 @@ func (b *promptBuilder) responsesItem(raw json.RawMessage) {var texts []stringfor _, p := range responsesContent(it.Output) {if p.image {- b.add(PartFiles, "image", "image", imageTokens, imagePiece)+ b.add(PartFiles, "image", "image", imageTokens, imgPiece(p.src))continue}n += tokensOf(p.text)@@ -921,6 +928,7 @@ func (b *promptBuilder) responsesItem(raw json.RawMessage) {type rPart struct {text stringimage bool+ src json.RawMessage // an image's image_url}func responsesContent(c json.RawMessage) []rPart {@@ -929,13 +937,14 @@ func responsesContent(c json.RawMessage) []rPart {return []rPart{{text: s}}}var ps []struct {- Type string `json:"type"`- Text string `json:"text"`+ Type string `json:"type"`+ Text string `json:"text"`+ ImageURL json.RawMessage `json:"image_url"`}_ = decodeJSON(c, &ps)out := make([]rPart, 0, len(ps))for _, p := range ps {- out = append(out, rPart{text: p.Text, image: p.Type == "input_image" || p.Type == "input_file" || p.Type == "image_url"})+ out = append(out, rPart{text: p.Text, image: p.Type == "input_image" || p.Type == "input_file" || p.Type == "image_url", src: p.ImageURL})}return out}@@ -978,7 +987,7 @@ func (b *promptBuilder) chatCompletions(body []byte) bool {}for _, p := range parts {if p.image {- b.add(PartFiles, "image", "image", imageTokens, imagePiece)+ b.add(PartFiles, "image", "image", imageTokens, imgPiece(p.src))continue}b.userText(p.text)@@ -988,7 +997,7 @@ func (b *promptBuilder) chatCompletions(body []byte) bool {var texts []stringfor _, p := range parts {if p.image {- b.add(PartFiles, "image", "image", imageTokens, imagePiece)+ b.add(PartFiles, "image", "image", imageTokens, imgPiece(p.src))continue}n += tokensOf(p.text)@@ -1086,7 +1095,7 @@ func (b *promptBuilder) gemini(body []byte) bool {}b.result(id, rawTokens(p.FunctionResponse.Response), rawPiece("result", "", p.FunctionResponse.Response))case len(p.InlineData) > 0 || len(p.FileData) > 0:- b.add(PartFiles, "image", "image", imageTokens, imagePiece)+ b.add(PartFiles, "image", "image", imageTokens, imgPiece(p.InlineData))case c.Role == "model":role := "assistant"if p.Thought {diff --git a/internal/gateway/prompt_text.go b/internal/gateway/prompt_text.goindex dd041b77f..08005829a 100644--- a/internal/gateway/prompt_text.go+++ b/internal/gateway/prompt_text.go@@ -166,6 +166,10 @@ func boolInt(b bool) int {// read is the piece with its JSON laid out to be read: indented, or a// string's text when the JSON is a string.func (pc PromptPiece) read() PromptPiece {+ if pc.img != nil {+ pc.Src = promptImageSrc(pc.img)+ pc.img = nil+ }if pc.raw == nil {return pc}@@ -192,6 +196,58 @@ func (pc PromptPiece) read() PromptPiece {return pc}+// promptImageSrc is an image's source as a data: URL the GUI can draw: an+// Anthropic source ({type: base64, media_type, data}), an image_url (a+// string, or {url}) or Gemini's inlineData ({mimeType, data}). "" for an+// image that is a link, a file id, or anything but base64 PNG, JPEG, GIF+// or WebP: the GUI never fetches a picture from elsewhere, and an SVG+// could carry script.+func promptImageSrc(raw []byte) string {+ var s string+ if json.Unmarshal(raw, &s) == nil {+ return dataImage(s)+ }+ var src struct {+ URL string `json:"url"`+ MediaType string `json:"media_type"`+ MimeType string `json:"mimeType"`+ Data string `json:"data"`+ }+ if json.Unmarshal(raw, &src) != nil {+ return ""+ }+ if src.URL != "" {+ return dataImage(src.URL)+ }+ mt := src.MediaType+ if mt == "" {+ mt = src.MimeType+ }+ return dataImage("data:" + mt + ";base64," + src.Data)+}++// dataImage is u when it is a base64 data: URL of a picture the GUI+// draws, else "".+func dataImage(u string) string {+ head, data, ok := strings.Cut(u, ",")+ if !ok || data == "" {+ return ""+ }+ switch strings.ToLower(head) {+ case "data:image/png;base64", "data:image/jpeg;base64", "data:image/jpg;base64",+ "data:image/gif;base64", "data:image/webp;base64":+ default:+ return ""+ }+ for i := 0; i < len(data); i++ {+ c := data[i]+ if !('A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' || c == '+' || c == '/' || c == '=') {+ return ""+ }+ }+ return u+}+// promptPieces reads a request's prompt keeping each item's text.func promptPieces(from provider.Protocol, body []byte) (b *promptBuilder, ok bool) {defer func() {diff --git a/internal/gateway/prompt_text_test.go b/internal/gateway/prompt_text_test.goindex 5cadebfbe..43fcb373b 100644--- a/internal/gateway/prompt_text_test.go+++ b/internal/gateway/prompt_text_test.go@@ -160,3 +160,63 @@ func TestPromptTextChatPieces(t *testing.T) {t.Fatalf("sizes differ")}}++// An image the request sent inline is shown as its picture in the text of+// the context window's Files (the owner: "这里要是能够显示图片就更牛逼了"),+// in each protocol's shape, a tool's result's image too. A linked image,+// an SVG or a source that isn't base64 stays a name: the GUI draws only+// what the request itself carried.+func TestPromptTextImagesArePictures(t *testing.T) {+ const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="+ const url = "data:image/png;base64," + png+ cases := []struct {+ from provider.Protocol+ body string+ want []string+ }{+ {provider.Anthropic, `{"messages":[{"role":"user","content":[+ {"type":"text","text":"look"},+ {"type":"image","source":{"type":"base64","media_type":"image/png","data":"` + png + `"}},+ {"type":"image","source":{"type":"url","url":"https://example.com/a.png"}},+ {"type":"image","source":{"type":"base64","media_type":"image/svg+xml","data":"PHN2Zz4="}}]},+ {"role":"assistant","content":[{"type":"tool_use","id":"c1","name":"Shot","input":{}}]},+ {"role":"user","content":[{"type":"tool_result","tool_use_id":"c1","content":[+ {"type":"image","source":{"type":"base64","media_type":"image/jpeg","data":"` + png + `"}}]}]}]}`,+ []string{url, "", "", "data:image/jpeg;base64," + png}},+ {provider.Responses, `{"input":[{"role":"user","content":[+ {"type":"input_text","text":"look"},+ {"type":"input_image","image_url":"` + url + `"},+ {"type":"input_image","image_url":"https://example.com/a.png"}]}]}`,+ []string{url, ""}},+ {provider.Chat, `{"messages":[{"role":"user","content":[+ {"type":"text","text":"look"},+ {"type":"image_url","image_url":{"url":"` + url + `"}},+ {"type":"image_url","image_url":{"url":"data:image/png;base64,<script>"}}]}]}`,+ []string{url, ""}},+ {provider.Gemini, `{"contents":[{"role":"user","parts":[+ {"text":"look"},+ {"inlineData":{"mimeType":"image/webp","data":"` + png + `"}},+ {"fileData":{"mimeType":"image/png","fileUri":"https://example.com/a.png"}}]}]}`,+ []string{"data:image/webp;base64," + png, ""}},+ }+ for _, c := range cases {+ b, ok := promptPieces(c.from, []byte(c.body))+ if !ok {+ t.Fatalf("%s: not read", c.from)+ }+ var got []string+ for key, pcs := range b.pieces {+ if !strings.HasPrefix(key, PartFiles+"\x00") {+ continue+ }+ for _, pc := range pcs {+ if pc = pc.read(); pc.Role == "image" {+ got = append(got, pc.Src)+ }+ }+ }+ if strings.Join(got, " | ") != strings.Join(c.want, " | ") {+ t.Fatalf("%s: srcs %q, want %q", c.from, got, c.want)+ }+ }+}