Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(embed): send images when an OpenAI-compatible row declares them (#4389)
JSJ1897705f77be962
diff --git a/internal/models/embedding/image_wire_test.go b/internal/models/embedding/image_wire_test.goindex 48d2e6b31..62444145f 100644--- a/internal/models/embedding/image_wire_test.go+++ b/internal/models/embedding/image_wire_test.go@@ -154,6 +154,8 @@ func TestImageEmbeddingNeedsBothTheModelAndTheEndpoint(t *testing.T) {{"text model on the OpenAI shape", "openai", "text-embedding-3-small", nil, false},{"generic endpoint, image input not declared", "generic", "my-clip", nil, false},{"declared on a generic (vLLM) endpoint", "generic", "my-clip", declare, true},+ {"declared on the OpenAI provider", "openai", "my-clip", declare, true},+ {"declared on LiteLLM", "litellm", "my-clip", declare, true},{"declared, but the row switched to an object format without a field", "generic", "my-clip",&types.ModelSpecOverride{@@ -188,6 +190,35 @@ func TestImageEmbeddingNeedsBothTheModelAndTheEndpoint(t *testing.T) {// image both pass the chat template and the prompt ends in <|endoftext|>,// the token its last-token pooling reads. A plain `input` string skips the// template and lands in a different space from the images.+// An OpenAI-compatible gateway has no image field of its own. Declaring+// image input, and nothing else, must still send the picture: vLLM takes it+// as a messages conversation. The text request stays on input.+func TestDeclaredImageInputOnOpenAIGatewayUsesMessages(t *testing.T) {+ up := newUpstream(t)+ declare := &types.ModelSpecOverride{Input: []string{"text", "image"}}+ e := imageEmbedder(t, up, "openai", "WeMM-Embedding-2B", "/v1", declare)+ ie, ok := AsImageEmbedder(e)+ require.True(t, ok)++ _, err := e.BatchEmbed(context.Background(), []string{"bridge"})+ require.NoError(t, err)+ images, err := ie.BatchEmbedImages(context.Background(), pngs(1))+ require.NoError(t, err)+ assert.Equal(t, [][]float32{{1}}, images)++ require.Len(t, up.requests, 2)+ assert.Equal(t, map[string]any{+ "model": "WeMM-Embedding-2B", "encoding_format": "float",+ "input": []any{"bridge"},+ }, up.requests[0].body)+ assert.Equal(t, map[string]any{+ "model": "WeMM-Embedding-2B", "encoding_format": "float",+ "messages": []any{map[string]any{"role": "user", "content": []any{+ map[string]any{"type": "image_url", "image_url": map[string]any{"url": pngs(1)[0].DataURI()}},+ }}},+ }, up.requests[1].body)+}+func TestSelfHostedQwen3VLEmbeddingSendsTextAndImagesThroughTheTemplate(t *testing.T) {up := newUpstream(t)e := imageEmbedder(t, up, "generic", "Qwen/Qwen3-VL-Embedding-8B", "/v1", nil)diff --git a/internal/models/runtime/resolve.go b/internal/models/runtime/resolve.goindex 0a721d063..04534736a 100644--- a/internal/models/runtime/resolve.go+++ b/internal/models/runtime/resolve.go@@ -492,6 +492,17 @@ func resolveEmbeddings("catalog: unknown embedding image_format %q on %s/%s (expected %q or %q)",settings.ImageFormat, vendor.ID, spec.ID, api.EmbeddingImageObject, api.EmbeddingImageMessages)}+ // A row that declares image input on the OpenAI embedding shape still+ // needs a wire format: that shape has none of its own, and only Jina+ // names one. vLLM, the usual server behind these gateways, takes one+ // chat conversation in messages. Jina and SGLang stay explicit+ // (image_format=object plus image_field). An object format without a+ // field is left alone, so a half-filled row is refused instead of resent+ // as messages. Text stays on input unless the row sets text_as_messages.+ if protocol == api.EmbeddingOpenAI && spec.AcceptsImages() &&+ settings.ImageFormat == "" && settings.ImageField == "" {+ settings.ImageFormat = api.EmbeddingImageMessages+ }out := &Resolved{Vendor: configcopy.Clone(vendor.Definition),diff --git a/website-docs/03-features/06-models.md b/website-docs/03-features/06-models.mdindex 89633da8e..ef1e4a810 100644--- a/website-docs/03-features/06-models.md+++ b/website-docs/03-features/06-models.md@@ -89,7 +89,7 @@| 需要附带厂商私有参数,如阿里云联网搜索 | `{"extra_body": {"enable_search": true}}` || 多轮对话回传思考内容时报错 | `{"replay_reasoning_content": false}` || 自建向量服务一次请求只接受 16 条 | `{"max_batch_size": 16}` |-| vLLM 部署的多模态向量模型(VLM2Vec、GME 等) | 打开「图片输入」即可,自定义厂商默认按 vLLM 的 `messages` 格式发图片;模板需要指令时加 `{"image_prompt": "Represent the given image."}`;文本也必须经过对话模板才和图片同空间时加 `{"text_as_messages": true}` |+| vLLM 部署的多模态向量模型(VLM2Vec、GME 等) | 打开「图片输入」即可。未填 `image_format` 和 `image_field` 时,OpenAI 兼容协议按 vLLM 的 `messages` 格式发图片。模板需要指令时加 `{"image_prompt": "Represent the given image."}`;文本也必须经过对话模板才和图片同空间时加 `{"text_as_messages": true}` || vLLM 部署的 Qwen3-VL-Embedding | 无需填写:自定义厂商下模型名含 `qwen3-vl-embedding` 时已内置图片输入与 `{"text_as_messages": true, "extra_body": {"add_special_tokens": true}}`。文本走普通 `input` 会绕过对话模板,与图片不在同一空间;不加 `add_special_tokens` 时末尾缺少模型取向量用的 `<\|endoftext\|>`,与[模型卡](https://huggingface.co/Qwen/Qwen3-VL-Embedding-8B)结果不符。已用该模型建立的索引需重建向量 || SGLang 部署的多模态向量模型 | 打开「图片输入」,并填 `{"image_format": "object", "image_field": "image"}` || 向量或重排服务响应慢,需要更长的超时 | `{"request_timeout_seconds": 120}` |@@ -235,8 +235,8 @@ WeMM 的模板已经包含所需特殊 token,保留 vLLM 的 `add_special_toke| `accepts_truncate_prompt_tokens` | bool | false | 服务是否支持 vLLM 的 `truncate_prompt_tokens` || `request_timeout_seconds` | int | 60 | 单次请求超时(秒) || `extra_body` | object | 空 | 附加到每次请求的字段 |-| `image_field` | string | 空 | 仅 `openai-embeddings`:`input` 里承载图片的对象键名(Jina、SGLang 为 `image`)。留空且 `image_format` 不是 `messages` 时表示该端点不收图片 |-| `image_format` | string | 空(自定义厂商为 `messages`) | 仅 `openai-embeddings`:图片的发送方式。`object` 把 `{image_field: 图片}` 放进 `input`;`messages` 把图片作为一轮对话放进 `messages`(vLLM 的多模态向量接口),每张图一个请求。留空时有 `image_field` 即按 `object` 处理 |+| `image_field` | string | 空 | 仅 `openai-embeddings`:`input` 里承载图片的对象键名(Jina、SGLang 为 `image`)。模型未声明图片输入、留空、且 `image_format` 不是 `messages` 时,端点不收图片 |+| `image_format` | string | 空 | 仅 `openai-embeddings`:图片的发送方式。`object` 把 `{image_field: 图片}` 放进 `input`;`messages` 把图片作为一轮对话放进 `messages`(vLLM 的多模态向量接口)。模型声明了图片输入、本字段和 `image_field` 都留空时,按 `messages` 发送。只填了 `image_field` 时按 `object` 处理。写成 `object` 却没有 `image_field` 时仍不发图片 || `image_prompt` | string | 空 | 仅 `messages` 格式:随图片一起发送的文字,部分模型的对话模板需要(如 VLM2Vec 的 `Represent the given image.`) || `text_as_messages` | bool | false | 仅 `openai-embeddings`:文本也按 `messages` 格式发送,每条一个请求。适用于必须经过对话模板、文本和图片才在同一向量空间的模型(自定义厂商下的 Qwen3-VL-Embedding 已内置);已有索引的模型改动后需要重建向量 || `max_image_batch_size` | int | 1 | 单次请求的最大图片数,超出自动分批 |