Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

feat(models): add WeMM embedding presets and Transformers serving (#4238)

lyingbug94fae6de43a3
diff --git a/internal/models/catalog/data/models.generated.json b/internal/models/catalog/data/models.generated.json
index 1851b122a..59f8631a3 100644
--- a/internal/models/catalog/data/models.generated.json
+++ b/internal/models/catalog/data/models.generated.json
@@ -2185,6 +2185,75 @@
"add_special_tokens": true
}
}
+ },
+ {
+ "id": "tencent/WeMM-Embedding-2B",
+ "match": "*/WeMM-Embedding-2B",
+ "type": "Embedding",
+ "name": "WeMM-Embedding-2B",
+ "aliases": [
+ "WeMM-Embedding-2B"
+ ],
+ "input": [
+ "text",
+ "image"
+ ],
+ "dimension": 2048,
+ "source": "https://huggingface.co/tencent/WeMM-Embedding-2B",
+ "compat": {
+ "api": "openai-embeddings",
+ "image_format": "messages",
+ "text_as_messages": true,
+ "batch_messages": true,
+ "accepts_truncate_prompt_tokens": false,
+ "max_image_batch_size": 5
+ }
+ },
+ {
+ "id": "tencent/WeMM-Embedding-4B",
+ "match": "*/WeMM-Embedding-4B",
+ "type": "Embedding",
+ "name": "WeMM-Embedding-4B",
+ "aliases": [
+ "WeMM-Embedding-4B"
+ ],
+ "input": [
+ "text",
+ "image"
+ ],
+ "dimension": 2560,
+ "source": "https://huggingface.co/tencent/WeMM-Embedding-4B",
+ "compat": {
+ "api": "openai-embeddings",
+ "image_format": "messages",
+ "text_as_messages": true,
+ "batch_messages": true,
+ "accepts_truncate_prompt_tokens": false,
+ "max_image_batch_size": 5
+ }
+ },
+ {
+ "id": "tencent/WeMM-Embedding-9B",
+ "match": "*/WeMM-Embedding-9B",
+ "type": "Embedding",
+ "name": "WeMM-Embedding-9B",
+ "aliases": [
+ "WeMM-Embedding-9B"
+ ],
+ "input": [
+ "text",
+ "image"
+ ],
+ "dimension": 4096,
+ "source": "https://huggingface.co/tencent/WeMM-Embedding-9B",
+ "compat": {
+ "api": "openai-embeddings",
+ "image_format": "messages",
+ "text_as_messages": true,
+ "batch_messages": true,
+ "accepts_truncate_prompt_tokens": false,
+ "max_image_batch_size": 5
+ }
}
],
"gpustack": [],
diff --git a/internal/models/catalog/data/overrides.json b/internal/models/catalog/data/overrides.json
index 1123f6600..fd49c67ab 100644
--- a/internal/models/catalog/data/overrides.json
+++ b/internal/models/catalog/data/overrides.json
@@ -1050,6 +1050,45 @@
"add_special_tokens": true
}
}
+ },
+ {
+ "id": "tencent/WeMM-Embedding-2B",
+ "match": "*/WeMM-Embedding-2B",
+ "type": "Embedding",
+ "compat": {
+ "api": "openai-embeddings",
+ "image_format": "messages",
+ "text_as_messages": true,
+ "batch_messages": true,
+ "accepts_truncate_prompt_tokens": false,
+ "max_image_batch_size": 5
+ }
+ },
+ {
+ "id": "tencent/WeMM-Embedding-4B",
+ "match": "*/WeMM-Embedding-4B",
+ "type": "Embedding",
+ "compat": {
+ "api": "openai-embeddings",
+ "image_format": "messages",
+ "text_as_messages": true,
+ "batch_messages": true,
+ "accepts_truncate_prompt_tokens": false,
+ "max_image_batch_size": 5
+ }
+ },
+ {
+ "id": "tencent/WeMM-Embedding-9B",
+ "match": "*/WeMM-Embedding-9B",
+ "type": "Embedding",
+ "compat": {
+ "api": "openai-embeddings",
+ "image_format": "messages",
+ "text_as_messages": true,
+ "batch_messages": true,
+ "accepts_truncate_prompt_tokens": false,
+ "max_image_batch_size": 5
+ }
}
],
"hunyuan": [
diff --git a/internal/models/catalog/data/seed.json b/internal/models/catalog/data/seed.json
index a2ca16392..02b4b0c55 100644
--- a/internal/models/catalog/data/seed.json
+++ b/internal/models/catalog/data/seed.json
@@ -1454,6 +1454,51 @@
"image"
],
"source": "https://huggingface.co/Qwen/Qwen3-VL-Embedding-8B"
+ },
+ {
+ "id": "tencent/WeMM-Embedding-2B",
+ "match": "*/WeMM-Embedding-2B",
+ "type": "Embedding",
+ "name": "WeMM-Embedding-2B",
+ "aliases": [
+ "WeMM-Embedding-2B"
+ ],
+ "input": [
+ "text",
+ "image"
+ ],
+ "dimension": 2048,
+ "source": "https://huggingface.co/tencent/WeMM-Embedding-2B"
+ },
+ {
+ "id": "tencent/WeMM-Embedding-4B",
+ "match": "*/WeMM-Embedding-4B",
+ "type": "Embedding",
+ "name": "WeMM-Embedding-4B",
+ "aliases": [
+ "WeMM-Embedding-4B"
+ ],
+ "input": [
+ "text",
+ "image"
+ ],
+ "dimension": 2560,
+ "source": "https://huggingface.co/tencent/WeMM-Embedding-4B"
+ },
+ {
+ "id": "tencent/WeMM-Embedding-9B",
+ "match": "*/WeMM-Embedding-9B",
+ "type": "Embedding",
+ "name": "WeMM-Embedding-9B",
+ "aliases": [
+ "WeMM-Embedding-9B"
+ ],
+ "input": [
+ "text",
+ "image"
+ ],
+ "dimension": 4096,
+ "source": "https://huggingface.co/tencent/WeMM-Embedding-9B"
}
],
"gpustack": [],
diff --git a/internal/models/embedding/wemm_test.go b/internal/models/embedding/wemm_test.go
new file mode 100644
index 000000000..012c2b799
--- /dev/null
+++ b/internal/models/embedding/wemm_test.go
@@ -0,0 +1,127 @@
+package embedding
+
+import (
+ "context"
+ "strings"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/models/api"
+ modelruntime "github.com/Tencent/WeKnora/internal/models/runtime"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/stretchr/testify/require"
+)
+
+// WeMM's vLLM template appends the dedicated <embedding> pooling token.
+// Both retrieval sides and images must use that template, including when
+// vLLM exposes the local model path instead of the Hugging Face repository ID.
+// https://huggingface.co/tencent/WeMM-Embedding-2B/blob/main/embedding_chat_template.jinja
+func TestWeMMVLLMPresets(t *testing.T) {
+ vendor, ok := modelruntime.Get("generic")
+ require.True(t, ok)
+ listed := vendor.ModelsByType(types.ModelTypeEmbedding)
+ for _, tc := range []struct {
+ size string
+ dims int
+ }{{"2B", 2048}, {"4B", 2560}, {"9B", 4096}} {
+ short := "WeMM-Embedding-" + tc.size
+ id := "tencent/" + short
+ t.Run(tc.size, func(t *testing.T) {
+ visible := false
+ for _, entry := range listed {
+ if entry.ID == id {
+ visible = true
+ require.Equal(t, tc.dims, entry.Dimension)
+ }
+ }
+ require.True(t, visible, "preset must appear in the embedding picker")
+ for _, name := range []string{id, short, "/models/" + short, strings.ToLower(id)} {
+ t.Run(name, func(t *testing.T) {
+ resolved, err := modelruntime.Resolve(modelruntime.Ref{
+ Provider: "generic", Model: name, ModelType: types.ModelTypeEmbedding,
+ })
+ require.NoError(t, err)
+ require.True(t, resolved.Cataloged)
+ require.Equal(t, tc.dims, resolved.Spec.Dimension)
+ require.Equal(t, api.EmbeddingOpenAI, resolved.EmbeddingAPI)
+ require.Equal(t, []string{"text", "image"}, resolved.Spec.Input)
+ require.True(t, resolved.Embeddings.BatchMessages)
+ require.Equal(t, 5, resolved.Embeddings.ImageBatchLimit())
+
+ up := newUpstream(t)
+ e, err := NewEmbedder(Config{
+ Source: types.ModelSourceRemote, Provider: "generic", ModelName: name,
+ BaseURL: up.url + "/v1", Dimensions: tc.dims, TruncatePromptTokens: 511,
+ }, nil, nil)
+ require.NoError(t, err)
+ require.Equal(t, tc.dims, e.GetDimensions())
+ ie, ok := AsImageEmbedder(e)
+ require.True(t, ok, "images must work without a row override")
+ contexts := []context.Context{context.Background(), types.WithEmbedQuery(context.Background())}
+ for _, ctx := range contexts {
+ vectors, err := e.BatchEmbed(ctx, []string{"ab", "c"})
+ require.NoError(t, err)
+ require.Equal(t, [][]float32{{2}, {1}}, vectors)
+ }
+ vectors, err := ie.BatchEmbedImages(context.Background(), pngs(6))
+ require.NoError(t, err)
+ require.Equal(t, [][]float32{{1}, {2}, {3}, {4}, {5}, {6}}, vectors)
+
+ conversation := func(part map[string]any) []any {
+ return []any{map[string]any{"role": "user", "content": []any{part}}}
+ }
+ textBatch := []any{
+ conversation(map[string]any{"type": "text", "text": "ab"}),
+ conversation(map[string]any{"type": "text", "text": "c"}),
+ }
+ imageBatch := []any{}
+ for _, img := range pngs(5) {
+ imageBatch = append(imageBatch, conversation(map[string]any{
+ "type": "image_url", "image_url": map[string]any{"url": img.DataURI()},
+ }))
+ }
+ messages := []any{textBatch, textBatch, imageBatch, conversation(map[string]any{
+ "type": "image_url", "image_url": map[string]any{"url": pngs(6)[5].DataURI()},
+ })}
+ require.Len(t, up.requests, len(messages), "batch texts and split six images into five and one")
+ for i, message := range messages {
+ require.Equal(t, "/v1/embeddings", up.requests[i].path)
+ require.Equal(t, map[string]any{
+ "model": name, "encoding_format": "float",
+ "messages": message,
+ }, up.requests[i].body, "no raw input, legacy truncation, or extra image prompt")
+ }
+ })
+ }
+ })
+ }
+}
+
+// vLLM reads matryoshka_dimensions from WeMM's config and truncates before
+// normalizing. Text and image requests must use the same requested width.
+func TestWeMMDimensionOverride(t *testing.T) {
+ for _, size := range []string{"2B", "4B", "9B"} {
+ t.Run(size, func(t *testing.T) {
+ up := newUpstream(t)
+ e, err := NewEmbedder(Config{
+ Source: types.ModelSourceRemote, Provider: "generic",
+ ModelName: "tencent/WeMM-Embedding-" + size, BaseURL: up.url + "/v1",
+ Dimensions: 1024, SupportsDimensionOverride: true,
+ }, nil, nil)
+ require.NoError(t, err)
+ _, err = e.Embed(types.WithEmbedQuery(context.Background()), "query")
+ require.NoError(t, err)
+ ie, ok := AsImageEmbedder(e)
+ require.True(t, ok)
+ _, err = ie.BatchEmbedImages(context.Background(), pngs(2))
+ require.NoError(t, err)
+ require.Len(t, up.requests, 2)
+ for _, req := range up.requests {
+ require.Equal(t, float64(1024), req.body["dimensions"])
+ require.NotContains(t, req.body, "truncate_prompt_tokens")
+ require.NotContains(t, req.body, "add_special_tokens")
+ require.NotContains(t, req.body, "add_generation_prompt")
+ require.NotContains(t, req.body, "use_activation")
+ }
+ })
+ }
+}
diff --git a/internal/models/parity/testdata/catalog-baseline.json b/internal/models/parity/testdata/catalog-baseline.json
index 42eae44a8..2e2936206 100644
--- a/internal/models/parity/testdata/catalog-baseline.json
+++ b/internal/models/parity/testdata/catalog-baseline.json
@@ -107,7 +107,16 @@
"gemini/KnowledgeQA/gemini-3.7-flash": "570b30391590c3fa380e98e8323bf2187635d18db427eae3d75f9e63def26c8d",
"gemini/KnowledgeQA/gemini-3.8-flash": "6d40802ed8d3b310e1e91c186f769a4f8a298138538853cf623ad43e59c29140",
"gemini/KnowledgeQA/gemini-3snapshot": "c5931f2304774b718637f47540e78b66cbf855b4a4189c9fc0cea2040a12951d",
+ "generic/Embedding/WeMM-Embedding-2B": "8b6d496d4e145cebe75c192377b9a428be2b049fb23419b07e2f4ed766ee744d",
+ "generic/Embedding/WeMM-Embedding-4B": "d1a6ae0fbde3a9828d6392069aef8707817061b553cfc871b2f5432a460b6e83",
+ "generic/Embedding/WeMM-Embedding-9B": "aa6fce5495ec995031fb677c0396671b79864ab099587ac6cbc3fa52c0868e39",
+ "generic/Embedding/snapshot/WeMM-Embedding-2B": "4832569c35ef95ad39fc75e903fa3a860127c373e06670148301dffbaaf65294",
+ "generic/Embedding/snapshot/WeMM-Embedding-4B": "39c3cfdf00aacdcd443309ae6d4940a0ec16300d52aa1944460feae0174353bd",
+ "generic/Embedding/snapshot/WeMM-Embedding-9B": "ad2407f2bedd74df45f4cc2debe4a8f44be7c9bf138d1228c55924c477ea9a61",
"generic/Embedding/snapshotqwen3-vl-embeddingsnapshot": "a1dedaaddb23ad1449d1aef38f0f032d7bcd834bdf079c511c65ec03cccfffe4",
+ "generic/Embedding/tencent/WeMM-Embedding-2B": "54141d358d96ea836ce5a0ede691db0e5dff87cf7d33319a4a59d17a0c1ce0c4",
+ "generic/Embedding/tencent/WeMM-Embedding-4B": "0dd3bbf5caadcd0c230f5bece1aff09e6ccf6180b095c785244580936aeec68b",
+ "generic/Embedding/tencent/WeMM-Embedding-9B": "7176482974a360aa14a1e97dc2e7b508b3ec2526d6f38acbf7f2bf4de595b2d3",
"generic/KnowledgeQA/gpt-5snapshot": "3e3712264316db008c1465a2d7c395072d85942d0352196e456342cef0b14eb0",
"generic/KnowledgeQA/o1snapshot": "52340886b4160a0cb9342ae33a7c81304c9dad5d22c2d1c1ce073f89860a30b1",
"generic/KnowledgeQA/o3snapshot": "782e836769e56c43d9c954ab2d563df7d2f476ae4fb4d588bb8bf3c88ae42ce6",
diff --git a/scripts/wemm-transformers/requirements.txt b/scripts/wemm-transformers/requirements.txt
new file mode 100644
index 000000000..c9a8dd403
--- /dev/null
+++ b/scripts/wemm-transformers/requirements.txt
@@ -0,0 +1,10 @@
+# Install torch/torchvision for the target CPU/CUDA/MPS platform separately.
+# Versions below match the WeMM model card and the validated service environment.
+transformers==5.2.0
+qwen-vl-utils==0.0.14
+accelerate>=1.1.0,<2
+fastapi>=0.115,<1
+pydantic>=2,<3
+uvicorn>=0.30,<1
+pillow>=10,<13
+httpx>=0.27,<1
diff --git a/scripts/wemm-transformers/serve.py b/scripts/wemm-transformers/serve.py
new file mode 100644
index 000000000..08b36ab3b
--- /dev/null
+++ b/scripts/wemm-transformers/serve.py
@@ -0,0 +1,367 @@
+#!/usr/bin/env python3
+"""Serve WeMM text/image embeddings with the official Transformers model code."""
+import argparse
+import base64
+import binascii
+import hmac
+import io
+import logging
+import os
+import threading
+import time
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Literal
+
+from fastapi import Depends, FastAPI, Header, HTTPException
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import JSONResponse
+from PIL import Image, UnidentifiedImageError
+from pydantic import BaseModel, ConfigDict, Field, StrictInt
+
+log = logging.getLogger("uvicorn.error")
+
+
+@dataclass(frozen=True)
+class Settings:
+ model: str = "tencent/WeMM-Embedding-2B"
+ served_model_name: str | None = None
+ revision: str | None = None
+ device: str = "auto"
+ dtype: str = "auto"
+ max_tokens: int = 8192
+ max_pixels: int = 1048576
+ max_batch_size: int = 32
+ cpu_threads: int = 8
+ local_files_only: bool = False
+ api_key: str = ""
+
+ def __post_init__(self):
+ if min(self.max_tokens, self.max_pixels, self.max_batch_size, self.cpu_threads) <= 0:
+ raise ValueError("Resource limits and cpu_threads must be positive.")
+ if self.max_pixels < 65536:
+ raise ValueError("max_pixels must be at least 65536 (the model processor's minimum).")
+
+ @property
+ def name(self):
+ if self.served_model_name:
+ return self.served_model_name
+ return Path(self.model).name if Path(self.model).is_dir() else self.model
+
+
+class StrictModel(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+
+class TextPart(StrictModel):
+ type: Literal["text"]
+ text: str
+
+
+class ImageURL(StrictModel):
+ url: str = Field(max_length=16 * 1024 * 1024)
+ detail: Literal["auto", "low", "high"] = "auto"
+
+
+class ImagePart(StrictModel):
+ type: Literal["image_url"]
+ image_url: ImageURL
+
+
+class Message(StrictModel):
+ role: Literal["system", "user", "assistant"]
+ content: str | list[TextPart | ImagePart] = Field(min_length=1)
+
+
+class EmbedRequest(StrictModel):
+ model: str
+ input: str | list[str] | None = None
+ messages: list[Message] | list[list[Message]] | None = None
+ dimensions: StrictInt | None = None
+ encoding_format: Literal["float", "base64"] = "float"
+ user: str | None = None
+
+
+def bad(message):
+ raise HTTPException(400, message)
+
+
+def parse_inputs(request, settings, dimensions, default_dimension):
+ if request.model != settings.name:
+ raise HTTPException(404, f"Model not found: {request.model}")
+ dimension = default_dimension if request.dimensions is None else request.dimensions
+ if dimension not in dimensions:
+ bad(f"dimensions must be one of {dimensions}.")
+ if (request.input is None) == (request.messages is None):
+ bad("Supply exactly one of input or messages.")
+ if request.input is not None:
+ texts = [request.input] if isinstance(request.input, str) else request.input
+ conversations = [[Message(role="user", content=[TextPart(type="text", text=text)])] for text in texts]
+ else:
+ messages = request.messages
+ conversations = [messages] if messages and isinstance(messages[0], Message) else messages
+ if not conversations or len(conversations) > settings.max_batch_size:
+ bad(f"Supply between 1 and {settings.max_batch_size} independent inputs.")
+ for conversation in conversations:
+ if not conversation:
+ bad("Each conversation must contain at least one message.")
+ chars, images = 0, 0
+ for message in conversation:
+ parts = message.content
+ if isinstance(parts, str):
+ chars += len(parts)
+ continue
+ for part in parts:
+ if isinstance(part, TextPart):
+ chars += len(part.text)
+ else:
+ images += 1
+ if not part.image_url.url.startswith("data:image/") or ";base64," not in part.image_url.url:
+ bad("Images must be base64 data:image/... URLs; network URLs and file paths are unsupported.")
+ if images > 5:
+ bad("At most 5 images per conversation.")
+ if chars > settings.max_tokens * 16:
+ bad("Text exceeds the character guard; reduce the input size.")
+ return conversations, dimension
+
+
+def decode_image(url):
+ try:
+ data = base64.b64decode(url.split(";base64,", 1)[1], validate=True)
+ with Image.open(io.BytesIO(data)) as image:
+ if image.width * image.height > 40_000_000:
+ bad("Image exceeds 40 megapixels before resizing.")
+ # Preserve alpha until qwen-vl-utils composites RGBA onto white,
+ # matching the official inference path instead of exposing hidden RGB.
+ return image.copy()
+ except (ValueError, binascii.Error, UnidentifiedImageError, OSError, Image.DecompressionBombError):
+ bad("Invalid image data.")
+
+
+def prepare_messages(conversation, max_pixels):
+ messages = []
+ for message in conversation:
+ parts = message.content
+ if isinstance(parts, str):
+ parts = [TextPart(type="text", text=parts)]
+ content = []
+ for part in parts:
+ if isinstance(part, TextPart):
+ content.append({"type": "text", "text": part.text})
+ else:
+ content.append({"type": "image", "image": decode_image(part.image_url.url), "max_pixels": max_pixels})
+ messages.append({"role": message.role, "content": content})
+ return messages
+
+
+def resolve_device_dtype(torch, settings):
+ name = settings.device
+ if name == "auto":
+ name = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu"
+ device = torch.device(name)
+ if device.type not in ("cuda", "mps", "cpu"):
+ raise ValueError("device must be auto, cpu, mps, cuda or cuda:N.")
+ if device.type == "cuda":
+ if not torch.cuda.is_available():
+ raise ValueError("CUDA is unavailable; install a CUDA-enabled PyTorch build or select another device.")
+ if device.index is not None and device.index >= torch.cuda.device_count():
+ raise ValueError("The requested CUDA device does not exist.")
+ with torch.cuda.device(device):
+ bf16 = torch.cuda.is_bf16_supported()
+ elif device.type == "mps":
+ if not torch.backends.mps.is_available():
+ raise ValueError("MPS is unavailable on this machine.")
+ bf16 = True
+ else:
+ bf16 = False
+ dtype = settings.dtype
+ if dtype == "auto":
+ dtype = "float32" if device.type == "cpu" else "bfloat16" if bf16 else "float16"
+ if device.type == "cuda" and dtype == "bfloat16" and not bf16:
+ raise ValueError("This CUDA device does not support bfloat16; use --dtype float16.")
+ return device, getattr(torch, dtype)
+
+
+class Engine:
+ def __init__(self, settings):
+ # Import after CLI setup so --mps-fallback takes effect before torch loads.
+ import torch
+ from transformers import AutoConfig, AutoModel, AutoProcessor
+ self.settings, self.torch = settings, torch
+ self.device, self.dtype = resolve_device_dtype(torch, settings)
+ self.lock = threading.Lock()
+ torch.set_num_threads(settings.cpu_threads)
+ options = {"trust_remote_code": True, "local_files_only": settings.local_files_only}
+ if settings.revision:
+ options["revision"] = settings.revision
+ config = AutoConfig.from_pretrained(settings.model, **options)
+ self.dimensions = tuple(getattr(config, "matryoshka_dimensions", ()))
+ self.default_dimension = config.text_config.hidden_size
+ if not self.dimensions or self.default_dimension not in self.dimensions:
+ raise ValueError("The model must provide WeMM matryoshka_dimensions including its hidden size.")
+ self.processor = AutoProcessor.from_pretrained(settings.model, **options)
+ self.model = AutoModel.from_pretrained(
+ settings.model, dtype=self.dtype, attn_implementation="sdpa", **options,
+ ).to(self.device).eval()
+ if not callable(getattr(self.model, "embedding", None)):
+ raise ValueError("The model must expose WeMM's embedding() method.")
+
+ def encode(self, conversations, dimension):
+ from qwen_vl_utils import process_vision_info
+ torch = self.torch
+ vectors, total_tokens = [], 0
+ # WeMM's model stores rope_deltas. Keep independent conversations serialized,
+ # and decode images only for the current input rather than the entire batch.
+ with self.lock, torch.inference_mode():
+ for conversation in conversations:
+ messages = prepare_messages(conversation, self.settings.max_pixels)
+ text = self.processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=False)
+ images, _ = process_vision_info(messages, image_patch_size=16)
+ inputs = self.processor(text=text, images=images, return_tensors="pt")
+ count = inputs["input_ids"].shape[-1]
+ if count > self.settings.max_tokens:
+ bad(f"Input has {count} tokens; limit is {self.settings.max_tokens}. Input was not truncated.")
+ last = self.processor.tokenizer.convert_ids_to_tokens(inputs["input_ids"][0, -1].item())
+ if last != "<embedding>":
+ raise RuntimeError("Processor did not append the official <embedding> token.")
+ inputs = inputs.to(self.device)
+ # Reset explicitly as well, so older official code revisions are safe
+ # when a text request follows a multimodal request.
+ self.model.model.rope_deltas = None
+ vector = self.model.embedding(**inputs, use_cache=False).float()
+ vector = torch.nn.functional.normalize(vector[:, :dimension], dim=-1).cpu()[0]
+ if vector.numel() != dimension or not torch.isfinite(vector).all() or vector.norm() < 0.99:
+ raise RuntimeError("Invalid embedding output.")
+ vectors.append(vector)
+ total_tokens += count
+ del inputs, images, messages
+ return vectors, total_tokens
+
+
+class BodyLimit:
+ """Bound JSON bodies, including chunked requests, before parsing/base64 decoding."""
+ def __init__(self, app, limit=32 * 1024 * 1024):
+ self.app, self.limit = app, limit
+
+ async def __call__(self, scope, receive, send):
+ if scope["type"] != "http" or scope["method"] != "POST":
+ return await self.app(scope, receive, send)
+ chunks, size = [], 0
+ while True:
+ message = await receive()
+ if message["type"] == "http.disconnect":
+ return
+ chunk = message.get("body", b"")
+ size += len(chunk)
+ if size > self.limit:
+ response = JSONResponse({"error": {"message": "Request body exceeds 32 MiB."}}, status_code=413)
+ return await response(scope, receive, send)
+ chunks.append(chunk)
+ if not message.get("more_body", False):
+ break
+ body = b"".join(chunks)
+
+ async def replay():
+ nonlocal body
+ if body is not None:
+ value, body = body, None
+ return {"type": "http.request", "body": value, "more_body": False}
+ return await receive()
+
+ await self.app(scope, replay, send)
+
+
+def create_app(settings, engine_factory=Engine):
+ @asynccontextmanager
+ async def lifespan(app):
+ started = time.monotonic()
+ app.state.engine = engine_factory(settings)
+ log.info("Loaded %s on %s/%s in %.1fs", settings.name, app.state.engine.device,
+ app.state.engine.dtype, time.monotonic() - started)
+ yield
+ del app.state.engine
+
+ app = FastAPI(title="WeMM Transformers embedding service", lifespan=lifespan)
+ app.add_middleware(BodyLimit)
+
+ def authorize(authorization: str | None = Header(default=None)):
+ expected = "Bearer " + settings.api_key
+ if settings.api_key and not hmac.compare_digest((authorization or "").encode(), expected.encode()):
+ raise HTTPException(401, "Invalid API key.", headers={"WWW-Authenticate": "Bearer"})
+
+ @app.exception_handler(HTTPException)
+ async def api_error(_request, exc):
+ return JSONResponse(status_code=exc.status_code, headers=exc.headers,
+ content={"error": {"message": exc.detail, "code": exc.status_code}})
+
+ @app.exception_handler(RequestValidationError)
+ async def validation_error(_request, exc):
+ return JSONResponse(status_code=400, content={"error": {
+ "message": "Invalid request: " + exc.errors()[0]["msg"], "code": 400,
+ }})
+
+ @app.get("/health")
+ def health():
+ engine = app.state.engine
+ return {"status": "ok", "model": settings.name, "backend": "transformers",
+ "device": str(engine.device), "dtype": str(engine.dtype), "revision": settings.revision,
+ "dimensions": engine.dimensions, "default_dimension": engine.default_dimension,
+ "max_tokens": settings.max_tokens, "max_image_pixels": settings.max_pixels}
+
+ @app.get("/v1/models", dependencies=[Depends(authorize)])
+ def models():
+ return {"object": "list", "data": [{"id": settings.name, "object": "model", "created": 0, "owned_by": "tencent"}]}
+
+ @app.post("/v1/embeddings", dependencies=[Depends(authorize)])
+ def embeddings(request: EmbedRequest):
+ engine = app.state.engine
+ conversations, dimension = parse_inputs(request, settings, engine.dimensions, engine.default_dimension)
+ started = time.monotonic()
+ try:
+ vectors, tokens = engine.encode(conversations, dimension)
+ except HTTPException:
+ raise
+ except Exception:
+ log.exception("WeMM inference failed")
+ raise HTTPException(500, "WeMM inference failed; inspect the server log.") from None
+ data = []
+ for index, vector in enumerate(vectors):
+ value = (base64.b64encode(vector.numpy().astype("<f4").tobytes()).decode()
+ if request.encoding_format == "base64" else vector.tolist())
+ data.append({"object": "embedding", "index": index, "embedding": value})
+ log.info("Embedded %d inputs, %d tokens, %d dimensions in %.2fs", len(data), tokens,
+ dimension, time.monotonic() - started)
+ return {"object": "list", "data": data, "model": settings.name,
+ "usage": {"prompt_tokens": tokens, "total_tokens": tokens}}
+
+ return app
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--model", default=Settings.model, help="WeMM Hugging Face ID or model directory")
+ parser.add_argument("--served-model-name", help="API model ID; defaults to the repo ID or directory basename")
+ parser.add_argument("--revision", help="Hugging Face revision (prefer a commit SHA for reproducibility)")
+ parser.add_argument("--device", default="auto", help="auto, cuda, cuda:N, mps or cpu")
+ parser.add_argument("--dtype", choices=("auto", "bfloat16", "float16", "float32"), default="auto")
+ parser.add_argument("--max-tokens", type=int, default=8192)
+ parser.add_argument("--max-pixels", type=int, default=1048576)
+ parser.add_argument("--max-batch-size", type=int, default=32)
+ parser.add_argument("--cpu-threads", type=int, default=8)
+ parser.add_argument("--local-files-only", action="store_true")
+ parser.add_argument("--mps-fallback", action="store_true", help="Allow PyTorch CPU fallback for unsupported MPS ops")
+ parser.add_argument("--host", default="127.0.0.1")
+ parser.add_argument("--port", type=int, default=18080)
+ args = vars(parser.parse_args())
+ host, port = args.pop("host"), args.pop("port")
+ if args.pop("mps_fallback"):
+ os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
+ settings = Settings(**args, api_key=os.environ.get("WEMM_API_KEY", ""))
+ import uvicorn
+ # A single process owns the model; do not add workers without separate GPUs.
+ uvicorn.run(create_app(settings), host=host, port=port, workers=1)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/wemm-transformers/smoke_test.py b/scripts/wemm-transformers/smoke_test.py
new file mode 100644
index 000000000..0384087de
--- /dev/null
+++ b/scripts/wemm-transformers/smoke_test.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+"""Check a running WeMM service with actual text and image inference."""
+import argparse
+import base64
+import io
+import json
+import math
+import os
+import time
+
+import httpx
+from PIL import Image, ImageDraw
+
+
+def dot(a, b):
+ return sum(x * y for x, y in zip(a, b))
+
+
+def text_message(text):
+ return [{"role": "user", "content": [{"type": "text", "text": text}]}]
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--base-url", default="http://127.0.0.1:18080", help="Service root URL, without /v1")
+ parser.add_argument("--timeout", type=float, default=180)
+ args = parser.parse_args()
+ headers = {"Authorization": "Bearer " + os.environ["WEMM_API_KEY"]} if os.environ.get("WEMM_API_KEY") else {}
+ results = []
+ with httpx.Client(base_url=args.base_url.rstrip("/"), timeout=args.timeout, headers=headers) as client:
+ health_response = client.get("/health")
+ health_response.raise_for_status()
+ health = health_response.json()
+ model = health["model"]
+ default_dimension = health["default_dimension"]
+ response = client.get("/v1/models")
+ response.raise_for_status()
+ assert model in [row["id"] for row in response.json()["data"]]
+
+ def run(name, **payload):
+ started = time.monotonic()
+ response = client.post("/v1/embeddings", json={"model": model, **payload})
+ response.raise_for_status()
+ data = response.json()
+ vectors = [row["embedding"] for row in data["data"]]
+ assert [row["index"] for row in data["data"]] == list(range(len(vectors)))
+ for vector in vectors:
+ assert len(vector) == payload.get("dimensions", default_dimension)
+ assert all(math.isfinite(x) for x in vector)
+ assert abs(math.sqrt(dot(vector, vector)) - 1) < 1e-4
+ results.append({"test": name, "seconds": round(time.monotonic() - started, 3),
+ "vectors": len(vectors), "dimension": len(vectors[0])})
+ print(json.dumps(results[-1]), flush=True)
+ return vectors
+
+ texts = ["A red square on a white background.", "A blue circle on a black background."]
+ text_vectors = run("batched text messages", messages=[text_message(text) for text in texts])
+ images = []
+ for color, shape, background in [("red", "rectangle", "white"), ("blue", "ellipse", "black")]:
+ image = Image.new("RGB", (256, 256), background)
+ getattr(ImageDraw.Draw(image), shape)((40, 40, 216, 216), fill=color)
+ buffer = io.BytesIO()
+ image.save(buffer, format="PNG")
+ images.append([{"role": "user", "content": [{"type": "image_url", "image_url": {
+ "url": "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode(),
+ }}]}])
+ image_vectors = run("batched image messages", messages=images)
+ scores = [[dot(text, image) for image in image_vectors] for text in text_vectors]
+ assert scores[0][0] > scores[0][1] and scores[1][1] > scores[1][0], scores
+ repeated = run("text after image state isolation", messages=text_message(texts[0]))[0]
+ assert dot(repeated, text_vectors[0]) > 0.9999
+ dimension = min(health["dimensions"])
+ short = run("Matryoshka dimensions", messages=text_message(texts[0]), dimensions=dimension)[0]
+ prefix = text_vectors[0][:dimension]
+ norm = math.sqrt(dot(prefix, prefix))
+ assert dot(short, [value / norm for value in prefix]) > 0.9999
+ raw = run("raw text compatibility", input=texts[0])[0]
+ assert dot(raw, text_vectors[0]) > 0.9999
+ for payload in ({"input": "hello", "dimensions": 3}, {"input": "a", "messages": []},
+ {"messages": [{"role": "user", "content": [{"type": "image_url", "image_url": {
+ "url": "https://example.com/image.png",
+ }}]}]}):
+ response = client.post("/v1/embeddings", json={"model": model, **payload})
+ assert response.status_code == 400, response.text
+ print(json.dumps({"status": "passed", "health": health, "tests": results, "text_image_scores": scores}), flush=True)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/wemm-transformers/test_serve.py b/scripts/wemm-transformers/test_serve.py
new file mode 100644
index 000000000..e08f5a53a
--- /dev/null
+++ b/scripts/wemm-transformers/test_serve.py
@@ -0,0 +1,169 @@
+"""Contract tests without downloading weights: python -m unittest discover -s scripts/wemm-transformers."""
+import asyncio
+import base64
+import io
+import struct
+import unittest
+from contextlib import nullcontext
+from unittest.mock import patch
+
+import torch
+from fastapi import HTTPException
+from fastapi.testclient import TestClient
+from PIL import Image
+from serve import BodyLimit, Message, Settings, create_app, decode_image, prepare_messages, resolve_device_dtype
+
+
+class FakeEngine:
+ device, dtype = "cpu", "float32"
+
+ def __init__(self, settings, dimensions=(64, 128, 256, 512, 1024, 2048)):
+ self.dimensions = dimensions
+ self.default_dimension = dimensions[-1]
+ self.calls = []
+
+ def encode(self, conversations, dimension):
+ self.calls.append(conversations)
+ vectors = []
+ for index, _ in enumerate(conversations):
+ vector = torch.zeros(dimension)
+ vector[index] = 1
+ vectors.append(vector)
+ return vectors, len(conversations) * 10
+
+
+class ServiceTests(unittest.TestCase):
+ def setUp(self):
+ self.settings = Settings()
+ self.engine = FakeEngine(self.settings)
+ self.client = self.enterContext(TestClient(create_app(self.settings, lambda _: self.engine)))
+
+ def post(self, **payload):
+ return self.client.post("/v1/embeddings", json={"model": self.settings.name, **payload})
+
+ def test_input_and_single_or_batched_messages(self):
+ message = {"role": "user", "content": [{"type": "text", "text": "hello"}]}
+ for payload, count in [({"input": "hello"}, 1), ({"input": ["a", "b"]}, 2),
+ ({"messages": [message]}, 1), ({"messages": [[message], [message]]}, 2)]:
+ with self.subTest(payload=payload):
+ response = self.post(**payload)
+ self.assertEqual(response.status_code, 200, response.text)
+ data = response.json()["data"]
+ self.assertEqual([x["index"] for x in data], list(range(count)))
+ self.assertTrue(all(len(x["embedding"]) == 2048 for x in data))
+ self.assertEqual([x["embedding"][i] for i, x in enumerate(data)], [1] * count)
+
+ def test_all_model_sizes_use_their_own_default_and_allowed_dimensions(self):
+ for size, dimensions in [("2B", (64, 128, 256, 512, 1024, 2048)),
+ ("4B", (64, 128, 256, 512, 1024, 2560)),
+ ("9B", (64, 128, 256, 512, 1024, 2048, 4096))]:
+ settings = Settings(model="tencent/WeMM-Embedding-" + size)
+ with self.subTest(size=size), TestClient(create_app(settings, lambda s: FakeEngine(s, dimensions))) as client:
+ for requested in (None, 1024):
+ payload = {"model": settings.name, "input": "hello"}
+ if requested is not None:
+ payload["dimensions"] = requested
+ response = client.post("/v1/embeddings", json=payload)
+ self.assertEqual(response.status_code, 200, response.text)
+ self.assertEqual(len(response.json()["data"][0]["embedding"]), requested or dimensions[-1])
+ payload["dimensions"] = 2049
+ self.assertEqual(client.post("/v1/embeddings", json=payload).status_code, 400)
+ self.assertEqual(client.get("/health").json()["default_dimension"], dimensions[-1])
+
+ def test_base64_is_little_endian_float32(self):
+ response = self.post(input="hello", dimensions=64, encoding_format="base64")
+ vector = struct.unpack("<64f", base64.b64decode(response.json()["data"][0]["embedding"]))
+ self.assertEqual(vector, (1,) + (0,) * 63)
+
+ def test_invalid_requests_never_reach_inference(self):
+ for payload in [dict(input="a", messages=[]), dict(input=[]), dict(messages=[[]]),
+ dict(input="a", dimensions=True), dict(input="a", dimensions=0),
+ dict(input="a", temperature=0), dict(input=["a"] * 33),
+ dict(messages=[{"role": "user", "content": [{"type": "video", "video": "x"}]}]),
+ dict(messages=[{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "file:///etc/passwd"}}]}])]:
+ with self.subTest(payload=payload):
+ self.assertEqual(self.post(**payload).status_code, 400)
+ self.assertEqual(self.post(model="wrong", input="a").status_code, 404)
+ self.assertEqual(self.engine.calls, [])
+
+ def test_authentication_on_models_and_embeddings(self):
+ settings = Settings(api_key="test-only-key")
+ with TestClient(create_app(settings, lambda _: self.engine)) as client:
+ self.assertEqual(client.get("/health").status_code, 200)
+ self.assertEqual(client.get("/v1/models").status_code, 401)
+ for token, status in [("Bearer wrong", 401), ("Bearer test-only-key", 200)]:
+ response = client.post("/v1/embeddings", json={"model": settings.name, "input": "a"},
+ headers={"Authorization": token})
+ self.assertEqual(response.status_code, status)
+ self.assertEqual(len(self.engine.calls), 1)
+
+ def test_image_decoding_and_invalid_bytes(self):
+ buffer = io.BytesIO()
+ Image.new("RGB", (64, 64), "red").save(buffer, format="PNG")
+ uri = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
+ self.assertEqual(decode_image(uri).size, (64, 64))
+ for data in ["!invalid!", base64.b64encode(b"not an image").decode()]:
+ with self.assertRaises(HTTPException) as error:
+ decode_image("data:image/png;base64," + data)
+ self.assertEqual(error.exception.status_code, 400)
+
+ def test_transparent_images_match_official_white_background(self):
+ from qwen_vl_utils import process_vision_info
+ image = Image.new("RGBA", (256, 256), (0, 0, 0, 0))
+ image.putpixel((128, 128), (255, 0, 0, 128))
+ buffer = io.BytesIO()
+ image.save(buffer, format="PNG")
+ uri = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
+ conversation = [Message.model_validate({"role": "user", "content": [
+ {"type": "image_url", "image_url": {"url": uri}},
+ ]})]
+ actual, _ = process_vision_info(prepare_messages(conversation, 1048576), image_patch_size=16)
+ expected, _ = process_vision_info([{"role": "user", "content": [
+ {"type": "image", "image": image, "max_pixels": 1048576},
+ ]}], image_patch_size=16)
+ self.assertEqual(actual[0].getpixel((0, 0)), (255, 255, 255))
+ self.assertEqual(actual[0].getpixel((128, 128)), (255, 127, 127))
+ self.assertEqual(actual[0].tobytes(), expected[0].tobytes())
+
+ def test_cpu_defaults_and_explicit_unavailable_device(self):
+ with patch.object(torch.cuda, "is_available", return_value=False), patch.object(torch.backends.mps, "is_available", return_value=False):
+ device, dtype = resolve_device_dtype(torch, Settings())
+ self.assertEqual(str(device), "cpu")
+ self.assertEqual(dtype, torch.float32)
+ for requested in ("cuda", "mps"):
+ with self.assertRaises(ValueError):
+ resolve_device_dtype(torch, Settings(device=requested))
+
+ def test_cuda_auto_dtype_and_explicit_bf16_guard(self):
+ with patch.object(torch.cuda, "is_available", return_value=True), patch.object(torch.cuda, "device", return_value=nullcontext()):
+ for supported, expected in [(True, torch.bfloat16), (False, torch.float16)]:
+ with patch.object(torch.cuda, "is_bf16_supported", return_value=supported):
+ device, dtype = resolve_device_dtype(torch, Settings())
+ self.assertEqual(str(device), "cuda")
+ self.assertEqual(dtype, expected)
+ if not supported:
+ with self.assertRaises(ValueError):
+ resolve_device_dtype(torch, Settings(device="cuda", dtype="bfloat16"))
+
+ def test_body_limit_counts_chunked_bodies(self):
+ async def run(chunks):
+ received, sent = [], []
+ events = iter({"type": "http.request", "body": chunk, "more_body": i < len(chunks) - 1}
+ for i, chunk in enumerate(chunks))
+ async def receive():
+ return next(events)
+ async def send(event):
+ sent.append(event)
+ async def downstream(scope, receive, send):
+ received.append((await receive())["body"])
+ await BodyLimit(downstream, limit=4)({"type": "http", "method": "POST"}, receive, send)
+ return received, sent
+ received, _ = asyncio.run(run([b"12", b"34"]))
+ self.assertEqual(received, [b"1234"])
+ received, sent = asyncio.run(run([b"12", b"345"]))
+ self.assertEqual(received, [])
+ self.assertEqual(sent[0]["status"], 413)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/website-docs/03-features/06-models.md b/website-docs/03-features/06-models.md
index ea8ddb2e7..68cd82a24 100644
--- a/website-docs/03-features/06-models.md
+++ b/website-docs/03-features/06-models.md
@@ -95,6 +95,46 @@
| 向量或重排服务响应慢,需要更长的超时 | `{"request_timeout_seconds": 120}` |
| 自建重排服务返回的是未归一化分数(logit) | `{"score_scale": "logit"}` |
+#### WeMM-Embedding
+
+自定义厂商(`generic`)的向量模型列表已内置以下预设,选择后会自动识别图片输入和原生向量维度:
+
+| 模型名称 | 原生维度 |
+| --- | --- |
+| `tencent/WeMM-Embedding-2B` | 2048 |
+| `tencent/WeMM-Embedding-4B` | 2560 |
+| `tencent/WeMM-Embedding-9B` | 4096 |
+
+可选择官方 vLLM 部署,或使用本仓库的 [Transformers 部署脚本](../06-development/06-wemm-transformers.md)在 CUDA、MPS 或 CPU 上提供同一 Embedding 接口。MPS 是可选计算设备,服务也可以部署在独立服务器上。
+
+按[官方部署说明](https://github.com/Tencent/WeMM-Embedding#serving)使用 vLLM `0.27.0`,并加载模型目录中的 `embedding_chat_template.jinja`。例如部署 2B,并通过 `--served-model-name` 让服务名称与预设一致:
+
+```bash
+MODEL_PATH=/path/to/WeMM-Embedding-2B
+vllm serve "$MODEL_PATH" \
+ --runner pooling \
+ --chat-template "$MODEL_PATH/embedding_chat_template.jinja" \
+ --served-model-name tencent/WeMM-Embedding-2B
+```
+
+4B、9B 分别替换模型路径与服务名称。在 WeKnora 中选择「自定义」厂商,Base URL 填写服务的 `http://主机:端口/v1`,选择对应向量模型,初次接入保持「自定义输出维度」关闭。若服务没有设置 `--served-model-name`,模型名称须填写服务实际返回的本地路径;短名称 `WeMM-Embedding-2B` 及以 `/WeMM-Embedding-2B` 结尾的路径也能识别预设,4B、9B 同理。任意自定义别名需要另行配置模型能力和协议覆盖。
+
+预设让文本和图片都通过 `messages` 进入模型模板,由模板追加取向量所需的 `<embedding>` 标记;启用 `batch_messages`,将同一批内的独立 conversations 合并请求,每条仍返回各自的向量。图片批量调用最多每批 5 张,超出自动拆分;这是 WeKnora 选择的批量大小,与现有 Qwen3-VL 预设一致,不代表模型或 vLLM 的官方上限,可用 `max_image_batch_size` 按服务承载能力调整。当前知识库图片索引任务仍逐张调用,不会因该配置自动合并多个任务。不发送历史默认的 `truncate_prompt_tokens: 511`;超出服务上下文上限时由服务报错,不会静默截断。无需额外填写兼容 JSON 或图片提示词。若此前用普通文本 `input` 为该模型建立过索引,需要重建向量。
+
+WeMM 的模板已经包含所需特殊 token,保留 vLLM 的 `add_special_tokens: false`、`add_generation_prompt: false` 默认行为,不要照搬 Qwen3-VL 预设的 `add_special_tokens: true`。vLLM 的 Embedding 默认使用末尾 token 并做 L2 归一化,无需额外发送 `normalize` 或关闭 `use_activation`。
+
+需要降维时,开启「自定义输出维度」并选择模型配置中的合法维度,WeKnora 会给文本和图片请求发送相同的 `dimensions`;vLLM 会先截取对应维度,再归一化:
+
+| 模型 | 可选维度 |
+| --- | --- |
+| 2B | 64、128、256、512、1024、2048 |
+| 4B | 64、128、256、512、1024、2560 |
+| 9B | 64、128、256、512、1024、2048、4096 |
+
+这些值来自[官方模型配置与说明](https://github.com/Tencent/WeMM-Embedding#model-zoo),vLLM 会拒绝列表之外的维度。更换输出维度后需重建已有向量索引。超时、并发与图片大小限制保持现有默认值,按实际服务性能调整。
+
+知识库还需开启图片向量索引才能直接检索图片内容。当前预设接入文本和图片;视频、任意图文交错输入及扫描 PDF 整页向量化不在当前业务链路的支持范围内。
+
#### 字段参考
下表的「协议默认」是没有任何厂商、目录或覆盖时的取值;具体厂商的实际值以「实际调用方式」面板为准。
diff --git a/website-docs/06-development/06-wemm-transformers.md b/website-docs/06-development/06-wemm-transformers.md
new file mode 100644
index 000000000..914f6d12a
--- /dev/null
+++ b/website-docs/06-development/06-wemm-transformers.md
@@ -0,0 +1,205 @@
+# WeMM Transformers 部署
+
+使用 [WeMM 官方 Transformers 推理代码](https://github.com/Tencent/WeMM-Embedding#transformers)加载模型,由仓库内的 `scripts/wemm-transformers/serve.py` 提供 HTTP Embedding 服务。服务可以与 WeKnora 部署在同一机器,也可以部署在独立 GPU 服务器上,通过 Base URL 接入。
+
+CUDA、Apple MPS 和 CPU 是可选的计算设备,使用同一份脚本和请求协议。此 HTTP 适配层由 WeKnora 提供,支持文本和图片;模型本身支持的视频等能力尚未在此接口开放。官方 vLLM 部署方式见[模型配置](../03-features/06-models.md#wemm-embedding)。
+
+## 安装依赖
+
+以下命令在 WeKnora 仓库根目录执行,使用 Python 3.12。虚拟环境和模型文件放在仓库外:
+
+```bash
+python3.12 -m venv "$HOME/.venvs/weknora-wemm"
+source "$HOME/.venvs/weknora-wemm/bin/activate"
+python -m pip install --upgrade pip
+```
+
+按设备选择一组 PyTorch 安装命令。以下使用已验证的 PyTorch 2.10.0 / torchvision 0.25.0,CUDA wheel 须与驱动兼容;其他 CUDA 版本可参考 [PyTorch 安装说明](https://pytorch.org/get-started/previous-versions/#v2100)。
+
+```bash
+# NVIDIA GPU:示例为 CUDA 12.8 wheel
+python -m pip install torch==2.10.0 torchvision==0.25.0 \
+ --index-url https://download.pytorch.org/whl/cu128
+
+# 或 Apple Silicon / MPS
+python -m pip install torch==2.10.0 torchvision==0.25.0
+
+# 或 Linux / Windows CPU wheel
+python -m pip install torch==2.10.0 torchvision==0.25.0 \
+ --index-url https://download.pytorch.org/whl/cpu
+```
+
+安装其余依赖:
+
+```bash
+python -m pip install -r scripts/wemm-transformers/requirements.txt
+```
+
+Transformers 固定为官方推荐的 `5.2.0`,`qwen-vl-utils` 固定为 `0.0.14`。不要随意替换 Transformers 版本,避免预处理行为变化。脚本会启用 `trust_remote_code=True` 来加载 WeMM 的 `embedding()` 实现,应使用官方权重或已审查的模型目录。
+
+## 启动服务
+
+以 CUDA 上的 2B 为例:
+
+```bash
+python scripts/wemm-transformers/serve.py \
+ --model tencent/WeMM-Embedding-2B \
+ --revision bbd6cd4bf52cfc6716f752a2df80b2706720bd95 \
+ --device cuda \
+ --host 127.0.0.1 \
+ --port 18080
+```
+
+首次启动会从 Hugging Face 下载模型到标准缓存,后续复用缓存。`--revision` 可固定模型代码与权重版本;示例 SHA 属于 2B,切换 4B、9B 时应选取对应仓库的版本,不能沿用此 SHA。未指定时使用该仓库默认版本。
+
+已有模型目录时可以离线启动,并指定 WeKnora 调用时使用的模型名称:
+
+```bash
+python scripts/wemm-transformers/serve.py \
+ --model /srv/models/WeMM-Embedding-2B \
+ --served-model-name tencent/WeMM-Embedding-2B \
+ --local-files-only \
+ --device cuda
+```
+
+也可以提前下载并固定版本,例如:
+
+```bash
+hf download tencent/WeMM-Embedding-2B \
+ --revision bbd6cd4bf52cfc6716f752a2df80b2706720bd95 \
+ --local-dir /srv/models/WeMM-Embedding-2B
+```
+
+其他设备只需更换参数:
+
+```bash
+# Apple Silicon,允许不支持的 MPS 算子回退到 CPU
+python scripts/wemm-transformers/serve.py \
+ --model tencent/WeMM-Embedding-2B --device mps --mps-fallback
+
+# CPU,默认使用 float32
+python scripts/wemm-transformers/serve.py \
+ --model tencent/WeMM-Embedding-2B --device cpu
+```
+
+4B、9B 将 `--model` 换成对应 ID 或路径。服务从模型配置读取原生维度及 Matryoshka 可选维度,不会将其他规格强制按 2048 维处理:
+
+| 模型 ID | 原生维度 |
+| --- | --- |
+| `tencent/WeMM-Embedding-2B` | 2048 |
+| `tencent/WeMM-Embedding-4B` | 2560 |
+| `tencent/WeMM-Embedding-9B` | 4096 |
+
+服务以前台方式运行,按 Ctrl+C 停止。常驻部署时由 systemd、容器或其他进程管理器托管同一命令。一个进程加载一份完整模型到一个设备,不提供多 GPU 切分;选择规格前应预留权重、视觉处理和推理激活所需的显存/内存。
+
+### 独立服务器与鉴权
+
+跨机器部署时使用 `--host 0.0.0.0`,并在服务进程环境中设置 `WEMM_API_KEY`。`/v1/models` 与 `/v1/embeddings` 会验证 `Authorization: Bearer <key>`;WeKnora 的 API Key 填写相同值。服务不直接终止 TLS,跨不可信网络时通过 HTTPS 反向代理访问。`/health` 仅提供运行状态和非敏感模型元数据,不要求 API Key。
+
+默认监听回环地址、未设置 API Key。WeKnora 若运行在 Docker 中,`127.0.0.1` 指向 WeKnora 容器自身,应填写容器能够访问的推理服务地址,例如同一容器网络的服务名,或已配置的宿主机地址。私网地址还需符合 WeKnora 的 SSRF 访问策略,配置方法见[安全说明](../01-getting-started/04-configuration.md)。
+
+## WeKnora 接入
+
+在「设置 → 模型管理」添加:
+
+| 字段 | 填写 |
+| --- | --- |
+| 类型 | Embedding |
+| 来源 / 服务商 | API / 自定义(OpenAI 兼容接口) |
+| 模型名称 | `/v1/models` 返回的 ID,推荐保留 `tencent/WeMM-Embedding-2B` 等官方名称 |
+| Base URL | `http://推理服务地址:18080/v1`,使用 HTTPS 代理时填写对应地址 |
+| API Key | 设置了 `WEMM_API_KEY` 时填写相同值 |
+| 图片输入 | 官方名称由预设识别;自定义别名需手动开启 |
+| 自定义输出维度 | 初次接入关闭,让连接检测识别原生维度;需要降维时再开启 |
+| 后台并发上限 | 建议从 1 开始 |
+
+现有 WeMM 预设的 `messages` 格式同时适用于此服务。推荐在「高级 → 协议兼容覆盖」设置较小批量和较长超时:
+
+```json
+{
+ "api": "openai-embeddings",
+ "image_format": "messages",
+ "text_as_messages": true,
+ "batch_messages": true,
+ "accepts_truncate_prompt_tokens": false,
+ "max_batch_size": 4,
+ "max_image_batch_size": 2,
+ "request_timeout_seconds": 180
+}
+```
+
+完整配置也适用于尚未包含 WeMM 预设的旧版本或自定义模型别名;此时应手动打开图片输入。点击「测试连接」,确认实际维度后保存。知识库还需开启图片向量索引,配置方式见[模型配置](../03-features/06-models.md)。切换模型或输出维度需要重建已有索引。
+
+## 请求格式与推理流程
+
+接口为 `POST /v1/embeddings`。纯文本可以使用 `input`:
+
+```json
+{
+ "model": "tencent/WeMM-Embedding-2B",
+ "input": ["第一段文本", "第二段文本"],
+ "encoding_format": "float"
+}
+```
+
+文本和图片也可使用 `messages`。以下二维数组包含两个独立输入,分别返回一个文本向量和一个图片向量:
+
+```json
+{
+ "model": "tencent/WeMM-Embedding-2B",
+ "dimensions": 1024,
+ "messages": [
+ [{"role": "user", "content": [{"type": "text", "text": "白色背景上的红色方块"}]}],
+ [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "data:image/png;base64,图片Base64"}}]}]
+ ]
+}
+```
+
+`input` 和 `messages` 只能选一个。单条 conversation 可以使用一维消息数组;文字与图片放在同一 conversation 中会得到一个联合向量。返回值采用 OpenAI Embedding 的 `data[].embedding`、`data[].index` 和 `usage` 结构,编码支持 `float` 与 float32 小端字节的 `base64`。
+
+服务会将文本包装成消息,把内联图片解码成 PIL 图片,使用官方 processor 的聊天模板,再转换为张量。Tokenizer 自动追加 `<embedding>`,调用方无需手动添加。推理调用 `model.embedding(..., use_cache=False)`,随后按请求维度截取并做 L2 归一化。这里使用 Transformers 自带的模型 tokenizer/processor;vLLM 的 `embedding_chat_template.jinja` 不作为本脚本的额外模板传入。
+
+HTTP 支持批量输入,但模型逐条执行,并串行处理推理请求,以隔离模型内部的可变位置状态。`batch_messages: true` 表示合并 HTTP 请求,不代表 GPU 张量批处理,也不提供 vLLM 的连续批处理调度。
+
+## 参数与限制
+
+| 参数 | 默认值 | 说明 |
+| --- | --- | --- |
+| `--device` | `auto` | 自动按 CUDA → MPS → CPU 选择;可显式指定 `cuda:N`、`mps`、`cpu`,指定不可用设备会报错 |
+| `--dtype` | `auto` | CUDA 支持 BF16 时用 BF16,否则 FP16;MPS 用 BF16;CPU 用 FP32。可显式指定 `bfloat16`、`float16`、`float32` |
+| `--max-tokens` | 8192 | 每个输入经过模板和图片处理后的总 token 上限,超限拒绝,不静默截断 |
+| `--max-pixels` | 1048576 | 每张图片缩放后的像素上限,至少 65536;较小上限会影响高分辨率文档效果 |
+| `--max-batch-size` | 32 | 每个 HTTP 请求的独立输入数上限;WeKnora 的批量设置不应超过它 |
+| `--cpu-threads` | 8 | PyTorch CPU 线程数 |
+| `--mps-fallback` | 关闭 | 允许 PyTorch 将不支持的 MPS 算子放到 CPU 执行 |
+| `--local-files-only` | 关闭 | 仅使用本机文件或已有缓存,不下载权重 |
+| `--host` / `--port` | `127.0.0.1` / `18080` | 监听地址与端口 |
+
+每个请求体最多 32 MiB,包含分块传输;每张图片的 data URI 最多 16 MiB,解码前原图最多 4000 万像素,每个 conversation 最多 5 张图片。图片只接受内联 Base64,不抓取远程 URL 或文件路径。`image_url.detail` 仅为格式兼容,实际分辨率由 `--max-pixels` 控制。所有上限都是此部署脚本的资源设置,不代表 WeMM 模型本身的上限。
+
+## 验证与排障
+
+运行无需下载权重的接口测试:
+
+```bash
+python -m unittest discover -s scripts/wemm-transformers -v
+```
+
+对运行中的服务做实际图文推理测试:
+
+```bash
+python scripts/wemm-transformers/smoke_test.py \
+ --base-url http://127.0.0.1:18080
+```
+
+设置了鉴权时,测试进程同样读取 `WEMM_API_KEY`。测试覆盖批量图文、输出维度与归一化、图文匹配、图片后文本状态隔离及降维;输出耗时和结果到标准输出,不修改知识库。
+
+当前在 Apple M4 Pro 上完成了 **WeMM 2B / MPS / BF16** 和 **WeMM 2B / CPU / FP32** 的完整图文推理测试。2B、4B、9B 的接口维度处理及 CUDA 精度选择有自动化覆盖;4B/9B 的完整权重推理和 CUDA 部署仍需在目标设备运行上述验证脚本。不要将接口测试视为这些设备和规格的性能验证。
+
+常见问题:
+
+- **CUDA/MPS 不可用**:检查 PyTorch wheel、驱动与设备,或显式改为 `--device cpu`;脚本不会将显式设备要求静默降级。
+- **MPS 算子不支持**:尝试 `--mps-fallback`;BF16 不可用时可使用 `--dtype float32`,同时检查内存余量。
+- **显存不足或超时**:减小模型规格、图片像素、输入长度和 WeKnora 批量。该服务为单 worker,增加 HTTP 并发不会自动增加推理吞吐。
+- **模型加载缓慢**:首次需要下载权重;先预下载并固定版本,离线启动时检查目录中同时包含权重、tokenizer、processor 和官方 Python 模型代码。
+- **模型名不匹配**:检查 `/v1/models`;使用自定义目录时建议显式设置 `--served-model-name`。