Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
feat(models): add WeMM embedding presets and Transformers serving (#4238)
lyingbug94fae6de43a3
diff --git a/internal/models/catalog/data/models.generated.json b/internal/models/catalog/data/models.generated.jsonindex 1851b122a..59f8631a3 100644--- a/internal/models/catalog/data/models.generated.json+++ b/internal/models/catalog/data/models.generated.json@@ -2185,6 +2185,75 @@"add_special_tokens": true}}+ },+ {+ "id": "tencent/WeMM-Embedding-2B",+ "match": "*/WeMM-Embedding-2B",+ "type": "Embedding",+ "name": "WeMM-Embedding-2B",+ "aliases": [+ "WeMM-Embedding-2B"+ ],+ "input": [+ "text",+ "image"+ ],+ "dimension": 2048,+ "source": "https://huggingface.co/tencent/WeMM-Embedding-2B",+ "compat": {+ "api": "openai-embeddings",+ "image_format": "messages",+ "text_as_messages": true,+ "batch_messages": true,+ "accepts_truncate_prompt_tokens": false,+ "max_image_batch_size": 5+ }+ },+ {+ "id": "tencent/WeMM-Embedding-4B",+ "match": "*/WeMM-Embedding-4B",+ "type": "Embedding",+ "name": "WeMM-Embedding-4B",+ "aliases": [+ "WeMM-Embedding-4B"+ ],+ "input": [+ "text",+ "image"+ ],+ "dimension": 2560,+ "source": "https://huggingface.co/tencent/WeMM-Embedding-4B",+ "compat": {+ "api": "openai-embeddings",+ "image_format": "messages",+ "text_as_messages": true,+ "batch_messages": true,+ "accepts_truncate_prompt_tokens": false,+ "max_image_batch_size": 5+ }+ },+ {+ "id": "tencent/WeMM-Embedding-9B",+ "match": "*/WeMM-Embedding-9B",+ "type": "Embedding",+ "name": "WeMM-Embedding-9B",+ "aliases": [+ "WeMM-Embedding-9B"+ ],+ "input": [+ "text",+ "image"+ ],+ "dimension": 4096,+ "source": "https://huggingface.co/tencent/WeMM-Embedding-9B",+ "compat": {+ "api": "openai-embeddings",+ "image_format": "messages",+ "text_as_messages": true,+ "batch_messages": true,+ "accepts_truncate_prompt_tokens": false,+ "max_image_batch_size": 5+ }}],"gpustack": [],diff --git a/internal/models/catalog/data/overrides.json b/internal/models/catalog/data/overrides.jsonindex 1123f6600..fd49c67ab 100644--- a/internal/models/catalog/data/overrides.json+++ b/internal/models/catalog/data/overrides.json@@ -1050,6 +1050,45 @@"add_special_tokens": true}}+ },+ {+ "id": "tencent/WeMM-Embedding-2B",+ "match": "*/WeMM-Embedding-2B",+ "type": "Embedding",+ "compat": {+ "api": "openai-embeddings",+ "image_format": "messages",+ "text_as_messages": true,+ "batch_messages": true,+ "accepts_truncate_prompt_tokens": false,+ "max_image_batch_size": 5+ }+ },+ {+ "id": "tencent/WeMM-Embedding-4B",+ "match": "*/WeMM-Embedding-4B",+ "type": "Embedding",+ "compat": {+ "api": "openai-embeddings",+ "image_format": "messages",+ "text_as_messages": true,+ "batch_messages": true,+ "accepts_truncate_prompt_tokens": false,+ "max_image_batch_size": 5+ }+ },+ {+ "id": "tencent/WeMM-Embedding-9B",+ "match": "*/WeMM-Embedding-9B",+ "type": "Embedding",+ "compat": {+ "api": "openai-embeddings",+ "image_format": "messages",+ "text_as_messages": true,+ "batch_messages": true,+ "accepts_truncate_prompt_tokens": false,+ "max_image_batch_size": 5+ }}],"hunyuan": [diff --git a/internal/models/catalog/data/seed.json b/internal/models/catalog/data/seed.jsonindex a2ca16392..02b4b0c55 100644--- a/internal/models/catalog/data/seed.json+++ b/internal/models/catalog/data/seed.json@@ -1454,6 +1454,51 @@"image"],"source": "https://huggingface.co/Qwen/Qwen3-VL-Embedding-8B"+ },+ {+ "id": "tencent/WeMM-Embedding-2B",+ "match": "*/WeMM-Embedding-2B",+ "type": "Embedding",+ "name": "WeMM-Embedding-2B",+ "aliases": [+ "WeMM-Embedding-2B"+ ],+ "input": [+ "text",+ "image"+ ],+ "dimension": 2048,+ "source": "https://huggingface.co/tencent/WeMM-Embedding-2B"+ },+ {+ "id": "tencent/WeMM-Embedding-4B",+ "match": "*/WeMM-Embedding-4B",+ "type": "Embedding",+ "name": "WeMM-Embedding-4B",+ "aliases": [+ "WeMM-Embedding-4B"+ ],+ "input": [+ "text",+ "image"+ ],+ "dimension": 2560,+ "source": "https://huggingface.co/tencent/WeMM-Embedding-4B"+ },+ {+ "id": "tencent/WeMM-Embedding-9B",+ "match": "*/WeMM-Embedding-9B",+ "type": "Embedding",+ "name": "WeMM-Embedding-9B",+ "aliases": [+ "WeMM-Embedding-9B"+ ],+ "input": [+ "text",+ "image"+ ],+ "dimension": 4096,+ "source": "https://huggingface.co/tencent/WeMM-Embedding-9B"}],"gpustack": [],diff --git a/internal/models/embedding/wemm_test.go b/internal/models/embedding/wemm_test.gonew file mode 100644index 000000000..012c2b799--- /dev/null+++ b/internal/models/embedding/wemm_test.go@@ -0,0 +1,127 @@+package embedding++import (+ "context"+ "strings"+ "testing"++ "github.com/Tencent/WeKnora/internal/models/api"+ modelruntime "github.com/Tencent/WeKnora/internal/models/runtime"+ "github.com/Tencent/WeKnora/internal/types"+ "github.com/stretchr/testify/require"+)++// WeMM's vLLM template appends the dedicated <embedding> pooling token.+// Both retrieval sides and images must use that template, including when+// vLLM exposes the local model path instead of the Hugging Face repository ID.+// https://huggingface.co/tencent/WeMM-Embedding-2B/blob/main/embedding_chat_template.jinja+func TestWeMMVLLMPresets(t *testing.T) {+ vendor, ok := modelruntime.Get("generic")+ require.True(t, ok)+ listed := vendor.ModelsByType(types.ModelTypeEmbedding)+ for _, tc := range []struct {+ size string+ dims int+ }{{"2B", 2048}, {"4B", 2560}, {"9B", 4096}} {+ short := "WeMM-Embedding-" + tc.size+ id := "tencent/" + short+ t.Run(tc.size, func(t *testing.T) {+ visible := false+ for _, entry := range listed {+ if entry.ID == id {+ visible = true+ require.Equal(t, tc.dims, entry.Dimension)+ }+ }+ require.True(t, visible, "preset must appear in the embedding picker")+ for _, name := range []string{id, short, "/models/" + short, strings.ToLower(id)} {+ t.Run(name, func(t *testing.T) {+ resolved, err := modelruntime.Resolve(modelruntime.Ref{+ Provider: "generic", Model: name, ModelType: types.ModelTypeEmbedding,+ })+ require.NoError(t, err)+ require.True(t, resolved.Cataloged)+ require.Equal(t, tc.dims, resolved.Spec.Dimension)+ require.Equal(t, api.EmbeddingOpenAI, resolved.EmbeddingAPI)+ require.Equal(t, []string{"text", "image"}, resolved.Spec.Input)+ require.True(t, resolved.Embeddings.BatchMessages)+ require.Equal(t, 5, resolved.Embeddings.ImageBatchLimit())++ up := newUpstream(t)+ e, err := NewEmbedder(Config{+ Source: types.ModelSourceRemote, Provider: "generic", ModelName: name,+ BaseURL: up.url + "/v1", Dimensions: tc.dims, TruncatePromptTokens: 511,+ }, nil, nil)+ require.NoError(t, err)+ require.Equal(t, tc.dims, e.GetDimensions())+ ie, ok := AsImageEmbedder(e)+ require.True(t, ok, "images must work without a row override")+ contexts := []context.Context{context.Background(), types.WithEmbedQuery(context.Background())}+ for _, ctx := range contexts {+ vectors, err := e.BatchEmbed(ctx, []string{"ab", "c"})+ require.NoError(t, err)+ require.Equal(t, [][]float32{{2}, {1}}, vectors)+ }+ vectors, err := ie.BatchEmbedImages(context.Background(), pngs(6))+ require.NoError(t, err)+ require.Equal(t, [][]float32{{1}, {2}, {3}, {4}, {5}, {6}}, vectors)++ conversation := func(part map[string]any) []any {+ return []any{map[string]any{"role": "user", "content": []any{part}}}+ }+ textBatch := []any{+ conversation(map[string]any{"type": "text", "text": "ab"}),+ conversation(map[string]any{"type": "text", "text": "c"}),+ }+ imageBatch := []any{}+ for _, img := range pngs(5) {+ imageBatch = append(imageBatch, conversation(map[string]any{+ "type": "image_url", "image_url": map[string]any{"url": img.DataURI()},+ }))+ }+ messages := []any{textBatch, textBatch, imageBatch, conversation(map[string]any{+ "type": "image_url", "image_url": map[string]any{"url": pngs(6)[5].DataURI()},+ })}+ require.Len(t, up.requests, len(messages), "batch texts and split six images into five and one")+ for i, message := range messages {+ require.Equal(t, "/v1/embeddings", up.requests[i].path)+ require.Equal(t, map[string]any{+ "model": name, "encoding_format": "float",+ "messages": message,+ }, up.requests[i].body, "no raw input, legacy truncation, or extra image prompt")+ }+ })+ }+ })+ }+}++// vLLM reads matryoshka_dimensions from WeMM's config and truncates before+// normalizing. Text and image requests must use the same requested width.+func TestWeMMDimensionOverride(t *testing.T) {+ for _, size := range []string{"2B", "4B", "9B"} {+ t.Run(size, func(t *testing.T) {+ up := newUpstream(t)+ e, err := NewEmbedder(Config{+ Source: types.ModelSourceRemote, Provider: "generic",+ ModelName: "tencent/WeMM-Embedding-" + size, BaseURL: up.url + "/v1",+ Dimensions: 1024, SupportsDimensionOverride: true,+ }, nil, nil)+ require.NoError(t, err)+ _, err = e.Embed(types.WithEmbedQuery(context.Background()), "query")+ require.NoError(t, err)+ ie, ok := AsImageEmbedder(e)+ require.True(t, ok)+ _, err = ie.BatchEmbedImages(context.Background(), pngs(2))+ require.NoError(t, err)+ require.Len(t, up.requests, 2)+ for _, req := range up.requests {+ require.Equal(t, float64(1024), req.body["dimensions"])+ require.NotContains(t, req.body, "truncate_prompt_tokens")+ require.NotContains(t, req.body, "add_special_tokens")+ require.NotContains(t, req.body, "add_generation_prompt")+ require.NotContains(t, req.body, "use_activation")+ }+ })+ }+}diff --git a/internal/models/parity/testdata/catalog-baseline.json b/internal/models/parity/testdata/catalog-baseline.jsonindex 42eae44a8..2e2936206 100644--- a/internal/models/parity/testdata/catalog-baseline.json+++ b/internal/models/parity/testdata/catalog-baseline.json@@ -107,7 +107,16 @@"gemini/KnowledgeQA/gemini-3.7-flash": "570b30391590c3fa380e98e8323bf2187635d18db427eae3d75f9e63def26c8d","gemini/KnowledgeQA/gemini-3.8-flash": "6d40802ed8d3b310e1e91c186f769a4f8a298138538853cf623ad43e59c29140","gemini/KnowledgeQA/gemini-3snapshot": "c5931f2304774b718637f47540e78b66cbf855b4a4189c9fc0cea2040a12951d",+ "generic/Embedding/WeMM-Embedding-2B": "8b6d496d4e145cebe75c192377b9a428be2b049fb23419b07e2f4ed766ee744d",+ "generic/Embedding/WeMM-Embedding-4B": "d1a6ae0fbde3a9828d6392069aef8707817061b553cfc871b2f5432a460b6e83",+ "generic/Embedding/WeMM-Embedding-9B": "aa6fce5495ec995031fb677c0396671b79864ab099587ac6cbc3fa52c0868e39",+ "generic/Embedding/snapshot/WeMM-Embedding-2B": "4832569c35ef95ad39fc75e903fa3a860127c373e06670148301dffbaaf65294",+ "generic/Embedding/snapshot/WeMM-Embedding-4B": "39c3cfdf00aacdcd443309ae6d4940a0ec16300d52aa1944460feae0174353bd",+ "generic/Embedding/snapshot/WeMM-Embedding-9B": "ad2407f2bedd74df45f4cc2debe4a8f44be7c9bf138d1228c55924c477ea9a61","generic/Embedding/snapshotqwen3-vl-embeddingsnapshot": "a1dedaaddb23ad1449d1aef38f0f032d7bcd834bdf079c511c65ec03cccfffe4",+ "generic/Embedding/tencent/WeMM-Embedding-2B": "54141d358d96ea836ce5a0ede691db0e5dff87cf7d33319a4a59d17a0c1ce0c4",+ "generic/Embedding/tencent/WeMM-Embedding-4B": "0dd3bbf5caadcd0c230f5bece1aff09e6ccf6180b095c785244580936aeec68b",+ "generic/Embedding/tencent/WeMM-Embedding-9B": "7176482974a360aa14a1e97dc2e7b508b3ec2526d6f38acbf7f2bf4de595b2d3","generic/KnowledgeQA/gpt-5snapshot": "3e3712264316db008c1465a2d7c395072d85942d0352196e456342cef0b14eb0","generic/KnowledgeQA/o1snapshot": "52340886b4160a0cb9342ae33a7c81304c9dad5d22c2d1c1ce073f89860a30b1","generic/KnowledgeQA/o3snapshot": "782e836769e56c43d9c954ab2d563df7d2f476ae4fb4d588bb8bf3c88ae42ce6",diff --git a/scripts/wemm-transformers/requirements.txt b/scripts/wemm-transformers/requirements.txtnew file mode 100644index 000000000..c9a8dd403--- /dev/null+++ b/scripts/wemm-transformers/requirements.txt@@ -0,0 +1,10 @@+# Install torch/torchvision for the target CPU/CUDA/MPS platform separately.+# Versions below match the WeMM model card and the validated service environment.+transformers==5.2.0+qwen-vl-utils==0.0.14+accelerate>=1.1.0,<2+fastapi>=0.115,<1+pydantic>=2,<3+uvicorn>=0.30,<1+pillow>=10,<13+httpx>=0.27,<1diff --git a/scripts/wemm-transformers/serve.py b/scripts/wemm-transformers/serve.pynew file mode 100644index 000000000..08b36ab3b--- /dev/null+++ b/scripts/wemm-transformers/serve.py@@ -0,0 +1,367 @@+#!/usr/bin/env python3+"""Serve WeMM text/image embeddings with the official Transformers model code."""+import argparse+import base64+import binascii+import hmac+import io+import logging+import os+import threading+import time+from contextlib import asynccontextmanager+from dataclasses import dataclass+from pathlib import Path+from typing import Literal++from fastapi import Depends, FastAPI, Header, HTTPException+from fastapi.exceptions import RequestValidationError+from fastapi.responses import JSONResponse+from PIL import Image, UnidentifiedImageError+from pydantic import BaseModel, ConfigDict, Field, StrictInt++log = logging.getLogger("uvicorn.error")+++@dataclass(frozen=True)+class Settings:+ model: str = "tencent/WeMM-Embedding-2B"+ served_model_name: str | None = None+ revision: str | None = None+ device: str = "auto"+ dtype: str = "auto"+ max_tokens: int = 8192+ max_pixels: int = 1048576+ max_batch_size: int = 32+ cpu_threads: int = 8+ local_files_only: bool = False+ api_key: str = ""++ def __post_init__(self):+ if min(self.max_tokens, self.max_pixels, self.max_batch_size, self.cpu_threads) <= 0:+ raise ValueError("Resource limits and cpu_threads must be positive.")+ if self.max_pixels < 65536:+ raise ValueError("max_pixels must be at least 65536 (the model processor's minimum).")++ @property+ def name(self):+ if self.served_model_name:+ return self.served_model_name+ return Path(self.model).name if Path(self.model).is_dir() else self.model+++class StrictModel(BaseModel):+ model_config = ConfigDict(extra="forbid")+++class TextPart(StrictModel):+ type: Literal["text"]+ text: str+++class ImageURL(StrictModel):+ url: str = Field(max_length=16 * 1024 * 1024)+ detail: Literal["auto", "low", "high"] = "auto"+++class ImagePart(StrictModel):+ type: Literal["image_url"]+ image_url: ImageURL+++class Message(StrictModel):+ role: Literal["system", "user", "assistant"]+ content: str | list[TextPart | ImagePart] = Field(min_length=1)+++class EmbedRequest(StrictModel):+ model: str+ input: str | list[str] | None = None+ messages: list[Message] | list[list[Message]] | None = None+ dimensions: StrictInt | None = None+ encoding_format: Literal["float", "base64"] = "float"+ user: str | None = None+++def bad(message):+ raise HTTPException(400, message)+++def parse_inputs(request, settings, dimensions, default_dimension):+ if request.model != settings.name:+ raise HTTPException(404, f"Model not found: {request.model}")+ dimension = default_dimension if request.dimensions is None else request.dimensions+ if dimension not in dimensions:+ bad(f"dimensions must be one of {dimensions}.")+ if (request.input is None) == (request.messages is None):+ bad("Supply exactly one of input or messages.")+ if request.input is not None:+ texts = [request.input] if isinstance(request.input, str) else request.input+ conversations = [[Message(role="user", content=[TextPart(type="text", text=text)])] for text in texts]+ else:+ messages = request.messages+ conversations = [messages] if messages and isinstance(messages[0], Message) else messages+ if not conversations or len(conversations) > settings.max_batch_size:+ bad(f"Supply between 1 and {settings.max_batch_size} independent inputs.")+ for conversation in conversations:+ if not conversation:+ bad("Each conversation must contain at least one message.")+ chars, images = 0, 0+ for message in conversation:+ parts = message.content+ if isinstance(parts, str):+ chars += len(parts)+ continue+ for part in parts:+ if isinstance(part, TextPart):+ chars += len(part.text)+ else:+ images += 1+ if not part.image_url.url.startswith("data:image/") or ";base64," not in part.image_url.url:+ bad("Images must be base64 data:image/... URLs; network URLs and file paths are unsupported.")+ if images > 5:+ bad("At most 5 images per conversation.")+ if chars > settings.max_tokens * 16:+ bad("Text exceeds the character guard; reduce the input size.")+ return conversations, dimension+++def decode_image(url):+ try:+ data = base64.b64decode(url.split(";base64,", 1)[1], validate=True)+ with Image.open(io.BytesIO(data)) as image:+ if image.width * image.height > 40_000_000:+ bad("Image exceeds 40 megapixels before resizing.")+ # Preserve alpha until qwen-vl-utils composites RGBA onto white,+ # matching the official inference path instead of exposing hidden RGB.+ return image.copy()+ except (ValueError, binascii.Error, UnidentifiedImageError, OSError, Image.DecompressionBombError):+ bad("Invalid image data.")+++def prepare_messages(conversation, max_pixels):+ messages = []+ for message in conversation:+ parts = message.content+ if isinstance(parts, str):+ parts = [TextPart(type="text", text=parts)]+ content = []+ for part in parts:+ if isinstance(part, TextPart):+ content.append({"type": "text", "text": part.text})+ else:+ content.append({"type": "image", "image": decode_image(part.image_url.url), "max_pixels": max_pixels})+ messages.append({"role": message.role, "content": content})+ return messages+++def resolve_device_dtype(torch, settings):+ name = settings.device+ if name == "auto":+ name = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu"+ device = torch.device(name)+ if device.type not in ("cuda", "mps", "cpu"):+ raise ValueError("device must be auto, cpu, mps, cuda or cuda:N.")+ if device.type == "cuda":+ if not torch.cuda.is_available():+ raise ValueError("CUDA is unavailable; install a CUDA-enabled PyTorch build or select another device.")+ if device.index is not None and device.index >= torch.cuda.device_count():+ raise ValueError("The requested CUDA device does not exist.")+ with torch.cuda.device(device):+ bf16 = torch.cuda.is_bf16_supported()+ elif device.type == "mps":+ if not torch.backends.mps.is_available():+ raise ValueError("MPS is unavailable on this machine.")+ bf16 = True+ else:+ bf16 = False+ dtype = settings.dtype+ if dtype == "auto":+ dtype = "float32" if device.type == "cpu" else "bfloat16" if bf16 else "float16"+ if device.type == "cuda" and dtype == "bfloat16" and not bf16:+ raise ValueError("This CUDA device does not support bfloat16; use --dtype float16.")+ return device, getattr(torch, dtype)+++class Engine:+ def __init__(self, settings):+ # Import after CLI setup so --mps-fallback takes effect before torch loads.+ import torch+ from transformers import AutoConfig, AutoModel, AutoProcessor+ self.settings, self.torch = settings, torch+ self.device, self.dtype = resolve_device_dtype(torch, settings)+ self.lock = threading.Lock()+ torch.set_num_threads(settings.cpu_threads)+ options = {"trust_remote_code": True, "local_files_only": settings.local_files_only}+ if settings.revision:+ options["revision"] = settings.revision+ config = AutoConfig.from_pretrained(settings.model, **options)+ self.dimensions = tuple(getattr(config, "matryoshka_dimensions", ()))+ self.default_dimension = config.text_config.hidden_size+ if not self.dimensions or self.default_dimension not in self.dimensions:+ raise ValueError("The model must provide WeMM matryoshka_dimensions including its hidden size.")+ self.processor = AutoProcessor.from_pretrained(settings.model, **options)+ self.model = AutoModel.from_pretrained(+ settings.model, dtype=self.dtype, attn_implementation="sdpa", **options,+ ).to(self.device).eval()+ if not callable(getattr(self.model, "embedding", None)):+ raise ValueError("The model must expose WeMM's embedding() method.")++ def encode(self, conversations, dimension):+ from qwen_vl_utils import process_vision_info+ torch = self.torch+ vectors, total_tokens = [], 0+ # WeMM's model stores rope_deltas. Keep independent conversations serialized,+ # and decode images only for the current input rather than the entire batch.+ with self.lock, torch.inference_mode():+ for conversation in conversations:+ messages = prepare_messages(conversation, self.settings.max_pixels)+ text = self.processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=False)+ images, _ = process_vision_info(messages, image_patch_size=16)+ inputs = self.processor(text=text, images=images, return_tensors="pt")+ count = inputs["input_ids"].shape[-1]+ if count > self.settings.max_tokens:+ bad(f"Input has {count} tokens; limit is {self.settings.max_tokens}. Input was not truncated.")+ last = self.processor.tokenizer.convert_ids_to_tokens(inputs["input_ids"][0, -1].item())+ if last != "<embedding>":+ raise RuntimeError("Processor did not append the official <embedding> token.")+ inputs = inputs.to(self.device)+ # Reset explicitly as well, so older official code revisions are safe+ # when a text request follows a multimodal request.+ self.model.model.rope_deltas = None+ vector = self.model.embedding(**inputs, use_cache=False).float()+ vector = torch.nn.functional.normalize(vector[:, :dimension], dim=-1).cpu()[0]+ if vector.numel() != dimension or not torch.isfinite(vector).all() or vector.norm() < 0.99:+ raise RuntimeError("Invalid embedding output.")+ vectors.append(vector)+ total_tokens += count+ del inputs, images, messages+ return vectors, total_tokens+++class BodyLimit:+ """Bound JSON bodies, including chunked requests, before parsing/base64 decoding."""+ def __init__(self, app, limit=32 * 1024 * 1024):+ self.app, self.limit = app, limit++ async def __call__(self, scope, receive, send):+ if scope["type"] != "http" or scope["method"] != "POST":+ return await self.app(scope, receive, send)+ chunks, size = [], 0+ while True:+ message = await receive()+ if message["type"] == "http.disconnect":+ return+ chunk = message.get("body", b"")+ size += len(chunk)+ if size > self.limit:+ response = JSONResponse({"error": {"message": "Request body exceeds 32 MiB."}}, status_code=413)+ return await response(scope, receive, send)+ chunks.append(chunk)+ if not message.get("more_body", False):+ break+ body = b"".join(chunks)++ async def replay():+ nonlocal body+ if body is not None:+ value, body = body, None+ return {"type": "http.request", "body": value, "more_body": False}+ return await receive()++ await self.app(scope, replay, send)+++def create_app(settings, engine_factory=Engine):+ @asynccontextmanager+ async def lifespan(app):+ started = time.monotonic()+ app.state.engine = engine_factory(settings)+ log.info("Loaded %s on %s/%s in %.1fs", settings.name, app.state.engine.device,+ app.state.engine.dtype, time.monotonic() - started)+ yield+ del app.state.engine++ app = FastAPI(title="WeMM Transformers embedding service", lifespan=lifespan)+ app.add_middleware(BodyLimit)++ def authorize(authorization: str | None = Header(default=None)):+ expected = "Bearer " + settings.api_key+ if settings.api_key and not hmac.compare_digest((authorization or "").encode(), expected.encode()):+ raise HTTPException(401, "Invalid API key.", headers={"WWW-Authenticate": "Bearer"})++ @app.exception_handler(HTTPException)+ async def api_error(_request, exc):+ return JSONResponse(status_code=exc.status_code, headers=exc.headers,+ content={"error": {"message": exc.detail, "code": exc.status_code}})++ @app.exception_handler(RequestValidationError)+ async def validation_error(_request, exc):+ return JSONResponse(status_code=400, content={"error": {+ "message": "Invalid request: " + exc.errors()[0]["msg"], "code": 400,+ }})++ @app.get("/health")+ def health():+ engine = app.state.engine+ return {"status": "ok", "model": settings.name, "backend": "transformers",+ "device": str(engine.device), "dtype": str(engine.dtype), "revision": settings.revision,+ "dimensions": engine.dimensions, "default_dimension": engine.default_dimension,+ "max_tokens": settings.max_tokens, "max_image_pixels": settings.max_pixels}++ @app.get("/v1/models", dependencies=[Depends(authorize)])+ def models():+ return {"object": "list", "data": [{"id": settings.name, "object": "model", "created": 0, "owned_by": "tencent"}]}++ @app.post("/v1/embeddings", dependencies=[Depends(authorize)])+ def embeddings(request: EmbedRequest):+ engine = app.state.engine+ conversations, dimension = parse_inputs(request, settings, engine.dimensions, engine.default_dimension)+ started = time.monotonic()+ try:+ vectors, tokens = engine.encode(conversations, dimension)+ except HTTPException:+ raise+ except Exception:+ log.exception("WeMM inference failed")+ raise HTTPException(500, "WeMM inference failed; inspect the server log.") from None+ data = []+ for index, vector in enumerate(vectors):+ value = (base64.b64encode(vector.numpy().astype("<f4").tobytes()).decode()+ if request.encoding_format == "base64" else vector.tolist())+ data.append({"object": "embedding", "index": index, "embedding": value})+ log.info("Embedded %d inputs, %d tokens, %d dimensions in %.2fs", len(data), tokens,+ dimension, time.monotonic() - started)+ return {"object": "list", "data": data, "model": settings.name,+ "usage": {"prompt_tokens": tokens, "total_tokens": tokens}}++ return app+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--model", default=Settings.model, help="WeMM Hugging Face ID or model directory")+ parser.add_argument("--served-model-name", help="API model ID; defaults to the repo ID or directory basename")+ parser.add_argument("--revision", help="Hugging Face revision (prefer a commit SHA for reproducibility)")+ parser.add_argument("--device", default="auto", help="auto, cuda, cuda:N, mps or cpu")+ parser.add_argument("--dtype", choices=("auto", "bfloat16", "float16", "float32"), default="auto")+ parser.add_argument("--max-tokens", type=int, default=8192)+ parser.add_argument("--max-pixels", type=int, default=1048576)+ parser.add_argument("--max-batch-size", type=int, default=32)+ parser.add_argument("--cpu-threads", type=int, default=8)+ parser.add_argument("--local-files-only", action="store_true")+ parser.add_argument("--mps-fallback", action="store_true", help="Allow PyTorch CPU fallback for unsupported MPS ops")+ parser.add_argument("--host", default="127.0.0.1")+ parser.add_argument("--port", type=int, default=18080)+ args = vars(parser.parse_args())+ host, port = args.pop("host"), args.pop("port")+ if args.pop("mps_fallback"):+ os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"+ settings = Settings(**args, api_key=os.environ.get("WEMM_API_KEY", ""))+ import uvicorn+ # A single process owns the model; do not add workers without separate GPUs.+ uvicorn.run(create_app(settings), host=host, port=port, workers=1)+++if __name__ == "__main__":+ main()diff --git a/scripts/wemm-transformers/smoke_test.py b/scripts/wemm-transformers/smoke_test.pynew file mode 100644index 000000000..0384087de--- /dev/null+++ b/scripts/wemm-transformers/smoke_test.py@@ -0,0 +1,89 @@+#!/usr/bin/env python3+"""Check a running WeMM service with actual text and image inference."""+import argparse+import base64+import io+import json+import math+import os+import time++import httpx+from PIL import Image, ImageDraw+++def dot(a, b):+ return sum(x * y for x, y in zip(a, b))+++def text_message(text):+ return [{"role": "user", "content": [{"type": "text", "text": text}]}]+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--base-url", default="http://127.0.0.1:18080", help="Service root URL, without /v1")+ parser.add_argument("--timeout", type=float, default=180)+ args = parser.parse_args()+ headers = {"Authorization": "Bearer " + os.environ["WEMM_API_KEY"]} if os.environ.get("WEMM_API_KEY") else {}+ results = []+ with httpx.Client(base_url=args.base_url.rstrip("/"), timeout=args.timeout, headers=headers) as client:+ health_response = client.get("/health")+ health_response.raise_for_status()+ health = health_response.json()+ model = health["model"]+ default_dimension = health["default_dimension"]+ response = client.get("/v1/models")+ response.raise_for_status()+ assert model in [row["id"] for row in response.json()["data"]]++ def run(name, **payload):+ started = time.monotonic()+ response = client.post("/v1/embeddings", json={"model": model, **payload})+ response.raise_for_status()+ data = response.json()+ vectors = [row["embedding"] for row in data["data"]]+ assert [row["index"] for row in data["data"]] == list(range(len(vectors)))+ for vector in vectors:+ assert len(vector) == payload.get("dimensions", default_dimension)+ assert all(math.isfinite(x) for x in vector)+ assert abs(math.sqrt(dot(vector, vector)) - 1) < 1e-4+ results.append({"test": name, "seconds": round(time.monotonic() - started, 3),+ "vectors": len(vectors), "dimension": len(vectors[0])})+ print(json.dumps(results[-1]), flush=True)+ return vectors++ texts = ["A red square on a white background.", "A blue circle on a black background."]+ text_vectors = run("batched text messages", messages=[text_message(text) for text in texts])+ images = []+ for color, shape, background in [("red", "rectangle", "white"), ("blue", "ellipse", "black")]:+ image = Image.new("RGB", (256, 256), background)+ getattr(ImageDraw.Draw(image), shape)((40, 40, 216, 216), fill=color)+ buffer = io.BytesIO()+ image.save(buffer, format="PNG")+ images.append([{"role": "user", "content": [{"type": "image_url", "image_url": {+ "url": "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode(),+ }}]}])+ image_vectors = run("batched image messages", messages=images)+ scores = [[dot(text, image) for image in image_vectors] for text in text_vectors]+ assert scores[0][0] > scores[0][1] and scores[1][1] > scores[1][0], scores+ repeated = run("text after image state isolation", messages=text_message(texts[0]))[0]+ assert dot(repeated, text_vectors[0]) > 0.9999+ dimension = min(health["dimensions"])+ short = run("Matryoshka dimensions", messages=text_message(texts[0]), dimensions=dimension)[0]+ prefix = text_vectors[0][:dimension]+ norm = math.sqrt(dot(prefix, prefix))+ assert dot(short, [value / norm for value in prefix]) > 0.9999+ raw = run("raw text compatibility", input=texts[0])[0]+ assert dot(raw, text_vectors[0]) > 0.9999+ for payload in ({"input": "hello", "dimensions": 3}, {"input": "a", "messages": []},+ {"messages": [{"role": "user", "content": [{"type": "image_url", "image_url": {+ "url": "https://example.com/image.png",+ }}]}]}):+ response = client.post("/v1/embeddings", json={"model": model, **payload})+ assert response.status_code == 400, response.text+ print(json.dumps({"status": "passed", "health": health, "tests": results, "text_image_scores": scores}), flush=True)+++if __name__ == "__main__":+ main()diff --git a/scripts/wemm-transformers/test_serve.py b/scripts/wemm-transformers/test_serve.pynew file mode 100644index 000000000..e08f5a53a--- /dev/null+++ b/scripts/wemm-transformers/test_serve.py@@ -0,0 +1,169 @@+"""Contract tests without downloading weights: python -m unittest discover -s scripts/wemm-transformers."""+import asyncio+import base64+import io+import struct+import unittest+from contextlib import nullcontext+from unittest.mock import patch++import torch+from fastapi import HTTPException+from fastapi.testclient import TestClient+from PIL import Image+from serve import BodyLimit, Message, Settings, create_app, decode_image, prepare_messages, resolve_device_dtype+++class FakeEngine:+ device, dtype = "cpu", "float32"++ def __init__(self, settings, dimensions=(64, 128, 256, 512, 1024, 2048)):+ self.dimensions = dimensions+ self.default_dimension = dimensions[-1]+ self.calls = []++ def encode(self, conversations, dimension):+ self.calls.append(conversations)+ vectors = []+ for index, _ in enumerate(conversations):+ vector = torch.zeros(dimension)+ vector[index] = 1+ vectors.append(vector)+ return vectors, len(conversations) * 10+++class ServiceTests(unittest.TestCase):+ def setUp(self):+ self.settings = Settings()+ self.engine = FakeEngine(self.settings)+ self.client = self.enterContext(TestClient(create_app(self.settings, lambda _: self.engine)))++ def post(self, **payload):+ return self.client.post("/v1/embeddings", json={"model": self.settings.name, **payload})++ def test_input_and_single_or_batched_messages(self):+ message = {"role": "user", "content": [{"type": "text", "text": "hello"}]}+ for payload, count in [({"input": "hello"}, 1), ({"input": ["a", "b"]}, 2),+ ({"messages": [message]}, 1), ({"messages": [[message], [message]]}, 2)]:+ with self.subTest(payload=payload):+ response = self.post(**payload)+ self.assertEqual(response.status_code, 200, response.text)+ data = response.json()["data"]+ self.assertEqual([x["index"] for x in data], list(range(count)))+ self.assertTrue(all(len(x["embedding"]) == 2048 for x in data))+ self.assertEqual([x["embedding"][i] for i, x in enumerate(data)], [1] * count)++ def test_all_model_sizes_use_their_own_default_and_allowed_dimensions(self):+ for size, dimensions in [("2B", (64, 128, 256, 512, 1024, 2048)),+ ("4B", (64, 128, 256, 512, 1024, 2560)),+ ("9B", (64, 128, 256, 512, 1024, 2048, 4096))]:+ settings = Settings(model="tencent/WeMM-Embedding-" + size)+ with self.subTest(size=size), TestClient(create_app(settings, lambda s: FakeEngine(s, dimensions))) as client:+ for requested in (None, 1024):+ payload = {"model": settings.name, "input": "hello"}+ if requested is not None:+ payload["dimensions"] = requested+ response = client.post("/v1/embeddings", json=payload)+ self.assertEqual(response.status_code, 200, response.text)+ self.assertEqual(len(response.json()["data"][0]["embedding"]), requested or dimensions[-1])+ payload["dimensions"] = 2049+ self.assertEqual(client.post("/v1/embeddings", json=payload).status_code, 400)+ self.assertEqual(client.get("/health").json()["default_dimension"], dimensions[-1])++ def test_base64_is_little_endian_float32(self):+ response = self.post(input="hello", dimensions=64, encoding_format="base64")+ vector = struct.unpack("<64f", base64.b64decode(response.json()["data"][0]["embedding"]))+ self.assertEqual(vector, (1,) + (0,) * 63)++ def test_invalid_requests_never_reach_inference(self):+ for payload in [dict(input="a", messages=[]), dict(input=[]), dict(messages=[[]]),+ dict(input="a", dimensions=True), dict(input="a", dimensions=0),+ dict(input="a", temperature=0), dict(input=["a"] * 33),+ dict(messages=[{"role": "user", "content": [{"type": "video", "video": "x"}]}]),+ dict(messages=[{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "file:///etc/passwd"}}]}])]:+ with self.subTest(payload=payload):+ self.assertEqual(self.post(**payload).status_code, 400)+ self.assertEqual(self.post(model="wrong", input="a").status_code, 404)+ self.assertEqual(self.engine.calls, [])++ def test_authentication_on_models_and_embeddings(self):+ settings = Settings(api_key="test-only-key")+ with TestClient(create_app(settings, lambda _: self.engine)) as client:+ self.assertEqual(client.get("/health").status_code, 200)+ self.assertEqual(client.get("/v1/models").status_code, 401)+ for token, status in [("Bearer wrong", 401), ("Bearer test-only-key", 200)]:+ response = client.post("/v1/embeddings", json={"model": settings.name, "input": "a"},+ headers={"Authorization": token})+ self.assertEqual(response.status_code, status)+ self.assertEqual(len(self.engine.calls), 1)++ def test_image_decoding_and_invalid_bytes(self):+ buffer = io.BytesIO()+ Image.new("RGB", (64, 64), "red").save(buffer, format="PNG")+ uri = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()+ self.assertEqual(decode_image(uri).size, (64, 64))+ for data in ["!invalid!", base64.b64encode(b"not an image").decode()]:+ with self.assertRaises(HTTPException) as error:+ decode_image("data:image/png;base64," + data)+ self.assertEqual(error.exception.status_code, 400)++ def test_transparent_images_match_official_white_background(self):+ from qwen_vl_utils import process_vision_info+ image = Image.new("RGBA", (256, 256), (0, 0, 0, 0))+ image.putpixel((128, 128), (255, 0, 0, 128))+ buffer = io.BytesIO()+ image.save(buffer, format="PNG")+ uri = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()+ conversation = [Message.model_validate({"role": "user", "content": [+ {"type": "image_url", "image_url": {"url": uri}},+ ]})]+ actual, _ = process_vision_info(prepare_messages(conversation, 1048576), image_patch_size=16)+ expected, _ = process_vision_info([{"role": "user", "content": [+ {"type": "image", "image": image, "max_pixels": 1048576},+ ]}], image_patch_size=16)+ self.assertEqual(actual[0].getpixel((0, 0)), (255, 255, 255))+ self.assertEqual(actual[0].getpixel((128, 128)), (255, 127, 127))+ self.assertEqual(actual[0].tobytes(), expected[0].tobytes())++ def test_cpu_defaults_and_explicit_unavailable_device(self):+ with patch.object(torch.cuda, "is_available", return_value=False), patch.object(torch.backends.mps, "is_available", return_value=False):+ device, dtype = resolve_device_dtype(torch, Settings())+ self.assertEqual(str(device), "cpu")+ self.assertEqual(dtype, torch.float32)+ for requested in ("cuda", "mps"):+ with self.assertRaises(ValueError):+ resolve_device_dtype(torch, Settings(device=requested))++ def test_cuda_auto_dtype_and_explicit_bf16_guard(self):+ with patch.object(torch.cuda, "is_available", return_value=True), patch.object(torch.cuda, "device", return_value=nullcontext()):+ for supported, expected in [(True, torch.bfloat16), (False, torch.float16)]:+ with patch.object(torch.cuda, "is_bf16_supported", return_value=supported):+ device, dtype = resolve_device_dtype(torch, Settings())+ self.assertEqual(str(device), "cuda")+ self.assertEqual(dtype, expected)+ if not supported:+ with self.assertRaises(ValueError):+ resolve_device_dtype(torch, Settings(device="cuda", dtype="bfloat16"))++ def test_body_limit_counts_chunked_bodies(self):+ async def run(chunks):+ received, sent = [], []+ events = iter({"type": "http.request", "body": chunk, "more_body": i < len(chunks) - 1}+ for i, chunk in enumerate(chunks))+ async def receive():+ return next(events)+ async def send(event):+ sent.append(event)+ async def downstream(scope, receive, send):+ received.append((await receive())["body"])+ await BodyLimit(downstream, limit=4)({"type": "http", "method": "POST"}, receive, send)+ return received, sent+ received, _ = asyncio.run(run([b"12", b"34"]))+ self.assertEqual(received, [b"1234"])+ received, sent = asyncio.run(run([b"12", b"345"]))+ self.assertEqual(received, [])+ self.assertEqual(sent[0]["status"], 413)+++if __name__ == "__main__":+ unittest.main()diff --git a/website-docs/03-features/06-models.md b/website-docs/03-features/06-models.mdindex ea8ddb2e7..68cd82a24 100644--- a/website-docs/03-features/06-models.md+++ b/website-docs/03-features/06-models.md@@ -95,6 +95,46 @@| 向量或重排服务响应慢,需要更长的超时 | `{"request_timeout_seconds": 120}` || 自建重排服务返回的是未归一化分数(logit) | `{"score_scale": "logit"}` |+#### WeMM-Embedding++自定义厂商(`generic`)的向量模型列表已内置以下预设,选择后会自动识别图片输入和原生向量维度:++| 模型名称 | 原生维度 |+| --- | --- |+| `tencent/WeMM-Embedding-2B` | 2048 |+| `tencent/WeMM-Embedding-4B` | 2560 |+| `tencent/WeMM-Embedding-9B` | 4096 |++可选择官方 vLLM 部署,或使用本仓库的 [Transformers 部署脚本](../06-development/06-wemm-transformers.md)在 CUDA、MPS 或 CPU 上提供同一 Embedding 接口。MPS 是可选计算设备,服务也可以部署在独立服务器上。++按[官方部署说明](https://github.com/Tencent/WeMM-Embedding#serving)使用 vLLM `0.27.0`,并加载模型目录中的 `embedding_chat_template.jinja`。例如部署 2B,并通过 `--served-model-name` 让服务名称与预设一致:++```bash+MODEL_PATH=/path/to/WeMM-Embedding-2B+vllm serve "$MODEL_PATH" \+ --runner pooling \+ --chat-template "$MODEL_PATH/embedding_chat_template.jinja" \+ --served-model-name tencent/WeMM-Embedding-2B+```++4B、9B 分别替换模型路径与服务名称。在 WeKnora 中选择「自定义」厂商,Base URL 填写服务的 `http://主机:端口/v1`,选择对应向量模型,初次接入保持「自定义输出维度」关闭。若服务没有设置 `--served-model-name`,模型名称须填写服务实际返回的本地路径;短名称 `WeMM-Embedding-2B` 及以 `/WeMM-Embedding-2B` 结尾的路径也能识别预设,4B、9B 同理。任意自定义别名需要另行配置模型能力和协议覆盖。++预设让文本和图片都通过 `messages` 进入模型模板,由模板追加取向量所需的 `<embedding>` 标记;启用 `batch_messages`,将同一批内的独立 conversations 合并请求,每条仍返回各自的向量。图片批量调用最多每批 5 张,超出自动拆分;这是 WeKnora 选择的批量大小,与现有 Qwen3-VL 预设一致,不代表模型或 vLLM 的官方上限,可用 `max_image_batch_size` 按服务承载能力调整。当前知识库图片索引任务仍逐张调用,不会因该配置自动合并多个任务。不发送历史默认的 `truncate_prompt_tokens: 511`;超出服务上下文上限时由服务报错,不会静默截断。无需额外填写兼容 JSON 或图片提示词。若此前用普通文本 `input` 为该模型建立过索引,需要重建向量。++WeMM 的模板已经包含所需特殊 token,保留 vLLM 的 `add_special_tokens: false`、`add_generation_prompt: false` 默认行为,不要照搬 Qwen3-VL 预设的 `add_special_tokens: true`。vLLM 的 Embedding 默认使用末尾 token 并做 L2 归一化,无需额外发送 `normalize` 或关闭 `use_activation`。++需要降维时,开启「自定义输出维度」并选择模型配置中的合法维度,WeKnora 会给文本和图片请求发送相同的 `dimensions`;vLLM 会先截取对应维度,再归一化:++| 模型 | 可选维度 |+| --- | --- |+| 2B | 64、128、256、512、1024、2048 |+| 4B | 64、128、256、512、1024、2560 |+| 9B | 64、128、256、512、1024、2048、4096 |++这些值来自[官方模型配置与说明](https://github.com/Tencent/WeMM-Embedding#model-zoo),vLLM 会拒绝列表之外的维度。更换输出维度后需重建已有向量索引。超时、并发与图片大小限制保持现有默认值,按实际服务性能调整。++知识库还需开启图片向量索引才能直接检索图片内容。当前预设接入文本和图片;视频、任意图文交错输入及扫描 PDF 整页向量化不在当前业务链路的支持范围内。+#### 字段参考下表的「协议默认」是没有任何厂商、目录或覆盖时的取值;具体厂商的实际值以「实际调用方式」面板为准。diff --git a/website-docs/06-development/06-wemm-transformers.md b/website-docs/06-development/06-wemm-transformers.mdnew file mode 100644index 000000000..914f6d12a--- /dev/null+++ b/website-docs/06-development/06-wemm-transformers.md@@ -0,0 +1,205 @@+# WeMM Transformers 部署++使用 [WeMM 官方 Transformers 推理代码](https://github.com/Tencent/WeMM-Embedding#transformers)加载模型,由仓库内的 `scripts/wemm-transformers/serve.py` 提供 HTTP Embedding 服务。服务可以与 WeKnora 部署在同一机器,也可以部署在独立 GPU 服务器上,通过 Base URL 接入。++CUDA、Apple MPS 和 CPU 是可选的计算设备,使用同一份脚本和请求协议。此 HTTP 适配层由 WeKnora 提供,支持文本和图片;模型本身支持的视频等能力尚未在此接口开放。官方 vLLM 部署方式见[模型配置](../03-features/06-models.md#wemm-embedding)。++## 安装依赖++以下命令在 WeKnora 仓库根目录执行,使用 Python 3.12。虚拟环境和模型文件放在仓库外:++```bash+python3.12 -m venv "$HOME/.venvs/weknora-wemm"+source "$HOME/.venvs/weknora-wemm/bin/activate"+python -m pip install --upgrade pip+```++按设备选择一组 PyTorch 安装命令。以下使用已验证的 PyTorch 2.10.0 / torchvision 0.25.0,CUDA wheel 须与驱动兼容;其他 CUDA 版本可参考 [PyTorch 安装说明](https://pytorch.org/get-started/previous-versions/#v2100)。++```bash+# NVIDIA GPU:示例为 CUDA 12.8 wheel+python -m pip install torch==2.10.0 torchvision==0.25.0 \+ --index-url https://download.pytorch.org/whl/cu128++# 或 Apple Silicon / MPS+python -m pip install torch==2.10.0 torchvision==0.25.0++# 或 Linux / Windows CPU wheel+python -m pip install torch==2.10.0 torchvision==0.25.0 \+ --index-url https://download.pytorch.org/whl/cpu+```++安装其余依赖:++```bash+python -m pip install -r scripts/wemm-transformers/requirements.txt+```++Transformers 固定为官方推荐的 `5.2.0`,`qwen-vl-utils` 固定为 `0.0.14`。不要随意替换 Transformers 版本,避免预处理行为变化。脚本会启用 `trust_remote_code=True` 来加载 WeMM 的 `embedding()` 实现,应使用官方权重或已审查的模型目录。++## 启动服务++以 CUDA 上的 2B 为例:++```bash+python scripts/wemm-transformers/serve.py \+ --model tencent/WeMM-Embedding-2B \+ --revision bbd6cd4bf52cfc6716f752a2df80b2706720bd95 \+ --device cuda \+ --host 127.0.0.1 \+ --port 18080+```++首次启动会从 Hugging Face 下载模型到标准缓存,后续复用缓存。`--revision` 可固定模型代码与权重版本;示例 SHA 属于 2B,切换 4B、9B 时应选取对应仓库的版本,不能沿用此 SHA。未指定时使用该仓库默认版本。++已有模型目录时可以离线启动,并指定 WeKnora 调用时使用的模型名称:++```bash+python scripts/wemm-transformers/serve.py \+ --model /srv/models/WeMM-Embedding-2B \+ --served-model-name tencent/WeMM-Embedding-2B \+ --local-files-only \+ --device cuda+```++也可以提前下载并固定版本,例如:++```bash+hf download tencent/WeMM-Embedding-2B \+ --revision bbd6cd4bf52cfc6716f752a2df80b2706720bd95 \+ --local-dir /srv/models/WeMM-Embedding-2B+```++其他设备只需更换参数:++```bash+# Apple Silicon,允许不支持的 MPS 算子回退到 CPU+python scripts/wemm-transformers/serve.py \+ --model tencent/WeMM-Embedding-2B --device mps --mps-fallback++# CPU,默认使用 float32+python scripts/wemm-transformers/serve.py \+ --model tencent/WeMM-Embedding-2B --device cpu+```++4B、9B 将 `--model` 换成对应 ID 或路径。服务从模型配置读取原生维度及 Matryoshka 可选维度,不会将其他规格强制按 2048 维处理:++| 模型 ID | 原生维度 |+| --- | --- |+| `tencent/WeMM-Embedding-2B` | 2048 |+| `tencent/WeMM-Embedding-4B` | 2560 |+| `tencent/WeMM-Embedding-9B` | 4096 |++服务以前台方式运行,按 Ctrl+C 停止。常驻部署时由 systemd、容器或其他进程管理器托管同一命令。一个进程加载一份完整模型到一个设备,不提供多 GPU 切分;选择规格前应预留权重、视觉处理和推理激活所需的显存/内存。++### 独立服务器与鉴权++跨机器部署时使用 `--host 0.0.0.0`,并在服务进程环境中设置 `WEMM_API_KEY`。`/v1/models` 与 `/v1/embeddings` 会验证 `Authorization: Bearer <key>`;WeKnora 的 API Key 填写相同值。服务不直接终止 TLS,跨不可信网络时通过 HTTPS 反向代理访问。`/health` 仅提供运行状态和非敏感模型元数据,不要求 API Key。++默认监听回环地址、未设置 API Key。WeKnora 若运行在 Docker 中,`127.0.0.1` 指向 WeKnora 容器自身,应填写容器能够访问的推理服务地址,例如同一容器网络的服务名,或已配置的宿主机地址。私网地址还需符合 WeKnora 的 SSRF 访问策略,配置方法见[安全说明](../01-getting-started/04-configuration.md)。++## WeKnora 接入++在「设置 → 模型管理」添加:++| 字段 | 填写 |+| --- | --- |+| 类型 | Embedding |+| 来源 / 服务商 | API / 自定义(OpenAI 兼容接口) |+| 模型名称 | `/v1/models` 返回的 ID,推荐保留 `tencent/WeMM-Embedding-2B` 等官方名称 |+| Base URL | `http://推理服务地址:18080/v1`,使用 HTTPS 代理时填写对应地址 |+| API Key | 设置了 `WEMM_API_KEY` 时填写相同值 |+| 图片输入 | 官方名称由预设识别;自定义别名需手动开启 |+| 自定义输出维度 | 初次接入关闭,让连接检测识别原生维度;需要降维时再开启 |+| 后台并发上限 | 建议从 1 开始 |++现有 WeMM 预设的 `messages` 格式同时适用于此服务。推荐在「高级 → 协议兼容覆盖」设置较小批量和较长超时:++```json+{+ "api": "openai-embeddings",+ "image_format": "messages",+ "text_as_messages": true,+ "batch_messages": true,+ "accepts_truncate_prompt_tokens": false,+ "max_batch_size": 4,+ "max_image_batch_size": 2,+ "request_timeout_seconds": 180+}+```++完整配置也适用于尚未包含 WeMM 预设的旧版本或自定义模型别名;此时应手动打开图片输入。点击「测试连接」,确认实际维度后保存。知识库还需开启图片向量索引,配置方式见[模型配置](../03-features/06-models.md)。切换模型或输出维度需要重建已有索引。++## 请求格式与推理流程++接口为 `POST /v1/embeddings`。纯文本可以使用 `input`:++```json+{+ "model": "tencent/WeMM-Embedding-2B",+ "input": ["第一段文本", "第二段文本"],+ "encoding_format": "float"+}+```++文本和图片也可使用 `messages`。以下二维数组包含两个独立输入,分别返回一个文本向量和一个图片向量:++```json+{+ "model": "tencent/WeMM-Embedding-2B",+ "dimensions": 1024,+ "messages": [+ [{"role": "user", "content": [{"type": "text", "text": "白色背景上的红色方块"}]}],+ [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "data:image/png;base64,图片Base64"}}]}]+ ]+}+```++`input` 和 `messages` 只能选一个。单条 conversation 可以使用一维消息数组;文字与图片放在同一 conversation 中会得到一个联合向量。返回值采用 OpenAI Embedding 的 `data[].embedding`、`data[].index` 和 `usage` 结构,编码支持 `float` 与 float32 小端字节的 `base64`。++服务会将文本包装成消息,把内联图片解码成 PIL 图片,使用官方 processor 的聊天模板,再转换为张量。Tokenizer 自动追加 `<embedding>`,调用方无需手动添加。推理调用 `model.embedding(..., use_cache=False)`,随后按请求维度截取并做 L2 归一化。这里使用 Transformers 自带的模型 tokenizer/processor;vLLM 的 `embedding_chat_template.jinja` 不作为本脚本的额外模板传入。++HTTP 支持批量输入,但模型逐条执行,并串行处理推理请求,以隔离模型内部的可变位置状态。`batch_messages: true` 表示合并 HTTP 请求,不代表 GPU 张量批处理,也不提供 vLLM 的连续批处理调度。++## 参数与限制++| 参数 | 默认值 | 说明 |+| --- | --- | --- |+| `--device` | `auto` | 自动按 CUDA → MPS → CPU 选择;可显式指定 `cuda:N`、`mps`、`cpu`,指定不可用设备会报错 |+| `--dtype` | `auto` | CUDA 支持 BF16 时用 BF16,否则 FP16;MPS 用 BF16;CPU 用 FP32。可显式指定 `bfloat16`、`float16`、`float32` |+| `--max-tokens` | 8192 | 每个输入经过模板和图片处理后的总 token 上限,超限拒绝,不静默截断 |+| `--max-pixels` | 1048576 | 每张图片缩放后的像素上限,至少 65536;较小上限会影响高分辨率文档效果 |+| `--max-batch-size` | 32 | 每个 HTTP 请求的独立输入数上限;WeKnora 的批量设置不应超过它 |+| `--cpu-threads` | 8 | PyTorch CPU 线程数 |+| `--mps-fallback` | 关闭 | 允许 PyTorch 将不支持的 MPS 算子放到 CPU 执行 |+| `--local-files-only` | 关闭 | 仅使用本机文件或已有缓存,不下载权重 |+| `--host` / `--port` | `127.0.0.1` / `18080` | 监听地址与端口 |++每个请求体最多 32 MiB,包含分块传输;每张图片的 data URI 最多 16 MiB,解码前原图最多 4000 万像素,每个 conversation 最多 5 张图片。图片只接受内联 Base64,不抓取远程 URL 或文件路径。`image_url.detail` 仅为格式兼容,实际分辨率由 `--max-pixels` 控制。所有上限都是此部署脚本的资源设置,不代表 WeMM 模型本身的上限。++## 验证与排障++运行无需下载权重的接口测试:++```bash+python -m unittest discover -s scripts/wemm-transformers -v+```++对运行中的服务做实际图文推理测试:++```bash+python scripts/wemm-transformers/smoke_test.py \+ --base-url http://127.0.0.1:18080+```++设置了鉴权时,测试进程同样读取 `WEMM_API_KEY`。测试覆盖批量图文、输出维度与归一化、图文匹配、图片后文本状态隔离及降维;输出耗时和结果到标准输出,不修改知识库。++当前在 Apple M4 Pro 上完成了 **WeMM 2B / MPS / BF16** 和 **WeMM 2B / CPU / FP32** 的完整图文推理测试。2B、4B、9B 的接口维度处理及 CUDA 精度选择有自动化覆盖;4B/9B 的完整权重推理和 CUDA 部署仍需在目标设备运行上述验证脚本。不要将接口测试视为这些设备和规格的性能验证。++常见问题:++- **CUDA/MPS 不可用**:检查 PyTorch wheel、驱动与设备,或显式改为 `--device cpu`;脚本不会将显式设备要求静默降级。+- **MPS 算子不支持**:尝试 `--mps-fallback`;BF16 不可用时可使用 `--dtype float32`,同时检查内存余量。+- **显存不足或超时**:减小模型规格、图片像素、输入长度和 WeKnora 批量。该服务为单 worker,增加 HTTP 并发不会自动增加推理吞吐。+- **模型加载缓慢**:首次需要下载权重;先预下载并固定版本,离线启动时检查目录中同时包含权重、tokenizer、processor 和官方 Python 模型代码。+- **模型名不匹配**:检查 `/v1/models`;使用自定义目录时建议显式设置 `--served-model-name`。