Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
refactor(asr): move the wire format into protocol packages, and cover every vendor whose docs offer ASR (#3503)
lyingbug4a9aa06274e6
diff --git a/Makefile b/Makefileindex 816ea5015..67f74ac7a 100644--- a/Makefile+++ b/Makefile@@ -117,7 +117,7 @@ test:.PHONY: model-catalog-checkmodel-catalog-check:go test ./internal/models/parity/ ./internal/models/vendors/ ./internal/models/catalog/ ./internal/models/api/... \- ./internal/models/rerank/ ./internal/models/embedding/+ ./internal/models/rerank/ ./internal/models/embedding/ ./internal/models/asr/# Vendor catalog: report where our model metadata differs from models.dev.# Development aid only — nothing is fetched at runtime and nothing is writtendiff --git a/go.mod b/go.modindex f41067f7f..20ae07003 100644--- a/go.mod+++ b/go.mod@@ -51,7 +51,6 @@ require (github.com/redis/go-redis/v9 v9.14.1github.com/robfig/cron/v3 v3.0.1github.com/santhosh-tekuri/jsonschema/v6 v6.0.2- github.com/sashabaranov/go-openai v1.41.2github.com/sirupsen/logrus v1.9.4github.com/slack-go/slack v0.23.1github.com/spf13/viper v1.21.0diff --git a/go.sum b/go.sumindex e32a11df7..528734cfd 100644--- a/go.sum+++ b/go.sum@@ -2516,8 +2516,6 @@ github.com/samber/lo v1.49.1 h1:4BIFyVfuQSEpluc7Fua+j1NolZHiEHEpaSEKdsH0tew=github.com/samber/lo v1.49.1/go.mod h1:dO6KHFzUKXgP8LDhU0oI8d2hekjXnGOu0DB8Jecxd6o=github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=-github.com/sashabaranov/go-openai v1.41.2 h1:vfPRBZNMpnqu8ELsclWcAvF19lDNgh1t6TVfFFOPiSM=-github.com/sashabaranov/go-openai v1.41.2/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g=github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=github.com/sebdah/goldie/v2 v2.8.0 h1:dZb9wR8q5++oplmEiJT+U/5KyotVD+HNGCAc5gNr8rc=diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.goindex 10ed2f4f7..6eac56512 100644--- a/internal/application/service/knowledge_process.go+++ b/internal/application/service/knowledge_process.go@@ -18,6 +18,7 @@ import ("github.com/Tencent/WeKnora/internal/infrastructure/chunker""github.com/Tencent/WeKnora/internal/infrastructure/docparser""github.com/Tencent/WeKnora/internal/logger"+ "github.com/Tencent/WeKnora/internal/models/asr""github.com/Tencent/WeKnora/internal/models/chat""github.com/Tencent/WeKnora/internal/models/embedding""github.com/Tencent/WeKnora/internal/searchutil"@@ -3699,7 +3700,10 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) ereturn nil}- transcriptionResult, err := asrModel.Transcribe(ctx, convertResult.AudioData, knowledge.FileName)+ // The knowledge base's language hint; the vendor declares where it+ // goes, or that it takes none.+ asrCtx := asr.WithLanguage(ctx, eff.ASRConfig.Language)+ transcriptionResult, err := asrModel.Transcribe(asrCtx, convertResult.AudioData, knowledge.FileName)if err != nil {logger.Errorf(ctx, "[ASR] Transcription failed: %v", err)if isLastRetry {diff --git a/internal/handler/model.go b/internal/handler/model.goindex c2f2a330e..2a73e723e 100644--- a/internal/handler/model.go+++ b/internal/handler/model.go@@ -759,8 +759,7 @@ func (h *ModelHandler) DeleteModel(c *gin.Context) {// validateCatalogParameters rejects protocol / compat overrides the catalog// cannot interpret (unknown extra_config.api, unknown compat keys, bad// reasoning levels) so a typo fails at save time instead of at the first-// call. Chat, VLM, embedding and rerank rows are checked; see-// catalog.ValidateRow.+// call. Every model type is checked; see catalog.ValidateRow.func validateCatalogParameters(name string, modelType types.ModelType, params *types.ModelParameters) error {return catalog.ValidateRow(name, modelType, params)}diff --git a/internal/models/api/endpoint.go b/internal/models/api/endpoint.goindex e9f7ddd9c..5075753a6 100644--- a/internal/models/api/endpoint.go+++ b/internal/models/api/endpoint.go@@ -100,24 +100,33 @@ func (e Endpoint) NewRequest(ctx context.Context, url string, body any, stream bif err != nil {return nil, nil, fmt.Errorf("marshal request: %w", err)}- if err := secutils.ValidateURLForSSRF(url); err != nil {- return nil, nil, fmt.Errorf("endpoint SSRF check failed: %w", err)- }- req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(data))+ req, err := e.newPost(ctx, url, data, "application/json")if err != nil {- return nil, nil, fmt.Errorf("create request: %w", err)+ return nil, nil, err}- req.Header.Set("Content-Type", "application/json")if stream {req.Header.Set("Accept", "text/event-stream")}+ return req, data, nil+}++// newPost prepares an authenticated POST of an already-encoded body.+func (e Endpoint) newPost(ctx context.Context, url string, data []byte, contentType string) (*http.Request, error) {+ if err := secutils.ValidateURLForSSRF(url); err != nil {+ return nil, fmt.Errorf("endpoint SSRF check failed: %w", err)+ }+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(data))+ if err != nil {+ return nil, fmt.Errorf("create request: %w", err)+ }+ req.Header.Set("Content-Type", contentType)if e.Auth != nil {e.Auth(req, data)}// User headers are applied last but the helper skips reserved names so// they cannot clobber auth or content negotiation.secutils.ApplyCustomHeaders(req, e.Headers)- return req, data, nil+ return req, nil}// Do sends the request and returns the response. Non-2xx responses are@@ -125,7 +134,7 @@ func (e Endpoint) NewRequest(ctx context.Context, url string, body any, stream bfunc (e Endpoint) Do(req *http.Request) (*http.Response, error) {resp, err := e.httpClient().Do(req)if err != nil {- return nil, fmt.Errorf("send request: %w", err)+ return nil, &TransportError{Op: "send request", Err: err}}if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))@@ -151,6 +160,11 @@ func (e Endpoint) PostJSON(ctx context.Context, url string, body, out any) error// prompt is what an operator debugs, while a rerank body is a query plus// every candidate chunk. The rerank layer logs a truncated line at Debug// instead, so this would have duplicated it at Info.+ return e.roundTrip(req, out)+}++// roundTrip sends a prepared request and decodes a 2xx reply into out.+func (e Endpoint) roundTrip(req *http.Request, out any) error {resp, err := e.Do(req)if err != nil {return err@@ -158,7 +172,9 @@ func (e Endpoint) PostJSON(ctx context.Context, url string, body, out any) errordefer func() { _ = resp.Body.Close() }()raw, err := io.ReadAll(resp.Body)if err != nil {- return fmt.Errorf("read response: %w", err)+ // The connection broke before the reply arrived whole: nothing was+ // received, so this is the network failing, not the vendor answering.+ return &TransportError{Op: "read response", Err: err}}if out == nil {return nil@@ -169,6 +185,18 @@ func (e Endpoint) PostJSON(ctx context.Context, url string, body, out any) errorreturn nil}+// TransportError is a request that never got a whole answer: DNS, connect,+// TLS, a reset or a timeout, while sending or while reading the reply. It is+// the only failure worth sending again.+type TransportError struct {+ // Op is the phase that failed: "send request" or "read response".+ Op string+ Err error+}++func (e *TransportError) Error() string { return e.Op + ": " + e.Err.Error() }+func (e *TransportError) Unwrap() error { return e.Err }+// HTTPError is a non-2xx vendor reply.type HTTPError struct {StatusCode intdiff --git a/internal/models/api/multipart.go b/internal/models/api/multipart.gonew file mode 100644index 000000000..be47a189e--- /dev/null+++ b/internal/models/api/multipart.go@@ -0,0 +1,76 @@+package api++import (+ "bytes"+ "context"+ "fmt"+ "mime/multipart"+ "net/textproto"+ "path/filepath"+ "strings"+)++// FormField is one text part of a multipart form. Order is kept.+type FormField struct {+ Name string+ Value string+}++// FormFile is the file part of a multipart form.+type FormFile struct {+ Field string+ FileName string+ Data []byte+}++// EncodeMultipart writes fields and one file into a multipart/form-data body+// and returns it with its Content-Type. It is encoded once: those bytes are+// what a signing vendor hashes and what every retry sends again.+//+// The file part carries a filename and no Content-Type of its own. Servers+// identify the format from the extension then — OpenAI asks for "an+// extension-bearing filename", and vox-box (GPUStack's audio backend) falls+// back to guessing from it — whereas a guessed type that a server does not+// list is rejected outright.+func EncodeMultipart(fields []FormField, file FormFile) ([]byte, string, error) {+ var buf bytes.Buffer+ w := multipart.NewWriter(&buf)+ header := make(textproto.MIMEHeader)+ header.Set("Content-Disposition", fmt.Sprintf(`form-data; name=%q; filename=%q`,+ file.Field, filepath.Base(file.FileName)))+ part, err := w.CreatePart(header)+ if err != nil {+ return nil, "", fmt.Errorf("encode form file: %w", err)+ }+ if _, err := part.Write(file.Data); err != nil {+ return nil, "", fmt.Errorf("encode form file: %w", err)+ }+ for _, f := range fields {+ if err := w.WriteField(f.Name, f.Value); err != nil {+ return nil, "", fmt.Errorf("encode form field %s: %w", f.Name, err)+ }+ }+ if err := w.Close(); err != nil {+ return nil, "", fmt.Errorf("encode form: %w", err)+ }+ return buf.Bytes(), w.FormDataContentType(), nil+}++// PostMultipartWithRetry sends an encoded multipart body, retrying transport+// failures only, and decodes the JSON reply into out. It has the guarantees+// of PostJSONWithRetry.+func (e Endpoint) PostMultipartWithRetry(+ ctx context.Context, url string, body []byte, contentType string,+ out any, policy RetryPolicy, label string,+) error {+ if !strings.HasPrefix(contentType, "multipart/") {+ return fmt.Errorf("%s: not a multipart content type: %q", label, contentType)+ }+ return withRetry(ctx, policy, label, func() error {+ req, err := e.newPost(ctx, url, body, contentType)+ if err != nil {+ return err+ }+ return e.roundTrip(req, out)+ })+}diff --git a/internal/models/api/openaichataudio/client.go b/internal/models/api/openaichataudio/client.gonew file mode 100644index 000000000..1e71ddbda--- /dev/null+++ b/internal/models/api/openaichataudio/client.go@@ -0,0 +1,152 @@+// Package openaichataudio implements speech recognition served through the+// chat completions endpoint: one user message whose only content part is+// {"type": "input_audio", "input_audio": {"data": "data:<mime>;base64,..."}},+// answered by an assistant message whose content is the transcript.+//+// Only dedicated recognition models are addressed this way — Alibaba's+// qwen3-asr-flash and Xiaomi's mimo-v2.5-asr — so no instruction is sent: the+// model transcribes whatever it is given. A general audio-capable chat model+// would need a prompt and would answer about the audio rather than transcribe+// it, which is why this is its own protocol and not a chat request.+//+// https://help.aliyun.com/zh/model-studio/qwen-asr-api-reference+// https://mimo.mi.com/docs/en-US/quick-start/usage-guide/audio/Speech-Recognition+package openaichataudio++import (+ "context"+ "encoding/base64"+ "fmt"+ "path/filepath"+ "strings"++ "github.com/Tencent/WeKnora/internal/models/api"+ "github.com/Tencent/WeKnora/internal/models/catalog"+)++// Config is everything the client needs, already resolved by the catalog.+type Config struct {+ Endpoint api.Endpoint+ Settings catalog.TranscriptionsSettings+ Retry api.RetryPolicy+}++// Client talks chat-served speech recognition to one endpoint.+type Client struct {+ cfg Config+}++// New builds a client.+func New(cfg Config) *Client { return &Client{cfg: cfg} }++const defaultPath = "/chat/completions"++func (c *Client) url() string {+ if c.cfg.Endpoint.URL != "" {+ return c.cfg.Endpoint.Resolve("")+ }+ path := defaultPath+ if strings.HasSuffix(strings.TrimRight(c.cfg.Endpoint.BaseURL, "/"), path) {+ path = ""+ }+ return c.cfg.Endpoint.Resolve(path)+}++// audioTypes maps the file extension onto the media type of the data URI.+// The two references show audio/wav and audio/mpeg; the rest are the+// registered types for the containers Alibaba lists. A fixed table rather+// than mime.TypeByExtension, which reads the host's tables and would make the+// request depend on the machine it runs on.+var audioTypes = map[string]string{+ ".wav": "audio/wav",+ ".mp3": "audio/mpeg",+ ".m4a": "audio/mp4",+ ".mp4": "audio/mp4",+ ".aac": "audio/aac",+ ".flac": "audio/flac",+ ".ogg": "audio/ogg",+ ".opus": "audio/opus",+ ".webm": "audio/webm",+ ".amr": "audio/amr",+}++// DataURI is the input_audio payload for one file. Both references show an+// audio media type in the URI, so a file whose extension names no audio+// container is refused rather than labelled application/octet-stream.+func DataURI(audio []byte, fileName string) (string, error) {+ mediaType, ok := audioTypes[strings.ToLower(filepath.Ext(fileName))]+ if !ok {+ return "", fmt.Errorf("cannot tell the audio format of %q from its extension", fileName)+ }+ return "data:" + mediaType + ";base64," + base64.StdEncoding.EncodeToString(audio), nil+}++// BuildRequestBody is the golden-test entry point.+func (c *Client) BuildRequestBody(req api.TranscriptionRequest) (map[string]any, error) {+ uri, err := DataURI(req.Audio, req.FileName)+ if err != nil {+ return nil, err+ }+ // The ceiling is on the whole string as sent — "encoded string size must+ // not exceed 10 MB" — so the data: prefix counts too.+ if limit := c.cfg.Settings.MaxEncodedBytes; limit > 0 && len(uri) > limit {+ return nil, fmt.Errorf(+ "the audio is %.1f MB and %.1f MB once base64-encoded; this model accepts at most %.1f MB encoded",+ float64(len(req.Audio))/(1<<20), float64(len(uri))/(1<<20), float64(limit)/(1<<20))+ }+ body := map[string]any{+ "model": c.cfg.Endpoint.Model,+ "messages": []any{map[string]any{+ "role": "user",+ "content": []any{map[string]any{+ "type": "input_audio",+ "input_audio": map[string]any{"data": uri},+ }},+ }},+ }+ if req.Language != "" && c.cfg.Settings.LanguageParam == catalog.LanguageASROptions {+ body["asr_options"] = map[string]any{"language": req.Language}+ }+ return body, nil+}++type response struct {+ Choices []struct {+ Message struct {+ // Content is a pointer for the same reason as the multipart+ // protocol's text: an absent transcript is an error, an empty one+ // is silence.+ Content *string `json:"content"`+ } `json:"message"`+ } `json:"choices"`+ // Usage.Seconds is the audio length, which Alibaba reports.+ Usage struct {+ Seconds float64 `json:"seconds"`+ } `json:"usage"`+ Error *struct {+ Message string `json:"message"`+ } `json:"error"`+}++// Transcribe sends one audio file.+func (c *Client) Transcribe(ctx context.Context, req api.TranscriptionRequest) (*api.Transcription, error) {+ body, err := c.BuildRequestBody(req)+ if err != nil {+ return nil, err+ }+ var decoded response+ err = c.cfg.Endpoint.PostJSONWithRetry(ctx, c.url(), body, &decoded, c.cfg.Retry, "transcription")+ if err != nil {+ return nil, err+ }+ if decoded.Error != nil && decoded.Error.Message != "" {+ return nil, fmt.Errorf("transcription API error: %s", decoded.Error.Message)+ }+ if len(decoded.Choices) == 0 || decoded.Choices[0].Message.Content == nil {+ return nil, fmt.Errorf("transcription reply carries no message content")+ }+ return &api.Transcription{+ Text: strings.TrimSpace(*decoded.Choices[0].Message.Content),+ Duration: decoded.Usage.Seconds,+ }, nil+}diff --git a/internal/models/api/openaichataudio/golden_test.go b/internal/models/api/openaichataudio/golden_test.gonew file mode 100644index 000000000..8515a02ff--- /dev/null+++ b/internal/models/api/openaichataudio/golden_test.go@@ -0,0 +1,162 @@+package openaichataudio++import (+ "context"+ "encoding/json"+ "net/http"+ "net/http/httptest"+ "testing"++ "github.com/Tencent/WeKnora/internal/models/api"+ "github.com/Tencent/WeKnora/internal/models/catalog"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// qwenASRResponse is the non-streaming response example from Alibaba's+// Qwen-ASR reference, copied from the page:+// https://help.aliyun.com/zh/model-studio/qwen-asr-api-reference+const qwenASRResponse = `{+ "choices": [{+ "finish_reason": "stop",+ "index": 0,+ "message": {+ "annotations": [{"emotion": "neutral", "language": "zh", "type": "audio_info"}],+ "content": "欢迎使用阿里云。",+ "role": "assistant"+ }+ }],+ "created": 1767683986,+ "id": "chatcmpl-487abe5f-d4f2-9363-a877-xxxxxxx",+ "model": "qwen3-asr-flash",+ "object": "chat.completion",+ "usage": {"completion_tokens": 12, "prompt_tokens": 42, "total_tokens": 54}+}`++// withSeconds is a reply carrying the usage.seconds that the reference's+// schema documents ("音频时长(秒)") but its example omits.+const withSeconds = `{"choices":[{"message":{"role":"assistant","content":"x"}}],"usage":{"seconds":3}}`++func newClient(url, model string) *Client {+ return newClientWith(url, model, catalog.TranscriptionsSettings{})+}++func newClientWith(url, model string, settings catalog.TranscriptionsSettings) *Client {+ return New(Config{+ Endpoint: api.Endpoint{BaseURL: url, Model: model, Auth: api.BearerAuth("k")},+ Settings: settings,+ })+}++// The request is the one both references show: a single user message whose+// only part is the audio as a data URI, no instruction, no stream flag.+func TestRequestBodyMatchesTheReferences(t *testing.T) {+ body, err := newClient("https://example.invalid/v1", "qwen3-asr-flash").+ BuildRequestBody(api.TranscriptionRequest{Audio: []byte("ID3"), FileName: "clip.MP3"})+ require.NoError(t, err)+ assert.Equal(t, map[string]any{+ "model": "qwen3-asr-flash",+ "messages": []any{map[string]any{+ "role": "user",+ "content": []any{map[string]any{+ "type": "input_audio",+ "input_audio": map[string]any{"data": "data:audio/mpeg;base64,SUQz"},+ }},+ }},+ }, body)+}++func TestDataURIMediaTypeFollowsTheExtension(t *testing.T) {+ uri, err := DataURI([]byte("RIFF"), "a.wav")+ require.NoError(t, err)+ assert.Equal(t, "data:audio/wav;base64,UklGRg==", uri)++ // No audio type can be named for it, and the references show one.+ _, err = DataURI([]byte("x"), "noext")+ assert.Error(t, err)+}++// Both references cap the string as sent, so the data: prefix counts. A file+// sized to exactly the encoded budget without the prefix goes over it.+func TestEncodedCeilingCountsTheWholeDataURI(t *testing.T) {+ const limit = 1024+ c := newClientWith("https://example.invalid", "m", catalog.TranscriptionsSettings{MaxEncodedBytes: limit})+ audio := make([]byte, limit*3/4) // base64 of this is exactly limit bytes+ _, err := c.BuildRequestBody(api.TranscriptionRequest{Audio: audio, FileName: "a.wav"})+ require.Error(t, err)++ prefix := len("data:audio/wav;base64,")+ fits := make([]byte, (limit-prefix)/4*3)+ _, err = c.BuildRequestBody(api.TranscriptionRequest{Audio: fits, FileName: "a.wav"})+ require.NoError(t, err)+}++func TestLanguageGoesInASROptionsWhereDeclared(t *testing.T) {+ req := api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav", Language: "zh"}++ body, err := newClient("https://example.invalid", "m").BuildRequestBody(req)+ require.NoError(t, err)+ assert.NotContains(t, body, "asr_options", "undeclared, so not sent")++ body, err = newClientWith("https://example.invalid", "m",+ catalog.TranscriptionsSettings{LanguageParam: catalog.LanguageASROptions}).BuildRequestBody(req)+ require.NoError(t, err)+ assert.Equal(t, map[string]any{"language": "zh"}, body["asr_options"])+}++func TestDecodesTheDocumentedResponse(t *testing.T) {+ t.Setenv("SSRF_WHITELIST", "127.0.0.1")+ var path string+ var sent map[string]any+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ path = r.URL.Path+ _ = json.NewDecoder(r.Body).Decode(&sent)+ _, _ = w.Write([]byte(qwenASRResponse))+ }))+ defer server.Close()++ out, err := newClient(server.URL+"/compatible-mode/v1", "qwen3-asr-flash").+ Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("RIFF"), FileName: "a.wav"})+ require.NoError(t, err)+ assert.Equal(t, "/compatible-mode/v1/chat/completions", path)+ assert.Equal(t, &api.Transcription{Text: "欢迎使用阿里云。"}, out)+}++// A reply with no assistant content is not silence.+func TestReplyWithoutContentIsAnError(t *testing.T) {+ t.Setenv("SSRF_WHITELIST", "127.0.0.1")+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ _, _ = w.Write([]byte(`{"choices":[]}`))+ }))+ defer server.Close()++ _, err := newClient(server.URL, "m").+ Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"})+ assert.Error(t, err)+}++func TestDurationFromUsageSeconds(t *testing.T) {+ t.Setenv("SSRF_WHITELIST", "127.0.0.1")+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ _, _ = w.Write([]byte(withSeconds))+ }))+ defer server.Close()++ out, err := newClient(server.URL, "m").+ Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"})+ require.NoError(t, err)+ assert.Equal(t, float64(3), out.Duration)+}++func TestEmptyContentIsSilence(t *testing.T) {+ t.Setenv("SSRF_WHITELIST", "127.0.0.1")+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ _, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":""}}]}`))+ }))+ defer server.Close()++ out, err := newClient(server.URL, "m").+ Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"})+ require.NoError(t, err)+ assert.Equal(t, "", out.Text)+}diff --git a/internal/models/api/openaitranscriptions/client.go b/internal/models/api/openaitranscriptions/client.gonew file mode 100644index 000000000..b09762cac--- /dev/null+++ b/internal/models/api/openaitranscriptions/client.go@@ -0,0 +1,131 @@+// Package openaitranscriptions implements the speech-to-text shape OpenAI+// defined and the compatible servers copied: a multipart POST to+// {base}/audio/transcriptions carrying file and model, answering {text}, or+// {text, segments} when response_format is verbose_json.+//+// response_format is the one optional field that matters, and it differs by+// model: gpt-4o-transcribe and gpt-4o-mini-transcribe accept only json,+// whisper-1 also serves verbose_json, SiliconFlow documents no such field at+// all. json is the default everywhere it is documented, so nothing is sent+// unless catalog.TranscriptionsSettings names a format.+//+// https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions/methods/create+package openaitranscriptions++import (+ "context"+ "fmt"+ "strings"++ "github.com/Tencent/WeKnora/internal/models/api"+ "github.com/Tencent/WeKnora/internal/models/catalog"+)++// Config is everything the client needs, already resolved by the catalog.+type Config struct {+ Endpoint api.Endpoint+ Settings catalog.TranscriptionsSettings+ Retry api.RetryPolicy+}++// Client talks the OpenAI transcription shape to one endpoint.+type Client struct {+ cfg Config+}++// New builds a client.+func New(cfg Config) *Client { return &Client{cfg: cfg} }++func (c *Client) url() string {+ if c.cfg.Endpoint.URL != "" {+ return c.cfg.Endpoint.Resolve("")+ }+ path := c.cfg.Settings.Path+ if strings.HasSuffix(strings.TrimRight(c.cfg.Endpoint.BaseURL, "/"), path) {+ path = ""+ }+ return c.cfg.Endpoint.Resolve(path)+}++// FormFields is the golden-test entry point: the text parts of the form, in+// the order they are written.+func (c *Client) FormFields(language string) []api.FormField {+ fields := []api.FormField{{Name: "model", Value: c.cfg.Endpoint.Model}}+ if format := c.cfg.Settings.ResponseFormat; format != "" {+ fields = append(fields, api.FormField{Name: "response_format", Value: format})+ }+ if language != "" && c.cfg.Settings.LanguageParam == catalog.LanguageForm {+ fields = append(fields, api.FormField{Name: "language", Value: language})+ }+ return fields+}++type response struct {+ // Text is a pointer because its absence is an error, not silence. A+ // transcript of silent audio is a present, empty string; a reply without+ // the field is some other document. vox-box (GPUStack's audio backend)+ // returns its HTTPException from the route instead of raising it, so a+ // failed transcription can arrive as a 2xx whose body is the exception.+ Text *string `json:"text"`+ Segments []struct {+ Start float64 `json:"start"`+ End float64 `json:"end"`+ Text string `json:"text"`+ } `json:"segments"`+ // Duration is top-level in OpenAI's verbose_json and in MiniMax's+ // default json; usage.seconds is where OpenAI, OpenRouter and Requesty+ // report it for models billed by duration.+ Duration float64 `json:"duration"`+ Usage struct {+ Seconds float64 `json:"seconds"`+ } `json:"usage"`+ Detail any `json:"detail"`+ Error *struct {+ Message string `json:"message"`+ } `json:"error"`+}++// Transcribe sends one audio file.+func (c *Client) Transcribe(ctx context.Context, req api.TranscriptionRequest) (*api.Transcription, error) {+ body, contentType, err := api.EncodeMultipart(c.FormFields(req.Language), api.FormFile{+ Field: "file", FileName: req.FileName, Data: req.Audio,+ })+ if err != nil {+ return nil, err+ }+ endpoint := c.cfg.Endpoint+ if req.Language != "" && c.cfg.Settings.LanguageParam == catalog.LanguageHeader {+ headers := make(map[string]string, len(endpoint.Headers)+1)+ for k, v := range endpoint.Headers {+ headers[k] = v+ }+ headers["language"] = req.Language+ endpoint.Headers = headers+ }+ var decoded response+ err = endpoint.PostMultipartWithRetry(+ ctx, c.url(), body, contentType, &decoded, c.cfg.Retry, "transcription",+ )+ if err != nil {+ return nil, err+ }+ if decoded.Error != nil && decoded.Error.Message != "" {+ return nil, fmt.Errorf("transcription API error: %s", decoded.Error.Message)+ }+ if decoded.Text == nil {+ if decoded.Detail != nil {+ return nil, fmt.Errorf("transcription reply carries no text: %v", decoded.Detail)+ }+ return nil, fmt.Errorf("transcription reply carries no text")+ }+ out := &api.Transcription{Text: strings.TrimSpace(*decoded.Text), Duration: decoded.Duration}+ if out.Duration == 0 {+ out.Duration = decoded.Usage.Seconds+ }+ for _, s := range decoded.Segments {+ out.Segments = append(out.Segments, api.TranscriptionSegment{+ Start: s.Start, End: s.End, Text: strings.TrimSpace(s.Text),+ })+ }+ return out, nil+}diff --git a/internal/models/api/openaitranscriptions/golden_test.go b/internal/models/api/openaitranscriptions/golden_test.gonew file mode 100644index 000000000..2e2252b77--- /dev/null+++ b/internal/models/api/openaitranscriptions/golden_test.go@@ -0,0 +1,219 @@+package openaitranscriptions++import (+ "context"+ "io"+ "mime"+ "mime/multipart"+ "net/http"+ "net/http/httptest"+ "testing"++ "github.com/Tencent/WeKnora/internal/models/api"+ "github.com/Tencent/WeKnora/internal/models/catalog"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// jsonResponse is the json example from the reference, copied from the page:+// https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions/methods/create+const jsonResponse = `{+ "text": "text",+ "languages": [{"code": "code"}],+ "logprobs": [{"token": "token", "bytes": [0], "logprob": 0}],+ "usage": {+ "input_tokens": 0, "output_tokens": 0, "total_tokens": 0, "type": "tokens",+ "input_token_details": {"audio_tokens": 0, "text_tokens": 0}+ }+}`++// verboseResponse is the "Segment timestamps" example from the same page,+// cut to its first segment where the page elides the rest with "...".+const verboseResponse = `{+ "task": "transcribe",+ "language": "english",+ "duration": 8.470000267028809,+ "text": "The beach was a popular spot on a hot summer day. ` ++ `People were swimming in the ocean, building sandcastles, and playing beach volleyball.",+ "segments": [+ {+ "id": 0,+ "seek": 0,+ "start": 0.0,+ "end": 3.319999933242798,+ "text": " The beach was a popular spot on a hot summer day.",+ "tokens": [50364, 440, 7534, 390, 257, 3743, 4008, 322, 257, 2368, 4266, 786, 13, 50530],+ "temperature": 0.0,+ "avg_logprob": -0.2860786020755768,+ "compression_ratio": 1.2363636493682861,+ "no_speech_prob": 0.00985979475080967+ }+ ],+ "usage": {"type": "duration", "seconds": 9}+}`++type captured struct {+ path string+ auth string+ language string+ fields map[string]string+ fileName string+ fileType string+ file []byte+}++func serve(t *testing.T, status int, reply string) (*httptest.Server, *captured) {+ t.Helper()+ t.Setenv("SSRF_WHITELIST", "127.0.0.1")+ got := &captured{fields: map[string]string{}}+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ got.path = r.URL.Path+ got.auth = r.Header.Get("Authorization")+ got.language = r.Header.Get("language")+ _, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))+ require.NoError(t, err)+ reader := multipart.NewReader(r.Body, params["boundary"])+ for {+ part, err := reader.NextPart()+ if err == io.EOF {+ break+ }+ require.NoError(t, err)+ data, _ := io.ReadAll(part)+ if part.FormName() == "file" {+ got.fileName, got.fileType, got.file = part.FileName(), part.Header.Get("Content-Type"), data+ continue+ }+ got.fields[part.FormName()] = string(data)+ }+ w.WriteHeader(status)+ _, _ = w.Write([]byte(reply))+ }))+ t.Cleanup(server.Close)+ return server, got+}++func newClient(url, model string, settings catalog.TranscriptionsSettings) *Client {+ if settings.Path == "" {+ settings.Path = catalog.DefaultTranscriptions().Path+ }+ return New(Config{+ Endpoint: api.Endpoint{BaseURL: url, Model: model, Auth: api.BearerAuth("k")},+ Settings: settings,+ })+}++// The baseline form is file and model. gpt-4o-transcribe accepts only json,+// which is the default, so no response_format is sent.+func TestBaselineFormIsFileAndModel(t *testing.T) {+ server, got := serve(t, http.StatusOK, jsonResponse)+ out, err := newClient(server.URL+"/v1", "gpt-4o-transcribe", catalog.TranscriptionsSettings{}).+ Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("RIFF"), FileName: "meeting.wav"})+ require.NoError(t, err)++ assert.Equal(t, "/v1/audio/transcriptions", got.path)+ assert.Equal(t, "Bearer k", got.auth)+ assert.Equal(t, map[string]string{"model": "gpt-4o-transcribe"}, got.fields)+ assert.Equal(t, "meeting.wav", got.fileName, "the extension is how the server identifies the format")+ assert.Empty(t, got.fileType, "a guessed type a server does not list is rejected outright")+ assert.Equal(t, []byte("RIFF"), got.file)+ assert.Equal(t, &api.Transcription{Text: "text"}, out)+}++// The language hint goes where the vendor documents it: a form field on+// OpenAI's shape, a request header on MiniMax's, nowhere otherwise.+func TestLanguageGoesWhereTheVendorDeclares(t *testing.T) {+ for _, tc := range []struct {+ param string+ wantField string+ wantHeader string+ }{+ {param: "", wantField: "", wantHeader: ""},+ {param: catalog.LanguageForm, wantField: "zh", wantHeader: ""},+ {param: catalog.LanguageHeader, wantField: "", wantHeader: "zh"},+ } {+ t.Run("param="+tc.param, func(t *testing.T) {+ server, got := serve(t, http.StatusOK, jsonResponse)+ _, err := newClient(server.URL, "m", catalog.TranscriptionsSettings{LanguageParam: tc.param}).+ Transcribe(context.Background(), api.TranscriptionRequest{+ Audio: []byte("x"), FileName: "a.wav", Language: "zh",+ })+ require.NoError(t, err)+ assert.Equal(t, tc.wantField, got.fields["language"])+ assert.Equal(t, tc.wantHeader, got.language)+ })+ }+}++// The audio length is read from wherever the reply states it, so usage can+// be reported without asking for segments.+func TestDurationIsReadFromTheReply(t *testing.T) {+ for name, reply := range map[string]string{+ // MiniMax's default json, from its reference.+ "top-level duration": `{"text": "x", "duration": 26.325, "trace_id": "021785229015510a2c883cf675b9804d"}`,+ // OpenRouter's reference example.+ "usage.seconds": `{"text": "x", "usage": {"cost": 0.000508, "input_tokens": 83, "output_tokens": 30, ` ++ `"seconds": 9.2, "total_tokens": 113}}`,+ } {+ t.Run(name, func(t *testing.T) {+ server, _ := serve(t, http.StatusOK, reply)+ out, err := newClient(server.URL, "m", catalog.TranscriptionsSettings{}).+ Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"})+ require.NoError(t, err)+ assert.NotZero(t, out.Duration)+ })+ }+}++func TestVerboseJSONCarriesSegments(t *testing.T) {+ server, got := serve(t, http.StatusOK, verboseResponse)+ settings := catalog.TranscriptionsSettings{ResponseFormat: "verbose_json"}+ out, err := newClient(server.URL+"/v1", "whisper-1", settings).+ Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("ID3"), FileName: "a.mp3"})+ require.NoError(t, err)++ assert.Equal(t, "verbose_json", got.fields["response_format"])+ require.Len(t, out.Segments, 1)+ assert.Equal(t, api.TranscriptionSegment{+ Start: 0, End: 3.319999933242798, Text: "The beach was a popular spot on a hot summer day.",+ }, out.Segments[0])+}++// Silence is an empty transcript, and that is a valid answer.+func TestEmptyTextIsSilence(t *testing.T) {+ server, _ := serve(t, http.StatusOK, `{"text": ""}`)+ out, err := newClient(server.URL, "m", catalog.TranscriptionsSettings{}).+ Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"})+ require.NoError(t, err)+ assert.Equal(t, "", out.Text)+}++// A reply without text is not silence. vox-box returns its HTTPException+// from the route rather than raising it, so a failure can arrive as a 2xx+// body shaped like this; reading it as an empty transcript would store "no+// speech detected" for a file that was never transcribed.+func TestReplyWithoutTextIsAnError(t *testing.T) {+ server, _ := serve(t, http.StatusOK,+ `{"status_code": 500, "detail": "Failed to transcribe audio, boom", "headers": null}`)+ _, err := newClient(server.URL, "m", catalog.TranscriptionsSettings{}).+ Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"})+ require.Error(t, err)+ assert.Contains(t, err.Error(), "Failed to transcribe audio")+}++func TestSurfacesTheVendorErrorBody(t *testing.T) {+ // A stand-in rejection in OpenAI's error envelope.+ server, _ := serve(t, http.StatusBadRequest, `{"error":{"message":"stand-in rejection"}}`)+ _, err := newClient(server.URL, "gpt-4o-transcribe", catalog.TranscriptionsSettings{}).+ Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"})+ require.Error(t, err)+ assert.Contains(t, err.Error(), "stand-in rejection")+}++func TestDoesNotDoubleThePath(t *testing.T) {+ server, got := serve(t, http.StatusOK, jsonResponse)+ _, err := newClient(server.URL+"/v1/audio/transcriptions", "m", catalog.TranscriptionsSettings{}).+ Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"})+ require.NoError(t, err)+ assert.Equal(t, "/v1/audio/transcriptions", got.path)+}diff --git a/internal/models/api/retry.go b/internal/models/api/retry.goindex 2dd9a02e8..64e61c75e 100644--- a/internal/models/api/retry.go+++ b/internal/models/api/retry.go@@ -49,12 +49,23 @@ func (p RetryPolicy) delay(attempt int) time.Duration {// Here the last error is captured in the enclosing scope on purpose and the// function cannot fall through without it.//-// Only transport failures are retried. A non-2xx reply is the vendor-// answering, and repeating a rejected request neither fixes it nor tells the-// operator anything new.+// Only transport failures are retried, including a reply cut off while it+// was being read. A non-2xx reply is the vendor answering, and repeating a+// rejected request neither fixes it nor tells the operator anything new; a+// reply that arrived whole but does not decode is the same answer every time,+// and sending it again would bill the request twice.func (e Endpoint) PostJSONWithRetry(ctx context.Context, url string, body, out any, policy RetryPolicy, label string,) error {+ return withRetry(ctx, policy, label, func() error {+ return e.PostJSON(ctx, url, body, out)+ })+}++// withRetry runs send until it succeeds, fails with anything other than a+// TransportError, or the budget runs out. The last error lives in the+// enclosing scope and the loop cannot fall through without it.+func withRetry(ctx context.Context, policy RetryPolicy, label string, send func() error) error {var lastErr errorfor attempt := 0; attempt <= policy.MaxRetries; attempt++ {if attempt > 0 {@@ -67,12 +78,12 @@ func (e Endpoint) PostJSONWithRetry(return ctx.Err()}}- err := e.PostJSON(ctx, url, body, out)+ err := send()if err == nil {return nil}- var httpErr *HTTPError- if errors.As(err, &httpErr) {+ var transportErr *TransportError+ if !errors.As(err, &transportErr) {return err}lastErr = errdiff --git a/internal/models/api/retry_test.go b/internal/models/api/retry_test.goindex 4d7c8d142..b056a4dab 100644--- a/internal/models/api/retry_test.go+++ b/internal/models/api/retry_test.go@@ -78,6 +78,25 @@ func TestPostJSONWithRetryDoesNotRetryAVendorRejection(t *testing.T) {assert.Equal(t, 1, attempts, "a rejection must not be repeated")}+// A reply that arrived but does not decode is not a transport failure: the+// vendor did the work, and sending the request again bills it again.+func TestPostJSONWithRetryDoesNotRetryAnUndecodableReply(t *testing.T) {+ t.Setenv("SSRF_WHITELIST", "127.0.0.1")+ attempts := 0+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ attempts+++ _, _ = w.Write([]byte(`not json`))+ }))+ defer server.Close()++ var out map[string]any+ err := Endpoint{}.PostJSONWithRetry(+ context.Background(), server.URL, map[string]any{}, &out, fastPolicy(), "test",+ )+ require.Error(t, err)+ assert.Equal(t, 1, attempts)+}+func TestPostJSONWithRetryStopsWhenTheContextIsDone(t *testing.T) {t.Setenv("SSRF_WHITELIST", "127.0.0.1")server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))@@ -101,3 +120,54 @@ func TestRetryPolicyBackoffDoublesAndIsCapped(t *testing.T) {assert.Equal(t, 4*time.Second, p.delay(4), "capped")assert.Zero(t, RetryPolicy{MaxRetries: 2}.delay(1), "no base delay means no waiting")}++// A reply cut off mid-body never arrived, so it is sent again.+func TestPostJSONWithRetryRetriesAReplyCutOffWhileReading(t *testing.T) {+ t.Setenv("SSRF_WHITELIST", "127.0.0.1")+ attempts := 0+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ attempts+++ if attempts == 1 {+ // Promise more than is sent, then hang up.+ w.Header().Set("Content-Length", "100")+ _, _ = w.Write([]byte(`{"a":`))+ w.(http.Flusher).Flush() // the status and half a body are on the wire+ hj, ok := w.(http.Hijacker)+ require.True(t, ok)+ conn, _, err := hj.Hijack()+ require.NoError(t, err)+ _ = conn.Close()+ return+ }+ _, _ = w.Write([]byte(`{"a":1}`))+ }))+ defer server.Close()++ var out map[string]any+ err := Endpoint{}.PostJSONWithRetry(+ context.Background(), server.URL, map[string]any{}, &out, fastPolicy(), "test",+ )+ require.NoError(t, err)+ assert.Equal(t, 2, attempts)+ assert.Equal(t, float64(1), out["a"])+}++// The phase that failed is named, so the log says whether the request or the+// reply was lost.+func TestReadFailureIsReportedAsSuch(t *testing.T) {+ t.Setenv("SSRF_WHITELIST", "127.0.0.1")+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ w.Header().Set("Content-Length", "100")+ _, _ = w.Write([]byte(`{"a":`))+ w.(http.Flusher).Flush()+ conn, _, err := w.(http.Hijacker).Hijack()+ require.NoError(t, err)+ _ = conn.Close()+ }))+ defer server.Close()++ err := Endpoint{}.PostJSON(context.Background(), server.URL, map[string]any{}, &map[string]any{})+ var transportErr *TransportError+ require.ErrorAs(t, err, &transportErr)+ assert.Equal(t, "read response", transportErr.Op)+}diff --git a/internal/models/api/transcriptions.go b/internal/models/api/transcriptions.gonew file mode 100644index 000000000..6ace55fa9--- /dev/null+++ b/internal/models/api/transcriptions.go@@ -0,0 +1,56 @@+package api++import "context"++// TranscriptionAPI names a speech-to-text wire protocol. Like RerankAPI and+// EmbeddingAPI it is its own type, so no other modality's value validates.+type TranscriptionAPI string++// The transcription protocols WeKnora speaks.+const (+ // TranscriptionOpenAI is POST {base}/audio/transcriptions as a multipart+ // form carrying file and model, answering {text} — or {text, segments}+ // when verbose_json is asked for.+ TranscriptionOpenAI TranscriptionAPI = "openai-transcriptions"+ // TranscriptionChatAudio is a dedicated speech-recognition model behind+ // POST {base}/chat/completions: the audio goes in as a base64 data URI in+ // an input_audio content part and the transcript comes back as the+ // assistant message. Alibaba's qwen3-asr-flash and Xiaomi's mimo-v2.5-asr+ // take no instruction; the model only transcribes.+ TranscriptionChatAudio TranscriptionAPI = "openai-chat-audio"+)++// Known reports whether the value names a protocol this build implements.+func (a TranscriptionAPI) Known() bool {+ return a == TranscriptionOpenAI || a == TranscriptionChatAudio+}++// TranscriptionSegment is one timed stretch of a transcript.+type TranscriptionSegment struct {+ Start float64+ End float64+ Text string+}++// Transcription is what a protocol client returns.+type Transcription struct {+ Text string+ Segments []TranscriptionSegment+ // Duration is the audio length in seconds when the reply states it —+ // OpenAI-shaped duration or usage.seconds — and 0 otherwise.+ Duration float64+}++// TranscriptionRequest is one file to transcribe.+type TranscriptionRequest struct {+ Audio []byte+ FileName string+ // Language is the operator's hint, empty for auto-detection. Where it is+ // sent, and whether at all, is the vendor's declaration.+ Language string+}++// Transcriber turns one audio file into text.+type Transcriber interface {+ Transcribe(ctx context.Context, req TranscriptionRequest) (*Transcription, error)+}diff --git a/internal/models/asr/asr.go b/internal/models/asr/asr.goindex c241d52ef..913e5e22a 100644--- a/internal/models/asr/asr.go+++ b/internal/models/asr/asr.go@@ -2,6 +2,7 @@ package asrimport ("context"+ "strings""github.com/Tencent/WeKnora/internal/types")@@ -17,6 +18,27 @@ type Segment struct {type TranscriptionResult struct {Text string `json:"text"`Segments []Segment `json:"segments,omitempty"`+ // Duration is the audio length in seconds when the vendor reports it,+ // which several do without segments; 0 when unknown.+ Duration float64 `json:"duration,omitempty"`+}++type languageKey struct{}++// WithLanguage attaches the operator's language hint (for example "zh") to+// a transcription. Empty and "auto" mean auto-detection, which is every+// vendor's default, and send nothing.+func WithLanguage(ctx context.Context, language string) context.Context {+ return context.WithValue(ctx, languageKey{}, language)+}++func languageFrom(ctx context.Context) string {+ language, _ := ctx.Value(languageKey{}).(string)+ language = strings.TrimSpace(language)+ if strings.EqualFold(language, "auto") {+ return ""+ }+ return language}// ASR defines the interface for Automatic Speech Recognition model operations.@@ -35,7 +57,10 @@ type Config struct {ModelName stringAPIKey stringModelID string- Language string // optional: specify language for transcription+ // Provider is the vendor id stored on the row; empty detects it from+ // BaseURL.+ Provider string+ ExtraConfig map[string]string// CustomHeaders 允许在调用远程 API 时附加自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。CustomHeaders map[string]string}@@ -53,13 +78,14 @@ func ConfigFromModel(m *types.Model) *Config {BaseURL: m.Parameters.BaseURL,ModelName: m.Name,Source: m.Source,+ Provider: m.Parameters.Provider,+ ExtraConfig: m.Parameters.ExtraConfig,CustomHeaders: m.Parameters.CustomHeaders,}}// NewASR creates an ASR instance based on the provided configuration.-// All ASR vendors use the OpenAI-compatible /v1/audio/transcriptions API.func NewASR(config *Config) (ASR, error) {- a, err := NewOpenAIASR(config)+ a, err := newASR(config)return wrapASRLangfuse(a, err)}diff --git a/internal/models/asr/config_from_model_test.go b/internal/models/asr/config_from_model_test.goindex 7d2275eca..ecf3bf1dc 100644--- a/internal/models/asr/config_from_model_test.go+++ b/internal/models/asr/config_from_model_test.go@@ -14,6 +14,8 @@ func TestConfigFromModel(t *testing.T) {Parameters: types.ModelParameters{BaseURL: "https://api.example.com/v1",APIKey: "sk",+ Provider: "siliconflow",+ ExtraConfig: map[string]string{"remote_model_name": "r"},CustomHeaders: map[string]string{"X": "y"},},}@@ -24,6 +26,11 @@ func TestConfigFromModel(t *testing.T) {if cfg.BaseURL != "https://api.example.com/v1" || cfg.APIKey != "sk" {t.Errorf("connection fields mismatch: %+v", cfg)}+ // The vendor decides the wire format; without it every row resolves to+ // the generic runtime.+ if cfg.Provider != "siliconflow" || cfg.ExtraConfig["remote_model_name"] != "r" {+ t.Errorf("vendor fields not propagated: %+v", cfg)+ }if cfg.CustomHeaders["X"] != "y" {t.Errorf("CustomHeaders not propagated: %+v", cfg.CustomHeaders)}diff --git a/internal/models/asr/langfuse_wrapper.go b/internal/models/asr/langfuse_wrapper.goindex 7564c0592..d0bc7ecfc 100644--- a/internal/models/asr/langfuse_wrapper.go+++ b/internal/models/asr/langfuse_wrapper.go@@ -8,7 +8,8 @@ import (// langfuseASR wraps an ASR implementation and reports each Transcribe call// as a Langfuse generation observation. Audio bytes are not uploaded — we-// record file name, audio size and duration (derived from segment end times)+// record file name, audio size and duration (as the vendor reports it, or+// from segment end times)// so usage can be billed by second/minute, which is how most ASR providers// price their services.type langfuseASR struct {@@ -44,8 +45,13 @@ func (l *langfuseASR) Transcribe(ctx context.Context, audioBytes []byte, fileNamif result != nil {output["text"] = result.Textoutput["segment_count"] = len(result.Segments)- if n := len(result.Segments); n > 0 {+ // The vendor's own figure when it gives one; otherwise the end of+ // the last segment, which only verbose replies carry.+ duration = result.Duration+ if n := len(result.Segments); duration == 0 && n > 0 {duration = result.Segments[n-1].End+ }+ if duration > 0 {output["duration_seconds"] = duration}}diff --git a/internal/models/asr/openai.go b/internal/models/asr/openai.godeleted file mode 100644index 66dee48e8..000000000--- a/internal/models/asr/openai.go+++ /dev/null@@ -1,150 +0,0 @@-package asr--import (- "bytes"- "context"- "encoding/json"- "fmt"- "path/filepath"- "strings"- "time"-- "github.com/Tencent/WeKnora/internal/logger"- secutils "github.com/Tencent/WeKnora/internal/utils"- openai "github.com/sashabaranov/go-openai"-)--const (- asrDefaultTimeout = 300 * time.Second // audio transcription can be slow-)--// OpenAIASR implements ASR via an OpenAI-compatible audio transcriptions API.-type OpenAIASR struct {- modelName string- modelID string- client *openai.Client- baseURL string- language string-}--// NewOpenAIASR creates an OpenAI-compatible ASR instance.-func NewOpenAIASR(config *Config) (*OpenAIASR, error) {- if err := validateASRBaseURL(config.BaseURL); err != nil {- return nil, err- }-- apiCfg := openai.DefaultConfig(config.APIKey)- if config.BaseURL != "" {- apiCfg.BaseURL = config.BaseURL- }- httpClient := newASRHTTPClient(asrDefaultTimeout)-- // 注入用户自定义 HTTP header(类似 OpenAI Python SDK 的 extra_headers)- if len(config.CustomHeaders) > 0 {- apiCfg.HTTPClient = secutils.WrapHTTPClientWithHeaders(httpClient, config.CustomHeaders)- } else {- apiCfg.HTTPClient = httpClient- }-- return &OpenAIASR{- modelName: config.ModelName,- modelID: config.ModelID,- client: openai.NewClientWithConfig(apiCfg),- baseURL: config.BaseURL,- language: config.Language,- }, nil-}--// Transcribe sends audio bytes to the OpenAI-compatible audio transcriptions API.-func (s *OpenAIASR) Transcribe(ctx context.Context, audioBytes []byte, fileName string) (*TranscriptionResult, error) {- if len(audioBytes) == 0 {- return nil, fmt.Errorf("audio bytes are empty")- }-- // Ensure fileName has an extension for proper MIME type detection- if fileName == "" {- fileName = "audio.mp3"- }-- logger.Infof(ctx, "[ASR] Calling OpenAI-compatible transcription API, model=%s, baseURL=%s, audioSize=%d, file=%s",- s.modelName, s.baseURL, len(audioBytes), fileName)-- req := openai.AudioRequest{- Model: s.modelName,- FilePath: fileName,- Reader: bytes.NewReader(audioBytes),- Format: openai.AudioResponseFormatVerboseJSON,- }-- if s.language != "" {- req.Language = s.language- }-- resp, err := s.client.CreateTranscription(ctx, req)- if err != nil {- return nil, fmt.Errorf("ASR transcription request failed: %w", err)- }-- jsonData, err := json.Marshal(resp)- if err != nil {- return nil, fmt.Errorf("marshal transcription response: %w", err)- }- logger.Debugf(ctx, "[ASR] Transcription response: %s", string(jsonData))-- text := strings.TrimSpace(resp.Text)- logger.Infof(ctx, "[ASR] Transcription completed, text length=%d", len(text))-- var segments []Segment- for _, s := range resp.Segments {- segments = append(segments, Segment{- Start: s.Start,- End: s.End,- Text: strings.TrimSpace(s.Text),- })- }-- return &TranscriptionResult{- Text: text,- Segments: segments,- }, nil-}--func (s *OpenAIASR) GetModelName() string { return s.modelName }-func (s *OpenAIASR) GetModelID() string { return s.modelID }--// DetectAudioFormat returns a file extension hint for the given audio bytes.-func DetectAudioFormat(data []byte, fileName string) string {- if fileName != "" {- ext := strings.ToLower(filepath.Ext(fileName))- if ext != "" {- return ext- }- }- // Try to detect from magic bytes- if len(data) >= 4 {- // MP3: starts with ID3 or 0xFF 0xFB- if data[0] == 'I' && data[1] == 'D' && data[2] == '3' {- return ".mp3"- }- if data[0] == 0xFF && (data[1]&0xE0) == 0xE0 {- return ".mp3"- }- // FLAC: starts with "fLaC"- if data[0] == 'f' && data[1] == 'L' && data[2] == 'a' && data[3] == 'C' {- return ".flac"- }- // OGG: starts with "OggS"- if data[0] == 'O' && data[1] == 'g' && data[2] == 'g' && data[3] == 'S' {- return ".ogg"- }- // WAV: starts with "RIFF"- if data[0] == 'R' && data[1] == 'I' && data[2] == 'F' && data[3] == 'F' {- return ".wav"- }- }- // M4A: check for ftyp box- if len(data) >= 8 && data[4] == 'f' && data[5] == 't' && data[6] == 'y' && data[7] == 'p' {- return ".m4a"- }- return ".mp3" // default-}diff --git a/internal/models/asr/protocol.go b/internal/models/asr/protocol.gonew file mode 100644index 000000000..b16b56ae3--- /dev/null+++ b/internal/models/asr/protocol.go@@ -0,0 +1,183 @@+package asr++import (+ "context"+ "fmt"+ "path/filepath"+ "slices"+ "strings"+ "time"++ "github.com/Tencent/WeKnora/internal/logger"+ "github.com/Tencent/WeKnora/internal/models/api"+ "github.com/Tencent/WeKnora/internal/models/api/openaichataudio"+ "github.com/Tencent/WeKnora/internal/models/api/openaitranscriptions"+ "github.com/Tencent/WeKnora/internal/models/catalog"+ // catalog.Resolve answers from the vendor catalog, which is empty until+ // the vendor packages have run their init.+ _ "github.com/Tencent/WeKnora/internal/models/vendors"+ "github.com/Tencent/WeKnora/internal/types"+)++// retryPolicy is the transport-error retry budget, which is none. The+// knowledge pipeline retries a failed transcription as a whole task, and a+// client-level retry on top would turn one hung 300-second upload into+// several.+var retryPolicy = func() api.RetryPolicy { return api.RetryPolicy{} }++// newASR resolves the catalog and returns the protocol client for the+// configured model. It mirrors rerank.newReranker and+// embedding.newRemoteEmbedder: the vendor's facts decide the protocol, the+// URL and the credential, and this function knows no vendor names.+func newASR(config *Config) (ASR, error) {+ if config == nil {+ return nil, fmt.Errorf("asr config is nil")+ }+ if strings.TrimSpace(config.ModelName) == "" {+ return nil, fmt.Errorf("model name is required")+ }+ resolved, err := catalog.Resolve(catalog.Ref{+ Provider: config.Provider,+ Model: config.ModelName,+ BaseURL: config.BaseURL,+ ModelType: types.ModelTypeASR,+ Extra: config.ExtraConfig,+ })+ if err != nil {+ return nil, err+ }+ // A vendor that does not declare speech recognition has not been checked+ // for it, and its Endpoint hook may still compute a path — Azure's does.+ // A row that named the vendor is refused; a row that named none and was+ // matched by its URL is what the pre-catalog client served: an+ // OpenAI-compatible endpoint at that URL.+ if !resolved.Vendor.SupportsType(types.ModelTypeASR) {+ if strings.TrimSpace(config.Provider) != "" {+ return nil, fmt.Errorf("%s does not offer speech recognition in this build", resolved.Vendor.Name)+ }+ resolved, err = catalog.Resolve(catalog.Ref{+ Provider: catalog.GenericID,+ Model: config.ModelName,+ BaseURL: config.BaseURL,+ ModelType: types.ModelTypeASR,+ Extra: config.ExtraConfig,+ })+ if err != nil {+ return nil, err+ }+ }+ if err := validateASRBaseURL(resolved.BaseURL); err != nil {+ return nil, err+ }++ vendor := resolved.Vendor+ creds := catalog.Credentials{APIKey: config.APIKey}+ if creds.APIKey == "" {+ creds.APIKey = vendor.DefaultAPIKey+ }+ // Vendor headers first so a user header cannot silently replace a vendor+ // beta flag, matching chat.NewRemoteChat.+ headers := make(map[string]string, len(vendor.Headers)+len(config.CustomHeaders))+ for k, v := range vendor.Headers {+ headers[k] = v+ }+ for k, v := range config.CustomHeaders {+ headers[k] = v+ }+ settings := resolved.Transcriptions+ endpoint := api.Endpoint{+ BaseURL: resolved.BaseURL,+ Model: resolved.RemoteModel,+ ModelID: config.ModelID,+ Auth: vendor.AuthFunc(vendor.API, creds),+ Headers: headers,+ Client: newASRHTTPClient(time.Duration(settings.RequestTimeout) * time.Second),+ }+ if vendor.Endpoint != nil {+ if url, query := vendor.Endpoint(catalog.EndpointRequest{+ BaseURL: resolved.BaseURL,+ Model: resolved.RemoteModel,+ ModelType: types.ModelTypeASR,+ API: vendor.API,+ Extra: config.ExtraConfig,+ }); url != "" {+ if err := validateASRBaseURL(url); err != nil {+ return nil, err+ }+ endpoint.URL, endpoint.Query = url, query+ }+ }++ var client api.Transcriber+ switch resolved.TranscriptionAPI {+ case api.TranscriptionOpenAI:+ client = openaitranscriptions.New(openaitranscriptions.Config{+ Endpoint: endpoint, Settings: settings, Retry: retryPolicy(),+ })+ case api.TranscriptionChatAudio:+ client = openaichataudio.New(openaichataudio.Config{+ Endpoint: endpoint, Settings: settings, Retry: retryPolicy(),+ })+ default:+ return nil, fmt.Errorf("unsupported transcription api %q for provider %s",+ resolved.TranscriptionAPI, vendor.ID)+ }+ return &protocolASR{+ inner: client,+ settings: settings,+ vendor: vendor.Name,+ endpoint: resolved.BaseURL,+ modelName: config.ModelName,+ modelID: config.ModelID,+ }, nil+}++// protocolASR adapts a protocol client to the ASR interface and refuses an+// upload the vendor has documented it will not take, before sending it.+type protocolASR struct {+ inner api.Transcriber+ settings catalog.TranscriptionsSettings+ vendor string+ endpoint string+ modelName string+ modelID string+}++func (a *protocolASR) GetModelName() string { return a.modelName }+func (a *protocolASR) GetModelID() string { return a.modelID }++func (a *protocolASR) Transcribe(ctx context.Context, audio []byte, fileName string) (*TranscriptionResult, error) {+ if len(audio) == 0 {+ return nil, fmt.Errorf("audio bytes are empty")+ }+ if limit := a.settings.MaxFileBytes; limit > 0 && len(audio) > limit {+ return nil, fmt.Errorf("%s transcription: the audio is %.1f MB; %s accepts at most %d MB per file",+ a.modelName, float64(len(audio))/(1<<20), a.vendor, limit>>20)+ }+ // The server identifies the format from the extension.+ if fileName == "" {+ fileName = "audio.mp3"+ }+ if formats := a.settings.Formats; len(formats) > 0 {+ ext := strings.TrimPrefix(strings.ToLower(filepath.Ext(fileName)), ".")+ if !slices.Contains(formats, ext) {+ return nil, fmt.Errorf("%s transcription: %s accepts %s audio, not %q",+ a.modelName, a.vendor, strings.Join(formats, "/"), fileName)+ }+ }+ logger.Infof(ctx, "[ASR] transcribing model=%s endpoint=%s size=%d file=%s",+ a.modelName, a.endpoint, len(audio), fileName)++ out, err := a.inner.Transcribe(ctx, api.TranscriptionRequest{+ Audio: audio, FileName: fileName, Language: languageFrom(ctx),+ })+ if err != nil {+ return nil, fmt.Errorf("ASR transcription request failed: %w", err)+ }+ result := &TranscriptionResult{Text: out.Text, Duration: out.Duration}+ for _, s := range out.Segments {+ result.Segments = append(result.Segments, Segment{Start: s.Start, End: s.End, Text: s.Text})+ }+ logger.Infof(ctx, "[ASR] transcription completed, text length=%d", len(result.Text))+ return result, nil+}diff --git a/internal/models/asr/transport_security_test.go b/internal/models/asr/transport_security_test.godeleted file mode 100644index a2d97025a..000000000--- a/internal/models/asr/transport_security_test.go+++ /dev/null@@ -1,26 +0,0 @@-package asr--import (- "testing"-- secutils "github.com/Tencent/WeKnora/internal/utils"-)--func withASRSSRFWhitelist(t *testing.T, raw string) {- t.Helper()- t.Setenv("SSRF_WHITELIST", raw)- secutils.ResetSSRFWhitelistForTest()- t.Cleanup(secutils.ResetSSRFWhitelistForTest)-}--func TestOpenAIASRRejectsInternalBaseURL(t *testing.T) {- withASRSSRFWhitelist(t, "")-- _, err := NewOpenAIASR(&Config{- BaseURL: "http://169.254.169.254/latest/meta-data/",- ModelName: "asr-test",- })- if err == nil {- t.Fatalf("NewOpenAIASR returned nil error for blocked internal BaseURL")- }-}diff --git a/internal/models/asr/wire_test.go b/internal/models/asr/wire_test.gonew file mode 100644index 000000000..503921fab--- /dev/null+++ b/internal/models/asr/wire_test.go@@ -0,0 +1,356 @@+package asr++import (+ "context"+ "encoding/json"+ "io"+ "mime"+ "mime/multipart"+ "net/http"+ "net/http/httptest"+ "sync/atomic"+ "testing"++ "github.com/Tencent/WeKnora/internal/models/catalog"+ "github.com/Tencent/WeKnora/internal/types"+ secutils "github.com/Tencent/WeKnora/internal/utils"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func allowLoopback(t *testing.T) {+ t.Helper()+ t.Setenv("SSRF_WHITELIST", "127.0.0.1")+ secutils.ResetSSRFWhitelistForTest()+ t.Cleanup(secutils.ResetSSRFWhitelistForTest)+}++type form struct {+ path string+ auth string+ fields map[string]string+ fileName string+ language string // the language request header+ // body is set instead of fields when the request was JSON: the+ // chat-served recognisers take the audio as a data URI.+ body map[string]any+}++// upstream records the form it receives and answers the documented json+// shape, adding a segment when verbose_json was asked for.+func upstream(t *testing.T) (string, *form, *atomic.Int32) {+ t.Helper()+ allowLoopback(t)+ got := &form{fields: map[string]string{}}+ calls := &atomic.Int32{}+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ calls.Add(1)+ got.path, got.auth, got.language = r.URL.Path, r.Header.Get("Authorization"), r.Header.Get("language")+ mediaType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))+ require.NoError(t, err)+ if mediaType == "application/json" {+ require.NoError(t, json.NewDecoder(r.Body).Decode(&got.body))+ _, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":" hello "}}]}`))+ return+ }+ reader := multipart.NewReader(r.Body, params["boundary"])+ for {+ part, err := reader.NextPart()+ if err == io.EOF {+ break+ }+ require.NoError(t, err)+ data, _ := io.ReadAll(part)+ if part.FormName() == "file" {+ got.fileName = part.FileName()+ continue+ }+ got.fields[part.FormName()] = string(data)+ }+ if got.fields["response_format"] == "verbose_json" {+ _, _ = w.Write([]byte(`{"text":" hello ","segments":[{"start":0,"end":1.5,"text":" hello "}]}`))+ return+ }+ _, _ = w.Write([]byte(`{"text":" hello "}`))+ }))+ t.Cleanup(server.Close)+ return server.URL, got, calls+}++// TestTranscriptionWireFormatPerVendor pins the form each ASR vendor is sent.+// response_format appears only where the vendor documents it for that model:+// the pre-catalog client sent verbose_json to every one of them, which+// gpt-4o-transcribe rejects ("the only supported format is json") and which+// vox-box's FunASR backend answers with an undecodable bare string.+func TestTranscriptionWireFormatPerVendor(t *testing.T) {+ chatAudio := func(model string) map[string]any {+ return map[string]any{+ "model": model,+ "messages": []any{map[string]any{+ "role": "user",+ "content": []any{map[string]any{+ "type": "input_audio",+ "input_audio": map[string]any{"data": "data:audio/wav;base64,UklGRg=="},+ }},+ }},+ }+ }+ cases := []struct {+ name, provider, model, base string+ wantPath string+ wantFields map[string]string+ wantBody map[string]any+ wantSegments int+ }{+ {+ name: "openai whisper-1 serves segments", provider: "openai", model: "whisper-1", base: "/v1",+ wantPath: "/v1/audio/transcriptions",+ wantFields: map[string]string{"model": "whisper-1", "response_format": "verbose_json"},+ wantSegments: 1,+ },+ {+ name: "openai gpt-4o-transcribe accepts only json", provider: "openai", model: "gpt-4o-transcribe",+ base: "/v1", wantPath: "/v1/audio/transcriptions",+ wantFields: map[string]string{"model": "gpt-4o-transcribe"},+ },+ {+ name: "openai model not yet in the catalog", provider: "openai", model: "gpt-5-transcribe",+ base: "/v1", wantPath: "/v1/audio/transcriptions",+ wantFields: map[string]string{"model": "gpt-5-transcribe"},+ },+ {+ name: "siliconflow documents file and model only", provider: "siliconflow",+ model: "FunAudioLLM/SenseVoiceSmall", base: "/v1", wantPath: "/v1/audio/transcriptions",+ wantFields: map[string]string{"model": "FunAudioLLM/SenseVoiceSmall"},+ },+ {+ name: "gpustack stays on json for its FunASR backend", provider: "gpustack",+ model: "SenseVoiceSmall", base: "/v1", wantPath: "/v1/audio/transcriptions",+ wantFields: map[string]string{"model": "SenseVoiceSmall"},+ },+ {+ name: "generic stays on json", provider: "generic", model: "whisper-large-v3",+ base: "/v1", wantPath: "/v1/audio/transcriptions",+ wantFields: map[string]string{"model": "whisper-large-v3"},+ },+ {+ name: "zhipu glm-asr on the OpenAI shape", provider: "zhipu", model: "glm-asr-2512",+ base: "/api/paas/v4", wantPath: "/api/paas/v4/audio/transcriptions",+ wantFields: map[string]string{"model": "glm-asr-2512"},+ },+ {+ name: "minimax on its own path", provider: "minimax", model: "asr-1.0",+ base: "/v1", wantPath: "/v1/speech_to_text",+ wantFields: map[string]string{"model": "asr-1.0"},+ },+ {+ name: "openrouter", provider: "openrouter", model: "openai/whisper-large-v3",+ base: "/api/v1", wantPath: "/api/v1/audio/transcriptions",+ wantFields: map[string]string{"model": "openai/whisper-large-v3"},+ },+ {+ name: "requesty whisper-1 serves segments", provider: "requesty", model: "openai/whisper-1",+ base: "/v1", wantPath: "/v1/audio/transcriptions",+ wantFields: map[string]string{"model": "openai/whisper-1", "response_format": "verbose_json"},+ wantSegments: 1,+ },+ {+ name: "litellm proxy", provider: "litellm", model: "whisper",+ base: "/v1", wantPath: "/v1/audio/transcriptions",+ wantFields: map[string]string{"model": "whisper"},+ },+ {+ name: "aliyun qwen3-asr-flash is served on chat", provider: "aliyun", model: "qwen3-asr-flash",+ base: "/compatible-mode/v1", wantPath: "/compatible-mode/v1/chat/completions",+ wantBody: chatAudio("qwen3-asr-flash"),+ },+ {+ name: "mimo asr is served on chat", provider: "mimo", model: "mimo-v2.5-asr",+ base: "/v1", wantPath: "/v1/chat/completions",+ wantBody: chatAudio("mimo-v2.5-asr"),+ },+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ url, got, _ := upstream(t)+ a, err := NewASR(&Config{+ Source: types.ModelSourceRemote, Provider: tc.provider, BaseURL: url + tc.base,+ ModelName: tc.model, APIKey: "k",+ })+ require.NoError(t, err)+ out, err := a.Transcribe(context.Background(), []byte("RIFF"), "meeting.wav")+ require.NoError(t, err)++ assert.Equal(t, tc.wantPath, got.path)+ assert.Equal(t, "Bearer k", got.auth)+ if tc.wantBody != nil {+ assert.Equal(t, tc.wantBody, got.body)+ } else {+ assert.Equal(t, tc.wantFields, got.fields)+ assert.Equal(t, "meeting.wav", got.fileName)+ }+ assert.Equal(t, "hello", out.Text)+ assert.Len(t, out.Segments, tc.wantSegments)+ })+ }+}++// A file over the documented ceiling is refused before it is uploaded: the+// vendor would reject it anyway, after the whole upload.+func TestOversizedAudioIsRefusedBeforeUpload(t *testing.T) {+ url, _, calls := upstream(t)+ a, err := NewASR(&Config{+ Source: types.ModelSourceRemote, Provider: "openai", BaseURL: url + "/v1",+ ModelName: "whisper-1", APIKey: "k",+ })+ require.NoError(t, err)++ _, err = a.Transcribe(context.Background(), make([]byte, 25<<20+1), "long.mp3")+ require.Error(t, err)+ assert.Contains(t, err.Error(), "at most 25 MB")+ assert.Zero(t, calls.Load(), "nothing should have been sent")++ _, err = a.Transcribe(context.Background(), make([]byte, 25<<20), "exactly.mp3")+ require.NoError(t, err, "the ceiling itself is allowed")+}++// The chat-served recognisers cap the data URI as sent at 10 MB, prefix+// included, so three quarters of 10 MB of audio is already too much.+func TestChatServedCeilingCountsTheWholeDataURI(t *testing.T) {+ url, _, calls := upstream(t)+ a, err := NewASR(&Config{+ Source: types.ModelSourceRemote, Provider: "aliyun", BaseURL: url + "/compatible-mode/v1",+ ModelName: "qwen3-asr-flash", APIKey: "k",+ })+ require.NoError(t, err)++ _, err = a.Transcribe(context.Background(), make([]byte, 10<<20*3/4), "long.wav")+ require.Error(t, err)+ assert.Zero(t, calls.Load())++ prefix := len("data:audio/wav;base64,")+ _, err = a.Transcribe(context.Background(), make([]byte, (10<<20-prefix)/4*3), "fits.wav")+ require.NoError(t, err)+}++// Only qwen3-asr-flash takes the audio in the request. Alibaba's other+// recognition models want a public URL or an asynchronous task, so a row+// naming one is refused with the reason rather than sent a chat request.+func TestAlibabaRecognitionModelsOtherThanQwenASRFlashAreRefused(t *testing.T) {+ _, err := NewASR(&Config{+ Source: types.ModelSourceRemote, Provider: "aliyun",+ BaseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1", ModelName: "paraformer-v2", APIKey: "k",+ })+ require.Error(t, err)+ assert.Contains(t, err.Error(), "public file URL")+}++// A vendor that does not declare speech recognition is not trusted with it.+// Azure's Endpoint hook still knows a transcription path, but nobody has+// verified it; a row that names Azure is refused, and a row that names no+// vendor gets what the pre-catalog client sent: the OpenAI shape at its URL.+func TestVendorsWithoutASRAreNotRoutedThroughTheirHooks(t *testing.T) {+ _, err := NewASR(&Config{+ Source: types.ModelSourceRemote, Provider: "azure_openai",+ BaseURL: "https://example.openai.azure.com", ModelName: "whisper", APIKey: "k",+ })+ require.Error(t, err)++ // Detection matches URL patterns as substrings, so a path carrying Azure's+ // host detects as Azure while the request still reaches the test server.+ url, got, _ := upstream(t)+ base := url + "/openai.azure.com/v1"+ require.Equal(t, "azure_openai", catalog.DetectByURL(base))+ a, err := NewASR(&Config{+ Source: types.ModelSourceRemote, BaseURL: base, ModelName: "whisper", APIKey: "k",+ })+ require.NoError(t, err)+ _, err = a.Transcribe(context.Background(), []byte("RIFF"), "a.wav")+ require.NoError(t, err)+ assert.Equal(t, "/openai.azure.com/v1/audio/transcriptions", got.path, "not Azure's /openai/v1 path")+ assert.Equal(t, "Bearer k", got.auth, "not Azure's api-key header")+}++// A format the vendor does not list is refused before upload.+func TestUndocumentedFormatIsRefusedBeforeUpload(t *testing.T) {+ url, _, calls := upstream(t)+ a, err := NewASR(&Config{+ Source: types.ModelSourceRemote, Provider: "zhipu", BaseURL: url + "/api/paas/v4",+ ModelName: "glm-asr-2512", APIKey: "k",+ })+ require.NoError(t, err)+ _, err = a.Transcribe(context.Background(), []byte("x"), "memo.m4a")+ require.Error(t, err)+ assert.Contains(t, err.Error(), "wav/mp3")+ assert.Zero(t, calls.Load())+}++// The knowledge base's language hint reaches each vendor where its reference+// puts it, and nowhere for a vendor that documents none.+func TestLanguageHintGoesWhereEachVendorDocumentsIt(t *testing.T) {+ for _, tc := range []struct {+ name, provider, model, base string+ check func(t *testing.T, got *form)+ }{+ {"openai form field", "openai", "whisper-1", "/v1", func(t *testing.T, got *form) {+ assert.Equal(t, "zh", got.fields["language"])+ assert.Empty(t, got.language)+ }},+ {"minimax header", "minimax", "asr-1.0", "/v1", func(t *testing.T, got *form) {+ assert.Equal(t, "zh", got.language)+ assert.NotContains(t, got.fields, "language")+ }},+ {"aliyun asr_options", "aliyun", "qwen3-asr-flash", "/compatible-mode/v1", func(t *testing.T, got *form) {+ assert.Equal(t, map[string]any{"language": "zh"}, got.body["asr_options"])+ }},+ {+ "siliconflow documents none", "siliconflow", "FunAudioLLM/SenseVoiceSmall", "/v1",+ func(t *testing.T, got *form) {+ assert.NotContains(t, got.fields, "language")+ assert.Empty(t, got.language)+ },+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ url, got, _ := upstream(t)+ a, err := NewASR(&Config{+ Source: types.ModelSourceRemote, Provider: tc.provider, BaseURL: url + tc.base,+ ModelName: tc.model, APIKey: "k",+ })+ require.NoError(t, err)+ _, err = a.Transcribe(WithLanguage(context.Background(), "zh"), []byte("RIFF"), "a.wav")+ require.NoError(t, err)+ tc.check(t, got)+ })+ }++ // "auto" is every vendor's default and is not sent.+ url, got, _ := upstream(t)+ a, err := NewASR(&Config{+ Source: types.ModelSourceRemote, Provider: "openai", BaseURL: url + "/v1", ModelName: "whisper-1", APIKey: "k",+ })+ require.NoError(t, err)+ _, err = a.Transcribe(WithLanguage(context.Background(), "auto"), []byte("RIFF"), "a.wav")+ require.NoError(t, err)+ assert.NotContains(t, got.fields, "language")+}++func TestEmptyAudioIsAnError(t *testing.T) {+ url, _, calls := upstream(t)+ a, err := NewASR(&Config{Source: types.ModelSourceRemote, Provider: "generic", BaseURL: url, ModelName: "m"})+ require.NoError(t, err)+ _, err = a.Transcribe(context.Background(), nil, "a.wav")+ assert.Error(t, err)+ assert.Zero(t, calls.Load())+}++func TestInternalBaseURLIsRefused(t *testing.T) {+ t.Setenv("SSRF_WHITELIST", "")+ secutils.ResetSSRFWhitelistForTest()+ t.Cleanup(secutils.ResetSSRFWhitelistForTest)+ _, err := NewASR(&Config{+ Source: types.ModelSourceRemote, Provider: "generic",+ BaseURL: "http://169.254.169.254/latest/meta-data/", ModelName: "asr-test",+ })+ assert.Error(t, err)+}diff --git a/internal/models/catalog/compat.go b/internal/models/catalog/compat.goindex 234d8a2b6..f2a479d07 100644--- a/internal/models/catalog/compat.go+++ b/internal/models/catalog/compat.go@@ -346,6 +346,8 @@ type VendorCompat struct {Rerank RerankCompat `json:"rerank,omitempty"`// Embeddings is the embedding protocol overlay, likewise one per vendor.Embeddings EmbeddingsCompat `json:"embeddings,omitempty"`+ // Transcriptions is the speech-to-text overlay, likewise one per vendor.+ Transcriptions TranscriptionsCompat `json:"transcriptions,omitempty"`}// apply writes the set fields of an overlay struct onto a settings structdiff --git a/internal/models/catalog/registry.go b/internal/models/catalog/registry.goindex 91767fde5..fc132a49a 100644--- a/internal/models/catalog/registry.go+++ b/internal/models/catalog/registry.go@@ -34,6 +34,9 @@ func Register(v *Vendor) {if v.RerankAPI == "" && v.SupportsType(types.ModelTypeRerank) {v.RerankAPI = api.RerankCohere}+ if v.TranscriptionAPI == "" && v.SupportsType(types.ModelTypeASR) {+ v.TranscriptionAPI = api.TranscriptionOpenAI+ }if v.EmbeddingAPI == "" && v.SupportsType(types.ModelTypeEmbedding) {v.EmbeddingAPI = api.EmbeddingOpenAI}diff --git a/internal/models/catalog/resolve.go b/internal/models/catalog/resolve.goindex ef6202278..fa8542c22 100644--- a/internal/models/catalog/resolve.go+++ b/internal/models/catalog/resolve.go@@ -81,6 +81,11 @@ type Resolved struct {// for the same reason.EmbeddingAPI api.EmbeddingAPIEmbeddings EmbeddingsSettings++ // TranscriptionAPI and Transcriptions are filled only for an ASR+ // reference.+ TranscriptionAPI api.TranscriptionAPI+ Transcriptions TranscriptionsSettings}// Resolve merges the vendor, catalog entry, extra-config and per-row@@ -122,6 +127,9 @@ func Resolve(ref Ref) (*Resolved, error) {if modelType == types.ModelTypeEmbedding {return resolveEmbeddings(ref, vendor, spec, cataloged, baseURL)}+ if modelType == types.ModelTypeASR {+ return resolveTranscriptions(ref, vendor, spec, cataloged, baseURL)+ }resolvedAPI := spec.APIif forced := strings.TrimSpace(ref.Extra[ExtraAPI]); forced != "" {@@ -537,3 +545,62 @@ func resolveEmbeddings(}return out, nil}++// resolveTranscriptions merges the speech-to-text layers, in the same order+// as resolveEmbeddings: protocol default, vendor compat, the catalog entry,+// the row's own compat, then remote_model_name.+func resolveTranscriptions(+ ref Ref, vendor *Vendor, spec ModelSpec, cataloged bool, baseURL string,+) (*Resolved, error) {+ protocol := vendor.TranscriptionAPI+ if protocol == "" {+ protocol = api.TranscriptionOpenAI+ }+ settings := DefaultTranscriptions()+ apply(&settings, &vendor.Compat.Transcriptions)+ for _, raw := range []json.RawMessage{spec.Compat, ref.Override.CompatJSON()} {+ if len(raw) == 0 {+ continue+ }+ overlay := &TranscriptionsCompat{}+ if err := decodeCompat(raw, overlay); err != nil {+ return nil, fmt.Errorf("transcriptions compat: %w", err)+ }+ apply(&settings, overlay)+ }+ if settings.API != "" {+ protocol = settings.API+ }+ if !protocol.Known() {+ return nil, fmt.Errorf("catalog: unknown transcription api %q for %s/%s", protocol, vendor.ID, spec.ID)+ }+ settings.API = protocol+ switch settings.LanguageParam {+ case "", LanguageForm, LanguageHeader, LanguageASROptions:+ default:+ return nil, fmt.Errorf("catalog: unknown language_param %q for %s/%s",+ settings.LanguageParam, vendor.ID, spec.ID)+ }+ // Checked after every layer: an entry may lift a vendor-wide refusal for+ // the one model that takes the audio in the request.+ if settings.UnsupportedReason != "" {+ return nil, fmt.Errorf(+ "catalog: %s does not serve %q through a protocol this build implements: %s",+ vendor.ID, spec.ID, settings.UnsupportedReason,+ )+ }++ out := &Resolved{+ Vendor: vendor,+ Spec: spec,+ Cataloged: cataloged,+ BaseURL: baseURL,+ RemoteModel: ref.Model,+ TranscriptionAPI: protocol,+ Transcriptions: settings,+ }+ if override := strings.TrimSpace(ref.Extra[ExtraRemoteModelName]); override != "" {+ out.RemoteModel = override+ }+ return out, nil+}diff --git a/internal/models/catalog/transcriptions.go b/internal/models/catalog/transcriptions.gonew file mode 100644index 000000000..0a7b74ce6--- /dev/null+++ b/internal/models/catalog/transcriptions.go@@ -0,0 +1,70 @@+package catalog++import "github.com/Tencent/WeKnora/internal/models/api"++// Where a transcription protocol puts the operator's language hint. Each+// vendor documents one place, or none.+const (+ // LanguageForm is a multipart field named language (OpenAI's shape,+ // ISO-639-1).+ LanguageForm = "form"+ // LanguageHeader is an HTTP request header named language (MiniMax,+ // BCP-47).+ LanguageHeader = "header"+ // LanguageASROptions is asr_options.language in a chat-served request+ // body (Alibaba, Xiaomi).+ LanguageASROptions = "asr_options"+)++// TranscriptionsCompat is the overlay form (every field optional) of the+// speech-to-text settings. JSON keys are the names used in models.json.+type TranscriptionsCompat struct {+ API *api.TranscriptionAPI `json:"api,omitempty"`+ Path *string `json:"path,omitempty"`+ // ResponseFormat is sent as response_format when non-empty. OpenAI's+ // gpt-4o-transcribe and gpt-4o-mini-transcribe accept only json, which is+ // also the default, so most models leave it unset.+ ResponseFormat *string `json:"response_format,omitempty"`+ // LanguageParam names where the language hint goes: form, header or+ // asr_options; empty sends none.+ LanguageParam *string `json:"language_param,omitempty"`+ // MaxFileBytes is the documented upload ceiling for the audio itself.+ MaxFileBytes *int `json:"max_file_bytes,omitempty"`+ // MaxEncodedBytes is the documented ceiling on the base64 data URI, for+ // the protocols that send the audio that way.+ MaxEncodedBytes *int `json:"max_encoded_bytes,omitempty"`+ // Formats lists the file extensions the vendor documents, without the+ // dot. Empty means the vendor documents no closed list.+ Formats []string `json:"formats,omitempty"`+ RequestTimeout *int `json:"request_timeout_seconds,omitempty"`+ // UnsupportedReason marks a model this build cannot call.+ UnsupportedReason *string `json:"unsupported_reason,omitempty"`+}++// TranscriptionsSettings is the resolved (fully defaulted) form.+type TranscriptionsSettings struct {+ API api.TranscriptionAPI+ Path string+ ResponseFormat string+ LanguageParam string+ // MaxFileBytes and MaxEncodedBytes refuse an upload the vendor would+ // reject anyway, before sending it; 0 leaves the check to the vendor.+ MaxFileBytes int+ MaxEncodedBytes int+ Formats []string+ // RequestTimeout caps one request, in seconds.+ RequestTimeout int+ // UnsupportedReason is set where a vendor's recognition models, or some+ // of them, cannot be called with the audio in the request — they take a+ // public URL or run as asynchronous tasks. Resolve refuses such a model+ // rather than sending it a shape it does not accept.+ UnsupportedReason string+}++// DefaultTranscriptions is the protocol baseline: a form with file and model+// and nothing else, which every speech-to-text endpoint in this catalog+// documents. The deadline is the one the pre-catalog client carried; audio+// transcription is slow.+func DefaultTranscriptions() TranscriptionsSettings {+ return TranscriptionsSettings{Path: "/audio/transcriptions", RequestTimeout: 300}+}diff --git a/internal/models/catalog/types.go b/internal/models/catalog/types.goindex 90856f424..bc2985be7 100644--- a/internal/models/catalog/types.go+++ b/internal/models/catalog/types.go@@ -251,6 +251,9 @@ type Vendor struct {// OpenAI shape for any vendor that serves embeddings without naming// another.EmbeddingAPI api.EmbeddingAPI+ // TranscriptionAPI is the speech-to-text protocol. Register defaults it+ // to the OpenAI shape for any vendor that serves ASR.+ TranscriptionAPI api.TranscriptionAPI// DefaultBaseURLs by model type; GetDefaultURL falls back to chat.DefaultBaseURLs map[types.ModelType]stringModelTypes []types.ModelTypediff --git a/internal/models/catalog/validate.go b/internal/models/catalog/validate.goindex eaff1ce68..5955ae8e8 100644--- a/internal/models/catalog/validate.go+++ b/internal/models/catalog/validate.go@@ -14,10 +14,9 @@ import (// protocol, a bad compat key or an invalid thinking level is rejected where// it is written instead of surfacing as a failed chat call much later.//-// Chat, VLM, embedding and rerank rows are checked; all four are built from-// catalog.Resolve, so a row that does not resolve cannot be used. ASR rows-// return nil: the ASR client does not resolve through the catalog yet, so a-// spec stored on such a row is inert — neither validated nor applied.+// Every model type is checked: all of them are built from catalog.Resolve,+// so a row that does not resolve cannot be used and should fail the save+// rather than the first call.func ValidateRow(modelName string, modelType types.ModelType, params *types.ModelParameters) error {if params == nil || !resolvesThroughCatalog(modelType) {return nil@@ -44,7 +43,8 @@ func ValidateRow(modelName string, modelType types.ModelType, params *types.Modefunc resolvesThroughCatalog(modelType types.ModelType) bool {switch modelType {- case types.ModelTypeKnowledgeQA, types.ModelTypeVLLM, types.ModelTypeEmbedding, types.ModelTypeRerank:+ case types.ModelTypeKnowledgeQA, types.ModelTypeVLLM, types.ModelTypeEmbedding, types.ModelTypeRerank,+ types.ModelTypeASR:return true}return falsediff --git a/internal/models/catalog/validate_test.go b/internal/models/catalog/validate_test.goindex 0e6186f96..68034a967 100644--- a/internal/models/catalog/validate_test.go+++ b/internal/models/catalog/validate_test.go@@ -56,9 +56,17 @@ func TestValidateRow(t *testing.T) {wantErr: true,},{- name: "asr rows are not resolved through the catalog", modelName: "whisper-1",+ name: "a catalogued asr row resolves", modelName: "whisper-1",modelType: types.ModelTypeASR,- params: &types.ModelParameters{Provider: "openai", Spec: &types.ModelSpecOverride{API: "nope"}},+ params: &types.ModelParameters{Provider: "openai"},+ },+ {+ name: "unknown asr compat key is rejected", modelName: "whisper-1",+ modelType: types.ModelTypeASR,+ params: &types.ModelParameters{+ Provider: "openai", Spec: &types.ModelSpecOverride{Compat: map[string]any{"dimensions_field": "x"}},+ },+ wantErr: true,},{name: "a catalogued chat row resolves", modelName: "deepseek-v4-pro",diff --git a/internal/models/vendors/aliyun/models.json b/internal/models/vendors/aliyun/models.jsonindex bff7b78f8..fd06a04dd 100644--- a/internal/models/vendors/aliyun/models.json+++ b/internal/models/vendors/aliyun/models.json@@ -55,6 +55,8 @@{"name": "qwen2.5-vl-embedding (fused vectors only)", "match": "qwen2.5-vl-embedding*", "type": "Embedding", "compat": {"api": "dashscope-embeddings", "dimensions_field": "dimension", "max_batch_size": 1}, "source": "https://help.aliyun.com/zh/model-studio/multimodal-embedding-api-reference"},{"name": "multimodal-embedding-v1", "match": "multimodal-embedding*", "type": "Embedding", "compat": {"api": "dashscope-embeddings", "dimensions_field": "", "max_batch_size": 20}, "source": "https://help.aliyun.com/zh/model-studio/multimodal-embedding-api-reference"},{"id": "qwen3-rerank", "name": "Qwen3 Rerank (separate endpoint, not implemented)", "type": "Rerank", "deprecated": true, "compat": {"unsupported_reason": "qwen3-rerank speaks a different protocol (/compatible-api/v1/reranks, flat request, results at the top level) that no protocol package implements yet"}, "source": "https://help.aliyun.com/zh/model-studio/text-rerank-api"},- {"id": "gte-rerank-v2", "name": "GTE Rerank v2", "type": "Rerank", "source": "https://help.aliyun.com/zh/model-studio/text-rerank-api"}+ {"id": "gte-rerank-v2", "name": "GTE Rerank v2", "type": "Rerank", "source": "https://help.aliyun.com/zh/model-studio/text-rerank-api"},+ {"id": "qwen3-asr-flash", "name": "Qwen3 ASR Flash", "type": "ASR", "compat": {"api": "openai-chat-audio", "max_encoded_bytes": 10485760, "language_param": "asr_options"}, "source": "https://help.aliyun.com/zh/model-studio/qwen-asr-api-reference"},+ {"name": "Other Alibaba recognition models (not callable with the audio in the request)", "match": "*", "type": "ASR", "compat": {"unsupported_reason": "Alibaba's other recognition models take a public file URL or run as asynchronous tasks; only qwen3-asr-flash accepts the audio in the request"}, "source": "https://help.aliyun.com/zh/model-studio/qwen-asr-api-reference"}]}diff --git a/internal/models/vendors/aliyun/vendor.go b/internal/models/vendors/aliyun/vendor.goindex 36d9da82d..9c5672580 100644--- a/internal/models/vendors/aliyun/vendor.go+++ b/internal/models/vendors/aliyun/vendor.go@@ -136,6 +136,7 @@ func init() {types.ModelTypeEmbedding,types.ModelTypeRerank,types.ModelTypeVLLM,+ types.ModelTypeASR,},RerankAPI: api.RerankDashScope,// Text and multimodal embeddings live under different roots of the@@ -160,6 +161,13 @@ func init() {return root + "/compatible-mode/v1/embeddings", nil},Compat: catalog.VendorCompat{+ // ASR: qwen3-asr-flash is the one recognition model callable with+ // the audio in the request, on the compatible chat endpoint as a+ // base64 data URI; its catalog entry declares that. Every other+ // ASR name falls to a catch-all entry that refuses it: Paraformer,+ // Fun-ASR and the *-filetrans models are asynchronous tasks that+ // take a public file URL, which no protocol here can send+ // (https://help.aliyun.com/zh/model-studio/qwen-asr-api-reference).// Text models: the OpenAI-compatible endpoint// (https://help.aliyun.com/zh/model-studio/embedding-interfaces-compatible-with-openai),// which takes model, input, dimensions and encoding_format.diff --git a/internal/models/vendors/azure_openai/vendor.go b/internal/models/vendors/azure_openai/vendor.goindex 8c3e34188..63e7909c4 100644--- a/internal/models/vendors/azure_openai/vendor.go+++ b/internal/models/vendors/azure_openai/vendor.go@@ -124,14 +124,16 @@ func init() {types.ModelTypeEmbedding: BaseURL,types.ModelTypeVLLM: BaseURL,},- // ASR is deliberately absent: Azure serves Whisper, but- // internal/models/asr builds every remote client with a plain- // OpenAI config (Bearer auth, {base}/audio/transcriptions) and has- // no provider or extra-config field to route through this vendor's- // api-key header and deployments path. Advertising the type would- // let the picker create a row that fails at first use. The Endpoint- // hook below already knows the ASR path, so re-adding the type is a- // one-line change once asr.Config carries the provider.+ // ASR is deliberately absent. The ASR client now resolves through+ // this vendor (api-key header, Endpoint hook below), but the only+ // v1 reference for audio is the preview one —+ // POST {endpoint}/openai/v1/audio/transcriptions?api-version=preview+ // (https://learn.microsoft.com/en-us/azure/foundry/openai/reference-preview-latest)+ // — while the hook sends a row without api_version to the v1 path+ // with no version at all. Until that is verified against a live+ // resource, advertising the type would let the picker create a row+ // that may fail at first use. A row with an explicit api_version+ // would take the documented deployments path.ModelTypes: []types.ModelType{types.ModelTypeKnowledgeQA,types.ModelTypeEmbedding,diff --git a/internal/models/vendors/generic/vendor.go b/internal/models/vendors/generic/vendor.goindex 8b7a0424b..145530fdc 100644--- a/internal/models/vendors/generic/vendor.go+++ b/internal/models/vendors/generic/vendor.go@@ -108,6 +108,17 @@ func init() {ModelTypes: []types.ModelType{types.ModelTypeRerank},}},Compat: catalog.VendorCompat{+ // Transcriptions keeps the baseline, json by default. The+ // pre-catalog client always asked for verbose_json, which not every+ // model or server can produce: OpenAI's gpt-4o transcribers accept+ // only json, and vox-box's FunASR backend answers verbose_json with+ // a bare JSON string. A row that wants segments sets+ // {"response_format": "verbose_json"} in its compat. The language+ // hint goes as OpenAI's language form field, which a server that+ // does not implement it ignores.+ Transcriptions: catalog.TranscriptionsCompat{+ LanguageParam: catalog.Ptr(catalog.LanguageForm),+ },Embeddings: catalog.EmbeddingsCompat{// Whatever the operator runs. Everything the pre-catalog client// sent stays: vLLM, SGLang, TEI and Ollama's OpenAI route alldiff --git a/internal/models/vendors/gpustack/vendor.go b/internal/models/vendors/gpustack/vendor.goindex 89f24dc0d..f42f3ff1c 100644--- a/internal/models/vendors/gpustack/vendor.go+++ b/internal/models/vendors/gpustack/vendor.go@@ -118,6 +118,17 @@ func init() {ModelTypes: []types.ModelType{types.ModelTypeRerank},}},Compat: catalog.VendorCompat{+ // Transcriptions keeps the baseline, json by default. GPUStack+ // serves audio through vox-box (https://github.com/gpustack/vox-box),+ // whose route wraps a json answer as {"text": ...} but returns+ // verbose_json as whatever the backend produced — and its FunASR+ // backend (SenseVoice, Paraformer) produces a bare string for every+ // format. The pre-catalog client's verbose_json therefore arrived+ // as a JSON string that could not be decoded. The route reads a+ // language form field.+ Transcriptions: catalog.TranscriptionsCompat{+ LanguageParam: catalog.Ptr(catalog.LanguageForm),+ },Embeddings: catalog.EmbeddingsCompat{// vLLM's embedding server: dimensions for Matryoshka models, the// row's truncation budget, and encoding_format.diff --git a/internal/models/vendors/litellm/vendor.go b/internal/models/vendors/litellm/vendor.goindex 02d0d96b3..23c24f2af 100644--- a/internal/models/vendors/litellm/vendor.go+++ b/internal/models/vendors/litellm/vendor.go@@ -77,6 +77,7 @@ func init() {types.ModelTypeEmbedding,types.ModelTypeRerank,types.ModelTypeVLLM,+ types.ModelTypeASR,},ExtraFields: []catalog.ExtraField{{Key: catalog.ExtraScoreScale,@@ -104,6 +105,9 @@ func init() {ModelTypes: []types.ModelType{types.ModelTypeRerank},}},Compat: catalog.VendorCompat{+ // Transcriptions keeps the baseline: the proxy serves the OpenAI+ // /audio/transcriptions route for whichever upstream it is+ // configured with (https://docs.litellm.ai/docs/audio_transcription).Embeddings: catalog.EmbeddingsCompat{// https://docs.litellm.ai/docs/embedding/supported_embedding:// model, input, user, dimensions, encoding_format; anything elsediff --git a/internal/models/vendors/mimo/models.json b/internal/models/vendors/mimo/models.jsonindex c54858e31..b2d86cd20 100644--- a/internal/models/vendors/mimo/models.json+++ b/internal/models/vendors/mimo/models.json@@ -5,6 +5,7 @@{"id": "mimo-v2.5-pro", "name": "MiMo V2.5 Pro", "reasoning": true, "input": ["text"], "context_window": 1048576, "max_output_tokens": 131072, "cost": {"input": 0.435, "output": 0.87, "cache_read": 0.0036}, "source": "https://mimo.mi.com/static/docs/price/pay-as-you-go.md"},{"id": "mimo-v2.5", "name": "MiMo V2.5", "reasoning": true, "input": ["text", "image", "audio", "video"], "context_window": 1048576, "max_output_tokens": 131072, "cost": {"input": 0.14, "output": 0.28, "cache_read": 0.0028}, "source": "https://mimo.mi.com/static/docs/price/pay-as-you-go.md"},{"id": "mimo-v2.5-pro-ultraspeed", "name": "MiMo V2.5 Pro Ultraspeed", "reasoning": true, "input": ["text"], "context_window": 1048576, "max_output_tokens": 131072, "cost": {"input": 1.305, "output": 2.61, "cache_read": 0.0108}, "source": "https://mimo.mi.com/static/docs/news/latest/beta-extended.md"},- {"id": "mimo-v2-flash", "name": "MiMo V2 Flash (retired 2026-06-30)", "reasoning": true, "input": ["text"], "context_window": 256000, "max_output_tokens": 256000, "cost": {"input": 0.1, "output": 0.3, "cache_read": 0.01}, "deprecated": true, "source": "https://mimo.mi.com/static/docs/updates/deprecate.md"}+ {"id": "mimo-v2-flash", "name": "MiMo V2 Flash (retired 2026-06-30)", "reasoning": true, "input": ["text"], "context_window": 256000, "max_output_tokens": 256000, "cost": {"input": 0.1, "output": 0.3, "cache_read": 0.01}, "deprecated": true, "source": "https://mimo.mi.com/static/docs/updates/deprecate.md"},+ {"id": "mimo-v2.5-asr", "name": "MiMo V2.5 ASR", "type": "ASR", "source": "https://mimo.mi.com/docs/en-US/quick-start/usage-guide/audio/Speech-Recognition"}]}diff --git a/internal/models/vendors/mimo/vendor.go b/internal/models/vendors/mimo/vendor.goindex f3c7ec138..48ece1f13 100644--- a/internal/models/vendors/mimo/vendor.go+++ b/internal/models/vendors/mimo/vendor.go@@ -71,10 +71,22 @@ func init() {DefaultBaseURLs: map[types.ModelType]string{types.ModelTypeKnowledgeQA: BaseURL,},+ TranscriptionAPI: api.TranscriptionChatAudio,ModelTypes: []types.ModelType{types.ModelTypeKnowledgeQA,+ types.ModelTypeASR,},Compat: catalog.VendorCompat{+ // ASR: mimo-v2.5-asr on the chat endpoint, WAV or MP3 as a base64+ // data URI whose "encoded string size must not exceed 10 MB"+ // (https://mimo.mi.com/docs/en-US/quick-start/usage-guide/audio/Speech-Recognition).+ Transcriptions: catalog.TranscriptionsCompat{+ // The ceiling counts the whole data URI as sent.+ MaxEncodedBytes: catalog.Ptr(10 << 20),+ Formats: []string{"wav", "mp3"},+ // asr_options.language: "auto|zh|en".+ LanguageParam: catalog.Ptr(catalog.LanguageASROptions),+ },OpenAICompletions: catalog.OpenAICompletionsCompat{ThinkingFormat: catalog.Ptr(catalog.ThinkingFormatThinkingType),PromptCacheAccounting: catalog.Ptr(true),diff --git a/internal/models/vendors/minimax/models.json b/internal/models/vendors/minimax/models.jsonindex 7aababf42..b518a6906 100644--- a/internal/models/vendors/minimax/models.json+++ b/internal/models/vendors/minimax/models.json@@ -9,6 +9,7 @@{"id": "MiniMax-M2.5-highspeed", "name": "MiniMax M2.5 Highspeed", "reasoning": true, "input": ["text"], "context_window": 204800, "max_output_tokens": 131072, "cost": {"input": 0.6, "output": 2.4, "cache_read": 0.06, "cache_write": 0.375}, "thinking_levels": {"off": null}},{"id": "MiniMax-M2.1", "name": "MiniMax M2.1", "reasoning": true, "input": ["text"], "context_window": 204800, "max_output_tokens": 131072, "cost": {"input": 0.3, "output": 1.2, "cache_read": 0.03, "cache_write": 0.375}, "thinking_levels": {"off": null}},{"id": "MiniMax-M2.1-highspeed", "name": "MiniMax M2.1 Highspeed", "reasoning": true, "input": ["text"], "context_window": 204800, "max_output_tokens": 131072, "thinking_levels": {"off": null}, "source": "https://platform.minimax.io/docs/guides/text-generation"},- {"id": "MiniMax-M2", "name": "MiniMax M2", "reasoning": true, "input": ["text"], "context_window": 204800, "max_output_tokens": 131072, "cost": {"input": 0.3, "output": 1.2}, "thinking_levels": {"off": null}}+ {"id": "MiniMax-M2", "name": "MiniMax M2", "reasoning": true, "input": ["text"], "context_window": 204800, "max_output_tokens": 131072, "cost": {"input": 0.3, "output": 1.2}, "thinking_levels": {"off": null}},+ {"id": "asr-1.0", "name": "MiniMax ASR 1.0", "type": "ASR", "source": "https://platform.minimax.io/docs/api-reference/speech-to-text"}]}diff --git a/internal/models/vendors/minimax/vendor.go b/internal/models/vendors/minimax/vendor.goindex 08652c127..d5a6f156c 100644--- a/internal/models/vendors/minimax/vendor.go+++ b/internal/models/vendors/minimax/vendor.go@@ -95,8 +95,23 @@ func init() {},ModelTypes: []types.ModelType{types.ModelTypeKnowledgeQA,+ types.ModelTypeASR,},Compat: catalog.VendorCompat{+ Transcriptions: catalog.TranscriptionsCompat{+ // https://platform.minimax.io/docs/api-reference/speech-to-text+ // (and the same page on platform.minimax.cn): multipart model+ // (asr-1.0) + file on /v1/speech_to_text rather than the OpenAI+ // path, answering {text, duration, trace_id} for the default+ // json. At most 50 MB and 500 seconds; "超出会返回 400 而不会被截断".+ Path: catalog.Ptr("/speech_to_text"),+ MaxFileBytes: catalog.Ptr(50 << 20),+ // "wav / aiff / flac / alac(m4a) / mp3 / aac / opus / ogg";+ // "不支持无容器的裸 PCM 数据". The language hint is a request+ // header, a BCP-47 tag, not a form field.+ Formats: []string{"wav", "aiff", "flac", "m4a", "mp3", "aac", "opus", "ogg"},+ LanguageParam: catalog.Ptr(catalog.LanguageHeader),+ },OpenAICompletions: catalog.OpenAICompletionsCompat{ThinkingFormat: catalog.Ptr(catalog.ThinkingFormatThinkingType),ThinkingEnabledValue: catalog.Ptr("adaptive"),diff --git a/internal/models/vendors/openai/models.json b/internal/models/vendors/openai/models.jsonindex be7775ce5..b67103769 100644--- a/internal/models/vendors/openai/models.json+++ b/internal/models/vendors/openai/models.json@@ -29,6 +29,10 @@{"id": "text-embedding-3-small", "name": "text-embedding-3-small", "type": "Embedding", "dimension": 1536, "context_window": 8191, "cost": {"input": 0.02, "output": 0}},{"id": "text-embedding-3-large", "name": "text-embedding-3-large", "type": "Embedding", "dimension": 3072, "context_window": 8191, "cost": {"input": 0.13, "output": 0}},+ {"id": "whisper-1", "name": "Whisper", "type": "ASR", "compat": {"response_format": "verbose_json"}, "source": "https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions/methods/create"},+ {"id": "gpt-transcribe", "name": "GPT Transcribe", "type": "ASR", "source": "https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions/methods/create"},+ {"id": "gpt-4o-transcribe", "name": "GPT-4o Transcribe", "type": "ASR", "source": "https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions/methods/create"},+ {"id": "gpt-4o-mini-transcribe", "name": "GPT-4o mini Transcribe", "type": "ASR", "source": "https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions/methods/create"},{"id": "text-embedding-ada-002", "name": "text-embedding-ada-002", "type": "Embedding", "dimension": 1536, "context_window": 8192, "cost": {"input": 0.1, "output": 0}, "compat": {"dimensions_field": ""}}]}diff --git a/internal/models/vendors/openai/vendor.go b/internal/models/vendors/openai/vendor.goindex a6f99249d..d1d02d539 100644--- a/internal/models/vendors/openai/vendor.go+++ b/internal/models/vendors/openai/vendor.go@@ -97,6 +97,19 @@ func init() {types.ModelTypeASR,},Compat: catalog.VendorCompat{+ Transcriptions: catalog.TranscriptionsCompat{+ // The transcription reference (URL in openaitranscriptions):+ // response_format differs per model — "For gpt-4o-transcribe+ // and gpt-4o-mini-transcribe, the only supported format is+ // json", the default — so it is declared per entry, not here.+ // "Files can be up to 25 MB"+ // (https://developers.openai.com/api/docs/guides/speech-to-text).+ MaxFileBytes: catalog.Ptr(25 << 20),+ // "flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm", and a+ // language field in ISO-639-1.+ Formats: []string{"flac", "mp3", "mp4", "mpeg", "mpga", "m4a", "ogg", "wav", "webm"},+ LanguageParam: catalog.Ptr(catalog.LanguageForm),+ },Embeddings: catalog.EmbeddingsCompat{// https://developers.openai.com/api/reference/resources/embeddings/methods/create:// model, input, dimensions (text-embedding-3 and later; the ada-002diff --git a/internal/models/vendors/openrouter/models.json b/internal/models/vendors/openrouter/models.jsonindex a76f2d054..23660cd7c 100644--- a/internal/models/vendors/openrouter/models.json+++ b/internal/models/vendors/openrouter/models.json@@ -19,6 +19,7 @@{"id": "x-ai/grok-4.6", "name": "Grok 4.6", "reasoning": true, "input": ["text", "image"], "context_window": 500000, "max_output_tokens": 450000, "cost": {"input": 2, "output": 6, "cache_read": 0.5}, "thinking_levels": {"off": null, "minimal": "low", "xhigh": "xhigh", "max": "xhigh"}},{"id": "openai/text-embedding-3-small", "name": "text-embedding-3-small", "type": "Embedding", "dimension": 1536, "context_window": 8192, "source": "https://openrouter.ai/docs/api_reference/embeddings"},{"id": "openai/text-embedding-3-large", "name": "text-embedding-3-large", "type": "Embedding", "dimension": 3072, "context_window": 8192, "source": "https://openrouter.ai/docs/api_reference/embeddings"},- {"id": "cohere/rerank-v3.5", "name": "Cohere Rerank v3.5", "type": "Rerank", "source": "https://openrouter.ai/docs/api/api-reference/rerank/submit-a-rerank-request"}+ {"id": "cohere/rerank-v3.5", "name": "Cohere Rerank v3.5", "type": "Rerank", "source": "https://openrouter.ai/docs/api/api-reference/rerank/submit-a-rerank-request"},+ {"id": "openai/whisper-large-v3", "name": "Whisper Large v3", "type": "ASR", "source": "https://openrouter.ai/docs/api/api-reference/stt/create-transcription"}]}diff --git a/internal/models/vendors/openrouter/vendor.go b/internal/models/vendors/openrouter/vendor.goindex c475ea4e7..38d62c260 100644--- a/internal/models/vendors/openrouter/vendor.go+++ b/internal/models/vendors/openrouter/vendor.go@@ -75,8 +75,18 @@ func init() {types.ModelTypeEmbedding,types.ModelTypeRerank,types.ModelTypeVLLM,+ types.ModelTypeASR,},Compat: catalog.VendorCompat{+ Transcriptions: catalog.TranscriptionsCompat{+ // https://openrouter.ai/docs/api/api-reference/stt/create-transcription:+ // OpenAI-style multipart file + model, json by default. "Max 25+ // MB; send larger files as base64 JSON via input_audio."+ MaxFileBytes: catalog.Ptr(25 << 20),+ // language is an ISO-639-1 form field. The format "is derived+ // from the filename extension", with no closed list.+ LanguageParam: catalog.Ptr(catalog.LanguageForm),+ },Embeddings: catalog.EmbeddingsCompat{// https://openrouter.ai/docs/api/api-reference/embeddings/create-embeddings:// model, input, dimensions, encoding_format, input_type, provider,diff --git a/internal/models/vendors/requesty/models.json b/internal/models/vendors/requesty/models.jsonindex 360c6a99b..c1d2d27e2 100644--- a/internal/models/vendors/requesty/models.json+++ b/internal/models/vendors/requesty/models.json@@ -12,6 +12,9 @@{"id": "google/gemini-3.6-flash", "name": "Gemini 3.6 Flash", "reasoning": true, "input": ["text", "image", "audio", "video"], "context_window": 1048576, "max_output_tokens": 65536, "thinking_levels": {"off": null, "minimal": "min", "xhigh": "high", "max": "high"}},{"id": "deepseek/deepseek-v4-pro-0813", "name": "DeepSeek V4 Pro", "reasoning": true, "input": ["text"], "context_window": 1000000, "max_output_tokens": 131072},{"id": "zai/glm-5.2", "name": "GLM-5.2", "reasoning": true, "input": ["text"], "context_window": 1048576, "max_output_tokens": 131072},- {"id": "moonshot/kimi-k3", "name": "Kimi K3", "reasoning": true, "input": ["text", "image", "video"], "context_window": 1048576, "max_output_tokens": 262144}+ {"id": "moonshot/kimi-k3", "name": "Kimi K3", "reasoning": true, "input": ["text", "image", "video"], "context_window": 1048576, "max_output_tokens": 262144},+ {"id": "openai/gpt-4o-transcribe", "name": "GPT-4o Transcribe", "type": "ASR", "source": "https://docs.requesty.ai/api-reference/endpoint/audio-transcriptions-create"},+ {"id": "openai/gpt-4o-mini-transcribe", "name": "GPT-4o mini Transcribe", "type": "ASR", "source": "https://docs.requesty.ai/api-reference/endpoint/audio-transcriptions-create"},+ {"id": "openai/whisper-1", "name": "Whisper", "type": "ASR", "compat": {"response_format": "verbose_json"}, "source": "https://docs.requesty.ai/api-reference/endpoint/audio-transcriptions-create"}]}diff --git a/internal/models/vendors/requesty/vendor.go b/internal/models/vendors/requesty/vendor.goindex 283f44c83..74be132d9 100644--- a/internal/models/vendors/requesty/vendor.go+++ b/internal/models/vendors/requesty/vendor.go@@ -77,8 +77,19 @@ func init() {types.ModelTypeKnowledgeQA,types.ModelTypeEmbedding,types.ModelTypeVLLM,+ types.ModelTypeASR,},Compat: catalog.VendorCompat{+ Transcriptions: catalog.TranscriptionsCompat{+ // https://docs.requesty.ai/api-reference/endpoint/audio-transcriptions-create:+ // OpenAI-compatible multipart, json by default, verbose_json+ // available. "The maximum upload size per request is 32 MB."+ MaxFileBytes: catalog.Ptr(32 << 20),+ // "flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, webm"; language+ // is an ISO 639-1 form field.+ Formats: []string{"flac", "mp3", "mp4", "mpeg", "mpga", "m4a", "ogg", "wav", "webm"},+ LanguageParam: catalog.Ptr(catalog.LanguageForm),+ },Embeddings: catalog.EmbeddingsCompat{// https://docs.requesty.ai/api-reference/endpoint/embeddings-create:// model, input, encoding_format, dimensions.diff --git a/internal/models/vendors/siliconflow/models.json b/internal/models/vendors/siliconflow/models.jsonindex e51084c6e..44cb98d2d 100644--- a/internal/models/vendors/siliconflow/models.json+++ b/internal/models/vendors/siliconflow/models.json@@ -24,6 +24,7 @@{"id": "Qwen/Qwen3-Embedding-0.6B", "name": "Qwen3 Embedding 0.6B", "type": "Embedding", "dimension": 1024, "source": "https://api-docs.siliconflow.cn/docs/api/embeddings-post"},{"id": "BAAI/bge-reranker-v2-m3", "name": "BGE Reranker v2 M3", "type": "Rerank", "source": "https://docs.siliconflow.cn/cn/api-reference/rerank/create-rerank"},{"id": "Qwen/Qwen3-Reranker-8B", "name": "Qwen3 Reranker 8B", "type": "Rerank", "source": "https://docs.siliconflow.cn/cn/api-reference/rerank/create-rerank"},- {"id": "FunAudioLLM/SenseVoiceSmall", "name": "SenseVoice Small", "type": "ASR", "source": "https://docs.siliconflow.cn/cn/api-reference/audio/create-audio-transcriptions"}+ {"id": "FunAudioLLM/SenseVoiceSmall", "name": "SenseVoice Small", "type": "ASR", "source": "https://api-docs.siliconflow.cn/docs/api/audio-transcriptions-post"},+ {"id": "TeleAI/TeleSpeechASR", "name": "TeleSpeechASR", "type": "ASR", "source": "https://api-docs.siliconflow.cn/docs/api/audio-transcriptions-post"}]}diff --git a/internal/models/vendors/siliconflow/vendor.go b/internal/models/vendors/siliconflow/vendor.goindex f332e6fce..1421d5d72 100644--- a/internal/models/vendors/siliconflow/vendor.go+++ b/internal/models/vendors/siliconflow/vendor.go@@ -77,6 +77,12 @@ func init() {types.ModelTypeASR,},Compat: catalog.VendorCompat{+ Transcriptions: catalog.TranscriptionsCompat{+ // https://api-docs.siliconflow.cn/docs/api/audio-transcriptions-post:+ // file and model only — no response_format — and a file of+ // "时长不超过 1 小时,文件大小不超过 50MB".+ MaxFileBytes: catalog.Ptr(50 << 20),+ },Embeddings: catalog.EmbeddingsCompat{// https://api-docs.siliconflow.cn/docs/api/embeddings-post: model,// input, encoding_format, and dimensions "仅 Qwen/Qwen3 系列支持"diff --git a/internal/models/vendors/vendors_test.go b/internal/models/vendors/vendors_test.goindex 452e6c390..f3ebb50fd 100644--- a/internal/models/vendors/vendors_test.go+++ b/internal/models/vendors/vendors_test.go@@ -110,6 +110,9 @@ func TestEveryCatalogEntryResolves(t *testing.T) {if modelType == types.ModelTypeKnowledgeQA && !r.API.Known() {t.Errorf("%s/%s: resolved to unknown API %q", id, name, r.API)}+ if modelType == types.ModelTypeASR && !r.TranscriptionAPI.Known() {+ t.Errorf("%s/%s: resolved to unknown transcription API %q", id, name, r.TranscriptionAPI)+ }if modelType == types.ModelTypeEmbedding && !r.EmbeddingAPI.Known() {t.Errorf("%s/%s: resolved to unknown embedding API %q", id, name, r.EmbeddingAPI)}diff --git a/internal/models/vendors/zhipu/models.json b/internal/models/vendors/zhipu/models.jsonindex 62c15a1eb..bd2ea0e3c 100644--- a/internal/models/vendors/zhipu/models.json+++ b/internal/models/vendors/zhipu/models.json@@ -15,6 +15,7 @@{"id": "glm-4.5-air", "name": "GLM-4.5 Air", "reasoning": true, "input": ["text"], "context_window": 131072, "max_output_tokens": 98304, "cost": {"input": 0.2, "output": 1.1, "cache_read": 0.03}, "compat": {"supports_reasoning_effort": false}},{"id": "embedding-3", "name": "Embedding-3", "type": "Embedding", "dimension": 2048, "compat": {"max_batch_size": 64}, "source": "https://docs.bigmodel.cn/api-reference/模型-api/文本嵌入"},{"id": "embedding-2", "name": "Embedding-2", "type": "Embedding", "dimension": 1024, "compat": {"dimensions_field": ""}, "source": "https://docs.bigmodel.cn/api-reference/模型-api/文本嵌入"},- {"id": "rerank", "name": "Rerank", "type": "Rerank", "source": "https://docs.bigmodel.cn/api-reference/模型-api/文本重排序"}+ {"id": "rerank", "name": "Rerank", "type": "Rerank", "source": "https://docs.bigmodel.cn/api-reference/模型-api/文本重排序"},+ {"id": "glm-asr-2512", "name": "GLM-ASR-2512 (at most 30 s per file)", "type": "ASR", "source": "https://docs.bigmodel.cn/api-reference/模型-api/语音转文本"}]}diff --git a/internal/models/vendors/zhipu/vendor.go b/internal/models/vendors/zhipu/vendor.goindex bf9942fa3..3b4e5d69b 100644--- a/internal/models/vendors/zhipu/vendor.go+++ b/internal/models/vendors/zhipu/vendor.go@@ -107,8 +107,19 @@ func init() {types.ModelTypeEmbedding,types.ModelTypeRerank,types.ModelTypeVLLM,+ types.ModelTypeASR,},Compat: catalog.VendorCompat{+ Transcriptions: catalog.TranscriptionsCompat{+ // https://docs.bigmodel.cn/api-reference/模型-api/语音转文本:+ // POST {base}/audio/transcriptions, multipart file + model+ // (glm-asr-2512), answering {text}. The file is ".wav / .mp3",+ // at most 25 MB and 30 seconds — the duration cannot be checked+ // here without decoding, so a longer file is the vendor's error.+ MaxFileBytes: catalog.Ptr(25 << 20),+ // No language field; prompt and hotwords are its only hints.+ Formats: []string{"wav", "mp3"},+ },Embeddings: catalog.EmbeddingsCompat{// https://docs.bigmodel.cn/api-reference/模型-api/文本嵌入: model,// input, dimensions — no encoding_format. embedding-2 is fixed atdiff --git a/internal/utils/extraheaders.go b/internal/utils/extraheaders.goindex b32d54f8e..c6f644b84 100644--- a/internal/utils/extraheaders.go+++ b/internal/utils/extraheaders.go@@ -44,44 +44,3 @@ func ApplyCustomHeaders(req *http.Request, headers map[string]string) {req.Header.Set(name, v)}}--// CustomHeadersRoundTripper 是一个 http.RoundTripper 包装器,-// 会在每个 HTTP 请求发出前注入用户自定义的 header。-// 用于无法直接拿到底层 *http.Request 的场景(如 go-openai SDK)。-type CustomHeadersRoundTripper struct {- Headers map[string]string- Base http.RoundTripper-}--// RoundTrip 实现 http.RoundTripper 接口。-func (t *CustomHeadersRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {- base := t.Base- if base == nil {- base = http.DefaultTransport- }- if len(t.Headers) > 0 {- // 复制请求以避免修改调用方传入的对象。- cloned := req.Clone(req.Context())- ApplyCustomHeaders(cloned, t.Headers)- return base.RoundTrip(cloned)- }- return base.RoundTrip(req)-}--// WrapHTTPClientWithHeaders 返回一个新的 *http.Client,在原有 client 基础上注入自定义 header。-// 如果 headers 为空则直接返回原 client,避免不必要的开销。-func WrapHTTPClientWithHeaders(client *http.Client, headers map[string]string) *http.Client {- if len(headers) == 0 {- return client- }- if client == nil {- client = NewSSRFSafeHTTPClient(DefaultSSRFSafeHTTPClientConfig())- }- base := client.Transport- if base == nil {- base = http.DefaultTransport- }- wrapped := *client- wrapped.Transport = &CustomHeadersRoundTripper{Headers: headers, Base: base}- return &wrapped-}diff --git a/internal/utils/extraheaders_test.go b/internal/utils/extraheaders_test.goindex 5f4ecf330..451779562 100644--- a/internal/utils/extraheaders_test.go+++ b/internal/utils/extraheaders_test.go@@ -1,10 +1,7 @@package utilsimport (- "io""net/http"- "net/http/httptest"- "strings""testing")@@ -48,47 +45,3 @@ func TestApplyCustomHeaders_NilSafe(t *testing.T) {t.Fatalf("unexpected headers added: %+v", req.Header)}}--func TestWrapHTTPClientWithHeaders(t *testing.T) {- t.Setenv("SSRF_WHITELIST", "127.0.0.1,::1,localhost")- ResetSSRFWhitelistForTest()- t.Cleanup(ResetSSRFWhitelistForTest)- gotTrace := ""- gotAuth := ""- srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {- gotTrace = r.Header.Get("X-Trace-Id")- gotAuth = r.Header.Get("Authorization")- io.Copy(io.Discard, r.Body)- w.WriteHeader(http.StatusOK)- }))- defer srv.Close()-- client := WrapHTTPClientWithHeaders(nil, map[string]string{- "X-Trace-Id": "rt-1",- "Authorization": "Bearer should-not-override",- })-- req, _ := http.NewRequest("POST", srv.URL, strings.NewReader("{}"))- req.Header.Set("Authorization", "Bearer kept")-- resp, err := client.Do(req)- if err != nil {- t.Fatalf("request failed: %v", err)- }- resp.Body.Close()-- if gotTrace != "rt-1" {- t.Fatalf("expected custom header injected, got %q", gotTrace)- }- if gotAuth != "Bearer kept" {- t.Fatalf("reserved header must not be overridden, got %q", gotAuth)- }-}--func TestWrapHTTPClientWithHeaders_EmptyReturnsOriginal(t *testing.T) {- orig := &http.Client{}- wrapped := WrapHTTPClientWithHeaders(orig, nil)- if wrapped != orig {- t.Fatalf("expected original client returned when headers empty")- }-}diff --git a/website-docs/02-architecture/01-overview.md b/website-docs/02-architecture/01-overview.mdindex 6687441cb..d49091db1 100644--- a/website-docs/02-architecture/01-overview.md+++ b/website-docs/02-architecture/01-overview.md@@ -58,7 +58,7 @@ WeKnora 采用"主服务 + 前端 + 文档解析微服务"的三进程核心架| 配置 | `github.com/spf13/viper` + `config/config.yaml` + 环境变量 | — || 可观测 | OpenTelemetry + Langfuse(`internal/tracing/langfuse`) | LLM 调用级 trace || gRPC | `google.golang.org/grpc` v1.81.0 | 调用 docreader |-| LLM 接入 | `sashabaranov/go-openai`、Ollama、腾讯云 LKE 等 | 18+ 模型提供商(OpenAI 兼容 / Ollama / 云厂商 SDK) |+| LLM 接入 | 自研协议层 `internal/models/api`(OpenAI / Anthropic / Gemini / DashScope 等线格式各一个包)、Ollama、腾讯云 LKE SDK 等 | 27 家厂商由 `internal/models/vendors` 声明,见[模型管理](../03-features/06-models.md) || 向量/检索 | pgvector、ES v7/v8、OpenSearch、Qdrant、Milvus、Weaviate、Doris、腾讯 VectorDB、sqlite-vec | 由 `RETRIEVE_DRIVER` 与 `vector_stores` 表动态装配 || 知识图谱 | `neo4j-go-driver/v6` | 可选 || 数据分析 | DuckDB(`duckdb-go/v2`)、`pg_query_go` SQL 校验 | Agent 数据分析工具 |diff --git a/website-docs/03-features/06-models.md b/website-docs/03-features/06-models.mdindex e0af171a1..3d8ccf7e6 100644--- a/website-docs/03-features/06-models.md+++ b/website-docs/03-features/06-models.md@@ -56,7 +56,7 @@ const (| `Embedding` | `embedding` | `internal/models/embedding` | `Embed` / `BatchEmbed`(含 `GetDimensions`) | 文本向量化,供向量检索索引与查询 || `Rerank` | `rerank` | `internal/models/rerank` | `Rerank(query, documents)` 返回 `RankResult` | 检索结果精排 || `VLLM` | `vllm` | `internal/models/vlm` | `Predict(imgBytes, prompt)` | 视觉语言模型(VLM),文档图片理解 / 多模态解析 |-| `ASR` | `asr` | `internal/models/asr` | `Transcribe(audioBytes, fileName)` 返回文本与分段时间戳 | 音频转写(自动语音识别) |+| `ASR` | `asr` | `internal/models/asr` | `Transcribe(audioBytes, fileName)` 返回文本;模型提供时附带分段时间戳(如 OpenAI `whisper-1`) | 音频转写(自动语音识别) |前后端类型映射见 `internal/handler/model.go` 的 `modelTypeToFrontend()`(`KnowledgeQA -> chat` 等)。@@ -176,7 +176,7 @@ builtin_models:| 层 | 位置 | 职责 ||----|------|------|-| 协议层 | `internal/models/api/<protocol>` | 一个 wire 协议一个包。对话:`openaicompletions`、`openairesponses`、`anthropicmessages`、`googlegenai`;重排:`cohererank`、`dashscoperank`、`nimrerank`;向量:`openaiembeddings`、`dashscopeembeddings`、`arkembeddings`、`googleembeddings`。各自持有请求/响应结构与解析,不含任何厂商名 |+| 协议层 | `internal/models/api/<protocol>` | 一个 wire 协议一个包。对话:`openaicompletions`、`openairesponses`、`anthropicmessages`、`googlegenai`;重排:`cohererank`、`dashscoperank`、`nimrerank`;向量:`openaiembeddings`、`dashscopeembeddings`、`arkembeddings`、`googleembeddings`;语音:`openaitranscriptions`。各自持有请求/响应结构与解析,不含任何厂商名 || 厂商层 | `internal/models/vendors/<id>/` | 一个厂商一个目录:`vendor.go`(注册 `catalog.Vendor`)、`models.json`(模型目录)、`icon.svg`(品牌图标,`go:embed` 进二进制) || 目录层 | `internal/models/catalog` | 合并厂商定义、模型条目、部署叠加与单行覆盖,`Resolve` 得出「这个模型到底怎么发请求」 |@@ -235,7 +235,7 @@ make model-catalog-check前端不需要任何改动:厂商下拉、图标、额外字段、内置模型列表都由 `GET /api/v1/models/providers` 动态渲染。-写入侧也有一道闸:`catalog.ValidateRow` 会在创建 / 更新模型(REST)和加载 `config/builtin_models.yaml`(启动)时解析这行配置(chat、VLM、embedding、rerank 四类;ASR 还没进目录),未知协议、拼错的 compat 键、非法的思考档位在写入时就被拒绝(YAML 行只打 WARN 不阻塞启动,避免一次重启把线上模型下线)。+写入侧也有一道闸:`catalog.ValidateRow` 会在创建 / 更新模型(REST)和加载 `config/builtin_models.yaml`(启动)时解析这行配置(全部模型类型),未知协议、拼错的 compat 键、非法的思考档位在写入时就被拒绝(YAML 行只打 WARN 不阻塞启动,避免一次重启把线上模型下线)。#### 厂商更新了模型怎么办@@ -280,7 +280,21 @@ Embedding 行也有几处按厂商文档纠正的行为变化(逐厂商的出7. **OpenAI 兼容回复里没有 `index` 时按顺序取**(和旧实现一致);有 `index` 就按它放回,重复或缺位报错。8. **Jina 的 `task`、Gemini 的 `taskType`、OpenRouter 的 `input_type`、火山的 `instructions`、百炼原生接口的 `text_type` / `instruct` 都不发**。它们会改变文档侧向量,开启后同一个知识库里新旧向量不在同一空间;需要按行显式开启的设计另见 Tencent/WeKnora#1401。-另外 Azure OpenAI 不再声明支持 ASR(ASR 客户端只会构造标准 OpenAI 客户端,根本无法带上 Azure 的 `api-key` 头和部署路径,这类行此前就调不通)。+ASR 行的变化(逐厂商的出站表单由 `internal/models/asr/wire_test.go` 钉住):++1. **不再一律发 `response_format=verbose_json`**。OpenAI 文档写明 gpt-4o-transcribe / gpt-4o-mini-transcribe「only supported format is json」,旧实现发给它们必然 400;GPUStack 的音频后端 vox-box 在 FunASR 模型(SenseVoice、Paraformer)上对 `verbose_json` 返回裸字符串,旧实现解不开。现在只有文档写明支持的模型声明它(OpenAI `whisper-1`),其余走默认 json,Langfuse 里的音频时长因此只在返回分段的模型上有。需要分段的自建行可以在 `spec.compat` 里写 `{"response_format": "verbose_json"}`。+2. **按厂商文档的上限和格式在上传前拒绝**:文件大小(OpenAI / 智谱 / OpenRouter 25 MB、Requesty 32 MB、SiliconFlow / MiniMax 50 MB;阿里与小米按整段 `data:` URI 计 10 MB),以及文档给出封闭格式清单的厂商(智谱、小米只收 wav/mp3,OpenAI、Requesty、MiniMax 各有列表)。+3. **回复里没有 `text` 字段即报错**,不再当成「未检测到语音」入库;静音音频返回的是空字符串 `text`,照常处理。+4. ASR 行现在带上 provider 走目录解析,go-openai 依赖随之移除。provider 没有声明 ASR 的行:显式填了该厂商的直接报错;没填、靠 URL 识别到的,按 OpenAI 形状打到它的 URL(与旧实现一致),不走该厂商的 Endpoint 钩子。+5. **支持 ASR 的厂商按文档补齐**。OpenAI 形状(multipart `file` + `model`):openai、siliconflow、gpustack、generic、智谱(`glm-asr-2512`,单文件 ≤30 秒)、MiniMax(`asr-1.0`,路径是 `/v1/speech_to_text`)、OpenRouter、Requesty、LiteLLM。经 chat completions 的 `input_audio`(base64 data URI,编码后 ≤10 MB):阿里云 `qwen3-asr-flash`、小米 `mimo-v2.5-asr`。+6. **知识库的「音频语言提示」终于发出去了**(此前从未接线)。位置按厂商文档:OpenAI / Requesty / OpenRouter / GPUStack / generic 是 `language` 表单字段,MiniMax 是 `language` 请求头,阿里与小米是 `asr_options.language`;智谱、SiliconFlow、LiteLLM 文档没有这个参数,不发。填 `auto` 等同留空。+7. **音频时长从回包里读**:MiniMax 的 `duration`、OpenAI 系与阿里的 `usage.seconds`,Langfuse 不再只靠 `verbose_json` 的分段。++阿里只有 `qwen3-asr-flash` 能直接带音频调用;其余 ASR 模型名由一条兜底条目拒绝并说明原因。++查过但没有接的:火山豆包语音(独立域名与密钥,当前文档的请求体只收音频 URL)、千帆(`vop_asr` 挂在应用实例下要 `app_id`;短语音接口 ≤60 秒且只收 pcm/wav/amr/m4a)、七牛(只收音频 URL)、Novita(GLM-ASR 自有接口,≤30 秒)、腾讯云 ASR(与混元不是同一套鉴权)、NVIDIA 托管 Riva(gRPC)、Gemini(没有专用转写接口,只能让对话模型听音频)、Azure(见下)。阿里 Paraformer / Fun-ASR / `*-filetrans` 是异步任务,同样要公网 URL。++Azure OpenAI 仍不声明 ASR:音频转写只出现在 v1 **preview** 参考里(`/openai/v1/audio/transcriptions?api-version=preview`),v1 GA 参考没有;未填 `api_version` 的行会被钩子发到不带版本的 v1 路径,能否调通没有核实。### 模型调用链