Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

refactor(asr): move the wire format into protocol packages, and cover every vendor whose docs offer ASR (#3503)

lyingbug4a9aa06274e6
diff --git a/Makefile b/Makefile
index 816ea5015..67f74ac7a 100644
--- a/Makefile
+++ b/Makefile
@@ -117,7 +117,7 @@ test:
.PHONY: model-catalog-check
model-catalog-check:
go test ./internal/models/parity/ ./internal/models/vendors/ ./internal/models/catalog/ ./internal/models/api/... \
- ./internal/models/rerank/ ./internal/models/embedding/
+ ./internal/models/rerank/ ./internal/models/embedding/ ./internal/models/asr/
# Vendor catalog: report where our model metadata differs from models.dev.
# Development aid only — nothing is fetched at runtime and nothing is written
diff --git a/go.mod b/go.mod
index f41067f7f..20ae07003 100644
--- a/go.mod
+++ b/go.mod
@@ -51,7 +51,6 @@ require (
github.com/redis/go-redis/v9 v9.14.1
github.com/robfig/cron/v3 v3.0.1
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
- github.com/sashabaranov/go-openai v1.41.2
github.com/sirupsen/logrus v1.9.4
github.com/slack-go/slack v0.23.1
github.com/spf13/viper v1.21.0
diff --git a/go.sum b/go.sum
index e32a11df7..528734cfd 100644
--- a/go.sum
+++ b/go.sum
@@ -2516,8 +2516,6 @@ github.com/samber/lo v1.49.1 h1:4BIFyVfuQSEpluc7Fua+j1NolZHiEHEpaSEKdsH0tew=
github.com/samber/lo v1.49.1/go.mod h1:dO6KHFzUKXgP8LDhU0oI8d2hekjXnGOu0DB8Jecxd6o=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
-github.com/sashabaranov/go-openai v1.41.2 h1:vfPRBZNMpnqu8ELsclWcAvF19lDNgh1t6TVfFFOPiSM=
-github.com/sashabaranov/go-openai v1.41.2/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/sebdah/goldie/v2 v2.8.0 h1:dZb9wR8q5++oplmEiJT+U/5KyotVD+HNGCAc5gNr8rc=
diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go
index 10ed2f4f7..6eac56512 100644
--- a/internal/application/service/knowledge_process.go
+++ b/internal/application/service/knowledge_process.go
@@ -18,6 +18,7 @@ import (
"github.com/Tencent/WeKnora/internal/infrastructure/chunker"
"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
"github.com/Tencent/WeKnora/internal/logger"
+ "github.com/Tencent/WeKnora/internal/models/asr"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/models/embedding"
"github.com/Tencent/WeKnora/internal/searchutil"
@@ -3699,7 +3700,10 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
return nil
}
- transcriptionResult, err := asrModel.Transcribe(ctx, convertResult.AudioData, knowledge.FileName)
+ // The knowledge base's language hint; the vendor declares where it
+ // goes, or that it takes none.
+ asrCtx := asr.WithLanguage(ctx, eff.ASRConfig.Language)
+ transcriptionResult, err := asrModel.Transcribe(asrCtx, convertResult.AudioData, knowledge.FileName)
if err != nil {
logger.Errorf(ctx, "[ASR] Transcription failed: %v", err)
if isLastRetry {
diff --git a/internal/handler/model.go b/internal/handler/model.go
index c2f2a330e..2a73e723e 100644
--- a/internal/handler/model.go
+++ b/internal/handler/model.go
@@ -759,8 +759,7 @@ func (h *ModelHandler) DeleteModel(c *gin.Context) {
// validateCatalogParameters rejects protocol / compat overrides the catalog
// cannot interpret (unknown extra_config.api, unknown compat keys, bad
// reasoning levels) so a typo fails at save time instead of at the first
-// call. Chat, VLM, embedding and rerank rows are checked; see
-// catalog.ValidateRow.
+// call. Every model type is checked; see catalog.ValidateRow.
func validateCatalogParameters(name string, modelType types.ModelType, params *types.ModelParameters) error {
return catalog.ValidateRow(name, modelType, params)
}
diff --git a/internal/models/api/endpoint.go b/internal/models/api/endpoint.go
index e9f7ddd9c..5075753a6 100644
--- a/internal/models/api/endpoint.go
+++ b/internal/models/api/endpoint.go
@@ -100,24 +100,33 @@ func (e Endpoint) NewRequest(ctx context.Context, url string, body any, stream b
if err != nil {
return nil, nil, fmt.Errorf("marshal request: %w", err)
}
- if err := secutils.ValidateURLForSSRF(url); err != nil {
- return nil, nil, fmt.Errorf("endpoint SSRF check failed: %w", err)
- }
- req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(data))
+ req, err := e.newPost(ctx, url, data, "application/json")
if err != nil {
- return nil, nil, fmt.Errorf("create request: %w", err)
+ return nil, nil, err
}
- req.Header.Set("Content-Type", "application/json")
if stream {
req.Header.Set("Accept", "text/event-stream")
}
+ return req, data, nil
+}
+
+// newPost prepares an authenticated POST of an already-encoded body.
+func (e Endpoint) newPost(ctx context.Context, url string, data []byte, contentType string) (*http.Request, error) {
+ if err := secutils.ValidateURLForSSRF(url); err != nil {
+ return nil, fmt.Errorf("endpoint SSRF check failed: %w", err)
+ }
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(data))
+ if err != nil {
+ return nil, fmt.Errorf("create request: %w", err)
+ }
+ req.Header.Set("Content-Type", contentType)
if e.Auth != nil {
e.Auth(req, data)
}
// User headers are applied last but the helper skips reserved names so
// they cannot clobber auth or content negotiation.
secutils.ApplyCustomHeaders(req, e.Headers)
- return req, data, nil
+ return req, nil
}
// Do sends the request and returns the response. Non-2xx responses are
@@ -125,7 +134,7 @@ func (e Endpoint) NewRequest(ctx context.Context, url string, body any, stream b
func (e Endpoint) Do(req *http.Request) (*http.Response, error) {
resp, err := e.httpClient().Do(req)
if err != nil {
- return nil, fmt.Errorf("send request: %w", err)
+ return nil, &TransportError{Op: "send request", Err: err}
}
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
@@ -151,6 +160,11 @@ func (e Endpoint) PostJSON(ctx context.Context, url string, body, out any) error
// prompt is what an operator debugs, while a rerank body is a query plus
// every candidate chunk. The rerank layer logs a truncated line at Debug
// instead, so this would have duplicated it at Info.
+ return e.roundTrip(req, out)
+}
+
+// roundTrip sends a prepared request and decodes a 2xx reply into out.
+func (e Endpoint) roundTrip(req *http.Request, out any) error {
resp, err := e.Do(req)
if err != nil {
return err
@@ -158,7 +172,9 @@ func (e Endpoint) PostJSON(ctx context.Context, url string, body, out any) error
defer func() { _ = resp.Body.Close() }()
raw, err := io.ReadAll(resp.Body)
if err != nil {
- return fmt.Errorf("read response: %w", err)
+ // The connection broke before the reply arrived whole: nothing was
+ // received, so this is the network failing, not the vendor answering.
+ return &TransportError{Op: "read response", Err: err}
}
if out == nil {
return nil
@@ -169,6 +185,18 @@ func (e Endpoint) PostJSON(ctx context.Context, url string, body, out any) error
return nil
}
+// TransportError is a request that never got a whole answer: DNS, connect,
+// TLS, a reset or a timeout, while sending or while reading the reply. It is
+// the only failure worth sending again.
+type TransportError struct {
+ // Op is the phase that failed: "send request" or "read response".
+ Op string
+ Err error
+}
+
+func (e *TransportError) Error() string { return e.Op + ": " + e.Err.Error() }
+func (e *TransportError) Unwrap() error { return e.Err }
+
// HTTPError is a non-2xx vendor reply.
type HTTPError struct {
StatusCode int
diff --git a/internal/models/api/multipart.go b/internal/models/api/multipart.go
new file mode 100644
index 000000000..be47a189e
--- /dev/null
+++ b/internal/models/api/multipart.go
@@ -0,0 +1,76 @@
+package api
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "mime/multipart"
+ "net/textproto"
+ "path/filepath"
+ "strings"
+)
+
+// FormField is one text part of a multipart form. Order is kept.
+type FormField struct {
+ Name string
+ Value string
+}
+
+// FormFile is the file part of a multipart form.
+type FormFile struct {
+ Field string
+ FileName string
+ Data []byte
+}
+
+// EncodeMultipart writes fields and one file into a multipart/form-data body
+// and returns it with its Content-Type. It is encoded once: those bytes are
+// what a signing vendor hashes and what every retry sends again.
+//
+// The file part carries a filename and no Content-Type of its own. Servers
+// identify the format from the extension then — OpenAI asks for "an
+// extension-bearing filename", and vox-box (GPUStack's audio backend) falls
+// back to guessing from it — whereas a guessed type that a server does not
+// list is rejected outright.
+func EncodeMultipart(fields []FormField, file FormFile) ([]byte, string, error) {
+ var buf bytes.Buffer
+ w := multipart.NewWriter(&buf)
+ header := make(textproto.MIMEHeader)
+ header.Set("Content-Disposition", fmt.Sprintf(`form-data; name=%q; filename=%q`,
+ file.Field, filepath.Base(file.FileName)))
+ part, err := w.CreatePart(header)
+ if err != nil {
+ return nil, "", fmt.Errorf("encode form file: %w", err)
+ }
+ if _, err := part.Write(file.Data); err != nil {
+ return nil, "", fmt.Errorf("encode form file: %w", err)
+ }
+ for _, f := range fields {
+ if err := w.WriteField(f.Name, f.Value); err != nil {
+ return nil, "", fmt.Errorf("encode form field %s: %w", f.Name, err)
+ }
+ }
+ if err := w.Close(); err != nil {
+ return nil, "", fmt.Errorf("encode form: %w", err)
+ }
+ return buf.Bytes(), w.FormDataContentType(), nil
+}
+
+// PostMultipartWithRetry sends an encoded multipart body, retrying transport
+// failures only, and decodes the JSON reply into out. It has the guarantees
+// of PostJSONWithRetry.
+func (e Endpoint) PostMultipartWithRetry(
+ ctx context.Context, url string, body []byte, contentType string,
+ out any, policy RetryPolicy, label string,
+) error {
+ if !strings.HasPrefix(contentType, "multipart/") {
+ return fmt.Errorf("%s: not a multipart content type: %q", label, contentType)
+ }
+ return withRetry(ctx, policy, label, func() error {
+ req, err := e.newPost(ctx, url, body, contentType)
+ if err != nil {
+ return err
+ }
+ return e.roundTrip(req, out)
+ })
+}
diff --git a/internal/models/api/openaichataudio/client.go b/internal/models/api/openaichataudio/client.go
new file mode 100644
index 000000000..1e71ddbda
--- /dev/null
+++ b/internal/models/api/openaichataudio/client.go
@@ -0,0 +1,152 @@
+// Package openaichataudio implements speech recognition served through the
+// chat completions endpoint: one user message whose only content part is
+// {"type": "input_audio", "input_audio": {"data": "data:<mime>;base64,..."}},
+// answered by an assistant message whose content is the transcript.
+//
+// Only dedicated recognition models are addressed this way — Alibaba's
+// qwen3-asr-flash and Xiaomi's mimo-v2.5-asr — so no instruction is sent: the
+// model transcribes whatever it is given. A general audio-capable chat model
+// would need a prompt and would answer about the audio rather than transcribe
+// it, which is why this is its own protocol and not a chat request.
+//
+// https://help.aliyun.com/zh/model-studio/qwen-asr-api-reference
+// https://mimo.mi.com/docs/en-US/quick-start/usage-guide/audio/Speech-Recognition
+package openaichataudio
+
+import (
+ "context"
+ "encoding/base64"
+ "fmt"
+ "path/filepath"
+ "strings"
+
+ "github.com/Tencent/WeKnora/internal/models/api"
+ "github.com/Tencent/WeKnora/internal/models/catalog"
+)
+
+// Config is everything the client needs, already resolved by the catalog.
+type Config struct {
+ Endpoint api.Endpoint
+ Settings catalog.TranscriptionsSettings
+ Retry api.RetryPolicy
+}
+
+// Client talks chat-served speech recognition to one endpoint.
+type Client struct {
+ cfg Config
+}
+
+// New builds a client.
+func New(cfg Config) *Client { return &Client{cfg: cfg} }
+
+const defaultPath = "/chat/completions"
+
+func (c *Client) url() string {
+ if c.cfg.Endpoint.URL != "" {
+ return c.cfg.Endpoint.Resolve("")
+ }
+ path := defaultPath
+ if strings.HasSuffix(strings.TrimRight(c.cfg.Endpoint.BaseURL, "/"), path) {
+ path = ""
+ }
+ return c.cfg.Endpoint.Resolve(path)
+}
+
+// audioTypes maps the file extension onto the media type of the data URI.
+// The two references show audio/wav and audio/mpeg; the rest are the
+// registered types for the containers Alibaba lists. A fixed table rather
+// than mime.TypeByExtension, which reads the host's tables and would make the
+// request depend on the machine it runs on.
+var audioTypes = map[string]string{
+ ".wav": "audio/wav",
+ ".mp3": "audio/mpeg",
+ ".m4a": "audio/mp4",
+ ".mp4": "audio/mp4",
+ ".aac": "audio/aac",
+ ".flac": "audio/flac",
+ ".ogg": "audio/ogg",
+ ".opus": "audio/opus",
+ ".webm": "audio/webm",
+ ".amr": "audio/amr",
+}
+
+// DataURI is the input_audio payload for one file. Both references show an
+// audio media type in the URI, so a file whose extension names no audio
+// container is refused rather than labelled application/octet-stream.
+func DataURI(audio []byte, fileName string) (string, error) {
+ mediaType, ok := audioTypes[strings.ToLower(filepath.Ext(fileName))]
+ if !ok {
+ return "", fmt.Errorf("cannot tell the audio format of %q from its extension", fileName)
+ }
+ return "data:" + mediaType + ";base64," + base64.StdEncoding.EncodeToString(audio), nil
+}
+
+// BuildRequestBody is the golden-test entry point.
+func (c *Client) BuildRequestBody(req api.TranscriptionRequest) (map[string]any, error) {
+ uri, err := DataURI(req.Audio, req.FileName)
+ if err != nil {
+ return nil, err
+ }
+ // The ceiling is on the whole string as sent — "encoded string size must
+ // not exceed 10 MB" — so the data: prefix counts too.
+ if limit := c.cfg.Settings.MaxEncodedBytes; limit > 0 && len(uri) > limit {
+ return nil, fmt.Errorf(
+ "the audio is %.1f MB and %.1f MB once base64-encoded; this model accepts at most %.1f MB encoded",
+ float64(len(req.Audio))/(1<<20), float64(len(uri))/(1<<20), float64(limit)/(1<<20))
+ }
+ body := map[string]any{
+ "model": c.cfg.Endpoint.Model,
+ "messages": []any{map[string]any{
+ "role": "user",
+ "content": []any{map[string]any{
+ "type": "input_audio",
+ "input_audio": map[string]any{"data": uri},
+ }},
+ }},
+ }
+ if req.Language != "" && c.cfg.Settings.LanguageParam == catalog.LanguageASROptions {
+ body["asr_options"] = map[string]any{"language": req.Language}
+ }
+ return body, nil
+}
+
+type response struct {
+ Choices []struct {
+ Message struct {
+ // Content is a pointer for the same reason as the multipart
+ // protocol's text: an absent transcript is an error, an empty one
+ // is silence.
+ Content *string `json:"content"`
+ } `json:"message"`
+ } `json:"choices"`
+ // Usage.Seconds is the audio length, which Alibaba reports.
+ Usage struct {
+ Seconds float64 `json:"seconds"`
+ } `json:"usage"`
+ Error *struct {
+ Message string `json:"message"`
+ } `json:"error"`
+}
+
+// Transcribe sends one audio file.
+func (c *Client) Transcribe(ctx context.Context, req api.TranscriptionRequest) (*api.Transcription, error) {
+ body, err := c.BuildRequestBody(req)
+ if err != nil {
+ return nil, err
+ }
+ var decoded response
+ err = c.cfg.Endpoint.PostJSONWithRetry(ctx, c.url(), body, &decoded, c.cfg.Retry, "transcription")
+ if err != nil {
+ return nil, err
+ }
+ if decoded.Error != nil && decoded.Error.Message != "" {
+ return nil, fmt.Errorf("transcription API error: %s", decoded.Error.Message)
+ }
+ if len(decoded.Choices) == 0 || decoded.Choices[0].Message.Content == nil {
+ return nil, fmt.Errorf("transcription reply carries no message content")
+ }
+ return &api.Transcription{
+ Text: strings.TrimSpace(*decoded.Choices[0].Message.Content),
+ Duration: decoded.Usage.Seconds,
+ }, nil
+}
diff --git a/internal/models/api/openaichataudio/golden_test.go b/internal/models/api/openaichataudio/golden_test.go
new file mode 100644
index 000000000..8515a02ff
--- /dev/null
+++ b/internal/models/api/openaichataudio/golden_test.go
@@ -0,0 +1,162 @@
+package openaichataudio
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/models/api"
+ "github.com/Tencent/WeKnora/internal/models/catalog"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// qwenASRResponse is the non-streaming response example from Alibaba's
+// Qwen-ASR reference, copied from the page:
+// https://help.aliyun.com/zh/model-studio/qwen-asr-api-reference
+const qwenASRResponse = `{
+ "choices": [{
+ "finish_reason": "stop",
+ "index": 0,
+ "message": {
+ "annotations": [{"emotion": "neutral", "language": "zh", "type": "audio_info"}],
+ "content": "欢迎使用阿里云。",
+ "role": "assistant"
+ }
+ }],
+ "created": 1767683986,
+ "id": "chatcmpl-487abe5f-d4f2-9363-a877-xxxxxxx",
+ "model": "qwen3-asr-flash",
+ "object": "chat.completion",
+ "usage": {"completion_tokens": 12, "prompt_tokens": 42, "total_tokens": 54}
+}`
+
+// withSeconds is a reply carrying the usage.seconds that the reference's
+// schema documents ("音频时长(秒)") but its example omits.
+const withSeconds = `{"choices":[{"message":{"role":"assistant","content":"x"}}],"usage":{"seconds":3}}`
+
+func newClient(url, model string) *Client {
+ return newClientWith(url, model, catalog.TranscriptionsSettings{})
+}
+
+func newClientWith(url, model string, settings catalog.TranscriptionsSettings) *Client {
+ return New(Config{
+ Endpoint: api.Endpoint{BaseURL: url, Model: model, Auth: api.BearerAuth("k")},
+ Settings: settings,
+ })
+}
+
+// The request is the one both references show: a single user message whose
+// only part is the audio as a data URI, no instruction, no stream flag.
+func TestRequestBodyMatchesTheReferences(t *testing.T) {
+ body, err := newClient("https://example.invalid/v1", "qwen3-asr-flash").
+ BuildRequestBody(api.TranscriptionRequest{Audio: []byte("ID3"), FileName: "clip.MP3"})
+ require.NoError(t, err)
+ assert.Equal(t, map[string]any{
+ "model": "qwen3-asr-flash",
+ "messages": []any{map[string]any{
+ "role": "user",
+ "content": []any{map[string]any{
+ "type": "input_audio",
+ "input_audio": map[string]any{"data": "data:audio/mpeg;base64,SUQz"},
+ }},
+ }},
+ }, body)
+}
+
+func TestDataURIMediaTypeFollowsTheExtension(t *testing.T) {
+ uri, err := DataURI([]byte("RIFF"), "a.wav")
+ require.NoError(t, err)
+ assert.Equal(t, "data:audio/wav;base64,UklGRg==", uri)
+
+ // No audio type can be named for it, and the references show one.
+ _, err = DataURI([]byte("x"), "noext")
+ assert.Error(t, err)
+}
+
+// Both references cap the string as sent, so the data: prefix counts. A file
+// sized to exactly the encoded budget without the prefix goes over it.
+func TestEncodedCeilingCountsTheWholeDataURI(t *testing.T) {
+ const limit = 1024
+ c := newClientWith("https://example.invalid", "m", catalog.TranscriptionsSettings{MaxEncodedBytes: limit})
+ audio := make([]byte, limit*3/4) // base64 of this is exactly limit bytes
+ _, err := c.BuildRequestBody(api.TranscriptionRequest{Audio: audio, FileName: "a.wav"})
+ require.Error(t, err)
+
+ prefix := len("data:audio/wav;base64,")
+ fits := make([]byte, (limit-prefix)/4*3)
+ _, err = c.BuildRequestBody(api.TranscriptionRequest{Audio: fits, FileName: "a.wav"})
+ require.NoError(t, err)
+}
+
+func TestLanguageGoesInASROptionsWhereDeclared(t *testing.T) {
+ req := api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav", Language: "zh"}
+
+ body, err := newClient("https://example.invalid", "m").BuildRequestBody(req)
+ require.NoError(t, err)
+ assert.NotContains(t, body, "asr_options", "undeclared, so not sent")
+
+ body, err = newClientWith("https://example.invalid", "m",
+ catalog.TranscriptionsSettings{LanguageParam: catalog.LanguageASROptions}).BuildRequestBody(req)
+ require.NoError(t, err)
+ assert.Equal(t, map[string]any{"language": "zh"}, body["asr_options"])
+}
+
+func TestDecodesTheDocumentedResponse(t *testing.T) {
+ t.Setenv("SSRF_WHITELIST", "127.0.0.1")
+ var path string
+ var sent map[string]any
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ path = r.URL.Path
+ _ = json.NewDecoder(r.Body).Decode(&sent)
+ _, _ = w.Write([]byte(qwenASRResponse))
+ }))
+ defer server.Close()
+
+ out, err := newClient(server.URL+"/compatible-mode/v1", "qwen3-asr-flash").
+ Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("RIFF"), FileName: "a.wav"})
+ require.NoError(t, err)
+ assert.Equal(t, "/compatible-mode/v1/chat/completions", path)
+ assert.Equal(t, &api.Transcription{Text: "欢迎使用阿里云。"}, out)
+}
+
+// A reply with no assistant content is not silence.
+func TestReplyWithoutContentIsAnError(t *testing.T) {
+ t.Setenv("SSRF_WHITELIST", "127.0.0.1")
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ _, _ = w.Write([]byte(`{"choices":[]}`))
+ }))
+ defer server.Close()
+
+ _, err := newClient(server.URL, "m").
+ Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"})
+ assert.Error(t, err)
+}
+
+func TestDurationFromUsageSeconds(t *testing.T) {
+ t.Setenv("SSRF_WHITELIST", "127.0.0.1")
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ _, _ = w.Write([]byte(withSeconds))
+ }))
+ defer server.Close()
+
+ out, err := newClient(server.URL, "m").
+ Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"})
+ require.NoError(t, err)
+ assert.Equal(t, float64(3), out.Duration)
+}
+
+func TestEmptyContentIsSilence(t *testing.T) {
+ t.Setenv("SSRF_WHITELIST", "127.0.0.1")
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ _, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":""}}]}`))
+ }))
+ defer server.Close()
+
+ out, err := newClient(server.URL, "m").
+ Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"})
+ require.NoError(t, err)
+ assert.Equal(t, "", out.Text)
+}
diff --git a/internal/models/api/openaitranscriptions/client.go b/internal/models/api/openaitranscriptions/client.go
new file mode 100644
index 000000000..b09762cac
--- /dev/null
+++ b/internal/models/api/openaitranscriptions/client.go
@@ -0,0 +1,131 @@
+// Package openaitranscriptions implements the speech-to-text shape OpenAI
+// defined and the compatible servers copied: a multipart POST to
+// {base}/audio/transcriptions carrying file and model, answering {text}, or
+// {text, segments} when response_format is verbose_json.
+//
+// response_format is the one optional field that matters, and it differs by
+// model: gpt-4o-transcribe and gpt-4o-mini-transcribe accept only json,
+// whisper-1 also serves verbose_json, SiliconFlow documents no such field at
+// all. json is the default everywhere it is documented, so nothing is sent
+// unless catalog.TranscriptionsSettings names a format.
+//
+// https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions/methods/create
+package openaitranscriptions
+
+import (
+ "context"
+ "fmt"
+ "strings"
+
+ "github.com/Tencent/WeKnora/internal/models/api"
+ "github.com/Tencent/WeKnora/internal/models/catalog"
+)
+
+// Config is everything the client needs, already resolved by the catalog.
+type Config struct {
+ Endpoint api.Endpoint
+ Settings catalog.TranscriptionsSettings
+ Retry api.RetryPolicy
+}
+
+// Client talks the OpenAI transcription shape to one endpoint.
+type Client struct {
+ cfg Config
+}
+
+// New builds a client.
+func New(cfg Config) *Client { return &Client{cfg: cfg} }
+
+func (c *Client) url() string {
+ if c.cfg.Endpoint.URL != "" {
+ return c.cfg.Endpoint.Resolve("")
+ }
+ path := c.cfg.Settings.Path
+ if strings.HasSuffix(strings.TrimRight(c.cfg.Endpoint.BaseURL, "/"), path) {
+ path = ""
+ }
+ return c.cfg.Endpoint.Resolve(path)
+}
+
+// FormFields is the golden-test entry point: the text parts of the form, in
+// the order they are written.
+func (c *Client) FormFields(language string) []api.FormField {
+ fields := []api.FormField{{Name: "model", Value: c.cfg.Endpoint.Model}}
+ if format := c.cfg.Settings.ResponseFormat; format != "" {
+ fields = append(fields, api.FormField{Name: "response_format", Value: format})
+ }
+ if language != "" && c.cfg.Settings.LanguageParam == catalog.LanguageForm {
+ fields = append(fields, api.FormField{Name: "language", Value: language})
+ }
+ return fields
+}
+
+type response struct {
+ // Text is a pointer because its absence is an error, not silence. A
+ // transcript of silent audio is a present, empty string; a reply without
+ // the field is some other document. vox-box (GPUStack's audio backend)
+ // returns its HTTPException from the route instead of raising it, so a
+ // failed transcription can arrive as a 2xx whose body is the exception.
+ Text *string `json:"text"`
+ Segments []struct {
+ Start float64 `json:"start"`
+ End float64 `json:"end"`
+ Text string `json:"text"`
+ } `json:"segments"`
+ // Duration is top-level in OpenAI's verbose_json and in MiniMax's
+ // default json; usage.seconds is where OpenAI, OpenRouter and Requesty
+ // report it for models billed by duration.
+ Duration float64 `json:"duration"`
+ Usage struct {
+ Seconds float64 `json:"seconds"`
+ } `json:"usage"`
+ Detail any `json:"detail"`
+ Error *struct {
+ Message string `json:"message"`
+ } `json:"error"`
+}
+
+// Transcribe sends one audio file.
+func (c *Client) Transcribe(ctx context.Context, req api.TranscriptionRequest) (*api.Transcription, error) {
+ body, contentType, err := api.EncodeMultipart(c.FormFields(req.Language), api.FormFile{
+ Field: "file", FileName: req.FileName, Data: req.Audio,
+ })
+ if err != nil {
+ return nil, err
+ }
+ endpoint := c.cfg.Endpoint
+ if req.Language != "" && c.cfg.Settings.LanguageParam == catalog.LanguageHeader {
+ headers := make(map[string]string, len(endpoint.Headers)+1)
+ for k, v := range endpoint.Headers {
+ headers[k] = v
+ }
+ headers["language"] = req.Language
+ endpoint.Headers = headers
+ }
+ var decoded response
+ err = endpoint.PostMultipartWithRetry(
+ ctx, c.url(), body, contentType, &decoded, c.cfg.Retry, "transcription",
+ )
+ if err != nil {
+ return nil, err
+ }
+ if decoded.Error != nil && decoded.Error.Message != "" {
+ return nil, fmt.Errorf("transcription API error: %s", decoded.Error.Message)
+ }
+ if decoded.Text == nil {
+ if decoded.Detail != nil {
+ return nil, fmt.Errorf("transcription reply carries no text: %v", decoded.Detail)
+ }
+ return nil, fmt.Errorf("transcription reply carries no text")
+ }
+ out := &api.Transcription{Text: strings.TrimSpace(*decoded.Text), Duration: decoded.Duration}
+ if out.Duration == 0 {
+ out.Duration = decoded.Usage.Seconds
+ }
+ for _, s := range decoded.Segments {
+ out.Segments = append(out.Segments, api.TranscriptionSegment{
+ Start: s.Start, End: s.End, Text: strings.TrimSpace(s.Text),
+ })
+ }
+ return out, nil
+}
diff --git a/internal/models/api/openaitranscriptions/golden_test.go b/internal/models/api/openaitranscriptions/golden_test.go
new file mode 100644
index 000000000..2e2252b77
--- /dev/null
+++ b/internal/models/api/openaitranscriptions/golden_test.go
@@ -0,0 +1,219 @@
+package openaitranscriptions
+
+import (
+ "context"
+ "io"
+ "mime"
+ "mime/multipart"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/models/api"
+ "github.com/Tencent/WeKnora/internal/models/catalog"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// jsonResponse is the json example from the reference, copied from the page:
+// https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions/methods/create
+const jsonResponse = `{
+ "text": "text",
+ "languages": [{"code": "code"}],
+ "logprobs": [{"token": "token", "bytes": [0], "logprob": 0}],
+ "usage": {
+ "input_tokens": 0, "output_tokens": 0, "total_tokens": 0, "type": "tokens",
+ "input_token_details": {"audio_tokens": 0, "text_tokens": 0}
+ }
+}`
+
+// verboseResponse is the "Segment timestamps" example from the same page,
+// cut to its first segment where the page elides the rest with "...".
+const verboseResponse = `{
+ "task": "transcribe",
+ "language": "english",
+ "duration": 8.470000267028809,
+ "text": "The beach was a popular spot on a hot summer day. ` +
+ `People were swimming in the ocean, building sandcastles, and playing beach volleyball.",
+ "segments": [
+ {
+ "id": 0,
+ "seek": 0,
+ "start": 0.0,
+ "end": 3.319999933242798,
+ "text": " The beach was a popular spot on a hot summer day.",
+ "tokens": [50364, 440, 7534, 390, 257, 3743, 4008, 322, 257, 2368, 4266, 786, 13, 50530],
+ "temperature": 0.0,
+ "avg_logprob": -0.2860786020755768,
+ "compression_ratio": 1.2363636493682861,
+ "no_speech_prob": 0.00985979475080967
+ }
+ ],
+ "usage": {"type": "duration", "seconds": 9}
+}`
+
+type captured struct {
+ path string
+ auth string
+ language string
+ fields map[string]string
+ fileName string
+ fileType string
+ file []byte
+}
+
+func serve(t *testing.T, status int, reply string) (*httptest.Server, *captured) {
+ t.Helper()
+ t.Setenv("SSRF_WHITELIST", "127.0.0.1")
+ got := &captured{fields: map[string]string{}}
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ got.path = r.URL.Path
+ got.auth = r.Header.Get("Authorization")
+ got.language = r.Header.Get("language")
+ _, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
+ require.NoError(t, err)
+ reader := multipart.NewReader(r.Body, params["boundary"])
+ for {
+ part, err := reader.NextPart()
+ if err == io.EOF {
+ break
+ }
+ require.NoError(t, err)
+ data, _ := io.ReadAll(part)
+ if part.FormName() == "file" {
+ got.fileName, got.fileType, got.file = part.FileName(), part.Header.Get("Content-Type"), data
+ continue
+ }
+ got.fields[part.FormName()] = string(data)
+ }
+ w.WriteHeader(status)
+ _, _ = w.Write([]byte(reply))
+ }))
+ t.Cleanup(server.Close)
+ return server, got
+}
+
+func newClient(url, model string, settings catalog.TranscriptionsSettings) *Client {
+ if settings.Path == "" {
+ settings.Path = catalog.DefaultTranscriptions().Path
+ }
+ return New(Config{
+ Endpoint: api.Endpoint{BaseURL: url, Model: model, Auth: api.BearerAuth("k")},
+ Settings: settings,
+ })
+}
+
+// The baseline form is file and model. gpt-4o-transcribe accepts only json,
+// which is the default, so no response_format is sent.
+func TestBaselineFormIsFileAndModel(t *testing.T) {
+ server, got := serve(t, http.StatusOK, jsonResponse)
+ out, err := newClient(server.URL+"/v1", "gpt-4o-transcribe", catalog.TranscriptionsSettings{}).
+ Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("RIFF"), FileName: "meeting.wav"})
+ require.NoError(t, err)
+
+ assert.Equal(t, "/v1/audio/transcriptions", got.path)
+ assert.Equal(t, "Bearer k", got.auth)
+ assert.Equal(t, map[string]string{"model": "gpt-4o-transcribe"}, got.fields)
+ assert.Equal(t, "meeting.wav", got.fileName, "the extension is how the server identifies the format")
+ assert.Empty(t, got.fileType, "a guessed type a server does not list is rejected outright")
+ assert.Equal(t, []byte("RIFF"), got.file)
+ assert.Equal(t, &api.Transcription{Text: "text"}, out)
+}
+
+// The language hint goes where the vendor documents it: a form field on
+// OpenAI's shape, a request header on MiniMax's, nowhere otherwise.
+func TestLanguageGoesWhereTheVendorDeclares(t *testing.T) {
+ for _, tc := range []struct {
+ param string
+ wantField string
+ wantHeader string
+ }{
+ {param: "", wantField: "", wantHeader: ""},
+ {param: catalog.LanguageForm, wantField: "zh", wantHeader: ""},
+ {param: catalog.LanguageHeader, wantField: "", wantHeader: "zh"},
+ } {
+ t.Run("param="+tc.param, func(t *testing.T) {
+ server, got := serve(t, http.StatusOK, jsonResponse)
+ _, err := newClient(server.URL, "m", catalog.TranscriptionsSettings{LanguageParam: tc.param}).
+ Transcribe(context.Background(), api.TranscriptionRequest{
+ Audio: []byte("x"), FileName: "a.wav", Language: "zh",
+ })
+ require.NoError(t, err)
+ assert.Equal(t, tc.wantField, got.fields["language"])
+ assert.Equal(t, tc.wantHeader, got.language)
+ })
+ }
+}
+
+// The audio length is read from wherever the reply states it, so usage can
+// be reported without asking for segments.
+func TestDurationIsReadFromTheReply(t *testing.T) {
+ for name, reply := range map[string]string{
+ // MiniMax's default json, from its reference.
+ "top-level duration": `{"text": "x", "duration": 26.325, "trace_id": "021785229015510a2c883cf675b9804d"}`,
+ // OpenRouter's reference example.
+ "usage.seconds": `{"text": "x", "usage": {"cost": 0.000508, "input_tokens": 83, "output_tokens": 30, ` +
+ `"seconds": 9.2, "total_tokens": 113}}`,
+ } {
+ t.Run(name, func(t *testing.T) {
+ server, _ := serve(t, http.StatusOK, reply)
+ out, err := newClient(server.URL, "m", catalog.TranscriptionsSettings{}).
+ Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"})
+ require.NoError(t, err)
+ assert.NotZero(t, out.Duration)
+ })
+ }
+}
+
+func TestVerboseJSONCarriesSegments(t *testing.T) {
+ server, got := serve(t, http.StatusOK, verboseResponse)
+ settings := catalog.TranscriptionsSettings{ResponseFormat: "verbose_json"}
+ out, err := newClient(server.URL+"/v1", "whisper-1", settings).
+ Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("ID3"), FileName: "a.mp3"})
+ require.NoError(t, err)
+
+ assert.Equal(t, "verbose_json", got.fields["response_format"])
+ require.Len(t, out.Segments, 1)
+ assert.Equal(t, api.TranscriptionSegment{
+ Start: 0, End: 3.319999933242798, Text: "The beach was a popular spot on a hot summer day.",
+ }, out.Segments[0])
+}
+
+// Silence is an empty transcript, and that is a valid answer.
+func TestEmptyTextIsSilence(t *testing.T) {
+ server, _ := serve(t, http.StatusOK, `{"text": ""}`)
+ out, err := newClient(server.URL, "m", catalog.TranscriptionsSettings{}).
+ Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"})
+ require.NoError(t, err)
+ assert.Equal(t, "", out.Text)
+}
+
+// A reply without text is not silence. vox-box returns its HTTPException
+// from the route rather than raising it, so a failure can arrive as a 2xx
+// body shaped like this; reading it as an empty transcript would store "no
+// speech detected" for a file that was never transcribed.
+func TestReplyWithoutTextIsAnError(t *testing.T) {
+ server, _ := serve(t, http.StatusOK,
+ `{"status_code": 500, "detail": "Failed to transcribe audio, boom", "headers": null}`)
+ _, err := newClient(server.URL, "m", catalog.TranscriptionsSettings{}).
+ Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"})
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "Failed to transcribe audio")
+}
+
+func TestSurfacesTheVendorErrorBody(t *testing.T) {
+ // A stand-in rejection in OpenAI's error envelope.
+ server, _ := serve(t, http.StatusBadRequest, `{"error":{"message":"stand-in rejection"}}`)
+ _, err := newClient(server.URL, "gpt-4o-transcribe", catalog.TranscriptionsSettings{}).
+ Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"})
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "stand-in rejection")
+}
+
+func TestDoesNotDoubleThePath(t *testing.T) {
+ server, got := serve(t, http.StatusOK, jsonResponse)
+ _, err := newClient(server.URL+"/v1/audio/transcriptions", "m", catalog.TranscriptionsSettings{}).
+ Transcribe(context.Background(), api.TranscriptionRequest{Audio: []byte("x"), FileName: "a.wav"})
+ require.NoError(t, err)
+ assert.Equal(t, "/v1/audio/transcriptions", got.path)
+}
diff --git a/internal/models/api/retry.go b/internal/models/api/retry.go
index 2dd9a02e8..64e61c75e 100644
--- a/internal/models/api/retry.go
+++ b/internal/models/api/retry.go
@@ -49,12 +49,23 @@ func (p RetryPolicy) delay(attempt int) time.Duration {
// Here the last error is captured in the enclosing scope on purpose and the
// function cannot fall through without it.
//
-// Only transport failures are retried. A non-2xx reply is the vendor
-// answering, and repeating a rejected request neither fixes it nor tells the
-// operator anything new.
+// Only transport failures are retried, including a reply cut off while it
+// was being read. A non-2xx reply is the vendor answering, and repeating a
+// rejected request neither fixes it nor tells the operator anything new; a
+// reply that arrived whole but does not decode is the same answer every time,
+// and sending it again would bill the request twice.
func (e Endpoint) PostJSONWithRetry(
ctx context.Context, url string, body, out any, policy RetryPolicy, label string,
) error {
+ return withRetry(ctx, policy, label, func() error {
+ return e.PostJSON(ctx, url, body, out)
+ })
+}
+
+// withRetry runs send until it succeeds, fails with anything other than a
+// TransportError, or the budget runs out. The last error lives in the
+// enclosing scope and the loop cannot fall through without it.
+func withRetry(ctx context.Context, policy RetryPolicy, label string, send func() error) error {
var lastErr error
for attempt := 0; attempt <= policy.MaxRetries; attempt++ {
if attempt > 0 {
@@ -67,12 +78,12 @@ func (e Endpoint) PostJSONWithRetry(
return ctx.Err()
}
}
- err := e.PostJSON(ctx, url, body, out)
+ err := send()
if err == nil {
return nil
}
- var httpErr *HTTPError
- if errors.As(err, &httpErr) {
+ var transportErr *TransportError
+ if !errors.As(err, &transportErr) {
return err
}
lastErr = err
diff --git a/internal/models/api/retry_test.go b/internal/models/api/retry_test.go
index 4d7c8d142..b056a4dab 100644
--- a/internal/models/api/retry_test.go
+++ b/internal/models/api/retry_test.go
@@ -78,6 +78,25 @@ func TestPostJSONWithRetryDoesNotRetryAVendorRejection(t *testing.T) {
assert.Equal(t, 1, attempts, "a rejection must not be repeated")
}
+// A reply that arrived but does not decode is not a transport failure: the
+// vendor did the work, and sending the request again bills it again.
+func TestPostJSONWithRetryDoesNotRetryAnUndecodableReply(t *testing.T) {
+ t.Setenv("SSRF_WHITELIST", "127.0.0.1")
+ attempts := 0
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ attempts++
+ _, _ = w.Write([]byte(`not json`))
+ }))
+ defer server.Close()
+
+ var out map[string]any
+ err := Endpoint{}.PostJSONWithRetry(
+ context.Background(), server.URL, map[string]any{}, &out, fastPolicy(), "test",
+ )
+ require.Error(t, err)
+ assert.Equal(t, 1, attempts)
+}
+
func TestPostJSONWithRetryStopsWhenTheContextIsDone(t *testing.T) {
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
@@ -101,3 +120,54 @@ func TestRetryPolicyBackoffDoublesAndIsCapped(t *testing.T) {
assert.Equal(t, 4*time.Second, p.delay(4), "capped")
assert.Zero(t, RetryPolicy{MaxRetries: 2}.delay(1), "no base delay means no waiting")
}
+
+// A reply cut off mid-body never arrived, so it is sent again.
+func TestPostJSONWithRetryRetriesAReplyCutOffWhileReading(t *testing.T) {
+ t.Setenv("SSRF_WHITELIST", "127.0.0.1")
+ attempts := 0
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ attempts++
+ if attempts == 1 {
+ // Promise more than is sent, then hang up.
+ w.Header().Set("Content-Length", "100")
+ _, _ = w.Write([]byte(`{"a":`))
+ w.(http.Flusher).Flush() // the status and half a body are on the wire
+ hj, ok := w.(http.Hijacker)
+ require.True(t, ok)
+ conn, _, err := hj.Hijack()
+ require.NoError(t, err)
+ _ = conn.Close()
+ return
+ }
+ _, _ = w.Write([]byte(`{"a":1}`))
+ }))
+ defer server.Close()
+
+ var out map[string]any
+ err := Endpoint{}.PostJSONWithRetry(
+ context.Background(), server.URL, map[string]any{}, &out, fastPolicy(), "test",
+ )
+ require.NoError(t, err)
+ assert.Equal(t, 2, attempts)
+ assert.Equal(t, float64(1), out["a"])
+}
+
+// The phase that failed is named, so the log says whether the request or the
+// reply was lost.
+func TestReadFailureIsReportedAsSuch(t *testing.T) {
+ t.Setenv("SSRF_WHITELIST", "127.0.0.1")
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Length", "100")
+ _, _ = w.Write([]byte(`{"a":`))
+ w.(http.Flusher).Flush()
+ conn, _, err := w.(http.Hijacker).Hijack()
+ require.NoError(t, err)
+ _ = conn.Close()
+ }))
+ defer server.Close()
+
+ err := Endpoint{}.PostJSON(context.Background(), server.URL, map[string]any{}, &map[string]any{})
+ var transportErr *TransportError
+ require.ErrorAs(t, err, &transportErr)
+ assert.Equal(t, "read response", transportErr.Op)
+}
diff --git a/internal/models/api/transcriptions.go b/internal/models/api/transcriptions.go
new file mode 100644
index 000000000..6ace55fa9
--- /dev/null
+++ b/internal/models/api/transcriptions.go
@@ -0,0 +1,56 @@
+package api
+
+import "context"
+
+// TranscriptionAPI names a speech-to-text wire protocol. Like RerankAPI and
+// EmbeddingAPI it is its own type, so no other modality's value validates.
+type TranscriptionAPI string
+
+// The transcription protocols WeKnora speaks.
+const (
+ // TranscriptionOpenAI is POST {base}/audio/transcriptions as a multipart
+ // form carrying file and model, answering {text} — or {text, segments}
+ // when verbose_json is asked for.
+ TranscriptionOpenAI TranscriptionAPI = "openai-transcriptions"
+ // TranscriptionChatAudio is a dedicated speech-recognition model behind
+ // POST {base}/chat/completions: the audio goes in as a base64 data URI in
+ // an input_audio content part and the transcript comes back as the
+ // assistant message. Alibaba's qwen3-asr-flash and Xiaomi's mimo-v2.5-asr
+ // take no instruction; the model only transcribes.
+ TranscriptionChatAudio TranscriptionAPI = "openai-chat-audio"
+)
+
+// Known reports whether the value names a protocol this build implements.
+func (a TranscriptionAPI) Known() bool {
+ return a == TranscriptionOpenAI || a == TranscriptionChatAudio
+}
+
+// TranscriptionSegment is one timed stretch of a transcript.
+type TranscriptionSegment struct {
+ Start float64
+ End float64
+ Text string
+}
+
+// Transcription is what a protocol client returns.
+type Transcription struct {
+ Text string
+ Segments []TranscriptionSegment
+ // Duration is the audio length in seconds when the reply states it —
+ // OpenAI-shaped duration or usage.seconds — and 0 otherwise.
+ Duration float64
+}
+
+// TranscriptionRequest is one file to transcribe.
+type TranscriptionRequest struct {
+ Audio []byte
+ FileName string
+ // Language is the operator's hint, empty for auto-detection. Where it is
+ // sent, and whether at all, is the vendor's declaration.
+ Language string
+}
+
+// Transcriber turns one audio file into text.
+type Transcriber interface {
+ Transcribe(ctx context.Context, req TranscriptionRequest) (*Transcription, error)
+}
diff --git a/internal/models/asr/asr.go b/internal/models/asr/asr.go
index c241d52ef..913e5e22a 100644
--- a/internal/models/asr/asr.go
+++ b/internal/models/asr/asr.go
@@ -2,6 +2,7 @@ package asr
import (
"context"
+ "strings"
"github.com/Tencent/WeKnora/internal/types"
)
@@ -17,6 +18,27 @@ type Segment struct {
type TranscriptionResult struct {
Text string `json:"text"`
Segments []Segment `json:"segments,omitempty"`
+ // Duration is the audio length in seconds when the vendor reports it,
+ // which several do without segments; 0 when unknown.
+ Duration float64 `json:"duration,omitempty"`
+}
+
+type languageKey struct{}
+
+// WithLanguage attaches the operator's language hint (for example "zh") to
+// a transcription. Empty and "auto" mean auto-detection, which is every
+// vendor's default, and send nothing.
+func WithLanguage(ctx context.Context, language string) context.Context {
+ return context.WithValue(ctx, languageKey{}, language)
+}
+
+func languageFrom(ctx context.Context) string {
+ language, _ := ctx.Value(languageKey{}).(string)
+ language = strings.TrimSpace(language)
+ if strings.EqualFold(language, "auto") {
+ return ""
+ }
+ return language
}
// ASR defines the interface for Automatic Speech Recognition model operations.
@@ -35,7 +57,10 @@ type Config struct {
ModelName string
APIKey string
ModelID string
- Language string // optional: specify language for transcription
+ // Provider is the vendor id stored on the row; empty detects it from
+ // BaseURL.
+ Provider string
+ ExtraConfig map[string]string
// CustomHeaders 允许在调用远程 API 时附加自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。
CustomHeaders map[string]string
}
@@ -53,13 +78,14 @@ func ConfigFromModel(m *types.Model) *Config {
BaseURL: m.Parameters.BaseURL,
ModelName: m.Name,
Source: m.Source,
+ Provider: m.Parameters.Provider,
+ ExtraConfig: m.Parameters.ExtraConfig,
CustomHeaders: m.Parameters.CustomHeaders,
}
}
// NewASR creates an ASR instance based on the provided configuration.
-// All ASR vendors use the OpenAI-compatible /v1/audio/transcriptions API.
func NewASR(config *Config) (ASR, error) {
- a, err := NewOpenAIASR(config)
+ a, err := newASR(config)
return wrapASRLangfuse(a, err)
}
diff --git a/internal/models/asr/config_from_model_test.go b/internal/models/asr/config_from_model_test.go
index 7d2275eca..ecf3bf1dc 100644
--- a/internal/models/asr/config_from_model_test.go
+++ b/internal/models/asr/config_from_model_test.go
@@ -14,6 +14,8 @@ func TestConfigFromModel(t *testing.T) {
Parameters: types.ModelParameters{
BaseURL: "https://api.example.com/v1",
APIKey: "sk",
+ Provider: "siliconflow",
+ ExtraConfig: map[string]string{"remote_model_name": "r"},
CustomHeaders: map[string]string{"X": "y"},
},
}
@@ -24,6 +26,11 @@ func TestConfigFromModel(t *testing.T) {
if cfg.BaseURL != "https://api.example.com/v1" || cfg.APIKey != "sk" {
t.Errorf("connection fields mismatch: %+v", cfg)
}
+ // The vendor decides the wire format; without it every row resolves to
+ // the generic runtime.
+ if cfg.Provider != "siliconflow" || cfg.ExtraConfig["remote_model_name"] != "r" {
+ t.Errorf("vendor fields not propagated: %+v", cfg)
+ }
if cfg.CustomHeaders["X"] != "y" {
t.Errorf("CustomHeaders not propagated: %+v", cfg.CustomHeaders)
}
diff --git a/internal/models/asr/langfuse_wrapper.go b/internal/models/asr/langfuse_wrapper.go
index 7564c0592..d0bc7ecfc 100644
--- a/internal/models/asr/langfuse_wrapper.go
+++ b/internal/models/asr/langfuse_wrapper.go
@@ -8,7 +8,8 @@ import (
// langfuseASR wraps an ASR implementation and reports each Transcribe call
// as a Langfuse generation observation. Audio bytes are not uploaded — we
-// record file name, audio size and duration (derived from segment end times)
+// record file name, audio size and duration (as the vendor reports it, or
+// from segment end times)
// so usage can be billed by second/minute, which is how most ASR providers
// price their services.
type langfuseASR struct {
@@ -44,8 +45,13 @@ func (l *langfuseASR) Transcribe(ctx context.Context, audioBytes []byte, fileNam
if result != nil {
output["text"] = result.Text
output["segment_count"] = len(result.Segments)
- if n := len(result.Segments); n > 0 {
+ // The vendor's own figure when it gives one; otherwise the end of
+ // the last segment, which only verbose replies carry.
+ duration = result.Duration
+ if n := len(result.Segments); duration == 0 && n > 0 {
duration = result.Segments[n-1].End
+ }
+ if duration > 0 {
output["duration_seconds"] = duration
}
}
diff --git a/internal/models/asr/openai.go b/internal/models/asr/openai.go
deleted file mode 100644
index 66dee48e8..000000000
--- a/internal/models/asr/openai.go
+++ /dev/null
@@ -1,150 +0,0 @@
-package asr
-
-import (
- "bytes"
- "context"
- "encoding/json"
- "fmt"
- "path/filepath"
- "strings"
- "time"
-
- "github.com/Tencent/WeKnora/internal/logger"
- secutils "github.com/Tencent/WeKnora/internal/utils"
- openai "github.com/sashabaranov/go-openai"
-)
-
-const (
- asrDefaultTimeout = 300 * time.Second // audio transcription can be slow
-)
-
-// OpenAIASR implements ASR via an OpenAI-compatible audio transcriptions API.
-type OpenAIASR struct {
- modelName string
- modelID string
- client *openai.Client
- baseURL string
- language string
-}
-
-// NewOpenAIASR creates an OpenAI-compatible ASR instance.
-func NewOpenAIASR(config *Config) (*OpenAIASR, error) {
- if err := validateASRBaseURL(config.BaseURL); err != nil {
- return nil, err
- }
-
- apiCfg := openai.DefaultConfig(config.APIKey)
- if config.BaseURL != "" {
- apiCfg.BaseURL = config.BaseURL
- }
- httpClient := newASRHTTPClient(asrDefaultTimeout)
-
- // 注入用户自定义 HTTP header(类似 OpenAI Python SDK 的 extra_headers)
- if len(config.CustomHeaders) > 0 {
- apiCfg.HTTPClient = secutils.WrapHTTPClientWithHeaders(httpClient, config.CustomHeaders)
- } else {
- apiCfg.HTTPClient = httpClient
- }
-
- return &OpenAIASR{
- modelName: config.ModelName,
- modelID: config.ModelID,
- client: openai.NewClientWithConfig(apiCfg),
- baseURL: config.BaseURL,
- language: config.Language,
- }, nil
-}
-
-// Transcribe sends audio bytes to the OpenAI-compatible audio transcriptions API.
-func (s *OpenAIASR) Transcribe(ctx context.Context, audioBytes []byte, fileName string) (*TranscriptionResult, error) {
- if len(audioBytes) == 0 {
- return nil, fmt.Errorf("audio bytes are empty")
- }
-
- // Ensure fileName has an extension for proper MIME type detection
- if fileName == "" {
- fileName = "audio.mp3"
- }
-
- logger.Infof(ctx, "[ASR] Calling OpenAI-compatible transcription API, model=%s, baseURL=%s, audioSize=%d, file=%s",
- s.modelName, s.baseURL, len(audioBytes), fileName)
-
- req := openai.AudioRequest{
- Model: s.modelName,
- FilePath: fileName,
- Reader: bytes.NewReader(audioBytes),
- Format: openai.AudioResponseFormatVerboseJSON,
- }
-
- if s.language != "" {
- req.Language = s.language
- }
-
- resp, err := s.client.CreateTranscription(ctx, req)
- if err != nil {
- return nil, fmt.Errorf("ASR transcription request failed: %w", err)
- }
-
- jsonData, err := json.Marshal(resp)
- if err != nil {
- return nil, fmt.Errorf("marshal transcription response: %w", err)
- }
- logger.Debugf(ctx, "[ASR] Transcription response: %s", string(jsonData))
-
- text := strings.TrimSpace(resp.Text)
- logger.Infof(ctx, "[ASR] Transcription completed, text length=%d", len(text))
-
- var segments []Segment
- for _, s := range resp.Segments {
- segments = append(segments, Segment{
- Start: s.Start,
- End: s.End,
- Text: strings.TrimSpace(s.Text),
- })
- }
-
- return &TranscriptionResult{
- Text: text,
- Segments: segments,
- }, nil
-}
-
-func (s *OpenAIASR) GetModelName() string { return s.modelName }
-func (s *OpenAIASR) GetModelID() string { return s.modelID }
-
-// DetectAudioFormat returns a file extension hint for the given audio bytes.
-func DetectAudioFormat(data []byte, fileName string) string {
- if fileName != "" {
- ext := strings.ToLower(filepath.Ext(fileName))
- if ext != "" {
- return ext
- }
- }
- // Try to detect from magic bytes
- if len(data) >= 4 {
- // MP3: starts with ID3 or 0xFF 0xFB
- if data[0] == 'I' && data[1] == 'D' && data[2] == '3' {
- return ".mp3"
- }
- if data[0] == 0xFF && (data[1]&0xE0) == 0xE0 {
- return ".mp3"
- }
- // FLAC: starts with "fLaC"
- if data[0] == 'f' && data[1] == 'L' && data[2] == 'a' && data[3] == 'C' {
- return ".flac"
- }
- // OGG: starts with "OggS"
- if data[0] == 'O' && data[1] == 'g' && data[2] == 'g' && data[3] == 'S' {
- return ".ogg"
- }
- // WAV: starts with "RIFF"
- if data[0] == 'R' && data[1] == 'I' && data[2] == 'F' && data[3] == 'F' {
- return ".wav"
- }
- }
- // M4A: check for ftyp box
- if len(data) >= 8 && data[4] == 'f' && data[5] == 't' && data[6] == 'y' && data[7] == 'p' {
- return ".m4a"
- }
- return ".mp3" // default
-}
diff --git a/internal/models/asr/protocol.go b/internal/models/asr/protocol.go
new file mode 100644
index 000000000..b16b56ae3
--- /dev/null
+++ b/internal/models/asr/protocol.go
@@ -0,0 +1,183 @@
+package asr
+
+import (
+ "context"
+ "fmt"
+ "path/filepath"
+ "slices"
+ "strings"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/logger"
+ "github.com/Tencent/WeKnora/internal/models/api"
+ "github.com/Tencent/WeKnora/internal/models/api/openaichataudio"
+ "github.com/Tencent/WeKnora/internal/models/api/openaitranscriptions"
+ "github.com/Tencent/WeKnora/internal/models/catalog"
+ // catalog.Resolve answers from the vendor catalog, which is empty until
+ // the vendor packages have run their init.
+ _ "github.com/Tencent/WeKnora/internal/models/vendors"
+ "github.com/Tencent/WeKnora/internal/types"
+)
+
+// retryPolicy is the transport-error retry budget, which is none. The
+// knowledge pipeline retries a failed transcription as a whole task, and a
+// client-level retry on top would turn one hung 300-second upload into
+// several.
+var retryPolicy = func() api.RetryPolicy { return api.RetryPolicy{} }
+
+// newASR resolves the catalog and returns the protocol client for the
+// configured model. It mirrors rerank.newReranker and
+// embedding.newRemoteEmbedder: the vendor's facts decide the protocol, the
+// URL and the credential, and this function knows no vendor names.
+func newASR(config *Config) (ASR, error) {
+ if config == nil {
+ return nil, fmt.Errorf("asr config is nil")
+ }
+ if strings.TrimSpace(config.ModelName) == "" {
+ return nil, fmt.Errorf("model name is required")
+ }
+ resolved, err := catalog.Resolve(catalog.Ref{
+ Provider: config.Provider,
+ Model: config.ModelName,
+ BaseURL: config.BaseURL,
+ ModelType: types.ModelTypeASR,
+ Extra: config.ExtraConfig,
+ })
+ if err != nil {
+ return nil, err
+ }
+ // A vendor that does not declare speech recognition has not been checked
+ // for it, and its Endpoint hook may still compute a path — Azure's does.
+ // A row that named the vendor is refused; a row that named none and was
+ // matched by its URL is what the pre-catalog client served: an
+ // OpenAI-compatible endpoint at that URL.
+ if !resolved.Vendor.SupportsType(types.ModelTypeASR) {
+ if strings.TrimSpace(config.Provider) != "" {
+ return nil, fmt.Errorf("%s does not offer speech recognition in this build", resolved.Vendor.Name)
+ }
+ resolved, err = catalog.Resolve(catalog.Ref{
+ Provider: catalog.GenericID,
+ Model: config.ModelName,
+ BaseURL: config.BaseURL,
+ ModelType: types.ModelTypeASR,
+ Extra: config.ExtraConfig,
+ })
+ if err != nil {
+ return nil, err
+ }
+ }
+ if err := validateASRBaseURL(resolved.BaseURL); err != nil {
+ return nil, err
+ }
+
+ vendor := resolved.Vendor
+ creds := catalog.Credentials{APIKey: config.APIKey}
+ if creds.APIKey == "" {
+ creds.APIKey = vendor.DefaultAPIKey
+ }
+ // Vendor headers first so a user header cannot silently replace a vendor
+ // beta flag, matching chat.NewRemoteChat.
+ headers := make(map[string]string, len(vendor.Headers)+len(config.CustomHeaders))
+ for k, v := range vendor.Headers {
+ headers[k] = v
+ }
+ for k, v := range config.CustomHeaders {
+ headers[k] = v
+ }
+ settings := resolved.Transcriptions
+ endpoint := api.Endpoint{
+ BaseURL: resolved.BaseURL,
+ Model: resolved.RemoteModel,
+ ModelID: config.ModelID,
+ Auth: vendor.AuthFunc(vendor.API, creds),
+ Headers: headers,
+ Client: newASRHTTPClient(time.Duration(settings.RequestTimeout) * time.Second),
+ }
+ if vendor.Endpoint != nil {
+ if url, query := vendor.Endpoint(catalog.EndpointRequest{
+ BaseURL: resolved.BaseURL,
+ Model: resolved.RemoteModel,
+ ModelType: types.ModelTypeASR,
+ API: vendor.API,
+ Extra: config.ExtraConfig,
+ }); url != "" {
+ if err := validateASRBaseURL(url); err != nil {
+ return nil, err
+ }
+ endpoint.URL, endpoint.Query = url, query
+ }
+ }
+
+ var client api.Transcriber
+ switch resolved.TranscriptionAPI {
+ case api.TranscriptionOpenAI:
+ client = openaitranscriptions.New(openaitranscriptions.Config{
+ Endpoint: endpoint, Settings: settings, Retry: retryPolicy(),
+ })
+ case api.TranscriptionChatAudio:
+ client = openaichataudio.New(openaichataudio.Config{
+ Endpoint: endpoint, Settings: settings, Retry: retryPolicy(),
+ })
+ default:
+ return nil, fmt.Errorf("unsupported transcription api %q for provider %s",
+ resolved.TranscriptionAPI, vendor.ID)
+ }
+ return &protocolASR{
+ inner: client,
+ settings: settings,
+ vendor: vendor.Name,
+ endpoint: resolved.BaseURL,
+ modelName: config.ModelName,
+ modelID: config.ModelID,
+ }, nil
+}
+
+// protocolASR adapts a protocol client to the ASR interface and refuses an
+// upload the vendor has documented it will not take, before sending it.
+type protocolASR struct {
+ inner api.Transcriber
+ settings catalog.TranscriptionsSettings
+ vendor string
+ endpoint string
+ modelName string
+ modelID string
+}
+
+func (a *protocolASR) GetModelName() string { return a.modelName }
+func (a *protocolASR) GetModelID() string { return a.modelID }
+
+func (a *protocolASR) Transcribe(ctx context.Context, audio []byte, fileName string) (*TranscriptionResult, error) {
+ if len(audio) == 0 {
+ return nil, fmt.Errorf("audio bytes are empty")
+ }
+ if limit := a.settings.MaxFileBytes; limit > 0 && len(audio) > limit {
+ return nil, fmt.Errorf("%s transcription: the audio is %.1f MB; %s accepts at most %d MB per file",
+ a.modelName, float64(len(audio))/(1<<20), a.vendor, limit>>20)
+ }
+ // The server identifies the format from the extension.
+ if fileName == "" {
+ fileName = "audio.mp3"
+ }
+ if formats := a.settings.Formats; len(formats) > 0 {
+ ext := strings.TrimPrefix(strings.ToLower(filepath.Ext(fileName)), ".")
+ if !slices.Contains(formats, ext) {
+ return nil, fmt.Errorf("%s transcription: %s accepts %s audio, not %q",
+ a.modelName, a.vendor, strings.Join(formats, "/"), fileName)
+ }
+ }
+ logger.Infof(ctx, "[ASR] transcribing model=%s endpoint=%s size=%d file=%s",
+ a.modelName, a.endpoint, len(audio), fileName)
+
+ out, err := a.inner.Transcribe(ctx, api.TranscriptionRequest{
+ Audio: audio, FileName: fileName, Language: languageFrom(ctx),
+ })
+ if err != nil {
+ return nil, fmt.Errorf("ASR transcription request failed: %w", err)
+ }
+ result := &TranscriptionResult{Text: out.Text, Duration: out.Duration}
+ for _, s := range out.Segments {
+ result.Segments = append(result.Segments, Segment{Start: s.Start, End: s.End, Text: s.Text})
+ }
+ logger.Infof(ctx, "[ASR] transcription completed, text length=%d", len(result.Text))
+ return result, nil
+}
diff --git a/internal/models/asr/transport_security_test.go b/internal/models/asr/transport_security_test.go
deleted file mode 100644
index a2d97025a..000000000
--- a/internal/models/asr/transport_security_test.go
+++ /dev/null
@@ -1,26 +0,0 @@
-package asr
-
-import (
- "testing"
-
- secutils "github.com/Tencent/WeKnora/internal/utils"
-)
-
-func withASRSSRFWhitelist(t *testing.T, raw string) {
- t.Helper()
- t.Setenv("SSRF_WHITELIST", raw)
- secutils.ResetSSRFWhitelistForTest()
- t.Cleanup(secutils.ResetSSRFWhitelistForTest)
-}
-
-func TestOpenAIASRRejectsInternalBaseURL(t *testing.T) {
- withASRSSRFWhitelist(t, "")
-
- _, err := NewOpenAIASR(&Config{
- BaseURL: "http://169.254.169.254/latest/meta-data/",
- ModelName: "asr-test",
- })
- if err == nil {
- t.Fatalf("NewOpenAIASR returned nil error for blocked internal BaseURL")
- }
-}
diff --git a/internal/models/asr/wire_test.go b/internal/models/asr/wire_test.go
new file mode 100644
index 000000000..503921fab
--- /dev/null
+++ b/internal/models/asr/wire_test.go
@@ -0,0 +1,356 @@
+package asr
+
+import (
+ "context"
+ "encoding/json"
+ "io"
+ "mime"
+ "mime/multipart"
+ "net/http"
+ "net/http/httptest"
+ "sync/atomic"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/models/catalog"
+ "github.com/Tencent/WeKnora/internal/types"
+ secutils "github.com/Tencent/WeKnora/internal/utils"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func allowLoopback(t *testing.T) {
+ t.Helper()
+ t.Setenv("SSRF_WHITELIST", "127.0.0.1")
+ secutils.ResetSSRFWhitelistForTest()
+ t.Cleanup(secutils.ResetSSRFWhitelistForTest)
+}
+
+type form struct {
+ path string
+ auth string
+ fields map[string]string
+ fileName string
+ language string // the language request header
+ // body is set instead of fields when the request was JSON: the
+ // chat-served recognisers take the audio as a data URI.
+ body map[string]any
+}
+
+// upstream records the form it receives and answers the documented json
+// shape, adding a segment when verbose_json was asked for.
+func upstream(t *testing.T) (string, *form, *atomic.Int32) {
+ t.Helper()
+ allowLoopback(t)
+ got := &form{fields: map[string]string{}}
+ calls := &atomic.Int32{}
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ calls.Add(1)
+ got.path, got.auth, got.language = r.URL.Path, r.Header.Get("Authorization"), r.Header.Get("language")
+ mediaType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
+ require.NoError(t, err)
+ if mediaType == "application/json" {
+ require.NoError(t, json.NewDecoder(r.Body).Decode(&got.body))
+ _, _ = w.Write([]byte(`{"choices":[{"message":{"role":"assistant","content":" hello "}}]}`))
+ return
+ }
+ reader := multipart.NewReader(r.Body, params["boundary"])
+ for {
+ part, err := reader.NextPart()
+ if err == io.EOF {
+ break
+ }
+ require.NoError(t, err)
+ data, _ := io.ReadAll(part)
+ if part.FormName() == "file" {
+ got.fileName = part.FileName()
+ continue
+ }
+ got.fields[part.FormName()] = string(data)
+ }
+ if got.fields["response_format"] == "verbose_json" {
+ _, _ = w.Write([]byte(`{"text":" hello ","segments":[{"start":0,"end":1.5,"text":" hello "}]}`))
+ return
+ }
+ _, _ = w.Write([]byte(`{"text":" hello "}`))
+ }))
+ t.Cleanup(server.Close)
+ return server.URL, got, calls
+}
+
+// TestTranscriptionWireFormatPerVendor pins the form each ASR vendor is sent.
+// response_format appears only where the vendor documents it for that model:
+// the pre-catalog client sent verbose_json to every one of them, which
+// gpt-4o-transcribe rejects ("the only supported format is json") and which
+// vox-box's FunASR backend answers with an undecodable bare string.
+func TestTranscriptionWireFormatPerVendor(t *testing.T) {
+ chatAudio := func(model string) map[string]any {
+ return map[string]any{
+ "model": model,
+ "messages": []any{map[string]any{
+ "role": "user",
+ "content": []any{map[string]any{
+ "type": "input_audio",
+ "input_audio": map[string]any{"data": "data:audio/wav;base64,UklGRg=="},
+ }},
+ }},
+ }
+ }
+ cases := []struct {
+ name, provider, model, base string
+ wantPath string
+ wantFields map[string]string
+ wantBody map[string]any
+ wantSegments int
+ }{
+ {
+ name: "openai whisper-1 serves segments", provider: "openai", model: "whisper-1", base: "/v1",
+ wantPath: "/v1/audio/transcriptions",
+ wantFields: map[string]string{"model": "whisper-1", "response_format": "verbose_json"},
+ wantSegments: 1,
+ },
+ {
+ name: "openai gpt-4o-transcribe accepts only json", provider: "openai", model: "gpt-4o-transcribe",
+ base: "/v1", wantPath: "/v1/audio/transcriptions",
+ wantFields: map[string]string{"model": "gpt-4o-transcribe"},
+ },
+ {
+ name: "openai model not yet in the catalog", provider: "openai", model: "gpt-5-transcribe",
+ base: "/v1", wantPath: "/v1/audio/transcriptions",
+ wantFields: map[string]string{"model": "gpt-5-transcribe"},
+ },
+ {
+ name: "siliconflow documents file and model only", provider: "siliconflow",
+ model: "FunAudioLLM/SenseVoiceSmall", base: "/v1", wantPath: "/v1/audio/transcriptions",
+ wantFields: map[string]string{"model": "FunAudioLLM/SenseVoiceSmall"},
+ },
+ {
+ name: "gpustack stays on json for its FunASR backend", provider: "gpustack",
+ model: "SenseVoiceSmall", base: "/v1", wantPath: "/v1/audio/transcriptions",
+ wantFields: map[string]string{"model": "SenseVoiceSmall"},
+ },
+ {
+ name: "generic stays on json", provider: "generic", model: "whisper-large-v3",
+ base: "/v1", wantPath: "/v1/audio/transcriptions",
+ wantFields: map[string]string{"model": "whisper-large-v3"},
+ },
+ {
+ name: "zhipu glm-asr on the OpenAI shape", provider: "zhipu", model: "glm-asr-2512",
+ base: "/api/paas/v4", wantPath: "/api/paas/v4/audio/transcriptions",
+ wantFields: map[string]string{"model": "glm-asr-2512"},
+ },
+ {
+ name: "minimax on its own path", provider: "minimax", model: "asr-1.0",
+ base: "/v1", wantPath: "/v1/speech_to_text",
+ wantFields: map[string]string{"model": "asr-1.0"},
+ },
+ {
+ name: "openrouter", provider: "openrouter", model: "openai/whisper-large-v3",
+ base: "/api/v1", wantPath: "/api/v1/audio/transcriptions",
+ wantFields: map[string]string{"model": "openai/whisper-large-v3"},
+ },
+ {
+ name: "requesty whisper-1 serves segments", provider: "requesty", model: "openai/whisper-1",
+ base: "/v1", wantPath: "/v1/audio/transcriptions",
+ wantFields: map[string]string{"model": "openai/whisper-1", "response_format": "verbose_json"},
+ wantSegments: 1,
+ },
+ {
+ name: "litellm proxy", provider: "litellm", model: "whisper",
+ base: "/v1", wantPath: "/v1/audio/transcriptions",
+ wantFields: map[string]string{"model": "whisper"},
+ },
+ {
+ name: "aliyun qwen3-asr-flash is served on chat", provider: "aliyun", model: "qwen3-asr-flash",
+ base: "/compatible-mode/v1", wantPath: "/compatible-mode/v1/chat/completions",
+ wantBody: chatAudio("qwen3-asr-flash"),
+ },
+ {
+ name: "mimo asr is served on chat", provider: "mimo", model: "mimo-v2.5-asr",
+ base: "/v1", wantPath: "/v1/chat/completions",
+ wantBody: chatAudio("mimo-v2.5-asr"),
+ },
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ url, got, _ := upstream(t)
+ a, err := NewASR(&Config{
+ Source: types.ModelSourceRemote, Provider: tc.provider, BaseURL: url + tc.base,
+ ModelName: tc.model, APIKey: "k",
+ })
+ require.NoError(t, err)
+ out, err := a.Transcribe(context.Background(), []byte("RIFF"), "meeting.wav")
+ require.NoError(t, err)
+
+ assert.Equal(t, tc.wantPath, got.path)
+ assert.Equal(t, "Bearer k", got.auth)
+ if tc.wantBody != nil {
+ assert.Equal(t, tc.wantBody, got.body)
+ } else {
+ assert.Equal(t, tc.wantFields, got.fields)
+ assert.Equal(t, "meeting.wav", got.fileName)
+ }
+ assert.Equal(t, "hello", out.Text)
+ assert.Len(t, out.Segments, tc.wantSegments)
+ })
+ }
+}
+
+// A file over the documented ceiling is refused before it is uploaded: the
+// vendor would reject it anyway, after the whole upload.
+func TestOversizedAudioIsRefusedBeforeUpload(t *testing.T) {
+ url, _, calls := upstream(t)
+ a, err := NewASR(&Config{
+ Source: types.ModelSourceRemote, Provider: "openai", BaseURL: url + "/v1",
+ ModelName: "whisper-1", APIKey: "k",
+ })
+ require.NoError(t, err)
+
+ _, err = a.Transcribe(context.Background(), make([]byte, 25<<20+1), "long.mp3")
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "at most 25 MB")
+ assert.Zero(t, calls.Load(), "nothing should have been sent")
+
+ _, err = a.Transcribe(context.Background(), make([]byte, 25<<20), "exactly.mp3")
+ require.NoError(t, err, "the ceiling itself is allowed")
+}
+
+// The chat-served recognisers cap the data URI as sent at 10 MB, prefix
+// included, so three quarters of 10 MB of audio is already too much.
+func TestChatServedCeilingCountsTheWholeDataURI(t *testing.T) {
+ url, _, calls := upstream(t)
+ a, err := NewASR(&Config{
+ Source: types.ModelSourceRemote, Provider: "aliyun", BaseURL: url + "/compatible-mode/v1",
+ ModelName: "qwen3-asr-flash", APIKey: "k",
+ })
+ require.NoError(t, err)
+
+ _, err = a.Transcribe(context.Background(), make([]byte, 10<<20*3/4), "long.wav")
+ require.Error(t, err)
+ assert.Zero(t, calls.Load())
+
+ prefix := len("data:audio/wav;base64,")
+ _, err = a.Transcribe(context.Background(), make([]byte, (10<<20-prefix)/4*3), "fits.wav")
+ require.NoError(t, err)
+}
+
+// Only qwen3-asr-flash takes the audio in the request. Alibaba's other
+// recognition models want a public URL or an asynchronous task, so a row
+// naming one is refused with the reason rather than sent a chat request.
+func TestAlibabaRecognitionModelsOtherThanQwenASRFlashAreRefused(t *testing.T) {
+ _, err := NewASR(&Config{
+ Source: types.ModelSourceRemote, Provider: "aliyun",
+ BaseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1", ModelName: "paraformer-v2", APIKey: "k",
+ })
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "public file URL")
+}
+
+// A vendor that does not declare speech recognition is not trusted with it.
+// Azure's Endpoint hook still knows a transcription path, but nobody has
+// verified it; a row that names Azure is refused, and a row that names no
+// vendor gets what the pre-catalog client sent: the OpenAI shape at its URL.
+func TestVendorsWithoutASRAreNotRoutedThroughTheirHooks(t *testing.T) {
+ _, err := NewASR(&Config{
+ Source: types.ModelSourceRemote, Provider: "azure_openai",
+ BaseURL: "https://example.openai.azure.com", ModelName: "whisper", APIKey: "k",
+ })
+ require.Error(t, err)
+
+ // Detection matches URL patterns as substrings, so a path carrying Azure's
+ // host detects as Azure while the request still reaches the test server.
+ url, got, _ := upstream(t)
+ base := url + "/openai.azure.com/v1"
+ require.Equal(t, "azure_openai", catalog.DetectByURL(base))
+ a, err := NewASR(&Config{
+ Source: types.ModelSourceRemote, BaseURL: base, ModelName: "whisper", APIKey: "k",
+ })
+ require.NoError(t, err)
+ _, err = a.Transcribe(context.Background(), []byte("RIFF"), "a.wav")
+ require.NoError(t, err)
+ assert.Equal(t, "/openai.azure.com/v1/audio/transcriptions", got.path, "not Azure's /openai/v1 path")
+ assert.Equal(t, "Bearer k", got.auth, "not Azure's api-key header")
+}
+
+// A format the vendor does not list is refused before upload.
+func TestUndocumentedFormatIsRefusedBeforeUpload(t *testing.T) {
+ url, _, calls := upstream(t)
+ a, err := NewASR(&Config{
+ Source: types.ModelSourceRemote, Provider: "zhipu", BaseURL: url + "/api/paas/v4",
+ ModelName: "glm-asr-2512", APIKey: "k",
+ })
+ require.NoError(t, err)
+ _, err = a.Transcribe(context.Background(), []byte("x"), "memo.m4a")
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "wav/mp3")
+ assert.Zero(t, calls.Load())
+}
+
+// The knowledge base's language hint reaches each vendor where its reference
+// puts it, and nowhere for a vendor that documents none.
+func TestLanguageHintGoesWhereEachVendorDocumentsIt(t *testing.T) {
+ for _, tc := range []struct {
+ name, provider, model, base string
+ check func(t *testing.T, got *form)
+ }{
+ {"openai form field", "openai", "whisper-1", "/v1", func(t *testing.T, got *form) {
+ assert.Equal(t, "zh", got.fields["language"])
+ assert.Empty(t, got.language)
+ }},
+ {"minimax header", "minimax", "asr-1.0", "/v1", func(t *testing.T, got *form) {
+ assert.Equal(t, "zh", got.language)
+ assert.NotContains(t, got.fields, "language")
+ }},
+ {"aliyun asr_options", "aliyun", "qwen3-asr-flash", "/compatible-mode/v1", func(t *testing.T, got *form) {
+ assert.Equal(t, map[string]any{"language": "zh"}, got.body["asr_options"])
+ }},
+ {
+ "siliconflow documents none", "siliconflow", "FunAudioLLM/SenseVoiceSmall", "/v1",
+ func(t *testing.T, got *form) {
+ assert.NotContains(t, got.fields, "language")
+ assert.Empty(t, got.language)
+ },
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ url, got, _ := upstream(t)
+ a, err := NewASR(&Config{
+ Source: types.ModelSourceRemote, Provider: tc.provider, BaseURL: url + tc.base,
+ ModelName: tc.model, APIKey: "k",
+ })
+ require.NoError(t, err)
+ _, err = a.Transcribe(WithLanguage(context.Background(), "zh"), []byte("RIFF"), "a.wav")
+ require.NoError(t, err)
+ tc.check(t, got)
+ })
+ }
+
+ // "auto" is every vendor's default and is not sent.
+ url, got, _ := upstream(t)
+ a, err := NewASR(&Config{
+ Source: types.ModelSourceRemote, Provider: "openai", BaseURL: url + "/v1", ModelName: "whisper-1", APIKey: "k",
+ })
+ require.NoError(t, err)
+ _, err = a.Transcribe(WithLanguage(context.Background(), "auto"), []byte("RIFF"), "a.wav")
+ require.NoError(t, err)
+ assert.NotContains(t, got.fields, "language")
+}
+
+func TestEmptyAudioIsAnError(t *testing.T) {
+ url, _, calls := upstream(t)
+ a, err := NewASR(&Config{Source: types.ModelSourceRemote, Provider: "generic", BaseURL: url, ModelName: "m"})
+ require.NoError(t, err)
+ _, err = a.Transcribe(context.Background(), nil, "a.wav")
+ assert.Error(t, err)
+ assert.Zero(t, calls.Load())
+}
+
+func TestInternalBaseURLIsRefused(t *testing.T) {
+ t.Setenv("SSRF_WHITELIST", "")
+ secutils.ResetSSRFWhitelistForTest()
+ t.Cleanup(secutils.ResetSSRFWhitelistForTest)
+ _, err := NewASR(&Config{
+ Source: types.ModelSourceRemote, Provider: "generic",
+ BaseURL: "http://169.254.169.254/latest/meta-data/", ModelName: "asr-test",
+ })
+ assert.Error(t, err)
+}
diff --git a/internal/models/catalog/compat.go b/internal/models/catalog/compat.go
index 234d8a2b6..f2a479d07 100644
--- a/internal/models/catalog/compat.go
+++ b/internal/models/catalog/compat.go
@@ -346,6 +346,8 @@ type VendorCompat struct {
Rerank RerankCompat `json:"rerank,omitempty"`
// Embeddings is the embedding protocol overlay, likewise one per vendor.
Embeddings EmbeddingsCompat `json:"embeddings,omitempty"`
+ // Transcriptions is the speech-to-text overlay, likewise one per vendor.
+ Transcriptions TranscriptionsCompat `json:"transcriptions,omitempty"`
}
// apply writes the set fields of an overlay struct onto a settings struct
diff --git a/internal/models/catalog/registry.go b/internal/models/catalog/registry.go
index 91767fde5..fc132a49a 100644
--- a/internal/models/catalog/registry.go
+++ b/internal/models/catalog/registry.go
@@ -34,6 +34,9 @@ func Register(v *Vendor) {
if v.RerankAPI == "" && v.SupportsType(types.ModelTypeRerank) {
v.RerankAPI = api.RerankCohere
}
+ if v.TranscriptionAPI == "" && v.SupportsType(types.ModelTypeASR) {
+ v.TranscriptionAPI = api.TranscriptionOpenAI
+ }
if v.EmbeddingAPI == "" && v.SupportsType(types.ModelTypeEmbedding) {
v.EmbeddingAPI = api.EmbeddingOpenAI
}
diff --git a/internal/models/catalog/resolve.go b/internal/models/catalog/resolve.go
index ef6202278..fa8542c22 100644
--- a/internal/models/catalog/resolve.go
+++ b/internal/models/catalog/resolve.go
@@ -81,6 +81,11 @@ type Resolved struct {
// for the same reason.
EmbeddingAPI api.EmbeddingAPI
Embeddings EmbeddingsSettings
+
+ // TranscriptionAPI and Transcriptions are filled only for an ASR
+ // reference.
+ TranscriptionAPI api.TranscriptionAPI
+ Transcriptions TranscriptionsSettings
}
// Resolve merges the vendor, catalog entry, extra-config and per-row
@@ -122,6 +127,9 @@ func Resolve(ref Ref) (*Resolved, error) {
if modelType == types.ModelTypeEmbedding {
return resolveEmbeddings(ref, vendor, spec, cataloged, baseURL)
}
+ if modelType == types.ModelTypeASR {
+ return resolveTranscriptions(ref, vendor, spec, cataloged, baseURL)
+ }
resolvedAPI := spec.API
if forced := strings.TrimSpace(ref.Extra[ExtraAPI]); forced != "" {
@@ -537,3 +545,62 @@ func resolveEmbeddings(
}
return out, nil
}
+
+// resolveTranscriptions merges the speech-to-text layers, in the same order
+// as resolveEmbeddings: protocol default, vendor compat, the catalog entry,
+// the row's own compat, then remote_model_name.
+func resolveTranscriptions(
+ ref Ref, vendor *Vendor, spec ModelSpec, cataloged bool, baseURL string,
+) (*Resolved, error) {
+ protocol := vendor.TranscriptionAPI
+ if protocol == "" {
+ protocol = api.TranscriptionOpenAI
+ }
+ settings := DefaultTranscriptions()
+ apply(&settings, &vendor.Compat.Transcriptions)
+ for _, raw := range []json.RawMessage{spec.Compat, ref.Override.CompatJSON()} {
+ if len(raw) == 0 {
+ continue
+ }
+ overlay := &TranscriptionsCompat{}
+ if err := decodeCompat(raw, overlay); err != nil {
+ return nil, fmt.Errorf("transcriptions compat: %w", err)
+ }
+ apply(&settings, overlay)
+ }
+ if settings.API != "" {
+ protocol = settings.API
+ }
+ if !protocol.Known() {
+ return nil, fmt.Errorf("catalog: unknown transcription api %q for %s/%s", protocol, vendor.ID, spec.ID)
+ }
+ settings.API = protocol
+ switch settings.LanguageParam {
+ case "", LanguageForm, LanguageHeader, LanguageASROptions:
+ default:
+ return nil, fmt.Errorf("catalog: unknown language_param %q for %s/%s",
+ settings.LanguageParam, vendor.ID, spec.ID)
+ }
+ // Checked after every layer: an entry may lift a vendor-wide refusal for
+ // the one model that takes the audio in the request.
+ if settings.UnsupportedReason != "" {
+ return nil, fmt.Errorf(
+ "catalog: %s does not serve %q through a protocol this build implements: %s",
+ vendor.ID, spec.ID, settings.UnsupportedReason,
+ )
+ }
+
+ out := &Resolved{
+ Vendor: vendor,
+ Spec: spec,
+ Cataloged: cataloged,
+ BaseURL: baseURL,
+ RemoteModel: ref.Model,
+ TranscriptionAPI: protocol,
+ Transcriptions: settings,
+ }
+ if override := strings.TrimSpace(ref.Extra[ExtraRemoteModelName]); override != "" {
+ out.RemoteModel = override
+ }
+ return out, nil
+}
diff --git a/internal/models/catalog/transcriptions.go b/internal/models/catalog/transcriptions.go
new file mode 100644
index 000000000..0a7b74ce6
--- /dev/null
+++ b/internal/models/catalog/transcriptions.go
@@ -0,0 +1,70 @@
+package catalog
+
+import "github.com/Tencent/WeKnora/internal/models/api"
+
+// Where a transcription protocol puts the operator's language hint. Each
+// vendor documents one place, or none.
+const (
+ // LanguageForm is a multipart field named language (OpenAI's shape,
+ // ISO-639-1).
+ LanguageForm = "form"
+ // LanguageHeader is an HTTP request header named language (MiniMax,
+ // BCP-47).
+ LanguageHeader = "header"
+ // LanguageASROptions is asr_options.language in a chat-served request
+ // body (Alibaba, Xiaomi).
+ LanguageASROptions = "asr_options"
+)
+
+// TranscriptionsCompat is the overlay form (every field optional) of the
+// speech-to-text settings. JSON keys are the names used in models.json.
+type TranscriptionsCompat struct {
+ API *api.TranscriptionAPI `json:"api,omitempty"`
+ Path *string `json:"path,omitempty"`
+ // ResponseFormat is sent as response_format when non-empty. OpenAI's
+ // gpt-4o-transcribe and gpt-4o-mini-transcribe accept only json, which is
+ // also the default, so most models leave it unset.
+ ResponseFormat *string `json:"response_format,omitempty"`
+ // LanguageParam names where the language hint goes: form, header or
+ // asr_options; empty sends none.
+ LanguageParam *string `json:"language_param,omitempty"`
+ // MaxFileBytes is the documented upload ceiling for the audio itself.
+ MaxFileBytes *int `json:"max_file_bytes,omitempty"`
+ // MaxEncodedBytes is the documented ceiling on the base64 data URI, for
+ // the protocols that send the audio that way.
+ MaxEncodedBytes *int `json:"max_encoded_bytes,omitempty"`
+ // Formats lists the file extensions the vendor documents, without the
+ // dot. Empty means the vendor documents no closed list.
+ Formats []string `json:"formats,omitempty"`
+ RequestTimeout *int `json:"request_timeout_seconds,omitempty"`
+ // UnsupportedReason marks a model this build cannot call.
+ UnsupportedReason *string `json:"unsupported_reason,omitempty"`
+}
+
+// TranscriptionsSettings is the resolved (fully defaulted) form.
+type TranscriptionsSettings struct {
+ API api.TranscriptionAPI
+ Path string
+ ResponseFormat string
+ LanguageParam string
+ // MaxFileBytes and MaxEncodedBytes refuse an upload the vendor would
+ // reject anyway, before sending it; 0 leaves the check to the vendor.
+ MaxFileBytes int
+ MaxEncodedBytes int
+ Formats []string
+ // RequestTimeout caps one request, in seconds.
+ RequestTimeout int
+ // UnsupportedReason is set where a vendor's recognition models, or some
+ // of them, cannot be called with the audio in the request — they take a
+ // public URL or run as asynchronous tasks. Resolve refuses such a model
+ // rather than sending it a shape it does not accept.
+ UnsupportedReason string
+}
+
+// DefaultTranscriptions is the protocol baseline: a form with file and model
+// and nothing else, which every speech-to-text endpoint in this catalog
+// documents. The deadline is the one the pre-catalog client carried; audio
+// transcription is slow.
+func DefaultTranscriptions() TranscriptionsSettings {
+ return TranscriptionsSettings{Path: "/audio/transcriptions", RequestTimeout: 300}
+}
diff --git a/internal/models/catalog/types.go b/internal/models/catalog/types.go
index 90856f424..bc2985be7 100644
--- a/internal/models/catalog/types.go
+++ b/internal/models/catalog/types.go
@@ -251,6 +251,9 @@ type Vendor struct {
// OpenAI shape for any vendor that serves embeddings without naming
// another.
EmbeddingAPI api.EmbeddingAPI
+ // TranscriptionAPI is the speech-to-text protocol. Register defaults it
+ // to the OpenAI shape for any vendor that serves ASR.
+ TranscriptionAPI api.TranscriptionAPI
// DefaultBaseURLs by model type; GetDefaultURL falls back to chat.
DefaultBaseURLs map[types.ModelType]string
ModelTypes []types.ModelType
diff --git a/internal/models/catalog/validate.go b/internal/models/catalog/validate.go
index eaff1ce68..5955ae8e8 100644
--- a/internal/models/catalog/validate.go
+++ b/internal/models/catalog/validate.go
@@ -14,10 +14,9 @@ import (
// protocol, a bad compat key or an invalid thinking level is rejected where
// it is written instead of surfacing as a failed chat call much later.
//
-// Chat, VLM, embedding and rerank rows are checked; all four are built from
-// catalog.Resolve, so a row that does not resolve cannot be used. ASR rows
-// return nil: the ASR client does not resolve through the catalog yet, so a
-// spec stored on such a row is inert — neither validated nor applied.
+// Every model type is checked: all of them are built from catalog.Resolve,
+// so a row that does not resolve cannot be used and should fail the save
+// rather than the first call.
func ValidateRow(modelName string, modelType types.ModelType, params *types.ModelParameters) error {
if params == nil || !resolvesThroughCatalog(modelType) {
return nil
@@ -44,7 +43,8 @@ func ValidateRow(modelName string, modelType types.ModelType, params *types.Mode
func resolvesThroughCatalog(modelType types.ModelType) bool {
switch modelType {
- case types.ModelTypeKnowledgeQA, types.ModelTypeVLLM, types.ModelTypeEmbedding, types.ModelTypeRerank:
+ case types.ModelTypeKnowledgeQA, types.ModelTypeVLLM, types.ModelTypeEmbedding, types.ModelTypeRerank,
+ types.ModelTypeASR:
return true
}
return false
diff --git a/internal/models/catalog/validate_test.go b/internal/models/catalog/validate_test.go
index 0e6186f96..68034a967 100644
--- a/internal/models/catalog/validate_test.go
+++ b/internal/models/catalog/validate_test.go
@@ -56,9 +56,17 @@ func TestValidateRow(t *testing.T) {
wantErr: true,
},
{
- name: "asr rows are not resolved through the catalog", modelName: "whisper-1",
+ name: "a catalogued asr row resolves", modelName: "whisper-1",
modelType: types.ModelTypeASR,
- params: &types.ModelParameters{Provider: "openai", Spec: &types.ModelSpecOverride{API: "nope"}},
+ params: &types.ModelParameters{Provider: "openai"},
+ },
+ {
+ name: "unknown asr compat key is rejected", modelName: "whisper-1",
+ modelType: types.ModelTypeASR,
+ params: &types.ModelParameters{
+ Provider: "openai", Spec: &types.ModelSpecOverride{Compat: map[string]any{"dimensions_field": "x"}},
+ },
+ wantErr: true,
},
{
name: "a catalogued chat row resolves", modelName: "deepseek-v4-pro",
diff --git a/internal/models/vendors/aliyun/models.json b/internal/models/vendors/aliyun/models.json
index bff7b78f8..fd06a04dd 100644
--- a/internal/models/vendors/aliyun/models.json
+++ b/internal/models/vendors/aliyun/models.json
@@ -55,6 +55,8 @@
{"name": "qwen2.5-vl-embedding (fused vectors only)", "match": "qwen2.5-vl-embedding*", "type": "Embedding", "compat": {"api": "dashscope-embeddings", "dimensions_field": "dimension", "max_batch_size": 1}, "source": "https://help.aliyun.com/zh/model-studio/multimodal-embedding-api-reference"},
{"name": "multimodal-embedding-v1", "match": "multimodal-embedding*", "type": "Embedding", "compat": {"api": "dashscope-embeddings", "dimensions_field": "", "max_batch_size": 20}, "source": "https://help.aliyun.com/zh/model-studio/multimodal-embedding-api-reference"},
{"id": "qwen3-rerank", "name": "Qwen3 Rerank (separate endpoint, not implemented)", "type": "Rerank", "deprecated": true, "compat": {"unsupported_reason": "qwen3-rerank speaks a different protocol (/compatible-api/v1/reranks, flat request, results at the top level) that no protocol package implements yet"}, "source": "https://help.aliyun.com/zh/model-studio/text-rerank-api"},
- {"id": "gte-rerank-v2", "name": "GTE Rerank v2", "type": "Rerank", "source": "https://help.aliyun.com/zh/model-studio/text-rerank-api"}
+ {"id": "gte-rerank-v2", "name": "GTE Rerank v2", "type": "Rerank", "source": "https://help.aliyun.com/zh/model-studio/text-rerank-api"},
+ {"id": "qwen3-asr-flash", "name": "Qwen3 ASR Flash", "type": "ASR", "compat": {"api": "openai-chat-audio", "max_encoded_bytes": 10485760, "language_param": "asr_options"}, "source": "https://help.aliyun.com/zh/model-studio/qwen-asr-api-reference"},
+ {"name": "Other Alibaba recognition models (not callable with the audio in the request)", "match": "*", "type": "ASR", "compat": {"unsupported_reason": "Alibaba's other recognition models take a public file URL or run as asynchronous tasks; only qwen3-asr-flash accepts the audio in the request"}, "source": "https://help.aliyun.com/zh/model-studio/qwen-asr-api-reference"}
]
}
diff --git a/internal/models/vendors/aliyun/vendor.go b/internal/models/vendors/aliyun/vendor.go
index 36d9da82d..9c5672580 100644
--- a/internal/models/vendors/aliyun/vendor.go
+++ b/internal/models/vendors/aliyun/vendor.go
@@ -136,6 +136,7 @@ func init() {
types.ModelTypeEmbedding,
types.ModelTypeRerank,
types.ModelTypeVLLM,
+ types.ModelTypeASR,
},
RerankAPI: api.RerankDashScope,
// Text and multimodal embeddings live under different roots of the
@@ -160,6 +161,13 @@ func init() {
return root + "/compatible-mode/v1/embeddings", nil
},
Compat: catalog.VendorCompat{
+ // ASR: qwen3-asr-flash is the one recognition model callable with
+ // the audio in the request, on the compatible chat endpoint as a
+ // base64 data URI; its catalog entry declares that. Every other
+ // ASR name falls to a catch-all entry that refuses it: Paraformer,
+ // Fun-ASR and the *-filetrans models are asynchronous tasks that
+ // take a public file URL, which no protocol here can send
+ // (https://help.aliyun.com/zh/model-studio/qwen-asr-api-reference).
// Text models: the OpenAI-compatible endpoint
// (https://help.aliyun.com/zh/model-studio/embedding-interfaces-compatible-with-openai),
// which takes model, input, dimensions and encoding_format.
diff --git a/internal/models/vendors/azure_openai/vendor.go b/internal/models/vendors/azure_openai/vendor.go
index 8c3e34188..63e7909c4 100644
--- a/internal/models/vendors/azure_openai/vendor.go
+++ b/internal/models/vendors/azure_openai/vendor.go
@@ -124,14 +124,16 @@ func init() {
types.ModelTypeEmbedding: BaseURL,
types.ModelTypeVLLM: BaseURL,
},
- // ASR is deliberately absent: Azure serves Whisper, but
- // internal/models/asr builds every remote client with a plain
- // OpenAI config (Bearer auth, {base}/audio/transcriptions) and has
- // no provider or extra-config field to route through this vendor's
- // api-key header and deployments path. Advertising the type would
- // let the picker create a row that fails at first use. The Endpoint
- // hook below already knows the ASR path, so re-adding the type is a
- // one-line change once asr.Config carries the provider.
+ // ASR is deliberately absent. The ASR client now resolves through
+ // this vendor (api-key header, Endpoint hook below), but the only
+ // v1 reference for audio is the preview one —
+ // POST {endpoint}/openai/v1/audio/transcriptions?api-version=preview
+ // (https://learn.microsoft.com/en-us/azure/foundry/openai/reference-preview-latest)
+ // — while the hook sends a row without api_version to the v1 path
+ // with no version at all. Until that is verified against a live
+ // resource, advertising the type would let the picker create a row
+ // that may fail at first use. A row with an explicit api_version
+ // would take the documented deployments path.
ModelTypes: []types.ModelType{
types.ModelTypeKnowledgeQA,
types.ModelTypeEmbedding,
diff --git a/internal/models/vendors/generic/vendor.go b/internal/models/vendors/generic/vendor.go
index 8b7a0424b..145530fdc 100644
--- a/internal/models/vendors/generic/vendor.go
+++ b/internal/models/vendors/generic/vendor.go
@@ -108,6 +108,17 @@ func init() {
ModelTypes: []types.ModelType{types.ModelTypeRerank},
}},
Compat: catalog.VendorCompat{
+ // Transcriptions keeps the baseline, json by default. The
+ // pre-catalog client always asked for verbose_json, which not every
+ // model or server can produce: OpenAI's gpt-4o transcribers accept
+ // only json, and vox-box's FunASR backend answers verbose_json with
+ // a bare JSON string. A row that wants segments sets
+ // {"response_format": "verbose_json"} in its compat. The language
+ // hint goes as OpenAI's language form field, which a server that
+ // does not implement it ignores.
+ Transcriptions: catalog.TranscriptionsCompat{
+ LanguageParam: catalog.Ptr(catalog.LanguageForm),
+ },
Embeddings: catalog.EmbeddingsCompat{
// Whatever the operator runs. Everything the pre-catalog client
// sent stays: vLLM, SGLang, TEI and Ollama's OpenAI route all
diff --git a/internal/models/vendors/gpustack/vendor.go b/internal/models/vendors/gpustack/vendor.go
index 89f24dc0d..f42f3ff1c 100644
--- a/internal/models/vendors/gpustack/vendor.go
+++ b/internal/models/vendors/gpustack/vendor.go
@@ -118,6 +118,17 @@ func init() {
ModelTypes: []types.ModelType{types.ModelTypeRerank},
}},
Compat: catalog.VendorCompat{
+ // Transcriptions keeps the baseline, json by default. GPUStack
+ // serves audio through vox-box (https://github.com/gpustack/vox-box),
+ // whose route wraps a json answer as {"text": ...} but returns
+ // verbose_json as whatever the backend produced — and its FunASR
+ // backend (SenseVoice, Paraformer) produces a bare string for every
+ // format. The pre-catalog client's verbose_json therefore arrived
+ // as a JSON string that could not be decoded. The route reads a
+ // language form field.
+ Transcriptions: catalog.TranscriptionsCompat{
+ LanguageParam: catalog.Ptr(catalog.LanguageForm),
+ },
Embeddings: catalog.EmbeddingsCompat{
// vLLM's embedding server: dimensions for Matryoshka models, the
// row's truncation budget, and encoding_format.
diff --git a/internal/models/vendors/litellm/vendor.go b/internal/models/vendors/litellm/vendor.go
index 02d0d96b3..23c24f2af 100644
--- a/internal/models/vendors/litellm/vendor.go
+++ b/internal/models/vendors/litellm/vendor.go
@@ -77,6 +77,7 @@ func init() {
types.ModelTypeEmbedding,
types.ModelTypeRerank,
types.ModelTypeVLLM,
+ types.ModelTypeASR,
},
ExtraFields: []catalog.ExtraField{{
Key: catalog.ExtraScoreScale,
@@ -104,6 +105,9 @@ func init() {
ModelTypes: []types.ModelType{types.ModelTypeRerank},
}},
Compat: catalog.VendorCompat{
+ // Transcriptions keeps the baseline: the proxy serves the OpenAI
+ // /audio/transcriptions route for whichever upstream it is
+ // configured with (https://docs.litellm.ai/docs/audio_transcription).
Embeddings: catalog.EmbeddingsCompat{
// https://docs.litellm.ai/docs/embedding/supported_embedding:
// model, input, user, dimensions, encoding_format; anything else
diff --git a/internal/models/vendors/mimo/models.json b/internal/models/vendors/mimo/models.json
index c54858e31..b2d86cd20 100644
--- a/internal/models/vendors/mimo/models.json
+++ b/internal/models/vendors/mimo/models.json
@@ -5,6 +5,7 @@
{"id": "mimo-v2.5-pro", "name": "MiMo V2.5 Pro", "reasoning": true, "input": ["text"], "context_window": 1048576, "max_output_tokens": 131072, "cost": {"input": 0.435, "output": 0.87, "cache_read": 0.0036}, "source": "https://mimo.mi.com/static/docs/price/pay-as-you-go.md"},
{"id": "mimo-v2.5", "name": "MiMo V2.5", "reasoning": true, "input": ["text", "image", "audio", "video"], "context_window": 1048576, "max_output_tokens": 131072, "cost": {"input": 0.14, "output": 0.28, "cache_read": 0.0028}, "source": "https://mimo.mi.com/static/docs/price/pay-as-you-go.md"},
{"id": "mimo-v2.5-pro-ultraspeed", "name": "MiMo V2.5 Pro Ultraspeed", "reasoning": true, "input": ["text"], "context_window": 1048576, "max_output_tokens": 131072, "cost": {"input": 1.305, "output": 2.61, "cache_read": 0.0108}, "source": "https://mimo.mi.com/static/docs/news/latest/beta-extended.md"},
- {"id": "mimo-v2-flash", "name": "MiMo V2 Flash (retired 2026-06-30)", "reasoning": true, "input": ["text"], "context_window": 256000, "max_output_tokens": 256000, "cost": {"input": 0.1, "output": 0.3, "cache_read": 0.01}, "deprecated": true, "source": "https://mimo.mi.com/static/docs/updates/deprecate.md"}
+ {"id": "mimo-v2-flash", "name": "MiMo V2 Flash (retired 2026-06-30)", "reasoning": true, "input": ["text"], "context_window": 256000, "max_output_tokens": 256000, "cost": {"input": 0.1, "output": 0.3, "cache_read": 0.01}, "deprecated": true, "source": "https://mimo.mi.com/static/docs/updates/deprecate.md"},
+ {"id": "mimo-v2.5-asr", "name": "MiMo V2.5 ASR", "type": "ASR", "source": "https://mimo.mi.com/docs/en-US/quick-start/usage-guide/audio/Speech-Recognition"}
]
}
diff --git a/internal/models/vendors/mimo/vendor.go b/internal/models/vendors/mimo/vendor.go
index f3c7ec138..48ece1f13 100644
--- a/internal/models/vendors/mimo/vendor.go
+++ b/internal/models/vendors/mimo/vendor.go
@@ -71,10 +71,22 @@ func init() {
DefaultBaseURLs: map[types.ModelType]string{
types.ModelTypeKnowledgeQA: BaseURL,
},
+ TranscriptionAPI: api.TranscriptionChatAudio,
ModelTypes: []types.ModelType{
types.ModelTypeKnowledgeQA,
+ types.ModelTypeASR,
},
Compat: catalog.VendorCompat{
+ // ASR: mimo-v2.5-asr on the chat endpoint, WAV or MP3 as a base64
+ // data URI whose "encoded string size must not exceed 10 MB"
+ // (https://mimo.mi.com/docs/en-US/quick-start/usage-guide/audio/Speech-Recognition).
+ Transcriptions: catalog.TranscriptionsCompat{
+ // The ceiling counts the whole data URI as sent.
+ MaxEncodedBytes: catalog.Ptr(10 << 20),
+ Formats: []string{"wav", "mp3"},
+ // asr_options.language: "auto|zh|en".
+ LanguageParam: catalog.Ptr(catalog.LanguageASROptions),
+ },
OpenAICompletions: catalog.OpenAICompletionsCompat{
ThinkingFormat: catalog.Ptr(catalog.ThinkingFormatThinkingType),
PromptCacheAccounting: catalog.Ptr(true),
diff --git a/internal/models/vendors/minimax/models.json b/internal/models/vendors/minimax/models.json
index 7aababf42..b518a6906 100644
--- a/internal/models/vendors/minimax/models.json
+++ b/internal/models/vendors/minimax/models.json
@@ -9,6 +9,7 @@
{"id": "MiniMax-M2.5-highspeed", "name": "MiniMax M2.5 Highspeed", "reasoning": true, "input": ["text"], "context_window": 204800, "max_output_tokens": 131072, "cost": {"input": 0.6, "output": 2.4, "cache_read": 0.06, "cache_write": 0.375}, "thinking_levels": {"off": null}},
{"id": "MiniMax-M2.1", "name": "MiniMax M2.1", "reasoning": true, "input": ["text"], "context_window": 204800, "max_output_tokens": 131072, "cost": {"input": 0.3, "output": 1.2, "cache_read": 0.03, "cache_write": 0.375}, "thinking_levels": {"off": null}},
{"id": "MiniMax-M2.1-highspeed", "name": "MiniMax M2.1 Highspeed", "reasoning": true, "input": ["text"], "context_window": 204800, "max_output_tokens": 131072, "thinking_levels": {"off": null}, "source": "https://platform.minimax.io/docs/guides/text-generation"},
- {"id": "MiniMax-M2", "name": "MiniMax M2", "reasoning": true, "input": ["text"], "context_window": 204800, "max_output_tokens": 131072, "cost": {"input": 0.3, "output": 1.2}, "thinking_levels": {"off": null}}
+ {"id": "MiniMax-M2", "name": "MiniMax M2", "reasoning": true, "input": ["text"], "context_window": 204800, "max_output_tokens": 131072, "cost": {"input": 0.3, "output": 1.2}, "thinking_levels": {"off": null}},
+ {"id": "asr-1.0", "name": "MiniMax ASR 1.0", "type": "ASR", "source": "https://platform.minimax.io/docs/api-reference/speech-to-text"}
]
}
diff --git a/internal/models/vendors/minimax/vendor.go b/internal/models/vendors/minimax/vendor.go
index 08652c127..d5a6f156c 100644
--- a/internal/models/vendors/minimax/vendor.go
+++ b/internal/models/vendors/minimax/vendor.go
@@ -95,8 +95,23 @@ func init() {
},
ModelTypes: []types.ModelType{
types.ModelTypeKnowledgeQA,
+ types.ModelTypeASR,
},
Compat: catalog.VendorCompat{
+ Transcriptions: catalog.TranscriptionsCompat{
+ // https://platform.minimax.io/docs/api-reference/speech-to-text
+ // (and the same page on platform.minimax.cn): multipart model
+ // (asr-1.0) + file on /v1/speech_to_text rather than the OpenAI
+ // path, answering {text, duration, trace_id} for the default
+ // json. At most 50 MB and 500 seconds; "超出会返回 400 而不会被截断".
+ Path: catalog.Ptr("/speech_to_text"),
+ MaxFileBytes: catalog.Ptr(50 << 20),
+ // "wav / aiff / flac / alac(m4a) / mp3 / aac / opus / ogg";
+ // "不支持无容器的裸 PCM 数据". The language hint is a request
+ // header, a BCP-47 tag, not a form field.
+ Formats: []string{"wav", "aiff", "flac", "m4a", "mp3", "aac", "opus", "ogg"},
+ LanguageParam: catalog.Ptr(catalog.LanguageHeader),
+ },
OpenAICompletions: catalog.OpenAICompletionsCompat{
ThinkingFormat: catalog.Ptr(catalog.ThinkingFormatThinkingType),
ThinkingEnabledValue: catalog.Ptr("adaptive"),
diff --git a/internal/models/vendors/openai/models.json b/internal/models/vendors/openai/models.json
index be7775ce5..b67103769 100644
--- a/internal/models/vendors/openai/models.json
+++ b/internal/models/vendors/openai/models.json
@@ -29,6 +29,10 @@
{"id": "text-embedding-3-small", "name": "text-embedding-3-small", "type": "Embedding", "dimension": 1536, "context_window": 8191, "cost": {"input": 0.02, "output": 0}},
{"id": "text-embedding-3-large", "name": "text-embedding-3-large", "type": "Embedding", "dimension": 3072, "context_window": 8191, "cost": {"input": 0.13, "output": 0}},
+ {"id": "whisper-1", "name": "Whisper", "type": "ASR", "compat": {"response_format": "verbose_json"}, "source": "https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions/methods/create"},
+ {"id": "gpt-transcribe", "name": "GPT Transcribe", "type": "ASR", "source": "https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions/methods/create"},
+ {"id": "gpt-4o-transcribe", "name": "GPT-4o Transcribe", "type": "ASR", "source": "https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions/methods/create"},
+ {"id": "gpt-4o-mini-transcribe", "name": "GPT-4o mini Transcribe", "type": "ASR", "source": "https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions/methods/create"},
{"id": "text-embedding-ada-002", "name": "text-embedding-ada-002", "type": "Embedding", "dimension": 1536, "context_window": 8192, "cost": {"input": 0.1, "output": 0}, "compat": {"dimensions_field": ""}}
]
}
diff --git a/internal/models/vendors/openai/vendor.go b/internal/models/vendors/openai/vendor.go
index a6f99249d..d1d02d539 100644
--- a/internal/models/vendors/openai/vendor.go
+++ b/internal/models/vendors/openai/vendor.go
@@ -97,6 +97,19 @@ func init() {
types.ModelTypeASR,
},
Compat: catalog.VendorCompat{
+ Transcriptions: catalog.TranscriptionsCompat{
+ // The transcription reference (URL in openaitranscriptions):
+ // response_format differs per model — "For gpt-4o-transcribe
+ // and gpt-4o-mini-transcribe, the only supported format is
+ // json", the default — so it is declared per entry, not here.
+ // "Files can be up to 25 MB"
+ // (https://developers.openai.com/api/docs/guides/speech-to-text).
+ MaxFileBytes: catalog.Ptr(25 << 20),
+ // "flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm", and a
+ // language field in ISO-639-1.
+ Formats: []string{"flac", "mp3", "mp4", "mpeg", "mpga", "m4a", "ogg", "wav", "webm"},
+ LanguageParam: catalog.Ptr(catalog.LanguageForm),
+ },
Embeddings: catalog.EmbeddingsCompat{
// https://developers.openai.com/api/reference/resources/embeddings/methods/create:
// model, input, dimensions (text-embedding-3 and later; the ada-002
diff --git a/internal/models/vendors/openrouter/models.json b/internal/models/vendors/openrouter/models.json
index a76f2d054..23660cd7c 100644
--- a/internal/models/vendors/openrouter/models.json
+++ b/internal/models/vendors/openrouter/models.json
@@ -19,6 +19,7 @@
{"id": "x-ai/grok-4.6", "name": "Grok 4.6", "reasoning": true, "input": ["text", "image"], "context_window": 500000, "max_output_tokens": 450000, "cost": {"input": 2, "output": 6, "cache_read": 0.5}, "thinking_levels": {"off": null, "minimal": "low", "xhigh": "xhigh", "max": "xhigh"}},
{"id": "openai/text-embedding-3-small", "name": "text-embedding-3-small", "type": "Embedding", "dimension": 1536, "context_window": 8192, "source": "https://openrouter.ai/docs/api_reference/embeddings"},
{"id": "openai/text-embedding-3-large", "name": "text-embedding-3-large", "type": "Embedding", "dimension": 3072, "context_window": 8192, "source": "https://openrouter.ai/docs/api_reference/embeddings"},
- {"id": "cohere/rerank-v3.5", "name": "Cohere Rerank v3.5", "type": "Rerank", "source": "https://openrouter.ai/docs/api/api-reference/rerank/submit-a-rerank-request"}
+ {"id": "cohere/rerank-v3.5", "name": "Cohere Rerank v3.5", "type": "Rerank", "source": "https://openrouter.ai/docs/api/api-reference/rerank/submit-a-rerank-request"},
+ {"id": "openai/whisper-large-v3", "name": "Whisper Large v3", "type": "ASR", "source": "https://openrouter.ai/docs/api/api-reference/stt/create-transcription"}
]
}
diff --git a/internal/models/vendors/openrouter/vendor.go b/internal/models/vendors/openrouter/vendor.go
index c475ea4e7..38d62c260 100644
--- a/internal/models/vendors/openrouter/vendor.go
+++ b/internal/models/vendors/openrouter/vendor.go
@@ -75,8 +75,18 @@ func init() {
types.ModelTypeEmbedding,
types.ModelTypeRerank,
types.ModelTypeVLLM,
+ types.ModelTypeASR,
},
Compat: catalog.VendorCompat{
+ Transcriptions: catalog.TranscriptionsCompat{
+ // https://openrouter.ai/docs/api/api-reference/stt/create-transcription:
+ // OpenAI-style multipart file + model, json by default. "Max 25
+ // MB; send larger files as base64 JSON via input_audio."
+ MaxFileBytes: catalog.Ptr(25 << 20),
+ // language is an ISO-639-1 form field. The format "is derived
+ // from the filename extension", with no closed list.
+ LanguageParam: catalog.Ptr(catalog.LanguageForm),
+ },
Embeddings: catalog.EmbeddingsCompat{
// https://openrouter.ai/docs/api/api-reference/embeddings/create-embeddings:
// model, input, dimensions, encoding_format, input_type, provider,
diff --git a/internal/models/vendors/requesty/models.json b/internal/models/vendors/requesty/models.json
index 360c6a99b..c1d2d27e2 100644
--- a/internal/models/vendors/requesty/models.json
+++ b/internal/models/vendors/requesty/models.json
@@ -12,6 +12,9 @@
{"id": "google/gemini-3.6-flash", "name": "Gemini 3.6 Flash", "reasoning": true, "input": ["text", "image", "audio", "video"], "context_window": 1048576, "max_output_tokens": 65536, "thinking_levels": {"off": null, "minimal": "min", "xhigh": "high", "max": "high"}},
{"id": "deepseek/deepseek-v4-pro-0813", "name": "DeepSeek V4 Pro", "reasoning": true, "input": ["text"], "context_window": 1000000, "max_output_tokens": 131072},
{"id": "zai/glm-5.2", "name": "GLM-5.2", "reasoning": true, "input": ["text"], "context_window": 1048576, "max_output_tokens": 131072},
- {"id": "moonshot/kimi-k3", "name": "Kimi K3", "reasoning": true, "input": ["text", "image", "video"], "context_window": 1048576, "max_output_tokens": 262144}
+ {"id": "moonshot/kimi-k3", "name": "Kimi K3", "reasoning": true, "input": ["text", "image", "video"], "context_window": 1048576, "max_output_tokens": 262144},
+ {"id": "openai/gpt-4o-transcribe", "name": "GPT-4o Transcribe", "type": "ASR", "source": "https://docs.requesty.ai/api-reference/endpoint/audio-transcriptions-create"},
+ {"id": "openai/gpt-4o-mini-transcribe", "name": "GPT-4o mini Transcribe", "type": "ASR", "source": "https://docs.requesty.ai/api-reference/endpoint/audio-transcriptions-create"},
+ {"id": "openai/whisper-1", "name": "Whisper", "type": "ASR", "compat": {"response_format": "verbose_json"}, "source": "https://docs.requesty.ai/api-reference/endpoint/audio-transcriptions-create"}
]
}
diff --git a/internal/models/vendors/requesty/vendor.go b/internal/models/vendors/requesty/vendor.go
index 283f44c83..74be132d9 100644
--- a/internal/models/vendors/requesty/vendor.go
+++ b/internal/models/vendors/requesty/vendor.go
@@ -77,8 +77,19 @@ func init() {
types.ModelTypeKnowledgeQA,
types.ModelTypeEmbedding,
types.ModelTypeVLLM,
+ types.ModelTypeASR,
},
Compat: catalog.VendorCompat{
+ Transcriptions: catalog.TranscriptionsCompat{
+ // https://docs.requesty.ai/api-reference/endpoint/audio-transcriptions-create:
+ // OpenAI-compatible multipart, json by default, verbose_json
+ // available. "The maximum upload size per request is 32 MB."
+ MaxFileBytes: catalog.Ptr(32 << 20),
+ // "flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, webm"; language
+ // is an ISO 639-1 form field.
+ Formats: []string{"flac", "mp3", "mp4", "mpeg", "mpga", "m4a", "ogg", "wav", "webm"},
+ LanguageParam: catalog.Ptr(catalog.LanguageForm),
+ },
Embeddings: catalog.EmbeddingsCompat{
// https://docs.requesty.ai/api-reference/endpoint/embeddings-create:
// model, input, encoding_format, dimensions.
diff --git a/internal/models/vendors/siliconflow/models.json b/internal/models/vendors/siliconflow/models.json
index e51084c6e..44cb98d2d 100644
--- a/internal/models/vendors/siliconflow/models.json
+++ b/internal/models/vendors/siliconflow/models.json
@@ -24,6 +24,7 @@
{"id": "Qwen/Qwen3-Embedding-0.6B", "name": "Qwen3 Embedding 0.6B", "type": "Embedding", "dimension": 1024, "source": "https://api-docs.siliconflow.cn/docs/api/embeddings-post"},
{"id": "BAAI/bge-reranker-v2-m3", "name": "BGE Reranker v2 M3", "type": "Rerank", "source": "https://docs.siliconflow.cn/cn/api-reference/rerank/create-rerank"},
{"id": "Qwen/Qwen3-Reranker-8B", "name": "Qwen3 Reranker 8B", "type": "Rerank", "source": "https://docs.siliconflow.cn/cn/api-reference/rerank/create-rerank"},
- {"id": "FunAudioLLM/SenseVoiceSmall", "name": "SenseVoice Small", "type": "ASR", "source": "https://docs.siliconflow.cn/cn/api-reference/audio/create-audio-transcriptions"}
+ {"id": "FunAudioLLM/SenseVoiceSmall", "name": "SenseVoice Small", "type": "ASR", "source": "https://api-docs.siliconflow.cn/docs/api/audio-transcriptions-post"},
+ {"id": "TeleAI/TeleSpeechASR", "name": "TeleSpeechASR", "type": "ASR", "source": "https://api-docs.siliconflow.cn/docs/api/audio-transcriptions-post"}
]
}
diff --git a/internal/models/vendors/siliconflow/vendor.go b/internal/models/vendors/siliconflow/vendor.go
index f332e6fce..1421d5d72 100644
--- a/internal/models/vendors/siliconflow/vendor.go
+++ b/internal/models/vendors/siliconflow/vendor.go
@@ -77,6 +77,12 @@ func init() {
types.ModelTypeASR,
},
Compat: catalog.VendorCompat{
+ Transcriptions: catalog.TranscriptionsCompat{
+ // https://api-docs.siliconflow.cn/docs/api/audio-transcriptions-post:
+ // file and model only — no response_format — and a file of
+ // "时长不超过 1 小时,文件大小不超过 50MB".
+ MaxFileBytes: catalog.Ptr(50 << 20),
+ },
Embeddings: catalog.EmbeddingsCompat{
// https://api-docs.siliconflow.cn/docs/api/embeddings-post: model,
// input, encoding_format, and dimensions "仅 Qwen/Qwen3 系列支持"
diff --git a/internal/models/vendors/vendors_test.go b/internal/models/vendors/vendors_test.go
index 452e6c390..f3ebb50fd 100644
--- a/internal/models/vendors/vendors_test.go
+++ b/internal/models/vendors/vendors_test.go
@@ -110,6 +110,9 @@ func TestEveryCatalogEntryResolves(t *testing.T) {
if modelType == types.ModelTypeKnowledgeQA && !r.API.Known() {
t.Errorf("%s/%s: resolved to unknown API %q", id, name, r.API)
}
+ if modelType == types.ModelTypeASR && !r.TranscriptionAPI.Known() {
+ t.Errorf("%s/%s: resolved to unknown transcription API %q", id, name, r.TranscriptionAPI)
+ }
if modelType == types.ModelTypeEmbedding && !r.EmbeddingAPI.Known() {
t.Errorf("%s/%s: resolved to unknown embedding API %q", id, name, r.EmbeddingAPI)
}
diff --git a/internal/models/vendors/zhipu/models.json b/internal/models/vendors/zhipu/models.json
index 62c15a1eb..bd2ea0e3c 100644
--- a/internal/models/vendors/zhipu/models.json
+++ b/internal/models/vendors/zhipu/models.json
@@ -15,6 +15,7 @@
{"id": "glm-4.5-air", "name": "GLM-4.5 Air", "reasoning": true, "input": ["text"], "context_window": 131072, "max_output_tokens": 98304, "cost": {"input": 0.2, "output": 1.1, "cache_read": 0.03}, "compat": {"supports_reasoning_effort": false}},
{"id": "embedding-3", "name": "Embedding-3", "type": "Embedding", "dimension": 2048, "compat": {"max_batch_size": 64}, "source": "https://docs.bigmodel.cn/api-reference/模型-api/文本嵌入"},
{"id": "embedding-2", "name": "Embedding-2", "type": "Embedding", "dimension": 1024, "compat": {"dimensions_field": ""}, "source": "https://docs.bigmodel.cn/api-reference/模型-api/文本嵌入"},
- {"id": "rerank", "name": "Rerank", "type": "Rerank", "source": "https://docs.bigmodel.cn/api-reference/模型-api/文本重排序"}
+ {"id": "rerank", "name": "Rerank", "type": "Rerank", "source": "https://docs.bigmodel.cn/api-reference/模型-api/文本重排序"},
+ {"id": "glm-asr-2512", "name": "GLM-ASR-2512 (at most 30 s per file)", "type": "ASR", "source": "https://docs.bigmodel.cn/api-reference/模型-api/语音转文本"}
]
}
diff --git a/internal/models/vendors/zhipu/vendor.go b/internal/models/vendors/zhipu/vendor.go
index bf9942fa3..3b4e5d69b 100644
--- a/internal/models/vendors/zhipu/vendor.go
+++ b/internal/models/vendors/zhipu/vendor.go
@@ -107,8 +107,19 @@ func init() {
types.ModelTypeEmbedding,
types.ModelTypeRerank,
types.ModelTypeVLLM,
+ types.ModelTypeASR,
},
Compat: catalog.VendorCompat{
+ Transcriptions: catalog.TranscriptionsCompat{
+ // https://docs.bigmodel.cn/api-reference/模型-api/语音转文本:
+ // POST {base}/audio/transcriptions, multipart file + model
+ // (glm-asr-2512), answering {text}. The file is ".wav / .mp3",
+ // at most 25 MB and 30 seconds — the duration cannot be checked
+ // here without decoding, so a longer file is the vendor's error.
+ MaxFileBytes: catalog.Ptr(25 << 20),
+ // No language field; prompt and hotwords are its only hints.
+ Formats: []string{"wav", "mp3"},
+ },
Embeddings: catalog.EmbeddingsCompat{
// https://docs.bigmodel.cn/api-reference/模型-api/文本嵌入: model,
// input, dimensions — no encoding_format. embedding-2 is fixed at
diff --git a/internal/utils/extraheaders.go b/internal/utils/extraheaders.go
index b32d54f8e..c6f644b84 100644
--- a/internal/utils/extraheaders.go
+++ b/internal/utils/extraheaders.go
@@ -44,44 +44,3 @@ func ApplyCustomHeaders(req *http.Request, headers map[string]string) {
req.Header.Set(name, v)
}
}
-
-// CustomHeadersRoundTripper 是一个 http.RoundTripper 包装器,
-// 会在每个 HTTP 请求发出前注入用户自定义的 header。
-// 用于无法直接拿到底层 *http.Request 的场景(如 go-openai SDK)。
-type CustomHeadersRoundTripper struct {
- Headers map[string]string
- Base http.RoundTripper
-}
-
-// RoundTrip 实现 http.RoundTripper 接口。
-func (t *CustomHeadersRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
- base := t.Base
- if base == nil {
- base = http.DefaultTransport
- }
- if len(t.Headers) > 0 {
- // 复制请求以避免修改调用方传入的对象。
- cloned := req.Clone(req.Context())
- ApplyCustomHeaders(cloned, t.Headers)
- return base.RoundTrip(cloned)
- }
- return base.RoundTrip(req)
-}
-
-// WrapHTTPClientWithHeaders 返回一个新的 *http.Client,在原有 client 基础上注入自定义 header。
-// 如果 headers 为空则直接返回原 client,避免不必要的开销。
-func WrapHTTPClientWithHeaders(client *http.Client, headers map[string]string) *http.Client {
- if len(headers) == 0 {
- return client
- }
- if client == nil {
- client = NewSSRFSafeHTTPClient(DefaultSSRFSafeHTTPClientConfig())
- }
- base := client.Transport
- if base == nil {
- base = http.DefaultTransport
- }
- wrapped := *client
- wrapped.Transport = &CustomHeadersRoundTripper{Headers: headers, Base: base}
- return &wrapped
-}
diff --git a/internal/utils/extraheaders_test.go b/internal/utils/extraheaders_test.go
index 5f4ecf330..451779562 100644
--- a/internal/utils/extraheaders_test.go
+++ b/internal/utils/extraheaders_test.go
@@ -1,10 +1,7 @@
package utils
import (
- "io"
"net/http"
- "net/http/httptest"
- "strings"
"testing"
)
@@ -48,47 +45,3 @@ func TestApplyCustomHeaders_NilSafe(t *testing.T) {
t.Fatalf("unexpected headers added: %+v", req.Header)
}
}
-
-func TestWrapHTTPClientWithHeaders(t *testing.T) {
- t.Setenv("SSRF_WHITELIST", "127.0.0.1,::1,localhost")
- ResetSSRFWhitelistForTest()
- t.Cleanup(ResetSSRFWhitelistForTest)
- gotTrace := ""
- gotAuth := ""
- srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- gotTrace = r.Header.Get("X-Trace-Id")
- gotAuth = r.Header.Get("Authorization")
- io.Copy(io.Discard, r.Body)
- w.WriteHeader(http.StatusOK)
- }))
- defer srv.Close()
-
- client := WrapHTTPClientWithHeaders(nil, map[string]string{
- "X-Trace-Id": "rt-1",
- "Authorization": "Bearer should-not-override",
- })
-
- req, _ := http.NewRequest("POST", srv.URL, strings.NewReader("{}"))
- req.Header.Set("Authorization", "Bearer kept")
-
- resp, err := client.Do(req)
- if err != nil {
- t.Fatalf("request failed: %v", err)
- }
- resp.Body.Close()
-
- if gotTrace != "rt-1" {
- t.Fatalf("expected custom header injected, got %q", gotTrace)
- }
- if gotAuth != "Bearer kept" {
- t.Fatalf("reserved header must not be overridden, got %q", gotAuth)
- }
-}
-
-func TestWrapHTTPClientWithHeaders_EmptyReturnsOriginal(t *testing.T) {
- orig := &http.Client{}
- wrapped := WrapHTTPClientWithHeaders(orig, nil)
- if wrapped != orig {
- t.Fatalf("expected original client returned when headers empty")
- }
-}
diff --git a/website-docs/02-architecture/01-overview.md b/website-docs/02-architecture/01-overview.md
index 6687441cb..d49091db1 100644
--- a/website-docs/02-architecture/01-overview.md
+++ b/website-docs/02-architecture/01-overview.md
@@ -58,7 +58,7 @@ WeKnora 采用"主服务 + 前端 + 文档解析微服务"的三进程核心架
| 配置 | `github.com/spf13/viper` + `config/config.yaml` + 环境变量 | — |
| 可观测 | OpenTelemetry + Langfuse(`internal/tracing/langfuse`) | LLM 调用级 trace |
| gRPC | `google.golang.org/grpc` v1.81.0 | 调用 docreader |
-| LLM 接入 | `sashabaranov/go-openai`、Ollama、腾讯云 LKE 等 | 18+ 模型提供商(OpenAI 兼容 / Ollama / 云厂商 SDK) |
+| LLM 接入 | 自研协议层 `internal/models/api`(OpenAI / Anthropic / Gemini / DashScope 等线格式各一个包)、Ollama、腾讯云 LKE SDK 等 | 27 家厂商由 `internal/models/vendors` 声明,见[模型管理](../03-features/06-models.md) |
| 向量/检索 | pgvector、ES v7/v8、OpenSearch、Qdrant、Milvus、Weaviate、Doris、腾讯 VectorDB、sqlite-vec | 由 `RETRIEVE_DRIVER` 与 `vector_stores` 表动态装配 |
| 知识图谱 | `neo4j-go-driver/v6` | 可选 |
| 数据分析 | DuckDB(`duckdb-go/v2`)、`pg_query_go` SQL 校验 | Agent 数据分析工具 |
diff --git a/website-docs/03-features/06-models.md b/website-docs/03-features/06-models.md
index e0af171a1..3d8ccf7e6 100644
--- a/website-docs/03-features/06-models.md
+++ b/website-docs/03-features/06-models.md
@@ -56,7 +56,7 @@ const (
| `Embedding` | `embedding` | `internal/models/embedding` | `Embed` / `BatchEmbed`(含 `GetDimensions`) | 文本向量化,供向量检索索引与查询 |
| `Rerank` | `rerank` | `internal/models/rerank` | `Rerank(query, documents)` 返回 `RankResult` | 检索结果精排 |
| `VLLM` | `vllm` | `internal/models/vlm` | `Predict(imgBytes, prompt)` | 视觉语言模型(VLM),文档图片理解 / 多模态解析 |
-| `ASR` | `asr` | `internal/models/asr` | `Transcribe(audioBytes, fileName)` 返回文本与分段时间戳 | 音频转写(自动语音识别) |
+| `ASR` | `asr` | `internal/models/asr` | `Transcribe(audioBytes, fileName)` 返回文本;模型提供时附带分段时间戳(如 OpenAI `whisper-1`) | 音频转写(自动语音识别) |
前后端类型映射见 `internal/handler/model.go` 的 `modelTypeToFrontend()`(`KnowledgeQA -> chat` 等)。
@@ -176,7 +176,7 @@ builtin_models:
| 层 | 位置 | 职责 |
|----|------|------|
-| 协议层 | `internal/models/api/<protocol>` | 一个 wire 协议一个包。对话:`openaicompletions`、`openairesponses`、`anthropicmessages`、`googlegenai`;重排:`cohererank`、`dashscoperank`、`nimrerank`;向量:`openaiembeddings`、`dashscopeembeddings`、`arkembeddings`、`googleembeddings`。各自持有请求/响应结构与解析,不含任何厂商名 |
+| 协议层 | `internal/models/api/<protocol>` | 一个 wire 协议一个包。对话:`openaicompletions`、`openairesponses`、`anthropicmessages`、`googlegenai`;重排:`cohererank`、`dashscoperank`、`nimrerank`;向量:`openaiembeddings`、`dashscopeembeddings`、`arkembeddings`、`googleembeddings`;语音:`openaitranscriptions`。各自持有请求/响应结构与解析,不含任何厂商名 |
| 厂商层 | `internal/models/vendors/<id>/` | 一个厂商一个目录:`vendor.go`(注册 `catalog.Vendor`)、`models.json`(模型目录)、`icon.svg`(品牌图标,`go:embed` 进二进制) |
| 目录层 | `internal/models/catalog` | 合并厂商定义、模型条目、部署叠加与单行覆盖,`Resolve` 得出「这个模型到底怎么发请求」 |
@@ -235,7 +235,7 @@ make model-catalog-check
前端不需要任何改动:厂商下拉、图标、额外字段、内置模型列表都由 `GET /api/v1/models/providers` 动态渲染。
-写入侧也有一道闸:`catalog.ValidateRow` 会在创建 / 更新模型(REST)和加载 `config/builtin_models.yaml`(启动)时解析这行配置(chat、VLM、embedding、rerank 四类;ASR 还没进目录),未知协议、拼错的 compat 键、非法的思考档位在写入时就被拒绝(YAML 行只打 WARN 不阻塞启动,避免一次重启把线上模型下线)。
+写入侧也有一道闸:`catalog.ValidateRow` 会在创建 / 更新模型(REST)和加载 `config/builtin_models.yaml`(启动)时解析这行配置(全部模型类型),未知协议、拼错的 compat 键、非法的思考档位在写入时就被拒绝(YAML 行只打 WARN 不阻塞启动,避免一次重启把线上模型下线)。
#### 厂商更新了模型怎么办
@@ -280,7 +280,21 @@ Embedding 行也有几处按厂商文档纠正的行为变化(逐厂商的出
7. **OpenAI 兼容回复里没有 `index` 时按顺序取**(和旧实现一致);有 `index` 就按它放回,重复或缺位报错。
8. **Jina 的 `task`、Gemini 的 `taskType`、OpenRouter 的 `input_type`、火山的 `instructions`、百炼原生接口的 `text_type` / `instruct` 都不发**。它们会改变文档侧向量,开启后同一个知识库里新旧向量不在同一空间;需要按行显式开启的设计另见 Tencent/WeKnora#1401。
-另外 Azure OpenAI 不再声明支持 ASR(ASR 客户端只会构造标准 OpenAI 客户端,根本无法带上 Azure 的 `api-key` 头和部署路径,这类行此前就调不通)。
+ASR 行的变化(逐厂商的出站表单由 `internal/models/asr/wire_test.go` 钉住):
+
+1. **不再一律发 `response_format=verbose_json`**。OpenAI 文档写明 gpt-4o-transcribe / gpt-4o-mini-transcribe「only supported format is json」,旧实现发给它们必然 400;GPUStack 的音频后端 vox-box 在 FunASR 模型(SenseVoice、Paraformer)上对 `verbose_json` 返回裸字符串,旧实现解不开。现在只有文档写明支持的模型声明它(OpenAI `whisper-1`),其余走默认 json,Langfuse 里的音频时长因此只在返回分段的模型上有。需要分段的自建行可以在 `spec.compat` 里写 `{"response_format": "verbose_json"}`。
+2. **按厂商文档的上限和格式在上传前拒绝**:文件大小(OpenAI / 智谱 / OpenRouter 25 MB、Requesty 32 MB、SiliconFlow / MiniMax 50 MB;阿里与小米按整段 `data:` URI 计 10 MB),以及文档给出封闭格式清单的厂商(智谱、小米只收 wav/mp3,OpenAI、Requesty、MiniMax 各有列表)。
+3. **回复里没有 `text` 字段即报错**,不再当成「未检测到语音」入库;静音音频返回的是空字符串 `text`,照常处理。
+4. ASR 行现在带上 provider 走目录解析,go-openai 依赖随之移除。provider 没有声明 ASR 的行:显式填了该厂商的直接报错;没填、靠 URL 识别到的,按 OpenAI 形状打到它的 URL(与旧实现一致),不走该厂商的 Endpoint 钩子。
+5. **支持 ASR 的厂商按文档补齐**。OpenAI 形状(multipart `file` + `model`):openai、siliconflow、gpustack、generic、智谱(`glm-asr-2512`,单文件 ≤30 秒)、MiniMax(`asr-1.0`,路径是 `/v1/speech_to_text`)、OpenRouter、Requesty、LiteLLM。经 chat completions 的 `input_audio`(base64 data URI,编码后 ≤10 MB):阿里云 `qwen3-asr-flash`、小米 `mimo-v2.5-asr`。
+6. **知识库的「音频语言提示」终于发出去了**(此前从未接线)。位置按厂商文档:OpenAI / Requesty / OpenRouter / GPUStack / generic 是 `language` 表单字段,MiniMax 是 `language` 请求头,阿里与小米是 `asr_options.language`;智谱、SiliconFlow、LiteLLM 文档没有这个参数,不发。填 `auto` 等同留空。
+7. **音频时长从回包里读**:MiniMax 的 `duration`、OpenAI 系与阿里的 `usage.seconds`,Langfuse 不再只靠 `verbose_json` 的分段。
+
+阿里只有 `qwen3-asr-flash` 能直接带音频调用;其余 ASR 模型名由一条兜底条目拒绝并说明原因。
+
+查过但没有接的:火山豆包语音(独立域名与密钥,当前文档的请求体只收音频 URL)、千帆(`vop_asr` 挂在应用实例下要 `app_id`;短语音接口 ≤60 秒且只收 pcm/wav/amr/m4a)、七牛(只收音频 URL)、Novita(GLM-ASR 自有接口,≤30 秒)、腾讯云 ASR(与混元不是同一套鉴权)、NVIDIA 托管 Riva(gRPC)、Gemini(没有专用转写接口,只能让对话模型听音频)、Azure(见下)。阿里 Paraformer / Fun-ASR / `*-filetrans` 是异步任务,同样要公网 URL。
+
+Azure OpenAI 仍不声明 ASR:音频转写只出现在 v1 **preview** 参考里(`/openai/v1/audio/transcriptions?api-version=preview`),v1 GA 参考没有;未填 `api_version` 的行会被钩子发到不带版本的 v1 路径,能否调通没有核实。
### 模型调用链