Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(gpustack): rework provider setup and fix driver issues (#21903)

foyouf9a0c3e83e16
diff --git a/internal/entity/models/gpustack.go b/internal/entity/models/gpustack.go
index 052f6a72db..776adab186 100644
--- a/internal/entity/models/gpustack.go
+++ b/internal/entity/models/gpustack.go
@@ -24,7 +24,11 @@ import (
"io"
"net/http"
"ragflow/internal/common"
+ "slices"
+ "sort"
"strings"
+
+ "go.uber.org/zap"
)
// GPUStackModel implements ModelDriver for GPUStack
@@ -108,12 +112,33 @@ func (g *GPUStackModel) ChatStreamlyWithSender(ctx context.Context, modelName st
})
}
-type gpustackModelInfo struct {
- ID string `json:"id"`
+type gpustackModelItem struct {
+ ModelListItem
+ Meta struct {
+ Voices []string `json:"voices"`
+ MaxModelLen *int `json:"max_model_len"`
+ } `json:"meta"`
}
type gpustackModelsResponse struct {
- Data []ModelListItem `json:"data"`
+ Data []gpustackModelItem `json:"data"`
+}
+
+const (
+ // gpustackImageCategory marks image-generation models, which RAGFlow has no
+ // model type for.
+ gpustackImageCategory = "image"
+ gpustackTTSCategory = "text_to_speech"
+)
+
+// gpustackCategoryModelTypes maps GPUStack model categories to RAGFlow model
+// types.
+var gpustackCategoryModelTypes = map[string]string{
+ "llm": modelTypeChat,
+ "embedding": modelTypeEmbedding,
+ "reranker": modelTypeRerank,
+ "speech_to_text": modelTypeASR,
+ gpustackTTSCategory: modelTypeTTS,
}
func (g *GPUStackModel) ListModels(ctx context.Context, apiConfig *APIConfig) ([]ListModelResponse, error) {
@@ -126,6 +151,31 @@ func (g *GPUStackModel) ListModels(ctx context.Context, apiConfig *APIConfig) ([
return nil, err
}
+ items, err := g.fetchModels(ctx, url+"?with_meta=true", apiConfig)
+ if err != nil {
+ return nil, err
+ }
+
+ categories := g.fetchModelCategories(ctx, url, apiConfig, items)
+ listItems := make([]ModelListItem, 0, len(items))
+ for _, item := range items {
+ if item.ContextLength == nil {
+ item.ContextLength = item.Meta.MaxModelLen
+ }
+ listItems = append(listItems, item.ModelListItem)
+ // Only TTS models declare voices, even when GPUStack filed them under "llm".
+ if len(item.Meta.Voices) > 0 {
+ if categories == nil {
+ categories = make(map[string][]string, len(items))
+ }
+ categories[strings.TrimSpace(item.ID)] = []string{gpustackTTSCategory}
+ }
+ }
+
+ return applyGPUStackCategories(ParseListModel(ModelList{Models: listItems}), categories), nil
+}
+
+func (g *GPUStackModel) fetchModels(ctx context.Context, url string, apiConfig *APIConfig) ([]gpustackModelItem, error) {
ctx, cancel := context.WithTimeout(ctx, nonStreamCallTimeout)
defer cancel()
@@ -156,8 +206,92 @@ func (g *GPUStackModel) ListModels(ctx context.Context, apiConfig *APIConfig) ([
if err = json.Unmarshal(body, &parsed); err != nil {
return nil, fmt.Errorf("failed to parse response: %w", err)
}
+ return parsed.Data, nil
+}
+
+// fetchModelCategories returns the GPUStack categories of each listed model.
+//
+// A GPUStack model is served under a route name its operator chose, so the name
+// says nothing reliable about what the model does. /v1/models does not return
+// categories but can filter by them, so each category is asked for in turn.
+//
+// The result is best effort: it is nil when a lookup fails or the server ignores
+// the filter, and routes created by hand may have no categories at all. Callers
+// fall back to the types derived from the model name for anything not returned.
+func (g *GPUStackModel) fetchModelCategories(ctx context.Context, url string, apiConfig *APIConfig, items []gpustackModelItem) map[string][]string {
+ if len(items) == 0 {
+ return nil
+ }
+
+ queried := make([]string, 0, len(gpustackCategoryModelTypes)+1)
+ for category := range gpustackCategoryModelTypes {
+ queried = append(queried, category)
+ }
+ queried = append(queried, gpustackImageCategory)
+ sort.Strings(queried)
+
+ categories := make(map[string][]string, len(items))
+ unfiltered := 0
+ for _, category := range queried {
+ matched, err := g.fetchModels(ctx, url+"?categories="+category, apiConfig)
+ if err != nil {
+ common.Warn("gpustack: failed to list models by category", zap.String("category", category), zap.Error(err))
+ return nil
+ }
+ // One full listing is a cluster serving only that category. Two mean
+ // the server is not applying the filter.
+ if len(matched) == len(items) {
+ unfiltered++
+ if unfiltered > 1 {
+ return nil
+ }
+ }
+ for _, item := range matched {
+ id := strings.TrimSpace(item.ID)
+ categories[id] = append(categories[id], category)
+ }
+ }
+ return categories
+}
+
+// applyGPUStackCategories overrides the name-derived types of models GPUStack
+// reported categories for, and drops models RAGFlow cannot use. Vision is kept
+// from the name-derived types because GPUStack has no category for it.
+func applyGPUStackCategories(listed []ListModelResponse, categories map[string][]string) []ListModelResponse {
+ if categories == nil {
+ return listed
+ }
+
+ result := make([]ListModelResponse, 0, len(listed))
+ for _, model := range listed {
+ modelCategories := categories[model.Name]
+ if len(modelCategories) == 0 {
+ result = append(result, model)
+ continue
+ }
- return ParseListModel(ModelList{Models: parsed.Data}), nil
+ modelTypes := make([]string, 0, len(modelCategories))
+ for _, category := range modelCategories {
+ if modelType, ok := gpustackCategoryModelTypes[category]; ok {
+ modelTypes = append(modelTypes, modelType)
+ }
+ }
+ if len(modelTypes) == 0 {
+ continue
+ }
+ // GPUStack files architectures it does not recognise under "llm", so that
+ // category alone does not outweigh a more specific type from the name.
+ if len(modelTypes) == 1 && modelTypes[0] == modelTypeChat && !slices.Contains(model.ModelTypes, modelTypeChat) {
+ result = append(result, model)
+ continue
+ }
+ if slices.Contains(modelTypes, modelTypeChat) && slices.Contains(model.ModelTypes, modelTypeVision) {
+ modelTypes = append(modelTypes, modelTypeVision)
+ }
+ model.ModelTypes = normalizeModelTypes(modelTypes)
+ result = append(result, model)
+ }
+ return result
}
func (g *GPUStackModel) CheckConnection(ctx context.Context, apiConfig *APIConfig) error {
@@ -173,7 +307,9 @@ type gpustackEmbeddingData struct {
// gpustackEmbeddingResponse is the JSON body returned by GPUStack embeddings API.
type gpustackEmbeddingResponse struct {
- Data []gpustackEmbeddingData `json:"data"`
+ ID string `json:"id"`
+ Data []gpustackEmbeddingData `json:"data"`
+ Usage *TokenUsage `json:"usage"`
}
// Embed requests embedding vectors via GPUStack's v1/embeddings endpoint.
@@ -277,6 +413,7 @@ func (g *GPUStackModel) Embed(
return nil, fmt.Errorf("gpustack: missing embedding for input index %d", i)
}
}
+ recordResponseUsage(modelUsage, parsed.ID, parsed.Usage, "embedding")
return embeddings, nil
}
@@ -350,24 +487,71 @@ func (g *GPUStackModel) Balance(ctx context.Context, apiConfig *APIConfig) (map[
return nil, fmt.Errorf("%s: no such method: Balance", g.Name())
}
+// audioModel reuses the OpenAI audio client. VoxBox, GPUStack's audio backend,
+// answers failures with HTTP 200 and the error in the JSON body.
+func (g *GPUStackModel) audioModel() *OpenAIModel {
+ return &OpenAIModel{baseModel: g.baseModel, skipTTSVoiceValidation: true, detectEmbeddedAudioErrors: true}
+}
+
func (g *GPUStackModel) TranscribeAudio(ctx context.Context, modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, modelUsage *common.ModelUsage) (*ASRResponse, error) {
- audioModel := OpenAIModel{baseModel: g.baseModel}
- return audioModel.TranscribeAudio(ctx, modelName, file, apiConfig, asrConfig, modelUsage)
+ return g.audioModel().TranscribeAudio(ctx, modelName, file, apiConfig, asrConfig, modelUsage)
}
func (g *GPUStackModel) TranscribeAudioWithSender(ctx context.Context, modelName *string, file *string, apiConfig *APIConfig, asrConfig *ASRConfig, modelUsage *common.ModelUsage, sender func(*string, *string) error) error {
- audioModel := OpenAIModel{baseModel: g.baseModel}
- return audioModel.TranscribeAudioWithSender(ctx, modelName, file, apiConfig, asrConfig, modelUsage, sender)
+ return g.audioModel().TranscribeAudioWithSender(ctx, modelName, file, apiConfig, asrConfig, modelUsage, sender)
}
func (g *GPUStackModel) AudioSpeech(ctx context.Context, modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, modelUsage *common.ModelUsage) (*TTSResponse, error) {
- audioModel := OpenAIModel{baseModel: g.baseModel, skipTTSVoiceValidation: true}
- return audioModel.AudioSpeech(ctx, modelName, audioContent, apiConfig, ttsConfig, modelUsage)
+ return g.audioModel().AudioSpeech(ctx, modelName, audioContent, apiConfig, g.withDefaultVoice(ctx, modelName, apiConfig, ttsConfig), modelUsage)
}
func (g *GPUStackModel) AudioSpeechWithSender(ctx context.Context, modelName *string, audioContent *string, apiConfig *APIConfig, ttsConfig *TTSConfig, modelUsage *common.ModelUsage, sender func(*string, *string) error) error {
- audioModel := OpenAIModel{baseModel: g.baseModel, skipTTSVoiceValidation: true}
- return audioModel.AudioSpeechWithSender(ctx, modelName, audioContent, apiConfig, ttsConfig, modelUsage, sender)
+ return g.audioModel().AudioSpeechWithSender(ctx, modelName, audioContent, apiConfig, g.withDefaultVoice(ctx, modelName, apiConfig, ttsConfig), modelUsage, sender)
+}
+
+// withDefaultVoice fills in the first voice the model declares when the caller
+// chose none: VoxBox requires one, while chat read-aloud never sends it. A failed
+// lookup leaves the config unchanged so the backend reports the problem.
+func (g *GPUStackModel) withDefaultVoice(ctx context.Context, modelName *string, apiConfig *APIConfig, ttsConfig *TTSConfig) *TTSConfig {
+ if ttsConfig != nil {
+ switch voice := ttsConfig.Params["voice"].(type) {
+ case nil:
+ case string:
+ if strings.TrimSpace(voice) != "" {
+ return ttsConfig
+ }
+ default:
+ return ttsConfig
+ }
+ }
+ if modelName == nil || strings.TrimSpace(g.baseModel.URLSuffix.Models) == "" {
+ return ttsConfig
+ }
+ url, err := g.baseModel.EndpointURL(apiConfig, g.baseModel.URLSuffix.Models)
+ if err != nil {
+ return ttsConfig
+ }
+ items, err := g.fetchModels(ctx, url+"?with_meta=true", apiConfig)
+ if err != nil {
+ common.Warn("gpustack: failed to look up model voices", zap.String("model", *modelName), zap.Error(err))
+ return ttsConfig
+ }
+ for _, item := range items {
+ if item.ID != *modelName || len(item.Meta.Voices) == 0 {
+ continue
+ }
+ withVoice := TTSConfig{Params: map[string]interface{}{"voice": item.Meta.Voices[0]}}
+ if ttsConfig != nil {
+ withVoice.Format = ttsConfig.Format
+ for key, value := range ttsConfig.Params {
+ if key != "voice" {
+ withVoice.Params[key] = value
+ }
+ }
+ }
+ return &withVoice
+ }
+ return ttsConfig
}
func (g *GPUStackModel) OCRFile(ctx context.Context, modelName *string, content []byte, url *string, apiConfig *APIConfig, ocrConfig *OCRConfig, modelUsage *common.ModelUsage) (*OCRFileResponse, error) {
diff --git a/internal/entity/models/gpustack_rerank_audio_test.go b/internal/entity/models/gpustack_rerank_audio_test.go
index cd70d092b9..ec5fccfcad 100644
--- a/internal/entity/models/gpustack_rerank_audio_test.go
+++ b/internal/entity/models/gpustack_rerank_audio_test.go
@@ -228,9 +228,6 @@ func TestGPUStackAudioSpeech(t *testing.T) {
t.Run(fmt.Sprintf("%s/stream=%t/auth=%t", tc.name, stream, authenticated), func(t *testing.T) {
before, _ := json.Marshal(tc.config)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- if r.Method != http.MethodPost || r.URL.Path != "/v1/audio/speech" {
- t.Errorf("request=%s %s", r.Method, r.URL.Path)
- }
wantAuth := ""
if authenticated {
wantAuth = "Bearer test-key"
@@ -238,6 +235,14 @@ func TestGPUStackAudioSpeech(t *testing.T) {
if r.Header.Get("Authorization") != wantAuth {
t.Errorf("auth=%q", r.Header.Get("Authorization"))
}
+ // The voice lookup: this model declares none.
+ if r.Method == http.MethodGet && r.URL.Path == "/v1/models" {
+ _, _ = w.Write([]byte(`{"data":[{"id":"tts-model"}]}`))
+ return
+ }
+ if r.Method != http.MethodPost || r.URL.Path != "/v1/audio/speech" {
+ t.Errorf("request=%s %s", r.Method, r.URL.Path)
+ }
var body map[string]interface{}
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
t.Error(err)
@@ -405,3 +410,114 @@ func TestGPUStackAudioSpeechSurfacesVoiceValidation(t *testing.T) {
})
}
}
+
+// gpustackVoiceServer serves a TTS model that declares voices, and reports a
+// missing voice the way VoxBox does.
+func gpustackVoiceServer(t *testing.T, gotVoice *any) *httptest.Server {
+ t.Helper()
+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/v1/models":
+ _ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{
+ {"id": "other", "meta": map[string]any{"voices": []string{"wrong"}}},
+ {"id": "voice-out", "meta": map[string]any{"voices": []string{"Chinese Female", "English Male"}}},
+ }})
+ case "/v1/audio/speech":
+ var body map[string]any
+ _ = json.NewDecoder(r.Body).Decode(&body)
+ *gotVoice = body["voice"]
+ w.Header().Set("Content-Type", "audio/mpeg")
+ _, _ = w.Write([]byte("audio"))
+ default:
+ t.Errorf("unexpected path %s", r.URL.Path)
+ }
+ }))
+}
+
+func TestGPUStackAudioSpeechDefaultsToDeclaredVoice(t *testing.T) {
+ withSSRFBypass(t)
+ model, text := "voice-out", "hello"
+ apiKey := "test-key"
+ for name, tc := range map[string]struct {
+ config *TTSConfig
+ want string
+ }{
+ "nil config": {nil, "Chinese Female"},
+ "read aloud": {&TTSConfig{Format: "mp3"}, "Chinese Female"},
+ "blank voice": {&TTSConfig{Params: map[string]interface{}{"voice": " "}}, "Chinese Female"},
+ "explicit voice": {&TTSConfig{Params: map[string]interface{}{"voice": "English Male"}}, "English Male"},
+ } {
+ t.Run(name, func(t *testing.T) {
+ var got any
+ srv := gpustackVoiceServer(t, &got)
+ defer srv.Close()
+
+ _, err := configuredGPUStackForTest(t, srv.URL).AudioSpeech(t.Context(), &model, &text, &APIConfig{ApiKey: &apiKey}, tc.config, nil)
+ if err != nil {
+ t.Fatalf("AudioSpeech: %v", err)
+ }
+ if got != tc.want {
+ t.Errorf("voice=%v, want %s", got, tc.want)
+ }
+ })
+ }
+}
+
+func TestGPUStackAudioSpeechDoesNotMutateCallerConfig(t *testing.T) {
+ withSSRFBypass(t)
+ var got any
+ srv := gpustackVoiceServer(t, &got)
+ defer srv.Close()
+
+ model, text, apiKey := "voice-out", "hello", "test-key"
+ config := &TTSConfig{Format: "mp3", Params: map[string]interface{}{"speed": 1.5}}
+ if _, err := configuredGPUStackForTest(t, srv.URL).AudioSpeech(t.Context(), &model, &text, &APIConfig{ApiKey: &apiKey}, config, nil); err != nil {
+ t.Fatalf("AudioSpeech: %v", err)
+ }
+ if _, polluted := config.Params["voice"]; polluted {
+ t.Errorf("caller config mutated: %v", config.Params)
+ }
+}
+
+// VoxBox returns HTTPException objects instead of raising them, so failures
+// arrive as HTTP 200 with the status in the body.
+func TestGPUStackAudioSurfacesEmbeddedErrors(t *testing.T) {
+ withSSRFBypass(t)
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ if r.URL.Path == "/v1/models" {
+ _, _ = w.Write([]byte(`{"data":[]}`))
+ return
+ }
+ _, _ = w.Write([]byte(`{"status_code":500,"detail":"Failed to generate speech","headers":null}`))
+ }))
+ defer srv.Close()
+
+ path := filepath.Join(t.TempDir(), "a.wav")
+ if err := os.WriteFile(path, []byte("audio"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ driver := configuredGPUStackForTest(t, srv.URL)
+ model, text, apiKey := "m", "hello", "test-key"
+ config := &APIConfig{ApiKey: &apiKey}
+ sender := func(*string, *string) error { return nil }
+ for name, call := range map[string]func() error{
+ "asr": func() error {
+ _, err := driver.TranscribeAudio(t.Context(), &model, &path, config, nil, nil)
+ return err
+ },
+ "asr stream": func() error {
+ return driver.TranscribeAudioWithSender(t.Context(), &model, &path, config, nil, nil, sender)
+ },
+ "tts": func() error { _, err := driver.AudioSpeech(t.Context(), &model, &text, config, nil, nil); return err },
+ "tts stream": func() error {
+ return driver.AudioSpeechWithSender(t.Context(), &model, &text, config, nil, nil, sender)
+ },
+ } {
+ t.Run(name, func(t *testing.T) {
+ if err := call(); err == nil || !strings.Contains(err.Error(), "Failed to generate speech") {
+ t.Errorf("err=%v, want the embedded failure", err)
+ }
+ })
+ }
+}
diff --git a/internal/entity/models/gpustack_test.go b/internal/entity/models/gpustack_test.go
index 5c85add65d..7e7bc1fc48 100644
--- a/internal/entity/models/gpustack_test.go
+++ b/internal/entity/models/gpustack_test.go
@@ -5,6 +5,10 @@ import (
"io"
"net/http"
"net/http/httptest"
+ "ragflow/internal/common"
+ "reflect"
+ "slices"
+ "sort"
"strings"
"testing"
)
@@ -498,6 +502,226 @@ func TestGPUStackListModelsHappyPath(t *testing.T) {
}
}
+// newGPUStackCategoryServer serves /v1/models like GPUStack: every route without
+// a filter, or only the routes carrying one of the requested categories.
+func newGPUStackCategoryServer(t *testing.T, routes map[string][]string) *httptest.Server {
+ t.Helper()
+ names := make([]string, 0, len(routes))
+ for name := range routes {
+ names = append(names, name)
+ }
+ sort.Strings(names)
+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/v1/models" {
+ t.Errorf("path=%s", r.URL.Path)
+ }
+ requested := r.URL.Query()["categories"]
+ data := []map[string]interface{}{}
+ for _, name := range names {
+ if len(requested) == 0 || slices.Contains(routes[name], requested[0]) {
+ data = append(data, map[string]interface{}{"id": name})
+ }
+ }
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{"data": data})
+ }))
+}
+
+func gpustackModelTypesByName(t *testing.T, baseURL string) map[string][]string {
+ t.Helper()
+ apiKey := "test-key"
+ listed, err := newGPUStackForTest(baseURL).ListModels(t.Context(), &APIConfig{ApiKey: &apiKey})
+ if err != nil {
+ t.Fatalf("ListModels: %v", err)
+ }
+ types := make(map[string][]string, len(listed))
+ for _, model := range listed {
+ types[model.Name] = model.ModelTypes
+ }
+ return types
+}
+
+func TestGPUStackListModelsUsesCategories(t *testing.T) {
+ withSSRFBypass(t)
+ srv := newGPUStackCategoryServer(t, map[string][]string{
+ "assistant": {"llm"},
+ "kb-encoder": {"embedding"},
+ "doc-sorter": {"reranker"},
+ "voice-in": {"speech_to_text"},
+ "voice-out": {"text_to_speech"},
+ "qwen2.5-vl": {"llm"},
+ "rerank-as-embedding": {"embedding"},
+ "whisper-small": {"llm"},
+ "bge-as-rerank": {"reranker"},
+ "painter": {"image"},
+ "no-category": {},
+ "bge-m3-route": {},
+ })
+ defer srv.Close()
+
+ got := gpustackModelTypesByName(t, srv.URL)
+ want := map[string][]string{
+ "assistant": {modelTypeChat},
+ "kb-encoder": {modelTypeEmbedding},
+ "doc-sorter": {modelTypeRerank},
+ "voice-in": {modelTypeASR},
+ "voice-out": {modelTypeTTS},
+ // Vision has no GPUStack category, so it still comes from the name.
+ "qwen2.5-vl": {modelTypeChat, modelTypeVision},
+ // A specific category beats a name that suggests something else.
+ "rerank-as-embedding": {modelTypeEmbedding},
+ "bge-as-rerank": {modelTypeRerank},
+ // "llm" is GPUStack's default for unknown architectures, so the name wins.
+ "whisper-small": {modelTypeASR},
+ // Routes without categories fall back to the name.
+ "no-category": {modelTypeChat},
+ "bge-m3-route": {modelTypeEmbedding},
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("types=%v\nwant %v", got, want)
+ }
+}
+
+// CosyVoice has no config.json, so GPUStack files it under "llm" by default.
+func TestGPUStackListModelsDetectsTTSFromVoices(t *testing.T) {
+ withSSRFBypass(t)
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ data := []map[string]interface{}{{"id": "plain"}, {"id": "voice-out"}}
+ switch r.URL.Query().Get("categories") {
+ case "":
+ if r.URL.Query().Get("with_meta") == "true" {
+ data[1]["meta"] = map[string]interface{}{"voices": []string{"Chinese Female"}}
+ }
+ case "llm":
+ default:
+ data = nil
+ }
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{"data": data})
+ }))
+ defer srv.Close()
+
+ got := gpustackModelTypesByName(t, srv.URL)
+ want := map[string][]string{"plain": {modelTypeChat}, "voice-out": {modelTypeTTS}}
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("types=%v, want %v", got, want)
+ }
+}
+
+func TestGPUStackListModelsReadsContextLengthFromMeta(t *testing.T) {
+ withSSRFBypass(t)
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ data := []map[string]interface{}{{"id": "served"}, {"id": "unknown"}}
+ if r.URL.Query().Get("with_meta") == "true" {
+ data[0]["meta"] = map[string]interface{}{"max_model_len": 8192}
+ }
+ if r.URL.Query().Has("categories") {
+ data = nil
+ }
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{"data": data})
+ }))
+ defer srv.Close()
+
+ apiKey := "test-key"
+ listed, err := newGPUStackForTest(srv.URL).ListModels(t.Context(), &APIConfig{ApiKey: &apiKey})
+ if err != nil {
+ t.Fatalf("ListModels: %v", err)
+ }
+ if len(listed) != 2 {
+ t.Fatalf("listed=%v", listed)
+ }
+ if listed[0].ContextLength == nil || *listed[0].ContextLength != 8192 {
+ t.Errorf("served context length=%v, want 8192", listed[0].ContextLength)
+ }
+ if listed[1].ContextLength != nil {
+ t.Errorf("unknown context length=%v, want nil", *listed[1].ContextLength)
+ }
+}
+
+func TestGPUStackListModelsSingleCategoryCluster(t *testing.T) {
+ withSSRFBypass(t)
+ srv := newGPUStackCategoryServer(t, map[string][]string{"a": {"embedding"}, "b": {"embedding"}})
+ defer srv.Close()
+
+ got := gpustackModelTypesByName(t, srv.URL)
+ want := map[string][]string{"a": {modelTypeEmbedding}, "b": {modelTypeEmbedding}}
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("types=%v, want %v", got, want)
+ }
+}
+
+func TestGPUStackListModelsIgnoresUnappliedCategoryFilter(t *testing.T) {
+ withSSRFBypass(t)
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "data": []map[string]interface{}{{"id": "bge-m3"}, {"id": "bge-reranker-v2-m3"}, {"id": "qwen3-8b"}},
+ })
+ }))
+ defer srv.Close()
+
+ got := gpustackModelTypesByName(t, srv.URL)
+ want := map[string][]string{
+ "bge-m3": {modelTypeEmbedding},
+ "bge-reranker-v2-m3": {modelTypeRerank},
+ "qwen3-8b": {modelTypeChat},
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("types=%v, want %v", got, want)
+ }
+}
+
+func TestGPUStackListModelsSurvivesCategoryLookupFailure(t *testing.T) {
+ withSSRFBypass(t)
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Query().Has("categories") {
+ http.Error(w, "boom", http.StatusInternalServerError)
+ return
+ }
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{"data": []map[string]interface{}{{"id": "bge-m3"}}})
+ }))
+ defer srv.Close()
+
+ got := gpustackModelTypesByName(t, srv.URL)
+ if !reflect.DeepEqual(got, map[string][]string{"bge-m3": {modelTypeEmbedding}}) {
+ t.Errorf("types=%v", got)
+ }
+}
+
+func TestGPUStackListModelsReportsAuthFailure(t *testing.T) {
+ withSSRFBypass(t)
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ http.Error(w, `{"code":401,"message":"Invalid authentication credentials"}`, http.StatusUnauthorized)
+ }))
+ defer srv.Close()
+
+ apiKey := "wrong"
+ _, err := newGPUStackForTest(srv.URL).ListModels(t.Context(), &APIConfig{ApiKey: &apiKey})
+ if err == nil || !strings.Contains(err.Error(), "401") {
+ t.Errorf("err=%v, want the 401 surfaced", err)
+ }
+}
+
+func TestGPUStackEmbedRecordsUsage(t *testing.T) {
+ withSSRFBypass(t)
+ srv := newGPUStackServer(t, gpustackEmbeddingsPath, func(t *testing.T, body map[string]interface{}, w http.ResponseWriter) {
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "id": "embd-1",
+ "data": []map[string]interface{}{{"embedding": []float64{0.1}, "index": 0}},
+ "usage": map[string]interface{}{"prompt_tokens": 8, "total_tokens": 8},
+ })
+ })
+ defer srv.Close()
+
+ model := "bge-m3"
+ apiKey := "test-key"
+ usage := &common.ModelUsage{}
+ _, err := configuredGPUStackForTest(t, srv.URL).Embed(t.Context(), &model, EmbedRequest{Texts: []string{"hello"}}, &APIConfig{ApiKey: &apiKey}, nil, usage)
+ if err != nil {
+ t.Fatalf("Embed: %v", err)
+ }
+ if usage.TotalTokens != 8 || usage.Type != "embedding" || usage.RequestID != "embd-1" {
+ t.Errorf("usage=%+v", usage)
+ }
+}
+
func TestGPUStackListModelsAllowsEmptyAPIKey(t *testing.T) {
withSSRFBypass(t)
ctx := t.Context()
diff --git a/internal/entity/models/openai.go b/internal/entity/models/openai.go
index 41806c6073..2ceefc597f 100644
--- a/internal/entity/models/openai.go
+++ b/internal/entity/models/openai.go
@@ -39,6 +39,9 @@ type OpenAIModel struct {
// skipTTSVoiceValidation delegates voice selection and validation to the
// model backend when this driver is reused by a local provider.
skipTTSVoiceValidation bool
+ // detectEmbeddedAudioErrors treats a 200 response whose JSON body carries an
+ // error status as a failure, for backends that report errors that way.
+ detectEmbeddedAudioErrors bool
}
// NewOpenAIModel creates a new OpenAI model instance.
@@ -343,6 +346,11 @@ func (o *OpenAIModel) TranscribeAudio(ctx context.Context, modelName *string, fi
if err != nil {
return nil, fmt.Errorf("failed to read response body: %w", err)
}
+ if o.detectEmbeddedAudioErrors {
+ if err := embeddedAudioError(respBody); err != nil {
+ return nil, err
+ }
+ }
return decodeOpenAIASRResponse(respBody, responseFormat)
}
@@ -377,6 +385,11 @@ func (o *OpenAIModel) TranscribeAudioWithSender(ctx context.Context, modelName *
if err != nil {
return fmt.Errorf("failed to read response body: %w", err)
}
+ if o.detectEmbeddedAudioErrors {
+ if err := embeddedAudioError(respBody); err != nil {
+ return err
+ }
+ }
response, err := decodeOpenAIASRResponse(respBody, responseFormat)
if err != nil {
return err
@@ -425,6 +438,19 @@ func (o *OpenAIModel) TranscribeAudioWithSender(ctx context.Context, modelName *
return sender(&done, nil)
}
+// embeddedAudioError returns the failure reported inside a 200 response body, or
+// nil when the body is not such a report (audio bytes included).
+func embeddedAudioError(body []byte) error {
+ var report struct {
+ StatusCode int `json:"status_code"`
+ Detail json.RawMessage `json:"detail"`
+ }
+ if err := json.Unmarshal(body, &report); err != nil || report.StatusCode < http.StatusBadRequest {
+ return nil
+ }
+ return fmt.Errorf("audio backend error %d: %s", report.StatusCode, string(report.Detail))
+}
+
func decodeOpenAIASRResponse(respBody []byte, responseFormat string) (*ASRResponse, error) {
switch responseFormat {
case "text", "srt", "vtt":
@@ -470,6 +496,12 @@ func (o *OpenAIModel) AudioSpeech(ctx context.Context, modelName *string, audioC
return nil, fmt.Errorf("failed to read response body: %w", err)
}
+ if o.detectEmbeddedAudioErrors {
+ if err := embeddedAudioError(body); err != nil {
+ return nil, err
+ }
+ }
+
return &TTSResponse{Audio: body}, nil
}
@@ -503,6 +535,16 @@ func (o *OpenAIModel) AudioSpeechWithSender(ctx context.Context, modelName *stri
if streamFormat == "sse" || strings.Contains(resp.Header.Get("Content-Type"), "text/event-stream") {
return readOpenAITTSSSEStream(resp.Body, sender)
}
+ if o.detectEmbeddedAudioErrors && strings.Contains(resp.Header.Get("Content-Type"), "application/json") {
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return fmt.Errorf("failed to read response body: %w", err)
+ }
+ if err := embeddedAudioError(body); err != nil {
+ return err
+ }
+ return readOpenAITTSRawStream(bytes.NewReader(body), sender)
+ }
return readOpenAITTSRawStream(resp.Body, sender)
}
diff --git a/internal/service/model_service.go b/internal/service/model_service.go
index f397219458..5e36af46cb 100644
--- a/internal/service/model_service.go
+++ b/internal/service/model_service.go
@@ -1496,6 +1496,7 @@ func verifyProviderModel(ctx context.Context, driver modelModule.ModelDriver, pr
zap.L().Warn("failed to list remote models from provider",
zap.Error(listErr),
)
+ return modelVerifyResult, fmt.Errorf("failed to list models from provider: %w", listErr)
}
if len(modelsToVerify) == 0 {
diff --git a/web/src/hooks/use-llm-request.tsx b/web/src/hooks/use-llm-request.tsx
index d8302d6091..d2c98e105a 100644
--- a/web/src/hooks/use-llm-request.tsx
+++ b/web/src/hooks/use-llm-request.tsx
@@ -242,6 +242,7 @@ export const useFetchInstanceModels = (
data,
isFetching: loading,
isSuccess,
+ isFetchedAfterMount,
} = useQuery<IInstanceModel[]>({
queryKey: LlmKeys.instanceModels(providerName, instanceName),
initialData: [],
@@ -259,7 +260,7 @@ export const useFetchInstanceModels = (
},
});
- return { data, loading, isSuccess };
+ return { data, loading, isSuccess, isFetchedAfterMount };
};
export type LlmItem = { name: string; logo: string } & IMyLlmValue;
@@ -387,6 +388,7 @@ export const useListProviderModels = () => {
use_saved_credentials,
api_key_config,
instance_name,
+ skipGlobalErrorNotification,
} = params;
// GET /api/v1/providers/<provider_name>/models
// The API accepts api_key and base_url as optional query parameters.
@@ -415,7 +417,13 @@ export const useListProviderModels = () => {
queryParams.instance_name = instance_name;
}
const { data } = await llmService.listProviderModels(
- { provider_name, params: queryParams },
+ {
+ provider_name,
+ params: queryParams,
+ ...(skipGlobalErrorNotification
+ ? { skipGlobalErrorNotification }
+ : {}),
+ },
true,
);
return data;
diff --git a/web/src/interfaces/request/llm.ts b/web/src/interfaces/request/llm.ts
index f34f3afdc2..96ea851317 100644
--- a/web/src/interfaces/request/llm.ts
+++ b/web/src/interfaces/request/llm.ts
@@ -156,4 +156,9 @@ export interface IListProviderModelsRequestBody {
* server applies the current form's non-secret configuration and endpoint.
*/
instance_name?: string;
+ /**
+ * Returns failures to the caller instead of raising the global notification,
+ * so they can be explained next to the form.
+ */
+ skipGlobalErrorNotification?: boolean;
}
diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts
index 5cfe20ad92..dfb858fb70 100644
--- a/web/src/locales/en.ts
+++ b/web/src/locales/en.ts
@@ -1365,6 +1365,55 @@ This auto-tagging feature enhances retrieval by adding another layer of domain-s
deleteSelectedConfirm: 'Delete the selected {{count}} session(s)?',
},
setting: {
+ gpustack: {
+ serverUrl: 'GPUStack server URL',
+ serverUrlTip: 'Must be reachable from the RAGFlow container.',
+ loopbackWarning:
+ 'localhost points to the RAGFlow container itself. Use the IP address or host.docker.internal instead.',
+ apiKey: 'API key',
+ apiKeyTip: 'Create one in GPUStack under Access Control > API Keys.',
+ apiKeyKeep: 'Leave blank to keep the saved key',
+ connect: 'Connect',
+ connecting: 'Connecting...',
+ connected: 'Connected. {{count}} running models found.',
+ connectedShort: '{{count}} running',
+ errorAuth:
+ 'GPUStack rejected this API key. Check it under Access Control > API Keys.',
+ errorUnreachable:
+ 'Cannot reach this server from RAGFlow. Check the URL and that the port is open to the RAGFlow container.',
+ errorOther: 'Could not list models: {{message}}',
+ emptyTitle: 'No running models found',
+ emptyBody:
+ 'GPUStack only exposes models that are running and have a route. Deploy a model with "Enable model route" turned on, then refresh.',
+ emptyLink: 'Read the integration guide',
+ addModels: 'Add {{count}} models',
+ addSelected: 'Add {{count}} selected',
+ adding: 'Adding...',
+ refresh: 'Refresh',
+ editInstance: 'Edit instance',
+ saveInstance: 'Save instance',
+ cancel: 'Cancel',
+ availableTitle: 'Available in GPUStack',
+ addedTitle: 'Added models',
+ modelsLoadFailed:
+ "Could not load this instance's models. Reload the page and try again.",
+ notRunning: 'Not running',
+ notRunningTip:
+ 'GPUStack is not serving this model right now, so calls to it will fail.',
+ tokens: '{{count}} tokens',
+ changeType: 'Model type for {{name}}',
+ select: 'Select {{name}}',
+ remove: 'Remove {{name}}',
+ noAdded: 'No models added yet.',
+ kind_chat: 'Chat',
+ kind_vision: 'Chat + Vision',
+ kind_embedding: 'Embedding',
+ kind_rerank: 'Rerank',
+ kind_asr: 'Speech to text',
+ kind_tts: 'Text to speech',
+ typeHint:
+ 'Types come from GPUStack. GPUStack cannot tell whether a model accepts images, so choose Chat + Vision for those.',
+ },
Verify: 'Verify',
keyValid: 'Your API Key is valid.',
keyInvalid: 'Your API Key is invalid.',
diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.ts
index 6ecc381fc1..7cb498c852 100644
--- a/web/src/locales/zh.ts
+++ b/web/src/locales/zh.ts
@@ -1244,6 +1244,53 @@ NER:使用 spaCy NER 和基于规则的关键词提取来抽取 Entities 和 R
deleteSelectedConfirm: '删除选中的 {{count}} 个会话?',
},
setting: {
+ gpustack: {
+ serverUrl: 'GPUStack 服务地址',
+ serverUrlTip: '需要能从 RAGFlow 容器访问到。',
+ loopbackWarning:
+ 'localhost 指向的是 RAGFlow 容器自身,请改用 IP 地址或 host.docker.internal。',
+ apiKey: 'API 密钥',
+ apiKeyTip: '在 GPUStack 的“访问控制 > API 密钥”中创建。',
+ apiKeyKeep: '留空则继续使用已保存的密钥',
+ connect: '连接',
+ connecting: '连接中...',
+ connected: '已连接,发现 {{count}} 个运行中的模型。',
+ connectedShort: '{{count}} 个运行中',
+ errorAuth:
+ 'GPUStack 拒绝了这个 API 密钥,请到“访问控制 > API 密钥”中检查。',
+ errorUnreachable:
+ 'RAGFlow 无法访问这个地址,请检查地址是否正确,以及端口是否对 RAGFlow 容器开放。',
+ errorOther: '无法获取模型列表:{{message}}',
+ emptyTitle: '没有发现运行中的模型',
+ emptyBody:
+ 'GPUStack 只对外提供正在运行且配置了路由的模型。请在部署模型时开启“启用模型路由”,然后刷新。',
+ emptyLink: '查看集成指南',
+ addModels: '添加 {{count}} 个模型',
+ addSelected: '添加选中的 {{count}} 个',
+ adding: '添加中...',
+ refresh: '刷新',
+ editInstance: '修改实例',
+ saveInstance: '保存实例',
+ cancel: '取消',
+ availableTitle: 'GPUStack 中可添加',
+ addedTitle: '已添加的模型',
+ modelsLoadFailed: '无法加载此实例的模型,请刷新页面后重试。',
+ notRunning: '未运行',
+ notRunningTip: 'GPUStack 目前没有提供这个模型,调用会失败。',
+ tokens: '{{count}} tokens',
+ changeType: '{{name}} 的模型类型',
+ select: '选择 {{name}}',
+ remove: '移除 {{name}}',
+ noAdded: '还没有添加模型。',
+ kind_chat: '对话',
+ kind_vision: '对话 + 视觉',
+ kind_embedding: '向量',
+ kind_rerank: '重排序',
+ kind_asr: '语音转文字',
+ kind_tts: '文字转语音',
+ typeHint:
+ '类型来自 GPUStack。GPUStack 无法判断模型是否支持图片,这类模型请选择“对话 + 视觉”。',
+ },
Verify: '验证',
keyValid: '你的 API 密钥有效。',
keyInvalid: '你的 API 密钥无效。',
diff --git a/web/src/pages/user-setting/setting-model/instance-card/gpustack-instance-card.test.tsx b/web/src/pages/user-setting/setting-model/instance-card/gpustack-instance-card.test.tsx
new file mode 100644
index 0000000000..51f18ada26
--- /dev/null
+++ b/web/src/pages/user-setting/setting-model/instance-card/gpustack-instance-card.test.tsx
@@ -0,0 +1,379 @@
+import { act, fireEvent, render, screen } from '@testing-library/react';
+import '@testing-library/jest-dom';
+
+import { GPUStackInstanceCard } from './gpustack-instance-card';
+
+const mockDiscover = jest.fn();
+const mockAddInstance = jest.fn();
+const mockAddModel = jest.fn();
+const mockPatchModel = jest.fn();
+const mockUpdateInstance = jest.fn();
+let mockInstanceModels: any[] = [];
+const loadedModelsState = {
+ loading: false,
+ isSuccess: true,
+ isFetchedAfterMount: true,
+};
+let mockInstanceModelsState = loadedModelsState;
+let mockDiscovering = false;
+
+jest.mock('react-i18next', () => ({
+ useTranslation: () => ({
+ t: (key: string, options?: Record<string, unknown>) =>
+ options?.count !== undefined ? `${key}:${options.count}` : key,
+ }),
+}));
+jest.mock('@/hooks/use-llm-request', () => ({
+ useListProviderModels: () => ({
+ loading: mockDiscovering,
+ listProviderModels: mockDiscover,
+ }),
+ useAddProviderInstance: () => ({ addProviderInstance: mockAddInstance }),
+ useUpdateProviderInstance: () => ({
+ updateProviderInstance: mockUpdateInstance,
+ }),
+ useAddInstanceModel: () => ({ addInstanceModel: mockAddModel }),
+ useDeleteInstanceModels: () => ({ deleteInstanceModels: jest.fn() }),
+ usePatchInstanceModel: () => ({ patchInstanceModel: mockPatchModel }),
+ useFetchInstanceModels: () => ({
+ data: mockInstanceModels,
+ ...mockInstanceModelsState,
+ }),
+}));
+jest.mock('../hooks', () => ({
+ useFetchInstanceNameSet: () => ({
+ instanceNameSet: new Set<string>(['taken']),
+ }),
+}));
+jest.mock('./hooks', () => ({
+ useDeleteInstance: () => jest.fn(),
+ useLazyInstanceDetails: () => ({
+ instanceDetails: { base_url: 'http://gs.local' },
+ }),
+}));
+jest.mock('@/components/confirm-delete-dialog', () => ({
+ ConfirmDeleteDialog: ({ children }: any) => children,
+}));
+jest.mock('@/components/ui/select', () => ({
+ Select: ({ onValueChange, children }: any) => (
+ <div>
+ <button onClick={() => onValueChange('vision')}>make-vision</button>
+ {children}
+ </div>
+ ),
+ SelectContent: () => null,
+ SelectItem: () => null,
+ SelectTrigger: () => null,
+ SelectValue: () => null,
+}));
+
+const served = [
+ { name: 'writer', model_types: ['chat'], max_tokens: 8192 },
+ { name: 'encoder', model_types: ['embedding'], max_tokens: 512 },
+];
+const instance = { id: 'id-1', instance_name: 'saved' } as any;
+
+function fillForm(name = 'my-gpustack') {
+ fireEvent.change(screen.getByTestId('gpustack-instance-name'), {
+ target: { value: name },
+ });
+ fireEvent.change(screen.getByTestId('gpustack-base-url'), {
+ target: { value: 'http://10.0.0.5:8090' },
+ });
+ fireEvent.change(screen.getByTestId('gpustack-api-key'), {
+ target: { value: 'secret' },
+ });
+}
+
+async function connect() {
+ fillForm();
+ await act(async () => {
+ fireEvent.click(screen.getByTestId('gpustack-connect'));
+ });
+}
+
+beforeEach(() => {
+ jest.clearAllMocks();
+ mockInstanceModels = [];
+ mockInstanceModelsState = loadedModelsState;
+ mockDiscovering = false;
+});
+
+describe('GPUStackInstanceCard draft', () => {
+ const renderDraft = (onSaved = jest.fn()) =>
+ render(
+ <GPUStackInstanceCard
+ providerName="GPUStack"
+ instance={{ id: '', instance_name: '' } as any}
+ isDraft
+ onSaved={onSaved}
+ />,
+ );
+
+ it('needs a name, URL and key before connecting', () => {
+ renderDraft();
+ expect(screen.getByTestId('gpustack-connect')).toBeDisabled();
+ fillForm(' ');
+ expect(screen.getByTestId('gpustack-connect')).toBeDisabled();
+ fillForm();
+ expect(screen.getByTestId('gpustack-connect')).toBeEnabled();
+ });
+
+ it('rejects a name that is already in use', () => {
+ renderDraft();
+ fillForm('taken');
+ expect(screen.getByText('instanceNameExists')).toBeInTheDocument();
+ expect(screen.getByTestId('gpustack-connect')).toBeDisabled();
+ });
+
+ it('cannot add while discovering again', async () => {
+ mockDiscover.mockResolvedValue({ code: 0, data: served });
+ const draft = () => (
+ <GPUStackInstanceCard
+ providerName="GPUStack"
+ instance={{ id: '', instance_name: '' } as any}
+ isDraft
+ />
+ );
+ const { rerender } = render(draft());
+ await connect();
+ expect(screen.getByTestId('gpustack-add-models')).toBeEnabled();
+ mockDiscovering = true;
+ rerender(draft());
+ expect(screen.getByTestId('gpustack-add-models')).toBeDisabled();
+ });
+
+ it('drops a pending result when the credentials change', async () => {
+ let resolve: (value: unknown) => void = () => {};
+ mockDiscover.mockReturnValue(
+ new Promise((r) => {
+ resolve = r;
+ }),
+ );
+ renderDraft();
+ await connect();
+ fireEvent.change(screen.getByTestId('gpustack-api-key'), {
+ target: { value: 'other' },
+ });
+ await act(async () => {
+ resolve({ code: 0, data: served });
+ });
+ expect(screen.queryByTestId('gpustack-add-models')).toBeNull();
+ });
+
+ it('lists models with the entered credential, keeping failures local', async () => {
+ mockDiscover.mockResolvedValue({ code: 0, data: served });
+ renderDraft();
+ await connect();
+ expect(mockDiscover).toHaveBeenCalledWith({
+ provider_name: 'GPUStack',
+ skipGlobalErrorNotification: true,
+ api_key: 'secret',
+ base_url: 'http://10.0.0.5:8090',
+ });
+ });
+
+ it('adds every discovered model in one step by default', async () => {
+ mockDiscover.mockResolvedValue({ code: 0, data: served });
+ mockAddInstance.mockResolvedValue({ code: 0 });
+ const onSaved = jest.fn();
+ renderDraft(onSaved);
+ await connect();
+
+ expect(screen.getByTestId('gpustack-group-chat')).toBeInTheDocument();
+ expect(screen.getByTestId('gpustack-group-embedding')).toBeInTheDocument();
+ await act(async () => {
+ fireEvent.click(screen.getByTestId('gpustack-add-models'));
+ });
+
+ expect(mockAddInstance).toHaveBeenCalledWith({
+ llm_factory: 'GPUStack',
+ instance_name: 'my-gpustack',
+ api_key: 'secret',
+ base_url: 'http://10.0.0.5:8090',
+ model_info: [
+ { model_name: 'writer', model_type: ['chat'], max_tokens: 8192 },
+ { model_name: 'encoder', model_type: ['embedding'], max_tokens: 512 },
+ ],
+ });
+ expect(onSaved).toHaveBeenCalled();
+ });
+
+ it('leaves out models the user unchecks', async () => {
+ mockDiscover.mockResolvedValue({ code: 0, data: served });
+ mockAddInstance.mockResolvedValue({ code: 0 });
+ renderDraft();
+ await connect();
+ fireEvent.click(screen.getAllByLabelText('gpustack.select')[0]);
+ expect(screen.getByTestId('gpustack-add-models')).toHaveTextContent(
+ 'gpustack.addModels:1',
+ );
+ });
+
+ it('explains a rejected key next to the form', async () => {
+ mockDiscover.mockResolvedValue({
+ code: 102,
+ message: 'API request failed with status 401: unauthorized',
+ });
+ renderDraft();
+ await connect();
+ expect(screen.getByTestId('gpustack-error')).toHaveTextContent(
+ 'gpustack.errorAuth',
+ );
+ expect(screen.queryByTestId('gpustack-add-models')).toBeNull();
+ });
+
+ it('explains how to get models when none are running', async () => {
+ mockDiscover.mockResolvedValue({ code: 0, data: [] });
+ renderDraft();
+ await connect();
+ expect(screen.getByTestId('gpustack-empty')).toBeInTheDocument();
+ expect(screen.queryByTestId('gpustack-add-models')).toBeNull();
+ });
+
+ it('hides the context length of audio models', async () => {
+ mockDiscover.mockResolvedValue({
+ code: 0,
+ data: [
+ { name: 'writer', model_types: ['chat'], max_tokens: 4096 },
+ { name: 'voice', model_types: ['tts'], max_tokens: 0 },
+ ],
+ });
+ renderDraft();
+ await connect();
+ expect(screen.getAllByText(/^gpustack\.tokens/)).toHaveLength(1);
+ });
+
+ it('warns about localhost', () => {
+ renderDraft();
+ fireEvent.change(screen.getByTestId('gpustack-base-url'), {
+ target: { value: 'http://localhost:8080' },
+ });
+ expect(screen.getByText('gpustack.loopbackWarning')).toBeInTheDocument();
+ });
+});
+
+describe('GPUStackInstanceCard saved', () => {
+ it('uses saved credentials and shows what changed on the server', async () => {
+ mockInstanceModels = [
+ { name: 'writer', model_type: ['chat'], max_tokens: 8192 },
+ { name: 'stopped', model_type: ['chat'], max_tokens: 8192 },
+ ];
+ mockDiscover.mockResolvedValue({ code: 0, data: served });
+ mockAddModel.mockResolvedValue({ code: 0 });
+ await act(async () => {
+ render(
+ <GPUStackInstanceCard
+ providerName="GPUStack"
+ instance={instance}
+ defaultOpen
+ />,
+ );
+ });
+
+ expect(mockDiscover).toHaveBeenCalledWith({
+ provider_name: 'GPUStack',
+ skipGlobalErrorNotification: true,
+ use_saved_credentials: true,
+ instance_name: 'saved',
+ });
+ expect(screen.getByText('gpustack.notRunning')).toBeInTheDocument();
+ await act(async () => {
+ fireEvent.click(screen.getByTestId('gpustack-add-selected'));
+ });
+ expect(mockAddModel).toHaveBeenCalledTimes(1);
+ expect(mockAddModel).toHaveBeenCalledWith(
+ expect.objectContaining({
+ instance_name: 'saved',
+ model_name: 'encoder',
+ model_type: ['embedding'],
+ max_tokens: 512,
+ }),
+ );
+ });
+
+ const savedModels = [
+ {
+ name: 'writer',
+ model_type: ['chat'],
+ max_tokens: 8192,
+ extra: { is_tools: true },
+ },
+ ];
+
+ async function renderSaved() {
+ mockInstanceModels = savedModels;
+ mockDiscover.mockResolvedValue({ code: 0, data: [served[0]] });
+ await act(async () => {
+ render(
+ <GPUStackInstanceCard
+ providerName="GPUStack"
+ instance={instance}
+ defaultOpen
+ />,
+ );
+ });
+ }
+
+ it('changes a saved type through the PATCH endpoint, keeping settings', async () => {
+ await renderSaved();
+ await act(async () => {
+ fireEvent.click(screen.getByText('make-vision'));
+ });
+ expect(mockPatchModel).toHaveBeenCalledWith({
+ provider_name: 'GPUStack',
+ instance_name: 'saved',
+ model_name: 'writer',
+ max_tokens: 8192,
+ model_type: ['chat', 'vision'],
+ extra: { is_tools: true },
+ });
+ });
+
+ it('keeps the saved models and key when editing the connection', async () => {
+ mockUpdateInstance.mockResolvedValue({ code: 0 });
+ await renderSaved();
+ fireEvent.click(screen.getByTestId('gpustack-edit'));
+ await act(async () => {
+ fireEvent.click(screen.getByTestId('gpustack-save-connection'));
+ });
+ const body = mockUpdateInstance.mock.calls[0][0];
+ expect(body).toMatchObject({
+ id: 'id-1',
+ instance_name: 'saved',
+ base_url: 'http://gs.local',
+ region: 'default',
+ model_info: [
+ {
+ model_name: 'writer',
+ model_type: ['chat'],
+ max_tokens: 8192,
+ extra: { is_tools: true },
+ },
+ ],
+ });
+ expect(body).not.toHaveProperty('api_key');
+ expect(screen.queryByTestId('gpustack-save-connection')).toBeNull();
+ });
+
+ it.each([
+ ['loading', { ...loadedModelsState, loading: true }, false],
+ [
+ 'not fetched',
+ { ...loadedModelsState, isFetchedAfterMount: false },
+ false,
+ ],
+ ['failed to load', { ...loadedModelsState, isSuccess: false }, true],
+ ])(
+ 'cannot save while the models have %s',
+ async (_name, state, explained) => {
+ mockInstanceModelsState = state;
+ await renderSaved();
+ fireEvent.click(screen.getByTestId('gpustack-edit'));
+ expect(screen.getByTestId('gpustack-save-connection')).toBeDisabled();
+ expect(!!screen.queryByTestId('gpustack-models-load-failed')).toBe(
+ explained,
+ );
+ },
+ );
+});
diff --git a/web/src/pages/user-setting/setting-model/instance-card/gpustack-instance-card.tsx b/web/src/pages/user-setting/setting-model/instance-card/gpustack-instance-card.tsx
new file mode 100644
index 0000000000..fb47a12de0
--- /dev/null
+++ b/web/src/pages/user-setting/setting-model/instance-card/gpustack-instance-card.tsx
@@ -0,0 +1,852 @@
+/*
+ * Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ConfirmDeleteDialog } from '@/components/confirm-delete-dialog';
+import { Button } from '@/components/ui/button';
+import { Checkbox } from '@/components/ui/checkbox';
+import { Input } from '@/components/ui/input';
+import {
+ Select,
+ SelectContent,
+ SelectItem,
+ SelectTrigger,
+ SelectValue,
+} from '@/components/ui/select';
+import {
+ useAddInstanceModel,
+ useAddProviderInstance,
+ useDeleteInstanceModels,
+ useFetchInstanceModels,
+ useListProviderModels,
+ usePatchInstanceModel,
+ useUpdateProviderInstance,
+} from '@/hooks/use-llm-request';
+import {
+ ChevronDown,
+ ChevronRight,
+ CircleCheck,
+ CircleAlert,
+ ExternalLink,
+ Loader2,
+ Minus,
+ RefreshCw,
+ ServerOff,
+ Trash2,
+} from 'lucide-react';
+import {
+ ChangeEvent,
+ forwardRef,
+ useCallback,
+ useEffect,
+ useImperativeHandle,
+ useMemo,
+ useRef,
+ useState,
+} from 'react';
+import { useTranslation } from 'react-i18next';
+import { useFetchInstanceNameSet } from '../hooks';
+import {
+ GPUSTACK_MODEL_KINDS,
+ GPUStackModel,
+ GPUStackModelKind,
+ classifyConnectionError,
+ diffModels,
+ groupModelsByKind,
+ isLoopbackUrl,
+ modelTypesFromKind,
+ toGPUStackModel,
+} from './gpustack-instance-utils';
+import { useDeleteInstance, useLazyInstanceDetails } from './hooks';
+import {
+ ProviderInstanceCardProps,
+ ProviderInstanceCardRef,
+} from './interface';
+
+const GUIDE_URL =
+ 'https://docs.gpustack.ai/latest/integrations/integrate-with-ragflow/';
+const URL_PLACEHOLDER = 'http://host.docker.internal:80';
+
+type TranslateFn = (key: string, options?: Record<string, unknown>) => string;
+
+function ConnectionError({ message, t }: { message: string; t: TranslateFn }) {
+ const kind = classifyConnectionError(message);
+ const text =
+ kind === 'auth'
+ ? t('gpustack.errorAuth')
+ : kind === 'unreachable'
+ ? t('gpustack.errorUnreachable')
+ : t('gpustack.errorOther', { message });
+ return (
+ <div
+ role="alert"
+ className="flex items-start gap-2 text-sm text-state-error"
+ data-testid="gpustack-error"
+ >
+ <CircleAlert className="size-4 mt-0.5 shrink-0" />
+ <span>{text}</span>
+ </div>
+ );
+}
+
+function EmptyModels({ t }: { t: TranslateFn }) {
+ return (
+ <div
+ className="flex flex-col items-center gap-2 rounded-lg border border-border-button p-6 text-center"
+ data-testid="gpustack-empty"
+ >
+ <ServerOff className="size-5 text-text-secondary" />
+ <div className="text-sm font-medium text-text-primary">
+ {t('gpustack.emptyTitle')}
+ </div>
+ <div className="text-sm text-text-secondary max-w-prose">
+ {t('gpustack.emptyBody')}
+ </div>
+ <a
+ href={GUIDE_URL}
+ target="_blank"
+ rel="noreferrer"
+ className="inline-flex items-center gap-1 text-sm text-accent-primary"
+ >
+ {t('gpustack.emptyLink')}
+ <ExternalLink className="size-3.5" />
+ </a>
+ </div>
+ );
+}
+
+interface ModelGroupsProps {
+ models: (GPUStackModel & { running?: boolean })[];
+ selected?: Set<string>;
+ onToggle?: (name: string) => void;
+ onKindChange: (name: string, kind: GPUStackModelKind) => void;
+ onRemove?: (name: string) => void;
+ t: TranslateFn;
+}
+
+interface ModelRowProps extends Omit<ModelGroupsProps, 'models'> {
+ model: GPUStackModel & { running?: boolean };
+}
+
+function ModelRow({
+ model,
+ selected,
+ onToggle,
+ onKindChange,
+ onRemove,
+ t,
+}: ModelRowProps) {
+ const handleToggle = useCallback(
+ () => onToggle?.(model.name),
+ [model.name, onToggle],
+ );
+ const handleKindChange = useCallback(
+ (kind: string) => onKindChange(model.name, kind as GPUStackModelKind),
+ [model.name, onKindChange],
+ );
+ const handleRemove = useCallback(
+ () => onRemove?.(model.name),
+ [model.name, onRemove],
+ );
+
+ return (
+ <li className="flex items-center gap-3 px-3 py-2">
+ {selected && onToggle && (
+ <Checkbox
+ checked={selected.has(model.name)}
+ onCheckedChange={handleToggle}
+ aria-label={t('gpustack.select', { name: model.name })}
+ />
+ )}
+ <span className="min-w-0 flex-1 truncate text-sm text-text-primary">
+ {model.name}
+ </span>
+ {model.running === false && (
+ <span
+ className="shrink-0 rounded px-1.5 py-0.5 text-xs text-state-warning border border-state-warning"
+ title={t('gpustack.notRunningTip')}
+ >
+ {t('gpustack.notRunning')}
+ </span>
+ )}
+ {model.kind !== 'asr' && model.kind !== 'tts' && (
+ <span className="shrink-0 text-xs text-text-secondary">
+ {t('gpustack.tokens', {
+ count: model.maxTokens.toLocaleString(),
+ })}
+ </span>
+ )}
+ <Select value={model.kind} onValueChange={handleKindChange}>
+ <SelectTrigger
+ className="w-40 shrink-0"
+ aria-label={t('gpustack.changeType', { name: model.name })}
+ >
+ <SelectValue />
+ </SelectTrigger>
+ <SelectContent>
+ {GPUSTACK_MODEL_KINDS.map((kind) => (
+ <SelectItem key={kind} value={kind}>
+ {t(`gpustack.kind_${kind}`)}
+ </SelectItem>
+ ))}
+ </SelectContent>
+ </Select>
+ {onRemove && (
+ <ConfirmDeleteDialog onOk={handleRemove}>
+ <Button
+ variant="ghost"
+ size="icon-sm"
+ className="shrink-0"
+ aria-label={t('gpustack.remove', { name: model.name })}
+ >
+ <Minus className="size-4" />
+ </Button>
+ </ConfirmDeleteDialog>
+ )}
+ </li>
+ );
+}
+
+function ModelGroups({
+ models,
+ selected,
+ onToggle,
+ onKindChange,
+ onRemove,
+ t,
+}: ModelGroupsProps) {
+ return (
+ <div className="flex flex-col gap-4">
+ {groupModelsByKind(models).map((group) => (
+ <section
+ key={group.kind}
+ data-testid={`gpustack-group-${group.kind}`}
+ className="flex flex-col gap-1.5"
+ >
+ <h4 className="text-xs font-medium text-text-secondary">
+ {t(`gpustack.kind_${group.kind}`)} ({group.models.length})
+ </h4>
+ <ul className="rounded-lg border border-border-button divide-y divide-border-button">
+ {group.models.map((model) => (
+ <ModelRow
+ key={model.name}
+ model={model}
+ selected={selected}
+ onToggle={onToggle}
+ onKindChange={onKindChange}
+ onRemove={onRemove}
+ t={t}
+ />
+ ))}
+ </ul>
+ </section>
+ ))}
+ </div>
+ );
+}
+
+function toModelInfo(model: GPUStackModel) {
+ return {
+ model_name: model.name,
+ model_type: modelTypesFromKind(model.kind),
+ max_tokens: model.maxTokens,
+ };
+}
+
+/**
+ * Instance card for GPUStack. A GPUStack server already knows which models it
+ * runs and what they are, so connecting is the whole setup: enter the server
+ * and key, review what was found, add it.
+ */
+export const GPUStackInstanceCard = forwardRef<
+ ProviderInstanceCardRef,
+ ProviderInstanceCardProps
+>(function GPUStackInstanceCard(
+ { providerName, instance, isDraft = false, onDelete, onSaved, defaultOpen },
+ ref,
+) {
+ const { t } = useTranslation('translation', { keyPrefix: 'setting' });
+ const [open, setOpen] = useState(isDraft || !!defaultOpen);
+ const [editing, setEditing] = useState(false);
+ const [baseUrl, setBaseUrl] = useState('');
+ const [apiKey, setApiKey] = useState('');
+ const [instanceName, setInstanceName] = useState('');
+ const [discovered, setDiscovered] = useState<GPUStackModel[] | null>(null);
+ const [error, setError] = useState('');
+ const [deselected, setDeselected] = useState<Set<string>>(new Set());
+ const [kindOverrides, setKindOverrides] = useState<
+ Record<string, GPUStackModelKind>
+ >({});
+ const [saving, setSaving] = useState(false);
+
+ const { instanceNameSet } = useFetchInstanceNameSet(providerName);
+ const { instanceDetails } = useLazyInstanceDetails(
+ providerName,
+ instance.id,
+ isDraft,
+ open,
+ );
+ const {
+ data: instanceModels,
+ loading: instanceModelsLoading,
+ isSuccess: instanceModelsLoaded,
+ isFetchedAfterMount: instanceModelsFetched,
+ } = useFetchInstanceModels(
+ providerName,
+ isDraft ? '' : instance.instance_name,
+ );
+ const { loading: discovering, listProviderModels } = useListProviderModels();
+ const { addProviderInstance } = useAddProviderInstance();
+ const { updateProviderInstance } = useUpdateProviderInstance();
+ const { addInstanceModel } = useAddInstanceModel();
+ const { deleteInstanceModels } = useDeleteInstanceModels();
+ const { patchInstanceModel } = usePatchInstanceModel();
+ const handleDelete = useDeleteInstance(
+ providerName,
+ instance.instance_name,
+ isDraft,
+ onDelete,
+ );
+
+ const savedBaseUrl = instanceDetails?.base_url ?? '';
+ useEffect(() => {
+ if (!isDraft && savedBaseUrl) setBaseUrl(savedBaseUrl);
+ }, [isDraft, savedBaseUrl]);
+
+ const trimmedName = instanceName.trim();
+ const nameTaken = instanceNameSet.has(trimmedName);
+ const nameValid = !!trimmedName && !nameTaken;
+
+ // Bumped when the inputs change, so a slower response for the old ones is
+ // dropped instead of being saved with the new credentials.
+ const discoverRequest = useRef(0);
+ const discover = useCallback(async () => {
+ const request = ++discoverRequest.current;
+ setError('');
+ try {
+ const ret = await listProviderModels({
+ provider_name: providerName,
+ skipGlobalErrorNotification: true,
+ ...(isDraft
+ ? { api_key: apiKey, base_url: baseUrl.trim() }
+ : {
+ use_saved_credentials: true,
+ instance_name: instance.instance_name,
+ }),
+ });
+ if (request !== discoverRequest.current) return;
+ if (ret?.code !== 0) {
+ setDiscovered(null);
+ setError(ret?.message || String(ret?.code));
+ return;
+ }
+ setDiscovered((ret.data ?? []).map(toGPUStackModel));
+ } catch (e) {
+ if (request !== discoverRequest.current) return;
+ setDiscovered(null);
+ setError(e instanceof Error ? e.message : String(e));
+ }
+ }, [
+ apiKey,
+ baseUrl,
+ instance.instance_name,
+ isDraft,
+ listProviderModels,
+ providerName,
+ ]);
+
+ // A saved connection always opens showing what the server serves right now.
+ const discoveredOnOpen = useRef(false);
+ useEffect(() => {
+ if (isDraft || !open || discoveredOnOpen.current) return;
+ discoveredOnOpen.current = true;
+ discover();
+ }, [discover, isDraft, open]);
+
+ const withKindOverride = useCallback(
+ (model: GPUStackModel) => ({
+ ...model,
+ kind: kindOverrides[model.name] ?? model.kind,
+ }),
+ [kindOverrides],
+ );
+ const { notAdded, added } = useMemo(
+ () =>
+ diffModels(
+ (discovered ?? []).map(withKindOverride),
+ (instanceModels ?? []).map((model) =>
+ toGPUStackModel({ ...model, model_types: model.model_type }),
+ ),
+ ),
+ [discovered, instanceModels, withKindOverride],
+ );
+ const selectedModels = notAdded.filter((m) => !deselected.has(m.name));
+ const selectedNames = useMemo(
+ () => new Set(selectedModels.map((m) => m.name)),
+ [selectedModels],
+ );
+
+ const toggle = useCallback((name: string) => {
+ setDeselected((prev) => {
+ const next = new Set(prev);
+ if (!next.delete(name)) next.add(name);
+ return next;
+ });
+ }, []);
+ const setKind = useCallback((name: string, kind: GPUStackModelKind) => {
+ setKindOverrides((prev) => ({ ...prev, [name]: kind }));
+ }, []);
+
+ const getSavePayload = useCallback(() => {
+ if (!isDraft || !nameValid || discovering || !selectedModels.length) {
+ return null;
+ }
+ return {
+ payload: {
+ llm_factory: providerName,
+ instance_name: trimmedName,
+ api_key: apiKey,
+ base_url: baseUrl.trim(),
+ model_info: selectedModels.map(toModelInfo),
+ },
+ instanceName: trimmedName,
+ isDraft: true,
+ apiKind: 'add' as const,
+ };
+ }, [
+ apiKey,
+ baseUrl,
+ discovering,
+ isDraft,
+ nameValid,
+ providerName,
+ selectedModels,
+ trimmedName,
+ ]);
+
+ useImperativeHandle(
+ ref,
+ () => ({
+ validate: async () => !!getSavePayload(),
+ getSavePayload,
+ markSaved: () => {},
+ }),
+ [getSavePayload],
+ );
+
+ const addConnection = useCallback(async () => {
+ const save = getSavePayload();
+ if (!save) return;
+ setSaving(true);
+ try {
+ const ret = await addProviderInstance(save.payload as any);
+ if (ret?.code === 0) onSaved?.(save);
+ } finally {
+ setSaving(false);
+ }
+ }, [addProviderInstance, getSavePayload, onSaved]);
+
+ const addToSaved = useCallback(async () => {
+ setSaving(true);
+ try {
+ await Promise.all(
+ selectedModels.map((model) =>
+ addInstanceModel({
+ provider_name: providerName,
+ instance_name: instance.instance_name,
+ model_name: model.name,
+ model_type: modelTypesFromKind(model.kind),
+ max_tokens: model.maxTokens,
+ } as any),
+ ),
+ );
+ } finally {
+ setSaving(false);
+ }
+ }, [addInstanceModel, instance.instance_name, providerName, selectedModels]);
+
+ const saveConnection = useCallback(async () => {
+ setSaving(true);
+ try {
+ const ret = await updateProviderInstance({
+ provider_name: providerName,
+ id: instance.id,
+ instance_name: instance.instance_name,
+ base_url: baseUrl.trim(),
+ ...(apiKey ? { api_key: apiKey } : {}),
+ region: instanceDetails?.region || 'default',
+ // Required by the endpoint: the instance's models stay as they are.
+ model_info: (instanceModels ?? []).map((model) => ({
+ model_name: model.name,
+ model_type: model.model_type,
+ max_tokens: model.max_tokens,
+ ...(model.extra ? { extra: model.extra } : {}),
+ })),
+ verify: false,
+ });
+ if (ret?.code === 0) {
+ setEditing(false);
+ setApiKey('');
+ await discover();
+ }
+ } finally {
+ setSaving(false);
+ }
+ }, [
+ apiKey,
+ baseUrl,
+ discover,
+ instance.id,
+ instance.instance_name,
+ instanceDetails?.region,
+ instanceModels,
+ providerName,
+ updateProviderInstance,
+ ]);
+
+ const removeModel = useCallback(
+ (name: string) =>
+ deleteInstanceModels({
+ provider_name: providerName,
+ instance_name: instance.instance_name,
+ model_name: [name],
+ }),
+ [deleteInstanceModels, instance.instance_name, providerName],
+ );
+ const changeSavedKind = useCallback(
+ (name: string, kind: GPUStackModelKind) => {
+ const saved = instanceModels?.find((model) => model.name === name);
+ return patchInstanceModel({
+ provider_name: providerName,
+ instance_name: instance.instance_name,
+ model_name: name,
+ max_tokens: saved?.max_tokens ?? 0,
+ model_type: modelTypesFromKind(kind),
+ extra: saved?.extra ?? {},
+ });
+ },
+ [instance.instance_name, instanceModels, patchInstanceModel, providerName],
+ );
+
+ const resetDiscovery = useCallback(() => {
+ discoverRequest.current += 1;
+ setDiscovered(null);
+ setError('');
+ setDeselected(new Set());
+ setKindOverrides({});
+ }, []);
+
+ const handleBaseUrlChange = useCallback(
+ (e: ChangeEvent<HTMLInputElement>) => {
+ setBaseUrl(e.target.value);
+ if (isDraft) resetDiscovery();
+ },
+ [isDraft, resetDiscovery],
+ );
+ const handleApiKeyChange = useCallback(
+ (e: ChangeEvent<HTMLInputElement>) => {
+ setApiKey(e.target.value);
+ if (isDraft) resetDiscovery();
+ },
+ [isDraft, resetDiscovery],
+ );
+ const handleNameChange = useCallback((e: ChangeEvent<HTMLInputElement>) => {
+ setInstanceName(e.target.value);
+ }, []);
+ const toggleOpen = useCallback(() => setOpen((value) => !value), []);
+ const toggleEditing = useCallback(() => setEditing((value) => !value), []);
+
+ // Saving sends the full model list and the server deletes anything missing
+ // from it, so never save the placeholder list used before the models load.
+ // That list counts as a success, and a paused fetch offline is not "loading".
+ const instanceModelsReady =
+ instanceModelsFetched && instanceModelsLoaded && !instanceModelsLoading;
+
+ const connectionForm = (
+ <div className="flex flex-col gap-3">
+ <label className="flex flex-col gap-1.5 text-sm font-medium text-text-primary">
+ {t('gpustack.serverUrl')}
+ <Input
+ value={baseUrl}
+ onChange={handleBaseUrlChange}
+ placeholder={URL_PLACEHOLDER}
+ autoComplete="off"
+ data-testid="gpustack-base-url"
+ />
+ <span
+ className={
+ isLoopbackUrl(baseUrl)
+ ? 'text-xs font-normal text-state-warning'
+ : 'text-xs font-normal text-text-secondary'
+ }
+ >
+ {isLoopbackUrl(baseUrl)
+ ? t('gpustack.loopbackWarning')
+ : t('gpustack.serverUrlTip')}
+ </span>
+ </label>
+ <label className="flex flex-col gap-1.5 text-sm font-medium text-text-primary">
+ {t('gpustack.apiKey')}
+ <Input
+ type="password"
+ value={apiKey}
+ onChange={handleApiKeyChange}
+ placeholder={isDraft ? '' : t('gpustack.apiKeyKeep')}
+ autoComplete="new-password"
+ data-testid="gpustack-api-key"
+ />
+ <span className="text-xs font-normal text-text-secondary">
+ {t('gpustack.apiKeyTip')}
+ </span>
+ </label>
+ </div>
+ );
+
+ if (isDraft) {
+ const canConnect =
+ nameValid && !!baseUrl.trim() && !!apiKey && !discovering;
+ return (
+ <div
+ className="rounded-lg bg-bg-card p-5 mb-5 flex flex-col gap-4"
+ data-testid="instance-card-draft"
+ >
+ <label className="flex flex-col gap-1.5 text-sm font-medium text-text-primary">
+ <span>
+ <span className="text-state-error mr-0.5">*</span>
+ {t('instanceName')}
+ </span>
+ <Input
+ value={instanceName}
+ onChange={handleNameChange}
+ placeholder={t('instanceNamePlaceholder')}
+ autoComplete="off"
+ data-testid="gpustack-instance-name"
+ />
+ {nameTaken && (
+ <span className="text-xs font-normal text-state-error">
+ {t('instanceNameExists')}
+ </span>
+ )}
+ </label>
+ {connectionForm}
+ <div className="flex items-center gap-2">
+ <Button
+ onClick={discover}
+ disabled={!canConnect}
+ data-testid="gpustack-connect"
+ >
+ {discovering && <Loader2 className="size-4 animate-spin" />}
+ {discovering ? t('gpustack.connecting') : t('gpustack.connect')}
+ </Button>
+ {onDelete && (
+ <Button variant="ghost" onClick={onDelete}>
+ {t('gpustack.cancel')}
+ </Button>
+ )}
+ </div>
+ {error && <ConnectionError message={error} t={t} />}
+ {discovered && discovered.length === 0 && <EmptyModels t={t} />}
+ {discovered && discovered.length > 0 && (
+ <>
+ <div
+ className="flex items-center gap-2 text-sm text-state-success"
+ data-testid="gpustack-connected"
+ >
+ <CircleCheck className="size-4" />
+ {t('gpustack.connected', { count: discovered.length })}
+ </div>
+ <ModelGroups
+ models={notAdded}
+ selected={selectedNames}
+ onToggle={toggle}
+ onKindChange={setKind}
+ t={t}
+ />
+ <p className="text-xs text-text-secondary">
+ {t('gpustack.typeHint')}
+ </p>
+ <div>
+ <Button
+ onClick={addConnection}
+ disabled={
+ saving ||
+ discovering ||
+ selectedModels.length === 0 ||
+ !nameValid
+ }
+ data-testid="gpustack-add-models"
+ >
+ {saving && <Loader2 className="size-4 animate-spin" />}
+ {saving
+ ? t('gpustack.adding')
+ : t('gpustack.addModels', { count: selectedModels.length })}
+ </Button>
+ </div>
+ </>
+ )}
+ </div>
+ );
+ }
+
+ return (
+ <div
+ className="rounded-lg bg-bg-card p-5 mb-5 flex flex-col gap-4"
+ data-testid={`instance-card-${instance.instance_name}`}
+ >
+ <div className="flex items-center gap-2">
+ <Button
+ type="button"
+ variant="static"
+ size="auto"
+ className="min-w-0 flex-1 shrink justify-start text-left"
+ onClick={toggleOpen}
+ aria-expanded={open}
+ >
+ {open ? (
+ <ChevronDown className="size-4 shrink-0" />
+ ) : (
+ <ChevronRight className="size-4 shrink-0" />
+ )}
+ <span className="truncate text-sm font-medium text-text-primary">
+ {instance.instance_name}
+ </span>
+ {open && savedBaseUrl && (
+ <span className="truncate text-xs text-text-secondary">
+ {savedBaseUrl}
+ </span>
+ )}
+ {discovered && (
+ <span className="shrink-0 text-xs text-state-success">
+ {t('gpustack.connectedShort', { count: discovered.length })}
+ </span>
+ )}
+ </Button>
+ {open && (
+ <>
+ <Button
+ variant="outline"
+ size="sm"
+ onClick={discover}
+ disabled={discovering}
+ data-testid="gpustack-refresh"
+ >
+ <RefreshCw
+ className={discovering ? 'size-4 animate-spin' : 'size-4'}
+ />
+ {t('gpustack.refresh')}
+ </Button>
+ <Button
+ variant="outline"
+ size="sm"
+ onClick={toggleEditing}
+ data-testid="gpustack-edit"
+ >
+ {t('gpustack.editInstance')}
+ </Button>
+ </>
+ )}
+ <ConfirmDeleteDialog onOk={handleDelete}>
+ <Button
+ variant="delete"
+ size="icon-sm"
+ aria-label={t('deleteInstance')}
+ >
+ <Trash2 className="size-4" />
+ </Button>
+ </ConfirmDeleteDialog>
+ </div>
+
+ {open && (
+ <>
+ {editing && (
+ <div className="flex flex-col gap-3">
+ {connectionForm}
+ {!instanceModelsLoaded && !instanceModelsLoading && (
+ <p
+ role="alert"
+ className="text-sm text-state-error"
+ data-testid="gpustack-models-load-failed"
+ >
+ {t('gpustack.modelsLoadFailed')}
+ </p>
+ )}
+ <div>
+ <Button
+ onClick={saveConnection}
+ disabled={saving || !baseUrl.trim() || !instanceModelsReady}
+ data-testid="gpustack-save-connection"
+ >
+ {t('gpustack.saveInstance')}
+ </Button>
+ </div>
+ </div>
+ )}
+ {error && <ConnectionError message={error} t={t} />}
+ {discovered && discovered.length === 0 && added.length === 0 && (
+ <EmptyModels t={t} />
+ )}
+ {notAdded.length > 0 && (
+ <div className="flex flex-col gap-3">
+ <h3 className="text-sm font-medium text-text-primary">
+ {t('gpustack.availableTitle')} ({notAdded.length})
+ </h3>
+ <ModelGroups
+ models={notAdded}
+ selected={selectedNames}
+ onToggle={toggle}
+ onKindChange={setKind}
+ t={t}
+ />
+ <div>
+ <Button
+ onClick={addToSaved}
+ disabled={saving || selectedModels.length === 0}
+ data-testid="gpustack-add-selected"
+ >
+ {saving
+ ? t('gpustack.adding')
+ : t('gpustack.addSelected', {
+ count: selectedModels.length,
+ })}
+ </Button>
+ </div>
+ </div>
+ )}
+ <div className="flex flex-col gap-3">
+ <h3 className="text-sm font-medium text-text-primary">
+ {t('gpustack.addedTitle')} ({added.length})
+ </h3>
+ {added.length === 0 ? (
+ <p className="text-sm text-text-secondary">
+ {t('gpustack.noAdded')}
+ </p>
+ ) : (
+ <ModelGroups
+ models={
+ discovered
+ ? added
+ : added.map((model) => ({ ...model, running: undefined }))
+ }
+ onKindChange={changeSavedKind}
+ onRemove={removeModel}
+ t={t}
+ />
+ )}
+ </div>
+ </>
+ )}
+ </div>
+ );
+});
diff --git a/web/src/pages/user-setting/setting-model/instance-card/gpustack-instance-utils.test.ts b/web/src/pages/user-setting/setting-model/instance-card/gpustack-instance-utils.test.ts
new file mode 100644
index 0000000000..f408035f8c
--- /dev/null
+++ b/web/src/pages/user-setting/setting-model/instance-card/gpustack-instance-utils.test.ts
@@ -0,0 +1,99 @@
+import {
+ classifyConnectionError,
+ diffModels,
+ groupModelsByKind,
+ isLoopbackUrl,
+ kindFromModelTypes,
+ modelTypesFromKind,
+ toGPUStackModel,
+} from './gpustack-instance-utils';
+
+describe('kindFromModelTypes', () => {
+ it.each([
+ [['chat'], 'chat'],
+ [['chat', 'vision'], 'vision'],
+ [['embedding'], 'embedding'],
+ [['rerank'], 'rerank'],
+ [['asr'], 'asr'],
+ [['tts'], 'tts'],
+ [[], 'chat'],
+ [undefined, 'chat'],
+ ])('maps %j to %s', (types, kind) => {
+ expect(kindFromModelTypes(types as string[] | undefined)).toBe(kind);
+ });
+
+ it('round-trips through modelTypesFromKind', () => {
+ expect(modelTypesFromKind('vision')).toEqual(['chat', 'vision']);
+ expect(kindFromModelTypes(modelTypesFromKind('vision'))).toBe('vision');
+ expect(modelTypesFromKind('tts')).toEqual(['tts']);
+ });
+});
+
+describe('toGPUStackModel', () => {
+ it('keeps the reported context length', () => {
+ expect(
+ toGPUStackModel({ name: 'm', model_types: ['chat'], max_tokens: 4096 }),
+ ).toEqual({ name: 'm', kind: 'chat', maxTokens: 4096 });
+ });
+
+ it('falls back when no context length is reported', () => {
+ expect(
+ toGPUStackModel({ name: 'm', model_types: ['asr'], max_tokens: 0 })
+ .maxTokens,
+ ).toBe(8192);
+ });
+});
+
+describe('groupModelsByKind', () => {
+ it('orders groups and omits empty ones', () => {
+ const groups = groupModelsByKind([
+ { name: 'r', kind: 'rerank' as const },
+ { name: 'c', kind: 'chat' as const },
+ ]);
+ expect(groups.map((group) => group.kind)).toEqual(['chat', 'rerank']);
+ });
+});
+
+describe('isLoopbackUrl', () => {
+ it.each([
+ ['http://localhost:8080', true],
+ ['http://127.0.0.1', true],
+ ['http://host.docker.internal:80', false],
+ ['http://10.0.0.5', false],
+ ['nonsense', false],
+ ])('%s -> %s', (url, expected) => {
+ expect(isLoopbackUrl(url)).toBe(expected);
+ });
+});
+
+describe('classifyConnectionError', () => {
+ it.each([
+ ['API request failed with status 401: {"code":401}', 'auth'],
+ ['failed to send request: dial tcp 10.0.0.5:80: connect', 'unreachable'],
+ ['context deadline exceeded', 'unreachable'],
+ ['API request failed with status 500: boom', 'other'],
+ [undefined, 'other'],
+ ])('%s -> %s', (message, expected) => {
+ expect(classifyConnectionError(message)).toBe(expected);
+ });
+});
+
+describe('diffModels', () => {
+ const model = (name: string) => ({
+ name,
+ kind: 'chat' as const,
+ maxTokens: 8192,
+ });
+
+ it('separates new models and marks stopped ones', () => {
+ const { notAdded, added } = diffModels(
+ [model('running'), model('new')],
+ [model('running'), model('stopped')],
+ );
+ expect(notAdded.map((m) => m.name)).toEqual(['new']);
+ expect(added).toEqual([
+ { ...model('running'), running: true },
+ { ...model('stopped'), running: false },
+ ]);
+ });
+});
diff --git a/web/src/pages/user-setting/setting-model/instance-card/gpustack-instance-utils.ts b/web/src/pages/user-setting/setting-model/instance-card/gpustack-instance-utils.ts
new file mode 100644
index 0000000000..20a39a0544
--- /dev/null
+++ b/web/src/pages/user-setting/setting-model/instance-card/gpustack-instance-utils.ts
@@ -0,0 +1,125 @@
+/*
+ * Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** What a model is used as. Vision is a chat model that also accepts images. */
+export type GPUStackModelKind =
+ | 'chat'
+ | 'vision'
+ | 'embedding'
+ | 'rerank'
+ | 'asr'
+ | 'tts';
+
+/** Display order of the model groups. */
+export const GPUSTACK_MODEL_KINDS: GPUStackModelKind[] = [
+ 'chat',
+ 'vision',
+ 'embedding',
+ 'rerank',
+ 'asr',
+ 'tts',
+];
+
+/** Used when GPUStack does not report a context length (audio models). */
+export const GPUSTACK_DEFAULT_MAX_TOKENS = 8192;
+
+export interface GPUStackModel {
+ name: string;
+ kind: GPUStackModelKind;
+ maxTokens: number;
+}
+
+export type GPUStackConnectionError = 'auth' | 'unreachable' | 'other';
+
+export function kindFromModelTypes(types?: string[] | null): GPUStackModelKind {
+ const list = types ?? [];
+ if (list.includes('vision') || list.includes('image2text')) return 'vision';
+ if (list.includes('embedding')) return 'embedding';
+ if (list.includes('rerank')) return 'rerank';
+ if (list.includes('asr') || list.includes('speech2text')) return 'asr';
+ if (list.includes('tts')) return 'tts';
+ return 'chat';
+}
+
+export function modelTypesFromKind(kind: GPUStackModelKind): string[] {
+ return kind === 'vision' ? ['chat', 'vision'] : [kind];
+}
+
+export function toGPUStackModel(item: {
+ name: string;
+ model_types?: string[] | null;
+ max_tokens?: number | null;
+}): GPUStackModel {
+ return {
+ name: item.name,
+ kind: kindFromModelTypes(item.model_types),
+ maxTokens: item.max_tokens || GPUSTACK_DEFAULT_MAX_TOKENS,
+ };
+}
+
+/** Group models by kind, keeping only the groups that have a model. */
+export function groupModelsByKind<T extends { kind: GPUStackModelKind }>(
+ models: T[],
+): { kind: GPUStackModelKind; models: T[] }[] {
+ return GPUSTACK_MODEL_KINDS.map((kind) => ({
+ kind,
+ models: models.filter((model) => model.kind === kind),
+ })).filter((group) => group.models.length > 0);
+}
+
+/** RAGFlow runs in a container, where localhost is not the host machine. */
+export function isLoopbackUrl(baseUrl: string): boolean {
+ try {
+ const { hostname } = new URL(baseUrl.trim());
+ return ['localhost', '127.0.0.1', '0.0.0.0', '[::1]'].includes(hostname);
+ } catch {
+ return false;
+ }
+}
+
+export function classifyConnectionError(
+ message?: string,
+): GPUStackConnectionError {
+ const text = (message ?? '').toLowerCase();
+ if (/status (401|403)\b/.test(text)) return 'auth';
+ if (
+ /failed to send request|connection refused|no such host|dial tcp|timeout|deadline exceeded|unreachable/.test(
+ text,
+ )
+ ) {
+ return 'unreachable';
+ }
+ return 'other';
+}
+
+/** Compare what GPUStack serves with what this connection already has. */
+export function diffModels(
+ discovered: GPUStackModel[],
+ added: GPUStackModel[],
+): {
+ notAdded: GPUStackModel[];
+ added: (GPUStackModel & { running: boolean })[];
+} {
+ const addedNames = new Set(added.map((model) => model.name));
+ const servedNames = new Set(discovered.map((model) => model.name));
+ return {
+ notAdded: discovered.filter((model) => !addedNames.has(model.name)),
+ added: added.map((model) => ({
+ ...model,
+ running: servedNames.has(model.name),
+ })),
+ };
+}
diff --git a/web/src/pages/user-setting/setting-model/instance-card/provider-instance-card.tsx b/web/src/pages/user-setting/setting-model/instance-card/provider-instance-card.tsx
index c0924b1304..8e68de22d4 100644
--- a/web/src/pages/user-setting/setting-model/instance-card/provider-instance-card.tsx
+++ b/web/src/pages/user-setting/setting-model/instance-card/provider-instance-card.tsx
@@ -31,6 +31,7 @@ import {
import { useFetchInstanceNameSet, useHideWhenInstanceExists } from '../hooks';
import { getProviderConfig } from '../provider-schema/field-config';
import { BedrockInstanceCard } from './bedrock-instance-card';
+import { GPUStackInstanceCard } from './gpustack-instance-card';
import { DraftModeCard } from './components/draft-mode-card';
import { InstanceNameSection } from './components/instance-name-section';
import { SavedModeCard } from './components/saved-mode-card';
@@ -327,6 +328,9 @@ export const ProviderInstanceCard = forwardRef<
if (props.providerName === 'Bedrock') {
return <BedrockInstanceCard {...props} ref={ref} />;
}
+ if (props.providerName === 'GPUStack') {
+ return <GPUStackInstanceCard {...props} ref={ref} />;
+ }
return <GenericProviderInstanceCard {...props} ref={ref} />;
});
diff --git a/web/src/pages/user-setting/setting-model/provider-schema/field-config/local-llm-configs.ts b/web/src/pages/user-setting/setting-model/provider-schema/field-config/local-llm-configs.ts
index a576a83350..5b654e32ac 100644
--- a/web/src/pages/user-setting/setting-model/provider-schema/field-config/local-llm-configs.ts
+++ b/web/src/pages/user-setting/setting-model/provider-schema/field-config/local-llm-configs.ts
@@ -181,7 +181,7 @@ export const LocalLlmConfigs: Record<string, ProviderConfig> = {
'GPUStack',
false,
undefined,
- 'https://docs.gpustack.ai/latest/quickstart',
+ 'https://docs.gpustack.ai/latest/integrations/integrate-with-ragflow/',
),
[LLMFactory.VLLM]: buildLocalConfig(
LLMFactory.VLLM,