Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat[provider]: apply model_type infer when model has no type (#18593)

Haruko3868ac8ca71daca
diff --git a/internal/entity/models/base_model.go b/internal/entity/models/base_model.go
index 5018e19ce2..e32651204a 100644
--- a/internal/entity/models/base_model.go
+++ b/internal/entity/models/base_model.go
@@ -423,10 +423,9 @@ func ParseListModel(modelList ModelList) []ListModelResponse {
if len(modelResponse.ModelTypes) == 0 {
modelResponse.ModelTypes = InferModelTypes(modelName)
}
-
models = append(models, modelResponse)
}
- return models
+ return FillMissingModelTypes(models)
}
// NewDriverHTTPClient returns an *http.Client with the standard connection-pool
diff --git a/internal/entity/models/base_model_test.go b/internal/entity/models/base_model_test.go
index 9312060006..ddd81346e6 100644
--- a/internal/entity/models/base_model_test.go
+++ b/internal/entity/models/base_model_test.go
@@ -23,8 +23,8 @@ import (
// ParseListModel feeds the provider-model list endpoint whose merge treats
// remote entries as authoritative: every returned entry must carry non-empty
-// model types. Catalog hits keep the catalog types; catalog misses fall back
-// to name-based inference (defaulting to chat), mirroring Python's
+// model types. Catalog hits are normalized; catalog misses fall back to
+// name-based inference (defaulting to chat), mirroring Python's
// OpenAIAPICompatible._format_model_list.
func TestParseListModelAssignsModelTypes(t *testing.T) {
dir, restore := setupProviderTestDir(t, "aliyun.json")
@@ -35,7 +35,7 @@ func TestParseListModelAssignsModelTypes(t *testing.T) {
}
got := ParseListModel(ModelList{Models: []ModelListItem{
- {ID: "qwen3-vl-plus"}, // catalog hit: chat+vision+ocr
+ {ID: "qwen3-vl-plus"}, // catalog hit: chat+vision+ocr -> chat+vision
{ID: "qwen-vl-plus"}, // catalog hit: chat+vision
{ID: "qwen3-vl-flash-2099-01-01"}, // miss: future dated variant -> inferred
{ID: "gpt-4o-2099-01-01"}, // miss -> inferred
@@ -47,7 +47,7 @@ func TestParseListModelAssignsModelTypes(t *testing.T) {
}})
want := map[string][]string{
- "qwen3-vl-plus": {"chat", "vision", "ocr"}, // catalog wins over inference
+ "qwen3-vl-plus": {"chat", "vision"},
"qwen-vl-plus": {"chat", "vision"},
"qwen3-vl-flash-2099-01-01": {"chat", "vision"},
"gpt-4o-2099-01-01": {"chat", "vision"},
diff --git a/internal/entity/models/gitee_test.go b/internal/entity/models/gitee_test.go
index f894a2e18b..07e7c0b69b 100644
--- a/internal/entity/models/gitee_test.go
+++ b/internal/entity/models/gitee_test.go
@@ -169,8 +169,9 @@ func TestGiteeListModelsMapsAllDeepSeekAliasesToModelMetadata(t *testing.T) {
if model.MaxOutput != nil && expected.MaxOutput != nil && *model.MaxOutput != *expected.MaxOutput {
t.Fatalf("models[%d] alias %q MaxOutput=%d, want %d", i, alias, *model.MaxOutput, *expected.MaxOutput)
}
- if strings.Join(model.ModelTypes, ",") != strings.Join(expected.ModelTypes, ",") {
- t.Fatalf("models[%d] alias %q ModelTypes=%v, want %v", i, alias, model.ModelTypes, expected.ModelTypes)
+ expectedModelTypes := normalizeModelTypes(expected.ModelTypes)
+ if strings.Join(model.ModelTypes, ",") != strings.Join(expectedModelTypes, ",") {
+ t.Fatalf("models[%d] alias %q ModelTypes=%v, want %v", i, alias, model.ModelTypes, expectedModelTypes)
}
if (model.Thinking == nil) != (expected.Thinking == nil) {
t.Fatalf("models[%d] alias %q Thinking nil=%t, want nil=%t", i, alias, model.Thinking == nil, expected.Thinking == nil)
diff --git a/internal/entity/models/model_type_infer.go b/internal/entity/models/model_type_infer.go
new file mode 100644
index 0000000000..3252877630
--- /dev/null
+++ b/internal/entity/models/model_type_infer.go
@@ -0,0 +1,384 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package models
+
+import (
+ "math"
+ "regexp"
+ "strconv"
+ "strings"
+)
+
+const (
+ modelTypeChat = "chat"
+ modelTypeEmbedding = "embedding"
+ modelTypeRerank = "rerank"
+ modelTypeASR = "asr"
+ modelTypeVision = "vision"
+ modelTypeTTS = "tts"
+ modelTypeOCR = "ocr"
+)
+
+const (
+ modelTypeSimilarityBaseline = 0.78
+ modelTypeSimilarityMargin = 0.005
+)
+
+var (
+ modelNameTokenRE = regexp.MustCompile(`[a-z0-9]+`)
+ modelNameLetterNumberTokenRE = regexp.MustCompile(`^([a-z]+)([0-9]+)$`)
+ modelNameNumberTokenRE = regexp.MustCompile(`^[0-9]+$`)
+)
+
+// InferModelTypes derives RAGFlow LLM model types from the model name using
+// keyword heuristics, covering all seven supported types: chat, embedding,
+// rerank, asr, tts, ocr, and vision (always combined with chat).
+func InferModelTypes(modelName string) []string {
+ modelName = strings.TrimSpace(modelName)
+ if modelName == "" {
+ return []string{modelTypeChat}
+ }
+ if modelTypes := inferModelTypesByName(modelName); len(modelTypes) > 0 {
+ return normalizeModelTypes(modelTypes)
+ }
+ return []string{modelTypeChat}
+}
+
+// FillMissingModelTypes fills missing model types using only the models present
+// in the provided list. Existing model types are normalized before returning.
+func FillMissingModelTypes(models []ListModelResponse) []ListModelResponse {
+ for i := range models {
+ if len(models[i].ModelTypes) > 0 {
+ models[i].ModelTypes = normalizeModelTypes(models[i].ModelTypes)
+ continue
+ }
+ if modelTypes := inferModelTypesByName(models[i].Name); len(modelTypes) > 0 {
+ models[i].ModelTypes = normalizeModelTypes(modelTypes)
+ }
+ }
+ candidates := append([]ListModelResponse(nil), models...)
+ inferred := make([][]string, len(models))
+ for i := range models {
+ if len(models[i].ModelTypes) == 0 {
+ if modelTypes := inferModelTypesBySimilarity(models[i].Name, candidates); len(modelTypes) > 0 {
+ inferred[i] = normalizeModelTypes(modelTypes)
+ }
+ }
+ }
+ for i := range models {
+ if len(models[i].ModelTypes) == 0 && len(inferred[i]) > 0 {
+ models[i].ModelTypes = inferred[i]
+ }
+ if len(models[i].ModelTypes) == 0 {
+ models[i].ModelTypes = []string{modelTypeChat}
+ }
+ }
+ return models
+}
+
+func inferModelTypesByName(modelName string) []string {
+ lowerName := strings.ToLower(modelName)
+ tokens := modelNameTokens(modelName)
+ hasToken := func(values ...string) bool {
+ for _, value := range values {
+ if _, ok := tokens[value]; ok {
+ return true
+ }
+ }
+ return false
+ }
+
+ switch {
+ case hasToken("rerank", "reranker"):
+ return []string{modelTypeRerank}
+ case hasToken("ocr"):
+ return []string{modelTypeOCR}
+ case hasToken("asr", "stt", "transcribe", "transcriber", "whisper", "audio"):
+ return []string{modelTypeASR}
+ case hasToken("tts") || strings.Contains(lowerName, "text-to-speech"):
+ return []string{modelTypeTTS}
+ case hasToken("embed", "embedding", "embeddings", "bge", "e5"):
+ return []string{modelTypeEmbedding}
+ case strings.Contains(lowerName, "qwen-vl") ||
+ strings.Contains(lowerName, "glm-4v") ||
+ strings.Contains(lowerName, "minicpm-v") ||
+ strings.Contains(lowerName, "gpt-4o") ||
+ hasToken("vl", "vlm", "vision", "llava", "internvl", "pixtral", "qvq", "image", "video"):
+ return []string{modelTypeChat, modelTypeVision}
+ default:
+ return nil
+ }
+}
+
+func modelNameTokens(modelName string) map[string]struct{} {
+ tokens := modelNameComparableTokens(modelName)
+ tokenSet := map[string]struct{}{}
+ for _, token := range tokens {
+ tokenSet[token] = struct{}{}
+ }
+ return tokenSet
+}
+
+func modelNameComparableTokens(modelName string) []string {
+ modelName = comparableModelNameText(modelName)
+ tokens := map[string]struct{}{}
+ ordered := []string{}
+ add := func(token string) {
+ if token == "" {
+ return
+ }
+ if _, ok := tokens[token]; ok {
+ return
+ }
+ tokens[token] = struct{}{}
+ ordered = append(ordered, token)
+ }
+ for _, token := range modelNameTokenRE.FindAllString(modelName, -1) {
+ if parts := modelNameLetterNumberTokenRE.FindStringSubmatch(token); len(parts) == 3 {
+ add(parts[1])
+ add(parts[2])
+ continue
+ }
+ add(token)
+ }
+ return ordered
+}
+
+func comparableModelNameText(modelName string) string {
+ modelName = strings.ToLower(strings.TrimSpace(modelName))
+ if idx := strings.LastIndex(modelName, "/"); idx >= 0 && idx+1 < len(modelName) {
+ modelName = modelName[idx+1:]
+ }
+ return modelName
+}
+
+func inferModelTypesBySimilarity(modelName string, models []ListModelResponse) []string {
+ targetHint := inferModelTypesByName(modelName)
+ target := newModelNameProfile(modelName)
+ if len(target.tokenSet) == 0 {
+ return nil
+ }
+
+ bestScore := 0.0
+ secondScore := 0.0
+ var bestTypes []string
+ var secondTypes []string
+ for _, model := range models {
+ if len(model.ModelTypes) == 0 {
+ continue
+ }
+ if strings.EqualFold(model.Name, modelName) {
+ continue
+ }
+ if !compatibleModelNameCapabilities(targetHint, inferModelTypesByName(model.Name)) {
+ continue
+ }
+ score := modelNameSimilarityScore(target, newModelNameProfile(model.Name))
+ if score > bestScore {
+ secondScore = bestScore
+ secondTypes = bestTypes
+ bestScore = score
+ bestTypes = normalizeModelTypes(model.ModelTypes)
+ continue
+ }
+ if score > secondScore {
+ secondScore = score
+ secondTypes = normalizeModelTypes(model.ModelTypes)
+ }
+ }
+ if bestScore < modelTypeSimilarityBaseline {
+ return nil
+ }
+ if secondScore > 0 && bestScore-secondScore < modelTypeSimilarityMargin && !sameModelTypes(bestTypes, secondTypes) {
+ return nil
+ }
+ return bestTypes
+}
+
+func compatibleModelNameCapabilities(targetHint, candidateHint []string) bool {
+ targetCapability := capabilityModelType(targetHint)
+ candidateCapability := capabilityModelType(candidateHint)
+ if targetCapability == "" {
+ return candidateCapability == ""
+ }
+ return targetCapability == candidateCapability
+}
+
+func capabilityModelType(modelTypes []string) string {
+ for _, modelType := range normalizeModelTypes(modelTypes) {
+ if modelType != modelTypeChat {
+ return modelType
+ }
+ }
+ return ""
+}
+
+type modelNameProfile struct {
+ tokens []string
+ tokenSet map[string]struct{}
+ brand string
+ series string
+ version float64
+ hasVersion bool
+}
+
+func newModelNameProfile(modelName string) modelNameProfile {
+ rawTokens := modelNameTokenRE.FindAllString(comparableModelNameText(modelName), -1)
+ profile := modelNameProfile{
+ tokens: modelNameComparableTokens(modelName),
+ tokenSet: map[string]struct{}{},
+ }
+ for _, token := range profile.tokens {
+ profile.tokenSet[token] = struct{}{}
+ }
+ if len(rawTokens) > 0 {
+ profile.brand = rawTokens[0]
+ profile.series = rawTokens[0]
+ }
+ for index, token := range rawTokens {
+ parts := modelNameLetterNumberTokenRE.FindStringSubmatch(token)
+ if len(parts) != 3 {
+ continue
+ }
+ versionText := parts[2]
+ if index+1 < len(rawTokens) && modelNameNumberTokenRE.MatchString(rawTokens[index+1]) {
+ versionText += "." + rawTokens[index+1]
+ }
+ version, err := strconv.ParseFloat(versionText, 64)
+ if err != nil {
+ continue
+ }
+ profile.series = parts[1]
+ profile.version = version
+ profile.hasVersion = true
+ if index > 0 {
+ profile.brand = rawTokens[0]
+ } else {
+ profile.brand = parts[1]
+ }
+ break
+ }
+ return profile
+}
+
+func modelNameSimilarityScore(a, b modelNameProfile) float64 {
+ if len(a.tokenSet) == 0 || len(b.tokenSet) == 0 || tokenSetIntersectionSize(a.tokenSet, b.tokenSet) == 0 {
+ return 0
+ }
+ jaccard := tokenSetJaccard(a.tokenSet, b.tokenSet)
+ series := modelNameSeriesScore(a, b)
+ version := modelNameVersionScore(a, b)
+ return 0.25*jaccard + 0.45*series + 0.30*version
+}
+
+func modelNameSeriesScore(a, b modelNameProfile) float64 {
+ if a.brand == "" || b.brand == "" || a.brand != b.brand {
+ return 0
+ }
+ if a.series != "" && a.series == b.series {
+ return 1
+ }
+ return 0.6
+}
+
+func modelNameVersionScore(a, b modelNameProfile) float64 {
+ if !a.hasVersion || !b.hasVersion || a.series == "" || a.series != b.series {
+ return 0
+ }
+ maxVersion := math.Max(1, math.Max(a.version, b.version))
+ score := 1 - math.Abs(a.version-b.version)/maxVersion
+ if score < 0 {
+ return 0
+ }
+ return score
+}
+
+func tokenSetIntersectionSize(a, b map[string]struct{}) int {
+ count := 0
+ for token := range a {
+ if _, ok := b[token]; ok {
+ count++
+ }
+ }
+ return count
+}
+
+func tokenSetJaccard(a, b map[string]struct{}) float64 {
+ intersection := tokenSetIntersectionSize(a, b)
+ union := len(a) + len(b) - intersection
+ if union == 0 {
+ return 0
+ }
+ return float64(intersection) / float64(union)
+}
+
+func normalizeModelTypes(modelTypes []string) []string {
+ seen := map[string]struct{}{}
+ unique := make([]string, 0, len(modelTypes))
+ for _, modelType := range modelTypes {
+ modelType = strings.TrimSpace(modelType)
+ if modelType == "" {
+ continue
+ }
+ if _, ok := seen[modelType]; ok {
+ continue
+ }
+ seen[modelType] = struct{}{}
+ unique = append(unique, modelType)
+ }
+ if len(unique) == 0 {
+ return []string{modelTypeChat}
+ }
+
+ hasChat := false
+ hasVision := false
+ hasOCR := false
+ for _, modelType := range unique {
+ hasChat = hasChat || modelType == modelTypeChat
+ hasVision = hasVision || modelType == modelTypeVision
+ hasOCR = hasOCR || modelType == modelTypeOCR
+ }
+ if hasChat && hasOCR {
+ return []string{modelTypeChat, modelTypeVision}
+ }
+ for _, modelType := range unique {
+ if modelType != modelTypeChat && modelType != modelTypeVision {
+ return []string{modelType}
+ }
+ }
+ if hasVision || hasChat {
+ if hasVision {
+ return []string{modelTypeChat, modelTypeVision}
+ }
+ return []string{modelTypeChat}
+ }
+ return []string{modelTypeChat}
+}
+
+func sameModelTypes(a, b []string) bool {
+ a = normalizeModelTypes(a)
+ b = normalizeModelTypes(b)
+ if len(a) != len(b) {
+ return false
+ }
+ for i := range a {
+ if a[i] != b[i] {
+ return false
+ }
+ }
+ return true
+}
diff --git a/internal/entity/models/openai_api_compatible.go b/internal/entity/models/openai_api_compatible.go
index dc54518260..5e48e06f50 100644
--- a/internal/entity/models/openai_api_compatible.go
+++ b/internal/entity/models/openai_api_compatible.go
@@ -160,60 +160,6 @@ func (m *OpenAIAPICompatibleModel) ChatStreamlyWithSender(ctx context.Context, m
})
}
-// Hint keywords for model type inference, matching Python's
-// OpenAIAPICompatible class-level hint constants.
-var (
- embeddingHints = []string{"embed", "embedding", "bge"}
- rerankHints = []string{"rerank", "reranker"}
- asrHints = []string{"asr", "stt", "transcribe", "transcriber", "whisper"}
- ttsHints = []string{"tts", "text-to-speech"}
- visionHints = []string{
- "vl", "vision", "llava", "internvl", "minicpm-v",
- "gpt-4o", "glm-4v", "qvq", "qwen-vl", "pixtral",
- }
- ocrHints = []string{"ocr"}
-)
-
-// containsHint checks whether modelName (lowercased) contains any of the
-// given hint substrings.
-func containsHint(modelName string, hints []string) bool {
- for _, hint := range hints {
- if strings.Contains(modelName, hint) {
- return true
- }
- }
- return false
-}
-
-// InferModelTypes derives RAGFlow LLM model types from the model name using
-// keyword heuristics, covering all seven supported types: chat, embedding,
-// rerank, asr, tts, ocr, and vision (always combined with chat).
-func InferModelTypes(modelName string) []string {
- lower := strings.ToLower(modelName)
-
- if containsHint(lower, rerankHints) {
- return []string{"rerank"}
- }
- if containsHint(lower, embeddingHints) {
- return []string{"embedding"}
- }
- if containsHint(lower, asrHints) {
- return []string{"asr"}
- }
- if containsHint(lower, ttsHints) {
- return []string{"tts"}
- }
- if containsHint(lower, ocrHints) {
- return []string{"ocr"}
- }
-
- types := []string{"chat"}
- if containsHint(lower, visionHints) {
- types = append(types, "vision")
- }
- return types
-}
-
// ttsVoiceForModel maps model names to appropriate TTS voices for
// OpenAI-compatible providers (e.g. SiliconFlow). Returns "alloy" as
// the generic fallback.
diff --git a/internal/handler/providers.go b/internal/handler/providers.go
index 718f9797dc..e9cb00db61 100644
--- a/internal/handler/providers.go
+++ b/internal/handler/providers.go
@@ -204,15 +204,23 @@ func (h *ProviderHandler) ListModels(c *gin.Context) {
}
for _, m := range remoteModels {
if name, ok := m["name"].(string); ok {
+ if len(providerModelMapTypes(m)) == 0 {
+ if existing := merged[name]; len(providerModelMapTypes(existing)) > 0 {
+ m["model_types"] = existing["model_types"]
+ }
+ }
merged[name] = m
}
}
- // 5. Sort by name
+ // 5. Fill missing model types using only the merged list.
result := make([]map[string]interface{}, 0, len(merged))
for _, m := range merged {
result = append(result, m)
}
+ fillProviderModelMapTypes(result)
+
+ // 6. Sort by name
sort.Slice(result, func(i, j int) bool {
ni, _ := result[i]["name"].(string)
nj, _ := result[j]["name"].(string)
@@ -222,6 +230,44 @@ func (h *ProviderHandler) ListModels(c *gin.Context) {
common.SuccessWithData(c, result, "success")
}
+func fillProviderModelMapTypes(result []map[string]interface{}) {
+ list := make([]models.ListModelResponse, 0, len(result))
+ indexes := make([]int, 0, len(result))
+ for i, model := range result {
+ name, _ := model["name"].(string)
+ name = strings.TrimSpace(name)
+ if name == "" {
+ continue
+ }
+ list = append(list, models.ListModelResponse{
+ Name: name,
+ ModelTypes: providerModelMapTypes(model),
+ })
+ indexes = append(indexes, i)
+ }
+ list = models.FillMissingModelTypes(list)
+ for i, model := range list {
+ result[indexes[i]]["model_types"] = model.ModelTypes
+ }
+}
+
+func providerModelMapTypes(model map[string]interface{}) []string {
+ switch modelTypes := model["model_types"].(type) {
+ case []string:
+ return modelTypes
+ case []interface{}:
+ types := make([]string, 0, len(modelTypes))
+ for _, modelType := range modelTypes {
+ if s, ok := modelType.(string); ok {
+ types = append(types, s)
+ }
+ }
+ return types
+ default:
+ return nil
+ }
+}
+
func (h *ProviderHandler) ShowModel(c *gin.Context) {
providerName := c.Param("provider_id_or_name")
if providerName == "" {