RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat[provider]: apply model_type infer when model has no type (#18593)
Haruko3868ac8ca71daca
diff --git a/internal/entity/models/base_model.go b/internal/entity/models/base_model.goindex 5018e19ce2..e32651204a 100644--- a/internal/entity/models/base_model.go+++ b/internal/entity/models/base_model.go@@ -423,10 +423,9 @@ func ParseListModel(modelList ModelList) []ListModelResponse {if len(modelResponse.ModelTypes) == 0 {modelResponse.ModelTypes = InferModelTypes(modelName)}-models = append(models, modelResponse)}- return models+ return FillMissingModelTypes(models)}// NewDriverHTTPClient returns an *http.Client with the standard connection-pooldiff --git a/internal/entity/models/base_model_test.go b/internal/entity/models/base_model_test.goindex 9312060006..ddd81346e6 100644--- a/internal/entity/models/base_model_test.go+++ b/internal/entity/models/base_model_test.go@@ -23,8 +23,8 @@ import (// ParseListModel feeds the provider-model list endpoint whose merge treats// remote entries as authoritative: every returned entry must carry non-empty-// model types. Catalog hits keep the catalog types; catalog misses fall back-// to name-based inference (defaulting to chat), mirroring Python's+// model types. Catalog hits are normalized; catalog misses fall back to+// name-based inference (defaulting to chat), mirroring Python's// OpenAIAPICompatible._format_model_list.func TestParseListModelAssignsModelTypes(t *testing.T) {dir, restore := setupProviderTestDir(t, "aliyun.json")@@ -35,7 +35,7 @@ func TestParseListModelAssignsModelTypes(t *testing.T) {}got := ParseListModel(ModelList{Models: []ModelListItem{- {ID: "qwen3-vl-plus"}, // catalog hit: chat+vision+ocr+ {ID: "qwen3-vl-plus"}, // catalog hit: chat+vision+ocr -> chat+vision{ID: "qwen-vl-plus"}, // catalog hit: chat+vision{ID: "qwen3-vl-flash-2099-01-01"}, // miss: future dated variant -> inferred{ID: "gpt-4o-2099-01-01"}, // miss -> inferred@@ -47,7 +47,7 @@ func TestParseListModelAssignsModelTypes(t *testing.T) {}})want := map[string][]string{- "qwen3-vl-plus": {"chat", "vision", "ocr"}, // catalog wins over inference+ "qwen3-vl-plus": {"chat", "vision"},"qwen-vl-plus": {"chat", "vision"},"qwen3-vl-flash-2099-01-01": {"chat", "vision"},"gpt-4o-2099-01-01": {"chat", "vision"},diff --git a/internal/entity/models/gitee_test.go b/internal/entity/models/gitee_test.goindex f894a2e18b..07e7c0b69b 100644--- a/internal/entity/models/gitee_test.go+++ b/internal/entity/models/gitee_test.go@@ -169,8 +169,9 @@ func TestGiteeListModelsMapsAllDeepSeekAliasesToModelMetadata(t *testing.T) {if model.MaxOutput != nil && expected.MaxOutput != nil && *model.MaxOutput != *expected.MaxOutput {t.Fatalf("models[%d] alias %q MaxOutput=%d, want %d", i, alias, *model.MaxOutput, *expected.MaxOutput)}- if strings.Join(model.ModelTypes, ",") != strings.Join(expected.ModelTypes, ",") {- t.Fatalf("models[%d] alias %q ModelTypes=%v, want %v", i, alias, model.ModelTypes, expected.ModelTypes)+ expectedModelTypes := normalizeModelTypes(expected.ModelTypes)+ if strings.Join(model.ModelTypes, ",") != strings.Join(expectedModelTypes, ",") {+ t.Fatalf("models[%d] alias %q ModelTypes=%v, want %v", i, alias, model.ModelTypes, expectedModelTypes)}if (model.Thinking == nil) != (expected.Thinking == nil) {t.Fatalf("models[%d] alias %q Thinking nil=%t, want nil=%t", i, alias, model.Thinking == nil, expected.Thinking == nil)diff --git a/internal/entity/models/model_type_infer.go b/internal/entity/models/model_type_infer.gonew file mode 100644index 0000000000..3252877630--- /dev/null+++ b/internal/entity/models/model_type_infer.go@@ -0,0 +1,384 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package models++import (+ "math"+ "regexp"+ "strconv"+ "strings"+)++const (+ modelTypeChat = "chat"+ modelTypeEmbedding = "embedding"+ modelTypeRerank = "rerank"+ modelTypeASR = "asr"+ modelTypeVision = "vision"+ modelTypeTTS = "tts"+ modelTypeOCR = "ocr"+)++const (+ modelTypeSimilarityBaseline = 0.78+ modelTypeSimilarityMargin = 0.005+)++var (+ modelNameTokenRE = regexp.MustCompile(`[a-z0-9]+`)+ modelNameLetterNumberTokenRE = regexp.MustCompile(`^([a-z]+)([0-9]+)$`)+ modelNameNumberTokenRE = regexp.MustCompile(`^[0-9]+$`)+)++// InferModelTypes derives RAGFlow LLM model types from the model name using+// keyword heuristics, covering all seven supported types: chat, embedding,+// rerank, asr, tts, ocr, and vision (always combined with chat).+func InferModelTypes(modelName string) []string {+ modelName = strings.TrimSpace(modelName)+ if modelName == "" {+ return []string{modelTypeChat}+ }+ if modelTypes := inferModelTypesByName(modelName); len(modelTypes) > 0 {+ return normalizeModelTypes(modelTypes)+ }+ return []string{modelTypeChat}+}++// FillMissingModelTypes fills missing model types using only the models present+// in the provided list. Existing model types are normalized before returning.+func FillMissingModelTypes(models []ListModelResponse) []ListModelResponse {+ for i := range models {+ if len(models[i].ModelTypes) > 0 {+ models[i].ModelTypes = normalizeModelTypes(models[i].ModelTypes)+ continue+ }+ if modelTypes := inferModelTypesByName(models[i].Name); len(modelTypes) > 0 {+ models[i].ModelTypes = normalizeModelTypes(modelTypes)+ }+ }+ candidates := append([]ListModelResponse(nil), models...)+ inferred := make([][]string, len(models))+ for i := range models {+ if len(models[i].ModelTypes) == 0 {+ if modelTypes := inferModelTypesBySimilarity(models[i].Name, candidates); len(modelTypes) > 0 {+ inferred[i] = normalizeModelTypes(modelTypes)+ }+ }+ }+ for i := range models {+ if len(models[i].ModelTypes) == 0 && len(inferred[i]) > 0 {+ models[i].ModelTypes = inferred[i]+ }+ if len(models[i].ModelTypes) == 0 {+ models[i].ModelTypes = []string{modelTypeChat}+ }+ }+ return models+}++func inferModelTypesByName(modelName string) []string {+ lowerName := strings.ToLower(modelName)+ tokens := modelNameTokens(modelName)+ hasToken := func(values ...string) bool {+ for _, value := range values {+ if _, ok := tokens[value]; ok {+ return true+ }+ }+ return false+ }++ switch {+ case hasToken("rerank", "reranker"):+ return []string{modelTypeRerank}+ case hasToken("ocr"):+ return []string{modelTypeOCR}+ case hasToken("asr", "stt", "transcribe", "transcriber", "whisper", "audio"):+ return []string{modelTypeASR}+ case hasToken("tts") || strings.Contains(lowerName, "text-to-speech"):+ return []string{modelTypeTTS}+ case hasToken("embed", "embedding", "embeddings", "bge", "e5"):+ return []string{modelTypeEmbedding}+ case strings.Contains(lowerName, "qwen-vl") ||+ strings.Contains(lowerName, "glm-4v") ||+ strings.Contains(lowerName, "minicpm-v") ||+ strings.Contains(lowerName, "gpt-4o") ||+ hasToken("vl", "vlm", "vision", "llava", "internvl", "pixtral", "qvq", "image", "video"):+ return []string{modelTypeChat, modelTypeVision}+ default:+ return nil+ }+}++func modelNameTokens(modelName string) map[string]struct{} {+ tokens := modelNameComparableTokens(modelName)+ tokenSet := map[string]struct{}{}+ for _, token := range tokens {+ tokenSet[token] = struct{}{}+ }+ return tokenSet+}++func modelNameComparableTokens(modelName string) []string {+ modelName = comparableModelNameText(modelName)+ tokens := map[string]struct{}{}+ ordered := []string{}+ add := func(token string) {+ if token == "" {+ return+ }+ if _, ok := tokens[token]; ok {+ return+ }+ tokens[token] = struct{}{}+ ordered = append(ordered, token)+ }+ for _, token := range modelNameTokenRE.FindAllString(modelName, -1) {+ if parts := modelNameLetterNumberTokenRE.FindStringSubmatch(token); len(parts) == 3 {+ add(parts[1])+ add(parts[2])+ continue+ }+ add(token)+ }+ return ordered+}++func comparableModelNameText(modelName string) string {+ modelName = strings.ToLower(strings.TrimSpace(modelName))+ if idx := strings.LastIndex(modelName, "/"); idx >= 0 && idx+1 < len(modelName) {+ modelName = modelName[idx+1:]+ }+ return modelName+}++func inferModelTypesBySimilarity(modelName string, models []ListModelResponse) []string {+ targetHint := inferModelTypesByName(modelName)+ target := newModelNameProfile(modelName)+ if len(target.tokenSet) == 0 {+ return nil+ }++ bestScore := 0.0+ secondScore := 0.0+ var bestTypes []string+ var secondTypes []string+ for _, model := range models {+ if len(model.ModelTypes) == 0 {+ continue+ }+ if strings.EqualFold(model.Name, modelName) {+ continue+ }+ if !compatibleModelNameCapabilities(targetHint, inferModelTypesByName(model.Name)) {+ continue+ }+ score := modelNameSimilarityScore(target, newModelNameProfile(model.Name))+ if score > bestScore {+ secondScore = bestScore+ secondTypes = bestTypes+ bestScore = score+ bestTypes = normalizeModelTypes(model.ModelTypes)+ continue+ }+ if score > secondScore {+ secondScore = score+ secondTypes = normalizeModelTypes(model.ModelTypes)+ }+ }+ if bestScore < modelTypeSimilarityBaseline {+ return nil+ }+ if secondScore > 0 && bestScore-secondScore < modelTypeSimilarityMargin && !sameModelTypes(bestTypes, secondTypes) {+ return nil+ }+ return bestTypes+}++func compatibleModelNameCapabilities(targetHint, candidateHint []string) bool {+ targetCapability := capabilityModelType(targetHint)+ candidateCapability := capabilityModelType(candidateHint)+ if targetCapability == "" {+ return candidateCapability == ""+ }+ return targetCapability == candidateCapability+}++func capabilityModelType(modelTypes []string) string {+ for _, modelType := range normalizeModelTypes(modelTypes) {+ if modelType != modelTypeChat {+ return modelType+ }+ }+ return ""+}++type modelNameProfile struct {+ tokens []string+ tokenSet map[string]struct{}+ brand string+ series string+ version float64+ hasVersion bool+}++func newModelNameProfile(modelName string) modelNameProfile {+ rawTokens := modelNameTokenRE.FindAllString(comparableModelNameText(modelName), -1)+ profile := modelNameProfile{+ tokens: modelNameComparableTokens(modelName),+ tokenSet: map[string]struct{}{},+ }+ for _, token := range profile.tokens {+ profile.tokenSet[token] = struct{}{}+ }+ if len(rawTokens) > 0 {+ profile.brand = rawTokens[0]+ profile.series = rawTokens[0]+ }+ for index, token := range rawTokens {+ parts := modelNameLetterNumberTokenRE.FindStringSubmatch(token)+ if len(parts) != 3 {+ continue+ }+ versionText := parts[2]+ if index+1 < len(rawTokens) && modelNameNumberTokenRE.MatchString(rawTokens[index+1]) {+ versionText += "." + rawTokens[index+1]+ }+ version, err := strconv.ParseFloat(versionText, 64)+ if err != nil {+ continue+ }+ profile.series = parts[1]+ profile.version = version+ profile.hasVersion = true+ if index > 0 {+ profile.brand = rawTokens[0]+ } else {+ profile.brand = parts[1]+ }+ break+ }+ return profile+}++func modelNameSimilarityScore(a, b modelNameProfile) float64 {+ if len(a.tokenSet) == 0 || len(b.tokenSet) == 0 || tokenSetIntersectionSize(a.tokenSet, b.tokenSet) == 0 {+ return 0+ }+ jaccard := tokenSetJaccard(a.tokenSet, b.tokenSet)+ series := modelNameSeriesScore(a, b)+ version := modelNameVersionScore(a, b)+ return 0.25*jaccard + 0.45*series + 0.30*version+}++func modelNameSeriesScore(a, b modelNameProfile) float64 {+ if a.brand == "" || b.brand == "" || a.brand != b.brand {+ return 0+ }+ if a.series != "" && a.series == b.series {+ return 1+ }+ return 0.6+}++func modelNameVersionScore(a, b modelNameProfile) float64 {+ if !a.hasVersion || !b.hasVersion || a.series == "" || a.series != b.series {+ return 0+ }+ maxVersion := math.Max(1, math.Max(a.version, b.version))+ score := 1 - math.Abs(a.version-b.version)/maxVersion+ if score < 0 {+ return 0+ }+ return score+}++func tokenSetIntersectionSize(a, b map[string]struct{}) int {+ count := 0+ for token := range a {+ if _, ok := b[token]; ok {+ count+++ }+ }+ return count+}++func tokenSetJaccard(a, b map[string]struct{}) float64 {+ intersection := tokenSetIntersectionSize(a, b)+ union := len(a) + len(b) - intersection+ if union == 0 {+ return 0+ }+ return float64(intersection) / float64(union)+}++func normalizeModelTypes(modelTypes []string) []string {+ seen := map[string]struct{}{}+ unique := make([]string, 0, len(modelTypes))+ for _, modelType := range modelTypes {+ modelType = strings.TrimSpace(modelType)+ if modelType == "" {+ continue+ }+ if _, ok := seen[modelType]; ok {+ continue+ }+ seen[modelType] = struct{}{}+ unique = append(unique, modelType)+ }+ if len(unique) == 0 {+ return []string{modelTypeChat}+ }++ hasChat := false+ hasVision := false+ hasOCR := false+ for _, modelType := range unique {+ hasChat = hasChat || modelType == modelTypeChat+ hasVision = hasVision || modelType == modelTypeVision+ hasOCR = hasOCR || modelType == modelTypeOCR+ }+ if hasChat && hasOCR {+ return []string{modelTypeChat, modelTypeVision}+ }+ for _, modelType := range unique {+ if modelType != modelTypeChat && modelType != modelTypeVision {+ return []string{modelType}+ }+ }+ if hasVision || hasChat {+ if hasVision {+ return []string{modelTypeChat, modelTypeVision}+ }+ return []string{modelTypeChat}+ }+ return []string{modelTypeChat}+}++func sameModelTypes(a, b []string) bool {+ a = normalizeModelTypes(a)+ b = normalizeModelTypes(b)+ if len(a) != len(b) {+ return false+ }+ for i := range a {+ if a[i] != b[i] {+ return false+ }+ }+ return true+}diff --git a/internal/entity/models/openai_api_compatible.go b/internal/entity/models/openai_api_compatible.goindex dc54518260..5e48e06f50 100644--- a/internal/entity/models/openai_api_compatible.go+++ b/internal/entity/models/openai_api_compatible.go@@ -160,60 +160,6 @@ func (m *OpenAIAPICompatibleModel) ChatStreamlyWithSender(ctx context.Context, m})}-// Hint keywords for model type inference, matching Python's-// OpenAIAPICompatible class-level hint constants.-var (- embeddingHints = []string{"embed", "embedding", "bge"}- rerankHints = []string{"rerank", "reranker"}- asrHints = []string{"asr", "stt", "transcribe", "transcriber", "whisper"}- ttsHints = []string{"tts", "text-to-speech"}- visionHints = []string{- "vl", "vision", "llava", "internvl", "minicpm-v",- "gpt-4o", "glm-4v", "qvq", "qwen-vl", "pixtral",- }- ocrHints = []string{"ocr"}-)--// containsHint checks whether modelName (lowercased) contains any of the-// given hint substrings.-func containsHint(modelName string, hints []string) bool {- for _, hint := range hints {- if strings.Contains(modelName, hint) {- return true- }- }- return false-}--// InferModelTypes derives RAGFlow LLM model types from the model name using-// keyword heuristics, covering all seven supported types: chat, embedding,-// rerank, asr, tts, ocr, and vision (always combined with chat).-func InferModelTypes(modelName string) []string {- lower := strings.ToLower(modelName)-- if containsHint(lower, rerankHints) {- return []string{"rerank"}- }- if containsHint(lower, embeddingHints) {- return []string{"embedding"}- }- if containsHint(lower, asrHints) {- return []string{"asr"}- }- if containsHint(lower, ttsHints) {- return []string{"tts"}- }- if containsHint(lower, ocrHints) {- return []string{"ocr"}- }-- types := []string{"chat"}- if containsHint(lower, visionHints) {- types = append(types, "vision")- }- return types-}-// ttsVoiceForModel maps model names to appropriate TTS voices for// OpenAI-compatible providers (e.g. SiliconFlow). Returns "alloy" as// the generic fallback.diff --git a/internal/handler/providers.go b/internal/handler/providers.goindex 718f9797dc..e9cb00db61 100644--- a/internal/handler/providers.go+++ b/internal/handler/providers.go@@ -204,15 +204,23 @@ func (h *ProviderHandler) ListModels(c *gin.Context) {}for _, m := range remoteModels {if name, ok := m["name"].(string); ok {+ if len(providerModelMapTypes(m)) == 0 {+ if existing := merged[name]; len(providerModelMapTypes(existing)) > 0 {+ m["model_types"] = existing["model_types"]+ }+ }merged[name] = m}}- // 5. Sort by name+ // 5. Fill missing model types using only the merged list.result := make([]map[string]interface{}, 0, len(merged))for _, m := range merged {result = append(result, m)}+ fillProviderModelMapTypes(result)++ // 6. Sort by namesort.Slice(result, func(i, j int) bool {ni, _ := result[i]["name"].(string)nj, _ := result[j]["name"].(string)@@ -222,6 +230,44 @@ func (h *ProviderHandler) ListModels(c *gin.Context) {common.SuccessWithData(c, result, "success")}+func fillProviderModelMapTypes(result []map[string]interface{}) {+ list := make([]models.ListModelResponse, 0, len(result))+ indexes := make([]int, 0, len(result))+ for i, model := range result {+ name, _ := model["name"].(string)+ name = strings.TrimSpace(name)+ if name == "" {+ continue+ }+ list = append(list, models.ListModelResponse{+ Name: name,+ ModelTypes: providerModelMapTypes(model),+ })+ indexes = append(indexes, i)+ }+ list = models.FillMissingModelTypes(list)+ for i, model := range list {+ result[indexes[i]]["model_types"] = model.ModelTypes+ }+}++func providerModelMapTypes(model map[string]interface{}) []string {+ switch modelTypes := model["model_types"].(type) {+ case []string:+ return modelTypes+ case []interface{}:+ types := make([]string, 0, len(modelTypes))+ for _, modelType := range modelTypes {+ if s, ok := modelType.(string); ok {+ types = append(types, s)+ }+ }+ return types+ default:+ return nil+ }+}+func (h *ProviderHandler) ShowModel(c *gin.Context) {providerName := c.Param("provider_id_or_name")if providerName == "" {