Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Refactor: move cache to transaction (#21023)

Lynn57d6a9f86017
diff --git a/internal/agent/tool/retrieval_nlp.go b/internal/agent/tool/retrieval_nlp.go
index 5d75ea462a..03be424622 100644
--- a/internal/agent/tool/retrieval_nlp.go
+++ b/internal/agent/tool/retrieval_nlp.go
@@ -70,6 +70,7 @@ import (
"ragflow/internal/agent/runtime"
"ragflow/internal/common"
"ragflow/internal/dao"
+ "ragflow/internal/dao/transaction"
"ragflow/internal/engine"
"ragflow/internal/entity"
modelModule "ragflow/internal/entity/models"
@@ -484,10 +485,10 @@ func validateEmbeddingModels(ctx context.Context, db *gorm.DB, kbs []*entity.Kno
return fmt.Errorf("retrieval: dataset record is nil")
}
}
- embdNameCache := make(map[string]string)
- firstKey := knowledgebaseEmbeddingKey(ctx, db, kbs[0], embdNameCache)
+ embdNames := transaction.NewEmbeddingBaseNameResolver(dao.NewKnowledgebaseDAO().EmbeddingBaseName)
+ firstKey := knowledgebaseEmbeddingKey(ctx, db, kbs[0], embdNames)
for _, kb := range kbs[1:] {
- if knowledgebaseEmbeddingKey(ctx, db, kb, embdNameCache) != firstKey {
+ if knowledgebaseEmbeddingKey(ctx, db, kb, embdNames) != firstKey {
return fmt.Errorf("retrieval: datasets use different embedding models")
}
}
@@ -499,11 +500,11 @@ func validateEmbeddingModels(ctx context.Context, db *gorm.DB, kbs []*entity.Kno
// validation, so datasets pointing at the same model through different
// provider instances or storage forms (tenant_model id vs legacy composite
// name) retrieve together.
-func knowledgebaseEmbeddingKey(ctx context.Context, db *gorm.DB, kb *entity.Knowledgebase, cache map[string]string) string {
+func knowledgebaseEmbeddingKey(ctx context.Context, db *gorm.DB, kb *entity.Knowledgebase, names *transaction.EmbeddingBaseNameResolver) string {
if strings.TrimSpace(kb.EmbdID) == "" && (kb.TenantEmbdID == nil || strings.TrimSpace(*kb.TenantEmbdID) == "") {
return "default:" + strings.TrimSpace(kb.TenantID)
}
- return "embedding:" + dao.NewKnowledgebaseDAO().EmbeddingBaseName(ctx, db, kb, cache)
+ return "embedding:" + names.Resolve(ctx, db, kb)
}
func (a *NLPRetrievalAdapter) resolveEmbeddingModel(
diff --git a/internal/dao/kb.go b/internal/dao/kb.go
index 7783bde7ec..49e2d42396 100644
--- a/internal/dao/kb.go
+++ b/internal/dao/kb.go
@@ -22,6 +22,7 @@ import (
"fmt"
"path"
"ragflow/internal/common"
+ "ragflow/internal/dao/transaction"
"ragflow/internal/entity"
"strconv"
@@ -99,48 +100,30 @@ func NewKnowledgebaseDAO() *KnowledgebaseDAO {
return &KnowledgebaseDAO{}
}
-// EmbeddingBaseName resolves a knowledge base's embedding model reference to
-// the bare model name (e.g. "BAAI/bge-m3") used to decide whether datasets
-// can be selected and searched together. tenant_embd_id — or embd_id itself
-// when it stores a raw tenant_model id — is resolved through tenant_model;
-// legacy composite "model@instance@provider" values are reduced with
-// common.BaseModelName. An id that no longer resolves falls back to the
-// composite base name when embd_id holds one, otherwise to the id itself so
-// only exact matches group together.
-func (dao *KnowledgebaseDAO) EmbeddingBaseName(ctx context.Context, db *gorm.DB, kb *entity.Knowledgebase, cache map[string]string) string {
- raw := strings.TrimSpace(kb.EmbdID)
- id := ""
- if kb.TenantEmbdID != nil {
- id = strings.TrimSpace(*kb.TenantEmbdID)
- }
- if id == "" && raw != "" && !strings.Contains(raw, "@") {
- id = raw
- }
+// EmbeddingBaseName resolves the tenant_model reference of one knowledge base
+// to the bare model name (e.g. "BAAI/bge-m3") used to decide whether datasets
+// can be selected and searched together. Which column carries the reference is
+// a business rule answered by transaction.EmbeddingModelRef; what remains here
+// is the read itself, once per call, so callers resolving a batch of datasets
+// memoize through transaction.EmbeddingBaseNameResolver. An id that no longer
+// resolves falls back to the composite base name when embd_id holds one,
+// otherwise to the id itself so only exact matches group together.
+func (dao *KnowledgebaseDAO) EmbeddingBaseName(ctx context.Context, db *gorm.DB, kb *entity.Knowledgebase) string {
+ id, raw := transaction.EmbeddingModelRef(kb)
if id == "" {
return common.BaseModelName(raw)
}
- if cache != nil {
- if cached, ok := cache[id]; ok {
- return cached
- }
- }
- base := ""
if db != nil {
if model, err := NewTenantModelDAO().GetByID(ctx, db, id); err == nil && model != nil {
- base = strings.TrimSpace(model.ModelName)
- }
- }
- if base == "" {
- if raw != "" && raw != id {
- base = common.BaseModelName(raw)
- } else {
- base = id
+ if base := strings.TrimSpace(model.ModelName); base != "" {
+ return base
+ }
}
}
- if cache != nil {
- cache[id] = base
+ if raw != "" && raw != id {
+ return common.BaseModelName(raw)
}
- return base
+ return id
}
// Create creates a new knowledge base record
diff --git a/internal/dao/transaction/embedding_base_name.go b/internal/dao/transaction/embedding_base_name.go
new file mode 100644
index 0000000000..8e6beaeb70
--- /dev/null
+++ b/internal/dao/transaction/embedding_base_name.go
@@ -0,0 +1,84 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package transaction
+
+import (
+ "context"
+ "strings"
+
+ "ragflow/internal/common"
+ "ragflow/internal/entity"
+
+ "gorm.io/gorm"
+)
+
+// EmbeddingModelRef extracts the tenant_model reference a dataset's embedding
+// columns point at: tenant_embd_id, or embd_id itself when it stores a raw
+// tenant_model id rather than a legacy "model@instance@provider" composite.
+// raw is the trimmed embd_id, kept as the fallback once id no longer resolves.
+func EmbeddingModelRef(kb *entity.Knowledgebase) (id, raw string) {
+ raw = strings.TrimSpace(kb.EmbdID)
+ if kb.TenantEmbdID != nil {
+ id = strings.TrimSpace(*kb.TenantEmbdID)
+ }
+ if id == "" && raw != "" && !strings.Contains(raw, "@") {
+ id = raw
+ }
+ return id, raw
+}
+
+// EmbeddingBaseNameRead resolves one dataset's embedding reference to a bare
+// model name through tenant_model. It is the read the resolver below memoizes.
+type EmbeddingBaseNameRead func(ctx context.Context, db *gorm.DB, kb *entity.Knowledgebase) string
+
+// EmbeddingBaseNameResolver answers "do these datasets share one embedding
+// model?" for a batch with a single tenant_model read per distinct reference.
+// Datasets commonly repeat one reference, and a request mixing N datasets would
+// otherwise pay N identical reads only to compare answers that were equal all
+// along.
+//
+// A resolver belongs to one request: build one, use it for the batch, drop it.
+// Resolutions are not stable across requests — an upgraded tenant_model row
+// must reach the next request.
+type EmbeddingBaseNameResolver struct {
+ read EmbeddingBaseNameRead
+ names map[string]string
+}
+
+// NewEmbeddingBaseNameResolver builds a resolver that answers misses through
+// read.
+func NewEmbeddingBaseNameResolver(read EmbeddingBaseNameRead) *EmbeddingBaseNameResolver {
+ return &EmbeddingBaseNameResolver{read: read, names: make(map[string]string)}
+}
+
+// Resolve reports the bare embedding model name of kb (e.g. "BAAI/bge-m3"). A
+// reference already resolved in this batch answers from the cache, and datasets
+// carrying no tenant_model reference cost no read at all: their legacy
+// composite form reduces to a base name in memory, so the circuit opens before
+// a query is composed.
+func (r *EmbeddingBaseNameResolver) Resolve(ctx context.Context, db *gorm.DB, kb *entity.Knowledgebase) string {
+ id, raw := EmbeddingModelRef(kb)
+ if id == "" {
+ return common.BaseModelName(raw)
+ }
+ if cached, ok := r.names[id]; ok {
+ return cached
+ }
+ base := r.read(ctx, db, kb)
+ r.names[id] = base
+ return base
+}
diff --git a/internal/service/chat.go b/internal/service/chat.go
index 8ffe9ea023..fc58020eb0 100644
--- a/internal/service/chat.go
+++ b/internal/service/chat.go
@@ -29,6 +29,7 @@ import (
"unicode/utf8"
"ragflow/internal/dao"
+ "ragflow/internal/dao/transaction"
)
var DefaultRerankModels = map[string]struct{}{
@@ -1022,10 +1023,10 @@ func (s *ChatService) validateRESTDatasetIDs(ctx context.Context, value interfac
embeddingModelIDs := make([]string, 0, len(kbs))
seenEmbedIDs := make(map[string]struct{})
- embdNameCache := make(map[string]string)
+ embdNames := transaction.NewEmbeddingBaseNameResolver(s.kbDAO.EmbeddingBaseName)
for _, kb := range kbs {
embeddingModelIDs = append(embeddingModelIDs, kb.EmbdID)
- seenEmbedIDs[s.kbDAO.EmbeddingBaseName(ctx, dao.DB, kb, embdNameCache)] = struct{}{}
+ seenEmbedIDs[embdNames.Resolve(ctx, dao.DB, kb)] = struct{}{}
}
if len(seenEmbedIDs) > 1 {
return nil, fmt.Errorf("datasets use different embedding models: %v", embeddingModelIDs)
diff --git a/internal/service/chunk/chunk.go b/internal/service/chunk/chunk.go
index 283dcaecb9..ac7c8a97a3 100644
--- a/internal/service/chunk/chunk.go
+++ b/internal/service/chunk/chunk.go
@@ -43,6 +43,7 @@ import (
"gorm.io/gorm"
"ragflow/internal/dao"
+ "ragflow/internal/dao/transaction"
"ragflow/internal/engine"
"ragflow/internal/engine/types"
"ragflow/internal/ingestion/knowledge_compile"
@@ -191,10 +192,10 @@ func (s *ChunkService) RetrievalTest(ctx context.Context, req *service.Retrieval
// Check if all kbs resolve to the same base embedding model
if len(kbRecords) > 1 {
- embdNameCache := make(map[string]string)
- firstEmbeddingKey := knowledgebaseEmbeddingKey(ctx, dao.DB, kbRecords[0], tenantIDs[0], embdNameCache)
+ embdNames := transaction.NewEmbeddingBaseNameResolver(dao.NewKnowledgebaseDAO().EmbeddingBaseName)
+ firstEmbeddingKey := knowledgebaseEmbeddingKey(ctx, dao.DB, kbRecords[0], tenantIDs[0], embdNames)
for i := 1; i < len(kbRecords); i++ {
- if knowledgebaseEmbeddingKey(ctx, dao.DB, kbRecords[i], tenantIDs[i], embdNameCache) != firstEmbeddingKey {
+ if knowledgebaseEmbeddingKey(ctx, dao.DB, kbRecords[i], tenantIDs[i], embdNames) != firstEmbeddingKey {
return nil, fmt.Errorf("cannot retrieve across datasets with different embedding models")
}
}
@@ -497,11 +498,11 @@ func resolveSearchbotRetrievalParams(req *service.RetrievalTestRequest, searchCo
// model name (e.g. "BAAI/bge-m3") so datasets pointing at the same model
// through different provider instances or storage forms (tenant_model id vs
// legacy composite name) retrieve together.
-func knowledgebaseEmbeddingKey(ctx context.Context, db *gorm.DB, kb *entity.Knowledgebase, tenantID string, cache map[string]string) string {
+func knowledgebaseEmbeddingKey(ctx context.Context, db *gorm.DB, kb *entity.Knowledgebase, tenantID string, names *transaction.EmbeddingBaseNameResolver) string {
if strings.TrimSpace(kb.EmbdID) == "" && (kb.TenantEmbdID == nil || strings.TrimSpace(*kb.TenantEmbdID) == "") {
return fmt.Sprintf("default:%s", tenantID)
}
- return "embd:" + dao.NewKnowledgebaseDAO().EmbeddingBaseName(ctx, db, kb, cache)
+ return "embd:" + names.Resolve(ctx, db, kb)
}
// hydrateChunkVectors replaces zero (placeholder) vectors in chunks with real
diff --git a/internal/service/chunk/chunk_test.go b/internal/service/chunk/chunk_test.go
index 9d42607699..aef63eb980 100644
--- a/internal/service/chunk/chunk_test.go
+++ b/internal/service/chunk/chunk_test.go
@@ -12,6 +12,7 @@ import (
"image/png"
"ragflow/internal/common"
"ragflow/internal/dao"
+ "ragflow/internal/dao/transaction"
"ragflow/internal/engine/types"
"ragflow/internal/entity"
"ragflow/internal/entity/models"
@@ -152,7 +153,8 @@ func TestKnowledgebaseEmbeddingKey(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- if got := knowledgebaseEmbeddingKey(t.Context(), db, tt.kb, tt.tenantID, map[string]string{}); got != tt.want {
+ names := transaction.NewEmbeddingBaseNameResolver(dao.NewKnowledgebaseDAO().EmbeddingBaseName)
+ if got := knowledgebaseEmbeddingKey(t.Context(), db, tt.kb, tt.tenantID, names); got != tt.want {
t.Fatalf("knowledgebaseEmbeddingKey() = %q, want %q", got, tt.want)
}
})
diff --git a/internal/service/pipeline_params.go b/internal/service/pipeline_params.go
index 4adddf06ac..82b62e66dd 100644
--- a/internal/service/pipeline_params.go
+++ b/internal/service/pipeline_params.go
@@ -8,6 +8,7 @@ import (
"strings"
"ragflow/internal/dao"
+ "ragflow/internal/dao/transaction"
"ragflow/internal/entity"
pipelinepkg "ragflow/internal/ingestion/pipeline"
@@ -108,12 +109,11 @@ func ValidateDatasetEmbeddingModels(ctx context.Context, db *gorm.DB, kbs []*ent
embdNames := make(map[string]struct{})
hasEmbd := false
noEmbd := false
- cache := make(map[string]string)
- kbDAO := dao.NewKnowledgebaseDAO()
+ names := transaction.NewEmbeddingBaseNameResolver(dao.NewKnowledgebaseDAO().EmbeddingBaseName)
for _, kb := range kbs {
if kb.EmbdID != "" {
hasEmbd = true
- embdNames[kbDAO.EmbeddingBaseName(ctx, db, kb, cache)] = struct{}{}
+ embdNames[names.Resolve(ctx, db, kb)] = struct{}{}
} else {
noEmbd = true
}