Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(ingestion): converge global singletons into injected dependencies (#21203)

xugangqiangb4b850f6f58c
diff --git a/internal/handler/agent.go b/internal/handler/agent.go
index 8116ffff9e..ddb04e9301 100644
--- a/internal/handler/agent.go
+++ b/internal/handler/agent.go
@@ -35,7 +35,9 @@ import (
"ragflow/internal/agent/canvas"
"ragflow/internal/common"
"ragflow/internal/dao"
+ "ragflow/internal/engine"
"ragflow/internal/entity"
+ "ragflow/internal/ingestion"
pipelinepkg "ragflow/internal/ingestion/pipeline"
"ragflow/internal/ingestion/task"
"ragflow/internal/service"
@@ -114,7 +116,12 @@ func NewAgentHandler(ctx context.Context, agentService *service.AgentService, fi
redisGet: func(key string) (string, error) { return kvrocks.Get().Get(ctx, key) },
redisStore: kvrocks.Get(),
newExecutor: func(taskCtx *task.TaskContext, canvasID string, docBulkSize int) (debugExecutor, error) {
- return task.NewPipelineExecutor(taskCtx, canvasID, docBulkSize)
+ return task.NewPipelineExecutor(taskCtx, canvasID, docBulkSize, &ingestion.Deps{
+ Engine: engine.Get(),
+ DB: dao.DB,
+ MQ: engine.GetMessageQueueEngine(),
+ KV: kvrocks.Get(),
+ })
},
}
}
diff --git a/internal/handler/memory_message_test.go b/internal/handler/memory_message_test.go
index 7062996fe7..9f4f4b6388 100644
--- a/internal/handler/memory_message_test.go
+++ b/internal/handler/memory_message_test.go
@@ -10,6 +10,8 @@ import (
"github.com/gin-gonic/gin"
"ragflow/internal/common"
+ "ragflow/internal/dao"
+ "ragflow/internal/engine"
"ragflow/internal/entity"
"ragflow/internal/service"
)
@@ -67,7 +69,7 @@ func TestParseMemoryMessagePath(t *testing.T) {
func TestForgetMessageRejectsMalformedPath(t *testing.T) {
gin.SetMode(gin.TestMode)
router := gin.New()
- h := NewMemoryHandler(service.NewMemoryService())
+ h := NewMemoryHandler(service.NewMemoryService(dao.DB, engine.Get()))
router.DELETE("/api/v1/messages/:memory_message", func(c *gin.Context) {
c.Set("user", &entity.User{ID: "user-1"})
h.ForgetMessage(c)
diff --git a/internal/ingestion/arch_test.go b/internal/ingestion/arch_test.go
new file mode 100644
index 0000000000..9827682c23
--- /dev/null
+++ b/internal/ingestion/arch_test.go
@@ -0,0 +1,399 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package ingestion
+
+import (
+ "go/ast"
+ "go/parser"
+ "go/token"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+// forbiddenPackage describes one package whose process-wide singleton
+// accessors orchestration-layer production code must not reach for directly.
+// The singletons are resolved once at the server assembly points (cmd, server)
+// and passed down explicitly (see ingestion.Deps).
+type forbiddenPackage struct {
+ // importPath is the package path. A file only counts as accessing the
+ // global when it also imports this package, which keeps unrelated
+ // identifiers named "db" or "engine" from false-positiving.
+ importPath string
+ // pkgName is the default package identifier (last element of importPath).
+ pkgName string
+ // symbols are the package-level singleton accessors that are forbidden
+ // outside the assembly points.
+ symbols []string
+}
+
+var forbiddenPackages = []forbiddenPackage{
+ {importPath: "ragflow/internal/dao", pkgName: "dao", symbols: []string{"DB"}},
+ {importPath: "ragflow/internal/engine", pkgName: "engine", symbols: []string{"Get*"}},
+ {importPath: "ragflow/internal/engine/kvrocks", pkgName: "kvrocks", symbols: []string{"Get"}},
+}
+
+// globalAccessWhitelist lists the few production files that legitimately
+// resolve a process-wide singleton. Each entry carries the justification;
+// new entries require reviewer sign-off.
+var globalAccessWhitelist = map[string]string{
+ // Client() is the sanctioned kvrocks resolver for the chunkcache.Store
+ // seam: it wraps kvrocks.Get() with typed-nil protection so callers can
+ // pass the result into Store parameters. Components reach it through the
+ // canvas runtime, which does not carry a kvrocks handle.
+ "internal/ingestion/chunkcache/chunkcache.go": "Client() is the sanctioned kvrocks resolver for the chunkcache.Store seam",
+ // MarkWikiDocumentDirty is a package-level API consumed by out-of-scope
+ // service packages (internal/service/document, internal/service/chunk)
+ // that run in the API server process, where Provision has not installed
+ // kcDB. The explicit kcDB (installed by Provision with an injected db) is
+ // preferred; dao.DB is the API-process fallback.
+ "internal/ingestion/knowledge_compile/wiki_dirty.go": "package-level API for out-of-scope callers; kcDB is the explicit Provision-installed dependency",
+ // testutil installs in-memory test doubles by swapping the process-wide
+ // singletons (ReplaceDBForTest). It is the sanctioned test seam, the only
+ // place besides the server assembly points that may touch the globals.
+ "internal/ingestion/testutil/helpers.go": "test-seam installer: swaps the process-wide singletons for in-memory doubles",
+}
+
+// scanRoots are the directory trees this guard covers: the ingestion
+// orchestration layers plus the Go memory service files. The wider
+// internal/service package is deliberately out of scope for this guard; it
+// converges separately.
+var scanRoots = []string{
+ "internal/ingestion",
+ "internal/service",
+}
+
+// memoryFilePrefix scopes the internal/service scan to the memory module
+// files only.
+const memoryFilePrefix = "memory"
+
+// findModuleRoot walks up from the working directory until it finds go.mod.
+func findModuleRoot(t *testing.T) string {
+ t.Helper()
+ dir, err := os.Getwd()
+ if err != nil {
+ t.Fatalf("getwd: %v", err)
+ }
+ for {
+ if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {
+ return dir
+ }
+ parent := filepath.Dir(dir)
+ if parent == dir {
+ t.Fatalf("go.mod not found above %s", dir)
+ }
+ dir = parent
+ }
+}
+
+// importedName maps an import spec to the identifier the file uses for it,
+// honouring explicit aliases (import kv "ragflow/internal/engine/kvrocks").
+// A blank or dot import cannot produce pkg.Symbol selectors and is skipped.
+func importedName(spec *ast.ImportSpec, fallback string) (string, bool) {
+ if spec.Name != nil {
+ switch spec.Name.Name {
+ case "_", ".":
+ return "", false
+ }
+ return spec.Name.Name, true
+ }
+ return fallback, true
+}
+
+// fileForbiddenHits returns the forbidden global accesses in one parsed file.
+// Detection is AST-based: only real selector expressions count, so comments
+// and string literals that merely mention e.g. "engine.Get()" are ignored.
+func fileForbiddenHits(fset *token.FileSet, file *ast.File) []string {
+ // Map each locally-used package identifier to the forbidden package it
+ // belongs to. Keying by the local name (alias or default) keeps aliased
+ // imports detectable while unrelated identifiers named "db" or "engine"
+ // in files that do not import the package are ignored.
+ imported := map[string]forbiddenPackage{}
+ for _, imp := range file.Imports {
+ path := strings.Trim(imp.Path.Value, `"`)
+ for _, fp := range forbiddenPackages {
+ if path != fp.importPath {
+ continue
+ }
+ name, ok := importedName(imp, fp.pkgName)
+ if !ok {
+ continue
+ }
+ imported[name] = fp
+ }
+ }
+ if len(imported) == 0 {
+ return nil
+ }
+
+ var hits []string
+ ast.Inspect(file, func(n ast.Node) bool {
+ sel, ok := n.(*ast.SelectorExpr)
+ if !ok {
+ return true
+ }
+ ident, ok := sel.X.(*ast.Ident)
+ if !ok {
+ return true
+ }
+ fp, ok := imported[ident.Name]
+ if !ok {
+ return true
+ }
+ for _, sym := range fp.symbols {
+ switch {
+ case strings.HasSuffix(sym, "*"):
+ // A trailing "*" matches every selector beginning with the
+ // prefix, so "engine.Get*" forbids engine.Get / GetMessageQueueEngine /
+ // GetEngineType — any process-wide engine singleton accessor.
+ if strings.HasPrefix(sel.Sel.Name, strings.TrimSuffix(sym, "*")) {
+ hits = append(hits, fp.pkgName+"."+sel.Sel.Name)
+ }
+ case sel.Sel.Name == sym:
+ hits = append(hits, fp.pkgName+"."+sym)
+ }
+ }
+ return true
+ })
+ return hits
+}
+
+// parseSource detects forbidden accesses in an in-memory source snippet. It
+// backs the detector self-tests.
+func parseSource(t *testing.T, src string) []string {
+ t.Helper()
+ fset := token.NewFileSet()
+ file, err := parser.ParseFile(fset, "snippet.go", src, parser.ParseComments)
+ if err != nil {
+ t.Fatalf("parse snippet: %v", err)
+ }
+ return fileForbiddenHits(fset, file)
+}
+
+func TestForbiddenGlobalDetectorIgnoresCommentsAndStrings(t *testing.T) {
+ src := `package snippet
+
+import (
+ "ragflow/internal/dao"
+ "ragflow/internal/engine"
+)
+
+// A comment mentioning dao.DB and engine.Get() must not count.
+const note = "dao.DB and kvrocks.Get() in a string literal must not count either"
+
+func f() {
+ _ = note
+ _ = dao.NewDocumentDAO
+ _ = engine.EngineInfinity
+}
+`
+ if hits := parseSource(t, src); len(hits) != 0 {
+ t.Fatalf("comments/strings/unrelated selectors flagged as violations: %v", hits)
+ }
+}
+
+func TestForbiddenGlobalDetectorFlagsRealAccesses(t *testing.T) {
+ cases := []struct {
+ name string
+ src string
+ want string
+ }{
+ {
+ name: "dao.DB passed as an argument",
+ src: `package snippet
+
+import "ragflow/internal/dao"
+
+func f() { _ = dao.NewTaskDAO().GetByID(nil, dao.DB, "id") }
+`,
+ want: "dao.DB",
+ },
+ {
+ name: "engine.Get assigned",
+ src: `package snippet
+
+import "ragflow/internal/engine"
+
+func f() { eng := engine.Get(); _ = eng }
+`,
+ want: "engine.Get",
+ },
+ {
+ name: "kvrocks.Get nil check",
+ src: `package snippet
+
+import "ragflow/internal/engine/kvrocks"
+
+func f() { if kvrocks.Get() != nil { _ = 1 } }
+`,
+ want: "kvrocks.Get",
+ },
+ {
+ name: "aliased import still detected",
+ src: `package snippet
+
+import eng "ragflow/internal/engine"
+
+func f() { _ = eng.GetMessageQueueEngine() }
+`,
+ want: "engine.GetMessageQueueEngine",
+ },
+ {
+ name: "engine.Get* prefix catches GetEngineType",
+ src: `package snippet
+
+import "ragflow/internal/engine"
+
+func f() { _ = engine.GetEngineType() }
+`,
+ want: "engine.GetEngineType",
+ },
+ {
+ name: "unrelated identifier named engine is not a violation",
+ src: `package snippet
+
+type engine struct{ Get int }
+
+func f() { _ = engine{}.Get }
+`,
+ want: "",
+ },
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ hits := parseSource(t, tc.src)
+ if tc.want == "" {
+ if len(hits) != 0 {
+ t.Fatalf("unexpected violations: %v", hits)
+ }
+ return
+ }
+ if len(hits) != 1 || hits[0] != tc.want {
+ t.Fatalf("hits = %v, want exactly [%s]", hits, tc.want)
+ }
+ })
+ }
+}
+
+func TestForbiddenGlobalDetectorSkipsTestFilesAndWhitelist(t *testing.T) {
+ // The detector is fed by the walker, so verify the walker's own rules:
+ // _test.go files are skipped, whitelisted files are skipped, and files
+ // under internal/service that are not memory files are skipped.
+ root := findModuleRoot(t)
+
+ scanned, skipped := 0, 0
+ for _, scanRoot := range scanRoots {
+ err := filepath.Walk(filepath.Join(root, scanRoot), func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ if info.IsDir() {
+ return nil
+ }
+ rel, err := filepath.Rel(root, path)
+ if err != nil {
+ return err
+ }
+ rel = filepath.ToSlash(rel)
+ if !strings.HasSuffix(rel, ".go") {
+ return nil
+ }
+ switch {
+ case strings.HasSuffix(rel, "_test.go"):
+ skipped++
+ return nil
+ case globalAccessWhitelist[rel] != "":
+ skipped++
+ return nil
+ case scanRoot == "internal/service" && !strings.HasPrefix(filepath.Base(rel), memoryFilePrefix):
+ skipped++
+ return nil
+ }
+ scanned++
+ return nil
+ })
+ if err != nil {
+ t.Fatalf("walk %s: %v", scanRoot, err)
+ }
+ }
+ if scanned == 0 {
+ t.Fatal("walker selected no files to scan; the guard would pass vacuously")
+ }
+ if skipped == 0 {
+ t.Fatal("walker skipped nothing; test-file/whitelist rules are not exercised")
+ }
+}
+
+// TestProductionCodeDoesNotAccessGlobalSingletons is the architecture guard:
+// ingestion orchestration layers and the memory module must resolve their
+// dependencies through the injected ingestion.Deps bundle (and the memory
+// service's constructor parameters), never through process-wide singletons.
+// The singletons are captured once at the server assembly points.
+func TestProductionCodeDoesNotAccessGlobalSingletons(t *testing.T) {
+ root := findModuleRoot(t)
+
+ var violations []string
+ scanned := 0
+ for _, scanRoot := range scanRoots {
+ err := filepath.Walk(filepath.Join(root, scanRoot), func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ if info.IsDir() {
+ return nil
+ }
+ rel, err := filepath.Rel(root, path)
+ if err != nil {
+ return err
+ }
+ rel = filepath.ToSlash(rel)
+ if !strings.HasSuffix(rel, ".go") || strings.HasSuffix(rel, "_test.go") {
+ return nil
+ }
+ if reason := globalAccessWhitelist[rel]; reason != "" {
+ return nil
+ }
+ if scanRoot == "internal/service" && !strings.HasPrefix(filepath.Base(rel), memoryFilePrefix) {
+ return nil
+ }
+ scanned++
+
+ fset := token.NewFileSet()
+ file, err := parser.ParseFile(fset, path, nil, parser.ParseComments)
+ if err != nil {
+ return err
+ }
+ for _, hit := range fileForbiddenHits(fset, file) {
+ violations = append(violations, rel+": "+hit)
+ }
+ return nil
+ })
+ if err != nil {
+ t.Fatalf("walk %s: %v", scanRoot, err)
+ }
+ }
+
+ if scanned == 0 {
+ t.Fatal("guard scanned no files; the guard would pass vacuously")
+ }
+ if len(violations) > 0 {
+ t.Fatalf("%d file(s) in the ingestion/memory orchestration layers still resolve process-wide singletons directly (%d files scanned):\n%s\n\n"+
+ "Resolve these through the injected ingestion.Deps bundle at the server assembly points instead.",
+ len(violations), scanned, strings.Join(violations, "\n"))
+ }
+}
diff --git a/internal/ingestion/component/document_storage.go b/internal/ingestion/component/document_storage.go
index cff4610c47..d3e6e896ab 100644
--- a/internal/ingestion/component/document_storage.go
+++ b/internal/ingestion/component/document_storage.go
@@ -108,7 +108,7 @@ func ResolveDocumentStorage(ctx context.Context, db *gorm.DB, docID string) (*Do
return ref, nil
}
-func resolveDocumentName(ctx context.Context, docID string) (string, error) {
+func resolveDocumentName(ctx context.Context, db *gorm.DB, docID string) (string, error) {
if ResolveDocumentStorageOverride != nil {
ref, err := ResolveDocumentStorageOverride(docID)
if err != nil {
@@ -118,7 +118,7 @@ func resolveDocumentName(ctx context.Context, docID string) (string, error) {
return ref.Name, nil
}
}
- doc, err := dao.NewDocumentDAO().GetByID(ctx, dao.DB, docID)
+ doc, err := dao.NewDocumentDAO().GetByID(ctx, db, docID)
if err != nil {
return "", err
}
diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.go
index bb430661af..4e3e76aa34 100644
--- a/internal/ingestion/component/extractor.go
+++ b/internal/ingestion/component/extractor.go
@@ -1652,9 +1652,6 @@ func extractorContextLength(ctx context.Context, db *gorm.DB, llmID string) int
if fn := getExtractorContextLengthOverride(); fn != nil {
return fn(ctx, llmID)
}
- if db == nil {
- db = dao.DB
- }
state, err := runtime.GetStateFromContext(ctx)
if err != nil || state == nil {
return 0
diff --git a/internal/ingestion/component/extractor_tag.go b/internal/ingestion/component/extractor_tag.go
index 807ac2dfaf..f2a0c8d75b 100644
--- a/internal/ingestion/component/extractor_tag.go
+++ b/internal/ingestion/component/extractor_tag.go
@@ -376,7 +376,7 @@ func (c *ExtractorComponent) runAutoTags(ctx context.Context, db *gorm.DB, in ex
lang = detectTextLanguage(in.chunks)
}
- indexed, ok := c.resolveTagSource(ctx, lang)
+ indexed, ok := c.resolveTagSource(ctx, db, lang)
if !ok || len(in.chunks) == 0 {
common.Info("extractor tags: skipped",
zap.Int("chunk_count", len(in.chunks)),
@@ -454,19 +454,19 @@ func (c *ExtractorComponent) runAutoTags(ctx context.Context, db *gorm.DB, in ex
return in.chunks, nil
}
-func (c *ExtractorComponent) resolveTagSource(ctx context.Context, lang string) (*MemoryTagIndex, bool) {
+func (c *ExtractorComponent) resolveTagSource(ctx context.Context, db *gorm.DB, lang string) (*MemoryTagIndex, bool) {
if c.Param.Tags.TagFileID == "" {
return nil, false
}
- return c.loadTagFileIndexed(ctx, lang)
+ return c.loadTagFileIndexed(ctx, db, lang)
}
-func (c *ExtractorComponent) loadTagFileIndexed(ctx context.Context, lang string) (*MemoryTagIndex, bool) {
+func (c *ExtractorComponent) loadTagFileIndexed(ctx context.Context, db *gorm.DB, lang string) (*MemoryTagIndex, bool) {
// Same IDOR guard as TagVocabularyFromTagFileID: tag_file_id is
// user-controlled through parser_config, so it must resolve inside the
// caller's tenant before the bytes are read.
tenantID := globals.GlobalOrInput(ctx, nil, "tenant_id", "")
- f, err := resolveTagSourceFile(ctx, c.Param.Tags.TagFileID, tenantID)
+ f, err := resolveTagSourceFile(ctx, db, c.Param.Tags.TagFileID, tenantID)
if err != nil {
common.Warn(fmt.Sprintf("extractor tags: resolve tag_file_id %q: %v", c.Param.Tags.TagFileID, err))
return nil, false
diff --git a/internal/ingestion/component/extractor_tag_test.go b/internal/ingestion/component/extractor_tag_test.go
index bd6e37ee7b..7f8c231afc 100644
--- a/internal/ingestion/component/extractor_tag_test.go
+++ b/internal/ingestion/component/extractor_tag_test.go
@@ -13,6 +13,7 @@ import (
"ragflow/internal/agent/runtime"
"ragflow/internal/common"
+ "ragflow/internal/dao"
"ragflow/internal/entity"
"ragflow/internal/ingestion/component/schema"
"ragflow/internal/tokenizer"
@@ -208,7 +209,7 @@ func TestTagVocabularyFromTagFileID_RejectsForeignTenantFile(t *testing.T) {
// A dataset whose tenant does not own the file must be denied, and must not
// get the bytes.
- vocab, err := TagVocabularyFromTagFileID(t.Context(), fileID, attackerTenant)
+ vocab, err := TagVocabularyFromTagFileID(t.Context(), dao.DB, fileID, attackerTenant)
if err == nil {
t.Fatalf("IDOR: foreign tenant read the tag source file, vocab=%v", vocab)
}
@@ -217,12 +218,12 @@ func TestTagVocabularyFromTagFileID_RejectsForeignTenantFile(t *testing.T) {
}
// An empty owner tenant fails closed as well.
- if _, err := TagVocabularyFromTagFileID(t.Context(), fileID, ""); err == nil {
+ if _, err := TagVocabularyFromTagFileID(t.Context(), dao.DB, fileID, ""); err == nil {
t.Fatal("empty owner tenant must fail closed")
}
// The owning tenant still gets the vocabulary.
- vocab, err = TagVocabularyFromTagFileID(t.Context(), fileID, ownerTenant)
+ vocab, err = TagVocabularyFromTagFileID(t.Context(), dao.DB, fileID, ownerTenant)
if err != nil {
t.Fatalf("owner tenant load failed: %v", err)
}
@@ -234,7 +235,7 @@ func TestTagVocabularyFromTagFileID_RejectsForeignTenantFile(t *testing.T) {
// file's ParentID is never compared against the referencing dataset), so any
// number of datasets in ownerTenant can point at this same tag source file —
// each passes its own dataset tenant (kb.TenantID) here.
- shared, err := TagVocabularyFromTagFileID(t.Context(), fileID, ownerTenant)
+ shared, err := TagVocabularyFromTagFileID(t.Context(), dao.DB, fileID, ownerTenant)
if err != nil {
t.Fatalf("a second dataset in the same tenant must resolve the shared file: %v", err)
}
diff --git a/internal/ingestion/component/file.go b/internal/ingestion/component/file.go
index fe88499010..bbadda72a8 100644
--- a/internal/ingestion/component/file.go
+++ b/internal/ingestion/component/file.go
@@ -133,7 +133,7 @@ func (c *FileComponent) Outputs() map[string]string {
func (c *FileComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {
// Parse the wire input through the schema type so the
// validation errors match the package convention.
- in, err := parseFileInputs(ctx, inputs)
+ in, err := parseFileInputs(ctx, db, inputs)
if err != nil {
return nil, err
}
@@ -183,7 +183,7 @@ type fileInputs struct {
// parseFileInputs parses and validates the upstream input map.
// Mirrors python's branching on `self._canvas._doc_id` vs.
// `kwargs.get("file")[0]`.
-func parseFileInputs(ctx context.Context, inputs map[string]any) (fileInputs, error) {
+func parseFileInputs(ctx context.Context, db *gorm.DB, inputs map[string]any) (fileInputs, error) {
if inputs == nil {
return fileInputs{}, fmt.Errorf("file: inputs map is nil")
}
@@ -253,7 +253,7 @@ func parseFileInputs(ctx context.Context, inputs map[string]any) (fileInputs, er
out.path = v
}
if out.docID != "" {
- name, err := resolveDocumentName(ctx, out.docID)
+ name, err := resolveDocumentName(ctx, db, out.docID)
if err != nil {
return fileInputs{}, fmt.Errorf("file: resolve doc_id %q: %w", out.docID, err)
}
diff --git a/internal/ingestion/component/tag_query.go b/internal/ingestion/component/tag_query.go
index 778adecdb8..14737f2386 100644
--- a/internal/ingestion/component/tag_query.go
+++ b/internal/ingestion/component/tag_query.go
@@ -25,6 +25,8 @@ import (
"ragflow/internal/entity"
"ragflow/internal/ingestion/component/schema"
"ragflow/internal/tokenizer"
+
+ "gorm.io/gorm"
)
const (
@@ -154,14 +156,14 @@ type resolvedTagSource struct {
// all_tags_in_portion / tag_query: Go has no tag dataset, so the records come
// from the tag source file. Returns (nil, nil) when refs is empty or yields no
// usable records.
-func LoadTagQueryIndex(ctx context.Context, refs []TagSourceRef) (*MemoryTagIndex, error) {
+func LoadTagQueryIndex(ctx context.Context, db *gorm.DB, refs []TagSourceRef) (*MemoryTagIndex, error) {
resolved := make([]resolvedTagSource, 0, len(refs))
tokens := make([]string, 0, len(refs))
for _, ref := range refs {
if ref.FileID == "" {
continue
}
- f, err := resolveTagSourceFile(ctx, ref.FileID, ref.TenantID)
+ f, err := resolveTagSourceFile(ctx, db, ref.FileID, ref.TenantID)
if err != nil {
return nil, err
}
diff --git a/internal/ingestion/component/tag_source.go b/internal/ingestion/component/tag_source.go
index 0d0a78a4d4..f157a3e1f2 100644
--- a/internal/ingestion/component/tag_source.go
+++ b/internal/ingestion/component/tag_source.go
@@ -51,6 +51,8 @@ import (
"ragflow/internal/entity"
"ragflow/internal/ingestion/component/schema"
"ragflow/internal/tokenizer"
+
+ "gorm.io/gorm"
)
// bgSmoothing is the Dirichlet smoothing constant for background prior tag probabilities.
@@ -663,8 +665,8 @@ func tagVocabularyFromLabels(labels []schema.TagLabel) map[string]int {
// be proven to belong to that tenant before its bytes are read (IDOR, CWE-639).
// An empty ownerTenantID fails closed. Note that storage.Get's tenant argument
// does not establish file ownership by itself.
-func readTagSourceFile(ctx context.Context, tagFileID, ownerTenantID string) (*entity.File, []byte, error) {
- f, err := resolveTagSourceFile(ctx, tagFileID, ownerTenantID)
+func readTagSourceFile(ctx context.Context, db *gorm.DB, tagFileID, ownerTenantID string) (*entity.File, []byte, error) {
+ f, err := resolveTagSourceFile(ctx, db, tagFileID, ownerTenantID)
if err != nil {
return nil, nil, err
}
@@ -757,13 +759,13 @@ func IsTagSourceNotFound(err error) bool {
// validates that it is a readable stored object. Resolved rows are reused for
// tagSourceFileRowTTL so repeated lookups of the same file do not each cost a
// query.
-func resolveTagSourceFile(ctx context.Context, tagFileID, ownerTenantID string) (*entity.File, error) {
+func resolveTagSourceFile(ctx context.Context, db *gorm.DB, tagFileID, ownerTenantID string) (*entity.File, error) {
key := tagFileID + "\x00" + ownerTenantID
now := time.Now()
if cached, ok := tagSourceFileRows.load(key, now); ok {
return cached, nil
}
- f, err := dao.NewFileDAO().GetByIDAndTenant(ctx, dao.DB, tagFileID, ownerTenantID)
+ f, err := dao.NewFileDAO().GetByIDAndTenant(ctx, db, tagFileID, ownerTenantID)
if err != nil {
// The DAO returns database errors unchanged, and only a genuine miss
// means this id is dead: wrapping a transient failure as well would
@@ -802,12 +804,12 @@ func fetchTagSourceBytes(ctx context.Context, f *entity.File, ownerTenantID stri
// selectable-tag list surfaced by the tag-options/aggregation API.
//
// It returns (nil, nil) when tagFileID is empty.
-func TagVocabularyFromTagFileID(ctx context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {
+func TagVocabularyFromTagFileID(ctx context.Context, db *gorm.DB, tagFileID, ownerTenantID string) (map[string]int, error) {
if tagFileID == "" {
return nil, nil
}
common.Info(fmt.Sprintf("tag_vocab: loading tag source file_id=%q", tagFileID))
- f, data, err := readTagSourceFile(ctx, tagFileID, ownerTenantID)
+ f, data, err := readTagSourceFile(ctx, db, tagFileID, ownerTenantID)
if err != nil {
return nil, err
}
diff --git a/internal/ingestion/component/tag_source_test.go b/internal/ingestion/component/tag_source_test.go
index 7795bb9e9e..2520e997f9 100644
--- a/internal/ingestion/component/tag_source_test.go
+++ b/internal/ingestion/component/tag_source_test.go
@@ -128,7 +128,7 @@ func TestIsTagSourceNotFoundExcludesDatabaseFailures(t *testing.T) {
t.Fatalf("close: %v", err)
}
- _, err = resolveTagSourceFile(context.Background(), "file-transient-db", "tenant-1")
+ _, err = resolveTagSourceFile(context.Background(), dao.DB, "file-transient-db", "tenant-1")
if err == nil {
t.Fatal("expected the query against a closed database to fail")
}
diff --git a/internal/ingestion/deps.go b/internal/ingestion/deps.go
new file mode 100644
index 0000000000..fcc9400ed2
--- /dev/null
+++ b/internal/ingestion/deps.go
@@ -0,0 +1,78 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+// Package ingestion holds the dependency bundle shared by the ingestion
+// orchestration layers (service, task, pipeline, knowledge_compile, component).
+//
+// The process-wide singletons (doc engine, database handle, message-queue
+// engine, kvrocks client) are resolved exactly once, at the server assembly
+// points (internal/server/api, internal/server/ingestor), and passed down
+// explicitly through Deps. Orchestration code must never call engine.Get(),
+// dao.DB, or kvrocks.Get() itself; arch_test.go guards that rule.
+package ingestion
+
+import (
+ "fmt"
+
+ "ragflow/internal/engine"
+ kvrocks "ragflow/internal/engine/kvrocks"
+
+ "gorm.io/gorm"
+)
+
+// Deps bundles the process-wide singletons an ingestion run needs. It is
+// constructed once per process at the assembly point and threaded through the
+// Ingestor, TaskContext, PipelineExecutor, and the knowledge-compile Consumer.
+//
+// KV is optional: a nil client disables the chunkcache and the memory id
+// generator, which fall back to their documented no-cache / local-id paths.
+// Engine, DB, and MQ are required for a production run; Validate reports the
+// missing ones at startup instead of failing mid-task.
+type Deps struct {
+ // Engine persists and searches chunks (document index, memory messages,
+ // compiled knowledge products).
+ Engine engine.DocEngine
+ // DB is the relational handle. DAO methods take it as a parameter so
+ // callers can substitute a transaction; the bundle value is the default.
+ DB *gorm.DB
+ // MQ is the task broker (NATS JetStream) the ingestor consumes.
+ MQ engine.MessageQueue
+ // KV is the kvrocks client (chunkcache, run trackers, memory ids). nil
+ // disables those optional accelerators.
+ KV *kvrocks.Client
+}
+
+// Validate reports the required dependencies that are missing. KV is allowed
+// to be nil (optional accelerator); Engine, DB, and MQ are not.
+func (d *Deps) Validate() error {
+ if d == nil {
+ return fmt.Errorf("ingestion deps: nil dependency bundle")
+ }
+ var missing []string
+ if d.Engine == nil {
+ missing = append(missing, "doc engine")
+ }
+ if d.DB == nil {
+ missing = append(missing, "database")
+ }
+ if d.MQ == nil {
+ missing = append(missing, "message queue")
+ }
+ if len(missing) > 0 {
+ return fmt.Errorf("ingestion deps: missing required dependencies: %v", missing)
+ }
+ return nil
+}
diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.go
index e7c2fc259e..2a2ee30522 100644
--- a/internal/ingestion/knowledge_compile/consumer.go
+++ b/internal/ingestion/knowledge_compile/consumer.go
@@ -37,10 +37,13 @@ import (
"gorm.io/gorm"
)
-// kcDB is the package-level MySQL handle installed by Provision. It backs the
-// doc enumeration used by RebuildDataset; it is nil (and docLister returns no
-// docs) when the scheduler was provisioned without a DB.
-var kcDB *gorm.DB
+// errEngineNotConfigured is returned by the engine-backed reader, writer, and
+// contribution store when they were assembled without a doc engine. The
+// consumer's storage dependencies are injected by the server assembly point
+// (see WithDocEngine); an unconfigured consumer fails loudly on its first
+// storage access instead of silently reading zero products or acking writes
+// that never happened.
+var errEngineNotConfigured = errors.New("knowledge_compile: doc engine is not configured; wire ingestion.Deps.Engine at the server assembly point")
// Consumer is the dataset-level post-processing worker (§11.5). Multiple
// instances compete on the MySQL scheduling rows; each KB is processed by at
@@ -55,6 +58,13 @@ type Consumer struct {
factory DeduperFactory
contributions wikiContributionStore
+ // storageConfigured is set by WithStorage and records that the consumer was
+ // assembled with an injected doc engine + db. An unconfigured consumer fails
+ // fast on its first storage access (errEngineNotConfigured); callers can
+ // assert this flag before relying on merges so a wiring regression is caught
+ // at assembly rather than as a silently-acked empty batch.
+ storageConfigured bool
+
ttl time.Duration
heartbeat time.Duration
pollInterval time.Duration
@@ -67,6 +77,18 @@ type Consumer struct {
// rebuildPause suppresses local claims while this consumer rebuilds a dataset.
rebuildPause map[string]bool
+ // db is the injected MySQL handle (wired by WithStorage). It backs the doc
+ // enumeration used by RebuildDataset and the dataset compile log. A consumer
+ // assembled without WithStorage has a nil db and its storage paths no-op,
+ // which decouples the consumer from Provision's call ordering and from the
+ // package-level kcDB global.
+ db *gorm.DB
+
+ // docEngine is the injected doc engine (wired by WithStorage). It is kept on
+ // the Consumer (mirroring db) so the assembly point can assert the exact
+ // engine reached the reader/writer, not just that WithStorage ran.
+ docEngine engine.DocEngine
+
// docLister enumerates every doc id in a KB so a rewrite can republish them.
docLister func(ctx context.Context, tenant, kb string) ([]string, error)
}
@@ -81,13 +103,18 @@ type rewriteScheduler interface {
// NewConsumer constructs a Consumer driven by the given Claimer. Tests pass a
// FakeScheduler and override the Reader/Writer/Deduper via options.
+//
+// The default reader, writer, and contribution store are unconfigured: the
+// production assembly point passes WithDocEngine(ingestion.Deps.Engine). A
+// consumer that reaches storage without it fails fast (errEngineNotConfigured)
+// rather than silently processing an empty dataset.
func NewConsumer(scheduler Claimer, opts ...Option) *Consumer {
c := &Consumer{
scheduler: scheduler,
- reader: engineReader{eng: engine.Get()},
- writer: engineWriter{eng: engine.Get(), commitService: file.NewFileCommitService(), resolveLanguage: datasetLanguageByID},
+ reader: engineReader{},
+ writer: engineWriter{commitService: file.NewFileCommitService()},
factory: defaultDeduperFactory,
- contributions: newWikiContributionStore(engine.Get()),
+ contributions: newWikiContributionStore(nil),
ttl: 2 * time.Minute,
heartbeat: 20 * time.Second,
pollInterval: 2 * time.Second,
@@ -95,7 +122,11 @@ func NewConsumer(scheduler Claimer, opts ...Option) *Consumer {
mergeThreshold: 0.85,
tombs: map[string]map[string]uint64{},
rebuildPause: map[string]bool{},
- docLister: defaultDocLister,
+ }
+ // docLister reads the injected db at call time (see defaultDocLister), so it
+ // reflects whatever WithStorage configured after construction.
+ c.docLister = func(ctx context.Context, tenant, kb string) ([]string, error) {
+ return c.defaultDocLister(ctx, tenant, kb)
}
for _, o := range opts {
o(c)
@@ -103,6 +134,29 @@ func NewConsumer(scheduler Claimer, opts ...Option) *Consumer {
return c
}
+// IsStorageConfigured reports whether the consumer was assembled with a doc
+// engine + db via WithStorage. The production ingestor must build it that way;
+// an unconfigured consumer fails fast on storage access, so this flag lets the
+// assembly point assert wiring before any batch is processed.
+func (c *Consumer) IsStorageConfigured() bool {
+ return c.storageConfigured
+}
+
+// StorageEngine returns the doc engine injected via WithStorage. It is exposed
+// so the assembly point can assert the exact engine the ingestor threads
+// through (not a zero value) actually reached the reader/writer — IsStorageConfigured
+// alone only reports that WithStorage ran, not that the deps propagated.
+func (c *Consumer) StorageEngine() engine.DocEngine {
+ return c.docEngine
+}
+
+// StorageDB returns the database handle injected via WithStorage. See
+// StorageEngine for why the injected value (not just the configured flag) is
+// observable.
+func (c *Consumer) StorageDB() *gorm.DB {
+ return c.db
+}
+
// Run is one owned worker loop (Option E §11.5/§11.7): it wakes on NATS notify
// and otherwise polls the scheduling table for a claimable KB, then claims the
// closed batch, processes it, and acks. It returns when ctx is cancelled.
@@ -173,7 +227,7 @@ func (c *Consumer) processClaim(ctx context.Context, cr ClaimResult) {
return
}
logTaskTypes := taskTypesForEntries(cr.Entries)
- if err := startDatasetCompileLog(ctx, cr.TenantID, datasetID, cr.Token, cr.Entries); err != nil {
+ if err := c.startDatasetCompileLog(ctx, cr.TenantID, datasetID, cr.Token, cr.Entries); err != nil {
common.Warn("knowledge_compile: failed to create dataset ingestion log",
zap.String("dataset_id", datasetID), zap.Error(err))
}
@@ -215,7 +269,7 @@ func (c *Consumer) processClaim(ctx context.Context, cr ClaimResult) {
case <-ctx.Done():
close(stopHb)
<-done
- if err := finishDatasetCompileLog(context.WithoutCancel(ctx), cr.Token, logTaskTypes, common.STOPPED, "Knowledge compilation stopped during shutdown", -1); err != nil {
+ if err := c.finishDatasetCompileLog(context.WithoutCancel(ctx), cr.Token, logTaskTypes, common.STOPPED, "Knowledge compilation stopped during shutdown", -1); err != nil {
common.Warn("knowledge_compile: failed to finalize stopped dataset ingestion log",
zap.String("dataset_id", datasetID), zap.Error(err))
}
@@ -223,7 +277,7 @@ func (c *Consumer) processClaim(ctx context.Context, cr ClaimResult) {
case <-hbFailed:
close(stopHb)
<-done
- if err := finishDatasetCompileLog(context.WithoutCancel(ctx), cr.Token, logTaskTypes, common.STOPPED, "Knowledge compilation lease was lost and will be retried", -1); err != nil {
+ if err := c.finishDatasetCompileLog(context.WithoutCancel(ctx), cr.Token, logTaskTypes, common.STOPPED, "Knowledge compilation lease was lost and will be retried", -1); err != nil {
common.Warn("knowledge_compile: failed to finalize stopped dataset ingestion log",
zap.String("dataset_id", datasetID), zap.Error(err))
}
@@ -247,7 +301,7 @@ func (c *Consumer) processClaim(ctx context.Context, cr ClaimResult) {
common.Warn("knowledge_compile: failed to record error_msg",
zap.String("dataset_id", datasetID), zap.Error(err))
}
- if err := finishDatasetCompileLog(ctx, cr.Token, logTaskTypes, common.FAILED, batchErr.Error(), -1); err != nil {
+ if err := c.finishDatasetCompileLog(ctx, cr.Token, logTaskTypes, common.FAILED, batchErr.Error(), -1); err != nil {
common.Warn("knowledge_compile: failed to finalize dataset ingestion log",
zap.String("dataset_id", datasetID), zap.Error(err))
}
@@ -256,13 +310,13 @@ func (c *Consumer) processClaim(ctx context.Context, cr ClaimResult) {
if _, err := c.scheduler.Ack(ctx, datasetID, cr.Token, cr.Entries); err != nil {
common.Warn("knowledge_compile: ack failed",
zap.String("dataset_id", datasetID), zap.Error(err))
- if logErr := finishDatasetCompileLog(ctx, cr.Token, logTaskTypes, common.FAILED, "Failed to acknowledge completed knowledge compilation: "+err.Error(), -1); logErr != nil {
+ if logErr := c.finishDatasetCompileLog(ctx, cr.Token, logTaskTypes, common.FAILED, "Failed to acknowledge completed knowledge compilation: "+err.Error(), -1); logErr != nil {
common.Warn("knowledge_compile: failed to finalize dataset ingestion log",
zap.String("dataset_id", datasetID), zap.Error(logErr))
}
return
}
- if err := finishDatasetCompileLog(ctx, cr.Token, logTaskTypes, common.COMPLETED, "Knowledge compilation completed", 1); err != nil {
+ if err := c.finishDatasetCompileLog(ctx, cr.Token, logTaskTypes, common.COMPLETED, "Knowledge compilation completed", 1); err != nil {
common.Warn("knowledge_compile: failed to finalize dataset ingestion log",
zap.String("dataset_id", datasetID), zap.Error(err))
}
@@ -275,7 +329,7 @@ func (c *Consumer) reportProgress(ctx context.Context, tenant, datasetID, token
zap.String("phase", phase),
zap.Error(err))
}
- if err := updateDatasetCompileLog(ctx, token, taskTypes, progress, message); err != nil {
+ if err := c.updateDatasetCompileLog(ctx, token, taskTypes, progress, message); err != nil {
common.Warn("knowledge_compile: failed to update dataset ingestion log",
zap.String("dataset_id", datasetID), zap.String("phase", phase), zap.Error(err))
}
@@ -412,12 +466,13 @@ func (c *Consumer) RebuildDataset(ctx context.Context, tenant, kb, mode string)
}
// defaultDocLister enumerates every doc id in a KB via the Document DAO. When
-// no DB was provisioned it returns no docs (a rewrite becomes a no-op clean).
-func defaultDocLister(ctx context.Context, tenant, kb string) ([]string, error) {
- if kcDB == nil {
+// no DB was provisioned (nil c.db) it returns no docs (a rewrite becomes a
+// no-op clean). The handle comes from WithStorage, not the package global kcDB.
+func (c *Consumer) defaultDocLister(ctx context.Context, tenant, kb string) ([]string, error) {
+ if c.db == nil {
return nil, nil
}
- return dao.NewDocumentDAO().ListIDsByKBIDWithOptions(ctx, kcDB, dao.DocumentListOptions{KbID: kb})
+ return dao.NewDocumentDAO().ListIDsByKBIDWithOptions(ctx, c.db, dao.DocumentListOptions{KbID: kb})
}
// recoverDocTypes reconstructs the compile-type and frontend task-type sets a
diff --git a/internal/ingestion/knowledge_compile/dataset_log.go b/internal/ingestion/knowledge_compile/dataset_log.go
index fb02f2a270..a77fe35f13 100644
--- a/internal/ingestion/knowledge_compile/dataset_log.go
+++ b/internal/ingestion/knowledge_compile/dataset_log.go
@@ -165,8 +165,8 @@ func sortedTaskTypes(seen map[string]struct{}) []string {
return taskTypes
}
-func startDatasetCompileLog(ctx context.Context, tenantID, datasetID, claimToken string, entries []BacklogEntry) error {
- if kcDB == nil || claimToken == "" {
+func (c *Consumer) startDatasetCompileLog(ctx context.Context, tenantID, datasetID, claimToken string, entries []BacklogEntry) error {
+ if c.db == nil || claimToken == "" {
return nil
}
now := time.Now()
@@ -187,7 +187,7 @@ func startDatasetCompileLog(ctx context.Context, tenantID, datasetID, claimToken
}
}
}
- return kcDB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ return c.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
for _, taskType := range sortedTaskTypesFromGroups(groups) {
entryData := make([]any, 0, len(groups[taskType]))
for _, entry := range groups[taskType] {
@@ -231,37 +231,37 @@ func startDatasetCompileLog(ctx context.Context, tenantID, datasetID, claimToken
})
}
-func updateDatasetCompileLog(ctx context.Context, claimToken string, taskTypes []string, progress float64, message string) error {
- if kcDB == nil || claimToken == "" {
+func (c *Consumer) updateDatasetCompileLog(ctx context.Context, claimToken string, taskTypes []string, progress float64, message string) error {
+ if c.db == nil || claimToken == "" {
return nil
}
for _, taskType := range taskTypes {
- if err := updateDatasetCompileLogForType(ctx, datasetCompileLogID(claimToken, taskType), progress, message); err != nil {
+ if err := c.updateDatasetCompileLogForType(ctx, datasetCompileLogID(claimToken, taskType), progress, message); err != nil {
return err
}
}
return nil
}
-func updateDatasetCompileLogForType(ctx context.Context, logID string, progress float64, message string) error {
- return recordDatasetCompileLogEvent(ctx, logID, progress, common.RUNNING, dao.EventTypeMessage, message)
+func (c *Consumer) updateDatasetCompileLogForType(ctx context.Context, logID string, progress float64, message string) error {
+ return c.recordDatasetCompileLogEvent(ctx, logID, progress, common.RUNNING, dao.EventTypeMessage, message)
}
-func finishDatasetCompileLog(ctx context.Context, claimToken string, taskTypes []string, operationStatus, message string, progress float64) error {
- if kcDB == nil || claimToken == "" {
+func (c *Consumer) finishDatasetCompileLog(ctx context.Context, claimToken string, taskTypes []string, operationStatus, message string, progress float64) error {
+ if c.db == nil || claimToken == "" {
return nil
}
for _, taskType := range taskTypes {
logID := datasetCompileLogID(claimToken, taskType)
- if err := recordDatasetCompileLogEvent(ctx, logID, progress, datasetLogOperationStatus(operationStatus), dao.EventTypeTerminal, message); err != nil {
+ if err := c.recordDatasetCompileLogEvent(ctx, logID, progress, datasetLogOperationStatus(operationStatus), dao.EventTypeTerminal, message); err != nil {
return err
}
}
return nil
}
-func recordDatasetCompileLogEvent(ctx context.Context, logID string, progress float64, operationStatus string, eventType int, message string) error {
- return kcDB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+func (c *Consumer) recordDatasetCompileLogEvent(ctx context.Context, logID string, progress float64, operationStatus string, eventType int, message string) error {
+ return c.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var log entity.PipelineOperationLog
if err := tx.WithContext(ctx).Where("id = ?", logID).First(&log).Error; err != nil {
return err
diff --git a/internal/ingestion/knowledge_compile/dataset_log_test.go b/internal/ingestion/knowledge_compile/dataset_log_test.go
index 8aa3559d5a..7949813fd4 100644
--- a/internal/ingestion/knowledge_compile/dataset_log_test.go
+++ b/internal/ingestion/knowledge_compile/dataset_log_test.go
@@ -22,24 +22,22 @@ func TestDatasetCompileLogLifecycle(t *testing.T) {
if err := db.AutoMigrate(&entity.PipelineOperationLog{}, &entity.IngestionTaskLog{}); err != nil {
t.Fatalf("migrate ingestion logs: %v", err)
}
- oldDB := kcDB
- kcDB = db
- t.Cleanup(func() { kcDB = oldDB })
+ consumer := &Consumer{db: db}
entries := []BacklogEntry{
{DocID: "doc-1", EventType: string(EventTypeCompleted), Variants: []string{"wiki"}, TaskTypes: []string{kccommon.TaskTypeWiki}},
{DocID: "doc-2", EventType: string(EventTypeCompleted), Variants: []string{"structure"}, TaskTypes: []string{kccommon.TaskTypeGraph}},
}
- if err := startDatasetCompileLog(t.Context(), "tenant-1", "kb-1", "claim-1", entries); err != nil {
+ if err := consumer.startDatasetCompileLog(t.Context(), "tenant-1", "kb-1", "claim-1", entries); err != nil {
t.Fatalf("start dataset log: %v", err)
}
- if err := startDatasetCompileLog(t.Context(), "tenant-1", "kb-1", "claim-1", entries); err != nil {
+ if err := consumer.startDatasetCompileLog(t.Context(), "tenant-1", "kb-1", "claim-1", entries); err != nil {
t.Fatalf("restart dataset log: %v", err)
}
- if err := updateDatasetCompileLog(t.Context(), "claim-1", []string{kccommon.TaskTypeWiki, kccommon.TaskTypeGraph}, 0.5, "Comparing knowledge contributions: 2 affected page(s)"); err != nil {
+ if err := consumer.updateDatasetCompileLog(t.Context(), "claim-1", []string{kccommon.TaskTypeWiki, kccommon.TaskTypeGraph}, 0.5, "Comparing knowledge contributions: 2 affected page(s)"); err != nil {
t.Fatalf("update dataset log: %v", err)
}
- if err := finishDatasetCompileLog(t.Context(), "claim-1", []string{kccommon.TaskTypeWiki, kccommon.TaskTypeGraph}, common.COMPLETED, "Knowledge compilation completed", 1); err != nil {
+ if err := consumer.finishDatasetCompileLog(t.Context(), "claim-1", []string{kccommon.TaskTypeWiki, kccommon.TaskTypeGraph}, common.COMPLETED, "Knowledge compilation completed", 1); err != nil {
t.Fatalf("finish dataset log: %v", err)
}
diff --git a/internal/ingestion/knowledge_compile/deps_failfast_test.go b/internal/ingestion/knowledge_compile/deps_failfast_test.go
new file mode 100644
index 0000000000..8c997d70e0
--- /dev/null
+++ b/internal/ingestion/knowledge_compile/deps_failfast_test.go
@@ -0,0 +1,224 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package knowledge_compile
+
+import (
+ "context"
+ "strings"
+ "testing"
+
+ "gorm.io/gorm"
+ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
+)
+
+// The doc engine is a hard dependency of the dataset-level compile reader,
+// writer, and contribution store. Before dependency injection, a nil engine
+// silently degraded: readers reported "no products", writers reported
+// "deleted successfully", and the contribution store silently fell back to an
+// in-memory map — a batch would be acked as merged while nothing was read,
+// written, or deleted. These tests pin the fail-fast contract: an unconfigured
+// engine is a loud error, never a silent success.
+
+const engineUnconfiguredNeedle = "engine"
+
+func requireEngineError(t *testing.T, err error) {
+ t.Helper()
+ if err == nil {
+ t.Fatal("expected a fail-fast error for the unconfigured doc engine, got nil (silent success)")
+ }
+ if !strings.Contains(err.Error(), engineUnconfiguredNeedle) {
+ t.Fatalf("error %q does not identify the missing doc engine", err.Error())
+ }
+}
+
+func TestEngineReaderWithoutEngineFailsFast(t *testing.T) {
+ ctx := context.Background()
+ r := engineReader{} // zero value: no engine injected
+
+ cases := []struct {
+ name string
+ call func() error
+ }{
+ {"LoadDocProducts", func() error {
+ _, err := r.LoadDocProducts(ctx, "tenant-1", "kb-1", "doc-1")
+ return err
+ }},
+ {"LoadMergedProduct", func() error {
+ _, err := r.LoadMergedProduct(ctx, "tenant-1", "kb-1", "product-1")
+ return err
+ }},
+ {"LoadMergedWikiPages", func() error {
+ _, err := r.LoadMergedWikiPages(ctx, "tenant-1", "kb-1")
+ return err
+ }},
+ {"LoadDocumentWikiPagesBySlugs", func() error {
+ _, err := r.LoadDocumentWikiPagesBySlugs(ctx, "tenant-1", "kb-1", []string{"slug-1"})
+ return err
+ }},
+ {"LoadDocumentWikiPagesBySlugs with empty slugs", func() error {
+ _, err := r.LoadDocumentWikiPagesBySlugs(ctx, "tenant-1", "kb-1", nil)
+ return err
+ }},
+ {"SearchSimilar", func() error {
+ _, _, err := r.SearchSimilar(ctx, "tenant-1", "kb-1", kccommon.VariantWiki, []float64{0.1, 0.2}, 5, 0.85)
+ return err
+ }},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ requireEngineError(t, tc.call())
+ })
+ }
+}
+
+func TestEngineWriterWithoutEngineFailsFast(t *testing.T) {
+ ctx := context.Background()
+ // Zero value: no engine injected. commitService and resolveLanguage stay
+ // nil so the engine check is the only failure mode under test.
+ w := engineWriter{}
+
+ oneDoc := []string{"doc-1"}
+ oneProduct := []kccommon.Product{{ID: "product-1"}}
+
+ cases := []struct {
+ name string
+ call func() error
+ }{
+ {"DeleteMergedWikiPages", func() error {
+ return w.DeleteMergedWikiPages(ctx, "tenant-1", "kb-1", []string{"page-1"})
+ }},
+ {"WriteMerged", func() error {
+ return w.WriteMerged(ctx, "tenant-1", "kb-1", oneProduct)
+ }},
+ {"WriteMergedStructure", func() error {
+ return w.WriteMergedStructure(ctx, "tenant-1", "kb-1", []StructureBucket{{}})
+ }},
+ {"DeleteStructureForDocs", func() error {
+ return w.DeleteStructureForDocs(ctx, "tenant-1", "kb-1", oneDoc)
+ }},
+ {"DeleteDocLevelForDocs", func() error {
+ return w.DeleteDocLevelForDocs(ctx, "tenant-1", "kb-1", oneDoc)
+ }},
+ {"StripMergedSources", func() error {
+ return w.StripMergedSources(ctx, "tenant-1", "kb-1", oneDoc)
+ }},
+ {"ProjectWikiGraph", func() error {
+ return w.ProjectWikiGraph(ctx, "tenant-1", "kb-1")
+ }},
+ {"DropWikiGraph", func() error {
+ return w.DropWikiGraph(ctx, "tenant-1", "kb-1")
+ }},
+ {"DeleteMerged", func() error {
+ return w.DeleteMerged(ctx, "tenant-1", "kb-1")
+ }},
+ {"DeleteMergedForVariant", func() error {
+ return w.DeleteMergedForVariant(ctx, "tenant-1", "kb-1", []kccommon.Variant{kccommon.VariantWiki})
+ }},
+ {"loadWikiPageContent", func() error {
+ _, err := w.loadWikiPageContent(ctx, "tenant-1", "kb-1", oneProduct)
+ return err
+ }},
+ {"loadActiveDocumentWikiPages", func() error {
+ _, err := w.loadActiveDocumentWikiPages(ctx, "tenant-1", "kb-1")
+ return err
+ }},
+ {"loadMergedWikiPages", func() error {
+ _, err := w.loadMergedWikiPages(ctx, "tenant-1", "kb-1")
+ return err
+ }},
+ {"dropWikiGraph", func() error {
+ return w.dropWikiGraph(ctx, "tenant-1", "kb-1")
+ }},
+ {"insertWikiGraphChunks", func() error {
+ return w.insertWikiGraphChunks(ctx, "tenant-1", "kb-1", []map[string]interface{}{{"id": "row-1"}})
+ }},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ requireEngineError(t, tc.call())
+ })
+ }
+}
+
+func TestNewConsumerWithoutConfiguredStorageFailsFast(t *testing.T) {
+ // A Consumer assembled without an injected doc engine must fail loudly on
+ // the first storage access. Previously it silently read zero products and
+ // acked the batch as merged.
+ consumer := NewConsumer(NewFakeScheduler())
+ if consumer.IsStorageConfigured() {
+ t.Fatal("unconfigured consumer must report IsStorageConfigured() == false")
+ }
+ err := consumer.processBatch(context.Background(), "tenant-1", "kb-1", "claim-token-1",
+ []BacklogEntry{{DocID: "doc-1", EventType: string(EventTypeCompleted), Variants: []string{"wiki"}}},
+ []string{kccommon.TaskTypeWiki})
+ requireEngineError(t, err)
+}
+
+// TestWithStorageConfiguresReaderWriter pins the contract that WithStorage
+// (the production wiring path) installs an engine-backed reader, writer, and
+// contribution store. Without this, a bare NewConsumer silently wires nothing
+// and the production merge path is dead.
+func TestWithStorageConfiguresReaderWriter(t *testing.T) {
+ consumer := NewConsumer(NewFakeScheduler(), WithStorage(&fakeEngine{engineType: "infinity"}, nil))
+ if !consumer.IsStorageConfigured() {
+ t.Fatal("WithStorage must report IsStorageConfigured() == true")
+ }
+}
+
+// TestWithStorageInjectsDatabaseHandle pins the contract that WithStorage wires
+// the database handle into the consumer, so the storage-backed paths (doc
+// enumeration, dataset compile log) read from the injected db rather than a
+// package-level global installed by Provision. This decouples the consumer from
+// Provision's call ordering and from the knowledge_compile-specific global kcDB.
+func TestWithStorageInjectsDatabaseHandle(t *testing.T) {
+ db := &gorm.DB{}
+ consumer := NewConsumer(NewFakeScheduler(), WithStorage(&fakeEngine{engineType: "infinity"}, db))
+ if consumer.db != db {
+ t.Fatalf("WithStorage did not inject the database handle: got %v, want %v", consumer.db, db)
+ }
+}
+
+func TestNewWikiContributionStoreWithoutEngineFailsFast(t *testing.T) {
+ // Previously a nil engine silently produced an in-memory store, so a
+ // mis-wired consumer diffed contributions against empty state.
+ ctx := context.Background()
+ store := newWikiContributionStore(nil)
+
+ cases := []struct {
+ name string
+ call func() error
+ }{
+ {"Get", func() error {
+ _, _, err := store.Get(ctx, "tenant-1", "kb-1", "doc-1")
+ return err
+ }},
+ {"Put", func() error {
+ return store.Put(ctx, "tenant-1", "kb-1", wikiDocumentContribution{DocumentID: "doc-1"})
+ }},
+ {"Delete", func() error {
+ return store.Delete(ctx, "tenant-1", "kb-1", "doc-1")
+ }},
+ {"ClearDataset", func() error {
+ return store.ClearDataset(ctx, "tenant-1", "kb-1")
+ }},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ requireEngineError(t, tc.call())
+ })
+ }
+}
diff --git a/internal/ingestion/knowledge_compile/fake_engine_test.go b/internal/ingestion/knowledge_compile/fake_engine_test.go
index 958bd744f8..3dbd726301 100644
--- a/internal/ingestion/knowledge_compile/fake_engine_test.go
+++ b/internal/ingestion/knowledge_compile/fake_engine_test.go
@@ -27,6 +27,10 @@ import (
// Search filter/SelectFields and the DeleteChunks condition so tests can assert
// the structural scope of a reader/writer call, and returns a canned chunk set.
type fakeEngine struct {
+ // engineType overrides GetType; defaults to "fake" (unsupported for
+ // dataset-structure writes). Tests of structure write/delete paths set
+ // this to "infinity" so the guard permits the operation.
+ engineType string
// searchChunks is the result returned by Search.
searchChunks []map[string]interface{}
// lastSearchReq captures the most recent SearchRequest for assertions.
@@ -114,8 +118,13 @@ func (f *fakeEngine) KNNScores(context.Context, []map[string]interface{}, []floa
func (f *fakeEngine) GetScores(map[string]interface{}) map[string]float64 { return nil }
func (f *fakeEngine) Ping(context.Context) error { return nil }
func (f *fakeEngine) Close() error { return nil }
-func (f *fakeEngine) GetType() string { return "fake" }
-func (f *fakeEngine) SupportsPageRank() bool { return false }
+
+// GetType reports the engine type. It defaults to "fake"; tests that exercise
+// dataset-structure write/delete paths override engineType to "infinity" so
+// the datasetStructureSupported guard (which mirrors the production engine
+// type) lets the operation proceed.
+func (f *fakeEngine) GetType() string { return f.engineType }
+func (f *fakeEngine) SupportsPageRank() bool { return false }
func (f *fakeEngine) FilterDocIdsByMetaPushdown(context.Context, *gorm.DB, []string, []map[string]interface{}, string) []string {
return nil
}
diff --git a/internal/ingestion/knowledge_compile/reader.go b/internal/ingestion/knowledge_compile/reader.go
index d1817ce150..4e8c683135 100644
--- a/internal/ingestion/knowledge_compile/reader.go
+++ b/internal/ingestion/knowledge_compile/reader.go
@@ -29,6 +29,8 @@ import (
"ragflow/internal/engine/types"
"ragflow/internal/entity"
kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
+
+ "gorm.io/gorm"
)
// Reader finds the compiled products needed for incremental dedup without
@@ -78,24 +80,29 @@ type documentWikiPageReader interface {
//
// The bool reports whether the database is configured. Offline/unit readers do
// not have document status available and retain the previous engine-only behavior.
-func enabledDocumentIDs(ctx context.Context, datasetID string) ([]string, bool, error) {
- if kcDB == nil {
+func enabledDocumentIDs(ctx context.Context, db *gorm.DB, datasetID string) ([]string, bool, error) {
+ if db == nil {
return nil, false, nil
}
var ids []string
- err := kcDB.WithContext(ctx).
+ err := db.WithContext(ctx).
Model(&entity.Document{}).
Where("kb_id = ? AND (status IS NULL OR status <> ?)", datasetID, "0").
Pluck("id", &ids).Error
return ids, true, err
}
-// engineReader loads the per-document compiled products through the global
-// DocEngine (§11.6 step 1, §11.7 incremental re-dedup). It depends on the
-// process-wide DocEngine obtained via engine.Get(); the engine abstraction owns
-// the storage schema, so this reader is not backend-specific.
+// engineReader loads the per-document compiled products through the injected
+// DocEngine (§11.6 step 1, §11.7 incremental re-dedup). The engine is wired by
+// the server assembly point (see WithDocEngine); the engine abstraction owns
+// the storage schema, so this reader is not backend-specific. A reader without
+// an engine fails fast instead of silently reporting "no products".
type engineReader struct {
eng engine.DocEngine
+ // db is the injected MySQL handle backing enabledDocumentIDs. It is wired by
+ // WithStorage; a zero-value reader has a nil db and enabledDocumentIDs
+ // reports "status unavailable" (the previous engine-only behavior).
+ db *gorm.DB
}
// compiledSelectFields are the columns needed to reconstruct a Product. Every
@@ -150,13 +157,10 @@ const loadDocProductsLimit = 5000
// results are paged so a document with more than loadDocProductsLimit rows is
// not silently truncated.
func (r engineReader) LoadDocProducts(ctx context.Context, tenant, kb, docID string) ([]kccommon.Product, error) {
- eng := r.eng
- if eng == nil {
- eng = engine.Get()
- }
- if eng == nil {
- return nil, nil
+ if r.eng == nil {
+ return nil, errEngineNotConfigured
}
+ eng := r.eng
var out []kccommon.Product
offset := 0
for {
@@ -213,11 +217,11 @@ func (r engineReader) LoadDocProducts(ctx context.Context, tenant, kb, docID str
}
func (r engineReader) LoadMergedProduct(ctx context.Context, tenant, kb, id string) (kccommon.Product, error) {
- eng := r.eng
- if eng == nil {
- eng = engine.Get()
+ if r.eng == nil {
+ return kccommon.Product{}, errEngineNotConfigured
}
- if eng == nil || id == "" {
+ eng := r.eng
+ if id == "" {
return kccommon.Product{}, nil
}
filter := map[string]interface{}{"id": id, "available_int": 1, "scope_kwd": "dataset"}
@@ -240,13 +244,10 @@ func (r engineReader) LoadMergedProduct(ctx context.Context, tenant, kb, id stri
}
func (r engineReader) LoadMergedWikiPages(ctx context.Context, tenant, kb string) ([]kccommon.Product, error) {
- eng := r.eng
- if eng == nil {
- eng = engine.Get()
- }
- if eng == nil {
- return nil, nil
+ if r.eng == nil {
+ return nil, errEngineNotConfigured
}
+ eng := r.eng
filter := map[string]interface{}{"available_int": 1, "scope_kwd": "dataset", "type_kwd": compileKwdWikiPage}
var pages []kccommon.Product
for offset := 0; ; offset += loadDocProductsLimit {
@@ -276,14 +277,14 @@ func (r engineReader) LoadMergedWikiPages(ctx context.Context, tenant, kb string
}
func (r engineReader) LoadDocumentWikiPagesBySlugs(ctx context.Context, tenant, kb string, slugs []string) ([]kccommon.Product, error) {
- eng := r.eng
- if eng == nil {
- eng = engine.Get()
+ if r.eng == nil {
+ return nil, errEngineNotConfigured
}
- if eng == nil || len(slugs) == 0 {
+ eng := r.eng
+ if len(slugs) == 0 {
return nil, nil
}
- enabledIDs, statusAvailable, err := enabledDocumentIDs(ctx, kb)
+ enabledIDs, statusAvailable, err := enabledDocumentIDs(ctx, r.db, kb)
if err != nil {
return nil, fmt.Errorf("load enabled Wiki documents: %w", err)
}
@@ -525,13 +526,10 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo
// SearchSimilar runs a dense KNN over the existing merged rows (available_int=1,
// compile_kwd=variant) of the KB and returns the closest hit above minScore.
func (r engineReader) SearchSimilar(ctx context.Context, tenant, kb string, variant kccommon.Variant, vector []float64, topN int, minScore float64) (kccommon.Product, float64, error) {
- eng := r.eng
- if eng == nil {
- eng = engine.Get()
- }
- if eng == nil {
- return kccommon.Product{}, 0, nil
+ if r.eng == nil {
+ return kccommon.Product{}, 0, errEngineNotConfigured
}
+ eng := r.eng
if topN <= 0 {
topN = 1
}
diff --git a/internal/ingestion/knowledge_compile/reader_test.go b/internal/ingestion/knowledge_compile/reader_test.go
index 42a21289aa..bba9280a58 100644
--- a/internal/ingestion/knowledge_compile/reader_test.go
+++ b/internal/ingestion/knowledge_compile/reader_test.go
@@ -45,16 +45,12 @@ func TestWikiReadersFilterDisabledDocuments(t *testing.T) {
t.Fatalf("create document %s: %v", doc.ID, err)
}
}
- previousDB := kcDB
- kcDB = db
- t.Cleanup(func() { kcDB = previousDB })
-
eng := &fakeEngine{searchChunks: []map[string]interface{}{{
"id": "page-1", "doc_id": "active-doc", "compile_kwd": "wiki_page",
"available_int": 0, "content_with_weight": "page content", "slug_kwd": "entity/page",
"page_type_kwd": "entity", "title_kwd": "page", "source_doc_ids": []string{"active-doc"},
}}}
- r := engineReader{eng: eng}
+ r := engineReader{eng: eng, db: db}
if _, err := r.LoadDocumentWikiPagesBySlugs(t.Context(), "tenant", "kb1", []string{"entity/page"}); err != nil {
t.Fatalf("load document Wiki pages: %v", err)
}
@@ -63,7 +59,7 @@ func TestWikiReadersFilterDisabledDocuments(t *testing.T) {
t.Fatalf("document Wiki filter = %#v, want active document IDs", wikiFilter)
}
- w := engineWriter{eng: eng}
+ w := engineWriter{eng: eng, db: db}
if _, err := w.loadActiveDocumentWikiPages(t.Context(), "tenant", "kb1"); err != nil {
t.Fatalf("load active document Wiki pages: %v", err)
}
diff --git a/internal/ingestion/knowledge_compile/service.go b/internal/ingestion/knowledge_compile/service.go
index 086cbe8c84..208f90bc3e 100644
--- a/internal/ingestion/knowledge_compile/service.go
+++ b/internal/ingestion/knowledge_compile/service.go
@@ -25,6 +25,7 @@ import (
"ragflow/internal/engine"
"ragflow/internal/entity"
kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
+ "ragflow/internal/service/file"
"gorm.io/gorm"
)
@@ -76,6 +77,23 @@ func WithReader(r Reader) Option { return func(c *Consumer) { c.reader = r } }
// WithWriter overrides the chunk Writer (used in tests).
func WithWriter(w Writer) Option { return func(c *Consumer) { c.writer = w } }
+// WithStorage wires the process-wide doc engine and database handle into the
+// consumer's default reader, writer, and wiki contribution store. The server
+// assembly point passes ingestion.Deps here; a consumer assembled without it
+// fails fast on its first storage access (errEngineNotConfigured) instead of
+// silently reading zero products. Apply it before any WithReader/WithWriter
+// overrides.
+func WithStorage(eng engine.DocEngine, db *gorm.DB) Option {
+ return func(c *Consumer) {
+ c.reader = engineReader{eng: eng, db: db}
+ c.writer = engineWriter{eng: eng, db: db, commitService: file.NewFileCommitService()}
+ c.contributions = newWikiContributionStore(eng)
+ c.db = db
+ c.docEngine = eng
+ c.storageConfigured = true
+ }
+}
+
// WithDeduperFactory overrides the per-tenant Deduper factory.
func WithDeduperFactory(f DeduperFactory) Option { return func(c *Consumer) { c.factory = f } }
@@ -141,7 +159,6 @@ func Provision(ctx context.Context, mq engine.MessageQueue, db *gorm.DB) error {
if db == nil {
return nil
}
- kcDB = db
s := newScheduler(db, mq, generateHolder(), 2*time.Minute)
// Bound the startup AutoMigrate so a slow/unreachable DB cannot block
// startup indefinitely. The caller's ctx is also honoured (cancelled on
diff --git a/internal/ingestion/knowledge_compile/wiki_contribution.go b/internal/ingestion/knowledge_compile/wiki_contribution.go
index 6df368136f..d0f897a574 100644
--- a/internal/ingestion/knowledge_compile/wiki_contribution.go
+++ b/internal/ingestion/knowledge_compile/wiki_contribution.go
@@ -55,13 +55,37 @@ type engineWikiContributionStore struct {
engine engine.DocEngine
}
+// newWikiContributionStore returns the engine-backed store. A nil engine is a
+// wiring mistake: the store then fails fast on every access instead of
+// silently degrading to an in-memory map, which would diff contributions
+// against empty state and ack a batch that merged nothing.
func newWikiContributionStore(docEngine engine.DocEngine) wikiContributionStore {
if docEngine == nil {
- return &memoryWikiContributionStore{items: map[string]wikiDocumentContribution{}}
+ return unconfiguredWikiContributionStore{}
}
return &engineWikiContributionStore{engine: docEngine}
}
+// unconfiguredWikiContributionStore fails fast when the consumer was assembled
+// without a doc engine (see WithDocEngine).
+type unconfiguredWikiContributionStore struct{}
+
+func (unconfiguredWikiContributionStore) Get(context.Context, string, string, string) (wikiDocumentContribution, bool, error) {
+ return wikiDocumentContribution{}, false, errEngineNotConfigured
+}
+
+func (unconfiguredWikiContributionStore) Put(context.Context, string, string, wikiDocumentContribution) error {
+ return errEngineNotConfigured
+}
+
+func (unconfiguredWikiContributionStore) Delete(context.Context, string, string, string) error {
+ return errEngineNotConfigured
+}
+
+func (unconfiguredWikiContributionStore) ClearDataset(context.Context, string, string) error {
+ return errEngineNotConfigured
+}
+
func (s *engineWikiContributionStore) Get(ctx context.Context, tenantID, datasetID, documentID string) (wikiDocumentContribution, bool, error) {
key := wikiContributionStateID(datasetID, documentID)
result, err := s.engine.Search(ctx, &enginetypes.SearchRequest{
@@ -280,7 +304,11 @@ type wikiContributionDiffResult struct {
func (c *Consumer) prepareWikiContributionDiff(ctx context.Context, tenantID, datasetID string, completed []BacklogEntry, deleted []string) (wikiContributionDiffResult, error) {
if c.contributions == nil {
- c.contributions = newWikiContributionStore(nil)
+ // NewConsumer always installs a store (the unconfigured sentinel when
+ // WithDocEngine was not passed); a nil here means the Consumer struct
+ // was assembled by hand. Fail fast rather than silently diffing
+ // against empty state.
+ return wikiContributionDiffResult{}, errEngineNotConfigured
}
result := wikiContributionDiffResult{
productsByDoc: make(map[string][]kccommon.Product, len(completed)),
diff --git a/internal/ingestion/knowledge_compile/wiki_dirty.go b/internal/ingestion/knowledge_compile/wiki_dirty.go
index eb71fb34a9..b09ebdca83 100644
--- a/internal/ingestion/knowledge_compile/wiki_dirty.go
+++ b/internal/ingestion/knowledge_compile/wiki_dirty.go
@@ -67,10 +67,10 @@ func SetWikiDirtyCompiler(compiler WikiDirtyCompiler) {
// mutations merge chunk ids and postpone execution until twenty seconds after
// the most recent successful mutation.
func MarkWikiDocumentDirty(ctx context.Context, tenantID, datasetID, documentID string, chunkIDs []string) error {
- db := kcDB
- if db == nil {
- db = dao.DB
- }
+ // The publishing path (wiki_refresh, chunk) calls this from processes that
+ // own the canonical DAO handle; it is an out-of-scope external API, so it
+ // reads the global dao.DB rather than a package-scoped kcDB.
+ db := dao.DB
if db == nil {
return fmt.Errorf("mark Wiki document dirty: database is unavailable")
}
diff --git a/internal/ingestion/knowledge_compile/wiki_dirty_test.go b/internal/ingestion/knowledge_compile/wiki_dirty_test.go
index 04cfe59820..64124e2d34 100644
--- a/internal/ingestion/knowledge_compile/wiki_dirty_test.go
+++ b/internal/ingestion/knowledge_compile/wiki_dirty_test.go
@@ -9,6 +9,7 @@ import (
"github.com/glebarez/sqlite"
"gorm.io/gorm"
+ "ragflow/internal/dao"
"ragflow/internal/entity"
)
@@ -20,9 +21,9 @@ func TestMarkWikiDocumentDirtyDebouncesAndMergesChunks(t *testing.T) {
if err := db.AutoMigrate(&entity.WikiDocumentDirty{}); err != nil {
t.Fatalf("migrate dirty table: %v", err)
}
- previousDB := kcDB
- kcDB = db
- t.Cleanup(func() { kcDB = previousDB })
+ previousDB := dao.DB
+ dao.DB = db
+ t.Cleanup(func() { dao.DB = previousDB })
if err := MarkWikiDocumentDirty(t.Context(), "tenant", "dataset", "document", []string{"chunk-b", "chunk-a"}); err != nil {
t.Fatalf("first dirty mark: %v", err)
@@ -62,9 +63,9 @@ func TestWikiDirtyRevisionPreventsDroppingConcurrentMutation(t *testing.T) {
if err := db.AutoMigrate(&entity.WikiDocumentDirty{}); err != nil {
t.Fatalf("migrate dirty table: %v", err)
}
- previousDB := kcDB
- kcDB = db
- t.Cleanup(func() { kcDB = previousDB })
+ previousDB := dao.DB
+ dao.DB = db
+ t.Cleanup(func() { dao.DB = previousDB })
if err := MarkWikiDocumentDirty(t.Context(), "tenant", "dataset", "document", []string{"chunk-a"}); err != nil {
t.Fatalf("dirty mark: %v", err)
diff --git a/internal/ingestion/knowledge_compile/wiki_map_store.go b/internal/ingestion/knowledge_compile/wiki_map_store.go
index 804bd8e2e3..669a563890 100644
--- a/internal/ingestion/knowledge_compile/wiki_map_store.go
+++ b/internal/ingestion/knowledge_compile/wiki_map_store.go
@@ -26,6 +26,8 @@ import (
"ragflow/internal/engine"
"ragflow/internal/engine/types"
kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
+
+ "gorm.io/gorm"
)
const (
@@ -36,9 +38,10 @@ const (
type wikiMapVersionStore struct {
engine engine.DocEngine
+ db *gorm.DB
resolveVectorSize func(context.Context) (int, error)
// resolveLanguage returns the dataset language a missing Infinity chunk
- // table is created with; nil reads it from the dataset row.
+ // table is created with; nil reads it from the dataset row via db.
resolveLanguage func(ctx context.Context, datasetID string) (string, error)
}
@@ -97,11 +100,13 @@ func (s *wikiMapVersionStore) PutWikiMapActiveState(ctx context.Context, state k
// rows. Elasticsearch does not need this resolver because it can create its
// mapping dynamically.
func NewWikiMapVersionStoreWithVectorSizeResolver(
+ db *gorm.DB,
docEngine engine.DocEngine,
resolveVectorSize func(context.Context) (int, error),
) kccommon.WikiMapVersionStore {
return &wikiMapVersionStore{
engine: docEngine,
+ db: db,
resolveVectorSize: resolveVectorSize,
}
}
@@ -132,7 +137,9 @@ func (s *wikiMapVersionStore) ensureInfinityChunkStore(ctx context.Context, tena
// analyzer here, so it has to be created with the dataset language.
resolveLanguage := s.resolveLanguage
if resolveLanguage == nil {
- resolveLanguage = datasetLanguageByID
+ resolveLanguage = func(ctx context.Context, datasetID string) (string, error) {
+ return datasetLanguageByID(ctx, s.db, datasetID)
+ }
}
language, err := resolveLanguage(ctx, datasetID)
if err != nil {
@@ -144,9 +151,13 @@ func (s *wikiMapVersionStore) ensureInfinityChunkStore(ctx context.Context, tena
return nil
}
-// datasetLanguageByID reads the dataset's language, "" when it is unset.
-func datasetLanguageByID(ctx context.Context, datasetID string) (string, error) {
- kb, err := dao.NewKnowledgebaseDAO().GetByID(ctx, dao.DB, datasetID)
+// datasetLanguageByID reads the dataset's language, "" when it is unset (or
+// when no database handle was wired — the caller then tokenizes as English).
+func datasetLanguageByID(ctx context.Context, db *gorm.DB, datasetID string) (string, error) {
+ if db == nil {
+ return "", nil
+ }
+ kb, err := dao.NewKnowledgebaseDAO().GetByID(ctx, db, datasetID)
if err != nil {
return "", err
}
diff --git a/internal/ingestion/knowledge_compile/wiki_map_store_test.go b/internal/ingestion/knowledge_compile/wiki_map_store_test.go
index ceec7df3ab..6051630785 100644
--- a/internal/ingestion/knowledge_compile/wiki_map_store_test.go
+++ b/internal/ingestion/knowledge_compile/wiki_map_store_test.go
@@ -147,7 +147,7 @@ func (e *wikiMapStoreEngine) InsertChunks(_ context.Context, chunks []map[string
func TestWikiMapVersionStoreUsesNonSearchableDocStoreRows(t *testing.T) {
engine := &wikiMapStoreEngine{rows: map[string]map[string]interface{}{}}
- store := NewWikiMapVersionStoreWithVectorSizeResolver(engine, nil)
+ store := NewWikiMapVersionStoreWithVectorSizeResolver(nil, engine, nil)
version := kccommon.WikiMapVersion{
Key: "version-a",
TenantID: "tenant-1",
@@ -219,7 +219,7 @@ func TestWikiMapVersionStoreSkipsExistingInfinityStoreBootstrap(t *testing.T) {
chunkStoreExists: true,
}
resolverCalls := 0
- store := NewWikiMapVersionStoreWithVectorSizeResolver(engine, func(context.Context) (int, error) {
+ store := NewWikiMapVersionStoreWithVectorSizeResolver(nil, engine, func(context.Context) (int, error) {
resolverCalls++
return 0, nil
})
@@ -238,7 +238,7 @@ func TestWikiMapVersionStoreSkipsExistingInfinityStoreBootstrap(t *testing.T) {
func TestWikiMapVersionStoreCreatesMissingInfinityStore(t *testing.T) {
engine := &wikiMapStoreEngine{rows: map[string]map[string]interface{}{}, engineType: "infinity"}
- store := NewWikiMapVersionStoreWithVectorSizeResolver(engine, func(context.Context) (int, error) {
+ store := NewWikiMapVersionStoreWithVectorSizeResolver(nil, engine, func(context.Context) (int, error) {
return 768, nil
})
// The table is the dataset's chunk table, so it is created with the
@@ -264,7 +264,7 @@ func TestWikiMapVersionStoreCreatesMissingInfinityStore(t *testing.T) {
func TestWikiMapActiveStateRequiresResolverOnlyWhenInfinityStoreMissing(t *testing.T) {
engine := &wikiMapStoreEngine{rows: map[string]map[string]interface{}{}, engineType: "infinity"}
- store := NewWikiMapVersionStoreWithVectorSizeResolver(engine, nil).(kccommon.WikiMapActiveStateStore)
+ store := NewWikiMapVersionStoreWithVectorSizeResolver(nil, engine, nil).(kccommon.WikiMapActiveStateStore)
state := kccommon.WikiMapActiveState{
Key: "active-a", TenantID: "tenant-1", DatasetID: "kb-1",
DocumentID: "doc-1", Payload: []byte(`{}`),
diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.go
index 371d95a3a1..1f441a13c5 100644
--- a/internal/ingestion/knowledge_compile/writer.go
+++ b/internal/ingestion/knowledge_compile/writer.go
@@ -27,6 +27,7 @@ import (
"github.com/cespare/xxhash/v2"
"go.uber.org/zap"
+ "gorm.io/gorm"
"ragflow/internal/common"
"ragflow/internal/engine"
@@ -119,15 +120,18 @@ type StructureBucket struct {
MentionCount int
}
-// engineWriter persists dataset-level merged products through the global
-// DocEngine (§11.7). Like engineReader, it depends on the process-wide DocEngine
-// obtained via engine.Get(); the storage schema lives behind the engine
-// abstraction rather than in this package.
+// engineWriter persists dataset-level merged products through the injected
+// DocEngine (§11.7). The engine is wired by the server assembly point (see
+// WithDocEngine); the storage schema lives behind the engine abstraction
+// rather than in this package. A writer without an engine fails fast instead
+// of silently reporting successful writes.
type engineWriter struct {
eng engine.DocEngine
+ db *gorm.DB
commitService pageCommitter
// resolveLanguage returns the dataset language the dataset-level rows are
- // tokenized with; nil tokenizes them as English.
+ // tokenized with; nil reads it from the dataset row via db, defaulting to
+ // English when db is nil.
resolveLanguage func(ctx context.Context, datasetID string) (string, error)
}
@@ -135,10 +139,13 @@ type engineWriter struct {
// share the dataset's chunk table with the parsed chunks and are retrieved by
// the same queries, so they tokenize like them.
func (w engineWriter) datasetTokenizer(ctx context.Context, kb string) (tokenizer.Tokenizer, error) {
- if w.resolveLanguage == nil {
- return tokenizer.New(""), nil
+ resolveLanguage := w.resolveLanguage
+ if resolveLanguage == nil {
+ resolveLanguage = func(ctx context.Context, datasetID string) (string, error) {
+ return datasetLanguageByID(ctx, w.db, datasetID)
+ }
}
- language, err := w.resolveLanguage(ctx, kb)
+ language, err := resolveLanguage(ctx, kb)
if err != nil {
return tokenizer.Tokenizer{}, fmt.Errorf("resolve dataset language: %w", err)
}
@@ -153,13 +160,10 @@ func (w engineWriter) DeleteMergedWikiPages(ctx context.Context, tenant, kb stri
if len(ids) == 0 {
return nil
}
- eng := w.eng
- if eng == nil {
- eng = engine.Get()
- }
- if eng == nil {
- return nil
+ if w.eng == nil {
+ return errEngineNotConfigured
}
+ eng := w.eng
baseName := fmt.Sprintf("ragflow_%s", tenant)
_, err := eng.DeleteChunks(ctx, map[string]interface{}{
"id": ids, "kb_id": kb, "available_int": 1, "scope_kwd": "dataset", "type_kwd": compileKwdWikiPage,
@@ -174,15 +178,18 @@ func (w engineWriter) DeleteMergedWikiPages(ctx context.Context, tenant, kb stri
// these filter keys and would reject the query or silently drop unknown fields.
// The guard is checked at every dataset-structure write/delete/rebuild entry so
// those engines never leave partial state or hit an unknown-filter error (plan §5).
-func datasetStructureSupported() bool {
- switch engine.GetEngineType() {
- case "infinity", "elasticsearch":
- return true
- case "":
+// It reads the engine injected via engineWriter.eng rather than the process
+// global, so the dependency is explicit.
+func (w engineWriter) datasetStructureSupported() bool {
+ if w.eng == nil {
// Engine not initialized (unit tests inject a fake engine directly via
// engineWriter.eng). The guard is only meaningful against a real engine
// type, so an empty type is treated as supported.
return true
+ }
+ switch w.eng.GetType() {
+ case "infinity", "elasticsearch", "":
+ return true
default:
return false
}
@@ -190,8 +197,12 @@ func datasetStructureSupported() bool {
// errDatasetStructureUnsupported returns an error (not a silent no-op) naming
// the engine when a dataset-structure operation runs on an unsupported engine.
-func errDatasetStructureUnsupported() error {
- return fmt.Errorf("dataset structure graph unsupported on doc engine %q", engine.GetEngineType())
+func (w engineWriter) errDatasetStructureUnsupported() error {
+ engineType := ""
+ if w.eng != nil {
+ engineType = w.eng.GetType()
+ }
+ return fmt.Errorf("dataset structure graph unsupported on doc engine %q", engineType)
}
// writeMergedBatchSize bounds how many rows each parallel InsertChunks call
@@ -230,13 +241,10 @@ func (w engineWriter) WriteMerged(ctx context.Context, tenant, kb string, produc
zap.Int("total", len(products)),
zap.Any("by_compile_kwd", byKwd),
)
- eng := w.eng
- if eng == nil {
- eng = engine.Get()
- }
- if eng == nil {
- return nil
+ if w.eng == nil {
+ return errEngineNotConfigured
}
+ eng := w.eng
baseName := fmt.Sprintf("ragflow_%s", tenant)
// Source-tracing fields (tasks/2026-08-07-wiki-duplicates-research.md §5.1):
// one runID + inputHash + timestamp per WriteMerged call, shared by every
@@ -315,14 +323,11 @@ type wikiPageCommitTarget struct {
// generated version, while an empty value means an existing page currently has
// empty content; callers must preserve that distinction.
func (w engineWriter) loadWikiPageContent(ctx context.Context, tenant, kb string, products []kccommon.Product) (map[string]string, error) {
- eng := w.eng
- if eng == nil {
- eng = engine.Get()
+ if w.eng == nil {
+ return nil, errEngineNotConfigured
}
+ eng := w.eng
result := make(map[string]string)
- if eng == nil {
- return result, nil
- }
pageSlugs := make(map[string]struct{})
for _, product := range products {
@@ -484,16 +489,13 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin
if len(buckets) == 0 {
return nil
}
- if !datasetStructureSupported() {
- return errDatasetStructureUnsupported()
+ if !w.datasetStructureSupported() {
+ return w.errDatasetStructureUnsupported()
}
- eng := w.eng
- if eng == nil {
- eng = engine.Get()
- }
- if eng == nil {
- return nil
+ if w.eng == nil {
+ return errEngineNotConfigured
}
+ eng := w.eng
baseName := fmt.Sprintf("ragflow_%s", tenant)
now := time.Now()
staleIDs, err := mergeExistingStructureBuckets(ctx, eng, baseName, tenant, kb, buckets)
@@ -826,16 +828,13 @@ func (w engineWriter) DeleteStructureForDocs(ctx context.Context, tenant, kb str
if len(deletedDocIDs) == 0 {
return nil
}
- if !datasetStructureSupported() {
- return errDatasetStructureUnsupported()
+ if !w.datasetStructureSupported() {
+ return w.errDatasetStructureUnsupported()
}
- eng := w.eng
- if eng == nil {
- eng = engine.Get()
- }
- if eng == nil {
- return nil
+ if w.eng == nil {
+ return errEngineNotConfigured
}
+ eng := w.eng
baseName := fmt.Sprintf("ragflow_%s", tenant)
deleted := make(map[string]bool, len(deletedDocIDs))
for _, d := range deletedDocIDs {
@@ -1069,13 +1068,10 @@ func (w engineWriter) DeleteDocLevelForDocs(ctx context.Context, tenant, kb stri
if len(deletedDocIDs) == 0 {
return nil
}
- eng := w.eng
- if eng == nil {
- eng = engine.Get()
- }
- if eng == nil {
- return nil
+ if w.eng == nil {
+ return errEngineNotConfigured
}
+ eng := w.eng
baseName := fmt.Sprintf("ragflow_%s", tenant)
_, err := eng.DeleteChunks(ctx, map[string]interface{}{
"doc_id": deletedDocIDs,
@@ -1094,13 +1090,10 @@ func (w engineWriter) StripMergedSources(ctx context.Context, tenant, kb string,
if len(deletedDocIDs) == 0 {
return nil
}
- eng := w.eng
- if eng == nil {
- eng = engine.Get()
- }
- if eng == nil {
- return nil
+ if w.eng == nil {
+ return errEngineNotConfigured
}
+ eng := w.eng
baseName := fmt.Sprintf("ragflow_%s", tenant)
delSet := make(map[string]bool, len(deletedDocIDs))
for _, d := range deletedDocIDs {
@@ -1453,14 +1446,11 @@ func (w engineWriter) ProjectWikiGraph(ctx context.Context, tenant, kb string) e
// LLM-composed dataset page, otherwise disabling one of several contributors
// would leave that contributor's entities and edges in the graph.
func (w engineWriter) loadActiveDocumentWikiPages(ctx context.Context, tenant, kb string) ([]wikiPageProjection, error) {
- eng := w.eng
- if eng == nil {
- eng = engine.Get()
+ if w.eng == nil {
+ return nil, errEngineNotConfigured
}
- if eng == nil {
- return nil, nil
- }
- enabledIDs, statusAvailable, err := enabledDocumentIDs(ctx, kb)
+ eng := w.eng
+ enabledIDs, statusAvailable, err := enabledDocumentIDs(ctx, w.db, kb)
if err != nil {
return nil, fmt.Errorf("load enabled Wiki documents for graph: %w", err)
}
@@ -1545,19 +1535,16 @@ func (w engineWriter) DeleteMerged(ctx context.Context, tenant, kb string) error
// DeleteMergedForVariant clears the selected dataset outputs, preserving
// document-level compilation inputs. An empty variant set clears all outputs.
func (w engineWriter) DeleteMergedForVariant(ctx context.Context, tenant, kb string, variants []kccommon.Variant) error {
- eng := w.eng
- if eng == nil {
- eng = engine.Get()
- }
- if eng == nil {
- return nil
+ if w.eng == nil {
+ return errEngineNotConfigured
}
+ eng := w.eng
baseName := fmt.Sprintf("ragflow_%s", tenant)
if len(variants) == 0 {
// Navigation roles and searchable wiki outputs are separate from the
// dataset-scoped structure sweep.
- if !datasetStructureSupported() {
- return errDatasetStructureUnsupported()
+ if !w.datasetStructureSupported() {
+ return w.errDatasetStructureUnsupported()
}
_, err := eng.DeleteChunks(ctx, map[string]interface{}{
"kb_id": kb,
@@ -1593,8 +1580,8 @@ func (w engineWriter) DeleteMergedForVariant(ctx context.Context, tenant, kb str
return fmt.Errorf("delete merged (wiki): %w", err)
}
case kccommon.VariantStructure, kccommon.VariantMindmap:
- if !datasetStructureSupported() {
- return errDatasetStructureUnsupported()
+ if !w.datasetStructureSupported() {
+ return w.errDatasetStructureUnsupported()
}
// structure/mindmap dataset rows are scope_kwd="dataset" +
// type_kwd ∈ {entity,relation,kg_build_meta} with a raw
@@ -1631,13 +1618,10 @@ func (w engineWriter) DeleteMergedForVariant(ctx context.Context, tenant, kb str
// accumulate stale state. The dataset-wide full delete of merged rows is owned
// by DeleteDocLevelForKb elsewhere and is not repeated here.
func (w engineWriter) dropWikiGraph(ctx context.Context, tenant, kb string) error {
- eng := w.eng
- if eng == nil {
- eng = engine.Get()
- }
- if eng == nil {
- return nil
+ if w.eng == nil {
+ return errEngineNotConfigured
}
+ eng := w.eng
baseName := fmt.Sprintf("ragflow_%s", tenant)
_, err := eng.DeleteChunks(ctx, map[string]interface{}{
"kb_id": kb,
@@ -1655,13 +1639,10 @@ func (w engineWriter) dropWikiGraph(ctx context.Context, tenant, kb string) erro
// title / entity_names_kwd (aliases) / summary_with_weight / outlinks_kwd /
// source_doc_ids / source_chunk_ids), never the JSON payload.
func (w engineWriter) loadMergedWikiPages(ctx context.Context, tenant, kb string) ([]wikiPageProjection, error) {
- eng := w.eng
- if eng == nil {
- eng = engine.Get()
- }
- if eng == nil {
- return nil, nil
+ if w.eng == nil {
+ return nil, errEngineNotConfigured
}
+ eng := w.eng
baseName := fmt.Sprintf("ragflow_%s", tenant)
const batchSize = 2000
var out []wikiPageProjection
@@ -1907,13 +1888,10 @@ func (w engineWriter) insertWikiGraphChunks(ctx context.Context, tenant, kb stri
if len(rows) == 0 {
return nil
}
- eng := w.eng
- if eng == nil {
- eng = engine.Get()
- }
- if eng == nil {
- return nil
+ if w.eng == nil {
+ return errEngineNotConfigured
}
+ eng := w.eng
baseName := fmt.Sprintf("ragflow_%s", tenant)
jobs := make([]CompilerJob, 0, (len(rows)+wikiGraphBatchSize-1)/wikiGraphBatchSize)
for start := 0; start < len(rows); start += wikiGraphBatchSize {
diff --git a/internal/ingestion/knowledge_compile/writer_test.go b/internal/ingestion/knowledge_compile/writer_test.go
index 142cd5c5d9..12e7be8a74 100644
--- a/internal/ingestion/knowledge_compile/writer_test.go
+++ b/internal/ingestion/knowledge_compile/writer_test.go
@@ -172,7 +172,7 @@ func TestProductFromChunkMapRestoresWikiFields(t *testing.T) {
func TestWriteMergedStructureMigratesTypeScopedEntityID(t *testing.T) {
oldID := "dataset_structure_old_type_identity"
- eng := &fakeEngine{searchChunks: []map[string]interface{}{
+ eng := &fakeEngine{engineType: "infinity", searchChunks: []map[string]interface{}{
{
"id": oldID,
"compile_kwd": "hypergraph",
@@ -238,7 +238,7 @@ func TestWriteMergedStructureMigratesTypeScopedEntityID(t *testing.T) {
}
func TestWriteMergedStructureKeepsEntityKeywordLowercase(t *testing.T) {
- eng := &fakeEngine{}
+ eng := &fakeEngine{engineType: "infinity"}
w := engineWriter{eng: eng}
if err := w.WriteMergedStructure(context.Background(), "t1", "kb1", []StructureBucket{{
Name: "Acme Corp", Type: "organization", Description: "company", CompileKwd: "knowledge_graph", TemplateKind: "knowledge_graph",
@@ -268,7 +268,7 @@ func TestWriteMergedStructureKeepsEntityKeywordLowercase(t *testing.T) {
}
func TestWriteMergedStructurePreservesDistinctRelationTypes(t *testing.T) {
- eng := &fakeEngine{}
+ eng := &fakeEngine{engineType: "infinity"}
w := engineWriter{eng: eng}
buckets := []StructureBucket{
{Name: "Acme Corp -> Beta Ltd", Type: "relation", FromEntity: "Acme Corp", ToEntity: "Beta Ltd", RelationType: "supplies", Description: "supply", CompileKwd: "knowledge_graph", TemplateKind: "knowledge_graph"},
@@ -302,7 +302,7 @@ func TestWriteMergedStructurePreservesDistinctRelationTypes(t *testing.T) {
func TestWriteMergedStructureDoesNotDeleteForInvalidBucket(t *testing.T) {
oldID := "dataset_structure_old"
- eng := &fakeEngine{searchChunks: []map[string]interface{}{
+ eng := &fakeEngine{engineType: "infinity", searchChunks: []map[string]interface{}{
{
"id": oldID,
"compile_kwd": "hypergraph",
@@ -332,7 +332,7 @@ func TestWriteMergedStructureDoesNotDeleteForInvalidBucket(t *testing.T) {
// the source of truth, so a wrong tenant/kb can never cascade to per-document
// rows or to rows of other variants.
func TestDeleteMergedScopesToMergedWikiRows(t *testing.T) {
- eng := &fakeEngine{}
+ eng := &fakeEngine{engineType: "infinity"}
w := engineWriter{eng: eng}
if err := w.DeleteMerged(context.Background(), "t1", "kb1"); err != nil {
t.Fatalf("DeleteMerged: %v", err)
@@ -360,7 +360,7 @@ func TestDeleteMergedScopesToMergedWikiRows(t *testing.T) {
// slice (full rebuild) clears the full managed set — nav + wiki + structure
// compile_kwds — plus the structure scope_kwd="dataset" sweep.
func TestDeleteMergedForVariant_EmptySetClearsAll(t *testing.T) {
- eng := &fakeEngine{}
+ eng := &fakeEngine{engineType: "infinity"}
w := engineWriter{eng: eng}
if err := w.DeleteMergedForVariant(context.Background(), "t1", "kb1", nil); err != nil {
t.Fatalf("DeleteMergedForVariant: %v", err)
@@ -399,7 +399,7 @@ func TestDeleteMergedForVariant_EmptySetClearsAll(t *testing.T) {
// wiki clean: it must issue a delete scoped to available_int=1 and the wiki
// page/section kwds (wiki merged rows carry available_int=1).
func TestDeleteMergedForVariant_WikiScopesToAvailableOne(t *testing.T) {
- eng := &fakeEngine{}
+ eng := &fakeEngine{engineType: "infinity"}
w := engineWriter{eng: eng}
if err := w.DeleteMergedForVariant(context.Background(), "t1", "kb1", []kccommon.Variant{kccommon.VariantWiki}); err != nil {
t.Fatalf("DeleteMergedForVariant: %v", err)
@@ -421,7 +421,7 @@ func TestDeleteMergedForVariant_WikiScopesToAvailableOne(t *testing.T) {
// structure clean: dataset rows are tagged scope_kwd="dataset" (compile_kwd may
// be the inferred sub-kind), so the delete is scoped to scope_kwd="dataset".
func TestDeleteMergedForVariant_StructureScopesToDataset(t *testing.T) {
- eng := &fakeEngine{}
+ eng := &fakeEngine{engineType: "infinity"}
w := engineWriter{eng: eng}
if err := w.DeleteMergedForVariant(context.Background(), "t1", "kb1", []kccommon.Variant{kccommon.VariantStructure}); err != nil {
t.Fatalf("DeleteMergedForVariant: %v", err)
@@ -440,7 +440,7 @@ func TestDeleteMergedForVariant_StructureScopesToDataset(t *testing.T) {
// "dataset"). A wiki+tree clean must produce one wiki delete (available_int=1)
// and one nav delete (compile_kwd=dataset_nav, no available_int).
func TestDeleteMergedForVariant_WikiPlusNavCoversNavRows(t *testing.T) {
- eng := &fakeEngine{}
+ eng := &fakeEngine{engineType: "infinity"}
w := engineWriter{eng: eng}
if err := w.DeleteMergedForVariant(context.Background(), "t1", "kb1",
[]kccommon.Variant{kccommon.VariantWiki, kccommon.VariantTree}); err != nil {
@@ -472,7 +472,7 @@ func TestDeleteMergedForVariant_WikiPlusNavCoversNavRows(t *testing.T) {
// whose source docs are ALL deleted are ghosts and removed; rows that still have
// a surviving source doc are kept.
func TestDeleteStructureForDocs_RemovesGhostOnly(t *testing.T) {
- eng := &fakeEngine{searchChunks: []map[string]interface{}{
+ eng := &fakeEngine{engineType: "infinity", searchChunks: []map[string]interface{}{
{"id": "row_ghost", "source_doc_ids": []any{"d1", "d2"}}, // all deleted -> ghost
{"id": "row_keep", "source_doc_ids": []any{"d1", "d3"}}, // d3 survives -> keep
}}
@@ -492,7 +492,7 @@ func TestDeleteStructureForDocs_RemovesGhostOnly(t *testing.T) {
// TestDeleteStructureForDocs_NoGhostsNoDelete covers G3: when every structure row
// still has a surviving source doc, no delete is issued.
func TestDeleteStructureForDocs_NoGhostsNoDelete(t *testing.T) {
- eng := &fakeEngine{searchChunks: []map[string]interface{}{
+ eng := &fakeEngine{engineType: "infinity", searchChunks: []map[string]interface{}{
{"id": "row_keep", "source_doc_ids": []any{"d1", "d3"}},
}}
before := len(eng.deleteConds)
diff --git a/internal/ingestion/pipeline/pipeline.go b/internal/ingestion/pipeline/pipeline.go
index 09e2d4638b..70a37ac9c3 100644
--- a/internal/ingestion/pipeline/pipeline.go
+++ b/internal/ingestion/pipeline/pipeline.go
@@ -42,6 +42,7 @@ type Pipeline struct {
// the document table for existing status projections.
canvas *canvas.Canvas
tracker *canvas.RunTracker // optional injected; nil -> resolve at Run
+ kv *kvrocks.Client // optional; nil -> no tracker fallback (unit/test runs)
factory runtime.ComponentFactory // optional instance-scoped component factory
sink ProgressSink // optional progress sink; nil -> drop events (DB-independent)
}
@@ -51,12 +52,20 @@ type Pipeline struct {
type PipelineOption func(*Pipeline)
// WithRunTracker injects a RunTracker for progress mirroring (terminal
-// run-status marks). When unset, Run resolves one from the global Kvrocks
-// client.
+// run-status marks). When unset and no Kvrocks client is configured, Run does
+// not mirror progress.
func WithRunTracker(t *canvas.RunTracker) PipelineOption {
return func(p *Pipeline) { p.tracker = t }
}
+// WithKV injects the Kvrocks client used to resolve a RunTracker when none is
+// injected. It replaces the previous global kvrocks.Get() lookup, so the
+// pipeline no longer reads a process-wide singleton (arch_test.go guards that
+// rule). A nil client disables the RunTracker fallback.
+func WithKV(kv *kvrocks.Client) PipelineOption {
+ return func(p *Pipeline) { p.kv = kv }
+}
+
// WithDocumentID binds the pipeline's owning document so numeric progress can
// be mirrored back into the document table.
// Pass the empty string to disable the mirror (e.g. headless/test runs where
@@ -268,8 +277,8 @@ func (p *Pipeline) resolveTracker() *canvas.RunTracker {
if p.tracker != nil {
return p.tracker
}
- if kvrocks.Get() != nil {
- return canvas.NewRunTracker(defaultTrackerTTL)
+ if p.kv != nil {
+ return canvas.NewRunTrackerWithClient(p.kv.GetClient(), defaultTrackerTTL)
}
return nil
}
diff --git a/internal/ingestion/service/execute_memory_task_panic_test.go b/internal/ingestion/service/execute_memory_task_panic_test.go
index 2cc0065d2a..fcf8410221 100644
--- a/internal/ingestion/service/execute_memory_task_panic_test.go
+++ b/internal/ingestion/service/execute_memory_task_panic_test.go
@@ -5,6 +5,8 @@ import (
"testing"
"ragflow/internal/common"
+ "ragflow/internal/dao"
+ "ragflow/internal/engine"
"ragflow/internal/entity"
taskpkg "ragflow/internal/ingestion/task"
"ragflow/internal/ingestion/testutil"
@@ -28,10 +30,10 @@ import (
func TestExecuteMemoryTask_PanicDoesNotPropagate(t *testing.T) {
_ = testutil.SetupTestDB(t) // ensure DB helpers are initialized; not exercised here
- ingestor := NewIngestor("test", 1, []string{"pdf"})
+ ingestor := NewIngestor("test", 1, []string{"pdf"}, nil)
// Real (non-nil) memory service so executeMemoryTask reaches the
// runMemoryTask dispatch path instead of early-Acking on nil service.
- ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService()))
+ ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService(dao.DB, engine.Get())))
// Inject a panicking memory runner through the same seam used in production
// (defaultRunMemoryTask calls memorySvc.HandleSaveToMemoryTask).
ingestor.runMemoryTask = func(_ context.Context, _, _ string) (servicepkg.MemoryTaskDisposition, error) {
@@ -68,8 +70,8 @@ func TestHandleAndExecute_SurvivesMemoryPanicAndKeepsServing(t *testing.T) {
_, _, _, taskID := testutil.SeedTestData(t, db, testutil.WithPipelineID("flow-1"))
- ingestor := NewIngestor("test", 1, []string{"pdf"})
- ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService()))
+ ingestor := NewIngestor("test", 1, []string{"pdf"}, nil)
+ ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService(dao.DB, engine.Get())))
ingestor.runMemoryTask = func(_ context.Context, _, _ string) (servicepkg.MemoryTaskDisposition, error) {
panic("simulated memory extraction panic")
}
diff --git a/internal/ingestion/service/handle_and_execute_test.go b/internal/ingestion/service/handle_and_execute_test.go
index 9d67e3cac6..47a2f39498 100644
--- a/internal/ingestion/service/handle_and_execute_test.go
+++ b/internal/ingestion/service/handle_and_execute_test.go
@@ -24,6 +24,7 @@ import (
"ragflow/internal/common"
"ragflow/internal/dao"
+ "ragflow/internal/engine"
"ragflow/internal/entity"
"ragflow/internal/ingestion/testutil"
servicepkg "ragflow/internal/service"
@@ -33,7 +34,7 @@ import (
// need only the task id because execution input lives in the database.
func TestHandleAndExecute_MemoryTaskIDInvokesRunner(t *testing.T) {
ingestor := newUnitIngestor("test-mem-malformed", 1, nil)
- ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService()))
+ ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService(dao.DB, engine.Get())))
runnerCalled := false
ingestor.runMemoryTask = func(ctx context.Context, taskID, leaseOwner string) (servicepkg.MemoryTaskDisposition, error) {
@@ -81,7 +82,7 @@ func TestHandleAndExecute_MemoryExtractorDisabledAcks(t *testing.T) {
// an empty task ID is ack-skipped.
func TestHandleAndExecute_MemoryEmptyTaskIDAcks(t *testing.T) {
ingestor := newUnitIngestor("test-mem-empty-id", 1, nil)
- ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService()))
+ ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService(dao.DB, engine.Get())))
handle := &fakeTaskHandle{msg: common.TaskMessage{
TaskID: "",
@@ -422,7 +423,7 @@ func TestHandleAndExecute_DocumentDuplicateClaimRenewsLease(t *testing.T) {
// verifies DB lease admission controls settlement for duplicate or failed work.
func TestHandleAndExecute_MemoryUnsettledDispositionDefersToDurableRecovery(t *testing.T) {
ingestor := newUnitIngestor("test-mem-dup", 1, nil)
- ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService()))
+ ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService(dao.DB, engine.Get())))
ingestor.runMemoryTask = func(context.Context, string, string) (servicepkg.MemoryTaskDisposition, error) {
return servicepkg.MemoryTaskLeaveUnsettled, nil
}
@@ -443,7 +444,7 @@ func TestHandleAndExecute_MemoryUnsettledDispositionDefersToDurableRecovery(t *t
// still acknowledged after the service durably schedules its retry.
func TestHandleAndExecute_MemoryPersistedRetryAcks(t *testing.T) {
ingestor := newUnitIngestor("test-mem-retry", 1, nil)
- ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService()))
+ ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService(dao.DB, engine.Get())))
ingestor.runMemoryTask = func(context.Context, string, string) (servicepkg.MemoryTaskDisposition, error) {
return servicepkg.MemoryTaskAcknowledge, errors.New("retry scheduled")
}
diff --git a/internal/ingestion/service/ingestion_service.go b/internal/ingestion/service/ingestion_service.go
index 8fdce682a0..44e5e6ad76 100644
--- a/internal/ingestion/service/ingestion_service.go
+++ b/internal/ingestion/service/ingestion_service.go
@@ -28,9 +28,8 @@ import (
"ragflow/internal/common"
"ragflow/internal/dao"
- "ragflow/internal/engine"
- kvrocks "ragflow/internal/engine/kvrocks"
"ragflow/internal/entity"
+ "ragflow/internal/ingestion"
"ragflow/internal/ingestion/knowledge_compile"
pipelinepkg "ragflow/internal/ingestion/pipeline"
taskpkg "ragflow/internal/ingestion/task"
@@ -139,6 +138,14 @@ type Ingestor struct {
// DB fallback when Redis is unavailable. Tests may override this to
// simulate cancel without Redis.
cancelCheck func(ctx context.Context, taskID string) bool
+
+ // deps bundles the process-wide singletons the ingestor depends on
+ // (document engine, relational handle, task broker, kvrocks). It is
+ // injected at construction (NewIngestor) rather than read from the
+ // package globals (arch_test.go guards that rule). A nil bundle is
+ // normalized to an empty Deps; a nil sub-value disables the matching
+ // capability (e.g. a nil DB skips startup task scheduling).
+ deps *ingestion.Deps
}
type worker struct {
@@ -156,10 +163,17 @@ type pendingDocumentCompileEvent struct {
taskTypes []string
}
-func NewIngestor(name string, maxConcurrency int32, supportedTypes []string) *Ingestor {
+// NewIngestor constructs the ingestion consumer. db, mq, and kv are the
+// process-wide relational handle, message-queue engine, and kvrocks client;
+// they are resolved once at the server assembly point and injected here rather
+// than read from the package globals (arch_test.go guards that rule).
+func NewIngestor(name string, maxConcurrency int32, supportedTypes []string, deps *ingestion.Deps) *Ingestor {
if maxConcurrency <= 0 {
maxConcurrency = int32(runtime.NumCPU())
}
+ if deps == nil {
+ deps = &ingestion.Deps{}
+ }
ctx, cancel := context.WithCancel(context.Background())
dispatchCtx, dispatchCancel := context.WithCancel(context.Background())
id := common.GenerateUUID()
@@ -183,6 +197,7 @@ func NewIngestor(name string, maxConcurrency int32, supportedTypes []string) *In
heartbeatInterval: defaultHeartbeatInterval,
memoryReconcileInterval: defaultMemoryTaskReconcileInterval,
memoryReconcileBatchSize: defaultMemoryTaskReconcileBatchSize,
+ deps: deps,
}
ingestor.runDocumentTask = ingestor.defaultRunDocumentTask
ingestor.runMemoryTask = ingestor.defaultRunMemoryTask
@@ -221,14 +236,14 @@ func (e *Ingestor) Start() error {
// pools, compile consumers, or consume loops, and the first initialization
// error is retained and returned to every later caller.
func (e *Ingestor) start() error {
- msgQueueEngine := engine.GetMessageQueueEngine()
+ msgQueueEngine := e.deps.MQ
if msgQueueEngine == nil {
return fmt.Errorf("message queue engine not initialized; run engine.InitMessageQueue first")
}
if err := msgQueueEngine.InitConsumer(common.TaskSubject); err != nil {
return err
}
- if dao.DB != nil {
+ if e.deps.DB != nil {
if err := e.ingestionTaskSvc.ScheduleCreatedTasks(e.ctx); err != nil {
common.Warn(fmt.Sprintf("schedule CREATED ingestion tasks at startup: %v", err))
}
@@ -253,7 +268,7 @@ func (e *Ingestor) start() error {
// startMemoryTaskReconciler starts the owned loop that recovers durable memory
// tasks whose initial wake-up was lost or whose retry became due.
func (e *Ingestor) startMemoryTaskReconciler() {
- if e.memorySvc == nil || dao.DB == nil || e.reconcileMemoryTasks == nil {
+ if e.memorySvc == nil || e.deps.DB == nil || e.reconcileMemoryTasks == nil {
return
}
e.memoryReconcileWg.Add(1)
@@ -301,7 +316,7 @@ func (e *Ingestor) defaultReconcileMemoryTasks(ctx context.Context) error {
// and limiting this ingestor to one outstanding broker request.
func (e *Ingestor) consumeLoop() {
defer e.dispatcherWg.Done()
- msgQueueEngine := engine.GetMessageQueueEngine()
+ msgQueueEngine := e.deps.MQ
for {
var w *worker
select {
@@ -406,17 +421,17 @@ func (e *Ingestor) SetKnowledgeCompileConcurrency(n int32) {
// is logged and the pipeline continues to write available_int=0 compiled
// chunks; they just won't be merged until a scheduler is available.
func (e *Ingestor) startDatasetKnowledgeCompile() {
- mq := engine.GetMessageQueueEngine()
+ mq := e.deps.MQ
if mq == nil {
common.Warn("message queue not initialized; dataset-level compile consumer disabled")
return
}
knowledge_compile.SetModelConfig(e.kcLLMID, e.kcEmbedding)
- if err := knowledge_compile.Provision(e.ctx, mq, dao.DB); err != nil {
+ if err := knowledge_compile.Provision(e.ctx, mq, e.deps.DB); err != nil {
common.Warn(fmt.Sprintf("dataset-level compile consumer unavailable; compiled chunks will not be merged: %v", err))
return
}
- e.knowledgeCompile = knowledge_compile.NewConsumer(knowledge_compile.DefaultClaimer())
+ e.knowledgeCompile = e.newKnowledgeCompileConsumer()
n := e.kcConcurrency
if n <= 0 {
n = int32(runtime.NumCPU())
@@ -432,6 +447,19 @@ func (e *Ingestor) startDatasetKnowledgeCompile() {
}
}
+// newKnowledgeCompileConsumer builds the dataset-level knowledge-compile
+// consumer with its storage dependencies injected from the ingestor's Deps
+// bundle. The consumer fails fast on storage access when WithStorage is omitted
+// (see knowledge_compile.NewConsumer), so the doc engine + db MUST be threaded
+// through here — never construct it with a bare NewConsumer in production. The
+// assembly point can assert c.IsStorageConfigured() before relying on merges.
+func (e *Ingestor) newKnowledgeCompileConsumer() *knowledge_compile.Consumer {
+ return knowledge_compile.NewConsumer(
+ knowledge_compile.DefaultClaimer(),
+ knowledge_compile.WithStorage(e.deps.Engine, e.deps.DB),
+ )
+}
+
// compileLoop is one of the owned dataset-level compile workers. It runs the
// Consumer's Run loop (poll scheduling rows + NATS notify → claim closed batch
// → merge → ack). kcConcurrency such goroutines run in parallel, so different
@@ -778,7 +806,7 @@ func (e *Ingestor) markStopped(ctx context.Context, taskID string) bool {
common.Error(fmt.Sprintf("markStopped: MarkStopped task %s: %v", taskID, err), err)
return false
}
- if rc := kvrocks.Get(); rc != nil {
+ if rc := e.deps.KV; rc != nil {
utility.BestEffort(fmt.Sprintf("clear cancel flag for %s", taskID), func() error {
rc.Delete(ctx, fmt.Sprintf("%s-cancel", taskID))
return nil // Delete returns bool; the bool does not distinguish "not found" from "error"
@@ -828,7 +856,7 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) bool
// by the DB status (STOPPING), which defaultCancelCheck falls back to
// when the Redis flag is absent. Clearing a stale flag here is safe:
// a genuine concurrent cancel sets the task to STOPPING in DB.
- if rc := kvrocks.Get(); rc != nil {
+ if rc := e.deps.KV; rc != nil {
key := fmt.Sprintf("%s-cancel", task.ID)
utility.BestEffort(fmt.Sprintf("clear stale cancel flag for %s", task.ID), func() error {
rc.Delete(ctx, key)
@@ -1048,7 +1076,7 @@ func (e *Ingestor) ackOrNack(taskCtx *taskpkg.TaskContext, terminal bool) {
// task status in DB when Redis is unavailable — a STOPPING status
// (set by RequestStop) is treated as a cancel signal.
func (e *Ingestor) defaultCancelCheck(ctx context.Context, taskID string) bool {
- rc := kvrocks.Get()
+ rc := e.deps.KV
if rc != nil {
if ok, _ := rc.Exist(ctx, fmt.Sprintf("%s-cancel", taskID)); ok {
return true
@@ -1171,7 +1199,7 @@ func (e *Ingestor) releaseTask(taskID string) {
func (e *Ingestor) defaultRunDocumentTask(ctx context.Context, ingestionTask *entity.IngestionTask) error {
const documentBulkSize = 32
- docTaskCtx, err := taskpkg.LoadFromIngestionTask(ctx, ingestionTask)
+ docTaskCtx, err := taskpkg.LoadFromIngestionTask(ctx, e.deps.DB, ingestionTask)
if err != nil {
return fmt.Errorf("load task context for %s: %w", ingestionTask.ID, err)
}
@@ -1195,7 +1223,7 @@ func (e *Ingestor) defaultRunDocumentTask(ctx context.Context, ingestionTask *en
// text, source-chunk references, images, or vectors, so sending the whole
// document through one Elasticsearch bulk request can exceed the request
// size limit even when the individual chunks are valid.
- executor, err := taskpkg.NewPipelineExecutor(docTaskCtx, pipelineID, documentBulkSize)
+ executor, err := taskpkg.NewPipelineExecutor(docTaskCtx, pipelineID, documentBulkSize, e.deps)
if err != nil {
return err
}
@@ -1278,7 +1306,7 @@ func (e *Ingestor) recordTerminalPipelineLog(ctx context.Context, ingestionTask
common.Warn(fmt.Sprintf("record terminal pipeline log for task %s: missing run identity", ingestionTask.ID))
return
}
- run, err := dao.NewPipelineOperationLogDAO().GetByID(ctx, dao.DB, *ingestionTask.PipelineLogID)
+ run, err := dao.NewPipelineOperationLogDAO().GetByID(ctx, e.deps.DB, *ingestionTask.PipelineLogID)
if err != nil {
common.Warn(fmt.Sprintf("record terminal pipeline log for task %s: load run %s: %v", ingestionTask.ID, *ingestionTask.PipelineLogID, err))
return
@@ -1293,7 +1321,7 @@ func (e *Ingestor) recordTerminalPipelineLog(ctx context.Context, ingestionTask
Status: status,
PipelineLogID: *ingestionTask.PipelineLogID,
}
- if err := taskpkg.RecordPipelineLog(ctx, dao.DB, input); err != nil {
+ if err := taskpkg.RecordPipelineLog(ctx, e.deps.DB, input); err != nil {
common.Warn(fmt.Sprintf("record terminal pipeline log for task %s document %s: %v", ingestionTask.ID, ingestionTask.DocumentID, err))
return
}
diff --git a/internal/ingestion/service/knowledge_compile_wiring_test.go b/internal/ingestion/service/knowledge_compile_wiring_test.go
new file mode 100644
index 0000000000..146e6909d6
--- /dev/null
+++ b/internal/ingestion/service/knowledge_compile_wiring_test.go
@@ -0,0 +1,68 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+//
+// limitations under the License.
+//
+
+package service
+
+import (
+ "testing"
+
+ "github.com/glebarez/sqlite"
+ "gorm.io/gorm"
+
+ "ragflow/internal/engine"
+)
+
+// stubEngine embeds engine.DocEngine so it satisfies the interface at compile
+// time (its methods delegate to the embedded nil value); only the methods the
+// test observes are overridden. It exists purely to give the wiring test a
+// comparable, non-nil engine value to assert propagation with.
+type stubEngine struct{ engine.DocEngine }
+
+func (stubEngine) GetType() string { return "stub" }
+
+// TestNewKnowledgeCompileConsumerWiresStorage guards the production assembly:
+// the dataset-level compile consumer MUST be built with WithStorage so its
+// reader/writer/contribution store are engine-backed. A regression that drops
+// WithStorage — or threads the wrong/nil engine or db through it — would
+// silently disable every production merge; this catches it by asserting the
+// exact deps the ingestor provides actually reach the consumer's reader/writer.
+func TestNewKnowledgeCompileConsumerWiresStorage(t *testing.T) {
+ ingestor := newUnitIngestor("test-kc-wiring", 1, nil)
+ // Provide the deps the production path threads through WithStorage and
+ // assert they propagate verbatim to the consumer (not just the flag).
+ eng := stubEngine{}
+ db, err := gorm.Open(sqlite.Open("file:test-kc-wiring?mode=memory&cache=shared"), &gorm.Config{})
+ if err != nil {
+ t.Fatalf("open sqlite: %v", err)
+ }
+ ingestor.deps.Engine = eng
+ ingestor.deps.DB = db
+
+ consumer := ingestor.newKnowledgeCompileConsumer()
+ if consumer == nil {
+ t.Fatal("newKnowledgeCompileConsumer returned nil")
+ }
+ if !consumer.IsStorageConfigured() {
+ t.Fatal("production consumer must be storage-configured (WithStorage applied)")
+ }
+ if consumer.StorageEngine() != eng {
+ t.Fatal("WithStorage did not propagate the ingestor's engine to the consumer reader/writer")
+ }
+ if consumer.StorageDB() != db {
+ t.Fatal("WithStorage did not propagate the ingestor's db to the consumer")
+ }
+}
diff --git a/internal/ingestion/service/test_helpers.go b/internal/ingestion/service/test_helpers_test.go
similarity index 63%
rename from internal/ingestion/service/test_helpers.go
rename to internal/ingestion/service/test_helpers_test.go
index 008f862399..3db110bc97 100644
--- a/internal/ingestion/service/test_helpers.go
+++ b/internal/ingestion/service/test_helpers_test.go
@@ -5,7 +5,7 @@
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
-// http://www.apache.org/licenses/LICENSE-2.0
+// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
@@ -16,9 +16,12 @@
package service
-// ──────────────────────────────────────────────────────────
-// Ingestor Test Helpers
-// ──────────────────────────────────────────────────────────
+import (
+ "ragflow/internal/dao"
+ "ragflow/internal/engine"
+ kvrocks "ragflow/internal/engine/kvrocks"
+ "ragflow/internal/ingestion"
+)
// NewTestIngestor creates a new Ingestor for testing with default values.
func NewTestIngestor() *Ingestor {
@@ -30,9 +33,18 @@ func NewTestIngestor() *Ingestor {
// IngestorOption configures a test Ingestor.
type IngestorOption func(*Ingestor)
+// newUnitIngestor builds a unit-test Ingestor. The dependency bundle reads the
+// process-wide handles the surrounding test swaps in (dao.DB via
+// testutil.ReplaceDBForTest, the message queue via engine.SetMessageQueueEngine),
+// so real values flow into the ingestor while nil values exercise the
+// nil-dependency guards. Engine stays nil: executor tests stub the index writer
+// or assert the downstream no-engine failure.
func newUnitIngestor(name string, maxConcurrency int32, supportedTypes []string) *Ingestor {
- ingestor := NewIngestor(name, maxConcurrency, supportedTypes)
- return ingestor
+ return NewIngestor(name, maxConcurrency, supportedTypes, &ingestion.Deps{
+ DB: dao.DB,
+ MQ: engine.GetMessageQueueEngine(),
+ KV: kvrocks.Get(),
+ })
}
// SetupTestIngestor creates a new test Ingestor with the given options.
diff --git a/internal/ingestion/service/worker_dispatcher_integration_test.go b/internal/ingestion/service/worker_dispatcher_integration_test.go
index 7ef954e143..010d973331 100644
--- a/internal/ingestion/service/worker_dispatcher_integration_test.go
+++ b/internal/ingestion/service/worker_dispatcher_integration_test.go
@@ -122,23 +122,26 @@ func TestIntegration_MultiInstanceSharedConsumerNoOverExecution(t *testing.T) {
return nil
}
+ // Point the global MQ engine to engine1 then construct ingestor1, then to
+ // engine2 and construct ingestor2. newUnitIngestor snapshots the global MQ
+ // engine at construction time and Start reads the injected deps.MQ, so the
+ // engine must be set on the global before each ingestor is built (not after).
+ previousEngine := engine.GetMessageQueueEngine()
+ t.Cleanup(func() { engine.SetMessageQueueEngine(previousEngine) })
+
+ engine.SetMessageQueueEngine(engine1)
ingestor1 := newUnitIngestor("instance-1", 2, []string{"pdf"})
ingestor1.runDocumentTask = runner
+ engine.SetMessageQueueEngine(engine2)
ingestor2 := newUnitIngestor("instance-2", 2, []string{"pdf"})
ingestor2.runDocumentTask = runner
- // Point global MQ engine to engine1 then start ingestor1, then engine2 and start ingestor2
- previousEngine := engine.GetMessageQueueEngine()
- t.Cleanup(func() { engine.SetMessageQueueEngine(previousEngine) })
-
- engine.SetMessageQueueEngine(engine1)
if err := ingestor1.Start(); err != nil {
t.Fatalf("ingestor1.Start: %v", err)
}
t.Cleanup(func() { ingestor1.Stop(context.Background()) })
- engine.SetMessageQueueEngine(engine2)
if err := ingestor2.Start(); err != nil {
t.Fatalf("ingestor2.Start: %v", err)
}
@@ -202,6 +205,12 @@ func TestIntegration_SlowTaskHeartbeatPreventsPrematureRedelivery(t *testing.T)
t.Fatalf("schedule task %s: %v", taskID, err)
}
+ // Set the global MQ engine before constructing the ingestor: newUnitIngestor
+ // snapshots the global at construction and Start reads the injected deps.MQ.
+ previousEngine := engine.GetMessageQueueEngine()
+ engine.SetMessageQueueEngine(mq)
+ t.Cleanup(func() { engine.SetMessageQueueEngine(previousEngine) })
+
ingestor := newUnitIngestor("slow-task-ingestor", 1, []string{"pdf"})
ingestor.heartbeatInterval = 1 * time.Second // comfortably below BackOff[0] = 5s
@@ -220,10 +229,6 @@ func TestIntegration_SlowTaskHeartbeatPreventsPrematureRedelivery(t *testing.T)
}
}
- previousEngine := engine.GetMessageQueueEngine()
- engine.SetMessageQueueEngine(mq)
- t.Cleanup(func() { engine.SetMessageQueueEngine(previousEngine) })
-
if err := ingestor.Start(); err != nil {
t.Fatalf("ingestor.Start: %v", err)
}
diff --git a/internal/ingestion/task/debug_log_sink_test.go b/internal/ingestion/task/debug_log_sink_test.go
index 586ed8ea87..1262817932 100644
--- a/internal/ingestion/task/debug_log_sink_test.go
+++ b/internal/ingestion/task/debug_log_sink_test.go
@@ -202,7 +202,7 @@ func TestDebugLogSink_OnComponentTotalIsNoOp(t *testing.T) {
func TestPipelineExecutor_DebugRunWritesLogViaSink(t *testing.T) {
ctx := t.Context()
taskCtx := NewDebugTaskContext("tenant-1", "c1", "doc.pdf", nil)
- exec, err := NewPipelineExecutor(taskCtx, "c1", 0)
+ exec, err := NewPipelineExecutor(taskCtx, "c1", 0, testPipelineDeps())
if err != nil {
t.Fatalf("NewPipelineExecutor: %v", err)
}
diff --git a/internal/ingestion/task/debug_pages_integration_test.go b/internal/ingestion/task/debug_pages_integration_test.go
index 3ff13703e7..98ab455e44 100644
--- a/internal/ingestion/task/debug_pages_integration_test.go
+++ b/internal/ingestion/task/debug_pages_integration_test.go
@@ -191,7 +191,7 @@ func TestExecute_DebugViaEntry_HonorsPagesCap_Integration(t *testing.T) {
defer func() { common.Logger = oldLogger }()
// Uncapped baseline: explicit pages=all → executor respects override.
- uncappedExec, err := NewPipelineExecutor(newDebugCtx(allPages), canvasID, 0)
+ uncappedExec, err := NewPipelineExecutor(newDebugCtx(allPages), canvasID, 0, testPipelineDeps())
if err != nil {
t.Fatalf("NewPipelineExecutor (uncapped): %v", err)
}
@@ -201,7 +201,7 @@ func TestExecute_DebugViaEntry_HonorsPagesCap_Integration(t *testing.T) {
}
// Capped: no pages override → executor injects [[1, debugPageCapPages]].
- cappedExec, err := NewPipelineExecutor(newDebugCtx(nil), canvasID, 0)
+ cappedExec, err := NewPipelineExecutor(newDebugCtx(nil), canvasID, 0, testPipelineDeps())
if err != nil {
t.Fatalf("NewPipelineExecutor (capped): %v", err)
}
diff --git a/internal/ingestion/task/debug_test.go b/internal/ingestion/task/debug_test.go
index 34f08a4e77..b250b4e93e 100644
--- a/internal/ingestion/task/debug_test.go
+++ b/internal/ingestion/task/debug_test.go
@@ -75,7 +75,7 @@ func TestExecute_DebugViaEntry(t *testing.T) {
insertCalled := false
- exec, err := NewPipelineExecutor(taskCtx, "canvas-1", 0)
+ exec, err := NewPipelineExecutor(taskCtx, "canvas-1", 0, testPipelineDeps())
if err != nil {
t.Fatalf("NewPipelineExecutor: %v", err)
}
@@ -313,7 +313,7 @@ func TestExecute_DebugViaEntry_NoKnowledgeCompilePublish(t *testing.T) {
taskCtx := NewDebugTaskContext("t1", "canvas-1", "doc.pdf", []byte("page one\fpage two\fpage three"))
- exec, err := NewPipelineExecutor(taskCtx, "canvas-1", 0)
+ exec, err := NewPipelineExecutor(taskCtx, "canvas-1", 0, testPipelineDeps())
if err != nil {
t.Fatalf("NewPipelineExecutor: %v", err)
}
diff --git a/internal/ingestion/task/embedder.go b/internal/ingestion/task/embedder.go
index 38c3dcef01..8c94c9cdde 100644
--- a/internal/ingestion/task/embedder.go
+++ b/internal/ingestion/task/embedder.go
@@ -40,6 +40,8 @@ import (
componentpkg "ragflow/internal/ingestion/component"
"ragflow/internal/service"
"ragflow/internal/tokenizer"
+
+ "gorm.io/gorm"
)
type embedder struct {
@@ -666,6 +668,24 @@ func newEmbedderResolver(
}
}
+// embedderResolverDB is the relational handle the production embedder resolver
+// uses to read a knowledgebase's configured embedding model id. It is bound once
+// at the server assembly point via InitEmbedderResolver; a nil value makes the
+// resolver return an error at call time instead of silently reading zero rows.
+// It replaces the previous process-wide dao.DB lookup so the task package (the
+// composition root for ingestion runs) no longer reaches the global directly
+// (arch_test.go guards that rule).
+var embedderResolverDB *gorm.DB
+
+// InitEmbedderResolver binds the relational handle used by the production
+// embedder resolver. The task package is the composition root for ingestion
+// runs; the server assembly point passes the process-wide db here rather than
+// letting the resolver read dao.DB. Call it once during server startup, before
+// any ingestion run.
+func InitEmbedderResolver(db *gorm.DB) {
+ embedderResolverDB = db
+}
+
// init wires the production embedder resolver into the component package. The
// component package must not import internal/service (dependency direction),
// so the concrete resolver is injected here - the task package is the
@@ -674,7 +694,10 @@ func init() {
modelFactory := service.NewModelFactory()
componentpkg.DefaultEmbedderResolver = newEmbedderResolver(
func(ctx context.Context, kbID string) (string, error) {
- kb, err := dao.NewKnowledgebaseDAO().GetByID(ctx, dao.DB, kbID)
+ if embedderResolverDB == nil {
+ return "", fmt.Errorf("embedder: resolver not initialized; call task.InitEmbedderResolver(dao.DB) at server startup")
+ }
+ kb, err := dao.NewKnowledgebaseDAO().GetByID(ctx, embedderResolverDB, kbID)
if err != nil {
return "", err
}
diff --git a/internal/ingestion/task/embedder_test.go b/internal/ingestion/task/embedder_test.go
index 16252cf92f..a508d1189c 100644
--- a/internal/ingestion/task/embedder_test.go
+++ b/internal/ingestion/task/embedder_test.go
@@ -22,6 +22,7 @@ import (
"testing"
"ragflow/internal/entity/models"
+ componentpkg "ragflow/internal/ingestion/component"
)
func makeEmbeddingModelForResolver() *models.EmbeddingModel {
@@ -78,6 +79,23 @@ func TestEmbedderResolver_EmptyKBEmbdIDReturnsNil(t *testing.T) {
}
}
+// TestEmbedderResolverFailsFastWhenUninitialized guards the nil-db guard in the
+// production resolver closure: if the server assembly point forgets to call
+// InitEmbedderResolver, the resolver must fail loud instead of dereferencing a
+// nil *gorm.DB at run time.
+func TestEmbedderResolverFailsFastWhenUninitialized(t *testing.T) {
+ saved := embedderResolverDB
+ defer func() { embedderResolverDB = saved }()
+ embedderResolverDB = nil
+ if componentpkg.DefaultEmbedderResolver == nil {
+ t.Fatal("DefaultEmbedderResolver was not wired by init()")
+ }
+ _, _, err := componentpkg.DefaultEmbedderResolver(t.Context(), "tenant-1", "kb-1")
+ if err == nil {
+ t.Fatal("expected fail-fast error when the resolver db is uninitialized, got nil")
+ }
+}
+
// stubDriver records the texts passed to Embed for verification.
type stubDriver struct {
capturedTexts []string
diff --git a/internal/ingestion/task/knowledge_compiler_wiring.go b/internal/ingestion/task/knowledge_compiler_wiring.go
index 5538a5ac3d..281cd92660 100644
--- a/internal/ingestion/task/knowledge_compiler_wiring.go
+++ b/internal/ingestion/task/knowledge_compiler_wiring.go
@@ -51,6 +51,29 @@ import (
// (pipeline / downstream tokenizer) handles any persistence, exactly as it does
// for ordinary chunks.
+// kcWiringDeps binds the relational handle and document engine the
+// knowledge-compiler composition-root wiring reads. The task package is the
+// composition root for ingestion runs, so these are bound once at the server
+// assembly point via InitKnowledgeCompilerDeps rather than read from the package
+// globals (arch_test.go guards that rule). A nil Engine makes the Wiki dirty
+// compiler and the Deps resolver fail loud; a nil DB makes the resolver panic on
+// a missing Infinity chunk table, which the runner surfaces as a failed compile.
+var kcWiringDeps struct {
+ DB *gorm.DB
+ Engine engine.DocEngine
+}
+
+// InitKnowledgeCompilerDeps binds the relational handle and document engine used
+// by the knowledge-compiler composition-root wiring (Wiki dirty compiler, Wiki
+// map version store, knowledge-compiler Deps resolver). The server assembly
+// point passes the process-wide db and engine here instead of the wiring reading
+// kcWiringDeps.DB / kcWiringDeps.Engine directly. Call it once during server startup, before any
+// ingestion run.
+func InitKnowledgeCompilerDeps(db *gorm.DB, eng engine.DocEngine) {
+ kcWiringDeps.DB = db
+ kcWiringDeps.Engine = eng
+}
+
func init() {
kc.SetDepsResolver(newKnowledgeCompilerDepsResolver())
kc.SetGroupResolver(newKnowledgeCompilerGroupResolver())
@@ -59,7 +82,7 @@ func init() {
}
func compileDirtyWikiDocument(ctx context.Context, request knowledge_compile.WikiDirtyRequest) error {
- doc, err := dao.NewDocumentDAO().GetByID(ctx, dao.DB, request.DocumentID)
+ doc, err := dao.NewDocumentDAO().GetByID(ctx, kcWiringDeps.DB, request.DocumentID)
if err != nil {
if err == gorm.ErrRecordNotFound {
return nil
@@ -100,7 +123,7 @@ func compileDirtyWikiDocument(ctx context.Context, request knowledge_compile.Wik
if err != nil {
return err
}
- output, err := component.Invoke(ctx, dao.DB, map[string]any{
+ output, err := component.Invoke(ctx, kcWiringDeps.DB, map[string]any{
"chunks": sourceChunks,
"tenant_id": request.TenantID,
"dataset_id": request.DatasetID,
@@ -133,7 +156,7 @@ func loadWikiCompilerParams(ctx context.Context, doc *entity.Document) (map[stri
if doc.PipelineID == nil || strings.TrimSpace(*doc.PipelineID) == "" {
return map[string]any{}, nil
}
- canvas, err := dao.NewUserCanvasDAO().GetByID(ctx, dao.DB, *doc.PipelineID)
+ canvas, err := dao.NewUserCanvasDAO().GetByID(ctx, kcWiringDeps.DB, *doc.PipelineID)
if err != nil {
if err == dao.ErrUserCanvasNotFound {
return map[string]any{}, nil
@@ -183,7 +206,7 @@ func resolveWikiTemplateIDs(ctx context.Context, tenantID string, params map[str
groupIDs := stringValues(params["compilation_template_group_id"])
groupIDs = append(groupIDs, stringValues(params["compilation_template_group_ids"])...)
if len(groupIDs) > 0 {
- resolved, err := dao.NewCompilationTemplateDAO().ResolveGroupTemplateIDs(ctx, dao.DB, tenantID, uniqueStrings(groupIDs))
+ resolved, err := dao.NewCompilationTemplateDAO().ResolveGroupTemplateIDs(ctx, kcWiringDeps.DB, tenantID, uniqueStrings(groupIDs))
if err != nil {
return nil, err
}
@@ -194,7 +217,7 @@ func resolveWikiTemplateIDs(ctx context.Context, tenantID string, params map[str
return nil, nil
}
var templates []entity.CompilationTemplate
- if err := dao.DB.WithContext(ctx).
+ if err := kcWiringDeps.DB.WithContext(ctx).
Where("id IN ? AND status = ?", ids, string(entity.StatusValid)).Find(&templates).Error; err != nil {
return nil, err
}
@@ -212,7 +235,7 @@ func resolveWikiTemplateIDs(ctx context.Context, tenantID string, params map[str
}
func loadActiveSourceChunks(ctx context.Context, request knowledge_compile.WikiDirtyRequest) ([]map[string]any, error) {
- docEngine := engine.Get()
+ docEngine := kcWiringDeps.Engine
if docEngine == nil {
return nil, fmt.Errorf("Wiki dirty compile: document engine is unavailable")
}
@@ -270,7 +293,7 @@ func wikiCompiledRows(output map[string]any) []map[string]any {
func replaceDirtyWikiProducts(ctx context.Context, request knowledge_compile.WikiDirtyRequest, rows []map[string]any, affectedSlugs, removedSlugs []string, activeStates []kc.WikiMapActiveState, fullReplace bool) error {
var dirty entity.WikiDocumentDirty
- if err := dao.DB.WithContext(ctx).Where("document_id = ?", request.DocumentID).First(&dirty).Error; err != nil {
+ if err := kcWiringDeps.DB.WithContext(ctx).Where("document_id = ?", request.DocumentID).First(&dirty).Error; err != nil {
if err == gorm.ErrRecordNotFound {
return nil
}
@@ -279,7 +302,7 @@ func replaceDirtyWikiProducts(ctx context.Context, request knowledge_compile.Wik
if dirty.Revision != request.Revision {
return nil
}
- docEngine := engine.Get()
+ docEngine := kcWiringDeps.Engine
if docEngine == nil {
return fmt.Errorf("Wiki dirty compile: document engine is unavailable")
}
@@ -384,7 +407,7 @@ func persistDirtyWikiActiveStates(ctx context.Context, request knowledge_compile
return nil
}
var dirty entity.WikiDocumentDirty
- if err := dao.DB.WithContext(ctx).Where("document_id = ?", request.DocumentID).First(&dirty).Error; err != nil {
+ if err := kcWiringDeps.DB.WithContext(ctx).Where("document_id = ?", request.DocumentID).First(&dirty).Error; err != nil {
if err == gorm.ErrRecordNotFound {
return nil
}
@@ -393,7 +416,7 @@ func persistDirtyWikiActiveStates(ctx context.Context, request knowledge_compile
if dirty.Revision != request.Revision {
return nil
}
- docEngine := engine.Get()
+ docEngine := kcWiringDeps.Engine
if docEngine == nil {
return fmt.Errorf("Wiki dirty compile: document engine is unavailable")
}
@@ -407,6 +430,7 @@ func putWikiActiveStates(ctx context.Context, docEngine engine.DocEngine, states
// Every state comes from one document's compile, so they share a tenant and
// dataset: the resolver only runs when the Infinity chunk table is missing.
store, ok := knowledge_compile.NewWikiMapVersionStoreWithVectorSizeResolver(
+ kcWiringDeps.DB,
docEngine,
wikiMapVectorSizeResolver(states[0].TenantID, states[0].DatasetID),
).(kc.WikiMapActiveStateStore)
@@ -427,7 +451,7 @@ func putWikiActiveStates(ctx context.Context, docEngine engine.DocEngine, states
func wikiMapVectorSizeResolver(tenantID, datasetID string) func(context.Context) (int, error) {
modelFactory := service.NewModelFactory()
return func(ctx context.Context) (int, error) {
- kb, err := dao.NewKnowledgebaseDAO().GetByID(ctx, dao.DB, datasetID)
+ kb, err := dao.NewKnowledgebaseDAO().GetByID(ctx, kcWiringDeps.DB, datasetID)
if err != nil {
return 0, fmt.Errorf("Wiki MAP chunk store: load dataset %s: %w", datasetID, err)
}
@@ -588,14 +612,14 @@ func newKnowledgeCompilerDepsResolver() kc.DepsResolver {
}
embedder := &kcEmbedder{factory: modelFactory, tenantID: tenantID, embdID: embeddingModel}
- mapStore := knowledge_compile.NewWikiMapVersionStoreWithVectorSizeResolver(engine.Get(), func(ctx context.Context) (int, error) {
+ mapStore := knowledge_compile.NewWikiMapVersionStoreWithVectorSizeResolver(kcWiringDeps.DB, kcWiringDeps.Engine, func(ctx context.Context) (int, error) {
return embedderVectorSize(ctx, embedder)
})
return kc.Deps{
Chat: &kcChatInvoker{factory: modelFactory, tenantID: tenantID, llmID: llmID},
Embed: embedder,
- WikiPages: &kcWikiPageStore{docEngine: engine.Get()},
+ WikiPages: &kcWikiPageStore{docEngine: kcWiringDeps.Engine},
WikiMapVersions: mapStore,
// HistoricalKNN / Redis are optional (wiki historical dedup,
// datasetnav lock). They are wired separately when the
diff --git a/internal/ingestion/task/pipeline_e2e_test.go b/internal/ingestion/task/pipeline_e2e_test.go
index 90569a5492..7ce27face9 100644
--- a/internal/ingestion/task/pipeline_e2e_test.go
+++ b/internal/ingestion/task/pipeline_e2e_test.go
@@ -209,7 +209,7 @@ func TestPipelineE2E_PipelineExecutor(t *testing.T) {
if err != nil {
t.Fatalf("GetByID failed: %v", err)
}
- taskCtx, err := LoadFromIngestionTask(ctx, ingestionTask)
+ taskCtx, err := LoadFromIngestionTask(ctx, dao.DB, ingestionTask)
if err != nil {
t.Fatalf("LoadFromIngestionTask failed: %v", err)
}
diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.go
index 1b6b8caae2..5d7634eddc 100644
--- a/internal/ingestion/task/pipeline_executor.go
+++ b/internal/ingestion/task/pipeline_executor.go
@@ -30,6 +30,7 @@ import (
"ragflow/internal/engine"
enginetypes "ragflow/internal/engine/types"
"ragflow/internal/entity"
+ "ragflow/internal/ingestion"
"ragflow/internal/ingestion/chunkcache"
"ragflow/internal/ingestion/component"
"ragflow/internal/ingestion/component/globals"
@@ -73,6 +74,7 @@ type PipelineExecutor struct {
taskCtx *TaskContext
canvasID string
docBulkSize int
+ deps *ingestion.Deps
indexWriter *chunkIndexWriter
deleteChunksFunc DeleteChunksFunc
@@ -137,10 +139,14 @@ func insertFinalChunksForIngestion(eng engine.DocEngine, language string) Insert
}
}
+// NewPipelineExecutor builds the executor for one document run. deps carries
+// the process-wide singletons (doc engine, database handle) resolved at the
+// server assembly point; the executor never resolves them itself.
func NewPipelineExecutor(
taskCtx *TaskContext,
canvasID string,
docBulkSize int,
+ deps *ingestion.Deps,
) (*PipelineExecutor, error) {
if err := validateTaskContext(taskCtx); err != nil {
return nil, err
@@ -148,18 +154,22 @@ func NewPipelineExecutor(
if strings.TrimSpace(canvasID) == "" {
return nil, fmt.Errorf("pipeline executor: empty canvas id")
}
+ if deps == nil {
+ return nil, fmt.Errorf("pipeline executor: nil dependency bundle")
+ }
svc := &PipelineExecutor{
taskCtx: taskCtx,
canvasID: canvasID,
docBulkSize: docBulkSize,
+ deps: deps,
indexWriter: newChunkIndexWriter(
- insertChunksForIngestion(engine.Get(), datasetLanguage(taskCtx)),
+ insertChunksForIngestion(deps.Engine, datasetLanguage(taskCtx)),
fmt.Sprintf("ragflow_%s", taskCtx.Tenant.ID),
taskCtx.Doc.KbID,
docBulkSize,
- ).withFinalInsertFunc(insertFinalChunksForIngestion(engine.Get(), datasetLanguage(taskCtx))),
+ ).withFinalInsertFunc(insertFinalChunksForIngestion(deps.Engine, datasetLanguage(taskCtx))),
deleteChunksFunc: func(ctx context.Context, condition map[string]any, baseName, datasetID string) (int64, error) {
- return engine.Get().DeleteChunks(ctx, condition, baseName, datasetID)
+ return deps.Engine.DeleteChunks(ctx, condition, baseName, datasetID)
},
}
svc.loadDSLFunc = svc.loadDSLFromCanvas
@@ -246,7 +256,7 @@ func (s *PipelineExecutor) Execute(ctx context.Context) (*PipelineResult, error)
d := s.terminalDuration(start)
terminalDuration = &d
}
- s.recordPipelineLog(context.WithoutCancel(ctx), dao.DB, s.taskCtx.Doc.ID, pipelineDSL, string(entity.TaskStatusDone), terminalDuration)
+ s.recordPipelineLog(context.WithoutCancel(ctx), s.deps.DB, s.taskCtx.Doc.ID, pipelineDSL, string(entity.TaskStatusDone), terminalDuration)
}
return result, nil
@@ -313,8 +323,8 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map
// Reload status immediately before write: LoadFromIngestionTask copied a
// snapshot that may be stale if BatchUpdateDocumentStatus ran mid-pipeline.
docStatus := s.taskCtx.Doc.Status
- if dao.DB != nil {
- if persisted, err := dao.NewDocumentDAO().GetByID(ctx, dao.DB, s.taskCtx.Doc.ID); err == nil && persisted != nil {
+ if s.deps.DB != nil {
+ if persisted, err := dao.NewDocumentDAO().GetByID(ctx, s.deps.DB, s.taskCtx.Doc.ID); err == nil && persisted != nil {
docStatus = persisted.Status
s.taskCtx.Doc.Status = persisted.Status
} else if err != nil {
@@ -341,7 +351,7 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map
if err != nil {
return nil, err
}
- if err := putWikiActiveStates(ctx, engine.Get(), activeStates); err != nil {
+ if err := putWikiActiveStates(ctx, s.deps.Engine, activeStates); err != nil {
return nil, fmt.Errorf("persist Wiki active MAP state: %w", err)
}
@@ -515,10 +525,10 @@ func markCompiledProductsHidden(chunks []map[string]any) {
// compiler generation before the new pipeline output is written. Reading first
// avoids relying on immediate search visibility after a bulk index write.
func (s *PipelineExecutor) loadDocumentCompiledState(ctx context.Context) ([]string, []string, []string, error) {
- docEngine := engine.Get()
- if docEngine == nil || s == nil || s.taskCtx == nil {
+ if s == nil || s.taskCtx == nil || s.deps == nil || s.deps.Engine == nil {
return nil, nil, nil, nil
}
+ docEngine := s.deps.Engine
const pageSize = 1000
indexName := fmt.Sprintf("ragflow_%s", s.taskCtx.Tenant.ID)
oldIDs := make([]string, 0)
@@ -560,10 +570,10 @@ func (s *PipelineExecutor) loadDocumentCompiledState(ctx context.Context) ([]str
// the last usable document Wiki; this method removes rows absent from the new
// successful generation immediately before the dataset completion event.
func (s *PipelineExecutor) reconcileDocumentCompiledProducts(ctx context.Context, oldIDs []string, chunks []map[string]any) error {
- docEngine := engine.Get()
- if docEngine == nil || s == nil || s.taskCtx == nil {
+ if s == nil || s.taskCtx == nil || s.deps == nil || s.deps.Engine == nil {
return nil
}
+ docEngine := s.deps.Engine
newIDs := make(map[string]struct{})
for _, chunk := range chunks {
if strings.TrimSpace(asCompiledKwd(chunk)) == "" {
@@ -1014,7 +1024,7 @@ func (s *PipelineExecutor) loadDSLFromCanvas(ctx context.Context, canvasID strin
if canvasID == "" {
return "", "", fmt.Errorf("pipeline executor: empty canvas id")
}
- canvas, err := dao.NewUserCanvasDAO().GetByID(ctx, dao.DB, canvasID)
+ canvas, err := dao.NewUserCanvasDAO().GetByID(ctx, s.deps.DB, canvasID)
if err != nil {
return "", "", fmt.Errorf("load canvas %s: %w", canvasID, err)
}
@@ -1106,7 +1116,8 @@ func (s *PipelineExecutor) runPipelineWithDSL(ctx context.Context, dsl string) (
}
pipe, err := pipelinepkg.NewPipelineFromDSL([]byte(dsl), pipelineID,
pipelinepkg.WithProgressSink(s.progressSink),
- pipelinepkg.WithDocumentID(s.taskCtx.Doc.ID))
+ pipelinepkg.WithDocumentID(s.taskCtx.Doc.ID),
+ pipelinepkg.WithKV(s.deps.KV))
if err != nil {
return nil, dsl, fmt.Errorf("compile pipeline dsl: %w", err)
}
diff --git a/internal/ingestion/task/pipeline_executor_persist_test.go b/internal/ingestion/task/pipeline_executor_persist_test.go
index 156f66ed86..0a528b05e4 100644
--- a/internal/ingestion/task/pipeline_executor_persist_test.go
+++ b/internal/ingestion/task/pipeline_executor_persist_test.go
@@ -46,7 +46,7 @@ func TestExecute_DebugSkipsPipelineLog(t *testing.T) {
Tenant: entity.Tenant{ID: "tenant-1"},
}
- exec, err := NewPipelineExecutor(taskCtx, "canvas-1", 10)
+ exec, err := NewPipelineExecutor(taskCtx, "canvas-1", 10, testPipelineDeps())
if err != nil {
t.Fatalf("NewPipelineExecutor: %v", err)
}
@@ -91,7 +91,7 @@ func TestExecute_DebugReturnsChunks(t *testing.T) {
Tenant: entity.Tenant{ID: "tenant-1"},
}
- exec, err := NewPipelineExecutor(taskCtx, "canvas-1", 10)
+ exec, err := NewPipelineExecutor(taskCtx, "canvas-1", 10, testPipelineDeps())
if err != nil {
t.Fatalf("NewPipelineExecutor: %v", err)
}
diff --git a/internal/ingestion/task/pipeline_executor_test.go b/internal/ingestion/task/pipeline_executor_test.go
index ffe8e85bba..1d93fab08b 100644
--- a/internal/ingestion/task/pipeline_executor_test.go
+++ b/internal/ingestion/task/pipeline_executor_test.go
@@ -16,6 +16,7 @@ import (
"ragflow/internal/agent/runtime"
"ragflow/internal/dao"
"ragflow/internal/entity"
+ "ragflow/internal/ingestion"
pipelinepkg "ragflow/internal/ingestion/pipeline"
indexdoc "ragflow/internal/ingestion/task/indexdoc"
)
@@ -155,9 +156,17 @@ func setupPipelineExecutorTestDB(t *testing.T) func() {
return func() { dao.DB = origDB }
}
+// testPipelineDeps builds the dependency bundle the package tests use: the
+// swapped-in test database (see setupPipelineExecutorDB) and no engine —
+// executor tests stub the index writer with WithInsertFunc, and the
+// engine-touching paths guard a nil engine.
+func testPipelineDeps() *ingestion.Deps {
+ return &ingestion.Deps{DB: dao.DB}
+}
+
func mustNewPipelineExecutor(t *testing.T, taskCtx *TaskContext, canvasID string, docBulkSize int) *PipelineExecutor {
t.Helper()
- svc, err := NewPipelineExecutor(taskCtx, canvasID, docBulkSize)
+ svc, err := NewPipelineExecutor(taskCtx, canvasID, docBulkSize, testPipelineDeps())
if err != nil {
t.Fatalf("NewPipelineExecutor: %v", err)
}
@@ -169,7 +178,7 @@ func mustNewPipelineExecutor(t *testing.T, taskCtx *TaskContext, canvasID string
// =============================================================================
func TestNewPipelineExecutor_Basic(t *testing.T) {
- svc, err := NewPipelineExecutor(makeTaskCtx(), "flow-1", 0)
+ svc, err := NewPipelineExecutor(makeTaskCtx(), "flow-1", 0, testPipelineDeps())
if err != nil {
t.Fatalf("NewPipelineExecutor: %v", err)
}
@@ -185,14 +194,14 @@ func TestNewPipelineExecutor_Basic(t *testing.T) {
}
func TestNewPipelineExecutor_RejectsNilTaskContext(t *testing.T) {
- _, err := NewPipelineExecutor(nil, "flow-1", 0)
+ _, err := NewPipelineExecutor(nil, "flow-1", 0, testPipelineDeps())
if err == nil {
t.Fatal("expected error for nil task context")
}
}
func TestNewPipelineExecutor_RejectsEmptyCanvasID(t *testing.T) {
- _, err := NewPipelineExecutor(makeTaskCtx(), "", 0)
+ _, err := NewPipelineExecutor(makeTaskCtx(), "", 0, testPipelineDeps())
if err == nil {
t.Fatal("expected error for empty canvas id")
}
@@ -213,7 +222,7 @@ func TestNewPipelineExecutor_RejectsIncompleteTaskContext(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
ctx := makeTaskCtx()
tt.mutate(ctx)
- _, err := NewPipelineExecutor(ctx, "flow-1", 0)
+ _, err := NewPipelineExecutor(ctx, "flow-1", 0, testPipelineDeps())
if err == nil {
t.Fatal("expected validation error")
}
@@ -229,7 +238,7 @@ func TestNewPipelineExecutor_AcceptsDebugTaskContext(t *testing.T) {
ctx := makeTaskCtx()
ctx.KB = entity.Knowledgebase{ID: ""}
ctx.Doc.KbID = ""
- if _, err := NewPipelineExecutor(ctx, "flow-1", 0); err != nil {
+ if _, err := NewPipelineExecutor(ctx, "flow-1", 0, testPipelineDeps()); err != nil {
t.Fatalf("debug TaskContext rejected: %v", err)
}
}
diff --git a/internal/ingestion/task/real_consumer_test.go b/internal/ingestion/task/real_consumer_test.go
index b3b56a7810..253e3f2e29 100644
--- a/internal/ingestion/task/real_consumer_test.go
+++ b/internal/ingestion/task/real_consumer_test.go
@@ -142,14 +142,14 @@ func TestRealProducerConsumer(t *testing.T) {
t.Fatalf("set pipeline_id: %v", err)
}
- tc, err := LoadFromIngestionTask(context.Background(), task)
+ tc, err := LoadFromIngestionTask(context.Background(), dao.DB, task)
if err != nil {
t.Fatalf("LoadFromIngestionTask: %v", err)
}
t.Logf("Consumer: Loaded Doc=%s Parser=%s KB=%s Tenant=%s",
tc.Doc.ID, tc.Doc.ParserID, tc.KB.ID, tc.Tenant.ID)
- svc, err := NewPipelineExecutor(tc, tc.PipelineID, 0)
+ svc, err := NewPipelineExecutor(tc, tc.PipelineID, 0, testPipelineDeps())
if err != nil {
t.Fatalf("NewPipelineExecutor: %v", err)
}
diff --git a/internal/ingestion/task/task_context.go b/internal/ingestion/task/task_context.go
index 74eb4b8c74..c418f18eae 100644
--- a/internal/ingestion/task/task_context.go
+++ b/internal/ingestion/task/task_context.go
@@ -24,6 +24,8 @@ import (
"ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/entity"
+
+ "gorm.io/gorm"
)
// TaskKind discriminates which execution path a queued TaskContext takes.
@@ -111,8 +113,10 @@ func (c *TaskContext) ID() string {
// LoadFromIngestionTask loads the full task context from an IngestionTask.
// It follows the FK chain: ingestion task -> document -> knowledgebase -> tenant.
-func LoadFromIngestionTask(ctx context.Context, ingestionTask *entity.IngestionTask) (*TaskContext, error) {
- doc, err := dao.NewDocumentDAO().GetByID(ctx, dao.DB, ingestionTask.DocumentID)
+// db is the relational handle from the ingestion dependency bundle; callers
+// pass it explicitly so the loader never reaches for the process-wide global.
+func LoadFromIngestionTask(ctx context.Context, db *gorm.DB, ingestionTask *entity.IngestionTask) (*TaskContext, error) {
+ doc, err := dao.NewDocumentDAO().GetByID(ctx, db, ingestionTask.DocumentID)
if err != nil {
return nil, fmt.Errorf("load document %s: %w", ingestionTask.DocumentID, err)
}
@@ -120,12 +124,12 @@ func LoadFromIngestionTask(ctx context.Context, ingestionTask *entity.IngestionT
return nil, fmt.Errorf("document %s not found", ingestionTask.DocumentID)
}
- kb, err := dao.NewKnowledgebaseDAO().GetByID(ctx, dao.DB, doc.KbID)
+ kb, err := dao.NewKnowledgebaseDAO().GetByID(ctx, db, doc.KbID)
if err != nil || kb == nil {
return nil, fmt.Errorf("error when load knowledgebase %s: %w", doc.KbID, err)
}
- tenant, err := dao.NewTenantDAO().GetByID(ctx, dao.DB, kb.TenantID)
+ tenant, err := dao.NewTenantDAO().GetByID(ctx, db, kb.TenantID)
if err != nil || tenant == nil {
return nil, fmt.Errorf("error when load tenant %s: %w", kb.TenantID, err)
}
diff --git a/internal/ingestion/task/task_context_test.go b/internal/ingestion/task/task_context_test.go
index 1515bacba5..91d5f45439 100644
--- a/internal/ingestion/task/task_context_test.go
+++ b/internal/ingestion/task/task_context_test.go
@@ -19,6 +19,7 @@ package task
import (
"testing"
+ "ragflow/internal/dao"
"ragflow/internal/entity"
"ragflow/internal/ingestion/testutil"
)
@@ -57,7 +58,7 @@ func TestLoadFromIngestionTask_DoesNotFallbackToKnowledgebasePipelineID(t *testi
}
ctx := t.Context()
- taskCtx, err := LoadFromIngestionTask(ctx, &entity.IngestionTask{
+ taskCtx, err := LoadFromIngestionTask(ctx, dao.DB, &entity.IngestionTask{
ID: "task-1",
DocumentID: "doc-1",
DatasetID: "kb-1",
@@ -106,7 +107,7 @@ func TestLoadFromIngestionTask_PrefersDocumentPipelineID(t *testing.T) {
}
ctx := t.Context()
- taskCtx, err := LoadFromIngestionTask(ctx, &entity.IngestionTask{
+ taskCtx, err := LoadFromIngestionTask(ctx, dao.DB, &entity.IngestionTask{
ID: "task-1",
DocumentID: "doc-1",
DatasetID: "kb-1",
diff --git a/internal/server/api/server.go b/internal/server/api/server.go
index fdfdf9e4fd..f2307d4fd6 100644
--- a/internal/server/api/server.go
+++ b/internal/server/api/server.go
@@ -55,6 +55,7 @@ import (
"ragflow/internal/dao"
"ragflow/internal/engine"
"ragflow/internal/engine/kvrocks"
+ "ragflow/internal/ingestion/task"
_ "ragflow/internal/ingestion/wire"
"ragflow/internal/server"
"ragflow/internal/server/bootstrap"
@@ -124,7 +125,7 @@ func startServer(ctx context.Context, serverName string, arguments *bootstrap.Se
searchService := service.NewSearchService()
searchService.SetTenantService(tenantService)
fileService := file.NewFileService(documentService)
- memoryService := service.NewMemoryService()
+ memoryService := service.NewMemoryService(dao.DB, engine.Get(), service.WithMemoryKvrocks(kvrocks.Get()))
mcpService := service.NewMCPService()
modelProviderService := service.NewModelProviderService()
modelFactory := service.NewModelFactory()
@@ -133,6 +134,14 @@ func startServer(ctx context.Context, serverName string, arguments *bootstrap.Se
// Wire the real MemorySaver so the Message component can persist
// conversation turns to memory stores declared in the canvas DSL.
component.SetMemorySaver(service.NewMemorySaverAdapter(memoryService))
+ // Bind the relational handle and document engine into the production
+ // embedder and knowledge-compiler resolvers. The API process shares the
+ // task-package composition root with the ingestor: without these, the
+ // embedder resolver would read a nil db and fail at run time when the API
+ // process runs ingestion runs (e.g. in a combined --api --ingestor deploy
+ // where the ingestor bindings do not apply).
+ task.InitEmbedderResolver(dao.DB)
+ task.InitKnowledgeCompilerDeps(dao.DB, engine.Get())
// Initialize doc engine for skill search
docEngine := engine.Get()
diff --git a/internal/server/ingestor/server.go b/internal/server/ingestor/server.go
index d6e3d4c491..22d224ff13 100644
--- a/internal/server/ingestor/server.go
+++ b/internal/server/ingestor/server.go
@@ -29,8 +29,13 @@ import (
"go.uber.org/zap"
"ragflow/internal/common"
+ "ragflow/internal/dao"
+ "ragflow/internal/engine"
+ kvrocks "ragflow/internal/engine/kvrocks"
+ ingestioncore "ragflow/internal/ingestion"
"ragflow/internal/ingestion/knowledge_compile"
ingestion "ragflow/internal/ingestion/service"
+ "ragflow/internal/ingestion/task"
"ragflow/internal/server"
"ragflow/internal/server/bootstrap"
"ragflow/internal/server/config"
@@ -149,7 +154,20 @@ func Run(ctx context.Context, cancel context.CancelFunc, serverName string, args
// Apply the configured compiler pool size (no-op when 0; the pool keeps its
// vCPU default, overridable via KC_COMPILE_CONCURRENCY).
knowledge_compile.SetCompilerConcurrency(ingestorCfg.CompilerPoolSize)
- ingestor := ingestion.NewIngestor(*args.Name, int32(workers), []string{"pdf", "docx", "txt"})
+ ingestorDeps := &ingestioncore.Deps{
+ Engine: engine.Get(),
+ DB: dao.DB,
+ MQ: engine.GetMessageQueueEngine(),
+ KV: kvrocks.Get(),
+ }
+ // Fail fast at the assembly point: a misconfigured ingestor (missing doc
+ // engine, database, or message queue) must never reach Start as a "healthy"
+ // service that silently cannot consume.
+ if err = ingestorDeps.Validate(); err != nil {
+ common.Error("Invalid ingestor dependencies", err)
+ return fmt.Errorf("ingestor dependencies invalid: %w", err)
+ }
+ ingestor := ingestion.NewIngestor(*args.Name, int32(workers), []string{"pdf", "docx", "txt"}, ingestorDeps)
ingestor.SetKnowledgeCompileModelConfig(
globalConfig.GetDefaultChatModel().Name,
globalConfig.GetDefaultEmbeddingModel().Name,
@@ -173,7 +191,13 @@ func Run(ctx context.Context, cancel context.CancelFunc, serverName string, args
// Memory extraction runs on the Ingestor's shared NATS consumer + worker
// pool (task_type="memory" dispatched by handleAndExecute -> executeMemoryTask),
// so there is no longer a dedicated Redis memory consumer to start.
- ingestor.SetMemoryMessageService(service.NewMemoryMessageService(service.NewMemoryService()))
+ ingestor.SetMemoryMessageService(service.NewMemoryMessageService(service.NewMemoryService(dao.DB, engine.Get(), service.WithMemoryKvrocks(kvrocks.Get()))))
+ // Bind the relational handle and document engine into the production
+ // embedder and knowledge-compiler resolvers. The task package is the
+ // composition root for ingestion runs; it reads these from the explicit
+ // bindings rather than the package globals.
+ task.InitEmbedderResolver(dao.DB)
+ task.InitKnowledgeCompilerDeps(dao.DB, engine.Get())
// Start returns immediately (it launches the owned consume/compile
// goroutines and joins them via Stop); a provisioning failure here must
diff --git a/internal/service/dataset/tags.go b/internal/service/dataset/tags.go
index a61f89daa2..960502dd00 100644
--- a/internal/service/dataset/tags.go
+++ b/internal/service/dataset/tags.go
@@ -34,7 +34,9 @@ func (d *DatasetService) AggregateTags(ctx context.Context, datasetIDs []string,
// chunk-level aggregation.
loader := d.tagVocabularyLoader
if loader == nil {
- loader = component.TagVocabularyFromTagFileID
+ loader = func(ctx context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {
+ return component.TagVocabularyFromTagFileID(ctx, dao.DB, tagFileID, ownerTenantID)
+ }
}
merged := make(map[string]int)
// The handler accepts repeated IDs, and distinct raw IDs can normalize to
diff --git a/internal/service/dataset/tags_aggregate_test.go b/internal/service/dataset/tags_aggregate_test.go
index 00925e4fd0..6a5c01490e 100644
--- a/internal/service/dataset/tags_aggregate_test.go
+++ b/internal/service/dataset/tags_aggregate_test.go
@@ -422,7 +422,7 @@ func TestDatasetServiceAggregateTagsCountsSharedFileOnce(t *testing.T) {
// loader, so the error carries the production sentinel rather than a stub's
// approximation of it.
func missingTagSourceLoader(ctx context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {
- return component.TagVocabularyFromTagFileID(ctx, tagFileID, ownerTenantID)
+ return component.TagVocabularyFromTagFileID(ctx, dao.DB, tagFileID, ownerTenantID)
}
// TestDatasetServiceAggregateTagsSkipsUnresolvableDocumentTagSource is the
diff --git a/internal/service/memory.go b/internal/service/memory.go
index b76d02cc88..cead57f89c 100644
--- a/internal/service/memory.go
+++ b/internal/service/memory.go
@@ -36,6 +36,7 @@ import (
"ragflow/internal/dao"
"ragflow/internal/engine"
+ "ragflow/internal/engine/kvrocks"
enginetypes "ragflow/internal/engine/types"
"ragflow/internal/permission"
"ragflow/internal/service/nlp"
@@ -284,19 +285,40 @@ func generateOutputFormat(typesToExtract []string) string {
// MemoryService handles business logic for memory operations
// It provides methods for creating, updating, deleting, and querying memories
type MemoryService struct {
- memoryDAO *dao.MemoryDAO
- docEngine engine.DocEngine
+ memoryDAO *dao.MemoryDAO
+ db *gorm.DB
+ docEngine engine.DocEngine
+ kvrocksClient *kvrocks.Client
}
-// NewMemoryService creates a new MemoryService instance
+// MemoryServiceOption configures a MemoryService at construction time.
+type MemoryServiceOption func(*MemoryService)
+
+// WithMemoryKvrocks binds the kvrocks client used to mint raw-message ids for
+// the transient MemoryMessageService the MemoryService constructs. It keeps the
+// kvrocks dependency explicit rather than read from a package global.
+func WithMemoryKvrocks(kv *kvrocks.Client) MemoryServiceOption {
+ return func(s *MemoryService) { s.kvrocksClient = kv }
+}
+
+// NewMemoryService creates a new MemoryService instance. db and eng are the
+// process-wide relational handle and doc engine; they are resolved once at the
+// server assembly point and injected here rather than read from the package
+// globals (arch_test.go guards that rule). kv is optional and defaults to nil,
+// in which case raw-message ids fall back to wall-clock values.
//
// Returns:
// - *MemoryService: Initialized service instance with DAO
-func NewMemoryService() *MemoryService {
- return &MemoryService{
+func NewMemoryService(db *gorm.DB, eng engine.DocEngine, opts ...MemoryServiceOption) *MemoryService {
+ s := &MemoryService{
memoryDAO: dao.NewMemoryDAO(),
- docEngine: engine.Get(),
+ db: db,
+ docEngine: eng,
+ }
+ for _, opt := range opts {
+ opt(s)
}
+ return s
}
// CreateMemoryRequest defines the request structure for creating a memory
@@ -380,14 +402,14 @@ type MemoryFiltersResponse struct {
}
func (s *MemoryService) ListMemoryFilters(ctx context.Context, userID string) (*MemoryFiltersResponse, error) {
- memoryIDs, err := accessibleMemoryIDs(ctx, userID, permission.OperationRead)
+ memoryIDs, err := s.accessibleMemoryIDs(ctx, userID, permission.OperationRead)
if err != nil {
return nil, err
}
if len(memoryIDs) == 0 {
return &MemoryFiltersResponse{}, nil
}
- memories, _, err := s.memoryDAO.GetByFilter(ctx, dao.DB, memoryIDs, nil, nil, "", "", 1, 0)
+ memories, _, err := s.memoryDAO.GetByFilter(ctx, s.db, memoryIDs, nil, nil, "", "", 1, 0)
if err != nil {
return nil, err
}
@@ -508,7 +530,7 @@ func (s *MemoryService) CreateMemory(ctx context.Context, userID string, req *Cr
}
memoryName, err := common.UniqueName(memoryName, MemoryNameLimit, func(candidate string) (bool, error) {
- existing, err := s.memoryDAO.GetByNameAndTenant(ctx, dao.DB, candidate, userID)
+ existing, err := s.memoryDAO.GetByNameAndTenant(ctx, s.db, candidate, userID)
if err != nil {
return false, err
}
@@ -545,11 +567,11 @@ func (s *MemoryService) CreateMemory(ctx context.Context, userID string, req *Cr
if req.TenantLLMID != nil {
memory.TenantLLMID = req.TenantLLMID
}
- if err = s.memoryDAO.Create(ctx, dao.DB, memory); err != nil {
+ if err = s.memoryDAO.Create(ctx, s.db, memory); err != nil {
return nil, errors.New("could not create new memory")
}
- createdMemory, err := s.memoryDAO.GetByID(ctx, dao.DB, newID)
+ createdMemory, err := s.memoryDAO.GetByID(ctx, s.db, newID)
if err != nil {
return nil, errors.New("could not create new memory")
}
@@ -575,11 +597,11 @@ func (s *MemoryService) CreateMemory(ctx context.Context, userID string, req *Cr
// resp, err := service.UpdateMemory("tenant123", "memory456", req)
func (s *MemoryService) UpdateMemory(ctx context.Context, userID string, memoryID string, req *UpdateMemoryRequest) (*CreateMemoryResponse, error) {
updateDict := make(map[string]interface{})
- if err := checkMemoryPermission(ctx, userID, memoryID, permission.OperationUpdate); err != nil {
+ if err := s.checkMemoryPermission(ctx, userID, memoryID, permission.OperationUpdate); err != nil {
return nil, err
}
- currentMemory, err := s.memoryDAO.GetByID(ctx, dao.DB, memoryID)
+ currentMemory, err := s.memoryDAO.GetByID(ctx, s.db, memoryID)
if err != nil {
return nil, fmt.Errorf("memory '%s' not found", memoryID)
}
@@ -591,7 +613,7 @@ func (s *MemoryService) UpdateMemory(ctx context.Context, userID string, memoryI
return nil, err
}
available, nameErr := common.NameAvailable(strings.TrimSpace(currentMemory.Name), memoryName, func(candidate string) (bool, error) {
- existing, err := s.memoryDAO.GetByNameAndTenant(ctx, dao.DB, candidate, ownerTenantID)
+ existing, err := s.memoryDAO.GetByNameAndTenant(ctx, s.db, candidate, ownerTenantID)
if err != nil {
return false, err
}
@@ -612,7 +634,7 @@ func (s *MemoryService) UpdateMemory(ctx context.Context, userID string, memoryI
return nil, fmt.Errorf("unknown permission '%s'", *req.Permissions)
}
if strings.ToLower(strings.TrimSpace(currentMemory.Permissions)) != string(perm) {
- if err := checkMemoryPermission(ctx, userID, memoryID, permission.OperationShare); err != nil {
+ if err := s.checkMemoryPermission(ctx, userID, memoryID, permission.OperationShare); err != nil {
return nil, err
}
}
@@ -857,11 +879,11 @@ func (s *MemoryService) UpdateMemory(ctx context.Context, userID string, memoryI
}
}
- if err = s.memoryDAO.UpdateByID(ctx, dao.DB, memoryID, updateDict); err != nil {
+ if err = s.memoryDAO.UpdateByID(ctx, s.db, memoryID, updateDict); err != nil {
return nil, errors.New("failed to update memory")
}
- updatedMemory, err := s.memoryDAO.GetByID(ctx, dao.DB, memoryID)
+ updatedMemory, err := s.memoryDAO.GetByID(ctx, s.db, memoryID)
if err != nil {
return nil, errors.New("failed to get updated memory")
}
@@ -936,7 +958,7 @@ func (s *MemoryService) DeleteMemory(ctx context.Context, userID, memoryID strin
}
// Delete memory record
- if err := s.memoryDAO.DeleteByID(ctx, dao.DB, memoryID); err != nil {
+ if err := s.memoryDAO.DeleteByID(ctx, s.db, memoryID); err != nil {
return errors.New("failed to delete memory")
}
@@ -1268,7 +1290,7 @@ func (s *MemoryService) queryMessage(ctx context.Context, memories []*entity.Mem
}
searchReq := &enginetypes.SearchRequest{
- IndexNames: memorySearchIndexNames(memories),
+ IndexNames: memorySearchIndexNames(memories, s.docEngine),
Offset: 0,
Limit: topN,
SelectFields: memoryMessageSelectFields(),
@@ -1304,14 +1326,14 @@ func (s *MemoryService) filterAccessibleMemories(ctx context.Context, userID str
return []*entity.Memory{}, nil
}
- accessibleIDs, err := filterMemoryIDs(ctx, userID, memoryIDs, permission.OperationUse)
+ accessibleIDs, err := s.filterMemoryIDs(ctx, userID, memoryIDs, permission.OperationUse)
if err != nil {
return nil, err
}
if len(accessibleIDs) == 0 {
return []*entity.Memory{}, nil
}
- return s.memoryDAO.GetByIDs(ctx, dao.DB, accessibleIDs)
+ return s.memoryDAO.GetByIDs(ctx, s.db, accessibleIDs)
}
func splitFilterValues(values interface{}) []string {
@@ -1416,7 +1438,7 @@ func MemoryIndexName(tenantID string) string {
return fmt.Sprintf("memory_%s_%s", prefix, tenantID)
}
-func memorySearchIndexNames(memories []*entity.Memory) []string {
+func memorySearchIndexNames(memories []*entity.Memory, docEngine engine.DocEngine) []string {
seen := make(map[string]struct{}, len(memories))
indexNames := make([]string, 0, len(memories))
for _, memory := range memories {
@@ -1424,7 +1446,7 @@ func memorySearchIndexNames(memories []*entity.Memory) []string {
continue
}
indexName := MemoryIndexName(memory.TenantID)
- if engine.GetEngineType() == "infinity" {
+ if docEngine != nil && docEngine.GetType() == "infinity" {
indexName = fmt.Sprintf("%s_%s", indexName, memory.ID)
}
if _, ok := seen[indexName]; ok {
@@ -1522,7 +1544,7 @@ func (s *MemoryService) getRecentMessage(ctx context.Context, memories []*entity
} else if limit > maxMessageLimit {
limit = maxMessageLimit
}
- indexNames := memorySearchIndexNames(memories)
+ indexNames := memorySearchIndexNames(memories, s.docEngine)
memoryIDs := make([]string, 0, len(memories))
for _, memory := range memories {
if memory == nil || strings.TrimSpace(memory.ID) == "" {
@@ -1586,10 +1608,10 @@ func (s *MemoryService) memoryWithAccess(ctx context.Context, userID, memoryID s
if err := ctx.Err(); err != nil {
return nil, err
}
- if err := checkMemoryPermission(ctx, userID, memoryID, operation); err != nil {
+ if err := s.checkMemoryPermission(ctx, userID, memoryID, operation); err != nil {
return nil, err
}
- memory, err := s.memoryDAO.GetByIDWithContext(ctx, dao.DB, memoryID)
+ memory, err := s.memoryDAO.GetByIDWithContext(ctx, s.db, memoryID)
if err != nil {
if dao.IsNotFoundErr(err) {
return nil, permission.ErrResourceNotFound
@@ -1619,7 +1641,7 @@ func (s *MemoryService) memoryWithAccess(ctx context.Context, userID, memoryID s
//
// resp, err := service.ListMemories("user123", []string{}, []string{"semantic"}, "table", "test", 1, 10)
func (s *MemoryService) ListMemories(ctx context.Context, userID string, tenantIDs []string, memoryTypes []string, storageType string, keywords string, page int, pageSize int, canvasID string) (*ListMemoryResponse, error) {
- memoryIDs, err := accessibleMemoryIDsForCanvas(ctx, userID, canvasID)
+ memoryIDs, err := s.accessibleMemoryIDsForCanvas(ctx, userID, canvasID)
if err != nil {
return nil, err
}
@@ -1627,7 +1649,7 @@ func (s *MemoryService) ListMemories(ctx context.Context, userID string, tenantI
return &ListMemoryResponse{MemoryList: []map[string]interface{}{}}, nil
}
- memories, total, err := s.memoryDAO.GetByFilter(ctx, dao.DB, memoryIDs, tenantIDs, memoryTypes, storageType, keywords, page, pageSize)
+ memories, total, err := s.memoryDAO.GetByFilter(ctx, s.db, memoryIDs, tenantIDs, memoryTypes, storageType, keywords, page, pageSize)
if err != nil {
return nil, err
}
@@ -1642,8 +1664,8 @@ func (s *MemoryService) ListMemories(ctx context.Context, userID string, tenantI
}
memoryMap := map[string]interface{}{
"id": resp.ID,
- "llm_id": ResolveTenantModelDisplayName(ctx, dao.DB, ptrStringValue(resp.TenantLLMID), resp.LLMID, modelNameCache),
- "embd_id": ResolveTenantModelDisplayName(ctx, dao.DB, ptrStringValue(resp.TenantEmbdID), resp.EmbdID, modelNameCache),
+ "llm_id": ResolveTenantModelDisplayName(ctx, s.db, ptrStringValue(resp.TenantLLMID), resp.LLMID, modelNameCache),
+ "embd_id": ResolveTenantModelDisplayName(ctx, s.db, ptrStringValue(resp.TenantEmbdID), resp.EmbdID, modelNameCache),
"name": resp.Name,
"avatar": resp.Avatar,
"tenant_id": resp.TenantID,
@@ -1724,7 +1746,7 @@ func (s *MemoryService) GetMemoryConfig(ctx context.Context, userID, memoryID st
// getMemoryConfig retrieves the full configuration of a memory without access
// control checks. This is for trusted internal callers such as queue processing.
func (s *MemoryService) getMemoryConfig(ctx context.Context, memoryID string) (*CreateMemoryResponse, error) {
- memory, err := s.memoryDAO.GetWithOwnerNameByID(ctx, dao.DB, memoryID)
+ memory, err := s.memoryDAO.GetWithOwnerNameByID(ctx, s.db, memoryID)
if err != nil {
return nil, fmt.Errorf("get memory %q: %w", memoryID, err)
}
@@ -1808,7 +1830,7 @@ func (s *MemoryService) listMemoryMessages(ctx context.Context, memory *entity.M
filter["session_id"] = keywords
}
filter["memory_id"] = []string{memoryID}
- indexNames := memorySearchIndexNames([]*entity.Memory{memory})
+ indexNames := memorySearchIndexNames([]*entity.Memory{memory}, s.docEngine)
rawReq := &enginetypes.SearchRequest{
IndexNames: indexNames,
@@ -1921,7 +1943,7 @@ func (s *MemoryService) memoryMessageAgentNames(messages []map[string]interface{
ID string `gorm:"column:id"`
Title *string `gorm:"column:title"`
}
- if err := dao.DB.Model(&entity.UserCanvas{}).Select("id, title").Where("id IN ?", agentIDs).Scan(&agents).Error; err != nil {
+ if err := s.db.Model(&entity.UserCanvas{}).Select("id, title").Where("id IN ?", agentIDs).Scan(&agents).Error; err != nil {
return nil, err
}
agentNames := make(map[string]string, len(agents))
@@ -1944,7 +1966,7 @@ func (s *MemoryService) memoryMessageTasks(memoryID string) (map[string]map[stri
ChunkIDs *string `gorm:"column:chunk_ids"`
CreateTime *int64 `gorm:"column:create_time"`
}
- if err := dao.DB.Model(&entity.Task{}).
+ if err := s.db.Model(&entity.Task{}).
Select("id, doc_id, from_page, progress, progress_msg, digest, chunk_ids, create_time").
Where("doc_id IN ?", []string{memoryID}).
Order("create_time ASC").
diff --git a/internal/service/memory_extractor.go b/internal/service/memory_extractor.go
index e072b6367c..d1fe1ab58d 100644
--- a/internal/service/memory_extractor.go
+++ b/internal/service/memory_extractor.go
@@ -45,6 +45,7 @@ import (
"ragflow/internal/common"
"ragflow/internal/dao"
+ "ragflow/internal/engine/kvrocks"
"ragflow/internal/entity"
"ragflow/internal/entity/models"
@@ -140,7 +141,7 @@ func (s *MemoryMessageService) HandleSaveToMemoryTask(ctx context.Context, taskI
s.memoryTaskDAO = dao.NewMemoryTaskDAO()
}
- task, acquired, err := s.memoryTaskDAO.Claim(ctx, dao.DB, taskID, leaseOwner, memoryNow(), memoryTaskLeaseTTL)
+ task, acquired, err := s.memoryTaskDAO.Claim(ctx, s.memories.db, taskID, leaseOwner, memoryNow(), memoryTaskLeaseTTL)
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return s.failTaskMissingDurableState(ctx, taskID)
@@ -223,11 +224,11 @@ func (s *MemoryMessageService) persistMemoryTaskFailure(ctx context.Context, tas
}
if errors.Is(runErr, errPermanentMemoryTask) || task.AttemptCount >= memoryTaskMaxAttempts {
action = "mark failed"
- updated, err = s.memoryTaskDAO.MarkFailed(persistCtx, dao.DB, task.TaskID, leaseOwner, reportedErr.Error(), now)
+ updated, err = s.memoryTaskDAO.MarkFailed(persistCtx, s.memories.db, task.TaskID, leaseOwner, reportedErr.Error(), now)
} else {
action = "schedule retry"
nextRetryAt := now.Add(memoryTaskRetryDelay(task.AttemptCount))
- updated, err = s.memoryTaskDAO.ScheduleRetry(persistCtx, dao.DB, task.TaskID, leaseOwner, now, nextRetryAt, reportedErr.Error())
+ updated, err = s.memoryTaskDAO.ScheduleRetry(persistCtx, s.memories.db, task.TaskID, leaseOwner, now, nextRetryAt, reportedErr.Error())
}
if err != nil {
return MemoryTaskLeaveUnsettled, errors.Join(reportedErr, fmt.Errorf("memory: %s for task %s: %w", action, task.TaskID, err))
@@ -244,7 +245,7 @@ func (s *MemoryMessageService) failTaskMissingDurableState(ctx context.Context,
if s.taskDAO == nil {
s.taskDAO = dao.NewTaskDAO()
}
- task, err := s.taskDAO.GetByID(ctx, dao.DB, taskID)
+ task, err := s.taskDAO.GetByID(ctx, s.memories.db, taskID)
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return MemoryTaskAcknowledge, fmt.Errorf("memory: task %s is not found", taskID)
@@ -258,7 +259,7 @@ func (s *MemoryMessageService) failTaskMissingDurableState(ctx context.Context,
failure := fmt.Errorf("memory: task %s cannot resume because it is missing durable execution state after the server upgrade; save the message again", taskID)
persistCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), memoryTaskFailureWriteTimeout)
defer cancel()
- updated, updateErr := s.taskDAO.MarkActiveMemoryTaskFailed(persistCtx, dao.DB, taskID, failure.Error())
+ updated, updateErr := s.taskDAO.MarkActiveMemoryTaskFailed(persistCtx, s.memories.db, taskID, failure.Error())
if updateErr != nil {
return MemoryTaskLeaveUnsettled, errors.Join(failure, fmt.Errorf("memory: mark task failed: %w", updateErr))
}
@@ -295,7 +296,7 @@ func (s *MemoryMessageService) renewMemoryTaskLease(ctx context.Context, cancel
case <-ticker.C:
renewedAt := memoryNow()
renewCtx, renewCancel := context.WithTimeout(ctx, memoryTaskLeaseRenewTimeout)
- renewed, err := s.memoryTaskDAO.RenewLease(renewCtx, dao.DB, taskID, leaseOwner, renewedAt, memoryTaskLeaseTTL)
+ renewed, err := s.memoryTaskDAO.RenewLease(renewCtx, s.memories.db, taskID, leaseOwner, renewedAt, memoryTaskLeaseTTL)
renewCancel()
if err == nil && renewed {
leaseExpiresAt = renewedAt.Add(memoryTaskLeaseTTL)
@@ -355,7 +356,7 @@ func (s *MemoryMessageService) resumeMemoryTask(ctx context.Context, task *entit
if encodeErr != nil {
return fmt.Errorf("memory: encode task %s extraction: %w", task.TaskID, encodeErr)
}
- updated, persistErr := s.memoryTaskDAO.PersistExtraction(ctx, dao.DB, task.TaskID, leaseOwner, memoryNow(), encoded)
+ updated, persistErr := s.memoryTaskDAO.PersistExtraction(ctx, s.memories.db, task.TaskID, leaseOwner, memoryNow(), encoded)
if persistErr != nil {
return fmt.Errorf("memory: persist task %s extraction: %w", task.TaskID, persistErr)
}
@@ -376,7 +377,7 @@ func (s *MemoryMessageService) resumeMemoryTask(ctx context.Context, task *entit
if err := s.storeMemoryTaskExtraction(ctx, task, msg); err != nil {
return err
}
- updated, storeErr := s.memoryTaskDAO.MarkStored(ctx, dao.DB, task.TaskID, leaseOwner, memoryNow())
+ updated, storeErr := s.memoryTaskDAO.MarkStored(ctx, s.memories.db, task.TaskID, leaseOwner, memoryNow())
if storeErr != nil {
return fmt.Errorf("memory: mark task %s stored: %w", task.TaskID, storeErr)
}
@@ -390,7 +391,7 @@ func (s *MemoryMessageService) resumeMemoryTask(ctx context.Context, task *entit
if len(task.Extraction) == 0 {
progressMsg = memoryNoExtractionProgress
}
- completed, completeErr := s.memoryTaskDAO.Complete(ctx, dao.DB, task.TaskID, leaseOwner, progressMsg, memoryNow())
+ completed, completeErr := s.memoryTaskDAO.Complete(ctx, s.memories.db, task.TaskID, leaseOwner, progressMsg, memoryNow())
if completeErr != nil {
return fmt.Errorf("memory: complete task %s: %w", task.TaskID, completeErr)
}
@@ -436,7 +437,7 @@ func (s *MemoryMessageService) extractMemoryTask(ctx context.Context, task *enti
common.Warn(fmt.Sprintf("memory: chat model call failed while extracting task %s: %v", task.TaskID, err))
return []extractedMemory{}, nil
}
- materialized := materializeMemoryExtraction(ctx, extracted, memoryNow())
+ materialized := materializeMemoryExtraction(ctx, extracted, memoryNow(), s.kvrocksClient)
return materialized, nil
}
@@ -539,11 +540,13 @@ func memoryLLMError(answer string) string {
}
// materializeMemoryExtraction assigns stable message ids and timestamp
-// fallbacks before the extraction checkpoint is persisted.
-func materializeMemoryExtraction(ctx context.Context, extracted []extractedMemory, now time.Time) []extractedMemory {
+// fallbacks before the extraction checkpoint is persisted. kv is the kvrocks
+// client used to mint raw-message ids; a nil client falls back to wall-clock
+// ids.
+func materializeMemoryExtraction(ctx context.Context, extracted []extractedMemory, now time.Time, kv *kvrocks.Client) []extractedMemory {
materialized := make([]extractedMemory, len(extracted))
for i, item := range extracted {
- item.MessageID = generateRawMessageID(ctx)
+ item.MessageID = generateRawMessageID(ctx, kv)
item.ValidAt = formatMemoryTime(item.ValidAt, now)
// An invalid_at the model made up normalizes to "", which persists as
// nil and leaves the memory valid (Python stores nothing via
@@ -729,7 +732,7 @@ func (s *MemoryMessageService) updateTaskProgress(ctx context.Context, taskID st
return errors.New("memory: empty task id")
}
stamped := time.Now().Format(memoryTimeLayout) + " " + msg
- if err := s.taskDAO.UpdateProgress(ctx, dao.DB, taskID, progress, stamped); err != nil {
+ if err := s.taskDAO.UpdateProgress(ctx, s.memories.db, taskID, progress, stamped); err != nil {
common.Warn("memory: update task progress failed", zap.Error(err))
return err
}
diff --git a/internal/service/memory_extractor_test.go b/internal/service/memory_extractor_test.go
index 84ea6a5243..53b3e18447 100644
--- a/internal/service/memory_extractor_test.go
+++ b/internal/service/memory_extractor_test.go
@@ -29,6 +29,7 @@ import (
"ragflow/internal/common"
"ragflow/internal/dao"
+ "ragflow/internal/engine"
"ragflow/internal/entity"
"ragflow/internal/ingestion/testutil"
@@ -105,7 +106,7 @@ func TestBuildExtractedMessageValidAtSemantics(t *testing.T) {
MessageType: "fact",
Content: "likes coffee",
ValidAt: "2026-08-20T10:05:00+08:00",
- }}, now)
+ }}, now, nil)
withLLMTime := buildExtractedMessage(42, "mem-1", msg, materialized[0])
if got := withLLMTime["valid_at"]; got != "2026-08-20 10:05:00" {
t.Fatalf("valid_at = %v, want LLM wall clock %q", got, "2026-08-20 10:05:00")
@@ -114,7 +115,7 @@ func TestBuildExtractedMessageValidAtSemantics(t *testing.T) {
materialized = materializeMemoryExtraction(t.Context(), []extractedMemory{{
MessageType: "fact",
Content: "likes coffee",
- }}, now)
+ }}, now, nil)
fallbackOnly := buildExtractedMessage(42, "mem-1", msg, materialized[0])
if got := fallbackOnly["valid_at"]; got != now.Format(memoryTimeLayout) {
t.Fatalf("valid_at = %v, want fallback %q", got, now.Format(memoryTimeLayout))
@@ -124,7 +125,7 @@ func TestBuildExtractedMessageValidAtSemantics(t *testing.T) {
MessageType: "fact",
Content: "likes coffee",
InvalidAt: " ",
- }}, now)
+ }}, now, nil)
blankInvalidAt := buildExtractedMessage(42, "mem-1", msg, materialized[0])
if got := blankInvalidAt["invalid_at"]; got != nil {
t.Fatalf("invalid_at = %#v, want nil for whitespace-only input", got)
@@ -153,7 +154,7 @@ func TestMaterializeMemoryExtractionInvalidAtStaysValidWhenUnparseable(t *testin
MessageType: "fact",
Content: "likes coffee",
InvalidAt: test.invalidAt,
- }}, now)
+ }}, now, nil)
got := buildExtractedMessage(42, "mem-1", msg, materialized[0])["invalid_at"]
if got != test.want {
t.Fatalf("invalid_at = %#v, want %#v", got, test.want)
@@ -210,7 +211,7 @@ func TestExtractMemoryTaskMarksMissingMemoryPermanent(t *testing.T) {
cleanup := testutil.ReplaceDBForTest(t, db)
defer cleanup()
- svc := NewMemoryMessageService(NewMemoryService())
+ svc := NewMemoryMessageService(NewMemoryService(dao.DB, engine.Get()))
_, err := svc.extractMemoryTask(t.Context(), &entity.MemoryTask{MemoryID: "missing-memory"}, MemoryMessage{})
if !errors.Is(err, errPermanentMemoryTask) || !errors.Is(err, gorm.ErrRecordNotFound) {
t.Fatalf("extractMemoryTask error = %v, want permanent missing-memory error", err)
@@ -224,7 +225,7 @@ func TestExtractMemoryTaskKeepsLookupFailureRetryable(t *testing.T) {
cleanup := testutil.ReplaceDBForTest(t, db)
defer cleanup()
- svc := NewMemoryMessageService(NewMemoryService())
+ svc := NewMemoryMessageService(NewMemoryService(dao.DB, engine.Get()))
_, err := svc.extractMemoryTask(t.Context(), &entity.MemoryTask{MemoryID: "memory-1"}, MemoryMessage{})
if err == nil || errors.Is(err, errPermanentMemoryTask) {
t.Fatalf("extractMemoryTask error = %v, want retryable database error", err)
@@ -238,7 +239,7 @@ func TestExtractByLLMMarksMissingModelPermanent(t *testing.T) {
cleanup := testutil.ReplaceDBForTest(t, db)
defer cleanup()
- svc := NewMemoryMessageService(NewMemoryService())
+ svc := NewMemoryMessageService(NewMemoryService(dao.DB, engine.Get()))
mem := &CreateMemoryResponse{Memory: entity.Memory{
TenantID: "tenant-1",
LLMID: "deleted-model-id",
@@ -278,7 +279,7 @@ func TestExtractByLLMMarksInvalidModelConfigPermanent(t *testing.T) {
t.Fatalf("create model instance: %v", err)
}
- svc := NewMemoryMessageService(NewMemoryService())
+ svc := NewMemoryMessageService(NewMemoryService(dao.DB, engine.Get()))
mem := &CreateMemoryResponse{Memory: entity.Memory{
TenantID: "tenant-1",
LLMID: "gpt-4o@default@OpenAI",
@@ -317,7 +318,7 @@ func TestHandleSaveToMemoryTaskResumesStoredCheckpoint(t *testing.T) {
t.Fatalf("create memory task: %v", err)
}
- svc := NewMemoryMessageService(nil)
+ svc := NewMemoryMessageService(NewMemoryService(dao.DB, engine.Get()))
disposition, err := svc.HandleSaveToMemoryTask(t.Context(), "task-1", "worker-1")
if err != nil || disposition != MemoryTaskAcknowledge {
t.Fatalf("HandleSaveToMemoryTask disposition=%v err=%v", disposition, err)
@@ -380,7 +381,7 @@ func TestHandleSaveToMemoryTaskRetriesCompletionFromStored(t *testing.T) {
t.Fatalf("create update trigger: %v", err)
}
- svc := NewMemoryMessageService(nil)
+ svc := NewMemoryMessageService(NewMemoryService(dao.DB, engine.Get()))
disposition, err := svc.HandleSaveToMemoryTask(t.Context(), "task-completion-retry", "worker-1")
if err == nil || disposition != MemoryTaskAcknowledge {
t.Fatalf("first attempt disposition=%v err=%v, want acknowledged scheduled retry", disposition, err)
@@ -439,7 +440,7 @@ func TestHandleSaveToMemoryTaskResumesExtractedWithoutLLM(t *testing.T) {
t.Fatalf("create memory task: %v", err)
}
- svc := NewMemoryMessageService(nil)
+ svc := NewMemoryMessageService(NewMemoryService(dao.DB, engine.Get()))
disposition, err := svc.HandleSaveToMemoryTask(t.Context(), "task-extracted-resume", "worker-1")
if err != nil || disposition != MemoryTaskAcknowledge {
t.Fatalf("HandleSaveToMemoryTask disposition=%v err=%v", disposition, err)
@@ -471,7 +472,7 @@ func TestHandleSaveToMemoryTaskLeavesActiveLeaseUnsettled(t *testing.T) {
}).Error; err != nil {
t.Fatalf("create memory task: %v", err)
}
- svc := NewMemoryMessageService(nil)
+ svc := NewMemoryMessageService(NewMemoryService(dao.DB, engine.Get()))
if _, acquired, err := svc.memoryTaskDAO.Claim(t.Context(), db, "task-live-lease", "worker-1", now, memoryTaskLeaseTTL); err != nil || !acquired {
t.Fatalf("Claim acquired=%v err=%v, want acquired", acquired, err)
}
@@ -500,7 +501,7 @@ func TestHandleSaveToMemoryTaskFailsTaskMissingDurableState(t *testing.T) {
t.Fatalf("create generic task: %v", err)
}
- svc := NewMemoryMessageService(nil)
+ svc := NewMemoryMessageService(NewMemoryService(dao.DB, engine.Get()))
disposition, err := svc.HandleSaveToMemoryTask(t.Context(), "task-missing-state", "worker-1")
if err == nil || disposition != MemoryTaskAcknowledge || !strings.Contains(err.Error(), "missing durable execution state") {
t.Fatalf("HandleSaveToMemoryTask disposition=%v err=%v, want acknowledged explicit failure", disposition, err)
@@ -522,7 +523,7 @@ func TestHandleSaveToMemoryTaskAcknowledgesUnknownID(t *testing.T) {
cleanup := testutil.ReplaceDBForTest(t, db)
defer cleanup()
- svc := NewMemoryMessageService(nil)
+ svc := NewMemoryMessageService(NewMemoryService(dao.DB, engine.Get()))
disposition, err := svc.HandleSaveToMemoryTask(t.Context(), "unknown-task", "worker-1")
if err == nil || disposition != MemoryTaskAcknowledge || !strings.Contains(err.Error(), "is not found") {
t.Fatalf("HandleSaveToMemoryTask disposition=%v err=%v, want acknowledged unknown task", disposition, err)
@@ -578,7 +579,7 @@ func TestHandleSaveToMemoryTaskDoesNotReportUnpersistedExtraction(t *testing.T)
t.Fatalf("create checkpoint trigger: %v", err)
}
- svc := NewMemoryMessageService(NewMemoryService())
+ svc := NewMemoryMessageService(NewMemoryService(dao.DB, engine.Get()))
disposition, err := svc.HandleSaveToMemoryTask(t.Context(), "task-checkpoint-failure", "worker-1")
if err == nil || disposition != MemoryTaskAcknowledge {
t.Fatalf("HandleSaveToMemoryTask disposition=%v err=%v, want acknowledged scheduled retry", disposition, err)
@@ -611,7 +612,7 @@ func TestPersistMemoryTaskFailureSchedulesRetry(t *testing.T) {
t.Fatalf("create memory task: %v", err)
}
- svc := NewMemoryMessageService(nil)
+ svc := NewMemoryMessageService(NewMemoryService(dao.DB, engine.Get()))
claimed, acquired, err := svc.memoryTaskDAO.Claim(t.Context(), db, "task-retry", "worker-1", now, memoryTaskLeaseTTL)
if err != nil || !acquired {
t.Fatalf("Claim acquired=%v err=%v, want acquired", acquired, err)
@@ -668,7 +669,7 @@ func TestPersistMemoryTaskFailureStopsAtMaxAttempts(t *testing.T) {
t.Fatalf("create memory task: %v", err)
}
- svc := NewMemoryMessageService(nil)
+ svc := NewMemoryMessageService(NewMemoryService(dao.DB, engine.Get()))
claimed, acquired, err := svc.memoryTaskDAO.Claim(t.Context(), db, "task-max-attempts", "worker-1", now, memoryTaskLeaseTTL)
if err != nil || !acquired {
t.Fatalf("Claim acquired=%v err=%v, want acquired", acquired, err)
@@ -714,7 +715,7 @@ func TestPersistMemoryTaskFailureLeavesDeliveryUnsettledWhenRetryWriteFails(t *t
t.Fatalf("create memory task: %v", err)
}
- svc := NewMemoryMessageService(nil)
+ svc := NewMemoryMessageService(NewMemoryService(dao.DB, engine.Get()))
claimed, acquired, err := svc.memoryTaskDAO.Claim(t.Context(), db, "task-retry-write-failure", "worker-1", now, memoryTaskLeaseTTL)
if err != nil || !acquired {
t.Fatalf("Claim acquired=%v err=%v, want acquired", acquired, err)
@@ -759,7 +760,7 @@ func TestRunClaimedMemoryTaskPersistsPanic(t *testing.T) {
t.Fatalf("create memory task: %v", err)
}
- svc := NewMemoryMessageService(nil)
+ svc := NewMemoryMessageService(NewMemoryService(dao.DB, engine.Get()))
svc.resumeTask = func(context.Context, *entity.MemoryTask, string) error {
panic("simulated state-machine panic")
}
@@ -812,7 +813,7 @@ func TestRenewMemoryTaskLeaseRetriesTransientError(t *testing.T) {
t.Fatalf("register renewal callback: %v", err)
}
- svc := NewMemoryMessageService(nil)
+ svc := NewMemoryMessageService(NewMemoryService(dao.DB, engine.Get()))
runCtx, cancel := context.WithCancel(t.Context())
renewErr := make(chan error, 1)
done := make(chan struct{})
@@ -887,7 +888,7 @@ func TestRenewMemoryTaskLeaseTimesOutNearExpiry(t *testing.T) {
t.Fatalf("register blocking update callback: %v", err)
}
- svc := NewMemoryMessageService(nil)
+ svc := NewMemoryMessageService(NewMemoryService(dao.DB, engine.Get()))
runCtx, cancel := context.WithCancel(t.Context())
defer cancel()
renewErr := make(chan error, 1)
@@ -943,7 +944,7 @@ func TestHandleSaveToMemoryTaskMarksInvalidInputFailed(t *testing.T) {
t.Fatalf("create memory task: %v", err)
}
- svc := NewMemoryMessageService(nil)
+ svc := NewMemoryMessageService(NewMemoryService(dao.DB, engine.Get()))
disposition, err := svc.HandleSaveToMemoryTask(t.Context(), "task-invalid", "worker-1")
if err == nil || disposition != MemoryTaskAcknowledge {
t.Fatalf("HandleSaveToMemoryTask disposition=%v err=%v, want acknowledged terminal failure", disposition, err)
@@ -989,7 +990,7 @@ func TestUpdateTaskProgressReturnsPersistenceError(t *testing.T) {
cleanup := testutil.ReplaceDBForTest(t, db)
defer cleanup()
- svc := &MemoryMessageService{taskDAO: dao.NewTaskDAO()}
+ svc := &MemoryMessageService{taskDAO: dao.NewTaskDAO(), memories: NewMemoryService(db, nil)}
if err := svc.updateTaskProgress(t.Context(), "mem-task-1", 1.0, "completed"); err == nil {
t.Fatal("updateTaskProgress() error = nil, want persistence error")
}
diff --git a/internal/service/memory_message_service.go b/internal/service/memory_message_service.go
index ca7f51b6ff..e6b0f79741 100644
--- a/internal/service/memory_message_service.go
+++ b/internal/service/memory_message_service.go
@@ -108,19 +108,35 @@ type MemoryMessageService struct {
memoryTaskDAO *dao.MemoryTaskDAO
taskPublisher TaskPublisher
resumeTask func(context.Context, *entity.MemoryTask, string) error
+ // kvrocksClient mints raw-message ids via Redis auto-increment. It is
+ // injected at construction (inherited from the owning MemoryService) rather
+ // than read from a package global, so the memory module holds no mutable
+ // package-level state and has no init-order race. A nil client falls back
+ // to wall-clock ids.
+ kvrocksClient *kvrocks.Client
}
// NewMemoryMessageService constructs a service bound to the
// supplied MemoryService. Caller is expected to register this as
// the default MemorySaver in the Message component via
// `component.SetMemorySaver(...)` at boot.
+//
+// The kvrocks client used to mint raw-message ids is inherited from the
+// supplied MemoryService (which receives it via WithMemoryKvrocks at the
+// composition root), so the memory module carries no package-level global.
+// When the MemoryService has no client bound, ids fall back to wall-clock
+// values — acceptable for local/test paths.
func NewMemoryMessageService(memories *MemoryService) *MemoryMessageService {
- return &MemoryMessageService{
+ s := &MemoryMessageService{
memories: memories,
taskDAO: dao.NewTaskDAO(),
memoryTaskDAO: dao.NewMemoryTaskDAO(),
taskPublisher: NewMessageQueueTaskPublisher(),
}
+ if memories != nil {
+ s.kvrocksClient = memories.kvrocksClient
+ }
+ return s
}
// queueSaveToMemoryTask runs the memory-persistence flow for the
@@ -163,7 +179,7 @@ func (s *MemoryMessageService) queueSaveToMemoryTask(ctx context.Context, memory
// keeps the same field set as Python:344-386 so the
// downstream extractor can consume the row without
// schema changes.
- rawMessageID := generateRawMessageID(ctx)
+ rawMessageID := generateRawMessageID(ctx, s.kvrocksClient)
rawMessage := buildRawMessage(rawMessageID, memoryID, msg)
task, memoryTask := buildMemoryTaskRecords(rawMessageID, memoryID, msg)
if err := s.insertMemoryTask(ctx, task, memoryTask); err != nil {
@@ -176,7 +192,7 @@ func (s *MemoryMessageService) queueSaveToMemoryTask(ctx context.Context, memory
if err := s.embedAndSave(ctx, mem, rawMessage); err != nil {
failure := fmt.Errorf("store raw memory message: %w", err)
- failed, failErr := s.memoryTaskDAO.MarkUnclaimedFailed(ctx, dao.DB, task.ID, failure.Error())
+ failed, failErr := s.memoryTaskDAO.MarkUnclaimedFailed(ctx, s.memories.db, task.ID, failure.Error())
if failErr != nil {
failure = errors.Join(failure, fmt.Errorf("mark memory task failed: %w", failErr))
} else if !failed {
@@ -203,6 +219,9 @@ func (s *MemoryMessageService) ReconcileMemoryTasks(ctx context.Context, limit i
if s == nil {
return errors.New("memory: nil MemoryMessageService")
}
+ if s.memories == nil {
+ return errors.New("memory: nil memory service dependency")
+ }
if s.memoryTaskDAO == nil {
s.memoryTaskDAO = dao.NewMemoryTaskDAO()
}
@@ -210,7 +229,7 @@ func (s *MemoryMessageService) ReconcileMemoryTasks(ctx context.Context, limit i
return errors.New("memory task publisher is not initialized")
}
- tasks, err := s.memoryTaskDAO.ListDue(ctx, dao.DB, memoryNow(), limit)
+ tasks, err := s.memoryTaskDAO.ListDue(ctx, s.memories.db, memoryNow(), limit)
if err != nil {
return fmt.Errorf("memory: list due tasks: %w", err)
}
@@ -227,10 +246,14 @@ func (s *MemoryMessageService) ReconcileMemoryTasks(ctx context.Context, limit i
}
// generateRawMessageID returns the Redis auto-increment id used by the Python
-// side (`REDIS_CONN.generate_auto_increment_id(namespace="memory")`).
-func generateRawMessageID(ctx context.Context) int64 {
- if redisClient := kvrocks.Get(); redisClient != nil {
- if id := redisClient.GenerateAutoIncrementID(ctx, "id_generator", "memory", 1, nil); id > 0 {
+// side (`REDIS_CONN.generate_auto_increment_id(namespace="memory")`). When kv
+// is nil it falls back to a wall-clock id, so callers that do not configure a
+// kvrocks client (local/test paths) still produce a unique-enough id. The
+// client is passed in explicitly rather than read from a package global, which
+// keeps the memory module free of mutable package-level state and its init race.
+func generateRawMessageID(ctx context.Context, kv *kvrocks.Client) int64 {
+ if kv != nil {
+ if id := kv.GenerateAutoIncrementID(ctx, "id_generator", "memory", 1, nil); id > 0 {
return id
}
}
@@ -380,7 +403,7 @@ func (s *MemoryMessageService) insertMemoryTask(ctx context.Context, task *entit
if s.memoryTaskDAO == nil {
s.memoryTaskDAO = dao.NewMemoryTaskDAO()
}
- return s.memoryTaskDAO.CreateWithTask(ctx, dao.DB, task, memoryTask)
+ return s.memoryTaskDAO.CreateWithTask(ctx, s.memories.db, task, memoryTask)
}
// newUUIDString is a thin wrapper so we can swap in a real UUID
diff --git a/internal/service/memory_message_service_test.go b/internal/service/memory_message_service_test.go
index 7932489d21..f0725f2189 100644
--- a/internal/service/memory_message_service_test.go
+++ b/internal/service/memory_message_service_test.go
@@ -26,6 +26,8 @@ import (
"testing"
"time"
+ "ragflow/internal/dao"
+ "ragflow/internal/engine"
"ragflow/internal/entity"
"ragflow/internal/ingestion/testutil"
)
@@ -78,7 +80,7 @@ func TestQueueSaveToMemoryTaskClassifiesMemoryLookupErrors(t *testing.T) {
cleanup := testutil.ReplaceDBForTest(t, db)
defer cleanup()
- res, err := NewMemoryMessageService(NewMemoryService()).queueSaveToMemoryTask(
+ res, err := NewMemoryMessageService(NewMemoryService(dao.DB, engine.Get())).queueSaveToMemoryTask(
t.Context(), []string{"missing-memory"}, MemoryMessage{AgentID: "agent-1"},
)
if err != nil {
@@ -94,7 +96,7 @@ func TestQueueSaveToMemoryTaskClassifiesMemoryLookupErrors(t *testing.T) {
cleanup := testutil.ReplaceDBForTest(t, db)
defer cleanup()
- res, err := NewMemoryMessageService(NewMemoryService()).queueSaveToMemoryTask(
+ res, err := NewMemoryMessageService(NewMemoryService(dao.DB, engine.Get())).queueSaveToMemoryTask(
t.Context(), []string{"memory-1"}, MemoryMessage{AgentID: "agent-1"},
)
if err != nil {
@@ -125,7 +127,7 @@ func TestQueueSaveToMemoryTaskMarksDurableTaskFailedWhenRawStorageFails(t *testi
}).Error; err != nil {
t.Fatalf("create memory: %v", err)
}
- memorySvc := NewMemoryService()
+ memorySvc := NewMemoryService(dao.DB, engine.Get())
memorySvc.docEngine = nil
svc := NewMemoryMessageService(memorySvc)
publisher := &recordingTaskPublisher{}
@@ -264,8 +266,8 @@ func TestBuildMemoryTaskRecordsShape(t *testing.T) {
// be added when the project's Redis client lands.)
func TestGenerateRawMessageID_Unique(t *testing.T) {
ctx := t.Context()
- a := generateRawMessageID(ctx)
- b := generateRawMessageID(ctx)
+ a := generateRawMessageID(ctx, nil)
+ b := generateRawMessageID(ctx, nil)
if a == b {
// Allow a 1-second tie when the clock hasn't ticked.
// GenerateRawMessageID uses Unix seconds; two calls
diff --git a/internal/service/memory_message_test.go b/internal/service/memory_message_test.go
index 8f2090191f..9e463aa60c 100644
--- a/internal/service/memory_message_test.go
+++ b/internal/service/memory_message_test.go
@@ -121,7 +121,7 @@ func TestMemoryWithAccessReturnsCanceledContext(t *testing.T) {
cancel()
err := ctx.Err()
- if _, gotErr := NewMemoryService().memoryWithAccess(ctx, "user-1", "memory-1", permission.OperationRead); !errors.Is(gotErr, err) {
+ if _, gotErr := NewMemoryService(dao.DB, engine.Get()).memoryWithAccess(ctx, "user-1", "memory-1", permission.OperationRead); !errors.Is(gotErr, err) {
t.Fatalf("memoryWithAccess error = %v, want %v", gotErr, err)
}
}
@@ -215,7 +215,7 @@ func TestListMemoryFiltersUsesAccessibleMemoryAggregates(t *testing.T) {
}
}
- filters, err := NewMemoryService().ListMemoryFilters(t.Context(), "user-1")
+ filters, err := NewMemoryService(dao.DB, engine.Get()).ListMemoryFilters(t.Context(), "user-1")
if err != nil {
t.Fatalf("ListMemoryFilters: %v", err)
}
@@ -243,7 +243,7 @@ func TestListMemoryFiltersUsesAccessibleMemoryAggregates(t *testing.T) {
t.Fatalf("storage type filter = %+v, want %+v", got, want)
}
- refreshed, err := NewMemoryService().ListMemoryFilters(t.Context(), "user-1")
+ refreshed, err := NewMemoryService(dao.DB, engine.Get()).ListMemoryFilters(t.Context(), "user-1")
if err != nil {
t.Fatalf("ListMemoryFilters refresh: %v", err)
}
@@ -278,7 +278,7 @@ func TestListMemoryFiltersKeepsCanonicalFacetOrder(t *testing.T) {
}
}
- filters, err := NewMemoryService().ListMemoryFilters(t.Context(), "user-1")
+ filters, err := NewMemoryService(dao.DB, engine.Get()).ListMemoryFilters(t.Context(), "user-1")
if err != nil {
t.Fatalf("ListMemoryFilters: %v", err)
}
@@ -307,7 +307,7 @@ func TestListMemoryFiltersKeepsCanonicalFacetOrder(t *testing.T) {
if err := dao.DB.Model(&entity.Memory{ID: "mem-semantic"}).Update("update_time", bumped).Error; err != nil {
t.Fatalf("bump update_time: %v", err)
}
- refreshed, err := NewMemoryService().ListMemoryFilters(t.Context(), "user-1")
+ refreshed, err := NewMemoryService(dao.DB, engine.Get()).ListMemoryFilters(t.Context(), "user-1")
if err != nil {
t.Fatalf("ListMemoryFilters refresh: %v", err)
}
@@ -355,7 +355,7 @@ func TestForgetMessageKeepsCompanionFieldForNonOceanBaseEngines(t *testing.T) {
} {
t.Run(test.engineType, func(t *testing.T) {
docEngine := &memoryMessageDocEngine{engineType: test.engineType}
- service := NewMemoryService()
+ service := NewMemoryService(dao.DB, engine.Get())
service.docEngine = docEngine
if err := service.ForgetMessage(t.Context(), "user-1", "memory-1", 42); err != nil {
@@ -413,7 +413,7 @@ func TestUpdateMemoryTeamMemberCannotChangePermissions(t *testing.T) {
t.Fatalf("seed user tenant: %v", err)
}
- svc := NewMemoryService()
+ svc := NewMemoryService(dao.DB, engine.Get())
samePermission := " TEAM "
if _, err := svc.UpdateMemory(t.Context(), "member-1", "mem-team", &UpdateMemoryRequest{
Description: sptr("member edit"),
@@ -496,7 +496,7 @@ func TestUpdateMemoryTeamMemberResolvesModelsAgainstOwnerTenant(t *testing.T) {
}
llmID := "gpt-4o@default@OpenAI"
- if _, err := NewMemoryService().UpdateMemory(t.Context(), "member-1", "mem-model", &UpdateMemoryRequest{
+ if _, err := NewMemoryService(dao.DB, engine.Get()).UpdateMemory(t.Context(), "member-1", "mem-model", &UpdateMemoryRequest{
LLMID: &llmID,
}); err != nil {
t.Fatalf("UpdateMemory model error = %v", err)
@@ -572,7 +572,7 @@ func TestListMemoriesUsesTenantModelIDForDisplayName(t *testing.T) {
}
ctx := t.Context()
- resp, err := NewMemoryService().ListMemories(ctx, "user-1", []string{"user-1"}, nil, "", "", 1, 10, "")
+ resp, err := NewMemoryService(dao.DB, engine.Get()).ListMemories(ctx, "user-1", []string{"user-1"}, nil, "", "", 1, 10, "")
if err != nil {
t.Fatalf("ListMemories: %v", err)
}
@@ -609,7 +609,7 @@ func TestListMemoriesFallsBackToRawModelIDWithoutTenantModelID(t *testing.T) {
}
ctx := t.Context()
- resp, err := NewMemoryService().ListMemories(ctx, "user-1", []string{"user-1"}, nil, "", "", 1, 10, "")
+ resp, err := NewMemoryService(dao.DB, engine.Get()).ListMemories(ctx, "user-1", []string{"user-1"}, nil, "", "", 1, 10, "")
if err != nil {
t.Fatalf("ListMemories: %v", err)
}
@@ -655,7 +655,7 @@ func TestListMemoriesRejectsUnjoinedTenantFilter(t *testing.T) {
t.Fatalf("seed memory: %v", err)
}
- resp, err := NewMemoryService().ListMemories(t.Context(), "user-1", []string{"tenant-2"}, nil, "", "", 1, 10, "")
+ resp, err := NewMemoryService(dao.DB, engine.Get()).ListMemories(t.Context(), "user-1", []string{"tenant-2"}, nil, "", "", 1, 10, "")
if err != nil {
t.Fatalf("ListMemories: %v", err)
}
@@ -718,7 +718,7 @@ func TestListMemoriesKeepsJoinedTenantFilter(t *testing.T) {
ctx := t.Context()
- resp, err := NewMemoryService().ListMemories(ctx, "user-1", []string{"tenant-2"}, nil, "", "", 1, 10, "")
+ resp, err := NewMemoryService(dao.DB, engine.Get()).ListMemories(ctx, "user-1", []string{"tenant-2"}, nil, "", "", 1, 10, "")
if err != nil {
t.Fatalf("ListMemories: %v", err)
}
@@ -726,7 +726,7 @@ func TestListMemoriesKeepsJoinedTenantFilter(t *testing.T) {
t.Fatalf("joined tenant filter must return its team memory, got total=%d list=%v", resp.TotalCount, resp.MemoryList)
}
- resp, err = NewMemoryService().ListMemories(ctx, "user-1", []string{"tenant-2", "tenant-3"}, nil, "", "", 1, 10, "")
+ resp, err = NewMemoryService(dao.DB, engine.Get()).ListMemories(ctx, "user-1", []string{"tenant-2", "tenant-3"}, nil, "", "", 1, 10, "")
if err != nil {
t.Fatalf("ListMemories: %v", err)
}
@@ -773,7 +773,7 @@ func TestListMemoriesForCanvasUsesCallerOwnerIntersection(t *testing.T) {
}
}
- response, err := NewMemoryService().ListMemories(t.Context(), "caller", nil, nil, "", "", 1, 30, canvas.ID)
+ response, err := NewMemoryService(dao.DB, engine.Get()).ListMemories(t.Context(), "caller", nil, nil, "", "", 1, 30, canvas.ID)
if err != nil {
t.Fatalf("ListMemories(canvas): %v", err)
}
@@ -833,7 +833,7 @@ func TestSaveAgentMessageChecksMemoryAccess(t *testing.T) {
t.Fatalf("seed memory: %v", err)
}
- svc := &MemoryService{memoryDAO: dao.NewMemoryDAO()}
+ svc := &MemoryService{memoryDAO: dao.NewMemoryDAO(), db: dao.DB}
msg := MemoryMessage{
AgentID: "agent-1",
SessionID: "session-1",
@@ -871,6 +871,7 @@ func TestGetMessagesFiltersAccessibleMemoryAndBuildsRecentSearch(t *testing.T) {
seedMemoryMessages(t)
docEngine := &memoryMessageDocEngine{
+ engineType: "infinity",
searchResp: &enginetypes.SearchResult{
Total: 1,
Chunks: []map[string]interface{}{
@@ -889,7 +890,7 @@ func TestGetMessagesFiltersAccessibleMemoryAndBuildsRecentSearch(t *testing.T) {
},
},
}
- svc := &MemoryService{memoryDAO: dao.NewMemoryDAO(), docEngine: docEngine}
+ svc := &MemoryService{memoryDAO: dao.NewMemoryDAO(), db: dao.DB, docEngine: docEngine}
got, code, err := svc.GetMessages(t.Context(), []string{"mem-owned", "mem-other"}, "user-1", "agent-1", "session-1", 3)
if err != nil {
@@ -910,11 +911,11 @@ func TestGetMessagesFiltersAccessibleMemoryAndBuildsRecentSearch(t *testing.T) {
t.Fatal("expected doc engine search request")
}
// The per-memory index name gains a `_<memoryID>` suffix on infinity
- // (see memorySearchIndexNames). The engine type is process-global and set
- // by sibling integration tests, so make the expectation engine-aware rather
- // than hard-coding the non-infinity name.
+ // (see memorySearchIndexNames). The service now reads the engine type from
+ // the injected doc engine rather than the process-global engine type, so
+ // derive the expectation from the injected docEngine's type.
wantIndexName := "memory_user-1"
- if engine.GetEngineType() == "infinity" {
+ if docEngine.GetType() == "infinity" {
wantIndexName = "memory_user-1_mem-owned"
}
if !reflect.DeepEqual(req.IndexNames, []string{wantIndexName}) {
@@ -959,7 +960,7 @@ func TestSearchMessageFiltersAccessibleMemoryAndDefaultsStatus(t *testing.T) {
},
},
}
- svc := &MemoryService{memoryDAO: dao.NewMemoryDAO(), docEngine: docEngine}
+ svc := &MemoryService{memoryDAO: dao.NewMemoryDAO(), db: dao.DB, docEngine: docEngine}
filter := map[string]interface{}{
"memory_id": []string{"mem-owned", "mem-other"},
"agent_id": "agent-1",
@@ -1010,7 +1011,7 @@ func TestUpdateMessageUpdatesStatusByMessageDocID(t *testing.T) {
seedMemoryMessages(t)
docEngine := &memoryMessageDocEngine{}
- svc := &MemoryService{memoryDAO: dao.NewMemoryDAO(), docEngine: docEngine}
+ svc := &MemoryService{memoryDAO: dao.NewMemoryDAO(), db: dao.DB, docEngine: docEngine}
ok, err := svc.UpdateMessage(t.Context(), "user-1", "mem-owned", 42, true)
if err != nil {
diff --git a/internal/service/memory_permission.go b/internal/service/memory_permission.go
index d5b434cb9a..1855037e60 100644
--- a/internal/service/memory_permission.go
+++ b/internal/service/memory_permission.go
@@ -20,12 +20,11 @@ import (
"context"
"strings"
- "ragflow/internal/dao"
"ragflow/internal/permission"
)
-func checkMemoryPermission(ctx context.Context, userID, memoryID string, operation permission.Operation) error {
- return permission.NewDatabaseChecker(dao.DB).CheckResource(
+func (s *MemoryService) checkMemoryPermission(ctx context.Context, userID, memoryID string, operation permission.Operation) error {
+ return permission.NewDatabaseChecker(s.db).CheckResource(
ctx,
permission.Subject{UserID: userID},
permission.ResourceRef{Kind: permission.ResourceKindMemory, ID: memoryID},
@@ -33,8 +32,8 @@ func checkMemoryPermission(ctx context.Context, userID, memoryID string, operati
)
}
-func accessibleMemoryIDs(ctx context.Context, userID string, operation permission.Operation) ([]string, error) {
- scope, err := permission.NewDatabaseChecker(dao.DB).Scope(ctx, permission.Subject{UserID: userID}, permission.ScopeQuery{
+func (s *MemoryService) accessibleMemoryIDs(ctx context.Context, userID string, operation permission.Operation) ([]string, error) {
+ scope, err := permission.NewDatabaseChecker(s.db).Scope(ctx, permission.Subject{UserID: userID}, permission.ScopeQuery{
Kind: permission.ResourceKindMemory,
Operation: operation,
})
@@ -47,12 +46,12 @@ func accessibleMemoryIDs(ctx context.Context, userID string, operation permissio
return scope.ResourceIDs, nil
}
-func accessibleMemoryIDsForCanvas(ctx context.Context, userID, canvasID string) ([]string, error) {
+func (s *MemoryService) accessibleMemoryIDsForCanvas(ctx context.Context, userID, canvasID string) ([]string, error) {
canvasID = strings.TrimSpace(canvasID)
if canvasID == "" {
- return accessibleMemoryIDs(ctx, userID, permission.OperationRead)
+ return s.accessibleMemoryIDs(ctx, userID, permission.OperationRead)
}
- scope, err := permission.NewDatabaseChecker(dao.DB).Scope(ctx, permission.Subject{UserID: userID}, permission.ScopeQuery{
+ scope, err := permission.NewDatabaseChecker(s.db).Scope(ctx, permission.Subject{UserID: userID}, permission.ScopeQuery{
Kind: permission.ResourceKindMemory,
Operation: permission.OperationUse,
Entry: &permission.ResourceRef{Kind: permission.ResourceKindCanvas, ID: canvasID},
@@ -67,12 +66,12 @@ func accessibleMemoryIDsForCanvas(ctx context.Context, userID, canvasID string)
return scope.ResourceIDs, nil
}
-func filterMemoryIDs(ctx context.Context, userID string, memoryIDs []string, operation permission.Operation) ([]string, error) {
+func (s *MemoryService) filterMemoryIDs(ctx context.Context, userID string, memoryIDs []string, operation permission.Operation) ([]string, error) {
refs := make([]permission.ResourceRef, 0, len(memoryIDs))
for _, memoryID := range memoryIDs {
refs = append(refs, permission.ResourceRef{Kind: permission.ResourceKindMemory, ID: memoryID})
}
- accessible, err := permission.NewDatabaseChecker(dao.DB).FilterResources(
+ accessible, err := permission.NewDatabaseChecker(s.db).FilterResources(
ctx,
permission.Subject{UserID: userID},
refs,
diff --git a/internal/service/memory_reconciler_test.go b/internal/service/memory_reconciler_test.go
index 055670ef6a..11a2cc113c 100644
--- a/internal/service/memory_reconciler_test.go
+++ b/internal/service/memory_reconciler_test.go
@@ -33,9 +33,6 @@ func TestReconcileMemoryTasksPublishesDueWakeups(t *testing.T) {
now := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
pinMemoryNow(t, now)
db := testutil.SetupTestDB(t, &entity.MemoryTask{})
- cleanup := testutil.ReplaceDBForTest(t, db)
- defer cleanup()
-
future := now.Add(time.Minute)
for _, task := range []*entity.MemoryTask{
{TaskID: "due", MemoryID: "memory-1", SourceID: 1, Input: entity.JSONMap{}, State: entity.MemoryTaskStatePending},
@@ -48,7 +45,7 @@ func TestReconcileMemoryTasksPublishesDueWakeups(t *testing.T) {
}
publisher := &recordingTaskPublisher{}
- svc := NewMemoryMessageService(nil)
+ svc := NewMemoryMessageService(NewMemoryService(db, nil))
svc.taskPublisher = publisher
if err := svc.ReconcileMemoryTasks(t.Context(), 100); err != nil {
t.Fatalf("ReconcileMemoryTasks: %v", err)
@@ -68,9 +65,6 @@ func TestReconcileMemoryTasksStopsPublishingWhenCanceled(t *testing.T) {
now := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
pinMemoryNow(t, now)
db := testutil.SetupTestDB(t, &entity.MemoryTask{})
- cleanup := testutil.ReplaceDBForTest(t, db)
- defer cleanup()
-
for _, taskID := range []string{"due-1", "due-2"} {
if err := db.Create(&entity.MemoryTask{
TaskID: taskID, MemoryID: "memory-1", SourceID: 1,
@@ -82,7 +76,7 @@ func TestReconcileMemoryTasksStopsPublishingWhenCanceled(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
publisher := &recordingTaskPublisher{beforeReturn: func(string) { cancel() }}
- svc := NewMemoryMessageService(nil)
+ svc := NewMemoryMessageService(NewMemoryService(db, nil))
svc.taskPublisher = publisher
err := svc.ReconcileMemoryTasks(ctx, 100)
if !errors.Is(err, context.Canceled) {
diff --git a/internal/service/memory_saver_adapter.go b/internal/service/memory_saver_adapter.go
index 2ad803ce0e..4597ab989f 100644
--- a/internal/service/memory_saver_adapter.go
+++ b/internal/service/memory_saver_adapter.go
@@ -23,7 +23,6 @@ import (
"ragflow/internal/agent/component"
"ragflow/internal/agent/runtime"
- "ragflow/internal/dao"
"ragflow/internal/permission"
)
@@ -64,7 +63,7 @@ func (a *memorySaverAdapter) Save(ctx context.Context, req component.MemorySaveR
if userID == "" {
return permission.ErrUnauthenticated
}
- checker := permission.NewDatabaseChecker(dao.DB)
+ checker := permission.NewDatabaseChecker(a.svc.db)
entry := permission.ResourceRef{Kind: permission.ResourceKindCanvas, ID: canvasID}
for _, memoryID := range req.MemoryIDs {
memoryID = strings.TrimSpace(memoryID)
diff --git a/internal/service/memory_saver_adapter_test.go b/internal/service/memory_saver_adapter_test.go
index f80a1d42d0..f1992c1112 100644
--- a/internal/service/memory_saver_adapter_test.go
+++ b/internal/service/memory_saver_adapter_test.go
@@ -23,6 +23,7 @@ import (
"ragflow/internal/agent/component"
"ragflow/internal/agent/runtime"
"ragflow/internal/dao"
+ "ragflow/internal/engine"
"ragflow/internal/entity"
"ragflow/internal/permission"
)
@@ -73,7 +74,7 @@ func TestMemorySaverAuthorizesWithAuthenticatedCanvasCaller(t *testing.T) {
// req.UserID is the memory-message partition key and must not be trusted as
// the permission subject. The canvas owner can access memory-1, but caller
// cannot, so using req.UserID here would incorrectly authorize this write.
- err := NewMemorySaverAdapter(NewMemoryService()).Save(ctx, component.MemorySaveRequest{
+ err := NewMemorySaverAdapter(NewMemoryService(dao.DB, engine.Get())).Save(ctx, component.MemorySaveRequest{
MemoryIDs: []string{memory.ID},
UserID: "canvas-owner",
AgentID: canvas.ID,
diff --git a/internal/service/tag.go b/internal/service/tag.go
index a6b57bdf8d..414b9b649f 100644
--- a/internal/service/tag.go
+++ b/internal/service/tag.go
@@ -23,6 +23,7 @@ import (
"go.uber.org/zap"
"ragflow/internal/common"
+ "ragflow/internal/dao"
"ragflow/internal/entity"
"ragflow/internal/ingestion/component"
)
@@ -109,7 +110,7 @@ func (s *MetadataService) LabelQuestion(ctx context.Context, question string, kb
zap.Int("kbCount", len(kbs)))
return nil
}
- idx, err := component.LoadTagQueryIndex(ctx, refs)
+ idx, err := component.LoadTagQueryIndex(ctx, dao.DB, refs)
if err != nil {
common.WarnCtx(ctx, "LabelQuestion: failed to load tag source index",
zap.Int("tagSources", len(refs)), zap.Error(err))