RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(memory): inject MQ engine into memory publish path (#21244)
xugangqiang904983591b64
diff --git a/internal/ingestion/arch_test.go b/internal/ingestion/arch_test.goindex 9827682c23..e51be18f20 100644--- a/internal/ingestion/arch_test.go+++ b/internal/ingestion/arch_test.go@@ -80,6 +80,20 @@ var scanRoots = []string{// memoryFilePrefix scopes the internal/service scan to the memory module// files only.+//+// Blind spots — read before trusting a green run:+// - internal/service is only scanned for files whose base name starts with+// "memory". Production files that feed the memory (or other DI-converged)+// modules but are not memory*-named — e.g. task_publisher.go (the memory+// module's MQ publisher) or document/document.go (which injects the+// knowledge_compile publisher) — are invisible to this guard and may still+// read process-wide singletons. Their global reads are only safe because a+// composition root reads the singleton to inject it; the guard cannot see+// that intent, so reviewers must check such sites manually.+// - Detection is AST-based and only matches literal pkg.Symbol selectors.+// A helper that wraps a singleton read (e.g. chunkcache.Client() wrapping+// kvrocks.Get()) is invisible even within a scanned file; it relies on the+// globalAccessWhitelist to be declared as an explicit, approved access.const memoryFilePrefix = "memory"// findModuleRoot walks up from the working directory until it finds go.mod.diff --git a/internal/ingestion/chunkcache/chunkcache.go b/internal/ingestion/chunkcache/chunkcache.goindex 6f0d8efb3b..f691e9274b 100644--- a/internal/ingestion/chunkcache/chunkcache.go+++ b/internal/ingestion/chunkcache/chunkcache.go@@ -66,6 +66,14 @@ type Store interface {// is not configured. Always resolve through this function: passing kvrocks.Get()// directly into a Store parameter would wrap a typed nil pointer in a non-nil// interface, defeating the nil guards below.+//+// Reading the process-wide kvrocks singleton here is intentional and correct+// (design-by-intent), NOT a convergence gap: the chunkcache.Store seam is the+// one place components legitimately reach the singleton, because the canvas+// runtime that carries chunk-cache callers does not thread a kvrocks handle.+// This is the sanctioned resolver whitelisted in internal/ingestion/arch_test.go,+// so the AST guard treats it as an explicit, reviewer-approved access rather+// than an unguarded global read.func Client() Store {if c := kvrocks.Get(); c != nil {return cdiff --git a/internal/server/api/server.go b/internal/server/api/server.goindex f2307d4fd6..07212a81d3 100644--- a/internal/server/api/server.go+++ b/internal/server/api/server.go@@ -125,7 +125,7 @@ func startServer(ctx context.Context, serverName string, arguments *bootstrap.SesearchService := service.NewSearchService()searchService.SetTenantService(tenantService)fileService := file.NewFileService(documentService)- memoryService := service.NewMemoryService(dao.DB, engine.Get(), service.WithMemoryKvrocks(kvrocks.Get()))+ memoryService := service.NewMemoryService(dao.DB, engine.Get(), service.WithMemoryKvrocks(kvrocks.Get()), service.WithMemoryMQEngine(engine.GetMessageQueueEngine()))mcpService := service.NewMCPService()modelProviderService := service.NewModelProviderService()modelFactory := service.NewModelFactory()diff --git a/internal/server/ingestor/server.go b/internal/server/ingestor/server.goindex 22d224ff13..6f9d5e9e01 100644--- a/internal/server/ingestor/server.go+++ b/internal/server/ingestor/server.go@@ -191,7 +191,7 @@ func Run(ctx context.Context, cancel context.CancelFunc, serverName string, args// Memory extraction runs on the Ingestor's shared NATS consumer + worker// pool (task_type="memory" dispatched by handleAndExecute -> executeMemoryTask),// so there is no longer a dedicated Redis memory consumer to start.- ingestor.SetMemoryMessageService(service.NewMemoryMessageService(service.NewMemoryService(dao.DB, engine.Get(), service.WithMemoryKvrocks(kvrocks.Get()))))+ ingestor.SetMemoryMessageService(service.NewMemoryMessageService(service.NewMemoryService(dao.DB, engine.Get(), service.WithMemoryKvrocks(kvrocks.Get()), service.WithMemoryMQEngine(engine.GetMessageQueueEngine()))))// Bind the relational handle and document engine into the production// embedder and knowledge-compiler resolvers. The task package is the// composition root for ingestion runs; it reads these from the explicitdiff --git a/internal/service/memory.go b/internal/service/memory.goindex cead57f89c..92f6792003 100644--- a/internal/service/memory.go+++ b/internal/service/memory.go@@ -289,6 +289,7 @@ type MemoryService struct {db *gorm.DBdocEngine engine.DocEnginekvrocksClient *kvrocks.Client+ mqEngine engine.MessageQueue}// MemoryServiceOption configures a MemoryService at construction time.@@ -301,6 +302,14 @@ func WithMemoryKvrocks(kv *kvrocks.Client) MemoryServiceOption {return func(s *MemoryService) { s.kvrocksClient = kv }}+// WithMemoryMQEngine binds the MQ engine used by the memory module's publish+// path (MemoryMessageService -> MessageQueueTaskPublisher). It keeps the+// dependency explicit rather than read from the process-wide+// engine.GetMessageQueueEngine() global.+func WithMemoryMQEngine(mq engine.MessageQueue) MemoryServiceOption {+ return func(s *MemoryService) { s.mqEngine = mq }+}+// NewMemoryService creates a new MemoryService instance. db and eng are the// process-wide relational handle and doc engine; they are resolved once at the// server assembly point and injected here rather than read from the packagediff --git a/internal/service/memory_message_service.go b/internal/service/memory_message_service.goindex e6b0f79741..f309107e2c 100644--- a/internal/service/memory_message_service.go+++ b/internal/service/memory_message_service.go@@ -61,6 +61,7 @@ import ("ragflow/internal/common""ragflow/internal/dao"+ "ragflow/internal/engine""ragflow/internal/engine/kvrocks""ragflow/internal/entity""ragflow/internal/entity/models"@@ -114,6 +115,11 @@ type MemoryMessageService struct {// package-level state and has no init-order race. A nil client falls back// to wall-clock ids.kvrocksClient *kvrocks.Client+ // mqEngine backs the publish path. It is injected at construction+ // (inherited from the owning MemoryService) rather than read from the+ // process-wide engine.GetMessageQueueEngine() global, so the memory+ // module's publish path carries no package-level global dependency.+ mqEngine engine.MessageQueue}// NewMemoryMessageService constructs a service bound to the@@ -127,16 +133,19 @@ type MemoryMessageService struct {// When the MemoryService has no client bound, ids fall back to wall-clock// values — acceptable for local/test paths.func NewMemoryMessageService(memories *MemoryService) *MemoryMessageService {- s := &MemoryMessageService{+ var mq engine.MessageQueue+ var kv *kvrocks.Client+ if memories != nil {+ mq = memories.mqEngine+ kv = memories.kvrocksClient+ }+ return &MemoryMessageService{memories: memories,taskDAO: dao.NewTaskDAO(),memoryTaskDAO: dao.NewMemoryTaskDAO(),- taskPublisher: NewMessageQueueTaskPublisher(),- }- if memories != nil {- s.kvrocksClient = memories.kvrocksClient+ taskPublisher: NewMessageQueueTaskPublisherWithEngine(mq),+ kvrocksClient: kv,}- return s}// queueSaveToMemoryTask runs the memory-persistence flow for thediff --git a/internal/service/task_publisher.go b/internal/service/task_publisher.goindex b8d14753f3..b9e70ad0e4 100644--- a/internal/service/task_publisher.go+++ b/internal/service/task_publisher.go@@ -12,14 +12,33 @@ type TaskPublisher interface {PublishTaskMessage(subject string, msg common.TaskMessage) error}-type MessageQueueTaskPublisher struct{}+type MessageQueueTaskPublisher struct {+ // mqEngine is the injected MQ engine. A nil value falls back to the+ // process-wide engine.GetMessageQueueEngine() for legacy callers that have+ // not migrated to injection. The memory module always injects, so its+ // publish path never reads the global.+ mqEngine engine.MessageQueue+}func NewMessageQueueTaskPublisher() *MessageQueueTaskPublisher {return &MessageQueueTaskPublisher{}}+// NewMessageQueueTaskPublisherWithEngine builds a publisher backed by the+// injected MQ engine. The memory module uses this so its publish path does not+// reach the process-wide engine.GetMessageQueueEngine() global.+func NewMessageQueueTaskPublisherWithEngine(mq engine.MessageQueue) *MessageQueueTaskPublisher {+ return &MessageQueueTaskPublisher{mqEngine: mq}+}+func (p *MessageQueueTaskPublisher) PublishTaskMessage(subject string, msg common.TaskMessage) error {- msgQueueEngine := engine.GetMessageQueueEngine()+ // Prefer the injected engine; fall back to the process-wide global only for+ // legacy callers that construct the publisher without injection. The memory+ // module always injects, so its publish path never reads the global.+ msgQueueEngine := p.mqEngine+ if msgQueueEngine == nil {+ msgQueueEngine = engine.GetMessageQueueEngine()+ }if msgQueueEngine == nil {return fmt.Errorf("message queue engine is not initialized")}diff --git a/internal/service/task_publisher_test.go b/internal/service/task_publisher_test.gonew file mode 100644index 0000000000..09b17b8493--- /dev/null+++ b/internal/service/task_publisher_test.go@@ -0,0 +1,105 @@+package service++import (+ "encoding/json"+ "sync"+ "testing"++ "ragflow/internal/common"+ "ragflow/internal/engine"+)++// recordingMessageQueue is a fake engine.MessageQueue that records publishes.+// It embeds the interface so it satisfies MessageQueue; only PublishTask is+// overridden. Other methods are never called by the publisher under test.+type recordingMessageQueue struct {+ engine.MessageQueue+ mu sync.Mutex+ subjects []string+ messages []common.TaskMessage+}++func (r *recordingMessageQueue) PublishTask(subject string, payload []byte) error {+ r.mu.Lock()+ defer r.mu.Unlock()+ r.subjects = append(r.subjects, subject)+ var m common.TaskMessage+ _ = json.Unmarshal(payload, &m)+ r.messages = append(r.messages, m)+ return nil+}++// TestMessageQueueTaskPublisherUsesInjectedEngineNotGlobal verifies the+// injected engine wins over the process-wide global: publishing through a+// publisher built with NewMessageQueueTaskPublisherWithEngine must hit the+// injected engine and must NOT touch engine.GetMessageQueueEngine(). This is+// the closed-DI guarantee for the memory module's publish path (F1).+func TestMessageQueueTaskPublisherUsesInjectedEngineNotGlobal(t *testing.T) {+ globalMQ := &recordingMessageQueue{}+ engine.SetMessageQueueEngine(globalMQ)+ defer engine.SetMessageQueueEngine(nil)++ injectedMQ := &recordingMessageQueue{}+ pub := NewMessageQueueTaskPublisherWithEngine(injectedMQ)++ if err := pub.PublishTaskMessage(common.TaskSubject, common.TaskMessage{+ TaskID: "task-1",+ TaskType: common.TaskTypeMemory,+ }); err != nil {+ t.Fatalf("PublishTaskMessage: %v", err)+ }++ if len(injectedMQ.messages) != 1 || injectedMQ.messages[0].TaskID != "task-1" {+ t.Fatalf("injected engine got %+v, want one publish for task-1", injectedMQ.messages)+ }+ if len(globalMQ.messages) != 0 {+ t.Fatalf("global engine was used (%d publishes); injected engine should win", len(globalMQ.messages))+ }+}++// TestMemoryMessageServicePublishesViaInjectedEngine verifies the inheritance+// chain MemoryService.WithMemoryMQEngine -> MemoryMessageService -> its+// publisher uses the injected engine, not the global (F1, service level).+func TestMemoryMessageServicePublishesViaInjectedEngine(t *testing.T) {+ globalMQ := &recordingMessageQueue{}+ engine.SetMessageQueueEngine(globalMQ)+ defer engine.SetMessageQueueEngine(nil)++ injectedMQ := &recordingMessageQueue{}+ memSvc := NewMemoryService(nil, nil, WithMemoryMQEngine(injectedMQ))+ svc := NewMemoryMessageService(memSvc)++ if svc.taskPublisher == nil {+ t.Fatal("MemoryMessageService.taskPublisher is nil")+ }+ if err := publishMemoryTaskWakeup(svc.taskPublisher, "task-1"); err != nil {+ t.Fatalf("publishMemoryTaskWakeup: %v", err)+ }+ if len(injectedMQ.messages) != 1 {+ t.Fatalf("injected engine got %d publishes, want 1", len(injectedMQ.messages))+ }+ if len(globalMQ.messages) != 0 {+ t.Fatalf("global engine was used; injected engine should win")+ }+}++// TestMessageQueueTaskPublisherNilEngineFallsBackToGlobal verifies the legacy+// no-arg constructor still resolves through the global when no engine is+// injected, so non-memory callers (out of the memory module's DI scope) keep+// working.+func TestMessageQueueTaskPublisherNilEngineFallsBackToGlobal(t *testing.T) {+ globalMQ := &recordingMessageQueue{}+ engine.SetMessageQueueEngine(globalMQ)+ defer engine.SetMessageQueueEngine(nil)++ pub := NewMessageQueueTaskPublisher()+ if err := pub.PublishTaskMessage(common.TaskSubject, common.TaskMessage{+ TaskID: "task-2",+ TaskType: common.TaskTypeMemory,+ }); err != nil {+ t.Fatalf("PublishTaskMessage: %v", err)+ }+ if len(globalMQ.messages) != 1 {+ t.Fatalf("global fallback got %d publishes, want 1", len(globalMQ.messages))+ }+}