Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(memory): inject MQ engine into memory publish path (#21244)

xugangqiang904983591b64
diff --git a/internal/ingestion/arch_test.go b/internal/ingestion/arch_test.go
index 9827682c23..e51be18f20 100644
--- a/internal/ingestion/arch_test.go
+++ b/internal/ingestion/arch_test.go
@@ -80,6 +80,20 @@ var scanRoots = []string{
// memoryFilePrefix scopes the internal/service scan to the memory module
// files only.
+//
+// Blind spots — read before trusting a green run:
+// - internal/service is only scanned for files whose base name starts with
+// "memory". Production files that feed the memory (or other DI-converged)
+// modules but are not memory*-named — e.g. task_publisher.go (the memory
+// module's MQ publisher) or document/document.go (which injects the
+// knowledge_compile publisher) — are invisible to this guard and may still
+// read process-wide singletons. Their global reads are only safe because a
+// composition root reads the singleton to inject it; the guard cannot see
+// that intent, so reviewers must check such sites manually.
+// - Detection is AST-based and only matches literal pkg.Symbol selectors.
+// A helper that wraps a singleton read (e.g. chunkcache.Client() wrapping
+// kvrocks.Get()) is invisible even within a scanned file; it relies on the
+// globalAccessWhitelist to be declared as an explicit, approved access.
const memoryFilePrefix = "memory"
// findModuleRoot walks up from the working directory until it finds go.mod.
diff --git a/internal/ingestion/chunkcache/chunkcache.go b/internal/ingestion/chunkcache/chunkcache.go
index 6f0d8efb3b..f691e9274b 100644
--- a/internal/ingestion/chunkcache/chunkcache.go
+++ b/internal/ingestion/chunkcache/chunkcache.go
@@ -66,6 +66,14 @@ type Store interface {
// is not configured. Always resolve through this function: passing kvrocks.Get()
// directly into a Store parameter would wrap a typed nil pointer in a non-nil
// interface, defeating the nil guards below.
+//
+// Reading the process-wide kvrocks singleton here is intentional and correct
+// (design-by-intent), NOT a convergence gap: the chunkcache.Store seam is the
+// one place components legitimately reach the singleton, because the canvas
+// runtime that carries chunk-cache callers does not thread a kvrocks handle.
+// This is the sanctioned resolver whitelisted in internal/ingestion/arch_test.go,
+// so the AST guard treats it as an explicit, reviewer-approved access rather
+// than an unguarded global read.
func Client() Store {
if c := kvrocks.Get(); c != nil {
return c
diff --git a/internal/server/api/server.go b/internal/server/api/server.go
index f2307d4fd6..07212a81d3 100644
--- a/internal/server/api/server.go
+++ b/internal/server/api/server.go
@@ -125,7 +125,7 @@ func startServer(ctx context.Context, serverName string, arguments *bootstrap.Se
searchService := service.NewSearchService()
searchService.SetTenantService(tenantService)
fileService := file.NewFileService(documentService)
- memoryService := service.NewMemoryService(dao.DB, engine.Get(), service.WithMemoryKvrocks(kvrocks.Get()))
+ memoryService := service.NewMemoryService(dao.DB, engine.Get(), service.WithMemoryKvrocks(kvrocks.Get()), service.WithMemoryMQEngine(engine.GetMessageQueueEngine()))
mcpService := service.NewMCPService()
modelProviderService := service.NewModelProviderService()
modelFactory := service.NewModelFactory()
diff --git a/internal/server/ingestor/server.go b/internal/server/ingestor/server.go
index 22d224ff13..6f9d5e9e01 100644
--- a/internal/server/ingestor/server.go
+++ b/internal/server/ingestor/server.go
@@ -191,7 +191,7 @@ func Run(ctx context.Context, cancel context.CancelFunc, serverName string, args
// Memory extraction runs on the Ingestor's shared NATS consumer + worker
// pool (task_type="memory" dispatched by handleAndExecute -> executeMemoryTask),
// so there is no longer a dedicated Redis memory consumer to start.
- ingestor.SetMemoryMessageService(service.NewMemoryMessageService(service.NewMemoryService(dao.DB, engine.Get(), service.WithMemoryKvrocks(kvrocks.Get()))))
+ ingestor.SetMemoryMessageService(service.NewMemoryMessageService(service.NewMemoryService(dao.DB, engine.Get(), service.WithMemoryKvrocks(kvrocks.Get()), service.WithMemoryMQEngine(engine.GetMessageQueueEngine()))))
// Bind the relational handle and document engine into the production
// embedder and knowledge-compiler resolvers. The task package is the
// composition root for ingestion runs; it reads these from the explicit
diff --git a/internal/service/memory.go b/internal/service/memory.go
index cead57f89c..92f6792003 100644
--- a/internal/service/memory.go
+++ b/internal/service/memory.go
@@ -289,6 +289,7 @@ type MemoryService struct {
db *gorm.DB
docEngine engine.DocEngine
kvrocksClient *kvrocks.Client
+ mqEngine engine.MessageQueue
}
// MemoryServiceOption configures a MemoryService at construction time.
@@ -301,6 +302,14 @@ func WithMemoryKvrocks(kv *kvrocks.Client) MemoryServiceOption {
return func(s *MemoryService) { s.kvrocksClient = kv }
}
+// WithMemoryMQEngine binds the MQ engine used by the memory module's publish
+// path (MemoryMessageService -> MessageQueueTaskPublisher). It keeps the
+// dependency explicit rather than read from the process-wide
+// engine.GetMessageQueueEngine() global.
+func WithMemoryMQEngine(mq engine.MessageQueue) MemoryServiceOption {
+ return func(s *MemoryService) { s.mqEngine = mq }
+}
+
// NewMemoryService creates a new MemoryService instance. db and eng are the
// process-wide relational handle and doc engine; they are resolved once at the
// server assembly point and injected here rather than read from the package
diff --git a/internal/service/memory_message_service.go b/internal/service/memory_message_service.go
index e6b0f79741..f309107e2c 100644
--- a/internal/service/memory_message_service.go
+++ b/internal/service/memory_message_service.go
@@ -61,6 +61,7 @@ import (
"ragflow/internal/common"
"ragflow/internal/dao"
+ "ragflow/internal/engine"
"ragflow/internal/engine/kvrocks"
"ragflow/internal/entity"
"ragflow/internal/entity/models"
@@ -114,6 +115,11 @@ type MemoryMessageService struct {
// package-level state and has no init-order race. A nil client falls back
// to wall-clock ids.
kvrocksClient *kvrocks.Client
+ // mqEngine backs the publish path. It is injected at construction
+ // (inherited from the owning MemoryService) rather than read from the
+ // process-wide engine.GetMessageQueueEngine() global, so the memory
+ // module's publish path carries no package-level global dependency.
+ mqEngine engine.MessageQueue
}
// NewMemoryMessageService constructs a service bound to the
@@ -127,16 +133,19 @@ type MemoryMessageService struct {
// When the MemoryService has no client bound, ids fall back to wall-clock
// values — acceptable for local/test paths.
func NewMemoryMessageService(memories *MemoryService) *MemoryMessageService {
- s := &MemoryMessageService{
+ var mq engine.MessageQueue
+ var kv *kvrocks.Client
+ if memories != nil {
+ mq = memories.mqEngine
+ kv = memories.kvrocksClient
+ }
+ return &MemoryMessageService{
memories: memories,
taskDAO: dao.NewTaskDAO(),
memoryTaskDAO: dao.NewMemoryTaskDAO(),
- taskPublisher: NewMessageQueueTaskPublisher(),
- }
- if memories != nil {
- s.kvrocksClient = memories.kvrocksClient
+ taskPublisher: NewMessageQueueTaskPublisherWithEngine(mq),
+ kvrocksClient: kv,
}
- return s
}
// queueSaveToMemoryTask runs the memory-persistence flow for the
diff --git a/internal/service/task_publisher.go b/internal/service/task_publisher.go
index b8d14753f3..b9e70ad0e4 100644
--- a/internal/service/task_publisher.go
+++ b/internal/service/task_publisher.go
@@ -12,14 +12,33 @@ type TaskPublisher interface {
PublishTaskMessage(subject string, msg common.TaskMessage) error
}
-type MessageQueueTaskPublisher struct{}
+type MessageQueueTaskPublisher struct {
+ // mqEngine is the injected MQ engine. A nil value falls back to the
+ // process-wide engine.GetMessageQueueEngine() for legacy callers that have
+ // not migrated to injection. The memory module always injects, so its
+ // publish path never reads the global.
+ mqEngine engine.MessageQueue
+}
func NewMessageQueueTaskPublisher() *MessageQueueTaskPublisher {
return &MessageQueueTaskPublisher{}
}
+// NewMessageQueueTaskPublisherWithEngine builds a publisher backed by the
+// injected MQ engine. The memory module uses this so its publish path does not
+// reach the process-wide engine.GetMessageQueueEngine() global.
+func NewMessageQueueTaskPublisherWithEngine(mq engine.MessageQueue) *MessageQueueTaskPublisher {
+ return &MessageQueueTaskPublisher{mqEngine: mq}
+}
+
func (p *MessageQueueTaskPublisher) PublishTaskMessage(subject string, msg common.TaskMessage) error {
- msgQueueEngine := engine.GetMessageQueueEngine()
+ // Prefer the injected engine; fall back to the process-wide global only for
+ // legacy callers that construct the publisher without injection. The memory
+ // module always injects, so its publish path never reads the global.
+ msgQueueEngine := p.mqEngine
+ if msgQueueEngine == nil {
+ msgQueueEngine = engine.GetMessageQueueEngine()
+ }
if msgQueueEngine == nil {
return fmt.Errorf("message queue engine is not initialized")
}
diff --git a/internal/service/task_publisher_test.go b/internal/service/task_publisher_test.go
new file mode 100644
index 0000000000..09b17b8493
--- /dev/null
+++ b/internal/service/task_publisher_test.go
@@ -0,0 +1,105 @@
+package service
+
+import (
+ "encoding/json"
+ "sync"
+ "testing"
+
+ "ragflow/internal/common"
+ "ragflow/internal/engine"
+)
+
+// recordingMessageQueue is a fake engine.MessageQueue that records publishes.
+// It embeds the interface so it satisfies MessageQueue; only PublishTask is
+// overridden. Other methods are never called by the publisher under test.
+type recordingMessageQueue struct {
+ engine.MessageQueue
+ mu sync.Mutex
+ subjects []string
+ messages []common.TaskMessage
+}
+
+func (r *recordingMessageQueue) PublishTask(subject string, payload []byte) error {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ r.subjects = append(r.subjects, subject)
+ var m common.TaskMessage
+ _ = json.Unmarshal(payload, &m)
+ r.messages = append(r.messages, m)
+ return nil
+}
+
+// TestMessageQueueTaskPublisherUsesInjectedEngineNotGlobal verifies the
+// injected engine wins over the process-wide global: publishing through a
+// publisher built with NewMessageQueueTaskPublisherWithEngine must hit the
+// injected engine and must NOT touch engine.GetMessageQueueEngine(). This is
+// the closed-DI guarantee for the memory module's publish path (F1).
+func TestMessageQueueTaskPublisherUsesInjectedEngineNotGlobal(t *testing.T) {
+ globalMQ := &recordingMessageQueue{}
+ engine.SetMessageQueueEngine(globalMQ)
+ defer engine.SetMessageQueueEngine(nil)
+
+ injectedMQ := &recordingMessageQueue{}
+ pub := NewMessageQueueTaskPublisherWithEngine(injectedMQ)
+
+ if err := pub.PublishTaskMessage(common.TaskSubject, common.TaskMessage{
+ TaskID: "task-1",
+ TaskType: common.TaskTypeMemory,
+ }); err != nil {
+ t.Fatalf("PublishTaskMessage: %v", err)
+ }
+
+ if len(injectedMQ.messages) != 1 || injectedMQ.messages[0].TaskID != "task-1" {
+ t.Fatalf("injected engine got %+v, want one publish for task-1", injectedMQ.messages)
+ }
+ if len(globalMQ.messages) != 0 {
+ t.Fatalf("global engine was used (%d publishes); injected engine should win", len(globalMQ.messages))
+ }
+}
+
+// TestMemoryMessageServicePublishesViaInjectedEngine verifies the inheritance
+// chain MemoryService.WithMemoryMQEngine -> MemoryMessageService -> its
+// publisher uses the injected engine, not the global (F1, service level).
+func TestMemoryMessageServicePublishesViaInjectedEngine(t *testing.T) {
+ globalMQ := &recordingMessageQueue{}
+ engine.SetMessageQueueEngine(globalMQ)
+ defer engine.SetMessageQueueEngine(nil)
+
+ injectedMQ := &recordingMessageQueue{}
+ memSvc := NewMemoryService(nil, nil, WithMemoryMQEngine(injectedMQ))
+ svc := NewMemoryMessageService(memSvc)
+
+ if svc.taskPublisher == nil {
+ t.Fatal("MemoryMessageService.taskPublisher is nil")
+ }
+ if err := publishMemoryTaskWakeup(svc.taskPublisher, "task-1"); err != nil {
+ t.Fatalf("publishMemoryTaskWakeup: %v", err)
+ }
+ if len(injectedMQ.messages) != 1 {
+ t.Fatalf("injected engine got %d publishes, want 1", len(injectedMQ.messages))
+ }
+ if len(globalMQ.messages) != 0 {
+ t.Fatalf("global engine was used; injected engine should win")
+ }
+}
+
+// TestMessageQueueTaskPublisherNilEngineFallsBackToGlobal verifies the legacy
+// no-arg constructor still resolves through the global when no engine is
+// injected, so non-memory callers (out of the memory module's DI scope) keep
+// working.
+func TestMessageQueueTaskPublisherNilEngineFallsBackToGlobal(t *testing.T) {
+ globalMQ := &recordingMessageQueue{}
+ engine.SetMessageQueueEngine(globalMQ)
+ defer engine.SetMessageQueueEngine(nil)
+
+ pub := NewMessageQueueTaskPublisher()
+ if err := pub.PublishTaskMessage(common.TaskSubject, common.TaskMessage{
+ TaskID: "task-2",
+ TaskType: common.TaskTypeMemory,
+ }); err != nil {
+ t.Fatalf("PublishTaskMessage: %v", err)
+ }
+ if len(globalMQ.messages) != 1 {
+ t.Fatalf("global fallback got %d publishes, want 1", len(globalMQ.messages))
+ }
+}