RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix knowledge compiler Go port regressions (MySQL 1101 + Start deadlock) (#17599)
Zhichang Yu650387b2fded
diff --git a/internal/engine/global.go b/internal/engine/global.goindex 8d9657b74d..32a6f0f6b1 100644--- a/internal/engine/global.go+++ b/internal/engine/global.go@@ -88,6 +88,13 @@ func GetMessageQueueEngine() MessageQueue {return messageQueueEngine}+// SetMessageQueueEngine installs the global message-queue engine. It exists+// primarily as a test seam so callers can drive Start() without a real server+// config; production code uses InitMessageQueueEngine.+func SetMessageQueueEngine(mq MessageQueue) {+ messageQueueEngine = mq+}+func InitMessageQueueEngine(messageQueueType string) error {config := server.GetConfig()switch messageQueueType {diff --git a/internal/entity/knowledge_compile_doc.go b/internal/entity/knowledge_compile_doc.goindex 80af64c9ab..5ac1c205e6 100644--- a/internal/entity/knowledge_compile_doc.go+++ b/internal/entity/knowledge_compile_doc.go@@ -30,10 +30,14 @@ import "time"// out-of-order / tombstone guards as the broker-based design without re-reading// the queue.type KnowledgeCompileDoc struct {- DatasetID string `gorm:"primaryKey;column:dataset_id;size:64" json:"dataset_id"`- TenantID string `gorm:"column:tenant_id;size:64;not null;default:''" json:"tenant_id"`- BacklogDocIDs string `gorm:"column:backlog_doc_ids;type:text;not null;default:'[]'" json:"backlog_doc_ids"`- InflightDocIDs string `gorm:"column:inflight_doc_ids;type:text;not null;default:'[]'" json:"inflight_doc_ids"`+ DatasetID string `gorm:"primaryKey;column:dataset_id;size:64" json:"dataset_id"`+ TenantID string `gorm:"column:tenant_id;size:64;not null;default:''" json:"tenant_id"`+ // The *_doc_ids columns store a JSON array as TEXT. No DDL default is set:+ // MySQL (8.0.13+) rejects a literal DEFAULT on TEXT/BLOB columns (Error+ // 1101), and the application always writes "[]" explicitly on insert/update+ // (scheduler.go FirstOrCreate / release paths), so the default is redundant.+ BacklogDocIDs string `gorm:"column:backlog_doc_ids;type:text;not null" json:"backlog_doc_ids"`+ InflightDocIDs string `gorm:"column:inflight_doc_ids;type:text;not null" json:"inflight_doc_ids"`ClaimOwner string `gorm:"column:claim_owner;size:64;not null;default:''" json:"claim_owner"`ClaimToken string `gorm:"column:claim_token;size:64;not null;default:''" json:"claim_token"`ClaimExpiresAt *time.Time `gorm:"column:claim_expires_at;default:null" json:"claim_expires_at"`diff --git a/internal/ingestion/service/ingestion_service.go b/internal/ingestion/service/ingestion_service.goindex d4b2b4bdeb..877c97186b 100644--- a/internal/ingestion/service/ingestion_service.go+++ b/internal/ingestion/service/ingestion_service.go@@ -64,10 +64,11 @@ type Ingestor struct {ShutdownCh chan struct{}// Worker pool- taskChan chan *taskpkg.TaskContext- workerWg sync.WaitGroup- startOnce sync.Once- stopOnce sync.Once // guards close(ShutdownCh) against double-close on repeated Stop+ taskChan chan *taskpkg.TaskContext+ workerWg sync.WaitGroup+ startOnce sync.Once+ workerOnce sync.Once // guards startWorkerPool; must NOT be startOnce (Start wraps start() in startOnce, and start() calls startWorkerPool -> re-entry deadlock)+ stopOnce sync.Once // guards close(ShutdownCh) against double-close on repeated StopingestionTaskSvc *servicepkg.IngestionTaskServicedocState *docStateUpdater@@ -382,7 +383,7 @@ func (e *Ingestor) processMessage(handle common.TaskHandle) {}func (e *Ingestor) startWorkerPool() {- e.startOnce.Do(func() {+ e.workerOnce.Do(func() {for i := int32(0); i < e.maxConcurrency; i++ {e.workerWg.Add(1)go e.workerLoop(i)diff --git a/internal/ingestion/service/ingestor_lifecycle_test.go b/internal/ingestion/service/ingestor_lifecycle_test.goindex 3c4ce2cab5..cb95f882a4 100644--- a/internal/ingestion/service/ingestor_lifecycle_test.go+++ b/internal/ingestion/service/ingestor_lifecycle_test.go@@ -22,6 +22,7 @@ import ("time""ragflow/internal/common"+ "ragflow/internal/engine""ragflow/internal/entity"taskpkg "ragflow/internal/ingestion/task""ragflow/internal/ingestion/testutil"@@ -200,3 +201,49 @@ func TestPollCancel_ExitsWhenDoneClosed(t *testing.T) {close(released) // cleanup}++// TestStart_FullPathReturnsAndStartsWorkers is a regression test for the+// sync.Once re-entrancy deadlock. Before the fix, Start() wrapped the whole+// startup (start()) in e.startOnce.Do, but start() also called startWorkerPool()+// which nested the SAME startOnce. sync.Once.Do blocks forever when re-entered+// from inside its own callback, so Start() hung after InitConsumer succeeded:+// no worker pool, no consumeLoop, and ingestion tasks were never consumed.+//+// The test drives the real Start() path (start -> startWorkerPool -> consumeLoop)+// against an embedded NATS server and asserts Start() returns within a deadline+// and that workers are actually up.+func TestStart_FullPathReturnsAndStartsWorkers(t *testing.T) {+ // SetMessageQueueEngine mutates process-global state; restore the previous+ // engine so later tests don't inherit a closed embedded NATS server.+ previousEngine := engine.GetMessageQueueEngine()+ engine.SetMessageQueueEngine(testutil.SetupNatsEngine(t))+ t.Cleanup(func() { engine.SetMessageQueueEngine(previousEngine) })++ const concurrency int32 = 2+ ing := NewIngestor("test-start-fullpath", concurrency, nil)+ t.Cleanup(func() { ing.Stop(context.Background()) })++ done := make(chan error, 1)+ go func() {+ done <- ing.Start()+ }()++ select {+ case err := <-done:+ if err != nil {+ t.Fatalf("Start() returned error: %v", err)+ }+ case <-time.After(10 * time.Second):+ t.Fatal("Start() did not return within 10s; sync.Once re-entrancy deadlock likely")+ }++ // Start() launches workers asynchronously and returns immediately; poll+ // briefly so we don't observe zero before a worker enters workerLoop.+ deadline := time.Now().Add(2 * time.Second)+ for time.Now().Before(deadline) && ing.activeWorkers.Load() <= 0 {+ time.Sleep(time.Millisecond)+ }+ if got := ing.activeWorkers.Load(); got <= 0 {+ t.Fatalf("expected activeWorkers > 0 after Start(), got %d", got)+ }+}