RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: synchronize document compilation and navigation projections (#20030)
buua4369e99f996ff7e
diff --git a/internal/entity/models/jiekouai.go b/internal/entity/models/jiekouai.goindex 926f26776a..e1c7484e3b 100644--- a/internal/entity/models/jiekouai.go+++ b/internal/entity/models/jiekouai.go@@ -25,7 +25,6 @@ import ("net/http""ragflow/internal/common""strings"- "time")type JieKouAIModel struct {@@ -33,16 +32,11 @@ type JieKouAIModel struct {}func NewJieKouAIModel(baseURL map[string]string, urlSuffix URLSuffix) *JieKouAIModel {- // JieKouAI's methods issue requests without a per-call context deadline, so- // keep an explicit 120s client-level timeout to bound them. Built on the- // shared transport via NewDriverHTTPClient.- client := common.GetSSRFHTTPClient()- client.Timeout = 120 * time.Secondreturn &JieKouAIModel{baseModel: BaseModel{BaseURL: baseURL,URLSuffix: urlSuffix,- httpClient: client,+ httpClient: common.GetSSRFHTTPClient(),},}}@@ -376,7 +370,10 @@ func (j *JieKouAIModel) ListModels(ctx context.Context, apiConfig *APIConfig) ([}url := fmt.Sprintf("%s/%s", resolvedBaseURL, j.baseModel.URLSuffix.Models)- req, err := http.NewRequest("GET", url, nil)+ ctx, cancel := context.WithTimeout(ctx, nonStreamCallTimeout)+ defer cancel()++ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)if err != nil {return nil, fmt.Errorf("failed to create request: %w", err)}diff --git a/internal/entity/models/jina.go b/internal/entity/models/jina.goindex 35022dc363..530de837f7 100644--- a/internal/entity/models/jina.go+++ b/internal/entity/models/jina.go@@ -27,7 +27,6 @@ import ("net/url""ragflow/internal/common""strings"- "time""unicode/utf8""go.uber.org/zap"@@ -41,13 +40,11 @@ func NewJinaModel(baseURL map[string]string, urlSuffix URLSuffix) *JinaModel {// Embed/Rerank/ListModels issue requests without a per-call context// deadline, so keep an explicit 90s client-level timeout to bound them.// Built on the shared transport via NewDriverHTTPClient.- client := common.GetSSRFHTTPClient()- client.Timeout = 90 * time.Secondreturn &JinaModel{baseModel: BaseModel{BaseURL: baseURL,URLSuffix: urlSuffix,- httpClient: client,+ httpClient: common.GetSSRFHTTPClient(),},}}diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.goindex b079d70e8f..f8598b783b 100644--- a/internal/ingestion/knowledge_compile/consumer.go+++ b/internal/ingestion/knowledge_compile/consumer.go@@ -775,14 +775,15 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e}}- // A0-4 dispatch: route tree/structure products to the dataset-navigation tree- // (NavService.UpsertDoc), and keep wiki (and only wiki) products on the- // existing dataset-level merge path below. This is the per-variant dispatch- // the plan requires: tree and structure both produce a nav by-product (B2),- // so both upsert their summary into the cross-document nav tree; wiki products- // continue to the page-specific evidence merge.+ // Route Tree and PageIndex products to the dataset-navigation tree. Wiki,+ // Graph, Timeline, and Mindmap products have no navigation projection.navIn := navInputFromProducts(kb, incoming)c.reportProgress(ctx, tenant, kb, token, taskTypes, 0.40, "merging_navigation", fmt.Sprintf("Merging navigation products: %d", len(navIn)))+ if docsWithoutNav := completedDocsWithoutNavInput(completed, navIn); len(docsWithoutNav) > 0 {+ if err := c.removeNavLocked(ctx, tenant, kb, token, docsWithoutNav); err != nil {+ return err+ }+ }if len(navIn) > 0 {ns := nav.GetNavService()if ns == nil {@@ -1171,6 +1172,20 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, ereturn nil}+func completedDocsWithoutNavInput(completed []BacklogEntry, inputs []nav.UpsertDocInput) []string {+ withNav := make(map[string]struct{}, len(inputs))+ for _, input := range inputs {+ withNav[input.DocID] = struct{}{}+ }+ withoutNav := make([]string, 0, len(completed))+ for _, entry := range completed {+ if _, ok := withNav[entry.DocID]; !ok {+ withoutNav = append(withoutNav, entry.DocID)+ }+ }+ return uniqueSortedStrings(withoutNav)+}+func copyMeta(input map[string]any) map[string]any {output := make(map[string]any, len(input)+2)for key, value := range input {@@ -1451,17 +1466,15 @@ func productsForVariants(products []kccommon.Product, variants []string) []kccomreturn out}-// navInputFromProducts extracts the dataset-navigation upsert inputs for the-// tree and structure products in a batch (B2: nav trigger is tree || structure).+// navInputFromProducts extracts the dataset-navigation upsert inputs for tree+// and PageIndex products in a batch.// Summary extraction matches the component's by-product hooks:// - tree: the root product (Meta.kind=="root") carries the document summary in// Content and its vector in Vector.-// - structure: the graph product (Meta.kind=="graph") carries the graph JSON in-// Content; its entity descriptions are folded into a document-level summary-// via pageIndexSummary.+// - PageIndex: entity descriptions are folded into a document-level summary.//// Each document contributes at most one nav input, keyed by DocID, so a doc that-// yields several tree/structure products upserts once. NavService embeds the+// yields several tree/PageIndex products upserts once. NavService embeds the// summary itself when Embedd is empty, so the consumer needs no embedder.func navInputFromProducts(kb string, products []kccommon.Product) []nav.UpsertDocInput {type acc struct {@@ -1486,11 +1499,9 @@ func navInputFromProducts(kb string, products []kccommon.Product) []nav.UpsertDoa.treeSummary = p.Contenta.treeEmbedd = p.Vectorcase kccommon.VariantStructure:- // Python's page_index path rebuilds the doc graph from the stored- // entity rows and folds their descriptions into the nav summary- // (runner.py rebuild_structure_graph_json + _page_index_graph_summary).- // The graph blob product (Meta.kind=="graph") is gone from the- // storage model, so fold directly from the entity rows.+ if p.Kind != "page_index" {+ continue+ }if kind, _ := p.Meta["kind"].(string); kind != "entity" {continue}@@ -1789,7 +1800,7 @@ func appendUnique(dst, values []string) []string {return dst}-// upsertNavLocked places the batch's tree/structure summaries into the dataset+// upsertNavLocked places the batch's Tree/PageIndex summaries into the dataset// navigation tree under the per-dataset write/rebuild lock, with the claim-token// check inside the lock (B7: nav writes are claim-fenced like any other// destructive write so a stale worker cannot repopulate a rebuilt tree).diff --git a/internal/ingestion/knowledge_compile/event.go b/internal/ingestion/knowledge_compile/event.goindex affe050870..8102e9501e 100644--- a/internal/ingestion/knowledge_compile/event.go+++ b/internal/ingestion/knowledge_compile/event.go@@ -118,13 +118,13 @@ func PublishCompleted(ctx context.Context, tenantID, datasetID, docID string, va}// PublishDeleted records a doc_deleted event the same way (Publish handles the-// append + notify pairing). taskTypes identifies the document-level artifacts-// that were removed before the event was published.-func PublishDeleted(ctx context.Context, tenantID, datasetID, docID string, taskTypes []string) error {+// append + notify pairing). variants and taskTypes identify the document-level+// artifacts that were removed before the event was published.+func PublishDeleted(ctx context.Context, tenantID, datasetID, docID string, variants, taskTypes []string) error {if defaultPublisher == nil {return nil}- return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeDeleted), nil, taskTypes)+ return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeDeleted), variants, taskTypes)}// PublishEnabled records that a document with compiled products becamediff --git a/internal/ingestion/knowledge_compile/nav_dispatch_test.go b/internal/ingestion/knowledge_compile/nav_dispatch_test.goindex fa2aea0bed..36e146ba98 100644--- a/internal/ingestion/knowledge_compile/nav_dispatch_test.go+++ b/internal/ingestion/knowledge_compile/nav_dispatch_test.go@@ -9,13 +9,10 @@ import ("ragflow/internal/service/nav")-// TestNavInputFromProducts_TreeAndStructure covers B2: nav is fed by BOTH tree-// and structure products (not tree alone). Tree uses the root product's summary;-// structure folds the per-document entity ROW descriptions into a summary-// (mirroring Python runner.py rebuild_structure_graph_json +-// _page_index_graph_summary — the graph blob product is gone from the storage-// model).-func TestNavInputFromProducts_TreeAndStructure(t *testing.T) {+// TestNavInputFromProducts_TreeAndPageIndex verifies that navigation is fed by+// tree roots and PageIndex entity rows. Other structure kinds do not create+// navigation entries.+func TestNavInputFromProducts_TreeAndPageIndex(t *testing.T) {products := []kccommon.Product{// tree: root product carries the doc summary + vector.{DocID: "d1", TenantID: "t1", Variant: kccommon.VariantTree,@@ -24,17 +21,21 @@ func TestNavInputFromProducts_TreeAndStructure(t *testing.T) {// tree: a non-root summary node is NOT a nav input.{DocID: "d1", TenantID: "t1", Variant: kccommon.VariantTree,Content: "section body", Meta: map[string]any{"kind": "summary", "level": 0}},- // structure: entity rows fold their descriptions into the doc summary.+ // PageIndex entity rows fold their descriptions into the doc summary.{DocID: "d2", TenantID: "t1", Variant: kccommon.VariantStructure,Content: `{"name":"Engine","type":"component","description":"a propulsion device"}`,- Meta: map[string]any{"kind": "entity", "compile_kwd": "page_index"}},+ Kind: "page_index", Meta: map[string]any{"kind": "entity"}},{DocID: "d2", TenantID: "t1", Variant: kccommon.VariantStructure,Content: `{"name":"Turbine","type":"component","description":"converts flow into rotation"}`,- Meta: map[string]any{"kind": "entity", "compile_kwd": "page_index"}},- // structure: a relation row is NOT a nav summary line.+ Kind: "page_index", Meta: map[string]any{"kind": "entity"}},+ // A PageIndex relation is not a nav summary line.{DocID: "d2", TenantID: "t1", Variant: kccommon.VariantStructure,Content: `{"from":"Engine","to":"Turbine","type":"drives"}`,- Meta: map[string]any{"kind": "relation", "compile_kwd": "page_index", "from": "Engine", "to": "Turbine"}},+ Kind: "page_index", Meta: map[string]any{"kind": "relation", "from": "Engine", "to": "Turbine"}},+ // Graph entities must not create navigation entries.+ {DocID: "d3", TenantID: "t1", Variant: kccommon.VariantStructure,+ Content: `{"name":"Graph entity","description":"must not become navigation"}`,+ Kind: "knowledge_graph", Meta: map[string]any{"kind": "entity"}},}got := navInputFromProducts("kb1", products)@@ -55,23 +56,26 @@ func TestNavInputFromProducts_TreeAndStructure(t *testing.T) {if !reflect.DeepEqual(treeIn.Embedd, []float32{0.1, 0.2}) {t.Errorf("tree embedd should be the root vector, got %v", treeIn.Embedd)}- structIn, ok := byDoc["d2"]+ pageIndexIn, ok := byDoc["d2"]if !ok {- t.Fatal("missing structure nav input for d2")+ t.Fatal("missing PageIndex nav input for d2")}- if !strings.Contains(structIn.Summary, "Engine: a propulsion device") {- t.Errorf("structure summary = %q, want folded entity descriptions", structIn.Summary)+ if _, found := byDoc["d3"]; found {+ t.Fatal("Graph structure product must not produce navigation input")}- if !strings.Contains(structIn.Summary, "Turbine: converts flow into rotation") {- t.Errorf("structure summary = %q, want both entity lines", structIn.Summary)+ if !strings.Contains(pageIndexIn.Summary, "Engine: a propulsion device") {+ t.Errorf("PageIndex summary = %q, want folded entity descriptions", pageIndexIn.Summary)}- if strings.Contains(structIn.Summary, "drives") {- t.Errorf("structure summary must not include relation rows, got %q", structIn.Summary)+ if !strings.Contains(pageIndexIn.Summary, "Turbine: converts flow into rotation") {+ t.Errorf("PageIndex summary = %q, want both entity lines", pageIndexIn.Summary)+ }+ if strings.Contains(pageIndexIn.Summary, "drives") {+ t.Errorf("PageIndex summary must not include relation rows, got %q", pageIndexIn.Summary)}// The entity rows' vectors are NOT the summary vector: leave Embedd empty so// NavService embeds the folded summary text.- if len(structIn.Embedd) != 0 {- t.Errorf("structure embedd should be empty (NavService re-embeds), got %v", structIn.Embedd)+ if len(pageIndexIn.Embedd) != 0 {+ t.Errorf("PageIndex embedd should be empty (NavService re-embeds), got %v", pageIndexIn.Embedd)}}@@ -95,9 +99,9 @@ func TestNavInputFromProducts_EmptySummaryDropped(t *testing.T) {products := []kccommon.Product{// tree root with empty content -> skip.{DocID: "d1", TenantID: "t1", Variant: kccommon.VariantTree, Meta: map[string]any{"kind": "root"}},- // structure entity with no description -> skip.+ // PageIndex entity with no description -> skip.{DocID: "d2", TenantID: "t1", Variant: kccommon.VariantStructure,- Content: `{"name":"A","description":""}`, Meta: map[string]any{"kind": "entity"}},+ Kind: "page_index", Content: `{"name":"A","description":""}`, Meta: map[string]any{"kind": "entity"}},}got := navInputFromProducts("kb1", products)if len(got) != 0 {@@ -109,7 +113,7 @@ func TestNavInputFromProducts_TreeSummaryWinsRegardlessOfOrder(t *testing.T) {products := []kccommon.Product{{DocID: "d1", TenantID: "t1", Variant: kccommon.VariantStructure,Content: `{"name":"A","description":"structure summary"}`,- Meta: map[string]any{"kind": "entity"}},+ Kind: "page_index", Meta: map[string]any{"kind": "entity"}},{DocID: "d1", TenantID: "t1", Variant: kccommon.VariantTree,Content: "tree summary", Vector: []float32{0.1},Meta: map[string]any{"kind": "root"}},diff --git a/internal/ingestion/knowledge_compile/nav_lifecycle_test.go b/internal/ingestion/knowledge_compile/nav_lifecycle_test.goindex 50e7d55d2d..fe752cebad 100644--- a/internal/ingestion/knowledge_compile/nav_lifecycle_test.go+++ b/internal/ingestion/knowledge_compile/nav_lifecycle_test.go@@ -140,3 +140,38 @@ func TestProcessBatchReaddsEnabledDocumentToNavigation(t *testing.T) {t.Fatalf("upserted navigation input = %+v, want enabled document summary", recorder.upserted[0])}}++func TestProcessBatchRemovesNavigationForGraphOnlyDocument(t *testing.T) {+ previous := nav.GetNavService()+ recorder := &recordingNavService{}+ nav.SetNavService(recorder)+ defer nav.SetNavService(previous)++ consumer := NewConsumer(+ NewFakeScheduler(),+ WithReader(&fakeReader{products: []kccommon.Product{{+ DocID: "doc-graph",+ TenantID: "tenant-1",+ Variant: kccommon.VariantStructure,+ Kind: "knowledge_graph",+ Content: `{"name":"Graph entity","description":"not navigation"}`,+ Meta: map[string]any{"kind": "entity"},+ }}}),+ WithWriter(&fakeWriter{}),+ WithDeduperFactory(func(string) (Deduper, error) { return NewNoopDeduper(), nil }),+ withWikiContributionStore(&memoryWikiContributionStore{items: map[string]wikiDocumentContribution{}}),+ )++ err := consumer.processBatch(context.Background(), "tenant-1", "kb-1", "", []BacklogEntry{{+ DocID: "doc-graph", EventType: string(EventTypeCompleted), Variants: []string{string(kccommon.VariantStructure)}, TaskTypes: []string{kccommon.TaskTypeGraph},+ }}, nil)+ if err != nil {+ t.Fatalf("processBatch failed: %v", err)+ }+ if len(recorder.upserted) != 0 {+ t.Fatalf("Graph-only document upserted navigation: %+v", recorder.upserted)+ }+ if len(recorder.removed) != 1 || recorder.removed[0] != "doc-graph" {+ t.Fatalf("removed navigation docs = %v, want [doc-graph]", recorder.removed)+ }+}diff --git a/internal/ingestion/knowledge_compile/scheduler.go b/internal/ingestion/knowledge_compile/scheduler.goindex a65149ffbe..c3d28403e7 100644--- a/internal/ingestion/knowledge_compile/scheduler.go+++ b/internal/ingestion/knowledge_compile/scheduler.go@@ -63,9 +63,9 @@ const (// into per-variant sub-batches and dispatch each to its own dataset-level// compile path. TaskTypes preserves the original frontend-facing category for// each template, so structure sub-kinds such as Graph, Timeline, and PageIndex-// can be logged separately. Both fields are empty (nil) for deleted events and-// for events published before they existed; the consumer falls back to the-// legacy unified path when the fields are absent.+// can be logged separately. Both fields are empty (nil) for events published+// before they existed; the consumer falls back to the legacy unified path when+// the fields are absent.type BacklogEntry struct {DocID string `json:"doc_id"`EventType string `json:"event_type"`@@ -96,7 +96,7 @@ type Publisher interface {// each other's backlog, and it always pairs the append with a notify.// variants carries the doc-level compile types (tree/structure/wiki/mindmap)// for completed events; taskTypes carries the frontend-facing categories for- // those templates. Both are nil for deleted events.+ // those templates, including the products removed by deleted events.Publish(ctx context.Context, tenantID, datasetID, docID, eventType string, variants, taskTypes []string) error}diff --git a/internal/ingestion/knowledge_compile/scheduler_test.go b/internal/ingestion/knowledge_compile/scheduler_test.goindex 2e3d3e4a03..c70f07e36b 100644--- a/internal/ingestion/knowledge_compile/scheduler_test.go+++ b/internal/ingestion/knowledge_compile/scheduler_test.go@@ -56,6 +56,38 @@ func TestFakeSchedulerReclaimsInterruptedClaim(t *testing.T) {}}+func TestPublishDeletedPreservesRemovedProductTypes(t *testing.T) {+ scheduler := NewFakeScheduler()+ previousPublisher, previousClaimer := defaultPublisher, defaultClaimer+ defaultPublisher, defaultClaimer = scheduler, scheduler+ t.Cleanup(func() {+ defaultPublisher, defaultClaimer = previousPublisher, previousClaimer+ })++ variants := []string{"mindmap", "structure"}+ taskTypes := []string{"Graph", "MindMap"}+ if err := PublishDeleted(t.Context(), "t1", "kb1", "d1", variants, taskTypes); err != nil {+ t.Fatalf("publish deleted: %v", err)+ }+ claim, ok, err := scheduler.Claim(t.Context(), "kb1")+ if err != nil || !ok {+ t.Fatalf("claim deleted event: ok=%v err=%v", ok, err)+ }+ if len(claim.Entries) != 1 {+ t.Fatalf("entries = %d, want 1", len(claim.Entries))+ }+ entry := claim.Entries[0]+ if entry.EventType != string(EventTypeDeleted) {+ t.Fatalf("event type = %q, want %q", entry.EventType, EventTypeDeleted)+ }+ if !reflect.DeepEqual(entry.Variants, variants) {+ t.Fatalf("variants = %v, want %v", entry.Variants, variants)+ }+ if !reflect.DeepEqual(entry.TaskTypes, taskTypes) {+ t.Fatalf("task types = %v, want %v", entry.TaskTypes, taskTypes)+ }+}+func TestFakeSchedulerProgressIsClaimScoped(t *testing.T) {f := NewFakeScheduler()if err := f.Publish(t.Context(), "t1", "kb1", "d1", string(EventTypeCompleted), nil, nil); err != nil {diff --git a/internal/ingestion/service/ingestion_service.go b/internal/ingestion/service/ingestion_service.goindex 89c4aaddef..ee59a79ed6 100644--- a/internal/ingestion/service/ingestion_service.go+++ b/internal/ingestion/service/ingestion_service.go@@ -70,12 +70,14 @@ type Ingestor struct {memoryReconcileBatchSize int// Runtime state- currentTasks map[string]struct{} // set of task IDs currently claimed by a worker- tasksMu sync.RWMutex- activeWorkers atomic.Int32 // number of worker goroutines currently in workerLoop- activeLeases map[*Heartbeat]*activeLease- leasesMu sync.Mutex- stopLeases atomic.Bool+ currentTasks map[string]struct{} // set of task IDs currently claimed by a worker+ tasksMu sync.RWMutex+ pendingCompileEvents map[string]pendingDocumentCompileEvent+ compileEventsMu sync.Mutex+ activeWorkers atomic.Int32 // number of worker goroutines currently in workerLoop+ activeLeases map[*Heartbeat]*activeLease+ leasesMu sync.Mutex+ stopLeases atomic.Bool// Shutdown channel - receive on this to trigger graceful shutdownShutdownCh chan struct{}@@ -148,6 +150,12 @@ type activeLease struct {abandoned atomic.Bool}+type pendingDocumentCompileEvent struct {+ tenantID string+ variants []string+ taskTypes []string+}+func NewIngestor(name string, maxConcurrency int32, supportedTypes []string) *Ingestor {if maxConcurrency <= 0 {maxConcurrency = int32(runtime.NumCPU())@@ -166,6 +174,7 @@ func NewIngestor(name string, maxConcurrency int32, supportedTypes []string) *InsupportedDocTypes: supportedTypes,version: "1.0.0",currentTasks: make(map[string]struct{}),+ pendingCompileEvents: make(map[string]pendingDocumentCompileEvent),activeLeases: make(map[*Heartbeat]*activeLease),workerQueue: make(chan *worker, maxConcurrency),ShutdownCh: make(chan struct{}, 1),@@ -828,6 +837,7 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) bool}if err := e.runDocumentTask(ctx, task); err != nil {+ e.clearPendingCompileEvent(task.ID)if errors.Is(err, context.Canceled) {if e.ctx.Err() != nil || e.dispatchCtx.Err() != nil {common.Info(fmt.Sprintf("Task %s pipeline interrupted by ingestor shutdown, leaving for redelivery", task.ID))@@ -863,6 +873,7 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) boolreturn false}e.recordTerminalPipelineLog(ctx, task, string(entity.TaskStatusDone), "Task completed.")+ e.publishPendingCompileEvent(ctx, task)common.Info(fmt.Sprintf("Task %s completed", task.ID))return true@@ -1214,10 +1225,50 @@ func (e *Ingestor) defaultRunDocumentTask(ctx context.Context, ingestionTask *enif err != nil {return err}+ e.stagePendingCompileEvent(ingestionTask.ID, docTaskCtx.Tenant.ID, result)e.docState.apply(ctx, result)return nil}+func (e *Ingestor) stagePendingCompileEvent(taskID, tenantID string, result *taskpkg.PipelineResult) {+ if taskID == "" {+ return+ }+ e.compileEventsMu.Lock()+ defer e.compileEventsMu.Unlock()+ if result == nil || len(result.CompiledVariants) == 0 {+ delete(e.pendingCompileEvents, taskID)+ return+ }+ e.pendingCompileEvents[taskID] = pendingDocumentCompileEvent{+ tenantID: tenantID,+ variants: append([]string(nil), result.CompiledVariants...),+ taskTypes: append([]string(nil), result.CompiledTaskTypes...),+ }+}++func (e *Ingestor) clearPendingCompileEvent(taskID string) {+ e.compileEventsMu.Lock()+ delete(e.pendingCompileEvents, taskID)+ e.compileEventsMu.Unlock()+}++func (e *Ingestor) publishPendingCompileEvent(ctx context.Context, task *entity.IngestionTask) {+ if task == nil {+ return+ }+ e.compileEventsMu.Lock()+ event, ok := e.pendingCompileEvents[task.ID]+ delete(e.pendingCompileEvents, task.ID)+ e.compileEventsMu.Unlock()+ if !ok || len(event.variants) == 0 {+ return+ }+ if err := knowledge_compile.PublishCompleted(ctx, event.tenantID, task.DatasetID, task.DocumentID, event.variants, event.taskTypes); err != nil {+ common.Logger.Warn(fmt.Sprintf("knowledge_compile: publish doc_completed for %s failed: %v", task.DocumentID, err))+ }+}+func (e *Ingestor) recordTerminalPipelineLog(ctx context.Context, ingestionTask *entity.IngestionTask, status, message string) {if ingestionTask == nil || status == "" {returndiff --git a/internal/ingestion/task/chunk_index_writer.go b/internal/ingestion/task/chunk_index_writer.goindex bfdffab918..ad86a0c0f5 100644--- a/internal/ingestion/task/chunk_index_writer.go+++ b/internal/ingestion/task/chunk_index_writer.go@@ -37,10 +37,11 @@ type InsertFunc func(ctx context.Context, chunks []map[string]any, baseName, dat// chunkIndexWriter batches chunks and writes them to the search engine in// bulkSize-sized batches. Progress is reported every 128 batches.type chunkIndexWriter struct {- insertFunc InsertFunc- baseName string- datasetID string- bulkSize int+ insertFunc InsertFunc+ finalInsertFunc InsertFunc+ baseName string+ datasetID string+ bulkSize int}// newChunkIndexWriter creates a chunkIndexWriter. When bulkSize is <= 0 the@@ -52,13 +53,19 @@ func newChunkIndexWriter(bulkSize int,) *chunkIndexWriter {return &chunkIndexWriter{- insertFunc: insertFunc,- baseName: baseName,- datasetID: datasetID,- bulkSize: bulkSize,+ insertFunc: insertFunc,+ finalInsertFunc: insertFunc,+ baseName: baseName,+ datasetID: datasetID,+ bulkSize: bulkSize,}}+func (w *chunkIndexWriter) withFinalInsertFunc(insertFunc InsertFunc) *chunkIndexWriter {+ w.finalInsertFunc = insertFunc+ return w+}+// Write inserts chunks in batches. An empty or nil slice is forwarded to the// backend as-is.func (w *chunkIndexWriter) Write(ctx context.Context, chunks []map[string]any) error {@@ -78,9 +85,13 @@ func (w *chunkIndexWriter) Write(ctx context.Context, chunks []map[string]any) eif err := ctx.Err(); err != nil {return err}+ insert := w.insertFunc+ if end == len(chunks) && w.finalInsertFunc != nil {+ insert = w.finalInsertFunc+ }var err errorfor attempt := 1; attempt <= chunkInsertAttempts; attempt++ {- _, err = w.insertFunc(ctx, chunks[b:end], w.baseName, w.datasetID)+ _, err = insert(ctx, chunks[b:end], w.baseName, w.datasetID)if err == nil {break}diff --git a/internal/ingestion/task/chunk_index_writer_test.go b/internal/ingestion/task/chunk_index_writer_test.goindex d3c05f485b..ad459429a8 100644--- a/internal/ingestion/task/chunk_index_writer_test.go+++ b/internal/ingestion/task/chunk_index_writer_test.go@@ -19,6 +19,8 @@ package taskimport ("context""errors"+ "fmt"+ "reflect""testing""time")@@ -97,6 +99,29 @@ func TestChunkIndexWriter_MultipleBatches(t *testing.T) {}}+func TestChunkIndexWriter_UsesFinalInserterForLastBatch(t *testing.T) {+ var calls []string+ w := newChunkIndexWriter(+ func(_ context.Context, chunks []map[string]any, _, _ string) ([]string, error) {+ calls = append(calls, fmt.Sprintf("regular:%d", len(chunks)))+ return nil, nil+ },+ "base",+ "kb-1",+ 3,+ ).withFinalInsertFunc(func(_ context.Context, chunks []map[string]any, _, _ string) ([]string, error) {+ calls = append(calls, fmt.Sprintf("final:%d", len(chunks)))+ return nil, nil+ })++ if err := w.Write(t.Context(), make([]map[string]any, 7)); err != nil {+ t.Fatalf("unexpected error: %v", err)+ }+ if want := []string{"regular:3", "regular:3", "final:1"}; !reflect.DeepEqual(calls, want) {+ t.Fatalf("insert calls = %v, want %v", calls, want)+ }+}+func TestChunkIndexWriter_BulkSizeZero(t *testing.T) {var lastBatchSize intw := newChunkIndexWriter(diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.goindex bfc224d4ce..e298cf59ce 100644--- a/internal/ingestion/task/pipeline_executor.go+++ b/internal/ingestion/task/pipeline_executor.go@@ -58,6 +58,11 @@ type PipelineResult struct {DocName stringBuiltInMetadataConfig []anyAutoMetadataEnabled bool+ // CompiledVariants and CompiledTaskTypes identify the document-level+ // products that the dataset consumer must merge after the ingestion task+ // reaches its terminal COMPLETED state.+ CompiledVariants []string+ CompiledTaskTypes []string// MessageID is the polling key for the debug-run log. The front-end reads// it from the run response and polls GET /agents/:id/logs/:message_id to// render progress; it is empty for non-debug (persist) runs.@@ -104,16 +109,13 @@ type noRefreshChunkInserter interface {InsertChunksNoRefresh(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string) ([]string, error)}-// insertChunksForIngestion writes chunks through the engine's no-refresh path-// when it offers one, falling back to the plain inserter otherwise.+// insertChunksForIngestion writes non-final chunks through the engine's+// no-refresh path when it offers one, falling back to the plain inserter+// otherwise.//-// Waiting for a refresh costs ~0.5s per write (median; p90 1.0s, max 5.0s) with-// the KB index's refresh_interval=1000ms, and it buys nothing during ingestion:-// the index publishes the chunks on its own refresh cycle a moment later, so the-// only difference is up to a second before they are searchable. It is also what-// Python's ingestion does - it inserts with refresh=False-// (rag/svr/task_executor_refactor/chunk_service.py:386 and :423) while its API-// default stays "wait_for" - so this keeps the two ingestion paths in step.+// Refreshing every batch adds unnecessary latency. The writer uses the regular+// inserter for its final batch, so all preceding batches become searchable+// before the document task is acknowledged and its completion event is sent.func insertChunksForIngestion(eng engine.DocEngine) InsertFunc {return func(ctx context.Context, chunks []map[string]any, baseName string, datasetID string) ([]string, error) {if bulk, ok := eng.(noRefreshChunkInserter); ok {@@ -123,6 +125,15 @@ func insertChunksForIngestion(eng engine.DocEngine) InsertFunc {}}+// insertFinalChunksForIngestion waits for the index refresh before returning.+// The final write makes every preceding no-refresh batch searchable before the+// document task is acknowledged and its completion event is published.+func insertFinalChunksForIngestion(eng engine.DocEngine) InsertFunc {+ return func(ctx context.Context, chunks []map[string]any, baseName string, datasetID string) ([]string, error) {+ return eng.InsertChunks(ctx, chunks, baseName, datasetID)+ }+}+func NewPipelineExecutor(taskCtx *TaskContext,canvasID string,@@ -138,7 +149,12 @@ func NewPipelineExecutor(taskCtx: taskCtx,canvasID: canvasID,docBulkSize: docBulkSize,- indexWriter: newChunkIndexWriter(insertChunksForIngestion(engine.Get()), fmt.Sprintf("ragflow_%s", taskCtx.Tenant.ID), taskCtx.Doc.KbID, docBulkSize),+ indexWriter: newChunkIndexWriter(+ insertChunksForIngestion(engine.Get()),+ fmt.Sprintf("ragflow_%s", taskCtx.Tenant.ID),+ taskCtx.Doc.KbID,+ docBulkSize,+ ).withFinalInsertFunc(insertFinalChunksForIngestion(engine.Get())),deleteChunksFunc: func(ctx context.Context, condition map[string]any, baseName, datasetID string) (int64, error) {return engine.Get().DeleteChunks(ctx, condition, baseName, datasetID)},@@ -153,6 +169,7 @@ type DeleteChunksFunc func(ctx context.Context, condition map[string]any, baseNafunc (s *PipelineExecutor) WithInsertFunc(f InsertFunc) *PipelineExecutor {s.indexWriter.insertFunc = f+ s.indexWriter.finalInsertFunc = freturn s}@@ -327,26 +344,13 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput mapreturn nil, fmt.Errorf("persist Wiki active MAP state: %w", err)}- // All chunks are now persisted. Notify the dataset-level post-processing consumer- // (§11) that this document is complete: its compiled products were written- // available_int=0 and the consumer later merges them into dataset-level products- // (available_int=1). The notification is sent only after a successful persist- // and is best-effort / non-fatal — a delivery failure is logged but does not- // fail the pipeline task.- //- // The variants passed to PublishCompleted are the union of the compile types- // in the previous and current document generations. They are derived from the- // authoritative `compilation_template_kind_kwd` the KnowledgeCompiler- // component stamps on each compiled product (the resolved template's kind →- // KindToVariant, O2a whitelist). Keeping the previous types lets the dataset- // consumer retract stale merged products when a template is removed.+ // All chunks are now persisted. Defer the dataset-level post-processing+ // notification until the enclosing ingestion task reaches COMPLETED. The+ // index writes intentionally do not wait for refresh, so publishing here lets+ // the consumer race the final task bookkeeping and read an incomplete view of+ // the document's products.eventVariants := mergeCompiledVariants(oldCompiledVariants, compiledVariants(chunks))eventTaskTypes := mergeTaskTypes(oldCompiledTaskTypes, compiledTaskTypes(chunks))- if len(eventVariants) > 0 {- if err := knowledge_compile.PublishCompleted(ctx, s.taskCtx.Tenant.ID, s.taskCtx.Doc.KbID, s.taskCtx.Doc.ID, eventVariants, eventTaskTypes); err != nil {- common.Logger.Warn(fmt.Sprintf("knowledge_compile: publish doc_completed for %s failed: %v", s.taskCtx.Doc.ID, err))- }- }// Compilation products are derived artifacts and must not inflate the// document's source chunk counter shown by the document list API.@@ -378,6 +382,8 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput mapDocName: docNameValue(s.taskCtx.Doc.Name),BuiltInMetadataConfig: builtInMetadata,AutoMetadataEnabled: autoMetaEnabled,+ CompiledVariants: eventVariants,+ CompiledTaskTypes: eventTaskTypes,}, nil}diff --git a/internal/service/document/document_crud.go b/internal/service/document/document_crud.goindex 73e396bd8e..f11463a6cb 100644--- a/internal/service/document/document_crud.go+++ b/internal/service/document/document_crud.go@@ -315,7 +315,7 @@ func (s *DocumentService) RemoveDocumentKeepFile(ctx context.Context, docID striif err != nil {return err}- _, taskTypes, typeErr := s.documentKnowledgeCompileTypes(ctx, kb.TenantID, kb.ID, docID)+ variants, taskTypes, typeErr := s.documentKnowledgeCompileTypes(ctx, kb.TenantID, kb.ID, docID)if typeErr != nil {common.Warn(fmt.Sprintf("RemoveDocumentKeepFile: failed to resolve knowledge compile types for %s: %v", docID, typeErr))}@@ -345,7 +345,7 @@ func (s *DocumentService) RemoveDocumentKeepFile(ctx context.Context, docID stri// deleted document's contribution in both paths.pubCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)defer cancel()- if err := knowledge_compile.PublishDeleted(pubCtx, kb.TenantID, kb.ID, docID, taskTypes); err != nil {+ if err := knowledge_compile.PublishDeleted(pubCtx, kb.TenantID, kb.ID, docID, variants, taskTypes); err != nil {common.Warn(fmt.Sprintf("RemoveDocumentKeepFile: publish doc_deleted for %s failed: %v", docID, err))}return nil@@ -415,7 +415,7 @@ func (s *DocumentService) deleteDocEngineData(ctx context.Context, docID, tenantreturn nil}indexName := fmt.Sprintf("ragflow_%s", tenantID)- _, taskTypes, typeErr := s.documentKnowledgeCompileTypes(ctx, tenantID, kbID, docID)+ variants, taskTypes, typeErr := s.documentKnowledgeCompileTypes(ctx, tenantID, kbID, docID)if typeErr != nil {common.Warn(fmt.Sprintf("deleteDocEngineData: failed to resolve knowledge compile types for %s: %v", docID, typeErr))}@@ -435,7 +435,7 @@ func (s *DocumentService) deleteDocEngineData(ctx context.Context, docID, tenant// never block the document delete, which already succeeded above.pubCtx, cancel := context.WithTimeout(ctx, 15*time.Second)defer cancel()- if err := knowledge_compile.PublishDeleted(pubCtx, tenantID, kbID, docID, taskTypes); err != nil {+ if err := knowledge_compile.PublishDeleted(pubCtx, tenantID, kbID, docID, variants, taskTypes); err != nil {common.Warn(fmt.Sprintf("deleteDocEngineData: publish doc_deleted for %s failed: %v", docID, err))}if s.metadataSvc != nil {diff --git a/internal/service/document/document_parse.go b/internal/service/document/document_parse.goindex 2a302978a6..6c8eb0676b 100644--- a/internal/service/document/document_parse.go+++ b/internal/service/document/document_parse.go@@ -186,7 +186,7 @@ func (s *DocumentService) clearDocumentParseResults(ctx context.Context, doc *enif !exists {return nil}- _, taskTypes, err := s.documentKnowledgeCompileTypes(ctx, tenantID, doc.KbID, doc.ID)+ variants, taskTypes, err := s.documentKnowledgeCompileTypes(ctx, tenantID, doc.KbID, doc.ID)if err != nil {return fmt.Errorf("resolve generated products for document %s: %w", doc.ID, err)}@@ -199,9 +199,13 @@ func (s *DocumentService) clearDocumentParseResults(ctx context.Context, doc *enif err = s.deleteSourceChunks(ctx, tenantID, doc.KbID, doc.ID); err != nil {return err}+ if len(variants) == 0 {+ common.Warn(fmt.Sprintf("skip document cleanup event for %s: existing knowledge products have no routing metadata", doc.ID))+ return nil+ }publishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)defer cancel()- if err := knowledge_compile.PublishDeleted(publishCtx, tenantID, doc.KbID, doc.ID, taskTypes); err != nil {+ if err := knowledge_compile.PublishDeleted(publishCtx, tenantID, doc.KbID, doc.ID, variants, taskTypes); err != nil {return fmt.Errorf("publish document cleanup for %s: %w", doc.ID, err)}return nil