Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: synchronize document compilation and navigation projections (#20030)

buua4369e99f996ff7e
diff --git a/internal/entity/models/jiekouai.go b/internal/entity/models/jiekouai.go
index 926f26776a..e1c7484e3b 100644
--- a/internal/entity/models/jiekouai.go
+++ b/internal/entity/models/jiekouai.go
@@ -25,7 +25,6 @@ import (
"net/http"
"ragflow/internal/common"
"strings"
- "time"
)
type JieKouAIModel struct {
@@ -33,16 +32,11 @@ type JieKouAIModel struct {
}
func NewJieKouAIModel(baseURL map[string]string, urlSuffix URLSuffix) *JieKouAIModel {
- // JieKouAI's methods issue requests without a per-call context deadline, so
- // keep an explicit 120s client-level timeout to bound them. Built on the
- // shared transport via NewDriverHTTPClient.
- client := common.GetSSRFHTTPClient()
- client.Timeout = 120 * time.Second
return &JieKouAIModel{
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: client,
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
@@ -376,7 +370,10 @@ func (j *JieKouAIModel) ListModels(ctx context.Context, apiConfig *APIConfig) ([
}
url := fmt.Sprintf("%s/%s", resolvedBaseURL, j.baseModel.URLSuffix.Models)
- req, err := http.NewRequest("GET", url, nil)
+ ctx, cancel := context.WithTimeout(ctx, nonStreamCallTimeout)
+ defer cancel()
+
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, fmt.Errorf("failed to create request: %w", err)
}
diff --git a/internal/entity/models/jina.go b/internal/entity/models/jina.go
index 35022dc363..530de837f7 100644
--- a/internal/entity/models/jina.go
+++ b/internal/entity/models/jina.go
@@ -27,7 +27,6 @@ import (
"net/url"
"ragflow/internal/common"
"strings"
- "time"
"unicode/utf8"
"go.uber.org/zap"
@@ -41,13 +40,11 @@ func NewJinaModel(baseURL map[string]string, urlSuffix URLSuffix) *JinaModel {
// Embed/Rerank/ListModels issue requests without a per-call context
// deadline, so keep an explicit 90s client-level timeout to bound them.
// Built on the shared transport via NewDriverHTTPClient.
- client := common.GetSSRFHTTPClient()
- client.Timeout = 90 * time.Second
return &JinaModel{
baseModel: BaseModel{
BaseURL: baseURL,
URLSuffix: urlSuffix,
- httpClient: client,
+ httpClient: common.GetSSRFHTTPClient(),
},
}
}
diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.go
index b079d70e8f..f8598b783b 100644
--- a/internal/ingestion/knowledge_compile/consumer.go
+++ b/internal/ingestion/knowledge_compile/consumer.go
@@ -775,14 +775,15 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
}
}
- // A0-4 dispatch: route tree/structure products to the dataset-navigation tree
- // (NavService.UpsertDoc), and keep wiki (and only wiki) products on the
- // existing dataset-level merge path below. This is the per-variant dispatch
- // the plan requires: tree and structure both produce a nav by-product (B2),
- // so both upsert their summary into the cross-document nav tree; wiki products
- // continue to the page-specific evidence merge.
+ // Route Tree and PageIndex products to the dataset-navigation tree. Wiki,
+ // Graph, Timeline, and Mindmap products have no navigation projection.
navIn := navInputFromProducts(kb, incoming)
c.reportProgress(ctx, tenant, kb, token, taskTypes, 0.40, "merging_navigation", fmt.Sprintf("Merging navigation products: %d", len(navIn)))
+ if docsWithoutNav := completedDocsWithoutNavInput(completed, navIn); len(docsWithoutNav) > 0 {
+ if err := c.removeNavLocked(ctx, tenant, kb, token, docsWithoutNav); err != nil {
+ return err
+ }
+ }
if len(navIn) > 0 {
ns := nav.GetNavService()
if ns == nil {
@@ -1171,6 +1172,20 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
return nil
}
+func completedDocsWithoutNavInput(completed []BacklogEntry, inputs []nav.UpsertDocInput) []string {
+ withNav := make(map[string]struct{}, len(inputs))
+ for _, input := range inputs {
+ withNav[input.DocID] = struct{}{}
+ }
+ withoutNav := make([]string, 0, len(completed))
+ for _, entry := range completed {
+ if _, ok := withNav[entry.DocID]; !ok {
+ withoutNav = append(withoutNav, entry.DocID)
+ }
+ }
+ return uniqueSortedStrings(withoutNav)
+}
+
func copyMeta(input map[string]any) map[string]any {
output := make(map[string]any, len(input)+2)
for key, value := range input {
@@ -1451,17 +1466,15 @@ func productsForVariants(products []kccommon.Product, variants []string) []kccom
return out
}
-// navInputFromProducts extracts the dataset-navigation upsert inputs for the
-// tree and structure products in a batch (B2: nav trigger is tree || structure).
+// navInputFromProducts extracts the dataset-navigation upsert inputs for tree
+// and PageIndex products in a batch.
// Summary extraction matches the component's by-product hooks:
// - tree: the root product (Meta.kind=="root") carries the document summary in
// Content and its vector in Vector.
-// - structure: the graph product (Meta.kind=="graph") carries the graph JSON in
-// Content; its entity descriptions are folded into a document-level summary
-// via pageIndexSummary.
+// - PageIndex: entity descriptions are folded into a document-level summary.
//
// Each document contributes at most one nav input, keyed by DocID, so a doc that
-// yields several tree/structure products upserts once. NavService embeds the
+// yields several tree/PageIndex products upserts once. NavService embeds the
// summary itself when Embedd is empty, so the consumer needs no embedder.
func navInputFromProducts(kb string, products []kccommon.Product) []nav.UpsertDocInput {
type acc struct {
@@ -1486,11 +1499,9 @@ func navInputFromProducts(kb string, products []kccommon.Product) []nav.UpsertDo
a.treeSummary = p.Content
a.treeEmbedd = p.Vector
case kccommon.VariantStructure:
- // Python's page_index path rebuilds the doc graph from the stored
- // entity rows and folds their descriptions into the nav summary
- // (runner.py rebuild_structure_graph_json + _page_index_graph_summary).
- // The graph blob product (Meta.kind=="graph") is gone from the
- // storage model, so fold directly from the entity rows.
+ if p.Kind != "page_index" {
+ continue
+ }
if kind, _ := p.Meta["kind"].(string); kind != "entity" {
continue
}
@@ -1789,7 +1800,7 @@ func appendUnique(dst, values []string) []string {
return dst
}
-// upsertNavLocked places the batch's tree/structure summaries into the dataset
+// upsertNavLocked places the batch's Tree/PageIndex summaries into the dataset
// navigation tree under the per-dataset write/rebuild lock, with the claim-token
// check inside the lock (B7: nav writes are claim-fenced like any other
// destructive write so a stale worker cannot repopulate a rebuilt tree).
diff --git a/internal/ingestion/knowledge_compile/event.go b/internal/ingestion/knowledge_compile/event.go
index affe050870..8102e9501e 100644
--- a/internal/ingestion/knowledge_compile/event.go
+++ b/internal/ingestion/knowledge_compile/event.go
@@ -118,13 +118,13 @@ func PublishCompleted(ctx context.Context, tenantID, datasetID, docID string, va
}
// PublishDeleted records a doc_deleted event the same way (Publish handles the
-// append + notify pairing). taskTypes identifies the document-level artifacts
-// that were removed before the event was published.
-func PublishDeleted(ctx context.Context, tenantID, datasetID, docID string, taskTypes []string) error {
+// append + notify pairing). variants and taskTypes identify the document-level
+// artifacts that were removed before the event was published.
+func PublishDeleted(ctx context.Context, tenantID, datasetID, docID string, variants, taskTypes []string) error {
if defaultPublisher == nil {
return nil
}
- return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeDeleted), nil, taskTypes)
+ return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeDeleted), variants, taskTypes)
}
// PublishEnabled records that a document with compiled products became
diff --git a/internal/ingestion/knowledge_compile/nav_dispatch_test.go b/internal/ingestion/knowledge_compile/nav_dispatch_test.go
index fa2aea0bed..36e146ba98 100644
--- a/internal/ingestion/knowledge_compile/nav_dispatch_test.go
+++ b/internal/ingestion/knowledge_compile/nav_dispatch_test.go
@@ -9,13 +9,10 @@ import (
"ragflow/internal/service/nav"
)
-// TestNavInputFromProducts_TreeAndStructure covers B2: nav is fed by BOTH tree
-// and structure products (not tree alone). Tree uses the root product's summary;
-// structure folds the per-document entity ROW descriptions into a summary
-// (mirroring Python runner.py rebuild_structure_graph_json +
-// _page_index_graph_summary — the graph blob product is gone from the storage
-// model).
-func TestNavInputFromProducts_TreeAndStructure(t *testing.T) {
+// TestNavInputFromProducts_TreeAndPageIndex verifies that navigation is fed by
+// tree roots and PageIndex entity rows. Other structure kinds do not create
+// navigation entries.
+func TestNavInputFromProducts_TreeAndPageIndex(t *testing.T) {
products := []kccommon.Product{
// tree: root product carries the doc summary + vector.
{DocID: "d1", TenantID: "t1", Variant: kccommon.VariantTree,
@@ -24,17 +21,21 @@ func TestNavInputFromProducts_TreeAndStructure(t *testing.T) {
// tree: a non-root summary node is NOT a nav input.
{DocID: "d1", TenantID: "t1", Variant: kccommon.VariantTree,
Content: "section body", Meta: map[string]any{"kind": "summary", "level": 0}},
- // structure: entity rows fold their descriptions into the doc summary.
+ // PageIndex entity rows fold their descriptions into the doc summary.
{DocID: "d2", TenantID: "t1", Variant: kccommon.VariantStructure,
Content: `{"name":"Engine","type":"component","description":"a propulsion device"}`,
- Meta: map[string]any{"kind": "entity", "compile_kwd": "page_index"}},
+ Kind: "page_index", Meta: map[string]any{"kind": "entity"}},
{DocID: "d2", TenantID: "t1", Variant: kccommon.VariantStructure,
Content: `{"name":"Turbine","type":"component","description":"converts flow into rotation"}`,
- Meta: map[string]any{"kind": "entity", "compile_kwd": "page_index"}},
- // structure: a relation row is NOT a nav summary line.
+ Kind: "page_index", Meta: map[string]any{"kind": "entity"}},
+ // A PageIndex relation is not a nav summary line.
{DocID: "d2", TenantID: "t1", Variant: kccommon.VariantStructure,
Content: `{"from":"Engine","to":"Turbine","type":"drives"}`,
- Meta: map[string]any{"kind": "relation", "compile_kwd": "page_index", "from": "Engine", "to": "Turbine"}},
+ Kind: "page_index", Meta: map[string]any{"kind": "relation", "from": "Engine", "to": "Turbine"}},
+ // Graph entities must not create navigation entries.
+ {DocID: "d3", TenantID: "t1", Variant: kccommon.VariantStructure,
+ Content: `{"name":"Graph entity","description":"must not become navigation"}`,
+ Kind: "knowledge_graph", Meta: map[string]any{"kind": "entity"}},
}
got := navInputFromProducts("kb1", products)
@@ -55,23 +56,26 @@ func TestNavInputFromProducts_TreeAndStructure(t *testing.T) {
if !reflect.DeepEqual(treeIn.Embedd, []float32{0.1, 0.2}) {
t.Errorf("tree embedd should be the root vector, got %v", treeIn.Embedd)
}
- structIn, ok := byDoc["d2"]
+ pageIndexIn, ok := byDoc["d2"]
if !ok {
- t.Fatal("missing structure nav input for d2")
+ t.Fatal("missing PageIndex nav input for d2")
}
- if !strings.Contains(structIn.Summary, "Engine: a propulsion device") {
- t.Errorf("structure summary = %q, want folded entity descriptions", structIn.Summary)
+ if _, found := byDoc["d3"]; found {
+ t.Fatal("Graph structure product must not produce navigation input")
}
- if !strings.Contains(structIn.Summary, "Turbine: converts flow into rotation") {
- t.Errorf("structure summary = %q, want both entity lines", structIn.Summary)
+ if !strings.Contains(pageIndexIn.Summary, "Engine: a propulsion device") {
+ t.Errorf("PageIndex summary = %q, want folded entity descriptions", pageIndexIn.Summary)
}
- if strings.Contains(structIn.Summary, "drives") {
- t.Errorf("structure summary must not include relation rows, got %q", structIn.Summary)
+ if !strings.Contains(pageIndexIn.Summary, "Turbine: converts flow into rotation") {
+ t.Errorf("PageIndex summary = %q, want both entity lines", pageIndexIn.Summary)
+ }
+ if strings.Contains(pageIndexIn.Summary, "drives") {
+ t.Errorf("PageIndex summary must not include relation rows, got %q", pageIndexIn.Summary)
}
// The entity rows' vectors are NOT the summary vector: leave Embedd empty so
// NavService embeds the folded summary text.
- if len(structIn.Embedd) != 0 {
- t.Errorf("structure embedd should be empty (NavService re-embeds), got %v", structIn.Embedd)
+ if len(pageIndexIn.Embedd) != 0 {
+ t.Errorf("PageIndex embedd should be empty (NavService re-embeds), got %v", pageIndexIn.Embedd)
}
}
@@ -95,9 +99,9 @@ func TestNavInputFromProducts_EmptySummaryDropped(t *testing.T) {
products := []kccommon.Product{
// tree root with empty content -> skip.
{DocID: "d1", TenantID: "t1", Variant: kccommon.VariantTree, Meta: map[string]any{"kind": "root"}},
- // structure entity with no description -> skip.
+ // PageIndex entity with no description -> skip.
{DocID: "d2", TenantID: "t1", Variant: kccommon.VariantStructure,
- Content: `{"name":"A","description":""}`, Meta: map[string]any{"kind": "entity"}},
+ Kind: "page_index", Content: `{"name":"A","description":""}`, Meta: map[string]any{"kind": "entity"}},
}
got := navInputFromProducts("kb1", products)
if len(got) != 0 {
@@ -109,7 +113,7 @@ func TestNavInputFromProducts_TreeSummaryWinsRegardlessOfOrder(t *testing.T) {
products := []kccommon.Product{
{DocID: "d1", TenantID: "t1", Variant: kccommon.VariantStructure,
Content: `{"name":"A","description":"structure summary"}`,
- Meta: map[string]any{"kind": "entity"}},
+ Kind: "page_index", Meta: map[string]any{"kind": "entity"}},
{DocID: "d1", TenantID: "t1", Variant: kccommon.VariantTree,
Content: "tree summary", Vector: []float32{0.1},
Meta: map[string]any{"kind": "root"}},
diff --git a/internal/ingestion/knowledge_compile/nav_lifecycle_test.go b/internal/ingestion/knowledge_compile/nav_lifecycle_test.go
index 50e7d55d2d..fe752cebad 100644
--- a/internal/ingestion/knowledge_compile/nav_lifecycle_test.go
+++ b/internal/ingestion/knowledge_compile/nav_lifecycle_test.go
@@ -140,3 +140,38 @@ func TestProcessBatchReaddsEnabledDocumentToNavigation(t *testing.T) {
t.Fatalf("upserted navigation input = %+v, want enabled document summary", recorder.upserted[0])
}
}
+
+func TestProcessBatchRemovesNavigationForGraphOnlyDocument(t *testing.T) {
+ previous := nav.GetNavService()
+ recorder := &recordingNavService{}
+ nav.SetNavService(recorder)
+ defer nav.SetNavService(previous)
+
+ consumer := NewConsumer(
+ NewFakeScheduler(),
+ WithReader(&fakeReader{products: []kccommon.Product{{
+ DocID: "doc-graph",
+ TenantID: "tenant-1",
+ Variant: kccommon.VariantStructure,
+ Kind: "knowledge_graph",
+ Content: `{"name":"Graph entity","description":"not navigation"}`,
+ Meta: map[string]any{"kind": "entity"},
+ }}}),
+ WithWriter(&fakeWriter{}),
+ WithDeduperFactory(func(string) (Deduper, error) { return NewNoopDeduper(), nil }),
+ withWikiContributionStore(&memoryWikiContributionStore{items: map[string]wikiDocumentContribution{}}),
+ )
+
+ err := consumer.processBatch(context.Background(), "tenant-1", "kb-1", "", []BacklogEntry{{
+ DocID: "doc-graph", EventType: string(EventTypeCompleted), Variants: []string{string(kccommon.VariantStructure)}, TaskTypes: []string{kccommon.TaskTypeGraph},
+ }}, nil)
+ if err != nil {
+ t.Fatalf("processBatch failed: %v", err)
+ }
+ if len(recorder.upserted) != 0 {
+ t.Fatalf("Graph-only document upserted navigation: %+v", recorder.upserted)
+ }
+ if len(recorder.removed) != 1 || recorder.removed[0] != "doc-graph" {
+ t.Fatalf("removed navigation docs = %v, want [doc-graph]", recorder.removed)
+ }
+}
diff --git a/internal/ingestion/knowledge_compile/scheduler.go b/internal/ingestion/knowledge_compile/scheduler.go
index a65149ffbe..c3d28403e7 100644
--- a/internal/ingestion/knowledge_compile/scheduler.go
+++ b/internal/ingestion/knowledge_compile/scheduler.go
@@ -63,9 +63,9 @@ const (
// into per-variant sub-batches and dispatch each to its own dataset-level
// compile path. TaskTypes preserves the original frontend-facing category for
// each template, so structure sub-kinds such as Graph, Timeline, and PageIndex
-// can be logged separately. Both fields are empty (nil) for deleted events and
-// for events published before they existed; the consumer falls back to the
-// legacy unified path when the fields are absent.
+// can be logged separately. Both fields are empty (nil) for events published
+// before they existed; the consumer falls back to the legacy unified path when
+// the fields are absent.
type BacklogEntry struct {
DocID string `json:"doc_id"`
EventType string `json:"event_type"`
@@ -96,7 +96,7 @@ type Publisher interface {
// each other's backlog, and it always pairs the append with a notify.
// variants carries the doc-level compile types (tree/structure/wiki/mindmap)
// for completed events; taskTypes carries the frontend-facing categories for
- // those templates. Both are nil for deleted events.
+ // those templates, including the products removed by deleted events.
Publish(ctx context.Context, tenantID, datasetID, docID, eventType string, variants, taskTypes []string) error
}
diff --git a/internal/ingestion/knowledge_compile/scheduler_test.go b/internal/ingestion/knowledge_compile/scheduler_test.go
index 2e3d3e4a03..c70f07e36b 100644
--- a/internal/ingestion/knowledge_compile/scheduler_test.go
+++ b/internal/ingestion/knowledge_compile/scheduler_test.go
@@ -56,6 +56,38 @@ func TestFakeSchedulerReclaimsInterruptedClaim(t *testing.T) {
}
}
+func TestPublishDeletedPreservesRemovedProductTypes(t *testing.T) {
+ scheduler := NewFakeScheduler()
+ previousPublisher, previousClaimer := defaultPublisher, defaultClaimer
+ defaultPublisher, defaultClaimer = scheduler, scheduler
+ t.Cleanup(func() {
+ defaultPublisher, defaultClaimer = previousPublisher, previousClaimer
+ })
+
+ variants := []string{"mindmap", "structure"}
+ taskTypes := []string{"Graph", "MindMap"}
+ if err := PublishDeleted(t.Context(), "t1", "kb1", "d1", variants, taskTypes); err != nil {
+ t.Fatalf("publish deleted: %v", err)
+ }
+ claim, ok, err := scheduler.Claim(t.Context(), "kb1")
+ if err != nil || !ok {
+ t.Fatalf("claim deleted event: ok=%v err=%v", ok, err)
+ }
+ if len(claim.Entries) != 1 {
+ t.Fatalf("entries = %d, want 1", len(claim.Entries))
+ }
+ entry := claim.Entries[0]
+ if entry.EventType != string(EventTypeDeleted) {
+ t.Fatalf("event type = %q, want %q", entry.EventType, EventTypeDeleted)
+ }
+ if !reflect.DeepEqual(entry.Variants, variants) {
+ t.Fatalf("variants = %v, want %v", entry.Variants, variants)
+ }
+ if !reflect.DeepEqual(entry.TaskTypes, taskTypes) {
+ t.Fatalf("task types = %v, want %v", entry.TaskTypes, taskTypes)
+ }
+}
+
func TestFakeSchedulerProgressIsClaimScoped(t *testing.T) {
f := NewFakeScheduler()
if err := f.Publish(t.Context(), "t1", "kb1", "d1", string(EventTypeCompleted), nil, nil); err != nil {
diff --git a/internal/ingestion/service/ingestion_service.go b/internal/ingestion/service/ingestion_service.go
index 89c4aaddef..ee59a79ed6 100644
--- a/internal/ingestion/service/ingestion_service.go
+++ b/internal/ingestion/service/ingestion_service.go
@@ -70,12 +70,14 @@ type Ingestor struct {
memoryReconcileBatchSize int
// Runtime state
- currentTasks map[string]struct{} // set of task IDs currently claimed by a worker
- tasksMu sync.RWMutex
- activeWorkers atomic.Int32 // number of worker goroutines currently in workerLoop
- activeLeases map[*Heartbeat]*activeLease
- leasesMu sync.Mutex
- stopLeases atomic.Bool
+ currentTasks map[string]struct{} // set of task IDs currently claimed by a worker
+ tasksMu sync.RWMutex
+ pendingCompileEvents map[string]pendingDocumentCompileEvent
+ compileEventsMu sync.Mutex
+ activeWorkers atomic.Int32 // number of worker goroutines currently in workerLoop
+ activeLeases map[*Heartbeat]*activeLease
+ leasesMu sync.Mutex
+ stopLeases atomic.Bool
// Shutdown channel - receive on this to trigger graceful shutdown
ShutdownCh chan struct{}
@@ -148,6 +150,12 @@ type activeLease struct {
abandoned atomic.Bool
}
+type pendingDocumentCompileEvent struct {
+ tenantID string
+ variants []string
+ taskTypes []string
+}
+
func NewIngestor(name string, maxConcurrency int32, supportedTypes []string) *Ingestor {
if maxConcurrency <= 0 {
maxConcurrency = int32(runtime.NumCPU())
@@ -166,6 +174,7 @@ func NewIngestor(name string, maxConcurrency int32, supportedTypes []string) *In
supportedDocTypes: supportedTypes,
version: "1.0.0",
currentTasks: make(map[string]struct{}),
+ pendingCompileEvents: make(map[string]pendingDocumentCompileEvent),
activeLeases: make(map[*Heartbeat]*activeLease),
workerQueue: make(chan *worker, maxConcurrency),
ShutdownCh: make(chan struct{}, 1),
@@ -828,6 +837,7 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) bool
}
if err := e.runDocumentTask(ctx, task); err != nil {
+ e.clearPendingCompileEvent(task.ID)
if errors.Is(err, context.Canceled) {
if e.ctx.Err() != nil || e.dispatchCtx.Err() != nil {
common.Info(fmt.Sprintf("Task %s pipeline interrupted by ingestor shutdown, leaving for redelivery", task.ID))
@@ -863,6 +873,7 @@ func (e *Ingestor) runTask(ctx context.Context, task *entity.IngestionTask) bool
return false
}
e.recordTerminalPipelineLog(ctx, task, string(entity.TaskStatusDone), "Task completed.")
+ e.publishPendingCompileEvent(ctx, task)
common.Info(fmt.Sprintf("Task %s completed", task.ID))
return true
@@ -1214,10 +1225,50 @@ func (e *Ingestor) defaultRunDocumentTask(ctx context.Context, ingestionTask *en
if err != nil {
return err
}
+ e.stagePendingCompileEvent(ingestionTask.ID, docTaskCtx.Tenant.ID, result)
e.docState.apply(ctx, result)
return nil
}
+func (e *Ingestor) stagePendingCompileEvent(taskID, tenantID string, result *taskpkg.PipelineResult) {
+ if taskID == "" {
+ return
+ }
+ e.compileEventsMu.Lock()
+ defer e.compileEventsMu.Unlock()
+ if result == nil || len(result.CompiledVariants) == 0 {
+ delete(e.pendingCompileEvents, taskID)
+ return
+ }
+ e.pendingCompileEvents[taskID] = pendingDocumentCompileEvent{
+ tenantID: tenantID,
+ variants: append([]string(nil), result.CompiledVariants...),
+ taskTypes: append([]string(nil), result.CompiledTaskTypes...),
+ }
+}
+
+func (e *Ingestor) clearPendingCompileEvent(taskID string) {
+ e.compileEventsMu.Lock()
+ delete(e.pendingCompileEvents, taskID)
+ e.compileEventsMu.Unlock()
+}
+
+func (e *Ingestor) publishPendingCompileEvent(ctx context.Context, task *entity.IngestionTask) {
+ if task == nil {
+ return
+ }
+ e.compileEventsMu.Lock()
+ event, ok := e.pendingCompileEvents[task.ID]
+ delete(e.pendingCompileEvents, task.ID)
+ e.compileEventsMu.Unlock()
+ if !ok || len(event.variants) == 0 {
+ return
+ }
+ if err := knowledge_compile.PublishCompleted(ctx, event.tenantID, task.DatasetID, task.DocumentID, event.variants, event.taskTypes); err != nil {
+ common.Logger.Warn(fmt.Sprintf("knowledge_compile: publish doc_completed for %s failed: %v", task.DocumentID, err))
+ }
+}
+
func (e *Ingestor) recordTerminalPipelineLog(ctx context.Context, ingestionTask *entity.IngestionTask, status, message string) {
if ingestionTask == nil || status == "" {
return
diff --git a/internal/ingestion/task/chunk_index_writer.go b/internal/ingestion/task/chunk_index_writer.go
index bfdffab918..ad86a0c0f5 100644
--- a/internal/ingestion/task/chunk_index_writer.go
+++ b/internal/ingestion/task/chunk_index_writer.go
@@ -37,10 +37,11 @@ type InsertFunc func(ctx context.Context, chunks []map[string]any, baseName, dat
// chunkIndexWriter batches chunks and writes them to the search engine in
// bulkSize-sized batches. Progress is reported every 128 batches.
type chunkIndexWriter struct {
- insertFunc InsertFunc
- baseName string
- datasetID string
- bulkSize int
+ insertFunc InsertFunc
+ finalInsertFunc InsertFunc
+ baseName string
+ datasetID string
+ bulkSize int
}
// newChunkIndexWriter creates a chunkIndexWriter. When bulkSize is <= 0 the
@@ -52,13 +53,19 @@ func newChunkIndexWriter(
bulkSize int,
) *chunkIndexWriter {
return &chunkIndexWriter{
- insertFunc: insertFunc,
- baseName: baseName,
- datasetID: datasetID,
- bulkSize: bulkSize,
+ insertFunc: insertFunc,
+ finalInsertFunc: insertFunc,
+ baseName: baseName,
+ datasetID: datasetID,
+ bulkSize: bulkSize,
}
}
+func (w *chunkIndexWriter) withFinalInsertFunc(insertFunc InsertFunc) *chunkIndexWriter {
+ w.finalInsertFunc = insertFunc
+ return w
+}
+
// Write inserts chunks in batches. An empty or nil slice is forwarded to the
// backend as-is.
func (w *chunkIndexWriter) Write(ctx context.Context, chunks []map[string]any) error {
@@ -78,9 +85,13 @@ func (w *chunkIndexWriter) Write(ctx context.Context, chunks []map[string]any) e
if err := ctx.Err(); err != nil {
return err
}
+ insert := w.insertFunc
+ if end == len(chunks) && w.finalInsertFunc != nil {
+ insert = w.finalInsertFunc
+ }
var err error
for attempt := 1; attempt <= chunkInsertAttempts; attempt++ {
- _, err = w.insertFunc(ctx, chunks[b:end], w.baseName, w.datasetID)
+ _, err = insert(ctx, chunks[b:end], w.baseName, w.datasetID)
if err == nil {
break
}
diff --git a/internal/ingestion/task/chunk_index_writer_test.go b/internal/ingestion/task/chunk_index_writer_test.go
index d3c05f485b..ad459429a8 100644
--- a/internal/ingestion/task/chunk_index_writer_test.go
+++ b/internal/ingestion/task/chunk_index_writer_test.go
@@ -19,6 +19,8 @@ package task
import (
"context"
"errors"
+ "fmt"
+ "reflect"
"testing"
"time"
)
@@ -97,6 +99,29 @@ func TestChunkIndexWriter_MultipleBatches(t *testing.T) {
}
}
+func TestChunkIndexWriter_UsesFinalInserterForLastBatch(t *testing.T) {
+ var calls []string
+ w := newChunkIndexWriter(
+ func(_ context.Context, chunks []map[string]any, _, _ string) ([]string, error) {
+ calls = append(calls, fmt.Sprintf("regular:%d", len(chunks)))
+ return nil, nil
+ },
+ "base",
+ "kb-1",
+ 3,
+ ).withFinalInsertFunc(func(_ context.Context, chunks []map[string]any, _, _ string) ([]string, error) {
+ calls = append(calls, fmt.Sprintf("final:%d", len(chunks)))
+ return nil, nil
+ })
+
+ if err := w.Write(t.Context(), make([]map[string]any, 7)); err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if want := []string{"regular:3", "regular:3", "final:1"}; !reflect.DeepEqual(calls, want) {
+ t.Fatalf("insert calls = %v, want %v", calls, want)
+ }
+}
+
func TestChunkIndexWriter_BulkSizeZero(t *testing.T) {
var lastBatchSize int
w := newChunkIndexWriter(
diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.go
index bfc224d4ce..e298cf59ce 100644
--- a/internal/ingestion/task/pipeline_executor.go
+++ b/internal/ingestion/task/pipeline_executor.go
@@ -58,6 +58,11 @@ type PipelineResult struct {
DocName string
BuiltInMetadataConfig []any
AutoMetadataEnabled bool
+ // CompiledVariants and CompiledTaskTypes identify the document-level
+ // products that the dataset consumer must merge after the ingestion task
+ // reaches its terminal COMPLETED state.
+ CompiledVariants []string
+ CompiledTaskTypes []string
// MessageID is the polling key for the debug-run log. The front-end reads
// it from the run response and polls GET /agents/:id/logs/:message_id to
// render progress; it is empty for non-debug (persist) runs.
@@ -104,16 +109,13 @@ type noRefreshChunkInserter interface {
InsertChunksNoRefresh(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string) ([]string, error)
}
-// insertChunksForIngestion writes chunks through the engine's no-refresh path
-// when it offers one, falling back to the plain inserter otherwise.
+// insertChunksForIngestion writes non-final chunks through the engine's
+// no-refresh path when it offers one, falling back to the plain inserter
+// otherwise.
//
-// Waiting for a refresh costs ~0.5s per write (median; p90 1.0s, max 5.0s) with
-// the KB index's refresh_interval=1000ms, and it buys nothing during ingestion:
-// the index publishes the chunks on its own refresh cycle a moment later, so the
-// only difference is up to a second before they are searchable. It is also what
-// Python's ingestion does - it inserts with refresh=False
-// (rag/svr/task_executor_refactor/chunk_service.py:386 and :423) while its API
-// default stays "wait_for" - so this keeps the two ingestion paths in step.
+// Refreshing every batch adds unnecessary latency. The writer uses the regular
+// inserter for its final batch, so all preceding batches become searchable
+// before the document task is acknowledged and its completion event is sent.
func insertChunksForIngestion(eng engine.DocEngine) InsertFunc {
return func(ctx context.Context, chunks []map[string]any, baseName string, datasetID string) ([]string, error) {
if bulk, ok := eng.(noRefreshChunkInserter); ok {
@@ -123,6 +125,15 @@ func insertChunksForIngestion(eng engine.DocEngine) InsertFunc {
}
}
+// insertFinalChunksForIngestion waits for the index refresh before returning.
+// The final write makes every preceding no-refresh batch searchable before the
+// document task is acknowledged and its completion event is published.
+func insertFinalChunksForIngestion(eng engine.DocEngine) InsertFunc {
+ return func(ctx context.Context, chunks []map[string]any, baseName string, datasetID string) ([]string, error) {
+ return eng.InsertChunks(ctx, chunks, baseName, datasetID)
+ }
+}
+
func NewPipelineExecutor(
taskCtx *TaskContext,
canvasID string,
@@ -138,7 +149,12 @@ func NewPipelineExecutor(
taskCtx: taskCtx,
canvasID: canvasID,
docBulkSize: docBulkSize,
- indexWriter: newChunkIndexWriter(insertChunksForIngestion(engine.Get()), fmt.Sprintf("ragflow_%s", taskCtx.Tenant.ID), taskCtx.Doc.KbID, docBulkSize),
+ indexWriter: newChunkIndexWriter(
+ insertChunksForIngestion(engine.Get()),
+ fmt.Sprintf("ragflow_%s", taskCtx.Tenant.ID),
+ taskCtx.Doc.KbID,
+ docBulkSize,
+ ).withFinalInsertFunc(insertFinalChunksForIngestion(engine.Get())),
deleteChunksFunc: func(ctx context.Context, condition map[string]any, baseName, datasetID string) (int64, error) {
return engine.Get().DeleteChunks(ctx, condition, baseName, datasetID)
},
@@ -153,6 +169,7 @@ type DeleteChunksFunc func(ctx context.Context, condition map[string]any, baseNa
func (s *PipelineExecutor) WithInsertFunc(f InsertFunc) *PipelineExecutor {
s.indexWriter.insertFunc = f
+ s.indexWriter.finalInsertFunc = f
return s
}
@@ -327,26 +344,13 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map
return nil, fmt.Errorf("persist Wiki active MAP state: %w", err)
}
- // All chunks are now persisted. Notify the dataset-level post-processing consumer
- // (§11) that this document is complete: its compiled products were written
- // available_int=0 and the consumer later merges them into dataset-level products
- // (available_int=1). The notification is sent only after a successful persist
- // and is best-effort / non-fatal — a delivery failure is logged but does not
- // fail the pipeline task.
- //
- // The variants passed to PublishCompleted are the union of the compile types
- // in the previous and current document generations. They are derived from the
- // authoritative `compilation_template_kind_kwd` the KnowledgeCompiler
- // component stamps on each compiled product (the resolved template's kind →
- // KindToVariant, O2a whitelist). Keeping the previous types lets the dataset
- // consumer retract stale merged products when a template is removed.
+ // All chunks are now persisted. Defer the dataset-level post-processing
+ // notification until the enclosing ingestion task reaches COMPLETED. The
+ // index writes intentionally do not wait for refresh, so publishing here lets
+ // the consumer race the final task bookkeeping and read an incomplete view of
+ // the document's products.
eventVariants := mergeCompiledVariants(oldCompiledVariants, compiledVariants(chunks))
eventTaskTypes := mergeTaskTypes(oldCompiledTaskTypes, compiledTaskTypes(chunks))
- if len(eventVariants) > 0 {
- if err := knowledge_compile.PublishCompleted(ctx, s.taskCtx.Tenant.ID, s.taskCtx.Doc.KbID, s.taskCtx.Doc.ID, eventVariants, eventTaskTypes); err != nil {
- common.Logger.Warn(fmt.Sprintf("knowledge_compile: publish doc_completed for %s failed: %v", s.taskCtx.Doc.ID, err))
- }
- }
// Compilation products are derived artifacts and must not inflate the
// document's source chunk counter shown by the document list API.
@@ -378,6 +382,8 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map
DocName: docNameValue(s.taskCtx.Doc.Name),
BuiltInMetadataConfig: builtInMetadata,
AutoMetadataEnabled: autoMetaEnabled,
+ CompiledVariants: eventVariants,
+ CompiledTaskTypes: eventTaskTypes,
}, nil
}
diff --git a/internal/service/document/document_crud.go b/internal/service/document/document_crud.go
index 73e396bd8e..f11463a6cb 100644
--- a/internal/service/document/document_crud.go
+++ b/internal/service/document/document_crud.go
@@ -315,7 +315,7 @@ func (s *DocumentService) RemoveDocumentKeepFile(ctx context.Context, docID stri
if err != nil {
return err
}
- _, taskTypes, typeErr := s.documentKnowledgeCompileTypes(ctx, kb.TenantID, kb.ID, docID)
+ variants, taskTypes, typeErr := s.documentKnowledgeCompileTypes(ctx, kb.TenantID, kb.ID, docID)
if typeErr != nil {
common.Warn(fmt.Sprintf("RemoveDocumentKeepFile: failed to resolve knowledge compile types for %s: %v", docID, typeErr))
}
@@ -345,7 +345,7 @@ func (s *DocumentService) RemoveDocumentKeepFile(ctx context.Context, docID stri
// deleted document's contribution in both paths.
pubCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)
defer cancel()
- if err := knowledge_compile.PublishDeleted(pubCtx, kb.TenantID, kb.ID, docID, taskTypes); err != nil {
+ if err := knowledge_compile.PublishDeleted(pubCtx, kb.TenantID, kb.ID, docID, variants, taskTypes); err != nil {
common.Warn(fmt.Sprintf("RemoveDocumentKeepFile: publish doc_deleted for %s failed: %v", docID, err))
}
return nil
@@ -415,7 +415,7 @@ func (s *DocumentService) deleteDocEngineData(ctx context.Context, docID, tenant
return nil
}
indexName := fmt.Sprintf("ragflow_%s", tenantID)
- _, taskTypes, typeErr := s.documentKnowledgeCompileTypes(ctx, tenantID, kbID, docID)
+ variants, taskTypes, typeErr := s.documentKnowledgeCompileTypes(ctx, tenantID, kbID, docID)
if typeErr != nil {
common.Warn(fmt.Sprintf("deleteDocEngineData: failed to resolve knowledge compile types for %s: %v", docID, typeErr))
}
@@ -435,7 +435,7 @@ func (s *DocumentService) deleteDocEngineData(ctx context.Context, docID, tenant
// never block the document delete, which already succeeded above.
pubCtx, cancel := context.WithTimeout(ctx, 15*time.Second)
defer cancel()
- if err := knowledge_compile.PublishDeleted(pubCtx, tenantID, kbID, docID, taskTypes); err != nil {
+ if err := knowledge_compile.PublishDeleted(pubCtx, tenantID, kbID, docID, variants, taskTypes); err != nil {
common.Warn(fmt.Sprintf("deleteDocEngineData: publish doc_deleted for %s failed: %v", docID, err))
}
if s.metadataSvc != nil {
diff --git a/internal/service/document/document_parse.go b/internal/service/document/document_parse.go
index 2a302978a6..6c8eb0676b 100644
--- a/internal/service/document/document_parse.go
+++ b/internal/service/document/document_parse.go
@@ -186,7 +186,7 @@ func (s *DocumentService) clearDocumentParseResults(ctx context.Context, doc *en
if !exists {
return nil
}
- _, taskTypes, err := s.documentKnowledgeCompileTypes(ctx, tenantID, doc.KbID, doc.ID)
+ variants, taskTypes, err := s.documentKnowledgeCompileTypes(ctx, tenantID, doc.KbID, doc.ID)
if err != nil {
return fmt.Errorf("resolve generated products for document %s: %w", doc.ID, err)
}
@@ -199,9 +199,13 @@ func (s *DocumentService) clearDocumentParseResults(ctx context.Context, doc *en
if err = s.deleteSourceChunks(ctx, tenantID, doc.KbID, doc.ID); err != nil {
return err
}
+ if len(variants) == 0 {
+ common.Warn(fmt.Sprintf("skip document cleanup event for %s: existing knowledge products have no routing metadata", doc.ID))
+ return nil
+ }
publishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)
defer cancel()
- if err := knowledge_compile.PublishDeleted(publishCtx, tenantID, doc.KbID, doc.ID, taskTypes); err != nil {
+ if err := knowledge_compile.PublishDeleted(publishCtx, tenantID, doc.KbID, doc.ID, variants, taskTypes); err != nil {
return fmt.Errorf("publish document cleanup for %s: %w", doc.ID, err)
}
return nil