Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: split knowledge compilation logs by task type (#19548)

buua436ec5a0136de42
diff --git a/internal/ingestion/component/knowledge_compiler/common/types.go b/internal/ingestion/component/knowledge_compiler/common/types.go
index 3fcda008a0..a7070a7471 100644
--- a/internal/ingestion/component/knowledge_compiler/common/types.go
+++ b/internal/ingestion/component/knowledge_compiler/common/types.go
@@ -24,6 +24,18 @@ const (
VariantMindmap Variant = "mindmap"
)
+// Task type labels are the values exposed by the pipeline operation log API.
+// They intentionally preserve the frontend categories instead of collapsing
+// every structure-based template into the internal structure variant.
+const (
+ TaskTypeWiki = "Wiki"
+ TaskTypeTree = "Tree"
+ TaskTypeGraph = "Graph"
+ TaskTypeMindmap = "Mindmap"
+ TaskTypeTimeline = "Timeline"
+ TaskTypePageIndex = "PageIndex"
+)
+
// Sentinel errors.
var (
ErrUnknownVariant = errors.New("knowledge_compiler: unknown variant")
@@ -230,6 +242,46 @@ func KindToVariant(kind string) (Variant, error) {
}
}
+// KindToTaskType maps the original compilation template kind to the category
+// shown in pipeline operation logs. Unlike KindToVariant, this mapping keeps
+// structure sub-kinds distinct so graph, timeline, and page-index logs are not
+// mislabeled as the same task.
+func KindToTaskType(kind string) (string, error) {
+ switch strings.ToLower(strings.TrimSpace(kind)) {
+ case "wiki":
+ return TaskTypeWiki, nil
+ case "tree":
+ return TaskTypeTree, nil
+ case "knowledge_graph", "knowledgegraph", "graph":
+ return TaskTypeGraph, nil
+ case "mind_map", "mindmap":
+ return TaskTypeMindmap, nil
+ case "timeline":
+ return TaskTypeTimeline, nil
+ case "page_index":
+ return TaskTypePageIndex, nil
+ default:
+ return "", fmt.Errorf("%w: %q", ErrUnknownVariant, kind)
+ }
+}
+
+// VariantToTaskType provides the best category available when reading legacy
+// rows that do not retain the original template kind.
+func VariantToTaskType(variant Variant) string {
+ switch variant {
+ case VariantWiki:
+ return TaskTypeWiki
+ case VariantTree:
+ return TaskTypeTree
+ case VariantMindmap:
+ return TaskTypeMindmap
+ case VariantStructure:
+ return TaskTypeGraph
+ default:
+ return ""
+ }
+}
+
// FirstNonEmpty returns the first string that is non-empty after trimming.
func FirstNonEmpty(vals ...string) string {
for _, v := range vals {
diff --git a/internal/ingestion/component/knowledge_compiler/common/types_test.go b/internal/ingestion/component/knowledge_compiler/common/types_test.go
index 38e3da1742..ef3ffa1f89 100644
--- a/internal/ingestion/component/knowledge_compiler/common/types_test.go
+++ b/internal/ingestion/component/knowledge_compiler/common/types_test.go
@@ -51,3 +51,24 @@ func TestKindToVariant_CanonicalForms(t *testing.T) {
t.Errorf("structure/knowledge_graph/graph should map to the same variant, got %q %q %q", a, b, c)
}
}
+
+func TestKindToTaskTypePreservesFrontendCategories(t *testing.T) {
+ tests := map[string]string{
+ "wiki": TaskTypeWiki,
+ "tree": TaskTypeTree,
+ "knowledge_graph": TaskTypeGraph,
+ "mind_map": TaskTypeMindmap,
+ "timeline": TaskTypeTimeline,
+ "page_index": TaskTypePageIndex,
+ }
+ for kind, want := range tests {
+ got, err := KindToTaskType(kind)
+ if err != nil {
+ t.Errorf("KindToTaskType(%q) unexpected error: %v", kind, err)
+ continue
+ }
+ if got != want {
+ t.Errorf("KindToTaskType(%q) = %q, want %q", kind, got, want)
+ }
+ }
+}
diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.go
index c2c7622a42..68312b11f3 100644
--- a/internal/ingestion/knowledge_compile/consumer.go
+++ b/internal/ingestion/knowledge_compile/consumer.go
@@ -73,7 +73,7 @@ type Consumer struct {
// satisfied by *mysqlScheduler (and FakeScheduler in tests). Using a narrow
// interface keeps the Claimer surface unchanged.
type rewriteScheduler interface {
- Publish(ctx context.Context, tenantID, datasetID, docID, eventType string, variants []string) error
+ Publish(ctx context.Context, tenantID, datasetID, docID, eventType string, variants, taskTypes []string) error
CancelInflight(ctx context.Context, datasetID, token string) error
}
@@ -170,6 +170,7 @@ func (c *Consumer) processClaim(ctx context.Context, cr ClaimResult) {
if paused {
return
}
+ logTaskTypes := taskTypesForEntries(cr.Entries)
if err := startDatasetCompileLog(ctx, cr.TenantID, datasetID, cr.Token, cr.Entries); err != nil {
common.Warn("knowledge_compile: failed to create dataset ingestion log",
zap.String("dataset_id", datasetID), zap.Error(err))
@@ -205,14 +206,14 @@ func (c *Consumer) processClaim(ctx context.Context, cr ClaimResult) {
var batchErr error
go func() {
defer close(done)
- batchErr = c.processBatch(ctx, cr.TenantID, datasetID, cr.Token, cr.Entries)
+ batchErr = c.processBatch(ctx, cr.TenantID, datasetID, cr.Token, cr.Entries, logTaskTypes)
}()
select {
case <-ctx.Done():
close(stopHb)
<-done
- if err := finishDatasetCompileLog(context.WithoutCancel(ctx), cr.Token, common.STOPPED, "Wiki dataset compilation stopped during shutdown", -1); err != nil {
+ if err := finishDatasetCompileLog(context.WithoutCancel(ctx), cr.Token, logTaskTypes, common.STOPPED, "Knowledge compilation stopped during shutdown", -1); err != nil {
common.Warn("knowledge_compile: failed to finalize stopped dataset ingestion log",
zap.String("dataset_id", datasetID), zap.Error(err))
}
@@ -220,7 +221,7 @@ func (c *Consumer) processClaim(ctx context.Context, cr ClaimResult) {
case <-hbFailed:
close(stopHb)
<-done
- if err := finishDatasetCompileLog(context.WithoutCancel(ctx), cr.Token, common.STOPPED, "Wiki dataset compilation lease was lost and will be retried", -1); err != nil {
+ if err := finishDatasetCompileLog(context.WithoutCancel(ctx), cr.Token, logTaskTypes, common.STOPPED, "Knowledge compilation lease was lost and will be retried", -1); err != nil {
common.Warn("knowledge_compile: failed to finalize stopped dataset ingestion log",
zap.String("dataset_id", datasetID), zap.Error(err))
}
@@ -244,7 +245,7 @@ func (c *Consumer) processClaim(ctx context.Context, cr ClaimResult) {
common.Warn("knowledge_compile: failed to record error_msg",
zap.String("dataset_id", datasetID), zap.Error(err))
}
- if err := finishDatasetCompileLog(ctx, cr.Token, common.FAILED, batchErr.Error(), -1); err != nil {
+ if err := finishDatasetCompileLog(ctx, cr.Token, logTaskTypes, common.FAILED, batchErr.Error(), -1); err != nil {
common.Warn("knowledge_compile: failed to finalize dataset ingestion log",
zap.String("dataset_id", datasetID), zap.Error(err))
}
@@ -253,26 +254,26 @@ func (c *Consumer) processClaim(ctx context.Context, cr ClaimResult) {
if _, err := c.scheduler.Ack(ctx, datasetID, cr.Token, cr.Entries); err != nil {
common.Warn("knowledge_compile: ack failed",
zap.String("dataset_id", datasetID), zap.Error(err))
- if logErr := finishDatasetCompileLog(ctx, cr.Token, common.FAILED, "Failed to acknowledge completed Wiki compilation: "+err.Error(), -1); logErr != nil {
+ if logErr := finishDatasetCompileLog(ctx, cr.Token, logTaskTypes, common.FAILED, "Failed to acknowledge completed knowledge compilation: "+err.Error(), -1); logErr != nil {
common.Warn("knowledge_compile: failed to finalize dataset ingestion log",
zap.String("dataset_id", datasetID), zap.Error(logErr))
}
return
}
- if err := finishDatasetCompileLog(ctx, cr.Token, common.COMPLETED, "Wiki dataset compilation completed", 1); err != nil {
+ if err := finishDatasetCompileLog(ctx, cr.Token, logTaskTypes, common.COMPLETED, "Knowledge compilation completed", 1); err != nil {
common.Warn("knowledge_compile: failed to finalize dataset ingestion log",
zap.String("dataset_id", datasetID), zap.Error(err))
}
}
-func (c *Consumer) reportProgress(ctx context.Context, tenant, datasetID, token string, progress float64, phase, message string) {
+func (c *Consumer) reportProgress(ctx context.Context, tenant, datasetID, token string, taskTypes []string, progress float64, phase, message string) {
if err := c.scheduler.UpdateProgress(ctx, datasetID, token, progress, phase, message); err != nil {
common.Warn("knowledge_compile: failed to update progress",
zap.String("dataset_id", datasetID),
zap.String("phase", phase),
zap.Error(err))
}
- if err := updateDatasetCompileLog(ctx, token, progress, message); err != nil {
+ if err := updateDatasetCompileLog(ctx, token, taskTypes, progress, message); err != nil {
common.Warn("knowledge_compile: failed to update dataset ingestion log",
zap.String("dataset_id", datasetID), zap.String("phase", phase), zap.Error(err))
}
@@ -382,7 +383,7 @@ func (c *Consumer) RebuildDataset(ctx context.Context, tenant, kb, mode string)
return fmt.Errorf("knowledge_compile: rebuild list docs: %w", err)
}
for _, docID := range docs {
- variants, rerr := c.recoverDocVariants(ctx, tenant, kb, docID)
+ variants, taskTypes, rerr := c.recoverDocTypes(ctx, tenant, kb, docID)
if rerr != nil {
// O2a hard failure: abort the rebuild so an unknown template kind
// cannot leave the doc's dataset-level products unrebuilt.
@@ -394,7 +395,7 @@ func (c *Consumer) RebuildDataset(ctx context.Context, tenant, kb, mode string)
// event that falls back to the legacy all-products path.
continue
}
- if perr := rs.Publish(ctx, tenant, kb, docID, string(EventTypeCompleted), variants); perr != nil {
+ if perr := rs.Publish(ctx, tenant, kb, docID, string(EventTypeCompleted), variants, taskTypes); perr != nil {
return fmt.Errorf("knowledge_compile: rebuild republish %s: %w", docID, perr)
}
}
@@ -417,9 +418,10 @@ func defaultDocLister(ctx context.Context, tenant, kb string) ([]string, error)
return dao.NewDocumentDAO().ListIDsByKBIDWithOptions(ctx, kcDB, dao.DocumentListOptions{KbID: kb})
}
-// recoverDocVariants reconstructs the compile-type set a document produced, by
-// reading its persisted doc-level products and mapping each product's
-// authoritative `compilation_template_kind_kwd` through common.KindToVariant.
+// recoverDocTypes reconstructs the compile-type and frontend task-type sets a
+// document produced by reading its persisted doc-level products and mapping
+// each product's authoritative `compilation_template_kind_kwd` through the
+// shared kind mappings.
// It is used by RebuildDataset (B1) so the republished completed events carry
// the variants needed to route the dataset-level re-compile (tree nav, structure
// merge, wiki merge) — a nil/empty Variants would fall back to the legacy
@@ -431,27 +433,34 @@ func defaultDocLister(ctx context.Context, tenant, kb string) ([]string, error)
// variant and leave the doc's dataset-level products unrebuilt. A product
// without an authoritative kind falls back to its reverse-mapped variant (which
// itself must be whitelist-valid, enforced by KwdToVariant in the Reader). The
-// returned slice is sorted and de-duplicated.
-func (c *Consumer) recoverDocVariants(ctx context.Context, tenant, kb, docID string) ([]string, error) {
+// Both returned slices are sorted and de-duplicated.
+func (c *Consumer) recoverDocTypes(ctx context.Context, tenant, kb, docID string) ([]string, []string, error) {
products, err := c.reader.LoadDocProducts(ctx, tenant, kb, docID)
if err != nil {
- return nil, fmt.Errorf("knowledge_compile: recover variants load doc %s: %w", docID, err)
+ return nil, nil, fmt.Errorf("knowledge_compile: recover variants load doc %s: %w", docID, err)
}
- seen := map[string]struct{}{}
- var out []string
+ seenVariants := map[string]struct{}{}
+ seenTaskTypes := map[string]struct{}{}
+ var variants []string
for _, p := range products {
// Authoritative: product.Kind (compilation_template_kind_kwd) when present.
if p.Kind != "" {
v, err := kccommon.KindToVariant(p.Kind)
if err != nil {
// O2a hard failure: do not continue with an incomplete variant set.
- return nil, fmt.Errorf("knowledge_compile: recover variants doc %s kind %q: %w", docID, p.Kind, err)
+ return nil, nil, fmt.Errorf("knowledge_compile: recover variants doc %s kind %q: %w", docID, p.Kind, err)
}
- if _, dup := seen[string(v)]; dup {
- continue
+ if _, dup := seenVariants[string(v)]; !dup {
+ seenVariants[string(v)] = struct{}{}
+ variants = append(variants, string(v))
+ }
+ taskType, taskTypeErr := kccommon.KindToTaskType(p.Kind)
+ if taskTypeErr != nil {
+ taskType = kccommon.VariantToTaskType(v)
+ }
+ if taskType != "" {
+ seenTaskTypes[taskType] = struct{}{}
}
- seen[string(v)] = struct{}{}
- out = append(out, string(v))
continue
}
// Fallback: the product's reverse-mapped variant (already whitelist-valid).
@@ -461,14 +470,17 @@ func (c *Consumer) recoverDocVariants(ctx context.Context, tenant, kb, docID str
// (which the consumer cannot route and defeats "nil = legacy").
continue
}
- if _, dup := seen[string(p.Variant)]; dup {
- continue
+ if _, dup := seenVariants[string(p.Variant)]; !dup {
+ seenVariants[string(p.Variant)] = struct{}{}
+ variants = append(variants, string(p.Variant))
+ }
+ if taskType := kccommon.VariantToTaskType(p.Variant); taskType != "" {
+ seenTaskTypes[taskType] = struct{}{}
}
- seen[string(p.Variant)] = struct{}{}
- out = append(out, string(p.Variant))
}
- sort.Strings(out)
- return out, nil
+ sort.Strings(variants)
+ taskTypes := sortedTaskTypes(seenTaskTypes)
+ return variants, taskTypes, nil
}
// filterWikiPageCandidates returns ONLY the wiki page products the dataset-level
@@ -498,8 +510,11 @@ func filterWikiPageCandidates(candidates []kccommon.Product) []kccommon.Product
// writes the dataset-level merged products for the claimed closed batch. It
// returns an error if any reader/dedup/writer step fails so the caller can
// leave the batch for reclamation instead of acking dropped work.
-func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, entries []BacklogEntry) error {
- c.reportProgress(ctx, tenant, kb, token, 0.05, "classifying", fmt.Sprintf("Classifying %d scheduling entries", len(entries)))
+func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, entries []BacklogEntry, taskTypes []string) error {
+ if len(taskTypes) == 0 {
+ taskTypes = taskTypesForEntries(entries)
+ }
+ c.reportProgress(ctx, tenant, kb, token, taskTypes, 0.05, "classifying", fmt.Sprintf("Classifying %d scheduling entries", len(entries)))
common.Info("knowledge_compile: processing claimed batch",
zap.String("dataset_id", kb),
zap.String("tenant_id", tenant),
@@ -589,19 +604,24 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
zap.Strings("entries", entryDetails))
if len(deleted) == 0 && len(disabled) == 0 && len(completed) == 0 {
- c.reportProgress(ctx, tenant, kb, token, 1, "completed", "No document products require merging")
+ c.reportProgress(ctx, tenant, kb, token, taskTypes, 1, "completed", "No document products require merging")
return nil
}
deletedSet := make(map[string]bool, len(deleted))
for _, d := range deleted {
deletedSet[d] = true
}
- wikiDiff, err := c.prepareWikiContributionDiff(ctx, tenant, kb, completed, deleted)
+ retractedDocIDs := append([]string(nil), deleted...)
+ for _, entry := range disabled {
+ retractedDocIDs = append(retractedDocIDs, entry.DocID)
+ }
+ retractedDocIDs = uniqueSortedStrings(retractedDocIDs)
+ wikiDiff, err := c.prepareWikiContributionDiff(ctx, tenant, kb, completed, retractedDocIDs)
if err != nil {
return err
}
- c.reportProgress(ctx, tenant, kb, token, 0.10, "comparing_wiki_contributions",
- fmt.Sprintf("Comparing Wiki contributions: %d affected page(s)", len(wikiDiff.affectedKeys)))
+ c.reportProgress(ctx, tenant, kb, token, taskTypes, 0.10, "comparing_knowledge_contributions",
+ fmt.Sprintf("Comparing knowledge contributions: %d affected page(s)", len(wikiDiff.affectedKeys)))
wikiOnly := len(completed) > 0
for _, entry := range completed {
if len(entry.Variants) != 1 || !variantsContain(entry.Variants, "wiki") {
@@ -609,17 +629,17 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
break
}
}
- if len(deleted) == 0 && wikiOnly && len(wikiDiff.affectedKeys) == 0 {
+ if len(retractedDocIDs) == 0 && wikiOnly && len(wikiDiff.affectedKeys) == 0 {
c.mu.Lock()
for _, docID := range pendingTombClear {
delete(c.tombs[kb], docID)
}
c.mu.Unlock()
- c.reportProgress(ctx, tenant, kb, token, 1, "completed", "Wiki contributions are unchanged; no dataset pages require updating")
+ c.reportProgress(ctx, tenant, kb, token, taskTypes, 1, "completed", "Knowledge contributions are unchanged; no dataset pages require updating")
return nil
}
- c.reportProgress(ctx, tenant, kb, token, 0.12, "deleting", fmt.Sprintf("Deleting products for %d document(s)", len(deleted)))
+ c.reportProgress(ctx, tenant, kb, token, taskTypes, 0.12, "deleting", fmt.Sprintf("Deleting products for %d document(s)", len(deleted)))
// --- Retraction (sequential DocEngine calls, no in-memory load) ---
// Deleted and disabled docs are handled before completion merges. Their
@@ -635,14 +655,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
// A merged product referencing several deleted docs is pruned in one pass,
// and an emptied product is removed exactly once.
if len(deleted) > 0 || len(disabled) > 0 {
- delIDs := make([]string, 0, len(deletedSet)+len(disabled))
- for d := range deletedSet {
- delIDs = append(delIDs, d)
- }
- for _, entry := range disabled {
- delIDs = append(delIDs, entry.DocID)
- }
- delIDs = uniqueSortedStrings(delIDs)
+ delIDs := retractedDocIDs
// Each destructive write runs under the scheduler's per-dataset
// write/rebuild lock with the generation check performed INSIDE the lock,
// so a rewrite can neither land between the check and the write nor
@@ -685,12 +698,15 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
}
}
if len(completed) == 0 && len(wikiDiff.affectedKeys) == 0 {
+ if err := c.commitWikiContributions(ctx, tenant, kb, wikiDiff.currentByDoc, retractedDocIDs); err != nil {
+ return err
+ }
c.mu.Lock()
for _, docID := range pendingTombClear {
delete(c.tombs[kb], docID)
}
c.mu.Unlock()
- c.reportProgress(ctx, tenant, kb, token, 1, "completed", "Document availability changes applied")
+ c.reportProgress(ctx, tenant, kb, token, taskTypes, 1, "completed", "Document availability changes applied")
return nil
}
deduper, err := c.factory(tenant)
@@ -716,7 +732,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
// completed and deleted is a stale tombstone: the deletion wins, so we skip
// its completion.
var incoming []kccommon.Product
- c.reportProgress(ctx, tenant, kb, token, 0.28, "loading_products", fmt.Sprintf("Loading products for %d document(s)", len(completed)))
+ c.reportProgress(ctx, tenant, kb, token, taskTypes, 0.28, "loading_products", fmt.Sprintf("Loading products for %d document(s)", len(completed)))
for _, e := range completed {
if deletedSet[e.DocID] {
continue
@@ -758,7 +774,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
// so both upsert their summary into the cross-document nav tree; wiki products
// continue to the page-specific evidence merge.
navIn := navInputFromProducts(kb, incoming)
- c.reportProgress(ctx, tenant, kb, token, 0.40, "merging_navigation", fmt.Sprintf("Merging navigation products: %d", len(navIn)))
+ c.reportProgress(ctx, tenant, kb, token, taskTypes, 0.40, "merging_navigation", fmt.Sprintf("Merging navigation products: %d", len(navIn)))
if len(navIn) > 0 {
ns := nav.GetNavService()
if ns == nil {
@@ -782,7 +798,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
}
return err
}
- c.reportProgress(ctx, tenant, kb, token, 0.50, "merging_structure", fmt.Sprintf("Merging structure products: %d", len(incoming)))
+ c.reportProgress(ctx, tenant, kb, token, taskTypes, 0.50, "merging_structure", fmt.Sprintf("Merging structure products: %d", len(incoming)))
// wiki_incremental port (M1): the dataset-level merge only processes wiki
// PAGES. A wiki doc yields both page and section products (Meta.kind
@@ -793,7 +809,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
// contract. Legacy rows whose kind is empty are derived in the Reader; only
// truly page-kind wiki products proceed.
candidates := filterWikiPageCandidates(incoming)
- c.reportProgress(ctx, tenant, kb, token, 0.54, "filtering_candidates", fmt.Sprintf("Filtering Wiki page candidates: %d", len(candidates)))
+ c.reportProgress(ctx, tenant, kb, token, taskTypes, 0.54, "filtering_candidates", fmt.Sprintf("Filtering knowledge page candidates: %d", len(candidates)))
entityMerged, topicCandidates, err := c.mergeEntityModeCandidates(ctx, tenant, kb, candidates)
if err != nil {
return err
@@ -811,7 +827,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
return err
}
}
- c.reportProgress(ctx, tenant, kb, token, 0.58, "merging_by_slug", fmt.Sprintf("Merging Wiki pages by slug: %d", len(entityMerged)))
+ c.reportProgress(ctx, tenant, kb, token, taskTypes, 0.58, "merging_by_slug", fmt.Sprintf("Merging knowledge pages by slug: %d", len(entityMerged)))
// Entity pages have stable page identity and must bypass topic-mode KNN and
// Page-specific entity/concept merging. Only topic-mode pages continue below.
// The slug-only merge above has already handled every Wiki page candidate.
@@ -977,7 +993,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
if err := runCompilerJobs(ctx, jobs); err != nil {
return err
}
- c.reportProgress(ctx, tenant, kb, token, 0.72, "routing_pages", fmt.Sprintf("Routing %d Wiki pages against existing products", len(candidates)))
+ c.reportProgress(ctx, tenant, kb, token, taskTypes, 0.72, "routing_pages", fmt.Sprintf("Routing %d knowledge pages against existing products", len(candidates)))
// KNN only narrows topic-page candidates. When the hit is a topic page,
// require an explicit topic match or an LLM topic-route decision before
@@ -1033,7 +1049,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
var newMerged []kccommon.Product
newMerged = append(newMerged, entityMerged...)
if len(groupsByID) > 0 {
- c.reportProgress(ctx, tenant, kb, token, 0.82, "llm_merge", fmt.Sprintf("Merging %d candidate groups with the LLM", len(groupsByID)))
+ c.reportProgress(ctx, tenant, kb, token, taskTypes, 0.82, "llm_merge", fmt.Sprintf("Merging %d candidate groups with the LLM", len(groupsByID)))
// Diagnostics: summarize the KNN groups before the LLM merge decision so
// the reader can see which existing merged rows were hit and by how many
// candidates (e.g. whether a "吕布" candidate hit an existing 吕布 row and
@@ -1085,7 +1101,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
mergedFinal = append(mergedFinal, unmatched...)
mergedFinal, staleTopicIDs := mergeTopicProducts(tenant, kb, mergedFinal)
mergedFinal = refreshWikiProductVectors(ctx, tenant, mergedFinal)
- c.reportProgress(ctx, tenant, kb, token, 0.90, "writing_products", fmt.Sprintf("Writing %d merged products", len(mergedFinal)))
+ c.reportProgress(ctx, tenant, kb, token, taskTypes, 0.90, "writing_products", fmt.Sprintf("Writing %d merged products", len(mergedFinal)))
if err := c.withWriteLock(ctx, kb, token, func() error {
if err := c.writer.WriteMerged(ctx, tenant, kb, mergedFinal); err != nil {
return err
@@ -1116,7 +1132,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
// then just drops any stale graph rows). The graph is reconstructible, so the
// cost of an occasional no-op reprojection is accepted. It is also a locked
// destructive side effect for the same TOCTOU reasons as WriteMerged.
- c.reportProgress(ctx, tenant, kb, token, 0.97, "projecting_graph", "Projecting the Wiki graph")
+ c.reportProgress(ctx, tenant, kb, token, taskTypes, 0.97, "projecting_graph", "Projecting the knowledge graph")
if err := c.withWriteLock(ctx, kb, token, func() error {
return c.writer.ProjectWikiGraph(ctx, tenant, kb)
}); err != nil {
@@ -1126,7 +1142,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
}
return err
}
- if err := c.commitWikiContributions(ctx, tenant, kb, wikiDiff.currentByDoc, deleted); err != nil {
+ if err := c.commitWikiContributions(ctx, tenant, kb, wikiDiff.currentByDoc, retractedDocIDs); err != nil {
return err
}
@@ -1143,7 +1159,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
zap.Int("completed_docs", len(completed)),
zap.Int("deleted_docs", len(deleted)),
zap.Int("merged_rows_written", len(mergedFinal)))
- c.reportProgress(ctx, tenant, kb, token, 1, "completed", fmt.Sprintf("Wiki compilation completed: %d merged products", len(mergedFinal)))
+ c.reportProgress(ctx, tenant, kb, token, taskTypes, 1, "completed", fmt.Sprintf("Knowledge compilation completed: %d merged products", len(mergedFinal)))
return nil
}
@@ -1446,7 +1462,10 @@ func productsForVariants(products []kccommon.Product, variants []string) []kccom
// summary itself when Embedd is empty, so the consumer needs no embedder.
func navInputFromProducts(kb string, products []kccommon.Product) []nav.UpsertDocInput {
type acc struct {
- in nav.UpsertDocInput
+ in nav.UpsertDocInput
+ treeSummary string
+ treeEmbedd []float32
+ structureLines []string
}
byDoc := make(map[string]*acc, len(products))
for i := range products {
@@ -1461,8 +1480,8 @@ func navInputFromProducts(kb string, products []kccommon.Product) []nav.UpsertDo
byDoc[p.DocID] = &acc{in: nav.UpsertDocInput{TenantID: p.TenantID, KbID: kb, DocID: p.DocID}}
a = byDoc[p.DocID]
}
- a.in.Summary = p.Content
- a.in.Embedd = p.Vector
+ a.treeSummary = p.Content
+ a.treeEmbedd = p.Vector
case kccommon.VariantStructure:
// Python's page_index path rebuilds the doc graph from the stored
// entity rows and folds their descriptions into the nav summary
@@ -1481,14 +1500,20 @@ func navInputFromProducts(kb string, products []kccommon.Product) []nav.UpsertDo
byDoc[p.DocID] = &acc{in: nav.UpsertDocInput{TenantID: p.TenantID, KbID: kb, DocID: p.DocID}}
a = byDoc[p.DocID]
}
- if a.in.Summary != "" {
- a.in.Summary += "\n"
- }
- a.in.Summary += line
+ a.structureLines = append(a.structureLines, line)
}
}
out := make([]nav.UpsertDocInput, 0, len(byDoc))
for _, a := range byDoc {
+ // Tree is the canonical navigation summary when both variants are present;
+ // otherwise use the folded structure/page-index summary. This makes the
+ // result independent of product iteration order.
+ if strings.TrimSpace(a.treeSummary) != "" {
+ a.in.Summary = a.treeSummary
+ a.in.Embedd = a.treeEmbedd
+ } else {
+ a.in.Summary = strings.Join(a.structureLines, "\n")
+ }
if strings.TrimSpace(a.in.Summary) == "" {
continue
}
diff --git a/internal/ingestion/knowledge_compile/dataset_log.go b/internal/ingestion/knowledge_compile/dataset_log.go
index 719a403d8f..b79fab49c3 100644
--- a/internal/ingestion/knowledge_compile/dataset_log.go
+++ b/internal/ingestion/knowledge_compile/dataset_log.go
@@ -20,14 +20,32 @@ import (
"crypto/sha256"
"encoding/hex"
"fmt"
+ "sort"
+ "strings"
"time"
"ragflow/internal/common"
"ragflow/internal/entity"
+ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
)
const datasetLogDocumentID = "graph_raptor_x"
+var allDatasetTaskTypes = []string{
+ kccommon.TaskTypeWiki,
+ kccommon.TaskTypeTree,
+ kccommon.TaskTypeGraph,
+ kccommon.TaskTypeMindmap,
+ kccommon.TaskTypeTimeline,
+ kccommon.TaskTypePageIndex,
+}
+
+var structureTaskTypes = []string{
+ kccommon.TaskTypeGraph,
+ kccommon.TaskTypeTimeline,
+ kccommon.TaskTypePageIndex,
+}
+
// Dataset ingestion logs use the status values exposed by the Python API and
// consumed by the shared frontend. The Go scheduler uses a different set of
// internal terminal names, so translate them at the persistence boundary.
@@ -44,52 +62,167 @@ func datasetLogOperationStatus(status string) string {
}
}
-func datasetCompileLogID(claimToken string) string {
- sum := sha256.Sum256([]byte("wiki-dataset-log\x00" + claimToken))
+func datasetCompileLogID(claimToken, taskType string) string {
+ sum := sha256.Sum256([]byte("knowledge-compile-dataset-log\x00" + claimToken + "\x00" + taskType))
return hex.EncodeToString(sum[:16])
}
+func taskTypesForVariants(variants []string) []string {
+ seen := make(map[string]struct{}, len(variants))
+ for _, variant := range variants {
+ var taskType string
+ switch strings.ToLower(strings.TrimSpace(variant)) {
+ case "wiki":
+ taskType = kccommon.TaskTypeWiki
+ case "tree":
+ taskType = kccommon.TaskTypeTree
+ case "mindmap", "mind_map":
+ taskType = kccommon.TaskTypeMindmap
+ case "structure":
+ // Legacy structure events predate the task-type field and cannot
+ // distinguish graph, timeline, and page-index products.
+ for _, structureTaskType := range structureTaskTypes {
+ seen[structureTaskType] = struct{}{}
+ }
+ }
+ if taskType != "" {
+ seen[taskType] = struct{}{}
+ }
+ }
+ return sortedTaskTypes(seen)
+}
+
+func taskTypesForEntry(entry BacklogEntry) []string {
+ seen := make(map[string]struct{}, len(entry.TaskTypes))
+ for _, taskType := range entry.TaskTypes {
+ if taskType = normalizeTaskType(taskType); taskType != "" {
+ seen[taskType] = struct{}{}
+ }
+ }
+ if len(seen) > 0 {
+ return sortedTaskTypes(seen)
+ }
+ if taskTypes := taskTypesForVariants(entry.Variants); len(taskTypes) > 0 {
+ return taskTypes
+ }
+ // An event with no routing metadata must remain visible in every possible
+ // dataset log. Returning the fallback per entry prevents an unknown entry
+ // from disappearing when other entries in the same batch are typed.
+ return append([]string(nil), allDatasetTaskTypes...)
+}
+
+func normalizeTaskType(taskType string) string {
+ switch strings.ToLower(strings.TrimSpace(taskType)) {
+ case "wiki":
+ return kccommon.TaskTypeWiki
+ case "tree":
+ return kccommon.TaskTypeTree
+ case "graph", "knowledge_graph", "knowledgegraph":
+ return kccommon.TaskTypeGraph
+ case "mindmap", "mind_map":
+ return kccommon.TaskTypeMindmap
+ case "timeline":
+ return kccommon.TaskTypeTimeline
+ case "pageindex", "page_index":
+ return kccommon.TaskTypePageIndex
+ default:
+ return ""
+ }
+}
+
+func taskTypesForEntries(entries []BacklogEntry) []string {
+ seen := make(map[string]struct{})
+ for _, entry := range entries {
+ for _, taskType := range taskTypesForEntry(entry) {
+ seen[taskType] = struct{}{}
+ }
+ }
+ if len(seen) == 0 {
+ // Legacy events did not carry routing metadata. A deletion can affect any
+ // dataset artifact, so expose one log per supported category rather than
+ // incorrectly labeling the event as Wiki.
+ for _, taskType := range allDatasetTaskTypes {
+ seen[taskType] = struct{}{}
+ }
+ }
+ return sortedTaskTypes(seen)
+}
+
+func sortedTaskTypes(seen map[string]struct{}) []string {
+ taskTypes := make([]string, 0, len(seen))
+ for taskType := range seen {
+ taskTypes = append(taskTypes, taskType)
+ }
+ sort.Strings(taskTypes)
+ return taskTypes
+}
+
func startDatasetCompileLog(ctx context.Context, tenantID, datasetID, claimToken string, entries []BacklogEntry) error {
if kcDB == nil || claimToken == "" {
return nil
}
now := time.Now()
status := "1"
- message := timestampProgressMessage(fmt.Sprintf("Created automatic Wiki dataset task for %d document event(s)", len(entries)))
- entryData := make([]any, 0, len(entries))
+ groups := make(map[string][]BacklogEntry)
for _, entry := range entries {
- entryData = append(entryData, map[string]any{
- "doc_id": entry.DocID,
- "event_type": entry.EventType,
- "variants": entry.Variants,
- })
- }
- log := entity.PipelineOperationLog{
- ID: datasetCompileLogID(claimToken),
- DocumentID: datasetLogDocumentID,
- TenantID: tenantID,
- KbID: datasetID,
- ParserID: "wiki",
- DocumentName: "Wiki",
- DocumentSuffix: "",
- DocumentType: "dataset",
- SourceFrom: "knowledgebase",
- Progress: 0,
- ProgressMsg: &message,
- ProcessBeginAt: &now,
- DSL: entity.JSONMap{"entries": entryData},
- TaskType: string(entity.PipelineTaskTypeWiki),
- OperationStatus: common.RUNNING,
- Status: &status,
- }
- return kcDB.WithContext(ctx).Where("id = ?", log.ID).FirstOrCreate(&log).Error
+ for _, taskType := range taskTypesForEntry(entry) {
+ groups[taskType] = append(groups[taskType], entry)
+ }
+ }
+ if len(groups) == 0 {
+ for _, taskType := range allDatasetTaskTypes {
+ groups[taskType] = entries
+ }
+ }
+ for _, taskType := range sortedTaskTypesFromGroups(groups) {
+ entryData := make([]any, 0, len(groups[taskType]))
+ for _, entry := range groups[taskType] {
+ entryData = append(entryData, map[string]any{
+ "doc_id": entry.DocID,
+ "event_type": entry.EventType,
+ "variants": entry.Variants,
+ "task_type": taskType,
+ })
+ }
+ message := timestampProgressMessage(fmt.Sprintf("Created automatic %s dataset task for %d document event(s)", taskType, len(groups[taskType])))
+ log := entity.PipelineOperationLog{
+ ID: datasetCompileLogID(claimToken, taskType),
+ DocumentID: datasetLogDocumentID,
+ TenantID: tenantID,
+ KbID: datasetID,
+ ParserID: "knowledge_compile",
+ DocumentName: taskType,
+ DocumentSuffix: "",
+ DocumentType: "dataset",
+ SourceFrom: "knowledgebase",
+ Progress: 0,
+ ProgressMsg: &message,
+ ProcessBeginAt: &now,
+ DSL: entity.JSONMap{"entries": entryData, "task_type": taskType},
+ TaskType: taskType,
+ OperationStatus: common.RUNNING,
+ Status: &status,
+ }
+ if err := kcDB.WithContext(ctx).Where("id = ?", log.ID).FirstOrCreate(&log).Error; err != nil {
+ return err
+ }
+ }
+ return nil
}
-func updateDatasetCompileLog(ctx context.Context, claimToken string, progress float64, message string) error {
+func updateDatasetCompileLog(ctx context.Context, claimToken string, taskTypes []string, progress float64, message string) error {
if kcDB == nil || claimToken == "" {
return nil
}
- logID := datasetCompileLogID(claimToken)
+ for _, taskType := range taskTypes {
+ if err := updateDatasetCompileLogForType(ctx, datasetCompileLogID(claimToken, taskType), progress, message); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func updateDatasetCompileLogForType(ctx context.Context, logID string, progress float64, message string) error {
var log entity.PipelineOperationLog
if err := kcDB.WithContext(ctx).Where("id = ?", logID).First(&log).Error; err != nil {
return err
@@ -111,28 +244,41 @@ func updateDatasetCompileLog(ctx context.Context, claimToken string, progress fl
}).Error
}
-func finishDatasetCompileLog(ctx context.Context, claimToken, operationStatus, message string, progress float64) error {
+func finishDatasetCompileLog(ctx context.Context, claimToken string, taskTypes []string, operationStatus, message string, progress float64) error {
if kcDB == nil || claimToken == "" {
return nil
}
- logID := datasetCompileLogID(claimToken)
- var log entity.PipelineOperationLog
- if err := kcDB.WithContext(ctx).Where("id = ?", logID).First(&log).Error; err != nil {
- return err
- }
- progressMessage := ""
- if log.ProgressMsg != nil {
- progressMessage = *log.ProgressMsg
+ for _, taskType := range taskTypes {
+ logID := datasetCompileLogID(claimToken, taskType)
+ var log entity.PipelineOperationLog
+ if err := kcDB.WithContext(ctx).Where("id = ?", logID).First(&log).Error; err != nil {
+ return err
+ }
+ progressMessage := ""
+ if log.ProgressMsg != nil {
+ progressMessage = *log.ProgressMsg
+ }
+ progressMessage = appendProgressMessage(progressMessage, timestampProgressMessage(message))
+ duration := log.ProcessDuration
+ if log.ProcessBeginAt != nil {
+ duration = max(0, time.Since(*log.ProcessBeginAt).Seconds())
+ }
+ if err := kcDB.WithContext(ctx).Model(&entity.PipelineOperationLog{}).Where("id = ?", logID).Updates(map[string]any{
+ "progress": progress,
+ "progress_msg": progressMessage,
+ "process_duration": duration,
+ "operation_status": datasetLogOperationStatus(operationStatus),
+ }).Error; err != nil {
+ return err
+ }
}
- progressMessage = appendProgressMessage(progressMessage, timestampProgressMessage(message))
- duration := log.ProcessDuration
- if log.ProcessBeginAt != nil {
- duration = max(0, time.Since(*log.ProcessBeginAt).Seconds())
+ return nil
+}
+
+func sortedTaskTypesFromGroups(groups map[string][]BacklogEntry) []string {
+ seen := make(map[string]struct{}, len(groups))
+ for taskType := range groups {
+ seen[taskType] = struct{}{}
}
- return kcDB.WithContext(ctx).Model(&entity.PipelineOperationLog{}).Where("id = ?", logID).Updates(map[string]any{
- "progress": progress,
- "progress_msg": progressMessage,
- "process_duration": duration,
- "operation_status": datasetLogOperationStatus(operationStatus),
- }).Error
+ return sortedTaskTypes(seen)
}
diff --git a/internal/ingestion/knowledge_compile/dataset_log_test.go b/internal/ingestion/knowledge_compile/dataset_log_test.go
index 072639b34e..a600fe619e 100644
--- a/internal/ingestion/knowledge_compile/dataset_log_test.go
+++ b/internal/ingestion/knowledge_compile/dataset_log_test.go
@@ -1,6 +1,7 @@
package knowledge_compile
import (
+ "reflect"
"strings"
"testing"
@@ -9,6 +10,7 @@ import (
"ragflow/internal/common"
"ragflow/internal/entity"
+ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
)
func TestDatasetCompileLogLifecycle(t *testing.T) {
@@ -23,28 +25,56 @@ func TestDatasetCompileLogLifecycle(t *testing.T) {
kcDB = db
t.Cleanup(func() { kcDB = oldDB })
- entries := []BacklogEntry{{DocID: "doc-1", EventType: string(EventTypeCompleted), Variants: []string{"wiki"}}}
+ entries := []BacklogEntry{
+ {DocID: "doc-1", EventType: string(EventTypeCompleted), Variants: []string{"wiki"}, TaskTypes: []string{kccommon.TaskTypeWiki}},
+ {DocID: "doc-2", EventType: string(EventTypeCompleted), Variants: []string{"structure"}, TaskTypes: []string{kccommon.TaskTypeGraph}},
+ }
if err := startDatasetCompileLog(t.Context(), "tenant-1", "kb-1", "claim-1", entries); err != nil {
t.Fatalf("start dataset log: %v", err)
}
- if err := updateDatasetCompileLog(t.Context(), "claim-1", 0.5, "Comparing Wiki contributions: 2 affected page(s)"); err != nil {
+ if err := updateDatasetCompileLog(t.Context(), "claim-1", []string{kccommon.TaskTypeWiki, kccommon.TaskTypeGraph}, 0.5, "Comparing knowledge contributions: 2 affected page(s)"); err != nil {
t.Fatalf("update dataset log: %v", err)
}
- if err := finishDatasetCompileLog(t.Context(), "claim-1", common.COMPLETED, "Wiki dataset compilation completed", 1); err != nil {
+ if err := finishDatasetCompileLog(t.Context(), "claim-1", []string{kccommon.TaskTypeWiki, kccommon.TaskTypeGraph}, common.COMPLETED, "Knowledge compilation completed", 1); err != nil {
t.Fatalf("finish dataset log: %v", err)
}
- var log entity.PipelineOperationLog
- if err := db.Where("id = ?", datasetCompileLogID("claim-1")).First(&log).Error; err != nil {
- t.Fatalf("load dataset log: %v", err)
+ var logs []entity.PipelineOperationLog
+ if err := db.Order("task_type").Find(&logs).Error; err != nil {
+ t.Fatalf("load dataset logs: %v", err)
}
- if log.DocumentID != datasetLogDocumentID || log.TaskType != string(entity.PipelineTaskTypeWiki) {
- t.Fatalf("unexpected dataset log identity: %+v", log)
+ if len(logs) != 2 {
+ t.Fatalf("want one log per task type, got %d: %+v", len(logs), logs)
}
- if log.OperationStatus != "DONE" || log.Progress != 1 {
- t.Fatalf("unexpected final state: status=%s progress=%v", log.OperationStatus, log.Progress)
+ for _, log := range logs {
+ if log.DocumentID != datasetLogDocumentID || log.OperationStatus != "DONE" || log.Progress != 1 {
+ t.Fatalf("unexpected dataset log state: %+v", log)
+ }
+ if log.ID != datasetCompileLogID("claim-1", log.TaskType) {
+ t.Fatalf("dataset log id is not task-type scoped: %+v", log)
+ }
+ if log.ProgressMsg == nil || !strings.Contains(*log.ProgressMsg, "2 affected page(s)") || !strings.Contains(*log.ProgressMsg, "completed") {
+ t.Fatalf("progress messages were not persisted: %v", log.ProgressMsg)
+ }
+ if log.DSL["task_type"] != log.TaskType {
+ t.Fatalf("dataset log DSL task type = %v, want %s", log.DSL["task_type"], log.TaskType)
+ }
+ entries, ok := log.DSL["entries"].([]any)
+ if !ok || len(entries) != 1 {
+ t.Fatalf("dataset log entries were not split by task type: %+v", log.DSL["entries"])
+ }
}
- if log.ProgressMsg == nil || !strings.Contains(*log.ProgressMsg, "2 affected page(s)") || !strings.Contains(*log.ProgressMsg, "completed") {
- t.Fatalf("progress messages were not persisted: %v", log.ProgressMsg)
+}
+
+func TestTaskTypesForEntryLegacyFallbacks(t *testing.T) {
+ got := taskTypesForEntry(BacklogEntry{Variants: []string{"structure"}})
+ want := []string{kccommon.TaskTypeGraph, kccommon.TaskTypePageIndex, kccommon.TaskTypeTimeline}
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("legacy structure task types = %v, want %v", got, want)
+ }
+
+ got = taskTypesForEntry(BacklogEntry{})
+ if !reflect.DeepEqual(got, allDatasetTaskTypes) {
+ t.Fatalf("unknown entry task types = %v, want %v", got, allDatasetTaskTypes)
}
}
diff --git a/internal/ingestion/knowledge_compile/event.go b/internal/ingestion/knowledge_compile/event.go
index bdc707b7f2..affe050870 100644
--- a/internal/ingestion/knowledge_compile/event.go
+++ b/internal/ingestion/knowledge_compile/event.go
@@ -110,37 +110,38 @@ func DefaultClaimer() Claimer { return defaultClaimer }
// doc so the consumer can dispatch to the matching dataset-level path. It is a
// no-op when no Publisher has been installed (e.g. DB unavailable). A failure is
// returned so callers can log but never fail the pipeline on it.
-func PublishCompleted(ctx context.Context, tenantID, datasetID, docID string, variants []string) error {
+func PublishCompleted(ctx context.Context, tenantID, datasetID, docID string, variants, taskTypes []string) error {
if defaultPublisher == nil {
return nil
}
- return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeCompleted), variants)
+ return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeCompleted), variants, taskTypes)
}
// PublishDeleted records a doc_deleted event the same way (Publish handles the
-// append + notify pairing). Deleted events carry no compile types.
-func PublishDeleted(ctx context.Context, tenantID, datasetID, docID string) error {
+// append + notify pairing). taskTypes identifies the document-level artifacts
+// that were removed before the event was published.
+func PublishDeleted(ctx context.Context, tenantID, datasetID, docID string, taskTypes []string) error {
if defaultPublisher == nil {
return nil
}
- return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeDeleted), nil)
+ return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeDeleted), nil, taskTypes)
}
// PublishEnabled records that a document with compiled products became
// available again and must be merged into dataset-level products.
-func PublishEnabled(ctx context.Context, tenantID, datasetID, docID string, variants []string) error {
+func PublishEnabled(ctx context.Context, tenantID, datasetID, docID string, variants, taskTypes []string) error {
if defaultPublisher == nil {
return nil
}
- return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeEnabled), variants)
+ return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeEnabled), variants, taskTypes)
}
// PublishDisabled records that a document with compiled products became
// unavailable. The consumer retracts only its dataset-level contributions;
// document-level compiled rows remain available for a later re-enable.
-func PublishDisabled(ctx context.Context, tenantID, datasetID, docID string, variants []string) error {
+func PublishDisabled(ctx context.Context, tenantID, datasetID, docID string, variants, taskTypes []string) error {
if defaultPublisher == nil {
return nil
}
- return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeDisabled), variants)
+ return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeDisabled), variants, taskTypes)
}
diff --git a/internal/ingestion/knowledge_compile/nav_dispatch_test.go b/internal/ingestion/knowledge_compile/nav_dispatch_test.go
index b1dad687e5..fa2aea0bed 100644
--- a/internal/ingestion/knowledge_compile/nav_dispatch_test.go
+++ b/internal/ingestion/knowledge_compile/nav_dispatch_test.go
@@ -105,6 +105,28 @@ func TestNavInputFromProducts_EmptySummaryDropped(t *testing.T) {
}
}
+func TestNavInputFromProducts_TreeSummaryWinsRegardlessOfOrder(t *testing.T) {
+ products := []kccommon.Product{
+ {DocID: "d1", TenantID: "t1", Variant: kccommon.VariantStructure,
+ Content: `{"name":"A","description":"structure summary"}`,
+ Meta: map[string]any{"kind": "entity"}},
+ {DocID: "d1", TenantID: "t1", Variant: kccommon.VariantTree,
+ Content: "tree summary", Vector: []float32{0.1},
+ Meta: map[string]any{"kind": "root"}},
+ }
+
+ got := navInputFromProducts("kb1", products)
+ if len(got) != 1 {
+ t.Fatalf("want one nav input, got %d", len(got))
+ }
+ if got[0].Summary != "tree summary" {
+ t.Fatalf("summary = %q, want tree summary", got[0].Summary)
+ }
+ if !reflect.DeepEqual(got[0].Embedd, []float32{0.1}) {
+ t.Fatalf("embedding = %v, want tree embedding", got[0].Embedd)
+ }
+}
+
// TestProductsForVariants_GatesDispatch covers the A0-4 per-variant gate: a doc
// whose event carries only ["wiki"] must NOT have its stale tree/structure
// doc-level products leak into the nav/structure paths. Empty variants (legacy
diff --git a/internal/ingestion/knowledge_compile/rebuild_variants_test.go b/internal/ingestion/knowledge_compile/rebuild_variants_test.go
index cef9c7b949..639719e055 100644
--- a/internal/ingestion/knowledge_compile/rebuild_variants_test.go
+++ b/internal/ingestion/knowledge_compile/rebuild_variants_test.go
@@ -9,7 +9,7 @@ import (
)
// fakeReader is a Reader double that returns a canned product set for
-// recoverDocVariants tests.
+// recoverDocTypes tests.
type fakeReader struct {
products []kccommon.Product
}
@@ -22,7 +22,7 @@ func (f *fakeReader) SearchSimilar(context.Context, string, string, kccommon.Var
return kccommon.Product{}, 0, nil
}
-// TestRecoverDocVariants_AuthoritativeKind covers B1a/O2a: the authoritative
+// TestRecoverDocTypes_AuthoritativeKind covers B1a/O2a: the authoritative
// Product.Kind (compilation_template_kind_kwd) is mapped through KindToVariant;
// results are sorted/deduped.
func TestRecoverDocVariants_AuthoritativeKind(t *testing.T) {
@@ -32,17 +32,20 @@ func TestRecoverDocVariants_AuthoritativeKind(t *testing.T) {
// duplicate variant, deduped
{DocID: "d1", Kind: "structure"},
}}}
- got, err := c.recoverDocVariants(context.Background(), "t1", "kb1", "d1")
+ got, taskTypes, err := c.recoverDocTypes(context.Background(), "t1", "kb1", "d1")
if err != nil {
- t.Fatalf("recoverDocVariants error: %v", err)
+ t.Fatalf("recoverDocTypes error: %v", err)
}
want := []string{string(kccommon.VariantStructure), string(kccommon.VariantTree)}
if !reflect.DeepEqual(got, want) {
- t.Fatalf("recoverDocVariants = %v, want %v", got, want)
+ t.Fatalf("recoverDocTypes variants = %v, want %v", got, want)
+ }
+ if !reflect.DeepEqual(taskTypes, []string{kccommon.TaskTypeGraph, kccommon.TaskTypeTree}) {
+ t.Fatalf("recoverDocTypes task types = %v, want [Graph Tree]", taskTypes)
}
}
-// TestRecoverDocVariants_UnknownKindHardFails covers O2a: a whitelist-out
+// TestRecoverDocTypes_UnknownKindHardFails covers O2a: a whitelist-out
// authoritative kind aborts recovery (returns an error) so the rebuild does not
// proceed with an incomplete variant set.
func TestRecoverDocVariants_UnknownKindHardFails(t *testing.T) {
@@ -50,22 +53,25 @@ func TestRecoverDocVariants_UnknownKindHardFails(t *testing.T) {
{DocID: "d1", Kind: "structure"},
{DocID: "d1", Kind: "garbage"},
}}}
- if _, err := c.recoverDocVariants(context.Background(), "t1", "kb1", "d1"); err == nil {
- t.Fatal("recoverDocVariants must hard-fail on an unknown authoritative kind (O2a)")
+ if _, _, err := c.recoverDocTypes(context.Background(), "t1", "kb1", "d1"); err == nil {
+ t.Fatal("recoverDocTypes must hard-fail on an unknown authoritative kind (O2a)")
}
}
-// TestRecoverDocVariants_FallbackVariant covers B1a: a product without an
+// TestRecoverDocTypes_FallbackVariant covers B1a: a product without an
// authoritative kind falls back to its reverse-mapped variant.
func TestRecoverDocVariants_FallbackVariant(t *testing.T) {
c := &Consumer{reader: &fakeReader{products: []kccommon.Product{
{DocID: "d1", Variant: kccommon.VariantWiki},
}}}
- got, err := c.recoverDocVariants(context.Background(), "t1", "kb1", "d1")
+ got, taskTypes, err := c.recoverDocTypes(context.Background(), "t1", "kb1", "d1")
if err != nil {
- t.Fatalf("recoverDocVariants error: %v", err)
+ t.Fatalf("recoverDocTypes error: %v", err)
}
if len(got) != 1 || got[0] != string(kccommon.VariantWiki) {
- t.Fatalf("recoverDocVariants fallback = %v, want [wiki]", got)
+ t.Fatalf("recoverDocTypes fallback = %v, want [wiki]", got)
+ }
+ if !reflect.DeepEqual(taskTypes, []string{kccommon.TaskTypeWiki}) {
+ t.Fatalf("recoverDocTypes fallback task types = %v, want [Wiki]", taskTypes)
}
}
diff --git a/internal/ingestion/knowledge_compile/scheduler.go b/internal/ingestion/knowledge_compile/scheduler.go
index 76430c9af8..a65149ffbe 100644
--- a/internal/ingestion/knowledge_compile/scheduler.go
+++ b/internal/ingestion/knowledge_compile/scheduler.go
@@ -61,13 +61,16 @@ const (
// Variants records the compile types (tree/structure/wiki/mindmap) produced by
// the doc-level compile for this doc, so the consumer can group a claimed batch
// into per-variant sub-batches and dispatch each to its own dataset-level
-// compile path. It is empty (nil) for deleted events and for events published
-// before this field existed; the consumer treats a nil/empty Variants as
-// "unknown/legacy" and falls back to the legacy unified path.
+// compile path. TaskTypes preserves the original frontend-facing category for
+// each template, so structure sub-kinds such as Graph, Timeline, and PageIndex
+// can be logged separately. Both fields are empty (nil) for deleted events and
+// for events published before they existed; the consumer falls back to the
+// legacy unified path when the fields are absent.
type BacklogEntry struct {
DocID string `json:"doc_id"`
EventType string `json:"event_type"`
Variants []string `json:"variants,omitempty"`
+ TaskTypes []string `json:"task_types,omitempty"`
}
// ClaimResult is returned by Scheduler.Claim: the KB's tenant plus the closed
@@ -92,8 +95,9 @@ type Publisher interface {
// idle workers. It is transactional so concurrent publishers do not clobber
// each other's backlog, and it always pairs the append with a notify.
// variants carries the doc-level compile types (tree/structure/wiki/mindmap)
- // for completed events; it is nil for deleted events.
- Publish(ctx context.Context, tenantID, datasetID, docID, eventType string, variants []string) error
+ // for completed events; taskTypes carries the frontend-facing categories for
+ // those templates. Both are nil for deleted events.
+ Publish(ctx context.Context, tenantID, datasetID, docID, eventType string, variants, taskTypes []string) error
}
// Claimer is the consumer-side role (Option E §11.5). A cluster of competing
@@ -216,8 +220,8 @@ func (s *mysqlScheduler) Provision(ctx context.Context) error {
// workers. The append is transactional (concurrent publishers do not clobber
// each other's backlog), and the notify is always paired with the append so a
// producer never needs a separate Notify call.
-func (s *mysqlScheduler) Publish(ctx context.Context, tenantID, datasetID, docID, eventType string, variants []string) error {
- return s.publishEntry(ctx, tenantID, datasetID, BacklogEntry{DocID: docID, EventType: eventType, Variants: variants})
+func (s *mysqlScheduler) Publish(ctx context.Context, tenantID, datasetID, docID, eventType string, variants, taskTypes []string) error {
+ return s.publishEntry(ctx, tenantID, datasetID, BacklogEntry{DocID: docID, EventType: eventType, Variants: variants, TaskTypes: taskTypes})
}
// loadRow fetches the dataset scheduling row (no lock). Returns nil (no error)
@@ -233,8 +237,8 @@ func (s *mysqlScheduler) loadRow(ctx context.Context, datasetID string) (*entity
return &row, nil
}
-func (s *mysqlScheduler) publish(ctx context.Context, tenantID, datasetID, docID, eventType string, variants []string) error {
- return s.publishEntry(ctx, tenantID, datasetID, BacklogEntry{DocID: docID, EventType: eventType, Variants: variants})
+func (s *mysqlScheduler) publish(ctx context.Context, tenantID, datasetID, docID, eventType string, variants, taskTypes []string) error {
+ return s.publishEntry(ctx, tenantID, datasetID, BacklogEntry{DocID: docID, EventType: eventType, Variants: variants, TaskTypes: taskTypes})
}
func (s *mysqlScheduler) publishEntry(ctx context.Context, tenantID, datasetID string, entry BacklogEntry) error {
@@ -783,8 +787,8 @@ func NewFakeScheduler() *FakeScheduler {
func (f *FakeScheduler) Provision(_ context.Context) error { return nil }
// Publish appends one doc event and pushes a notify (same contract as MySQL).
-func (f *FakeScheduler) Publish(_ context.Context, tenantID, datasetID, docID, eventType string, variants []string) error {
- return f.publishEntry(tenantID, datasetID, BacklogEntry{DocID: docID, EventType: eventType, Variants: variants})
+func (f *FakeScheduler) Publish(_ context.Context, tenantID, datasetID, docID, eventType string, variants, taskTypes []string) error {
+ return f.publishEntry(tenantID, datasetID, BacklogEntry{DocID: docID, EventType: eventType, Variants: variants, TaskTypes: taskTypes})
}
// PublishedCount returns the total number of backlog events published across
diff --git a/internal/ingestion/knowledge_compile/scheduler_test.go b/internal/ingestion/knowledge_compile/scheduler_test.go
index fe592b1773..2e3d3e4a03 100644
--- a/internal/ingestion/knowledge_compile/scheduler_test.go
+++ b/internal/ingestion/knowledge_compile/scheduler_test.go
@@ -36,7 +36,7 @@ func TestRemoveEntriesMatchesByDocAndEventType(t *testing.T) {
func TestFakeSchedulerReclaimsInterruptedClaim(t *testing.T) {
f := NewFakeScheduler()
- if err := f.Publish(t.Context(), "t1", "kb1", "d1", string(EventTypeCompleted), []string{"wiki"}); err != nil {
+ if err := f.Publish(t.Context(), "t1", "kb1", "d1", string(EventTypeCompleted), []string{"wiki"}, []string{"Wiki"}); err != nil {
t.Fatalf("publish: %v", err)
}
first, ok, err := f.Claim(t.Context(), "kb1")
@@ -58,7 +58,7 @@ func TestFakeSchedulerReclaimsInterruptedClaim(t *testing.T) {
func TestFakeSchedulerProgressIsClaimScoped(t *testing.T) {
f := NewFakeScheduler()
- if err := f.Publish(t.Context(), "t1", "kb1", "d1", string(EventTypeCompleted), nil); err != nil {
+ if err := f.Publish(t.Context(), "t1", "kb1", "d1", string(EventTypeCompleted), nil, nil); err != nil {
t.Fatalf("publish: %v", err)
}
claim, ok, err := f.Claim(t.Context(), "kb1")
@@ -81,7 +81,7 @@ func TestFakeSchedulerProgressIsClaimScoped(t *testing.T) {
func TestWithWriteLockRejectsSupersededClaim(t *testing.T) {
f := NewFakeScheduler()
- if err := f.Publish(t.Context(), "t1", "kb1", "d1", string(EventTypeCompleted), []string{"wiki"}); err != nil {
+ if err := f.Publish(t.Context(), "t1", "kb1", "d1", string(EventTypeCompleted), []string{"wiki"}, []string{"Wiki"}); err != nil {
t.Fatalf("publish: %v", err)
}
claim, ok, err := f.Claim(t.Context(), "kb1")
@@ -131,14 +131,17 @@ func TestBacklogEntryJSONBackwardCompat(t *testing.T) {
t.Fatalf("legacy backlog should unmarshal Variants as nil, got %v", e.Variants)
}
- // A completed event with variants round-trips intact.
- withVariants := []byte(`{"doc_id":"d1","event_type":"doc_completed","variants":["tree","wiki"]}`)
+ // A completed event with variants and frontend task types round-trips intact.
+ withVariants := []byte(`{"doc_id":"d1","event_type":"doc_completed","variants":["tree","wiki"],"task_types":["Tree","Wiki"]}`)
if err := json.Unmarshal(withVariants, &e); err != nil {
t.Fatalf("unmarshal variants backlog: %v", err)
}
if !reflect.DeepEqual(e.Variants, []string{"tree", "wiki"}) {
t.Fatalf("variants not restored: %+v", e.Variants)
}
+ if !reflect.DeepEqual(e.TaskTypes, []string{"Tree", "Wiki"}) {
+ t.Fatalf("task types not restored: %+v", e.TaskTypes)
+ }
// omitempty: a nil Variants serializes without the key (matches legacy format).
enc, err := json.Marshal(BacklogEntry{DocID: "d2", EventType: "doc_deleted"})
if err != nil {
@@ -151,16 +154,19 @@ func TestBacklogEntryJSONBackwardCompat(t *testing.T) {
if _, hasVariants := decoded["variants"]; hasVariants {
t.Fatalf("nil Variants should be omitted, got %s", enc)
}
+ if _, hasTaskTypes := decoded["task_types"]; hasTaskTypes {
+ t.Fatalf("nil TaskTypes should be omitted, got %s", enc)
+ }
}
// TestFakeSchedulerPublishCarriesVariants locks the A0-2 contract: FakeScheduler
// records the variants on the backlog entry, so the consumer sees them.
func TestFakeSchedulerPublishCarriesVariants(t *testing.T) {
f := NewFakeScheduler()
- if err := f.Publish(t.Context(), "t1", "kb1", "d1", string(EventTypeCompleted), []string{"tree", "structure"}); err != nil {
+ if err := f.Publish(t.Context(), "t1", "kb1", "d1", string(EventTypeCompleted), []string{"tree", "structure"}, []string{"Tree", "Graph"}); err != nil {
t.Fatalf("publish: %v", err)
}
- if err := f.Publish(t.Context(), "t1", "kb1", "d2", string(EventTypeDeleted), nil); err != nil {
+ if err := f.Publish(t.Context(), "t1", "kb1", "d2", string(EventTypeDeleted), nil, nil); err != nil {
t.Fatalf("publish deleted: %v", err)
}
res, ok, err := f.Claim(t.Context(), "kb1")
@@ -173,6 +179,9 @@ func TestFakeSchedulerPublishCarriesVariants(t *testing.T) {
if !reflect.DeepEqual(res.Entries[0].Variants, []string{"tree", "structure"}) {
t.Fatalf("completed variants not carried: %+v", res.Entries[0])
}
+ if !reflect.DeepEqual(res.Entries[0].TaskTypes, []string{"Tree", "Graph"}) {
+ t.Fatalf("completed task types not carried: %+v", res.Entries[0])
+ }
if res.Entries[1].Variants != nil {
t.Fatalf("deleted event should carry nil variants, got %+v", res.Entries[1])
}
diff --git a/internal/ingestion/knowledge_compile/wiki_contribution_test.go b/internal/ingestion/knowledge_compile/wiki_contribution_test.go
index db4531db14..5ba6d3143f 100644
--- a/internal/ingestion/knowledge_compile/wiki_contribution_test.go
+++ b/internal/ingestion/knowledge_compile/wiki_contribution_test.go
@@ -83,7 +83,7 @@ func TestProcessBatchSkipsUnchangedWikiOnlyEvent(t *testing.T) {
)
if err := consumer.processBatch(context.Background(), "tenant-1", "kb-1", "token-1", []BacklogEntry{{
DocID: "doc-1", EventType: string(EventTypeCompleted), Variants: []string{"wiki"},
- }}); err != nil {
+ }}, []string{kccommon.TaskTypeWiki}); err != nil {
t.Fatalf("processBatch failed: %v", err)
}
if factoryCalled {
diff --git a/internal/ingestion/task/knowledge_compiler_wiring.go b/internal/ingestion/task/knowledge_compiler_wiring.go
index d6734e54b9..1444cce9cd 100644
--- a/internal/ingestion/task/knowledge_compiler_wiring.go
+++ b/internal/ingestion/task/knowledge_compiler_wiring.go
@@ -363,7 +363,7 @@ func replaceDirtyWikiProducts(ctx context.Context, request knowledge_compile.Wik
if len(rows) == 0 && (existing == nil || len(existing.Chunks) == 0) {
return nil
}
- return knowledge_compile.PublishCompleted(ctx, request.TenantID, request.DatasetID, request.DocumentID, []string{"wiki"})
+ return knowledge_compile.PublishCompleted(ctx, request.TenantID, request.DatasetID, request.DocumentID, []string{"wiki"}, []string{kc.TaskTypeWiki})
}
func clearWikiActiveStates(ctx context.Context, docEngine engine.DocEngine, request knowledge_compile.WikiDirtyRequest) error {
diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.go
index de5ae141bc..8afabef237 100644
--- a/internal/ingestion/task/pipeline_executor.go
+++ b/internal/ingestion/task/pipeline_executor.go
@@ -278,7 +278,7 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map
}
applyDocumentAvailability(chunks, docStatus)
- oldCompiledProductIDs, oldCompiledVariants, err := s.loadDocumentCompiledState(ctx)
+ oldCompiledProductIDs, oldCompiledVariants, oldCompiledTaskTypes, err := s.loadDocumentCompiledState(ctx)
if err != nil {
return nil, err
}
@@ -310,8 +310,9 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map
// KindToVariant, O2a whitelist). Keeping the previous types lets the dataset
// consumer retract stale merged products when a template is removed.
eventVariants := mergeCompiledVariants(oldCompiledVariants, compiledVariants(chunks))
+ eventTaskTypes := mergeTaskTypes(oldCompiledTaskTypes, compiledTaskTypes(chunks))
if len(eventVariants) > 0 {
- if err := knowledge_compile.PublishCompleted(ctx, s.taskCtx.Tenant.ID, s.taskCtx.Doc.KbID, s.taskCtx.Doc.ID, eventVariants); err != nil {
+ if err := knowledge_compile.PublishCompleted(ctx, s.taskCtx.Tenant.ID, s.taskCtx.Doc.KbID, s.taskCtx.Doc.ID, eventVariants, eventTaskTypes); err != nil {
common.Logger.Warn(fmt.Sprintf("knowledge_compile: publish doc_completed for %s failed: %v", s.taskCtx.Doc.ID, err))
}
}
@@ -463,10 +464,10 @@ func markCompiledProductsHidden(chunks []map[string]any) {
// loadDocumentCompiledState snapshots the previous successful document
// compiler generation before the new pipeline output is written. Reading first
// avoids relying on immediate search visibility after a bulk index write.
-func (s *PipelineExecutor) loadDocumentCompiledState(ctx context.Context) ([]string, []string, error) {
+func (s *PipelineExecutor) loadDocumentCompiledState(ctx context.Context) ([]string, []string, []string, error) {
docEngine := engine.Get()
if docEngine == nil || s == nil || s.taskCtx == nil {
- return nil, nil, nil
+ return nil, nil, nil, nil
}
const pageSize = 1000
indexName := fmt.Sprintf("ragflow_%s", s.taskCtx.Tenant.ID)
@@ -482,7 +483,7 @@ func (s *PipelineExecutor) loadDocumentCompiledState(ctx context.Context) ([]str
Filter: map[string]any{"doc_id": []string{s.taskCtx.Doc.ID}},
})
if err != nil {
- return nil, nil, fmt.Errorf("load document compiler products: %w", err)
+ return nil, nil, nil, fmt.Errorf("load document compiler products: %w", err)
}
if result == nil || len(result.Chunks) == 0 {
break
@@ -500,7 +501,7 @@ func (s *PipelineExecutor) loadDocumentCompiledState(ctx context.Context) ([]str
break
}
}
- return oldIDs, compiledVariants(oldProducts), nil
+ return oldIDs, compiledVariants(oldProducts), compiledTaskTypes(oldProducts), nil
}
// reconcileDocumentCompiledProducts advances the document-level compiler
@@ -606,6 +607,54 @@ func compiledVariants(chunks []map[string]any) []string {
return out
}
+// compiledTaskTypes returns the sorted, de-duplicated frontend categories of
+// the compiled products in chunks. The raw template kind is preferred because
+// several kinds intentionally share the structure execution variant.
+func compiledTaskTypes(chunks []map[string]any) []string {
+ seen := map[string]struct{}{}
+ for _, ck := range chunks {
+ if _, ok := ck["compile_kwd"]; !ok {
+ continue
+ }
+ taskType := ""
+ if kind, ok := ck["compilation_template_kind_kwd"].(string); ok && strings.TrimSpace(kind) != "" {
+ if mapped, err := kccommon.KindToTaskType(kind); err == nil {
+ taskType = mapped
+ }
+ }
+ if taskType == "" {
+ if mapped, err := knowledge_compile.KwdToVariant(asCompiledKwd(ck)); err == nil {
+ taskType = kccommon.VariantToTaskType(mapped)
+ }
+ }
+ if taskType != "" {
+ seen[taskType] = struct{}{}
+ }
+ }
+ return sortedTaskTypes(seen)
+}
+
+func mergeTaskTypes(groups ...[]string) []string {
+ seen := map[string]struct{}{}
+ for _, group := range groups {
+ for _, taskType := range group {
+ if taskType = strings.TrimSpace(taskType); taskType != "" {
+ seen[taskType] = struct{}{}
+ }
+ }
+ }
+ return sortedTaskTypes(seen)
+}
+
+func sortedTaskTypes(seen map[string]struct{}) []string {
+ out := make([]string, 0, len(seen))
+ for taskType := range seen {
+ out = append(out, taskType)
+ }
+ sort.Strings(out)
+ return out
+}
+
// mergeCompiledVariants returns the union of the previous and current
// document-level compiler variants. A reparse that removes a compiler template
// still needs to notify the dataset consumer so it can retract stale merged
diff --git a/internal/service/document/document_crud.go b/internal/service/document/document_crud.go
index add8615436..502eb518c2 100644
--- a/internal/service/document/document_crud.go
+++ b/internal/service/document/document_crud.go
@@ -338,6 +338,10 @@ func (s *DocumentService) RemoveDocumentKeepFile(ctx context.Context, docID stri
if err != nil {
return err
}
+ _, taskTypes, typeErr := s.documentKnowledgeCompileTypes(ctx, kb.TenantID, kb.ID, docID)
+ if typeErr != nil {
+ common.Warn(fmt.Sprintf("RemoveDocumentKeepFile: failed to resolve knowledge compile types for %s: %v", docID, typeErr))
+ }
if _, delErr := s.taskDAO.DeleteByDocIDs(ctx, dao.DB, []string{docID}); delErr != nil {
if errors.Is(delErr, context.Canceled) || errors.Is(delErr, context.DeadlineExceeded) {
return fmt.Errorf("RemoveDocumentKeepFile: failed to delete tasks for %s: %w", docID, delErr)
@@ -348,11 +352,11 @@ func (s *DocumentService) RemoveDocumentKeepFile(ctx context.Context, docID stri
return err
}
// File replacement/deletion uses this path instead of deleteDocumentFull.
- // Publish the same deletion event so the dataset-level Wiki consumer removes
- // the deleted document's contribution in both paths.
+ // Publish the same deletion event so the dataset-level consumer removes the
+ // deleted document's contribution in both paths.
pubCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)
defer cancel()
- if err := knowledge_compile.PublishDeleted(pubCtx, kb.TenantID, kb.ID, docID); err != nil {
+ if err := knowledge_compile.PublishDeleted(pubCtx, kb.TenantID, kb.ID, docID, taskTypes); err != nil {
common.Warn(fmt.Sprintf("RemoveDocumentKeepFile: publish doc_deleted for %s failed: %v", docID, err))
}
return nil
@@ -422,6 +426,10 @@ func (s *DocumentService) deleteDocEngineData(ctx context.Context, docID, tenant
return
}
indexName := fmt.Sprintf("ragflow_%s", tenantID)
+ _, taskTypes, typeErr := s.documentKnowledgeCompileTypes(ctx, tenantID, kbID, docID)
+ if typeErr != nil {
+ common.Warn(fmt.Sprintf("deleteDocEngineData: failed to resolve knowledge compile types for %s: %v", docID, typeErr))
+ }
if _, delErr := s.docEngine.DeleteChunks(ctx, map[string]interface{}{"doc_id": docID}, indexName, kbID); delErr != nil {
common.Warn(fmt.Sprintf("deleteDocEngineData: failed to delete chunks for %s: %v", docID, delErr))
}
@@ -434,7 +442,7 @@ func (s *DocumentService) deleteDocEngineData(ctx context.Context, docID, tenant
// never block the document delete, which already succeeded above.
pubCtx, cancel := context.WithTimeout(ctx, 15*time.Second)
defer cancel()
- if err := knowledge_compile.PublishDeleted(pubCtx, tenantID, kbID, docID); err != nil {
+ if err := knowledge_compile.PublishDeleted(pubCtx, tenantID, kbID, docID, taskTypes); err != nil {
common.Warn(fmt.Sprintf("deleteDocEngineData: publish doc_deleted for %s failed: %v", docID, err))
}
if s.metadataSvc != nil {
diff --git a/internal/service/document/knowledge_compile_status.go b/internal/service/document/knowledge_compile_status.go
index cc0478b82a..24f39c8684 100644
--- a/internal/service/document/knowledge_compile_status.go
+++ b/internal/service/document/knowledge_compile_status.go
@@ -34,10 +34,16 @@ import (
// products do not wake the consumer because they cannot contribute to any
// dataset-level knowledge artifact.
func (s *DocumentService) publishKnowledgeCompileStatusChange(ctx context.Context, tenantID, datasetID, documentID string, status int) {
- variants, err := s.documentKnowledgeCompileVariants(ctx, tenantID, datasetID, documentID)
+ variants, taskTypes, err := s.documentKnowledgeCompileTypes(ctx, tenantID, datasetID, documentID)
if err != nil {
common.Warn("document mutation: failed to resolve knowledge compile variants",
zap.String("document_id", documentID), zap.Error(err))
+ if status == 0 {
+ if publishErr := knowledge_compile.PublishDisabled(ctx, tenantID, datasetID, documentID, nil, nil); publishErr != nil {
+ common.Warn("document mutation: failed to publish fallback knowledge compile disable",
+ zap.String("document_id", documentID), zap.Error(publishErr))
+ }
+ }
return
}
if len(variants) == 0 {
@@ -45,9 +51,9 @@ func (s *DocumentService) publishKnowledgeCompileStatusChange(ctx context.Contex
}
var publishErr error
if status == 0 {
- publishErr = knowledge_compile.PublishDisabled(ctx, tenantID, datasetID, documentID, variants)
+ publishErr = knowledge_compile.PublishDisabled(ctx, tenantID, datasetID, documentID, variants, taskTypes)
} else {
- publishErr = knowledge_compile.PublishEnabled(ctx, tenantID, datasetID, documentID, variants)
+ publishErr = knowledge_compile.PublishEnabled(ctx, tenantID, datasetID, documentID, variants, taskTypes)
}
if publishErr != nil {
common.Warn("document mutation: failed to publish knowledge compile status change",
@@ -55,12 +61,13 @@ func (s *DocumentService) publishKnowledgeCompileStatusChange(ctx context.Contex
}
}
-func (s *DocumentService) documentKnowledgeCompileVariants(ctx context.Context, tenantID, datasetID, documentID string) ([]string, error) {
+func (s *DocumentService) documentKnowledgeCompileTypes(ctx context.Context, tenantID, datasetID, documentID string) ([]string, []string, error) {
if s.docEngine == nil {
- return nil, nil
+ return nil, nil, nil
}
indexName := fmt.Sprintf("ragflow_%s", tenantID)
- seen := make(map[string]struct{})
+ seenVariants := make(map[string]struct{})
+ seenTaskTypes := make(map[string]struct{})
for offset := 0; ; offset += 1000 {
result, err := s.docEngine.Search(ctx, &types.SearchRequest{
IndexNames: []string{indexName},
@@ -71,7 +78,7 @@ func (s *DocumentService) documentKnowledgeCompileVariants(ctx context.Context,
Filter: map[string]any{"doc_id": []string{documentID}},
})
if err != nil {
- return nil, err
+ return nil, nil, err
}
if result == nil || len(result.Chunks) == 0 {
break
@@ -83,17 +90,29 @@ func (s *DocumentService) documentKnowledgeCompileVariants(ctx context.Context,
variant, variantErr = knowledge_compile.KwdToVariant(strings.TrimSpace(documentStoreString(row["compile_kwd"])))
}
if variantErr == nil {
- seen[string(variant)] = struct{}{}
+ seenVariants[string(variant)] = struct{}{}
+ taskType, taskTypeErr := kccommon.KindToTaskType(kind)
+ if taskTypeErr != nil {
+ taskType = kccommon.VariantToTaskType(variant)
+ }
+ if taskType != "" {
+ seenTaskTypes[taskType] = struct{}{}
+ }
}
}
if int64(offset+len(result.Chunks)) >= result.Total {
break
}
}
- variants := make([]string, 0, len(seen))
- for variant := range seen {
+ variants := make([]string, 0, len(seenVariants))
+ for variant := range seenVariants {
variants = append(variants, variant)
}
sort.Strings(variants)
- return variants, nil
+ taskTypes := make([]string, 0, len(seenTaskTypes))
+ for taskType := range seenTaskTypes {
+ taskTypes = append(taskTypes, taskType)
+ }
+ sort.Strings(taskTypes)
+ return variants, taskTypes, nil
}