RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: split knowledge compilation logs by task type (#19548)
buua436ec5a0136de42
diff --git a/internal/ingestion/component/knowledge_compiler/common/types.go b/internal/ingestion/component/knowledge_compiler/common/types.goindex 3fcda008a0..a7070a7471 100644--- a/internal/ingestion/component/knowledge_compiler/common/types.go+++ b/internal/ingestion/component/knowledge_compiler/common/types.go@@ -24,6 +24,18 @@ const (VariantMindmap Variant = "mindmap")+// Task type labels are the values exposed by the pipeline operation log API.+// They intentionally preserve the frontend categories instead of collapsing+// every structure-based template into the internal structure variant.+const (+ TaskTypeWiki = "Wiki"+ TaskTypeTree = "Tree"+ TaskTypeGraph = "Graph"+ TaskTypeMindmap = "Mindmap"+ TaskTypeTimeline = "Timeline"+ TaskTypePageIndex = "PageIndex"+)+// Sentinel errors.var (ErrUnknownVariant = errors.New("knowledge_compiler: unknown variant")@@ -230,6 +242,46 @@ func KindToVariant(kind string) (Variant, error) {}}+// KindToTaskType maps the original compilation template kind to the category+// shown in pipeline operation logs. Unlike KindToVariant, this mapping keeps+// structure sub-kinds distinct so graph, timeline, and page-index logs are not+// mislabeled as the same task.+func KindToTaskType(kind string) (string, error) {+ switch strings.ToLower(strings.TrimSpace(kind)) {+ case "wiki":+ return TaskTypeWiki, nil+ case "tree":+ return TaskTypeTree, nil+ case "knowledge_graph", "knowledgegraph", "graph":+ return TaskTypeGraph, nil+ case "mind_map", "mindmap":+ return TaskTypeMindmap, nil+ case "timeline":+ return TaskTypeTimeline, nil+ case "page_index":+ return TaskTypePageIndex, nil+ default:+ return "", fmt.Errorf("%w: %q", ErrUnknownVariant, kind)+ }+}++// VariantToTaskType provides the best category available when reading legacy+// rows that do not retain the original template kind.+func VariantToTaskType(variant Variant) string {+ switch variant {+ case VariantWiki:+ return TaskTypeWiki+ case VariantTree:+ return TaskTypeTree+ case VariantMindmap:+ return TaskTypeMindmap+ case VariantStructure:+ return TaskTypeGraph+ default:+ return ""+ }+}+// FirstNonEmpty returns the first string that is non-empty after trimming.func FirstNonEmpty(vals ...string) string {for _, v := range vals {diff --git a/internal/ingestion/component/knowledge_compiler/common/types_test.go b/internal/ingestion/component/knowledge_compiler/common/types_test.goindex 38e3da1742..ef3ffa1f89 100644--- a/internal/ingestion/component/knowledge_compiler/common/types_test.go+++ b/internal/ingestion/component/knowledge_compiler/common/types_test.go@@ -51,3 +51,24 @@ func TestKindToVariant_CanonicalForms(t *testing.T) {t.Errorf("structure/knowledge_graph/graph should map to the same variant, got %q %q %q", a, b, c)}}++func TestKindToTaskTypePreservesFrontendCategories(t *testing.T) {+ tests := map[string]string{+ "wiki": TaskTypeWiki,+ "tree": TaskTypeTree,+ "knowledge_graph": TaskTypeGraph,+ "mind_map": TaskTypeMindmap,+ "timeline": TaskTypeTimeline,+ "page_index": TaskTypePageIndex,+ }+ for kind, want := range tests {+ got, err := KindToTaskType(kind)+ if err != nil {+ t.Errorf("KindToTaskType(%q) unexpected error: %v", kind, err)+ continue+ }+ if got != want {+ t.Errorf("KindToTaskType(%q) = %q, want %q", kind, got, want)+ }+ }+}diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.goindex c2c7622a42..68312b11f3 100644--- a/internal/ingestion/knowledge_compile/consumer.go+++ b/internal/ingestion/knowledge_compile/consumer.go@@ -73,7 +73,7 @@ type Consumer struct {// satisfied by *mysqlScheduler (and FakeScheduler in tests). Using a narrow// interface keeps the Claimer surface unchanged.type rewriteScheduler interface {- Publish(ctx context.Context, tenantID, datasetID, docID, eventType string, variants []string) error+ Publish(ctx context.Context, tenantID, datasetID, docID, eventType string, variants, taskTypes []string) errorCancelInflight(ctx context.Context, datasetID, token string) error}@@ -170,6 +170,7 @@ func (c *Consumer) processClaim(ctx context.Context, cr ClaimResult) {if paused {return}+ logTaskTypes := taskTypesForEntries(cr.Entries)if err := startDatasetCompileLog(ctx, cr.TenantID, datasetID, cr.Token, cr.Entries); err != nil {common.Warn("knowledge_compile: failed to create dataset ingestion log",zap.String("dataset_id", datasetID), zap.Error(err))@@ -205,14 +206,14 @@ func (c *Consumer) processClaim(ctx context.Context, cr ClaimResult) {var batchErr errorgo func() {defer close(done)- batchErr = c.processBatch(ctx, cr.TenantID, datasetID, cr.Token, cr.Entries)+ batchErr = c.processBatch(ctx, cr.TenantID, datasetID, cr.Token, cr.Entries, logTaskTypes)}()select {case <-ctx.Done():close(stopHb)<-done- if err := finishDatasetCompileLog(context.WithoutCancel(ctx), cr.Token, common.STOPPED, "Wiki dataset compilation stopped during shutdown", -1); err != nil {+ if err := finishDatasetCompileLog(context.WithoutCancel(ctx), cr.Token, logTaskTypes, common.STOPPED, "Knowledge compilation stopped during shutdown", -1); err != nil {common.Warn("knowledge_compile: failed to finalize stopped dataset ingestion log",zap.String("dataset_id", datasetID), zap.Error(err))}@@ -220,7 +221,7 @@ func (c *Consumer) processClaim(ctx context.Context, cr ClaimResult) {case <-hbFailed:close(stopHb)<-done- if err := finishDatasetCompileLog(context.WithoutCancel(ctx), cr.Token, common.STOPPED, "Wiki dataset compilation lease was lost and will be retried", -1); err != nil {+ if err := finishDatasetCompileLog(context.WithoutCancel(ctx), cr.Token, logTaskTypes, common.STOPPED, "Knowledge compilation lease was lost and will be retried", -1); err != nil {common.Warn("knowledge_compile: failed to finalize stopped dataset ingestion log",zap.String("dataset_id", datasetID), zap.Error(err))}@@ -244,7 +245,7 @@ func (c *Consumer) processClaim(ctx context.Context, cr ClaimResult) {common.Warn("knowledge_compile: failed to record error_msg",zap.String("dataset_id", datasetID), zap.Error(err))}- if err := finishDatasetCompileLog(ctx, cr.Token, common.FAILED, batchErr.Error(), -1); err != nil {+ if err := finishDatasetCompileLog(ctx, cr.Token, logTaskTypes, common.FAILED, batchErr.Error(), -1); err != nil {common.Warn("knowledge_compile: failed to finalize dataset ingestion log",zap.String("dataset_id", datasetID), zap.Error(err))}@@ -253,26 +254,26 @@ func (c *Consumer) processClaim(ctx context.Context, cr ClaimResult) {if _, err := c.scheduler.Ack(ctx, datasetID, cr.Token, cr.Entries); err != nil {common.Warn("knowledge_compile: ack failed",zap.String("dataset_id", datasetID), zap.Error(err))- if logErr := finishDatasetCompileLog(ctx, cr.Token, common.FAILED, "Failed to acknowledge completed Wiki compilation: "+err.Error(), -1); logErr != nil {+ if logErr := finishDatasetCompileLog(ctx, cr.Token, logTaskTypes, common.FAILED, "Failed to acknowledge completed knowledge compilation: "+err.Error(), -1); logErr != nil {common.Warn("knowledge_compile: failed to finalize dataset ingestion log",zap.String("dataset_id", datasetID), zap.Error(logErr))}return}- if err := finishDatasetCompileLog(ctx, cr.Token, common.COMPLETED, "Wiki dataset compilation completed", 1); err != nil {+ if err := finishDatasetCompileLog(ctx, cr.Token, logTaskTypes, common.COMPLETED, "Knowledge compilation completed", 1); err != nil {common.Warn("knowledge_compile: failed to finalize dataset ingestion log",zap.String("dataset_id", datasetID), zap.Error(err))}}-func (c *Consumer) reportProgress(ctx context.Context, tenant, datasetID, token string, progress float64, phase, message string) {+func (c *Consumer) reportProgress(ctx context.Context, tenant, datasetID, token string, taskTypes []string, progress float64, phase, message string) {if err := c.scheduler.UpdateProgress(ctx, datasetID, token, progress, phase, message); err != nil {common.Warn("knowledge_compile: failed to update progress",zap.String("dataset_id", datasetID),zap.String("phase", phase),zap.Error(err))}- if err := updateDatasetCompileLog(ctx, token, progress, message); err != nil {+ if err := updateDatasetCompileLog(ctx, token, taskTypes, progress, message); err != nil {common.Warn("knowledge_compile: failed to update dataset ingestion log",zap.String("dataset_id", datasetID), zap.String("phase", phase), zap.Error(err))}@@ -382,7 +383,7 @@ func (c *Consumer) RebuildDataset(ctx context.Context, tenant, kb, mode string)return fmt.Errorf("knowledge_compile: rebuild list docs: %w", err)}for _, docID := range docs {- variants, rerr := c.recoverDocVariants(ctx, tenant, kb, docID)+ variants, taskTypes, rerr := c.recoverDocTypes(ctx, tenant, kb, docID)if rerr != nil {// O2a hard failure: abort the rebuild so an unknown template kind// cannot leave the doc's dataset-level products unrebuilt.@@ -394,7 +395,7 @@ func (c *Consumer) RebuildDataset(ctx context.Context, tenant, kb, mode string)// event that falls back to the legacy all-products path.continue}- if perr := rs.Publish(ctx, tenant, kb, docID, string(EventTypeCompleted), variants); perr != nil {+ if perr := rs.Publish(ctx, tenant, kb, docID, string(EventTypeCompleted), variants, taskTypes); perr != nil {return fmt.Errorf("knowledge_compile: rebuild republish %s: %w", docID, perr)}}@@ -417,9 +418,10 @@ func defaultDocLister(ctx context.Context, tenant, kb string) ([]string, error)return dao.NewDocumentDAO().ListIDsByKBIDWithOptions(ctx, kcDB, dao.DocumentListOptions{KbID: kb})}-// recoverDocVariants reconstructs the compile-type set a document produced, by-// reading its persisted doc-level products and mapping each product's-// authoritative `compilation_template_kind_kwd` through common.KindToVariant.+// recoverDocTypes reconstructs the compile-type and frontend task-type sets a+// document produced by reading its persisted doc-level products and mapping+// each product's authoritative `compilation_template_kind_kwd` through the+// shared kind mappings.// It is used by RebuildDataset (B1) so the republished completed events carry// the variants needed to route the dataset-level re-compile (tree nav, structure// merge, wiki merge) — a nil/empty Variants would fall back to the legacy@@ -431,27 +433,34 @@ func defaultDocLister(ctx context.Context, tenant, kb string) ([]string, error)// variant and leave the doc's dataset-level products unrebuilt. A product// without an authoritative kind falls back to its reverse-mapped variant (which// itself must be whitelist-valid, enforced by KwdToVariant in the Reader). The-// returned slice is sorted and de-duplicated.-func (c *Consumer) recoverDocVariants(ctx context.Context, tenant, kb, docID string) ([]string, error) {+// Both returned slices are sorted and de-duplicated.+func (c *Consumer) recoverDocTypes(ctx context.Context, tenant, kb, docID string) ([]string, []string, error) {products, err := c.reader.LoadDocProducts(ctx, tenant, kb, docID)if err != nil {- return nil, fmt.Errorf("knowledge_compile: recover variants load doc %s: %w", docID, err)+ return nil, nil, fmt.Errorf("knowledge_compile: recover variants load doc %s: %w", docID, err)}- seen := map[string]struct{}{}- var out []string+ seenVariants := map[string]struct{}{}+ seenTaskTypes := map[string]struct{}{}+ var variants []stringfor _, p := range products {// Authoritative: product.Kind (compilation_template_kind_kwd) when present.if p.Kind != "" {v, err := kccommon.KindToVariant(p.Kind)if err != nil {// O2a hard failure: do not continue with an incomplete variant set.- return nil, fmt.Errorf("knowledge_compile: recover variants doc %s kind %q: %w", docID, p.Kind, err)+ return nil, nil, fmt.Errorf("knowledge_compile: recover variants doc %s kind %q: %w", docID, p.Kind, err)}- if _, dup := seen[string(v)]; dup {- continue+ if _, dup := seenVariants[string(v)]; !dup {+ seenVariants[string(v)] = struct{}{}+ variants = append(variants, string(v))+ }+ taskType, taskTypeErr := kccommon.KindToTaskType(p.Kind)+ if taskTypeErr != nil {+ taskType = kccommon.VariantToTaskType(v)+ }+ if taskType != "" {+ seenTaskTypes[taskType] = struct{}{}}- seen[string(v)] = struct{}{}- out = append(out, string(v))continue}// Fallback: the product's reverse-mapped variant (already whitelist-valid).@@ -461,14 +470,17 @@ func (c *Consumer) recoverDocVariants(ctx context.Context, tenant, kb, docID str// (which the consumer cannot route and defeats "nil = legacy").continue}- if _, dup := seen[string(p.Variant)]; dup {- continue+ if _, dup := seenVariants[string(p.Variant)]; !dup {+ seenVariants[string(p.Variant)] = struct{}{}+ variants = append(variants, string(p.Variant))+ }+ if taskType := kccommon.VariantToTaskType(p.Variant); taskType != "" {+ seenTaskTypes[taskType] = struct{}{}}- seen[string(p.Variant)] = struct{}{}- out = append(out, string(p.Variant))}- sort.Strings(out)- return out, nil+ sort.Strings(variants)+ taskTypes := sortedTaskTypes(seenTaskTypes)+ return variants, taskTypes, nil}// filterWikiPageCandidates returns ONLY the wiki page products the dataset-level@@ -498,8 +510,11 @@ func filterWikiPageCandidates(candidates []kccommon.Product) []kccommon.Product// writes the dataset-level merged products for the claimed closed batch. It// returns an error if any reader/dedup/writer step fails so the caller can// leave the batch for reclamation instead of acking dropped work.-func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, entries []BacklogEntry) error {- c.reportProgress(ctx, tenant, kb, token, 0.05, "classifying", fmt.Sprintf("Classifying %d scheduling entries", len(entries)))+func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, entries []BacklogEntry, taskTypes []string) error {+ if len(taskTypes) == 0 {+ taskTypes = taskTypesForEntries(entries)+ }+ c.reportProgress(ctx, tenant, kb, token, taskTypes, 0.05, "classifying", fmt.Sprintf("Classifying %d scheduling entries", len(entries)))common.Info("knowledge_compile: processing claimed batch",zap.String("dataset_id", kb),zap.String("tenant_id", tenant),@@ -589,19 +604,24 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, ezap.Strings("entries", entryDetails))if len(deleted) == 0 && len(disabled) == 0 && len(completed) == 0 {- c.reportProgress(ctx, tenant, kb, token, 1, "completed", "No document products require merging")+ c.reportProgress(ctx, tenant, kb, token, taskTypes, 1, "completed", "No document products require merging")return nil}deletedSet := make(map[string]bool, len(deleted))for _, d := range deleted {deletedSet[d] = true}- wikiDiff, err := c.prepareWikiContributionDiff(ctx, tenant, kb, completed, deleted)+ retractedDocIDs := append([]string(nil), deleted...)+ for _, entry := range disabled {+ retractedDocIDs = append(retractedDocIDs, entry.DocID)+ }+ retractedDocIDs = uniqueSortedStrings(retractedDocIDs)+ wikiDiff, err := c.prepareWikiContributionDiff(ctx, tenant, kb, completed, retractedDocIDs)if err != nil {return err}- c.reportProgress(ctx, tenant, kb, token, 0.10, "comparing_wiki_contributions",- fmt.Sprintf("Comparing Wiki contributions: %d affected page(s)", len(wikiDiff.affectedKeys)))+ c.reportProgress(ctx, tenant, kb, token, taskTypes, 0.10, "comparing_knowledge_contributions",+ fmt.Sprintf("Comparing knowledge contributions: %d affected page(s)", len(wikiDiff.affectedKeys)))wikiOnly := len(completed) > 0for _, entry := range completed {if len(entry.Variants) != 1 || !variantsContain(entry.Variants, "wiki") {@@ -609,17 +629,17 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, ebreak}}- if len(deleted) == 0 && wikiOnly && len(wikiDiff.affectedKeys) == 0 {+ if len(retractedDocIDs) == 0 && wikiOnly && len(wikiDiff.affectedKeys) == 0 {c.mu.Lock()for _, docID := range pendingTombClear {delete(c.tombs[kb], docID)}c.mu.Unlock()- c.reportProgress(ctx, tenant, kb, token, 1, "completed", "Wiki contributions are unchanged; no dataset pages require updating")+ c.reportProgress(ctx, tenant, kb, token, taskTypes, 1, "completed", "Knowledge contributions are unchanged; no dataset pages require updating")return nil}- c.reportProgress(ctx, tenant, kb, token, 0.12, "deleting", fmt.Sprintf("Deleting products for %d document(s)", len(deleted)))+ c.reportProgress(ctx, tenant, kb, token, taskTypes, 0.12, "deleting", fmt.Sprintf("Deleting products for %d document(s)", len(deleted)))// --- Retraction (sequential DocEngine calls, no in-memory load) ---// Deleted and disabled docs are handled before completion merges. Their@@ -635,14 +655,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e// A merged product referencing several deleted docs is pruned in one pass,// and an emptied product is removed exactly once.if len(deleted) > 0 || len(disabled) > 0 {- delIDs := make([]string, 0, len(deletedSet)+len(disabled))- for d := range deletedSet {- delIDs = append(delIDs, d)- }- for _, entry := range disabled {- delIDs = append(delIDs, entry.DocID)- }- delIDs = uniqueSortedStrings(delIDs)+ delIDs := retractedDocIDs// Each destructive write runs under the scheduler's per-dataset// write/rebuild lock with the generation check performed INSIDE the lock,// so a rewrite can neither land between the check and the write nor@@ -685,12 +698,15 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e}}if len(completed) == 0 && len(wikiDiff.affectedKeys) == 0 {+ if err := c.commitWikiContributions(ctx, tenant, kb, wikiDiff.currentByDoc, retractedDocIDs); err != nil {+ return err+ }c.mu.Lock()for _, docID := range pendingTombClear {delete(c.tombs[kb], docID)}c.mu.Unlock()- c.reportProgress(ctx, tenant, kb, token, 1, "completed", "Document availability changes applied")+ c.reportProgress(ctx, tenant, kb, token, taskTypes, 1, "completed", "Document availability changes applied")return nil}deduper, err := c.factory(tenant)@@ -716,7 +732,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e// completed and deleted is a stale tombstone: the deletion wins, so we skip// its completion.var incoming []kccommon.Product- c.reportProgress(ctx, tenant, kb, token, 0.28, "loading_products", fmt.Sprintf("Loading products for %d document(s)", len(completed)))+ c.reportProgress(ctx, tenant, kb, token, taskTypes, 0.28, "loading_products", fmt.Sprintf("Loading products for %d document(s)", len(completed)))for _, e := range completed {if deletedSet[e.DocID] {continue@@ -758,7 +774,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e// so both upsert their summary into the cross-document nav tree; wiki products// continue to the page-specific evidence merge.navIn := navInputFromProducts(kb, incoming)- c.reportProgress(ctx, tenant, kb, token, 0.40, "merging_navigation", fmt.Sprintf("Merging navigation products: %d", len(navIn)))+ c.reportProgress(ctx, tenant, kb, token, taskTypes, 0.40, "merging_navigation", fmt.Sprintf("Merging navigation products: %d", len(navIn)))if len(navIn) > 0 {ns := nav.GetNavService()if ns == nil {@@ -782,7 +798,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e}return err}- c.reportProgress(ctx, tenant, kb, token, 0.50, "merging_structure", fmt.Sprintf("Merging structure products: %d", len(incoming)))+ c.reportProgress(ctx, tenant, kb, token, taskTypes, 0.50, "merging_structure", fmt.Sprintf("Merging structure products: %d", len(incoming)))// wiki_incremental port (M1): the dataset-level merge only processes wiki// PAGES. A wiki doc yields both page and section products (Meta.kind@@ -793,7 +809,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e// contract. Legacy rows whose kind is empty are derived in the Reader; only// truly page-kind wiki products proceed.candidates := filterWikiPageCandidates(incoming)- c.reportProgress(ctx, tenant, kb, token, 0.54, "filtering_candidates", fmt.Sprintf("Filtering Wiki page candidates: %d", len(candidates)))+ c.reportProgress(ctx, tenant, kb, token, taskTypes, 0.54, "filtering_candidates", fmt.Sprintf("Filtering knowledge page candidates: %d", len(candidates)))entityMerged, topicCandidates, err := c.mergeEntityModeCandidates(ctx, tenant, kb, candidates)if err != nil {return err@@ -811,7 +827,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, ereturn err}}- c.reportProgress(ctx, tenant, kb, token, 0.58, "merging_by_slug", fmt.Sprintf("Merging Wiki pages by slug: %d", len(entityMerged)))+ c.reportProgress(ctx, tenant, kb, token, taskTypes, 0.58, "merging_by_slug", fmt.Sprintf("Merging knowledge pages by slug: %d", len(entityMerged)))// Entity pages have stable page identity and must bypass topic-mode KNN and// Page-specific entity/concept merging. Only topic-mode pages continue below.// The slug-only merge above has already handled every Wiki page candidate.@@ -977,7 +993,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, eif err := runCompilerJobs(ctx, jobs); err != nil {return err}- c.reportProgress(ctx, tenant, kb, token, 0.72, "routing_pages", fmt.Sprintf("Routing %d Wiki pages against existing products", len(candidates)))+ c.reportProgress(ctx, tenant, kb, token, taskTypes, 0.72, "routing_pages", fmt.Sprintf("Routing %d knowledge pages against existing products", len(candidates)))// KNN only narrows topic-page candidates. When the hit is a topic page,// require an explicit topic match or an LLM topic-route decision before@@ -1033,7 +1049,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, evar newMerged []kccommon.ProductnewMerged = append(newMerged, entityMerged...)if len(groupsByID) > 0 {- c.reportProgress(ctx, tenant, kb, token, 0.82, "llm_merge", fmt.Sprintf("Merging %d candidate groups with the LLM", len(groupsByID)))+ c.reportProgress(ctx, tenant, kb, token, taskTypes, 0.82, "llm_merge", fmt.Sprintf("Merging %d candidate groups with the LLM", len(groupsByID)))// Diagnostics: summarize the KNN groups before the LLM merge decision so// the reader can see which existing merged rows were hit and by how many// candidates (e.g. whether a "吕布" candidate hit an existing 吕布 row and@@ -1085,7 +1101,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, emergedFinal = append(mergedFinal, unmatched...)mergedFinal, staleTopicIDs := mergeTopicProducts(tenant, kb, mergedFinal)mergedFinal = refreshWikiProductVectors(ctx, tenant, mergedFinal)- c.reportProgress(ctx, tenant, kb, token, 0.90, "writing_products", fmt.Sprintf("Writing %d merged products", len(mergedFinal)))+ c.reportProgress(ctx, tenant, kb, token, taskTypes, 0.90, "writing_products", fmt.Sprintf("Writing %d merged products", len(mergedFinal)))if err := c.withWriteLock(ctx, kb, token, func() error {if err := c.writer.WriteMerged(ctx, tenant, kb, mergedFinal); err != nil {return err@@ -1116,7 +1132,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e// then just drops any stale graph rows). The graph is reconstructible, so the// cost of an occasional no-op reprojection is accepted. It is also a locked// destructive side effect for the same TOCTOU reasons as WriteMerged.- c.reportProgress(ctx, tenant, kb, token, 0.97, "projecting_graph", "Projecting the Wiki graph")+ c.reportProgress(ctx, tenant, kb, token, taskTypes, 0.97, "projecting_graph", "Projecting the knowledge graph")if err := c.withWriteLock(ctx, kb, token, func() error {return c.writer.ProjectWikiGraph(ctx, tenant, kb)}); err != nil {@@ -1126,7 +1142,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e}return err}- if err := c.commitWikiContributions(ctx, tenant, kb, wikiDiff.currentByDoc, deleted); err != nil {+ if err := c.commitWikiContributions(ctx, tenant, kb, wikiDiff.currentByDoc, retractedDocIDs); err != nil {return err}@@ -1143,7 +1159,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, ezap.Int("completed_docs", len(completed)),zap.Int("deleted_docs", len(deleted)),zap.Int("merged_rows_written", len(mergedFinal)))- c.reportProgress(ctx, tenant, kb, token, 1, "completed", fmt.Sprintf("Wiki compilation completed: %d merged products", len(mergedFinal)))+ c.reportProgress(ctx, tenant, kb, token, taskTypes, 1, "completed", fmt.Sprintf("Knowledge compilation completed: %d merged products", len(mergedFinal)))return nil}@@ -1446,7 +1462,10 @@ func productsForVariants(products []kccommon.Product, variants []string) []kccom// summary itself when Embedd is empty, so the consumer needs no embedder.func navInputFromProducts(kb string, products []kccommon.Product) []nav.UpsertDocInput {type acc struct {- in nav.UpsertDocInput+ in nav.UpsertDocInput+ treeSummary string+ treeEmbedd []float32+ structureLines []string}byDoc := make(map[string]*acc, len(products))for i := range products {@@ -1461,8 +1480,8 @@ func navInputFromProducts(kb string, products []kccommon.Product) []nav.UpsertDobyDoc[p.DocID] = &acc{in: nav.UpsertDocInput{TenantID: p.TenantID, KbID: kb, DocID: p.DocID}}a = byDoc[p.DocID]}- a.in.Summary = p.Content- a.in.Embedd = p.Vector+ a.treeSummary = p.Content+ a.treeEmbedd = p.Vectorcase kccommon.VariantStructure:// Python's page_index path rebuilds the doc graph from the stored// entity rows and folds their descriptions into the nav summary@@ -1481,14 +1500,20 @@ func navInputFromProducts(kb string, products []kccommon.Product) []nav.UpsertDobyDoc[p.DocID] = &acc{in: nav.UpsertDocInput{TenantID: p.TenantID, KbID: kb, DocID: p.DocID}}a = byDoc[p.DocID]}- if a.in.Summary != "" {- a.in.Summary += "\n"- }- a.in.Summary += line+ a.structureLines = append(a.structureLines, line)}}out := make([]nav.UpsertDocInput, 0, len(byDoc))for _, a := range byDoc {+ // Tree is the canonical navigation summary when both variants are present;+ // otherwise use the folded structure/page-index summary. This makes the+ // result independent of product iteration order.+ if strings.TrimSpace(a.treeSummary) != "" {+ a.in.Summary = a.treeSummary+ a.in.Embedd = a.treeEmbedd+ } else {+ a.in.Summary = strings.Join(a.structureLines, "\n")+ }if strings.TrimSpace(a.in.Summary) == "" {continue}diff --git a/internal/ingestion/knowledge_compile/dataset_log.go b/internal/ingestion/knowledge_compile/dataset_log.goindex 719a403d8f..b79fab49c3 100644--- a/internal/ingestion/knowledge_compile/dataset_log.go+++ b/internal/ingestion/knowledge_compile/dataset_log.go@@ -20,14 +20,32 @@ import ("crypto/sha256""encoding/hex""fmt"+ "sort"+ "strings""time""ragflow/internal/common""ragflow/internal/entity"+ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common")const datasetLogDocumentID = "graph_raptor_x"+var allDatasetTaskTypes = []string{+ kccommon.TaskTypeWiki,+ kccommon.TaskTypeTree,+ kccommon.TaskTypeGraph,+ kccommon.TaskTypeMindmap,+ kccommon.TaskTypeTimeline,+ kccommon.TaskTypePageIndex,+}++var structureTaskTypes = []string{+ kccommon.TaskTypeGraph,+ kccommon.TaskTypeTimeline,+ kccommon.TaskTypePageIndex,+}+// Dataset ingestion logs use the status values exposed by the Python API and// consumed by the shared frontend. The Go scheduler uses a different set of// internal terminal names, so translate them at the persistence boundary.@@ -44,52 +62,167 @@ func datasetLogOperationStatus(status string) string {}}-func datasetCompileLogID(claimToken string) string {- sum := sha256.Sum256([]byte("wiki-dataset-log\x00" + claimToken))+func datasetCompileLogID(claimToken, taskType string) string {+ sum := sha256.Sum256([]byte("knowledge-compile-dataset-log\x00" + claimToken + "\x00" + taskType))return hex.EncodeToString(sum[:16])}+func taskTypesForVariants(variants []string) []string {+ seen := make(map[string]struct{}, len(variants))+ for _, variant := range variants {+ var taskType string+ switch strings.ToLower(strings.TrimSpace(variant)) {+ case "wiki":+ taskType = kccommon.TaskTypeWiki+ case "tree":+ taskType = kccommon.TaskTypeTree+ case "mindmap", "mind_map":+ taskType = kccommon.TaskTypeMindmap+ case "structure":+ // Legacy structure events predate the task-type field and cannot+ // distinguish graph, timeline, and page-index products.+ for _, structureTaskType := range structureTaskTypes {+ seen[structureTaskType] = struct{}{}+ }+ }+ if taskType != "" {+ seen[taskType] = struct{}{}+ }+ }+ return sortedTaskTypes(seen)+}++func taskTypesForEntry(entry BacklogEntry) []string {+ seen := make(map[string]struct{}, len(entry.TaskTypes))+ for _, taskType := range entry.TaskTypes {+ if taskType = normalizeTaskType(taskType); taskType != "" {+ seen[taskType] = struct{}{}+ }+ }+ if len(seen) > 0 {+ return sortedTaskTypes(seen)+ }+ if taskTypes := taskTypesForVariants(entry.Variants); len(taskTypes) > 0 {+ return taskTypes+ }+ // An event with no routing metadata must remain visible in every possible+ // dataset log. Returning the fallback per entry prevents an unknown entry+ // from disappearing when other entries in the same batch are typed.+ return append([]string(nil), allDatasetTaskTypes...)+}++func normalizeTaskType(taskType string) string {+ switch strings.ToLower(strings.TrimSpace(taskType)) {+ case "wiki":+ return kccommon.TaskTypeWiki+ case "tree":+ return kccommon.TaskTypeTree+ case "graph", "knowledge_graph", "knowledgegraph":+ return kccommon.TaskTypeGraph+ case "mindmap", "mind_map":+ return kccommon.TaskTypeMindmap+ case "timeline":+ return kccommon.TaskTypeTimeline+ case "pageindex", "page_index":+ return kccommon.TaskTypePageIndex+ default:+ return ""+ }+}++func taskTypesForEntries(entries []BacklogEntry) []string {+ seen := make(map[string]struct{})+ for _, entry := range entries {+ for _, taskType := range taskTypesForEntry(entry) {+ seen[taskType] = struct{}{}+ }+ }+ if len(seen) == 0 {+ // Legacy events did not carry routing metadata. A deletion can affect any+ // dataset artifact, so expose one log per supported category rather than+ // incorrectly labeling the event as Wiki.+ for _, taskType := range allDatasetTaskTypes {+ seen[taskType] = struct{}{}+ }+ }+ return sortedTaskTypes(seen)+}++func sortedTaskTypes(seen map[string]struct{}) []string {+ taskTypes := make([]string, 0, len(seen))+ for taskType := range seen {+ taskTypes = append(taskTypes, taskType)+ }+ sort.Strings(taskTypes)+ return taskTypes+}+func startDatasetCompileLog(ctx context.Context, tenantID, datasetID, claimToken string, entries []BacklogEntry) error {if kcDB == nil || claimToken == "" {return nil}now := time.Now()status := "1"- message := timestampProgressMessage(fmt.Sprintf("Created automatic Wiki dataset task for %d document event(s)", len(entries)))- entryData := make([]any, 0, len(entries))+ groups := make(map[string][]BacklogEntry)for _, entry := range entries {- entryData = append(entryData, map[string]any{- "doc_id": entry.DocID,- "event_type": entry.EventType,- "variants": entry.Variants,- })- }- log := entity.PipelineOperationLog{- ID: datasetCompileLogID(claimToken),- DocumentID: datasetLogDocumentID,- TenantID: tenantID,- KbID: datasetID,- ParserID: "wiki",- DocumentName: "Wiki",- DocumentSuffix: "",- DocumentType: "dataset",- SourceFrom: "knowledgebase",- Progress: 0,- ProgressMsg: &message,- ProcessBeginAt: &now,- DSL: entity.JSONMap{"entries": entryData},- TaskType: string(entity.PipelineTaskTypeWiki),- OperationStatus: common.RUNNING,- Status: &status,- }- return kcDB.WithContext(ctx).Where("id = ?", log.ID).FirstOrCreate(&log).Error+ for _, taskType := range taskTypesForEntry(entry) {+ groups[taskType] = append(groups[taskType], entry)+ }+ }+ if len(groups) == 0 {+ for _, taskType := range allDatasetTaskTypes {+ groups[taskType] = entries+ }+ }+ for _, taskType := range sortedTaskTypesFromGroups(groups) {+ entryData := make([]any, 0, len(groups[taskType]))+ for _, entry := range groups[taskType] {+ entryData = append(entryData, map[string]any{+ "doc_id": entry.DocID,+ "event_type": entry.EventType,+ "variants": entry.Variants,+ "task_type": taskType,+ })+ }+ message := timestampProgressMessage(fmt.Sprintf("Created automatic %s dataset task for %d document event(s)", taskType, len(groups[taskType])))+ log := entity.PipelineOperationLog{+ ID: datasetCompileLogID(claimToken, taskType),+ DocumentID: datasetLogDocumentID,+ TenantID: tenantID,+ KbID: datasetID,+ ParserID: "knowledge_compile",+ DocumentName: taskType,+ DocumentSuffix: "",+ DocumentType: "dataset",+ SourceFrom: "knowledgebase",+ Progress: 0,+ ProgressMsg: &message,+ ProcessBeginAt: &now,+ DSL: entity.JSONMap{"entries": entryData, "task_type": taskType},+ TaskType: taskType,+ OperationStatus: common.RUNNING,+ Status: &status,+ }+ if err := kcDB.WithContext(ctx).Where("id = ?", log.ID).FirstOrCreate(&log).Error; err != nil {+ return err+ }+ }+ return nil}-func updateDatasetCompileLog(ctx context.Context, claimToken string, progress float64, message string) error {+func updateDatasetCompileLog(ctx context.Context, claimToken string, taskTypes []string, progress float64, message string) error {if kcDB == nil || claimToken == "" {return nil}- logID := datasetCompileLogID(claimToken)+ for _, taskType := range taskTypes {+ if err := updateDatasetCompileLogForType(ctx, datasetCompileLogID(claimToken, taskType), progress, message); err != nil {+ return err+ }+ }+ return nil+}++func updateDatasetCompileLogForType(ctx context.Context, logID string, progress float64, message string) error {var log entity.PipelineOperationLogif err := kcDB.WithContext(ctx).Where("id = ?", logID).First(&log).Error; err != nil {return err@@ -111,28 +244,41 @@ func updateDatasetCompileLog(ctx context.Context, claimToken string, progress fl}).Error}-func finishDatasetCompileLog(ctx context.Context, claimToken, operationStatus, message string, progress float64) error {+func finishDatasetCompileLog(ctx context.Context, claimToken string, taskTypes []string, operationStatus, message string, progress float64) error {if kcDB == nil || claimToken == "" {return nil}- logID := datasetCompileLogID(claimToken)- var log entity.PipelineOperationLog- if err := kcDB.WithContext(ctx).Where("id = ?", logID).First(&log).Error; err != nil {- return err- }- progressMessage := ""- if log.ProgressMsg != nil {- progressMessage = *log.ProgressMsg+ for _, taskType := range taskTypes {+ logID := datasetCompileLogID(claimToken, taskType)+ var log entity.PipelineOperationLog+ if err := kcDB.WithContext(ctx).Where("id = ?", logID).First(&log).Error; err != nil {+ return err+ }+ progressMessage := ""+ if log.ProgressMsg != nil {+ progressMessage = *log.ProgressMsg+ }+ progressMessage = appendProgressMessage(progressMessage, timestampProgressMessage(message))+ duration := log.ProcessDuration+ if log.ProcessBeginAt != nil {+ duration = max(0, time.Since(*log.ProcessBeginAt).Seconds())+ }+ if err := kcDB.WithContext(ctx).Model(&entity.PipelineOperationLog{}).Where("id = ?", logID).Updates(map[string]any{+ "progress": progress,+ "progress_msg": progressMessage,+ "process_duration": duration,+ "operation_status": datasetLogOperationStatus(operationStatus),+ }).Error; err != nil {+ return err+ }}- progressMessage = appendProgressMessage(progressMessage, timestampProgressMessage(message))- duration := log.ProcessDuration- if log.ProcessBeginAt != nil {- duration = max(0, time.Since(*log.ProcessBeginAt).Seconds())+ return nil+}++func sortedTaskTypesFromGroups(groups map[string][]BacklogEntry) []string {+ seen := make(map[string]struct{}, len(groups))+ for taskType := range groups {+ seen[taskType] = struct{}{}}- return kcDB.WithContext(ctx).Model(&entity.PipelineOperationLog{}).Where("id = ?", logID).Updates(map[string]any{- "progress": progress,- "progress_msg": progressMessage,- "process_duration": duration,- "operation_status": datasetLogOperationStatus(operationStatus),- }).Error+ return sortedTaskTypes(seen)}diff --git a/internal/ingestion/knowledge_compile/dataset_log_test.go b/internal/ingestion/knowledge_compile/dataset_log_test.goindex 072639b34e..a600fe619e 100644--- a/internal/ingestion/knowledge_compile/dataset_log_test.go+++ b/internal/ingestion/knowledge_compile/dataset_log_test.go@@ -1,6 +1,7 @@package knowledge_compileimport (+ "reflect""strings""testing"@@ -9,6 +10,7 @@ import ("ragflow/internal/common""ragflow/internal/entity"+ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common")func TestDatasetCompileLogLifecycle(t *testing.T) {@@ -23,28 +25,56 @@ func TestDatasetCompileLogLifecycle(t *testing.T) {kcDB = dbt.Cleanup(func() { kcDB = oldDB })- entries := []BacklogEntry{{DocID: "doc-1", EventType: string(EventTypeCompleted), Variants: []string{"wiki"}}}+ entries := []BacklogEntry{+ {DocID: "doc-1", EventType: string(EventTypeCompleted), Variants: []string{"wiki"}, TaskTypes: []string{kccommon.TaskTypeWiki}},+ {DocID: "doc-2", EventType: string(EventTypeCompleted), Variants: []string{"structure"}, TaskTypes: []string{kccommon.TaskTypeGraph}},+ }if err := startDatasetCompileLog(t.Context(), "tenant-1", "kb-1", "claim-1", entries); err != nil {t.Fatalf("start dataset log: %v", err)}- if err := updateDatasetCompileLog(t.Context(), "claim-1", 0.5, "Comparing Wiki contributions: 2 affected page(s)"); err != nil {+ if err := updateDatasetCompileLog(t.Context(), "claim-1", []string{kccommon.TaskTypeWiki, kccommon.TaskTypeGraph}, 0.5, "Comparing knowledge contributions: 2 affected page(s)"); err != nil {t.Fatalf("update dataset log: %v", err)}- if err := finishDatasetCompileLog(t.Context(), "claim-1", common.COMPLETED, "Wiki dataset compilation completed", 1); err != nil {+ if err := finishDatasetCompileLog(t.Context(), "claim-1", []string{kccommon.TaskTypeWiki, kccommon.TaskTypeGraph}, common.COMPLETED, "Knowledge compilation completed", 1); err != nil {t.Fatalf("finish dataset log: %v", err)}- var log entity.PipelineOperationLog- if err := db.Where("id = ?", datasetCompileLogID("claim-1")).First(&log).Error; err != nil {- t.Fatalf("load dataset log: %v", err)+ var logs []entity.PipelineOperationLog+ if err := db.Order("task_type").Find(&logs).Error; err != nil {+ t.Fatalf("load dataset logs: %v", err)}- if log.DocumentID != datasetLogDocumentID || log.TaskType != string(entity.PipelineTaskTypeWiki) {- t.Fatalf("unexpected dataset log identity: %+v", log)+ if len(logs) != 2 {+ t.Fatalf("want one log per task type, got %d: %+v", len(logs), logs)}- if log.OperationStatus != "DONE" || log.Progress != 1 {- t.Fatalf("unexpected final state: status=%s progress=%v", log.OperationStatus, log.Progress)+ for _, log := range logs {+ if log.DocumentID != datasetLogDocumentID || log.OperationStatus != "DONE" || log.Progress != 1 {+ t.Fatalf("unexpected dataset log state: %+v", log)+ }+ if log.ID != datasetCompileLogID("claim-1", log.TaskType) {+ t.Fatalf("dataset log id is not task-type scoped: %+v", log)+ }+ if log.ProgressMsg == nil || !strings.Contains(*log.ProgressMsg, "2 affected page(s)") || !strings.Contains(*log.ProgressMsg, "completed") {+ t.Fatalf("progress messages were not persisted: %v", log.ProgressMsg)+ }+ if log.DSL["task_type"] != log.TaskType {+ t.Fatalf("dataset log DSL task type = %v, want %s", log.DSL["task_type"], log.TaskType)+ }+ entries, ok := log.DSL["entries"].([]any)+ if !ok || len(entries) != 1 {+ t.Fatalf("dataset log entries were not split by task type: %+v", log.DSL["entries"])+ }}- if log.ProgressMsg == nil || !strings.Contains(*log.ProgressMsg, "2 affected page(s)") || !strings.Contains(*log.ProgressMsg, "completed") {- t.Fatalf("progress messages were not persisted: %v", log.ProgressMsg)+}++func TestTaskTypesForEntryLegacyFallbacks(t *testing.T) {+ got := taskTypesForEntry(BacklogEntry{Variants: []string{"structure"}})+ want := []string{kccommon.TaskTypeGraph, kccommon.TaskTypePageIndex, kccommon.TaskTypeTimeline}+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("legacy structure task types = %v, want %v", got, want)+ }++ got = taskTypesForEntry(BacklogEntry{})+ if !reflect.DeepEqual(got, allDatasetTaskTypes) {+ t.Fatalf("unknown entry task types = %v, want %v", got, allDatasetTaskTypes)}}diff --git a/internal/ingestion/knowledge_compile/event.go b/internal/ingestion/knowledge_compile/event.goindex bdc707b7f2..affe050870 100644--- a/internal/ingestion/knowledge_compile/event.go+++ b/internal/ingestion/knowledge_compile/event.go@@ -110,37 +110,38 @@ func DefaultClaimer() Claimer { return defaultClaimer }// doc so the consumer can dispatch to the matching dataset-level path. It is a// no-op when no Publisher has been installed (e.g. DB unavailable). A failure is// returned so callers can log but never fail the pipeline on it.-func PublishCompleted(ctx context.Context, tenantID, datasetID, docID string, variants []string) error {+func PublishCompleted(ctx context.Context, tenantID, datasetID, docID string, variants, taskTypes []string) error {if defaultPublisher == nil {return nil}- return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeCompleted), variants)+ return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeCompleted), variants, taskTypes)}// PublishDeleted records a doc_deleted event the same way (Publish handles the-// append + notify pairing). Deleted events carry no compile types.-func PublishDeleted(ctx context.Context, tenantID, datasetID, docID string) error {+// append + notify pairing). taskTypes identifies the document-level artifacts+// that were removed before the event was published.+func PublishDeleted(ctx context.Context, tenantID, datasetID, docID string, taskTypes []string) error {if defaultPublisher == nil {return nil}- return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeDeleted), nil)+ return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeDeleted), nil, taskTypes)}// PublishEnabled records that a document with compiled products became// available again and must be merged into dataset-level products.-func PublishEnabled(ctx context.Context, tenantID, datasetID, docID string, variants []string) error {+func PublishEnabled(ctx context.Context, tenantID, datasetID, docID string, variants, taskTypes []string) error {if defaultPublisher == nil {return nil}- return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeEnabled), variants)+ return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeEnabled), variants, taskTypes)}// PublishDisabled records that a document with compiled products became// unavailable. The consumer retracts only its dataset-level contributions;// document-level compiled rows remain available for a later re-enable.-func PublishDisabled(ctx context.Context, tenantID, datasetID, docID string, variants []string) error {+func PublishDisabled(ctx context.Context, tenantID, datasetID, docID string, variants, taskTypes []string) error {if defaultPublisher == nil {return nil}- return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeDisabled), variants)+ return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeDisabled), variants, taskTypes)}diff --git a/internal/ingestion/knowledge_compile/nav_dispatch_test.go b/internal/ingestion/knowledge_compile/nav_dispatch_test.goindex b1dad687e5..fa2aea0bed 100644--- a/internal/ingestion/knowledge_compile/nav_dispatch_test.go+++ b/internal/ingestion/knowledge_compile/nav_dispatch_test.go@@ -105,6 +105,28 @@ func TestNavInputFromProducts_EmptySummaryDropped(t *testing.T) {}}+func TestNavInputFromProducts_TreeSummaryWinsRegardlessOfOrder(t *testing.T) {+ products := []kccommon.Product{+ {DocID: "d1", TenantID: "t1", Variant: kccommon.VariantStructure,+ Content: `{"name":"A","description":"structure summary"}`,+ Meta: map[string]any{"kind": "entity"}},+ {DocID: "d1", TenantID: "t1", Variant: kccommon.VariantTree,+ Content: "tree summary", Vector: []float32{0.1},+ Meta: map[string]any{"kind": "root"}},+ }++ got := navInputFromProducts("kb1", products)+ if len(got) != 1 {+ t.Fatalf("want one nav input, got %d", len(got))+ }+ if got[0].Summary != "tree summary" {+ t.Fatalf("summary = %q, want tree summary", got[0].Summary)+ }+ if !reflect.DeepEqual(got[0].Embedd, []float32{0.1}) {+ t.Fatalf("embedding = %v, want tree embedding", got[0].Embedd)+ }+}+// TestProductsForVariants_GatesDispatch covers the A0-4 per-variant gate: a doc// whose event carries only ["wiki"] must NOT have its stale tree/structure// doc-level products leak into the nav/structure paths. Empty variants (legacydiff --git a/internal/ingestion/knowledge_compile/rebuild_variants_test.go b/internal/ingestion/knowledge_compile/rebuild_variants_test.goindex cef9c7b949..639719e055 100644--- a/internal/ingestion/knowledge_compile/rebuild_variants_test.go+++ b/internal/ingestion/knowledge_compile/rebuild_variants_test.go@@ -9,7 +9,7 @@ import ()// fakeReader is a Reader double that returns a canned product set for-// recoverDocVariants tests.+// recoverDocTypes tests.type fakeReader struct {products []kccommon.Product}@@ -22,7 +22,7 @@ func (f *fakeReader) SearchSimilar(context.Context, string, string, kccommon.Varreturn kccommon.Product{}, 0, nil}-// TestRecoverDocVariants_AuthoritativeKind covers B1a/O2a: the authoritative+// TestRecoverDocTypes_AuthoritativeKind covers B1a/O2a: the authoritative// Product.Kind (compilation_template_kind_kwd) is mapped through KindToVariant;// results are sorted/deduped.func TestRecoverDocVariants_AuthoritativeKind(t *testing.T) {@@ -32,17 +32,20 @@ func TestRecoverDocVariants_AuthoritativeKind(t *testing.T) {// duplicate variant, deduped{DocID: "d1", Kind: "structure"},}}}- got, err := c.recoverDocVariants(context.Background(), "t1", "kb1", "d1")+ got, taskTypes, err := c.recoverDocTypes(context.Background(), "t1", "kb1", "d1")if err != nil {- t.Fatalf("recoverDocVariants error: %v", err)+ t.Fatalf("recoverDocTypes error: %v", err)}want := []string{string(kccommon.VariantStructure), string(kccommon.VariantTree)}if !reflect.DeepEqual(got, want) {- t.Fatalf("recoverDocVariants = %v, want %v", got, want)+ t.Fatalf("recoverDocTypes variants = %v, want %v", got, want)+ }+ if !reflect.DeepEqual(taskTypes, []string{kccommon.TaskTypeGraph, kccommon.TaskTypeTree}) {+ t.Fatalf("recoverDocTypes task types = %v, want [Graph Tree]", taskTypes)}}-// TestRecoverDocVariants_UnknownKindHardFails covers O2a: a whitelist-out+// TestRecoverDocTypes_UnknownKindHardFails covers O2a: a whitelist-out// authoritative kind aborts recovery (returns an error) so the rebuild does not// proceed with an incomplete variant set.func TestRecoverDocVariants_UnknownKindHardFails(t *testing.T) {@@ -50,22 +53,25 @@ func TestRecoverDocVariants_UnknownKindHardFails(t *testing.T) {{DocID: "d1", Kind: "structure"},{DocID: "d1", Kind: "garbage"},}}}- if _, err := c.recoverDocVariants(context.Background(), "t1", "kb1", "d1"); err == nil {- t.Fatal("recoverDocVariants must hard-fail on an unknown authoritative kind (O2a)")+ if _, _, err := c.recoverDocTypes(context.Background(), "t1", "kb1", "d1"); err == nil {+ t.Fatal("recoverDocTypes must hard-fail on an unknown authoritative kind (O2a)")}}-// TestRecoverDocVariants_FallbackVariant covers B1a: a product without an+// TestRecoverDocTypes_FallbackVariant covers B1a: a product without an// authoritative kind falls back to its reverse-mapped variant.func TestRecoverDocVariants_FallbackVariant(t *testing.T) {c := &Consumer{reader: &fakeReader{products: []kccommon.Product{{DocID: "d1", Variant: kccommon.VariantWiki},}}}- got, err := c.recoverDocVariants(context.Background(), "t1", "kb1", "d1")+ got, taskTypes, err := c.recoverDocTypes(context.Background(), "t1", "kb1", "d1")if err != nil {- t.Fatalf("recoverDocVariants error: %v", err)+ t.Fatalf("recoverDocTypes error: %v", err)}if len(got) != 1 || got[0] != string(kccommon.VariantWiki) {- t.Fatalf("recoverDocVariants fallback = %v, want [wiki]", got)+ t.Fatalf("recoverDocTypes fallback = %v, want [wiki]", got)+ }+ if !reflect.DeepEqual(taskTypes, []string{kccommon.TaskTypeWiki}) {+ t.Fatalf("recoverDocTypes fallback task types = %v, want [Wiki]", taskTypes)}}diff --git a/internal/ingestion/knowledge_compile/scheduler.go b/internal/ingestion/knowledge_compile/scheduler.goindex 76430c9af8..a65149ffbe 100644--- a/internal/ingestion/knowledge_compile/scheduler.go+++ b/internal/ingestion/knowledge_compile/scheduler.go@@ -61,13 +61,16 @@ const (// Variants records the compile types (tree/structure/wiki/mindmap) produced by// the doc-level compile for this doc, so the consumer can group a claimed batch// into per-variant sub-batches and dispatch each to its own dataset-level-// compile path. It is empty (nil) for deleted events and for events published-// before this field existed; the consumer treats a nil/empty Variants as-// "unknown/legacy" and falls back to the legacy unified path.+// compile path. TaskTypes preserves the original frontend-facing category for+// each template, so structure sub-kinds such as Graph, Timeline, and PageIndex+// can be logged separately. Both fields are empty (nil) for deleted events and+// for events published before they existed; the consumer falls back to the+// legacy unified path when the fields are absent.type BacklogEntry struct {DocID string `json:"doc_id"`EventType string `json:"event_type"`Variants []string `json:"variants,omitempty"`+ TaskTypes []string `json:"task_types,omitempty"`}// ClaimResult is returned by Scheduler.Claim: the KB's tenant plus the closed@@ -92,8 +95,9 @@ type Publisher interface {// idle workers. It is transactional so concurrent publishers do not clobber// each other's backlog, and it always pairs the append with a notify.// variants carries the doc-level compile types (tree/structure/wiki/mindmap)- // for completed events; it is nil for deleted events.- Publish(ctx context.Context, tenantID, datasetID, docID, eventType string, variants []string) error+ // for completed events; taskTypes carries the frontend-facing categories for+ // those templates. Both are nil for deleted events.+ Publish(ctx context.Context, tenantID, datasetID, docID, eventType string, variants, taskTypes []string) error}// Claimer is the consumer-side role (Option E §11.5). A cluster of competing@@ -216,8 +220,8 @@ func (s *mysqlScheduler) Provision(ctx context.Context) error {// workers. The append is transactional (concurrent publishers do not clobber// each other's backlog), and the notify is always paired with the append so a// producer never needs a separate Notify call.-func (s *mysqlScheduler) Publish(ctx context.Context, tenantID, datasetID, docID, eventType string, variants []string) error {- return s.publishEntry(ctx, tenantID, datasetID, BacklogEntry{DocID: docID, EventType: eventType, Variants: variants})+func (s *mysqlScheduler) Publish(ctx context.Context, tenantID, datasetID, docID, eventType string, variants, taskTypes []string) error {+ return s.publishEntry(ctx, tenantID, datasetID, BacklogEntry{DocID: docID, EventType: eventType, Variants: variants, TaskTypes: taskTypes})}// loadRow fetches the dataset scheduling row (no lock). Returns nil (no error)@@ -233,8 +237,8 @@ func (s *mysqlScheduler) loadRow(ctx context.Context, datasetID string) (*entityreturn &row, nil}-func (s *mysqlScheduler) publish(ctx context.Context, tenantID, datasetID, docID, eventType string, variants []string) error {- return s.publishEntry(ctx, tenantID, datasetID, BacklogEntry{DocID: docID, EventType: eventType, Variants: variants})+func (s *mysqlScheduler) publish(ctx context.Context, tenantID, datasetID, docID, eventType string, variants, taskTypes []string) error {+ return s.publishEntry(ctx, tenantID, datasetID, BacklogEntry{DocID: docID, EventType: eventType, Variants: variants, TaskTypes: taskTypes})}func (s *mysqlScheduler) publishEntry(ctx context.Context, tenantID, datasetID string, entry BacklogEntry) error {@@ -783,8 +787,8 @@ func NewFakeScheduler() *FakeScheduler {func (f *FakeScheduler) Provision(_ context.Context) error { return nil }// Publish appends one doc event and pushes a notify (same contract as MySQL).-func (f *FakeScheduler) Publish(_ context.Context, tenantID, datasetID, docID, eventType string, variants []string) error {- return f.publishEntry(tenantID, datasetID, BacklogEntry{DocID: docID, EventType: eventType, Variants: variants})+func (f *FakeScheduler) Publish(_ context.Context, tenantID, datasetID, docID, eventType string, variants, taskTypes []string) error {+ return f.publishEntry(tenantID, datasetID, BacklogEntry{DocID: docID, EventType: eventType, Variants: variants, TaskTypes: taskTypes})}// PublishedCount returns the total number of backlog events published acrossdiff --git a/internal/ingestion/knowledge_compile/scheduler_test.go b/internal/ingestion/knowledge_compile/scheduler_test.goindex fe592b1773..2e3d3e4a03 100644--- a/internal/ingestion/knowledge_compile/scheduler_test.go+++ b/internal/ingestion/knowledge_compile/scheduler_test.go@@ -36,7 +36,7 @@ func TestRemoveEntriesMatchesByDocAndEventType(t *testing.T) {func TestFakeSchedulerReclaimsInterruptedClaim(t *testing.T) {f := NewFakeScheduler()- if err := f.Publish(t.Context(), "t1", "kb1", "d1", string(EventTypeCompleted), []string{"wiki"}); err != nil {+ if err := f.Publish(t.Context(), "t1", "kb1", "d1", string(EventTypeCompleted), []string{"wiki"}, []string{"Wiki"}); err != nil {t.Fatalf("publish: %v", err)}first, ok, err := f.Claim(t.Context(), "kb1")@@ -58,7 +58,7 @@ func TestFakeSchedulerReclaimsInterruptedClaim(t *testing.T) {func TestFakeSchedulerProgressIsClaimScoped(t *testing.T) {f := NewFakeScheduler()- if err := f.Publish(t.Context(), "t1", "kb1", "d1", string(EventTypeCompleted), nil); err != nil {+ if err := f.Publish(t.Context(), "t1", "kb1", "d1", string(EventTypeCompleted), nil, nil); err != nil {t.Fatalf("publish: %v", err)}claim, ok, err := f.Claim(t.Context(), "kb1")@@ -81,7 +81,7 @@ func TestFakeSchedulerProgressIsClaimScoped(t *testing.T) {func TestWithWriteLockRejectsSupersededClaim(t *testing.T) {f := NewFakeScheduler()- if err := f.Publish(t.Context(), "t1", "kb1", "d1", string(EventTypeCompleted), []string{"wiki"}); err != nil {+ if err := f.Publish(t.Context(), "t1", "kb1", "d1", string(EventTypeCompleted), []string{"wiki"}, []string{"Wiki"}); err != nil {t.Fatalf("publish: %v", err)}claim, ok, err := f.Claim(t.Context(), "kb1")@@ -131,14 +131,17 @@ func TestBacklogEntryJSONBackwardCompat(t *testing.T) {t.Fatalf("legacy backlog should unmarshal Variants as nil, got %v", e.Variants)}- // A completed event with variants round-trips intact.- withVariants := []byte(`{"doc_id":"d1","event_type":"doc_completed","variants":["tree","wiki"]}`)+ // A completed event with variants and frontend task types round-trips intact.+ withVariants := []byte(`{"doc_id":"d1","event_type":"doc_completed","variants":["tree","wiki"],"task_types":["Tree","Wiki"]}`)if err := json.Unmarshal(withVariants, &e); err != nil {t.Fatalf("unmarshal variants backlog: %v", err)}if !reflect.DeepEqual(e.Variants, []string{"tree", "wiki"}) {t.Fatalf("variants not restored: %+v", e.Variants)}+ if !reflect.DeepEqual(e.TaskTypes, []string{"Tree", "Wiki"}) {+ t.Fatalf("task types not restored: %+v", e.TaskTypes)+ }// omitempty: a nil Variants serializes without the key (matches legacy format).enc, err := json.Marshal(BacklogEntry{DocID: "d2", EventType: "doc_deleted"})if err != nil {@@ -151,16 +154,19 @@ func TestBacklogEntryJSONBackwardCompat(t *testing.T) {if _, hasVariants := decoded["variants"]; hasVariants {t.Fatalf("nil Variants should be omitted, got %s", enc)}+ if _, hasTaskTypes := decoded["task_types"]; hasTaskTypes {+ t.Fatalf("nil TaskTypes should be omitted, got %s", enc)+ }}// TestFakeSchedulerPublishCarriesVariants locks the A0-2 contract: FakeScheduler// records the variants on the backlog entry, so the consumer sees them.func TestFakeSchedulerPublishCarriesVariants(t *testing.T) {f := NewFakeScheduler()- if err := f.Publish(t.Context(), "t1", "kb1", "d1", string(EventTypeCompleted), []string{"tree", "structure"}); err != nil {+ if err := f.Publish(t.Context(), "t1", "kb1", "d1", string(EventTypeCompleted), []string{"tree", "structure"}, []string{"Tree", "Graph"}); err != nil {t.Fatalf("publish: %v", err)}- if err := f.Publish(t.Context(), "t1", "kb1", "d2", string(EventTypeDeleted), nil); err != nil {+ if err := f.Publish(t.Context(), "t1", "kb1", "d2", string(EventTypeDeleted), nil, nil); err != nil {t.Fatalf("publish deleted: %v", err)}res, ok, err := f.Claim(t.Context(), "kb1")@@ -173,6 +179,9 @@ func TestFakeSchedulerPublishCarriesVariants(t *testing.T) {if !reflect.DeepEqual(res.Entries[0].Variants, []string{"tree", "structure"}) {t.Fatalf("completed variants not carried: %+v", res.Entries[0])}+ if !reflect.DeepEqual(res.Entries[0].TaskTypes, []string{"Tree", "Graph"}) {+ t.Fatalf("completed task types not carried: %+v", res.Entries[0])+ }if res.Entries[1].Variants != nil {t.Fatalf("deleted event should carry nil variants, got %+v", res.Entries[1])}diff --git a/internal/ingestion/knowledge_compile/wiki_contribution_test.go b/internal/ingestion/knowledge_compile/wiki_contribution_test.goindex db4531db14..5ba6d3143f 100644--- a/internal/ingestion/knowledge_compile/wiki_contribution_test.go+++ b/internal/ingestion/knowledge_compile/wiki_contribution_test.go@@ -83,7 +83,7 @@ func TestProcessBatchSkipsUnchangedWikiOnlyEvent(t *testing.T) {)if err := consumer.processBatch(context.Background(), "tenant-1", "kb-1", "token-1", []BacklogEntry{{DocID: "doc-1", EventType: string(EventTypeCompleted), Variants: []string{"wiki"},- }}); err != nil {+ }}, []string{kccommon.TaskTypeWiki}); err != nil {t.Fatalf("processBatch failed: %v", err)}if factoryCalled {diff --git a/internal/ingestion/task/knowledge_compiler_wiring.go b/internal/ingestion/task/knowledge_compiler_wiring.goindex d6734e54b9..1444cce9cd 100644--- a/internal/ingestion/task/knowledge_compiler_wiring.go+++ b/internal/ingestion/task/knowledge_compiler_wiring.go@@ -363,7 +363,7 @@ func replaceDirtyWikiProducts(ctx context.Context, request knowledge_compile.Wikif len(rows) == 0 && (existing == nil || len(existing.Chunks) == 0) {return nil}- return knowledge_compile.PublishCompleted(ctx, request.TenantID, request.DatasetID, request.DocumentID, []string{"wiki"})+ return knowledge_compile.PublishCompleted(ctx, request.TenantID, request.DatasetID, request.DocumentID, []string{"wiki"}, []string{kc.TaskTypeWiki})}func clearWikiActiveStates(ctx context.Context, docEngine engine.DocEngine, request knowledge_compile.WikiDirtyRequest) error {diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.goindex de5ae141bc..8afabef237 100644--- a/internal/ingestion/task/pipeline_executor.go+++ b/internal/ingestion/task/pipeline_executor.go@@ -278,7 +278,7 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map}applyDocumentAvailability(chunks, docStatus)- oldCompiledProductIDs, oldCompiledVariants, err := s.loadDocumentCompiledState(ctx)+ oldCompiledProductIDs, oldCompiledVariants, oldCompiledTaskTypes, err := s.loadDocumentCompiledState(ctx)if err != nil {return nil, err}@@ -310,8 +310,9 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map// KindToVariant, O2a whitelist). Keeping the previous types lets the dataset// consumer retract stale merged products when a template is removed.eventVariants := mergeCompiledVariants(oldCompiledVariants, compiledVariants(chunks))+ eventTaskTypes := mergeTaskTypes(oldCompiledTaskTypes, compiledTaskTypes(chunks))if len(eventVariants) > 0 {- if err := knowledge_compile.PublishCompleted(ctx, s.taskCtx.Tenant.ID, s.taskCtx.Doc.KbID, s.taskCtx.Doc.ID, eventVariants); err != nil {+ if err := knowledge_compile.PublishCompleted(ctx, s.taskCtx.Tenant.ID, s.taskCtx.Doc.KbID, s.taskCtx.Doc.ID, eventVariants, eventTaskTypes); err != nil {common.Logger.Warn(fmt.Sprintf("knowledge_compile: publish doc_completed for %s failed: %v", s.taskCtx.Doc.ID, err))}}@@ -463,10 +464,10 @@ func markCompiledProductsHidden(chunks []map[string]any) {// loadDocumentCompiledState snapshots the previous successful document// compiler generation before the new pipeline output is written. Reading first// avoids relying on immediate search visibility after a bulk index write.-func (s *PipelineExecutor) loadDocumentCompiledState(ctx context.Context) ([]string, []string, error) {+func (s *PipelineExecutor) loadDocumentCompiledState(ctx context.Context) ([]string, []string, []string, error) {docEngine := engine.Get()if docEngine == nil || s == nil || s.taskCtx == nil {- return nil, nil, nil+ return nil, nil, nil, nil}const pageSize = 1000indexName := fmt.Sprintf("ragflow_%s", s.taskCtx.Tenant.ID)@@ -482,7 +483,7 @@ func (s *PipelineExecutor) loadDocumentCompiledState(ctx context.Context) ([]strFilter: map[string]any{"doc_id": []string{s.taskCtx.Doc.ID}},})if err != nil {- return nil, nil, fmt.Errorf("load document compiler products: %w", err)+ return nil, nil, nil, fmt.Errorf("load document compiler products: %w", err)}if result == nil || len(result.Chunks) == 0 {break@@ -500,7 +501,7 @@ func (s *PipelineExecutor) loadDocumentCompiledState(ctx context.Context) ([]strbreak}}- return oldIDs, compiledVariants(oldProducts), nil+ return oldIDs, compiledVariants(oldProducts), compiledTaskTypes(oldProducts), nil}// reconcileDocumentCompiledProducts advances the document-level compiler@@ -606,6 +607,54 @@ func compiledVariants(chunks []map[string]any) []string {return out}+// compiledTaskTypes returns the sorted, de-duplicated frontend categories of+// the compiled products in chunks. The raw template kind is preferred because+// several kinds intentionally share the structure execution variant.+func compiledTaskTypes(chunks []map[string]any) []string {+ seen := map[string]struct{}{}+ for _, ck := range chunks {+ if _, ok := ck["compile_kwd"]; !ok {+ continue+ }+ taskType := ""+ if kind, ok := ck["compilation_template_kind_kwd"].(string); ok && strings.TrimSpace(kind) != "" {+ if mapped, err := kccommon.KindToTaskType(kind); err == nil {+ taskType = mapped+ }+ }+ if taskType == "" {+ if mapped, err := knowledge_compile.KwdToVariant(asCompiledKwd(ck)); err == nil {+ taskType = kccommon.VariantToTaskType(mapped)+ }+ }+ if taskType != "" {+ seen[taskType] = struct{}{}+ }+ }+ return sortedTaskTypes(seen)+}++func mergeTaskTypes(groups ...[]string) []string {+ seen := map[string]struct{}{}+ for _, group := range groups {+ for _, taskType := range group {+ if taskType = strings.TrimSpace(taskType); taskType != "" {+ seen[taskType] = struct{}{}+ }+ }+ }+ return sortedTaskTypes(seen)+}++func sortedTaskTypes(seen map[string]struct{}) []string {+ out := make([]string, 0, len(seen))+ for taskType := range seen {+ out = append(out, taskType)+ }+ sort.Strings(out)+ return out+}+// mergeCompiledVariants returns the union of the previous and current// document-level compiler variants. A reparse that removes a compiler template// still needs to notify the dataset consumer so it can retract stale mergeddiff --git a/internal/service/document/document_crud.go b/internal/service/document/document_crud.goindex add8615436..502eb518c2 100644--- a/internal/service/document/document_crud.go+++ b/internal/service/document/document_crud.go@@ -338,6 +338,10 @@ func (s *DocumentService) RemoveDocumentKeepFile(ctx context.Context, docID striif err != nil {return err}+ _, taskTypes, typeErr := s.documentKnowledgeCompileTypes(ctx, kb.TenantID, kb.ID, docID)+ if typeErr != nil {+ common.Warn(fmt.Sprintf("RemoveDocumentKeepFile: failed to resolve knowledge compile types for %s: %v", docID, typeErr))+ }if _, delErr := s.taskDAO.DeleteByDocIDs(ctx, dao.DB, []string{docID}); delErr != nil {if errors.Is(delErr, context.Canceled) || errors.Is(delErr, context.DeadlineExceeded) {return fmt.Errorf("RemoveDocumentKeepFile: failed to delete tasks for %s: %w", docID, delErr)@@ -348,11 +352,11 @@ func (s *DocumentService) RemoveDocumentKeepFile(ctx context.Context, docID strireturn err}// File replacement/deletion uses this path instead of deleteDocumentFull.- // Publish the same deletion event so the dataset-level Wiki consumer removes- // the deleted document's contribution in both paths.+ // Publish the same deletion event so the dataset-level consumer removes the+ // deleted document's contribution in both paths.pubCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)defer cancel()- if err := knowledge_compile.PublishDeleted(pubCtx, kb.TenantID, kb.ID, docID); err != nil {+ if err := knowledge_compile.PublishDeleted(pubCtx, kb.TenantID, kb.ID, docID, taskTypes); err != nil {common.Warn(fmt.Sprintf("RemoveDocumentKeepFile: publish doc_deleted for %s failed: %v", docID, err))}return nil@@ -422,6 +426,10 @@ func (s *DocumentService) deleteDocEngineData(ctx context.Context, docID, tenantreturn}indexName := fmt.Sprintf("ragflow_%s", tenantID)+ _, taskTypes, typeErr := s.documentKnowledgeCompileTypes(ctx, tenantID, kbID, docID)+ if typeErr != nil {+ common.Warn(fmt.Sprintf("deleteDocEngineData: failed to resolve knowledge compile types for %s: %v", docID, typeErr))+ }if _, delErr := s.docEngine.DeleteChunks(ctx, map[string]interface{}{"doc_id": docID}, indexName, kbID); delErr != nil {common.Warn(fmt.Sprintf("deleteDocEngineData: failed to delete chunks for %s: %v", docID, delErr))}@@ -434,7 +442,7 @@ func (s *DocumentService) deleteDocEngineData(ctx context.Context, docID, tenant// never block the document delete, which already succeeded above.pubCtx, cancel := context.WithTimeout(ctx, 15*time.Second)defer cancel()- if err := knowledge_compile.PublishDeleted(pubCtx, tenantID, kbID, docID); err != nil {+ if err := knowledge_compile.PublishDeleted(pubCtx, tenantID, kbID, docID, taskTypes); err != nil {common.Warn(fmt.Sprintf("deleteDocEngineData: publish doc_deleted for %s failed: %v", docID, err))}if s.metadataSvc != nil {diff --git a/internal/service/document/knowledge_compile_status.go b/internal/service/document/knowledge_compile_status.goindex cc0478b82a..24f39c8684 100644--- a/internal/service/document/knowledge_compile_status.go+++ b/internal/service/document/knowledge_compile_status.go@@ -34,10 +34,16 @@ import (// products do not wake the consumer because they cannot contribute to any// dataset-level knowledge artifact.func (s *DocumentService) publishKnowledgeCompileStatusChange(ctx context.Context, tenantID, datasetID, documentID string, status int) {- variants, err := s.documentKnowledgeCompileVariants(ctx, tenantID, datasetID, documentID)+ variants, taskTypes, err := s.documentKnowledgeCompileTypes(ctx, tenantID, datasetID, documentID)if err != nil {common.Warn("document mutation: failed to resolve knowledge compile variants",zap.String("document_id", documentID), zap.Error(err))+ if status == 0 {+ if publishErr := knowledge_compile.PublishDisabled(ctx, tenantID, datasetID, documentID, nil, nil); publishErr != nil {+ common.Warn("document mutation: failed to publish fallback knowledge compile disable",+ zap.String("document_id", documentID), zap.Error(publishErr))+ }+ }return}if len(variants) == 0 {@@ -45,9 +51,9 @@ func (s *DocumentService) publishKnowledgeCompileStatusChange(ctx context.Contex}var publishErr errorif status == 0 {- publishErr = knowledge_compile.PublishDisabled(ctx, tenantID, datasetID, documentID, variants)+ publishErr = knowledge_compile.PublishDisabled(ctx, tenantID, datasetID, documentID, variants, taskTypes)} else {- publishErr = knowledge_compile.PublishEnabled(ctx, tenantID, datasetID, documentID, variants)+ publishErr = knowledge_compile.PublishEnabled(ctx, tenantID, datasetID, documentID, variants, taskTypes)}if publishErr != nil {common.Warn("document mutation: failed to publish knowledge compile status change",@@ -55,12 +61,13 @@ func (s *DocumentService) publishKnowledgeCompileStatusChange(ctx context.Contex}}-func (s *DocumentService) documentKnowledgeCompileVariants(ctx context.Context, tenantID, datasetID, documentID string) ([]string, error) {+func (s *DocumentService) documentKnowledgeCompileTypes(ctx context.Context, tenantID, datasetID, documentID string) ([]string, []string, error) {if s.docEngine == nil {- return nil, nil+ return nil, nil, nil}indexName := fmt.Sprintf("ragflow_%s", tenantID)- seen := make(map[string]struct{})+ seenVariants := make(map[string]struct{})+ seenTaskTypes := make(map[string]struct{})for offset := 0; ; offset += 1000 {result, err := s.docEngine.Search(ctx, &types.SearchRequest{IndexNames: []string{indexName},@@ -71,7 +78,7 @@ func (s *DocumentService) documentKnowledgeCompileVariants(ctx context.Context,Filter: map[string]any{"doc_id": []string{documentID}},})if err != nil {- return nil, err+ return nil, nil, err}if result == nil || len(result.Chunks) == 0 {break@@ -83,17 +90,29 @@ func (s *DocumentService) documentKnowledgeCompileVariants(ctx context.Context,variant, variantErr = knowledge_compile.KwdToVariant(strings.TrimSpace(documentStoreString(row["compile_kwd"])))}if variantErr == nil {- seen[string(variant)] = struct{}{}+ seenVariants[string(variant)] = struct{}{}+ taskType, taskTypeErr := kccommon.KindToTaskType(kind)+ if taskTypeErr != nil {+ taskType = kccommon.VariantToTaskType(variant)+ }+ if taskType != "" {+ seenTaskTypes[taskType] = struct{}{}+ }}}if int64(offset+len(result.Chunks)) >= result.Total {break}}- variants := make([]string, 0, len(seen))- for variant := range seen {+ variants := make([]string, 0, len(seenVariants))+ for variant := range seenVariants {variants = append(variants, variant)}sort.Strings(variants)- return variants, nil+ taskTypes := make([]string, 0, len(seenTaskTypes))+ for taskType := range seenTaskTypes {+ taskTypes = append(taskTypes, taskType)+ }+ sort.Strings(taskTypes)+ return variants, taskTypes, nil}