Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: support hierarchical Wiki topic navigation (#18721)

buua4363a836911d1a8
diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.py
index 75e43f17f0..914b21cb54 100644
--- a/api/apps/services/dataset_api_service.py
+++ b/api/apps/services/dataset_api_service.py
@@ -2653,46 +2653,13 @@ async def list_wiki_topics(
if not counts:
return True, {"total": 0, "items": []}
- # Resolve display metadata (title/slug) from the topic landing pages; fall
- # back to the raw topic name when a topic has no ``page_type="topic"`` row.
- meta: dict[str, dict] = {}
- try:
- meta_fields = ["topic_kwd", "title_kwd", "slug_kwd"]
- _BATCH = 1000
- _offset = 0
- while True:
- meta_res = settings.docStoreConn.search(
- select_fields=meta_fields,
- highlight_fields=[],
- condition={"compile_kwd": [WIKI_PAGE_COMPILE_KWD], "page_type_kwd": ["topic"]},
- match_expressions=[],
- order_by=OrderByExpr(),
- offset=_offset,
- limit=_BATCH,
- index_names=index_nm,
- knowledgebase_ids=[dataset_id],
- )
- rows = settings.docStoreConn.get_fields(meta_res, meta_fields) or {}
- if not rows:
- break
- for row in rows.values():
- t = _scalar(row.get("topic_kwd"))
- if t:
- meta[t] = {
- "title": _scalar(row.get("title_kwd")) or t,
- "slug": _scalar(row.get("slug_kwd")) or t,
- }
- _offset += _BATCH
- except Exception:
- logging.exception("list_wiki_topics: topic metadata lookup failed for kb=%s", dataset_id)
-
# Rank topics by page count (descending), then title for a stable order.
ranked = sorted(
(
{
"topic": t,
- "title": (meta.get(t) or {}).get("title") or t,
- "slug": (meta.get(t) or {}).get("slug") or t,
+ "title": t.rsplit("/", 1)[-1],
+ "slug": t,
"page_count": c,
}
for t, c in counts.items()
diff --git a/internal/entity/models/base_model.go b/internal/entity/models/base_model.go
index e32651204a..5242c716d9 100644
--- a/internal/entity/models/base_model.go
+++ b/internal/entity/models/base_model.go
@@ -450,7 +450,7 @@ func NewDriverHTTPClient(allowPrivate bool) *http.Client {
t.MaxIdleConnsPerHost = 10
t.IdleConnTimeout = 90 * time.Second
t.DisableCompression = false
- t.ResponseHeaderTimeout = 2 * 60 * time.Second
+ t.ResponseHeaderTimeout = 5 * 60 * time.Second
t.TLSHandshakeTimeout = 30 * time.Second
var rt http.RoundTripper = t
diff --git a/internal/ingestion/component/knowledge_compiler/common/types.go b/internal/ingestion/component/knowledge_compiler/common/types.go
index 6f803cb9ba..3fcda008a0 100644
--- a/internal/ingestion/component/knowledge_compiler/common/types.go
+++ b/internal/ingestion/component/knowledge_compiler/common/types.go
@@ -50,9 +50,9 @@ type Param struct {
MaxWorkers int
EnableHistoricalDedup bool
// Plan selects the wiki compilation mode. nil means the DSL did not specify a
- // mode, so template config may decide; the effective default is false (Mode A).
- // true selects B-mode (LLM planning + reconcile), while false selects A-mode
- // (one entity/concept = one deterministic flat page).
+ // mode, so template config may decide; the effective default is false
+ // (entity mode). true selects topic mode, which allows related identities to
+ // be merged during PLAN; both modes assign pages to MAP topics.
Plan *bool
// Extra carries arbitrary caller-provided overrides merged into the
// resolved template config.
@@ -187,8 +187,8 @@ func ParseParam(m map[string]any) (Param, error) {
return p, nil
}
-// PlanEnabled reports whether wiki B-mode is selected. An unset value defaults
-// to Mode A.
+// PlanEnabled reports whether wiki topic mode is selected. An unset value
+// defaults to entity mode.
func (p Param) PlanEnabled() bool {
return p.Plan != nil && *p.Plan
}
diff --git a/internal/ingestion/component/knowledge_compiler/common/wiki_topic.go b/internal/ingestion/component/knowledge_compiler/common/wiki_topic.go
new file mode 100644
index 0000000000..cbc03bbbc2
--- /dev/null
+++ b/internal/ingestion/component/knowledge_compiler/common/wiki_topic.go
@@ -0,0 +1,36 @@
+package common
+
+import "strings"
+
+const wikiTopicPathSeparator = "/"
+
+// GeneralWikiTopic is the fallback topic for pages that cannot be assigned to
+// a topic emitted by the MAP stage.
+const GeneralWikiTopic = "General"
+
+// NormalizeWikiTopicPath canonicalizes a materialized Wiki topic path while
+// preserving its human-readable segments. A one-segment topic is a valid root
+// path; empty segments are discarded.
+func NormalizeWikiTopicPath(topic string) string {
+ parts := strings.Split(topic, wikiTopicPathSeparator)
+ normalized := make([]string, 0, len(parts))
+ for _, part := range parts {
+ part = strings.Join(strings.Fields(strings.TrimSpace(part)), " ")
+ if part != "" {
+ normalized = append(normalized, part)
+ }
+ }
+ return strings.Join(normalized, wikiTopicPathSeparator)
+}
+
+// WikiTopicLeaf returns the last segment of a normalized Wiki topic path.
+func WikiTopicLeaf(topic string) string {
+ topic = NormalizeWikiTopicPath(topic)
+ if topic == "" {
+ return ""
+ }
+ if i := strings.LastIndex(topic, wikiTopicPathSeparator); i >= 0 {
+ return topic[i+1:]
+ }
+ return topic
+}
diff --git a/internal/ingestion/component/knowledge_compiler/common/wiki_topic_test.go b/internal/ingestion/component/knowledge_compiler/common/wiki_topic_test.go
new file mode 100644
index 0000000000..2902effc9b
--- /dev/null
+++ b/internal/ingestion/component/knowledge_compiler/common/wiki_topic_test.go
@@ -0,0 +1,30 @@
+package common
+
+import "testing"
+
+func TestNormalizeWikiTopicPath(t *testing.T) {
+ tests := []struct {
+ name string
+ input string
+ want string
+ }{
+ {name: "flat", input: "蜀汉人物", want: "蜀汉人物"},
+ {name: "nested", input: " 三国演义 / 人物 / 蜀汉人物 ", want: "三国演义/人物/蜀汉人物"},
+ {name: "empty segments", input: "三国演义//人物/", want: "三国演义/人物"},
+ {name: "spaces", input: "Three Kingdoms / Major Figures", want: "Three Kingdoms/Major Figures"},
+ {name: "depth", input: "History/China/Three Kingdoms/People", want: "History/China/Three Kingdoms/People"},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := NormalizeWikiTopicPath(tt.input); got != tt.want {
+ t.Fatalf("NormalizeWikiTopicPath(%q) = %q, want %q", tt.input, got, tt.want)
+ }
+ })
+ }
+}
+
+func TestWikiTopicLeaf(t *testing.T) {
+ if got := WikiTopicLeaf("三国演义 / 人物 / 蜀汉人物"); got != "蜀汉人物" {
+ t.Fatalf("WikiTopicLeaf() = %q, want %q", got, "蜀汉人物")
+ }
+}
diff --git a/internal/ingestion/component/knowledge_compiler/wiki/page.go b/internal/ingestion/component/knowledge_compiler/wiki/page.go
index a13b6d7c14..9f2ba076fa 100644
--- a/internal/ingestion/component/knowledge_compiler/wiki/page.go
+++ b/internal/ingestion/component/knowledge_compiler/wiki/page.go
@@ -189,7 +189,7 @@ func buildWikiPageProducts(tenantID, docID string, pages []wikiPageResult) []com
"slug": slug,
"title": title,
"page_type": pageType,
- "topic": strings.TrimSpace(page.Topic),
+ "topic": common.NormalizeWikiTopicPath(page.Topic),
"summary": summary,
"entity_names": uniqueStrings(page.EntityNames),
"related_kb_pages": uniqueStrings(page.RelatedKBPages),
diff --git a/internal/ingestion/component/knowledge_compiler/wiki/prompt.go b/internal/ingestion/component/knowledge_compiler/wiki/prompt.go
index 206df964ec..a42f7f77c1 100644
--- a/internal/ingestion/component/knowledge_compiler/wiki/prompt.go
+++ b/internal/ingestion/component/knowledge_compiler/wiki/prompt.go
@@ -57,7 +57,13 @@ exact schema:
"source_chunk_id": "string"
}
],
- "topics": ["string"]
+ "topics": [
+ {
+ "path": "root/subtopic/leaf",
+ "description": "string",
+ "source_chunk_id": "string"
+ }
+ ]
}
## Customization
@@ -78,7 +84,12 @@ Rules:
- Extract entities and concepts from human-readable text only.
- Claims should be liberal: every factual sentence about an entity or concept is a claim.
- Relations should be explicit links only.
-- Topics should capture coherent subtopics that could become wiki sections.
+- Each topic.path must be a normalized navigation-path string. Use "/" as the
+ hierarchy separator, keep each path to at most 3 segments, and do not use "/"
+ inside a segment.
+- Reuse the same complete path for repeated topics instead of emitting
+ alternate spellings or isolated leaf labels.
+- Every topic must cite a supporting chunk with source_chunk_id.
- Return empty arrays [] for categories with no findings.
- Return ONLY the JSON object, no markdown fences, no commentary.`
@@ -111,22 +122,12 @@ Return a JSON compilation plan with one or more page entries:
{
"pages": [
{
- "action": "CREATE",
- "slug": "string",
"title": "string",
"page_type": "entity | concept | topic",
- "topic": "string",
- "entity_names": ["string"],
- "related_kb_pages": ["string"],
- "priority": 1,
- "lead": "string",
- "sections": [
- { "heading": "string", "points": ["string"] }
- ]
+ "topic": "root/subtopic/leaf",
+ "entity_names": ["string"]
}
- ],
- "estimated_page_count": 1,
- "compilation_notes": "string"
+ ]
}
Rules:
@@ -141,13 +142,21 @@ Rules:
but list every represented identity in entity_names and do not repeat any of
them on another page. Merge minor or weakly-supported facts into such a page
instead of emitting tiny standalone pages.
-- Identity ownership does not limit linking: related_kb_pages should include
- every directly related canonical page supported by the input (up to the
- available-page budget), and links must target canonical slugs only.
+- Page identity and topic assignment are the only planning decisions. Slugs,
+ actions, priorities, related-page links, summaries, and sections are filled
+ by the compiler or the later writing stage.
+- {planning_mode_rules}
- Use page_type=entity for entity pages, page_type=concept for concept pages, and page_type=topic for cross-cutting themes.
+- topic must be selected from the complete paths in the reduced input topics.
+ A topic may contain up to 3 hierarchical segments, and the selected path
+ must be reused exactly for every page assigned to it. Do not create a new
+ topic during PLAN. If no reduced topic fits, use exactly "General".
+- Do not use an entity or concept title as its topic. The page title identifies
+ the page; topic is selected from the MAP topic candidates. Never use the
+ page title itself, or a path created only by appending the page title, as its
+ topic.
+- A topic path segment must not contain "/"; "/" is reserved for hierarchy.
- entity_names must name the entities and concepts that justify the page.
-- related_kb_pages should list other slugs from the same plan that the page should cross-link to.
-- Keep lead concise (one sentence) and keep sections compact (no more than 4 sections, no more than 3 short points per section).
- Keep the page in the source language.
- Return ONLY the JSON object.`
@@ -175,8 +184,9 @@ Rules:
- If action is UPDATE, entity_names must contain the complete membership set
for the target page. Remove members that no longer belong and include newly
routed members. If membership is unchanged, repeat the candidate members.
-- topic must be the final topic label for the page. If action is CREATE, use a
- concise topic label grounded in the planned page and candidates.
+- topic must be the complete materialized topic path. If action is UPDATE, copy
+ the selected candidate's topic exactly. If action is CREATE, preserve the
+ planned page's topic path.
- Return only JSON.`
const wikiRefineWriterExample = `Each page must be a proper encyclopedic article, not a flat bullet list:
@@ -340,6 +350,9 @@ const defaultRelationSchemaBody = `"from": "string — source entity/concept nam
"source_chunk_id": "string — exact value from the chunk_id list above"`
func wikiTemplateCustomRules(parserConfig map[string]any, language string) string {
+ if rules := common.Localize(common.Get(parserConfig, "global_rules"), language); strings.TrimSpace(rules) != "" {
+ return strings.TrimSpace(rules)
+ }
guideline := common.GetMap(parserConfig, "guideline")
if len(guideline) == 0 {
return ""
diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki.go
index c9bb1ca648..74f057e41d 100644
--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go
+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki.go
@@ -125,7 +125,7 @@ type wikiExtract struct {
Concepts []wikiConcept `json:"concepts"`
Claims []wikiClaim `json:"claims"`
Relations []wikiRelation `json:"relations"`
- Topics []string `json:"topics"`
+ Topics []wikiTopic `json:"topics"`
Mode string `json:"mode,omitempty"`
}
@@ -166,6 +166,12 @@ type wikiRelation struct {
SourceChunkIDs []string `json:"source_chunk_ids,omitempty"`
}
+type wikiTopic struct {
+ Path string `json:"path"`
+ Description string `json:"description,omitempty"`
+ SourceChunkIDs []string `json:"source_chunk_ids,omitempty"`
+}
+
type wikiPlan struct {
Title string `json:"title"`
Slug string `json:"slug"`
@@ -397,25 +403,17 @@ func (p *wikiPipeline) run() error {
zap.Int("entities", len(p.reduced.Entities)),
zap.Int("claims", len(p.reduced.Claims)))
runtime.ReportProgressMessage(p.ctx, "Compiler", fmt.Sprintf("Wiki REDUCE Done: output_entities=%d output_concepts=%d output_claims=%d output_relations=%d", len(p.reduced.Entities), len(p.reduced.Concepts), len(p.reduced.Claims), len(p.reduced.Relations)))
- // PLAN (B-mode only): LLM-based page plan + reconcile against existing pages.
- // Mode A (param.PlanEnabled() == false, the default) skips the planner entirely — every
- // extracted entity/concept becomes its own flat page (1 identity = 1 page), so
- // the wiki is a flat encyclopedia without PLAN-grouped pages. See buildModeAPlan.
+ // PLAN uses the same topic-assignment path in both modes. Entity mode keeps
+ // one identity per page, while topic mode may merge related identities in
+ // the planner before assigning the resulting pages to MAP topics.
var plan wikiPlan
var err error
runtime.ReportProgressMessage(p.ctx, "Compiler", fmt.Sprintf("Wiki PLAN Started: input_entities=%d input_concepts=%d input_relations=%d", len(p.reduced.Entities), len(p.reduced.Concepts), len(p.reduced.Relations)))
- if p.param.PlanEnabled() {
- plan, err = p.runPlan()
- if err != nil {
- return err
- }
- appcommon.Info("wiki: PLAN (B-mode) done", zap.String("dataset_id", p.datasetID), zap.String("doc_id", p.runKey()), zap.Int("plan_pages", len(plan.Pages)))
- } else {
- // wiki_incremental port (T1 + M1 Mode A): deterministic flat plan from the
- // reduced graph. No LLM, no reconcile.
- plan = p.buildModeAPlan()
- appcommon.Info("wiki: PLAN (A-mode flat) done", zap.String("dataset_id", p.datasetID), zap.String("doc_id", p.runKey()), zap.Int("plan_pages", len(plan.Pages)))
+ plan, err = p.runPlan()
+ if err != nil {
+ return err
}
+ appcommon.Info("wiki: PLAN done", zap.String("dataset_id", p.datasetID), zap.String("doc_id", p.runKey()), zap.String("mode", p.wikiMode()), zap.Int("plan_pages", len(plan.Pages)))
p.plan = plan
p.selectAffectedPages(plan.Pages)
appcommon.Info("wiki: PLAN done",
@@ -502,9 +500,9 @@ func countExtractTopics(extracts []wikiExtract) int {
func countPlanTopics(pages []wikiPlanPage) int {
topics := make(map[string]struct{})
for _, page := range pages {
- topic := strings.TrimSpace(page.Topic)
+ topic := common.NormalizeWikiTopicPath(page.Topic)
if topic != "" {
- topics[topic] = struct{}{}
+ topics[wikiTopicKey(topic)] = struct{}{}
}
}
return len(topics)
@@ -1002,16 +1000,21 @@ func maxPagesForBatch(quota int) int {
}
func (p *wikiPipeline) runPlanBatch(batch wikiExtract, batchIndex, batchTotal, quota, maxTokens int) (wikiPlan, error) {
+ planningModeRules := "Entity mode: create exactly one page for each extracted entity or concept. Do not merge multiple identities into one page, and each page's entity_names must contain only its own identity."
+ if p.wikiMode() == "topic" {
+ planningModeRules = "Topic mode: closely related extracted entities or concepts may be combined into one page when the source evidence supports it. Include every combined identity in entity_names."
+ }
user := renderWikiTemplate(wikiPlanBatchUserTemplate, map[string]string{
- "doc_id": p.docID,
- "batch_index": fmt.Sprintf("%d", batchIndex),
- "batch_total": fmt.Sprintf("%d", batchTotal),
- "max_pages": fmt.Sprintf("%d", maxPagesForBatch(quota)),
- "entities": mustJSON(batch.Entities),
- "concepts": mustJSON(batch.Concepts),
- "claims": mustJSON(batch.Claims),
- "relations": mustJSON(batch.Relations),
- "topics": mustJSON(batch.Topics),
+ "doc_id": p.docID,
+ "batch_index": fmt.Sprintf("%d", batchIndex),
+ "batch_total": fmt.Sprintf("%d", batchTotal),
+ "max_pages": fmt.Sprintf("%d", maxPagesForBatch(quota)),
+ "entities": mustJSON(batch.Entities),
+ "concepts": mustJSON(batch.Concepts),
+ "claims": mustJSON(batch.Claims),
+ "relations": mustJSON(batch.Relations),
+ "topics": mustJSON(batch.Topics),
+ "planning_mode_rules": planningModeRules,
})
raw, err := common.GenJSON(p.ctx, p.deps.Chat, common.ChatRequest{
LLMID: p.llmID,
@@ -1069,10 +1072,68 @@ func (p *wikiPipeline) mergePlanCandidates(plans []wikiPlan, reduced wikiExtract
}
}
merged = normalizeWikiPlan(merged, p.docID, reduced)
+ merged.Pages = assembleWikiPlanRelatedPages(merged.Pages, reduced.Relations)
merged.Pages = normalizeWikiPlanPageLinks(merged.Pages)
return merged
}
+// assembleWikiPlanRelatedPages derives page links from the reduced relation
+// graph. The planner only decides page identity and topic; generated slugs,
+// priorities, actions, and cross-page links stay deterministic and do not
+// consume LLM output tokens.
+func assembleWikiPlanRelatedPages(pages []wikiPlanPage, relations []wikiRelation) []wikiPlanPage {
+ if len(pages) == 0 || len(relations) == 0 {
+ return pages
+ }
+
+ pageByName := make(map[string][]int)
+ for i, page := range pages {
+ names := uniqueStrings(append([]string{}, page.EntityNames...))
+ if len(names) == 0 && strings.TrimSpace(page.Title) != "" {
+ names = []string{page.Title}
+ }
+ for _, name := range names {
+ if key := normKey(name); key != "" {
+ pageByName[key] = append(pageByName[key], i)
+ }
+ }
+ }
+
+ related := make([]map[string]struct{}, len(pages))
+ for i := range related {
+ related[i] = make(map[string]struct{})
+ for _, slug := range pages[i].RelatedKB {
+ if slug = strings.TrimSpace(slug); slug != "" {
+ related[i][slug] = struct{}{}
+ }
+ }
+ }
+ for _, relation := range relations {
+ fromPages := pageByName[normKey(relation.From)]
+ toPages := pageByName[normKey(relation.To)]
+ for _, from := range fromPages {
+ for _, to := range toPages {
+ if from == to || strings.TrimSpace(pages[to].Slug) == "" {
+ continue
+ }
+ related[from][pages[to].Slug] = struct{}{}
+ if strings.TrimSpace(pages[from].Slug) != "" {
+ related[to][pages[from].Slug] = struct{}{}
+ }
+ }
+ }
+ }
+
+ for i := range pages {
+ pages[i].RelatedKB = make([]string, 0, len(related[i]))
+ for slug := range related[i] {
+ pages[i].RelatedKB = append(pages[i].RelatedKB, slug)
+ }
+ sort.Strings(pages[i].RelatedKB)
+ }
+ return pages
+}
+
func (p *wikiPipeline) reconcilePlan(plan wikiPlan) (wikiPlan, error) {
if p.deps.WikiPages == nil || p.deps.Embed == nil || strings.TrimSpace(p.datasetID) == "" || len(plan.Pages) == 0 {
return plan, nil
@@ -1113,9 +1174,9 @@ func (p *wikiPipeline) reconcilePlan(plan wikiPlan) (wikiPlan, error) {
page.Title = firstNonEmpty(existing.Title, page.Title)
page.PageType = firstNonEmpty(existing.PageType, page.PageType)
if existing.RoutedTopic != "" {
- page.Topic = existing.RoutedTopic
+ page.Topic = common.NormalizeWikiTopicPath(existing.RoutedTopic)
} else {
- page.Topic = firstNonEmpty(existing.Topic, page.Topic)
+ page.Topic = common.NormalizeWikiTopicPath(firstNonEmpty(existing.Topic, page.Topic))
}
page.RelatedKB = mergeStrings(page.RelatedKB, existing.RelatedKBPages)
if len(existing.RoutedEntityNames) > 0 {
@@ -1155,7 +1216,7 @@ func (p *wikiPipeline) reconcilePlanPage(page wikiPlanPage, queryVec []float32)
cands = rerankWikiPlanCandidates(page, cands)
targetNames := normalizedStringSet(page.EntityNames)
pageTitle := normKey(page.Title)
- pageTopic := normKey(page.Topic)
+ pageTopic := wikiTopicKey(page.Topic)
maybe := make([]common.WikiPageCandidate, 0, len(cands))
for i := range cands {
cand := cands[i]
@@ -1165,7 +1226,7 @@ func (p *wikiPipeline) reconcilePlanPage(page wikiPlanPage, queryVec []float32)
if cand.Score < wikiPlanMaybeThreshold {
continue
}
- if normKey(cand.Title) == pageTitle || normKey(cand.Topic) == pageTopic {
+ if normKey(cand.Title) == pageTitle || wikiTopicKey(cand.Topic) == pageTopic {
return &cand, nil
}
if intersectsNormalized(targetNames, cand.EntityNames) {
@@ -1246,7 +1307,7 @@ func (p *wikiPipeline) expandTopicNeighborCandidates(page wikiPlanPage, candidat
func (p *wikiPipeline) sourceChunksForPlanPage(page wikiPlanPage) []string {
names := normalizedStringSet(page.EntityNames)
- for _, name := range []string{page.Title, page.Topic} {
+ for _, name := range []string{page.Title, page.Topic, common.WikiTopicLeaf(page.Topic)} {
if key := normKey(name); key != "" {
names[key] = struct{}{}
}
@@ -1294,7 +1355,7 @@ func rerankWikiPlanCandidates(page wikiPlanPage, candidates []common.WikiPageCan
targetNames := normalizedStringSet(page.EntityNames)
pageType := normKey(page.PageType)
pageTitle := normKey(page.Title)
- pageTopic := normKey(page.Topic)
+ pageTopic := wikiTopicKey(page.Topic)
scoredCands := make([]scored, 0, len(candidates))
for _, cand := range candidates {
boost := cand.Score
@@ -1304,7 +1365,7 @@ func rerankWikiPlanCandidates(page wikiPlanPage, candidates []common.WikiPageCan
if pageTitle != "" && normKey(cand.Title) == pageTitle {
boost += 0.12
}
- if pageTopic != "" && normKey(cand.Topic) == pageTopic {
+ if pageTopic != "" && wikiTopicKey(cand.Topic) == pageTopic {
boost += 0.08
}
if overlap := normalizedOverlapCount(targetNames, cand.EntityNames); overlap > 0 {
@@ -1350,7 +1411,10 @@ func (p *wikiPipeline) resolveMaybePlanPage(page wikiPlanPage, candidates []comm
}
for i := range candidates {
if candidates[i].Slug == slug {
- if topic := strings.TrimSpace(firstString(raw["topic"])); topic != "" {
+ if topic := common.NormalizeWikiTopicPath(candidates[i].Topic); topic != "" {
+ // UPDATE preserves the existing page's materialized topic path.
+ candidates[i].RoutedTopic = topic
+ } else if topic := common.NormalizeWikiTopicPath(firstString(raw["topic"])); topic != "" {
candidates[i].RoutedTopic = topic
}
if names := parseWikiStrings(raw["entity_names"]); len(names) > 0 {
@@ -1516,7 +1580,7 @@ func reduceExtracts(extracts []wikiExtract) wikiExtract {
entities := map[entityKey]*wikiEntity{}
concepts := map[conceptKey]*wikiConcept{}
relationIndexes := map[string]int{}
- seenTopics := map[string]bool{}
+ topics := map[string]*wikiTopic{}
for _, ex := range extracts {
for _, e := range ex.Entities {
@@ -1566,16 +1630,22 @@ func reduceExtracts(extracts []wikiExtract) wikiExtract {
out.Relations = append(out.Relations, r)
}
for _, t := range ex.Topics {
- t = strings.TrimSpace(t)
- if t == "" {
+ t.Path = common.NormalizeWikiTopicPath(t.Path)
+ if t.Path == "" {
continue
}
- key := normKey(t)
- if seenTopics[key] {
+ key := normKey(t.Path)
+ if current := topics[key]; current != nil {
+ if current.Description == "" {
+ current.Description = strings.TrimSpace(t.Description)
+ }
+ current.SourceChunkIDs = mergeStrings(current.SourceChunkIDs, t.SourceChunkIDs)
continue
}
- seenTopics[key] = true
- out.Topics = append(out.Topics, t)
+ t.Description = strings.TrimSpace(t.Description)
+ t.SourceChunkIDs = uniqueStrings(t.SourceChunkIDs)
+ topic := t
+ topics[key] = &topic
}
}
@@ -1602,6 +1672,14 @@ func reduceExtracts(extracts []wikiExtract) wikiExtract {
for _, k := range ckeys {
out.Concepts = append(out.Concepts, *concepts[k])
}
+ topicKeys := make([]string, 0, len(topics))
+ for key := range topics {
+ topicKeys = append(topicKeys, key)
+ }
+ sort.Strings(topicKeys)
+ for _, key := range topicKeys {
+ out.Topics = append(out.Topics, *topics[key])
+ }
return out
}
@@ -1611,7 +1689,7 @@ func parseWikiExtract(raw map[string]any) wikiExtract {
out.Concepts = parseWikiConcepts(raw["concepts"])
out.Claims = parseWikiClaims(raw["claims"])
out.Relations = parseWikiRelations(raw["relations"])
- out.Topics = parseWikiStrings(raw["topics"])
+ out.Topics = parseWikiTopics(raw["topics"])
return out
}
@@ -1656,7 +1734,7 @@ func parseWikiPlan(raw map[string]any, docID string, reduced wikiExtract) wikiPl
}
if plan.Title == "" {
if len(reduced.Topics) > 0 {
- plan.Title = reduced.Topics[0]
+ plan.Title = common.WikiTopicLeaf(reduced.Topics[0].Path)
} else if len(reduced.Entities) > 0 {
plan.Title = reduced.Entities[0].Name
} else if len(reduced.Concepts) > 0 {
@@ -1704,6 +1782,7 @@ func normalizeWikiPlanPages(pages []wikiPlanPage, reduced wikiExtract) []wikiPla
out := make([]wikiPlanPage, 0, len(pages))
for _, page := range pages {
page = normalizeWikiPlanPage(page)
+ page.Topic = selectWikiTopicPath(page.Topic, page.Title, page.PageType, wikiTopicPaths(reduced.Topics))
if page.Slug == "" {
continue
}
@@ -1731,6 +1810,7 @@ func normalizeWikiPlanPages(pages []wikiPlanPage, reduced wikiExtract) []wikiPla
continue
}
page = normalizeWikiPlanPage(page)
+ page.Topic = selectWikiTopicPath(page.Topic, page.Title, page.PageType, wikiTopicPaths(reduced.Topics))
idx := len(out)
key := wikiTitleKey(page.PageType, page.Title)
if page.PageType == "entity" && len(page.EntityNames) == 1 {
@@ -1783,6 +1863,69 @@ func normalizeWikiPlanPageLinks(pages []wikiPlanPage) []wikiPlanPage {
return out
}
+func normalizeWikiTopicPaths(topics []string) []string {
+ out := make([]string, 0, len(topics))
+ seen := make(map[string]struct{}, len(topics))
+ for _, topic := range topics {
+ topic = common.NormalizeWikiTopicPath(topic)
+ key := normKey(topic)
+ if key == "" {
+ continue
+ }
+ if _, exists := seen[key]; exists {
+ continue
+ }
+ seen[key] = struct{}{}
+ out = append(out, topic)
+ }
+ return out
+}
+
+func wikiTopicPaths(topics []wikiTopic) []string {
+ paths := make([]string, 0, len(topics))
+ for _, topic := range topics {
+ paths = append(paths, topic.Path)
+ }
+ return normalizeWikiTopicPaths(paths)
+}
+
+func wikiTopicKey(topic string) string {
+ return normKey(common.NormalizeWikiTopicPath(topic))
+}
+
+// selectWikiTopicPath resolves a planner topic to a MAP candidate. PLAN is not
+// allowed to create topics: unmatched pages are placed in General. A unique
+// leaf match still expands to its complete MAP path, which preserves
+// multi-level topics while accepting concise LLM references.
+func selectWikiTopicPath(topic, title, pageType string, candidates []string) string {
+ topic = common.NormalizeWikiTopicPath(topic)
+ candidates = normalizeWikiTopicPaths(candidates)
+ pageType = normKey(pageType)
+ title = strings.TrimSpace(title)
+ selfTopic := func(candidate string) bool {
+ return (pageType == "entity" || pageType == "concept") &&
+ normKey(common.WikiTopicLeaf(candidate)) == normKey(title)
+ }
+ if topic != "" {
+ for _, candidate := range candidates {
+ if normKey(candidate) == normKey(topic) && !selfTopic(candidate) {
+ return candidate
+ }
+ }
+ leafMatches := make([]string, 0, 1)
+ for _, candidate := range candidates {
+ if normKey(common.WikiTopicLeaf(candidate)) == normKey(topic) && !selfTopic(candidate) {
+ leafMatches = append(leafMatches, candidate)
+ }
+ }
+ if len(leafMatches) == 1 {
+ return leafMatches[0]
+ }
+ return common.GeneralWikiTopic
+ }
+ return common.GeneralWikiTopic
+}
+
func normalizeWikiPlanPage(page wikiPlanPage) wikiPlanPage {
page.Action = strings.ToUpper(strings.TrimSpace(page.Action))
if page.Action == "" {
@@ -1805,9 +1948,9 @@ func normalizeWikiPlanPage(page wikiPlanPage) wikiPlanPage {
if page.PageType == "" {
page.PageType = "concept"
}
- page.Topic = strings.TrimSpace(page.Topic)
+ page.Topic = common.NormalizeWikiTopicPath(page.Topic)
if page.Topic == "" {
- page.Topic = page.Title
+ page.Topic = common.GeneralWikiTopic
}
page.EntityNames = uniqueStrings(page.EntityNames)
page.RelatedKB = uniqueStrings(page.RelatedKB)
@@ -1845,89 +1988,6 @@ func wikiTitleKey(pageType, title string) string {
return strings.TrimSpace(pageType) + "\x00" + normKey(title)
}
-// buildModeAPlan synthesizes a FLAT wiki plan (Mode A) directly from the reduced
-// extract graph — no LLM planner, no reconcile. Every extracted entity and
-// concept becomes its own canonical page (1 identity = 1 page), slugged as
-// "entity/<slug>" / "concept/<slug>". Cross-links (RelatedKB) are derived purely
-// from the reduced relations: each relation end is mapped to its full-slug page
-// and the counterpart is added to the page's RelatedKB. This is the deterministic
-// counterpart of Python's no_plan wiki mode.
-func (p *wikiPipeline) buildModeAPlan() wikiPlan {
- reduced := p.reduced
- // Build the full-slug index for every entity/concept so relation endpoints
- // (which are bare names) resolve to canonical pages.
- fullSlugFor := func(name, pageType string) string {
- return pageType + "/" + normalizeWikiSlugHyphens(slugify(name))
- }
- entitySlugFor := func(e wikiEntity) string { return entityPageSlug(e.Name, e.Type) }
- entitySlugsByName := map[string][]string{}
- slugToIndex := map[string]int{}
- var pages []wikiPlanPage
- addPage := func(slug, title, pageType string, entityNames []string) {
- if _, ok := slugToIndex[slug]; ok {
- return
- }
- page := wikiPlanPage{
- Action: "CREATE",
- Slug: slug,
- Title: title,
- PageType: pageType,
- Topic: title,
- EntityNames: entityNames,
- Priority: len(pages) + 1,
- }
- slugToIndex[slug] = len(pages)
- pages = append(pages, page)
- }
- for _, e := range reduced.Entities {
- name := strings.TrimSpace(e.Name)
- if name == "" {
- continue
- }
- slug := entitySlugFor(e)
- entitySlugsByName[normKey(name)] = append(entitySlugsByName[normKey(name)], slug)
- addPage(slug, name, "entity", []string{name})
- }
- for _, c := range reduced.Concepts {
- term := strings.TrimSpace(c.Term)
- if term == "" {
- continue
- }
- slug := fullSlugFor(term, "concept")
- addPage(slug, term, "concept", []string{term})
- }
- // Resolve relations into RelatedKB (full-slug cross-links) on both endpoints.
- for _, rel := range reduced.Relations {
- from := strings.TrimSpace(rel.From)
- to := strings.TrimSpace(rel.To)
- if from == "" || to == "" {
- continue
- }
- fromSlugs := uniqueStrings(entitySlugsByName[normKey(from)])
- toSlugs := uniqueStrings(entitySlugsByName[normKey(to)])
- if len(fromSlugs) != 1 || len(toSlugs) != 1 {
- continue
- }
- fromSlug := fromSlugs[0]
- toSlug := toSlugs[0]
- if fromSlug == toSlug {
- continue
- }
- if i, ok := slugToIndex[fromSlug]; ok {
- pages[i].RelatedKB = append(pages[i].RelatedKB, toSlug)
- }
- if i, ok := slugToIndex[toSlug]; ok {
- pages[i].RelatedKB = append(pages[i].RelatedKB, fromSlug)
- }
- }
- for i := range pages {
- pages[i].RelatedKB = uniqueStrings(pages[i].RelatedKB)
- }
- plan := wikiPlan{Pages: pages}
- plan.Pages = normalizeWikiPlanPages(plan.Pages, reduced)
- return plan
-}
-
func buildWikiFallbackPages(reduced wikiExtract) []wikiPlanPage {
var out []wikiPlanPage
seen := map[string]bool{}
@@ -1967,15 +2027,17 @@ func buildWikiFallbackPages(reduced wikiExtract) []wikiPlanPage {
}
if len(out) == 0 {
for _, t := range reduced.Topics {
+ title := common.WikiTopicLeaf(t.Path)
+ lead := firstNonEmpty(t.Description, title)
appendPage(wikiPlanPage{
Action: "CREATE",
- Slug: "topic/" + slugify(t),
- Title: t,
+ Slug: "topic/" + slugify(title),
+ Title: title,
PageType: "topic",
- Topic: t,
+ Topic: t.Path,
Priority: len(out) + 1,
- Lead: t,
- Sections: []wikiPlanSection{{Heading: "Overview", Points: []string{t}}},
+ Lead: lead,
+ Sections: []wikiPlanSection{{Heading: "Overview", Points: []string{lead}}},
})
}
}
@@ -2001,7 +2063,7 @@ func firstPlanTitle(pages []wikiPlanPage, reduced wikiExtract, docID string) str
}
}
if len(reduced.Topics) > 0 {
- return reduced.Topics[0]
+ return common.WikiTopicLeaf(reduced.Topics[0].Path)
}
if len(reduced.Entities) > 0 {
return reduced.Entities[0].Name
@@ -2218,6 +2280,65 @@ func parseWikiRelations(raw any) []wikiRelation {
return out
}
+func parseWikiTopics(raw any) []wikiTopic {
+ arr, ok := raw.([]any)
+ if !ok {
+ return nil
+ }
+ out := make([]wikiTopic, 0, len(arr))
+ for _, item := range arr {
+ // Accept the old string form as a read-only input compatibility path.
+ // Newly generated MAP payloads still use the structured object form so
+ // source-chunk provenance is retained.
+ if path := common.NormalizeWikiTopicPath(firstString(item)); path != "" {
+ out = append(out, wikiTopic{Path: path})
+ continue
+ }
+ m, ok := item.(map[string]any)
+ if !ok {
+ continue
+ }
+ topic := wikiTopic{
+ Path: common.NormalizeWikiTopicPath(firstString(m["path"])),
+ Description: strings.TrimSpace(firstString(m["description"])),
+ SourceChunkIDs: parseWikiStrings(m["source_chunk_ids"]),
+ }
+ if len(topic.SourceChunkIDs) == 0 {
+ if chunkID := strings.TrimSpace(firstString(m["source_chunk_id"])); chunkID != "" {
+ topic.SourceChunkIDs = []string{chunkID}
+ }
+ }
+ if topic.Path != "" {
+ out = append(out, topic)
+ }
+ }
+ return normalizeWikiTopics(out)
+}
+
+func normalizeWikiTopics(topics []wikiTopic) []wikiTopic {
+ out := make([]wikiTopic, 0, len(topics))
+ indexes := make(map[string]int, len(topics))
+ for _, topic := range topics {
+ topic.Path = common.NormalizeWikiTopicPath(topic.Path)
+ key := normKey(topic.Path)
+ if key == "" {
+ continue
+ }
+ topic.Description = strings.TrimSpace(topic.Description)
+ topic.SourceChunkIDs = uniqueStrings(topic.SourceChunkIDs)
+ if index, exists := indexes[key]; exists {
+ if out[index].Description == "" {
+ out[index].Description = topic.Description
+ }
+ out[index].SourceChunkIDs = mergeStrings(out[index].SourceChunkIDs, topic.SourceChunkIDs)
+ continue
+ }
+ indexes[key] = len(out)
+ out = append(out, topic)
+ }
+ return out
+}
+
func parseWikiPlanSections(raw any) []wikiPlanSection {
arr, ok := raw.([]any)
if !ok {
@@ -2324,7 +2445,7 @@ func buildEvidenceChecklist(reduced wikiExtract) string {
lines = append(lines, fmt.Sprintf("- relation: %s -> %s (%s)", r.From, r.To, r.Type))
}
for _, t := range reduced.Topics {
- lines = append(lines, fmt.Sprintf("- topic: %s", t))
+ lines = append(lines, fmt.Sprintf("- topic: %s", t.Path))
}
if len(lines) == 0 {
return "- (none)"
@@ -2974,6 +3095,11 @@ func (e wikiExtract) sourceChunkIDs() []string {
add(id)
}
}
+ for _, item := range e.Topics {
+ for _, id := range item.SourceChunkIDs {
+ add(id)
+ }
+ }
return out
}
diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_incremental_state.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_incremental_state.go
index aa497772c8..79ddc55dd1 100644
--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki_incremental_state.go
+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_incremental_state.go
@@ -51,7 +51,8 @@ func (p *wikiPipeline) addAffectedExtractTerms(extract wikiExtract) {
add(relation.To)
}
for _, topic := range extract.Topics {
- add(topic)
+ add(topic.Path)
+ add(common.WikiTopicLeaf(topic.Path))
}
}
@@ -97,6 +98,11 @@ func (p *wikiPipeline) pageTouchesAffectedTerms(page wikiPlanPage) bool {
return true
}
}
+ if leaf := normalizedWikiTerm(common.WikiTopicLeaf(page.Topic)); leaf != "" {
+ if _, ok := p.affectedTerms[leaf]; ok {
+ return true
+ }
+ }
for _, name := range page.EntityNames {
if _, ok := p.affectedTerms[normalizedWikiTerm(name)]; ok {
return true
diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache.go
index cf4eafbbcc..ae61daa703 100644
--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache.go
+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache.go
@@ -26,6 +26,8 @@ import (
"ragflow/internal/ingestion/component/knowledge_compiler/common"
)
+const wikiMapActiveStateSchemaVersion = "topic-path-v2"
+
type wikiMapChunkVersion struct {
index int
chunk common.Chunk
@@ -43,7 +45,8 @@ func (p *wikiPipeline) runVersionedMap() error {
}
llmFingerprint := wikiMapHash(p.llmID)
p.activeStateKey = wikiMapHash(strings.Join([]string{
- p.tenantID, p.datasetID, p.docID, templateFingerprint, llmFingerprint, "active",
+ p.tenantID, p.datasetID, p.docID, templateFingerprint, llmFingerprint,
+ wikiMapActiveStateSchemaVersion, "active",
}, "\x00"))
p.previousActiveState = wikiMapActiveSnapshot{Chunks: map[string]wikiMapActiveChunk{}}
if activeStore, ok := p.deps.WikiMapVersions.(common.WikiMapActiveStateStore); ok {
@@ -275,9 +278,14 @@ func splitWikiExtractByChunk(extract wikiExtract, batch []common.Chunk) map[stri
out[chunkID] = part
}
}
- for chunkID, part := range out {
- part.Topics = append(part.Topics, extract.Topics...)
- out[chunkID] = part
+ for _, topic := range extract.Topics {
+ for _, chunkID := range wikiMapItemChunkIDs(topic.SourceChunkIDs, known) {
+ part := out[chunkID]
+ topicCopy := topic
+ topicCopy.SourceChunkIDs = []string{chunkID}
+ part.Topics = append(part.Topics, topicCopy)
+ out[chunkID] = part
+ }
}
return out
}
diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache_test.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache_test.go
index 8a890e6eb9..31ae146247 100644
--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache_test.go
+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache_test.go
@@ -193,6 +193,10 @@ func TestSplitWikiExtractByChunkDropsUnknownProvenance(t *testing.T) {
{Name: "Ghost", SourceChunkIDs: []string{"unknown"}},
},
Claims: []wikiClaim{{Statement: "shared", Subject: "Alpha"}},
+ Topics: []wikiTopic{
+ {Path: "Knowledge/Alpha", SourceChunkIDs: []string{"c1"}},
+ {Path: "Knowledge/Shared"},
+ },
}, []common.Chunk{{ID: "c1"}, {ID: "c2"}})
if got := len(parts["c1"].Entities); got != 1 {
@@ -207,4 +211,10 @@ func TestSplitWikiExtractByChunkDropsUnknownProvenance(t *testing.T) {
if got := len(parts["c2"].Claims); got != 1 {
t.Fatalf("c2 claims = %d, want source-less claim fallback", got)
}
+ if got := len(parts["c1"].Topics); got != 2 {
+ t.Fatalf("c1 topics = %d, want cited and source-less topics", got)
+ }
+ if got := len(parts["c2"].Topics); got != 1 || parts["c2"].Topics[0].Path != "Knowledge/Shared" {
+ t.Fatalf("c2 topics = %#v, want only source-less fallback", parts["c2"].Topics)
+ }
}
diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_test.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_test.go
index 11b12dfcec..6bfacd93d1 100644
--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki_test.go
+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_test.go
@@ -2,6 +2,7 @@ package wiki
import (
"context"
+ "reflect"
"slices"
"strings"
"sync"
@@ -33,6 +34,59 @@ func TestReduceExtracts_MergesProvenance(t *testing.T) {
}
}
+func TestParseWikiExtractNormalizesTopicPathAndProvenance(t *testing.T) {
+ extract := parseWikiExtract(map[string]any{
+ "topics": []any{map[string]any{
+ "path": " 三国演义 / 人物 / 蜀汉人物 ", "description": "蜀汉人物", "source_chunk_id": "c1",
+ }},
+ })
+ if len(extract.Topics) != 1 {
+ t.Fatalf("topics = %#v, want one", extract.Topics)
+ }
+ topic := extract.Topics[0]
+ if topic.Path != "三国演义/人物/蜀汉人物" || !slices.Equal(topic.SourceChunkIDs, []string{"c1"}) {
+ t.Fatalf("topic = %#v", topic)
+ }
+}
+
+func TestParseWikiExtractAcceptsLegacyStringTopics(t *testing.T) {
+ extract := parseWikiExtract(map[string]any{
+ "topics": []any{"人物/汉末/曹魏", "桃园结义"},
+ })
+ if len(extract.Topics) != 2 {
+ t.Fatalf("topics = %#v, want two", extract.Topics)
+ }
+ if extract.Topics[0].Path != "人物/汉末/曹魏" || extract.Topics[1].Path != "桃园结义" {
+ t.Fatalf("topics = %#v", extract.Topics)
+ }
+}
+
+func TestWikiTemplateCustomRulesPrefersGlobalRules(t *testing.T) {
+ got := wikiTemplateCustomRules(map[string]any{
+ "global_rules": " Use configured topics. ",
+ "guideline": map[string]any{
+ "rules_for_entities": "entity fallback",
+ "rules_for_relations": "relation fallback",
+ },
+ }, "English")
+ if got != "Use configured topics." {
+ t.Fatalf("custom rules = %q, want global rules", got)
+ }
+}
+
+func TestWikiTemplateCustomRulesFallsBackToGuideline(t *testing.T) {
+ got := wikiTemplateCustomRules(map[string]any{
+ "global_rules": " ",
+ "guideline": map[string]any{
+ "rules_for_entities": "extract configured entities",
+ "rules_for_relations": "extract configured relations",
+ },
+ }, "English")
+ if !strings.Contains(got, "extract configured entities") || !strings.Contains(got, "extract configured relations") {
+ t.Fatalf("custom rules = %q, want guideline fallback", got)
+ }
+}
+
func TestPackWikiPlanBatches_SplitsLargeInput(t *testing.T) {
reduced := wikiExtract{
Entities: []wikiEntity{
@@ -103,7 +157,7 @@ func TestRunMapBatches_PreservesBatchOrderWithSubmitter(t *testing.T) {
if batch[0].ID == "slow" {
time.Sleep(25 * time.Millisecond)
}
- return wikiExtract{Topics: []string{batch[0].ID}}, nil
+ return wikiExtract{Topics: []wikiTopic{{Path: batch[0].ID}}}, nil
})
if err != nil {
t.Fatalf("runMapBatches err = %v", err)
@@ -112,7 +166,7 @@ func TestRunMapBatches_PreservesBatchOrderWithSubmitter(t *testing.T) {
t.Fatalf("runMapBatches len = %d, want %d", len(got), len(batches))
}
for i, want := range []string{"slow", "fast-1", "fast-2"} {
- if len(got[i].Topics) != 1 || got[i].Topics[0] != want {
+ if len(got[i].Topics) != 1 || got[i].Topics[0].Path != want {
t.Fatalf("runMapBatches[%d] = %#v, want topic %q", i, got[i], want)
}
}
@@ -145,6 +199,86 @@ func TestNormalizeWikiPlanPages_FallbacksToEntitiesAndConcepts(t *testing.T) {
}
}
+func TestNormalizeWikiPlanPagesExpandsUniqueMAPTopicLeaf(t *testing.T) {
+ pages := normalizeWikiPlanPages([]wikiPlanPage{{
+ Slug: "entity/刘备", Title: "刘备", PageType: "entity", Topic: "蜀汉人物",
+ }}, wikiExtract{Topics: []wikiTopic{{Path: "三国演义 / 人物 / 蜀汉人物"}}})
+ if len(pages) != 1 {
+ t.Fatalf("pages = %#v, want one page", pages)
+ }
+ if got := pages[0].Topic; got != "三国演义/人物/蜀汉人物" {
+ t.Fatalf("topic = %q, want complete MAP topic path", got)
+ }
+}
+
+func TestNormalizeWikiPlanPagesFallsBackForUnknownTopic(t *testing.T) {
+ pages := normalizeWikiPlanPages([]wikiPlanPage{{
+ Slug: "entity/刘备", Title: "刘备", PageType: "entity", Topic: "历史 / 人物 / 蜀汉人物",
+ }}, wikiExtract{Topics: []wikiTopic{{Path: "文学/人物"}}})
+ if got := pages[0].Topic; got != common.GeneralWikiTopic {
+ t.Fatalf("topic = %q, want %q", got, common.GeneralWikiTopic)
+ }
+}
+
+func TestNormalizeWikiPlanPagesDoesNotUseEntityTitleAsTopic(t *testing.T) {
+ pages := normalizeWikiPlanPages([]wikiPlanPage{{
+ Slug: "entity/person/曹操", Title: "曹操", PageType: "entity", Topic: "人物/曹操",
+ }}, wikiExtract{Topics: []wikiTopic{{Path: "人物/曹操"}, {Path: "人物/汉末/曹魏"}}})
+ if got := pages[0].Topic; got != common.GeneralWikiTopic {
+ t.Fatalf("topic = %q, want %q", got, common.GeneralWikiTopic)
+ }
+}
+
+type topicPathEmbedStub struct{}
+
+func (topicPathEmbedStub) Encode(_ context.Context, texts []string) ([][]float32, error) {
+ out := make([][]float32, len(texts))
+ for i, text := range texts {
+ switch {
+ case strings.Contains(text, "刘备"), strings.Contains(text, "蜀汉"):
+ out[i] = []float32{1, 0}
+ case strings.Contains(text, "曹操"), strings.Contains(text, "曹魏"):
+ out[i] = []float32{0, 1}
+ default:
+ out[i] = []float32{0.5, 0.5}
+ }
+ }
+ return out, nil
+}
+
+func (topicPathEmbedStub) Dimensions() int { return 2 }
+
+func TestBuildTopicCandidateCommunitiesAssignsMAPTopicPaths(t *testing.T) {
+ p := &wikiPipeline{
+ ctx: context.Background(),
+ deps: common.Deps{Embed: topicPathEmbedStub{}},
+ reduced: wikiExtract{
+ Entities: []wikiEntity{{Name: "刘备"}, {Name: "曹操"}},
+ Topics: []wikiTopic{
+ {Path: "三国演义/人物/蜀汉人物"},
+ {Path: "三国演义/人物/曹魏人物"},
+ },
+ },
+ }
+ communities := p.buildTopicCandidateCommunities()
+ if len(communities) != 2 {
+ t.Fatalf("communities = %#v, want two", communities)
+ }
+ for _, community := range communities {
+ if len(community.Entities) != 1 || len(community.Topics) != 1 {
+ t.Fatalf("community = %#v, want one entity and one topic", community)
+ }
+ entity := community.Entities[0].Name
+ topic := community.Topics[0].Path
+ if entity == "刘备" && topic != "三国演义/人物/蜀汉人物" {
+ t.Fatalf("刘备 topic = %q", topic)
+ }
+ if entity == "曹操" && topic != "三国演义/人物/曹魏人物" {
+ t.Fatalf("曹操 topic = %q", topic)
+ }
+ }
+}
+
func TestNormalizeWikiPlanPages_DedupSameTypeSameTitle(t *testing.T) {
// Scheme A: two entity pages with the same title but different slug
// transliterations (lu-bu vs lv-bu) collapse to one, and their RelatedKB
@@ -279,6 +413,22 @@ func TestMergePlanCandidates_DeduplicatesWithoutLLMMerge(t *testing.T) {
}
}
+func TestAssembleWikiPlanRelatedPagesFromRelations(t *testing.T) {
+ pages := []wikiPlanPage{
+ {Slug: "entity/alpha", Title: "Alpha", EntityNames: []string{"Alpha"}},
+ {Slug: "entity/beta", Title: "Beta", EntityNames: []string{"Beta"}},
+ }
+ relations := []wikiRelation{{From: "Alpha", To: "Beta", Type: "related"}}
+
+ got := assembleWikiPlanRelatedPages(pages, relations)
+ if want := []string{"entity/beta"}; !reflect.DeepEqual(got[0].RelatedKB, want) {
+ t.Fatalf("Alpha related links = %v, want %v", got[0].RelatedKB, want)
+ }
+ if want := []string{"entity/alpha"}; !reflect.DeepEqual(got[1].RelatedKB, want) {
+ t.Fatalf("Beta related links = %v, want %v", got[1].RelatedKB, want)
+ }
+}
+
type reconcileChatStub struct {
resp string
}
diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_topic_plan.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_topic_plan.go
index 77f691b22f..46b59e5cd7 100644
--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki_topic_plan.go
+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_topic_plan.go
@@ -6,6 +6,7 @@ import (
"strings"
appcommon "ragflow/internal/common"
+ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
"go.uber.org/zap"
)
@@ -44,12 +45,22 @@ func (p *wikiPipeline) buildTopicCandidateCommunities() []wikiExtract {
if len(items) < 2 || p.deps.Embed == nil {
return []wikiExtract{p.reduced}
}
- texts := make([]string, len(items))
+ topics := wikiTopicPaths(p.reduced.Topics)
+ texts := make([]string, 0, len(items)+len(topics))
for i := range items {
- texts[i] = items[i].name
+ texts = append(texts, items[i].name)
+ }
+ for i, topic := range topics {
+ text := topic
+ if i < len(p.reduced.Topics) {
+ if description := strings.TrimSpace(p.reduced.Topics[i].Description); description != "" {
+ text += "\n" + description
+ }
+ }
+ texts = append(texts, text)
}
vectors, err := p.deps.Embed.Encode(p.ctx, texts)
- if err != nil || len(vectors) != len(items) {
+ if err != nil || len(vectors) != len(texts) {
return []wikiExtract{p.reduced}
}
for i := range items {
@@ -77,10 +88,10 @@ func (p *wikiPipeline) buildTopicCandidateCommunities() []wikiExtract {
communities[placed] = append(communities[placed], item)
}
}
+ topicCommunityIndexes := p.assignTopicCommunities(communities, topics, vectors[len(items):])
residual := wikiExtract{}
assignedClaims := make([]bool, len(p.reduced.Claims))
assignedRelations := make([]bool, len(p.reduced.Relations))
- assignedTopics := make([]bool, len(p.reduced.Topics))
out := make([]wikiExtract, 0, len(communities))
for i, community := range communities {
nameSet := make(map[string]struct{}, len(community))
@@ -113,23 +124,74 @@ func (p *wikiPipeline) buildTopicCandidateCommunities() []wikiExtract {
}
}
for topicIndex, topic := range p.reduced.Topics {
- if _, ok := nameSet[normKey(topic)]; ok {
+ if topicIndex < len(topicCommunityIndexes) && topicCommunityIndexes[topicIndex] == i {
extract.Topics = append(extract.Topics, topic)
- assignedTopics[topicIndex] = true
- } else if i == len(communities)-1 && !assignedTopics[topicIndex] {
+ } else if i == len(communities)-1 && (topicIndex >= len(topicCommunityIndexes) || topicCommunityIndexes[topicIndex] < 0) {
residual.Topics = append(residual.Topics, topic)
}
}
- extract.Topics = uniqueStrings(extract.Topics)
+ extract.Topics = normalizeWikiTopics(extract.Topics)
out = append(out, extract)
}
if len(residual.Claims) > 0 || len(residual.Relations) > 0 || len(residual.Topics) > 0 {
- residual.Topics = uniqueStrings(residual.Topics)
+ residual.Topics = normalizeWikiTopics(residual.Topics)
out = append(out, residual)
}
return out
}
+// assignTopicCommunities attaches every MAP topic path to the most relevant
+// entity/concept community. Embedding is used only for candidate placement;
+// the planner still decides the final page-to-topic assignment.
+func (p *wikiPipeline) assignTopicCommunities(communities [][]topicCommunityItem, topics []string, vectors [][]float32) []int {
+ assignments := make([]int, len(p.reduced.Topics))
+ for i := range assignments {
+ assignments[i] = -1
+ }
+ if len(assignments) == 0 || len(communities) == 0 {
+ return assignments
+ }
+
+ if len(topics) != len(p.reduced.Topics) {
+ return assignments
+ }
+ if len(vectors) == len(topics) {
+ for topicIndex, vector := range vectors {
+ bestCommunity := -1
+ bestScore := -1.0
+ for communityIndex, community := range communities {
+ for _, member := range community {
+ score := cosine32(vector, member.vector)
+ if score > bestScore {
+ bestScore = score
+ bestCommunity = communityIndex
+ }
+ }
+ }
+ assignments[topicIndex] = bestCommunity
+ }
+ return assignments
+ }
+
+ // Missing topic vectors must not block compilation. Retain an exact
+ // leaf/name fallback and leave unmatched paths for the residual planner batch.
+ for topicIndex, topic := range topics {
+ leaf := normKey(kccommon.WikiTopicLeaf(topic))
+ for communityIndex, community := range communities {
+ for _, member := range community {
+ if leaf != "" && leaf == normKey(member.name) {
+ assignments[topicIndex] = communityIndex
+ break
+ }
+ }
+ if assignments[topicIndex] >= 0 {
+ break
+ }
+ }
+ }
+ return assignments
+}
+
func (p *wikiPipeline) runTopicPlan() (wikiPlan, error) {
communities := p.buildTopicCandidateCommunities()
if len(communities) == 0 {
diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.go
index ba55469cd9..c020f1200f 100644
--- a/internal/ingestion/knowledge_compile/consumer.go
+++ b/internal/ingestion/knowledge_compile/consumer.go
@@ -1173,9 +1173,11 @@ func (c *Consumer) mergeWikiProductsBySlug(ctx context.Context, tenant, kb strin
merged := make([]kccommon.Product, 0, len(order))
rewriteIndexes := make([]int, 0, len(order))
for _, key := range order {
+ group := groups[key]
+ topic := selectMergedWikiTopicPath(group)
var current kccommon.Product
exists := false
- for _, candidate := range groups[key] {
+ for _, candidate := range group {
candidate.Merged = true
candidate.DocID = kb
if !exists || current.ID == "" {
@@ -1190,10 +1192,12 @@ func (c *Consumer) mergeWikiProductsBySlug(ctx context.Context, tenant, kb strin
}
}
if current.ID != "" {
+ current.Meta = copyMeta(current.Meta)
+ current.Meta["topic"] = topic
current.Merged = true
current.DocID = kb
merged = append(merged, current)
- if len(groups[key]) > 1 {
+ if len(group) > 1 {
rewriteIndexes = append(rewriteIndexes, len(merged)-1)
}
}
diff --git a/internal/ingestion/knowledge_compile/topic_merge.go b/internal/ingestion/knowledge_compile/topic_merge.go
index c84042d338..1638a2be3d 100644
--- a/internal/ingestion/knowledge_compile/topic_merge.go
+++ b/internal/ingestion/knowledge_compile/topic_merge.go
@@ -8,7 +8,7 @@ import (
kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
)
-const topicRouteSystemPrompt = "You route incoming Wiki pages to an existing knowledge-base topic. Decide by semantic topic relevance, not title similarity alone. Return only JSON: {\"merge\":true|false,\"topic\":\"canonical topic label\"}. Set merge=true only when the incoming page belongs to the candidate topic. Set merge=false when it should remain a separate topic."
+const topicRouteSystemPrompt = "You route incoming Wiki pages to an existing knowledge-base topic path. A topic uses '/' as a hierarchy separator. Decide by semantic topic relevance, not title similarity alone. Return only JSON: {\"merge\":true|false,\"topic\":\"canonical/materialized/topic/path\"}. Set merge=true only when the incoming page belongs to the candidate topic path. When merge=true, copy the candidate topic path exactly. Set merge=false when it should remain a separate topic."
type topicRouter interface {
RouteTopic(ctx context.Context, incoming, existing kccommon.Product) (bool, string, error)
@@ -29,7 +29,12 @@ func (x *llmDeduper) RouteTopic(ctx context.Context, incoming, existing kccommon
}
merge, _ := result["merge"].(bool)
topic, _ := result["topic"].(string)
- return merge, strings.TrimSpace(topic), nil
+ if merge {
+ if candidateTopic := productTopic(existing); candidateTopic != "" {
+ return true, candidateTopic, nil
+ }
+ }
+ return merge, kccommon.NormalizeWikiTopicPath(topic), nil
}
func isTopicPage(product kccommon.Product) bool {
@@ -37,7 +42,7 @@ func isTopicPage(product kccommon.Product) bool {
}
func productTopic(product kccommon.Product) string {
- return strings.TrimSpace(metaString(product.Meta, "topic"))
+ return kccommon.NormalizeWikiTopicPath(metaString(product.Meta, "topic"))
}
func productTitle(product kccommon.Product) string {
@@ -65,7 +70,7 @@ func firstTopicString(values ...string) string {
}
func topicKey(topic string) string {
- return strings.ToLower(strings.Join(strings.Fields(strings.TrimSpace(topic)), " "))
+ return strings.ToLower(kccommon.NormalizeWikiTopicPath(topic))
}
// prepareTopicProduct normalizes topic metadata for legacy routing code. The
@@ -73,7 +78,7 @@ func topicKey(topic string) string {
// caller-provided stable page identity and is never replaced by a hash.
func prepareTopicProduct(product kccommon.Product, topic string) kccommon.Product {
product.Meta = copyMeta(product.Meta)
- topic = strings.TrimSpace(topic)
+ topic = kccommon.NormalizeWikiTopicPath(topic)
if topic == "" {
topic = productTopic(product)
}
@@ -149,7 +154,7 @@ func mergeTopicPage(existing, incoming kccommon.Product) kccommon.Product {
merged.Meta = copyMeta(merged.Meta)
merged.Meta["slug"] = metaString(existing.Meta, "slug")
merged.Meta["page_type"] = "topic"
- topic := firstTopicString(metaString(incoming.Meta, "topic"), metaString(existing.Meta, "topic"))
+ topic := kccommon.NormalizeWikiTopicPath(firstTopicString(metaString(incoming.Meta, "topic"), metaString(existing.Meta, "topic")))
merged.Meta["topic"] = topic
merged.Meta["title"] = firstTopicString(metaString(existing.Meta, "title"), metaString(incoming.Meta, "title"), topic)
merged.Meta["entity_names"] = unionStrs(metaStringSlice(existing.Meta, "entity_names"), metaStringSlice(incoming.Meta, "entity_names"))
diff --git a/internal/ingestion/knowledge_compile/topic_merge_test.go b/internal/ingestion/knowledge_compile/topic_merge_test.go
index 0a6dd86eac..a886f5a1dc 100644
--- a/internal/ingestion/knowledge_compile/topic_merge_test.go
+++ b/internal/ingestion/knowledge_compile/topic_merge_test.go
@@ -58,6 +58,13 @@ func TestMergeTopicProductsKeepsDifferentTopicsSeparate(t *testing.T) {
}
}
+func TestTopicKeyNormalizesMaterializedPath(t *testing.T) {
+ want := topicKey("三国演义/人物/蜀汉人物")
+ if got := topicKey(" 三国演义 / 人物 / 蜀汉人物 "); got != want {
+ t.Fatalf("topicKey() = %q, want %q", got, want)
+ }
+}
+
func TestMergeTopicProductsRemovesSupersededDatasetPage(t *testing.T) {
products := []kccommon.Product{
{ID: "old-page", DocID: "kb", Merged: true, Variant: kccommon.VariantWiki, Content: "old", Meta: map[string]any{
diff --git a/internal/ingestion/knowledge_compile/wiki_contribution.go b/internal/ingestion/knowledge_compile/wiki_contribution.go
index 000a4a349a..ebca83b223 100644
--- a/internal/ingestion/knowledge_compile/wiki_contribution.go
+++ b/internal/ingestion/knowledge_compile/wiki_contribution.go
@@ -246,7 +246,7 @@ func wikiContributionSignature(product kccommon.Product) string {
}{
Content: strings.TrimSpace(product.Content),
Title: strings.TrimSpace(metaString(product.Meta, "title")),
- Topic: strings.TrimSpace(metaString(product.Meta, "topic")),
+ Topic: kccommon.NormalizeWikiTopicPath(metaString(product.Meta, "topic")),
Summary: strings.TrimSpace(metaString(product.Meta, "summary")),
EntityNames: sortedUniqueCopy(metaStringSlice(product.Meta, "entity_names")),
RelatedKBPages: sortedUniqueCopy(metaStringSlice(product.Meta, "related_kb_pages")),
diff --git a/internal/ingestion/knowledge_compile/wiki_merge.go b/internal/ingestion/knowledge_compile/wiki_merge.go
index 4a843d21e7..df1d622236 100644
--- a/internal/ingestion/knowledge_compile/wiki_merge.go
+++ b/internal/ingestion/knowledge_compile/wiki_merge.go
@@ -42,6 +42,61 @@ func wikiEntityMerge(existing, incoming kccommon.Product) kccommon.Product {
return merged
}
+// selectMergedWikiTopicPath chooses the materialized path with the strongest
+// source support when document pages with the same canonical slug disagree.
+// No additional metadata is persisted: the selected path remains the page's
+// single topic value.
+func selectMergedWikiTopicPath(products []kccommon.Product) string {
+ type support struct {
+ topic string
+ docs map[string]struct{}
+ chunks map[string]struct{}
+ products int
+ }
+ byKey := make(map[string]*support)
+ for _, product := range products {
+ topic := productTopic(product)
+ key := topicKey(topic)
+ if key == "" {
+ continue
+ }
+ current := byKey[key]
+ if current == nil {
+ current = &support{topic: topic, docs: make(map[string]struct{}), chunks: make(map[string]struct{})}
+ byKey[key] = current
+ }
+ current.products++
+ docIDs := metaStringSliceAny(product.Meta, "source_doc_ids")
+ if len(docIDs) == 0 && product.DocID != "" {
+ docIDs = []string{product.DocID}
+ }
+ for _, docID := range docIDs {
+ if docID = strings.TrimSpace(docID); docID != "" {
+ current.docs[docID] = struct{}{}
+ }
+ }
+ for _, chunkID := range metaStringSliceAny(product.Meta, "source_chunk_ids") {
+ if chunkID = strings.TrimSpace(chunkID); chunkID != "" {
+ current.chunks[chunkID] = struct{}{}
+ }
+ }
+ }
+
+ var best *support
+ for _, candidate := range byKey {
+ if best == nil || len(candidate.docs) > len(best.docs) ||
+ (len(candidate.docs) == len(best.docs) && len(candidate.chunks) > len(best.chunks)) ||
+ (len(candidate.docs) == len(best.docs) && len(candidate.chunks) == len(best.chunks) && candidate.products > best.products) ||
+ (len(candidate.docs) == len(best.docs) && len(candidate.chunks) == len(best.chunks) && candidate.products == best.products && topicKey(candidate.topic) < topicKey(best.topic)) {
+ best = candidate
+ }
+ }
+ if best == nil {
+ return ""
+ }
+ return best.topic
+}
+
func unionWikiMarkdown(left, right string) string {
left, right = strings.TrimSpace(left), strings.TrimSpace(right)
if left == "" {
diff --git a/internal/ingestion/knowledge_compile/wiki_merge_test.go b/internal/ingestion/knowledge_compile/wiki_merge_test.go
index 9361e69638..e0f1a2cbdb 100644
--- a/internal/ingestion/knowledge_compile/wiki_merge_test.go
+++ b/internal/ingestion/knowledge_compile/wiki_merge_test.go
@@ -35,3 +35,20 @@ func TestMergeWikiProductsBySlugKeepsSingleDocumentPageContent(t *testing.T) {
t.Fatalf("dataset page identity not applied: doc_id=%q merged=%t", merged[0].DocID, merged[0].Merged)
}
}
+
+func TestSelectMergedWikiTopicPathUsesSourceSupport(t *testing.T) {
+ products := []kccommon.Product{
+ {DocID: "doc-1", Meta: map[string]any{
+ "topic": "三国演义/人物/蜀汉人物", "source_doc_ids": []string{"doc-1"}, "source_chunk_ids": []string{"c1"},
+ }},
+ {DocID: "doc-2", Meta: map[string]any{
+ "topic": "三国演义 / 人物 / 蜀汉人物", "source_doc_ids": []string{"doc-2"}, "source_chunk_ids": []string{"c2"},
+ }},
+ {DocID: "doc-3", Meta: map[string]any{
+ "topic": "文学/人物", "source_doc_ids": []string{"doc-3"}, "source_chunk_ids": []string{"c3", "c4", "c5"},
+ }},
+ }
+ if got := selectMergedWikiTopicPath(products); got != "三国演义/人物/蜀汉人物" {
+ t.Fatalf("selected topic = %q, want path supported by two documents", got)
+ }
+}
diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.go
index 437d8ce2a5..0c35a2437e 100644
--- a/internal/ingestion/knowledge_compile/writer.go
+++ b/internal/ingestion/knowledge_compile/writer.go
@@ -699,7 +699,7 @@ func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccomm
if pageType != "" {
m["page_type_kwd"] = pageType
}
- if v := metaString(p.Meta, "topic"); v != "" {
+ if v := kccommon.NormalizeWikiTopicPath(metaString(p.Meta, "topic")); v != "" {
m["topic_kwd"] = v
}
if v := metaString(p.Meta, "plan_group"); v != "" {
diff --git a/internal/ingestion/knowledge_compile/writer_test.go b/internal/ingestion/knowledge_compile/writer_test.go
index 9056c1970b..32a284c7fe 100644
--- a/internal/ingestion/knowledge_compile/writer_test.go
+++ b/internal/ingestion/knowledge_compile/writer_test.go
@@ -22,7 +22,7 @@ func TestMergedChunkMapKeepsWikiFields(t *testing.T) {
"slug": "entity/alpha",
"title": "Alpha",
"page_type": "entity",
- "topic": "Alpha",
+ "topic": " Knowledge / Core / Alpha ",
"summary": "A page about Alpha",
"entity_names": []string{"Alpha"},
"related_kb_pages": []string{"entity/beta"},
@@ -38,7 +38,7 @@ func TestMergedChunkMapKeepsWikiFields(t *testing.T) {
"artifact_slug_kwd": "entity/alpha",
"title_kwd": "Alpha",
"page_type_kwd": "entity",
- "topic_kwd": "Alpha",
+ "topic_kwd": "Knowledge/Core/Alpha",
"summary_with_weight": "A page about Alpha",
"plan_kwd": "run-abc",
"input_hash_kwd": "hash-123",
diff --git a/internal/service/compilation_template_service.go b/internal/service/compilation_template_service.go
index b647dce249..6d77141107 100644
--- a/internal/service/compilation_template_service.go
+++ b/internal/service/compilation_template_service.go
@@ -25,6 +25,7 @@ import (
"sort"
"strings"
"unicode"
+ "unicode/utf8"
"ragflow/internal/dao"
"ragflow/internal/entity"
@@ -199,7 +200,7 @@ func ValidateTemplatePayload(req map[string]interface{}, requireAll bool) error
}
}
if desc, ok := req["description"]; ok {
- if descStr, ok2 := desc.(string); !ok2 || len(descStr) > 1024 {
+ if descStr, ok2 := desc.(string); !ok2 || utf8.RuneCountInString(descStr) > 1024 {
return errors.New("invalid template description")
}
}
@@ -223,7 +224,7 @@ func ValidateTemplatePayload(req map[string]interface{}, requireAll bool) error
default:
return errors.New("invalid template config")
}
- if len(fmt.Sprint(configMap["global_rules"])) > 4096 {
+ if utf8.RuneCountInString(yamlStr(configMap["global_rules"])) > 4096 {
return errors.New("global compilation rules is too long")
}
for _, section := range []string{"entity", "relation"} {
@@ -243,10 +244,10 @@ func ValidateTemplatePayload(req map[string]interface{}, requireAll bool) error
if strings.TrimSpace(yamlStr(fm["description"])) == "" {
return fmt.Errorf("%s field description is required", capitalizeTitle(section))
}
- if len(yamlStr(fm["description"])) > 1024 {
+ if utf8.RuneCountInString(yamlStr(fm["description"])) > 1024 {
return fmt.Errorf("%s field description is too long", capitalizeTitle(section))
}
- if len(yamlStr(fm["rule"])) > 1024 {
+ if utf8.RuneCountInString(yamlStr(fm["rule"])) > 1024 {
return fmt.Errorf("%s field rule is too long", capitalizeTitle(section))
}
}
@@ -265,10 +266,10 @@ func ValidateTemplatePayload(req map[string]interface{}, requireAll bool) error
if strings.TrimSpace(yamlStr(fm["subject"])) == "" {
return errors.New("claim subject is required")
}
- if len(yamlStr(fm["statement"])) > 1024 {
+ if utf8.RuneCountInString(yamlStr(fm["statement"])) > 1024 {
return errors.New("claim statement is too long")
}
- if len(yamlStr(fm["subject"])) > 1024 {
+ if utf8.RuneCountInString(yamlStr(fm["subject"])) > 1024 {
return errors.New("claim subject is too long")
}
case "concept":
@@ -278,10 +279,10 @@ func ValidateTemplatePayload(req map[string]interface{}, requireAll bool) error
if strings.TrimSpace(yamlStr(fm["definition_excerpt"])) == "" {
return errors.New("concept definition excerpt is required")
}
- if len(yamlStr(fm["term"])) > 1024 {
+ if utf8.RuneCountInString(yamlStr(fm["term"])) > 1024 {
return errors.New("concept term is too long")
}
- if len(yamlStr(fm["definition_excerpt"])) > 1024 {
+ if utf8.RuneCountInString(yamlStr(fm["definition_excerpt"])) > 1024 {
return errors.New("concept definition excerpt is too long")
}
}
diff --git a/internal/service/compilation_template_service_test.go b/internal/service/compilation_template_service_test.go
index 7f714fee5c..710ab3f6d0 100644
--- a/internal/service/compilation_template_service_test.go
+++ b/internal/service/compilation_template_service_test.go
@@ -3,6 +3,7 @@ package service
import (
"encoding/json"
"os"
+ "strings"
"testing"
"ragflow/internal/entity"
@@ -70,6 +71,37 @@ func TestValidateTemplatePayload_AcceptsJSONMapConfig(t *testing.T) {
}
}
+func TestValidateTemplatePayloadCountsTextLimitsByRune(t *testing.T) {
+ valid := map[string]interface{}{
+ "name": "模板",
+ "description": strings.Repeat("描", 1024),
+ "kind": "graph",
+ "config": map[string]interface{}{
+ "global_rules": strings.Repeat("规", 4096),
+ "entity": map[string]interface{}{
+ "fields": []interface{}{map[string]interface{}{
+ "type": "person", "description": strings.Repeat("人", 1024), "rule": strings.Repeat("则", 1024),
+ }},
+ },
+ "relation": map[string]interface{}{"fields": []interface{}{}},
+ },
+ }
+ if err := ValidateTemplatePayload(valid, true); err != nil {
+ t.Fatalf("valid Unicode lengths rejected: %v", err)
+ }
+
+ invalid := map[string]interface{}{
+ "name": "模板",
+ "kind": "graph",
+ "config": map[string]interface{}{
+ "global_rules": strings.Repeat("规", 4097),
+ },
+ }
+ if err := ValidateTemplatePayload(invalid, true); err == nil {
+ t.Fatal("global_rules over 4096 Unicode characters should be rejected")
+ }
+}
+
// TestLoadWikiPresets_FrontendContract pins the Python API contract of
// /v1/compilation-templates/wiki-presets: every preset must expose the
// "example" JSON key filled from the yaml "example" key. The Go port once
diff --git a/internal/service/dataset_artifact_service.go b/internal/service/dataset_artifact_service.go
index ea531b4491..6c0ecb4141 100644
--- a/internal/service/dataset_artifact_service.go
+++ b/internal/service/dataset_artifact_service.go
@@ -28,6 +28,7 @@ import (
"ragflow/internal/engine"
"ragflow/internal/engine/types"
"ragflow/internal/entity"
+ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
"ragflow/internal/service/nav"
)
@@ -174,7 +175,7 @@ func (s *DatasetArtifactService) ListWikiPages(ctx context.Context, tenantID, da
filter["page_type_kwd"] = []string{pageType}
}
if topic != "" {
- filter["topic_kwd"] = []string{topic}
+ filter["topic_kwd"] = []string{kccommon.NormalizeWikiTopicPath(topic)}
}
offset := (page - 1) * pageSize
chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID, filter,
@@ -197,7 +198,7 @@ func (s *DatasetArtifactService) ListWikiPages(ctx context.Context, tenantID, da
Slug: bareSlug,
Title: firstStringValue(c["title_kwd"]),
PageType: pageType,
- Topic: firstStringValue(c["topic_kwd"]),
+ Topic: kccommon.NormalizeWikiTopicPath(firstStringValue(c["topic_kwd"])),
Summary: firstStringValue(c["summary_with_weight"]),
})
}
@@ -264,7 +265,7 @@ func (s *DatasetArtifactService) GetWikiPage(ctx context.Context, tenantID, data
Slug: detailSlug,
Title: firstStringValue(c["title_kwd"]),
PageType: detailPageType,
- Topic: firstStringValue(c["topic_kwd"]),
+ Topic: kccommon.NormalizeWikiTopicPath(firstStringValue(c["topic_kwd"])),
ContentMd: content,
Summary: firstStringValue(c["summary_with_weight"]),
EntityNames: toStringSlice(c["entity_names_kwd"]),
@@ -334,44 +335,29 @@ type WikiTopicItem struct {
PageCount int `json:"page_count"`
}
-// ListWikiTopics aggregates wiki topics for a dataset.
+// ListWikiTopics aggregates materialized Wiki topic paths for a dataset. Topic
+// is the complete path and Title is its leaf segment; the frontend may derive a
+// navigation tree by splitting Topic on '/'.
func (s *DatasetArtifactService) ListWikiTopics(ctx context.Context, tenantID, datasetID string) ([]WikiTopicItem, int64, error) {
filter := map[string]interface{}{
"compile_kwd": []string{CompileKwdWikiPage},
- "page_type_kwd": []string{"concept", "entity"},
+ "page_type_kwd": []string{"concept", "entity", "topic"},
"available_int": 1, // count only merged pages, not per-doc source rows
}
- chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter,
- []string{"topic_kwd", "title_kwd", "slug_kwd", "page_type_kwd"}, 0, 1000, nil)
- if err != nil {
- return nil, 0, err
- }
- counts := map[string]int{}
- metas := map[string]WikiTopicItem{}
- for _, c := range chunks {
- t := firstStringValue(c["topic_kwd"])
- if t == "" {
- continue
- }
- pageType := firstStringValue(c["page_type_kwd"])
- bareSlug := firstStringValue(c["slug_kwd"])
- if pageType != "" {
- bareSlug = strings.TrimPrefix(bareSlug, pageType+"/")
+ const batchSize = 1000
+ fields := []string{"topic_kwd"}
+ chunks := make([]map[string]interface{}, 0, batchSize)
+ for offset := 0; ; offset += batchSize {
+ batch, total, err := s.searchCompiled(ctx, tenantID, datasetID, filter, fields, offset, batchSize, nil)
+ if err != nil {
+ return nil, 0, err
}
- counts[t]++
- if _, ok := metas[t]; !ok {
- metas[t] = WikiTopicItem{
- Topic: t,
- Title: firstStringValue(c["title_kwd"]),
- Slug: bareSlug,
- }
+ chunks = append(chunks, batch...)
+ if len(batch) == 0 || int64(len(chunks)) >= total {
+ break
}
}
- items := make([]WikiTopicItem, 0, len(metas))
- for t, it := range metas {
- it.PageCount = counts[t]
- items = append(items, it)
- }
+ items := aggregateWikiTopicItems(chunks)
// Sort topics by a deterministic rule. Plain UTF-8 byte order is chaotic for
// CJK (it sorts by Unicode code point, unrelated to pinyin/stroke). We use a
// CLDR-based collator (golang.org/x/text/collate) with the Chinese locale,
@@ -383,6 +369,37 @@ func (s *DatasetArtifactService) ListWikiTopics(ctx context.Context, tenantID, d
return items, int64(len(items)), nil
}
+func aggregateWikiTopicItems(chunks []map[string]interface{}) []WikiTopicItem {
+ type aggregate struct {
+ topic string
+ count int
+ }
+ byKey := make(map[string]*aggregate)
+ for _, c := range chunks {
+ t := kccommon.NormalizeWikiTopicPath(firstStringValue(c["topic_kwd"]))
+ if t == "" {
+ continue
+ }
+ key := strings.ToLower(t)
+ item := byKey[key]
+ if item == nil {
+ item = &aggregate{topic: t}
+ byKey[key] = item
+ }
+ item.count++
+ }
+ items := make([]WikiTopicItem, 0, len(byKey))
+ for _, aggregate := range byKey {
+ items = append(items, WikiTopicItem{
+ Topic: aggregate.topic,
+ Title: kccommon.WikiTopicLeaf(aggregate.topic),
+ Slug: aggregate.topic,
+ PageCount: aggregate.count,
+ })
+ }
+ return items
+}
+
// wikiTopicCollator is a process-wide collator for wiki topics. language.Chinese
// selects the CLDR zh collation (pinyin-based for Han), while Latin/digits and
// other scripts sort naturally, case-insensitively. It is safe for concurrent
diff --git a/internal/service/dataset_artifact_topic_test.go b/internal/service/dataset_artifact_topic_test.go
new file mode 100644
index 0000000000..60c397ffa3
--- /dev/null
+++ b/internal/service/dataset_artifact_topic_test.go
@@ -0,0 +1,36 @@
+package service
+
+import "testing"
+
+func TestAggregateWikiTopicItemsUsesMaterializedPaths(t *testing.T) {
+ items := aggregateWikiTopicItems([]map[string]interface{}{
+ {"topic_kwd": " 三国演义 / 人物 / 蜀汉人物 "},
+ {"topic_kwd": "三国演义/人物/蜀汉人物"},
+ {"topic_kwd": "三国演义/人物/曹魏人物"},
+ })
+ if len(items) != 2 {
+ t.Fatalf("topics = %#v, want 2 complete paths", items)
+ }
+ byTopic := make(map[string]WikiTopicItem, len(items))
+ for _, item := range items {
+ byTopic[item.Topic] = item
+ }
+ shuhan := byTopic["三国演义/人物/蜀汉人物"]
+ if shuhan.Title != "蜀汉人物" || shuhan.Slug != shuhan.Topic || shuhan.PageCount != 2 {
+ t.Fatalf("蜀汉人物 item = %#v", shuhan)
+ }
+ caowei := byTopic["三国演义/人物/曹魏人物"]
+ if caowei.Title != "曹魏人物" || caowei.PageCount != 1 {
+ t.Fatalf("曹魏人物 item = %#v", caowei)
+ }
+}
+
+func TestAggregateWikiTopicItemsIgnoresPathCase(t *testing.T) {
+ items := aggregateWikiTopicItems([]map[string]interface{}{
+ {"topic_kwd": "Knowledge/Core"},
+ {"topic_kwd": "knowledge / core"},
+ })
+ if len(items) != 1 || items[0].PageCount != 2 {
+ t.Fatalf("topics = %#v, want one case-insensitive path", items)
+ }
+}
diff --git a/web/src/interfaces/database/dataset.ts b/web/src/interfaces/database/dataset.ts
index b12d397621..1b5ef2dffd 100644
--- a/web/src/interfaces/database/dataset.ts
+++ b/web/src/interfaces/database/dataset.ts
@@ -231,6 +231,7 @@ export interface IArtifactTopic {
topic: string;
title: string;
slug: string;
+ page_count?: number;
}
export interface IArtifactPage {
diff --git a/web/src/pages/dataset/compilation/wiki-left-panel/hooks/use-wiki-navigation.ts b/web/src/pages/dataset/compilation/wiki-left-panel/hooks/use-wiki-navigation.ts
index 8c3adf511b..de9a1a428a 100644
--- a/web/src/pages/dataset/compilation/wiki-left-panel/hooks/use-wiki-navigation.ts
+++ b/web/src/pages/dataset/compilation/wiki-left-panel/hooks/use-wiki-navigation.ts
@@ -8,11 +8,51 @@ import { useCallback, useMemo, useRef, useState } from 'react';
export type WikiPageType = 'concept' | 'entity' | 'topic';
+const normalizeTopicPath = (topic: string) =>
+ topic
+ .split('/')
+ .map((segment) => segment.trim())
+ .filter(Boolean)
+ .join('/');
+
+const topicLeaf = (topic: string) => {
+ const segments = topic.split('/');
+ return segments[segments.length - 1] ?? topic;
+};
+
+function getTopicChildren(topics: IArtifactTopic[], parentPath: string) {
+ const normalizedParent = normalizeTopicPath(parentPath);
+ const prefix = normalizedParent ? `${normalizedParent}/` : '';
+ const children = new Map<string, number>();
+
+ topics.forEach((topic) => {
+ const path = normalizeTopicPath(topic.topic);
+ if (!path.startsWith(prefix) || path === normalizedParent) {
+ return;
+ }
+
+ const remainder = path.slice(prefix.length);
+ const child = remainder.split('/')[0];
+ const childPath = `${prefix}${child}`;
+ children.set(
+ childPath,
+ (children.get(childPath) ?? 0) + (topic.page_count ?? 0),
+ );
+ });
+
+ return Array.from(children, ([path, pageCount]) => ({
+ topic: path,
+ title: topicLeaf(path),
+ slug: path,
+ page_count: pageCount,
+ })).sort((a, b) => a.title.localeCompare(b.title));
+}
+
export function useWikiNavigation() {
const scrollRef = useRef<HTMLDivElement>(null);
const [searchString, setSearchString] = useState('');
const debouncedSearchString = useDebounce(searchString, { wait: 500 });
- const [selectedTopic, setSelectedTopic] = useState<IArtifactTopic | null>(
+ const [selectedTopicPath, setSelectedTopicPath] = useState<string | null>(
null,
);
@@ -25,6 +65,26 @@ export function useWikiNavigation() {
keywords: debouncedSearchString,
});
+ const visibleTopics = useMemo(
+ () => getTopicChildren(topics, selectedTopicPath ?? ''),
+ [topics, selectedTopicPath],
+ );
+
+ const selectedTopic = useMemo<IArtifactTopic | null>(() => {
+ if (!selectedTopicPath) {
+ return null;
+ }
+ return {
+ topic: selectedTopicPath,
+ title: topicLeaf(selectedTopicPath),
+ slug: selectedTopicPath,
+ };
+ }, [selectedTopicPath]);
+
+ const showArtifacts = Boolean(
+ selectedTopicPath && visibleTopics.length === 0,
+ );
+
const {
artifacts,
loading: artifactLoading,
@@ -32,8 +92,8 @@ export function useWikiNavigation() {
hasMore: artifactHasMore,
} = useFetchArtifactList({
keywords: debouncedSearchString,
- topic: selectedTopic?.topic,
- enabled: !!selectedTopic,
+ topic: showArtifacts ? (selectedTopicPath ?? undefined) : undefined,
+ enabled: showArtifacts,
});
const handleSearchChange = useCallback(
@@ -51,30 +111,37 @@ export function useWikiNavigation() {
const handleSelectTopic = useCallback(
(topic: IArtifactTopic) => {
- setSelectedTopic(topic);
+ setSelectedTopicPath(normalizeTopicPath(topic.topic));
+ resetScroll();
+ },
+ [resetScroll],
+ );
+
+ const handleSelectTopicPath = useCallback(
+ (path: string | null) => {
+ setSelectedTopicPath(path ? normalizeTopicPath(path) : null);
resetScroll();
},
[resetScroll],
);
const handleBackToTopics = useCallback(() => {
- setSelectedTopic(null);
- resetScroll();
- }, [resetScroll]);
+ handleSelectTopicPath(null);
+ }, [handleSelectTopicPath]);
const handleScroll = useCallback(
(e: React.UIEvent<HTMLDivElement>) => {
- if (selectedTopic) {
+ if (showArtifacts) {
handleArtifactScroll(e);
} else {
handleTopicScroll(e);
}
},
- [selectedTopic, handleArtifactScroll, handleTopicScroll],
+ [showArtifacts, handleArtifactScroll, handleTopicScroll],
);
- const loading = selectedTopic ? artifactLoading : topicLoading;
- const hasMore = selectedTopic ? artifactHasMore : topicHasMore;
+ const loading = showArtifacts ? artifactLoading : topicLoading;
+ const hasMore = showArtifacts ? artifactHasMore : topicHasMore;
return useMemo(
() => ({
@@ -82,12 +149,15 @@ export function useWikiNavigation() {
searchString,
debouncedSearchString,
selectedTopic,
- topics,
+ selectedTopicPath,
+ visibleTopics,
+ showArtifacts,
artifacts,
loading,
hasMore,
handleSearchChange,
handleSelectTopic,
+ handleSelectTopicPath,
handleBackToTopics,
handleScroll,
}),
@@ -95,12 +165,15 @@ export function useWikiNavigation() {
searchString,
debouncedSearchString,
selectedTopic,
- topics,
+ selectedTopicPath,
+ visibleTopics,
+ showArtifacts,
artifacts,
loading,
hasMore,
handleSearchChange,
handleSelectTopic,
+ handleSelectTopicPath,
handleBackToTopics,
handleScroll,
],
diff --git a/web/src/pages/dataset/compilation/wiki-left-panel/wiki-nav-bar.tsx b/web/src/pages/dataset/compilation/wiki-left-panel/wiki-nav-bar.tsx
index dbb0e7d65b..31762914b3 100644
--- a/web/src/pages/dataset/compilation/wiki-left-panel/wiki-nav-bar.tsx
+++ b/web/src/pages/dataset/compilation/wiki-left-panel/wiki-nav-bar.tsx
@@ -10,6 +10,7 @@ import { SearchInput } from '@/components/ui/input';
import { IArtifact } from '@/interfaces/database/dataset';
import { cn } from '@/lib/utils';
import { Plus } from 'lucide-react';
+import { Fragment } from 'react';
import { useTranslation } from 'react-i18next';
import { CreateDirectoryDialog } from '../create-directory-dialog';
@@ -32,12 +33,15 @@ export function WikiNavBar({
scrollRef,
searchString,
selectedTopic,
- topics,
+ selectedTopicPath,
+ visibleTopics,
+ showArtifacts,
artifacts,
loading,
hasMore,
handleSearchChange,
handleSelectTopic,
+ handleSelectTopicPath,
handleBackToTopics,
handleScroll,
} = useWikiNavigation();
@@ -50,6 +54,10 @@ export function WikiNavBar({
handleOk: handleCreateOk,
} = useCreateDirectory();
+ const selectedTopicSegments = selectedTopicPath
+ ? selectedTopicPath.split('/').filter(Boolean)
+ : [];
+
return (
<div className="size-full flex flex-col gap-3">
<SearchInput
@@ -74,16 +82,28 @@ export function WikiNavBar({
{t('knowledgeCompilation.topics')}
</BreadcrumbLink>
</BreadcrumbItem>
- {selectedTopic && (
- <>
+ {selectedTopicSegments.map((segment, index) => (
+ <Fragment key={`${segment}-${index}`}>
<BreadcrumbSeparator />
<BreadcrumbItem>
- <BreadcrumbLink className="text-text-primary text-sm cursor-default">
- {selectedTopic.title}
+ <BreadcrumbLink
+ onClick={() =>
+ handleSelectTopicPath(
+ selectedTopicSegments.slice(0, index + 1).join('/'),
+ )
+ }
+ className={cn(
+ 'text-sm',
+ index === selectedTopicSegments.length - 1
+ ? 'text-text-primary cursor-default'
+ : 'text-text-secondary cursor-pointer',
+ )}
+ >
+ {segment}
</BreadcrumbLink>
</BreadcrumbItem>
- </>
- )}
+ </Fragment>
+ ))}
</BreadcrumbList>
</Breadcrumb>
{!selectedTopic && (
@@ -110,7 +130,7 @@ export function WikiNavBar({
className="flex-1 min-h-0 overflow-y-auto pb-3"
onScroll={handleScroll}
>
- {selectedTopic ? (
+ {showArtifacts ? (
<WikiArtifactList
artifacts={artifacts}
selectedArtifact={selectedArtifact}
@@ -120,7 +140,7 @@ export function WikiNavBar({
/>
) : (
<WikiTopicList
- topics={topics}
+ topics={visibleTopics}
loading={loading}
hasMore={hasMore}
onSelectTopic={handleSelectTopic}