Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(wiki): remove duplicate Markdown storage (#20597)

adws83c6ebdf2b5e
diff --git a/conf/infinity_mapping.json b/conf/infinity_mapping.json
index 9fdbd269f7..56eb58c966 100644
--- a/conf/infinity_mapping.json
+++ b/conf/infinity_mapping.json
@@ -55,7 +55,6 @@
"chunk_hash_kwd": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},
"input_hash_kwd": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},
"wiki_slug_kwd": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},
- "md_with_weight": {"type": "varchar", "default": ""},
"summary_with_weight": {"type": "varchar", "default": ""},
"skill_with_weight": {"type": "varchar", "default": ""},
"skill_kwd": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},
diff --git a/internal/engine/infinity/chunk_transform_test.go b/internal/engine/infinity/chunk_transform_test.go
index 6543b34f49..4febde65ab 100644
--- a/internal/engine/infinity/chunk_transform_test.go
+++ b/internal/engine/infinity/chunk_transform_test.go
@@ -214,7 +214,6 @@ func compiledRowForTransform() map[string]interface{} {
"compile_kwd": "tree",
"raptor_kwd": "root",
"raptor_layer_int": 2,
- "md_with_weight": "# page",
"extra": map[string]interface{}{"raptor_method": "gmm"},
"q_3_vec": []float64{0.1, 0.2, 0.3},
}
diff --git a/internal/engine/types/compilation.go b/internal/engine/types/compilation.go
index 20aa641ce5..62cfb1f682 100644
--- a/internal/engine/types/compilation.go
+++ b/internal/engine/types/compilation.go
@@ -75,6 +75,16 @@ func WikiPageCategory(row map[string]any) string {
return compilationString(row["page_type_kwd"])
}
+// WikiPageContent prefers content_with_weight and falls back to md_with_weight.
+func WikiPageContent(row map[string]any) string {
+ for _, field := range []string{"content_with_weight", "md_with_weight"} {
+ if values := compilationValues(row[field]); len(values) > 0 && values[0] != "" {
+ return values[0]
+ }
+ }
+ return ""
+}
+
func IsNavigationRow(row map[string]any) bool {
role := compilationString(row["type_kwd"])
return role == "nav_doc" || role == "nav_cluster" || compilationString(row["compile_kwd"]) == "dataset_nav"
diff --git a/internal/engine/types/compilation_test.go b/internal/engine/types/compilation_test.go
index 694afa5b2a..78e61b224b 100644
--- a/internal/engine/types/compilation_test.go
+++ b/internal/engine/types/compilation_test.go
@@ -2,6 +2,30 @@ package types
import "testing"
+func TestWikiPageContent(t *testing.T) {
+ const body = " # Alpha\n\n[Beta](artifact/kb1/entity/beta)\n"
+ for _, test := range []struct {
+ name string
+ row map[string]any
+ want string
+ }{
+ {"canonical", map[string]any{"content_with_weight": body}, body},
+ {"canonical wins", map[string]any{"content_with_weight": body, "md_with_weight": "stale body"}, body},
+ {"Markdown only", map[string]any{"md_with_weight": body}, body},
+ {"empty canonical", map[string]any{"content_with_weight": "", "md_with_weight": body}, body},
+ {"blank canonical", map[string]any{"content_with_weight": " \n", "md_with_weight": body}, " \n"},
+ {"list values", map[string]any{"content_with_weight": []string{""}, "md_with_weight": []any{body}}, body},
+ {"canonical list", map[string]any{"content_with_weight": []string{body}, "md_with_weight": "stale body"}, body},
+ {"missing", nil, ""},
+ } {
+ t.Run(test.name, func(t *testing.T) {
+ if got := WikiPageContent(test.row); got != test.want {
+ t.Fatalf("WikiPageContent() = %q, want %q", got, test.want)
+ }
+ })
+ }
+}
+
func TestCompilationKindPrefersCanonicalValue(t *testing.T) {
for _, test := range []struct {
row map[string]any
diff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.go
index 49cbeae434..a4fe62d74e 100644
--- a/internal/ingestion/component/knowledge_compiler/component.go
+++ b/internal/ingestion/component/knowledge_compiler/component.go
@@ -540,14 +540,6 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {
return err
}
}
- // md_with_weight is the page-body column Python writes with the same
- // value as content_with_weight (wiki_incremental.py:2190-2191), i.e. the
- // rendered body; section rows have no page body of their own.
- if metaString(p.Meta, "kind") == "page" && p.Content != "" {
- if err := doc.SetExtraValue("md_with_weight", p.Content); err != nil {
- return err
- }
- }
// Section depth goes to depth_int; there is no section_level_int column.
if v, ok := metaInt(p.Meta, "section_level"); ok {
if err := doc.SetExtraValue("depth_int", v); err != nil {
diff --git a/internal/ingestion/component/knowledge_compiler/component_test.go b/internal/ingestion/component/knowledge_compiler/component_test.go
index 0090599cb7..fd604e3261 100644
--- a/internal/ingestion/component/knowledge_compiler/component_test.go
+++ b/internal/ingestion/component/knowledge_compiler/component_test.go
@@ -15,6 +15,7 @@ import (
"ragflow/internal/agent/runtime"
"ragflow/internal/ingestion/component/globals"
"ragflow/internal/ingestion/component/knowledge_compiler/common"
+ "ragflow/internal/ingestion/task/indexdoc"
"gorm.io/gorm"
)
@@ -1444,13 +1445,13 @@ func TestKnowledgeCompiler_BuildInputsAcceptsMapSliceChunks(t *testing.T) {
}
}
-// TestProductsToChunkDocs_PageVsSectionCompileKWD locks the page/section
-// discriminator (compile_kwd) and the schema-column contract: the page body goes
-// to md_with_weight and no Go-only field (kc_*, tenant_id) is emitted.
+// TestProductsToChunkDocs_WikiPageAndSectionRoles locks the page/section
+// discriminator (type_kwd) and the schema-column contract: Markdown stays in
+// content_with_weight without a duplicate body column or non-schema fields.
func TestProductsToChunkDocs_WikiPageAndSectionRoles(t *testing.T) {
page := common.Product{
ID: "page-id", DocID: "d1", TenantID: "t1", Variant: common.VariantWiki,
- Content: "# Alpha\n\nBody", ParentID: "",
+ Content: "# Alpha\n\n**Body** links to [Beta](artifact/kb1/entity/beta).", ParentID: "",
Meta: map[string]any{"kind": "page", "slug": "entity/alpha", "title": "Alpha", "page_type": "entity", "summary": "Body", "source_chunk_ids": []string{"c1"}},
}
section := common.Product{
@@ -1465,6 +1466,8 @@ func TestProductsToChunkDocs_WikiPageAndSectionRoles(t *testing.T) {
var pageKWD, sectionKWD string
var sectionParent, pageBody, sectionBody string
for _, d := range docs {
+ row := d.ToMap()
+ indexdoc.RenameTextToContentWithWeight(row)
if compiled, _ := d.GetExtraString("compile_kwd"); compiled != "wiki" {
t.Fatalf("compile_kwd = %q, want wiki", compiled)
}
@@ -1473,12 +1476,11 @@ func TestProductsToChunkDocs_WikiPageAndSectionRoles(t *testing.T) {
t.Fatalf("retired field %s was written", field)
}
}
- // compile_kwd IS the page/section discriminator (wiki_page /
- // wiki_section); there is no separate kind column.
+ // type_kwd distinguishes pages from sections within compile_kwd=wiki.
kind, _ := d.GetExtraString("type_kwd")
if kind == "wiki_page" {
pageKWD = kind
- pageBody, _ = d.GetExtraString("md_with_weight")
+ pageBody, _ = row["content_with_weight"].(string)
if summary, _ := d.GetExtraString("summary_with_weight"); summary != "Body" {
t.Errorf("page summary_with_weight = %q, want Body", summary)
}
@@ -1491,7 +1493,10 @@ func TestProductsToChunkDocs_WikiPageAndSectionRoles(t *testing.T) {
if kind == "wiki_section" {
sectionKWD = kind
sectionParent, _ = d.GetExtraString("parent_kwd")
- sectionBody, _ = d.GetExtraString("md_with_weight")
+ sectionBody, _ = row["content_with_weight"].(string)
+ }
+ if _, ok := row["md_with_weight"]; ok {
+ t.Error("Wiki rows must not duplicate the body in md_with_weight")
}
// No Go-only field: Infinity rejects an unknown column.
for k := range d.Extra {
@@ -1504,19 +1509,19 @@ func TestProductsToChunkDocs_WikiPageAndSectionRoles(t *testing.T) {
}
}
if pageKWD != "wiki_page" {
- t.Errorf("page compile_kwd = %q, want wiki_page", pageKWD)
+ t.Errorf("page type_kwd = %q, want wiki_page", pageKWD)
}
if sectionKWD != "wiki_section" {
- t.Errorf("section compile_kwd = %q, want wiki_section (schema-backed page/section discriminator)", sectionKWD)
+ t.Errorf("section type_kwd = %q, want wiki_section (schema-backed page/section discriminator)", sectionKWD)
}
if sectionParent != "page-id" {
t.Errorf("section parent_kwd = %q, want page-id", sectionParent)
}
- if pageBody != "# Alpha\n\nBody" {
- t.Errorf("page md_with_weight = %q, want the rendered page body", pageBody)
+ if pageBody != page.Content {
+ t.Errorf("page content_with_weight = %q, want %q", pageBody, page.Content)
}
- if sectionBody != "" {
- t.Errorf("section md_with_weight = %q, want empty (only page rows carry the page body)", sectionBody)
+ if sectionBody != section.Content {
+ t.Errorf("section content_with_weight = %q, want %q", sectionBody, section.Content)
}
}
diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.go
index 24bfd3079e..5b427e599d 100644
--- a/internal/ingestion/knowledge_compile/writer.go
+++ b/internal/ingestion/knowledge_compile/writer.go
@@ -333,7 +333,7 @@ func (w engineWriter) loadWikiPageContent(ctx context.Context, tenant, kb string
Offset: offset,
Limit: pageBatchSize,
SelectFields: []string{
- "slug_kwd", "page_type_kwd", "compile_kwd", "entity_type_kwd", "type_kwd", "md_with_weight", "content_with_weight",
+ "id", "slug_kwd", "page_type_kwd", "compile_kwd", "entity_type_kwd", "type_kwd", "content_with_weight",
},
Filter: map[string]interface{}{
"type_kwd": compileKwdWikiPage,
@@ -357,9 +357,14 @@ func (w engineWriter) loadWikiPageContent(ctx context.Context, tenant, kb string
if !ok {
continue
}
- content := pageEngineString(row["md_with_weight"])
+ content := types.WikiPageContent(row)
if content == "" {
- content = pageEngineString(row["content_with_weight"])
+ raw, err := eng.GetChunk(ctx, baseName, pageEngineString(row["id"]), []string{kb})
+ if err != nil {
+ return nil, err
+ }
+ stored, _ := raw.(map[string]interface{})
+ content = types.WikiPageContent(stored)
}
result[target.key] = content
}
@@ -1014,11 +1019,6 @@ func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccomm
if v := metaString(p.Meta, "summary"); v != "" {
m["summary_with_weight"] = v
}
- // The merged wiki page keeps the page body in md_with_weight (the column
- // GetWikiPage / the artifact API read), as Python does; sections have none.
- if p.Variant == kccommon.VariantWiki && metaString(p.Meta, "kind") == "page" && p.Content != "" {
- m["md_with_weight"] = p.Content
- }
if v := metaStringSlice(p.Meta, "entity_names"); len(v) > 0 {
m["entity_names_kwd"] = v
}
diff --git a/internal/ingestion/knowledge_compile/writer_test.go b/internal/ingestion/knowledge_compile/writer_test.go
index cf85feac25..47ac4bb258 100644
--- a/internal/ingestion/knowledge_compile/writer_test.go
+++ b/internal/ingestion/knowledge_compile/writer_test.go
@@ -61,13 +61,13 @@ func TestWriteMergedRecordsGeneratedPageCommitOnly(t *testing.T) {
// TestMergedChunkMapKeepsWikiFields locks the fix for the merged-row metadata
// gap: the dataset-level merged row written by mergedChunkMap must carry the
-// wiki page fields (page_type_kwd/topic_kwd/title_kwd/slug_kwd/...) that the
+// wiki page fields (entity_type_kwd/topic_kwd/title_kwd/slug_kwd/...) that the
// artifact API (ListArtifacts/ListWikiTopics) and page renderers read. Without
// them the compilation page surfaces no wiki pages from the merged products.
func TestMergedChunkMapKeepsWikiFields(t *testing.T) {
p := kccommon.Product{
ID: "merged-1", DocID: "kb1", TenantID: "t1", Variant: kccommon.VariantWiki,
- Content: "# Alpha\n\nBody",
+ Content: "# Alpha\n\n**Body** links to [Beta](entity/beta).",
Vector: []float32{0.1, 0.2, 0.3},
Meta: map[string]any{
"kind": "page",
@@ -91,12 +91,9 @@ func TestMergedChunkMapKeepsWikiFields(t *testing.T) {
"entity_type_kwd": "entity",
"topic_kwd": "Knowledge/Core/Alpha",
"summary_with_weight": "A page about Alpha",
- // The wiki page body goes to md_with_weight, the column Python writes
- // and reads (wiki_incremental.py:2190); artifact_slug_kwd was a Go-only
- // column that exists in no engine mapping.
- "md_with_weight": "# Alpha\n\nBody",
- "plan_kwd": "run-abc",
- "input_hash_kwd": "hash-123",
+ "content_with_weight": p.Content,
+ "plan_kwd": "run-abc",
+ "input_hash_kwd": "hash-123",
}
// The wall-clock audit fields must be stamped from `now`, not omitted.
if m["create_time"] != now.Format("2006-01-02 15:04:05") {
@@ -110,6 +107,9 @@ func TestMergedChunkMapKeepsWikiFields(t *testing.T) {
t.Errorf("merged row[%q] = %q, want %q", k, got, want)
}
}
+ if _, ok := m["md_with_weight"]; ok {
+ t.Fatal("merged Wiki rows must not duplicate the body in md_with_weight")
+ }
if v, _ := m["entity_names_kwd"].([]string); len(v) != 1 || v[0] != "Alpha" {
t.Errorf("entity_names_kwd = %#v, want [Alpha]", m["entity_names_kwd"])
}
diff --git a/internal/ingestion/task/knowledge_compiler_wiring.go b/internal/ingestion/task/knowledge_compiler_wiring.go
index aab3a92ba4..4ed4e6353a 100644
--- a/internal/ingestion/task/knowledge_compiler_wiring.go
+++ b/internal/ingestion/task/knowledge_compiler_wiring.go
@@ -845,11 +845,8 @@ func (s *kcWikiPageStore) FindSimilarPages(ctx context.Context, tenantID, datase
IndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},
KbIDs: []string{datasetID},
Limit: k,
- SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "compile_kwd", "type_kwd", "entity_type_kwd", "topic_kwd", "plan_group_kwd", "summary_with_weight", "content_with_weight", "md_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "_score"},
- // compile_kwd="wiki_page" is the schema-backed discriminator for wiki
- // pages (sections carry compile_kwd="wiki_section"); there is no
- // "kc_kind" column in the chunk schema, so filtering on it would return
- // empty on Infinity.
+ SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "compile_kwd", "type_kwd", "entity_type_kwd", "topic_kwd", "plan_group_kwd", "summary_with_weight", "content_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "_score"},
+ // Select pages, not sections, by their role within the Wiki compilation.
Filter: map[string]interface{}{
"type_kwd": "wiki_page",
},
@@ -868,7 +865,11 @@ func (s *kcWikiPageStore) FindSimilarPages(ctx context.Context, tenantID, datase
}
out := make([]kc.WikiPageCandidate, 0, len(res.Chunks))
for _, row := range res.Chunks {
- out = append(out, wikiPageCandidateFromRow(row))
+ candidate, err := s.loadPageCandidate(ctx, req.IndexNames[0], req.KbIDs, row)
+ if err != nil {
+ return nil, err
+ }
+ out = append(out, candidate)
}
return out, nil
}
@@ -881,7 +882,7 @@ func (s *kcWikiPageStore) GetPageBySlug(ctx context.Context, tenantID, datasetID
IndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},
KbIDs: []string{datasetID},
Limit: 1,
- SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "compile_kwd", "type_kwd", "entity_type_kwd", "topic_kwd", "plan_group_kwd", "summary_with_weight", "content_with_weight", "md_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "_score"},
+ SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "compile_kwd", "type_kwd", "entity_type_kwd", "topic_kwd", "plan_group_kwd", "summary_with_weight", "content_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "_score"},
Filter: map[string]interface{}{
"type_kwd": "wiki_page",
"slug_kwd": slug,
@@ -891,7 +892,10 @@ func (s *kcWikiPageStore) GetPageBySlug(ctx context.Context, tenantID, datasetID
if err != nil || res == nil || len(res.Chunks) == 0 {
return nil, err
}
- page := wikiPageCandidateFromRow(res.Chunks[0])
+ page, err := s.loadPageCandidate(ctx, req.IndexNames[0], req.KbIDs, res.Chunks[0])
+ if err != nil {
+ return nil, err
+ }
return &page, nil
}
@@ -903,7 +907,7 @@ func (s *kcWikiPageStore) FindPagesBySourceChunks(ctx context.Context, tenantID,
IndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},
KbIDs: []string{datasetID},
Limit: k,
- SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "compile_kwd", "type_kwd", "entity_type_kwd", "topic_kwd", "summary_with_weight", "content_with_weight", "md_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "_score"},
+ SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "compile_kwd", "type_kwd", "entity_type_kwd", "topic_kwd", "summary_with_weight", "content_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "_score"},
Filter: map[string]interface{}{
"type_kwd": "wiki_page",
"source_chunk_ids": chunkIDs,
@@ -915,7 +919,10 @@ func (s *kcWikiPageStore) FindPagesBySourceChunks(ctx context.Context, tenantID,
}
out := make([]kc.WikiPageCandidate, 0, len(res.Chunks))
for _, row := range res.Chunks {
- candidate := wikiPageCandidateFromRow(row)
+ candidate, err := s.loadPageCandidate(ctx, req.IndexNames[0], req.KbIDs, row)
+ if err != nil {
+ return nil, err
+ }
if candidate.Score == 0 {
candidate.Score = 0.68
}
@@ -924,20 +931,34 @@ func (s *kcWikiPageStore) FindPagesBySourceChunks(ctx context.Context, tenantID,
return out, nil
}
+func (s *kcWikiPageStore) loadPageCandidate(ctx context.Context, indexName string, datasetIDs []string, row map[string]interface{}) (kc.WikiPageCandidate, error) {
+ page := wikiPageCandidateFromRow(row)
+ if enginetypes.WikiPageContent(row) != "" {
+ return page, nil
+ }
+ raw, err := s.docEngine.GetChunk(ctx, indexName, page.ID, datasetIDs)
+ if err != nil {
+ return kc.WikiPageCandidate{}, err
+ }
+ stored, _ := raw.(map[string]interface{})
+ content := strings.TrimSpace(enginetypes.WikiPageContent(stored))
+ page.ContentMD = content
+ page.ContentMDRaw = content
+ return page, nil
+}
+
func wikiPageCandidateFromRow(row map[string]interface{}) kc.WikiPageCandidate {
+ content := strings.TrimSpace(enginetypes.WikiPageContent(row))
return kc.WikiPageCandidate{
- ID: strings.TrimSpace(anyString(row["id"])),
- Slug: strings.TrimSpace(anyString(row["slug_kwd"])),
- Title: strings.TrimSpace(anyString(row["title_kwd"])),
- PageType: enginetypes.WikiPageCategory(row),
- Topic: strings.TrimSpace(anyString(row["topic_kwd"])),
- PlanGroup: strings.TrimSpace(anyString(row["plan_group_kwd"])),
- Summary: strings.TrimSpace(anyString(row["summary_with_weight"])),
- ContentMD: strings.TrimSpace(anyString(row["content_with_weight"])),
- // md_with_weight is the page-body column Python writes and reads
- // (wiki_incremental.py:2190/:2251); page rows fall back to
- // content_with_weight when it was not stamped.
- ContentMDRaw: firstNonEmptyString(row["md_with_weight"], row["content_with_weight"]),
+ ID: strings.TrimSpace(anyString(row["id"])),
+ Slug: strings.TrimSpace(anyString(row["slug_kwd"])),
+ Title: strings.TrimSpace(anyString(row["title_kwd"])),
+ PageType: enginetypes.WikiPageCategory(row),
+ Topic: strings.TrimSpace(anyString(row["topic_kwd"])),
+ PlanGroup: strings.TrimSpace(anyString(row["plan_group_kwd"])),
+ Summary: strings.TrimSpace(anyString(row["summary_with_weight"])),
+ ContentMD: content,
+ ContentMDRaw: content,
EntityNames: anyStrings(row["entity_names_kwd"]),
RelatedKBPages: anyStrings(row["related_kb_pages_kwd"]),
Outlinks: anyStrings(row["outlinks_kwd"]),
@@ -946,16 +967,6 @@ func wikiPageCandidateFromRow(row map[string]interface{}) kc.WikiPageCandidate {
}
}
-// firstNonEmptyString returns the first candidate that renders non-empty.
-func firstNonEmptyString(values ...interface{}) string {
- for _, v := range values {
- if s := strings.TrimSpace(anyString(v)); s != "" {
- return s
- }
- }
- return ""
-}
-
func anyString(v interface{}) string {
switch x := v.(type) {
case string:
diff --git a/internal/ingestion/task/knowledge_compiler_wiring_test.go b/internal/ingestion/task/knowledge_compiler_wiring_test.go
index 99b565bdd6..ca18752ea2 100644
--- a/internal/ingestion/task/knowledge_compiler_wiring_test.go
+++ b/internal/ingestion/task/knowledge_compiler_wiring_test.go
@@ -17,10 +17,15 @@
package task
import (
+ "context"
+ "errors"
"testing"
"ragflow/internal/agent/runtime"
+ "ragflow/internal/engine"
+ "ragflow/internal/engine/types"
"ragflow/internal/entity"
+ kc "ragflow/internal/ingestion/component/knowledge_compiler/common"
)
// TestKnowledgeCompilerRegisteredByWiring locks the composition-root contract:
@@ -43,6 +48,103 @@ func TestKnowledgeCompilerRegisteredByWiring(t *testing.T) {
}
}
+type wikiPageStoreEngine struct {
+ engine.DocEngine
+ row map[string]interface{}
+ getErr error
+ getCalls int
+ getIndex string
+ getID string
+ getKbIDs []string
+}
+
+func (e *wikiPageStoreEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
+ row := make(map[string]interface{})
+ for _, field := range req.SelectFields {
+ if value, ok := e.row[field]; ok {
+ row[field] = value
+ }
+ }
+ return &types.SearchResult{Chunks: []map[string]interface{}{row}}, nil
+}
+
+func (e *wikiPageStoreEngine) GetChunk(_ context.Context, index, id string, kbIDs []string) (interface{}, error) {
+ e.getCalls++
+ e.getIndex = index
+ e.getID = id
+ e.getKbIDs = append([]string(nil), kbIDs...)
+ return e.row, e.getErr
+}
+
+func TestWikiPageStoreReadsBody(t *testing.T) {
+ const body = "# Alpha\n\n**Body** links to [Beta](artifact/kb1/entity/beta)."
+ getErr := errors.New("get chunk failed")
+ paths := []struct {
+ name string
+ read func(context.Context, *kcWikiPageStore) ([]kc.WikiPageCandidate, error)
+ }{
+ {name: "similar", read: func(ctx context.Context, store *kcWikiPageStore) ([]kc.WikiPageCandidate, error) {
+ return store.FindSimilarPages(ctx, "t1", "kb1", []float32{1, 2}, 3)
+ }},
+ {name: "slug", read: func(ctx context.Context, store *kcWikiPageStore) ([]kc.WikiPageCandidate, error) {
+ page, err := store.GetPageBySlug(ctx, "t1", "kb1", "entity/alpha")
+ if page == nil {
+ return nil, err
+ }
+ return []kc.WikiPageCandidate{*page}, err
+ }},
+ {name: "source chunks", read: func(ctx context.Context, store *kcWikiPageStore) ([]kc.WikiPageCandidate, error) {
+ return store.FindPagesBySourceChunks(ctx, "t1", "kb1", []string{"source1"}, 3)
+ }},
+ }
+ cases := []struct {
+ name string
+ canonical string
+ hasCanonical bool
+ getErr error
+ wantBody string
+ wantGetCalls int
+ }{
+ {name: "stored markdown", wantBody: body, wantGetCalls: 1},
+ {name: "empty canonical", hasCanonical: true, wantBody: body, wantGetCalls: 1},
+ {name: "canonical wins", canonical: "# Updated body", hasCanonical: true, wantBody: "# Updated body"},
+ {name: "whitespace canonical", canonical: "\n\t ", hasCanonical: true},
+ {name: "load failure", getErr: getErr, wantGetCalls: 1},
+ }
+ for _, path := range paths {
+ t.Run(path.name, func(t *testing.T) {
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ row := map[string]interface{}{
+ "id": "page1", "slug_kwd": "entity/alpha", "md_with_weight": body,
+ "compile_kwd": "wiki_page", "page_type_kwd": "entity",
+ }
+ if tc.hasCanonical {
+ row["content_with_weight"] = tc.canonical
+ }
+ eng := &wikiPageStoreEngine{row: row, getErr: tc.getErr}
+ pages, err := path.read(t.Context(), &kcWikiPageStore{docEngine: eng})
+ if !errors.Is(err, tc.getErr) {
+ t.Fatalf("read error = %v, want %v", err, tc.getErr)
+ }
+ if eng.getCalls != tc.wantGetCalls {
+ t.Fatalf("GetChunk calls = %d, want %d", eng.getCalls, tc.wantGetCalls)
+ }
+ if eng.getCalls > 0 && (eng.getIndex != "ragflow_t1" || eng.getID != "page1" || len(eng.getKbIDs) != 1 || eng.getKbIDs[0] != "kb1") {
+ t.Fatalf("GetChunk scope = (%q, %q, %v), want (ragflow_t1, page1, [kb1])", eng.getIndex, eng.getID, eng.getKbIDs)
+ }
+ if err != nil {
+ return
+ }
+ if len(pages) != 1 || pages[0].ContentMD != tc.wantBody || pages[0].ContentMDRaw != tc.wantBody {
+ t.Fatalf("pages = %+v, want one page with body %q", pages, tc.wantBody)
+ }
+ })
+ }
+ })
+ }
+}
+
// TestDatasetEmbeddingIDBindsTheDatasetModel mirrors Python's compile-task
// binding: the tenant-level model wins over the dataset's own, a blank tenant id
// falls back to the dataset model, and an empty id defers to the tenant default.
diff --git a/internal/service/dataset_artifact_body_test.go b/internal/service/dataset_artifact_body_test.go
new file mode 100644
index 0000000000..59dfde8af9
--- /dev/null
+++ b/internal/service/dataset_artifact_body_test.go
@@ -0,0 +1,51 @@
+package service
+
+import (
+ "context"
+ "errors"
+ "testing"
+
+ "ragflow/internal/engine"
+)
+
+type wikiPageBodyEngine struct {
+ engine.DocEngine
+ getChunk func(string, string, []string) (interface{}, error)
+}
+
+func (e wikiPageBodyEngine) GetChunk(_ context.Context, index, id string, kbIDs []string) (interface{}, error) {
+ return e.getChunk(index, id, kbIDs)
+}
+
+func TestLoadWikiPageBody(t *testing.T) {
+ const body = "# Page\n\n[Link](entity/example)"
+ lookupErr := errors.New("lookup failed")
+ for _, test := range []struct {
+ name, content, want string
+ stored map[string]interface{}
+ err error
+ calls int
+ }{
+ {"content", body, body, nil, nil, 0},
+ {"blank content", "\n\n", "\n\n", nil, nil, 0},
+ {"stored Markdown", "", body, map[string]interface{}{"md_with_weight": body}, nil, 1},
+ {"no Markdown column", "", "", map[string]interface{}{"content_with_weight": ""}, nil, 1},
+ {"lookup error", "", "", nil, lookupErr, 1},
+ } {
+ t.Run(test.name, func(t *testing.T) {
+ calls := 0
+ eng := wikiPageBodyEngine{getChunk: func(index, id string, kbIDs []string) (interface{}, error) {
+ calls++
+ if index != "ragflow_t1" || id != "page1" || len(kbIDs) != 1 || kbIDs[0] != "kb1" {
+ t.Fatalf("unexpected page lookup: %s, %s, %v", index, id, kbIDs)
+ }
+ return test.stored, test.err
+ }}
+ row := map[string]interface{}{"id": "page1", "content_with_weight": test.content}
+ got, err := loadWikiPageBody(t.Context(), eng, "ragflow_t1", "kb1", row)
+ if got != test.want || !errors.Is(err, test.err) || calls != test.calls {
+ t.Fatalf("body = %q, calls = %d, err = %v; want %q, %d, %v", got, calls, err, test.want, test.calls, test.err)
+ }
+ })
+ }
+}
diff --git a/internal/service/dataset_artifact_service.go b/internal/service/dataset_artifact_service.go
index b013c60ce4..ae8750e106 100644
--- a/internal/service/dataset_artifact_service.go
+++ b/internal/service/dataset_artifact_service.go
@@ -281,8 +281,8 @@ func containsFold(s, lowerKeyword string) bool {
}
// WikiPageDetail is the full wiki page payload. The content field is exposed as
-// content_md_rendered to match the frontend IArtifactPage contract (and Python's
-// get_wiki_page), which renders it directly.
+// content_md_rendered to match the frontend IArtifactPage contract, which
+// renders the page's Markdown body directly.
type WikiPageDetail struct {
Slug string `json:"slug"`
Title string `json:"title"`
@@ -311,7 +311,7 @@ func (s *DatasetArtifactService) GetWikiPage(ctx context.Context, tenantID, data
"available_int": 1, // merged dataset-level page, not the per-doc source row
}
chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter,
- []string{"slug_kwd", "title_kwd", "page_type_kwd", "compile_kwd", "entity_type_kwd", "type_kwd", "topic_kwd", "md_with_weight",
+ []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "compile_kwd", "entity_type_kwd", "type_kwd", "topic_kwd",
"content_with_weight", "summary_with_weight", "entity_names_kwd", "outlinks_kwd",
"related_kb_pages_kwd", "source_chunk_ids", "source_doc_ids"},
0, 1, nil)
@@ -322,11 +322,9 @@ func (s *DatasetArtifactService) GetWikiPage(ctx context.Context, tenantID, data
return nil, nil
}
c := chunks[0]
- // Python stores the page body in md_with_weight (incremental writer), falling
- // back to content_with_weight for legacy rows; mirror that here.
- content := firstStringValue(c["md_with_weight"])
- if content == "" {
- content = firstStringValue(c["content_with_weight"])
+ content, err := loadWikiPageBody(ctx, engine.Get(), wikiIndexName(tenantID), datasetID, c)
+ if err != nil {
+ return nil, err
}
// slug_kwd is the full "<page_type>/<slug>" form; expose the bare slug so a
// client can pass it straight back to GetWikiPage/UpdateWikiPage without the
@@ -352,6 +350,18 @@ func (s *DatasetArtifactService) GetWikiPage(ctx context.Context, tenantID, data
return detail, nil
}
+func loadWikiPageBody(ctx context.Context, docEngine engine.DocEngine, indexName, datasetID string, row map[string]interface{}) (string, error) {
+ if content := types.WikiPageContent(row); content != "" {
+ return content, nil
+ }
+ raw, err := docEngine.GetChunk(ctx, indexName, firstStringValue(row["id"]), []string{datasetID})
+ if err != nil {
+ return "", err
+ }
+ stored, _ := raw.(map[string]interface{})
+ return types.WikiPageContent(stored), nil
+}
+
// UpdateWikiPage performs a partial field update of a wiki page's content,
// title and outlinks through the document engine, then returns the refreshed
// page.
@@ -382,9 +392,6 @@ func (s *DatasetArtifactService) UpdateWikiPage(ctx context.Context, tenantID, d
}
update := map[string]interface{}{}
if contentMd != "" {
- // GetWikiPage prefers md_with_weight and falls back to content_with_weight,
- // so write both to keep the edit readable regardless of the row's writer.
- update["md_with_weight"] = contentMd
update["content_with_weight"] = contentMd
}
if title != "" {
diff --git a/internal/service/file/file_commit.go b/internal/service/file/file_commit.go
index 81c805390a..7f8e551d88 100644
--- a/internal/service/file/file_commit.go
+++ b/internal/service/file/file_commit.go
@@ -707,7 +707,7 @@ func (s *FileCommitService) readCurrentPageContent(ctx context.Context, tenantID
IndexNames: []string{wikiIndexName(tenantID)},
KbIDs: []string{datasetID},
Limit: 1,
- SelectFields: []string{"md_with_weight", "content_with_weight"},
+ SelectFields: []string{"id", "content_with_weight"},
Filter: map[string]interface{}{
"type_kwd": []string{"wiki_page"},
"page_type_kwd": []string{pageType},
@@ -715,14 +715,24 @@ func (s *FileCommitService) readCurrentPageContent(ctx context.Context, tenantID
"available_int": 1,
},
})
- if err != nil || result == nil || len(result.Chunks) == 0 {
+ if err != nil {
+ common.Warn("failed to read current Wiki page content", zap.Error(err))
+ return ""
+ }
+ if result == nil || len(result.Chunks) == 0 {
return ""
}
- content := pageContentValue(result.Chunks[0]["md_with_weight"])
- if content == "" {
- content = pageContentValue(result.Chunks[0]["content_with_weight"])
+ row := result.Chunks[0]
+ if content := enginetypes.WikiPageContent(row); content != "" {
+ return content
+ }
+ raw, err := docEngine.GetChunk(ctx, wikiIndexName(tenantID), pageContentValue(row["id"]), []string{datasetID})
+ if err != nil {
+ common.Warn("failed to read stored Wiki page content", zap.Error(err))
+ return ""
}
- return content
+ stored, _ := raw.(map[string]interface{})
+ return enginetypes.WikiPageContent(stored)
}
func pageContentValue(value interface{}) string {