RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(wiki): remove duplicate Markdown storage (#20597)
adws83c6ebdf2b5e
diff --git a/conf/infinity_mapping.json b/conf/infinity_mapping.jsonindex 9fdbd269f7..56eb58c966 100644--- a/conf/infinity_mapping.json+++ b/conf/infinity_mapping.json@@ -55,7 +55,6 @@"chunk_hash_kwd": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},"input_hash_kwd": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},"wiki_slug_kwd": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},- "md_with_weight": {"type": "varchar", "default": ""},"summary_with_weight": {"type": "varchar", "default": ""},"skill_with_weight": {"type": "varchar", "default": ""},"skill_kwd": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},diff --git a/internal/engine/infinity/chunk_transform_test.go b/internal/engine/infinity/chunk_transform_test.goindex 6543b34f49..4febde65ab 100644--- a/internal/engine/infinity/chunk_transform_test.go+++ b/internal/engine/infinity/chunk_transform_test.go@@ -214,7 +214,6 @@ func compiledRowForTransform() map[string]interface{} {"compile_kwd": "tree","raptor_kwd": "root","raptor_layer_int": 2,- "md_with_weight": "# page","extra": map[string]interface{}{"raptor_method": "gmm"},"q_3_vec": []float64{0.1, 0.2, 0.3},}diff --git a/internal/engine/types/compilation.go b/internal/engine/types/compilation.goindex 20aa641ce5..62cfb1f682 100644--- a/internal/engine/types/compilation.go+++ b/internal/engine/types/compilation.go@@ -75,6 +75,16 @@ func WikiPageCategory(row map[string]any) string {return compilationString(row["page_type_kwd"])}+// WikiPageContent prefers content_with_weight and falls back to md_with_weight.+func WikiPageContent(row map[string]any) string {+ for _, field := range []string{"content_with_weight", "md_with_weight"} {+ if values := compilationValues(row[field]); len(values) > 0 && values[0] != "" {+ return values[0]+ }+ }+ return ""+}+func IsNavigationRow(row map[string]any) bool {role := compilationString(row["type_kwd"])return role == "nav_doc" || role == "nav_cluster" || compilationString(row["compile_kwd"]) == "dataset_nav"diff --git a/internal/engine/types/compilation_test.go b/internal/engine/types/compilation_test.goindex 694afa5b2a..78e61b224b 100644--- a/internal/engine/types/compilation_test.go+++ b/internal/engine/types/compilation_test.go@@ -2,6 +2,30 @@ package typesimport "testing"+func TestWikiPageContent(t *testing.T) {+ const body = " # Alpha\n\n[Beta](artifact/kb1/entity/beta)\n"+ for _, test := range []struct {+ name string+ row map[string]any+ want string+ }{+ {"canonical", map[string]any{"content_with_weight": body}, body},+ {"canonical wins", map[string]any{"content_with_weight": body, "md_with_weight": "stale body"}, body},+ {"Markdown only", map[string]any{"md_with_weight": body}, body},+ {"empty canonical", map[string]any{"content_with_weight": "", "md_with_weight": body}, body},+ {"blank canonical", map[string]any{"content_with_weight": " \n", "md_with_weight": body}, " \n"},+ {"list values", map[string]any{"content_with_weight": []string{""}, "md_with_weight": []any{body}}, body},+ {"canonical list", map[string]any{"content_with_weight": []string{body}, "md_with_weight": "stale body"}, body},+ {"missing", nil, ""},+ } {+ t.Run(test.name, func(t *testing.T) {+ if got := WikiPageContent(test.row); got != test.want {+ t.Fatalf("WikiPageContent() = %q, want %q", got, test.want)+ }+ })+ }+}+func TestCompilationKindPrefersCanonicalValue(t *testing.T) {for _, test := range []struct {row map[string]anydiff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.goindex 49cbeae434..a4fe62d74e 100644--- a/internal/ingestion/component/knowledge_compiler/component.go+++ b/internal/ingestion/component/knowledge_compiler/component.go@@ -540,14 +540,6 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {return err}}- // md_with_weight is the page-body column Python writes with the same- // value as content_with_weight (wiki_incremental.py:2190-2191), i.e. the- // rendered body; section rows have no page body of their own.- if metaString(p.Meta, "kind") == "page" && p.Content != "" {- if err := doc.SetExtraValue("md_with_weight", p.Content); err != nil {- return err- }- }// Section depth goes to depth_int; there is no section_level_int column.if v, ok := metaInt(p.Meta, "section_level"); ok {if err := doc.SetExtraValue("depth_int", v); err != nil {diff --git a/internal/ingestion/component/knowledge_compiler/component_test.go b/internal/ingestion/component/knowledge_compiler/component_test.goindex 0090599cb7..fd604e3261 100644--- a/internal/ingestion/component/knowledge_compiler/component_test.go+++ b/internal/ingestion/component/knowledge_compiler/component_test.go@@ -15,6 +15,7 @@ import ("ragflow/internal/agent/runtime""ragflow/internal/ingestion/component/globals""ragflow/internal/ingestion/component/knowledge_compiler/common"+ "ragflow/internal/ingestion/task/indexdoc""gorm.io/gorm")@@ -1444,13 +1445,13 @@ func TestKnowledgeCompiler_BuildInputsAcceptsMapSliceChunks(t *testing.T) {}}-// TestProductsToChunkDocs_PageVsSectionCompileKWD locks the page/section-// discriminator (compile_kwd) and the schema-column contract: the page body goes-// to md_with_weight and no Go-only field (kc_*, tenant_id) is emitted.+// TestProductsToChunkDocs_WikiPageAndSectionRoles locks the page/section+// discriminator (type_kwd) and the schema-column contract: Markdown stays in+// content_with_weight without a duplicate body column or non-schema fields.func TestProductsToChunkDocs_WikiPageAndSectionRoles(t *testing.T) {page := common.Product{ID: "page-id", DocID: "d1", TenantID: "t1", Variant: common.VariantWiki,- Content: "# Alpha\n\nBody", ParentID: "",+ Content: "# Alpha\n\n**Body** links to [Beta](artifact/kb1/entity/beta).", ParentID: "",Meta: map[string]any{"kind": "page", "slug": "entity/alpha", "title": "Alpha", "page_type": "entity", "summary": "Body", "source_chunk_ids": []string{"c1"}},}section := common.Product{@@ -1465,6 +1466,8 @@ func TestProductsToChunkDocs_WikiPageAndSectionRoles(t *testing.T) {var pageKWD, sectionKWD stringvar sectionParent, pageBody, sectionBody stringfor _, d := range docs {+ row := d.ToMap()+ indexdoc.RenameTextToContentWithWeight(row)if compiled, _ := d.GetExtraString("compile_kwd"); compiled != "wiki" {t.Fatalf("compile_kwd = %q, want wiki", compiled)}@@ -1473,12 +1476,11 @@ func TestProductsToChunkDocs_WikiPageAndSectionRoles(t *testing.T) {t.Fatalf("retired field %s was written", field)}}- // compile_kwd IS the page/section discriminator (wiki_page /- // wiki_section); there is no separate kind column.+ // type_kwd distinguishes pages from sections within compile_kwd=wiki.kind, _ := d.GetExtraString("type_kwd")if kind == "wiki_page" {pageKWD = kind- pageBody, _ = d.GetExtraString("md_with_weight")+ pageBody, _ = row["content_with_weight"].(string)if summary, _ := d.GetExtraString("summary_with_weight"); summary != "Body" {t.Errorf("page summary_with_weight = %q, want Body", summary)}@@ -1491,7 +1493,10 @@ func TestProductsToChunkDocs_WikiPageAndSectionRoles(t *testing.T) {if kind == "wiki_section" {sectionKWD = kindsectionParent, _ = d.GetExtraString("parent_kwd")- sectionBody, _ = d.GetExtraString("md_with_weight")+ sectionBody, _ = row["content_with_weight"].(string)+ }+ if _, ok := row["md_with_weight"]; ok {+ t.Error("Wiki rows must not duplicate the body in md_with_weight")}// No Go-only field: Infinity rejects an unknown column.for k := range d.Extra {@@ -1504,19 +1509,19 @@ func TestProductsToChunkDocs_WikiPageAndSectionRoles(t *testing.T) {}}if pageKWD != "wiki_page" {- t.Errorf("page compile_kwd = %q, want wiki_page", pageKWD)+ t.Errorf("page type_kwd = %q, want wiki_page", pageKWD)}if sectionKWD != "wiki_section" {- t.Errorf("section compile_kwd = %q, want wiki_section (schema-backed page/section discriminator)", sectionKWD)+ t.Errorf("section type_kwd = %q, want wiki_section (schema-backed page/section discriminator)", sectionKWD)}if sectionParent != "page-id" {t.Errorf("section parent_kwd = %q, want page-id", sectionParent)}- if pageBody != "# Alpha\n\nBody" {- t.Errorf("page md_with_weight = %q, want the rendered page body", pageBody)+ if pageBody != page.Content {+ t.Errorf("page content_with_weight = %q, want %q", pageBody, page.Content)}- if sectionBody != "" {- t.Errorf("section md_with_weight = %q, want empty (only page rows carry the page body)", sectionBody)+ if sectionBody != section.Content {+ t.Errorf("section content_with_weight = %q, want %q", sectionBody, section.Content)}}diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.goindex 24bfd3079e..5b427e599d 100644--- a/internal/ingestion/knowledge_compile/writer.go+++ b/internal/ingestion/knowledge_compile/writer.go@@ -333,7 +333,7 @@ func (w engineWriter) loadWikiPageContent(ctx context.Context, tenant, kb stringOffset: offset,Limit: pageBatchSize,SelectFields: []string{- "slug_kwd", "page_type_kwd", "compile_kwd", "entity_type_kwd", "type_kwd", "md_with_weight", "content_with_weight",+ "id", "slug_kwd", "page_type_kwd", "compile_kwd", "entity_type_kwd", "type_kwd", "content_with_weight",},Filter: map[string]interface{}{"type_kwd": compileKwdWikiPage,@@ -357,9 +357,14 @@ func (w engineWriter) loadWikiPageContent(ctx context.Context, tenant, kb stringif !ok {continue}- content := pageEngineString(row["md_with_weight"])+ content := types.WikiPageContent(row)if content == "" {- content = pageEngineString(row["content_with_weight"])+ raw, err := eng.GetChunk(ctx, baseName, pageEngineString(row["id"]), []string{kb})+ if err != nil {+ return nil, err+ }+ stored, _ := raw.(map[string]interface{})+ content = types.WikiPageContent(stored)}result[target.key] = content}@@ -1014,11 +1019,6 @@ func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccommif v := metaString(p.Meta, "summary"); v != "" {m["summary_with_weight"] = v}- // The merged wiki page keeps the page body in md_with_weight (the column- // GetWikiPage / the artifact API read), as Python does; sections have none.- if p.Variant == kccommon.VariantWiki && metaString(p.Meta, "kind") == "page" && p.Content != "" {- m["md_with_weight"] = p.Content- }if v := metaStringSlice(p.Meta, "entity_names"); len(v) > 0 {m["entity_names_kwd"] = v}diff --git a/internal/ingestion/knowledge_compile/writer_test.go b/internal/ingestion/knowledge_compile/writer_test.goindex cf85feac25..47ac4bb258 100644--- a/internal/ingestion/knowledge_compile/writer_test.go+++ b/internal/ingestion/knowledge_compile/writer_test.go@@ -61,13 +61,13 @@ func TestWriteMergedRecordsGeneratedPageCommitOnly(t *testing.T) {// TestMergedChunkMapKeepsWikiFields locks the fix for the merged-row metadata// gap: the dataset-level merged row written by mergedChunkMap must carry the-// wiki page fields (page_type_kwd/topic_kwd/title_kwd/slug_kwd/...) that the+// wiki page fields (entity_type_kwd/topic_kwd/title_kwd/slug_kwd/...) that the// artifact API (ListArtifacts/ListWikiTopics) and page renderers read. Without// them the compilation page surfaces no wiki pages from the merged products.func TestMergedChunkMapKeepsWikiFields(t *testing.T) {p := kccommon.Product{ID: "merged-1", DocID: "kb1", TenantID: "t1", Variant: kccommon.VariantWiki,- Content: "# Alpha\n\nBody",+ Content: "# Alpha\n\n**Body** links to [Beta](entity/beta).",Vector: []float32{0.1, 0.2, 0.3},Meta: map[string]any{"kind": "page",@@ -91,12 +91,9 @@ func TestMergedChunkMapKeepsWikiFields(t *testing.T) {"entity_type_kwd": "entity","topic_kwd": "Knowledge/Core/Alpha","summary_with_weight": "A page about Alpha",- // The wiki page body goes to md_with_weight, the column Python writes- // and reads (wiki_incremental.py:2190); artifact_slug_kwd was a Go-only- // column that exists in no engine mapping.- "md_with_weight": "# Alpha\n\nBody",- "plan_kwd": "run-abc",- "input_hash_kwd": "hash-123",+ "content_with_weight": p.Content,+ "plan_kwd": "run-abc",+ "input_hash_kwd": "hash-123",}// The wall-clock audit fields must be stamped from `now`, not omitted.if m["create_time"] != now.Format("2006-01-02 15:04:05") {@@ -110,6 +107,9 @@ func TestMergedChunkMapKeepsWikiFields(t *testing.T) {t.Errorf("merged row[%q] = %q, want %q", k, got, want)}}+ if _, ok := m["md_with_weight"]; ok {+ t.Fatal("merged Wiki rows must not duplicate the body in md_with_weight")+ }if v, _ := m["entity_names_kwd"].([]string); len(v) != 1 || v[0] != "Alpha" {t.Errorf("entity_names_kwd = %#v, want [Alpha]", m["entity_names_kwd"])}diff --git a/internal/ingestion/task/knowledge_compiler_wiring.go b/internal/ingestion/task/knowledge_compiler_wiring.goindex aab3a92ba4..4ed4e6353a 100644--- a/internal/ingestion/task/knowledge_compiler_wiring.go+++ b/internal/ingestion/task/knowledge_compiler_wiring.go@@ -845,11 +845,8 @@ func (s *kcWikiPageStore) FindSimilarPages(ctx context.Context, tenantID, dataseIndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},KbIDs: []string{datasetID},Limit: k,- SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "compile_kwd", "type_kwd", "entity_type_kwd", "topic_kwd", "plan_group_kwd", "summary_with_weight", "content_with_weight", "md_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "_score"},- // compile_kwd="wiki_page" is the schema-backed discriminator for wiki- // pages (sections carry compile_kwd="wiki_section"); there is no- // "kc_kind" column in the chunk schema, so filtering on it would return- // empty on Infinity.+ SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "compile_kwd", "type_kwd", "entity_type_kwd", "topic_kwd", "plan_group_kwd", "summary_with_weight", "content_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "_score"},+ // Select pages, not sections, by their role within the Wiki compilation.Filter: map[string]interface{}{"type_kwd": "wiki_page",},@@ -868,7 +865,11 @@ func (s *kcWikiPageStore) FindSimilarPages(ctx context.Context, tenantID, datase}out := make([]kc.WikiPageCandidate, 0, len(res.Chunks))for _, row := range res.Chunks {- out = append(out, wikiPageCandidateFromRow(row))+ candidate, err := s.loadPageCandidate(ctx, req.IndexNames[0], req.KbIDs, row)+ if err != nil {+ return nil, err+ }+ out = append(out, candidate)}return out, nil}@@ -881,7 +882,7 @@ func (s *kcWikiPageStore) GetPageBySlug(ctx context.Context, tenantID, datasetIDIndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},KbIDs: []string{datasetID},Limit: 1,- SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "compile_kwd", "type_kwd", "entity_type_kwd", "topic_kwd", "plan_group_kwd", "summary_with_weight", "content_with_weight", "md_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "_score"},+ SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "compile_kwd", "type_kwd", "entity_type_kwd", "topic_kwd", "plan_group_kwd", "summary_with_weight", "content_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "_score"},Filter: map[string]interface{}{"type_kwd": "wiki_page","slug_kwd": slug,@@ -891,7 +892,10 @@ func (s *kcWikiPageStore) GetPageBySlug(ctx context.Context, tenantID, datasetIDif err != nil || res == nil || len(res.Chunks) == 0 {return nil, err}- page := wikiPageCandidateFromRow(res.Chunks[0])+ page, err := s.loadPageCandidate(ctx, req.IndexNames[0], req.KbIDs, res.Chunks[0])+ if err != nil {+ return nil, err+ }return &page, nil}@@ -903,7 +907,7 @@ func (s *kcWikiPageStore) FindPagesBySourceChunks(ctx context.Context, tenantID,IndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},KbIDs: []string{datasetID},Limit: k,- SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "compile_kwd", "type_kwd", "entity_type_kwd", "topic_kwd", "summary_with_weight", "content_with_weight", "md_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "_score"},+ SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "compile_kwd", "type_kwd", "entity_type_kwd", "topic_kwd", "summary_with_weight", "content_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "_score"},Filter: map[string]interface{}{"type_kwd": "wiki_page","source_chunk_ids": chunkIDs,@@ -915,7 +919,10 @@ func (s *kcWikiPageStore) FindPagesBySourceChunks(ctx context.Context, tenantID,}out := make([]kc.WikiPageCandidate, 0, len(res.Chunks))for _, row := range res.Chunks {- candidate := wikiPageCandidateFromRow(row)+ candidate, err := s.loadPageCandidate(ctx, req.IndexNames[0], req.KbIDs, row)+ if err != nil {+ return nil, err+ }if candidate.Score == 0 {candidate.Score = 0.68}@@ -924,20 +931,34 @@ func (s *kcWikiPageStore) FindPagesBySourceChunks(ctx context.Context, tenantID,return out, nil}+func (s *kcWikiPageStore) loadPageCandidate(ctx context.Context, indexName string, datasetIDs []string, row map[string]interface{}) (kc.WikiPageCandidate, error) {+ page := wikiPageCandidateFromRow(row)+ if enginetypes.WikiPageContent(row) != "" {+ return page, nil+ }+ raw, err := s.docEngine.GetChunk(ctx, indexName, page.ID, datasetIDs)+ if err != nil {+ return kc.WikiPageCandidate{}, err+ }+ stored, _ := raw.(map[string]interface{})+ content := strings.TrimSpace(enginetypes.WikiPageContent(stored))+ page.ContentMD = content+ page.ContentMDRaw = content+ return page, nil+}+func wikiPageCandidateFromRow(row map[string]interface{}) kc.WikiPageCandidate {+ content := strings.TrimSpace(enginetypes.WikiPageContent(row))return kc.WikiPageCandidate{- ID: strings.TrimSpace(anyString(row["id"])),- Slug: strings.TrimSpace(anyString(row["slug_kwd"])),- Title: strings.TrimSpace(anyString(row["title_kwd"])),- PageType: enginetypes.WikiPageCategory(row),- Topic: strings.TrimSpace(anyString(row["topic_kwd"])),- PlanGroup: strings.TrimSpace(anyString(row["plan_group_kwd"])),- Summary: strings.TrimSpace(anyString(row["summary_with_weight"])),- ContentMD: strings.TrimSpace(anyString(row["content_with_weight"])),- // md_with_weight is the page-body column Python writes and reads- // (wiki_incremental.py:2190/:2251); page rows fall back to- // content_with_weight when it was not stamped.- ContentMDRaw: firstNonEmptyString(row["md_with_weight"], row["content_with_weight"]),+ ID: strings.TrimSpace(anyString(row["id"])),+ Slug: strings.TrimSpace(anyString(row["slug_kwd"])),+ Title: strings.TrimSpace(anyString(row["title_kwd"])),+ PageType: enginetypes.WikiPageCategory(row),+ Topic: strings.TrimSpace(anyString(row["topic_kwd"])),+ PlanGroup: strings.TrimSpace(anyString(row["plan_group_kwd"])),+ Summary: strings.TrimSpace(anyString(row["summary_with_weight"])),+ ContentMD: content,+ ContentMDRaw: content,EntityNames: anyStrings(row["entity_names_kwd"]),RelatedKBPages: anyStrings(row["related_kb_pages_kwd"]),Outlinks: anyStrings(row["outlinks_kwd"]),@@ -946,16 +967,6 @@ func wikiPageCandidateFromRow(row map[string]interface{}) kc.WikiPageCandidate {}}-// firstNonEmptyString returns the first candidate that renders non-empty.-func firstNonEmptyString(values ...interface{}) string {- for _, v := range values {- if s := strings.TrimSpace(anyString(v)); s != "" {- return s- }- }- return ""-}-func anyString(v interface{}) string {switch x := v.(type) {case string:diff --git a/internal/ingestion/task/knowledge_compiler_wiring_test.go b/internal/ingestion/task/knowledge_compiler_wiring_test.goindex 99b565bdd6..ca18752ea2 100644--- a/internal/ingestion/task/knowledge_compiler_wiring_test.go+++ b/internal/ingestion/task/knowledge_compiler_wiring_test.go@@ -17,10 +17,15 @@package taskimport (+ "context"+ "errors""testing""ragflow/internal/agent/runtime"+ "ragflow/internal/engine"+ "ragflow/internal/engine/types""ragflow/internal/entity"+ kc "ragflow/internal/ingestion/component/knowledge_compiler/common")// TestKnowledgeCompilerRegisteredByWiring locks the composition-root contract:@@ -43,6 +48,103 @@ func TestKnowledgeCompilerRegisteredByWiring(t *testing.T) {}}+type wikiPageStoreEngine struct {+ engine.DocEngine+ row map[string]interface{}+ getErr error+ getCalls int+ getIndex string+ getID string+ getKbIDs []string+}++func (e *wikiPageStoreEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {+ row := make(map[string]interface{})+ for _, field := range req.SelectFields {+ if value, ok := e.row[field]; ok {+ row[field] = value+ }+ }+ return &types.SearchResult{Chunks: []map[string]interface{}{row}}, nil+}++func (e *wikiPageStoreEngine) GetChunk(_ context.Context, index, id string, kbIDs []string) (interface{}, error) {+ e.getCalls+++ e.getIndex = index+ e.getID = id+ e.getKbIDs = append([]string(nil), kbIDs...)+ return e.row, e.getErr+}++func TestWikiPageStoreReadsBody(t *testing.T) {+ const body = "# Alpha\n\n**Body** links to [Beta](artifact/kb1/entity/beta)."+ getErr := errors.New("get chunk failed")+ paths := []struct {+ name string+ read func(context.Context, *kcWikiPageStore) ([]kc.WikiPageCandidate, error)+ }{+ {name: "similar", read: func(ctx context.Context, store *kcWikiPageStore) ([]kc.WikiPageCandidate, error) {+ return store.FindSimilarPages(ctx, "t1", "kb1", []float32{1, 2}, 3)+ }},+ {name: "slug", read: func(ctx context.Context, store *kcWikiPageStore) ([]kc.WikiPageCandidate, error) {+ page, err := store.GetPageBySlug(ctx, "t1", "kb1", "entity/alpha")+ if page == nil {+ return nil, err+ }+ return []kc.WikiPageCandidate{*page}, err+ }},+ {name: "source chunks", read: func(ctx context.Context, store *kcWikiPageStore) ([]kc.WikiPageCandidate, error) {+ return store.FindPagesBySourceChunks(ctx, "t1", "kb1", []string{"source1"}, 3)+ }},+ }+ cases := []struct {+ name string+ canonical string+ hasCanonical bool+ getErr error+ wantBody string+ wantGetCalls int+ }{+ {name: "stored markdown", wantBody: body, wantGetCalls: 1},+ {name: "empty canonical", hasCanonical: true, wantBody: body, wantGetCalls: 1},+ {name: "canonical wins", canonical: "# Updated body", hasCanonical: true, wantBody: "# Updated body"},+ {name: "whitespace canonical", canonical: "\n\t ", hasCanonical: true},+ {name: "load failure", getErr: getErr, wantGetCalls: 1},+ }+ for _, path := range paths {+ t.Run(path.name, func(t *testing.T) {+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ row := map[string]interface{}{+ "id": "page1", "slug_kwd": "entity/alpha", "md_with_weight": body,+ "compile_kwd": "wiki_page", "page_type_kwd": "entity",+ }+ if tc.hasCanonical {+ row["content_with_weight"] = tc.canonical+ }+ eng := &wikiPageStoreEngine{row: row, getErr: tc.getErr}+ pages, err := path.read(t.Context(), &kcWikiPageStore{docEngine: eng})+ if !errors.Is(err, tc.getErr) {+ t.Fatalf("read error = %v, want %v", err, tc.getErr)+ }+ if eng.getCalls != tc.wantGetCalls {+ t.Fatalf("GetChunk calls = %d, want %d", eng.getCalls, tc.wantGetCalls)+ }+ if eng.getCalls > 0 && (eng.getIndex != "ragflow_t1" || eng.getID != "page1" || len(eng.getKbIDs) != 1 || eng.getKbIDs[0] != "kb1") {+ t.Fatalf("GetChunk scope = (%q, %q, %v), want (ragflow_t1, page1, [kb1])", eng.getIndex, eng.getID, eng.getKbIDs)+ }+ if err != nil {+ return+ }+ if len(pages) != 1 || pages[0].ContentMD != tc.wantBody || pages[0].ContentMDRaw != tc.wantBody {+ t.Fatalf("pages = %+v, want one page with body %q", pages, tc.wantBody)+ }+ })+ }+ })+ }+}+// TestDatasetEmbeddingIDBindsTheDatasetModel mirrors Python's compile-task// binding: the tenant-level model wins over the dataset's own, a blank tenant id// falls back to the dataset model, and an empty id defers to the tenant default.diff --git a/internal/service/dataset_artifact_body_test.go b/internal/service/dataset_artifact_body_test.gonew file mode 100644index 0000000000..59dfde8af9--- /dev/null+++ b/internal/service/dataset_artifact_body_test.go@@ -0,0 +1,51 @@+package service++import (+ "context"+ "errors"+ "testing"++ "ragflow/internal/engine"+)++type wikiPageBodyEngine struct {+ engine.DocEngine+ getChunk func(string, string, []string) (interface{}, error)+}++func (e wikiPageBodyEngine) GetChunk(_ context.Context, index, id string, kbIDs []string) (interface{}, error) {+ return e.getChunk(index, id, kbIDs)+}++func TestLoadWikiPageBody(t *testing.T) {+ const body = "# Page\n\n[Link](entity/example)"+ lookupErr := errors.New("lookup failed")+ for _, test := range []struct {+ name, content, want string+ stored map[string]interface{}+ err error+ calls int+ }{+ {"content", body, body, nil, nil, 0},+ {"blank content", "\n\n", "\n\n", nil, nil, 0},+ {"stored Markdown", "", body, map[string]interface{}{"md_with_weight": body}, nil, 1},+ {"no Markdown column", "", "", map[string]interface{}{"content_with_weight": ""}, nil, 1},+ {"lookup error", "", "", nil, lookupErr, 1},+ } {+ t.Run(test.name, func(t *testing.T) {+ calls := 0+ eng := wikiPageBodyEngine{getChunk: func(index, id string, kbIDs []string) (interface{}, error) {+ calls+++ if index != "ragflow_t1" || id != "page1" || len(kbIDs) != 1 || kbIDs[0] != "kb1" {+ t.Fatalf("unexpected page lookup: %s, %s, %v", index, id, kbIDs)+ }+ return test.stored, test.err+ }}+ row := map[string]interface{}{"id": "page1", "content_with_weight": test.content}+ got, err := loadWikiPageBody(t.Context(), eng, "ragflow_t1", "kb1", row)+ if got != test.want || !errors.Is(err, test.err) || calls != test.calls {+ t.Fatalf("body = %q, calls = %d, err = %v; want %q, %d, %v", got, calls, err, test.want, test.calls, test.err)+ }+ })+ }+}diff --git a/internal/service/dataset_artifact_service.go b/internal/service/dataset_artifact_service.goindex b013c60ce4..ae8750e106 100644--- a/internal/service/dataset_artifact_service.go+++ b/internal/service/dataset_artifact_service.go@@ -281,8 +281,8 @@ func containsFold(s, lowerKeyword string) bool {}// WikiPageDetail is the full wiki page payload. The content field is exposed as-// content_md_rendered to match the frontend IArtifactPage contract (and Python's-// get_wiki_page), which renders it directly.+// content_md_rendered to match the frontend IArtifactPage contract, which+// renders the page's Markdown body directly.type WikiPageDetail struct {Slug string `json:"slug"`Title string `json:"title"`@@ -311,7 +311,7 @@ func (s *DatasetArtifactService) GetWikiPage(ctx context.Context, tenantID, data"available_int": 1, // merged dataset-level page, not the per-doc source row}chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter,- []string{"slug_kwd", "title_kwd", "page_type_kwd", "compile_kwd", "entity_type_kwd", "type_kwd", "topic_kwd", "md_with_weight",+ []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "compile_kwd", "entity_type_kwd", "type_kwd", "topic_kwd","content_with_weight", "summary_with_weight", "entity_names_kwd", "outlinks_kwd","related_kb_pages_kwd", "source_chunk_ids", "source_doc_ids"},0, 1, nil)@@ -322,11 +322,9 @@ func (s *DatasetArtifactService) GetWikiPage(ctx context.Context, tenantID, datareturn nil, nil}c := chunks[0]- // Python stores the page body in md_with_weight (incremental writer), falling- // back to content_with_weight for legacy rows; mirror that here.- content := firstStringValue(c["md_with_weight"])- if content == "" {- content = firstStringValue(c["content_with_weight"])+ content, err := loadWikiPageBody(ctx, engine.Get(), wikiIndexName(tenantID), datasetID, c)+ if err != nil {+ return nil, err}// slug_kwd is the full "<page_type>/<slug>" form; expose the bare slug so a// client can pass it straight back to GetWikiPage/UpdateWikiPage without the@@ -352,6 +350,18 @@ func (s *DatasetArtifactService) GetWikiPage(ctx context.Context, tenantID, datareturn detail, nil}+func loadWikiPageBody(ctx context.Context, docEngine engine.DocEngine, indexName, datasetID string, row map[string]interface{}) (string, error) {+ if content := types.WikiPageContent(row); content != "" {+ return content, nil+ }+ raw, err := docEngine.GetChunk(ctx, indexName, firstStringValue(row["id"]), []string{datasetID})+ if err != nil {+ return "", err+ }+ stored, _ := raw.(map[string]interface{})+ return types.WikiPageContent(stored), nil+}+// UpdateWikiPage performs a partial field update of a wiki page's content,// title and outlinks through the document engine, then returns the refreshed// page.@@ -382,9 +392,6 @@ func (s *DatasetArtifactService) UpdateWikiPage(ctx context.Context, tenantID, d}update := map[string]interface{}{}if contentMd != "" {- // GetWikiPage prefers md_with_weight and falls back to content_with_weight,- // so write both to keep the edit readable regardless of the row's writer.- update["md_with_weight"] = contentMdupdate["content_with_weight"] = contentMd}if title != "" {diff --git a/internal/service/file/file_commit.go b/internal/service/file/file_commit.goindex 81c805390a..7f8e551d88 100644--- a/internal/service/file/file_commit.go+++ b/internal/service/file/file_commit.go@@ -707,7 +707,7 @@ func (s *FileCommitService) readCurrentPageContent(ctx context.Context, tenantIDIndexNames: []string{wikiIndexName(tenantID)},KbIDs: []string{datasetID},Limit: 1,- SelectFields: []string{"md_with_weight", "content_with_weight"},+ SelectFields: []string{"id", "content_with_weight"},Filter: map[string]interface{}{"type_kwd": []string{"wiki_page"},"page_type_kwd": []string{pageType},@@ -715,14 +715,24 @@ func (s *FileCommitService) readCurrentPageContent(ctx context.Context, tenantID"available_int": 1,},})- if err != nil || result == nil || len(result.Chunks) == 0 {+ if err != nil {+ common.Warn("failed to read current Wiki page content", zap.Error(err))+ return ""+ }+ if result == nil || len(result.Chunks) == 0 {return ""}- content := pageContentValue(result.Chunks[0]["md_with_weight"])- if content == "" {- content = pageContentValue(result.Chunks[0]["content_with_weight"])+ row := result.Chunks[0]+ if content := enginetypes.WikiPageContent(row); content != "" {+ return content+ }+ raw, err := docEngine.GetChunk(ctx, wikiIndexName(tenantID), pageContentValue(row["id"]), []string{datasetID})+ if err != nil {+ common.Warn("failed to read stored Wiki page content", zap.Error(err))+ return ""}- return content+ stored, _ := raw.(map[string]interface{})+ return enginetypes.WikiPageContent(stored)}func pageContentValue(value interface{}) string {