RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(wiki): use plain-text summaries (#20598)
adws0c7f4ef5eacd
diff --git a/internal/ingestion/component/knowledge_compiler/component_test.go b/internal/ingestion/component/knowledge_compiler/component_test.goindex 6a784fcc86..0090599cb7 100644--- a/internal/ingestion/component/knowledge_compiler/component_test.go+++ b/internal/ingestion/component/knowledge_compiler/component_test.go@@ -1451,7 +1451,7 @@ func TestProductsToChunkDocs_WikiPageAndSectionRoles(t *testing.T) {page := common.Product{ID: "page-id", DocID: "d1", TenantID: "t1", Variant: common.VariantWiki,Content: "# Alpha\n\nBody", ParentID: "",- Meta: map[string]any{"kind": "page", "slug": "entity/alpha", "title": "Alpha", "page_type": "entity", "source_chunk_ids": []string{"c1"}},+ Meta: map[string]any{"kind": "page", "slug": "entity/alpha", "title": "Alpha", "page_type": "entity", "summary": "Body", "source_chunk_ids": []string{"c1"}},}section := common.Product{ID: "section-id", DocID: "d1", TenantID: "t1", Variant: common.VariantWiki,@@ -1479,6 +1479,9 @@ func TestProductsToChunkDocs_WikiPageAndSectionRoles(t *testing.T) {if kind == "wiki_page" {pageKWD = kindpageBody, _ = d.GetExtraString("md_with_weight")+ if summary, _ := d.GetExtraString("summary_with_weight"); summary != "Body" {+ t.Errorf("page summary_with_weight = %q, want Body", summary)+ }// Python's page row has only title_tks; the Infinity writer folds// title_sm_tks into docnm.if _, ok := d.ToMap()["title_sm_tks"]; ok {diff --git a/internal/ingestion/component/knowledge_compiler/wiki/page.go b/internal/ingestion/component/knowledge_compiler/wiki/page.goindex 9f2ba076fa..946807382b 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/page.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/page.go@@ -22,7 +22,6 @@ type wikiPageResult struct {RelatedKBPages []stringContentRaw stringContent string- Summary stringOutlinks []stringSourceChunkIDs []stringSourceDocIDs []string@@ -111,7 +110,6 @@ func buildPageProducts(tenantID, docID, page string, sourceChunkIDs []string) []Action: "CREATE",ContentRaw: page,Content: page,- Summary: firstParagraph(page),SourceChunkIDs: sourceChunkIDs,SourceDocIDs: []string{docID},}})@@ -167,10 +165,7 @@ func buildWikiPageProducts(tenantID, docID string, pages []wikiPageResult) []comif strings.TrimSpace(content) == "" {content = page.ContentRaw}- summary := strings.TrimSpace(page.Summary)- if summary == "" {- summary = firstParagraph(content)- }+ summary := summarizeMarkdown(content, title)sourceChunkIDs := uniqueStrings(page.SourceChunkIDs)sourceDocIDs := uniqueStrings(page.SourceDocIDs)if len(sourceDocIDs) == 0 && docID != "" {diff --git a/internal/ingestion/component/knowledge_compiler/wiki/summary.go b/internal/ingestion/component/knowledge_compiler/wiki/summary.gonew file mode 100644index 0000000000..fb5af5e589--- /dev/null+++ b/internal/ingestion/component/knowledge_compiler/wiki/summary.go@@ -0,0 +1,120 @@+package wiki++import (+ "bytes"+ "strings"+ "unicode"+ "unicode/utf8"++ "github.com/yuin/goldmark"+ "github.com/yuin/goldmark/ast"+ "github.com/yuin/goldmark/extension"+ extast "github.com/yuin/goldmark/extension/ast"+ gmhtml "github.com/yuin/goldmark/renderer/html"+ "github.com/yuin/goldmark/text"+ "golang.org/x/net/html"+)++// summarizeMarkdown extracts the first non-heading body block as plain text,+// falling back to the page title when there is no visible body text.+// Parse before truncating so long link destinations cannot leak into summaries.+func summarizeMarkdown(content, title string) string {+ md := goldmark.New(+ goldmark.WithExtensions(extension.GFM),+ // Raw HTML stays in an intermediate buffer only; tokenization removes tags.+ goldmark.WithRendererOptions(gmhtml.WithUnsafe()),+ )+ var summary string+ for i, input := range []string{content, title} {+ source := []byte(input)+ doc := md.Parser().Parse(text.NewReader(source))+ _ = ast.Walk(doc, func(node ast.Node, entering bool) (ast.WalkStatus, error) {+ if !entering {+ return ast.WalkContinue, nil+ }+ if i == 0 && node.Kind() == ast.KindHeading {+ return ast.WalkSkipChildren, nil+ }+ switch node.Kind() {+ case ast.KindHeading, ast.KindParagraph, ast.KindTextBlock, ast.KindCodeBlock,+ ast.KindFencedCodeBlock, ast.KindHTMLBlock, extast.KindTable:+ var rendered bytes.Buffer+ if err := md.Renderer().Render(&rendered, source, node); err != nil {+ return ast.WalkStop, err+ }+ summary = strings.Join(strings.Fields(extractHTMLText(rendered.Bytes())), " ")+ if summary != "" {+ return ast.WalkStop, nil+ }+ return ast.WalkSkipChildren, nil+ }+ return ast.WalkContinue, nil+ })+ if summary != "" {+ break+ }+ }+ if summary == "" {+ // A title consisting only of punctuation is still a literal page name.+ literalTitle := strings.Join(strings.Fields(title), " ")+ for _, r := range literalTitle {+ if !unicode.IsPunct(r) && !unicode.IsSymbol(r) && !unicode.IsSpace(r) {+ literalTitle = ""+ break+ }+ }+ summary = literalTitle+ }+ const maxSummaryBytes = 300+ if len(summary) > maxSummaryBytes {+ end := maxSummaryBytes+ for !utf8.RuneStart(summary[end]) {+ end--+ }+ summary = strings.TrimSpace(summary[:end])+ }+ return summary+}++// extractHTMLText reads rendered text, not link targets or formatting tags.+// Tokenization decodes entities once and preserves escaped HTML in code spans.+func extractHTMLText(source []byte) string {+ tokens := html.NewTokenizer(bytes.NewReader(source))+ var out strings.Builder+ var hiddenTag string+ for {+ switch tokens.Next() {+ case html.ErrorToken:+ return out.String()+ case html.TextToken:+ if hiddenTag == "" {+ out.Write(tokens.Text())+ }+ case html.StartTagToken, html.EndTagToken, html.SelfClosingTagToken:+ token := tokens.Token()+ if token.Data == "script" || token.Data == "style" {+ if token.Type == html.EndTagToken {+ hiddenTag = ""+ } else {+ hiddenTag = token.Data+ }+ }+ if hiddenTag != "" {+ continue+ }+ switch token.Data {+ case "img":+ for _, attr := range token.Attr {+ if attr.Key == "alt" {+ out.WriteString(attr.Val)+ break+ }+ }+ case "br", "hr", "p", "div", "li", "tr", "th", "td", "pre", "blockquote",+ "section", "article", "header", "footer", "main", "aside", "nav",+ "h1", "h2", "h3", "h4", "h5", "h6", "dl", "dt", "dd", "figure", "figcaption":+ out.WriteByte(' ')+ }+ }+ }+}diff --git a/internal/ingestion/component/knowledge_compiler/wiki/summary_test.go b/internal/ingestion/component/knowledge_compiler/wiki/summary_test.gonew file mode 100644index 0000000000..14e8c830b3--- /dev/null+++ b/internal/ingestion/component/knowledge_compiler/wiki/summary_test.go@@ -0,0 +1,142 @@+package wiki++import (+ "strings"+ "testing"+ "unicode/utf8"+)++func TestSummarizeMarkdown(t *testing.T) {+ tests := []struct {+ name string+ content string+ want string+ }{+ {name: "empty", content: " \n\n", want: ""},+ {name: "only headings", content: "# Title\n\n## Details", want: ""},+ {name: "plain text", content: "# Title\n\n齿轮是钟楼的零件。", want: "齿轮是钟楼的零件。"},+ {+ name: "wiki link and emphasis",+ content: "# 齿轮\n\n**齿轮**是[海雾镇](artifact/b157a72656fa4d61b6d5ad49a002a3d2/entity/location/海雾镇)钟楼的*核心*零件。",+ want: "齿轮是海雾镇钟楼的核心零件。",+ },+ {name: "nested link label", content: "See [**the _map_**](https://example.com/a_(b)) now.", want: "See the map now."},+ {name: "reference link", content: "See [the map][map].\n\n[map]: artifact/kb/entity/map", want: "See the map."},+ {name: "inline code", content: "Use `content_with_weight` and ~~old~~ new text.", want: "Use content_with_weight and old new text."},+ {name: "literal punctuation", content: `Keep a_b, [brackets], 2 * 3 and \*literal\*.`, want: "Keep a_b, [brackets], 2 * 3 and *literal*."},+ {name: "entities and escapes", content: `& \& &amp;`, want: "& & &"},+ {name: "code stays literal", content: "Keep `<b> & \\*literal\\*`.", want: "Keep <b> & \\*literal\\*."},+ {name: "escaped HTML stays literal", content: `Keep \<b\> and <b>.`, want: "Keep <b> and <b>."},+ {name: "inline HTML", content: "<span>First</span><br>second.", want: "First second."},+ {name: "HTML block", content: "<div><b>First</b> paragraph.<br>More.</div>", want: "First paragraph. More."},+ {name: "HTML block boundaries", content: "<section>First</section><section>Second</section>", want: "First Second"},+ {name: "autolink labels", content: "Visit <https://example.com> or <[email protected]>.", want: "Visit https://example.com or [email protected]."},+ {name: "image alt text", content: "See .", want: "See the map."},+ {name: "first paragraph", content: "# Title\n\nFirst line\nsecond line.\n\nOther paragraph.", want: "First line second line."},+ {name: "list", content: "# Title\n\n- **First** item\n- Second item", want: "First item"},+ {name: "quote", content: "# Title\n\n> **First** paragraph.\n\nOther paragraph.", want: "First paragraph."},+ {name: "code block", content: "# Title\n\n```go\nvalue := \"<b>\"\n```", want: `value := "<b>"`},+ {name: "table", content: "# Title\n\n| Name | Value |\n| --- | --- |\n| **Map** | [Town](artifact/kb/entity/town) |", want: "Name Value Map Town"},+ {name: "skip non-text blocks", content: "# Title\n\n---\n\n<!-- comment -->\n\n**Body**.", want: "Body."},+ {name: "skip script", content: "<script>ignored()</script>\n\n**Body**.", want: "Body."},+ {name: "skip style", content: "<style>.ignored { color: red; }</style>\n\n**Body**.", want: "Body."},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ if got := summarizeMarkdown(tt.content, ""); got != tt.want {+ t.Fatalf("summarizeMarkdown() = %q, want %q", got, tt.want)+ }+ })+ }+}++func TestSummarizeMarkdownTruncatesAfterRemovingLinks(t *testing.T) {+ content := "**齿轮**是[海雾镇](artifact/" + strings.Repeat("a", 400) + "/entity/location/海雾镇)钟楼的零件。"+ if got := summarizeMarkdown(content, ""); got != "齿轮是海雾镇钟楼的零件。" {+ t.Fatalf("summary = %q, want the complete visible text", got)+ }++ for _, runeText := range []string{"中", "🔔"} {+ t.Run(runeText, func(t *testing.T) {+ prefix := strings.Repeat("a", 299)+ got := summarizeMarkdown(prefix+runeText+" tail", "")+ if got != prefix || !utf8.ValidString(got) {+ t.Fatalf("summary = %q, want a valid UTF-8 prefix at the 300-byte limit", got)+ }+ })+ }+ if got := summarizeMarkdown(strings.Repeat("中", 101), ""); got != strings.Repeat("中", 100) {+ t.Fatalf("Chinese summary length = %d bytes, want 300", len(got))+ }+ if got := summarizeMarkdown(strings.Repeat("a", 301), ""); len(got) != 300 {+ t.Fatalf("ASCII summary length = %d bytes, want 300", len(got))+ }+}++func TestSummarizeMarkdownTitleDoesNotRestoreHiddenContent(t *testing.T) {+ for _, title := range []string{+ `<img src="artifact/kb/entity/gear" alt="">`,+ "<script>ignored()</script>",+ "<style>.ignored { color: red; }</style>",+ } {+ t.Run(title, func(t *testing.T) {+ if got := summarizeMarkdown("", title); got != "" {+ t.Fatalf("summary = %q, want empty text for a title with no visible content", got)+ }+ })+ }+}++func TestBuildWikiPageProductsGeneratesPlainTextSummary(t *testing.T) {+ const body = "# 齿轮\n\n**齿轮**是[海雾镇](artifact/kb/entity/location/海雾镇)钟楼的零件。\n\n## Details\n\nSee `a_b`."+ for _, rawOnly := range []bool{false, true} {+ page := wikiPageResult{+ Slug: "entity/equipment/齿轮",+ Title: "齿轮",+ PageType: "entity",+ Topic: "Clock/Parts",+ ContentRaw: body,+ Outlinks: []string{"entity/location/海雾镇"},+ }+ if !rawOnly {+ page.Content = body+ }+ products := buildWikiPageProducts("tenant", "doc", []wikiPageResult{page})+ if len(products) == 0 {+ t.Fatal("no page product")+ }+ product := products[0]+ if got := product.Meta["summary"]; got != "齿轮是海雾镇钟楼的零件。" {+ t.Fatalf("summary = %q, want plain text", got)+ }+ if product.Content != body || product.Meta["content_md_raw"] != body {+ t.Fatal("summary generation changed the Markdown body")+ }+ if product.Meta["topic"] != "Clock/Parts" {+ t.Fatalf("topic = %q, want unchanged topic", product.Meta["topic"])+ }+ links := product.Meta["outlinks"].([]string)+ if len(links) != 1 || links[0] != "entity/location/海雾镇" {+ t.Fatalf("outlinks = %#v, want unchanged outlinks", links)+ }+ }+}++func TestBuildWikiPageProductsSummaryFallsBackToTitle(t *testing.T) {+ for _, title := range []string{"**齿轮**", "# 齿轮", "---"} {+ t.Run(title, func(t *testing.T) {+ products := buildWikiPageProducts("tenant", "doc", []wikiPageResult{{+ Slug: "concept/gear",+ Title: title,+ Content: "# 齿轮\n\n<!-- no body -->",+ }})+ want := "齿轮"+ if title == "---" {+ want = title+ }+ if len(products) == 0 || products[0].Meta["summary"] != want {+ t.Fatalf("products = %#v, want plain-text title summary %q", products, want)+ }+ })+ }+}diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki.goindex ce0054b767..8d40926172 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki.go@@ -1045,10 +1045,6 @@ func (p *wikiPipeline) runRefinePage(// full-slug is linked once, only if not already present in Outlinks.contentRendered, outlinks = appendWikiSeeAlso(contentRendered, outlinks, planItem.RelatedKB, p.datasetID, pageTitles, slugToPageType)sourceDocIDs := collectWikiSourceDocIDs(p.inputs.Chunks, sourceChunkIDs, p.docID)- summary := firstParagraph(contentRendered)- if summary == "" {- summary = firstNonEmpty(planItem.Title, planItem.Slug)- }topic := firstNonEmpty(planItem.Topic, planItem.Title, planItem.Slug)return wikiPageResult{Slug: planItem.Slug,@@ -1060,7 +1056,6 @@ func (p *wikiPipeline) runRefinePage(RelatedKBPages: uniqueStrings(planItem.RelatedKB),ContentRaw: contentRaw,Content: contentRendered,- Summary: summary,Outlinks: outlinks,SourceChunkIDs: sourceChunkIDs,SourceDocIDs: sourceDocIDs,diff --git a/internal/ingestion/knowledge_compile/wiki_graph_test.go b/internal/ingestion/knowledge_compile/wiki_graph_test.goindex e79e1f4566..3bbf07826b 100644--- a/internal/ingestion/knowledge_compile/wiki_graph_test.go+++ b/internal/ingestion/knowledge_compile/wiki_graph_test.go@@ -65,6 +65,15 @@ func TestProjectWikiGraphRowsWritesSearchContent(t *testing.T) {if entity == nil {t.Fatal("missing entity")}+ var payload struct {+ Description string `json:"description"`+ }+ if err := json.Unmarshal([]byte(metaString(entity, "content_with_weight")), &payload); err != nil {+ t.Fatalf("decode graph entity: %v", err)+ }+ if payload.Description != p.Summary {+ t.Fatalf("description = %q, want stored summary %q", payload.Description, p.Summary)+ }// Python tokenizes slug + description into content_ltks and writes no title// token column on the entity row (dataset_wiki_generator.py:750-773); the// row's title stays in title_kwd / the payload.diff --git a/web/src/components/artifact-force-graph/__tests__/index.test.tsx b/web/src/components/artifact-force-graph/__tests__/index.test.tsxnew file mode 100644index 0000000000..285b69c3ea--- /dev/null+++ b/web/src/components/artifact-force-graph/__tests__/index.test.tsx@@ -0,0 +1,79 @@+import { render, screen } from '@testing-library/react';+import { createElement } from 'react';++import ArtifactForceGraph from '..';++jest.mock('react-force-graph-2d', () => {+ const ReactLib = jest.requireActual<typeof import('react')>('react');+ const MockForceGraph = ReactLib.forwardRef<+ HTMLDivElement,+ {+ graphData: { nodes: Array<{ description?: string }> };+ nodeLabel: (node: { description?: string }) => string;+ }+ >(({ graphData, nodeLabel }, ref) => {+ const label = nodeLabel(graphData.nodes[0]);+ if (!label) return null;++ // Match the graph library's HTML tooltip sink instead of rendering text.+ return ReactLib.createElement('div', {+ ref,+ 'data-testid': 'node-tooltip',+ dangerouslySetInnerHTML: { __html: label },+ });+ });+ MockForceGraph.displayName = 'MockForceGraph';+ return { __esModule: true, default: MockForceGraph };+});++jest.mock('../use-container-dimensions', () => ({+ useContainerDimensions: () => ({ width: 800, height: 600 }),+}));++jest.mock('../use-center-gravity', () => ({+ useCenterGravity: jest.fn(),+}));++jest.mock('react-i18next', () => ({+ useTranslation: () => ({ t: (key: string) => key }),+}));++function renderGraph(description: string) {+ return render(+ createElement(ArtifactForceGraph, {+ data: {+ entities: [+ {+ slug: 'entity/齿轮',+ name: '齿轮',+ aliases: [],+ description,+ type: 'entity',+ weight: 1,+ },+ ],+ relations: [],+ },+ }),+ );+}++describe('ArtifactForceGraph node tooltip', () => {+ it.each([+ ['HTML-like text', '<script>alert("bad")</script><b>齿轮</b> & text'],+ ['literal HTML entities', '<b>齿轮</b> &'],+ ['plain Chinese', '齿轮是海雾镇钟楼报时机械的核心零件。'],+ ])('shows %s as plain text', (_, description) => {+ renderGraph(description);++ const tooltip = screen.getByTestId('node-tooltip');+ expect(tooltip.textContent).toBe(description);+ expect(tooltip.querySelector('script, b')).toBeNull();+ });++ it('hides the tooltip for an empty description', () => {+ renderGraph('');++ expect(screen.queryByTestId('node-tooltip')).toBeNull();+ });+});diff --git a/web/src/components/artifact-force-graph/index.tsx b/web/src/components/artifact-force-graph/index.tsxindex ceac72af28..c3b36b4478 100644--- a/web/src/components/artifact-force-graph/index.tsx+++ b/web/src/components/artifact-force-graph/index.tsx@@ -16,6 +16,7 @@import { type IArtifactGraphEntity } from '@/interfaces/database/dataset';import { cn } from '@/lib/utils';+import escape from 'lodash/escape';import { memo, useCallback, useEffect, useMemo, useRef } from 'react';import { useTranslation } from 'react-i18next';import ForceGraph2D, { type ForceGraphMethods } from 'react-force-graph-2d';@@ -112,9 +113,9 @@ function ArtifactForceGraph<TNodeValue = IArtifactGraphEntity>({[],);- // Hover tooltip shows the entity description; empty string hides it+ // ForceGraph treats tooltip strings as HTML; descriptions are plain text.const getNodeLabel = useCallback(- (node: ArtifactGraphNode) => node.description ?? '',+ (node: ArtifactGraphNode) => escape(node.description ?? ''),[],);