Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(wiki): use plain-text summaries (#20598)

adws0c7f4ef5eacd
diff --git a/internal/ingestion/component/knowledge_compiler/component_test.go b/internal/ingestion/component/knowledge_compiler/component_test.go
index 6a784fcc86..0090599cb7 100644
--- a/internal/ingestion/component/knowledge_compiler/component_test.go
+++ b/internal/ingestion/component/knowledge_compiler/component_test.go
@@ -1451,7 +1451,7 @@ func TestProductsToChunkDocs_WikiPageAndSectionRoles(t *testing.T) {
page := common.Product{
ID: "page-id", DocID: "d1", TenantID: "t1", Variant: common.VariantWiki,
Content: "# Alpha\n\nBody", ParentID: "",
- Meta: map[string]any{"kind": "page", "slug": "entity/alpha", "title": "Alpha", "page_type": "entity", "source_chunk_ids": []string{"c1"}},
+ Meta: map[string]any{"kind": "page", "slug": "entity/alpha", "title": "Alpha", "page_type": "entity", "summary": "Body", "source_chunk_ids": []string{"c1"}},
}
section := common.Product{
ID: "section-id", DocID: "d1", TenantID: "t1", Variant: common.VariantWiki,
@@ -1479,6 +1479,9 @@ func TestProductsToChunkDocs_WikiPageAndSectionRoles(t *testing.T) {
if kind == "wiki_page" {
pageKWD = kind
pageBody, _ = d.GetExtraString("md_with_weight")
+ if summary, _ := d.GetExtraString("summary_with_weight"); summary != "Body" {
+ t.Errorf("page summary_with_weight = %q, want Body", summary)
+ }
// Python's page row has only title_tks; the Infinity writer folds
// title_sm_tks into docnm.
if _, ok := d.ToMap()["title_sm_tks"]; ok {
diff --git a/internal/ingestion/component/knowledge_compiler/wiki/page.go b/internal/ingestion/component/knowledge_compiler/wiki/page.go
index 9f2ba076fa..946807382b 100644
--- a/internal/ingestion/component/knowledge_compiler/wiki/page.go
+++ b/internal/ingestion/component/knowledge_compiler/wiki/page.go
@@ -22,7 +22,6 @@ type wikiPageResult struct {
RelatedKBPages []string
ContentRaw string
Content string
- Summary string
Outlinks []string
SourceChunkIDs []string
SourceDocIDs []string
@@ -111,7 +110,6 @@ func buildPageProducts(tenantID, docID, page string, sourceChunkIDs []string) []
Action: "CREATE",
ContentRaw: page,
Content: page,
- Summary: firstParagraph(page),
SourceChunkIDs: sourceChunkIDs,
SourceDocIDs: []string{docID},
}})
@@ -167,10 +165,7 @@ func buildWikiPageProducts(tenantID, docID string, pages []wikiPageResult) []com
if strings.TrimSpace(content) == "" {
content = page.ContentRaw
}
- summary := strings.TrimSpace(page.Summary)
- if summary == "" {
- summary = firstParagraph(content)
- }
+ summary := summarizeMarkdown(content, title)
sourceChunkIDs := uniqueStrings(page.SourceChunkIDs)
sourceDocIDs := uniqueStrings(page.SourceDocIDs)
if len(sourceDocIDs) == 0 && docID != "" {
diff --git a/internal/ingestion/component/knowledge_compiler/wiki/summary.go b/internal/ingestion/component/knowledge_compiler/wiki/summary.go
new file mode 100644
index 0000000000..fb5af5e589
--- /dev/null
+++ b/internal/ingestion/component/knowledge_compiler/wiki/summary.go
@@ -0,0 +1,120 @@
+package wiki
+
+import (
+ "bytes"
+ "strings"
+ "unicode"
+ "unicode/utf8"
+
+ "github.com/yuin/goldmark"
+ "github.com/yuin/goldmark/ast"
+ "github.com/yuin/goldmark/extension"
+ extast "github.com/yuin/goldmark/extension/ast"
+ gmhtml "github.com/yuin/goldmark/renderer/html"
+ "github.com/yuin/goldmark/text"
+ "golang.org/x/net/html"
+)
+
+// summarizeMarkdown extracts the first non-heading body block as plain text,
+// falling back to the page title when there is no visible body text.
+// Parse before truncating so long link destinations cannot leak into summaries.
+func summarizeMarkdown(content, title string) string {
+ md := goldmark.New(
+ goldmark.WithExtensions(extension.GFM),
+ // Raw HTML stays in an intermediate buffer only; tokenization removes tags.
+ goldmark.WithRendererOptions(gmhtml.WithUnsafe()),
+ )
+ var summary string
+ for i, input := range []string{content, title} {
+ source := []byte(input)
+ doc := md.Parser().Parse(text.NewReader(source))
+ _ = ast.Walk(doc, func(node ast.Node, entering bool) (ast.WalkStatus, error) {
+ if !entering {
+ return ast.WalkContinue, nil
+ }
+ if i == 0 && node.Kind() == ast.KindHeading {
+ return ast.WalkSkipChildren, nil
+ }
+ switch node.Kind() {
+ case ast.KindHeading, ast.KindParagraph, ast.KindTextBlock, ast.KindCodeBlock,
+ ast.KindFencedCodeBlock, ast.KindHTMLBlock, extast.KindTable:
+ var rendered bytes.Buffer
+ if err := md.Renderer().Render(&rendered, source, node); err != nil {
+ return ast.WalkStop, err
+ }
+ summary = strings.Join(strings.Fields(extractHTMLText(rendered.Bytes())), " ")
+ if summary != "" {
+ return ast.WalkStop, nil
+ }
+ return ast.WalkSkipChildren, nil
+ }
+ return ast.WalkContinue, nil
+ })
+ if summary != "" {
+ break
+ }
+ }
+ if summary == "" {
+ // A title consisting only of punctuation is still a literal page name.
+ literalTitle := strings.Join(strings.Fields(title), " ")
+ for _, r := range literalTitle {
+ if !unicode.IsPunct(r) && !unicode.IsSymbol(r) && !unicode.IsSpace(r) {
+ literalTitle = ""
+ break
+ }
+ }
+ summary = literalTitle
+ }
+ const maxSummaryBytes = 300
+ if len(summary) > maxSummaryBytes {
+ end := maxSummaryBytes
+ for !utf8.RuneStart(summary[end]) {
+ end--
+ }
+ summary = strings.TrimSpace(summary[:end])
+ }
+ return summary
+}
+
+// extractHTMLText reads rendered text, not link targets or formatting tags.
+// Tokenization decodes entities once and preserves escaped HTML in code spans.
+func extractHTMLText(source []byte) string {
+ tokens := html.NewTokenizer(bytes.NewReader(source))
+ var out strings.Builder
+ var hiddenTag string
+ for {
+ switch tokens.Next() {
+ case html.ErrorToken:
+ return out.String()
+ case html.TextToken:
+ if hiddenTag == "" {
+ out.Write(tokens.Text())
+ }
+ case html.StartTagToken, html.EndTagToken, html.SelfClosingTagToken:
+ token := tokens.Token()
+ if token.Data == "script" || token.Data == "style" {
+ if token.Type == html.EndTagToken {
+ hiddenTag = ""
+ } else {
+ hiddenTag = token.Data
+ }
+ }
+ if hiddenTag != "" {
+ continue
+ }
+ switch token.Data {
+ case "img":
+ for _, attr := range token.Attr {
+ if attr.Key == "alt" {
+ out.WriteString(attr.Val)
+ break
+ }
+ }
+ case "br", "hr", "p", "div", "li", "tr", "th", "td", "pre", "blockquote",
+ "section", "article", "header", "footer", "main", "aside", "nav",
+ "h1", "h2", "h3", "h4", "h5", "h6", "dl", "dt", "dd", "figure", "figcaption":
+ out.WriteByte(' ')
+ }
+ }
+ }
+}
diff --git a/internal/ingestion/component/knowledge_compiler/wiki/summary_test.go b/internal/ingestion/component/knowledge_compiler/wiki/summary_test.go
new file mode 100644
index 0000000000..14e8c830b3
--- /dev/null
+++ b/internal/ingestion/component/knowledge_compiler/wiki/summary_test.go
@@ -0,0 +1,142 @@
+package wiki
+
+import (
+ "strings"
+ "testing"
+ "unicode/utf8"
+)
+
+func TestSummarizeMarkdown(t *testing.T) {
+ tests := []struct {
+ name string
+ content string
+ want string
+ }{
+ {name: "empty", content: " \n\n", want: ""},
+ {name: "only headings", content: "# Title\n\n## Details", want: ""},
+ {name: "plain text", content: "# Title\n\n齿轮是钟楼的零件。", want: "齿轮是钟楼的零件。"},
+ {
+ name: "wiki link and emphasis",
+ content: "# 齿轮\n\n**齿轮**是[海雾镇](artifact/b157a72656fa4d61b6d5ad49a002a3d2/entity/location/海雾镇)钟楼的*核心*零件。",
+ want: "齿轮是海雾镇钟楼的核心零件。",
+ },
+ {name: "nested link label", content: "See [**the _map_**](https://example.com/a_(b)) now.", want: "See the map now."},
+ {name: "reference link", content: "See [the map][map].\n\n[map]: artifact/kb/entity/map", want: "See the map."},
+ {name: "inline code", content: "Use `content_with_weight` and ~~old~~ new text.", want: "Use content_with_weight and old new text."},
+ {name: "literal punctuation", content: `Keep a_b, [brackets], 2 * 3 and \*literal\*.`, want: "Keep a_b, [brackets], 2 * 3 and *literal*."},
+ {name: "entities and escapes", content: `& \& &`, want: "& & &"},
+ {name: "code stays literal", content: "Keep `<b> &amp; \\*literal\\*`.", want: "Keep <b> &amp; \\*literal\\*."},
+ {name: "escaped HTML stays literal", content: `Keep \<b\> and &lt;b&gt;.`, want: "Keep <b> and <b>."},
+ {name: "inline HTML", content: "<span>First</span><br>second.", want: "First second."},
+ {name: "HTML block", content: "<div><b>First</b> paragraph.<br>More.</div>", want: "First paragraph. More."},
+ {name: "HTML block boundaries", content: "<section>First</section><section>Second</section>", want: "First Second"},
+ {name: "autolink labels", content: "Visit <https://example.com> or <[email protected]>.", want: "Visit https://example.com or [email protected]."},
+ {name: "image alt text", content: "See ![the **map**](artifact/kb/map.png).", want: "See the map."},
+ {name: "first paragraph", content: "# Title\n\nFirst line\nsecond line.\n\nOther paragraph.", want: "First line second line."},
+ {name: "list", content: "# Title\n\n- **First** item\n- Second item", want: "First item"},
+ {name: "quote", content: "# Title\n\n> **First** paragraph.\n\nOther paragraph.", want: "First paragraph."},
+ {name: "code block", content: "# Title\n\n```go\nvalue := \"<b>\"\n```", want: `value := "<b>"`},
+ {name: "table", content: "# Title\n\n| Name | Value |\n| --- | --- |\n| **Map** | [Town](artifact/kb/entity/town) |", want: "Name Value Map Town"},
+ {name: "skip non-text blocks", content: "# Title\n\n---\n\n<!-- comment -->\n\n**Body**.", want: "Body."},
+ {name: "skip script", content: "<script>ignored()</script>\n\n**Body**.", want: "Body."},
+ {name: "skip style", content: "<style>.ignored { color: red; }</style>\n\n**Body**.", want: "Body."},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := summarizeMarkdown(tt.content, ""); got != tt.want {
+ t.Fatalf("summarizeMarkdown() = %q, want %q", got, tt.want)
+ }
+ })
+ }
+}
+
+func TestSummarizeMarkdownTruncatesAfterRemovingLinks(t *testing.T) {
+ content := "**齿轮**是[海雾镇](artifact/" + strings.Repeat("a", 400) + "/entity/location/海雾镇)钟楼的零件。"
+ if got := summarizeMarkdown(content, ""); got != "齿轮是海雾镇钟楼的零件。" {
+ t.Fatalf("summary = %q, want the complete visible text", got)
+ }
+
+ for _, runeText := range []string{"中", "🔔"} {
+ t.Run(runeText, func(t *testing.T) {
+ prefix := strings.Repeat("a", 299)
+ got := summarizeMarkdown(prefix+runeText+" tail", "")
+ if got != prefix || !utf8.ValidString(got) {
+ t.Fatalf("summary = %q, want a valid UTF-8 prefix at the 300-byte limit", got)
+ }
+ })
+ }
+ if got := summarizeMarkdown(strings.Repeat("中", 101), ""); got != strings.Repeat("中", 100) {
+ t.Fatalf("Chinese summary length = %d bytes, want 300", len(got))
+ }
+ if got := summarizeMarkdown(strings.Repeat("a", 301), ""); len(got) != 300 {
+ t.Fatalf("ASCII summary length = %d bytes, want 300", len(got))
+ }
+}
+
+func TestSummarizeMarkdownTitleDoesNotRestoreHiddenContent(t *testing.T) {
+ for _, title := range []string{
+ `<img src="artifact/kb/entity/gear" alt="">`,
+ "<script>ignored()</script>",
+ "<style>.ignored { color: red; }</style>",
+ } {
+ t.Run(title, func(t *testing.T) {
+ if got := summarizeMarkdown("", title); got != "" {
+ t.Fatalf("summary = %q, want empty text for a title with no visible content", got)
+ }
+ })
+ }
+}
+
+func TestBuildWikiPageProductsGeneratesPlainTextSummary(t *testing.T) {
+ const body = "# 齿轮\n\n**齿轮**是[海雾镇](artifact/kb/entity/location/海雾镇)钟楼的零件。\n\n## Details\n\nSee `a_b`."
+ for _, rawOnly := range []bool{false, true} {
+ page := wikiPageResult{
+ Slug: "entity/equipment/齿轮",
+ Title: "齿轮",
+ PageType: "entity",
+ Topic: "Clock/Parts",
+ ContentRaw: body,
+ Outlinks: []string{"entity/location/海雾镇"},
+ }
+ if !rawOnly {
+ page.Content = body
+ }
+ products := buildWikiPageProducts("tenant", "doc", []wikiPageResult{page})
+ if len(products) == 0 {
+ t.Fatal("no page product")
+ }
+ product := products[0]
+ if got := product.Meta["summary"]; got != "齿轮是海雾镇钟楼的零件。" {
+ t.Fatalf("summary = %q, want plain text", got)
+ }
+ if product.Content != body || product.Meta["content_md_raw"] != body {
+ t.Fatal("summary generation changed the Markdown body")
+ }
+ if product.Meta["topic"] != "Clock/Parts" {
+ t.Fatalf("topic = %q, want unchanged topic", product.Meta["topic"])
+ }
+ links := product.Meta["outlinks"].([]string)
+ if len(links) != 1 || links[0] != "entity/location/海雾镇" {
+ t.Fatalf("outlinks = %#v, want unchanged outlinks", links)
+ }
+ }
+}
+
+func TestBuildWikiPageProductsSummaryFallsBackToTitle(t *testing.T) {
+ for _, title := range []string{"**齿轮**", "# 齿轮", "---"} {
+ t.Run(title, func(t *testing.T) {
+ products := buildWikiPageProducts("tenant", "doc", []wikiPageResult{{
+ Slug: "concept/gear",
+ Title: title,
+ Content: "# 齿轮\n\n<!-- no body -->",
+ }})
+ want := "齿轮"
+ if title == "---" {
+ want = title
+ }
+ if len(products) == 0 || products[0].Meta["summary"] != want {
+ t.Fatalf("products = %#v, want plain-text title summary %q", products, want)
+ }
+ })
+ }
+}
diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki.go
index ce0054b767..8d40926172 100644
--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go
+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki.go
@@ -1045,10 +1045,6 @@ func (p *wikiPipeline) runRefinePage(
// full-slug is linked once, only if not already present in Outlinks.
contentRendered, outlinks = appendWikiSeeAlso(contentRendered, outlinks, planItem.RelatedKB, p.datasetID, pageTitles, slugToPageType)
sourceDocIDs := collectWikiSourceDocIDs(p.inputs.Chunks, sourceChunkIDs, p.docID)
- summary := firstParagraph(contentRendered)
- if summary == "" {
- summary = firstNonEmpty(planItem.Title, planItem.Slug)
- }
topic := firstNonEmpty(planItem.Topic, planItem.Title, planItem.Slug)
return wikiPageResult{
Slug: planItem.Slug,
@@ -1060,7 +1056,6 @@ func (p *wikiPipeline) runRefinePage(
RelatedKBPages: uniqueStrings(planItem.RelatedKB),
ContentRaw: contentRaw,
Content: contentRendered,
- Summary: summary,
Outlinks: outlinks,
SourceChunkIDs: sourceChunkIDs,
SourceDocIDs: sourceDocIDs,
diff --git a/internal/ingestion/knowledge_compile/wiki_graph_test.go b/internal/ingestion/knowledge_compile/wiki_graph_test.go
index e79e1f4566..3bbf07826b 100644
--- a/internal/ingestion/knowledge_compile/wiki_graph_test.go
+++ b/internal/ingestion/knowledge_compile/wiki_graph_test.go
@@ -65,6 +65,15 @@ func TestProjectWikiGraphRowsWritesSearchContent(t *testing.T) {
if entity == nil {
t.Fatal("missing entity")
}
+ var payload struct {
+ Description string `json:"description"`
+ }
+ if err := json.Unmarshal([]byte(metaString(entity, "content_with_weight")), &payload); err != nil {
+ t.Fatalf("decode graph entity: %v", err)
+ }
+ if payload.Description != p.Summary {
+ t.Fatalf("description = %q, want stored summary %q", payload.Description, p.Summary)
+ }
// Python tokenizes slug + description into content_ltks and writes no title
// token column on the entity row (dataset_wiki_generator.py:750-773); the
// row's title stays in title_kwd / the payload.
diff --git a/web/src/components/artifact-force-graph/__tests__/index.test.tsx b/web/src/components/artifact-force-graph/__tests__/index.test.tsx
new file mode 100644
index 0000000000..285b69c3ea
--- /dev/null
+++ b/web/src/components/artifact-force-graph/__tests__/index.test.tsx
@@ -0,0 +1,79 @@
+import { render, screen } from '@testing-library/react';
+import { createElement } from 'react';
+
+import ArtifactForceGraph from '..';
+
+jest.mock('react-force-graph-2d', () => {
+ const ReactLib = jest.requireActual<typeof import('react')>('react');
+ const MockForceGraph = ReactLib.forwardRef<
+ HTMLDivElement,
+ {
+ graphData: { nodes: Array<{ description?: string }> };
+ nodeLabel: (node: { description?: string }) => string;
+ }
+ >(({ graphData, nodeLabel }, ref) => {
+ const label = nodeLabel(graphData.nodes[0]);
+ if (!label) return null;
+
+ // Match the graph library's HTML tooltip sink instead of rendering text.
+ return ReactLib.createElement('div', {
+ ref,
+ 'data-testid': 'node-tooltip',
+ dangerouslySetInnerHTML: { __html: label },
+ });
+ });
+ MockForceGraph.displayName = 'MockForceGraph';
+ return { __esModule: true, default: MockForceGraph };
+});
+
+jest.mock('../use-container-dimensions', () => ({
+ useContainerDimensions: () => ({ width: 800, height: 600 }),
+}));
+
+jest.mock('../use-center-gravity', () => ({
+ useCenterGravity: jest.fn(),
+}));
+
+jest.mock('react-i18next', () => ({
+ useTranslation: () => ({ t: (key: string) => key }),
+}));
+
+function renderGraph(description: string) {
+ return render(
+ createElement(ArtifactForceGraph, {
+ data: {
+ entities: [
+ {
+ slug: 'entity/齿轮',
+ name: '齿轮',
+ aliases: [],
+ description,
+ type: 'entity',
+ weight: 1,
+ },
+ ],
+ relations: [],
+ },
+ }),
+ );
+}
+
+describe('ArtifactForceGraph node tooltip', () => {
+ it.each([
+ ['HTML-like text', '<script>alert("bad")</script><b>齿轮</b> & text'],
+ ['literal HTML entities', '&lt;b&gt;齿轮&lt;/b&gt; &amp;'],
+ ['plain Chinese', '齿轮是海雾镇钟楼报时机械的核心零件。'],
+ ])('shows %s as plain text', (_, description) => {
+ renderGraph(description);
+
+ const tooltip = screen.getByTestId('node-tooltip');
+ expect(tooltip.textContent).toBe(description);
+ expect(tooltip.querySelector('script, b')).toBeNull();
+ });
+
+ it('hides the tooltip for an empty description', () => {
+ renderGraph('');
+
+ expect(screen.queryByTestId('node-tooltip')).toBeNull();
+ });
+});
diff --git a/web/src/components/artifact-force-graph/index.tsx b/web/src/components/artifact-force-graph/index.tsx
index ceac72af28..c3b36b4478 100644
--- a/web/src/components/artifact-force-graph/index.tsx
+++ b/web/src/components/artifact-force-graph/index.tsx
@@ -16,6 +16,7 @@
import { type IArtifactGraphEntity } from '@/interfaces/database/dataset';
import { cn } from '@/lib/utils';
+import escape from 'lodash/escape';
import { memo, useCallback, useEffect, useMemo, useRef } from 'react';
import { useTranslation } from 'react-i18next';
import ForceGraph2D, { type ForceGraphMethods } from 'react-force-graph-2d';
@@ -112,9 +113,9 @@ function ArtifactForceGraph<TNodeValue = IArtifactGraphEntity>({
[],
);
- // Hover tooltip shows the entity description; empty string hides it
+ // ForceGraph treats tooltip strings as HTML; descriptions are plain text.
const getNodeLabel = useCallback(
- (node: ArtifactGraphNode) => node.description ?? '',
+ (node: ArtifactGraphNode) => escape(node.description ?? ''),
[],
);