Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(agent): preserve Markdown headings and lists in DOCX (#22078)

Sbaaoui Idriss3a594af0ddcf
diff --git a/internal/agent/component/io/docx_writer.go b/internal/agent/component/io/docx_writer.go
index 0eec48c52c..19f4c001f5 100644
--- a/internal/agent/component/io/docx_writer.go
+++ b/internal/agent/component/io/docx_writer.go
@@ -26,7 +26,7 @@
// - //go:embed — the static XML parts (Content_Types, _rels, styles)
//
// The output is a Word-compatible .docx. The XML is intentionally
-// minimal: a single font, a single style, paragraphs, and native tables.
+// minimal: a single font, heading styles, paragraphs, and native tables.
package io
import (
@@ -54,6 +54,7 @@ var relsXML []byte
//go:embed templates/document_rels.xml.tmpl
//go:embed templates/styles.xml.tmpl
//go:embed templates/document.xml.tmpl
+//go:embed templates/numbering.xml.tmpl
//go:embed templates/header.xml.tmpl
//go:embed templates/footer.xml.tmpl
var tmplFS embed.FS
@@ -74,6 +75,7 @@ type DOCXOptions struct {
// DOCXOptions and Document flatten into this when the writer is
// invoked.
type docModel struct {
+ Lists []*docxList
Blocks []docxBlock
HeaderText string
FooterText string
@@ -89,8 +91,18 @@ type docModel struct {
}
type docxBlock struct {
- Text string
- Rows [][]string
+ Text string
+ Rows [][]string
+ HeadingLevel int
+ List *docxList
+ Indent int
+}
+
+type docxList struct {
+ ID int
+ Ordered bool
+ Start int
+ Indent int
}
// tmplFuncs registers the {{xml}} helper used to escape user content
@@ -138,7 +150,14 @@ func WriteDOCX(content string, opts DOCXOptions) ([]byte, error) {
Timestamp: nowUTC(),
}
- // Pre-parse all 5 templates once; FuncMaps are shared.
+ for _, block := range model.Blocks {
+ if block.List != nil && block.List.ID == 0 {
+ block.List.ID = len(model.Lists) + 1
+ model.Lists = append(model.Lists, block.List)
+ }
+ }
+
+ // Pre-parse templates once; FuncMaps are shared.
docTmpl, err := template.New("document.xml.tmpl").Funcs(tmplFuncs).ParseFS(tmplFS, "templates/document.xml.tmpl")
if err != nil {
return nil, fmt.Errorf("DOCX: parse document template: %w", err)
@@ -160,6 +179,11 @@ func WriteDOCX(content string, opts DOCXOptions) ([]byte, error) {
return nil, fmt.Errorf("DOCX: parse document_rels template: %w", err)
}
+ numberingTmpl, err := template.New("numbering.xml.tmpl").ParseFS(tmplFS, "templates/numbering.xml.tmpl")
+ if err != nil {
+ return nil, fmt.Errorf("DOCX: parse numbering template: %w", err)
+ }
+
buf := &bytes.Buffer{}
zw := zip.NewWriter(buf)
@@ -176,6 +200,9 @@ func WriteDOCX(content string, opts DOCXOptions) ([]byte, error) {
if err := writeZipTmpl(zw, "word/_rels/document.xml.rels", docRelsTmpl, model); err != nil {
return nil, err
}
+ if err := writeZipTmpl(zw, "word/numbering.xml", numberingTmpl, model); err != nil {
+ return nil, err
+ }
// word/styles.xml
if err := writeZipTmpl(zw, "word/styles.xml", stylesTmpl, model); err != nil {
return nil, err
@@ -207,9 +234,41 @@ func markdownDOCXBlocks(content string) []docxBlock {
content = normalizeMarkdownTableSeparators(content)
source := []byte(content)
document := goldmark.New(goldmark.WithExtensions(extension.Table)).Parser().Parse(gmtext.NewReader(source))
- blocks := make([]docxBlock, 0, document.ChildCount())
- for node := document.FirstChild(); node != nil; node = node.NextSibling() {
+ blocks := docxChildBlocks(document, source)
+ if len(blocks) == 0 {
+ return []docxBlock{{}}
+ }
+ return blocks
+}
+
+func docxChildBlocks(parent ast.Node, source []byte) []docxBlock {
+ var blocks []docxBlock
+ for node := parent.FirstChild(); node != nil; node = node.NextSibling() {
switch n := node.(type) {
+ case *ast.Heading:
+ blocks = append(blocks, docxBlock{Text: inlinePDFText(n, source), HeadingLevel: n.Level})
+ case *ast.List:
+ depth := 1
+ for ancestor := n.Parent(); ancestor != nil; ancestor = ancestor.Parent() {
+ if _, ok := ancestor.(*ast.List); ok {
+ depth++
+ }
+ }
+ list := &docxList{Ordered: n.IsOrdered(), Start: max(1, n.Start), Indent: depth * 720}
+ for item := n.FirstChild(); item != nil; item = item.NextSibling() {
+ children := docxChildBlocks(item, source)
+ for i := range children {
+ if children[i].Indent == 0 && children[i].Rows == nil {
+ children[i].Indent = list.Indent
+ if i == 0 {
+ children[i].List = list
+ }
+ }
+ }
+ blocks = append(blocks, children...)
+ }
+ case *ast.ListItem, *ast.Blockquote:
+ blocks = append(blocks, docxChildBlocks(n, source)...)
case *ast.FencedCodeBlock:
text := rawPDFBlockText(n, source)
if strings.EqualFold(string(n.Language(source)), "markdown") || strings.EqualFold(string(n.Language(source)), "md") {
@@ -232,14 +291,11 @@ func markdownDOCXBlocks(content string) []docxBlock {
}
blocks = append(blocks, docxBlock{Rows: rows})
default:
- if text := rawPDFBlockText(n, source); text != "" || node == document.FirstChild() {
+ if text := rawPDFBlockText(n, source); text != "" || node == parent.FirstChild() {
blocks = append(blocks, docxBlock{Text: text})
}
}
}
- if len(blocks) == 0 {
- return []docxBlock{{}}
- }
return blocks
}
diff --git a/internal/agent/component/io/docx_writer_test.go b/internal/agent/component/io/docx_writer_test.go
index 48921c6235..3783f08200 100644
--- a/internal/agent/component/io/docx_writer_test.go
+++ b/internal/agent/component/io/docx_writer_test.go
@@ -247,3 +247,91 @@ func truncate(s string, n int) string {
}
return s[:n] + "..."
}
+
+func TestDOCXWriter_MarkdownHeadings(t *testing.T) {
+ content := "# Report\n\n## 4. Findings\n\n### 4.2 **Risk & controls**\n\nBody retained.\n\n> #### Quoted heading\n>\n> Quoted body.\n\n- ##### Nested heading\n\n Nested body.\n\n```markdown\n###### Fenced heading\n```"
+ doc, err := WriteDOCX(content, DOCXOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ zr, err := zip.NewReader(bytes.NewReader(doc), int64(len(doc)))
+ if err != nil {
+ t.Fatal(err)
+ }
+ body, _ := readZipFile(t, zr, "word/document.xml")
+ var document struct {
+ Body struct {
+ Paragraphs []struct {
+ Properties struct {
+ Style struct {
+ Value string `xml:"val,attr"`
+ } `xml:"pStyle"`
+ } `xml:"pPr"`
+ Runs []struct {
+ Text string `xml:"t"`
+ } `xml:"r"`
+ } `xml:"p"`
+ } `xml:"body"`
+ }
+ if err := xml.Unmarshal([]byte(body), &document); err != nil {
+ t.Fatal(err)
+ }
+ want := []string{"Report", "4. Findings", "4.2 Risk & controls", "Body retained.", "Quoted heading", "Quoted body.", "Nested heading", "Nested body.", "Fenced heading"}
+ styles := []string{"Heading1", "Heading2", "Heading3", "", "Heading4", "", "Heading5", "", "Heading6"}
+ if len(document.Body.Paragraphs) != len(want) {
+ t.Fatalf("got %d paragraphs, want %d: %s", len(document.Body.Paragraphs), len(want), body)
+ }
+ for i, paragraph := range document.Body.Paragraphs {
+ var text strings.Builder
+ for _, run := range paragraph.Runs {
+ text.WriteString(run.Text)
+ }
+ if text.String() != want[i] || paragraph.Properties.Style.Value != styles[i] {
+ t.Errorf("paragraph %d = %q (%s), want %q (%s)", i, text.String(), paragraph.Properties.Style.Value, want[i], styles[i])
+ }
+ }
+ styleXML, _ := readZipFile(t, zr, "word/styles.xml")
+ for _, style := range styles {
+ if style != "" && !strings.Contains(styleXML, `w:styleId="`+style+`"`) {
+ t.Errorf("missing style definition %s", style)
+ }
+ }
+}
+
+func TestDOCXWriter_MarkdownLists(t *testing.T) {
+ doc, err := WriteDOCX("## 4.2 Findings\n\n- First bullet\n - Nested bullet\n\n Continuation.\n- Second bullet\n\n## 4.3 Actions\n\n3. Third action\n4. Fourth action\n\nNext list:\n\n1. Separate list\n", DOCXOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ zr, err := zip.NewReader(bytes.NewReader(doc), int64(len(doc)))
+ if err != nil {
+ t.Fatal(err)
+ }
+ body, _ := readZipFile(t, zr, "word/document.xml")
+ if count := strings.Count(body, "<w:numPr>"); count != 6 {
+ t.Errorf("numbered paragraphs = %d, want 6", count)
+ }
+ for _, text := range []string{"First bullet", "Nested bullet", "Continuation.", "Second bullet", "Third action", "Fourth action", "Separate list"} {
+ if !strings.Contains(body, text) {
+ t.Errorf("missing %q", text)
+ }
+ }
+ if !strings.Contains(body, "<w:ind w:left=\"1440\"/></w:pPr><w:r>") {
+ t.Error("nested continuation lost its indentation")
+ }
+ numbering, ok := readZipFile(t, zr, "word/numbering.xml")
+ if !ok {
+ t.Fatal("missing numbering.xml")
+ }
+ for _, value := range []string{`w:val="bullet"`, `w:val="decimal"`, `w:start w:val="3"`, `w:left="1440"`} {
+ if !strings.Contains(numbering, value) {
+ t.Errorf("missing list definition %s", value)
+ }
+ }
+ for name, want := range map[string]string{"[Content_Types].xml": "/word/numbering.xml", "word/_rels/document.xml.rels": "relationships/numbering"} {
+ part, _ := readZipFile(t, zr, name)
+ if !strings.Contains(part, want) {
+ t.Errorf("%s missing %s", name, want)
+ }
+ }
+}
diff --git a/internal/agent/component/io/templates/content_types.xml b/internal/agent/component/io/templates/content_types.xml
index a71383baec..a38e10fb6f 100644
--- a/internal/agent/component/io/templates/content_types.xml
+++ b/internal/agent/component/io/templates/content_types.xml
@@ -7,4 +7,5 @@
<Override PartName="/word/header1.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.header+xml"/>
<Override PartName="/word/footer1.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.footer+xml"/>
<Override PartName="/word/_rels/document.xml.rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>
+<Override PartName="/word/numbering.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.numbering+xml"/>
</Types>
diff --git a/internal/agent/component/io/templates/document.xml.tmpl b/internal/agent/component/io/templates/document.xml.tmpl
index a7397243e0..df493a2705 100644
--- a/internal/agent/component/io/templates/document.xml.tmpl
+++ b/internal/agent/component/io/templates/document.xml.tmpl
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">
<w:body>
-{{range .Blocks}}{{if .Rows}}<w:tbl><w:tblPr><w:tblStyle w:val="TableGrid"/></w:tblPr>{{range .Rows}}<w:tr>{{range .}}<w:tc><w:tcPr><w:tcW w:w="0" w:type="auto"/></w:tcPr><w:p><w:r><w:rPr><w:sz w:val="{{pt $.FontSize}}"/><w:szCs w:val="{{pt $.FontSize}}"/><w:rFonts w:ascii="{{$.FontFamily}}" w:eastAsia="{{$.FontFamily}}" w:hAnsi="{{$.FontFamily}}"/></w:rPr><w:t xml:space="preserve">{{xml .}}</w:t></w:r></w:p></w:tc>{{end}}</w:tr>{{end}}</w:tbl>{{else}}<w:p><w:r><w:rPr><w:sz w:val="{{pt $.FontSize}}"/><w:szCs w:val="{{pt $.FontSize}}"/><w:rFonts w:ascii="{{$.FontFamily}}" w:eastAsia="{{$.FontFamily}}" w:hAnsi="{{$.FontFamily}}"/></w:rPr><w:t xml:space="preserve">{{xml .Text}}</w:t></w:r></w:p>{{end}}
+{{range .Blocks}}{{if .Rows}}<w:tbl><w:tblPr><w:tblStyle w:val="TableGrid"/></w:tblPr>{{range .Rows}}<w:tr>{{range .}}<w:tc><w:tcPr><w:tcW w:w="0" w:type="auto"/></w:tcPr><w:p><w:r><w:rPr><w:sz w:val="{{pt $.FontSize}}"/><w:szCs w:val="{{pt $.FontSize}}"/><w:rFonts w:ascii="{{$.FontFamily}}" w:eastAsia="{{$.FontFamily}}" w:hAnsi="{{$.FontFamily}}"/></w:rPr><w:t xml:space="preserve">{{xml .}}</w:t></w:r></w:p></w:tc>{{end}}</w:tr>{{end}}</w:tbl>{{else}}<w:p>{{if or .HeadingLevel .Indent}}<w:pPr>{{if .HeadingLevel}}<w:pStyle w:val="Heading{{.HeadingLevel}}"/>{{end}}{{if .List}}<w:numPr><w:ilvl w:val="0"/><w:numId w:val="{{.List.ID}}"/></w:numPr>{{end}}{{if .Indent}}<w:ind w:left="{{.Indent}}"{{if .List}} w:hanging="360"{{end}}/>{{end}}</w:pPr>{{end}}<w:r><w:rPr><w:sz w:val="{{pt $.FontSize}}"/><w:szCs w:val="{{pt $.FontSize}}"/><w:rFonts w:ascii="{{$.FontFamily}}" w:eastAsia="{{$.FontFamily}}" w:hAnsi="{{$.FontFamily}}"/></w:rPr><w:t xml:space="preserve">{{xml .Text}}</w:t></w:r></w:p>{{end}}
{{end}}<w:sectPr>
{{if .HasHeader}}<w:headerReference w:type="default" r:id="rIdHeader1"/>{{end}}
{{if .HasFooter}}<w:footerReference w:type="default" r:id="rIdFooter1"/>{{end}}
diff --git a/internal/agent/component/io/templates/document_rels.xml.tmpl b/internal/agent/component/io/templates/document_rels.xml.tmpl
index 1893220747..80cbe5a966 100644
--- a/internal/agent/component/io/templates/document_rels.xml.tmpl
+++ b/internal/agent/component/io/templates/document_rels.xml.tmpl
@@ -3,4 +3,5 @@
<Relationship Id="rIdStyles" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/styles" Target="styles.xml"/>
{{if .HasHeader}}<Relationship Id="rIdHeader1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/header" Target="header1.xml"/>{{end}}
{{if .HasFooter}}<Relationship Id="rIdFooter1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/footer" Target="footer1.xml"/>{{end}}
+<Relationship Id="rIdNumbering" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/numbering" Target="numbering.xml"/>
</Relationships>
diff --git a/internal/agent/component/io/templates/numbering.xml.tmpl b/internal/agent/component/io/templates/numbering.xml.tmpl
new file mode 100644
index 0000000000..b8c41edc51
--- /dev/null
+++ b/internal/agent/component/io/templates/numbering.xml.tmpl
@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<w:numbering xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">
+{{range .Lists}}<w:abstractNum w:abstractNumId="{{.ID}}"><w:multiLevelType w:val="singleLevel"/><w:lvl w:ilvl="0"><w:start w:val="{{.Start}}"/><w:numFmt w:val="{{if .Ordered}}decimal{{else}}bullet{{end}}"/><w:lvlText w:val="{{if .Ordered}}%1.{{else}}•{{end}}"/><w:lvlJc w:val="left"/><w:pPr><w:tabs><w:tab w:val="num" w:pos="{{.Indent}}"/></w:tabs><w:ind w:left="{{.Indent}}" w:hanging="360"/></w:pPr></w:lvl></w:abstractNum>
+{{end}}{{range .Lists}}<w:num w:numId="{{.ID}}"><w:abstractNumId w:val="{{.ID}}"/></w:num>
+{{end}}</w:numbering>
diff --git a/internal/agent/component/io/templates/styles.xml.tmpl b/internal/agent/component/io/templates/styles.xml.tmpl
index efed99b295..1b1732dd14 100644
--- a/internal/agent/component/io/templates/styles.xml.tmpl
+++ b/internal/agent/component/io/templates/styles.xml.tmpl
@@ -7,4 +7,10 @@
<w:style w:type="paragraph" w:default="1" w:styleId="Normal"><w:name w:val="Normal"/><w:qFormat/></w:style>
<w:style w:type="paragraph" w:styleId="Header"><w:name w:val="header"/><w:basedOn w:val="Normal"/><w:pPr><w:tabs><w:tab w:val="center" w:pos="4680"/><w:tab w:val="right" w:pos="9360"/></w:tabs><w:spacing w:after="0"/></w:pPr></w:style>
<w:style w:type="paragraph" w:styleId="Footer"><w:name w:val="footer"/><w:basedOn w:val="Normal"/><w:pPr><w:tabs><w:tab w:val="center" w:pos="4680"/><w:tab w:val="right" w:pos="9360"/></w:tabs><w:spacing w:after="0"/></w:pPr></w:style>
+<w:style w:type="paragraph" w:styleId="Heading1"><w:name w:val="heading 1"/><w:basedOn w:val="Normal"/><w:next w:val="Normal"/><w:qFormat/><w:pPr><w:keepNext/><w:keepLines/><w:spacing w:before="240" w:after="120"/><w:outlineLvl w:val="0"/></w:pPr><w:rPr><w:b/><w:bCs/></w:rPr></w:style>
+<w:style w:type="paragraph" w:styleId="Heading2"><w:name w:val="heading 2"/><w:basedOn w:val="Normal"/><w:next w:val="Normal"/><w:qFormat/><w:pPr><w:keepNext/><w:keepLines/><w:spacing w:before="240" w:after="120"/><w:outlineLvl w:val="1"/></w:pPr><w:rPr><w:b/><w:bCs/></w:rPr></w:style>
+<w:style w:type="paragraph" w:styleId="Heading3"><w:name w:val="heading 3"/><w:basedOn w:val="Normal"/><w:next w:val="Normal"/><w:qFormat/><w:pPr><w:keepNext/><w:keepLines/><w:spacing w:before="240" w:after="120"/><w:outlineLvl w:val="2"/></w:pPr><w:rPr><w:b/><w:bCs/></w:rPr></w:style>
+<w:style w:type="paragraph" w:styleId="Heading4"><w:name w:val="heading 4"/><w:basedOn w:val="Normal"/><w:next w:val="Normal"/><w:qFormat/><w:pPr><w:keepNext/><w:keepLines/><w:spacing w:before="240" w:after="120"/><w:outlineLvl w:val="3"/></w:pPr><w:rPr><w:b/><w:bCs/></w:rPr></w:style>
+<w:style w:type="paragraph" w:styleId="Heading5"><w:name w:val="heading 5"/><w:basedOn w:val="Normal"/><w:next w:val="Normal"/><w:qFormat/><w:pPr><w:keepNext/><w:keepLines/><w:spacing w:before="240" w:after="120"/><w:outlineLvl w:val="4"/></w:pPr><w:rPr><w:b/><w:bCs/></w:rPr></w:style>
+<w:style w:type="paragraph" w:styleId="Heading6"><w:name w:val="heading 6"/><w:basedOn w:val="Normal"/><w:next w:val="Normal"/><w:qFormat/><w:pPr><w:keepNext/><w:keepLines/><w:spacing w:before="240" w:after="120"/><w:outlineLvl w:val="5"/></w:pPr><w:rPr><w:b/><w:bCs/></w:rPr></w:style>
</w:styles>