RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(agent): preserve Markdown headings and lists in DOCX (#22078)
Sbaaoui Idriss3a594af0ddcf
diff --git a/internal/agent/component/io/docx_writer.go b/internal/agent/component/io/docx_writer.goindex 0eec48c52c..19f4c001f5 100644--- a/internal/agent/component/io/docx_writer.go+++ b/internal/agent/component/io/docx_writer.go@@ -26,7 +26,7 @@// - //go:embed — the static XML parts (Content_Types, _rels, styles)//// The output is a Word-compatible .docx. The XML is intentionally-// minimal: a single font, a single style, paragraphs, and native tables.+// minimal: a single font, heading styles, paragraphs, and native tables.package ioimport (@@ -54,6 +54,7 @@ var relsXML []byte//go:embed templates/document_rels.xml.tmpl//go:embed templates/styles.xml.tmpl//go:embed templates/document.xml.tmpl+//go:embed templates/numbering.xml.tmpl//go:embed templates/header.xml.tmpl//go:embed templates/footer.xml.tmplvar tmplFS embed.FS@@ -74,6 +75,7 @@ type DOCXOptions struct {// DOCXOptions and Document flatten into this when the writer is// invoked.type docModel struct {+ Lists []*docxListBlocks []docxBlockHeaderText stringFooterText string@@ -89,8 +91,18 @@ type docModel struct {}type docxBlock struct {- Text string- Rows [][]string+ Text string+ Rows [][]string+ HeadingLevel int+ List *docxList+ Indent int+}++type docxList struct {+ ID int+ Ordered bool+ Start int+ Indent int}// tmplFuncs registers the {{xml}} helper used to escape user content@@ -138,7 +150,14 @@ func WriteDOCX(content string, opts DOCXOptions) ([]byte, error) {Timestamp: nowUTC(),}- // Pre-parse all 5 templates once; FuncMaps are shared.+ for _, block := range model.Blocks {+ if block.List != nil && block.List.ID == 0 {+ block.List.ID = len(model.Lists) + 1+ model.Lists = append(model.Lists, block.List)+ }+ }++ // Pre-parse templates once; FuncMaps are shared.docTmpl, err := template.New("document.xml.tmpl").Funcs(tmplFuncs).ParseFS(tmplFS, "templates/document.xml.tmpl")if err != nil {return nil, fmt.Errorf("DOCX: parse document template: %w", err)@@ -160,6 +179,11 @@ func WriteDOCX(content string, opts DOCXOptions) ([]byte, error) {return nil, fmt.Errorf("DOCX: parse document_rels template: %w", err)}+ numberingTmpl, err := template.New("numbering.xml.tmpl").ParseFS(tmplFS, "templates/numbering.xml.tmpl")+ if err != nil {+ return nil, fmt.Errorf("DOCX: parse numbering template: %w", err)+ }+buf := &bytes.Buffer{}zw := zip.NewWriter(buf)@@ -176,6 +200,9 @@ func WriteDOCX(content string, opts DOCXOptions) ([]byte, error) {if err := writeZipTmpl(zw, "word/_rels/document.xml.rels", docRelsTmpl, model); err != nil {return nil, err}+ if err := writeZipTmpl(zw, "word/numbering.xml", numberingTmpl, model); err != nil {+ return nil, err+ }// word/styles.xmlif err := writeZipTmpl(zw, "word/styles.xml", stylesTmpl, model); err != nil {return nil, err@@ -207,9 +234,41 @@ func markdownDOCXBlocks(content string) []docxBlock {content = normalizeMarkdownTableSeparators(content)source := []byte(content)document := goldmark.New(goldmark.WithExtensions(extension.Table)).Parser().Parse(gmtext.NewReader(source))- blocks := make([]docxBlock, 0, document.ChildCount())- for node := document.FirstChild(); node != nil; node = node.NextSibling() {+ blocks := docxChildBlocks(document, source)+ if len(blocks) == 0 {+ return []docxBlock{{}}+ }+ return blocks+}++func docxChildBlocks(parent ast.Node, source []byte) []docxBlock {+ var blocks []docxBlock+ for node := parent.FirstChild(); node != nil; node = node.NextSibling() {switch n := node.(type) {+ case *ast.Heading:+ blocks = append(blocks, docxBlock{Text: inlinePDFText(n, source), HeadingLevel: n.Level})+ case *ast.List:+ depth := 1+ for ancestor := n.Parent(); ancestor != nil; ancestor = ancestor.Parent() {+ if _, ok := ancestor.(*ast.List); ok {+ depth+++ }+ }+ list := &docxList{Ordered: n.IsOrdered(), Start: max(1, n.Start), Indent: depth * 720}+ for item := n.FirstChild(); item != nil; item = item.NextSibling() {+ children := docxChildBlocks(item, source)+ for i := range children {+ if children[i].Indent == 0 && children[i].Rows == nil {+ children[i].Indent = list.Indent+ if i == 0 {+ children[i].List = list+ }+ }+ }+ blocks = append(blocks, children...)+ }+ case *ast.ListItem, *ast.Blockquote:+ blocks = append(blocks, docxChildBlocks(n, source)...)case *ast.FencedCodeBlock:text := rawPDFBlockText(n, source)if strings.EqualFold(string(n.Language(source)), "markdown") || strings.EqualFold(string(n.Language(source)), "md") {@@ -232,14 +291,11 @@ func markdownDOCXBlocks(content string) []docxBlock {}blocks = append(blocks, docxBlock{Rows: rows})default:- if text := rawPDFBlockText(n, source); text != "" || node == document.FirstChild() {+ if text := rawPDFBlockText(n, source); text != "" || node == parent.FirstChild() {blocks = append(blocks, docxBlock{Text: text})}}}- if len(blocks) == 0 {- return []docxBlock{{}}- }return blocks}diff --git a/internal/agent/component/io/docx_writer_test.go b/internal/agent/component/io/docx_writer_test.goindex 48921c6235..3783f08200 100644--- a/internal/agent/component/io/docx_writer_test.go+++ b/internal/agent/component/io/docx_writer_test.go@@ -247,3 +247,91 @@ func truncate(s string, n int) string {}return s[:n] + "..."}++func TestDOCXWriter_MarkdownHeadings(t *testing.T) {+ content := "# Report\n\n## 4. Findings\n\n### 4.2 **Risk & controls**\n\nBody retained.\n\n> #### Quoted heading\n>\n> Quoted body.\n\n- ##### Nested heading\n\n Nested body.\n\n```markdown\n###### Fenced heading\n```"+ doc, err := WriteDOCX(content, DOCXOptions{})+ if err != nil {+ t.Fatal(err)+ }+ zr, err := zip.NewReader(bytes.NewReader(doc), int64(len(doc)))+ if err != nil {+ t.Fatal(err)+ }+ body, _ := readZipFile(t, zr, "word/document.xml")+ var document struct {+ Body struct {+ Paragraphs []struct {+ Properties struct {+ Style struct {+ Value string `xml:"val,attr"`+ } `xml:"pStyle"`+ } `xml:"pPr"`+ Runs []struct {+ Text string `xml:"t"`+ } `xml:"r"`+ } `xml:"p"`+ } `xml:"body"`+ }+ if err := xml.Unmarshal([]byte(body), &document); err != nil {+ t.Fatal(err)+ }+ want := []string{"Report", "4. Findings", "4.2 Risk & controls", "Body retained.", "Quoted heading", "Quoted body.", "Nested heading", "Nested body.", "Fenced heading"}+ styles := []string{"Heading1", "Heading2", "Heading3", "", "Heading4", "", "Heading5", "", "Heading6"}+ if len(document.Body.Paragraphs) != len(want) {+ t.Fatalf("got %d paragraphs, want %d: %s", len(document.Body.Paragraphs), len(want), body)+ }+ for i, paragraph := range document.Body.Paragraphs {+ var text strings.Builder+ for _, run := range paragraph.Runs {+ text.WriteString(run.Text)+ }+ if text.String() != want[i] || paragraph.Properties.Style.Value != styles[i] {+ t.Errorf("paragraph %d = %q (%s), want %q (%s)", i, text.String(), paragraph.Properties.Style.Value, want[i], styles[i])+ }+ }+ styleXML, _ := readZipFile(t, zr, "word/styles.xml")+ for _, style := range styles {+ if style != "" && !strings.Contains(styleXML, `w:styleId="`+style+`"`) {+ t.Errorf("missing style definition %s", style)+ }+ }+}++func TestDOCXWriter_MarkdownLists(t *testing.T) {+ doc, err := WriteDOCX("## 4.2 Findings\n\n- First bullet\n - Nested bullet\n\n Continuation.\n- Second bullet\n\n## 4.3 Actions\n\n3. Third action\n4. Fourth action\n\nNext list:\n\n1. Separate list\n", DOCXOptions{})+ if err != nil {+ t.Fatal(err)+ }+ zr, err := zip.NewReader(bytes.NewReader(doc), int64(len(doc)))+ if err != nil {+ t.Fatal(err)+ }+ body, _ := readZipFile(t, zr, "word/document.xml")+ if count := strings.Count(body, "<w:numPr>"); count != 6 {+ t.Errorf("numbered paragraphs = %d, want 6", count)+ }+ for _, text := range []string{"First bullet", "Nested bullet", "Continuation.", "Second bullet", "Third action", "Fourth action", "Separate list"} {+ if !strings.Contains(body, text) {+ t.Errorf("missing %q", text)+ }+ }+ if !strings.Contains(body, "<w:ind w:left=\"1440\"/></w:pPr><w:r>") {+ t.Error("nested continuation lost its indentation")+ }+ numbering, ok := readZipFile(t, zr, "word/numbering.xml")+ if !ok {+ t.Fatal("missing numbering.xml")+ }+ for _, value := range []string{`w:val="bullet"`, `w:val="decimal"`, `w:start w:val="3"`, `w:left="1440"`} {+ if !strings.Contains(numbering, value) {+ t.Errorf("missing list definition %s", value)+ }+ }+ for name, want := range map[string]string{"[Content_Types].xml": "/word/numbering.xml", "word/_rels/document.xml.rels": "relationships/numbering"} {+ part, _ := readZipFile(t, zr, name)+ if !strings.Contains(part, want) {+ t.Errorf("%s missing %s", name, want)+ }+ }+}diff --git a/internal/agent/component/io/templates/content_types.xml b/internal/agent/component/io/templates/content_types.xmlindex a71383baec..a38e10fb6f 100644--- a/internal/agent/component/io/templates/content_types.xml+++ b/internal/agent/component/io/templates/content_types.xml@@ -7,4 +7,5 @@<Override PartName="/word/header1.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.header+xml"/><Override PartName="/word/footer1.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.footer+xml"/><Override PartName="/word/_rels/document.xml.rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>+<Override PartName="/word/numbering.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.numbering+xml"/></Types>diff --git a/internal/agent/component/io/templates/document.xml.tmpl b/internal/agent/component/io/templates/document.xml.tmplindex a7397243e0..df493a2705 100644--- a/internal/agent/component/io/templates/document.xml.tmpl+++ b/internal/agent/component/io/templates/document.xml.tmpl@@ -1,7 +1,7 @@<?xml version="1.0" encoding="UTF-8" standalone="yes"?><w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><w:body>-{{range .Blocks}}{{if .Rows}}<w:tbl><w:tblPr><w:tblStyle w:val="TableGrid"/></w:tblPr>{{range .Rows}}<w:tr>{{range .}}<w:tc><w:tcPr><w:tcW w:w="0" w:type="auto"/></w:tcPr><w:p><w:r><w:rPr><w:sz w:val="{{pt $.FontSize}}"/><w:szCs w:val="{{pt $.FontSize}}"/><w:rFonts w:ascii="{{$.FontFamily}}" w:eastAsia="{{$.FontFamily}}" w:hAnsi="{{$.FontFamily}}"/></w:rPr><w:t xml:space="preserve">{{xml .}}</w:t></w:r></w:p></w:tc>{{end}}</w:tr>{{end}}</w:tbl>{{else}}<w:p><w:r><w:rPr><w:sz w:val="{{pt $.FontSize}}"/><w:szCs w:val="{{pt $.FontSize}}"/><w:rFonts w:ascii="{{$.FontFamily}}" w:eastAsia="{{$.FontFamily}}" w:hAnsi="{{$.FontFamily}}"/></w:rPr><w:t xml:space="preserve">{{xml .Text}}</w:t></w:r></w:p>{{end}}+{{range .Blocks}}{{if .Rows}}<w:tbl><w:tblPr><w:tblStyle w:val="TableGrid"/></w:tblPr>{{range .Rows}}<w:tr>{{range .}}<w:tc><w:tcPr><w:tcW w:w="0" w:type="auto"/></w:tcPr><w:p><w:r><w:rPr><w:sz w:val="{{pt $.FontSize}}"/><w:szCs w:val="{{pt $.FontSize}}"/><w:rFonts w:ascii="{{$.FontFamily}}" w:eastAsia="{{$.FontFamily}}" w:hAnsi="{{$.FontFamily}}"/></w:rPr><w:t xml:space="preserve">{{xml .}}</w:t></w:r></w:p></w:tc>{{end}}</w:tr>{{end}}</w:tbl>{{else}}<w:p>{{if or .HeadingLevel .Indent}}<w:pPr>{{if .HeadingLevel}}<w:pStyle w:val="Heading{{.HeadingLevel}}"/>{{end}}{{if .List}}<w:numPr><w:ilvl w:val="0"/><w:numId w:val="{{.List.ID}}"/></w:numPr>{{end}}{{if .Indent}}<w:ind w:left="{{.Indent}}"{{if .List}} w:hanging="360"{{end}}/>{{end}}</w:pPr>{{end}}<w:r><w:rPr><w:sz w:val="{{pt $.FontSize}}"/><w:szCs w:val="{{pt $.FontSize}}"/><w:rFonts w:ascii="{{$.FontFamily}}" w:eastAsia="{{$.FontFamily}}" w:hAnsi="{{$.FontFamily}}"/></w:rPr><w:t xml:space="preserve">{{xml .Text}}</w:t></w:r></w:p>{{end}}{{end}}<w:sectPr>{{if .HasHeader}}<w:headerReference w:type="default" r:id="rIdHeader1"/>{{end}}{{if .HasFooter}}<w:footerReference w:type="default" r:id="rIdFooter1"/>{{end}}diff --git a/internal/agent/component/io/templates/document_rels.xml.tmpl b/internal/agent/component/io/templates/document_rels.xml.tmplindex 1893220747..80cbe5a966 100644--- a/internal/agent/component/io/templates/document_rels.xml.tmpl+++ b/internal/agent/component/io/templates/document_rels.xml.tmpl@@ -3,4 +3,5 @@<Relationship Id="rIdStyles" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/styles" Target="styles.xml"/>{{if .HasHeader}}<Relationship Id="rIdHeader1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/header" Target="header1.xml"/>{{end}}{{if .HasFooter}}<Relationship Id="rIdFooter1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/footer" Target="footer1.xml"/>{{end}}+<Relationship Id="rIdNumbering" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/numbering" Target="numbering.xml"/></Relationships>diff --git a/internal/agent/component/io/templates/numbering.xml.tmpl b/internal/agent/component/io/templates/numbering.xml.tmplnew file mode 100644index 0000000000..b8c41edc51--- /dev/null+++ b/internal/agent/component/io/templates/numbering.xml.tmpl@@ -0,0 +1,5 @@+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>+<w:numbering xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">+{{range .Lists}}<w:abstractNum w:abstractNumId="{{.ID}}"><w:multiLevelType w:val="singleLevel"/><w:lvl w:ilvl="0"><w:start w:val="{{.Start}}"/><w:numFmt w:val="{{if .Ordered}}decimal{{else}}bullet{{end}}"/><w:lvlText w:val="{{if .Ordered}}%1.{{else}}•{{end}}"/><w:lvlJc w:val="left"/><w:pPr><w:tabs><w:tab w:val="num" w:pos="{{.Indent}}"/></w:tabs><w:ind w:left="{{.Indent}}" w:hanging="360"/></w:pPr></w:lvl></w:abstractNum>+{{end}}{{range .Lists}}<w:num w:numId="{{.ID}}"><w:abstractNumId w:val="{{.ID}}"/></w:num>+{{end}}</w:numbering>diff --git a/internal/agent/component/io/templates/styles.xml.tmpl b/internal/agent/component/io/templates/styles.xml.tmplindex efed99b295..1b1732dd14 100644--- a/internal/agent/component/io/templates/styles.xml.tmpl+++ b/internal/agent/component/io/templates/styles.xml.tmpl@@ -7,4 +7,10 @@<w:style w:type="paragraph" w:default="1" w:styleId="Normal"><w:name w:val="Normal"/><w:qFormat/></w:style><w:style w:type="paragraph" w:styleId="Header"><w:name w:val="header"/><w:basedOn w:val="Normal"/><w:pPr><w:tabs><w:tab w:val="center" w:pos="4680"/><w:tab w:val="right" w:pos="9360"/></w:tabs><w:spacing w:after="0"/></w:pPr></w:style><w:style w:type="paragraph" w:styleId="Footer"><w:name w:val="footer"/><w:basedOn w:val="Normal"/><w:pPr><w:tabs><w:tab w:val="center" w:pos="4680"/><w:tab w:val="right" w:pos="9360"/></w:tabs><w:spacing w:after="0"/></w:pPr></w:style>+<w:style w:type="paragraph" w:styleId="Heading1"><w:name w:val="heading 1"/><w:basedOn w:val="Normal"/><w:next w:val="Normal"/><w:qFormat/><w:pPr><w:keepNext/><w:keepLines/><w:spacing w:before="240" w:after="120"/><w:outlineLvl w:val="0"/></w:pPr><w:rPr><w:b/><w:bCs/></w:rPr></w:style>+<w:style w:type="paragraph" w:styleId="Heading2"><w:name w:val="heading 2"/><w:basedOn w:val="Normal"/><w:next w:val="Normal"/><w:qFormat/><w:pPr><w:keepNext/><w:keepLines/><w:spacing w:before="240" w:after="120"/><w:outlineLvl w:val="1"/></w:pPr><w:rPr><w:b/><w:bCs/></w:rPr></w:style>+<w:style w:type="paragraph" w:styleId="Heading3"><w:name w:val="heading 3"/><w:basedOn w:val="Normal"/><w:next w:val="Normal"/><w:qFormat/><w:pPr><w:keepNext/><w:keepLines/><w:spacing w:before="240" w:after="120"/><w:outlineLvl w:val="2"/></w:pPr><w:rPr><w:b/><w:bCs/></w:rPr></w:style>+<w:style w:type="paragraph" w:styleId="Heading4"><w:name w:val="heading 4"/><w:basedOn w:val="Normal"/><w:next w:val="Normal"/><w:qFormat/><w:pPr><w:keepNext/><w:keepLines/><w:spacing w:before="240" w:after="120"/><w:outlineLvl w:val="3"/></w:pPr><w:rPr><w:b/><w:bCs/></w:rPr></w:style>+<w:style w:type="paragraph" w:styleId="Heading5"><w:name w:val="heading 5"/><w:basedOn w:val="Normal"/><w:next w:val="Normal"/><w:qFormat/><w:pPr><w:keepNext/><w:keepLines/><w:spacing w:before="240" w:after="120"/><w:outlineLvl w:val="4"/></w:pPr><w:rPr><w:b/><w:bCs/></w:rPr></w:style>+<w:style w:type="paragraph" w:styleId="Heading6"><w:name w:val="heading 6"/><w:basedOn w:val="Normal"/><w:next w:val="Normal"/><w:qFormat/><w:pPr><w:keepNext/><w:keepLines/><w:spacing w:before="240" w:after="120"/><w:outlineLvl w:val="5"/></w:pPr><w:rPr><w:b/><w:bCs/></w:rPr></w:style></w:styles>