RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): preserve table structure across document formats (#21504)
jay777216e2214aaa8cd
diff --git a/internal/ingestion/component/chunker/page.go b/internal/ingestion/component/chunker/page.goindex 10ae52dac1..f0428c38f4 100644--- a/internal/ingestion/component/chunker/page.go+++ b/internal/ingestion/component/chunker/page.go@@ -21,11 +21,8 @@//// Unlike TokenChunker (which merges slides into a single chunk) or// OneChunker (which collapses many slides into one), PageChunker-// keeps each slide as the unit of chunking. The upstream parser produces-// one record per slide with text and slide_number; this chunker passes-// each through unchanged. Note that the PPTX/PPT path does not emit image-// or position information (unlike the PDF path), so slide chunks carry no-// image and no bbox-based preview positioning.+// preserves the upstream slide records, including structured tables and+// images. Every record retains its slide_number for source-page identity.package chunkerimport (diff --git a/internal/ingestion/component/chunker/page_pptx_cgo_test.go b/internal/ingestion/component/chunker/page_pptx_cgo_test.gonew file mode 100644index 0000000000..32925a965c--- /dev/null+++ b/internal/ingestion/component/chunker/page_pptx_cgo_test.go@@ -0,0 +1,73 @@+//go:build cgo++package chunker++import (+ "archive/zip"+ "bytes"+ "io"+ "strings"+ "testing"++ officeOxide "github.com/yfedoseev/office_oxide/go"+ "ragflow/internal/parser/parser"+)++func TestPageChunkerPPTXTableEndToEnd(t *testing.T) {+ writer := officeOxide.NewPptxWriter()+ defer writer.Close()+ slide := writer.AddSlide()+ writer.SetSlideTitle(slide, "Revenue 2026")+ writer.AddSlideText(slide, "Revenue rose")+ original, err := writer.ToBytes()+ if err != nil {+ t.Fatal(err)+ }+ zr, err := zip.NewReader(bytes.NewReader(original), int64(len(original)))+ if err != nil {+ t.Fatal(err)+ }+ var buf bytes.Buffer+ zw := zip.NewWriter(&buf)+ for _, file := range zr.File {+ r, err := file.Open()+ if err != nil {+ t.Fatal(err)+ }+ data, err := io.ReadAll(r)+ r.Close()+ if err != nil {+ t.Fatal(err)+ }+ if file.Name == "ppt/slides/slide1.xml" {+ table := `<p:graphicFrame><p:nvGraphicFramePr><p:cNvPr id="50" name="Table 1"/><p:cNvGraphicFramePr/><p:nvPr/></p:nvGraphicFramePr><p:xfrm><a:off x="0" y="0"/><a:ext cx="4000000" cy="1000000"/></p:xfrm><a:graphic><a:graphicData uri="http://schemas.openxmlformats.org/drawingml/2006/table"><a:tbl><a:tblPr/><a:tblGrid><a:gridCol w="2000000"/><a:gridCol w="2000000"/></a:tblGrid><a:tr h="400000"><a:tc><a:txBody><a:bodyPr/><a:lstStyle/><a:p><a:r><a:t>Q1</a:t></a:r></a:p></a:txBody><a:tcPr/></a:tc><a:tc><a:txBody><a:bodyPr/><a:lstStyle/><a:p><a:r><a:t>100</a:t></a:r></a:p></a:txBody><a:tcPr/></a:tc></a:tr></a:tbl></a:graphicData></a:graphic></p:graphicFrame>`+ source := string(data)+ if !strings.Contains(source, "</p:sp>") {+ t.Fatalf("slide does not contain shapes: %s", source)+ }+ data = []byte(strings.Replace(source, "</p:sp>", "</p:sp>"+table, 1))+ }+ w, err := zw.Create(file.Name)+ if err != nil {+ t.Fatal(err)+ }+ if _, err := w.Write(data); err != nil {+ t.Fatal(err)+ }+ }+ if err := zw.Close(); err != nil {+ t.Fatal(err)+ }+ parsed := parser.NewPPTXParser().ParseWithResult(t.Context(), "deck.pptx", buf.Bytes())+ if parsed.Err != nil {+ t.Fatal(parsed.Err)+ }+ chunks := slideChunksOf(t, map[string]any{"name": "deck.pptx", "file_type": "pptx", "output_format": "json", "json": parsed.JSON})+ if len(chunks) != 3 {+ t.Fatalf("items=%#v chunks=%#v", parser.ItemsToMaps(parsed.JSON), chunks)+ }+ text, _ := chunks[1]["text"].(string)+ if chunks[0]["text"] != "Revenue 2026" || chunks[1]["doc_type_kwd"] != "table" || chunks[1]["table"] == nil || !strings.Contains(text, "<th>Q1</th><th>100</th>") || chunks[2]["text"] != "Revenue rose" {+ t.Fatalf("PPTX slide contents lost: items=%#v chunks=%#v", parser.ItemsToMaps(parsed.JSON), chunks)+ }+}diff --git a/internal/ingestion/component/chunker/page_test.go b/internal/ingestion/component/chunker/page_test.goindex 447bf9a208..e93f3df296 100644--- a/internal/ingestion/component/chunker/page_test.go+++ b/internal/ingestion/component/chunker/page_test.go@@ -18,8 +18,12 @@ package chunkerimport ("context"- "ragflow/internal/ingestion/component/schema"+ "fmt""testing"++ "ragflow/internal/entity"+ "ragflow/internal/ingestion/component/schema"+ "ragflow/internal/parser/parser")// slideChunksOf drives PageChunker.Invoke and returns the emitted maps.@@ -118,3 +122,56 @@ func TestPageChunker_NilInput(t *testing.T) {t.Fatalf("expected 0 chunks for nil input, got %d", len(chunks))}}++func TestPageChunkerPreservesSlideTableItems(t *testing.T) {+ table := map[string]any{"grid": []any{[]any{map[string]any{"text": "cell"}}}, "header_rows": 1}+ chunks := slideChunksOf(t, map[string]any{"name": "deck.pptx", "output_format": "json", "json": []map[string]any{+ {"text": "Before", "doc_type_kwd": "text", "slide_number": 1},+ {"doc_type_kwd": "table", "slide_number": 1, "source_table_id": "table-1", "table": table},+ {"doc_type_kwd": "image", "slide_number": 1, "parent_table_id": "table-1", "image": "one"},+ {"doc_type_kwd": "table", "slide_number": 1, "source_table_id": "table-2", "table": table},+ {"doc_type_kwd": "image", "slide_number": 1, "parent_table_id": "table-2", "image": "two"},+ {"text": "After", "doc_type_kwd": "text", "slide_number": 1},+ }})+ if len(chunks) != 6 {+ t.Fatalf("expected ordered text, tables and images: %#v", chunks)+ }+ if chunks[0]["text"] != "Before" || chunks[5]["text"] != "After" {+ t.Fatalf("text order lost: %#v", chunks)+ }+ for _, index := range []int{1, 3} {+ if chunks[index]["doc_type_kwd"] != "table" || chunks[index]["table"] == nil || chunks[index+1]["parent_table_id"] != chunks[index]["source_table_id"] {+ t.Fatalf("table structure or image association lost: %#v", chunks)+ }+ }+ for _, chunk := range chunks {+ if got := fmt.Sprint(chunk["slide_number"]); got != "1" {+ t.Fatalf("slide_number lost: %#v", chunk)+ }+ }+}++// TestPageChunkerPreservesTypedSlideTableItems covers the in-memory+// []parser.DocItem leg, which bypasses the JSON-map decode path.+func TestPageChunkerPreservesTypedSlideTableItems(t *testing.T) {+ table := &entity.TableData{+ Grid: [][]entity.TableCell{{{Text: "cell"}}},+ HeaderRows: 1,+ }+ maps := []map[string]any{+ {"text": "Before", "doc_type_kwd": "text", "slide_number": 1},+ {"doc_type_kwd": "table", "slide_number": 1, "source_table_id": "table-1", "table": table},+ {"doc_type_kwd": "image", "slide_number": 1, "parent_table_id": "table-1", "image": "one"},+ }+ chunks := slideChunksOf(t, map[string]any{+ "name": "deck.pptx", "output_format": "json", "json": parser.MapsToItems(maps),+ })+ if len(chunks) != len(maps) {+ t.Fatalf("chunks = %d, want %d: %#v", len(chunks), len(maps), chunks)+ }+ for _, chunk := range chunks {+ if got := fmt.Sprint(chunk["slide_number"]); got != "1" {+ t.Fatalf("slide_number lost through typed DocItem input: %#v", chunk)+ }+ }+}diff --git a/internal/ingestion/component/chunker/parser_table_test.go b/internal/ingestion/component/chunker/parser_table_test.gonew file mode 100644index 0000000000..b640ec13a3--- /dev/null+++ b/internal/ingestion/component/chunker/parser_table_test.go@@ -0,0 +1,115 @@+package chunker++import (+ "archive/zip"+ "bytes"+ "encoding/json"+ "os"+ "strings"+ "testing"++ "ragflow/internal/parser/parser"+)++func TestGeneralChunkerPreservesRetrievalHandbookTables(t *testing.T) {+ data, err := os.ReadFile("testdata/tables/retrieval_md_20_ret.md")+ if err != nil {+ t.Fatal(err)+ }+ p, err := parser.NewMarkdownParser(parser.GoMarkdown)+ if err != nil {+ t.Fatal(err)+ }+ parsed := p.ParseWithResult(t.Context(), "retrieval_md_20_ret.md", data)+ if parsed.Err != nil {+ t.Fatal(parsed.Err)+ }+ wire, err := json.Marshal(parsed.JSON)+ if err != nil {+ t.Fatal(err)+ }+ var items []map[string]any+ if err := json.Unmarshal(wire, &items); err != nil {+ t.Fatal(err)+ }+ component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 512})+ if err != nil {+ t.Fatal(err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "retrieval_md_20_ret.md", "file_type": "md", "output_format": "json", "json": items,+ })+ if err != nil {+ t.Fatal(err)+ }+ tables := 0+ metadataCopies := 0+ for _, chunk := range outputChunks(t, out) {+ text, _ := chunk["text"].(string)+ if chunk["doc_type_kwd"] == "table" {+ tables+++ }+ if strings.Contains(text, "Document ID") {+ metadataCopies+++ if chunk["doc_type_kwd"] != "table" {+ t.Fatalf("metadata flattened: %#v", chunk)+ }+ for _, row := range []string{+ "<td>Document ID</td><td>retrieval_md_20_ret.md</td>",+ "<td>Component code</td><td>RET</td>",+ "<td>Owner</td><td>Platform Reliability (ret-team)</td>",+ "<td>Last reviewed</td><td>2026-09-10</td>",+ "<td>Review cycle</td><td>every 90 days</td>",+ } {+ if !strings.Contains(text, row) {+ t.Errorf("metadata missing row %q: %s", row, text)+ }+ }+ }+ }+ if tables != 4 || metadataCopies != 1 {+ t.Fatalf("tables=%d metadata copies=%d, want 4 and 1", tables, metadataCopies)+ }+}++func TestGeneralChunkerPreservesEPUBParagraphBoundaries(t *testing.T) {+ var data bytes.Buffer+ writer := zip.NewWriter(&data)+ for name, content := range map[string]string{+ "META-INF/container.xml": `<container><rootfiles><rootfile full-path="OEBPS/content.opf"/></rootfiles></container>`,+ "OEBPS/content.opf": `<package><manifest><item id="a" href="chapter.xhtml"/></manifest><spine><itemref idref="a"/></spine></package>`,+ "OEBPS/chapter.xhtml": `<html><body><p>Alpha</p><p>Beta</p><table><tr><td>Item</td><td>Value</td></tr></table><p>Gamma</p><p>Delta</p></body></html>`,+ } {+ entry, err := writer.Create(name)+ if err != nil {+ t.Fatal(err)+ }+ if _, err := entry.Write([]byte(content)); err != nil {+ t.Fatal(err)+ }+ }+ if err := writer.Close(); err != nil {+ t.Fatal(err)+ }+ parsed := parser.NewEPUBParser().ParseWithResult(t.Context(), "book.epub", data.Bytes())+ if parsed.Err != nil {+ t.Fatal(parsed.Err)+ }+ component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 512})+ if err != nil {+ t.Fatal(err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{"name": "book.epub", "output_format": "json", "json": parsed.JSON})+ if err != nil {+ t.Fatal(err)+ }+ chunks := outputChunks(t, out)+ if len(chunks) != 3 {+ t.Fatalf("chunks=%#v", chunks)+ }+ before, _ := chunks[0]["text"].(string)+ after, _ := chunks[2]["text"].(string)+ if before != "Alpha Beta" || chunks[1]["doc_type_kwd"] != "table" || after != "Gamma Delta" {+ t.Fatalf("paragraph or table boundaries lost: %#v", chunks)+ }+}diff --git a/internal/ingestion/component/chunker/table.go b/internal/ingestion/component/chunker/table.goindex 0da8aa90ce..c662b275af 100644--- a/internal/ingestion/component/chunker/table.go+++ b/internal/ingestion/component/chunker/table.go@@ -153,7 +153,6 @@ func expandHTMLTableRows(item schema.ChunkDoc) []schema.ChunkDoc {out.TableData = nilreturn []schema.ChunkDoc{out}}- names := rows[0]// R1: a row chunk must carry only its own tuple. That is only sound when// the item carries spreadsheet identity and its matrix was built// row-aligned (one five-field tuple per row, header included); a@@ -177,7 +176,14 @@ func expandHTMLTableRows(item schema.ChunkDoc) []schema.ChunkDoc {for i, row := range rows[headerCount:] {// Skip empty rows (no cell content); the HTML render of an empty row// would be a header-only table that duplicates the header-only chunk.- if tableRowRecordText(names, row) == "" {+ hasText := false+ for _, cell := range row {+ if strings.TrimSpace(cell) != "" {+ hasText = true+ break+ }+ }+ if !hasText {continue}doc := item@@ -231,29 +237,6 @@ func tableDataOf(item schema.ChunkDoc) *entity.TableData {return item.TableData}-// tableRowRecordText renders one row in the Python table chunker's line format:-// "- {field}: {value}" per non-empty cell, newline joined. A cell whose-// column has no header name keeps its value alone rather than being dropped.-func tableRowRecordText(names, cells []string) string {- lines := make([]string, 0, len(cells))- for j, cell := range cells {- value := strings.TrimSpace(cell)- if value == "" {- continue- }- name := ""- if j < len(names) {- name = strings.TrimSpace(names[j])- }- if name != "" {- lines = append(lines, "- "+name+": "+value)- continue- }- lines = append(lines, "- "+value)- }- return strings.Join(lines, "\n")-}-func init() {MustRegisterChunker(ComponentNameTableChunker)}diff --git a/internal/ingestion/component/chunker/testdata/tables/retrieval_md_20_ret.md b/internal/ingestion/component/chunker/testdata/tables/retrieval_md_20_ret.mdnew file mode 100644index 0000000000..c107c3ac19--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/tables/retrieval_md_20_ret.md@@ -0,0 +1,92 @@+# Retrieval Evaluation Handbook++| | |+|---|---|+| Document ID | `retrieval_md_20_ret.md` |+| Component code | `RET` |+| Owner | Platform Reliability (ret-team) |+| Last reviewed | 2026-09-10 |+| Review cycle | every 90 days |++## 1. Purpose++This handbook covers **Retrieval Evaluation Handbook**. It exists to measure and improve retrieval quality, including recall, ranking and the effect of configuration changes.++It is written for on-call engineers who already have access to the platform+console and to the read-only observability dashboards. Everything in this+document describes steady-state behaviour; emergency procedures are owned by+the incident commander and are not repeated here.++## 2. Service endpoints++| Endpoint | Description |+|---|---|+| `POST /v1/eval/runs` | 提交一次离线评测 |+| `GET /v1/eval/runs/{id}` | 读取评测指标与逐条明细 |++All endpoints require a bearer token and are subject to the platform rate+limit. Requests without a valid token are rejected before any work is done.++## 3. Configuration defaults++Unless a site profile overrides them, the following values apply to every+deployment of Retrieval Evaluation Handbook. These are the values the operations team should quote+when sizing, tuning or comparing environments.++- **default worker concurrency**: 16 workers+- **request timeout**: 80 seconds+- **maximum retry count**: 2 retries+- **health check interval**: 35 seconds+- **maintenance batch size**: 384 items++Changing any of the values above is a configuration change and must go through+the normal review process. Values that differ from the defaults should be+recorded in the site profile so that later comparisons remain meaningful.++## 4. Other tuning parameters++The parameters below are environment-specific and are normally left at the+values shown. They are listed here for completeness.++| Parameter | Typical value |+|---|---|+| Maximum concurrent uploads | 2 |+| Log retention | 34 days |+| Slow query threshold | 400 ms |+| Circuit breaker threshold | 5 consecutive failures |+| Trace sampling ratio | 0.1 |++## 5. Normal operation++1. Confirm that the health endpoint reports the component as serving before+ starting any maintenance work.+2. Verify that no rebalance, rebuild or compaction is already in progress.+3. Apply changes to one node or one replica group at a time.+4. Watch the error rate and the latency dashboards for at least one full window+ after each step.+5. Record the change in the operations log with the exact parameter values+ used, so that the next reviewer can reconstruct the state.++## 6. Failure handling++| Symptom | First action |+|---|---|+| 评测结果不可复现 | 固定随机种子与候选集快照 |+| 标注与文档版本不匹配 | 用文档指纹校验标注时效 |++If none of the actions above resolves the symptom, escalate to the component+owner rather than retrying indefinitely: repeated retries of a saturated+component make the original symptom harder to observe.++## 7. Maintenance window procedure++- Announce the window at least one business day in advance.+- Take a restorable snapshot before touching configuration.+- Prefer draining traffic over restarting under load.+- After the window, re-run the standard smoke checks and the retrieval+ evaluation job before declaring the window closed.++## 8. Related documents++- `retrieval_md_05_chk.md`+- `retrieval_md_09_rank.md`diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex 517ceeeede..4cafa80311 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -247,9 +247,9 @@ func (c *TokenChunkerComponent) invoke(ctx context.Context, db *gorm.DB, inputsfunc decodeChunkerFromUpstream(inputs map[string]any) (schema.ChunkerFromUpstream, error) {var out schema.ChunkerFromUpstream// Fast path: the parser emits typed []parser.DocItem. Convert it to- // []ChunkDoc directly via schema.ChunkDocsFromDocItem (which reads the- // item's typed fields and never calls DocItem.ToMap), then let the JSON- // marshal below serialize the already-typed ChunkDoc. This keeps the+ // []ChunkDoc directly via schema.ChunkDocsFromDocItem (which projects each+ // item through its canonical ToMap wire shape), then let the JSON marshal+ // below serialize the already-typed ChunkDoc. This keeps the// chunker free of the historical map intermediate for the in-memory// Parser -> Chunker leg. When inputs["json"] is already []map[string]any// (legacy fixtures, round-tripped JSON, or absent), the original path runs.diff --git a/internal/parser/parser/charset_test.go b/internal/parser/parser/charset_test.goindex e1b3b06e7a..23c1c0988b 100644--- a/internal/parser/parser/charset_test.go+++ b/internal/parser/parser/charset_test.go@@ -217,7 +217,7 @@ func TestEPUBParser_GB2312(t *testing.T) {if len(res.JSON) == 0 {t.Fatal("want non-empty JSON items from EPUB")}- text, _ := res.JSON[0].ToMap()["text"].(string)+ text := epubTestText(res)if strings.ContainsRune(text, '\ufffd') {t.Errorf("EPUB text contains replacement runes: %s", text)}diff --git a/internal/parser/parser/docx_caption_test.go b/internal/parser/parser/docx_caption_test.goindex 194c45296b..362dc1c2c5 100644--- a/internal/parser/parser/docx_caption_test.go+++ b/internal/parser/parser/docx_caption_test.go@@ -16,8 +16,7 @@ const docxCaptionTableBody = `{"type":"table","rows":[{"cells":[{"content":[{"ty// docxCaptionFindTableCaption returns the Caption of the (single) table// section, which buildDOCXJSONSections stores on the structured-// entity.TableData (after projectPDFTable parses the injected <caption>-// out of the table HTML).+// entity.TableData through the Office IR table projection.func docxCaptionFindTableCaption(t *testing.T, sections []map[string]any) string {t.Helper()for _, s := range sections {diff --git a/internal/parser/parser/docx_ir.go b/internal/parser/parser/docx_ir.goindex 3bb904be0a..dfe3707ad1 100644--- a/internal/parser/parser/docx_ir.go+++ b/internal/parser/parser/docx_ir.go@@ -28,8 +28,10 @@ import ("encoding/base64""encoding/json""fmt"- "html""strings"++ "ragflow/internal/entity"+ "ragflow/internal/parser/tableutil")// DOCXFigure represents one embedded image plus its surrounding text@@ -188,29 +190,17 @@ func joinCellText(cell docxIRCell) string {return strings.Join(parts, "\n")}-// docxIRTableToHTML converts a table IR element to an HTML table string.-// A non-empty caption is emitted as a leading <caption> element so the-// structured TableData contract (ParseTableHTML) carries it through to the-// chunk renderer, which escapes it again on output.-func docxIRTableToHTML(el docxIRElement, caption string) string {- var sb strings.Builder- sb.WriteString("<table>")- if caption != "" {- sb.WriteString("<caption>")- sb.WriteString(html.EscapeString(caption))- sb.WriteString("</caption>")- }+// officeIRTableData projects native Office rows directly onto the table contract.+func officeIRTableData(el docxIRElement, caption string) *entity.TableData {+ rows := make([][]tableutil.CellInput, 0, len(el.Rows))for _, row := range el.Rows {- sb.WriteString("<tr>")+ cells := make([]tableutil.CellInput, 0, len(row.Cells))for _, cell := range row.Cells {- sb.WriteString("<td>")- sb.WriteString(html.EscapeString(joinCellText(cell)))- sb.WriteString("</td>")+ cells = append(cells, tableutil.CellInput{Text: joinCellText(cell)})}- sb.WriteString("</tr>")+ rows = append(rows, cells)}- sb.WriteString("</table>")- return sb.String()+ return tableutil.BuildTableData(rows, nil, strings.TrimSpace(caption))}type docxTableImage struct {@@ -423,15 +413,22 @@ func buildDOCXJSONSectionsCore(irJSON string, budget *embeddedMediaBudget, table}var sections []map[string]anytableSequence := 0+ captionSequence := 0var headings []docxHeading- for si := range ir.Sections {- sec := &ir.Sections[si]- for ei := 0; ei < len(sec.Elements); ei++ {- el := sec.Elements[ei]- // TEMPORARY: this merge is a stopgap. Revisit it once office_oxide- // populates `TableRow.is_header` for .doc tables (upstream issue+ var emitElements func([]docxIRElement, bool)+ emitElements = func(elements []docxIRElement, topLevel bool) {+ if !topLevel {+ parentHeadings := headings+ headings = append([]docxHeading(nil), headings...)+ defer func() { headings = parentHeadings }()+ }+ for ei := 0; ei < len(elements); ei++ {+ el := elements[ei]+ // TEMPORARY: this merge is a stopgap. Revisit it once docxIRRow+ // carries a three-state header field and office_oxide populates+ // `TableRow.is_header` for .doc tables (upstream issue// yfedoseev/office_oxide#456) — at that point prefer rejoining via- // the real `is_header` flag and drop the structural proxy below.+ // the real header flag and drop the structural proxy below.//// office_oxide splits a .doc table whose first row is a styled// header/title row into a 1-row "header" table immediately@@ -466,8 +463,8 @@ func buildDOCXJSONSectionsCore(irJSON string, budget *embeddedMediaBudget, table// separate.// - adjacency (no element between): any heading/paragraph/image// in between keeps the two tables distinct.- if el.Type == "table" && len(el.Rows) == 1 && ei+1 < len(sec.Elements) {- if nxt := sec.Elements[ei+1]; nxt.Type == "table" &&+ if topLevel && el.Type == "table" && len(el.Rows) == 1 && ei+1 < len(elements) {+ if nxt := elements[ei+1]; nxt.Type == "table" &&docxIRTableCols(el) > 0 && docxIRTableCols(el) == docxIRTableCols(nxt) {merged := nxtrows := make([]docxIRRow, 0, len(nxt.Rows)+len(el.Rows))@@ -493,8 +490,8 @@ func buildDOCXJSONSectionsCore(irJSON string, budget *embeddedMediaBudget, table// to that table as the caption instead. The text match keeps// this safe if the document scan and the IR element order// ever desync: a non-matching paragraph is never dropped.- if tableSequence < len(tableCaptions) {- if explicit := strings.TrimSpace(tableCaptions[tableSequence]); explicit != "" &&+ if topLevel && captionSequence < len(tableCaptions) {+ if explicit := strings.TrimSpace(tableCaptions[captionSequence]); explicit != "" &&strings.TrimSpace(docxElementText(el, " ")) == explicit {continue}@@ -506,6 +503,9 @@ func buildDOCXJSONSectionsCore(irJSON string, budget *embeddedMediaBudget, tablecase "table":tableSequence+++ if topLevel {+ captionSequence+++ }if len(el.Rows) == 0 {continue}@@ -513,71 +513,58 @@ func buildDOCXJSONSectionsCore(irJSON string, budget *embeddedMediaBudget, table// stopgap, see docx_caption_scan.go) over the synthesized// heading-location caption.caption := docxHeadingLocationCaption(headings)- if idx := tableSequence - 1; idx >= 0 && idx < len(tableCaptions) {+ if idx := captionSequence - 1; topLevel && idx >= 0 && idx < len(tableCaptions) {if explicit := strings.TrimSpace(tableCaptions[idx]); explicit != "" {caption = explicit}}- html := docxIRTableToHTML(el, caption)- table := map[string]any{- "text": html,- "image": nil,- "doc_type_kwd": "table",- }- projectPDFTable(table)- tableID := ""- foundImage := false- var tableImages []docxTableImage- forEachDOCXTableImage(el, func(tableImage docxTableImage) bool {- foundImage = true- included, keepWalking := budget.include(tableImage.data)- if included || keepWalking {- tableImage.included = included- tableImages = append(tableImages, tableImage)- }- return keepWalking- })- if foundImage {- tableID = fmt.Sprintf("docx-table-%d", tableSequence)- table["source_table_id"] = tableID- }- sections = append(sections, table)- for mediaOrder, tableImage := range tableImages {- metadata := map[string]any{- "parent_table_id": tableID,- "row_index": tableImage.row,- "column_index": tableImage.column,- "media_order": mediaOrder + 1,- }- sections = appendDOCXImagePayload(sections, tableImage.data, tableImage.included, metadata)- }+ sections = appendOfficeTableSections(sections, el, caption, fmt.Sprintf("docx-table-%d", tableSequence), budget)case "list":for _, item := range el.Items {- text := extractTextFromListItem(item)- if text != "" {- sections = append(sections, map[string]any{- "text": text,- "image": nil,- "doc_type_kwd": "text",- })+ emitElements(item.Content, false)+ if item.Nested != nil {+ emitElements([]docxIRElement{{Type: "list", Items: item.Nested.Items}}, false)}- sections = appendDOCXListImageSections(sections, docxIRList{Items: []docxIRListItem{item}}, budget)}-case "text_box":- text := extractTextFromBlockElements(el.contentBlocks())- if text != "" {- sections = append(sections, map[string]any{- "text": text,- "image": nil,- "doc_type_kwd": "text",- })- }- sections = appendDOCXIRImageSections(sections, el.contentBlocks(), budget)+ emitElements(el.contentBlocks(), false)}}}+ for _, sec := range ir.Sections {+ emitElements(sec.Elements, true)+ }+ return sections+}++// appendOfficeTableSections emits a table and its embedded images together.+func appendOfficeTableSections(sections []map[string]any, el docxIRElement, caption, tableID string, budget *embeddedMediaBudget) []map[string]any {+ table := NewTableDataJSONItem(officeIRTableData(el, caption), "", nil).ToMap()++ var tableImages []docxTableImage+ forEachDOCXTableImage(el, func(tableImage docxTableImage) bool {+ included, keepWalking := budget.include(tableImage.data)+ if included || keepWalking {+ tableImage.included = included+ tableImages = append(tableImages, tableImage)+ }+ return keepWalking+ })+ if len(tableImages) > 0 {+ table["source_table_id"] = tableID+ }+ sections = append(sections, table)+ for mediaOrder, tableImage := range tableImages {+ metadata := map[string]any{+ "parent_table_id": tableID,+ "row_index": tableImage.row,+ "column_index": tableImage.column,+ "media_order": mediaOrder + 1,+ }+ sections = appendDOCXImagePayload(sections, tableImage.data, tableImage.included, metadata)+ }+return sections}@@ -603,8 +590,10 @@ func appendDOCXParagraphSections(sections []map[string]any, el docxIRElement, he}sections = append(sections, item)}- appendImage := func(data []byte) {- sections, _ = appendDOCXImageSection(sections, data, budget, nil)+ appendImage := func(data []byte) bool {+ var keepWalking bool+ sections, keepWalking = appendDOCXImageSection(sections, data, budget, nil)+ return keepWalking}for _, run := range el.contentRuns() {switch run.Type {@@ -615,12 +604,8 @@ func appendDOCXParagraphSections(sections []map[string]any, el docxIRElement, hecase "image":flushText()appendImage(run.Data)- for _, nested := range run.Content {- if nested.Type == "image" {- appendImage(nested.Data)- }- }}+ forEachDOCXIRImage(run.Content, func(data []byte) bool { flushText(); return appendImage(data) })}flushText()return sections@@ -637,25 +622,6 @@ func appendDOCXImageSection(sections []map[string]any, data []byte, budget *embereturn appendDOCXImagePayload(sections, data, included, metadata), keepWalking}-func appendDOCXListImageSections(sections []map[string]any, list docxIRList, budget *embeddedMediaBudget) []map[string]any {- for _, item := range list.Items {- sections = appendDOCXIRImageSections(sections, item.Content, budget)- if item.Nested != nil {- sections = appendDOCXListImageSections(sections, *item.Nested, budget)- }- }- return sections-}--func appendDOCXIRImageSections(sections []map[string]any, elements []docxIRElement, budget *embeddedMediaBudget) []map[string]any {- forEachDOCXIRImage(elements, func(data []byte) bool {- var keepWalking bool- sections, keepWalking = appendDOCXImageSection(sections, data, budget, nil)- return keepWalking- })- return sections-}-func appendDOCXImagePayload(sections []map[string]any, data []byte, included bool, metadata map[string]any) []map[string]any {item := map[string]any{"text": "",@@ -694,18 +660,7 @@ func extractDOCXFiguresFromIR(irJSON string, budget *embeddedMediaBudget) []DOCXvar flat []flatBlockfor _, sec := range ir.Sections {- for _, el := range sec.Elements {- if el.Type == "image" {- flat = append(flat, flatBlock{imageData: el.Data})- continue- }- if el.Type == "paragraph" || el.Type == "heading" {- flat = append(flat, docxInlineFlatBlocks(el)...)- continue- }- text := docxElementText(el, "\n")- flat = append(flat, flatBlock{text: text})- }+ flat = appendDOCXFigureBlocks(flat, sec.Elements)}var figures []DOCXFigure@@ -750,6 +705,43 @@ func extractDOCXFiguresFromIR(irJSON string, budget *embeddedMediaBudget) []DOCXreturn figures}+// appendDOCXFigureBlocks flattens IR elements in document order, including+// images nested in tables, lists and text boxes. Section emission recurses+// through the same containers, so figure metadata must use the same traversal+// instead of recording only top-level images.+func appendDOCXFigureBlocks(flat []flatBlock, elements []docxIRElement) []flatBlock {+ for _, el := range elements {+ switch el.Type {+ case "image":+ if len(el.Data) > 0 {+ flat = append(flat, flatBlock{imageData: el.Data})+ }+ case "paragraph", "heading":+ flat = append(flat, docxInlineFlatBlocks(el)...)+ case "table":+ for _, row := range el.Rows {+ for _, cell := range row.Cells {+ flat = appendDOCXFigureBlocks(flat, cell.Content)+ }+ }+ case "list":+ for _, item := range el.Items {+ flat = appendDOCXFigureBlocks(flat, item.Content)+ if item.Nested != nil {+ flat = appendDOCXFigureBlocks(flat, []docxIRElement{{Type: "list", Items: item.Nested.Items}})+ }+ }+ case "text_box":+ flat = appendDOCXFigureBlocks(flat, el.contentBlocks())+ default:+ if text := docxElementText(el, "\n"); text != "" {+ flat = append(flat, flatBlock{text: text})+ }+ }+ }+ return flat+}+// docxInlineFlatBlocks preserves the text/image sequence inside a paragraph// for figure context extraction. Top-level element order alone is not enough// for DOCX because office_oxide can encode an inline image as a paragraph run.diff --git a/internal/parser/parser/docx_ir_test.go b/internal/parser/parser/docx_ir_test.goindex d26229c900..866ee36097 100644--- a/internal/parser/parser/docx_ir_test.go+++ b/internal/parser/parser/docx_ir_test.go@@ -34,7 +34,7 @@ func TestBuildDOCXJSONSections_FromJSON(t *testing.T) {{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},// The table sits under heading "Title", so it carries the// heading-location caption (mirrors Python __get_nearest_title).- {"image": nil, "doc_type_kwd": "table", "table": &entity.TableData{Grid: [][]entity.TableCell{{{Text: "cell", ColSpan: 1, RowSpan: 1}}}, HeaderRows: 1, Caption: "Table Location: Title"}},+ {"doc_type_kwd": "table", "table": &entity.TableData{Grid: [][]entity.TableCell{{{Text: "cell", ColSpan: 1, RowSpan: 1}}}, HeaderRows: 1, Caption: "Table Location: Title"}},{"text": "item1", "image": nil, "doc_type_kwd": "text"},}if !reflect.DeepEqual(got, want) {@@ -516,6 +516,28 @@ func TestExtractDOCXFiguresFromIR_NonParagraphContext(t *testing.T) {}}+// TestExtractDOCXFiguresFromIR_NestedImages verifies that images inside+// tables and text boxes reach the figure metadata. Section emission recurses+// into these containers, so a top-level-only figure walk would omit images+// the parser actually emits.+func TestExtractDOCXFiguresFromIR_NestedImages(t *testing.T) {+ irJSON := `{"sections":[{"elements":[+ {"type":"paragraph","content":[{"type":"text","text":"before"}]},+ {"type":"table","rows":[{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"cell"},{"type":"image","data":"aGVsbG8="}]}]}]}]},+ {"type":"text_box","content":[{"type":"paragraph","content":[{"type":"image","data":"d29ybGQ="}]}]}+ ]}]}`+ figs := extractDOCXFiguresFromIR(irJSON, newEmbeddedMediaBudget())+ if len(figs) != 2 {+ t.Fatalf("expected 2 nested figures, got %d: %+v", len(figs), figs)+ }+ if figs[0].Image != "aGVsbG8=" || figs[1].Image != "d29ybGQ=" {+ t.Fatalf("nested figures = %+v", figs)+ }+ if !strings.Contains(figs[0].ContextAbove, "cell") {+ t.Errorf("ContextAbove = %q, want the nested cell text", figs[0].ContextAbove)+ }+}+// TestCollectDOCXText_BoundedByMaxLen pins that the joined context// never exceeds maxLen runes, even though newline separators are// inserted between blocks (they are not counted by the per-block@@ -611,8 +633,7 @@ func TestExtractTextFromListItem_NestedNull(t *testing.T) {}// TestExtractTextFromListItem_NestedTable verifies that a table nested-// inside a list item's content is flattened, not silently dropped. The old-// path only recognized paragraph/heading blocks in item.Content.+// inside a list item remains structured beside the preceding text.func TestExtractTextFromListItem_NestedTable(t *testing.T) {irJSON := `{"sections":[{"elements":[{"type":"list","items":[@@ -628,12 +649,10 @@ func TestExtractTextFromListItem_NestedTable(t *testing.T) {]}]}]}`sections := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())- if len(sections) != 1 {- t.Fatalf("expected 1 section, got %d: %+v", len(sections), sections)- }- if got := sections[0]["text"]; got != "before\nc1\nc2" {- t.Errorf("list item with nested table text = %q, want %q", got, "before\nc1\nc2")+ if len(sections) != 2 || sections[0]["text"] != "before" {+ t.Fatalf("sections=%#v", sections)}+ requireTableRows(t, sections[1], [][]string{{"c1", "c2"}})}// TestJoinCellText_NestedList verifies that a list nested inside a table@@ -682,9 +701,7 @@ func TestBuildDOCXJSONSections_TextBoxTable(t *testing.T) {if len(sections) != 1 {t.Fatalf("expected 1 section, got %d: %+v", len(sections), sections)}- if got := sections[0]["text"]; got != "Q1\nA1" {- t.Errorf("text_box table text = %q, want %q", got, "Q1\nA1")- }+ requireTableRows(t, sections[0], [][]string{{"Q1", "A1"}})}// TestBuildDOCXJSONSections_NestedList verifies end-to-end that a@@ -700,11 +717,13 @@ func TestBuildDOCXJSONSections_NestedList(t *testing.T) {]}]}]}`sections := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())- if len(sections) != 1 {- t.Fatalf("expected 1 section, got %d: %+v", len(sections), sections)+ if len(sections) != 3 {+ t.Fatalf("sections=%#v", sections)}- if got := sections[0]["text"]; got != "L1\nL2a\nL2b" {- t.Errorf("nested list text = %q, want %q", got, "L1\nL2a\nL2b")+ for i, want := range []string{"L1", "L2a", "L2b"} {+ if sections[i]["text"] != want {+ t.Fatalf("section[%d]=%#v", i, sections[i])+ }}}diff --git a/internal/parser/parser/docx_parser_test.go b/internal/parser/parser/docx_parser_test.goindex d724bf05aa..a4398a6b1b 100644--- a/internal/parser/parser/docx_parser_test.go+++ b/internal/parser/parser/docx_parser_test.go@@ -20,7 +20,6 @@ package parserimport ("encoding/base64"- "encoding/json""strings""testing"@@ -225,53 +224,3 @@ func TestBuildDOCXJSONSections_EmptySkipped(t *testing.T) {t.Errorf("text = %q, want %q", got, "Only this matters")}}--func TestDocxIRTableToHTML_Empty(t *testing.T) {- el := docxIRElement{Type: "table", Rows: []docxIRRow{}}- html := docxIRTableToHTML(el, "")- if html != "<table></table>" {- t.Errorf("empty table html = %q, want %q", html, "<table></table>")- }-}--func TestDocxIRTableToHTML_Single(t *testing.T) {- el := docxIRElement{- Type: "table",- Rows: []docxIRRow{{- Cells: []docxIRCell{{- Content: []docxIRElement{{- Type: "paragraph",- Content: json.RawMessage(`[{"type":"text","text":"hello"}]`),- }},- }},- }},- }- html := docxIRTableToHTML(el, "")- want := "<table><tr><td>hello</td></tr></table>"- if html != want {- t.Errorf("single cell html = %q, want %q", html, want)- }-}--func TestDocxIRTableToHTML_MultiRowCol(t *testing.T) {- cell := func(text string) docxIRCell {- return docxIRCell{- Content: []docxIRElement{{- Type: "paragraph",- Content: rawJSON([]docxIRRun{{Type: "text", Text: text}}),- }},- }- }- el := docxIRElement{- Type: "table",- Rows: []docxIRRow{- {Cells: []docxIRCell{cell("A1"), cell("B1")}},- {Cells: []docxIRCell{cell("A2"), cell("B2")}},- },- }- html := docxIRTableToHTML(el, "")- want := "<table><tr><td>A1</td><td>B1</td></tr><tr><td>A2</td><td>B2</td></tr></table>"- if html != want {- t.Errorf("multi cell html = %q, want %q", html, want)- }-}diff --git a/internal/parser/parser/epub_parser.go b/internal/parser/parser/epub_parser.goindex 7e0381f54d..df2ac36922 100644--- a/internal/parser/parser/epub_parser.go+++ b/internal/parser/parser/epub_parser.go@@ -20,8 +20,7 @@// 1. Opens the EPUB as a ZIP archive.// 2. Reads META-INF/container.xml to locate the OPF manifest.// 3. Walks the OPF spine to collect content documents in order.-// 4. Strips HTML tags from each content document to produce plain-// text items, emitting one JSON item per spine entry.+// 4. Emits text, tables and inline images from each chapter in document order.//// The output format is "json", matching the epub default in// ParserParam.Defaults().@@ -34,12 +33,13 @@ import ("context""encoding/xml""fmt"- "html""io""net/url""path""regexp""strings"++ "golang.org/x/net/html")// EPUBParser extracts text from EPUB archives.@@ -54,8 +54,7 @@ func (p *EPUBParser) String() string {}// ParseWithResult implements ParseResultProducer. It extracts XHTML-// content from the EPUB spine and emits one JSON item per spine entry-// with {text, doc_type_kwd:"text"}.+// content from the EPUB spine using the shared HTML structure walker.func (p *EPUBParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {if len(data) == 0 {return ParseResult{@@ -83,7 +82,7 @@ func (p *EPUBParser) ParseWithResult(ctx context.Context, filename string, datareturn ParseResult{Err: fmt.Errorf("epub: opf: %w", err)}}- items, detectedEnc, err := extractEPUBTextItems(reader, opfPath, spineItems)+ items, detectedEnc, warnings, err := extractEPUBTextItems(reader, opfPath, spineItems)if err != nil {return ParseResult{Err: fmt.Errorf("epub: %w", err)}}@@ -96,7 +95,8 @@ func (p *EPUBParser) ParseWithResult(ctx context.Context, filename string, data"size": len(data),"encoding": detectedEnc,},- JSON: MapsToItems(items),+ JSON: MapsToItems(items),+ Warnings: warnings,}}@@ -189,25 +189,28 @@ func readOPFSpine(r *zip.Reader, opfPath string) ([]string, error) {// --- text extraction ----var (- epubScriptRe = regexp.MustCompile(`(?is)<script[^>]*>.*?</script>`)- epubStyleRe = regexp.MustCompile(`(?is)<style[^>]*>.*?</style>`)- epubTagRe = regexp.MustCompile(`<[^>]+>`)- epubEntityRe = regexp.MustCompile(`&(?:[a-zA-Z][a-zA-Z0-9]*|#[0-9]+|#[xX][0-9a-fA-F]+);`)- epubWSRe = regexp.MustCompile(`\s+`)- // Closing tags of block elements, or a void <hr>, that end a run of text.- epubBlockEndRe = regexp.MustCompile(`^(?:</(?:title|main|section|article|header|footer|aside|nav|figure|figcaption|caption|table|pre|address|dl|dt|dd|ul|ol)[\s>]|<hr[\s/>])`)-)+var epubSpaceEntity = regexp.MustCompile(`&(?:nbsp|#0*160|#[xX]0*[aA]0);`)-// extractEPUBTextItems reads every spine item and returns one text item per-// item that has text. An item that cannot be read is skipped; when items were+// epubSelfClosingRawText matches a self-closing non-void XHTML element whose+// content HTML parsers treat as raw text. HTML5 parsing turns <script src="x"/>+// into an unclosed start tag, so the rest of the chapter becomes script text+// and the walker drops it. Restoring the end tag keeps the chapter body.+var epubSelfClosingRawText = regexp.MustCompile(`(?is)<(script|style|title|textarea)(\s[^<>]*?)?\s*/>`)++func normalizeEPUBSelfClosing(source string) string {+ return epubSelfClosingRawText.ReplaceAllString(source, "<$1$2></$1>")+}++// extractEPUBTextItems reads chapters in spine order and preserves their blocks.+// An item that cannot be read is skipped; when items were// skipped and no other item could be read, it returns an error instead of an// empty result. An item that holds no text (an image-only chapter) still counts// as read, and an empty item counts as neither.-func extractEPUBTextItems(r *zip.Reader, opfDir string, spineHrefs []string) ([]map[string]any, string, error) {+func extractEPUBTextItems(r *zip.Reader, opfDir string, spineHrefs []string) ([]map[string]any, string, []string, error) {var items []map[string]anyvar failures []stringread := 0+ state := &htmlWalkState{}detectedEncoding := "utf-8"for _, href := range spineHrefs {raw, err := readEPUBContentFile(r, opfDir, href)@@ -223,19 +226,52 @@ func extractEPUBTextItems(r *zip.Reader, opfDir string, spineHrefs []string) ([]if detectedEncoding == "utf-8" && enc != "" && enc != "utf-8" {detectedEncoding = enc}- text := stripHTMLTags(string(decoded))- if strings.TrimSpace(text) == "" {+ // Retain chapter text normalization for named quotes and spaces.+ source := epubSpaceEntity.ReplaceAllString(string(decoded), " ")+ source = strings.NewReplacer("‘", "'", "’", "'", "“", "\"", "”", "\"").Replace(source)+ source = normalizeEPUBSelfClosing(source)+ doc, err := html.Parse(strings.NewReader(source))+ if err != nil {+ failures = append(failures, fmt.Sprintf("%s: %v", href, err))continue}- items = append(items, map[string]any{- "text": text,- "doc_type_kwd": "text",- })+ items = appendEPUBChapter(items, doc, state)}if len(failures) > 0 && read == 0 {- return nil, detectedEncoding, fmt.Errorf("no readable content: %d of %d content items could not be read (%s)", len(failures), len(spineHrefs), failures[0])+ return nil, detectedEncoding, state.warnings(), fmt.Errorf("no readable content: %d of %d content items could not be read (%s)", len(failures), len(spineHrefs), failures[0])}- return items, detectedEncoding, nil+ return items, detectedEncoding, state.warnings(), nil+}++// appendEPUBChapter keeps continuous chapter prose together, splitting only+// at tables and images. The shared leaf walker preserves their source order.+func appendEPUBChapter(items []map[string]any, doc *html.Node, state *htmlWalkState) []map[string]any {+ var chapter []map[string]any+ for root := doc.FirstChild; root != nil; root = root.NextSibling {+ if root.Type != html.ElementNode {+ continue+ }+ for part := root.FirstChild; part != nil; part = part.NextSibling {+ htmlLeafTextWithState(part, &chapter, "text", true, state)+ }+ }+ start := len(items)+ for _, item := range chapter {+ if item["doc_type_kwd"] != "text" {+ items = append(items, item)+ continue+ }+ text := strings.Join(strings.FieldsFunc(stringValue(item["text"]), isCollapsibleWS), " ")+ if text == "" {+ continue+ }+ if len(items) > start && items[len(items)-1]["doc_type_kwd"] == "text" {+ items[len(items)-1]["text"] = stringValue(items[len(items)-1]["text"]) + " " + text+ } else {+ items = append(items, map[string]any{"text": text, "doc_type_kwd": "text"})+ }+ }+ return items}// readEPUBContentFile resolves a spine href (relative to the OPF@@ -271,84 +307,3 @@ func readEPUBContentFile(r *zip.Reader, opfDir, href string) ([]byte, error) {defer f.Close()return io.ReadAll(f)}--// stripHTMLTags removes HTML markup and returns normalized plain text.-func stripHTMLTags(s string) string {- // Remove script/style blocks first (including their content).- s = epubScriptRe.ReplaceAllString(s, "")- s = epubStyleRe.ReplaceAllString(s, "")- // Replace <br>, <p>, </p>, </div>, </tr>, </li> etc. with newlines.- s = epubTagRe.ReplaceAllStringFunc(s, func(tag string) string {- lower := strings.ToLower(tag)- switch {- case strings.HasPrefix(lower, "<br"),- strings.HasPrefix(lower, "</p"),- strings.HasPrefix(lower, "</div"),- strings.HasPrefix(lower, "</h1"),- strings.HasPrefix(lower, "</h2"),- strings.HasPrefix(lower, "</h3"),- strings.HasPrefix(lower, "</h4"),- strings.HasPrefix(lower, "</h5"),- strings.HasPrefix(lower, "</h6"),- strings.HasPrefix(lower, "</tr"),- strings.HasPrefix(lower, "</li"),- strings.HasPrefix(lower, "</blockquote"),- // Other elements a reader shows as blocks. Without a break, the- // head's <title> ran into the first word of the body, and a term- // into its definition ("TermDefinition").- epubBlockEndRe.MatchString(lower):- return "\n"- case strings.HasPrefix(lower, "</td"),- strings.HasPrefix(lower, "</th"):- return " "- }- return ""- })- // Decode common XML entities.- s = epubEntityRe.ReplaceAllStringFunc(s, xmlEntityValue)- // Normalize whitespace.- s = epubWSRe.ReplaceAllString(s, " ")- return strings.TrimSpace(s)-}--// xmlEntityValue decodes a few common XML named entities. A full-// entity table isn't needed for the typical EPUB vocabulary.-func xmlEntityValue(entity string) string {- switch entity {- case "&":- return "&"- case "<":- return "<"- case ">":- return ">"- case """:- return "\""- case "'":- return "'"- case " ":- return " "- case "–":- return "–"- case "—":- return "—"- case "‘":- return "'"- case "’":- return "'"- case "“":- return "\""- case "”":- return "\""- case "…":- return "…"- default:- // Numeric references (’, —, and   for a- // non-breaking space, which XHTML has no named entity for) and the- // other HTML named entities. One html cannot resolve stays as written.- decoded := html.UnescapeString(entity)- if decoded == "\u00a0" {- return " "- }- return decoded- }-}diff --git a/internal/parser/parser/epub_parser_test.go b/internal/parser/parser/epub_parser_test.goindex 8fba6626ca..f1603bf8ef 100644--- a/internal/parser/parser/epub_parser_test.go+++ b/internal/parser/parser/epub_parser_test.go@@ -91,6 +91,29 @@ func parseTestEPUB(t *testing.T, chapters []epubTestChapter) ParseResult {return NewEPUBParser().ParseWithResult(context.Background(), "book.epub", buildTestEPUB(t, chapters))}+func TestEPUBParserReportsDocumentImageLimitAcrossChapters(t *testing.T) {+ image := `<img src="data:image/png;base64,aGVsbG8=">`+ res := parseTestEPUB(t, []epubTestChapter{+ {name: "one.xhtml", content: epubTestHTML(strings.Repeat(image, 128))},+ {name: "two.xhtml", content: epubTestHTML(strings.Repeat(image, 129) + "<p>After images.</p>")},+ })+ if res.Err != nil {+ t.Fatal(res.Err)+ }+ images := 0+ for _, item := range res.JSON {+ if item.ToMap()["doc_type_kwd"] == "image" {+ images+++ }+ }+ if images != 256 || !strings.Contains(epubTestText(res), "After images.") {+ t.Fatalf("images=%d text=%q", images, epubTestText(res))+ }+ if !strings.Contains(strings.Join(res.Warnings, "\n"), "256-item document limit") {+ t.Fatalf("missing document image limit warning: %v", res.Warnings)+ }+}+func epubTestText(res ParseResult) string {var parts []stringfor _, item := range ItemsToMaps(res.JSON) {@@ -168,7 +191,7 @@ func TestEPUBParser_NoReadableChapterIsAnError(t *testing.T) {func TestEPUBParser_ImageOnlyChapterCountsAsRead(t *testing.T) {// The image-only chapter was read and has no text; the book is not unreadable.- // No <title>: stripHTMLTags keeps its text, and the chapter would not be text-free.+ // The chapter has no body text.imageOnly := `<?xml version="1.0" encoding="utf-8"?><html xmlns="http://www.w3.org/1999/xhtml"><body><img src="plate.png" alt=""/></body></html>`res := parseTestEPUB(t, []epubTestChapter{{name: "ch1.xhtml", content: imageOnly},@@ -207,9 +230,7 @@ func TestEPUBParser_ResolvesManifestHrefsAsURLs(t *testing.T) {}}-// The head's <title> and block elements outside the old list ended without a-// break, so their text ran into the next word: "Testchapter text" and-// "TermDefinition".+// Chapter title text and block boundaries retain their text representation.func TestEPUBParser_SeparatesBlockElements(t *testing.T) {res := parseTestEPUB(t, []epubTestChapter{{name: "ch1.xhtml", content: epubTestHTML(@@ -224,6 +245,31 @@ func TestEPUBParser_SeparatesBlockElements(t *testing.T) {}}+// A self-closing raw-text element is valid XHTML but an unclosed start tag+// under HTML5 rules, which would swallow the rest of the chapter.+func TestEPUBParser_SelfClosingRawTextKeepsFollowingContent(t *testing.T) {+ res := parseTestEPUB(t, []epubTestChapter{+ {name: "ch1.xhtml", content: epubTestHTML(+ `<script src="chapter.js"/><style type="text/css"/><p>After self-closing.</p>` ++ `<table><tr><td>Owner</td><td>RET</td></tr></table>`)},+ })+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if text := epubTestText(res); !strings.Contains(text, "After self-closing.") {+ t.Fatalf("body after self-closing raw-text element lost: %q", text)+ }+ tables := 0+ for _, item := range res.JSON {+ if item.ToMap()["doc_type_kwd"] == "table" {+ tables+++ }+ }+ if tables != 1 {+ t.Fatalf("table after self-closing raw-text element lost: %#v", ItemsToMaps(res.JSON))+ }+}+// XHTML has no and EPUB tools often write numeric character// references; only a fixed list of named entities was decoded, so// "don’t" reached the index as written.@@ -239,3 +285,20 @@ func TestEPUBParser_DecodesCharacterReferences(t *testing.T) {t.Errorf("text = %q, want it to contain %q", text, want)}}++func TestEPUBParserPreservesChapterTextUnits(t *testing.T) {+ paragraph := strings.Repeat("chapter prose ", 600)+ res := parseTestEPUB(t, []epubTestChapter{+ {name: "one.xhtml", content: epubTestHTML("<p>" + paragraph + "</p><p>End one.</p>")},+ {name: "two.xhtml", content: epubTestHTML("<p>Second “chapter”.</p><p>\u3000End two.</p>")},+ })+ if res.Err != nil || len(res.JSON) != 2 {+ t.Fatalf("expected one text item per chapter: items=%d err=%v", len(res.JSON), res.Err)+ }+ if got, want := res.JSON[0].ToMap()["text"], "Test "+paragraph+"End one."; got != want {+ t.Fatal("first chapter text changed")+ }+ if got, want := res.JSON[1].ToMap()["text"], "Test Second \"chapter\". \u3000End two."; got != want {+ t.Fatalf("second chapter text=%q, want %q", got, want)+ }+}diff --git a/internal/parser/parser/html_media_test.go b/internal/parser/parser/html_media_test.goindex c0cc61f7c8..9950e39551 100644--- a/internal/parser/parser/html_media_test.go+++ b/internal/parser/parser/html_media_test.go@@ -80,3 +80,23 @@ func TestHTMLParser_RejectsOversizedInlineImagePayload(t *testing.T) {t.Fatal("oversized inline image should produce a parser warning")}}++func TestHTMLParser_DocumentImageLimit(t *testing.T) {+ input := "<p>" + strings.Repeat(`<img src="`+htmlMediaDataURI+`">`, maxEmbeddedMediaItems+44) + "</p>"+ result := NewHTMLParser().ParseWithResult(context.Background(), "many.html", []byte(input))+ if result.Err != nil {+ t.Fatalf("ParseWithResult: %v", result.Err)+ }+ images := 0+ for _, item := range result.JSON {+ if item.ToMap()["doc_type_kwd"] == "image" {+ images+++ }+ }+ if images != maxEmbeddedMediaItems {+ t.Fatalf("images = %d, want document limit %d", images, maxEmbeddedMediaItems)+ }+ if !strings.Contains(strings.Join(result.Warnings, "\n"), "256-item document limit") {+ t.Fatalf("missing document image limit warning: %v", result.Warnings)+ }+}diff --git a/internal/parser/parser/html_parser.go b/internal/parser/parser/html_parser.goindex ac68c4b095..3989302f7d 100644--- a/internal/parser/parser/html_parser.go+++ b/internal/parser/parser/html_parser.go@@ -110,13 +110,36 @@ type htmlWalkState struct {tableSequence intmediaOrder intskippedInlineImages int+ mediaBudget *embeddedMediaBudget+ resolveImage func(string) (string, bool)}func (s *htmlWalkState) warnings() []string {- if s.skippedInlineImages == 0 {- return nil+ warnings := s.mediaBudget.warnings()+ if s.skippedInlineImages > 0 {+ warnings = append(warnings, fmt.Sprintf("document parser skipped %d invalid or oversized inline image(s)", s.skippedInlineImages))}- return []string{fmt.Sprintf("HTML parser skipped %d invalid or oversized inline image(s)", s.skippedInlineImages)}+ return warnings+}++func (s *htmlWalkState) imageSource(src string) (string, bool) {+ if s.resolveImage != nil {+ return s.resolveImage(src)+ }+ if !isHTMLDataImageSource(src) {+ return "", false+ }+ if s.mediaBudget == nil {+ s.mediaBudget = newEmbeddedMediaBudget()+ }+ if !s.mediaBudget.reserveItem() {+ return "", false+ }+ normalized, size, ok := normalizeHTMLImageSource(src)+ if !ok || !s.mediaBudget.includeReservedSize(size) {+ return "", false+ }+ return normalized, true}// walkHTMLBlocksWithState emits normalized text and image items in document@@ -369,9 +392,11 @@ func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any, layout str}if n.Data == "img" {src := htmlAttribute(n, "src")- normalized, ok := normalizeHTMLImageSource(src)+ normalized, ok := state.imageSource(src)if !ok {- if isHTMLDataImageSource(src) {+ if state.resolveImage != nil {+ w.writeText(htmlAttribute(n, "alt"))+ } else if isHTMLDataImageSource(src) {state.skippedInlineImages++}return@@ -426,7 +451,7 @@ type htmlTableImage struct {}func emitHTMLTable(n *html.Node, out *[]map[string]any, state *htmlWalkState) {- images, skippedInlineImages := htmlTableImages(n)+ images, skippedInlineImages := htmlTableImages(n, state)state.skippedInlineImages += skippedInlineImagesmarkup := renderTableHTML(n)if strings.TrimSpace(markup) == "" {@@ -455,7 +480,7 @@ func emitHTMLTable(n *html.Node, out *[]map[string]any, state *htmlWalkState) {}}-func htmlTableImages(table *html.Node) ([]htmlTableImage, int) {+func htmlTableImages(table *html.Node, state *htmlWalkState) ([]htmlTableImage, int) {var images []htmlTableImageskippedInlineImages := 0rowCount := 0@@ -473,9 +498,12 @@ func htmlTableImages(table *html.Node) ([]htmlTableImage, int) {column = columnCounts[row]case "img":src := htmlAttribute(node, "src")- normalized, ok := normalizeHTMLImageSource(src)+ normalized, ok := state.imageSource(src)if !ok {- if isHTMLDataImageSource(src) {+ if state.resolveImage != nil {+ alt := htmlAttribute(node, "alt")+ node.Type, node.Data, node.Attr = html.TextNode, alt, nil+ } else if isHTMLDataImageSource(src) {skippedInlineImages++removeHTMLImageSource(node)}@@ -527,20 +555,20 @@ func appendHTMLImageItem(out *[]map[string]any, state *htmlWalkState, src, alt,*out = append(*out, item)}-func normalizeHTMLImageSource(src string) (string, bool) {+func normalizeHTMLImageSource(src string) (string, int, bool) {src = strings.TrimSpace(src)if !isHTMLDataImageSource(src) {- return "", false+ return "", 0, false}separator := strings.IndexByte(src, ',')if separator <= len("data:image/") || separator > 128 || !strings.EqualFold(src[separator-len(";base64"):separator], ";base64") {- return "", false+ return "", 0, false}- payload, _, ok := normalizeBase64ImagePayload(src[separator+1:], maxEmbeddedImageBytes)+ payload, size, ok := normalizeBase64ImagePayload(src[separator+1:], maxEmbeddedImageBytes)if !ok {- return "", false+ return "", 0, false}- return strings.ToLower(src[:separator+1]) + payload, true+ return strings.ToLower(src[:separator+1]) + payload, int(size), true}func isHTMLDataImageSource(src string) bool {diff --git a/internal/parser/parser/markdown_parser.go b/internal/parser/parser/markdown_parser.goindex 9f835465ad..71d0cdadb9 100644--- a/internal/parser/parser/markdown_parser.go+++ b/internal/parser/parser/markdown_parser.go@@ -31,8 +31,9 @@ import (markdownlib "github.com/gomarkdown/markdown""github.com/gomarkdown/markdown/ast"+ mdhtml "github.com/gomarkdown/markdown/html"mdparser "github.com/gomarkdown/markdown/parser"- "ragflow/internal/parser/tableutil"+ "golang.org/x/net/html")// dataURIPrefix is the MIME prefix for data URI images.@@ -92,30 +93,80 @@ func (p *MarkdownParser) ConfigureFromSetup(setup map[string]any) {// the base64-encoded image payload. The legacy debug-print path has// been removed; callers consume ParseResult directly.func (p *MarkdownParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {- decoded, _ := DecodeToUTF8(data, "text/markdown")- rawText := string(decoded)- // Render any GFM/HTML table inline as an HTML block before parsing. This- // keeps the document as one item per top-level block (the table becomes a- // normal text item) instead of collapsing the whole document into a single- // item. The result mirrors Python's `_markdown` (separate_tables=False),- // which also inlines tables into the surrounding text. When no table is- // present renderMarkdownTablesInlineText returns the input unchanged.- rendered := renderMarkdownTablesInlineText(rawText)+ state := &htmlWalkState{}+ result := p.parseWithState(ctx, filename, data, state)+ result.Warnings = append(state.warnings(), result.Warnings...)+ return result+}- doc := markdownNew().Parse([]byte(rendered))+// parseWithState keeps media identity and budget across fragments of one document.+// Its warnings describe this fragment; the caller adds state.warnings once.+func (p *MarkdownParser) parseWithState(ctx context.Context, filename string, data []byte, state *htmlWalkState) ParseResult {+ decoded, _ := DecodeToUTF8(data, "text/markdown")+ // The final blank line also terminates a raw HTML block at EOF.+ rawText := string(decoded) + "\n\n"+ blockParser := markdownNew()+ blockParser.Block(mdparser.NormalizeNewlines([]byte(rawText)))+ // Expand tight-list tables before the inline pass. gomarkdown can also+ // end a list before its indented table, leaving uneven paragraph lines.+ ast.WalkFunc(blockParser.Doc, func(node ast.Node, entering bool) ast.WalkStatus {+ if !entering || node.AsContainer() == nil {+ return ast.GoToNext+ }+ var children []ast.Node+ changed := false+ for index, child := range node.GetChildren() {+ if paragraph, ok := child.(*ast.Paragraph); ok && bytes.Contains(paragraph.Content, []byte("|")) {+ _, inList := node.(*ast.ListItem)+ afterList := false+ if index > 0 {+ _, afterList = node.GetChildren()[index-1].(*ast.List)+ }+ content := paragraph.Content+ if afterList {+ content = dedentMarkdownTable(content)+ }+ if !inList && bytes.Equal(content, paragraph.Content) {+ children = append(children, child)+ continue+ }+ blocks := markdownNew()+ blocks.Block(content)+ if containsMarkdownTable(blocks.Doc) {+ children = append(children, blocks.Doc.GetChildren()...)+ changed = true+ continue+ }+ }+ children = append(children, child)+ }+ if !changed {+ return ast.GoToNext+ }+ node.SetChildren(nil)+ for _, child := range children {+ child.SetParent(nil)+ ast.AppendChild(node, child)+ }+ return ast.GoToNext+ })+ doc := blockParser.Parse(nil)var items []map[string]any- mediaBudget := newEmbeddedMediaBudget()+ if state.mediaBudget == nil {+ state.mediaBudget = newEmbeddedMediaBudget()+ }+ mediaBudget := state.mediaBudgetimageCtx, cancel := context.WithTimeout(ctx, 60*time.Second)defer cancel()- unresolvedImages := walkMarkdownBlocksWithImages(imageCtx, doc, &items, p.FlattenMediaToText, p.FetchRemoteImages, mediaBudget)+ unresolvedImages := walkMarkdownBlocksWithImages(imageCtx, doc, &items, p.FlattenMediaToText, p.FetchRemoteImages, state)if err := ctx.Err(); err != nil {return ParseResult{Err: err}}if items == nil {items = []map[string]any{{"text": "", "doc_type_kwd": "text"}}}- warnings := mediaBudget.warnings()+ var warnings []stringif mediaBudget.items > 0 && imageCtx.Err() == context.DeadlineExceeded && ctx.Err() == nil {warnings = append(warnings, "Markdown image processing reached the document time budget")}@@ -149,63 +200,113 @@ func markdownNew() *mdparser.Parser {// ast.MathBlock defensively so that turning the extension back on// cannot lose text again.// OrderedListStart is not part of CommonExtensions, and without it the parser leaves- // ast.List.Start at zero for every ordered list: listEntries then falls back to 1 and+ // ast.List.Start at zero for every ordered list: markdownListMarker then falls back to 1 and// renumbers the entries 1..N, so a list that the source spells "996. graphql: ..." is// indexed as "1. graphql: ...". The numbers are then gone from the index (a query for// "996" cannot match anything, and the gap counts against coverage), while Python's// _extract_list_block keeps the raw lines - markers included - verbatim. Enabling it// makes Start carry the number the source wrote (gomarkdown normalises a literal 1 to 0,- // which the fallback in listEntries turns back into 1).+ // which markdownListMarker turns back into 1).extensions := mdparser.CommonExtensions&^mdparser.MathJax | mdparser.AutoHeadingIDs | mdparser.NoEmptyLineBeforeBlock | mdparser.OrderedListStart- return mdparser.NewWithExtensions(extensions)+ p := mdparser.NewWithExtensions(extensions)+ p.Opts.ParserHook = parseWideMarkdownTable+ return p}-func renderMarkdownTablesInline(text string) (string, bool) {- lines := strings.SplitAfter(strings.ReplaceAll(strings.ReplaceAll(text, "\r\n", "\n"), "\r", "\n"), "\n")- var buf strings.Builder- changed := false- inFence := false- var fenceChar byte- var fenceLen int-- for i := 0; i < len(lines); {- line := strings.TrimRight(lines[i], "\n")- if ch, n, ok := markdownFenceMarker(line); ok {- if inFence && ch == fenceChar && n >= fenceLen {- inFence = false- } else if !inFence {- inFence = true- fenceChar = ch- fenceLen = n+// Pad over-wide tables only when the Markdown parser reaches a real block.+// Fences, raw HTML, lists and indentation remain the parser's responsibility.+func parseWideMarkdownTable(data []byte) (ast.Node, []byte, int) {+ end := bytes.IndexByte(data, '\n')+ if end < 0 || bytes.HasPrefix(data, []byte("\t")) || bytes.HasPrefix(data, []byte(" ")) {+ return nil, nil, 0+ }+ header := string(data[:end+1])+ if !isMarkdownTableRow(header) {+ return nil, nil, 0+ }+ next := bytes.IndexByte(data[end+1:], '\n')+ if next < 0 {+ return nil, nil, 0+ }+ consumed := end + next + 2+ separator := string(data[end+1 : consumed])+ if !isMarkdownTableSeparator(separator) {+ return nil, nil, 0+ }+ lines := []string{header, separator}+ for consumed < len(data) {+ next = bytes.IndexByte(data[consumed:], '\n')+ if next < 0 {+ next = len(data) - consumed - 1+ }+ line := string(data[consumed : consumed+next+1])+ if !isMarkdownTableRow(line) {+ break+ }+ lines = append(lines, line)+ consumed += next + 1+ }+ balanced := strings.Join(balanceTableColumns(lines), "")+ if balanced == string(data[:consumed]) {+ return nil, nil, 0+ }+ if strings.HasPrefix(strings.TrimSpace(header), "<") {+ // The hook runs before HTML recognition. Let the library distinguish+ // raw HTML from a table whose first cell contains inline markup.+ probe := markdownNew()+ first := true+ probe.Opts.ParserHook = func(rest []byte) (ast.Node, []byte, int) {+ if first {+ first = false+ return nil, nil, 0}- buf.WriteString(lines[i])- i++- continue+ // The first block determines HTML priority; do not parse the suffix.+ return nil, nil, len(rest)}- if !inFence && i+1 < len(lines) && isMarkdownTableRow(line) && isMarkdownTableSeparator(strings.TrimRight(lines[i+1], "\n")) {- start := i- i += 2- for i < len(lines) && isMarkdownTableRow(strings.TrimRight(lines[i], "\n")) {- i+++ probe.Block(data)+ if children := probe.Doc.GetChildren(); len(children) > 0 {+ if _, table := children[0].(*ast.Table); !table {+ return nil, nil, 0}- for i < len(lines) && strings.TrimSpace(lines[i]) == "" {- i++- }- tableHTML := markdownlib.ToHTML([]byte(strings.Join(balanceTableColumns(lines[start:i]), "")), markdownNew(), nil)- // Wrap the inlined <table> HTML in blank lines so gomarkdown- // keeps it as a single HTML block (one item) instead of- // re-parsing it into scattered cell text. See- // PARSER_ALIGNMENT_HANDOFF.md §3.1 (Markdown session A, 方案 Y).- ensureTrailingBlankLine(&buf)- buf.WriteString(strings.TrimRight(string(tableHTML), "\r\n"))- buf.WriteString("\n\n")- changed = true- continue}- buf.WriteString(lines[i])- i++}- return buf.String(), changed+ tableParser := markdownNew()+ tableParser.Opts.ParserHook = nil+ tableParser.Block([]byte(balanced))+ children := tableParser.Doc.GetChildren()+ if len(children) == 0 {+ return nil, nil, 0+ }+ table, ok := children[0].(*ast.Table)+ if !ok {+ return nil, nil, 0+ }+ // Keep cell Content for the document parser's inline pass and references.+ // Detach before AddBlock so ast.AppendChild does not clear the subtree.+ table.SetParent(nil)+ return table, nil, consumed+}++// List paragraphs can have a dedented first line but indented continuation+// lines. Normalize only a table paragraph with a shared continuation prefix.+func dedentMarkdownTable(data []byte) []byte {+ lines := strings.Split(string(data), "\n")+ if len(lines) < 2 || !isMarkdownTableRow(lines[0]) || !isMarkdownTableSeparator(lines[1]) {+ return data+ }+ prefix := lines[1][:len(lines[1])-len(strings.TrimLeft(lines[1], " \t"))]+ if prefix == "" {+ return data+ }+ for _, line := range lines[1:] {+ if strings.TrimSpace(line) != "" && !strings.HasPrefix(line, prefix) {+ return data+ }+ }+ for i := 1; i < len(lines); i++ {+ lines[i] = strings.TrimPrefix(lines[i], prefix)+ }+ return mdparser.NormalizeNewlines([]byte(strings.Join(lines, "\n")))}// balanceTableColumns pads the header row and its separator up to the widest row's cell@@ -226,31 +327,7 @@ func balanceTableColumns(tableLines []string) []string {if len(tableLines) < 2 {return tableLines}- cells := func(line string) int {- s := strings.TrimRight(line, "\n")- pipes, escaped := 0, false- for i := 0; i < len(s); i++ {- switch {- case escaped:- escaped = false- case s[i] == '\\':- escaped = true- case s[i] == '|':- pipes++- }- }- trimmed := strings.TrimSpace(s)- if strings.HasPrefix(trimmed, "|") {- pipes--- }- if strings.HasSuffix(trimmed, "|") {- pipes--- }- if pipes < 0 {- return 0- }- return pipes + 1- }+ cells := func(line string) int { return len(markdownTableCells(line)) }head := cells(tableLines[0])widest := head@@ -286,59 +363,8 @@ func balanceTableColumns(tableLines []string) []string {return out}-// renderMarkdownTablesInlineText renders every GFM/HTML table inline as an-// HTML block and returns the rewritten text. When no table is present the-// input is returned unchanged. Unlike renderMarkdownTablesInline it always-// returns the full text (ignoring the changed flag) so callers can parse the-// result uniformly and emit one item per top-level block.-func renderMarkdownTablesInlineText(text string) string {- out, _ := renderMarkdownTablesInline(text)- return out-}--// ensureTrailingBlankLine makes sure b ends with a blank line (two-// newlines) so the next block is separated from what precedes it. gomarkdown-// only treats a <table> as a standalone HTML block (rather than re-parsing it-// into scattered cell nodes) when it is surrounded by blank lines.-func ensureTrailingBlankLine(b *strings.Builder) {- s := b.String()- switch {- case strings.HasSuffix(s, "\n\n"):- // already separated.- case strings.HasSuffix(s, "\n"):- b.WriteByte('\n')- default:- b.WriteString("\n\n")- }-}--func markdownFenceMarker(line string) (byte, int, bool) {- trimmed := strings.TrimLeft(line, " \t")- if len(line)-len(trimmed) > 3 || len(trimmed) < 3 {- return 0, 0, false- }- ch := trimmed[0]- if ch != '`' && ch != '~' {- return 0, 0, false- }- n := 0- for n < len(trimmed) && trimmed[n] == ch {- n++- }- return ch, n, n >= 3-}-func isMarkdownTableRow(line string) bool {- cells := markdownTableCells(line)- if len(cells) < 2 {- return false- }- for _, cell := range cells {- if strings.TrimSpace(cell) != "" {- return true- }- }- return false+ return len(markdownTableCells(line)) >= 2}func isMarkdownTableSeparator(line string) bool {@@ -359,13 +385,52 @@ func isMarkdownTableSeparator(line string) bool {return true}+// markdownTableCells follows the Markdown library's table-row rules: only+// a code span at the start of a cell can hide its internal pipes.func markdownTableCells(line string) []string {line = strings.TrimSpace(line)- if !strings.Contains(line, "|") {+ var cells []string+ start := 0+ atCellStart := true+ for i := 0; i < len(line); i++ {+ switch line[i] {+ case ' ':+ continue+ case '\\':+ i+++ atCellStart = false+ case '`':+ if !atCellStart {+ continue+ }+ atCellStart = false+ end := i + 1+ for end < len(line) && line[end] == '`' {+ end+++ }+ marker := line[i:end]+ if close := strings.Index(line[end:], marker); close >= 0 {+ i = end + close + len(marker) - 1+ } else {+ i = end - 1+ }+ case '|':+ if i > 0 || start > 0 {+ cells = append(cells, line[start:i])+ }+ start = i + 1+ atCellStart = true+ default:+ atCellStart = false+ }+ }+ if start == 0 {return nil}- line = strings.TrimPrefix(strings.TrimSuffix(line, "|"), "|")- return strings.Split(line, "|")+ if start < len(line) {+ cells = append(cells, line[start:])+ }+ return cells}// walkMarkdownBlocksWithImages emits one normalized item per@@ -374,78 +439,115 @@ func markdownTableCells(line string) []string {// reference (), the image data is resolved via// findBlockImage (per-block AST walk) and the item carries// `doc_type_kwd: "image"` together with the base64-encoded image-// payload. When flatten is true, all items are forced to-// doc_type_kwd="text" (mirrors Python parser.py:1034-// flatten_media_to_text).+// payload. When flatten is true, image items use doc_type_kwd="text".//-// Tables: a GFM/HTML table is rendered inline as a single <table> HTML-// block by renderMarkdownTablesInlineText and kept as one HTML block.-// It is emitted as ONE structured item (doc_type_kwd:"table")-// in its original document position — there is no-// duplicate doc_type_kwd:"text" copy. The downstream chunker consumes-// doc_type_kwd:"table" to keep the table whole and attach table context-// to neighbouring chunks (chunker/token.go). Non-table HTML blocks-// (<div>, <style>, …) are emitted as ordinary text with no layout.-func walkMarkdownBlocksWithImages(ctx context.Context, doc ast.Node, out *[]map[string]any, flatten, fetchRemoteImages bool, budget *embeddedMediaBudget) int {+// Tables are emitted through the HTML walker as structured items in source+// order, with embedded images linked to their parent table.+func walkMarkdownBlocksWithImages(ctx context.Context, doc ast.Node, out *[]map[string]any, flatten, fetchRemoteImages bool, htmlState *htmlWalkState) int {+ if htmlState.mediaBudget == nil {+ htmlState.mediaBudget = newEmbeddedMediaBudget()+ }+ budget := htmlState.mediaBudget+ start := len(*out)+ defer func() {+ if flatten {+ for _, item := range (*out)[start:] {+ if item["doc_type_kwd"] == "image" {+ item["doc_type_kwd"] = "text"+ }+ }+ }+ }()unresolvedImages := 0+ previousResolver := htmlState.resolveImage+ htmlState.resolveImage = func(src string) (string, bool) {+ remote := strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://")+ if remote && !fetchRemoteImages {+ return "", false+ }+ encoded, raw, omitted, resolved := resolveImageURL(ctx, src, budget)+ if !resolved {+ if !omitted {+ unresolvedImages+++ }+ return "", false+ }+ if raw != nil {+ mime := http.DetectContentType(raw)+ if !strings.HasPrefix(mime, "image/") {+ mime = "image/octet-stream"+ }+ return "data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(raw), true+ }+ return strings.ToLower(src[:strings.IndexByte(src, ',')+1]) + encoded, true+ }+ defer func() { htmlState.resolveImage = previousResolver }()for _, child := range doc.GetChildren() {var layout stringvar docTypeKwd stringvar txt stringswitch n := child.(type) {+ case *ast.Table:+ markup := markdownlib.Render(n, mdhtml.NewRenderer(mdhtml.RendererOptions{}))+ tableDoc, _ := html.Parse(bytes.NewReader(markup))+ walkHTMLBlocksWithState(tableDoc, out, htmlState)+ continuecase *ast.Heading:txt = headingText(n)layout = "heading"docTypeKwd = "text"case *ast.Paragraph:+ if containsMarkdownTable(n) {+ markup := markdownlib.Render(n, mdhtml.NewRenderer(mdhtml.RendererOptions{}))+ doc, _ := html.Parse(bytes.NewReader(markup))+ walkHTMLBlocksWithState(doc, out, htmlState)+ continue+ }txt = leafText(n)docTypeKwd = "text"case *ast.List:- // Python's _markdown keeps one item per list entry, marker included- // ("- Eat a light diet before the exam?"), so a list never collapses into one blob.- // Emit the same shape: the reconstructed marker plus the entry's own text, one item- // each. The boundaries matter beyond alignment - the marker lives in the list node- // rather than in any leaf literal, and concatenating entries glued their words- // ("ChinaChina"), which destroys the word boundaries the tokenizer and the retrieval- // index depend on.- for _, entry := range listEntries(n) {- *out = append(*out, map[string]any{- "text": entry,- "doc_type_kwd": "text",- "layout": "list",- })+ // Keep ordinary entries whole, and visit entries containing tables+ // as blocks so their cells are not flattened into the list text.+ for i, entry := range n.GetChildren() {+ if containsMarkdownTable(entry) {+ start := len(*out)+ unresolvedImages += walkMarkdownBlocksWithImages(ctx, entry, out, flatten, fetchRemoteImages, htmlState)+ for _, item := range (*out)[start:] {+ if item["doc_type_kwd"] == "text" && item["layout"] == nil {+ item["layout"] = "list"+ }+ }+ if len(*out) > start && (*out)[start]["doc_type_kwd"] == "text" {+ (*out)[start]["text"] = markdownListMarker(n, i, entry) + stringValue((*out)[start]["text"])+ }+ continue+ }+ text := strings.TrimSpace(leafText(entry))+ if text != "" {+ *out = append(*out, map[string]any{"text": markdownListMarker(n, i, entry) + text, "doc_type_kwd": "text", "layout": "list"})+ }}+continue+ case *ast.BlockQuote:+ if containsMarkdownTable(n) {+ unresolvedImages += walkMarkdownBlocksWithImages(ctx, n, out, flatten, fetchRemoteImages, htmlState)+ continue+ }+ txt = leafText(n)+ docTypeKwd = "text"case *ast.CodeBlock:txt = leafText(n)layout = "code"docTypeKwd = "text"case *ast.HTMLBlock:- // An HTML block is either an inlined table (kept as a single- // HTML block thanks to the blank lines renderMarkdownTablesInline- // wraps around it) or a plain HTML block such as <div>/<style>.- // Only a table is emitted as a structured table item; everything- // else is treated as ordinary text (no layout). We emit exactly- // ONE item (doc_type_kwd:"table") in document- // order — no duplicate doc_type_kwd:"text" copy — so the table is- // embedded once and its markup does not pollute prose chunks.txt = leafText(n)if isTableHTML(txt) {- // Emit the structured TableData contract so the chunker reads- // rows directly instead of re-parsing markup. ParseTableHTML- // mirrors the legacy chunker extraction. html.Parse is- // infallible, so the parse always yields a non-nil TableData- // and no legacy HTML `text` is kept.- td, _ := tableutil.ParseTableHTML(txt)- table := map[string]any{- "doc_type_kwd": "table",- "table": td,- }- *out = append(*out, table)+ tableDoc, _ := html.Parse(strings.NewReader(txt))+ walkHTMLBlocksWithState(tableDoc, out, htmlState)continue}- // Non-table HTML block: ordinary text, no layout.docTypeKwd = "text"default:txt = leafText(n)@@ -473,29 +575,17 @@ func walkMarkdownBlocksWithImages(ctx context.Context, doc ast.Node, out *[]map[if imgURL, ok := findBlockImage(child); ok {isRemote := strings.HasPrefix(imgURL, "http://") || strings.HasPrefix(imgURL, "https://")if fetchRemoteImages || !isRemote {- switch {- case isRemote && ctx.Err() != nil:- item["media_omitted"] = true- case !budget.reserveItem():+ encoded, raw, omitted, resolved := resolveImageURL(ctx, imgURL, budget)+ if omitted {item["media_omitted"] = true- default:- encoded, raw, size, resolved := resolveImageURL(ctx, imgURL)- if !resolved {- unresolvedImages++- break- }- if !budget.includeReservedSize(size) {- item["media_omitted"] = true- break- }+ } else if !resolved {+ unresolvedImages+++ } else {if raw != nil {- item["image"] = base64.StdEncoding.EncodeToString(raw)- } else {- item["image"] = encoded- }- if !flatten {- item["doc_type_kwd"] = "image"+ encoded = base64.StdEncoding.EncodeToString(raw)}+ item["image"] = encoded+ item["doc_type_kwd"] = "image"}}}@@ -505,11 +595,13 @@ func walkMarkdownBlocksWithImages(ctx context.Context, doc ast.Node, out *[]map[return unresolvedImages}-// isTableHTML reports whether block text is an outer <table> element (the-// inlined GFM/HTML table). Only such blocks are emitted as structured table-// items; other raw HTML (e.g. <div>, <style>) is plain text.+// isTableHTML reports whether block text starts with a table element.func isTableHTML(s string) bool {- return strings.HasPrefix(strings.TrimSpace(strings.ToLower(s)), "<table")+ s = strings.TrimSpace(strings.ToLower(s))+ if !strings.HasPrefix(s, "<table") || len(s) <= len("<table") {+ return false+ }+ return strings.ContainsRune("> \t\r\n", rune(s[len("<table")]))}// findBlockImage returns the destination URL of the first image node found@@ -537,40 +629,39 @@ func findBlockImage(n ast.Node) (string, bool) {return url, found}-// resolveImageURL resolves a Markdown image URL to an encoded payload or raw-// bytes plus the image size in bytes.-// Supports:-// - data:image/... URIs → validated, normalized encoded payload-// - http:// / https:// URLs → fetched (with basic SSRF filtering)-//-// Local / relative paths are not fetched (security). Returns-// exactly one of encoded and raw populated on success.-func resolveImageURL(ctx context.Context, imageURL string) (encoded string, raw []byte, size int, ok bool) {+// resolveImageURL resolves one bounded Markdown image attempt. It returns+// either normalized base64 or downloaded bytes, and distinguishes a budget+// or timeout omission from an unresolved source. Local paths are never fetched.+func resolveImageURL(ctx context.Context, imageURL string, budget *embeddedMediaBudget) (encoded string, raw []byte, omitted, resolved bool) {+ if ctx.Err() != nil || !budget.reserveItem() {+ return "", nil, true, false+ }+ var size intif strings.HasPrefix(strings.ToLower(imageURL), dataURIPrefix) {- // data:image/png;base64,xxxx- idx := strings.Index(imageURL, "base64,")- if idx < 0 {- idx = strings.Index(strings.ToLower(imageURL), "base64,")- }+ idx := strings.Index(strings.ToLower(imageURL), "base64,")if idx < 0 {- return "", nil, 0, false+ return "", nil, false, false}var decodedSize int64- encoded, decodedSize, ok = normalizeBase64ImagePayload(imageURL[idx+len("base64,"):], MaxImagePayloadBytes)- if !ok {- return "", nil, 0, false- }- return encoded, nil, int(decodedSize), true- }- if strings.HasPrefix(imageURL, "http://") || strings.HasPrefix(imageURL, "https://") {- raw, err := fetchImage(ctx, imageURL)+ encoded, decodedSize, resolved = normalizeBase64ImagePayload(imageURL[idx+len("base64,"):], MaxImagePayloadBytes)+ if !resolved {+ return "", nil, false, false+ }+ size = int(decodedSize)+ } else if strings.HasPrefix(imageURL, "http://") || strings.HasPrefix(imageURL, "https://") {+ var err error+ raw, err = fetchImage(ctx, imageURL)if err != nil {- return "", nil, 0, false+ return "", nil, false, false}- return "", raw, len(raw), true+ size = len(raw)+ } else {+ return "", nil, false, false+ }+ if !budget.includeReservedSize(size) {+ return "", nil, true, false}- // Local / relative paths — not fetched for security.- return "", nil, 0, false+ return encoded, raw, false, true}// fetchImage fetches a bounded HTTP(S) image payload. Local/private addresses@@ -728,44 +819,49 @@ func leafText(n ast.Node) string {return strings.TrimSpace(buf.String())}-// listEntries renders a list's entries the way the source spells them: the marker first, then the-// entry text, one string per entry (Python's _markdown shape, see the en golden). The marker has to-// be reconstructed because the AST keeps it on the list node - BulletChar for bullet lists,-// Start/Delimiter for ordered ones - rather than in any leaf literal.-func listEntries(list *ast.List) []string {- entries := make([]string, 0, len(list.GetChildren()))- ordered := list.ListFlags&ast.ListTypeOrdered != 0- for i, item := range list.GetChildren() {- text := strings.TrimSpace(leafText(item))- if text == "" {- continue+// markdownListMarker reconstructs the marker stored on the list and item nodes.+func markdownListMarker(list *ast.List, index int, item ast.Node) string {+ if list.ListFlags&ast.ListTypeOrdered != 0 {+ start := list.Start+ if start == 0 {+ start = 1}- var marker string- switch {- case ordered:- start := list.Start- if start == 0 {- start = 1- }- delim := list.Delimiter- if delim == 0 {- delim = '.'- }- marker = fmt.Sprintf("%d%c ", start+i, delim)- default:- // The markdown parser records the bullet on the ITEM (parser/block.go sets- // ListItem.BulletChar), not on the list, so read it there first.- bullet := list.BulletChar- if li, ok := item.(*ast.ListItem); ok && li.BulletChar != 0 {- bullet = li.BulletChar- }- if bullet != 0 {- marker = string(bullet) + " "- }+ delimiter := list.Delimiter+ if delimiter == 0 {+ delimiter = '.'}- entries = append(entries, marker+text)+ return fmt.Sprintf("%d%c ", start+index, delimiter)}- return entries+ bullet := list.BulletChar+ if entry, ok := item.(*ast.ListItem); ok && entry.BulletChar != 0 {+ bullet = entry.BulletChar+ }+ if bullet != 0 {+ return string(bullet) + " "+ }+ return ""+}++func containsMarkdownTable(node ast.Node) bool {+ found := false+ ast.WalkFunc(node, func(node ast.Node, entering bool) ast.WalkStatus {+ if !entering {+ return ast.GoToNext+ }+ switch n := node.(type) {+ case *ast.Table:+ found = true+ case *ast.HTMLSpan:+ found = isTableHTML(string(n.Literal))+ case *ast.HTMLBlock:+ found = isTableHTML(string(n.Literal)) || isTableHTML(string(n.Content))+ }+ if found {+ return ast.Terminate+ }+ return ast.GoToNext+ })+ return found}// writeListItems emits a list's items separated by a newline. Without the separator the item textsdiff --git a/internal/parser/parser/markdown_parser_test.go b/internal/parser/parser/markdown_parser_test.goindex 2fd89dbea5..9cdc4249e1 100644--- a/internal/parser/parser/markdown_parser_test.go+++ b/internal/parser/parser/markdown_parser_test.go@@ -370,8 +370,7 @@ func TestMarkdownParser_ParseWithResult_InlineRawHTMLSurvives(t *testing.T) {}var sb strings.Builderfor _, item := range ItemsToMaps(res.JSON) {- text, _ := item["text"].(string)- sb.WriteString(text)+ sb.WriteString(renderedText(item))sb.WriteString("\n")}got := sb.String()@@ -383,8 +382,7 @@ func TestMarkdownParser_ParseWithResult_InlineRawHTMLSurvives(t *testing.T) {t.Errorf("parsed text lost %q\n--- got ---\n%s", want, got)}}- // The inline HTML payload must not vanish either: the walker emits the raw- // markup, exactly as it stands in the source.+ // The table content must survive in the structured item.if !strings.Contains(got, "binoculars") {t.Errorf("inline raw HTML payload was dropped: %q", got)}@@ -456,21 +454,22 @@ func TestMarkdownParser_FlattenMediaToText(t *testing.T) {func TestResolveImageURL_DataURI(t *testing.T) {b64 := base64.StdEncoding.EncodeToString([]byte("fakeimage"))- result, raw, size, found := resolveImageURL(context.Background(), "data:image/png;base64,"+b64)+ budget := newEmbeddedMediaBudget()+ result, raw, omitted, found := resolveImageURL(context.Background(), "data:image/png;base64,"+b64, budget)if !found {t.Fatal("expected image found for data URI")}if result != b64 || raw != nil {t.Fatalf("got %q, want %q", result, b64)}- if size != len("fakeimage") {- t.Errorf("size = %d, want %d", size, len("fakeimage"))+ if omitted || budget.totalBytes != len("fakeimage") {+ t.Errorf("charged bytes = %d, want %d", budget.totalBytes, len("fakeimage"))}}func TestResolveImageURL_LocalPathNotFetched(t *testing.T) {// Local / relative paths are not fetched (security); resolution fails.- if _, _, _, found := resolveImageURL(context.Background(), "./local/image.png"); found {+ if _, _, _, found := resolveImageURL(context.Background(), "./local/image.png", newEmbeddedMediaBudget()); found {t.Fatal("expected no image resolved for a local path")}}@@ -489,15 +488,16 @@ func TestResolveImageURL_HTTPImage(t *testing.T) {}))defer ts.Close()- encoded, result, size, found := resolveImageURL(context.Background(), ts.URL+"/image.png")+ budget := newEmbeddedMediaBudget()+ encoded, result, omitted, found := resolveImageURL(context.Background(), ts.URL+"/image.png", budget)if !found {t.Fatal("expected image found for HTTP URL")}if encoded != "" || string(result) != "fake-png-bytes" {t.Fatalf("got encoded=%q raw=%q, want raw image bytes", encoded, result)}- if size != len("fake-png-bytes") {- t.Errorf("size = %d, want %d", size, len("fake-png-bytes"))+ if omitted || budget.totalBytes != len("fake-png-bytes") {+ t.Errorf("charged bytes = %d, want %d", budget.totalBytes, len("fake-png-bytes"))}}@@ -520,7 +520,7 @@ func TestMarkdownImageBudgetSkipsFetchAfterCountLimit(t *testing.T) {doc := markdownNew().Parse([]byte(markdown))budget := &embeddedMediaBudget{maxImageBytes: 4, maxTotalBytes: 4, maxItems: 2}var items []map[string]any- if unresolved := walkMarkdownBlocksWithImages(t.Context(), doc, &items, false, true, budget); unresolved != 0 {+ if unresolved := walkMarkdownBlocksWithImages(t.Context(), doc, &items, false, true, &htmlWalkState{mediaBudget: budget}); unresolved != 0 {t.Fatalf("unresolved images = %d, want 0", unresolved)}if got := fetches.Load(); got != 2 {@@ -756,8 +756,8 @@ func TestMarkdownParser_NonTableHTMLBlockNotTable(t *testing.T) {// TestMarkdownParser_TableInCodeFenceNotRendered guards against a regression// where pipe rows INSIDE a fenced code block would be mis-identified as a GFM-// table and rewritten into <table> HTML, corrupting the code. renderMarkdownTablesInline-// tracks fence state (inFence) so the table detector must skip lines inside a fence.+// table, corrupting the code. The block parser consumes fences before+// processing their contents as table candidates.func TestMarkdownParser_TableInCodeFenceNotRendered(t *testing.T) {ctx := t.Context()p, _ := NewMarkdownParser(GoMarkdown)@@ -790,7 +790,7 @@ func TestMarkdownParser_TableInCodeFenceNotRendered(t *testing.T) {}// TestMarkdownParser_RawHTMLTableHandled covers a user-written raw <table> HTML-// block (not a GFM pipe table). renderMarkdownTablesInline only rewrites GFM+// block (not a GFM pipe table). The table hook only pads GFM// pipe tables, so the raw <table> passes through and is caught by isTableHTML// as an HTMLBlock, producing a single doc_type_kwd:"table" item in document// order. There must be NO redundant inlined doc_type_kwd:"text" copy.diff --git a/internal/parser/parser/pdf_parser_common.go b/internal/parser/parser/pdf_parser_common.goindex 59d0fbe68d..f09a34d638 100644--- a/internal/parser/parser/pdf_parser_common.go+++ b/internal/parser/parser/pdf_parser_common.go@@ -391,7 +391,7 @@ func (p *PDFParser) parseWithResult(ctx context.Context, filename string, data [case "opendataloader":return parsePDFWithOpenDataLoader(ctx, filename, data, p)case "somark":- return parsePDFWithSoMark(filename, data, p)+ return parsePDFWithSoMark(ctx, filename, data, p)case "tcadp":return parsePDFWithTCADP(ctx, filename, data, p)}diff --git a/internal/parser/parser/pdf_parser_somark.go b/internal/parser/parser/pdf_parser_somark.goindex f5af8d8382..be6b28f439 100644--- a/internal/parser/parser/pdf_parser_somark.go+++ b/internal/parser/parser/pdf_parser_somark.go@@ -15,7 +15,7 @@ import (// parsePDFWithSoMark submits a PDF and converts the task result into parser// output. Explicit tenant configuration never borrows a global endpoint or key.-func parsePDFWithSoMark(filename string, data []byte, parser *PDFParser) ParseResult {+func parsePDFWithSoMark(ctx context.Context, filename string, data []byte, parser *PDFParser) ParseResult {if len(data) == 0 {return emptyPDFResult(filename)}@@ -38,11 +38,16 @@ func parsePDFWithSoMark(filename string, data []byte, parser *PDFParser) ParseReif err != nil {return ParseResult{Err: err}}- items, pageCount := soMarkItems(result, parser.SoMarkKeepHeaderFooter)+ items, pageCount, warnings := soMarkItems(ctx, result, parser.SoMarkKeepHeaderFooter)+ if err := ctx.Err(); err != nil {+ return ParseResult{Err: err}+ }if len(items) == 0 {return ParseResult{Err: fmt.Errorf("parser: SoMark returned no usable blocks")}}- return pdfItemsToResult(filename, items, pageCount)+ parsed := pdfItemsToResult(filename, items, pageCount)+ parsed.Warnings = warnings+ return parsed}// soMarkSubmit uploads a PDF with the supplied credentials and parser options,@@ -155,38 +160,47 @@ func soMarkPoll(baseURL, taskID, apiKey string) (map[string]any, error) {return result, nil}-func soMarkItems(result map[string]any, keepHeaderFooter bool) ([]map[string]any, int) {+func soMarkItems(ctx context.Context, result map[string]any, keepHeaderFooter bool) ([]map[string]any, int, []string) {outputs, _ := result["outputs"].(map[string]any)jsonPayload, _ := outputs["json"].(map[string]any)pages, _ := jsonPayload["pages"].([]any)items := make([]map[string]any, 0)- for _, pageRaw := range pages {+ state := &htmlWalkState{}+ var warnings []string+ for pageIndex, pageRaw := range pages {page, ok := pageRaw.(map[string]any)if !ok {continue}+ pageNumber := pageIndex + 1+ if n := int(numberValue(page["page_number"])); n > 0 {+ pageNumber = n+ }blocks, _ := page["blocks"].([]any)for _, blockRaw := range blocks {block, ok := blockRaw.(map[string]any)if !ok {continue}- if item := soMarkBlockToItem(block, keepHeaderFooter); item != nil {+ blockItems, blockWarnings := soMarkBlockToItems(ctx, block, keepHeaderFooter, state)+ warnings = append(warnings, blockWarnings...)+ for _, item := range blockItems {+ item["page_number"] = pageNumberitems = append(items, item)}}}- return items, len(pages)+ return items, len(pages), append(state.warnings(), warnings...)}-func soMarkBlockToItem(block map[string]any, keepHeaderFooter bool) map[string]any {+func soMarkBlockToItems(ctx context.Context, block map[string]any, keepHeaderFooter bool, state *htmlWalkState) ([]map[string]any, []string) {blockType := strings.ToLower(strings.TrimSpace(stringValue(block["type"])))switch blockType {case "cate", "cate_item", "blank":- return nil+ return nil, nilcase "header", "footer":if !keepHeaderFooter {- return nil+ return nil, nil}}content := strings.TrimSpace(stringValue(block["content"]))@@ -195,33 +209,39 @@ func soMarkBlockToItem(block map[string]any, keepHeaderFooter bool) map[string]aif content == "" {content = "[Image]"}- return map[string]any{"text": content, "doc_type_kwd": "image", "layout": "figure"}+ return []map[string]any{{"text": content, "doc_type_kwd": "image", "layout": "figure"}}, nilcase "table":if content == "" {- return nil+ return nil, nil+ }+ markdown := &MarkdownParser{libType: GoMarkdown}+ parsed := markdown.parseWithState(ctx, "", []byte(content), state)+ items := ItemsToMaps(parsed.JSON)+ for _, item := range items {+ if item["doc_type_kwd"] == "table" {+ item["layout"] = "table"+ }}- item := map[string]any{"text": content, "doc_type_kwd": pdfTableDocType(content), "layout": "table"}- projectPDFTable(item)- return item+ return items, parsed.Warningscase "equation":if content == "" {- return nil+ return nil, nil}- return map[string]any{"text": content, "doc_type_kwd": "text", "layout": "equation"}+ return []map[string]any{{"text": content, "doc_type_kwd": "text", "layout": "equation"}}, nilcase "title":level := int(numberValue(block["title_level"]))if level < 1 || level > 6 {level = 1}if content == "" {- return nil+ return nil, nil}- return map[string]any{"text": strings.Repeat("#", level) + " " + content, "doc_type_kwd": "text", "layout": "title"}+ return []map[string]any{{"text": strings.Repeat("#", level) + " " + content, "doc_type_kwd": "text", "layout": "title"}}, nildefault:if content == "" {- return nil+ return nil, nil}- return map[string]any{"text": content, "doc_type_kwd": "text", "layout": "text"}+ return []map[string]any{{"text": content, "doc_type_kwd": "text", "layout": "text"}}, nil}}diff --git a/internal/parser/parser/pdf_parser_somark_test.go b/internal/parser/parser/pdf_parser_somark_test.goindex db33a1aa09..f2dd57d1c7 100644--- a/internal/parser/parser/pdf_parser_somark_test.go+++ b/internal/parser/parser/pdf_parser_somark_test.go@@ -89,7 +89,7 @@ func TestPDFParser_ParseWithResult_SoMarkRequiresBaseURL(t *testing.T) {}func TestSoMarkBlockToItem_DropsHeaderByDefault(t *testing.T) {- if item := soMarkBlockToItem(map[string]any{"type": "header", "content": "x"}, false); item != nil {+ if item, _ := soMarkBlockToItems(t.Context(), map[string]any{"type": "header", "content": "x"}, false, &htmlWalkState{}); item != nil {t.Fatalf("item = %#v, want nil", item)}}diff --git a/internal/parser/parser/pdf_parser_table_contract_test.go b/internal/parser/parser/pdf_parser_table_contract_test.goindex 5e7e114498..9a3d222df4 100644--- a/internal/parser/parser/pdf_parser_table_contract_test.go+++ b/internal/parser/parser/pdf_parser_table_contract_test.go@@ -73,11 +73,13 @@ func TestPDFRemoteParsers_TableItemsCarryTableMarkup(t *testing.T) {return tcadpAnyToItems(map[string]any{"type": "table", "content": pipeText, "table_data": blankRows})}, "text"},{"somark html", func() []map[string]any {- return []map[string]any{soMarkBlockToItem(map[string]any{"type": "table", "content": tableHTML}, false)}+ items, _ := soMarkBlockToItems(t.Context(), map[string]any{"type": "table", "content": tableHTML}, false, &htmlWalkState{})+ return items}, "table"},{"somark markdown", func() []map[string]any {- return []map[string]any{soMarkBlockToItem(map[string]any{"type": "table", "content": markdownTable}, false)}- }, "text"},+ items, _ := soMarkBlockToItems(t.Context(), map[string]any{"type": "table", "content": markdownTable}, false, &htmlWalkState{})+ return items+ }, "table"},}for _, tc := range cases {t.Run(tc.name, func(t *testing.T) {diff --git a/internal/parser/parser/pdf_table_no_html_text_test.go b/internal/parser/parser/pdf_table_no_html_text_test.goindex c5f29ae81b..d88deedf55 100644--- a/internal/parser/parser/pdf_table_no_html_text_test.go+++ b/internal/parser/parser/pdf_table_no_html_text_test.go@@ -33,7 +33,8 @@ func TestPDFTableItemsOmitHTMLText(t *testing.T) {return tcadpAnyToItems(map[string]any{"type": "table", "content": tableHTML})}},{"somark html", func() []map[string]any {- return []map[string]any{soMarkBlockToItem(map[string]any{"type": "table", "content": tableHTML}, false)}+ items, _ := soMarkBlockToItems(t.Context(), map[string]any{"type": "table", "content": tableHTML}, false, &htmlWalkState{})+ return items}},{"docx table", func() []map[string]any {return buildDOCXJSONSections(docxIR, newEmbeddedMediaBudget())diff --git a/internal/parser/parser/pptx_ir.go b/internal/parser/parser/pptx_ir.goindex 81a20ae2db..677686932c 100644--- a/internal/parser/parser/pptx_ir.go+++ b/internal/parser/parser/pptx_ir.go@@ -17,24 +17,16 @@package parserimport (- "encoding/base64""encoding/json""fmt""strings"++ "ragflow/internal/entity")-// buildPPTXJSONSections converts office_oxide presentation IR JSON into one-// text item per slide followed by that slide's image items. The IR carries-// exactly one section per slide, so section index + 1 is the slide number.-// Per-slide text is flattened with the shared IR walker (docxElementText),-// which covers paragraphs, headings, text boxes, tables, and (nested) lists.-//-// A slide without extractable text (blank layout, image-only) still-// yields an item with an empty text field, so slide_number stays-// aligned with the source deck.-//-// Pure Go (no cgo) so the IR → items transform is unit-testable without-// the office_oxide native library.+// buildPPTXJSONSections emits text runs, tables and images in slide order.+// Every item retains its slide number; a blank slide emits an empty text item.+// Container elements are traversed recursively so grouped tables remain structured.func buildPPTXJSONSections(irJSON string, budget *embeddedMediaBudget) ([]map[string]any, error) {var ir docxIRDocumentif err := json.Unmarshal([]byte(irJSON), &ir); err != nil {@@ -45,54 +37,81 @@ func buildPPTXJSONSections(irJSON string, budget *embeddedMediaBudget) ([]map[st}items := make([]map[string]any, 0, len(ir.Sections))for i, sec := range ir.Sections {+ start := len(items)var lines []string- for _, el := range sec.Elements {- // Each element's full text (paragraphs, table cells, list- // items, etc.) becomes a single value; the shared IR walker- // already inserts newlines between rows/items (cellSep "\n"),- // so internal newlines are preserved as-is. Only the- // element-level split is collapsed (one element → one value or- // none if empty).- if text := strings.TrimSpace(docxElementText(el, "\n")); text != "" {- lines = append(lines, text)+ flushText := func() {+ if len(lines) > 0 {+ items = append(items, map[string]any{"text": strings.Join(lines, "\n"), "doc_type_kwd": "text"})+ lines = nil}}- items = append(items, map[string]any{- "text": strings.Join(lines, "\n"),- "doc_type_kwd": "text",- "slide_number": i + 1,- })- mediaOrder := 0- forEachDOCXIRImage(sec.Elements, func(data []byte) bool {- mediaOrder++- imageItem := map[string]any{- "text": "",- "image": nil,- "doc_type_kwd": "image",- "slide_number": i + 1,- "media_order": mediaOrder,+ tableSequence := 0+ var emitElements func([]docxIRElement)+ emitElements = func(elements []docxIRElement) {+ for _, el := range elements {+ var units []map[string]any+ switch el.Type {+ case "table":+ if len(el.Rows) == 0 {+ continue+ }+ flushText()+ tableSequence+++ items = appendOfficeTableSections(items, el, "", fmt.Sprintf("pptx-slide-%d-table-%d", i+1, tableSequence), budget)+ continue+ case "text_box":+ emitElements(el.contentBlocks())+ continue+ case "list":+ for _, item := range el.Items {+ emitElements(item.Content)+ if item.Nested != nil {+ emitElements([]docxIRElement{{Type: "list", Items: item.Nested.Items}})+ }+ }+ continue+ case "paragraph", "heading":+ units = appendDOCXParagraphSections(nil, el, false, budget)+ case "image":+ units, _ = appendDOCXImageSection(nil, el.Data, budget, nil)+ default:+ if text := strings.TrimSpace(docxElementText(el, "\n")); text != "" {+ lines = append(lines, text)+ }+ }+ for _, unit := range units {+ if unit["doc_type_kwd"] == "text" {+ lines = append(lines, unit["text"].(string))+ continue+ }+ flushText()+ items = append(items, unit)+ }}- included, keepWalking := budget.include(data)- if included {- imageItem["image"] = base64.StdEncoding.EncodeToString(data)- } else if keepWalking {- imageItem["media_omitted"] = true- } else {- return false+ }+ emitElements(sec.Elements)+ flushText()+ if len(items) == start {+ items = append(items, map[string]any{"text": "", "doc_type_kwd": "text"})+ }+ mediaOrder := 0+ for _, item := range items[start:] {+ item["slide_number"] = i + 1+ if item["doc_type_kwd"] == "image" {+ mediaOrder+++ item["media_order"] = mediaOrder}- items = append(items, imageItem)- return keepWalking- })+ }}return items, nil}-// itemsAllEmpty reports whether every item lacks both text and image payloads.-// A deck whose IR sections all flatten to "" and contain no images must fall-// back to PlainText instead of emitting empty chunks that the Tokenizer later-// filters, silently yielding 0 chunks.+// itemsAllEmpty reports whether a presentation has no text, tables or images.func itemsAllEmpty(items []map[string]any) bool {for _, it := range items {+ if table, ok := it["table"].(*entity.TableData); ok && table != nil && len(table.Grid) > 0 {+ return false+ }if text, _ := it["text"].(string); strings.TrimSpace(text) != "" {return false}diff --git a/internal/parser/parser/pptx_ir_test.go b/internal/parser/parser/pptx_ir_test.goindex dfa37c01a4..7c6240024f 100644--- a/internal/parser/parser/pptx_ir_test.go+++ b/internal/parser/parser/pptx_ir_test.go@@ -10,6 +10,7 @@ func TestBuildPPTXJSONSections(t *testing.T) {name stringirJSON stringwantTexts []string // one entry per expected slide item+ wantRows [][]stringwantErr bool}{{@@ -55,7 +56,7 @@ func TestBuildPPTXJSONSections(t *testing.T) {]}]}]}]}`,- wantTexts: []string{"H1\nH2\na\nb"},+ wantRows: [][]string{{"H1", "H2"}, {"a", "b"}},},{name: "hard line break keeps newline",@@ -116,7 +117,7 @@ func TestBuildPPTXJSONSections(t *testing.T) {]}]}]}]}]}`,- wantTexts: []string{"Q1\nA1"},+ wantRows: [][]string{{"Q1", "A1"}},},{// Same flattening path with a list inside the text_box: the@@ -156,7 +157,7 @@ func TestBuildPPTXJSONSections(t *testing.T) {]}]}]}]}]}`,- wantTexts: []string{"head\nli1\nli2"},+ wantRows: [][]string{{"head\nli1\nli2"}},},{name: "invalid JSON",@@ -181,6 +182,16 @@ func TestBuildPPTXJSONSections(t *testing.T) {if err != nil {t.Fatalf("buildPPTXJSONSections: %v", err)}+ if tt.wantRows != nil {+ if len(items) != 1 {+ t.Fatalf("items=%#v", items)+ }+ requireTableRows(t, items[0], tt.wantRows)+ if items[0]["slide_number"] != 1 {+ t.Fatalf("slide lost: %#v", items[0])+ }+ return+ }if len(items) != len(tt.wantTexts) {t.Fatalf("items = %d, want %d: %+v", len(items), len(tt.wantTexts), items)}@@ -219,8 +230,8 @@ func TestBuildPPTXJSONSections_ExtractsSlideImagesInOrder(t *testing.T) {}{{docType: "text", text: "slide text", slide: 1},{docType: "image", image: "aGVsbG8=", slide: 1, order: 1},+ {docType: "table", slide: 1},{docType: "image", image: "aW1hZ2U=", slide: 1, order: 2},- {docType: "text", slide: 2},{docType: "image", image: "c2Vjb25k", slide: 2, order: 1},}if len(items) != len(want) {diff --git a/internal/parser/parser/table_regression_test.go b/internal/parser/parser/table_regression_test.gonew file mode 100644index 0000000000..8440123900--- /dev/null+++ b/internal/parser/parser/table_regression_test.go@@ -0,0 +1,349 @@+package parser++import (+ "encoding/json"+ "strings"+ "testing"++ "ragflow/internal/entity"+ "ragflow/internal/parser/tableutil"+)++func TestTableRegressionMarkdownMixedBackticksKeepTrailingCells(t *testing.T) {+ for _, cell := range []string{"a\\|b", "`a|b`", "``a|`b``", "`a``|b``", "`a\\`|b`", "a\\\\|b", "``a`|b``", "`a``|b`", "[a|b](u)", ""} {+ for _, prefix := range []string{"", " ", " ", " ", " ", "- "} {+ p, _ := NewMarkdownParser(GoMarkdown)+ res := p.ParseWithResult(t.Context(), "table.md", []byte(prefix+"| H1 | H2 |\n"+prefix+"|---|---|\n"+prefix+"| Owner | "+cell+" | TAIL_SENTINEL |\n"))+ if res.Err != nil {+ t.Fatal(res.Err)+ }+ wire, _ := json.Marshal(res.JSON)+ if !strings.Contains(string(wire), "TAIL_SENTINEL") {+ t.Fatalf("trailing cell dropped for cell=%q prefix=%q: %s", cell, prefix, wire)+ }+ }+ }+}++func TestTableRegressionDOCXCaptionAfterNestedTable(t *testing.T) {+ ir := `{"sections":[{"elements":[{"type":"text_box","content":[{"type":"table","rows":[{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"Nested"}]}]}]}]}]},{"type":"paragraph","content":[{"type":"text","text":"Outer Caption"}]},{"type":"table","rows":[{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"Outer"}]}]}]}]}]}]}`++ after := buildDOCXJSONSectionsWithCaptions(ir, nil, []string{"Outer Caption"})+ if len(after) != 2 {+ t.Fatalf("expected two tables without duplicate caption text: %#v", after)+ }+ for _, item := range after {+ if td, ok := item["table"].(*entity.TableData); ok {+ if tableRowsContain(td, "Nested") && td.Caption != "" {+ t.Error("nested table stole the outer caption")+ }+ if tableRowsContain(td, "Outer") && td.Caption != "Outer Caption" {+ t.Error("outer table lost its caption")+ }+ }+ }+}++func TestTableRegressionMarkdownFlattenBesideTable(t *testing.T) {+ source := " Before <table><tr><td>A</td><td>B</td></tr></table> After"+ p, _ := NewMarkdownParser(GoMarkdown)+ p.FlattenMediaToText = true+ after := p.ParseWithResult(t.Context(), "x.md", []byte(source))+ if after.Err != nil || len(after.JSON) == 0 {+ t.Fatalf("result=%+v", after)+ }+ payloads := 0+ for _, item := range after.JSON {+ if item.ToMap()["image"] != nil {+ payloads+++ }+ if item.ToMap()["doc_type_kwd"] == "image" {+ t.Error("flatten_media_to_text ignored")+ }+ }+ if payloads != 1 {+ t.Fatalf("image payloads=%d, want 1", payloads)+ }+}++func TestTableRegressionPPTXNestedImageRun(t *testing.T) {+ ir := `{"sections":[{"elements":[{"type":"paragraph","content":[{"type":"text","text":"before"},{"type":"image","content":[{"type":"text_box","content":[{"type":"image","data":"aGVsbG8="}]}]},{"type":"text","text":"after"}]}]}]}`++ after, err := buildPPTXJSONSections(ir, nil)+ if err != nil || len(after) != 3 || after[0]["text"] != "before" || after[2]["text"] != "after" {+ t.Fatalf("text/image order lost: %#v err=%v", after, err)+ }+ images := 0+ for _, item := range after {+ if item["doc_type_kwd"] == "image" {+ images+++ }+ }+ if images != 1 {+ t.Error("image inside a nested run was dropped")+ }+}++func TestTableRegressionSoMarkTableImageIDs(t *testing.T) {+ markup := "<table><tr><td>Logo</td><td><img src=\"data:image/png;base64,aGVsbG8=\"></td></tr></table>"+ items, _, _ := soMarkItems(t.Context(), map[string]any{"outputs": map[string]any{"json": map[string]any{"pages": []any{map[string]any{"blocks": []any{+ map[string]any{"type": "table", "content": markup}, map[string]any{"type": "table", "content": markup},+ }}}}}}, false)++ seen := map[string]bool{}+ for _, item := range items {+ if id, ok := item["source_table_id"].(string); ok {+ if seen[id] {+ t.Errorf("duplicate table image association id: %q", id)+ }+ seen[id] = true+ }+ }+ if len(items) != 4 || len(seen) != 2 {+ t.Fatalf("items=%#v", items)+ }+}++func TestTableRegressionSoMarkDocumentImageBudget(t *testing.T) {+ markup := "| Field | Value |\n|---|---|\n| Logo |  |"+ blocks := make([]any, maxEmbeddedMediaItems+1)+ for i := range blocks {+ blocks[i] = map[string]any{"type": "table", "content": markup}+ }+ items, _, warnings := soMarkItems(t.Context(), map[string]any{"outputs": map[string]any{"json": map[string]any{"pages": []any{map[string]any{"blocks": blocks}}}}}, false)+ images := 0+ for _, item := range items {+ if item["doc_type_kwd"] == "image" && item["image"] != nil {+ images+++ }+ }+ t.Logf("retained images=%d document item limit=%d", images, maxEmbeddedMediaItems)+ if images != maxEmbeddedMediaItems || !strings.Contains(strings.Join(warnings, "\n"), "256-item") {+ t.Errorf("retained=%d warnings=%v, want document-wide item limit", images, warnings)+ }+}++func TestTableRegressionHTMLTheadWithTDRetainsDefaultHeader(t *testing.T) {+ td, err := tableutil.ParseTableHTML("<table><thead><tr><td>Field</td><td>Value</td></tr><tr><td>Subfield</td><td>Subvalue</td></tr></thead><tbody><tr><td>Owner</td><td>RET</td></tr></tbody></table>")+ if err != nil {+ t.Fatal(err)+ }++ if td.HeaderRows != 1 {+ t.Errorf("header_rows=%d, want the default 1", td.HeaderRows)+ }+}++func TestTableRegressionMarkdownRawHTMLImageBudget(t *testing.T) {+ source := "<table><tr><td>" + strings.Repeat("<img src=\"data:image/png;base64,aGVsbG8=\">", maxEmbeddedMediaItems+1) + "</td></tr></table>\n\n"+ doc := markdownNew().Parse([]byte(source))+ budget := newEmbeddedMediaBudget()+ var items []map[string]any+ walkMarkdownBlocksWithImages(t.Context(), doc, &items, false, true, &htmlWalkState{mediaBudget: budget})+ images := 0+ for _, item := range items {+ if item["doc_type_kwd"] == "image" && item["image"] != nil {+ images+++ }+ }+ t.Logf("retained images=%d budget attempts=%d document item limit=%d", images, budget.items, maxEmbeddedMediaItems)+ if images != maxEmbeddedMediaItems || budget.items != maxEmbeddedMediaItems || !budget.countExhausted {+ t.Errorf("raw HTML budget not enforced: retained=%d budget=%+v", images, budget)+ }+}++func TestTableRegressionMarkdownStateResetsForNewDocument(t *testing.T) {+ p, _ := NewMarkdownParser(GoMarkdown)+ source := []byte("| Field | Value |\n|---|---|\n| Logo |  |")+ for range 2 {+ result := p.ParseWithResult(t.Context(), "doc.md", source)+ if result.Err != nil || len(result.JSON) != 2 {+ t.Fatalf("result=%+v", result)+ }+ table, img := result.JSON[0].ToMap(), result.JSON[1].ToMap()+ if table["source_table_id"] != "html-table-1" || img["parent_table_id"] != table["source_table_id"] {+ t.Fatalf("document state leaked: %#v", ItemsToMaps(result.JSON))+ }+ }+}++func TestMarkdownTableAfterContainerKeepsBlockBoundary(t *testing.T) {+ for _, prefix := range []string{"- Item\n", "> Quote\n"} {+ t.Run(strings.TrimSpace(prefix), func(t *testing.T) {+ p, _ := NewMarkdownParser(GoMarkdown)+ result := p.ParseWithResult(t.Context(), "table.md", []byte(prefix+"| H | V |\n|---|---|\n| OWNER | TAIL |\n\nAfter."))+ if result.Err != nil || len(result.JSON) != 3 {+ t.Fatalf("result=%+v", result)+ }+ wantPrefix := strings.TrimSpace(prefix)+ if strings.HasPrefix(wantPrefix, "> ") {+ wantPrefix = strings.TrimPrefix(wantPrefix, "> ")+ }+ if result.JSON[0].ToMap()["text"] != wantPrefix || result.JSON[2].ToMap()["text"] != "After." {+ t.Fatalf("surrounding text changed: %#v", ItemsToMaps(result.JSON))+ }+ requireTableRows(t, result.JSON[1].ToMap(), [][]string{{"H", "V"}, {"OWNER", "TAIL"}})+ })+ }+}++func TestMarkdownTableHookPreservesCodeAndHTMLBlocks(t *testing.T) {+ for _, source := range []string{+ " intro\n | H | V |\n |---|---|\n | x | y | extra |\n",+ "\tintro\n\t| H | V |\n\t|---|---|\n\t| x | y | extra |\n",+ "```\nintro\n| H | V |\n|---|---|\n| x | y | extra |\n```\n",+ "<pre>\nintro\n| H | V |\n|---|---|\n| x | y | extra |\n</pre>\n",+ "<div>\nintro\n| H | V |\n|---|---|\n| x | y | extra |\n</div>\n",+ "<div>| H |\n|---|---|\n| x | y | extra |\n</div>\n",+ "<div>\n<div>inner</div>\n| H | V |\n|---|---|\n| x | y | extra |\n</div>\n",+ "<!--\nintro\n| H | V |\n|---|---|\n| x | y | extra |\n-->\n",+ "- Item\n\n ```\n | H | V |\n |---|---|\n | x | y | extra |\n ```\n",+ "- Item\n\n | H | V |\n |---|---|\n | x | y | extra |\n",+ "1. Item\n\n | H | V |\n |---|---|\n | x | y | extra |\n",+ "| H | V |\n |---|---|\n | x | y | extra |\n",+ } {+ p, _ := NewMarkdownParser(GoMarkdown)+ result := p.ParseWithResult(t.Context(), "blocks.md", []byte(source))+ if result.Err != nil {+ t.Fatal(result.Err)+ }+ for _, item := range result.JSON {+ if item.ToMap()["doc_type_kwd"] == "table" {+ t.Fatalf("code or HTML content became a table: %q", source)+ }+ }+ wire, _ := json.Marshal(result.JSON)+ if !strings.HasPrefix(source, "<!--") && !strings.Contains(string(wire), "| x | y | extra |") {+ t.Fatalf("protected content lost: %s", wire)+ }+ if strings.HasPrefix(source, "<") && !strings.HasPrefix(source, "<!--") && (len(result.JSON) != 1 || result.JSON[0].ToMap()["text"] != strings.TrimSpace(source)) {+ t.Fatalf("raw HTML changed: %s", wire)+ }+ }+}++func TestMarkdownTableHookBalancesIndentedListTable(t *testing.T) {+ for _, tc := range []struct{ marker, indent string }{+ {"- ", " "}, {"1. ", " "}, {"10. ", " "}, {"- ", " "}, {"- ", "\t"},+ } {+ t.Run(tc.marker+tc.indent, func(t *testing.T) {+ for _, wide := range []bool{false, true} {+ row := "| x | y |"+ want := [][]string{{"H", "V"}, {"x", "y"}}+ if wide {+ row = "| x | y | TAIL |"+ want = [][]string{{"H", "V", ""}, {"x", "y", "TAIL"}}+ }+ source := tc.marker + "Item\n\n" + tc.indent + "| H | V |\n" + tc.indent + "|---|---|\n" + tc.indent + row + "\n"+ p, _ := NewMarkdownParser(GoMarkdown)+ result := p.ParseWithResult(t.Context(), "list.md", []byte(source))+ if result.Err != nil || len(result.JSON) != 2 || result.JSON[0].ToMap()["text"] != tc.marker+"Item" {+ t.Fatalf("list content lost: %#v err=%v", ItemsToMaps(result.JSON), result.Err)+ }+ if result.JSON[0].ToMap()["layout"] != "list" {+ t.Fatalf("list layout lost: %#v", result.JSON[0].ToMap())+ }+ requireTableRows(t, result.JSON[1].ToMap(), want)+ }+ })+ }+}++func TestMarkdownWideTableAtQuoteDepthLimit(t *testing.T) {+ prefix := strings.Repeat("> ", 63)+ source := prefix + "| H | V |\n" + prefix + "|---|---|\n" + prefix + "| x | y | TAIL |\n"+ p, _ := NewMarkdownParser(GoMarkdown)+ result := p.ParseWithResult(t.Context(), "nested.md", []byte(source))+ if result.Err != nil || len(result.JSON) != 1 {+ t.Fatalf("result=%+v", result)+ }+ requireTableRows(t, result.JSON[0].ToMap(), [][]string{{"H", "V", ""}, {"x", "y", "TAIL"}})+}++func TestMarkdownTableInNestedListPreservesReferences(t *testing.T) {+ source := "- Outer\n\n - Inner\n\n | A | B |\n |---|---|\n | [team][ref] | y | TAIL |\n\n[ref]: https://example.org\n"+ p, _ := NewMarkdownParser(GoMarkdown)+ result := p.ParseWithResult(t.Context(), "nested-list.md", []byte(source))+ if result.Err != nil || len(result.JSON) != 3 {+ t.Fatalf("result=%#v err=%v", ItemsToMaps(result.JSON), result.Err)+ }+ if result.JSON[0].ToMap()["text"] != "- Outer" || result.JSON[1].ToMap()["text"] != "- Inner" {+ t.Fatalf("list text lost: %#v", ItemsToMaps(result.JSON))+ }+ requireTableRows(t, result.JSON[2].ToMap(), [][]string{{"A", "B", ""}, {"team", "y", "TAIL"}})+}++func TestMarkdownTableHookBalancesMarkdownInsideHTMLContainer(t *testing.T) {+ source := "<div>\n\n| H | V |\n|---|---|\n| x | y | TAIL |\n\n</div>\n"+ p, _ := NewMarkdownParser(GoMarkdown)+ result := p.ParseWithResult(t.Context(), "table.md", []byte(source))+ for _, item := range result.JSON {+ if item.ToMap()["doc_type_kwd"] == "table" {+ requireTableRows(t, item.ToMap(), [][]string{{"H", "V", ""}, {"x", "y", "TAIL"}})+ return+ }+ }+ t.Fatalf("missing table in Markdown-enabled HTML container: %+v", result)+}++func TestMarkdownTableHookResumesAfterProtectedBlock(t *testing.T) {+ source := " code\n | H | V |\n |---|---|\n | x | y |\n\n| H | V |\n|---|---|\n| x | y | TAIL |\n"+ p, _ := NewMarkdownParser(GoMarkdown)+ result := p.ParseWithResult(t.Context(), "table.md", []byte(source))+ if result.Err != nil || len(result.JSON) != 2 || result.JSON[0].ToMap()["text"] != "code\n| H | V |\n|---|---|\n| x | y |" {+ t.Fatalf("code block changed: %+v", result)+ }+ requireTableRows(t, result.JSON[1].ToMap(), [][]string{{"H", "V", ""}, {"x", "y", "TAIL"}})+}++func TestMarkdownTableHookResumesAfterHTMLComments(t *testing.T) {+ source := "<div>\n<!-- <div> is an example -->\n</div>\n\n| H | V |\n|---|---|\n| x | y | TAIL |\n"+ p, _ := NewMarkdownParser(GoMarkdown)+ result := p.ParseWithResult(t.Context(), "table.md", []byte(source))+ for _, item := range result.JSON {+ if item.ToMap()["doc_type_kwd"] == "table" {+ requireTableRows(t, item.ToMap(), [][]string{{"H", "V", ""}, {"x", "y", "TAIL"}})+ return+ }+ }+ t.Fatalf("table after HTML block lost: %+v", result)+}++func TestDOCXContainerHeadingDoesNotReplaceBodyHeading(t *testing.T) {+ table := `{"type":"table","rows":[{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"value"}]}]}]}]}`+ for _, container := range []string{"text_box", "list"} {+ t.Run(container, func(t *testing.T) {+ content := `{"type":"heading","level":1,"content":[{"type":"text","text":"Sidebar"}]},` + table+ wrapped := `{"type":"text_box","content":[` + content + `]}`+ if container == "list" {+ wrapped = `{"type":"list","items":[{"content":[` + content + `]}]}`+ }+ ir := `{"sections":[{"elements":[{"type":"heading","level":1,"content":[{"type":"text","text":"Chapter"}]},` + wrapped + `,` + table + `]}]}`+ items := buildDOCXJSONSections(ir, nil)+ var captions []string+ for _, item := range items {+ if td, ok := item["table"].(*entity.TableData); ok {+ captions = append(captions, td.Caption)+ }+ }+ if len(captions) != 2 || captions[0] != "Table Location: Sidebar" || captions[1] != "Table Location: Chapter" {+ t.Fatalf("table captions=%q", captions)+ }+ })+ }+}++func TestSoMarkPreservesImageResolutionWarnings(t *testing.T) {+ source := `<table><tr><td><img alt="logo" src="data:image/png;base64,!!!"></td></tr></table>`+ items, _, warnings := soMarkItems(t.Context(), map[string]any{"outputs": map[string]any{"json": map[string]any{"pages": []any{map[string]any{"blocks": []any{+ map[string]any{"type": "table", "content": source},+ map[string]any{"type": "table", "content": "| A | B |\n|---|---|\n| x | y |"},+ }}}}}}, false)+ if len(items) != 2 {+ t.Fatalf("items=%#v", items)+ }+ requireTableRows(t, items[0], [][]string{{"logo"}})+ if !strings.Contains(strings.Join(warnings, "\n"), "could not resolve 1 image source") {+ t.Fatalf("missing image resolution warning: %v", warnings)+ }+}diff --git a/internal/parser/parser/table_structure_test.go b/internal/parser/parser/table_structure_test.gonew file mode 100644index 0000000000..745eeffd95--- /dev/null+++ b/internal/parser/parser/table_structure_test.go@@ -0,0 +1,311 @@+package parser++import (+ "context"+ "encoding/json"+ "net/http"+ "net/http/httptest"+ "reflect"+ "strings"+ "testing"++ "ragflow/internal/entity"+)++func requireTableRows(t *testing.T, item map[string]any, want [][]string) {+ t.Helper()+ td, ok := item["table"].(*entity.TableData)+ if item["doc_type_kwd"] != "table" || !ok || td == nil {+ t.Fatalf("expected structured table, got %#v", item)+ }+ if got := td.CellTexts(); !reflect.DeepEqual(got, want) {+ t.Fatalf("table rows = %#v, want %#v", got, want)+ }+}++func TestMarkdownParserPreservesEmptyTableCells(t *testing.T) {+ p, _ := NewMarkdownParser(GoMarkdown)+ for _, md := range []string{+ "| | |\n|---|---|\n| Document ID | demo |\n| Owner | RET |",+ "| | |\r\n|---|---|\r\n| Document ID | demo |\r\n| Owner | RET |\r\n",+ } {+ res := p.ParseWithResult(t.Context(), "metadata.md", []byte(md))+ if res.Err != nil || len(res.JSON) != 1 {+ t.Fatalf("result = %+v", res)+ }+ requireTableRows(t, res.JSON[0].ToMap(), [][]string{{"", ""}, {"Document ID", "demo"}, {"Owner", "RET"}})+ }+ res := p.ParseWithResult(t.Context(), "blank-row.md", []byte("| A | B |\n|---|---|\n| | |\n| x | y |"))+ if len(res.JSON) != 1 {+ t.Fatalf("items = %d, want 1", len(res.JSON))+ }+ requireTableRows(t, res.JSON[0].ToMap(), [][]string{{"A", "B"}, {"", ""}, {"x", "y"}})+}++func TestMarkdownParserPreservesHTMLTableAtEOF(t *testing.T) {+ p, _ := NewMarkdownParser(GoMarkdown)+ res := p.ParseWithResult(t.Context(), "table.md", []byte("<table><tr><td>Owner</td><td>RET</td></tr></table>"))+ if res.Err != nil || len(res.JSON) != 1 {+ t.Fatalf("result = %+v", res)+ }+ requireTableRows(t, res.JSON[0].ToMap(), [][]string{{"Owner", "RET"}})+}++func TestMarkdownParserPreservesInlineHTMLTable(t *testing.T) {+ p, _ := NewMarkdownParser(GoMarkdown)+ res := p.ParseWithResult(t.Context(), "table.md", []byte("Before. <table><tr><td>Owner</td><td>RET</td></tr></table> After."))+ if res.Err != nil || len(res.JSON) != 3 {+ t.Fatalf("result = %+v", res)+ }+ if res.JSON[0].ToMap()["text"] != "Before." || res.JSON[2].ToMap()["text"] != "After." {+ t.Fatalf("items=%#v", ItemsToMaps(res.JSON))+ }+ requireTableRows(t, res.JSON[1].ToMap(), [][]string{{"Owner", "RET"}})+}++func TestMarkdownParserPreservesWideTableWithInlineHeaderAndReference(t *testing.T) {+ p, _ := NewMarkdownParser(GoMarkdown)+ res := p.ParseWithResult(t.Context(), "table.md", []byte("<b>Field</b> | Value\n---|---\nOwner | [team][ref] | TAIL\n\n[ref]: https://example.org\n"))+ if res.Err != nil || len(res.JSON) != 1 {+ t.Fatalf("result=%+v", res)+ }+ requireTableRows(t, res.JSON[0].ToMap(), [][]string{{"Field", "Value", ""}, {"Owner", "team", "TAIL"}})+}++func TestMarkdownParserPreservesEscapedPipesInHTMLAttributes(t *testing.T) {+ for _, destination := range []string{`x\|y`, `x|y`} {+ p, _ := NewMarkdownParser(GoMarkdown)+ res := p.ParseWithResult(t.Context(), "table.md", []byte("| A | B |\n|---|---|\n| <a href=\""+destination+"\">link</a> | TAIL |\n"))+ if res.Err != nil || len(res.JSON) != 1 {+ t.Fatalf("result=%+v", res)+ }+ requireTableRows(t, res.JSON[0].ToMap(), [][]string{{"A", "B"}, {"link", "TAIL"}})+ }+}++func TestMarkdownParserDoesNotTreatTablePrefixAsTable(t *testing.T) {+ p, _ := NewMarkdownParser(GoMarkdown)+ source := "Before <tableau>value</tableau> after."+ res := p.ParseWithResult(t.Context(), "text.md", []byte(source))+ if res.Err != nil || len(res.JSON) != 1 || res.JSON[0].ToMap()["text"] != source {+ t.Fatalf("non-table HTML changed: %+v", res)+ }+}++func TestMarkdownTableImageTimeoutDoesNotConsumeBudget(t *testing.T) {+ ctx, cancel := context.WithCancel(t.Context())+ cancel()+ doc := markdownNew().Parse([]byte("| A | B |\n|---|---|\n| Logo |  |\n"))+ budget := newEmbeddedMediaBudget()+ var items []map[string]any+ unresolved := walkMarkdownBlocksWithImages(ctx, doc, &items, false, true, &htmlWalkState{mediaBudget: budget})+ if budget.items != 0 || unresolved != 0 {+ t.Fatalf("timed out image consumed budget: attempts=%d unresolved=%d", budget.items, unresolved)+ }+}++func TestMarkdownParserPreservesImageBesideInlineTable(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "image/png")+ _, _ = w.Write([]byte("hello"))+ }))+ defer server.Close()+ old := ssrfAllowLoopback+ ssrfAllowLoopback = true+ defer func() { ssrfAllowLoopback = old }()+ p, _ := NewMarkdownParser(GoMarkdown)+ res := p.ParseWithResult(t.Context(), "table.md", []byte(" <table><tr><td>Owner</td><td>RET</td></tr></table>"))+ if res.Err != nil {+ t.Fatal(res.Err)+ }+ images, tables := 0, 0+ for _, item := range res.JSON {+ m := item.ToMap()+ if m["doc_type_kwd"] == "table" {+ tables+++ }+ if m["doc_type_kwd"] == "image" && m["image"] == "data:image/octet-stream;base64,aGVsbG8=" {+ images+++ }+ }+ if images != 1 || tables != 1 {+ t.Fatalf("images=%d tables=%d items=%#v", images, tables, ItemsToMaps(res.JSON))+ }+}++func TestMarkdownParserPreservesPipesInsideCells(t *testing.T) {+ p, _ := NewMarkdownParser(GoMarkdown)+ for _, cell := range []string{"a\\|b", "`a|b`", "``a|b``"} {+ res := p.ParseWithResult(t.Context(), "table.md", []byte("| Field | Value |\n|---|---|\n| Owner | "+cell+" |"))+ if res.Err != nil || len(res.JSON) != 1 {+ t.Fatalf("result = %+v", res)+ }+ requireTableRows(t, res.JSON[0].ToMap(), [][]string{{"Field", "Value"}, {"Owner", "a|b"}})+ }+}++func TestMarkdownParserKeepsTableImageAssociation(t *testing.T) {+ p, _ := NewMarkdownParser(GoMarkdown)+ for _, source := range []string{+ "| Field | Value |\n|---|---|\n| Logo |  |",+ "<table><tr><td>Logo</td><td><img alt=\"logo\" src=\"data:image/png;base64,aGVsbG8=\"></td></tr></table>",+ } {+ res := p.ParseWithResult(t.Context(), "table.md", []byte(source))+ if res.Err != nil || len(res.JSON) != 2 {+ t.Fatalf("result = %+v", res)+ }+ table, img := res.JSON[0].ToMap(), res.JSON[1].ToMap()+ if table["doc_type_kwd"] != "table" || img["doc_type_kwd"] != "image" || table["source_table_id"] == nil || img["parent_table_id"] != table["source_table_id"] {+ t.Fatalf("table/image association lost: %#v", ItemsToMaps(res.JSON))+ }+ }+}++func TestPDFMarkdownPreservesEmptyHeaderTable(t *testing.T) {+ res := pdfMarkdownToResult(t.Context(), "table.pdf", "| | |\n|---|---|\n| Owner | RET |", 2)+ if res.Err != nil || len(res.JSON) != 1 {+ t.Fatalf("result = %+v", res)+ }+ requireTableRows(t, res.JSON[0].ToMap(), [][]string{{"", ""}, {"Owner", "RET"}})+}++func TestSoMarkPreservesMarkdownTableAndNotes(t *testing.T) {+ items, pages, _ := soMarkItems(t.Context(), map[string]any{"outputs": map[string]any{"json": map[string]any{"pages": []any{map[string]any{"blocks": []any{+ map[string]any{"type": "table", "content": "Before.\n\n| | |\n|---|---|\n| Owner | RET |\n\nAfter."},+ }}}}}}, false)+ if pages != 1 || len(items) != 3 {+ t.Fatalf("pages=%d items=%#v", pages, items)+ }+ if items[0]["text"] != "Before." || items[2]["text"] != "After." {+ t.Fatalf("notes = %#v", items)+ }+ requireTableRows(t, items[1], [][]string{{"", ""}, {"Owner", "RET"}})+ if items[1]["layout"] != "table" || items[0]["layout"] == "table" || items[2]["layout"] == "table" {+ t.Fatalf("table layout misplaced: %#v", items)+ }+}++func TestSoMarkPreservesPageOnSplitTableItems(t *testing.T) {+ items, _, _ := soMarkItems(t.Context(), map[string]any{"outputs": map[string]any{"json": map[string]any{"pages": []any{+ map[string]any{"blocks": []any{}},+ map[string]any{"blocks": []any{map[string]any{"type": "table", "content": "Before.\n\n| A | B |\n|---|---|\n| Owner | RET |\n\nAfter."}}},+ }}}}, false)+ if len(items) != 3 {+ t.Fatalf("items=%#v", items)+ }+ for _, item := range items {+ if item["page_number"] != 2 {+ t.Fatalf("page lost: %#v", item)+ }+ }+}++func tableStructureIR(t *testing.T, container string) string {+ t.Helper()+ table := `{"type":"table","rows":[{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"Owner"}]}]},{"content":[{"type":"paragraph","content":[{"type":"text","text":"RET"}]}]}]}]}`+ if container == "text_box" {+ table = `{"type":"text_box","content":[` + table + `]}`+ }+ if container == "list" {+ table = `{"type":"list","items":[{"content":[` + table + `]}]}`+ }+ return `{"sections":[{"elements":[{"type":"paragraph","content":[{"type":"text","text":"Before."}]},` + table + `,{"type":"paragraph","content":[{"type":"text","text":"After."}]}]}]}`+}++func TestOfficePreservesNestedTables(t *testing.T) {+ for _, container := range []string{"table", "text_box", "list"} {+ t.Run(container, func(t *testing.T) {+ ir := tableStructureIR(t, container)+ for _, format := range []string{"docx", "pptx"} {+ t.Run(format, func(t *testing.T) {+ var items []map[string]any+ if format == "docx" {+ items = buildDOCXJSONSections(ir, nil)+ } else {+ var err error+ items, err = buildPPTXJSONSections(ir, nil)+ if err != nil {+ t.Fatal(err)+ }+ }+ if len(items) != 3 {+ t.Fatalf("items = %#v, want before/table/after", items)+ }+ requireTableRows(t, items[1], [][]string{{"Owner", "RET"}})+ if items[0]["text"] != "Before." || items[2]["text"] != "After." {+ t.Fatalf("notes = %#v", items)+ }+ if format == "pptx" {+ for _, item := range items {+ if item["slide_number"] != 1 {+ t.Fatalf("slide lost: %#v", item)+ }+ }+ }+ })+ }+ })+ }+}++func TestPPTXTableOnlySlideIsNotEmpty(t *testing.T) {+ if itemsAllEmpty([]map[string]any{{"doc_type_kwd": "table", "table": &entity.TableData{Grid: [][]entity.TableCell{{{Text: "RET"}}}}}}) {+ t.Fatal("table-only slide was considered empty")+ }+}++func TestOfficeTableRetainsDefaultHeaderCount(t *testing.T) {+ ir := `{"sections":[{"elements":[{"type":"table","rows":[{"is_header":true,"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"H1"}]}]}]},{"is_header":true,"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"H2"}]}]}]}]}]}]}`+ items := buildDOCXJSONSections(ir, nil)+ if len(items) != 1 {+ t.Fatalf("items=%#v", items)+ }+ td := items[0]["table"].(*entity.TableData)+ if td.HeaderRows != 1 || len(td.Grid) != 2 {+ t.Fatalf("table=%+v, want two rows and one default header", td)+ }+}++func TestOfficeTablePreservesLiteralCellText(t *testing.T) {+ el := docxIRElement{Type: "table", Rows: []docxIRRow{{Cells: []docxIRCell{{Content: []docxIRElement{{Type: "paragraph", Content: json.RawMessage(`[{"type":"text","text":" <b>A & B</b>\nsecond line "}]`)}}}}}}}+ items := appendOfficeTableSections(nil, el, " A & <B> ", "table-1", newEmbeddedMediaBudget())+ if len(items) != 1 {+ t.Fatalf("items=%#v", items)+ }+ requireTableRows(t, items[0], [][]string{{"<b>A & B</b>\nsecond line"}})+ td := items[0]["table"].(*entity.TableData)+ if td.HeaderRows != 1 || td.Caption != "A & <B>" {+ t.Fatalf("table metadata=%+v", td)+ }+}++func TestEPUBPreservesTablesInChapterOrder(t *testing.T) {+ res := parseTestEPUB(t, []epubTestChapter{+ {name: "one.xhtml", content: "<p>Before.</p><table><tr><td>Owner</td><td>RET</td></tr></table><p>After.</p>"},+ {name: "two.xhtml", content: "<p>Next chapter.</p>"},+ })+ if res.Err != nil || len(res.JSON) != 4 {+ t.Fatalf("result = %+v", res)+ }+ requireTableRows(t, res.JSON[1].ToMap(), [][]string{{"Owner", "RET"}})+ if res.JSON[0].ToMap()["text"] != "Before." || res.JSON[2].ToMap()["text"] != "After." || res.JSON[3].ToMap()["text"] != "Next chapter." {+ t.Fatalf("items = %#v", ItemsToMaps(res.JSON))+ }+ b, err := json.Marshal(res.JSON)+ if err != nil {+ t.Fatal(err)+ }+ if strings.Count(string(b), "Owner") != 1 {+ t.Fatalf("table duplicated: %s", b)+ }+}++func TestEPUBNormalizesChapterWhitespace(t *testing.T) {+ res := parseTestEPUB(t, []epubTestChapter{{name: "one.xhtml", content: epubTestHTML("<textarea> first\n second</textarea>")}})+ if res.Err != nil || len(res.JSON) != 1 {+ t.Fatalf("result=%+v", res)+ }+ if got := res.JSON[0].ToMap()["text"]; got != "Test first second" {+ t.Fatalf("chapter text=%q", got)+ }+}diff --git a/internal/service/document/document_artifact.go b/internal/service/document/document_artifact.goindex b7f339d05d..b0e506dfed 100644--- a/internal/service/document/document_artifact.go+++ b/internal/service/document/document_artifact.go@@ -11,7 +11,9 @@ import ("ragflow/internal/common""ragflow/internal/dao"enginetypes "ragflow/internal/engine/types"+ "ragflow/internal/entity""ragflow/internal/parser/parser"+ "ragflow/internal/parser/tableutil""ragflow/internal/permission""ragflow/internal/service""ragflow/internal/storage"@@ -321,13 +323,25 @@ func extractDOCPreviewText(ctx context.Context, filename string, data []byte) (sif res.Err != nil {return "", res.Err}+ return docPreviewText(res)+}++// docPreviewText flattens parser items for the legacy .doc preview. Table+// items carry only TableData, so render them instead of dropping the table+// content from the preview.+func docPreviewText(res parser.ParseResult) (string, error) {if len(res.JSON) == 0 {return "", fmt.Errorf("doc preview: no content extracted")}parts := make([]string, 0, len(res.JSON))for _, item := range res.JSON {- if text, ok := item.ToMap()["text"].(string); ok {+ m := item.ToMap()+ if text, ok := m["text"].(string); ok && text != "" {parts = append(parts, text)+ continue+ }+ if table, ok := m["table"].(*entity.TableData); ok && table != nil {+ parts = append(parts, tableutil.RenderTableHTML(table))}}return strings.Join(parts, "\n"), nildiff --git a/internal/service/document/document_artifact_test.go b/internal/service/document/document_artifact_test.gonew file mode 100644index 0000000000..2791a30852--- /dev/null+++ b/internal/service/document/document_artifact_test.go@@ -0,0 +1,46 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package document++import (+ "strings"+ "testing"++ "ragflow/internal/entity"+ "ragflow/internal/parser/parser"+)++func TestDocPreviewTextRendersTableItems(t *testing.T) {+ table := &entity.TableData{+ Grid: [][]entity.TableCell{{{Text: "Owner"}, {Text: "RET"}}},+ HeaderRows: 1,+ }+ res := parser.ParseResult{JSON: parser.MapsToItems([]map[string]any{+ {"text": "Before", "doc_type_kwd": "text"},+ {"doc_type_kwd": "table", "table": table},+ })}++ text, err := docPreviewText(res)+ if err != nil {+ t.Fatalf("docPreviewText: %v", err)+ }+ for _, want := range []string{"Before", "Owner", "RET"} {+ if !strings.Contains(text, want) {+ t.Fatalf("preview %q missing %q", text, want)+ }+ }+}