Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): preserve DOCX XML character references (#20359)

jay77721630482567694
diff --git a/build.sh b/build.sh
index e3d2ecf309..7de4e775a2 100755
--- a/build.sh
+++ b/build.sh
@@ -27,7 +27,7 @@ SYSTEM_DEPS="/opt/ragflow-native-libs"
# office_oxide native library settings — static linking
OFFICE_OXIDE_PREFIX="${HOME}/ragflow-native-libs/office_oxide"
-OFFICE_OXIDE_VERSION="0.1.9"
+OFFICE_OXIDE_VERSION="0.1.12"
# pdfium native library settings — static linking (kognitos/pdfium-static)
PDFIUM_STATIC_PREFIX="${HOME}/ragflow-native-libs/pdfium-static"
diff --git a/go.mod b/go.mod
index 58b936ecb5..211edf0197 100644
--- a/go.mod
+++ b/go.mod
@@ -69,7 +69,7 @@ require (
github.com/spf13/viper v1.18.2
github.com/ucloud/ucloud-sandbox-sdk-go v0.0.0-20260807065450-08464aef9ed5
github.com/xuri/excelize/v2 v2.11.0
- github.com/yfedoseev/office_oxide/go v0.1.9
+ github.com/yfedoseev/office_oxide/go v0.1.12
github.com/yfedoseev/pdf_oxide/go v0.3.73
github.com/yuin/goldmark v1.7.1
github.com/zeebo/xxh3 v1.0.2
diff --git a/go.sum b/go.sum
index ed9fb57165..90696378cb 100644
--- a/go.sum
+++ b/go.sum
@@ -612,8 +612,8 @@ github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 h1:+C0TIdyyYmzadGaL/HBL
github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ=
github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc=
github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA=
-github.com/yfedoseev/office_oxide/go v0.1.9 h1:gvTMCvJDYEZNF4dKd1cdfvU+9SRDSpnbNhCpZJbZOaE=
-github.com/yfedoseev/office_oxide/go v0.1.9/go.mod h1:YLtMlKUkRCp/Q96wsy7D6yoBKDeJnP66UH+c9Bb+E+M=
+github.com/yfedoseev/office_oxide/go v0.1.12 h1:QLsiB8Aj7LB42D9Vwu1rslRuZg/vSXc4IkVXCDKQv6o=
+github.com/yfedoseev/office_oxide/go v0.1.12/go.mod h1:YLtMlKUkRCp/Q96wsy7D6yoBKDeJnP66UH+c9Bb+E+M=
github.com/yfedoseev/pdf_oxide/go v0.3.73 h1:hRad+X4W0ILW3bCQeCvdUKcoy8yMj2TFS6/XtoMk55I=
github.com/yfedoseev/pdf_oxide/go v0.3.73/go.mod h1:QbJ/nLbez0al2EnqEdEPIlGflFprWmiuUM4mo9rNNOI=
github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4=
diff --git a/internal/parser/parser/doc_parser_synth_test.go b/internal/parser/parser/doc_parser_synth_test.go
index 8c7d1bafed..6f2ae1c0b2 100644
--- a/internal/parser/parser/doc_parser_synth_test.go
+++ b/internal/parser/parser/doc_parser_synth_test.go
@@ -91,6 +91,8 @@ func buildSyntheticDoc(t *testing.T, text string) []byte {
copy(dir[1*128:], dirEntry("WordDocument", 2, 2, uint64(len(wordDoc))))
copy(dir[2*128:], dirEntry("0Table", 2, 3, uint64(len(clx))))
copy(dir[3*128:], dirEntry("", 0, 0xFFFFFFFF, 0))
+ binary.LittleEndian.PutUint32(dir[0*128+0x4C:], 1) // root child: WordDocument
+ binary.LittleEndian.PutUint32(dir[1*128+0x44:], 2) // left sibling: 0Table
// ── Assemble ──────────────────────────────────────────────────────────
tableSector := make([]byte, sector)
diff --git a/internal/parser/parser/docx_parser_cgo_test.go b/internal/parser/parser/docx_parser_cgo_test.go
index 65ff846f7f..6132e1eff7 100644
--- a/internal/parser/parser/docx_parser_cgo_test.go
+++ b/internal/parser/parser/docx_parser_cgo_test.go
@@ -5,6 +5,7 @@ package parser
import (
"archive/zip"
"bytes"
+ "strings"
"testing"
)
@@ -129,3 +130,60 @@ func writeZipFile(t *testing.T, zw *zip.Writer, name, body string) {
t.Fatalf("write zip entry %s: %v", name, err)
}
}
+
+// TestDOCXParser_PreservesXMLCharacterReferences pins the office_oxide version.
+// v0.1.9 deleted every XML character reference in DOCX run text instead of
+// decoding it, so <w:t>&lt;SEP&gt;</w:t> parsed as "SEP". That reached the
+// chunker intact, so a delimiter configured as "`<SEP>`" could never match and
+// the marker survived into the chunk body. v0.1.10 was the correctness release
+// that fixed this defect class; v0.1.12 is the pinned version.
+func TestDOCXParser_PreservesXMLCharacterReferences(t *testing.T) {
+ cases := []struct {
+ name string
+ run string // run text as it appears inside <w:t>, entities unescaped
+ want string
+ wantMarkdown string
+ }{
+ {"named", "&lt;SEP&gt;", "<SEP>", `\<SEP\>`},
+ {"decimal", "&#60;SEP&#62;", "<SEP>", `\<SEP\>`},
+ {"hex", "&#x3C;SEP&#x3E;", "<SEP>", `\<SEP\>`},
+ {"ampersand", "A&amp;B", "A&B", "A&B"},
+ {"apostrophe", "it&apos;s", "it's", "it's"},
+ {"emdash", "a&#8212;b", "a—b", "a—b"},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ data := minimalDOCX(t, tc.run)
+
+ jsonParser := NewDOCXParser()
+ jsonParser.ConfigureFromSetup(map[string]any{"output_format": "json"})
+ jsonRes := jsonParser.ParseWithResult(t.Context(), "entities.docx", data)
+ if jsonRes.Err != nil {
+ t.Fatalf("ParseWithResult(json): %v", jsonRes.Err)
+ }
+ if got := joinItemTexts(jsonRes.JSON); !strings.Contains(got, tc.want) {
+ t.Errorf("json text = %q, want it to contain %q", got, tc.want)
+ }
+
+ mdParser := NewDOCXParser()
+ mdParser.ConfigureFromSetup(map[string]any{"output_format": "markdown"})
+ mdRes := mdParser.ParseWithResult(t.Context(), "entities.docx", data)
+ if mdRes.Err != nil {
+ t.Fatalf("ParseWithResult(markdown): %v", mdRes.Err)
+ }
+ if !strings.Contains(mdRes.Markdown, tc.wantMarkdown) {
+ t.Errorf("markdown = %q, want it to contain %q", mdRes.Markdown, tc.wantMarkdown)
+ }
+ })
+ }
+}
+
+func joinItemTexts(items []map[string]any) string {
+ parts := make([]string, 0, len(items))
+ for _, item := range items {
+ if text, ok := item["text"].(string); ok {
+ parts = append(parts, text)
+ }
+ }
+ return strings.Join(parts, "\n")
+}
diff --git a/ragflow_deps/download_deps.py b/ragflow_deps/download_deps.py
index 2a25cf5405..fdb8ad2536 100755
--- a/ragflow_deps/download_deps.py
+++ b/ragflow_deps/download_deps.py
@@ -121,7 +121,7 @@ def get_urls(use_china_mirrors=False) -> list[str | list[str]]:
# network access during CI.
["https://github.com/kognitos/pdfium-static/releases/download/chromium%2F7809/pdfium-linux-x64-static.tgz", "pdfium-linux-x64-static.tgz"],
["https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.73/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"],
- ["https://github.com/yfedoseev/office_oxide/releases/download/v0.1.9/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"],
+ ["https://github.com/yfedoseev/office_oxide/releases/download/v0.1.12/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"],
# ONNX Runtime static archives for the Go in-process (DeepDoc)
# backend. Statically linked into the server binary (see build.sh:
# ONNXRUNTIME_STATIC_PREFIX — no --whole-archive, so unreferenced
@@ -171,7 +171,7 @@ def get_urls(use_china_mirrors=False) -> list[str | list[str]]:
# network access during CI.
["https://github.com/kognitos/pdfium-static/releases/download/chromium%2F7809/pdfium-linux-x64-static.tgz", "pdfium-linux-x64-static.tgz"],
["https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.73/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"],
- ["https://github.com/yfedoseev/office_oxide/releases/download/v0.1.9/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"],
+ ["https://github.com/yfedoseev/office_oxide/releases/download/v0.1.12/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"],
# ONNX Runtime static archives for the Go in-process (DeepDoc)
# backend. Statically linked into the server binary (see build.sh:
# ONNXRUNTIME_STATIC_PREFIX — no --whole-archive, so unreferenced
diff --git a/ragflow_deps/download_go_deps.py b/ragflow_deps/download_go_deps.py
index 90d5f1cf81..a23b27b0bf 100755
--- a/ragflow_deps/download_go_deps.py
+++ b/ragflow_deps/download_go_deps.py
@@ -226,7 +226,7 @@ def get_urls(use_china_mirrors=False) -> list[str | list[str]]:
# functions — pre-downloaded to avoid network access during CI.
["https://gh-proxy.com/https://github.com/kognitos/pdfium-static/releases/download/chromium%2F7809/pdfium-linux-x64-static.tgz", "pdfium-linux-x64-static.tgz"],
["https://gh-proxy.com/https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.73/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"],
- ["https://gh-proxy.com/https://github.com/yfedoseev/office_oxide/releases/download/v0.1.9/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"],
+ ["https://gh-proxy.com/https://github.com/yfedoseev/office_oxide/releases/download/v0.1.12/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"],
[
f"https://gh-proxy.com/https://github.com/infiniflow/ragflow-build/releases/download/onnxruntime-v{ORT_VERSION}/{_ort_asset_name(ORT_VERSION)}",
_ort_asset_name(ORT_VERSION),
@@ -255,7 +255,7 @@ def get_urls(use_china_mirrors=False) -> list[str | list[str]]:
# functions — pre-downloaded to avoid network access during CI.
["https://github.com/kognitos/pdfium-static/releases/download/chromium%2F7809/pdfium-linux-x64-static.tgz", "pdfium-linux-x64-static.tgz"],
["https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.73/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"],
- ["https://github.com/yfedoseev/office_oxide/releases/download/v0.1.9/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"],
+ ["https://github.com/yfedoseev/office_oxide/releases/download/v0.1.12/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"],
[
f"https://github.com/infiniflow/ragflow-build/releases/download/onnxruntime-v{ORT_VERSION}/{_ort_asset_name(ORT_VERSION)}",
_ort_asset_name(ORT_VERSION),