RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): preserve DOCX XML character references (#20359)
jay77721630482567694
diff --git a/build.sh b/build.shindex e3d2ecf309..7de4e775a2 100755--- a/build.sh+++ b/build.sh@@ -27,7 +27,7 @@ SYSTEM_DEPS="/opt/ragflow-native-libs"# office_oxide native library settings — static linkingOFFICE_OXIDE_PREFIX="${HOME}/ragflow-native-libs/office_oxide"-OFFICE_OXIDE_VERSION="0.1.9"+OFFICE_OXIDE_VERSION="0.1.12"# pdfium native library settings — static linking (kognitos/pdfium-static)PDFIUM_STATIC_PREFIX="${HOME}/ragflow-native-libs/pdfium-static"diff --git a/go.mod b/go.modindex 58b936ecb5..211edf0197 100644--- a/go.mod+++ b/go.mod@@ -69,7 +69,7 @@ require (github.com/spf13/viper v1.18.2github.com/ucloud/ucloud-sandbox-sdk-go v0.0.0-20260807065450-08464aef9ed5github.com/xuri/excelize/v2 v2.11.0- github.com/yfedoseev/office_oxide/go v0.1.9+ github.com/yfedoseev/office_oxide/go v0.1.12github.com/yfedoseev/pdf_oxide/go v0.3.73github.com/yuin/goldmark v1.7.1github.com/zeebo/xxh3 v1.0.2diff --git a/go.sum b/go.sumindex ed9fb57165..90696378cb 100644--- a/go.sum+++ b/go.sum@@ -612,8 +612,8 @@ github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 h1:+C0TIdyyYmzadGaL/HBLgithub.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ=github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc=github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA=-github.com/yfedoseev/office_oxide/go v0.1.9 h1:gvTMCvJDYEZNF4dKd1cdfvU+9SRDSpnbNhCpZJbZOaE=-github.com/yfedoseev/office_oxide/go v0.1.9/go.mod h1:YLtMlKUkRCp/Q96wsy7D6yoBKDeJnP66UH+c9Bb+E+M=+github.com/yfedoseev/office_oxide/go v0.1.12 h1:QLsiB8Aj7LB42D9Vwu1rslRuZg/vSXc4IkVXCDKQv6o=+github.com/yfedoseev/office_oxide/go v0.1.12/go.mod h1:YLtMlKUkRCp/Q96wsy7D6yoBKDeJnP66UH+c9Bb+E+M=github.com/yfedoseev/pdf_oxide/go v0.3.73 h1:hRad+X4W0ILW3bCQeCvdUKcoy8yMj2TFS6/XtoMk55I=github.com/yfedoseev/pdf_oxide/go v0.3.73/go.mod h1:QbJ/nLbez0al2EnqEdEPIlGflFprWmiuUM4mo9rNNOI=github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4=diff --git a/internal/parser/parser/doc_parser_synth_test.go b/internal/parser/parser/doc_parser_synth_test.goindex 8c7d1bafed..6f2ae1c0b2 100644--- a/internal/parser/parser/doc_parser_synth_test.go+++ b/internal/parser/parser/doc_parser_synth_test.go@@ -91,6 +91,8 @@ func buildSyntheticDoc(t *testing.T, text string) []byte {copy(dir[1*128:], dirEntry("WordDocument", 2, 2, uint64(len(wordDoc))))copy(dir[2*128:], dirEntry("0Table", 2, 3, uint64(len(clx))))copy(dir[3*128:], dirEntry("", 0, 0xFFFFFFFF, 0))+ binary.LittleEndian.PutUint32(dir[0*128+0x4C:], 1) // root child: WordDocument+ binary.LittleEndian.PutUint32(dir[1*128+0x44:], 2) // left sibling: 0Table// ── Assemble ──────────────────────────────────────────────────────────tableSector := make([]byte, sector)diff --git a/internal/parser/parser/docx_parser_cgo_test.go b/internal/parser/parser/docx_parser_cgo_test.goindex 65ff846f7f..6132e1eff7 100644--- a/internal/parser/parser/docx_parser_cgo_test.go+++ b/internal/parser/parser/docx_parser_cgo_test.go@@ -5,6 +5,7 @@ package parserimport ("archive/zip""bytes"+ "strings""testing")@@ -129,3 +130,60 @@ func writeZipFile(t *testing.T, zw *zip.Writer, name, body string) {t.Fatalf("write zip entry %s: %v", name, err)}}++// TestDOCXParser_PreservesXMLCharacterReferences pins the office_oxide version.+// v0.1.9 deleted every XML character reference in DOCX run text instead of+// decoding it, so <w:t><SEP></w:t> parsed as "SEP". That reached the+// chunker intact, so a delimiter configured as "`<SEP>`" could never match and+// the marker survived into the chunk body. v0.1.10 was the correctness release+// that fixed this defect class; v0.1.12 is the pinned version.+func TestDOCXParser_PreservesXMLCharacterReferences(t *testing.T) {+ cases := []struct {+ name string+ run string // run text as it appears inside <w:t>, entities unescaped+ want string+ wantMarkdown string+ }{+ {"named", "<SEP>", "<SEP>", `\<SEP\>`},+ {"decimal", "<SEP>", "<SEP>", `\<SEP\>`},+ {"hex", "<SEP>", "<SEP>", `\<SEP\>`},+ {"ampersand", "A&B", "A&B", "A&B"},+ {"apostrophe", "it's", "it's", "it's"},+ {"emdash", "a—b", "a—b", "a—b"},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ data := minimalDOCX(t, tc.run)++ jsonParser := NewDOCXParser()+ jsonParser.ConfigureFromSetup(map[string]any{"output_format": "json"})+ jsonRes := jsonParser.ParseWithResult(t.Context(), "entities.docx", data)+ if jsonRes.Err != nil {+ t.Fatalf("ParseWithResult(json): %v", jsonRes.Err)+ }+ if got := joinItemTexts(jsonRes.JSON); !strings.Contains(got, tc.want) {+ t.Errorf("json text = %q, want it to contain %q", got, tc.want)+ }++ mdParser := NewDOCXParser()+ mdParser.ConfigureFromSetup(map[string]any{"output_format": "markdown"})+ mdRes := mdParser.ParseWithResult(t.Context(), "entities.docx", data)+ if mdRes.Err != nil {+ t.Fatalf("ParseWithResult(markdown): %v", mdRes.Err)+ }+ if !strings.Contains(mdRes.Markdown, tc.wantMarkdown) {+ t.Errorf("markdown = %q, want it to contain %q", mdRes.Markdown, tc.wantMarkdown)+ }+ })+ }+}++func joinItemTexts(items []map[string]any) string {+ parts := make([]string, 0, len(items))+ for _, item := range items {+ if text, ok := item["text"].(string); ok {+ parts = append(parts, text)+ }+ }+ return strings.Join(parts, "\n")+}diff --git a/ragflow_deps/download_deps.py b/ragflow_deps/download_deps.pyindex 2a25cf5405..fdb8ad2536 100755--- a/ragflow_deps/download_deps.py+++ b/ragflow_deps/download_deps.py@@ -121,7 +121,7 @@ def get_urls(use_china_mirrors=False) -> list[str | list[str]]:# network access during CI.["https://github.com/kognitos/pdfium-static/releases/download/chromium%2F7809/pdfium-linux-x64-static.tgz", "pdfium-linux-x64-static.tgz"],["https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.73/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"],- ["https://github.com/yfedoseev/office_oxide/releases/download/v0.1.9/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"],+ ["https://github.com/yfedoseev/office_oxide/releases/download/v0.1.12/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"],# ONNX Runtime static archives for the Go in-process (DeepDoc)# backend. Statically linked into the server binary (see build.sh:# ONNXRUNTIME_STATIC_PREFIX — no --whole-archive, so unreferenced@@ -171,7 +171,7 @@ def get_urls(use_china_mirrors=False) -> list[str | list[str]]:# network access during CI.["https://github.com/kognitos/pdfium-static/releases/download/chromium%2F7809/pdfium-linux-x64-static.tgz", "pdfium-linux-x64-static.tgz"],["https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.73/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"],- ["https://github.com/yfedoseev/office_oxide/releases/download/v0.1.9/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"],+ ["https://github.com/yfedoseev/office_oxide/releases/download/v0.1.12/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"],# ONNX Runtime static archives for the Go in-process (DeepDoc)# backend. Statically linked into the server binary (see build.sh:# ONNXRUNTIME_STATIC_PREFIX — no --whole-archive, so unreferenceddiff --git a/ragflow_deps/download_go_deps.py b/ragflow_deps/download_go_deps.pyindex 90d5f1cf81..a23b27b0bf 100755--- a/ragflow_deps/download_go_deps.py+++ b/ragflow_deps/download_go_deps.py@@ -226,7 +226,7 @@ def get_urls(use_china_mirrors=False) -> list[str | list[str]]:# functions — pre-downloaded to avoid network access during CI.["https://gh-proxy.com/https://github.com/kognitos/pdfium-static/releases/download/chromium%2F7809/pdfium-linux-x64-static.tgz", "pdfium-linux-x64-static.tgz"],["https://gh-proxy.com/https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.73/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"],- ["https://gh-proxy.com/https://github.com/yfedoseev/office_oxide/releases/download/v0.1.9/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"],+ ["https://gh-proxy.com/https://github.com/yfedoseev/office_oxide/releases/download/v0.1.12/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"],[f"https://gh-proxy.com/https://github.com/infiniflow/ragflow-build/releases/download/onnxruntime-v{ORT_VERSION}/{_ort_asset_name(ORT_VERSION)}",_ort_asset_name(ORT_VERSION),@@ -255,7 +255,7 @@ def get_urls(use_china_mirrors=False) -> list[str | list[str]]:# functions — pre-downloaded to avoid network access during CI.["https://github.com/kognitos/pdfium-static/releases/download/chromium%2F7809/pdfium-linux-x64-static.tgz", "pdfium-linux-x64-static.tgz"],["https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.73/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"],- ["https://github.com/yfedoseev/office_oxide/releases/download/v0.1.9/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"],+ ["https://github.com/yfedoseev/office_oxide/releases/download/v0.1.12/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"],[f"https://github.com/infiniflow/ragflow-build/releases/download/onnxruntime-v{ORT_VERSION}/{_ort_asset_name(ORT_VERSION)}",_ort_asset_name(ORT_VERSION),