RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix the One and Book chunkers when Tika extracts nothing from a .doc (#19692)
JessYanc0976fe0480a
diff --git a/rag/app/book.py b/rag/app/book.pyindex 5eb82e4a52..98fca04741 100644--- a/rag/app/book.py+++ b/rag/app/book.py@@ -166,6 +166,11 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=sections = [(line, "") for line in sections if line]remove_contents_table(sections, eng=is_english(random_choices([t for t, _ in sections], k=200)))callback(0.8, "Finish parsing.")+ else:+ error_msg = f"tika.parser got empty content from {filename}."+ callback(0.8, error_msg)+ logging.warning(error_msg)+ return []else:raise NotImplementedError("file type not supported yet(doc, docx, pdf, txt supported)")diff --git a/rag/app/one.py b/rag/app/one.pyindex 11c4bbf66a..4b5c7042b6 100644--- a/rag/app/one.py+++ b/rag/app/one.py@@ -176,7 +176,12 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=if doc_parsed.get("content", None) is not None:sections = doc_parsed["content"].split("\n")sections = [s for s in sections if s]- callback(0.8, "Finish parsing.")+ callback(0.8, "Finish parsing.")+ else:+ error_msg = f"tika.parser got empty content from {filename}."+ callback(0.8, error_msg)+ logging.warning(error_msg)+ return []else:raise NotImplementedError("file type not supported yet(doc, docx, pdf, txt supported)")diff --git a/test/unit_test/rag/app/test_doc_tika_empty_content.py b/test/unit_test/rag/app/test_doc_tika_empty_content.pynew file mode 100644index 0000000000..db7a52c76d--- /dev/null+++ b/test/unit_test/rag/app/test_doc_tika_empty_content.py@@ -0,0 +1,146 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#++"""`tika.parser.from_buffer` answers with `content=None` for a `.doc` it cannot+extract any text from: an empty or unreadable body, an encrypted document, or a+Tika server that answers with nothing.++#3564 added the guard for that to `naive.py`, closing #3229+("'NoneType' object has no attribute 'split'"). #11807 carried it into+`laws.py`. #12073 carried it into `one.py` and `book.py` but kept only its+`if content is not None:` half, so those two have been falling through ever+since.++Every `rag/app` chunker whose `.doc` branch goes through tika is parametrized+here; `manual.py` has a `.doc` branch too but rejects the format outright.+`laws` and `naive` are the control group: they already behave correctly, so they+must stay green both before and after the fix.+"""++from __future__ import annotations++import importlib+import sys++import pytest++# A display name, not a path on disk, plus the OLE2 magic a real .doc starts with.+DOC_NAME = "legacy.doc"+DOC_BINARY = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"+EMPTY_CONTENT_MESSAGE = f"tika.parser got empty content from {DOC_NAME}."++DOC_MODULES = ["book", "laws", "naive", "one"]+++class _Callback:+ """Records what the chunker reports, the way `task_executor` does."""++ def __init__(self):+ self.messages: list[str] = []++ def __call__(self, prog=None, msg=""):+ if msg:+ self.messages.append(msg)+++def _load(module_name: str):+ """Import a `rag.app` module lazily, inside the test that needs it."""+ return importlib.import_module(f"rag.app.{module_name}")+++def _deepdoc_and_chunker_modules():+ """The `sys.modules` keys this module swaps out, as a fresh list."""+ return [name for name in list(sys.modules) if name in ("deepdoc", "rag.app") or name.startswith(("deepdoc.", "rag.app."))]++[email protected](scope="module", autouse=True)+def _real_pdf_parser():+ """Swap the `deepdoc.parser.pdf_parser` stub for the real package, then put+ the stub back.++ `test/unit_test/rag/conftest.py` installs that stub at conftest import time+ so the tests that only need `rag.nlp` do not pull in deepdoc. The chunkers+ here import `deepdoc.parser` for real, which fails against the stub, so drop+ it and let the import machinery find the package again.++ The `rag.app` modules go with it: a copy built against the stub subclasses+ the stub's `RAGFlowPdfParser`, so leaving behind a copy that derives from+ the other `deepdoc` would make a later `monkeypatch.setattr` on+ `deepdoc.parser.PdfParser` land on a class that copy no longer inherits+ from. Restoring on the way out — including the `app` attribute on the `rag`+ package, which `sys.modules` alone does not cover — leaves the session+ exactly as this module found it.+ """+ saved_modules = {name: sys.modules[name] for name in _deepdoc_and_chunker_modules()}+ rag_package = sys.modules.get("rag")+ unset = object()+ saved_app_attr = getattr(rag_package, "app", unset) if rag_package is not None else unset++ for name in _deepdoc_and_chunker_modules():+ del sys.modules[name]+ importlib.invalidate_caches()+ importlib.import_module("deepdoc.parser.pdf_parser")++ yield++ for name in _deepdoc_and_chunker_modules():+ del sys.modules[name]+ sys.modules.update(saved_modules)+ if rag_package is not None:+ if saved_app_attr is unset:+ rag_package.__dict__.pop("app", None)+ else:+ rag_package.app = saved_app_attr+ importlib.invalidate_caches()+++def tika_content(monkeypatch):+ """Drive `from_buffer`'s return value without a Tika server or a JVM."""+ import tika.parser++ def _set(content):+ monkeypatch.setattr(tika.parser, "from_buffer", lambda *args, **kwargs: {"content": content, "metadata": {}, "status": 200})++ return _set++[email protected]("module_name", DOC_MODULES)+def test_doc_without_tika_content_reports_and_returns_nothing(module_name, tika_content, caplog):+ """No crash, no chunks, and the user is told why."""+ tika_content(None)+ callback = _Callback()++ with caplog.at_level("WARNING"):+ chunks = _load(module_name).chunk(DOC_NAME, binary=DOC_BINARY, callback=callback)++ assert chunks == []+ assert EMPTY_CONTENT_MESSAGE in callback.messages+ # Reporting success and then failing is what made this hard to diagnose.+ assert "Finish parsing." not in callback.messages+ assert EMPTY_CONTENT_MESSAGE in caplog.text++[email protected]("module_name", DOC_MODULES)+def test_doc_with_tika_content_is_unaffected(module_name, tika_content):+ """The guard must not cost the path that does have content."""+ tika_content("First paragraph.\nSecond paragraph.\nThird paragraph.\n")+ callback = _Callback()++ chunks = _load(module_name).chunk(DOC_NAME, binary=DOC_BINARY, callback=callback)++ assert chunks+ assert "Finish parsing." in callback.messages