Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(docreader): prevent XLSX chunking timeouts (#4683)

杰尼龟8a52858d8771
diff --git a/docreader/parser/markitdown_parser.py b/docreader/parser/markitdown_parser.py
index a1576ef52..df1694d3f 100644
--- a/docreader/parser/markitdown_parser.py
+++ b/docreader/parser/markitdown_parser.py
@@ -1,5 +1,6 @@
import io
import logging
+import time
from markitdown import MarkItDown
@@ -44,13 +45,15 @@ class StdMarkitdownParser(BaseParser):
ext = self.file_type
ft = (ext or "").lstrip(".").lower()
pptx_bytes: bytes | None = None
+ preprocess_started = time.monotonic()
+ xlsx = ft in ("xlsx", "xlsm")
if ft in ("ppt", "pptx"):
content, ext = normalize_ppt_bytes(content, ft)
pptx_bytes = content
ft = "pptx"
elif ft == "docx":
content = fill_vertical_merged_cells_docx(content)
- elif ft in ("xlsx", "xlsm"):
+ elif xlsx:
# MarkItDown reads spreadsheets through pandas/openpyxl, so the
# ranges openpyxl cannot parse (#3599) and the non-conforming
# styles.xml fills (#3637) that break the builtin ExcelParser break
@@ -63,18 +66,73 @@ class StdMarkitdownParser(BaseParser):
sanitized = sanitize_xlsx_styles(content)
if sanitized is not None:
content = sanitized
+ if xlsx:
+ logger.info(
+ "MarkItDown XLSX preprocessing completed in %.3fs (bytes=%d)",
+ time.monotonic() - preprocess_started,
+ len(content),
+ )
if ext and not ext.startswith("."):
ext = "." + ext
+ # Embedded data URIs can make spreadsheet output and subsequent chunking
+ # disproportionately expensive. XLSX conversion therefore starts with
+ # them disabled; all other formats retain the historical image-first
+ # behavior and its single bounded fallback.
+ first_keep_data_uris = not xlsx
with parser_worker_limit("markitdown", CONFIG.markitdown_max_workers):
- result = self._convert_markitdown(content, ext, keep_data_uris=True)
- if result is None:
+ started = time.monotonic()
+ try:
+ result = self._convert_markitdown(
+ content, ext, keep_data_uris=first_keep_data_uris
+ )
+ except Exception as exc:
+ logger.exception(
+ "MarkItDown conversion failed (format=%s bytes=%d keep_data_uris=%s)",
+ ft or ext,
+ len(content),
+ first_keep_data_uris,
+ )
+ raise RuntimeError(
+ f"MarkItDown conversion failed for {ft or ext or 'document'}"
+ ) from exc
+ logger.info(
+ "MarkItDown conversion completed in %.3fs (format=%s bytes=%d text=%d keep_data_uris=%s)",
+ time.monotonic() - started,
+ ft or ext,
+ len(content),
+ len(result.text_content) if result is not None else 0,
+ first_keep_data_uris,
+ )
+ if result is None and first_keep_data_uris:
logger.warning(
"MarkItDown failed with embedded images for %s; retrying without data URIs",
ft or ext,
)
- result = self._convert_markitdown(content, ext, keep_data_uris=False)
+ started = time.monotonic()
+ try:
+ result = self._convert_markitdown(
+ content, ext, keep_data_uris=False
+ )
+ except Exception as exc:
+ logger.exception(
+ "MarkItDown fallback failed (format=%s bytes=%d)",
+ ft or ext,
+ len(content),
+ )
+ raise RuntimeError(
+ f"MarkItDown conversion failed for {ft or ext or 'document'}"
+ ) from exc
+ logger.info(
+ "MarkItDown fallback completed in %.3fs (format=%s bytes=%d text=%d keep_data_uris=false)",
+ time.monotonic() - started,
+ ft or ext,
+ len(content),
+ len(result.text_content) if result is not None else 0,
+ )
+ if result is None:
+ raise RuntimeError(f"MarkItDown conversion failed for {ft or ext or 'document'}")
text = result.text_content
images: dict[str, str] = {}
if pptx_bytes is not None and markdown_needs_pptx_media_attach(text):
diff --git a/docreader/tests/test_docx_merge.py b/docreader/tests/test_docx_merge.py
index a86f901b8..504285a40 100644
--- a/docreader/tests/test_docx_merge.py
+++ b/docreader/tests/test_docx_merge.py
@@ -120,6 +120,52 @@ class DocxVerticalMergeFillTest(unittest.TestCase):
class DocxMarkitdownDispatchTest(unittest.TestCase):
+ def test_xlsx_starts_without_embedded_data_uris(self):
+ parser = StdMarkitdownParser(file_type="xlsx")
+ result = SimpleNamespace(text_content="parsed")
+ with patch.object(parser, "_convert_markitdown", return_value=result) as convert:
+ document = parser.parse_into_text(b"payload")
+
+ self.assertEqual(document.content, "parsed")
+ convert.assert_called_once_with(b"payload", ".xlsx", keep_data_uris=False)
+
+ def test_xlsx_conversion_failure_is_not_retried_with_data_uris(self):
+ parser = StdMarkitdownParser(file_type="xlsx")
+ with patch.object(parser, "_convert_markitdown", return_value=None) as convert:
+ with self.assertRaisesRegex(RuntimeError, "MarkItDown conversion failed"):
+ parser.parse_into_text(b"payload")
+
+ convert.assert_called_once_with(b"payload", ".xlsx", keep_data_uris=False)
+
+ def test_xlsx_conversion_exception_is_wrapped_without_retry(self):
+ parser = StdMarkitdownParser(file_type="xlsx")
+ with patch.object(
+ parser,
+ "_convert_markitdown",
+ side_effect=ValueError("synthetic parser failure"),
+ ) as convert:
+ with self.assertRaisesRegex(RuntimeError, "MarkItDown conversion failed"):
+ parser.parse_into_text(b"payload")
+
+ convert.assert_called_once_with(b"payload", ".xlsx", keep_data_uris=False)
+
+ def test_non_xlsx_parser_has_one_bounded_fallback(self):
+ parser = StdMarkitdownParser(file_type="docx")
+ result = SimpleNamespace(text_content="parsed")
+ with patch.object(
+ parser, "_convert_markitdown", side_effect=[None, result]
+ ) as convert:
+ document = parser.parse_into_text(b"payload")
+
+ self.assertEqual(document.content, "parsed")
+ self.assertEqual(convert.call_count, 2)
+ self.assertEqual(
+ convert.call_args_list[0].kwargs["keep_data_uris"], True
+ )
+ self.assertEqual(
+ convert.call_args_list[1].kwargs["keep_data_uris"], False
+ )
+
def test_non_docx_types_do_not_run_docx_preprocessor(self):
for file_type in ("doc", "txt", "md"):
with self.subTest(file_type=file_type):
diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go
index f8d26796b..fa309944e 100644
--- a/internal/application/service/knowledge_process.go
+++ b/internal/application/service/knowledge_process.go
@@ -4135,12 +4135,19 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
}
}
sanitizeReadResult(convertResult)
+ normalizeStarted := time.Now()
if convertResult != nil {
convertResult.MarkdownContent = chunker.NormalizeLineEndings(convertResult.MarkdownContent)
if convertResult.MarkdownContent != "" {
convertResult.MarkdownContent = docparser.NormalizeHTMLTables(convertResult.MarkdownContent)
}
}
+ normalizedBytes := 0
+ if convertResult != nil {
+ normalizedBytes = len(convertResult.MarkdownContent)
+ }
+ logger.Infof(ctx, "Document text normalization completed in %s (knowledge=%s input_bytes=%d output_bytes=%d)",
+ time.Since(normalizeStarted), knowledge.ID, len(parsedMarkdown), normalizedBytes)
// Step 2: Store images and update markdown references
var storedImages []docparser.StoredImage
@@ -4213,6 +4220,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
// Step 3: Split into chunks using Go chunker. Line endings and inline
// HTML tables were normalized before image resolution above.
chunkCfg := buildSplitterConfigFromChunking(eff.ChunkingConfig)
+ chunkStarted := time.Now()
multimodalImages := imagesForMultimodal(storedImages)
processOpts := ProcessChunksOptions{
@@ -4266,6 +4274,8 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
}
logger.Infof(ctx, "Split document into %d chunks for knowledge %s", len(chunks), knowledge.ID)
}
+ logger.Infof(ctx, "Document chunking completed in %s (knowledge=%s input_bytes=%d chunks=%d)",
+ time.Since(chunkStarted), knowledge.ID, len(convertResult.MarkdownContent), len(chunks))
// Step 4: Process chunks (vectorize + index + enqueue async tasks)
return s.processChunks(ctx, kb, knowledge, chunks, processOpts)