Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(docreader): prevent XLSX chunking timeouts (#4683)
杰尼龟8a52858d8771
diff --git a/docreader/parser/markitdown_parser.py b/docreader/parser/markitdown_parser.pyindex a1576ef52..df1694d3f 100644--- a/docreader/parser/markitdown_parser.py+++ b/docreader/parser/markitdown_parser.py@@ -1,5 +1,6 @@import ioimport logging+import timefrom markitdown import MarkItDown@@ -44,13 +45,15 @@ class StdMarkitdownParser(BaseParser):ext = self.file_typeft = (ext or "").lstrip(".").lower()pptx_bytes: bytes | None = None+ preprocess_started = time.monotonic()+ xlsx = ft in ("xlsx", "xlsm")if ft in ("ppt", "pptx"):content, ext = normalize_ppt_bytes(content, ft)pptx_bytes = contentft = "pptx"elif ft == "docx":content = fill_vertical_merged_cells_docx(content)- elif ft in ("xlsx", "xlsm"):+ elif xlsx:# MarkItDown reads spreadsheets through pandas/openpyxl, so the# ranges openpyxl cannot parse (#3599) and the non-conforming# styles.xml fills (#3637) that break the builtin ExcelParser break@@ -63,18 +66,73 @@ class StdMarkitdownParser(BaseParser):sanitized = sanitize_xlsx_styles(content)if sanitized is not None:content = sanitized+ if xlsx:+ logger.info(+ "MarkItDown XLSX preprocessing completed in %.3fs (bytes=%d)",+ time.monotonic() - preprocess_started,+ len(content),+ )if ext and not ext.startswith("."):ext = "." + ext+ # Embedded data URIs can make spreadsheet output and subsequent chunking+ # disproportionately expensive. XLSX conversion therefore starts with+ # them disabled; all other formats retain the historical image-first+ # behavior and its single bounded fallback.+ first_keep_data_uris = not xlsxwith parser_worker_limit("markitdown", CONFIG.markitdown_max_workers):- result = self._convert_markitdown(content, ext, keep_data_uris=True)- if result is None:+ started = time.monotonic()+ try:+ result = self._convert_markitdown(+ content, ext, keep_data_uris=first_keep_data_uris+ )+ except Exception as exc:+ logger.exception(+ "MarkItDown conversion failed (format=%s bytes=%d keep_data_uris=%s)",+ ft or ext,+ len(content),+ first_keep_data_uris,+ )+ raise RuntimeError(+ f"MarkItDown conversion failed for {ft or ext or 'document'}"+ ) from exc+ logger.info(+ "MarkItDown conversion completed in %.3fs (format=%s bytes=%d text=%d keep_data_uris=%s)",+ time.monotonic() - started,+ ft or ext,+ len(content),+ len(result.text_content) if result is not None else 0,+ first_keep_data_uris,+ )+ if result is None and first_keep_data_uris:logger.warning("MarkItDown failed with embedded images for %s; retrying without data URIs",ft or ext,)- result = self._convert_markitdown(content, ext, keep_data_uris=False)+ started = time.monotonic()+ try:+ result = self._convert_markitdown(+ content, ext, keep_data_uris=False+ )+ except Exception as exc:+ logger.exception(+ "MarkItDown fallback failed (format=%s bytes=%d)",+ ft or ext,+ len(content),+ )+ raise RuntimeError(+ f"MarkItDown conversion failed for {ft or ext or 'document'}"+ ) from exc+ logger.info(+ "MarkItDown fallback completed in %.3fs (format=%s bytes=%d text=%d keep_data_uris=false)",+ time.monotonic() - started,+ ft or ext,+ len(content),+ len(result.text_content) if result is not None else 0,+ )+ if result is None:+ raise RuntimeError(f"MarkItDown conversion failed for {ft or ext or 'document'}")text = result.text_contentimages: dict[str, str] = {}if pptx_bytes is not None and markdown_needs_pptx_media_attach(text):diff --git a/docreader/tests/test_docx_merge.py b/docreader/tests/test_docx_merge.pyindex a86f901b8..504285a40 100644--- a/docreader/tests/test_docx_merge.py+++ b/docreader/tests/test_docx_merge.py@@ -120,6 +120,52 @@ class DocxVerticalMergeFillTest(unittest.TestCase):class DocxMarkitdownDispatchTest(unittest.TestCase):+ def test_xlsx_starts_without_embedded_data_uris(self):+ parser = StdMarkitdownParser(file_type="xlsx")+ result = SimpleNamespace(text_content="parsed")+ with patch.object(parser, "_convert_markitdown", return_value=result) as convert:+ document = parser.parse_into_text(b"payload")++ self.assertEqual(document.content, "parsed")+ convert.assert_called_once_with(b"payload", ".xlsx", keep_data_uris=False)++ def test_xlsx_conversion_failure_is_not_retried_with_data_uris(self):+ parser = StdMarkitdownParser(file_type="xlsx")+ with patch.object(parser, "_convert_markitdown", return_value=None) as convert:+ with self.assertRaisesRegex(RuntimeError, "MarkItDown conversion failed"):+ parser.parse_into_text(b"payload")++ convert.assert_called_once_with(b"payload", ".xlsx", keep_data_uris=False)++ def test_xlsx_conversion_exception_is_wrapped_without_retry(self):+ parser = StdMarkitdownParser(file_type="xlsx")+ with patch.object(+ parser,+ "_convert_markitdown",+ side_effect=ValueError("synthetic parser failure"),+ ) as convert:+ with self.assertRaisesRegex(RuntimeError, "MarkItDown conversion failed"):+ parser.parse_into_text(b"payload")++ convert.assert_called_once_with(b"payload", ".xlsx", keep_data_uris=False)++ def test_non_xlsx_parser_has_one_bounded_fallback(self):+ parser = StdMarkitdownParser(file_type="docx")+ result = SimpleNamespace(text_content="parsed")+ with patch.object(+ parser, "_convert_markitdown", side_effect=[None, result]+ ) as convert:+ document = parser.parse_into_text(b"payload")++ self.assertEqual(document.content, "parsed")+ self.assertEqual(convert.call_count, 2)+ self.assertEqual(+ convert.call_args_list[0].kwargs["keep_data_uris"], True+ )+ self.assertEqual(+ convert.call_args_list[1].kwargs["keep_data_uris"], False+ )+def test_non_docx_types_do_not_run_docx_preprocessor(self):for file_type in ("doc", "txt", "md"):with self.subTest(file_type=file_type):diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.goindex f8d26796b..fa309944e 100644--- a/internal/application/service/knowledge_process.go+++ b/internal/application/service/knowledge_process.go@@ -4135,12 +4135,19 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e}}sanitizeReadResult(convertResult)+ normalizeStarted := time.Now()if convertResult != nil {convertResult.MarkdownContent = chunker.NormalizeLineEndings(convertResult.MarkdownContent)if convertResult.MarkdownContent != "" {convertResult.MarkdownContent = docparser.NormalizeHTMLTables(convertResult.MarkdownContent)}}+ normalizedBytes := 0+ if convertResult != nil {+ normalizedBytes = len(convertResult.MarkdownContent)+ }+ logger.Infof(ctx, "Document text normalization completed in %s (knowledge=%s input_bytes=%d output_bytes=%d)",+ time.Since(normalizeStarted), knowledge.ID, len(parsedMarkdown), normalizedBytes)// Step 2: Store images and update markdown referencesvar storedImages []docparser.StoredImage@@ -4213,6 +4220,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e// Step 3: Split into chunks using Go chunker. Line endings and inline// HTML tables were normalized before image resolution above.chunkCfg := buildSplitterConfigFromChunking(eff.ChunkingConfig)+ chunkStarted := time.Now()multimodalImages := imagesForMultimodal(storedImages)processOpts := ProcessChunksOptions{@@ -4266,6 +4274,8 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e}logger.Infof(ctx, "Split document into %d chunks for knowledge %s", len(chunks), knowledge.ID)}+ logger.Infof(ctx, "Document chunking completed in %s (knowledge=%s input_bytes=%d chunks=%d)",+ time.Since(chunkStarted), knowledge.ID, len(convertResult.MarkdownContent), len(chunks))// Step 4: Process chunks (vectorize + index + enqueue async tasks)return s.processChunks(ctx, kb, knowledge, chunks, processOpts)