Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix(api): preserve headings inside fenced Markdown (#43434)
lorenzozanedaba8dc38198
diff --git a/api/core/rag/extractor/markdown_extractor.py b/api/core/rag/extractor/markdown_extractor.pyindex 662da9db27f..195fe56ff1c 100644--- a/api/core/rag/extractor/markdown_extractor.py+++ b/api/core/rag/extractor/markdown_extractor.py@@ -57,16 +57,30 @@ class MarkdownExtractor(BaseExtractor):current_header = Nonecurrent_text = ""- code_block_flag = False+ fence_marker = None+ fence_length = 0for line in lines:- if line.startswith("```"):- code_block_flag = not code_block_flag- current_text += line + "\n"- continue- if code_block_flag:+ fence_match = re.match(r"^ {0,3}(`{3,}|~{3,})(.*)$", line)+ if fence_marker is not None:+ if (+ fence_match+ and fence_match.group(1)[0] == fence_marker+ and len(fence_match.group(1)) >= fence_length+ and re.fullmatch(r"[ \t]*", fence_match.group(2))+ ):+ fence_marker = None+ fence_length = 0current_text += line + "\n"continue+ if fence_match:+ marker = fence_match.group(1)+ info = fence_match.group(2)+ if marker[0] == "~" or "`" not in info:+ fence_marker = marker[0]+ fence_length = len(marker)+ current_text += line + "\n"+ continueheader_match = re.match(r"^#+\s", line)if header_match:markdown_tups.append((current_header, current_text))diff --git a/api/tests/unit_tests/core/rag/extractor/test_markdown_extractor.py b/api/tests/unit_tests/core/rag/extractor/test_markdown_extractor.pyindex 2ca4c3b0e07..8f2a5e2cf75 100644--- a/api/tests/unit_tests/core/rag/extractor/test_markdown_extractor.py+++ b/api/tests/unit_tests/core/rag/extractor/test_markdown_extractor.py@@ -47,6 +47,37 @@ afterassert tups[1][0] == "Header"assert "# this is not a heading" in tups[1][1]+ @pytest.mark.parametrize(+ "fence",+ [+ "~~~python\n# example comment\n~~~~ \t",+ " ```python\n# example comment\n ```",+ "````python\n```\n# example comment\n````",+ "~~~python\n```\n# example comment\n~~~",+ ],+ )+ def test_extract_ignores_headings_inside_fenced_code(self, fence: str, tmp_path: Path):+ markdown = f"# Intro\nbefore\n{fence}\n# Next\nafter"+ markdown_file = tmp_path / "example.md"+ markdown_file.write_text(markdown, encoding="utf-8")++ documents = MarkdownExtractor(+ file_path=str(markdown_file), encoding="utf-8", autodetect_encoding=False+ ).extract()+ nonempty_documents = [document.page_content for document in documents if document.page_content]++ assert len(nonempty_documents) == 2+ assert nonempty_documents[0].startswith("\n\nIntro\n")+ assert "# example comment" in nonempty_documents[0]+ assert nonempty_documents[1].startswith("\n\nNext\n")++ def test_markdown_to_tups_does_not_open_backtick_fence_with_backtick_in_info(self):+ markdown = "# Intro\n```py`thon\n# Next\nafter"++ tups = MarkdownExtractor(file_path="dummy_path").markdown_to_tups(markdown)++ assert [header for header, _ in tups if header is not None] == ["Intro", "Next"]+def test_remove_images_and_hyperlinks(self):extractor = MarkdownExtractor(file_path="dummy_path")