Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(api): preserve headings inside fenced Markdown (#43434)

lorenzozanedaba8dc38198
diff --git a/api/core/rag/extractor/markdown_extractor.py b/api/core/rag/extractor/markdown_extractor.py
index 662da9db27f..195fe56ff1c 100644
--- a/api/core/rag/extractor/markdown_extractor.py
+++ b/api/core/rag/extractor/markdown_extractor.py
@@ -57,16 +57,30 @@ class MarkdownExtractor(BaseExtractor):
current_header = None
current_text = ""
- code_block_flag = False
+ fence_marker = None
+ fence_length = 0
for line in lines:
- if line.startswith("```"):
- code_block_flag = not code_block_flag
- current_text += line + "\n"
- continue
- if code_block_flag:
+ fence_match = re.match(r"^ {0,3}(`{3,}|~{3,})(.*)$", line)
+ if fence_marker is not None:
+ if (
+ fence_match
+ and fence_match.group(1)[0] == fence_marker
+ and len(fence_match.group(1)) >= fence_length
+ and re.fullmatch(r"[ \t]*", fence_match.group(2))
+ ):
+ fence_marker = None
+ fence_length = 0
current_text += line + "\n"
continue
+ if fence_match:
+ marker = fence_match.group(1)
+ info = fence_match.group(2)
+ if marker[0] == "~" or "`" not in info:
+ fence_marker = marker[0]
+ fence_length = len(marker)
+ current_text += line + "\n"
+ continue
header_match = re.match(r"^#+\s", line)
if header_match:
markdown_tups.append((current_header, current_text))
diff --git a/api/tests/unit_tests/core/rag/extractor/test_markdown_extractor.py b/api/tests/unit_tests/core/rag/extractor/test_markdown_extractor.py
index 2ca4c3b0e07..8f2a5e2cf75 100644
--- a/api/tests/unit_tests/core/rag/extractor/test_markdown_extractor.py
+++ b/api/tests/unit_tests/core/rag/extractor/test_markdown_extractor.py
@@ -47,6 +47,37 @@ after
assert tups[1][0] == "Header"
assert "# this is not a heading" in tups[1][1]
+ @pytest.mark.parametrize(
+ "fence",
+ [
+ "~~~python\n# example comment\n~~~~ \t",
+ " ```python\n# example comment\n ```",
+ "````python\n```\n# example comment\n````",
+ "~~~python\n```\n# example comment\n~~~",
+ ],
+ )
+ def test_extract_ignores_headings_inside_fenced_code(self, fence: str, tmp_path: Path):
+ markdown = f"# Intro\nbefore\n{fence}\n# Next\nafter"
+ markdown_file = tmp_path / "example.md"
+ markdown_file.write_text(markdown, encoding="utf-8")
+
+ documents = MarkdownExtractor(
+ file_path=str(markdown_file), encoding="utf-8", autodetect_encoding=False
+ ).extract()
+ nonempty_documents = [document.page_content for document in documents if document.page_content]
+
+ assert len(nonempty_documents) == 2
+ assert nonempty_documents[0].startswith("\n\nIntro\n")
+ assert "# example comment" in nonempty_documents[0]
+ assert nonempty_documents[1].startswith("\n\nNext\n")
+
+ def test_markdown_to_tups_does_not_open_backtick_fence_with_backtick_in_info(self):
+ markdown = "# Intro\n```py`thon\n# Next\nafter"
+
+ tups = MarkdownExtractor(file_path="dummy_path").markdown_to_tups(markdown)
+
+ assert [header for header, _ in tups if header is not None] == ["Intro", "Next"]
+
def test_remove_images_and_hyperlinks(self):
extractor = MarkdownExtractor(file_path="dummy_path")