Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): preserve unwrapped Markdown code fences (#19552)

Wenchao An37d88a65c2d3
diff --git a/common/string_utils.py b/common/string_utils.py
index 5271709b00..fdc513af28 100644
--- a/common/string_utils.py
+++ b/common/string_utils.py
@@ -52,7 +52,7 @@ def clean_markdown_block(text):
This function cleans Markdown code blocks by removing:
- Opening ```Markdown tags (with optional whitespace and newlines)
- - Closing ``` tags (with optional whitespace and newlines)
+ - Closing ``` tags only when an opening Markdown tag was removed
Args:
text (str): Input text that may be wrapped in Markdown code blocks
@@ -63,11 +63,12 @@ def clean_markdown_block(text):
"""
# Remove opening ```Markdown tag with optional whitespace and newlines
# Matches: optional whitespace + ```markdown + optional whitespace + optional newline
- text = re.sub(r"^\s*```markdown\s*\n?", "", text)
+ text, opening_count = re.subn(r"^\s*```markdown\s*\n?", "", text)
# Remove closing ``` tag with optional whitespace and newlines
# Matches: optional newline + optional whitespace + ``` + optional whitespace at end
- text = re.sub(r"\n?\s*```\s*$", "", text)
+ if opening_count:
+ text = re.sub(r"\n?\s*```\s*$", "", text)
# Return text with surrounding whitespace removed
return text.strip()
diff --git a/internal/ingestion/component/vision_enhancement.go b/internal/ingestion/component/vision_enhancement.go
index 2584ffa853..aa19aff7f6 100644
--- a/internal/ingestion/component/vision_enhancement.go
+++ b/internal/ingestion/component/vision_enhancement.go
@@ -91,9 +91,7 @@ func (s *promptDirState) resolve(requestedRoot string) (string, error) {
}
// cleanMarkdownBlock mirrors Python common/string_utils.py:clean_markdown_block
-//
-// re.sub(r"^\s*```markdown\s*\n?", "", text)
-// re.sub(r"\n?\s*```\s*$", "", text)
+// and removes a closing fence only after recognizing an opening markdown wrapper.
//
// Matches Python without re.MULTILINE so ^/$ anchor only the whole text.
var (
@@ -102,8 +100,10 @@ var (
)
func cleanMarkdownBlock(s string) string {
- s = reMarkdownOpen.ReplaceAllString(s, "")
- s = reMarkdownClose.ReplaceAllString(s, "")
+ if reMarkdownOpen.MatchString(s) {
+ s = reMarkdownOpen.ReplaceAllString(s, "")
+ s = reMarkdownClose.ReplaceAllString(s, "")
+ }
return strings.TrimSpace(s)
}
diff --git a/internal/ingestion/component/vision_enhancement_test.go b/internal/ingestion/component/vision_enhancement_test.go
index 79872116e2..553e38df36 100644
--- a/internal/ingestion/component/vision_enhancement_test.go
+++ b/internal/ingestion/component/vision_enhancement_test.go
@@ -688,6 +688,46 @@ func TestCleanMarkdownBlock_EdgeCases(t *testing.T) {
input: "Just plain text without markdown block",
want: "Just plain text without markdown block",
},
+ {
+ name: "unwrapped python block",
+ input: "```python\nprint('hello')\n```",
+ want: "```python\nprint('hello')\n```",
+ },
+ {
+ name: "unwrapped JSON block",
+ input: "```json\n{\"value\": 1}\n```",
+ want: "```json\n{\"value\": 1}\n```",
+ },
+ {
+ name: "unlabeled code block",
+ input: "```\ncode without a language\n```",
+ want: "```\ncode without a language\n```",
+ },
+ {
+ name: "prose ending with code block",
+ input: "Example:\n```python\nprint('hello')\n```",
+ want: "Example:\n```python\nprint('hello')\n```",
+ },
+ {
+ name: "multiple unwrapped code blocks",
+ input: "```python\nfirst()\n```\n\n```python\nsecond()\n```",
+ want: "```python\nfirst()\n```\n\n```python\nsecond()\n```",
+ },
+ {
+ name: "unwrapped CRLF block",
+ input: " ```python\r\nprint('hello')\r\n``` ",
+ want: "```python\r\nprint('hello')\r\n```",
+ },
+ {
+ name: "closing fence without markdown opener",
+ input: "Unopened block\n```",
+ want: "Unopened block\n```",
+ },
+ {
+ name: "markdown wrapper around code example",
+ input: "```markdown\nExample:\n```python\nprint('hello')\n```\n```",
+ want: "Example:\n```python\nprint('hello')\n```",
+ },
}
for _, tc := range cases {
@@ -696,6 +736,10 @@ func TestCleanMarkdownBlock_EdgeCases(t *testing.T) {
if got != tc.want {
t.Errorf("cleanMarkdownBlock(%q) = %q, want %q", tc.input, got, tc.want)
}
+ resp := &modelModule.ChatResponse{Answer: &tc.input}
+ if got := extractVisionAnswer(resp); got != tc.want {
+ t.Errorf("extractVisionAnswer(%q) = %q, want %q", tc.input, got, tc.want)
+ }
})
}
}
diff --git a/test/unit_test/common/test_string_utils.py b/test/unit_test/common/test_string_utils.py
index 8833578adb..cb682a53cc 100644
--- a/test/unit_test/common/test_string_utils.py
+++ b/test/unit_test/common/test_string_utils.py
@@ -297,8 +297,9 @@ class TestCleanMarkdownBlock:
expected = "Unclosed block"
assert clean_markdown_block(input_text) == expected
+ # Without a markdown opener, the closing fence may belong to content.
input_text = "Unopened block\n```"
- expected = "Unopened block"
+ expected = "Unopened block\n```"
assert clean_markdown_block(input_text) == expected
def test_mixed_whitespace_characters(self):
@@ -354,3 +355,23 @@ class TestCleanMarkdownBlock:
input_text = "```markdown\nFirst line\n```\n```markdown\nSecond line\n```"
expected = "First line\n```\n```markdown\nSecond line"
assert clean_markdown_block(input_text) == expected
+
+ @pytest.mark.parametrize(
+ "text",
+ [
+ "```python\nprint('hello')\n```",
+ '```json\n{"value": 1}\n```',
+ "```\ncode without a language\n```",
+ "An explanation.\n\n```python\nprint('hello')\n```",
+ "```python\nfirst()\n```\n\n```python\nsecond()\n```",
+ " ```python\r\nprint('hello')\r\n``` ",
+ ],
+ )
+ def test_preserves_code_fences_without_markdown_wrapper(self, text):
+ """Preserve code blocks returned directly by a vision model."""
+ assert clean_markdown_block(text) == text.strip()
+
+ def test_preserves_code_fences_inside_markdown_wrapper(self):
+ """Remove only the outer markdown fences around a code example."""
+ content = "Example:\n```python\nprint('hello')\n```"
+ assert clean_markdown_block(f"```markdown\n{content}\n```") == content
diff --git a/test/unit_test/rag/app/test_picture_video.py b/test/unit_test/rag/app/test_picture_video.py
index e36866893d..3614b6f402 100644
--- a/test/unit_test/rag/app/test_picture_video.py
+++ b/test/unit_test/rag/app/test_picture_video.py
@@ -222,3 +222,27 @@ def test_ocr_only_fallback_is_reported_as_degraded():
progressed = [args[0] for args, _kwargs in callback_calls if args]
assert 0.8 not in progressed, "the fallback still reports a clean 0.8 success"
+
+
[email protected]("wrapped", [False, True])
+def test_vision_markdown_preserves_code_fences(wrapped):
+ """Keep a model's code example intact when cleaning image-derived text."""
+ from common.string_utils import clean_markdown_block
+
+ picture = _load_picture_module([])
+ picture.clean_markdown_block = clean_markdown_block
+ content = "Example:\n```python\nprint('hello')\n```"
+ model_text = f"```markdown\n{content}\n```" if wrapped else content
+ calls = []
+
+ def describe(binary, prompt):
+ """Check that the real image encoding path reached the model boundary."""
+ with Image.open(io.BytesIO(binary)) as decoded:
+ assert decoded.size == (64, 64)
+ calls.append(prompt)
+ return model_text
+
+ with Image.new("RGB", (64, 64), "white") as image:
+ text = picture.vision_llm_chunk(image, SimpleNamespace(describe_with_prompt=describe), prompt="Read this image")
+ assert text == "\n" + content
+ assert calls == ["Read this image"]