RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): preserve unwrapped Markdown code fences (#19552)
Wenchao An37d88a65c2d3
diff --git a/common/string_utils.py b/common/string_utils.pyindex 5271709b00..fdc513af28 100644--- a/common/string_utils.py+++ b/common/string_utils.py@@ -52,7 +52,7 @@ def clean_markdown_block(text):This function cleans Markdown code blocks by removing:- Opening ```Markdown tags (with optional whitespace and newlines)- - Closing ``` tags (with optional whitespace and newlines)+ - Closing ``` tags only when an opening Markdown tag was removedArgs:text (str): Input text that may be wrapped in Markdown code blocks@@ -63,11 +63,12 @@ def clean_markdown_block(text):"""# Remove opening ```Markdown tag with optional whitespace and newlines# Matches: optional whitespace + ```markdown + optional whitespace + optional newline- text = re.sub(r"^\s*```markdown\s*\n?", "", text)+ text, opening_count = re.subn(r"^\s*```markdown\s*\n?", "", text)# Remove closing ``` tag with optional whitespace and newlines# Matches: optional newline + optional whitespace + ``` + optional whitespace at end- text = re.sub(r"\n?\s*```\s*$", "", text)+ if opening_count:+ text = re.sub(r"\n?\s*```\s*$", "", text)# Return text with surrounding whitespace removedreturn text.strip()diff --git a/internal/ingestion/component/vision_enhancement.go b/internal/ingestion/component/vision_enhancement.goindex 2584ffa853..aa19aff7f6 100644--- a/internal/ingestion/component/vision_enhancement.go+++ b/internal/ingestion/component/vision_enhancement.go@@ -91,9 +91,7 @@ func (s *promptDirState) resolve(requestedRoot string) (string, error) {}// cleanMarkdownBlock mirrors Python common/string_utils.py:clean_markdown_block-//-// re.sub(r"^\s*```markdown\s*\n?", "", text)-// re.sub(r"\n?\s*```\s*$", "", text)+// and removes a closing fence only after recognizing an opening markdown wrapper.//// Matches Python without re.MULTILINE so ^/$ anchor only the whole text.var (@@ -102,8 +100,10 @@ var ()func cleanMarkdownBlock(s string) string {- s = reMarkdownOpen.ReplaceAllString(s, "")- s = reMarkdownClose.ReplaceAllString(s, "")+ if reMarkdownOpen.MatchString(s) {+ s = reMarkdownOpen.ReplaceAllString(s, "")+ s = reMarkdownClose.ReplaceAllString(s, "")+ }return strings.TrimSpace(s)}diff --git a/internal/ingestion/component/vision_enhancement_test.go b/internal/ingestion/component/vision_enhancement_test.goindex 79872116e2..553e38df36 100644--- a/internal/ingestion/component/vision_enhancement_test.go+++ b/internal/ingestion/component/vision_enhancement_test.go@@ -688,6 +688,46 @@ func TestCleanMarkdownBlock_EdgeCases(t *testing.T) {input: "Just plain text without markdown block",want: "Just plain text without markdown block",},+ {+ name: "unwrapped python block",+ input: "```python\nprint('hello')\n```",+ want: "```python\nprint('hello')\n```",+ },+ {+ name: "unwrapped JSON block",+ input: "```json\n{\"value\": 1}\n```",+ want: "```json\n{\"value\": 1}\n```",+ },+ {+ name: "unlabeled code block",+ input: "```\ncode without a language\n```",+ want: "```\ncode without a language\n```",+ },+ {+ name: "prose ending with code block",+ input: "Example:\n```python\nprint('hello')\n```",+ want: "Example:\n```python\nprint('hello')\n```",+ },+ {+ name: "multiple unwrapped code blocks",+ input: "```python\nfirst()\n```\n\n```python\nsecond()\n```",+ want: "```python\nfirst()\n```\n\n```python\nsecond()\n```",+ },+ {+ name: "unwrapped CRLF block",+ input: " ```python\r\nprint('hello')\r\n``` ",+ want: "```python\r\nprint('hello')\r\n```",+ },+ {+ name: "closing fence without markdown opener",+ input: "Unopened block\n```",+ want: "Unopened block\n```",+ },+ {+ name: "markdown wrapper around code example",+ input: "```markdown\nExample:\n```python\nprint('hello')\n```\n```",+ want: "Example:\n```python\nprint('hello')\n```",+ },}for _, tc := range cases {@@ -696,6 +736,10 @@ func TestCleanMarkdownBlock_EdgeCases(t *testing.T) {if got != tc.want {t.Errorf("cleanMarkdownBlock(%q) = %q, want %q", tc.input, got, tc.want)}+ resp := &modelModule.ChatResponse{Answer: &tc.input}+ if got := extractVisionAnswer(resp); got != tc.want {+ t.Errorf("extractVisionAnswer(%q) = %q, want %q", tc.input, got, tc.want)+ }})}}diff --git a/test/unit_test/common/test_string_utils.py b/test/unit_test/common/test_string_utils.pyindex 8833578adb..cb682a53cc 100644--- a/test/unit_test/common/test_string_utils.py+++ b/test/unit_test/common/test_string_utils.py@@ -297,8 +297,9 @@ class TestCleanMarkdownBlock:expected = "Unclosed block"assert clean_markdown_block(input_text) == expected+ # Without a markdown opener, the closing fence may belong to content.input_text = "Unopened block\n```"- expected = "Unopened block"+ expected = "Unopened block\n```"assert clean_markdown_block(input_text) == expecteddef test_mixed_whitespace_characters(self):@@ -354,3 +355,23 @@ class TestCleanMarkdownBlock:input_text = "```markdown\nFirst line\n```\n```markdown\nSecond line\n```"expected = "First line\n```\n```markdown\nSecond line"assert clean_markdown_block(input_text) == expected++ @pytest.mark.parametrize(+ "text",+ [+ "```python\nprint('hello')\n```",+ '```json\n{"value": 1}\n```',+ "```\ncode without a language\n```",+ "An explanation.\n\n```python\nprint('hello')\n```",+ "```python\nfirst()\n```\n\n```python\nsecond()\n```",+ " ```python\r\nprint('hello')\r\n``` ",+ ],+ )+ def test_preserves_code_fences_without_markdown_wrapper(self, text):+ """Preserve code blocks returned directly by a vision model."""+ assert clean_markdown_block(text) == text.strip()++ def test_preserves_code_fences_inside_markdown_wrapper(self):+ """Remove only the outer markdown fences around a code example."""+ content = "Example:\n```python\nprint('hello')\n```"+ assert clean_markdown_block(f"```markdown\n{content}\n```") == contentdiff --git a/test/unit_test/rag/app/test_picture_video.py b/test/unit_test/rag/app/test_picture_video.pyindex e36866893d..3614b6f402 100644--- a/test/unit_test/rag/app/test_picture_video.py+++ b/test/unit_test/rag/app/test_picture_video.py@@ -222,3 +222,27 @@ def test_ocr_only_fallback_is_reported_as_degraded():progressed = [args[0] for args, _kwargs in callback_calls if args]assert 0.8 not in progressed, "the fallback still reports a clean 0.8 success"++[email protected]("wrapped", [False, True])+def test_vision_markdown_preserves_code_fences(wrapped):+ """Keep a model's code example intact when cleaning image-derived text."""+ from common.string_utils import clean_markdown_block++ picture = _load_picture_module([])+ picture.clean_markdown_block = clean_markdown_block+ content = "Example:\n```python\nprint('hello')\n```"+ model_text = f"```markdown\n{content}\n```" if wrapped else content+ calls = []++ def describe(binary, prompt):+ """Check that the real image encoding path reached the model boundary."""+ with Image.open(io.BytesIO(binary)) as decoded:+ assert decoded.size == (64, 64)+ calls.append(prompt)+ return model_text++ with Image.new("RGB", (64, 64), "white") as image:+ text = picture.vision_llm_chunk(image, SimpleNamespace(describe_with_prompt=describe), prompt="Read this image")+ assert text == "\n" + content+ assert calls == ["Read this image"]