Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix: more edge cases in codeblock stripping (#7511)

Myles Scolnick1614a2a65c0a
diff --git a/marimo/_server/ai/providers.py b/marimo/_server/ai/providers.py
index 111969135b..8743e825a3 100644
--- a/marimo/_server/ai/providers.py
+++ b/marimo/_server/ai/providers.py
@@ -1248,22 +1248,31 @@ async def merge_backticks(
) -> AsyncGenerator[str, None]:
buffer: Optional[str] = None
+ def only_whitespace_or_newlines(text: str) -> bool:
+ return all(char.isspace() or char == "\n" for char in text)
+
async for chunk in chunks:
if buffer is None:
buffer = chunk
- else:
- # If buffer contains backticks, keep merging until we have no backticks,
- # encounter a newline, or run out of chunks
- if "`" in buffer:
- buffer += chunk
- # If we've hit a newline or no more backticks, yield the buffer
- if "\n" in chunk or "`" not in buffer:
- yield buffer
- buffer = None
- else:
- # No backticks in buffer, yield it separately
+ continue
+
+ # Combine whitespace
+ if only_whitespace_or_newlines(buffer):
+ buffer += chunk
+ continue
+
+ # If buffer contains backticks, keep merging until we have no backticks,
+ # encounter a newline, or run out of chunks
+ if "`" in buffer:
+ buffer += chunk
+ # If we've hit a newline or no more backticks, yield the buffer
+ if "\n" in chunk or "`" not in buffer:
yield buffer
- buffer = chunk
+ buffer = None
+ else:
+ # No backticks in buffer, yield it separately
+ yield buffer
+ buffer = chunk
# Return the last chunk if there's anything left
if buffer is not None:
@@ -1276,17 +1285,21 @@ async def without_wrapping_backticks(
"""
Removes the first and last backticks (```) from a stream of text chunks.
+ This function removes opening backticks (with optional language identifier)
+ from the start of the stream and closing backticks from the end of the stream.
+ It does not remove backticks that appear in the middle of the content.
+
Args:
chunks: An async iterator of text chunks
Yields:
Text chunks with the first and last backticks removed if they exist
"""
-
- # First, merge backticks across chunks
+ # First, merge backticks across chunks to avoid split patterns
chunks = merge_backticks(chunks)
- langs = ["python", "sql"]
+ # Supported language identifiers
+ langs = ["python", "sql", "markdown"]
first_chunk = True
buffer: Optional[str] = None
@@ -1330,8 +1343,9 @@ async def without_wrapping_backticks(
trailing_space = buffer[len(stripped_buffer) :]
# Remove ending newline if present
- if stripped_buffer.endswith("\n```"):
- buffer = stripped_buffer[:-4] + trailing_space
- elif has_starting_backticks and stripped_buffer.endswith("```"):
- buffer = stripped_buffer[:-3] + trailing_space
+ if has_starting_backticks:
+ if stripped_buffer.endswith("\n```"):
+ buffer = stripped_buffer[:-4] + trailing_space
+ elif stripped_buffer.endswith("```"):
+ buffer = stripped_buffer[:-3] + trailing_space
yield buffer
diff --git a/tests/_server/api/endpoints/test_ai.py b/tests/_server/api/endpoints/test_ai.py
index 0009a09da6..dcb78e57d6 100644
--- a/tests/_server/api/endpoints/test_ai.py
+++ b/tests/_server/api/endpoints/test_ai.py
@@ -974,6 +974,7 @@ class TestGetFinishReason(unittest.TestCase):
@pytest.mark.parametrize(
("chunks", "expected"),
[
+ # Basic cases
(["Hello", " world", "!"], "Hello world!"),
(
["```", "print('hello')", "print('world')"],
@@ -994,7 +995,7 @@ class TestGetFinishReason(unittest.TestCase):
),
(
["``", "`", "\n", "print('hello')", "\n", "``", "`"],
- "print('hello')\n",
+ "print('hello')",
),
(
["```\n", "print('hello')", "print('world')", "\n```"],
@@ -1053,6 +1054,22 @@ class TestGetFinishReason(unittest.TestCase):
["\t```", "code", "```"],
"code",
),
+ (
+ ["\n", "\n", "```\n", "code", "\n```\n"],
+ "code\n",
+ ),
+ (
+ ["\n", "\n", "```python\n", "code", "\n```\n"],
+ "code\n",
+ ),
+ (
+ ["\n``", "`python\n", "code", "\n```\n"],
+ "code\n",
+ ),
+ (
+ ["\n`", "`", "`python\n", "code", "\n```\n"],
+ "code\n",
+ ),
# Test opening backticks with extra characters after language (language stripped, rest preserved)
(
["```python ", "code", "```"],
@@ -1062,6 +1079,52 @@ class TestGetFinishReason(unittest.TestCase):
["```python\t", "code", "```"],
"\tcode",
),
+ # Empty code block
+ (["```\n", "```"], ""),
+ (["```python\n", "```"], ""),
+ # Only opening backticks
+ (["```\n", "code"], "code"),
+ (["```python\n", "code"], "code"),
+ # Only closing backticks
+ (["code", "\n```"], "code\n```"),
+ # Multiple consecutive code blocks (keeps middle fences)
+ (
+ ["```\n", "x = 1\n", "```\n", "```\n", "y = 2\n", "```"],
+ "x = 1\n```\n```\ny = 2\n",
+ ),
+ # Code block with inline backticks in content
+ (
+ ["```python\n", "s = 'use `backticks`'\n", "```"],
+ "s = 'use `backticks`'\n",
+ ),
+ # Backticks split across multiple chunks
+ (["``", "`\n", "code\n", "``", "`"], "code\n"),
+ # Language identifier split from backticks
+ (["```", "python\n", "code\n", "```"], "code\n"),
+ # No newline after opening backticks
+ (["```", "code", "```"], "code"),
+ # Text before opening backticks (not at start, so backticks kept)
+ (["prefix ", "```\n", "code\n", "```"], "prefix ```\ncode\n```"),
+ # Text after closing backticks (backticks kept because followed by text)
+ (["```\n", "code\n", "```", " suffix"], "code\n``` suffix"),
+ # Multiple newlines
+ (["```\n\n", "code\n\n", "```"], "\ncode\n\n"),
+ # Whitespace handling (preserved inside code block)
+ (["```\n", " code \n", "```"], " code \n"),
+ # Tab characters (preserved inside code block)
+ (["```\n", "\tcode\t\n", "```"], "\tcode\t\n"),
+ # Mixed opening styles in same stream (middle ```python\n is kept)
+ (
+ ["```\n", "x\n", "```\n", "```python\n", "y\n", "```"],
+ "x\n```\n```python\ny\n",
+ ),
+ # Unsupported language identifier (should keep it as text)
+ (
+ ["```javascript\n", "console.log()\n", "```"],
+ "javascript\nconsole.log()\n",
+ ),
+ # Markdown language identifier (supported)
+ (["```markdown\n", "# Title\n", "```"], "# Title\n"),
],
)
async def test_without_wrapping_backticks(