A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix: more edge cases in codeblock stripping (#7511)
Myles Scolnick1614a2a65c0a
diff --git a/marimo/_server/ai/providers.py b/marimo/_server/ai/providers.pyindex 111969135b..8743e825a3 100644--- a/marimo/_server/ai/providers.py+++ b/marimo/_server/ai/providers.py@@ -1248,22 +1248,31 @@ async def merge_backticks() -> AsyncGenerator[str, None]:buffer: Optional[str] = None+ def only_whitespace_or_newlines(text: str) -> bool:+ return all(char.isspace() or char == "\n" for char in text)+async for chunk in chunks:if buffer is None:buffer = chunk- else:- # If buffer contains backticks, keep merging until we have no backticks,- # encounter a newline, or run out of chunks- if "`" in buffer:- buffer += chunk- # If we've hit a newline or no more backticks, yield the buffer- if "\n" in chunk or "`" not in buffer:- yield buffer- buffer = None- else:- # No backticks in buffer, yield it separately+ continue++ # Combine whitespace+ if only_whitespace_or_newlines(buffer):+ buffer += chunk+ continue++ # If buffer contains backticks, keep merging until we have no backticks,+ # encounter a newline, or run out of chunks+ if "`" in buffer:+ buffer += chunk+ # If we've hit a newline or no more backticks, yield the buffer+ if "\n" in chunk or "`" not in buffer:yield buffer- buffer = chunk+ buffer = None+ else:+ # No backticks in buffer, yield it separately+ yield buffer+ buffer = chunk# Return the last chunk if there's anything leftif buffer is not None:@@ -1276,17 +1285,21 @@ async def without_wrapping_backticks("""Removes the first and last backticks (```) from a stream of text chunks.+ This function removes opening backticks (with optional language identifier)+ from the start of the stream and closing backticks from the end of the stream.+ It does not remove backticks that appear in the middle of the content.+Args:chunks: An async iterator of text chunksYields:Text chunks with the first and last backticks removed if they exist"""-- # First, merge backticks across chunks+ # First, merge backticks across chunks to avoid split patternschunks = merge_backticks(chunks)- langs = ["python", "sql"]+ # Supported language identifiers+ langs = ["python", "sql", "markdown"]first_chunk = Truebuffer: Optional[str] = None@@ -1330,8 +1343,9 @@ async def without_wrapping_backticks(trailing_space = buffer[len(stripped_buffer) :]# Remove ending newline if present- if stripped_buffer.endswith("\n```"):- buffer = stripped_buffer[:-4] + trailing_space- elif has_starting_backticks and stripped_buffer.endswith("```"):- buffer = stripped_buffer[:-3] + trailing_space+ if has_starting_backticks:+ if stripped_buffer.endswith("\n```"):+ buffer = stripped_buffer[:-4] + trailing_space+ elif stripped_buffer.endswith("```"):+ buffer = stripped_buffer[:-3] + trailing_spaceyield bufferdiff --git a/tests/_server/api/endpoints/test_ai.py b/tests/_server/api/endpoints/test_ai.pyindex 0009a09da6..dcb78e57d6 100644--- a/tests/_server/api/endpoints/test_ai.py+++ b/tests/_server/api/endpoints/test_ai.py@@ -974,6 +974,7 @@ class TestGetFinishReason(unittest.TestCase):@pytest.mark.parametrize(("chunks", "expected"),[+ # Basic cases(["Hello", " world", "!"], "Hello world!"),(["```", "print('hello')", "print('world')"],@@ -994,7 +995,7 @@ class TestGetFinishReason(unittest.TestCase):),(["``", "`", "\n", "print('hello')", "\n", "``", "`"],- "print('hello')\n",+ "print('hello')",),(["```\n", "print('hello')", "print('world')", "\n```"],@@ -1053,6 +1054,22 @@ class TestGetFinishReason(unittest.TestCase):["\t```", "code", "```"],"code",),+ (+ ["\n", "\n", "```\n", "code", "\n```\n"],+ "code\n",+ ),+ (+ ["\n", "\n", "```python\n", "code", "\n```\n"],+ "code\n",+ ),+ (+ ["\n``", "`python\n", "code", "\n```\n"],+ "code\n",+ ),+ (+ ["\n`", "`", "`python\n", "code", "\n```\n"],+ "code\n",+ ),# Test opening backticks with extra characters after language (language stripped, rest preserved)(["```python ", "code", "```"],@@ -1062,6 +1079,52 @@ class TestGetFinishReason(unittest.TestCase):["```python\t", "code", "```"],"\tcode",),+ # Empty code block+ (["```\n", "```"], ""),+ (["```python\n", "```"], ""),+ # Only opening backticks+ (["```\n", "code"], "code"),+ (["```python\n", "code"], "code"),+ # Only closing backticks+ (["code", "\n```"], "code\n```"),+ # Multiple consecutive code blocks (keeps middle fences)+ (+ ["```\n", "x = 1\n", "```\n", "```\n", "y = 2\n", "```"],+ "x = 1\n```\n```\ny = 2\n",+ ),+ # Code block with inline backticks in content+ (+ ["```python\n", "s = 'use `backticks`'\n", "```"],+ "s = 'use `backticks`'\n",+ ),+ # Backticks split across multiple chunks+ (["``", "`\n", "code\n", "``", "`"], "code\n"),+ # Language identifier split from backticks+ (["```", "python\n", "code\n", "```"], "code\n"),+ # No newline after opening backticks+ (["```", "code", "```"], "code"),+ # Text before opening backticks (not at start, so backticks kept)+ (["prefix ", "```\n", "code\n", "```"], "prefix ```\ncode\n```"),+ # Text after closing backticks (backticks kept because followed by text)+ (["```\n", "code\n", "```", " suffix"], "code\n``` suffix"),+ # Multiple newlines+ (["```\n\n", "code\n\n", "```"], "\ncode\n\n"),+ # Whitespace handling (preserved inside code block)+ (["```\n", " code \n", "```"], " code \n"),+ # Tab characters (preserved inside code block)+ (["```\n", "\tcode\t\n", "```"], "\tcode\t\n"),+ # Mixed opening styles in same stream (middle ```python\n is kept)+ (+ ["```\n", "x\n", "```\n", "```python\n", "y\n", "```"],+ "x\n```\n```python\ny\n",+ ),+ # Unsupported language identifier (should keep it as text)+ (+ ["```javascript\n", "console.log()\n", "```"],+ "javascript\nconsole.log()\n",+ ),+ # Markdown language identifier (supported)+ (["```markdown\n", "# Title\n", "```"], "# Title\n"),],)async def test_without_wrapping_backticks(