Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix: preserve multiline string indentation in cell loader (#9877)

ChidiebereNjoku1f1bb633df7e
diff --git a/marimo/_ast/cell_manager.py b/marimo/_ast/cell_manager.py
index e3c02bcbb8..06b7e508ea 100644
--- a/marimo/_ast/cell_manager.py
+++ b/marimo/_ast/cell_manager.py
@@ -19,9 +19,9 @@ from marimo._ast.compiler import (
ir_cell_factory,
toplevel_cell_factory,
)
+from marimo._ast.dedent import fixed_dedent
from marimo._ast.models import CellData
from marimo._ast.names import DEFAULT_CELL_NAME, SETUP_CELL_NAME
-from marimo._ast.parse import fixed_dedent
from marimo._ast.pytest import process_for_pytest
from marimo._messaging.notebook.changes import (
CreateCell,
diff --git a/marimo/_ast/compiler.py b/marimo/_ast/compiler.py
index de2b836b5d..93061a9c9a 100644
--- a/marimo/_ast/compiler.py
+++ b/marimo/_ast/compiler.py
@@ -24,6 +24,7 @@ from marimo._ast.cell import (
ImportWorkspace,
SourcePosition,
)
+from marimo._ast.dedent import smart_dedent
from marimo._ast.names import SETUP_CELL_NAME, TOPLEVEL_CELL_PREFIX
from marimo._ast.pytest import has_fixture_decorator
from marimo._ast.transformers import ContainedExtractWithBlock
@@ -528,7 +529,7 @@ def context_cell_factory(
end_line = end_node.end_lineno
if start_node == end_node and lines[end_line - 1].strip() == "pass":
end_line -= 1
- cell_code = textwrap.dedent("\n".join(lines[entry_line:end_line])).rstrip()
+ cell_code = smart_dedent("\n".join(lines[entry_line:end_line])).rstrip()
source_position = None
if not anonymous_file:
@@ -561,7 +562,7 @@ def toplevel_cell_factory(
definition. As such, signature and return type are important.
"""
code, lnum = inspect.getsourcelines(obj)
- function_code = textwrap.dedent("".join(code))
+ function_code = smart_dedent("".join(code))
# We need to scrub through the initial decorator. Since we don't care about
# indentation etc, easiest just to use AST.
@@ -570,9 +571,7 @@ def toplevel_cell_factory(
try:
decorator = tree.body[0].decorator_list.pop(0) # type: ignore
# NB. We don't unparse from the AST because it strips comments.
- cell_code = textwrap.dedent(
- "".join(code[decorator.end_lineno :])
- ).strip()
+ cell_code = smart_dedent("".join(code[decorator.end_lineno :])).strip()
except (IndexError, AttributeError) as e:
raise ValueError(
"Unexpected usage (expected decorated function)"
@@ -645,7 +644,7 @@ def cell_factory(
signature, marimo will autofix them on save.
"""
code, lnum = inspect.getsourcelines(f)
- function_code = textwrap.dedent("".join(code))
+ function_code = smart_dedent("".join(code))
extractor = parse.Extractor(contents=function_code)
func_ast = parse.ast_parse(function_code).body[0]
diff --git a/marimo/_ast/dedent.py b/marimo/_ast/dedent.py
new file mode 100644
index 0000000000..cb55ac7475
--- /dev/null
+++ b/marimo/_ast/dedent.py
@@ -0,0 +1,201 @@
+# Copyright 2026 Marimo. All rights reserved.
+"""Token-aware dedent that preserves multiline string literal whitespace."""
+
+from __future__ import annotations
+
+import io
+import re
+import textwrap
+import tokenize
+from tokenize import TokenError
+
+_NEWLINE_RE = re.compile(r"\r\n|\r|\n")
+
+
+def split_source_lines(text: str, keepends: bool = False) -> list[str]:
+ """Split source into lines the way `ast`/`tokenize` count them.
+
+ Unlike `str.splitlines()`, this only treats `\n`, `\r`, and `\r\n` as
+ line breaks. `str.splitlines()` additionally splits on `\f`, `\v`, the
+ `\x1c`-`\x1e` separators, and Unicode line separators.
+
+ When `keepends` is `True`, line terminators are preserved on each line so
+ that `"".join(...)` reconstructs `text` exactly. The element count matches
+ the `keepends=False` result, so the two can be indexed against each other.
+ """
+ if not keepends:
+ return _NEWLINE_RE.sub("\n", text).split("\n")
+
+ lines: list[str] = []
+ start = 0
+ for match in _NEWLINE_RE.finditer(text):
+ lines.append(text[start : match.end()])
+ start = match.end()
+ lines.append(text[start:])
+ return lines
+
+
+def _get_protected_lines(
+ code: str, tokens: list[tokenize.TokenInfo]
+) -> list[bool]:
+ """Return a bool list marking lines inside multiline string literals.
+
+ Handles both regular triple-quoted strings (STRING token) and f-strings
+ (FSTRING_START/FSTRING_MIDDLE/FSTRING_END tokens in Python 3.12+).
+ """
+ lines = split_source_lines(code)
+ n = len(lines)
+ protected = [False] * n
+
+ # Token type values for f-string tokens (Python 3.12+)
+ FSTRING_START = getattr(tokenize, "FSTRING_START", None)
+ FSTRING_END = getattr(tokenize, "FSTRING_END", None)
+
+ fstring_start_line: int | None = None
+
+ for tok_type, tok_string, tok_start, tok_end, _ in tokens:
+ # Regular triple-quoted strings: single STRING token spanning lines
+ if tok_type == tokenize.STRING and _NEWLINE_RE.search(tok_string):
+ start_line = tok_start[0] - 1
+ end_line = tok_end[0] - 1
+ for i in range(start_line + 1, end_line + 1):
+ if i < n:
+ protected[i] = True
+
+ # Python 3.12+ f-strings: track FSTRING_START to FSTRING_END span
+ elif FSTRING_START and tok_type == FSTRING_START:
+ fstring_start_line = tok_start[0] - 1
+
+ elif FSTRING_END and tok_type == FSTRING_END:
+ if fstring_start_line is not None:
+ end_line = tok_end[0] - 1
+ for i in range(fstring_start_line + 1, end_line + 1):
+ if i < n:
+ protected[i] = True
+ fstring_start_line = None
+
+ return protected
+
+
+def _strip_indent(line: str, shift: int) -> str:
+ """Remove up to `shift` leading spaces/tabs.
+
+ Never touches the line's content or terminator, so blank lines shorter
+ than `shift` keep their newline instead of being swallowed.
+ """
+ i = 0
+ while i < shift and i < len(line) and line[i] in " \t":
+ i += 1
+ return line[i:]
+
+
+def _under_indented_string_lines(
+ lines: list[str], protected: list[bool], base_shift: int
+) -> list[bool]:
+ """Mark protected lines belonging to an under-indented string block.
+
+ A multiline string's interior is dedented with the surrounding code
+ (the common case, equivalent to `textwrap.dedent`) unless the block is
+ under-indented — i.e. some non-blank interior line has less leading
+ whitespace than `base_shift`. Dedenting such a block would eat into its
+ content, so the whole block is left untouched instead.
+ """
+ n = len(lines)
+ keep = [False] * n
+ i = 0
+ while i < n:
+ if not protected[i]:
+ i += 1
+ continue
+ j = i
+ block_min: float = float("inf")
+ while j < n and protected[j]:
+ line = lines[j]
+ if line.strip():
+ block_min = min(block_min, len(line) - len(line.lstrip()))
+ j += 1
+ if block_min != float("inf") and block_min < base_shift:
+ for k in range(i, j):
+ keep[k] = True
+ i = j
+ return keep
+
+
+def smart_dedent(code: str) -> str:
+ """Token-aware dedent.
+
+ Strips base indentation (computed from code lines only) from every line.
+ In the common case this is identical to `textwrap.dedent`. The exception
+ is a multiline string whose interior is under-indented relative to the
+ surrounding code: dedenting it would eat into the string's content, so its
+ interior is left completely untouched.
+ """
+ try:
+ tokens = list(tokenize.generate_tokens(io.StringIO(code).readline))
+ except TokenError:
+ return textwrap.dedent(code)
+
+ lines = split_source_lines(code, keepends=True)
+ protected = _get_protected_lines(code, tokens)
+
+ min_indent: float = float("inf")
+ for i, line in enumerate(lines):
+ if protected[i] or not line.strip():
+ continue
+ min_indent = min(min_indent, len(line) - len(line.lstrip()))
+
+ if min_indent in (0, float("inf")):
+ return code
+
+ base_shift = int(min_indent)
+ keep = _under_indented_string_lines(lines, protected, base_shift)
+
+ return "".join(
+ line if keep[i] else _strip_indent(line, base_shift)
+ for i, line in enumerate(lines)
+ )
+
+
+def fixed_dedent(text: str) -> str:
+ """Dedent with robustness for AI-generated code with inconsistent indentation.
+
+ Unlike textwrap.dedent, this pads lines that are under-indented to the
+ base indentation level before dedenting, while preserving whitespace
+ inside multiline string literals whose internal structure doesn't match
+ the surrounding code's indentation.
+ """
+ lines = split_source_lines(text)
+
+ try:
+ tokens = list(tokenize.generate_tokens(io.StringIO(text).readline))
+ except TokenError:
+ tokens = []
+
+ protected = _get_protected_lines(text, tokens)
+
+ # Mirror str.splitlines(): a final line terminator yields a trailing ""
+ # that we drop, so the result carries no spurious trailing newline. (The
+ # trailing line is never inside a string, so `protected` stays aligned.)
+ n = len(lines)
+ if n > 1 and lines[-1] == "":
+ n -= 1
+
+ indent = None
+ for i in range(n):
+ if protected[i]:
+ continue
+ if content := lines[i].lstrip():
+ indent = lines[i][: len(lines[i]) - len(content)]
+ break
+
+ if indent is None:
+ # No code line to key off of (all-blank input): fall back to a plain
+ # dedent, matching textwrap's normalization of whitespace-only lines.
+ return textwrap.dedent(text)
+
+ def refill(i: int, ln: str) -> str:
+ if protected[i] or ln.startswith(indent):
+ return ln
+ return indent + ln
+
+ return smart_dedent("\n".join(refill(i, lines[i]) for i in range(n)))
diff --git a/marimo/_ast/parse.py b/marimo/_ast/parse.py
index b7169126c2..4668fe54a7 100644
--- a/marimo/_ast/parse.py
+++ b/marimo/_ast/parse.py
@@ -6,7 +6,6 @@ import io
import token as token_types
import warnings
from pathlib import Path
-from textwrap import dedent
from tokenize import TokenInfo, tokenize
from typing import (
TYPE_CHECKING,
@@ -16,6 +15,7 @@ from typing import (
cast,
)
+from marimo._ast.dedent import fixed_dedent
from marimo._ast.names import DEFAULT_CELL_NAME, SETUP_CELL_NAME
from marimo._schemas.serialization import (
AppInstantiation,
@@ -64,26 +64,6 @@ def split_source_lines(text: str) -> list[str]:
return text.replace("\r\n", "\n").replace("\r", "\n").split("\n")
-def fixed_dedent(text: str) -> str:
- """Manually edited code, can dedent"""
- # Added robustness for AI generated code
- lines = text.splitlines()
- for line in lines:
- if content := line.lstrip():
- indent = line[: len(line) - len(content)]
- break
- else:
- # Quit early, no clear leading spaces
- return dedent(text)
-
- def refill(line: str) -> str:
- if not line.startswith(indent):
- return indent + line
- return line
-
- return dedent("\n".join(map(refill, lines)))
-
-
def unwrap_cell_body(formatted: str) -> str:
"""Extract the body of a wrapped `def _():` cell after ruff formatting.
diff --git a/marimo/_ast/scanner.py b/marimo/_ast/scanner.py
index 8fcc040f43..dae9592c9b 100644
--- a/marimo/_ast/scanner.py
+++ b/marimo/_ast/scanner.py
@@ -400,7 +400,7 @@ def _extract_body_code(cell_source: str, kind: str) -> str:
Strips decorator lines, def/class/with header, and trailing return.
Used when a cell fails ast.parse() to get the body for UnparsableCell.
"""
- from marimo._ast.parse import fixed_dedent
+ from marimo._ast.dedent import fixed_dedent
if kind == "unparsable":
# Already app._unparsable_cell(...) — return as-is
diff --git a/tests/_ast/test_dedent.py b/tests/_ast/test_dedent.py
new file mode 100644
index 0000000000..429213852e
--- /dev/null
+++ b/tests/_ast/test_dedent.py
@@ -0,0 +1,157 @@
+"""Tests for token-aware smart_dedent and fixed_dedent."""
+
+from __future__ import annotations
+
+from marimo._ast.dedent import (
+ fixed_dedent,
+ smart_dedent,
+ split_source_lines,
+)
+
+
+class TestSplitSourceLines:
+ def test_default_drops_terminators(self):
+ assert split_source_lines("a\nb\n") == ["a", "b", ""]
+
+ def test_normalizes_carriage_returns(self):
+ assert split_source_lines("a\r\nb\rc") == ["a", "b", "c"]
+
+ def test_keepends_roundtrips(self):
+ for text in ["a\n b\n", "a\r\nb", "a\rb\n", "x = 1", "", "a\n\nb"]:
+ assert "".join(split_source_lines(text, keepends=True)) == text
+
+ def test_keepends_preserves_terminators(self):
+ assert split_source_lines("a\r\nb\n", keepends=True) == [
+ "a\r\n",
+ "b\n",
+ "",
+ ]
+
+ def test_keepends_element_count_matches_default(self):
+ for text in ["a\nb\n", "a\r\nb", "a\rb\n", "x = 1", "", "a\n\nb"]:
+ assert len(split_source_lines(text, keepends=True)) == len(
+ split_source_lines(text)
+ )
+
+
+class TestSmartDedent:
+ def test_basic_dedent(self):
+ assert smart_dedent(" x = 1\n y = 2\n") == "x = 1\ny = 2\n"
+
+ def test_no_indent_noop(self):
+ code = "x = 1\ny = 2\n"
+ assert smart_dedent(code) == code
+
+ def test_multiline_string_interior_preserved(self):
+ code = ' x = """line0\n two_spaces\n four_spaces\n six_spaces"""\n'
+ result = smart_dedent(code)
+ assert (
+ result
+ == 'x = """line0\n two_spaces\n four_spaces\n six_spaces"""\n'
+ )
+
+ def test_fstring_interior_preserved(self):
+ code = ' name = "world"\n x = f"""hello\n indented line\n more indented"""\n'
+ result = smart_dedent(code)
+ assert " indented line" in result
+ assert " more indented" in result
+ assert result.startswith('name = "world"')
+
+ def test_raw_string_interior_preserved(self):
+ code = ' x = r"""line0\n raw content\n more raw"""\n'
+ result = smart_dedent(code)
+ assert " raw content" in result
+ assert " more raw" in result
+
+ def test_docstring_content_at_column_zero(self):
+ code = ' def f():\n """Cell to compute sum\ne.g. a=1; b=2\nfoo\n"""\n return a + b\n'
+ result = smart_dedent(code)
+ assert "e.g. a=1; b=2" in result
+ assert result.startswith("def f():")
+
+ def test_single_line_string_not_protected(self):
+ code = " x = 'hello world'\n y = 2\n"
+ assert smart_dedent(code) == "x = 'hello world'\ny = 2\n"
+
+ def test_mixed_code_and_multiline_string_deeper_indent(self):
+ # String block indented MORE than surrounding code (block_min >=
+ # base_shift) participates in the same dedent, preserving its own
+ # internal alignment between SELECT and FROM.
+ code = ' def f():\n sql = """\n SELECT *\n FROM t\n """\n return sql\n'
+ result = smart_dedent(code)
+ assert result.startswith("def f():")
+ assert " SELECT *" in result
+ assert " FROM t" in result
+
+ def test_multiline_string_under_indented_relative_to_code(self):
+ # String block indented LESS than the surrounding code in places
+ # (block_min < base_shift) is left fully untouched, since it can't
+ # safely absorb the same dedent without losing structure.
+ code = ' def f():\n text = """\n shallow\n deep\n """\n return text\n'
+ result = smart_dedent(code)
+ assert result.startswith("def f():")
+ assert " shallow" in result
+ assert " deep" in result
+
+ def test_top_level_function_with_multiline_docstring(self):
+ # Already at zero indent — no-op
+ code = '@app.function\ndef foo():\n """I wonder what\nHappens to this ?\n"""\n return ...\n'
+ result = smart_dedent(code)
+ assert result == code
+
+ def test_indented_top_level_function_with_docstring(self):
+ code = ' @app.function\n def foo():\n """Also wonder what\nHappens to that ?\n"""\n return ...\n'
+ result = smart_dedent(code)
+ assert result.startswith("@app.function")
+ assert "Happens to that ?" in result
+
+ def test_under_indented_docstring_preserved(self):
+ # Docstring under-indented relative to the code (a line with only two
+ # spaces) is left completely untouched; only the code is dedented.
+ code = (
+ ' """Cell to compute sum\n'
+ " e.g. a=1; b=2\n"
+ " foo\n"
+ ' """\n'
+ " a + b\n"
+ )
+ assert smart_dedent(code) == (
+ '"""Cell to compute sum\n'
+ " e.g. a=1; b=2\n"
+ " foo\n"
+ ' """\n'
+ "a + b\n"
+ )
+
+ def test_blank_line_in_docstring_preserved(self):
+ # A blank line inside an in-step docstring keeps its newline instead of
+ # being swallowed (the line block is still dedented with the code).
+ code = ' """hello\n\n world\n """\n x = 1\n'
+ assert smart_dedent(code) == '"""hello\n\nworld\n"""\nx = 1\n'
+
+
+class TestFixedDedent:
+ def test_basic(self):
+ assert fixed_dedent(" x = 1\n y = 2") == "x = 1\ny = 2"
+
+ def test_inconsistent_indentation(self):
+ result = fixed_dedent(" x = 1\ny = 2")
+ assert result == "x = 1\ny = 2"
+
+ def test_multiline_string_preserved(self):
+ code = ' x = """line0\n two_spaces\n four_spaces"""\n'
+ result = fixed_dedent(code)
+ assert " two_spaces" in result
+ assert " four_spaces" in result
+
+ def test_fstring_preserved(self):
+ code = ' x = f"""hello\n indented\n more"""\n'
+ result = fixed_dedent(code)
+ assert " indented" in result
+ assert " more" in result
+
+ def test_no_spurious_trailing_newline(self):
+ # A trailing newline must not survive into the result (it would add a
+ # spurious blank line before the auto-generated `return` on save).
+ assert fixed_dedent(" import x\n") == "import x"
+ assert fixed_dedent(" x = 1\n y = 2\n") == "x = 1\ny = 2"