A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix: preserve multiline string indentation in cell loader (#9877)
ChidiebereNjoku1f1bb633df7e
diff --git a/marimo/_ast/cell_manager.py b/marimo/_ast/cell_manager.pyindex e3c02bcbb8..06b7e508ea 100644--- a/marimo/_ast/cell_manager.py+++ b/marimo/_ast/cell_manager.py@@ -19,9 +19,9 @@ from marimo._ast.compiler import (ir_cell_factory,toplevel_cell_factory,)+from marimo._ast.dedent import fixed_dedentfrom marimo._ast.models import CellDatafrom marimo._ast.names import DEFAULT_CELL_NAME, SETUP_CELL_NAME-from marimo._ast.parse import fixed_dedentfrom marimo._ast.pytest import process_for_pytestfrom marimo._messaging.notebook.changes import (CreateCell,diff --git a/marimo/_ast/compiler.py b/marimo/_ast/compiler.pyindex de2b836b5d..93061a9c9a 100644--- a/marimo/_ast/compiler.py+++ b/marimo/_ast/compiler.py@@ -24,6 +24,7 @@ from marimo._ast.cell import (ImportWorkspace,SourcePosition,)+from marimo._ast.dedent import smart_dedentfrom marimo._ast.names import SETUP_CELL_NAME, TOPLEVEL_CELL_PREFIXfrom marimo._ast.pytest import has_fixture_decoratorfrom marimo._ast.transformers import ContainedExtractWithBlock@@ -528,7 +529,7 @@ def context_cell_factory(end_line = end_node.end_linenoif start_node == end_node and lines[end_line - 1].strip() == "pass":end_line -= 1- cell_code = textwrap.dedent("\n".join(lines[entry_line:end_line])).rstrip()+ cell_code = smart_dedent("\n".join(lines[entry_line:end_line])).rstrip()source_position = Noneif not anonymous_file:@@ -561,7 +562,7 @@ def toplevel_cell_factory(definition. As such, signature and return type are important."""code, lnum = inspect.getsourcelines(obj)- function_code = textwrap.dedent("".join(code))+ function_code = smart_dedent("".join(code))# We need to scrub through the initial decorator. Since we don't care about# indentation etc, easiest just to use AST.@@ -570,9 +571,7 @@ def toplevel_cell_factory(try:decorator = tree.body[0].decorator_list.pop(0) # type: ignore# NB. We don't unparse from the AST because it strips comments.- cell_code = textwrap.dedent(- "".join(code[decorator.end_lineno :])- ).strip()+ cell_code = smart_dedent("".join(code[decorator.end_lineno :])).strip()except (IndexError, AttributeError) as e:raise ValueError("Unexpected usage (expected decorated function)"@@ -645,7 +644,7 @@ def cell_factory(signature, marimo will autofix them on save."""code, lnum = inspect.getsourcelines(f)- function_code = textwrap.dedent("".join(code))+ function_code = smart_dedent("".join(code))extractor = parse.Extractor(contents=function_code)func_ast = parse.ast_parse(function_code).body[0]diff --git a/marimo/_ast/dedent.py b/marimo/_ast/dedent.pynew file mode 100644index 0000000000..cb55ac7475--- /dev/null+++ b/marimo/_ast/dedent.py@@ -0,0 +1,201 @@+# Copyright 2026 Marimo. All rights reserved.+"""Token-aware dedent that preserves multiline string literal whitespace."""++from __future__ import annotations++import io+import re+import textwrap+import tokenize+from tokenize import TokenError++_NEWLINE_RE = re.compile(r"\r\n|\r|\n")+++def split_source_lines(text: str, keepends: bool = False) -> list[str]:+ """Split source into lines the way `ast`/`tokenize` count them.++ Unlike `str.splitlines()`, this only treats `\n`, `\r`, and `\r\n` as+ line breaks. `str.splitlines()` additionally splits on `\f`, `\v`, the+ `\x1c`-`\x1e` separators, and Unicode line separators.++ When `keepends` is `True`, line terminators are preserved on each line so+ that `"".join(...)` reconstructs `text` exactly. The element count matches+ the `keepends=False` result, so the two can be indexed against each other.+ """+ if not keepends:+ return _NEWLINE_RE.sub("\n", text).split("\n")++ lines: list[str] = []+ start = 0+ for match in _NEWLINE_RE.finditer(text):+ lines.append(text[start : match.end()])+ start = match.end()+ lines.append(text[start:])+ return lines+++def _get_protected_lines(+ code: str, tokens: list[tokenize.TokenInfo]+) -> list[bool]:+ """Return a bool list marking lines inside multiline string literals.++ Handles both regular triple-quoted strings (STRING token) and f-strings+ (FSTRING_START/FSTRING_MIDDLE/FSTRING_END tokens in Python 3.12+).+ """+ lines = split_source_lines(code)+ n = len(lines)+ protected = [False] * n++ # Token type values for f-string tokens (Python 3.12+)+ FSTRING_START = getattr(tokenize, "FSTRING_START", None)+ FSTRING_END = getattr(tokenize, "FSTRING_END", None)++ fstring_start_line: int | None = None++ for tok_type, tok_string, tok_start, tok_end, _ in tokens:+ # Regular triple-quoted strings: single STRING token spanning lines+ if tok_type == tokenize.STRING and _NEWLINE_RE.search(tok_string):+ start_line = tok_start[0] - 1+ end_line = tok_end[0] - 1+ for i in range(start_line + 1, end_line + 1):+ if i < n:+ protected[i] = True++ # Python 3.12+ f-strings: track FSTRING_START to FSTRING_END span+ elif FSTRING_START and tok_type == FSTRING_START:+ fstring_start_line = tok_start[0] - 1++ elif FSTRING_END and tok_type == FSTRING_END:+ if fstring_start_line is not None:+ end_line = tok_end[0] - 1+ for i in range(fstring_start_line + 1, end_line + 1):+ if i < n:+ protected[i] = True+ fstring_start_line = None++ return protected+++def _strip_indent(line: str, shift: int) -> str:+ """Remove up to `shift` leading spaces/tabs.++ Never touches the line's content or terminator, so blank lines shorter+ than `shift` keep their newline instead of being swallowed.+ """+ i = 0+ while i < shift and i < len(line) and line[i] in " \t":+ i += 1+ return line[i:]+++def _under_indented_string_lines(+ lines: list[str], protected: list[bool], base_shift: int+) -> list[bool]:+ """Mark protected lines belonging to an under-indented string block.++ A multiline string's interior is dedented with the surrounding code+ (the common case, equivalent to `textwrap.dedent`) unless the block is+ under-indented — i.e. some non-blank interior line has less leading+ whitespace than `base_shift`. Dedenting such a block would eat into its+ content, so the whole block is left untouched instead.+ """+ n = len(lines)+ keep = [False] * n+ i = 0+ while i < n:+ if not protected[i]:+ i += 1+ continue+ j = i+ block_min: float = float("inf")+ while j < n and protected[j]:+ line = lines[j]+ if line.strip():+ block_min = min(block_min, len(line) - len(line.lstrip()))+ j += 1+ if block_min != float("inf") and block_min < base_shift:+ for k in range(i, j):+ keep[k] = True+ i = j+ return keep+++def smart_dedent(code: str) -> str:+ """Token-aware dedent.++ Strips base indentation (computed from code lines only) from every line.+ In the common case this is identical to `textwrap.dedent`. The exception+ is a multiline string whose interior is under-indented relative to the+ surrounding code: dedenting it would eat into the string's content, so its+ interior is left completely untouched.+ """+ try:+ tokens = list(tokenize.generate_tokens(io.StringIO(code).readline))+ except TokenError:+ return textwrap.dedent(code)++ lines = split_source_lines(code, keepends=True)+ protected = _get_protected_lines(code, tokens)++ min_indent: float = float("inf")+ for i, line in enumerate(lines):+ if protected[i] or not line.strip():+ continue+ min_indent = min(min_indent, len(line) - len(line.lstrip()))++ if min_indent in (0, float("inf")):+ return code++ base_shift = int(min_indent)+ keep = _under_indented_string_lines(lines, protected, base_shift)++ return "".join(+ line if keep[i] else _strip_indent(line, base_shift)+ for i, line in enumerate(lines)+ )+++def fixed_dedent(text: str) -> str:+ """Dedent with robustness for AI-generated code with inconsistent indentation.++ Unlike textwrap.dedent, this pads lines that are under-indented to the+ base indentation level before dedenting, while preserving whitespace+ inside multiline string literals whose internal structure doesn't match+ the surrounding code's indentation.+ """+ lines = split_source_lines(text)++ try:+ tokens = list(tokenize.generate_tokens(io.StringIO(text).readline))+ except TokenError:+ tokens = []++ protected = _get_protected_lines(text, tokens)++ # Mirror str.splitlines(): a final line terminator yields a trailing ""+ # that we drop, so the result carries no spurious trailing newline. (The+ # trailing line is never inside a string, so `protected` stays aligned.)+ n = len(lines)+ if n > 1 and lines[-1] == "":+ n -= 1++ indent = None+ for i in range(n):+ if protected[i]:+ continue+ if content := lines[i].lstrip():+ indent = lines[i][: len(lines[i]) - len(content)]+ break++ if indent is None:+ # No code line to key off of (all-blank input): fall back to a plain+ # dedent, matching textwrap's normalization of whitespace-only lines.+ return textwrap.dedent(text)++ def refill(i: int, ln: str) -> str:+ if protected[i] or ln.startswith(indent):+ return ln+ return indent + ln++ return smart_dedent("\n".join(refill(i, lines[i]) for i in range(n)))diff --git a/marimo/_ast/parse.py b/marimo/_ast/parse.pyindex b7169126c2..4668fe54a7 100644--- a/marimo/_ast/parse.py+++ b/marimo/_ast/parse.py@@ -6,7 +6,6 @@ import ioimport token as token_typesimport warningsfrom pathlib import Path-from textwrap import dedentfrom tokenize import TokenInfo, tokenizefrom typing import (TYPE_CHECKING,@@ -16,6 +15,7 @@ from typing import (cast,)+from marimo._ast.dedent import fixed_dedentfrom marimo._ast.names import DEFAULT_CELL_NAME, SETUP_CELL_NAMEfrom marimo._schemas.serialization import (AppInstantiation,@@ -64,26 +64,6 @@ def split_source_lines(text: str) -> list[str]:return text.replace("\r\n", "\n").replace("\r", "\n").split("\n")-def fixed_dedent(text: str) -> str:- """Manually edited code, can dedent"""- # Added robustness for AI generated code- lines = text.splitlines()- for line in lines:- if content := line.lstrip():- indent = line[: len(line) - len(content)]- break- else:- # Quit early, no clear leading spaces- return dedent(text)-- def refill(line: str) -> str:- if not line.startswith(indent):- return indent + line- return line-- return dedent("\n".join(map(refill, lines)))--def unwrap_cell_body(formatted: str) -> str:"""Extract the body of a wrapped `def _():` cell after ruff formatting.diff --git a/marimo/_ast/scanner.py b/marimo/_ast/scanner.pyindex 8fcc040f43..dae9592c9b 100644--- a/marimo/_ast/scanner.py+++ b/marimo/_ast/scanner.py@@ -400,7 +400,7 @@ def _extract_body_code(cell_source: str, kind: str) -> str:Strips decorator lines, def/class/with header, and trailing return.Used when a cell fails ast.parse() to get the body for UnparsableCell."""- from marimo._ast.parse import fixed_dedent+ from marimo._ast.dedent import fixed_dedentif kind == "unparsable":# Already app._unparsable_cell(...) — return as-isdiff --git a/tests/_ast/test_dedent.py b/tests/_ast/test_dedent.pynew file mode 100644index 0000000000..429213852e--- /dev/null+++ b/tests/_ast/test_dedent.py@@ -0,0 +1,157 @@+"""Tests for token-aware smart_dedent and fixed_dedent."""++from __future__ import annotations++from marimo._ast.dedent import (+ fixed_dedent,+ smart_dedent,+ split_source_lines,+)+++class TestSplitSourceLines:+ def test_default_drops_terminators(self):+ assert split_source_lines("a\nb\n") == ["a", "b", ""]++ def test_normalizes_carriage_returns(self):+ assert split_source_lines("a\r\nb\rc") == ["a", "b", "c"]++ def test_keepends_roundtrips(self):+ for text in ["a\n b\n", "a\r\nb", "a\rb\n", "x = 1", "", "a\n\nb"]:+ assert "".join(split_source_lines(text, keepends=True)) == text++ def test_keepends_preserves_terminators(self):+ assert split_source_lines("a\r\nb\n", keepends=True) == [+ "a\r\n",+ "b\n",+ "",+ ]++ def test_keepends_element_count_matches_default(self):+ for text in ["a\nb\n", "a\r\nb", "a\rb\n", "x = 1", "", "a\n\nb"]:+ assert len(split_source_lines(text, keepends=True)) == len(+ split_source_lines(text)+ )+++class TestSmartDedent:+ def test_basic_dedent(self):+ assert smart_dedent(" x = 1\n y = 2\n") == "x = 1\ny = 2\n"++ def test_no_indent_noop(self):+ code = "x = 1\ny = 2\n"+ assert smart_dedent(code) == code++ def test_multiline_string_interior_preserved(self):+ code = ' x = """line0\n two_spaces\n four_spaces\n six_spaces"""\n'+ result = smart_dedent(code)+ assert (+ result+ == 'x = """line0\n two_spaces\n four_spaces\n six_spaces"""\n'+ )++ def test_fstring_interior_preserved(self):+ code = ' name = "world"\n x = f"""hello\n indented line\n more indented"""\n'+ result = smart_dedent(code)+ assert " indented line" in result+ assert " more indented" in result+ assert result.startswith('name = "world"')++ def test_raw_string_interior_preserved(self):+ code = ' x = r"""line0\n raw content\n more raw"""\n'+ result = smart_dedent(code)+ assert " raw content" in result+ assert " more raw" in result++ def test_docstring_content_at_column_zero(self):+ code = ' def f():\n """Cell to compute sum\ne.g. a=1; b=2\nfoo\n"""\n return a + b\n'+ result = smart_dedent(code)+ assert "e.g. a=1; b=2" in result+ assert result.startswith("def f():")++ def test_single_line_string_not_protected(self):+ code = " x = 'hello world'\n y = 2\n"+ assert smart_dedent(code) == "x = 'hello world'\ny = 2\n"++ def test_mixed_code_and_multiline_string_deeper_indent(self):+ # String block indented MORE than surrounding code (block_min >=+ # base_shift) participates in the same dedent, preserving its own+ # internal alignment between SELECT and FROM.+ code = ' def f():\n sql = """\n SELECT *\n FROM t\n """\n return sql\n'+ result = smart_dedent(code)+ assert result.startswith("def f():")+ assert " SELECT *" in result+ assert " FROM t" in result++ def test_multiline_string_under_indented_relative_to_code(self):+ # String block indented LESS than the surrounding code in places+ # (block_min < base_shift) is left fully untouched, since it can't+ # safely absorb the same dedent without losing structure.+ code = ' def f():\n text = """\n shallow\n deep\n """\n return text\n'+ result = smart_dedent(code)+ assert result.startswith("def f():")+ assert " shallow" in result+ assert " deep" in result++ def test_top_level_function_with_multiline_docstring(self):+ # Already at zero indent — no-op+ code = '@app.function\ndef foo():\n """I wonder what\nHappens to this ?\n"""\n return ...\n'+ result = smart_dedent(code)+ assert result == code++ def test_indented_top_level_function_with_docstring(self):+ code = ' @app.function\n def foo():\n """Also wonder what\nHappens to that ?\n"""\n return ...\n'+ result = smart_dedent(code)+ assert result.startswith("@app.function")+ assert "Happens to that ?" in result++ def test_under_indented_docstring_preserved(self):+ # Docstring under-indented relative to the code (a line with only two+ # spaces) is left completely untouched; only the code is dedented.+ code = (+ ' """Cell to compute sum\n'+ " e.g. a=1; b=2\n"+ " foo\n"+ ' """\n'+ " a + b\n"+ )+ assert smart_dedent(code) == (+ '"""Cell to compute sum\n'+ " e.g. a=1; b=2\n"+ " foo\n"+ ' """\n'+ "a + b\n"+ )++ def test_blank_line_in_docstring_preserved(self):+ # A blank line inside an in-step docstring keeps its newline instead of+ # being swallowed (the line block is still dedented with the code).+ code = ' """hello\n\n world\n """\n x = 1\n'+ assert smart_dedent(code) == '"""hello\n\nworld\n"""\nx = 1\n'+++class TestFixedDedent:+ def test_basic(self):+ assert fixed_dedent(" x = 1\n y = 2") == "x = 1\ny = 2"++ def test_inconsistent_indentation(self):+ result = fixed_dedent(" x = 1\ny = 2")+ assert result == "x = 1\ny = 2"++ def test_multiline_string_preserved(self):+ code = ' x = """line0\n two_spaces\n four_spaces"""\n'+ result = fixed_dedent(code)+ assert " two_spaces" in result+ assert " four_spaces" in result++ def test_fstring_preserved(self):+ code = ' x = f"""hello\n indented\n more"""\n'+ result = fixed_dedent(code)+ assert " indented" in result+ assert " more" in result++ def test_no_spurious_trailing_newline(self):+ # A trailing newline must not survive into the result (it would add a+ # spurious blank line before the auto-generated `return` on save).+ assert fixed_dedent(" import x\n") == "import x"+ assert fixed_dedent(" x = 1\n y = 2\n") == "x = 1\ny = 2"