Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix: keep string values intact through `marimo check --fix` (#11581)

Dylan Madisetti20d77c9d251a
diff --git a/marimo/_ast/codegen.py b/marimo/_ast/codegen.py
index 3e7e023c8b..978acacb82 100644
--- a/marimo/_ast/codegen.py
+++ b/marimo/_ast/codegen.py
@@ -13,7 +13,11 @@ from typing import TYPE_CHECKING, Any, Literal
from marimo import _loggers
from marimo._ast.app_config import _AppConfig
from marimo._ast.cell import CellConfig, CellImpl
-from marimo._ast.compiler import compile_cell
+from marimo._ast.compiler import (
+ compile_cell,
+ dedent_markdown,
+ extract_markdown,
+)
from marimo._ast.names import (
DEFAULT_CELL_NAME,
SETUP_CELL_NAME,
@@ -193,12 +197,22 @@ def to_decorator(
)
+_STRING_START_TYPES = {tokenize.STRING}
+_STRING_END_TYPES: set[int] = set()
+if sys.version_info >= (3, 12):
+ _STRING_START_TYPES.add(tokenize.FSTRING_START)
+ _STRING_END_TYPES.add(tokenize.FSTRING_END)
+if sys.version_info >= (3, 14):
+ _STRING_START_TYPES.add(tokenize.TSTRING_START)
+ _STRING_END_TYPES.add(tokenize.TSTRING_END)
+
+
def format_markdown(cell: CellImpl) -> str:
- markdown = cell.markdown or ""
- # AST does not preserve string quote types or types, so directly use
+ # The markdown is reflowed from the literal's source text rather than
+ # its evaluated value, so escape sequences come out exactly as written.
+ # AST does not preserve string quote types or prefixes, so directly use
# tokenize.
tokens = tokenize.tokenize(io.BytesIO(cell.code.encode("utf-8")).readline)
- tag = ""
# Anything outside the `mo.md(...)` call (comments, and the whitespace
# around them) is kept verbatim, so track where the call starts and ends
# in the source rather than rebuilding it from tokens, which drop
@@ -214,42 +228,50 @@ def format_markdown(cell: CellImpl) -> str:
call_start = 0
call_end = len(cell.code)
depth = 0
- tokenizes_fstring = sys.version_info >= (3, 12)
- start_tokens = (
- (tokenize.STRING, tokenize.FSTRING_START)
- if tokenizes_fstring
- else (tokenize.STRING,)
- )
- fstring = False
+ literals = 0
+ prefix = ""
+ quote = ""
+ body_start = body_end = 0
for tok in tokens:
- # if string
- if tok.type in start_tokens:
- tag = ""
+ if tok.type in _STRING_START_TYPES:
+ literals += 1
# rf"""/ f"/ r"/ "more
- start = tok.string[:5]
- for _ in range(2):
- if start[0].lower() in "rtf":
- tag += start[0]
- start = start[1:]
- fstring = "f" in tag.lower()
- elif tok.string == "mo":
+ opening = tok.string.lstrip("rRfFtTuU")
+ prefix = tok.string[: len(tok.string) - len(opening)]
+ quote = (
+ opening[:3] if opening[:3] in ('"""', "'''") else opening[0]
+ )
+ body_start = offset(tok.start) + len(prefix) + len(quote)
+ if tok.type == tokenize.STRING:
+ body_end = offset(tok.end) - len(quote)
+ elif tok.type in _STRING_END_TYPES:
+ body_end = offset(tok.start)
+ elif tok.type == tokenize.NAME and tok.string == "mo":
call_start = offset(tok.start)
depth = 0
- elif tok.string == "(":
+ elif tok.type == tokenize.OP and tok.string == "(":
depth += 1
- elif tok.string == ")":
+ elif tok.type == tokenize.OP and tok.string == ")":
depth -= 1
if depth == 0:
call_end = offset(tok.end)
- if fstring:
- # We can blanket replace, because cell.markdown is not set
- # on f-strings with values.
- markdown = markdown.replace("{", "{{").replace("}", "}}")
- markdown = markdown.replace('""', '"\\"')
+ # Implicitly concatenated literals have no single body to reflow.
+ if literals != 1:
+ return cell.code
- # We always use """ as per front end.
- body = construct_markdown_call(markdown, '"""', tag)
+ markdown = dedent_markdown(cell.code[body_start:body_end])
+ # We always use """ as per front end, unless the body holds a bare
+ # triple quote, which only the other triple quote can hold.
+ if '"""' in markdown and quote != '"""':
+ quote = "'''"
+ else:
+ quote = '"""'
+ body = construct_markdown_call(markdown, quote, prefix)
+ # Dedenting source lines can disagree with dedenting the value when an
+ # escaped newline hides a line break; leave such a cell as written.
+ if extract_markdown(body) != cell.markdown:
+ return cell.code
return "".join([cell.code[:call_start], body, cell.code[call_end:]])
diff --git a/marimo/_ast/compiler.py b/marimo/_ast/compiler.py
index c9d7f7fdd8..7ec35eb04c 100644
--- a/marimo/_ast/compiler.py
+++ b/marimo/_ast/compiler.py
@@ -218,22 +218,24 @@ def _extract_markdown(tree: ast.Module) -> str | None:
assert value.func.value.id == "mo" # type: ignore[attr-defined]
if not value.args: # Handle mo.md() with no arguments
return None
- md_lines = _extract_const_string(value.args).split("\n")
+ text = _extract_const_string(value.args)
except (AssertionError, AttributeError, ValueError):
# No reason to explicitly catch exceptions if we can't parse out
# markdown. Just handle it as a code block.
return None
+ return dedent_markdown(text)
+
+def dedent_markdown(text: str) -> str:
# Dedent behavior is a little different that in marimo js, so handle
# accordingly.
- md_lines = [line.rstrip() for line in md_lines]
+ lines = [line.rstrip() for line in text.split("\n")]
md = (
- textwrap.dedent(md_lines[0])
+ textwrap.dedent(lines[0])
+ "\n"
- + textwrap.dedent("\n".join(md_lines[1:]))
+ + textwrap.dedent("\n".join(lines[1:]))
)
- md = md.strip()
- return md
+ return md.strip()
def extract_markdown(code: str) -> str | None:
diff --git a/marimo/_ast/parse.py b/marimo/_ast/parse.py
index 978c64b22a..8f4ed6264e 100644
--- a/marimo/_ast/parse.py
+++ b/marimo/_ast/parse.py
@@ -318,6 +318,10 @@ class Extractor:
else:
end_lineno = node.body[-1].lineno - 1
end_col_offset = None
+ else:
+ # The last statement's own end offset stops short of a comment
+ # that shares its line.
+ end_col_offset = len(self.lines[end_lineno - 1])
# NB. node.[end_]lineno only captures the starting line of the
# captured node, same with col_offset.
diff --git a/tests/_ast/test_codegen.py b/tests/_ast/test_codegen.py
index f105498946..75a293abed 100644
--- a/tests/_ast/test_codegen.py
+++ b/tests/_ast/test_codegen.py
@@ -4,6 +4,7 @@ from __future__ import annotations
import ast
import json
import sys
+import warnings
from functools import partial
from inspect import cleandoc
from pathlib import Path
@@ -2410,3 +2411,71 @@ class TestLazyImports:
]
_, defs = run_path(str(path))["app"].run()
assert defs["y"] == "1"
+
+
+def _markdown_value(code: str) -> str:
+ (stmt,) = ast.parse(code).body
+ (arg,) = stmt.value.args # type: ignore[attr-defined]
+ if isinstance(arg, ast.JoinedStr):
+ (arg,) = arg.values
+ assert isinstance(arg, ast.Constant)
+ assert isinstance(arg.value, str)
+ return arg.value
+
+
+ ("code", "expected"),
+ [
+ # Raw string: a doubled quote is content, not an escape
+ (
+ 'mo.md(\n r"""\n !!! tip ""\n """\n)',
+ 'mo.md(r"""\n!!! tip ""\n""")',
+ ),
+ # Raw string: a code sample with an empty JS string stays terminated
+ (
+ 'mo.md(\n r"""\n const myVar = "";\n """\n)',
+ 'mo.md(r"""\nconst myVar = "";\n""")',
+ ),
+ # Non-raw string: an escaped backslash stays escaped
+ (
+ 'mo.md(\n """\n `field=\'value\\\\.data\'`\n """\n)',
+ 'mo.md("""\n`field=\'value\\\\.data\'`\n""")',
+ ),
+ # Non-raw string: escape sequences are kept as written
+ ('mo.md("tab\\there")', 'mo.md("""\ntab\\there\n""")'),
+ # f-string: doubled braces stay doubled
+ ('mo.md(f"{{x}}")', 'mo.md(f"""\n{{x}}\n""")'),
+ # A bare triple quote in the body cannot move into """ quotes
+ ('mo.md(r\'has """\')', "mo.md(r'''\nhas \"\"\"\n''')"),
+ ('mo.md(\'has """\')', "mo.md('''\nhas \"\"\"\n''')"),
+ ],
+)
+def test_format_markdown_preserves_string_value(
+ code: str, expected: str
+) -> None:
+ formatted = codegen.format_markdown(compile_cell(code))
+ assert formatted == expected
+ with warnings.catch_warnings():
+ # An invalid escape sequence is only a SyntaxWarning
+ warnings.simplefilter("error")
+ compile(formatted, "<cell>", "exec")
+ assert (
+ _markdown_value(formatted)
+ == "\n" + dedent(_markdown_value(code)).strip() + "\n"
+ )
+
+
+def test_format_markdown_keeps_cell_with_escaped_newline() -> None:
+ # Dedenting the physical lines would leave `bar` indented but not `baz`.
+ code = 'mo.md("""foo\\n bar\n baz""")'
+ assert codegen.format_markdown(compile_cell(code)) == code
+
+
+def test_trailing_comment_on_final_expression_round_trips() -> None:
+ from marimo._convert.converters import MarimoConvert
+
+ source = wrap_generate_filecontents(
+ ["x = 1", "x # Uses the default renderer"], ["a", "b"]
+ )
+ assert " x # Uses the default renderer\n" in source
+ assert MarimoConvert.from_py(source).to_py() == source
diff --git a/tests/_ast/test_parse.py b/tests/_ast/test_parse.py
index 6625f9f70f..74a6714db3 100644
--- a/tests/_ast/test_parse.py
+++ b/tests/_ast/test_parse.py
@@ -361,6 +361,45 @@ if __name__ == "__main__":
# end 1"""
)
+ @staticmethod
+ def test_cell_without_return_keeps_inline_comment() -> None:
+ """A cell whose last line carries a comment keeps it without a return."""
+ code = """import marimo
+
+__generated_with = "0.0.0"
+app = marimo.App()
+
+
+def _():
+ x = 1
+ x # the comment
+
+
+if __name__ == "__main__":
+ app.run()
+"""
+ notebook = parse_notebook(code)
+ assert notebook.cells[0].code == "x = 1\nx # the comment"
+
+ @staticmethod
+ def test_one_line_cell_keeps_inline_comment() -> None:
+ code = """import marimo
+
+__generated_with = "0.0.0"
+app = marimo.App()
+
+
+def _(): x # the comment
+
+
+if __name__ == "__main__":
+ app.run()
+"""
+ notebook = parse_notebook(code)
+ assert notebook.cells[0].code == "x # the comment"
+
@staticmethod
def test_class_definition_with_trailing_comment() -> None:
"""@app.class_definition should preserve trailing comments at body indentation."""