A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix: keep string values intact through `marimo check --fix` (#11581)
Dylan Madisetti20d77c9d251a
diff --git a/marimo/_ast/codegen.py b/marimo/_ast/codegen.pyindex 3e7e023c8b..978acacb82 100644--- a/marimo/_ast/codegen.py+++ b/marimo/_ast/codegen.py@@ -13,7 +13,11 @@ from typing import TYPE_CHECKING, Any, Literalfrom marimo import _loggersfrom marimo._ast.app_config import _AppConfigfrom marimo._ast.cell import CellConfig, CellImpl-from marimo._ast.compiler import compile_cell+from marimo._ast.compiler import (+ compile_cell,+ dedent_markdown,+ extract_markdown,+)from marimo._ast.names import (DEFAULT_CELL_NAME,SETUP_CELL_NAME,@@ -193,12 +197,22 @@ def to_decorator()+_STRING_START_TYPES = {tokenize.STRING}+_STRING_END_TYPES: set[int] = set()+if sys.version_info >= (3, 12):+ _STRING_START_TYPES.add(tokenize.FSTRING_START)+ _STRING_END_TYPES.add(tokenize.FSTRING_END)+if sys.version_info >= (3, 14):+ _STRING_START_TYPES.add(tokenize.TSTRING_START)+ _STRING_END_TYPES.add(tokenize.TSTRING_END)++def format_markdown(cell: CellImpl) -> str:- markdown = cell.markdown or ""- # AST does not preserve string quote types or types, so directly use+ # The markdown is reflowed from the literal's source text rather than+ # its evaluated value, so escape sequences come out exactly as written.+ # AST does not preserve string quote types or prefixes, so directly use# tokenize.tokens = tokenize.tokenize(io.BytesIO(cell.code.encode("utf-8")).readline)- tag = ""# Anything outside the `mo.md(...)` call (comments, and the whitespace# around them) is kept verbatim, so track where the call starts and ends# in the source rather than rebuilding it from tokens, which drop@@ -214,42 +228,50 @@ def format_markdown(cell: CellImpl) -> str:call_start = 0call_end = len(cell.code)depth = 0- tokenizes_fstring = sys.version_info >= (3, 12)- start_tokens = (- (tokenize.STRING, tokenize.FSTRING_START)- if tokenizes_fstring- else (tokenize.STRING,)- )- fstring = False+ literals = 0+ prefix = ""+ quote = ""+ body_start = body_end = 0for tok in tokens:- # if string- if tok.type in start_tokens:- tag = ""+ if tok.type in _STRING_START_TYPES:+ literals += 1# rf"""/ f"/ r"/ "more- start = tok.string[:5]- for _ in range(2):- if start[0].lower() in "rtf":- tag += start[0]- start = start[1:]- fstring = "f" in tag.lower()- elif tok.string == "mo":+ opening = tok.string.lstrip("rRfFtTuU")+ prefix = tok.string[: len(tok.string) - len(opening)]+ quote = (+ opening[:3] if opening[:3] in ('"""', "'''") else opening[0]+ )+ body_start = offset(tok.start) + len(prefix) + len(quote)+ if tok.type == tokenize.STRING:+ body_end = offset(tok.end) - len(quote)+ elif tok.type in _STRING_END_TYPES:+ body_end = offset(tok.start)+ elif tok.type == tokenize.NAME and tok.string == "mo":call_start = offset(tok.start)depth = 0- elif tok.string == "(":+ elif tok.type == tokenize.OP and tok.string == "(":depth += 1- elif tok.string == ")":+ elif tok.type == tokenize.OP and tok.string == ")":depth -= 1if depth == 0:call_end = offset(tok.end)- if fstring:- # We can blanket replace, because cell.markdown is not set- # on f-strings with values.- markdown = markdown.replace("{", "{{").replace("}", "}}")- markdown = markdown.replace('""', '"\\"')+ # Implicitly concatenated literals have no single body to reflow.+ if literals != 1:+ return cell.code- # We always use """ as per front end.- body = construct_markdown_call(markdown, '"""', tag)+ markdown = dedent_markdown(cell.code[body_start:body_end])+ # We always use """ as per front end, unless the body holds a bare+ # triple quote, which only the other triple quote can hold.+ if '"""' in markdown and quote != '"""':+ quote = "'''"+ else:+ quote = '"""'+ body = construct_markdown_call(markdown, quote, prefix)+ # Dedenting source lines can disagree with dedenting the value when an+ # escaped newline hides a line break; leave such a cell as written.+ if extract_markdown(body) != cell.markdown:+ return cell.codereturn "".join([cell.code[:call_start], body, cell.code[call_end:]])diff --git a/marimo/_ast/compiler.py b/marimo/_ast/compiler.pyindex c9d7f7fdd8..7ec35eb04c 100644--- a/marimo/_ast/compiler.py+++ b/marimo/_ast/compiler.py@@ -218,22 +218,24 @@ def _extract_markdown(tree: ast.Module) -> str | None:assert value.func.value.id == "mo" # type: ignore[attr-defined]if not value.args: # Handle mo.md() with no argumentsreturn None- md_lines = _extract_const_string(value.args).split("\n")+ text = _extract_const_string(value.args)except (AssertionError, AttributeError, ValueError):# No reason to explicitly catch exceptions if we can't parse out# markdown. Just handle it as a code block.return None+ return dedent_markdown(text)++def dedent_markdown(text: str) -> str:# Dedent behavior is a little different that in marimo js, so handle# accordingly.- md_lines = [line.rstrip() for line in md_lines]+ lines = [line.rstrip() for line in text.split("\n")]md = (- textwrap.dedent(md_lines[0])+ textwrap.dedent(lines[0])+ "\n"- + textwrap.dedent("\n".join(md_lines[1:]))+ + textwrap.dedent("\n".join(lines[1:])))- md = md.strip()- return md+ return md.strip()def extract_markdown(code: str) -> str | None:diff --git a/marimo/_ast/parse.py b/marimo/_ast/parse.pyindex 978c64b22a..8f4ed6264e 100644--- a/marimo/_ast/parse.py+++ b/marimo/_ast/parse.py@@ -318,6 +318,10 @@ class Extractor:else:end_lineno = node.body[-1].lineno - 1end_col_offset = None+ else:+ # The last statement's own end offset stops short of a comment+ # that shares its line.+ end_col_offset = len(self.lines[end_lineno - 1])# NB. node.[end_]lineno only captures the starting line of the# captured node, same with col_offset.diff --git a/tests/_ast/test_codegen.py b/tests/_ast/test_codegen.pyindex f105498946..75a293abed 100644--- a/tests/_ast/test_codegen.py+++ b/tests/_ast/test_codegen.py@@ -4,6 +4,7 @@ from __future__ import annotationsimport astimport jsonimport sys+import warningsfrom functools import partialfrom inspect import cleandocfrom pathlib import Path@@ -2410,3 +2411,71 @@ class TestLazyImports:]_, defs = run_path(str(path))["app"].run()assert defs["y"] == "1"+++def _markdown_value(code: str) -> str:+ (stmt,) = ast.parse(code).body+ (arg,) = stmt.value.args # type: ignore[attr-defined]+ if isinstance(arg, ast.JoinedStr):+ (arg,) = arg.values+ assert isinstance(arg, ast.Constant)+ assert isinstance(arg.value, str)+ return arg.value+++ ("code", "expected"),+ [+ # Raw string: a doubled quote is content, not an escape+ (+ 'mo.md(\n r"""\n !!! tip ""\n """\n)',+ 'mo.md(r"""\n!!! tip ""\n""")',+ ),+ # Raw string: a code sample with an empty JS string stays terminated+ (+ 'mo.md(\n r"""\n const myVar = "";\n """\n)',+ 'mo.md(r"""\nconst myVar = "";\n""")',+ ),+ # Non-raw string: an escaped backslash stays escaped+ (+ 'mo.md(\n """\n `field=\'value\\\\.data\'`\n """\n)',+ 'mo.md("""\n`field=\'value\\\\.data\'`\n""")',+ ),+ # Non-raw string: escape sequences are kept as written+ ('mo.md("tab\\there")', 'mo.md("""\ntab\\there\n""")'),+ # f-string: doubled braces stay doubled+ ('mo.md(f"{{x}}")', 'mo.md(f"""\n{{x}}\n""")'),+ # A bare triple quote in the body cannot move into """ quotes+ ('mo.md(r\'has """\')', "mo.md(r'''\nhas \"\"\"\n''')"),+ ('mo.md(\'has """\')', "mo.md('''\nhas \"\"\"\n''')"),+ ],+)+def test_format_markdown_preserves_string_value(+ code: str, expected: str+) -> None:+ formatted = codegen.format_markdown(compile_cell(code))+ assert formatted == expected+ with warnings.catch_warnings():+ # An invalid escape sequence is only a SyntaxWarning+ warnings.simplefilter("error")+ compile(formatted, "<cell>", "exec")+ assert (+ _markdown_value(formatted)+ == "\n" + dedent(_markdown_value(code)).strip() + "\n"+ )+++def test_format_markdown_keeps_cell_with_escaped_newline() -> None:+ # Dedenting the physical lines would leave `bar` indented but not `baz`.+ code = 'mo.md("""foo\\n bar\n baz""")'+ assert codegen.format_markdown(compile_cell(code)) == code+++def test_trailing_comment_on_final_expression_round_trips() -> None:+ from marimo._convert.converters import MarimoConvert++ source = wrap_generate_filecontents(+ ["x = 1", "x # Uses the default renderer"], ["a", "b"]+ )+ assert " x # Uses the default renderer\n" in source+ assert MarimoConvert.from_py(source).to_py() == sourcediff --git a/tests/_ast/test_parse.py b/tests/_ast/test_parse.pyindex 6625f9f70f..74a6714db3 100644--- a/tests/_ast/test_parse.py+++ b/tests/_ast/test_parse.py@@ -361,6 +361,45 @@ if __name__ == "__main__":# end 1""")+ @staticmethod+ def test_cell_without_return_keeps_inline_comment() -> None:+ """A cell whose last line carries a comment keeps it without a return."""+ code = """import marimo++__generated_with = "0.0.0"+app = marimo.App()+++def _():+ x = 1+ x # the comment+++if __name__ == "__main__":+ app.run()+"""+ notebook = parse_notebook(code)+ assert notebook.cells[0].code == "x = 1\nx # the comment"++ @staticmethod+ def test_one_line_cell_keeps_inline_comment() -> None:+ code = """import marimo++__generated_with = "0.0.0"+app = marimo.App()+++def _(): x # the comment+++if __name__ == "__main__":+ app.run()+"""+ notebook = parse_notebook(code)+ assert notebook.cells[0].code == "x # the comment"+@staticmethoddef test_class_definition_with_trailing_comment() -> None:"""@app.class_definition should preserve trailing comments at body indentation."""