A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix: handle bang statement edgecases in ipynb convert (#8126)
Dylan Madisetti9054cebe29fb
diff --git a/marimo/_convert/ipynb/to_ir.py b/marimo/_convert/ipynb/to_ir.pyindex 4af7fd81a2..6ff7c0c8af 100644--- a/marimo/_convert/ipynb/to_ir.py+++ b/marimo/_convert/ipynb/to_ir.py@@ -403,6 +403,12 @@ class ExclamationMarkResult:needs_subprocess: bool+def _pass_if_indented(indent_level: int) -> str:+ if indent_level > 0:+ return "pass "+ return ""++def _normalize_git_url_package(package: str) -> str:"""Normalize git URL packages to PEP 508 format.@@ -456,43 +462,115 @@ def _normalize_git_url_package(package: str) -> str:def _extract_pip_install(- command_line: str, command_tokens: list[str]+ command_line: str, command_tokens: list[str], indent_level: int = 0) -> ExclamationCommandResult:pip_packages: list[str] = []if "install" not in command_tokens:- return _shlex_to_subprocess_call(command_line, command_tokens)+ return _shlex_to_subprocess_call(+ command_line, command_tokens, indent_level+ )install_idx = command_tokens.index("install")- packages = [- p for p in command_tokens[install_idx + 1 :] if not p.startswith("-")- ]++ # Collect packages and items for display, skipping flags+ packages = [] # Actual packages (no templates)+ templates = [] # Template placeholders+ for token in command_tokens[install_idx + 1 :]:+ # Skip flags (starting with -)+ if token.startswith("-"):+ continue+ # Template placeholders stop package collection+ if token.startswith("{") and token.endswith("}"):+ templates.append(token)+ break+ packages.append(token)+# Normalize git URLs to PEP 508 formatpip_packages = [_normalize_git_url_package(p) for p in packages]- # Comment out the pip command+ # For display: show templates only if there are no real packages+ display_items = packages if packages else templates++ # Comment out the pip command, showing items in comment+ # Add pass for indented commands to prevent empty blocksreplacement = (- "# packages added via marimo's package management: "- f"{' '.join(packages)} !{command_line}"+ f"{_pass_if_indented(indent_level)}# packages added via marimo's "+ f"package management: {' '.join(display_items)} !{command_line}")return ExclamationCommandResult(replacement, pip_packages, False)+def _is_compilable_expression(expr: str) -> bool:+ """Check if expression is valid Python that can be compiled.++ Args:+ expr: The expression to check (without surrounding braces)++ Returns:+ True if the expression can be compiled as valid Python, False otherwise+ """+ try:+ compile(expr, "<string>", "eval")+ return True+ except (SyntaxError, ValueError):+ return False++def _shlex_to_subprocess_call(- command_line: str, command_tokens: list[str]+ command_line: str, command_tokens: list[str], indent_level: int = 0) -> ExclamationCommandResult:- """Convert a shell command to subprocess.call([...])"""+ """Convert a shell command to subprocess.call([...])++ Template placeholders {expr} are converted to str(expr) if expr is valid Python.+ If any template contains invalid Python, the entire command is commented out.++ Args:+ command_line: The command string+ command_tokens: Tokenized command+ indent_level: Number of indents for the examined line.+ """+ # First pass: check if any template is invalid+ for token in command_tokens:+ if token.startswith("{") and token.endswith("}"):+ expr = token[1:-1]+ if not _is_compilable_expression(expr):+ # Comment out entire command if any template is invalid+ # Always add pass to prevent empty blocks+ return ExclamationCommandResult(+ f"# Note: Command contains invalid template expression\n"+ f"{_pass_if_indented(indent_level)}# !{command_line}",+ [],+ False, # No subprocess needed+ )++ # Second pass: convert templates to str() calls+ processed_tokens = []+ for token in command_tokens:+ if token.startswith("{") and token.endswith("}"):+ expr = token[1:-1]+ # Convert to str() call+ processed_tokens.append(f"str({expr})")+ else:+ processed_tokens.append(repr(token))++ # Build the subprocess call with processed tokens+ tokens_str = "[" + ", ".join(processed_tokens) + "]"command = "\n".join(- [f"#! {command_line}", f"subprocess.call({command_tokens!r})"]+ [f"#! {command_line}", f"subprocess.call({tokens_str})"])return ExclamationCommandResult(command, [], True)def _handle_exclamation_command(- command_line: str,+ command_line: str, indent_level: int = 0) -> ExclamationCommandResult:"""Process an exclamation command line.+ Args:+ command_line: The command to process (without the leading !)+ indent_level: Column position of the ! (0 = top-level, >0 = indented)+Returns: (replacement_text, pip_packages, needs_subprocess)"""import shlex@@ -510,10 +588,135 @@ def _handle_exclamation_command(# For instance in the case `uv pip install ...`for i, token in enumerate(command_tokens):if token.startswith("pip"):- return _extract_pip_install(command_line, command_tokens[i:])+ # Pip installs always use marimo's package management (never subprocess)+ return _extract_pip_install(+ command_line, command_tokens[i:], indent_level+ )# Replace with subprocess.call()- return _shlex_to_subprocess_call(command_line, command_tokens)+ return _shlex_to_subprocess_call(+ command_line, command_tokens, indent_level+ )+++def _normalize_package_name(name: str) -> str:+ """Normalize a package name per PEP 503.++ PEP 503 specifies that package names should be normalized by:+ - Converting to lowercase+ - Replacing underscores, periods, and consecutive dashes with single dashes++ Args:+ name: Package name to normalize++ Returns:+ Normalized package name+ """+ return re.sub(r"[-_.]+", "-", name.lower())+++def _extract_package_name(pkg: str) -> str:+ """Extract and normalize the base package name from a package specification.++ Handles version specifiers, extras, and VCS URLs.+ Returns normalized names per PEP 503.++ Args:+ pkg: Package specification (e.g., "numpy>=1.0", "package[extra]", "name @ git+...")++ Returns:+ Normalized base package name (e.g., "numpy", "package", "name")+ """+ # Handle PEP 508 URL format: "name @ git+..."+ if " @ " in pkg:+ name = pkg.split(" @ ")[0].strip()+ return _normalize_package_name(name)++ # Strip version specifiers and extras+ name = re.split(r"[=<>~\[]+", pkg)[0].strip()+ return _normalize_package_name(name)+++def _resolve_pip_packages(packages: list[str]) -> list[str]:+ """Resolve pip packages using uv for validation, returning only direct packages.++ Uses uv pip compile to validate packages and resolve conflicts, but filters+ the output to only include packages that were originally requested (not+ transitive dependencies).++ For git URLs, preserves the full PEP 508 format (e.g., "name @ git+...").++ Args:+ packages: List of package specifications (may have duplicates/conflicts)++ Returns:+ Resolved and sorted list of direct packages only+ """+ if not packages:+ return []++ import subprocess+ import tempfile+ from pathlib import Path++ # Build a mapping from normalized name to original package spec+ # For git URLs, we want to preserve the full URL format+ original_specs: dict[str, str] = {}+ for pkg in packages:+ name = _extract_package_name(pkg)+ # For git URLs (PEP 508 format), preserve the full spec+ if " @ " in pkg:+ original_specs[name] = pkg+ else:+ # For regular packages, just use the normalized name+ original_specs[name] = name++ original_names = set(original_specs.keys())++ # Try using uv pip compile to validate/resolve conflicts+ try:+ with tempfile.TemporaryDirectory() as tmpdir:+ req_file = Path(tmpdir) / "requirements.in"+ out_file = Path(tmpdir) / "requirements.txt"++ # Write packages to temp file+ req_file.write_text("\n".join(packages))++ # Run uv pip compile+ result = subprocess.run(+ [+ "uv",+ "pip",+ "compile",+ str(req_file),+ "--output-file",+ str(out_file),+ ],+ capture_output=True,+ text=True,+ timeout=10,+ )++ if result.returncode == 0 and out_file.exists():+ # Parse resolved requirements, filtering to only direct packages+ resolved = []+ for line in out_file.read_text().splitlines():+ line = line.strip()+ # Skip comments and empty lines+ if line and not line.startswith("#"):+ # Extract package name+ pkg_name = _extract_package_name(line)+ # Only include if it was in the original request+ if pkg_name in original_names:+ # Use the original spec (preserves git URLs)+ resolved.append(original_specs[pkg_name])+ return sorted(set(resolved))+ except (FileNotFoundError, subprocess.TimeoutExpired, Exception):+ # uv not available or failed, fall through to unpinning+ pass++ # Fallback: deduplicate and return original specs+ return sorted(original_specs.values())def transform_exclamation_mark(sources: list[str]) -> ExclamationMarkResult:@@ -553,6 +756,7 @@ def transform_exclamation_mark(sources: list[str]) -> ExclamationMarkResult:exclaim_tokens: list[TokenInfo] = []trailing_comment = Noneexclaim_line_num = None+ exclaim_indent_level = 0for i, token in enumerate(tokens):# Check for newline + ! pattern (or ! at start after ENCODING)@@ -568,6 +772,9 @@ def transform_exclamation_mark(sources: list[str]) -> ExclamationMarkResult:exclaim_tokens = []trailing_comment = Noneexclaim_line_num = token.start[0]+ exclaim_indent_level = token.start[+ 1+ ] # Column position = indentationcontinue # Skip the ! tokenelif in_exclaim:@@ -577,20 +784,58 @@ def transform_exclamation_mark(sources: list[str]) -> ExclamationMarkResult:if exclaim_tokens and exclaim_tokens[-1].type == COMMENT:trailing_comment = exclaim_tokens.pop()- # Extract command from collected tokens by joining their strings+ # Extract command from collected tokens by reconstructing from sourceif exclaim_tokens:- # Reconstruct command from tokens+ # Reconstruct command from original source using token positions+ # This properly handles multi-line commands with backslash continuationsfirst_token = exclaim_tokens[0]last_token = exclaim_tokens[-1]- # Get the source line and extract the command portion- line_text = first_token.line++ # Get line numbers (1-indexed)+ start_line_num = first_token.start[0]+ end_line_num = last_token.end[0]start_col = first_token.start[1]end_col = last_token.end[1]- command_line = line_text[start_col:end_col].strip()++ # Split cell into lines for extraction+ cell_lines = cell.split("\n")++ if start_line_num == end_line_num:+ # Single line command+ command_line = cell_lines[start_line_num - 1][+ start_col:end_col+ ].strip()+ else:+ # Multi-line command - extract across lines+ parts = []+ # First line+ parts.append(+ cell_lines[start_line_num - 1][start_col:]+ )+ # Middle lines+ for line_idx in range(+ start_line_num, end_line_num - 1+ ):+ parts.append(cell_lines[line_idx])+ # Last line+ parts.append(+ cell_lines[end_line_num - 1][:end_col]+ )++ # Join and remove backslash continuations+ full_text = "\n".join(parts)+ # Remove backslash line continuations+ command_line = full_text.replace(+ "\\\n", " "+ ).strip()+ # Normalize whitespace+ command_line = " ".join(command_line.split())else:command_line = ""- result = _handle_exclamation_command(command_line)+ result = _handle_exclamation_command(+ command_line, indent_level=exclaim_indent_level+ )all_pip_packages.extend(result.pip_packages)any_needs_subprocess |= result.needs_subprocess@@ -605,6 +850,16 @@ def transform_exclamation_mark(sources: list[str]) -> ExclamationMarkResult:"\n" + trailing_comment.string)+ # For multi-line commands, mark continuation lines for removal+ if exclaim_tokens:+ last_token = exclaim_tokens[-1]+ end_line_num = last_token.end[0]+ # Mark all continuation lines (after the first) as removed+ for line_num in range(+ exclaim_line_num + 1, end_line_num + 1+ ):+ line_replacements[line_num] = None # type: ignore+in_exclaim = Falseelse:# Collect tokens that are part of the command@@ -616,9 +871,13 @@ def transform_exclamation_mark(sources: list[str]) -> ExclamationMarkResult:new_lines = []for line_num, line in enumerate(lines, start=1):if line_num in line_replacements:+ replacement = line_replacements[line_num]+ # None means skip this line (multi-line continuation)+ if replacement is None:+ continue# Preserve indentation from original lineindent = (len(line) - len(line.lstrip())) * " "- replacements = line_replacements[line_num].split("\n")+ replacements = replacement.split("\n")# Add indentation to replacementindented_replacement = [indent + replacement for replacement in replacements@@ -630,9 +889,12 @@ def transform_exclamation_mark(sources: list[str]) -> ExclamationMarkResult:else:transformed_sources.append(cell)+ # Resolve packages using uv (or unpin if unavailable)+ resolved_packages = _resolve_pip_packages(all_pip_packages)+return ExclamationMarkResult(transformed_sources=transformed_sources,- pip_packages=all_pip_packages,+ pip_packages=resolved_packages,needs_subprocess=any_needs_subprocess,)diff --git a/tests/_convert/ipynb/snapshots/converted_pip_commands.py.txt b/tests/_convert/ipynb/snapshots/converted_pip_commands.py.txtindex 2927b80dce..0d787a2b8b 100644--- a/tests/_convert/ipynb/snapshots/converted_pip_commands.py.txt+++ b/tests/_convert/ipynb/snapshots/converted_pip_commands.py.txt@@ -1,5 +1,5 @@# /// script-# dependencies = ["transformers", "pandas", "numpy", "matplotlib", "scikit-learn", "seaborn"]+# dependencies = ["matplotlib", "numpy", "pandas", "scikit-learn", "seaborn", "transformers"]# ///import marimodiff --git a/tests/_convert/ipynb/test_ipynb_to_ir.py b/tests/_convert/ipynb/test_ipynb_to_ir.pyindex 76dd020226..1169c5103e 100644--- a/tests/_convert/ipynb/test_ipynb_to_ir.py+++ b/tests/_convert/ipynb/test_ipynb_to_ir.py@@ -272,6 +272,29 @@ def test_transform_exclamation_mark():assert result.needs_subprocess is True+def test_complex_exclamation_mark():+ sources = [+ """!uv pip install --upgrade \\+ openpipe-art[backend,langgraph]==0.4.11 langchain-core langgraph langchain_openai tenacity datasets pillow==11.3.0 protobuf==5.29.5 {get_vllm} {get_numpy} --prerelease allow --no-cache-dir"""+ ]+ result = transform_exclamation_mark(sources)+ # Packages are unpinned, normalized per PEP 503, and sorted+ assert result.pip_packages == [+ "datasets",+ "langchain-core",+ "langchain-openai", # normalized from langchain_openai+ "langgraph",+ "openpipe-art",+ "pillow",+ "protobuf",+ "tenacity",+ ]+ assert result.transformed_sources == [+ "# packages added via marimo's package management: openpipe-art[backend,langgraph]==0.4.11 langchain-core langgraph langchain_openai tenacity datasets pillow==11.3.0 protobuf==5.29.5 !uv pip install --upgrade openpipe-art[backend,langgraph]==0.4.11 langchain-core langgraph langchain_openai tenacity datasets pillow==11.3.0 protobuf==5.29.5 {get_vllm} {get_numpy} --prerelease allow --no-cache-dir",+ ]+ assert result.needs_subprocess is False++def test_transform_duplicate_definitions():sources = ["a = 1",@@ -404,15 +427,35 @@ def test_transform_exclamation_mark_complex():def test_transform_exclamation_mark_with_indentation():+ # Indented pip installs get pass, non-pip commands use subprocesssources = ["if True:\n !pip install numpy","for i in range(10):\n !echo test",]result = transform_exclamation_mark(sources)+ # Pip gets pass + package management, echo uses subprocessassert result.transformed_sources == [- "if True:\n # packages added via marimo's package management: numpy !pip install numpy",+ "if True:\n pass # packages added via marimo's package management: numpy !pip install numpy","for i in range(10):\n #! echo test\n subprocess.call(['echo', 'test'])",]+ # Packages extracted even when indented+ assert result.pip_packages == ["numpy"]+ assert result.needs_subprocess is True+++def test_transform_exclamation_mark_indented_multiline():+ """Test indented multi-line command with backslash continuation"""+ sources = ["if something:\n !echo 12\\\n 34"]+ result = transform_exclamation_mark(sources)+ # Should handle backslash continuation and preserve indentation+ assert "subprocess.call" in result.transformed_sources[0]+ assert "echo" in result.transformed_sources[0]+ assert result.needs_subprocess is True+ # Verify indentation is preserved for the subprocess call+ assert " subprocess.call" in result.transformed_sources[0]+ # The command should have combined "12" and "34" after removing continuation+ assert "12" in result.transformed_sources[0]+ assert "34" in result.transformed_sources[0]def test_transform_exclamation_mark_preserves_comments():@@ -514,6 +557,21 @@ def test_transform_exclamation_in_multiline_string():assert result.transformed_sources[0] == sources[0]+def test_transform_exclamation_mark_pip_in_string():+ """Ensure !pip install inside strings is NOT transformed"""+ sources = [+ '''"""+!pip install numpy+"""'''+ ]+ result = transform_exclamation_mark(sources)+ # Should NOT be transformed - it's inside a string+ assert result.transformed_sources[0] == sources[0]+ # Should NOT extract packages from strings+ assert result.pip_packages == []+ assert result.needs_subprocess is False++def test_transform_duplicate_definitions_complex():sources = ["x = 1 # comment unaffected",@@ -578,13 +636,82 @@ def test_transform_exclamation_mark_with_variables():sources = ["package = 'numpy'\n!pip install {package}","command = 'echo \"Hello, World!\"'\n!{command}",+ "value = 42\n!echo {value}",]result = transform_exclamation_mark(sources)- # These get transformed (the {var} syntax won't work in subprocess.call anyway)+ # These get transformed (template variables become str() calls in subprocess)assert result.transformed_sources == ["package = 'numpy'\n# packages added via marimo's package management: {package} !pip install {package}",- "command = 'echo \"Hello, World!\"'\n#! {command}\nsubprocess.call(['{command}'])",+ "command = 'echo \"Hello, World!\"'\n#! {command}\nsubprocess.call([str(command)])",+ "value = 42\n#! echo {value}\nsubprocess.call(['echo', str(value)])",+ ]+++def test_transform_exclamation_mark_with_invalid_expressions():+ # Test that invalid Python expressions in templates are caught+ # Most syntax errors fail tokenization and remain unchanged+ # But some pass tokenization and are caught by compilation check+ sources = [+ "!wget {1+*2}", # Invalid operator sequence - caught by compilation check+ "!echo {1/0}", # Valid syntax but would fail at runtime - still converts+ ]+ result = transform_exclamation_mark(sources)+ # First is invalid and gets pass (always for invalid commands)+ # Second is valid Python syntax (runtime error is OK)+ assert result.transformed_sources == [+ "# Note: Command contains invalid template expression\n# !wget {1+*2}",+ "#! echo {1/0}\nsubprocess.call(['echo', str(1/0)])",+ ]+ assert result.pip_packages == []+ # Second command needs subprocess+ assert result.needs_subprocess is True+++def test_transform_exclamation_mark_with_safe_templates():+ # Test that safe template placeholders are converted to str() calls+ sources = [+ "!echo {value}",+ "!ls {directory}",+ "!cat {config[file]}", # Note: shlex removes quotes, so use unquoted]+ result = transform_exclamation_mark(sources)+ # Safe templates are converted to str() calls+ assert result.transformed_sources == [+ "#! echo {value}\nsubprocess.call(['echo', str(value)])",+ "#! ls {directory}\nsubprocess.call(['ls', str(directory)])",+ "#! cat {config[file]}\nsubprocess.call(['cat', str(config[file])])",+ ]+ assert result.needs_subprocess is True+++def test_transform_exclamation_mark_indented_pip_install():+ """Test that indented pip installs use package management with pass"""+ sources = [+ "if True:\n !python -m uv pip install --upgrade weave\n\nprint('done')"+ ]+ result = transform_exclamation_mark(sources)+ # Should use marimo's package management with pass+ assert (+ "pass # packages added via marimo's package management"+ in result.transformed_sources[0]+ )+ assert "weave" in result.transformed_sources[0]+ assert result.pip_packages == ["weave"] # Packages extracted+ assert result.needs_subprocess is False+++def test_transform_exclamation_mark_indented_invalid_command():+ """Test that indented invalid commands get pass statement"""+ sources = ["if False:\n !wget {1+*2}\n\nprint('done')"]+ result = transform_exclamation_mark(sources)+ # Invalid expression in indented context gets pass statement+ assert "pass # !wget {1+*2}" in result.transformed_sources[0]+ assert (+ "Note: Command contains invalid template expression"+ in result.transformed_sources[0]+ )+ assert result.pip_packages == []+ assert result.needs_subprocess is Falsedef test_transform_duplicate_definitions_with_comprehensions():