Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

feat: Add !pip metadata from ipynb convert and handle ! commands (#7141)

Dylan Madisetti7f4f695cee18
diff --git a/marimo/_convert/ipynb.py b/marimo/_convert/ipynb.py
index 9d2e871b3a..6af5f4b0c4 100644
--- a/marimo/_convert/ipynb.py
+++ b/marimo/_convert/ipynb.py
@@ -124,6 +124,43 @@ def transform_add_marimo_import(sources: list[CodeCell]) -> list[CodeCell]:
return sources
+def transform_add_subprocess_import(
+ sources: list[CodeCell], exclamation_metadata: ExclamationMarkResult | None
+) -> list[CodeCell]:
+ """
+ Add an import statement for subprocess if any cell uses subprocess.call.
+ """
+ # Check if subprocess import is needed
+ if not exclamation_metadata or not exclamation_metadata.needs_subprocess:
+ return sources
+
+ def has_subprocess_import(cell: str) -> bool:
+ # Quick check
+ if "import subprocess" not in cell:
+ return False
+
+ def is_in_import_line(line: str) -> bool:
+ stripped = line.strip()
+ if stripped.startswith("import subprocess"):
+ return True
+ return False
+
+ # Slow check
+ lines = cell.strip().split("\n")
+ if any(is_in_import_line(line) for line in lines):
+ return True
+ return False
+
+ already_has_subprocess_import = any(
+ has_subprocess_import(cell.source) for cell in sources
+ )
+ if already_has_subprocess_import:
+ return sources
+
+ # Add subprocess import at the beginning
+ return [CodeCell("import subprocess")] + sources
+
+
def transform_magic_commands(sources: list[str]) -> list[str]:
"""
Transform Jupyter magic commands to their marimo equivalents
@@ -348,20 +385,199 @@ def transform_magic_commands(sources: list[str]) -> list[str]:
return [transform(cell) for cell in sources]
-def transform_exclamation_mark(sources: list[str]) -> list[str]:
+@dataclass
+class ExclamationCommandResult:
+ """Result of processing a single exclamation command."""
+
+ replacement: str
+ pip_packages: list[str]
+ needs_subprocess: bool
+
+
+@dataclass
+class ExclamationMarkResult:
+ """Result of processing all exclamation mark commands in sources."""
+
+ transformed_sources: list[str]
+ pip_packages: list[str]
+ needs_subprocess: bool
+
+
+def _extract_pip_install(
+ command_line: str, command_tokens: list[str]
+) -> ExclamationCommandResult:
+ pip_packages: list[str] = []
+ if "install" not in command_tokens:
+ return _shlex_to_subprocess_call(command_line, command_tokens)
+
+ install_idx = command_tokens.index("install")
+ packages = [
+ p for p in command_tokens[install_idx + 1 :] if not p.startswith("-")
+ ]
+ pip_packages = packages
+
+ # Comment out the pip command
+ replacement = (
+ "# packages added via marimo's package management: "
+ f"{' '.join(packages)} !{command_line}"
+ )
+ return ExclamationCommandResult(replacement, pip_packages, False)
+
+
+def _shlex_to_subprocess_call(
+ command_line: str, command_tokens: list[str]
+) -> ExclamationCommandResult:
+ """Convert a shell command to subprocess.call([...])"""
+ command = "\n".join(
+ [f"#! {command_line}", f"subprocess.call({command_tokens!r})"]
+ )
+ return ExclamationCommandResult(command, [], True)
+
+
+def _handle_exclamation_command(
+ command_line: str,
+) -> ExclamationCommandResult:
"""
- Handle exclamation mark commands.
+ Process an exclamation command line.
+
+ Returns: (replacement_text, pip_packages, needs_subprocess)
"""
+ import shlex
- def transform(cell: str) -> str:
- if "!pip" in cell:
- cell = cell.replace(
- "!pip",
- "# (use marimo's built-in package management features instead) !pip", # noqa: E501
- )
- return cell
+ # Split command to check first token
+ try:
+ command_tokens = shlex.split(command_line)
+ except ValueError:
+ command_tokens = command_line.split()
- return [transform(cell) for cell in sources]
+ if not command_tokens:
+ return ExclamationCommandResult(f"# !{command_line}", [], False)
+
+ if command_tokens[0].startswith("pip"):
+ return _extract_pip_install(command_line, command_tokens)
+ # Replace with subprocess.call()
+ return _shlex_to_subprocess_call(command_line, command_tokens)
+
+
+def transform_exclamation_mark(sources: list[str]) -> ExclamationMarkResult:
+ """
+ Implementation of exclamation mark transform.
+
+ Uses tokenization to detect: newline + "!" + optional whitespace + command
+ """
+ import io
+ from tokenize import (
+ COMMENT,
+ ENCODING,
+ ERRORTOKEN,
+ INDENT,
+ NEWLINE,
+ NL,
+ OP,
+ TokenInfo,
+ tokenize,
+ )
+
+ all_pip_packages: list[str] = []
+ any_needs_subprocess = False
+ transformed_sources: list[str] = []
+
+ for cell in sources:
+ try:
+ tokens = list(tokenize(io.BytesIO(cell.encode("utf-8")).readline))
+ except Exception:
+ # If tokenization fails, return cell unchanged
+ transformed_sources.append(cell)
+ continue
+
+ # Track which lines have ! commands and their replacements
+ line_replacements: dict[int, str] = {} # line_num -> replacement_text
+ in_exclaim = False
+ exclaim_tokens: list[TokenInfo] = []
+ trailing_comment = None
+ exclaim_line_num = None
+
+ for i, token in enumerate(tokens):
+ # Check for newline + ! pattern (or ! at start after ENCODING)
+ # ! is OP in Python 3.12+ and ERRORTOKEN in Python 3.10/3.11
+ # Can also appear after INDENT for indented cells
+ if (
+ token.string == "!"
+ and token.type in (OP, ERRORTOKEN)
+ and i > 0
+ and tokens[i - 1].type in (NEWLINE, NL, ENCODING, INDENT)
+ ):
+ in_exclaim = True
+ exclaim_tokens = []
+ trailing_comment = None
+ exclaim_line_num = token.start[0]
+ continue # Skip the ! token
+
+ elif in_exclaim:
+ if token.type in (NEWLINE, NL):
+ # End of exclamation command - process it
+ # Check if last token was a comment
+ if exclaim_tokens and exclaim_tokens[-1].type == COMMENT:
+ trailing_comment = exclaim_tokens.pop()
+
+ # Extract command from collected tokens by joining their strings
+ if exclaim_tokens:
+ # Reconstruct command from tokens
+ first_token = exclaim_tokens[0]
+ last_token = exclaim_tokens[-1]
+ # Get the source line and extract the command portion
+ line_text = first_token.line
+ start_col = first_token.start[1]
+ end_col = last_token.end[1]
+ command_line = line_text[start_col:end_col].strip()
+ else:
+ command_line = ""
+
+ result = _handle_exclamation_command(command_line)
+
+ all_pip_packages.extend(result.pip_packages)
+ any_needs_subprocess |= result.needs_subprocess
+
+ if exclaim_line_num is not None:
+ line_replacements[exclaim_line_num] = (
+ result.replacement
+ )
+ # Store replacement for this line
+ if trailing_comment:
+ line_replacements[exclaim_line_num] += (
+ "\n" + trailing_comment.string
+ )
+
+ in_exclaim = False
+ else:
+ # Collect tokens that are part of the command
+ exclaim_tokens.append(token)
+
+ # Apply replacements to the cell
+ if line_replacements:
+ lines = cell.split("\n")
+ new_lines = []
+ for line_num, line in enumerate(lines, start=1):
+ if line_num in line_replacements:
+ # Preserve indentation from original line
+ indent = (len(line) - len(line.lstrip())) * " "
+ replacements = line_replacements[line_num].split("\n")
+ # Add indentation to replacement
+ indented_replacement = [
+ indent + replacement for replacement in replacements
+ ]
+ new_lines.extend(indented_replacement)
+ else:
+ new_lines.append(line)
+ transformed_sources.append("\n".join(new_lines))
+ else:
+ transformed_sources.append(cell)
+
+ return ExclamationMarkResult(
+ transformed_sources=transformed_sources,
+ pip_packages=all_pip_packages,
+ needs_subprocess=any_needs_subprocess,
+ )
class Renamer:
@@ -708,9 +924,47 @@ def extract_inline_meta(script: str) -> tuple[str | None, str]:
return None, script
+def build_metadata(
+ old_metadata: str | None,
+ extra_metadata: ExclamationMarkResult | None = None,
+) -> str:
+ """
+ Build PEP 723 metadata, adding pip packages if found.
+
+ Uses PyProjectReader utilities to parse existing metadata and properly
+ merge dependencies.
+ """
+ from marimo._utils.inline_script_metadata import PyProjectReader
+ from marimo._utils.scripts import write_pyproject_to_script
+
+ # If no exclamation mark processing happened or no packages found
+ if not extra_metadata or not extra_metadata.pip_packages:
+ return old_metadata or ""
+
+ pip_packages = extra_metadata.pip_packages
+
+ # Parse existing metadata if present
+ if old_metadata:
+ reader = PyProjectReader.from_script(old_metadata)
+ existing_deps = reader.dependencies
+
+ # Merge new pip packages with existing dependencies
+ merged_deps = list(set(existing_deps + pip_packages))
+
+ # Update the project dict with merged dependencies
+ project = reader.project.copy()
+ project["dependencies"] = merged_deps
+ else:
+ # Create new project dict
+ project = {"dependencies": pip_packages}
+
+ # Convert project dict to PEP 723 format using utility
+ return write_pyproject_to_script(project)
+
+
def _transform_sources(
sources: list[str], metadata: list[dict[str, Any]], hide_flags: list[bool]
-) -> list[CodeCell]:
+) -> tuple[list[CodeCell], ExclamationMarkResult | None]:
"""
Process raw sources and metadata into finalized cells.
@@ -720,6 +974,10 @@ def _transform_sources(
3. Cell-level transforms (e.g., inserting imports, removing empty cells)
After this step, cells are ready for execution or rendering.
+
+ Returns:
+ A tuple of (cells, exclamation_metadata) where exclamation_metadata
+ contains pip packages and subprocess import information.
"""
from marimo._convert.comment_preserver import CommentPreserver
@@ -727,10 +985,9 @@ def _transform_sources(
simple_transforms = [
transform_strip_whitespace,
transform_magic_commands,
- transform_exclamation_mark,
]
- # Define transforms that should preserve comments
+ # Define transforms that should preserve comments (excluding exclamation_mark)
comment_preserving_transforms = [
transform_remove_duplicate_imports,
transform_fixup_multiple_definitions,
@@ -757,18 +1014,19 @@ def _transform_sources(
)
sources = new_sources
+ # Handle exclamation_mark specially since it returns ExclamationMarkResult
+ exclamation_result = transform_exclamation_mark(sources)
+ sources = exclamation_result.transformed_sources
+ exclamation_metadata = exclamation_result
+
cells = bind_cell_metadata(sources, metadata, hide_flags)
# may change cell count
- cell_transforms: list[CellsTransform] = [
- transform_add_marimo_import,
- transform_remove_empty_cells,
- ]
+ cells = transform_add_subprocess_import(cells, exclamation_metadata)
+ cells = transform_add_marimo_import(cells)
+ cells = transform_remove_empty_cells(cells)
- for cell_transform in cell_transforms:
- cells = cell_transform(cells)
-
- return cells
+ return cells, exclamation_metadata
def convert_from_ipynb_to_notebook_ir(
@@ -801,11 +1059,13 @@ def convert_from_ipynb_to_notebook_ir(
metadata.append(cell.get("metadata", {}))
hide_flags.append(is_markdown)
- transformed_cells = _transform_sources(sources, metadata, hide_flags)
+ transformed_cells, extra_metadata = _transform_sources(
+ sources, metadata, hide_flags
+ )
return NotebookSerializationV1(
app=AppInstantiation(),
- header=Header(value=inline_meta or ""),
+ header=Header(value=build_metadata(inline_meta, extra_metadata)),
cells=[
CellDef(
code=cell.source,
diff --git a/marimo/_convert/markdown/markdown.py b/marimo/_convert/markdown/markdown.py
index 791f1443d7..6723fd781a 100644
--- a/marimo/_convert/markdown/markdown.py
+++ b/marimo/_convert/markdown/markdown.py
@@ -244,10 +244,12 @@ def _tree_to_ir(root: Element) -> SafeWrap[NotebookSerializationV1]:
)
sources.append(get_source_from_tag(child))
+ from marimo._utils.scripts import wrap_script_metadata
+
header = root.get("header", None)
pyproject = root.get("pyproject", None)
if pyproject and not header:
- header = "\n# ".join(["# /// script", *pyproject.splitlines(), "///"])
+ header = wrap_script_metadata(pyproject)
notebook = NotebookSerializationV1(
app=AppInstantiation(options=app_config),
cells=[
diff --git a/marimo/_utils/inline_script_metadata.py b/marimo/_utils/inline_script_metadata.py
index fbc24a78e5..d19ec8a4ab 100644
--- a/marimo/_utils/inline_script_metadata.py
+++ b/marimo/_utils/inline_script_metadata.py
@@ -217,14 +217,15 @@ def get_headers_from_markdown(contents: str) -> dict[str, str]:
def get_headers_from_frontmatter(
frontmatter: dict[str, Any],
) -> dict[str, str]:
+ from marimo._utils.scripts import wrap_script_metadata
+
headers = {"pyproject": "", "header": ""}
pyproject = frontmatter.get("pyproject", "")
if pyproject:
if not pyproject.startswith("#"):
- pyproject = "\n# ".join(
- [r"# /// script", *pyproject.splitlines(), r"///"]
- )
+ # Wrap raw TOML content in PEP 723 format
+ pyproject = wrap_script_metadata(pyproject)
headers["pyproject"] = pyproject
headers["header"] = frontmatter.get("header", "")
return headers
diff --git a/marimo/_utils/scripts.py b/marimo/_utils/scripts.py
index f1ea4a5145..393deca036 100644
--- a/marimo/_utils/scripts.py
+++ b/marimo/_utils/scripts.py
@@ -33,3 +33,61 @@ def read_pyproject_from_script(script: str) -> dict[str, Any] | None:
return pyproject
else:
return None
+
+
+def write_pyproject_to_script(project: dict[str, Any]) -> str:
+ """
+ Convert a project dict to PEP 723 inline script metadata format.
+
+ Adapted from https://peps.python.org/pep-0723/#reference-implementation
+
+ Args:
+ project: Dictionary containing project metadata (dependencies,
+ requires-python, etc.)
+
+ Returns:
+ PEP 723 formatted string with "# /// script" markers
+ """
+ from marimo._dependencies.dependencies import DependencyManager
+
+ # Convert project dict to TOML
+ if DependencyManager.tomlkit.has():
+ import tomlkit
+
+ toml_content = tomlkit.dumps(project)
+ else:
+ # Fallback to simple formatting if tomlkit not available
+ # This should rarely happen in practice
+ import json
+
+ lines = []
+ if "dependencies" in project:
+ lines.append(
+ f"dependencies = {json.dumps(project['dependencies'])}"
+ )
+ if "requires-python" in project:
+ lines.append(
+ f"requires-python = {json.dumps(project['requires-python'])}"
+ )
+ toml_content = "\n".join(lines) + "\n" if lines else ""
+
+ # Wrap in PEP 723 comment format
+ return wrap_script_metadata(toml_content)
+
+
+def wrap_script_metadata(toml_content: str) -> str:
+ """
+ Wrap TOML content in PEP 723 inline script metadata markers.
+
+ Args:
+ toml_content: Raw TOML content string
+
+ Returns:
+ PEP 723 formatted string with "# /// script" markers
+ """
+ result_lines = ["# /// script"]
+ for line in toml_content.rstrip().split("\n"):
+ result_lines.append(f"# {line}")
+ result_lines.append("# ///")
+
+ return "\n".join(result_lines)
diff --git a/scripts/pycheck.sh b/scripts/pycheck.sh
index 070b09ae49..116c1946fa 100755
--- a/scripts/pycheck.sh
+++ b/scripts/pycheck.sh
@@ -1,14 +1,14 @@
#!/bin/sh
echo "[check: typos]"
-hatch run typos -w
+uvx hatch run typos -w
echo "[check: copyright]"
./scripts/pycopyright.sh
echo "[check: lint]"
-hatch run ruff check --fix
+uvx hatch run ruff check --fix
echo "[check: format]"
-hatch run ruff format
+uvx hatch run ruff format
echo "[check: typecheck]"
-hatch run typecheck:check
+uvx hatch run typecheck:check
echo "[check: update-lock]"
-hatch run pixi lock
+uvx hatch run pixi lock
diff --git a/tests/_convert/snapshots/convert_pip_commands.py.txt b/tests/_convert/snapshots/convert_pip_commands.py.txt
index d4a568552f..0a966294d3 100644
--- a/tests/_convert/snapshots/convert_pip_commands.py.txt
+++ b/tests/_convert/snapshots/convert_pip_commands.py.txt
@@ -1,3 +1,7 @@
+# /// script
+# dependencies = ["transformers", "pandas", "numpy", "matplotlib", "scikit-learn", "seaborn"]
+# ///
+
import marimo
app = marimo.App()
@@ -5,33 +9,39 @@ app = marimo.App()
@app.cell
def _():
- # (use marimo's built-in package management features instead) !pip install transformers
+ import subprocess
+ return (subprocess,)
+
+
+def _():
+ # packages added via marimo's package management: transformers !pip install transformers
return
@app.cell
def _():
- # (use marimo's built-in package management features instead) !pip install pandas numpy matplotlib
+ # packages added via marimo's package management: pandas numpy matplotlib !pip install pandas numpy matplotlib
return
@app.cell
def _():
# Mixed cell with pip and other commands
- # (use marimo's built-in package management features instead) !pip install scikit-learn
+ # packages added via marimo's package management: scikit-learn !pip install scikit-learn
import numpy as np
- # (use marimo's built-in package management features instead) !pip install seaborn
+ # packages added via marimo's package management: seaborn !pip install seaborn
return
-app._unparsable_cell(
- r"""
+def _(subprocess):
# Non-pip exclamation commands should remain unchanged
- !ls -la
- !echo 'Hello World'
- """,
- name="_"
-)
+ #! ls -la
+ subprocess.call(['ls', '-la'])
+ #! echo 'Hello World'
+ subprocess.call(['echo', 'Hello World'])
+ return
@app.cell
diff --git a/tests/_convert/test_ipynb.py b/tests/_convert/test_ipynb.py
index af0e02d471..f22be0cca9 100644
--- a/tests/_convert/test_ipynb.py
+++ b/tests/_convert/test_ipynb.py
@@ -9,7 +9,9 @@ from marimo._convert.ipynb import (
CellsTransform,
CodeCell,
Transform,
+ convert_from_ipynb_to_notebook_ir,
transform_add_marimo_import,
+ transform_add_subprocess_import,
transform_duplicate_definitions,
transform_exclamation_mark,
transform_fixup_multiple_definitions,
@@ -261,10 +263,12 @@ def test_transform_exclamation_mark():
"!ls -l",
]
result = transform_exclamation_mark(sources)
- assert result == [
- "# (use marimo's built-in package management features instead) !pip install package", # noqa: E501
- "!ls -l",
+ assert result.transformed_sources == [
+ "# packages added via marimo's package management: package !pip install package",
+ "#! ls -l\nsubprocess.call(['ls', '-l'])",
]
+ assert result.pip_packages == ["package"]
+ assert result.needs_subprocess is True
def test_transform_duplicate_definitions():
@@ -381,18 +385,128 @@ def test_transform_magic_commands_complex():
def test_transform_exclamation_mark_complex():
sources = [
"!pip install package1 package2",
- "! ls -l | grep '.py'",
- "result = !echo 'Hello, World!'",
+ "!ls -l",
+ "!echo 'Hello, World!'",
"!python script.py arg1 arg2",
]
result = transform_exclamation_mark(sources)
- assert result == [
- "# (use marimo's built-in package management features instead) !pip install package1 package2", # noqa: E501
- # These are currently unhandled.
- "! ls -l | grep '.py'",
- "result = !echo 'Hello, World!'",
- "!python script.py arg1 arg2",
+ assert result.transformed_sources == [
+ "# packages added via marimo's package management: package1 package2 !pip install package1 package2",
+ "#! ls -l\nsubprocess.call(['ls', '-l'])",
+ "#! echo 'Hello, World!'\nsubprocess.call(['echo', 'Hello, World!'])",
+ "#! python script.py arg1 arg2\nsubprocess.call(['python', 'script.py', 'arg1', 'arg2'])",
+ ]
+
+
+def test_transform_exclamation_mark_with_indentation():
+ sources = [
+ "if True:\n !pip install numpy",
+ "for i in range(10):\n !echo test",
+ ]
+ result = transform_exclamation_mark(sources)
+ assert result.transformed_sources == [
+ "if True:\n # packages added via marimo's package management: numpy !pip install numpy",
+ "for i in range(10):\n #! echo test\n subprocess.call(['echo', 'test'])",
+ ]
+
+
+def test_transform_exclamation_mark_preserves_comments():
+ sources = [
+ "!ls -l # list files",
+ "!pip install numpy # install numpy",
]
+ result = transform_exclamation_mark(sources)
+ # Comments should be preserved on the next line
+ assert "# list files" in result.transformed_sources[0]
+ assert "subprocess.call" in result.transformed_sources[0]
+ assert "# install numpy" in result.transformed_sources[1]
+
+
+def test_transform_exclamation_mark_pip_variants():
+ sources = [
+ "!pip install numpy",
+ "!pip3 install pandas",
+ "!pip install -U matplotlib",
+ "!pip install --upgrade seaborn",
+ ]
+ result = transform_exclamation_mark(sources)
+ assert all("# packages" in r for r in result.transformed_sources)
+
+
+def test_transform_exclamation_mark_pip_non_install():
+ sources = [
+ "!pip upgrade",
+ "!pip --version",
+ "!pip",
+ ]
+ result = transform_exclamation_mark(sources)
+ # Non-install pip commands should be treated as subprocess calls
+ assert all(
+ "# packages" not in r and "#! pip" in r and "subprocess.call" in r
+ for r in result.transformed_sources
+ )
+
+
+def test_transform_exclamation_mark_not_at_line_start():
+ # ! must be at the start of a line (after newline)
+ sources = [
+ 'x = "!pip install numpy"', # In string
+ "y = 5 !", # Not at line start
+ ]
+ result = transform_exclamation_mark(sources)
+ # Should not transform these
+ assert result.transformed_sources[0] == 'x = "!pip install numpy"'
+ # Second one might cause tokenization error, so just check it doesn't crash
+
+
+def test_transform_exclamation_mark_multiline_cell():
+ sources = [
+ "import os\n!pip install numpy\nprint('done')",
+ "x = 1\n!ls -l\ny = 2",
+ ]
+ result = transform_exclamation_mark(sources)
+ assert (
+ "# packages added via marimo's package management:"
+ in result.transformed_sources[0]
+ )
+ assert "print('done')" in result.transformed_sources[0]
+ assert "subprocess.call" in result.transformed_sources[1]
+ assert "x = 1" in result.transformed_sources[1]
+ assert "y = 2" in result.transformed_sources[1]
+
+
+def test_transform_exclamation_mark_quoted_args():
+ sources = [
+ '!echo "hello world"',
+ "!grep 'pattern' file.txt",
+ ]
+ result = transform_exclamation_mark(sources)
+ assert result.transformed_sources == [
+ "#! echo \"hello world\"\nsubprocess.call(['echo', 'hello world'])",
+ "#! grep 'pattern' file.txt\nsubprocess.call(['grep', 'pattern', 'file.txt'])",
+ ]
+
+
+def test_transform_exclamation_mark_empty_command():
+ sources = [
+ "!",
+ "! ",
+ ]
+ result = transform_exclamation_mark(sources)
+ # Should handle gracefully
+ assert len(result.transformed_sources) == 2
+
+
+def test_transform_exclamation_in_multiline_string():
+ sources = [
+ """'''
+! this is just a string'''
+"""
+ ]
+ result = transform_exclamation_mark(sources)
+ # Should handle gracefully - returns unchanged source
+ assert len(result.transformed_sources) == 1
+ assert result.transformed_sources[0] == sources[0]
def test_transform_duplicate_definitions_complex():
@@ -454,14 +568,17 @@ def test_transform_magic_commands_unsupported():
def test_transform_exclamation_mark_with_variables():
+ # Note: Jupyter's {variable} interpolation in ! commands doesn't work in Python
+ # We transform them anyway since they're ! commands
sources = [
"package = 'numpy'\n!pip install {package}",
"command = 'echo \"Hello, World!\"'\n!{command}",
]
result = transform_exclamation_mark(sources)
- assert result == [
- "package = 'numpy'\n# (use marimo's built-in package management features instead) !pip install {package}", # noqa: E501
- "command = 'echo \"Hello, World!\"'\n!{command}",
+ # These get transformed (the {var} syntax won't work in subprocess.call anyway)
+ assert result.transformed_sources == [
+ "package = 'numpy'\n# packages added via marimo's package management: {package} !pip install {package}",
+ "command = 'echo \"Hello, World!\"'\n#! {command}\nsubprocess.call(['{command}'])",
]
@@ -889,3 +1006,173 @@ def test_transform_duplicate_definitions_numbered():
"df_3",
],
)
+
+
+def test_transform_add_subprocess_import():
+ # After transforming exclamation marks, subprocess import should be added
+ sources = [
+ "!ls -l",
+ "print('hello')",
+ ]
+ # First transform exclamation marks
+ exclamation_result = transform_exclamation_mark(sources)
+
+ # Then test subprocess import addition
+ cells = [
+ CodeCell("subprocess.call(['ls', '-l'])"),
+ CodeCell("print('hello')"),
+ ]
+ result = transform_add_subprocess_import(cells, exclamation_result)
+
+ # Should have subprocess import at the beginning
+ assert len(result) == 3
+ assert result[0].source == "import subprocess"
+ assert "subprocess.call" in result[1].source
+
+
+def test_transform_add_subprocess_import_already_exists():
+ sources = [
+ "!echo test",
+ ]
+ exclamation_result = transform_exclamation_mark(sources)
+
+ cells = [
+ CodeCell("import subprocess"),
+ CodeCell("subprocess.call(['echo', 'test'])"),
+ ]
+ result = transform_add_subprocess_import(cells, exclamation_result)
+
+ # Should not add duplicate import
+ assert len(result) == 2
+ assert result[0].source == "import subprocess"
+
+
+def test_transform_add_subprocess_import_not_needed():
+ sources = [
+ "!pip install numpy",
+ ]
+ exclamation_result = transform_exclamation_mark(sources)
+
+ cells = [
+ CodeCell(
+ "# packages added via marimo's package management: numpy !pip install numpy"
+ ),
+ ]
+ result = transform_add_subprocess_import(cells, exclamation_result)
+
+ # Should not add import since only pip commands were used
+ assert len(result) == 1
+
+
+def test_build_metadata_with_pip_packages():
+ import json
+
+ # Create a minimal notebook with pip install
+ notebook = {
+ "cells": [
+ {
+ "cell_type": "code",
+ "source": "!pip install numpy pandas",
+ "metadata": {},
+ }
+ ],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 2,
+ }
+
+ result = convert_from_ipynb_to_notebook_ir(json.dumps(notebook))
+
+ # Check that metadata contains the pip packages
+ assert 'dependencies = ["numpy", "pandas"]' in result.header.value
+
+
+def test_build_metadata_no_pip_packages():
+ import json
+
+ # Create a notebook without pip install
+ notebook = {
+ "cells": [
+ {
+ "cell_type": "code",
+ "source": "print('hello')",
+ "metadata": {},
+ }
+ ],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 2,
+ }
+
+ result = convert_from_ipynb_to_notebook_ir(json.dumps(notebook))
+
+ # Should not have metadata
+ assert result.header.value == ""
+
+
+def test_build_metadata_with_existing_metadata():
+ import json
+
+ # Create a notebook with existing PEP 723 metadata
+ notebook = {
+ "cells": [
+ {
+ "cell_type": "code",
+ "source": "# /// script\n# requires-python = '>=3.9'\n# ///\n\n!pip install requests",
+ "metadata": {},
+ }
+ ],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 2,
+ }
+
+ result = convert_from_ipynb_to_notebook_ir(json.dumps(notebook))
+
+ # Should add dependencies to existing metadata
+ assert 'dependencies = ["requests"]' in result.header.value
+ assert "requires-python" in result.header.value
+
+
+def test_integration_exclamation_marks_full_pipeline():
+ import json
+
+ # Test the full pipeline with mixed commands
+ notebook = {
+ "cells": [
+ {
+ "cell_type": "code",
+ "source": "!pip install numpy\nimport numpy as np",
+ "metadata": {},
+ },
+ {
+ "cell_type": "code",
+ "source": "!echo 'Processing data'",
+ "metadata": {},
+ },
+ {
+ "cell_type": "code",
+ "source": "result = np.array([1, 2, 3])",
+ "metadata": {},
+ },
+ ],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 2,
+ }
+
+ result = convert_from_ipynb_to_notebook_ir(json.dumps(notebook))
+
+ # Check metadata has pip packages
+ assert 'dependencies = ["numpy"]' in result.header.value
+
+ # Check subprocess import was added
+ sources = [cell.code for cell in result.cells]
+ assert any("import subprocess" in s for s in sources)
+
+ # Check transformations applied
+ assert any("subprocess.call" in s for s in sources)
+ assert any(
+ "# packages added via marimo's package management:" in s
+ for s in sources
+ )