Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix(directory-scanner): detect marimo notebooks with long module docstrings (#9647) (#9652)

Myles Scolnick4a6fb52b0267
diff --git a/marimo/_server/files/directory_scanner.py b/marimo/_server/files/directory_scanner.py
index 4c04739e08..447cc05d6d 100644
--- a/marimo/_server/files/directory_scanner.py
+++ b/marimo/_server/files/directory_scanner.py
@@ -19,42 +19,49 @@ def is_marimo_app(full_path: str) -> bool:
Detect whether a file is a marimo app.
Rules:
- - Markdown (`.md`/`.qmd`) files are marimo apps if the first 512 bytes
- contain `marimo-version:`.
- - Python (`.py`) files are marimo apps if the header (first 512 bytes)
- contains both `marimo.App` and `import marimo`.
- - If the header contains `# /// script`, read the full file and check for
- the same Python markers, to handle large script headers.
+ - Markdown (`.md`/`.qmd`) files are marimo apps if they contain
+ `marimo-version:` (frontmatter marker).
+ - Python (`.py`) files are marimo apps if they contain both
+ `marimo.App` and `import marimo`.
+ - In both cases the first 512 bytes are scanned first (fast path);
+ on a miss we read up to 1 MB of the file looking for the markers.
+ Above `import marimo` there's only ever a shebang, comments, a
+ module docstring, and/or a `# /// script` block — none of which
+ realistically exceed a few hundred KB.
- Any errors while reading result in `False`.
"""
- READ_LIMIT = 512
-
- def contains_marimo_app(content: bytes) -> bool:
- return b"marimo.App" in content and b"import marimo" in content
+ FAST_PATH_BYTES = 512
+ # Cap on how far we'll read looking for markers. Marimo notebooks
+ # put `import marimo` near the top of the file, so this is just a
+ # guard against scanning huge unrelated Python files in full.
+ MAX_SCAN_BYTES = 1 * 1024 * 1024 # 1 MB
try:
path = MarimoPath(full_path)
- # Fast extension check to avoid I/O for unrelated files
- if not path.is_python() and not path.is_markdown():
+ # Fast extension check to avoid I/O for unrelated files.
+ if path.is_markdown():
+ markers: tuple[bytes, ...] = (b"marimo-version:",)
+ elif path.is_python():
+ markers = (b"import marimo", b"marimo.App")
+ else:
return False
- with open(full_path, "rb") as f:
- header = f.read(READ_LIMIT)
-
- if path.is_markdown():
- return b"marimo-version:" in header
+ def matches(content: bytes) -> bool:
+ return all(m in content for m in markers)
- if path.is_python():
- if contains_marimo_app(header):
+ with open(full_path, "rb") as f:
+ header = f.read(FAST_PATH_BYTES)
+ if matches(header):
return True
-
- if b"# /// script" in header:
- full_content = path.read_bytes()
- if contains_marimo_app(full_content):
- return True
-
- return False
+ # Fast path missed. If the file is smaller than the window,
+ # we've already seen everything.
+ if len(header) < FAST_PATH_BYTES:
+ return False
+ # Read further, bounded by MAX_SCAN_BYTES. If markers are
+ # past that, the file isn't shaped like a marimo notebook.
+ rest = f.read(MAX_SCAN_BYTES - FAST_PATH_BYTES)
+ return matches(header + rest)
except Exception as e:
LOGGER.debug("Error reading file %s: %s", full_path, e)
return False
diff --git a/tests/_server/test_directory_scanner.py b/tests/_server/test_directory_scanner.py
index 6f08f46806..c761d3f47c 100644
--- a/tests/_server/test_directory_scanner.py
+++ b/tests/_server/test_directory_scanner.py
@@ -2,213 +2,190 @@
from __future__ import annotations
import os
-import shutil
-import tempfile
-import unittest
+from typing import TYPE_CHECKING
+
+import pytest
from marimo._server.files.directory_scanner import (
DirectoryScanner,
is_marimo_app,
)
+from marimo._server.models.files import FileInfo
+
+if TYPE_CHECKING:
+ from pathlib import Path
+
+MARIMO_APP = "import marimo\napp = marimo.App()\n"
+MARIMO_MD = "---\nmarimo-version: 0.1.0\n---\n"
+
+
+def _write(path: Path, content: str) -> Path:
+ path.write_text(content)
+ return path
+
+
+def _file_names(files: list[FileInfo]) -> list[str]:
+ return [f.name for f in files if not f.is_directory]
+
+
+def _count_files(files: list[FileInfo]) -> int:
+ return sum(
+ _count_files(f.children or []) if f.is_directory else 1 for f in files
+ )
+
+
+def test_python_app_detected(tmp_path: Path):
+ f = _write(tmp_path / "app.py", MARIMO_APP)
+ assert is_marimo_app(str(f))
+
+
+def test_python_non_app(tmp_path: Path):
+ f = _write(tmp_path / "other.py", "import sys\nprint('hi')\n")
+ assert not is_marimo_app(str(f))
+
+
+def test_markdown_app_detected(tmp_path: Path):
+ f = _write(tmp_path / "notebook.md", MARIMO_MD)
+ assert is_marimo_app(str(f))
+
+def test_markdown_app_detected_with_long_frontmatter(tmp_path: Path):
+ """Long YAML frontmatter must not hide the marimo-version marker."""
+ padding = "\n".join(f"key{i}: value{i}" for i in range(50))
+ content = f"---\n{padding}\nmarimo-version: 0.1.0\n---\n"
+ assert len(content.encode()) > 512 # ensure we exercise the slow path
+ f = _write(tmp_path / "notebook.md", content)
+ assert is_marimo_app(str(f))
-def test_python_app_detected():
- """Test that Python marimo apps are detected."""
- with tempfile.NamedTemporaryFile(
- mode="w", suffix=".py", delete=False
- ) as f:
- f.write("import marimo\napp = marimo.App()\n")
- f.flush()
- assert is_marimo_app(f.name)
- os.unlink(f.name)
-
-
-def test_python_non_app():
- """Test that non-marimo Python files return False."""
- with tempfile.NamedTemporaryFile(
- mode="w", suffix=".py", delete=False
- ) as f:
- f.write("import sys\nprint('hello')\n")
- f.flush()
- assert not is_marimo_app(f.name)
- os.unlink(f.name)
-
-
-def test_markdown_app_detected():
- """Test that markdown files with marimo-version are detected."""
- with tempfile.NamedTemporaryFile(
- mode="w", suffix=".md", delete=False
- ) as f:
- f.write("---\nmarimo-version: 0.1.0\n---\n")
- f.flush()
- assert is_marimo_app(f.name)
- os.unlink(f.name)
-
-
-class TestDirectoryScanner(unittest.TestCase):
- def setUp(self):
- self.temp_root = tempfile.mkdtemp()
- self.test_dir = os.path.join(self.temp_root, "test_directory")
- os.makedirs(self.test_dir)
-
- # Create marimo files
- with open(os.path.join(self.test_dir, "app1.py"), "w") as f:
- f.write("import marimo\napp = marimo.App()\n")
- with open(os.path.join(self.test_dir, "app2.py"), "w") as f:
- f.write("import marimo\napp = marimo.App()\n")
-
- # Create markdown file
- with open(os.path.join(self.test_dir, "notebook.md"), "w") as f:
- f.write("---\nmarimo-version: 0.1.0\n---\n")
-
- # Create nested directory with marimo file
- self.nested_dir = os.path.join(self.test_dir, "nested")
- os.makedirs(self.nested_dir)
- with open(os.path.join(self.nested_dir, "nested_app.py"), "w") as f:
- f.write("import marimo\napp = marimo.App()\n")
-
- def tearDown(self):
- shutil.rmtree(self.temp_root)
-
- def test_basic_scan(self):
- """Test basic directory scanning."""
- scanner = DirectoryScanner(self.test_dir)
- files = scanner.scan()
- file_names = [f.name for f in files if not f.is_directory]
- assert len(file_names) == 2
- assert "app1.py" in file_names
- assert "app2.py" in file_names
-
- def test_scan_with_markdown(self):
- """Test scanning with markdown files included."""
- scanner = DirectoryScanner(self.test_dir, include_markdown=True)
- files = scanner.scan()
- file_names = [f.name for f in files if not f.is_directory]
- assert len(file_names) == 3
- assert "notebook.md" in file_names
-
- def test_scan_nested_directories(self):
- """Test scanning nested directories."""
- scanner = DirectoryScanner(self.test_dir)
- files = scanner.scan()
- nested_dirs = [
+
+def test_python_app_detected_with_long_docstring(tmp_path: Path):
+ """Long module docstrings must not hide marimo markers (issue #9647)."""
+ f = _write(tmp_path / "app.py", f'"""{"x" * 1024}"""\n' + MARIMO_APP)
+ assert is_marimo_app(str(f))
+
+
+def test_python_app_detected_with_script_header(tmp_path: Path):
+ """PEP 723 script headers larger than the fast-path window are still detected."""
+ padding = "# " + ("x" * 80) + "\n"
+ header = "# /// script\n" + (padding * 10) + "# ///\n"
+ f = _write(tmp_path / "app.py", header + MARIMO_APP)
+ assert is_marimo_app(str(f))
+
+
+def test_python_non_app_with_long_content(tmp_path: Path):
+ """Slow-path scan still rejects non-marimo Python files."""
+ content = '"""' + ("x" * 1024) + '"""\nimport sys\nprint("hi")\n'
+ f = _write(tmp_path / "other.py", content)
+ assert not is_marimo_app(str(f))
+
+
+def test_dir(tmp_path: Path) -> Path:
+ """Fixture dir: two marimo apps, a markdown notebook, and a nested app."""
+ _write(tmp_path / "app1.py", MARIMO_APP)
+ _write(tmp_path / "app2.py", MARIMO_APP)
+ _write(tmp_path / "notebook.md", MARIMO_MD)
+ nested = tmp_path / "nested"
+ nested.mkdir()
+ _write(nested / "nested_app.py", MARIMO_APP)
+ return tmp_path
+
+
+class TestDirectoryScanner:
+ def test_basic_scan(self, test_dir: Path):
+ files = DirectoryScanner(str(test_dir)).scan()
+ assert set(_file_names(files)) == {"app1.py", "app2.py"}
+
+ def test_scan_with_markdown(self, test_dir: Path):
+ files = DirectoryScanner(str(test_dir), include_markdown=True).scan()
+ assert set(_file_names(files)) == {
+ "app1.py",
+ "app2.py",
+ "notebook.md",
+ }
+
+ def test_scan_nested_directories(self, test_dir: Path):
+ files = DirectoryScanner(str(test_dir)).scan()
+ nested = next(
f for f in files if f.is_directory and f.name == "nested"
- ]
- assert len(nested_dirs) == 1
- assert nested_dirs[0].children is not None
- assert nested_dirs[0].children[0].name == "nested_app.py"
+ )
+ assert nested.children is not None
+ assert nested.children[0].name == "nested_app.py"
- def test_max_files_limit(self):
- """Test that max_files limit is enforced."""
+ def test_max_files_limit(self, test_dir: Path):
for i in range(10):
- with open(os.path.join(self.test_dir, f"app{i + 3}.py"), "w") as f:
- f.write("import marimo\napp = marimo.App()\n")
-
- scanner = DirectoryScanner(self.test_dir, max_files=5)
- files = scanner.scan()
-
- def count_files(file_list: list) -> int:
- total = 0
- for f in file_list:
- if f.is_directory:
- if f.children:
- total += count_files(f.children)
- else:
- total += 1
- return total
-
- assert count_files(files) == 5
-
- def test_skip_common_directories(self):
- """Test that common directories are skipped."""
- for dirname in [
+ _write(test_dir / f"app{i + 3}.py", MARIMO_APP)
+ files = DirectoryScanner(str(test_dir), max_files=5).scan()
+ assert _count_files(files) == 5
+
+ def test_skip_common_directories(self, test_dir: Path):
+ skip_names = (
"venv",
"node_modules",
"__pycache__",
".git",
"winpython",
- ]:
- skip_dir = os.path.join(self.test_dir, dirname)
- os.makedirs(skip_dir)
- with open(os.path.join(skip_dir, "app.py"), "w") as f:
- f.write("import marimo\napp = marimo.App()\n")
-
- scanner = DirectoryScanner(self.test_dir)
- files = scanner.scan()
- file_paths = [f.path for f in files if not f.is_directory]
- for path in file_paths:
- assert "venv" not in path
- assert "node_modules" not in path
- assert "__pycache__" not in path
- assert ".git" not in path
- assert "winpython" not in path
-
- def test_skip_common_directories_case_insensitive(self):
- """Test that SKIP_DIRS matching is case-insensitive."""
- # WinPython distributions often use varied casing like "WinPython"
- skip_dir = os.path.join(self.test_dir, "WinPython")
- os.makedirs(skip_dir)
- with open(os.path.join(skip_dir, "app.py"), "w") as f:
- f.write("import marimo\napp = marimo.App()\n")
-
- scanner = DirectoryScanner(self.test_dir)
- files = scanner.scan()
- file_paths = [f.path for f in files if not f.is_directory]
- for path in file_paths:
- assert "winpython" not in path.lower()
-
- def test_skip_hidden_files(self):
- """Test that hidden files are skipped."""
- with open(os.path.join(self.test_dir, ".hidden_app.py"), "w") as f:
- f.write("import marimo\napp = marimo.App()\n")
-
- scanner = DirectoryScanner(self.test_dir)
- files = scanner.scan()
- file_names = [f.name for f in files if not f.is_directory]
- assert ".hidden_app.py" not in file_names
-
- def test_skip_symlinks(self):
- """Test that symlinks are skipped to avoid cycles and broken links."""
+ )
+ for name in skip_names:
+ d = test_dir / name
+ d.mkdir()
+ _write(d / "app.py", MARIMO_APP)
+ scanner = DirectoryScanner(str(test_dir))
+ scanner.scan()
+ # Skipped dirs must not contribute marimo files anywhere in the tree.
+ for f in scanner.partial_results:
+ for name in skip_names:
+ assert name not in f.path
+
+ def test_skip_common_directories_case_insensitive(self, test_dir: Path):
+ """WinPython distributions often vary case."""
+ d = test_dir / "WinPython"
+ d.mkdir()
+ _write(d / "app.py", MARIMO_APP)
+ files = DirectoryScanner(str(test_dir)).scan()
+ for f in files:
+ assert "winpython" not in f.path.lower()
+
+ def test_skip_hidden_files(self, test_dir: Path):
+ _write(test_dir / ".hidden_app.py", MARIMO_APP)
+ files = DirectoryScanner(str(test_dir)).scan()
+ assert ".hidden_app.py" not in _file_names(files)
+
+ def test_skip_symlinks(self, test_dir: Path):
+ """Broken links, cycles, and valid links are all skipped."""
try:
- # Create a symlink that points to a non-existent target (broken)
- broken_link = os.path.join(self.test_dir, "broken_link.py")
- broken_target = os.path.join(self.test_dir, "nonexistent_app.py")
- os.symlink(broken_target, broken_link)
-
- # Create a directory symlink that would cause a cycle
- cycle_link = os.path.join(self.test_dir, "cycle")
- os.symlink(self.test_dir, cycle_link, target_is_directory=True)
-
- # Create a symlink to a valid marimo file
- valid_target = os.path.join(self.test_dir, "app1.py")
- valid_link = os.path.join(self.test_dir, "linked_app.py")
- os.symlink(valid_target, valid_link)
+ os.symlink(
+ test_dir / "nonexistent.py", test_dir / "broken_link.py"
+ )
+ os.symlink(test_dir, test_dir / "cycle", target_is_directory=True)
+ os.symlink(test_dir / "app1.py", test_dir / "linked_app.py")
except (OSError, NotImplementedError):
- self.skipTest("Symlinks not supported on this platform")
+ pytest.skip("Symlinks not supported on this platform")
- scanner = DirectoryScanner(self.test_dir)
- files = scanner.scan()
- file_names = [f.name for f in files if not f.is_directory]
+ files = DirectoryScanner(str(test_dir)).scan()
+ names = _file_names(files)
dir_names = [f.name for f in files if f.is_directory]
- # Symlinked files and directories should be skipped
- assert "broken_link.py" not in file_names
- assert "linked_app.py" not in file_names
+ assert "broken_link.py" not in names
+ assert "linked_app.py" not in names
assert "cycle" not in dir_names
-
- # Original files should still be found
- assert "app1.py" in file_names
- assert "app2.py" in file_names
-
- def test_partial_results_populated_during_scan(self):
- """Test that partial_results is populated during scanning."""
- scanner = DirectoryScanner(self.test_dir)
- # partial_results starts empty
+ assert "app1.py" in names
+ assert "app2.py" in names
+
+ def test_scan_includes_notebook_with_long_docstring(self, test_dir: Path):
+ """Folder scans surface notebooks with long docstrings (issue #9647)."""
+ _write(
+ test_dir / "long_docstring_app.py",
+ f'"""{"x" * 1024}"""\n' + MARIMO_APP,
+ )
+ files = DirectoryScanner(str(test_dir)).scan()
+ assert "long_docstring_app.py" in _file_names(files)
+
+ def test_partial_results_populated_during_scan(self, test_dir: Path):
+ scanner = DirectoryScanner(str(test_dir))
assert scanner.partial_results == []
- files = scanner.scan()
- # After scan, partial_results contains all found files (flat list)
+ scanner.scan()
assert len(scanner.partial_results) >= 2
- # All items in partial_results should be non-directory files
for f in scanner.partial_results:
assert not f.is_directory
assert f.is_marimo_file