A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix(directory-scanner): detect marimo notebooks with long module docstrings (#9647) (#9652)
Myles Scolnick4a6fb52b0267
diff --git a/marimo/_server/files/directory_scanner.py b/marimo/_server/files/directory_scanner.pyindex 4c04739e08..447cc05d6d 100644--- a/marimo/_server/files/directory_scanner.py+++ b/marimo/_server/files/directory_scanner.py@@ -19,42 +19,49 @@ def is_marimo_app(full_path: str) -> bool:Detect whether a file is a marimo app.Rules:- - Markdown (`.md`/`.qmd`) files are marimo apps if the first 512 bytes- contain `marimo-version:`.- - Python (`.py`) files are marimo apps if the header (first 512 bytes)- contains both `marimo.App` and `import marimo`.- - If the header contains `# /// script`, read the full file and check for- the same Python markers, to handle large script headers.+ - Markdown (`.md`/`.qmd`) files are marimo apps if they contain+ `marimo-version:` (frontmatter marker).+ - Python (`.py`) files are marimo apps if they contain both+ `marimo.App` and `import marimo`.+ - In both cases the first 512 bytes are scanned first (fast path);+ on a miss we read up to 1 MB of the file looking for the markers.+ Above `import marimo` there's only ever a shebang, comments, a+ module docstring, and/or a `# /// script` block — none of which+ realistically exceed a few hundred KB.- Any errors while reading result in `False`."""- READ_LIMIT = 512-- def contains_marimo_app(content: bytes) -> bool:- return b"marimo.App" in content and b"import marimo" in content+ FAST_PATH_BYTES = 512+ # Cap on how far we'll read looking for markers. Marimo notebooks+ # put `import marimo` near the top of the file, so this is just a+ # guard against scanning huge unrelated Python files in full.+ MAX_SCAN_BYTES = 1 * 1024 * 1024 # 1 MBtry:path = MarimoPath(full_path)- # Fast extension check to avoid I/O for unrelated files- if not path.is_python() and not path.is_markdown():+ # Fast extension check to avoid I/O for unrelated files.+ if path.is_markdown():+ markers: tuple[bytes, ...] = (b"marimo-version:",)+ elif path.is_python():+ markers = (b"import marimo", b"marimo.App")+ else:return False- with open(full_path, "rb") as f:- header = f.read(READ_LIMIT)-- if path.is_markdown():- return b"marimo-version:" in header+ def matches(content: bytes) -> bool:+ return all(m in content for m in markers)- if path.is_python():- if contains_marimo_app(header):+ with open(full_path, "rb") as f:+ header = f.read(FAST_PATH_BYTES)+ if matches(header):return True-- if b"# /// script" in header:- full_content = path.read_bytes()- if contains_marimo_app(full_content):- return True-- return False+ # Fast path missed. If the file is smaller than the window,+ # we've already seen everything.+ if len(header) < FAST_PATH_BYTES:+ return False+ # Read further, bounded by MAX_SCAN_BYTES. If markers are+ # past that, the file isn't shaped like a marimo notebook.+ rest = f.read(MAX_SCAN_BYTES - FAST_PATH_BYTES)+ return matches(header + rest)except Exception as e:LOGGER.debug("Error reading file %s: %s", full_path, e)return Falsediff --git a/tests/_server/test_directory_scanner.py b/tests/_server/test_directory_scanner.pyindex 6f08f46806..c761d3f47c 100644--- a/tests/_server/test_directory_scanner.py+++ b/tests/_server/test_directory_scanner.py@@ -2,213 +2,190 @@from __future__ import annotationsimport os-import shutil-import tempfile-import unittest+from typing import TYPE_CHECKING++import pytestfrom marimo._server.files.directory_scanner import (DirectoryScanner,is_marimo_app,)+from marimo._server.models.files import FileInfo++if TYPE_CHECKING:+ from pathlib import Path++MARIMO_APP = "import marimo\napp = marimo.App()\n"+MARIMO_MD = "---\nmarimo-version: 0.1.0\n---\n"+++def _write(path: Path, content: str) -> Path:+ path.write_text(content)+ return path+++def _file_names(files: list[FileInfo]) -> list[str]:+ return [f.name for f in files if not f.is_directory]+++def _count_files(files: list[FileInfo]) -> int:+ return sum(+ _count_files(f.children or []) if f.is_directory else 1 for f in files+ )+++def test_python_app_detected(tmp_path: Path):+ f = _write(tmp_path / "app.py", MARIMO_APP)+ assert is_marimo_app(str(f))+++def test_python_non_app(tmp_path: Path):+ f = _write(tmp_path / "other.py", "import sys\nprint('hi')\n")+ assert not is_marimo_app(str(f))+++def test_markdown_app_detected(tmp_path: Path):+ f = _write(tmp_path / "notebook.md", MARIMO_MD)+ assert is_marimo_app(str(f))++def test_markdown_app_detected_with_long_frontmatter(tmp_path: Path):+ """Long YAML frontmatter must not hide the marimo-version marker."""+ padding = "\n".join(f"key{i}: value{i}" for i in range(50))+ content = f"---\n{padding}\nmarimo-version: 0.1.0\n---\n"+ assert len(content.encode()) > 512 # ensure we exercise the slow path+ f = _write(tmp_path / "notebook.md", content)+ assert is_marimo_app(str(f))-def test_python_app_detected():- """Test that Python marimo apps are detected."""- with tempfile.NamedTemporaryFile(- mode="w", suffix=".py", delete=False- ) as f:- f.write("import marimo\napp = marimo.App()\n")- f.flush()- assert is_marimo_app(f.name)- os.unlink(f.name)---def test_python_non_app():- """Test that non-marimo Python files return False."""- with tempfile.NamedTemporaryFile(- mode="w", suffix=".py", delete=False- ) as f:- f.write("import sys\nprint('hello')\n")- f.flush()- assert not is_marimo_app(f.name)- os.unlink(f.name)---def test_markdown_app_detected():- """Test that markdown files with marimo-version are detected."""- with tempfile.NamedTemporaryFile(- mode="w", suffix=".md", delete=False- ) as f:- f.write("---\nmarimo-version: 0.1.0\n---\n")- f.flush()- assert is_marimo_app(f.name)- os.unlink(f.name)---class TestDirectoryScanner(unittest.TestCase):- def setUp(self):- self.temp_root = tempfile.mkdtemp()- self.test_dir = os.path.join(self.temp_root, "test_directory")- os.makedirs(self.test_dir)-- # Create marimo files- with open(os.path.join(self.test_dir, "app1.py"), "w") as f:- f.write("import marimo\napp = marimo.App()\n")- with open(os.path.join(self.test_dir, "app2.py"), "w") as f:- f.write("import marimo\napp = marimo.App()\n")-- # Create markdown file- with open(os.path.join(self.test_dir, "notebook.md"), "w") as f:- f.write("---\nmarimo-version: 0.1.0\n---\n")-- # Create nested directory with marimo file- self.nested_dir = os.path.join(self.test_dir, "nested")- os.makedirs(self.nested_dir)- with open(os.path.join(self.nested_dir, "nested_app.py"), "w") as f:- f.write("import marimo\napp = marimo.App()\n")-- def tearDown(self):- shutil.rmtree(self.temp_root)-- def test_basic_scan(self):- """Test basic directory scanning."""- scanner = DirectoryScanner(self.test_dir)- files = scanner.scan()- file_names = [f.name for f in files if not f.is_directory]- assert len(file_names) == 2- assert "app1.py" in file_names- assert "app2.py" in file_names-- def test_scan_with_markdown(self):- """Test scanning with markdown files included."""- scanner = DirectoryScanner(self.test_dir, include_markdown=True)- files = scanner.scan()- file_names = [f.name for f in files if not f.is_directory]- assert len(file_names) == 3- assert "notebook.md" in file_names-- def test_scan_nested_directories(self):- """Test scanning nested directories."""- scanner = DirectoryScanner(self.test_dir)- files = scanner.scan()- nested_dirs = [++def test_python_app_detected_with_long_docstring(tmp_path: Path):+ """Long module docstrings must not hide marimo markers (issue #9647)."""+ f = _write(tmp_path / "app.py", f'"""{"x" * 1024}"""\n' + MARIMO_APP)+ assert is_marimo_app(str(f))+++def test_python_app_detected_with_script_header(tmp_path: Path):+ """PEP 723 script headers larger than the fast-path window are still detected."""+ padding = "# " + ("x" * 80) + "\n"+ header = "# /// script\n" + (padding * 10) + "# ///\n"+ f = _write(tmp_path / "app.py", header + MARIMO_APP)+ assert is_marimo_app(str(f))+++def test_python_non_app_with_long_content(tmp_path: Path):+ """Slow-path scan still rejects non-marimo Python files."""+ content = '"""' + ("x" * 1024) + '"""\nimport sys\nprint("hi")\n'+ f = _write(tmp_path / "other.py", content)+ assert not is_marimo_app(str(f))+++def test_dir(tmp_path: Path) -> Path:+ """Fixture dir: two marimo apps, a markdown notebook, and a nested app."""+ _write(tmp_path / "app1.py", MARIMO_APP)+ _write(tmp_path / "app2.py", MARIMO_APP)+ _write(tmp_path / "notebook.md", MARIMO_MD)+ nested = tmp_path / "nested"+ nested.mkdir()+ _write(nested / "nested_app.py", MARIMO_APP)+ return tmp_path+++class TestDirectoryScanner:+ def test_basic_scan(self, test_dir: Path):+ files = DirectoryScanner(str(test_dir)).scan()+ assert set(_file_names(files)) == {"app1.py", "app2.py"}++ def test_scan_with_markdown(self, test_dir: Path):+ files = DirectoryScanner(str(test_dir), include_markdown=True).scan()+ assert set(_file_names(files)) == {+ "app1.py",+ "app2.py",+ "notebook.md",+ }++ def test_scan_nested_directories(self, test_dir: Path):+ files = DirectoryScanner(str(test_dir)).scan()+ nested = next(f for f in files if f.is_directory and f.name == "nested"- ]- assert len(nested_dirs) == 1- assert nested_dirs[0].children is not None- assert nested_dirs[0].children[0].name == "nested_app.py"+ )+ assert nested.children is not None+ assert nested.children[0].name == "nested_app.py"- def test_max_files_limit(self):- """Test that max_files limit is enforced."""+ def test_max_files_limit(self, test_dir: Path):for i in range(10):- with open(os.path.join(self.test_dir, f"app{i + 3}.py"), "w") as f:- f.write("import marimo\napp = marimo.App()\n")-- scanner = DirectoryScanner(self.test_dir, max_files=5)- files = scanner.scan()-- def count_files(file_list: list) -> int:- total = 0- for f in file_list:- if f.is_directory:- if f.children:- total += count_files(f.children)- else:- total += 1- return total-- assert count_files(files) == 5-- def test_skip_common_directories(self):- """Test that common directories are skipped."""- for dirname in [+ _write(test_dir / f"app{i + 3}.py", MARIMO_APP)+ files = DirectoryScanner(str(test_dir), max_files=5).scan()+ assert _count_files(files) == 5++ def test_skip_common_directories(self, test_dir: Path):+ skip_names = ("venv","node_modules","__pycache__",".git","winpython",- ]:- skip_dir = os.path.join(self.test_dir, dirname)- os.makedirs(skip_dir)- with open(os.path.join(skip_dir, "app.py"), "w") as f:- f.write("import marimo\napp = marimo.App()\n")-- scanner = DirectoryScanner(self.test_dir)- files = scanner.scan()- file_paths = [f.path for f in files if not f.is_directory]- for path in file_paths:- assert "venv" not in path- assert "node_modules" not in path- assert "__pycache__" not in path- assert ".git" not in path- assert "winpython" not in path-- def test_skip_common_directories_case_insensitive(self):- """Test that SKIP_DIRS matching is case-insensitive."""- # WinPython distributions often use varied casing like "WinPython"- skip_dir = os.path.join(self.test_dir, "WinPython")- os.makedirs(skip_dir)- with open(os.path.join(skip_dir, "app.py"), "w") as f:- f.write("import marimo\napp = marimo.App()\n")-- scanner = DirectoryScanner(self.test_dir)- files = scanner.scan()- file_paths = [f.path for f in files if not f.is_directory]- for path in file_paths:- assert "winpython" not in path.lower()-- def test_skip_hidden_files(self):- """Test that hidden files are skipped."""- with open(os.path.join(self.test_dir, ".hidden_app.py"), "w") as f:- f.write("import marimo\napp = marimo.App()\n")-- scanner = DirectoryScanner(self.test_dir)- files = scanner.scan()- file_names = [f.name for f in files if not f.is_directory]- assert ".hidden_app.py" not in file_names-- def test_skip_symlinks(self):- """Test that symlinks are skipped to avoid cycles and broken links."""+ )+ for name in skip_names:+ d = test_dir / name+ d.mkdir()+ _write(d / "app.py", MARIMO_APP)+ scanner = DirectoryScanner(str(test_dir))+ scanner.scan()+ # Skipped dirs must not contribute marimo files anywhere in the tree.+ for f in scanner.partial_results:+ for name in skip_names:+ assert name not in f.path++ def test_skip_common_directories_case_insensitive(self, test_dir: Path):+ """WinPython distributions often vary case."""+ d = test_dir / "WinPython"+ d.mkdir()+ _write(d / "app.py", MARIMO_APP)+ files = DirectoryScanner(str(test_dir)).scan()+ for f in files:+ assert "winpython" not in f.path.lower()++ def test_skip_hidden_files(self, test_dir: Path):+ _write(test_dir / ".hidden_app.py", MARIMO_APP)+ files = DirectoryScanner(str(test_dir)).scan()+ assert ".hidden_app.py" not in _file_names(files)++ def test_skip_symlinks(self, test_dir: Path):+ """Broken links, cycles, and valid links are all skipped."""try:- # Create a symlink that points to a non-existent target (broken)- broken_link = os.path.join(self.test_dir, "broken_link.py")- broken_target = os.path.join(self.test_dir, "nonexistent_app.py")- os.symlink(broken_target, broken_link)-- # Create a directory symlink that would cause a cycle- cycle_link = os.path.join(self.test_dir, "cycle")- os.symlink(self.test_dir, cycle_link, target_is_directory=True)-- # Create a symlink to a valid marimo file- valid_target = os.path.join(self.test_dir, "app1.py")- valid_link = os.path.join(self.test_dir, "linked_app.py")- os.symlink(valid_target, valid_link)+ os.symlink(+ test_dir / "nonexistent.py", test_dir / "broken_link.py"+ )+ os.symlink(test_dir, test_dir / "cycle", target_is_directory=True)+ os.symlink(test_dir / "app1.py", test_dir / "linked_app.py")except (OSError, NotImplementedError):- self.skipTest("Symlinks not supported on this platform")+ pytest.skip("Symlinks not supported on this platform")- scanner = DirectoryScanner(self.test_dir)- files = scanner.scan()- file_names = [f.name for f in files if not f.is_directory]+ files = DirectoryScanner(str(test_dir)).scan()+ names = _file_names(files)dir_names = [f.name for f in files if f.is_directory]- # Symlinked files and directories should be skipped- assert "broken_link.py" not in file_names- assert "linked_app.py" not in file_names+ assert "broken_link.py" not in names+ assert "linked_app.py" not in namesassert "cycle" not in dir_names-- # Original files should still be found- assert "app1.py" in file_names- assert "app2.py" in file_names-- def test_partial_results_populated_during_scan(self):- """Test that partial_results is populated during scanning."""- scanner = DirectoryScanner(self.test_dir)- # partial_results starts empty+ assert "app1.py" in names+ assert "app2.py" in names++ def test_scan_includes_notebook_with_long_docstring(self, test_dir: Path):+ """Folder scans surface notebooks with long docstrings (issue #9647)."""+ _write(+ test_dir / "long_docstring_app.py",+ f'"""{"x" * 1024}"""\n' + MARIMO_APP,+ )+ files = DirectoryScanner(str(test_dir)).scan()+ assert "long_docstring_app.py" in _file_names(files)++ def test_partial_results_populated_during_scan(self, test_dir: Path):+ scanner = DirectoryScanner(str(test_dir))assert scanner.partial_results == []- files = scanner.scan()- # After scan, partial_results contains all found files (flat list)+ scanner.scan()assert len(scanner.partial_results) >= 2- # All items in partial_results should be non-directory filesfor f in scanner.partial_results:assert not f.is_directoryassert f.is_marimo_file