A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
# Copyright 2024 Marimo. All rights reserved.
from __future__ import annotations
import os
from typing import TYPE_CHECKING
import pytest
from marimo._server.files.directory_scanner import (
DirectoryScanner,
is_marimo_app,
)
from marimo._server.models.files import FileInfo
if TYPE_CHECKING:
from pathlib import Path
MARIMO_APP = "import marimo\napp = marimo.App()\n"
MARIMO_MD = "---\nmarimo-version: 0.1.0\n---\n"
def _write(path: Path, content: str) -> Path:
path.write_text(content)
return path
def _file_names(files: list[FileInfo]) -> list[str]:
return [f.name for f in files if not f.is_directory]
def _count_files(files: list[FileInfo]) -> int:
return sum(
_count_files(f.children or []) if f.is_directory else 1 for f in files
)
def test_python_app_detected(tmp_path: Path):
f = _write(tmp_path / "app.py", MARIMO_APP)
assert is_marimo_app(str(f))
def test_python_non_app(tmp_path: Path):
f = _write(tmp_path / "other.py", "import sys\nprint('hi')\n")
assert not is_marimo_app(str(f))
def test_markdown_app_detected(tmp_path: Path):
f = _write(tmp_path / "notebook.md", MARIMO_MD)
assert is_marimo_app(str(f))
def test_markdown_app_detected_with_long_frontmatter(tmp_path: Path):
"""Long YAML frontmatter must not hide the marimo-version marker."""
padding = "\n".join(f"key{i}: value{i}" for i in range(50))
content = f"---\n{padding}\nmarimo-version: 0.1.0\n---\n"
assert len(content.encode()) > 512 # ensure we exercise the slow path
f = _write(tmp_path / "notebook.md", content)
assert is_marimo_app(str(f))
def test_python_app_detected_with_long_docstring(tmp_path: Path):
"""Long module docstrings must not hide marimo markers (issue #9647)."""
f = _write(tmp_path / "app.py", f'"""{"x" * 1024}"""\n' + MARIMO_APP)
assert is_marimo_app(str(f))
def test_python_app_detected_with_script_header(tmp_path: Path):
"""PEP 723 script headers larger than the fast-path window are still detected."""
padding = "# " + ("x" * 80) + "\n"
header = "# /// script\n" + (padding * 10) + "# ///\n"
f = _write(tmp_path / "app.py", header + MARIMO_APP)
assert is_marimo_app(str(f))
def test_python_non_app_with_long_content(tmp_path: Path):
"""Slow-path scan still rejects non-marimo Python files."""
content = '"""' + ("x" * 1024) + '"""\nimport sys\nprint("hi")\n'
f = _write(tmp_path / "other.py", content)
assert not is_marimo_app(str(f))
@pytest.fixture
def test_dir(tmp_path: Path) -> Path:
"""Fixture dir: two marimo apps, a markdown notebook, and a nested app."""
_write(tmp_path / "app1.py", MARIMO_APP)
_write(tmp_path / "app2.py", MARIMO_APP)
_write(tmp_path / "notebook.md", MARIMO_MD)
nested = tmp_path / "nested"
nested.mkdir()
_write(nested / "nested_app.py", MARIMO_APP)
return tmp_path
class TestDirectoryScanner:
def test_basic_scan(self, test_dir: Path):
files = DirectoryScanner(str(test_dir)).scan()
assert set(_file_names(files)) == {"app1.py", "app2.py"}
def test_scan_with_markdown(self, test_dir: Path):
files = DirectoryScanner(str(test_dir), include_markdown=True).scan()
assert set(_file_names(files)) == {
"app1.py",
"app2.py",
"notebook.md",
}
def test_scan_nested_directories(self, test_dir: Path):
files = DirectoryScanner(str(test_dir)).scan()
nested = next(
f for f in files if f.is_directory and f.name == "nested"
)
assert nested.children is not None
assert nested.children[0].name == "nested_app.py"
def test_max_files_limit(self, test_dir: Path):
for i in range(10):
_write(test_dir / f"app{i + 3}.py", MARIMO_APP)
files = DirectoryScanner(str(test_dir), max_files=5).scan()
assert _count_files(files) == 5
def test_max_files_limit_recursion_at_boundary(
self, test_dir: Path, monkeypatch: pytest.MonkeyPatch
):
"""A subdir recursion that reaches max_files must not let another
top-level file slip in. Forces the ordering that only flaked in CI."""
import marimo._server.files.directory_scanner as scanner_mod
for i in range(10):
_write(test_dir / f"app{i + 3}.py", MARIMO_APP)
real_scandir = os.scandir
def ordered_scandir(path: str): # type: ignore[no-untyped-def]
entries = list(real_scandir(path))
dirs = [e for e in entries if e.is_dir()]
files = sorted(
(e for e in entries if not e.is_dir()), key=lambda e: e.name
)
# 4 files, then the nested dir (reaches the limit), then the rest.
return files[:4] + dirs + files[4:]
monkeypatch.setattr(scanner_mod.os, "scandir", ordered_scandir)
files = DirectoryScanner(str(test_dir), max_files=5).scan()
assert _count_files(files) == 5
def test_skip_common_directories(self, test_dir: Path):
skip_names = (
"venv",
"node_modules",
"__pycache__",
".git",
"winpython",
)
for name in skip_names:
d = test_dir / name
d.mkdir()
_write(d / "app.py", MARIMO_APP)
scanner = DirectoryScanner(str(test_dir))
scanner.scan()
# Skipped dirs must not contribute marimo files anywhere in the tree.
for f in scanner.partial_results:
for name in skip_names:
assert name not in f.path
def test_skip_common_directories_case_insensitive(self, test_dir: Path):
"""WinPython distributions often vary case."""
d = test_dir / "WinPython"
d.mkdir()
_write(d / "app.py", MARIMO_APP)
files = DirectoryScanner(str(test_dir)).scan()
for f in files:
assert "winpython" not in f.path.lower()
def test_skip_hidden_files(self, test_dir: Path):
_write(test_dir / ".hidden_app.py", MARIMO_APP)
files = DirectoryScanner(str(test_dir)).scan()
assert ".hidden_app.py" not in _file_names(files)
def test_skip_symlinks(self, test_dir: Path):
"""Broken links, cycles, and valid links are all skipped."""
try:
os.symlink(
test_dir / "nonexistent.py", test_dir / "broken_link.py"
)
os.symlink(test_dir, test_dir / "cycle", target_is_directory=True)
os.symlink(test_dir / "app1.py", test_dir / "linked_app.py")
except (OSError, NotImplementedError):
pytest.skip("Symlinks not supported on this platform")
files = DirectoryScanner(str(test_dir)).scan()
names = _file_names(files)
dir_names = [f.name for f in files if f.is_directory]
assert "broken_link.py" not in names
assert "linked_app.py" not in names
assert "cycle" not in dir_names
assert "app1.py" in names
assert "app2.py" in names
def test_scan_includes_notebook_with_long_docstring(self, test_dir: Path):
"""Folder scans surface notebooks with long docstrings (issue #9647)."""
_write(
test_dir / "long_docstring_app.py",
f'"""{"x" * 1024}"""\n' + MARIMO_APP,
)
files = DirectoryScanner(str(test_dir)).scan()
assert "long_docstring_app.py" in _file_names(files)
def test_truncated_when_folders_are_deeper_than_max_depth(
self, tmp_path: Path
):
"""Notebooks past `max_depth` are dropped, but not silently (#10064).
The folder holding them looks exactly like an empty folder, so the
scanner has to report that it stopped looking.
"""
deep = tmp_path / "projects" / "analysis"
deep.mkdir(parents=True)
_write(deep / "app.py", MARIMO_APP)
scanner = DirectoryScanner(str(tmp_path), max_depth=1)
files = scanner.scan()
assert _count_files(files) == 0
assert scanner.truncated
def test_not_truncated_when_tree_is_fully_scanned(self, test_dir: Path):
scanner = DirectoryScanner(str(test_dir))
scanner.scan()
assert not scanner.truncated
def test_truncated_when_max_files_reached(self, test_dir: Path):
for i in range(10):
_write(test_dir / f"app{i + 3}.py", MARIMO_APP)
scanner = DirectoryScanner(str(test_dir), max_files=5)
scanner.scan()
assert scanner.truncated
def test_truncated_is_reset_between_scans(self, tmp_path: Path):
deep = tmp_path / "projects" / "analysis"
deep.mkdir(parents=True)
_write(deep / "app.py", MARIMO_APP)
scanner = DirectoryScanner(str(tmp_path), max_depth=1)
scanner.scan()
assert scanner.truncated
# Deep enough to reach the notebook: no truncation this time.
scanner.max_depth = 2
files = scanner.scan()
assert _count_files(files) == 1
assert not scanner.truncated
def test_truncated_when_an_entry_cannot_be_read(
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
):
"""An entry we fail to stat may have been a notebook, or a folder
holding notebooks, so it counts as a truncated scan too."""
import marimo._server.files.directory_scanner as scanner_mod
(tmp_path / "projects").mkdir()
_write(tmp_path / "projects" / "app.py", MARIMO_APP)
class UnreadableEntry:
name = "projects"
path = str(tmp_path / "projects")
def is_symlink(self) -> bool:
return False
def is_dir(self) -> bool:
raise PermissionError("denied")
monkeypatch.setattr(
scanner_mod.os, "scandir", lambda _path: iter([UnreadableEntry()])
)
scanner = DirectoryScanner(str(tmp_path))
files = scanner.scan()
assert files == []
assert scanner.truncated
def test_partial_results_populated_during_scan(self, test_dir: Path):
scanner = DirectoryScanner(str(test_dir))
assert scanner.partial_results == []
scanner.scan()
assert len(scanner.partial_results) >= 2
for f in scanner.partial_results:
assert not f.is_directory
assert f.is_marimo_file