A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
improvement: add file limit, warning, and more performant file detection (#7368)
Myles Scolnick4062b0fd585e
diff --git a/frontend/src/components/pages/home-page.tsx b/frontend/src/components/pages/home-page.tsxindex 6d963b3ae8..83dad963fc 100644--- a/frontend/src/components/pages/home-page.tsx+++ b/frontend/src/components/pages/home-page.tsx@@ -169,6 +169,12 @@ const WorkspaceNotebooks: React.FC = () => {return (<WorkspaceRootContext value={workspace.root}><div className="flex flex-col gap-2">+ {workspace.hasMore && (+ <Banner kind="warn" className="rounded p-4">+ Showing first {workspace.fileCount} files. Your workspace has more+ files.+ </Banner>+ )}<HeaderIcon={BookTextIcon}control={diff --git a/marimo/_server/api/endpoints/home.py b/marimo/_server/api/endpoints/home.pyindex b3a099a237..29033bb1a0 100644--- a/marimo/_server/api/endpoints/home.py+++ b/marimo/_server/api/endpoints/home.py@@ -1,6 +1,7 @@# Copyright 2024 Marimo. All rights reserved.from __future__ import annotations+import asyncioimport osimport tempfilefrom typing import TYPE_CHECKING@@ -11,7 +12,11 @@ from starlette.responses import JSONResponsefrom marimo import _loggersfrom marimo._server.api.deps import AppStatefrom marimo._server.api.utils import parse_request-from marimo._server.file_router import LazyListOfFilesAppFileRouter+from marimo._server.file_router import (+ MAX_FILES,+ LazyListOfFilesAppFileRouter,+ count_files,+)from marimo._server.model import ConnectionStatefrom marimo._server.models.home import (MarimoFile,@@ -89,8 +94,18 @@ async def workspace_files()root = session_manager.file_router.directory- files = session_manager.file_router.files- return WorkspaceFilesResponse(files=files, root=root)+ # Run file scanning in thread pool to avoid blocking the server+ files = await asyncio.to_thread(lambda: session_manager.file_router.files)++ file_count = count_files(files)+ has_more = file_count >= MAX_FILES++ return WorkspaceFilesResponse(+ files=files,+ root=root,+ has_more=has_more,+ file_count=file_count,+ )def _get_active_sessions(app_state: AppState) -> list[MarimoFile]:diff --git a/marimo/_server/file_router.py b/marimo/_server/file_router.pyindex f3d64b7fcc..2598f2a618 100644--- a/marimo/_server/file_router.py+++ b/marimo/_server/file_router.py@@ -23,6 +23,9 @@ if TYPE_CHECKING:LOGGER = _loggers.marimo_logger()+# Maximum number of files to return in workspace scan+MAX_FILES = 1000+# Some unique identifier for a fileMarimoFileKey = str@@ -260,14 +263,31 @@ class LazyListOfFilesAppFileRouter(AppFileRouter):@propertydef files(self) -> list[FileInfo]:if self._lazy_files is None:- self._lazy_files = self._load_files()+ try:+ self._lazy_files = self._load_files()+ except HTTPException as e:+ if e.status_code == HTTPStatus.REQUEST_TIMEOUT:+ # Return partial results on timeout+ LOGGER.warning(+ "Timeout during file scan, returning partial results"+ )+ # self._lazy_files will already contain partial results+ # set in _load_files before the exception was raised+ if self._lazy_files is None:+ self._lazy_files = []+ else:+ raisereturn self._lazy_filesdef _load_files(self) -> list[FileInfo]:import timestart_time = time.time()- MAX_EXECUTION_TIME = 5 # 5 seconds timeout+ MAX_EXECUTION_TIME = 10 # 10 seconds timeout+ file_count = [0] # Use list for closure mutability+ accumulated_results: list[+ FileInfo+ ] = [] # For partial results on timeoutdef recurse(directory: str, depth: int = 0@@ -275,10 +295,17 @@ class LazyListOfFilesAppFileRouter(AppFileRouter):if depth > MAX_DEPTH:return None+ # Check file limit+ if file_count[0] >= MAX_FILES:+ LOGGER.warning(f"Reached maximum file limit ({MAX_FILES})")+ return None+if time.time() - start_time > MAX_EXECUTION_TIME:+ # Store accumulated results before raising timeout+ self._lazy_files = accumulated_resultsraise HTTPException(status_code=HTTPStatus.REQUEST_TIMEOUT,- detail="Request timed out: Loading workspace files took too long.", # noqa: E501+ detail=f"Request timed out: Loading workspace files took too long. Showing first {file_count[0]} files.", # noqa: E501)try:@@ -310,18 +337,23 @@ class LazyListOfFilesAppFileRouter(AppFileRouter):children=children,))- elif entry.name.endswith(tuple(allowed_extensions)):- if self._is_marimo_app(entry.path):- files.append(- FileInfo(- id=entry.path,- path=entry.path,- name=entry.name,- is_directory=False,- is_marimo_file=True,- last_modified=entry.stat().st_mtime,- )+ elif entry.name.endswith(allowed_extensions):+ if is_marimo_app(entry.path):+ file_count[0] += 1+ file_info = FileInfo(+ id=entry.path,+ path=entry.path,+ name=entry.name,+ is_directory=False,+ is_marimo_file=True,+ last_modified=entry.stat().st_mtime,)+ files.append(file_info)+ # Also add to accumulated results for partial loading+ accumulated_results.append(file_info)+ # Check if we've reached the limit+ if file_count[0] >= MAX_FILES:+ break# Sort folders then files, based on natural sort (alpha, then num)return sorted(folders, key=natural_sort_file) + sorted(@@ -330,11 +362,26 @@ class LazyListOfFilesAppFileRouter(AppFileRouter):MAX_DEPTH = 5skip_dirs = {+ # Python virtual environments"venv",+ ".venv",+ ".virtualenv",+ "__pypackages__",+ # Python cache and build"__pycache__",+ "build",+ "dist",+ "eggs",+ # Package management"node_modules","site-packages",- "eggs",+ # Testing and tooling+ ".tox",+ ".nox",+ ".pytest_cache",+ ".mypy_cache",+ # Version control+ ".git",}allowed_extensions = ((".py", ".md", ".qmd") if self.include_markdown else (".py",)@@ -342,19 +389,6 @@ class LazyListOfFilesAppFileRouter(AppFileRouter):return recurse(self.directory) or []- def _is_marimo_app(self, full_path: str) -> bool:- try:- path = MarimoPath(full_path)- contents = path.read_text()- if path.is_markdown():- return "marimo-version:" in contents- if path.is_python():- return "marimo.App" in contents and "import marimo" in contents- return False- except Exception as e:- LOGGER.debug("Error reading file %s: %s", full_path, e)- return False-def get_unique_file_key(self) -> str | None:return None@@ -480,3 +514,56 @@ def validate_inside_directory(directory: Path, filepath: Path) -> None:status_code=HTTPStatus.SERVER_ERROR,detail=f"Unexpected error validating path: {str(e)}",) from e+++def is_marimo_app(full_path: str) -> bool:+ """+ Detect whether a file is a marimo app.++ Rules:+ - Markdown (`.md`/`.qmd`) files are marimo apps if the first 512 bytes+ contain `marimo-version:`.+ - Python (`.py`) files are marimo apps if the header (first 512 bytes)+ contains both `marimo.App` and `import marimo`.+ - If the header contains `# /// script`, read the full file and check for+ the same Python markers, to handle large script headers.+ - Any errors while reading result in `False`.+ """+ READ_LIMIT = 512++ def contains_marimo_app(content: bytes) -> bool:+ return b"marimo.App" in content and b"import marimo" in content++ try:+ path = MarimoPath(full_path)++ with open(full_path, "rb") as f:+ header = f.read(READ_LIMIT)++ if path.is_markdown():+ return b"marimo-version:" in header++ if path.is_python():+ if contains_marimo_app(header):+ return True++ if b"# /// script" in header:+ full_content = path.read_bytes()+ if contains_marimo_app(full_content):+ return True++ return False+ except Exception as e:+ LOGGER.debug("Error reading file %s: %s", full_path, e)+ return False+++# Count total marimo files (not directories)+def count_files(file_list: list[FileInfo]) -> int:+ count = 0+ for item in file_list:+ if not item.is_directory:+ count += 1+ if item.children:+ count += count_files(item.children)+ return countdiff --git a/marimo/_server/models/home.py b/marimo/_server/models/home.pyindex 6f440bf47e..7ec173b1cb 100644--- a/marimo/_server/models/home.py+++ b/marimo/_server/models/home.py@@ -43,6 +43,10 @@ class WorkspaceFilesRequest(msgspec.Struct, rename="camel"):class WorkspaceFilesResponse(msgspec.Struct, rename="camel"):root: strfiles: list[FileInfo]+ # Indicates if limit was reached+ has_more: bool = False+ # Total files found+ file_count: int = 0class ShutdownSessionRequest(msgspec.Struct, rename="camel"):diff --git a/marimo/_utils/marimo_path.py b/marimo/_utils/marimo_path.pyindex bd39c10802..a6fdae6956 100644--- a/marimo/_utils/marimo_path.py+++ b/marimo/_utils/marimo_path.py@@ -85,6 +85,9 @@ class MarimoPath:def read_text(self, encoding: str = "utf-8") -> str:return self.path.read_text(encoding)+ def read_bytes(self) -> bytes:+ return self.path.read_bytes()+@propertydef short_name(self) -> str:return self.path.namediff --git a/packages/openapi/api.yaml b/packages/openapi/api.yamlindex 1e765aabe0..58f17b4d4c 100644--- a/packages/openapi/api.yaml+++ b/packages/openapi/api.yaml@@ -3692,10 +3692,16 @@ components:type: objectWorkspaceFilesResponse:properties:+ fileCount:+ default: 0+ type: integerfiles:items:$ref: '#/components/schemas/FileInfo'type: array+ hasMore:+ default: false+ type: booleanroot:type: stringrequired:@@ -3755,7 +3761,7 @@ components:type: objectinfo:title: marimo API- version: 0.17.6+ version: 0.18.1openapi: 3.1.0paths:/@file/{filename_and_length}:diff --git a/packages/openapi/src/api.ts b/packages/openapi/src/api.tsindex ae91fd7b91..f2d0b86994 100644--- a/packages/openapi/src/api.ts+++ b/packages/openapi/src/api.ts@@ -4750,7 +4750,11 @@ export interface components {};/** WorkspaceFilesResponse */WorkspaceFilesResponse: {+ /** @default 0 */+ fileCount?: number;files: components["schemas"]["FileInfo"][];+ /** @default false */+ hasMore?: boolean;root: string;};/**diff --git a/pixi.lock b/pixi.lockindex 06962b9302..ddc976285e 100644--- a/pixi.lock+++ b/pixi.lock@@ -1101,8 +1101,8 @@ packages:timestamp: 1727801725384- pypi: ./name: marimo- version: 0.18.0- sha256: 38a7a4e5f4bd2017f356d0959dc2b5f0049990f4e416e6bf9ea6d35eeb67c9e7+ version: 0.18.1+ sha256: b89cff1b4327d087653d929b56b08026b0b690464da595eb8bb4d5dbadc971e1requires_dist:- click>=8.0,<9- jedi>=0.18.0diff --git a/tests/_server/api/endpoints/test_home.py b/tests/_server/api/endpoints/test_home.pyindex a9bf907bcf..d6c03024fb 100644--- a/tests/_server/api/endpoints/test_home.py+++ b/tests/_server/api/endpoints/test_home.py@@ -35,6 +35,11 @@ def test_workspace_files(client: TestClient) -> None:files = body["files"]assert len(files) == 1assert files[0]["path"] == current_filename+ # Check that new fields are present+ assert "hasMore" in body+ assert "fileCount" in body+ assert body["hasMore"] is False+ assert body["fileCount"] == 1@with_session(SESSION_ID)diff --git a/tests/_server/test_file_router.py b/tests/_server/test_file_router.pyindex 2529dfea73..4ce410c842 100644--- a/tests/_server/test_file_router.py+++ b/tests/_server/test_file_router.py@@ -14,6 +14,8 @@ from marimo._server.file_router import (LazyListOfFilesAppFileRouter,ListOfFilesAppFileRouter,NewFileAppFileRouter,+ count_files,+ is_marimo_app,validate_inside_directory,)from marimo._server.models.home import MarimoFile@@ -402,3 +404,133 @@ class TestValidateInsideDirectory(unittest.TestCase):with pytest.raises(HTTPException) as exc_info:validate_inside_directory(directory, filepath)assert exc_info.value.status_code == HTTPStatus.FORBIDDEN+++def test_python_app_detected_in_header(tmp_path: Path):+ f = tmp_path / "app.py"+ content = b"import marimo\napp = marimo.App()\n"+ f.write_bytes(content)+ assert is_marimo_app(str(f)) is True+++def test_python_app_detected_with_script_header_full_read(tmp_path: Path):+ f = tmp_path / "script_app.py"+ header = b"# /// script\n# lots of stuff before markers\n" + (b"x" * 600)+ # Ensure markers are only in full content beyond header limit+ body = b"\nimport marimo\napp = marimo.App()\n"+ f.write_bytes(header + body)+ assert is_marimo_app(str(f)) is True+++def test_python_non_app_returns_false(tmp_path: Path):+ f = tmp_path / "not_app.py"+ f.write_bytes(b"print('hello')\n")+ assert is_marimo_app(str(f)) is False+++def test_markdown_with_marimo_version_detected(tmp_path: Path):+ f = tmp_path / "notebook.md"+ # Place marker in the first 512 bytes+ f.write_bytes(b"---\nmarimo-version: 0.1\n---\n")+ assert is_marimo_app(str(f)) is True+++def test_markdown_without_marimo_version_returns_false(tmp_path: Path):+ f = tmp_path / "plain.md"+ f.write_bytes(b"# Title\nSome content\n")+ assert is_marimo_app(str(f)) is False+++def test_error_path_returns_false_and_logs(tmp_path: Path):+ # Point to a directory to trigger open error+ d = tmp_path / "adir"+ d.mkdir()+ assert is_marimo_app(str(d)) is False+++def test_lazy_router_respects_max_files(tmp_path: Path):+ """Test that LazyListOfFilesAppFileRouter enforces MAX_FILES limit"""+ # Create a directory with more files than MAX_FILES+ # To make this test fast, we'll use a monkey-patch approach+ # by temporarily reducing MAX_FILES+ import marimo._server.file_router as file_router_module++ original_max_files = file_router_module.MAX_FILES+ try:+ # Set a small limit for testing+ file_router_module.MAX_FILES = 5++ # Create 10 marimo files+ for i in range(10):+ f = tmp_path / f"app_{i}.py"+ f.write_text("import marimo\napp = marimo.App()\n")++ router = LazyListOfFilesAppFileRouter(+ str(tmp_path), include_markdown=False+ )+ files = router.files++ # Should only get MAX_FILES worth of files+ # Count actual marimo files (not directories)+ file_count = sum(1 for f in files if not f.is_directory)+ assert file_count <= 5++ finally:+ # Restore original value+ file_router_module.MAX_FILES = original_max_files+++def test_lazy_router_skips_common_dirs(tmp_path: Path):+ """Test that LazyListOfFilesAppFileRouter skips common directories"""+ # Create directories that should be skipped+ skip_dirs = [+ ".venv",+ ".git",+ "__pycache__",+ "node_modules",+ ".tox",+ ".pytest_cache",+ ]++ for skip_dir in skip_dirs:+ dir_path = tmp_path / skip_dir+ dir_path.mkdir()+ # Create a marimo file inside+ f = dir_path / "app.py"+ f.write_text("import marimo\napp = marimo.App()\n")++ # Create a valid marimo file in the root+ root_file = tmp_path / "root_app.py"+ root_file.write_text("import marimo\napp = marimo.App()\n")++ router = LazyListOfFilesAppFileRouter(+ str(tmp_path), include_markdown=False+ )+ files = router.files++ # Should only find the root file, not files in skipped directories+ file_paths = [f.path for f in files if not f.is_directory]+ assert len(file_paths) == 1+ assert str(root_file) in file_paths+++def test_lazy_router_counts_nested_files(tmp_path: Path):+ """Test that file counting works correctly with nested directories"""+ # Create nested structure+ nested_dir = tmp_path / "subdir"+ nested_dir.mkdir()++ # Create files at different levels+ root_file = tmp_path / "root.py"+ root_file.write_text("import marimo\napp = marimo.App()\n")++ nested_file = nested_dir / "nested.py"+ nested_file.write_text("import marimo\napp = marimo.App()\n")++ router = LazyListOfFilesAppFileRouter(+ str(tmp_path), include_markdown=False+ )+ files = router.files++ total_files = count_files(files)+ assert total_files == 2diff --git a/tests/_utils/test_marimo_path.py b/tests/_utils/test_marimo_path.pyindex 6f768ae18d..38f04646f9 100644--- a/tests/_utils/test_marimo_path.py+++ b/tests/_utils/test_marimo_path.py@@ -79,6 +79,20 @@ def test_write_text(tmp_path: Path):assert tmp_file.read_text() == text+def test_read_bytes(tmp_path: Path):+ # Create a temporary file with binary content+ tmp_file = tmp_path / "test.py"+ binary_content = b"import marimo\napp = marimo.App()\n"+ tmp_file.write_bytes(binary_content)++ mp = MarimoPath(tmp_file)++ # Read bytes and verify+ content = mp.read_bytes()+ assert content == binary_content+ assert isinstance(content, bytes)++def test_properties():mp = MarimoPath(__file__)